From 295831a481241d3d06b49f646a40f27b1297fab5 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Sat, 8 Oct 2022 12:30:12 +0200 Subject: [PATCH 001/793] Init --- README.md | 37 + proto/generate.proto | 83 + router/.gitignore | 1 + router/Cargo.lock | 2373 +++++++++++++++++ router/Cargo.toml | 28 + router/client/Cargo.toml | 19 + router/client/build.rs | 14 + router/client/src/client.rs | 104 + router/client/src/lib.rs | 32 + router/client/src/pb/.gitignore | 1 + router/client/src/sharded_client.rs | 106 + router/src/db.rs | 129 + router/src/infer.rs | 130 + router/src/main.rs | 125 + server/.DS_Store | Bin 0 -> 6148 bytes server/Makefile | 20 + server/README.md | 15 + server/bloom_inference/.DS_Store | Bin 0 -> 6148 bytes server/bloom_inference/__init__.py | 0 .../__pycache__/__init__.cpython-39.pyc | Bin 0 -> 158 bytes .../__pycache__/cache.cpython-39.pyc | Bin 0 -> 2296 bytes .../__pycache__/model.cpython-39.pyc | Bin 0 -> 9759 bytes .../__pycache__/server.cpython-39.pyc | Bin 0 -> 3683 bytes .../__pycache__/shard_model.cpython-39.pyc | Bin 0 -> 3058 bytes .../__pycache__/utils.cpython-39.pyc | Bin 0 -> 3184 bytes server/bloom_inference/cache.py | 48 + server/bloom_inference/main.py | 30 + server/bloom_inference/model.py | 428 +++ server/bloom_inference/pb/__init__.py | 0 server/bloom_inference/pb/__init__.py-e | 0 .../pb/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 161 bytes .../__pycache__/generate_pb2.cpython-39.pyc | Bin 0 -> 2182 bytes .../generate_pb2_grpc.cpython-39.pyc | Bin 0 -> 4355 bytes server/bloom_inference/pb/generate_pb2.py | 43 + server/bloom_inference/pb/generate_pb2.py-e | 43 + .../bloom_inference/pb/generate_pb2_grpc.py | 169 ++ .../bloom_inference/pb/generate_pb2_grpc.py-e | 169 ++ server/bloom_inference/prepare_weights.py | 124 + server/bloom_inference/server.py | 91 + server/bloom_inference/shard_model.py | 102 + server/bloom_inference/utils.py | 95 + server/poetry.lock | 480 ++++ server/pyproject.toml | 21 + 43 files changed, 5060 insertions(+) create mode 100644 README.md create mode 100644 proto/generate.proto create mode 100644 router/.gitignore create mode 100644 router/Cargo.lock create mode 100644 router/Cargo.toml create mode 100644 router/client/Cargo.toml create mode 100644 router/client/build.rs create mode 100644 router/client/src/client.rs create mode 100644 router/client/src/lib.rs create mode 100644 router/client/src/pb/.gitignore create mode 100644 router/client/src/sharded_client.rs create mode 100644 router/src/db.rs create mode 100644 router/src/infer.rs create mode 100644 router/src/main.rs create mode 100644 server/.DS_Store create mode 100644 server/Makefile create mode 100644 server/README.md create mode 100644 server/bloom_inference/.DS_Store create mode 100644 server/bloom_inference/__init__.py create mode 100644 server/bloom_inference/__pycache__/__init__.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/cache.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/model.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/server.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/shard_model.cpython-39.pyc create mode 100644 server/bloom_inference/__pycache__/utils.cpython-39.pyc create mode 100644 server/bloom_inference/cache.py create mode 100644 server/bloom_inference/main.py create mode 100644 server/bloom_inference/model.py create mode 100644 server/bloom_inference/pb/__init__.py create mode 100644 server/bloom_inference/pb/__init__.py-e create mode 100644 server/bloom_inference/pb/__pycache__/__init__.cpython-39.pyc create mode 100644 server/bloom_inference/pb/__pycache__/generate_pb2.cpython-39.pyc create mode 100644 server/bloom_inference/pb/__pycache__/generate_pb2_grpc.cpython-39.pyc create mode 100644 server/bloom_inference/pb/generate_pb2.py create mode 100644 server/bloom_inference/pb/generate_pb2.py-e create mode 100644 server/bloom_inference/pb/generate_pb2_grpc.py create mode 100644 server/bloom_inference/pb/generate_pb2_grpc.py-e create mode 100644 server/bloom_inference/prepare_weights.py create mode 100644 server/bloom_inference/server.py create mode 100644 server/bloom_inference/shard_model.py create mode 100644 server/bloom_inference/utils.py create mode 100644 server/poetry.lock create mode 100644 server/pyproject.toml diff --git a/README.md b/README.md new file mode 100644 index 00000000..fe7bb443 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# BLOOM Inference + +A Rust and gRPC server for BLOOM Inference. + +## Install + +```shell +cd server +pip install . +``` + +``` +cd router +cargo build --release +``` + +## Run + +```shell +python server/bloom_inference/main.py bigscience/bloom --num-gpus 8 --shard-directory /dev/shm/models +``` + +```shell +./router/target/release/router +``` + +## TODO: + +- [ ] Improve model download + - Store "shardable" layers separately and layer by layer +- [ ] Add batching args to router CLI +- [ ] Add docstrings + comments everywhere as the codebase is fairly complicated +- [ ] Add tests +- [ ] Add shutdown logic in router and server +- [ ] Improve multi-processing logic in server +- [ ] Improve error handling everywhere +- [ ] Improve past key layer indexing? \ No newline at end of file diff --git a/proto/generate.proto b/proto/generate.proto new file mode 100644 index 00000000..324b9206 --- /dev/null +++ b/proto/generate.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +package generate.v1; + +service TextGeneration { + /// Service discovery + rpc ServiceDiscovery(Empty) returns (ServiceDiscoveryResponse) {} + /// Empties batch cache + rpc ClearCache(Empty) returns (Empty); + /// Generate tokens for a batch without cache + rpc Generate(Batch) returns (Response); + /// Generate tokens for a batch with cache + rpc GenerateWithCache(BatchCached) returns (Response); +} + +message ServiceDiscoveryResponse { + repeated string urls = 1; +} + +message LogitsWarperParameters { + float temperature = 1; + uint32 top_k = 2; + float top_p = 3; + bool do_sample = 4; +} + +message Request { + /// Request ID + uint64 id = 1; + /// The generation context + string inputs = 2; + /// Logits Warper Parameters + LogitsWarperParameters parameters = 3; + /// Stopping criteria + uint32 max_new_tokens = 4; +} + +message Batch { + /// Batch ID + uint64 id = 1; + /// Individual requests + repeated Request requests = 2; +} + +message BatchCached { + /// Batch ID + uint64 id = 1; + /// Request ids within cache + repeated uint64 request_ids = 2; + /// Cache IDs + repeated uint64 batch_cached_ids = 3; + /// Batch size (sum of all batch sizes) + uint32 total_batch_size = 4; + /// Max sequence length + uint32 max_sequence_length = 5; +} + +message FinishedGeneration { + /// ID of the original request + uint64 id = 1; + /// Output + string output = 2; +} + +message CacheEntry { + /// Cache ID; same as batch ID + uint64 id = 1; + /// Requests present in cache entry + repeated uint64 request_ids = 2; + /// Sequence length + uint32 sequence_length = 3; +} + +message Response { + /// Finished requests (optional) + repeated FinishedGeneration finished = 1; + /// Cache entry (optional) + optional CacheEntry cache_entry = 2; +} + + +// Represent an empty message. +message Empty {} \ No newline at end of file diff --git a/router/.gitignore b/router/.gitignore new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/router/.gitignore @@ -0,0 +1 @@ +/target diff --git a/router/Cargo.lock b/router/Cargo.lock new file mode 100644 index 00000000..79761345 --- /dev/null +++ b/router/Cargo.lock @@ -0,0 +1,2373 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" + +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bloom-inference" +version = "0.1.0" +dependencies = [ + "bloom-inference-client", + "futures", + "parking_lot", + "poem", + "serde", + "serde_json", + "tokenizers", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "bloom-inference-client" +version = "0.1.0" +dependencies = [ + "futures", + "prost", + "thiserror", + "tokio", + "tonic", + "tonic-build", + "tower", + "tracing", + "tracing-error", +] + +[[package]] +name = "bumpalo" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" + +[[package]] +name = "bzip2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cached-path" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f1c56d30236522ab3393a08746b138d4e16372001f42d29c88d513aeb8ab7ef" +dependencies = [ + "flate2", + "fs2", + "glob", + "indicatif 0.16.2", + "log", + "rand", + "reqwest", + "serde", + "serde_json", + "sha2", + "tar", + "tempfile", + "thiserror", + "zip", + "zip-extensions", +] + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "console" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "terminal_size", + "unicode-width", + "winapi", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.9.3", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "derive_builder" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0" +dependencies = [ + "darling", + "derive_builder_core", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dirs" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f748b253ceca9fed5f42f8b5ceb3851e93102199bc25b64b65369f76e5c0a35" +dependencies = [ + "cc", +] + +[[package]] +name = "fastrand" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +dependencies = [ + "instant", +] + +[[package]] +name = "filetime" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" + +[[package]] +name = "futures-executor" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" + +[[package]] +name = "futures-macro" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" + +[[package]] +name = "futures-task" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" + +[[package]] +name = "futures-util" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "h2" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util 0.7.4", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "headers" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" +dependencies = [ + "base64", + "bitflags", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha1", +] + +[[package]] +name = "headers-core" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +dependencies = [ + "http", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "indicatif" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +dependencies = [ + "console", + "lazy_static", + "number_prefix 0.3.0", + "regex", +] + +[[package]] +name = "indicatif" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" +dependencies = [ + "console", + "lazy_static", + "number_prefix 0.4.0", + "regex", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" + +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.134" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "macro_rules_attribute" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258c86475e1616d6f2d8f5227cfaabd3dae1f6d5388b9597df8a199d4497aba7" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26a8d2502d5aa4d411ef494ba7470eb299f05725179ce3b5de77aa01a9ffdea" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "number_prefix" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" + +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "openssl" +version = "0.10.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + +[[package]] +name = "paste" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "petgraph" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + +[[package]] +name = "poem" +version = "1.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2992ba72908e36200671c0f3a692992ced894b3b2bbe2b2dc6dfbffea6e2c85a" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "headers", + "http", + "hyper", + "mime", + "parking_lot", + "percent-encoding", + "pin-project-lite", + "poem-derive", + "regex", + "rfc7239", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util 0.7.4", + "tracing", +] + +[[package]] +name = "poem-derive" +version = "1.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f535d4331a22610b98ca48f98bae9bda0c654da89b9ae10a1830fa9edfd8f36" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "proc-macro-crate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" +dependencies = [ + "once_cell", + "thiserror", + "toml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +dependencies = [ + "bytes", + "heck", + "itertools 0.10.5", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" +dependencies = [ + "bytes", + "prost", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd1259362c9065e5ea39a789ef40b1e3fd934c94beb7b5ab3ac6629d3b5e7cb7" +dependencies = [ + "either", + "itertools 0.8.2", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rfc7239" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "087317b3cf7eb481f13bd9025d729324b7cd068d6f470e2d76d049e191f5ba47" +dependencies = [ + "uncased", +] + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + +[[package]] +name = "syn" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokenizers" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b08ede6742d7a59d58c71da8a6fa21bedc433dca2e855e439274d08df1170" +dependencies = [ + "aho-corasick", + "cached-path", + "clap", + "derive_builder", + "dirs", + "esaxx-rs", + "getrandom", + "indicatif 0.15.0", + "itertools 0.9.0", + "lazy_static", + "log", + "macro_rules_attribute", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "reqwest", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tokio" +version = "1.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "socket2", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + +[[package]] +name = "tonic" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" +dependencies = [ + "async-stream", + "async-trait", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" +dependencies = [ + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util 0.7.4", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" +dependencies = [ + "ansi_term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "uncased" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-ident" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "which" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +dependencies = [ + "either", + "libc", + "once_cell", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] + +[[package]] +name = "zip" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +dependencies = [ + "byteorder", + "bzip2", + "crc32fast", + "flate2", + "thiserror", + "time", +] + +[[package]] +name = "zip-extensions" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64c3c977bc3434ce2d4bcea8ad3c644672de0f2c402b72b9171ca80a8885d14" +dependencies = [ + "zip", +] diff --git a/router/Cargo.toml b/router/Cargo.toml new file mode 100644 index 00000000..901a14f2 --- /dev/null +++ b/router/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "bloom-inference" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bloom-inference-client = { path = "client" } +futures = "0.3.24" +parking_lot = "0.12.1" +poem = "1.3.45" +serde = "1.0.145" +serde_json = "1.0.85" +tokenizers = "0.13.0" +tokio = { version = "1.21.1", features = ["rt-multi-thread", "parking_lot", "sync"] } +tracing = "0.1.36" +tracing-subscriber = "0.3.15" + +[workspace] +members = [ + "client", +] + +[profile.release] +debug = 1 +incremental = true +lto = "off" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml new file mode 100644 index 00000000..e4f2fa0f --- /dev/null +++ b/router/client/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bloom-inference-client" +version = "0.1.0" +edition = "2021" + +[dependencies] +futures = "0.3.24" +#grpc-error-details = { path = "../../grpc-error-details" } +#grpc-metadata = { path = "../../grpc-metadata" } +prost = "^0.9" +thiserror = "1.0.37" +tokio = { version = "1.21.2", features = ["sync"] } +tonic = "^0.6" +tower = "^0.4" +tracing = "^0.1" +tracing-error = "^0.2" + +[build-dependencies] +tonic-build = "0.6.2" diff --git a/router/client/build.rs b/router/client/build.rs new file mode 100644 index 00000000..1876b217 --- /dev/null +++ b/router/client/build.rs @@ -0,0 +1,14 @@ +use std::fs; + +fn main() -> Result<(), Box> { + fs::create_dir("src/pb").unwrap_or(()); + tonic_build::configure() + .build_client(true) + .build_server(false) + .out_dir("src/pb") + .include_file("mod.rs") + .compile(&["../../proto/generate.proto"], &["../../proto"]) + .unwrap_or_else(|e| panic!("protobuf compilation failed: {}", e)); + + Ok(()) +} diff --git a/router/client/src/client.rs b/router/client/src/client.rs new file mode 100644 index 00000000..7f68e252 --- /dev/null +++ b/router/client/src/client.rs @@ -0,0 +1,104 @@ +use crate::pb::generate::v1::text_generation_client::TextGenerationClient; +use crate::pb::generate::v1::*; +use crate::Result; +use std::time::Duration; +use tonic::transport::{Channel, Uri}; +use tower::timeout::Timeout; +use tracing::*; + +/// BLOOM Inference gRPC client +#[derive(Clone)] +pub struct Client { + stub: TextGenerationClient>, +} + +impl Client { + /// Returns a client connected to the given url. Requests exceeding timeout will fail. + pub async fn connect(uri: Uri, timeout: Duration) -> Self { + let channel = Channel::builder(uri) + .connect() + .await + .expect("Transport error"); + let timeout_channel = Timeout::new(channel, timeout); + + Self { + stub: TextGenerationClient::new(timeout_channel), + } + } + + /// Returns a client connected to the given unix socket. Requests exceeding timeout will fail. + pub async fn connect_uds(path: String, timeout: Duration) -> Self { + let channel = Channel::from_shared(format!("http://[::]:50051")) + .unwrap() + .connect_with_connector(tower::service_fn(move |_: Uri| { + tokio::net::UnixStream::connect(path.clone()) + })) + .await + .expect("Transport error"); + let timeout_channel = Timeout::new(channel, timeout); + + Self { + stub: TextGenerationClient::new(timeout_channel), + } + } + + #[instrument(skip(self))] + pub async fn service_discovery(&mut self) -> Result> { + let request = tonic::Request::new(Empty {}); + let response = self + .stub + .service_discovery(request) + .instrument(info_span!("service_discovery")) + .await?; + let urls = response + .into_inner() + .urls + .into_iter() + .map(|url| match url.strip_prefix("unix://") { + None => url, + Some(stripped_url) => stripped_url.to_string(), + }) + .collect(); + Ok(urls) + } + + #[instrument(skip(self))] + pub async fn clear_cache(&mut self) -> Result<()> { + let request = tonic::Request::new(Empty {}); + self.stub + .clear_cache(request) + .instrument(info_span!("clear_cache")) + .await?; + Ok(()) + } + + #[instrument(skip(self))] + pub async fn generate( + &mut self, + request: Batch, + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(request); + let response = self + .stub + .generate(request) + .instrument(info_span!("generate")) + .await? + .into_inner(); + Ok((response.finished, response.cache_entry)) + } + + #[instrument(skip(self))] + pub async fn generate_with_cache( + &mut self, + request: BatchCached, + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(request); + let response = self + .stub + .generate_with_cache(request) + .instrument(info_span!("generate_with_cache")) + .await? + .into_inner(); + Ok((response.finished, response.cache_entry)) + } +} diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs new file mode 100644 index 00000000..aca9f65b --- /dev/null +++ b/router/client/src/lib.rs @@ -0,0 +1,32 @@ +//! BLOOM Inference gRPC client library + +mod client; +mod pb; +mod sharded_client; + +pub use client::Client; +pub use pb::generate::v1::{ + Batch, BatchCached, CacheEntry, FinishedGeneration, LogitsWarperParameters, Request, +}; +pub use sharded_client::ShardedClient; +use thiserror::Error; +pub use tonic::transport::Uri; +use tonic::Status; + +#[derive(Error, Debug, Clone)] +#[error("Text generation client error: {msg:?}")] +pub struct ClientError { + msg: String, + // source: Status, +} + +impl From for ClientError { + fn from(err: Status) -> Self { + Self { + msg: err.to_string(), + // source: err, + } + } +} + +pub type Result = std::result::Result; diff --git a/router/client/src/pb/.gitignore b/router/client/src/pb/.gitignore new file mode 100644 index 00000000..b46a4c42 --- /dev/null +++ b/router/client/src/pb/.gitignore @@ -0,0 +1 @@ +*.rs \ No newline at end of file diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs new file mode 100644 index 00000000..6856a9bc --- /dev/null +++ b/router/client/src/sharded_client.rs @@ -0,0 +1,106 @@ +use crate::Result; +use crate::{Batch, BatchCached, CacheEntry, Client, FinishedGeneration}; +use futures::future::join_all; +use std::time::Duration; +use tokio::sync::{broadcast, mpsc}; +use tonic::transport::Uri; + +#[derive(Clone, Debug)] +enum Command { + Generate( + Batch, + mpsc::Sender, Option)>>, + ), + GenerateWithCache( + BatchCached, + mpsc::Sender, Option)>>, + ), + ClearCache(mpsc::Sender>), +} + +async fn client_task(mut client: Client, mut request_subscriber: broadcast::Receiver) { + while let Ok(message) = request_subscriber.recv().await { + match message { + Command::Generate(batch, response_tx) => { + let result = client.generate(batch).await; + response_tx.try_send(result).unwrap_or(()); + } + Command::GenerateWithCache(batch_cached, response_tx) => { + let result = client.generate_with_cache(batch_cached).await; + response_tx.try_send(result).unwrap_or(()); + } + Command::ClearCache(response_tx) => { + let result = client.clear_cache().await; + response_tx.try_send(result).unwrap_or(()); + } + }; + } +} + +pub struct ShardedClient { + request_tx: broadcast::Sender, +} + +impl ShardedClient { + fn new(mut clients: Vec) -> Self { + let (request_tx, _) = broadcast::channel(1); + + for client in clients.drain(..) { + let request_subscriber = request_tx.subscribe(); + tokio::spawn(client_task(client, request_subscriber)); + } + + Self { request_tx } + } + + async fn from_master_client(mut master_client: Client) -> Self { + let uris = master_client.service_discovery().await.unwrap(); + let futures = uris + .into_iter() + .map(|path| Client::connect_uds(path, Duration::from_secs(5))); + let clients = join_all(futures).await; + Self::new(clients) + } + + /// Returns a client connected to the given url. Requests exceeding timeout will fail. + pub async fn connect(uri: Uri, timeout: Duration) -> Self { + let master_client = Client::connect(uri, timeout).await; + Self::from_master_client(master_client).await + } + + /// Returns a client connected to the given unix socket. Requests exceeding timeout will fail. + pub async fn connect_uds(path: String, timeout: Duration) -> Self { + let master_client = Client::connect_uds(path, timeout).await; + Self::from_master_client(master_client).await + } + + pub async fn generate( + &self, + batch: Batch, + ) -> Result<(Vec, Option)> { + let (response_tx, mut response_rx) = mpsc::channel(1); + self.request_tx + .send(Command::Generate(batch, response_tx)) + .unwrap(); + response_rx.recv().await.unwrap() + } + + pub async fn generate_with_cache( + &self, + batch_cached: BatchCached, + ) -> Result<(Vec, Option)> { + let (response_tx, mut response_rx) = mpsc::channel(1); + self.request_tx + .send(Command::GenerateWithCache(batch_cached, response_tx)) + .unwrap(); + response_rx.recv().await.unwrap() + } + + pub async fn clear_cache(&self) -> Result<()> { + let (response_tx, mut response_rx) = mpsc::channel(1); + self.request_tx + .send(Command::ClearCache(response_tx)) + .unwrap(); + response_rx.recv().await.unwrap() + } +} diff --git a/router/src/db.rs b/router/src/db.rs new file mode 100644 index 00000000..3dd98d94 --- /dev/null +++ b/router/src/db.rs @@ -0,0 +1,129 @@ +/// This code is massively inspired by Tokio mini-redis +use crate::GenerateRequest; +use bloom_inference_client::{Batch, ClientError, LogitsWarperParameters, Request}; +use parking_lot::RwLock; +use std::collections::BTreeMap; +use std::sync::Arc; +use tokio::sync::oneshot::Sender; + +#[derive(Debug, Clone)] +pub(crate) struct Db { + pub shared: Arc, +} + +#[derive(Debug)] +pub struct Shared { + state: RwLock, +} + +#[derive(Debug)] +struct State { + entries: BTreeMap>)>, + + /// Identifier to use for the next expiration. Each expiration is associated + /// with a unique identifier. See above for why. + next_id: u64, + + next_batch_id: u64, + + /// Current batch id + next_batch_start_id: u64, +} + +impl Db { + pub(crate) fn new() -> Self { + let shared = Arc::new(Shared { + state: RwLock::new(State { + entries: BTreeMap::new(), + next_id: 0, + next_batch_id: 0, + next_batch_start_id: 0, + }), + }); + + Self { shared } + } + + pub(crate) fn append(&self, request: GenerateRequest, sender: Sender>) { + let mut state = self.shared.state.write(); + + let id = state.next_id; + state.next_id += 1; + + let parameters = Some(LogitsWarperParameters { + temperature: request.parameters.temperature, + top_k: request.parameters.top_k, + top_p: request.parameters.top_p, + do_sample: request.parameters.do_sample, + }); + let request = Request { + id, + inputs: request.inputs, + parameters, + max_new_tokens: request.parameters.max_new_tokens, + }; + state.entries.insert(id, (request, sender)); + } + + pub(crate) fn remove(&self, id: &u64) -> Option<(Request, Sender>)> { + let mut state = self.shared.state.write(); + state.entries.remove(id) + } + + pub(crate) fn len(&self) -> usize { + let state = self.shared.state.read(); + state.entries.len() + } + + fn next_requests(&self, max_size: usize) -> Option<(u64, Vec)> { + let state = self.shared.state.read(); + + let requests: Vec = state + .entries + .range(state.next_batch_start_id..) + .take(max_size) + .map(|(_, (request, _))| request.clone()) + .collect(); + + if requests.is_empty() { + None + } else { + let last_id = requests.last().unwrap().id; + Some((last_id, requests)) + } + } + + pub(crate) fn next_batch(&self, max_size: usize) -> Option { + if let Some((last_id, requests)) = self.next_requests(max_size) { + let mut state = self.shared.state.write(); + let batch = Batch { + id: state.next_batch_id, + requests, + }; + state.next_batch_start_id = last_id + 1; + state.next_batch_id += 1; + return Some(batch); + } + None + } + + pub(crate) fn next_batch_minimum_size( + &self, + min_size: usize, + max_size: usize, + ) -> Option { + if let Some((last_id, requests)) = self.next_requests(max_size) { + if requests.len() >= min_size { + let mut state = self.shared.state.write(); + let batch = Batch { + id: state.next_batch_id, + requests, + }; + state.next_batch_start_id = last_id + 1; + state.next_batch_id += 1; + return Some(batch); + } + } + None + } +} diff --git a/router/src/infer.rs b/router/src/infer.rs new file mode 100644 index 00000000..2a7aa0ac --- /dev/null +++ b/router/src/infer.rs @@ -0,0 +1,130 @@ +use crate::{Db, GenerateRequest}; +use bloom_inference_client::{Batch, BatchCached, CacheEntry, ClientError, FinishedGeneration, ShardedClient}; +use std::sync::Arc; +use tokio::sync::{oneshot, Notify}; + +const MAX_LENGTH: usize = 128; + +pub struct InferError {} + +#[derive(Clone)] +pub(crate) struct Infer { + db: Db, + shared: Arc, +} + +struct Shared { + batching_task: Notify, +} + +impl Infer { + pub(crate) fn new(client: ShardedClient) -> Self { + let db = Db::new(); + let shared = Arc::new(Shared { + batching_task: Notify::new(), + }); + + tokio::spawn(batching_task(client, db.clone(), shared.clone())); + + Self { db, shared } + } + + pub(crate) async fn infer(&self, request: GenerateRequest) -> Result { + if self.db.len() > MAX_LENGTH { + return Err(InferError {}); + } + let (request_tx, request_rx) = oneshot::channel(); + self.db.append(request, request_tx); + self.shared.batching_task.notify_waiters(); + match request_rx.await.unwrap() { + Ok(output) => Ok(output), + Err(_) => Err(InferError {}) + } + } +} + +async fn batching_task(client: ShardedClient, db: Db, shared: Arc) { + loop { + shared.batching_task.notified().await; + + if let Some(batch) = db.next_batch(32) { + let mut cache_entry = infer_batch(batch, &client, &db).await; + + loop { + if let Some(entry) = cache_entry { + let mut batch_cached_ids = vec![entry.id]; + let mut total_batch_size = entry.request_ids.len(); + let mut max_sequence_length = entry.sequence_length; + let mut request_ids = entry.request_ids; + + if total_batch_size <= 16 { + if let Some(batch) = db.next_batch_minimum_size(16, 48) { + let other_cache_entry = infer_batch(batch, &client, &db).await; + + if let Some(entry) = other_cache_entry { + batch_cached_ids.push(entry.id); + total_batch_size += entry.request_ids.len(); + max_sequence_length = + max_sequence_length.max(entry.sequence_length); + request_ids.extend(entry.request_ids.into_iter()); + } + } + } + + let batch_cached = BatchCached { + id: entry.id, + batch_cached_ids, + total_batch_size: total_batch_size as u32, + max_sequence_length, + request_ids, + }; + cache_entry = infer_batch_cached(batch_cached, &client, &db).await; + } else { + break; + } + } + } + } +} + +async fn infer_batch_cached(batch: BatchCached, client: &ShardedClient, db: &Db) -> Option { + match client.generate_with_cache(batch.clone()).await { + Ok((finished, cache_entry)) => { + send_finished(finished, db); + cache_entry + } + Err(err) => { + println!("{:?}", err); + send_error(err, batch.request_ids, &db); + None + } + } +} + +async fn infer_batch(batch: Batch, client: &ShardedClient, db: &Db) -> Option { + match client.generate(batch.clone()).await { + Ok((finished, cache_entry)) => { + send_finished(finished, db); + cache_entry + } + Err(err) => { + println!("{:?}", err); + send_error(err, batch.requests.into_iter().map(|req| req.id).collect(), &db); + None + } + } +} + +fn send_error(error: ClientError, request_ids: Vec, db: &Db) { + request_ids.into_iter().for_each(|id| { + let (_, response_tx) = db.remove(&id).unwrap(); + response_tx.send(Err(error.clone())).unwrap_or(()); + }); +} + +fn send_finished(finished: Vec, db: &Db) { + finished.into_iter().for_each(|output| { + let (_, response_tx) = db.remove(&output.id).unwrap(); + response_tx.send(Ok(output.output)).unwrap_or(()); + }); +} diff --git a/router/src/main.rs b/router/src/main.rs new file mode 100644 index 00000000..5d87cd46 --- /dev/null +++ b/router/src/main.rs @@ -0,0 +1,125 @@ +use tokio::time::Instant; + +use poem; +use poem::middleware::AddData; +use poem::web::Data; +use poem::{handler, listener::TcpListener, post, web::Json, EndpointExt, Result, Route, Server}; + +use bloom_inference_client::ShardedClient; +use serde::Deserialize; +use std::time::Duration; +use poem::http::StatusCode; +use tracing::instrument; + +mod db; + +use db::Db; + +mod infer; + +use infer::Infer; + +#[derive(Clone, Debug, Deserialize)] +struct GenerateParameters { + #[serde(default = "default_temperature")] + temperature: f32, + #[serde(default = "default_top_k")] + top_k: u32, + #[serde(default = "default_top_p")] + top_p: f32, + #[serde(default = "default_do_sample")] + do_sample: bool, + #[serde(default = "default_max_new_tokens")] + max_new_tokens: u32, +} + +fn default_temperature() -> f32 { + 1.0 +} + +fn default_top_k() -> u32 { + 0 +} + +fn default_top_p() -> f32 { + 1.0 +} + +fn default_do_sample() -> bool { + false +} + +fn default_max_new_tokens() -> u32 { + 20 +} + +#[derive(Clone, Debug, Deserialize)] +struct GenerateRequest { + inputs: String, + #[serde(default = "default_parameters")] + parameters: GenerateParameters, +} + +fn default_parameters() -> GenerateParameters { + GenerateParameters { + temperature: default_temperature(), + top_k: default_top_k(), + top_p: default_top_p(), + do_sample: default_do_sample(), + max_new_tokens: default_max_new_tokens(), + } +} + +#[handler] +#[instrument(skip(infer), fields(time, time_per_token))] +async fn generate( + infer: Data<&Infer>, + req: Json, +) -> Result> { + let start = Instant::now(); + + let output = infer + .infer(GenerateRequest { + inputs: req.inputs.clone(), + parameters: req.parameters.clone(), + }) + .await; + + match output { + Ok(generated_text) => { + tracing::Span::current().record("time", format!("{:?}", start.elapsed())); + tracing::Span::current().record("time_per_token", format!("{:?}", start.elapsed() / req.parameters.max_new_tokens)); + tracing::info!("response: {}", generated_text); + + Ok(Json(serde_json::json!({ + "generated_text": generated_text, + }))) + } + Err(_) => { + Err(poem::Error::from_status(StatusCode::INTERNAL_SERVER_ERROR)) + } + } +} + +#[tokio::main] +async fn main() -> Result<(), std::io::Error> { + tracing_subscriber::fmt::init(); + + let sharded_client = + ShardedClient::connect_uds("/tmp/bloom-inference-0".to_string(), Duration::from_secs(5)) + .await; + sharded_client + .clear_cache() + .await + .expect("Unable to clear cache"); + tracing::info!("Connected"); + + let infer = Infer::new(sharded_client); + + let app = Route::new() + .at("/generate", post(generate)) + .with(AddData::new(infer)); + Server::new(TcpListener::bind("127.0.0.1:3000")) + .run(app) + .await +} diff --git a/server/.DS_Store b/server/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..09937716d14643b9416026cef19e7ff913d0ca36 GIT binary patch literal 6148 zcmeHKOG-mQ5UkcL0xrzb zHN&gL>ofpc9v-iO1%Nr-5$_(R=Fi0Qmp=ZhdVmDUXH#D_~ z==O2mi1Z@T!Hu%EFfv6xlba0Z`{^=VuNNz%&5~Av_cFTv+DB3WDnJFO02QDD(@`Lg z@@_euN8*!E0V*&N1?>A!;KrJ4LH~3h_y_g`kg4sXJl0o!i5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HserR!OQL%o0 zPKtg~PJVtagw)N-hwZU{2>--fIdSHOgv5K}BuzhX^q$}I8|U}Fopw4c0@rtIJJ}yTA%CK>dciZY z1Ej~m1QE0#3Eo*jCoEwUWv}oie&SCWNn;WufqUi`%}JPqPHq&@q?NQL?W9eKWIqrQ zh~^~`%?oxxlNj_+M4(3(Ea?ckM+U9G0F8_W)J7sTX;!3K4?OEX%QIucpQa|S%CxZl zkGVE*C_a>>RH>1Cy8q39+1^N=nCI0?S?-RjN=s#Xd#0LB^YU<4OVW1FCd+Ivns^qeo=~T@~*le_gU3W57+i!vT zHxM~{x)O+~pc{uF+E3x88!y#cru9k%Q?$`hKtasO)(V%NX&dYQnIkzp&CzY5LD94;1&(dOU@$%Bx zHO|wrtW1j2ra4y)#Kja^Pa9BhY%!a4)f1Rcn9bk`tf5y-_@Z&?CrufM;1zZA<|iRc zZ3J3Nw80jMSae|eTB0k~;NBKJ=#@vd;c|7WWA2Sf4n4zRTUq4tD$a7xi zhV%E)^LeLt9SSsgZ5l9FWg94IW~$8GDL9T7%D8=Zfi!j_G&p%p&tWlLu|=OU^$A$4 zH?5}EOkqarwz=IR>N7BW1hjA?s@;IMpwl0N;T|%k+#$aS0%Fdn+BnCd1MLVF;xY>m z1Nz`_L3TdiU#=A2&5Ng{!GjuuA~d9Db1&dIc!4c(&kM|f^8%Z{#{TvOS>Y;B17x@3 zwSka@G!jjVkmEY!+VKBnzDe{Itc1HkLcI%_s)y>Wz63C&1z@`dbT8b)Tc#KB>h1fQ zUBUKR6xMSKsIIR6Ql9DuF>%s1S1>y@=6r{a2C;e@tm++P?;`sY*$2q53iY1B=1`v_ z`vMupP>=9UQg5dBDOleF(pZs@hTewP^}8NSE?Nd%^&#ABd->ItT451wU{3M#>()n& zjoZZb-ACl+J29N@yA7B;zH6A7ywEF4#;r)iT&exiO&f02n{69s*iBE!V!8p;^BC=| F{|781%S8YH literal 0 HcmV?d00001 diff --git a/server/bloom_inference/__pycache__/model.cpython-39.pyc b/server/bloom_inference/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0aeff1a277f9b6e70276691afd54342324fa30c GIT binary patch literal 9759 zcmbtaO^h7Jb?*P^>G|2&+1Wpml2)>nU!}Xhui;eiQru{3u%w85=F5-^=9YknC?`Xby>ijf1 zM%VOBop@%)>e{}|X{+OObAGPt`ffMx=ewToG2iYKx<$X(E%~Kx*)KEC>CAO2eudMy z&V0A(S2^u=7P>XR#_4=#vAg6ib(j5Re)l>n-Bo|Jd&WQ0J?o$Cp7YP?TBMC16WX4R zl=*#46h!f^CW^bpuI@k1oRTPmQ)bQ+C^080c(3dl{uW@pA`BPt-kO% zMy+ks^(-&KBy4oTIL3?hPMB=ht;~9}6(^Z}bvWomnfvx2Y4v+yC$lcM8cE&EO5YtO z{j2@!QLnWV$*e}NZ}mmgdA%<$g~K@Py!lq<5%*HR*KBQ}=DA)!X!Tl2a4TwUZ70;2 zcPVUaN8jru@($jLTTw5PVG;#{&6ny%R*1JlDS|Fln-#lZ(%25-VYAt~js8`xB>lml z)!VuxTS+8aVOH6QZYSKOOWXZ^j9#6i=2~F~T?~@G#ODHolC8~Q5{Ya+j*>t`&2ZSk zGr2Q}8YGd~%fZV<-0>JB(tKTLzVR+*AhLvhL5u9jT-A^^kanV6=^f{bnqqc{m#BF^%y-e9PbrLfb%WE{LM^dLHj2-6ufl#5r|Xnm(Pr%g^fV?D8uW49fqIi@btywLAiTq=iB9(V;|+|!A3 zV<|OKTbRNU_C0f4Oi|k?pXP3w3c+>t7+|mTgE;JBQ((cCyW#Di7u`~liy@ojYTfY5N)|y9_2Ryav)mvQ0xMN6 zLk6GkkH;4mUw9Xr zCw`&d5ie|Z`u(n=&$oKbNJhOz^a8SPMpCh{iVoPkQP&4|vQks_yMbDSgH7si{0fL> zSvvnrPcPzU>4xs8-(&=LOEaUH`2e-^Wbr^u`fzd)ce;g;lkgyRb)jE3Nu0YQUbTD^&Mn^cIgEBTVNZQIZAA z_^Zo9j%2zu7k@ta5MgoI}6Jgq$HEDC-3aCbl&y8N{o%|X8)MJrvYY3Y(E?$PQv z>@%;2gEzExP?F~0NJ*Id`}&Pvqc`PmX`{l8+bQj)l{5$Q;gT6)6H?mvT*tPRM0WDL z;n&`{ytcEj*6Sy0&Hk__)>^$a-u`uy^>0=V6IctvSV1?8uV-_E5JvlYbSJnOc3{98 zRN28r+KrS%OgCxuRJ&Fl>7*KV_4hPbKTOvZO_I)ql`=a~4;s^0KcCr3Umnw6`3nTR zZV%+EZG7<`xV4?YqY3V>05n^_(CIhAPW6kos9=0Z1ihHcLZUgYQy_*NCv)U^9EH=|Y@@63DA3qT89?;vr_95(SOR z#}LFMd+zw~^tNx&a`5v9iz4<5oK$Ng(}T?2jzR%++x3?SY((OukRYRgO2vL_Ct)-{TTrMvr>Gyh$TGY~6?68jW}%v+@Z-!^?}Lr{QsxFh4-hQ~GA|%U zW!Rx~F$iu9!_FkfZ1s|CE(pRN_J2rwJPra$Kv@z-$xOH;@^wl%R~a!f`eSqlm_u$E zFyMAy$P#^0BjOTaQ1=q0ibQC~_yz9nCBjBRUM2Q)uXBn33)ev_G$wH987`)#s+Vbk zAQw+J9+jUm^r~m*T(W^eM=sJm7jeggh)(f}$U7pB2$)@%4|KxdcP&3B2w&&2>g({z zV)XC(%G-lHGH>dpWKMsWz%Qxm3f9dkK1!3^E>YuEkVFHZ(*WYNwh3pzKm*1Gz!dFQ z8GzI+CADkB_`^dk+}t?oG-rgvKn}_Q`t2W3EG8YR={qYYJG;(>HtMc?m+E_u$d8El zM1D+!CyJX4>GiZJpxnR59g~YmQ{WnscqfEygLx5m`~#2!VMiD(1ZxNrg0(~;vVq}r z#!R-)j%bvRfK=e1!mbfQ5iZ=tyzo%IC@jdS2xsxlx8Ht?yl@8YypT_z=tFV8kLF3- zV`C54IMK%F!XAVln7tyu0WQ#6Vzup5C(r91rX_Ha9DpZEj@>;CkPj2Z<=}m_y>_8p zM42L4tGd0BS&g9x;kyimnbnD!$wS>QPJE{zZi%Q~;(a0o`jQn}aS+}NTb*#T6RFKW z+ajA|MKypACc{<_+kibF0@0ZjLAS~0C~5PUedypgQV|Au1$?ZX5AcvQ4Kwq`s{?zT z!xzCnAu}$6XwY9}Xs3!%gLc{}vq0Amifug9+5R<1h6fj=Y07yC(_V%gXc6o1S0PdO zuQWO4#?^G`%BmN=ypB(^T(d84Aw+^z{sG>PeQ>P*a)2WLf;%R4p+QY6 z6>5CBQ&Js2s10_UP18xgid-ZTvWYu>33~)HukUG}J_WBG+pwv%w2v{vztTU^N5+o; z@+|yz5kc!!Uex0A=YdV=_hZ~}G0ZLPXF8sTF@eOw*LG^O9XxZmUEKLydz6EBpCbr; z?Qa2~-S_QLe&me`?2Z)DQtFKhv>GR{k|Fu7F)E{XMdX$V%EEC$dRLeKjGiLml9tAE z0(`YnM{}t&t|apSadW#2d~Xa=b6iPXa!@|j$5mkq=dPA40O)D$8epU=a?3Poq<0N+ zQrNAr@J*RhHd5@~P)+ys_!U4-Y{e_sir7a*a$ex37%g^f09XG8V>QVwNfA>5$c4l4 zk>1o1`El;LsUeE1h?VT>$SER6Lr#fvOyZ8_nFF^8WfiB&oVgj!Lhrf5Hn@IOtEv$> z_Glh7H>!fVqXq6I%B|8YFH(=-4lR-EA#zQ~2yTMnt?cPrhRD%qhy%B06*Zhm=hI4R zr*J=Z9k?w{TBFg|D;tcA85LfwBcc^wZw-PN(KkfuCXtj=;V$jZxB{{Tv0rmX*ff7) z+`x93E_7^+@1THuhsadz^7i#EbP)V&^t*#sS7Aux@1e}jGiv1@1|B)EH3M)DKat<0 zDhc2{1fmAelC&|8^piFPY3l{t@fryCa|H_)n<3Rd)-bIG(WY^t-zN7vvw-*!5K;&X znz_o`Jz$^)@Q6TLtcFD+k`m+@3?*izVmSLs-H??$xX6rAF87I$DfbD^0wxIV!m2jY z6Ifg-@o*x;wMNKD8wbds*Us0 zHURAzf8`LN-jbnURmqc`nLDikizRb9{jF9KPaO&sAtFUCp9g{63KKRq8G@n>1g*2& zBp|1p2r}>TGL^Jt*xQQO;AB?6he>^d^2jhLLq-~!orLml zFa88XL*(&1+z4_daGM17C%^x;=A18peGh)-IkRlKu#(=sV^#6pvoQ8WxC^dX)g1sf zW8XF5>^$-;I6ZiCPzF4>F+~;AG%b7$2WH=RWH`oudBD>Y@tpj=u-u*6Y1*fbU}Wi3 zY}vs8B!LIDKG_o>g`<24US7O#J|x~$`*DiIA~!+)5qC`IVy<2;R?w&FBjz{2I&zWP zpeud_WC}d#+r zz?!?$AjNrb(Etdm5ic%71teMmT)B%t=N?c)V(x11o4||~ZkzBWEHHvu*9mLp5}db; zJ>b(nNL(NSatrKVXupK5LN0=t?c&JnRXIRj?0khIs0o7B%cRBe0Xn;_e zhLZkrTb+{sgV}uU>l571@;wT`y5qIf21dfU%J|7WbtVI6VdN!G;mibx2rYbq z(AP~-0R~$_U>N+gKz#X`D2zPheU^gCn-mB`2<>Y)an$#W@z?i&l+rv0&Tgi;@i!Er zLh5m}kFfCgX_U}{-%6eKGt{GTeNQ_>oM(^slD&kgLJvDOxeDKr5tfN;64?U5*-I%I z1h@LKg8&GfPf={=gax5J!(f7ZU^a*q6+I&?yE_Xiuw==r%iM4??ssr%2nGR+3;~&B zduEUE_9A;MrdDOAKD+u4-GbB51wvwJh5clx&ioKyzU?XZ#f1q_VN zgF2n4^EvGf*yf*8R~&VmTDNw-K|XD87)e}5luX{41`*d)!JV_miU|ZAqJzvDzXjV+ z=fNx6$GERB`+&=ksQ%jsmkn`P#!5u8iXzWED-C2Ha3zL8#gH-!ChvOYO{XSv5a@*2 znR;|`8{erwoV(fV^uy%EZ^Ff)#LF+KkVlJ3us=ucS5}xk&RyJ;{ZKS0Zra~$L$w41 z-Xy6y_9B&m!a*n60~M%Y1WJK1Ii|aDL*1YE7@OCs&iTimPE20`-`XYGY8G%>|HsF-t?=? z1kOYppdTD|6+yv1N1%bh)fqc*ZaxS&{)BMV)QHd&|BzNW?F2g8Vn z8e(%WnjBekts;z|OU4l~Jcqbe5oyE#jd$QTd-%q}_m<&0MerQ`9P+C8?JSqL3sCdm704YF?mYtzr!8`6ZqFoTnpy!4n8_4wT$Md^ zjlG^z719ZWPOkOGIMtg)=!U|n9!IaSzONrBB>6*BPT>*-QUhKOGm}QD>8~(pvtG{? zWtP_`ZvSa|N#3JAv+ENh1zbXL2L(R}Aj&@{@&*wSy!;~~uM_!G5Z+e77FH;>`F-M! zh)^J_o>%Le94hvt)Jefl>2(9duiV5L!=@SstMbg7aA=Eg+q@+In!aG|J3#8sXkC(i z&?w5QeqTB{T2mgNSWRES3Y);LW5Xxdl|L1B(YjKK?GSrqrOD1{L_ofi-NF6&ODfF$ z`D3Ksq5cq%Ld5$GNu2sqY6|E19AbS|ffWw_h69IyoFDq*K(gIwZA$7w=HM?X^e+Oc zPm&E&Sde4LMc&AmNt}bkN>|dhP}=Y8Z00%#zEDpvOj`Jp0I9fakNYEM!ye-1IJ-dDj^QnXT7s=C zycvVb!X`Oq?gg}nxcjS+wH7}O!bb*BDl7OuK@ke3h{K1KoCmaD`HA)ds9XMtrvEQ6 C84{xa literal 0 HcmV?d00001 diff --git a/server/bloom_inference/__pycache__/server.cpython-39.pyc b/server/bloom_inference/__pycache__/server.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3ffb8e5e5b523a5b6787ee24c398b63cccea77e GIT binary patch literal 3683 zcmb_f%WoUU8K2n~mrsc@^>EN8OprDqQcF&n<)m6vuj~M}}jlu@Rb) z?5OAz)w>mzqOwy~<3d=8s!lbkIkjlbSyOX%xE|G=dbHteQ1S_3MOHc`ti*}@PjWF(D-bzK7!NbkM-{Si%EePI+<^#FAKA|@ z9BXjz!<}vc8EGTpk`V<;M>hx~^B(q(Z4y5ZM&(s1!s0F{__Cn@~d!%S(LWi%dh`ze_k_z~pQURHR z_+(0Fgc7Mu^%;0Z)M2`1DBpm$L+%NsOlBTLeR(3#r>g>%_Hmm6H!i0N5VS;xS675v z?7JWqdIFPx66+BRBnX9&>1jQH3{YdHkJV^uj`SJ*oi;YcCahS~!pNLa5Z~?<8D-ij z85wuTgpN$n9+}h9jKlyqrhlt}hm5vQ;q>{1mOxPt;ZD)gM_`ZNSH{44N@nCQ+7kop zfyO?A*p&%{g7G0+m&7fd}{~AW^`%6(b|K<`l&~yqLZ5S-DG1C+#^r>+H6B=M7$Ar}@ ztYG3UV8Z@sFd=>p!id+9TvT4X4s+MwUby`I|Kakl0(q>$=4HMYZ@?L?V(#b0lAjd} z7`l6rK!1sHyoCfak9Zpi`ckxzTof>?&n^9eKYn@crrN5Tn*UGucB_J^HTELzx>?C} zp;Qb)99LZT$-oQq6=9<&1tb{onVw1^Hu3c;lAj@Y2g$FHEJRi#M3txl!8^r%!lqlM zt=ooORr41>qhbyA?BnKLAk36RQmIKD%Ap43P-n)eIo73t31x)!Nt-}o!5#q;h1I?b zC9kAPp3*BTqmC4kY{om%8sV&tvrNs6%p=$(3kc`2C5zLN zERS?qnLq-Z=p$6K9-Umsc1Mjk3K+MJPlp#7&< z3$|DRxC5UFSOnI#7i-Esc6<2hAP&C%Xm4*%M*Y=FeC?a@@cN3Kn$aMY%?@uu(rG?B z=5h1fqRk*}J`@9fDK%q7K;Nr2LZ#-~n1TVopfkFZoT0I@x`~K9+0sN4KFAE=#gEUZ z(?A*BH1QvE=}I2;LujfNSxN=!E#!ce0NQ|EsLW57^00b^mI1o4mj);n69v9u1IsQK z%~X0qa@O=@^P6#NCo4Vj!T~=JA`t>Xo0%Zm5zk7`5)m>t#U%Ycp7H?_6(A@}W+~P} zl;OpB(__rN7=*|U^15)LDxGh|z{sOZ^&X`=2Z#F)?tSs_{sU(nx4`z)?VKn$JH_rG z4AoHR$f9z1@ZizC{R6js`||^5hsRzA0ysa}U39lj9YqPCgYgFf+S5eH%v3sOW?uxc zbQ;e*Xt2N?U~drxv8S41TfB?Xy@%x2NU(iUO^{PmMGqTE@excq)n!Qzb7y^syH#Du z`_9}YIGbJvLQI1i)P`RPe)e-quW2~8fEu*++%PKCfRa;zxf=Y2uPDV9E>*Ajp)Kuh zyJZPHHZwX&5(FU0zc% zyrgwx0q5)WypFD3F8S!i1zL(B)r`WOhHJEPg-Z?*$_f#P{xpL^IDGJ-1-I z2Scm?DUd`3KE`#h;&XBX=0nhMfwfR8W@(owTe3r`95L$mqr(ftYq0I>_cuQljEl`I zrklHQmL(-!kJ3KlEDhNve161ux$G}24+2j8M1$POk5nTGWH|6k;jTty;9_2fdDm8+ z;q=oWVam?ITGf=f|7pf!>WgU1RE5f;oP{>9&z~ij=wN9>#R2O)3yFasm#!-0u{X zv&uP|RHai?&#LEIrdzmo(J9{f$CpHsDKWCDQ*iC1Cf&!Rk7*4wy3{K*o?^aM)F375 zz~L3%)LKEbJ|yC=AE4i3{eBZo>eAb<7qy~J>$Cw#8nRJ9qD(E|`={oUP5pis?Y#Xv zGHFg)MXQupfTU9_93!iB);dRL?Q>K(Me_!lk}Y)b)hSMwX1JOLSL+hj-wduc;A$7` zIWCa)&`5)o?;vYGj|@0J`wq6BFJJ?yGi-nPZ`e-p!58VH8McnWc5Q}@@K#Z~vPSz( zTO$z#RL2DTusUf<@2peQ&heGCAo1VnkLU5*xjAVrN}2oQlpL%S5V3dAXSD+^oAO#= z7tS$666D^|Ho)jit`(jArTy!W8xCE1LNEe-%LS~S!@RK1UV$0p5lH($=Ti6j`1Rl1 z`|!j2YXZ_W>PO+4(4`Qidus^`2SFN%B*gO^hJRfMy4J445I+U72i3ZBdxEEUivEZw z)GAOsB|yoaom=P+pVz@>tlO8>jT}$<&h~|sAsbAwyHuAX@;)F8z7lC zHG!sARyWs;PzFJI+pP zf(v|x&e10bqXX!h4-l`wcj!3|I|QYoOLzwsR80hrn6K;Jpf8C*SKHcSDLc&hwl4So zz(C{MF9$Yf9KU)m3yi~qn6rQ$t%X52V07(K5RaI62)?Pyj6*vzJwZS^zX^kD&%7XBd;AwL#3~SoeP@Cpd3C=g z&w1{mY?v2ww}nAUBN|9{ft4pDOiT;C@*jT%M6PMsT8Rx|?K9}|LFg)?M8tZuO@N6L0Kx8Kk#Md-x$Mv8vM z6UH~7!&9FnyNv39iP_$-TXEtKAat{LuIojgR5W_AcD}#&xCy~6ki}osL|0uIAMGU* zflDu0<6F;}Z;YH9w_x@iWB1%_{u+pSs`37SF#m6$=1{)`O2#){3e@;|AqI1pdL1gD zng?)xOQBp-iGDWhuBZw`W{)W=h+|dBd6Y`7^{95AXNjM42BC{mM(4Rkqfjbaw-04Q z`Uu#W9;phHXAUQ}sWEuV<4AISP^%hCM-X*LQ{_-5p(<4+jI)#}Z_Icm{5Tqd`f-#i zr^Kf!$&f}|*}6d}C&+V_((cmp-K-7Ld){Re5wK0)JbFu}QwYB-4GmZDH-VSGrQuY} zvZ`kznU7?7&+y=u6AsP1*EYKu^~-_~{93t7_jBb%>3kDk*6KTDSGrPV@~dkvV7;wX zoAzsZTfN+2pz8K4VjzgLi1UqOFJTwk#Kz9Ea^Aomthl&A$Twcy!yTA+%x@XrELU9* z>s=k79elZ3UB)fEYBlgGaj>^|#Q9o)bz!nZ-6a(zF6u=@VEvm5P+Ff#W#bK4s? zHle!&o246{>WdpRIb(r#dAFnUNDks?m+NDUuWO=@0o7Qja$&N_T=kSCBv+QsEG^&? zfWG+oe(5UhO`}7(yZtx|G~)6tMxjWmx9 z$(3Vabg6tQ>U)pU(f<;!Jrut6Toh=3A4hAg<>XQ>$mRFVNAi3A$aH0;$Iw2$vX%X& z&)7eRSZ+RuEp+`?41x(>uu1#RC*0BA#B=|?`}c~V3@2e3O`@_h>6Gy#=Im1@d=Wfl zA~^9TT~MKjKt-T>pgJN36@%)7>WUty9;kuni@{SiS&@8yC|1PqDLdhlD}p~@qbo0< z4;zmHd;NYr&5iy-)tS_~R`+vl?5%BC&ZJ7sT*>zP!&HG`SGVig7q4+XU*=@A^K$b* zY@zEL7$hURVJjCNbn}G|Z2}uVNXuD~SJRB77cIbq`dxJW6AXs!^L=mMZ#b=g<{f+B z1dT`jdFGvl!aw83enWeP;}`!$d(N#d^6~}8>Jhj7YF;Lq&XRf*+0fJ~+p}I(+1~EF z%1mBYX<<9M-Zf?Vwe6Sl!sJz5=8pT-T*@N}bptJnT^kk7bvB%-dPh(CmHgTyraq9B ze$Lc)aN@tWHvXuk(i?RlHg<}-F5B_lyxNsYR+-$u=R>Jlc5=aTAW6QOr$zfk-NGJDy`|wM z@o4uIbWK5sxugHDa7LdhDaDglf=PQ)Ik;dBFlqQc3IbuIg!IR%1D=X8AWl)LrU<~u zQ`d-lod$AfJ2+*@3Sv2B89%zxo_XO7^~TEumzKVT50s1xJBfbHPD;Ej5_%iM%e0L} zAi}2{i5rOy5;zbZwC~#0F@nEMvA?@l*9huAu?p>9AC2#2VLNvtSxK~W!{0G6?1UYM zO>i6`tJ$2ZJCxlAzPe+0!%z4zZ^Bd0Am>l`8RtxRkOXEx@)2b|Bp*2ms)OL!g^cfH zn#RX&Of^m>1QjmHgbhogCQ^ShofB3CCO%={bOnP^(O7*C`faQ8+Cj0*k^= zqLA8fx2RKth|OlQ5{g2s$UsF7w4KKeIJCiO%52L_^YdpU9JXE)_Op!y>cWL-B4c}V! zl;E^Llfg0XU01}(&|hifo-OoO3!)>CgRUT`FsL9RI3u>J50EZ1#OtogjZ}I1f-3Gc zM4%LMK3>HH{$&X;0V7JG^{Gof&klhGNhKej5resCI|G497SVkca9S1b_fGp~@Mgnz>GJN4ZSdvCNF1A$VSy;_V0*3T-jfbnnhmVN8D?F|A{*49^^8rxkaNQ=%Q2ZXi~%g{La`h$3K9O z!h3W>us<=*9@00jaW_ykxR`I^#fzZwF|g6O@#?aX!aSKtd$sjjYqw*KJR7>nq_5LM zDH2CqGFtr(8a*P_sOp#p^nxw1=lG*l^-C~qFRLqq>$%8x&d+!5SzmI`gyl@+~y@fu9pH(qicICtf>cO2DEd(lx9xvS0LrmHmPM}Sm&aG6%=R4S*`k^wKv h)yBVSpK8A*&ufZijN6NTZ^XmSFdV))9K^Q+{y$$P-bVlc literal 0 HcmV?d00001 diff --git a/server/bloom_inference/cache.py b/server/bloom_inference/cache.py new file mode 100644 index 00000000..5ef1f1da --- /dev/null +++ b/server/bloom_inference/cache.py @@ -0,0 +1,48 @@ +import torch + +from dataclasses import dataclass +from typing import Dict, Optional, List + +from bloom_inference.pb import generate_pb2 +from bloom_inference.utils import NextTokenChooser, StoppingCriteria + + +@dataclass +class CacheEntry: + batch_id: int + request_ids: List[int] + input_ids: Dict[str, torch.Tensor] + all_input_ids: List[torch.Tensor] + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + + def __len__(self): + return len(self.request_ids) + + def to_pb(self): + return generate_pb2.CacheEntry( + id=self.batch_id, + request_ids=self.request_ids, + sequence_length=max(len(entry) for entry in self.all_input_ids), + ) + + +class Cache: + def __init__(self): + self.cache: Dict[str, CacheEntry] = {} + + def pop(self, batch_id: str) -> Optional[CacheEntry]: + return self.cache.pop(batch_id, None) + + def set(self, entry: CacheEntry): + if entry is not None: + self.cache[entry.batch_id] = entry + + def delete(self, batch_id: str): + del self.cache[batch_id] + + def clear(self): + self.cache.clear() + + def __len__(self): + return len(self.cache.keys()) diff --git a/server/bloom_inference/main.py b/server/bloom_inference/main.py new file mode 100644 index 00000000..b81ed040 --- /dev/null +++ b/server/bloom_inference/main.py @@ -0,0 +1,30 @@ +import typer + +from pathlib import Path +from torch.distributed.launcher import launch_agent, LaunchConfig +from typing import Optional + +from bloom_inference.server import serve + + +def main( + model_name: str, + num_gpus: int = 1, + shard_directory: Optional[Path] = None, +): + if num_gpus == 1: + serve(model_name, False, shard_directory) + + else: + config = LaunchConfig( + min_nodes=1, + max_nodes=1, + nproc_per_node=num_gpus, + rdzv_backend="c10d", + max_restarts=0, + ) + launch_agent(config, serve, [model_name, True, shard_directory]) + + +if __name__ == "__main__": + typer.run(main) diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py new file mode 100644 index 00000000..a762a792 --- /dev/null +++ b/server/bloom_inference/model.py @@ -0,0 +1,428 @@ +import torch +import torch.distributed + +from dataclasses import dataclass +from pathlib import Path +from typing import List, Tuple, Optional, Dict + +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig +from transformers.modeling_utils import no_init_weights + +from bloom_inference.cache import CacheEntry +from bloom_inference.pb import generate_pb2 +from bloom_inference.shard_model import shard_model, match_suffix +from bloom_inference.utils import ( + StoppingCriteria, + NextTokenChooser, + initialize_torch_distributed, + set_default_dtype, +) + +torch.manual_seed(0) + + +@dataclass +class Batch: + batch_id: int + request_ids: List[int] + input_ids: Dict[str, torch.Tensor] + all_input_ids: List[torch.Tensor] + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + + @classmethod + def from_batch_pb( + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + ) -> "Batch": + request_ids = [] + inputs = [] + next_token_choosers = [] + stopping_criterias = [] + + # Parse batch + for r in pb.requests: + request_ids.append(r.id) + inputs.append(r.inputs) + next_token_choosers.append( + NextTokenChooser( + temperature=r.parameters.temperature, + top_k=r.parameters.top_k, + top_p=r.parameters.top_p, + do_sample=r.parameters.do_sample, + ) + ) + stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) + + input_ids = tokenizer(inputs, return_tensors="pt", padding=True).to(device) + all_input_ids = input_ids["input_ids"].unsqueeze(-1) + + return cls( + pb.id, + request_ids, + input_ids, + all_input_ids, + next_token_choosers, + stopping_criterias, + ) + + @classmethod + def from_cache_entry(cls, cache_entry: CacheEntry) -> "Batch": + return cls( + cache_entry.batch_id, + cache_entry.request_ids, + cache_entry.input_ids, + cache_entry.all_input_ids, + cache_entry.next_token_choosers, + cache_entry.stopping_criterias, + ) + + @classmethod + def from_batch_cached_pb(cls, pb: generate_pb2.BatchCached, cache) -> "Batch": + if len(pb.batch_cached_ids) == 1: + cache_entry = cache.pop(pb.batch_cached_ids[0]) + if cache_entry is None: + raise ValueError(f"Batch ID {pb.batch_id} not found in cache") + return cls.from_cache_entry(cache_entry) + + total_batch_size = pb.total_batch_size + max_sequence_length = pb.max_sequence_length + input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} + request_ids = [] + all_input_ids = [] + next_token_choosers = [] + stopping_criterias = [] + start_index = 0 + for i, batch_id in enumerate(pb.batch_cached_ids): + cache_entry = cache.pop(batch_id) + if cache_entry is None: + raise ValueError(f"Batch ID {batch_id} not found in cache") + request_ids.extend(cache_entry.request_ids) + all_input_ids.extend(cache_entry.all_input_ids) + next_token_choosers.extend(cache_entry.next_token_choosers) + stopping_criterias.extend(cache_entry.stopping_criterias) + + batch_size = len(cache_entry.request_ids) + end_index = start_index + batch_size + sequence_length = max(len(entry) for entry in cache_entry.all_input_ids) + + if input_ids["input_ids"] is None: + input_ids["input_ids"] = torch.empty( + (total_batch_size, 1), + dtype=cache_entry.input_ids["input_ids"].dtype, + device=cache_entry.input_ids["input_ids"].device, + ) + + input_ids["input_ids"][start_index:end_index] = cache_entry.input_ids[ + "input_ids" + ] + + if input_ids["attention_mask"] is None: + input_ids["attention_mask"] = torch.zeros( + (total_batch_size, max_sequence_length), + dtype=cache_entry.input_ids["attention_mask"].dtype, + device=cache_entry.input_ids["attention_mask"].device, + ) + + input_ids["attention_mask"][ + start_index:end_index, -sequence_length: + ] = cache_entry.input_ids["attention_mask"][:, -sequence_length:] + + for j, past in enumerate(cache_entry.input_ids["past_key_values"]): + # TODO: this could be done without the views by using indices + past_keys = past[0] + past_values = past[1] + + _, head_dim, padded_sequence_length = past_keys.shape + + past_keys = past_keys.view( + batch_size, -1, head_dim, padded_sequence_length + ) + past_values = past_values.view( + batch_size, -1, padded_sequence_length, head_dim + ) + num_heads = past_keys.shape[1] + + if j == len(input_ids["past_key_values"]): + padded_past_keys = torch.zeros( + ( + total_batch_size, + num_heads, + head_dim, + max_sequence_length - 1, + ), + dtype=past_keys.dtype, + device=past_keys.device, + ) + padded_past_values = torch.zeros( + ( + total_batch_size, + num_heads, + max_sequence_length - 1, + head_dim, + ), + dtype=past_values.dtype, + device=past_values.device, + ) + input_ids["past_key_values"].append( + [padded_past_keys, padded_past_values] + ) + + input_ids["past_key_values"][j][0][ + start_index:end_index, :, :, -(sequence_length - 1): + ] = past_keys[:, :, :, -(sequence_length - 1):] + + input_ids["past_key_values"][j][1][ + start_index:end_index, :, -(sequence_length - 1):, : + ] = past_values[:, :, -(sequence_length - 1):, :] + + if (i + 1) == len(pb.batch_cached_ids): + input_ids["past_key_values"][j][0] = input_ids["past_key_values"][ + j + ][0].view(total_batch_size * num_heads, head_dim, -1) + input_ids["past_key_values"][j][1] = input_ids["past_key_values"][ + j + ][1].view(total_batch_size * num_heads, -1, head_dim) + + start_index += batch_size + + assert pb.request_ids == request_ids + + return cls( + pb.id, + request_ids, + input_ids, + all_input_ids, + next_token_choosers, + stopping_criterias, + ) + + +@dataclass +class FinishedGeneration: + request_id: str + output: str + + def to_pb(self) -> generate_pb2.FinishedGeneration: + return generate_pb2.FinishedGeneration(id=self.request_id, output=self.output) + + +class BLOOM: + def __init__(self, model_name: str): + if torch.cuda.is_available(): + self.device = torch.device("cuda") + else: + self.device = torch.device("cpu") + + self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + self.model = ( + AutoModelForCausalLM.from_pretrained(model_name).eval().to(self.device) + ) + self.num_heads = self.model.base_model.num_heads + + def forward(self, input_ids, attention_mask, past_key_values: Optional = None): + # Model Forward + return self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + def generate_token( + self, batch: Batch + ) -> Tuple[List[FinishedGeneration], Optional[CacheEntry]]: + with torch.no_grad(): + outputs = self.forward(**batch.input_ids) + + # List of indices to cache + cache_indices = [] + cache_past_indices = [] + + # New input_ids for next forward; keep in cache + cache_next_input_ids = [] + cache_all_input_ids = [] + + # Finished requests + finished_generations: List[FinishedGeneration] = [] + + # Zipped iterator + iterator = zip( + batch.request_ids, + outputs.logits, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + ) + + # For each member of the batch + for i, ( + request_id, + logits, + next_token_chooser, + stopping_criteria, + all_tokens, + ) in enumerate(iterator): + # Select next token + next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) + + # Append next token to all tokens + all_tokens = torch.cat([all_tokens, next_token]) + + # Evaluate stopping criteria + if stopping_criteria(all_tokens): + # Decode all tokens + output = self.tokenizer.decode( + all_tokens.squeeze(-1), skip_special_tokens=True + ) + # Add to the list of finished generations with the original request id + finished_generations.append(FinishedGeneration(request_id, output)) + # must be added to the cache + else: + cache_indices.append(i) + cache_past_indices.extend([j for j in range(i * self.num_heads, (i + 1) * self.num_heads)]) + cache_next_input_ids.append(next_token) + cache_all_input_ids.append(all_tokens) + + # No cache is needed, we finished all generations in the batch + if not cache_indices: + return finished_generations, None + + # If we finished at least one generation + cache_input_ids = {"input_ids": torch.cat(cache_next_input_ids, dim=0)} + if finished_generations: + # Apply indices to attention mask, past key values and other items that need to be cached + cache_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ + cache_indices + ] + cache_input_ids["past_key_values"] = [ + (keys[cache_past_indices], values[cache_past_indices]) + for keys, values in outputs["past_key_values"] + ] + cache_request_ids = [batch.request_ids[i] for i in cache_indices] + cache_next_token_choosers = [ + batch.next_token_choosers[i] for i in cache_indices + ] + cache_stopping_criterias = [ + batch.stopping_criterias[i] for i in cache_indices + ] + else: + cache_input_ids["attention_mask"] = batch.input_ids["attention_mask"] + cache_input_ids["past_key_values"] = outputs["past_key_values"] + cache_request_ids = batch.request_ids + cache_next_token_choosers = batch.next_token_choosers + cache_stopping_criterias = batch.stopping_criterias + + # Update attention_mask with padding as we added a new token to input_ids + cache_input_ids["attention_mask"] = torch.cat( + [ + cache_input_ids["attention_mask"], + torch.ones((cache_input_ids["attention_mask"].shape[0], 1)).to( + cache_input_ids["attention_mask"].device + ), + ], + dim=1, + ) + + cache_entry = CacheEntry( + batch.batch_id, + cache_request_ids, + cache_input_ids, + cache_all_input_ids, + cache_next_token_choosers, + cache_stopping_criterias, + ) + return finished_generations, cache_entry + + +class BLOOMSharded(BLOOM): + def __init__(self, model_name: str, shard_directory: Path): + super(BLOOM, self).__init__() + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + self.device = torch.device(f"cuda:{self.rank}") + dtype = torch.bfloat16 + else: + self.device = torch.device("cpu") + dtype = torch.float32 + + self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + + # shard state_dict + if self.master: + # TODO @thomasw21 do some caching + shard_state_dict_paths = shard_model( + model_name, shard_directory, tp_world_size=self.world_size, dtype=dtype + ) + shard_state_dict_paths = [ + str(path.absolute()) for path in shard_state_dict_paths + ] + else: + shard_state_dict_paths = [None] * self.world_size + + torch.distributed.broadcast_object_list( + shard_state_dict_paths, src=0, group=self.process_group + ) + shard_state_dict_path = shard_state_dict_paths[self.rank] + + config = AutoConfig.from_pretrained( + model_name, slow_but_exact=False, tp_parallel=True + ) + config.pad_token_id = 3 + + # The flag below controls whether to allow TF32 on matmul. This flag defaults to False + # in PyTorch 1.12 and later. + torch.backends.cuda.matmul.allow_tf32 = True + + # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. + torch.backends.cudnn.allow_tf32 = True + + with set_default_dtype(dtype): + with no_init_weights(): + # we can probably set the device to `meta` here? + model = AutoModelForCausalLM.from_config(config).to(dtype) + + torch.distributed.barrier(group=self.process_group) + # print_rank_0(f"Initialized model") + state_dict = torch.load(shard_state_dict_path) + # TODO @thomasw21: HACK in order to transpose all weight prior + for key in state_dict.keys(): + do_transpose = False + if not match_suffix(key, "weight"): + continue + + for potential_suffix in [ + "self_attention.query_key_value.weight", + "self_attention.dense.weight", + "dense_h_to_4h.weight", + "dense_4h_to_h.weight", + ]: + if match_suffix(key, potential_suffix): + do_transpose = True + + if do_transpose: + state_dict[key] = state_dict[key].transpose(1, 0).contiguous() + + model.load_state_dict(state_dict) + self.model = model.to(self.device).eval() + self.num_heads = config.n_head // self.process_group.size() + torch.distributed.barrier(group=self.process_group) + + def forward(self, input_ids, attention_mask, past_key_values: Optional = None): + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + logits_shard = outputs.logits[:, -1, :].contiguous() + + batch_size, vocab_shard_size = logits_shard.shape + vocab_size = self.world_size * vocab_shard_size + logits = [torch.empty_like(logits_shard) for _ in range(self.world_size)] + torch.distributed.all_gather(logits, logits_shard, group=self.process_group) + logits = torch.cat(logits, dim=1).view(batch_size, 1, vocab_size) + + outputs.logits = logits + return outputs diff --git a/server/bloom_inference/pb/__init__.py b/server/bloom_inference/pb/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/bloom_inference/pb/__init__.py-e b/server/bloom_inference/pb/__init__.py-e new file mode 100644 index 00000000..e69de29b diff --git a/server/bloom_inference/pb/__pycache__/__init__.cpython-39.pyc b/server/bloom_inference/pb/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1d54d8ecf2bddf9eba6c7de3fe9a4ae98c02952 GIT binary patch literal 161 zcmYe~<>g`kg4Mzf$sqbMh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vFKeRZts8~Ng zCq+LgCqF+ILh5GbrKJ|7<|U`<1EtGSiy*@B2;qVx{dgcZvm`!Vub}c4hfQvNN@-52 L9mxF8K+FIDsFx>q literal 0 HcmV?d00001 diff --git a/server/bloom_inference/pb/__pycache__/generate_pb2.cpython-39.pyc b/server/bloom_inference/pb/__pycache__/generate_pb2.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..936aca2917ee5f063bbf8244d43713e2612dce31 GIT binary patch literal 2182 zcma)7&2HO95GM7n)t^Ntvf?DRtD=Q-sNJ|NdT3BYmS{&sV!M}W*q-;xu15|`Q&d<)w$KmeKDwT+Wzu)KY z4*n=A$}6c%|I(=3#lLuiz!XzqDoZ%(C}Ad5DJPxOC~amUIfZi8%tdk<<-AE$1;|O^ zg<=*VC+{aEmU*F?Wk{Y>pbT?AD=!kbCV5hYGRsCf=7r368p}OR`2Vszl<7PpaD`rF z1;iRF!UD=A#2PC@9pwsQjm^O|l&gp}rokf0^N2Ne1+Jrf6|u%@a0BH9#2Tx^O_Z-8 z*4QE}p?n>&#%{ne$~O^fEZJ|EhF*tdKS{5{LJMi786 zw0TkpV1za+82fdDAkC3(iFutuH(^%RYot7~o>3QmpaFjhu88WE z8efp~rWFhhXR3Zka(+xk)=jP!Hm=Onm@sYpND5JK%NiU4n@NFPnWUgLi?gJ(I+^Rs zVssD%M4lw(k30bnEQiK^VLy)>Q_`-Jc}ZMIKJEah1MV<5tfysP8y^#G%XV#n)h8Ru z=I#s+Y#<*8*ul7gjV(eVwz?bm$1^K0WxPNt7c0wGSZe&FlHA!=xlM8h6B^NWDQqmy z^y?Ciw@D#t0tHDVB3Ia0n29>)c~AL8RX59|7&k7CX=F)SA9=y?#;*xd?!mKQss+1b z?&8MQw3+Nt=;8-6wu>RB57{zI#WOekMBbpM=HXWCBpHzp=IRdKpK{>um`g3BO5riC{0Q2=7Vw zR>Bhre@XaTLM4Gvm9Q_tln_ezQ^G3=|44Y7#L2Z1W+`QSKxtSXaz1n*9zQta)@&E& zhHE)Ku2#Yvos77!a$%v5iIlKF&nKOa15F9%rYzCy^W_%>%@^ihm~F#>HFg5zf-cJX zVYSH^zGT~S@PM%`+kud@^=`}P?DckyaA{Lc+Xo%TaK74 z>~U|(i|Z$mBGl-^u1;I@Ja-o#lzRalPhno8UEO%pY3c1wx3&97Hy#^$cW-y6tA|xQ z=%@6k2rNGcE7K-$nYlpk?{0T`-TkJqryF}sqxnGZ=||w%&I*AI12Q?c$eAj;nQa<5wj%({r=eU!UHUYu<5~yEEc! e?7)}uC5t-F(d)M_W+x9H8GfVd@ z#MW{lxp3eQAP&hf;ve;uQ*PXlED(Iv``)Y_jM7$@yQ+G+tGb>WR;wig3(Q(GYSrDpB zx0tzZGmF`f)L{-JE2v}l`2GA=|D-re*YfVh-jzGClq3x#%%T`qFrQ^nw+$7-9J?TN zn-D)v{koRd-;5JpkLcFnpmhOr1yd#fJ+pJtJ^6hiH9{HR2RC z#{tLF-z-Q=Z{em`u*6#{~bj%Vi$G)^V7`yIiQ3 z`&O1bV0=v8d-I^9R}vnIl`z`od*=_@H?SYyyYydneI4i9`p{;rs>2syb9N zAfHl&)pjS}QAeR+UyFO$S}x!i>g24_ceM*ku1YsW)*kFyn%$*JV~ted(=d;=l{fm- z8>*qonx$oA>gbR>k{d7VNS~)_3$Q89gP=@Kd=bEK9n&&h)BW2k9JJbfZQ4yJueX5!of0JbU&Yc_G8%f7OCh081|xBGIuY`f1~!dc$)x)M;~anSq6F;b5j0u`^Y~I*%u=D`wf2>(5QCb@KpMnnefAb z9^WH5W!?Oc`W%NYS7!7%yN^D@+h2$_m!GfAAyuJz{mG0f7oVfbC^I%|o;ZV##aRMA zfpY}NUj-Rm&`SxrW5Ol0b7;SK9gGW@l3FGY>BKA2pCI~FaUbR#9p^uSn;R{&XSA%` zros;Bwu;RB*&fin5#5=hFMwXs`^!`G3h32IIp zHem*`h|@%2cxhOAC#$Sl9u;2{fI&sBIrE-n_tmXITlH+IuPuo;p{hDFQ8NSNoS?1K zQ-HpKC0z86tzxHGByg0#5`iNG#%6el(3c5lgPg!hot-{Q23njr8Xm4rhK!Afbot;` zS#u~C>Rv=BmsaRHmY)?*sv|RJPPhVjfjUoD%y+pU�-v(YLTuz6W4fj_K%%=%6AV zu=>M24*ox=Q6J!|h(4Vh3qY=6%Fh5MYZP}_Rd?Sg80!UKi@NtF%msEt_uhmRfi3C& zn=p^raEha9aat2yv$V#|o00xyfa_?`nhc~}`W-CDbst$h^sVSxC)~Pc_3U1u=k(lO zvFG&`9#?UX)f7|<5P#Z5D*S#2o~AqIL{z{LV~5agpdEr-<4H--ZOY9z>% literal 0 HcmV?d00001 diff --git a/server/bloom_inference/pb/generate_pb2.py b/server/bloom_inference/pb/generate_pb2.py new file mode 100644 index 00000000..e55a2c77 --- /dev/null +++ b/server/bloom_inference/pb/generate_pb2.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: generate.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0egenerate.proto\x12\x0bgenerate.v1\"(\n\x18ServiceDiscoveryResponse\x12\x0c\n\x04urls\x18\x01 \x03(\t\"^\n\x16LogitsWarperParameters\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_k\x18\x02 \x01(\r\x12\r\n\x05top_p\x18\x03 \x01(\x02\x12\x11\n\tdo_sample\x18\x04 \x01(\x08\"v\n\x07Request\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06inputs\x18\x02 \x01(\t\x12\x37\n\nparameters\x18\x03 \x01(\x0b\x32#.generate.v1.LogitsWarperParameters\x12\x16\n\x0emax_new_tokens\x18\x04 \x01(\r\";\n\x05\x42\x61tch\x12\n\n\x02id\x18\x01 \x01(\x04\x12&\n\x08requests\x18\x02 \x03(\x0b\x32\x14.generate.v1.Request\"\x7f\n\x0b\x42\x61tchCached\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x18\n\x10\x62\x61tch_cached_ids\x18\x03 \x03(\x04\x12\x18\n\x10total_batch_size\x18\x04 \x01(\r\x12\x1b\n\x13max_sequence_length\x18\x05 \x01(\r\"0\n\x12\x46inishedGeneration\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06output\x18\x02 \x01(\t\"F\n\nCacheEntry\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x17\n\x0fsequence_length\x18\x03 \x01(\r\"\x80\x01\n\x08Response\x12\x31\n\x08\x66inished\x18\x01 \x03(\x0b\x32\x1f.generate.v1.FinishedGeneration\x12\x31\n\x0b\x63\x61\x63he_entry\x18\x02 \x01(\x0b\x32\x17.generate.v1.CacheEntryH\x00\x88\x01\x01\x42\x0e\n\x0c_cache_entry\"\x07\n\x05\x45mpty2\x94\x02\n\x0eTextGeneration\x12O\n\x10ServiceDiscovery\x12\x12.generate.v1.Empty\x1a%.generate.v1.ServiceDiscoveryResponse\"\x00\x12\x34\n\nClearCache\x12\x12.generate.v1.Empty\x1a\x12.generate.v1.Empty\x12\x35\n\x08Generate\x12\x12.generate.v1.Batch\x1a\x15.generate.v1.Response\x12\x44\n\x11GenerateWithCache\x12\x18.generate.v1.BatchCached\x1a\x15.generate.v1.Responseb\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'generate_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _SERVICEDISCOVERYRESPONSE._serialized_start=31 + _SERVICEDISCOVERYRESPONSE._serialized_end=71 + _LOGITSWARPERPARAMETERS._serialized_start=73 + _LOGITSWARPERPARAMETERS._serialized_end=167 + _REQUEST._serialized_start=169 + _REQUEST._serialized_end=287 + _BATCH._serialized_start=289 + _BATCH._serialized_end=348 + _BATCHCACHED._serialized_start=350 + _BATCHCACHED._serialized_end=477 + _FINISHEDGENERATION._serialized_start=479 + _FINISHEDGENERATION._serialized_end=527 + _CACHEENTRY._serialized_start=529 + _CACHEENTRY._serialized_end=599 + _RESPONSE._serialized_start=602 + _RESPONSE._serialized_end=730 + _EMPTY._serialized_start=732 + _EMPTY._serialized_end=739 + _TEXTGENERATION._serialized_start=742 + _TEXTGENERATION._serialized_end=1018 +# @@protoc_insertion_point(module_scope) diff --git a/server/bloom_inference/pb/generate_pb2.py-e b/server/bloom_inference/pb/generate_pb2.py-e new file mode 100644 index 00000000..e55a2c77 --- /dev/null +++ b/server/bloom_inference/pb/generate_pb2.py-e @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: generate.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0egenerate.proto\x12\x0bgenerate.v1\"(\n\x18ServiceDiscoveryResponse\x12\x0c\n\x04urls\x18\x01 \x03(\t\"^\n\x16LogitsWarperParameters\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_k\x18\x02 \x01(\r\x12\r\n\x05top_p\x18\x03 \x01(\x02\x12\x11\n\tdo_sample\x18\x04 \x01(\x08\"v\n\x07Request\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06inputs\x18\x02 \x01(\t\x12\x37\n\nparameters\x18\x03 \x01(\x0b\x32#.generate.v1.LogitsWarperParameters\x12\x16\n\x0emax_new_tokens\x18\x04 \x01(\r\";\n\x05\x42\x61tch\x12\n\n\x02id\x18\x01 \x01(\x04\x12&\n\x08requests\x18\x02 \x03(\x0b\x32\x14.generate.v1.Request\"\x7f\n\x0b\x42\x61tchCached\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x18\n\x10\x62\x61tch_cached_ids\x18\x03 \x03(\x04\x12\x18\n\x10total_batch_size\x18\x04 \x01(\r\x12\x1b\n\x13max_sequence_length\x18\x05 \x01(\r\"0\n\x12\x46inishedGeneration\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06output\x18\x02 \x01(\t\"F\n\nCacheEntry\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x17\n\x0fsequence_length\x18\x03 \x01(\r\"\x80\x01\n\x08Response\x12\x31\n\x08\x66inished\x18\x01 \x03(\x0b\x32\x1f.generate.v1.FinishedGeneration\x12\x31\n\x0b\x63\x61\x63he_entry\x18\x02 \x01(\x0b\x32\x17.generate.v1.CacheEntryH\x00\x88\x01\x01\x42\x0e\n\x0c_cache_entry\"\x07\n\x05\x45mpty2\x94\x02\n\x0eTextGeneration\x12O\n\x10ServiceDiscovery\x12\x12.generate.v1.Empty\x1a%.generate.v1.ServiceDiscoveryResponse\"\x00\x12\x34\n\nClearCache\x12\x12.generate.v1.Empty\x1a\x12.generate.v1.Empty\x12\x35\n\x08Generate\x12\x12.generate.v1.Batch\x1a\x15.generate.v1.Response\x12\x44\n\x11GenerateWithCache\x12\x18.generate.v1.BatchCached\x1a\x15.generate.v1.Responseb\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'generate_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _SERVICEDISCOVERYRESPONSE._serialized_start=31 + _SERVICEDISCOVERYRESPONSE._serialized_end=71 + _LOGITSWARPERPARAMETERS._serialized_start=73 + _LOGITSWARPERPARAMETERS._serialized_end=167 + _REQUEST._serialized_start=169 + _REQUEST._serialized_end=287 + _BATCH._serialized_start=289 + _BATCH._serialized_end=348 + _BATCHCACHED._serialized_start=350 + _BATCHCACHED._serialized_end=477 + _FINISHEDGENERATION._serialized_start=479 + _FINISHEDGENERATION._serialized_end=527 + _CACHEENTRY._serialized_start=529 + _CACHEENTRY._serialized_end=599 + _RESPONSE._serialized_start=602 + _RESPONSE._serialized_end=730 + _EMPTY._serialized_start=732 + _EMPTY._serialized_end=739 + _TEXTGENERATION._serialized_start=742 + _TEXTGENERATION._serialized_end=1018 +# @@protoc_insertion_point(module_scope) diff --git a/server/bloom_inference/pb/generate_pb2_grpc.py b/server/bloom_inference/pb/generate_pb2_grpc.py new file mode 100644 index 00000000..e8eb63f0 --- /dev/null +++ b/server/bloom_inference/pb/generate_pb2_grpc.py @@ -0,0 +1,169 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from . import generate_pb2 as generate__pb2 + + +class TextGenerationStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ServiceDiscovery = channel.unary_unary( + '/generate.v1.TextGeneration/ServiceDiscovery', + request_serializer=generate__pb2.Empty.SerializeToString, + response_deserializer=generate__pb2.ServiceDiscoveryResponse.FromString, + ) + self.ClearCache = channel.unary_unary( + '/generate.v1.TextGeneration/ClearCache', + request_serializer=generate__pb2.Empty.SerializeToString, + response_deserializer=generate__pb2.Empty.FromString, + ) + self.Generate = channel.unary_unary( + '/generate.v1.TextGeneration/Generate', + request_serializer=generate__pb2.Batch.SerializeToString, + response_deserializer=generate__pb2.Response.FromString, + ) + self.GenerateWithCache = channel.unary_unary( + '/generate.v1.TextGeneration/GenerateWithCache', + request_serializer=generate__pb2.BatchCached.SerializeToString, + response_deserializer=generate__pb2.Response.FromString, + ) + + +class TextGenerationServicer(object): + """Missing associated documentation comment in .proto file.""" + + def ServiceDiscovery(self, request, context): + """/ Service discovery + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ClearCache(self, request, context): + """/ Empties batch cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Generate(self, request, context): + """/ Generate tokens for a batch without cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GenerateWithCache(self, request, context): + """/ Generate tokens for a batch with cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_TextGenerationServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ServiceDiscovery': grpc.unary_unary_rpc_method_handler( + servicer.ServiceDiscovery, + request_deserializer=generate__pb2.Empty.FromString, + response_serializer=generate__pb2.ServiceDiscoveryResponse.SerializeToString, + ), + 'ClearCache': grpc.unary_unary_rpc_method_handler( + servicer.ClearCache, + request_deserializer=generate__pb2.Empty.FromString, + response_serializer=generate__pb2.Empty.SerializeToString, + ), + 'Generate': grpc.unary_unary_rpc_method_handler( + servicer.Generate, + request_deserializer=generate__pb2.Batch.FromString, + response_serializer=generate__pb2.Response.SerializeToString, + ), + 'GenerateWithCache': grpc.unary_unary_rpc_method_handler( + servicer.GenerateWithCache, + request_deserializer=generate__pb2.BatchCached.FromString, + response_serializer=generate__pb2.Response.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'generate.v1.TextGeneration', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class TextGeneration(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def ServiceDiscovery(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ServiceDiscovery', + generate__pb2.Empty.SerializeToString, + generate__pb2.ServiceDiscoveryResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ClearCache(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ClearCache', + generate__pb2.Empty.SerializeToString, + generate__pb2.Empty.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Generate(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/Generate', + generate__pb2.Batch.SerializeToString, + generate__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GenerateWithCache(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/GenerateWithCache', + generate__pb2.BatchCached.SerializeToString, + generate__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/server/bloom_inference/pb/generate_pb2_grpc.py-e b/server/bloom_inference/pb/generate_pb2_grpc.py-e new file mode 100644 index 00000000..541a6b8c --- /dev/null +++ b/server/bloom_inference/pb/generate_pb2_grpc.py-e @@ -0,0 +1,169 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import generate_pb2 as generate__pb2 + + +class TextGenerationStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ServiceDiscovery = channel.unary_unary( + '/generate.v1.TextGeneration/ServiceDiscovery', + request_serializer=generate__pb2.Empty.SerializeToString, + response_deserializer=generate__pb2.ServiceDiscoveryResponse.FromString, + ) + self.ClearCache = channel.unary_unary( + '/generate.v1.TextGeneration/ClearCache', + request_serializer=generate__pb2.Empty.SerializeToString, + response_deserializer=generate__pb2.Empty.FromString, + ) + self.Generate = channel.unary_unary( + '/generate.v1.TextGeneration/Generate', + request_serializer=generate__pb2.Batch.SerializeToString, + response_deserializer=generate__pb2.Response.FromString, + ) + self.GenerateWithCache = channel.unary_unary( + '/generate.v1.TextGeneration/GenerateWithCache', + request_serializer=generate__pb2.BatchCached.SerializeToString, + response_deserializer=generate__pb2.Response.FromString, + ) + + +class TextGenerationServicer(object): + """Missing associated documentation comment in .proto file.""" + + def ServiceDiscovery(self, request, context): + """/ Service discovery + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ClearCache(self, request, context): + """/ Empties batch cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Generate(self, request, context): + """/ Generate tokens for a batch without cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GenerateWithCache(self, request, context): + """/ Generate tokens for a batch with cache + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_TextGenerationServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ServiceDiscovery': grpc.unary_unary_rpc_method_handler( + servicer.ServiceDiscovery, + request_deserializer=generate__pb2.Empty.FromString, + response_serializer=generate__pb2.ServiceDiscoveryResponse.SerializeToString, + ), + 'ClearCache': grpc.unary_unary_rpc_method_handler( + servicer.ClearCache, + request_deserializer=generate__pb2.Empty.FromString, + response_serializer=generate__pb2.Empty.SerializeToString, + ), + 'Generate': grpc.unary_unary_rpc_method_handler( + servicer.Generate, + request_deserializer=generate__pb2.Batch.FromString, + response_serializer=generate__pb2.Response.SerializeToString, + ), + 'GenerateWithCache': grpc.unary_unary_rpc_method_handler( + servicer.GenerateWithCache, + request_deserializer=generate__pb2.BatchCached.FromString, + response_serializer=generate__pb2.Response.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'generate.v1.TextGeneration', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class TextGeneration(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def ServiceDiscovery(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ServiceDiscovery', + generate__pb2.Empty.SerializeToString, + generate__pb2.ServiceDiscoveryResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ClearCache(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ClearCache', + generate__pb2.Empty.SerializeToString, + generate__pb2.Empty.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Generate(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/Generate', + generate__pb2.Batch.SerializeToString, + generate__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GenerateWithCache(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/GenerateWithCache', + generate__pb2.BatchCached.SerializeToString, + generate__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/server/bloom_inference/prepare_weights.py b/server/bloom_inference/prepare_weights.py new file mode 100644 index 00000000..6fd4ec50 --- /dev/null +++ b/server/bloom_inference/prepare_weights.py @@ -0,0 +1,124 @@ +import torch + +from pathlib import Path +from tqdm import tqdm + +MODEL_NAME = "bigscience/bloom" + + +def match_suffix(text, suffix): + return text[-len(suffix) :] == suffix + + +def prepare_weights(hub_path: Path, save_path: Path, tp_world_size: int): + save_paths = [ + save_path / f"{MODEL_NAME}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" + for tp_rank in range(tp_world_size) + ] + + if all(save_path.exists() for save_path in save_paths): + print("Weights are already prepared") + return + + shards_state_dicts = [{} for _ in range(tp_world_size)] + + for weight_path in tqdm(hub_path.glob("*.bin")): + state_dict = torch.load(weight_path, map_location="cpu") + + keys = list(state_dict.keys()) + for state_name in keys: + state = state_dict[state_name] + if any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.query_key_value.weight", + "self_attention.query_key_value.bias", + "mlp.dense_h_to_4h.weight", + "mlp.dense_h_to_4h.bias", + "word_embeddings.weight", + "lm_head.weight", + ] + ): + output_size = state.shape[0] + assert output_size % tp_world_size == 0 + block_size = output_size // tp_world_size + sharded_weights = torch.split(state, block_size, dim=0) + assert len(sharded_weights) == tp_world_size + for tp_rank, shard in enumerate(sharded_weights): + assert shard.shape[0] == block_size + if match_suffix(state_name, "lm_head.weight"): + shards_state_dicts[tp_rank][state_name] = shard.detach().clone() + else: + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = shard.detach().clone() + elif any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.weight", + "mlp.dense_4h_to_h.weight", + "lm_head.weight", + ] + ): + input_size = state.shape[1] + assert input_size % tp_world_size == 0 + block_size = input_size // tp_world_size + sharded_weights = torch.split(state, block_size, dim=1) + assert len(sharded_weights) == tp_world_size + for tp_rank, shard in enumerate(sharded_weights): + assert shard.shape[1] == block_size + if match_suffix(state_name, "lm_head.weight"): + shards_state_dicts[tp_rank][state_name] = shard.detach().clone() + else: + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = shard.detach().clone() + elif any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.bias", + "mlp.dense_4h_to_h.bias", + ] + ): + shards_state_dicts[0][ + "transformer." + state_name + ] = state.detach().clone() + for tp_rank in range(1, tp_world_size): + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = torch.zeros_like(state) + else: + # We duplicate parameters across tp ranks + for tp_rank in range(tp_world_size): + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = state.detach().clone() + + del state_dict[state_name] # delete key from state_dict + del state # delete tensor + + # we save state_dict + for tp_rank, (save_path, shard_state_dict) in enumerate( + zip(save_paths, shards_state_dicts) + ): + save_paths.append(save_path) + save_path.parent.mkdir(parents=True, exist_ok=True) + if save_path.exists(): + print(f"Skipping {save_path} as it already exists") + else: + torch.save(shard_state_dict, save_path) + + return save_paths + + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser() + + parser.add_argument("--hub-path", required=True, type=str) + parser.add_argument("--save-path", required=True, type=str) + parser.add_argument("--world-size", required=True, type=int) + args = parser.parse_args() + + prepare_weights(Path(args.hub_path), Path(args.save_path), args.world_size) diff --git a/server/bloom_inference/server.py b/server/bloom_inference/server.py new file mode 100644 index 00000000..c652468e --- /dev/null +++ b/server/bloom_inference/server.py @@ -0,0 +1,91 @@ +import asyncio +from grpc import aio + +from grpc_reflection.v1alpha import reflection +from pathlib import Path +from typing import Optional, List + +from bloom_inference.cache import Cache +from bloom_inference.model import BLOOM, Batch, BLOOMSharded +from bloom_inference.pb import generate_pb2_grpc, generate_pb2 + + +class TextGeneration(generate_pb2_grpc.TextGenerationServicer): + def __init__(self, model: BLOOM, cache: Cache, server_urls: List[str]): + self.cache = cache + self.model = model + self.server_urls = server_urls + + async def ServiceDiscovery(self, request, context): + return generate_pb2.ServiceDiscoveryResponse(urls=self.server_urls) + + async def ClearCache(self, request, context): + self.cache.clear() + return generate_pb2.Empty() + + async def Generate(self, request, context): + batch = Batch.from_batch_pb(request, self.model.tokenizer, self.model.device) + finished_generations, cache_entry = self.model.generate_token(batch) + self.cache.set(cache_entry) + + return generate_pb2.Response( + finished=[ + finished_generation.to_pb() + for finished_generation in finished_generations + ], + cache_entry=cache_entry.to_pb() if cache_entry else None, + ) + + async def GenerateWithCache(self, request, context): + batch = Batch.from_batch_cached_pb(request, self.cache) + finished_generations, cache_entry = self.model.generate_token(batch) + self.cache.set(cache_entry) + + return generate_pb2.Response( + finished=[ + finished_generation.to_pb() + for finished_generation in finished_generations + ], + cache_entry=cache_entry.to_pb() if cache_entry else None, + ) + + +def serve(model_name, sharded, shard_directory): + async def serve_inner( + model_name: str, + sharded: bool = False, + shard_directory: Optional[Path] = None, + ): + unix_socket_template = "unix:///tmp/bloom-inference-{}" + if sharded: + if shard_directory is None: + raise ValueError("shard_directory must be set when sharded is True") + model = BLOOMSharded(model_name, shard_directory) + server_urls = [ + unix_socket_template.format(rank) for rank in range(model.world_size) + ] + local_url = unix_socket_template.format(model.rank) + else: + model = BLOOM(model_name) + local_url = unix_socket_template.format(0) + server_urls = [local_url] + + server = aio.server() + generate_pb2_grpc.add_TextGenerationServicer_to_server( + TextGeneration(model, Cache(), server_urls), server + ) + SERVICE_NAMES = ( + generate_pb2.DESCRIPTOR.services_by_name["TextGeneration"].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port(local_url) + await server.start() + print("Server started at {}".format(local_url)) + await server.wait_for_termination() + + asyncio.run(serve_inner(model_name, sharded, shard_directory)) + + +if __name__ == "__main__": + serve("bigscience/bloom-560m", True, Path("/tmp/models")) diff --git a/server/bloom_inference/shard_model.py b/server/bloom_inference/shard_model.py new file mode 100644 index 00000000..360e8962 --- /dev/null +++ b/server/bloom_inference/shard_model.py @@ -0,0 +1,102 @@ +from pathlib import Path + +import torch +from torch import nn +from transformers import AutoModelForCausalLM + + +def match_suffix(text, suffix): + return text[-len(suffix) :] == suffix + + +def shard_model(model_name: str, path: Path, tp_world_size: int, dtype: torch.dtype): + """BLOOM specific sharding mechanism""" + save_paths = [ + path / f"{model_name}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" + for tp_rank in range(tp_world_size) + ] + if all(save_path.exists() for save_path in save_paths): + print("Loading already cached values") + return save_paths + + model: nn.Module = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype=dtype, local_files_only=True + ) + + shards_state_dicts = [{} for _ in range(tp_world_size)] + state_dict = model.state_dict() + keys = list(state_dict.keys()) + for state_name in keys: + print(state_name) + state = state_dict[state_name] + if any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.query_key_value.weight", + "self_attention.query_key_value.bias", + "mlp.dense_h_to_4h.weight", + "mlp.dense_h_to_4h.bias", + "transformer.word_embeddings.weight", + "lm_head.weight", + ] + ): + output_size = state.shape[0] + assert output_size % tp_world_size == 0 + block_size = output_size // tp_world_size + sharded_weights = torch.split(state, block_size, dim=0) + assert len(sharded_weights) == tp_world_size + for tp_rank, shard in enumerate(sharded_weights): + assert shard.shape[0] == block_size + shards_state_dicts[tp_rank][state_name] = shard.detach().clone() + elif any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.weight", + "mlp.dense_4h_to_h.weight", + "lm_head.weight", + ] + ): + input_size = state.shape[1] + assert input_size % tp_world_size == 0 + block_size = input_size // tp_world_size + sharded_weights = torch.split(state, block_size, dim=1) + assert len(sharded_weights) == tp_world_size + for tp_rank, shard in enumerate(sharded_weights): + assert shard.shape[1] == block_size + shards_state_dicts[tp_rank][state_name] = shard.detach().clone() + elif any( + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.bias", + "mlp.dense_4h_to_h.bias", + ] + ): + shards_state_dicts[0][state_name] = state.detach().clone() + for tp_rank in range(1, tp_world_size): + shards_state_dicts[tp_rank][state_name] = torch.zeros_like(state) + else: + # We duplicate parameters across tp ranks + for tp_rank in range(tp_world_size): + shards_state_dicts[tp_rank][state_name] = state.detach().clone() + + del state_dict[state_name] # delete key from state_dict + del state # delete tensor + + # we save state_dict + for tp_rank, (save_path, shard_state_dict) in enumerate( + zip(save_paths, shards_state_dicts) + ): + save_path.parent.mkdir(parents=True, exist_ok=True) + torch.save(shard_state_dict, save_path) + save_paths.append(save_path) + + return save_paths + + +if __name__ == "__main__": + model_name = "bigscience/bloom" + save_path = Path("/data/shards") + tp_world_size = 8 + dtype = torch.bfloat16 + + shard_model(model_name, save_path, tp_world_size=tp_world_size, dtype=dtype) diff --git a/server/bloom_inference/utils.py b/server/bloom_inference/utils.py new file mode 100644 index 00000000..db02dadb --- /dev/null +++ b/server/bloom_inference/utils.py @@ -0,0 +1,95 @@ +import os +import contextlib +import torch +import torch.distributed +from transformers.generation_logits_process import ( + LogitsProcessorList, + TemperatureLogitsWarper, + TopPLogitsWarper, + TopKLogitsWarper, +) + + +class Sampling: + def __call__(self, logits): + probs = torch.nn.functional.softmax(logits, dim=-1) + next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1) + return next_tokens + + +class Greedy: + def __call__(self, logits): + return logits.argmax(dim=-1) + + +class NextTokenChooser: + def __init__(self, temperature=1.0, top_k=None, top_p=None, do_sample=False): + warpers = LogitsProcessorList() + # the following idea is largely copied from this PR: https://github.com/huggingface/transformers/pull/5420/files + # all samplers can be found in `generation_utils_samplers.py` + sampling = do_sample + if temperature is not None and temperature != 1.0: + temperature = float(temperature) + warpers.append(TemperatureLogitsWarper(temperature)) + sampling = True + if top_k is not None and top_k != 0: + warpers.append(TopKLogitsWarper(top_k=top_k)) + sampling = True + if top_p is not None and top_p < 1.0: + warpers.append(TopPLogitsWarper(top_p=top_p)) + sampling = True + + self.warpers = warpers + self.choice = Sampling() if sampling else Greedy() + + def __call__(self, input_ids, scores): + scores = self.warpers(input_ids, scores) + next_ids = self.choice(scores) + return next_ids.unsqueeze(-1) + + +class StoppingCriteria: + def __init__(self, max_new_tokens=20): + self.max_new_tokens = max_new_tokens + self.current_tokens = 0 + + def __call__(self, all_ids): + self.current_tokens += 1 + if self.current_tokens >= self.max_new_tokens: + return True + return False + + +def initialize_torch_distributed(): + rank = int(os.getenv("RANK", "0")) + world_size = int(os.getenv("WORLD_SIZE", "1")) + + if torch.cuda.is_available(): + # initialized `torch.distributed` + # Set the device id. + assert world_size <= torch.cuda.device_count(), "Each process is one gpu" + device = rank % torch.cuda.device_count() + torch.cuda.set_device(device) + backend = "nccl" + else: + backend = "gloo" + + # Call the init process. + torch.distributed.init_process_group( + backend=backend, + world_size=world_size, + rank=rank, + init_method="tcp://localhost:6000", + ) + + return torch.distributed.distributed_c10d._get_default_group(), rank, world_size + + +@contextlib.contextmanager +def set_default_dtype(dtype): + saved_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + try: + yield + finally: + torch.set_default_dtype(saved_dtype) diff --git a/server/poetry.lock b/server/poetry.lock new file mode 100644 index 00000000..ea20ef26 --- /dev/null +++ b/server/poetry.lock @@ -0,0 +1,480 @@ +[[package]] +name = "accelerate" +version = "0.12.0" +description = "Accelerate" +category = "main" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +numpy = ">=1.17" +packaging = ">=20.0" +psutil = "*" +pyyaml = "*" +torch = ">=1.4.0" + +[package.extras] +dev = ["black (>=22.0,<23.0)", "datasets", "deepspeed (<0.7.0)", "evaluate", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scipy", "sklearn", "tqdm", "transformers"] +quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)"] +sagemaker = ["sagemaker"] +test_dev = ["datasets", "deepspeed (<0.7.0)", "evaluate", "scipy", "sklearn", "tqdm", "transformers"] +test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] +test_trackers = ["comet-ml", "tensorboard", "wandb"] +testing = ["datasets", "deepspeed (<0.7.0)", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scipy", "sklearn", "tqdm", "transformers"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.5" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "grpcio" +version = "1.49.1" +description = "HTTP/2-based RPC framework" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +six = ">=1.5.2" + +[package.extras] +protobuf = ["grpcio-tools (>=1.49.1)"] + +[[package]] +name = "grpcio-reflection" +version = "1.49.1" +description = "Standard Protobuf Reflection Service for gRPC" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +grpcio = ">=1.49.1" +protobuf = ">=4.21.3" + +[[package]] +name = "grpcio-tools" +version = "1.49.1" +description = "Protobuf code generator for gRPC" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +grpcio = ">=1.49.1" +protobuf = ">=4.21.3,<5.0dev" +setuptools = "*" + +[[package]] +name = "numpy" +version = "1.23.3" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.8" + +[[package]] +name = "packaging" +version = "21.3" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" + +[[package]] +name = "protobuf" +version = "4.21.7" +description = "" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "psutil" +version = "5.9.2" +description = "Cross-platform lib for process and system monitoring in Python." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "PyYAML" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "setuptools" +version = "65.4.1" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "torch" +version = "1.12.1" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +category = "main" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +typing-extensions = "*" + +[[package]] +name = "typer" +version = "0.6.1" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +click = ">=7.1.1,<9.0.0" + +[package.extras] +all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] + +[[package]] +name = "typing-extensions" +version = "4.3.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[metadata] +lock-version = "1.1" +python-versions = "^3.9" +content-hash = "cedd0aebeb3731e2bbddf017a2ee6074c285866354272f8dfe930e9606437a25" + +[metadata.files] +accelerate = [ + {file = "accelerate-0.12.0-py3-none-any.whl", hash = "sha256:7742ca5c9f15dd1e0a283305599c196e260af4717a561d1f544aeab27d828af6"}, + {file = "accelerate-0.12.0.tar.gz", hash = "sha256:e8b119c94fac31877620d5f9de311164ec81fa9dc9e175f0d0d4f50fc8d79473"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] +colorama = [ + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, +] +grpcio = [ + {file = "grpcio-1.49.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:fd86040232e805b8e6378b2348c928490ee595b058ce9aaa27ed8e4b0f172b20"}, + {file = "grpcio-1.49.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6fd0c9cede9552bf00f8c5791d257d5bf3790d7057b26c59df08be5e7a1e021d"}, + {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:d0d402e158d4e84e49c158cb5204119d55e1baf363ee98d6cb5dce321c3a065d"}, + {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:822ceec743d42a627e64ea266059a62d214c5a3cdfcd0d7fe2b7a8e4e82527c7"}, + {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2106d9c16527f0a85e2eea6e6b91a74fc99579c60dd810d8690843ea02bc0f5f"}, + {file = "grpcio-1.49.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:52dd02b7e7868233c571b49bc38ebd347c3bb1ff8907bb0cb74cb5f00c790afc"}, + {file = "grpcio-1.49.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:120fecba2ec5d14b5a15d11063b39783fda8dc8d24addd83196acb6582cabd9b"}, + {file = "grpcio-1.49.1-cp310-cp310-win32.whl", hash = "sha256:f1a3b88e3c53c1a6e6bed635ec1bbb92201bb6a1f2db186179f7f3f244829788"}, + {file = "grpcio-1.49.1-cp310-cp310-win_amd64.whl", hash = "sha256:a7d0017b92d3850abea87c1bdec6ea41104e71c77bca44c3e17f175c6700af62"}, + {file = "grpcio-1.49.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:9fb17ff8c0d56099ac6ebfa84f670c5a62228d6b5c695cf21c02160c2ac1446b"}, + {file = "grpcio-1.49.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:075f2d06e3db6b48a2157a1bcd52d6cbdca980dd18988fe6afdb41795d51625f"}, + {file = "grpcio-1.49.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46d93a1b4572b461a227f1db6b8d35a88952db1c47e5fadcf8b8a2f0e1dd9201"}, + {file = "grpcio-1.49.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc79b2b37d779ac42341ddef40ad5bf0966a64af412c89fc2b062e3ddabb093f"}, + {file = "grpcio-1.49.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5f8b3a971c7820ea9878f3fd70086240a36aeee15d1b7e9ecbc2743b0e785568"}, + {file = "grpcio-1.49.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49b301740cf5bc8fed4fee4c877570189ae3951432d79fa8e524b09353659811"}, + {file = "grpcio-1.49.1-cp311-cp311-win32.whl", hash = "sha256:1c66a25afc6c71d357867b341da594a5587db5849b48f4b7d5908d236bb62ede"}, + {file = "grpcio-1.49.1-cp311-cp311-win_amd64.whl", hash = "sha256:6b6c3a95d27846f4145d6967899b3ab25fffc6ae99544415e1adcacef84842d2"}, + {file = "grpcio-1.49.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:1cc400c8a2173d1c042997d98a9563e12d9bb3fb6ad36b7f355bc77c7663b8af"}, + {file = "grpcio-1.49.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:34f736bd4d0deae90015c0e383885b431444fe6b6c591dea288173df20603146"}, + {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:196082b9c89ebf0961dcd77cb114bed8171964c8e3063b9da2fb33536a6938ed"}, + {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c9f89c42749890618cd3c2464e1fbf88446e3d2f67f1e334c8e5db2f3272bbd"}, + {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64419cb8a5b612cdb1550c2fd4acbb7d4fb263556cf4625f25522337e461509e"}, + {file = "grpcio-1.49.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8a5272061826e6164f96e3255405ef6f73b88fd3e8bef464c7d061af8585ac62"}, + {file = "grpcio-1.49.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ea9d0172445241ad7cb49577314e39d0af2c5267395b3561d7ced5d70458a9f3"}, + {file = "grpcio-1.49.1-cp37-cp37m-win32.whl", hash = "sha256:2070e87d95991473244c72d96d13596c751cb35558e11f5df5414981e7ed2492"}, + {file = "grpcio-1.49.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fcedcab49baaa9db4a2d240ac81f2d57eb0052b1c6a9501b46b8ae912720fbf"}, + {file = "grpcio-1.49.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:afbb3475cf7f4f7d380c2ca37ee826e51974f3e2665613996a91d6a58583a534"}, + {file = "grpcio-1.49.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:a4f9ba141380abde6c3adc1727f21529137a2552002243fa87c41a07e528245c"}, + {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:cf0a1fb18a7204b9c44623dfbd1465b363236ce70c7a4ed30402f9f60d8b743b"}, + {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17bb6fe72784b630728c6cff9c9d10ccc3b6d04e85da6e0a7b27fb1d135fac62"}, + {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18305d5a082d1593b005a895c10041f833b16788e88b02bb81061f5ebcc465df"}, + {file = "grpcio-1.49.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b6a1b39e59ac5a3067794a0e498911cf2e37e4b19ee9e9977dc5e7051714f13f"}, + {file = "grpcio-1.49.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e20d59aafc086b1cc68400463bddda6e41d3e5ed30851d1e2e0f6a2e7e342d3"}, + {file = "grpcio-1.49.1-cp38-cp38-win32.whl", hash = "sha256:e1e83233d4680863a421f3ee4a7a9b80d33cd27ee9ed7593bc93f6128302d3f2"}, + {file = "grpcio-1.49.1-cp38-cp38-win_amd64.whl", hash = "sha256:221d42c654d2a41fa31323216279c73ed17d92f533bc140a3390cc1bd78bf63c"}, + {file = "grpcio-1.49.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:fa9e6e61391e99708ac87fc3436f6b7b9c6b845dc4639b406e5e61901e1aacde"}, + {file = "grpcio-1.49.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9b449e966ef518ce9c860d21f8afe0b0f055220d95bc710301752ac1db96dd6a"}, + {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:aa34d2ad9f24e47fa9a3172801c676e4037d862247e39030165fe83821a7aafd"}, + {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5207f4eed1b775d264fcfe379d8541e1c43b878f2b63c0698f8f5c56c40f3d68"}, + {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b24a74651438d45619ac67004638856f76cc13d78b7478f2457754cbcb1c8ad"}, + {file = "grpcio-1.49.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:fe763781669790dc8b9618e7e677c839c87eae6cf28b655ee1fa69ae04eea03f"}, + {file = "grpcio-1.49.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f2ff7ba0f8f431f32d4b4bc3a3713426949d3533b08466c4ff1b2b475932ca8"}, + {file = "grpcio-1.49.1-cp39-cp39-win32.whl", hash = "sha256:08ff74aec8ff457a89b97152d36cb811dcc1d17cd5a92a65933524e363327394"}, + {file = "grpcio-1.49.1-cp39-cp39-win_amd64.whl", hash = "sha256:274ffbb39717918c514b35176510ae9be06e1d93121e84d50b350861dcb9a705"}, + {file = "grpcio-1.49.1.tar.gz", hash = "sha256:d4725fc9ec8e8822906ae26bb26f5546891aa7fbc3443de970cc556d43a5c99f"}, +] +grpcio-reflection = [ + {file = "grpcio-reflection-1.49.1.tar.gz", hash = "sha256:b755dfe61d5255a02fb8d0d845bd0027847dee68bf0763a2b286d664ed07ec4d"}, + {file = "grpcio_reflection-1.49.1-py3-none-any.whl", hash = "sha256:70a325a83c1c1ab583d368711e5733cbef5e068ad2c17cbe77df6e47e0311d1f"}, +] +grpcio-tools = [ + {file = "grpcio-tools-1.49.1.tar.gz", hash = "sha256:84cc64e5b46bad43d5d7bd2fd772b656eba0366961187a847e908e2cb735db91"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:2dfb6c7ece84d46bd690b23d3e060d18115c8bc5047d2e8a33e6747ed323a348"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:8f452a107c054a04db2570f7851a07f060313c6e841b0d394ce6030d598290e6"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:6a198871b582287213c4d70792bf275e1d7cf34eed1d019f534ddf4cd15ab039"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0cca67a7d0287bdc855d81fdd38dc949c4273273a74f832f9e520abe4f20bc6"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdaff4c89eecb37c247b93025410db68114d97fa093cbb028e9bd7cda5912473"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bb8773118ad315db317d7b22b5ff75d649ca20931733281209e7cbd8c0fad53e"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cc5534023735b8a8f56760b7c533918f874ce5a9064d7c5456d2709ae2b31f9"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-win32.whl", hash = "sha256:d277642acbe305f5586f9597b78fb9970d6633eb9f89c61e429c92c296c37129"}, + {file = "grpcio_tools-1.49.1-cp310-cp310-win_amd64.whl", hash = "sha256:eed599cf08fc1a06c72492d3c5750c32f58de3750eddd984af1f257c14326701"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:9e5c13809ab2f245398e8446c4c3b399a62d591db651e46806cccf52a700452e"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:ab3d0ee9623720ee585fdf3753b3755d3144a4a8ae35bca8e3655fa2f41056be"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ba87e3512bc91d78bf9febcfb522eadda171d2d4ddaf886066b0f01aa4929ad"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e13b3643e7577a3ec13b79689eb4d7548890b1e104c04b9ed6557a3c3dd452"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:324f67d9cb4b7058b6ce45352fb64c20cc1fa04c34d97ad44772cfe6a4ae0cf5"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a64bab81b220c50033f584f57978ebbea575f09c1ccee765cd5c462177988098"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-win32.whl", hash = "sha256:f632d376f92f23e5931697a3acf1b38df7eb719774213d93c52e02acd2d529ac"}, + {file = "grpcio_tools-1.49.1-cp311-cp311-win_amd64.whl", hash = "sha256:28ff2b978d9509474928b9c096a0cce4eaa9c8f7046136aee1545f6211ed8126"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:46afd3cb7e555187451a5d283f108cdef397952a662cb48680afc615b158864a"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:9284568b728e41fa8f7e9c2e7399545d605f75d8072ef0e9aa2a05655cb679eb"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:aa34442cf787732cb41f2aa6172007e24f480b8b9d3dc5166de80d63e9072ea4"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b8c9eb5a4250905414cd53a68caea3eb8f0c515aadb689e6e81b71ebe9ab5c6"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab15db024051bf21feb21c29cb2c3ea0a2e4f5cf341d46ef76e17fcf6aaef164"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:502084b622f758bef620a9107c2db9fcdf66d26c7e0e481d6bb87db4dc917d70"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4085890b77c640085f82bf1e90a0ea166ce48000bc2f5180914b974783c9c0a8"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-win32.whl", hash = "sha256:da0edb984699769ce02e18e3392d54b59a7a3f93acd285a68043f5bde4fc028e"}, + {file = "grpcio_tools-1.49.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9887cd622770271101a7dd1832845d64744c3f88fd11ccb2620394079197a42e"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:8440fe7dae6a40c279e3a24b82793735babd38ecbb0d07bb712ff9c8963185d9"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:b5de2bb7dd6b6231da9b1556ade981513330b740e767f1d902c71ceee0a7d196"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:1e6f06a763aea7836b63d9c117347f2bf7038008ceef72758815c9e09c5fb1fc"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e31562f90120318c5395aabec0f2f69ad8c14b6676996b7730d9d2eaf9415d57"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49ef9a4e389a618157a9daa9fafdfeeaef1ece9adda7f50f85db928f24d4b3e8"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b384cb8e8d9bcb55ee8f9b064374561c7a1a05d848249581403d36fc7060032f"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73732f77943ac3e898879cbb29c27253aa3c47566b8a59780fd24c6a54de1b66"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-win32.whl", hash = "sha256:b594b2745a5ba9e7a76ce561bc5ab40bc65bb44743c505529b1e4f12af29104d"}, + {file = "grpcio_tools-1.49.1-cp38-cp38-win_amd64.whl", hash = "sha256:680fbc88f8709ddcabb88f86749f2d8e429160890cff2c70680880a6970d4eef"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:e8c3869121860f6767eedb7d24fc54dfd71e737fdfbb26e1334684606f3274fd"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:73e9d7c886ba10e20c97d1dab0ff961ba5800757ae5e31be21b1cda8130c52f8"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:1760de2dd2c4f08de87b039043a4797f3c17193656e7e3eb84e92f0517083c0c"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd4b1e216dd04d9245ee8f4e601a1f98c25e6e417ea5cf8d825c50589a8b447e"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1c28751ab5955cae563d07677e799233f0fe1c0fc49d9cbd61ff1957e83617f"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c24239c3ee9ed16314c14b4e24437b5079ebc344f343f33629a582f8699f583b"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:892d3dacf1942820f0b7a868a30e6fbcdf5bec08543b682c7274b0101cee632d"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-win32.whl", hash = "sha256:704d21509ec06efc9d034dbe70e7152715aac004941f4f0f553cf3a0aff15bd5"}, + {file = "grpcio_tools-1.49.1-cp39-cp39-win_amd64.whl", hash = "sha256:1efa0c221c719433f441ac0e026fc3c4dbc9a1a08a552ecdc707775e2f2fbbae"}, +] +numpy = [ + {file = "numpy-1.23.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c9f707b5bb73bf277d812ded9896f9512a43edff72712f31667d0a8c2f8e71ee"}, + {file = "numpy-1.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ffcf105ecdd9396e05a8e58e81faaaf34d3f9875f137c7372450baa5d77c9a54"}, + {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ea3f98a0ffce3f8f57675eb9119f3f4edb81888b6874bc1953f91e0b1d4f440"}, + {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:004f0efcb2fe1c0bd6ae1fcfc69cc8b6bf2407e0f18be308612007a0762b4089"}, + {file = "numpy-1.23.3-cp310-cp310-win32.whl", hash = "sha256:98dcbc02e39b1658dc4b4508442a560fe3ca5ca0d989f0df062534e5ca3a5c1a"}, + {file = "numpy-1.23.3-cp310-cp310-win_amd64.whl", hash = "sha256:39a664e3d26ea854211867d20ebcc8023257c1800ae89773cbba9f9e97bae036"}, + {file = "numpy-1.23.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1f27b5322ac4067e67c8f9378b41c746d8feac8bdd0e0ffede5324667b8a075c"}, + {file = "numpy-1.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ad3ec9a748a8943e6eb4358201f7e1c12ede35f510b1a2221b70af4bb64295c"}, + {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdc9febce3e68b697d931941b263c59e0c74e8f18861f4064c1f712562903411"}, + {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301c00cf5e60e08e04d842fc47df641d4a181e651c7135c50dc2762ffe293dbd"}, + {file = "numpy-1.23.3-cp311-cp311-win32.whl", hash = "sha256:7cd1328e5bdf0dee621912f5833648e2daca72e3839ec1d6695e91089625f0b4"}, + {file = "numpy-1.23.3-cp311-cp311-win_amd64.whl", hash = "sha256:8355fc10fd33a5a70981a5b8a0de51d10af3688d7a9e4a34fcc8fa0d7467bb7f"}, + {file = "numpy-1.23.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc6e8da415f359b578b00bcfb1d08411c96e9a97f9e6c7adada554a0812a6cc6"}, + {file = "numpy-1.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:22d43376ee0acd547f3149b9ec12eec2f0ca4a6ab2f61753c5b29bb3e795ac4d"}, + {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a64403f634e5ffdcd85e0b12c08f04b3080d3e840aef118721021f9b48fc1460"}, + {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd9d3abe5774404becdb0748178b48a218f1d8c44e0375475732211ea47c67e"}, + {file = "numpy-1.23.3-cp38-cp38-win32.whl", hash = "sha256:f8c02ec3c4c4fcb718fdf89a6c6f709b14949408e8cf2a2be5bfa9c49548fd85"}, + {file = "numpy-1.23.3-cp38-cp38-win_amd64.whl", hash = "sha256:e868b0389c5ccfc092031a861d4e158ea164d8b7fdbb10e3b5689b4fc6498df6"}, + {file = "numpy-1.23.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09f6b7bdffe57fc61d869a22f506049825d707b288039d30f26a0d0d8ea05164"}, + {file = "numpy-1.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8c79d7cf86d049d0c5089231a5bcd31edb03555bd93d81a16870aa98c6cfb79d"}, + {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5d5420053bbb3dd64c30e58f9363d7a9c27444c3648e61460c1237f9ec3fa14"}, + {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5422d6a1ea9b15577a9432e26608c73a78faf0b9039437b075cf322c92e98e7"}, + {file = "numpy-1.23.3-cp39-cp39-win32.whl", hash = "sha256:c1ba66c48b19cc9c2975c0d354f24058888cdc674bebadceb3cdc9ec403fb5d1"}, + {file = "numpy-1.23.3-cp39-cp39-win_amd64.whl", hash = "sha256:78a63d2df1d947bd9d1b11d35564c2f9e4b57898aae4626638056ec1a231c40c"}, + {file = "numpy-1.23.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:17c0e467ade9bda685d5ac7f5fa729d8d3e76b23195471adae2d6a6941bd2c18"}, + {file = "numpy-1.23.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91b8d6768a75247026e951dce3b2aac79dc7e78622fc148329135ba189813584"}, + {file = "numpy-1.23.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:94c15ca4e52671a59219146ff584488907b1f9b3fc232622b47e2cf832e94fb8"}, + {file = "numpy-1.23.3.tar.gz", hash = "sha256:51bf49c0cd1d52be0a240aa66f3458afc4b95d8993d2d04f0d91fa60c10af6cd"}, +] +packaging = [ + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, +] +protobuf = [ + {file = "protobuf-4.21.7-cp310-abi3-win32.whl", hash = "sha256:c7cb105d69a87416bd9023e64324e1c089593e6dae64d2536f06bcbe49cd97d8"}, + {file = "protobuf-4.21.7-cp310-abi3-win_amd64.whl", hash = "sha256:3ec85328a35a16463c6f419dbce3c0fc42b3e904d966f17f48bae39597c7a543"}, + {file = "protobuf-4.21.7-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:db9056b6a11cb5131036d734bcbf91ef3ef9235d6b681b2fc431cbfe5a7f2e56"}, + {file = "protobuf-4.21.7-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:ca200645d6235ce0df3ccfdff1567acbab35c4db222a97357806e015f85b5744"}, + {file = "protobuf-4.21.7-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:b019c79e23a80735cc8a71b95f76a49a262f579d6b84fd20a0b82279f40e2cc1"}, + {file = "protobuf-4.21.7-cp37-cp37m-win32.whl", hash = "sha256:d3f89ccf7182293feba2de2739c8bf34fed1ed7c65a5cf987be00311acac57c1"}, + {file = "protobuf-4.21.7-cp37-cp37m-win_amd64.whl", hash = "sha256:a74d96cd960b87b4b712797c741bb3ea3a913f5c2dc4b6cbe9c0f8360b75297d"}, + {file = "protobuf-4.21.7-cp38-cp38-win32.whl", hash = "sha256:8e09d1916386eca1ef1353767b6efcebc0a6859ed7f73cb7fb974feba3184830"}, + {file = "protobuf-4.21.7-cp38-cp38-win_amd64.whl", hash = "sha256:9e355f2a839d9930d83971b9f562395e13493f0e9211520f8913bd11efa53c02"}, + {file = "protobuf-4.21.7-cp39-cp39-win32.whl", hash = "sha256:f370c0a71712f8965023dd5b13277444d3cdfecc96b2c778b0e19acbfd60df6e"}, + {file = "protobuf-4.21.7-cp39-cp39-win_amd64.whl", hash = "sha256:9643684232b6b340b5e63bb69c9b4904cdd39e4303d498d1a92abddc7e895b7f"}, + {file = "protobuf-4.21.7-py2.py3-none-any.whl", hash = "sha256:8066322588d4b499869bf9f665ebe448e793036b552f68c585a9b28f1e393f66"}, + {file = "protobuf-4.21.7-py3-none-any.whl", hash = "sha256:58b81358ec6c0b5d50df761460ae2db58405c063fd415e1101209221a0a810e1"}, + {file = "protobuf-4.21.7.tar.gz", hash = "sha256:71d9dba03ed3432c878a801e2ea51e034b0ea01cf3a4344fb60166cb5f6c8757"}, +] +psutil = [ + {file = "psutil-5.9.2-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:8f024fbb26c8daf5d70287bb3edfafa22283c255287cf523c5d81721e8e5d82c"}, + {file = "psutil-5.9.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:b2f248ffc346f4f4f0d747ee1947963613216b06688be0be2e393986fe20dbbb"}, + {file = "psutil-5.9.2-cp27-cp27m-win32.whl", hash = "sha256:b1928b9bf478d31fdffdb57101d18f9b70ed4e9b0e41af751851813547b2a9ab"}, + {file = "psutil-5.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:404f4816c16a2fcc4eaa36d7eb49a66df2d083e829d3e39ee8759a411dbc9ecf"}, + {file = "psutil-5.9.2-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:94e621c6a4ddb2573d4d30cba074f6d1aa0186645917df42c811c473dd22b339"}, + {file = "psutil-5.9.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:256098b4f6ffea6441eb54ab3eb64db9ecef18f6a80d7ba91549195d55420f84"}, + {file = "psutil-5.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:614337922702e9be37a39954d67fdb9e855981624d8011a9927b8f2d3c9625d9"}, + {file = "psutil-5.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39ec06dc6c934fb53df10c1672e299145ce609ff0611b569e75a88f313634969"}, + {file = "psutil-5.9.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3ac2c0375ef498e74b9b4ec56df3c88be43fe56cac465627572dbfb21c4be34"}, + {file = "psutil-5.9.2-cp310-cp310-win32.whl", hash = "sha256:e4c4a7636ffc47b7141864f1c5e7d649f42c54e49da2dd3cceb1c5f5d29bfc85"}, + {file = "psutil-5.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:f4cb67215c10d4657e320037109939b1c1d2fd70ca3d76301992f89fe2edb1f1"}, + {file = "psutil-5.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dc9bda7d5ced744622f157cc8d8bdd51735dafcecff807e928ff26bdb0ff097d"}, + {file = "psutil-5.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75291912b945a7351d45df682f9644540d564d62115d4a20d45fa17dc2d48f8"}, + {file = "psutil-5.9.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4018d5f9b6651f9896c7a7c2c9f4652e4eea53f10751c4e7d08a9093ab587ec"}, + {file = "psutil-5.9.2-cp36-cp36m-win32.whl", hash = "sha256:f40ba362fefc11d6bea4403f070078d60053ed422255bd838cd86a40674364c9"}, + {file = "psutil-5.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9770c1d25aee91417eba7869139d629d6328a9422ce1cdd112bd56377ca98444"}, + {file = "psutil-5.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:42638876b7f5ef43cef8dcf640d3401b27a51ee3fa137cb2aa2e72e188414c32"}, + {file = "psutil-5.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91aa0dac0c64688667b4285fa29354acfb3e834e1fd98b535b9986c883c2ce1d"}, + {file = "psutil-5.9.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fb54941aac044a61db9d8eb56fc5bee207db3bc58645d657249030e15ba3727"}, + {file = "psutil-5.9.2-cp37-cp37m-win32.whl", hash = "sha256:7cbb795dcd8ed8fd238bc9e9f64ab188f3f4096d2e811b5a82da53d164b84c3f"}, + {file = "psutil-5.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:5d39e3a2d5c40efa977c9a8dd4f679763c43c6c255b1340a56489955dbca767c"}, + {file = "psutil-5.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fd331866628d18223a4265371fd255774affd86244fc307ef66eaf00de0633d5"}, + {file = "psutil-5.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b315febaebae813326296872fdb4be92ad3ce10d1d742a6b0c49fb619481ed0b"}, + {file = "psutil-5.9.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7929a516125f62399d6e8e026129c8835f6c5a3aab88c3fff1a05ee8feb840d"}, + {file = "psutil-5.9.2-cp38-cp38-win32.whl", hash = "sha256:561dec454853846d1dd0247b44c2e66a0a0c490f937086930ec4b8f83bf44f06"}, + {file = "psutil-5.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:67b33f27fc0427483b61563a16c90d9f3b547eeb7af0ef1b9fe024cdc9b3a6ea"}, + {file = "psutil-5.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b3591616fa07b15050b2f87e1cdefd06a554382e72866fcc0ab2be9d116486c8"}, + {file = "psutil-5.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14b29f581b5edab1f133563272a6011925401804d52d603c5c606936b49c8b97"}, + {file = "psutil-5.9.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4642fd93785a29353d6917a23e2ac6177308ef5e8be5cc17008d885cb9f70f12"}, + {file = "psutil-5.9.2-cp39-cp39-win32.whl", hash = "sha256:ed29ea0b9a372c5188cdb2ad39f937900a10fb5478dc077283bf86eeac678ef1"}, + {file = "psutil-5.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:68b35cbff92d1f7103d8f1db77c977e72f49fcefae3d3d2b91c76b0e7aef48b8"}, + {file = "psutil-5.9.2.tar.gz", hash = "sha256:feb861a10b6c3bb00701063b37e4afc754f8217f0f09c42280586bd6ac712b5c"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +PyYAML = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +setuptools = [ + {file = "setuptools-65.4.1-py3-none-any.whl", hash = "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012"}, + {file = "setuptools-65.4.1.tar.gz", hash = "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +torch = [ + {file = "torch-1.12.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:9c038662db894a23e49e385df13d47b2a777ffd56d9bcd5b832593fab0a7e286"}, + {file = "torch-1.12.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4e1b9c14cf13fd2ab8d769529050629a0e68a6fc5cb8e84b4a3cc1dd8c4fe541"}, + {file = "torch-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:e9c8f4a311ac29fc7e8e955cfb7733deb5dbe1bdaabf5d4af2765695824b7e0d"}, + {file = "torch-1.12.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:976c3f997cea38ee91a0dd3c3a42322785414748d1761ef926b789dfa97c6134"}, + {file = "torch-1.12.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:68104e4715a55c4bb29a85c6a8d57d820e0757da363be1ba680fa8cc5be17b52"}, + {file = "torch-1.12.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:743784ccea0dc8f2a3fe6a536bec8c4763bd82c1352f314937cb4008d4805de1"}, + {file = "torch-1.12.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b5dbcca369800ce99ba7ae6dee3466607a66958afca3b740690d88168752abcf"}, + {file = "torch-1.12.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f3b52a634e62821e747e872084ab32fbcb01b7fa7dbb7471b6218279f02a178a"}, + {file = "torch-1.12.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:8a34a2fbbaa07c921e1b203f59d3d6e00ed379f2b384445773bd14e328a5b6c8"}, + {file = "torch-1.12.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:42f639501928caabb9d1d55ddd17f07cd694de146686c24489ab8c615c2871f2"}, + {file = "torch-1.12.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0b44601ec56f7dd44ad8afc00846051162ef9c26a8579dda0a02194327f2d55e"}, + {file = "torch-1.12.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:cd26d8c5640c3a28c526d41ccdca14cf1cbca0d0f2e14e8263a7ac17194ab1d2"}, + {file = "torch-1.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:42e115dab26f60c29e298559dbec88444175528b729ae994ec4c65d56fe267dd"}, + {file = "torch-1.12.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:a8320ba9ad87e80ca5a6a016e46ada4d1ba0c54626e135d99b2129a4541c509d"}, + {file = "torch-1.12.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:03e31c37711db2cd201e02de5826de875529e45a55631d317aadce2f1ed45aa8"}, + {file = "torch-1.12.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:9b356aea223772cd754edb4d9ecf2a025909b8615a7668ac7d5130f86e7ec421"}, + {file = "torch-1.12.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:6cf6f54b43c0c30335428195589bd00e764a6d27f3b9ba637aaa8c11aaf93073"}, + {file = "torch-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:f00c721f489089dc6364a01fd84906348fe02243d0af737f944fddb36003400d"}, + {file = "torch-1.12.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:bfec2843daa654f04fda23ba823af03e7b6f7650a873cdb726752d0e3718dada"}, + {file = "torch-1.12.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:69fe2cae7c39ccadd65a123793d30e0db881f1c1927945519c5c17323131437e"}, +] +typer = [ + {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, + {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, +] +typing-extensions = [ + {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, + {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, +] diff --git a/server/pyproject.toml b/server/pyproject.toml new file mode 100644 index 00000000..9d14ce6c --- /dev/null +++ b/server/pyproject.toml @@ -0,0 +1,21 @@ +[tool.poetry] +name = "bloom-inference" +version = "0.1.0" +description = "BLOOM Inference Python gRPC Server" +authors = ["Olivier Dehaene "] + +[tool.poetry.dependencies] +python = "^3.9" +protobuf = "^4.21.7" +grpcio = "^1.49.1" +torch = "^1.12.1" +typer = "^0.6.1" +grpcio-reflection = "^1.49.1" +accelerate = "^0.12.0" + +[tool.poetry.group.dev.dependencies] +grpcio-tools = "^1.49.1" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" From 1d986983d54c011cd2dd716e21219dd8cc2f45eb Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Sat, 8 Oct 2022 12:34:25 +0200 Subject: [PATCH 002/793] fix: cleanup --- server/.DS_Store | Bin 6148 -> 0 bytes server/bloom_inference/.DS_Store | Bin 6148 -> 0 bytes .../__pycache__/__init__.cpython-39.pyc | Bin 158 -> 0 bytes .../__pycache__/cache.cpython-39.pyc | Bin 2296 -> 0 bytes .../__pycache__/model.cpython-39.pyc | Bin 9759 -> 0 bytes .../__pycache__/server.cpython-39.pyc | Bin 3683 -> 0 bytes .../__pycache__/shard_model.cpython-39.pyc | Bin 3058 -> 0 bytes .../__pycache__/utils.cpython-39.pyc | Bin 3184 -> 0 bytes server/bloom_inference/pb/.gitignore | 2 + server/bloom_inference/pb/__init__.py | 0 server/bloom_inference/pb/__init__.py-e | 0 .../pb/__pycache__/__init__.cpython-39.pyc | Bin 161 -> 0 bytes .../__pycache__/generate_pb2.cpython-39.pyc | Bin 2182 -> 0 bytes .../generate_pb2_grpc.cpython-39.pyc | Bin 4355 -> 0 bytes server/bloom_inference/pb/generate_pb2.py | 43 ----- server/bloom_inference/pb/generate_pb2.py-e | 43 ----- .../bloom_inference/pb/generate_pb2_grpc.py | 169 ------------------ .../bloom_inference/pb/generate_pb2_grpc.py-e | 169 ------------------ 18 files changed, 2 insertions(+), 424 deletions(-) delete mode 100644 server/.DS_Store delete mode 100644 server/bloom_inference/.DS_Store delete mode 100644 server/bloom_inference/__pycache__/__init__.cpython-39.pyc delete mode 100644 server/bloom_inference/__pycache__/cache.cpython-39.pyc delete mode 100644 server/bloom_inference/__pycache__/model.cpython-39.pyc delete mode 100644 server/bloom_inference/__pycache__/server.cpython-39.pyc delete mode 100644 server/bloom_inference/__pycache__/shard_model.cpython-39.pyc delete mode 100644 server/bloom_inference/__pycache__/utils.cpython-39.pyc create mode 100644 server/bloom_inference/pb/.gitignore delete mode 100644 server/bloom_inference/pb/__init__.py delete mode 100644 server/bloom_inference/pb/__init__.py-e delete mode 100644 server/bloom_inference/pb/__pycache__/__init__.cpython-39.pyc delete mode 100644 server/bloom_inference/pb/__pycache__/generate_pb2.cpython-39.pyc delete mode 100644 server/bloom_inference/pb/__pycache__/generate_pb2_grpc.cpython-39.pyc delete mode 100644 server/bloom_inference/pb/generate_pb2.py delete mode 100644 server/bloom_inference/pb/generate_pb2.py-e delete mode 100644 server/bloom_inference/pb/generate_pb2_grpc.py delete mode 100644 server/bloom_inference/pb/generate_pb2_grpc.py-e diff --git a/server/.DS_Store b/server/.DS_Store deleted file mode 100644 index 09937716d14643b9416026cef19e7ff913d0ca36..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOG-mQ5UkcL0xrzb zHN&gL>ofpc9v-iO1%Nr-5$_(R=Fi0Qmp=ZhdVmDUXH#D_~ z==O2mi1Z@T!Hu%EFfv6xlba0Z`{^=VuNNz%&5~Av_cFTv+DB3WDnJFO02QDD(@`Lg z@@_euN8*!E0V*&N1?>A!;KrJ4LH~3h_y_g`kg4sXJl0o!i5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HserR!OQL%o0 zPKtg~PJVtagw)N-hwZU{2>--fIdSHOgv5K}BuzhX^q$}I8|U}Fopw4c0@rtIJJ}yTA%CK>dciZY z1Ej~m1QE0#3Eo*jCoEwUWv}oie&SCWNn;WufqUi`%}JPqPHq&@q?NQL?W9eKWIqrQ zh~^~`%?oxxlNj_+M4(3(Ea?ckM+U9G0F8_W)J7sTX;!3K4?OEX%QIucpQa|S%CxZl zkGVE*C_a>>RH>1Cy8q39+1^N=nCI0?S?-RjN=s#Xd#0LB^YU<4OVW1FCd+Ivns^qeo=~T@~*le_gU3W57+i!vT zHxM~{x)O+~pc{uF+E3x88!y#cru9k%Q?$`hKtasO)(V%NX&dYQnIkzp&CzY5LD94;1&(dOU@$%Bx zHO|wrtW1j2ra4y)#Kja^Pa9BhY%!a4)f1Rcn9bk`tf5y-_@Z&?CrufM;1zZA<|iRc zZ3J3Nw80jMSae|eTB0k~;NBKJ=#@vd;c|7WWA2Sf4n4zRTUq4tD$a7xi zhV%E)^LeLt9SSsgZ5l9FWg94IW~$8GDL9T7%D8=Zfi!j_G&p%p&tWlLu|=OU^$A$4 zH?5}EOkqarwz=IR>N7BW1hjA?s@;IMpwl0N;T|%k+#$aS0%Fdn+BnCd1MLVF;xY>m z1Nz`_L3TdiU#=A2&5Ng{!GjuuA~d9Db1&dIc!4c(&kM|f^8%Z{#{TvOS>Y;B17x@3 zwSka@G!jjVkmEY!+VKBnzDe{Itc1HkLcI%_s)y>Wz63C&1z@`dbT8b)Tc#KB>h1fQ zUBUKR6xMSKsIIR6Ql9DuF>%s1S1>y@=6r{a2C;e@tm++P?;`sY*$2q53iY1B=1`v_ z`vMupP>=9UQg5dBDOleF(pZs@hTewP^}8NSE?Nd%^&#ABd->ItT451wU{3M#>()n& zjoZZb-ACl+J29N@yA7B;zH6A7ywEF4#;r)iT&exiO&f02n{69s*iBE!V!8p;^BC=| F{|781%S8YH diff --git a/server/bloom_inference/__pycache__/model.cpython-39.pyc b/server/bloom_inference/__pycache__/model.cpython-39.pyc deleted file mode 100644 index f0aeff1a277f9b6e70276691afd54342324fa30c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9759 zcmbtaO^h7Jb?*P^>G|2&+1Wpml2)>nU!}Xhui;eiQru{3u%w85=F5-^=9YknC?`Xby>ijf1 zM%VOBop@%)>e{}|X{+OObAGPt`ffMx=ewToG2iYKx<$X(E%~Kx*)KEC>CAO2eudMy z&V0A(S2^u=7P>XR#_4=#vAg6ib(j5Re)l>n-Bo|Jd&WQ0J?o$Cp7YP?TBMC16WX4R zl=*#46h!f^CW^bpuI@k1oRTPmQ)bQ+C^080c(3dl{uW@pA`BPt-kO% zMy+ks^(-&KBy4oTIL3?hPMB=ht;~9}6(^Z}bvWomnfvx2Y4v+yC$lcM8cE&EO5YtO z{j2@!QLnWV$*e}NZ}mmgdA%<$g~K@Py!lq<5%*HR*KBQ}=DA)!X!Tl2a4TwUZ70;2 zcPVUaN8jru@($jLTTw5PVG;#{&6ny%R*1JlDS|Fln-#lZ(%25-VYAt~js8`xB>lml z)!VuxTS+8aVOH6QZYSKOOWXZ^j9#6i=2~F~T?~@G#ODHolC8~Q5{Ya+j*>t`&2ZSk zGr2Q}8YGd~%fZV<-0>JB(tKTLzVR+*AhLvhL5u9jT-A^^kanV6=^f{bnqqc{m#BF^%y-e9PbrLfb%WE{LM^dLHj2-6ufl#5r|Xnm(Pr%g^fV?D8uW49fqIi@btywLAiTq=iB9(V;|+|!A3 zV<|OKTbRNU_C0f4Oi|k?pXP3w3c+>t7+|mTgE;JBQ((cCyW#Di7u`~liy@ojYTfY5N)|y9_2Ryav)mvQ0xMN6 zLk6GkkH;4mUw9Xr zCw`&d5ie|Z`u(n=&$oKbNJhOz^a8SPMpCh{iVoPkQP&4|vQks_yMbDSgH7si{0fL> zSvvnrPcPzU>4xs8-(&=LOEaUH`2e-^Wbr^u`fzd)ce;g;lkgyRb)jE3Nu0YQUbTD^&Mn^cIgEBTVNZQIZAA z_^Zo9j%2zu7k@ta5MgoI}6Jgq$HEDC-3aCbl&y8N{o%|X8)MJrvYY3Y(E?$PQv z>@%;2gEzExP?F~0NJ*Id`}&Pvqc`PmX`{l8+bQj)l{5$Q;gT6)6H?mvT*tPRM0WDL z;n&`{ytcEj*6Sy0&Hk__)>^$a-u`uy^>0=V6IctvSV1?8uV-_E5JvlYbSJnOc3{98 zRN28r+KrS%OgCxuRJ&Fl>7*KV_4hPbKTOvZO_I)ql`=a~4;s^0KcCr3Umnw6`3nTR zZV%+EZG7<`xV4?YqY3V>05n^_(CIhAPW6kos9=0Z1ihHcLZUgYQy_*NCv)U^9EH=|Y@@63DA3qT89?;vr_95(SOR z#}LFMd+zw~^tNx&a`5v9iz4<5oK$Ng(}T?2jzR%++x3?SY((OukRYRgO2vL_Ct)-{TTrMvr>Gyh$TGY~6?68jW}%v+@Z-!^?}Lr{QsxFh4-hQ~GA|%U zW!Rx~F$iu9!_FkfZ1s|CE(pRN_J2rwJPra$Kv@z-$xOH;@^wl%R~a!f`eSqlm_u$E zFyMAy$P#^0BjOTaQ1=q0ibQC~_yz9nCBjBRUM2Q)uXBn33)ev_G$wH987`)#s+Vbk zAQw+J9+jUm^r~m*T(W^eM=sJm7jeggh)(f}$U7pB2$)@%4|KxdcP&3B2w&&2>g({z zV)XC(%G-lHGH>dpWKMsWz%Qxm3f9dkK1!3^E>YuEkVFHZ(*WYNwh3pzKm*1Gz!dFQ z8GzI+CADkB_`^dk+}t?oG-rgvKn}_Q`t2W3EG8YR={qYYJG;(>HtMc?m+E_u$d8El zM1D+!CyJX4>GiZJpxnR59g~YmQ{WnscqfEygLx5m`~#2!VMiD(1ZxNrg0(~;vVq}r z#!R-)j%bvRfK=e1!mbfQ5iZ=tyzo%IC@jdS2xsxlx8Ht?yl@8YypT_z=tFV8kLF3- zV`C54IMK%F!XAVln7tyu0WQ#6Vzup5C(r91rX_Ha9DpZEj@>;CkPj2Z<=}m_y>_8p zM42L4tGd0BS&g9x;kyimnbnD!$wS>QPJE{zZi%Q~;(a0o`jQn}aS+}NTb*#T6RFKW z+ajA|MKypACc{<_+kibF0@0ZjLAS~0C~5PUedypgQV|Au1$?ZX5AcvQ4Kwq`s{?zT z!xzCnAu}$6XwY9}Xs3!%gLc{}vq0Amifug9+5R<1h6fj=Y07yC(_V%gXc6o1S0PdO zuQWO4#?^G`%BmN=ypB(^T(d84Aw+^z{sG>PeQ>P*a)2WLf;%R4p+QY6 z6>5CBQ&Js2s10_UP18xgid-ZTvWYu>33~)HukUG}J_WBG+pwv%w2v{vztTU^N5+o; z@+|yz5kc!!Uex0A=YdV=_hZ~}G0ZLPXF8sTF@eOw*LG^O9XxZmUEKLydz6EBpCbr; z?Qa2~-S_QLe&me`?2Z)DQtFKhv>GR{k|Fu7F)E{XMdX$V%EEC$dRLeKjGiLml9tAE z0(`YnM{}t&t|apSadW#2d~Xa=b6iPXa!@|j$5mkq=dPA40O)D$8epU=a?3Poq<0N+ zQrNAr@J*RhHd5@~P)+ys_!U4-Y{e_sir7a*a$ex37%g^f09XG8V>QVwNfA>5$c4l4 zk>1o1`El;LsUeE1h?VT>$SER6Lr#fvOyZ8_nFF^8WfiB&oVgj!Lhrf5Hn@IOtEv$> z_Glh7H>!fVqXq6I%B|8YFH(=-4lR-EA#zQ~2yTMnt?cPrhRD%qhy%B06*Zhm=hI4R zr*J=Z9k?w{TBFg|D;tcA85LfwBcc^wZw-PN(KkfuCXtj=;V$jZxB{{Tv0rmX*ff7) z+`x93E_7^+@1THuhsadz^7i#EbP)V&^t*#sS7Aux@1e}jGiv1@1|B)EH3M)DKat<0 zDhc2{1fmAelC&|8^piFPY3l{t@fryCa|H_)n<3Rd)-bIG(WY^t-zN7vvw-*!5K;&X znz_o`Jz$^)@Q6TLtcFD+k`m+@3?*izVmSLs-H??$xX6rAF87I$DfbD^0wxIV!m2jY z6Ifg-@o*x;wMNKD8wbds*Us0 zHURAzf8`LN-jbnURmqc`nLDikizRb9{jF9KPaO&sAtFUCp9g{63KKRq8G@n>1g*2& zBp|1p2r}>TGL^Jt*xQQO;AB?6he>^d^2jhLLq-~!orLml zFa88XL*(&1+z4_daGM17C%^x;=A18peGh)-IkRlKu#(=sV^#6pvoQ8WxC^dX)g1sf zW8XF5>^$-;I6ZiCPzF4>F+~;AG%b7$2WH=RWH`oudBD>Y@tpj=u-u*6Y1*fbU}Wi3 zY}vs8B!LIDKG_o>g`<24US7O#J|x~$`*DiIA~!+)5qC`IVy<2;R?w&FBjz{2I&zWP zpeud_WC}d#+r zz?!?$AjNrb(Etdm5ic%71teMmT)B%t=N?c)V(x11o4||~ZkzBWEHHvu*9mLp5}db; zJ>b(nNL(NSatrKVXupK5LN0=t?c&JnRXIRj?0khIs0o7B%cRBe0Xn;_e zhLZkrTb+{sgV}uU>l571@;wT`y5qIf21dfU%J|7WbtVI6VdN!G;mibx2rYbq z(AP~-0R~$_U>N+gKz#X`D2zPheU^gCn-mB`2<>Y)an$#W@z?i&l+rv0&Tgi;@i!Er zLh5m}kFfCgX_U}{-%6eKGt{GTeNQ_>oM(^slD&kgLJvDOxeDKr5tfN;64?U5*-I%I z1h@LKg8&GfPf={=gax5J!(f7ZU^a*q6+I&?yE_Xiuw==r%iM4??ssr%2nGR+3;~&B zduEUE_9A;MrdDOAKD+u4-GbB51wvwJh5clx&ioKyzU?XZ#f1q_VN zgF2n4^EvGf*yf*8R~&VmTDNw-K|XD87)e}5luX{41`*d)!JV_miU|ZAqJzvDzXjV+ z=fNx6$GERB`+&=ksQ%jsmkn`P#!5u8iXzWED-C2Ha3zL8#gH-!ChvOYO{XSv5a@*2 znR;|`8{erwoV(fV^uy%EZ^Ff)#LF+KkVlJ3us=ucS5}xk&RyJ;{ZKS0Zra~$L$w41 z-Xy6y_9B&m!a*n60~M%Y1WJK1Ii|aDL*1YE7@OCs&iTimPE20`-`XYGY8G%>|HsF-t?=? z1kOYppdTD|6+yv1N1%bh)fqc*ZaxS&{)BMV)QHd&|BzNW?F2g8Vn z8e(%WnjBekts;z|OU4l~Jcqbe5oyE#jd$QTd-%q}_m<&0MerQ`9P+C8?JSqL3sCdm704YF?mYtzr!8`6ZqFoTnpy!4n8_4wT$Md^ zjlG^z719ZWPOkOGIMtg)=!U|n9!IaSzONrBB>6*BPT>*-QUhKOGm}QD>8~(pvtG{? zWtP_`ZvSa|N#3JAv+ENh1zbXL2L(R}Aj&@{@&*wSy!;~~uM_!G5Z+e77FH;>`F-M! zh)^J_o>%Le94hvt)Jefl>2(9duiV5L!=@SstMbg7aA=Eg+q@+In!aG|J3#8sXkC(i z&?w5QeqTB{T2mgNSWRES3Y);LW5Xxdl|L1B(YjKK?GSrqrOD1{L_ofi-NF6&ODfF$ z`D3Ksq5cq%Ld5$GNu2sqY6|E19AbS|ffWw_h69IyoFDq*K(gIwZA$7w=HM?X^e+Oc zPm&E&Sde4LMc&AmNt}bkN>|dhP}=Y8Z00%#zEDpvOj`Jp0I9fakNYEM!ye-1IJ-dDj^QnXT7s=C zycvVb!X`Oq?gg}nxcjS+wH7}O!bb*BDl7OuK@ke3h{K1KoCmaD`HA)ds9XMtrvEQ6 C84{xa diff --git a/server/bloom_inference/__pycache__/server.cpython-39.pyc b/server/bloom_inference/__pycache__/server.cpython-39.pyc deleted file mode 100644 index e3ffb8e5e5b523a5b6787ee24c398b63cccea77e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3683 zcmb_f%WoUU8K2n~mrsc@^>EN8OprDqQcF&n<)m6vuj~M}}jlu@Rb) z?5OAz)w>mzqOwy~<3d=8s!lbkIkjlbSyOX%xE|G=dbHteQ1S_3MOHc`ti*}@PjWF(D-bzK7!NbkM-{Si%EePI+<^#FAKA|@ z9BXjz!<}vc8EGTpk`V<;M>hx~^B(q(Z4y5ZM&(s1!s0F{__Cn@~d!%S(LWi%dh`ze_k_z~pQURHR z_+(0Fgc7Mu^%;0Z)M2`1DBpm$L+%NsOlBTLeR(3#r>g>%_Hmm6H!i0N5VS;xS675v z?7JWqdIFPx66+BRBnX9&>1jQH3{YdHkJV^uj`SJ*oi;YcCahS~!pNLa5Z~?<8D-ij z85wuTgpN$n9+}h9jKlyqrhlt}hm5vQ;q>{1mOxPt;ZD)gM_`ZNSH{44N@nCQ+7kop zfyO?A*p&%{g7G0+m&7fd}{~AW^`%6(b|K<`l&~yqLZ5S-DG1C+#^r>+H6B=M7$Ar}@ ztYG3UV8Z@sFd=>p!id+9TvT4X4s+MwUby`I|Kakl0(q>$=4HMYZ@?L?V(#b0lAjd} z7`l6rK!1sHyoCfak9Zpi`ckxzTof>?&n^9eKYn@crrN5Tn*UGucB_J^HTELzx>?C} zp;Qb)99LZT$-oQq6=9<&1tb{onVw1^Hu3c;lAj@Y2g$FHEJRi#M3txl!8^r%!lqlM zt=ooORr41>qhbyA?BnKLAk36RQmIKD%Ap43P-n)eIo73t31x)!Nt-}o!5#q;h1I?b zC9kAPp3*BTqmC4kY{om%8sV&tvrNs6%p=$(3kc`2C5zLN zERS?qnLq-Z=p$6K9-Umsc1Mjk3K+MJPlp#7&< z3$|DRxC5UFSOnI#7i-Esc6<2hAP&C%Xm4*%M*Y=FeC?a@@cN3Kn$aMY%?@uu(rG?B z=5h1fqRk*}J`@9fDK%q7K;Nr2LZ#-~n1TVopfkFZoT0I@x`~K9+0sN4KFAE=#gEUZ z(?A*BH1QvE=}I2;LujfNSxN=!E#!ce0NQ|EsLW57^00b^mI1o4mj);n69v9u1IsQK z%~X0qa@O=@^P6#NCo4Vj!T~=JA`t>Xo0%Zm5zk7`5)m>t#U%Ycp7H?_6(A@}W+~P} zl;OpB(__rN7=*|U^15)LDxGh|z{sOZ^&X`=2Z#F)?tSs_{sU(nx4`z)?VKn$JH_rG z4AoHR$f9z1@ZizC{R6js`||^5hsRzA0ysa}U39lj9YqPCgYgFf+S5eH%v3sOW?uxc zbQ;e*Xt2N?U~drxv8S41TfB?Xy@%x2NU(iUO^{PmMGqTE@excq)n!Qzb7y^syH#Du z`_9}YIGbJvLQI1i)P`RPe)e-quW2~8fEu*++%PKCfRa;zxf=Y2uPDV9E>*Ajp)Kuh zyJZPHHZwX&5(FU0zc% zyrgwx0q5)WypFD3F8S!i1zL(B)r`WOhHJEPg-Z?*$_f#P{xpL^IDGJ-1-I z2Scm?DUd`3KE`#h;&XBX=0nhMfwfR8W@(owTe3r`95L$mqr(ftYq0I>_cuQljEl`I zrklHQmL(-!kJ3KlEDhNve161ux$G}24+2j8M1$POk5nTGWH|6k;jTty;9_2fdDm8+ z;q=oWVam?ITGf=f|7pf!>WgU1RE5f;oP{>9&z~ij=wN9>#R2O)3yFasm#!-0u{X zv&uP|RHai?&#LEIrdzmo(J9{f$CpHsDKWCDQ*iC1Cf&!Rk7*4wy3{K*o?^aM)F375 zz~L3%)LKEbJ|yC=AE4i3{eBZo>eAb<7qy~J>$Cw#8nRJ9qD(E|`={oUP5pis?Y#Xv zGHFg)MXQupfTU9_93!iB);dRL?Q>K(Me_!lk}Y)b)hSMwX1JOLSL+hj-wduc;A$7` zIWCa)&`5)o?;vYGj|@0J`wq6BFJJ?yGi-nPZ`e-p!58VH8McnWc5Q}@@K#Z~vPSz( zTO$z#RL2DTusUf<@2peQ&heGCAo1VnkLU5*xjAVrN}2oQlpL%S5V3dAXSD+^oAO#= z7tS$666D^|Ho)jit`(jArTy!W8xCE1LNEe-%LS~S!@RK1UV$0p5lH($=Ti6j`1Rl1 z`|!j2YXZ_W>PO+4(4`Qidus^`2SFN%B*gO^hJRfMy4J445I+U72i3ZBdxEEUivEZw z)GAOsB|yoaom=P+pVz@>tlO8>jT}$<&h~|sAsbAwyHuAX@;)F8z7lC zHG!sARyWs;PzFJI+pP zf(v|x&e10bqXX!h4-l`wcj!3|I|QYoOLzwsR80hrn6K;Jpf8C*SKHcSDLc&hwl4So zz(C{MF9$Yf9KU)m3yi~qn6rQ$t%X52V07(K5RaI62)?Pyj6*vzJwZS^zX^kD&%7XBd;AwL#3~SoeP@Cpd3C=g z&w1{mY?v2ww}nAUBN|9{ft4pDOiT;C@*jT%M6PMsT8Rx|?K9}|LFg)?M8tZuO@N6L0Kx8Kk#Md-x$Mv8vM z6UH~7!&9FnyNv39iP_$-TXEtKAat{LuIojgR5W_AcD}#&xCy~6ki}osL|0uIAMGU* zflDu0<6F;}Z;YH9w_x@iWB1%_{u+pSs`37SF#m6$=1{)`O2#){3e@;|AqI1pdL1gD zng?)xOQBp-iGDWhuBZw`W{)W=h+|dBd6Y`7^{95AXNjM42BC{mM(4Rkqfjbaw-04Q z`Uu#W9;phHXAUQ}sWEuV<4AISP^%hCM-X*LQ{_-5p(<4+jI)#}Z_Icm{5Tqd`f-#i zr^Kf!$&f}|*}6d}C&+V_((cmp-K-7Ld){Re5wK0)JbFu}QwYB-4GmZDH-VSGrQuY} zvZ`kznU7?7&+y=u6AsP1*EYKu^~-_~{93t7_jBb%>3kDk*6KTDSGrPV@~dkvV7;wX zoAzsZTfN+2pz8K4VjzgLi1UqOFJTwk#Kz9Ea^Aomthl&A$Twcy!yTA+%x@XrELU9* z>s=k79elZ3UB)fEYBlgGaj>^|#Q9o)bz!nZ-6a(zF6u=@VEvm5P+Ff#W#bK4s? zHle!&o246{>WdpRIb(r#dAFnUNDks?m+NDUuWO=@0o7Qja$&N_T=kSCBv+QsEG^&? zfWG+oe(5UhO`}7(yZtx|G~)6tMxjWmx9 z$(3Vabg6tQ>U)pU(f<;!Jrut6Toh=3A4hAg<>XQ>$mRFVNAi3A$aH0;$Iw2$vX%X& z&)7eRSZ+RuEp+`?41x(>uu1#RC*0BA#B=|?`}c~V3@2e3O`@_h>6Gy#=Im1@d=Wfl zA~^9TT~MKjKt-T>pgJN36@%)7>WUty9;kuni@{SiS&@8yC|1PqDLdhlD}p~@qbo0< z4;zmHd;NYr&5iy-)tS_~R`+vl?5%BC&ZJ7sT*>zP!&HG`SGVig7q4+XU*=@A^K$b* zY@zEL7$hURVJjCNbn}G|Z2}uVNXuD~SJRB77cIbq`dxJW6AXs!^L=mMZ#b=g<{f+B z1dT`jdFGvl!aw83enWeP;}`!$d(N#d^6~}8>Jhj7YF;Lq&XRf*+0fJ~+p}I(+1~EF z%1mBYX<<9M-Zf?Vwe6Sl!sJz5=8pT-T*@N}bptJnT^kk7bvB%-dPh(CmHgTyraq9B ze$Lc)aN@tWHvXuk(i?RlHg<}-F5B_lyxNsYR+-$u=R>Jlc5=aTAW6QOr$zfk-NGJDy`|wM z@o4uIbWK5sxugHDa7LdhDaDglf=PQ)Ik;dBFlqQc3IbuIg!IR%1D=X8AWl)LrU<~u zQ`d-lod$AfJ2+*@3Sv2B89%zxo_XO7^~TEumzKVT50s1xJBfbHPD;Ej5_%iM%e0L} zAi}2{i5rOy5;zbZwC~#0F@nEMvA?@l*9huAu?p>9AC2#2VLNvtSxK~W!{0G6?1UYM zO>i6`tJ$2ZJCxlAzPe+0!%z4zZ^Bd0Am>l`8RtxRkOXEx@)2b|Bp*2ms)OL!g^cfH zn#RX&Of^m>1QjmHgbhogCQ^ShofB3CCO%={bOnP^(O7*C`faQ8+Cj0*k^= zqLA8fx2RKth|OlQ5{g2s$UsF7w4KKeIJCiO%52L_^YdpU9JXE)_Op!y>cWL-B4c}V! zl;E^Llfg0XU01}(&|hifo-OoO3!)>CgRUT`FsL9RI3u>J50EZ1#OtogjZ}I1f-3Gc zM4%LMK3>HH{$&X;0V7JG^{Gof&klhGNhKej5resCI|G497SVkca9S1b_fGp~@Mgnz>GJN4ZSdvCNF1A$VSy;_V0*3T-jfbnnhmVN8D?F|A{*49^^8rxkaNQ=%Q2ZXi~%g{La`h$3K9O z!h3W>us<=*9@00jaW_ykxR`I^#fzZwF|g6O@#?aX!aSKtd$sjjYqw*KJR7>nq_5LM zDH2CqGFtr(8a*P_sOp#p^nxw1=lG*l^-C~qFRLqq>$%8x&d+!5SzmI`gyl@+~y@fu9pH(qicICtf>cO2DEd(lx9xvS0LrmHmPM}Sm&aG6%=R4S*`k^wKv h)yBVSpK8A*&ufZijN6NTZ^XmSFdV))9K^Q+{y$$P-bVlc diff --git a/server/bloom_inference/pb/.gitignore b/server/bloom_inference/pb/.gitignore new file mode 100644 index 00000000..a9feac81 --- /dev/null +++ b/server/bloom_inference/pb/.gitignore @@ -0,0 +1,2 @@ +*.py +*.py-e diff --git a/server/bloom_inference/pb/__init__.py b/server/bloom_inference/pb/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/server/bloom_inference/pb/__init__.py-e b/server/bloom_inference/pb/__init__.py-e deleted file mode 100644 index e69de29b..00000000 diff --git a/server/bloom_inference/pb/__pycache__/__init__.cpython-39.pyc b/server/bloom_inference/pb/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index a1d54d8ecf2bddf9eba6c7de3fe9a4ae98c02952..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 161 zcmYe~<>g`kg4Mzf$sqbMh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vFKeRZts8~Ng zCq+LgCqF+ILh5GbrKJ|7<|U`<1EtGSiy*@B2;qVx{dgcZvm`!Vub}c4hfQvNN@-52 L9mxF8K+FIDsFx>q diff --git a/server/bloom_inference/pb/__pycache__/generate_pb2.cpython-39.pyc b/server/bloom_inference/pb/__pycache__/generate_pb2.cpython-39.pyc deleted file mode 100644 index 936aca2917ee5f063bbf8244d43713e2612dce31..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2182 zcma)7&2HO95GM7n)t^Ntvf?DRtD=Q-sNJ|NdT3BYmS{&sV!M}W*q-;xu15|`Q&d<)w$KmeKDwT+Wzu)KY z4*n=A$}6c%|I(=3#lLuiz!XzqDoZ%(C}Ad5DJPxOC~amUIfZi8%tdk<<-AE$1;|O^ zg<=*VC+{aEmU*F?Wk{Y>pbT?AD=!kbCV5hYGRsCf=7r368p}OR`2Vszl<7PpaD`rF z1;iRF!UD=A#2PC@9pwsQjm^O|l&gp}rokf0^N2Ne1+Jrf6|u%@a0BH9#2Tx^O_Z-8 z*4QE}p?n>&#%{ne$~O^fEZJ|EhF*tdKS{5{LJMi786 zw0TkpV1za+82fdDAkC3(iFutuH(^%RYot7~o>3QmpaFjhu88WE z8efp~rWFhhXR3Zka(+xk)=jP!Hm=Onm@sYpND5JK%NiU4n@NFPnWUgLi?gJ(I+^Rs zVssD%M4lw(k30bnEQiK^VLy)>Q_`-Jc}ZMIKJEah1MV<5tfysP8y^#G%XV#n)h8Ru z=I#s+Y#<*8*ul7gjV(eVwz?bm$1^K0WxPNt7c0wGSZe&FlHA!=xlM8h6B^NWDQqmy z^y?Ciw@D#t0tHDVB3Ia0n29>)c~AL8RX59|7&k7CX=F)SA9=y?#;*xd?!mKQss+1b z?&8MQw3+Nt=;8-6wu>RB57{zI#WOekMBbpM=HXWCBpHzp=IRdKpK{>um`g3BO5riC{0Q2=7Vw zR>Bhre@XaTLM4Gvm9Q_tln_ezQ^G3=|44Y7#L2Z1W+`QSKxtSXaz1n*9zQta)@&E& zhHE)Ku2#Yvos77!a$%v5iIlKF&nKOa15F9%rYzCy^W_%>%@^ihm~F#>HFg5zf-cJX zVYSH^zGT~S@PM%`+kud@^=`}P?DckyaA{Lc+Xo%TaK74 z>~U|(i|Z$mBGl-^u1;I@Ja-o#lzRalPhno8UEO%pY3c1wx3&97Hy#^$cW-y6tA|xQ z=%@6k2rNGcE7K-$nYlpk?{0T`-TkJqryF}sqxnGZ=||w%&I*AI12Q?c$eAj;nQa<5wj%({r=eU!UHUYu<5~yEEc! e?7)}uC5t-F(d)M_W+x9H8GfVd@ z#MW{lxp3eQAP&hf;ve;uQ*PXlED(Iv``)Y_jM7$@yQ+G+tGb>WR;wig3(Q(GYSrDpB zx0tzZGmF`f)L{-JE2v}l`2GA=|D-re*YfVh-jzGClq3x#%%T`qFrQ^nw+$7-9J?TN zn-D)v{koRd-;5JpkLcFnpmhOr1yd#fJ+pJtJ^6hiH9{HR2RC z#{tLF-z-Q=Z{em`u*6#{~bj%Vi$G)^V7`yIiQ3 z`&O1bV0=v8d-I^9R}vnIl`z`od*=_@H?SYyyYydneI4i9`p{;rs>2syb9N zAfHl&)pjS}QAeR+UyFO$S}x!i>g24_ceM*ku1YsW)*kFyn%$*JV~ted(=d;=l{fm- z8>*qonx$oA>gbR>k{d7VNS~)_3$Q89gP=@Kd=bEK9n&&h)BW2k9JJbfZQ4yJueX5!of0JbU&Yc_G8%f7OCh081|xBGIuY`f1~!dc$)x)M;~anSq6F;b5j0u`^Y~I*%u=D`wf2>(5QCb@KpMnnefAb z9^WH5W!?Oc`W%NYS7!7%yN^D@+h2$_m!GfAAyuJz{mG0f7oVfbC^I%|o;ZV##aRMA zfpY}NUj-Rm&`SxrW5Ol0b7;SK9gGW@l3FGY>BKA2pCI~FaUbR#9p^uSn;R{&XSA%` zros;Bwu;RB*&fin5#5=hFMwXs`^!`G3h32IIp zHem*`h|@%2cxhOAC#$Sl9u;2{fI&sBIrE-n_tmXITlH+IuPuo;p{hDFQ8NSNoS?1K zQ-HpKC0z86tzxHGByg0#5`iNG#%6el(3c5lgPg!hot-{Q23njr8Xm4rhK!Afbot;` zS#u~C>Rv=BmsaRHmY)?*sv|RJPPhVjfjUoD%y+pU�-v(YLTuz6W4fj_K%%=%6AV zu=>M24*ox=Q6J!|h(4Vh3qY=6%Fh5MYZP}_Rd?Sg80!UKi@NtF%msEt_uhmRfi3C& zn=p^raEha9aat2yv$V#|o00xyfa_?`nhc~}`W-CDbst$h^sVSxC)~Pc_3U1u=k(lO zvFG&`9#?UX)f7|<5P#Z5D*S#2o~AqIL{z{LV~5agpdEr-<4H--ZOY9z>% diff --git a/server/bloom_inference/pb/generate_pb2.py b/server/bloom_inference/pb/generate_pb2.py deleted file mode 100644 index e55a2c77..00000000 --- a/server/bloom_inference/pb/generate_pb2.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: generate.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0egenerate.proto\x12\x0bgenerate.v1\"(\n\x18ServiceDiscoveryResponse\x12\x0c\n\x04urls\x18\x01 \x03(\t\"^\n\x16LogitsWarperParameters\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_k\x18\x02 \x01(\r\x12\r\n\x05top_p\x18\x03 \x01(\x02\x12\x11\n\tdo_sample\x18\x04 \x01(\x08\"v\n\x07Request\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06inputs\x18\x02 \x01(\t\x12\x37\n\nparameters\x18\x03 \x01(\x0b\x32#.generate.v1.LogitsWarperParameters\x12\x16\n\x0emax_new_tokens\x18\x04 \x01(\r\";\n\x05\x42\x61tch\x12\n\n\x02id\x18\x01 \x01(\x04\x12&\n\x08requests\x18\x02 \x03(\x0b\x32\x14.generate.v1.Request\"\x7f\n\x0b\x42\x61tchCached\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x18\n\x10\x62\x61tch_cached_ids\x18\x03 \x03(\x04\x12\x18\n\x10total_batch_size\x18\x04 \x01(\r\x12\x1b\n\x13max_sequence_length\x18\x05 \x01(\r\"0\n\x12\x46inishedGeneration\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06output\x18\x02 \x01(\t\"F\n\nCacheEntry\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x17\n\x0fsequence_length\x18\x03 \x01(\r\"\x80\x01\n\x08Response\x12\x31\n\x08\x66inished\x18\x01 \x03(\x0b\x32\x1f.generate.v1.FinishedGeneration\x12\x31\n\x0b\x63\x61\x63he_entry\x18\x02 \x01(\x0b\x32\x17.generate.v1.CacheEntryH\x00\x88\x01\x01\x42\x0e\n\x0c_cache_entry\"\x07\n\x05\x45mpty2\x94\x02\n\x0eTextGeneration\x12O\n\x10ServiceDiscovery\x12\x12.generate.v1.Empty\x1a%.generate.v1.ServiceDiscoveryResponse\"\x00\x12\x34\n\nClearCache\x12\x12.generate.v1.Empty\x1a\x12.generate.v1.Empty\x12\x35\n\x08Generate\x12\x12.generate.v1.Batch\x1a\x15.generate.v1.Response\x12\x44\n\x11GenerateWithCache\x12\x18.generate.v1.BatchCached\x1a\x15.generate.v1.Responseb\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'generate_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - _SERVICEDISCOVERYRESPONSE._serialized_start=31 - _SERVICEDISCOVERYRESPONSE._serialized_end=71 - _LOGITSWARPERPARAMETERS._serialized_start=73 - _LOGITSWARPERPARAMETERS._serialized_end=167 - _REQUEST._serialized_start=169 - _REQUEST._serialized_end=287 - _BATCH._serialized_start=289 - _BATCH._serialized_end=348 - _BATCHCACHED._serialized_start=350 - _BATCHCACHED._serialized_end=477 - _FINISHEDGENERATION._serialized_start=479 - _FINISHEDGENERATION._serialized_end=527 - _CACHEENTRY._serialized_start=529 - _CACHEENTRY._serialized_end=599 - _RESPONSE._serialized_start=602 - _RESPONSE._serialized_end=730 - _EMPTY._serialized_start=732 - _EMPTY._serialized_end=739 - _TEXTGENERATION._serialized_start=742 - _TEXTGENERATION._serialized_end=1018 -# @@protoc_insertion_point(module_scope) diff --git a/server/bloom_inference/pb/generate_pb2.py-e b/server/bloom_inference/pb/generate_pb2.py-e deleted file mode 100644 index e55a2c77..00000000 --- a/server/bloom_inference/pb/generate_pb2.py-e +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: generate.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0egenerate.proto\x12\x0bgenerate.v1\"(\n\x18ServiceDiscoveryResponse\x12\x0c\n\x04urls\x18\x01 \x03(\t\"^\n\x16LogitsWarperParameters\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_k\x18\x02 \x01(\r\x12\r\n\x05top_p\x18\x03 \x01(\x02\x12\x11\n\tdo_sample\x18\x04 \x01(\x08\"v\n\x07Request\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06inputs\x18\x02 \x01(\t\x12\x37\n\nparameters\x18\x03 \x01(\x0b\x32#.generate.v1.LogitsWarperParameters\x12\x16\n\x0emax_new_tokens\x18\x04 \x01(\r\";\n\x05\x42\x61tch\x12\n\n\x02id\x18\x01 \x01(\x04\x12&\n\x08requests\x18\x02 \x03(\x0b\x32\x14.generate.v1.Request\"\x7f\n\x0b\x42\x61tchCached\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x18\n\x10\x62\x61tch_cached_ids\x18\x03 \x03(\x04\x12\x18\n\x10total_batch_size\x18\x04 \x01(\r\x12\x1b\n\x13max_sequence_length\x18\x05 \x01(\r\"0\n\x12\x46inishedGeneration\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0e\n\x06output\x18\x02 \x01(\t\"F\n\nCacheEntry\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0brequest_ids\x18\x02 \x03(\x04\x12\x17\n\x0fsequence_length\x18\x03 \x01(\r\"\x80\x01\n\x08Response\x12\x31\n\x08\x66inished\x18\x01 \x03(\x0b\x32\x1f.generate.v1.FinishedGeneration\x12\x31\n\x0b\x63\x61\x63he_entry\x18\x02 \x01(\x0b\x32\x17.generate.v1.CacheEntryH\x00\x88\x01\x01\x42\x0e\n\x0c_cache_entry\"\x07\n\x05\x45mpty2\x94\x02\n\x0eTextGeneration\x12O\n\x10ServiceDiscovery\x12\x12.generate.v1.Empty\x1a%.generate.v1.ServiceDiscoveryResponse\"\x00\x12\x34\n\nClearCache\x12\x12.generate.v1.Empty\x1a\x12.generate.v1.Empty\x12\x35\n\x08Generate\x12\x12.generate.v1.Batch\x1a\x15.generate.v1.Response\x12\x44\n\x11GenerateWithCache\x12\x18.generate.v1.BatchCached\x1a\x15.generate.v1.Responseb\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'generate_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - _SERVICEDISCOVERYRESPONSE._serialized_start=31 - _SERVICEDISCOVERYRESPONSE._serialized_end=71 - _LOGITSWARPERPARAMETERS._serialized_start=73 - _LOGITSWARPERPARAMETERS._serialized_end=167 - _REQUEST._serialized_start=169 - _REQUEST._serialized_end=287 - _BATCH._serialized_start=289 - _BATCH._serialized_end=348 - _BATCHCACHED._serialized_start=350 - _BATCHCACHED._serialized_end=477 - _FINISHEDGENERATION._serialized_start=479 - _FINISHEDGENERATION._serialized_end=527 - _CACHEENTRY._serialized_start=529 - _CACHEENTRY._serialized_end=599 - _RESPONSE._serialized_start=602 - _RESPONSE._serialized_end=730 - _EMPTY._serialized_start=732 - _EMPTY._serialized_end=739 - _TEXTGENERATION._serialized_start=742 - _TEXTGENERATION._serialized_end=1018 -# @@protoc_insertion_point(module_scope) diff --git a/server/bloom_inference/pb/generate_pb2_grpc.py b/server/bloom_inference/pb/generate_pb2_grpc.py deleted file mode 100644 index e8eb63f0..00000000 --- a/server/bloom_inference/pb/generate_pb2_grpc.py +++ /dev/null @@ -1,169 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -from . import generate_pb2 as generate__pb2 - - -class TextGenerationStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.ServiceDiscovery = channel.unary_unary( - '/generate.v1.TextGeneration/ServiceDiscovery', - request_serializer=generate__pb2.Empty.SerializeToString, - response_deserializer=generate__pb2.ServiceDiscoveryResponse.FromString, - ) - self.ClearCache = channel.unary_unary( - '/generate.v1.TextGeneration/ClearCache', - request_serializer=generate__pb2.Empty.SerializeToString, - response_deserializer=generate__pb2.Empty.FromString, - ) - self.Generate = channel.unary_unary( - '/generate.v1.TextGeneration/Generate', - request_serializer=generate__pb2.Batch.SerializeToString, - response_deserializer=generate__pb2.Response.FromString, - ) - self.GenerateWithCache = channel.unary_unary( - '/generate.v1.TextGeneration/GenerateWithCache', - request_serializer=generate__pb2.BatchCached.SerializeToString, - response_deserializer=generate__pb2.Response.FromString, - ) - - -class TextGenerationServicer(object): - """Missing associated documentation comment in .proto file.""" - - def ServiceDiscovery(self, request, context): - """/ Service discovery - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ClearCache(self, request, context): - """/ Empties batch cache - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Generate(self, request, context): - """/ Generate tokens for a batch without cache - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GenerateWithCache(self, request, context): - """/ Generate tokens for a batch with cache - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_TextGenerationServicer_to_server(servicer, server): - rpc_method_handlers = { - 'ServiceDiscovery': grpc.unary_unary_rpc_method_handler( - servicer.ServiceDiscovery, - request_deserializer=generate__pb2.Empty.FromString, - response_serializer=generate__pb2.ServiceDiscoveryResponse.SerializeToString, - ), - 'ClearCache': grpc.unary_unary_rpc_method_handler( - servicer.ClearCache, - request_deserializer=generate__pb2.Empty.FromString, - response_serializer=generate__pb2.Empty.SerializeToString, - ), - 'Generate': grpc.unary_unary_rpc_method_handler( - servicer.Generate, - request_deserializer=generate__pb2.Batch.FromString, - response_serializer=generate__pb2.Response.SerializeToString, - ), - 'GenerateWithCache': grpc.unary_unary_rpc_method_handler( - servicer.GenerateWithCache, - request_deserializer=generate__pb2.BatchCached.FromString, - response_serializer=generate__pb2.Response.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'generate.v1.TextGeneration', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class TextGeneration(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def ServiceDiscovery(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ServiceDiscovery', - generate__pb2.Empty.SerializeToString, - generate__pb2.ServiceDiscoveryResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def ClearCache(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ClearCache', - generate__pb2.Empty.SerializeToString, - generate__pb2.Empty.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Generate(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/Generate', - generate__pb2.Batch.SerializeToString, - generate__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def GenerateWithCache(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/GenerateWithCache', - generate__pb2.BatchCached.SerializeToString, - generate__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/server/bloom_inference/pb/generate_pb2_grpc.py-e b/server/bloom_inference/pb/generate_pb2_grpc.py-e deleted file mode 100644 index 541a6b8c..00000000 --- a/server/bloom_inference/pb/generate_pb2_grpc.py-e +++ /dev/null @@ -1,169 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -import generate_pb2 as generate__pb2 - - -class TextGenerationStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.ServiceDiscovery = channel.unary_unary( - '/generate.v1.TextGeneration/ServiceDiscovery', - request_serializer=generate__pb2.Empty.SerializeToString, - response_deserializer=generate__pb2.ServiceDiscoveryResponse.FromString, - ) - self.ClearCache = channel.unary_unary( - '/generate.v1.TextGeneration/ClearCache', - request_serializer=generate__pb2.Empty.SerializeToString, - response_deserializer=generate__pb2.Empty.FromString, - ) - self.Generate = channel.unary_unary( - '/generate.v1.TextGeneration/Generate', - request_serializer=generate__pb2.Batch.SerializeToString, - response_deserializer=generate__pb2.Response.FromString, - ) - self.GenerateWithCache = channel.unary_unary( - '/generate.v1.TextGeneration/GenerateWithCache', - request_serializer=generate__pb2.BatchCached.SerializeToString, - response_deserializer=generate__pb2.Response.FromString, - ) - - -class TextGenerationServicer(object): - """Missing associated documentation comment in .proto file.""" - - def ServiceDiscovery(self, request, context): - """/ Service discovery - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ClearCache(self, request, context): - """/ Empties batch cache - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Generate(self, request, context): - """/ Generate tokens for a batch without cache - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GenerateWithCache(self, request, context): - """/ Generate tokens for a batch with cache - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_TextGenerationServicer_to_server(servicer, server): - rpc_method_handlers = { - 'ServiceDiscovery': grpc.unary_unary_rpc_method_handler( - servicer.ServiceDiscovery, - request_deserializer=generate__pb2.Empty.FromString, - response_serializer=generate__pb2.ServiceDiscoveryResponse.SerializeToString, - ), - 'ClearCache': grpc.unary_unary_rpc_method_handler( - servicer.ClearCache, - request_deserializer=generate__pb2.Empty.FromString, - response_serializer=generate__pb2.Empty.SerializeToString, - ), - 'Generate': grpc.unary_unary_rpc_method_handler( - servicer.Generate, - request_deserializer=generate__pb2.Batch.FromString, - response_serializer=generate__pb2.Response.SerializeToString, - ), - 'GenerateWithCache': grpc.unary_unary_rpc_method_handler( - servicer.GenerateWithCache, - request_deserializer=generate__pb2.BatchCached.FromString, - response_serializer=generate__pb2.Response.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'generate.v1.TextGeneration', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class TextGeneration(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def ServiceDiscovery(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ServiceDiscovery', - generate__pb2.Empty.SerializeToString, - generate__pb2.ServiceDiscoveryResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def ClearCache(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/ClearCache', - generate__pb2.Empty.SerializeToString, - generate__pb2.Empty.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def Generate(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/Generate', - generate__pb2.Batch.SerializeToString, - generate__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def GenerateWithCache(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/generate.v1.TextGeneration/GenerateWithCache', - generate__pb2.BatchCached.SerializeToString, - generate__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) From fa9a08846739ed7508f5c802df577242c5f44768 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Tue, 11 Oct 2022 10:36:51 +0200 Subject: [PATCH 003/793] Add load testing --- .gitignore | 1 + k6/load_test.js | 97 ++++++++++++++++++++++++ router/src/{infer.rs => batcher.rs} | 53 +++++++------ router/src/db.rs | 13 +++- router/src/main.rs | 112 +++------------------------- router/src/server.rs | 111 +++++++++++++++++++++++++++ 6 files changed, 260 insertions(+), 127 deletions(-) create mode 100644 .gitignore create mode 100644 k6/load_test.js rename router/src/{infer.rs => batcher.rs} (69%) create mode 100644 router/src/server.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..723ef36f --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea \ No newline at end of file diff --git a/k6/load_test.js b/k6/load_test.js new file mode 100644 index 00000000..3f3791af --- /dev/null +++ b/k6/load_test.js @@ -0,0 +1,97 @@ +import http from 'k6/http'; +import {check, sleep} from 'k6'; + +export const options = { + stages: [ + {duration: '1m', target: 50}, + {duration: '2m', target: 100}, + {duration: '1m', target: 0}, + ], + hosts: { + 'text-generation-inference.huggingface.co': '127.0.0.1:3000', + }, +}; +const SLEEP_DURATION = 1; + +function greedy_example(inputs, max_new_tokens, name) { + let body = JSON.stringify({ + inputs: inputs, + parameters: { + max_new_tokens: max_new_tokens, + do_sample: false, + } + }); + let params = { + headers: { + 'Content-Type': 'application/json', + }, + tags: { + name: name + } + }; + return http.post('http://text-generation-inference.huggingface.co/generate', body, params); +} + +function sample_example(inputs, max_new_tokens, name) { + let body = JSON.stringify({ + inputs: inputs, + parameters: { + max_new_tokens: max_new_tokens, + do_sample: true, + top_p: 0.9 + } + }); + let params = { + headers: { + 'Content-Type': 'application/json', + }, + tags: { + name: name + } + }; + return http.post('http://text-generation-inference.huggingface.co/generate', body, params); +} + +export default function () { + const response_1 = sample_example('A "whatpu" is a small, furry animal native to Tanzania. An example of a sentence that uses the word whatpu is: We were traveling in Africa and we saw these very cute whatpus. To do a "farduddle" means to jump up and down really fast. An example of a sentence that uses the word farduddle is:', 32, 'example-1'); + check(response_1, { + 'is status 200': (r) => r.status === 200, + }); + sleep(SLEEP_DURATION); + + const response_2 = sample_example("A poem about the beauty of science by Alfred Edgar Brittle\\nTitle: The Magic Craft\\nIn the old times", 50, "example-2"); + check(response_2, { + 'is status 200': (r) => r.status === 200, + }); + sleep(SLEEP_DURATION); + + const response_3 = greedy_example("استخراج العدد العاملي في لغة بايثون: ", 30, "example-3"); + check(response_3, { + 'is status 200': (r) => r.status === 200, + }); + sleep(SLEEP_DURATION); + + const response_4 = sample_example("Pour déguster un ortolan, il faut tout d'abord", 32, "example-4"); + check(response_4, { + 'is status 200': (r) => r.status === 200, + }); + sleep(SLEEP_DURATION); + + const response_5 = sample_example("Traduce español de España a español de Argentina\nEl coche es rojo - el auto es rojo\nEl ordenador es nuevo - la computadora es nueva\nel boligrafo es negro -", 16, "example-5"); + check(response_5, { + 'is status 200': (r) => r.status === 200, + }); + sleep(SLEEP_DURATION); + + const response_6 = sample_example("Question: If I put cheese into the fridge, will it melt?\nAnswer:", 32, "example-6"); + check(response_6, { + 'is status 200': (r) => r.status === 200, + }); + sleep(SLEEP_DURATION); + + const response_7 = greedy_example("Question: Where does the Greek Goddess Persephone spend half of the year when she is not with her mother?\nAnswer:", 24, "example-7"); + check(response_7, { + 'is status 200': (r) => r.status === 200, + }); + sleep(SLEEP_DURATION); +} \ No newline at end of file diff --git a/router/src/infer.rs b/router/src/batcher.rs similarity index 69% rename from router/src/infer.rs rename to router/src/batcher.rs index 2a7aa0ac..2da47dfc 100644 --- a/router/src/infer.rs +++ b/router/src/batcher.rs @@ -1,14 +1,17 @@ -use crate::{Db, GenerateRequest}; -use bloom_inference_client::{Batch, BatchCached, CacheEntry, ClientError, FinishedGeneration, ShardedClient}; +use crate::Db; +use bloom_inference_client::{ + Batch, BatchCached, CacheEntry, ClientError, FinishedGeneration, ShardedClient, +}; use std::sync::Arc; -use tokio::sync::{oneshot, Notify}; +use tokio::sync::{Notify, oneshot}; +use crate::server::GenerateRequest; const MAX_LENGTH: usize = 128; pub struct InferError {} #[derive(Clone)] -pub(crate) struct Infer { +pub(crate) struct Batcher { db: Db, shared: Arc, } @@ -17,7 +20,7 @@ struct Shared { batching_task: Notify, } -impl Infer { +impl Batcher { pub(crate) fn new(client: ShardedClient) -> Self { let db = Db::new(); let shared = Arc::new(Shared { @@ -38,7 +41,7 @@ impl Infer { self.shared.batching_task.notify_waiters(); match request_rx.await.unwrap() { Ok(output) => Ok(output), - Err(_) => Err(InferError {}) + Err(_) => Err(InferError {}), } } } @@ -57,19 +60,19 @@ async fn batching_task(client: ShardedClient, db: Db, shared: Arc) { let mut max_sequence_length = entry.sequence_length; let mut request_ids = entry.request_ids; - if total_batch_size <= 16 { - if let Some(batch) = db.next_batch_minimum_size(16, 48) { - let other_cache_entry = infer_batch(batch, &client, &db).await; - - if let Some(entry) = other_cache_entry { - batch_cached_ids.push(entry.id); - total_batch_size += entry.request_ids.len(); - max_sequence_length = - max_sequence_length.max(entry.sequence_length); - request_ids.extend(entry.request_ids.into_iter()); - } - } - } + // if total_batch_size <= 16 { + // if let Some(batch) = db.next_batch_minimum_size(16, 48) { + // let other_cache_entry = infer_batch(batch, &client, &db).await; + // + // if let Some(entry) = other_cache_entry { + // batch_cached_ids.push(entry.id); + // total_batch_size += entry.request_ids.len(); + // max_sequence_length = + // max_sequence_length.max(entry.sequence_length); + // request_ids.extend(entry.request_ids.into_iter()); + // } + // } + // } let batch_cached = BatchCached { id: entry.id, @@ -87,7 +90,11 @@ async fn batching_task(client: ShardedClient, db: Db, shared: Arc) { } } -async fn infer_batch_cached(batch: BatchCached, client: &ShardedClient, db: &Db) -> Option { +async fn infer_batch_cached( + batch: BatchCached, + client: &ShardedClient, + db: &Db, +) -> Option { match client.generate_with_cache(batch.clone()).await { Ok((finished, cache_entry)) => { send_finished(finished, db); @@ -109,7 +116,11 @@ async fn infer_batch(batch: Batch, client: &ShardedClient, db: &Db) -> Option { println!("{:?}", err); - send_error(err, batch.requests.into_iter().map(|req| req.id).collect(), &db); + send_error( + err, + batch.requests.into_iter().map(|req| req.id).collect(), + &db, + ); None } } diff --git a/router/src/db.rs b/router/src/db.rs index 3dd98d94..b6d218e2 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,5 +1,5 @@ /// This code is massively inspired by Tokio mini-redis -use crate::GenerateRequest; +use crate::server::GenerateRequest; use bloom_inference_client::{Batch, ClientError, LogitsWarperParameters, Request}; use parking_lot::RwLock; use std::collections::BTreeMap; @@ -44,7 +44,11 @@ impl Db { Self { shared } } - pub(crate) fn append(&self, request: GenerateRequest, sender: Sender>) { + pub(crate) fn append( + &self, + request: GenerateRequest, + sender: Sender>, + ) { let mut state = self.shared.state.write(); let id = state.next_id; @@ -65,7 +69,10 @@ impl Db { state.entries.insert(id, (request, sender)); } - pub(crate) fn remove(&self, id: &u64) -> Option<(Request, Sender>)> { + pub(crate) fn remove( + &self, + id: &u64, + ) -> Option<(Request, Sender>)> { let mut state = self.shared.state.write(); state.entries.remove(id) } diff --git a/router/src/main.rs b/router/src/main.rs index 5d87cd46..97ccb571 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,105 +1,15 @@ -use tokio::time::Instant; - -use poem; -use poem::middleware::AddData; -use poem::web::Data; -use poem::{handler, listener::TcpListener, post, web::Json, EndpointExt, Result, Route, Server}; - use bloom_inference_client::ShardedClient; -use serde::Deserialize; +use poem; +use poem::listener::TcpListener; use std::time::Duration; -use poem::http::StatusCode; -use tracing::instrument; -mod db; +mod server; +mod db; use db::Db; -mod infer; - -use infer::Infer; - -#[derive(Clone, Debug, Deserialize)] -struct GenerateParameters { - #[serde(default = "default_temperature")] - temperature: f32, - #[serde(default = "default_top_k")] - top_k: u32, - #[serde(default = "default_top_p")] - top_p: f32, - #[serde(default = "default_do_sample")] - do_sample: bool, - #[serde(default = "default_max_new_tokens")] - max_new_tokens: u32, -} - -fn default_temperature() -> f32 { - 1.0 -} - -fn default_top_k() -> u32 { - 0 -} - -fn default_top_p() -> f32 { - 1.0 -} - -fn default_do_sample() -> bool { - false -} - -fn default_max_new_tokens() -> u32 { - 20 -} - -#[derive(Clone, Debug, Deserialize)] -struct GenerateRequest { - inputs: String, - #[serde(default = "default_parameters")] - parameters: GenerateParameters, -} - -fn default_parameters() -> GenerateParameters { - GenerateParameters { - temperature: default_temperature(), - top_k: default_top_k(), - top_p: default_top_p(), - do_sample: default_do_sample(), - max_new_tokens: default_max_new_tokens(), - } -} - -#[handler] -#[instrument(skip(infer), fields(time, time_per_token))] -async fn generate( - infer: Data<&Infer>, - req: Json, -) -> Result> { - let start = Instant::now(); - - let output = infer - .infer(GenerateRequest { - inputs: req.inputs.clone(), - parameters: req.parameters.clone(), - }) - .await; - - match output { - Ok(generated_text) => { - tracing::Span::current().record("time", format!("{:?}", start.elapsed())); - tracing::Span::current().record("time_per_token", format!("{:?}", start.elapsed() / req.parameters.max_new_tokens)); - tracing::info!("response: {}", generated_text); - - Ok(Json(serde_json::json!({ - "generated_text": generated_text, - }))) - } - Err(_) => { - Err(poem::Error::from_status(StatusCode::INTERNAL_SERVER_ERROR)) - } - } -} +mod batcher; +use batcher::Batcher; #[tokio::main] async fn main() -> Result<(), std::io::Error> { @@ -114,12 +24,8 @@ async fn main() -> Result<(), std::io::Error> { .expect("Unable to clear cache"); tracing::info!("Connected"); - let infer = Infer::new(sharded_client); + let addr = "127.0.0.1:3000".to_string(); + let listener = TcpListener::bind(addr); - let app = Route::new() - .at("/generate", post(generate)) - .with(AddData::new(infer)); - Server::new(TcpListener::bind("127.0.0.1:3000")) - .run(app) - .await + server::run(sharded_client, listener).await } diff --git a/router/src/server.rs b/router/src/server.rs new file mode 100644 index 00000000..0daf8df3 --- /dev/null +++ b/router/src/server.rs @@ -0,0 +1,111 @@ +use poem::{EndpointExt, handler, post, Route, Server}; +use poem::http::StatusCode; +use poem::listener::TcpListener; +use poem::middleware::AddData; +use poem::web::{Data, Json}; +use tokio::time::Instant; +use crate::{Batcher, ShardedClient}; +use tracing::instrument; +use serde::Deserialize; + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct GenerateParameters { + #[serde(default = "default_temperature")] + pub temperature: f32, + #[serde(default = "default_top_k")] + pub top_k: u32, + #[serde(default = "default_top_p")] + pub top_p: f32, + #[serde(default = "default_do_sample")] + pub do_sample: bool, + #[serde(default = "default_max_new_tokens")] + pub max_new_tokens: u32, +} + +fn default_temperature() -> f32 { + 1.0 +} + +fn default_top_k() -> u32 { + 0 +} + +fn default_top_p() -> f32 { + 1.0 +} + +fn default_do_sample() -> bool { + false +} + +fn default_max_new_tokens() -> u32 { + 20 +} + +fn default_parameters() -> GenerateParameters { + GenerateParameters { + temperature: default_temperature(), + top_k: default_top_k(), + top_p: default_top_p(), + do_sample: default_do_sample(), + max_new_tokens: default_max_new_tokens(), + } +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct GenerateRequest { + pub inputs: String, + #[serde(default = "default_parameters")] + pub parameters: GenerateParameters, +} + + +#[handler] +#[instrument(skip(infer), fields(time, time_per_token))] +async fn generate( + infer: Data<&Batcher>, + req: Json, +) -> poem::Result> { + let start = Instant::now(); + + let output = infer + .infer(GenerateRequest { + inputs: req.inputs.clone(), + parameters: req.parameters.clone(), + }) + .await; + + match output { + Ok(generated_text) => { + tracing::Span::current().record("time", format!("{:?}", start.elapsed())); + tracing::Span::current().record( + "time_per_token", + format!("{:?}", start.elapsed() / req.parameters.max_new_tokens), + ); + tracing::info!("response: {}", generated_text); + + Ok(Json(serde_json::json!({ + "generated_text": generated_text, + }))) + } + Err(_) => Err(poem::Error::from_status(StatusCode::INTERNAL_SERVER_ERROR)), + } +} + +pub async fn run(client: ShardedClient, listener: TcpListener) -> Result<(), std::io::Error> { + client + .clear_cache() + .await + .expect("Unable to clear cache"); + tracing::info!("Connected"); + + let infer = Batcher::new(client); + + let app = Route::new() + .at("/generate", post(generate)) + .with(AddData::new(infer)); + + Server::new(listener) + .run(app) + .await +} \ No newline at end of file From 4c693e65245058a4d0ca227ee30b6d8a35d115f1 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Tue, 11 Oct 2022 16:50:54 +0200 Subject: [PATCH 004/793] Refactored gRPC interface Added validation logic --- README.md | 21 +- proto/generate.proto | 113 +++++++---- router/client/src/client.rs | 61 ++++-- router/client/src/lib.rs | 4 +- router/client/src/sharded_client.rs | 64 ++++-- router/src/batcher.rs | 122 +++++------- router/src/db.rs | 10 + router/src/main.rs | 42 ++-- router/src/server.rs | 40 ++-- router/src/validation.rs | 65 +++++++ server/bloom_inference/cache.py | 37 +--- server/bloom_inference/model.py | 292 +++++++++++++++------------- server/bloom_inference/server.py | 104 +++++++--- 13 files changed, 613 insertions(+), 362 deletions(-) create mode 100644 router/src/validation.rs diff --git a/README.md b/README.md index fe7bb443..ad8adaa9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,23 @@ -# BLOOM Inference +# Text Generation Inference -A Rust and gRPC server for BLOOM Inference. +A Rust and gRPC server for text generation inference. + +## Load Tests + +See `k6/load_test.js` +We send the default examples with a 1 second delay between each request. + +Stages: +- Ramp up to 50 concurrent requests per second in 1min +- Ramp up from 50 to 100 concurrent requests per second in 2min +- Ramp down to 0 concurrent requests per second in 1min + + +| | avg | min | med | max | p(90) | p(95) | RPS | +|------------------------|-----------|-----------|-----------|------------|-----------|-----------|----------| +| Original code | 8.9s | 1s | 9.12s | 16.69s | 13.7s | 14.26s | 5.9 | +| ISO with original code | 8.88s | 959.53ms | 8.89s | 17.08s | 13.34s | 14.12s | 5.94 | +| New batching logic | **5.44s** | **1.27s** | **5.28s** | **13.12s** | **7.78s** | **8.92s** | **9.08** | ## Install diff --git a/proto/generate.proto b/proto/generate.proto index 324b9206..8c5221b4 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -2,21 +2,35 @@ syntax = "proto3"; package generate.v1; -service TextGeneration { +service TextGenerationService { /// Service discovery - rpc ServiceDiscovery(Empty) returns (ServiceDiscoveryResponse) {} + rpc ServiceDiscovery (ServiceDiscoveryRequest) returns (ServiceDiscoveryResponse) {} /// Empties batch cache - rpc ClearCache(Empty) returns (Empty); - /// Generate tokens for a batch without cache - rpc Generate(Batch) returns (Response); - /// Generate tokens for a batch with cache - rpc GenerateWithCache(BatchCached) returns (Response); + rpc ClearCache (ClearCacheRequest) returns (ClearCacheResponse); + /// Generate tokens for a batch + rpc Generate (GenerateRequest) returns (GenerateResponse); + /// Generate tokens for a list of cached batches + rpc GenerateWithCache (GenerateWithCacheRequest) returns (GenerateWithCacheResponse); + /// Generate tokens until the text of at least one request of the batch is generated + rpc GenerateUntilFinished (GenerateUntilFinishedRequest) returns (GenerateUntilFinishedResponse); + /// Generate tokens until the text of at least one request of the cached batches i finished + rpc GenerateUntilFinishedWithCache (GenerateUntilFinishedWithCacheRequest) returns (GenerateUntilFinishedWithCacheResponse); } +/// Empty request +message ServiceDiscoveryRequest {} + message ServiceDiscoveryResponse { + /// Other shards urls repeated string urls = 1; } +/// Empty request +message ClearCacheRequest {} + +/// Empty response +message ClearCacheResponse {} + message LogitsWarperParameters { float temperature = 1; uint32 top_k = 2; @@ -29,10 +43,12 @@ message Request { uint64 id = 1; /// The generation context string inputs = 2; + /// The number of tokens inside inputs + uint32 input_length = 3; /// Logits Warper Parameters - LogitsWarperParameters parameters = 3; + LogitsWarperParameters parameters = 4; /// Stopping criteria - uint32 max_new_tokens = 4; + uint32 max_new_tokens = 5; } message Batch { @@ -40,44 +56,63 @@ message Batch { uint64 id = 1; /// Individual requests repeated Request requests = 2; + /// Batch size (==len(requests)) + uint32 size = 3; + /// Length of the longest sequence within the batch (used for padding) + uint32 max_sequence_length = 4; } -message BatchCached { - /// Batch ID - uint64 id = 1; - /// Request ids within cache - repeated uint64 request_ids = 2; - /// Cache IDs - repeated uint64 batch_cached_ids = 3; - /// Batch size (sum of all batch sizes) - uint32 total_batch_size = 4; - /// Max sequence length - uint32 max_sequence_length = 5; -} - -message FinishedGeneration { - /// ID of the original request - uint64 id = 1; +message GeneratedText { + /// Request + Request request = 1; /// Output string output = 2; } -message CacheEntry { - /// Cache ID; same as batch ID - uint64 id = 1; - /// Requests present in cache entry - repeated uint64 request_ids = 2; - /// Sequence length - uint32 sequence_length = 3; +message GenerateRequest { + /// Batch + Batch batch = 1; } -message Response { - /// Finished requests (optional) - repeated FinishedGeneration finished = 1; - /// Cache entry (optional) - optional CacheEntry cache_entry = 2; +message GenerateResponse { + /// Finished requests + repeated GeneratedText generated_texts = 1; + /// Next batch (cached) + optional Batch batch = 2; } +message GenerateWithCacheRequest { + /// Cached batches + repeated Batch batches = 1; +} -// Represent an empty message. -message Empty {} \ No newline at end of file +message GenerateWithCacheResponse { + /// Finished requests + repeated GeneratedText generated_texts = 1; + /// Next batch (cached) + optional Batch batch = 2; +} + +message GenerateUntilFinishedRequest { + /// Batch + Batch batch = 1; +} + +message GenerateUntilFinishedResponse { + /// Finished requests + repeated GeneratedText generated_texts = 1; + /// Next batch (cached) + optional Batch batch = 2; +} + +message GenerateUntilFinishedWithCacheRequest { + /// Cached batches + repeated Batch batches = 1; +} + +message GenerateUntilFinishedWithCacheResponse { + /// Finished requests + repeated GeneratedText generated_texts = 1; + /// Next batch (cached) + optional Batch batch = 2; +} diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 7f68e252..712c13da 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -1,4 +1,4 @@ -use crate::pb::generate::v1::text_generation_client::TextGenerationClient; +use crate::pb::generate::v1::text_generation_service_client::TextGenerationServiceClient; use crate::pb::generate::v1::*; use crate::Result; use std::time::Duration; @@ -9,7 +9,7 @@ use tracing::*; /// BLOOM Inference gRPC client #[derive(Clone)] pub struct Client { - stub: TextGenerationClient>, + stub: TextGenerationServiceClient>, } impl Client { @@ -22,13 +22,13 @@ impl Client { let timeout_channel = Timeout::new(channel, timeout); Self { - stub: TextGenerationClient::new(timeout_channel), + stub: TextGenerationServiceClient::new(timeout_channel), } } /// Returns a client connected to the given unix socket. Requests exceeding timeout will fail. pub async fn connect_uds(path: String, timeout: Duration) -> Self { - let channel = Channel::from_shared(format!("http://[::]:50051")) + let channel = Channel::from_shared("http://[::]:50051".to_string()) .unwrap() .connect_with_connector(tower::service_fn(move |_: Uri| { tokio::net::UnixStream::connect(path.clone()) @@ -38,13 +38,13 @@ impl Client { let timeout_channel = Timeout::new(channel, timeout); Self { - stub: TextGenerationClient::new(timeout_channel), + stub: TextGenerationServiceClient::new(timeout_channel), } } #[instrument(skip(self))] pub async fn service_discovery(&mut self) -> Result> { - let request = tonic::Request::new(Empty {}); + let request = tonic::Request::new(ServiceDiscoveryRequest {}); let response = self .stub .service_discovery(request) @@ -64,7 +64,7 @@ impl Client { #[instrument(skip(self))] pub async fn clear_cache(&mut self) -> Result<()> { - let request = tonic::Request::new(Empty {}); + let request = tonic::Request::new(ClearCacheRequest {}); self.stub .clear_cache(request) .instrument(info_span!("clear_cache")) @@ -73,32 +73,59 @@ impl Client { } #[instrument(skip(self))] - pub async fn generate( - &mut self, - request: Batch, - ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(request); + pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { + let request = tonic::Request::new(GenerateRequest { batch: Some(batch) }); let response = self .stub .generate(request) .instrument(info_span!("generate")) .await? .into_inner(); - Ok((response.finished, response.cache_entry)) + Ok((response.generated_texts, response.batch)) } #[instrument(skip(self))] pub async fn generate_with_cache( &mut self, - request: BatchCached, - ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(request); + batches: Vec, + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(GenerateWithCacheRequest { batches }); let response = self .stub .generate_with_cache(request) .instrument(info_span!("generate_with_cache")) .await? .into_inner(); - Ok((response.finished, response.cache_entry)) + Ok((response.generated_texts, response.batch)) + } + + #[instrument(skip(self))] + pub async fn generate_until_finished( + &mut self, + batch: Batch, + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(GenerateUntilFinishedRequest { batch: Some(batch) }); + let response = self + .stub + .generate_until_finished(request) + .instrument(info_span!("generate_until_finished")) + .await? + .into_inner(); + Ok((response.generated_texts, response.batch)) + } + + #[instrument(skip(self))] + pub async fn generate_until_finished_with_cache( + &mut self, + batches: Vec, + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(GenerateUntilFinishedWithCacheRequest { batches }); + let response = self + .stub + .generate_until_finished_with_cache(request) + .instrument(info_span!("generate_until_finished_with_cache")) + .await? + .into_inner(); + Ok((response.generated_texts, response.batch)) } } diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index aca9f65b..bb9919e2 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -5,9 +5,7 @@ mod pb; mod sharded_client; pub use client::Client; -pub use pb::generate::v1::{ - Batch, BatchCached, CacheEntry, FinishedGeneration, LogitsWarperParameters, Request, -}; +pub use pb::generate::v1::{Batch, GeneratedText, LogitsWarperParameters, Request}; pub use sharded_client::ShardedClient; use thiserror::Error; pub use tonic::transport::Uri; diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 6856a9bc..7af741f0 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,5 +1,5 @@ use crate::Result; -use crate::{Batch, BatchCached, CacheEntry, Client, FinishedGeneration}; +use crate::{Batch, Client, GeneratedText}; use futures::future::join_all; use std::time::Duration; use tokio::sync::{broadcast, mpsc}; @@ -9,11 +9,19 @@ use tonic::transport::Uri; enum Command { Generate( Batch, - mpsc::Sender, Option)>>, + mpsc::Sender, Option)>>, ), GenerateWithCache( - BatchCached, - mpsc::Sender, Option)>>, + Vec, + mpsc::Sender, Option)>>, + ), + GenerateUntilFinished( + Batch, + mpsc::Sender, Option)>>, + ), + GenerateUntilFinishedWithCache( + Vec, + mpsc::Sender, Option)>>, ), ClearCache(mpsc::Sender>), } @@ -25,8 +33,16 @@ async fn client_task(mut client: Client, mut request_subscriber: broadcast::Rece let result = client.generate(batch).await; response_tx.try_send(result).unwrap_or(()); } - Command::GenerateWithCache(batch_cached, response_tx) => { - let result = client.generate_with_cache(batch_cached).await; + Command::GenerateWithCache(batches, response_tx) => { + let result = client.generate_with_cache(batches).await; + response_tx.try_send(result).unwrap_or(()); + } + Command::GenerateUntilFinished(batch, response_tx) => { + let result = client.generate_until_finished(batch).await; + response_tx.try_send(result).unwrap_or(()); + } + Command::GenerateUntilFinishedWithCache(batches, response_tx) => { + let result = client.generate_until_finished_with_cache(batches).await; response_tx.try_send(result).unwrap_or(()); } Command::ClearCache(response_tx) => { @@ -74,10 +90,7 @@ impl ShardedClient { Self::from_master_client(master_client).await } - pub async fn generate( - &self, - batch: Batch, - ) -> Result<(Vec, Option)> { + pub async fn generate(&self, batch: Batch) -> Result<(Vec, Option)> { let (response_tx, mut response_rx) = mpsc::channel(1); self.request_tx .send(Command::Generate(batch, response_tx)) @@ -87,11 +100,36 @@ impl ShardedClient { pub async fn generate_with_cache( &self, - batch_cached: BatchCached, - ) -> Result<(Vec, Option)> { + batches: Vec, + ) -> Result<(Vec, Option)> { + let (response_tx, mut response_rx) = mpsc::channel(1); + self.request_tx + .send(Command::GenerateWithCache(batches, response_tx)) + .unwrap(); + response_rx.recv().await.unwrap() + } + + pub async fn generate_until_finished( + &self, + batch: Batch, + ) -> Result<(Vec, Option)> { + let (response_tx, mut response_rx) = mpsc::channel(1); + self.request_tx + .send(Command::GenerateUntilFinished(batch, response_tx)) + .unwrap(); + response_rx.recv().await.unwrap() + } + + pub async fn generate_until_finished_with_cache( + &self, + batches: Vec, + ) -> Result<(Vec, Option)> { let (response_tx, mut response_rx) = mpsc::channel(1); self.request_tx - .send(Command::GenerateWithCache(batch_cached, response_tx)) + .send(Command::GenerateUntilFinishedWithCache( + batches, + response_tx, + )) .unwrap(); response_rx.recv().await.unwrap() } diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 2da47dfc..a044e26c 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -1,10 +1,9 @@ +use crate::server::GenerateRequest; use crate::Db; -use bloom_inference_client::{ - Batch, BatchCached, CacheEntry, ClientError, FinishedGeneration, ShardedClient, -}; +use bloom_inference_client::{Batch, ClientError, GeneratedText, ShardedClient}; +use std::future::Future; use std::sync::Arc; -use tokio::sync::{Notify, oneshot}; -use crate::server::GenerateRequest; +use tokio::sync::{oneshot, Notify}; const MAX_LENGTH: usize = 128; @@ -32,12 +31,16 @@ impl Batcher { Self { db, shared } } - pub(crate) async fn infer(&self, request: GenerateRequest) -> Result { + pub(crate) async fn infer( + &self, + input_length: usize, + request: GenerateRequest, + ) -> Result { if self.db.len() > MAX_LENGTH { return Err(InferError {}); } let (request_tx, request_rx) = oneshot::channel(); - self.db.append(request, request_tx); + self.db.append(input_length, request, request_tx); self.shared.batching_task.notify_waiters(); match request_rx.await.unwrap() { Ok(output) => Ok(output), @@ -51,76 +54,57 @@ async fn batching_task(client: ShardedClient, db: Db, shared: Arc) { shared.batching_task.notified().await; if let Some(batch) = db.next_batch(32) { - let mut cache_entry = infer_batch(batch, &client, &db).await; - - loop { - if let Some(entry) = cache_entry { - let mut batch_cached_ids = vec![entry.id]; - let mut total_batch_size = entry.request_ids.len(); - let mut max_sequence_length = entry.sequence_length; - let mut request_ids = entry.request_ids; - - // if total_batch_size <= 16 { - // if let Some(batch) = db.next_batch_minimum_size(16, 48) { - // let other_cache_entry = infer_batch(batch, &client, &db).await; - // - // if let Some(entry) = other_cache_entry { - // batch_cached_ids.push(entry.id); - // total_batch_size += entry.request_ids.len(); - // max_sequence_length = - // max_sequence_length.max(entry.sequence_length); - // request_ids.extend(entry.request_ids.into_iter()); - // } - // } - // } - - let batch_cached = BatchCached { - id: entry.id, - batch_cached_ids, - total_batch_size: total_batch_size as u32, - max_sequence_length, - request_ids, - }; - cache_entry = infer_batch_cached(batch_cached, &client, &db).await; - } else { - break; + let request_ids = batch.requests.iter().map(|req| req.id).collect(); + let mut cached_batch = match batch.size { + size if size > 16 => { + wrap_future(client.generate_until_finished(batch), request_ids, &db).await + } + _ => wrap_future(client.generate(batch), request_ids, &db).await, + }; + + while let Some(batch) = cached_batch { + let batch_size = batch.size; + let mut request_ids: Vec = batch.requests.iter().map(|req| req.id).collect(); + let mut batches = vec![batch]; + + if batch_size <= 16 { + if let Some(new_batch) = db.next_batch_minimum_size(16, 48) { + let new_batch_request_ids = + new_batch.requests.iter().map(|req| req.id).collect(); + let new_cached_batch = + wrap_future(client.generate(new_batch), new_batch_request_ids, &db) + .await; + if let Some(new_cached_batch) = new_cached_batch { + request_ids.extend(new_cached_batch.requests.iter().map(|req| req.id)); + batches.push(new_cached_batch); + } + } } + + cached_batch = match batch_size { + size if size > 16 => { + wrap_future(client.generate_until_finished_with_cache(batches), request_ids, &db).await + } + _ => wrap_future(client.generate_with_cache(batches), request_ids, &db).await, + }; } } } } -async fn infer_batch_cached( - batch: BatchCached, - client: &ShardedClient, +async fn wrap_future( + future: impl Future, Option), ClientError>>, + request_ids: Vec, db: &Db, -) -> Option { - match client.generate_with_cache(batch.clone()).await { - Ok((finished, cache_entry)) => { - send_finished(finished, db); - cache_entry - } - Err(err) => { - println!("{:?}", err); - send_error(err, batch.request_ids, &db); - None - } - } -} - -async fn infer_batch(batch: Batch, client: &ShardedClient, db: &Db) -> Option { - match client.generate(batch.clone()).await { - Ok((finished, cache_entry)) => { - send_finished(finished, db); - cache_entry +) -> Option { + match future.await { + Ok((generated_texts, next_batch)) => { + send_generated(generated_texts, db); + next_batch } Err(err) => { println!("{:?}", err); - send_error( - err, - batch.requests.into_iter().map(|req| req.id).collect(), - &db, - ); + send_error(err, request_ids, db); None } } @@ -133,9 +117,9 @@ fn send_error(error: ClientError, request_ids: Vec, db: &Db) { }); } -fn send_finished(finished: Vec, db: &Db) { +fn send_generated(finished: Vec, db: &Db) { finished.into_iter().for_each(|output| { - let (_, response_tx) = db.remove(&output.id).unwrap(); + let (_, response_tx) = db.remove(&output.request.unwrap().id).unwrap(); response_tx.send(Ok(output.output)).unwrap_or(()); }); } diff --git a/router/src/db.rs b/router/src/db.rs index b6d218e2..5118b2fc 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -46,6 +46,7 @@ impl Db { pub(crate) fn append( &self, + input_length: usize, request: GenerateRequest, sender: Sender>, ) { @@ -63,6 +64,7 @@ impl Db { let request = Request { id, inputs: request.inputs, + input_length: input_length as u32, parameters, max_new_tokens: request.parameters.max_new_tokens, }; @@ -103,9 +105,13 @@ impl Db { pub(crate) fn next_batch(&self, max_size: usize) -> Option { if let Some((last_id, requests)) = self.next_requests(max_size) { let mut state = self.shared.state.write(); + let size = requests.len(); + let max_sequence_length = requests.iter().map(|r| r.input_length).max().unwrap(); let batch = Batch { id: state.next_batch_id, requests, + size: size as u32, + max_sequence_length, }; state.next_batch_start_id = last_id + 1; state.next_batch_id += 1; @@ -122,9 +128,13 @@ impl Db { if let Some((last_id, requests)) = self.next_requests(max_size) { if requests.len() >= min_size { let mut state = self.shared.state.write(); + let size = requests.len(); + let max_sequence_length = requests.iter().map(|r| r.input_length).max().unwrap(); let batch = Batch { id: state.next_batch_id, requests, + size: size as u32, + max_sequence_length, }; state.next_batch_start_id = last_id + 1; state.next_batch_id += 1; diff --git a/router/src/main.rs b/router/src/main.rs index 97ccb571..fe82d059 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,31 +1,45 @@ use bloom_inference_client::ShardedClient; -use poem; use poem::listener::TcpListener; use std::time::Duration; +use tokenizers::Tokenizer; mod server; +mod validation; + +use validation::Validation; mod db; + use db::Db; mod batcher; + use batcher::Batcher; -#[tokio::main] -async fn main() -> Result<(), std::io::Error> { - tracing_subscriber::fmt::init(); +fn main() -> Result<(), std::io::Error> { + let tokenizer = Tokenizer::from_pretrained("bigscience/bloom", None).unwrap(); + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + tracing_subscriber::fmt::init(); - let sharded_client = - ShardedClient::connect_uds("/tmp/bloom-inference-0".to_string(), Duration::from_secs(5)) + let sharded_client = ShardedClient::connect_uds( + "/tmp/bloom-inference-0".to_string(), + Duration::from_secs(5), + ) .await; - sharded_client - .clear_cache() - .await - .expect("Unable to clear cache"); - tracing::info!("Connected"); + sharded_client + .clear_cache() + .await + .expect("Unable to clear cache"); + tracing::info!("Connected"); - let addr = "127.0.0.1:3000".to_string(); - let listener = TcpListener::bind(addr); + let addr = "127.0.0.1:3000".to_string(); + let listener = TcpListener::bind(addr); - server::run(sharded_client, listener).await + server::run(sharded_client, tokenizer, listener).await + }) } diff --git a/router/src/server.rs b/router/src/server.rs index 0daf8df3..14e81709 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,12 +1,13 @@ -use poem::{EndpointExt, handler, post, Route, Server}; +use crate::{Batcher, ShardedClient, Validation}; use poem::http::StatusCode; use poem::listener::TcpListener; use poem::middleware::AddData; use poem::web::{Data, Json}; +use poem::{handler, post, EndpointExt, Route, Server}; +use serde::Deserialize; +use tokenizers::Tokenizer; use tokio::time::Instant; -use crate::{Batcher, ShardedClient}; use tracing::instrument; -use serde::Deserialize; #[derive(Clone, Debug, Deserialize)] pub(crate) struct GenerateParameters { @@ -59,21 +60,24 @@ pub(crate) struct GenerateRequest { pub parameters: GenerateParameters, } - #[handler] -#[instrument(skip(infer), fields(time, time_per_token))] +#[instrument(skip(validation, infer), fields(time, time_per_token))] async fn generate( + validation: Data<&Validation>, infer: Data<&Batcher>, req: Json, ) -> poem::Result> { let start = Instant::now(); - let output = infer - .infer(GenerateRequest { + let (input_length, validated_request) = validation + .validate(GenerateRequest { inputs: req.inputs.clone(), parameters: req.parameters.clone(), }) - .await; + .await + .unwrap(); + + let output = infer.infer(input_length, validated_request).await; match output { Ok(generated_text) => { @@ -92,20 +96,22 @@ async fn generate( } } -pub async fn run(client: ShardedClient, listener: TcpListener) -> Result<(), std::io::Error> { - client - .clear_cache() - .await - .expect("Unable to clear cache"); +pub async fn run( + client: ShardedClient, + tokenizer: Tokenizer, + listener: TcpListener, +) -> Result<(), std::io::Error> { + client.clear_cache().await.expect("Unable to clear cache"); tracing::info!("Connected"); let infer = Batcher::new(client); + let validation = Validation::new(tokenizer); + let app = Route::new() .at("/generate", post(generate)) + .with(AddData::new(validation)) .with(AddData::new(infer)); - Server::new(listener) - .run(app) - .await -} \ No newline at end of file + Server::new(listener).run(app).await +} diff --git a/router/src/validation.rs b/router/src/validation.rs new file mode 100644 index 00000000..6987894d --- /dev/null +++ b/router/src/validation.rs @@ -0,0 +1,65 @@ +use crate::server::GenerateRequest; +use tokenizers::tokenizer::Tokenizer; +use tokio::sync::{mpsc, oneshot}; + +#[derive(Debug)] +pub struct ValidationError {} + +type ValidationRequest = ( + GenerateRequest, + oneshot::Sender>, +); + +#[derive(Debug, Clone)] +pub(crate) struct Validation { + sender: mpsc::Sender, +} + +impl Validation { + pub(crate) fn new(tokenizer: Tokenizer) -> Self { + let (validation_sender, validation_receiver) = mpsc::channel(128); + + tokio::spawn(validation_task(tokenizer, validation_receiver)); + + Self { + sender: validation_sender, + } + } + + pub(crate) async fn validate( + &self, + request: GenerateRequest, + ) -> Result<(usize, GenerateRequest), ValidationError> { + let (sender, receiver) = oneshot::channel(); + self.sender.send((request, sender)).await.unwrap(); + receiver.await.unwrap() + } +} + +async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver) { + while let Some((request, response_tx)) = receiver.recv().await { + if request.parameters.temperature < 0.0 { + response_tx.send(Err(ValidationError {})).unwrap_or(()); + continue; + } + if request.parameters.top_p <= 0.0 || request.parameters.top_p > 1.0 { + response_tx.send(Err(ValidationError {})).unwrap_or(()); + continue; + } + if request.parameters.max_new_tokens > 512 { + response_tx.send(Err(ValidationError {})).unwrap_or(()); + continue; + } + + let inputs = tokenizer.encode(request.inputs.clone(), false).unwrap(); + let input_length = inputs.len(); + + if input_length > 512 { + response_tx.send(Err(ValidationError {})).unwrap_or(()); + continue; + } + + response_tx.send(Ok((input_length, request))).unwrap_or(()); + } + println!("drop here"); +} diff --git a/server/bloom_inference/cache.py b/server/bloom_inference/cache.py index 5ef1f1da..6812b306 100644 --- a/server/bloom_inference/cache.py +++ b/server/bloom_inference/cache.py @@ -1,44 +1,19 @@ -import torch - -from dataclasses import dataclass -from typing import Dict, Optional, List - -from bloom_inference.pb import generate_pb2 -from bloom_inference.utils import NextTokenChooser, StoppingCriteria - - -@dataclass -class CacheEntry: - batch_id: int - request_ids: List[int] - input_ids: Dict[str, torch.Tensor] - all_input_ids: List[torch.Tensor] - next_token_choosers: List[NextTokenChooser] - stopping_criterias: List[StoppingCriteria] - - def __len__(self): - return len(self.request_ids) - - def to_pb(self): - return generate_pb2.CacheEntry( - id=self.batch_id, - request_ids=self.request_ids, - sequence_length=max(len(entry) for entry in self.all_input_ids), - ) +from bloom_inference.model import Batch +from typing import Dict, Optional class Cache: def __init__(self): - self.cache: Dict[str, CacheEntry] = {} + self.cache: Dict[int, Batch] = {} - def pop(self, batch_id: str) -> Optional[CacheEntry]: + def pop(self, batch_id: int) -> Optional[Batch]: return self.cache.pop(batch_id, None) - def set(self, entry: CacheEntry): + def set(self, entry: Batch): if entry is not None: self.cache[entry.batch_id] = entry - def delete(self, batch_id: str): + def delete(self, batch_id: int): del self.cache[batch_id] def clear(self): diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py index a762a792..21cf1154 100644 --- a/server/bloom_inference/model.py +++ b/server/bloom_inference/model.py @@ -8,7 +8,6 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig from transformers.modeling_utils import no_init_weights -from bloom_inference.cache import CacheEntry from bloom_inference.pb import generate_pb2 from bloom_inference.shard_model import shard_model, match_suffix from bloom_inference.utils import ( @@ -24,25 +23,35 @@ @dataclass class Batch: batch_id: int - request_ids: List[int] + requests: List[generate_pb2.Request] input_ids: Dict[str, torch.Tensor] all_input_ids: List[torch.Tensor] next_token_choosers: List[NextTokenChooser] stopping_criterias: List[StoppingCriteria] + size: int + max_sequence_length: int + + def to_pb(self): + return generate_pb2.Batch( + id=self.batch_id, + requests=self.requests, + size=self.size, + max_sequence_length=self.max_sequence_length, + ) @classmethod - def from_batch_pb( - cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + def from_pb( + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device ) -> "Batch": - request_ids = [] inputs = [] next_token_choosers = [] stopping_criterias = [] + input_lengths = [] # Parse batch for r in pb.requests: - request_ids.append(r.id) inputs.append(r.inputs) + input_lengths.append(r.input_length) next_token_choosers.append( NextTokenChooser( temperature=r.parameters.temperature, @@ -54,94 +63,93 @@ def from_batch_pb( stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) input_ids = tokenizer(inputs, return_tensors="pt", padding=True).to(device) - all_input_ids = input_ids["input_ids"].unsqueeze(-1) + # Remove padding from all_input_ids + all_input_ids = [ + input_ids.squeeze(0)[-length:].unsqueeze(-1) + for length, input_ids in zip( + input_lengths, input_ids["input_ids"].split(1, dim=0) + ) + ] return cls( - pb.id, - request_ids, - input_ids, - all_input_ids, - next_token_choosers, - stopping_criterias, + batch_id=pb.id, + requests=pb.requests, + input_ids=input_ids, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=pb.size, + max_sequence_length=pb.max_sequence_length, ) @classmethod - def from_cache_entry(cls, cache_entry: CacheEntry) -> "Batch": - return cls( - cache_entry.batch_id, - cache_entry.request_ids, - cache_entry.input_ids, - cache_entry.all_input_ids, - cache_entry.next_token_choosers, - cache_entry.stopping_criterias, - ) + def concatenate(cls, batches: List["Batch"]) -> "Batch": + # Used for padding + total_batch_size = sum(batch.size for batch in batches) + max_sequence_length = max(batch.max_sequence_length for batch in batches) - @classmethod - def from_batch_cached_pb(cls, pb: generate_pb2.BatchCached, cache) -> "Batch": - if len(pb.batch_cached_ids) == 1: - cache_entry = cache.pop(pb.batch_cached_ids[0]) - if cache_entry is None: - raise ValueError(f"Batch ID {pb.batch_id} not found in cache") - return cls.from_cache_entry(cache_entry) - - total_batch_size = pb.total_batch_size - max_sequence_length = pb.max_sequence_length + # Batch attributes input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} - request_ids = [] + requests = [] all_input_ids = [] next_token_choosers = [] stopping_criterias = [] + + # Used for slicing correctly inside the tensors + # Equivalent to a cumsum on batch sizes start_index = 0 - for i, batch_id in enumerate(pb.batch_cached_ids): - cache_entry = cache.pop(batch_id) - if cache_entry is None: - raise ValueError(f"Batch ID {batch_id} not found in cache") - request_ids.extend(cache_entry.request_ids) - all_input_ids.extend(cache_entry.all_input_ids) - next_token_choosers.extend(cache_entry.next_token_choosers) - stopping_criterias.extend(cache_entry.stopping_criterias) - - batch_size = len(cache_entry.request_ids) - end_index = start_index + batch_size - sequence_length = max(len(entry) for entry in cache_entry.all_input_ids) - - if input_ids["input_ids"] is None: + for i, batch in enumerate(batches): + requests.extend(batch.requests) + all_input_ids.extend(batch.all_input_ids) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + + # Slicing end index for this batch + end_index = start_index + batch.size + + # We only concatenate batches that did at least one step + if batch.input_ids["input_ids"].shape[1] > 1: + raise ValueError("Batch input_ids should be of shape (batch_size, 1)") + + # Initialize tensors + if i == 0: input_ids["input_ids"] = torch.empty( (total_batch_size, 1), - dtype=cache_entry.input_ids["input_ids"].dtype, - device=cache_entry.input_ids["input_ids"].device, + dtype=batch.input_ids["input_ids"].dtype, + device=batch.input_ids["input_ids"].device, ) - - input_ids["input_ids"][start_index:end_index] = cache_entry.input_ids[ - "input_ids" - ] - - if input_ids["attention_mask"] is None: input_ids["attention_mask"] = torch.zeros( (total_batch_size, max_sequence_length), - dtype=cache_entry.input_ids["attention_mask"].dtype, - device=cache_entry.input_ids["attention_mask"].device, + dtype=batch.input_ids["attention_mask"].dtype, + device=batch.input_ids["attention_mask"].device, ) + # input_ids["input_ids"] is always of shape [batch_size, 1] + # We do not need to pad it + input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] + + # We need to slice the attention mask to remove padding from previous steps input_ids["attention_mask"][ - start_index:end_index, -sequence_length: - ] = cache_entry.input_ids["attention_mask"][:, -sequence_length:] + start_index:end_index, -batch.max_sequence_length : + ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length :] - for j, past in enumerate(cache_entry.input_ids["past_key_values"]): - # TODO: this could be done without the views by using indices + for j, past in enumerate(batch.input_ids["past_key_values"]): past_keys = past[0] past_values = past[1] _, head_dim, padded_sequence_length = past_keys.shape + # Reshape the tensors to make slicing easier past_keys = past_keys.view( - batch_size, -1, head_dim, padded_sequence_length + batch.size, -1, head_dim, padded_sequence_length ) past_values = past_values.view( - batch_size, -1, padded_sequence_length, head_dim + batch.size, -1, padded_sequence_length, head_dim ) num_heads = past_keys.shape[1] + # Initialize tensors + # This will run only once per layer if j == len(input_ids["past_key_values"]): padded_past_keys = torch.zeros( ( @@ -167,15 +175,17 @@ def from_batch_cached_pb(cls, pb: generate_pb2.BatchCached, cache) -> "Batch": [padded_past_keys, padded_past_values] ) + # We slice the past keys and values to remove the padding from previous batches input_ids["past_key_values"][j][0][ - start_index:end_index, :, :, -(sequence_length - 1): - ] = past_keys[:, :, :, -(sequence_length - 1):] + start_index:end_index, :, :, -(batch.max_sequence_length - 1) : + ] = past_keys[:, :, :, -(batch.max_sequence_length - 1) :] input_ids["past_key_values"][j][1][ - start_index:end_index, :, -(sequence_length - 1):, : - ] = past_values[:, :, -(sequence_length - 1):, :] + start_index:end_index, :, -(batch.max_sequence_length - 1) :, : + ] = past_values[:, :, -(batch.max_sequence_length - 1) :, :] - if (i + 1) == len(pb.batch_cached_ids): + # If we are on the last batch, we need to reshape the tensors + if (i + 1) == len(batches): input_ids["past_key_values"][j][0] = input_ids["past_key_values"][ j ][0].view(total_batch_size * num_heads, head_dim, -1) @@ -183,27 +193,27 @@ def from_batch_cached_pb(cls, pb: generate_pb2.BatchCached, cache) -> "Batch": j ][1].view(total_batch_size * num_heads, -1, head_dim) - start_index += batch_size - - assert pb.request_ids == request_ids + start_index += batch.size return cls( - pb.id, - request_ids, - input_ids, - all_input_ids, - next_token_choosers, - stopping_criterias, + batch_id=batches[0].batch_id, + requests=requests, + input_ids=input_ids, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=total_batch_size, + max_sequence_length=max_sequence_length, ) @dataclass -class FinishedGeneration: - request_id: str +class GeneratedText: + request: generate_pb2.Request output: str - def to_pb(self) -> generate_pb2.FinishedGeneration: - return generate_pb2.FinishedGeneration(id=self.request_id, output=self.output) + def to_pb(self) -> generate_pb2.GeneratedText: + return generate_pb2.GeneratedText(request=self.request, output=self.output) class BLOOM: @@ -229,25 +239,28 @@ def forward(self, input_ids, attention_mask, past_key_values: Optional = None): ) def generate_token( - self, batch: Batch - ) -> Tuple[List[FinishedGeneration], Optional[CacheEntry]]: + self, batch: Batch + ) -> Tuple[List[GeneratedText], Optional[Batch]]: with torch.no_grad(): outputs = self.forward(**batch.input_ids) # List of indices to cache - cache_indices = [] - cache_past_indices = [] + next_batch_keep_indices = [] + next_batch_past_keep_indices = [] + + # New input_ids for next forward + next_batch_input_ids = [] + next_batch_all_input_ids = [] - # New input_ids for next forward; keep in cache - cache_next_input_ids = [] - cache_all_input_ids = [] + next_batch_size = 0 + next_batch_max_sequence_length = 0 # Finished requests - finished_generations: List[FinishedGeneration] = [] + generated_texts: List[GeneratedText] = [] # Zipped iterator iterator = zip( - batch.request_ids, + batch.requests, outputs.logits, batch.next_token_choosers, batch.stopping_criterias, @@ -256,11 +269,11 @@ def generate_token( # For each member of the batch for i, ( - request_id, - logits, - next_token_chooser, - stopping_criteria, - all_tokens, + request, + logits, + next_token_chooser, + stopping_criteria, + all_tokens, ) in enumerate(iterator): # Select next token next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) @@ -274,64 +287,75 @@ def generate_token( output = self.tokenizer.decode( all_tokens.squeeze(-1), skip_special_tokens=True ) - # Add to the list of finished generations with the original request id - finished_generations.append(FinishedGeneration(request_id, output)) - # must be added to the cache + # Add to the list of finished generations with the original request + generated_texts.append(GeneratedText(request, output)) + # add to the next batch else: - cache_indices.append(i) - cache_past_indices.extend([j for j in range(i * self.num_heads, (i + 1) * self.num_heads)]) - cache_next_input_ids.append(next_token) - cache_all_input_ids.append(all_tokens) + next_batch_keep_indices.append(i) + # past_key_values is of shape [batch_size * num_heads, ...] + # so we need to take into account the `num_heads` stride here + next_batch_past_keep_indices.extend( + [j for j in range(i * self.num_heads, (i + 1) * self.num_heads)] + ) + next_batch_input_ids.append(next_token) + next_batch_all_input_ids.append(all_tokens) + next_batch_size += 1 + next_batch_max_sequence_length = max( + next_batch_max_sequence_length, len(all_tokens) + ) - # No cache is needed, we finished all generations in the batch - if not cache_indices: - return finished_generations, None + # We finished all generations in the batch; there is no next batch + if not next_batch_keep_indices: + return generated_texts, None # If we finished at least one generation - cache_input_ids = {"input_ids": torch.cat(cache_next_input_ids, dim=0)} - if finished_generations: + next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} + if generated_texts: # Apply indices to attention mask, past key values and other items that need to be cached - cache_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ - cache_indices + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ + next_batch_keep_indices ] - cache_input_ids["past_key_values"] = [ - (keys[cache_past_indices], values[cache_past_indices]) + next_batch_input_ids["past_key_values"] = [ + ( + keys[next_batch_past_keep_indices], + values[next_batch_past_keep_indices], + ) for keys, values in outputs["past_key_values"] ] - cache_request_ids = [batch.request_ids[i] for i in cache_indices] - cache_next_token_choosers = [ - batch.next_token_choosers[i] for i in cache_indices + next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] + next_batch_next_token_choosers = [ + batch.next_token_choosers[i] for i in next_batch_keep_indices ] - cache_stopping_criterias = [ - batch.stopping_criterias[i] for i in cache_indices + next_batch_stopping_criterias = [ + batch.stopping_criterias[i] for i in next_batch_keep_indices ] else: - cache_input_ids["attention_mask"] = batch.input_ids["attention_mask"] - cache_input_ids["past_key_values"] = outputs["past_key_values"] - cache_request_ids = batch.request_ids - cache_next_token_choosers = batch.next_token_choosers - cache_stopping_criterias = batch.stopping_criterias + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] + next_batch_input_ids["past_key_values"] = outputs["past_key_values"] + next_batch_requests = batch.requests + next_batch_next_token_choosers = batch.next_token_choosers + next_batch_stopping_criterias = batch.stopping_criterias # Update attention_mask with padding as we added a new token to input_ids - cache_input_ids["attention_mask"] = torch.cat( + next_batch_input_ids["attention_mask"] = torch.cat( [ - cache_input_ids["attention_mask"], - torch.ones((cache_input_ids["attention_mask"].shape[0], 1)).to( - cache_input_ids["attention_mask"].device - ), + next_batch_input_ids["attention_mask"], + torch.ones((next_batch_size, 1)).to(self.device), ], dim=1, ) - cache_entry = CacheEntry( - batch.batch_id, - cache_request_ids, - cache_input_ids, - cache_all_input_ids, - cache_next_token_choosers, - cache_stopping_criterias, + next_batch = Batch( + batch_id=batch.batch_id, + requests=next_batch_requests, + input_ids=next_batch_input_ids, + all_input_ids=next_batch_all_input_ids, + next_token_choosers=next_batch_next_token_choosers, + stopping_criterias=next_batch_stopping_criterias, + size=next_batch_size, + max_sequence_length=next_batch_max_sequence_length, ) - return finished_generations, cache_entry + return generated_texts, next_batch class BLOOMSharded(BLOOM): diff --git a/server/bloom_inference/server.py b/server/bloom_inference/server.py index c652468e..3a509169 100644 --- a/server/bloom_inference/server.py +++ b/server/bloom_inference/server.py @@ -10,7 +10,7 @@ from bloom_inference.pb import generate_pb2_grpc, generate_pb2 -class TextGeneration(generate_pb2_grpc.TextGenerationServicer): +class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): def __init__(self, model: BLOOM, cache: Cache, server_urls: List[str]): self.cache = cache self.model = model @@ -21,32 +21,90 @@ async def ServiceDiscovery(self, request, context): async def ClearCache(self, request, context): self.cache.clear() - return generate_pb2.Empty() + return generate_pb2.ClearCacheResponse() async def Generate(self, request, context): - batch = Batch.from_batch_pb(request, self.model.tokenizer, self.model.device) - finished_generations, cache_entry = self.model.generate_token(batch) - self.cache.set(cache_entry) - - return generate_pb2.Response( - finished=[ - finished_generation.to_pb() - for finished_generation in finished_generations + batch = Batch.from_pb(request.batch, self.model.tokenizer, self.model.device) + + generated_texts, next_batch = self.model.generate_token(batch) + self.cache.set(next_batch) + + return generate_pb2.GenerateResponse( + generated_texts=[ + generated_text.to_pb() for generated_text in generated_texts ], - cache_entry=cache_entry.to_pb() if cache_entry else None, + batch=next_batch.to_pb() if next_batch else None, ) async def GenerateWithCache(self, request, context): - batch = Batch.from_batch_cached_pb(request, self.cache) - finished_generations, cache_entry = self.model.generate_token(batch) - self.cache.set(cache_entry) - - return generate_pb2.Response( - finished=[ - finished_generation.to_pb() - for finished_generation in finished_generations + if len(request.batches) == 0: + raise ValueError("Must provide at least one batch") + + batches = [] + for batch_pb in request.batches: + batch = self.cache.pop(batch_pb.id) + if batch is None: + raise ValueError(f"Batch ID {batch_pb.id} not found in cache.") + batches.append(batch) + + if len(batches) > 1: + batch = Batch.concatenate(batches) + else: + batch = batches[0] + + generated_texts, next_batch = self.model.generate_token(batch) + self.cache.set(next_batch) + + return generate_pb2.GenerateWithCacheResponse( + generated_texts=[ + generated_text.to_pb() for generated_text in generated_texts + ], + batch=next_batch.to_pb() if next_batch else None, + ) + + async def GenerateUntilFinished(self, request, context): + batch = Batch.from_pb(request.batch, self.model.tokenizer, self.model.device) + + generated_texts = [] + while not generated_texts: + generated_texts, next_batch = self.model.generate_token(batch) + batch = next_batch + self.cache.set(next_batch) + + return generate_pb2.GenerateUntilFinishedResponse( + generated_texts=[ + generated_text.to_pb() for generated_text in generated_texts + ], + batch=next_batch.to_pb() if next_batch else None, + ) + + async def GenerateUntilFinishedWithCache(self, request, context): + if len(request.batches) == 0: + raise ValueError("Must provide at least one batch") + + batches = [] + for batch_pb in request.batches: + batch = self.cache.pop(batch_pb.id) + if batch is None: + raise ValueError(f"Batch ID {batch_pb.id} not found in cache.") + batches.append(batch) + + if len(batches) > 1: + batch = Batch.concatenate(batches) + else: + batch = batches[0] + + generated_texts = [] + while not generated_texts: + generated_texts, next_batch = self.model.generate_token(batch) + batch = next_batch + self.cache.set(next_batch) + + return generate_pb2.GenerateUntilFinishedWithCacheResponse( + generated_texts=[ + generated_text.to_pb() for generated_text in generated_texts ], - cache_entry=cache_entry.to_pb() if cache_entry else None, + batch=next_batch.to_pb() if next_batch else None, ) @@ -71,11 +129,11 @@ async def serve_inner( server_urls = [local_url] server = aio.server() - generate_pb2_grpc.add_TextGenerationServicer_to_server( - TextGeneration(model, Cache(), server_urls), server + generate_pb2_grpc.add_TextGenerationServiceServicer_to_server( + TextGenerationService(model, Cache(), server_urls), server ) SERVICE_NAMES = ( - generate_pb2.DESCRIPTOR.services_by_name["TextGeneration"].full_name, + generate_pb2.DESCRIPTOR.services_by_name["TextGenerationService"].full_name, reflection.SERVICE_NAME, ) reflection.enable_server_reflection(SERVICE_NAMES, server) From e86ecbac63ab8cec773e07549a286a77edaac1d4 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Tue, 11 Oct 2022 16:53:40 +0200 Subject: [PATCH 005/793] ValidationError was not correctly handled --- router/src/server.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/router/src/server.rs b/router/src/server.rs index 14e81709..64b37ff8 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -69,13 +69,15 @@ async fn generate( ) -> poem::Result> { let start = Instant::now(); - let (input_length, validated_request) = validation + let (input_length, validated_request) = match validation .validate(GenerateRequest { inputs: req.inputs.clone(), parameters: req.parameters.clone(), }) - .await - .unwrap(); + .await { + Ok(result) => result, + Err(_) => return Err(poem::Error::from_status(StatusCode::INTERNAL_SERVER_ERROR)) + }; let output = infer.infer(input_length, validated_request).await; From 39df4d99754b4fdf5a30f0f17711ef3db17c9117 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Tue, 11 Oct 2022 18:14:39 +0200 Subject: [PATCH 006/793] Use axum --- router/Cargo.lock | 202 ++++++++++++++++----------------------- router/Cargo.toml | 4 +- router/client/Cargo.toml | 6 +- router/src/main.rs | 8 +- router/src/server.rs | 49 ++++++---- 5 files changed, 121 insertions(+), 148 deletions(-) diff --git a/router/Cargo.lock b/router/Cargo.lock index 79761345..1f00df14 100644 --- a/router/Cargo.lock +++ b/router/Cargo.lock @@ -81,6 +81,53 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e3356844c4d6a6d6467b8da2cffb4a2820be256f50a3a386c9d152bab31043" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f0c0a60006f2a293d82d571f635042a72edf927539b7685bd62d361963839b" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "tower-layer", + "tower-service", +] + [[package]] name = "base64" version = "0.13.0" @@ -106,10 +153,10 @@ dependencies = [ name = "bloom-inference" version = "0.1.0" dependencies = [ + "axum", "bloom-inference-client", "futures", "parking_lot", - "poem", "serde", "serde_json", "tokenizers", @@ -661,31 +708,6 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -[[package]] -name = "headers" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" -dependencies = [ - "base64", - "bitflags", - "bytes", - "headers-core", - "http", - "httpdate", - "mime", - "sha1", -] - -[[package]] -name = "headers-core" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" -dependencies = [ - "http", -] - [[package]] name = "heck" version = "0.3.3" @@ -726,6 +748,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + [[package]] name = "httparse" version = "1.8.0" @@ -941,6 +969,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26a8d2502d5aa4d411ef494ba7470eb299f05725179ce3b5de77aa01a9ffdea" +[[package]] +name = "matchit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" + [[package]] name = "memchr" version = "2.5.0" @@ -1201,65 +1235,12 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" -[[package]] -name = "poem" -version = "1.3.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2992ba72908e36200671c0f3a692992ced894b3b2bbe2b2dc6dfbffea6e2c85a" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "headers", - "http", - "hyper", - "mime", - "parking_lot", - "percent-encoding", - "pin-project-lite", - "poem-derive", - "regex", - "rfc7239", - "serde", - "serde_json", - "serde_urlencoded", - "smallvec", - "thiserror", - "tokio", - "tokio-stream", - "tokio-util 0.7.4", - "tracing", -] - -[[package]] -name = "poem-derive" -version = "1.3.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f535d4331a22610b98ca48f98bae9bda0c654da89b9ae10a1830fa9edfd8f36" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "ppv-lite86" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" -[[package]] -name = "proc-macro-crate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" -dependencies = [ - "once_cell", - "thiserror", - "toml", -] - [[package]] name = "proc-macro2" version = "1.0.46" @@ -1479,15 +1460,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "rfc7239" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "087317b3cf7eb481f13bd9025d729324b7cd068d6f470e2d76d049e191f5ba47" -dependencies = [ - "uncased", -] - [[package]] name = "ryu" version = "1.0.11" @@ -1576,17 +1548,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha1" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" version = "0.10.6" @@ -1667,6 +1628,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + [[package]] name = "tar" version = "0.4.38" @@ -1890,15 +1857,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "toml" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" -dependencies = [ - "serde", -] - [[package]] name = "tonic" version = "0.6.2" @@ -1962,6 +1920,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.1" @@ -2065,15 +2042,6 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" -[[package]] -name = "uncased" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622" -dependencies = [ - "version_check", -] - [[package]] name = "unicode-bidi" version = "0.3.8" diff --git a/router/Cargo.toml b/router/Cargo.toml index 901a14f2..c5e5bb89 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -3,13 +3,11 @@ name = "bloom-inference" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] +axum = { version = "0.5.16", features = ["json", "serde_json"] } bloom-inference-client = { path = "client" } futures = "0.3.24" parking_lot = "0.12.1" -poem = "1.3.45" serde = "1.0.145" serde_json = "1.0.85" tokenizers = "0.13.0" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index e4f2fa0f..7760c8cb 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -4,12 +4,12 @@ version = "0.1.0" edition = "2021" [dependencies] -futures = "0.3.24" +futures = "^0.3" #grpc-error-details = { path = "../../grpc-error-details" } #grpc-metadata = { path = "../../grpc-metadata" } prost = "^0.9" -thiserror = "1.0.37" -tokio = { version = "1.21.2", features = ["sync"] } +thiserror = "^1.0" +tokio = { version = "^1.21", features = ["sync"] } tonic = "^0.6" tower = "^0.4" tracing = "^0.1" diff --git a/router/src/main.rs b/router/src/main.rs index fe82d059..803753b1 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,5 +1,5 @@ +use std::net::SocketAddr; use bloom_inference_client::ShardedClient; -use poem::listener::TcpListener; use std::time::Duration; use tokenizers::Tokenizer; @@ -37,9 +37,9 @@ fn main() -> Result<(), std::io::Error> { .expect("Unable to clear cache"); tracing::info!("Connected"); - let addr = "127.0.0.1:3000".to_string(); - let listener = TcpListener::bind(addr); + let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); - server::run(sharded_client, tokenizer, listener).await + server::run(sharded_client, tokenizer, addr).await; + Ok(()) }) } diff --git a/router/src/server.rs b/router/src/server.rs index 64b37ff8..61c57069 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,9 +1,9 @@ +use std::net::SocketAddr; +use axum::{Router, Json}; +use axum::http::StatusCode; +use axum::extract::Extension; +use axum::routing::post; use crate::{Batcher, ShardedClient, Validation}; -use poem::http::StatusCode; -use poem::listener::TcpListener; -use poem::middleware::AddData; -use poem::web::{Data, Json}; -use poem::{handler, post, EndpointExt, Route, Server}; use serde::Deserialize; use tokenizers::Tokenizer; use tokio::time::Instant; @@ -60,26 +60,24 @@ pub(crate) struct GenerateRequest { pub parameters: GenerateParameters, } -#[handler] -#[instrument(skip(validation, infer), fields(time, time_per_token))] +#[instrument(skip(state), fields(time, time_per_token))] async fn generate( - validation: Data<&Validation>, - infer: Data<&Batcher>, + state: Extension, req: Json, -) -> poem::Result> { +) -> Result, StatusCode> { let start = Instant::now(); - let (input_length, validated_request) = match validation + let (input_length, validated_request) = match state.validation .validate(GenerateRequest { inputs: req.inputs.clone(), parameters: req.parameters.clone(), }) .await { Ok(result) => result, - Err(_) => return Err(poem::Error::from_status(StatusCode::INTERNAL_SERVER_ERROR)) + Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR) }; - let output = infer.infer(input_length, validated_request).await; + let output = state.infer.infer(input_length, validated_request).await; match output { Ok(generated_text) => { @@ -94,15 +92,21 @@ async fn generate( "generated_text": generated_text, }))) } - Err(_) => Err(poem::Error::from_status(StatusCode::INTERNAL_SERVER_ERROR)), + Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR), } } +#[derive(Clone)] +struct ServerState { + validation: Validation, + infer: Batcher, +} + pub async fn run( client: ShardedClient, tokenizer: Tokenizer, - listener: TcpListener, -) -> Result<(), std::io::Error> { + addr: SocketAddr, +) { client.clear_cache().await.expect("Unable to clear cache"); tracing::info!("Connected"); @@ -110,10 +114,13 @@ pub async fn run( let validation = Validation::new(tokenizer); - let app = Route::new() - .at("/generate", post(generate)) - .with(AddData::new(validation)) - .with(AddData::new(infer)); + let shared_state = ServerState { + validation, + infer, + }; + + let app = Router::new().route("/generate", post(generate)).layer(Extension(shared_state)); - Server::new(listener).run(app).await + axum::Server::bind(&addr) + .serve(app.into_make_service()).await.unwrap(); } From bf99afe9160f97124ba2ec28afaa97e4364846a8 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Fri, 14 Oct 2022 15:56:21 +0200 Subject: [PATCH 007/793] feat: Docker image --- .dockerignore | 1 + Dockerfile | 59 ++++++++ README.md | 2 - router/rust-toolchain.toml | 3 + router/src/batcher.rs | 7 +- router/src/main.rs | 4 +- router/src/server.rs | 62 +++++--- run.sh | 21 +++ server/bloom_inference/model.py | 7 +- server/bloom_inference/prepare_weights.py | 164 +++++++++++++++------- server/bloom_inference/shard_model.py | 102 -------------- server/poetry.lock | 14 +- server/pyproject.toml | 1 + 13 files changed, 266 insertions(+), 181 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 router/rust-toolchain.toml create mode 100755 run.sh delete mode 100644 server/bloom_inference/shard_model.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..d2704bf1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +router/target \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a5161020 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,59 @@ +FROM rust:1.64 as builder + +WORKDIR /usr/src + +COPY proto proto +COPY router router + +WORKDIR /usr/src/router + +RUN cargo install --path . + +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 + +ENV LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + DEBIAN_FRONTEND=noninteractive \ + MODEL_BASE_PATH=/var/azureml-model \ + MODEL_NAME=bigscience/bloom \ + NUM_GPUS=8 \ + CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ + CUDA_HOME=/usr/local/cuda \ + LD_LIBRARY_PATH="/opt/miniconda/envs/text-generation/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH" \ + CONDA_DEFAULT_ENV=text-generation \ + PATH=$PATH:/opt/miniconda/envs/text-generation/bin:/opt/miniconda/bin:/usr/local/cuda/bin + +SHELL ["/bin/bash", "-c"] + +RUN apt-get update && apt-get install -y unzip wget libssl-dev && rm -rf /var/lib/apt/lists/* + +RUN cd ~ && \ + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + chmod +x Miniconda3-latest-Linux-x86_64.sh && \ + bash ./Miniconda3-latest-Linux-x86_64.sh -bf -p /opt/miniconda && \ + conda create -n text-generation python=3.9 -y + +# Install specific version of torch +RUN /opt/miniconda/envs/text-generation/bin/pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir + +# Install specific version of transformers +RUN wget https://github.com/huggingface/transformers/archive/46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip && \ + unzip 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip && \ + rm 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip && \ + cd transformers-46d37bece7d3ffdef97b1ee4a3170c0a0627d921 && \ + /opt/miniconda/envs/text-generation/bin/python setup.py install + +WORKDIR /usr/src + +# Install server +COPY server server +RUN cd server && \ + /opt/miniconda/envs/text-generation/bin/pip install . --no-cache-dir + +# Install router +COPY --from=builder /usr/local/cargo/bin/bloom-inference /usr/local/bin/bloom-inference + +COPY run.sh . +RUN chmod +x run.sh + +CMD ["./run.sh"] \ No newline at end of file diff --git a/README.md b/README.md index ad8adaa9..a62daca1 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,6 @@ python server/bloom_inference/main.py bigscience/bloom --num-gpus 8 --shard-dire ## TODO: -- [ ] Improve model download - - Store "shardable" layers separately and layer by layer - [ ] Add batching args to router CLI - [ ] Add docstrings + comments everywhere as the codebase is fairly complicated - [ ] Add tests diff --git a/router/rust-toolchain.toml b/router/rust-toolchain.toml new file mode 100644 index 00000000..30355814 --- /dev/null +++ b/router/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.64.0" +components = ["rustfmt", "clippy"] \ No newline at end of file diff --git a/router/src/batcher.rs b/router/src/batcher.rs index a044e26c..2025cf62 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -83,7 +83,12 @@ async fn batching_task(client: ShardedClient, db: Db, shared: Arc) { cached_batch = match batch_size { size if size > 16 => { - wrap_future(client.generate_until_finished_with_cache(batches), request_ids, &db).await + wrap_future( + client.generate_until_finished_with_cache(batches), + request_ids, + &db, + ) + .await } _ => wrap_future(client.generate_with_cache(batches), request_ids, &db).await, }; diff --git a/router/src/main.rs b/router/src/main.rs index 803753b1..2fe02944 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,5 +1,5 @@ -use std::net::SocketAddr; use bloom_inference_client::ShardedClient; +use std::net::SocketAddr; use std::time::Duration; use tokenizers::Tokenizer; @@ -37,7 +37,7 @@ fn main() -> Result<(), std::io::Error> { .expect("Unable to clear cache"); tracing::info!("Connected"); - let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); + let addr = SocketAddr::from(([0, 0, 0, 0], 3000)); server::run(sharded_client, tokenizer, addr).await; Ok(()) diff --git a/router/src/server.rs b/router/src/server.rs index 61c57069..1cbec333 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,10 +1,10 @@ -use std::net::SocketAddr; -use axum::{Router, Json}; -use axum::http::StatusCode; +use crate::{Batcher, ShardedClient, Validation}; use axum::extract::Extension; +use axum::http::StatusCode; use axum::routing::post; -use crate::{Batcher, ShardedClient, Validation}; +use axum::{Json, Router}; use serde::Deserialize; +use std::net::SocketAddr; use tokenizers::Tokenizer; use tokio::time::Instant; use tracing::instrument; @@ -60,6 +60,31 @@ pub(crate) struct GenerateRequest { pub parameters: GenerateParameters, } +#[instrument(skip(state), fields(time, time_per_token))] +async fn liveness(state: Extension) -> Result<(), StatusCode> { + let output = state + .infer + .infer( + 1, + GenerateRequest { + inputs: "liveness".to_string(), + parameters: GenerateParameters { + temperature: 1.0, + top_k: 0, + top_p: 1.0, + do_sample: false, + max_new_tokens: 1, + }, + }, + ) + .await; + + match output { + Ok(_) => Ok(()), + Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR), + } +} + #[instrument(skip(state), fields(time, time_per_token))] async fn generate( state: Extension, @@ -67,14 +92,16 @@ async fn generate( ) -> Result, StatusCode> { let start = Instant::now(); - let (input_length, validated_request) = match state.validation + let (input_length, validated_request) = match state + .validation .validate(GenerateRequest { inputs: req.inputs.clone(), parameters: req.parameters.clone(), }) - .await { + .await + { Ok(result) => result, - Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR) + Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR), }; let output = state.infer.infer(input_length, validated_request).await; @@ -102,11 +129,7 @@ struct ServerState { infer: Batcher, } -pub async fn run( - client: ShardedClient, - tokenizer: Tokenizer, - addr: SocketAddr, -) { +pub async fn run(client: ShardedClient, tokenizer: Tokenizer, addr: SocketAddr) { client.clear_cache().await.expect("Unable to clear cache"); tracing::info!("Connected"); @@ -114,13 +137,16 @@ pub async fn run( let validation = Validation::new(tokenizer); - let shared_state = ServerState { - validation, - infer, - }; + let shared_state = ServerState { validation, infer }; - let app = Router::new().route("/generate", post(generate)).layer(Extension(shared_state)); + let app = Router::new() + .route("/generate", post(generate)) + .layer(Extension(shared_state.clone())) + .route("/health", post(liveness)) + .layer(Extension(shared_state.clone())); axum::Server::bind(&addr) - .serve(app.into_make_service()).await.unwrap(); + .serve(app.into_make_service()) + .await + .unwrap(); } diff --git a/run.sh b/run.sh new file mode 100755 index 00000000..3b095541 --- /dev/null +++ b/run.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +server_cmd="python server/bloom_inference/main.py $MODEL_NAME --num-gpus $NUM_GPUS --shard-directory $MODEL_BASE_PATH" +$server_cmd & + +FILE=/tmp/bloom-inference-0 + +while : + do + if test -S "$FILE"; then + echo "Text Generation Python gRPC server started" + break + else + echo "Waiting for Text Generation Python gRPC server to start" + sleep 5 + fi + done + +sleep 1 + +exec "bloom-inference" diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py index 21cf1154..40d69e8b 100644 --- a/server/bloom_inference/model.py +++ b/server/bloom_inference/model.py @@ -220,12 +220,14 @@ class BLOOM: def __init__(self, model_name: str): if torch.cuda.is_available(): self.device = torch.device("cuda") + dtype = torch.bfloat16 else: self.device = torch.device("cpu") + dtype = torch.float32 self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") self.model = ( - AutoModelForCausalLM.from_pretrained(model_name).eval().to(self.device) + AutoModelForCausalLM.from_pretrained(model_name).eval().to(self.device).to(dtype) ) self.num_heads = self.model.base_model.num_heads @@ -427,7 +429,8 @@ def __init__(self, model_name: str, shard_directory: Path): if do_transpose: state_dict[key] = state_dict[key].transpose(1, 0).contiguous() - model.load_state_dict(state_dict) + model.load_state_dict(state_dict, strict=False) + model.tie_weights() self.model = model.to(self.device).eval() self.num_heads = config.n_head // self.process_group.size() torch.distributed.barrier(group=self.process_group) diff --git a/server/bloom_inference/prepare_weights.py b/server/bloom_inference/prepare_weights.py index 6fd4ec50..5fa2be51 100644 --- a/server/bloom_inference/prepare_weights.py +++ b/server/bloom_inference/prepare_weights.py @@ -1,18 +1,62 @@ import torch +import os +import tempfile +import json +from typing import BinaryIO +from joblib import Parallel, delayed +from functools import partial from pathlib import Path from tqdm import tqdm -MODEL_NAME = "bigscience/bloom" +from huggingface_hub import hf_hub_url +from huggingface_hub.file_download import _request_wrapper, hf_raise_for_status def match_suffix(text, suffix): - return text[-len(suffix) :] == suffix - - -def prepare_weights(hub_path: Path, save_path: Path, tp_world_size: int): + return text[-len(suffix):] == suffix + + +def http_get( + url: str, + temp_file: BinaryIO, + *, + timeout=10.0, + max_retries=0, +): + """ + Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub. + """ + r = _request_wrapper( + method="GET", + url=url, + stream=True, + timeout=timeout, + max_retries=max_retries, + ) + hf_raise_for_status(r) + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + temp_file.write(chunk) + + +def cache_download_url(url: str, root_dir: Path): + filename = root_dir / url.split("/")[-1] + + if not filename.exists(): + temp_file_manager = partial( + tempfile.NamedTemporaryFile, mode="wb", dir=root_dir, delete=False + ) + with temp_file_manager() as temp_file: + http_get(url, temp_file) + + os.replace(temp_file.name, filename) + return filename + + +def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world_size: int): save_paths = [ - save_path / f"{MODEL_NAME}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" + save_path / f"{model_name}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" for tp_rank in range(tp_world_size) ] @@ -20,45 +64,67 @@ def prepare_weights(hub_path: Path, save_path: Path, tp_world_size: int): print("Weights are already prepared") return + cache_path.mkdir(parents=True, exist_ok=True) + if model_name == "bigscience/bloom-560m": + url = hf_hub_url(model_name, filename="pytorch_model.bin") + cache_download_url(url, cache_path) + elif model_name == "bigscience/bloom": + url = hf_hub_url(model_name, filename="pytorch_model.bin.index.json") + index_path = cache_download_url(url, cache_path) + with index_path.open("r") as f: + index = json.load(f) + + # Get unique file names + weight_files = list(set([filename for filename in index["weight_map"].values()])) + urls = [hf_hub_url(model_name, filename=filename) for filename in weight_files] + + Parallel(n_jobs=5)(delayed(cache_download_url)(url, cache_path) for url in tqdm(urls)) + else: + raise ValueError(f"Unknown model name: {model_name}") + shards_state_dicts = [{} for _ in range(tp_world_size)] - for weight_path in tqdm(hub_path.glob("*.bin")): + for weight_path in tqdm(Path(cache_path).glob("*.bin")): state_dict = torch.load(weight_path, map_location="cpu") keys = list(state_dict.keys()) for state_name in keys: state = state_dict[state_name] if any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.query_key_value.weight", - "self_attention.query_key_value.bias", - "mlp.dense_h_to_4h.weight", - "mlp.dense_h_to_4h.bias", - "word_embeddings.weight", - "lm_head.weight", - ] + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.query_key_value.weight", + "self_attention.query_key_value.bias", + "mlp.dense_h_to_4h.weight", + "mlp.dense_h_to_4h.bias", + "word_embeddings.weight", + ] ): output_size = state.shape[0] assert output_size % tp_world_size == 0 block_size = output_size // tp_world_size sharded_weights = torch.split(state, block_size, dim=0) assert len(sharded_weights) == tp_world_size + for tp_rank, shard in enumerate(sharded_weights): - assert shard.shape[0] == block_size - if match_suffix(state_name, "lm_head.weight"): - shards_state_dicts[tp_rank][state_name] = shard.detach().clone() - else: - shards_state_dicts[tp_rank][ - "transformer." + state_name - ] = shard.detach().clone() + shards_state_dicts[tp_rank]["transformer." + state_name] = shard.detach().clone() + + elif match_suffix(state_name, "lm_head.weight"): + output_size = state.shape[0] + assert output_size % tp_world_size == 0 + block_size = output_size // tp_world_size + sharded_weights = torch.split(state, block_size, dim=0) + assert len(sharded_weights) == tp_world_size + + for tp_rank, shard in enumerate(sharded_weights): + shards_state_dicts[tp_rank][state_name] = shard.detach().clone() + elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.weight", - "mlp.dense_4h_to_h.weight", - "lm_head.weight", - ] + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.weight", + "mlp.dense_4h_to_h.weight", + ] ): input_size = state.shape[1] assert input_size % tp_world_size == 0 @@ -66,40 +132,31 @@ def prepare_weights(hub_path: Path, save_path: Path, tp_world_size: int): sharded_weights = torch.split(state, block_size, dim=1) assert len(sharded_weights) == tp_world_size for tp_rank, shard in enumerate(sharded_weights): - assert shard.shape[1] == block_size - if match_suffix(state_name, "lm_head.weight"): - shards_state_dicts[tp_rank][state_name] = shard.detach().clone() - else: - shards_state_dicts[tp_rank][ - "transformer." + state_name - ] = shard.detach().clone() + shards_state_dicts[tp_rank]["transformer." + state_name] = shard.detach().clone() + elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.bias", - "mlp.dense_4h_to_h.bias", - ] + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.bias", + "mlp.dense_4h_to_h.bias", + ] ): - shards_state_dicts[0][ - "transformer." + state_name - ] = state.detach().clone() + shards_state_dicts[0]["transformer." + state_name] = state.detach().clone() for tp_rank in range(1, tp_world_size): - shards_state_dicts[tp_rank][ - "transformer." + state_name - ] = torch.zeros_like(state) + shards_state_dicts[tp_rank]["transformer." + state_name] = torch.zeros_like(state) + else: # We duplicate parameters across tp ranks for tp_rank in range(tp_world_size): - shards_state_dicts[tp_rank][ - "transformer." + state_name - ] = state.detach().clone() + shards_state_dicts[tp_rank]["transformer." + state_name] = state.detach().clone() del state_dict[state_name] # delete key from state_dict del state # delete tensor + del state_dict # we save state_dict for tp_rank, (save_path, shard_state_dict) in enumerate( - zip(save_paths, shards_state_dicts) + zip(save_paths, shards_state_dicts) ): save_paths.append(save_path) save_path.parent.mkdir(parents=True, exist_ok=True) @@ -116,9 +173,10 @@ def prepare_weights(hub_path: Path, save_path: Path, tp_world_size: int): parser = ArgumentParser() - parser.add_argument("--hub-path", required=True, type=str) + parser.add_argument("--model-name", required=True, type=str) + parser.add_argument("--cache-path", required=True, type=str) parser.add_argument("--save-path", required=True, type=str) parser.add_argument("--world-size", required=True, type=int) args = parser.parse_args() - prepare_weights(Path(args.hub_path), Path(args.save_path), args.world_size) + prepare_weights(args.model_name, Path(args.cache_path), Path(args.save_path), args.world_size) diff --git a/server/bloom_inference/shard_model.py b/server/bloom_inference/shard_model.py deleted file mode 100644 index 360e8962..00000000 --- a/server/bloom_inference/shard_model.py +++ /dev/null @@ -1,102 +0,0 @@ -from pathlib import Path - -import torch -from torch import nn -from transformers import AutoModelForCausalLM - - -def match_suffix(text, suffix): - return text[-len(suffix) :] == suffix - - -def shard_model(model_name: str, path: Path, tp_world_size: int, dtype: torch.dtype): - """BLOOM specific sharding mechanism""" - save_paths = [ - path / f"{model_name}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" - for tp_rank in range(tp_world_size) - ] - if all(save_path.exists() for save_path in save_paths): - print("Loading already cached values") - return save_paths - - model: nn.Module = AutoModelForCausalLM.from_pretrained( - model_name, torch_dtype=dtype, local_files_only=True - ) - - shards_state_dicts = [{} for _ in range(tp_world_size)] - state_dict = model.state_dict() - keys = list(state_dict.keys()) - for state_name in keys: - print(state_name) - state = state_dict[state_name] - if any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.query_key_value.weight", - "self_attention.query_key_value.bias", - "mlp.dense_h_to_4h.weight", - "mlp.dense_h_to_4h.bias", - "transformer.word_embeddings.weight", - "lm_head.weight", - ] - ): - output_size = state.shape[0] - assert output_size % tp_world_size == 0 - block_size = output_size // tp_world_size - sharded_weights = torch.split(state, block_size, dim=0) - assert len(sharded_weights) == tp_world_size - for tp_rank, shard in enumerate(sharded_weights): - assert shard.shape[0] == block_size - shards_state_dicts[tp_rank][state_name] = shard.detach().clone() - elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.weight", - "mlp.dense_4h_to_h.weight", - "lm_head.weight", - ] - ): - input_size = state.shape[1] - assert input_size % tp_world_size == 0 - block_size = input_size // tp_world_size - sharded_weights = torch.split(state, block_size, dim=1) - assert len(sharded_weights) == tp_world_size - for tp_rank, shard in enumerate(sharded_weights): - assert shard.shape[1] == block_size - shards_state_dicts[tp_rank][state_name] = shard.detach().clone() - elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.bias", - "mlp.dense_4h_to_h.bias", - ] - ): - shards_state_dicts[0][state_name] = state.detach().clone() - for tp_rank in range(1, tp_world_size): - shards_state_dicts[tp_rank][state_name] = torch.zeros_like(state) - else: - # We duplicate parameters across tp ranks - for tp_rank in range(tp_world_size): - shards_state_dicts[tp_rank][state_name] = state.detach().clone() - - del state_dict[state_name] # delete key from state_dict - del state # delete tensor - - # we save state_dict - for tp_rank, (save_path, shard_state_dict) in enumerate( - zip(save_paths, shards_state_dicts) - ): - save_path.parent.mkdir(parents=True, exist_ok=True) - torch.save(shard_state_dict, save_path) - save_paths.append(save_path) - - return save_paths - - -if __name__ == "__main__": - model_name = "bigscience/bloom" - save_path = Path("/data/shards") - tp_world_size = 8 - dtype = torch.bfloat16 - - shard_model(model_name, save_path, tp_world_size=tp_world_size, dtype=dtype) diff --git a/server/poetry.lock b/server/poetry.lock index ea20ef26..8100c200 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -80,6 +80,14 @@ grpcio = ">=1.49.1" protobuf = ">=4.21.3,<5.0dev" setuptools = "*" +[[package]] +name = "joblib" +version = "1.2.0" +description = "Lightweight pipelining with Python functions" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "numpy" version = "1.23.3" @@ -197,7 +205,7 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "cedd0aebeb3731e2bbddf017a2ee6074c285866354272f8dfe930e9606437a25" +content-hash = "f3dc5b2420183f2e7e9257e372489409d7bd26d1dcc535fc2558ebca50c988c2" [metadata.files] accelerate = [ @@ -310,6 +318,10 @@ grpcio-tools = [ {file = "grpcio_tools-1.49.1-cp39-cp39-win32.whl", hash = "sha256:704d21509ec06efc9d034dbe70e7152715aac004941f4f0f553cf3a0aff15bd5"}, {file = "grpcio_tools-1.49.1-cp39-cp39-win_amd64.whl", hash = "sha256:1efa0c221c719433f441ac0e026fc3c4dbc9a1a08a552ecdc707775e2f2fbbae"}, ] +joblib = [ + {file = "joblib-1.2.0-py3-none-any.whl", hash = "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385"}, + {file = "joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"}, +] numpy = [ {file = "numpy-1.23.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c9f707b5bb73bf277d812ded9896f9512a43edff72712f31667d0a8c2f8e71ee"}, {file = "numpy-1.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ffcf105ecdd9396e05a8e58e81faaaf34d3f9875f137c7372450baa5d77c9a54"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 9d14ce6c..80d95426 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -12,6 +12,7 @@ torch = "^1.12.1" typer = "^0.6.1" grpcio-reflection = "^1.49.1" accelerate = "^0.12.0" +joblib = "^1.2.0" [tool.poetry.group.dev.dependencies] grpcio-tools = "^1.49.1" From bcb53903b8b7d70db19e583cd24b4a1b13352487 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Sat, 15 Oct 2022 20:21:50 +0200 Subject: [PATCH 008/793] feat: Add AML deployment --- .dockerignore | 1 + aml/README.md | 7 ++++ aml/deployment.yaml | 39 +++++++++++++++++++++++ aml/endpoint.yaml | 3 ++ router/src/server.rs | 7 ++-- server/bloom_inference/model.py | 6 ++-- server/bloom_inference/prepare_weights.py | 2 +- 7 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 aml/README.md create mode 100644 aml/deployment.yaml create mode 100644 aml/endpoint.yaml diff --git a/.dockerignore b/.dockerignore index d2704bf1..38e8f824 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ +aml router/target \ No newline at end of file diff --git a/aml/README.md b/aml/README.md new file mode 100644 index 00000000..c38f9fef --- /dev/null +++ b/aml/README.md @@ -0,0 +1,7 @@ +```shell +docker build . -t db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.1 +docker push db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.1 + +az ml online-endpoint create -f endpoint.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace +az ml online-deployment create -f deployment.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace +``` \ No newline at end of file diff --git a/aml/deployment.yaml b/aml/deployment.yaml new file mode 100644 index 00000000..f6b55faa --- /dev/null +++ b/aml/deployment.yaml @@ -0,0 +1,39 @@ +$schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json +name: bloom-deployment +endpoint_name: bloom-inference +model: + name: bloom + path: ./bloom +model_mount_path: /var/azureml-model +environment_variables: + MODEL_BASE_PATH: /var/azureml-model/bloom + MODEL_NAME: bigscience/bloom + NUM_GPUS: 8 +environment: + image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.1 + inference_config: + liveness_route: + port: 3000 + path: /health + readiness_route: + port: 3000 + path: /health + scoring_route: + port: 3000 + path: /generate +instance_type: Standard_ND96amsr_A100_v4 +request_settings: + request_timeout_ms: 90000 +liveness_probe: + initial_delay: 300 + timeout: 20 + period: 60 + success_threshold: 1 + failure_threshold: 60 +readiness_probe: + initial_delay: 300 + timeout: 20 + period: 60 + success_threshold: 1 + failure_threshold: 60 +instance_count: 1 diff --git a/aml/endpoint.yaml b/aml/endpoint.yaml new file mode 100644 index 00000000..934b31ad --- /dev/null +++ b/aml/endpoint.yaml @@ -0,0 +1,3 @@ +$schema: https://azuremlsdk2.blob.core.windows.net/latest/managedOnlineEndpoint.schema.json +name: bloom-inference +auth_mode: aml_token diff --git a/router/src/server.rs b/router/src/server.rs index 1cbec333..f113ce44 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,7 +1,8 @@ -use crate::{Batcher, ShardedClient, Validation}; +use bloom_inference_client::ShardedClient; +use crate::{Batcher, Validation}; use axum::extract::Extension; use axum::http::StatusCode; -use axum::routing::post; +use axum::routing::{get, post}; use axum::{Json, Router}; use serde::Deserialize; use std::net::SocketAddr; @@ -142,7 +143,7 @@ pub async fn run(client: ShardedClient, tokenizer: Tokenizer, addr: SocketAddr) let app = Router::new() .route("/generate", post(generate)) .layer(Extension(shared_state.clone())) - .route("/health", post(liveness)) + .route("/health", get(liveness)) .layer(Extension(shared_state.clone())); axum::Server::bind(&addr) diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py index 40d69e8b..8b0e7ab0 100644 --- a/server/bloom_inference/model.py +++ b/server/bloom_inference/model.py @@ -9,7 +9,7 @@ from transformers.modeling_utils import no_init_weights from bloom_inference.pb import generate_pb2 -from bloom_inference.shard_model import shard_model, match_suffix +from bloom_inference.prepare_weights import prepare_weights, match_suffix from bloom_inference.utils import ( StoppingCriteria, NextTokenChooser, @@ -377,8 +377,8 @@ def __init__(self, model_name: str, shard_directory: Path): # shard state_dict if self.master: # TODO @thomasw21 do some caching - shard_state_dict_paths = shard_model( - model_name, shard_directory, tp_world_size=self.world_size, dtype=dtype + shard_state_dict_paths = prepare_weights( + model_name, shard_directory / "cache", shard_directory, tp_world_size=self.world_size ) shard_state_dict_paths = [ str(path.absolute()) for path in shard_state_dict_paths diff --git a/server/bloom_inference/prepare_weights.py b/server/bloom_inference/prepare_weights.py index 5fa2be51..7cf3dbb5 100644 --- a/server/bloom_inference/prepare_weights.py +++ b/server/bloom_inference/prepare_weights.py @@ -62,7 +62,7 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world if all(save_path.exists() for save_path in save_paths): print("Weights are already prepared") - return + return save_paths cache_path.mkdir(parents=True, exist_ok=True) if model_name == "bigscience/bloom-560m": From 00e6ce44b165555087265286d416cad4826c3791 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Mon, 17 Oct 2022 10:39:59 +0200 Subject: [PATCH 009/793] Update aml deployment --- aml/README.md | 1 + aml/deployment.yaml | 5 ++--- aml/endpoint.yaml | 2 +- aml/model.yaml | 5 +++++ 4 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 aml/model.yaml diff --git a/aml/README.md b/aml/README.md index c38f9fef..959e2942 100644 --- a/aml/README.md +++ b/aml/README.md @@ -2,6 +2,7 @@ docker build . -t db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.1 docker push db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.1 +az ml model create -f model.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace az ml online-endpoint create -f endpoint.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace az ml online-deployment create -f deployment.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace ``` \ No newline at end of file diff --git a/aml/deployment.yaml b/aml/deployment.yaml index f6b55faa..31cb09c5 100644 --- a/aml/deployment.yaml +++ b/aml/deployment.yaml @@ -1,9 +1,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json name: bloom-deployment endpoint_name: bloom-inference -model: - name: bloom - path: ./bloom +model: azureml:bloom:1 model_mount_path: /var/azureml-model environment_variables: MODEL_BASE_PATH: /var/azureml-model/bloom @@ -24,6 +22,7 @@ environment: instance_type: Standard_ND96amsr_A100_v4 request_settings: request_timeout_ms: 90000 + max_concurrent_requests_per_instance: 256 liveness_probe: initial_delay: 300 timeout: 20 diff --git a/aml/endpoint.yaml b/aml/endpoint.yaml index 934b31ad..f2f01d5e 100644 --- a/aml/endpoint.yaml +++ b/aml/endpoint.yaml @@ -1,3 +1,3 @@ $schema: https://azuremlsdk2.blob.core.windows.net/latest/managedOnlineEndpoint.schema.json name: bloom-inference -auth_mode: aml_token +auth_mode: key diff --git a/aml/model.yaml b/aml/model.yaml new file mode 100644 index 00000000..e4f1ded2 --- /dev/null +++ b/aml/model.yaml @@ -0,0 +1,5 @@ +$schema: https://azuremlschemas.azureedge.net/latest/model.schema.json +name: bloom +version: 1 +path: ./bloom +type: custom_model From 5e5d8766a2c2a28b166fbecf7543f19a4bf2c9d8 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Mon, 17 Oct 2022 14:59:00 +0200 Subject: [PATCH 010/793] feat: Improve error handling --- Dockerfile | 3 +- README.md | 1 - aml/deployment.yaml | 14 +++---- router/Cargo.lock | 33 ++++++++-------- router/Cargo.toml | 10 ++++- router/client/src/client.rs | 34 +++++++---------- router/client/src/lib.rs | 22 ++++++----- router/client/src/sharded_client.rs | 23 +++++------ router/src/batcher.rs | 44 ++++++++++++++++----- router/src/db.rs | 59 ++++++++++++++++------------- router/src/lib.rs | 8 ++++ router/src/main.rs | 23 ++--------- router/src/server.rs | 59 +++++++++++------------------ router/src/validation.rs | 42 ++++++++++++++++---- run.sh | 17 +++++++-- server/bloom_inference/cli.py | 42 ++++++++++++++++++++ server/bloom_inference/main.py | 30 --------------- server/bloom_inference/server.py | 6 +-- server/bloom_inference/utils.py | 3 ++ server/pyproject.toml | 3 ++ 20 files changed, 270 insertions(+), 206 deletions(-) create mode 100644 router/src/lib.rs mode change 100755 => 100644 run.sh create mode 100644 server/bloom_inference/cli.py delete mode 100644 server/bloom_inference/main.py diff --git a/Dockerfile b/Dockerfile index a5161020..57c19ee2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ ENV LANG=C.UTF-8 \ MODEL_NAME=bigscience/bloom \ NUM_GPUS=8 \ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ + NCCL_ASYNC_ERROR_HANDLING=1 \ CUDA_HOME=/usr/local/cuda \ LD_LIBRARY_PATH="/opt/miniconda/envs/text-generation/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH" \ CONDA_DEFAULT_ENV=text-generation \ @@ -51,7 +52,7 @@ RUN cd server && \ /opt/miniconda/envs/text-generation/bin/pip install . --no-cache-dir # Install router -COPY --from=builder /usr/local/cargo/bin/bloom-inference /usr/local/bin/bloom-inference +COPY --from=builder /usr/local/cargo/bin/text-generation-router /usr/local/bin/text-generation-router COPY run.sh . RUN chmod +x run.sh diff --git a/README.md b/README.md index a62daca1..78da4db2 100644 --- a/README.md +++ b/README.md @@ -48,5 +48,4 @@ python server/bloom_inference/main.py bigscience/bloom --num-gpus 8 --shard-dire - [ ] Add tests - [ ] Add shutdown logic in router and server - [ ] Improve multi-processing logic in server -- [ ] Improve error handling everywhere - [ ] Improve past key layer indexing? \ No newline at end of file diff --git a/aml/deployment.yaml b/aml/deployment.yaml index 31cb09c5..be28ceef 100644 --- a/aml/deployment.yaml +++ b/aml/deployment.yaml @@ -8,7 +8,7 @@ environment_variables: MODEL_NAME: bigscience/bloom NUM_GPUS: 8 environment: - image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.1 + image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.3 inference_config: liveness_route: port: 3000 @@ -24,15 +24,15 @@ request_settings: request_timeout_ms: 90000 max_concurrent_requests_per_instance: 256 liveness_probe: - initial_delay: 300 + initial_delay: 600 timeout: 20 - period: 60 + period: 120 success_threshold: 1 - failure_threshold: 60 + failure_threshold: 3 readiness_probe: - initial_delay: 300 + initial_delay: 600 timeout: 20 - period: 60 + period: 120 success_threshold: 1 - failure_threshold: 60 + failure_threshold: 3 instance_count: 1 diff --git a/router/Cargo.lock b/router/Cargo.lock index 1f00df14..c912be89 100644 --- a/router/Cargo.lock +++ b/router/Cargo.lock @@ -149,22 +149,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bloom-inference" -version = "0.1.0" -dependencies = [ - "axum", - "bloom-inference-client", - "futures", - "parking_lot", - "serde", - "serde_json", - "tokenizers", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "bloom-inference-client" version = "0.1.0" @@ -1669,6 +1653,23 @@ dependencies = [ "winapi", ] +[[package]] +name = "text-generation-router" +version = "0.1.0" +dependencies = [ + "axum", + "bloom-inference-client", + "futures", + "parking_lot", + "serde", + "serde_json", + "thiserror", + "tokenizers", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "textwrap" version = "0.11.0" diff --git a/router/Cargo.toml b/router/Cargo.toml index c5e5bb89..95666dcf 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,8 +1,15 @@ [package] -name = "bloom-inference" +name = "text-generation-router" version = "0.1.0" edition = "2021" +[lib] +path = "src/lib.rs" + +[[bin]] +name = "text-generation-router" +path = "src/main.rs" + [dependencies] axum = { version = "0.5.16", features = ["json", "serde_json"] } bloom-inference-client = { path = "client" } @@ -10,6 +17,7 @@ futures = "0.3.24" parking_lot = "0.12.1" serde = "1.0.145" serde_json = "1.0.85" +thiserror = "1.0.37" tokenizers = "0.13.0" tokio = { version = "1.21.1", features = ["rt-multi-thread", "parking_lot", "sync"] } tracing = "0.1.36" diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 712c13da..e7189b89 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -1,45 +1,37 @@ use crate::pb::generate::v1::text_generation_service_client::TextGenerationServiceClient; use crate::pb::generate::v1::*; use crate::Result; -use std::time::Duration; use tonic::transport::{Channel, Uri}; -use tower::timeout::Timeout; use tracing::*; /// BLOOM Inference gRPC client #[derive(Clone)] pub struct Client { - stub: TextGenerationServiceClient>, + stub: TextGenerationServiceClient, } impl Client { - /// Returns a client connected to the given url. Requests exceeding timeout will fail. - pub async fn connect(uri: Uri, timeout: Duration) -> Self { - let channel = Channel::builder(uri) - .connect() - .await - .expect("Transport error"); - let timeout_channel = Timeout::new(channel, timeout); + /// Returns a client connected to the given url + pub async fn connect(uri: Uri) -> Result { + let channel = Channel::builder(uri).connect().await?; - Self { - stub: TextGenerationServiceClient::new(timeout_channel), - } + Ok(Self { + stub: TextGenerationServiceClient::new(channel), + }) } - /// Returns a client connected to the given unix socket. Requests exceeding timeout will fail. - pub async fn connect_uds(path: String, timeout: Duration) -> Self { + /// Returns a client connected to the given unix socket + pub async fn connect_uds(path: String) -> Result { let channel = Channel::from_shared("http://[::]:50051".to_string()) .unwrap() .connect_with_connector(tower::service_fn(move |_: Uri| { tokio::net::UnixStream::connect(path.clone()) })) - .await - .expect("Transport error"); - let timeout_channel = Timeout::new(channel, timeout); + .await?; - Self { - stub: TextGenerationServiceClient::new(timeout_channel), - } + Ok(Self { + stub: TextGenerationServiceClient::new(channel), + }) } #[instrument(skip(self))] diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index bb9919e2..48b2650d 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -8,22 +8,26 @@ pub use client::Client; pub use pb::generate::v1::{Batch, GeneratedText, LogitsWarperParameters, Request}; pub use sharded_client::ShardedClient; use thiserror::Error; -pub use tonic::transport::Uri; +pub use tonic::transport; use tonic::Status; #[derive(Error, Debug, Clone)] -#[error("Text generation client error: {msg:?}")] -pub struct ClientError { - msg: String, - // source: Status, +pub enum ClientError { + #[error("Could not connect to Text Generation server: {0:?}")] + Connection(String), + #[error("Server error: {0:?}")] + Generation(String), } impl From for ClientError { fn from(err: Status) -> Self { - Self { - msg: err.to_string(), - // source: err, - } + Self::Generation(err.to_string()) + } +} + +impl From for ClientError { + fn from(err: transport::Error) -> Self { + Self::Connection(err.to_string()) } } diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 7af741f0..7134551e 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,7 +1,6 @@ use crate::Result; use crate::{Batch, Client, GeneratedText}; use futures::future::join_all; -use std::time::Duration; use tokio::sync::{broadcast, mpsc}; use tonic::transport::Uri; @@ -69,24 +68,22 @@ impl ShardedClient { Self { request_tx } } - async fn from_master_client(mut master_client: Client) -> Self { + async fn from_master_client(mut master_client: Client) -> Result { let uris = master_client.service_discovery().await.unwrap(); - let futures = uris - .into_iter() - .map(|path| Client::connect_uds(path, Duration::from_secs(5))); - let clients = join_all(futures).await; - Self::new(clients) + let futures = uris.into_iter().map(|path| Client::connect_uds(path)); + let clients: Result> = join_all(futures).await.into_iter().collect(); + Ok(Self::new(clients?)) } - /// Returns a client connected to the given url. Requests exceeding timeout will fail. - pub async fn connect(uri: Uri, timeout: Duration) -> Self { - let master_client = Client::connect(uri, timeout).await; + /// Returns a client connected to the given url + pub async fn connect(uri: Uri) -> Result { + let master_client = Client::connect(uri).await?; Self::from_master_client(master_client).await } - /// Returns a client connected to the given unix socket. Requests exceeding timeout will fail. - pub async fn connect_uds(path: String, timeout: Duration) -> Self { - let master_client = Client::connect_uds(path, timeout).await; + /// Returns a client connected to the given unix socket + pub async fn connect_uds(path: String) -> Result { + let master_client = Client::connect_uds(path).await?; Self::from_master_client(master_client).await } diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 2025cf62..94c72cc5 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -1,13 +1,30 @@ use crate::server::GenerateRequest; -use crate::Db; +use crate::{Db, Entry}; +use axum::http::StatusCode; use bloom_inference_client::{Batch, ClientError, GeneratedText, ShardedClient}; use std::future::Future; use std::sync::Arc; +use thiserror::Error; use tokio::sync::{oneshot, Notify}; const MAX_LENGTH: usize = 128; -pub struct InferError {} +#[derive(Debug, Error)] +pub enum InferError { + #[error("Request failed during generation: {0}")] + GenerationError(String), + #[error("Model is overloaded")] + Overloaded, +} + +impl From for (StatusCode, String) { + fn from(err: InferError) -> Self { + match err { + InferError::GenerationError(_) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), + InferError::Overloaded => (StatusCode::TOO_MANY_REQUESTS, err.to_string()), + } + } +} #[derive(Clone)] pub(crate) struct Batcher { @@ -37,14 +54,18 @@ impl Batcher { request: GenerateRequest, ) -> Result { if self.db.len() > MAX_LENGTH { - return Err(InferError {}); + return Err(InferError::Overloaded); } let (request_tx, request_rx) = oneshot::channel(); - self.db.append(input_length, request, request_tx); + self.db.append(Entry { + request, + response_tx: request_tx, + input_length, + }); self.shared.batching_task.notify_waiters(); match request_rx.await.unwrap() { Ok(output) => Ok(output), - Err(_) => Err(InferError {}), + Err(err) => Err(InferError::GenerationError(err.to_string())), } } } @@ -108,7 +129,6 @@ async fn wrap_future( next_batch } Err(err) => { - println!("{:?}", err); send_error(err, request_ids, db); None } @@ -117,14 +137,18 @@ async fn wrap_future( fn send_error(error: ClientError, request_ids: Vec, db: &Db) { request_ids.into_iter().for_each(|id| { - let (_, response_tx) = db.remove(&id).unwrap(); - response_tx.send(Err(error.clone())).unwrap_or(()); + let entry = db.remove(&id).expect("ID not found in db. This is a bug."); + // unwrap_or is valid here as we don't care if the receiver is gone. + entry.response_tx.send(Err(error.clone())).unwrap_or(()); }); } fn send_generated(finished: Vec, db: &Db) { finished.into_iter().for_each(|output| { - let (_, response_tx) = db.remove(&output.request.unwrap().id).unwrap(); - response_tx.send(Ok(output.output)).unwrap_or(()); + let entry = db + .remove(&output.request.unwrap().id) + .expect("ID not found in db. This is a bug."); + // unwrap_or is valid here as we don't care if the receiver is gone. + entry.response_tx.send(Ok(output.output)).unwrap_or(()); }); } diff --git a/router/src/db.rs b/router/src/db.rs index 5118b2fc..03593fc0 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,11 +1,29 @@ /// This code is massively inspired by Tokio mini-redis -use crate::server::GenerateRequest; +use crate::server::{GenerateParameters, GenerateRequest}; use bloom_inference_client::{Batch, ClientError, LogitsWarperParameters, Request}; use parking_lot::RwLock; use std::collections::BTreeMap; use std::sync::Arc; use tokio::sync::oneshot::Sender; +#[derive(Debug)] +pub(crate) struct Entry { + pub request: GenerateRequest, + pub response_tx: Sender>, + pub input_length: usize, +} + +impl From for LogitsWarperParameters { + fn from(parameters: GenerateParameters) -> Self { + Self { + temperature: parameters.temperature, + top_k: parameters.top_k as u32, + top_p: parameters.top_p, + do_sample: parameters.do_sample, + } + } +} + #[derive(Debug, Clone)] pub(crate) struct Db { pub shared: Arc, @@ -18,7 +36,7 @@ pub struct Shared { #[derive(Debug)] struct State { - entries: BTreeMap>)>, + entries: BTreeMap, /// Identifier to use for the next expiration. Each expiration is associated /// with a unique identifier. See above for why. @@ -44,37 +62,16 @@ impl Db { Self { shared } } - pub(crate) fn append( - &self, - input_length: usize, - request: GenerateRequest, - sender: Sender>, - ) { + pub(crate) fn append(&self, entry: Entry) { let mut state = self.shared.state.write(); let id = state.next_id; state.next_id += 1; - let parameters = Some(LogitsWarperParameters { - temperature: request.parameters.temperature, - top_k: request.parameters.top_k, - top_p: request.parameters.top_p, - do_sample: request.parameters.do_sample, - }); - let request = Request { - id, - inputs: request.inputs, - input_length: input_length as u32, - parameters, - max_new_tokens: request.parameters.max_new_tokens, - }; - state.entries.insert(id, (request, sender)); + state.entries.insert(id, entry); } - pub(crate) fn remove( - &self, - id: &u64, - ) -> Option<(Request, Sender>)> { + pub(crate) fn remove(&self, id: &u64) -> Option { let mut state = self.shared.state.write(); state.entries.remove(id) } @@ -91,7 +88,15 @@ impl Db { .entries .range(state.next_batch_start_id..) .take(max_size) - .map(|(_, (request, _))| request.clone()) + .map(|(id, entry)| Request { + id: *id, + inputs: entry.request.inputs.clone(), + input_length: entry.input_length as u32, + parameters: Some(LogitsWarperParameters::from( + entry.request.parameters.clone(), + )), + max_new_tokens: entry.request.parameters.max_new_tokens, + }) .collect(); if requests.is_empty() { diff --git a/router/src/lib.rs b/router/src/lib.rs new file mode 100644 index 00000000..09dbdd12 --- /dev/null +++ b/router/src/lib.rs @@ -0,0 +1,8 @@ +mod batcher; +mod db; +pub mod server; +mod validation; + +use batcher::Batcher; +use db::{Db, Entry}; +use validation::Validation; diff --git a/router/src/main.rs b/router/src/main.rs index 2fe02944..5169a071 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,21 +1,8 @@ use bloom_inference_client::ShardedClient; use std::net::SocketAddr; -use std::time::Duration; +use text_generation_router::server; use tokenizers::Tokenizer; -mod server; -mod validation; - -use validation::Validation; - -mod db; - -use db::Db; - -mod batcher; - -use batcher::Batcher; - fn main() -> Result<(), std::io::Error> { let tokenizer = Tokenizer::from_pretrained("bigscience/bloom", None).unwrap(); @@ -26,11 +13,9 @@ fn main() -> Result<(), std::io::Error> { .block_on(async { tracing_subscriber::fmt::init(); - let sharded_client = ShardedClient::connect_uds( - "/tmp/bloom-inference-0".to_string(), - Duration::from_secs(5), - ) - .await; + let sharded_client = ShardedClient::connect_uds("/tmp/bloom-inference-0".to_string()) + .await + .expect("Could not connect to server"); sharded_client .clear_cache() .await diff --git a/router/src/server.rs b/router/src/server.rs index f113ce44..b8331da1 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,9 +1,9 @@ -use bloom_inference_client::ShardedClient; use crate::{Batcher, Validation}; use axum::extract::Extension; use axum::http::StatusCode; use axum::routing::{get, post}; use axum::{Json, Router}; +use bloom_inference_client::ShardedClient; use serde::Deserialize; use std::net::SocketAddr; use tokenizers::Tokenizer; @@ -15,7 +15,7 @@ pub(crate) struct GenerateParameters { #[serde(default = "default_temperature")] pub temperature: f32, #[serde(default = "default_top_k")] - pub top_k: u32, + pub top_k: i32, #[serde(default = "default_top_p")] pub top_p: f32, #[serde(default = "default_do_sample")] @@ -28,7 +28,7 @@ fn default_temperature() -> f32 { 1.0 } -fn default_top_k() -> u32 { +fn default_top_k() -> i32 { 0 } @@ -62,8 +62,8 @@ pub(crate) struct GenerateRequest { } #[instrument(skip(state), fields(time, time_per_token))] -async fn liveness(state: Extension) -> Result<(), StatusCode> { - let output = state +async fn liveness(state: Extension) -> Result<(), (StatusCode, String)> { + state .infer .infer( 1, @@ -78,50 +78,37 @@ async fn liveness(state: Extension) -> Result<(), StatusCode> { }, }, ) - .await; - - match output { - Ok(_) => Ok(()), - Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR), - } + .await?; + Ok(()) } #[instrument(skip(state), fields(time, time_per_token))] async fn generate( state: Extension, req: Json, -) -> Result, StatusCode> { +) -> Result, (StatusCode, String)> { let start = Instant::now(); - let (input_length, validated_request) = match state + let (input_length, validated_request) = state .validation .validate(GenerateRequest { inputs: req.inputs.clone(), parameters: req.parameters.clone(), }) - .await - { - Ok(result) => result, - Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR), - }; - - let output = state.infer.infer(input_length, validated_request).await; - - match output { - Ok(generated_text) => { - tracing::Span::current().record("time", format!("{:?}", start.elapsed())); - tracing::Span::current().record( - "time_per_token", - format!("{:?}", start.elapsed() / req.parameters.max_new_tokens), - ); - tracing::info!("response: {}", generated_text); - - Ok(Json(serde_json::json!({ - "generated_text": generated_text, - }))) - } - Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR), - } + .await?; + + let generated_text = state.infer.infer(input_length, validated_request).await?; + + tracing::Span::current().record("time", format!("{:?}", start.elapsed())); + tracing::Span::current().record( + "time_per_token", + format!("{:?}", start.elapsed() / req.parameters.max_new_tokens), + ); + tracing::info!("response: {}", generated_text); + + Ok(Json(serde_json::json!({ + "generated_text": generated_text, + }))) } #[derive(Clone)] diff --git a/router/src/validation.rs b/router/src/validation.rs index 6987894d..c9d391aa 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,9 +1,28 @@ use crate::server::GenerateRequest; +use axum::http::StatusCode; +use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokio::sync::{mpsc, oneshot}; -#[derive(Debug)] -pub struct ValidationError {} +#[derive(Error, Debug)] +pub enum ValidationError { + #[error("Temperature must be strictly positive")] + Temperature, + #[error("Top p must be <= 0.0 or > 1.0")] + TopP, + #[error("Top k must be strictly positive")] + TopK, + #[error("Max New Tokens must be < 512")] + MaxNewTokens, + #[error("Inputs must have less than 512 tokens. Given: {0}")] + InputLength(usize), +} + +impl From for (StatusCode, String) { + fn from(err: ValidationError) -> Self { + (StatusCode::BAD_REQUEST, err.to_string()) + } +} type ValidationRequest = ( GenerateRequest, @@ -39,15 +58,23 @@ impl Validation { async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver) { while let Some((request, response_tx)) = receiver.recv().await { if request.parameters.temperature < 0.0 { - response_tx.send(Err(ValidationError {})).unwrap_or(()); + response_tx + .send(Err(ValidationError::Temperature)) + .unwrap_or(()); continue; } if request.parameters.top_p <= 0.0 || request.parameters.top_p > 1.0 { - response_tx.send(Err(ValidationError {})).unwrap_or(()); + response_tx.send(Err(ValidationError::TopP)).unwrap_or(()); + continue; + } + if request.parameters.top_k < 0 { + response_tx.send(Err(ValidationError::TopK)).unwrap_or(()); continue; } if request.parameters.max_new_tokens > 512 { - response_tx.send(Err(ValidationError {})).unwrap_or(()); + response_tx + .send(Err(ValidationError::MaxNewTokens)) + .unwrap_or(()); continue; } @@ -55,11 +82,12 @@ async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver 512 { - response_tx.send(Err(ValidationError {})).unwrap_or(()); + response_tx + .send(Err(ValidationError::InputLength(input_length))) + .unwrap_or(()); continue; } response_tx.send(Ok((input_length, request))).unwrap_or(()); } - println!("drop here"); } diff --git a/run.sh b/run.sh old mode 100755 new mode 100644 index 3b095541..30303501 --- a/run.sh +++ b/run.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash -server_cmd="python server/bloom_inference/main.py $MODEL_NAME --num-gpus $NUM_GPUS --shard-directory $MODEL_BASE_PATH" -$server_cmd & +server_cmd="bloom-inference-server launcher $MODEL_NAME --num-gpus $NUM_GPUS --shard-directory $MODEL_BASE_PATH" -FILE=/tmp/bloom-inference-0 +# Run in background +$server_cmd 2>&1 > /dev/null & +# Check if server is running by checking if the unix socket is created +FILE=/tmp/bloom-inference-0 while : do if test -S "$FILE"; then @@ -18,4 +20,11 @@ while : sleep 1 -exec "bloom-inference" +# Run in background +text-generation-router & + +# Wait for any process to exit +wait -n + +# Exit with status of process that exited first +exit $? \ No newline at end of file diff --git a/server/bloom_inference/cli.py b/server/bloom_inference/cli.py new file mode 100644 index 00000000..a5f84e77 --- /dev/null +++ b/server/bloom_inference/cli.py @@ -0,0 +1,42 @@ +import typer + +from pathlib import Path +from torch.distributed.launcher import launch_agent, LaunchConfig +from typing import Optional + +from bloom_inference import server + +app = typer.Typer() + + +@app.command() +def launcher( + model_name: str, + num_gpus: int = 1, + shard_directory: Optional[Path] = None, +): + if num_gpus == 1: + serve(model_name, False, shard_directory) + + else: + config = LaunchConfig( + min_nodes=1, + max_nodes=1, + nproc_per_node=num_gpus, + rdzv_backend="c10d", + max_restarts=0, + ) + launch_agent(config, server.serve, [model_name, True, shard_directory]) + + +@app.command() +def serve( + model_name: str, + sharded: bool = False, + shard_directory: Optional[Path] = None, +): + server.serve(model_name, sharded, shard_directory) + + +if __name__ == "__main__": + app() diff --git a/server/bloom_inference/main.py b/server/bloom_inference/main.py deleted file mode 100644 index b81ed040..00000000 --- a/server/bloom_inference/main.py +++ /dev/null @@ -1,30 +0,0 @@ -import typer - -from pathlib import Path -from torch.distributed.launcher import launch_agent, LaunchConfig -from typing import Optional - -from bloom_inference.server import serve - - -def main( - model_name: str, - num_gpus: int = 1, - shard_directory: Optional[Path] = None, -): - if num_gpus == 1: - serve(model_name, False, shard_directory) - - else: - config = LaunchConfig( - min_nodes=1, - max_nodes=1, - nproc_per_node=num_gpus, - rdzv_backend="c10d", - max_restarts=0, - ) - launch_agent(config, serve, [model_name, True, shard_directory]) - - -if __name__ == "__main__": - typer.run(main) diff --git a/server/bloom_inference/server.py b/server/bloom_inference/server.py index 3a509169..e89706e0 100644 --- a/server/bloom_inference/server.py +++ b/server/bloom_inference/server.py @@ -1,4 +1,6 @@ import asyncio +import os + from grpc import aio from grpc_reflection.v1alpha import reflection @@ -143,7 +145,3 @@ async def serve_inner( await server.wait_for_termination() asyncio.run(serve_inner(model_name, sharded, shard_directory)) - - -if __name__ == "__main__": - serve("bigscience/bloom-560m", True, Path("/tmp/models")) diff --git a/server/bloom_inference/utils.py b/server/bloom_inference/utils.py index db02dadb..fe2c913e 100644 --- a/server/bloom_inference/utils.py +++ b/server/bloom_inference/utils.py @@ -2,6 +2,8 @@ import contextlib import torch import torch.distributed + +from datetime import timedelta from transformers.generation_logits_process import ( LogitsProcessorList, TemperatureLogitsWarper, @@ -79,6 +81,7 @@ def initialize_torch_distributed(): backend=backend, world_size=world_size, rank=rank, + timeout=timedelta(seconds=60), init_method="tcp://localhost:6000", ) diff --git a/server/pyproject.toml b/server/pyproject.toml index 80d95426..1dd8ae27 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -4,6 +4,9 @@ version = "0.1.0" description = "BLOOM Inference Python gRPC Server" authors = ["Olivier Dehaene "] +[tool.poetry.scripts] +bloom-inference-server = 'bloom_inference.cli:app' + [tool.poetry.dependencies] python = "^3.9" protobuf = "^4.21.7" From 92c1ecd0089d329d4b5a5e2b9327da828b888d34 Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Mon, 17 Oct 2022 18:27:33 +0200 Subject: [PATCH 011/793] feat: Add arguments to CLI --- Dockerfile | 2 +- README.md | 1 - router/Cargo.lock | 102 ++++++++++++++++++++++++++++++++++++++- router/Cargo.toml | 3 ++ router/src/batcher.rs | 28 ++++++----- router/src/lib.rs | 4 +- router/src/main.rs | 36 ++++++++++++-- router/src/server.rs | 16 +++--- router/src/validation.rs | 6 +-- 9 files changed, 163 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 57c19ee2..9b8a2054 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ WORKDIR /usr/src/router RUN cargo install --path . -FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 +FROM nvidia/cuda:11.6.1-devel-ubuntu18.04 ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ diff --git a/README.md b/README.md index 78da4db2..6d23d9c5 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ python server/bloom_inference/main.py bigscience/bloom --num-gpus 8 --shard-dire ## TODO: -- [ ] Add batching args to router CLI - [ ] Add docstrings + comments everywhere as the codebase is fairly complicated - [ ] Add tests - [ ] Add shutdown logic in router and server diff --git a/router/Cargo.lock b/router/Cargo.lock index c912be89..eda42908 100644 --- a/router/Cargo.lock +++ b/router/Cargo.lock @@ -253,6 +253,43 @@ dependencies = [ "vec_map", ] +[[package]] +name = "clap" +version = "4.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf8832993da70a4c6d13c581f4463c2bdda27b9bf1c5498dc4365543abe6d6f" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "once_cell", + "strsim 0.10.0", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42f169caba89a7d512b5418b09864543eeb4d497416c917d7137863bd2076ad" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "console" version = "0.15.2" @@ -701,6 +738,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1136,6 +1179,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "os_str_bytes" +version = "6.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1225,6 +1274,30 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.46" @@ -1251,7 +1324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" dependencies = [ "bytes", - "heck", + "heck 0.3.3", "itertools 0.10.5", "lazy_static", "log", @@ -1601,6 +1674,12 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "syn" version = "1.0.101" @@ -1643,6 +1722,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.1.17" @@ -1659,6 +1747,7 @@ version = "0.1.0" dependencies = [ "axum", "bloom-inference-client", + "clap 4.0.15", "futures", "parking_lot", "serde", @@ -1742,7 +1831,7 @@ checksum = "3d7b08ede6742d7a59d58c71da8a6fa21bedc433dca2e855e439274d08df1170" dependencies = [ "aho-corasick", "cached-path", - "clap", + "clap 2.34.0", "derive_builder", "dirs", "esaxx-rs", @@ -2251,6 +2340,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/router/Cargo.toml b/router/Cargo.toml index 95666dcf..37f319e9 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -2,6 +2,8 @@ name = "text-generation-router" version = "0.1.0" edition = "2021" +authors = ["Olivier Dehaene"] +description = "Text Generation Webserver" [lib] path = "src/lib.rs" @@ -13,6 +15,7 @@ path = "src/main.rs" [dependencies] axum = { version = "0.5.16", features = ["json", "serde_json"] } bloom-inference-client = { path = "client" } +clap = { version = "4.0.15", features = ["derive", "env"] } futures = "0.3.24" parking_lot = "0.12.1" serde = "1.0.145" diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 94c72cc5..ebd81730 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -27,7 +27,7 @@ impl From for (StatusCode, String) { } #[derive(Clone)] -pub(crate) struct Batcher { +pub struct Batcher { db: Db, shared: Arc, } @@ -37,13 +37,13 @@ struct Shared { } impl Batcher { - pub(crate) fn new(client: ShardedClient) -> Self { + pub(crate) fn new(client: ShardedClient, max_batch_size: usize) -> Self { let db = Db::new(); let shared = Arc::new(Shared { batching_task: Notify::new(), }); - tokio::spawn(batching_task(client, db.clone(), shared.clone())); + tokio::spawn(batching_task(max_batch_size, client, db.clone(), shared.clone())); Self { db, shared } } @@ -70,40 +70,46 @@ impl Batcher { } } -async fn batching_task(client: ShardedClient, db: Db, shared: Arc) { +async fn batching_task(max_batch_size: usize, + client: ShardedClient, + db: Db, + shared: Arc) { + let limit_min_batch_size = (max_batch_size / 2) as u32; + loop { shared.batching_task.notified().await; - if let Some(batch) = db.next_batch(32) { + if let Some(batch) = db.next_batch(max_batch_size) { let request_ids = batch.requests.iter().map(|req| req.id).collect(); let mut cached_batch = match batch.size { - size if size > 16 => { + size if size > limit_min_batch_size => { wrap_future(client.generate_until_finished(batch), request_ids, &db).await } _ => wrap_future(client.generate(batch), request_ids, &db).await, }; while let Some(batch) = cached_batch { - let batch_size = batch.size; + let mut current_batch_size = batch.size; let mut request_ids: Vec = batch.requests.iter().map(|req| req.id).collect(); let mut batches = vec![batch]; - if batch_size <= 16 { - if let Some(new_batch) = db.next_batch_minimum_size(16, 48) { + if current_batch_size <= limit_min_batch_size { + if let Some(new_batch) = db.next_batch_minimum_size(limit_min_batch_size as usize, max_batch_size) { let new_batch_request_ids = new_batch.requests.iter().map(|req| req.id).collect(); let new_cached_batch = wrap_future(client.generate(new_batch), new_batch_request_ids, &db) .await; if let Some(new_cached_batch) = new_cached_batch { + current_batch_size += new_cached_batch.size; request_ids.extend(new_cached_batch.requests.iter().map(|req| req.id)); batches.push(new_cached_batch); } } } - cached_batch = match batch_size { - size if size > 16 => { + cached_batch = match current_batch_size { + size if size > limit_min_batch_size => { wrap_future( client.generate_until_finished_with_cache(batches), request_ids, diff --git a/router/src/lib.rs b/router/src/lib.rs index 09dbdd12..14dc5724 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,8 +1,8 @@ mod batcher; mod db; -pub mod server; mod validation; +pub mod server; -use batcher::Batcher; use db::{Db, Entry}; +use batcher::Batcher; use validation::Validation; diff --git a/router/src/main.rs b/router/src/main.rs index 5169a071..89cd4731 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,10 +1,36 @@ use bloom_inference_client::ShardedClient; -use std::net::SocketAddr; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use text_generation_router::server; use tokenizers::Tokenizer; +use clap::Parser; + +/// App Configuration +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Args { + #[clap(default_value = "32", long, short, env)] + max_batch_size: usize, + #[clap(default_value = "3000", long, short, env)] + port: u16, + #[clap(default_value = "/tmp/bloom-inference-0", long, env)] + shard_uds_path: String, + #[clap(default_value = "bigscience/bloom", long, env)] + tokenizer_name: String, +} fn main() -> Result<(), std::io::Error> { - let tokenizer = Tokenizer::from_pretrained("bigscience/bloom", None).unwrap(); + // Get args + let args = Args::parse(); +// Pattern match configuration + let Args { + max_batch_size, + port, + shard_uds_path, + tokenizer_name, + } = args; + + + let tokenizer = Tokenizer::from_pretrained(tokenizer_name, None).unwrap(); tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -13,7 +39,7 @@ fn main() -> Result<(), std::io::Error> { .block_on(async { tracing_subscriber::fmt::init(); - let sharded_client = ShardedClient::connect_uds("/tmp/bloom-inference-0".to_string()) + let sharded_client = ShardedClient::connect_uds(shard_uds_path) .await .expect("Could not connect to server"); sharded_client @@ -22,9 +48,9 @@ fn main() -> Result<(), std::io::Error> { .expect("Unable to clear cache"); tracing::info!("Connected"); - let addr = SocketAddr::from(([0, 0, 0, 0], 3000)); + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port); - server::run(sharded_client, tokenizer, addr).await; + server::run(max_batch_size, sharded_client, tokenizer, addr).await; Ok(()) }) } diff --git a/router/src/server.rs b/router/src/server.rs index b8331da1..0fdfd58b 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -64,7 +64,7 @@ pub(crate) struct GenerateRequest { #[instrument(skip(state), fields(time, time_per_token))] async fn liveness(state: Extension) -> Result<(), (StatusCode, String)> { state - .infer + .batcher .infer( 1, GenerateRequest { @@ -97,7 +97,7 @@ async fn generate( }) .await?; - let generated_text = state.infer.infer(input_length, validated_request).await?; + let generated_text = state.batcher.infer(input_length, validated_request).await?; tracing::Span::current().record("time", format!("{:?}", start.elapsed())); tracing::Span::current().record( @@ -114,18 +114,14 @@ async fn generate( #[derive(Clone)] struct ServerState { validation: Validation, - infer: Batcher, + batcher: Batcher, } -pub async fn run(client: ShardedClient, tokenizer: Tokenizer, addr: SocketAddr) { - client.clear_cache().await.expect("Unable to clear cache"); - tracing::info!("Connected"); - - let infer = Batcher::new(client); - +pub async fn run(max_batch_size: usize, client: ShardedClient, tokenizer: Tokenizer, addr: SocketAddr) { + let batcher = Batcher::new(client, max_batch_size); let validation = Validation::new(tokenizer); - let shared_state = ServerState { validation, infer }; + let shared_state = ServerState { validation, batcher }; let app = Router::new() .route("/generate", post(generate)) diff --git a/router/src/validation.rs b/router/src/validation.rs index c9d391aa..45b108fd 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -14,7 +14,7 @@ pub enum ValidationError { TopK, #[error("Max New Tokens must be < 512")] MaxNewTokens, - #[error("Inputs must have less than 512 tokens. Given: {0}")] + #[error("Inputs must have less than 1000 tokens. Given: {0}")] InputLength(usize), } @@ -30,7 +30,7 @@ type ValidationRequest = ( ); #[derive(Debug, Clone)] -pub(crate) struct Validation { +pub struct Validation { sender: mpsc::Sender, } @@ -81,7 +81,7 @@ async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver 512 { + if input_length > 1000 { response_tx .send(Err(ValidationError::InputLength(input_length))) .unwrap_or(()); From f16f2f5ae15a1b728559e1ac1a8b840f15a54efd Mon Sep 17 00:00:00 2001 From: Olivier Dehaene Date: Tue, 18 Oct 2022 15:19:03 +0200 Subject: [PATCH 012/793] v0.1.0 --- .dockerignore | 2 +- .gitignore | 3 +- router/Cargo.lock => Cargo.lock | 212 ++++++++--- Cargo.toml | 11 + Dockerfile | 36 +- Makefile | 19 + README.md | 55 +-- aml/deployment.yaml | 2 +- assets/architecture.jpg | Bin 0 -> 135442 bytes launcher/Cargo.toml | 13 + launcher/src/main.rs | 358 ++++++++++++++++++ proto/generate.proto | 28 -- router/.gitignore | 1 - router/Cargo.toml | 11 +- router/client/Cargo.toml | 2 - router/client/src/client.rs | 44 +-- router/client/src/lib.rs | 7 +- router/client/src/sharded_client.rs | 96 ++--- router/src/batcher.rs | 167 +++++--- router/src/db.rs | 185 +++++---- router/src/lib.rs | 64 +++- router/src/main.rs | 52 ++- router/src/server.rs | 166 ++++---- router/src/validation.rs | 131 +++++-- run.sh | 30 -- ...rust-toolchain.toml => rust-toolchain.toml | 0 server/.gitignore | 155 ++++++++ server/Makefile | 21 +- server/bloom_inference/cli.py | 56 +-- server/bloom_inference/model.py | 36 +- server/bloom_inference/pb/.gitignore | 2 +- server/bloom_inference/prepare_weights.py | 97 ++--- server/bloom_inference/server.py | 67 +--- server/bloom_inference/utils.py | 1 - server/poetry.lock | 2 +- server/pyproject.toml | 1 - 36 files changed, 1506 insertions(+), 627 deletions(-) rename router/Cargo.lock => Cargo.lock (91%) create mode 100644 Cargo.toml create mode 100644 Makefile create mode 100644 assets/architecture.jpg create mode 100644 launcher/Cargo.toml create mode 100644 launcher/src/main.rs delete mode 100644 router/.gitignore delete mode 100644 run.sh rename router/rust-toolchain.toml => rust-toolchain.toml (100%) create mode 100644 server/.gitignore diff --git a/.dockerignore b/.dockerignore index 38e8f824..5aa1aa3a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,2 @@ aml -router/target \ No newline at end of file +target \ No newline at end of file diff --git a/.gitignore b/.gitignore index 723ef36f..ec376bb8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.idea \ No newline at end of file +.idea +target \ No newline at end of file diff --git a/router/Cargo.lock b/Cargo.lock similarity index 91% rename from router/Cargo.lock rename to Cargo.lock index eda42908..551f7aeb 100644 --- a/router/Cargo.lock +++ b/Cargo.lock @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" dependencies = [ "proc-macro2", "quote", @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.0" +version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "byteorder" @@ -255,9 +255,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.15" +version = "4.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bf8832993da70a4c6d13c581f4463c2bdda27b9bf1c5498dc4365543abe6d6f" +checksum = "06badb543e734a2d6568e19a40af66ed5364360b9226184926f89d229b4b4267" dependencies = [ "atty", "bitflags", @@ -391,6 +391,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctrlc" +version = "3.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +dependencies = [ + "nix", + "winapi", +] + [[package]] name = "darling" version = "0.10.2" @@ -529,7 +539,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -936,9 +946,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "js-sys" @@ -957,9 +967,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.134" +version = "0.2.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" +checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" [[package]] name = "lock_api" @@ -1047,7 +1057,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -1074,6 +1084,18 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +dependencies = [ + "autocfg", + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.1" @@ -1084,6 +1106,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num_cpus" version = "1.13.1" @@ -1185,6 +1217,12 @@ version = "6.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1197,15 +1235,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1300,9 +1338,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -1530,7 +1568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -1584,9 +1622,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" +checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" dependencies = [ "itoa", "ryu", @@ -1625,6 +1663,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.7" @@ -1680,11 +1727,21 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "syn" -version = "1.0.101" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" +checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" dependencies = [ "proc-macro2", "quote", @@ -1741,13 +1798,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "text-generation-launcher" +version = "0.1.0" +dependencies = [ + "clap 4.0.17", + "ctrlc", + "subprocess", + "tracing", + "tracing-subscriber", +] + [[package]] name = "text-generation-router" version = "0.1.0" dependencies = [ "axum", "bloom-inference-client", - "clap 4.0.15", + "clap 4.0.17", "futures", "parking_lot", "serde", @@ -1872,6 +1940,7 @@ dependencies = [ "num_cpus", "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "winapi", @@ -1910,9 +1979,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" +checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", "pin-project-lite", @@ -2031,9 +2100,9 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" [[package]] name = "tower-service" @@ -2043,9 +2112,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.36" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", @@ -2056,9 +2125,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", @@ -2067,9 +2136,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -2108,11 +2177,11 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ - "ansi_term", + "nu-ansi-term", "sharded-slab", "smallvec", "thread_local", @@ -2140,9 +2209,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] name = "unicode-normalization" @@ -2361,43 +2430,100 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + [[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + [[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + [[package]] name = "winreg" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..d3f7dfb0 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +[workspace] +members = [ + "router", + "router/client", + "launcher" +] + +[profile.release] +debug = 1 +incremental = true +lto = "off" diff --git a/Dockerfile b/Dockerfile index 9b8a2054..70b91bc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.64 as builder +FROM rust:1.64 as router-builder WORKDIR /usr/src @@ -9,7 +9,17 @@ WORKDIR /usr/src/router RUN cargo install --path . -FROM nvidia/cuda:11.6.1-devel-ubuntu18.04 +FROM rust:1.64 as launcher-builder + +WORKDIR /usr/src + +COPY launcher launcher + +WORKDIR /usr/src/launcher + +RUN cargo install --path . + +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ @@ -34,17 +44,15 @@ RUN cd ~ && \ bash ./Miniconda3-latest-Linux-x86_64.sh -bf -p /opt/miniconda && \ conda create -n text-generation python=3.9 -y +WORKDIR /usr/src + +COPY server/Makefile server/Makefile + # Install specific version of torch -RUN /opt/miniconda/envs/text-generation/bin/pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir +RUN cd server && make install-torch # Install specific version of transformers -RUN wget https://github.com/huggingface/transformers/archive/46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip && \ - unzip 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip && \ - rm 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip && \ - cd transformers-46d37bece7d3ffdef97b1ee4a3170c0a0627d921 && \ - /opt/miniconda/envs/text-generation/bin/python setup.py install - -WORKDIR /usr/src +RUN cd server && make install-transformers # Install server COPY server server @@ -52,9 +60,7 @@ RUN cd server && \ /opt/miniconda/envs/text-generation/bin/pip install . --no-cache-dir # Install router -COPY --from=builder /usr/local/cargo/bin/text-generation-router /usr/local/bin/text-generation-router - -COPY run.sh . -RUN chmod +x run.sh +COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/local/bin/text-generation-router +COPY --from=launcher-builder /usr/local/cargo/bin/text-generation-launcher /usr/local/bin/text-generation-launcher -CMD ["./run.sh"] \ No newline at end of file +CMD text-generation-launcher --model-name $MODEL_NAME --num-shard $NUM_GPUS --shard-directory $MODEL_BASE_PATH \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..3a80a12d --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +install-server: + cd server && make pip-install + +install-router: + cd router && cargo install --path . + +install-launcher: + cd launcher && cargo install --path . + +install: + make install-server + make install-router + make install-launcher + +run-bloom-560m: + text-generation-launcher --model-name bigscience/bloom-560m --shard-directory /tmp/models --num-shard 2 + +run-bloom: + text-generation-launcher --model-name bigscience/bloom --shard-directory /tmp/models --num-shard 8 diff --git a/README.md b/README.md index 6d23d9c5..18e9865d 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,51 @@ -# Text Generation Inference +# LLM Text Generation Inference -A Rust and gRPC server for text generation inference. +
-## Load Tests +![architecture](assets/architecture.jpg) + +
+ +A Rust and gRPC server for large language models text generation inference. + +## Load Tests for BLOOM See `k6/load_test.js` -We send the default examples with a 1 second delay between each request. +We send the default examples with a 1 second delay between requests. Stages: -- Ramp up to 50 concurrent requests per second in 1min -- Ramp up from 50 to 100 concurrent requests per second in 2min -- Ramp down to 0 concurrent requests per second in 1min +- Ramp up to 50 vus in 1min +- Ramp up from 50 to 100 vus in 2min +- Ramp down to 0 vus in 1min -| | avg | min | med | max | p(90) | p(95) | RPS | -|------------------------|-----------|-----------|-----------|------------|-----------|-----------|----------| -| Original code | 8.9s | 1s | 9.12s | 16.69s | 13.7s | 14.26s | 5.9 | -| ISO with original code | 8.88s | 959.53ms | 8.89s | 17.08s | 13.34s | 14.12s | 5.94 | -| New batching logic | **5.44s** | **1.27s** | **5.28s** | **13.12s** | **7.78s** | **8.92s** | **9.08** | +| | avg | min | med | max | p(90) | p(95) | RPS | +|--------------------------------------------------------------|-----------|--------------|-----------|------------|-----------|-----------|----------| +| [Original code](https://github.com/huggingface/transformers_bloom_parallel) | 8.9s | 1s | 9.12s | 16.69s | 13.7s | 14.26s | 5.9 | +| ISO with original code | 8.88s | **959.53ms** | 8.89s | 17.08s | 13.34s | 14.12s | 5.94 | +| New batching logic | **5.44s** | 1.27s | **5.28s** | **13.12s** | **7.78s** | **8.92s** | **9.08** | ## Install ```shell -cd server -pip install . +make install ``` -``` -cd router -cargo build --release -``` - -## Run +## Run ```shell -python server/bloom_inference/main.py bigscience/bloom --num-gpus 8 --shard-directory /dev/shm/models +make run-bloom-560m ``` +## Test + ```shell -./router/target/release/router +curl 127.0.0.1:3000/generate \ + -X POST \ + -d '{"inputs":"Testing API","parameters":{"max_new_tokens":9}}' \ + -H 'Content-Type: application/json' ``` ## TODO: -- [ ] Add docstrings + comments everywhere as the codebase is fairly complicated -- [ ] Add tests -- [ ] Add shutdown logic in router and server -- [ ] Improve multi-processing logic in server -- [ ] Improve past key layer indexing? \ No newline at end of file +- [ ] Add tests for the `server/model` logic \ No newline at end of file diff --git a/aml/deployment.yaml b/aml/deployment.yaml index be28ceef..35d19006 100644 --- a/aml/deployment.yaml +++ b/aml/deployment.yaml @@ -8,7 +8,7 @@ environment_variables: MODEL_NAME: bigscience/bloom NUM_GPUS: 8 environment: - image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.3 + image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.2 inference_config: liveness_route: port: 3000 diff --git a/assets/architecture.jpg b/assets/architecture.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e0a5f7c6ffbb34de6ee877bdf2d74fd3581c75da GIT binary patch literal 135442 zcmeFZ2Ut_xmN$No5~>lTw;)B7suTe!!A21w6lqeT(m_C_DuEzMZvp}WQWO#C(mRok z^dh~6BGMBO2#|z-oS8fG-aGHTpYJ;}|2xm~eTRpH=dkx#Wv{*WTEDgSIUYNn15Vvk z(^Lb<$N+$h^aC7E0@ncY6F;w?pX8*Af{NnjMNLITNkv0VOG`sdLqkgsrl+MlNk>D& zz{qeCeCibADO!4_(@dvMlm0&S^C4tEdy-R7lRBQFqoE_+{e#!>XMmCVgb`H`IT<5x zf{~1zk?gnyfROm4B>R^O__vGf1UUsI6^NRKmX6e*{1kA4MEe8Hh!) zBPG+Bi&v;l>)iwKKVp`87W0-`;A(Lri~aycQ2PGk02*3WHg*n9p|is0&Wp&%UXqhn zxP0yU4K;NQ&6@_d4ULRVOz&7(+t}LKJ2<+zdw6Mj8QWeGJR}#{;yif--0!>`&4DMY3NLEZ|Q`_V0rIhg=f?JvkX^@W>ef zC_pgCIN?tVoCE&f*Z)(&|GieA;L3u(jk`B@)wpezvF;*zPcOFSLy@Z^@;tXLDA^HS zDY+h(rGd3Tcw#1BtvbJtMI*_XEKX^{{AEEURp7V!z z{&UYc`=|8t=V<-8vH0iG`fu0IKeUaoW8nXHWBnB)`-e>bnF;pK9n}90L!_*Q;m)P6 zJNQHkFIc==Ci*yVM!*kwsxnT7s}agkHVmFN0qqNYFEVYvyGp8bpu%sR0{8kWUu#xT zz!*o~l~ZBB289+~rCKP%32d7oCj|C3aeJQ;EI4o$LbhKREj!E6 zv7unFZ>^DZdoId5$nY3&^yfz2Pme+}z3w}@N*wc(8Jr0DM7A{aA1)A0AxD^t zsyuQ*(76;;2|V~_m)D%)^ivOoopKAyr7tR30`3t+QB~foEWd7KX+k*dm23nJhKb_) z7JALp8gJ2GjMoNSC-7%*u=|(B5f?D=RX@USRKONrky#O05}T2S!ZV!5K;tb4{sw8o z$#38~1_o3xqeR-f$H4qK;1~$B!ORjv_k&Ov1JE(Rs0zZ1LW7QhAFw*vV<6x)6c+|S z(WVHwW8j@Hx(LtQcnq}Yl6ETEE;Nd8dW!)$_W?}g_Xp#QkPM_1Bi-BS7@)j>#4-cO zrc#{iF>w8;8IC*u{1^zyApbp!@4mk^@84+Y-^}}W2JSy=>Cc+?@38dm zGwv^1`bR8@{QVjCZ(5oq(q@nr0sEhefZd;51pZn}=cn-;IOs>a4bt*};BiZ9@C)VuBJ6Os`(T%30$pQ(VzN)yWm?v}?9iC1%H3xFraA*i`zNJ(#j+{6zCa0@SU z4anW?`^3~_rt6mEgi=4a|Gxjl1wuT^DLSxc8Bb3r_Ha6~uL{<{rdHSapCf{0>*h$J z6f-T915&U_;k`^8D-pJF;9cS^BU}H!WfLTugO@=!(lLp9h$Mup zhj?EY!quZ(tWNx0^yO_$2*Ko4M94L!s$TQw@oE`h>U1=0BlXB4uokzxV1)_)%x3iv zli&~A3+xugxxkf}sygCdQ$bdle`=cQ7_dP;L!J=72M4@QwkfM!3>uNtHD6`ozMHPx zW}rNLspv?CY)$5$>TY-ow?&Ud&w<(^599X(PdEGVGkj|;#395}2fN+`Kg{fUwRqA( z>ru1h9bh}f7rr+|h}LZ=XbWX!=Kpp#i_%$V=SWez_Nx*~ zdi|5;rS1jKVPTGq!QvLfJ!U(j(Iss{=VGsU@#A&xM7%8RG0-1{#Mm4I6L~w)xRBXh z(oP(A4D1ER5ef)>?L+|CtXo8&UOon>;Dp@Yws`i38qkoOmOuRj9MqvT%^Q6TybvsU zjdmkszOh~)J~4<6sq2x= zYdll$(V1IVOy4TLMsZWBce903S3&TNE~uG!fsoT6O&p#%-+QP`N|9O%#&yi^SVOZh9u&4Jb^zAX4CaAbg;|2M#SG7q)7- z=E3w3eLz3wqHyry`63@XX;_Z|XGta;yzM2PictK6T8U-BAmqpFDT&h>qipiVOuijS z^;gEsM6PRxbb!Wg|5MkF|HB>u0?nf$Eh#ajvc)Lpn;SYK0WCC_-azUW1Gys@W7VD- zcYQO5C)gA;r=(eA7R~JK4wsKD6hvogX-yw+U!P^iJI`0$x&!mK30AMBqRaabW!myc zu5+E|YVI7<<16Icqdi^W(qg@r-UgPRcQ}7XzvQW$d#sVgmR00}XSFDr4`q(FrSt1;{mmx~kK^BgSPvAYkB^O^UGY&eR+D+UX{y5+3B zO%Qj7(*~(w;jO;<%UT1nsrTIVkSv)~S!El}k1osPuuDDQACEg1b?ve>c9w@--GY5JD4nrM!p?4lIR=kE@x3+i@$ zZ+We+ORGXoiqvZGf}`cIKtI+|cbd6+tah0ZQF{1&%mXom;>U~IL9Ov`v|I+CpK%ij z$?zUeT4cKJ53$7a;+8SX3$4K+<>^73&5nn#pxtqVvc$X4aG%=O837Kw)1^uL5c}Iq zcd4xJ*Tz|W0wMp2`xIF-{w_`uv$jz5d{c=9E1(*#SNw`K;O>bj(@r&te$9xsg|83Q z={Ynh$iE?jp{#yF#1<9Fp$1yg23&V|3^dj2=sBPvyqDt$Z;%0lKei23S7JTM;>Foj zbV1XndRSUCm6+bgXuKU5cxI$8MJ@0#1JwV&693@f6~-Ec&Jjektt{$H&gd|zON#DY zw_HaaE(52{ITGxGI_;IzM2z6!zOk#v0D3y%gJIs)7f(L)ofxk3Sm=x79K)c3_!3Hd$&NVW9m=kbK|L>-=Sz`6<}fPfoGmcfOaayDAeVMdh4uLoG(_u<@Y* zq*>~>)H!(IF+lC!u?uH0pR|~Ld9%;`+4z`M$i3ya?ecB^rI|RlgA8hAxtF13v6Y*qyLhrpLb4#s+3!i7jqB{Ho zKMO8aP$?8%uCr6>>0$_Iu4K!M8yR?dX;53#>v8d$;>~ij4yPBy?dTHj=3%RCf1Tb( zQ4MmzLH0s^n>|`KsKu~{JURKD*aX(P^6^?tKW^0P8DpKBm^gf|nk_C9!@sB0E9q_u zaSSls5EOoMcdmtSWF0*ZjP`w5HfJ-m*wJ_GX*}=>UmM(8jZ=V+NF8`Zlbi)$*X%r= z20PTuwds5%nX%&f6lKq=7f8CN+}4nqu|?}sD)SNB4bqK1*!_gHjVld(rQi2Fhb?64 z*=rO6m5mQFmr)@QaCvm_MCF(Vc=k?TLRqb0)2+-&oQbuKk6jht=#uG-dPv9Gk;p_V zUdl}NP-QJUxztZWfI&&AK)m)%xp%gLax@-kb_|T(Cf+ZEFmZdSPwf&e_)xSmm3S{lk%Ss8F4 zrj^s5g+-k<0|2;zb0=DVI#Qr^NCj+T$L2_38s|bX(uQKpLigwmz7Gq82?D>s$_&g`sPmWTu=NK%}D0TSDdZgoovidg2vz#DoezLfd z$o*0w$fShc2&?NZ^ZX!R0;z^DURJ4i?Ag&L%R&{P~R9SLvHWAog~;MF2_$3zEo@g+b`5+NYf=Dq)>lYNXvLT0rLg zgrTuEv}o-Us`OH6HkYzl^Ec{D6qESaD{z|qCpJ)2$jmXY-9P7%+1zc&dvnZ8NHGVV zXJ$4ic?{g#rO$m-_I4DxHnb}b4!vM`2X9yv-c~7RY;+Y1@!WbnR=P;>or*c8o$J|F z@_`Fq;7!x1g!@yO-Sxhyo3RHD9}=ru!*t|>H$05-;Pw&?BbUiRG#c3=I!m^O z>E5|{XxL0qUrFy758duRnUYvk)v*PSc)0n*K&bWnrD?hMiHSa5@Th4RIbEptli6&8 zYFs-udDJ*)xp9Wo#O1*vVS0Lj&*Rne)MD~BG6?dk%~L9F#4FdJNGQE@295{QyVdkba{|HP>w_>y*;O5Dhk~h_5-~cGK(%g2BO!HhpXw`2Ei=> zikX{FiLX`a{$SQ!g@y1QZ!-^@SYD7PM60kk#@)fmsU|3*LND(YAE>H#6svtVky_yX zrq@A7w@AJgO^t&!&ro1`ljgQ)RvxZQs6;>fPb3DDN zDm1Mx`p)ih8<-oHw-<`pZ*n$h@?*lP)pZFxDV;UFk=I3?{0&l3XCEmOWmC2NDa+`V z$cCOOhb24i7{D$xa`IFLDRE#++iR#IZBXkIo{95XU1s)PwTkOu3MV9%ZLOYtxepWU zr^?358gE=V2Gq}BwT=Ng26d7?e104u`(iCgfI|dfMlIYaTgzNAG>W zl36KO|?LI?#Md?B=KQtP0#H^o(=;6ac^fl$4Pr|*Y}yAhwyj{EhRoC&+4 z3aL(~MHtK%aU&Ek`w>b`a|sx~(gKsL-QE}but#Up1P@}*Tg(T(cuVVgjq}yxFFVdt zs+VhkPOS9|UnQoZ^buOV;Qh$_&?e-T)_c3a!j=d3m_$lq1NRoGC?e*f%E)%M*3Td5 zdS~vXV%S1FT)9^cm80r2R_6ybWU8#gBukZU4k(xkdeWQU0?Qw$^VpM%?qLXpN@hnF zR~nc~a6VbzC!*@-6~hy~%jJcVzV3xi}+i3V%Oo!V|mG>T2eJ=|ao#{93M@>i~@;X@Z zRu7e%h!=WHrc6G`st%+{FdgBK&3(&SOhp;rV8=m0e+r!Y5y$|)Mm>N}IJM7~?uYv^ z+gs8N$%xh>xg9H6?}R%VZFgYXxhulx)f9Bkw>B?+atUt-@j-O{#VbcxVB7bmaq*9^ zPq+4b2(KErn&*!J))jqC)*N5&tc`-{3xc=x_(ej-1C&Z?J|YuA0jl!pOn!su7`^mo zcaZqA#{h>4HO|cdQ&4-p!AC7LnL_ZSb$J?0IF@QZGEvYY16c41VykK{4n1Qx=itfOF*a|Kk`7xj!jbGi!oC{J0#`8SplsL*>;c z36+k|vWjbT5w#&Cs7)Wn!S$*njlCcAn%EBELRg)BoA1{(%d>nQEBTq4m9CmSrfB8b z3DK&l?RDmi`+oOnix290G9b7ApxgNO(GKqAFroln3Bzsa6@_?$o;}(2&}((#J?rzd zH=SLo+(md9()=4DOS>~CbnY&dlDi#SAO34XpbZv^xuq{h=FoN6GUDs%7c6T{rdN73V*-c7Ang>Hzw_u~~mxB;XmXjJl2E78xr1Rx5D zcHZl0VBv0uTA2l|%_8@@EUsOc_JfHM1&8}0{cz*ly2RaK_Jm`glvKKa z0`=9!x9e8nvF&{5Nr2mTP#Ufk_+>kar9xQhW45VdqRU9smPMAR=%iiFC@u2^OM=}K zqy^L<9NXRsXF=%H@^WInr9WbL(R_sPXM#+~s#rUmb??5~(& zM}`%a{4AoV@7^7W$`ZRv0&E7%gg30IwW%ab68wy{MnJvLyH~z&tBsROX)|nUw~IGw z7WaL5>!JEr%ob%c^@#Arn+7X9gS+2n=jwhqmC_}C^x5_BjT)Vo48~LDelMBvMvG?z zP@Hi!tEvpNv47g;_oZRH}RRdu9=syx4-wY73`;< zn`+N$oZaTpF!eFObb&_TEcy$OPDGiXe2(cH$mG6e;yERm>DW8tboj#zA}3)8 zb+g*2A;zeb{UK);Ohza}+M7lD!)cQ0#6GOGTZ*S2ftxDygp)*PGy|nY9-CZeKXu`X zXDDnogZdb-p7p;pTRPjM1!0&enf6XMblre4%_i*dej{EQ{<$N+G9z0vGvWVyZCjXSYeDQ`M8W1w0>$e z7g5oY&&r{yB0}nj!1G{{8_(yUj0h&|a$7p11BzA&lxyO|gtKg-)|%<_OiVFStfVZM z*SxpG$y3Bt7S*VK?VBz+YOWN%?as?aN?_A=9RoU-0|9U0_2MI2ye=UIi54cM=Up>! zW+Z?1MbSFknK)HmmC|)2nZ9)3cMEF;wf7yLej{_z6mz5fdIKY(yk3GoA?FgHrw|PD zuH!QhYI`U%-hkw`G2)ONDe%(K%-6_!PdfW2O(xjHA>*uy{8lE5F8S-9l9XNnM^Yqo z9Zw|0iIdPGEx%bP2a!&SkPgE#>pSkzM39HFexrcvezLqJO>ErHpyR9$%-&!jf&9`b z*T0@&Yv96}qQ}ovhX&nGxW-7YOir_hK<`>#*Jg=ZFbh(X#@w##x**G($_IH>G0C= zPkio7(R!*(t0->LXt+)+EVzdHaS*#3#7i--XLTHBD!(GsM&>nL(%tx+TTO%ZH0kYA zq&{}{bm)>9Ti{BM4ESYyWFVc=Bpt@T;gr!$B~{+!w3h_Kas z<{%1xQi1^|dbaXW=4v`QUj4nG@#%5P)zOOF&m|h)Cwwx_Xb3WauAWrvrh~9%RNz|SPmGtSGd2J)0nCJ^>8X%Nk8@w!s@PWuA8RlW8 z=p)Iv!lwOU?b6;{NJQf`!Qvpn7w&f-YBp1G$W!?H3H~cZlGsz=SJ64u;EwhG4QCuawJ`I z;e=MUrP-R%^~o>X9eSg^{vB^`7IUTF-E9SYj2n-E;8}`ez?L)zo~CX!M_%j1=$b{J zwt!#C7%i5$Pq`y(Ds=@W!UC1#wyOsaZ|5M~#BpRLZ(gByI~vtu;&&RaFtnN64%^;% zJ~gFLfp0LtE1VDAwjM|KU$>@aiTTmv^&=0fhp z*lSBoFLDd#1|<_ZS1l~QZl1fs#}2K4Z?GMy?h5m21(r30;CPafgAZYi-rVJcCHvnnO*t=23@_;Dd_1f!wNsVXb^?#F<{m~u}`c+isn%qB~s56GnP%xxx8 zT%m=jU6|dV!_e&o4?^uA6LFTMShbXe6XEE-)7NM-IjmL8&)v-3WZ*w|P?$A5tuz0E ziL8e>0xgrh1FCVeNWzV;7fU#D4 zLAdxFVjr}`=wr>L)VErT-p%tg7GJC%8)lU5BJw`kC?>sHKbLExV3?aDT4Ez%(M2Qy zAOYUQzRTx)VeL;n3$UOTIrZ-PVf>YxTidYnM?J+n#aUVf^#DhpvM4=Y-$da38xj7UK+rMp zyA%ZeS{J{e;Qjj)en(0FD)Zkw>F;XwFD#M#)8BH`@*lGOB4xI}Xz6b$|2<{=dxyU@ zfM4i<`mM$MR!gqGI}U%VrN0Or{P)J;4EhgJ_`4(h*J}NHru66+Wd5sJ@>`jI*)o2| zikJUmzWw5C_>E=!mDMjz6(NsK5<58~7@QwrjBnhyd*y10B@(mHZoqUlo?7 zw(G6->~M`oVqb(?B*%8zCfYKy#;dEpHt?)Tr{rmUFHNE>7GyjNA}8EiW;*@CCE295 z7PciJds+HB>N%XvkHc>aN@Lt|$Ky>5*_raMKe~b-j)Si8PqfEY$!*N(rP5RnUW&Dg zH``Wu1-G56!eRoi zSH!;gkG{@F@Yna{z2wn#>LVp1%e>g($+FDRCEub2C&_aFch6I;LCup=$1Aj?e zbDA)w`I@hL!*%_};7AvCG<)XUhrufyVrue*hf2=$zH%?x2FjM+IT?2U(CIT7U-PMO z;%+pw?0xS+&>-<;i12vBZi@@J{(!b(!}5Co*z(ejKF%)meYK?|?C=1(&Gr==tqyIC zv!5-!eQ~1>>lH>vmXIuZeL|(diHH)$jl70|UpX5<;KMhN8zf*XhFhxENgwWYa14CD z+y!SwaNzFQ?$zkL&CZNhY)z7r?o_w+=u_yUSLGev)bIB83U;Wvw@g|$y=LMn)rN`R z1})WZIifG*4md64wBgeC64F#L-q%QO@0|JUn~ICQ0tWtW{Rsk5Z?HQD(IL|DX#XtN zEiELT(gBCMmQZNF_r8IYcv5-}$Eu*_Q<{-QtEDqty|Wh^ZtodFM56CfB2LO(XtkA9 zE);Qr^<}>sN5m?1#@iY0XisHX9RrY&V_?p zAL>9yMhNSHhBJ7jK@WnW&QP7S>$@)2a#wokq=lNA*Cin~@t)3Asx)^Wb49)Ize5P| zQ}QDPdK4%$3HS_(!ATzjH2ZsfdtQ5Tc!L_xygfoXJHhHUrm+4)MQYy4q5GW5l1D68eBiWIf=P$F#rlP~ zt{|=uKJKlJz@wY1>1D@2TdHfjuKC*D3=10>Lh+sB)8{(L?kA}4d1*Ym_<1MABjI{9 zpW@xJ6jE$AwuKXJ4^k1q$?e_2R0gYvn&9q?IKgGhWc!6G5)@>D`89-+&Gbixv;~w( zo)k;d2zaNCe{eAI_l1yrI0X2}+F}AhODOZBBMunf8adR-y}r!)c3xX_-fM$yo}G5pPX#L$mOq6Lx#K_!*-UaV_4{<6{Oi@~2^oQT_B@lVj&r}tv9cFjCKsx{#ohn6pQ zGA)5YwIP+RkV@u@SQbSj5`bWQ6q zG^x4qH%VBFmYWA0sd8b{Qz*#cL|Orrcr47AihWS5_O$bv3|kMHNDeY!r5u zkSKEv=ar}oGjWeJYhfK2zivz&$=SV0wmYfBN1z0JF_RUkRlV|kxB5X{@DkWYyyOPU zJdz!uh}{o#pQZIqY~H2}zsaViULC@w6*w`+YM%<+1!;on0S_dJRcrY8^LbEEd89#G z8b!a7qM+yCl=eJ>%DEp08yfnzw)mZmC#aB5IlLEjL2bz5`29${<`E;Yzk$A@y0&IW zrswiTnIA{v6R{aUX;^tX`5TiHp_Orx#o!p&(?X66292Zn9C-D#!ee zr8iO|7JaWQ@k~4>QSbRTQBk|QUpf|ZGENq?AxohFco237JrYL>*U?y`>IK2C`9*cg z*^bGw-!erjC*rzp#Pz*W6->C0Q9&k}_DGcn*5+g{ftt%j(ekHVSgoPZ2at6!tMu6z zK1h?l%6F`KuuM9;x04|zq5gq{%(P>tugbOn{$Xbz50b@76%v=mfkPJ%yHR8+r0Lzj zhq{Q1YBs)X8KlMHaKBcZVKF8uH-P;`uKJV41tou9XEf~DVR};^P8bqWM`ve`!96#? z*kF)Y}*r^Tj}w2OdF+m!m0h;Dz+&0^tSWjmG%`w>HfjQ@FQ} z&ak^vkD9S-xo|=l5%G>pusD$N6OI0pR=}MoA0Qfy6Gi$jB$0pYPyGy?v5-1=MWi6d z>4Ij<{wL~6$hM6?Ya-D}C?`_gX|MFT3XE|}8Ilbr{`7{kH#4EI2Xv)Lfo}X>cx^vu z;_s)Pe*Lr}FT>G8yv0`{tzS1$66^QOb7W$;`{K%F>&}M*u*dV#{At}WNrG^9W-{YF zbKJ!b2pZgZ^en}l@sRSf6J!hj4HFrvMd;VT-7d3h9ynyhIPOE zL;Ku64h0Xcc6R&q!7DezCa8~u0~?fA67^Ou`BMhUrpQV?=>%afEkd>|G<+hSn%sQT zU;5oud)nm4v8IO}#_~PcGds)D9QNMKV&B?$+S$(H!-^X&A7pdd9y#YV1ozmwMs-s|F)2IW6R#Z_FL=$v50S?}JQzeM zI+_iQ`REOw_biZ1&-M0c*^jm2+7x#YeDKufS>fda&qra2%1>pjkE(i5tsI>R{4U$! z$g_qd=myjd{UNlLj+0D0214_Z#cq;^pqyjCUke|4Sns-obOYm55gy0DRohxpl)8e# zB}0GH=K1O_Du&PvMW4Wq5@?dQh;$!*9$zB1cB8)YW)sOIx50TMMD=|sCrL6K+Lg~1 z<5`s}q^~)&^qI6%d**UnvVM;mUq#L_9Rp|bm7VaCn|9-d1bSbMKalR8t{H*X{w6j^s3vpU=2b=fHJNx`^z zifIKKEnjV{tZ61v#FYOa37=7$d^LZOe3Zz8U?hYgyui)lE_m`IL;DfbDcKar$vbPt zj4Q%dIpdpO-_o7s@?fKl6ARP0@wm8SlKlKl;E0XLO+ubYXFCpl)?82}dmRIO{h)zk z;1tRROsWSxjkF`dXAW4UfoWa*CSrb_I<@VW%D1g-BP#2!Xa~T_k zVpsWYn~Qt|xPFi-6+`_-kfmqSj|>pX#Tf0rFesX*X2?ThSJC;piEnvYZrFhXxc zg&jAzGU2pVpmHgJD@TLujmQ_S>wS3d`vqS_V_W?1;iQQ)D<4-;jPA3c>1Ek%1grVL zZ&3sF38|0B)gAMLMM~IWy3}hKmNFt(A(!Sf_;{m_jENn0yfzGSyjh*$ZAI@)ywz4$ z&GJl0FpzRNwVl&r(inW*)|q@PcaKPXv_0qVJ-m-U9XB8O?Uf3b+{TmnY@8Dp8E>%c z`=~D`DvZFp^HLdP&&$s7G4cSuUYHa#sq~1}8+;4rHYcvU;T|b&X7~DNDOv4)pI9_r zr(09lU;gKdczApPMB>p6WTO&-5}V$}gujKc3zx)G&L1UF>>GRPKB?H31|K{a2vXD3 zQE-10_k|)>1C;cI2&4_E{a5yJLPzsiGyJ_p5b0bqJG26Ki`jJGb=W#7mt%h*VT_zm8Ym^IxJ8RMEL(a5`!&D;Y{7V)5)Fx%Et81D4(_*R z!YZGw3ysJ?*WXI57Co44l}m0g@lKHM7@Iol)(1cg~yS3hh9le9IJgm60rWF867>gaBh=_g5ctVr0=1YJ#-(BEj`kdUlzz4h>?$t5C zNfgCv4f_dsFG?#H?Bdnr3*HpYN;}m>3x-u23I%*r;ya@Ota(I_-*%iW?+rY<4dFR} zkJb2$MbaHfdZpBi`1q1CDLZPjyGO-K>9r9%FUhw4XWWn-xs8pv0;4swc5quXRZev4 zEOog&Lhr+^%(xSm3AvX@?{l$XvA#ETE$T%-Xb^f5;gN~Z#;^qw&v-xCcbMTH%J}Ns zc=x7Q2$z)=?p5r2PmvpdyZe9HV*<6$Vs3k9rtEnPpySB}a6>nZ4Eh5~Pn^Ffz2b=J z)g`$bN(=MIHhDZ*?#c|u0A3t9uZ`4;m^z#8P_5zbabkH4V^dc-+X@zc}zRb-j3g`jDKw#4w4_p1}W4(sP*luT&{%EVZj@Zu<&ccg=K4f zRQ5I2#VUSZeC-|!hqAk<9>hHMj8~YDe&bJ8l>fHzz}eC9VcvP+Vfpd>is+&gCC<8Q z5!rGa23lXTk!g9`qTk__pfGWdQwTFO&)LmZe|?z+p}Gz)`U>kOESTB`c%m@m!p9bj z5l=HA?folMy)BBOV6EuwZSRDV#I3u@+co%2U)WqUhkg1fglM*RdvnRH3BN23Z~XU8 zjfi}`f#Ci4;a77nH97?ErU$g4Mdn1iz-IH&uJzF5(H)f{hL9kU*aDcXi}A%zf7x;W z?9mQT=WKc7MZbP%385P)mbVbfPk(vcvw|LaO0*JGd%!ug>YHQv?9Y<{HP4pay0*}_ zF^#F5OSparZuS9T)2i7OjNa6|&h+@MrvKFqF9*i?`$qmI^Fg)%EpuJZ#%%3nl3XqQ zzIo5``S^i4QNP3JJ&B|oIwx@*wXutywR9Ckr8aixBl?M$TEL@5 z9nVs`hr-Yzn|KzY>2Xk_;U7*p1s>2u)iNV^cr(U{}RLJig{HA zDv+SAk-iop)b-IWsyOnLu*84tZT%`R?DIBURHeA>YXnJ9>g$Ymc;qbG8kOmtc*^8GgF=|7+fsfCQ0>uO zBycJxL0Hh==MSGh`}Gjy*IJSp7!8&Yyl&PH$U5DwF8(UMSY932-X*4g-q>LMb_}i` z)CQJj zYNBf|Yfy#B+jIGPf6@LL8`nnHm2;+s*zQNX-BNO-y;H26eE$X6>fKMF69tCZH)e^T z-3n4YWxDMymMOt3;pPx&Z@Mv#^ciw4^Lo@zNI*GMH(a#G*!w9C75sp)h?&iW^rdf{ zd&Oi;SA6v*vx|MZUV<)Q6S=Uxf*!4S;SAcTEJ}2(K3Hg0y7!_xukJ&+hthSnmPC;p z=L5ru8aUG(WpiT@N9a}KSH54CRU~i+nB~Sa=pd1Eb)uL!`8Cra!ZFyI(I4Esyd)Wr zZ$(jctZ$8him zLBjX(F`!>wF;q}6vFY^X!|i}`EM78^XA})j1ZbW1r*cZ}-COk$dGu%*roA=T&O-9r zXfR-6c(6YdlSSH<7vi%&<>8Aqr4EO|%kZa{>cp{%=%OgevqN|mEM2n-Url9nZvum) zQDtqc{5I8|$<)hgK5J~Oh>gy}5zTk#JE`jSnou?OVX9zK!mar#f^^RB9eNIoKmP!Q zTXI?_Rh@_)MW~ctI~bjbgq=i~;S3fSwB6yu|?F_*>) zH8pA_TpBKd%1&Ws!=bbdD&8r#NoC#MrJpM*?wa`)?iRQf)O~yx0A$7A>RzVd)%YT~ z%p3pXz|#5EJ%^#s=+(-C_^*YNz6bi#X=fG5UK_u`8s=mTkLTIf!>=O<9&V$_hemxo z0voeq`cH&$H}YIP??FN~S*|EdsPsH|sH`AJJd3)xAd{fNO1yEZX@k^yO`+^LqlE7`>Ftw{mnx@c_G4k z(#a^JyLs3h^cTT7viTHV(tG9LB{?LUgNUwsTJmCz1MwEN9uQjZIIWQamWrewYaN?b zpdokbh_7V}RTW(G(QMwZMot5#1{thMT!usL+G$J2#Ns=jc^X!e1n*WW2R(UvO!`KvFtL>4ze0Bm)^l*WDVJ32%uMxMhRr8<@n_EgHwxa&Kqz+@VVT{**^Gq;PZNfw)b5Lv`51426`IOd+Fyu9WF(46rNX z0(uQ2mCf{cj<)NhEEGL!X{>nTq{|HIuX4`ZDNv5oJ}@;eDqgv)Q}ljq!kx1rF!zq* zB{tCpx;)JhdmFf+hv0=n&Yllt)=5NhYhwb>1iJV!>;rX3br z#=Oo>Iu

E4|EVZEx-`z1Q9F4Sxy>se{hf^G~+$vX06SthDOrNA@RtJI&3y$ytO1 zuNr>pywbQzy`h=SW-D;jUqM@VzOShs*;0a&zKTmiYpb`MQYzMZKjCy~$Wov+UG@&k z?n@p*cY^!<7B5H5FAFJNCQU_z_(kpq8sG}HO} zx;ba6KyN>m)`u#%2F!Be8L?Z9+3iU1Qhl5j+vD-mUapH@zO^w|cEz>#D&*#943!8nY-(JWVDSqtF;(TqeK!Yxtc9=jK()&& z(r;I>c?~wxW=`wyM=^@Mr>_lp8>ZY7S0ZB>1_0%MCDWgEbpMO}KwX^nPi`HBaCAp{cp<{uvPQ{(-D21L8@xyLCvo{+6~y-vkNA-lFQU{QlE4{@3~&(g z!R)x>;MPr7*6%M_!oJ?0SwGR^5Z79_oOf08022FtZZUnvmx`*Ub>dZOHYsBZ$H2P0 z10pS%Tp>3lyp6F!btE{ev7X4GxLp#w9(r^xoh1tuNt8oK46WxE+z;}Vn9*`l3pNom zJFOAY8qX{e8Ds zBU=2WuMOu`R4!qb%0Gl>nVI_*F5D`sjA9OZ_?VX`W-&nZ3`f9Q4zi$AUPIlBXN4Z|LU0$*^_(kzS#5p(O~g^=g|m#w!o3-fi(7F;g8_!Lpg z?JdSa`&B@pZOgK#D!rU-T^qC22%+EfJBJ4i72zf4B`F~rL97$+w|(DHkDD$s!7kd{ z1m5nB^nG=UWYlxtFsi(zQ|qck61$eh<=H@2OdwK zi*9p$aR(KqBIGT4WMxo}6>5E9KoZSja^sn^vxUI}sdwqg#tH&TyPw%N+wP0#C+qrx zMJ1Z_iZnal)uOx>pTaiij;QezLqsqxr3DVgL*)W@i@*;1oZWJoAE&0?ueqM?7}*wT z>3++7LM}4DpuFU^)TP@O<1Mbg-%J_^>-s37|B5AfbaPj6YYa9&8mc0Jr>sC>J%)Ss z3eW^K-;=&qzugn8?BY}xdt5zn63xBBA!hoz&Ej=waua&QT57m6=%663(dQnejyFx=XU_`vCTQLh=ZN>b#wqZMi7EC+ z7B`E`Vv!sHY0;~o@XgTt{)V<6bW~@imSBLM9e3Ju!!#r$tEFJzO$MqFy%4=2tFLE; zqI7JlnKaZh#Gm!wugVI^6ZSg$RJXA|pd#&qi{wQ7ZrNEUD-i(borI^{$0uZL`i6&nsIl>((M7cRI}OCx{n=uB)1c z7y|SdO25gVrv65QJrnPb?OmI-oA{aRZytHWEqxP)aju6)i7}ni=DXi2rKKbapYG_7 zYqIYfb48Km$Y>k|?@11Am2CwKPR~p#={erno$kr2)V$Alh4J1-!+EyV52$3s!vaur zv*4~?te?&bQ5%wnp;QnPHyYW1Hq9n0?`VIeH;!pBS%0-IPyxGkFps)`R$BEI>rAE5 zJh~hmU*POzoormbW#c!hQY+i2R>5hYH4|02ZC0cO*I7P)qkvsljb7#LJ= z+NXuKMCV{*rXj+9Ewd$T>nm8NJ!i~Tq#qw%F5GBjmu72letp&WrF6*V()IcC@oFr$ zAA)YKv>4~mM&3x)(bl>aiCe)uZzer5dqWh1Y)J=X#SOIKY>w$x_5D2yy#=&WW^LG| zFSb#JRW{8DXW|~eiB8Dr8O;__dB)b8Y*&wLgrOEkTu8yPVm7Jd4`zafhG;`W4%z0G z@v)Mm2y3kJ19C;ogLHyjYbtR*y@iC$8aGEX;?;)dVtwFXhatGq#h0A6Hx8^QFm#uO zzwEwaT#gN+C|pMf9EtmNg&&#msPCl`vbT15o_HL6nICkIaTBT&r*OI5uy}WksR6eT zO8aanR?DTZbGSZwb0ztvdBZx)5l2ddQA`n1m2A){d9EkCo$(y=Ea3!@UTIxbf^3X> zr%;e4hpFaK>s8c|;j4>={DciXk@pEpFr08@+ZH=+w3$~3Cyl-7RQ><4_ugSqblJOT zBO(F zPdPJZ=1VhYX3pGuf6sI0FCMC^cI~R#d#|NXAWTve7B{4n7BTMa`YzO z*)i7~^pAUOI>~J!&17;|qq+<6?XS#qBT*gVjeR|vEWARL@yF`^h%8(T*`H3x{v~HeY_+V>vy)5 zdK%7!R-6^CLT`MIyWNpz?wFRlK4usp?2~NgEj3H~ZK~4!&{*qI&(-t-c~Qoxsc}GX z!5nW%;3XoV4m2*5?TQm&Ii)ijhkH$LR!5uH(H-SyIg-lXpHiY}c=C|jj@-&?wYONO zINE_tI1m7vu&X#BOiNoA8W13`3TSB*N>Yq}yD_Rm{#~ZvzS)GL*LP;7s>m|XEH~#$ zL3pNoPwly;4x^DE3AiO%x(RyfyYJq-aDil4yqlQ1UF3xJR33a&y{te<&{hQG!(aYP z$Y8fMDcxJE2ZLzRCy@odp=<DTNv_OJF3wa;Or@y`E;D!-6!c928J^E*spcTF7S)#WY&z-r%DP zuXN){pQ4`r0^%cIa+}y8>5OdxL(q1G)aPRLNp0rKDAH)_#y)Lq+E0qvWZfL>bG+x= z7Ox$~?ZpBGv_j%OuWk;Iz|($dcMkmBs&Ac{XP69Q-E2J}j!h#1Z_Wuu@si;8Zo+S( z(op-Y<6uK$x?6txwEfpgzIlz^_$C?F`zo5-FF{1-e}@PGNDkZ)bt5OpV0@zW=jx%a zt=EI`O|H9$%_OotX#&&L7Kbau#K*Ln8vNF0*Og-mg#8n-VZlrXX3RuDQ>0+^f9%E$ zN2b{Jc!D@UM{>bsBkQbWJ8adn4BEeWv0Q@#G*A9GQcxusR`QP|23fq+Om7ZI+#iw8 zyfNDRb;?KA-Aks8*>)uJIVeyU>|mhkr>ky!zR`1gRM^Pi7cAbxw~_vid;M+m*dN+}YvZ^(jo9w5}!C|WuC z*p2DUI@^_L4t~7a(YzPGhM`Yf`N~`SdlSK`Fzt~87jmPIqm`0+8+yYMi;R;~7Tp87 zG%-;zxh#;`e#)7d@(u!X`0bjJ$H;IfPchwV?Am-;C5Goc#Vej~VuA0^1uqDa^WP0t zG7+o-1w~j*SjXH7FSt zvYhXf8#2{Z<8<6c-hy}FbqNmupWk=61AB*{>I9tGIN)PAU@iX$osSMxZrUOWkyye@9SVdlxi1!y$qZ zx9nmc&K}E+>{pVo5&l+j-N2(RvG3@)4Ts`gkx0`)>gfbL;P@|^~!a->SX`3Tu&0-nmiz}l*$IBzWV;QO%-_Sp|m0wqS8)Jk;t0lJ=7VgYW8eXn-L6fFV} zs|Dtf<1^R~xO8;lEbEKcYpC+WNL8J=Eov@&7)FrkyM}|H=DJ^AT=|5uSiAX1%r0e0 z>q4B$#@uVeLRUw{p<{>m(DvcimoBO zv6Fw;TM=4!`r|9DcDpUl1T8bm)O*>5S59|}YGz7s*Sy{Sn*31Hp6-VG4K*)PeJ$7- zE{^kh0~f~mWfPRqXyLA_m6q-TCpi7IB_&$zW;+E`;^@e1DA`LPxl`IojrpaT2|2n( zu~+wou$PFhgG7KU?cW1l2cp6!g2eaj`^M0n&Vn_UNBm(X)zw$YcDL-gp0*bsK4}e! zd3OBr{n-M!CAmscZ5Aszi(#}PyOGD3*Y2)Ek#E-mcqq|nm)^@p^nG+B72b`RzJThaFUd| zj^{0$n+at@R`_f>&X-cJ9T#cO^Qbf*j~4j7cTAtXKWs5!%GgeT*wS!2ihOOo5*-RX z-)_tTzmhnyx)#m$toerM#{KQOI3Oy4$nl715mQx_-Z_qD>9*JSy_=aqz3f@ht2VR@ zZxWQ=E}pwBMcJ8QmvBcxchqs8)trS}#<}&q(s8aQG*_HP7y?a*elAkr1Qmthr!!^0|XyIUX z%rech!F&VHTLD7At|vp9Lm#7^y%Ucgz6dK^xpVCOl!ThKwle7`?!vRexs%}2e3+NV zRu;h86-k)Gjf0Q&fbskTba-|B2WXi62MD?V$X~==#{mtzd13h0Kv#E^1q~ z9Wqa;G(}H8%6;S==QkVD<6%?p?v#|w2|3g$3{u^lM1-!pXtm&!O8nL~fJRDtcu{$d zY0_bjUqI=7J#U8S4awo2v3~pzn*=H0L-yx!iZ|OxpKx^Dp#LHT^_yed+%qk9n)xfdfVZJl7 ztl{(EFjVFKG+n<`UW26z&*LAU3+i2-{53NfgkFw@YEvtxqy>dMuC%WPuv-QrmGt1C z_8%Z#fXf?zeOpzHE6#VSC{97nza5rxs2P~@6XHu;I`jHfvO;`t8;2!-ntLfFr@!B+ zM3b2CegxX+m2EoK_*0q)U4pZ}r#N}sWr@p07m%&|$ksr5v=2#}?FkS~Sq?$>QzmDSE#*o*skO3 z>97O?AU~^S;>Zdkz~v>hFzopl+3(`~un>S*d~=F4jj9wP#kuFT2`#cw)1TVdS_e4u zfm-fIWA-lRo0k)CXJ@{B*LK8vkOmYiI%x~K`~p;IXEtXhPDTZ|_(E@p17tN;ZW43g z((U_NfH#D`LPIBiiGr)FvO+@ zkk=gK`fB#4pkvu|O9eTO!Vcm*@z*U8Ileya>JK17k1jlUS|lwP-hZY43#F%%;~;TM z1or+keqc!+;i}VxjE1t4n1jAd2(`TQioOiLJTJg>*0x-C&O^(c(F#KE&ah1@0o|Sx zD?0ba<2;v+Gd)f$Ce_yd#AP?PIWlGg%?s~d*v#Fc`wXH`nW5dI{}4EE`@X{~0)mKy zljAjUd^mA{qOZx9x@8eaETs5Ujy&mhQjU( z@>{i5pluVcTqQ0Ky`!PKyH*{qnmA#RZ?trA&sM`=ki+VhedJxvHNzaM@j_Xh_JGyC zyhaXq0QztXj8}AqGVF7S${o?AzcX7`h9^*Q_U>My@;wYDKT#4>{3O}y3niatZ+f1A zM%?`TrmrG_F??<&yr_{8m|slxgIkMZRk|K7O41zU+B_K-TCY90{*_!Xfwk%A!z=uo z>UQnD^ZQacqb-QpY)(DoYv3U>;Q@IT8g4~B^Y$M<)m&@o<5u+6Wvt@5z3;;VSLQ`J6{oYCK8{6vrQ8C?@(K6RaPao;W|9qiw$b5FtOX`?9 z?JzdC@hm|e-Pha~#!1<-L${WKHgi0N>wjgH2mJY@>^=KL@-y4x_tiQ}{CaxzZia1d z!wt}~pOxubU^vm0SXI%vfYU{fa|cSKU5ek3**?47;`r!GEu<@C;iTy3QS8sejE*R9u>(^XAcUD_~f;)i) zMxOY;V~~}d2`GvB90o{;P1IjRf4hy7Z63U@;G5T;7XT*8CBwFpVhT96FMCt6ta5<` z60`*~64k*rKdl}_IH(Uv;YQE~7X23xFNO0TB~G;U)~_~pSi(rZn^)z zJMp($*8Rg<0=qW+sUL{dQZ1k#R)6Zp2{VLS?6|P^HzeB@u*|t>51$Qk0j&J`G$pP8 zJvM)v77C0e-UkpqhW7HWX@LY!ZR%>6jJ!(D@r6^t%J2Wd)_gkvoFv zXVL`BS0?-z8KDfqP)-okE>lVpjJ3g8Ii3k%h0!=3UPDyL0oZBl7ubS2@BfHw)E_3Z zD1n-0w4v@t>xx`n5o-7hop1j2k$dhVJociWf*2V9hRwGdS3zGdpR{98r}*$k1u z9TcrWK>c!nwmpdrAoI)DI;OZ=cDTzu3>vt|5X}BchtJicEDz-8(_$TJeKtrM7XzgR zJotv?igyr>R`KVvSKJ%T=@{jJRLFClT8uh4Z>9nOzDk==)Y8B~Zh~RMi&m0&#p9Zf zA72RQ4J{i|mp>)fTi3jLiJJb}7g}|&El`fd%g`9oVvRPE=4vVxYD@Gj1(5PLuUr;s z;GHhE0_>Bn)K459D>79-mZ+K99fPMTFe9rl8^H&$$w4B% z`OOpgTYG4?<_Y_%oQ*Wz)S}M!Q*BF?PeeX^at?5$Cd_7Q3!VN>QF{gjPaM~iF>~zP z+&ap#cWgIXcel*lIYe0dZCO`}5aOT_2<&-h=&^?6JuO_>f%U8LNYY6nc<~YVg99NA z@dULk@&lAF{(gvq;0H$uoT(;`LZLlb7w4Qmp=_il5@6xg6B);drk&>lTs;?eTyvdu zrS`}?2J7uywa;Fefu8Mi5gg$a3i= zEG!N>`x3gVe((c?=h{Z?M}DoVpr0>=jbxb^5GXf}1vZZ7)QrIGci4sqZgaGrP5zZ* z3}ze0F1@LqLUm$a<6%siOEf3QDQur!=4ijXX3TL(Tl+D8F3a^xkoO&(;1ANRy09ig zTnP3SE+-S*2H|K-3FQB})!fiZcP6pr^f#8~kFG2sWeF-zR~0lZ>fGausvbP-la+kG zHbZNq`H_SIzrqGS;w?j#;=+)8yyi#|U$CKl$|f+lbHhkL|zM1HjlNzU8nq zZ(Ch$E3hDdCH^IW^B73r=$r}^xx?|*4p*&i&EjinhVJ@)`c&~fwYZlZ%YK^IzlKl7 zXXdVO@F+*FHrwzrF;K5rC+EAAIt0MG-I;^RG;fu<+Ko*I-N$vbWC@|~7fVIV7qjl% zY>XB8`YEd9;7nHoz19`uU@sv(I;zsi3%FYLuR&wvqEcmth}FBGs_H7^rnKEt#9GXMn=NIfwew3xhiyO(|Z>Y-fbq|cpQ(@2x4d`J1J z>M4;$2i&^vO^J6RJ3Z#oQq{XvV%-IKx)I(kr!(Oxj=G6@EWkq2ZBUz)b2;8_r=^a-M za#_r0pG!>2%1+2HR0G6qtnE)_;)NPBolb`hnGT(c6%iEMHb~q*QPx~HQo~FVc#dWx zuz;}$boCRs(p>aaz&f6_s~wa}SD~8bfqZEn)S%8`5#D61F^URy|7P4pr7&45K3*~1 z$U=CC)5W_IkP6asqQOT?3 zOGtlu!wF+~%gn=Q#B4q zOAQz)S<;eb0)1v9&M|7A*DqemwS9ClG9%|odOn+X`^3|7W$WKYW*OjiB886UH-yW~ zGl2z>&6W8;#=&7&;e!E$>zcaMGG61u{qs5aK8BQw+w;yfY( zkhpc&lE%<2>ku^_ZmA08egCRt_w@9&Udo%BG2=dTd3gu&jiT@J^dNb|LsOau-$Y2% z)%G#d%m97ovj~v3d+GMFvFVV`gkwhqS254r3tZB{CFnj&&kl4lLZ@XZ?&1}*oF*{Fl)uMNr5hrogt-FaUxc!Rb%jhL-5kD3FWX0h%gze4~k`xY%7xYcPiRsB1cfP}~ zt7DPUL?WiCrL}RpQff_}wNZ}7!}x~B-n}y8!p{%ajNy$;_ykAiICsO9ncdeH>IcI}-z?f~Yx@80s-1!Syn0 ztc5OTu3US1t!9tvQINisx%bXF^EcGjdrUa?uITIaJd@XDB}tLK`%427s>dtMZsXOT zv&1nX5pbit+vZlPqbE+wbtwsmt9-?OnOaJv_N*4$7v9WwW|!YmI@i4%7G6B_#ULXp=~10od;&+jZN@e7kns*Nz-7)fy>!U0>FSDM}er*yHnEj_9ea zmC2$`7Ch@80>o0|#gKd!JP-ji$#Ce7y+AkwWR%Ki&!w*&9b5FStUXIQCJ2axA^@9g0qGwDMQsXL(uC-JMfHXjXi9r5&xB+K6U&`nkGAGg+8$Qi4 z=DK4;8Xa);vf`guzdI0I@wt>=_HIch%jM7GqC4u~Y`}Z&gMtvN;czJ6j@}0%=jeVi zfPu7!rX)T00b&h;Fu+xw0=Ay~-DxKPO0yaqIQb0^uwVg9>~v|8qB_+dTr5qH9s3ME zJ~u-T-0oHRpB>#lxR&zooCn-#6$`|WvT+^7oVCSI926>$2OPci`8Hut0; z$d@qcywVszMPdn^LIk2sA>74x8(>;h%ln+pG+SaMTM$x-V&G-%8M~mhA&y=yc0Nv2 z`B=veuk*AxhH~=&vpqrM7QuVgTF&Z9I{&CdSick-NYpKpEBJNta45xX5Pja?Pi#Q| z`0r_0$MR2%kCAO~v=b$c_xvgLiKBJ;2qq8W`Sw;}5@&S%x$VHidGKJc$}ht_I@2}; z3yBp+xAc(UjuCHeC%k@2#|K-x40<0)0G1C*qps14McHvW=4x)p>(s68Jm0!c54LpM2rl z=MU!j#d`67U1Ii!q48f2f&r|xNo$LB1*?MT4lk}uyH$8V8vJOz*d7ddiB`iqWl{Ib zEA(#kNR4`C)%Yf(ia{>@f4%xIl`ddm(x5{AeCn1h+GsX!y|R|~W%V7I5tq(xKrObd z@nD<$^IEKSS}>&ims#UhOatoqHi4sb0;fEGUjC@E?%jTY{iW+WXKtRmuNEAb+fTVX z_8Ey2wr+(nujiZN_PnRr2VcKT8QOBSsX^8A?crOcxo;rqbuN-tzc_)2Y&cJJ%V%pU zxDs0^ga>ESr7>i)*x`y)Esb(Z?!BYcfw?G>FGn^E^q&`s*~8kS9Xj0KZkQ@Z18@t$ zDjR+beZK#e%Io1qN`&?61i7RL(3z4Vx(?l)I)C{k&5g&$+@CT8FDe9RKO9)%L^RXm z&O=_kvwV;;Q`r|f(=wVUROBSgU=%U7Jw$sj$~a;d0Vq!^8DC3zPur zVC9(meVCF;fQFl{%V7{oHfu15*%v&Uzhw+?D3tZ8yh?lg>oyv){wWWoIrGx;2auln%n%pfu~eia6VO zz~HUPTDAq2t}2J6x(Kz%v?lbqVqSY>$Gv`O7px~6>~G4VbY53`-=V;QE>G#Qmo7JI zvkwjI=PQfWYC+J}8~Ef;Ir&2;RgWGD`aG$hta)oyv>Fv;evA9vB5zKj*Sc60_n47{ z5XZgM_a0M-SEkr39D^hNGC`~nZK3PvSZw(vbwcvsiziRRgttq$ir^JYSqlYyy@1U5 z{P5yUk)D3DZkg7>M;raCjWlc4OqJkp3s`ugl+_yB^tr*;mLkJ)lanv<8YAV6wU-47>kAcN)K;4`x!ZO<#&;Wo%}C{{>v&?f>l+p zPl-vYJeaTZV{P7?#beLvecoz6#PSM%pc`yG=kc|a{V{!1LJ5bR$*ptP0?pN&h-9$W zIzUp*V7|z^>z%Of7g~sD(uT0FdDqf5rp|&RK1W4gv5c;m?4P`l1*e zg_YG)ee20Q2eAixN~zdTmM(kgjYVyD{6%;W@wL2*q7ktW!oQZ%$WfN1ra99@;<)lo z$oIK#-peOdaa&fCC3L(%In~rBq*z)W6Ld{!FiIB%bCPedS%HfX`uhg6}t zW~<7Q8Qy6@=`JjJdG)qbXD+_GrGcdefp5&abWUYu*t|(H`EsO4q6Q)~;k`GAZ(Bg! ziRgR#53*~@e=a};JYfc%^0iZlr8sCoX@~07g3ec$5kmkbm3JLa#~Y0v1dxA1eArkl z^9u==FwHah-%VE@wGX!#NBipyzR-pAI8{TU`=iY+={^`ib`O=+VKnnkRSg9s~#Y%-0N;HlEr(;F*4nuMm%e#|a zH>M@YI_7`8<@0uuGl-+10W~x|f7FFaGq!wiyM!|PpeHbv?EWXhibRUYg;cqk3BAsM z0xf63tm?7GOnNS8^F6e0o(^46jq!n-W#iUw}-v7KUW2 z8_HpZ7cO(0)IVwR(L3d$AFIiS$+iUVYfsoj3O>I%JmK`S<I@IdiQI3%uj~U-tQSgsefh&N#Yjp3UCo(c4h1bUwq5D+uW5D|Ax2xjlg&H zxXII%px1FG6O+LS!i6*=K4@)GppCFmXrJcL?thmpX7Qx5-p9rmLBUO~%T1Ks7PF8LW39Y3owzJDg=OSgKR z@2tA2^&Dzb4IQFBkW-IK&INPfJ{}SUOh&wP3i24gL!}se$f~^`b<eB;yqk1AP6^oyngHmB4nWf~20AeXV_ZJocxECg3r5LMwcT zjI`uH9S!$G`t$pCe)i{A?ZE!@ zTz}A?-?j6r=lWfL{@l*r`a|KOg;ge|d8uesq|ZquGJa~uI|_$s9G;0eO;{z*B&K=+ zNs{knPyFTUjq`!XX z-+CthTq*89e>vG+IDy%SbaQ5&4PR?b%9K1$z6u3a&RtzqAk@AqPEQoM0b2W=J5Qgy z1Dk-(=BDtv^77vAN8o|iBmZcaLjs7|OxV}x)R;=?$QjDz@U|e1L6_x^ zBLv3yB1Rb532hNNvsX(Hvs>pC+ppadNYcLK`X175@TeeX`_X(*1=ah?DjhE(;ttH{ zmD{B3d^URhd;X?1Mt|dNV6ojpDUIxKWX5e`x?d1dB!N<7c_IC{NSdHwD_uIWgept^ z0V?@Oq_HM%JOwu!Bgsq!4S_2IKc?HydzD-AP+lE!%T3`Cx}qv`k!t9PpFLLc$p-P93pML7Q4y;=ze!*V^W^o*1_MP>vo3aQo zrWUrFmiX=Fv0lAw5^y55MM|PuLG~V%HuKq%hM@3a(;f`vnR}PQXeOrR`7?8?_nnK} ziYo)t>QH&9%U-8-jE7A2M3C3M;KDFg ze$}3r>pr{Jc^mhPxRiuwMd={r)yX9*i%}U1LJZu2ac*61ZD85iLHq%snlLapIoaY& zZ8_O45Pgb8%ZX@B8S*-{KZcElgo8rH;C^6l(T`ZQvLwNg@1meWgU^`MtXLh{M&<{Ls505 z)tdd@AVdylJNxlfHY5*G;&`ZBOdS0FeC%dr>^VXE37s7;J96=mn3`Ggfp@C)@w$7v{fvCRmI# z+Ot9$0GDrPo|RO=jXKBWko;Ol^ZI5_mwe~lS#;JZz!v+TYWm%y|K$w;e<(Xp=v#;* z#bPE>2aPmQs4koS!(YaO=G{oEm=jTeCX*?N*9tF<=r?M$#)%*>+2sGj{-@_B7NcCH zgMWb7Q4Rp$Y6jL-TcN7nC4J`at7u;L=YEWYHesc8y%^t>*0gmNj*@@q=wXX9jwPS3ilgSC7& zu%*K&$I}qHwIC7nzC{pft}lX9-0HRk6V|yh`K9>#jF*$=J!<$?_zpniJ5HeUWK`5N z7eQnRFelJa3-%7>ymc!$!n)FBFtvKjOB$zDue=f`S9eotf8YGJr~rPTdNf$ot;qyQ zPZGsh&Wl7hsuAvu$RW7b{2OyzB`sTH?H=EHPDb72(r~q8^WD*P+hp(s!C-TAUko5G zWQ}EkMJmIOb0HToj|=Zt*M@E$3cA+3;{Oz*^y(>?kCjXPjQ7XWpR?(-+OjU-%xA4B zCvY9J9qq8^wXv;RqFeb1x>qC(bpr2IzspiecUHPA;r`Rl592k8+q~*@hHXS1yQ?*e``wrCRXX`;Mf?i|>Oy$VHo6%@!Q*BJm!j^x)&e`I=XCy)E ziEk4ABhA;=9%3r*Kc@X?1DlpkXbDLJEePRC(H--sRus=Vf|taPYi!XOl=piaOZ&i| zf2O$dta_VnS5=XnT+x;4ZV$T6r*#*|wyet>TGg!EIDMkw96s~FXtIDkJFttrJSuM{ z7_+{j_Y#^^&cR~-3ZB!+aa8sh5Gcz7oF{r@z2^^*$Q&Mu7<=3B1C(pnPw5YU>jpF9 zZxroZV67a6mjPA>V5Fl#A_P?g7BC9g$ma;OAppJuk;wn~2rIBm&%Oa~uHiC`6r z|66|tc+P)h5r1#)9RFW2cVD4Su^~H{nOU|d2*<=p(!lUT_|}CQYKl#|3UwNAEyuzw z5q!}w*)rfJ&9r|Dx(Gk_;p517alP%FkQGhq#N#8&7XhEhV!I!^z4Uvc=yAzqV}#a^ zWv$k8S80XUBp}a5UH?B9B;-nkrvX&rY$>9y_wcH?T<4@kHe}T<=H>z z(=^spEkTT2nG&=(xG+3*$+=GX9GAe>#G1rE`U1E^5;Xv-3LgS^(wsdo1gjsDVOjnT zC;-@30N6w8#~#TdBcKOMj(`s^x=%JmU_H)qC%Mx=)5={Wm zJocOru*`?5{-`ivp+NsCK)UmE6TZo8M*#nBm<*Tb?ymNI*UIYVp04aG!T4_#DohEj zS-DxB>R|n#UgQT@YitzM9wQxnjdT{Bk#1+6|AMG+piG))#lP3m29w9#x|B`tN-QMN z!j*9%Z{ZZ^z7`a{M!qGgbxN|tK_sy4d+?J+M|vs0OFXBp>r*n5?Kc1b$7gszNXH@S zG#r*kkRF&c#7O1Yjj|d@cAG4z&I=?M2o}Ak*6eM`SfmD3pl6zurJLNIr74EmeF~!i zpK;`yzP*3t%Uhj@SbVRY=K&BRk-$KnSu55HV676YhoJZS&JZ*P;g%(R1*>U&FQg>5 zN=U8wU15S@SLSRrq9_DKRk`D%K>iJF-%%eGo2=R4N_FV{tdkf-sF!hBLa*f$ITMS} z8d?T(fjc%2W$d-cg?wmpgB^i)&Ww1-rK?O!Za{ohxv}LWWM)5V)aG4gc0=nFLj05l zWl8tSk^U(F;XsBr%3!oQ5FxyB1C{RXxxR)j_6Ou528!(TJ-Q+QO=tc~#Ps^c14x?R=>>OZB}1`9{9 z5JWIpW7uGv^cyo=!t#bTr!Z=6vT&rr`IB*rmV_+yONqiep;`)dN_POGsfQ0IJq#L6TF!U?pR>)kt_jIb;t3kEm}xJiAV_?~h3DbY zuUCJ#-d^Zt@0Huhs2OVTmPEPR#gmh{BIwcrx`TrP+0F$g(jX^5QHDlBc!%~UVP|p1 zlINbbAFDhYTC{wQH81<($F0Q>pG&iput?s1{zQHYn4aK{6UD&V?|>_DCXK>qM0CBi zR0Ca|OOYiIz&Lbx@gKdtZoPt#yzmJ0U4N^08^rMz2V1D#Tw6e}^#Gijp3+w6NnDFb z>K&a+2hShldt`QR`(?k<wRUA|=8NvQ?AKUAX#&s!HiYoX)-Upt;I% zi!Un=C~~%XeuC76;3PQC*Yz>L*>Qbm^x@ah5|OE;oT0im2FjME_&{_%!q-6zf)7F8 z)j`M9zk&|_(bE5K{dUwsq80@fEH?}daS8~E^iq<@_QB0RKu5w^j6XmhSi(t(d7#I? z3rHxx9)fohblmX50O?PMGZ^yjADX4#_wC4Xp>GqpAb{ccDJ73+#ZI&1!t`mab?Uwh z60|rM7BjWJ$3G3W%sE^9@*9Vkh@L!4t_btKg8+D|sRRyj+k{Q1V7)LuKq2zJnq##? zMVdKpZ(o?LG%5sL-UyYV6maiQWoqh#OLPte(jd_oxA7fPAlwBgTV6Z>8y76X1n6w*9fkoEkFD)cqNxT}OuQVEk&d7>= zKfFt)3I5sQ#{X_{KhPvU&f@|Q1H1r)t@kz-+z28fIqOxbA@Sh^RZA@3g&?iD5OYdu zKX1()1)ZA*ayEox95BeKI#-X2VSJ1CfQlhjVisy;tbGQj2h$pH@}vB3AOHIv|3{5S z4iN&g>8xcuYw}#_bN}M5MJ*SHxqO)~fy^rB1&NMDk}y8(!`|U7fWm7%d18{pP)JX8 zhd_&S2;5y)PI{yBH2I1#|Fj5n+O{g`G@41&^xQ?rW86f%79x&p87IxI^2WXl9FHISe><@E18f5SOYOy6>n8Aqw zWnI$|0P!s}m6R~`4@IcJDgg803UskINbm<}EIFT)YS~ZegSI4HUwH!EH2;Z_r6_|o z-~9|UAP&S&fF16OKvc4Y)?dtps}5K0TZ!nWEMt4I-l4)$PXv-K7|OEmol+CLwDuI( zTELU|l*C2g#!E-Q@AF}NUI>B(16?Amp>;v*|t3P`nx&f3+m9DW(uok;YNqC7nx=7sLnkBiMuFoTq|$PIw_c0;IYo3-UpHfAj!cX z2KYJjOw$bg_tgoWDN>G4!L;_f^|X?l^C>&}wQ+>Oud|>)5P7VJHXl#VAG_`FqxnP2 zzRAN`O{7a|R-ex3_KH4aQP=U&`;Ko(_W;-R)87IF`V@#<3%F%w^6KnLtB&01ESZlD z=#Os&X1gls1*=Q@RN=?hDDSC*js8C7{5LS}-w%Jlmk>=F(8V6oIYeC^P=Pr(7z_!) zSp!+ap=N9Bi4TC2^EjI>>vx{U(VYd7p*CPM?OK7WZ1pKe{u90VE1dpsG&c4)np2y{ z_?G>v7!dEiy zQn)zQu%*57c5a$>z(kc@Nl8k>c&90gW-qlx#pjt>MBC|sT(>ifY#EY^Cz09o)~OKs z-uaSM#9}{CR~5^(&K|o&Fg?mBDV>`NP><6nUL8x37@x#GHzyCIVuyzM`WDfNP)Oa@ zRl0oiwsb9!q$Ri3o0VuHTbL9KjA9Ys9mj4Uj@Oah#x3&zMC3WP-*hndE9mu4V5{4a zV#Dvj*1wIYzdBM7K!4-uGels?Mg{nXM+o_nkm|tN8huB_5NF|1D_d z4Z!TomWCl7(1&qWw?LV%V=>lZU5){qrxMz?=xj2{1XN%Y9*Uy&U#0+1_i9TQ-jBcu zML)y6m>r32i&lhL3%A@EI{5N%`O`BHTaS$W3$kFJcJ>TH-{JF9JAghnpvn=2Falhu zK_D?5^^;6TWBoHXlLH>i$oZ3 zg$|4)3qUddKW>e}N^9P2t1#f93tSD)u5BAAO2d3elBmi$DN=Wy0kBKt%w(&G^1jWk z1q3;+&;+-I*=Y4XK5Q*LJ5ijb;nq{~!Xd7GnyESElKze0Q>Q7aRW1P9sDI59Py%$; zDjXHY2&1dP~7L}xnT3b3UcfWi&s z<$(ez(I}F3{xj4j1z`Z-{bvUD%wV`#=hli8R)1l!~{_w(`gwt(sO~3X}4{K`KRc^hjB_{7#qJ$3L z7WsY4Xl8Dh@JD@KmVofUb#b{&w=g6WBt5@^ioZfM49m|Dsz=3@F1b$Uc}Mb0xYE3i zX^+zytpbenLO(3XV)iz# z4e`4jeM9y8h0(f=MYvk><)+?))D}Pd_Zf$&=oIXfWlhnCZx&)>hqcbd#}7OQr%8ho z%);xwZVt~ce3eibyNQ@R&r!BtD(3a6gM!yjRDO*rh~Pbw2bFOFT&)^UnyRdiI1~E%=O*X8;buLDVudV=Xv;Od1_;Q zbuD5pPlcK`@J30TU#Yoi@`yAsF zv2V6n$sFnOP75bWoiBx1C)Z0kYFAuH6eIzc_Hh%bYc5?Er&x-K2D`{f`ugjTkh}n2SL&vvZ;qxV z-`CqEcOJa0GSRV8orINtTMM(+OXAKe)IfwHd8Fz!3a`$gSle8f5BY?uzF)nYp!a26 zPiE@*=iXCFMW5WF)@=(rC}Xvi962PV@bXnNI+j+F1($9%jigo-9wr3!d*3<1sb4!h zehL0BN&qaPtkB5g6(}^J zwo_**x$idJC!T5QOGJ)!9Ro2P+m?{?9CLtp@}n&NxZtj{;QHz&j@16AAfw@3CGv=c zW3!WUa$8nAu4OyY0|z56}sh} z{Q%kYrUK@zte?{US3cjr;?skrN#CHmfP>`_i$Bjgb1cINI)3_}HPOE{eE*-Jga1(r zzeEziiP7`=-dWJRe|<6eF zHk8EoMP`>VfA`;q_~9%UXpmu9vMGI*+LBu$h_dyIACy9qKY+<=GmZ&)WfQ3LbKRqG zwU?i4q_T@AP+aw#5QU= z=H61$lHZ&78E2%#P`x9td6`_0?$%1wU8I}XyoFM`m9nwD(^p1aHKj`blEd|si8W`% z0WCJp{o8iL4X{l6t_FW6lT7UeKyQyP=9%e&lb+Ptc-E9tdVsbAv=8&Ts--wrbF9 z+@>>eYXAsM1R5~;ZPqjdg+aDv-cw=I0UO2_-i!3m<`=3UxE8Zy$5fuL2IVWv-)5tq z8?{c(UEz_NP}@HKN>rJ7HKR3Y%>knU+iQ8@5Z36GRGjqON+e0#Y0p_Xba-GwHNB*Y zqe)nkv}ub=2cucYt3d=4gZPC{=?r2AcEgs3Z;uIjTmZ-N5P$ltU(%Bwcg$&>=P%!d8IVPIBjK;G*y zYl*>q>&`4`;a7+p=fb@8lmtsF%kN@3%pnwdT0cNVywAADR*WAVJgS=F9B@yvN^?k> zJFk;+qZ{vy%8f1l^@M_`0UmG;qO)NYiM?cwSd*Lr1&03J^?+;-;GzhSZm!J3h<+A` zHLnJc4}t=<7~{)CYRd$$dTrZmy08EC0syfFwrE?-C-s4``jV)9J2|q0f6-6*R~9dz z>3^F-x(!_tOqs#ngppF@L4gK;_l5m6#Q9~%$RZZNoWRK533nw}F2pjJJ@8kH-2T+i zkZBXGb_!-95}YHL06GbTytewl*NOMlX=O|GhMs^>!f)VSW0e%-w^MUfO>U}P-ST5b zpcse0y2otl=GD*>6csz%jxY2@fN_j7glL1U*HeM>%a?%?vL4JNDN1%;i*&LYmq57&LuUs@f1lw&t+qe}5wkjoU3uOKp0R1}FVMM~Mis&emT>?~tI2^8_yIa$3Fy7N!}f1CqW}8@ z(f|Dh{~bl>Wpz|_YFifvE8=VrkPc;WdB`U0wN$Qsq0uC_q9$4d5L%ul~g^NwwiI^gF1&o~||3Y^^1x({MhFLpH;1vWIg>bBq; zQ~gH5`*=|_1eq)~Ove&?pN9oS-cC($V188T=}UB=u{RGKv_c{scKyvup&U<*$5bKG z-Fqm&)Qyt3Z&!!Fz4E8zqriMxu$hUFeozp>VW@wR@e))d!CgV*91a}0+(~R zU0>&vSqV3T+o4OxO)P@Wue%Y*(fO?#(ow2xg&Hhv30&B!%A|)b&uXSMFT#kl!chd!(jKLIl?PCa_8wA!eK^PfOetpS^U zJYJP*zi~5>@9tF0H9k_+Vn_6a$9Y?hch7Mg)}rPBf;1DwOQRR{j)eesTElXN8SaA1 zVVqd=x9WL|e3S?KHt*~!$i4Eo*KdI@`Wxg`k@NlATCEXRqBx(Sk}tI#ccNO-`pRSq zU$m{Ukl)o7t6 zZX2bwZ;uilR-6|g^M9@Cwg0$Y<%86Ih5u5`@Znmu#CwrtT{RBSk|*3UcEF=AS!kem z%-`#-Wr@PWBHL+_p7(7X0kw*Z2JC{693C~zZ$2TMa)%IG{=?@JQ+odS?_c6)1oS4$ zMBJ{OJzR0`G<#>^^_h99eILbrsJHApuHbJL)8l#I=zxWg!EdH}rHjNU*bv{a{mE3Z09m=Z4~971@qdcVbXvqiXV>BSVfKQNx1>8vr*B!?pl!Wj2P_ z>2ZO02%Srs&IL>m@jZj=dCmiFn1pOs0w6Bt83oSioJ?}(FHjgWDh$AnJJKR(0tX#X zs<1h(tz#;{-RZM_U@c)S~<2T3@xP2Nh zCJo0r$wp5H9eHx?Zrr;a42%Rtb(sakep6srgMXRtghnO9+4o^n>7>Yp(7VJ3VJW22 zh=%LOv=t`y2yV^6Oe_WKAQ6!lUyMD2gRK5mE+Md#$3S)gjFt)A%(6tSbvZDu|6_sr zU;Qt}RbW&&pPGtolRW^mY-$!p;2{(A|#pwTIh58>o z$biRGaBtQE%2Imb>NB4l9inscloy)IaZMQeB||kP<)K1+C^{Z3c)!RxyrSq<;^>nc zF-~QY0-1;(JUdGUjsfzz4Low5Rghz40YE1>V^u$2Yw5<#7hO>O`=Zj&sH>OxKtzxRCh|DY=8HC9r--pDr| zwBqexTwJ2)*}IRLI=D9cmK>ooDf`4q(GYL>{bKJ54?QS5d^QU&5WY_;jcRw3;b)>( zh8Kxs@0fqsaekgwS^s=c%m4X>o#w5AtH@EMn+7@y50!053s6#qhwPcfN;5bW11x=c z`Z&Cs3>3CkDxuTxwx6|pcV6U8>zYEE>{FZfyCk5u@NML<*mPjW%H=6l@1`9@QBP(% z-lfVJ5LIG57cpq6cFDP%MW-^(-|{3q&THqFvA?JZ@`NkLENYAjlvtEk+isVp#g_i8 z?KI}LC~cP=TGkg?dYAa`oap>Ekh_>)*t<};Wp&Kp#uq4^gz(Cjh zu4@1=&~*-eWB^PNS^tfA`{%CeKQ~2mXhWitONl8s@C$_EA~ejRW~4-HTxjM&gMmX{ zyjk8FCx@iCuA~e;t}s;N^;K22sTn$xDh|#~OKHLK z(VZ5Gm|g*?#(cf}MBTzbFhES0oU&=`hLXPuarxWc@&6xmOzSStr@r&cxlaHLinEIy z1FpS>Ee4swhRr(MW`y91DOxcXz6D)ppG#z%LNG4SAUG-QQU{x5qWg%VFtZe3Es@ z71gz862`G+YMxU?(6?##OzeS8j7b^iOJjx{s=EWK+{u1aL9G}Sn4#+iv_0Jdm?<)9 znF>FQpo-6j&F@K}mbQRWfm`n=Jpna{0gMW%uTl26U!W8sY6U{bL2{#p$^gA*HH2yf zS$6@*bS>1bBQV%{jfRP!!`lHPEL2M@+3FWayOVzBJBsM_50?XzkdQ+ZHyzj}{^3;( z)AN=vKUV=b7}&!^ApiE7;?N7|){_}eVJx`U;oz+l0pQIr=G7>R-dRV}4Waa#TGv zonm^moDSmrx^^1mGk;P`ntHj6Y`BmvIUm`?SzS|2*iPpvGbphX?$~hVv9*7zl7{PXb=SGg&8gol6s1CK&$s;S4 z24T!nGQ%}2JSV{5_~s>VE?}Ak1a&HnhLqF#Of0mrY2~;QO5RZEl3nPraKhNnR8zkXN)48?rh&d z)6WpA+XG9{7L%c(BZ`LFgRtR>rEqmaMd{a9O1?>$53pK_sjO-&fJ_Odovs2@C@nUR zyuZlCwLT|8ywrHP{Bx+O9KVxOSZ(rgzm|(~+J#Raw=Lx{FkTA&9bry1Z9_3@K%9us z>$I{8j1W1weJgjIDlIx2lU&`kT_j~fZ1cTfxU~JqCtJfSiw#E0Mv74Hq6DcN!*o`1 zwgQR_OT1@p*j_hyG7?vHS-k^E(N5^T7!z#9vh>vGIGdmj5e4XeeSwe-<&J*O4hu+> z2FwxHXm2#-q4~(Jg!+rJCcn z*Zwh3U#`pL{QfqV!72-k-!pv63}CMH~ci!@}tw7)&w_REXo--)Fc9L4YUhk6#6_=p|eyY6;^#q%Vb(lQ0^+b zAu$sV_~o>r_+~ax6_uD4AnM%GI0&LwR#jh7-7qHlDqeJ~MDDwd)Cmy(O_hbBbD({; zDOo&gBat6Q^nIyeycne3&W8?By!#}L7JThof&F>ci-=36?z{8qem}-SZi2DEE(U@6 zt<1GYHJ=6SLvu*EH|{nn3ZhlTTnz5*mhujt&hJ}S(t zWiCsY+u$tL40H4AlE0noI#196A)5b3{f`Pvi0Vg{BQ&T11?M6vd`~}T!^!ib9w~7z zm{S6m%y3$ZlrlOBWD>EknE)tnWEF$YXRgE1kER)R0csg1YoO%1`zZ+TbJZdR4oSaSh<9B2rBR9Uw1Uwty3N zBR^c2UVQ@Xyac)blYziU8vXM*XG%&Zb&pU1KCI1|@duAX+)+3)RI_#kuKEtGL_--- zH0+9yewKQa^dfyfXGW&wsAyYNp_F_XnSkhO5=C5?7)OPy4EUvDcV_F*R6k?Vu-JO} z_QQxpIF$>o*RqZ|-gJ)q!n7&1NZ+BK+^-~0W+|EQhc97!+A_skW(ttSCr2HGpgyPr1!MXbh%gC zO^nEaWGy}dt*ei3K33*dMH#04VoM1xC@%d*PLn^^rZW>~W>-3ena*VVaU~EPgJt$g z2)OQ)r_oBhV##^PsS|6$5j2Eq5d#MPfmL*KgA0|hy+f!o6t3! z7I3aD#G?++MT5shXTj8qz~m#_kqR(+L>7V5b>27c>-z$C)=3PNE0i9`iJPaN;s$sD zhtEOxtp3lv0OUvo`uFeP`X!)UTgaueTtlF6#(ujhwN5HGszz@$1%G~V7Q0|}wD-QP zL0YG5n5s@|*yCJuKnfv;n|>Q>Izvu#dp~XXAo@pjgvurLgu;*VvtL5InjFV@ zI@zReW8{G-ugBDut8ZPvMq}oL3Gi8JrjE6lc8oUPOAmQ=V7dR(0;rDJvUfXy(wOBt znS1d>87{Ej#qd#}X`H_6)0mur-L{0}$L4XKmrhA}#vGJ3_PED)F5q6psB$)C6i;u~ z`)CQJUoNOGYQr?Bf;$=C=(7SxqVf!<)GMk`01M{>i>pV)6j1dn%idvq> z$6KkqlV*9Snc;U?RuNTS{!{n9(}ULD%r|LBy?JOkrj0TWIQ_DecyC#c?>k9GDs1WA zc&VFk`-Pbb2)`1GmkvI$_zm>fbSaf?#ZnGrfROnNy@&S>6}Fb0>M^J!8`0R-%Y%y+ z(2bFK`O%Gb&Hv?Ih%E{0ggul+m4j($3`MijXW;-PY!L*jlpIp}k8rmCfF%CyO8C9v zD<_+?*iHc`ClBonbS=(tp3D#2@ST4+5d7cr5Mw*^`OcIc2|;$GT=84&6gok6vdwi$ zdr(r5>>EG(Q2j=~FOwX@l7vTb>+66auoa;i2);`5-U)xiKB?#$W}?hJ9T#Tp>n(3J z%P8{2GDrQ4D9AQagT?czpdM8YvRMuoQ$+xB;;%DUy80gg@c+a^0@dx1sc6!5vMJ#w zoE%m{y*L`17iJ?m95ZiqIqj;Pz|}*n|M&N7K3oika-Ps>7u(uho9KQYl#JZV+wXAK zHr0*5@2T9h%%;l-{|RV0f)RI+Fd@e5TljEAQWAEs+}!(k^+~vgDs#*bco@TrvW2rC z`;{=qk7t&-7jGO*ww0WJ=DRDKA|(98M8#(H$!}ZTqHlL^@_-n(03?KQkgny2ZWlU7 zz6Lb=&3xyIipVdDEhY`CMx{k+Yksa=RE)pNAfkuu)9|`}^)#8Ausgc2+KfF-mayTv z;%MQQa`*LTW>QLDHH+nFpw_9iETAuH4>*y!FfT~PFGAwHOQ&W!uxBbVeA|3Ee$=Ie zg|%lUdE4-B#rQva@)Pb+uD)EYD)z|$6bPJb4YCwdWWO3Vw294u(+_1#h@~1?l2OKA z3{1Z?RH#xUJM?wFhmCQ#oocmV0UZ~FE=GiM-w6f0)vcji=(ESoK1ppgus@&BbqPyY z3f6e<86kNX=fGG)ZpB%79FSg;j{`+6PgCN}q2@Uq64vk!vc# z7{mI8L3zs)hh_Im(Iq=^M<@vDJqTobb)>jIGG)=Q!u(YKp#S_;AKm&CDYp~4Fa?G# z>N5gs$pRWZjPQb-v|NORRhIB7#Jb+D9DU!|_$5v0g!)@+H$-1yAu;Ckl(6Jg!O~_7 zH(9uSPA*A9jH0m7#2huyqmQcgc4cnPJ>k*QbVJ9Z&)ht+ndhYEn9(uF7uj&C4z7fI z@%|&>A#cIUJ9J?u?unwnX9V7@jPp|p)?erOouB{SrJ|x?1b&Aw8WAH@Z#^hPTm|Ba zEZb1*4PqZ1swSEQg5>q+6X@Hizxlr!;w&@&!138;a6ls3H05hG~hC1U7 zeQ8QRNmwagSi_H$ER~s=C|JHty64HWi?4B&UXru-D0p}@^c$6z2#%zmLKqVPV`9uX zgfuZ}wmJsk>0Z2cG?A36*#9DASI21d_DS}aM#ZZcSy}-s!JgChVR%}mhA$q7bHtVd zHFg_>iHx?uJR0`y=WiWaFQ1MqYnpiKnx^;=bm}|@Pq42W!w!ICD3*ZR0(A3>$*6_A z9K!mXK~mV`u?oM6ne?#-T}kiJ??d`d=x^&;fcB<=1r7KD0p}wbIXJ)nqZIXntAb&x zpTa@!z0*E}JC|u7kVxun!XA)T+as%cIWwDAi+Oqp#Q1In9?FPg((v2 z>McAmTtrX77MnhwZW2YEB!@hxDQf~P zb(HQFa$I4M;K%GoP6Yqa8u*TAsryBcP-7{uL_OlUXrj;HdA#7B z{9J(b#oNNq;6X3AB;*L0m?1c|g9W)yAjk(0Y{LhYxLRh(f0Ru|zdtZT7W5 zyVMVkYNGGIIZWH;h7|^VbGA0ifxrIX{47^hCeI`1lnYuJ zIG!Gpw+PDgOMs=)WM`~}lYtvQot=%Rn>4HCE$|kf$k;vbyHa5p{t83CBj4k5C{1M?y|{z+atlc=M_>O=Nv5Q7B{{vkh<2H@>tQzb@<5p8GcHl;gklN3z+P=@b>=>s3=QfGmocNp-)cpR3F?7@ z2VUH&Q>e7*FVPwCwS;(IS5Q8a+ul$hb?>8>*W;cZ-D}6(t_l9&VFL=!|Bt3G{@&|u z`VX%=&Oc-3nLxX29Q3hyW`t@%cb?9r{GzvCzQFRY#-4O*zZD)o>U(?pVZpJ-ne(tP zoIoe1;eEtstiCd;R?Jn;epW~jd7*h2bFraeWR&Pc80{mXqaa*B!xp8s#0NkCCCCew z-z}%5(@J7|ZibzmZN4xY=Xqf#hM_%3#x&Vp)sR~|`opO2BqmUFSf(>JuG4lZOy}PF zG)K!jaH#V4pFv1G={dc-iQWbUM&K*xgJR#B#z}0#gVPACyla2CkFxVBZ>Qq=2z`bW zPipE-UMHdduW+Ga=fJCE8)ooA2JV|YLD1XJ$`26R!(x3G$>7bq&G~O9I}$T z*?WRfMYs}yG;TAKT69TbE|7jcX{b1|;X6my8+x*P^i3i6-b9e;z9)FmHwK`-A5v9_ zur|o%{1zKox58-_zM1KpL_(LhjQ9tVs`_W#>m55hLq{6PdYQL02Sk9R_b?oqEbiKY z;d5{!V_KkP;OPBEp@o_#is9I{%#%c$vu?%7$K+T}o)^lB`r+rGGc|Zb=@;Nw_r(C7 zc&@A$dITbhwW7~GflmJ-h7j*>IzfUzq4-Y8uOuk@XFkl38+FxsU!DV-?oD2d z@y|eC2kcm(IB7y$#A3E13BHw~G-I@@(N8zc;Araq#$Q&?`*lE#DN<7DrPB!eu53H- zw%`#|b$l--410{u>xAJ#7}^TAdem84yChs_Ew)oxN_g_}OxyXpQ)rbR;k-3J4X60T zt*qWP4pO z>;shPxYahW<^4%oUEIhX~ z8Wcg=pylmLURbo^CkYZC9&uXZ-40?!%Bm&SbxQj_c-G!8ZFJrbaMIJfwPZEuKug0u zca4BlB+fYm*e2~nAk1^K6ArY;ZeDrGayC4@n)d$%U7JFRp7p-W; zf50ka+xdt(`#9iz0mS{wbc&vN^i7H^3H`01kzgpBitR~M*>$gWpiB+&NW{uIOFn7= zzM@$46b8Q?Ptiw6R*>WK9HCR3ien_@Y1iqdk>{(iO}NA-F0=f)_+O+UmYwll4qy{@GFE)0?=ysSnvbpMnA(cGxG%r%x|BNGLB7LV^n2% zb9ndhtfIlxa{o}QH!xv5BMwU9BM;*-Xvi_kPUx9WB{GV4 zi)YPUtE$X6dtz-Ql;Y%n3Oyro;qUmoT6tp?zbE?m>fZ<<$Q1{;5dFeDeHL)kVs!U? ziV)5t2PvMsaIR2{lRwa2q<1WK1Pg)PTJu``Sgw8UDMt$A8KqDIU9!lcwD6dKRHVn0$#8^n;m9Y$^(xH5J#{mJeR6c&0TF z_wm&3R7SVRV+F^qt}gIf`T(FTGV~Xv$O8C*1Aii)y9kp13uZ-YQa&3`>lz3^;FGf0 z8mUm+7Puu%vpRw*pELEt!Zyv{hm`DAd{?*W#eI-w5a76(4FGUV)`F{FQ#GZK6O zvEeipXCJ=VB$7eC-&h~wUU8?!I6DzKUc+JnBB2SWfm z;o|-xasptvlhV%jF0HAyu;RDlnYbf~8>dc2sW58lYp{re_Uit^7XCB(aCp$s2HpBW z_>JpA#`DTm3AHc^*LVDA=pi47g8W-WLl!1wlYJd=Y%@*^fQ8y#P1B$g-U?+9*fnG= z(o|U=rIg4R^L8bg_&qQUM49`${|KWqDKAR9wqUri_O2Wb7H21FJ#(H-H+@v-eQaS+ z#%OkPEtoOoTWQ>>V+6Ka_e@P>P8VE4XCv9@Ll7n{HHROCR1Kn4<@c7~57eIbU2Lil zE9^a`9?q(ZOHxsJqWVc+r_*yJ>}J1}P~^?e|K2)e0xcsiBJPv@3F+EE1I3Q8%6IUQ ziz%p0IY_{4xAXM&%x}I=+%mjGx^&OTG*L^NmV^|4h~ZQxyB(m@2t{MMa(sBY7(tw4d(9q2Lf(#^y>DquMStgP%ed(J!e^BfMax0B zQ;eKEq~}6$(cK8)1v9hlIF)RkG$O;=LsbB~9rXtjm>jlRMk&vazI@#N=kMun96YPj z;vWI3YzSk5OeG)#Xjf`+R<^7Nbo4v^-lpeP>e^bf70mAL%~TyjmGVBopLtM#9D-tS z0?ud+-#`+Yi+XV!-uko4EKxJuZ2Fe`z--|fc2hTXX)N=``^9Pd(J_wyEpq*r!B_48 zCdYiTue{herWkTH>q`c*W2q7J8ZAbZM^hTeI~8$p;@R*XsST8^oD~@`jc?ujQ15 zAkX%j?bBT+k&ES84+07vi``?rH@pX5{YD^Cud|3gx@F`U zW3lS*P+C2xNZC2FdYP1=*FGmO(jXRyPoKTZeSLJ(ZtRoHFyGbW;S2tdXCa+T z#XiB7F*X{e5Ro^v(goY2ZTE`wO<$ItD@6%+Kyi&wfSEdym_OA1BPq7(D8qG`dUueP z8%B3+27v!zKYmgbEkjFq!=3Eeg5s)}^1j?7a_->u;1m1!?-%vhYvbfZV){TgMC1I< z%C|lUX?X7lD+TyyM&m-fg zF1&&QYWikYUSeN+53L z%1wT_i?`eOWyx*_FQd-8ZCYlHfzb8N@=4VR1VGyz2WH?C{fD8OJ|n0%V{cRUGW_f> zcOYzuB6A#M_@cDOx4!aDyT@U;HyuQM`IGl_th*#ePT(;ql>xOv=T!PSk}f+3))(a^ zmsk!=sL0>iIUQPX*VQTL2)-PM$xJcT5ON|VBcH{`d%KU>#q0AN_^KRR$u7SUvzFB` z?rQz{b5+{P%8E`n&jlbwT3A3rBF;SWJV;e!!tjkbjP7#B_Qgx+zTb5kP@2K<~@s zBjm?viLV7HmD*}${hWH(ccUz#A!+P2!+P<_qe4@>qgUBVEyzaf*avtF|E1bEXC*krydYCn*UMr0Vj@W~pbx3LACtgZ% zE^rKZbK2F{(UZq7FE*U%OXkT>UemreFl@l$*AyE=@*(WV!xSa`LPO6FexAn32e;x} z8_ucrvB{SC^_N+Xp0_C&v=f_u`OH@#vp`-FR~2SU(M2`uYM>V3{LjZLFku^_nJM-f zP9p66uHqqz5`HZ`E&#Ig>8)l#H$~zUFbMe#l2zIE6p5nGOe>at#Y{WlxVN|$zNS5x zC@)ZqYqE~H_)yA~qrMy6Fmg?!tW0B3QB&8cnw6HTYHas;T)`VkFcFhOFc2z23;jmM zv67NK{}zcTOtKIoxzWBLMUeg6Vw-db(aw8^Mc41DcB+X=C(X@~9bJ-@Z+TFWbmybT z!TZ#WU!dxP_EaP{av-fgGf6Vmwc}0`|L8rbRJKgE%pS0Y_7{+z?$iCV$$SF)XOB3K z`v6L5b*dD4p?Yh0fi2=RljwHp0!;p{ru`+E*1F-gPw z&`wEg3<=Km$&SpN=TJ?Tc6ucoE^S`M)G~0+oMA51gAtpc%SA1A%TL3IDPzmhfUH;( za%kQkdveoRKe2n{vI2xN!KMDhi|0ql`Pnl+et&+hOy*FjtWwvxkf%z}5`>HV`srs&N2-=Gf?=30-Dx9)3HoUEkU!mP`9l4Rr6|K%kxf}%+hd89DH&7Hc_lpKe-^jf{42vPf0XZtfi^i?C6zoN#I2}j2y%*VdpP9VYb9HoCrSWma+B{|P z>5^Yo7jOgJ8=t%vD*;Qh^ign+@wOjHi)h0}V!xDbju-i@l46N28FweA5`c z*gw4+(}1S;gaq!DUj0V>M*9GyduXYXBT1!9RDrmS79k--$I%NnMT1I`t%zm8o;Doa z#2|T9$eY515UwM)Qf{nOEB_FwgNH2%T^w6}a%Uy1Iqa0CL7z>RMiOhF5qJ}Dll^CH z#UD_#lHSeu+@x`K20J?y5bK9D5Vek0rFHocHHg?}F&HvofQi7IuH+9A^nTi0-88;! zYb=&LE2w<~BrMFHJRtfo7@0HlcaZi!UgJNf$^Nqt*MO6%$;{9#W&(1fB|s^w4i%|T zR0BAE&eoqbE5)&J$+K#Isn64!mEs4xoCNpP`ECLH1=I92^b~y*dYWpAV>{=#e!f0K zfhe7X{|M@5E&@znRE)X@S#&^vU@_v=ACWhllR$fV9}sc^ET+bFk_%uNmx%=y5Adge zfO^~_1W2zrwhv_MoQ0DAuv}t0Lrcf5iIoCONdaKR3e3~^Mf{Oxb4?QkY)vpa;K0<{ z2DRHcItA>CWQo7n^cj4kLjLuM>-rMUx+ z_Rm%=&g*gduJDNYsX(P|ec31(OcDg#6AT>@aQxW` zEn&a6kE@CnH(HT15$>GG&4pd%2Au#Em%q;)WT8weU6;wk& zQ@JAA1FXKh-J&`#oK(C#eCO=;<2027ZOw$s;PJ@)0{he`laE2MH&>Z}@QVK(hQ{s0 z((uF%3tlf4Q2P za_6~i?kf}*7@`rx%^wcxL$1IhV|_k8@Nu=Lc~fbyK5~vs+f(*T*JQo4vZ>Q4zNo?Lmcn!B~eysV{Et%7~;`b5^rh4K2?T-fSdFH?S4lABzD{lA} zlgFm0@wL9kg}(~m%<`_4vd|QEAw0-l>}d@fqVOGZe9+V3?-o6y-={CTURt#I1(JT? zdpu4XNeL6J3n@v_B=kLNNhuGQv7ez*>jNARB}mSe`yIV!^>=l18gG7RJeZ?&0Q||c zL6FGz%!AHv7piC&r0|TTcDxREsQ1e_1*5_!say~Us2Rq4b*fFI58$bLig+IDT%ICP zZrfcAW0C|}l@Q$yFP9DV-r+(>5|CvDvnYqdgX$x}lD6HgF(CQw=9>BE!on+lAl8sN zId6Nu{>xBo*K+ULy-cAiZ;S!srzuO)Fzqe!bT2{Of|wHCAc;et*wEsN-Xc}|80q9+ z_sAX6cm6$6PV%Of*$e&#JYV<&Ulu}5ROYZw)#y1WSnpeHy9i)xwEI)jhNMgEz!LdM z9t2nv^{zW9_KuTk@oJ6C_D@>cqNQ!Ju8*CgptXsQSPD=HEm%i}gqiCIs_-QgMa1M{ z4;>Hy!+h)q*^+E;$oBGy{Mz`GcdOCK`5w7DLvqKhx z$0_dc(g%jkohBg;0F?wctr^-}@%4=_xhHNH`>-5ROA6(C5oU&uy^6n7He^(kZ>5to zmULS)claBz^f=i7C{98`5l&-tF>+6kNh=Qe(uJe<)^Kj(^Azy2@~b60%>4S5wl!}3 z7mMR>6kpXBp7Su1c*ah~0QC`z`)Fw2hFT)|T<;MTiQi#kCEK( zn>sRjry_3ptexr;j^DXB{KW(&(=V&4cYi10PBweAn*+(Yc##*V3&wTpK(x1c_cg@L5eLtI4Pp!(|5ml zIkxXb*EO-XXRQ#k`8j%{ML}dMkE%IPjnf6A2@w1NfU8ITvu|askwWA3P@ar6G%YW7HP)oR!jV*GXPv zZX%fUgfJMT;Y=uPt(z$I2RP`tIrHMgn;g6Mjm&C?>GlWLz*~lNC7(N7EXxEXvN4rW&>7W=u~wZ2z9+ap!^tE9}m`muT-@ zx#np-p!z&UCejn}m?_Ves$}JBJBTY^*3w?W+gajkf!mUiv+XM-{@eJIC-}REanDAy zVlw@tA1AD@JdNTp>K2J>e!#_&puYf8pEw2qBb$v;Ed#W?CN*ln=tAnAApj%O6S&o2Te(T=mmgKwsUld(>e)4$fz91&gC^_ zc9%+EZfAlW<4Qo-#AH8eFfOcJ(>2I4FuN3V_)u5x>pbBu^Ea(ah-YL;Ahqw=!SuVM z{F!}zhj2U>rvaCHS?6sLUzw@3!cz;AF8tqL=Ji{@f4&A?CzSnuW%(_zW*Vo#!BBRb} zypGZrNwD%0e_*oCsc&!<*ji1^o1H20vx6Mg+YNVvr1HmKDy>0pSjb`84v!o5LCNA@ z&}2Y{SDv`wA|(BW@jL2+87eqGjBLMlW-9XqcM33)+dJdf~fj z#{(?ng>>TKE{42$-EdV-mg0NRJ>i&d=-dzX<|Z7CPTpp+r`01Y?YL= zt2J(hQbZHGLujv?nY}nu8)z*qcSZn_-q1ju>U>sn|HI0g#|ro$c0ixrtchCc^#X{D zjbhZ=;WKx5`&rnlWnhIRo=g5gfs!+6Pwr~W$<>6-3w#kxO1(?wM)i%`Xeq9#)hD2u z%@!L`Q$Jf1uIg8z+esa6U{8A=F3j_#pqn{hBKB&2Es6W(wn8tK*$!Uc18II!Y*7}Sd4{0Wh$281Tw?LpKCUwhm42MvsG9J+F5*ZjLdodgtvc?3D+_sQ zjd>Hyus2DS-XV*FX?&(YXI1B7iwq0v$QqkwPge&~lpF0xAQ*$fIQr_;`X*!?}LG01VPAOTP)IIL{qW zxftoOl0a8yVvgF?0kDT^>_Qe~&+(BUFwFX;7eLehu=EIP1kelMXU;9c4*hI0oEi_{ zA^*iMkwFT|30%AG#qN7rTEp2~p%BX*_SP#w?kr0~$JKYHMgjU8mo7Ukiw_Hr#rbz0 z!NMCjnQv5!!{iJ!qIgO_x+?nNCxWk_Tp%Da6qmAD5477S$eh>_?5DMbjV(nsn6 z_sQwMt>i;aQXO!j7SK{qo6@DIwIa1B46|Ym^+u4~!~DEI!8Tq#Y$)Qw;_3Mr`q|-q z>4k8LZH%$YvB#?c@0Q$kzNNuDW5=*|=qbEO;y@D_vS6Pm8l4yF%r6I>lL)TH!_us} zy%vrH_766#DALb<)4?o!3U0D-alP;~0<4Pfb*~6yVM6R~%i6%o5`8JhkloycMas<^ zwoD!xB_@CcEs)*?-Mfk1#TKG=4TVOm+V=y3Lg|7#l=A?M zd+`^jhYlcU-3o#FzHCLKn=nTT1HEhF&eMnmDvR0U!@7=aacZUqcmnMxgWJ6PDLglS z6GoAo64qmD4q^`#z^Jaip}Q=hb7lN}A!YgB)}*j@Y*W3kl))-8?}9TeChcrR&8jNH zP;gMEf@g2-qaMMy)|&+I3k_&B1>CI0nqIz5OEe4@Y*>vglpBOvq*vXGvJ?(@@h5-_ z?t63&m>hcR1R)ui0s%aaKc&G>tu7+vH<^ZjCivf!0=uwjJl!y25lxGlj-W?A1Fc|f z(T7R^CkteSI|4hV=(G5Z5jx+V=AG26zln?fW$cLIr(qW(oyijS+#M4(vN}&wTF=e& zk;NZEtVdL}|Ldfs&&dVEK^(N%P*V{SiB$SL+^{@*yqr^d46oAXrpu?tyYTrT>yi;T zJRiuS##Mb9K(G_i7gM67$D>3iGO}{6T@duaDEqEt4KV^6Ps16 zhJTP=5{j3lpM+8*fZgnW?Rh{AGfFgxB{DDeP;ZPh36M*h%bk+%(&`4=&OvPbw=eQ* zkUQ?&&x(anZo*U-7{3BdJY zm!Jji^M}rFzJ8s`7-Kk^zX}mXj%ZyWM-!_Tcafd*C-*;ASQ_U`AH?pzgQmLLQS_^s z!=&<5FNo-N_A8&_S8%vA=zUT<3F@kd>`aL+FlIMhf0Ka)Dn|HdV|~Wi6e=f{WK&7# zb&w}}nl&^z*v2>+Z*Ve?+e?{T{b(}p;lkj_I$Gg!y8CxxWqM<;`6FJvT75uiX^rHJ<^eOAEHYI-TkqM zeF4=wgq%eV`~tCKolBVzzYW7$(4kt~lT_~f>nP2n&8?NWJMY(h-=4fj6!GCWEc{sJ z;#>%Na!qp|7m;K-cnPz=(eu8%?X^5}*!n`Ig@f3}H< zfD8o`H!T*hkOnyi>59g4BkJl7gewnDD9eFdd{ph8+oncpr2WhnQ&rWYKj*F14#r2?KCjxu)L~Lco1uW3 zk3(-JD--X`PhS7}6?T@-*(mB6E%3(*8cyU6M=fN{R~!=q4w$%N4Ov2G%jD+@i}tw; zg?oxy_1$Ou$^_eXlcyS9nuM!zbtP!=FtuE189vDIhi(k5zmp?lW(S7R@b#U&pPIfV zWLPeOoffMBCrsYNCet+~n*(rZyJpGCwV4KvOUc?r2iJTr+)tLszF%(R-hAKNsi0|X zBk62J%dNCE%+g(2aK)BZFUf=u+n*8JzzOd!cxctlb*sc(yeaed-7{AuyQibn zK$r3S-k9E9F)4lP_mh~FAhw73e!#8+0ruNwBfw_a3q}#Fz$`0Xy7XtutK;kFgmO}h z-LEiJnv|W3?oVF3CE?g{>jz*qP)!m7+yRO) zgfMdb&6-h&G=!9$dMj*KiRGiO6qy{ zlrgRan}#rG*RAr}xbCRCPJ6d9I9~H4I`_r><+{=Bs7sEBPP2o;?ei`+!SdfJrmJU= zbH$FH;y)EW%_h`W&a9=FemgR0d?!#6=w(R_OSuD3HFC}>K`zjv@r^MY(4_YHK!7R& z2nf#?@svdMMq$WoRQ5<*(MH7;OSoB2*20@NpTVOqOs}_6TYnENc-c1m&bShOm z6ROL?oUtbCE*IT#M>@*(rv-3_r{AejPWkd7QRasAsp+g34$qj(o4-IBK^QPW;ERSq zCw?XOfz>sgrU%}3vm?$G)3+uJtSlck$+%;Ry=9AVqjv-6CXrTN8}t8-z4r`ja@*F1 zgP>8W(tA)4P@2+ff`uj`Sm;7jItW;3YDf^IHvs_wAt)e76={k{=m?1PF1_~zFa}fB zJ6-!-YwxvJ`S#gopYM8qoF8yqA>_&P%rfQ}_qfMB`ie0vu>G#cgJ=%FzPkI0#O_H_ zq;|hzfu(Fp5@+fI-)N>pvun?v+6PPU)oDrX3>)a4JHMN~wt@1-cMx5HFBpM>;capo z!AT6T_*!0<(W61VvotV$N~#1#iW)-_lE;G?)?HztwBTqkgVO zuM9mrNnc}0K271lP7eis6%y~blzge8#?srsA%-O`Nt#Jdpz_}AJf(o2i|g37{Ec0k zF4mq}#vjX56OnDPq^`Hlho4~*QXZ8uw5V+rxOZFk`9)vlB!h+kc?0;%O79cv;*@}xCt8;2IZv7a6zL6%~?^LbZ=g=#n-^;6feV}3NRJ?UY^w)KZ0k1LY zXsLYFnL3m?nTcXSav(5WCMA;vQI?hX2dm6ho2UDqP~+}2M?Bw}*VDFmVsK9AOd?rlkwojNUxr`z&t1i@ylE>imNt2MX=3IG zXR~1Rd$0CeSD5L$w9SO`bEG~VKFZ^Bzp4ch^4+(2Gq_fQS&%g*tNKkK#~de#@%+UhnhLC8UVY9^Y3nV>vhsT) z^Y*+?9@*N;5w8K@}7hJ01o zKi@lzaLL%mpt?uHOG0F-B!yR0TS1T>z z3vV}I##9VbEk(BGEE=NP1l`}ysqNnGOU?hvWx0L$^VQEw9biQ{$qAsm384*`gMNml z-D~TV1I4wudBs_pJvF6#?%0E2Gq*a!U?Iol%`@J5x#qhXeBIwB^#tRF9YMCh_yMeS zfO46nH>-hU@gvuJz#Fv|rWty|8N!&3zcbVQqAl58KVorhA)UWH^>%#EBL{cB1~*ul z=lXNPq8<{8?t$^6R9=z{ue%knu~^S6UQASr{A|Z~{3X7qUNrXIvqP>$$31oi!LAu5 zB1fF2FcGx<=&mH}0g8FvvdaA$HguPAx^%oXnx<#pXuz*R7lO8Z*W$)Mz-6#~#C|vZ z6LvV1c#nWx8^C9o;^ks0jJRnuP*|z3($kjRDgC-o&fWdsC5IU0@#=kI?k)Qli-nKm z7s%SHwHb^HrxUWKB|_k8DCI#oAF2FYl6tvYrQhv^vGY;FY6+SiW+u14*b5)X%T>8> z_2Uy@VoRpF0ze498EhqnWUe_tD68|e<$iN{Ukc{c8PRb|>liLm{_&3@{{1pXN*H+| zCwq=R5#P;D@3<2*koACwriN2I@y5i6gc)KkVb4$#(JCoa)2e8E?hVW7Mbq?wg~xjH zYlpZmos#EXOf{8i$$_6OWKSQ6eeL$-)89$Tpytes3Si*5542A(A{?_u92g54r$oDa z)#{pETr0vS5JUT(`Zem1#j6A#$as%IN+-_5JWS(Fgnkn|DDShb2>=YVExHF$8rqB- z*>I^aT$Ll~Yf1`yv)aIN@Xj(!F@^0q3*=sXE(Vz+XbY84I54v(NS=f?6i2?zz59q* zQ5&@?WiGkyqzcahZoh!9PeIM|>d$%&zs9wH4|eFbfZn<f%I?jIrvY)@;zQyn`oh*(Q1p(WLW zG8m4oCW_NA*YnRmNA}=>=;p8$t&}vjW*C{Pf{n`FPuGd=fNi;h6g*%%w-Y&(!K9X7E+W&1zdH})|8C8X)N8C)I z*k!rsT^f=_3+m)JSEdOj8>=dlJEjW!z2!FC=Vaz5uUcFz4a)&lyt>J69H2HLalj>t1`4p^Lab@4FziAXH{q-gt`xd{nI}eiXV+`< z5rX|18J;DSURF_=<|D7KAAh0rBSY3)UC7Lgs@;ueUV7BxSh-Q;&{xv4yOqfBMXS@u z(6777ao=?AwU+-NWi!-+9Rvm3so`mF0N0zDjz7h4-N&TNJHYDDxEJu+VvYd?%9+e% z_;Ju1h-q~Y>)NLjsa7a=ZJSSX_~_|)@wvmv$D17VMKuq)v|Nx;JpUq>hwo-rB*qRS zw1Oxb`q9pOM08eb?Y{rEtCaV2WUix|TUE-f8!?y2rVe}54jVJasvH}5#X zbeYm6N_TPE5noa6k0^m-;kjQ-7lqq8D7qw26#OWIAHzdAOAxA!%n|Du3(ql+%TDZS zk89<1ihIrXI$z(^QPuQ@Ea$xGR=rKiKoR4i#k{rXbb6cKnv2g}YQ?)fng>$~M$h;= zsOp}QX3zASbb~Y=ubT?cy|?G3L34R~)RUPUp_Hvq3KtX2Xz?sap^O>OE?+4vTUN(^ zHUTSKZIffSv{P`5E$Ony#@L$+QXuurE@_qW?rhejswE^;C6rZf7FGI?Sp}XQ zJ$I#K)0VjPrhnn(+ixZ9`9ttx&W`)`sZ3U1Fs>=6&0VJ|t&A+6OV5;~W~ z5Q-$brHxY4kZ%5IE}fXgZ)n)2u+MeDPP5#vQg^<7c>lGwvTAD;7uQR=(ei*rTBafK z-qnuRQCtiVI!YO7Xca~Gpt$^nac9Q<0h z`YRn}`bD5ZrbkKXYQlcsDhKH9>V3!tjS15xh?k&7sv~Lft=d9~e}N7Q8xib?q20CY zlrwl(W9Q{Ys}I!HSUH`<4+Hu1T&41QyfPU`mLu^zk88f~p<7k_^hxZx|Ol=O3y+|KH##H6b}PWpYVnG{)O_8q0+VjYxlm zhUI+TFZTCd`&UeR=6B4${&w*<+PHfmH%LQ`y`=Z`)!%|e`se%X2g^H(wB&B^J#b7F zk2;qndeqlL@GB?K7_IXpi5owp9D2oe2xUKbjga_--J_yUB>Y8RKYimlfrY0#8W%Y7 z4mU&mi=3QXa!1cxzwX9TQBf^z(#XgVqj)hOKPvz-MW@XH=0O-kNSHEiM8*okBDG23 z&(liY+$*{MfFVTSJ)f9SAweH8C6=`>I^wW{dlr9hk-9A2kqh5XI60pedjNP0H`u~Q zb)UOCWYb1qht_Ij9OQM}bF4dW3%+()sykYJuIjXsLl^_=;nDpJda+5>!m=!GM#%45 zs!7O8vh8J6o^rUIUB05Jl28~F=UOrXytUlMb@7EBFKf1gt11bf>L&;rJG+ND0RGV| zB>)`{Hl@F2!nxOKG2Ke(xM2|vK@uaYd!1Cw>$A3@uF~kf>NUwRVjAvE!PrSKz#qI0 zI4qHRHNEJ8`Fpi`qlf)!I#X=Md>1`*Z%4&cG#|KQU|70r8e0D7`MBcUpCEK-P5>Lm zk&K_CZ;=h8Pl zx7lKFP(`*F;7lnQhg$6bwMepU+gi!sxOjZaio(+cZ9s@TajCGUCXCP5p3}8j;qs~Q z=YmEm7j$9+Y4ilSMp&qoaRAC_R#NlAsGyXCoxgB0aUb{X{KAv^ftK|0bmgj2iA;|z z6y6jlUGgxnnr-Y$b8JPYU{18{ZV3qFcmA8o3x8m+A-8l_E# z$jLdSh~J)D^wFDWQEjWy@${5+v*>^Zf6o?_^Y^$8lk6pH?N`T^n&F+13UD^|CFca-t0r8qxFI(8Uj^kqfh5c z@-jlE0L#o=criP%T<;?ee?@LrB^~tgb(h#RG6=^S429eY1}1c9fBPlv9E*czypFuc zQ8f$7y(qz5*f*f~&B(_=_X5i~*-pL$=lcx3d}0+*C)bMG5hB_pPn+=YQxSP9{f$z! zpD!Yh@`=?NcHQmI&2Z$*YU7WYa8dbIJ|gonG0W=U;NasBh7*pvVeqE8h?Ne~W%2e& zWdSzMzG}}2m*UUnZgV{`qDn0@iTV9Lqa8Zi=h{-acAy!*JwN`PSASMaFq*g|@N2~L z2N#A(0enplGZ(mwA*#4w$YufXyg!c*{`*b#plUP5VZsdfm+zkYJ!lq|=u=$3+{Ckw z<;dW|wKwBN4eL?Dxv~WtS6Op8>3X__2LZk=B|-OleoZo&)GaPT`e9J zWxd_QnQTQrhl>iRQ!_|blChDyU9&nZEz3gmxYFQrZ=sswoppP^1Tzr%GFkS|#aIkl zq}vGYGrRow)Y7Rr0;^q664-E3*K4UW&m{LGr4>6@cZ}<0t#X@p+9_C%%TzR+yY|F8 znwU@#dh3?+p7kF`JTx9X1lzia;KZz%^&@r+O`u9-X-r!cMG-iFcE%t%Ds?5+HW@$k ze8sEtw~o(mU$LcnVCIk3a)IodjT|j${7Tv9W(r1hOkt)OaA@a`oC?$~SsNux8h=GvJxvXlap@J z*Br{16qxtxMx>(c=1(MXFgRA{4ZM7@+;0W{aDmx~@w00MpKG-iE|;og_tY~2gEvu_ z?yoc+iERk7;n;BND;Z5Y!-f#4B-`T0;hw57Y#qzCT}+thu|66dwzBok*hsEcun%r&2Ws+uko9(Ll;y9bm%T&;7)WsdU&FP@`6nDbM zF?BdtGT&?9g-b}oIW8gB&)b;LuS@=i{+2oU?Y4q}0J7eAr+|zbvD&tN{iqE9tqZuE zl{c|x*`L1guR33zk@;XDywHVqxTjEwDt=ojCfAK?zC0+a`3O{(awW)Cz|6<#n9su> zm%VnM5pqK7cc8~_C-`4{wQEdFvYUw=T_rl#BZ{{>LNdOAQv9-+LHJ zvEWXAY=-z=`J)g2d|?=%uhgGL%zZ|Kn((2-Tw1bqH`EtzP19d|h*+}(ET)|#b;w%Y zZ{U^Rj>I(OF8=a#Zc{8L$*lz~j~`v($360Z_vKu0N;=6AYA%q-D;=ON3Ym(EJzW>z z`&Z9{8Y81g0u}^hoh8a(y4iExnBLWgCKlC_lF%Nee(nmGMAL*$csEs&ewfCNI@e8; zL#cJw9$sQUTs0`eZh6G~i+e6TdtEF2f*y1B>@)jYIuP$2u#4LPU9KzSdr35H52oQv zwt+_5)cPqU>qvWPGiyh)2(wXD>9m1wuK3knv?Yl>*X)KaW9ueIoj`3PRpm3@?Cs%!8?pX_kq;`RR&TK~-E3o* z5M4ao@sS~@4nipTD?dGj0Hk#sw=H8@r_l-7q@c^a=4J$ekuQeVz3}h!4waZ{&fXSP z1NJcqWQx$vjM-zfZWy=SWB)<*-h*u2x`=ehoyjs}1X_|JPJKnuBg^`A%pCm|qiC_S zSv72nJh#>5I&tN$+plOEU$-KomOb^nbkDeTM~lfgY!_Xm8M-Jxz8FQY$As9(nTB%o{xX?ZJM^BTV%-l z%=SE!5z7hJL>(cWNLnW-v?}vUC%f7*`B;j-jXPEE5OUOoaWQ)Ac7}pBCq;v-Phlar ze-3q+#)Ye2cz%1vV7{{N@S}I}E_RGIjO(9n(!Xk4SG}$+#GOu1sk>hls{|yieZysP z2^S)7R{4Iv5wd+P_>sfac;%_^7fZ&IoRHe1Bv6$KB}JaXCx@d9+^-jqGR%VX%O9V7 z@QyR@_*({VZ^5|1009DYCbSVi!KdADhcvBom&`8V5;?{VLJ|q-rI{USt*Ry$Byy?2-*i|e*`f?7C~he*?zA#6DD{gzou7^*zQcEk` zF2s}Ur`wa7KP6x)wm+H(x2S13({BLvPauaOMlD9n4WlJNKwp%HSxU-ebt75;%9SSx zgv&QtupkmPdnNoqZo1@5|8goU6)p_i-B#o%J)o_XO~F`cJ+^ZAbZlrvwAsww*ef;h zoZ)hx447|=eLSRnJ?hq;Sx&kUlr}XTbJIAmmSX^Z6ew`N8lvc<5N=1Q1JUE-J>#DI zn{i2Mb4kY-^AjopK0^6453=Rp?6EOClQ0^y3r%UT=y{N3#k*v~{wAqPmA7HXwmE&6 zV+9Tkm(mAF(hq!fe{iN-glzV{aCo`<>$dF1B`(VFLv7Yx?+*gD6=aXlf)Q8cx!KR< zZw(UQX;0#`VTu6#FK3HF@sQ6_#7U;p>314B&JKMQa8Y{4b@8PaSwE`bb5mIN!_N@A z)fh?omHiTdD@QX%o#(@M&2^b`mvkTwAn3B2hOHZ~AxP)SG3zp8@K)kd4RxeN^TSvf z))m;Pv4SS_VSMgd=S=k7hS80ztQ8aE+^`a9*^CLlM%&Yc+-DDmoD!MZI>1TShuTXh zdV+zVh1}rEgqutc@<`|w1KF?E_@nO*)o}T%BJ|)BsZ)ubtB+D5+&Enp*@hEhn%(#N>T(CaN!BZ)4m(5lgRTb@Gsx zSm#&2nCuINp!z__c}xahLXBU>>{R|I_*zb+AY>0p>iHxoxykAYKRWrP^@oA+{`HE( z;rlGZiS(=5FbP^?l6qA$n;^=VT3B<&;pkgJZMN5keQM^csowACi}zGd%glU1L`_ZxxAN(7T_wpWA{BXPtz5T3B^XbtW;p%Oo z*QWI}IqDc99tn=WP)zF)mLLoI3A7+MYY&rxh`prTY=fL}zZaceC3}~)*{v`1&@r65 z+mll#GG(t=xNk(~{?4oX+w*#7&5xH7xtvY@8ubc8cLN<^PjytHZq%l^z*R#3324z@JQ$r&^M?=fHI>P?z({84Nnet* z_Y&P}A-rE?Do7i~hSDS5B^VNq;s)Sils$x)FdC$PgMWeCU@Kay|03`PV*bm=#h9ht9E_6$lSi7<4&bj3KE+#cF@&|E^50sAVGWs>BF*FrcU z`QyeG0I2FkXAYR8fsd9)7n1M20Hl^O<0y|pZ>D=P9+`1(;v0`7dR?F2Ij)bef>6c= zUIdO^si~?6T;RR&Hcm8VX>SVWNjiJ_tcUf&?=tuHdPkd|>MyEDXS-b8SHYylFgRpZ zrg-^`$kNA^QYI{_rla2DQ@Gg}MQhGAJhL9X)5 zN!RMrJ&`>%#}R)cx1rM+kmz8qK;Y=qjKp6WR$@wH3sxRA8XcW9oG~|a8MfCsX3?O= zt48 zQ+qe`Aw_`{p@bL(Zb>te7u^MyBbDZ1ZW`FVtWD2W+TlNcvh9fU%``q!N_p^=3~gN+ zL6L6I&vmug7g2!=go}gn*Qe;iDD~dWFD`Hi1B#X+_vK~2xVKj7|et(!~@YNr5ITGXRb2V|~`P_>;L9gD}D1Jpqy^k0DLdSvC&RnHf2dlp0>0K?b6b z)BYDUazBJNZ0iV(zuB^jyAife4Pqa2G5x}+X2|`}b)zXncm@ykMLBF#L5HM`<*ZS0 z$vPTUtiGR+}(%zzg)BVQ+XX)naZ&#oTLils4X&9d-g4lMP$5J+&1^704! zqomZh?b{C)ckepxmSz}(PBbVpd{*r;h~hhkcs5=z z_j(7*vW|0N5-<)L zEgbz+S)#EbY7%Aj*<07-Q^JwtL&cgEmvUO_a0&;FAbvx^ID2EZ75GnsiNRbsOXZ}5S49goIM*BY z@edlrPGWZ*_e(tN4bb8tHV!JW1pp}y7_AdVU+n2sHuL`7+Ks1IOVizJ24r~BzJ;d9 zbhp783f1``ePAaCI-rgihiol#r`EoqDRKwT4Dm08bx!f|1qqzhdD}WUMzLH7P>YSU zg0|oe0Du9msW**@d8y`Iq+rrFU6!_9;4L-c|K>}}QSXnk+0}~@^GuzPQ)E?i1N;CG zfR8;*3MM2sp^prFmGLx+B`fduY}a`?Oc&|bck6~eGL3JTelJtK82uUI{)8q33r&e) zPEFsa=!iaZ7C-mqi$!5k;u)^4C-$aGA!p;5=vuD$>qDRVd0r*?ldn@;2*%L}XKWNd zCRCFn(|C^bK)ZVAdMStQr~30~JJvmbT7U6fO20i_0AwfRv_D<2P#1eR0M#Z zcuC@jKrDOtzWz%eqxCF$nz(Q6`gR+VTff|X-!ZB>p8{ICeO<>8M?vk80(UT;Oyf+B z)C4^Nr`N@e*9A^i9;g|U%-)5cdHlXI0b5h3VY1JhqFF{Os%nLO8~O?22I>ujErHy_ z*la9$6&wpci?c|BUGGk4rG5n(pG&|DgeGAIDoL}n?>~GApv=;NSQ$gz{t4obA^zxn zHIM!3UaJbgF&xm-wXu*Q$IqL}(K#b)1Te4E_?8+C7=lY-$ChGiVn)Zibb z{Gq*6I`6_s(ImdhZxkhNLCBMhG*}Pn*z;jOx5m_)>Ob85oT8jUsYfAVyrwX1jjadW zAkO$NIn2;B$}P|zgA3hF<43ZfjPNtIO#YpA82WqWmF$8JQ%{*ow9h@uau-M}1wF%> zf%N+rjOlhZil3U}1>^KXpWq(sYOR!4YR+d3CqBJoAXYc!8vNbACV`&{7LJ7zv}PdQDfs!O{)qtWN!M07K< zx+=f^Wiu?~kZ-ZELSrQFYX>JUgx<%g5osV3`)a`S%JDo1GXx4Tjk5Zc@bb@T2xcr7sjLyb4^N$ps6iH4 zEvxGU7F?aUT08uLxt>R2JVmcR0ySt3D0w;9rdX+~G{jj}@D6d(aGD;ITmT&4x!j9Qsyq&q;yxGJ)lnDw=|I3Wl+Z5~Pb zPMk9?5>!?)&{+LqXXp)hYs)K7xWv`6rTNRX?H|?uLVs`#s&S~x6++Q_`!-`^#=78D zhRaKzcmttGg;F(XgSq$%ceP;`X`d%wOEh64aSjBPm`5~LeD-?>^9;Ei@^Ba8U2yHp zW7YWc%<^vv0|T_mW})?%P#6Qo88dRX5yM)e#+uHYT3qQ-Y}|F%ZqV&!5JrqmC{s8iZW@R zSv%89JB}#alQn$VjCk)8b+;2DoAZO|O?`$|AeGD**Re4!(^T-WF&-?$SJS(j@uLr{ zqso$WT zxCg@}F7GcRH%MJc`3LtHK`s$m8xS!-dO%|pCxmb+VHpNBV1yU+Gjc3fZI6~ms?K!^ zXh80aTs!5s&tPBgei8b%im|Sk)Xnu={M}}xK#f)|$-_wZo@lS4lwtjyn?+e{Z$GY_ zOZGvSu&xwKS}H#oxyq;1s*9t3A=N zg)OVAd+x|w$hf{j*SUf@fN=w9V;Gv3;zEj>8kalYSFUBlkg61uj&V`g z+IxnGLmMJz<#|L*ptlKCQ zjq2Mk*^2s&=yToslr}=ijsWCW!tV&5=$~SjPw#p6KR0?7~5X@-H8Q0|Y-c|YX0&Di; z`qTh5Yrx5dVVq-QZTEqG^lsBhiTK4cc)@~uo{X+1$0sM--$;i=5d_y*BOCWJGR^v| zKuW=XO|A<~$&J35o!ta$wc0nTu%eSy>-fT`|7oeabQk$wd&-{`NnpOo>%l7~C}^^G zl9PlXmXQ<)?}0))Tws|8$wA?psbH@S_AlPO<8sHz=i{T|)5lA1)mk_VkDZIocZQG* z@NTnmF*p}c-$|4AQ_#ns>{DEL-REU3RsQzvTgUL(ZKrtU<4t14^23_C%F*wBuruiA z9+WWn8P32coQNh{P694;?Q6RD=Jbtl_u{NC@5nb*9kfhC9WU_;pW^B>oYtfmx7^EJ z=K%YA5>#L63`NxLHHLq9DZ6anUCb-6#Pezk`A|LPRxVpN+X;{gf{TB6Q`1+x(D#+e zpva?W{6*@4tYcyxtU!ksXne3h&~@l07%yhpgR-wz3|GDFjbpJp+h3(@6Ln5A#rSUR zkw;ZNY|GlhKS6$2y8jZl{>*j>@eV0^$~a`QGPzaDxOGChPm#&6NY6a*K*b}ed=1GX#4(m^uFgSIi2D` zyNE{eUQ55BI%L^qOKjvQJWV|^@_=~vHRleovJvYq@4Q&jr!>Q!If={H#0AiG(b};4 zF-|2haKRz8B)WU@Ag&U1!J7|0?(}N4rK|1HUMa=s5BAsQsyg3_FqL0>mC0;#*_^}y zTgTynY|w$6Wg?otN6ARTD0QVXvhl%v4{NV*(UbDU!iua0U#?6g){6)?lbzl8k-;F= z>UwA^PT0FBV4)XL)k)#9hj(;g&ak0(cwNz_6MwGYE~BmW zM)J3)5rGa7k+0Et^edYqcWFi)(^b?+W!U$l&h(3HBLMyeu#vJy3M*T5yow2eGpP125uusZ$r&#u@0l zX__PG#z{t{Xr_h?dJ-&~qC*fb^LnkEmgASYjtaiX@F{GIUbIZof)Fo%LG|Olq4n+D z#vD*G>)D(6+ECJ6Ng&dz!t#8V zjfloLkMd=iNB0*K~( zVi3B|`=Jpe6{0dV6n)swwY3G#NP1vburwOaQ&Cej`1MFUq}k%Gsz`|0rEJiQC_o3| zj%lPre*_`1{te*Oeg@ftHcq-{7zgZd>~u-AreYPGr$^D4eH^I+5868vR>JKT`84}< z#+5r)(fH6~TT_AhylQ|6eC*HMv42BV_vg~8f8-U>chnw3cTcdA3}?IODR9ER#3CCc zX=BII($K-9gp=7mM#TQ;Lxkq5hJ7OTvl}6hul#@NI)jQd8)~h6nibwD0L|MhnvGT= zw4}Yfx_(3DR8l*EHg*;>rAW6kpPo~Ub=eynU31IYTK9D$rz@AJ4zH<+Dg*+Iu@d;1 z!c?lPZO&juRik0Vtz_qfM-dQ8I{-aHlrsy3(Z&_Q|Qwsx-;e*(>BUV`K4UnRK<8xjPj z7+?p{3VzTTGU=ru5B}KGqJT1XrVq3?o&qOnt0sStMEeVtAR#su!0-*Rqx(J^#vWY8 zj$|CZGUi!4d?crR3RNJvEYAuzn`2&g`0Pv{ON~X?ZyrK-s-UH+hcw;Y3T12DX9Ag2 zZB+*j%O8?{h>O~NIx5eDO)0B`_U`=Hg~A5ptC+Uel#75$Xl(-WBQn~8Lu;i!LBuCP zWm0oA0yJ(dx}%)MevOf$yS{!CR}>2Iq9a=zpV+7Wk{hm$;kJln=tcE@H$v{+$9;faUh)vhNK;)NO!aJYm~8H~J$M^bZq1_dSfO ztJ{k^K!K5NPvJtt@mXi_&F}CpvU}3yzfNpPSlVQ0Pu#vyS9Y>qbur2Wnpm<1B_9_r zuCAsiVG^4H`&WiSCVlS4huG?o)y85@MnJyCas$qU?`$ljs$ugG$!ZkI_~sOEpG^At zk+Z=AvsbqG`1ZV!-Sb$r74U!D8YXzVDLm7iDh;-LrDWlRY^zo$2hRn~j4P7QO?}m* zhaq>Gu$Q$37enBT6a~C*toLL%O#t6DOfi}{gr zWM@L_E_YX+zae{J{3AK;tp-nA2=7!>D&*_Azp#kEGr0eZH~egj=@$XBiSwHepz`X) zjP~7bYLGLUkfSEPnxS!{lhg z7?q~Uk!b;dRRjmQ4#QvQ16kjdzYplYWW_^JC1+!sv3k$QjVz5`j1AdSt~i8CzqSiGmZjL$h@~yd|ZB?!}GC z$?Ko^*vOO+_g?7SV;0E$50lNX2&^ESo&-8Ztu<9E#gr6G?0X;7B$$0rS3JhA9x~sw z%nPNLN@xMT=wt*;0OKk52rZ6xdISpFNsvjumEH1nskO@58~0x5+rh7kxC<1EE<$lR zkO#319iC%xnh$ai&F_DLK-#870Cm>6IJJlvubI0=L}k&e?%i8_=}f(GHScHP}jC_boh zk`Lp~HVSRTl}j{rv3w!f)d{ouZF(_@kIx&6d5DEym!(VgdT49VuzsEQR1_o=mie=L z@dpF^pU~0UoROS9&fc|+afxnmOlVHS>L5lbE;4{R!e2OXi;kyQNb;0G>(uApMvk zPAKgD*wY3J^%mE=9GGow5Xfey4P&IWVcs-Pww8p}!bqhAj+xkqhrLS5k*CY5s&|i6 zhTQ25Tu<6N=b~#8s6Mv?;WUn|6`=KDc{E*K(Rwffj`-FtNR#WD#f6|1E4?F_3c9p*9*h^8npQp> zRI*hLe6z(QB1SLTf;?2iMLq+f3>Exlt8r8xNdd?M$QegCL;)9XRx>|JWI*G6g6QWNCe zJyDT>1E$_Lr!64p(7#PI-8N4!ZeE_mffg)4(PpB0cs^9b~Gd0bS^cLA43bb%D%sY3;Doy`K9yU*U<0n%l`lul#_1j zLn|daevH-@4xXSTlP&+Rmi@o|9z96iJ7c0~>ZDV!V75RwhpVl!KgN_><#ZQlYk0;u znPE+LH|_SQHKVd%H}v%{`{?&$@K@vWkM9ZUwM>Q&V)oT?lG0|}zvfljFn;XhdUxZ+ zy^%+^S}f?x6^kP-u+n{&+=Fc}78s_WNLtVR7LoXK%fwo-3WeIZyqnb~FLX`C66pp4 zLoZ!%u)MGr;_8Bd)*f0)cF%LhF`*7$_iDXs?Dl+2=P*1ckuHfUpkx6!<|ev>;K6_43-DZOED$nHX%bNB=9QD z8nG-(XaO8q_uWf9vHa>Mw)Qto@`c_wnD(*Xdfp~Dcks_|2Ut9MU=GHhv~!p>0`Lv@ zi~)_W>Y!0wI?3vz&Md1C>X)1LGED}TXdm#Uxo1@*Lc^3 zhFfOV7FLp3(e`J)i5-z|@}_$tcu*TMgYf9umBp0lQKRnwH^DIlP*hL=HHp^&{0<%^ z1odcqfMyWw0H_NV&zVMiC-viYs$D@hiiS!I6T(3*60U>?xj;PT+yF|Uc*Qu6G4lg<(o|J6N|X>p41Aq71p{uryWI4|!$Zjz2|AOH z4p-J#XY!VeIK(>ZN!yv^O7A0zFf&4?OT0<|`QZ&DQZZpQduPYK4Jc}% zH?<=##yZ@B)=ql4eTS5dMXEx`9$0nci$S1-vJ z!9l=YGYeJyWiS4}Tux{XEbNnbYs|<=#CMDS|0(C~7Lp5SH=Q6IS2QCf05?2thN;Zy z8oE<@?2G#s^(vO?%e3pXLBBYkY z9%CJJ91$DcP#v5>)V;?2>ZSrCN1&#i*Ihcs|dcH$mdE}p z0{SeiWP@(fRaCIY`boegR>Xdf_hexaAHiwdc;EHTYnNc-vfewhw(AKmBt z|Dl%jAMnHeeyHu=kXhe&?)D&0G*yDvKJJVZ+busu$kO$Y38j$z@v7a|FLFYluQ?%x zoR?d=yVwl`qU6;BdG8;*-%annzP^6M`hkULh}iTcCC03!y%2-Hg+TwJRt0vncGS9L zByHx2$ty=;j{ZrAkF~GecY19>%vzc+*XCLue$@GbDN4fyuoCZa*rD@*=I=!szm{W4 z-Y;A9x0ChrE4pn(b|F0)P52L^rJ`wS?&FkSgIg4-B5GQ`WsDexIe}O;Pwch zLyU*U!uBS9duO5ZBU;g23+7bH!wS; zATtv2PeJ0}hxZ@h;Q#6lS*UTrlR>^GR`;pqy=iv|OMaWtaSJ9Ax3M_-Q`-AytQu2b z;Trz#{{GP?`1Pj$>W^0j%F|VBdrQ2u9NVQjSac^04GZp>-!Wp2FzpGwb>*?%^Gvpf z6>KB1#8PS^#X4^l_a-?E8df8kcYcU1bKHo}dAr9U#F6Q+3FmzH3R(_tQ%j8qEUfW~H zL!^Z(@>Z7RIA?3VbL;E|a^p^C+mDEB&8}om`vmxh{@DZnM2$NxIQ@@KR5uWhgnf^q0tjrDXovY|Q&-Bvh!)#V%S>U^hwC_2iSey(JgzR9-( zA9)!t^_4AZ^ze-~AG+n%!*_c$hy1>P)ot-90<#>Rua^^?tSB@5{Z2~npWV;TkK0NX zYXih@Br%&bLZFQ_jmiXBlO)c*J5#Ne6>4|Ey6VGE|*kX!{jKe@&4Y=@7?fO7Vl~X%U4fTPnWMIh8ZckQEV-O zutsCC?S-e~t4nVsi#>_`AzSM^-PIzq2f>a~BRIsAIk&&O5^~ZCh}ab$}o;wrj=cHAO?t|%!& za>p}F-9N8GSN?^-ylu+v74};yk_@q`kNZlC_%lA6!xMm>_YNDs-veU;GPuG*Nk-zz zLl1Y^%$wPzRcF@pM1nqw1ky2>3g^3?Tk6le{-@S=jBH7{GqB?fEV@(Iql1+N1dY~q zFS3REWcu+@6Vr_x(Wjrsn+;mJ`A=(W^C{oGaePbG^u5;h6kGiVrYVVrXy|lIV25O% z$oM_)s(l;lpO;^1(VQe8i;Vw=z4wl5a?RF;gQ$QsL5kE!6QnBATR^3W5J8$CAcE2a zMCk^Cpfu?!C?G)*kuDtp3B7lu*M!~^N(iL*Jsr^kF4o7!X->16gFP6wDIQcxab_pJzJ z*6*rJwf~t^*^hC9DvR6?kPI*p@J6*Yf77=M&h=%0fwQ3gq6Msyvdi=!kUabP(W`O) zKB~f>Brv~POTR4czxE^$Vp-HA060e$m91iiDd>ApEDnRSZByM7rb7SyzX5}?vr(pZHSFM|81c{^Z9p&m& zgW!^i^=Ft2oL8}4{)t9{UDFnCXMQ00yc-vw&uA8Zgd6e7Pk`bLh92}d`fuRI6n}W= z@*@5XLCu#Djf_#@z*08tIA>@3a;{rDtvEd_Y7~oL)@05$(qZI8yv$yu5r_oscKmre z{65+G>Cy40wRH+^%NUX-ho!}D)yrGVG}^G2aKOum!rrd(ZT=+vzQr63zRfW7MkrGGry_&Yx z4Q0Y5UvWP$JH!skr3F;rH~mnGbHlc_Q{}Axyv-TGP-`3pXsdHS5d2ToZ|X&-pLLd5rp&3c!E^R44rm$)_8$_1v#2F zGELW8!Zc*n%ipIJ8th+5p9!qesAC&_Tv&kyGiM8fzrXg$4AIT&DRC-y{ll( zKn5q#?i}Jx%-z@4ALp#gq8JO7SPpD#6`S39%x;hh;Px|tPenh^_x*lZ`SW*ESJwkr+QRek%;$29_|WGHGNE`E zo9EV&O%Q58-?lKqKUcp=UX3q8#>I|2@|w^W>)N;~I>bKorp9PFtPzC(3}Bnm&5TFL z+{Ck=Uo7x<93PTpxk1Jzu&JR&%2l0*L^96Zxrgf;#e@+7VjQY%1cgPkGI9X8*;Y6o zUjmfrR>i2DVsE=M%>jJW$FoJvqBLQ&!RX;o$qK>J(vr(DFE9UnMa2^~k@pct1RXOd zOte9u^PrFXzve!l7*BhXMGva>H$}>Zuar5(u>e2C6PDcLs z-SJ|$Lp*)qG!IqFesW>s*2#t|W)=<+;`Tv}QrL4Zk3>>zoo!rKy76U#`O~oSTH8 zykJ}ad|Ni#O9Wd0V|10Cj}j_)RQb@oBJ!jIx;AF!Hm z>rYLkZK($Xp!vUh7R*O_r>43Jp%lR6uKW=414c@joihx)KF7T)_uD#>z(%Y0~LktOq5~a&lTUXPI?&w0W<2 z5W83uA5wsQJ^{)Ni53RAKyR6gcqIu9-B86;0EXGh;cW!DH}oIM6DxbXFEB0eAD!1^ zP8Uuv&^ZG(cLZYcVj*fna^y-M+|K><%DC6dJ6tY$#=h@DFZ3I-jUJ=yI{{jT2@zoU ztR}K(e~rBSdc~ZpnyTgXilWEu7MBvMtsx+_sW_S*_PnxKFue={xG0%8PkYQ0dz zzxWQaHbop-xvTTc)Vm)Dq#XsAL=JjD55jzzviK(-8)bW-gs;F{SVSP8Jz$A<#j*`* zV0xYhEcwPllO8qS33Q~3cuTIQg`Ft6H4COxf;2bdV-%5H( z>rUdJ2L)*S?)uG;gPt3vV|itIw2wP|2z`@RiUk5po*WNSczXi}$1~;6uYr%jX#Bf} zvM?JN6^nM=pobQ*>t+N|=ci@8b-i?aZpeETE^rcCXC!FJ9X^`|E}FvW5qTodLYj}4 zeQJ%Zce`+S{kSr`UIuBO-<*!IE_4G72uA?f_1t6-6z|=;-iE|(-9=YI))3)tR6A#% zOob7ZU2u->czih)sC4!Qo6M*VnAEL+r}c39b}6ycYc~L0EPa0<2NhlVaZ2dSEjkxvA1`>v-*iC164kd*n@6w+Sam6S%7FJ}9v+4e*rOF*C^8hHwL27JY zldrV!$@GzOAjy(A)mWbL9kf*_CA^c)FrnL5s()l+-JdPi?jz6=RmqnaU?IK)W+^`X z)dfzOh($IaLlMjy!xL!6AWudne#z0E%&cbR7Rcy5qjo?Lfj>wk$7WO*cO_Zt6)=-( zn>zpWYqcSsLWB~NejMI-B(Tk9SJQKg$OwqW&V2)>E&rc;hN1!mA05Hn!UT-g_6Jhq z?R&Bwe%xfvUJLy8 z_x>eVSbMDm(FEY*+k{(#FojRmSXOz@4yZukI9EI4iWn_@ISFh5+L-kcYuY7y(GPr} zJ<-1-kNi_W?62;fKltQ7LGT9ZGO*YryhB$4x&9hK5_kTY`1BvSJVhxmOc?Kl)gy?J z*{Ghbmm$~n3d_v{vV1m$=EUXVK@6f1k9YoG&TX^ z2wuJG`3CMx=H@B@1RDygKTBFEH@>^%xWsXKwvP(=x9ih?$=V9S{!qOL1Jl)5p=>De zNb`gdQ*3lw*`R<9r>3Np;kBsaOnOzHgukdhIYWv3$tUo4Tebhlw)yQQ`rD8GdFs_1 z*+|zF5J3j+z>3{^{yPV!9Wo%-j zJIh*TUXfj^Qx+7{8E^mO)BOk5*8-r!zD0}!7JDOJ}rkDwCwnkck*|bO@9c~{cErd zaL@m}kt~3*=RwfXV0;$4jn%r&bs%I^aV`PjP)sh_^1&WrqA*7(mX zXQ~bWGBcbxI8$XqiXOKi5B-#p^Dmg%U9e$=JJ?VYZ(;YJkojE3hs&G-W-aUe9Oz#) ze7F%W{xE1ZTWsgb7ZKz5ctn*k<i7~-$Rd4^ zpY3#)#P;?>hDGICWA1WBfQ9=9PPkuftN(Ek?gZGaK7iUK0YwWpuFX8-6&duLIWcCl zg*7Q_;CuWWE-kG-8D)ODDd)&bzSqqDtf0NBe;$_n*B?FWg6#ZSXZi~a9;<;%>#v z=E!Bf$u_xnOWD`|7*FkAv&DY95<<7!_byy@ zKZmaPoO!*2$+3=LXi*)N%TL}URVU!v#YlV!49CuqJC0S5qe1}&^uHt>{LACx&%09X z0RYNG!rN1bm%f910Q74xGZPd*g+Y>avz{W?kJ$q5RV(T|s(;y2`k!L$wgLRZU|vyt zKjX9a7$hL`S+A`=q)BQo*0%LUNOfFlLr8BNJc{3u(**gG{_-M!6_$RxXE?}wi8?a$ z29EZQ^D#a5evr!f&IBuQF0|@4(i$0hJ4)LJ;*30i@Y4zan9Vl0lSL7@X@~D14Os|A5&k;h zUd0abYuI1@OzcKYFFDv5KKt-H==@=~`x5*yBi@`BIDp;_S^>!a(;1=(c)|}sI{NK9 zsBVF5`yYP`jQOnM1CW+KR(}VD&>-ms?tepT$4q?O#!eIjf8CmXU=LS4bj4z3;x)ywiq6ugw%0e3CL4vkV=X{?hd)V; zegUs3)3ESZm5Z~Fp`~jhC_#_c`56ZvlSo6mbJ=zKg~GBun42<+Sfim0);z>}l2y{1 zXE03_rJAt?2eb{~w|wzyVPiKfL9Q9>GF53R5Os1uVN@xkOv83G84urku7aki^3#{p zkt)j|%#((yy!BE_9NzBb8JrsrE(Y*>szbX)yac*A2%prPrka3T(?7nh5g}~44OGxm zPdHlCwm7y-Bu}<(lOBO17hkHbxk!|iO_)x6R-w0ieDy`O#lDy&dC>LD1wdgxMrR^q zVHy}l^om>BVNF%J;y#rds_&<_>*2%P4NnHg5-Hsp7=t0~*T*fKuq(+aM@(fj%f757 z)JYl{gb|t;(#-;>AB?aBZl&A) zqz^T)%URJ)bh5jqcOvEyzdyWGT4Bi*mgd)Ry5^nIH>96ll|>;|a@xfJr{JML)3=|I zqcUB41$6?H)<}lre+RAQz%Q6FM$o$j^x6zkjXe9!f%iY3c@r&3i2H$J#4C6>_V`kZ zQu!<4pe^zgPi6s~k3&~9E%6qv@hsiY(Hb@%!BWue0+bQxdRU%a~>^8noI0zl9 zJ>aUhTJah1TI638IhjyhMAVOapyZ>t94H@InS4{!{Js)t{naK;&jc?2@wr<8L58TEon)Rbs>L=#0^%Rd^;d{8+oIQm6 z?1~25Xx0jfy)Arn#jJ3}v0h84^S$iCM!ADK%1|z%(*RMh6U7)dez&Z$eax&WQ9I6q=-bswF7 zdV1Q`#w37l7<|g`@zsHyimb#|Clolm_89(SbjB?w4^sAxZWk`Rk>&%Xmnxwx0kJT% z{slkW&8BQrmi{C6PQm2Blq~lbYjy=OXJe!$U97uMfs#b-{!$(#o727)D>D7}>laEB zTyqS3&lf4ur+F_SQ1u}2Yjhe*=Bd&@z_d{f!MN-8w+Hu z{d&eo|FFnLUrQTZhuYGxu1L%|ceyj_{6Fd~v{z@}jR#B7EncVrns`#YAa@fAFqTyha zM;%6?H_Y+wa0|-ybo*GVi|Q8ZM3trIpxdoR8cv&<(1_jdAc0nwS?KwH%hTrt%pw*S z4hNbc?oVH=oI_oJ1Id|V6%z-2`EHMV^yoIRyVjb^`y!xc-Tj+$^0y)W$MiL18%T&8 zVhJK_Gl$Z^pndRfa-mw!hqJ;XOQ9wF#I`S5KwVenF<^gxTpqSkT$+2cG0 zW|UiIZ5}at;v>g49CN>Ac_(=d`R`eN6|Ye6z{6dk$;ugBz@0&IX}?<=z*}B_onKJU zInM?Wy%hsmdr||d{qU3(J4v4^&UXgS_`Z)-<2XRtZhAzF=h+NH{_VC=pyb91%C(f1dtQw&M{1QK=a{ec}Y{P~K zwD(@!LacB)4_j>*Q+OA?m8Gu0EUku9CGLwi$A4?SbO3g|=y)@Z% z_G{%r$CIU*t5oWcXY3G7sxNNvOCO6jY3eN8c9O`)fxDESlpJbY~f~%=Z*_tKRN7Hee{C4D(~&QjKWYi%Z=^aKwBw1OeX>%x?OD3VC!GP+K+Yi zPZ`uI2;&s-FG0PBtnUE2>I7g>TYKWXdq^PV-7_Jmx8lJ-0rnp7seSvCa|4jeahW4R zPSgSTIan z=WHU#LSa`w(=VvMmWKMsN%Mya9el9csN*gIF@x0Xpc6N6N%$<~*C_eR|FCMhTKd1E1BLnB& zXz$Xb> z_m-V}5+05QtC=_Hc__QaEb-8|&%T)X!hO=WU@|G6+W5dn_mTgH?lY|v{BxA)9OLP{7o*G=s6sE^B3BF- zx{SAYV!W_MOWE%?fl0CbURvMFje$o8k~_7^^?ZZqRh#nY65`Z$V-F4~UIdBliC2i$ z&%fk8zuKAmxZ2B|%pn=U+bt@C!Im0`wE(FT>>>)kP>h*7Tl5KJEDhduoZtcAeXtvQ zNZX{?(#070`q>v*A*6W%>DA8K_Dq%H1&(MX{~JSOs-AtN%$^GLsa$J8G1Z}tHuUL4 znWLYVbL+?s+3oFx?JjqOpEoJ(N%$#O3non>3kq*EzMRtCQOla}BJl?}{w8Ih*hUgr ze8@UK)IROJfiGpi*k$D3M1mNPld8pSu9M;^!G$dcNI78A= zIzyPs)30LvCRV#O@FX@X0&WNF7mJBiK#9Rq>@h0Z;{N84=Z&BY4?_3pb1~*YRy3r3 z*4_l%21)J;W!lc2xgNzJ?PbxInr22C>bFZeaecr6qI-8e2c#n~0X+(}0 zOd?(&E+-$f+1FftQJ`bInBnes&Prr&z_dIG@1@8FX1#o6i?HAzhQ*!jmmrBN% z<}K)UKJ#qm(09;+Az%{<-|Mm{!)Iai>Z5W|?hLvz@!4Y@?)H>B-#CQ$4VH$h&SAmy z3lie2Y~$lsMS+8dB{^x)q8TLuXMlvQ?w#N39k;RiV*0%C@K|Q}nOJG`W#!mQpmB=B zP}U_o6^-U^rVRtvcii6`<|j?RiM}@AJ{yfZ)eY00b7#dD?fZt@`ckc5-y~Gtly{th zsepyshw{r_VNB~He!Y8!-K4Yg?srg7uFyU=tIM~1Z1roeC5}_0@YX#f$qTwDio){c z_kLMBynTrIP4AiVAEFfU`Vchn0X1eADztW(-KBw5UX~%-aK?jGSNky_1iPj)N|K(| z=&0Nv2oV*&(T2LX%IV97`q>os3RO}KhCexEQs>?KaTEX((-UY&P%YO@i&b7#tanoS zgmsTZ{yg%F`O{)B>egP}d|J@M2O^Y#)9C5VY2|9&xip$MUGAdK_Fe*fPqB(JvOqw+Wv}eN;Aa^dvn<6N8)u zH_SY(OdaqJXu76aVioB08EL; zp%@?(Fl|thgZ$xSN@4tU3c%D4_XH4NSA_sc+Y6!y2(SWrto|L8&I=qYLG5G~5WGP( z9p1+Xgq%A2@1PTaVH0m3WSI(xH?z|4XFyPjbp-EH4gx`h1JVElB<)hhLka-u!qXv8 zKrs6E`Qs>KczjYt z?HIn|qt%RNMoEcgma})fOca()m?A;?!i<06m204Y{$GWHuLN@)AbkL))*YaXTJ;~{ zY@MH8a(~u)WcC9+sRA;DayS?wASS`jYEp(%uO z&PtN01e>O+={Fw~d(>`7y^X>ht)sx5ec^rlF?*N{k8;B_?*Q?_hfDlL1LEnQ%cAIx zh6+-LU`$xIw_m(?7yy;6u1=Qa2O7DQX*&CPn- z@=Qqm@vqbi3J1*wGG_;e)NaD#yRemx>G` zC1~qc4-$K&Hm>V35FOkW1QfQ0i<*pZyAD@zi~0epO+7089-F?gMCpB!!FSMsK0uu; zJ3taI1ItbWasb>tH$TW2WQGWt4g2-al){@70XhhP9>8%Z(hH!7Xe3j^8>QgifF$tc z40%HP*DnG7SG_`P2+nR>IK~~X@8jpj1sEAg3TZP!beX17F5Vf2LVQc{WrmJdI}Vra=5vpb2$X;3(6II+z1q zuehMbg<*^((^|Iy5)TY8k8CG*F79~B^V<{Tqbrbwv|Wn5{2zOQsvV9ol6uCjrihnt zor}J}t2aYAB|8~ksGRq4{6(sNwmCxe6>pFlm;{`OE?bd+46G#o;4Y9KZy@Qmy2e?F zQMRD8YmA-hQ-PbtFA`1D83&<3Kslk^03Scne5zp_IqLN-HlQaA*2y#JOK3jsA0KPx6z99X&3Qy`sBBS84b*4 zm}CjM#~&0@oZ<<{+M_oWkHl|tr+4=DQaxwvRN$Lw2g8#;UjIxx&OM}6XKxzm1) z1Zhkz_#{D&9^{G_Ln9mE;lmaC(^S~RH+d5)j{_$pO2^FQk7#XsL_M>^bAnocsIqYi zP{y<7htlAE&`Dv1dp8lCq0?fYVN2w7*L`~xPa{Xlk689t7(0rVKrhv-COlZ0`_@P- zJD+*egFS_(G|+yX*5m2rFA*vyKua}zzzD}|MQqt1Dwc|<3piewEsnX+xA0nJ2zO4! zUZfOW=mn*Hqxd=p4-Y(!cg_4*HrLDQRK=4TyPz#Gk5U&EZ;q)u9c`gWMPdz>JEr-G9 zsx9V|4eH6V8j1HVbykDs1XLZ504|E;3ouCx!`r(#_}*3vtyuT$E5aoNlR8~Vfipbq z;2GOsY4te)8Bi@N9*IG=ka@qUn4oNUmd8V1d}tY1W!HU}Q|_2!UOMsFj>{y*oFWKn zr_SpN*h?buH?gOhIESBjp`*bDm9b5(@0Xcmmb!;*`DG)$j>0>2If7%=mHVjP`R>d) zG{8gUA2^X!vAFmQt4R^xjRN7WW}3Qy=ZUnb#3s9tUci>!7v5yS7`gk{{iLshdC8E> zMAOW%sSlmvMhvOa*pV}bOp%~|-~d+(#P$*qVe7zdw@%(C$Lpzs-Ty0@KBAoGY&9zS zNig97XfK`dg&LO)V*E0$5{d8pa6F|BNq1kU&r`V%S%v&|*YuPw$8raot|GXBPsQzr zq0HY42*1#9{)wMte1UX47K>)w_(D)2UI=%$p?`b1#b@r5X0%2*aD2GBa%fjaf(X*T zZ-7!RJdQok6{;Y;08YBh=)ca*weziRt`ZKG5sN*!wtrUHk*Xv7qF`_E6^}yfg?b!_ zSqBuK&v~&>mfFql;G;uW$N7O~^xOBfID*g+5--^P*v@%iV*Y^7;#?ctX1{ea)z|Sq9E4g=<=A(W*D9oMaqL z9nAmhR{XlWp#U}rk(HgSKcfj8dLuv|pzmIGV#8ywx>+ah57YvzM)XK%Y=W?>$j(C8 zeG~P?8n#!uA1l+o-sl%@t}~OIEzpf-aqE#kStWZ z?=8otEJYnt@V2d+JH0BlS)6Jh%pUUW{w+mO<*A#w&IA2xWp8%ykkeRAm?fw4{)chwF-=8xafbEluoUa97bDSC@iDCS!N`)UR zCRJ^n5d1ad)6aIm&*Bz!(=#D6K?bUP5De&V(Q5*?n;eB`f#c=F{u7M(|HX4uUDM3? zdaOK3c;nfe)$PDue^S|g^M=yQ^$JW>Qh#P=@BI&|O%K;6&KyX-E}-D*wQZy713WQo zc;U-kH9(ym;t2l}(DnD9F&>6ELZdm!FCM=GV4Wh!bGJNME1L+tl<=I0*$m31wzJyII>oiLSFAo-LNm?>`6(AOP(FM{V|rR|td{lkfO zufjSu-x+U$A!6Qc{Eiol@te zA#gh-9qxzKm|*8@Dtz~#5#y8NQF><%P=oL8MGi~755lW{D;y3|F+-U}>_S7wshYtd zfs*)(Xen;r5`s6$ZF4t($4kLWs+9lrl|eqW8xL>3B=taNCex3MG!@eM0s(h~cxp=z zqxJb?rv@e`ZVHUlm}W zZ#sm*c8XuO&=Yix;m~W*;Gq^SN~ed9FEvbaVJ4AOL^BlWM5F$Yc&&gF+xbZg%MX$g z^UNj9uP8;X>2B}NP4-doVU^ZeIrle;#5p02dSG5Kjj!ps-6>(iA->{ZM%lrxY8zQ| z<>LWwFTTx!h@>+VqI>{N*EsS(E4m;X^IQ+Z7LvUT{piA06+K}(k78H#H zZIaItp@79)t~+L~9zw~FkDYU87^ThTy;ojZ(Z8`c`oQg;>1fwc2j{{wQOplPR0tw< zh-nzE7CZf>kf_>E1fR{EU6yn&wUWb1ZoDPpUtVoCWPc=9t~EWMx+z035ZG!V87&C7 zHVX%wZO<&jBl5%wP_vKu5;yTJq&$^?(CLa z{0=f1am~M55=nQoRA(VNTsVr@?CDUc zKJSQ8%r+O~yh57H1x$`y1E_8;9UY|h^slB2yve{|Xm9H?Zxd1!@pc_E1$j6OK@3x; z`Jw88T_Q(ywm|T5e?Gnb{!^6pqzs6Y%fk6(cu{E8S&uSaLQ^7!zi8ROjsqNi7zt`c z;tdHFe@p70MA^~D+fNrI^Ht@R2l*M`;i z?2cyS<%Y5*>01UGL^`|Gh9I@vUzg-PSTg14?R@m)obwDAeh z{#Bsf-U-2Burtlw8qv|yspEmsr^?FESLWOAosKTzraKzvdiFZ>&30i&rRgEJtinL1 zzkgQsW}uxV^A2~9-g(nhHEPw2;kD|qnVccVBr%|s!g!i!hzYH5LP4Gdvi8F?Ke!l@ z-s5Ow{q97aJnuH2vfgDY@*sbkr;%wxVqs&s>-@L7p(a9iUEl5nnieBh7P5DU7Z8hL z;aMSz+4546DOsmmXj~=TT_lYQ3I!2L&K>&og%yzUS~YmT42Fu#&WzGWmQ|;Zi>M;Vnl`cgdH218 zZ+tyw`+*i}IJFiqy~g!LgOyDJ{MA>RfV{wff~Lfa zg;>cVmp8iL8@@Zuwfq=({T4HRWtSkb_m#0RX}oy6ty}yEn%3c!VfiZ|ACaTSx$TdW zMX}!mL~}0q-y)0-<#S7w<+EgUdRLfL?2+9cEkG#(`vDDN#)#=tMcCFSWNJL6QNOGd zpeAGeAgpV9!4WZ^WN3EU)3%MHMf3&vb~VN}K($o9(t99$b*5nhyqdBuzfg3fwk#mM!?y zX#-hd755fq7;L(%JwfInhmEH3j+7!;9XTB$Ww?-KPRY%w^0*rZ_tWk(r~A&h1a-c; z{h*F1O>yuP5#Ei_&1j$BA4`bJb5FQm1O7E4= zj+Gs?d-jS!y;b!ggDU51n5?i<`*)CyKx;-+3tFJtfTnP*YZ43IP$f7Kxd)-qwX)=% zX)br#OKa{A4)yW6{0RmE4)Kzbu+ph#ll>dw9n~SyAr~4^tF;!7e8^vE(=KvoJR1J^ z1%^b8?U^-7g->(hD`xeuTF&oZChx^yQ|&dhN>dL)iRQ2UaG=+%V0j?28F zah@Mg68$Uvxttgiz7iD#?g^<#g#!b%s=x=lUzR6d#7<$tEtmaW=rjcjf<>0%j^9nt zeAvz@P;L8=$zDs@u%RPo(XI1KXJqSTMyY#AGSvbh0gU7mCUej9J$+kITaD<>1s)bn z?uDx_2n{W?wU&L?v_J>Oi)FRUoHvnc^R}I@jrR*O7D&PeZ!VGnZJE;6_5Pd{9JmY` ze7{BmB^H`CAjRVAFG3_dN9@HXP_{D2ZEA2eQx@kcNGyEzSr-{Gd%sSAtP$INYqXd$ zBckV~;5PFCZydZ>_dc+H&gH6>v z$yOm9Q1Ap)LG(iJG|HFZtA(?9WDfIslI&Nj;o2nY`v)D9+6EUFG8yhp<4EAWz8wfB z(HHMG_x7r)Wi#Tq+Kdm~kue|;K8^7$I(_eUWsEG3Rg6cP%XA%P5hDI^8Rd~$b96uI(O*I)YdAt(1B!g8@Z2r_&vV4%1<`6eEIVT{y zQSKoakU?OCR-okanjF*wDgm&}V0Z}c5cy^aCue(kA2!*yaueE$wRt8!yCsKoe?qt za>U((O<=r=y~Zlrml6eys`1LGcLm4>VWnIoqhaM&_ns{TQ^iw06K&QtjH6cQpdB#v+kt7jWq|m(~dBlKOsbB>Co<8$Iruo?&i{o*v0$OwTtAal_ ziZN*49|^do#`f~dU19iI+xYcN_Io{rb%^qjCM!zx$LAOyTq<4$#+u_B>btQ^(&~AL zD-?__K5r(;!FlamywL3rQ9;sLj~Xsr_Y&z?p&8JBq>+|OumNCZvvmOZcLQ)6`oZ1Y z?yIU=!`*kjp%ZlEbSyo@xH24K)ZBZD8v3LCQxo`8Sr+d3hdq&i1AV@_{+ z1&9tD+COFDY!??@XJOX4ZaycN>LVL9*D1`}7n-;3#>~QY2gIG=d=v&n$4-QxN16a} zSk9!z*iiz?q!0k(ip}$TzI-Z@>!A?vXOvc$yJWoXCwM@RD$L;KK35NA67%CZ@$ycg z+zl+_1}!0wc&fymop=i`fXa7oD1T>mJ<-eGS8~()qz- zCms%n9~RD5v_Pl<%6fn;_Ei0&tM@lMHDzmNY4RmcaiQXlsN2T*&IW&Ypy^?fDSB}} zKI;AKdan#A2kz2+3g^Y?5GoVPsE3Z_T3h8Z=tn(#XNb7d&Ud}bM?o|^L((MFj3fW^ zsG0S)TRXc!O4?UO(J-_;k(JV@gjb*pe_=M)5zMUp@fudJD=f`qi!z(?(>+sMx9d8_ zY2poVZt@>EZanIP+=EZ>)k@+4Zas<-_;`+W&K8CzTU4v15t__wkJ~fl60ubw)wU9^ z4nmkx?}j%JLdQk>)-2wf?h!xhk=r1pLcJScObmp>MXsDJnH0Ty7?|sN zryx!+@^VXCL07$naMX6|B4h7qTGq>>#tW7wJ}3&=jg?7gwQ3I*l(`x#NUX2lew-4K ziuzFs*nA-H@XWglB0G~d_ziHk2+c1qf)nr!gr*31jTlV*`Y zQAQ#!2;yzer3fbt z*FZ^#Ps&c60_;!TGmGkmfj**Bf(|N0A3nNO@>IYqkzu+0L+0J=C149dk>S>|ct9kz zMt5$8*?KKGeMCxvJmgLIu>Id(cbs0XtatCMX=<5j4-B>3`K3oe)jrLO ze>+D21TDiNYS|}!bI8yW7QSxEpJWo|*0CL~`R0|U*S-Lvnsw1WS5pH&Wx2Y!p})R5 zM3|4z1pOM40*DP}@OBvK^>@%Z41CB_=I`gWeZZp-Ft&f}G1c{`j>sSQVY}cg1t6%W zwu-1QTi>z8>gnO3chSmUZLWXY5*P$dC7Dq+SEcDiZR+@%iO}^DiYi$zj`=8VV1D

k?DG530+4eph&f>^(i$tl%zN8K|`SrE__61ld z0Nj(sPGhJ>A_JyM$|6cQ4*H&_f;lds5J< z>`*L2v}c7q4%A z2o*ShW)ti(d5g=PHLy?qEOli9gnmqar=qNDIna>YuIHK7mGtm7^mEAlfZKE+AMrfCId1~rcy(Z}eOC|4eA6-)nSdO@SkgW0!p0b)YOF^goeJiRzYOrGgIgQQZ7@Jv z#U5ZgTtq(yGj9@v5xCfr2&xy+DMQFqjNOe9(jBZ z-UVV;)LC{RV>7KlOFJ_U57=_Iu=Nw=(b2A6-R_P1O5$dBoGkQr@@|xbfM3%}URt=! z__gc-Xofkkds=i5=9fpjjOW1muhq%d3VFzt-9Egg6dgT3N4BJ%Kch9DDD*|Js73lA z#WMT^k!P-#%1J&B_=R_m=AwQdC1TIuX_Vx$9fftiEyzlu2g(xmt` zGXDFr`5)&0{O%VhmJp><%_3^FL{==k=?l-%L%9@zF|pa)X*Ge0OR99er`4MJsALIL z#BoXJ`)XhQ@J4E6v8f;qVorp|YE!qq;*4;(hna5o$6d(@F1st-8O=()dd; zGFZ>@^{u;iBfjNku{Mp!Q)6oup07=Pxfp5Zl!c;Db1*n9=<%)$#WQTl?DQ+Fd!sy?O=ZQ< z09EApqH)*5b5!TM&iiPMsYV#-f)-v?e^zLHn!P2~3UGBl!U|b2186!i95c=f{k46y zZ0hWAd2P1~droaOR*C(Y8YQ_AOuGkRhI+}q^f`B8rfWpHQ?1bV39)ixWBO9g(&M1) zs54Unu9wixL)kIej=7T0iB7gv;nSwWHL!! zH_{>!+#H*ubit+$kwJ{1#L)sGu~3XW2}t#J7rZR&Akk7u^R|iKK~=3w2PF{P2L1(R zsS!*8)1Td9wKlh2Q#(@@%5zauKA)#6;Z2CPSGh`M3|n|-;t_erEA|U@Qd9;Hb2542 z6{9U4yc5y$DJBKlU>K5c(wN!`fcQO@R97V&e!n|^rL^+v`J!WqVky}L{8P)<9=;Rx zI~w;uBfDAmYH6}eoyK6JmNdCw)6a%@ii)hmiv+-+XCvhJo2Fx7ShK8y)G=)gxNtf4 zHYo@2ZWpctKs=&@6S|jarJUVjDa=cmN4f`nUmva#cREoaWpibiH2<` zFY{Z?j&jGf7L#X6(ngQ7+%LAmZ;!k{nXQKv5Vf%~(t4R?oB3$Rxw6KdVedXcgCj*I z4AwpOkJOzsPPo{D@0~WgZml!l34ip=OQ^hX9*K&Cf0l1_z1-Zl1}W2S<{uw<4jj@d z76$Kp4addoyKd;8yiv}-$qTAx`S8H?nl|6DIZ=`EC>fobuENS_=QZJqJv_Wlt*d}? zSZsO)$N5ss_0yQ;)Rh;S4~lv@t0L~i+eg$1a3x&PzVS^u_`JHzpo^_c+Ct=IHGpds z{2mzYs$NFolvW^zbVJgkP`l+52X@^G?lGKA*YWihcHD|31 zznGf3r4V!aOjJ#RqQZ$BMZvU&_nA6I&r`>B-KHbhsTFI@bBp(S8<3In%t*Q#^T1>Y zfV#5dqcltfG*;CX_NAjB#{U?2AwW8pB8^W ztNh8>+V9>OiuN<6k9vxiHQPs?B{>P>g?*Nz1)eljxZfi$1|Tt1sHvHT!caLio*{IM zg!Q767rm}PO1E*^HBiuQM^n6*YsV&iu(t5Qjpjh%#-%q^p=`JMKIFr#M%r_U>jO?1 zI9~b==LNj5Xh)A5js!0l8M@fV|E2kx`Te_fkch(W?fZ5aZ%_*ymNSv=?WKs>sow0? zz8eiD`ZiN=e>0nicJjc?w%ZKwc7Uw~*^RLvhc76l%o)0yY4nq3K$z;1+X5C|GX zMX{+djX0Lfc0uI7oVqDmYVEbyR5m&ek^^ zP-G+#Q;v<0!o(rF!UZrSenvnE%&|+=z<3~73*hZVJ^vYPsWhuH~cEd}?`@l$%iaAYNAm8O_tg7Qj6CuA3kC30)gGv!c z$z9fL)x~ATHCX=6e49?9QMtSfd9KP^`4B~2Uu6IZ|`8_x=x0e5Zf=2-Ar^H%JW>BUAp=5o_%^943%W8NTb zZhWhc#7MFj^6s;zW;S?_-l-1>s{t$4wp}pr4_)k{5L9c6REQ04=_Q=StJ-WF&n2`# zX(yd@v!Jub$T^{o)IC!WMr^PS>15!U)~XNomlR)y%;xmi;b~Aw*12(RyQZ-qQ5+QewYVXI=o^+_&3rLj*7aSg&5oO1v^Gh!4Ovq7uUohchuVMlKU?l|-%;s|4P^ zs4b$Y=%pP(mqUB!+OClc@|Y2Aa_ZE}=;d~?<~^m}5OBqQolRu6Q|6k*7{(zyla{+G zuoPB#ZSqJFP!Q(YR?fPKv=Yi#ADB3;z8Z;_&Jamko5KP0*a)7}(&e$f8dPUu`PP!b z%oi;YDKkaFK>%O?+-iEM8X+J&JokYxz^LA>KH}Ia{d9U}Ren(LJ~o0f+a&r3cjzs@ zkOkO6z1hoVn`937%tFG=IzP@?#810; zOShhivv=a4yN4D&W=_TR@>89Fm{GIQ%M32vQ=3E2oo;5V^+r!~;WN>_pvh$<70kmG z1!3y;V3^WEQ;6o*60c>55X|2R>2P#BdDVle*Sa26XlQj>;{$1K!94<5LIpq;ov6(+ ztRAq-%jIMC-ac+Lm0lfT^SNkHUU5M^RF~;`&stnglbDzoQxx62_h6G@wk(ytV0VU* zw6}-qKF%5cHJ|7}N|A97KSaf-x;^HqluA3vBHLWu&h!!l`v2Pd?x?2THQgvE2x^pG zgeYCA^iHr*L_k2PLQoI{r1zF!0i;GyK#?K>B3(KN66w82CrIysKtc(D6z_J{+%xB% zIdkss&N;JY?plX`SPS;v+1dO3zPCKj^S;I;R-Dp_FG{REFLUxuJ*+tU)hfz>p$$J4 z!00JL5e^XJ$YsVHp(5lU;Gb$o12Og)E^%60x^>ZLWks=9PjyAhMUIuXF5f-kcsbaj zm)?%uTXQ$M_Wm0(hwQyzcply|TTaMB6o*_NX}lDWsDHC~?>)u4@8N+W1hZ(nQy=Kr zaRC`Nr!airZzjN3mXvs_r?X4YG}wGj;{vsJw#u-z^b~*>ur#=ktN;ys2LBi~i3q$*Q~A*WD|eZtrbweb;*OuNmn zRz+*gxJ|fSC%TSq)vK_HbAWY|M;o{dmYd{3wjdsjtDI&(GmoIxxfRFNZ1;5;x{h%> zJ-an;@^mLrR+9^;w-{_chv6#W>ga&i#?`xWu;U+H6i-Cl+d6nS1hreC5w3k283dxh z|4L*J1o#Z8)up*>Dv!phiZKdfT%m9D)%!+!3oqPh&lXI78}{MsN|=Tg(Fx&8vZ9Wh z!j~{AN47x169)1E9y;AEz@%o+>3V%`QPsRue}1_sA@MsXH0s|%`oAb@fbN{6LpHEx zk+pFK^>YxWrj>R0#o^WAk~g9A7e3!D&*+tX6?_it3_7$UFyPzk^N8L+Tv(yl$~iz5 zMA6F&4@Hol`+kyDoXowwZ}cpihyP0E0-zXK0|X7yS;8{`mU?sPOZM}sPH;A2c%F^3WPxGQS&wMQSk`+$Jzp$4{XrjIFOyoEKOdg{@&G3bmmtOAJ5?ASFI7z|9 zvckoG_LQ8b*Q&S#+A2VIf9Ihu@)wBb1K0q!9YKc9^YG5L%&`4cTzb0XY@KJ}DAZRf{0*-DoDGyv3fVi2U=D0ZBkT>}N&?yA# zN_T>5k3-iN*r;3}(jCwT(J;{5F-^w5W=nMav+Rh6rN@Y*)}0V-9No)(L`0)fLPPsl zgT{*dy8*Lg)<-`(@NI8DG z>nO!;Tay{lECn1T1au@o;vA`DKpwUu0Vo1u0GFkUTFWK_YGM>Sf*b?r0Qi74XY76? z@(XklvE4TN3&i(Zf<_N0@ZM9Sp-q`2Jg}#9RswQ^KX^$9BlTzXEZb(I$a-=n_x|=t z%#0{fC$^!HOZOgsm5g$f&@5NT;V%7LdQqLylqN(O=pg_rgT0`tZ$R1i;_T+Wk8EO} zY=7Wd`H-8~Johc>G;!NSWGaT6++D;0|5e5%uXOG8_ok!!U4K;%m3B$OtM| zfS<(m+d~#<=%2SYocQJN6|7Ibt@eWMY`8NhJvzmism0bfH4_iSQ1zkO+%t6w~9g zR|V5<^&V4pTO09i?v~0$$#&;oHR-tKNirjC4tzlTMUJ?VMK(Xmu4g5mI)ZirRA7I~ zb2tatg?Qp4`Q}K2#X_8HtICKX=F=(HnSRcPUo$w~Hve$E{%;}}zzV)9TN~vlZK_*eLi0iu;%wnlL={XcyMp)gP$L$*699z<* zEL~hIuA#dN;hX3Y+VJ@$CYrXZrr`@riIN9hWbwYHSgL?#f^ZM zY*yQ6-_%XP06n|kf2QlR_No1X{9A_mx>VEQ?0T&G-M~rhS!g=^y4ora1Cwps$Kfq9EjaGAI#XBk4MT1jVt}BN?a^Ux%2|0oM`6zr1418 zy0zwatnXZ6h(?{g>zhFU3JZ5BL{7>XhSI(E z7ApLt2IL8z{R=(B3(+0N$f0UoS@_HwAj+xW|9>+L#%Euod_0GZ;eUm~?>EV#_+q+UkVfVH; z#tvA;*6X>6?T6OEO%eJkM}0ZGk0W~>BW%E`S%X=wL zVQ>B_xc!o>U0~&ec1UNuZ*6^Fhe^rKD6!Y@)?)1a^yC#?*o_b)Kx0g)8G-YYVZ<(y zcF!k0*Bs(&ljH%tz{W$f^t=EH;bJxIUe{t)M|ND5ibwV+_t?S14-O%!V*n)V2;_SH z@nrQyL#bOsaAARzAnVN`aSvU&ol}bsJCLbHp5W5An%k)bF;nw=iw)X>1G@5qBuqoa zTcR*wXcXJ10v`f~QlcD7PrezI^s&BKTD2{tiJvNLc(N{}v7t@NR8NDx$zvck+$?3M z<@;#d?o&z~yO2*N?q25MMH?W(D{G$m1=1-M^_pEk8ICf7@iIWqg5ajq>dbj}I_ekF>(r(BsH!Ri5C$}>EDrVs*E#+B*)dSKaDq4dFpPX{x zbpZc7{Hbd}bNT5!!kg}LkauT-g9jAbNmW#85aMau7A7KK@AQL3{V;v+QHQsDQz+3u zOsg}Ki>QMdA$ne8zwX9+d;rr^fEwzf?PoA{HAu;Z&1D?neU6Lin2B}XOKgJc*2RFq z>ccdU;MM1~x*4b1&(g2AEI1n{8u?qtJpekuYQLg3!KLwv0QPN%V4WaFZv}sDP*=#$ zT`%v|W36N}>MnCrdle`4bgllbtw>4i6Bf@?CDoGUx18H&$xKHaA1SGM0fS=t3s$RM z(5Qyb?9jxg#Jeld_Cp3#yDhLSU575^Qi08ipr8xz>DgG#ADARLi&EoneDJeTg2>F1J^tH!ZU zmL(&_@R@`gWjMvAOm^b#0y|b253_C@@=?uyS~vc}FU3s5;{3HAIs$?MuRcDReO(XQ zHPd5RHPSk#XVVg=*CmjE2JQ{0oh|kG0Hvw*BE;IEi?}YUas#jq9!cr}A@qk7)>QksA^u)vifFcI!PvM+ z$;?}adq>|mU2YZHT#Y0mi`34StZNA4+`<>sPd;8Au2NS(Z;dW0n=rQ*)spE;INRFw zW!Z{02Vyk1F-JC4!0vI2_|!kA??TzlkMcQ4ZhOx8s`6-~&3ixIAn(bqjp*BE1A)Yf zkFI63##ZT-l|N?^f&sM3?LC5J8faH}%4IIN)-^<1F={J&NccVPeY4z4=Tz;_KNTAC z0vkm9iI4tU$rC{p39#$nic4wd=5d~F-RXQ(A8LWt%@bJcWlFqJ;Mr2hsx&wL#rHxu zD721rY8}93GxF!ugQOU;tq9P zqCw`+hrmR{5bcH<6ygkJ?t6yF^r8>QWESkTZQ6IH?umNC}>)&WJ2gzdD z7DWG+ApKA8J~`p(U0o#}o^N9~8(pBpa_r9loPU??0igjaeB!HQ8A+XNhbzTa%uQ=p zr7N63wjj=l7mu&h4(HgG-On^NDU&griJ$P9)`Xa@1nBpa|e8k&w7T3k)*?`Zirz9cy{KBm~}y-U5nQuoriZf#LalZClg z(TDh(ppZyck|2B>aq1~<_B#^U$TLFbKgMtP1s_-VZu29R<3-u^OSaihpT%_0e-k;5 zAl?PgBk<8>$O&Nh*)%|C;JX8a_YBXe3|9fe;GaWLNDY8t+yV^a4x&j1x|S0H7zoNg zlN<#k2clr&WjYEA^4TxY!PYKdRljw};lPlNfE?ljV0^C-KXiydG6CO)9x?&8_y6M^9#?FV zB8oKTXmkYQW9GGzXw%fkGM&_h+f|cvU9Y{~=ikue9VaPLd#K!$%wk{thpxaL(*~Ev zU$T&&WpdV*oxZXp;H)d%k>R)X&_JexfrSSB`5(!D{#Uf~ul!O%#iCdn6mfdEs6C{c}Pwt|MX} zah|AZJsgp_DCVke>=L7s9eyuo!mih&z{;u@;&=L$4pKmmZ?!QdE=Z%@pzleb(%S*X zsH2g43kTFjz)vTvaUlQKiM%g~;^a?iqk=1!an^QUxl6_paNb2mN2TBY1H=G!+36c9R$t&Sop zg~_j=Q3!(Y2K4Brgy$etJk)4?0reEBeRzRx6IlL2nRD)>D4^ABH>R>3!3C4*Y>@!k zT_Xo}9aE`O8nk$D3z&5cT5Z@oS`~R8*YIl6=Sqv z>BF0aPXtDFzA{f@DR5zp2L^ZT%X|)i8U`$qEWK|CRCzC~L73H8KN1g`4|v_)l~{E6 zpIp%jH~^n3{5gK}5xPavtF1~CY#Yz31$y$~9a^^IyxW5W#(z=8GR6#3o zAw|WC&6C4aK;_V2i(iUVV___exRUdH$T(hxfypmb&Ei_1tC=yw-V7BsPChdLY>t2u zqCZ3Fhl$k}V%7Y#921kCweyj>%F1O0F;?mRyI;7&X5PPg^qz4M@UGh8A{blq(Q?*;tVVe+;R@#`{sNgakmT-6T^A>?t$k3f*9pIIs6QdCTXwv2 z3&ZTPp+}EiDaTj+$Sf>pzd= zvVKhPdeqqE=^#n27B@IgZdC<|f?WHW1mgopy;3~mdHUMtbq%bCmr(^3<3l}N5OuT2 z{BE;cRmm|F233+*`+bCsc;o z&Gt|`MPIC&Z=c)vn{?!Gtqro`J4t;LdmwOYH{Z61boC;0>|D_JYC=9kk{v_&a*UIr z%OuR~QDHR!6RH}UO(!B&|G2sy>!o}$BM4Iqpal{(xwlW=IOs*}02oN`V!W_4&@KZ; z&fsgxzK-Bbo-+RV0V~|apV>+p&rR{rU{=l9m@E7F!=R5OR8<~opV4=rw&YTAN}oyV zgH1H5JtI?q#Kq+nb2eke0ij~jTz21;w`F{$aX6w@jIm{3@MRjM*e22WXJK*DOle^w zTFeFLESHaDi#5#hn%qU3&sF{suXm{|y<}S{? z195WC6?Pjx-i{4kxP25iWO!n%sDiHYrv4|nUB|a`83Fw3_p31Ng$B~gGHGwBP1W)i zN;hfdXPE+4Ru>RWs3@-47BI7oYLB;2GROBF<(_i-rFfl~v@Hgb;eodDOon{BS`xdA zqoo3lI;sadOvb%v#0yoU|C$1Ck z=q6zkiAMW1VIe2xp%EJAha1k!#qmnWY7H&y6k`{=FuD#7Y84esy{(D50^_GwMH9V} zSI*R_Ry?C*0b?1ZB4#U-pSynodYs4T>lys?!CH0}{G4)YUp{zu7j+K*wPUIx;gG7pd{r zhaK(65z1!Y5TKJCgd0Rpk}`Lnolb4Tc!N=XR{X*J5zkc_Hb7e$&wuSZbC<)#NAaKw z`*@Sm>>qxz>Dn^mSPPO8z~46rpwY)El7(QnIE16idCaUcwV&ibd8N+|tXw4LZbiK( zskTFz@eB;J>s8g9MtNVi4n3#FoKB=T>UTcyFwdbqh=C^qX*wt#hFj+t)zA8j=tBT)Nf>G-wo5;1mM31_Jiu$6s zq3YHJUOrv)m}IWu<1JmO%`rhr{yZgjAWnK|UHdk*joQCKN=72kYO{n2q!d1T15!s& zIE;kWbq2e7b7q_%-Gukr@DT?V^!eLo7>L6Q;K-vbq{9=t_XcN;&3nVC?vmy|FO)$7 zV*N@AIKp+6x)pDaF&mjWuO{12n^D{io^Gk^ZOAEbo62~k-c#?qtFC;M0_n+jt74gc zx*AF4?IsbG>p4AyGRt=1p_y|;<+<(1OooAGLmEpSSzR_+v_Kqbl5^PL9A(F_4|ZBH?@*vQ)rvzF;(UHR%H0l;M=O0C;(8*QDnc z$S7{Y{o8Q7a|HiH`>pZXd5s(6UYUSuM`SIt^5I(0Xibb%gYd$@=}ws5*po#Wj`sb~ zqNfSvoY%#~3dQVWS6Ohxlxvvr+m}Z?)V5{`?D(3vn^k$(Q>0q~*&V6s(PhuI%xBQk zxl=C^O?S;Z%}O<=Yi!dt_B&gEm6C!rCf-9jP%^0}z)uk;iyN5`%|?`(CHY03>zgo# z>DA6YYGH4P>ooFPzp9yf&;e*EjtxbE4h{16@^@YV&UIgFJnOCzv+OV*V7Fjq*}W0w=pj3uOrr zUgX}0@yu#W@qQ5O_9zvz-u+H;X09vDyu^?GJ8xC`efvU_r9{l-d){&H_NqMJhW{Br z^?Pnpl|MOUU=m5~XabsxOqt~+Z%ZW%M^0y&hTJ78<7OAlB4I6%^JFQPLdzZ7gosJ( zeIQzrDdAFBADmmHJ>M2gXLOiF(m^z-YVe1MlFXttIJiqWi8gATcx_%^F@ay8PYEKF zou|kQ>5QC@Pg8$)F9aNlaflG*LuH@vowSbZfC0DXAw?ccjt>Y<(ilPXJ9RcQxnm^u zzlK!(0Wj?(ELu%AD`21{H4|VDd}{mbq|#PYZB|_#TlA$f*TT@V38T!TEq4rG3IOhL zjs!HubyG8H{W05XkL-(9p zw07k9(kB>8V_6ZuZ7_n(t*Ebgt?CeQQ)JC^BA*}s0jy3`C&`mJiA<7$M9;Yi@lv-P zAykG|ZN0rP#XR|%g-Kp_EgdK=<1x+fzdSOv4I9b_4b}k1g?1oYx5TTN&IpYa_Y&|* zts=`|5vK%6`En@fII#6wC7Bxnqef{TFoGYIm!onS3Su)#kI0CUh*I+9ffq2KW{XP# zc-w_I1Yvdx=&D(X`htaz5{jvgnJb&KEAEFitA1PFsWxMN%PW5SQWM52E6Zlv_964a zNC)CLDKJ18&O6;v8gP9+IWAZ->+@s}ZMEmm8T74qe1B}nU>lZiD!pkTh4F=7Nr~a3 zh!C_gJTeojQ5J`+7?U={!*CsANAc--n4{4C*M;U&GZQn-gad`~^Wq7H%=u>8TC_6d ziS+!mOlLfWhQF$7y-CVw3#Ds8(}nxIm4h&@1;LS@EVp*VizE_f=xe0?rno?b$qsf; zy9D;-ZW$P0qorfn#qTB984XubV1{)=zAoIsQ`{3aGb`u?!Nki8zM^Yu-y=+ouhQB+ zyPBntK8Q9p4d$e>&+pYD1O|(EZc3yMy za<}pOhV-s*B@@$ULG zR(bK+_XKoJyilcI%En>Z>+kaz_Y*tAJF2MTfC|7L*WFgR9SH=Ld-OTc91bqrH{z|5 zUY_Sn)l}^LE*mn$_5_e=>niu>U{t$)&ZKks+{@32&-gG1q$*T~7$7&?{`dsV{ z*JSVZJWxk|5L-sd0rz$K+e76hW-Bv$gT$REg{iO0e)g$=pPEcw0R2@`JiBvOSlY zZS6Buf{>qAGo6oYF7$a7Y4Db?b3gYj7|@v%njU>)X6{1B3RJ~zWq0*?1av1oumZy- zn^kA^Tu_dAx)x3=eOTVEP4O4TnduYtr&nnS6tVrrvt6P~$`7+%5|@7F)KoZ$!MX%P zN4FhDkx>KQ^bSfj#+l7YDe0x9Gt_OF{ZOODe@9jSf2F_wU(`Ey=n{#q33rKS7BZu_ z0Qkt{yF$!@zQVfH&9+J;tv_@2NnPP6dR+^1nq*F;V>M_0FY^I_*YO|kj5MuK7pch_ zLrONBH#az6(`J7AxjwN$lSDxRwHYtF?SrLHUHycR|pHv)2j6lpHAyQ41Jg zKiLm~ju)jT5Lv_+-C85=pPS799Z$(VB;U2+_;+N&;1{|(bwuxXMqMGg+)~=r`=WLm^WE-Z*xYGVU|MGm{47}j z=l^{>(o(y{kj8>Tu1|>PlXg?`ie2yP?v`yGY@*@S?;zj@`!}ESU%rF0G>h!tQ4t89 zN)8{XmuS#8HV3mW4H2dhm~LHWr+^$7k&(srM+G07DO$Q%bok@;$p^4AhRnQ{FY#JH zmCG+g3~@H^1!N(!O=~a_QCX7g+hg~*!kg3to~%B(M4lKBsyW)*?&~fE-~`%5YEN!Y zSD%<4icB*@p0ol~Jcd5`0>NufZ>O`(((4c3|$tP#h~ zsl6&&geJL%f1me|_G})GWu8qEM0{=cEgWl=+tBYito{X(30lAMhg|C4d_jMc-&QL& zq4Mm{$Z&u8Hvi-G^`|4j?|`;+hLhgr#Mi!gAy0^c`CxE}EiQm*-M_57JvB!x4Sk<^ zC+%62MdN*G?L%%+ptwF6Ln!#6p=vo3px27FBANj4KBLv|%AakAs7|Cy4+E8E3XkRye0iWD|^S?@al+&Bba#pgP&y;={t!QzHVQ z4OhA}ymawGN)u#?fndm1`W=Z5ZfTkS1!^T3e`y`(&{wtUYVLBEycOYfmfcfJji1R+ zgl@#=AM^J9&Z`{bX>0qI+fZX>(sBA+HqeUZ%H10@f^RcGG>Q745ac2ft-TeL=5`2F)TrKJnzEx~Ow@G{?>)Bh=I^}uKb?Nk|0P*Hn^*%hoC$BZ zZl98BQ|2Tehpnef+9olmWEPYGb#KQO2p0xXrWl_IiR+SA z``C)X%kI7|HGb)Se)|W@iyrar=3E59mWnsk8eagnvY2Bl+pnP}QgzUFHRwQX+k|A3 zPWtzaUJH#8N%FTu&#(DwiHdd}Rac7qO^5KOo%?UyJXnP|g`=5w#GaZr1>mvmGdB@T zClyi>7F2sU=}9>K*sM8wui`LHb5Bi(_%sGp<+yo+Y8K@x4~|37xoUS6Se23uee^uh zMv3+}IG1aLSVC~I5};5{TrlAo@g{yzhpV&&$~o!mRu)-X=CD)h>Wwz%KHGq=KcRbp zc`Qtw>F@JOf9t#b!T;e);~`aQEE~k zgv&s+r_B^$l10uTiTE*7Ij<9diDlU{FtWfL0Iz;NN(V24ADsn``h5=K$F>xe$peAE z8)^`_6ImW<`wJ9t2WSnkc8~(_u#}+00*$=uhu0^se?hc}z|pgN!9Zo{MH&Pc0u6jb z)@IfObp8$0)%{2vsQxq?fT^O${6Xp+!ktYBt{YNaL*d;4O8mk=L6TK_WDOw_EaC?y zJ_1i!QIUZN+-m@Lh|b3NK;NjpQKcixL}H77fwUV?MBU%t!f0Ow(#_k=^ZcJ*;-7Eg zpXT&`tcfVw-^r$ntIk_BZIU;}(=s7~%RPcG+LZ3|SwZ8-*;JQPH$z_m%P>g02{{on zH`^=`8WI>H?|y+edubp_&BV{$EzxlMM&43veBZM$sZldF=?M5aTF`jErS+Ou)>wgo=U(Iga)o=VazCJJ?>~py}F2Ot};4 zC>d4hOb3KI;)yIL>iKcW6SX5D>I&-;X>o13R#89F?@I4XWZ4$oi$4PjUCwn*h1cZ* zGID*iwK;|jKW#YxnNGgUBBMkQy-I$3eERQAKSKKO*#ugmDqhi?R~dI>eD-vMC*!-h z@rEn#fp;jBIKw^jr9_LzByuUro{m>HKjm1R~`XgcCIwFwb>wm5#|5o z1NcLcFXI{GAytYDfwA5xVr}Kw%|u1stT(R^zyR(49!Gl`>)#6J#Z_k<)kIZcRdvdc z+dpTg1snAC{V#(1!!&&G9b^{fxnyKvCO!E|)i_)<;NvypGm6cAQaq4kAUtv zmxPVlKo4rd5$<5Lr8GE7{*2-r-E+yn#3{qKGaCozdHvq8lqPj>o@C=(=J3?y9meq! zG>H(B1{e#WBR-xb#%s1Owg%P56yI`qvdwo56ys-b!}-HO*Giar!GF(#`Ge7k{pUwf zi%fElMn3zrFLz>OXTmgSagJCBU}?^jVkr$e_(+CEiZ#wLY7{Zgjpw0!`&NFfsBh`^ zHWy#G-P=cJnw3mxh4et?ICh}Y5f7_c#aaaBg|Om*i2?A_0fIl|^+>_y7FC)ho)O6k zm-1|{g=tN^syo(be5{8$UBeZ*IZv_z7#=0!ZgQayrzMixSFgUm6EC*Hcww*`f1|@Q zwoQXJx`;_r5&YJkW&q)gnnJX|s$r38>Nv)E$B5iP06UKf-1ddB+gMIC(Oh)>Tq59m zugAXrid~I!mXszW56Vd9(%=b^$7QzQL)^6^d{lY(u>!UOga9T9A~yY9$2XAA-ajBb zx&Ud-3LqT_gN9&rLUhsYt?oK9ctsw|Lt^$shculnk$kskD`*Z@8 zPQxO1~G4E2gv&600QC! zP)UpX1^ST|-~yb)4Wa4+Ohq^brRjP>Srzz!0B z6KccD0O}(*5UnZU#;I%yQ1WYlBq;$<>SrmO9T6y88Sww5RDetQ1knW4?AH3#fTnTb zaeyNb_^qn?=S}|eN&eF&|G9Af!CTO&-#Z!5&lDh8^UjuUhwW|dkzi35-$M?zFk$Cs zsSHsIz9az~UZV6oXm)33_x|mk{g!?xz8eIvIv*jXT-skiTeqhyt$Ae+u;k0x;=+(= zi`Sc*DC45XG#_vWWM&rvBF4nIHmv}$ z`B`A%);eMW1txQTBzko`8+7uMo)SwS>wh`Z@pEaA?)ra{R`3tV?^*t!H-w}ia3(P3 zbr@Gj;GE&4$vi5;wc$FRnRLUt%cnSU*w`F-q_GvvBQ5v7_C~Y<|Es$+Kt}w`ifn+} zdN+p+COjSwQKX9q37(^9ZLLrzQ|!@+@BF&NR$q&YvkCDW7r)_|2J#3JikZ)hLf}jx zZ0iuNOEn|RH_9dE`v5l3lUHg#tgOC$rJq{11T-<`cg< zIiN_igbjf_Ny6Z?K5H42(8RY|3p)4!M*Iz=@DJ6){$N!0_ZQy&|NWri{I<`ApQf8~ zX0AJh3NQ&)wGMJ-M;PbuS3Y?2HKIwZ&g-O%Y%1umh46q9T?9WCu-yh_CByJ%5id0q z@T%3=ha0!qT-_3-1FQ4i3)yJ5Tk$hwY0~&ij{BPaiU< z=Od+&N9zVRYmSkcfLn~Tj?DE>blq3Qk9iZaDUoCdNz293z4eT6TWaJ}a%gsuuFJw1 z6^Z3!g=xq;4xfi>-sLwFECQCpK%hVqM3Yh@e~8YgT<&jioHJll8!c$LMthYT9$k=B z&F^;NiM3k%5w_u-0(5k#Ad>gh!Ab<, + #[clap(long, env)] + shard_directory: Option, + #[clap(default_value = "128", long, env)] + max_concurrent_requests: usize, + #[clap(default_value = "1000", long, env)] + max_input_length: usize, + #[clap(default_value = "32", long, env)] + max_batch_size: usize, + #[clap(default_value = "5", long, env)] + max_waiting_time: u64, + #[clap(default_value = "3000", long, short, env)] + port: u16, + #[clap(default_value = "/tmp/text-generation-server", long, env)] + shard_uds_path: String, + #[clap(default_value = "localhost", long, env)] + master_addr: String, + #[clap(default_value = "29500", long, env)] + master_port: usize, +} + +fn main() -> ExitCode { + tracing_subscriber::fmt::init(); + + // Pattern match configuration + let Args { + model_name, + num_shard, + shard_directory, + max_concurrent_requests, + max_input_length, + max_batch_size, + max_waiting_time, + port, + shard_uds_path, + master_addr, + master_port, + } = Args::parse(); + + // By default we only have one master shard + let num_shard = num_shard.unwrap_or(1); + + // Signal handler + let running = Arc::new(AtomicBool::new(true)); + let r = running.clone(); + ctrlc::set_handler(move || { + r.store(false, Ordering::SeqCst); + }) + .expect("Error setting Ctrl-C handler"); + + // Shared shutdown bool + let shutdown = Arc::new(Mutex::new(false)); + // Shared shutdown channel + // When shutting down, the main thread will wait for all senders to be dropped + let (shutdown_sender, shutdown_receiver) = mpsc::channel(); + + // Shared channel to track shard status + let (status_sender, status_receiver) = mpsc::channel(); + + // Start shard processes + for rank in 0..num_shard { + let model_name = model_name.clone(); + let uds_path = shard_uds_path.clone(); + let shard_directory = shard_directory.clone(); + let master_addr = master_addr.clone(); + let status_sender = status_sender.clone(); + let shutdown = shutdown.clone(); + let shutdown_sender = shutdown_sender.clone(); + thread::spawn(move || { + shard_manager( + model_name, + uds_path, + shard_directory, + rank, + num_shard, + master_addr, + master_port, + status_sender, + shutdown, + shutdown_sender, + ) + }); + } + drop(shutdown_sender); + + // Wait for shard to start + let mut shard_ready = 0; + while running.load(Ordering::SeqCst) { + match status_receiver.try_recv() { + Ok(ShardStatus::Ready) => { + shard_ready += 1; + if shard_ready == num_shard { + break; + } + } + Err(TryRecvError::Empty) => { + sleep(Duration::from_millis(100)); + } + Ok(ShardStatus::Failed((rank, err))) => { + tracing::error!("Shard {} failed to start:\n{}", rank, err); + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::FAILURE; + } + Err(TryRecvError::Disconnected) => { + tracing::error!("Shard status channel disconnected"); + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::FAILURE; + } + } + } + + // We might have received a termination signal + if !running.load(Ordering::SeqCst) { + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::SUCCESS; + } + + // All shard started + // Start webserver + tracing::info!("Starting Webserver"); + let mut webserver = match Popen::create( + &[ + "text-generation-router", + "--max-concurrent-requests", + &max_concurrent_requests.to_string(), + "--max-input-length", + &max_input_length.to_string(), + "--max-batch-size", + &max_batch_size.to_string(), + "--max-waiting-time", + &max_waiting_time.to_string(), + "--port", + &port.to_string(), + "--master-shard-uds-path", + &format!("{}-0", shard_uds_path), + "--tokenizer-name", + &model_name, + ], + PopenConfig { + stdout: Redirection::Pipe, + stderr: Redirection::Pipe, + // Needed for the shutdown procedure + setpgid: true, + ..Default::default() + }, + ) { + Ok(p) => p, + Err(err) => { + tracing::error!("Failed to start webserver: {}", err); + if let PopenError::IoError(err) = err { + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("text-generation-router not found in PATH"); + tracing::error!("Please install it with `make install-router`") + } + } + + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::FAILURE; + } + }; + + // Redirect STDOUT and STDERR to the console + let webserver_stdout = webserver.stdout.take().unwrap(); + let webserver_stderr = webserver.stderr.take().unwrap(); + + thread::spawn(move || { + let stdout = BufReader::new(webserver_stdout); + let stderr = BufReader::new(webserver_stderr); + for line in stdout.lines() { + println!("{}", line.unwrap()); + } + for line in stderr.lines() { + println!("{}", line.unwrap()); + } + }); + + // Default exit code + let mut exit_code = ExitCode::SUCCESS; + + while running.load(Ordering::SeqCst) { + if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { + tracing::error!("Shard {} failed:\n{}", rank, err); + exit_code = ExitCode::FAILURE; + break; + }; + + match webserver.poll() { + Some(_) => { + tracing::error!("Webserver Crashed"); + shutdown_shards(shutdown, &shutdown_receiver); + return ExitCode::FAILURE; + } + None => { + sleep(Duration::from_millis(100)); + } + }; + } + + // Graceful termination + webserver.terminate().unwrap(); + tracing::info!("Waiting for webserver to gracefully shutdown"); + webserver.wait_timeout(Duration::from_secs(90)).unwrap(); + tracing::info!("Webserver terminated"); + shutdown_shards(shutdown, &shutdown_receiver); + + exit_code +} + +#[derive(Debug)] +enum ShardStatus { + Ready, + Failed((usize, String)), +} + +#[allow(clippy::too_many_arguments)] +fn shard_manager( + model_name: String, + uds_path: String, + shard_directory: Option, + rank: usize, + world_size: usize, + master_addr: String, + master_port: usize, + status_sender: mpsc::Sender, + shutdown: Arc>, + _shutdown_sender: mpsc::Sender<()>, +) { + // Get UDS path + let uds_string = format!("{}-{}", uds_path, rank); + let uds = Path::new(&uds_string); + // Clean previous runs + fs::remove_file(uds).unwrap_or_default(); + + // Process args + let mut shard_argv = vec![ + "bloom-inference-server".to_string(), + "serve".to_string(), + model_name, + "--uds-path".to_string(), + uds_path, + ]; + + if world_size > 1 { + shard_argv.push("--sharded".to_string()); + } + + if let Some(shard_directory) = shard_directory { + shard_argv.push("--shard-directory".to_string()); + shard_argv.push(shard_directory); + } + + // Start process + tracing::info!("Starting shard {}", rank); + let mut p = match Popen::create( + &shard_argv, + PopenConfig { + stdout: Redirection::Pipe, + stderr: Redirection::Pipe, + // Needed for the shutdown procedure + setpgid: true, + // NCCL env vars + env: Some(vec![ + ("RANK".parse().unwrap(), rank.to_string().parse().unwrap()), + ( + "WORLD_SIZE".parse().unwrap(), + world_size.to_string().parse().unwrap(), + ), + ("MASTER_ADDR".parse().unwrap(), master_addr.parse().unwrap()), + ( + "MASTER_PORT".parse().unwrap(), + master_port.to_string().parse().unwrap(), + ), + ]), + ..Default::default() + }, + ) { + Ok(p) => p, + Err(err) => { + if let PopenError::IoError(ref err) = err { + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("bloom-inference-server not found in PATH"); + tracing::error!("Please install it with `make install-server`") + } + } + status_sender + .send(ShardStatus::Failed((rank, err.to_string()))) + .unwrap(); + return; + } + }; + + let mut ready = false; + let start_time = Instant::now(); + loop { + // Process exited + if p.poll().is_some() { + let mut err = String::new(); + p.stderr.take().unwrap().read_to_string(&mut err).unwrap(); + status_sender + .send(ShardStatus::Failed((rank, err))) + .unwrap(); + return; + } + + // We received a shutdown signal + if *shutdown.lock().unwrap() { + p.terminate().unwrap(); + let _ = p.wait_timeout(Duration::from_secs(90)); + tracing::info!("Shard {} terminated", rank); + return; + } + + // Shard is ready + if uds.exists() && !ready { + tracing::info!("Shard {} ready in {:?}", rank, start_time.elapsed()); + status_sender.send(ShardStatus::Ready).unwrap(); + ready = true; + } else if !ready { + tracing::info!("Waiting for shard {} to be ready...", rank); + } + sleep(Duration::from_secs(5)); + } +} + +fn shutdown_shards(shutdown: Arc>, shutdown_receiver: &mpsc::Receiver<()>) { + tracing::info!("Shutting down shards"); + // Update shutdown value to true + // This will be picked up by the shard manager + { + let mut shutdown = shutdown.lock().unwrap(); + *shutdown = true; + } + + // Wait for shards to shutdown + // This will block till all shutdown_sender are dropped + let _ = shutdown_receiver.recv(); +} diff --git a/proto/generate.proto b/proto/generate.proto index 8c5221b4..45afca82 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -11,10 +11,6 @@ service TextGenerationService { rpc Generate (GenerateRequest) returns (GenerateResponse); /// Generate tokens for a list of cached batches rpc GenerateWithCache (GenerateWithCacheRequest) returns (GenerateWithCacheResponse); - /// Generate tokens until the text of at least one request of the batch is generated - rpc GenerateUntilFinished (GenerateUntilFinishedRequest) returns (GenerateUntilFinishedResponse); - /// Generate tokens until the text of at least one request of the cached batches i finished - rpc GenerateUntilFinishedWithCache (GenerateUntilFinishedWithCacheRequest) returns (GenerateUntilFinishedWithCacheResponse); } /// Empty request @@ -92,27 +88,3 @@ message GenerateWithCacheResponse { /// Next batch (cached) optional Batch batch = 2; } - -message GenerateUntilFinishedRequest { - /// Batch - Batch batch = 1; -} - -message GenerateUntilFinishedResponse { - /// Finished requests - repeated GeneratedText generated_texts = 1; - /// Next batch (cached) - optional Batch batch = 2; -} - -message GenerateUntilFinishedWithCacheRequest { - /// Cached batches - repeated Batch batches = 1; -} - -message GenerateUntilFinishedWithCacheResponse { - /// Finished requests - repeated GeneratedText generated_texts = 1; - /// Next batch (cached) - optional Batch batch = 2; -} diff --git a/router/.gitignore b/router/.gitignore deleted file mode 100644 index ea8c4bf7..00000000 --- a/router/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/router/Cargo.toml b/router/Cargo.toml index 37f319e9..5820c138 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -22,16 +22,7 @@ serde = "1.0.145" serde_json = "1.0.85" thiserror = "1.0.37" tokenizers = "0.13.0" -tokio = { version = "1.21.1", features = ["rt-multi-thread", "parking_lot", "sync"] } +tokio = { version = "1.21.1", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } tracing = "0.1.36" tracing-subscriber = "0.3.15" -[workspace] -members = [ - "client", -] - -[profile.release] -debug = 1 -incremental = true -lto = "off" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 7760c8cb..633f82a9 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -5,8 +5,6 @@ edition = "2021" [dependencies] futures = "^0.3" -#grpc-error-details = { path = "../../grpc-error-details" } -#grpc-metadata = { path = "../../grpc-metadata" } prost = "^0.9" thiserror = "^1.0" tokio = { version = "^1.21", features = ["sync"] } diff --git a/router/client/src/client.rs b/router/client/src/client.rs index e7189b89..172d0bf7 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -1,10 +1,11 @@ +/// Single shard Client use crate::pb::generate::v1::text_generation_service_client::TextGenerationServiceClient; use crate::pb::generate::v1::*; use crate::Result; use tonic::transport::{Channel, Uri}; use tracing::*; -/// BLOOM Inference gRPC client +/// Text Generation Inference gRPC client #[derive(Clone)] pub struct Client { stub: TextGenerationServiceClient, @@ -34,6 +35,7 @@ impl Client { }) } + /// Returns a list of uris or unix sockets of all shards #[instrument(skip(self))] pub async fn service_discovery(&mut self) -> Result> { let request = tonic::Request::new(ServiceDiscoveryRequest {}); @@ -46,6 +48,7 @@ impl Client { .into_inner() .urls .into_iter() + // Remove unix socket prefix .map(|url| match url.strip_prefix("unix://") { None => url, Some(stripped_url) => stripped_url.to_string(), @@ -54,6 +57,7 @@ impl Client { Ok(urls) } + /// Clear the past generations cache #[instrument(skip(self))] pub async fn clear_cache(&mut self) -> Result<()> { let request = tonic::Request::new(ClearCacheRequest {}); @@ -64,6 +68,10 @@ impl Client { Ok(()) } + /// Generate one token for each request in the given batch + /// + /// Returns a list of generated texts of request that met their stopping criteria + /// and the next cached batch #[instrument(skip(self))] pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { let request = tonic::Request::new(GenerateRequest { batch: Some(batch) }); @@ -76,6 +84,10 @@ impl Client { Ok((response.generated_texts, response.batch)) } + /// Generate one token for each request in the given cached batch + /// + /// Returns a list of generated texts of request that met their stopping criteria + /// and the next cached batch #[instrument(skip(self))] pub async fn generate_with_cache( &mut self, @@ -90,34 +102,4 @@ impl Client { .into_inner(); Ok((response.generated_texts, response.batch)) } - - #[instrument(skip(self))] - pub async fn generate_until_finished( - &mut self, - batch: Batch, - ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(GenerateUntilFinishedRequest { batch: Some(batch) }); - let response = self - .stub - .generate_until_finished(request) - .instrument(info_span!("generate_until_finished")) - .await? - .into_inner(); - Ok((response.generated_texts, response.batch)) - } - - #[instrument(skip(self))] - pub async fn generate_until_finished_with_cache( - &mut self, - batches: Vec, - ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(GenerateUntilFinishedWithCacheRequest { batches }); - let response = self - .stub - .generate_until_finished_with_cache(request) - .instrument(info_span!("generate_until_finished_with_cache")) - .await? - .into_inner(); - Ok((response.generated_texts, response.batch)) - } } diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 48b2650d..0f1f96bc 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -1,6 +1,7 @@ -//! BLOOM Inference gRPC client library +//! Text Generation gRPC client library mod client; +#[allow(clippy::derive_partial_eq_without_eq)] mod pb; mod sharded_client; @@ -8,7 +9,7 @@ pub use client::Client; pub use pb::generate::v1::{Batch, GeneratedText, LogitsWarperParameters, Request}; pub use sharded_client::ShardedClient; use thiserror::Error; -pub use tonic::transport; +use tonic::transport; use tonic::Status; #[derive(Error, Debug, Clone)] @@ -21,7 +22,7 @@ pub enum ClientError { impl From for ClientError { fn from(err: Status) -> Self { - Self::Generation(err.to_string()) + Self::Generation(err.message().to_string()) } } diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 7134551e..916e72b4 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,9 +1,11 @@ +/// Multi shard Client use crate::Result; use crate::{Batch, Client, GeneratedText}; use futures::future::join_all; use tokio::sync::{broadcast, mpsc}; use tonic::transport::Uri; +/// List of all available commands that can be sent through the command channel #[derive(Clone, Debug)] enum Command { Generate( @@ -14,36 +16,32 @@ enum Command { Vec, mpsc::Sender, Option)>>, ), - GenerateUntilFinished( - Batch, - mpsc::Sender, Option)>>, - ), - GenerateUntilFinishedWithCache( - Vec, - mpsc::Sender, Option)>>, - ), ClearCache(mpsc::Sender>), } +/// Tokio task that handles the communication with a single shard +/// +/// We subscribe on a broadcast channel to receive commands that will be sent by +/// the ShardedClient. +/// +/// Each command is fan out to all shards. +/// +/// The result of the command is sent back to the ShardedClient through a mpsc channel (multi +/// producer = the shards, single consumer = the ShardedClient). async fn client_task(mut client: Client, mut request_subscriber: broadcast::Receiver) { while let Ok(message) = request_subscriber.recv().await { match message { Command::Generate(batch, response_tx) => { let result = client.generate(batch).await; + // We can unwrap_or(()) here because the only error that can happen is if the + // receiver is dropped, which means that the ShardedClient already received a + // response from another shard response_tx.try_send(result).unwrap_or(()); } Command::GenerateWithCache(batches, response_tx) => { let result = client.generate_with_cache(batches).await; response_tx.try_send(result).unwrap_or(()); } - Command::GenerateUntilFinished(batch, response_tx) => { - let result = client.generate_until_finished(batch).await; - response_tx.try_send(result).unwrap_or(()); - } - Command::GenerateUntilFinishedWithCache(batches, response_tx) => { - let result = client.generate_until_finished_with_cache(batches).await; - response_tx.try_send(result).unwrap_or(()); - } Command::ClearCache(response_tx) => { let result = client.clear_cache().await; response_tx.try_send(result).unwrap_or(()); @@ -52,30 +50,42 @@ async fn client_task(mut client: Client, mut request_subscriber: broadcast::Rece } } +/// Text Generation Inference gRPC multi client pub struct ShardedClient { + _clients: Vec, request_tx: broadcast::Sender, } impl ShardedClient { - fn new(mut clients: Vec) -> Self { + fn new(clients: Vec) -> Self { + // The broadcast channel to communicate with the shards + // We use a capacity of one as the shards are not asynchronous and can only process one + // command at a time let (request_tx, _) = broadcast::channel(1); - for client in clients.drain(..) { + // Spawn client tasks + for client in clients.iter() { let request_subscriber = request_tx.subscribe(); - tokio::spawn(client_task(client, request_subscriber)); + tokio::spawn(client_task(client.clone(), request_subscriber)); } - Self { request_tx } + Self { + _clients: clients, + request_tx, + } } + /// Create a new ShardedClient from a master client. The master client will communicate with + /// the other shards and returns all uris/unix sockets with the `service_discovery` gRPC method. async fn from_master_client(mut master_client: Client) -> Result { + // Get all uris/unix sockets from the master client let uris = master_client.service_discovery().await.unwrap(); - let futures = uris.into_iter().map(|path| Client::connect_uds(path)); + let futures = uris.into_iter().map(Client::connect_uds); let clients: Result> = join_all(futures).await.into_iter().collect(); Ok(Self::new(clients?)) } - /// Returns a client connected to the given url + /// Returns a client connected to the given uri pub async fn connect(uri: Uri) -> Result { let master_client = Client::connect(uri).await?; Self::from_master_client(master_client).await @@ -87,51 +97,43 @@ impl ShardedClient { Self::from_master_client(master_client).await } + /// Generate one token for each request in the given batch + /// + /// Returns a list of generated texts of request that met their stopping criteria + /// and the next cached batch pub async fn generate(&self, batch: Batch) -> Result<(Vec, Option)> { + // Create a channel to receive the response from the shards + // We will only ever receive one message on this channel let (response_tx, mut response_rx) = mpsc::channel(1); self.request_tx .send(Command::Generate(batch, response_tx)) .unwrap(); + // As soon as we receive one response, we can return as all shards will return the same response_rx.recv().await.unwrap() } + /// Generate one token for each request in the given cached batch + /// + /// Returns a list of generated texts of request that met their stopping criteria + /// and the next cached batch pub async fn generate_with_cache( &self, batches: Vec, ) -> Result<(Vec, Option)> { + // Create a channel to receive the response from the shards + // We will only ever receive one message on this channel let (response_tx, mut response_rx) = mpsc::channel(1); self.request_tx .send(Command::GenerateWithCache(batches, response_tx)) .unwrap(); + // As soon as we receive one response, we can return as all shards will return the same response_rx.recv().await.unwrap() } - pub async fn generate_until_finished( - &self, - batch: Batch, - ) -> Result<(Vec, Option)> { - let (response_tx, mut response_rx) = mpsc::channel(1); - self.request_tx - .send(Command::GenerateUntilFinished(batch, response_tx)) - .unwrap(); - response_rx.recv().await.unwrap() - } - - pub async fn generate_until_finished_with_cache( - &self, - batches: Vec, - ) -> Result<(Vec, Option)> { - let (response_tx, mut response_rx) = mpsc::channel(1); - self.request_tx - .send(Command::GenerateUntilFinishedWithCache( - batches, - response_tx, - )) - .unwrap(); - response_rx.recv().await.unwrap() - } - + /// Clear the past generations cache pub async fn clear_cache(&self) -> Result<()> { + // Create a channel to receive the response from the shards + // We will only ever receive one message on this channel let (response_tx, mut response_rx) = mpsc::channel(1); self.request_tx .send(Command::ClearCache(response_tx)) diff --git a/router/src/batcher.rs b/router/src/batcher.rs index ebd81730..4523dbff 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -1,129 +1,158 @@ -use crate::server::GenerateRequest; +/// Batching and inference logic +use crate::GenerateRequest; use crate::{Db, Entry}; use axum::http::StatusCode; use bloom_inference_client::{Batch, ClientError, GeneratedText, ShardedClient}; use std::future::Future; use std::sync::Arc; +use std::time::Duration; use thiserror::Error; use tokio::sync::{oneshot, Notify}; +use tokio::time::Instant; +use tracing::instrument; -const MAX_LENGTH: usize = 128; - -#[derive(Debug, Error)] -pub enum InferError { - #[error("Request failed during generation: {0}")] - GenerationError(String), - #[error("Model is overloaded")] - Overloaded, -} - -impl From for (StatusCode, String) { - fn from(err: InferError) -> Self { - match err { - InferError::GenerationError(_) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), - InferError::Overloaded => (StatusCode::TOO_MANY_REQUESTS, err.to_string()), - } - } -} - +/// Batcher #[derive(Clone)] pub struct Batcher { + /// Request database db: Db, + /// Shared state shared: Arc, } +/// Batcher shared state struct Shared { + /// Batching background Tokio task notifier batching_task: Notify, } impl Batcher { - pub(crate) fn new(client: ShardedClient, max_batch_size: usize) -> Self { + pub(crate) fn new( + client: ShardedClient, + max_batch_size: usize, + max_waiting_time: Duration, + ) -> Self { + // Batcher shared state let db = Db::new(); let shared = Arc::new(Shared { batching_task: Notify::new(), }); - tokio::spawn(batching_task(max_batch_size, client, db.clone(), shared.clone())); + // Spawn batching background task that contains all the inference logic + tokio::spawn(batching_task( + max_batch_size, + max_waiting_time, + client, + db.clone(), + shared.clone(), + )); Self { db, shared } } + /// Add a new request to the database and return a future that will generate the text pub(crate) async fn infer( &self, input_length: usize, request: GenerateRequest, ) -> Result { - if self.db.len() > MAX_LENGTH { - return Err(InferError::Overloaded); - } - let (request_tx, request_rx) = oneshot::channel(); + // One shot channel to communicate with the background batching task + let (response_tx, response_rx) = oneshot::channel(); + + // Try to append the request to the database self.db.append(Entry { request, - response_tx: request_tx, + response_tx, input_length, + time: Instant::now(), }); + + // Notify the background task that we have a new entry in the database that needs + // to be batched self.shared.batching_task.notify_waiters(); - match request_rx.await.unwrap() { + + // Await on the response from the background task + // We can safely unwrap as the background task will never drop the sender + match response_rx.await.unwrap() { Ok(output) => Ok(output), Err(err) => Err(InferError::GenerationError(err.to_string())), } } } -async fn batching_task(max_batch_size: usize, - client: ShardedClient, - db: Db, - shared: Arc) { +/// Batching logic +/// Will be launched in a background Tokio task +/// +/// Batches requests and sends them to the inference server +#[instrument(skip(client, db, shared))] +async fn batching_task( + max_batch_size: usize, + max_waiting_time: Duration, + client: ShardedClient, + db: Db, + shared: Arc, +) { + // Minimum batch size after which we try to add more requests let limit_min_batch_size = (max_batch_size / 2) as u32; + // Infinite loop loop { + // Wait for a notification from the Batcher struct shared.batching_task.notified().await; - if let Some(batch) = db.next_batch(max_batch_size) { - let request_ids = batch.requests.iter().map(|req| req.id).collect(); - let mut cached_batch = match batch.size { - size if size > limit_min_batch_size => { - wrap_future(client.generate_until_finished(batch), request_ids, &db).await - } - _ => wrap_future(client.generate(batch), request_ids, &db).await, - }; + // Get the next batch from the DB + // This batch might be smaller than the maximum batch size if there are not enough requests + // waiting in the DB + if let Some((request_ids, batch)) = db.next_batch(None, max_batch_size, None) { + let mut cached_batch = wrap_future(client.generate(batch), request_ids, &db).await; + // We loop until we do not receive any cached batch from the inference server (== until + // all requests have met their stopping criteria) while let Some(batch) = cached_batch { - let mut current_batch_size = batch.size; + // Get current batch info + let batch_size = batch.size; let mut request_ids: Vec = batch.requests.iter().map(|req| req.id).collect(); let mut batches = vec![batch]; - if current_batch_size <= limit_min_batch_size { - if let Some(new_batch) = db.next_batch_minimum_size(limit_min_batch_size as usize, max_batch_size) { - let new_batch_request_ids = - new_batch.requests.iter().map(|req| req.id).collect(); + // If the current batch is too small, we try to add more requests to it + if batch_size <= limit_min_batch_size { + // Get the next batch from the DB that meet our minimum size criteria + if let Some((new_request_ids, new_batch)) = + db.next_batch(Some(limit_min_batch_size as usize), max_batch_size, None) + { + // Generate one token for this new batch to have the attention past in cache let new_cached_batch = - wrap_future(client.generate(new_batch), new_batch_request_ids, &db) - .await; + wrap_future(client.generate(new_batch), new_request_ids, &db).await; + // Extend current batch with the new batch + if let Some(new_cached_batch) = new_cached_batch { + request_ids.extend(new_cached_batch.requests.iter().map(|req| req.id)); + batches.push(new_cached_batch); + } + } + // If we don't have enough requests to meet the minimum size criteria, we + // try to get the next batch from the DB that have been waiting over + // the max_waiting_time + else if let Some((new_request_ids, new_batch)) = + db.next_batch(None, max_batch_size, Some(max_waiting_time)) + { + let new_cached_batch = + wrap_future(client.generate(new_batch), new_request_ids, &db).await; + // Extend current batch with the new batch if let Some(new_cached_batch) = new_cached_batch { - current_batch_size += new_cached_batch.size; request_ids.extend(new_cached_batch.requests.iter().map(|req| req.id)); batches.push(new_cached_batch); } } } - cached_batch = match current_batch_size { - size if size > limit_min_batch_size => { - wrap_future( - client.generate_until_finished_with_cache(batches), - request_ids, - &db, - ) - .await - } - _ => wrap_future(client.generate_with_cache(batches), request_ids, &db).await, - }; + cached_batch = + wrap_future(client.generate_with_cache(batches), request_ids, &db).await; } } } } +/// Wrap a future inside a match statement to handle errors and send the response to the Batcher async fn wrap_future( future: impl Future, Option), ClientError>>, request_ids: Vec, @@ -134,6 +163,7 @@ async fn wrap_future( send_generated(generated_texts, db); next_batch } + // If we have an error, we discard the whole batch Err(err) => { send_error(err, request_ids, db); None @@ -141,16 +171,20 @@ async fn wrap_future( } } +/// Send errors to the Batcher for all `request_ids` fn send_error(error: ClientError, request_ids: Vec, db: &Db) { request_ids.into_iter().for_each(|id| { + // We can `expect` here as the request id should always be in the DB let entry = db.remove(&id).expect("ID not found in db. This is a bug."); // unwrap_or is valid here as we don't care if the receiver is gone. entry.response_tx.send(Err(error.clone())).unwrap_or(()); }); } +/// Send `generated_text` to the Batcher for all `finished` fn send_generated(finished: Vec, db: &Db) { finished.into_iter().for_each(|output| { + // We can `expect` here as the request id should always be in the DB let entry = db .remove(&output.request.unwrap().id) .expect("ID not found in db. This is a bug."); @@ -158,3 +192,18 @@ fn send_generated(finished: Vec, db: &Db) { entry.response_tx.send(Ok(output.output)).unwrap_or(()); }); } + +#[derive(Debug, Error)] +pub enum InferError { + #[error("Request failed during generation: {0}")] + GenerationError(String), +} + +/// Convert to Axum supported format +impl From for (StatusCode, String) { + fn from(err: InferError) -> Self { + match err { + InferError::GenerationError(_) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), + } + } +} diff --git a/router/src/db.rs b/router/src/db.rs index 03593fc0..9518fa1d 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,57 +1,107 @@ /// This code is massively inspired by Tokio mini-redis -use crate::server::{GenerateParameters, GenerateRequest}; +use crate::{GenerateParameters, GenerateRequest}; use bloom_inference_client::{Batch, ClientError, LogitsWarperParameters, Request}; -use parking_lot::RwLock; +use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; +use std::time::Duration; use tokio::sync::oneshot::Sender; +use tokio::time::Instant; +/// Database entry #[derive(Debug)] pub(crate) struct Entry { + /// Request pub request: GenerateRequest, + /// Response sender to communicate between the Batcher and the batching_task pub response_tx: Sender>, + /// Number of tokens in the input pub input_length: usize, + /// Instant when this entry was created + pub time: Instant, } -impl From for LogitsWarperParameters { - fn from(parameters: GenerateParameters) -> Self { - Self { - temperature: parameters.temperature, - top_k: parameters.top_k as u32, - top_p: parameters.top_p, - do_sample: parameters.do_sample, - } - } -} - +/// Request Database #[derive(Debug, Clone)] pub(crate) struct Db { pub shared: Arc, } +/// Shared state #[derive(Debug)] pub struct Shared { - state: RwLock, + state: Mutex, } +/// Database State #[derive(Debug)] struct State { + /// Database entries organized in a BTreeMap to be able to iterate over them in order entries: BTreeMap, - /// Identifier to use for the next expiration. Each expiration is associated - /// with a unique identifier. See above for why. + /// Id of the next entry next_id: u64, + /// Id of the next batch next_batch_id: u64, - /// Current batch id + /// Start ID of the next batch. Used to iterate inside the entries BTreeMap next_batch_start_id: u64, } +impl State { + /// Get the next requests + fn next_requests( + &self, + max_size: usize, + min_waiting_time: Option, + ) -> Option<(Vec, Vec)> { + // Iterates for max_size over the BTreemap starting from next_batch_start_id + let mut requests = Vec::new(); + let mut ids = Vec::new(); + + for (id, entry) in self + .entries + // Start from next_batch_start_id + .range(self.next_batch_start_id..) + // Take max_size + .take(max_size) + { + if let Some(min_waiting_time) = min_waiting_time { + // Only take entries that waited for at least min_waiting_time + if entry.time.elapsed() < min_waiting_time { + // Since entries are ordered, we already know that all following entries won't + // satisfy the condition + break; + } + } + + requests.push(Request { + id: *id, + inputs: entry.request.inputs.clone(), + input_length: entry.input_length as u32, + parameters: Some(LogitsWarperParameters::from( + entry.request.parameters.clone(), + )), + max_new_tokens: entry.request.parameters.max_new_tokens, + }); + + ids.push(*id); + } + + if requests.is_empty() { + None + } else { + Some((ids, requests)) + } + } +} + impl Db { pub(crate) fn new() -> Self { + // Shared state let shared = Arc::new(Shared { - state: RwLock::new(State { + state: Mutex::new(State { entries: BTreeMap::new(), next_id: 0, next_batch_id: 0, @@ -62,55 +112,46 @@ impl Db { Self { shared } } + /// Append an entry to the database pub(crate) fn append(&self, entry: Entry) { - let mut state = self.shared.state.write(); + // Acquire lock + let mut state = self.shared.state.lock(); + // Insert entry let id = state.next_id; state.next_id += 1; - state.entries.insert(id, entry); } + /// Remove an entry from the database if it exists pub(crate) fn remove(&self, id: &u64) -> Option { - let mut state = self.shared.state.write(); + let mut state = self.shared.state.lock(); state.entries.remove(id) } - pub(crate) fn len(&self) -> usize { - let state = self.shared.state.read(); - state.entries.len() - } - - fn next_requests(&self, max_size: usize) -> Option<(u64, Vec)> { - let state = self.shared.state.read(); - - let requests: Vec = state - .entries - .range(state.next_batch_start_id..) - .take(max_size) - .map(|(id, entry)| Request { - id: *id, - inputs: entry.request.inputs.clone(), - input_length: entry.input_length as u32, - parameters: Some(LogitsWarperParameters::from( - entry.request.parameters.clone(), - )), - max_new_tokens: entry.request.parameters.max_new_tokens, - }) - .collect(); - - if requests.is_empty() { - None - } else { - let last_id = requests.last().unwrap().id; - Some((last_id, requests)) - } - } + // Get the next batch + pub(crate) fn next_batch( + &self, + min_size: Option, + max_size: usize, + min_waiting_time: Option, + ) -> Option<(Vec, Batch)> { + // Acquire lock + let mut state = self.shared.state.lock(); + + // Get requests from the database + if let Some((ids, requests)) = state.next_requests(max_size, min_waiting_time) { + if let Some(min_size) = min_size { + // If min_size is set, only return a batch if there are enough requests + if requests.len() < min_size { + return None; + } + } - pub(crate) fn next_batch(&self, max_size: usize) -> Option { - if let Some((last_id, requests)) = self.next_requests(max_size) { - let mut state = self.shared.state.write(); + // Batch size let size = requests.len(); + // Longest input length for all requests in batch size + // Used for padding inside the inference server let max_sequence_length = requests.iter().map(|r| r.input_length).max().unwrap(); let batch = Batch { id: state.next_batch_id, @@ -118,34 +159,24 @@ impl Db { size: size as u32, max_sequence_length, }; - state.next_batch_start_id = last_id + 1; + // Update next_batch_start_id to the last id in the batch + 1 + state.next_batch_start_id = ids.last().unwrap() + 1; + // Increment batch id state.next_batch_id += 1; - return Some(batch); + + return Some((ids, batch)); } None } +} - pub(crate) fn next_batch_minimum_size( - &self, - min_size: usize, - max_size: usize, - ) -> Option { - if let Some((last_id, requests)) = self.next_requests(max_size) { - if requests.len() >= min_size { - let mut state = self.shared.state.write(); - let size = requests.len(); - let max_sequence_length = requests.iter().map(|r| r.input_length).max().unwrap(); - let batch = Batch { - id: state.next_batch_id, - requests, - size: size as u32, - max_sequence_length, - }; - state.next_batch_start_id = last_id + 1; - state.next_batch_id += 1; - return Some(batch); - } +impl From for LogitsWarperParameters { + fn from(parameters: GenerateParameters) -> Self { + Self { + temperature: parameters.temperature, + top_k: parameters.top_k as u32, + top_p: parameters.top_p, + do_sample: parameters.do_sample, } - None } } diff --git a/router/src/lib.rs b/router/src/lib.rs index 14dc5724..02b912a3 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,8 +1,68 @@ +/// Text Generation Inference Webserver mod batcher; mod db; -mod validation; pub mod server; +mod validation; -use db::{Db, Entry}; use batcher::Batcher; +use db::{Db, Entry}; +use serde::{Deserialize, Serialize}; use validation::Validation; + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct GenerateParameters { + #[serde(default = "default_temperature")] + pub temperature: f32, + #[serde(default = "default_top_k")] + pub top_k: i32, + #[serde(default = "default_top_p")] + pub top_p: f32, + #[serde(default = "default_do_sample")] + pub do_sample: bool, + #[serde(default = "default_max_new_tokens")] + pub max_new_tokens: u32, +} + +fn default_temperature() -> f32 { + 1.0 +} + +fn default_top_k() -> i32 { + 0 +} + +fn default_top_p() -> f32 { + 1.0 +} + +fn default_do_sample() -> bool { + false +} + +fn default_max_new_tokens() -> u32 { + 20 +} + +fn default_parameters() -> GenerateParameters { + GenerateParameters { + temperature: default_temperature(), + top_k: default_top_k(), + top_p: default_top_p(), + do_sample: default_do_sample(), + max_new_tokens: default_max_new_tokens(), + } +} + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct GenerateRequest { + pub inputs: String, + #[serde(default = "default_parameters")] + pub parameters: GenerateParameters, +} + +#[derive(Serialize)] +pub(crate) struct GeneratedText { + pub generated_text: String, +} + +pub(crate) type GenerateResponse = Vec; diff --git a/router/src/main.rs b/router/src/main.rs index 89cd4731..49051b37 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,37 +1,61 @@ +/// Text Generation Inference webserver entrypoint use bloom_inference_client::ShardedClient; +use clap::Parser; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::time::Duration; use text_generation_router::server; use tokenizers::Tokenizer; -use clap::Parser; /// App Configuration #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] struct Args { - #[clap(default_value = "32", long, short, env)] + #[clap(default_value = "128", long, env)] + max_concurrent_requests: usize, + #[clap(default_value = "1000", long, env)] + max_input_length: usize, + #[clap(default_value = "32", long, env)] max_batch_size: usize, + #[clap(default_value = "5", long, env)] + max_waiting_time: u64, #[clap(default_value = "3000", long, short, env)] port: u16, #[clap(default_value = "/tmp/bloom-inference-0", long, env)] - shard_uds_path: String, + master_shard_uds_path: String, #[clap(default_value = "bigscience/bloom", long, env)] tokenizer_name: String, + #[clap(default_value = "2", long, env)] + validation_workers: usize, } fn main() -> Result<(), std::io::Error> { // Get args let args = Args::parse(); -// Pattern match configuration + // Pattern match configuration let Args { + max_concurrent_requests, + max_input_length, max_batch_size, + max_waiting_time, port, - shard_uds_path, + master_shard_uds_path, tokenizer_name, + validation_workers, } = args; + if validation_workers == 1 { + panic!("validation_workers must be > 0"); + } + + let max_waiting_time = Duration::from_secs(max_waiting_time); + // Download and instantiate tokenizer + // This will only be used to validate payloads + // + // We need to download it outside of the Tokio runtime let tokenizer = Tokenizer::from_pretrained(tokenizer_name, None).unwrap(); + // Launch Tokio runtime tokio::runtime::Builder::new_multi_thread() .enable_all() .build() @@ -39,18 +63,32 @@ fn main() -> Result<(), std::io::Error> { .block_on(async { tracing_subscriber::fmt::init(); - let sharded_client = ShardedClient::connect_uds(shard_uds_path) + // Instantiate sharded client from the master unix socket + let sharded_client = ShardedClient::connect_uds(master_shard_uds_path) .await .expect("Could not connect to server"); + // Clear the cache; useful if the webserver rebooted sharded_client .clear_cache() .await .expect("Unable to clear cache"); tracing::info!("Connected"); + // Binds on localhost let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port); - server::run(max_batch_size, sharded_client, tokenizer, addr).await; + // Run server + server::run( + max_concurrent_requests, + max_input_length, + max_batch_size, + max_waiting_time, + sharded_client, + tokenizer, + validation_workers, + addr, + ) + .await; Ok(()) }) } diff --git a/router/src/server.rs b/router/src/server.rs index 0fdfd58b..42258313 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,68 +1,44 @@ -use crate::{Batcher, Validation}; +use crate::{ + Batcher, GenerateParameters, GenerateRequest, GenerateResponse, GeneratedText, Validation, +}; use axum::extract::Extension; use axum::http::StatusCode; use axum::routing::{get, post}; use axum::{Json, Router}; use bloom_inference_client::ShardedClient; -use serde::Deserialize; use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; use tokenizers::Tokenizer; +use tokio::signal; +use tokio::sync::Semaphore; use tokio::time::Instant; use tracing::instrument; -#[derive(Clone, Debug, Deserialize)] -pub(crate) struct GenerateParameters { - #[serde(default = "default_temperature")] - pub temperature: f32, - #[serde(default = "default_top_k")] - pub top_k: i32, - #[serde(default = "default_top_p")] - pub top_p: f32, - #[serde(default = "default_do_sample")] - pub do_sample: bool, - #[serde(default = "default_max_new_tokens")] - pub max_new_tokens: u32, -} - -fn default_temperature() -> f32 { - 1.0 -} - -fn default_top_k() -> i32 { - 0 -} - -fn default_top_p() -> f32 { - 1.0 -} - -fn default_do_sample() -> bool { - false -} - -fn default_max_new_tokens() -> u32 { - 20 -} - -fn default_parameters() -> GenerateParameters { - GenerateParameters { - temperature: default_temperature(), - top_k: default_top_k(), - top_p: default_top_p(), - do_sample: default_do_sample(), - max_new_tokens: default_max_new_tokens(), - } -} - -#[derive(Clone, Debug, Deserialize)] -pub(crate) struct GenerateRequest { - pub inputs: String, - #[serde(default = "default_parameters")] - pub parameters: GenerateParameters, +// Server shared state +#[derive(Clone)] +struct ServerState { + validation: Validation, + batcher: Batcher, + limit_concurrent_requests: Arc, } +/// Health check method #[instrument(skip(state), fields(time, time_per_token))] -async fn liveness(state: Extension) -> Result<(), (StatusCode, String)> { +async fn health(state: Extension) -> Result<(), (StatusCode, String)> { + // TODO: while this is the best health check we can do, it is a bit on the heavy side and might + // be a bit too slow for a health check. + // What we should do instead if check if the gRPC channels are still healthy. + + // Limit concurrent requests by acquiring a permit from the semaphore + let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { + ( + StatusCode::TOO_MANY_REQUESTS, + "Model is overloaded".to_string(), + ) + })?; + + // Send a small inference request state .batcher .infer( @@ -82,23 +58,35 @@ async fn liveness(state: Extension) -> Result<(), (StatusCode, Stri Ok(()) } +/// Generate method #[instrument(skip(state), fields(time, time_per_token))] async fn generate( state: Extension, req: Json, -) -> Result, (StatusCode, String)> { +) -> Result, (StatusCode, String)> { let start = Instant::now(); + // Limit concurrent requests by acquiring a permit from the semaphore + let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { + ( + StatusCode::TOO_MANY_REQUESTS, + "Model is overloaded".to_string(), + ) + })?; + // Validate request let (input_length, validated_request) = state .validation + // FIXME: can't we get rid of the cloning here?? .validate(GenerateRequest { inputs: req.inputs.clone(), parameters: req.parameters.clone(), }) .await?; + // Inference let generated_text = state.batcher.infer(input_length, validated_request).await?; + // Tracing metadata tracing::Span::current().record("time", format!("{:?}", start.elapsed())); tracing::Span::current().record( "time_per_token", @@ -106,31 +94,71 @@ async fn generate( ); tracing::info!("response: {}", generated_text); - Ok(Json(serde_json::json!({ - "generated_text": generated_text, - }))) + // Send response + let response = vec![GeneratedText { generated_text }]; + Ok(Json(response)) } -#[derive(Clone)] -struct ServerState { - validation: Validation, - batcher: Batcher, -} - -pub async fn run(max_batch_size: usize, client: ShardedClient, tokenizer: Tokenizer, addr: SocketAddr) { - let batcher = Batcher::new(client, max_batch_size); - let validation = Validation::new(tokenizer); - - let shared_state = ServerState { validation, batcher }; - +/// Serving method +#[allow(clippy::too_many_arguments)] +pub async fn run( + max_concurrent_requests: usize, + max_input_length: usize, + max_batch_size: usize, + max_waiting_time: Duration, + client: ShardedClient, + tokenizer: Tokenizer, + validation_workers: usize, + addr: SocketAddr, +) { + // Create state + let batcher = Batcher::new(client, max_batch_size, max_waiting_time); + let validation = Validation::new(validation_workers, tokenizer, max_input_length); + let shared_state = ServerState { + validation, + batcher, + limit_concurrent_requests: Arc::new(Semaphore::new(max_concurrent_requests)), + }; + + // Create router let app = Router::new() .route("/generate", post(generate)) .layer(Extension(shared_state.clone())) - .route("/health", get(liveness)) + .route("/health", get(health)) .layer(Extension(shared_state.clone())); + // Run server axum::Server::bind(&addr) .serve(app.into_make_service()) + // Wait until all requests are finished to shut down + .with_graceful_shutdown(shutdown_signal()) .await .unwrap(); } + +/// Shutdown signal handler +async fn shutdown_signal() { + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + } + + tracing::info!("signal received, starting graceful shutdown"); +} diff --git a/router/src/validation.rs b/router/src/validation.rs index 45b108fd..49a46b62 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,62 +1,105 @@ -use crate::server::GenerateRequest; +/// Payload validation logic +use crate::GenerateRequest; use axum::http::StatusCode; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; +use tokenizers::{ + DecoderWrapper, ModelWrapper, NormalizerWrapper, PostProcessorWrapper, PreTokenizerWrapper, + TokenizerImpl, +}; use tokio::sync::{mpsc, oneshot}; -#[derive(Error, Debug)] -pub enum ValidationError { - #[error("Temperature must be strictly positive")] - Temperature, - #[error("Top p must be <= 0.0 or > 1.0")] - TopP, - #[error("Top k must be strictly positive")] - TopK, - #[error("Max New Tokens must be < 512")] - MaxNewTokens, - #[error("Inputs must have less than 1000 tokens. Given: {0}")] - InputLength(usize), -} - -impl From for (StatusCode, String) { - fn from(err: ValidationError) -> Self { - (StatusCode::BAD_REQUEST, err.to_string()) - } -} - -type ValidationRequest = ( - GenerateRequest, - oneshot::Sender>, -); - +/// Validation #[derive(Debug, Clone)] pub struct Validation { + /// Channel to communicate with the background validation task sender: mpsc::Sender, } impl Validation { - pub(crate) fn new(tokenizer: Tokenizer) -> Self { + pub(crate) fn new(workers: usize, tokenizer: Tokenizer, max_input_length: usize) -> Self { + // Crate channel let (validation_sender, validation_receiver) = mpsc::channel(128); - tokio::spawn(validation_task(tokenizer, validation_receiver)); + // Launch background validation task + tokio::spawn(validation_task( + workers, + tokenizer, + max_input_length, + validation_receiver, + )); Self { sender: validation_sender, } } + /// Validate a payload and get the number of tokens in the input pub(crate) async fn validate( &self, request: GenerateRequest, ) -> Result<(usize, GenerateRequest), ValidationError> { + // Create response channel let (sender, receiver) = oneshot::channel(); + // Send request to the background validation task + // Unwrap is safe here self.sender.send((request, sender)).await.unwrap(); + // Await on response channel + // Unwrap is safe here receiver.await.unwrap() } } -async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver) { - while let Some((request, response_tx)) = receiver.recv().await { +/// Validation task +/// Load balance the validation requests between multiple validation workers +async fn validation_task( + workers: usize, + tokenizer: Tokenizer, + max_input_length: usize, + mut receiver: mpsc::Receiver, +) { + let mut workers_senders = Vec::with_capacity(workers); + + // Create workers + for _ in 0..workers { + let tokenizer_clone = tokenizer.clone(); + // Create channel to communicate with worker + let (worker_sender, worker_receiver) = mpsc::channel(workers); + workers_senders.push(worker_sender); + + // Spawn worker + tokio::task::spawn_blocking(move || { + validation_worker(tokenizer_clone, max_input_length, worker_receiver) + }); + } + + loop { + // Load balance requests between workers + for sender in workers_senders.iter() { + if let Some(validation_request) = receiver.recv().await { + sender.send(validation_request).await.unwrap(); + } else { + return; + } + } + } +} + +/// Check the parameters inside the payload and get the number of tokens inside the input using +/// the tokenizer +fn validation_worker( + tokenizer: TokenizerImpl< + ModelWrapper, + NormalizerWrapper, + PreTokenizerWrapper, + PostProcessorWrapper, + DecoderWrapper, + >, + max_input_length: usize, + mut receiver: mpsc::Receiver, +) { + // Loop over requests + while let Some((request, response_tx)) = receiver.blocking_recv() { if request.parameters.temperature < 0.0 { response_tx .send(Err(ValidationError::Temperature)) @@ -78,10 +121,11 @@ async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver 1000 { + if input_length > max_input_length { response_tx .send(Err(ValidationError::InputLength(input_length))) .unwrap_or(()); @@ -91,3 +135,28 @@ async fn validation_task(tokenizer: Tokenizer, mut receiver: mpsc::Receiver>, +); + +#[derive(Error, Debug)] +pub enum ValidationError { + #[error("Temperature must be strictly positive")] + Temperature, + #[error("Top p must be <= 0.0 or > 1.0")] + TopP, + #[error("Top k must be strictly positive")] + TopK, + #[error("Max New Tokens must be < 512")] + MaxNewTokens, + #[error("Inputs must have less than 1000 tokens. Given: {0}")] + InputLength(usize), +} + +impl From for (StatusCode, String) { + fn from(err: ValidationError) -> Self { + (StatusCode::BAD_REQUEST, err.to_string()) + } +} diff --git a/run.sh b/run.sh deleted file mode 100644 index 30303501..00000000 --- a/run.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -server_cmd="bloom-inference-server launcher $MODEL_NAME --num-gpus $NUM_GPUS --shard-directory $MODEL_BASE_PATH" - -# Run in background -$server_cmd 2>&1 > /dev/null & - -# Check if server is running by checking if the unix socket is created -FILE=/tmp/bloom-inference-0 -while : - do - if test -S "$FILE"; then - echo "Text Generation Python gRPC server started" - break - else - echo "Waiting for Text Generation Python gRPC server to start" - sleep 5 - fi - done - -sleep 1 - -# Run in background -text-generation-router & - -# Wait for any process to exit -wait -n - -# Exit with status of process that exited first -exit $? \ No newline at end of file diff --git a/router/rust-toolchain.toml b/rust-toolchain.toml similarity index 100% rename from router/rust-toolchain.toml rename to rust-toolchain.toml diff --git a/server/.gitignore b/server/.gitignore new file mode 100644 index 00000000..0ebf3670 --- /dev/null +++ b/server/.gitignore @@ -0,0 +1,155 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +bloom_inference/__pycache__/ +bloom_inference/pb/__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/server/Makefile b/server/Makefile index d959dbb3..52b4d405 100644 --- a/server/Makefile +++ b/server/Makefile @@ -4,17 +4,28 @@ gen-server: find bloom_inference/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; touch bloom_inference/pb/__init__.py -unit-tests: - python -m pytest --cov=bloom_inference tests +install-transformers: + # Install specific version of transformers + rm transformers || true + wget https://github.com/huggingface/transformers/archive/46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip + unzip 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip + rm 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip + mv transformers-46d37bece7d3ffdef97b1ee4a3170c0a0627d921 transformers + cd transformers && python setup.py install -unit-tests-reporting: - python -m pytest --junitxml=report.xml --cov=bloom_inference tests +install-torch: + # Install specific version of torch + pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir pip-install: pip install grpcio-tools make gen-server + make install-torch + make install-transformers pip install . install: poetry install - make gen-server \ No newline at end of file + make gen-server + make install-torch + make install-transformers diff --git a/server/bloom_inference/cli.py b/server/bloom_inference/cli.py index a5f84e77..751485bb 100644 --- a/server/bloom_inference/cli.py +++ b/server/bloom_inference/cli.py @@ -1,41 +1,51 @@ +import os import typer from pathlib import Path -from torch.distributed.launcher import launch_agent, LaunchConfig from typing import Optional -from bloom_inference import server +from bloom_inference import prepare_weights, server app = typer.Typer() @app.command() -def launcher( - model_name: str, - num_gpus: int = 1, - shard_directory: Optional[Path] = None, +def serve( + model_name: str, + sharded: bool = False, + shard_directory: Optional[Path] = None, + uds_path: Path = "/tmp/bloom-inference", ): - if num_gpus == 1: - serve(model_name, False, shard_directory) - - else: - config = LaunchConfig( - min_nodes=1, - max_nodes=1, - nproc_per_node=num_gpus, - rdzv_backend="c10d", - max_restarts=0, - ) - launch_agent(config, server.serve, [model_name, True, shard_directory]) + if sharded: + assert ( + shard_directory is not None + ), "shard_directory must be set when sharded is True" + assert ( + os.getenv("RANK", None) is not None + ), "RANK must be set when sharded is True" + assert ( + os.getenv("WORLD_SIZE", None) is not None + ), "WORLD_SIZE must be set when sharded is True" + assert ( + os.getenv("MASTER_ADDR", None) is not None + ), "MASTER_ADDR must be set when sharded is True" + assert ( + os.getenv("MASTER_PORT", None) is not None + ), "MASTER_PORT must be set when sharded is True" + + server.serve(model_name, sharded, uds_path, shard_directory) @app.command() -def serve( - model_name: str, - sharded: bool = False, - shard_directory: Optional[Path] = None, +def prepare_weights( + model_name: str, + shard_directory: Path, + cache_directory: Path, + num_shard: int = 1, ): - server.serve(model_name, sharded, shard_directory) + prepare_weights.prepare_weights( + model_name, cache_directory, shard_directory, num_shard + ) if __name__ == "__main__": diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py index 8b0e7ab0..0ba90cee 100644 --- a/server/bloom_inference/model.py +++ b/server/bloom_inference/model.py @@ -24,6 +24,7 @@ class Batch: batch_id: int requests: List[generate_pb2.Request] + all_input_lengths: List[int] input_ids: Dict[str, torch.Tensor] all_input_ids: List[torch.Tensor] next_token_choosers: List[NextTokenChooser] @@ -46,12 +47,12 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] - input_lengths = [] + all_input_lengths = [] # Parse batch for r in pb.requests: inputs.append(r.inputs) - input_lengths.append(r.input_length) + all_input_lengths.append(r.input_length) next_token_choosers.append( NextTokenChooser( temperature=r.parameters.temperature, @@ -63,17 +64,12 @@ def from_pb( stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) input_ids = tokenizer(inputs, return_tensors="pt", padding=True).to(device) - # Remove padding from all_input_ids - all_input_ids = [ - input_ids.squeeze(0)[-length:].unsqueeze(-1) - for length, input_ids in zip( - input_lengths, input_ids["input_ids"].split(1, dim=0) - ) - ] + all_input_ids = input_ids["input_ids"].unsqueeze(-1) return cls( batch_id=pb.id, requests=pb.requests, + all_input_lengths=all_input_lengths, input_ids=input_ids, all_input_ids=all_input_ids, next_token_choosers=next_token_choosers, @@ -91,6 +87,7 @@ def concatenate(cls, batches: List["Batch"]) -> "Batch": # Batch attributes input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} requests = [] + all_input_lengths = [] all_input_ids = [] next_token_choosers = [] stopping_criterias = [] @@ -100,6 +97,7 @@ def concatenate(cls, batches: List["Batch"]) -> "Batch": start_index = 0 for i, batch in enumerate(batches): requests.extend(batch.requests) + all_input_lengths.extend(batch.all_input_lengths) all_input_ids.extend(batch.all_input_ids) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -198,6 +196,7 @@ def concatenate(cls, batches: List["Batch"]) -> "Batch": return cls( batch_id=batches[0].batch_id, requests=requests, + all_input_lengths=all_input_lengths, input_ids=input_ids, all_input_ids=all_input_ids, next_token_choosers=next_token_choosers, @@ -227,7 +226,10 @@ def __init__(self, model_name: str): self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") self.model = ( - AutoModelForCausalLM.from_pretrained(model_name).eval().to(self.device).to(dtype) + AutoModelForCausalLM.from_pretrained(model_name) + .eval() + .to(self.device) + .to(dtype) ) self.num_heads = self.model.base_model.num_heads @@ -253,6 +255,7 @@ def generate_token( # New input_ids for next forward next_batch_input_ids = [] next_batch_all_input_ids = [] + next_all_input_lengths = [] next_batch_size = 0 next_batch_max_sequence_length = 0 @@ -263,6 +266,7 @@ def generate_token( # Zipped iterator iterator = zip( batch.requests, + batch.all_input_lengths, outputs.logits, batch.next_token_choosers, batch.stopping_criterias, @@ -272,6 +276,7 @@ def generate_token( # For each member of the batch for i, ( request, + input_length, logits, next_token_chooser, stopping_criteria, @@ -302,8 +307,10 @@ def generate_token( next_batch_input_ids.append(next_token) next_batch_all_input_ids.append(all_tokens) next_batch_size += 1 + new_input_length = input_length + 1 + next_all_input_lengths.append(new_input_length) next_batch_max_sequence_length = max( - next_batch_max_sequence_length, len(all_tokens) + next_batch_max_sequence_length, new_input_length ) # We finished all generations in the batch; there is no next batch @@ -350,6 +357,7 @@ def generate_token( next_batch = Batch( batch_id=batch.batch_id, requests=next_batch_requests, + all_input_lengths=next_all_input_lengths, input_ids=next_batch_input_ids, all_input_ids=next_batch_all_input_ids, next_token_choosers=next_batch_next_token_choosers, @@ -378,7 +386,10 @@ def __init__(self, model_name: str, shard_directory: Path): if self.master: # TODO @thomasw21 do some caching shard_state_dict_paths = prepare_weights( - model_name, shard_directory / "cache", shard_directory, tp_world_size=self.world_size + model_name, + shard_directory / "cache", + shard_directory, + tp_world_size=self.world_size, ) shard_state_dict_paths = [ str(path.absolute()) for path in shard_state_dict_paths @@ -443,6 +454,7 @@ def forward(self, input_ids, attention_mask, past_key_values: Optional = None): use_cache=True, ) + # Logits are sharded, so we need to gather them logits_shard = outputs.logits[:, -1, :].contiguous() batch_size, vocab_shard_size = logits_shard.shape diff --git a/server/bloom_inference/pb/.gitignore b/server/bloom_inference/pb/.gitignore index a9feac81..8527ad13 100644 --- a/server/bloom_inference/pb/.gitignore +++ b/server/bloom_inference/pb/.gitignore @@ -1,2 +1,2 @@ *.py -*.py-e +*.py-e \ No newline at end of file diff --git a/server/bloom_inference/prepare_weights.py b/server/bloom_inference/prepare_weights.py index 7cf3dbb5..5594998d 100644 --- a/server/bloom_inference/prepare_weights.py +++ b/server/bloom_inference/prepare_weights.py @@ -14,15 +14,15 @@ def match_suffix(text, suffix): - return text[-len(suffix):] == suffix + return text[-len(suffix) :] == suffix def http_get( - url: str, - temp_file: BinaryIO, - *, - timeout=10.0, - max_retries=0, + url: str, + temp_file: BinaryIO, + *, + timeout=10.0, + max_retries=0, ): """ Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub. @@ -54,7 +54,9 @@ def cache_download_url(url: str, root_dir: Path): return filename -def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world_size: int): +def prepare_weights( + model_name: str, cache_path: Path, save_path: Path, tp_world_size: int +): save_paths = [ save_path / f"{model_name}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" for tp_rank in range(tp_world_size) @@ -68,6 +70,7 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world if model_name == "bigscience/bloom-560m": url = hf_hub_url(model_name, filename="pytorch_model.bin") cache_download_url(url, cache_path) + elif model_name == "bigscience/bloom": url = hf_hub_url(model_name, filename="pytorch_model.bin.index.json") index_path = cache_download_url(url, cache_path) @@ -75,10 +78,14 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world index = json.load(f) # Get unique file names - weight_files = list(set([filename for filename in index["weight_map"].values()])) + weight_files = list( + set([filename for filename in index["weight_map"].values()]) + ) urls = [hf_hub_url(model_name, filename=filename) for filename in weight_files] - Parallel(n_jobs=5)(delayed(cache_download_url)(url, cache_path) for url in tqdm(urls)) + Parallel(n_jobs=5)( + delayed(cache_download_url)(url, cache_path) for url in tqdm(urls) + ) else: raise ValueError(f"Unknown model name: {model_name}") @@ -91,14 +98,14 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world for state_name in keys: state = state_dict[state_name] if any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.query_key_value.weight", - "self_attention.query_key_value.bias", - "mlp.dense_h_to_4h.weight", - "mlp.dense_h_to_4h.bias", - "word_embeddings.weight", - ] + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.query_key_value.weight", + "self_attention.query_key_value.bias", + "mlp.dense_h_to_4h.weight", + "mlp.dense_h_to_4h.bias", + "word_embeddings.weight", + ] ): output_size = state.shape[0] assert output_size % tp_world_size == 0 @@ -107,7 +114,9 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world assert len(sharded_weights) == tp_world_size for tp_rank, shard in enumerate(sharded_weights): - shards_state_dicts[tp_rank]["transformer." + state_name] = shard.detach().clone() + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = shard.detach().clone() elif match_suffix(state_name, "lm_head.weight"): output_size = state.shape[0] @@ -120,11 +129,11 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world shards_state_dicts[tp_rank][state_name] = shard.detach().clone() elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.weight", - "mlp.dense_4h_to_h.weight", - ] + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.weight", + "mlp.dense_4h_to_h.weight", + ] ): input_size = state.shape[1] assert input_size % tp_world_size == 0 @@ -132,23 +141,31 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world sharded_weights = torch.split(state, block_size, dim=1) assert len(sharded_weights) == tp_world_size for tp_rank, shard in enumerate(sharded_weights): - shards_state_dicts[tp_rank]["transformer." + state_name] = shard.detach().clone() + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = shard.detach().clone() elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.bias", - "mlp.dense_4h_to_h.bias", - ] + match_suffix(state_name, candidate) + for candidate in [ + "self_attention.dense.bias", + "mlp.dense_4h_to_h.bias", + ] ): - shards_state_dicts[0]["transformer." + state_name] = state.detach().clone() + shards_state_dicts[0][ + "transformer." + state_name + ] = state.detach().clone() for tp_rank in range(1, tp_world_size): - shards_state_dicts[tp_rank]["transformer." + state_name] = torch.zeros_like(state) + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = torch.zeros_like(state) else: # We duplicate parameters across tp ranks for tp_rank in range(tp_world_size): - shards_state_dicts[tp_rank]["transformer." + state_name] = state.detach().clone() + shards_state_dicts[tp_rank][ + "transformer." + state_name + ] = state.detach().clone() del state_dict[state_name] # delete key from state_dict del state # delete tensor @@ -156,7 +173,7 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world # we save state_dict for tp_rank, (save_path, shard_state_dict) in enumerate( - zip(save_paths, shards_state_dicts) + zip(save_paths, shards_state_dicts) ): save_paths.append(save_path) save_path.parent.mkdir(parents=True, exist_ok=True) @@ -166,17 +183,3 @@ def prepare_weights(model_name: str, cache_path: Path, save_path: Path, tp_world torch.save(shard_state_dict, save_path) return save_paths - - -if __name__ == "__main__": - from argparse import ArgumentParser - - parser = ArgumentParser() - - parser.add_argument("--model-name", required=True, type=str) - parser.add_argument("--cache-path", required=True, type=str) - parser.add_argument("--save-path", required=True, type=str) - parser.add_argument("--world-size", required=True, type=int) - args = parser.parse_args() - - prepare_weights(args.model_name, Path(args.cache_path), Path(args.save_path), args.world_size) diff --git a/server/bloom_inference/server.py b/server/bloom_inference/server.py index e89706e0..734aba43 100644 --- a/server/bloom_inference/server.py +++ b/server/bloom_inference/server.py @@ -64,70 +64,31 @@ async def GenerateWithCache(self, request, context): batch=next_batch.to_pb() if next_batch else None, ) - async def GenerateUntilFinished(self, request, context): - batch = Batch.from_pb(request.batch, self.model.tokenizer, self.model.device) - - generated_texts = [] - while not generated_texts: - generated_texts, next_batch = self.model.generate_token(batch) - batch = next_batch - self.cache.set(next_batch) - - return generate_pb2.GenerateUntilFinishedResponse( - generated_texts=[ - generated_text.to_pb() for generated_text in generated_texts - ], - batch=next_batch.to_pb() if next_batch else None, - ) - - async def GenerateUntilFinishedWithCache(self, request, context): - if len(request.batches) == 0: - raise ValueError("Must provide at least one batch") - - batches = [] - for batch_pb in request.batches: - batch = self.cache.pop(batch_pb.id) - if batch is None: - raise ValueError(f"Batch ID {batch_pb.id} not found in cache.") - batches.append(batch) - - if len(batches) > 1: - batch = Batch.concatenate(batches) - else: - batch = batches[0] - - generated_texts = [] - while not generated_texts: - generated_texts, next_batch = self.model.generate_token(batch) - batch = next_batch - self.cache.set(next_batch) - - return generate_pb2.GenerateUntilFinishedWithCacheResponse( - generated_texts=[ - generated_text.to_pb() for generated_text in generated_texts - ], - batch=next_batch.to_pb() if next_batch else None, - ) - -def serve(model_name, sharded, shard_directory): +def serve( + model_name: str, + sharded: bool, + uds_path: Path, + shard_directory: Optional[Path] = None, +): async def serve_inner( model_name: str, sharded: bool = False, shard_directory: Optional[Path] = None, ): - unix_socket_template = "unix:///tmp/bloom-inference-{}" + unix_socket_template = "unix://{}-{}" if sharded: if shard_directory is None: raise ValueError("shard_directory must be set when sharded is True") model = BLOOMSharded(model_name, shard_directory) server_urls = [ - unix_socket_template.format(rank) for rank in range(model.world_size) + unix_socket_template.format(uds_path, rank) + for rank in range(model.world_size) ] - local_url = unix_socket_template.format(model.rank) + local_url = server_urls[model.rank] else: model = BLOOM(model_name) - local_url = unix_socket_template.format(0) + local_url = unix_socket_template.format(uds_path, 0) server_urls = [local_url] server = aio.server() @@ -142,6 +103,10 @@ async def serve_inner( server.add_insecure_port(local_url) await server.start() print("Server started at {}".format(local_url)) - await server.wait_for_termination() + try: + await server.wait_for_termination() + except KeyboardInterrupt: + print("Signal received. Shutting down") + await server.stop(0) asyncio.run(serve_inner(model_name, sharded, shard_directory)) diff --git a/server/bloom_inference/utils.py b/server/bloom_inference/utils.py index fe2c913e..c351806a 100644 --- a/server/bloom_inference/utils.py +++ b/server/bloom_inference/utils.py @@ -82,7 +82,6 @@ def initialize_torch_distributed(): world_size=world_size, rank=rank, timeout=timedelta(seconds=60), - init_method="tcp://localhost:6000", ) return torch.distributed.distributed_c10d._get_default_group(), rank, world_size diff --git a/server/poetry.lock b/server/poetry.lock index 8100c200..3feee60d 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -205,7 +205,7 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "f3dc5b2420183f2e7e9257e372489409d7bd26d1dcc535fc2558ebca50c988c2" +content-hash = "a4eef5f52e8d046aa883082c865b0865047f611a3240b18250487d4b6e831496" [metadata.files] accelerate = [ diff --git a/server/pyproject.toml b/server/pyproject.toml index 1dd8ae27..3d38f512 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -11,7 +11,6 @@ bloom-inference-server = 'bloom_inference.cli:app' python = "^3.9" protobuf = "^4.21.7" grpcio = "^1.49.1" -torch = "^1.12.1" typer = "^0.6.1" grpcio-reflection = "^1.49.1" accelerate = "^0.12.0" From 895a341d064c9930b2a9bd60cff0df42f91b52fa Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 21 Oct 2022 10:59:15 +0200 Subject: [PATCH 013/793] fix(validation): Fix error messages --- router/src/validation.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/router/src/validation.rs b/router/src/validation.rs index 49a46b62..31b4c49f 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -127,7 +127,7 @@ fn validation_worker( if input_length > max_input_length { response_tx - .send(Err(ValidationError::InputLength(input_length))) + .send(Err(ValidationError::InputLength(input_length, max_input_length))) .unwrap_or(()); continue; } @@ -145,14 +145,14 @@ type ValidationRequest = ( pub enum ValidationError { #[error("Temperature must be strictly positive")] Temperature, - #[error("Top p must be <= 0.0 or > 1.0")] + #[error("Top p must be >= 0.0 or < 1.0")] TopP, #[error("Top k must be strictly positive")] TopK, - #[error("Max New Tokens must be < 512")] + #[error("Max New Tokens must be <= 512")] MaxNewTokens, - #[error("Inputs must have less than 1000 tokens. Given: {0}")] - InputLength(usize), + #[error("Inputs must have less than {1} tokens. Given: {0}")] + InputLength(usize, usize), } impl From for (StatusCode, String) { From c8378933700e3b2b66534ca89c53a69c7b9468d3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 21 Oct 2022 16:40:05 +0200 Subject: [PATCH 014/793] feat(router): Add max_waiting_tokens --- launcher/src/main.rs | 12 +++---- router/src/batcher.rs | 59 ++++++++++++++++++------------ router/src/db.rs | 28 ++++++--------- router/src/lib.rs | 4 +-- router/src/main.rs | 15 ++++---- router/src/server.rs | 77 ++++++++++++++++++++++++++++++---------- router/src/validation.rs | 5 ++- 7 files changed, 121 insertions(+), 79 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 0821a456..0aedf563 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -28,8 +28,8 @@ struct Args { max_input_length: usize, #[clap(default_value = "32", long, env)] max_batch_size: usize, - #[clap(default_value = "5", long, env)] - max_waiting_time: u64, + #[clap(default_value = "20", long, env)] + max_waiting_tokens: usize, #[clap(default_value = "3000", long, short, env)] port: u16, #[clap(default_value = "/tmp/text-generation-server", long, env)] @@ -41,7 +41,7 @@ struct Args { } fn main() -> ExitCode { - tracing_subscriber::fmt::init(); + tracing_subscriber::fmt().compact().with_ansi(false).init(); // Pattern match configuration let Args { @@ -51,7 +51,7 @@ fn main() -> ExitCode { max_concurrent_requests, max_input_length, max_batch_size, - max_waiting_time, + max_waiting_tokens, port, shard_uds_path, master_addr, @@ -148,8 +148,8 @@ fn main() -> ExitCode { &max_input_length.to_string(), "--max-batch-size", &max_batch_size.to_string(), - "--max-waiting-time", - &max_waiting_time.to_string(), + "--max-waiting-tokens", + &max_waiting_tokens.to_string(), "--port", &port.to_string(), "--master-shard-uds-path", diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 4523dbff..052716a4 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -5,7 +5,6 @@ use axum::http::StatusCode; use bloom_inference_client::{Batch, ClientError, GeneratedText, ShardedClient}; use std::future::Future; use std::sync::Arc; -use std::time::Duration; use thiserror::Error; use tokio::sync::{oneshot, Notify}; use tokio::time::Instant; @@ -30,7 +29,7 @@ impl Batcher { pub(crate) fn new( client: ShardedClient, max_batch_size: usize, - max_waiting_time: Duration, + max_waiting_tokens: usize, ) -> Self { // Batcher shared state let db = Db::new(); @@ -41,7 +40,7 @@ impl Batcher { // Spawn batching background task that contains all the inference logic tokio::spawn(batching_task( max_batch_size, - max_waiting_time, + max_waiting_tokens, client, db.clone(), shared.clone(), @@ -55,7 +54,7 @@ impl Batcher { &self, input_length: usize, request: GenerateRequest, - ) -> Result { + ) -> Result { // One shot channel to communicate with the background batching task let (response_tx, response_rx) = oneshot::channel(); @@ -65,6 +64,7 @@ impl Batcher { response_tx, input_length, time: Instant::now(), + batch_time: None, }); // Notify the background task that we have a new entry in the database that needs @@ -87,7 +87,7 @@ impl Batcher { #[instrument(skip(client, db, shared))] async fn batching_task( max_batch_size: usize, - max_waiting_time: Duration, + max_waiting_tokens: usize, client: ShardedClient, db: Db, shared: Arc, @@ -103,8 +103,10 @@ async fn batching_task( // Get the next batch from the DB // This batch might be smaller than the maximum batch size if there are not enough requests // waiting in the DB - if let Some((request_ids, batch)) = db.next_batch(None, max_batch_size, None) { + let mut waiting_tokens = 0; + if let Some((request_ids, batch)) = db.next_batch(None, max_batch_size) { let mut cached_batch = wrap_future(client.generate(batch), request_ids, &db).await; + waiting_tokens += 1; // We loop until we do not receive any cached batch from the inference server (== until // all requests have met their stopping criteria) @@ -116,10 +118,20 @@ async fn batching_task( // If the current batch is too small, we try to add more requests to it if batch_size <= limit_min_batch_size { - // Get the next batch from the DB that meet our minimum size criteria + let min_size = match waiting_tokens { + // If we didn't onboard any new requests since >= max_waiting_tokens, we try + // to add a new batch even though its size might be small + _ if waiting_tokens >= max_waiting_tokens => None, + // Minimum size criteria + _ => Some(limit_min_batch_size as usize), + }; + + // Try to get a new batch if let Some((new_request_ids, new_batch)) = - db.next_batch(Some(limit_min_batch_size as usize), max_batch_size, None) + db.next_batch(min_size, max_batch_size) { + // Reset waiting counter + waiting_tokens = 0; // Generate one token for this new batch to have the attention past in cache let new_cached_batch = wrap_future(client.generate(new_batch), new_request_ids, &db).await; @@ -129,24 +141,11 @@ async fn batching_task( batches.push(new_cached_batch); } } - // If we don't have enough requests to meet the minimum size criteria, we - // try to get the next batch from the DB that have been waiting over - // the max_waiting_time - else if let Some((new_request_ids, new_batch)) = - db.next_batch(None, max_batch_size, Some(max_waiting_time)) - { - let new_cached_batch = - wrap_future(client.generate(new_batch), new_request_ids, &db).await; - // Extend current batch with the new batch - if let Some(new_cached_batch) = new_cached_batch { - request_ids.extend(new_cached_batch.requests.iter().map(|req| req.id)); - batches.push(new_cached_batch); - } - } } cached_batch = wrap_future(client.generate_with_cache(batches), request_ids, &db).await; + waiting_tokens += 1; } } } @@ -188,11 +187,25 @@ fn send_generated(finished: Vec, db: &Db) { let entry = db .remove(&output.request.unwrap().id) .expect("ID not found in db. This is a bug."); + let response = InferResponse { + output: output.output, + queued: entry.time, + start: entry.batch_time.unwrap(), // unwrap is always valid + end: Instant::now(), + }; // unwrap_or is valid here as we don't care if the receiver is gone. - entry.response_tx.send(Ok(output.output)).unwrap_or(()); + entry.response_tx.send(Ok(response)).unwrap_or(()); }); } +#[derive(Debug)] +pub(crate) struct InferResponse { + pub(crate) output: String, + pub(crate) queued: Instant, + pub(crate) start: Instant, + pub(crate) end: Instant, +} + #[derive(Debug, Error)] pub enum InferError { #[error("Request failed during generation: {0}")] diff --git a/router/src/db.rs b/router/src/db.rs index 9518fa1d..76a08ae0 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,10 +1,10 @@ +use crate::InferResponse; /// This code is massively inspired by Tokio mini-redis use crate::{GenerateParameters, GenerateRequest}; use bloom_inference_client::{Batch, ClientError, LogitsWarperParameters, Request}; use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; -use std::time::Duration; use tokio::sync::oneshot::Sender; use tokio::time::Instant; @@ -14,11 +14,13 @@ pub(crate) struct Entry { /// Request pub request: GenerateRequest, /// Response sender to communicate between the Batcher and the batching_task - pub response_tx: Sender>, + pub response_tx: Sender>, /// Number of tokens in the input pub input_length: usize, /// Instant when this entry was created pub time: Instant, + /// Instant when this entry was added to a batch + pub batch_time: Option, } /// Request Database @@ -51,11 +53,7 @@ struct State { impl State { /// Get the next requests - fn next_requests( - &self, - max_size: usize, - min_waiting_time: Option, - ) -> Option<(Vec, Vec)> { + fn next_requests(&self, max_size: usize) -> Option<(Vec, Vec)> { // Iterates for max_size over the BTreemap starting from next_batch_start_id let mut requests = Vec::new(); let mut ids = Vec::new(); @@ -67,15 +65,6 @@ impl State { // Take max_size .take(max_size) { - if let Some(min_waiting_time) = min_waiting_time { - // Only take entries that waited for at least min_waiting_time - if entry.time.elapsed() < min_waiting_time { - // Since entries are ordered, we already know that all following entries won't - // satisfy the condition - break; - } - } - requests.push(Request { id: *id, inputs: entry.request.inputs.clone(), @@ -134,19 +123,22 @@ impl Db { &self, min_size: Option, max_size: usize, - min_waiting_time: Option, ) -> Option<(Vec, Batch)> { // Acquire lock let mut state = self.shared.state.lock(); // Get requests from the database - if let Some((ids, requests)) = state.next_requests(max_size, min_waiting_time) { + if let Some((ids, requests)) = state.next_requests(max_size) { if let Some(min_size) = min_size { // If min_size is set, only return a batch if there are enough requests if requests.len() < min_size { return None; } } + ids.iter().for_each(|id| { + // Set batch_time for each request + state.entries.get_mut(id).unwrap().batch_time = Some(Instant::now()); + }); // Batch size let size = requests.len(); diff --git a/router/src/lib.rs b/router/src/lib.rs index 02b912a3..6604a91f 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -4,7 +4,7 @@ mod db; pub mod server; mod validation; -use batcher::Batcher; +use batcher::{Batcher, InferResponse}; use db::{Db, Entry}; use serde::{Deserialize, Serialize}; use validation::Validation; @@ -64,5 +64,3 @@ pub(crate) struct GenerateRequest { pub(crate) struct GeneratedText { pub generated_text: String, } - -pub(crate) type GenerateResponse = Vec; diff --git a/router/src/main.rs b/router/src/main.rs index 49051b37..6d1a0fb9 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -2,7 +2,6 @@ use bloom_inference_client::ShardedClient; use clap::Parser; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::time::Duration; use text_generation_router::server; use tokenizers::Tokenizer; @@ -16,8 +15,8 @@ struct Args { max_input_length: usize, #[clap(default_value = "32", long, env)] max_batch_size: usize, - #[clap(default_value = "5", long, env)] - max_waiting_time: u64, + #[clap(default_value = "20", long, env)] + max_waiting_tokens: usize, #[clap(default_value = "3000", long, short, env)] port: u16, #[clap(default_value = "/tmp/bloom-inference-0", long, env)] @@ -36,19 +35,19 @@ fn main() -> Result<(), std::io::Error> { max_concurrent_requests, max_input_length, max_batch_size, - max_waiting_time, + max_waiting_tokens, port, master_shard_uds_path, tokenizer_name, validation_workers, } = args; + tracing_subscriber::fmt().compact().with_ansi(false).init(); + if validation_workers == 1 { panic!("validation_workers must be > 0"); } - let max_waiting_time = Duration::from_secs(max_waiting_time); - // Download and instantiate tokenizer // This will only be used to validate payloads // @@ -61,8 +60,6 @@ fn main() -> Result<(), std::io::Error> { .build() .unwrap() .block_on(async { - tracing_subscriber::fmt::init(); - // Instantiate sharded client from the master unix socket let sharded_client = ShardedClient::connect_uds(master_shard_uds_path) .await @@ -82,7 +79,7 @@ fn main() -> Result<(), std::io::Error> { max_concurrent_requests, max_input_length, max_batch_size, - max_waiting_time, + max_waiting_tokens, sharded_client, tokenizer, validation_workers, diff --git a/router/src/server.rs b/router/src/server.rs index 42258313..02c4a497 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,14 +1,12 @@ -use crate::{ - Batcher, GenerateParameters, GenerateRequest, GenerateResponse, GeneratedText, Validation, -}; +use crate::{Batcher, GenerateParameters, GenerateRequest, GeneratedText, Validation}; use axum::extract::Extension; -use axum::http::StatusCode; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; use axum::routing::{get, post}; use axum::{Json, Router}; use bloom_inference_client::ShardedClient; use std::net::SocketAddr; use std::sync::Arc; -use std::time::Duration; use tokenizers::Tokenizer; use tokio::signal; use tokio::sync::Semaphore; @@ -59,12 +57,21 @@ async fn health(state: Extension) -> Result<(), (StatusCode, String } /// Generate method -#[instrument(skip(state), fields(time, time_per_token))] +#[instrument( + skip(state), + fields( + total_time, + validation_time, + queue_time, + inference_time, + time_per_token + ) +)] async fn generate( state: Extension, req: Json, -) -> Result, (StatusCode, String)> { - let start = Instant::now(); +) -> Result { + let start_time = Instant::now(); // Limit concurrent requests by acquiring a permit from the semaphore let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { ( @@ -84,19 +91,51 @@ async fn generate( .await?; // Inference - let generated_text = state.batcher.infer(input_length, validated_request).await?; + let response = state.batcher.infer(input_length, validated_request).await?; + + // Timings + let total_time = start_time.elapsed(); + let validation_time = response.queued - start_time; + let queue_time = response.start - response.queued; + let inference_time = response.end - response.start; + let time_per_token = inference_time / req.parameters.max_new_tokens; + + // Headers + let mut headers = HeaderMap::new(); + headers.insert( + "x-total-time", + total_time.as_millis().to_string().parse().unwrap(), + ); + headers.insert( + "x-validation-time", + validation_time.as_millis().to_string().parse().unwrap(), + ); + headers.insert( + "x-queue-time", + queue_time.as_millis().to_string().parse().unwrap(), + ); + headers.insert( + "x-inference-time", + inference_time.as_millis().to_string().parse().unwrap(), + ); + headers.insert( + "x-time-per-token", + time_per_token.as_millis().to_string().parse().unwrap(), + ); // Tracing metadata - tracing::Span::current().record("time", format!("{:?}", start.elapsed())); - tracing::Span::current().record( - "time_per_token", - format!("{:?}", start.elapsed() / req.parameters.max_new_tokens), - ); - tracing::info!("response: {}", generated_text); + tracing::Span::current().record("total_time", format!("{:?}", total_time)); + tracing::Span::current().record("validation_time", format!("{:?}", validation_time)); + tracing::Span::current().record("queue_time", format!("{:?}", queue_time)); + tracing::Span::current().record("inference_time", format!("{:?}", inference_time)); + tracing::Span::current().record("time_per_token", format!("{:?}", time_per_token)); + tracing::info!("Output: {}", response.output); // Send response - let response = vec![GeneratedText { generated_text }]; - Ok(Json(response)) + let response = vec![GeneratedText { + generated_text: response.output, + }]; + Ok((headers, Json(response))) } /// Serving method @@ -105,14 +144,14 @@ pub async fn run( max_concurrent_requests: usize, max_input_length: usize, max_batch_size: usize, - max_waiting_time: Duration, + max_waiting_tokens: usize, client: ShardedClient, tokenizer: Tokenizer, validation_workers: usize, addr: SocketAddr, ) { // Create state - let batcher = Batcher::new(client, max_batch_size, max_waiting_time); + let batcher = Batcher::new(client, max_batch_size, max_waiting_tokens); let validation = Validation::new(validation_workers, tokenizer, max_input_length); let shared_state = ServerState { validation, diff --git a/router/src/validation.rs b/router/src/validation.rs index 31b4c49f..11712d0a 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -127,7 +127,10 @@ fn validation_worker( if input_length > max_input_length { response_tx - .send(Err(ValidationError::InputLength(input_length, max_input_length))) + .send(Err(ValidationError::InputLength( + input_length, + max_input_length, + ))) .unwrap_or(()); continue; } From be8827fe41bdebae77118b3bda0c1e1ec2c8d0e0 Mon Sep 17 00:00:00 2001 From: Thomas Wang <24695242+thomasw21@users.noreply.github.com> Date: Sat, 22 Oct 2022 10:44:52 +0200 Subject: [PATCH 015/793] Create LICENSE (#2) --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..7d0e8034 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Hugging Face + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From c8ce9b25153e727e9b2de3efa0acf5d69922ac62 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Sat, 22 Oct 2022 20:00:15 +0200 Subject: [PATCH 016/793] feat(server): Use safetensors Co-authored-by: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> --- Cargo.toml | 3 + Dockerfile | 15 +- Makefile | 17 +- README.md | 12 +- launcher/src/main.rs | 53 ++-- server/.gitignore | 3 + server/Makefile | 32 +- server/README.md | 6 +- server/bloom_inference/cli.py | 18 +- server/bloom_inference/model.py | 145 +++++---- server/bloom_inference/prepare_weights.py | 185 ----------- server/bloom_inference/server.py | 8 +- server/bloom_inference/utils.py | 54 +++- server/poetry.lock | 370 +++++++++++----------- 14 files changed, 420 insertions(+), 501 deletions(-) delete mode 100644 server/bloom_inference/prepare_weights.py diff --git a/Cargo.toml b/Cargo.toml index d3f7dfb0..e23b802e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,9 @@ members = [ "router/client", "launcher" ] +exclude = [ + "server/safetensors", +] [profile.release] debug = 1 diff --git a/Dockerfile b/Dockerfile index 70b91bc2..78f66c0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,10 +36,10 @@ ENV LANG=C.UTF-8 \ SHELL ["/bin/bash", "-c"] -RUN apt-get update && apt-get install -y unzip wget libssl-dev && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y unzip curl libssl-dev && rm -rf /var/lib/apt/lists/* RUN cd ~ && \ - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + curl -L -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ chmod +x Miniconda3-latest-Linux-x86_64.sh && \ bash ./Miniconda3-latest-Linux-x86_64.sh -bf -p /opt/miniconda && \ conda create -n text-generation python=3.9 -y @@ -54,13 +54,22 @@ RUN cd server && make install-torch # Install specific version of transformers RUN cd server && make install-transformers +# Install specific version of safetensors +# FIXME: This is a temporary fix while we wait for a new release +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" +RUN cd server && make install-safetensors + # Install server +COPY proto proto COPY server server RUN cd server && \ + make gen-server && \ /opt/miniconda/envs/text-generation/bin/pip install . --no-cache-dir # Install router COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/local/bin/text-generation-router +# Install launcher COPY --from=launcher-builder /usr/local/cargo/bin/text-generation-launcher /usr/local/bin/text-generation-launcher -CMD text-generation-launcher --model-name $MODEL_NAME --num-shard $NUM_GPUS --shard-directory $MODEL_BASE_PATH \ No newline at end of file +CMD HUGGINGFACE_HUB_CACHE=$MODEL_BASE_PATH text-generation-launcher --model-name $MODEL_NAME --num-shard $NUM_GPUS \ No newline at end of file diff --git a/Makefile b/Makefile index 3a80a12d..e22309f9 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ install-server: - cd server && make pip-install + cd server && make install install-router: cd router && cargo install --path . @@ -7,13 +7,16 @@ install-router: install-launcher: cd launcher && cargo install --path . -install: - make install-server - make install-router - make install-launcher +install: install-server install-router install-launcher + +server-dev: + cd server && make run-dev + +router-dev: + cd router && cargo run run-bloom-560m: - text-generation-launcher --model-name bigscience/bloom-560m --shard-directory /tmp/models --num-shard 2 + text-generation-launcher --model-name bigscience/bloom-560m --num-shard 2 run-bloom: - text-generation-launcher --model-name bigscience/bloom --shard-directory /tmp/models --num-shard 8 + text-generation-launcher --model-name bigscience/bloom --num-shard 8 diff --git a/README.md b/README.md index 18e9865d..1af90dce 100644 --- a/README.md +++ b/README.md @@ -41,11 +41,21 @@ make run-bloom-560m ```shell curl 127.0.0.1:3000/generate \ + -v \ -X POST \ -d '{"inputs":"Testing API","parameters":{"max_new_tokens":9}}' \ -H 'Content-Type: application/json' ``` +## Develop + +```shell +make server-dev +make router-dev +``` + ## TODO: -- [ ] Add tests for the `server/model` logic \ No newline at end of file +- [ ] Add tests for the `server/model` logic +- [ ] Backport custom CUDA kernels to Transformers +- [ ] Install safetensors with pip \ No newline at end of file diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 0aedf563..6b529d60 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -1,4 +1,5 @@ use clap::Parser; +use std::env; use std::io::{BufRead, BufReader, Read}; use std::path::Path; use std::process::ExitCode; @@ -20,8 +21,6 @@ struct Args { model_name: String, #[clap(long, env)] num_shard: Option, - #[clap(long, env)] - shard_directory: Option, #[clap(default_value = "128", long, env)] max_concurrent_requests: usize, #[clap(default_value = "1000", long, env)] @@ -47,7 +46,6 @@ fn main() -> ExitCode { let Args { model_name, num_shard, - shard_directory, max_concurrent_requests, max_input_length, max_batch_size, @@ -82,7 +80,6 @@ fn main() -> ExitCode { for rank in 0..num_shard { let model_name = model_name.clone(); let uds_path = shard_uds_path.clone(); - let shard_directory = shard_directory.clone(); let master_addr = master_addr.clone(); let status_sender = status_sender.clone(); let shutdown = shutdown.clone(); @@ -91,7 +88,6 @@ fn main() -> ExitCode { shard_manager( model_name, uds_path, - shard_directory, rank, num_shard, master_addr, @@ -237,7 +233,6 @@ enum ShardStatus { fn shard_manager( model_name: String, uds_path: String, - shard_directory: Option, rank: usize, world_size: usize, master_addr: String, @@ -265,10 +260,27 @@ fn shard_manager( shard_argv.push("--sharded".to_string()); } - if let Some(shard_directory) = shard_directory { - shard_argv.push("--shard-directory".to_string()); - shard_argv.push(shard_directory); - } + let mut env = vec![ + ("RANK".parse().unwrap(), rank.to_string().parse().unwrap()), + ( + "WORLD_SIZE".parse().unwrap(), + world_size.to_string().parse().unwrap(), + ), + ("MASTER_ADDR".parse().unwrap(), master_addr.parse().unwrap()), + ( + "MASTER_PORT".parse().unwrap(), + master_port.to_string().parse().unwrap(), + ), + ]; + + // If the HUGGINGFACE_HUB_CACHE env var is set, pass it to the shard + // Useful when running inside a docker container + if let Ok(huggingface_hub_cache) = env::var("HUGGINGFACE_HUB_CACHE") { + env.push(( + "HUGGINGFACE_HUB_CACHE".parse().unwrap(), + huggingface_hub_cache.parse().unwrap(), + )); + }; // Start process tracing::info!("Starting shard {}", rank); @@ -280,18 +292,7 @@ fn shard_manager( // Needed for the shutdown procedure setpgid: true, // NCCL env vars - env: Some(vec![ - ("RANK".parse().unwrap(), rank.to_string().parse().unwrap()), - ( - "WORLD_SIZE".parse().unwrap(), - world_size.to_string().parse().unwrap(), - ), - ("MASTER_ADDR".parse().unwrap(), master_addr.parse().unwrap()), - ( - "MASTER_PORT".parse().unwrap(), - master_port.to_string().parse().unwrap(), - ), - ]), + env: Some(env), ..Default::default() }, ) { @@ -312,6 +313,7 @@ fn shard_manager( let mut ready = false; let start_time = Instant::now(); + let mut wait_time = Instant::now(); loop { // Process exited if p.poll().is_some() { @@ -337,9 +339,12 @@ fn shard_manager( status_sender.send(ShardStatus::Ready).unwrap(); ready = true; } else if !ready { - tracing::info!("Waiting for shard {} to be ready...", rank); + if wait_time.elapsed() > Duration::from_secs(10) { + tracing::info!("Waiting for shard {} to be ready...", rank); + wait_time = Instant::now(); + } } - sleep(Duration::from_secs(5)); + sleep(Duration::from_millis(100)); } } diff --git a/server/.gitignore b/server/.gitignore index 0ebf3670..88bbea8f 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -153,3 +153,6 @@ dmypy.json # Cython debug symbols cython_debug/ + +transformers +safetensors \ No newline at end of file diff --git a/server/Makefile b/server/Makefile index 52b4d405..9c7a2c43 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,4 +1,6 @@ gen-server: + # Compile protos + pip install grpcio-tools==1.49.1 --no-cache-dir mkdir bloom_inference/pb || true python -m grpc_tools.protoc -I../proto --python_out=bloom_inference/pb --grpc_python_out=bloom_inference/pb ../proto/generate.proto find bloom_inference/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; @@ -7,25 +9,31 @@ gen-server: install-transformers: # Install specific version of transformers rm transformers || true - wget https://github.com/huggingface/transformers/archive/46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip + rm transformers-46d37bece7d3ffdef97b1ee4a3170c0a0627d921 || true + curl -L -O https://github.com/huggingface/transformers/archive/46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip unzip 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip rm 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip mv transformers-46d37bece7d3ffdef97b1ee4a3170c0a0627d921 transformers cd transformers && python setup.py install +install-safetensors: + # Install specific version of safetensors + pip install setuptools_rust + rm safetensors || true + rm safetensors-634deccbcbad5eaf417935281f8b3be7ebca69c5 || true + curl -L -O https://github.com/huggingface/safetensors/archive/634deccbcbad5eaf417935281f8b3be7ebca69c5.zip + unzip 634deccbcbad5eaf417935281f8b3be7ebca69c5.zip + rm 634deccbcbad5eaf417935281f8b3be7ebca69c5.zip + mv safetensors-634deccbcbad5eaf417935281f8b3be7ebca69c5 safetensors + cd safetensors/bindings/python && python setup.py develop + install-torch: # Install specific version of torch pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir -pip-install: - pip install grpcio-tools - make gen-server - make install-torch - make install-transformers - pip install . +install: gen-server install-torch install-transformers install-safetensors + pip install pip --upgrade + pip install -e . --no-cache-dir -install: - poetry install - make gen-server - make install-torch - make install-transformers +run-dev: + python -m torch.distributed.run --nproc_per_node=2 bloom_inference/cli.py serve bigscience/bloom-560m --sharded \ No newline at end of file diff --git a/server/README.md b/server/README.md index 9127b978..c7054b8d 100644 --- a/server/README.md +++ b/server/README.md @@ -2,14 +2,14 @@ A Python gRPC server for BLOOM Inference -## Local Install (with poetry) +## Install ```shell make install ``` -## Local Install (with pip) +## Run ```shell -make pip-install +make run-dev ``` \ No newline at end of file diff --git a/server/bloom_inference/cli.py b/server/bloom_inference/cli.py index 751485bb..600c5273 100644 --- a/server/bloom_inference/cli.py +++ b/server/bloom_inference/cli.py @@ -2,9 +2,8 @@ import typer from pathlib import Path -from typing import Optional -from bloom_inference import prepare_weights, server +from bloom_inference import server, utils app = typer.Typer() @@ -13,13 +12,9 @@ def serve( model_name: str, sharded: bool = False, - shard_directory: Optional[Path] = None, uds_path: Path = "/tmp/bloom-inference", ): if sharded: - assert ( - shard_directory is not None - ), "shard_directory must be set when sharded is True" assert ( os.getenv("RANK", None) is not None ), "RANK must be set when sharded is True" @@ -33,19 +28,14 @@ def serve( os.getenv("MASTER_PORT", None) is not None ), "MASTER_PORT must be set when sharded is True" - server.serve(model_name, sharded, uds_path, shard_directory) + server.serve(model_name, sharded, uds_path) @app.command() -def prepare_weights( +def download_weights( model_name: str, - shard_directory: Path, - cache_directory: Path, - num_shard: int = 1, ): - prepare_weights.prepare_weights( - model_name, cache_directory, shard_directory, num_shard - ) + utils.download_weights(model_name) if __name__ == "__main__": diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py index 0ba90cee..eac54002 100644 --- a/server/bloom_inference/model.py +++ b/server/bloom_inference/model.py @@ -2,19 +2,24 @@ import torch.distributed from dataclasses import dataclass -from pathlib import Path from typing import List, Tuple, Optional, Dict +from accelerate import init_empty_weights +from safetensors import safe_open from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig -from transformers.modeling_utils import no_init_weights +from transformers.models.bloom.parallel_layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, +) from bloom_inference.pb import generate_pb2 -from bloom_inference.prepare_weights import prepare_weights, match_suffix from bloom_inference.utils import ( StoppingCriteria, NextTokenChooser, initialize_torch_distributed, - set_default_dtype, + weight_files, + download_weights ) torch.manual_seed(0) @@ -63,7 +68,7 @@ def from_pb( ) stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) - input_ids = tokenizer(inputs, return_tensors="pt", padding=True).to(device) + input_ids = tokenizer(inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8).to(device) all_input_ids = input_ids["input_ids"].unsqueeze(-1) return cls( @@ -369,7 +374,7 @@ def generate_token( class BLOOMSharded(BLOOM): - def __init__(self, model_name: str, shard_directory: Path): + def __init__(self, model_name: str): super(BLOOM, self).__init__() self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 @@ -382,30 +387,11 @@ def __init__(self, model_name: str, shard_directory: Path): self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") - # shard state_dict - if self.master: - # TODO @thomasw21 do some caching - shard_state_dict_paths = prepare_weights( - model_name, - shard_directory / "cache", - shard_directory, - tp_world_size=self.world_size, - ) - shard_state_dict_paths = [ - str(path.absolute()) for path in shard_state_dict_paths - ] - else: - shard_state_dict_paths = [None] * self.world_size - - torch.distributed.broadcast_object_list( - shard_state_dict_paths, src=0, group=self.process_group - ) - shard_state_dict_path = shard_state_dict_paths[self.rank] - config = AutoConfig.from_pretrained( model_name, slow_but_exact=False, tp_parallel=True ) config.pad_token_id = 3 + self.num_heads = config.n_head // self.process_group.size() # The flag below controls whether to allow TF32 on matmul. This flag defaults to False # in PyTorch 1.12 and later. @@ -414,37 +400,88 @@ def __init__(self, model_name: str, shard_directory: Path): # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. torch.backends.cudnn.allow_tf32 = True - with set_default_dtype(dtype): - with no_init_weights(): - # we can probably set the device to `meta` here? - model = AutoModelForCausalLM.from_config(config).to(dtype) + # Only download weights for small models + if self.master and model_name == "bigscience/bloom-560m": + download_weights(model_name) torch.distributed.barrier(group=self.process_group) - # print_rank_0(f"Initialized model") - state_dict = torch.load(shard_state_dict_path) - # TODO @thomasw21: HACK in order to transpose all weight prior - for key in state_dict.keys(): - do_transpose = False - if not match_suffix(key, "weight"): - continue - - for potential_suffix in [ - "self_attention.query_key_value.weight", - "self_attention.dense.weight", - "dense_h_to_4h.weight", - "dense_4h_to_h.weight", - ]: - if match_suffix(key, potential_suffix): - do_transpose = True - - if do_transpose: - state_dict[key] = state_dict[key].transpose(1, 0).contiguous() - - model.load_state_dict(state_dict, strict=False) - model.tie_weights() - self.model = model.to(self.device).eval() - self.num_heads = config.n_head // self.process_group.size() + filenames = weight_files(model_name) + + with init_empty_weights(): + model = AutoModelForCausalLM.from_config(config) + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + device=self.device, + rank=self.rank, + world_size=self.world_size, + ) + self.model = model.eval().to(dtype) + torch.distributed.barrier(group=self.process_group) + + @staticmethod + def load_weights( + model, filenames: List[str], device: torch.device, rank: int, world_size: int + ): + parameters = dict(model.named_parameters()) + for file in filenames: + with safe_open(file, framework="pt", device=str(device)) as f: + for name in f.keys(): + full_name = f"transformer.{name}" + + module_name, param_name = full_name.rsplit(".", 1) + module = model.get_submodule(module_name) + current_tensor = parameters[full_name] + + slice_ = f.get_slice(name) + + if isinstance(module, TensorParallelColumnLinear): + if param_name == "weight": + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + tensor = tensor.transpose(1, 0) + else: + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif isinstance(module, TensorParallelRowLinear): + if param_name == "weight": + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + tensor = tensor.transpose(1, 0) + else: + tensor = slice_[:] + # XXX: Hack for Rowlinear to add the bias only once. + if rank != 0: + tensor = torch.zeros_like(tensor) + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + else: + tensor = slice_[:] + + if current_tensor.shape != tensor.shape: + raise ValueError( + f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" + ) + + tensor = tensor.contiguous() + module._parameters[param_name] = tensor + if name == "word_embeddings.weight": + model.lm_head._parameters["weight"] = tensor def forward(self, input_ids, attention_mask, past_key_values: Optional = None): outputs = self.model.forward( diff --git a/server/bloom_inference/prepare_weights.py b/server/bloom_inference/prepare_weights.py deleted file mode 100644 index 5594998d..00000000 --- a/server/bloom_inference/prepare_weights.py +++ /dev/null @@ -1,185 +0,0 @@ -import torch -import os -import tempfile -import json - -from typing import BinaryIO -from joblib import Parallel, delayed -from functools import partial -from pathlib import Path -from tqdm import tqdm - -from huggingface_hub import hf_hub_url -from huggingface_hub.file_download import _request_wrapper, hf_raise_for_status - - -def match_suffix(text, suffix): - return text[-len(suffix) :] == suffix - - -def http_get( - url: str, - temp_file: BinaryIO, - *, - timeout=10.0, - max_retries=0, -): - """ - Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub. - """ - r = _request_wrapper( - method="GET", - url=url, - stream=True, - timeout=timeout, - max_retries=max_retries, - ) - hf_raise_for_status(r) - for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks - temp_file.write(chunk) - - -def cache_download_url(url: str, root_dir: Path): - filename = root_dir / url.split("/")[-1] - - if not filename.exists(): - temp_file_manager = partial( - tempfile.NamedTemporaryFile, mode="wb", dir=root_dir, delete=False - ) - with temp_file_manager() as temp_file: - http_get(url, temp_file) - - os.replace(temp_file.name, filename) - return filename - - -def prepare_weights( - model_name: str, cache_path: Path, save_path: Path, tp_world_size: int -): - save_paths = [ - save_path / f"{model_name}_tp-rank-{tp_rank}-of-{tp_world_size}.pty" - for tp_rank in range(tp_world_size) - ] - - if all(save_path.exists() for save_path in save_paths): - print("Weights are already prepared") - return save_paths - - cache_path.mkdir(parents=True, exist_ok=True) - if model_name == "bigscience/bloom-560m": - url = hf_hub_url(model_name, filename="pytorch_model.bin") - cache_download_url(url, cache_path) - - elif model_name == "bigscience/bloom": - url = hf_hub_url(model_name, filename="pytorch_model.bin.index.json") - index_path = cache_download_url(url, cache_path) - with index_path.open("r") as f: - index = json.load(f) - - # Get unique file names - weight_files = list( - set([filename for filename in index["weight_map"].values()]) - ) - urls = [hf_hub_url(model_name, filename=filename) for filename in weight_files] - - Parallel(n_jobs=5)( - delayed(cache_download_url)(url, cache_path) for url in tqdm(urls) - ) - else: - raise ValueError(f"Unknown model name: {model_name}") - - shards_state_dicts = [{} for _ in range(tp_world_size)] - - for weight_path in tqdm(Path(cache_path).glob("*.bin")): - state_dict = torch.load(weight_path, map_location="cpu") - - keys = list(state_dict.keys()) - for state_name in keys: - state = state_dict[state_name] - if any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.query_key_value.weight", - "self_attention.query_key_value.bias", - "mlp.dense_h_to_4h.weight", - "mlp.dense_h_to_4h.bias", - "word_embeddings.weight", - ] - ): - output_size = state.shape[0] - assert output_size % tp_world_size == 0 - block_size = output_size // tp_world_size - sharded_weights = torch.split(state, block_size, dim=0) - assert len(sharded_weights) == tp_world_size - - for tp_rank, shard in enumerate(sharded_weights): - shards_state_dicts[tp_rank][ - "transformer." + state_name - ] = shard.detach().clone() - - elif match_suffix(state_name, "lm_head.weight"): - output_size = state.shape[0] - assert output_size % tp_world_size == 0 - block_size = output_size // tp_world_size - sharded_weights = torch.split(state, block_size, dim=0) - assert len(sharded_weights) == tp_world_size - - for tp_rank, shard in enumerate(sharded_weights): - shards_state_dicts[tp_rank][state_name] = shard.detach().clone() - - elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.weight", - "mlp.dense_4h_to_h.weight", - ] - ): - input_size = state.shape[1] - assert input_size % tp_world_size == 0 - block_size = input_size // tp_world_size - sharded_weights = torch.split(state, block_size, dim=1) - assert len(sharded_weights) == tp_world_size - for tp_rank, shard in enumerate(sharded_weights): - shards_state_dicts[tp_rank][ - "transformer." + state_name - ] = shard.detach().clone() - - elif any( - match_suffix(state_name, candidate) - for candidate in [ - "self_attention.dense.bias", - "mlp.dense_4h_to_h.bias", - ] - ): - shards_state_dicts[0][ - "transformer." + state_name - ] = state.detach().clone() - for tp_rank in range(1, tp_world_size): - shards_state_dicts[tp_rank][ - "transformer." + state_name - ] = torch.zeros_like(state) - - else: - # We duplicate parameters across tp ranks - for tp_rank in range(tp_world_size): - shards_state_dicts[tp_rank][ - "transformer." + state_name - ] = state.detach().clone() - - del state_dict[state_name] # delete key from state_dict - del state # delete tensor - del state_dict - - # we save state_dict - for tp_rank, (save_path, shard_state_dict) in enumerate( - zip(save_paths, shards_state_dicts) - ): - save_paths.append(save_path) - save_path.parent.mkdir(parents=True, exist_ok=True) - if save_path.exists(): - print(f"Skipping {save_path} as it already exists") - else: - torch.save(shard_state_dict, save_path) - - return save_paths diff --git a/server/bloom_inference/server.py b/server/bloom_inference/server.py index 734aba43..fb02db6d 100644 --- a/server/bloom_inference/server.py +++ b/server/bloom_inference/server.py @@ -69,18 +69,14 @@ def serve( model_name: str, sharded: bool, uds_path: Path, - shard_directory: Optional[Path] = None, ): async def serve_inner( model_name: str, sharded: bool = False, - shard_directory: Optional[Path] = None, ): unix_socket_template = "unix://{}-{}" if sharded: - if shard_directory is None: - raise ValueError("shard_directory must be set when sharded is True") - model = BLOOMSharded(model_name, shard_directory) + model = BLOOMSharded(model_name) server_urls = [ unix_socket_template.format(uds_path, rank) for rank in range(model.world_size) @@ -109,4 +105,4 @@ async def serve_inner( print("Signal received. Shutting down") await server.stop(0) - asyncio.run(serve_inner(model_name, sharded, shard_directory)) + asyncio.run(serve_inner(model_name, sharded)) diff --git a/server/bloom_inference/utils.py b/server/bloom_inference/utils.py index c351806a..cca5403f 100644 --- a/server/bloom_inference/utils.py +++ b/server/bloom_inference/utils.py @@ -1,9 +1,14 @@ import os -import contextlib import torch import torch.distributed from datetime import timedelta + +from functools import partial +from joblib import Parallel, delayed +from huggingface_hub import HfApi, hf_hub_download, try_to_load_from_cache +from huggingface_hub.utils import LocalEntryNotFoundError +from tqdm import tqdm from transformers.generation_logits_process import ( LogitsProcessorList, TemperatureLogitsWarper, @@ -87,11 +92,42 @@ def initialize_torch_distributed(): return torch.distributed.distributed_c10d._get_default_group(), rank, world_size -@contextlib.contextmanager -def set_default_dtype(dtype): - saved_dtype = torch.get_default_dtype() - torch.set_default_dtype(dtype) - try: - yield - finally: - torch.set_default_dtype(saved_dtype) +def weight_hub_files(model_name): + """Get the safetensors filenames on the hub""" + api = HfApi() + info = api.model_info(model_name) + filenames = [ + s.rfilename for s in info.siblings if s.rfilename.endswith(".safetensors") + ] + return filenames + + +def weight_files(model_name): + """Get the local safetensors filenames""" + filenames = weight_hub_files(model_name) + files = [] + for filename in filenames: + cache_file = try_to_load_from_cache(model_name, filename=filename) + if cache_file is None: + raise LocalEntryNotFoundError( + f"File {filename} of model {model_name} not found in " + f"{os.getenv('HUGGINGFACE_HUB_CACHE', 'the local cache')}. " + f"Please run `bloom-inference-server download-weights {model_name}` first." + ) + files.append(cache_file) + + return files + + +def download_weights(model_name): + """Download the safetensors files from the hub""" + filenames = weight_hub_files(model_name) + + download_function = partial( + hf_hub_download, repo_id=model_name, local_files_only=False + ) + # FIXME: fix the overlapping progress bars + files = Parallel(n_jobs=5)( + delayed(download_function)(filename=filename) for filename in tqdm(filenames) + ) + return files diff --git a/server/poetry.lock b/server/poetry.lock index 3feee60d..38f14d35 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -43,7 +43,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "grpcio" -version = "1.49.1" +version = "1.50.0" description = "HTTP/2-based RPC framework" category = "main" optional = false @@ -53,31 +53,31 @@ python-versions = ">=3.7" six = ">=1.5.2" [package.extras] -protobuf = ["grpcio-tools (>=1.49.1)"] +protobuf = ["grpcio-tools (>=1.50.0)"] [[package]] name = "grpcio-reflection" -version = "1.49.1" +version = "1.50.0" description = "Standard Protobuf Reflection Service for gRPC" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -grpcio = ">=1.49.1" -protobuf = ">=4.21.3" +grpcio = ">=1.50.0" +protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.49.1" +version = "1.50.0" description = "Protobuf code generator for gRPC" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -grpcio = ">=1.49.1" -protobuf = ">=4.21.3,<5.0dev" +grpcio = ">=1.50.0" +protobuf = ">=4.21.6,<5.0dev" setuptools = "*" [[package]] @@ -90,7 +90,7 @@ python-versions = ">=3.7" [[package]] name = "numpy" -version = "1.23.3" +version = "1.23.4" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = false @@ -109,7 +109,7 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "protobuf" -version = "4.21.7" +version = "4.21.8" description = "" category = "main" optional = false @@ -117,7 +117,7 @@ python-versions = ">=3.7" [[package]] name = "psutil" -version = "5.9.2" +version = "5.9.3" description = "Cross-platform lib for process and system monitoring in Python." category = "main" optional = false @@ -147,7 +147,7 @@ python-versions = ">=3.6" [[package]] name = "setuptools" -version = "65.4.1" +version = "65.5.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false @@ -196,7 +196,7 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6. [[package]] name = "typing-extensions" -version = "4.3.0" +version = "4.4.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -221,190 +221,194 @@ colorama = [ {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] grpcio = [ - {file = "grpcio-1.49.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:fd86040232e805b8e6378b2348c928490ee595b058ce9aaa27ed8e4b0f172b20"}, - {file = "grpcio-1.49.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6fd0c9cede9552bf00f8c5791d257d5bf3790d7057b26c59df08be5e7a1e021d"}, - {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:d0d402e158d4e84e49c158cb5204119d55e1baf363ee98d6cb5dce321c3a065d"}, - {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:822ceec743d42a627e64ea266059a62d214c5a3cdfcd0d7fe2b7a8e4e82527c7"}, - {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2106d9c16527f0a85e2eea6e6b91a74fc99579c60dd810d8690843ea02bc0f5f"}, - {file = "grpcio-1.49.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:52dd02b7e7868233c571b49bc38ebd347c3bb1ff8907bb0cb74cb5f00c790afc"}, - {file = "grpcio-1.49.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:120fecba2ec5d14b5a15d11063b39783fda8dc8d24addd83196acb6582cabd9b"}, - {file = "grpcio-1.49.1-cp310-cp310-win32.whl", hash = "sha256:f1a3b88e3c53c1a6e6bed635ec1bbb92201bb6a1f2db186179f7f3f244829788"}, - {file = "grpcio-1.49.1-cp310-cp310-win_amd64.whl", hash = "sha256:a7d0017b92d3850abea87c1bdec6ea41104e71c77bca44c3e17f175c6700af62"}, - {file = "grpcio-1.49.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:9fb17ff8c0d56099ac6ebfa84f670c5a62228d6b5c695cf21c02160c2ac1446b"}, - {file = "grpcio-1.49.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:075f2d06e3db6b48a2157a1bcd52d6cbdca980dd18988fe6afdb41795d51625f"}, - {file = "grpcio-1.49.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46d93a1b4572b461a227f1db6b8d35a88952db1c47e5fadcf8b8a2f0e1dd9201"}, - {file = "grpcio-1.49.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc79b2b37d779ac42341ddef40ad5bf0966a64af412c89fc2b062e3ddabb093f"}, - {file = "grpcio-1.49.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5f8b3a971c7820ea9878f3fd70086240a36aeee15d1b7e9ecbc2743b0e785568"}, - {file = "grpcio-1.49.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49b301740cf5bc8fed4fee4c877570189ae3951432d79fa8e524b09353659811"}, - {file = "grpcio-1.49.1-cp311-cp311-win32.whl", hash = "sha256:1c66a25afc6c71d357867b341da594a5587db5849b48f4b7d5908d236bb62ede"}, - {file = "grpcio-1.49.1-cp311-cp311-win_amd64.whl", hash = "sha256:6b6c3a95d27846f4145d6967899b3ab25fffc6ae99544415e1adcacef84842d2"}, - {file = "grpcio-1.49.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:1cc400c8a2173d1c042997d98a9563e12d9bb3fb6ad36b7f355bc77c7663b8af"}, - {file = "grpcio-1.49.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:34f736bd4d0deae90015c0e383885b431444fe6b6c591dea288173df20603146"}, - {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:196082b9c89ebf0961dcd77cb114bed8171964c8e3063b9da2fb33536a6938ed"}, - {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c9f89c42749890618cd3c2464e1fbf88446e3d2f67f1e334c8e5db2f3272bbd"}, - {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64419cb8a5b612cdb1550c2fd4acbb7d4fb263556cf4625f25522337e461509e"}, - {file = "grpcio-1.49.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8a5272061826e6164f96e3255405ef6f73b88fd3e8bef464c7d061af8585ac62"}, - {file = "grpcio-1.49.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ea9d0172445241ad7cb49577314e39d0af2c5267395b3561d7ced5d70458a9f3"}, - {file = "grpcio-1.49.1-cp37-cp37m-win32.whl", hash = "sha256:2070e87d95991473244c72d96d13596c751cb35558e11f5df5414981e7ed2492"}, - {file = "grpcio-1.49.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fcedcab49baaa9db4a2d240ac81f2d57eb0052b1c6a9501b46b8ae912720fbf"}, - {file = "grpcio-1.49.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:afbb3475cf7f4f7d380c2ca37ee826e51974f3e2665613996a91d6a58583a534"}, - {file = "grpcio-1.49.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:a4f9ba141380abde6c3adc1727f21529137a2552002243fa87c41a07e528245c"}, - {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:cf0a1fb18a7204b9c44623dfbd1465b363236ce70c7a4ed30402f9f60d8b743b"}, - {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17bb6fe72784b630728c6cff9c9d10ccc3b6d04e85da6e0a7b27fb1d135fac62"}, - {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18305d5a082d1593b005a895c10041f833b16788e88b02bb81061f5ebcc465df"}, - {file = "grpcio-1.49.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b6a1b39e59ac5a3067794a0e498911cf2e37e4b19ee9e9977dc5e7051714f13f"}, - {file = "grpcio-1.49.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e20d59aafc086b1cc68400463bddda6e41d3e5ed30851d1e2e0f6a2e7e342d3"}, - {file = "grpcio-1.49.1-cp38-cp38-win32.whl", hash = "sha256:e1e83233d4680863a421f3ee4a7a9b80d33cd27ee9ed7593bc93f6128302d3f2"}, - {file = "grpcio-1.49.1-cp38-cp38-win_amd64.whl", hash = "sha256:221d42c654d2a41fa31323216279c73ed17d92f533bc140a3390cc1bd78bf63c"}, - {file = "grpcio-1.49.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:fa9e6e61391e99708ac87fc3436f6b7b9c6b845dc4639b406e5e61901e1aacde"}, - {file = "grpcio-1.49.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9b449e966ef518ce9c860d21f8afe0b0f055220d95bc710301752ac1db96dd6a"}, - {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:aa34d2ad9f24e47fa9a3172801c676e4037d862247e39030165fe83821a7aafd"}, - {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5207f4eed1b775d264fcfe379d8541e1c43b878f2b63c0698f8f5c56c40f3d68"}, - {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b24a74651438d45619ac67004638856f76cc13d78b7478f2457754cbcb1c8ad"}, - {file = "grpcio-1.49.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:fe763781669790dc8b9618e7e677c839c87eae6cf28b655ee1fa69ae04eea03f"}, - {file = "grpcio-1.49.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f2ff7ba0f8f431f32d4b4bc3a3713426949d3533b08466c4ff1b2b475932ca8"}, - {file = "grpcio-1.49.1-cp39-cp39-win32.whl", hash = "sha256:08ff74aec8ff457a89b97152d36cb811dcc1d17cd5a92a65933524e363327394"}, - {file = "grpcio-1.49.1-cp39-cp39-win_amd64.whl", hash = "sha256:274ffbb39717918c514b35176510ae9be06e1d93121e84d50b350861dcb9a705"}, - {file = "grpcio-1.49.1.tar.gz", hash = "sha256:d4725fc9ec8e8822906ae26bb26f5546891aa7fbc3443de970cc556d43a5c99f"}, + {file = "grpcio-1.50.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:906f4d1beb83b3496be91684c47a5d870ee628715227d5d7c54b04a8de802974"}, + {file = "grpcio-1.50.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:2d9fd6e38b16c4d286a01e1776fdf6c7a4123d99ae8d6b3f0b4a03a34bf6ce45"}, + {file = "grpcio-1.50.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:4b123fbb7a777a2fedec684ca0b723d85e1d2379b6032a9a9b7851829ed3ca9a"}, + {file = "grpcio-1.50.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2f77a90ba7b85bfb31329f8eab9d9540da2cf8a302128fb1241d7ea239a5469"}, + {file = "grpcio-1.50.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eea18a878cffc804506d39c6682d71f6b42ec1c151d21865a95fae743fda500"}, + {file = "grpcio-1.50.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b71916fa8f9eb2abd93151fafe12e18cebb302686b924bd4ec39266211da525"}, + {file = "grpcio-1.50.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:95ce51f7a09491fb3da8cf3935005bff19983b77c4e9437ef77235d787b06842"}, + {file = "grpcio-1.50.0-cp310-cp310-win32.whl", hash = "sha256:f7025930039a011ed7d7e7ef95a1cb5f516e23c5a6ecc7947259b67bea8e06ca"}, + {file = "grpcio-1.50.0-cp310-cp310-win_amd64.whl", hash = "sha256:05f7c248e440f538aaad13eee78ef35f0541e73498dd6f832fe284542ac4b298"}, + {file = "grpcio-1.50.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:ca8a2254ab88482936ce941485c1c20cdeaef0efa71a61dbad171ab6758ec998"}, + {file = "grpcio-1.50.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:3b611b3de3dfd2c47549ca01abfa9bbb95937eb0ea546ea1d762a335739887be"}, + {file = "grpcio-1.50.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a4cd8cb09d1bc70b3ea37802be484c5ae5a576108bad14728f2516279165dd7"}, + {file = "grpcio-1.50.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:156f8009e36780fab48c979c5605eda646065d4695deea4cfcbcfdd06627ddb6"}, + {file = "grpcio-1.50.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de411d2b030134b642c092e986d21aefb9d26a28bf5a18c47dd08ded411a3bc5"}, + {file = "grpcio-1.50.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d144ad10eeca4c1d1ce930faa105899f86f5d99cecfe0d7224f3c4c76265c15e"}, + {file = "grpcio-1.50.0-cp311-cp311-win32.whl", hash = "sha256:92d7635d1059d40d2ec29c8bf5ec58900120b3ce5150ef7414119430a4b2dd5c"}, + {file = "grpcio-1.50.0-cp311-cp311-win_amd64.whl", hash = "sha256:ce8513aee0af9c159319692bfbf488b718d1793d764798c3d5cff827a09e25ef"}, + {file = "grpcio-1.50.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:8e8999a097ad89b30d584c034929f7c0be280cd7851ac23e9067111167dcbf55"}, + {file = "grpcio-1.50.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:a50a1be449b9e238b9bd43d3857d40edf65df9416dea988929891d92a9f8a778"}, + {file = "grpcio-1.50.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:cf151f97f5f381163912e8952eb5b3afe89dec9ed723d1561d59cabf1e219a35"}, + {file = "grpcio-1.50.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a23d47f2fc7111869f0ff547f771733661ff2818562b04b9ed674fa208e261f4"}, + {file = "grpcio-1.50.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84d04dec64cc4ed726d07c5d17b73c343c8ddcd6b59c7199c801d6bbb9d9ed1"}, + {file = "grpcio-1.50.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:67dd41a31f6fc5c7db097a5c14a3fa588af54736ffc174af4411d34c4f306f68"}, + {file = "grpcio-1.50.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8d4c8e73bf20fb53fe5a7318e768b9734cf122fe671fcce75654b98ba12dfb75"}, + {file = "grpcio-1.50.0-cp37-cp37m-win32.whl", hash = "sha256:7489dbb901f4fdf7aec8d3753eadd40839c9085967737606d2c35b43074eea24"}, + {file = "grpcio-1.50.0-cp37-cp37m-win_amd64.whl", hash = "sha256:531f8b46f3d3db91d9ef285191825d108090856b3bc86a75b7c3930f16ce432f"}, + {file = "grpcio-1.50.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:d534d169673dd5e6e12fb57cc67664c2641361e1a0885545495e65a7b761b0f4"}, + {file = "grpcio-1.50.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:1d8d02dbb616c0a9260ce587eb751c9c7dc689bc39efa6a88cc4fa3e9c138a7b"}, + {file = "grpcio-1.50.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:baab51dcc4f2aecabf4ed1e2f57bceab240987c8b03533f1cef90890e6502067"}, + {file = "grpcio-1.50.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40838061e24f960b853d7bce85086c8e1b81c6342b1f4c47ff0edd44bbae2722"}, + {file = "grpcio-1.50.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:931e746d0f75b2a5cff0a1197d21827a3a2f400c06bace036762110f19d3d507"}, + {file = "grpcio-1.50.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:15f9e6d7f564e8f0776770e6ef32dac172c6f9960c478616c366862933fa08b4"}, + {file = "grpcio-1.50.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a4c23e54f58e016761b576976da6a34d876420b993f45f66a2bfb00363ecc1f9"}, + {file = "grpcio-1.50.0-cp38-cp38-win32.whl", hash = "sha256:3e4244c09cc1b65c286d709658c061f12c61c814be0b7030a2d9966ff02611e0"}, + {file = "grpcio-1.50.0-cp38-cp38-win_amd64.whl", hash = "sha256:8e69aa4e9b7f065f01d3fdcecbe0397895a772d99954bb82eefbb1682d274518"}, + {file = "grpcio-1.50.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:af98d49e56605a2912cf330b4627e5286243242706c3a9fa0bcec6e6f68646fc"}, + {file = "grpcio-1.50.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:080b66253f29e1646ac53ef288c12944b131a2829488ac3bac8f52abb4413c0d"}, + {file = "grpcio-1.50.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:ab5d0e3590f0a16cb88de4a3fa78d10eb66a84ca80901eb2c17c1d2c308c230f"}, + {file = "grpcio-1.50.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb11464f480e6103c59d558a3875bd84eed6723f0921290325ebe97262ae1347"}, + {file = "grpcio-1.50.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e07fe0d7ae395897981d16be61f0db9791f482f03fee7d1851fe20ddb4f69c03"}, + {file = "grpcio-1.50.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d75061367a69808ab2e84c960e9dce54749bcc1e44ad3f85deee3a6c75b4ede9"}, + {file = "grpcio-1.50.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ae23daa7eda93c1c49a9ecc316e027ceb99adbad750fbd3a56fa9e4a2ffd5ae0"}, + {file = "grpcio-1.50.0-cp39-cp39-win32.whl", hash = "sha256:177afaa7dba3ab5bfc211a71b90da1b887d441df33732e94e26860b3321434d9"}, + {file = "grpcio-1.50.0-cp39-cp39-win_amd64.whl", hash = "sha256:ea8ccf95e4c7e20419b7827aa5b6da6f02720270686ac63bd3493a651830235c"}, + {file = "grpcio-1.50.0.tar.gz", hash = "sha256:12b479839a5e753580b5e6053571de14006157f2ef9b71f38c56dc9b23b95ad6"}, ] grpcio-reflection = [ - {file = "grpcio-reflection-1.49.1.tar.gz", hash = "sha256:b755dfe61d5255a02fb8d0d845bd0027847dee68bf0763a2b286d664ed07ec4d"}, - {file = "grpcio_reflection-1.49.1-py3-none-any.whl", hash = "sha256:70a325a83c1c1ab583d368711e5733cbef5e068ad2c17cbe77df6e47e0311d1f"}, + {file = "grpcio-reflection-1.50.0.tar.gz", hash = "sha256:d75ff6a12f70f28ba6465d5a29781cf2d9c740eb809711990f61699019af1f72"}, + {file = "grpcio_reflection-1.50.0-py3-none-any.whl", hash = "sha256:92fea4a9527a8bcc52b615b6a7af655a5a118ab0f57227cc814dc2192f8a7455"}, ] grpcio-tools = [ - {file = "grpcio-tools-1.49.1.tar.gz", hash = "sha256:84cc64e5b46bad43d5d7bd2fd772b656eba0366961187a847e908e2cb735db91"}, - {file = "grpcio_tools-1.49.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:2dfb6c7ece84d46bd690b23d3e060d18115c8bc5047d2e8a33e6747ed323a348"}, - {file = "grpcio_tools-1.49.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:8f452a107c054a04db2570f7851a07f060313c6e841b0d394ce6030d598290e6"}, - {file = "grpcio_tools-1.49.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:6a198871b582287213c4d70792bf275e1d7cf34eed1d019f534ddf4cd15ab039"}, - {file = "grpcio_tools-1.49.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0cca67a7d0287bdc855d81fdd38dc949c4273273a74f832f9e520abe4f20bc6"}, - {file = "grpcio_tools-1.49.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdaff4c89eecb37c247b93025410db68114d97fa093cbb028e9bd7cda5912473"}, - {file = "grpcio_tools-1.49.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bb8773118ad315db317d7b22b5ff75d649ca20931733281209e7cbd8c0fad53e"}, - {file = "grpcio_tools-1.49.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cc5534023735b8a8f56760b7c533918f874ce5a9064d7c5456d2709ae2b31f9"}, - {file = "grpcio_tools-1.49.1-cp310-cp310-win32.whl", hash = "sha256:d277642acbe305f5586f9597b78fb9970d6633eb9f89c61e429c92c296c37129"}, - {file = "grpcio_tools-1.49.1-cp310-cp310-win_amd64.whl", hash = "sha256:eed599cf08fc1a06c72492d3c5750c32f58de3750eddd984af1f257c14326701"}, - {file = "grpcio_tools-1.49.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:9e5c13809ab2f245398e8446c4c3b399a62d591db651e46806cccf52a700452e"}, - {file = "grpcio_tools-1.49.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:ab3d0ee9623720ee585fdf3753b3755d3144a4a8ae35bca8e3655fa2f41056be"}, - {file = "grpcio_tools-1.49.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ba87e3512bc91d78bf9febcfb522eadda171d2d4ddaf886066b0f01aa4929ad"}, - {file = "grpcio_tools-1.49.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e13b3643e7577a3ec13b79689eb4d7548890b1e104c04b9ed6557a3c3dd452"}, - {file = "grpcio_tools-1.49.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:324f67d9cb4b7058b6ce45352fb64c20cc1fa04c34d97ad44772cfe6a4ae0cf5"}, - {file = "grpcio_tools-1.49.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a64bab81b220c50033f584f57978ebbea575f09c1ccee765cd5c462177988098"}, - {file = "grpcio_tools-1.49.1-cp311-cp311-win32.whl", hash = "sha256:f632d376f92f23e5931697a3acf1b38df7eb719774213d93c52e02acd2d529ac"}, - {file = "grpcio_tools-1.49.1-cp311-cp311-win_amd64.whl", hash = "sha256:28ff2b978d9509474928b9c096a0cce4eaa9c8f7046136aee1545f6211ed8126"}, - {file = "grpcio_tools-1.49.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:46afd3cb7e555187451a5d283f108cdef397952a662cb48680afc615b158864a"}, - {file = "grpcio_tools-1.49.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:9284568b728e41fa8f7e9c2e7399545d605f75d8072ef0e9aa2a05655cb679eb"}, - {file = "grpcio_tools-1.49.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:aa34442cf787732cb41f2aa6172007e24f480b8b9d3dc5166de80d63e9072ea4"}, - {file = "grpcio_tools-1.49.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b8c9eb5a4250905414cd53a68caea3eb8f0c515aadb689e6e81b71ebe9ab5c6"}, - {file = "grpcio_tools-1.49.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab15db024051bf21feb21c29cb2c3ea0a2e4f5cf341d46ef76e17fcf6aaef164"}, - {file = "grpcio_tools-1.49.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:502084b622f758bef620a9107c2db9fcdf66d26c7e0e481d6bb87db4dc917d70"}, - {file = "grpcio_tools-1.49.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4085890b77c640085f82bf1e90a0ea166ce48000bc2f5180914b974783c9c0a8"}, - {file = "grpcio_tools-1.49.1-cp37-cp37m-win32.whl", hash = "sha256:da0edb984699769ce02e18e3392d54b59a7a3f93acd285a68043f5bde4fc028e"}, - {file = "grpcio_tools-1.49.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9887cd622770271101a7dd1832845d64744c3f88fd11ccb2620394079197a42e"}, - {file = "grpcio_tools-1.49.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:8440fe7dae6a40c279e3a24b82793735babd38ecbb0d07bb712ff9c8963185d9"}, - {file = "grpcio_tools-1.49.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:b5de2bb7dd6b6231da9b1556ade981513330b740e767f1d902c71ceee0a7d196"}, - {file = "grpcio_tools-1.49.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:1e6f06a763aea7836b63d9c117347f2bf7038008ceef72758815c9e09c5fb1fc"}, - {file = "grpcio_tools-1.49.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e31562f90120318c5395aabec0f2f69ad8c14b6676996b7730d9d2eaf9415d57"}, - {file = "grpcio_tools-1.49.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49ef9a4e389a618157a9daa9fafdfeeaef1ece9adda7f50f85db928f24d4b3e8"}, - {file = "grpcio_tools-1.49.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b384cb8e8d9bcb55ee8f9b064374561c7a1a05d848249581403d36fc7060032f"}, - {file = "grpcio_tools-1.49.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73732f77943ac3e898879cbb29c27253aa3c47566b8a59780fd24c6a54de1b66"}, - {file = "grpcio_tools-1.49.1-cp38-cp38-win32.whl", hash = "sha256:b594b2745a5ba9e7a76ce561bc5ab40bc65bb44743c505529b1e4f12af29104d"}, - {file = "grpcio_tools-1.49.1-cp38-cp38-win_amd64.whl", hash = "sha256:680fbc88f8709ddcabb88f86749f2d8e429160890cff2c70680880a6970d4eef"}, - {file = "grpcio_tools-1.49.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:e8c3869121860f6767eedb7d24fc54dfd71e737fdfbb26e1334684606f3274fd"}, - {file = "grpcio_tools-1.49.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:73e9d7c886ba10e20c97d1dab0ff961ba5800757ae5e31be21b1cda8130c52f8"}, - {file = "grpcio_tools-1.49.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:1760de2dd2c4f08de87b039043a4797f3c17193656e7e3eb84e92f0517083c0c"}, - {file = "grpcio_tools-1.49.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd4b1e216dd04d9245ee8f4e601a1f98c25e6e417ea5cf8d825c50589a8b447e"}, - {file = "grpcio_tools-1.49.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1c28751ab5955cae563d07677e799233f0fe1c0fc49d9cbd61ff1957e83617f"}, - {file = "grpcio_tools-1.49.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c24239c3ee9ed16314c14b4e24437b5079ebc344f343f33629a582f8699f583b"}, - {file = "grpcio_tools-1.49.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:892d3dacf1942820f0b7a868a30e6fbcdf5bec08543b682c7274b0101cee632d"}, - {file = "grpcio_tools-1.49.1-cp39-cp39-win32.whl", hash = "sha256:704d21509ec06efc9d034dbe70e7152715aac004941f4f0f553cf3a0aff15bd5"}, - {file = "grpcio_tools-1.49.1-cp39-cp39-win_amd64.whl", hash = "sha256:1efa0c221c719433f441ac0e026fc3c4dbc9a1a08a552ecdc707775e2f2fbbae"}, + {file = "grpcio-tools-1.50.0.tar.gz", hash = "sha256:88b75f2afd889c7c6939f58d76b58ab84de4723c7de882a1f8448af6632e256f"}, + {file = "grpcio_tools-1.50.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:fad4f44220847691605e4079ec67b6663064d5b1283bee5a89dd6f7726672d08"}, + {file = "grpcio_tools-1.50.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:4fe1cd201f0e0f601551c20bea2a8c1e109767ce341ac263e88d5a0acd0a124c"}, + {file = "grpcio_tools-1.50.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:66113bc60db8ccb15e2d7b79efd5757bba33b15e653f20f473c776857bd4415d"}, + {file = "grpcio_tools-1.50.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:71e98662f76d3bcf031f36f727ffee594c57006d5dce7b6d031d030d194f054d"}, + {file = "grpcio_tools-1.50.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1891aca6042fc403c1d492d03ce1c4bac0053c26132063b7b0171665a0aa6086"}, + {file = "grpcio_tools-1.50.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:54528cec27238761055b28a34ea29392c8e90ac8d8966ee3ccfc9f906eb26fd3"}, + {file = "grpcio_tools-1.50.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:89010e8e0d158966af478d9b345d29d98e52d62464cb93ef192125e94fb904c1"}, + {file = "grpcio_tools-1.50.0-cp310-cp310-win32.whl", hash = "sha256:e42ac5aa09debbf7b72444af7f390713e85b8808557e920d9d8071f14127c0eb"}, + {file = "grpcio_tools-1.50.0-cp310-cp310-win_amd64.whl", hash = "sha256:c876dbaa6897e45ed074df1562006ac9fdeb5c7235f623cda2c29894fe4f8fba"}, + {file = "grpcio_tools-1.50.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:3995d96635d6decaf9d5d2e071677782b08ea3186d7cb8ad93f6c2dd97ec9097"}, + {file = "grpcio_tools-1.50.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1736defd9fc54f942f500dd71555816487e52f07d25b0a58ff4089c375f0b5c3"}, + {file = "grpcio_tools-1.50.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fecbc5e773c1c1971641227d8fcd5de97533dbbc0bd7854b436ceb57243f4b2a"}, + {file = "grpcio_tools-1.50.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8178a475de70e6ae0a7e3f6d7ec1415c9dd10680654dbea9a1caf79665b0ed4"}, + {file = "grpcio_tools-1.50.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d9b8b59a76f67270ce8a4fd7af005732356d5b4ebaffa4598343ea63677bad63"}, + {file = "grpcio_tools-1.50.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ead51910d83123648b1e36e2de8483b66e9ba358ca1d8965005ba7713cbcb7db"}, + {file = "grpcio_tools-1.50.0-cp311-cp311-win32.whl", hash = "sha256:4c928610ceda7890cf7ddd3cb4cc16fb81995063bf4fdb5171bba572d508079b"}, + {file = "grpcio_tools-1.50.0-cp311-cp311-win_amd64.whl", hash = "sha256:56c28f50b9c88cbfc62861de86a1f8b2454d3b48cff85a73d523ec22f130635d"}, + {file = "grpcio_tools-1.50.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:675634367846fc906e8b6016830b3d361f9e788bce7820b4e6432d914cb0e25d"}, + {file = "grpcio_tools-1.50.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:f6dbd9d86aa2fbeb8a104ae8edd23c55f182829db58cae3f28842024e08b1071"}, + {file = "grpcio_tools-1.50.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:2869fba92670d5be3730f1ad108cbacd3f9a7b7afa57187a3c39379513b3fde7"}, + {file = "grpcio_tools-1.50.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f211a8a6f159b496097cabbf4bdbb6d01db0fd7a8d516dd0e577ee2ddccded0"}, + {file = "grpcio_tools-1.50.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27f734e44631d755481a179ffd5f2659eaf1e9755cf3539e291d7a28d8373cc8"}, + {file = "grpcio_tools-1.50.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9bc97b52d577406c240b0d6acf68ab5100c67c903792f1f9ce04d871ca83de53"}, + {file = "grpcio_tools-1.50.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a9816a6c8d1b7adb65217d01624006704faf02b5daa15b6e145b63a5e559d791"}, + {file = "grpcio_tools-1.50.0-cp37-cp37m-win32.whl", hash = "sha256:c5dde47bae615554349dd09185eaaeffd9a28e961de347cf0b3b98a853b43a6c"}, + {file = "grpcio_tools-1.50.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e51133b78b67bd4c06e99bea9c631f29acd52af2b2a254926d192f277f9067b5"}, + {file = "grpcio_tools-1.50.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fec4336e067f2bdd45d74c2466dcd06f245e972d2139187d87cfccf847dbaf50"}, + {file = "grpcio_tools-1.50.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8d74672162474e2592c50708408b4ee145994451ee5171c9b8f580504b987ad3"}, + {file = "grpcio_tools-1.50.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:e6baae6fa9a88c1895193f9610b54d483753ad3e3f91f5d5a10a506afe342a43"}, + {file = "grpcio_tools-1.50.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:927c8cbbfbf3bca46cd05bbabfd8cb6c99f5dbaeb860f2786f9286ce6894ad3d"}, + {file = "grpcio_tools-1.50.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fe40a66d3ccde959f7290b881c78e8c420fd54289fede5c9119d993703e1a49"}, + {file = "grpcio_tools-1.50.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:86a9e9b2cc30797472bd421c75b42fe1062519ef5675243cb33d99de33af279c"}, + {file = "grpcio_tools-1.50.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c1646bfec6d707bcaafea69088770151ba291ede788f2cda5efdc09c1eac5c1c"}, + {file = "grpcio_tools-1.50.0-cp38-cp38-win32.whl", hash = "sha256:dedfe2947ad4ccc8cb2f7d0cac5766dd623901f29e1953a2854126559c317885"}, + {file = "grpcio_tools-1.50.0-cp38-cp38-win_amd64.whl", hash = "sha256:73ba21fe500e2476d391cc92aa05545fdc71bdeedb5b9ec2b22e9a3ccc298a2f"}, + {file = "grpcio_tools-1.50.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:23381ac6a19de0d3cb470f4d1fb450267014234a3923e3c43abb858049f5caf4"}, + {file = "grpcio_tools-1.50.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:c61730ae21533ef992a89df0321ebe59e344f9f0d1d54572b92d9468969c75b3"}, + {file = "grpcio_tools-1.50.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:e5adcbec235dce997efacb44cca5cde24db47ad9c432fcbca875185d2095c55a"}, + {file = "grpcio_tools-1.50.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c40f918d1fe0cb094bfb09ab7567a92f5368fa0b9935235680fe205ddc37518"}, + {file = "grpcio_tools-1.50.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2313c8c73a5003d789fe05f4d01112ccf8330d3d141f5b77b514200d742193f6"}, + {file = "grpcio_tools-1.50.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:578f9137ebca0f152136b47b4895a6b4dabcf17e41f957b03637d1ae096ff67b"}, + {file = "grpcio_tools-1.50.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d8aaff094d94d2119fffe7e0d08ca59d42076971f9d85857e99288e500ab31f0"}, + {file = "grpcio_tools-1.50.0-cp39-cp39-win32.whl", hash = "sha256:e1a8f9a57bbcc2e633aaf327e39830527f3c1f7add18c7580f3058fe9a0fa780"}, + {file = "grpcio_tools-1.50.0-cp39-cp39-win_amd64.whl", hash = "sha256:b7eb7a84d9171c0ae1550833f4a6ca52372bed9db0fa10f8c9dbe6ca65f97a8c"}, ] joblib = [ {file = "joblib-1.2.0-py3-none-any.whl", hash = "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385"}, {file = "joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"}, ] numpy = [ - {file = "numpy-1.23.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c9f707b5bb73bf277d812ded9896f9512a43edff72712f31667d0a8c2f8e71ee"}, - {file = "numpy-1.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ffcf105ecdd9396e05a8e58e81faaaf34d3f9875f137c7372450baa5d77c9a54"}, - {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ea3f98a0ffce3f8f57675eb9119f3f4edb81888b6874bc1953f91e0b1d4f440"}, - {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:004f0efcb2fe1c0bd6ae1fcfc69cc8b6bf2407e0f18be308612007a0762b4089"}, - {file = "numpy-1.23.3-cp310-cp310-win32.whl", hash = "sha256:98dcbc02e39b1658dc4b4508442a560fe3ca5ca0d989f0df062534e5ca3a5c1a"}, - {file = "numpy-1.23.3-cp310-cp310-win_amd64.whl", hash = "sha256:39a664e3d26ea854211867d20ebcc8023257c1800ae89773cbba9f9e97bae036"}, - {file = "numpy-1.23.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1f27b5322ac4067e67c8f9378b41c746d8feac8bdd0e0ffede5324667b8a075c"}, - {file = "numpy-1.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ad3ec9a748a8943e6eb4358201f7e1c12ede35f510b1a2221b70af4bb64295c"}, - {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdc9febce3e68b697d931941b263c59e0c74e8f18861f4064c1f712562903411"}, - {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301c00cf5e60e08e04d842fc47df641d4a181e651c7135c50dc2762ffe293dbd"}, - {file = "numpy-1.23.3-cp311-cp311-win32.whl", hash = "sha256:7cd1328e5bdf0dee621912f5833648e2daca72e3839ec1d6695e91089625f0b4"}, - {file = "numpy-1.23.3-cp311-cp311-win_amd64.whl", hash = "sha256:8355fc10fd33a5a70981a5b8a0de51d10af3688d7a9e4a34fcc8fa0d7467bb7f"}, - {file = "numpy-1.23.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc6e8da415f359b578b00bcfb1d08411c96e9a97f9e6c7adada554a0812a6cc6"}, - {file = "numpy-1.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:22d43376ee0acd547f3149b9ec12eec2f0ca4a6ab2f61753c5b29bb3e795ac4d"}, - {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a64403f634e5ffdcd85e0b12c08f04b3080d3e840aef118721021f9b48fc1460"}, - {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd9d3abe5774404becdb0748178b48a218f1d8c44e0375475732211ea47c67e"}, - {file = "numpy-1.23.3-cp38-cp38-win32.whl", hash = "sha256:f8c02ec3c4c4fcb718fdf89a6c6f709b14949408e8cf2a2be5bfa9c49548fd85"}, - {file = "numpy-1.23.3-cp38-cp38-win_amd64.whl", hash = "sha256:e868b0389c5ccfc092031a861d4e158ea164d8b7fdbb10e3b5689b4fc6498df6"}, - {file = "numpy-1.23.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09f6b7bdffe57fc61d869a22f506049825d707b288039d30f26a0d0d8ea05164"}, - {file = "numpy-1.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8c79d7cf86d049d0c5089231a5bcd31edb03555bd93d81a16870aa98c6cfb79d"}, - {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5d5420053bbb3dd64c30e58f9363d7a9c27444c3648e61460c1237f9ec3fa14"}, - {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5422d6a1ea9b15577a9432e26608c73a78faf0b9039437b075cf322c92e98e7"}, - {file = "numpy-1.23.3-cp39-cp39-win32.whl", hash = "sha256:c1ba66c48b19cc9c2975c0d354f24058888cdc674bebadceb3cdc9ec403fb5d1"}, - {file = "numpy-1.23.3-cp39-cp39-win_amd64.whl", hash = "sha256:78a63d2df1d947bd9d1b11d35564c2f9e4b57898aae4626638056ec1a231c40c"}, - {file = "numpy-1.23.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:17c0e467ade9bda685d5ac7f5fa729d8d3e76b23195471adae2d6a6941bd2c18"}, - {file = "numpy-1.23.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91b8d6768a75247026e951dce3b2aac79dc7e78622fc148329135ba189813584"}, - {file = "numpy-1.23.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:94c15ca4e52671a59219146ff584488907b1f9b3fc232622b47e2cf832e94fb8"}, - {file = "numpy-1.23.3.tar.gz", hash = "sha256:51bf49c0cd1d52be0a240aa66f3458afc4b95d8993d2d04f0d91fa60c10af6cd"}, + {file = "numpy-1.23.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d79ada05005f6f4f337d3bb9de8a7774f259341c70bc88047a1f7b96a4bcb2"}, + {file = "numpy-1.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926db372bc4ac1edf81cfb6c59e2a881606b409ddc0d0920b988174b2e2a767f"}, + {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c237129f0e732885c9a6076a537e974160482eab8f10db6292e92154d4c67d71"}, + {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8365b942f9c1a7d0f0dc974747d99dd0a0cdfc5949a33119caf05cb314682d3"}, + {file = "numpy-1.23.4-cp310-cp310-win32.whl", hash = "sha256:2341f4ab6dba0834b685cce16dad5f9b6606ea8a00e6da154f5dbded70fdc4dd"}, + {file = "numpy-1.23.4-cp310-cp310-win_amd64.whl", hash = "sha256:d331afac87c92373826af83d2b2b435f57b17a5c74e6268b79355b970626e329"}, + {file = "numpy-1.23.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:488a66cb667359534bc70028d653ba1cf307bae88eab5929cd707c761ff037db"}, + {file = "numpy-1.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce03305dd694c4873b9429274fd41fc7eb4e0e4dea07e0af97a933b079a5814f"}, + {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8981d9b5619569899666170c7c9748920f4a5005bf79c72c07d08c8a035757b0"}, + {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a70a7d3ce4c0e9284e92285cba91a4a3f5214d87ee0e95928f3614a256a1488"}, + {file = "numpy-1.23.4-cp311-cp311-win32.whl", hash = "sha256:5e13030f8793e9ee42f9c7d5777465a560eb78fa7e11b1c053427f2ccab90c79"}, + {file = "numpy-1.23.4-cp311-cp311-win_amd64.whl", hash = "sha256:7607b598217745cc40f751da38ffd03512d33ec06f3523fb0b5f82e09f6f676d"}, + {file = "numpy-1.23.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ab46e4e7ec63c8a5e6dbf5c1b9e1c92ba23a7ebecc86c336cb7bf3bd2fb10e5"}, + {file = "numpy-1.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8aae2fb3180940011b4862b2dd3756616841c53db9734b27bb93813cd79fce6"}, + {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c053d7557a8f022ec823196d242464b6955a7e7e5015b719e76003f63f82d0f"}, + {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0882323e0ca4245eb0a3d0a74f88ce581cc33aedcfa396e415e5bba7bf05f68"}, + {file = "numpy-1.23.4-cp38-cp38-win32.whl", hash = "sha256:dada341ebb79619fe00a291185bba370c9803b1e1d7051610e01ed809ef3a4ba"}, + {file = "numpy-1.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:0fe563fc8ed9dc4474cbf70742673fc4391d70f4363f917599a7fa99f042d5a8"}, + {file = "numpy-1.23.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c67b833dbccefe97cdd3f52798d430b9d3430396af7cdb2a0c32954c3ef73894"}, + {file = "numpy-1.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f76025acc8e2114bb664294a07ede0727aa75d63a06d2fae96bf29a81747e4a7"}, + {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12ac457b63ec8ded85d85c1e17d85efd3c2b0967ca39560b307a35a6703a4735"}, + {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95de7dc7dc47a312f6feddd3da2500826defdccbc41608d0031276a24181a2c0"}, + {file = "numpy-1.23.4-cp39-cp39-win32.whl", hash = "sha256:f2f390aa4da44454db40a1f0201401f9036e8d578a25f01a6e237cea238337ef"}, + {file = "numpy-1.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:f260da502d7441a45695199b4e7fd8ca87db659ba1c78f2bbf31f934fe76ae0e"}, + {file = "numpy-1.23.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:61be02e3bf810b60ab74e81d6d0d36246dbfb644a462458bb53b595791251911"}, + {file = "numpy-1.23.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296d17aed51161dbad3c67ed6d164e51fcd18dbcd5dd4f9d0a9c6055dce30810"}, + {file = "numpy-1.23.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4d52914c88b4930dafb6c48ba5115a96cbab40f45740239d9f4159c4ba779962"}, + {file = "numpy-1.23.4.tar.gz", hash = "sha256:ed2cc92af0efad20198638c69bb0fc2870a58dabfba6eb722c933b48556c686c"}, ] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] protobuf = [ - {file = "protobuf-4.21.7-cp310-abi3-win32.whl", hash = "sha256:c7cb105d69a87416bd9023e64324e1c089593e6dae64d2536f06bcbe49cd97d8"}, - {file = "protobuf-4.21.7-cp310-abi3-win_amd64.whl", hash = "sha256:3ec85328a35a16463c6f419dbce3c0fc42b3e904d966f17f48bae39597c7a543"}, - {file = "protobuf-4.21.7-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:db9056b6a11cb5131036d734bcbf91ef3ef9235d6b681b2fc431cbfe5a7f2e56"}, - {file = "protobuf-4.21.7-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:ca200645d6235ce0df3ccfdff1567acbab35c4db222a97357806e015f85b5744"}, - {file = "protobuf-4.21.7-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:b019c79e23a80735cc8a71b95f76a49a262f579d6b84fd20a0b82279f40e2cc1"}, - {file = "protobuf-4.21.7-cp37-cp37m-win32.whl", hash = "sha256:d3f89ccf7182293feba2de2739c8bf34fed1ed7c65a5cf987be00311acac57c1"}, - {file = "protobuf-4.21.7-cp37-cp37m-win_amd64.whl", hash = "sha256:a74d96cd960b87b4b712797c741bb3ea3a913f5c2dc4b6cbe9c0f8360b75297d"}, - {file = "protobuf-4.21.7-cp38-cp38-win32.whl", hash = "sha256:8e09d1916386eca1ef1353767b6efcebc0a6859ed7f73cb7fb974feba3184830"}, - {file = "protobuf-4.21.7-cp38-cp38-win_amd64.whl", hash = "sha256:9e355f2a839d9930d83971b9f562395e13493f0e9211520f8913bd11efa53c02"}, - {file = "protobuf-4.21.7-cp39-cp39-win32.whl", hash = "sha256:f370c0a71712f8965023dd5b13277444d3cdfecc96b2c778b0e19acbfd60df6e"}, - {file = "protobuf-4.21.7-cp39-cp39-win_amd64.whl", hash = "sha256:9643684232b6b340b5e63bb69c9b4904cdd39e4303d498d1a92abddc7e895b7f"}, - {file = "protobuf-4.21.7-py2.py3-none-any.whl", hash = "sha256:8066322588d4b499869bf9f665ebe448e793036b552f68c585a9b28f1e393f66"}, - {file = "protobuf-4.21.7-py3-none-any.whl", hash = "sha256:58b81358ec6c0b5d50df761460ae2db58405c063fd415e1101209221a0a810e1"}, - {file = "protobuf-4.21.7.tar.gz", hash = "sha256:71d9dba03ed3432c878a801e2ea51e034b0ea01cf3a4344fb60166cb5f6c8757"}, + {file = "protobuf-4.21.8-cp310-abi3-win32.whl", hash = "sha256:c252c55ee15175aa1b21b7b9896e6add5162d066d5202e75c39f96136f08cce3"}, + {file = "protobuf-4.21.8-cp310-abi3-win_amd64.whl", hash = "sha256:809ca0b225d3df42655a12f311dd0f4148a943c51f1ad63c38343e457492b689"}, + {file = "protobuf-4.21.8-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bbececaf3cfea9ea65ebb7974e6242d310d2a7772a6f015477e0d79993af4511"}, + {file = "protobuf-4.21.8-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:b02eabb9ebb1a089ed20626a90ad7a69cee6bcd62c227692466054b19c38dd1f"}, + {file = "protobuf-4.21.8-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:4761201b93e024bb70ee3a6a6425d61f3152ca851f403ba946fb0cde88872661"}, + {file = "protobuf-4.21.8-cp37-cp37m-win32.whl", hash = "sha256:f2d55ff22ec300c4d954d3b0d1eeb185681ec8ad4fbecff8a5aee6a1cdd345ba"}, + {file = "protobuf-4.21.8-cp37-cp37m-win_amd64.whl", hash = "sha256:c5f94911dd8feb3cd3786fc90f7565c9aba7ce45d0f254afd625b9628f578c3f"}, + {file = "protobuf-4.21.8-cp38-cp38-win32.whl", hash = "sha256:b37b76efe84d539f16cba55ee0036a11ad91300333abd213849cbbbb284b878e"}, + {file = "protobuf-4.21.8-cp38-cp38-win_amd64.whl", hash = "sha256:2c92a7bfcf4ae76a8ac72e545e99a7407e96ffe52934d690eb29a8809ee44d7b"}, + {file = "protobuf-4.21.8-cp39-cp39-win32.whl", hash = "sha256:89d641be4b5061823fa0e463c50a2607a97833e9f8cfb36c2f91ef5ccfcc3861"}, + {file = "protobuf-4.21.8-cp39-cp39-win_amd64.whl", hash = "sha256:bc471cf70a0f53892fdd62f8cd4215f0af8b3f132eeee002c34302dff9edd9b6"}, + {file = "protobuf-4.21.8-py2.py3-none-any.whl", hash = "sha256:a55545ce9eec4030cf100fcb93e861c622d927ef94070c1a3c01922902464278"}, + {file = "protobuf-4.21.8-py3-none-any.whl", hash = "sha256:0f236ce5016becd989bf39bd20761593e6d8298eccd2d878eda33012645dc369"}, + {file = "protobuf-4.21.8.tar.gz", hash = "sha256:427426593b55ff106c84e4a88cac855175330cb6eb7e889e85aaa7b5652b686d"}, ] psutil = [ - {file = "psutil-5.9.2-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:8f024fbb26c8daf5d70287bb3edfafa22283c255287cf523c5d81721e8e5d82c"}, - {file = "psutil-5.9.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:b2f248ffc346f4f4f0d747ee1947963613216b06688be0be2e393986fe20dbbb"}, - {file = "psutil-5.9.2-cp27-cp27m-win32.whl", hash = "sha256:b1928b9bf478d31fdffdb57101d18f9b70ed4e9b0e41af751851813547b2a9ab"}, - {file = "psutil-5.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:404f4816c16a2fcc4eaa36d7eb49a66df2d083e829d3e39ee8759a411dbc9ecf"}, - {file = "psutil-5.9.2-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:94e621c6a4ddb2573d4d30cba074f6d1aa0186645917df42c811c473dd22b339"}, - {file = "psutil-5.9.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:256098b4f6ffea6441eb54ab3eb64db9ecef18f6a80d7ba91549195d55420f84"}, - {file = "psutil-5.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:614337922702e9be37a39954d67fdb9e855981624d8011a9927b8f2d3c9625d9"}, - {file = "psutil-5.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39ec06dc6c934fb53df10c1672e299145ce609ff0611b569e75a88f313634969"}, - {file = "psutil-5.9.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3ac2c0375ef498e74b9b4ec56df3c88be43fe56cac465627572dbfb21c4be34"}, - {file = "psutil-5.9.2-cp310-cp310-win32.whl", hash = "sha256:e4c4a7636ffc47b7141864f1c5e7d649f42c54e49da2dd3cceb1c5f5d29bfc85"}, - {file = "psutil-5.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:f4cb67215c10d4657e320037109939b1c1d2fd70ca3d76301992f89fe2edb1f1"}, - {file = "psutil-5.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dc9bda7d5ced744622f157cc8d8bdd51735dafcecff807e928ff26bdb0ff097d"}, - {file = "psutil-5.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75291912b945a7351d45df682f9644540d564d62115d4a20d45fa17dc2d48f8"}, - {file = "psutil-5.9.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4018d5f9b6651f9896c7a7c2c9f4652e4eea53f10751c4e7d08a9093ab587ec"}, - {file = "psutil-5.9.2-cp36-cp36m-win32.whl", hash = "sha256:f40ba362fefc11d6bea4403f070078d60053ed422255bd838cd86a40674364c9"}, - {file = "psutil-5.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9770c1d25aee91417eba7869139d629d6328a9422ce1cdd112bd56377ca98444"}, - {file = "psutil-5.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:42638876b7f5ef43cef8dcf640d3401b27a51ee3fa137cb2aa2e72e188414c32"}, - {file = "psutil-5.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91aa0dac0c64688667b4285fa29354acfb3e834e1fd98b535b9986c883c2ce1d"}, - {file = "psutil-5.9.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fb54941aac044a61db9d8eb56fc5bee207db3bc58645d657249030e15ba3727"}, - {file = "psutil-5.9.2-cp37-cp37m-win32.whl", hash = "sha256:7cbb795dcd8ed8fd238bc9e9f64ab188f3f4096d2e811b5a82da53d164b84c3f"}, - {file = "psutil-5.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:5d39e3a2d5c40efa977c9a8dd4f679763c43c6c255b1340a56489955dbca767c"}, - {file = "psutil-5.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fd331866628d18223a4265371fd255774affd86244fc307ef66eaf00de0633d5"}, - {file = "psutil-5.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b315febaebae813326296872fdb4be92ad3ce10d1d742a6b0c49fb619481ed0b"}, - {file = "psutil-5.9.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7929a516125f62399d6e8e026129c8835f6c5a3aab88c3fff1a05ee8feb840d"}, - {file = "psutil-5.9.2-cp38-cp38-win32.whl", hash = "sha256:561dec454853846d1dd0247b44c2e66a0a0c490f937086930ec4b8f83bf44f06"}, - {file = "psutil-5.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:67b33f27fc0427483b61563a16c90d9f3b547eeb7af0ef1b9fe024cdc9b3a6ea"}, - {file = "psutil-5.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b3591616fa07b15050b2f87e1cdefd06a554382e72866fcc0ab2be9d116486c8"}, - {file = "psutil-5.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14b29f581b5edab1f133563272a6011925401804d52d603c5c606936b49c8b97"}, - {file = "psutil-5.9.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4642fd93785a29353d6917a23e2ac6177308ef5e8be5cc17008d885cb9f70f12"}, - {file = "psutil-5.9.2-cp39-cp39-win32.whl", hash = "sha256:ed29ea0b9a372c5188cdb2ad39f937900a10fb5478dc077283bf86eeac678ef1"}, - {file = "psutil-5.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:68b35cbff92d1f7103d8f1db77c977e72f49fcefae3d3d2b91c76b0e7aef48b8"}, - {file = "psutil-5.9.2.tar.gz", hash = "sha256:feb861a10b6c3bb00701063b37e4afc754f8217f0f09c42280586bd6ac712b5c"}, + {file = "psutil-5.9.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b4a247cd3feaae39bb6085fcebf35b3b8ecd9b022db796d89c8f05067ca28e71"}, + {file = "psutil-5.9.3-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:5fa88e3d5d0b480602553d362c4b33a63e0c40bfea7312a7bf78799e01e0810b"}, + {file = "psutil-5.9.3-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:767ef4fa33acda16703725c0473a91e1832d296c37c63896c7153ba81698f1ab"}, + {file = "psutil-5.9.3-cp27-cp27m-win32.whl", hash = "sha256:9a4af6ed1094f867834f5f07acd1250605a0874169a5fcadbcec864aec2496a6"}, + {file = "psutil-5.9.3-cp27-cp27m-win_amd64.whl", hash = "sha256:fa5e32c7d9b60b2528108ade2929b115167fe98d59f89555574715054f50fa31"}, + {file = "psutil-5.9.3-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:fe79b4ad4836e3da6c4650cb85a663b3a51aef22e1a829c384e18fae87e5e727"}, + {file = "psutil-5.9.3-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:db8e62016add2235cc87fb7ea000ede9e4ca0aa1f221b40cef049d02d5d2593d"}, + {file = "psutil-5.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:941a6c2c591da455d760121b44097781bc970be40e0e43081b9139da485ad5b7"}, + {file = "psutil-5.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:71b1206e7909792d16933a0d2c1c7f04ae196186c51ba8567abae1d041f06dcb"}, + {file = "psutil-5.9.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f57d63a2b5beaf797b87024d018772439f9d3103a395627b77d17a8d72009543"}, + {file = "psutil-5.9.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7507f6c7b0262d3e7b0eeda15045bf5881f4ada70473b87bc7b7c93b992a7d7"}, + {file = "psutil-5.9.3-cp310-cp310-win32.whl", hash = "sha256:1b540599481c73408f6b392cdffef5b01e8ff7a2ac8caae0a91b8222e88e8f1e"}, + {file = "psutil-5.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:547ebb02031fdada635452250ff39942db8310b5c4a8102dfe9384ee5791e650"}, + {file = "psutil-5.9.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d8c3cc6bb76492133474e130a12351a325336c01c96a24aae731abf5a47fe088"}, + {file = "psutil-5.9.3-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07d880053c6461c9b89cd5d4808f3b8336665fa3acdefd6777662c5ed73a851a"}, + {file = "psutil-5.9.3-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e8b50241dd3c2ed498507f87a6602825073c07f3b7e9560c58411c14fe1e1c9"}, + {file = "psutil-5.9.3-cp36-cp36m-win32.whl", hash = "sha256:828c9dc9478b34ab96be75c81942d8df0c2bb49edbb481f597314d92b6441d89"}, + {file = "psutil-5.9.3-cp36-cp36m-win_amd64.whl", hash = "sha256:ed15edb14f52925869250b1375f0ff58ca5c4fa8adefe4883cfb0737d32f5c02"}, + {file = "psutil-5.9.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d266cd05bd4a95ca1c2b9b5aac50d249cf7c94a542f47e0b22928ddf8b80d1ef"}, + {file = "psutil-5.9.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e4939ff75149b67aef77980409f156f0082fa36accc475d45c705bb00c6c16a"}, + {file = "psutil-5.9.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68fa227c32240c52982cb931801c5707a7f96dd8927f9102d6c7771ea1ff5698"}, + {file = "psutil-5.9.3-cp37-cp37m-win32.whl", hash = "sha256:beb57d8a1ca0ae0eb3d08ccaceb77e1a6d93606f0e1754f0d60a6ebd5c288837"}, + {file = "psutil-5.9.3-cp37-cp37m-win_amd64.whl", hash = "sha256:12500d761ac091f2426567f19f95fd3f15a197d96befb44a5c1e3cbe6db5752c"}, + {file = "psutil-5.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba38cf9984d5462b506e239cf4bc24e84ead4b1d71a3be35e66dad0d13ded7c1"}, + {file = "psutil-5.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:46907fa62acaac364fff0b8a9da7b360265d217e4fdeaca0a2397a6883dffba2"}, + {file = "psutil-5.9.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a04a1836894c8279e5e0a0127c0db8e198ca133d28be8a2a72b4db16f6cf99c1"}, + {file = "psutil-5.9.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a4e07611997acf178ad13b842377e3d8e9d0a5bac43ece9bfc22a96735d9a4f"}, + {file = "psutil-5.9.3-cp38-cp38-win32.whl", hash = "sha256:6ced1ad823ecfa7d3ce26fe8aa4996e2e53fb49b7fed8ad81c80958501ec0619"}, + {file = "psutil-5.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:35feafe232d1aaf35d51bd42790cbccb882456f9f18cdc411532902370d660df"}, + {file = "psutil-5.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:538fcf6ae856b5e12d13d7da25ad67f02113c96f5989e6ad44422cb5994ca7fc"}, + {file = "psutil-5.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a3d81165b8474087bb90ec4f333a638ccfd1d69d34a9b4a1a7eaac06648f9fbe"}, + {file = "psutil-5.9.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a7826e68b0cf4ce2c1ee385d64eab7d70e3133171376cac53d7c1790357ec8f"}, + {file = "psutil-5.9.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ec296f565191f89c48f33d9544d8d82b0d2af7dd7d2d4e6319f27a818f8d1cc"}, + {file = "psutil-5.9.3-cp39-cp39-win32.whl", hash = "sha256:9ec95df684583b5596c82bb380c53a603bb051cf019d5c849c47e117c5064395"}, + {file = "psutil-5.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:4bd4854f0c83aa84a5a40d3b5d0eb1f3c128f4146371e03baed4589fe4f3c931"}, + {file = "psutil-5.9.3.tar.gz", hash = "sha256:7ccfcdfea4fc4b0a02ca2c31de7fcd186beb9cff8207800e14ab66f79c773af6"}, ] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, @@ -453,8 +457,8 @@ PyYAML = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] setuptools = [ - {file = "setuptools-65.4.1-py3-none-any.whl", hash = "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012"}, - {file = "setuptools-65.4.1.tar.gz", hash = "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"}, + {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, + {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, @@ -487,6 +491,6 @@ typer = [ {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, ] typing-extensions = [ - {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, - {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] From beb552127a1547081a24263d110ba6f47c6ac72a Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Sat, 22 Oct 2022 23:40:05 +0200 Subject: [PATCH 017/793] feat(client): Simplify sharded logic --- router/client/src/sharded_client.rs | 111 +++++++--------------------- router/src/batcher.rs | 4 +- router/src/main.rs | 2 +- 3 files changed, 29 insertions(+), 88 deletions(-) diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 916e72b4..6fddfd7e 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -2,76 +2,18 @@ use crate::Result; use crate::{Batch, Client, GeneratedText}; use futures::future::join_all; -use tokio::sync::{broadcast, mpsc}; +use futures::future::select_all; use tonic::transport::Uri; -/// List of all available commands that can be sent through the command channel -#[derive(Clone, Debug)] -enum Command { - Generate( - Batch, - mpsc::Sender, Option)>>, - ), - GenerateWithCache( - Vec, - mpsc::Sender, Option)>>, - ), - ClearCache(mpsc::Sender>), -} - -/// Tokio task that handles the communication with a single shard -/// -/// We subscribe on a broadcast channel to receive commands that will be sent by -/// the ShardedClient. -/// -/// Each command is fan out to all shards. -/// -/// The result of the command is sent back to the ShardedClient through a mpsc channel (multi -/// producer = the shards, single consumer = the ShardedClient). -async fn client_task(mut client: Client, mut request_subscriber: broadcast::Receiver) { - while let Ok(message) = request_subscriber.recv().await { - match message { - Command::Generate(batch, response_tx) => { - let result = client.generate(batch).await; - // We can unwrap_or(()) here because the only error that can happen is if the - // receiver is dropped, which means that the ShardedClient already received a - // response from another shard - response_tx.try_send(result).unwrap_or(()); - } - Command::GenerateWithCache(batches, response_tx) => { - let result = client.generate_with_cache(batches).await; - response_tx.try_send(result).unwrap_or(()); - } - Command::ClearCache(response_tx) => { - let result = client.clear_cache().await; - response_tx.try_send(result).unwrap_or(()); - } - }; - } -} - /// Text Generation Inference gRPC multi client pub struct ShardedClient { - _clients: Vec, - request_tx: broadcast::Sender, + clients: Vec, } impl ShardedClient { fn new(clients: Vec) -> Self { - // The broadcast channel to communicate with the shards - // We use a capacity of one as the shards are not asynchronous and can only process one - // command at a time - let (request_tx, _) = broadcast::channel(1); - - // Spawn client tasks - for client in clients.iter() { - let request_subscriber = request_tx.subscribe(); - tokio::spawn(client_task(client.clone(), request_subscriber)); - } - Self { - _clients: clients, - request_tx, + clients, } } @@ -101,15 +43,15 @@ impl ShardedClient { /// /// Returns a list of generated texts of request that met their stopping criteria /// and the next cached batch - pub async fn generate(&self, batch: Batch) -> Result<(Vec, Option)> { - // Create a channel to receive the response from the shards - // We will only ever receive one message on this channel - let (response_tx, mut response_rx) = mpsc::channel(1); - self.request_tx - .send(Command::Generate(batch, response_tx)) - .unwrap(); + pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| Box::pin(client.generate(batch.clone()))) + .collect(); // As soon as we receive one response, we can return as all shards will return the same - response_rx.recv().await.unwrap() + let (result, _, _) = select_all(futures).await; + result } /// Generate one token for each request in the given cached batch @@ -117,27 +59,26 @@ impl ShardedClient { /// Returns a list of generated texts of request that met their stopping criteria /// and the next cached batch pub async fn generate_with_cache( - &self, + &mut self, batches: Vec, ) -> Result<(Vec, Option)> { - // Create a channel to receive the response from the shards - // We will only ever receive one message on this channel - let (response_tx, mut response_rx) = mpsc::channel(1); - self.request_tx - .send(Command::GenerateWithCache(batches, response_tx)) - .unwrap(); + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| Box::pin(client.generate_with_cache(batches.clone()))) + .collect(); // As soon as we receive one response, we can return as all shards will return the same - response_rx.recv().await.unwrap() + let (result, _, _) = select_all(futures).await; + result } /// Clear the past generations cache - pub async fn clear_cache(&self) -> Result<()> { - // Create a channel to receive the response from the shards - // We will only ever receive one message on this channel - let (response_tx, mut response_rx) = mpsc::channel(1); - self.request_tx - .send(Command::ClearCache(response_tx)) - .unwrap(); - response_rx.recv().await.unwrap() + pub async fn clear_cache(&mut self) -> Result<()> { + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| client.clear_cache()) + .collect(); + join_all(futures).await.into_iter().collect() } } diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 052716a4..f71428e5 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -39,9 +39,9 @@ impl Batcher { // Spawn batching background task that contains all the inference logic tokio::spawn(batching_task( + client, max_batch_size, max_waiting_tokens, - client, db.clone(), shared.clone(), )); @@ -86,9 +86,9 @@ impl Batcher { /// Batches requests and sends them to the inference server #[instrument(skip(client, db, shared))] async fn batching_task( + mut client: ShardedClient, max_batch_size: usize, max_waiting_tokens: usize, - client: ShardedClient, db: Db, shared: Arc, ) { diff --git a/router/src/main.rs b/router/src/main.rs index 6d1a0fb9..ea7ebd12 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -61,7 +61,7 @@ fn main() -> Result<(), std::io::Error> { .unwrap() .block_on(async { // Instantiate sharded client from the master unix socket - let sharded_client = ShardedClient::connect_uds(master_shard_uds_path) + let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path) .await .expect("Could not connect to server"); // Clear the cache; useful if the webserver rebooted From 09674e6df99918dad5169b5cc81241c884c543d0 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 27 Oct 2022 14:25:29 +0200 Subject: [PATCH 018/793] feat(server): Support bitsandbytes --- Dockerfile | 3 +- Makefile | 9 +++ README.md | 42 ++++++++++--- launcher/src/main.rs | 19 ++++-- router/client/src/sharded_client.rs | 4 +- router/src/batcher.rs | 12 +++- router/src/lib.rs | 5 ++ router/src/server.rs | 32 +++++++--- router/src/validation.rs | 22 ++++--- server/bloom_inference/cli.py | 3 +- server/bloom_inference/model.py | 92 ++++++++++++++++++++++++++--- server/bloom_inference/server.py | 10 +++- server/poetry.lock | 14 ++++- server/pyproject.toml | 1 + 14 files changed, 221 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index 78f66c0f..08561b68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ ENV LANG=C.UTF-8 \ DEBIAN_FRONTEND=noninteractive \ MODEL_BASE_PATH=/var/azureml-model \ MODEL_NAME=bigscience/bloom \ + QUANTIZE=false \ NUM_GPUS=8 \ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ NCCL_ASYNC_ERROR_HANDLING=1 \ @@ -72,4 +73,4 @@ COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/loca # Install launcher COPY --from=launcher-builder /usr/local/cargo/bin/text-generation-launcher /usr/local/bin/text-generation-launcher -CMD HUGGINGFACE_HUB_CACHE=$MODEL_BASE_PATH text-generation-launcher --model-name $MODEL_NAME --num-shard $NUM_GPUS \ No newline at end of file +CMD HUGGINGFACE_HUB_CACHE=$MODEL_BASE_PATH text-generation-launcher --num-shard $NUM_GPUS \ No newline at end of file diff --git a/Makefile b/Makefile index e22309f9..a52ed2b1 100644 --- a/Makefile +++ b/Makefile @@ -18,5 +18,14 @@ router-dev: run-bloom-560m: text-generation-launcher --model-name bigscience/bloom-560m --num-shard 2 +run-bloom-560m-quantize: + text-generation-launcher --model-name bigscience/bloom-560m --num-shard 2 --quantize + +download-bloom: + bloom-inference-server download-weights bigscience/bloom + run-bloom: text-generation-launcher --model-name bigscience/bloom --num-shard 8 + +run-bloom-quantize: + text-generation-launcher --model-name bigscience/bloom --num-shard 8 --quantize \ No newline at end of file diff --git a/README.md b/README.md index 1af90dce..e51d8c81 100644 --- a/README.md +++ b/README.md @@ -8,22 +8,26 @@ A Rust and gRPC server for large language models text generation inference. -## Load Tests for BLOOM +## Features -See `k6/load_test.js` -We send the default examples with a 1 second delay between requests. +- Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) +- [Dynamic bathing of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput +- [Safetensors](https://github.com/huggingface/safetensors) weight loading +- 45ms per token generation for BLOOM with 8xA100 80GB + +## Supported models -Stages: -- Ramp up to 50 vus in 1min -- Ramp up from 50 to 100 vus in 2min -- Ramp down to 0 vus in 1min +- BLOOM +- BLOOM-560m + +## Load Tests for BLOOM +See `k6/load_test.js` | | avg | min | med | max | p(90) | p(95) | RPS | |--------------------------------------------------------------|-----------|--------------|-----------|------------|-----------|-----------|----------| | [Original code](https://github.com/huggingface/transformers_bloom_parallel) | 8.9s | 1s | 9.12s | 16.69s | 13.7s | 14.26s | 5.9 | -| ISO with original code | 8.88s | **959.53ms** | 8.89s | 17.08s | 13.34s | 14.12s | 5.94 | -| New batching logic | **5.44s** | 1.27s | **5.28s** | **13.12s** | **7.78s** | **8.92s** | **9.08** | +| New batching logic | **5.44s** | **959.53ms** | **5.28s** | **13.12s** | **7.78s** | **8.92s** | **9.08** | ## Install @@ -33,10 +37,30 @@ make install ## Run +### BLOOM 560-m + ```shell make run-bloom-560m ``` +### BLOOM + +First you need to download the weights: + +```shell +make download-bloom +``` + +```shell +make run-bloom # Requires 8xA100 80GB +``` + +You can also quantize the weights with bitsandbytes to reduce the VRAM requirement: + +```shell +make run-bloom-quantize # Requires 8xA100 40GB +``` + ## Test ```shell diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 6b529d60..3e82516a 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -21,6 +21,8 @@ struct Args { model_name: String, #[clap(long, env)] num_shard: Option, + #[clap(long, env)] + quantize: bool, #[clap(default_value = "128", long, env)] max_concurrent_requests: usize, #[clap(default_value = "1000", long, env)] @@ -46,6 +48,7 @@ fn main() -> ExitCode { let Args { model_name, num_shard, + quantize, max_concurrent_requests, max_input_length, max_batch_size, @@ -87,6 +90,7 @@ fn main() -> ExitCode { thread::spawn(move || { shard_manager( model_name, + quantize, uds_path, rank, num_shard, @@ -169,6 +173,8 @@ fn main() -> ExitCode { tracing::error!("text-generation-router not found in PATH"); tracing::error!("Please install it with `make install-router`") } + } else { + tracing::error!("{}", err); } shutdown_shards(shutdown, &shutdown_receiver); @@ -232,6 +238,7 @@ enum ShardStatus { #[allow(clippy::too_many_arguments)] fn shard_manager( model_name: String, + quantize: bool, uds_path: String, rank: usize, world_size: usize, @@ -260,6 +267,10 @@ fn shard_manager( shard_argv.push("--sharded".to_string()); } + if quantize { + shard_argv.push("--quantize".to_string()) + } + let mut env = vec![ ("RANK".parse().unwrap(), rank.to_string().parse().unwrap()), ( @@ -338,11 +349,9 @@ fn shard_manager( tracing::info!("Shard {} ready in {:?}", rank, start_time.elapsed()); status_sender.send(ShardStatus::Ready).unwrap(); ready = true; - } else if !ready { - if wait_time.elapsed() > Duration::from_secs(10) { - tracing::info!("Waiting for shard {} to be ready...", rank); - wait_time = Instant::now(); - } + } else if !ready && wait_time.elapsed() > Duration::from_secs(10) { + tracing::info!("Waiting for shard {} to be ready...", rank); + wait_time = Instant::now(); } sleep(Duration::from_millis(100)); } diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 6fddfd7e..6c70afca 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -12,9 +12,7 @@ pub struct ShardedClient { impl ShardedClient { fn new(clients: Vec) -> Self { - Self { - clients, - } + Self { clients } } /// Create a new ShardedClient from a master client. The master client will communicate with diff --git a/router/src/batcher.rs b/router/src/batcher.rs index f71428e5..f131bf99 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -1,7 +1,8 @@ /// Batching and inference logic -use crate::GenerateRequest; use crate::{Db, Entry}; +use crate::{ErrorResponse, GenerateRequest}; use axum::http::StatusCode; +use axum::Json; use bloom_inference_client::{Batch, ClientError, GeneratedText, ShardedClient}; use std::future::Future; use std::sync::Arc; @@ -213,10 +214,15 @@ pub enum InferError { } /// Convert to Axum supported format -impl From for (StatusCode, String) { +impl From for (StatusCode, Json) { fn from(err: InferError) -> Self { match err { - InferError::GenerationError(_) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), + InferError::GenerationError(_) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: err.to_string(), + }), + ), } } } diff --git a/router/src/lib.rs b/router/src/lib.rs index 6604a91f..8646ad59 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -64,3 +64,8 @@ pub(crate) struct GenerateRequest { pub(crate) struct GeneratedText { pub generated_text: String, } + +#[derive(Serialize)] +pub(crate) struct ErrorResponse { + pub error: String, +} diff --git a/router/src/server.rs b/router/src/server.rs index 02c4a497..5698f4ec 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,4 +1,6 @@ -use crate::{Batcher, GenerateParameters, GenerateRequest, GeneratedText, Validation}; +use crate::{ + Batcher, ErrorResponse, GenerateParameters, GenerateRequest, GeneratedText, Validation, +}; use axum::extract::Extension; use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; @@ -23,7 +25,7 @@ struct ServerState { /// Health check method #[instrument(skip(state), fields(time, time_per_token))] -async fn health(state: Extension) -> Result<(), (StatusCode, String)> { +async fn health(state: Extension) -> Result<(), (StatusCode, Json)> { // TODO: while this is the best health check we can do, it is a bit on the heavy side and might // be a bit too slow for a health check. // What we should do instead if check if the gRPC channels are still healthy. @@ -32,7 +34,9 @@ async fn health(state: Extension) -> Result<(), (StatusCode, String let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { ( StatusCode::TOO_MANY_REQUESTS, - "Model is overloaded".to_string(), + Json(ErrorResponse { + error: "Model is overloaded".to_string(), + }), ) })?; @@ -70,13 +74,16 @@ async fn health(state: Extension) -> Result<(), (StatusCode, String async fn generate( state: Extension, req: Json, -) -> Result { +) -> Result)> { let start_time = Instant::now(); // Limit concurrent requests by acquiring a permit from the semaphore let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { + tracing::error!("Model is overloaded"); ( StatusCode::TOO_MANY_REQUESTS, - "Model is overloaded".to_string(), + Json(ErrorResponse { + error: "Model is overloaded".to_string(), + }), ) })?; @@ -88,10 +95,21 @@ async fn generate( inputs: req.inputs.clone(), parameters: req.parameters.clone(), }) - .await?; + .await + .map_err(|err| { + tracing::error!("{}", err.to_string()); + err + })?; // Inference - let response = state.batcher.infer(input_length, validated_request).await?; + let response = state + .batcher + .infer(input_length, validated_request) + .await + .map_err(|err| { + tracing::error!("{}", err.to_string()); + err + })?; // Timings let total_time = start_time.elapsed(); diff --git a/router/src/validation.rs b/router/src/validation.rs index 11712d0a..43c246f0 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,6 +1,7 @@ /// Payload validation logic -use crate::GenerateRequest; +use crate::{ErrorResponse, GenerateRequest}; use axum::http::StatusCode; +use axum::Json; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokenizers::{ @@ -146,20 +147,25 @@ type ValidationRequest = ( #[derive(Error, Debug)] pub enum ValidationError { - #[error("Temperature must be strictly positive")] + #[error("temperature must be strictly positive")] Temperature, - #[error("Top p must be >= 0.0 or < 1.0")] + #[error("top_p must be >= 0.0 or < 1.0")] TopP, - #[error("Top k must be strictly positive")] + #[error("top_k must be strictly positive")] TopK, - #[error("Max New Tokens must be <= 512")] + #[error("max_new_tokens must be <= 512")] MaxNewTokens, - #[error("Inputs must have less than {1} tokens. Given: {0}")] + #[error("inputs must have less than {1} tokens. Given: {0}")] InputLength(usize, usize), } -impl From for (StatusCode, String) { +impl From for (StatusCode, Json) { fn from(err: ValidationError) -> Self { - (StatusCode::BAD_REQUEST, err.to_string()) + ( + StatusCode::BAD_REQUEST, + Json(ErrorResponse { + error: err.to_string(), + }), + ) } } diff --git a/server/bloom_inference/cli.py b/server/bloom_inference/cli.py index 600c5273..6360aec0 100644 --- a/server/bloom_inference/cli.py +++ b/server/bloom_inference/cli.py @@ -12,6 +12,7 @@ def serve( model_name: str, sharded: bool = False, + quantize: bool = False, uds_path: Path = "/tmp/bloom-inference", ): if sharded: @@ -28,7 +29,7 @@ def serve( os.getenv("MASTER_PORT", None) is not None ), "MASTER_PORT must be set when sharded is True" - server.serve(model_name, sharded, uds_path) + server.serve(model_name, sharded, quantize, uds_path) @app.command() diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py index eac54002..a0450052 100644 --- a/server/bloom_inference/model.py +++ b/server/bloom_inference/model.py @@ -19,9 +19,16 @@ NextTokenChooser, initialize_torch_distributed, weight_files, - download_weights + download_weights, ) +HAS_BITS_AND_BYTES = True +try: + import bitsandbytes as bnb + from bitsandbytes.nn import Int8Params +except Exception as e: + HAS_BITS_AND_BYTES = False + torch.manual_seed(0) @@ -68,7 +75,9 @@ def from_pb( ) stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) - input_ids = tokenizer(inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8).to(device) + input_ids = tokenizer( + inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 + ).to(device) all_input_ids = input_ids["input_ids"].unsqueeze(-1) return cls( @@ -250,7 +259,7 @@ def forward(self, input_ids, attention_mask, past_key_values: Optional = None): def generate_token( self, batch: Batch ) -> Tuple[List[GeneratedText], Optional[Batch]]: - with torch.no_grad(): + with torch.inference_mode(): outputs = self.forward(**batch.input_ids) # List of indices to cache @@ -374,13 +383,13 @@ def generate_token( class BLOOMSharded(BLOOM): - def __init__(self, model_name: str): + def __init__(self, model_name: str, quantize: bool = False): super(BLOOM, self).__init__() self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 if torch.cuda.is_available(): self.device = torch.device(f"cuda:{self.rank}") - dtype = torch.bfloat16 + dtype = torch.float16 else: self.device = torch.device("cpu") dtype = torch.float32 @@ -414,6 +423,7 @@ def __init__(self, model_name: str): self.load_weights( model, filenames, + quantize=quantize, device=self.device, rank=self.rank, world_size=self.world_size, @@ -423,11 +433,18 @@ def __init__(self, model_name: str): @staticmethod def load_weights( - model, filenames: List[str], device: torch.device, rank: int, world_size: int + model, + filenames: List[str], + quantize: bool, + device: torch.device, + rank: int, + world_size: int, ): parameters = dict(model.named_parameters()) for file in filenames: - with safe_open(file, framework="pt", device=str(device)) as f: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: for name in f.keys(): full_name = f"transformer.{name}" @@ -479,6 +496,67 @@ def load_weights( ) tensor = tensor.contiguous() + + if quantize: + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine" + ) + + if ( + type(module) + in [TensorParallelRowLinear, TensorParallelColumnLinear] + and param_name == "weight" + ): + tensor = Int8Params( + tensor.transpose(1, 0), + has_fp16_weights=False, + requires_grad=False, + ).to(device) + state = bnb.MatmulLtState() + state.threshold = 6.0 + state.has_fp16_weights = False + state.memory_efficient_backward = False + state.use_pool = True + state.CB = tensor.CB + state.SCB = tensor.SCB + tensor.CB = None + tensor.SCB = None + + def replace_linear(state, in_features, out_features): + def linear(input, weight, bias): + size_out = input.size()[:-1] + (out_features,) + input = input.view(-1, in_features) + out = torch.empty( + size_out, device=input.device, dtype=input.dtype + ) + out = bnb.matmul( + input, + weight, + out=out.view(-1, out_features), + state=state, + threshold=state.threshold, + bias=bias, + ) + + if state.CB is not None: + # we converted 8-bit row major to turing/ampere format + # in the first inference pass + # we no longer need the row-major weight + del state.CB + weight.data = state.CxB + + return out.view(size_out) + + return linear + + module.linear = replace_linear( + state, module.in_features, module.out_features + ) + + else: + tensor = tensor.to(device) + module._parameters[param_name] = tensor if name == "word_embeddings.weight": model.lm_head._parameters["weight"] = tensor diff --git a/server/bloom_inference/server.py b/server/bloom_inference/server.py index fb02db6d..ad40a52a 100644 --- a/server/bloom_inference/server.py +++ b/server/bloom_inference/server.py @@ -68,21 +68,27 @@ async def GenerateWithCache(self, request, context): def serve( model_name: str, sharded: bool, + quantize: bool, uds_path: Path, ): async def serve_inner( model_name: str, sharded: bool = False, + quantize: bool = False, ): unix_socket_template = "unix://{}-{}" if sharded: - model = BLOOMSharded(model_name) + model = BLOOMSharded(model_name, quantize) server_urls = [ unix_socket_template.format(uds_path, rank) for rank in range(model.world_size) ] local_url = server_urls[model.rank] else: + if quantize: + raise ValueError( + "bitsandbytes quantization is only available when running in `sharded` mode." + ) model = BLOOM(model_name) local_url = unix_socket_template.format(uds_path, 0) server_urls = [local_url] @@ -105,4 +111,4 @@ async def serve_inner( print("Signal received. Shutting down") await server.stop(0) - asyncio.run(serve_inner(model_name, sharded)) + asyncio.run(serve_inner(model_name, sharded, quantize)) diff --git a/server/poetry.lock b/server/poetry.lock index 38f14d35..5e635a67 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -22,6 +22,14 @@ test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] test_trackers = ["comet-ml", "tensorboard", "wandb"] testing = ["datasets", "deepspeed (<0.7.0)", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scipy", "sklearn", "tqdm", "transformers"] +[[package]] +name = "bitsandbytes" +version = "0.35.1" +description = "8-bit optimizers and matrix multiplication routines." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "click" version = "8.1.3" @@ -205,13 +213,17 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "a4eef5f52e8d046aa883082c865b0865047f611a3240b18250487d4b6e831496" +content-hash = "50d9d44577a0222f125c770732d5f88807378573bd7386036eb5c79fc2a7c552" [metadata.files] accelerate = [ {file = "accelerate-0.12.0-py3-none-any.whl", hash = "sha256:7742ca5c9f15dd1e0a283305599c196e260af4717a561d1f544aeab27d828af6"}, {file = "accelerate-0.12.0.tar.gz", hash = "sha256:e8b119c94fac31877620d5f9de311164ec81fa9dc9e175f0d0d4f50fc8d79473"}, ] +bitsandbytes = [ + {file = "bitsandbytes-0.35.1-py3-none-any.whl", hash = "sha256:4506a9e3778359a743938aa5592d8d043fa91d1df66cd01ba8cc6486e64dea45"}, + {file = "bitsandbytes-0.35.1.tar.gz", hash = "sha256:63a6f59c87b713a731a685e43d68c19789ee6381e62196cafab293b87eca5d46"}, +] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 3d38f512..4e5e98b3 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -15,6 +15,7 @@ typer = "^0.6.1" grpcio-reflection = "^1.49.1" accelerate = "^0.12.0" joblib = "^1.2.0" +bitsandbytes = "^0.35.1" [tool.poetry.group.dev.dependencies] grpcio-tools = "^1.49.1" From 3cf6368c7722525fb8d069edb1e0ed70c42cf268 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 28 Oct 2022 19:24:00 +0200 Subject: [PATCH 019/793] feat(server): Support all AutoModelForCausalLM on a best effort basis --- Cargo.lock | 156 ++--- Dockerfile | 2 +- Makefile | 2 +- README.md | 4 +- aml/model.yaml | 4 +- launcher/src/main.rs | 4 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- router/src/batcher.rs | 2 +- router/src/db.rs | 2 +- router/src/main.rs | 6 +- router/src/server.rs | 2 +- server/.gitignore | 4 +- server/Makefile | 10 +- server/bloom_inference/model.py | 582 ------------------ server/pyproject.toml | 7 +- .../__init__.py | 0 .../cache.py | 3 +- .../cli.py | 7 +- server/text_generation/models/__init__.py | 22 + server/text_generation/models/bloom.py | 231 +++++++ server/text_generation/models/model.py | 166 +++++ server/text_generation/models/types.py | 206 +++++++ .../pb/.gitignore | 0 .../server.py | 23 +- .../utils.py | 16 +- 26 files changed, 755 insertions(+), 710 deletions(-) delete mode 100644 server/bloom_inference/model.py rename server/{bloom_inference => text_generation}/__init__.py (100%) rename server/{bloom_inference => text_generation}/cache.py (91%) rename server/{bloom_inference => text_generation}/cli.py (83%) create mode 100644 server/text_generation/models/__init__.py create mode 100644 server/text_generation/models/bloom.py create mode 100644 server/text_generation/models/model.py create mode 100644 server/text_generation/models/types.py rename server/{bloom_inference => text_generation}/pb/.gitignore (100%) rename server/{bloom_inference => text_generation}/server.py (83%) rename server/{bloom_inference => text_generation}/utils.py (89%) diff --git a/Cargo.lock b/Cargo.lock index 551f7aeb..33d75f0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "async-stream" @@ -83,9 +83,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e3356844c4d6a6d6467b8da2cffb4a2820be256f50a3a386c9d152bab31043" +checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43" dependencies = [ "async-trait", "axum-core", @@ -114,9 +114,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f0c0a60006f2a293d82d571f635042a72edf927539b7685bd62d361963839b" +checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc" dependencies = [ "async-trait", "bytes", @@ -130,9 +130,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "bitflags" @@ -149,21 +149,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bloom-inference-client" -version = "0.1.0" -dependencies = [ - "futures", - "prost", - "thiserror", - "tokio", - "tonic", - "tonic-build", - "tower", - "tracing", - "tracing-error", -] - [[package]] name = "bumpalo" version = "3.11.1" @@ -255,9 +240,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.17" +version = "4.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06badb543e734a2d6568e19a40af66ed5364360b9226184926f89d229b4b4267" +checksum = "335867764ed2de42325fafe6d18b8af74ba97ee0c590fa016f157535b42ab04b" dependencies = [ "atty", "bitflags", @@ -270,9 +255,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.0.13" +version = "4.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f169caba89a7d512b5418b09864543eeb4d497416c917d7137863bd2076ad" +checksum = "16a1b0f6422af32d5da0c58e2703320f379216ee70198241c84173a8c5ac28f3" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -532,14 +517,14 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" +checksum = "4b9663d381d07ae25dc88dbdf27df458faa83a9b25336bcac83d5e452b5fc9d3" dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys 0.36.1", + "windows-sys 0.42.0", ] [[package]] @@ -600,9 +585,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" dependencies = [ "futures-channel", "futures-core", @@ -615,9 +600,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", "futures-sink", @@ -625,15 +610,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" [[package]] name = "futures-executor" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" dependencies = [ "futures-core", "futures-task", @@ -642,15 +627,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" [[package]] name = "futures-macro" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" dependencies = [ "proc-macro2", "quote", @@ -659,21 +644,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" [[package]] name = "futures-task" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" [[package]] name = "futures-util" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-channel", "futures-core", @@ -699,9 +684,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", @@ -716,9 +701,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" dependencies = [ "bytes", "fnv", @@ -967,9 +952,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.135" +version = "0.2.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" [[package]] name = "lock_api" @@ -992,9 +977,9 @@ dependencies = [ [[package]] name = "macro_rules_attribute" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258c86475e1616d6f2d8f5227cfaabd3dae1f6d5388b9597df8a199d4497aba7" +checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862" dependencies = [ "macro_rules_attribute-proc_macro", "paste", @@ -1002,9 +987,9 @@ dependencies = [ [[package]] name = "macro_rules_attribute-proc_macro" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26a8d2502d5aa4d411ef494ba7470eb299f05725179ce3b5de77aa01a9ffdea" +checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" [[package]] name = "matchit" @@ -1050,14 +1035,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.36.1", + "windows-sys 0.42.0", ] [[package]] @@ -1200,9 +1185,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.76" +version = "0.9.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" +checksum = "b03b84c3b2d099b81f0953422b4d4ad58761589d0229b5506356afca05a3670a" dependencies = [ "autocfg", "cc", @@ -1213,9 +1198,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.3.0" +version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" [[package]] name = "overload" @@ -1302,9 +1287,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "ppv-lite86" @@ -1602,18 +1587,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.145" +version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.145" +version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" dependencies = [ "proc-macro2", "quote", @@ -1622,9 +1607,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" +checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45" dependencies = [ "itoa", "ryu", @@ -1739,9 +1724,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" dependencies = [ "proc-macro2", "quote", @@ -1798,11 +1783,26 @@ dependencies = [ "winapi", ] +[[package]] +name = "text-generation-client" +version = "0.1.0" +dependencies = [ + "futures", + "prost", + "thiserror", + "tokio", + "tonic", + "tonic-build", + "tower", + "tracing", + "tracing-error", +] + [[package]] name = "text-generation-launcher" version = "0.1.0" dependencies = [ - "clap 4.0.17", + "clap 4.0.18", "ctrlc", "subprocess", "tracing", @@ -1814,12 +1814,12 @@ name = "text-generation-router" version = "0.1.0" dependencies = [ "axum", - "bloom-inference-client", - "clap 4.0.17", + "clap 4.0.18", "futures", "parking_lot", "serde", "serde_json", + "text-generation-client", "thiserror", "tokenizers", "tokio", diff --git a/Dockerfile b/Dockerfile index 08561b68..a2bf199a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,7 @@ COPY proto proto COPY server server RUN cd server && \ make gen-server && \ - /opt/miniconda/envs/text-generation/bin/pip install . --no-cache-dir + /opt/miniconda/envs/text-generation/bin/pip install ".[bnb]" --no-cache-dir # Install router COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/local/bin/text-generation-router diff --git a/Makefile b/Makefile index a52ed2b1..d427ff87 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ run-bloom-560m-quantize: text-generation-launcher --model-name bigscience/bloom-560m --num-shard 2 --quantize download-bloom: - bloom-inference-server download-weights bigscience/bloom + text-generation-server download-weights bigscience/bloom run-bloom: text-generation-launcher --model-name bigscience/bloom --num-shard 8 diff --git a/README.md b/README.md index e51d8c81..2d2d49d3 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,13 @@ A Rust and gRPC server for large language models text generation inference. - [Safetensors](https://github.com/huggingface/safetensors) weight loading - 45ms per token generation for BLOOM with 8xA100 80GB -## Supported models +## Officially supported models - BLOOM - BLOOM-560m +Other models are supported on a best-effort basis using `AutoModelForCausalLM.from_pretrained(, torch_dtype=torch.float16, device_map="auto")`. + ## Load Tests for BLOOM See `k6/load_test.js` diff --git a/aml/model.yaml b/aml/model.yaml index e4f1ded2..bd490f1a 100644 --- a/aml/model.yaml +++ b/aml/model.yaml @@ -1,5 +1,5 @@ $schema: https://azuremlschemas.azureedge.net/latest/model.schema.json -name: bloom +name: bloom-safetensors version: 1 -path: ./bloom +path: ./bloom-safetensors type: custom_model diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 3e82516a..f94dd589 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -256,7 +256,7 @@ fn shard_manager( // Process args let mut shard_argv = vec![ - "bloom-inference-server".to_string(), + "text-generation-server".to_string(), "serve".to_string(), model_name, "--uds-path".to_string(), @@ -311,7 +311,7 @@ fn shard_manager( Err(err) => { if let PopenError::IoError(ref err) = err { if err.kind() == io::ErrorKind::NotFound { - tracing::error!("bloom-inference-server not found in PATH"); + tracing::error!("text-generation-server not found in PATH"); tracing::error!("Please install it with `make install-server`") } } diff --git a/router/Cargo.toml b/router/Cargo.toml index 5820c138..da9518bf 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -14,7 +14,7 @@ path = "src/main.rs" [dependencies] axum = { version = "0.5.16", features = ["json", "serde_json"] } -bloom-inference-client = { path = "client" } +text-generation-client = { path = "client" } clap = { version = "4.0.15", features = ["derive", "env"] } futures = "0.3.24" parking_lot = "0.12.1" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 633f82a9..fdd32494 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "bloom-inference-client" +name = "text-generation-client" version = "0.1.0" edition = "2021" diff --git a/router/src/batcher.rs b/router/src/batcher.rs index f131bf99..e381986a 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -3,9 +3,9 @@ use crate::{Db, Entry}; use crate::{ErrorResponse, GenerateRequest}; use axum::http::StatusCode; use axum::Json; -use bloom_inference_client::{Batch, ClientError, GeneratedText, ShardedClient}; use std::future::Future; use std::sync::Arc; +use text_generation_client::{Batch, ClientError, GeneratedText, ShardedClient}; use thiserror::Error; use tokio::sync::{oneshot, Notify}; use tokio::time::Instant; diff --git a/router/src/db.rs b/router/src/db.rs index 76a08ae0..af36614e 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,10 +1,10 @@ use crate::InferResponse; /// This code is massively inspired by Tokio mini-redis use crate::{GenerateParameters, GenerateRequest}; -use bloom_inference_client::{Batch, ClientError, LogitsWarperParameters, Request}; use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; +use text_generation_client::{Batch, ClientError, LogitsWarperParameters, Request}; use tokio::sync::oneshot::Sender; use tokio::time::Instant; diff --git a/router/src/main.rs b/router/src/main.rs index ea7ebd12..b24ec4c9 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,7 +1,7 @@ -/// Text Generation Inference webserver entrypoint -use bloom_inference_client::ShardedClient; use clap::Parser; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +/// Text Generation Inference webserver entrypoint +use text_generation_client::ShardedClient; use text_generation_router::server; use tokenizers::Tokenizer; @@ -19,7 +19,7 @@ struct Args { max_waiting_tokens: usize, #[clap(default_value = "3000", long, short, env)] port: u16, - #[clap(default_value = "/tmp/bloom-inference-0", long, env)] + #[clap(default_value = "/tmp/text-generation-0", long, env)] master_shard_uds_path: String, #[clap(default_value = "bigscience/bloom", long, env)] tokenizer_name: String, diff --git a/router/src/server.rs b/router/src/server.rs index 5698f4ec..d31c9494 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -6,9 +6,9 @@ use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; use axum::routing::{get, post}; use axum::{Json, Router}; -use bloom_inference_client::ShardedClient; use std::net::SocketAddr; use std::sync::Arc; +use text_generation_client::ShardedClient; use tokenizers::Tokenizer; use tokio::signal; use tokio::sync::Semaphore; diff --git a/server/.gitignore b/server/.gitignore index 88bbea8f..5758ba92 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -1,7 +1,7 @@ # Byte-compiled / optimized / DLL files __pycache__/ -bloom_inference/__pycache__/ -bloom_inference/pb/__pycache__/ +text_generation/__pycache__/ +text_generation/pb/__pycache__/ *.py[cod] *$py.class diff --git a/server/Makefile b/server/Makefile index 9c7a2c43..4fa966e2 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,10 +1,10 @@ gen-server: # Compile protos pip install grpcio-tools==1.49.1 --no-cache-dir - mkdir bloom_inference/pb || true - python -m grpc_tools.protoc -I../proto --python_out=bloom_inference/pb --grpc_python_out=bloom_inference/pb ../proto/generate.proto - find bloom_inference/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; - touch bloom_inference/pb/__init__.py + mkdir text_generation/pb || true + python -m grpc_tools.protoc -I../proto --python_out=text_generation/pb --grpc_python_out=text_generation/pb ../proto/generate.proto + find text_generation/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; + touch text_generation/pb/__init__.py install-transformers: # Install specific version of transformers @@ -36,4 +36,4 @@ install: gen-server install-torch install-transformers install-safetensors pip install -e . --no-cache-dir run-dev: - python -m torch.distributed.run --nproc_per_node=2 bloom_inference/cli.py serve bigscience/bloom-560m --sharded \ No newline at end of file + python -m torch.distributed.run --nproc_per_node=2 text_generation/cli.py serve bigscience/bloom-560m --sharded \ No newline at end of file diff --git a/server/bloom_inference/model.py b/server/bloom_inference/model.py deleted file mode 100644 index a0450052..00000000 --- a/server/bloom_inference/model.py +++ /dev/null @@ -1,582 +0,0 @@ -import torch -import torch.distributed - -from dataclasses import dataclass -from typing import List, Tuple, Optional, Dict - -from accelerate import init_empty_weights -from safetensors import safe_open -from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig -from transformers.models.bloom.parallel_layers import ( - TensorParallelColumnLinear, - TensorParallelEmbedding, - TensorParallelRowLinear, -) - -from bloom_inference.pb import generate_pb2 -from bloom_inference.utils import ( - StoppingCriteria, - NextTokenChooser, - initialize_torch_distributed, - weight_files, - download_weights, -) - -HAS_BITS_AND_BYTES = True -try: - import bitsandbytes as bnb - from bitsandbytes.nn import Int8Params -except Exception as e: - HAS_BITS_AND_BYTES = False - -torch.manual_seed(0) - - -@dataclass -class Batch: - batch_id: int - requests: List[generate_pb2.Request] - all_input_lengths: List[int] - input_ids: Dict[str, torch.Tensor] - all_input_ids: List[torch.Tensor] - next_token_choosers: List[NextTokenChooser] - stopping_criterias: List[StoppingCriteria] - size: int - max_sequence_length: int - - def to_pb(self): - return generate_pb2.Batch( - id=self.batch_id, - requests=self.requests, - size=self.size, - max_sequence_length=self.max_sequence_length, - ) - - @classmethod - def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device - ) -> "Batch": - inputs = [] - next_token_choosers = [] - stopping_criterias = [] - all_input_lengths = [] - - # Parse batch - for r in pb.requests: - inputs.append(r.inputs) - all_input_lengths.append(r.input_length) - next_token_choosers.append( - NextTokenChooser( - temperature=r.parameters.temperature, - top_k=r.parameters.top_k, - top_p=r.parameters.top_p, - do_sample=r.parameters.do_sample, - ) - ) - stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) - - input_ids = tokenizer( - inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 - ).to(device) - all_input_ids = input_ids["input_ids"].unsqueeze(-1) - - return cls( - batch_id=pb.id, - requests=pb.requests, - all_input_lengths=all_input_lengths, - input_ids=input_ids, - all_input_ids=all_input_ids, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - size=pb.size, - max_sequence_length=pb.max_sequence_length, - ) - - @classmethod - def concatenate(cls, batches: List["Batch"]) -> "Batch": - # Used for padding - total_batch_size = sum(batch.size for batch in batches) - max_sequence_length = max(batch.max_sequence_length for batch in batches) - - # Batch attributes - input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} - requests = [] - all_input_lengths = [] - all_input_ids = [] - next_token_choosers = [] - stopping_criterias = [] - - # Used for slicing correctly inside the tensors - # Equivalent to a cumsum on batch sizes - start_index = 0 - for i, batch in enumerate(batches): - requests.extend(batch.requests) - all_input_lengths.extend(batch.all_input_lengths) - all_input_ids.extend(batch.all_input_ids) - next_token_choosers.extend(batch.next_token_choosers) - stopping_criterias.extend(batch.stopping_criterias) - - # Slicing end index for this batch - end_index = start_index + batch.size - - # We only concatenate batches that did at least one step - if batch.input_ids["input_ids"].shape[1] > 1: - raise ValueError("Batch input_ids should be of shape (batch_size, 1)") - - # Initialize tensors - if i == 0: - input_ids["input_ids"] = torch.empty( - (total_batch_size, 1), - dtype=batch.input_ids["input_ids"].dtype, - device=batch.input_ids["input_ids"].device, - ) - input_ids["attention_mask"] = torch.zeros( - (total_batch_size, max_sequence_length), - dtype=batch.input_ids["attention_mask"].dtype, - device=batch.input_ids["attention_mask"].device, - ) - - # input_ids["input_ids"] is always of shape [batch_size, 1] - # We do not need to pad it - input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] - - # We need to slice the attention mask to remove padding from previous steps - input_ids["attention_mask"][ - start_index:end_index, -batch.max_sequence_length : - ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length :] - - for j, past in enumerate(batch.input_ids["past_key_values"]): - past_keys = past[0] - past_values = past[1] - - _, head_dim, padded_sequence_length = past_keys.shape - - # Reshape the tensors to make slicing easier - past_keys = past_keys.view( - batch.size, -1, head_dim, padded_sequence_length - ) - past_values = past_values.view( - batch.size, -1, padded_sequence_length, head_dim - ) - num_heads = past_keys.shape[1] - - # Initialize tensors - # This will run only once per layer - if j == len(input_ids["past_key_values"]): - padded_past_keys = torch.zeros( - ( - total_batch_size, - num_heads, - head_dim, - max_sequence_length - 1, - ), - dtype=past_keys.dtype, - device=past_keys.device, - ) - padded_past_values = torch.zeros( - ( - total_batch_size, - num_heads, - max_sequence_length - 1, - head_dim, - ), - dtype=past_values.dtype, - device=past_values.device, - ) - input_ids["past_key_values"].append( - [padded_past_keys, padded_past_values] - ) - - # We slice the past keys and values to remove the padding from previous batches - input_ids["past_key_values"][j][0][ - start_index:end_index, :, :, -(batch.max_sequence_length - 1) : - ] = past_keys[:, :, :, -(batch.max_sequence_length - 1) :] - - input_ids["past_key_values"][j][1][ - start_index:end_index, :, -(batch.max_sequence_length - 1) :, : - ] = past_values[:, :, -(batch.max_sequence_length - 1) :, :] - - # If we are on the last batch, we need to reshape the tensors - if (i + 1) == len(batches): - input_ids["past_key_values"][j][0] = input_ids["past_key_values"][ - j - ][0].view(total_batch_size * num_heads, head_dim, -1) - input_ids["past_key_values"][j][1] = input_ids["past_key_values"][ - j - ][1].view(total_batch_size * num_heads, -1, head_dim) - - start_index += batch.size - - return cls( - batch_id=batches[0].batch_id, - requests=requests, - all_input_lengths=all_input_lengths, - input_ids=input_ids, - all_input_ids=all_input_ids, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - size=total_batch_size, - max_sequence_length=max_sequence_length, - ) - - -@dataclass -class GeneratedText: - request: generate_pb2.Request - output: str - - def to_pb(self) -> generate_pb2.GeneratedText: - return generate_pb2.GeneratedText(request=self.request, output=self.output) - - -class BLOOM: - def __init__(self, model_name: str): - if torch.cuda.is_available(): - self.device = torch.device("cuda") - dtype = torch.bfloat16 - else: - self.device = torch.device("cpu") - dtype = torch.float32 - - self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") - self.model = ( - AutoModelForCausalLM.from_pretrained(model_name) - .eval() - .to(self.device) - .to(dtype) - ) - self.num_heads = self.model.base_model.num_heads - - def forward(self, input_ids, attention_mask, past_key_values: Optional = None): - # Model Forward - return self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - use_cache=True, - ) - - def generate_token( - self, batch: Batch - ) -> Tuple[List[GeneratedText], Optional[Batch]]: - with torch.inference_mode(): - outputs = self.forward(**batch.input_ids) - - # List of indices to cache - next_batch_keep_indices = [] - next_batch_past_keep_indices = [] - - # New input_ids for next forward - next_batch_input_ids = [] - next_batch_all_input_ids = [] - next_all_input_lengths = [] - - next_batch_size = 0 - next_batch_max_sequence_length = 0 - - # Finished requests - generated_texts: List[GeneratedText] = [] - - # Zipped iterator - iterator = zip( - batch.requests, - batch.all_input_lengths, - outputs.logits, - batch.next_token_choosers, - batch.stopping_criterias, - batch.all_input_ids, - ) - - # For each member of the batch - for i, ( - request, - input_length, - logits, - next_token_chooser, - stopping_criteria, - all_tokens, - ) in enumerate(iterator): - # Select next token - next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) - - # Append next token to all tokens - all_tokens = torch.cat([all_tokens, next_token]) - - # Evaluate stopping criteria - if stopping_criteria(all_tokens): - # Decode all tokens - output = self.tokenizer.decode( - all_tokens.squeeze(-1), skip_special_tokens=True - ) - # Add to the list of finished generations with the original request - generated_texts.append(GeneratedText(request, output)) - # add to the next batch - else: - next_batch_keep_indices.append(i) - # past_key_values is of shape [batch_size * num_heads, ...] - # so we need to take into account the `num_heads` stride here - next_batch_past_keep_indices.extend( - [j for j in range(i * self.num_heads, (i + 1) * self.num_heads)] - ) - next_batch_input_ids.append(next_token) - next_batch_all_input_ids.append(all_tokens) - next_batch_size += 1 - new_input_length = input_length + 1 - next_all_input_lengths.append(new_input_length) - next_batch_max_sequence_length = max( - next_batch_max_sequence_length, new_input_length - ) - - # We finished all generations in the batch; there is no next batch - if not next_batch_keep_indices: - return generated_texts, None - - # If we finished at least one generation - next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} - if generated_texts: - # Apply indices to attention mask, past key values and other items that need to be cached - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ - next_batch_keep_indices - ] - next_batch_input_ids["past_key_values"] = [ - ( - keys[next_batch_past_keep_indices], - values[next_batch_past_keep_indices], - ) - for keys, values in outputs["past_key_values"] - ] - next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] - next_batch_next_token_choosers = [ - batch.next_token_choosers[i] for i in next_batch_keep_indices - ] - next_batch_stopping_criterias = [ - batch.stopping_criterias[i] for i in next_batch_keep_indices - ] - else: - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] - next_batch_input_ids["past_key_values"] = outputs["past_key_values"] - next_batch_requests = batch.requests - next_batch_next_token_choosers = batch.next_token_choosers - next_batch_stopping_criterias = batch.stopping_criterias - - # Update attention_mask with padding as we added a new token to input_ids - next_batch_input_ids["attention_mask"] = torch.cat( - [ - next_batch_input_ids["attention_mask"], - torch.ones((next_batch_size, 1)).to(self.device), - ], - dim=1, - ) - - next_batch = Batch( - batch_id=batch.batch_id, - requests=next_batch_requests, - all_input_lengths=next_all_input_lengths, - input_ids=next_batch_input_ids, - all_input_ids=next_batch_all_input_ids, - next_token_choosers=next_batch_next_token_choosers, - stopping_criterias=next_batch_stopping_criterias, - size=next_batch_size, - max_sequence_length=next_batch_max_sequence_length, - ) - return generated_texts, next_batch - - -class BLOOMSharded(BLOOM): - def __init__(self, model_name: str, quantize: bool = False): - super(BLOOM, self).__init__() - self.process_group, self.rank, self.world_size = initialize_torch_distributed() - self.master = self.rank == 0 - if torch.cuda.is_available(): - self.device = torch.device(f"cuda:{self.rank}") - dtype = torch.float16 - else: - self.device = torch.device("cpu") - dtype = torch.float32 - - self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") - - config = AutoConfig.from_pretrained( - model_name, slow_but_exact=False, tp_parallel=True - ) - config.pad_token_id = 3 - self.num_heads = config.n_head // self.process_group.size() - - # The flag below controls whether to allow TF32 on matmul. This flag defaults to False - # in PyTorch 1.12 and later. - torch.backends.cuda.matmul.allow_tf32 = True - - # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. - torch.backends.cudnn.allow_tf32 = True - - # Only download weights for small models - if self.master and model_name == "bigscience/bloom-560m": - download_weights(model_name) - - torch.distributed.barrier(group=self.process_group) - filenames = weight_files(model_name) - - with init_empty_weights(): - model = AutoModelForCausalLM.from_config(config) - - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=self.device, - rank=self.rank, - world_size=self.world_size, - ) - self.model = model.eval().to(dtype) - torch.distributed.barrier(group=self.process_group) - - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: bool, - device: torch.device, - rank: int, - world_size: int, - ): - parameters = dict(model.named_parameters()) - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" - ) as f: - for name in f.keys(): - full_name = f"transformer.{name}" - - module_name, param_name = full_name.rsplit(".", 1) - module = model.get_submodule(module_name) - current_tensor = parameters[full_name] - - slice_ = f.get_slice(name) - - if isinstance(module, TensorParallelColumnLinear): - if param_name == "weight": - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - tensor = tensor.transpose(1, 0) - else: - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif isinstance(module, TensorParallelRowLinear): - if param_name == "weight": - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - tensor = tensor.transpose(1, 0) - else: - tensor = slice_[:] - # XXX: Hack for Rowlinear to add the bias only once. - if rank != 0: - tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - tensor = slice_[:] - - if current_tensor.shape != tensor.shape: - raise ValueError( - f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" - ) - - tensor = tensor.contiguous() - - if quantize: - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine" - ) - - if ( - type(module) - in [TensorParallelRowLinear, TensorParallelColumnLinear] - and param_name == "weight" - ): - tensor = Int8Params( - tensor.transpose(1, 0), - has_fp16_weights=False, - requires_grad=False, - ).to(device) - state = bnb.MatmulLtState() - state.threshold = 6.0 - state.has_fp16_weights = False - state.memory_efficient_backward = False - state.use_pool = True - state.CB = tensor.CB - state.SCB = tensor.SCB - tensor.CB = None - tensor.SCB = None - - def replace_linear(state, in_features, out_features): - def linear(input, weight, bias): - size_out = input.size()[:-1] + (out_features,) - input = input.view(-1, in_features) - out = torch.empty( - size_out, device=input.device, dtype=input.dtype - ) - out = bnb.matmul( - input, - weight, - out=out.view(-1, out_features), - state=state, - threshold=state.threshold, - bias=bias, - ) - - if state.CB is not None: - # we converted 8-bit row major to turing/ampere format - # in the first inference pass - # we no longer need the row-major weight - del state.CB - weight.data = state.CxB - - return out.view(size_out) - - return linear - - module.linear = replace_linear( - state, module.in_features, module.out_features - ) - - else: - tensor = tensor.to(device) - - module._parameters[param_name] = tensor - if name == "word_embeddings.weight": - model.lm_head._parameters["weight"] = tensor - - def forward(self, input_ids, attention_mask, past_key_values: Optional = None): - outputs = self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - use_cache=True, - ) - - # Logits are sharded, so we need to gather them - logits_shard = outputs.logits[:, -1, :].contiguous() - - batch_size, vocab_shard_size = logits_shard.shape - vocab_size = self.world_size * vocab_shard_size - logits = [torch.empty_like(logits_shard) for _ in range(self.world_size)] - torch.distributed.all_gather(logits, logits_shard, group=self.process_group) - logits = torch.cat(logits, dim=1).view(batch_size, 1, vocab_size) - - outputs.logits = logits - return outputs diff --git a/server/pyproject.toml b/server/pyproject.toml index 4e5e98b3..50f99398 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,11 +1,11 @@ [tool.poetry] -name = "bloom-inference" +name = "text-generation" version = "0.1.0" description = "BLOOM Inference Python gRPC Server" authors = ["Olivier Dehaene "] [tool.poetry.scripts] -bloom-inference-server = 'bloom_inference.cli:app' +text-generation-server = 'text_generation.cli:app' [tool.poetry.dependencies] python = "^3.9" @@ -17,6 +17,9 @@ accelerate = "^0.12.0" joblib = "^1.2.0" bitsandbytes = "^0.35.1" +[tool.poetry.extras] +bnb = ["bitsandbytes"] + [tool.poetry.group.dev.dependencies] grpcio-tools = "^1.49.1" diff --git a/server/bloom_inference/__init__.py b/server/text_generation/__init__.py similarity index 100% rename from server/bloom_inference/__init__.py rename to server/text_generation/__init__.py diff --git a/server/bloom_inference/cache.py b/server/text_generation/cache.py similarity index 91% rename from server/bloom_inference/cache.py rename to server/text_generation/cache.py index 6812b306..65ec3e7c 100644 --- a/server/bloom_inference/cache.py +++ b/server/text_generation/cache.py @@ -1,6 +1,7 @@ -from bloom_inference.model import Batch from typing import Dict, Optional +from text_generation.models.types import Batch + class Cache: def __init__(self): diff --git a/server/bloom_inference/cli.py b/server/text_generation/cli.py similarity index 83% rename from server/bloom_inference/cli.py rename to server/text_generation/cli.py index 6360aec0..c41b9751 100644 --- a/server/bloom_inference/cli.py +++ b/server/text_generation/cli.py @@ -3,7 +3,7 @@ from pathlib import Path -from bloom_inference import server, utils +from text_generation import server, utils app = typer.Typer() @@ -13,7 +13,7 @@ def serve( model_name: str, sharded: bool = False, quantize: bool = False, - uds_path: Path = "/tmp/bloom-inference", + uds_path: Path = "/tmp/text-generation", ): if sharded: assert ( @@ -35,8 +35,9 @@ def serve( @app.command() def download_weights( model_name: str, + extension: str = ".safetensors", ): - utils.download_weights(model_name) + utils.download_weights(model_name, extension) if __name__ == "__main__": diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py new file mode 100644 index 00000000..d4f3cf8b --- /dev/null +++ b/server/text_generation/models/__init__.py @@ -0,0 +1,22 @@ +from text_generation.models.model import Model +from text_generation.models.bloom import BLOOMSharded + +__all__ = ["Model", "BLOOMSharded"] + + +def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: + + if model_name.startswith("bigscience/bloom"): + if sharded: + return BLOOMSharded(model_name, quantize) + else: + if quantize: + raise ValueError("quantization is not supported for non-sharded BLOOM") + return Model(model_name) + else: + if sharded: + raise ValueError("sharded is only supported for BLOOM") + if quantize: + raise ValueError("Quantization is only supported for BLOOM models") + + return Model(model_name) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py new file mode 100644 index 00000000..172ca38d --- /dev/null +++ b/server/text_generation/models/bloom.py @@ -0,0 +1,231 @@ +import torch +import torch.distributed + +from typing import List, Optional + +from accelerate import init_empty_weights +from safetensors import safe_open +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig +from transformers.models.bloom.parallel_layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, +) + +from text_generation.models import Model +from text_generation.utils import ( + initialize_torch_distributed, + weight_files, + download_weights, +) + +HAS_BITS_AND_BYTES = True +try: + import bitsandbytes as bnb + from bitsandbytes.nn import Int8Params +except Exception as e: + HAS_BITS_AND_BYTES = False + +torch.manual_seed(0) + + +class BLOOMSharded(Model): + def __init__(self, model_name: str, quantize: bool = False): + super(Model, self).__init__() + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + self.device = torch.device(f"cuda:{self.rank}") + dtype = torch.float16 + else: + self.device = torch.device("cpu") + dtype = torch.float32 + + self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + + config = AutoConfig.from_pretrained( + model_name, slow_but_exact=False, tp_parallel=True + ) + config.pad_token_id = 3 + self.num_heads = config.n_head // self.process_group.size() + + # The flag below controls whether to allow TF32 on matmul. This flag defaults to False + # in PyTorch 1.12 and later. + torch.backends.cuda.matmul.allow_tf32 = True + + # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. + torch.backends.cudnn.allow_tf32 = True + + # Only download weights for small models + if self.master and model_name == "bigscience/bloom-560m": + download_weights(model_name, extension=".safetensors") + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_name, extension=".safetensors") + + with init_empty_weights(): + model = AutoModelForCausalLM.from_config(config) + + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + quantize=quantize, + device=self.device, + rank=self.rank, + world_size=self.world_size, + ) + self.model = model.eval().to(dtype) + torch.distributed.barrier(group=self.process_group) + + @staticmethod + def load_weights( + model, + filenames: List[str], + quantize: bool, + device: torch.device, + rank: int, + world_size: int, + ): + parameters = dict(model.named_parameters()) + for file in filenames: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: + for name in f.keys(): + full_name = f"transformer.{name}" + + module_name, param_name = full_name.rsplit(".", 1) + module = model.get_submodule(module_name) + current_tensor = parameters[full_name] + + slice_ = f.get_slice(name) + + if isinstance(module, TensorParallelColumnLinear): + if param_name == "weight": + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + tensor = tensor.transpose(1, 0) + else: + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif isinstance(module, TensorParallelRowLinear): + if param_name == "weight": + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + tensor = tensor.transpose(1, 0) + else: + tensor = slice_[:] + # XXX: Hack for Rowlinear to add the bias only once. + if rank != 0: + tensor = torch.zeros_like(tensor) + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + else: + tensor = slice_[:] + + if current_tensor.shape != tensor.shape: + raise ValueError( + f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" + ) + + tensor = tensor.contiguous() + + if quantize: + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine either because it is not installed " + "or you don't have a GPU.\n" + "You can install it with `pip install bitsandbytes`." + ) + + if ( + type(module) + in [TensorParallelRowLinear, TensorParallelColumnLinear] + and param_name == "weight" + ): + tensor = Int8Params( + tensor.transpose(1, 0), + has_fp16_weights=False, + requires_grad=False, + ).to(device) + state = bnb.MatmulLtState() + state.threshold = 6.0 + state.has_fp16_weights = False + state.memory_efficient_backward = False + state.use_pool = True + state.CB = tensor.CB + state.SCB = tensor.SCB + tensor.CB = None + tensor.SCB = None + + def replace_linear(state, in_features, out_features): + def linear(input, weight, bias): + size_out = input.size()[:-1] + (out_features,) + input = input.view(-1, in_features) + out = torch.empty( + size_out, device=input.device, dtype=input.dtype + ) + out = bnb.matmul( + input, + weight, + out=out.view(-1, out_features), + state=state, + threshold=state.threshold, + bias=bias, + ) + + if state.CB is not None: + # we converted 8-bit row major to turing/ampere format + # in the first inference pass + # we no longer need the row-major weight + del state.CB + weight.data = state.CxB + + return out.view(size_out) + + return linear + + module.linear = replace_linear( + state, module.in_features, module.out_features + ) + + else: + tensor = tensor.to(device) + + module._parameters[param_name] = tensor + if name == "word_embeddings.weight": + model.lm_head._parameters["weight"] = tensor + + def forward(self, input_ids, attention_mask, past_key_values: Optional = None): + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + # Logits are sharded, so we need to gather them + logits_shard = outputs.logits[:, -1, :].contiguous() + + batch_size, vocab_shard_size = logits_shard.shape + vocab_size = self.world_size * vocab_shard_size + logits = [torch.empty_like(logits_shard) for _ in range(self.world_size)] + torch.distributed.all_gather(logits, logits_shard, group=self.process_group) + logits = torch.cat(logits, dim=1).view(batch_size, 1, vocab_size) + + outputs.logits = logits + return outputs diff --git a/server/text_generation/models/model.py b/server/text_generation/models/model.py new file mode 100644 index 00000000..db1367b9 --- /dev/null +++ b/server/text_generation/models/model.py @@ -0,0 +1,166 @@ +import torch +import torch.distributed + +from typing import List, Tuple, Optional +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig +from transformers.modeling_outputs import CausalLMOutputWithPast + +from text_generation.models.types import Batch, GeneratedText + + +class Model: + def __init__(self, model_name: str): + if torch.cuda.is_available(): + self.device = torch.device("cuda") + dtype = torch.float16 + else: + self.device = torch.device("cpu") + dtype = torch.float32 + + self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + self.tokenizer.add_special_tokens({"pad_token": "[PAD]"}) + self.model = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype=dtype, device_map="auto" + ).eval() + + self.num_heads = self.model.config.num_attention_heads + + def forward( + self, input_ids, attention_mask, past_key_values: Optional = None + ) -> CausalLMOutputWithPast: + # Model Forward + return self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + def generate_token( + self, batch: Batch + ) -> Tuple[List[GeneratedText], Optional[Batch]]: + # For some reason, inference_mode does not work well with GLOO which we use on CPU + context_manager = ( + torch.no_grad if self.device.type == "cpu" else torch.inference_mode + ) + with context_manager(): + outputs = self.forward(**batch.input_ids) + + # List of indices to cache + next_batch_keep_indices = [] + next_batch_past_keep_indices = [] + + # New input_ids for next forward + next_batch_input_ids = [] + next_batch_all_input_ids = [] + next_all_input_lengths = [] + + next_batch_size = 0 + next_batch_max_sequence_length = 0 + + # Finished requests + generated_texts: List[GeneratedText] = [] + + # Zipped iterator + iterator = zip( + batch.requests, + batch.all_input_lengths, + outputs.logits, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + ) + + # For each member of the batch + for i, ( + request, + input_length, + logits, + next_token_chooser, + stopping_criteria, + all_tokens, + ) in enumerate(iterator): + # Select next token + next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) + + # Append next token to all tokens + all_tokens = torch.cat([all_tokens, next_token]) + + # Evaluate stopping criteria + if stopping_criteria(all_tokens): + # Decode all tokens + output = self.tokenizer.decode( + all_tokens.squeeze(-1), skip_special_tokens=True + ) + # Add to the list of finished generations with the original request + generated_texts.append(GeneratedText(request, output)) + # add to the next batch + else: + next_batch_keep_indices.append(i) + # past_key_values is of shape [batch_size * num_heads, ...] + # so we need to take into account the `num_heads` stride here + next_batch_past_keep_indices.extend( + [j for j in range(i * self.num_heads, (i + 1) * self.num_heads)] + ) + next_batch_input_ids.append(next_token) + next_batch_all_input_ids.append(all_tokens) + next_batch_size += 1 + new_input_length = input_length + 1 + next_all_input_lengths.append(new_input_length) + next_batch_max_sequence_length = max( + next_batch_max_sequence_length, new_input_length + ) + + # We finished all generations in the batch; there is no next batch + if not next_batch_keep_indices: + return generated_texts, None + + # If we finished at least one generation + next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} + if generated_texts: + # Apply indices to attention mask, past key values and other items that need to be cached + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ + next_batch_keep_indices + ] + next_batch_input_ids["past_key_values"] = [ + ( + keys[next_batch_past_keep_indices], + values[next_batch_past_keep_indices], + ) + for keys, values in outputs["past_key_values"] + ] + next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] + next_batch_next_token_choosers = [ + batch.next_token_choosers[i] for i in next_batch_keep_indices + ] + next_batch_stopping_criterias = [ + batch.stopping_criterias[i] for i in next_batch_keep_indices + ] + else: + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] + next_batch_input_ids["past_key_values"] = outputs["past_key_values"] + next_batch_requests = batch.requests + next_batch_next_token_choosers = batch.next_token_choosers + next_batch_stopping_criterias = batch.stopping_criterias + + # Update attention_mask with padding as we added a new token to input_ids + next_batch_input_ids["attention_mask"] = torch.cat( + [ + next_batch_input_ids["attention_mask"], + torch.ones((next_batch_size, 1)).to(self.device), + ], + dim=1, + ) + + next_batch = Batch( + batch_id=batch.batch_id, + requests=next_batch_requests, + all_input_lengths=next_all_input_lengths, + input_ids=next_batch_input_ids, + all_input_ids=next_batch_all_input_ids, + next_token_choosers=next_batch_next_token_choosers, + stopping_criterias=next_batch_stopping_criterias, + size=next_batch_size, + max_sequence_length=next_batch_max_sequence_length, + ) + return generated_texts, next_batch diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py new file mode 100644 index 00000000..39c33ab7 --- /dev/null +++ b/server/text_generation/models/types.py @@ -0,0 +1,206 @@ +import torch + +from dataclasses import dataclass +from typing import List, Dict + +from transformers import AutoTokenizer + +from text_generation.pb import generate_pb2 +from text_generation.utils import NextTokenChooser, StoppingCriteria + + +@dataclass +class Batch: + batch_id: int + requests: List[generate_pb2.Request] + all_input_lengths: List[int] + input_ids: Dict[str, torch.Tensor] + all_input_ids: List[torch.Tensor] + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + size: int + max_sequence_length: int + + def to_pb(self): + return generate_pb2.Batch( + id=self.batch_id, + requests=self.requests, + size=self.size, + max_sequence_length=self.max_sequence_length, + ) + + @classmethod + def from_pb( + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + ) -> "Batch": + inputs = [] + next_token_choosers = [] + stopping_criterias = [] + all_input_lengths = [] + + # Parse batch + for r in pb.requests: + inputs.append(r.inputs) + all_input_lengths.append(r.input_length) + next_token_choosers.append( + NextTokenChooser( + temperature=r.parameters.temperature, + top_k=r.parameters.top_k, + top_p=r.parameters.top_p, + do_sample=r.parameters.do_sample, + ) + ) + stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) + + input_ids = tokenizer( + inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 + ).to(device) + all_input_ids = input_ids["input_ids"].unsqueeze(-1) + + return cls( + batch_id=pb.id, + requests=pb.requests, + all_input_lengths=all_input_lengths, + input_ids=input_ids, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=pb.size, + max_sequence_length=pb.max_sequence_length, + ) + + @classmethod + def concatenate(cls, batches: List["Batch"]) -> "Batch": + # Used for padding + total_batch_size = sum(batch.size for batch in batches) + max_sequence_length = max(batch.max_sequence_length for batch in batches) + + # Batch attributes + input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} + requests = [] + all_input_lengths = [] + all_input_ids = [] + next_token_choosers = [] + stopping_criterias = [] + + # Used for slicing correctly inside the tensors + # Equivalent to a cumsum on batch sizes + start_index = 0 + for i, batch in enumerate(batches): + requests.extend(batch.requests) + all_input_lengths.extend(batch.all_input_lengths) + all_input_ids.extend(batch.all_input_ids) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + + # Slicing end index for this batch + end_index = start_index + batch.size + + # We only concatenate batches that did at least one step + if batch.input_ids["input_ids"].shape[1] > 1: + raise ValueError("Batch input_ids should be of shape (batch_size, 1)") + + # Initialize tensors + if i == 0: + input_ids["input_ids"] = torch.empty( + (total_batch_size, 1), + dtype=batch.input_ids["input_ids"].dtype, + device=batch.input_ids["input_ids"].device, + ) + input_ids["attention_mask"] = torch.zeros( + (total_batch_size, max_sequence_length), + dtype=batch.input_ids["attention_mask"].dtype, + device=batch.input_ids["attention_mask"].device, + ) + + # input_ids["input_ids"] is always of shape [batch_size, 1] + # We do not need to pad it + input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] + + # We need to slice the attention mask to remove padding from previous steps + input_ids["attention_mask"][ + start_index:end_index, -batch.max_sequence_length : + ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length :] + + for j, past in enumerate(batch.input_ids["past_key_values"]): + past_keys = past[0] + past_values = past[1] + + _, head_dim, padded_sequence_length = past_keys.shape + + # Reshape the tensors to make slicing easier + past_keys = past_keys.view( + batch.size, -1, head_dim, padded_sequence_length + ) + past_values = past_values.view( + batch.size, -1, padded_sequence_length, head_dim + ) + num_heads = past_keys.shape[1] + + # Initialize tensors + # This will run only once per layer + if j == len(input_ids["past_key_values"]): + padded_past_keys = torch.zeros( + ( + total_batch_size, + num_heads, + head_dim, + max_sequence_length - 1, + ), + dtype=past_keys.dtype, + device=past_keys.device, + ) + padded_past_values = torch.zeros( + ( + total_batch_size, + num_heads, + max_sequence_length - 1, + head_dim, + ), + dtype=past_values.dtype, + device=past_values.device, + ) + input_ids["past_key_values"].append( + [padded_past_keys, padded_past_values] + ) + + # We slice the past keys and values to remove the padding from previous batches + input_ids["past_key_values"][j][0][ + start_index:end_index, :, :, -(batch.max_sequence_length - 1) : + ] = past_keys[:, :, :, -(batch.max_sequence_length - 1) :] + + input_ids["past_key_values"][j][1][ + start_index:end_index, :, -(batch.max_sequence_length - 1) :, : + ] = past_values[:, :, -(batch.max_sequence_length - 1) :, :] + + # If we are on the last batch, we need to reshape the tensors + if (i + 1) == len(batches): + input_ids["past_key_values"][j][0] = input_ids["past_key_values"][ + j + ][0].view(total_batch_size * num_heads, head_dim, -1) + input_ids["past_key_values"][j][1] = input_ids["past_key_values"][ + j + ][1].view(total_batch_size * num_heads, -1, head_dim) + + start_index += batch.size + + return cls( + batch_id=batches[0].batch_id, + requests=requests, + all_input_lengths=all_input_lengths, + input_ids=input_ids, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=total_batch_size, + max_sequence_length=max_sequence_length, + ) + + +@dataclass +class GeneratedText: + request: generate_pb2.Request + output: str + + def to_pb(self) -> generate_pb2.GeneratedText: + return generate_pb2.GeneratedText(request=self.request, output=self.output) diff --git a/server/bloom_inference/pb/.gitignore b/server/text_generation/pb/.gitignore similarity index 100% rename from server/bloom_inference/pb/.gitignore rename to server/text_generation/pb/.gitignore diff --git a/server/bloom_inference/server.py b/server/text_generation/server.py similarity index 83% rename from server/bloom_inference/server.py rename to server/text_generation/server.py index ad40a52a..fffeb0ba 100644 --- a/server/bloom_inference/server.py +++ b/server/text_generation/server.py @@ -5,15 +5,16 @@ from grpc_reflection.v1alpha import reflection from pathlib import Path -from typing import Optional, List +from typing import List -from bloom_inference.cache import Cache -from bloom_inference.model import BLOOM, Batch, BLOOMSharded -from bloom_inference.pb import generate_pb2_grpc, generate_pb2 +from text_generation.cache import Cache +from text_generation.models import Model, get_model +from text_generation.models.types import Batch +from text_generation.pb import generate_pb2_grpc, generate_pb2 class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): - def __init__(self, model: BLOOM, cache: Cache, server_urls: List[str]): + def __init__(self, model: Model, cache: Cache, server_urls: List[str]): self.cache = cache self.model = model self.server_urls = server_urls @@ -78,21 +79,17 @@ async def serve_inner( ): unix_socket_template = "unix://{}-{}" if sharded: - model = BLOOMSharded(model_name, quantize) server_urls = [ unix_socket_template.format(uds_path, rank) - for rank in range(model.world_size) + for rank in range(int(os.environ["WORLD_SIZE"])) ] - local_url = server_urls[model.rank] + local_url = server_urls[int(os.environ["RANK"])] else: - if quantize: - raise ValueError( - "bitsandbytes quantization is only available when running in `sharded` mode." - ) - model = BLOOM(model_name) local_url = unix_socket_template.format(uds_path, 0) server_urls = [local_url] + model = get_model(model_name, sharded, quantize) + server = aio.server() generate_pb2_grpc.add_TextGenerationServiceServicer_to_server( TextGenerationService(model, Cache(), server_urls), server diff --git a/server/bloom_inference/utils.py b/server/text_generation/utils.py similarity index 89% rename from server/bloom_inference/utils.py rename to server/text_generation/utils.py index cca5403f..926fc69c 100644 --- a/server/bloom_inference/utils.py +++ b/server/text_generation/utils.py @@ -92,19 +92,17 @@ def initialize_torch_distributed(): return torch.distributed.distributed_c10d._get_default_group(), rank, world_size -def weight_hub_files(model_name): +def weight_hub_files(model_name, extension=".safetensors"): """Get the safetensors filenames on the hub""" api = HfApi() info = api.model_info(model_name) - filenames = [ - s.rfilename for s in info.siblings if s.rfilename.endswith(".safetensors") - ] + filenames = [s.rfilename for s in info.siblings if s.rfilename.endswith(extension)] return filenames -def weight_files(model_name): +def weight_files(model_name, extension=".safetensors"): """Get the local safetensors filenames""" - filenames = weight_hub_files(model_name) + filenames = weight_hub_files(model_name, extension) files = [] for filename in filenames: cache_file = try_to_load_from_cache(model_name, filename=filename) @@ -112,16 +110,16 @@ def weight_files(model_name): raise LocalEntryNotFoundError( f"File {filename} of model {model_name} not found in " f"{os.getenv('HUGGINGFACE_HUB_CACHE', 'the local cache')}. " - f"Please run `bloom-inference-server download-weights {model_name}` first." + f"Please run `text-generation-server download-weights {model_name}` first." ) files.append(cache_file) return files -def download_weights(model_name): +def download_weights(model_name, extension=".safetensors"): """Download the safetensors files from the hub""" - filenames = weight_hub_files(model_name) + filenames = weight_hub_files(model_name, extension) download_function = partial( hf_hub_download, repo_id=model_name, local_files_only=False From b3b7ea0d74627d30a0b739c5e7e4a74b4f2f4437 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 2 Nov 2022 17:29:56 +0100 Subject: [PATCH 020/793] feat: Use json formatter by default in docker image --- Cargo.lock | 13 ++++++ Dockerfile | 2 +- README.md | 6 +-- launcher/Cargo.toml | 2 +- launcher/src/main.rs | 51 ++++++++++++++--------- router/Cargo.toml | 2 +- router/src/main.rs | 11 ++++- server/poetry.lock | 17 ++------ server/pyproject.toml | 1 - server/text_generation/models/__init__.py | 2 +- server/text_generation/models/model.py | 2 +- server/text_generation/utils.py | 13 +++--- 12 files changed, 74 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33d75f0e..8f09c3a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2175,6 +2175,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.16" @@ -2182,11 +2192,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ "nu-ansi-term", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] diff --git a/Dockerfile b/Dockerfile index a2bf199a..ebe79609 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,4 +73,4 @@ COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/loca # Install launcher COPY --from=launcher-builder /usr/local/cargo/bin/text-generation-launcher /usr/local/bin/text-generation-launcher -CMD HUGGINGFACE_HUB_CACHE=$MODEL_BASE_PATH text-generation-launcher --num-shard $NUM_GPUS \ No newline at end of file +CMD HUGGINGFACE_HUB_CACHE=$MODEL_BASE_PATH text-generation-launcher --num-shard $NUM_GPUS --json-output \ No newline at end of file diff --git a/README.md b/README.md index 2d2d49d3..0c4f6f71 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# LLM Text Generation Inference +# Text Generation Inference

@@ -6,12 +6,12 @@
-A Rust and gRPC server for large language models text generation inference. +A Rust and gRPC server for text generation inference. ## Features -- Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Dynamic bathing of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput +- Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading - 45ms per token generation for BLOOM with 8xA100 80GB diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index ae3558b1..1779c051 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -10,4 +10,4 @@ clap = { version = "4.0.15", features = ["derive", "env"] } ctrlc = "3.2.3" subprocess = "0.2.9" tracing = "0.1.37" -tracing-subscriber = "0.3.16" +tracing-subscriber = { version = "0.3.16", features = ["json"] } diff --git a/launcher/src/main.rs b/launcher/src/main.rs index f94dd589..ed6cf8e5 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -39,11 +39,11 @@ struct Args { master_addr: String, #[clap(default_value = "29500", long, env)] master_port: usize, + #[clap(long, env)] + json_output: bool, } fn main() -> ExitCode { - tracing_subscriber::fmt().compact().with_ansi(false).init(); - // Pattern match configuration let Args { model_name, @@ -57,8 +57,15 @@ fn main() -> ExitCode { shard_uds_path, master_addr, master_port, + json_output, } = Args::parse(); + if json_output { + tracing_subscriber::fmt().json().init(); + } else { + tracing_subscriber::fmt().compact().init(); + } + // By default we only have one master shard let num_shard = num_shard.unwrap_or(1); @@ -139,24 +146,30 @@ fn main() -> ExitCode { // All shard started // Start webserver tracing::info!("Starting Webserver"); + let mut argv = vec![ + "text-generation-router".to_string(), + "--max-concurrent-requests".to_string(), + max_concurrent_requests.to_string(), + "--max-input-length".to_string(), + max_input_length.to_string(), + "--max-batch-size".to_string(), + max_batch_size.to_string(), + "--max-waiting-tokens".to_string(), + max_waiting_tokens.to_string(), + "--port".to_string(), + port.to_string(), + "--master-shard-uds-path".to_string(), + format!("{}-0", shard_uds_path), + "--tokenizer-name".to_string(), + model_name, + ]; + + if json_output { + argv.push("--json-output".to_string()); + } + let mut webserver = match Popen::create( - &[ - "text-generation-router", - "--max-concurrent-requests", - &max_concurrent_requests.to_string(), - "--max-input-length", - &max_input_length.to_string(), - "--max-batch-size", - &max_batch_size.to_string(), - "--max-waiting-tokens", - &max_waiting_tokens.to_string(), - "--port", - &port.to_string(), - "--master-shard-uds-path", - &format!("{}-0", shard_uds_path), - "--tokenizer-name", - &model_name, - ], + &argv, PopenConfig { stdout: Redirection::Pipe, stderr: Redirection::Pipe, diff --git a/router/Cargo.toml b/router/Cargo.toml index da9518bf..f99069d3 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -24,5 +24,5 @@ thiserror = "1.0.37" tokenizers = "0.13.0" tokio = { version = "1.21.1", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } tracing = "0.1.36" -tracing-subscriber = "0.3.15" +tracing-subscriber = { version = "0.3.15", features = ["json"] } diff --git a/router/src/main.rs b/router/src/main.rs index b24ec4c9..6497f23e 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -25,6 +25,8 @@ struct Args { tokenizer_name: String, #[clap(default_value = "2", long, env)] validation_workers: usize, + #[clap(long, env)] + json_output: bool, } fn main() -> Result<(), std::io::Error> { @@ -40,11 +42,16 @@ fn main() -> Result<(), std::io::Error> { master_shard_uds_path, tokenizer_name, validation_workers, + json_output, } = args; - tracing_subscriber::fmt().compact().with_ansi(false).init(); + if json_output { + tracing_subscriber::fmt().json().init(); + } else { + tracing_subscriber::fmt().compact().init(); + } - if validation_workers == 1 { + if validation_workers == 0 { panic!("validation_workers must be > 0"); } diff --git a/server/poetry.lock b/server/poetry.lock index 5e635a67..3c92903e 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -88,14 +88,6 @@ grpcio = ">=1.50.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" -[[package]] -name = "joblib" -version = "1.2.0" -description = "Lightweight pipelining with Python functions" -category = "main" -optional = false -python-versions = ">=3.7" - [[package]] name = "numpy" version = "1.23.4" @@ -210,10 +202,13 @@ category = "main" optional = false python-versions = ">=3.7" +[extras] +bnb = ["bitsandbytes"] + [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "50d9d44577a0222f125c770732d5f88807378573bd7386036eb5c79fc2a7c552" +content-hash = "224b1e379d6105fe911bff4563946a90dfa6ff5918cf2e7be59f8d4f7c5cd7cf" [metadata.files] accelerate = [ @@ -330,10 +325,6 @@ grpcio-tools = [ {file = "grpcio_tools-1.50.0-cp39-cp39-win32.whl", hash = "sha256:e1a8f9a57bbcc2e633aaf327e39830527f3c1f7add18c7580f3058fe9a0fa780"}, {file = "grpcio_tools-1.50.0-cp39-cp39-win_amd64.whl", hash = "sha256:b7eb7a84d9171c0ae1550833f4a6ca52372bed9db0fa10f8c9dbe6ca65f97a8c"}, ] -joblib = [ - {file = "joblib-1.2.0-py3-none-any.whl", hash = "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385"}, - {file = "joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"}, -] numpy = [ {file = "numpy-1.23.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d79ada05005f6f4f337d3bb9de8a7774f259341c70bc88047a1f7b96a4bcb2"}, {file = "numpy-1.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926db372bc4ac1edf81cfb6c59e2a881606b409ddc0d0920b988174b2e2a767f"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 50f99398..e2ba98a7 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -14,7 +14,6 @@ grpcio = "^1.49.1" typer = "^0.6.1" grpcio-reflection = "^1.49.1" accelerate = "^0.12.0" -joblib = "^1.2.0" bitsandbytes = "^0.35.1" [tool.poetry.extras] diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index d4f3cf8b..1f141c3c 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -15,7 +15,7 @@ def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: return Model(model_name) else: if sharded: - raise ValueError("sharded is only supported for BLOOM") + raise ValueError("sharded is only supported for BLOOM models") if quantize: raise ValueError("Quantization is only supported for BLOOM models") diff --git a/server/text_generation/models/model.py b/server/text_generation/models/model.py index db1367b9..e585e476 100644 --- a/server/text_generation/models/model.py +++ b/server/text_generation/models/model.py @@ -20,7 +20,7 @@ def __init__(self, model_name: str): self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") self.tokenizer.add_special_tokens({"pad_token": "[PAD]"}) self.model = AutoModelForCausalLM.from_pretrained( - model_name, torch_dtype=dtype, device_map="auto" + model_name, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None ).eval() self.num_heads = self.model.config.num_attention_heads diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 926fc69c..50cd4124 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -1,11 +1,13 @@ +import concurrent import os +import signal import torch import torch.distributed from datetime import timedelta +from concurrent.futures import ThreadPoolExecutor from functools import partial -from joblib import Parallel, delayed from huggingface_hub import HfApi, hf_hub_download, try_to_load_from_cache from huggingface_hub.utils import LocalEntryNotFoundError from tqdm import tqdm @@ -124,8 +126,9 @@ def download_weights(model_name, extension=".safetensors"): download_function = partial( hf_hub_download, repo_id=model_name, local_files_only=False ) - # FIXME: fix the overlapping progress bars - files = Parallel(n_jobs=5)( - delayed(download_function)(filename=filename) for filename in tqdm(filenames) - ) + + executor = ThreadPoolExecutor(max_workers=5) + futures = [executor.submit(download_function, filename=filename) for filename in filenames] + files = [file for file in tqdm(concurrent.futures.as_completed(futures), total=len(futures))] + return files From 755fc0e4035af1d5d4ecd9530c057afccc16b93b Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 3 Nov 2022 16:07:54 +0100 Subject: [PATCH 021/793] fix(models): Revert buggy support for AutoModel --- README.md | 5 +- server/Makefile | 10 +- server/text_generation/models/__init__.py | 14 +- server/text_generation/models/bloom.py | 323 +++++++++++++++++++++- server/text_generation/models/model.py | 171 +----------- server/text_generation/models/types.py | 127 +-------- server/text_generation/server.py | 4 +- 7 files changed, 339 insertions(+), 315 deletions(-) diff --git a/README.md b/README.md index 0c4f6f71..c7733d05 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,11 @@ A Rust and gRPC server for text generation inference. - [Safetensors](https://github.com/huggingface/safetensors) weight loading - 45ms per token generation for BLOOM with 8xA100 80GB -## Officially supported models +## Supported models - BLOOM - BLOOM-560m -Other models are supported on a best-effort basis using `AutoModelForCausalLM.from_pretrained(, torch_dtype=torch.float16, device_map="auto")`. - ## Load Tests for BLOOM See `k6/load_test.js` @@ -82,6 +80,7 @@ make router-dev ## TODO: +- [ ] Support AutoModelForSeq2SeqLM - [ ] Add tests for the `server/model` logic - [ ] Backport custom CUDA kernels to Transformers - [ ] Install safetensors with pip \ No newline at end of file diff --git a/server/Makefile b/server/Makefile index 4fa966e2..99764028 100644 --- a/server/Makefile +++ b/server/Makefile @@ -9,11 +9,11 @@ gen-server: install-transformers: # Install specific version of transformers rm transformers || true - rm transformers-46d37bece7d3ffdef97b1ee4a3170c0a0627d921 || true - curl -L -O https://github.com/huggingface/transformers/archive/46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip - unzip 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip - rm 46d37bece7d3ffdef97b1ee4a3170c0a0627d921.zip - mv transformers-46d37bece7d3ffdef97b1ee4a3170c0a0627d921 transformers + rm transformers-7302a24535e8dc5637ea5b4e4572fc971d404098 || true + curl -L -O https://github.com/OlivierDehaene/transformers/archive/7302a24535e8dc5637ea5b4e4572fc971d404098.zip + unzip 7302a24535e8dc5637ea5b4e4572fc971d404098.zip + rm 7302a24535e8dc5637ea5b4e4572fc971d404098.zip + mv transformers-7302a24535e8dc5637ea5b4e4572fc971d404098 transformers cd transformers && python setup.py install install-safetensors: diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 1f141c3c..55315ec6 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -1,22 +1,16 @@ from text_generation.models.model import Model -from text_generation.models.bloom import BLOOMSharded +from text_generation.models.bloom import BLOOM, BLOOMSharded -__all__ = ["Model", "BLOOMSharded"] +__all__ = ["Model", "BLOOM", "BLOOMSharded"] def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: - if model_name.startswith("bigscience/bloom"): if sharded: return BLOOMSharded(model_name, quantize) else: if quantize: raise ValueError("quantization is not supported for non-sharded BLOOM") - return Model(model_name) + return BLOOM(model_name) else: - if sharded: - raise ValueError("sharded is only supported for BLOOM models") - if quantize: - raise ValueError("Quantization is only supported for BLOOM models") - - return Model(model_name) + raise ValueError(f"model {model_name} is not supported yet") diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 172ca38d..ac2c6169 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -1,7 +1,7 @@ import torch import torch.distributed -from typing import List, Optional +from typing import List, Optional, Tuple, Type from accelerate import init_empty_weights from safetensors import safe_open @@ -11,8 +11,10 @@ TensorParallelEmbedding, TensorParallelRowLinear, ) +from transformers.modeling_outputs import CausalLMOutputWithPast from text_generation.models import Model +from text_generation.models.types import Batch, GeneratedText from text_generation.utils import ( initialize_torch_distributed, weight_files, @@ -29,9 +31,306 @@ torch.manual_seed(0) -class BLOOMSharded(Model): +class BloomBatch(Batch): + @classmethod + def concatenate(cls, batches: List["Batch"]) -> "BloomBatch": + # Used for padding + total_batch_size = sum(batch.size for batch in batches) + max_sequence_length = max(batch.max_sequence_length for batch in batches) + + # Batch attributes + input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} + requests = [] + all_input_lengths = [] + all_input_ids = [] + next_token_choosers = [] + stopping_criterias = [] + + # Used for slicing correctly inside the tensors + # Equivalent to a cumsum on batch sizes + start_index = 0 + for i, batch in enumerate(batches): + requests.extend(batch.requests) + all_input_lengths.extend(batch.all_input_lengths) + all_input_ids.extend(batch.all_input_ids) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + + # Slicing end index for this batch + end_index = start_index + batch.size + + # We only concatenate batches that did at least one step + if batch.input_ids["input_ids"].shape[1] > 1: + raise ValueError("Batch input_ids should be of shape (batch_size, 1)") + + # Initialize tensors + if i == 0: + input_ids["input_ids"] = torch.empty( + (total_batch_size, 1), + dtype=batch.input_ids["input_ids"].dtype, + device=batch.input_ids["input_ids"].device, + ) + input_ids["attention_mask"] = torch.zeros( + (total_batch_size, max_sequence_length), + dtype=batch.input_ids["attention_mask"].dtype, + device=batch.input_ids["attention_mask"].device, + ) + + # input_ids["input_ids"] is always of shape [batch_size, 1] + # We do not need to pad it + input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] + + # We need to slice the attention mask to remove padding from previous steps + input_ids["attention_mask"][ + start_index:end_index, -batch.max_sequence_length: + ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length:] + + for j, past in enumerate(batch.input_ids["past_key_values"]): + past_keys = past[0] + past_values = past[1] + + _, head_dim, padded_sequence_length = past_keys.shape + + # Reshape the tensors to make slicing easier + past_keys = past_keys.view( + batch.size, -1, head_dim, padded_sequence_length + ) + past_values = past_values.view( + batch.size, -1, padded_sequence_length, head_dim + ) + num_heads = past_keys.shape[1] + + # Initialize tensors + # This will run only once per layer + if j == len(input_ids["past_key_values"]): + padded_past_keys = torch.zeros( + ( + total_batch_size, + num_heads, + head_dim, + max_sequence_length - 1, + ), + dtype=past_keys.dtype, + device=past_keys.device, + ) + padded_past_values = torch.zeros( + ( + total_batch_size, + num_heads, + max_sequence_length - 1, + head_dim, + ), + dtype=past_values.dtype, + device=past_values.device, + ) + input_ids["past_key_values"].append( + [padded_past_keys, padded_past_values] + ) + + # We slice the past keys and values to remove the padding from previous batches + input_ids["past_key_values"][j][0][ + start_index:end_index, :, :, -(batch.max_sequence_length - 1): + ] = past_keys[:, :, :, -(batch.max_sequence_length - 1):] + + input_ids["past_key_values"][j][1][ + start_index:end_index, :, -(batch.max_sequence_length - 1):, : + ] = past_values[:, :, -(batch.max_sequence_length - 1):, :] + + # If we are on the last batch, we need to reshape the tensors + if (i + 1) == len(batches): + input_ids["past_key_values"][j][0] = input_ids["past_key_values"][ + j + ][0].view(total_batch_size * num_heads, head_dim, -1) + input_ids["past_key_values"][j][1] = input_ids["past_key_values"][ + j + ][1].view(total_batch_size * num_heads, -1, head_dim) + + start_index += batch.size + + return cls( + batch_id=batches[0].batch_id, + requests=requests, + all_input_lengths=all_input_lengths, + input_ids=input_ids, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=total_batch_size, + max_sequence_length=max_sequence_length, + ) + + +class BLOOM(Model): + def __init__(self, model_name: str): + if not model_name.startswith("bigscience/bloom"): + raise ValueError(f"Model {model_name} is not supported") + + if torch.cuda.is_available(): + self.device = torch.device("cuda") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + else: + self.device = torch.device("cpu") + dtype = torch.float32 + + self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + self.tokenizer.add_special_tokens({"pad_token": "[PAD]"}) + self.model = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None + ).eval() + + self.num_heads = self.model.config.num_attention_heads + + @property + def batch_type(self) -> Type[BloomBatch]: + return BloomBatch + + def forward( + self, input_ids, attention_mask, past_key_values: Optional = None + ) -> CausalLMOutputWithPast: + # Model Forward + return self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + def generate_token( + self, batch: BloomBatch + ) -> Tuple[List[GeneratedText], Optional[BloomBatch]]: + # For some reason, inference_mode does not work well with GLOO which we use on CPU + context_manager = ( + torch.no_grad if self.device.type == "cpu" else torch.inference_mode + ) + with context_manager(): + outputs = self.forward(**batch.input_ids) + + # List of indices to cache + next_batch_keep_indices = [] + next_batch_past_keep_indices = [] + + # New input_ids for next forward + next_batch_input_ids = [] + next_batch_all_input_ids = [] + next_all_input_lengths = [] + + next_batch_size = 0 + next_batch_max_sequence_length = 0 + + # Finished requests + generated_texts: List[GeneratedText] = [] + + # Zipped iterator + iterator = zip( + batch.requests, + batch.all_input_lengths, + outputs.logits, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + ) + + # For each member of the batch + for i, ( + request, + input_length, + logits, + next_token_chooser, + stopping_criteria, + all_tokens, + ) in enumerate(iterator): + # Select next token + next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) + + # Append next token to all tokens + all_tokens = torch.cat([all_tokens, next_token]) + + # Evaluate stopping criteria + if stopping_criteria(all_tokens): + # Decode all tokens + output = self.tokenizer.decode( + all_tokens.squeeze(-1), skip_special_tokens=True + ) + # Add to the list of finished generations with the original request + generated_texts.append(GeneratedText(request, output)) + # add to the next batch + else: + next_batch_keep_indices.append(i) + # past_key_values is of shape [batch_size * num_heads, ...] + # so we need to take into account the `num_heads` stride here + next_batch_past_keep_indices.extend( + [j for j in range(i * self.num_heads, (i + 1) * self.num_heads)] + ) + next_batch_input_ids.append(next_token) + next_batch_all_input_ids.append(all_tokens) + next_batch_size += 1 + new_input_length = input_length + 1 + next_all_input_lengths.append(new_input_length) + next_batch_max_sequence_length = max( + next_batch_max_sequence_length, new_input_length + ) + + # We finished all generations in the batch; there is no next batch + if not next_batch_keep_indices: + return generated_texts, None + + # If we finished at least one generation + next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} + if generated_texts: + # Apply indices to attention mask, past key values and other items that need to be cached + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ + next_batch_keep_indices + ] + next_batch_input_ids["past_key_values"] = [ + ( + keys[next_batch_past_keep_indices], + values[next_batch_past_keep_indices], + ) + for keys, values in outputs["past_key_values"] + ] + next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] + next_batch_next_token_choosers = [ + batch.next_token_choosers[i] for i in next_batch_keep_indices + ] + next_batch_stopping_criterias = [ + batch.stopping_criterias[i] for i in next_batch_keep_indices + ] + else: + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] + next_batch_input_ids["past_key_values"] = outputs["past_key_values"] + next_batch_requests = batch.requests + next_batch_next_token_choosers = batch.next_token_choosers + next_batch_stopping_criterias = batch.stopping_criterias + + # Update attention_mask with padding as we added a new token to input_ids + next_batch_input_ids["attention_mask"] = torch.cat( + [ + next_batch_input_ids["attention_mask"], + torch.ones((next_batch_size, 1)).to(self.device), + ], + dim=1, + ) + + next_batch = BloomBatch( + batch_id=batch.batch_id, + requests=next_batch_requests, + all_input_lengths=next_all_input_lengths, + input_ids=next_batch_input_ids, + all_input_ids=next_batch_all_input_ids, + next_token_choosers=next_batch_next_token_choosers, + stopping_criterias=next_batch_stopping_criterias, + size=next_batch_size, + max_sequence_length=next_batch_max_sequence_length, + ) + return generated_texts, next_batch + + +class BLOOMSharded(BLOOM): def __init__(self, model_name: str, quantize: bool = False): super(Model, self).__init__() + if not model_name.startswith("bigscience/bloom"): + raise ValueError(f"Model {model_name} is not supported") + self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 if torch.cuda.is_available(): @@ -80,17 +379,17 @@ def __init__(self, model_name: str, quantize: bool = False): @staticmethod def load_weights( - model, - filenames: List[str], - quantize: bool, - device: torch.device, - rank: int, - world_size: int, + model, + filenames: List[str], + quantize: bool, + device: torch.device, + rank: int, + world_size: int, ): parameters = dict(model.named_parameters()) for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if not quantize else "cpu" ) as f: for name in f.keys(): full_name = f"transformer.{name}" @@ -153,9 +452,9 @@ def load_weights( ) if ( - type(module) - in [TensorParallelRowLinear, TensorParallelColumnLinear] - and param_name == "weight" + type(module) + in [TensorParallelRowLinear, TensorParallelColumnLinear] + and param_name == "weight" ): tensor = Int8Params( tensor.transpose(1, 0), diff --git a/server/text_generation/models/model.py b/server/text_generation/models/model.py index e585e476..ad094197 100644 --- a/server/text_generation/models/model.py +++ b/server/text_generation/models/model.py @@ -1,166 +1,19 @@ -import torch -import torch.distributed - -from typing import List, Tuple, Optional -from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig -from transformers.modeling_outputs import CausalLMOutputWithPast +from abc import ABC, abstractmethod +from typing import List, Tuple, Optional, TypeVar, Type from text_generation.models.types import Batch, GeneratedText +B = TypeVar("B", bound=Batch) -class Model: - def __init__(self, model_name: str): - if torch.cuda.is_available(): - self.device = torch.device("cuda") - dtype = torch.float16 - else: - self.device = torch.device("cpu") - dtype = torch.float32 - - self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") - self.tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - self.model = AutoModelForCausalLM.from_pretrained( - model_name, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None - ).eval() - self.num_heads = self.model.config.num_attention_heads - - def forward( - self, input_ids, attention_mask, past_key_values: Optional = None - ) -> CausalLMOutputWithPast: - # Model Forward - return self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - use_cache=True, - ) +class Model(ABC): + @property + @abstractmethod + def batch_type(self) -> Type[B]: + raise NotImplementedError + @abstractmethod def generate_token( - self, batch: Batch - ) -> Tuple[List[GeneratedText], Optional[Batch]]: - # For some reason, inference_mode does not work well with GLOO which we use on CPU - context_manager = ( - torch.no_grad if self.device.type == "cpu" else torch.inference_mode - ) - with context_manager(): - outputs = self.forward(**batch.input_ids) - - # List of indices to cache - next_batch_keep_indices = [] - next_batch_past_keep_indices = [] - - # New input_ids for next forward - next_batch_input_ids = [] - next_batch_all_input_ids = [] - next_all_input_lengths = [] - - next_batch_size = 0 - next_batch_max_sequence_length = 0 - - # Finished requests - generated_texts: List[GeneratedText] = [] - - # Zipped iterator - iterator = zip( - batch.requests, - batch.all_input_lengths, - outputs.logits, - batch.next_token_choosers, - batch.stopping_criterias, - batch.all_input_ids, - ) - - # For each member of the batch - for i, ( - request, - input_length, - logits, - next_token_chooser, - stopping_criteria, - all_tokens, - ) in enumerate(iterator): - # Select next token - next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) - - # Append next token to all tokens - all_tokens = torch.cat([all_tokens, next_token]) - - # Evaluate stopping criteria - if stopping_criteria(all_tokens): - # Decode all tokens - output = self.tokenizer.decode( - all_tokens.squeeze(-1), skip_special_tokens=True - ) - # Add to the list of finished generations with the original request - generated_texts.append(GeneratedText(request, output)) - # add to the next batch - else: - next_batch_keep_indices.append(i) - # past_key_values is of shape [batch_size * num_heads, ...] - # so we need to take into account the `num_heads` stride here - next_batch_past_keep_indices.extend( - [j for j in range(i * self.num_heads, (i + 1) * self.num_heads)] - ) - next_batch_input_ids.append(next_token) - next_batch_all_input_ids.append(all_tokens) - next_batch_size += 1 - new_input_length = input_length + 1 - next_all_input_lengths.append(new_input_length) - next_batch_max_sequence_length = max( - next_batch_max_sequence_length, new_input_length - ) - - # We finished all generations in the batch; there is no next batch - if not next_batch_keep_indices: - return generated_texts, None - - # If we finished at least one generation - next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} - if generated_texts: - # Apply indices to attention mask, past key values and other items that need to be cached - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ - next_batch_keep_indices - ] - next_batch_input_ids["past_key_values"] = [ - ( - keys[next_batch_past_keep_indices], - values[next_batch_past_keep_indices], - ) - for keys, values in outputs["past_key_values"] - ] - next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] - next_batch_next_token_choosers = [ - batch.next_token_choosers[i] for i in next_batch_keep_indices - ] - next_batch_stopping_criterias = [ - batch.stopping_criterias[i] for i in next_batch_keep_indices - ] - else: - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] - next_batch_input_ids["past_key_values"] = outputs["past_key_values"] - next_batch_requests = batch.requests - next_batch_next_token_choosers = batch.next_token_choosers - next_batch_stopping_criterias = batch.stopping_criterias - - # Update attention_mask with padding as we added a new token to input_ids - next_batch_input_ids["attention_mask"] = torch.cat( - [ - next_batch_input_ids["attention_mask"], - torch.ones((next_batch_size, 1)).to(self.device), - ], - dim=1, - ) - - next_batch = Batch( - batch_id=batch.batch_id, - requests=next_batch_requests, - all_input_lengths=next_all_input_lengths, - input_ids=next_batch_input_ids, - all_input_ids=next_batch_all_input_ids, - next_token_choosers=next_batch_next_token_choosers, - stopping_criterias=next_batch_stopping_criterias, - size=next_batch_size, - max_sequence_length=next_batch_max_sequence_length, - ) - return generated_texts, next_batch + self, batch: B + ) -> Tuple[List[GeneratedText], Optional[B]]: + raise NotImplementedError diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index 39c33ab7..c2647a8d 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -1,5 +1,6 @@ import torch +from abc import abstractmethod from dataclasses import dataclass from typing import List, Dict @@ -70,131 +71,9 @@ def from_pb( ) @classmethod + @abstractmethod def concatenate(cls, batches: List["Batch"]) -> "Batch": - # Used for padding - total_batch_size = sum(batch.size for batch in batches) - max_sequence_length = max(batch.max_sequence_length for batch in batches) - - # Batch attributes - input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} - requests = [] - all_input_lengths = [] - all_input_ids = [] - next_token_choosers = [] - stopping_criterias = [] - - # Used for slicing correctly inside the tensors - # Equivalent to a cumsum on batch sizes - start_index = 0 - for i, batch in enumerate(batches): - requests.extend(batch.requests) - all_input_lengths.extend(batch.all_input_lengths) - all_input_ids.extend(batch.all_input_ids) - next_token_choosers.extend(batch.next_token_choosers) - stopping_criterias.extend(batch.stopping_criterias) - - # Slicing end index for this batch - end_index = start_index + batch.size - - # We only concatenate batches that did at least one step - if batch.input_ids["input_ids"].shape[1] > 1: - raise ValueError("Batch input_ids should be of shape (batch_size, 1)") - - # Initialize tensors - if i == 0: - input_ids["input_ids"] = torch.empty( - (total_batch_size, 1), - dtype=batch.input_ids["input_ids"].dtype, - device=batch.input_ids["input_ids"].device, - ) - input_ids["attention_mask"] = torch.zeros( - (total_batch_size, max_sequence_length), - dtype=batch.input_ids["attention_mask"].dtype, - device=batch.input_ids["attention_mask"].device, - ) - - # input_ids["input_ids"] is always of shape [batch_size, 1] - # We do not need to pad it - input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] - - # We need to slice the attention mask to remove padding from previous steps - input_ids["attention_mask"][ - start_index:end_index, -batch.max_sequence_length : - ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length :] - - for j, past in enumerate(batch.input_ids["past_key_values"]): - past_keys = past[0] - past_values = past[1] - - _, head_dim, padded_sequence_length = past_keys.shape - - # Reshape the tensors to make slicing easier - past_keys = past_keys.view( - batch.size, -1, head_dim, padded_sequence_length - ) - past_values = past_values.view( - batch.size, -1, padded_sequence_length, head_dim - ) - num_heads = past_keys.shape[1] - - # Initialize tensors - # This will run only once per layer - if j == len(input_ids["past_key_values"]): - padded_past_keys = torch.zeros( - ( - total_batch_size, - num_heads, - head_dim, - max_sequence_length - 1, - ), - dtype=past_keys.dtype, - device=past_keys.device, - ) - padded_past_values = torch.zeros( - ( - total_batch_size, - num_heads, - max_sequence_length - 1, - head_dim, - ), - dtype=past_values.dtype, - device=past_values.device, - ) - input_ids["past_key_values"].append( - [padded_past_keys, padded_past_values] - ) - - # We slice the past keys and values to remove the padding from previous batches - input_ids["past_key_values"][j][0][ - start_index:end_index, :, :, -(batch.max_sequence_length - 1) : - ] = past_keys[:, :, :, -(batch.max_sequence_length - 1) :] - - input_ids["past_key_values"][j][1][ - start_index:end_index, :, -(batch.max_sequence_length - 1) :, : - ] = past_values[:, :, -(batch.max_sequence_length - 1) :, :] - - # If we are on the last batch, we need to reshape the tensors - if (i + 1) == len(batches): - input_ids["past_key_values"][j][0] = input_ids["past_key_values"][ - j - ][0].view(total_batch_size * num_heads, head_dim, -1) - input_ids["past_key_values"][j][1] = input_ids["past_key_values"][ - j - ][1].view(total_batch_size * num_heads, -1, head_dim) - - start_index += batch.size - - return cls( - batch_id=batches[0].batch_id, - requests=requests, - all_input_lengths=all_input_lengths, - input_ids=input_ids, - all_input_ids=all_input_ids, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - size=total_batch_size, - max_sequence_length=max_sequence_length, - ) + raise NotImplementedError @dataclass diff --git a/server/text_generation/server.py b/server/text_generation/server.py index fffeb0ba..b2b34cb1 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -27,7 +27,7 @@ async def ClearCache(self, request, context): return generate_pb2.ClearCacheResponse() async def Generate(self, request, context): - batch = Batch.from_pb(request.batch, self.model.tokenizer, self.model.device) + batch = self.model.batch_type.from_pb(request.batch, self.model.tokenizer, self.model.device) generated_texts, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) @@ -51,7 +51,7 @@ async def GenerateWithCache(self, request, context): batches.append(batch) if len(batches) > 1: - batch = Batch.concatenate(batches) + batch = self.model.batch_type.concatenate(batches) else: batch = batches[0] From c5665f5c8be801237b0f7bac34da42371a43b9ad Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 4 Nov 2022 14:22:47 +0100 Subject: [PATCH 022/793] feat(server): Support generic AutoModelForCausalLM --- README.md | 1 + proto/generate.proto | 2 + router/src/batcher.rs | 2 + router/src/server.rs | 2 +- server/text_generation/models/__init__.py | 13 +- server/text_generation/models/bloom.py | 315 +-------------------- server/text_generation/models/causal_lm.py | 38 +++ server/text_generation/models/model.py | 138 ++++++++- server/text_generation/models/types.py | 171 ++++++++++- server/text_generation/server.py | 4 +- server/text_generation/utils.py | 18 +- 11 files changed, 372 insertions(+), 332 deletions(-) create mode 100644 server/text_generation/models/causal_lm.py diff --git a/README.md b/README.md index c7733d05..29e58357 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ A Rust and gRPC server for text generation inference. ## Supported models - BLOOM +- BLOOMZ - BLOOM-560m ## Load Tests for BLOOM diff --git a/proto/generate.proto b/proto/generate.proto index 45afca82..68dfa151 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -63,6 +63,8 @@ message GeneratedText { Request request = 1; /// Output string output = 2; + /// Number of generated tokens + uint32 tokens = 3; } message GenerateRequest { diff --git a/router/src/batcher.rs b/router/src/batcher.rs index e381986a..074c54bc 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -190,6 +190,7 @@ fn send_generated(finished: Vec, db: &Db) { .expect("ID not found in db. This is a bug."); let response = InferResponse { output: output.output, + tokens: output.tokens, queued: entry.time, start: entry.batch_time.unwrap(), // unwrap is always valid end: Instant::now(), @@ -202,6 +203,7 @@ fn send_generated(finished: Vec, db: &Db) { #[derive(Debug)] pub(crate) struct InferResponse { pub(crate) output: String, + pub(crate) tokens: u32, pub(crate) queued: Instant, pub(crate) start: Instant, pub(crate) end: Instant, diff --git a/router/src/server.rs b/router/src/server.rs index d31c9494..72b720ef 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -116,7 +116,7 @@ async fn generate( let validation_time = response.queued - start_time; let queue_time = response.start - response.queued; let inference_time = response.end - response.start; - let time_per_token = inference_time / req.parameters.max_new_tokens; + let time_per_token = inference_time / response.tokens; // Headers let mut headers = HeaderMap::new(); diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 55315ec6..0371ab4c 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -1,7 +1,8 @@ from text_generation.models.model import Model -from text_generation.models.bloom import BLOOM, BLOOMSharded +from text_generation.models.bloom import BLOOMSharded +from text_generation.models.causal_lm import CausalLM -__all__ = ["Model", "BLOOM", "BLOOMSharded"] +__all__ = ["Model", "BLOOMSharded", "CausalLM"] def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: @@ -11,6 +12,10 @@ def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: else: if quantize: raise ValueError("quantization is not supported for non-sharded BLOOM") - return BLOOM(model_name) + return CausalLM(model_name) else: - raise ValueError(f"model {model_name} is not supported yet") + if sharded: + raise ValueError("sharded is not supported for AutoModel") + if quantize: + raise ValueError("quantize is not supported for AutoModel") + return CausalLM(model_name) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index ac2c6169..37f55f2f 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -1,7 +1,7 @@ import torch import torch.distributed -from typing import List, Optional, Tuple, Type +from typing import List, Optional from accelerate import init_empty_weights from safetensors import safe_open @@ -11,10 +11,8 @@ TensorParallelEmbedding, TensorParallelRowLinear, ) -from transformers.modeling_outputs import CausalLMOutputWithPast from text_generation.models import Model -from text_generation.models.types import Batch, GeneratedText from text_generation.utils import ( initialize_torch_distributed, weight_files, @@ -31,322 +29,26 @@ torch.manual_seed(0) -class BloomBatch(Batch): - @classmethod - def concatenate(cls, batches: List["Batch"]) -> "BloomBatch": - # Used for padding - total_batch_size = sum(batch.size for batch in batches) - max_sequence_length = max(batch.max_sequence_length for batch in batches) - - # Batch attributes - input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} - requests = [] - all_input_lengths = [] - all_input_ids = [] - next_token_choosers = [] - stopping_criterias = [] - - # Used for slicing correctly inside the tensors - # Equivalent to a cumsum on batch sizes - start_index = 0 - for i, batch in enumerate(batches): - requests.extend(batch.requests) - all_input_lengths.extend(batch.all_input_lengths) - all_input_ids.extend(batch.all_input_ids) - next_token_choosers.extend(batch.next_token_choosers) - stopping_criterias.extend(batch.stopping_criterias) - - # Slicing end index for this batch - end_index = start_index + batch.size - - # We only concatenate batches that did at least one step - if batch.input_ids["input_ids"].shape[1] > 1: - raise ValueError("Batch input_ids should be of shape (batch_size, 1)") - - # Initialize tensors - if i == 0: - input_ids["input_ids"] = torch.empty( - (total_batch_size, 1), - dtype=batch.input_ids["input_ids"].dtype, - device=batch.input_ids["input_ids"].device, - ) - input_ids["attention_mask"] = torch.zeros( - (total_batch_size, max_sequence_length), - dtype=batch.input_ids["attention_mask"].dtype, - device=batch.input_ids["attention_mask"].device, - ) - - # input_ids["input_ids"] is always of shape [batch_size, 1] - # We do not need to pad it - input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] - - # We need to slice the attention mask to remove padding from previous steps - input_ids["attention_mask"][ - start_index:end_index, -batch.max_sequence_length: - ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length:] - - for j, past in enumerate(batch.input_ids["past_key_values"]): - past_keys = past[0] - past_values = past[1] - - _, head_dim, padded_sequence_length = past_keys.shape - - # Reshape the tensors to make slicing easier - past_keys = past_keys.view( - batch.size, -1, head_dim, padded_sequence_length - ) - past_values = past_values.view( - batch.size, -1, padded_sequence_length, head_dim - ) - num_heads = past_keys.shape[1] - - # Initialize tensors - # This will run only once per layer - if j == len(input_ids["past_key_values"]): - padded_past_keys = torch.zeros( - ( - total_batch_size, - num_heads, - head_dim, - max_sequence_length - 1, - ), - dtype=past_keys.dtype, - device=past_keys.device, - ) - padded_past_values = torch.zeros( - ( - total_batch_size, - num_heads, - max_sequence_length - 1, - head_dim, - ), - dtype=past_values.dtype, - device=past_values.device, - ) - input_ids["past_key_values"].append( - [padded_past_keys, padded_past_values] - ) - - # We slice the past keys and values to remove the padding from previous batches - input_ids["past_key_values"][j][0][ - start_index:end_index, :, :, -(batch.max_sequence_length - 1): - ] = past_keys[:, :, :, -(batch.max_sequence_length - 1):] - - input_ids["past_key_values"][j][1][ - start_index:end_index, :, -(batch.max_sequence_length - 1):, : - ] = past_values[:, :, -(batch.max_sequence_length - 1):, :] - - # If we are on the last batch, we need to reshape the tensors - if (i + 1) == len(batches): - input_ids["past_key_values"][j][0] = input_ids["past_key_values"][ - j - ][0].view(total_batch_size * num_heads, head_dim, -1) - input_ids["past_key_values"][j][1] = input_ids["past_key_values"][ - j - ][1].view(total_batch_size * num_heads, -1, head_dim) - - start_index += batch.size - - return cls( - batch_id=batches[0].batch_id, - requests=requests, - all_input_lengths=all_input_lengths, - input_ids=input_ids, - all_input_ids=all_input_ids, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - size=total_batch_size, - max_sequence_length=max_sequence_length, - ) - - -class BLOOM(Model): - def __init__(self, model_name: str): - if not model_name.startswith("bigscience/bloom"): - raise ValueError(f"Model {model_name} is not supported") - - if torch.cuda.is_available(): - self.device = torch.device("cuda") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 - else: - self.device = torch.device("cpu") - dtype = torch.float32 - - self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") - self.tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - self.model = AutoModelForCausalLM.from_pretrained( - model_name, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None - ).eval() - - self.num_heads = self.model.config.num_attention_heads - - @property - def batch_type(self) -> Type[BloomBatch]: - return BloomBatch - - def forward( - self, input_ids, attention_mask, past_key_values: Optional = None - ) -> CausalLMOutputWithPast: - # Model Forward - return self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - use_cache=True, - ) - - def generate_token( - self, batch: BloomBatch - ) -> Tuple[List[GeneratedText], Optional[BloomBatch]]: - # For some reason, inference_mode does not work well with GLOO which we use on CPU - context_manager = ( - torch.no_grad if self.device.type == "cpu" else torch.inference_mode - ) - with context_manager(): - outputs = self.forward(**batch.input_ids) - - # List of indices to cache - next_batch_keep_indices = [] - next_batch_past_keep_indices = [] - - # New input_ids for next forward - next_batch_input_ids = [] - next_batch_all_input_ids = [] - next_all_input_lengths = [] - - next_batch_size = 0 - next_batch_max_sequence_length = 0 - - # Finished requests - generated_texts: List[GeneratedText] = [] - - # Zipped iterator - iterator = zip( - batch.requests, - batch.all_input_lengths, - outputs.logits, - batch.next_token_choosers, - batch.stopping_criterias, - batch.all_input_ids, - ) - - # For each member of the batch - for i, ( - request, - input_length, - logits, - next_token_chooser, - stopping_criteria, - all_tokens, - ) in enumerate(iterator): - # Select next token - next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) - - # Append next token to all tokens - all_tokens = torch.cat([all_tokens, next_token]) - - # Evaluate stopping criteria - if stopping_criteria(all_tokens): - # Decode all tokens - output = self.tokenizer.decode( - all_tokens.squeeze(-1), skip_special_tokens=True - ) - # Add to the list of finished generations with the original request - generated_texts.append(GeneratedText(request, output)) - # add to the next batch - else: - next_batch_keep_indices.append(i) - # past_key_values is of shape [batch_size * num_heads, ...] - # so we need to take into account the `num_heads` stride here - next_batch_past_keep_indices.extend( - [j for j in range(i * self.num_heads, (i + 1) * self.num_heads)] - ) - next_batch_input_ids.append(next_token) - next_batch_all_input_ids.append(all_tokens) - next_batch_size += 1 - new_input_length = input_length + 1 - next_all_input_lengths.append(new_input_length) - next_batch_max_sequence_length = max( - next_batch_max_sequence_length, new_input_length - ) - - # We finished all generations in the batch; there is no next batch - if not next_batch_keep_indices: - return generated_texts, None - - # If we finished at least one generation - next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} - if generated_texts: - # Apply indices to attention mask, past key values and other items that need to be cached - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ - next_batch_keep_indices - ] - next_batch_input_ids["past_key_values"] = [ - ( - keys[next_batch_past_keep_indices], - values[next_batch_past_keep_indices], - ) - for keys, values in outputs["past_key_values"] - ] - next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] - next_batch_next_token_choosers = [ - batch.next_token_choosers[i] for i in next_batch_keep_indices - ] - next_batch_stopping_criterias = [ - batch.stopping_criterias[i] for i in next_batch_keep_indices - ] - else: - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] - next_batch_input_ids["past_key_values"] = outputs["past_key_values"] - next_batch_requests = batch.requests - next_batch_next_token_choosers = batch.next_token_choosers - next_batch_stopping_criterias = batch.stopping_criterias - - # Update attention_mask with padding as we added a new token to input_ids - next_batch_input_ids["attention_mask"] = torch.cat( - [ - next_batch_input_ids["attention_mask"], - torch.ones((next_batch_size, 1)).to(self.device), - ], - dim=1, - ) - - next_batch = BloomBatch( - batch_id=batch.batch_id, - requests=next_batch_requests, - all_input_lengths=next_all_input_lengths, - input_ids=next_batch_input_ids, - all_input_ids=next_batch_all_input_ids, - next_token_choosers=next_batch_next_token_choosers, - stopping_criterias=next_batch_stopping_criterias, - size=next_batch_size, - max_sequence_length=next_batch_max_sequence_length, - ) - return generated_texts, next_batch - - -class BLOOMSharded(BLOOM): +class BLOOMSharded(Model): def __init__(self, model_name: str, quantize: bool = False): - super(Model, self).__init__() if not model_name.startswith("bigscience/bloom"): raise ValueError(f"Model {model_name} is not supported") self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 if torch.cuda.is_available(): - self.device = torch.device(f"cuda:{self.rank}") + device = torch.device(f"cuda:{self.rank}") dtype = torch.float16 else: - self.device = torch.device("cpu") + device = torch.device("cpu") dtype = torch.float32 - self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") config = AutoConfig.from_pretrained( model_name, slow_but_exact=False, tp_parallel=True ) config.pad_token_id = 3 - self.num_heads = config.n_head // self.process_group.size() # The flag below controls whether to allow TF32 on matmul. This flag defaults to False # in PyTorch 1.12 and later. @@ -370,12 +72,14 @@ def __init__(self, model_name: str, quantize: bool = False): model, filenames, quantize=quantize, - device=self.device, + device=device, rank=self.rank, world_size=self.world_size, ) self.model = model.eval().to(dtype) torch.distributed.barrier(group=self.process_group) + super(BLOOMSharded, self).__init__(tokenizer=tokenizer, num_heads=config.n_head // self.process_group.size(), + device=device) @staticmethod def load_weights( @@ -526,5 +230,4 @@ def forward(self, input_ids, attention_mask, past_key_values: Optional = None): torch.distributed.all_gather(logits, logits_shard, group=self.process_group) logits = torch.cat(logits, dim=1).view(batch_size, 1, vocab_size) - outputs.logits = logits - return outputs + return logits, outputs.past_key_values diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py new file mode 100644 index 00000000..39bb450f --- /dev/null +++ b/server/text_generation/models/causal_lm.py @@ -0,0 +1,38 @@ +import torch + +from transformers import AutoTokenizer, AutoModelForCausalLM +from typing import Optional, Tuple, List + +from text_generation.models import Model + + +class CausalLM(Model): + def __init__(self, model_name: str): + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + else: + device = torch.device("cpu") + dtype = torch.float32 + + tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + tokenizer.add_special_tokens({"pad_token": "[PAD]"}) + self.model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=dtype, + device_map="auto" if torch.cuda.is_available() else None, + ).eval() + + super(CausalLM, self).__init__(tokenizer=tokenizer, num_heads=self.model.config.num_attention_heads, device=device) + + def forward( + self, input_ids, attention_mask, past_key_values: Optional = None + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + # Model Forward + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + return outputs.logits, outputs.past_key_values diff --git a/server/text_generation/models/model.py b/server/text_generation/models/model.py index ad094197..d6b3c73e 100644 --- a/server/text_generation/models/model.py +++ b/server/text_generation/models/model.py @@ -1,19 +1,139 @@ +import torch + from abc import ABC, abstractmethod -from typing import List, Tuple, Optional, TypeVar, Type +from typing import List, Tuple, Optional +from tokenizers import Tokenizer from text_generation.models.types import Batch, GeneratedText -B = TypeVar("B", bound=Batch) - class Model(ABC): - @property + def __init__(self, tokenizer: Tokenizer, num_heads: int, device: torch.device): + self.tokenizer = tokenizer + self.num_heads = num_heads + self.device = device + @abstractmethod - def batch_type(self) -> Type[B]: + def forward(self, input_ids, attention_mask, past_key_values: Optional = None) -> Tuple[torch.Tensor, List[Tuple]]: raise NotImplementedError - @abstractmethod def generate_token( - self, batch: B - ) -> Tuple[List[GeneratedText], Optional[B]]: - raise NotImplementedError + self, batch: Batch + ) -> Tuple[List[GeneratedText], Optional[Batch]]: + # For some reason, inference_mode does not work well with GLOO which we use on CPU + context_manager = ( + torch.no_grad if self.device.type == "cpu" else torch.inference_mode + ) + with context_manager(): + logits, past = self.forward(**batch.input_ids) + + # List of indices to cache + next_batch_keep_indices = [] + + # New input_ids for next forward + next_batch_input_ids = [] + next_batch_all_input_ids = [] + next_all_input_lengths = [] + + next_batch_size = 0 + next_batch_max_sequence_length = 0 + + # Finished requests + generated_texts: List[GeneratedText] = [] + + # Zipped iterator + iterator = zip( + batch.requests, + batch.all_input_lengths, + logits, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + ) + + # For each member of the batch + for i, ( + request, + input_length, + logits, + next_token_chooser, + stopping_criteria, + all_tokens, + ) in enumerate(iterator): + # Select next token + next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) + + # Append next token to all tokens + all_tokens = torch.cat([all_tokens, next_token]) + + # Evaluate stopping criteria + if stopping_criteria(all_tokens): + # Decode all tokens + output = self.tokenizer.decode( + all_tokens.squeeze(-1), skip_special_tokens=True + ) + # Add to the list of finished generations with the original request + generated_texts.append(GeneratedText(request, output, stopping_criteria.current_tokens)) + # add to the next batch + else: + next_batch_keep_indices.append(i) + next_batch_input_ids.append(next_token) + next_batch_all_input_ids.append(all_tokens) + next_batch_size += 1 + new_input_length = input_length + 1 + next_all_input_lengths.append(new_input_length) + next_batch_max_sequence_length = max( + next_batch_max_sequence_length, new_input_length + ) + + # We finished all generations in the batch; there is no next batch + if not next_batch_keep_indices: + return generated_texts, None + + # If we finished at least one generation + next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} + if generated_texts: + # Apply indices to attention mask, past key values and other items that need to be cached + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ + next_batch_keep_indices + ] + # Force past to be of dim [batch_size, num_heads, ...] for easy indexing + next_batch_input_ids["past_key_values"] = [ + [t.view(-1, self.num_heads, *t.shape[-2:])[next_batch_keep_indices] for t in layer] + for layer in past + ] + next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] + next_batch_next_token_choosers = [ + batch.next_token_choosers[i] for i in next_batch_keep_indices + ] + next_batch_stopping_criterias = [ + batch.stopping_criterias[i] for i in next_batch_keep_indices + ] + else: + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] + next_batch_input_ids["past_key_values"] = past + next_batch_requests = batch.requests + next_batch_next_token_choosers = batch.next_token_choosers + next_batch_stopping_criterias = batch.stopping_criterias + + # Update attention_mask with padding as we added a new token to input_ids + next_batch_input_ids["attention_mask"] = torch.cat( + [ + next_batch_input_ids["attention_mask"], + torch.ones((next_batch_size, 1)).to(self.device), + ], + dim=1, + ) + + next_batch = Batch( + batch_id=batch.batch_id, + requests=next_batch_requests, + all_input_lengths=next_all_input_lengths, + input_ids=next_batch_input_ids, + all_input_ids=next_batch_all_input_ids, + next_token_choosers=next_batch_next_token_choosers, + stopping_criterias=next_batch_stopping_criterias, + size=next_batch_size, + max_sequence_length=next_batch_max_sequence_length, + ) + return generated_texts, next_batch diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index c2647a8d..0c17e555 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -1,6 +1,5 @@ import torch -from abc import abstractmethod from dataclasses import dataclass from typing import List, Dict @@ -32,7 +31,7 @@ def to_pb(self): @classmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device ) -> "Batch": inputs = [] next_token_choosers = [] @@ -51,7 +50,11 @@ def from_pb( do_sample=r.parameters.do_sample, ) ) - stopping_criterias.append(StoppingCriteria(max_new_tokens=r.max_new_tokens)) + stopping_criterias.append( + StoppingCriteria( + eos_token_id=tokenizer.eos_token_id, max_new_tokens=r.max_new_tokens + ) + ) input_ids = tokenizer( inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 @@ -71,15 +74,171 @@ def from_pb( ) @classmethod - @abstractmethod def concatenate(cls, batches: List["Batch"]) -> "Batch": - raise NotImplementedError + # Used for padding + total_batch_size = sum(batch.size for batch in batches) + max_sequence_length = max(batch.max_sequence_length for batch in batches) + # Only needed for Seq2SeqLM + max_encoded_sequence_length = None + + # Batch attributes + input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} + requests = [] + all_input_lengths = [] + all_input_ids = [] + next_token_choosers = [] + stopping_criterias = [] + + # Used for slicing correctly inside the tensors + # Equivalent to a cumsum on batch sizes + start_index = 0 + for i, batch in enumerate(batches): + requests.extend(batch.requests) + all_input_lengths.extend(batch.all_input_lengths) + all_input_ids.extend(batch.all_input_ids) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + + # Slicing end index for this batch + end_index = start_index + batch.size + + # We only concatenate batches that did at least one step + if batch.input_ids["input_ids"].shape[1] > 1: + raise ValueError("Batch input_ids should be of shape (batch_size, 1)") + + # Initialize tensors + if i == 0: + input_ids["input_ids"] = torch.empty( + (total_batch_size, 1), + dtype=batch.input_ids["input_ids"].dtype, + device=batch.input_ids["input_ids"].device, + ) + input_ids["attention_mask"] = torch.zeros( + (total_batch_size, max_sequence_length), + dtype=batch.input_ids["attention_mask"].dtype, + device=batch.input_ids["attention_mask"].device, + ) + + # input_ids["input_ids"] is always of shape [batch_size, 1] + # We do not need to pad it + input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] + + # We need to slice the attention mask to remove padding from previous steps + input_ids["attention_mask"][ + start_index:end_index, -batch.max_sequence_length: + ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length:] + + for j, past in enumerate(batch.input_ids["past_key_values"]): + # Shenanigans to get dimensions because BLOOM outputs a past with a different shape + # BLOOM: [batch_size * num_heads, ...] vs [batch_size, num_heads, ...] + head_dim, padded_sequence_length = past[0].shape[-2:] + num_heads = ( + past[0] + .view(batch.size, -1, head_dim, padded_sequence_length) + .shape[1] + ) + + # This will run only once per layer + if j == len(input_ids["past_key_values"]): + input_ids["past_key_values"].append([]) + + # Decoder past + for k, t in enumerate(past[:2]): + # Needed because BLOOM past shapes are not the same for keys and values + # Keys: [batch_size * num_heads, head_dim, seq_length] + # Values: [batch_size * num_heads, seq_length, head_dim] + head_dim_last = False + if t.shape[-2] == head_dim: + t = t.view( + batch.size, num_heads, head_dim, padded_sequence_length + ) + padded_t_shape = ( + total_batch_size, + num_heads, + head_dim, + max_sequence_length - 1, + ) + elif t.shape[-1] == head_dim: + head_dim_last = True + t = t.view( + batch.size, num_heads, padded_sequence_length, head_dim + ) + padded_t_shape = ( + total_batch_size, + num_heads, + max_sequence_length - 1, + head_dim, + ) + else: + raise ValueError(f"shape {t.shape} is not valid") + + # Initialize tensors + # This will run only once per layer and per past tensor + if k == len(input_ids["past_key_values"][j]): + input_ids["past_key_values"][j].append( + torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) + ) + + # We slice the past keys and values to remove the padding from previous batches + if not head_dim_last: + input_ids["past_key_values"][j][k][ + start_index:end_index, + :, + :, + -(batch.max_sequence_length - 1):, + ] = t[:, :, :, -(batch.max_sequence_length - 1):] + else: + input_ids["past_key_values"][j][k][ + start_index:end_index, + :, + -(batch.max_sequence_length - 1):, + :, + ] = t[:, :, -(batch.max_sequence_length - 1):, :] + + # Seq2SeqLM specific past (encoder past) + for k, t in enumerate(past[2:]): + if max_encoded_sequence_length is None: + max_encoded_sequence_length = max(max(batch.all_input_lengths) for batch in batches) + batch_max_encoded_sequence_length = max(batch.all_input_lengths) + + padded_t_shape = (total_batch_size, num_heads, max_encoded_sequence_length, head_dim) + + idx = k + 2 + + # Initialize tensors + # This will run only once per layer and per past tensor + if idx == len(input_ids["past_key_values"][j]): + input_ids["past_key_values"][j].append( + torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) + ) + + input_ids["past_key_values"][j][idx][ + start_index:end_index, + :, + -batch_max_encoded_sequence_length:, + : + ] = t[:, :, -batch_max_encoded_sequence_length:, :] + + start_index += batch.size + + return cls( + batch_id=batches[0].batch_id, + requests=requests, + all_input_lengths=all_input_lengths, + input_ids=input_ids, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=total_batch_size, + max_sequence_length=max_sequence_length, + ) @dataclass class GeneratedText: request: generate_pb2.Request output: str + tokens: int def to_pb(self) -> generate_pb2.GeneratedText: - return generate_pb2.GeneratedText(request=self.request, output=self.output) + return generate_pb2.GeneratedText(request=self.request, output=self.output, tokens=self.tokens) diff --git a/server/text_generation/server.py b/server/text_generation/server.py index b2b34cb1..fffeb0ba 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -27,7 +27,7 @@ async def ClearCache(self, request, context): return generate_pb2.ClearCacheResponse() async def Generate(self, request, context): - batch = self.model.batch_type.from_pb(request.batch, self.model.tokenizer, self.model.device) + batch = Batch.from_pb(request.batch, self.model.tokenizer, self.model.device) generated_texts, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) @@ -51,7 +51,7 @@ async def GenerateWithCache(self, request, context): batches.append(batch) if len(batches) > 1: - batch = self.model.batch_type.concatenate(batches) + batch = Batch.concatenate(batches) else: batch = batches[0] diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 50cd4124..0e2d9ae4 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -58,7 +58,8 @@ def __call__(self, input_ids, scores): class StoppingCriteria: - def __init__(self, max_new_tokens=20): + def __init__(self, eos_token_id, max_new_tokens=20): + self.eos_token_id = eos_token_id self.max_new_tokens = max_new_tokens self.current_tokens = 0 @@ -66,6 +67,8 @@ def __call__(self, all_ids): self.current_tokens += 1 if self.current_tokens >= self.max_new_tokens: return True + if self.eos_token_id is not None and all_ids[-1] == self.eos_token_id: + return True return False @@ -124,11 +127,18 @@ def download_weights(model_name, extension=".safetensors"): filenames = weight_hub_files(model_name, extension) download_function = partial( - hf_hub_download, repo_id=model_name, local_files_only=False + hf_hub_download, + repo_id=model_name, + local_files_only=False, ) executor = ThreadPoolExecutor(max_workers=5) - futures = [executor.submit(download_function, filename=filename) for filename in filenames] - files = [file for file in tqdm(concurrent.futures.as_completed(futures), total=len(futures))] + futures = [ + executor.submit(download_function, filename=filename) for filename in filenames + ] + files = [ + file + for file in tqdm(concurrent.futures.as_completed(futures), total=len(futures)) + ] return files From 427d7cc44459b5d06421300c8d74a07e1db1773b Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 4 Nov 2022 18:03:04 +0100 Subject: [PATCH 023/793] feat(server): Support AutoModelForSeq2SeqLM --- README.md | 11 +- proto/generate.proto | 2 - router/src/db.rs | 4 - server/text_generation/cache.py | 10 +- server/text_generation/models/__init__.py | 10 +- server/text_generation/models/bloom.py | 31 +- server/text_generation/models/causal_lm.py | 342 +++++++++++++- server/text_generation/models/model.py | 130 +----- server/text_generation/models/seq2seq_lm.py | 488 ++++++++++++++++++++ server/text_generation/models/types.py | 233 +--------- server/text_generation/server.py | 7 +- 11 files changed, 892 insertions(+), 376 deletions(-) create mode 100644 server/text_generation/models/seq2seq_lm.py diff --git a/README.md b/README.md index 29e58357..eadf5e50 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,20 @@ A Rust and gRPC server for text generation inference. - [Safetensors](https://github.com/huggingface/safetensors) weight loading - 45ms per token generation for BLOOM with 8xA100 80GB -## Supported models +## Officialy supported models - BLOOM - BLOOMZ - BLOOM-560m +Other models are supported on a best effort basis using: + +`AutoModelForCausalLM.from_pretrained(, device_map="auto")` + +or + +`AutoModelForSeq2SeqLM.from_pretrained(, device_map="auto")` + ## Load Tests for BLOOM See `k6/load_test.js` @@ -81,7 +89,6 @@ make router-dev ## TODO: -- [ ] Support AutoModelForSeq2SeqLM - [ ] Add tests for the `server/model` logic - [ ] Backport custom CUDA kernels to Transformers - [ ] Install safetensors with pip \ No newline at end of file diff --git a/proto/generate.proto b/proto/generate.proto index 68dfa151..14f6f66b 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -54,8 +54,6 @@ message Batch { repeated Request requests = 2; /// Batch size (==len(requests)) uint32 size = 3; - /// Length of the longest sequence within the batch (used for padding) - uint32 max_sequence_length = 4; } message GeneratedText { diff --git a/router/src/db.rs b/router/src/db.rs index af36614e..0701206b 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -142,14 +142,10 @@ impl Db { // Batch size let size = requests.len(); - // Longest input length for all requests in batch size - // Used for padding inside the inference server - let max_sequence_length = requests.iter().map(|r| r.input_length).max().unwrap(); let batch = Batch { id: state.next_batch_id, requests, size: size as u32, - max_sequence_length, }; // Update next_batch_start_id to the last id in the batch + 1 state.next_batch_start_id = ids.last().unwrap() + 1; diff --git a/server/text_generation/cache.py b/server/text_generation/cache.py index 65ec3e7c..5a3a8d31 100644 --- a/server/text_generation/cache.py +++ b/server/text_generation/cache.py @@ -1,16 +1,18 @@ -from typing import Dict, Optional +from typing import Dict, Optional, TypeVar from text_generation.models.types import Batch +B = TypeVar("B", bound=Batch) + class Cache: def __init__(self): - self.cache: Dict[int, Batch] = {} + self.cache: Dict[int, B] = {} - def pop(self, batch_id: int) -> Optional[Batch]: + def pop(self, batch_id: int) -> Optional[B]: return self.cache.pop(batch_id, None) - def set(self, entry: Batch): + def set(self, entry: B): if entry is not None: self.cache[entry.batch_id] = entry diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 0371ab4c..ade22d4e 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -1,8 +1,9 @@ from text_generation.models.model import Model -from text_generation.models.bloom import BLOOMSharded from text_generation.models.causal_lm import CausalLM +from text_generation.models.bloom import BLOOMSharded +from text_generation.models.seq2seq_lm import Seq2SeqLM -__all__ = ["Model", "BLOOMSharded", "CausalLM"] +__all__ = ["Model", "BLOOMSharded", "CausalLM", "Seq2SeqLM"] def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: @@ -18,4 +19,7 @@ def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: raise ValueError("sharded is not supported for AutoModel") if quantize: raise ValueError("quantize is not supported for AutoModel") - return CausalLM(model_name) + try: + return CausalLM(model_name) + except Exception as e: + return Seq2SeqLM(model_name) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 37f55f2f..730958c1 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -12,7 +12,7 @@ TensorParallelRowLinear, ) -from text_generation.models import Model +from text_generation.models import CausalLM from text_generation.utils import ( initialize_torch_distributed, weight_files, @@ -29,7 +29,7 @@ torch.manual_seed(0) -class BLOOMSharded(Model): +class BLOOMSharded(CausalLM): def __init__(self, model_name: str, quantize: bool = False): if not model_name.startswith("bigscience/bloom"): raise ValueError(f"Model {model_name} is not supported") @@ -78,22 +78,25 @@ def __init__(self, model_name: str, quantize: bool = False): ) self.model = model.eval().to(dtype) torch.distributed.barrier(group=self.process_group) - super(BLOOMSharded, self).__init__(tokenizer=tokenizer, num_heads=config.n_head // self.process_group.size(), - device=device) + super(CausalLM, self).__init__( + tokenizer=tokenizer, + num_heads=config.n_head // self.process_group.size(), + device=device, + ) @staticmethod def load_weights( - model, - filenames: List[str], - quantize: bool, - device: torch.device, - rank: int, - world_size: int, + model, + filenames: List[str], + quantize: bool, + device: torch.device, + rank: int, + world_size: int, ): parameters = dict(model.named_parameters()) for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if not quantize else "cpu" ) as f: for name in f.keys(): full_name = f"transformer.{name}" @@ -156,9 +159,9 @@ def load_weights( ) if ( - type(module) - in [TensorParallelRowLinear, TensorParallelColumnLinear] - and param_name == "weight" + type(module) + in [TensorParallelRowLinear, TensorParallelColumnLinear] + and param_name == "weight" ): tensor = Int8Params( tensor.transpose(1, 0), diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 39bb450f..b07537bd 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -1,9 +1,211 @@ import torch +from dataclasses import dataclass from transformers import AutoTokenizer, AutoModelForCausalLM -from typing import Optional, Tuple, List +from typing import Optional, Tuple, List, Dict, Type from text_generation.models import Model +from text_generation.models.types import GeneratedText +from text_generation.pb import generate_pb2 +from text_generation.utils import NextTokenChooser, StoppingCriteria + + +@dataclass +class CausalLMBatch: + batch_id: int + requests: List[generate_pb2.Request] + all_input_lengths: List[int] + input_ids: Dict[str, torch.Tensor] + all_input_ids: List[torch.Tensor] + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + size: int + max_sequence_length: int + + def to_pb(self): + return generate_pb2.Batch( + id=self.batch_id, + requests=self.requests, + size=self.size, + ) + + @classmethod + def from_pb( + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + ) -> "CausalLMBatch": + inputs = [] + next_token_choosers = [] + stopping_criterias = [] + all_input_lengths = [] + + # Parse batch + for r in pb.requests: + inputs.append(r.inputs) + all_input_lengths.append(r.input_length) + next_token_choosers.append( + NextTokenChooser( + temperature=r.parameters.temperature, + top_k=r.parameters.top_k, + top_p=r.parameters.top_p, + do_sample=r.parameters.do_sample, + ) + ) + stopping_criterias.append( + StoppingCriteria( + eos_token_id=tokenizer.eos_token_id, max_new_tokens=r.max_new_tokens + ) + ) + + input_ids = tokenizer( + inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 + ).to(device) + all_input_ids = input_ids["input_ids"].unsqueeze(-1) + + return cls( + batch_id=pb.id, + requests=pb.requests, + all_input_lengths=all_input_lengths, + input_ids=input_ids, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=pb.size, + max_sequence_length=max(all_input_lengths), + ) + + @classmethod + def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": + # Used for padding + total_batch_size = sum(batch.size for batch in batches) + max_sequence_length = max(batch.max_sequence_length for batch in batches) + + # Batch attributes + input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} + requests = [] + all_input_lengths = [] + all_input_ids = [] + next_token_choosers = [] + stopping_criterias = [] + + # Used for slicing correctly inside the tensors + # Equivalent to a cumsum on batch sizes + start_index = 0 + for i, batch in enumerate(batches): + requests.extend(batch.requests) + all_input_lengths.extend(batch.all_input_lengths) + all_input_ids.extend(batch.all_input_ids) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + + # Slicing end index for this batch + end_index = start_index + batch.size + + # We only concatenate batches that did at least one step + if batch.input_ids["input_ids"].shape[1] > 1: + raise ValueError("Batch input_ids should be of shape (batch_size, 1)") + + # Initialize tensors + if i == 0: + input_ids["input_ids"] = torch.empty( + (total_batch_size, 1), + dtype=batch.input_ids["input_ids"].dtype, + device=batch.input_ids["input_ids"].device, + ) + input_ids["attention_mask"] = torch.zeros( + (total_batch_size, max_sequence_length), + dtype=batch.input_ids["attention_mask"].dtype, + device=batch.input_ids["attention_mask"].device, + ) + + # input_ids["input_ids"] is always of shape [batch_size, 1] + # We do not need to pad it + input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] + + # We need to slice the attention mask to remove padding from previous steps + input_ids["attention_mask"][ + start_index:end_index, -batch.max_sequence_length : + ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length :] + + for j, past in enumerate(batch.input_ids["past_key_values"]): + # Shenanigans to get dimensions because BLOOM outputs a past with a different shape + # BLOOM: [batch_size * num_heads, ...] vs [batch_size, num_heads, ...] + head_dim, padded_sequence_length = past[0].shape[-2:] + num_heads = ( + past[0] + .view(batch.size, -1, head_dim, padded_sequence_length) + .shape[1] + ) + + # This will run only once per layer + if j == len(input_ids["past_key_values"]): + input_ids["past_key_values"].append([]) + + # Decoder past + for k, t in enumerate(past): + # Needed because BLOOM past shapes are not the same for keys and values + # Keys: [batch_size * num_heads, head_dim, seq_length] + # Values: [batch_size * num_heads, seq_length, head_dim] + head_dim_last = False + if t.shape[-2] == head_dim: + t = t.view( + batch.size, num_heads, head_dim, padded_sequence_length + ) + padded_t_shape = ( + total_batch_size, + num_heads, + head_dim, + max_sequence_length - 1, + ) + elif t.shape[-1] == head_dim: + head_dim_last = True + t = t.view( + batch.size, num_heads, padded_sequence_length, head_dim + ) + padded_t_shape = ( + total_batch_size, + num_heads, + max_sequence_length - 1, + head_dim, + ) + else: + raise ValueError(f"shape {t.shape} is not valid") + + # Initialize tensors + # This will run only once per layer and per past tensor + if k == len(input_ids["past_key_values"][j]): + input_ids["past_key_values"][j].append( + torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) + ) + + # We slice the past keys and values to remove the padding from previous batches + if not head_dim_last: + input_ids["past_key_values"][j][k][ + start_index:end_index, + :, + :, + -(batch.max_sequence_length - 1) :, + ] = t[:, :, :, -(batch.max_sequence_length - 1) :] + else: + input_ids["past_key_values"][j][k][ + start_index:end_index, + :, + -(batch.max_sequence_length - 1) :, + :, + ] = t[:, :, -(batch.max_sequence_length - 1) :, :] + + start_index += batch.size + + return cls( + batch_id=batches[0].batch_id, + requests=requests, + all_input_lengths=all_input_lengths, + input_ids=input_ids, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=total_batch_size, + max_sequence_length=max_sequence_length, + ) class CausalLM(Model): @@ -23,10 +225,18 @@ def __init__(self, model_name: str): device_map="auto" if torch.cuda.is_available() else None, ).eval() - super(CausalLM, self).__init__(tokenizer=tokenizer, num_heads=self.model.config.num_attention_heads, device=device) + super(CausalLM, self).__init__( + tokenizer=tokenizer, + num_heads=self.model.config.num_attention_heads, + device=device, + ) + + @property + def batch_type(self) -> Type[CausalLMBatch]: + return CausalLMBatch def forward( - self, input_ids, attention_mask, past_key_values: Optional = None + self, input_ids, attention_mask, past_key_values: Optional = None ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: # Model Forward outputs = self.model.forward( @@ -36,3 +246,129 @@ def forward( use_cache=True, ) return outputs.logits, outputs.past_key_values + + def generate_token( + self, batch: CausalLMBatch + ) -> Tuple[List[GeneratedText], Optional[CausalLMBatch]]: + # For some reason, inference_mode does not work well with GLOO which we use on CPU + context_manager = ( + torch.no_grad if self.device.type == "cpu" else torch.inference_mode + ) + with context_manager(): + logits, past = self.forward(**batch.input_ids) + + # List of indices to cache + next_batch_keep_indices = [] + + # New input_ids for next forward + next_batch_input_ids = [] + next_batch_all_input_ids = [] + next_all_input_lengths = [] + + next_batch_size = 0 + next_batch_max_sequence_length = 0 + + # Finished requests + generated_texts: List[GeneratedText] = [] + + # Zipped iterator + iterator = zip( + batch.requests, + batch.all_input_lengths, + logits, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + ) + + # For each member of the batch + for i, ( + request, + input_length, + logits, + next_token_chooser, + stopping_criteria, + all_tokens, + ) in enumerate(iterator): + # Select next token + next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) + + # Append next token to all tokens + all_tokens = torch.cat([all_tokens, next_token]) + + # Evaluate stopping criteria + if stopping_criteria(all_tokens): + # Decode all tokens + output = self.tokenizer.decode( + all_tokens.squeeze(-1), skip_special_tokens=True + ) + # Add to the list of finished generations with the original request + generated_texts.append( + GeneratedText(request, output, stopping_criteria.current_tokens) + ) + # add to the next batch + else: + next_batch_keep_indices.append(i) + next_batch_input_ids.append(next_token) + next_batch_all_input_ids.append(all_tokens) + next_batch_size += 1 + new_input_length = input_length + 1 + next_all_input_lengths.append(new_input_length) + next_batch_max_sequence_length = max( + next_batch_max_sequence_length, new_input_length + ) + + # We finished all generations in the batch; there is no next batch + if not next_batch_keep_indices: + return generated_texts, None + + # If we finished at least one generation + next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} + if generated_texts: + # Apply indices to attention mask, past key values and other items that need to be cached + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ + next_batch_keep_indices + ] + # Force past to be of dim [batch_size, num_heads, ...] for easy indexing + next_batch_input_ids["past_key_values"] = [ + [ + t.view(-1, self.num_heads, *t.shape[-2:])[next_batch_keep_indices] + for t in layer + ] + for layer in past + ] + next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] + next_batch_next_token_choosers = [ + batch.next_token_choosers[i] for i in next_batch_keep_indices + ] + next_batch_stopping_criterias = [ + batch.stopping_criterias[i] for i in next_batch_keep_indices + ] + else: + next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] + next_batch_input_ids["past_key_values"] = past + next_batch_requests = batch.requests + next_batch_next_token_choosers = batch.next_token_choosers + next_batch_stopping_criterias = batch.stopping_criterias + + # Update attention_mask with padding as we added a new token to input_ids + next_batch_input_ids["attention_mask"] = torch.cat( + [ + next_batch_input_ids["attention_mask"], + torch.ones((next_batch_size, 1)).to(self.device), + ], + dim=1, + ) + + next_batch = CausalLMBatch( + batch_id=batch.batch_id, + requests=next_batch_requests, + all_input_lengths=next_all_input_lengths, + input_ids=next_batch_input_ids, + all_input_ids=next_batch_all_input_ids, + next_token_choosers=next_batch_next_token_choosers, + stopping_criterias=next_batch_stopping_criterias, + size=next_batch_size, + max_sequence_length=next_batch_max_sequence_length, + ) + return generated_texts, next_batch diff --git a/server/text_generation/models/model.py b/server/text_generation/models/model.py index d6b3c73e..7fb8142c 100644 --- a/server/text_generation/models/model.py +++ b/server/text_generation/models/model.py @@ -1,11 +1,13 @@ import torch from abc import ABC, abstractmethod -from typing import List, Tuple, Optional +from typing import List, Tuple, Optional, TypeVar, Type from tokenizers import Tokenizer from text_generation.models.types import Batch, GeneratedText +B = TypeVar("B", bound=Batch) + class Model(ABC): def __init__(self, tokenizer: Tokenizer, num_heads: int, device: torch.device): @@ -13,127 +15,11 @@ def __init__(self, tokenizer: Tokenizer, num_heads: int, device: torch.device): self.num_heads = num_heads self.device = device + @property @abstractmethod - def forward(self, input_ids, attention_mask, past_key_values: Optional = None) -> Tuple[torch.Tensor, List[Tuple]]: + def batch_type(self) -> Type[B]: raise NotImplementedError - def generate_token( - self, batch: Batch - ) -> Tuple[List[GeneratedText], Optional[Batch]]: - # For some reason, inference_mode does not work well with GLOO which we use on CPU - context_manager = ( - torch.no_grad if self.device.type == "cpu" else torch.inference_mode - ) - with context_manager(): - logits, past = self.forward(**batch.input_ids) - - # List of indices to cache - next_batch_keep_indices = [] - - # New input_ids for next forward - next_batch_input_ids = [] - next_batch_all_input_ids = [] - next_all_input_lengths = [] - - next_batch_size = 0 - next_batch_max_sequence_length = 0 - - # Finished requests - generated_texts: List[GeneratedText] = [] - - # Zipped iterator - iterator = zip( - batch.requests, - batch.all_input_lengths, - logits, - batch.next_token_choosers, - batch.stopping_criterias, - batch.all_input_ids, - ) - - # For each member of the batch - for i, ( - request, - input_length, - logits, - next_token_chooser, - stopping_criteria, - all_tokens, - ) in enumerate(iterator): - # Select next token - next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) - - # Append next token to all tokens - all_tokens = torch.cat([all_tokens, next_token]) - - # Evaluate stopping criteria - if stopping_criteria(all_tokens): - # Decode all tokens - output = self.tokenizer.decode( - all_tokens.squeeze(-1), skip_special_tokens=True - ) - # Add to the list of finished generations with the original request - generated_texts.append(GeneratedText(request, output, stopping_criteria.current_tokens)) - # add to the next batch - else: - next_batch_keep_indices.append(i) - next_batch_input_ids.append(next_token) - next_batch_all_input_ids.append(all_tokens) - next_batch_size += 1 - new_input_length = input_length + 1 - next_all_input_lengths.append(new_input_length) - next_batch_max_sequence_length = max( - next_batch_max_sequence_length, new_input_length - ) - - # We finished all generations in the batch; there is no next batch - if not next_batch_keep_indices: - return generated_texts, None - - # If we finished at least one generation - next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} - if generated_texts: - # Apply indices to attention mask, past key values and other items that need to be cached - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ - next_batch_keep_indices - ] - # Force past to be of dim [batch_size, num_heads, ...] for easy indexing - next_batch_input_ids["past_key_values"] = [ - [t.view(-1, self.num_heads, *t.shape[-2:])[next_batch_keep_indices] for t in layer] - for layer in past - ] - next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] - next_batch_next_token_choosers = [ - batch.next_token_choosers[i] for i in next_batch_keep_indices - ] - next_batch_stopping_criterias = [ - batch.stopping_criterias[i] for i in next_batch_keep_indices - ] - else: - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] - next_batch_input_ids["past_key_values"] = past - next_batch_requests = batch.requests - next_batch_next_token_choosers = batch.next_token_choosers - next_batch_stopping_criterias = batch.stopping_criterias - - # Update attention_mask with padding as we added a new token to input_ids - next_batch_input_ids["attention_mask"] = torch.cat( - [ - next_batch_input_ids["attention_mask"], - torch.ones((next_batch_size, 1)).to(self.device), - ], - dim=1, - ) - - next_batch = Batch( - batch_id=batch.batch_id, - requests=next_batch_requests, - all_input_lengths=next_all_input_lengths, - input_ids=next_batch_input_ids, - all_input_ids=next_batch_all_input_ids, - next_token_choosers=next_batch_next_token_choosers, - stopping_criterias=next_batch_stopping_criterias, - size=next_batch_size, - max_sequence_length=next_batch_max_sequence_length, - ) - return generated_texts, next_batch + @abstractmethod + def generate_token(self, batch: B) -> Tuple[List[GeneratedText], Optional[B]]: + raise NotImplementedError diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py new file mode 100644 index 00000000..0607b3d5 --- /dev/null +++ b/server/text_generation/models/seq2seq_lm.py @@ -0,0 +1,488 @@ +import torch + +from dataclasses import dataclass +from transformers import AutoTokenizer, AutoModelForSeq2SeqLM +from typing import Optional, Tuple, List, Type + +from text_generation.models import Model +from text_generation.models.types import GeneratedText +from text_generation.pb import generate_pb2 +from text_generation.utils import NextTokenChooser, StoppingCriteria + + +@dataclass +class Seq2SeqLMBatch: + batch_id: int + requests: List[generate_pb2.Request] + + input_ids: torch.Tensor + attention_mask: torch.Tensor + + decoder_input_ids: torch.Tensor + decoder_attention_mask: Optional[torch.Tensor] + encoder_last_hidden_state: Optional[torch.Tensor] + + past_key_values: Optional[List[Tuple]] + + input_lengths: List[int] + decoder_input_lengths: List[int] + + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + + size: int + max_input_length: int + max_decoder_input_length: int + + def to_pb(self): + return generate_pb2.Batch( + id=self.batch_id, + requests=self.requests, + size=self.size, + ) + + @classmethod + def from_pb( + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + ) -> "Seq2SeqLMBatch": + inputs = [] + next_token_choosers = [] + stopping_criterias = [] + input_lengths = [] + + decoder_input_ids = [] + decoder_input_lengths = [] + + # Parse batch + for r in pb.requests: + inputs.append(r.inputs) + input_lengths.append(r.input_length) + decoder_input_ids.append(tokenizer.bos_token_id) + decoder_input_lengths.append(1) + next_token_choosers.append( + NextTokenChooser( + temperature=r.parameters.temperature, + top_k=r.parameters.top_k, + top_p=r.parameters.top_p, + do_sample=r.parameters.do_sample, + ) + ) + stopping_criterias.append( + StoppingCriteria( + eos_token_id=tokenizer.eos_token_id, max_new_tokens=r.max_new_tokens + ) + ) + + tokenized_inputs = tokenizer( + inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 + ).to(device) + decoder_input_ids = torch.tensor(decoder_input_ids).to(device).unsqueeze(-1) + + return cls( + batch_id=pb.id, + requests=pb.requests, + input_ids=tokenized_inputs["input_ids"], + attention_mask=tokenized_inputs["attention_mask"], + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=None, + encoder_last_hidden_state=None, + past_key_values=None, + input_lengths=input_lengths, + decoder_input_lengths=decoder_input_lengths, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=len(pb.requests), + max_input_length=max(input_lengths), + max_decoder_input_length=1, + ) + + @classmethod + def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": + # Used for padding + total_batch_size = sum(batch.size for batch in batches) + max_input_length = max(batch.max_input_length for batch in batches) + max_decoder_input_length = max( + batch.max_decoder_input_length for batch in batches + ) + + # Batch attributes + requests = [] + input_lengths = [] + decoder_input_lengths = [] + next_token_choosers = [] + stopping_criterias = [] + + input_ids = None + attention_mask = None + decoder_input_ids = None + decoder_attention_mask = None + encoder_last_hidden_state = None + past_key_values = [] + + # Used for slicing correctly inside the tensors + # Equivalent to a cumsum on batch sizes + start_index = 0 + for i, batch in enumerate(batches): + requests.extend(batch.requests) + input_lengths.extend(batch.input_lengths) + decoder_input_lengths.extend(batch.decoder_input_lengths) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + + # Slicing end index for this batch + end_index = start_index + batch.size + + # We only concatenate batches that did at least one step + if batch.encoder_last_hidden_state is None: + raise ValueError("Batch encoder_last_hidden_state cannot be None") + + if input_ids is None: + input_ids = torch.zeros( + (total_batch_size, max_input_length), + dtype=batch.input_ids.dtype, + device=batch.input_ids.device, + ) + input_ids[ + start_index:end_index, -batch.max_input_length : + ] = batch.input_ids[:, -batch.max_input_length :] + + if attention_mask is None: + attention_mask = torch.zeros( + (total_batch_size, max_input_length), + dtype=batch.attention_mask.dtype, + device=batch.attention_mask.device, + ) + attention_mask[ + start_index:end_index, -batch.max_input_length : + ] = batch.attention_mask[:, -batch.max_input_length :] + + if decoder_input_ids is None: + decoder_input_ids = torch.zeros( + (total_batch_size, max_decoder_input_length), + dtype=batch.decoder_input_ids.dtype, + device=batch.decoder_input_ids.device, + ) + decoder_input_ids[ + start_index:end_index, -batch.max_decoder_input_length : + ] = batch.decoder_input_ids[:, -batch.max_decoder_input_length :] + + if decoder_attention_mask is None: + decoder_attention_mask = torch.zeros( + (total_batch_size, max_decoder_input_length), + dtype=batch.attention_mask.dtype, + device=batch.attention_mask.device, + ) + if batch.decoder_attention_mask is None: + decoder_attention_mask[ + start_index:end_index, -batch.max_decoder_input_length : + ] = 1 + else: + decoder_attention_mask[ + start_index:end_index, -batch.max_decoder_input_length : + ] = batch.decoder_attention_mask[:, -batch.max_decoder_input_length :] + + if encoder_last_hidden_state is None: + encoder_last_hidden_state = torch.zeros( + ( + total_batch_size, + max_input_length, + batch.encoder_last_hidden_state.shape[-1], + ), + dtype=batch.encoder_last_hidden_state.dtype, + device=batch.encoder_last_hidden_state.device, + ) + + encoder_last_hidden_state[ + start_index:end_index, -batch.max_decoder_input_length :, : + ] = batch.encoder_last_hidden_state[:, -batch.max_decoder_input_length :, :] + + for j, past in enumerate(batch.past_key_values): + _, num_heads, _, head_dim = past[0].shape + + # This will run only once per layer + if j == len(past_key_values): + past_key_values.append([]) + + # Decoder past + for k, t in enumerate(past[:2]): + padded_t_shape = ( + total_batch_size, + num_heads, + (max_decoder_input_length - 1), + head_dim, + ) + + # Initialize tensors + # This will run only once per layer and per past tensor + if k == len(past_key_values[j]): + past_key_values[j].append( + torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) + ) + + # We slice the past keys and values to remove the padding from previous batches + past_key_values[j][k][ + start_index:end_index, + :, + -(batch.max_decoder_input_length - 1) :, + :, + ] = t[:, :, -(batch.max_decoder_input_length - 1) :, :] + + # encoder past + for k, t in enumerate(past[2:]): + padded_t_shape = ( + total_batch_size, + num_heads, + max_input_length, + head_dim, + ) + + idx = k + 2 + + # Initialize tensors + # This will run only once per layer and per past tensor + if idx == len(past_key_values[j]): + past_key_values[j].append( + torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) + ) + + past_key_values[j][idx][ + start_index:end_index, :, -batch.max_input_length :, : + ] = t[:, :, -batch.max_input_length :, :] + + start_index += batch.size + + return cls( + batch_id=batches[0].batch_id, + requests=requests, + input_ids=input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + encoder_last_hidden_state=encoder_last_hidden_state, + past_key_values=past_key_values, + input_lengths=input_lengths, + decoder_input_lengths=decoder_input_lengths, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=total_batch_size, + max_input_length=max_input_length, + max_decoder_input_length=max_decoder_input_length, + ) + + +class Seq2SeqLM(Model): + def __init__(self, model_name: str): + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + else: + device = torch.device("cpu") + dtype = torch.float32 + + self.model = AutoModelForSeq2SeqLM.from_pretrained( + model_name, + torch_dtype=dtype, + device_map="auto" if torch.cuda.is_available() else None, + ).eval() + tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + tokenizer.bos_token_id = self.model.config.decoder_start_token_id + + super(Seq2SeqLM, self).__init__( + tokenizer=tokenizer, + num_heads=self.model.config.num_attention_heads, + device=device, + ) + + @property + def batch_type(self) -> Type[Seq2SeqLMBatch]: + return Seq2SeqLMBatch + + def forward( + self, + input_ids, + attention_mask, + decoder_input_ids, + decoder_attention_mask: Optional, + encoder_last_hidden_state: Optional, + past_key_values: Optional = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]], + ]: + # Model Forward + if past_key_values is not None: + decoder_input_ids = decoder_input_ids[:, -1].unsqueeze(-1) + + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + encoder_outputs=[encoder_last_hidden_state] + if encoder_last_hidden_state is not None + else None, + past_key_values=past_key_values, + use_cache=True, + ) + return ( + outputs.logits, + outputs.encoder_last_hidden_state, + outputs.past_key_values, + ) + + def generate_token( + self, batch: Seq2SeqLMBatch + ) -> Tuple[List[GeneratedText], Optional[Seq2SeqLMBatch]]: + # For some reason, inference_mode does not work well with GLOO which we use on CPU + context_manager = ( + torch.no_grad if self.device.type == "cpu" else torch.inference_mode + ) + with context_manager(): + logits, encoder_last_hidden_state, past = self.forward( + batch.input_ids, + batch.attention_mask, + batch.decoder_input_ids, + batch.decoder_attention_mask, + batch.encoder_last_hidden_state, + batch.past_key_values, + ) + + # List of indices to cache + next_batch_keep_indices = [] + + # New input_ids for next forward + next_batch_input_lengths = [] + next_batch_decoder_input_ids = [] + next_batch_decoder_input_lengths = [] + + next_batch_size = 0 + next_batch_max_input_length = 0 + next_batch_max_decoder_input_length = 0 + + # Finished requests + generated_texts: List[GeneratedText] = [] + + # Zipped iterator + iterator = zip( + batch.requests, + batch.input_lengths, + batch.decoder_input_lengths, + logits, + batch.next_token_choosers, + batch.stopping_criterias, + batch.input_ids, + batch.decoder_input_ids, + ) + + # For each member of the batch + for i, ( + request, + input_length, + decoder_input_length, + logits, + next_token_chooser, + stopping_criteria, + input_tokens, + decoder_tokens, + ) in enumerate(iterator): + all_tokens = torch.cat([input_tokens, decoder_tokens]) + # Select next token + next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) + + # Append next token to decoder tokens + decoder_tokens = torch.cat([decoder_tokens, next_token.squeeze(1)]) + + # Evaluate stopping criteria + if stopping_criteria(decoder_tokens): + # Decode all tokens + output = self.tokenizer.decode(decoder_tokens, skip_special_tokens=True) + # Add to the list of finished generations with the original request + generated_texts.append( + GeneratedText(request, output, stopping_criteria.current_tokens) + ) + # add to the next batch + else: + next_batch_keep_indices.append(i) + next_batch_decoder_input_ids.append(decoder_tokens.unsqueeze(0)) + next_batch_size += 1 + new_decoder_input_length = decoder_input_length + 1 + next_batch_input_lengths.append(input_length) + next_batch_decoder_input_lengths.append(new_decoder_input_length) + next_batch_max_input_length = max( + next_batch_max_input_length, input_length + ) + next_batch_max_decoder_input_length = max( + next_batch_max_decoder_input_length, new_decoder_input_length + ) + + # We finished all generations in the batch; there is no next batch + if not next_batch_keep_indices: + return generated_texts, None + + # If we finished at least one generation + next_batch_decoder_input_ids = torch.cat(next_batch_decoder_input_ids) + if generated_texts: + next_batch_input_ids = batch.input_ids[next_batch_keep_indices] + next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] + + if batch.decoder_attention_mask is not None: + next_batch_decoder_attention_mask = batch.decoder_attention_mask[ + next_batch_keep_indices + ] + else: + next_batch_decoder_attention_mask = None + + next_batch_encoder_last_hidden_state = encoder_last_hidden_state[ + next_batch_keep_indices + ] + + next_batch_past_key_values = [ + [t[next_batch_keep_indices] for t in layer] for layer in past + ] + next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] + next_batch_next_token_choosers = [ + batch.next_token_choosers[i] for i in next_batch_keep_indices + ] + next_batch_stopping_criterias = [ + batch.stopping_criterias[i] for i in next_batch_keep_indices + ] + else: + next_batch_input_ids = batch.input_ids + next_batch_attention_mask = batch.attention_mask + next_batch_decoder_attention_mask = batch.decoder_attention_mask + next_batch_encoder_last_hidden_state = encoder_last_hidden_state + next_batch_past_key_values = past + + next_batch_requests = batch.requests + next_batch_next_token_choosers = batch.next_token_choosers + next_batch_stopping_criterias = batch.stopping_criterias + + # Update attention_mask with padding as we added a new token to input_ids + if next_batch_decoder_attention_mask is not None: + next_batch_decoder_attention_mask = torch.cat( + [ + next_batch_decoder_attention_mask, + torch.ones((next_batch_size, 1)).to(self.device), + ], + dim=1, + ) + + next_batch = Seq2SeqLMBatch( + batch_id=batch.batch_id, + requests=next_batch_requests, + input_ids=next_batch_input_ids, + attention_mask=next_batch_attention_mask, + decoder_input_ids=next_batch_decoder_input_ids, + decoder_attention_mask=next_batch_decoder_attention_mask, + encoder_last_hidden_state=next_batch_encoder_last_hidden_state, + past_key_values=next_batch_past_key_values, + input_lengths=next_batch_input_lengths, + decoder_input_lengths=next_batch_decoder_input_lengths, + next_token_choosers=next_batch_next_token_choosers, + stopping_criterias=next_batch_stopping_criterias, + size=next_batch_size, + max_input_length=next_batch_max_input_length, + max_decoder_input_length=next_batch_max_decoder_input_length, + ) + return generated_texts, next_batch diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index 0c17e555..7c25bf67 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -1,237 +1,30 @@ import torch +from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import List, Dict +from typing import List from transformers import AutoTokenizer from text_generation.pb import generate_pb2 -from text_generation.utils import NextTokenChooser, StoppingCriteria -@dataclass -class Batch: - batch_id: int - requests: List[generate_pb2.Request] - all_input_lengths: List[int] - input_ids: Dict[str, torch.Tensor] - all_input_ids: List[torch.Tensor] - next_token_choosers: List[NextTokenChooser] - stopping_criterias: List[StoppingCriteria] - size: int - max_sequence_length: int - - def to_pb(self): - return generate_pb2.Batch( - id=self.batch_id, - requests=self.requests, - size=self.size, - max_sequence_length=self.max_sequence_length, - ) +class Batch(ABC): + @abstractmethod + def to_pb(self) -> generate_pb2.Batch: + raise NotImplementedError @classmethod + @abstractmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device ) -> "Batch": - inputs = [] - next_token_choosers = [] - stopping_criterias = [] - all_input_lengths = [] - - # Parse batch - for r in pb.requests: - inputs.append(r.inputs) - all_input_lengths.append(r.input_length) - next_token_choosers.append( - NextTokenChooser( - temperature=r.parameters.temperature, - top_k=r.parameters.top_k, - top_p=r.parameters.top_p, - do_sample=r.parameters.do_sample, - ) - ) - stopping_criterias.append( - StoppingCriteria( - eos_token_id=tokenizer.eos_token_id, max_new_tokens=r.max_new_tokens - ) - ) - - input_ids = tokenizer( - inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 - ).to(device) - all_input_ids = input_ids["input_ids"].unsqueeze(-1) - - return cls( - batch_id=pb.id, - requests=pb.requests, - all_input_lengths=all_input_lengths, - input_ids=input_ids, - all_input_ids=all_input_ids, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - size=pb.size, - max_sequence_length=pb.max_sequence_length, - ) + raise NotImplementedError @classmethod + @abstractmethod def concatenate(cls, batches: List["Batch"]) -> "Batch": - # Used for padding - total_batch_size = sum(batch.size for batch in batches) - max_sequence_length = max(batch.max_sequence_length for batch in batches) - # Only needed for Seq2SeqLM - max_encoded_sequence_length = None - - # Batch attributes - input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} - requests = [] - all_input_lengths = [] - all_input_ids = [] - next_token_choosers = [] - stopping_criterias = [] - - # Used for slicing correctly inside the tensors - # Equivalent to a cumsum on batch sizes - start_index = 0 - for i, batch in enumerate(batches): - requests.extend(batch.requests) - all_input_lengths.extend(batch.all_input_lengths) - all_input_ids.extend(batch.all_input_ids) - next_token_choosers.extend(batch.next_token_choosers) - stopping_criterias.extend(batch.stopping_criterias) - - # Slicing end index for this batch - end_index = start_index + batch.size - - # We only concatenate batches that did at least one step - if batch.input_ids["input_ids"].shape[1] > 1: - raise ValueError("Batch input_ids should be of shape (batch_size, 1)") - - # Initialize tensors - if i == 0: - input_ids["input_ids"] = torch.empty( - (total_batch_size, 1), - dtype=batch.input_ids["input_ids"].dtype, - device=batch.input_ids["input_ids"].device, - ) - input_ids["attention_mask"] = torch.zeros( - (total_batch_size, max_sequence_length), - dtype=batch.input_ids["attention_mask"].dtype, - device=batch.input_ids["attention_mask"].device, - ) - - # input_ids["input_ids"] is always of shape [batch_size, 1] - # We do not need to pad it - input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] - - # We need to slice the attention mask to remove padding from previous steps - input_ids["attention_mask"][ - start_index:end_index, -batch.max_sequence_length: - ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length:] - - for j, past in enumerate(batch.input_ids["past_key_values"]): - # Shenanigans to get dimensions because BLOOM outputs a past with a different shape - # BLOOM: [batch_size * num_heads, ...] vs [batch_size, num_heads, ...] - head_dim, padded_sequence_length = past[0].shape[-2:] - num_heads = ( - past[0] - .view(batch.size, -1, head_dim, padded_sequence_length) - .shape[1] - ) - - # This will run only once per layer - if j == len(input_ids["past_key_values"]): - input_ids["past_key_values"].append([]) - - # Decoder past - for k, t in enumerate(past[:2]): - # Needed because BLOOM past shapes are not the same for keys and values - # Keys: [batch_size * num_heads, head_dim, seq_length] - # Values: [batch_size * num_heads, seq_length, head_dim] - head_dim_last = False - if t.shape[-2] == head_dim: - t = t.view( - batch.size, num_heads, head_dim, padded_sequence_length - ) - padded_t_shape = ( - total_batch_size, - num_heads, - head_dim, - max_sequence_length - 1, - ) - elif t.shape[-1] == head_dim: - head_dim_last = True - t = t.view( - batch.size, num_heads, padded_sequence_length, head_dim - ) - padded_t_shape = ( - total_batch_size, - num_heads, - max_sequence_length - 1, - head_dim, - ) - else: - raise ValueError(f"shape {t.shape} is not valid") - - # Initialize tensors - # This will run only once per layer and per past tensor - if k == len(input_ids["past_key_values"][j]): - input_ids["past_key_values"][j].append( - torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) - ) - - # We slice the past keys and values to remove the padding from previous batches - if not head_dim_last: - input_ids["past_key_values"][j][k][ - start_index:end_index, - :, - :, - -(batch.max_sequence_length - 1):, - ] = t[:, :, :, -(batch.max_sequence_length - 1):] - else: - input_ids["past_key_values"][j][k][ - start_index:end_index, - :, - -(batch.max_sequence_length - 1):, - :, - ] = t[:, :, -(batch.max_sequence_length - 1):, :] - - # Seq2SeqLM specific past (encoder past) - for k, t in enumerate(past[2:]): - if max_encoded_sequence_length is None: - max_encoded_sequence_length = max(max(batch.all_input_lengths) for batch in batches) - batch_max_encoded_sequence_length = max(batch.all_input_lengths) - - padded_t_shape = (total_batch_size, num_heads, max_encoded_sequence_length, head_dim) - - idx = k + 2 - - # Initialize tensors - # This will run only once per layer and per past tensor - if idx == len(input_ids["past_key_values"][j]): - input_ids["past_key_values"][j].append( - torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) - ) - - input_ids["past_key_values"][j][idx][ - start_index:end_index, - :, - -batch_max_encoded_sequence_length:, - : - ] = t[:, :, -batch_max_encoded_sequence_length:, :] - - start_index += batch.size - - return cls( - batch_id=batches[0].batch_id, - requests=requests, - all_input_lengths=all_input_lengths, - input_ids=input_ids, - all_input_ids=all_input_ids, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - size=total_batch_size, - max_sequence_length=max_sequence_length, - ) + raise NotImplementedError @dataclass @@ -241,4 +34,6 @@ class GeneratedText: tokens: int def to_pb(self) -> generate_pb2.GeneratedText: - return generate_pb2.GeneratedText(request=self.request, output=self.output, tokens=self.tokens) + return generate_pb2.GeneratedText( + request=self.request, output=self.output, tokens=self.tokens + ) diff --git a/server/text_generation/server.py b/server/text_generation/server.py index fffeb0ba..699cef08 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -9,7 +9,6 @@ from text_generation.cache import Cache from text_generation.models import Model, get_model -from text_generation.models.types import Batch from text_generation.pb import generate_pb2_grpc, generate_pb2 @@ -27,7 +26,9 @@ async def ClearCache(self, request, context): return generate_pb2.ClearCacheResponse() async def Generate(self, request, context): - batch = Batch.from_pb(request.batch, self.model.tokenizer, self.model.device) + batch = self.model.batch_type.from_pb( + request.batch, self.model.tokenizer, self.model.device + ) generated_texts, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) @@ -51,7 +52,7 @@ async def GenerateWithCache(self, request, context): batches.append(batch) if len(batches) > 1: - batch = Batch.concatenate(batches) + batch = self.model.batch_type.concatenate(batches) else: batch = batches[0] From cea6051effa58c18609e6be55957bdfdcc8cd3a0 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 4 Nov 2022 18:31:08 +0100 Subject: [PATCH 024/793] feat(launcher): Pass CUDA_VISIBLE_DEVICES to the shard --- launcher/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index ed6cf8e5..b42ed0c5 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -306,6 +306,14 @@ fn shard_manager( )); }; + // If the CUDA_VISIBLE_DEVICES env var is set, pass it to the shard + if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { + env.push(( + "CUDA_VISIBLE_DEVICES".parse().unwrap(), + cuda_visible_devices.parse().unwrap(), + )); + }; + // Start process tracing::info!("Starting shard {}", rank); let mut p = match Popen::create( From 4236e41b0d896dce9435074dde39be74f16ff95e Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:53:56 +0100 Subject: [PATCH 025/793] feat(server): Improved doc --- Dockerfile | 7 +- README.md | 14 +- launcher/src/main.rs | 4 + server/Makefile | 15 +-- server/poetry.lock | 47 ++++++- server/pyproject.toml | 1 + server/text_generation/models/__init__.py | 12 +- server/text_generation/models/causal_lm.py | 134 ++++++++++++-------- server/text_generation/models/seq2seq_lm.py | 56 ++++++-- 9 files changed, 192 insertions(+), 98 deletions(-) diff --git a/Dockerfile b/Dockerfile index ebe79609..9b6ef835 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,7 @@ ENV LANG=C.UTF-8 \ MODEL_NAME=bigscience/bloom \ QUANTIZE=false \ NUM_GPUS=8 \ + SAFETENSORS_FAST_GPU=1 \ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ NCCL_ASYNC_ERROR_HANDLING=1 \ CUDA_HOME=/usr/local/cuda \ @@ -55,12 +56,6 @@ RUN cd server && make install-torch # Install specific version of transformers RUN cd server && make install-transformers -# Install specific version of safetensors -# FIXME: This is a temporary fix while we wait for a new release -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y -ENV PATH="/root/.cargo/bin:${PATH}" -RUN cd server && make install-safetensors - # Install server COPY proto proto COPY server server diff --git a/README.md b/README.md index eadf5e50..cd9e2176 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ -A Rust and gRPC server for text generation inference. +A Rust and gRPC server for text generation inference. Used in production at [HuggingFace](https://huggingface.co) +to power Bloom, BloomZ and MT0-XXL api-inference widgets. ## Features @@ -15,11 +16,11 @@ A Rust and gRPC server for text generation inference. - [Safetensors](https://github.com/huggingface/safetensors) weight loading - 45ms per token generation for BLOOM with 8xA100 80GB -## Officialy supported models +## Officially supported models -- BLOOM -- BLOOMZ -- BLOOM-560m +- [BLOOM](https://huggingface.co/bigscience/bloom) +- [BLOOMZ](https://huggingface.co/bigscience/bloomz) +- [MT0-XXL](https://huggingface.co/bigscience/mt0-xxl) Other models are supported on a best effort basis using: @@ -90,5 +91,4 @@ make router-dev ## TODO: - [ ] Add tests for the `server/model` logic -- [ ] Backport custom CUDA kernels to Transformers -- [ ] Install safetensors with pip \ No newline at end of file +- [ ] Backport custom CUDA kernels to Transformers \ No newline at end of file diff --git a/launcher/src/main.rs b/launcher/src/main.rs index b42ed0c5..016a28eb 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -295,6 +295,10 @@ fn shard_manager( "MASTER_PORT".parse().unwrap(), master_port.to_string().parse().unwrap(), ), + ( + "SAFETENSORS_FAST_GPU".parse().unwrap(), + "1".to_string().parse().unwrap(), + ), ]; // If the HUGGINGFACE_HUB_CACHE env var is set, pass it to the shard diff --git a/server/Makefile b/server/Makefile index 99764028..39a98b65 100644 --- a/server/Makefile +++ b/server/Makefile @@ -16,24 +16,13 @@ install-transformers: mv transformers-7302a24535e8dc5637ea5b4e4572fc971d404098 transformers cd transformers && python setup.py install -install-safetensors: - # Install specific version of safetensors - pip install setuptools_rust - rm safetensors || true - rm safetensors-634deccbcbad5eaf417935281f8b3be7ebca69c5 || true - curl -L -O https://github.com/huggingface/safetensors/archive/634deccbcbad5eaf417935281f8b3be7ebca69c5.zip - unzip 634deccbcbad5eaf417935281f8b3be7ebca69c5.zip - rm 634deccbcbad5eaf417935281f8b3be7ebca69c5.zip - mv safetensors-634deccbcbad5eaf417935281f8b3be7ebca69c5 safetensors - cd safetensors/bindings/python && python setup.py develop - install-torch: # Install specific version of torch pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir -install: gen-server install-torch install-transformers install-safetensors +install: gen-server install-torch install-transformers pip install pip --upgrade pip install -e . --no-cache-dir run-dev: - python -m torch.distributed.run --nproc_per_node=2 text_generation/cli.py serve bigscience/bloom-560m --sharded \ No newline at end of file + SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation/cli.py serve bigscience/bloom-560m --sharded \ No newline at end of file diff --git a/server/poetry.lock b/server/poetry.lock index 3c92903e..ebd64ea9 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -145,6 +145,18 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "safetensors" +version = "0.2.4" +description = "Fast and Safe Tensor serialization" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +dev = ["black (==22.3)", "flake8 (>=3.8.3)", "huggingface-hub", "isort (>=5.5.4)", "numpy", "pytest", "setuptools-rust"] +testing = ["black (==22.3)", "flake8 (>=3.8.3)", "huggingface-hub", "isort (>=5.5.4)", "numpy", "pytest", "setuptools-rust"] + [[package]] name = "setuptools" version = "65.5.0" @@ -208,7 +220,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "224b1e379d6105fe911bff4563946a90dfa6ff5918cf2e7be59f8d4f7c5cd7cf" +content-hash = "3266187ef14fe8f9e29b3b6530d07781ea952aa670c0fe0de34be43efa231a67" [metadata.files] accelerate = [ @@ -459,6 +471,39 @@ PyYAML = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +safetensors = [ + {file = "safetensors-0.2.4-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:79c4a7610d7699c64d8531c43f758ded4990ebaa7b0887c2078640e6de44e726"}, + {file = "safetensors-0.2.4-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ef425a4ddd29612fe733a6eeca6ad8f3ee3939f530a032114974aac4c4667b89"}, + {file = "safetensors-0.2.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77758f8ba4de6e20bf394dd964854a926dee2efee82eaa95e6c0893e2a7d960c"}, + {file = "safetensors-0.2.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb956e9090cce515649f00b491b5ddc0f9c3d989139016a8d69f9dcf57e8d3d9"}, + {file = "safetensors-0.2.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e31b02d27249bd519f05ec9d189097c59fc6851c59daa1a86ef347659e33ac3"}, + {file = "safetensors-0.2.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c2fead03a1497042efea4358574f3d7acf501b0c82e54d605f393f2b4e2aafe"}, + {file = "safetensors-0.2.4-cp310-cp310-win32.whl", hash = "sha256:dce6ed3c7d13aafa574737eb3309c928adcb6781e879b41f0861be83b439cf3e"}, + {file = "safetensors-0.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:1dfe727325a1342767c6725dc2cc1f00463eb40a1f5df37c338d8e03957e27ce"}, + {file = "safetensors-0.2.4-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:c066bc7b90a582a01ec468fef61a7581b5c726bf12c50491cb6ea5db215ea5e0"}, + {file = "safetensors-0.2.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca6ed53dad5d7d0e67eb676528ff2ad345cac3a34010e4dc1e3736972de294a5"}, + {file = "safetensors-0.2.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ada03b44acbb036cfabe7066a8df4ad9b1ac05bb585a6b6c0f285f08e016381d"}, + {file = "safetensors-0.2.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58a0902708daa7ec2b2293b46e85df61f4fa359ddfe648e7ac025a79e6f59627"}, + {file = "safetensors-0.2.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e0a4e38f7cbb4bfc513588e52f349b906c941e74fbbe192f2b19fc34221d448"}, + {file = "safetensors-0.2.4-cp37-cp37m-win32.whl", hash = "sha256:4f8695b77dd847203258f035f8468f8b701c90621cb6b457e109f8d89c27f16c"}, + {file = "safetensors-0.2.4-cp37-cp37m-win_amd64.whl", hash = "sha256:16b08f33c753c7da64b3999beea7c30d58204a0820961e33881d05a331e3f5c0"}, + {file = "safetensors-0.2.4-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:a381606804f23db9eede51135f5fbd1f75dda02100415ee150fd39eb1cd6be4c"}, + {file = "safetensors-0.2.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7aceae84d0c7233d83923029aaf8d184848561e0211ec98c5317327b3db025d6"}, + {file = "safetensors-0.2.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:da48fc929485cbd9ee22621e388764a7cef27b0205e73aee2ad75aadd7d67662"}, + {file = "safetensors-0.2.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2619b88f934c4de6b59de90c9dc00eae2d0e30f254a1daebd6eb232ac1f9a7a7"}, + {file = "safetensors-0.2.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1f78b987ae1f6b71da8ea110164e4cab2ee31b53835d2a66279df89c5d73f0e"}, + {file = "safetensors-0.2.4-cp38-cp38-win32.whl", hash = "sha256:34b3e60b5130fb0fe07114705e51d30aa2c7eae4c1d1e77d6f260fa4ade70ede"}, + {file = "safetensors-0.2.4-cp38-cp38-win_amd64.whl", hash = "sha256:debaa4fa98a7af44ba6dcb6945efee77b8480284c2cb05918ab97cf511c40826"}, + {file = "safetensors-0.2.4-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:90baaafc0c872a736124b341db54b0bdd61765cbf3a61418371066a37905b18d"}, + {file = "safetensors-0.2.4-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:b4bf7e23191d6a3ff00de141512869fc776e8ff159c872cb44af018cb04d45eb"}, + {file = "safetensors-0.2.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf11a3aba8796e548ceb0a65f34dcd334dcf0c4c891dccabe18a8b53918ae8ab"}, + {file = "safetensors-0.2.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:95c31935ea71d63a38c546654136d7f0dbf1e7aeb6564dbc2201bc1fe9b34e4c"}, + {file = "safetensors-0.2.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef31776e2e081d6f075408eed34a0fbd524cbd19e50268bef02c238b209213b7"}, + {file = "safetensors-0.2.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06bb1d68148f6d6934352124d8cbfcf0db092f969db7187e348bd5cbf183db5"}, + {file = "safetensors-0.2.4-cp39-cp39-win32.whl", hash = "sha256:5d546152b9a5bd58eae97c2ddefba394404d37ddedec305f7639c9b6054513e5"}, + {file = "safetensors-0.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:553ecfd895d379c1e03a7c9241f7343b3af66573436969ed7eb95df81dfbe9af"}, + {file = "safetensors-0.2.4.tar.gz", hash = "sha256:35c0719a898f1f1292464f4cd9370bb6c2698032f1db4d677489f078b66b5a75"}, +] setuptools = [ {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index e2ba98a7..cdf89869 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -15,6 +15,7 @@ typer = "^0.6.1" grpcio-reflection = "^1.49.1" accelerate = "^0.12.0" bitsandbytes = "^0.35.1" +safetensors = "^0.2.4" [tool.poetry.extras] bnb = ["bitsandbytes"] diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index ade22d4e..bf44115c 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -9,17 +9,13 @@ def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: if model_name.startswith("bigscience/bloom"): if sharded: - return BLOOMSharded(model_name, quantize) + return BLOOMSharded(model_name, quantize=quantize) else: - if quantize: - raise ValueError("quantization is not supported for non-sharded BLOOM") - return CausalLM(model_name) + return CausalLM(model_name, quantize=quantize) else: if sharded: raise ValueError("sharded is not supported for AutoModel") - if quantize: - raise ValueError("quantize is not supported for AutoModel") try: - return CausalLM(model_name) + return CausalLM(model_name, quantize=quantize) except Exception as e: - return Seq2SeqLM(model_name) + return Seq2SeqLM(model_name, quantize=quantize) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index b07537bd..2ba36b1a 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from transformers import AutoTokenizer, AutoModelForCausalLM -from typing import Optional, Tuple, List, Dict, Type +from typing import Optional, Tuple, List, Type from text_generation.models import Model from text_generation.models.types import GeneratedText @@ -14,11 +14,23 @@ class CausalLMBatch: batch_id: int requests: List[generate_pb2.Request] - all_input_lengths: List[int] - input_ids: Dict[str, torch.Tensor] + + # Decoder values + input_ids: torch.Tensor + attention_mask: torch.Tensor + past_key_values: Optional[List[Tuple]] + + # All tokens all_input_ids: List[torch.Tensor] + + # Lengths of all generations present in the batch + input_lengths: List[int] + + # Generation helpers next_token_choosers: List[NextTokenChooser] stopping_criterias: List[StoppingCriteria] + + # Metadata used for padding size: int max_sequence_length: int @@ -36,12 +48,12 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] - all_input_lengths = [] + input_lengths = [] # Parse batch for r in pb.requests: inputs.append(r.inputs) - all_input_lengths.append(r.input_length) + input_lengths.append(r.input_length) next_token_choosers.append( NextTokenChooser( temperature=r.parameters.temperature, @@ -56,21 +68,23 @@ def from_pb( ) ) - input_ids = tokenizer( + tokenized_inputs = tokenizer( inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 ).to(device) - all_input_ids = input_ids["input_ids"].unsqueeze(-1) + all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) return cls( batch_id=pb.id, requests=pb.requests, - all_input_lengths=all_input_lengths, - input_ids=input_ids, + input_ids=tokenized_inputs["input_ids"], + attention_mask=tokenized_inputs["attention_mask"], + past_key_values=None, all_input_ids=all_input_ids, + input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=pb.size, - max_sequence_length=max(all_input_lengths), + max_sequence_length=max(input_lengths), ) @classmethod @@ -80,19 +94,23 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": max_sequence_length = max(batch.max_sequence_length for batch in batches) # Batch attributes - input_ids = {"input_ids": None, "attention_mask": None, "past_key_values": []} requests = [] - all_input_lengths = [] + input_lengths = [] all_input_ids = [] next_token_choosers = [] stopping_criterias = [] + # Batch tensors + input_ids = None + attention_mask = None + past_key_values = [] + # Used for slicing correctly inside the tensors # Equivalent to a cumsum on batch sizes start_index = 0 for i, batch in enumerate(batches): requests.extend(batch.requests) - all_input_lengths.extend(batch.all_input_lengths) + input_lengths.extend(batch.input_lengths) all_input_ids.extend(batch.all_input_ids) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -101,32 +119,35 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": end_index = start_index + batch.size # We only concatenate batches that did at least one step - if batch.input_ids["input_ids"].shape[1] > 1: + if batch.input_ids.shape[1] > 1: raise ValueError("Batch input_ids should be of shape (batch_size, 1)") - # Initialize tensors - if i == 0: - input_ids["input_ids"] = torch.empty( + # Create empty tensor + # input_ids is always of shape [batch_size, 1] + # We do not need to pad it + if input_ids is None: + input_ids = torch.empty( (total_batch_size, 1), - dtype=batch.input_ids["input_ids"].dtype, - device=batch.input_ids["input_ids"].device, + dtype=batch.input_ids.dtype, + device=batch.input_ids.device, ) - input_ids["attention_mask"] = torch.zeros( + # Copy to correct indices + input_ids[start_index:end_index] = batch.input_ids + + # Create padded tensor + if attention_mask is None: + attention_mask = torch.zeros( (total_batch_size, max_sequence_length), - dtype=batch.input_ids["attention_mask"].dtype, - device=batch.input_ids["attention_mask"].device, + dtype=batch.attention_mask.dtype, + device=batch.attention_mask.device, ) - # input_ids["input_ids"] is always of shape [batch_size, 1] - # We do not need to pad it - input_ids["input_ids"][start_index:end_index] = batch.input_ids["input_ids"] - # We need to slice the attention mask to remove padding from previous steps - input_ids["attention_mask"][ + attention_mask[ start_index:end_index, -batch.max_sequence_length : - ] = batch.input_ids["attention_mask"][:, -batch.max_sequence_length :] + ] = batch.attention_mask[:, -batch.max_sequence_length :] - for j, past in enumerate(batch.input_ids["past_key_values"]): + for j, past in enumerate(batch.past_key_values): # Shenanigans to get dimensions because BLOOM outputs a past with a different shape # BLOOM: [batch_size * num_heads, ...] vs [batch_size, num_heads, ...] head_dim, padded_sequence_length = past[0].shape[-2:] @@ -137,8 +158,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": ) # This will run only once per layer - if j == len(input_ids["past_key_values"]): - input_ids["past_key_values"].append([]) + if j == len(past_key_values): + past_key_values.append([]) # Decoder past for k, t in enumerate(past): @@ -172,21 +193,21 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # Initialize tensors # This will run only once per layer and per past tensor - if k == len(input_ids["past_key_values"][j]): - input_ids["past_key_values"][j].append( + if k == len(past_key_values[j]): + past_key_values[j].append( torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) ) # We slice the past keys and values to remove the padding from previous batches if not head_dim_last: - input_ids["past_key_values"][j][k][ + past_key_values[j][k][ start_index:end_index, :, :, -(batch.max_sequence_length - 1) :, ] = t[:, :, :, -(batch.max_sequence_length - 1) :] else: - input_ids["past_key_values"][j][k][ + past_key_values[j][k][ start_index:end_index, :, -(batch.max_sequence_length - 1) :, @@ -198,9 +219,11 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": return cls( batch_id=batches[0].batch_id, requests=requests, - all_input_lengths=all_input_lengths, input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, all_input_ids=all_input_ids, + input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=total_batch_size, @@ -209,7 +232,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": class CausalLM(Model): - def __init__(self, model_name: str): + def __init__(self, model_name: str, quantize=False): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -223,6 +246,7 @@ def __init__(self, model_name: str): model_name, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, + load_in_8bit=quantize, ).eval() super(CausalLM, self).__init__( @@ -255,16 +279,19 @@ def generate_token( torch.no_grad if self.device.type == "cpu" else torch.inference_mode ) with context_manager(): - logits, past = self.forward(**batch.input_ids) + logits, past = self.forward( + batch.input_ids, batch.attention_mask, batch.past_key_values + ) # List of indices to cache next_batch_keep_indices = [] - # New input_ids for next forward + # New values for next forward + next_batch_input_lengths = [] next_batch_input_ids = [] next_batch_all_input_ids = [] - next_all_input_lengths = [] + # Metadata next_batch_size = 0 next_batch_max_sequence_length = 0 @@ -274,7 +301,7 @@ def generate_token( # Zipped iterator iterator = zip( batch.requests, - batch.all_input_lengths, + batch.input_lengths, logits, batch.next_token_choosers, batch.stopping_criterias, @@ -313,7 +340,7 @@ def generate_token( next_batch_all_input_ids.append(all_tokens) next_batch_size += 1 new_input_length = input_length + 1 - next_all_input_lengths.append(new_input_length) + next_batch_input_lengths.append(new_input_length) next_batch_max_sequence_length = max( next_batch_max_sequence_length, new_input_length ) @@ -322,15 +349,14 @@ def generate_token( if not next_batch_keep_indices: return generated_texts, None - # If we finished at least one generation - next_batch_input_ids = {"input_ids": torch.cat(next_batch_input_ids, dim=0)} + next_batch_input_ids = torch.cat(next_batch_input_ids, dim=0) + # If we finished at least one generation, we need to evict the indices of the generations that finished + # from the values of the next batch if generated_texts: # Apply indices to attention mask, past key values and other items that need to be cached - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"][ - next_batch_keep_indices - ] + next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] # Force past to be of dim [batch_size, num_heads, ...] for easy indexing - next_batch_input_ids["past_key_values"] = [ + next_batch_past_key_values = [ [ t.view(-1, self.num_heads, *t.shape[-2:])[next_batch_keep_indices] for t in layer @@ -345,16 +371,16 @@ def generate_token( batch.stopping_criterias[i] for i in next_batch_keep_indices ] else: - next_batch_input_ids["attention_mask"] = batch.input_ids["attention_mask"] - next_batch_input_ids["past_key_values"] = past + next_batch_attention_mask = batch.attention_mask + next_batch_past_key_values = past next_batch_requests = batch.requests next_batch_next_token_choosers = batch.next_token_choosers next_batch_stopping_criterias = batch.stopping_criterias # Update attention_mask with padding as we added a new token to input_ids - next_batch_input_ids["attention_mask"] = torch.cat( + next_batch_attention_mask = torch.cat( [ - next_batch_input_ids["attention_mask"], + next_batch_attention_mask, torch.ones((next_batch_size, 1)).to(self.device), ], dim=1, @@ -363,9 +389,11 @@ def generate_token( next_batch = CausalLMBatch( batch_id=batch.batch_id, requests=next_batch_requests, - all_input_lengths=next_all_input_lengths, input_ids=next_batch_input_ids, + attention_mask=next_batch_attention_mask, + past_key_values=next_batch_past_key_values, all_input_ids=next_batch_all_input_ids, + input_lengths=next_batch_input_lengths, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 0607b3d5..cb1291ab 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -15,26 +15,33 @@ class Seq2SeqLMBatch: batch_id: int requests: List[generate_pb2.Request] + # Encoder values input_ids: torch.Tensor attention_mask: torch.Tensor + # Decoder values decoder_input_ids: torch.Tensor decoder_attention_mask: Optional[torch.Tensor] encoder_last_hidden_state: Optional[torch.Tensor] + # Seq2SeqLM keeps track of both encoder and decoder attention keys and values past_key_values: Optional[List[Tuple]] + # Lengths of all generations present in the batch input_lengths: List[int] decoder_input_lengths: List[int] + # Generation helpers next_token_choosers: List[NextTokenChooser] stopping_criterias: List[StoppingCriteria] + # Metadata used for padding size: int max_input_length: int max_decoder_input_length: int def to_pb(self): + """Convert a Seq2SeqLMBatch to a text_generation.v1.Batch protobuf""" return generate_pb2.Batch( id=self.batch_id, requests=self.requests, @@ -45,6 +52,7 @@ def to_pb(self): def from_pb( cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device ) -> "Seq2SeqLMBatch": + """Convert a text_generation.v1.Batch protobuf to a Seq2SeqLMBatch""" inputs = [] next_token_choosers = [] stopping_criterias = [] @@ -57,6 +65,7 @@ def from_pb( for r in pb.requests: inputs.append(r.inputs) input_lengths.append(r.input_length) + # Decoder sequence only contains the bos_token decoder_input_ids.append(tokenizer.bos_token_id) decoder_input_lengths.append(1) next_token_choosers.append( @@ -73,9 +82,11 @@ def from_pb( ) ) + # Tokenize batch tokenized_inputs = tokenizer( inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 ).to(device) + # Convert decoder_input_ids to torch tensor of size [batch_size, 1] decoder_input_ids = torch.tensor(decoder_input_ids).to(device).unsqueeze(-1) return cls( @@ -98,6 +109,8 @@ def from_pb( @classmethod def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": + """Concatenate multiple batches together by padding internal torch tensors""" + # Used for padding total_batch_size = sum(batch.size for batch in batches) max_input_length = max(batch.max_input_length for batch in batches) @@ -112,6 +125,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": next_token_choosers = [] stopping_criterias = [] + # Batch tensors input_ids = None attention_mask = None decoder_input_ids = None @@ -122,7 +136,9 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Used for slicing correctly inside the tensors # Equivalent to a cumsum on batch sizes start_index = 0 + for i, batch in enumerate(batches): + # Extend all list attributes requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) decoder_input_lengths.extend(batch.decoder_input_lengths) @@ -136,51 +152,62 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": if batch.encoder_last_hidden_state is None: raise ValueError("Batch encoder_last_hidden_state cannot be None") + # Create padded tensor if input_ids is None: input_ids = torch.zeros( (total_batch_size, max_input_length), dtype=batch.input_ids.dtype, device=batch.input_ids.device, ) + # Copy to correct indices input_ids[ start_index:end_index, -batch.max_input_length : ] = batch.input_ids[:, -batch.max_input_length :] + # Create padded tensor if attention_mask is None: attention_mask = torch.zeros( (total_batch_size, max_input_length), dtype=batch.attention_mask.dtype, device=batch.attention_mask.device, ) + # Copy to correct indices attention_mask[ start_index:end_index, -batch.max_input_length : ] = batch.attention_mask[:, -batch.max_input_length :] + # Create padded tensor if decoder_input_ids is None: decoder_input_ids = torch.zeros( (total_batch_size, max_decoder_input_length), dtype=batch.decoder_input_ids.dtype, device=batch.decoder_input_ids.device, ) + # Copy to correct indices decoder_input_ids[ start_index:end_index, -batch.max_decoder_input_length : ] = batch.decoder_input_ids[:, -batch.max_decoder_input_length :] + # Create padded tensor if decoder_attention_mask is None: decoder_attention_mask = torch.zeros( (total_batch_size, max_decoder_input_length), - dtype=batch.attention_mask.dtype, - device=batch.attention_mask.device, + dtype=batch.attention_mask.dtype, # As decoder_attention_mask might not exist, + device=batch.attention_mask.device, # we use `batch.attention_maks` for device here ) + # If the decoder mask does not exist yet, all generations started at the same time and we never concatenated + # this batch. All generations are of length `batch.max_decoder_input_length`. if batch.decoder_attention_mask is None: decoder_attention_mask[ start_index:end_index, -batch.max_decoder_input_length : ] = 1 + # If it exists, we need to index else: decoder_attention_mask[ start_index:end_index, -batch.max_decoder_input_length : ] = batch.decoder_attention_mask[:, -batch.max_decoder_input_length :] + # Create padded tensor if encoder_last_hidden_state is None: encoder_last_hidden_state = torch.zeros( ( @@ -192,10 +219,12 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": device=batch.encoder_last_hidden_state.device, ) + # Copy to correct indices encoder_last_hidden_state[ start_index:end_index, -batch.max_decoder_input_length :, : ] = batch.encoder_last_hidden_state[:, -batch.max_decoder_input_length :, :] + # Iterate over attention layers for j, past in enumerate(batch.past_key_values): _, num_heads, _, head_dim = past[0].shape @@ -271,7 +300,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": class Seq2SeqLM(Model): - def __init__(self, model_name: str): + def __init__(self, model_name: str, quantize=False): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -283,6 +312,7 @@ def __init__(self, model_name: str): model_name, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, + load_in_8bit=quantize, ).eval() tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") tokenizer.bos_token_id = self.model.config.decoder_start_token_id @@ -314,14 +344,17 @@ def forward( if past_key_values is not None: decoder_input_ids = decoder_input_ids[:, -1].unsqueeze(-1) + # Wrap `encoder_last_hidden_state` because for some reason, Transformers does a `encoder_last_hidden_state[0]` + # internally... + if encoder_last_hidden_state is not None: + encoder_last_hidden_state = [encoder_last_hidden_state] + outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask, decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, - encoder_outputs=[encoder_last_hidden_state] - if encoder_last_hidden_state is not None - else None, + encoder_outputs=encoder_last_hidden_state, past_key_values=past_key_values, use_cache=True, ) @@ -351,11 +384,12 @@ def generate_token( # List of indices to cache next_batch_keep_indices = [] - # New input_ids for next forward + # New values for next forward next_batch_input_lengths = [] next_batch_decoder_input_ids = [] next_batch_decoder_input_lengths = [] + # Metadata next_batch_size = 0 next_batch_max_input_length = 0 next_batch_max_decoder_input_length = 0 @@ -395,7 +429,7 @@ def generate_token( # Evaluate stopping criteria if stopping_criteria(decoder_tokens): - # Decode all tokens + # Decode tokens output = self.tokenizer.decode(decoder_tokens, skip_special_tokens=True) # Add to the list of finished generations with the original request generated_texts.append( @@ -420,9 +454,11 @@ def generate_token( if not next_batch_keep_indices: return generated_texts, None - # If we finished at least one generation next_batch_decoder_input_ids = torch.cat(next_batch_decoder_input_ids) + # If we finished at least one generation, we need to evict the indices of the generations that finished + # from the values of the next batch if generated_texts: + # Apply indices to attention mask, past key values and other items that need to be cached next_batch_input_ids = batch.input_ids[next_batch_keep_indices] next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] @@ -458,7 +494,7 @@ def generate_token( next_batch_next_token_choosers = batch.next_token_choosers next_batch_stopping_criterias = batch.stopping_criterias - # Update attention_mask with padding as we added a new token to input_ids + # Update decoder_attention_mask with padding as we added a new token to input_ids if next_batch_decoder_attention_mask is not None: next_batch_decoder_attention_mask = torch.cat( [ From fa43fb71be75064de58784e96b16b14f7b3b4912 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Tue, 8 Nov 2022 17:42:38 +0100 Subject: [PATCH 026/793] fix(server): Fix Transformers fork version --- .dockerignore | 3 ++- Dockerfile | 2 ++ aml/deployment.yaml | 10 +++++----- server/Makefile | 12 ++++++------ server/text_generation/models/bloom.py | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.dockerignore b/.dockerignore index 5aa1aa3a..fcfaad0d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ aml -target \ No newline at end of file +target +server/transformers \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9b6ef835..214a0ace 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM rust:1.64 as router-builder WORKDIR /usr/src +COPY rust-toolchain.toml rust-toolchain.toml COPY proto proto COPY router router @@ -13,6 +14,7 @@ FROM rust:1.64 as launcher-builder WORKDIR /usr/src +COPY rust-toolchain.toml rust-toolchain.toml COPY launcher launcher WORKDIR /usr/src/launcher diff --git a/aml/deployment.yaml b/aml/deployment.yaml index 35d19006..51e124b6 100644 --- a/aml/deployment.yaml +++ b/aml/deployment.yaml @@ -8,7 +8,7 @@ environment_variables: MODEL_NAME: bigscience/bloom NUM_GPUS: 8 environment: - image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.2 + image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.3.1 inference_config: liveness_route: port: 3000 @@ -25,14 +25,14 @@ request_settings: max_concurrent_requests_per_instance: 256 liveness_probe: initial_delay: 600 - timeout: 20 + timeout: 90 period: 120 success_threshold: 1 - failure_threshold: 3 + failure_threshold: 5 readiness_probe: initial_delay: 600 - timeout: 20 + timeout: 90 period: 120 success_threshold: 1 - failure_threshold: 3 + failure_threshold: 5 instance_count: 1 diff --git a/server/Makefile b/server/Makefile index 39a98b65..57dea48f 100644 --- a/server/Makefile +++ b/server/Makefile @@ -7,13 +7,13 @@ gen-server: touch text_generation/pb/__init__.py install-transformers: - # Install specific version of transformers + # Install specific version of transformers with custom cuda kernels rm transformers || true - rm transformers-7302a24535e8dc5637ea5b4e4572fc971d404098 || true - curl -L -O https://github.com/OlivierDehaene/transformers/archive/7302a24535e8dc5637ea5b4e4572fc971d404098.zip - unzip 7302a24535e8dc5637ea5b4e4572fc971d404098.zip - rm 7302a24535e8dc5637ea5b4e4572fc971d404098.zip - mv transformers-7302a24535e8dc5637ea5b4e4572fc971d404098 transformers + rm transformers-b55f16c5b71aeef47a66a4270e19c154f050a7a7 || true + curl -L -O https://github.com/OlivierDehaene/transformers/archive/b55f16c5b71aeef47a66a4270e19c154f050a7a7.zip + unzip b55f16c5b71aeef47a66a4270e19c154f050a7a7.zip + rm b55f16c5b71aeef47a66a4270e19c154f050a7a7.zip + mv transformers-b55f16c5b71aeef47a66a4270e19c154f050a7a7 transformers cd transformers && python setup.py install install-torch: diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 730958c1..38ef8ef7 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -38,7 +38,7 @@ def __init__(self, model_name: str, quantize: bool = False): self.master = self.rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{self.rank}") - dtype = torch.float16 + dtype = torch.bfloat16 else: device = torch.device("cpu") dtype = torch.float32 From dccd5c2b1acdee242d847e1fbeea2edabfdde15f Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 9 Nov 2022 18:24:07 +0100 Subject: [PATCH 027/793] feat(server): Clarify CausalLMBatch concatenate method --- Cargo.lock | 52 ++++----- server/text_generation/models/causal_lm.py | 122 +++++++++++---------- 2 files changed, 88 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8f09c3a5..0fd5c4bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -213,9 +213,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.73" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f" [[package]] name = "cfg-if" @@ -240,9 +240,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.18" +version = "4.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335867764ed2de42325fafe6d18b8af74ba97ee0c590fa016f157535b42ab04b" +checksum = "91b9970d7505127a162fdaa9b96428d28a479ba78c9ec7550a63a5d9863db682" dependencies = [ "atty", "bitflags", @@ -255,9 +255,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.0.18" +version = "4.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a1b0f6422af32d5da0c58e2703320f379216ee70198241c84173a8c5ac28f3" +checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -790,9 +790,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.20" +version = "0.14.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" dependencies = [ "bytes", "futures-channel", @@ -898,9 +898,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" [[package]] name = "itertools" @@ -1053,9 +1053,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "native-tls" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" dependencies = [ "lazy_static", "libc", @@ -1103,9 +1103,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ "hermit-abi", "libc", @@ -1125,9 +1125,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "onig" @@ -1293,9 +1293,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro-error" @@ -1479,9 +1479,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "aho-corasick", "memchr", @@ -1490,9 +1490,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" @@ -1802,7 +1802,7 @@ dependencies = [ name = "text-generation-launcher" version = "0.1.0" dependencies = [ - "clap 4.0.18", + "clap 4.0.22", "ctrlc", "subprocess", "tracing", @@ -1814,7 +1814,7 @@ name = "text-generation-router" version = "0.1.0" dependencies = [ "axum", - "clap 4.0.18", + "clap 4.0.22", "futures", "parking_lot", "serde", @@ -1893,9 +1893,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokenizers" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d7b08ede6742d7a59d58c71da8a6fa21bedc433dca2e855e439274d08df1170" +checksum = "f4ff2dd291eac98dcea13e8cf7a0b28c373a90dc9210ccdab0fa9e69ee0cac69" dependencies = [ "aho-corasick", "cached-path", diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 2ba36b1a..c1057635 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -148,71 +148,73 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": ] = batch.attention_mask[:, -batch.max_sequence_length :] for j, past in enumerate(batch.past_key_values): + past_keys, past_values = past + # Shenanigans to get dimensions because BLOOM outputs a past with a different shape - # BLOOM: [batch_size * num_heads, ...] vs [batch_size, num_heads, ...] - head_dim, padded_sequence_length = past[0].shape[-2:] - num_heads = ( - past[0] - .view(batch.size, -1, head_dim, padded_sequence_length) - .shape[1] + # BLOOM Keys: [batch_size * num_heads, head_dim, seq_length] + # BLOOM Values: [batch_size * num_heads, seq_length, head_dim] + past_keys = past_keys.view(batch.size, -1, *past_keys.shape[-2:]) + past_values = past_values.view(batch.size, -1, *past_values.shape[-2:]) + + _, num_heads, head_dim, padded_sequence_length = past_keys.shape + + padded_past_keys_shape = ( + total_batch_size, + num_heads, + head_dim, + max_sequence_length - 1, ) + # head_dim is last for BLOOM + if past_values.shape[-1] == head_dim: + past_values_head_dim_last = True + padded_past_values_shape = ( + total_batch_size, + num_heads, + max_sequence_length - 1, + head_dim, + ) + elif past_values.shape[-2] == head_dim: + past_values_head_dim_last = False + padded_past_values_shape = padded_past_keys_shape + else: + raise ValueError( + f"past_values shape {past_values.shape} is not valid" + ) + # This will run only once per layer if j == len(past_key_values): - past_key_values.append([]) - - # Decoder past - for k, t in enumerate(past): - # Needed because BLOOM past shapes are not the same for keys and values - # Keys: [batch_size * num_heads, head_dim, seq_length] - # Values: [batch_size * num_heads, seq_length, head_dim] - head_dim_last = False - if t.shape[-2] == head_dim: - t = t.view( - batch.size, num_heads, head_dim, padded_sequence_length - ) - padded_t_shape = ( - total_batch_size, - num_heads, - head_dim, - max_sequence_length - 1, - ) - elif t.shape[-1] == head_dim: - head_dim_last = True - t = t.view( - batch.size, num_heads, padded_sequence_length, head_dim - ) - padded_t_shape = ( - total_batch_size, - num_heads, - max_sequence_length - 1, - head_dim, - ) - else: - raise ValueError(f"shape {t.shape} is not valid") - - # Initialize tensors - # This will run only once per layer and per past tensor - if k == len(past_key_values[j]): - past_key_values[j].append( - torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) - ) - - # We slice the past keys and values to remove the padding from previous batches - if not head_dim_last: - past_key_values[j][k][ - start_index:end_index, - :, - :, - -(batch.max_sequence_length - 1) :, - ] = t[:, :, :, -(batch.max_sequence_length - 1) :] - else: - past_key_values[j][k][ - start_index:end_index, - :, - -(batch.max_sequence_length - 1) :, - :, - ] = t[:, :, -(batch.max_sequence_length - 1) :, :] + padded_past_keys = torch.zeros( + padded_past_keys_shape, + dtype=past_keys.dtype, + device=past_keys.device, + ) + padded_past_values = torch.zeros( + padded_past_values_shape, + dtype=past_values.dtype, + device=past_values.device, + ) + past_key_values.append((padded_past_keys, padded_past_values)) + + # We slice the past keys and values to remove the padding from previous batches + past_key_values[j][0][ + start_index:end_index, :, :, -(batch.max_sequence_length - 1) : + ] = past_keys[:, :, :, -(batch.max_sequence_length - 1) :] + + if past_values_head_dim_last: + past_key_values[j][1][ + start_index:end_index, + :, + -(batch.max_sequence_length - 1) :, + :, + ] = past_values[:, :, -(batch.max_sequence_length - 1) :, :] + else: + past_key_values[j][1][ + start_index:end_index, + :, + :, + -(batch.max_sequence_length - 1) :, + ] = past_values[:, :, :, -(batch.max_sequence_length - 1) :] start_index += batch.size From 6c781025ae1c015a175eb2e319bc6003b5d5bdab Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 14 Nov 2022 13:59:56 +0100 Subject: [PATCH 028/793] feat(rust): Update to 1.65 --- Cargo.toml | 1 + Dockerfile | 4 ++-- rust-toolchain.toml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e23b802e..684a5da4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,3 +12,4 @@ exclude = [ debug = 1 incremental = true lto = "off" +panic = "abort" diff --git a/Dockerfile b/Dockerfile index 214a0ace..e8455b03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.64 as router-builder +FROM rust:1.65 as router-builder WORKDIR /usr/src @@ -10,7 +10,7 @@ WORKDIR /usr/src/router RUN cargo install --path . -FROM rust:1.64 as launcher-builder +FROM rust:1.65 as launcher-builder WORKDIR /usr/src diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 30355814..122f54dd 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.64.0" +channel = "1.65.0" components = ["rustfmt", "clippy"] \ No newline at end of file From 91f5f862804a2cb5ab4cb65d9634ab9017168a67 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 14 Nov 2022 14:34:15 +0100 Subject: [PATCH 029/793] fix(router): Fix HTTP status codes --- router/src/batcher.rs | 2 +- router/src/validation.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 074c54bc..aacc9634 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -220,7 +220,7 @@ impl From for (StatusCode, Json) { fn from(err: InferError) -> Self { match err { InferError::GenerationError(_) => ( - StatusCode::INTERNAL_SERVER_ERROR, + StatusCode::FAILED_DEPENDENCY, Json(ErrorResponse { error: err.to_string(), }), diff --git a/router/src/validation.rs b/router/src/validation.rs index 43c246f0..a8437e39 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -101,7 +101,7 @@ fn validation_worker( ) { // Loop over requests while let Some((request, response_tx)) = receiver.blocking_recv() { - if request.parameters.temperature < 0.0 { + if request.parameters.temperature <= 0.0 { response_tx .send(Err(ValidationError::Temperature)) .unwrap_or(()); @@ -162,7 +162,7 @@ pub enum ValidationError { impl From for (StatusCode, Json) { fn from(err: ValidationError) -> Self { ( - StatusCode::BAD_REQUEST, + StatusCode::UNPROCESSABLE_ENTITY, Json(ErrorResponse { error: err.to_string(), }), From feb7806ca460416e179b34ea1d991bab131ee4c4 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 14 Nov 2022 16:22:10 +0100 Subject: [PATCH 030/793] fix(readme): Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd9e2176..d081ad7f 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ to power Bloom, BloomZ and MT0-XXL api-inference widgets. ## Features -- [Dynamic bathing of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput +- [Dynamic batching of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading - 45ms per token generation for BLOOM with 8xA100 80GB From d6d5b12e0309433e0f2797afde901a8c0c8d6deb Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:15:19 +0100 Subject: [PATCH 031/793] fix(router): Handle tokenizer errors --- router/src/validation.rs | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/router/src/validation.rs b/router/src/validation.rs index a8437e39..a105ddf3 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -123,20 +123,26 @@ fn validation_worker( } // Get the number of tokens in the input - let inputs = tokenizer.encode(request.inputs.clone(), false).unwrap(); - let input_length = inputs.len(); - - if input_length > max_input_length { - response_tx - .send(Err(ValidationError::InputLength( - input_length, - max_input_length, - ))) - .unwrap_or(()); - continue; - } - - response_tx.send(Ok((input_length, request))).unwrap_or(()); + match tokenizer.encode(request.inputs.clone(), false) { + Ok(inputs) => { + let input_length = inputs.len(); + + if input_length > max_input_length { + response_tx + .send(Err(ValidationError::InputLength( + input_length, + max_input_length, + ))) + .unwrap_or(()); + continue; + } + + response_tx.send(Ok((input_length, request))).unwrap_or(()); + } + Err(err) => response_tx + .send(Err(ValidationError::Tokenizer(err.to_string()))) + .unwrap_or(()), + }; } } @@ -157,6 +163,8 @@ pub enum ValidationError { MaxNewTokens, #[error("inputs must have less than {1} tokens. Given: {0}")] InputLength(usize, usize), + #[error("tokenizer error {0}")] + Tokenizer(String), } impl From for (StatusCode, Json) { From daa1d81d5ec4ef9bc59a4d6e850687b788732c90 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 1 Dec 2022 19:31:54 +0100 Subject: [PATCH 032/793] feat(server): Support Galactica (#4) --- README.md | 1 + server/Makefile | 10 +- server/text_generation/models/__init__.py | 6 + server/text_generation/models/bloom.py | 2 + server/text_generation/models/causal_lm.py | 46 ++- server/text_generation/models/galactica.py | 346 +++++++++++++++++++++ server/text_generation/utils.py | 2 +- 7 files changed, 383 insertions(+), 30 deletions(-) create mode 100644 server/text_generation/models/galactica.py diff --git a/README.md b/README.md index d081ad7f..e76e38d0 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ to power Bloom, BloomZ and MT0-XXL api-inference widgets. - [BLOOM](https://huggingface.co/bigscience/bloom) - [BLOOMZ](https://huggingface.co/bigscience/bloomz) - [MT0-XXL](https://huggingface.co/bigscience/mt0-xxl) +- ~~[Galactica](https://huggingface.co/facebook/galactica-120b)~~ (deactivated) Other models are supported on a best effort basis using: diff --git a/server/Makefile b/server/Makefile index 57dea48f..10028dad 100644 --- a/server/Makefile +++ b/server/Makefile @@ -9,11 +9,11 @@ gen-server: install-transformers: # Install specific version of transformers with custom cuda kernels rm transformers || true - rm transformers-b55f16c5b71aeef47a66a4270e19c154f050a7a7 || true - curl -L -O https://github.com/OlivierDehaene/transformers/archive/b55f16c5b71aeef47a66a4270e19c154f050a7a7.zip - unzip b55f16c5b71aeef47a66a4270e19c154f050a7a7.zip - rm b55f16c5b71aeef47a66a4270e19c154f050a7a7.zip - mv transformers-b55f16c5b71aeef47a66a4270e19c154f050a7a7 transformers + rm transformers-text_generation_inference || true + curl -L -O https://github.com/OlivierDehaene/transformers/archive/refs/heads/text_generation_inference.zip + unzip text_generation_inference.zip + rm text_generation_inference.zip + mv transformers-text_generation_inference transformers cd transformers && python setup.py install install-torch: diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index bf44115c..b364309a 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -2,6 +2,7 @@ from text_generation.models.causal_lm import CausalLM from text_generation.models.bloom import BLOOMSharded from text_generation.models.seq2seq_lm import Seq2SeqLM +from text_generation.models.galactica import Galactica, GalacticaSharded __all__ = ["Model", "BLOOMSharded", "CausalLM", "Seq2SeqLM"] @@ -12,6 +13,11 @@ def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: return BLOOMSharded(model_name, quantize=quantize) else: return CausalLM(model_name, quantize=quantize) + elif model_name.startswith("facebook/galactica"): + if sharded: + return GalacticaSharded(model_name, quantize=quantize) + else: + return Galactica(model_name, quantize=quantize) else: if sharded: raise ValueError("sharded is not supported for AutoModel") diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 38ef8ef7..008288f8 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -63,6 +63,8 @@ def __init__(self, model_name: str, quantize: bool = False): torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_name, extension=".safetensors") + if not filenames: + raise ValueError("No safetensors weights found") with init_empty_weights(): model = AutoModelForCausalLM.from_config(config) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index c1057635..ca8ea575 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -156,31 +156,29 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": past_keys = past_keys.view(batch.size, -1, *past_keys.shape[-2:]) past_values = past_values.view(batch.size, -1, *past_values.shape[-2:]) - _, num_heads, head_dim, padded_sequence_length = past_keys.shape + _, num_heads, padded_sequence_length, head_dim = past_values.shape - padded_past_keys_shape = ( + padded_past_values_shape = ( total_batch_size, num_heads, - head_dim, max_sequence_length - 1, + head_dim, ) - # head_dim is last for BLOOM - if past_values.shape[-1] == head_dim: - past_values_head_dim_last = True - padded_past_values_shape = ( + # seq_length is last for BLOOM + if past_keys.shape[-2] == head_dim: + past_keys_head_dim_last = False + padded_past_keys_shape = ( total_batch_size, num_heads, - max_sequence_length - 1, head_dim, + max_sequence_length - 1, ) - elif past_values.shape[-2] == head_dim: - past_values_head_dim_last = False - padded_past_values_shape = padded_past_keys_shape + elif past_keys.shape[-1] == head_dim: + past_keys_head_dim_last = True + padded_past_keys_shape = padded_past_values_shape else: - raise ValueError( - f"past_values shape {past_values.shape} is not valid" - ) + raise ValueError(f"past_keys shape {past_keys.shape} is not valid") # This will run only once per layer if j == len(past_key_values): @@ -197,24 +195,24 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": past_key_values.append((padded_past_keys, padded_past_values)) # We slice the past keys and values to remove the padding from previous batches - past_key_values[j][0][ - start_index:end_index, :, :, -(batch.max_sequence_length - 1) : - ] = past_keys[:, :, :, -(batch.max_sequence_length - 1) :] - - if past_values_head_dim_last: - past_key_values[j][1][ + if past_keys_head_dim_last: + past_key_values[j][0][ start_index:end_index, :, -(batch.max_sequence_length - 1) :, :, - ] = past_values[:, :, -(batch.max_sequence_length - 1) :, :] + ] = past_keys[:, :, -(batch.max_sequence_length - 1) :, :] else: - past_key_values[j][1][ + past_key_values[j][0][ start_index:end_index, :, :, -(batch.max_sequence_length - 1) :, - ] = past_values[:, :, :, -(batch.max_sequence_length - 1) :] + ] = past_keys[:, :, :, -(batch.max_sequence_length - 1) :] + + past_key_values[j][1][ + start_index:end_index, :, -(batch.max_sequence_length - 1) :, : + ] = past_values[:, :, -(batch.max_sequence_length - 1) :, :] start_index += batch.size @@ -243,13 +241,13 @@ def __init__(self, model_name: str, quantize=False): dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") - tokenizer.add_special_tokens({"pad_token": "[PAD]"}) self.model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, load_in_8bit=quantize, ).eval() + tokenizer.pad_token_id = self.model.config.pad_token_id super(CausalLM, self).__init__( tokenizer=tokenizer, diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py new file mode 100644 index 00000000..abc3c36c --- /dev/null +++ b/server/text_generation/models/galactica.py @@ -0,0 +1,346 @@ +import re +import torch +import torch.distributed + +from typing import List, Optional, Type + +from accelerate import init_empty_weights +from safetensors import safe_open +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig +from transformers.models.opt.parallel_layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, +) + +from text_generation.models import CausalLM +from text_generation.pb import generate_pb2 +from text_generation.models.causal_lm import CausalLMBatch +from text_generation.utils import ( + NextTokenChooser, + StoppingCriteria, + initialize_torch_distributed, + weight_files, + download_weights, +) + +HAS_BITS_AND_BYTES = True +try: + import bitsandbytes as bnb + from bitsandbytes.nn import Int8Params +except Exception as e: + HAS_BITS_AND_BYTES = False + +torch.manual_seed(0) + +# CREDIT: Papers with code => https://github.com/paperswithcode/galai/blob/main/galai/utils.py + +# we split individual characters inside special tokens like [START_DNA] +CUSTOM_SEQ_RE = re.compile(r"(\[START_(DNA|SMILES|I_SMILES|AMINO)])(.*?)(\[END_\2])") + +# token added to implement a custom sequence tokenization. This token is added at +# corpus cleaning step and removed in pretokenization. The digits are added to increase the chance +# that they do not occur in the corpus. The digits are escaped so that the token does not appear +# literally in the source code in case we ever include it in the training data. +SPLIT_MARKER = f"SPL{1}T-TH{1}S-Pl3A5E" + + +def _insert_split_marker(m: re.Match): + """ + Applies split marker based on a regex match of special tokens such as + [START_DNA]. + Parameters + ---------- + n : str + Input text to split + Returns + ---------- + str - the text with the split token added + """ + start_token, _, sequence, end_token = m.groups() + sequence = re.sub(r"(.)", rf"{SPLIT_MARKER}\1", sequence, flags=re.DOTALL) + return f"{start_token}{sequence}{SPLIT_MARKER}{end_token}" + + +def escape_custom_split_sequence(text): + """ + Applies custom splitting to the text for GALILEO's tokenization + Parameters + ---------- + text : str + Input text to split + Returns + ---------- + str - the text with the split token added + """ + return CUSTOM_SEQ_RE.sub(_insert_split_marker, text) + + +# END CREDIT + + +class GalacticaCausalLMBatch(CausalLMBatch): + @classmethod + def from_pb( + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + ) -> "CausalLMBatch": + inputs = [] + next_token_choosers = [] + stopping_criterias = [] + input_lengths = [] + + # Parse batch + for r in pb.requests: + # Add escape_custom_split_sequence to the CausalLMBatch logic + inputs.append(escape_custom_split_sequence(r.inputs)) + input_lengths.append(r.input_length) + next_token_choosers.append( + NextTokenChooser( + temperature=r.parameters.temperature, + top_k=r.parameters.top_k, + top_p=r.parameters.top_p, + do_sample=r.parameters.do_sample, + ) + ) + stopping_criterias.append( + StoppingCriteria( + eos_token_id=tokenizer.eos_token_id, max_new_tokens=r.max_new_tokens + ) + ) + + tokenized_inputs = tokenizer( + inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 + ).to(device) + all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) + + return cls( + batch_id=pb.id, + requests=pb.requests, + input_ids=tokenized_inputs["input_ids"], + attention_mask=tokenized_inputs["attention_mask"], + past_key_values=None, + all_input_ids=all_input_ids, + input_lengths=input_lengths, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + size=pb.size, + max_sequence_length=max(input_lengths), + ) + + +class Galactica(CausalLM): + @property + def batch_type(self) -> Type[CausalLMBatch]: + return GalacticaCausalLMBatch + + +class GalacticaSharded(Galactica): + def __init__(self, model_name: str, quantize: bool = False): + if not model_name.startswith("facebook/galactica"): + raise ValueError(f"Model {model_name} is not supported") + + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + device = torch.device(f"cuda:{self.rank}") + dtype = torch.bfloat16 + else: + device = torch.device("cpu") + dtype = torch.float32 + + tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + + config = AutoConfig.from_pretrained(model_name, tp_parallel=True) + tokenizer.pad_token_id = config.pad_token_id + + # The flag below controls whether to allow TF32 on matmul. This flag defaults to False + # in PyTorch 1.12 and later. + torch.backends.cuda.matmul.allow_tf32 = True + + # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. + torch.backends.cudnn.allow_tf32 = True + + # Only download weights for small models + if self.master and model_name == "facebook/galactica-125m": + download_weights(model_name, extension=".safetensors") + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_name, extension=".safetensors") + if not filenames: + raise ValueError("No safetensors weights found") + + with init_empty_weights(): + model = AutoModelForCausalLM.from_config(config) + + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + quantize=quantize, + device=device, + rank=self.rank, + world_size=self.world_size, + ) + self.model = model.eval().to(dtype) + torch.distributed.barrier(group=self.process_group) + super(CausalLM, self).__init__( + tokenizer=tokenizer, + num_heads=config.num_attention_heads // self.process_group.size(), + device=device, + ) + + @staticmethod + def load_weights( + model, + filenames: List[str], + quantize: bool, + device: torch.device, + rank: int, + world_size: int, + ): + parameters = dict(model.named_parameters()) + for file in filenames: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: + for name in f.keys(): + if name == "lm_head.weight": + continue + + module_name, param_name = name.rsplit(".", 1) + try: + module = model.get_submodule(module_name) + except Exception as e: + print(type(model), name, module_name, param_name) + raise e + current_tensor = parameters[name] + + slice_ = f.get_slice(name) + + if isinstance(module, TensorParallelColumnLinear): + if param_name == "weight": + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + tensor = tensor.transpose(1, 0) + else: + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif isinstance(module, TensorParallelRowLinear): + if param_name == "weight": + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + tensor = tensor.transpose(1, 0) + else: + tensor = slice_[:] + # XXX: Hack for Rowlinear to add the bias only once. + if rank != 0: + tensor = torch.zeros_like(tensor) + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + else: + tensor = slice_[:] + + if current_tensor.shape != tensor.shape: + raise ValueError( + f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" + ) + + tensor = tensor.contiguous() + + if quantize: + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine either because it is not installed " + "or you don't have a GPU.\n" + "You can install it with `pip install bitsandbytes`." + ) + + if ( + type(module) + in [TensorParallelRowLinear, TensorParallelColumnLinear] + and param_name == "weight" + ): + tensor = Int8Params( + tensor.transpose(1, 0), + has_fp16_weights=False, + requires_grad=False, + ).to(device) + state = bnb.MatmulLtState() + state.threshold = 6.0 + state.has_fp16_weights = False + state.memory_efficient_backward = False + state.use_pool = True + state.CB = tensor.CB + state.SCB = tensor.SCB + tensor.CB = None + tensor.SCB = None + + def replace_linear(state, in_features, out_features): + def linear(input, weight, bias): + size_out = input.size()[:-1] + (out_features,) + input = input.view(-1, in_features) + out = torch.empty( + size_out, device=input.device, dtype=input.dtype + ) + out = bnb.matmul( + input, + weight, + out=out.view(-1, out_features), + state=state, + threshold=state.threshold, + bias=bias, + ) + + if state.CB is not None: + # we converted 8-bit row major to turing/ampere format + # in the first inference pass + # we no longer need the row-major weight + del state.CB + weight.data = state.CxB + + return out.view(size_out) + + return linear + + module.linear = replace_linear( + state, module.in_features, module.out_features + ) + + else: + tensor = tensor.to(device) + + module._parameters[param_name] = tensor + if name == "model.decoder.embed_tokens.weight": + model.lm_head._parameters["weight"] = tensor + + def forward(self, input_ids, attention_mask, past_key_values: Optional = None): + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + # Logits are sharded, so we need to gather them + logits_shard = outputs.logits[:, -1, :].contiguous() + + batch_size, vocab_shard_size = logits_shard.shape + vocab_size = self.world_size * vocab_shard_size + logits = [torch.empty_like(logits_shard) for _ in range(self.world_size)] + torch.distributed.all_gather(logits, logits_shard, group=self.process_group) + logits = torch.cat(logits, dim=1).view(batch_size, 1, vocab_size) + + return logits, outputs.past_key_values diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 0e2d9ae4..e6bfc391 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -11,7 +11,7 @@ from huggingface_hub import HfApi, hf_hub_download, try_to_load_from_cache from huggingface_hub.utils import LocalEntryNotFoundError from tqdm import tqdm -from transformers.generation_logits_process import ( +from transformers.generation.logits_process import ( LogitsProcessorList, TemperatureLogitsWarper, TopPLogitsWarper, From 31d76e238df7654157ab1e372b7d57ef859daaa7 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Mon, 5 Dec 2022 01:10:59 -0800 Subject: [PATCH 033/793] fix(batching): Avoid theoretical hang in batcher loop (#5) - Avoid theoretical hang in batcher loop - Avoid a couple of clones in the router generate method - Keep attention mask tensors as integers - Remove num_heads attribute Co-authored-by: OlivierDehaene --- router/src/batcher.rs | 9 ++++----- router/src/server.rs | 6 +----- router/src/validation.rs | 2 +- server/text_generation/models/bloom.py | 1 - server/text_generation/models/causal_lm.py | 5 ++--- server/text_generation/models/galactica.py | 1 - server/text_generation/models/model.py | 3 +-- server/text_generation/models/seq2seq_lm.py | 5 ++--- 8 files changed, 11 insertions(+), 21 deletions(-) diff --git a/router/src/batcher.rs b/router/src/batcher.rs index aacc9634..a9b892cc 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -104,10 +104,9 @@ async fn batching_task( // Get the next batch from the DB // This batch might be smaller than the maximum batch size if there are not enough requests // waiting in the DB - let mut waiting_tokens = 0; - if let Some((request_ids, batch)) = db.next_batch(None, max_batch_size) { + while let Some((request_ids, batch)) = db.next_batch(None, max_batch_size) { let mut cached_batch = wrap_future(client.generate(batch), request_ids, &db).await; - waiting_tokens += 1; + let mut waiting_tokens = 1; // We loop until we do not receive any cached batch from the inference server (== until // all requests have met their stopping criteria) @@ -131,11 +130,11 @@ async fn batching_task( if let Some((new_request_ids, new_batch)) = db.next_batch(min_size, max_batch_size) { - // Reset waiting counter - waiting_tokens = 0; // Generate one token for this new batch to have the attention past in cache let new_cached_batch = wrap_future(client.generate(new_batch), new_request_ids, &db).await; + // Reset waiting counter + waiting_tokens = 1; // Extend current batch with the new batch if let Some(new_cached_batch) = new_cached_batch { request_ids.extend(new_cached_batch.requests.iter().map(|req| req.id)); diff --git a/router/src/server.rs b/router/src/server.rs index 72b720ef..9f4a75c9 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -90,11 +90,7 @@ async fn generate( // Validate request let (input_length, validated_request) = state .validation - // FIXME: can't we get rid of the cloning here?? - .validate(GenerateRequest { - inputs: req.inputs.clone(), - parameters: req.parameters.clone(), - }) + .validate(req.0) .await .map_err(|err| { tracing::error!("{}", err.to_string()); diff --git a/router/src/validation.rs b/router/src/validation.rs index a105ddf3..0ddfe544 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -155,7 +155,7 @@ type ValidationRequest = ( pub enum ValidationError { #[error("temperature must be strictly positive")] Temperature, - #[error("top_p must be >= 0.0 or < 1.0")] + #[error("top_p must be > 0.0 and <= 1.0")] TopP, #[error("top_k must be strictly positive")] TopK, diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 008288f8..2a7405d3 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -82,7 +82,6 @@ def __init__(self, model_name: str, quantize: bool = False): torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( tokenizer=tokenizer, - num_heads=config.n_head // self.process_group.size(), device=device, ) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index ca8ea575..4e66ae3a 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -251,7 +251,6 @@ def __init__(self, model_name: str, quantize=False): super(CausalLM, self).__init__( tokenizer=tokenizer, - num_heads=self.model.config.num_attention_heads, device=device, ) @@ -358,7 +357,7 @@ def generate_token( # Force past to be of dim [batch_size, num_heads, ...] for easy indexing next_batch_past_key_values = [ [ - t.view(-1, self.num_heads, *t.shape[-2:])[next_batch_keep_indices] + t.view(batch.size, -1, *t.shape[-2:])[next_batch_keep_indices] for t in layer ] for layer in past @@ -381,7 +380,7 @@ def generate_token( next_batch_attention_mask = torch.cat( [ next_batch_attention_mask, - torch.ones((next_batch_size, 1)).to(self.device), + next_batch_attention_mask.new_ones(next_batch_size, 1), ], dim=1, ) diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index abc3c36c..5de75ab4 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -185,7 +185,6 @@ def __init__(self, model_name: str, quantize: bool = False): torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( tokenizer=tokenizer, - num_heads=config.num_attention_heads // self.process_group.size(), device=device, ) diff --git a/server/text_generation/models/model.py b/server/text_generation/models/model.py index 7fb8142c..0331e193 100644 --- a/server/text_generation/models/model.py +++ b/server/text_generation/models/model.py @@ -10,9 +10,8 @@ class Model(ABC): - def __init__(self, tokenizer: Tokenizer, num_heads: int, device: torch.device): + def __init__(self, tokenizer: Tokenizer, device: torch.device): self.tokenizer = tokenizer - self.num_heads = num_heads self.device = device @property diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index cb1291ab..e9c65596 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -87,7 +87,7 @@ def from_pb( inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 ).to(device) # Convert decoder_input_ids to torch tensor of size [batch_size, 1] - decoder_input_ids = torch.tensor(decoder_input_ids).to(device).unsqueeze(-1) + decoder_input_ids = torch.tensor(decoder_input_ids, device=device).unsqueeze(-1) return cls( batch_id=pb.id, @@ -319,7 +319,6 @@ def __init__(self, model_name: str, quantize=False): super(Seq2SeqLM, self).__init__( tokenizer=tokenizer, - num_heads=self.model.config.num_attention_heads, device=device, ) @@ -499,7 +498,7 @@ def generate_token( next_batch_decoder_attention_mask = torch.cat( [ next_batch_decoder_attention_mask, - torch.ones((next_batch_size, 1)).to(self.device), + next_batch_decoder_attention_mask.new_ones(next_batch_size, 1), ], dim=1, ) From a2985036aa3d1b50e01682841e9ea24096a5bcad Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 8 Dec 2022 18:49:33 +0100 Subject: [PATCH 034/793] feat(server): Add model tests (#6) --- README.md | 7 +- router/src/batcher.rs | 2 +- server/Makefile | 5 +- server/poetry.lock | 99 ++++++- server/pyproject.toml | 1 + server/tests/conftest.py | 36 +++ server/tests/models/test_bloom.py | 279 ++++++++++++++++++ server/tests/models/test_causal_lm.py | 296 +++++++++++++++++++ server/tests/models/test_seq2seq_lm.py | 306 ++++++++++++++++++++ server/tests/test_utils.py | 34 +++ server/text_generation/models/__init__.py | 6 +- server/text_generation/models/bloom.py | 24 +- server/text_generation/models/causal_lm.py | 26 +- server/text_generation/models/galactica.py | 2 +- server/text_generation/models/seq2seq_lm.py | 7 +- server/text_generation/utils.py | 4 +- 16 files changed, 1105 insertions(+), 29 deletions(-) create mode 100644 server/tests/conftest.py create mode 100644 server/tests/models/test_bloom.py create mode 100644 server/tests/models/test_causal_lm.py create mode 100644 server/tests/models/test_seq2seq_lm.py create mode 100644 server/tests/test_utils.py diff --git a/README.md b/README.md index e76e38d0..bb58e281 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,4 @@ curl 127.0.0.1:3000/generate \ ```shell make server-dev make router-dev -``` - -## TODO: - -- [ ] Add tests for the `server/model` logic -- [ ] Backport custom CUDA kernels to Transformers \ No newline at end of file +``` \ No newline at end of file diff --git a/router/src/batcher.rs b/router/src/batcher.rs index a9b892cc..0c85a406 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -70,7 +70,7 @@ impl Batcher { // Notify the background task that we have a new entry in the database that needs // to be batched - self.shared.batching_task.notify_waiters(); + self.shared.batching_task.notify_one(); // Await on the response from the background task // We can safely unwrap as the background task will never drop the sender diff --git a/server/Makefile b/server/Makefile index 10028dad..ac9cb785 100644 --- a/server/Makefile +++ b/server/Makefile @@ -8,8 +8,9 @@ gen-server: install-transformers: # Install specific version of transformers with custom cuda kernels - rm transformers || true - rm transformers-text_generation_inference || true + pip uninstall transformers -y || true + rm -rf transformers || true + rm -rf transformers-text_generation_inference || true curl -L -O https://github.com/OlivierDehaene/transformers/archive/refs/heads/text_generation_inference.zip unzip text_generation_inference.zip rm text_generation_inference.zip diff --git a/server/poetry.lock b/server/poetry.lock index ebd64ea9..c141612e 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -22,6 +22,20 @@ test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] test_trackers = ["comet-ml", "tensorboard", "wandb"] testing = ["datasets", "deepspeed (<0.7.0)", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scipy", "sklearn", "tqdm", "transformers"] +[[package]] +name = "attrs" +version = "22.1.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] + [[package]] name = "bitsandbytes" version = "0.35.1" @@ -49,6 +63,17 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "exceptiongroup" +version = "1.0.4" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "grpcio" version = "1.50.0" @@ -88,6 +113,14 @@ grpcio = ">=1.50.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "numpy" version = "1.23.4" @@ -107,6 +140,18 @@ python-versions = ">=3.6" [package.dependencies] pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "protobuf" version = "4.21.8" @@ -137,6 +182,26 @@ python-versions = ">=3.6.8" [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pytest" +version = "7.2.0" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + [[package]] name = "PyYAML" version = "6.0" @@ -178,6 +243,14 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + [[package]] name = "torch" version = "1.12.1" @@ -220,13 +293,17 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "3266187ef14fe8f9e29b3b6530d07781ea952aa670c0fe0de34be43efa231a67" +content-hash = "51693654531e3229ac64bee250932ace20a60e8d45af074ae7b860ed32b25ef8" [metadata.files] accelerate = [ {file = "accelerate-0.12.0-py3-none-any.whl", hash = "sha256:7742ca5c9f15dd1e0a283305599c196e260af4717a561d1f544aeab27d828af6"}, {file = "accelerate-0.12.0.tar.gz", hash = "sha256:e8b119c94fac31877620d5f9de311164ec81fa9dc9e175f0d0d4f50fc8d79473"}, ] +attrs = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] bitsandbytes = [ {file = "bitsandbytes-0.35.1-py3-none-any.whl", hash = "sha256:4506a9e3778359a743938aa5592d8d043fa91d1df66cd01ba8cc6486e64dea45"}, {file = "bitsandbytes-0.35.1.tar.gz", hash = "sha256:63a6f59c87b713a731a685e43d68c19789ee6381e62196cafab293b87eca5d46"}, @@ -239,6 +316,10 @@ colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] +exceptiongroup = [ + {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, + {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, +] grpcio = [ {file = "grpcio-1.50.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:906f4d1beb83b3496be91684c47a5d870ee628715227d5d7c54b04a8de802974"}, {file = "grpcio-1.50.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:2d9fd6e38b16c4d286a01e1776fdf6c7a4123d99ae8d6b3f0b4a03a34bf6ce45"}, @@ -337,6 +418,10 @@ grpcio-tools = [ {file = "grpcio_tools-1.50.0-cp39-cp39-win32.whl", hash = "sha256:e1a8f9a57bbcc2e633aaf327e39830527f3c1f7add18c7580f3058fe9a0fa780"}, {file = "grpcio_tools-1.50.0-cp39-cp39-win_amd64.whl", hash = "sha256:b7eb7a84d9171c0ae1550833f4a6ca52372bed9db0fa10f8c9dbe6ca65f97a8c"}, ] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] numpy = [ {file = "numpy-1.23.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d79ada05005f6f4f337d3bb9de8a7774f259341c70bc88047a1f7b96a4bcb2"}, {file = "numpy-1.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926db372bc4ac1edf81cfb6c59e2a881606b409ddc0d0920b988174b2e2a767f"}, @@ -371,6 +456,10 @@ packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] protobuf = [ {file = "protobuf-4.21.8-cp310-abi3-win32.whl", hash = "sha256:c252c55ee15175aa1b21b7b9896e6add5162d066d5202e75c39f96136f08cce3"}, {file = "protobuf-4.21.8-cp310-abi3-win_amd64.whl", hash = "sha256:809ca0b225d3df42655a12f311dd0f4148a943c51f1ad63c38343e457492b689"}, @@ -429,6 +518,10 @@ pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] +pytest = [ + {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, + {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, +] PyYAML = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, @@ -512,6 +605,10 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] torch = [ {file = "torch-1.12.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:9c038662db894a23e49e385df13d47b2a777ffd56d9bcd5b832593fab0a7e286"}, {file = "torch-1.12.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4e1b9c14cf13fd2ab8d769529050629a0e68a6fc5cb8e84b4a3cc1dd8c4fe541"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index cdf89869..f2628854 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -22,6 +22,7 @@ bnb = ["bitsandbytes"] [tool.poetry.group.dev.dependencies] grpcio-tools = "^1.49.1" +pytest = "^7.2.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/server/tests/conftest.py b/server/tests/conftest.py new file mode 100644 index 00000000..0640d45d --- /dev/null +++ b/server/tests/conftest.py @@ -0,0 +1,36 @@ +import pytest + +from transformers import AutoTokenizer + +from text_generation.pb import generate_pb2 + + +@pytest.fixture +def default_pb_parameters(): + return generate_pb2.LogitsWarperParameters( + temperature=1.0, + top_k=0, + top_p=1.0, + do_sample=False, + ) + + +@pytest.fixture(scope="session") +def bloom_560m_tokenizer(): + return AutoTokenizer.from_pretrained("bigscience/bloom-560m", padding_side="left") + + +@pytest.fixture(scope="session") +def gpt2_tokenizer(): + tokenizer = AutoTokenizer.from_pretrained("gpt2", padding_side="left") + tokenizer.pad_token_id = 50256 + return tokenizer + + +@pytest.fixture(scope="session") +def mt0_small_tokenizer(): + tokenizer = AutoTokenizer.from_pretrained( + "bigscience/mt0-small", padding_side="left" + ) + tokenizer.bos_token_id = 0 + return tokenizer diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py new file mode 100644 index 00000000..49dabb14 --- /dev/null +++ b/server/tests/models/test_bloom.py @@ -0,0 +1,279 @@ +import pytest +import torch + +from copy import copy + +from text_generation.pb import generate_pb2 +from text_generation.models.causal_lm import CausalLMBatch +from text_generation.models.bloom import BloomCausalLMBatch, BLOOM + + +@pytest.fixture +def default_pb_request(default_pb_parameters): + return generate_pb2.Request( + id=0, + inputs="Test", + input_length=1, + parameters=default_pb_parameters, + max_new_tokens=10, + ) + + +@pytest.fixture +def default_pb_batch(default_pb_request): + return generate_pb2.Batch(id=0, requests=[default_pb_request], size=1) + + +@pytest.fixture +def default_bloom_batch(default_pb_batch, bloom_560m_tokenizer): + return BloomCausalLMBatch.from_pb( + default_pb_batch, bloom_560m_tokenizer, torch.device("cpu") + ) + + +@pytest.fixture +def default_multi_requests_bloom_batch(default_pb_request, bloom_560m_tokenizer): + req_0 = copy(default_pb_request) + req_1 = default_pb_request + req_1.id = 1 + req_1.max_new_tokens = 5 + + batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) + return BloomCausalLMBatch.from_pb( + batch_pb, bloom_560m_tokenizer, torch.device("cpu") + ) + + +@pytest.fixture(scope="session") +def default_bloom(): + return BLOOM("bigscience/bloom-560m") + + +def test_batch_from_pb(default_pb_batch, default_bloom_batch): + batch = default_bloom_batch + + assert batch.batch_id == default_pb_batch.id + assert batch.requests == default_pb_batch.requests + + assert len(batch.input_ids) == default_pb_batch.size + assert len(batch.input_ids[0]) == 8 + assert batch.input_ids[0][-1] == 10264 + assert torch.all(batch.input_ids[0][:-1] == 3) + + assert batch.attention_mask[0][-1] == 1 + assert torch.all(batch.attention_mask[0][:-1] == 0) + + assert batch.past_key_values is None + + assert torch.equal(batch.input_ids, batch.all_input_ids[:, :, 0]) + + assert batch.input_lengths == [1] + + assert batch.size == default_pb_batch.size + assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == batch.size + + assert batch.max_sequence_length == batch.input_lengths[0] + + +def test_batch_concatenate_no_prefill(default_bloom_batch): + with pytest.raises(ValueError): + BloomCausalLMBatch.concatenate([default_bloom_batch, default_bloom_batch]) + + +def test_causal_lm_batch_type(default_bloom): + assert default_bloom.batch_type == BloomCausalLMBatch + + +def test_causal_lm_generate_token(default_bloom, default_bloom_batch): + generated_texts, next_batch = default_bloom.generate_token(default_bloom_batch) + + assert generated_texts == [] + assert isinstance(next_batch, CausalLMBatch) + assert not next_batch.keys_head_dim_last + + assert len(next_batch.all_input_ids) == next_batch.size + assert len(next_batch.all_input_ids[0]) == len(next_batch.attention_mask[0]) == 9 + assert torch.all(next_batch.all_input_ids[0][-2:] == 10264) + assert torch.all(next_batch.all_input_ids[0][:-2] == 3) + + assert torch.all(next_batch.attention_mask[0][-2:] == 1) + assert torch.all(next_batch.attention_mask[0][:-2] == 0) + + assert next_batch.input_ids.shape == (next_batch.size, 1) + assert next_batch.input_ids[0, 0] == 10264 + + assert next_batch.input_lengths == [2] + assert next_batch.max_sequence_length == next_batch.input_lengths[0] + + assert next_batch.past_key_values is not None + assert all([p[0].shape == (16, 64, 8) for p in next_batch.past_key_values]) + assert all([p[1].shape == (16, 8, 64) for p in next_batch.past_key_values]) + + +def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch): + next_batch = default_bloom_batch + for _ in range(default_bloom_batch.stopping_criterias[0].max_new_tokens - 1): + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "TestTestTestTestTestTestTestTestTestTestTest" + assert generated_texts[0].request == default_bloom_batch.requests[0] + assert ( + generated_texts[0].tokens + == default_bloom_batch.stopping_criterias[0].max_new_tokens + ) + + +def test_causal_lm_generate_token_completion_multi( + default_bloom, default_multi_requests_bloom_batch +): + next_batch = default_multi_requests_bloom_batch + + for i in range( + default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 1 + ): + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert next_batch is not None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "TestTestTestTestTestTest" + assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] + assert ( + generated_texts[0].tokens + == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens + ) + + for _ in range( + default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens + - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens + - 1 + ): + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "TestTestTestTestTestTestTestTestTestTestTest" + assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] + assert ( + generated_texts[0].tokens + == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens + ) + + +def test_batch_concatenate( + default_bloom, default_bloom_batch, default_multi_requests_bloom_batch +): + next_batch_0 = default_bloom_batch + _, next_batch_0 = default_bloom.generate_token(next_batch_0) + _, next_batch_0 = default_bloom.generate_token(next_batch_0) + + next_batch_1 = default_multi_requests_bloom_batch + _, next_batch_1 = default_bloom.generate_token(next_batch_1) + + next_batch = BloomCausalLMBatch.concatenate([next_batch_0, next_batch_1]) + + assert torch.equal(next_batch.all_input_ids[0], next_batch_0.all_input_ids[0]) + assert torch.equal(next_batch.all_input_ids[1], next_batch_1.all_input_ids[0]) + assert torch.equal(next_batch.all_input_ids[2], next_batch_1.all_input_ids[1]) + + assert torch.all(next_batch.attention_mask[0] == 1) + assert torch.all(next_batch.attention_mask[1:, -2:] == 1) + assert torch.all(next_batch.attention_mask[1:, :-2] == 0) + + assert next_batch.batch_id == 0 + assert torch.all(next_batch.input_ids == 10264) + + assert next_batch.input_lengths == [3, 2, 2] + assert next_batch.max_sequence_length == 3 + + assert next_batch.requests[0] == next_batch_0.requests[0] + assert next_batch.requests[1:] == next_batch_1.requests + + assert next_batch.next_token_choosers[0] == next_batch_0.next_token_choosers[0] + assert next_batch.next_token_choosers[1:] == next_batch_1.next_token_choosers + + assert next_batch.stopping_criterias[0] == next_batch_0.stopping_criterias[0] + assert next_batch.stopping_criterias[1:] == next_batch_1.stopping_criterias + + assert next_batch.past_key_values is not None + assert all([p[0].shape == (3, 16, 64, 2) for p in next_batch.past_key_values]) + assert all([p[1].shape == (3, 16, 2, 64) for p in next_batch.past_key_values]) + + for i, past in enumerate(next_batch.past_key_values): + assert torch.equal(next_batch_0.past_key_values[i][0][:, :, -2:], past[0][0]) + assert torch.equal( + next_batch_1.past_key_values[i][0][:, :, -1:], + past[0][1:, :, :, -1].reshape(-1, 64, 1), + ) + + assert torch.equal(next_batch_0.past_key_values[i][1][:, -2:, :], past[1][0]) + assert torch.equal( + next_batch_1.past_key_values[i][1][:, -1:, :], + past[1][1:, :, -1, :].reshape(-1, 1, 64), + ) + + for _ in range( + default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 2 + ): + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert next_batch is not None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "TestTestTestTestTestTest" + assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] + assert ( + generated_texts[0].tokens + == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens + ) + + for _ in range( + default_bloom_batch.stopping_criterias[0].max_new_tokens + - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens + - 2 + ): + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert next_batch is not None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "TestTestTestTestTestTestTestTestTestTestTest" + assert generated_texts[0].request == default_bloom_batch.requests[0] + assert ( + generated_texts[0].tokens + == default_bloom_batch.stopping_criterias[0].max_new_tokens + ) + + for _ in range( + default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens + - default_bloom_batch.stopping_criterias[0].max_new_tokens + - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens + - 4 + ): + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "TestTestTestTestTestTestTestTestTestTestTest" + assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] + assert ( + generated_texts[0].tokens + == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens + ) diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py new file mode 100644 index 00000000..1bf3e5e6 --- /dev/null +++ b/server/tests/models/test_causal_lm.py @@ -0,0 +1,296 @@ +import pytest +import torch + +from copy import copy + +from text_generation.pb import generate_pb2 +from text_generation.models.causal_lm import CausalLM, CausalLMBatch + + +@pytest.fixture +def default_pb_request(default_pb_parameters): + return generate_pb2.Request( + id=0, + inputs="Test", + input_length=1, + parameters=default_pb_parameters, + max_new_tokens=10, + ) + + +@pytest.fixture +def default_pb_batch(default_pb_request): + return generate_pb2.Batch(id=0, requests=[default_pb_request], size=1) + + +@pytest.fixture +def default_causal_lm_batch(default_pb_batch, gpt2_tokenizer): + return CausalLMBatch.from_pb(default_pb_batch, gpt2_tokenizer, torch.device("cpu")) + + +@pytest.fixture +def default_multi_requests_causal_lm_batch(default_pb_request, gpt2_tokenizer): + req_0 = copy(default_pb_request) + req_1 = default_pb_request + req_1.id = 1 + req_1.max_new_tokens = 5 + + batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) + return CausalLMBatch.from_pb(batch_pb, gpt2_tokenizer, torch.device("cpu")) + + +@pytest.fixture(scope="session") +def default_causal_lm(): + return CausalLM("gpt2") + + +def test_batch_from_pb(default_pb_batch, default_causal_lm_batch): + batch = default_causal_lm_batch + + assert batch.batch_id == default_pb_batch.id + assert batch.requests == default_pb_batch.requests + + assert len(batch.input_ids) == default_pb_batch.size + assert len(batch.input_ids[0]) == 8 + assert batch.input_ids[0][-1] == 14402 + assert torch.all(batch.input_ids[0][:-1] == 50256) + + assert batch.attention_mask[0][-1] == 1 + assert torch.all(batch.attention_mask[0][:-1] == 0) + + assert batch.past_key_values is None + + assert torch.equal(batch.input_ids, batch.all_input_ids[:, :, 0]) + + assert batch.input_lengths == [1] + + assert batch.size == default_pb_batch.size + assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == batch.size + + assert batch.max_sequence_length == batch.input_lengths[0] + + +def test_batch_concatenate_no_prefill(default_causal_lm_batch): + with pytest.raises(ValueError): + CausalLMBatch.concatenate([default_causal_lm_batch, default_causal_lm_batch]) + + +def test_causal_lm_batch_type(default_causal_lm): + assert default_causal_lm.batch_type == CausalLMBatch + + +def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): + generated_texts, next_batch = default_causal_lm.generate_token( + default_causal_lm_batch + ) + + assert generated_texts == [] + assert isinstance(next_batch, CausalLMBatch) + + assert len(next_batch.all_input_ids) == next_batch.size + assert len(next_batch.all_input_ids[0]) == len(next_batch.attention_mask[0]) == 9 + assert next_batch.all_input_ids[0][-1] == 6208 + assert next_batch.all_input_ids[0][-2] == 14402 + assert torch.all(next_batch.all_input_ids[0][:-2] == 50256) + + assert torch.all(next_batch.attention_mask[0][-2:] == 1) + assert torch.all(next_batch.attention_mask[0][:-2] == 0) + + assert next_batch.input_ids.shape == (next_batch.size, 1) + assert next_batch.input_ids[0, 0] == 6208 + + assert next_batch.input_lengths == [2] + assert next_batch.max_sequence_length == next_batch.input_lengths[0] + + assert next_batch.past_key_values is not None + assert all([p[0].shape == (1, 12, 8, 64) for p in next_batch.past_key_values]) + assert all([p[1].shape == (1, 12, 8, 64) for p in next_batch.past_key_values]) + + +def test_causal_lm_generate_token_completion( + default_causal_lm, default_causal_lm_batch +): + next_batch = default_causal_lm_batch + for _ in range(default_causal_lm_batch.stopping_criterias[0].max_new_tokens - 1): + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert ( + generated_texts[0].output + == "Test Test Test Test Test Test Test Test Test Test Test" + ) + assert generated_texts[0].request == default_causal_lm_batch.requests[0] + assert ( + generated_texts[0].tokens + == default_causal_lm_batch.stopping_criterias[0].max_new_tokens + ) + + +def test_causal_lm_generate_token_completion_multi( + default_causal_lm, default_multi_requests_causal_lm_batch +): + next_batch = default_multi_requests_causal_lm_batch + + for i in range( + default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 1 + ): + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert next_batch is not None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "Test Test Test Test Test Test" + assert ( + generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] + ) + assert ( + generated_texts[0].tokens + == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens + ) + + for _ in range( + default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens + - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens + - 1 + ): + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert ( + generated_texts[0].output + == "Test Test Test Test Test Test Test Test Test Test Test" + ) + assert ( + generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] + ) + assert ( + generated_texts[0].tokens + == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens + ) + + +def test_batch_concatenate( + default_causal_lm, default_causal_lm_batch, default_multi_requests_causal_lm_batch +): + next_batch_0 = default_causal_lm_batch + _, next_batch_0 = default_causal_lm.generate_token(next_batch_0) + _, next_batch_0 = default_causal_lm.generate_token(next_batch_0) + + next_batch_1 = default_multi_requests_causal_lm_batch + _, next_batch_1 = default_causal_lm.generate_token(next_batch_1) + + next_batch = CausalLMBatch.concatenate([next_batch_0, next_batch_1]) + + assert torch.equal(next_batch.all_input_ids[0], next_batch_0.all_input_ids[0]) + assert torch.equal(next_batch.all_input_ids[1], next_batch_1.all_input_ids[0]) + assert torch.equal(next_batch.all_input_ids[2], next_batch_1.all_input_ids[1]) + + assert torch.all(next_batch.attention_mask[0] == 1) + assert torch.all(next_batch.attention_mask[1:, -2:] == 1) + assert torch.all(next_batch.attention_mask[1:, :-2] == 0) + + assert next_batch.batch_id == 0 + assert torch.all(next_batch.input_ids == 6208) + + assert next_batch.input_lengths == [3, 2, 2] + assert next_batch.max_sequence_length == 3 + + assert next_batch.requests[0] == next_batch_0.requests[0] + assert next_batch.requests[1:] == next_batch_1.requests + + assert next_batch.next_token_choosers[0] == next_batch_0.next_token_choosers[0] + assert next_batch.next_token_choosers[1:] == next_batch_1.next_token_choosers + + assert next_batch.stopping_criterias[0] == next_batch_0.stopping_criterias[0] + assert next_batch.stopping_criterias[1:] == next_batch_1.stopping_criterias + + assert next_batch.past_key_values is not None + assert all([p[0].shape == (3, 12, 2, 64) for p in next_batch.past_key_values]) + assert all([p[1].shape == (3, 12, 2, 64) for p in next_batch.past_key_values]) + + for i, past in enumerate(next_batch.past_key_values): + assert torch.equal(next_batch_0.past_key_values[i][0][0, :, -2:], past[0][0]) + assert torch.equal( + next_batch_1.past_key_values[i][0][:, :, -1:], past[0][1:, :, -1:, :] + ) + + assert torch.equal(next_batch_0.past_key_values[i][1][0, :, -2:], past[1][0]) + assert torch.equal( + next_batch_1.past_key_values[i][1][:, :, -1:], past[1][1:, :, -1:, :] + ) + + for _ in range( + default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2 + ): + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert next_batch is not None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "Test Test Test Test Test Test" + assert ( + generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] + ) + assert ( + generated_texts[0].tokens + == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens + ) + + for _ in range( + default_causal_lm_batch.stopping_criterias[0].max_new_tokens + - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens + - 2 + ): + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert next_batch is not None + + assert len(generated_texts) == 1 + assert ( + generated_texts[0].output + == "Test Test Test Test Test Test Test Test Test Test Test" + ) + assert generated_texts[0].request == default_causal_lm_batch.requests[0] + assert ( + generated_texts[0].tokens + == default_causal_lm_batch.stopping_criterias[0].max_new_tokens + ) + + for _ in range( + default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens + - default_causal_lm_batch.stopping_criterias[0].max_new_tokens + - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens + - 4 + ): + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert ( + generated_texts[0].output + == "Test Test Test Test Test Test Test Test Test Test Test" + ) + assert ( + generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] + ) + assert ( + generated_texts[0].tokens + == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens + ) diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py new file mode 100644 index 00000000..7e4c7fdd --- /dev/null +++ b/server/tests/models/test_seq2seq_lm.py @@ -0,0 +1,306 @@ +import pytest +import torch + +from copy import copy + +from text_generation.pb import generate_pb2 +from text_generation.models.seq2seq_lm import Seq2SeqLM, Seq2SeqLMBatch + + +@pytest.fixture +def default_pb_request(default_pb_parameters): + return generate_pb2.Request( + id=0, + inputs="Test", + input_length=2, + parameters=default_pb_parameters, + max_new_tokens=10, + ) + + +@pytest.fixture +def default_pb_batch(default_pb_request): + return generate_pb2.Batch(id=0, requests=[default_pb_request], size=1) + + +@pytest.fixture +def default_seq2seq_lm_batch(default_pb_batch, mt0_small_tokenizer): + return Seq2SeqLMBatch.from_pb( + default_pb_batch, mt0_small_tokenizer, torch.device("cpu") + ) + + +@pytest.fixture +def default_multi_requests_seq2seq_lm_batch(default_pb_request, mt0_small_tokenizer): + req_0 = copy(default_pb_request) + req_1 = default_pb_request + req_1.id = 1 + req_1.max_new_tokens = 5 + + batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) + return Seq2SeqLMBatch.from_pb(batch_pb, mt0_small_tokenizer, torch.device("cpu")) + + +@pytest.fixture(scope="session") +def default_seq2seq_lm(): + return Seq2SeqLM("bigscience/mt0-small") + + +def test_batch_from_pb(default_pb_batch, default_seq2seq_lm_batch): + batch = default_seq2seq_lm_batch + + assert batch.batch_id == default_pb_batch.id + assert batch.requests == default_pb_batch.requests + + assert batch.input_ids.shape == (default_pb_batch.size, 8) + assert batch.input_ids[0][-2] == 4268 + assert batch.input_ids[0][-1] == 1 + assert torch.all(batch.input_ids[0][:-2] == 0) + + assert torch.all(batch.attention_mask[0][-2:] == 1) + assert torch.all(batch.attention_mask[0][:-2] == 0) + + assert batch.decoder_input_ids.shape == (default_pb_batch.size, 1) + assert batch.decoder_attention_mask is None + assert batch.encoder_last_hidden_state is None + + assert batch.past_key_values is None + + assert batch.input_lengths == [2] + assert batch.decoder_input_lengths == [1] + + assert batch.size == default_pb_batch.size + assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == batch.size + + assert batch.max_input_length == batch.input_lengths[0] + assert batch.max_decoder_input_length == batch.decoder_input_lengths[0] + + +def test_batch_concatenate_no_prefill(default_seq2seq_lm_batch): + with pytest.raises(ValueError): + Seq2SeqLMBatch.concatenate([default_seq2seq_lm_batch, default_seq2seq_lm_batch]) + + +def test_seq2seq_lm_batch_type(default_seq2seq_lm): + assert default_seq2seq_lm.batch_type == Seq2SeqLMBatch + + +def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch): + generated_texts, next_batch = default_seq2seq_lm.generate_token( + default_seq2seq_lm_batch + ) + + assert generated_texts == [] + assert isinstance(next_batch, Seq2SeqLMBatch) + + assert torch.equal(next_batch.input_ids, default_seq2seq_lm_batch.input_ids) + assert torch.equal( + next_batch.attention_mask, default_seq2seq_lm_batch.attention_mask + ) + assert next_batch.input_lengths == default_seq2seq_lm_batch.input_lengths + assert next_batch.max_input_length == default_seq2seq_lm_batch.max_input_length + assert ( + next_batch.next_token_choosers == default_seq2seq_lm_batch.next_token_choosers + ) + assert next_batch.stopping_criterias == default_seq2seq_lm_batch.stopping_criterias + + assert next_batch.decoder_input_ids.shape == (next_batch.size, 2) + assert next_batch.decoder_input_ids[0, 0] == 0 + assert next_batch.decoder_input_ids[0, 1] == 259 + assert next_batch.decoder_attention_mask is None + assert next_batch.encoder_last_hidden_state.shape == (1, 8, 512) + + assert next_batch.decoder_input_lengths == [2] + assert next_batch.max_decoder_input_length == 2 + + assert next_batch.past_key_values is not None + assert all( + [p[0].shape == (next_batch.size, 6, 1, 64) for p in next_batch.past_key_values] + ) + assert all( + [p[1].shape == (next_batch.size, 6, 1, 64) for p in next_batch.past_key_values] + ) + assert all( + [p[2].shape == (next_batch.size, 6, 8, 64) for p in next_batch.past_key_values] + ) + assert all( + [p[3].shape == (next_batch.size, 6, 8, 64) for p in next_batch.past_key_values] + ) + + +def test_seq2seq_lm_generate_token_completion( + default_seq2seq_lm, default_seq2seq_lm_batch +): + next_batch = default_seq2seq_lm_batch + for _ in range(6): + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "a few weeks" + assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] + assert generated_texts[0].tokens == 7 + + +def test_seq2seq_lm_generate_token_completion_multi( + default_seq2seq_lm, default_multi_requests_seq2seq_lm_batch +): + next_batch = default_multi_requests_seq2seq_lm_batch + + for i in range(4): + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert next_batch is not None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "a few " + assert ( + generated_texts[0].request + == default_multi_requests_seq2seq_lm_batch.requests[1] + ) + assert generated_texts[0].tokens == 5 + + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "a few weeks" + assert ( + generated_texts[0].request + == default_multi_requests_seq2seq_lm_batch.requests[0] + ) + assert generated_texts[0].tokens == 7 + + +def test_batch_concatenate( + default_seq2seq_lm, + default_seq2seq_lm_batch, + default_multi_requests_seq2seq_lm_batch, +): + next_batch_0 = default_seq2seq_lm_batch + _, next_batch_0 = default_seq2seq_lm.generate_token(next_batch_0) + _, next_batch_0 = default_seq2seq_lm.generate_token(next_batch_0) + + next_batch_1 = default_multi_requests_seq2seq_lm_batch + _, next_batch_1 = default_seq2seq_lm.generate_token(next_batch_1) + + next_batch = Seq2SeqLMBatch.concatenate([next_batch_0, next_batch_1]) + + assert next_batch.batch_id == 0 + + assert torch.all(next_batch.input_ids[:, 0] == 4268) + assert torch.all(next_batch.input_ids[:, 1] == 1) + + assert torch.all(next_batch.attention_mask == 1) + + assert torch.equal( + next_batch.decoder_input_ids[0], next_batch_0.decoder_input_ids[0] + ) + assert torch.all(next_batch.decoder_input_ids[1:, 0] == 0) + assert torch.equal( + next_batch.decoder_input_ids[1:, -2:], next_batch_1.decoder_input_ids + ) + + assert torch.all(next_batch.decoder_attention_mask[0] == 1) + assert torch.all(next_batch.decoder_attention_mask[1:, 0] == 0) + assert torch.all(next_batch.decoder_attention_mask[1:, -2:] == 1) + + assert torch.equal( + next_batch.encoder_last_hidden_state[0], + next_batch_0.encoder_last_hidden_state[0, -2:], + ) + assert torch.equal( + next_batch.encoder_last_hidden_state[1:], + next_batch_1.encoder_last_hidden_state[:, -2:], + ) + + assert next_batch.input_lengths == [2, 2, 2] + assert next_batch.decoder_input_lengths == [3, 2, 2] + assert next_batch.max_input_length == 2 + assert next_batch.max_decoder_input_length == 3 + + assert next_batch.requests[0] == next_batch_0.requests[0] + assert next_batch.requests[1:] == next_batch_1.requests + + assert next_batch.next_token_choosers[0] == next_batch_0.next_token_choosers[0] + assert next_batch.next_token_choosers[1:] == next_batch_1.next_token_choosers + + assert next_batch.stopping_criterias[0] == next_batch_0.stopping_criterias[0] + assert next_batch.stopping_criterias[1:] == next_batch_1.stopping_criterias + + assert next_batch.past_key_values is not None + assert all( + [p[0].shape == (next_batch.size, 6, 2, 64) for p in next_batch.past_key_values] + ) + assert all( + [p[1].shape == (next_batch.size, 6, 2, 64) for p in next_batch.past_key_values] + ) + assert all( + [p[2].shape == (next_batch.size, 6, 2, 64) for p in next_batch.past_key_values] + ) + assert all( + [p[3].shape == (next_batch.size, 6, 2, 64) for p in next_batch.past_key_values] + ) + + for i, past in enumerate(next_batch.past_key_values): + assert torch.equal(next_batch_0.past_key_values[i][0][0, :, -2:, :], past[0][0]) + assert torch.equal( + next_batch_1.past_key_values[i][0][:, :, -1:, :], past[0][1:, :, -1:, :] + ) + + assert torch.equal(next_batch_0.past_key_values[i][1][0, :, -2:, :], past[1][0]) + assert torch.equal( + next_batch_1.past_key_values[i][1][:, :, -1:, :], past[1][1:, :, -1:, :] + ) + + assert torch.equal(next_batch_0.past_key_values[i][2][0, :, -2:, :], past[2][0]) + assert torch.equal( + next_batch_1.past_key_values[i][2][:, :, -2:, :], past[2][1:] + ) + + assert torch.equal(next_batch_0.past_key_values[i][3][0, :, -2:, :], past[3][0]) + assert torch.equal( + next_batch_1.past_key_values[i][3][:, :, -2:, :], past[3][1:] + ) + + for _ in range(3): + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert next_batch is not None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "a few " + assert ( + generated_texts[0].request + == default_multi_requests_seq2seq_lm_batch.requests[1] + ) + assert generated_texts[0].tokens == 5 + + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert next_batch is not None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "a few weeks" + assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] + assert generated_texts[0].tokens == 7 + + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert generated_texts[0].output == "a few weeks" + assert ( + generated_texts[0].request + == default_multi_requests_seq2seq_lm_batch.requests[0] + ) + assert generated_texts[0].tokens == 7 diff --git a/server/tests/test_utils.py b/server/tests/test_utils.py new file mode 100644 index 00000000..e630ebda --- /dev/null +++ b/server/tests/test_utils.py @@ -0,0 +1,34 @@ +import pytest + +from text_generation.utils import ( + weight_hub_files, + download_weights, + weight_files, + LocalEntryNotFoundError, +) + + +def test_weight_hub_files(): + filenames = weight_hub_files("bigscience/bloom-560m") + assert filenames == ["model.safetensors"] + + +def test_weight_hub_files_llm(): + filenames = weight_hub_files("bigscience/bloom") + assert filenames == [f"model_{i:05d}-of-00072.safetensors" for i in range(1, 73)] + + +def test_weight_hub_files_empty(): + filenames = weight_hub_files("bigscience/bloom", ".errors") + assert filenames == [] + + +def test_download_weights(): + files = download_weights("bigscience/bloom-560m") + local_files = weight_files("bigscience/bloom-560m") + assert files == local_files + + +def test_weight_files_error(): + with pytest.raises(LocalEntryNotFoundError): + weight_files("bert-base-uncased") diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index b364309a..935d74ef 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -1,10 +1,10 @@ from text_generation.models.model import Model from text_generation.models.causal_lm import CausalLM -from text_generation.models.bloom import BLOOMSharded +from text_generation.models.bloom import BLOOM, BLOOMSharded from text_generation.models.seq2seq_lm import Seq2SeqLM from text_generation.models.galactica import Galactica, GalacticaSharded -__all__ = ["Model", "BLOOMSharded", "CausalLM", "Seq2SeqLM"] +__all__ = ["Model", "BLOOM", "BLOOMSharded", "CausalLM", "Seq2SeqLM"] def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: @@ -12,7 +12,7 @@ def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: if sharded: return BLOOMSharded(model_name, quantize=quantize) else: - return CausalLM(model_name, quantize=quantize) + return BLOOM(model_name, quantize=quantize) elif model_name.startswith("facebook/galactica"): if sharded: return GalacticaSharded(model_name, quantize=quantize) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 2a7405d3..20e26419 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -1,7 +1,7 @@ import torch import torch.distributed -from typing import List, Optional +from typing import List, Optional, Type from accelerate import init_empty_weights from safetensors import safe_open @@ -13,6 +13,8 @@ ) from text_generation.models import CausalLM +from text_generation.models.causal_lm import CausalLMBatch +from text_generation.pb import generate_pb2 from text_generation.utils import ( initialize_torch_distributed, weight_files, @@ -29,7 +31,25 @@ torch.manual_seed(0) -class BLOOMSharded(CausalLM): +class BloomCausalLMBatch(CausalLMBatch): + @classmethod + def from_pb( + cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + ) -> "CausalLMBatch": + batch = super(BloomCausalLMBatch, cls).from_pb( + pb=pb, tokenizer=tokenizer, device=device + ) + batch.keys_head_dim_last = False + return batch + + +class BLOOM(CausalLM): + @property + def batch_type(self) -> Type[CausalLMBatch]: + return BloomCausalLMBatch + + +class BLOOMSharded(BLOOM): def __init__(self, model_name: str, quantize: bool = False): if not model_name.startswith("bigscience/bloom"): raise ValueError(f"Model {model_name} is not supported") diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 4e66ae3a..2a88c781 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -34,6 +34,9 @@ class CausalLMBatch: size: int max_sequence_length: int + # Past metadata + keys_head_dim_last: bool = True + def to_pb(self): return generate_pb2.Batch( id=self.batch_id, @@ -165,20 +168,16 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": head_dim, ) + if batch.keys_head_dim_last: + padded_past_keys_shape = padded_past_values_shape # seq_length is last for BLOOM - if past_keys.shape[-2] == head_dim: - past_keys_head_dim_last = False + else: padded_past_keys_shape = ( total_batch_size, num_heads, head_dim, max_sequence_length - 1, ) - elif past_keys.shape[-1] == head_dim: - past_keys_head_dim_last = True - padded_past_keys_shape = padded_past_values_shape - else: - raise ValueError(f"past_keys shape {past_keys.shape} is not valid") # This will run only once per layer if j == len(past_key_values): @@ -195,7 +194,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": past_key_values.append((padded_past_keys, padded_past_values)) # We slice the past keys and values to remove the padding from previous batches - if past_keys_head_dim_last: + if batch.keys_head_dim_last: past_key_values[j][0][ start_index:end_index, :, @@ -228,6 +227,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": stopping_criterias=stopping_criterias, size=total_batch_size, max_sequence_length=max_sequence_length, + keys_head_dim_last=batches[0].keys_head_dim_last, ) @@ -237,6 +237,9 @@ def __init__(self, model_name: str, quantize=False): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: + if quantize: + raise ValueError("quantization is not available on CPU") + device = torch.device("cpu") dtype = torch.float32 @@ -247,7 +250,11 @@ def __init__(self, model_name: str, quantize=False): device_map="auto" if torch.cuda.is_available() else None, load_in_8bit=quantize, ).eval() - tokenizer.pad_token_id = self.model.config.pad_token_id + tokenizer.pad_token_id = ( + self.model.config.pad_token_id + if self.model.config.pad_token_id is not None + else self.model.config.eos_token_id + ) super(CausalLM, self).__init__( tokenizer=tokenizer, @@ -397,5 +404,6 @@ def generate_token( stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, max_sequence_length=next_batch_max_sequence_length, + keys_head_dim_last=batch.keys_head_dim_last, ) return generated_texts, next_batch diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index 5de75ab4..8aec1bc7 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -83,7 +83,7 @@ class GalacticaCausalLMBatch(CausalLMBatch): @classmethod def from_pb( cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device - ) -> "CausalLMBatch": + ) -> "GalacticaCausalLMBatch": inputs = [] next_token_choosers = [] stopping_criterias = [] diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index e9c65596..3302138f 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -221,8 +221,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Copy to correct indices encoder_last_hidden_state[ - start_index:end_index, -batch.max_decoder_input_length :, : - ] = batch.encoder_last_hidden_state[:, -batch.max_decoder_input_length :, :] + start_index:end_index, -batch.max_input_length :, : + ] = batch.encoder_last_hidden_state[:, -batch.max_input_length :, :] # Iterate over attention layers for j, past in enumerate(batch.past_key_values): @@ -305,6 +305,9 @@ def __init__(self, model_name: str, quantize=False): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: + if quantize: + raise ValueError("quantization is not available on CPU") + device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index e6bfc391..e55eeb64 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -137,8 +137,8 @@ def download_weights(model_name, extension=".safetensors"): executor.submit(download_function, filename=filename) for filename in filenames ] files = [ - file - for file in tqdm(concurrent.futures.as_completed(futures), total=len(futures)) + future.result() + for future in tqdm(concurrent.futures.as_completed(futures), total=len(futures)) ] return files From 042180d88f91d4bc9acd42ae4de3c0236d272de4 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 8 Dec 2022 19:37:37 +0100 Subject: [PATCH 035/793] fix(server): Only pad to multiple of 8 on GPUs --- server/text_generation/models/causal_lm.py | 3 ++- server/text_generation/models/seq2seq_lm.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 2a88c781..1cd999f0 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -71,8 +71,9 @@ def from_pb( ) ) + pad_to_multiple_of = 8 if "gpu" in str(device) else None tokenized_inputs = tokenizer( - inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 + inputs, return_tensors="pt", padding=True, pad_to_multiple_of=pad_to_multiple_of ).to(device) all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 3302138f..cbfe7ccf 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -83,8 +83,9 @@ def from_pb( ) # Tokenize batch + pad_to_multiple_of = 8 if "gpu" in str(device) else None tokenized_inputs = tokenizer( - inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 + inputs, return_tensors="pt", padding=True, pad_to_multiple_of=pad_to_multiple_of ).to(device) # Convert decoder_input_ids to torch tensor of size [batch_size, 1] decoder_input_ids = torch.tensor(decoder_input_ids, device=device).unsqueeze(-1) From 718096f6952147681559ba0a3962040f8655af1f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 12 Dec 2022 18:25:22 +0100 Subject: [PATCH 036/793] feat: Support stop sequences (#7) --- README.md | 2 + proto/generate.proto | 17 ++++- router/client/src/lib.rs | 4 +- router/src/batcher.rs | 2 + router/src/db.rs | 17 ++++- router/src/lib.rs | 3 + router/src/server.rs | 9 ++- router/src/validation.rs | 10 +++ server/tests/conftest.py | 5 ++ server/tests/models/test_bloom.py | 22 +++++-- server/tests/models/test_causal_lm.py | 53 ++++++++------- server/tests/models/test_seq2seq_lm.py | 22 +++++-- server/tests/test_utils.py | 45 +++++++++++++ server/text_generation/models/causal_lm.py | 29 ++++----- server/text_generation/models/galactica.py | 19 +----- server/text_generation/models/seq2seq_lm.py | 25 ++++---- server/text_generation/models/types.py | 6 +- server/text_generation/utils.py | 71 ++++++++++++++++++--- 18 files changed, 254 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index bb58e281..92879707 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ to power Bloom, BloomZ and MT0-XXL api-inference widgets. - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading - 45ms per token generation for BLOOM with 8xA100 80GB +- Logits warpers (temperature scaling, topk ...) +- Stop sequences ## Officially supported models diff --git a/proto/generate.proto b/proto/generate.proto index 14f6f66b..0c67de03 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -28,12 +28,23 @@ message ClearCacheRequest {} message ClearCacheResponse {} message LogitsWarperParameters { + /// exponential scaling output probability distribution float temperature = 1; + /// restricting to the k highest probability elements uint32 top_k = 2; + /// restricting to top tokens summing to prob_cut_off <= prob_cut_off float top_p = 3; + /// apply sampling on the logits bool do_sample = 4; } +message StoppingCriteriaParameters { + /// Maximum number of generated tokens + uint32 max_new_tokens = 1; + /// Optional stopping sequences + repeated string stop_sequences = 2; +} + message Request { /// Request ID uint64 id = 1; @@ -43,8 +54,8 @@ message Request { uint32 input_length = 3; /// Logits Warper Parameters LogitsWarperParameters parameters = 4; - /// Stopping criteria - uint32 max_new_tokens = 5; + /// Stopping Criteria Parameters + StoppingCriteriaParameters stopping_parameters = 5; } message Batch { @@ -63,6 +74,8 @@ message GeneratedText { string output = 2; /// Number of generated tokens uint32 tokens = 3; + /// Finish reason + string finish_reason = 4; } message GenerateRequest { diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 0f1f96bc..ae337dd6 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -6,7 +6,9 @@ mod pb; mod sharded_client; pub use client::Client; -pub use pb::generate::v1::{Batch, GeneratedText, LogitsWarperParameters, Request}; +pub use pb::generate::v1::{ + Batch, GeneratedText, LogitsWarperParameters, Request, StoppingCriteriaParameters, +}; pub use sharded_client::ShardedClient; use thiserror::Error; use tonic::transport; diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 0c85a406..a72a6e44 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -190,6 +190,7 @@ fn send_generated(finished: Vec, db: &Db) { let response = InferResponse { output: output.output, tokens: output.tokens, + finish_reason: output.finish_reason, queued: entry.time, start: entry.batch_time.unwrap(), // unwrap is always valid end: Instant::now(), @@ -203,6 +204,7 @@ fn send_generated(finished: Vec, db: &Db) { pub(crate) struct InferResponse { pub(crate) output: String, pub(crate) tokens: u32, + pub(crate) finish_reason: String, pub(crate) queued: Instant, pub(crate) start: Instant, pub(crate) end: Instant, diff --git a/router/src/db.rs b/router/src/db.rs index 0701206b..24fb7a09 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -4,7 +4,9 @@ use crate::{GenerateParameters, GenerateRequest}; use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; -use text_generation_client::{Batch, ClientError, LogitsWarperParameters, Request}; +use text_generation_client::{ + Batch, ClientError, LogitsWarperParameters, Request, StoppingCriteriaParameters, +}; use tokio::sync::oneshot::Sender; use tokio::time::Instant; @@ -72,7 +74,9 @@ impl State { parameters: Some(LogitsWarperParameters::from( entry.request.parameters.clone(), )), - max_new_tokens: entry.request.parameters.max_new_tokens, + stopping_parameters: Some(StoppingCriteriaParameters::from( + entry.request.parameters.clone(), + )), }); ids.push(*id); @@ -168,3 +172,12 @@ impl From for LogitsWarperParameters { } } } + +impl From for StoppingCriteriaParameters { + fn from(parameters: GenerateParameters) -> Self { + Self { + stop_sequences: parameters.stop, + max_new_tokens: parameters.max_new_tokens, + } + } +} diff --git a/router/src/lib.rs b/router/src/lib.rs index 8646ad59..b6c694ee 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -21,6 +21,7 @@ pub(crate) struct GenerateParameters { pub do_sample: bool, #[serde(default = "default_max_new_tokens")] pub max_new_tokens: u32, + pub stop: Vec, } fn default_temperature() -> f32 { @@ -50,6 +51,7 @@ fn default_parameters() -> GenerateParameters { top_p: default_top_p(), do_sample: default_do_sample(), max_new_tokens: default_max_new_tokens(), + stop: vec![], } } @@ -63,6 +65,7 @@ pub(crate) struct GenerateRequest { #[derive(Serialize)] pub(crate) struct GeneratedText { pub generated_text: String, + pub finish_reason: String, } #[derive(Serialize)] diff --git a/router/src/server.rs b/router/src/server.rs index 9f4a75c9..59296269 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -53,6 +53,7 @@ async fn health(state: Extension) -> Result<(), (StatusCode, Json 4 { + response_tx + .send(Err(ValidationError::StopSequence( + request.parameters.stop.len(), + ))) + .unwrap_or(()); + continue; + } // Get the number of tokens in the input match tokenizer.encode(request.inputs.clone(), false) { @@ -163,6 +171,8 @@ pub enum ValidationError { MaxNewTokens, #[error("inputs must have less than {1} tokens. Given: {0}")] InputLength(usize, usize), + #[error("stop supports up to 4 stop sequences. Given: {0}")] + StopSequence(usize), #[error("tokenizer error {0}")] Tokenizer(String), } diff --git a/server/tests/conftest.py b/server/tests/conftest.py index 0640d45d..24cdafac 100644 --- a/server/tests/conftest.py +++ b/server/tests/conftest.py @@ -15,6 +15,11 @@ def default_pb_parameters(): ) +@pytest.fixture +def default_pb_stop_parameters(): + return generate_pb2.StoppingCriteriaParameters(stop_sequences=[], max_new_tokens=10) + + @pytest.fixture(scope="session") def bloom_560m_tokenizer(): return AutoTokenizer.from_pretrained("bigscience/bloom-560m", padding_side="left") diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 49dabb14..c5dbaa3e 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -9,13 +9,13 @@ @pytest.fixture -def default_pb_request(default_pb_parameters): +def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", input_length=1, parameters=default_pb_parameters, - max_new_tokens=10, + stopping_parameters=default_pb_stop_parameters, ) @@ -36,7 +36,7 @@ def default_multi_requests_bloom_batch(default_pb_request, bloom_560m_tokenizer) req_0 = copy(default_pb_request) req_1 = default_pb_request req_1.id = 1 - req_1.max_new_tokens = 5 + req_1.stopping_parameters.max_new_tokens = 5 batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) return BloomCausalLMBatch.from_pb( @@ -56,7 +56,6 @@ def test_batch_from_pb(default_pb_batch, default_bloom_batch): assert batch.requests == default_pb_batch.requests assert len(batch.input_ids) == default_pb_batch.size - assert len(batch.input_ids[0]) == 8 assert batch.input_ids[0][-1] == 10264 assert torch.all(batch.input_ids[0][:-1] == 3) @@ -85,6 +84,7 @@ def test_causal_lm_batch_type(default_bloom): def test_causal_lm_generate_token(default_bloom, default_bloom_batch): + sequence_length = len(default_bloom_batch.all_input_ids[0]) generated_texts, next_batch = default_bloom.generate_token(default_bloom_batch) assert generated_texts == [] @@ -92,7 +92,11 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): assert not next_batch.keys_head_dim_last assert len(next_batch.all_input_ids) == next_batch.size - assert len(next_batch.all_input_ids[0]) == len(next_batch.attention_mask[0]) == 9 + assert ( + len(next_batch.all_input_ids[0]) + == len(next_batch.attention_mask[0]) + == sequence_length + 1 + ) assert torch.all(next_batch.all_input_ids[0][-2:] == 10264) assert torch.all(next_batch.all_input_ids[0][:-2] == 3) @@ -106,8 +110,12 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): assert next_batch.max_sequence_length == next_batch.input_lengths[0] assert next_batch.past_key_values is not None - assert all([p[0].shape == (16, 64, 8) for p in next_batch.past_key_values]) - assert all([p[1].shape == (16, 8, 64) for p in next_batch.past_key_values]) + assert all( + [p[0].shape == (16, 64, sequence_length) for p in next_batch.past_key_values] + ) + assert all( + [p[1].shape == (16, sequence_length, 64) for p in next_batch.past_key_values] + ) def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch): diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 1bf3e5e6..f38776cc 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -8,13 +8,13 @@ @pytest.fixture -def default_pb_request(default_pb_parameters): +def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", input_length=1, parameters=default_pb_parameters, - max_new_tokens=10, + stopping_parameters=default_pb_stop_parameters, ) @@ -33,7 +33,7 @@ def default_multi_requests_causal_lm_batch(default_pb_request, gpt2_tokenizer): req_0 = copy(default_pb_request) req_1 = default_pb_request req_1.id = 1 - req_1.max_new_tokens = 5 + req_1.stopping_parameters.max_new_tokens = 5 batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) return CausalLMBatch.from_pb(batch_pb, gpt2_tokenizer, torch.device("cpu")) @@ -51,7 +51,6 @@ def test_batch_from_pb(default_pb_batch, default_causal_lm_batch): assert batch.requests == default_pb_batch.requests assert len(batch.input_ids) == default_pb_batch.size - assert len(batch.input_ids[0]) == 8 assert batch.input_ids[0][-1] == 14402 assert torch.all(batch.input_ids[0][:-1] == 50256) @@ -80,6 +79,7 @@ def test_causal_lm_batch_type(default_causal_lm): def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): + sequence_length = len(default_causal_lm_batch.all_input_ids[0]) generated_texts, next_batch = default_causal_lm.generate_token( default_causal_lm_batch ) @@ -88,8 +88,12 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): assert isinstance(next_batch, CausalLMBatch) assert len(next_batch.all_input_ids) == next_batch.size - assert len(next_batch.all_input_ids[0]) == len(next_batch.attention_mask[0]) == 9 - assert next_batch.all_input_ids[0][-1] == 6208 + assert ( + len(next_batch.all_input_ids[0]) + == len(next_batch.attention_mask[0]) + == sequence_length + 1 + ) + assert next_batch.all_input_ids[0][-1] == 13 assert next_batch.all_input_ids[0][-2] == 14402 assert torch.all(next_batch.all_input_ids[0][:-2] == 50256) @@ -97,14 +101,18 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): assert torch.all(next_batch.attention_mask[0][:-2] == 0) assert next_batch.input_ids.shape == (next_batch.size, 1) - assert next_batch.input_ids[0, 0] == 6208 + assert next_batch.input_ids[0, 0] == 13 assert next_batch.input_lengths == [2] assert next_batch.max_sequence_length == next_batch.input_lengths[0] assert next_batch.past_key_values is not None - assert all([p[0].shape == (1, 12, 8, 64) for p in next_batch.past_key_values]) - assert all([p[1].shape == (1, 12, 8, 64) for p in next_batch.past_key_values]) + assert all( + [p[0].shape == (1, 12, sequence_length, 64) for p in next_batch.past_key_values] + ) + assert all( + [p[1].shape == (1, 12, sequence_length, 64) for p in next_batch.past_key_values] + ) def test_causal_lm_generate_token_completion( @@ -119,10 +127,7 @@ def test_causal_lm_generate_token_completion( assert next_batch is None assert len(generated_texts) == 1 - assert ( - generated_texts[0].output - == "Test Test Test Test Test Test Test Test Test Test Test" - ) + assert generated_texts[0].output == "Test.java:784) at net.minecraft." assert generated_texts[0].request == default_causal_lm_batch.requests[0] assert ( generated_texts[0].tokens @@ -145,7 +150,7 @@ def test_causal_lm_generate_token_completion_multi( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "Test Test Test Test Test Test" + assert generated_texts[0].output == "Test.java:784)" assert ( generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] ) @@ -166,10 +171,7 @@ def test_causal_lm_generate_token_completion_multi( assert next_batch is None assert len(generated_texts) == 1 - assert ( - generated_texts[0].output - == "Test Test Test Test Test Test Test Test Test Test Test" - ) + assert generated_texts[0].output == "Test.java:784) at net.minecraft." assert ( generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] ) @@ -200,7 +202,8 @@ def test_batch_concatenate( assert torch.all(next_batch.attention_mask[1:, :-2] == 0) assert next_batch.batch_id == 0 - assert torch.all(next_batch.input_ids == 6208) + assert next_batch.input_ids[0, 0] == 12355 + assert torch.all(next_batch.input_ids[1:] == 13) assert next_batch.input_lengths == [3, 2, 2] assert next_batch.max_sequence_length == 3 @@ -239,7 +242,7 @@ def test_batch_concatenate( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "Test Test Test Test Test Test" + assert generated_texts[0].output == "Test.java:784)" assert ( generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] ) @@ -260,10 +263,7 @@ def test_batch_concatenate( assert next_batch is not None assert len(generated_texts) == 1 - assert ( - generated_texts[0].output - == "Test Test Test Test Test Test Test Test Test Test Test" - ) + assert generated_texts[0].output == "Test.java:784) at net.minecraft." assert generated_texts[0].request == default_causal_lm_batch.requests[0] assert ( generated_texts[0].tokens @@ -283,10 +283,7 @@ def test_batch_concatenate( assert next_batch is None assert len(generated_texts) == 1 - assert ( - generated_texts[0].output - == "Test Test Test Test Test Test Test Test Test Test Test" - ) + assert generated_texts[0].output == "Test.java:784) at net.minecraft." assert ( generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] ) diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 7e4c7fdd..94ec70d5 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -8,13 +8,13 @@ @pytest.fixture -def default_pb_request(default_pb_parameters): +def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", input_length=2, parameters=default_pb_parameters, - max_new_tokens=10, + stopping_parameters=default_pb_stop_parameters, ) @@ -35,7 +35,7 @@ def default_multi_requests_seq2seq_lm_batch(default_pb_request, mt0_small_tokeni req_0 = copy(default_pb_request) req_1 = default_pb_request req_1.id = 1 - req_1.max_new_tokens = 5 + req_1.stopping_parameters.max_new_tokens = 5 batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) return Seq2SeqLMBatch.from_pb(batch_pb, mt0_small_tokenizer, torch.device("cpu")) @@ -48,11 +48,12 @@ def default_seq2seq_lm(): def test_batch_from_pb(default_pb_batch, default_seq2seq_lm_batch): batch = default_seq2seq_lm_batch + sequence_length = len(default_seq2seq_lm_batch.input_ids[0]) assert batch.batch_id == default_pb_batch.id assert batch.requests == default_pb_batch.requests - assert batch.input_ids.shape == (default_pb_batch.size, 8) + assert batch.input_ids.shape == (default_pb_batch.size, sequence_length) assert batch.input_ids[0][-2] == 4268 assert batch.input_ids[0][-1] == 1 assert torch.all(batch.input_ids[0][:-2] == 0) @@ -86,6 +87,7 @@ def test_seq2seq_lm_batch_type(default_seq2seq_lm): def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch): + sequence_length = len(default_seq2seq_lm_batch.input_ids[0]) generated_texts, next_batch = default_seq2seq_lm.generate_token( default_seq2seq_lm_batch ) @@ -108,7 +110,7 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) assert next_batch.decoder_input_ids[0, 0] == 0 assert next_batch.decoder_input_ids[0, 1] == 259 assert next_batch.decoder_attention_mask is None - assert next_batch.encoder_last_hidden_state.shape == (1, 8, 512) + assert next_batch.encoder_last_hidden_state.shape == (1, sequence_length, 512) assert next_batch.decoder_input_lengths == [2] assert next_batch.max_decoder_input_length == 2 @@ -121,10 +123,16 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) [p[1].shape == (next_batch.size, 6, 1, 64) for p in next_batch.past_key_values] ) assert all( - [p[2].shape == (next_batch.size, 6, 8, 64) for p in next_batch.past_key_values] + [ + p[2].shape == (next_batch.size, 6, sequence_length, 64) + for p in next_batch.past_key_values + ] ) assert all( - [p[3].shape == (next_batch.size, 6, 8, 64) for p in next_batch.past_key_values] + [ + p[3].shape == (next_batch.size, 6, sequence_length, 64) + for p in next_batch.past_key_values + ] ) diff --git a/server/tests/test_utils.py b/server/tests/test_utils.py index e630ebda..ed6e896a 100644 --- a/server/tests/test_utils.py +++ b/server/tests/test_utils.py @@ -4,10 +4,55 @@ weight_hub_files, download_weights, weight_files, + StopSequenceCriteria, + StoppingCriteria, LocalEntryNotFoundError, ) +def test_stop_sequence_criteria(): + criteria = StopSequenceCriteria([1, 2, 3]) + + assert not criteria(1) + assert criteria.current_token_idx == 1 + assert not criteria(2) + assert criteria.current_token_idx == 2 + assert criteria(3) + assert criteria.current_token_idx == 3 + + +def test_stop_sequence_criteria_reset(): + criteria = StopSequenceCriteria([1, 2, 3]) + + assert not criteria(1) + assert criteria.current_token_idx == 1 + assert not criteria(2) + assert criteria.current_token_idx == 2 + assert not criteria(4) + assert criteria.current_token_idx == 0 + + +def test_stop_sequence_criteria_empty(): + with pytest.raises(ValueError): + StopSequenceCriteria([]) + + +def test_stopping_criteria(): + criteria = StoppingCriteria([StopSequenceCriteria([1, 2, 3])], max_new_tokens=5) + assert criteria([1]) == (False, None) + assert criteria([1, 2]) == (False, None) + assert criteria([1, 2, 3]) == (True, "stop_sequence") + + +def test_stopping_criteria_max(): + criteria = StoppingCriteria([StopSequenceCriteria([1, 2, 3])], max_new_tokens=5) + assert criteria([1]) == (False, None) + assert criteria([1, 1]) == (False, None) + assert criteria([1, 1, 1]) == (False, None) + assert criteria([1, 1, 1, 1]) == (False, None) + assert criteria([1, 1, 1, 1, 1]) == (True, "length") + + def test_weight_hub_files(): filenames = weight_hub_files("bigscience/bloom-560m") assert filenames == ["model.safetensors"] diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 1cd999f0..72095858 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -57,23 +57,17 @@ def from_pb( for r in pb.requests: inputs.append(r.inputs) input_lengths.append(r.input_length) - next_token_choosers.append( - NextTokenChooser( - temperature=r.parameters.temperature, - top_k=r.parameters.top_k, - top_p=r.parameters.top_p, - do_sample=r.parameters.do_sample, - ) - ) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters)) stopping_criterias.append( - StoppingCriteria( - eos_token_id=tokenizer.eos_token_id, max_new_tokens=r.max_new_tokens - ) + StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) pad_to_multiple_of = 8 if "gpu" in str(device) else None tokenized_inputs = tokenizer( - inputs, return_tensors="pt", padding=True, pad_to_multiple_of=pad_to_multiple_of + inputs, + return_tensors="pt", + padding=True, + pad_to_multiple_of=pad_to_multiple_of, ).to(device) all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) @@ -123,8 +117,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": end_index = start_index + batch.size # We only concatenate batches that did at least one step - if batch.input_ids.shape[1] > 1: - raise ValueError("Batch input_ids should be of shape (batch_size, 1)") + if batch.past_key_values is None: + raise ValueError("only concatenate prefilled batches") # Create empty tensor # input_ids is always of shape [batch_size, 1] @@ -331,14 +325,17 @@ def generate_token( all_tokens = torch.cat([all_tokens, next_token]) # Evaluate stopping criteria - if stopping_criteria(all_tokens): + stop, reason = stopping_criteria(all_tokens) + if stop: # Decode all tokens output = self.tokenizer.decode( all_tokens.squeeze(-1), skip_special_tokens=True ) # Add to the list of finished generations with the original request generated_texts.append( - GeneratedText(request, output, stopping_criteria.current_tokens) + GeneratedText( + request, output, stopping_criteria.current_tokens, reason + ) ) # add to the next batch else: diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index 8aec1bc7..680ea43e 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -94,18 +94,9 @@ def from_pb( # Add escape_custom_split_sequence to the CausalLMBatch logic inputs.append(escape_custom_split_sequence(r.inputs)) input_lengths.append(r.input_length) - next_token_choosers.append( - NextTokenChooser( - temperature=r.parameters.temperature, - top_k=r.parameters.top_k, - top_p=r.parameters.top_p, - do_sample=r.parameters.do_sample, - ) - ) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters)) stopping_criterias.append( - StoppingCriteria( - eos_token_id=tokenizer.eos_token_id, max_new_tokens=r.max_new_tokens - ) + StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) tokenized_inputs = tokenizer( @@ -207,11 +198,7 @@ def load_weights( continue module_name, param_name = name.rsplit(".", 1) - try: - module = model.get_submodule(module_name) - except Exception as e: - print(type(model), name, module_name, param_name) - raise e + module = model.get_submodule(module_name) current_tensor = parameters[name] slice_ = f.get_slice(name) diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index cbfe7ccf..93e31b4a 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -68,24 +68,18 @@ def from_pb( # Decoder sequence only contains the bos_token decoder_input_ids.append(tokenizer.bos_token_id) decoder_input_lengths.append(1) - next_token_choosers.append( - NextTokenChooser( - temperature=r.parameters.temperature, - top_k=r.parameters.top_k, - top_p=r.parameters.top_p, - do_sample=r.parameters.do_sample, - ) - ) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters)) stopping_criterias.append( - StoppingCriteria( - eos_token_id=tokenizer.eos_token_id, max_new_tokens=r.max_new_tokens - ) + StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) # Tokenize batch pad_to_multiple_of = 8 if "gpu" in str(device) else None tokenized_inputs = tokenizer( - inputs, return_tensors="pt", padding=True, pad_to_multiple_of=pad_to_multiple_of + inputs, + return_tensors="pt", + padding=True, + pad_to_multiple_of=pad_to_multiple_of, ).to(device) # Convert decoder_input_ids to torch tensor of size [batch_size, 1] decoder_input_ids = torch.tensor(decoder_input_ids, device=device).unsqueeze(-1) @@ -431,12 +425,15 @@ def generate_token( decoder_tokens = torch.cat([decoder_tokens, next_token.squeeze(1)]) # Evaluate stopping criteria - if stopping_criteria(decoder_tokens): + stop, reason = stopping_criteria(decoder_tokens) + if stop: # Decode tokens output = self.tokenizer.decode(decoder_tokens, skip_special_tokens=True) # Add to the list of finished generations with the original request generated_texts.append( - GeneratedText(request, output, stopping_criteria.current_tokens) + GeneratedText( + request, output, stopping_criteria.current_tokens, reason + ) ) # add to the next batch else: diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index 7c25bf67..91ec75d0 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -32,8 +32,12 @@ class GeneratedText: request: generate_pb2.Request output: str tokens: int + reason: str def to_pb(self) -> generate_pb2.GeneratedText: return generate_pb2.GeneratedText( - request=self.request, output=self.output, tokens=self.tokens + request=self.request, + output=self.output, + tokens=self.tokens, + finish_reason=self.reason, ) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index e55eeb64..661df05e 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -1,6 +1,5 @@ import concurrent import os -import signal import torch import torch.distributed @@ -11,6 +10,8 @@ from huggingface_hub import HfApi, hf_hub_download, try_to_load_from_cache from huggingface_hub.utils import LocalEntryNotFoundError from tqdm import tqdm +from typing import List, Optional, Tuple +from transformers import AutoTokenizer from transformers.generation.logits_process import ( LogitsProcessorList, TemperatureLogitsWarper, @@ -18,6 +19,8 @@ TopKLogitsWarper, ) +from text_generation.pb import generate_pb2 + class Sampling: def __call__(self, logits): @@ -56,20 +59,72 @@ def __call__(self, input_ids, scores): next_ids = self.choice(scores) return next_ids.unsqueeze(-1) + @classmethod + def from_pb(cls, pb: generate_pb2.LogitsWarperParameters) -> "NextTokenChooser": + return NextTokenChooser( + temperature=pb.temperature, + top_k=pb.top_k, + top_p=pb.top_p, + do_sample=pb.do_sample, + ) + + +class StopSequenceCriteria: + def __init__(self, tokens: List[int]): + if not tokens: + raise ValueError("tokens cannot be empty") + + self.tokens = tokens + self.current_token_idx = 0 + + def __call__(self, last_token: int) -> bool: + if last_token == self.tokens[self.current_token_idx]: + # Increase idx to go to next token + self.current_token_idx += 1 + else: + # Reset to first token of the stopping sequence + self.current_token_idx = 0 + + if self.current_token_idx == len(self.tokens): + # We matched the entire sequence without resetting + return True + return False + class StoppingCriteria: - def __init__(self, eos_token_id, max_new_tokens=20): - self.eos_token_id = eos_token_id + def __init__( + self, stop_sequence_criterias: List[StopSequenceCriteria], max_new_tokens=20 + ): + self.stop_sequence_criterias = stop_sequence_criterias self.max_new_tokens = max_new_tokens self.current_tokens = 0 - def __call__(self, all_ids): + def __call__(self, all_ids) -> Tuple[bool, Optional[str]]: self.current_tokens += 1 if self.current_tokens >= self.max_new_tokens: - return True - if self.eos_token_id is not None and all_ids[-1] == self.eos_token_id: - return True - return False + return True, "length" + + last_token = all_ids[-1] + for stop_sequence_criteria in self.stop_sequence_criterias: + if stop_sequence_criteria(last_token): + return True, "stop_sequence" + + return False, None + + @classmethod + def from_pb( + cls, pb: generate_pb2.StoppingCriteriaParameters, tokenizer: AutoTokenizer + ) -> "StoppingCriteria": + stop_sequence_criterias = [] + for stop_sequence in pb.stop_sequences: + tokens = tokenizer( + stop_sequence, padding=False, return_attention_mask=False + ).input_ids + if tokens: + stop_sequence_criterias.append(StopSequenceCriteria(tokens)) + stop_sequence_criterias.append(StopSequenceCriteria([tokenizer.eos_token_id])) + + return StoppingCriteria(stop_sequence_criterias, pb.max_new_tokens) def initialize_torch_distributed(): From 32a253063dae768e71a0b0aa099cfbbe962032d1 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 15 Dec 2022 17:03:56 +0100 Subject: [PATCH 037/793] feat: Return logprobs (#8) --- .github/workflows/server-tests.yaml | 30 ++++++++++++ README.md | 1 + proto/generate.proto | 18 ++++--- router/client/src/lib.rs | 2 +- router/src/batcher.rs | 13 +++-- router/src/db.rs | 6 +-- router/src/lib.rs | 14 +++++- router/src/server.rs | 31 ++++++++++-- server/tests/conftest.py | 2 +- server/tests/models/test_bloom.py | 32 ++++++++----- server/tests/models/test_causal_lm.py | 25 +++++----- server/tests/models/test_seq2seq_lm.py | 24 +++++----- server/text_generation/models/bloom.py | 10 ++-- server/text_generation/models/causal_lm.py | 53 +++++++++++++++++---- server/text_generation/models/galactica.py | 10 ++-- server/text_generation/models/seq2seq_lm.py | 50 +++++++++++++++---- server/text_generation/models/types.py | 12 +++-- server/text_generation/utils.py | 8 +++- 18 files changed, 247 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/server-tests.yaml diff --git a/.github/workflows/server-tests.yaml b/.github/workflows/server-tests.yaml new file mode 100644 index 00000000..5bb4653a --- /dev/null +++ b/.github/workflows/server-tests.yaml @@ -0,0 +1,30 @@ +name: Server Tests + +on: + pull_request: + paths: + - "server/**" + - "proto/**" + +jobs: + run_tests: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.9 + - name: Loading cache. + uses: actions/cache@v2 + id: model_cache + with: + path: ~/.cache/huggingface/ + key: models + - name: Install server dependencies + run: | + make install-server + - name: Run tests + run: | + pip install pytest + pytest -sv server/tests diff --git a/README.md b/README.md index 92879707..8496ce8f 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ to power Bloom, BloomZ and MT0-XXL api-inference widgets. - 45ms per token generation for BLOOM with 8xA100 80GB - Logits warpers (temperature scaling, topk ...) - Stop sequences +- Log probabilities ## Officially supported models diff --git a/proto/generate.proto b/proto/generate.proto index 0c67de03..16539f8b 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -27,7 +27,7 @@ message ClearCacheRequest {} /// Empty response message ClearCacheResponse {} -message LogitsWarperParameters { +message NextTokenChooserParameters { /// exponential scaling output probability distribution float temperature = 1; /// restricting to the k highest probability elements @@ -52,8 +52,8 @@ message Request { string inputs = 2; /// The number of tokens inside inputs uint32 input_length = 3; - /// Logits Warper Parameters - LogitsWarperParameters parameters = 4; + /// Next Token Chooser Parameters + NextTokenChooserParameters parameters = 4; /// Stopping Criteria Parameters StoppingCriteriaParameters stopping_parameters = 5; } @@ -71,11 +71,17 @@ message GeneratedText { /// Request Request request = 1; /// Output - string output = 2; + string output_text = 2; /// Number of generated tokens - uint32 tokens = 3; + uint32 generated_tokens = 3; + /// Tokens + repeated string tokens = 4; + /// Token IDs + repeated uint32 token_ids = 5; + /// Logprobs + repeated float logprobs = 6; /// Finish reason - string finish_reason = 4; + string finish_reason = 7; } message GenerateRequest { diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index ae337dd6..295b009b 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -7,7 +7,7 @@ mod sharded_client; pub use client::Client; pub use pb::generate::v1::{ - Batch, GeneratedText, LogitsWarperParameters, Request, StoppingCriteriaParameters, + Batch, GeneratedText, NextTokenChooserParameters, Request, StoppingCriteriaParameters, }; pub use sharded_client::ShardedClient; use thiserror::Error; diff --git a/router/src/batcher.rs b/router/src/batcher.rs index a72a6e44..1484434c 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -187,9 +187,13 @@ fn send_generated(finished: Vec, db: &Db) { let entry = db .remove(&output.request.unwrap().id) .expect("ID not found in db. This is a bug."); + let response = InferResponse { - output: output.output, + output_text: output.output_text, + generated_tokens: output.generated_tokens, + token_ids: output.token_ids, tokens: output.tokens, + logprobs: output.logprobs, finish_reason: output.finish_reason, queued: entry.time, start: entry.batch_time.unwrap(), // unwrap is always valid @@ -202,8 +206,11 @@ fn send_generated(finished: Vec, db: &Db) { #[derive(Debug)] pub(crate) struct InferResponse { - pub(crate) output: String, - pub(crate) tokens: u32, + pub(crate) output_text: String, + pub(crate) generated_tokens: u32, + pub(crate) token_ids: Vec, + pub(crate) tokens: Vec, + pub(crate) logprobs: Vec, pub(crate) finish_reason: String, pub(crate) queued: Instant, pub(crate) start: Instant, diff --git a/router/src/db.rs b/router/src/db.rs index 24fb7a09..df9f2b8e 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -5,7 +5,7 @@ use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; use text_generation_client::{ - Batch, ClientError, LogitsWarperParameters, Request, StoppingCriteriaParameters, + Batch, ClientError, NextTokenChooserParameters, Request, StoppingCriteriaParameters, }; use tokio::sync::oneshot::Sender; use tokio::time::Instant; @@ -71,7 +71,7 @@ impl State { id: *id, inputs: entry.request.inputs.clone(), input_length: entry.input_length as u32, - parameters: Some(LogitsWarperParameters::from( + parameters: Some(NextTokenChooserParameters::from( entry.request.parameters.clone(), )), stopping_parameters: Some(StoppingCriteriaParameters::from( @@ -162,7 +162,7 @@ impl Db { } } -impl From for LogitsWarperParameters { +impl From for NextTokenChooserParameters { fn from(parameters: GenerateParameters) -> Self { Self { temperature: parameters.temperature, diff --git a/router/src/lib.rs b/router/src/lib.rs index b6c694ee..03711580 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -21,7 +21,10 @@ pub(crate) struct GenerateParameters { pub do_sample: bool, #[serde(default = "default_max_new_tokens")] pub max_new_tokens: u32, + #[serde(default)] pub stop: Vec, + #[serde(default)] + pub details: bool, } fn default_temperature() -> f32 { @@ -52,6 +55,7 @@ fn default_parameters() -> GenerateParameters { do_sample: default_do_sample(), max_new_tokens: default_max_new_tokens(), stop: vec![], + details: false, } } @@ -62,10 +66,18 @@ pub(crate) struct GenerateRequest { pub parameters: GenerateParameters, } +#[derive(Serialize)] +pub(crate) struct Details { + pub finish_reason: String, + pub generated_tokens: u32, + pub tokens: Vec<(u32, String, f32)>, +} + #[derive(Serialize)] pub(crate) struct GeneratedText { pub generated_text: String, - pub finish_reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub details: Option
, } #[derive(Serialize)] diff --git a/router/src/server.rs b/router/src/server.rs index 59296269..2e6c473f 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,5 +1,5 @@ use crate::{ - Batcher, ErrorResponse, GenerateParameters, GenerateRequest, GeneratedText, Validation, + Batcher, Details, ErrorResponse, GenerateParameters, GenerateRequest, GeneratedText, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, StatusCode}; @@ -54,6 +54,7 @@ async fn health(state: Extension) -> Result<(), (StatusCode, Json { + let tokens = response + .token_ids + .into_iter() + .zip(response.tokens.into_iter()) + .zip(response.logprobs.into_iter()) + .map(|((id, text), logprob)| (id, text, logprob)) + .collect(); + Some(Details { + finish_reason: response.finish_reason, + generated_tokens: response.generated_tokens, + tokens, + }) + } + false => None, + }; + // Timings let total_time = start_time.elapsed(); let validation_time = response.queued - start_time; let queue_time = response.start - response.queued; let inference_time = response.end - response.start; - let time_per_token = inference_time / response.tokens; + let time_per_token = inference_time / response.generated_tokens; // Headers let mut headers = HeaderMap::new(); @@ -141,12 +162,12 @@ async fn generate( tracing::Span::current().record("queue_time", format!("{:?}", queue_time)); tracing::Span::current().record("inference_time", format!("{:?}", inference_time)); tracing::Span::current().record("time_per_token", format!("{:?}", time_per_token)); - tracing::info!("Output: {}", response.output); + tracing::info!("Output: {}", response.output_text); // Send response let response = vec![GeneratedText { - generated_text: response.output, - finish_reason: response.finish_reason, + generated_text: response.output_text, + details, }]; Ok((headers, Json(response))) } diff --git a/server/tests/conftest.py b/server/tests/conftest.py index 24cdafac..eb72b8a2 100644 --- a/server/tests/conftest.py +++ b/server/tests/conftest.py @@ -7,7 +7,7 @@ @pytest.fixture def default_pb_parameters(): - return generate_pb2.LogitsWarperParameters( + return generate_pb2.NextTokenChooserParameters( temperature=1.0, top_k=0, top_p=1.0, diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index c5dbaa3e..2a6e670e 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -128,10 +128,12 @@ def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch) assert next_batch is None assert len(generated_texts) == 1 - assert generated_texts[0].output == "TestTestTestTestTestTestTestTestTestTestTest" + assert ( + generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + ) assert generated_texts[0].request == default_bloom_batch.requests[0] assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -151,10 +153,10 @@ def test_causal_lm_generate_token_completion_multi( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "TestTestTestTestTestTest" + assert generated_texts[0].output_text == "TestTestTestTestTestTest" assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) @@ -170,10 +172,12 @@ def test_causal_lm_generate_token_completion_multi( assert next_batch is None assert len(generated_texts) == 1 - assert generated_texts[0].output == "TestTestTestTestTestTestTestTestTestTestTest" + assert ( + generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + ) assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -240,10 +244,10 @@ def test_batch_concatenate( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "TestTestTestTestTestTest" + assert generated_texts[0].output_text == "TestTestTestTestTestTest" assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) @@ -259,10 +263,12 @@ def test_batch_concatenate( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "TestTestTestTestTestTestTestTestTestTestTest" + assert ( + generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + ) assert generated_texts[0].request == default_bloom_batch.requests[0] assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -279,9 +285,11 @@ def test_batch_concatenate( assert next_batch is None assert len(generated_texts) == 1 - assert generated_texts[0].output == "TestTestTestTestTestTestTestTestTestTestTest" + assert ( + generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + ) assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index f38776cc..683d9fdd 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -127,10 +127,11 @@ def test_causal_lm_generate_token_completion( assert next_batch is None assert len(generated_texts) == 1 - assert generated_texts[0].output == "Test.java:784) at net.minecraft." + assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." assert generated_texts[0].request == default_causal_lm_batch.requests[0] + assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -150,12 +151,12 @@ def test_causal_lm_generate_token_completion_multi( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "Test.java:784)" + assert generated_texts[0].output_text == "Test.java:784)" assert ( generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] ) assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) @@ -171,12 +172,12 @@ def test_causal_lm_generate_token_completion_multi( assert next_batch is None assert len(generated_texts) == 1 - assert generated_texts[0].output == "Test.java:784) at net.minecraft." + assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." assert ( generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] ) assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -242,12 +243,12 @@ def test_batch_concatenate( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "Test.java:784)" + assert generated_texts[0].output_text == "Test.java:784)" assert ( generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] ) assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) @@ -263,10 +264,10 @@ def test_batch_concatenate( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "Test.java:784) at net.minecraft." + assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." assert generated_texts[0].request == default_causal_lm_batch.requests[0] assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -283,11 +284,11 @@ def test_batch_concatenate( assert next_batch is None assert len(generated_texts) == 1 - assert generated_texts[0].output == "Test.java:784) at net.minecraft." + assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." assert ( generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] ) assert ( - generated_texts[0].tokens + generated_texts[0].generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 94ec70d5..f1b11bc2 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -148,9 +148,9 @@ def test_seq2seq_lm_generate_token_completion( assert next_batch is None assert len(generated_texts) == 1 - assert generated_texts[0].output == "a few weeks" + assert generated_texts[0].output_text == "a few weeks" assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] - assert generated_texts[0].tokens == 7 + assert generated_texts[0].generated_tokens == 7 def test_seq2seq_lm_generate_token_completion_multi( @@ -166,12 +166,12 @@ def test_seq2seq_lm_generate_token_completion_multi( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "a few " + assert generated_texts[0].output_text == "a few " assert ( generated_texts[0].request == default_multi_requests_seq2seq_lm_batch.requests[1] ) - assert generated_texts[0].tokens == 5 + assert generated_texts[0].generated_tokens == 5 generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) assert generated_texts == [] @@ -180,12 +180,12 @@ def test_seq2seq_lm_generate_token_completion_multi( assert next_batch is None assert len(generated_texts) == 1 - assert generated_texts[0].output == "a few weeks" + assert generated_texts[0].output_text == "a few weeks" assert ( generated_texts[0].request == default_multi_requests_seq2seq_lm_batch.requests[0] ) - assert generated_texts[0].tokens == 7 + assert generated_texts[0].generated_tokens == 7 def test_batch_concatenate( @@ -287,28 +287,28 @@ def test_batch_concatenate( assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "a few " + assert generated_texts[0].output_text == "a few " assert ( generated_texts[0].request == default_multi_requests_seq2seq_lm_batch.requests[1] ) - assert generated_texts[0].tokens == 5 + assert generated_texts[0].generated_tokens == 5 generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None assert len(generated_texts) == 1 - assert generated_texts[0].output == "a few weeks" + assert generated_texts[0].output_text == "a few weeks" assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] - assert generated_texts[0].tokens == 7 + assert generated_texts[0].generated_tokens == 7 generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None assert len(generated_texts) == 1 - assert generated_texts[0].output == "a few weeks" + assert generated_texts[0].output_text == "a few weeks" assert ( generated_texts[0].request == default_multi_requests_seq2seq_lm_batch.requests[0] ) - assert generated_texts[0].tokens == 7 + assert generated_texts[0].generated_tokens == 7 diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 20e26419..3561a8ea 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -246,12 +246,8 @@ def forward(self, input_ids, attention_mask, past_key_values: Optional = None): ) # Logits are sharded, so we need to gather them - logits_shard = outputs.logits[:, -1, :].contiguous() - - batch_size, vocab_shard_size = logits_shard.shape - vocab_size = self.world_size * vocab_shard_size - logits = [torch.empty_like(logits_shard) for _ in range(self.world_size)] - torch.distributed.all_gather(logits, logits_shard, group=self.process_group) - logits = torch.cat(logits, dim=1).view(batch_size, 1, vocab_size) + logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] + torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) + logits = torch.cat(logits, dim=2) return logits, outputs.past_key_values diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 72095858..5d62137f 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -22,6 +22,7 @@ class CausalLMBatch: # All tokens all_input_ids: List[torch.Tensor] + all_logprobs: List[Optional[torch.Tensor]] # Lengths of all generations present in the batch input_lengths: List[int] @@ -52,6 +53,7 @@ def from_pb( next_token_choosers = [] stopping_criterias = [] input_lengths = [] + all_logprobs = [] # Parse batch for r in pb.requests: @@ -61,6 +63,7 @@ def from_pb( stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) + all_logprobs.append(None) pad_to_multiple_of = 8 if "gpu" in str(device) else None tokenized_inputs = tokenizer( @@ -78,6 +81,7 @@ def from_pb( attention_mask=tokenized_inputs["attention_mask"], past_key_values=None, all_input_ids=all_input_ids, + all_logprobs=all_logprobs, input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, @@ -95,6 +99,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests = [] input_lengths = [] all_input_ids = [] + all_logprobs = [] next_token_choosers = [] stopping_criterias = [] @@ -110,6 +115,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) all_input_ids.extend(batch.all_input_ids) + all_logprobs.extend(batch.all_logprobs) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -217,6 +223,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": attention_mask=attention_mask, past_key_values=past_key_values, all_input_ids=all_input_ids, + all_logprobs=all_logprobs, input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, @@ -291,6 +298,7 @@ def generate_token( next_batch_input_lengths = [] next_batch_input_ids = [] next_batch_all_input_ids = [] + next_batch_all_logprobs = [] # Metadata next_batch_size = 0 @@ -307,6 +315,7 @@ def generate_token( batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, + batch.all_logprobs, ) # For each member of the batch @@ -316,34 +325,59 @@ def generate_token( logits, next_token_chooser, stopping_criteria, - all_tokens, + all_input_ids, + all_logprobs, ) in enumerate(iterator): # Select next token - next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) + tokens, logprobs = next_token_chooser(all_input_ids, logits) + next_token = tokens[-1].view(1, 1) # Append next token to all tokens - all_tokens = torch.cat([all_tokens, next_token]) + all_input_ids = torch.cat([all_input_ids, next_token]) + new_input_length = input_length + 1 + + if all_logprobs is None: + # logprobs of all prompt tokens (except the first one) and the generated token + all_logprobs = logprobs.gather(1, all_input_ids[1:]) + else: + # logprob of the generated token + next_token_logprob = logprobs[-1, next_token] + all_logprobs = torch.cat([all_logprobs, next_token_logprob]) # Evaluate stopping criteria - stop, reason = stopping_criteria(all_tokens) + stop, reason = stopping_criteria(all_input_ids) if stop: # Decode all tokens - output = self.tokenizer.decode( - all_tokens.squeeze(-1), skip_special_tokens=True + output_text = self.tokenizer.decode( + all_input_ids.squeeze(-1), skip_special_tokens=True ) + # Slice with input_length to remove padding + token_ids = all_input_ids[-new_input_length:] + tokens = self.tokenizer.batch_decode(token_ids) + # Add NaN for the first prompt token + logprobs = [float("nan")] + all_logprobs[-new_input_length:].squeeze( + 1 + ).tolist() + # Add to the list of finished generations with the original request generated_texts.append( GeneratedText( - request, output, stopping_criteria.current_tokens, reason + request=request, + output_text=output_text, + generated_tokens=stopping_criteria.current_tokens, + tokens=tokens, + token_ids=token_ids.squeeze(1).tolist(), + logprobs=logprobs, + reason=reason, ) ) # add to the next batch else: next_batch_keep_indices.append(i) next_batch_input_ids.append(next_token) - next_batch_all_input_ids.append(all_tokens) + next_batch_all_input_ids.append(all_input_ids) + next_batch_all_logprobs.append(all_logprobs) next_batch_size += 1 - new_input_length = input_length + 1 next_batch_input_lengths.append(new_input_length) next_batch_max_sequence_length = max( next_batch_max_sequence_length, new_input_length @@ -397,6 +431,7 @@ def generate_token( attention_mask=next_batch_attention_mask, past_key_values=next_batch_past_key_values, all_input_ids=next_batch_all_input_ids, + all_logprobs=next_batch_all_logprobs, input_lengths=next_batch_input_lengths, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index 680ea43e..a713e69e 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -321,12 +321,8 @@ def forward(self, input_ids, attention_mask, past_key_values: Optional = None): ) # Logits are sharded, so we need to gather them - logits_shard = outputs.logits[:, -1, :].contiguous() - - batch_size, vocab_shard_size = logits_shard.shape - vocab_size = self.world_size * vocab_shard_size - logits = [torch.empty_like(logits_shard) for _ in range(self.world_size)] - torch.distributed.all_gather(logits, logits_shard, group=self.process_group) - logits = torch.cat(logits, dim=1).view(batch_size, 1, vocab_size) + logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] + torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) + logits = torch.cat(logits, dim=2) return logits, outputs.past_key_values diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 93e31b4a..e51ce60b 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -30,6 +30,7 @@ class Seq2SeqLMBatch: # Lengths of all generations present in the batch input_lengths: List[int] decoder_input_lengths: List[int] + decoder_logprobs: List[Optional[torch.Tensor]] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -60,6 +61,7 @@ def from_pb( decoder_input_ids = [] decoder_input_lengths = [] + decoder_logprobs = [] # Parse batch for r in pb.requests: @@ -72,6 +74,7 @@ def from_pb( stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) + decoder_logprobs.append(None) # Tokenize batch pad_to_multiple_of = 8 if "gpu" in str(device) else None @@ -95,6 +98,7 @@ def from_pb( past_key_values=None, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, + decoder_logprobs=decoder_logprobs, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=len(pb.requests), @@ -117,6 +121,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests = [] input_lengths = [] decoder_input_lengths = [] + decoder_logprobs = [] next_token_choosers = [] stopping_criterias = [] @@ -137,6 +142,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) decoder_input_lengths.extend(batch.decoder_input_lengths) + decoder_logprobs.extend(batch.decoder_logprobs) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -286,6 +292,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": past_key_values=past_key_values, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, + decoder_logprobs=decoder_logprobs, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=total_batch_size, @@ -385,6 +392,7 @@ def generate_token( next_batch_input_lengths = [] next_batch_decoder_input_ids = [] next_batch_decoder_input_lengths = [] + next_batch_decoder_logprobs = [] # Metadata next_batch_size = 0 @@ -399,6 +407,7 @@ def generate_token( batch.requests, batch.input_lengths, batch.decoder_input_lengths, + batch.decoder_logprobs, logits, batch.next_token_choosers, batch.stopping_criterias, @@ -411,38 +420,58 @@ def generate_token( request, input_length, decoder_input_length, + decoder_logprobs, logits, next_token_chooser, stopping_criteria, input_tokens, - decoder_tokens, + decoder_input_ids, ) in enumerate(iterator): - all_tokens = torch.cat([input_tokens, decoder_tokens]) # Select next token - next_token = next_token_chooser(all_tokens, logits.unsqueeze(0)[:, -1]) + next_token, logprobs = next_token_chooser(decoder_input_ids, logits) # Append next token to decoder tokens - decoder_tokens = torch.cat([decoder_tokens, next_token.squeeze(1)]) + decoder_input_ids = torch.cat([decoder_input_ids, next_token]) + new_decoder_input_length = decoder_input_length + 1 + + next_token_logprob = logprobs[-1, next_token] + if decoder_logprobs is None: + decoder_logprobs = next_token_logprob + else: + decoder_logprobs = torch.cat([decoder_logprobs, next_token_logprob]) # Evaluate stopping criteria - stop, reason = stopping_criteria(decoder_tokens) + stop, reason = stopping_criteria(decoder_input_ids) if stop: - # Decode tokens - output = self.tokenizer.decode(decoder_tokens, skip_special_tokens=True) + # Slice with decoder_input_length to remove padding + # Decode all tokens + token_ids = decoder_input_ids[-new_decoder_input_length:] + output_text = self.tokenizer.decode(token_ids, skip_special_tokens=True) + tokens = self.tokenizer.batch_decode(token_ids) + # Add NaN for the bos token + logprobs = [float("nan")] + decoder_logprobs[ + -new_decoder_input_length: + ].tolist() # Add to the list of finished generations with the original request generated_texts.append( GeneratedText( - request, output, stopping_criteria.current_tokens, reason + request=request, + output_text=output_text, + generated_tokens=stopping_criteria.current_tokens, + tokens=tokens, + token_ids=token_ids.tolist(), + logprobs=logprobs, + reason=reason, ) ) # add to the next batch else: next_batch_keep_indices.append(i) - next_batch_decoder_input_ids.append(decoder_tokens.unsqueeze(0)) + next_batch_decoder_input_ids.append(decoder_input_ids.unsqueeze(0)) next_batch_size += 1 - new_decoder_input_length = decoder_input_length + 1 next_batch_input_lengths.append(input_length) next_batch_decoder_input_lengths.append(new_decoder_input_length) + next_batch_decoder_logprobs.append(decoder_logprobs) next_batch_max_input_length = max( next_batch_max_input_length, input_length ) @@ -515,6 +544,7 @@ def generate_token( past_key_values=next_batch_past_key_values, input_lengths=next_batch_input_lengths, decoder_input_lengths=next_batch_decoder_input_lengths, + decoder_logprobs=next_batch_decoder_logprobs, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index 91ec75d0..e76cf697 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -30,14 +30,20 @@ def concatenate(cls, batches: List["Batch"]) -> "Batch": @dataclass class GeneratedText: request: generate_pb2.Request - output: str - tokens: int + output_text: str + generated_tokens: int + tokens: List[str] + token_ids: List[int] + logprobs: List[float] reason: str def to_pb(self) -> generate_pb2.GeneratedText: return generate_pb2.GeneratedText( request=self.request, - output=self.output, + output_text=self.output_text, + generated_tokens=self.generated_tokens, tokens=self.tokens, + token_ids=self.token_ids, + logprobs=self.logprobs, finish_reason=self.reason, ) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 661df05e..9dd91151 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -55,12 +55,16 @@ def __init__(self, temperature=1.0, top_k=None, top_p=None, do_sample=False): self.choice = Sampling() if sampling else Greedy() def __call__(self, input_ids, scores): + # Warp logits scores = self.warpers(input_ids, scores) + # Compute logprobs + logprobs = torch.log_softmax(scores, -1) + # Choose tokens next_ids = self.choice(scores) - return next_ids.unsqueeze(-1) + return next_ids, logprobs @classmethod - def from_pb(cls, pb: generate_pb2.LogitsWarperParameters) -> "NextTokenChooser": + def from_pb(cls, pb: generate_pb2.NextTokenChooserParameters) -> "NextTokenChooser": return NextTokenChooser( temperature=pb.temperature, top_k=pb.top_k, From 3e2e6240b8f557e4db104fbb4e05dc43456b0b48 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 16 Dec 2022 11:29:36 +0100 Subject: [PATCH 038/793] feat(launcher): Add integration tests (#9) --- .../{server-tests.yaml => tests.yaml} | 17 +- Cargo.lock | 30 ++-- launcher/Cargo.toml | 8 +- launcher/tests/bloom_560m.json | 121 ++++++++++++++ launcher/tests/integration_tests.rs | 156 ++++++++++++++++++ launcher/tests/mt0_base.json | 116 +++++++++++++ 6 files changed, 434 insertions(+), 14 deletions(-) rename .github/workflows/{server-tests.yaml => tests.yaml} (60%) create mode 100644 launcher/tests/bloom_560m.json create mode 100644 launcher/tests/integration_tests.rs create mode 100644 launcher/tests/mt0_base.json diff --git a/.github/workflows/server-tests.yaml b/.github/workflows/tests.yaml similarity index 60% rename from .github/workflows/server-tests.yaml rename to .github/workflows/tests.yaml index 5bb4653a..7bdd3b73 100644 --- a/.github/workflows/server-tests.yaml +++ b/.github/workflows/tests.yaml @@ -5,6 +5,8 @@ on: paths: - "server/**" - "proto/**" + - "router/**" + - "launcher/**" jobs: run_tests: @@ -15,16 +17,25 @@ jobs: uses: actions/setup-python@v1 with: python-version: 3.9 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.65.0 + override: true + components: rustfmt, clippy - name: Loading cache. uses: actions/cache@v2 id: model_cache with: path: ~/.cache/huggingface/ key: models - - name: Install server dependencies + - name: Install run: | - make install-server - - name: Run tests + make install + - name: Run server tests run: | pip install pytest pytest -sv server/tests + - name: Run Rust tests + run: | + cargo test diff --git a/Cargo.lock b/Cargo.lock index 0fd5c4bf..752c4886 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -543,6 +543,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float_eq" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853" + [[package]] name = "fnv" version = "1.0.7" @@ -1505,9 +1511,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ "base64", "bytes", @@ -1587,18 +1593,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.147" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" dependencies = [ "proc-macro2", "quote", @@ -1607,9 +1613,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.87" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45" +checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" dependencies = [ "itoa", "ryu", @@ -1724,9 +1730,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.103" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", @@ -1804,6 +1810,10 @@ version = "0.1.0" dependencies = [ "clap 4.0.22", "ctrlc", + "float_eq", + "reqwest", + "serde", + "serde_json", "subprocess", "tracing", "tracing-subscriber", diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 1779c051..ecdef831 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -7,7 +7,13 @@ description = "Text Generation Launcher" [dependencies] clap = { version = "4.0.15", features = ["derive", "env"] } -ctrlc = "3.2.3" +ctrlc = { version = "3.2.3", features = ["termination"] } subprocess = "0.2.9" tracing = "0.1.37" tracing-subscriber = { version = "0.3.16", features = ["json"] } + +[dev-dependencies] +float_eq = "1.0.1" +reqwest = { version = "0.11.13", features = ["blocking", "json"] } +serde = "1.0.150" +serde_json = "1.0.89" diff --git a/launcher/tests/bloom_560m.json b/launcher/tests/bloom_560m.json new file mode 100644 index 00000000..d17f1ed4 --- /dev/null +++ b/launcher/tests/bloom_560m.json @@ -0,0 +1,121 @@ +[ + { + "details": { + "finish_reason": "length", + "generated_tokens": 20, + "tokens": [ + [ + 10264, + "Test", + null + ], + [ + 8821, + " request", + -11.895094 + ], + [ + 17, + ".", + -1.8267941 + ], + [ + 1587, + "get", + -2.4674964 + ], + [ + 11, + "(", + -1.9060438 + ], + [ + 5, + "\"", + -1.2279553 + ], + [ + 4899, + "action", + -4.170306 + ], + [ + 5, + "\"", + -0.3247902 + ], + [ + 12, + ")", + -1.0773602 + ], + [ + 30, + ";", + -0.27640444 + ], + [ + 837, + "\n ", + -1.6970599 + ], + [ + 1320, + " if", + -1.4495552 + ], + [ + 375, + " (", + -0.2360998 + ], + [ + 4899, + "action", + -1.1916926 + ], + [ + 3535, + " ==", + -0.8918663 + ], + [ + 5109, + " null", + -0.39334255 + ], + [ + 12, + ")", + -0.4321134 + ], + [ + 731, + " {", + -0.17701954 + ], + [ + 1260, + "\n ", + -0.07027287 + ], + [ + 10519, + " throw", + -1.3915133 + ], + [ + 2084, + " new", + -0.042013377 + ], + [ + 150858, + " RuntimeException", + -1.7330077 + ] + ] + }, + "generated_text": "Test request.get(\"action\");\n if (action == null) {\n throw new RuntimeException" + } +] \ No newline at end of file diff --git a/launcher/tests/integration_tests.rs b/launcher/tests/integration_tests.rs new file mode 100644 index 00000000..3e68f6be --- /dev/null +++ b/launcher/tests/integration_tests.rs @@ -0,0 +1,156 @@ +use std::fs::File; +use serde_json::Value; +use std::io::{BufRead, BufReader}; +use std::path::PathBuf; +use std::thread; +use std::thread::sleep; +use std::time::Duration; +use float_eq::assert_float_eq; +use subprocess::{Popen, PopenConfig, Redirection}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Details { + finish_reason: String, + generated_tokens: u32, + tokens: Vec<(u32, String, Option)>, +} + +#[derive(Deserialize)] +struct GeneratedText { + generated_text: String, + details: Details, +} + + +fn start_launcher(model_name: String, num_shard: usize, port: usize, master_port: usize) -> Popen { + let argv = vec![ + "text-generation-launcher".to_string(), + "--model-name".to_string(), + model_name.clone(), + "--num-shard".to_string(), + num_shard.to_string(), + "--port".to_string(), + port.to_string(), + "--master-port".to_string(), + master_port.to_string(), + "--shard-uds-path".to_string(), + format!("/tmp/test-{}-{}-{}", num_shard, port, master_port), + ]; + + let mut launcher = Popen::create( + &argv, + PopenConfig { + stdout: Redirection::Pipe, + stderr: Redirection::Pipe, + ..Default::default() + }, + ) + .expect("Could not start launcher"); + + // Redirect STDOUT and STDERR to the console + let launcher_stdout = launcher.stdout.take().unwrap(); + let launcher_stderr = launcher.stderr.take().unwrap(); + + thread::spawn(move || { + let stdout = BufReader::new(launcher_stdout); + let stderr = BufReader::new(launcher_stderr); + for line in stdout.lines() { + println!("{}", line.unwrap()); + } + for line in stderr.lines() { + println!("{}", line.unwrap()); + } + }); + + for _ in 0..30 { + let health = reqwest::blocking::get(format!("http://localhost:{}/health", port)); + if health.is_ok() { + return launcher; + } + sleep(Duration::from_secs(2)); + } + + launcher.terminate().unwrap(); + launcher.wait().unwrap(); + panic!("failed to launch {}", model_name) +} + +fn test_model(model_name: String, num_shard: usize, port: usize, master_port: usize) -> GeneratedText { + let mut launcher = start_launcher(model_name, num_shard, port, master_port); + + let data = r#" + { + "inputs": "Test request", + "parameters": { + "details": true + } + }"#; + let req: Value = serde_json::from_str(data).unwrap(); + + let client = reqwest::blocking::Client::new(); + let res = client + .post(format!("http://localhost:{}/generate", port)) + .json(&req) + .send(); + + launcher.terminate().unwrap(); + launcher.wait().unwrap(); + + let mut results: Vec = res.unwrap().json().unwrap(); + results.pop().unwrap() +} + + +fn read_json(name: &str) -> GeneratedText { + let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + d.push("tests/"); + d.push(name); + + let file = File::open(d).unwrap(); + let reader = BufReader::new(file); + + let mut results: Vec = serde_json::from_reader(reader).unwrap(); + results.pop().unwrap() +} + +fn compare_results(result: GeneratedText, expected: GeneratedText) { + assert_eq!(result.generated_text, expected.generated_text); + assert_eq!(result.details.finish_reason, expected.details.finish_reason); + assert_eq!(result.details.generated_tokens, expected.details.generated_tokens); + + for (token, expected_token) in result.details.tokens.into_iter().zip(expected.details.tokens.into_iter()) { + assert_eq!(token.0, expected_token.0); + assert_eq!(token.1, expected_token.1); + if let Some(logprob) = token.2 { + let expected_logprob = expected_token.2.unwrap(); + assert_float_eq!(logprob, expected_logprob, abs <= 0.001); + } else { + assert_eq!(token.2, expected_token.2); + } + } +} + +#[test] +fn test_bloom_560m() { + let expected = read_json("bloom_560m.json"); + + let result = test_model("bigscience/bloom-560m".to_string(), 1, 3000, 29500); + compare_results(result, expected); +} + +#[test] +fn test_bloom_560m_distributed() { + let expected = read_json("bloom_560m.json"); + + let result = test_model("bigscience/bloom-560m".to_string(), 2, 3001, 29501); + compare_results(result, expected); +} + +#[test] +fn test_mt0_base() { + let expected = read_json("mt0_base.json"); + + let result = test_model("bigscience/mt0-base".to_string(), 1, 3002, 29502); + compare_results(result, expected); +} diff --git a/launcher/tests/mt0_base.json b/launcher/tests/mt0_base.json new file mode 100644 index 00000000..1b772282 --- /dev/null +++ b/launcher/tests/mt0_base.json @@ -0,0 +1,116 @@ +[ + { + "details": { + "finish_reason": "length", + "generated_tokens": 20, + "tokens": [ + [ + 0, + "", + null + ], + [ + 259, + "", + -1.3656927 + ], + [ + 215100, + "\"\"\"", + -2.6551573 + ], + [ + 46138, + "Test", + -1.8059857 + ], + [ + 287, + "the", + -1.2102449 + ], + [ + 259, + "", + -1.6057279 + ], + [ + 49076, + "contents", + -3.6060903 + ], + [ + 304, + "of", + -0.5270343 + ], + [ + 287, + "the", + -0.62522805 + ], + [ + 259, + "", + -1.4069618 + ], + [ + 49076, + "contents", + -2.621994 + ], + [ + 304, + "of", + -1.3172221 + ], + [ + 287, + "the", + -0.3501925 + ], + [ + 259, + "", + -0.7219573 + ], + [ + 49076, + "contents", + -1.0494149 + ], + [ + 260, + ".", + -1.0803378 + ], + [ + 259, + "", + -0.32933083 + ], + [ + 215100, + "\"\"\"", + -0.11268901 + ], + [ + 2978, + "test", + -1.5846587 + ], + [ + 290, + "_", + -0.49796978 + ], + [ + 4125, + "test", + -2.0026445 + ] + ] + }, + "generated_text": "\"\"\"Test the contents of the contents of the contents. \"\"\" test_test" + } +] \ No newline at end of file From 611e21cb137e08b0b9a35c76676eb46de9d30627 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 16 Dec 2022 16:03:39 +0100 Subject: [PATCH 039/793] fix(server): Fix stop sequences (#11) --- launcher/tests/integration_tests.rs | 33 ++++++++----- server/tests/test_utils.py | 51 +++++++------------ server/text_generation/models/causal_lm.py | 7 ++- server/text_generation/models/seq2seq_lm.py | 7 ++- server/text_generation/utils.py | 55 +++++++++------------ 5 files changed, 77 insertions(+), 76 deletions(-) diff --git a/launcher/tests/integration_tests.rs b/launcher/tests/integration_tests.rs index 3e68f6be..9f0416c0 100644 --- a/launcher/tests/integration_tests.rs +++ b/launcher/tests/integration_tests.rs @@ -1,13 +1,13 @@ -use std::fs::File; +use float_eq::assert_float_eq; +use serde::Deserialize; use serde_json::Value; +use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::PathBuf; use std::thread; use std::thread::sleep; use std::time::Duration; -use float_eq::assert_float_eq; use subprocess::{Popen, PopenConfig, Redirection}; -use serde::Deserialize; #[derive(Deserialize)] struct Details { @@ -22,7 +22,6 @@ struct GeneratedText { details: Details, } - fn start_launcher(model_name: String, num_shard: usize, port: usize, master_port: usize) -> Popen { let argv = vec![ "text-generation-launcher".to_string(), @@ -46,7 +45,7 @@ fn start_launcher(model_name: String, num_shard: usize, port: usize, master_port ..Default::default() }, ) - .expect("Could not start launcher"); + .expect("Could not start launcher"); // Redirect STDOUT and STDERR to the console let launcher_stdout = launcher.stdout.take().unwrap(); @@ -63,7 +62,7 @@ fn start_launcher(model_name: String, num_shard: usize, port: usize, master_port } }); - for _ in 0..30 { + for _ in 0..60 { let health = reqwest::blocking::get(format!("http://localhost:{}/health", port)); if health.is_ok() { return launcher; @@ -76,7 +75,12 @@ fn start_launcher(model_name: String, num_shard: usize, port: usize, master_port panic!("failed to launch {}", model_name) } -fn test_model(model_name: String, num_shard: usize, port: usize, master_port: usize) -> GeneratedText { +fn test_model( + model_name: String, + num_shard: usize, + port: usize, + master_port: usize, +) -> GeneratedText { let mut launcher = start_launcher(model_name, num_shard, port, master_port); let data = r#" @@ -101,7 +105,6 @@ fn test_model(model_name: String, num_shard: usize, port: usize, master_port: us results.pop().unwrap() } - fn read_json(name: &str) -> GeneratedText { let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); d.push("tests/"); @@ -117,9 +120,17 @@ fn read_json(name: &str) -> GeneratedText { fn compare_results(result: GeneratedText, expected: GeneratedText) { assert_eq!(result.generated_text, expected.generated_text); assert_eq!(result.details.finish_reason, expected.details.finish_reason); - assert_eq!(result.details.generated_tokens, expected.details.generated_tokens); - - for (token, expected_token) in result.details.tokens.into_iter().zip(expected.details.tokens.into_iter()) { + assert_eq!( + result.details.generated_tokens, + expected.details.generated_tokens + ); + + for (token, expected_token) in result + .details + .tokens + .into_iter() + .zip(expected.details.tokens.into_iter()) + { assert_eq!(token.0, expected_token.0); assert_eq!(token.1, expected_token.1); if let Some(logprob) = token.2 { diff --git a/server/tests/test_utils.py b/server/tests/test_utils.py index ed6e896a..643cb834 100644 --- a/server/tests/test_utils.py +++ b/server/tests/test_utils.py @@ -11,46 +11,33 @@ def test_stop_sequence_criteria(): - criteria = StopSequenceCriteria([1, 2, 3]) + criteria = StopSequenceCriteria("/test;") - assert not criteria(1) - assert criteria.current_token_idx == 1 - assert not criteria(2) - assert criteria.current_token_idx == 2 - assert criteria(3) - assert criteria.current_token_idx == 3 + assert not criteria("/") + assert not criteria("/test") + assert criteria("/test;") + assert not criteria("/test; ") -def test_stop_sequence_criteria_reset(): - criteria = StopSequenceCriteria([1, 2, 3]) - - assert not criteria(1) - assert criteria.current_token_idx == 1 - assert not criteria(2) - assert criteria.current_token_idx == 2 - assert not criteria(4) - assert criteria.current_token_idx == 0 - - -def test_stop_sequence_criteria_empty(): - with pytest.raises(ValueError): - StopSequenceCriteria([]) +def test_stopping_criteria(): + criteria = StoppingCriteria(0, [StopSequenceCriteria("/test;")], max_new_tokens=5) + assert criteria(65827, "/test") == (False, None) + assert criteria(30, ";") == (True, "stop_sequence") -def test_stopping_criteria(): - criteria = StoppingCriteria([StopSequenceCriteria([1, 2, 3])], max_new_tokens=5) - assert criteria([1]) == (False, None) - assert criteria([1, 2]) == (False, None) - assert criteria([1, 2, 3]) == (True, "stop_sequence") +def test_stopping_criteria_eos(): + criteria = StoppingCriteria(0, [StopSequenceCriteria("/test;")], max_new_tokens=5) + assert criteria(1, "") == (False, None) + assert criteria(0, "") == (True, "eos_token") def test_stopping_criteria_max(): - criteria = StoppingCriteria([StopSequenceCriteria([1, 2, 3])], max_new_tokens=5) - assert criteria([1]) == (False, None) - assert criteria([1, 1]) == (False, None) - assert criteria([1, 1, 1]) == (False, None) - assert criteria([1, 1, 1, 1]) == (False, None) - assert criteria([1, 1, 1, 1, 1]) == (True, "length") + criteria = StoppingCriteria(0, [StopSequenceCriteria("/test;")], max_new_tokens=5) + assert criteria(1, "") == (False, None) + assert criteria(1, "") == (False, None) + assert criteria(1, "") == (False, None) + assert criteria(1, "") == (False, None) + assert criteria(1, "") == (True, "length") def test_weight_hub_files(): diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 5d62137f..aeecf127 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -345,7 +345,12 @@ def generate_token( all_logprobs = torch.cat([all_logprobs, next_token_logprob]) # Evaluate stopping criteria - stop, reason = stopping_criteria(all_input_ids) + stop, reason = stopping_criteria( + next_token.squeeze(), + self.tokenizer.decode( + next_token.squeeze(), clean_up_tokenization_spaces=False + ), + ) if stop: # Decode all tokens output_text = self.tokenizer.decode( diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index e51ce60b..fc80c60c 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -441,7 +441,12 @@ def generate_token( decoder_logprobs = torch.cat([decoder_logprobs, next_token_logprob]) # Evaluate stopping criteria - stop, reason = stopping_criteria(decoder_input_ids) + stop, reason = stopping_criteria( + next_token.squeeze(), + self.tokenizer.decode( + next_token.squeeze(), clean_up_tokenization_spaces=False + ), + ) if stop: # Slice with decoder_input_length to remove padding # Decode all tokens diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 9dd91151..cc779736 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -1,5 +1,6 @@ import concurrent import os +import re import torch import torch.distributed @@ -74,43 +75,39 @@ def from_pb(cls, pb: generate_pb2.NextTokenChooserParameters) -> "NextTokenChoos class StopSequenceCriteria: - def __init__(self, tokens: List[int]): - if not tokens: - raise ValueError("tokens cannot be empty") - - self.tokens = tokens - self.current_token_idx = 0 - - def __call__(self, last_token: int) -> bool: - if last_token == self.tokens[self.current_token_idx]: - # Increase idx to go to next token - self.current_token_idx += 1 - else: - # Reset to first token of the stopping sequence - self.current_token_idx = 0 - - if self.current_token_idx == len(self.tokens): - # We matched the entire sequence without resetting + def __init__(self, stop_sequence: str): + self.regex = re.compile(f".*{stop_sequence}$") + + def __call__(self, output: str) -> bool: + if self.regex.findall(output): return True return False class StoppingCriteria: def __init__( - self, stop_sequence_criterias: List[StopSequenceCriteria], max_new_tokens=20 + self, + eos_token_id: int, + stop_sequence_criterias: List[StopSequenceCriteria], + max_new_tokens=20, ): + self.eos_token_id = eos_token_id self.stop_sequence_criterias = stop_sequence_criterias self.max_new_tokens = max_new_tokens self.current_tokens = 0 + self.current_output = "" - def __call__(self, all_ids) -> Tuple[bool, Optional[str]]: + def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[str]]: self.current_tokens += 1 if self.current_tokens >= self.max_new_tokens: return True, "length" - last_token = all_ids[-1] + if last_token == self.eos_token_id: + return True, "eos_token" + + self.current_output += last_output for stop_sequence_criteria in self.stop_sequence_criterias: - if stop_sequence_criteria(last_token): + if stop_sequence_criteria(self.current_output): return True, "stop_sequence" return False, None @@ -119,16 +116,12 @@ def __call__(self, all_ids) -> Tuple[bool, Optional[str]]: def from_pb( cls, pb: generate_pb2.StoppingCriteriaParameters, tokenizer: AutoTokenizer ) -> "StoppingCriteria": - stop_sequence_criterias = [] - for stop_sequence in pb.stop_sequences: - tokens = tokenizer( - stop_sequence, padding=False, return_attention_mask=False - ).input_ids - if tokens: - stop_sequence_criterias.append(StopSequenceCriteria(tokens)) - stop_sequence_criterias.append(StopSequenceCriteria([tokenizer.eos_token_id])) - - return StoppingCriteria(stop_sequence_criterias, pb.max_new_tokens) + stop_sequence_criterias = [ + StopSequenceCriteria(sequence) for sequence in pb.stop_sequences + ] + return StoppingCriteria( + tokenizer.eos_token_id, stop_sequence_criterias, pb.max_new_tokens + ) def initialize_torch_distributed(): From 686cc6671705c666b767fffe71b2ed9c9b6fccd1 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Fri, 30 Dec 2022 10:30:42 -0800 Subject: [PATCH 040/793] fix(server): Check for device type correctly when determining initial padding (#16) AFAIK there is no torch device type called "gpu". --- server/text_generation/models/causal_lm.py | 2 +- server/text_generation/models/seq2seq_lm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index aeecf127..3ad36219 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -65,7 +65,7 @@ def from_pb( ) all_logprobs.append(None) - pad_to_multiple_of = 8 if "gpu" in str(device) else None + pad_to_multiple_of = 8 if device.type == "cuda" else None tokenized_inputs = tokenizer( inputs, return_tensors="pt", diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index fc80c60c..4095db92 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -77,7 +77,7 @@ def from_pb( decoder_logprobs.append(None) # Tokenize batch - pad_to_multiple_of = 8 if "gpu" in str(device) else None + pad_to_multiple_of = 8 if device.type == "cuda" else None tokenized_inputs = tokenizer( inputs, return_tensors="pt", From 3efa5bbbfd5868695da4d5d9ad23d81f48f1e5a8 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Fri, 30 Dec 2022 10:31:44 -0800 Subject: [PATCH 041/793] fix(router): Include special tokens when tokenizing (#14) There's currently a discrepancy in the tokenization between the router and python server code. The latter includes special tokens but former does not. This results in a token count mismatch for seq2seq models such as mt0 where the tokenizer emits an EOS token at the end. This in turn results in some unexpected/incorrect output, in particular when batch concatenation is involved, because the python code uses the input length passed from the router for each row. As far as I can tell, it is better to include this token in the encoder `input_ids`, so I guess it's best to just adjust on the router side. --- router/src/validation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/src/validation.rs b/router/src/validation.rs index ff659b3e..f6da1913 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -131,7 +131,7 @@ fn validation_worker( } // Get the number of tokens in the input - match tokenizer.encode(request.inputs.clone(), false) { + match tokenizer.encode(request.inputs.clone(), true) { Ok(inputs) => { let input_length = inputs.len(); From 60472f9d2b03822e9e771d9a4a4885e2496c5537 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 3 Jan 2023 01:41:22 -0800 Subject: [PATCH 042/793] feat(router): Add const parameters to validation logic (#15) I noticed some opportunity to collapse some of the logic, in case you are interested. --- router/src/db.rs | 12 ++--- router/src/validation.rs | 113 +++++++++++++++++---------------------- 2 files changed, 53 insertions(+), 72 deletions(-) diff --git a/router/src/db.rs b/router/src/db.rs index df9f2b8e..1d7df627 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -71,12 +71,8 @@ impl State { id: *id, inputs: entry.request.inputs.clone(), input_length: entry.input_length as u32, - parameters: Some(NextTokenChooserParameters::from( - entry.request.parameters.clone(), - )), - stopping_parameters: Some(StoppingCriteriaParameters::from( - entry.request.parameters.clone(), - )), + parameters: Some((&entry.request.parameters).into()), + stopping_parameters: Some(entry.request.parameters.clone().into()), }); ids.push(*id); @@ -162,8 +158,8 @@ impl Db { } } -impl From for NextTokenChooserParameters { - fn from(parameters: GenerateParameters) -> Self { +impl From<&GenerateParameters> for NextTokenChooserParameters { + fn from(parameters: &GenerateParameters) -> Self { Self { temperature: parameters.temperature, top_k: parameters.top_k as u32, diff --git a/router/src/validation.rs b/router/src/validation.rs index f6da1913..4a9d0c23 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -4,12 +4,11 @@ use axum::http::StatusCode; use axum::Json; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; -use tokenizers::{ - DecoderWrapper, ModelWrapper, NormalizerWrapper, PostProcessorWrapper, PreTokenizerWrapper, - TokenizerImpl, -}; use tokio::sync::{mpsc, oneshot}; +const MAX_MAX_NEW_TOKENS: u32 = 512; +const MAX_STOP_SEQUENCES: usize = 4; + /// Validation #[derive(Debug, Clone)] pub struct Validation { @@ -63,7 +62,7 @@ async fn validation_task( // Create workers for _ in 0..workers { - let tokenizer_clone = tokenizer.clone(); + let tokenizer_clone: Tokenizer = tokenizer.clone().into(); // Create channel to communicate with worker let (worker_sender, worker_receiver) = mpsc::channel(workers); workers_senders.push(worker_sender); @@ -89,68 +88,54 @@ async fn validation_task( /// Check the parameters inside the payload and get the number of tokens inside the input using /// the tokenizer fn validation_worker( - tokenizer: TokenizerImpl< - ModelWrapper, - NormalizerWrapper, - PreTokenizerWrapper, - PostProcessorWrapper, - DecoderWrapper, - >, + tokenizer: Tokenizer, max_input_length: usize, mut receiver: mpsc::Receiver, ) { // Loop over requests while let Some((request, response_tx)) = receiver.blocking_recv() { - if request.parameters.temperature <= 0.0 { - response_tx - .send(Err(ValidationError::Temperature)) - .unwrap_or(()); - continue; - } - if request.parameters.top_p <= 0.0 || request.parameters.top_p > 1.0 { - response_tx.send(Err(ValidationError::TopP)).unwrap_or(()); - continue; - } - if request.parameters.top_k < 0 { - response_tx.send(Err(ValidationError::TopK)).unwrap_or(()); - continue; - } - if request.parameters.max_new_tokens > 512 { - response_tx - .send(Err(ValidationError::MaxNewTokens)) - .unwrap_or(()); - continue; - } - if request.parameters.stop.len() > 4 { - response_tx - .send(Err(ValidationError::StopSequence( - request.parameters.stop.len(), - ))) - .unwrap_or(()); - continue; - } + response_tx.send(validate(request, &tokenizer, max_input_length)).unwrap_or(()) + } +} - // Get the number of tokens in the input - match tokenizer.encode(request.inputs.clone(), true) { - Ok(inputs) => { - let input_length = inputs.len(); - - if input_length > max_input_length { - response_tx - .send(Err(ValidationError::InputLength( - input_length, - max_input_length, - ))) - .unwrap_or(()); - continue; - } - - response_tx.send(Ok((input_length, request))).unwrap_or(()); +fn validate( + request: GenerateRequest, + tokenizer: &Tokenizer, + max_input_length: usize, +) -> Result<(usize, GenerateRequest), ValidationError> { + if request.parameters.temperature <= 0.0 { + return Err(ValidationError::Temperature); + } + if request.parameters.top_p <= 0.0 || request.parameters.top_p > 1.0 { + return Err(ValidationError::TopP); + } + if request.parameters.top_k < 0 { + return Err(ValidationError::TopK); + } + if request.parameters.max_new_tokens > MAX_MAX_NEW_TOKENS { + return Err(ValidationError::MaxNewTokens(MAX_MAX_NEW_TOKENS)); + } + if request.parameters.stop.len() > MAX_STOP_SEQUENCES { + return Err(ValidationError::StopSequence( + MAX_STOP_SEQUENCES, request.parameters.stop.len(), + )) + } + + // Get the number of tokens in the input + match tokenizer.encode(request.inputs.clone(), true) { + Ok(inputs) => { + let input_length = inputs.len(); + + if input_length > max_input_length { + Err(ValidationError::InputLength( + input_length, + max_input_length, + )) + } else { + Ok((input_length, request)) } - Err(err) => response_tx - .send(Err(ValidationError::Tokenizer(err.to_string()))) - .unwrap_or(()), - }; + }, + Err(err) => Err(ValidationError::Tokenizer(err.to_string())), } } @@ -167,12 +152,12 @@ pub enum ValidationError { TopP, #[error("top_k must be strictly positive")] TopK, - #[error("max_new_tokens must be <= 512")] - MaxNewTokens, + #[error("max_new_tokens must be <= {0}")] + MaxNewTokens(u32), #[error("inputs must have less than {1} tokens. Given: {0}")] InputLength(usize, usize), - #[error("stop supports up to 4 stop sequences. Given: {0}")] - StopSequence(usize), + #[error("stop supports up to {0} stop sequences. Given: {1}")] + StopSequence(usize, usize), #[error("tokenizer error {0}")] Tokenizer(String), } From b94f30215fab97c6ecbff2f647f911fa6ac38bd6 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 3 Jan 2023 11:07:05 +0100 Subject: [PATCH 043/793] fix(server): Use cleanup_tokenization_spaces=False for lossless decoding (#13) Fixes #12 in the easiest way I could think of. --- server/text_generation/models/causal_lm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 3ad36219..b352eb6b 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -354,7 +354,8 @@ def generate_token( if stop: # Decode all tokens output_text = self.tokenizer.decode( - all_input_ids.squeeze(-1), skip_special_tokens=True + all_input_ids.squeeze(-1), skip_special_tokens=True, + cleanup_tokenization_spaces=False ) # Slice with input_length to remove padding token_ids = all_input_ids[-new_input_length:] From fcc2c5fcbf1bff5cdbba1508818650db13982ed2 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 5 Jan 2023 12:01:23 +0100 Subject: [PATCH 044/793] feat(launcher): Log server stdout (#19) Co-authored-by: Nick Hill --- launcher/Cargo.toml | 2 +- launcher/src/main.rs | 22 + launcher/tests/integration_tests.rs | 8 +- server/poetry.lock | 728 +++++++++++++--------- server/pyproject.toml | 7 +- server/text_generation/cli.py | 15 + server/text_generation/interceptor.py | 29 + server/text_generation/models/__init__.py | 2 +- server/text_generation/server.py | 8 +- 9 files changed, 512 insertions(+), 309 deletions(-) create mode 100644 server/text_generation/interceptor.py diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index ecdef831..21d5d3ee 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -8,6 +8,7 @@ description = "Text Generation Launcher" [dependencies] clap = { version = "4.0.15", features = ["derive", "env"] } ctrlc = { version = "3.2.3", features = ["termination"] } +serde_json = "1.0.89" subprocess = "0.2.9" tracing = "0.1.37" tracing-subscriber = { version = "0.3.16", features = ["json"] } @@ -16,4 +17,3 @@ tracing-subscriber = { version = "0.3.16", features = ["json"] } float_eq = "1.0.1" reqwest = { version = "0.11.13", features = ["blocking", "json"] } serde = "1.0.150" -serde_json = "1.0.89" diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 016a28eb..86335875 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -11,6 +11,7 @@ use std::thread; use std::thread::sleep; use std::time::{Duration, Instant}; use std::{fs, io}; +use serde_json::Value; use subprocess::{Popen, PopenConfig, PopenError, Redirection}; /// App Configuration @@ -274,6 +275,9 @@ fn shard_manager( model_name, "--uds-path".to_string(), uds_path, + "--logger-level".to_string(), + "ERROR".to_string(), + "--json-output".to_string(), ]; if world_size > 1 { @@ -347,6 +351,24 @@ fn shard_manager( } }; + // Redirect STDOUT to the console + let shard_stdout = p.stdout.take().unwrap(); + + thread::spawn(move || { + // Enter shard-manager tracing span + let stdout = BufReader::new(shard_stdout); + let _span = tracing::span!(tracing::Level::INFO, "shard-manager", rank = rank).entered(); + for line in stdout.lines() { + // Parse loguru logs + if let Ok(value) = serde_json::from_str::(&line.unwrap()) { + if let Some(text) = value.get("text") { + // Format escaped newlines + tracing::error!("{}", text.to_string().replace("\\n", "\n")); + } + } + } + }); + let mut ready = false; let start_time = Instant::now(); let mut wait_time = Instant::now(); diff --git a/launcher/tests/integration_tests.rs b/launcher/tests/integration_tests.rs index 9f0416c0..6b270659 100644 --- a/launcher/tests/integration_tests.rs +++ b/launcher/tests/integration_tests.rs @@ -41,25 +41,21 @@ fn start_launcher(model_name: String, num_shard: usize, port: usize, master_port &argv, PopenConfig { stdout: Redirection::Pipe, - stderr: Redirection::Pipe, + stderr: Redirection::Merge, ..Default::default() }, ) .expect("Could not start launcher"); // Redirect STDOUT and STDERR to the console + // (STDERR is merged into STDOUT) let launcher_stdout = launcher.stdout.take().unwrap(); - let launcher_stderr = launcher.stderr.take().unwrap(); thread::spawn(move || { let stdout = BufReader::new(launcher_stdout); - let stderr = BufReader::new(launcher_stderr); for line in stdout.lines() { println!("{}", line.unwrap()); } - for line in stderr.lines() { - println!("{}", line.unwrap()); - } }); for _ in 0..60 { diff --git a/server/poetry.lock b/server/poetry.lock index c141612e..d610fcdf 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -24,21 +24,23 @@ testing = ["datasets", "deepspeed (<0.7.0)", "evaluate", "parameterized", "pytes [[package]] name = "attrs" -version = "22.1.0" +version = "22.2.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] +tests = ["attrs[tests-no-zope]", "zope.interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests_no_zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] [[package]] name = "bitsandbytes" -version = "0.35.1" +version = "0.35.4" description = "8-bit optimizers and matrix multiplication routines." category = "main" optional = false @@ -57,15 +59,15 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.5" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" [[package]] name = "exceptiongroup" -version = "1.0.4" +version = "1.1.0" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false @@ -75,41 +77,79 @@ python-versions = ">=3.7" test = ["pytest (>=6)"] [[package]] -name = "grpcio" -version = "1.50.0" -description = "HTTP/2-based RPC framework" +name = "googleapis-common-protos" +version = "1.57.0" +description = "Common protobufs used in Google APIs" category = "main" optional = false python-versions = ">=3.7" [package.dependencies] -six = ">=1.5.2" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] + +[[package]] +name = "grpc-interceptor" +version = "0.15.0" +description = "Simplifies gRPC interceptors" +category = "main" +optional = false +python-versions = ">=3.6.1,<4.0.0" + +[package.dependencies] +grpcio = ">=1.32.0,<2.0.0" + +[package.extras] +testing = ["protobuf (>=3.6.0)"] + +[[package]] +name = "grpcio" +version = "1.51.1" +description = "HTTP/2-based RPC framework" +category = "main" +optional = false +python-versions = ">=3.7" [package.extras] -protobuf = ["grpcio-tools (>=1.50.0)"] +protobuf = ["grpcio-tools (>=1.51.1)"] [[package]] name = "grpcio-reflection" -version = "1.50.0" +version = "1.51.1" description = "Standard Protobuf Reflection Service for gRPC" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -grpcio = ">=1.50.0" +grpcio = ">=1.51.1" +protobuf = ">=4.21.6" + +[[package]] +name = "grpcio-status" +version = "1.51.1" +description = "Status proto mapping for gRPC" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.51.1" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.50.0" +version = "1.51.1" description = "Protobuf code generator for gRPC" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -grpcio = ">=1.50.0" +grpcio = ">=1.51.1" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -121,24 +161,84 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "loguru" +version = "0.6.0" +description = "Python logging made (stupidly) simple" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} +win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} + +[package.extras] +dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] + [[package]] name = "numpy" -version = "1.23.4" -description = "NumPy is the fundamental package for array computing with Python." +version = "1.24.1" +description = "Fundamental package for array computing in Python" category = "main" optional = false python-versions = ">=3.8" [[package]] -name = "packaging" -version = "21.3" -description = "Core utilities for Python packages" +name = "nvidia-cublas-cu11" +version = "11.10.3.66" +description = "CUBLAS native runtime libraries" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3" [package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +setuptools = "*" +wheel = "*" + +[[package]] +name = "nvidia-cuda-nvrtc-cu11" +version = "11.7.99" +description = "NVRTC native runtime libraries" +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + +[[package]] +name = "nvidia-cuda-runtime-cu11" +version = "11.7.99" +description = "CUDA Runtime native Libraries" +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + +[[package]] +name = "nvidia-cudnn-cu11" +version = "8.5.0.96" +description = "cuDNN runtime libraries" +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + +[[package]] +name = "packaging" +version = "22.0" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.7" [[package]] name = "pluggy" @@ -154,7 +254,7 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.21.8" +version = "4.21.12" description = "" category = "main" optional = false @@ -162,7 +262,7 @@ python-versions = ">=3.7" [[package]] name = "psutil" -version = "5.9.3" +version = "5.9.4" description = "Cross-platform lib for process and system monitoring in Python." category = "main" optional = false @@ -171,17 +271,6 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - [[package]] name = "pytest" version = "7.2.0" @@ -212,37 +301,35 @@ python-versions = ">=3.6" [[package]] name = "safetensors" -version = "0.2.4" +version = "0.2.7" description = "Fast and Safe Tensor serialization" category = "main" optional = false python-versions = "*" [package.extras] -dev = ["black (==22.3)", "flake8 (>=3.8.3)", "huggingface-hub", "isort (>=5.5.4)", "numpy", "pytest", "setuptools-rust"] -testing = ["black (==22.3)", "flake8 (>=3.8.3)", "huggingface-hub", "isort (>=5.5.4)", "numpy", "pytest", "setuptools-rust"] +all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax", "h5py", "huggingface-hub", "isort (>=5.5.4)", "jax", "numpy", "pytest", "pytest-benchmark", "setuptools-rust", "tensorflow", "torch"] +dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax", "h5py", "huggingface-hub", "isort (>=5.5.4)", "jax", "numpy", "pytest", "pytest-benchmark", "setuptools-rust", "tensorflow", "torch"] +jax = ["flax", "jax"] +numpy = ["numpy"] +quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] +tensorflow = ["tensorflow"] +testing = ["h5py", "huggingface-hub", "numpy", "pytest", "pytest-benchmark", "setuptools-rust"] +torch = ["torch"] [[package]] name = "setuptools" -version = "65.5.0" +version = "65.6.3" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - [[package]] name = "tomli" version = "2.0.1" @@ -253,15 +340,22 @@ python-versions = ">=3.7" [[package]] name = "torch" -version = "1.12.1" +version = "1.13.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" category = "main" optional = false python-versions = ">=3.7.0" [package.dependencies] +nvidia-cublas-cu11 = {version = "11.10.3.66", markers = "platform_system == \"Linux\""} +nvidia-cuda-nvrtc-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""} +nvidia-cuda-runtime-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""} +nvidia-cudnn-cu11 = {version = "8.5.0.96", markers = "platform_system == \"Linux\""} typing-extensions = "*" +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] + [[package]] name = "typer" version = "0.6.1" @@ -287,13 +381,35 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "wheel" +version = "0.38.4" +description = "A built-package format for Python" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=3.0.0)"] + +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + [extras] bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "51693654531e3229ac64bee250932ace20a60e8d45af074ae7b860ed32b25ef8" +content-hash = "bc59a70e1e112ce2e173289c0c5285121550511236d4866f0053ae18f90c98aa" [metadata.files] accelerate = [ @@ -301,222 +417,229 @@ accelerate = [ {file = "accelerate-0.12.0.tar.gz", hash = "sha256:e8b119c94fac31877620d5f9de311164ec81fa9dc9e175f0d0d4f50fc8d79473"}, ] attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, ] bitsandbytes = [ - {file = "bitsandbytes-0.35.1-py3-none-any.whl", hash = "sha256:4506a9e3778359a743938aa5592d8d043fa91d1df66cd01ba8cc6486e64dea45"}, - {file = "bitsandbytes-0.35.1.tar.gz", hash = "sha256:63a6f59c87b713a731a685e43d68c19789ee6381e62196cafab293b87eca5d46"}, + {file = "bitsandbytes-0.35.4-py3-none-any.whl", hash = "sha256:201f168538ccfbd7594568a2f86c149cec8352782301076a15a783695ecec7fb"}, + {file = "bitsandbytes-0.35.4.tar.gz", hash = "sha256:b23db6b91cd73cb14faf9841a66bffa5c1722f9b8b57039ef2fb461ac22dd2a6"}, ] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] exceptiongroup = [ - {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, - {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, + {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, + {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, +] +googleapis-common-protos = [ + {file = "googleapis-common-protos-1.57.0.tar.gz", hash = "sha256:27a849d6205838fb6cc3c1c21cb9800707a661bb21c6ce7fb13e99eb1f8a0c46"}, + {file = "googleapis_common_protos-1.57.0-py2.py3-none-any.whl", hash = "sha256:a9f4a1d7f6d9809657b7f1316a1aa527f6664891531bcfcc13b6696e685f443c"}, +] +grpc-interceptor = [ + {file = "grpc-interceptor-0.15.0.tar.gz", hash = "sha256:5c1aa9680b1d7e12259960c38057b121826860b05ebbc1001c74343b7ad1455e"}, + {file = "grpc_interceptor-0.15.0-py3-none-any.whl", hash = "sha256:63e390162e64df96c39c40508eb697def76a7cafac32a7eaf9272093eec1109e"}, ] grpcio = [ - {file = "grpcio-1.50.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:906f4d1beb83b3496be91684c47a5d870ee628715227d5d7c54b04a8de802974"}, - {file = "grpcio-1.50.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:2d9fd6e38b16c4d286a01e1776fdf6c7a4123d99ae8d6b3f0b4a03a34bf6ce45"}, - {file = "grpcio-1.50.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:4b123fbb7a777a2fedec684ca0b723d85e1d2379b6032a9a9b7851829ed3ca9a"}, - {file = "grpcio-1.50.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2f77a90ba7b85bfb31329f8eab9d9540da2cf8a302128fb1241d7ea239a5469"}, - {file = "grpcio-1.50.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eea18a878cffc804506d39c6682d71f6b42ec1c151d21865a95fae743fda500"}, - {file = "grpcio-1.50.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b71916fa8f9eb2abd93151fafe12e18cebb302686b924bd4ec39266211da525"}, - {file = "grpcio-1.50.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:95ce51f7a09491fb3da8cf3935005bff19983b77c4e9437ef77235d787b06842"}, - {file = "grpcio-1.50.0-cp310-cp310-win32.whl", hash = "sha256:f7025930039a011ed7d7e7ef95a1cb5f516e23c5a6ecc7947259b67bea8e06ca"}, - {file = "grpcio-1.50.0-cp310-cp310-win_amd64.whl", hash = "sha256:05f7c248e440f538aaad13eee78ef35f0541e73498dd6f832fe284542ac4b298"}, - {file = "grpcio-1.50.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:ca8a2254ab88482936ce941485c1c20cdeaef0efa71a61dbad171ab6758ec998"}, - {file = "grpcio-1.50.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:3b611b3de3dfd2c47549ca01abfa9bbb95937eb0ea546ea1d762a335739887be"}, - {file = "grpcio-1.50.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a4cd8cb09d1bc70b3ea37802be484c5ae5a576108bad14728f2516279165dd7"}, - {file = "grpcio-1.50.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:156f8009e36780fab48c979c5605eda646065d4695deea4cfcbcfdd06627ddb6"}, - {file = "grpcio-1.50.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de411d2b030134b642c092e986d21aefb9d26a28bf5a18c47dd08ded411a3bc5"}, - {file = "grpcio-1.50.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d144ad10eeca4c1d1ce930faa105899f86f5d99cecfe0d7224f3c4c76265c15e"}, - {file = "grpcio-1.50.0-cp311-cp311-win32.whl", hash = "sha256:92d7635d1059d40d2ec29c8bf5ec58900120b3ce5150ef7414119430a4b2dd5c"}, - {file = "grpcio-1.50.0-cp311-cp311-win_amd64.whl", hash = "sha256:ce8513aee0af9c159319692bfbf488b718d1793d764798c3d5cff827a09e25ef"}, - {file = "grpcio-1.50.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:8e8999a097ad89b30d584c034929f7c0be280cd7851ac23e9067111167dcbf55"}, - {file = "grpcio-1.50.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:a50a1be449b9e238b9bd43d3857d40edf65df9416dea988929891d92a9f8a778"}, - {file = "grpcio-1.50.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:cf151f97f5f381163912e8952eb5b3afe89dec9ed723d1561d59cabf1e219a35"}, - {file = "grpcio-1.50.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a23d47f2fc7111869f0ff547f771733661ff2818562b04b9ed674fa208e261f4"}, - {file = "grpcio-1.50.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84d04dec64cc4ed726d07c5d17b73c343c8ddcd6b59c7199c801d6bbb9d9ed1"}, - {file = "grpcio-1.50.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:67dd41a31f6fc5c7db097a5c14a3fa588af54736ffc174af4411d34c4f306f68"}, - {file = "grpcio-1.50.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8d4c8e73bf20fb53fe5a7318e768b9734cf122fe671fcce75654b98ba12dfb75"}, - {file = "grpcio-1.50.0-cp37-cp37m-win32.whl", hash = "sha256:7489dbb901f4fdf7aec8d3753eadd40839c9085967737606d2c35b43074eea24"}, - {file = "grpcio-1.50.0-cp37-cp37m-win_amd64.whl", hash = "sha256:531f8b46f3d3db91d9ef285191825d108090856b3bc86a75b7c3930f16ce432f"}, - {file = "grpcio-1.50.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:d534d169673dd5e6e12fb57cc67664c2641361e1a0885545495e65a7b761b0f4"}, - {file = "grpcio-1.50.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:1d8d02dbb616c0a9260ce587eb751c9c7dc689bc39efa6a88cc4fa3e9c138a7b"}, - {file = "grpcio-1.50.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:baab51dcc4f2aecabf4ed1e2f57bceab240987c8b03533f1cef90890e6502067"}, - {file = "grpcio-1.50.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40838061e24f960b853d7bce85086c8e1b81c6342b1f4c47ff0edd44bbae2722"}, - {file = "grpcio-1.50.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:931e746d0f75b2a5cff0a1197d21827a3a2f400c06bace036762110f19d3d507"}, - {file = "grpcio-1.50.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:15f9e6d7f564e8f0776770e6ef32dac172c6f9960c478616c366862933fa08b4"}, - {file = "grpcio-1.50.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a4c23e54f58e016761b576976da6a34d876420b993f45f66a2bfb00363ecc1f9"}, - {file = "grpcio-1.50.0-cp38-cp38-win32.whl", hash = "sha256:3e4244c09cc1b65c286d709658c061f12c61c814be0b7030a2d9966ff02611e0"}, - {file = "grpcio-1.50.0-cp38-cp38-win_amd64.whl", hash = "sha256:8e69aa4e9b7f065f01d3fdcecbe0397895a772d99954bb82eefbb1682d274518"}, - {file = "grpcio-1.50.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:af98d49e56605a2912cf330b4627e5286243242706c3a9fa0bcec6e6f68646fc"}, - {file = "grpcio-1.50.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:080b66253f29e1646ac53ef288c12944b131a2829488ac3bac8f52abb4413c0d"}, - {file = "grpcio-1.50.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:ab5d0e3590f0a16cb88de4a3fa78d10eb66a84ca80901eb2c17c1d2c308c230f"}, - {file = "grpcio-1.50.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb11464f480e6103c59d558a3875bd84eed6723f0921290325ebe97262ae1347"}, - {file = "grpcio-1.50.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e07fe0d7ae395897981d16be61f0db9791f482f03fee7d1851fe20ddb4f69c03"}, - {file = "grpcio-1.50.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d75061367a69808ab2e84c960e9dce54749bcc1e44ad3f85deee3a6c75b4ede9"}, - {file = "grpcio-1.50.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ae23daa7eda93c1c49a9ecc316e027ceb99adbad750fbd3a56fa9e4a2ffd5ae0"}, - {file = "grpcio-1.50.0-cp39-cp39-win32.whl", hash = "sha256:177afaa7dba3ab5bfc211a71b90da1b887d441df33732e94e26860b3321434d9"}, - {file = "grpcio-1.50.0-cp39-cp39-win_amd64.whl", hash = "sha256:ea8ccf95e4c7e20419b7827aa5b6da6f02720270686ac63bd3493a651830235c"}, - {file = "grpcio-1.50.0.tar.gz", hash = "sha256:12b479839a5e753580b5e6053571de14006157f2ef9b71f38c56dc9b23b95ad6"}, + {file = "grpcio-1.51.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:cc2bece1737b44d878cc1510ea04469a8073dbbcdd762175168937ae4742dfb3"}, + {file = "grpcio-1.51.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:e223a9793522680beae44671b9ed8f6d25bbe5ddf8887e66aebad5e0686049ef"}, + {file = "grpcio-1.51.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:24ac1154c4b2ab4a0c5326a76161547e70664cd2c39ba75f00fc8a2170964ea2"}, + {file = "grpcio-1.51.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4ef09f8997c4be5f3504cefa6b5c6cc3cf648274ce3cede84d4342a35d76db6"}, + {file = "grpcio-1.51.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8a0b77e992c64880e6efbe0086fe54dfc0bbd56f72a92d9e48264dcd2a3db98"}, + {file = "grpcio-1.51.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:eacad297ea60c72dd280d3353d93fb1dcca952ec11de6bb3c49d12a572ba31dd"}, + {file = "grpcio-1.51.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:16c71740640ba3a882f50b01bf58154681d44b51f09a5728180a8fdc66c67bd5"}, + {file = "grpcio-1.51.1-cp310-cp310-win32.whl", hash = "sha256:29cb97d41a4ead83b7bcad23bdb25bdd170b1e2cba16db6d3acbb090bc2de43c"}, + {file = "grpcio-1.51.1-cp310-cp310-win_amd64.whl", hash = "sha256:9ff42c5620b4e4530609e11afefa4a62ca91fa0abb045a8957e509ef84e54d30"}, + {file = "grpcio-1.51.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:bc59f7ba87972ab236f8669d8ca7400f02a0eadf273ca00e02af64d588046f02"}, + {file = "grpcio-1.51.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:3c2b3842dcf870912da31a503454a33a697392f60c5e2697c91d133130c2c85d"}, + {file = "grpcio-1.51.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22b011674090594f1f3245960ced7386f6af35485a38901f8afee8ad01541dbd"}, + {file = "grpcio-1.51.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49d680356a975d9c66a678eb2dde192d5dc427a7994fb977363634e781614f7c"}, + {file = "grpcio-1.51.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:094e64236253590d9d4075665c77b329d707b6fca864dd62b144255e199b4f87"}, + {file = "grpcio-1.51.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:257478300735ce3c98d65a930bbda3db172bd4e00968ba743e6a1154ea6edf10"}, + {file = "grpcio-1.51.1-cp311-cp311-win32.whl", hash = "sha256:5a6ebcdef0ef12005d56d38be30f5156d1cb3373b52e96f147f4a24b0ddb3a9d"}, + {file = "grpcio-1.51.1-cp311-cp311-win_amd64.whl", hash = "sha256:3f9b0023c2c92bebd1be72cdfca23004ea748be1813a66d684d49d67d836adde"}, + {file = "grpcio-1.51.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:cd3baccea2bc5c38aeb14e5b00167bd4e2373a373a5e4d8d850bd193edad150c"}, + {file = "grpcio-1.51.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:17ec9b13cec4a286b9e606b48191e560ca2f3bbdf3986f91e480a95d1582e1a7"}, + {file = "grpcio-1.51.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:fbdbe9a849854fe484c00823f45b7baab159bdd4a46075302281998cb8719df5"}, + {file = "grpcio-1.51.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:31bb6bc7ff145e2771c9baf612f4b9ebbc9605ccdc5f3ff3d5553de7fc0e0d79"}, + {file = "grpcio-1.51.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e473525c28251558337b5c1ad3fa969511e42304524a4e404065e165b084c9e4"}, + {file = "grpcio-1.51.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6f0b89967ee11f2b654c23b27086d88ad7bf08c0b3c2a280362f28c3698b2896"}, + {file = "grpcio-1.51.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7942b32a291421460d6a07883033e392167d30724aa84987e6956cd15f1a21b9"}, + {file = "grpcio-1.51.1-cp37-cp37m-win32.whl", hash = "sha256:f96ace1540223f26fbe7c4ebbf8a98e3929a6aa0290c8033d12526847b291c0f"}, + {file = "grpcio-1.51.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f1fec3abaf274cdb85bf3878167cfde5ad4a4d97c68421afda95174de85ba813"}, + {file = "grpcio-1.51.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:0e1a9e1b4a23808f1132aa35f968cd8e659f60af3ffd6fb00bcf9a65e7db279f"}, + {file = "grpcio-1.51.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:6df3b63538c362312bc5fa95fb965069c65c3ea91d7ce78ad9c47cab57226f54"}, + {file = "grpcio-1.51.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:172405ca6bdfedd6054c74c62085946e45ad4d9cec9f3c42b4c9a02546c4c7e9"}, + {file = "grpcio-1.51.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:506b9b7a4cede87d7219bfb31014d7b471cfc77157da9e820a737ec1ea4b0663"}, + {file = "grpcio-1.51.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fb93051331acbb75b49a2a0fd9239c6ba9528f6bdc1dd400ad1cb66cf864292"}, + {file = "grpcio-1.51.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5dca372268c6ab6372d37d6b9f9343e7e5b4bc09779f819f9470cd88b2ece3c3"}, + {file = "grpcio-1.51.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:471d39d3370ca923a316d49c8aac66356cea708a11e647e3bdc3d0b5de4f0a40"}, + {file = "grpcio-1.51.1-cp38-cp38-win32.whl", hash = "sha256:75e29a90dc319f0ad4d87ba6d20083615a00d8276b51512e04ad7452b5c23b04"}, + {file = "grpcio-1.51.1-cp38-cp38-win_amd64.whl", hash = "sha256:f1158bccbb919da42544a4d3af5d9296a3358539ffa01018307337365a9a0c64"}, + {file = "grpcio-1.51.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:59dffade859f157bcc55243714d57b286da6ae16469bf1ac0614d281b5f49b67"}, + {file = "grpcio-1.51.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:dad6533411d033b77f5369eafe87af8583178efd4039c41d7515d3336c53b4f1"}, + {file = "grpcio-1.51.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:4c4423ea38a7825b8fed8934d6d9aeebdf646c97e3c608c3b0bcf23616f33877"}, + {file = "grpcio-1.51.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0dc5354e38e5adf2498312f7241b14c7ce3484eefa0082db4297189dcbe272e6"}, + {file = "grpcio-1.51.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97d67983189e2e45550eac194d6234fc38b8c3b5396c153821f2d906ed46e0ce"}, + {file = "grpcio-1.51.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:538d981818e49b6ed1e9c8d5e5adf29f71c4e334e7d459bf47e9b7abb3c30e09"}, + {file = "grpcio-1.51.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9235dcd5144a83f9ca6f431bd0eccc46b90e2c22fe27b7f7d77cabb2fb515595"}, + {file = "grpcio-1.51.1-cp39-cp39-win32.whl", hash = "sha256:aacb54f7789ede5cbf1d007637f792d3e87f1c9841f57dd51abf89337d1b8472"}, + {file = "grpcio-1.51.1-cp39-cp39-win_amd64.whl", hash = "sha256:2b170eaf51518275c9b6b22ccb59450537c5a8555326fd96ff7391b5dd75303c"}, + {file = "grpcio-1.51.1.tar.gz", hash = "sha256:e6dfc2b6567b1c261739b43d9c59d201c1b89e017afd9e684d85aa7a186c9f7a"}, ] grpcio-reflection = [ - {file = "grpcio-reflection-1.50.0.tar.gz", hash = "sha256:d75ff6a12f70f28ba6465d5a29781cf2d9c740eb809711990f61699019af1f72"}, - {file = "grpcio_reflection-1.50.0-py3-none-any.whl", hash = "sha256:92fea4a9527a8bcc52b615b6a7af655a5a118ab0f57227cc814dc2192f8a7455"}, + {file = "grpcio-reflection-1.51.1.tar.gz", hash = "sha256:c07a93c0c36ef88fe475744289863b4787005eff4de0cc04213ecad718b01aae"}, + {file = "grpcio_reflection-1.51.1-py3-none-any.whl", hash = "sha256:b70af764a83e42a44f65df1edb232e972ab69e72bc7fbbad481e66c29a9d8cb8"}, +] +grpcio-status = [ + {file = "grpcio-status-1.51.1.tar.gz", hash = "sha256:ac2617a3095935ebd785e2228958f24b10a0d527a0c9eb5a0863c784f648a816"}, + {file = "grpcio_status-1.51.1-py3-none-any.whl", hash = "sha256:a52cbdc4b18f325bfc13d319ae7c7ae7a0fee07f3d9a005504d6097896d7a495"}, ] grpcio-tools = [ - {file = "grpcio-tools-1.50.0.tar.gz", hash = "sha256:88b75f2afd889c7c6939f58d76b58ab84de4723c7de882a1f8448af6632e256f"}, - {file = "grpcio_tools-1.50.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:fad4f44220847691605e4079ec67b6663064d5b1283bee5a89dd6f7726672d08"}, - {file = "grpcio_tools-1.50.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:4fe1cd201f0e0f601551c20bea2a8c1e109767ce341ac263e88d5a0acd0a124c"}, - {file = "grpcio_tools-1.50.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:66113bc60db8ccb15e2d7b79efd5757bba33b15e653f20f473c776857bd4415d"}, - {file = "grpcio_tools-1.50.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:71e98662f76d3bcf031f36f727ffee594c57006d5dce7b6d031d030d194f054d"}, - {file = "grpcio_tools-1.50.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1891aca6042fc403c1d492d03ce1c4bac0053c26132063b7b0171665a0aa6086"}, - {file = "grpcio_tools-1.50.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:54528cec27238761055b28a34ea29392c8e90ac8d8966ee3ccfc9f906eb26fd3"}, - {file = "grpcio_tools-1.50.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:89010e8e0d158966af478d9b345d29d98e52d62464cb93ef192125e94fb904c1"}, - {file = "grpcio_tools-1.50.0-cp310-cp310-win32.whl", hash = "sha256:e42ac5aa09debbf7b72444af7f390713e85b8808557e920d9d8071f14127c0eb"}, - {file = "grpcio_tools-1.50.0-cp310-cp310-win_amd64.whl", hash = "sha256:c876dbaa6897e45ed074df1562006ac9fdeb5c7235f623cda2c29894fe4f8fba"}, - {file = "grpcio_tools-1.50.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:3995d96635d6decaf9d5d2e071677782b08ea3186d7cb8ad93f6c2dd97ec9097"}, - {file = "grpcio_tools-1.50.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1736defd9fc54f942f500dd71555816487e52f07d25b0a58ff4089c375f0b5c3"}, - {file = "grpcio_tools-1.50.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fecbc5e773c1c1971641227d8fcd5de97533dbbc0bd7854b436ceb57243f4b2a"}, - {file = "grpcio_tools-1.50.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8178a475de70e6ae0a7e3f6d7ec1415c9dd10680654dbea9a1caf79665b0ed4"}, - {file = "grpcio_tools-1.50.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d9b8b59a76f67270ce8a4fd7af005732356d5b4ebaffa4598343ea63677bad63"}, - {file = "grpcio_tools-1.50.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ead51910d83123648b1e36e2de8483b66e9ba358ca1d8965005ba7713cbcb7db"}, - {file = "grpcio_tools-1.50.0-cp311-cp311-win32.whl", hash = "sha256:4c928610ceda7890cf7ddd3cb4cc16fb81995063bf4fdb5171bba572d508079b"}, - {file = "grpcio_tools-1.50.0-cp311-cp311-win_amd64.whl", hash = "sha256:56c28f50b9c88cbfc62861de86a1f8b2454d3b48cff85a73d523ec22f130635d"}, - {file = "grpcio_tools-1.50.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:675634367846fc906e8b6016830b3d361f9e788bce7820b4e6432d914cb0e25d"}, - {file = "grpcio_tools-1.50.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:f6dbd9d86aa2fbeb8a104ae8edd23c55f182829db58cae3f28842024e08b1071"}, - {file = "grpcio_tools-1.50.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:2869fba92670d5be3730f1ad108cbacd3f9a7b7afa57187a3c39379513b3fde7"}, - {file = "grpcio_tools-1.50.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f211a8a6f159b496097cabbf4bdbb6d01db0fd7a8d516dd0e577ee2ddccded0"}, - {file = "grpcio_tools-1.50.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27f734e44631d755481a179ffd5f2659eaf1e9755cf3539e291d7a28d8373cc8"}, - {file = "grpcio_tools-1.50.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9bc97b52d577406c240b0d6acf68ab5100c67c903792f1f9ce04d871ca83de53"}, - {file = "grpcio_tools-1.50.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a9816a6c8d1b7adb65217d01624006704faf02b5daa15b6e145b63a5e559d791"}, - {file = "grpcio_tools-1.50.0-cp37-cp37m-win32.whl", hash = "sha256:c5dde47bae615554349dd09185eaaeffd9a28e961de347cf0b3b98a853b43a6c"}, - {file = "grpcio_tools-1.50.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e51133b78b67bd4c06e99bea9c631f29acd52af2b2a254926d192f277f9067b5"}, - {file = "grpcio_tools-1.50.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fec4336e067f2bdd45d74c2466dcd06f245e972d2139187d87cfccf847dbaf50"}, - {file = "grpcio_tools-1.50.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8d74672162474e2592c50708408b4ee145994451ee5171c9b8f580504b987ad3"}, - {file = "grpcio_tools-1.50.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:e6baae6fa9a88c1895193f9610b54d483753ad3e3f91f5d5a10a506afe342a43"}, - {file = "grpcio_tools-1.50.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:927c8cbbfbf3bca46cd05bbabfd8cb6c99f5dbaeb860f2786f9286ce6894ad3d"}, - {file = "grpcio_tools-1.50.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fe40a66d3ccde959f7290b881c78e8c420fd54289fede5c9119d993703e1a49"}, - {file = "grpcio_tools-1.50.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:86a9e9b2cc30797472bd421c75b42fe1062519ef5675243cb33d99de33af279c"}, - {file = "grpcio_tools-1.50.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c1646bfec6d707bcaafea69088770151ba291ede788f2cda5efdc09c1eac5c1c"}, - {file = "grpcio_tools-1.50.0-cp38-cp38-win32.whl", hash = "sha256:dedfe2947ad4ccc8cb2f7d0cac5766dd623901f29e1953a2854126559c317885"}, - {file = "grpcio_tools-1.50.0-cp38-cp38-win_amd64.whl", hash = "sha256:73ba21fe500e2476d391cc92aa05545fdc71bdeedb5b9ec2b22e9a3ccc298a2f"}, - {file = "grpcio_tools-1.50.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:23381ac6a19de0d3cb470f4d1fb450267014234a3923e3c43abb858049f5caf4"}, - {file = "grpcio_tools-1.50.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:c61730ae21533ef992a89df0321ebe59e344f9f0d1d54572b92d9468969c75b3"}, - {file = "grpcio_tools-1.50.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:e5adcbec235dce997efacb44cca5cde24db47ad9c432fcbca875185d2095c55a"}, - {file = "grpcio_tools-1.50.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c40f918d1fe0cb094bfb09ab7567a92f5368fa0b9935235680fe205ddc37518"}, - {file = "grpcio_tools-1.50.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2313c8c73a5003d789fe05f4d01112ccf8330d3d141f5b77b514200d742193f6"}, - {file = "grpcio_tools-1.50.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:578f9137ebca0f152136b47b4895a6b4dabcf17e41f957b03637d1ae096ff67b"}, - {file = "grpcio_tools-1.50.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d8aaff094d94d2119fffe7e0d08ca59d42076971f9d85857e99288e500ab31f0"}, - {file = "grpcio_tools-1.50.0-cp39-cp39-win32.whl", hash = "sha256:e1a8f9a57bbcc2e633aaf327e39830527f3c1f7add18c7580f3058fe9a0fa780"}, - {file = "grpcio_tools-1.50.0-cp39-cp39-win_amd64.whl", hash = "sha256:b7eb7a84d9171c0ae1550833f4a6ca52372bed9db0fa10f8c9dbe6ca65f97a8c"}, + {file = "grpcio-tools-1.51.1.tar.gz", hash = "sha256:8e62d23d3fed9d4f81738f98dd193dbd2e21aed4a8f0dd715e75b5439e649727"}, + {file = "grpcio_tools-1.51.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:ecf1494cb695afead36995534f787761ee33fb9e116b23030113a37fe6057a83"}, + {file = "grpcio_tools-1.51.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:16b8b915625dc6eb2ea7efdfb06f1fae44a9066c9016453a2ca120c034f33090"}, + {file = "grpcio_tools-1.51.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:d5e033c04b416afcddd5231b3ff94a34fb5d26fba2416eb940e69b05f22cfd25"}, + {file = "grpcio_tools-1.51.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a218f64e667f3332b74080bdc5440aaf0fa6700ae07a0b54ecf085aaef2aa9f"}, + {file = "grpcio_tools-1.51.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7b186183515ad6b8584ffe4bd820b72b00f6e7d121fb1c36294edeea9092313"}, + {file = "grpcio_tools-1.51.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ccd37165d7a3e93f460096a2eb62b7a9c1ebe5c424eaee42d8e92740d0c8f6bc"}, + {file = "grpcio_tools-1.51.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:531586c5598a99658249f3c5e92826d6d2bb117abd6ffc88527d1e1d9eaef924"}, + {file = "grpcio_tools-1.51.1-cp310-cp310-win32.whl", hash = "sha256:392ad4cd004f7b843cf7d916d9a15b2d6585965bfef235be1c88d8f8649777e5"}, + {file = "grpcio_tools-1.51.1-cp310-cp310-win_amd64.whl", hash = "sha256:14e82c2b3ee7e300611c2c729d411b3b911e4cca5f4ec14787457a2fb72ff9d4"}, + {file = "grpcio_tools-1.51.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:2281180490c475d09b7aa05dabafa5e09de9902176931e7295113f636c2b5360"}, + {file = "grpcio_tools-1.51.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:c4649af7f5d9553975ee66b6bfae20a84be779f13e163fa835e782961895e63c"}, + {file = "grpcio_tools-1.51.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f06bb0753b7cecbff154b523cfb8f45dee2c31b0a4c72bed7da44c57f1cba113"}, + {file = "grpcio_tools-1.51.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a671466158ed74c07ee070fb940ed783acf59ba6e6e53cb4de8fd63819c6c7f"}, + {file = "grpcio_tools-1.51.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:048793747339f327ea091d8f022c6756d89713d8080dffde5ce7380cc348ea8e"}, + {file = "grpcio_tools-1.51.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f6caf36e7752728329a28f93afec7c4ec9015fc1c6e4460bd1eb0f3737e1c55a"}, + {file = "grpcio_tools-1.51.1-cp311-cp311-win32.whl", hash = "sha256:67b304282cad38642587ebae68617e450e1ad4fa1c0c8b19e9e30274dbb32716"}, + {file = "grpcio_tools-1.51.1-cp311-cp311-win_amd64.whl", hash = "sha256:674b340f2f7bb2adbc3f15144bd37ce5ea83239f78b68dbbd0ea3cba00107e2b"}, + {file = "grpcio_tools-1.51.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:055819992ddd30c642a7fd6f344a03747be3afa95cb910f8a2e5efaabd41cde5"}, + {file = "grpcio_tools-1.51.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:4e3249a2ec435b3b972610c66c8a714c188844500d564c910f57a2771dc61978"}, + {file = "grpcio_tools-1.51.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:794f26a09b70f4f101df5cf54c6c12dc1b65747ab1dee5bda02c2991389ade56"}, + {file = "grpcio_tools-1.51.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4957f1ffa16598aa5379505fcbaeb47d65693a46b0817f4ee61db76707092aeb"}, + {file = "grpcio_tools-1.51.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9906fb6bf6d9c30c23d85153f12d130f44325afe8f9ebe58aa7a6c82ecade9d8"}, + {file = "grpcio_tools-1.51.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87bc5f3e3698c65907d397003c64d25c3ea84e3d6aa46dac133bd98bf66835ee"}, + {file = "grpcio_tools-1.51.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a66b3a5d18a7615f0f828b72e2d2935751459c89cc4725e56bdfb3d2cd93281f"}, + {file = "grpcio_tools-1.51.1-cp37-cp37m-win32.whl", hash = "sha256:566809d9942e78821b279af70f3cf159a328127f9f3d5fee8d83ad8b2d27b2fe"}, + {file = "grpcio_tools-1.51.1-cp37-cp37m-win_amd64.whl", hash = "sha256:aab24a342642329de38139cb26f8492882ca0d8551bb87f6530bcc613945a0d0"}, + {file = "grpcio_tools-1.51.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:6b83d7fc2597c6d392c225177d1fbbcff74900f8cc40b33236987fd1ff841330"}, + {file = "grpcio_tools-1.51.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:79c06d2577cb4d977922bbf01234de3b20f73d1784d3cbe3179deee1bdb9a60b"}, + {file = "grpcio_tools-1.51.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:e9abc03d67793b1bf33dc766caa69a3333f9db029869ba6e8fc6cd9c251c0080"}, + {file = "grpcio_tools-1.51.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64d8ad369417759f5fdb8ffb7cbd6374fecc06ab51c9a226dee9bbd7d311c3b5"}, + {file = "grpcio_tools-1.51.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de51a0a71845b854f6a5967756c893c96bd03e37f39e5dce87b4f409dac36ee2"}, + {file = "grpcio_tools-1.51.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9dfe6c12b0e2c07f6a4a91a9912ef4e5bd007672533891a44e6f433ffbf7c3b1"}, + {file = "grpcio_tools-1.51.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:27113b354f7587684eb55125733e6e5be1f489458abfe12344dabd918d8dcc54"}, + {file = "grpcio_tools-1.51.1-cp38-cp38-win32.whl", hash = "sha256:98777b5031f1b3c58b688815ffa83435c103b2152c26eb144f80f4a4bb34addb"}, + {file = "grpcio_tools-1.51.1-cp38-cp38-win_amd64.whl", hash = "sha256:1c44b57a6770b78a1eafe355878ff1ec59a2fa07455a2cbd522c071eedae04d4"}, + {file = "grpcio_tools-1.51.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:49624394805568acd7d767dea5a00d970fca5ad8f395fe0161eeea0de5133eba"}, + {file = "grpcio_tools-1.51.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:6d6626a6e4dbe843df96dc8c08dd244d2191a75324f54bfa4ebaa3e76b0b1958"}, + {file = "grpcio_tools-1.51.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:b4fb8ed6d29f2d6cf03ef99ffaad635bbc132a59be77013691392fe557e67144"}, + {file = "grpcio_tools-1.51.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8cc862a1ad30f94528d66cc6f95fb9e659005e568313e54a23550535b649573"}, + {file = "grpcio_tools-1.51.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e72a30be1746ea0749a8486d0ca0120c0b2757fe84fc246a5144b1ef66d7b89"}, + {file = "grpcio_tools-1.51.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:331a897306adeec3c67470431ea8d8b4972b689d32966f94506d91f4dac20952"}, + {file = "grpcio_tools-1.51.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f336ad9be661d92fa45940e74e8ff3d78e67ebe9b4f7ea8774b2d680c17aeb6c"}, + {file = "grpcio_tools-1.51.1-cp39-cp39-win32.whl", hash = "sha256:40ef70e8c5d0310dedff9af502b520b4c7e215bce94094527fb959150a0c594a"}, + {file = "grpcio_tools-1.51.1-cp39-cp39-win_amd64.whl", hash = "sha256:15b8acf4eaa0ebe37e2f69108de49efd935b7abe9c7e58ba737490b99906aa76"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +loguru = [ + {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, + {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, +] numpy = [ - {file = "numpy-1.23.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d79ada05005f6f4f337d3bb9de8a7774f259341c70bc88047a1f7b96a4bcb2"}, - {file = "numpy-1.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926db372bc4ac1edf81cfb6c59e2a881606b409ddc0d0920b988174b2e2a767f"}, - {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c237129f0e732885c9a6076a537e974160482eab8f10db6292e92154d4c67d71"}, - {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8365b942f9c1a7d0f0dc974747d99dd0a0cdfc5949a33119caf05cb314682d3"}, - {file = "numpy-1.23.4-cp310-cp310-win32.whl", hash = "sha256:2341f4ab6dba0834b685cce16dad5f9b6606ea8a00e6da154f5dbded70fdc4dd"}, - {file = "numpy-1.23.4-cp310-cp310-win_amd64.whl", hash = "sha256:d331afac87c92373826af83d2b2b435f57b17a5c74e6268b79355b970626e329"}, - {file = "numpy-1.23.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:488a66cb667359534bc70028d653ba1cf307bae88eab5929cd707c761ff037db"}, - {file = "numpy-1.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce03305dd694c4873b9429274fd41fc7eb4e0e4dea07e0af97a933b079a5814f"}, - {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8981d9b5619569899666170c7c9748920f4a5005bf79c72c07d08c8a035757b0"}, - {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a70a7d3ce4c0e9284e92285cba91a4a3f5214d87ee0e95928f3614a256a1488"}, - {file = "numpy-1.23.4-cp311-cp311-win32.whl", hash = "sha256:5e13030f8793e9ee42f9c7d5777465a560eb78fa7e11b1c053427f2ccab90c79"}, - {file = "numpy-1.23.4-cp311-cp311-win_amd64.whl", hash = "sha256:7607b598217745cc40f751da38ffd03512d33ec06f3523fb0b5f82e09f6f676d"}, - {file = "numpy-1.23.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ab46e4e7ec63c8a5e6dbf5c1b9e1c92ba23a7ebecc86c336cb7bf3bd2fb10e5"}, - {file = "numpy-1.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8aae2fb3180940011b4862b2dd3756616841c53db9734b27bb93813cd79fce6"}, - {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c053d7557a8f022ec823196d242464b6955a7e7e5015b719e76003f63f82d0f"}, - {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0882323e0ca4245eb0a3d0a74f88ce581cc33aedcfa396e415e5bba7bf05f68"}, - {file = "numpy-1.23.4-cp38-cp38-win32.whl", hash = "sha256:dada341ebb79619fe00a291185bba370c9803b1e1d7051610e01ed809ef3a4ba"}, - {file = "numpy-1.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:0fe563fc8ed9dc4474cbf70742673fc4391d70f4363f917599a7fa99f042d5a8"}, - {file = "numpy-1.23.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c67b833dbccefe97cdd3f52798d430b9d3430396af7cdb2a0c32954c3ef73894"}, - {file = "numpy-1.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f76025acc8e2114bb664294a07ede0727aa75d63a06d2fae96bf29a81747e4a7"}, - {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12ac457b63ec8ded85d85c1e17d85efd3c2b0967ca39560b307a35a6703a4735"}, - {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95de7dc7dc47a312f6feddd3da2500826defdccbc41608d0031276a24181a2c0"}, - {file = "numpy-1.23.4-cp39-cp39-win32.whl", hash = "sha256:f2f390aa4da44454db40a1f0201401f9036e8d578a25f01a6e237cea238337ef"}, - {file = "numpy-1.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:f260da502d7441a45695199b4e7fd8ca87db659ba1c78f2bbf31f934fe76ae0e"}, - {file = "numpy-1.23.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:61be02e3bf810b60ab74e81d6d0d36246dbfb644a462458bb53b595791251911"}, - {file = "numpy-1.23.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296d17aed51161dbad3c67ed6d164e51fcd18dbcd5dd4f9d0a9c6055dce30810"}, - {file = "numpy-1.23.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4d52914c88b4930dafb6c48ba5115a96cbab40f45740239d9f4159c4ba779962"}, - {file = "numpy-1.23.4.tar.gz", hash = "sha256:ed2cc92af0efad20198638c69bb0fc2870a58dabfba6eb722c933b48556c686c"}, + {file = "numpy-1.24.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7"}, + {file = "numpy-1.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9"}, + {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"}, + {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398"}, + {file = "numpy-1.24.1-cp310-cp310-win32.whl", hash = "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2"}, + {file = "numpy-1.24.1-cp310-cp310-win_amd64.whl", hash = "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2"}, + {file = "numpy-1.24.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8"}, + {file = "numpy-1.24.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032"}, + {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1"}, + {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9"}, + {file = "numpy-1.24.1-cp311-cp311-win32.whl", hash = "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36"}, + {file = "numpy-1.24.1-cp311-cp311-win_amd64.whl", hash = "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51"}, + {file = "numpy-1.24.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407"}, + {file = "numpy-1.24.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954"}, + {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36"}, + {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7"}, + {file = "numpy-1.24.1-cp38-cp38-win32.whl", hash = "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1"}, + {file = "numpy-1.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c"}, + {file = "numpy-1.24.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6"}, + {file = "numpy-1.24.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7"}, + {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700"}, + {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf"}, + {file = "numpy-1.24.1-cp39-cp39-win32.whl", hash = "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f"}, + {file = "numpy-1.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e"}, + {file = "numpy-1.24.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d"}, + {file = "numpy-1.24.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086"}, + {file = "numpy-1.24.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566"}, + {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, +] +nvidia-cublas-cu11 = [ + {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, + {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-win_amd64.whl", hash = "sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e"}, +] +nvidia-cuda-nvrtc-cu11 = [ + {file = "nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03"}, + {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7"}, + {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:f2effeb1309bdd1b3854fc9b17eaf997808f8b25968ce0c7070945c4265d64a3"}, +] +nvidia-cuda-runtime-cu11 = [ + {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31"}, + {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7"}, +] +nvidia-cudnn-cu11 = [ + {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, + {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, ] packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, + {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, + {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] protobuf = [ - {file = "protobuf-4.21.8-cp310-abi3-win32.whl", hash = "sha256:c252c55ee15175aa1b21b7b9896e6add5162d066d5202e75c39f96136f08cce3"}, - {file = "protobuf-4.21.8-cp310-abi3-win_amd64.whl", hash = "sha256:809ca0b225d3df42655a12f311dd0f4148a943c51f1ad63c38343e457492b689"}, - {file = "protobuf-4.21.8-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bbececaf3cfea9ea65ebb7974e6242d310d2a7772a6f015477e0d79993af4511"}, - {file = "protobuf-4.21.8-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:b02eabb9ebb1a089ed20626a90ad7a69cee6bcd62c227692466054b19c38dd1f"}, - {file = "protobuf-4.21.8-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:4761201b93e024bb70ee3a6a6425d61f3152ca851f403ba946fb0cde88872661"}, - {file = "protobuf-4.21.8-cp37-cp37m-win32.whl", hash = "sha256:f2d55ff22ec300c4d954d3b0d1eeb185681ec8ad4fbecff8a5aee6a1cdd345ba"}, - {file = "protobuf-4.21.8-cp37-cp37m-win_amd64.whl", hash = "sha256:c5f94911dd8feb3cd3786fc90f7565c9aba7ce45d0f254afd625b9628f578c3f"}, - {file = "protobuf-4.21.8-cp38-cp38-win32.whl", hash = "sha256:b37b76efe84d539f16cba55ee0036a11ad91300333abd213849cbbbb284b878e"}, - {file = "protobuf-4.21.8-cp38-cp38-win_amd64.whl", hash = "sha256:2c92a7bfcf4ae76a8ac72e545e99a7407e96ffe52934d690eb29a8809ee44d7b"}, - {file = "protobuf-4.21.8-cp39-cp39-win32.whl", hash = "sha256:89d641be4b5061823fa0e463c50a2607a97833e9f8cfb36c2f91ef5ccfcc3861"}, - {file = "protobuf-4.21.8-cp39-cp39-win_amd64.whl", hash = "sha256:bc471cf70a0f53892fdd62f8cd4215f0af8b3f132eeee002c34302dff9edd9b6"}, - {file = "protobuf-4.21.8-py2.py3-none-any.whl", hash = "sha256:a55545ce9eec4030cf100fcb93e861c622d927ef94070c1a3c01922902464278"}, - {file = "protobuf-4.21.8-py3-none-any.whl", hash = "sha256:0f236ce5016becd989bf39bd20761593e6d8298eccd2d878eda33012645dc369"}, - {file = "protobuf-4.21.8.tar.gz", hash = "sha256:427426593b55ff106c84e4a88cac855175330cb6eb7e889e85aaa7b5652b686d"}, + {file = "protobuf-4.21.12-cp310-abi3-win32.whl", hash = "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1"}, + {file = "protobuf-4.21.12-cp310-abi3-win_amd64.whl", hash = "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2"}, + {file = "protobuf-4.21.12-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791"}, + {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97"}, + {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7"}, + {file = "protobuf-4.21.12-cp37-cp37m-win32.whl", hash = "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717"}, + {file = "protobuf-4.21.12-cp37-cp37m-win_amd64.whl", hash = "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574"}, + {file = "protobuf-4.21.12-cp38-cp38-win32.whl", hash = "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec"}, + {file = "protobuf-4.21.12-cp38-cp38-win_amd64.whl", hash = "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30"}, + {file = "protobuf-4.21.12-cp39-cp39-win32.whl", hash = "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc"}, + {file = "protobuf-4.21.12-cp39-cp39-win_amd64.whl", hash = "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b"}, + {file = "protobuf-4.21.12-py2.py3-none-any.whl", hash = "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5"}, + {file = "protobuf-4.21.12-py3-none-any.whl", hash = "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462"}, + {file = "protobuf-4.21.12.tar.gz", hash = "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab"}, ] psutil = [ - {file = "psutil-5.9.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b4a247cd3feaae39bb6085fcebf35b3b8ecd9b022db796d89c8f05067ca28e71"}, - {file = "psutil-5.9.3-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:5fa88e3d5d0b480602553d362c4b33a63e0c40bfea7312a7bf78799e01e0810b"}, - {file = "psutil-5.9.3-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:767ef4fa33acda16703725c0473a91e1832d296c37c63896c7153ba81698f1ab"}, - {file = "psutil-5.9.3-cp27-cp27m-win32.whl", hash = "sha256:9a4af6ed1094f867834f5f07acd1250605a0874169a5fcadbcec864aec2496a6"}, - {file = "psutil-5.9.3-cp27-cp27m-win_amd64.whl", hash = "sha256:fa5e32c7d9b60b2528108ade2929b115167fe98d59f89555574715054f50fa31"}, - {file = "psutil-5.9.3-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:fe79b4ad4836e3da6c4650cb85a663b3a51aef22e1a829c384e18fae87e5e727"}, - {file = "psutil-5.9.3-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:db8e62016add2235cc87fb7ea000ede9e4ca0aa1f221b40cef049d02d5d2593d"}, - {file = "psutil-5.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:941a6c2c591da455d760121b44097781bc970be40e0e43081b9139da485ad5b7"}, - {file = "psutil-5.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:71b1206e7909792d16933a0d2c1c7f04ae196186c51ba8567abae1d041f06dcb"}, - {file = "psutil-5.9.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f57d63a2b5beaf797b87024d018772439f9d3103a395627b77d17a8d72009543"}, - {file = "psutil-5.9.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7507f6c7b0262d3e7b0eeda15045bf5881f4ada70473b87bc7b7c93b992a7d7"}, - {file = "psutil-5.9.3-cp310-cp310-win32.whl", hash = "sha256:1b540599481c73408f6b392cdffef5b01e8ff7a2ac8caae0a91b8222e88e8f1e"}, - {file = "psutil-5.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:547ebb02031fdada635452250ff39942db8310b5c4a8102dfe9384ee5791e650"}, - {file = "psutil-5.9.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d8c3cc6bb76492133474e130a12351a325336c01c96a24aae731abf5a47fe088"}, - {file = "psutil-5.9.3-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07d880053c6461c9b89cd5d4808f3b8336665fa3acdefd6777662c5ed73a851a"}, - {file = "psutil-5.9.3-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e8b50241dd3c2ed498507f87a6602825073c07f3b7e9560c58411c14fe1e1c9"}, - {file = "psutil-5.9.3-cp36-cp36m-win32.whl", hash = "sha256:828c9dc9478b34ab96be75c81942d8df0c2bb49edbb481f597314d92b6441d89"}, - {file = "psutil-5.9.3-cp36-cp36m-win_amd64.whl", hash = "sha256:ed15edb14f52925869250b1375f0ff58ca5c4fa8adefe4883cfb0737d32f5c02"}, - {file = "psutil-5.9.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d266cd05bd4a95ca1c2b9b5aac50d249cf7c94a542f47e0b22928ddf8b80d1ef"}, - {file = "psutil-5.9.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e4939ff75149b67aef77980409f156f0082fa36accc475d45c705bb00c6c16a"}, - {file = "psutil-5.9.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68fa227c32240c52982cb931801c5707a7f96dd8927f9102d6c7771ea1ff5698"}, - {file = "psutil-5.9.3-cp37-cp37m-win32.whl", hash = "sha256:beb57d8a1ca0ae0eb3d08ccaceb77e1a6d93606f0e1754f0d60a6ebd5c288837"}, - {file = "psutil-5.9.3-cp37-cp37m-win_amd64.whl", hash = "sha256:12500d761ac091f2426567f19f95fd3f15a197d96befb44a5c1e3cbe6db5752c"}, - {file = "psutil-5.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba38cf9984d5462b506e239cf4bc24e84ead4b1d71a3be35e66dad0d13ded7c1"}, - {file = "psutil-5.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:46907fa62acaac364fff0b8a9da7b360265d217e4fdeaca0a2397a6883dffba2"}, - {file = "psutil-5.9.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a04a1836894c8279e5e0a0127c0db8e198ca133d28be8a2a72b4db16f6cf99c1"}, - {file = "psutil-5.9.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a4e07611997acf178ad13b842377e3d8e9d0a5bac43ece9bfc22a96735d9a4f"}, - {file = "psutil-5.9.3-cp38-cp38-win32.whl", hash = "sha256:6ced1ad823ecfa7d3ce26fe8aa4996e2e53fb49b7fed8ad81c80958501ec0619"}, - {file = "psutil-5.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:35feafe232d1aaf35d51bd42790cbccb882456f9f18cdc411532902370d660df"}, - {file = "psutil-5.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:538fcf6ae856b5e12d13d7da25ad67f02113c96f5989e6ad44422cb5994ca7fc"}, - {file = "psutil-5.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a3d81165b8474087bb90ec4f333a638ccfd1d69d34a9b4a1a7eaac06648f9fbe"}, - {file = "psutil-5.9.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a7826e68b0cf4ce2c1ee385d64eab7d70e3133171376cac53d7c1790357ec8f"}, - {file = "psutil-5.9.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ec296f565191f89c48f33d9544d8d82b0d2af7dd7d2d4e6319f27a818f8d1cc"}, - {file = "psutil-5.9.3-cp39-cp39-win32.whl", hash = "sha256:9ec95df684583b5596c82bb380c53a603bb051cf019d5c849c47e117c5064395"}, - {file = "psutil-5.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:4bd4854f0c83aa84a5a40d3b5d0eb1f3c128f4146371e03baed4589fe4f3c931"}, - {file = "psutil-5.9.3.tar.gz", hash = "sha256:7ccfcdfea4fc4b0a02ca2c31de7fcd186beb9cff8207800e14ab66f79c773af6"}, -] -pyparsing = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, + {file = "psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8"}, + {file = "psutil-5.9.4-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe"}, + {file = "psutil-5.9.4-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549"}, + {file = "psutil-5.9.4-cp27-cp27m-win32.whl", hash = "sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad"}, + {file = "psutil-5.9.4-cp27-cp27m-win_amd64.whl", hash = "sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94"}, + {file = "psutil-5.9.4-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24"}, + {file = "psutil-5.9.4-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7"}, + {file = "psutil-5.9.4-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7"}, + {file = "psutil-5.9.4-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1"}, + {file = "psutil-5.9.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08"}, + {file = "psutil-5.9.4-cp36-abi3-win32.whl", hash = "sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff"}, + {file = "psutil-5.9.4-cp36-abi3-win_amd64.whl", hash = "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4"}, + {file = "psutil-5.9.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e"}, + {file = "psutil-5.9.4.tar.gz", hash = "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62"}, ] pytest = [ {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, @@ -565,71 +688,76 @@ PyYAML = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] safetensors = [ - {file = "safetensors-0.2.4-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:79c4a7610d7699c64d8531c43f758ded4990ebaa7b0887c2078640e6de44e726"}, - {file = "safetensors-0.2.4-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ef425a4ddd29612fe733a6eeca6ad8f3ee3939f530a032114974aac4c4667b89"}, - {file = "safetensors-0.2.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77758f8ba4de6e20bf394dd964854a926dee2efee82eaa95e6c0893e2a7d960c"}, - {file = "safetensors-0.2.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb956e9090cce515649f00b491b5ddc0f9c3d989139016a8d69f9dcf57e8d3d9"}, - {file = "safetensors-0.2.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e31b02d27249bd519f05ec9d189097c59fc6851c59daa1a86ef347659e33ac3"}, - {file = "safetensors-0.2.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c2fead03a1497042efea4358574f3d7acf501b0c82e54d605f393f2b4e2aafe"}, - {file = "safetensors-0.2.4-cp310-cp310-win32.whl", hash = "sha256:dce6ed3c7d13aafa574737eb3309c928adcb6781e879b41f0861be83b439cf3e"}, - {file = "safetensors-0.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:1dfe727325a1342767c6725dc2cc1f00463eb40a1f5df37c338d8e03957e27ce"}, - {file = "safetensors-0.2.4-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:c066bc7b90a582a01ec468fef61a7581b5c726bf12c50491cb6ea5db215ea5e0"}, - {file = "safetensors-0.2.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca6ed53dad5d7d0e67eb676528ff2ad345cac3a34010e4dc1e3736972de294a5"}, - {file = "safetensors-0.2.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ada03b44acbb036cfabe7066a8df4ad9b1ac05bb585a6b6c0f285f08e016381d"}, - {file = "safetensors-0.2.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58a0902708daa7ec2b2293b46e85df61f4fa359ddfe648e7ac025a79e6f59627"}, - {file = "safetensors-0.2.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e0a4e38f7cbb4bfc513588e52f349b906c941e74fbbe192f2b19fc34221d448"}, - {file = "safetensors-0.2.4-cp37-cp37m-win32.whl", hash = "sha256:4f8695b77dd847203258f035f8468f8b701c90621cb6b457e109f8d89c27f16c"}, - {file = "safetensors-0.2.4-cp37-cp37m-win_amd64.whl", hash = "sha256:16b08f33c753c7da64b3999beea7c30d58204a0820961e33881d05a331e3f5c0"}, - {file = "safetensors-0.2.4-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:a381606804f23db9eede51135f5fbd1f75dda02100415ee150fd39eb1cd6be4c"}, - {file = "safetensors-0.2.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7aceae84d0c7233d83923029aaf8d184848561e0211ec98c5317327b3db025d6"}, - {file = "safetensors-0.2.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:da48fc929485cbd9ee22621e388764a7cef27b0205e73aee2ad75aadd7d67662"}, - {file = "safetensors-0.2.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2619b88f934c4de6b59de90c9dc00eae2d0e30f254a1daebd6eb232ac1f9a7a7"}, - {file = "safetensors-0.2.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1f78b987ae1f6b71da8ea110164e4cab2ee31b53835d2a66279df89c5d73f0e"}, - {file = "safetensors-0.2.4-cp38-cp38-win32.whl", hash = "sha256:34b3e60b5130fb0fe07114705e51d30aa2c7eae4c1d1e77d6f260fa4ade70ede"}, - {file = "safetensors-0.2.4-cp38-cp38-win_amd64.whl", hash = "sha256:debaa4fa98a7af44ba6dcb6945efee77b8480284c2cb05918ab97cf511c40826"}, - {file = "safetensors-0.2.4-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:90baaafc0c872a736124b341db54b0bdd61765cbf3a61418371066a37905b18d"}, - {file = "safetensors-0.2.4-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:b4bf7e23191d6a3ff00de141512869fc776e8ff159c872cb44af018cb04d45eb"}, - {file = "safetensors-0.2.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf11a3aba8796e548ceb0a65f34dcd334dcf0c4c891dccabe18a8b53918ae8ab"}, - {file = "safetensors-0.2.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:95c31935ea71d63a38c546654136d7f0dbf1e7aeb6564dbc2201bc1fe9b34e4c"}, - {file = "safetensors-0.2.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef31776e2e081d6f075408eed34a0fbd524cbd19e50268bef02c238b209213b7"}, - {file = "safetensors-0.2.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06bb1d68148f6d6934352124d8cbfcf0db092f969db7187e348bd5cbf183db5"}, - {file = "safetensors-0.2.4-cp39-cp39-win32.whl", hash = "sha256:5d546152b9a5bd58eae97c2ddefba394404d37ddedec305f7639c9b6054513e5"}, - {file = "safetensors-0.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:553ecfd895d379c1e03a7c9241f7343b3af66573436969ed7eb95df81dfbe9af"}, - {file = "safetensors-0.2.4.tar.gz", hash = "sha256:35c0719a898f1f1292464f4cd9370bb6c2698032f1db4d677489f078b66b5a75"}, + {file = "safetensors-0.2.7-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:3123fdbb35fdedaedd39cd50a44493783f204821ae8b79012820020bb7e9ea1e"}, + {file = "safetensors-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d8dd3d47609c51b6bd6e473f32d74eeb90a59482f194df6db570ebbb829a948f"}, + {file = "safetensors-0.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:041aa5e13f1bbc0c0d441f692b4103531b9e867ccc281c2c3946fe51deb7eccd"}, + {file = "safetensors-0.2.7-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f2d730ac6716ab4105000aa85e7be7d771b199ec9ab77df50eff6cc0ddce6fcd"}, + {file = "safetensors-0.2.7-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:acef097785a9f03172151c7a8a4c6743d7ec77d1473c38da3aebc8f004d620a8"}, + {file = "safetensors-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f54630c845e489123a00bfb93787086c96de9abc5e72fbec52c1d2a55f8147fa"}, + {file = "safetensors-0.2.7-cp310-cp310-win32.whl", hash = "sha256:3c4c0d7f3d6922dcae178f19daf52c877674906d60944273e0fdf7a73b7a33e7"}, + {file = "safetensors-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:3c99cccbbe1da7a1bdeb7e4333046577c9785c8b4bb81912b8a134a66570fc0f"}, + {file = "safetensors-0.2.7-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:0a6f3a1db227aeb1152fb8676d94fea3f97d9e017b4b82f7ce5447b88c3a2126"}, + {file = "safetensors-0.2.7-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:8338b875e7e86f50cbffd8c79f28fb0fe2ed56bebe1f87c95a26c501ee50fc51"}, + {file = "safetensors-0.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8403e93665d87ff1006b5678c23566e7a4bf7f2cfdb2666c3955350b2841379"}, + {file = "safetensors-0.2.7-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d993db8e3f63cd71994544c130c1e227a6522c63ddaf2b9f85a33c8e789283b0"}, + {file = "safetensors-0.2.7-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c4901cac6c4bd3d52cb68c86311f86a28d33fa2503bd7c32012c1489cd9a52c"}, + {file = "safetensors-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3b9acd8d864c284d8fe300b290092a2cc7ae76319e7bdd5cbba00a8b4ec2dc0"}, + {file = "safetensors-0.2.7-cp311-cp311-win32.whl", hash = "sha256:dd63fc6bb6f7b78a957c84692afdbaa39168978aac61dfd5ea8c2bbbd459f8a6"}, + {file = "safetensors-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:848af8bca3ee8e4f68bb828e7fbc1d4022d3cde17e8bd098324ef93ace4779e6"}, + {file = "safetensors-0.2.7-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:44ab41d080be381550a0cdc1ded9afc5de899dd733f341c902834571a6f60ca7"}, + {file = "safetensors-0.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:964357808ad70e001d4d6779418e48616eb1f103acf6acdb72bb6a51c05a9da4"}, + {file = "safetensors-0.2.7-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb21eb7b86692cb1d5bc95b0926acd0356df408990de63ae09c3242594170156"}, + {file = "safetensors-0.2.7-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c3c421b55b3baf2ce373836155ffb58d530ec0752837c5fec2f8b841019b49c"}, + {file = "safetensors-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5f21fe636693f9f40594b357f3d4586cb23e98e044e5e78b21e814543d69c3b"}, + {file = "safetensors-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:1212ec6e113625d9eea662815a25c993a76860ec51f6cc26ac33f4e317abecb5"}, + {file = "safetensors-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:6b8076a21241dcad8848b42d90dc7fa1401c89bee622465d019988df62175ae1"}, + {file = "safetensors-0.2.7-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:e27907078ab3924c8135f229c9dff06126fe123b609a22cf2ae96593ca4833bc"}, + {file = "safetensors-0.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89cc90cb042654b224fa98a8acbfc5f795f33bd8c5ff6431ad9884370c9c0caf"}, + {file = "safetensors-0.2.7-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:092c34408391d6fc96617093d59f00ffd23ca624530591d225b0f824cb105367"}, + {file = "safetensors-0.2.7-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:795310f19396f01cfbe00170575daa52bc1019068ef75b13ba487155fba4e9bd"}, + {file = "safetensors-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3924794458e716dc7d2f3e264c98455de3d59b71e0335cbabef727083d537535"}, + {file = "safetensors-0.2.7-cp38-cp38-win32.whl", hash = "sha256:9205b4a12939db1135fff92b4ea4f62bf131bfc9d54df31881fb6a50eb2ba5fa"}, + {file = "safetensors-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:b84662f8d8dd49443a8f8d1d9a89a79974a5f03f4e06526de170ef02f8511236"}, + {file = "safetensors-0.2.7-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:657b01a122da21028a1d32570e0173fa0ef1865d3480cf5bcca80ec92cae421c"}, + {file = "safetensors-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:788a242ea5fab371dcc42cb10ed21c60ea8c5ff7fad41c9fb3d334420c80f156"}, + {file = "safetensors-0.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6bfbc7123d0c08754b6a11848b0298070705f60ab450e3f249624bba1571040"}, + {file = "safetensors-0.2.7-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1982f00d34d8c72e3339c458e5e2fb3eaf9458c55eae6d3bddd61649666db130"}, + {file = "safetensors-0.2.7-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:330570d27ebf5bdb3f63580baa4a4bf92ad11e1df17b96703181cfaa11cae90e"}, + {file = "safetensors-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f112b20899902dfc5ba27cdea49e57c275c7824a3e24764fca93d3ad436a160"}, + {file = "safetensors-0.2.7-cp39-cp39-win32.whl", hash = "sha256:29fbc1d1c802a5eced1675e93e130367109b2af3bf74af8c415a141b0f5fe568"}, + {file = "safetensors-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:054306d050453a92d3cc6cde15e7cb282ef4299f602433bbd70a9b1b6963c9f4"}, + {file = "safetensors-0.2.7.tar.gz", hash = "sha256:37192b456fdfe09762d6a2ef3322d2379fee52eb2db6d887ebac6e3349de596d"}, ] setuptools = [ - {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, - {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, + {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, ] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] torch = [ - {file = "torch-1.12.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:9c038662db894a23e49e385df13d47b2a777ffd56d9bcd5b832593fab0a7e286"}, - {file = "torch-1.12.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4e1b9c14cf13fd2ab8d769529050629a0e68a6fc5cb8e84b4a3cc1dd8c4fe541"}, - {file = "torch-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:e9c8f4a311ac29fc7e8e955cfb7733deb5dbe1bdaabf5d4af2765695824b7e0d"}, - {file = "torch-1.12.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:976c3f997cea38ee91a0dd3c3a42322785414748d1761ef926b789dfa97c6134"}, - {file = "torch-1.12.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:68104e4715a55c4bb29a85c6a8d57d820e0757da363be1ba680fa8cc5be17b52"}, - {file = "torch-1.12.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:743784ccea0dc8f2a3fe6a536bec8c4763bd82c1352f314937cb4008d4805de1"}, - {file = "torch-1.12.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b5dbcca369800ce99ba7ae6dee3466607a66958afca3b740690d88168752abcf"}, - {file = "torch-1.12.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f3b52a634e62821e747e872084ab32fbcb01b7fa7dbb7471b6218279f02a178a"}, - {file = "torch-1.12.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:8a34a2fbbaa07c921e1b203f59d3d6e00ed379f2b384445773bd14e328a5b6c8"}, - {file = "torch-1.12.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:42f639501928caabb9d1d55ddd17f07cd694de146686c24489ab8c615c2871f2"}, - {file = "torch-1.12.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0b44601ec56f7dd44ad8afc00846051162ef9c26a8579dda0a02194327f2d55e"}, - {file = "torch-1.12.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:cd26d8c5640c3a28c526d41ccdca14cf1cbca0d0f2e14e8263a7ac17194ab1d2"}, - {file = "torch-1.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:42e115dab26f60c29e298559dbec88444175528b729ae994ec4c65d56fe267dd"}, - {file = "torch-1.12.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:a8320ba9ad87e80ca5a6a016e46ada4d1ba0c54626e135d99b2129a4541c509d"}, - {file = "torch-1.12.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:03e31c37711db2cd201e02de5826de875529e45a55631d317aadce2f1ed45aa8"}, - {file = "torch-1.12.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:9b356aea223772cd754edb4d9ecf2a025909b8615a7668ac7d5130f86e7ec421"}, - {file = "torch-1.12.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:6cf6f54b43c0c30335428195589bd00e764a6d27f3b9ba637aaa8c11aaf93073"}, - {file = "torch-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:f00c721f489089dc6364a01fd84906348fe02243d0af737f944fddb36003400d"}, - {file = "torch-1.12.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:bfec2843daa654f04fda23ba823af03e7b6f7650a873cdb726752d0e3718dada"}, - {file = "torch-1.12.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:69fe2cae7c39ccadd65a123793d30e0db881f1c1927945519c5c17323131437e"}, + {file = "torch-1.13.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:fd12043868a34a8da7d490bf6db66991108b00ffbeecb034228bfcbbd4197143"}, + {file = "torch-1.13.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d9fe785d375f2e26a5d5eba5de91f89e6a3be5d11efb497e76705fdf93fa3c2e"}, + {file = "torch-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:98124598cdff4c287dbf50f53fb455f0c1e3a88022b39648102957f3445e9b76"}, + {file = "torch-1.13.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:393a6273c832e047581063fb74335ff50b4c566217019cc6ace318cd79eb0566"}, + {file = "torch-1.13.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:0122806b111b949d21fa1a5f9764d1fd2fcc4a47cb7f8ff914204fd4fc752ed5"}, + {file = "torch-1.13.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:22128502fd8f5b25ac1cd849ecb64a418382ae81dd4ce2b5cebaa09ab15b0d9b"}, + {file = "torch-1.13.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:76024be052b659ac1304ab8475ab03ea0a12124c3e7626282c9c86798ac7bc11"}, + {file = "torch-1.13.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ea8dda84d796094eb8709df0fcd6b56dc20b58fdd6bc4e8d7109930dafc8e419"}, + {file = "torch-1.13.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2ee7b81e9c457252bddd7d3da66fb1f619a5d12c24d7074de91c4ddafb832c93"}, + {file = "torch-1.13.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:0d9b8061048cfb78e675b9d2ea8503bfe30db43d583599ae8626b1263a0c1380"}, + {file = "torch-1.13.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:f402ca80b66e9fbd661ed4287d7553f7f3899d9ab54bf5c67faada1555abde28"}, + {file = "torch-1.13.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:727dbf00e2cf858052364c0e2a496684b9cb5aa01dc8a8bc8bbb7c54502bdcdd"}, + {file = "torch-1.13.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:df8434b0695e9ceb8cc70650afc1310d8ba949e6db2a0525ddd9c3b2b181e5fe"}, + {file = "torch-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:5e1e722a41f52a3f26f0c4fcec227e02c6c42f7c094f32e49d4beef7d1e213ea"}, + {file = "torch-1.13.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:33e67eea526e0bbb9151263e65417a9ef2d8fa53cbe628e87310060c9dcfa312"}, + {file = "torch-1.13.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:eeeb204d30fd40af6a2d80879b46a7efbe3cf43cdbeb8838dd4f3d126cc90b2b"}, + {file = "torch-1.13.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:50ff5e76d70074f6653d191fe4f6a42fdbe0cf942fbe2a3af0b75eaa414ac038"}, + {file = "torch-1.13.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2c3581a3fd81eb1f0f22997cddffea569fea53bafa372b2c0471db373b26aafc"}, + {file = "torch-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:0aa46f0ac95050c604bcf9ef71da9f1172e5037fdf2ebe051962d47b123848e7"}, + {file = "torch-1.13.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6930791efa8757cb6974af73d4996b6b50c592882a324b8fb0589c6a9ba2ddaf"}, + {file = "torch-1.13.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:e0df902a7c7dd6c795698532ee5970ce898672625635d885eade9976e5a04949"}, ] typer = [ {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, @@ -639,3 +767,11 @@ typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] +wheel = [ + {file = "wheel-0.38.4-py3-none-any.whl", hash = "sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8"}, + {file = "wheel-0.38.4.tar.gz", hash = "sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac"}, +] +win32-setctime = [ + {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, + {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, +] diff --git a/server/pyproject.toml b/server/pyproject.toml index f2628854..4a2ccce7 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -10,12 +10,15 @@ text-generation-server = 'text_generation.cli:app' [tool.poetry.dependencies] python = "^3.9" protobuf = "^4.21.7" -grpcio = "^1.49.1" +grpcio = "^1.51.1" +grpcio-status = "^1.51.1" +grpcio-reflection = "^1.51.1" +grpc-interceptor = "^0.15.0" typer = "^0.6.1" -grpcio-reflection = "^1.49.1" accelerate = "^0.12.0" bitsandbytes = "^0.35.1" safetensors = "^0.2.4" +loguru = "^0.6.0" [tool.poetry.extras] bnb = ["bitsandbytes"] diff --git a/server/text_generation/cli.py b/server/text_generation/cli.py index c41b9751..8101cea4 100644 --- a/server/text_generation/cli.py +++ b/server/text_generation/cli.py @@ -1,7 +1,9 @@ import os +import sys import typer from pathlib import Path +from loguru import logger from text_generation import server, utils @@ -14,7 +16,20 @@ def serve( sharded: bool = False, quantize: bool = False, uds_path: Path = "/tmp/text-generation", + logger_level: str = "INFO", + json_output: bool = False, ): + # Remove default handler + logger.remove() + logger.add( + sys.stdout, + format="{message}", + filter="text_generation", + level=logger_level, + serialize=json_output, + backtrace=True, + diagnose=False, + ) if sharded: assert ( os.getenv("RANK", None) is not None diff --git a/server/text_generation/interceptor.py b/server/text_generation/interceptor.py new file mode 100644 index 00000000..a3247d19 --- /dev/null +++ b/server/text_generation/interceptor.py @@ -0,0 +1,29 @@ +import grpc + +from google.rpc import status_pb2, code_pb2 +from grpc_status import rpc_status +from grpc_interceptor.server import AsyncServerInterceptor +from loguru import logger +from typing import Callable, Any + + +class ExceptionInterceptor(AsyncServerInterceptor): + async def intercept( + self, + method: Callable, + request_or_iterator: Any, + context: grpc.ServicerContext, + method_name: str, + ) -> Any: + try: + response = method(request_or_iterator, context) + return await response + except Exception as err: + method_name = method_name.split("/")[-1] + logger.exception(f"Method {method_name} encountered an error.") + + await context.abort_with_status( + rpc_status.to_status( + status_pb2.Status(code=code_pb2.INTERNAL, message=str(err)) + ) + ) diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 935d74ef..f561ad47 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -4,7 +4,7 @@ from text_generation.models.seq2seq_lm import Seq2SeqLM from text_generation.models.galactica import Galactica, GalacticaSharded -__all__ = ["Model", "BLOOM", "BLOOMSharded", "CausalLM", "Seq2SeqLM"] +__all__ = ["Model", "BLOOM", "BLOOMSharded", "CausalLM", "Seq2SeqLM", "get_model"] def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: diff --git a/server/text_generation/server.py b/server/text_generation/server.py index 699cef08..5fd3072e 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -2,12 +2,14 @@ import os from grpc import aio +from loguru import logger from grpc_reflection.v1alpha import reflection from pathlib import Path from typing import List from text_generation.cache import Cache +from text_generation.interceptor import ExceptionInterceptor from text_generation.models import Model, get_model from text_generation.pb import generate_pb2_grpc, generate_pb2 @@ -91,7 +93,7 @@ async def serve_inner( model = get_model(model_name, sharded, quantize) - server = aio.server() + server = aio.server(interceptors=[ExceptionInterceptor()]) generate_pb2_grpc.add_TextGenerationServiceServicer_to_server( TextGenerationService(model, Cache(), server_urls), server ) @@ -102,11 +104,11 @@ async def serve_inner( reflection.enable_server_reflection(SERVICE_NAMES, server) server.add_insecure_port(local_url) await server.start() - print("Server started at {}".format(local_url)) + logger.info("Server started at {}".format(local_url)) try: await server.wait_for_termination() except KeyboardInterrupt: - print("Signal received. Shutting down") + logger.info("Signal received. Shutting down") await server.stop(0) asyncio.run(serve_inner(model_name, sharded, quantize)) From e6d3eb5d5d257fb20caf1f86fb3cd2ef530fb555 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 17 Jan 2023 00:10:22 -0800 Subject: [PATCH 045/793] fix(server): Minor refactorization using new_zeros (#24) - Fix some type hints, in particular base tokenizer class - Make use of `tensor.new_zero/empty` methods - Simplify env var string parsing in launcher --- launcher/src/main.rs | 25 ++++--------- router/src/batcher.rs | 7 ++-- server/text_generation/models/__init__.py | 2 +- server/text_generation/models/bloom.py | 8 ++--- server/text_generation/models/causal_lm.py | 34 ++++++------------ server/text_generation/models/galactica.py | 8 ++--- server/text_generation/models/model.py | 4 +-- server/text_generation/models/seq2seq_lm.py | 39 +++++++-------------- server/text_generation/models/types.py | 4 +-- server/text_generation/utils.py | 4 +-- 10 files changed, 46 insertions(+), 89 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 86335875..51ddad20 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -289,36 +289,25 @@ fn shard_manager( } let mut env = vec![ - ("RANK".parse().unwrap(), rank.to_string().parse().unwrap()), - ( - "WORLD_SIZE".parse().unwrap(), - world_size.to_string().parse().unwrap(), - ), - ("MASTER_ADDR".parse().unwrap(), master_addr.parse().unwrap()), - ( - "MASTER_PORT".parse().unwrap(), - master_port.to_string().parse().unwrap(), - ), - ( - "SAFETENSORS_FAST_GPU".parse().unwrap(), - "1".to_string().parse().unwrap(), - ), + ("RANK".into(), rank.to_string().into()), + ("WORLD_SIZE".into(), world_size.to_string().into()), + ("MASTER_ADDR".into(), master_addr.into()), + ("MASTER_PORT".into(), master_port.to_string().into()), + ("SAFETENSORS_FAST_GPU".into(), "1".into()), ]; // If the HUGGINGFACE_HUB_CACHE env var is set, pass it to the shard // Useful when running inside a docker container if let Ok(huggingface_hub_cache) = env::var("HUGGINGFACE_HUB_CACHE") { env.push(( - "HUGGINGFACE_HUB_CACHE".parse().unwrap(), - huggingface_hub_cache.parse().unwrap(), + "HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into(), )); }; // If the CUDA_VISIBLE_DEVICES env var is set, pass it to the shard if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { env.push(( - "CUDA_VISIBLE_DEVICES".parse().unwrap(), - cuda_visible_devices.parse().unwrap(), + "CUDA_VISIBLE_DEVICES".into(), cuda_visible_devices.into(), )); }; diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 1484434c..3f2a8668 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -74,10 +74,9 @@ impl Batcher { // Await on the response from the background task // We can safely unwrap as the background task will never drop the sender - match response_rx.await.unwrap() { - Ok(output) => Ok(output), - Err(err) => Err(InferError::GenerationError(err.to_string())), - } + response_rx.await.unwrap().map_err( + |err| InferError::GenerationError(err.to_string()) + ) } } diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index f561ad47..b615eb76 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -23,5 +23,5 @@ def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: raise ValueError("sharded is not supported for AutoModel") try: return CausalLM(model_name, quantize=quantize) - except Exception as e: + except Exception: return Seq2SeqLM(model_name, quantize=quantize) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 3561a8ea..1135e565 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -5,7 +5,7 @@ from accelerate import init_empty_weights from safetensors import safe_open -from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, PreTrainedTokenizerBase from transformers.models.bloom.parallel_layers import ( TensorParallelColumnLinear, TensorParallelEmbedding, @@ -34,7 +34,7 @@ class BloomCausalLMBatch(CausalLMBatch): @classmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device ) -> "CausalLMBatch": batch = super(BloomCausalLMBatch, cls).from_pb( pb=pb, tokenizer=tokenizer, device=device @@ -203,9 +203,7 @@ def replace_linear(state, in_features, out_features): def linear(input, weight, bias): size_out = input.size()[:-1] + (out_features,) input = input.view(-1, in_features) - out = torch.empty( - size_out, device=input.device, dtype=input.dtype - ) + out = input.new_empty(size_out) out = bnb.matmul( input, weight, diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index b352eb6b..6bebcc36 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -1,17 +1,17 @@ import torch from dataclasses import dataclass -from transformers import AutoTokenizer, AutoModelForCausalLM +from transformers import AutoTokenizer, AutoModelForCausalLM, PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type from text_generation.models import Model -from text_generation.models.types import GeneratedText +from text_generation.models.types import GeneratedText, Batch from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria @dataclass -class CausalLMBatch: +class CausalLMBatch(Batch): batch_id: int requests: List[generate_pb2.Request] @@ -38,7 +38,7 @@ class CausalLMBatch: # Past metadata keys_head_dim_last: bool = True - def to_pb(self): + def to_pb(self) -> generate_pb2.Batch: return generate_pb2.Batch( id=self.batch_id, requests=self.requests, @@ -47,7 +47,7 @@ def to_pb(self): @classmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device ) -> "CausalLMBatch": inputs = [] next_token_choosers = [] @@ -130,20 +130,14 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # input_ids is always of shape [batch_size, 1] # We do not need to pad it if input_ids is None: - input_ids = torch.empty( - (total_batch_size, 1), - dtype=batch.input_ids.dtype, - device=batch.input_ids.device, - ) + input_ids = batch.input_ids.new_empty((total_batch_size, 1)) # Copy to correct indices input_ids[start_index:end_index] = batch.input_ids # Create padded tensor if attention_mask is None: - attention_mask = torch.zeros( + attention_mask = batch.attention_mask.new_zeros( (total_batch_size, max_sequence_length), - dtype=batch.attention_mask.dtype, - device=batch.attention_mask.device, ) # We need to slice the attention mask to remove padding from previous steps @@ -171,8 +165,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": if batch.keys_head_dim_last: padded_past_keys_shape = padded_past_values_shape - # seq_length is last for BLOOM else: + # seq_length is last for BLOOM padded_past_keys_shape = ( total_batch_size, num_heads, @@ -182,16 +176,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # This will run only once per layer if j == len(past_key_values): - padded_past_keys = torch.zeros( - padded_past_keys_shape, - dtype=past_keys.dtype, - device=past_keys.device, - ) - padded_past_values = torch.zeros( - padded_past_values_shape, - dtype=past_values.dtype, - device=past_values.device, - ) + padded_past_keys = past_keys.new_zeros(padded_past_keys_shape) + padded_past_values = past_values.new_zeros(padded_past_values_shape) past_key_values.append((padded_past_keys, padded_past_values)) # We slice the past keys and values to remove the padding from previous batches diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index a713e69e..76a1b1ab 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -6,7 +6,7 @@ from accelerate import init_empty_weights from safetensors import safe_open -from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, PreTrainedTokenizerBase from transformers.models.opt.parallel_layers import ( TensorParallelColumnLinear, TensorParallelEmbedding, @@ -82,7 +82,7 @@ def escape_custom_split_sequence(text): class GalacticaCausalLMBatch(CausalLMBatch): @classmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device ) -> "GalacticaCausalLMBatch": inputs = [] next_token_choosers = [] @@ -278,9 +278,7 @@ def replace_linear(state, in_features, out_features): def linear(input, weight, bias): size_out = input.size()[:-1] + (out_features,) input = input.view(-1, in_features) - out = torch.empty( - size_out, device=input.device, dtype=input.dtype - ) + out = input.new_empty(size_out) out = bnb.matmul( input, weight, diff --git a/server/text_generation/models/model.py b/server/text_generation/models/model.py index 0331e193..ef6a5682 100644 --- a/server/text_generation/models/model.py +++ b/server/text_generation/models/model.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import List, Tuple, Optional, TypeVar, Type -from tokenizers import Tokenizer +from transformers import PreTrainedTokenizerBase from text_generation.models.types import Batch, GeneratedText @@ -10,7 +10,7 @@ class Model(ABC): - def __init__(self, tokenizer: Tokenizer, device: torch.device): + def __init__(self, tokenizer: PreTrainedTokenizerBase, device: torch.device): self.tokenizer = tokenizer self.device = device diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 4095db92..c561aebe 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -1,17 +1,17 @@ import torch from dataclasses import dataclass -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM +from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type from text_generation.models import Model -from text_generation.models.types import GeneratedText +from text_generation.models.types import GeneratedText, Batch from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria @dataclass -class Seq2SeqLMBatch: +class Seq2SeqLMBatch(Batch): batch_id: int requests: List[generate_pb2.Request] @@ -41,7 +41,7 @@ class Seq2SeqLMBatch: max_input_length: int max_decoder_input_length: int - def to_pb(self): + def to_pb(self) -> generate_pb2.Batch: """Convert a Seq2SeqLMBatch to a text_generation.v1.Batch protobuf""" return generate_pb2.Batch( id=self.batch_id, @@ -51,7 +51,7 @@ def to_pb(self): @classmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device ) -> "Seq2SeqLMBatch": """Convert a text_generation.v1.Batch protobuf to a Seq2SeqLMBatch""" inputs = [] @@ -155,10 +155,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Create padded tensor if input_ids is None: - input_ids = torch.zeros( + input_ids = batch.input_ids.new_zeros( (total_batch_size, max_input_length), - dtype=batch.input_ids.dtype, - device=batch.input_ids.device, ) # Copy to correct indices input_ids[ @@ -167,10 +165,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Create padded tensor if attention_mask is None: - attention_mask = torch.zeros( + attention_mask = batch.attention_mask.new_zeros( (total_batch_size, max_input_length), - dtype=batch.attention_mask.dtype, - device=batch.attention_mask.device, ) # Copy to correct indices attention_mask[ @@ -179,10 +175,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Create padded tensor if decoder_input_ids is None: - decoder_input_ids = torch.zeros( + decoder_input_ids = batch.decoder_input_ids.new_zeros( (total_batch_size, max_decoder_input_length), - dtype=batch.decoder_input_ids.dtype, - device=batch.decoder_input_ids.device, ) # Copy to correct indices decoder_input_ids[ @@ -191,10 +185,9 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Create padded tensor if decoder_attention_mask is None: - decoder_attention_mask = torch.zeros( + # As decoder_attention_mask might not exist, we use `batch.attention_mask` for device here + decoder_attention_mask = batch.attention_mask.new_zeros( (total_batch_size, max_decoder_input_length), - dtype=batch.attention_mask.dtype, # As decoder_attention_mask might not exist, - device=batch.attention_mask.device, # we use `batch.attention_maks` for device here ) # If the decoder mask does not exist yet, all generations started at the same time and we never concatenated # this batch. All generations are of length `batch.max_decoder_input_length`. @@ -210,14 +203,12 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Create padded tensor if encoder_last_hidden_state is None: - encoder_last_hidden_state = torch.zeros( + encoder_last_hidden_state = batch.encoder_last_hidden_state.new_zeros( ( total_batch_size, max_input_length, batch.encoder_last_hidden_state.shape[-1], ), - dtype=batch.encoder_last_hidden_state.dtype, - device=batch.encoder_last_hidden_state.device, ) # Copy to correct indices @@ -245,9 +236,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Initialize tensors # This will run only once per layer and per past tensor if k == len(past_key_values[j]): - past_key_values[j].append( - torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) - ) + past_key_values[j].append(t.new_zeros(padded_t_shape)) # We slice the past keys and values to remove the padding from previous batches past_key_values[j][k][ @@ -271,9 +260,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Initialize tensors # This will run only once per layer and per past tensor if idx == len(past_key_values[j]): - past_key_values[j].append( - torch.zeros(padded_t_shape, dtype=t.dtype, device=t.device) - ) + past_key_values[j].append(t.new_zeros(padded_t_shape)) past_key_values[j][idx][ start_index:end_index, :, -batch.max_input_length :, : diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index e76cf697..fa0dc9a0 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import List -from transformers import AutoTokenizer +from transformers import PreTrainedTokenizerBase from text_generation.pb import generate_pb2 @@ -17,7 +17,7 @@ def to_pb(self) -> generate_pb2.Batch: @classmethod @abstractmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: AutoTokenizer, device: torch.device + cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device ) -> "Batch": raise NotImplementedError diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index cc779736..b0b5b072 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -12,7 +12,7 @@ from huggingface_hub.utils import LocalEntryNotFoundError from tqdm import tqdm from typing import List, Optional, Tuple -from transformers import AutoTokenizer +from transformers import PreTrainedTokenizerBase from transformers.generation.logits_process import ( LogitsProcessorList, TemperatureLogitsWarper, @@ -114,7 +114,7 @@ def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[st @classmethod def from_pb( - cls, pb: generate_pb2.StoppingCriteriaParameters, tokenizer: AutoTokenizer + cls, pb: generate_pb2.StoppingCriteriaParameters, tokenizer: PreTrainedTokenizerBase ) -> "StoppingCriteria": stop_sequence_criterias = [ StopSequenceCriteria(sequence) for sequence in pb.stop_sequences From f7ac394935f9ce502d827a1e8b3be2396c44f950 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 17 Jan 2023 00:11:21 -0800 Subject: [PATCH 046/793] fix(router): Obey max batch size (#23) --- router/src/batcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 3f2a8668..90ee4091 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -127,7 +127,7 @@ async fn batching_task( // Try to get a new batch if let Some((new_request_ids, new_batch)) = - db.next_batch(min_size, max_batch_size) + db.next_batch(min_size, max_batch_size - batch_size as usize) { // Generate one token for this new batch to have the attention past in cache let new_cached_batch = From 15511edc01a0725d374840f0e77d085eb5821483 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 20 Jan 2023 12:24:39 +0100 Subject: [PATCH 047/793] feat(server): Support SantaCoder (#26) --- README.md | 1 + launcher/src/main.rs | 10 +-- router/src/batcher.rs | 7 +- router/src/validation.rs | 16 ++-- server/tests/conftest.py | 23 ----- server/tests/models/test_bloom.py | 16 ++-- server/tests/models/test_causal_lm.py | 18 ++-- server/tests/models/test_santacoder.py | 93 +++++++++++++++++++++ server/tests/models/test_seq2seq_lm.py | 21 +++-- server/text_generation/models/__init__.py | 22 ++++- server/text_generation/models/bloom.py | 19 +++-- server/text_generation/models/causal_lm.py | 19 +++-- server/text_generation/models/galactica.py | 26 +++++- server/text_generation/models/santacoder.py | 87 +++++++++++++++++++ server/text_generation/models/seq2seq_lm.py | 11 ++- server/text_generation/models/types.py | 5 +- server/text_generation/utils.py | 4 +- 17 files changed, 320 insertions(+), 78 deletions(-) create mode 100644 server/tests/models/test_santacoder.py create mode 100644 server/text_generation/models/santacoder.py diff --git a/README.md b/README.md index 8496ce8f..2635f641 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ to power Bloom, BloomZ and MT0-XXL api-inference widgets. - [BLOOMZ](https://huggingface.co/bigscience/bloomz) - [MT0-XXL](https://huggingface.co/bigscience/mt0-xxl) - ~~[Galactica](https://huggingface.co/facebook/galactica-120b)~~ (deactivated) +- [SantaCoder](https://huggingface.co/bigcode/santacoder) Other models are supported on a best effort basis using: diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 51ddad20..f897d2cd 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -1,4 +1,5 @@ use clap::Parser; +use serde_json::Value; use std::env; use std::io::{BufRead, BufReader, Read}; use std::path::Path; @@ -11,7 +12,6 @@ use std::thread; use std::thread::sleep; use std::time::{Duration, Instant}; use std::{fs, io}; -use serde_json::Value; use subprocess::{Popen, PopenConfig, PopenError, Redirection}; /// App Configuration @@ -299,16 +299,12 @@ fn shard_manager( // If the HUGGINGFACE_HUB_CACHE env var is set, pass it to the shard // Useful when running inside a docker container if let Ok(huggingface_hub_cache) = env::var("HUGGINGFACE_HUB_CACHE") { - env.push(( - "HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into(), - )); + env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); }; // If the CUDA_VISIBLE_DEVICES env var is set, pass it to the shard if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { - env.push(( - "CUDA_VISIBLE_DEVICES".into(), cuda_visible_devices.into(), - )); + env.push(("CUDA_VISIBLE_DEVICES".into(), cuda_visible_devices.into())); }; // Start process diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 90ee4091..ee83d899 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -74,9 +74,10 @@ impl Batcher { // Await on the response from the background task // We can safely unwrap as the background task will never drop the sender - response_rx.await.unwrap().map_err( - |err| InferError::GenerationError(err.to_string()) - ) + response_rx + .await + .unwrap() + .map_err(|err| InferError::GenerationError(err.to_string())) } } diff --git a/router/src/validation.rs b/router/src/validation.rs index 4a9d0c23..aabc82a6 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -94,7 +94,9 @@ fn validation_worker( ) { // Loop over requests while let Some((request, response_tx)) = receiver.blocking_recv() { - response_tx.send(validate(request, &tokenizer, max_input_length)).unwrap_or(()) + response_tx + .send(validate(request, &tokenizer, max_input_length)) + .unwrap_or(()) } } @@ -117,8 +119,9 @@ fn validate( } if request.parameters.stop.len() > MAX_STOP_SEQUENCES { return Err(ValidationError::StopSequence( - MAX_STOP_SEQUENCES, request.parameters.stop.len(), - )) + MAX_STOP_SEQUENCES, + request.parameters.stop.len(), + )); } // Get the number of tokens in the input @@ -127,14 +130,11 @@ fn validate( let input_length = inputs.len(); if input_length > max_input_length { - Err(ValidationError::InputLength( - input_length, - max_input_length, - )) + Err(ValidationError::InputLength(input_length, max_input_length)) } else { Ok((input_length, request)) } - }, + } Err(err) => Err(ValidationError::Tokenizer(err.to_string())), } } diff --git a/server/tests/conftest.py b/server/tests/conftest.py index eb72b8a2..e0ed76b4 100644 --- a/server/tests/conftest.py +++ b/server/tests/conftest.py @@ -1,7 +1,5 @@ import pytest -from transformers import AutoTokenizer - from text_generation.pb import generate_pb2 @@ -18,24 +16,3 @@ def default_pb_parameters(): @pytest.fixture def default_pb_stop_parameters(): return generate_pb2.StoppingCriteriaParameters(stop_sequences=[], max_new_tokens=10) - - -@pytest.fixture(scope="session") -def bloom_560m_tokenizer(): - return AutoTokenizer.from_pretrained("bigscience/bloom-560m", padding_side="left") - - -@pytest.fixture(scope="session") -def gpt2_tokenizer(): - tokenizer = AutoTokenizer.from_pretrained("gpt2", padding_side="left") - tokenizer.pad_token_id = 50256 - return tokenizer - - -@pytest.fixture(scope="session") -def mt0_small_tokenizer(): - tokenizer = AutoTokenizer.from_pretrained( - "bigscience/mt0-small", padding_side="left" - ) - tokenizer.bos_token_id = 0 - return tokenizer diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 2a6e670e..1a788ce5 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -2,12 +2,23 @@ import torch from copy import copy +from transformers import AutoTokenizer from text_generation.pb import generate_pb2 from text_generation.models.causal_lm import CausalLMBatch from text_generation.models.bloom import BloomCausalLMBatch, BLOOM +@pytest.fixture(scope="session") +def default_bloom(): + return BLOOM("bigscience/bloom-560m") + + +@pytest.fixture(scope="session") +def bloom_560m_tokenizer(): + return AutoTokenizer.from_pretrained("bigscience/bloom-560m", padding_side="left") + + @pytest.fixture def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( @@ -44,11 +55,6 @@ def default_multi_requests_bloom_batch(default_pb_request, bloom_560m_tokenizer) ) -@pytest.fixture(scope="session") -def default_bloom(): - return BLOOM("bigscience/bloom-560m") - - def test_batch_from_pb(default_pb_batch, default_bloom_batch): batch = default_bloom_batch diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 683d9fdd..bedb65ba 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -2,11 +2,24 @@ import torch from copy import copy +from transformers import AutoTokenizer from text_generation.pb import generate_pb2 from text_generation.models.causal_lm import CausalLM, CausalLMBatch +@pytest.fixture(scope="session") +def default_causal_lm(): + return CausalLM("gpt2") + + +@pytest.fixture(scope="session") +def gpt2_tokenizer(): + tokenizer = AutoTokenizer.from_pretrained("gpt2", padding_side="left") + tokenizer.pad_token_id = 50256 + return tokenizer + + @pytest.fixture def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( @@ -39,11 +52,6 @@ def default_multi_requests_causal_lm_batch(default_pb_request, gpt2_tokenizer): return CausalLMBatch.from_pb(batch_pb, gpt2_tokenizer, torch.device("cpu")) -@pytest.fixture(scope="session") -def default_causal_lm(): - return CausalLM("gpt2") - - def test_batch_from_pb(default_pb_batch, default_causal_lm_batch): batch = default_causal_lm_batch diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py new file mode 100644 index 00000000..c3a83753 --- /dev/null +++ b/server/tests/models/test_santacoder.py @@ -0,0 +1,93 @@ +import pytest + +from text_generation.pb import generate_pb2 +from text_generation.models.causal_lm import CausalLMBatch +from text_generation.models.santacoder import SantaCoder + + +@pytest.fixture(scope="session") +def default_santacoder(): + return SantaCoder("bigcode/santacoder") + + +@pytest.fixture +def default_pb_request(default_pb_parameters, default_pb_stop_parameters): + return generate_pb2.Request( + id=0, + inputs="def", + input_length=1, + parameters=default_pb_parameters, + stopping_parameters=default_pb_stop_parameters, + ) + + +@pytest.fixture +def default_pb_batch(default_pb_request): + return generate_pb2.Batch(id=0, requests=[default_pb_request], size=1) + + +@pytest.fixture +def default_fim_pb_request(default_pb_parameters, default_pb_stop_parameters): + return generate_pb2.Request( + id=0, + inputs="defworld", + input_length=5, + parameters=default_pb_parameters, + stopping_parameters=default_pb_stop_parameters, + ) + + +@pytest.fixture +def default_fim_pb_batch(default_fim_pb_request): + return generate_pb2.Batch(id=0, requests=[default_fim_pb_request], size=1) + + +def test_santacoder_generate_token_completion(default_santacoder, default_pb_batch): + batch = CausalLMBatch.from_pb( + default_pb_batch, default_santacoder.tokenizer, default_santacoder.device + ) + next_batch = batch + + for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): + generated_texts, next_batch = default_santacoder.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_santacoder.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "def test_get_all_users_with_" + assert generated_texts[0].request == batch.requests[0] + assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) + assert ( + generated_texts[0].generated_tokens + == batch.stopping_criterias[0].max_new_tokens + ) + + +def test_fim_santacoder_generate_token_completion( + default_santacoder, default_fim_pb_batch +): + batch = CausalLMBatch.from_pb( + default_fim_pb_batch, default_santacoder.tokenizer, default_santacoder.device + ) + next_batch = batch + + for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): + generated_texts, next_batch = default_santacoder.generate_token(next_batch) + assert generated_texts == [] + + generated_texts, next_batch = default_santacoder.generate_token(next_batch) + assert next_batch is None + + assert len(generated_texts) == 1 + assert ( + generated_texts[0].output_text + == """defworldineProperty(exports, "__esModule", { value""" + ) + assert generated_texts[0].request == batch.requests[0] + assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) + assert ( + generated_texts[0].generated_tokens + == batch.stopping_criterias[0].max_new_tokens + ) diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index f1b11bc2..de1a4829 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -3,10 +3,26 @@ from copy import copy +from transformers import AutoTokenizer + from text_generation.pb import generate_pb2 from text_generation.models.seq2seq_lm import Seq2SeqLM, Seq2SeqLMBatch +@pytest.fixture(scope="session") +def mt0_small_tokenizer(): + tokenizer = AutoTokenizer.from_pretrained( + "bigscience/mt0-small", padding_side="left" + ) + tokenizer.bos_token_id = 0 + return tokenizer + + +@pytest.fixture(scope="session") +def default_seq2seq_lm(): + return Seq2SeqLM("bigscience/mt0-small") + + @pytest.fixture def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( @@ -41,11 +57,6 @@ def default_multi_requests_seq2seq_lm_batch(default_pb_request, mt0_small_tokeni return Seq2SeqLMBatch.from_pb(batch_pb, mt0_small_tokenizer, torch.device("cpu")) -@pytest.fixture(scope="session") -def default_seq2seq_lm(): - return Seq2SeqLM("bigscience/mt0-small") - - def test_batch_from_pb(default_pb_batch, default_seq2seq_lm_batch): batch = default_seq2seq_lm_batch sequence_length = len(default_seq2seq_lm_batch.input_ids[0]) diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index b615eb76..41d73815 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -1,10 +1,28 @@ +import torch + from text_generation.models.model import Model from text_generation.models.causal_lm import CausalLM from text_generation.models.bloom import BLOOM, BLOOMSharded from text_generation.models.seq2seq_lm import Seq2SeqLM from text_generation.models.galactica import Galactica, GalacticaSharded +from text_generation.models.santacoder import SantaCoder + +__all__ = [ + "Model", + "BLOOM", + "BLOOMSharded", + "CausalLM", + "Seq2SeqLM", + "SantaCoder", + "get_model", +] + +# The flag below controls whether to allow TF32 on matmul. This flag defaults to False +# in PyTorch 1.12 and later. +torch.backends.cuda.matmul.allow_tf32 = True -__all__ = ["Model", "BLOOM", "BLOOMSharded", "CausalLM", "Seq2SeqLM", "get_model"] +# The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. +torch.backends.cudnn.allow_tf32 = True def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: @@ -18,6 +36,8 @@ def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: return GalacticaSharded(model_name, quantize=quantize) else: return Galactica(model_name, quantize=quantize) + elif "santacoder" in model_name: + return SantaCoder(model_name, quantize) else: if sharded: raise ValueError("sharded is not supported for AutoModel") diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 1135e565..375fff4b 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -5,7 +5,12 @@ from accelerate import init_empty_weights from safetensors import safe_open -from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, PreTrainedTokenizerBase +from transformers import ( + AutoTokenizer, + AutoModelForCausalLM, + AutoConfig, + PreTrainedTokenizerBase, +) from transformers.models.bloom.parallel_layers import ( TensorParallelColumnLinear, TensorParallelEmbedding, @@ -34,7 +39,10 @@ class BloomCausalLMBatch(CausalLMBatch): @classmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + device: torch.device, ) -> "CausalLMBatch": batch = super(BloomCausalLMBatch, cls).from_pb( pb=pb, tokenizer=tokenizer, device=device @@ -70,13 +78,6 @@ def __init__(self, model_name: str, quantize: bool = False): ) config.pad_token_id = 3 - # The flag below controls whether to allow TF32 on matmul. This flag defaults to False - # in PyTorch 1.12 and later. - torch.backends.cuda.matmul.allow_tf32 = True - - # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. - torch.backends.cudnn.allow_tf32 = True - # Only download weights for small models if self.master and model_name == "bigscience/bloom-560m": download_weights(model_name, extension=".safetensors") diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 6bebcc36..93f35204 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -47,7 +47,10 @@ def to_pb(self) -> generate_pb2.Batch: @classmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + device: torch.device, ) -> "CausalLMBatch": inputs = [] next_token_choosers = [] @@ -71,6 +74,7 @@ def from_pb( return_tensors="pt", padding=True, pad_to_multiple_of=pad_to_multiple_of, + return_token_type_ids=False, ).to(device) all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) @@ -253,6 +257,11 @@ def __init__(self, model_name: str, quantize=False): def batch_type(self) -> Type[CausalLMBatch]: return CausalLMBatch + def decode(self, generated_ids: List[int]) -> str: + return self.tokenizer.decode( + generated_ids, skip_special_tokens=True, cleanup_tokenization_spaces=False + ) + def forward( self, input_ids, attention_mask, past_key_values: Optional = None ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: @@ -338,11 +347,11 @@ def generate_token( ), ) if stop: - # Decode all tokens - output_text = self.tokenizer.decode( - all_input_ids.squeeze(-1), skip_special_tokens=True, - cleanup_tokenization_spaces=False + # Decode generated tokens + generated_text = self.decode( + all_input_ids[-stopping_criteria.current_tokens :, 0] ) + output_text = request.inputs + generated_text # Slice with input_length to remove padding token_ids = all_input_ids[-new_input_length:] tokens = self.tokenizer.batch_decode(token_ids) diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index 76a1b1ab..26fb3dd6 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -6,7 +6,12 @@ from accelerate import init_empty_weights from safetensors import safe_open -from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, PreTrainedTokenizerBase +from transformers import ( + AutoTokenizer, + AutoModelForCausalLM, + AutoConfig, + PreTrainedTokenizerBase, +) from transformers.models.opt.parallel_layers import ( TensorParallelColumnLinear, TensorParallelEmbedding, @@ -82,7 +87,10 @@ def escape_custom_split_sequence(text): class GalacticaCausalLMBatch(CausalLMBatch): @classmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + device: torch.device, ) -> "GalacticaCausalLMBatch": inputs = [] next_token_choosers = [] @@ -99,8 +107,14 @@ def from_pb( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) + # Tokenize batch + pad_to_multiple_of = 8 if device.type == "cuda" else None tokenized_inputs = tokenizer( - inputs, return_tensors="pt", padding=True, pad_to_multiple_of=8 + inputs, + return_tensors="pt", + padding=True, + pad_to_multiple_of=pad_to_multiple_of, + return_token_type_ids=False, ).to(device) all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) @@ -124,6 +138,12 @@ class Galactica(CausalLM): def batch_type(self) -> Type[CausalLMBatch]: return GalacticaCausalLMBatch + def decode(self, generated_ids: List[int]) -> str: + # Do not skip special tokens as they are used for custom parsing rules of the generated text + return self.tokenizer.decode( + generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False + ) + class GalacticaSharded(Galactica): def __init__(self, model_name: str, quantize: bool = False): diff --git a/server/text_generation/models/santacoder.py b/server/text_generation/models/santacoder.py new file mode 100644 index 00000000..e1d8e6ac --- /dev/null +++ b/server/text_generation/models/santacoder.py @@ -0,0 +1,87 @@ +import torch +import torch.distributed + +from typing import Optional, List, Tuple +from transformers import AutoTokenizer, AutoModelForCausalLM + +from text_generation.models import CausalLM + +FIM_PREFIX = "" +FIM_MIDDLE = "" +FIM_SUFFIX = "" +FIM_PAD = "" +EOD = "<|endoftext|>" + + +class SantaCoder(CausalLM): + def __init__(self, model_name: str, quantize=False): + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + else: + if quantize: + raise ValueError("quantization is not available on CPU") + + device = torch.device("cpu") + dtype = torch.float32 + + tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + tokenizer.add_special_tokens( + { + "additional_special_tokens": [ + EOD, + FIM_PREFIX, + FIM_MIDDLE, + FIM_SUFFIX, + FIM_PAD, + ], + "pad_token": EOD, + } + ) + + self.model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=dtype, + device_map="auto" if torch.cuda.is_available() else None, + load_in_8bit=quantize, + trust_remote_code=True, # required + ).eval() + + super(CausalLM, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + def decode(self, generated_ids: List[int]) -> str: + # Do not skip special tokens as they are used for custom parsing rules of the generated text + return self.tokenizer.decode( + generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False + ) + + def forward( + self, input_ids, attention_mask, past_key_values: Optional = None + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + # FIXME: current forward with past is bugged for bigcode/santacoder because past_key_values does not have + # the correct shape ([batch_size, D, seq_length] instead of [batch_size, seq_length D] + # this leads to position_ids being wrong + + input_length = input_ids.shape[-1] + past_key_values_length = ( + 0 if past_key_values is None else past_key_values[0][0].shape[-1] + ) + position_ids = torch.arange( + past_key_values_length, + input_length + past_key_values_length, + dtype=torch.long, + device=input_ids.device, + ).view(1, input_length) + + # Model Forward + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + position_ids=position_ids, + use_cache=True, + ) + return outputs.logits, outputs.past_key_values diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index c561aebe..2980c74a 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -51,7 +51,10 @@ def to_pb(self) -> generate_pb2.Batch: @classmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + device: torch.device, ) -> "Seq2SeqLMBatch": """Convert a text_generation.v1.Batch protobuf to a Seq2SeqLMBatch""" inputs = [] @@ -83,6 +86,7 @@ def from_pb( return_tensors="pt", padding=True, pad_to_multiple_of=pad_to_multiple_of, + return_token_type_ids=False, ).to(device) # Convert decoder_input_ids to torch tensor of size [batch_size, 1] decoder_input_ids = torch.tensor(decoder_input_ids, device=device).unsqueeze(-1) @@ -318,6 +322,9 @@ def __init__(self, model_name: str, quantize=False): def batch_type(self) -> Type[Seq2SeqLMBatch]: return Seq2SeqLMBatch + def decode(self, decoder_ids: List[int]) -> str: + return self.tokenizer.decode(decoder_ids, skip_special_tokens=True) + def forward( self, input_ids, @@ -438,7 +445,7 @@ def generate_token( # Slice with decoder_input_length to remove padding # Decode all tokens token_ids = decoder_input_ids[-new_decoder_input_length:] - output_text = self.tokenizer.decode(token_ids, skip_special_tokens=True) + output_text = self.decode(token_ids) tokens = self.tokenizer.batch_decode(token_ids) # Add NaN for the bos token logprobs = [float("nan")] + decoder_logprobs[ diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index fa0dc9a0..6bf64e05 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -17,7 +17,10 @@ def to_pb(self) -> generate_pb2.Batch: @classmethod @abstractmethod def from_pb( - cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + device: torch.device, ) -> "Batch": raise NotImplementedError diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index b0b5b072..1ddeed6e 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -114,7 +114,9 @@ def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[st @classmethod def from_pb( - cls, pb: generate_pb2.StoppingCriteriaParameters, tokenizer: PreTrainedTokenizerBase + cls, + pb: generate_pb2.StoppingCriteriaParameters, + tokenizer: PreTrainedTokenizerBase, ) -> "StoppingCriteria": stop_sequence_criterias = [ StopSequenceCriteria(sequence) for sequence in pb.stop_sequences From 1f570d181f4836a430f9b92f001a7b834ea561e3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 20 Jan 2023 15:35:22 +0100 Subject: [PATCH 048/793] fix(server): Fix position ids (#28) --- server/tests/models/test_santacoder.py | 2 ++ server/text_generation/models/bloom.py | 3 +- server/text_generation/models/causal_lm.py | 25 +++++++++++++++-- server/text_generation/models/galactica.py | 6 +++- server/text_generation/models/santacoder.py | 31 +-------------------- server/text_generation/models/seq2seq_lm.py | 2 +- 6 files changed, 33 insertions(+), 36 deletions(-) diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index c3a83753..acebec04 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -42,6 +42,7 @@ def default_fim_pb_batch(default_fim_pb_request): return generate_pb2.Batch(id=0, requests=[default_fim_pb_request], size=1) +@pytest.mark.skip def test_santacoder_generate_token_completion(default_santacoder, default_pb_batch): batch = CausalLMBatch.from_pb( default_pb_batch, default_santacoder.tokenizer, default_santacoder.device @@ -65,6 +66,7 @@ def test_santacoder_generate_token_completion(default_santacoder, default_pb_bat ) +@pytest.mark.skip def test_fim_santacoder_generate_token_completion( default_santacoder, default_fim_pb_batch ): diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 375fff4b..2218b91b 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -236,10 +236,11 @@ def linear(input, weight, bias): if name == "word_embeddings.weight": model.lm_head._parameters["weight"] = tensor - def forward(self, input_ids, attention_mask, past_key_values: Optional = None): + def forward(self, input_ids, attention_mask, position_ids, past_key_values: Optional = None): outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask, + position_ids=position_ids, past_key_values=past_key_values, use_cache=True, ) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 93f35204..6e35b2ad 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -18,6 +18,7 @@ class CausalLMBatch(Batch): # Decoder values input_ids: torch.Tensor attention_mask: torch.Tensor + position_ids: torch.Tensor past_key_values: Optional[List[Tuple]] # All tokens @@ -76,6 +77,8 @@ def from_pb( pad_to_multiple_of=pad_to_multiple_of, return_token_type_ids=False, ).to(device) + position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 + position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) return cls( @@ -83,6 +86,7 @@ def from_pb( requests=pb.requests, input_ids=tokenized_inputs["input_ids"], attention_mask=tokenized_inputs["attention_mask"], + position_ids=position_ids, past_key_values=None, all_input_ids=all_input_ids, all_logprobs=all_logprobs, @@ -110,6 +114,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # Batch tensors input_ids = None attention_mask = None + position_ids = None past_key_values = [] # Used for slicing correctly inside the tensors @@ -149,6 +154,12 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": start_index:end_index, -batch.max_sequence_length : ] = batch.attention_mask[:, -batch.max_sequence_length :] + # Create empty tensor + # position_ids is always of shape [batch_size, 1] + if position_ids is None: + position_ids = batch.position_ids.new_empty((total_batch_size, 1)) + position_ids[start_index:end_index] = batch.position_ids + for j, past in enumerate(batch.past_key_values): past_keys, past_values = past @@ -211,6 +222,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests=requests, input_ids=input_ids, attention_mask=attention_mask, + position_ids=position_ids, past_key_values=past_key_values, all_input_ids=all_input_ids, all_logprobs=all_logprobs, @@ -263,12 +275,13 @@ def decode(self, generated_ids: List[int]) -> str: ) def forward( - self, input_ids, attention_mask, past_key_values: Optional = None + self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: # Model Forward outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask, + position_ids=position_ids, past_key_values=past_key_values, use_cache=True, ) @@ -283,7 +296,7 @@ def generate_token( ) with context_manager(): logits, past = self.forward( - batch.input_ids, batch.attention_mask, batch.past_key_values + batch.input_ids, batch.attention_mask, batch.position_ids, batch.past_key_values ) # List of indices to cache @@ -356,7 +369,7 @@ def generate_token( token_ids = all_input_ids[-new_input_length:] tokens = self.tokenizer.batch_decode(token_ids) # Add NaN for the first prompt token - logprobs = [float("nan")] + all_logprobs[-new_input_length:].squeeze( + logprobs = [float("nan")] + all_logprobs[-input_length:].squeeze( 1 ).tolist() @@ -394,6 +407,7 @@ def generate_token( if generated_texts: # Apply indices to attention mask, past key values and other items that need to be cached next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] + next_batch_position_ids = batch.position_ids[next_batch_keep_indices] # Force past to be of dim [batch_size, num_heads, ...] for easy indexing next_batch_past_key_values = [ [ @@ -411,6 +425,7 @@ def generate_token( ] else: next_batch_attention_mask = batch.attention_mask + next_batch_position_ids = batch.position_ids next_batch_past_key_values = past next_batch_requests = batch.requests next_batch_next_token_choosers = batch.next_token_choosers @@ -425,11 +440,15 @@ def generate_token( dim=1, ) + # Update position_ids + next_batch_position_ids = next_batch_position_ids[:, -1:] + 1 + next_batch = CausalLMBatch( batch_id=batch.batch_id, requests=next_batch_requests, input_ids=next_batch_input_ids, attention_mask=next_batch_attention_mask, + position_ids=next_batch_position_ids, past_key_values=next_batch_past_key_values, all_input_ids=next_batch_all_input_ids, all_logprobs=next_batch_all_logprobs, diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index 26fb3dd6..b56fc748 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -116,6 +116,8 @@ def from_pb( pad_to_multiple_of=pad_to_multiple_of, return_token_type_ids=False, ).to(device) + position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 + position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) return cls( @@ -123,6 +125,7 @@ def from_pb( requests=pb.requests, input_ids=tokenized_inputs["input_ids"], attention_mask=tokenized_inputs["attention_mask"], + position_ids=position_ids, past_key_values=None, all_input_ids=all_input_ids, input_lengths=input_lengths, @@ -330,10 +333,11 @@ def linear(input, weight, bias): if name == "model.decoder.embed_tokens.weight": model.lm_head._parameters["weight"] = tensor - def forward(self, input_ids, attention_mask, past_key_values: Optional = None): + def forward(self, input_ids, attention_mask, position_ids, past_key_values: Optional = None): outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask, + position_ids=position_ids, past_key_values=past_key_values, use_cache=True, ) diff --git a/server/text_generation/models/santacoder.py b/server/text_generation/models/santacoder.py index e1d8e6ac..cf9f450c 100644 --- a/server/text_generation/models/santacoder.py +++ b/server/text_generation/models/santacoder.py @@ -42,10 +42,9 @@ def __init__(self, model_name: str, quantize=False): self.model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=dtype, - device_map="auto" if torch.cuda.is_available() else None, load_in_8bit=quantize, trust_remote_code=True, # required - ).eval() + ).to(device).eval() super(CausalLM, self).__init__( tokenizer=tokenizer, @@ -57,31 +56,3 @@ def decode(self, generated_ids: List[int]) -> str: return self.tokenizer.decode( generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False ) - - def forward( - self, input_ids, attention_mask, past_key_values: Optional = None - ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: - # FIXME: current forward with past is bugged for bigcode/santacoder because past_key_values does not have - # the correct shape ([batch_size, D, seq_length] instead of [batch_size, seq_length D] - # this leads to position_ids being wrong - - input_length = input_ids.shape[-1] - past_key_values_length = ( - 0 if past_key_values is None else past_key_values[0][0].shape[-1] - ) - position_ids = torch.arange( - past_key_values_length, - input_length + past_key_values_length, - dtype=torch.long, - device=input_ids.device, - ).view(1, input_length) - - # Model Forward - outputs = self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - position_ids=position_ids, - use_cache=True, - ) - return outputs.logits, outputs.past_key_values diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 2980c74a..8390f89b 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -449,7 +449,7 @@ def generate_token( tokens = self.tokenizer.batch_decode(token_ids) # Add NaN for the bos token logprobs = [float("nan")] + decoder_logprobs[ - -new_decoder_input_length: + -decoder_input_length: ].tolist() # Add to the list of finished generations with the original request generated_texts.append( From f9d0ec376a6ddafb07828913968b9ed7c367fb0c Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 23 Jan 2023 17:11:27 +0100 Subject: [PATCH 049/793] feat(docker): Make the image compatible with api-inference (#29) --- Dockerfile | 6 +++--- aml/deployment.yaml | 2 +- router/src/server.rs | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e8455b03..932d85ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,10 +26,10 @@ FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ DEBIAN_FRONTEND=noninteractive \ - MODEL_BASE_PATH=/var/azureml-model \ - MODEL_NAME=bigscience/bloom \ + MODEL_BASE_PATH=/data \ + MODEL_ID=bigscience/bloom-560m \ QUANTIZE=false \ - NUM_GPUS=8 \ + NUM_GPUS=1 \ SAFETENSORS_FAST_GPU=1 \ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ NCCL_ASYNC_ERROR_HANDLING=1 \ diff --git a/aml/deployment.yaml b/aml/deployment.yaml index 51e124b6..59fdf59a 100644 --- a/aml/deployment.yaml +++ b/aml/deployment.yaml @@ -5,7 +5,7 @@ model: azureml:bloom:1 model_mount_path: /var/azureml-model environment_variables: MODEL_BASE_PATH: /var/azureml-model/bloom - MODEL_NAME: bigscience/bloom + MODEL_ID: bigscience/bloom NUM_GPUS: 8 environment: image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.3.1 diff --git a/router/src/server.rs b/router/src/server.rs index 2e6c473f..db6c369b 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -197,6 +197,7 @@ pub async fn run( let app = Router::new() .route("/generate", post(generate)) .layer(Extension(shared_state.clone())) + .route("/", get(health)) .route("/health", get(health)) .layer(Extension(shared_state.clone())); From ab2ad91da38aec963e25917cf2e3f7688e6600db Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 23 Jan 2023 17:33:08 +0100 Subject: [PATCH 050/793] fix(docker): fix api-inference deployment (#30) --- Dockerfile | 3 ++- aml/deployment.yaml | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 932d85ac..801f29d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,7 @@ ENV LANG=C.UTF-8 \ QUANTIZE=false \ NUM_GPUS=1 \ SAFETENSORS_FAST_GPU=1 \ + PORT=80 \ CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ NCCL_ASYNC_ERROR_HANDLING=1 \ CUDA_HOME=/usr/local/cuda \ @@ -70,4 +71,4 @@ COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/loca # Install launcher COPY --from=launcher-builder /usr/local/cargo/bin/text-generation-launcher /usr/local/bin/text-generation-launcher -CMD HUGGINGFACE_HUB_CACHE=$MODEL_BASE_PATH text-generation-launcher --num-shard $NUM_GPUS --json-output \ No newline at end of file +CMD HUGGINGFACE_HUB_CACHE=$MODEL_BASE_PATH text-generation-launcher --num-shard $NUM_GPUS --model-name $MODEL_ID --json-output \ No newline at end of file diff --git a/aml/deployment.yaml b/aml/deployment.yaml index 59fdf59a..67690722 100644 --- a/aml/deployment.yaml +++ b/aml/deployment.yaml @@ -11,13 +11,13 @@ environment: image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.3.1 inference_config: liveness_route: - port: 3000 + port: 80 path: /health readiness_route: - port: 3000 + port: 80 path: /health scoring_route: - port: 3000 + port: 80 path: /generate instance_type: Standard_ND96amsr_A100_v4 request_settings: From 5c01e2544c0a2a91b2e9aff8a24c18da9b485a37 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 23 Jan 2023 17:42:14 +0100 Subject: [PATCH 051/793] fix(router): fix api-inference deployment (#31) --- router/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/src/server.rs b/router/src/server.rs index db6c369b..623dd07c 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -195,8 +195,8 @@ pub async fn run( // Create router let app = Router::new() + .route("/", post(generate)) .route("/generate", post(generate)) - .layer(Extension(shared_state.clone())) .route("/", get(health)) .route("/health", get(health)) .layer(Extension(shared_state.clone())); From 13e7044ab7eb2f39d8d4418f225d8786befa082b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 24 Jan 2023 19:52:39 +0100 Subject: [PATCH 052/793] fix(dockerfile): fix docker build (#32) --- Dockerfile | 2 +- server/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 801f29d4..99fe6dde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,7 @@ COPY server/Makefile server/Makefile RUN cd server && make install-torch # Install specific version of transformers -RUN cd server && make install-transformers +RUN cd server && BUILD_EXTENSIONS="True" make install-transformers # Install server COPY proto proto diff --git a/server/Makefile b/server/Makefile index ac9cb785..6961178b 100644 --- a/server/Makefile +++ b/server/Makefile @@ -19,7 +19,7 @@ install-transformers: install-torch: # Install specific version of torch - pip install torch --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir + pip install torch --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir install: gen-server install-torch install-transformers pip install pip --upgrade From ce960be0a518262093f6038f10b4e2110da242d3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 26 Jan 2023 15:33:45 +0100 Subject: [PATCH 053/793] feat(bloom): use torch.nn.Linear and torch.nn.GELU (#33) --- server/text_generation/models/bloom.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 2218b91b..463c0406 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -136,7 +136,6 @@ def load_weights( start = rank * block_size stop = (rank + 1) * block_size tensor = slice_[start:stop] - tensor = tensor.transpose(1, 0) else: size = slice_.get_shape()[0] block_size = size // world_size @@ -150,7 +149,6 @@ def load_weights( start = rank * block_size stop = (rank + 1) * block_size tensor = slice_[:, start:stop] - tensor = tensor.transpose(1, 0) else: tensor = slice_[:] # XXX: Hack for Rowlinear to add the bias only once. @@ -186,7 +184,7 @@ def load_weights( and param_name == "weight" ): tensor = Int8Params( - tensor.transpose(1, 0), + tensor, has_fp16_weights=False, requires_grad=False, ).to(device) From 1539d3cbbef294df9c7ee9db07f45c97e6370f7b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 26 Jan 2023 16:29:13 +0100 Subject: [PATCH 054/793] feat(router): Remove second lock from batcher hot path (#27) @njhill --- Cargo.lock | 7 +++++++ router/Cargo.toml | 1 + router/src/batcher.rs | 38 +++++++++++++++++--------------------- router/src/db.rs | 27 ++++++++++++++------------- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 752c4886..33f5d181 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1087,6 +1087,12 @@ dependencies = [ "libc", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.1" @@ -1826,6 +1832,7 @@ dependencies = [ "axum", "clap 4.0.22", "futures", + "nohash-hasher", "parking_lot", "serde", "serde_json", diff --git a/router/Cargo.toml b/router/Cargo.toml index f99069d3..546f127f 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -17,6 +17,7 @@ axum = { version = "0.5.16", features = ["json", "serde_json"] } text-generation-client = { path = "client" } clap = { version = "4.0.15", features = ["derive", "env"] } futures = "0.3.24" +nohash-hasher = "0.2.0" parking_lot = "0.12.1" serde = "1.0.145" serde_json = "1.0.85" diff --git a/router/src/batcher.rs b/router/src/batcher.rs index ee83d899..624ac82d 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -3,6 +3,7 @@ use crate::{Db, Entry}; use crate::{ErrorResponse, GenerateRequest}; use axum::http::StatusCode; use axum::Json; +use nohash_hasher::IntMap; use std::future::Future; use std::sync::Arc; use text_generation_client::{Batch, ClientError, GeneratedText, ShardedClient}; @@ -104,8 +105,8 @@ async fn batching_task( // Get the next batch from the DB // This batch might be smaller than the maximum batch size if there are not enough requests // waiting in the DB - while let Some((request_ids, batch)) = db.next_batch(None, max_batch_size) { - let mut cached_batch = wrap_future(client.generate(batch), request_ids, &db).await; + while let Some((mut entries, batch)) = db.next_batch(None, max_batch_size) { + let mut cached_batch = wrap_future(client.generate(batch), &mut entries).await; let mut waiting_tokens = 1; // We loop until we do not receive any cached batch from the inference server (== until @@ -113,7 +114,6 @@ async fn batching_task( while let Some(batch) = cached_batch { // Get current batch info let batch_size = batch.size; - let mut request_ids: Vec = batch.requests.iter().map(|req| req.id).collect(); let mut batches = vec![batch]; // If the current batch is too small, we try to add more requests to it @@ -127,24 +127,23 @@ async fn batching_task( }; // Try to get a new batch - if let Some((new_request_ids, new_batch)) = + if let Some((mut new_entries, new_batch)) = db.next_batch(min_size, max_batch_size - batch_size as usize) { // Generate one token for this new batch to have the attention past in cache let new_cached_batch = - wrap_future(client.generate(new_batch), new_request_ids, &db).await; + wrap_future(client.generate(new_batch), &mut new_entries).await; // Reset waiting counter waiting_tokens = 1; // Extend current batch with the new batch if let Some(new_cached_batch) = new_cached_batch { - request_ids.extend(new_cached_batch.requests.iter().map(|req| req.id)); + entries.extend(new_entries); batches.push(new_cached_batch); } } } - cached_batch = - wrap_future(client.generate_with_cache(batches), request_ids, &db).await; + cached_batch = wrap_future(client.generate_with_cache(batches), &mut entries).await; waiting_tokens += 1; } } @@ -154,39 +153,36 @@ async fn batching_task( /// Wrap a future inside a match statement to handle errors and send the response to the Batcher async fn wrap_future( future: impl Future, Option), ClientError>>, - request_ids: Vec, - db: &Db, + entries: &mut IntMap, ) -> Option { match future.await { Ok((generated_texts, next_batch)) => { - send_generated(generated_texts, db); + send_generated(generated_texts, entries); next_batch } // If we have an error, we discard the whole batch Err(err) => { - send_error(err, request_ids, db); + send_error(err, entries); None } } } -/// Send errors to the Batcher for all `request_ids` -fn send_error(error: ClientError, request_ids: Vec, db: &Db) { - request_ids.into_iter().for_each(|id| { - // We can `expect` here as the request id should always be in the DB - let entry = db.remove(&id).expect("ID not found in db. This is a bug."); +/// Send errors to the Batcher for all `entries` +fn send_error(error: ClientError, entries: &mut IntMap) { + entries.drain().for_each(|(_, entry)| { // unwrap_or is valid here as we don't care if the receiver is gone. entry.response_tx.send(Err(error.clone())).unwrap_or(()); }); } /// Send `generated_text` to the Batcher for all `finished` -fn send_generated(finished: Vec, db: &Db) { +fn send_generated(finished: Vec, entries: &mut IntMap) { finished.into_iter().for_each(|output| { - // We can `expect` here as the request id should always be in the DB - let entry = db + // We can `expect` here as the request id should always be in the entries + let entry = entries .remove(&output.request.unwrap().id) - .expect("ID not found in db. This is a bug."); + .expect("ID not found in entries. This is a bug."); let response = InferResponse { output_text: output.output_text, diff --git a/router/src/db.rs b/router/src/db.rs index 1d7df627..51de9d05 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,6 +1,7 @@ -use crate::InferResponse; /// This code is massively inspired by Tokio mini-redis +use crate::InferResponse; use crate::{GenerateParameters, GenerateRequest}; +use nohash_hasher::{BuildNoHashHasher, IntMap}; use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; @@ -112,18 +113,12 @@ impl Db { state.entries.insert(id, entry); } - /// Remove an entry from the database if it exists - pub(crate) fn remove(&self, id: &u64) -> Option { - let mut state = self.shared.state.lock(); - state.entries.remove(id) - } - // Get the next batch pub(crate) fn next_batch( &self, min_size: Option, max_size: usize, - ) -> Option<(Vec, Batch)> { + ) -> Option<(IntMap, Batch)> { // Acquire lock let mut state = self.shared.state.lock(); @@ -135,13 +130,19 @@ impl Db { return None; } } + // Batch size + let size = requests.len(); + + let mut entries = IntMap::with_capacity_and_hasher(size, BuildNoHashHasher::default()); ids.iter().for_each(|id| { - // Set batch_time for each request - state.entries.get_mut(id).unwrap().batch_time = Some(Instant::now()); + // Remove entry from db + let mut entry = state.entries.remove(id).unwrap(); + // Set batch_time + entry.batch_time = Some(Instant::now()); + // Insert in entries IntMap + entries.insert(*id, entry); }); - // Batch size - let size = requests.len(); let batch = Batch { id: state.next_batch_id, requests, @@ -152,7 +153,7 @@ impl Db { // Increment batch id state.next_batch_id += 1; - return Some((ids, batch)); + return Some((entries, batch)); } None } From cd298bc5e5f1beb300c05099eb8d13460f867863 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 30 Jan 2023 15:36:16 +0100 Subject: [PATCH 055/793] feat: Support sampling seeding (#37) Co-authored-by: Yannic Kilcher --- proto/generate.proto | 4 ++++ router/client/build.rs | 1 + router/src/batcher.rs | 2 ++ router/src/db.rs | 1 + router/src/lib.rs | 4 ++++ router/src/server.rs | 6 +++++- server/text_generation/models/bloom.py | 4 +++- server/text_generation/models/causal_lm.py | 14 ++++++++++-- server/text_generation/models/galactica.py | 4 +++- server/text_generation/models/santacoder.py | 16 ++++++++------ server/text_generation/models/seq2seq_lm.py | 10 ++++++++- server/text_generation/models/types.py | 4 +++- server/text_generation/utils.py | 24 ++++++++++++++++++--- 13 files changed, 78 insertions(+), 16 deletions(-) diff --git a/proto/generate.proto b/proto/generate.proto index 16539f8b..921bd5c0 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -36,6 +36,8 @@ message NextTokenChooserParameters { float top_p = 3; /// apply sampling on the logits bool do_sample = 4; + /// random seed for sampling + optional uint64 seed = 5; } message StoppingCriteriaParameters { @@ -82,6 +84,8 @@ message GeneratedText { repeated float logprobs = 6; /// Finish reason string finish_reason = 7; + /// Seed + optional uint64 seed = 8; } message GenerateRequest { diff --git a/router/client/build.rs b/router/client/build.rs index 1876b217..7b577fcc 100644 --- a/router/client/build.rs +++ b/router/client/build.rs @@ -1,6 +1,7 @@ use std::fs; fn main() -> Result<(), Box> { + println!("cargo:rerun-if-changed=../../proto/generate.proto"); fs::create_dir("src/pb").unwrap_or(()); tonic_build::configure() .build_client(true) diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 624ac82d..baf58af4 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -191,6 +191,7 @@ fn send_generated(finished: Vec, entries: &mut IntMap tokens: output.tokens, logprobs: output.logprobs, finish_reason: output.finish_reason, + seed: output.seed, queued: entry.time, start: entry.batch_time.unwrap(), // unwrap is always valid end: Instant::now(), @@ -208,6 +209,7 @@ pub(crate) struct InferResponse { pub(crate) tokens: Vec, pub(crate) logprobs: Vec, pub(crate) finish_reason: String, + pub(crate) seed: Option, pub(crate) queued: Instant, pub(crate) start: Instant, pub(crate) end: Instant, diff --git a/router/src/db.rs b/router/src/db.rs index 51de9d05..15007b64 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -166,6 +166,7 @@ impl From<&GenerateParameters> for NextTokenChooserParameters { top_k: parameters.top_k as u32, top_p: parameters.top_p, do_sample: parameters.do_sample, + seed: parameters.seed, } } } diff --git a/router/src/lib.rs b/router/src/lib.rs index 03711580..1aeac302 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -25,6 +25,8 @@ pub(crate) struct GenerateParameters { pub stop: Vec, #[serde(default)] pub details: bool, + #[serde(default)] + pub seed: Option, } fn default_temperature() -> f32 { @@ -56,6 +58,7 @@ fn default_parameters() -> GenerateParameters { max_new_tokens: default_max_new_tokens(), stop: vec![], details: false, + seed: None, } } @@ -70,6 +73,7 @@ pub(crate) struct GenerateRequest { pub(crate) struct Details { pub finish_reason: String, pub generated_tokens: u32, + pub seed: Option, pub tokens: Vec<(u32, String, f32)>, } diff --git a/router/src/server.rs b/router/src/server.rs index 623dd07c..86041b96 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -55,6 +55,7 @@ async fn health(state: Extension) -> Result<(), (StatusCode, Json) -> Result<(), (StatusCode, Json generate_pb2.GeneratedText: return generate_pb2.GeneratedText( @@ -49,4 +50,5 @@ def to_pb(self) -> generate_pb2.GeneratedText: token_ids=self.token_ids, logprobs=self.logprobs, finish_reason=self.reason, + seed=self.seed, ) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 1ddeed6e..1d087a42 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -24,11 +24,24 @@ class Sampling: + def __init__(self, seed: Optional[int] = None): + self.generator = torch.Generator() + if seed is not None: + self.generator.manual_seed(seed) + else: + self.generator.seed() + def __call__(self, logits): probs = torch.nn.functional.softmax(logits, dim=-1) - next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1) + next_tokens = torch.multinomial( + probs, num_samples=1, generator=self.generator + ).squeeze(1) return next_tokens + @property + def seed(self) -> int: + return self.generator.initial_seed() + class Greedy: def __call__(self, logits): @@ -36,7 +49,9 @@ def __call__(self, logits): class NextTokenChooser: - def __init__(self, temperature=1.0, top_k=None, top_p=None, do_sample=False): + def __init__( + self, temperature=1.0, top_k=None, top_p=None, do_sample=False, seed=None + ): warpers = LogitsProcessorList() # the following idea is largely copied from this PR: https://github.com/huggingface/transformers/pull/5420/files # all samplers can be found in `generation_utils_samplers.py` @@ -53,7 +68,7 @@ def __init__(self, temperature=1.0, top_k=None, top_p=None, do_sample=False): sampling = True self.warpers = warpers - self.choice = Sampling() if sampling else Greedy() + self.choice = Sampling(seed) if sampling else Greedy() def __call__(self, input_ids, scores): # Warp logits @@ -66,11 +81,14 @@ def __call__(self, input_ids, scores): @classmethod def from_pb(cls, pb: generate_pb2.NextTokenChooserParameters) -> "NextTokenChooser": + # handle protobuf making default values 0 + seed = pb.seed if pb.HasField("seed") else None return NextTokenChooser( temperature=pb.temperature, top_k=pb.top_k, top_p=pb.top_p, do_sample=pb.do_sample, + seed=seed, ) From 7fbfbb0dc5d9f928ee4c496a04bff4a0eca8a9c8 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 31 Jan 2023 11:49:43 +0100 Subject: [PATCH 056/793] feat: Add token streaming using ServerSideEvents support (#36) Add token streaming using ServerSideEvents (SSE). The signature of the SSE events is: ```rust struct Details { finish_reason: String, generated_tokens: u32, seed: Option, } struct StreamResponse { token: Token, generated_text: Option, details: Option
, } struct ErrorResponse { error: String, } ``` --- Cargo.lock | 2 + launcher/Cargo.toml | 2 +- launcher/tests/bloom_560m.json | 6 +- launcher/tests/mt0_base.json | 6 +- proto/generate.proto | 66 ++-- router/Cargo.toml | 2 + router/client/src/client.rs | 28 +- router/client/src/lib.rs | 3 +- router/client/src/sharded_client.rs | 38 +-- router/src/batcher.rs | 236 ------------- router/src/db.rs | 14 +- router/src/infer.rs | 354 ++++++++++++++++++++ router/src/lib.rs | 21 +- router/src/server.rs | 307 +++++++++++------ router/src/validation.rs | 15 +- server/tests/models/test_bloom.py | 105 +++--- server/tests/models/test_causal_lm.py | 100 +++--- server/tests/models/test_santacoder.py | 30 +- server/tests/models/test_seq2seq_lm.py | 93 ++--- server/text_generation/models/causal_lm.py | 110 +++--- server/text_generation/models/seq2seq_lm.py | 102 +++--- server/text_generation/models/types.py | 59 +++- server/text_generation/server.py | 20 +- 23 files changed, 1017 insertions(+), 702 deletions(-) delete mode 100644 router/src/batcher.rs create mode 100644 router/src/infer.rs diff --git a/Cargo.lock b/Cargo.lock index 33f5d181..4c44820b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1829,6 +1829,7 @@ dependencies = [ name = "text-generation-router" version = "0.1.0" dependencies = [ + "async-stream", "axum", "clap 4.0.22", "futures", @@ -1840,6 +1841,7 @@ dependencies = [ "thiserror", "tokenizers", "tokio", + "tokio-stream", "tracing", "tracing-subscriber", ] diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 21d5d3ee..58df28d9 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -16,4 +16,4 @@ tracing-subscriber = { version = "0.3.16", features = ["json"] } [dev-dependencies] float_eq = "1.0.1" reqwest = { version = "0.11.13", features = ["blocking", "json"] } -serde = "1.0.150" +serde = { version = "1.0.150", features = ["derive"] } diff --git a/launcher/tests/bloom_560m.json b/launcher/tests/bloom_560m.json index d17f1ed4..a81d1982 100644 --- a/launcher/tests/bloom_560m.json +++ b/launcher/tests/bloom_560m.json @@ -3,7 +3,7 @@ "details": { "finish_reason": "length", "generated_tokens": 20, - "tokens": [ + "prefill": [ [ 10264, "Test", @@ -13,7 +13,9 @@ 8821, " request", -11.895094 - ], + ] + ], + "tokens": [ [ 17, ".", diff --git a/launcher/tests/mt0_base.json b/launcher/tests/mt0_base.json index 1b772282..51cb8b5c 100644 --- a/launcher/tests/mt0_base.json +++ b/launcher/tests/mt0_base.json @@ -3,12 +3,14 @@ "details": { "finish_reason": "length", "generated_tokens": 20, - "tokens": [ + "prefill": [ [ 0, "", null - ], + ] + ], + "tokens": [ [ 259, "", diff --git a/proto/generate.proto b/proto/generate.proto index 921bd5c0..32ec9681 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -7,10 +7,10 @@ service TextGenerationService { rpc ServiceDiscovery (ServiceDiscoveryRequest) returns (ServiceDiscoveryResponse) {} /// Empties batch cache rpc ClearCache (ClearCacheRequest) returns (ClearCacheResponse); - /// Generate tokens for a batch - rpc Generate (GenerateRequest) returns (GenerateResponse); - /// Generate tokens for a list of cached batches - rpc GenerateWithCache (GenerateWithCacheRequest) returns (GenerateWithCacheResponse); + /// Prefill batch and decode first token + rpc Prefill (PrefillRequest) returns (PrefillResponse); + /// Decode token for a list of prefilled batches + rpc Decode (DecodeRequest) returns (DecodeResponse); } /// Empty request @@ -70,44 +70,60 @@ message Batch { } message GeneratedText { - /// Request - Request request = 1; /// Output - string output_text = 2; + string text = 1; /// Number of generated tokens - uint32 generated_tokens = 3; - /// Tokens - repeated string tokens = 4; - /// Token IDs - repeated uint32 token_ids = 5; - /// Logprobs - repeated float logprobs = 6; + uint32 generated_tokens = 2; /// Finish reason - string finish_reason = 7; + string finish_reason = 3; /// Seed - optional uint64 seed = 8; + optional uint64 seed = 4; } -message GenerateRequest { +message PrefillTokens { + /// Prefill Token IDs + repeated uint32 ids = 1; + /// Prefill Logprobs + repeated float logprobs = 2; + /// Prefill tokens + repeated string texts = 3; +} + +message Generation { + /// Request ID + uint64 request_id = 1; + /// Prefill tokens (optional) + PrefillTokens prefill_tokens = 2; + /// Token ID + uint32 token_id = 3; + /// Logprob + float token_logprob = 4; + /// Text + string token_text = 5; + /// Complete generated text + GeneratedText generated_text = 6; +} + +message PrefillRequest { /// Batch Batch batch = 1; } -message GenerateResponse { - /// Finished requests - repeated GeneratedText generated_texts = 1; +message PrefillResponse { + /// Generation + repeated Generation generations = 1; /// Next batch (cached) optional Batch batch = 2; } -message GenerateWithCacheRequest { +message DecodeRequest { /// Cached batches repeated Batch batches = 1; } -message GenerateWithCacheResponse { - /// Finished requests - repeated GeneratedText generated_texts = 1; +message DecodeResponse { + /// Decodes + repeated Generation generations = 1; /// Next batch (cached) optional Batch batch = 2; -} +} \ No newline at end of file diff --git a/router/Cargo.toml b/router/Cargo.toml index 546f127f..d30d3b48 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -13,6 +13,7 @@ name = "text-generation-router" path = "src/main.rs" [dependencies] +async-stream = "0.3.3" axum = { version = "0.5.16", features = ["json", "serde_json"] } text-generation-client = { path = "client" } clap = { version = "4.0.15", features = ["derive", "env"] } @@ -24,6 +25,7 @@ serde_json = "1.0.85" thiserror = "1.0.37" tokenizers = "0.13.0" tokio = { version = "1.21.1", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } +tokio-stream = "0.1.11" tracing = "0.1.36" tracing-subscriber = { version = "0.3.15", features = ["json"] } diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 172d0bf7..77a43110 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -70,36 +70,36 @@ impl Client { /// Generate one token for each request in the given batch /// - /// Returns a list of generated texts of request that met their stopping criteria + /// Returns Generation for each request in batch /// and the next cached batch #[instrument(skip(self))] - pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { - let request = tonic::Request::new(GenerateRequest { batch: Some(batch) }); + pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { + let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }); let response = self .stub - .generate(request) - .instrument(info_span!("generate")) + .prefill(request) + .instrument(info_span!("prefill")) .await? .into_inner(); - Ok((response.generated_texts, response.batch)) + Ok((response.generations, response.batch)) } - /// Generate one token for each request in the given cached batch + /// Generate one token for each request in the given cached batches /// - /// Returns a list of generated texts of request that met their stopping criteria + /// Returns Generation for each request in batches /// and the next cached batch #[instrument(skip(self))] - pub async fn generate_with_cache( + pub async fn decode( &mut self, batches: Vec, - ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(GenerateWithCacheRequest { batches }); + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(DecodeRequest { batches }); let response = self .stub - .generate_with_cache(request) - .instrument(info_span!("generate_with_cache")) + .decode(request) + .instrument(info_span!("decode")) .await? .into_inner(); - Ok((response.generated_texts, response.batch)) + Ok((response.generations, response.batch)) } } diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 295b009b..e0546b16 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -7,7 +7,8 @@ mod sharded_client; pub use client::Client; pub use pb::generate::v1::{ - Batch, GeneratedText, NextTokenChooserParameters, Request, StoppingCriteriaParameters, + Batch, GeneratedText, Generation, NextTokenChooserParameters, PrefillTokens, Request, + StoppingCriteriaParameters, }; pub use sharded_client::ShardedClient; use thiserror::Error; diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 6c70afca..56335f92 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,6 +1,6 @@ /// Multi shard Client use crate::Result; -use crate::{Batch, Client, GeneratedText}; +use crate::{Batch, Client, Generation}; use futures::future::join_all; use futures::future::select_all; use tonic::transport::Uri; @@ -37,46 +37,46 @@ impl ShardedClient { Self::from_master_client(master_client).await } + /// Clear the past generations cache + pub async fn clear_cache(&mut self) -> Result<()> { + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| client.clear_cache()) + .collect(); + join_all(futures).await.into_iter().collect() + } + /// Generate one token for each request in the given batch /// - /// Returns a list of generated texts of request that met their stopping criteria + /// Returns Generation for each request in batch /// and the next cached batch - pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { + pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| Box::pin(client.generate(batch.clone()))) + .map(|client| Box::pin(client.prefill(batch.clone()))) .collect(); // As soon as we receive one response, we can return as all shards will return the same let (result, _, _) = select_all(futures).await; result } - /// Generate one token for each request in the given cached batch + /// Generate one token for each request in the given cached batches /// - /// Returns a list of generated texts of request that met their stopping criteria + /// Returns Generation for each request in batches /// and the next cached batch - pub async fn generate_with_cache( + pub async fn decode( &mut self, batches: Vec, - ) -> Result<(Vec, Option)> { + ) -> Result<(Vec, Option)> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| Box::pin(client.generate_with_cache(batches.clone()))) + .map(|client| Box::pin(client.decode(batches.clone()))) .collect(); // As soon as we receive one response, we can return as all shards will return the same let (result, _, _) = select_all(futures).await; result } - - /// Clear the past generations cache - pub async fn clear_cache(&mut self) -> Result<()> { - let futures: Vec<_> = self - .clients - .iter_mut() - .map(|client| client.clear_cache()) - .collect(); - join_all(futures).await.into_iter().collect() - } } diff --git a/router/src/batcher.rs b/router/src/batcher.rs deleted file mode 100644 index baf58af4..00000000 --- a/router/src/batcher.rs +++ /dev/null @@ -1,236 +0,0 @@ -/// Batching and inference logic -use crate::{Db, Entry}; -use crate::{ErrorResponse, GenerateRequest}; -use axum::http::StatusCode; -use axum::Json; -use nohash_hasher::IntMap; -use std::future::Future; -use std::sync::Arc; -use text_generation_client::{Batch, ClientError, GeneratedText, ShardedClient}; -use thiserror::Error; -use tokio::sync::{oneshot, Notify}; -use tokio::time::Instant; -use tracing::instrument; - -/// Batcher -#[derive(Clone)] -pub struct Batcher { - /// Request database - db: Db, - /// Shared state - shared: Arc, -} - -/// Batcher shared state -struct Shared { - /// Batching background Tokio task notifier - batching_task: Notify, -} - -impl Batcher { - pub(crate) fn new( - client: ShardedClient, - max_batch_size: usize, - max_waiting_tokens: usize, - ) -> Self { - // Batcher shared state - let db = Db::new(); - let shared = Arc::new(Shared { - batching_task: Notify::new(), - }); - - // Spawn batching background task that contains all the inference logic - tokio::spawn(batching_task( - client, - max_batch_size, - max_waiting_tokens, - db.clone(), - shared.clone(), - )); - - Self { db, shared } - } - - /// Add a new request to the database and return a future that will generate the text - pub(crate) async fn infer( - &self, - input_length: usize, - request: GenerateRequest, - ) -> Result { - // One shot channel to communicate with the background batching task - let (response_tx, response_rx) = oneshot::channel(); - - // Try to append the request to the database - self.db.append(Entry { - request, - response_tx, - input_length, - time: Instant::now(), - batch_time: None, - }); - - // Notify the background task that we have a new entry in the database that needs - // to be batched - self.shared.batching_task.notify_one(); - - // Await on the response from the background task - // We can safely unwrap as the background task will never drop the sender - response_rx - .await - .unwrap() - .map_err(|err| InferError::GenerationError(err.to_string())) - } -} - -/// Batching logic -/// Will be launched in a background Tokio task -/// -/// Batches requests and sends them to the inference server -#[instrument(skip(client, db, shared))] -async fn batching_task( - mut client: ShardedClient, - max_batch_size: usize, - max_waiting_tokens: usize, - db: Db, - shared: Arc, -) { - // Minimum batch size after which we try to add more requests - let limit_min_batch_size = (max_batch_size / 2) as u32; - - // Infinite loop - loop { - // Wait for a notification from the Batcher struct - shared.batching_task.notified().await; - - // Get the next batch from the DB - // This batch might be smaller than the maximum batch size if there are not enough requests - // waiting in the DB - while let Some((mut entries, batch)) = db.next_batch(None, max_batch_size) { - let mut cached_batch = wrap_future(client.generate(batch), &mut entries).await; - let mut waiting_tokens = 1; - - // We loop until we do not receive any cached batch from the inference server (== until - // all requests have met their stopping criteria) - while let Some(batch) = cached_batch { - // Get current batch info - let batch_size = batch.size; - let mut batches = vec![batch]; - - // If the current batch is too small, we try to add more requests to it - if batch_size <= limit_min_batch_size { - let min_size = match waiting_tokens { - // If we didn't onboard any new requests since >= max_waiting_tokens, we try - // to add a new batch even though its size might be small - _ if waiting_tokens >= max_waiting_tokens => None, - // Minimum size criteria - _ => Some(limit_min_batch_size as usize), - }; - - // Try to get a new batch - if let Some((mut new_entries, new_batch)) = - db.next_batch(min_size, max_batch_size - batch_size as usize) - { - // Generate one token for this new batch to have the attention past in cache - let new_cached_batch = - wrap_future(client.generate(new_batch), &mut new_entries).await; - // Reset waiting counter - waiting_tokens = 1; - // Extend current batch with the new batch - if let Some(new_cached_batch) = new_cached_batch { - entries.extend(new_entries); - batches.push(new_cached_batch); - } - } - } - - cached_batch = wrap_future(client.generate_with_cache(batches), &mut entries).await; - waiting_tokens += 1; - } - } - } -} - -/// Wrap a future inside a match statement to handle errors and send the response to the Batcher -async fn wrap_future( - future: impl Future, Option), ClientError>>, - entries: &mut IntMap, -) -> Option { - match future.await { - Ok((generated_texts, next_batch)) => { - send_generated(generated_texts, entries); - next_batch - } - // If we have an error, we discard the whole batch - Err(err) => { - send_error(err, entries); - None - } - } -} - -/// Send errors to the Batcher for all `entries` -fn send_error(error: ClientError, entries: &mut IntMap) { - entries.drain().for_each(|(_, entry)| { - // unwrap_or is valid here as we don't care if the receiver is gone. - entry.response_tx.send(Err(error.clone())).unwrap_or(()); - }); -} - -/// Send `generated_text` to the Batcher for all `finished` -fn send_generated(finished: Vec, entries: &mut IntMap) { - finished.into_iter().for_each(|output| { - // We can `expect` here as the request id should always be in the entries - let entry = entries - .remove(&output.request.unwrap().id) - .expect("ID not found in entries. This is a bug."); - - let response = InferResponse { - output_text: output.output_text, - generated_tokens: output.generated_tokens, - token_ids: output.token_ids, - tokens: output.tokens, - logprobs: output.logprobs, - finish_reason: output.finish_reason, - seed: output.seed, - queued: entry.time, - start: entry.batch_time.unwrap(), // unwrap is always valid - end: Instant::now(), - }; - // unwrap_or is valid here as we don't care if the receiver is gone. - entry.response_tx.send(Ok(response)).unwrap_or(()); - }); -} - -#[derive(Debug)] -pub(crate) struct InferResponse { - pub(crate) output_text: String, - pub(crate) generated_tokens: u32, - pub(crate) token_ids: Vec, - pub(crate) tokens: Vec, - pub(crate) logprobs: Vec, - pub(crate) finish_reason: String, - pub(crate) seed: Option, - pub(crate) queued: Instant, - pub(crate) start: Instant, - pub(crate) end: Instant, -} - -#[derive(Debug, Error)] -pub enum InferError { - #[error("Request failed during generation: {0}")] - GenerationError(String), -} - -/// Convert to Axum supported format -impl From for (StatusCode, Json) { - fn from(err: InferError) -> Self { - match err { - InferError::GenerationError(_) => ( - StatusCode::FAILED_DEPENDENCY, - Json(ErrorResponse { - error: err.to_string(), - }), - ), - } - } -} diff --git a/router/src/db.rs b/router/src/db.rs index 15007b64..f0a62d65 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,14 +1,16 @@ /// This code is massively inspired by Tokio mini-redis -use crate::InferResponse; +use crate::infer::InferError; +use crate::infer::InferStreamResponse; use crate::{GenerateParameters, GenerateRequest}; use nohash_hasher::{BuildNoHashHasher, IntMap}; use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; use text_generation_client::{ - Batch, ClientError, NextTokenChooserParameters, Request, StoppingCriteriaParameters, + Batch, NextTokenChooserParameters, Request, StoppingCriteriaParameters, }; -use tokio::sync::oneshot::Sender; +use tokio::sync::mpsc::UnboundedSender; +use tokio::sync::OwnedSemaphorePermit; use tokio::time::Instant; /// Database entry @@ -16,14 +18,16 @@ use tokio::time::Instant; pub(crate) struct Entry { /// Request pub request: GenerateRequest, - /// Response sender to communicate between the Batcher and the batching_task - pub response_tx: Sender>, + /// Response sender to communicate between the Infer struct and the batching_task + pub response_tx: UnboundedSender>, /// Number of tokens in the input pub input_length: usize, /// Instant when this entry was created pub time: Instant, /// Instant when this entry was added to a batch pub batch_time: Option, + /// Permit + pub _permit: OwnedSemaphorePermit, } /// Request Database diff --git a/router/src/infer.rs b/router/src/infer.rs new file mode 100644 index 00000000..4c4a7eb8 --- /dev/null +++ b/router/src/infer.rs @@ -0,0 +1,354 @@ +/// Batching and inference logic +use crate::validation::{Validation, ValidationError}; +use crate::GenerateRequest; +use crate::{Db, Entry, Token}; +use nohash_hasher::IntMap; +use std::future::Future; +use std::sync::Arc; +use text_generation_client::{ + Batch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, +}; +use thiserror::Error; +use tokio::sync::{mpsc, Notify, Semaphore, TryAcquireError}; +use tokio::time::Instant; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tokio_stream::StreamExt; +use tracing::instrument; + +/// Inference struct +#[derive(Clone)] +pub struct Infer { + /// Validation + validation: Validation, + /// Request database + db: Db, + /// Shared state + shared: Arc, + /// Inference limit + limit_concurrent_requests: Arc, +} + +/// Infer shared state +struct Shared { + /// Batching background Tokio task notifier + batching_task: Notify, +} + +impl Infer { + pub(crate) fn new( + client: ShardedClient, + validation: Validation, + max_batch_size: usize, + max_waiting_tokens: usize, + max_concurrent_requests: usize, + ) -> Self { + // Infer shared state + let db = Db::new(); + let shared = Arc::new(Shared { + batching_task: Notify::new(), + }); + + // Spawn batching background task that contains all the inference logic + tokio::spawn(batching_task( + client, + max_batch_size, + max_waiting_tokens, + db.clone(), + shared.clone(), + )); + + // Inference limit with a semaphore + let semaphore = Arc::new(Semaphore::new(max_concurrent_requests)); + + Self { + validation, + db, + shared, + limit_concurrent_requests: semaphore, + } + } + + /// Add a new request to the database and return a stream of InferStreamResponse + pub(crate) async fn generate_stream( + &self, + request: GenerateRequest, + ) -> Result>, InferError> { + // Limit concurrent requests by acquiring a permit from the semaphore + // This permit will live as long as Entry + let permit = self.clone().limit_concurrent_requests.try_acquire_owned()?; + + // Validate request + let (input_length, validated_request) = self.validation.validate(request).await?; + + // MPSC channel to communicate with the background batching task + let (response_tx, response_rx) = mpsc::unbounded_channel(); + + // Append the request to the database + self.db.append(Entry { + request: validated_request, + response_tx, + input_length, + time: Instant::now(), + batch_time: None, + _permit: permit, + }); + + // Notify the background task that we have a new entry in the database that needs + // to be batched + self.shared.batching_task.notify_one(); + + // Return stream + Ok(UnboundedReceiverStream::new(response_rx)) + } + + /// Add a new request to the database and return a InferResponse + pub(crate) async fn generate( + &self, + request: GenerateRequest, + ) -> Result { + // Create stream + let mut stream = self.generate_stream(request).await?; + + // Return values + let mut result_prefill = Vec::new(); + let mut result_tokens = Vec::new(); + let mut result_generated_text = None; + let mut result_start = None; + let mut result_queued = None; + + // Iterate on stream + while let Some(response) = stream.next().await { + match response? { + // Add prefill tokens + InferStreamResponse::Prefill(tokens) => { + // Create Token objects + // We do that here instead of in the Python code as Rust for loops are faster + result_prefill = tokens + .ids + .into_iter() + .zip(tokens.logprobs.into_iter()) + .zip(tokens.texts.into_iter()) + .map(|((id, logprob), text)| Token(id, text, logprob)) + .collect(); + } + // Push last token + InferStreamResponse::Token(token) => result_tokens.push(token), + // Final message + // Set return values + InferStreamResponse::End { + token, + generated_text, + start, + queued, + } => { + result_tokens.push(token); + result_generated_text = Some(generated_text); + result_start = Some(start); + result_queued = Some(queued) + } + } + } + + // Check that we received a `InferStreamResponse::End` message + if let (Some(generated_text), Some(queued), Some(start)) = + (result_generated_text, result_queued, result_start) + { + Ok(InferResponse { + prefill: result_prefill, + tokens: result_tokens, + generated_text, + queued, + start, + }) + } else { + Err(InferError::IncompleteGeneration) + } + } +} + +/// Batching logic +/// Will be launched in a background Tokio task +/// +/// Batches requests and sends them to the inference server +#[instrument(skip(client, db, shared))] +async fn batching_task( + mut client: ShardedClient, + max_batch_size: usize, + max_waiting_tokens: usize, + db: Db, + shared: Arc, +) { + // Minimum batch size after which we try to add more requests + let limit_min_batch_size = (max_batch_size / 2) as u32; + + // Infinite loop + loop { + // Wait for a notification from the Infer struct + shared.batching_task.notified().await; + + // Get the next batch from the DB + // This batch might be smaller than the maximum batch size if there are not enough requests + // waiting in the DB + while let Some((mut entries, batch)) = db.next_batch(None, max_batch_size) { + let mut cached_batch = wrap_future(client.prefill(batch), &mut entries).await; + let mut waiting_tokens = 1; + + // We loop until we do not receive any cached batch from the inference server (== until + // all requests have met their stopping criteria) + while let Some(batch) = cached_batch { + // Get current batch info + let batch_size = batch.size; + let mut batches = vec![batch]; + + // If the current batch is too small, we try to add more requests to it + if batch_size <= limit_min_batch_size { + let min_size = match waiting_tokens { + // If we didn't onboard any new requests since >= max_waiting_tokens, we try + // to add a new batch even though its size might be small + _ if waiting_tokens >= max_waiting_tokens => None, + // Minimum size criteria + _ => Some(limit_min_batch_size as usize), + }; + + // Try to get a new batch + if let Some((mut new_entries, new_batch)) = + db.next_batch(min_size, max_batch_size - batch_size as usize) + { + // Generate one token for this new batch to have the attention past in cache + let new_cached_batch = + wrap_future(client.prefill(new_batch), &mut new_entries).await; + // Reset waiting counter + waiting_tokens = 1; + // Extend current batch with the new batch + if let Some(new_cached_batch) = new_cached_batch { + entries.extend(new_entries); + batches.push(new_cached_batch); + } + } + } + + cached_batch = wrap_future(client.decode(batches), &mut entries).await; + waiting_tokens += 1; + } + } + } +} + +/// Wrap a future inside a match statement to handle errors and send the responses to Infer +async fn wrap_future( + future: impl Future, Option), ClientError>>, + entries: &mut IntMap, +) -> Option { + match future.await { + Ok((generations, next_batch)) => { + send_generations(generations, entries); + next_batch + } + // If we have an error, we discard the whole batch + Err(err) => { + send_error(err, entries); + None + } + } +} + +/// Send errors to Infer for all `entries` +fn send_error(error: ClientError, entries: &mut IntMap) { + entries.drain().for_each(|(_, entry)| { + // unwrap_or is valid here as we don't care if the receiver is gone. + entry + .response_tx + .send(Err(InferError::GenerationError(error.to_string()))) + .unwrap_or(()); + }); +} + +/// Send one or multiple `InferStreamResponse` to Infer for all `entries` +fn send_generations(generations: Vec, entries: &mut IntMap) { + generations.into_iter().for_each(|generation| { + // Get entry + // We can `expect` here as the request id should always be in the entries + let entry = entries + .get(&generation.request_id) + .expect("ID not found in entries. This is a bug."); + + if let Some(prefill_tokens) = generation.prefill_tokens { + // Send message + // unwrap_or is valid here as we don't care if the receiver is gone. + entry + .response_tx + .send(Ok(InferStreamResponse::Prefill(prefill_tokens))) + .unwrap_or(()); + } + + // Create last Token + let token = Token( + generation.token_id, + generation.token_text, + generation.token_logprob, + ); + + if let Some(generated_text) = generation.generated_text { + // Remove entry as this is the last message + // We can `expect` here as the request id should always be in the entries + let entry = entries + .remove(&generation.request_id) + .expect("ID not found in entries. This is a bug."); + + // Send message + // unwrap_or is valid here as we don't care if the receiver is gone. + entry + .response_tx + .send(Ok(InferStreamResponse::End { + token, + generated_text, + queued: entry.time, + start: entry.batch_time.unwrap(), + })) + .unwrap_or(()); + } else { + // Send message + // unwrap_or is valid here as we don't care if the receiver is gone. + entry + .response_tx + .send(Ok(InferStreamResponse::Token(token))) + .unwrap_or(()); + } + }); +} + +#[derive(Debug)] +pub(crate) enum InferStreamResponse { + // Optional first message + Prefill(PrefillTokens), + // Intermediate messages + Token(Token), + // Last message + End { + token: Token, + generated_text: GeneratedText, + start: Instant, + queued: Instant, + }, +} + +#[derive(Debug)] +pub(crate) struct InferResponse { + pub(crate) prefill: Vec, + pub(crate) tokens: Vec, + pub(crate) generated_text: GeneratedText, + pub(crate) queued: Instant, + pub(crate) start: Instant, +} + +#[derive(Debug, Error)] +pub enum InferError { + #[error("Request failed during generation: {0}")] + GenerationError(String), + #[error("Model is overloaded")] + Overloaded(#[from] TryAcquireError), + #[error("Input validation error: {0}")] + ValidationError(#[from] ValidationError), + #[error("Incomplete generation")] + IncompleteGeneration, +} diff --git a/router/src/lib.rs b/router/src/lib.rs index 1aeac302..beab7138 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,11 +1,11 @@ /// Text Generation Inference Webserver -mod batcher; mod db; +mod infer; pub mod server; mod validation; -use batcher::{Batcher, InferResponse}; use db::{Db, Entry}; +use infer::Infer; use serde::{Deserialize, Serialize}; use validation::Validation; @@ -69,21 +69,34 @@ pub(crate) struct GenerateRequest { pub parameters: GenerateParameters, } +#[derive(Debug, Serialize)] +pub struct Token(u32, String, f32); + #[derive(Serialize)] pub(crate) struct Details { pub finish_reason: String, pub generated_tokens: u32, pub seed: Option, - pub tokens: Vec<(u32, String, f32)>, + #[serde(skip_serializing_if = "Option::is_none")] + pub prefill: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub tokens: Option>, } #[derive(Serialize)] -pub(crate) struct GeneratedText { +pub(crate) struct GenerateResponse { pub generated_text: String, #[serde(skip_serializing_if = "Option::is_none")] pub details: Option
, } +#[derive(Serialize)] +pub(crate) struct StreamResponse { + pub token: Token, + pub generated_text: Option, + pub details: Option
, +} + #[derive(Serialize)] pub(crate) struct ErrorResponse { pub error: String, diff --git a/router/src/server.rs b/router/src/server.rs index 86041b96..ef3782d6 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,71 +1,54 @@ +/// HTTP Server logic +use crate::infer::{InferError, InferStreamResponse}; use crate::{ - Batcher, Details, ErrorResponse, GenerateParameters, GenerateRequest, GeneratedText, Validation, + Details, ErrorResponse, GenerateParameters, GenerateRequest, GenerateResponse, Infer, + StreamResponse, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, StatusCode}; +use axum::response::sse::{Event, KeepAlive, Sse}; use axum::response::IntoResponse; use axum::routing::{get, post}; use axum::{Json, Router}; +use futures::Stream; +use std::convert::Infallible; use std::net::SocketAddr; -use std::sync::Arc; use text_generation_client::ShardedClient; use tokenizers::Tokenizer; use tokio::signal; -use tokio::sync::Semaphore; use tokio::time::Instant; +use tokio_stream::StreamExt; use tracing::instrument; -// Server shared state -#[derive(Clone)] -struct ServerState { - validation: Validation, - batcher: Batcher, - limit_concurrent_requests: Arc, -} - /// Health check method -#[instrument(skip(state), fields(time, time_per_token))] -async fn health(state: Extension) -> Result<(), (StatusCode, Json)> { +#[instrument(skip(infer))] +async fn health(infer: Extension) -> Result<(), (StatusCode, Json)> { // TODO: while this is the best health check we can do, it is a bit on the heavy side and might // be a bit too slow for a health check. // What we should do instead if check if the gRPC channels are still healthy. - // Limit concurrent requests by acquiring a permit from the semaphore - let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { - ( - StatusCode::TOO_MANY_REQUESTS, - Json(ErrorResponse { - error: "Model is overloaded".to_string(), - }), - ) - })?; - // Send a small inference request - state - .batcher - .infer( - 1, - GenerateRequest { - inputs: "liveness".to_string(), - parameters: GenerateParameters { - temperature: 1.0, - top_k: 0, - top_p: 1.0, - do_sample: false, - max_new_tokens: 1, - stop: vec![], - details: false, - seed: None, - }, + infer + .generate(GenerateRequest { + inputs: "liveness".to_string(), + parameters: GenerateParameters { + temperature: 1.0, + top_k: 0, + top_p: 1.0, + do_sample: false, + max_new_tokens: 1, + stop: vec![], + details: false, + seed: None, }, - ) + }) .await?; Ok(()) } /// Generate method #[instrument( - skip(state), + skip(infer), fields( total_time, validation_time, @@ -76,56 +59,28 @@ async fn health(state: Extension) -> Result<(), (StatusCode, Json, + infer: Extension, req: Json, ) -> Result)> { + let span = tracing::Span::current(); let start_time = Instant::now(); - // Limit concurrent requests by acquiring a permit from the semaphore - let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { - tracing::error!("Model is overloaded"); - ( - StatusCode::TOO_MANY_REQUESTS, - Json(ErrorResponse { - error: "Model is overloaded".to_string(), - }), - ) - })?; - - // Validate request - let details = req.0.parameters.details; - let (input_length, validated_request) = - state.validation.validate(req.0).await.map_err(|err| { - tracing::error!("{}", err.to_string()); - err - })?; // Inference - let response = state - .batcher - .infer(input_length, validated_request) - .await - .map_err(|err| { - tracing::error!("{}", err.to_string()); - err - })?; + let details = req.0.parameters.details; + let response = infer.generate(req.0).await.map_err(|err| { + tracing::error!("{}", err.to_string()); + err + })?; // Token details let details = match details { - true => { - let tokens = response - .token_ids - .into_iter() - .zip(response.tokens.into_iter()) - .zip(response.logprobs.into_iter()) - .map(|((id, text), logprob)| (id, text, logprob)) - .collect(); - Some(Details { - seed: response.seed, - finish_reason: response.finish_reason, - generated_tokens: response.generated_tokens, - tokens, - }) - } + true => Some(Details { + finish_reason: response.generated_text.finish_reason, + generated_tokens: response.generated_text.generated_tokens, + prefill: Some(response.prefill), + tokens: Some(response.tokens), + seed: response.generated_text.seed, + }), false => None, }; @@ -133,8 +88,8 @@ async fn generate( let total_time = start_time.elapsed(); let validation_time = response.queued - start_time; let queue_time = response.start - response.queued; - let inference_time = response.end - response.start; - let time_per_token = inference_time / response.generated_tokens; + let inference_time = Instant::now() - response.start; + let time_per_token = inference_time / response.generated_text.generated_tokens; // Headers let mut headers = HeaderMap::new(); @@ -160,22 +115,143 @@ async fn generate( ); // Tracing metadata - tracing::Span::current().record("total_time", format!("{:?}", total_time)); - tracing::Span::current().record("validation_time", format!("{:?}", validation_time)); - tracing::Span::current().record("queue_time", format!("{:?}", queue_time)); - tracing::Span::current().record("inference_time", format!("{:?}", inference_time)); - tracing::Span::current().record("time_per_token", format!("{:?}", time_per_token)); - tracing::Span::current().record("seed", format!("{:?}", response.seed)); - tracing::info!("Output: {}", response.output_text); + span.record("total_time", format!("{:?}", total_time)); + span.record("validation_time", format!("{:?}", validation_time)); + span.record("queue_time", format!("{:?}", queue_time)); + span.record("inference_time", format!("{:?}", inference_time)); + span.record("time_per_token", format!("{:?}", time_per_token)); + span.record("seed", format!("{:?}", response.generated_text.seed)); + tracing::info!("Output: {}", response.generated_text.text); // Send response - let response = vec![GeneratedText { - generated_text: response.output_text, + let response = vec![GenerateResponse { + generated_text: response.generated_text.text, details, }]; Ok((headers, Json(response))) } +/// Generate stream method +#[instrument( + skip(infer), + fields( + total_time, + validation_time, + queue_time, + inference_time, + time_per_token + ) +)] +async fn generate_stream( + infer: Extension, + req: Json, +) -> Sse>> { + let span = tracing::Span::current(); + let start_time = Instant::now(); + + let stream = async_stream::stream! { + // Inference + let mut end_reached = false; + let mut error = false; + let details = req.0.parameters.details; + + match infer.generate_stream(req.0).await { + Ok(mut response_stream) => { + // Server Side Event stream + while let Some(response) = response_stream.next().await { + match response { + Ok(response) => { + match response { + // Prefill is ignored + InferStreamResponse::Prefill(_) => {} + // Yield event for every new token + InferStreamResponse::Token(token) => { + // StreamResponse + let stream_token = StreamResponse { + token, + generated_text: None, + details: None, + }; + + yield Ok(Event::default().json_data(stream_token).unwrap()) + } + // Yield event for last token and compute timings + InferStreamResponse::End { + token, + generated_text, + start, + queued, + } => { + // Token details + let details = match details { + true => Some(Details { + finish_reason: generated_text.finish_reason, + generated_tokens: generated_text.generated_tokens, + prefill: None, + tokens: None, + seed: generated_text.seed, + }), + false => None, + }; + + // Timings + let total_time = start_time.elapsed(); + let validation_time = queued - start_time; + let queue_time = start - queued; + let inference_time = Instant::now() - start; + let time_per_token = inference_time / generated_text.generated_tokens; + + // Tracing metadata + span.record("total_time", format!("{:?}", total_time)); + span + .record("validation_time", format!("{:?}", validation_time)); + span.record("queue_time", format!("{:?}", queue_time)); + span + .record("inference_time", format!("{:?}", inference_time)); + span + .record("time_per_token", format!("{:?}", time_per_token)); + tracing::info!(parent: &span, "Output: {}", generated_text.text); + + // StreamResponse + end_reached = true; + let stream_token = StreamResponse { + token, + generated_text: Some(generated_text.text), + details + }; + + yield Ok(Event::default().json_data(stream_token).unwrap()) + } + } + } + // Trace and yield error + Err(err) => { + error = true; + tracing::error!("{}", err.to_string()); + yield Ok(Event::from(err)) + } + } + } + }, + // Trace and yield error + Err(err) => { + error = true; + tracing::error!("{}", err.to_string()); + yield Ok(Event::from(err)) + } + } + // Check if generation reached the end + // Skip if we already sent an error + if !end_reached && !error { + let err = InferError::IncompleteGeneration; + tracing::error!("{}", err.to_string()); + yield Ok(Event::from(err)) + } + }; + + Sse::new(stream).keep_alive(KeepAlive::default()) +} + /// Serving method #[allow(clippy::too_many_arguments)] pub async fn run( @@ -189,21 +265,23 @@ pub async fn run( addr: SocketAddr, ) { // Create state - let batcher = Batcher::new(client, max_batch_size, max_waiting_tokens); let validation = Validation::new(validation_workers, tokenizer, max_input_length); - let shared_state = ServerState { + let infer = Infer::new( + client, validation, - batcher, - limit_concurrent_requests: Arc::new(Semaphore::new(max_concurrent_requests)), - }; + max_batch_size, + max_waiting_tokens, + max_concurrent_requests, + ); // Create router let app = Router::new() .route("/", post(generate)) .route("/generate", post(generate)) + .route("/generate_stream", post(generate_stream)) .route("/", get(health)) .route("/health", get(health)) - .layer(Extension(shared_state.clone())); + .layer(Extension(infer)); // Run server axum::Server::bind(&addr) @@ -240,3 +318,32 @@ async fn shutdown_signal() { tracing::info!("signal received, starting graceful shutdown"); } + +/// Convert to Axum supported formats +impl From for (StatusCode, Json) { + fn from(err: InferError) -> Self { + let status_code = match err { + InferError::GenerationError(_) => StatusCode::FAILED_DEPENDENCY, + InferError::Overloaded(_) => StatusCode::TOO_MANY_REQUESTS, + InferError::ValidationError(_) => StatusCode::UNPROCESSABLE_ENTITY, + InferError::IncompleteGeneration => StatusCode::INTERNAL_SERVER_ERROR, + }; + + ( + status_code, + Json(ErrorResponse { + error: err.to_string(), + }), + ) + } +} + +impl From for Event { + fn from(err: InferError) -> Self { + Event::default() + .json_data(ErrorResponse { + error: err.to_string(), + }) + .unwrap() + } +} diff --git a/router/src/validation.rs b/router/src/validation.rs index aabc82a6..d2287168 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,7 +1,5 @@ /// Payload validation logic -use crate::{ErrorResponse, GenerateRequest}; -use axum::http::StatusCode; -use axum::Json; +use crate::GenerateRequest; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokio::sync::{mpsc, oneshot}; @@ -161,14 +159,3 @@ pub enum ValidationError { #[error("tokenizer error {0}")] Tokenizer(String), } - -impl From for (StatusCode, Json) { - fn from(err: ValidationError) -> Self { - ( - StatusCode::UNPROCESSABLE_ENTITY, - Json(ErrorResponse { - error: err.to_string(), - }), - ) - } -} diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 1a788ce5..9f96efc3 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -91,9 +91,9 @@ def test_causal_lm_batch_type(default_bloom): def test_causal_lm_generate_token(default_bloom, default_bloom_batch): sequence_length = len(default_bloom_batch.all_input_ids[0]) - generated_texts, next_batch = default_bloom.generate_token(default_bloom_batch) + generations, next_batch = default_bloom.generate_token(default_bloom_batch) - assert generated_texts == [] + assert len(generations) == len(default_bloom_batch) assert isinstance(next_batch, CausalLMBatch) assert not next_batch.keys_head_dim_last @@ -122,24 +122,30 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): assert all( [p[1].shape == (16, sequence_length, 64) for p in next_batch.past_key_values] ) + assert all([generation.generated_text is None for generation in generations]) + assert all([len(generation.prefill_tokens) == 1 for generation in generations]) + assert all([generation.token_id.item() == 10264 for generation in generations]) + assert all([generation.token_text == "Test" for generation in generations]) + assert generations[0].request_id == 0 def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch): next_batch = default_bloom_batch for _ in range(default_bloom_batch.stopping_criterias[0].max_new_tokens - 1): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(default_bloom_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 + assert len(generations) == 1 assert ( - generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + generations[0].generated_text.text + == "TestTestTestTestTestTestTestTestTestTestTest" ) - assert generated_texts[0].request == default_bloom_batch.requests[0] + assert generations[0].request_id == default_bloom_batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -152,17 +158,19 @@ def test_causal_lm_generate_token_completion_multi( for i in range( default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 1 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(default_multi_requests_bloom_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "TestTestTestTestTestTest" - assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] + assert len(generations) == 2 + assert generations[1].generated_text.text == "TestTestTestTestTestTest" assert ( - generated_texts[0].generated_tokens + generations[1].request_id == default_multi_requests_bloom_batch.requests[1].id + ) + assert ( + generations[1].generated_text.generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) @@ -171,19 +179,22 @@ def test_causal_lm_generate_token_completion_multi( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 1 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 + assert len(generations) == 1 + assert ( + generations[0].generated_text.text + == "TestTestTestTestTestTestTestTestTestTestTest" + ) assert ( - generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id ) - assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -243,17 +254,19 @@ def test_batch_concatenate( for _ in range( default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 2 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "TestTestTestTestTestTest" - assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] + assert len(generations) == 3 + assert generations[2].generated_text.text == "TestTestTestTestTestTest" assert ( - generated_texts[0].generated_tokens + generations[2].request_id == default_multi_requests_bloom_batch.requests[1].id + ) + assert ( + generations[2].generated_text.generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) @@ -262,19 +275,20 @@ def test_batch_concatenate( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 2 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 + assert len(generations) == 2 assert ( - generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + generations[0].generated_text.text + == "TestTestTestTestTestTestTestTestTestTestTest" ) - assert generated_texts[0].request == default_bloom_batch.requests[0] + assert generations[0].request_id == default_bloom_batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -284,18 +298,21 @@ def test_batch_concatenate( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 4 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 + assert len(generations) == 1 + assert ( + generations[0].generated_text.text + == "TestTestTestTestTestTestTestTestTestTestTest" + ) assert ( - generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id ) - assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index bedb65ba..f9762b30 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -88,11 +88,9 @@ def test_causal_lm_batch_type(default_causal_lm): def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): sequence_length = len(default_causal_lm_batch.all_input_ids[0]) - generated_texts, next_batch = default_causal_lm.generate_token( - default_causal_lm_batch - ) + generations, next_batch = default_causal_lm.generate_token(default_causal_lm_batch) - assert generated_texts == [] + assert len(generations) == len(next_batch) assert isinstance(next_batch, CausalLMBatch) assert len(next_batch.all_input_ids) == next_batch.size @@ -121,6 +119,11 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): assert all( [p[1].shape == (1, 12, sequence_length, 64) for p in next_batch.past_key_values] ) + assert all([generation.generated_text is None for generation in generations]) + assert all([len(generation.prefill_tokens) == 1 for generation in generations]) + assert all([generation.token_id.item() == 13 for generation in generations]) + assert all([generation.token_text == "." for generation in generations]) + assert generations[0].request_id == 0 def test_causal_lm_generate_token_completion( @@ -128,18 +131,17 @@ def test_causal_lm_generate_token_completion( ): next_batch = default_causal_lm_batch for _ in range(default_causal_lm_batch.stopping_criterias[0].max_new_tokens - 1): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." - assert generated_texts[0].request == default_causal_lm_batch.requests[0] - assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) + assert len(generations) == 1 + assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert generations[0].request_id == default_causal_lm_batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -152,19 +154,20 @@ def test_causal_lm_generate_token_completion_multi( for i in range( default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 1 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784)" + assert len(generations) == 2 + assert generations[1].generated_text.text == "Test.java:784)" assert ( - generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] + generations[1].request_id + == default_multi_requests_causal_lm_batch.requests[1].id ) assert ( - generated_texts[0].generated_tokens + generations[1].generated_text.generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) @@ -173,19 +176,20 @@ def test_causal_lm_generate_token_completion_multi( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 1 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." + assert len(generations) == 1 + assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." assert ( - generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] + generations[0].request_id + == default_multi_requests_causal_lm_batch.requests[0].id ) assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -244,19 +248,20 @@ def test_batch_concatenate( for _ in range( default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784)" + assert len(generations) == 3 + assert generations[2].generated_text.text == "Test.java:784)" assert ( - generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] + generations[2].request_id + == default_multi_requests_causal_lm_batch.requests[1].id ) assert ( - generated_texts[0].generated_tokens + generations[2].generated_text.generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) @@ -265,17 +270,17 @@ def test_batch_concatenate( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." - assert generated_texts[0].request == default_causal_lm_batch.requests[0] + assert len(generations) == 2 + assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert generations[0].request_id == default_causal_lm_batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -285,18 +290,19 @@ def test_batch_concatenate( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 4 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." + assert len(generations) == 1 + assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." assert ( - generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] + generations[0].request_id + == default_multi_requests_causal_lm_batch.requests[0].id ) assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index acebec04..1b69477d 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -50,18 +50,17 @@ def test_santacoder_generate_token_completion(default_santacoder, default_pb_bat next_batch = batch for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): - generated_texts, next_batch = default_santacoder.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_santacoder.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_santacoder.generate_token(next_batch) + generations, next_batch = default_santacoder.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "def test_get_all_users_with_" - assert generated_texts[0].request == batch.requests[0] - assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) + assert len(generations) == 1 + assert generations[0].generated_text.text == "def test_get_all_users_with_" + assert generations[0].request_id == batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == batch.stopping_criterias[0].max_new_tokens ) @@ -76,20 +75,19 @@ def test_fim_santacoder_generate_token_completion( next_batch = batch for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): - generated_texts, next_batch = default_santacoder.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_santacoder.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_santacoder.generate_token(next_batch) + generations, next_batch = default_santacoder.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 + assert len(generations) == 1 assert ( - generated_texts[0].output_text + generations[0].generated_text.text == """defworldineProperty(exports, "__esModule", { value""" ) - assert generated_texts[0].request == batch.requests[0] - assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) + assert generations[0].request_id == batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index de1a4829..22c6ac9c 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -99,11 +99,11 @@ def test_seq2seq_lm_batch_type(default_seq2seq_lm): def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch): sequence_length = len(default_seq2seq_lm_batch.input_ids[0]) - generated_texts, next_batch = default_seq2seq_lm.generate_token( + generations, next_batch = default_seq2seq_lm.generate_token( default_seq2seq_lm_batch ) - assert generated_texts == [] + assert len(generations) == len(next_batch) assert isinstance(next_batch, Seq2SeqLMBatch) assert torch.equal(next_batch.input_ids, default_seq2seq_lm_batch.input_ids) @@ -145,6 +145,11 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) for p in next_batch.past_key_values ] ) + assert all([generation.generated_text is None for generation in generations]) + assert all([len(generation.prefill_tokens) == 1 for generation in generations]) + assert all([generation.token_id.item() == 259 for generation in generations]) + assert all([generation.token_text == "" for generation in generations]) + assert generations[0].request_id == 0 def test_seq2seq_lm_generate_token_completion( @@ -152,16 +157,16 @@ def test_seq2seq_lm_generate_token_completion( ): next_batch = default_seq2seq_lm_batch for _ in range(6): - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few weeks" - assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] - assert generated_texts[0].generated_tokens == 7 + assert len(generations) == 1 + assert generations[0].generated_text.text == "a few weeks" + assert generations[0].request_id == default_seq2seq_lm_batch.requests[0].id + assert generations[0].generated_text.generated_tokens == 7 def test_seq2seq_lm_generate_token_completion_multi( @@ -170,33 +175,33 @@ def test_seq2seq_lm_generate_token_completion_multi( next_batch = default_multi_requests_seq2seq_lm_batch for i in range(4): - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few " + assert len(generations) == 2 + assert generations[1].generated_text.text == "a few " assert ( - generated_texts[0].request - == default_multi_requests_seq2seq_lm_batch.requests[1] + generations[1].request_id + == default_multi_requests_seq2seq_lm_batch.requests[1].id ) - assert generated_texts[0].generated_tokens == 5 + assert generations[1].generated_text.generated_tokens == 5 - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few weeks" + assert len(generations) == 1 + assert generations[0].generated_text.text == "a few weeks" assert ( - generated_texts[0].request - == default_multi_requests_seq2seq_lm_batch.requests[0] + generations[0].request_id + == default_multi_requests_seq2seq_lm_batch.requests[0].id ) - assert generated_texts[0].generated_tokens == 7 + assert generations[0].generated_text.generated_tokens == 7 def test_batch_concatenate( @@ -291,35 +296,35 @@ def test_batch_concatenate( ) for _ in range(3): - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few " + assert len(generations) == 3 + assert generations[2].generated_text.text == "a few " assert ( - generated_texts[0].request - == default_multi_requests_seq2seq_lm_batch.requests[1] + generations[2].request_id + == default_multi_requests_seq2seq_lm_batch.requests[1].id ) - assert generated_texts[0].generated_tokens == 5 + assert generations[2].generated_text.generated_tokens == 5 - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few weeks" - assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] - assert generated_texts[0].generated_tokens == 7 + assert len(generations) == 2 + assert generations[0].generated_text.text == "a few weeks" + assert generations[0].request_id == default_seq2seq_lm_batch.requests[0].id + assert generations[0].generated_text.generated_tokens == 7 - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few weeks" + assert len(generations) == 1 + assert generations[0].generated_text.text == "a few weeks" assert ( - generated_texts[0].request - == default_multi_requests_seq2seq_lm_batch.requests[0] + generations[0].request_id + == default_multi_requests_seq2seq_lm_batch.requests[0].id ) - assert generated_texts[0].generated_tokens == 7 + assert generations[0].generated_text.generated_tokens == 7 diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index ccd4c3ba..a8fc23fe 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -5,7 +5,7 @@ from typing import Optional, Tuple, List, Type from text_generation.models import Model -from text_generation.models.types import GeneratedText, Batch +from text_generation.models.types import Batch, PrefillTokens, Generation, GeneratedText from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -23,7 +23,6 @@ class CausalLMBatch(Batch): # All tokens all_input_ids: List[torch.Tensor] - all_logprobs: List[Optional[torch.Tensor]] # Lengths of all generations present in the batch input_lengths: List[int] @@ -57,7 +56,6 @@ def from_pb( next_token_choosers = [] stopping_criterias = [] input_lengths = [] - all_logprobs = [] # Parse batch for r in pb.requests: @@ -67,7 +65,6 @@ def from_pb( stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) - all_logprobs.append(None) pad_to_multiple_of = 8 if device.type == "cuda" else None tokenized_inputs = tokenizer( @@ -89,7 +86,6 @@ def from_pb( position_ids=position_ids, past_key_values=None, all_input_ids=all_input_ids, - all_logprobs=all_logprobs, input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, @@ -107,7 +103,6 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests = [] input_lengths = [] all_input_ids = [] - all_logprobs = [] next_token_choosers = [] stopping_criterias = [] @@ -124,7 +119,6 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) all_input_ids.extend(batch.all_input_ids) - all_logprobs.extend(batch.all_logprobs) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -225,7 +219,6 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": position_ids=position_ids, past_key_values=past_key_values, all_input_ids=all_input_ids, - all_logprobs=all_logprobs, input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, @@ -234,6 +227,9 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": keys_head_dim_last=batches[0].keys_head_dim_last, ) + def __len__(self): + return len(self.requests) + class CausalLM(Model): def __init__(self, model_name: str, quantize=False): @@ -289,7 +285,7 @@ def forward( def generate_token( self, batch: CausalLMBatch - ) -> Tuple[List[GeneratedText], Optional[CausalLMBatch]]: + ) -> Tuple[List[Generation], Optional[CausalLMBatch]]: # For some reason, inference_mode does not work well with GLOO which we use on CPU context_manager = ( torch.no_grad if self.device.type == "cpu" else torch.inference_mode @@ -309,14 +305,13 @@ def generate_token( next_batch_input_lengths = [] next_batch_input_ids = [] next_batch_all_input_ids = [] - next_batch_all_logprobs = [] # Metadata next_batch_size = 0 next_batch_max_sequence_length = 0 - # Finished requests - generated_texts: List[GeneratedText] = [] + # Results + generations: List[Generation] = [] # Zipped iterator iterator = zip( @@ -326,7 +321,6 @@ def generate_token( batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, - batch.all_logprobs, ) # For each member of the batch @@ -337,44 +331,36 @@ def generate_token( next_token_chooser, stopping_criteria, all_input_ids, - all_logprobs, ) in enumerate(iterator): # Select next token tokens, logprobs = next_token_chooser(all_input_ids, logits) - next_token = tokens[-1].view(1, 1) + next_token_id = tokens[-1].view(1, 1) # Append next token to all tokens - all_input_ids = torch.cat([all_input_ids, next_token]) + all_input_ids = torch.cat([all_input_ids, next_token_id]) new_input_length = input_length + 1 - if all_logprobs is None: - # logprobs of all prompt tokens (except the first one) and the generated token - all_logprobs = logprobs.gather(1, all_input_ids[1:]) - else: - # logprob of the generated token - next_token_logprob = logprobs[-1, next_token] - all_logprobs = torch.cat([all_logprobs, next_token_logprob]) + # Generated token + next_token_logprob = logprobs[-1, next_token_id] + next_token_id_squeezed = next_token_id.squeeze() + next_token_text = self.tokenizer.decode( + next_token_id_squeezed, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) # Evaluate stopping criteria stop, reason = stopping_criteria( - next_token.squeeze(), - self.tokenizer.decode( - next_token.squeeze(), clean_up_tokenization_spaces=False - ), + next_token_id_squeezed, + next_token_text, ) + if stop: # Decode generated tokens generated_text = self.decode( all_input_ids[-stopping_criteria.current_tokens :, 0] ) output_text = request.inputs + generated_text - # Slice with input_length to remove padding - token_ids = all_input_ids[-new_input_length:] - tokens = self.tokenizer.batch_decode(token_ids) - # Add NaN for the first prompt token - logprobs = [float("nan")] + all_logprobs[-input_length:].squeeze( - 1 - ).tolist() # Get seed if isinstance(next_token_chooser.choice, Sampling): @@ -382,39 +368,58 @@ def generate_token( else: seed = None - # Add to the list of finished generations with the original request - generated_texts.append( - GeneratedText( - request=request, - output_text=output_text, - generated_tokens=stopping_criteria.current_tokens, - tokens=tokens, - token_ids=token_ids.squeeze(1).tolist(), - logprobs=logprobs, - reason=reason, - seed=seed, - ) + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed ) - # add to the next batch else: + # Keep request in the batch + generated_text = None next_batch_keep_indices.append(i) - next_batch_input_ids.append(next_token) + next_batch_input_ids.append(next_token_id) next_batch_all_input_ids.append(all_input_ids) - next_batch_all_logprobs.append(all_logprobs) next_batch_size += 1 next_batch_input_lengths.append(new_input_length) next_batch_max_sequence_length = max( next_batch_max_sequence_length, new_input_length ) + # Prefill + if stopping_criteria.current_tokens == 1: + # Remove generated token to only have prefill and add nan for first prompt token + prefill_logprobs = [float("nan")] + logprobs.gather( + 1, all_input_ids[1:] + ).squeeze(1)[-new_input_length:-1].tolist() + prefill_token_ids = all_input_ids[-new_input_length:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = PrefillTokens( + prefill_token_ids, prefill_logprobs, prefill_texts + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id_squeezed, + next_token_logprob, + next_token_text, + generated_text, + ) + + generations.append(generation) + # We finished all generations in the batch; there is no next batch if not next_batch_keep_indices: - return generated_texts, None + return generations, None next_batch_input_ids = torch.cat(next_batch_input_ids, dim=0) # If we finished at least one generation, we need to evict the indices of the generations that finished # from the values of the next batch - if generated_texts: + if len(next_batch_keep_indices) != len(batch): # Apply indices to attention mask, past key values and other items that need to be cached next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] next_batch_position_ids = batch.position_ids[next_batch_keep_indices] @@ -461,7 +466,6 @@ def generate_token( position_ids=next_batch_position_ids, past_key_values=next_batch_past_key_values, all_input_ids=next_batch_all_input_ids, - all_logprobs=next_batch_all_logprobs, input_lengths=next_batch_input_lengths, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, @@ -469,4 +473,4 @@ def generate_token( max_sequence_length=next_batch_max_sequence_length, keys_head_dim_last=batch.keys_head_dim_last, ) - return generated_texts, next_batch + return generations, next_batch diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index f965ea88..41657af2 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -5,7 +5,7 @@ from typing import Optional, Tuple, List, Type from text_generation.models import Model -from text_generation.models.types import GeneratedText, Batch +from text_generation.models.types import GeneratedText, Batch, Generation, PrefillTokens from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -30,7 +30,6 @@ class Seq2SeqLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] decoder_input_lengths: List[int] - decoder_logprobs: List[Optional[torch.Tensor]] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -64,7 +63,6 @@ def from_pb( decoder_input_ids = [] decoder_input_lengths = [] - decoder_logprobs = [] # Parse batch for r in pb.requests: @@ -77,7 +75,6 @@ def from_pb( stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) - decoder_logprobs.append(None) # Tokenize batch pad_to_multiple_of = 8 if device.type == "cuda" else None @@ -102,7 +99,6 @@ def from_pb( past_key_values=None, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, - decoder_logprobs=decoder_logprobs, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=len(pb.requests), @@ -125,7 +121,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests = [] input_lengths = [] decoder_input_lengths = [] - decoder_logprobs = [] next_token_choosers = [] stopping_criterias = [] @@ -146,7 +141,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) decoder_input_lengths.extend(batch.decoder_input_lengths) - decoder_logprobs.extend(batch.decoder_logprobs) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -283,7 +277,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": past_key_values=past_key_values, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, - decoder_logprobs=decoder_logprobs, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=total_batch_size, @@ -291,6 +284,9 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": max_decoder_input_length=max_decoder_input_length, ) + def __len__(self): + return len(self.requests) + class Seq2SeqLM(Model): def __init__(self, model_name: str, quantize=False): @@ -364,7 +360,7 @@ def forward( def generate_token( self, batch: Seq2SeqLMBatch - ) -> Tuple[List[GeneratedText], Optional[Seq2SeqLMBatch]]: + ) -> Tuple[List[Generation], Optional[Seq2SeqLMBatch]]: # For some reason, inference_mode does not work well with GLOO which we use on CPU context_manager = ( torch.no_grad if self.device.type == "cpu" else torch.inference_mode @@ -386,7 +382,6 @@ def generate_token( next_batch_input_lengths = [] next_batch_decoder_input_ids = [] next_batch_decoder_input_lengths = [] - next_batch_decoder_logprobs = [] # Metadata next_batch_size = 0 @@ -394,14 +389,13 @@ def generate_token( next_batch_max_decoder_input_length = 0 # Finished requests - generated_texts: List[GeneratedText] = [] + generations: List[Generation] = [] # Zipped iterator iterator = zip( batch.requests, batch.input_lengths, batch.decoder_input_lengths, - batch.decoder_logprobs, logits, batch.next_token_choosers, batch.stopping_criterias, @@ -414,7 +408,6 @@ def generate_token( request, input_length, decoder_input_length, - decoder_logprobs, logits, next_token_chooser, stopping_criteria, @@ -422,35 +415,28 @@ def generate_token( decoder_input_ids, ) in enumerate(iterator): # Select next token - next_token, logprobs = next_token_chooser(decoder_input_ids, logits) + next_token_id, logprobs = next_token_chooser(decoder_input_ids, logits) # Append next token to decoder tokens - decoder_input_ids = torch.cat([decoder_input_ids, next_token]) + decoder_input_ids = torch.cat([decoder_input_ids, next_token_id]) new_decoder_input_length = decoder_input_length + 1 - next_token_logprob = logprobs[-1, next_token] - if decoder_logprobs is None: - decoder_logprobs = next_token_logprob - else: - decoder_logprobs = torch.cat([decoder_logprobs, next_token_logprob]) + # Generated token + next_token_logprob = logprobs[-1, next_token_id] + next_token_id_squeezed = next_token_id.squeeze() + next_token_text = self.tokenizer.decode( + next_token_id_squeezed, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) # Evaluate stopping criteria - stop, reason = stopping_criteria( - next_token.squeeze(), - self.tokenizer.decode( - next_token.squeeze(), clean_up_tokenization_spaces=False - ), - ) + stop, reason = stopping_criteria(next_token_id, next_token_text) + if stop: # Slice with decoder_input_length to remove padding # Decode all tokens - token_ids = decoder_input_ids[-new_decoder_input_length:] - output_text = self.decode(token_ids) - tokens = self.tokenizer.batch_decode(token_ids) - # Add NaN for the bos token - logprobs = [float("nan")] + decoder_logprobs[ - -decoder_input_length: - ].tolist() + output_text = self.decode(decoder_input_ids[-new_decoder_input_length:]) # Get seed if isinstance(next_token_chooser.choice, Sampling): @@ -458,27 +444,17 @@ def generate_token( else: seed = None - # Add to the list of finished generations with the original request - generated_texts.append( - GeneratedText( - request=request, - output_text=output_text, - generated_tokens=stopping_criteria.current_tokens, - tokens=tokens, - token_ids=token_ids.tolist(), - logprobs=logprobs, - reason=reason, - seed=seed, - ) + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed ) - # add to the next batch else: + # Keep request in the batch + generated_text = None next_batch_keep_indices.append(i) next_batch_decoder_input_ids.append(decoder_input_ids.unsqueeze(0)) next_batch_size += 1 next_batch_input_lengths.append(input_length) next_batch_decoder_input_lengths.append(new_decoder_input_length) - next_batch_decoder_logprobs.append(decoder_logprobs) next_batch_max_input_length = max( next_batch_max_input_length, input_length ) @@ -486,14 +462,39 @@ def generate_token( next_batch_max_decoder_input_length, new_decoder_input_length ) + # Prefill + if stopping_criteria.current_tokens == 1: + prefill_token_ids = decoder_input_ids[-new_decoder_input_length:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = PrefillTokens( + prefill_token_ids, [float("nan")], prefill_texts + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id_squeezed, + next_token_logprob, + next_token_text, + generated_text, + ) + + generations.append(generation) + # We finished all generations in the batch; there is no next batch if not next_batch_keep_indices: - return generated_texts, None + return generations, None next_batch_decoder_input_ids = torch.cat(next_batch_decoder_input_ids) # If we finished at least one generation, we need to evict the indices of the generations that finished # from the values of the next batch - if generated_texts: + if len(next_batch_keep_indices) != len(batch): # Apply indices to attention mask, past key values and other items that need to be cached next_batch_input_ids = batch.input_ids[next_batch_keep_indices] next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] @@ -551,11 +552,10 @@ def generate_token( past_key_values=next_batch_past_key_values, input_lengths=next_batch_input_lengths, decoder_input_lengths=next_batch_decoder_input_lengths, - decoder_logprobs=next_batch_decoder_logprobs, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, max_input_length=next_batch_max_input_length, max_decoder_input_length=next_batch_max_decoder_input_length, ) - return generated_texts, next_batch + return generations, next_batch diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index 4ee3cb32..30cd716a 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -29,26 +29,61 @@ def from_pb( def concatenate(cls, batches: List["Batch"]) -> "Batch": raise NotImplementedError + @abstractmethod + def __len__(self): + raise NotImplementedError + @dataclass class GeneratedText: - request: generate_pb2.Request - output_text: str + text: str generated_tokens: int - tokens: List[str] - token_ids: List[int] - logprobs: List[float] - reason: str + finish_reason: str seed: Optional[int] def to_pb(self) -> generate_pb2.GeneratedText: return generate_pb2.GeneratedText( - request=self.request, - output_text=self.output_text, + text=self.text, generated_tokens=self.generated_tokens, - tokens=self.tokens, - token_ids=self.token_ids, - logprobs=self.logprobs, - finish_reason=self.reason, + finish_reason=self.finish_reason, seed=self.seed, ) + + +@dataclass +class PrefillTokens: + token_ids: List[int] + logprobs: List[float] + texts: List[str] + + def to_pb(self) -> generate_pb2.PrefillTokens: + return generate_pb2.PrefillTokens( + ids=self.token_ids, logprobs=self.logprobs, texts=self.texts + ) + + def __len__(self): + return len(self.token_ids) + + +@dataclass +class Generation: + request_id: int + prefill_tokens: Optional[PrefillTokens] + token_id: int + token_logprob: float + token_text: str + generated_text: Optional[GeneratedText] + + def to_pb(self) -> generate_pb2.Generation: + return generate_pb2.Generation( + request_id=self.request_id, + prefill_tokens=self.prefill_tokens.to_pb() + if self.prefill_tokens is not None + else None, + token_id=self.token_id, + token_logprob=self.token_logprob, + token_text=self.token_text, + generated_text=self.generated_text.to_pb() + if self.generated_text is not None + else None, + ) diff --git a/server/text_generation/server.py b/server/text_generation/server.py index 5fd3072e..a2bad8a7 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -27,22 +27,20 @@ async def ClearCache(self, request, context): self.cache.clear() return generate_pb2.ClearCacheResponse() - async def Generate(self, request, context): + async def Prefill(self, request, context): batch = self.model.batch_type.from_pb( request.batch, self.model.tokenizer, self.model.device ) - generated_texts, next_batch = self.model.generate_token(batch) + generations, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) - return generate_pb2.GenerateResponse( - generated_texts=[ - generated_text.to_pb() for generated_text in generated_texts - ], + return generate_pb2.PrefillResponse( + generations=[generation.to_pb() for generation in generations], batch=next_batch.to_pb() if next_batch else None, ) - async def GenerateWithCache(self, request, context): + async def Decode(self, request, context): if len(request.batches) == 0: raise ValueError("Must provide at least one batch") @@ -58,13 +56,11 @@ async def GenerateWithCache(self, request, context): else: batch = batches[0] - generated_texts, next_batch = self.model.generate_token(batch) + generations, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) - return generate_pb2.GenerateWithCacheResponse( - generated_texts=[ - generated_text.to_pb() for generated_text in generated_texts - ], + return generate_pb2.DecodeResponse( + generations=[generation.to_pb() for generation in generations], batch=next_batch.to_pb() if next_batch else None, ) From 4f9ac67cfa06348ab2375054af3e0ce62da2854e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 31 Jan 2023 14:21:51 +0100 Subject: [PATCH 057/793] Revert "feat: Add token streaming using ServerSideEvents support" (#40) Reverts huggingface/text-generation-inference#36 --- Cargo.lock | 2 - launcher/Cargo.toml | 2 +- launcher/tests/bloom_560m.json | 6 +- launcher/tests/mt0_base.json | 6 +- proto/generate.proto | 66 ++-- router/Cargo.toml | 2 - router/client/src/client.rs | 28 +- router/client/src/lib.rs | 3 +- router/client/src/sharded_client.rs | 38 +-- router/src/batcher.rs | 236 +++++++++++++ router/src/db.rs | 14 +- router/src/infer.rs | 354 -------------------- router/src/lib.rs | 21 +- router/src/server.rs | 307 ++++++----------- router/src/validation.rs | 15 +- server/tests/models/test_bloom.py | 105 +++--- server/tests/models/test_causal_lm.py | 100 +++--- server/tests/models/test_santacoder.py | 30 +- server/tests/models/test_seq2seq_lm.py | 93 +++-- server/text_generation/models/causal_lm.py | 110 +++--- server/text_generation/models/seq2seq_lm.py | 102 +++--- server/text_generation/models/types.py | 59 +--- server/text_generation/server.py | 20 +- 23 files changed, 702 insertions(+), 1017 deletions(-) create mode 100644 router/src/batcher.rs delete mode 100644 router/src/infer.rs diff --git a/Cargo.lock b/Cargo.lock index 4c44820b..33f5d181 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1829,7 +1829,6 @@ dependencies = [ name = "text-generation-router" version = "0.1.0" dependencies = [ - "async-stream", "axum", "clap 4.0.22", "futures", @@ -1841,7 +1840,6 @@ dependencies = [ "thiserror", "tokenizers", "tokio", - "tokio-stream", "tracing", "tracing-subscriber", ] diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 58df28d9..21d5d3ee 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -16,4 +16,4 @@ tracing-subscriber = { version = "0.3.16", features = ["json"] } [dev-dependencies] float_eq = "1.0.1" reqwest = { version = "0.11.13", features = ["blocking", "json"] } -serde = { version = "1.0.150", features = ["derive"] } +serde = "1.0.150" diff --git a/launcher/tests/bloom_560m.json b/launcher/tests/bloom_560m.json index a81d1982..d17f1ed4 100644 --- a/launcher/tests/bloom_560m.json +++ b/launcher/tests/bloom_560m.json @@ -3,7 +3,7 @@ "details": { "finish_reason": "length", "generated_tokens": 20, - "prefill": [ + "tokens": [ [ 10264, "Test", @@ -13,9 +13,7 @@ 8821, " request", -11.895094 - ] - ], - "tokens": [ + ], [ 17, ".", diff --git a/launcher/tests/mt0_base.json b/launcher/tests/mt0_base.json index 51cb8b5c..1b772282 100644 --- a/launcher/tests/mt0_base.json +++ b/launcher/tests/mt0_base.json @@ -3,14 +3,12 @@ "details": { "finish_reason": "length", "generated_tokens": 20, - "prefill": [ + "tokens": [ [ 0, "", null - ] - ], - "tokens": [ + ], [ 259, "", diff --git a/proto/generate.proto b/proto/generate.proto index 32ec9681..921bd5c0 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -7,10 +7,10 @@ service TextGenerationService { rpc ServiceDiscovery (ServiceDiscoveryRequest) returns (ServiceDiscoveryResponse) {} /// Empties batch cache rpc ClearCache (ClearCacheRequest) returns (ClearCacheResponse); - /// Prefill batch and decode first token - rpc Prefill (PrefillRequest) returns (PrefillResponse); - /// Decode token for a list of prefilled batches - rpc Decode (DecodeRequest) returns (DecodeResponse); + /// Generate tokens for a batch + rpc Generate (GenerateRequest) returns (GenerateResponse); + /// Generate tokens for a list of cached batches + rpc GenerateWithCache (GenerateWithCacheRequest) returns (GenerateWithCacheResponse); } /// Empty request @@ -70,60 +70,44 @@ message Batch { } message GeneratedText { + /// Request + Request request = 1; /// Output - string text = 1; + string output_text = 2; /// Number of generated tokens - uint32 generated_tokens = 2; + uint32 generated_tokens = 3; + /// Tokens + repeated string tokens = 4; + /// Token IDs + repeated uint32 token_ids = 5; + /// Logprobs + repeated float logprobs = 6; /// Finish reason - string finish_reason = 3; + string finish_reason = 7; /// Seed - optional uint64 seed = 4; + optional uint64 seed = 8; } -message PrefillTokens { - /// Prefill Token IDs - repeated uint32 ids = 1; - /// Prefill Logprobs - repeated float logprobs = 2; - /// Prefill tokens - repeated string texts = 3; -} - -message Generation { - /// Request ID - uint64 request_id = 1; - /// Prefill tokens (optional) - PrefillTokens prefill_tokens = 2; - /// Token ID - uint32 token_id = 3; - /// Logprob - float token_logprob = 4; - /// Text - string token_text = 5; - /// Complete generated text - GeneratedText generated_text = 6; -} - -message PrefillRequest { +message GenerateRequest { /// Batch Batch batch = 1; } -message PrefillResponse { - /// Generation - repeated Generation generations = 1; +message GenerateResponse { + /// Finished requests + repeated GeneratedText generated_texts = 1; /// Next batch (cached) optional Batch batch = 2; } -message DecodeRequest { +message GenerateWithCacheRequest { /// Cached batches repeated Batch batches = 1; } -message DecodeResponse { - /// Decodes - repeated Generation generations = 1; +message GenerateWithCacheResponse { + /// Finished requests + repeated GeneratedText generated_texts = 1; /// Next batch (cached) optional Batch batch = 2; -} \ No newline at end of file +} diff --git a/router/Cargo.toml b/router/Cargo.toml index d30d3b48..546f127f 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -13,7 +13,6 @@ name = "text-generation-router" path = "src/main.rs" [dependencies] -async-stream = "0.3.3" axum = { version = "0.5.16", features = ["json", "serde_json"] } text-generation-client = { path = "client" } clap = { version = "4.0.15", features = ["derive", "env"] } @@ -25,7 +24,6 @@ serde_json = "1.0.85" thiserror = "1.0.37" tokenizers = "0.13.0" tokio = { version = "1.21.1", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } -tokio-stream = "0.1.11" tracing = "0.1.36" tracing-subscriber = { version = "0.3.15", features = ["json"] } diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 77a43110..172d0bf7 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -70,36 +70,36 @@ impl Client { /// Generate one token for each request in the given batch /// - /// Returns Generation for each request in batch + /// Returns a list of generated texts of request that met their stopping criteria /// and the next cached batch #[instrument(skip(self))] - pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { - let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }); + pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { + let request = tonic::Request::new(GenerateRequest { batch: Some(batch) }); let response = self .stub - .prefill(request) - .instrument(info_span!("prefill")) + .generate(request) + .instrument(info_span!("generate")) .await? .into_inner(); - Ok((response.generations, response.batch)) + Ok((response.generated_texts, response.batch)) } - /// Generate one token for each request in the given cached batches + /// Generate one token for each request in the given cached batch /// - /// Returns Generation for each request in batches + /// Returns a list of generated texts of request that met their stopping criteria /// and the next cached batch #[instrument(skip(self))] - pub async fn decode( + pub async fn generate_with_cache( &mut self, batches: Vec, - ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(DecodeRequest { batches }); + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(GenerateWithCacheRequest { batches }); let response = self .stub - .decode(request) - .instrument(info_span!("decode")) + .generate_with_cache(request) + .instrument(info_span!("generate_with_cache")) .await? .into_inner(); - Ok((response.generations, response.batch)) + Ok((response.generated_texts, response.batch)) } } diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index e0546b16..295b009b 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -7,8 +7,7 @@ mod sharded_client; pub use client::Client; pub use pb::generate::v1::{ - Batch, GeneratedText, Generation, NextTokenChooserParameters, PrefillTokens, Request, - StoppingCriteriaParameters, + Batch, GeneratedText, NextTokenChooserParameters, Request, StoppingCriteriaParameters, }; pub use sharded_client::ShardedClient; use thiserror::Error; diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 56335f92..6c70afca 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,6 +1,6 @@ /// Multi shard Client use crate::Result; -use crate::{Batch, Client, Generation}; +use crate::{Batch, Client, GeneratedText}; use futures::future::join_all; use futures::future::select_all; use tonic::transport::Uri; @@ -37,46 +37,46 @@ impl ShardedClient { Self::from_master_client(master_client).await } - /// Clear the past generations cache - pub async fn clear_cache(&mut self) -> Result<()> { - let futures: Vec<_> = self - .clients - .iter_mut() - .map(|client| client.clear_cache()) - .collect(); - join_all(futures).await.into_iter().collect() - } - /// Generate one token for each request in the given batch /// - /// Returns Generation for each request in batch + /// Returns a list of generated texts of request that met their stopping criteria /// and the next cached batch - pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { + pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| Box::pin(client.prefill(batch.clone()))) + .map(|client| Box::pin(client.generate(batch.clone()))) .collect(); // As soon as we receive one response, we can return as all shards will return the same let (result, _, _) = select_all(futures).await; result } - /// Generate one token for each request in the given cached batches + /// Generate one token for each request in the given cached batch /// - /// Returns Generation for each request in batches + /// Returns a list of generated texts of request that met their stopping criteria /// and the next cached batch - pub async fn decode( + pub async fn generate_with_cache( &mut self, batches: Vec, - ) -> Result<(Vec, Option)> { + ) -> Result<(Vec, Option)> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| Box::pin(client.decode(batches.clone()))) + .map(|client| Box::pin(client.generate_with_cache(batches.clone()))) .collect(); // As soon as we receive one response, we can return as all shards will return the same let (result, _, _) = select_all(futures).await; result } + + /// Clear the past generations cache + pub async fn clear_cache(&mut self) -> Result<()> { + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| client.clear_cache()) + .collect(); + join_all(futures).await.into_iter().collect() + } } diff --git a/router/src/batcher.rs b/router/src/batcher.rs new file mode 100644 index 00000000..baf58af4 --- /dev/null +++ b/router/src/batcher.rs @@ -0,0 +1,236 @@ +/// Batching and inference logic +use crate::{Db, Entry}; +use crate::{ErrorResponse, GenerateRequest}; +use axum::http::StatusCode; +use axum::Json; +use nohash_hasher::IntMap; +use std::future::Future; +use std::sync::Arc; +use text_generation_client::{Batch, ClientError, GeneratedText, ShardedClient}; +use thiserror::Error; +use tokio::sync::{oneshot, Notify}; +use tokio::time::Instant; +use tracing::instrument; + +/// Batcher +#[derive(Clone)] +pub struct Batcher { + /// Request database + db: Db, + /// Shared state + shared: Arc, +} + +/// Batcher shared state +struct Shared { + /// Batching background Tokio task notifier + batching_task: Notify, +} + +impl Batcher { + pub(crate) fn new( + client: ShardedClient, + max_batch_size: usize, + max_waiting_tokens: usize, + ) -> Self { + // Batcher shared state + let db = Db::new(); + let shared = Arc::new(Shared { + batching_task: Notify::new(), + }); + + // Spawn batching background task that contains all the inference logic + tokio::spawn(batching_task( + client, + max_batch_size, + max_waiting_tokens, + db.clone(), + shared.clone(), + )); + + Self { db, shared } + } + + /// Add a new request to the database and return a future that will generate the text + pub(crate) async fn infer( + &self, + input_length: usize, + request: GenerateRequest, + ) -> Result { + // One shot channel to communicate with the background batching task + let (response_tx, response_rx) = oneshot::channel(); + + // Try to append the request to the database + self.db.append(Entry { + request, + response_tx, + input_length, + time: Instant::now(), + batch_time: None, + }); + + // Notify the background task that we have a new entry in the database that needs + // to be batched + self.shared.batching_task.notify_one(); + + // Await on the response from the background task + // We can safely unwrap as the background task will never drop the sender + response_rx + .await + .unwrap() + .map_err(|err| InferError::GenerationError(err.to_string())) + } +} + +/// Batching logic +/// Will be launched in a background Tokio task +/// +/// Batches requests and sends them to the inference server +#[instrument(skip(client, db, shared))] +async fn batching_task( + mut client: ShardedClient, + max_batch_size: usize, + max_waiting_tokens: usize, + db: Db, + shared: Arc, +) { + // Minimum batch size after which we try to add more requests + let limit_min_batch_size = (max_batch_size / 2) as u32; + + // Infinite loop + loop { + // Wait for a notification from the Batcher struct + shared.batching_task.notified().await; + + // Get the next batch from the DB + // This batch might be smaller than the maximum batch size if there are not enough requests + // waiting in the DB + while let Some((mut entries, batch)) = db.next_batch(None, max_batch_size) { + let mut cached_batch = wrap_future(client.generate(batch), &mut entries).await; + let mut waiting_tokens = 1; + + // We loop until we do not receive any cached batch from the inference server (== until + // all requests have met their stopping criteria) + while let Some(batch) = cached_batch { + // Get current batch info + let batch_size = batch.size; + let mut batches = vec![batch]; + + // If the current batch is too small, we try to add more requests to it + if batch_size <= limit_min_batch_size { + let min_size = match waiting_tokens { + // If we didn't onboard any new requests since >= max_waiting_tokens, we try + // to add a new batch even though its size might be small + _ if waiting_tokens >= max_waiting_tokens => None, + // Minimum size criteria + _ => Some(limit_min_batch_size as usize), + }; + + // Try to get a new batch + if let Some((mut new_entries, new_batch)) = + db.next_batch(min_size, max_batch_size - batch_size as usize) + { + // Generate one token for this new batch to have the attention past in cache + let new_cached_batch = + wrap_future(client.generate(new_batch), &mut new_entries).await; + // Reset waiting counter + waiting_tokens = 1; + // Extend current batch with the new batch + if let Some(new_cached_batch) = new_cached_batch { + entries.extend(new_entries); + batches.push(new_cached_batch); + } + } + } + + cached_batch = wrap_future(client.generate_with_cache(batches), &mut entries).await; + waiting_tokens += 1; + } + } + } +} + +/// Wrap a future inside a match statement to handle errors and send the response to the Batcher +async fn wrap_future( + future: impl Future, Option), ClientError>>, + entries: &mut IntMap, +) -> Option { + match future.await { + Ok((generated_texts, next_batch)) => { + send_generated(generated_texts, entries); + next_batch + } + // If we have an error, we discard the whole batch + Err(err) => { + send_error(err, entries); + None + } + } +} + +/// Send errors to the Batcher for all `entries` +fn send_error(error: ClientError, entries: &mut IntMap) { + entries.drain().for_each(|(_, entry)| { + // unwrap_or is valid here as we don't care if the receiver is gone. + entry.response_tx.send(Err(error.clone())).unwrap_or(()); + }); +} + +/// Send `generated_text` to the Batcher for all `finished` +fn send_generated(finished: Vec, entries: &mut IntMap) { + finished.into_iter().for_each(|output| { + // We can `expect` here as the request id should always be in the entries + let entry = entries + .remove(&output.request.unwrap().id) + .expect("ID not found in entries. This is a bug."); + + let response = InferResponse { + output_text: output.output_text, + generated_tokens: output.generated_tokens, + token_ids: output.token_ids, + tokens: output.tokens, + logprobs: output.logprobs, + finish_reason: output.finish_reason, + seed: output.seed, + queued: entry.time, + start: entry.batch_time.unwrap(), // unwrap is always valid + end: Instant::now(), + }; + // unwrap_or is valid here as we don't care if the receiver is gone. + entry.response_tx.send(Ok(response)).unwrap_or(()); + }); +} + +#[derive(Debug)] +pub(crate) struct InferResponse { + pub(crate) output_text: String, + pub(crate) generated_tokens: u32, + pub(crate) token_ids: Vec, + pub(crate) tokens: Vec, + pub(crate) logprobs: Vec, + pub(crate) finish_reason: String, + pub(crate) seed: Option, + pub(crate) queued: Instant, + pub(crate) start: Instant, + pub(crate) end: Instant, +} + +#[derive(Debug, Error)] +pub enum InferError { + #[error("Request failed during generation: {0}")] + GenerationError(String), +} + +/// Convert to Axum supported format +impl From for (StatusCode, Json) { + fn from(err: InferError) -> Self { + match err { + InferError::GenerationError(_) => ( + StatusCode::FAILED_DEPENDENCY, + Json(ErrorResponse { + error: err.to_string(), + }), + ), + } + } +} diff --git a/router/src/db.rs b/router/src/db.rs index f0a62d65..15007b64 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,16 +1,14 @@ /// This code is massively inspired by Tokio mini-redis -use crate::infer::InferError; -use crate::infer::InferStreamResponse; +use crate::InferResponse; use crate::{GenerateParameters, GenerateRequest}; use nohash_hasher::{BuildNoHashHasher, IntMap}; use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; use text_generation_client::{ - Batch, NextTokenChooserParameters, Request, StoppingCriteriaParameters, + Batch, ClientError, NextTokenChooserParameters, Request, StoppingCriteriaParameters, }; -use tokio::sync::mpsc::UnboundedSender; -use tokio::sync::OwnedSemaphorePermit; +use tokio::sync::oneshot::Sender; use tokio::time::Instant; /// Database entry @@ -18,16 +16,14 @@ use tokio::time::Instant; pub(crate) struct Entry { /// Request pub request: GenerateRequest, - /// Response sender to communicate between the Infer struct and the batching_task - pub response_tx: UnboundedSender>, + /// Response sender to communicate between the Batcher and the batching_task + pub response_tx: Sender>, /// Number of tokens in the input pub input_length: usize, /// Instant when this entry was created pub time: Instant, /// Instant when this entry was added to a batch pub batch_time: Option, - /// Permit - pub _permit: OwnedSemaphorePermit, } /// Request Database diff --git a/router/src/infer.rs b/router/src/infer.rs deleted file mode 100644 index 4c4a7eb8..00000000 --- a/router/src/infer.rs +++ /dev/null @@ -1,354 +0,0 @@ -/// Batching and inference logic -use crate::validation::{Validation, ValidationError}; -use crate::GenerateRequest; -use crate::{Db, Entry, Token}; -use nohash_hasher::IntMap; -use std::future::Future; -use std::sync::Arc; -use text_generation_client::{ - Batch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, -}; -use thiserror::Error; -use tokio::sync::{mpsc, Notify, Semaphore, TryAcquireError}; -use tokio::time::Instant; -use tokio_stream::wrappers::UnboundedReceiverStream; -use tokio_stream::StreamExt; -use tracing::instrument; - -/// Inference struct -#[derive(Clone)] -pub struct Infer { - /// Validation - validation: Validation, - /// Request database - db: Db, - /// Shared state - shared: Arc, - /// Inference limit - limit_concurrent_requests: Arc, -} - -/// Infer shared state -struct Shared { - /// Batching background Tokio task notifier - batching_task: Notify, -} - -impl Infer { - pub(crate) fn new( - client: ShardedClient, - validation: Validation, - max_batch_size: usize, - max_waiting_tokens: usize, - max_concurrent_requests: usize, - ) -> Self { - // Infer shared state - let db = Db::new(); - let shared = Arc::new(Shared { - batching_task: Notify::new(), - }); - - // Spawn batching background task that contains all the inference logic - tokio::spawn(batching_task( - client, - max_batch_size, - max_waiting_tokens, - db.clone(), - shared.clone(), - )); - - // Inference limit with a semaphore - let semaphore = Arc::new(Semaphore::new(max_concurrent_requests)); - - Self { - validation, - db, - shared, - limit_concurrent_requests: semaphore, - } - } - - /// Add a new request to the database and return a stream of InferStreamResponse - pub(crate) async fn generate_stream( - &self, - request: GenerateRequest, - ) -> Result>, InferError> { - // Limit concurrent requests by acquiring a permit from the semaphore - // This permit will live as long as Entry - let permit = self.clone().limit_concurrent_requests.try_acquire_owned()?; - - // Validate request - let (input_length, validated_request) = self.validation.validate(request).await?; - - // MPSC channel to communicate with the background batching task - let (response_tx, response_rx) = mpsc::unbounded_channel(); - - // Append the request to the database - self.db.append(Entry { - request: validated_request, - response_tx, - input_length, - time: Instant::now(), - batch_time: None, - _permit: permit, - }); - - // Notify the background task that we have a new entry in the database that needs - // to be batched - self.shared.batching_task.notify_one(); - - // Return stream - Ok(UnboundedReceiverStream::new(response_rx)) - } - - /// Add a new request to the database and return a InferResponse - pub(crate) async fn generate( - &self, - request: GenerateRequest, - ) -> Result { - // Create stream - let mut stream = self.generate_stream(request).await?; - - // Return values - let mut result_prefill = Vec::new(); - let mut result_tokens = Vec::new(); - let mut result_generated_text = None; - let mut result_start = None; - let mut result_queued = None; - - // Iterate on stream - while let Some(response) = stream.next().await { - match response? { - // Add prefill tokens - InferStreamResponse::Prefill(tokens) => { - // Create Token objects - // We do that here instead of in the Python code as Rust for loops are faster - result_prefill = tokens - .ids - .into_iter() - .zip(tokens.logprobs.into_iter()) - .zip(tokens.texts.into_iter()) - .map(|((id, logprob), text)| Token(id, text, logprob)) - .collect(); - } - // Push last token - InferStreamResponse::Token(token) => result_tokens.push(token), - // Final message - // Set return values - InferStreamResponse::End { - token, - generated_text, - start, - queued, - } => { - result_tokens.push(token); - result_generated_text = Some(generated_text); - result_start = Some(start); - result_queued = Some(queued) - } - } - } - - // Check that we received a `InferStreamResponse::End` message - if let (Some(generated_text), Some(queued), Some(start)) = - (result_generated_text, result_queued, result_start) - { - Ok(InferResponse { - prefill: result_prefill, - tokens: result_tokens, - generated_text, - queued, - start, - }) - } else { - Err(InferError::IncompleteGeneration) - } - } -} - -/// Batching logic -/// Will be launched in a background Tokio task -/// -/// Batches requests and sends them to the inference server -#[instrument(skip(client, db, shared))] -async fn batching_task( - mut client: ShardedClient, - max_batch_size: usize, - max_waiting_tokens: usize, - db: Db, - shared: Arc, -) { - // Minimum batch size after which we try to add more requests - let limit_min_batch_size = (max_batch_size / 2) as u32; - - // Infinite loop - loop { - // Wait for a notification from the Infer struct - shared.batching_task.notified().await; - - // Get the next batch from the DB - // This batch might be smaller than the maximum batch size if there are not enough requests - // waiting in the DB - while let Some((mut entries, batch)) = db.next_batch(None, max_batch_size) { - let mut cached_batch = wrap_future(client.prefill(batch), &mut entries).await; - let mut waiting_tokens = 1; - - // We loop until we do not receive any cached batch from the inference server (== until - // all requests have met their stopping criteria) - while let Some(batch) = cached_batch { - // Get current batch info - let batch_size = batch.size; - let mut batches = vec![batch]; - - // If the current batch is too small, we try to add more requests to it - if batch_size <= limit_min_batch_size { - let min_size = match waiting_tokens { - // If we didn't onboard any new requests since >= max_waiting_tokens, we try - // to add a new batch even though its size might be small - _ if waiting_tokens >= max_waiting_tokens => None, - // Minimum size criteria - _ => Some(limit_min_batch_size as usize), - }; - - // Try to get a new batch - if let Some((mut new_entries, new_batch)) = - db.next_batch(min_size, max_batch_size - batch_size as usize) - { - // Generate one token for this new batch to have the attention past in cache - let new_cached_batch = - wrap_future(client.prefill(new_batch), &mut new_entries).await; - // Reset waiting counter - waiting_tokens = 1; - // Extend current batch with the new batch - if let Some(new_cached_batch) = new_cached_batch { - entries.extend(new_entries); - batches.push(new_cached_batch); - } - } - } - - cached_batch = wrap_future(client.decode(batches), &mut entries).await; - waiting_tokens += 1; - } - } - } -} - -/// Wrap a future inside a match statement to handle errors and send the responses to Infer -async fn wrap_future( - future: impl Future, Option), ClientError>>, - entries: &mut IntMap, -) -> Option { - match future.await { - Ok((generations, next_batch)) => { - send_generations(generations, entries); - next_batch - } - // If we have an error, we discard the whole batch - Err(err) => { - send_error(err, entries); - None - } - } -} - -/// Send errors to Infer for all `entries` -fn send_error(error: ClientError, entries: &mut IntMap) { - entries.drain().for_each(|(_, entry)| { - // unwrap_or is valid here as we don't care if the receiver is gone. - entry - .response_tx - .send(Err(InferError::GenerationError(error.to_string()))) - .unwrap_or(()); - }); -} - -/// Send one or multiple `InferStreamResponse` to Infer for all `entries` -fn send_generations(generations: Vec, entries: &mut IntMap) { - generations.into_iter().for_each(|generation| { - // Get entry - // We can `expect` here as the request id should always be in the entries - let entry = entries - .get(&generation.request_id) - .expect("ID not found in entries. This is a bug."); - - if let Some(prefill_tokens) = generation.prefill_tokens { - // Send message - // unwrap_or is valid here as we don't care if the receiver is gone. - entry - .response_tx - .send(Ok(InferStreamResponse::Prefill(prefill_tokens))) - .unwrap_or(()); - } - - // Create last Token - let token = Token( - generation.token_id, - generation.token_text, - generation.token_logprob, - ); - - if let Some(generated_text) = generation.generated_text { - // Remove entry as this is the last message - // We can `expect` here as the request id should always be in the entries - let entry = entries - .remove(&generation.request_id) - .expect("ID not found in entries. This is a bug."); - - // Send message - // unwrap_or is valid here as we don't care if the receiver is gone. - entry - .response_tx - .send(Ok(InferStreamResponse::End { - token, - generated_text, - queued: entry.time, - start: entry.batch_time.unwrap(), - })) - .unwrap_or(()); - } else { - // Send message - // unwrap_or is valid here as we don't care if the receiver is gone. - entry - .response_tx - .send(Ok(InferStreamResponse::Token(token))) - .unwrap_or(()); - } - }); -} - -#[derive(Debug)] -pub(crate) enum InferStreamResponse { - // Optional first message - Prefill(PrefillTokens), - // Intermediate messages - Token(Token), - // Last message - End { - token: Token, - generated_text: GeneratedText, - start: Instant, - queued: Instant, - }, -} - -#[derive(Debug)] -pub(crate) struct InferResponse { - pub(crate) prefill: Vec, - pub(crate) tokens: Vec, - pub(crate) generated_text: GeneratedText, - pub(crate) queued: Instant, - pub(crate) start: Instant, -} - -#[derive(Debug, Error)] -pub enum InferError { - #[error("Request failed during generation: {0}")] - GenerationError(String), - #[error("Model is overloaded")] - Overloaded(#[from] TryAcquireError), - #[error("Input validation error: {0}")] - ValidationError(#[from] ValidationError), - #[error("Incomplete generation")] - IncompleteGeneration, -} diff --git a/router/src/lib.rs b/router/src/lib.rs index beab7138..1aeac302 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,11 +1,11 @@ /// Text Generation Inference Webserver +mod batcher; mod db; -mod infer; pub mod server; mod validation; +use batcher::{Batcher, InferResponse}; use db::{Db, Entry}; -use infer::Infer; use serde::{Deserialize, Serialize}; use validation::Validation; @@ -69,34 +69,21 @@ pub(crate) struct GenerateRequest { pub parameters: GenerateParameters, } -#[derive(Debug, Serialize)] -pub struct Token(u32, String, f32); - #[derive(Serialize)] pub(crate) struct Details { pub finish_reason: String, pub generated_tokens: u32, pub seed: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub prefill: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub tokens: Option>, + pub tokens: Vec<(u32, String, f32)>, } #[derive(Serialize)] -pub(crate) struct GenerateResponse { +pub(crate) struct GeneratedText { pub generated_text: String, #[serde(skip_serializing_if = "Option::is_none")] pub details: Option
, } -#[derive(Serialize)] -pub(crate) struct StreamResponse { - pub token: Token, - pub generated_text: Option, - pub details: Option
, -} - #[derive(Serialize)] pub(crate) struct ErrorResponse { pub error: String, diff --git a/router/src/server.rs b/router/src/server.rs index ef3782d6..86041b96 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,54 +1,71 @@ -/// HTTP Server logic -use crate::infer::{InferError, InferStreamResponse}; use crate::{ - Details, ErrorResponse, GenerateParameters, GenerateRequest, GenerateResponse, Infer, - StreamResponse, Validation, + Batcher, Details, ErrorResponse, GenerateParameters, GenerateRequest, GeneratedText, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, StatusCode}; -use axum::response::sse::{Event, KeepAlive, Sse}; use axum::response::IntoResponse; use axum::routing::{get, post}; use axum::{Json, Router}; -use futures::Stream; -use std::convert::Infallible; use std::net::SocketAddr; +use std::sync::Arc; use text_generation_client::ShardedClient; use tokenizers::Tokenizer; use tokio::signal; +use tokio::sync::Semaphore; use tokio::time::Instant; -use tokio_stream::StreamExt; use tracing::instrument; +// Server shared state +#[derive(Clone)] +struct ServerState { + validation: Validation, + batcher: Batcher, + limit_concurrent_requests: Arc, +} + /// Health check method -#[instrument(skip(infer))] -async fn health(infer: Extension) -> Result<(), (StatusCode, Json)> { +#[instrument(skip(state), fields(time, time_per_token))] +async fn health(state: Extension) -> Result<(), (StatusCode, Json)> { // TODO: while this is the best health check we can do, it is a bit on the heavy side and might // be a bit too slow for a health check. // What we should do instead if check if the gRPC channels are still healthy. + // Limit concurrent requests by acquiring a permit from the semaphore + let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { + ( + StatusCode::TOO_MANY_REQUESTS, + Json(ErrorResponse { + error: "Model is overloaded".to_string(), + }), + ) + })?; + // Send a small inference request - infer - .generate(GenerateRequest { - inputs: "liveness".to_string(), - parameters: GenerateParameters { - temperature: 1.0, - top_k: 0, - top_p: 1.0, - do_sample: false, - max_new_tokens: 1, - stop: vec![], - details: false, - seed: None, + state + .batcher + .infer( + 1, + GenerateRequest { + inputs: "liveness".to_string(), + parameters: GenerateParameters { + temperature: 1.0, + top_k: 0, + top_p: 1.0, + do_sample: false, + max_new_tokens: 1, + stop: vec![], + details: false, + seed: None, + }, }, - }) + ) .await?; Ok(()) } /// Generate method #[instrument( - skip(infer), + skip(state), fields( total_time, validation_time, @@ -59,28 +76,56 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json, + state: Extension, req: Json, ) -> Result)> { - let span = tracing::Span::current(); let start_time = Instant::now(); + // Limit concurrent requests by acquiring a permit from the semaphore + let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { + tracing::error!("Model is overloaded"); + ( + StatusCode::TOO_MANY_REQUESTS, + Json(ErrorResponse { + error: "Model is overloaded".to_string(), + }), + ) + })?; - // Inference + // Validate request let details = req.0.parameters.details; - let response = infer.generate(req.0).await.map_err(|err| { - tracing::error!("{}", err.to_string()); - err - })?; + let (input_length, validated_request) = + state.validation.validate(req.0).await.map_err(|err| { + tracing::error!("{}", err.to_string()); + err + })?; + + // Inference + let response = state + .batcher + .infer(input_length, validated_request) + .await + .map_err(|err| { + tracing::error!("{}", err.to_string()); + err + })?; // Token details let details = match details { - true => Some(Details { - finish_reason: response.generated_text.finish_reason, - generated_tokens: response.generated_text.generated_tokens, - prefill: Some(response.prefill), - tokens: Some(response.tokens), - seed: response.generated_text.seed, - }), + true => { + let tokens = response + .token_ids + .into_iter() + .zip(response.tokens.into_iter()) + .zip(response.logprobs.into_iter()) + .map(|((id, text), logprob)| (id, text, logprob)) + .collect(); + Some(Details { + seed: response.seed, + finish_reason: response.finish_reason, + generated_tokens: response.generated_tokens, + tokens, + }) + } false => None, }; @@ -88,8 +133,8 @@ async fn generate( let total_time = start_time.elapsed(); let validation_time = response.queued - start_time; let queue_time = response.start - response.queued; - let inference_time = Instant::now() - response.start; - let time_per_token = inference_time / response.generated_text.generated_tokens; + let inference_time = response.end - response.start; + let time_per_token = inference_time / response.generated_tokens; // Headers let mut headers = HeaderMap::new(); @@ -115,143 +160,22 @@ async fn generate( ); // Tracing metadata - span.record("total_time", format!("{:?}", total_time)); - span.record("validation_time", format!("{:?}", validation_time)); - span.record("queue_time", format!("{:?}", queue_time)); - span.record("inference_time", format!("{:?}", inference_time)); - span.record("time_per_token", format!("{:?}", time_per_token)); - span.record("seed", format!("{:?}", response.generated_text.seed)); - tracing::info!("Output: {}", response.generated_text.text); + tracing::Span::current().record("total_time", format!("{:?}", total_time)); + tracing::Span::current().record("validation_time", format!("{:?}", validation_time)); + tracing::Span::current().record("queue_time", format!("{:?}", queue_time)); + tracing::Span::current().record("inference_time", format!("{:?}", inference_time)); + tracing::Span::current().record("time_per_token", format!("{:?}", time_per_token)); + tracing::Span::current().record("seed", format!("{:?}", response.seed)); + tracing::info!("Output: {}", response.output_text); // Send response - let response = vec![GenerateResponse { - generated_text: response.generated_text.text, + let response = vec![GeneratedText { + generated_text: response.output_text, details, }]; Ok((headers, Json(response))) } -/// Generate stream method -#[instrument( - skip(infer), - fields( - total_time, - validation_time, - queue_time, - inference_time, - time_per_token - ) -)] -async fn generate_stream( - infer: Extension, - req: Json, -) -> Sse>> { - let span = tracing::Span::current(); - let start_time = Instant::now(); - - let stream = async_stream::stream! { - // Inference - let mut end_reached = false; - let mut error = false; - let details = req.0.parameters.details; - - match infer.generate_stream(req.0).await { - Ok(mut response_stream) => { - // Server Side Event stream - while let Some(response) = response_stream.next().await { - match response { - Ok(response) => { - match response { - // Prefill is ignored - InferStreamResponse::Prefill(_) => {} - // Yield event for every new token - InferStreamResponse::Token(token) => { - // StreamResponse - let stream_token = StreamResponse { - token, - generated_text: None, - details: None, - }; - - yield Ok(Event::default().json_data(stream_token).unwrap()) - } - // Yield event for last token and compute timings - InferStreamResponse::End { - token, - generated_text, - start, - queued, - } => { - // Token details - let details = match details { - true => Some(Details { - finish_reason: generated_text.finish_reason, - generated_tokens: generated_text.generated_tokens, - prefill: None, - tokens: None, - seed: generated_text.seed, - }), - false => None, - }; - - // Timings - let total_time = start_time.elapsed(); - let validation_time = queued - start_time; - let queue_time = start - queued; - let inference_time = Instant::now() - start; - let time_per_token = inference_time / generated_text.generated_tokens; - - // Tracing metadata - span.record("total_time", format!("{:?}", total_time)); - span - .record("validation_time", format!("{:?}", validation_time)); - span.record("queue_time", format!("{:?}", queue_time)); - span - .record("inference_time", format!("{:?}", inference_time)); - span - .record("time_per_token", format!("{:?}", time_per_token)); - tracing::info!(parent: &span, "Output: {}", generated_text.text); - - // StreamResponse - end_reached = true; - let stream_token = StreamResponse { - token, - generated_text: Some(generated_text.text), - details - }; - - yield Ok(Event::default().json_data(stream_token).unwrap()) - } - } - } - // Trace and yield error - Err(err) => { - error = true; - tracing::error!("{}", err.to_string()); - yield Ok(Event::from(err)) - } - } - } - }, - // Trace and yield error - Err(err) => { - error = true; - tracing::error!("{}", err.to_string()); - yield Ok(Event::from(err)) - } - } - // Check if generation reached the end - // Skip if we already sent an error - if !end_reached && !error { - let err = InferError::IncompleteGeneration; - tracing::error!("{}", err.to_string()); - yield Ok(Event::from(err)) - } - }; - - Sse::new(stream).keep_alive(KeepAlive::default()) -} - /// Serving method #[allow(clippy::too_many_arguments)] pub async fn run( @@ -265,23 +189,21 @@ pub async fn run( addr: SocketAddr, ) { // Create state + let batcher = Batcher::new(client, max_batch_size, max_waiting_tokens); let validation = Validation::new(validation_workers, tokenizer, max_input_length); - let infer = Infer::new( - client, + let shared_state = ServerState { validation, - max_batch_size, - max_waiting_tokens, - max_concurrent_requests, - ); + batcher, + limit_concurrent_requests: Arc::new(Semaphore::new(max_concurrent_requests)), + }; // Create router let app = Router::new() .route("/", post(generate)) .route("/generate", post(generate)) - .route("/generate_stream", post(generate_stream)) .route("/", get(health)) .route("/health", get(health)) - .layer(Extension(infer)); + .layer(Extension(shared_state.clone())); // Run server axum::Server::bind(&addr) @@ -318,32 +240,3 @@ async fn shutdown_signal() { tracing::info!("signal received, starting graceful shutdown"); } - -/// Convert to Axum supported formats -impl From for (StatusCode, Json) { - fn from(err: InferError) -> Self { - let status_code = match err { - InferError::GenerationError(_) => StatusCode::FAILED_DEPENDENCY, - InferError::Overloaded(_) => StatusCode::TOO_MANY_REQUESTS, - InferError::ValidationError(_) => StatusCode::UNPROCESSABLE_ENTITY, - InferError::IncompleteGeneration => StatusCode::INTERNAL_SERVER_ERROR, - }; - - ( - status_code, - Json(ErrorResponse { - error: err.to_string(), - }), - ) - } -} - -impl From for Event { - fn from(err: InferError) -> Self { - Event::default() - .json_data(ErrorResponse { - error: err.to_string(), - }) - .unwrap() - } -} diff --git a/router/src/validation.rs b/router/src/validation.rs index d2287168..aabc82a6 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,5 +1,7 @@ /// Payload validation logic -use crate::GenerateRequest; +use crate::{ErrorResponse, GenerateRequest}; +use axum::http::StatusCode; +use axum::Json; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokio::sync::{mpsc, oneshot}; @@ -159,3 +161,14 @@ pub enum ValidationError { #[error("tokenizer error {0}")] Tokenizer(String), } + +impl From for (StatusCode, Json) { + fn from(err: ValidationError) -> Self { + ( + StatusCode::UNPROCESSABLE_ENTITY, + Json(ErrorResponse { + error: err.to_string(), + }), + ) + } +} diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 9f96efc3..1a788ce5 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -91,9 +91,9 @@ def test_causal_lm_batch_type(default_bloom): def test_causal_lm_generate_token(default_bloom, default_bloom_batch): sequence_length = len(default_bloom_batch.all_input_ids[0]) - generations, next_batch = default_bloom.generate_token(default_bloom_batch) + generated_texts, next_batch = default_bloom.generate_token(default_bloom_batch) - assert len(generations) == len(default_bloom_batch) + assert generated_texts == [] assert isinstance(next_batch, CausalLMBatch) assert not next_batch.keys_head_dim_last @@ -122,30 +122,24 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): assert all( [p[1].shape == (16, sequence_length, 64) for p in next_batch.past_key_values] ) - assert all([generation.generated_text is None for generation in generations]) - assert all([len(generation.prefill_tokens) == 1 for generation in generations]) - assert all([generation.token_id.item() == 10264 for generation in generations]) - assert all([generation.token_text == "Test" for generation in generations]) - assert generations[0].request_id == 0 def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch): next_batch = default_bloom_batch for _ in range(default_bloom_batch.stopping_criterias[0].max_new_tokens - 1): - generations, next_batch = default_bloom.generate_token(next_batch) - assert len(generations) == len(default_bloom_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_bloom.generate_token(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 + assert len(generated_texts) == 1 assert ( - generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTestTest" + generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" ) - assert generations[0].request_id == default_bloom_batch.requests[0].id + assert generated_texts[0].request == default_bloom_batch.requests[0] assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -158,19 +152,17 @@ def test_causal_lm_generate_token_completion_multi( for i in range( default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 1 ): - generations, next_batch = default_bloom.generate_token(next_batch) - assert len(generations) == len(default_multi_requests_bloom_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_bloom.generate_token(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) assert next_batch is not None - assert len(generations) == 2 - assert generations[1].generated_text.text == "TestTestTestTestTestTest" + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "TestTestTestTestTestTest" + assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] assert ( - generations[1].request_id == default_multi_requests_bloom_batch.requests[1].id - ) - assert ( - generations[1].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) @@ -179,22 +171,19 @@ def test_causal_lm_generate_token_completion_multi( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 1 ): - generations, next_batch = default_bloom.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_bloom.generate_token(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 - assert ( - generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTestTest" - ) + assert len(generated_texts) == 1 assert ( - generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id + generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" ) + assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -254,19 +243,17 @@ def test_batch_concatenate( for _ in range( default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 2 ): - generations, next_batch = default_bloom.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_bloom.generate_token(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) assert next_batch is not None - assert len(generations) == 3 - assert generations[2].generated_text.text == "TestTestTestTestTestTest" + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "TestTestTestTestTestTest" + assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] assert ( - generations[2].request_id == default_multi_requests_bloom_batch.requests[1].id - ) - assert ( - generations[2].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) @@ -275,20 +262,19 @@ def test_batch_concatenate( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 2 ): - generations, next_batch = default_bloom.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_bloom.generate_token(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) assert next_batch is not None - assert len(generations) == 2 + assert len(generated_texts) == 1 assert ( - generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTestTest" + generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" ) - assert generations[0].request_id == default_bloom_batch.requests[0].id + assert generated_texts[0].request == default_bloom_batch.requests[0] assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -298,21 +284,18 @@ def test_batch_concatenate( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 4 ): - generations, next_batch = default_bloom.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_bloom.generate_token(next_batch) + generated_texts, next_batch = default_bloom.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 - assert ( - generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTestTest" - ) + assert len(generated_texts) == 1 assert ( - generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id + generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" ) + assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index f9762b30..bedb65ba 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -88,9 +88,11 @@ def test_causal_lm_batch_type(default_causal_lm): def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): sequence_length = len(default_causal_lm_batch.all_input_ids[0]) - generations, next_batch = default_causal_lm.generate_token(default_causal_lm_batch) + generated_texts, next_batch = default_causal_lm.generate_token( + default_causal_lm_batch + ) - assert len(generations) == len(next_batch) + assert generated_texts == [] assert isinstance(next_batch, CausalLMBatch) assert len(next_batch.all_input_ids) == next_batch.size @@ -119,11 +121,6 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): assert all( [p[1].shape == (1, 12, sequence_length, 64) for p in next_batch.past_key_values] ) - assert all([generation.generated_text is None for generation in generations]) - assert all([len(generation.prefill_tokens) == 1 for generation in generations]) - assert all([generation.token_id.item() == 13 for generation in generations]) - assert all([generation.token_text == "." for generation in generations]) - assert generations[0].request_id == 0 def test_causal_lm_generate_token_completion( @@ -131,17 +128,18 @@ def test_causal_lm_generate_token_completion( ): next_batch = default_causal_lm_batch for _ in range(default_causal_lm_batch.stopping_criterias[0].max_new_tokens - 1): - generations, next_batch = default_causal_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_causal_lm.generate_token(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 - assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." - assert generations[0].request_id == default_causal_lm_batch.requests[0].id + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." + assert generated_texts[0].request == default_causal_lm_batch.requests[0] + assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -154,20 +152,19 @@ def test_causal_lm_generate_token_completion_multi( for i in range( default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 1 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_causal_lm.generate_token(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is not None - assert len(generations) == 2 - assert generations[1].generated_text.text == "Test.java:784)" + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "Test.java:784)" assert ( - generations[1].request_id - == default_multi_requests_causal_lm_batch.requests[1].id + generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] ) assert ( - generations[1].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) @@ -176,20 +173,19 @@ def test_causal_lm_generate_token_completion_multi( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 1 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_causal_lm.generate_token(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 - assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." assert ( - generations[0].request_id - == default_multi_requests_causal_lm_batch.requests[0].id + generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] ) assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -248,20 +244,19 @@ def test_batch_concatenate( for _ in range( default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_causal_lm.generate_token(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is not None - assert len(generations) == 3 - assert generations[2].generated_text.text == "Test.java:784)" + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "Test.java:784)" assert ( - generations[2].request_id - == default_multi_requests_causal_lm_batch.requests[1].id + generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] ) assert ( - generations[2].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) @@ -270,17 +265,17 @@ def test_batch_concatenate( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_causal_lm.generate_token(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is not None - assert len(generations) == 2 - assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." - assert generations[0].request_id == default_causal_lm_batch.requests[0].id + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." + assert generated_texts[0].request == default_causal_lm_batch.requests[0] assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -290,19 +285,18 @@ def test_batch_concatenate( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 4 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_causal_lm.generate_token(next_batch) + generated_texts, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 - assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." assert ( - generations[0].request_id - == default_multi_requests_causal_lm_batch.requests[0].id + generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] ) assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index 1b69477d..acebec04 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -50,17 +50,18 @@ def test_santacoder_generate_token_completion(default_santacoder, default_pb_bat next_batch = batch for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): - generations, next_batch = default_santacoder.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_santacoder.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_santacoder.generate_token(next_batch) + generated_texts, next_batch = default_santacoder.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 - assert generations[0].generated_text.text == "def test_get_all_users_with_" - assert generations[0].request_id == batch.requests[0].id + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "def test_get_all_users_with_" + assert generated_texts[0].request == batch.requests[0] + assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == batch.stopping_criterias[0].max_new_tokens ) @@ -75,19 +76,20 @@ def test_fim_santacoder_generate_token_completion( next_batch = batch for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): - generations, next_batch = default_santacoder.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_santacoder.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_santacoder.generate_token(next_batch) + generated_texts, next_batch = default_santacoder.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 + assert len(generated_texts) == 1 assert ( - generations[0].generated_text.text + generated_texts[0].output_text == """defworldineProperty(exports, "__esModule", { value""" ) - assert generations[0].request_id == batch.requests[0].id + assert generated_texts[0].request == batch.requests[0] + assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) assert ( - generations[0].generated_text.generated_tokens + generated_texts[0].generated_tokens == batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 22c6ac9c..de1a4829 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -99,11 +99,11 @@ def test_seq2seq_lm_batch_type(default_seq2seq_lm): def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch): sequence_length = len(default_seq2seq_lm_batch.input_ids[0]) - generations, next_batch = default_seq2seq_lm.generate_token( + generated_texts, next_batch = default_seq2seq_lm.generate_token( default_seq2seq_lm_batch ) - assert len(generations) == len(next_batch) + assert generated_texts == [] assert isinstance(next_batch, Seq2SeqLMBatch) assert torch.equal(next_batch.input_ids, default_seq2seq_lm_batch.input_ids) @@ -145,11 +145,6 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) for p in next_batch.past_key_values ] ) - assert all([generation.generated_text is None for generation in generations]) - assert all([len(generation.prefill_tokens) == 1 for generation in generations]) - assert all([generation.token_id.item() == 259 for generation in generations]) - assert all([generation.token_text == "" for generation in generations]) - assert generations[0].request_id == 0 def test_seq2seq_lm_generate_token_completion( @@ -157,16 +152,16 @@ def test_seq2seq_lm_generate_token_completion( ): next_batch = default_seq2seq_lm_batch for _ in range(6): - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 - assert generations[0].generated_text.text == "a few weeks" - assert generations[0].request_id == default_seq2seq_lm_batch.requests[0].id - assert generations[0].generated_text.generated_tokens == 7 + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "a few weeks" + assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] + assert generated_texts[0].generated_tokens == 7 def test_seq2seq_lm_generate_token_completion_multi( @@ -175,33 +170,33 @@ def test_seq2seq_lm_generate_token_completion_multi( next_batch = default_multi_requests_seq2seq_lm_batch for i in range(4): - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None - assert len(generations) == 2 - assert generations[1].generated_text.text == "a few " + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "a few " assert ( - generations[1].request_id - == default_multi_requests_seq2seq_lm_batch.requests[1].id + generated_texts[0].request + == default_multi_requests_seq2seq_lm_batch.requests[1] ) - assert generations[1].generated_text.generated_tokens == 5 + assert generated_texts[0].generated_tokens == 5 - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 - assert generations[0].generated_text.text == "a few weeks" + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "a few weeks" assert ( - generations[0].request_id - == default_multi_requests_seq2seq_lm_batch.requests[0].id + generated_texts[0].request + == default_multi_requests_seq2seq_lm_batch.requests[0] ) - assert generations[0].generated_text.generated_tokens == 7 + assert generated_texts[0].generated_tokens == 7 def test_batch_concatenate( @@ -296,35 +291,35 @@ def test_batch_concatenate( ) for _ in range(3): - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert len(generations) == len(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert generated_texts == [] - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None - assert len(generations) == 3 - assert generations[2].generated_text.text == "a few " + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "a few " assert ( - generations[2].request_id - == default_multi_requests_seq2seq_lm_batch.requests[1].id + generated_texts[0].request + == default_multi_requests_seq2seq_lm_batch.requests[1] ) - assert generations[2].generated_text.generated_tokens == 5 + assert generated_texts[0].generated_tokens == 5 - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None - assert len(generations) == 2 - assert generations[0].generated_text.text == "a few weeks" - assert generations[0].request_id == default_seq2seq_lm_batch.requests[0].id - assert generations[0].generated_text.generated_tokens == 7 + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "a few weeks" + assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] + assert generated_texts[0].generated_tokens == 7 - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None - assert len(generations) == 1 - assert generations[0].generated_text.text == "a few weeks" + assert len(generated_texts) == 1 + assert generated_texts[0].output_text == "a few weeks" assert ( - generations[0].request_id - == default_multi_requests_seq2seq_lm_batch.requests[0].id + generated_texts[0].request + == default_multi_requests_seq2seq_lm_batch.requests[0] ) - assert generations[0].generated_text.generated_tokens == 7 + assert generated_texts[0].generated_tokens == 7 diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index a8fc23fe..ccd4c3ba 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -5,7 +5,7 @@ from typing import Optional, Tuple, List, Type from text_generation.models import Model -from text_generation.models.types import Batch, PrefillTokens, Generation, GeneratedText +from text_generation.models.types import GeneratedText, Batch from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -23,6 +23,7 @@ class CausalLMBatch(Batch): # All tokens all_input_ids: List[torch.Tensor] + all_logprobs: List[Optional[torch.Tensor]] # Lengths of all generations present in the batch input_lengths: List[int] @@ -56,6 +57,7 @@ def from_pb( next_token_choosers = [] stopping_criterias = [] input_lengths = [] + all_logprobs = [] # Parse batch for r in pb.requests: @@ -65,6 +67,7 @@ def from_pb( stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) + all_logprobs.append(None) pad_to_multiple_of = 8 if device.type == "cuda" else None tokenized_inputs = tokenizer( @@ -86,6 +89,7 @@ def from_pb( position_ids=position_ids, past_key_values=None, all_input_ids=all_input_ids, + all_logprobs=all_logprobs, input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, @@ -103,6 +107,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests = [] input_lengths = [] all_input_ids = [] + all_logprobs = [] next_token_choosers = [] stopping_criterias = [] @@ -119,6 +124,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) all_input_ids.extend(batch.all_input_ids) + all_logprobs.extend(batch.all_logprobs) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -219,6 +225,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": position_ids=position_ids, past_key_values=past_key_values, all_input_ids=all_input_ids, + all_logprobs=all_logprobs, input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, @@ -227,9 +234,6 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": keys_head_dim_last=batches[0].keys_head_dim_last, ) - def __len__(self): - return len(self.requests) - class CausalLM(Model): def __init__(self, model_name: str, quantize=False): @@ -285,7 +289,7 @@ def forward( def generate_token( self, batch: CausalLMBatch - ) -> Tuple[List[Generation], Optional[CausalLMBatch]]: + ) -> Tuple[List[GeneratedText], Optional[CausalLMBatch]]: # For some reason, inference_mode does not work well with GLOO which we use on CPU context_manager = ( torch.no_grad if self.device.type == "cpu" else torch.inference_mode @@ -305,13 +309,14 @@ def generate_token( next_batch_input_lengths = [] next_batch_input_ids = [] next_batch_all_input_ids = [] + next_batch_all_logprobs = [] # Metadata next_batch_size = 0 next_batch_max_sequence_length = 0 - # Results - generations: List[Generation] = [] + # Finished requests + generated_texts: List[GeneratedText] = [] # Zipped iterator iterator = zip( @@ -321,6 +326,7 @@ def generate_token( batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, + batch.all_logprobs, ) # For each member of the batch @@ -331,36 +337,44 @@ def generate_token( next_token_chooser, stopping_criteria, all_input_ids, + all_logprobs, ) in enumerate(iterator): # Select next token tokens, logprobs = next_token_chooser(all_input_ids, logits) - next_token_id = tokens[-1].view(1, 1) + next_token = tokens[-1].view(1, 1) # Append next token to all tokens - all_input_ids = torch.cat([all_input_ids, next_token_id]) + all_input_ids = torch.cat([all_input_ids, next_token]) new_input_length = input_length + 1 - # Generated token - next_token_logprob = logprobs[-1, next_token_id] - next_token_id_squeezed = next_token_id.squeeze() - next_token_text = self.tokenizer.decode( - next_token_id_squeezed, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) + if all_logprobs is None: + # logprobs of all prompt tokens (except the first one) and the generated token + all_logprobs = logprobs.gather(1, all_input_ids[1:]) + else: + # logprob of the generated token + next_token_logprob = logprobs[-1, next_token] + all_logprobs = torch.cat([all_logprobs, next_token_logprob]) # Evaluate stopping criteria stop, reason = stopping_criteria( - next_token_id_squeezed, - next_token_text, + next_token.squeeze(), + self.tokenizer.decode( + next_token.squeeze(), clean_up_tokenization_spaces=False + ), ) - if stop: # Decode generated tokens generated_text = self.decode( all_input_ids[-stopping_criteria.current_tokens :, 0] ) output_text = request.inputs + generated_text + # Slice with input_length to remove padding + token_ids = all_input_ids[-new_input_length:] + tokens = self.tokenizer.batch_decode(token_ids) + # Add NaN for the first prompt token + logprobs = [float("nan")] + all_logprobs[-input_length:].squeeze( + 1 + ).tolist() # Get seed if isinstance(next_token_chooser.choice, Sampling): @@ -368,58 +382,39 @@ def generate_token( else: seed = None - generated_text = GeneratedText( - output_text, stopping_criteria.current_tokens, reason, seed + # Add to the list of finished generations with the original request + generated_texts.append( + GeneratedText( + request=request, + output_text=output_text, + generated_tokens=stopping_criteria.current_tokens, + tokens=tokens, + token_ids=token_ids.squeeze(1).tolist(), + logprobs=logprobs, + reason=reason, + seed=seed, + ) ) + # add to the next batch else: - # Keep request in the batch - generated_text = None next_batch_keep_indices.append(i) - next_batch_input_ids.append(next_token_id) + next_batch_input_ids.append(next_token) next_batch_all_input_ids.append(all_input_ids) + next_batch_all_logprobs.append(all_logprobs) next_batch_size += 1 next_batch_input_lengths.append(new_input_length) next_batch_max_sequence_length = max( next_batch_max_sequence_length, new_input_length ) - # Prefill - if stopping_criteria.current_tokens == 1: - # Remove generated token to only have prefill and add nan for first prompt token - prefill_logprobs = [float("nan")] + logprobs.gather( - 1, all_input_ids[1:] - ).squeeze(1)[-new_input_length:-1].tolist() - prefill_token_ids = all_input_ids[-new_input_length:-1] - prefill_texts = self.tokenizer.batch_decode( - prefill_token_ids, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) - prefill_tokens = PrefillTokens( - prefill_token_ids, prefill_logprobs, prefill_texts - ) - else: - prefill_tokens = None - - generation = Generation( - request.id, - prefill_tokens, - next_token_id_squeezed, - next_token_logprob, - next_token_text, - generated_text, - ) - - generations.append(generation) - # We finished all generations in the batch; there is no next batch if not next_batch_keep_indices: - return generations, None + return generated_texts, None next_batch_input_ids = torch.cat(next_batch_input_ids, dim=0) # If we finished at least one generation, we need to evict the indices of the generations that finished # from the values of the next batch - if len(next_batch_keep_indices) != len(batch): + if generated_texts: # Apply indices to attention mask, past key values and other items that need to be cached next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] next_batch_position_ids = batch.position_ids[next_batch_keep_indices] @@ -466,6 +461,7 @@ def generate_token( position_ids=next_batch_position_ids, past_key_values=next_batch_past_key_values, all_input_ids=next_batch_all_input_ids, + all_logprobs=next_batch_all_logprobs, input_lengths=next_batch_input_lengths, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, @@ -473,4 +469,4 @@ def generate_token( max_sequence_length=next_batch_max_sequence_length, keys_head_dim_last=batch.keys_head_dim_last, ) - return generations, next_batch + return generated_texts, next_batch diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 41657af2..f965ea88 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -5,7 +5,7 @@ from typing import Optional, Tuple, List, Type from text_generation.models import Model -from text_generation.models.types import GeneratedText, Batch, Generation, PrefillTokens +from text_generation.models.types import GeneratedText, Batch from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -30,6 +30,7 @@ class Seq2SeqLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] decoder_input_lengths: List[int] + decoder_logprobs: List[Optional[torch.Tensor]] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -63,6 +64,7 @@ def from_pb( decoder_input_ids = [] decoder_input_lengths = [] + decoder_logprobs = [] # Parse batch for r in pb.requests: @@ -75,6 +77,7 @@ def from_pb( stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) + decoder_logprobs.append(None) # Tokenize batch pad_to_multiple_of = 8 if device.type == "cuda" else None @@ -99,6 +102,7 @@ def from_pb( past_key_values=None, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, + decoder_logprobs=decoder_logprobs, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=len(pb.requests), @@ -121,6 +125,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests = [] input_lengths = [] decoder_input_lengths = [] + decoder_logprobs = [] next_token_choosers = [] stopping_criterias = [] @@ -141,6 +146,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) decoder_input_lengths.extend(batch.decoder_input_lengths) + decoder_logprobs.extend(batch.decoder_logprobs) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -277,6 +283,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": past_key_values=past_key_values, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, + decoder_logprobs=decoder_logprobs, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=total_batch_size, @@ -284,9 +291,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": max_decoder_input_length=max_decoder_input_length, ) - def __len__(self): - return len(self.requests) - class Seq2SeqLM(Model): def __init__(self, model_name: str, quantize=False): @@ -360,7 +364,7 @@ def forward( def generate_token( self, batch: Seq2SeqLMBatch - ) -> Tuple[List[Generation], Optional[Seq2SeqLMBatch]]: + ) -> Tuple[List[GeneratedText], Optional[Seq2SeqLMBatch]]: # For some reason, inference_mode does not work well with GLOO which we use on CPU context_manager = ( torch.no_grad if self.device.type == "cpu" else torch.inference_mode @@ -382,6 +386,7 @@ def generate_token( next_batch_input_lengths = [] next_batch_decoder_input_ids = [] next_batch_decoder_input_lengths = [] + next_batch_decoder_logprobs = [] # Metadata next_batch_size = 0 @@ -389,13 +394,14 @@ def generate_token( next_batch_max_decoder_input_length = 0 # Finished requests - generations: List[Generation] = [] + generated_texts: List[GeneratedText] = [] # Zipped iterator iterator = zip( batch.requests, batch.input_lengths, batch.decoder_input_lengths, + batch.decoder_logprobs, logits, batch.next_token_choosers, batch.stopping_criterias, @@ -408,6 +414,7 @@ def generate_token( request, input_length, decoder_input_length, + decoder_logprobs, logits, next_token_chooser, stopping_criteria, @@ -415,28 +422,35 @@ def generate_token( decoder_input_ids, ) in enumerate(iterator): # Select next token - next_token_id, logprobs = next_token_chooser(decoder_input_ids, logits) + next_token, logprobs = next_token_chooser(decoder_input_ids, logits) # Append next token to decoder tokens - decoder_input_ids = torch.cat([decoder_input_ids, next_token_id]) + decoder_input_ids = torch.cat([decoder_input_ids, next_token]) new_decoder_input_length = decoder_input_length + 1 - # Generated token - next_token_logprob = logprobs[-1, next_token_id] - next_token_id_squeezed = next_token_id.squeeze() - next_token_text = self.tokenizer.decode( - next_token_id_squeezed, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) + next_token_logprob = logprobs[-1, next_token] + if decoder_logprobs is None: + decoder_logprobs = next_token_logprob + else: + decoder_logprobs = torch.cat([decoder_logprobs, next_token_logprob]) # Evaluate stopping criteria - stop, reason = stopping_criteria(next_token_id, next_token_text) - + stop, reason = stopping_criteria( + next_token.squeeze(), + self.tokenizer.decode( + next_token.squeeze(), clean_up_tokenization_spaces=False + ), + ) if stop: # Slice with decoder_input_length to remove padding # Decode all tokens - output_text = self.decode(decoder_input_ids[-new_decoder_input_length:]) + token_ids = decoder_input_ids[-new_decoder_input_length:] + output_text = self.decode(token_ids) + tokens = self.tokenizer.batch_decode(token_ids) + # Add NaN for the bos token + logprobs = [float("nan")] + decoder_logprobs[ + -decoder_input_length: + ].tolist() # Get seed if isinstance(next_token_chooser.choice, Sampling): @@ -444,17 +458,27 @@ def generate_token( else: seed = None - generated_text = GeneratedText( - output_text, stopping_criteria.current_tokens, reason, seed + # Add to the list of finished generations with the original request + generated_texts.append( + GeneratedText( + request=request, + output_text=output_text, + generated_tokens=stopping_criteria.current_tokens, + tokens=tokens, + token_ids=token_ids.tolist(), + logprobs=logprobs, + reason=reason, + seed=seed, + ) ) + # add to the next batch else: - # Keep request in the batch - generated_text = None next_batch_keep_indices.append(i) next_batch_decoder_input_ids.append(decoder_input_ids.unsqueeze(0)) next_batch_size += 1 next_batch_input_lengths.append(input_length) next_batch_decoder_input_lengths.append(new_decoder_input_length) + next_batch_decoder_logprobs.append(decoder_logprobs) next_batch_max_input_length = max( next_batch_max_input_length, input_length ) @@ -462,39 +486,14 @@ def generate_token( next_batch_max_decoder_input_length, new_decoder_input_length ) - # Prefill - if stopping_criteria.current_tokens == 1: - prefill_token_ids = decoder_input_ids[-new_decoder_input_length:-1] - prefill_texts = self.tokenizer.batch_decode( - prefill_token_ids, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) - prefill_tokens = PrefillTokens( - prefill_token_ids, [float("nan")], prefill_texts - ) - else: - prefill_tokens = None - - generation = Generation( - request.id, - prefill_tokens, - next_token_id_squeezed, - next_token_logprob, - next_token_text, - generated_text, - ) - - generations.append(generation) - # We finished all generations in the batch; there is no next batch if not next_batch_keep_indices: - return generations, None + return generated_texts, None next_batch_decoder_input_ids = torch.cat(next_batch_decoder_input_ids) # If we finished at least one generation, we need to evict the indices of the generations that finished # from the values of the next batch - if len(next_batch_keep_indices) != len(batch): + if generated_texts: # Apply indices to attention mask, past key values and other items that need to be cached next_batch_input_ids = batch.input_ids[next_batch_keep_indices] next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] @@ -552,10 +551,11 @@ def generate_token( past_key_values=next_batch_past_key_values, input_lengths=next_batch_input_lengths, decoder_input_lengths=next_batch_decoder_input_lengths, + decoder_logprobs=next_batch_decoder_logprobs, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, max_input_length=next_batch_max_input_length, max_decoder_input_length=next_batch_max_decoder_input_length, ) - return generations, next_batch + return generated_texts, next_batch diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index 30cd716a..4ee3cb32 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -29,61 +29,26 @@ def from_pb( def concatenate(cls, batches: List["Batch"]) -> "Batch": raise NotImplementedError - @abstractmethod - def __len__(self): - raise NotImplementedError - @dataclass class GeneratedText: - text: str + request: generate_pb2.Request + output_text: str generated_tokens: int - finish_reason: str + tokens: List[str] + token_ids: List[int] + logprobs: List[float] + reason: str seed: Optional[int] def to_pb(self) -> generate_pb2.GeneratedText: return generate_pb2.GeneratedText( - text=self.text, + request=self.request, + output_text=self.output_text, generated_tokens=self.generated_tokens, - finish_reason=self.finish_reason, + tokens=self.tokens, + token_ids=self.token_ids, + logprobs=self.logprobs, + finish_reason=self.reason, seed=self.seed, ) - - -@dataclass -class PrefillTokens: - token_ids: List[int] - logprobs: List[float] - texts: List[str] - - def to_pb(self) -> generate_pb2.PrefillTokens: - return generate_pb2.PrefillTokens( - ids=self.token_ids, logprobs=self.logprobs, texts=self.texts - ) - - def __len__(self): - return len(self.token_ids) - - -@dataclass -class Generation: - request_id: int - prefill_tokens: Optional[PrefillTokens] - token_id: int - token_logprob: float - token_text: str - generated_text: Optional[GeneratedText] - - def to_pb(self) -> generate_pb2.Generation: - return generate_pb2.Generation( - request_id=self.request_id, - prefill_tokens=self.prefill_tokens.to_pb() - if self.prefill_tokens is not None - else None, - token_id=self.token_id, - token_logprob=self.token_logprob, - token_text=self.token_text, - generated_text=self.generated_text.to_pb() - if self.generated_text is not None - else None, - ) diff --git a/server/text_generation/server.py b/server/text_generation/server.py index a2bad8a7..5fd3072e 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -27,20 +27,22 @@ async def ClearCache(self, request, context): self.cache.clear() return generate_pb2.ClearCacheResponse() - async def Prefill(self, request, context): + async def Generate(self, request, context): batch = self.model.batch_type.from_pb( request.batch, self.model.tokenizer, self.model.device ) - generations, next_batch = self.model.generate_token(batch) + generated_texts, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) - return generate_pb2.PrefillResponse( - generations=[generation.to_pb() for generation in generations], + return generate_pb2.GenerateResponse( + generated_texts=[ + generated_text.to_pb() for generated_text in generated_texts + ], batch=next_batch.to_pb() if next_batch else None, ) - async def Decode(self, request, context): + async def GenerateWithCache(self, request, context): if len(request.batches) == 0: raise ValueError("Must provide at least one batch") @@ -56,11 +58,13 @@ async def Decode(self, request, context): else: batch = batches[0] - generations, next_batch = self.model.generate_token(batch) + generated_texts, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) - return generate_pb2.DecodeResponse( - generations=[generation.to_pb() for generation in generations], + return generate_pb2.GenerateWithCacheResponse( + generated_texts=[ + generated_text.to_pb() for generated_text in generated_texts + ], batch=next_batch.to_pb() if next_batch else None, ) From 03bdf18290b5552eab94d64e7cd954b373340b07 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 31 Jan 2023 14:30:33 +0100 Subject: [PATCH 058/793] fix(server): fix seeding on gpu (#42) --- server/text_generation/models/causal_lm.py | 2 +- server/text_generation/models/galactica.py | 2 +- server/text_generation/models/seq2seq_lm.py | 2 +- server/text_generation/utils.py | 19 ++++++++++++++----- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index ccd4c3ba..bf49d134 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -63,7 +63,7 @@ def from_pb( for r in pb.requests: inputs.append(r.inputs) input_lengths.append(r.input_length) - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters)) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index d047ccb6..9bec1dde 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -102,7 +102,7 @@ def from_pb( # Add escape_custom_split_sequence to the CausalLMBatch logic inputs.append(escape_custom_split_sequence(r.inputs)) input_lengths.append(r.input_length) - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters)) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index f965ea88..26ebc7d7 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -73,7 +73,7 @@ def from_pb( # Decoder sequence only contains the bos_token decoder_input_ids.append(tokenizer.bos_token_id) decoder_input_lengths.append(1) - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters)) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 1d087a42..c93e783b 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -24,8 +24,8 @@ class Sampling: - def __init__(self, seed: Optional[int] = None): - self.generator = torch.Generator() + def __init__(self, seed: Optional[int] = None, device: str = "cpu"): + self.generator = torch.Generator(device) if seed is not None: self.generator.manual_seed(seed) else: @@ -50,7 +50,13 @@ def __call__(self, logits): class NextTokenChooser: def __init__( - self, temperature=1.0, top_k=None, top_p=None, do_sample=False, seed=None + self, + temperature=1.0, + top_k=None, + top_p=None, + do_sample=False, + seed=None, + device="cpu", ): warpers = LogitsProcessorList() # the following idea is largely copied from this PR: https://github.com/huggingface/transformers/pull/5420/files @@ -68,7 +74,7 @@ def __init__( sampling = True self.warpers = warpers - self.choice = Sampling(seed) if sampling else Greedy() + self.choice = Sampling(seed, device) if sampling else Greedy() def __call__(self, input_ids, scores): # Warp logits @@ -80,7 +86,9 @@ def __call__(self, input_ids, scores): return next_ids, logprobs @classmethod - def from_pb(cls, pb: generate_pb2.NextTokenChooserParameters) -> "NextTokenChooser": + def from_pb( + cls, pb: generate_pb2.NextTokenChooserParameters, device: torch.device + ) -> "NextTokenChooser": # handle protobuf making default values 0 seed = pb.seed if pb.HasField("seed") else None return NextTokenChooser( @@ -89,6 +97,7 @@ def from_pb(cls, pb: generate_pb2.NextTokenChooserParameters) -> "NextTokenChoos top_p=pb.top_p, do_sample=pb.do_sample, seed=seed, + device=str(device), ) From 54fec9319371b2792526e0cbfebe6cee66ed3980 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 31 Jan 2023 16:01:15 +0100 Subject: [PATCH 059/793] fix(server): fix seeding with multiple shards (#44) --- Cargo.lock | 1 + proto/generate.proto | 2 +- router/Cargo.toml | 1 + router/src/db.rs | 3 +- router/src/validation.rs | 15 ++- server/poetry.lock | 132 +++++++++++---------- server/pyproject.toml | 2 +- server/text_generation/models/bloom.py | 2 - server/text_generation/models/galactica.py | 1 - server/text_generation/utils.py | 18 +-- 10 files changed, 91 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33f5d181..1030e8fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1834,6 +1834,7 @@ dependencies = [ "futures", "nohash-hasher", "parking_lot", + "rand", "serde", "serde_json", "text-generation-client", diff --git a/proto/generate.proto b/proto/generate.proto index 921bd5c0..81039a7c 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -37,7 +37,7 @@ message NextTokenChooserParameters { /// apply sampling on the logits bool do_sample = 4; /// random seed for sampling - optional uint64 seed = 5; + uint64 seed = 5; } message StoppingCriteriaParameters { diff --git a/router/Cargo.toml b/router/Cargo.toml index 546f127f..17724bcc 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -19,6 +19,7 @@ clap = { version = "4.0.15", features = ["derive", "env"] } futures = "0.3.24" nohash-hasher = "0.2.0" parking_lot = "0.12.1" +rand = "0.8.5" serde = "1.0.145" serde_json = "1.0.85" thiserror = "1.0.37" diff --git a/router/src/db.rs b/router/src/db.rs index 15007b64..442d7b9c 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -166,7 +166,8 @@ impl From<&GenerateParameters> for NextTokenChooserParameters { top_k: parameters.top_k as u32, top_p: parameters.top_p, do_sample: parameters.do_sample, - seed: parameters.seed, + // FIXME: remove unwrap + seed: parameters.seed.unwrap(), } } } diff --git a/router/src/validation.rs b/router/src/validation.rs index aabc82a6..674635d3 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -2,6 +2,8 @@ use crate::{ErrorResponse, GenerateRequest}; use axum::http::StatusCode; use axum::Json; +use rand::rngs::ThreadRng; +use rand::Rng; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokio::sync::{mpsc, oneshot}; @@ -92,18 +94,22 @@ fn validation_worker( max_input_length: usize, mut receiver: mpsc::Receiver, ) { + // Seed rng + let mut rng = rand::thread_rng(); + // Loop over requests while let Some((request, response_tx)) = receiver.blocking_recv() { response_tx - .send(validate(request, &tokenizer, max_input_length)) + .send(validate(request, &tokenizer, max_input_length, &mut rng)) .unwrap_or(()) } } fn validate( - request: GenerateRequest, + mut request: GenerateRequest, tokenizer: &Tokenizer, max_input_length: usize, + rng: &mut ThreadRng, ) -> Result<(usize, GenerateRequest), ValidationError> { if request.parameters.temperature <= 0.0 { return Err(ValidationError::Temperature); @@ -124,6 +130,11 @@ fn validate( )); } + // If seed is None, assign a random one + if request.parameters.seed.is_none() { + request.parameters.seed = Some(rng.gen()); + } + // Get the number of tokens in the input match tokenizer.encode(request.inputs.clone(), true) { Ok(inputs) => { diff --git a/server/poetry.lock b/server/poetry.lock index d610fcdf..b236e0a7 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "accelerate" -version = "0.12.0" +version = "0.15.0" description = "Accelerate" category = "main" optional = false @@ -14,13 +14,14 @@ pyyaml = "*" torch = ">=1.4.0" [package.extras] -dev = ["black (>=22.0,<23.0)", "datasets", "deepspeed (<0.7.0)", "evaluate", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scipy", "sklearn", "tqdm", "transformers"] +dev = ["black (>=22.0,<23.0)", "datasets", "deepspeed (<0.7.0)", "evaluate", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "rich", "scikit-learn", "scipy", "tqdm", "transformers"] quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)"] +rich = ["rich"] sagemaker = ["sagemaker"] -test_dev = ["datasets", "deepspeed (<0.7.0)", "evaluate", "scipy", "sklearn", "tqdm", "transformers"] +test_dev = ["datasets", "deepspeed (<0.7.0)", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] test_trackers = ["comet-ml", "tensorboard", "wandb"] -testing = ["datasets", "deepspeed (<0.7.0)", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scipy", "sklearn", "tqdm", "transformers"] +testing = ["datasets", "deepspeed (<0.7.0)", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] [[package]] name = "attrs" @@ -78,7 +79,7 @@ test = ["pytest (>=6)"] [[package]] name = "googleapis-common-protos" -version = "1.57.0" +version = "1.58.0" description = "Common protobufs used in Google APIs" category = "main" optional = false @@ -155,11 +156,11 @@ setuptools = "*" [[package]] name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.7" [[package]] name = "loguru" @@ -234,7 +235,7 @@ wheel = "*" [[package]] name = "packaging" -version = "22.0" +version = "23.0" description = "Core utilities for Python packages" category = "main" optional = false @@ -273,7 +274,7 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "pytest" -version = "7.2.0" +version = "7.2.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false @@ -301,7 +302,7 @@ python-versions = ">=3.6" [[package]] name = "safetensors" -version = "0.2.7" +version = "0.2.8" description = "Fast and Safe Tensor serialization" category = "main" optional = false @@ -319,14 +320,14 @@ torch = ["torch"] [[package]] name = "setuptools" -version = "65.6.3" +version = "67.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] @@ -409,12 +410,12 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "bc59a70e1e112ce2e173289c0c5285121550511236d4866f0053ae18f90c98aa" +content-hash = "c920067f39ade631d1022c0560c3a4336c82d8c28355509bdfff751bbcfc01cd" [metadata.files] accelerate = [ - {file = "accelerate-0.12.0-py3-none-any.whl", hash = "sha256:7742ca5c9f15dd1e0a283305599c196e260af4717a561d1f544aeab27d828af6"}, - {file = "accelerate-0.12.0.tar.gz", hash = "sha256:e8b119c94fac31877620d5f9de311164ec81fa9dc9e175f0d0d4f50fc8d79473"}, + {file = "accelerate-0.15.0-py3-none-any.whl", hash = "sha256:014833307424cd0a22f89815802e00653756257c45dfdba2453e52d428931c65"}, + {file = "accelerate-0.15.0.tar.gz", hash = "sha256:438e25a01afa6e3ffbd25353e76a68be49677c3050f10bfac7beafaf53503efc"}, ] attrs = [ {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, @@ -437,8 +438,8 @@ exceptiongroup = [ {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, ] googleapis-common-protos = [ - {file = "googleapis-common-protos-1.57.0.tar.gz", hash = "sha256:27a849d6205838fb6cc3c1c21cb9800707a661bb21c6ce7fb13e99eb1f8a0c46"}, - {file = "googleapis_common_protos-1.57.0-py2.py3-none-any.whl", hash = "sha256:a9f4a1d7f6d9809657b7f1316a1aa527f6664891531bcfcc13b6696e685f443c"}, + {file = "googleapis-common-protos-1.58.0.tar.gz", hash = "sha256:c727251ec025947d545184ba17e3578840fc3a24a0516a020479edab660457df"}, + {file = "googleapis_common_protos-1.58.0-py2.py3-none-any.whl", hash = "sha256:ca3befcd4580dab6ad49356b46bf165bb68ff4b32389f028f1abd7c10ab9519a"}, ] grpc-interceptor = [ {file = "grpc-interceptor-0.15.0.tar.gz", hash = "sha256:5c1aa9680b1d7e12259960c38057b121826860b05ebbc1001c74343b7ad1455e"}, @@ -547,8 +548,8 @@ grpcio-tools = [ {file = "grpcio_tools-1.51.1-cp39-cp39-win_amd64.whl", hash = "sha256:15b8acf4eaa0ebe37e2f69108de49efd935b7abe9c7e58ba737490b99906aa76"}, ] iniconfig = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] loguru = [ {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, @@ -602,8 +603,8 @@ nvidia-cudnn-cu11 = [ {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, ] packaging = [ - {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, - {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, @@ -642,8 +643,8 @@ psutil = [ {file = "psutil-5.9.4.tar.gz", hash = "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62"}, ] pytest = [ - {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, - {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, + {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, + {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, ] PyYAML = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, @@ -688,49 +689,50 @@ PyYAML = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] safetensors = [ - {file = "safetensors-0.2.7-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:3123fdbb35fdedaedd39cd50a44493783f204821ae8b79012820020bb7e9ea1e"}, - {file = "safetensors-0.2.7-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d8dd3d47609c51b6bd6e473f32d74eeb90a59482f194df6db570ebbb829a948f"}, - {file = "safetensors-0.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:041aa5e13f1bbc0c0d441f692b4103531b9e867ccc281c2c3946fe51deb7eccd"}, - {file = "safetensors-0.2.7-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f2d730ac6716ab4105000aa85e7be7d771b199ec9ab77df50eff6cc0ddce6fcd"}, - {file = "safetensors-0.2.7-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:acef097785a9f03172151c7a8a4c6743d7ec77d1473c38da3aebc8f004d620a8"}, - {file = "safetensors-0.2.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f54630c845e489123a00bfb93787086c96de9abc5e72fbec52c1d2a55f8147fa"}, - {file = "safetensors-0.2.7-cp310-cp310-win32.whl", hash = "sha256:3c4c0d7f3d6922dcae178f19daf52c877674906d60944273e0fdf7a73b7a33e7"}, - {file = "safetensors-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:3c99cccbbe1da7a1bdeb7e4333046577c9785c8b4bb81912b8a134a66570fc0f"}, - {file = "safetensors-0.2.7-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:0a6f3a1db227aeb1152fb8676d94fea3f97d9e017b4b82f7ce5447b88c3a2126"}, - {file = "safetensors-0.2.7-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:8338b875e7e86f50cbffd8c79f28fb0fe2ed56bebe1f87c95a26c501ee50fc51"}, - {file = "safetensors-0.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8403e93665d87ff1006b5678c23566e7a4bf7f2cfdb2666c3955350b2841379"}, - {file = "safetensors-0.2.7-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d993db8e3f63cd71994544c130c1e227a6522c63ddaf2b9f85a33c8e789283b0"}, - {file = "safetensors-0.2.7-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c4901cac6c4bd3d52cb68c86311f86a28d33fa2503bd7c32012c1489cd9a52c"}, - {file = "safetensors-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3b9acd8d864c284d8fe300b290092a2cc7ae76319e7bdd5cbba00a8b4ec2dc0"}, - {file = "safetensors-0.2.7-cp311-cp311-win32.whl", hash = "sha256:dd63fc6bb6f7b78a957c84692afdbaa39168978aac61dfd5ea8c2bbbd459f8a6"}, - {file = "safetensors-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:848af8bca3ee8e4f68bb828e7fbc1d4022d3cde17e8bd098324ef93ace4779e6"}, - {file = "safetensors-0.2.7-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:44ab41d080be381550a0cdc1ded9afc5de899dd733f341c902834571a6f60ca7"}, - {file = "safetensors-0.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:964357808ad70e001d4d6779418e48616eb1f103acf6acdb72bb6a51c05a9da4"}, - {file = "safetensors-0.2.7-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb21eb7b86692cb1d5bc95b0926acd0356df408990de63ae09c3242594170156"}, - {file = "safetensors-0.2.7-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c3c421b55b3baf2ce373836155ffb58d530ec0752837c5fec2f8b841019b49c"}, - {file = "safetensors-0.2.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5f21fe636693f9f40594b357f3d4586cb23e98e044e5e78b21e814543d69c3b"}, - {file = "safetensors-0.2.7-cp37-cp37m-win32.whl", hash = "sha256:1212ec6e113625d9eea662815a25c993a76860ec51f6cc26ac33f4e317abecb5"}, - {file = "safetensors-0.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:6b8076a21241dcad8848b42d90dc7fa1401c89bee622465d019988df62175ae1"}, - {file = "safetensors-0.2.7-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:e27907078ab3924c8135f229c9dff06126fe123b609a22cf2ae96593ca4833bc"}, - {file = "safetensors-0.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89cc90cb042654b224fa98a8acbfc5f795f33bd8c5ff6431ad9884370c9c0caf"}, - {file = "safetensors-0.2.7-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:092c34408391d6fc96617093d59f00ffd23ca624530591d225b0f824cb105367"}, - {file = "safetensors-0.2.7-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:795310f19396f01cfbe00170575daa52bc1019068ef75b13ba487155fba4e9bd"}, - {file = "safetensors-0.2.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3924794458e716dc7d2f3e264c98455de3d59b71e0335cbabef727083d537535"}, - {file = "safetensors-0.2.7-cp38-cp38-win32.whl", hash = "sha256:9205b4a12939db1135fff92b4ea4f62bf131bfc9d54df31881fb6a50eb2ba5fa"}, - {file = "safetensors-0.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:b84662f8d8dd49443a8f8d1d9a89a79974a5f03f4e06526de170ef02f8511236"}, - {file = "safetensors-0.2.7-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:657b01a122da21028a1d32570e0173fa0ef1865d3480cf5bcca80ec92cae421c"}, - {file = "safetensors-0.2.7-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:788a242ea5fab371dcc42cb10ed21c60ea8c5ff7fad41c9fb3d334420c80f156"}, - {file = "safetensors-0.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6bfbc7123d0c08754b6a11848b0298070705f60ab450e3f249624bba1571040"}, - {file = "safetensors-0.2.7-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1982f00d34d8c72e3339c458e5e2fb3eaf9458c55eae6d3bddd61649666db130"}, - {file = "safetensors-0.2.7-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:330570d27ebf5bdb3f63580baa4a4bf92ad11e1df17b96703181cfaa11cae90e"}, - {file = "safetensors-0.2.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f112b20899902dfc5ba27cdea49e57c275c7824a3e24764fca93d3ad436a160"}, - {file = "safetensors-0.2.7-cp39-cp39-win32.whl", hash = "sha256:29fbc1d1c802a5eced1675e93e130367109b2af3bf74af8c415a141b0f5fe568"}, - {file = "safetensors-0.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:054306d050453a92d3cc6cde15e7cb282ef4299f602433bbd70a9b1b6963c9f4"}, - {file = "safetensors-0.2.7.tar.gz", hash = "sha256:37192b456fdfe09762d6a2ef3322d2379fee52eb2db6d887ebac6e3349de596d"}, + {file = "safetensors-0.2.8-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:8df8af89a0b6b535b47c077e33e5cd4941ef4b067e7c1dd1a05647dec0cf2eea"}, + {file = "safetensors-0.2.8-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:9cabae651542b22d229b6439029fb4a3abc7868cb123af336b2877541ad9ab39"}, + {file = "safetensors-0.2.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:384df328523c567a8b2193ebb02456300a43a3adbc316823d4c0d16f7ac9e89d"}, + {file = "safetensors-0.2.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa5f4b31f2e283b83894b388298368860367a1cb773228f9bb65f8db65da1549"}, + {file = "safetensors-0.2.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70f322d170a17b6ecb9c85b15e67f4a9aa3e561594e2dfc7709c0ae0000ebfff"}, + {file = "safetensors-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50f7e90ed02ef5e4263aadce23d39a3595156816c42c4643003791b45f81fd31"}, + {file = "safetensors-0.2.8-cp310-cp310-win32.whl", hash = "sha256:726ad66231286157dd505b5ab85fd903114dcb5f9180e79fd5540d68d6249dd0"}, + {file = "safetensors-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:d14e7c15e5acac6efcd5f5565e38b1b33600101387e5d16059de44adab87405f"}, + {file = "safetensors-0.2.8-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:6273dd3edd5db05b2da0090bc3d491bf25f6d1d7e8a4423477377649e9e38c37"}, + {file = "safetensors-0.2.8-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34705920c8a02f9ea6101bae8403db5f4aa18ec3eaccd8eab6701b1c88ee5bed"}, + {file = "safetensors-0.2.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efff5ce2d3f349d5360a5ca5901ae56c7e24f8da623d57cd08f8e8b1cd9cb1f8"}, + {file = "safetensors-0.2.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dec20b9b1fc90b7b4e588b4f0e9e266bd8f26d405e08b0b6ecad3136d92d007a"}, + {file = "safetensors-0.2.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7716bab34ca9651895126c720df1bf07f464184a7409138179f38db488ca9f15"}, + {file = "safetensors-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb28e5e6257f705828fd39b9ba28248b593f46e722d8d6beedbc8d1f194e2297"}, + {file = "safetensors-0.2.8-cp311-cp311-win32.whl", hash = "sha256:c71e147a8c2942690e8707adbcc4ab60775bc78dfdbd7f9e696dd411adef82bb"}, + {file = "safetensors-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:2f40604c50d4a08a4c74f37fef735cd1e203a59aeda66ea23a01d76fb35cf407"}, + {file = "safetensors-0.2.8-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:f3c7242debe713a87ca6deaadb0d7120e61c92435414142d526e8c32da970020"}, + {file = "safetensors-0.2.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9366140785d567049395c1c03ac69ee8d322fabcdc8fab7d389e933596b383da"}, + {file = "safetensors-0.2.8-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5d720c7640ad5f95f47780bdc35777ed8371afa14d8d63d6375cfe36df83fb4"}, + {file = "safetensors-0.2.8-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0c7ed4d75c2f248791dbe64309c98ada6f40a6949147ca2eaebd278906c918b"}, + {file = "safetensors-0.2.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b905cf3563da4fe8dd46fa777321516f5aa8f6bc1b884158be03a235478e96d"}, + {file = "safetensors-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:2a47327cfe57b6ee8c5dc246f9141f4f6b368e4444dd7f174c025b1d34446730"}, + {file = "safetensors-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:02e67b906ad9bbb87308a34e4d2d33c9eb69baf342b7e5c872728556baf3f0b6"}, + {file = "safetensors-0.2.8-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:ba7c2496765de45a84f5c79b2b12a14a568643d8966ef9bb3f8b16217a39457c"}, + {file = "safetensors-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:90cd22387b1520c4465033b986f79f0d24cc41aabae1903a22eff3b42cee9ad5"}, + {file = "safetensors-0.2.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f279e0fd917a886e1936d534734593f780afdddc6ed62f8ebf2f59de811cdd7c"}, + {file = "safetensors-0.2.8-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af4fce0565574ec3dbe997b021ed32d3dc229547c4f7fca2459be1f725c26f88"}, + {file = "safetensors-0.2.8-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ff0096f5a765e6e3f7f3156f568f59499edeade19e813d610a124ca70b42cdda"}, + {file = "safetensors-0.2.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c02c1cab1d23e6cc8ac1ef865efe76f2d0800e203c877288ebd31747f47a6940"}, + {file = "safetensors-0.2.8-cp38-cp38-win32.whl", hash = "sha256:832f27f6f379fb15d87f3a10eb87e2480434a89c483d7edc8a0c262364ba72c1"}, + {file = "safetensors-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:89da823f9801e92b2c48e8fad1e2f7f0cb696a8a93dab4c6700e8de06fe87392"}, + {file = "safetensors-0.2.8-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:ee8e169040468d176172b4f84a160ece2064abcc77294c4994a9d2bb5255cd75"}, + {file = "safetensors-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ac2197dbbf7cbd269faf8cebab4220dba5aa2ac8beacbce8fdcb9800776781ca"}, + {file = "safetensors-0.2.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f90513eee0a11902df7e51b07c3d9c328828b9dd692d6c74140bed932e7a491"}, + {file = "safetensors-0.2.8-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2bd2e4c6258dd0f4e3d554f2f59789f25ef4757431e83c927016de6339e54811"}, + {file = "safetensors-0.2.8-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:01000a4bfdf0474bb1bdf369de1284c93b4e6047524fe9dc55d77586cb9d0243"}, + {file = "safetensors-0.2.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:466f92a384e4fcb6b1b9811e7488516c4638119c302a959b89bbe1be826d5e25"}, + {file = "safetensors-0.2.8-cp39-cp39-win32.whl", hash = "sha256:2f16e5ee70ae4218474493eff8d998e632a896a6b8aff9273511d654cdf367ab"}, + {file = "safetensors-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:ba3dc236a2344b7feadc9868307f42ba5e4804c9d68a80a35aac831349b31f6f"}, + {file = "safetensors-0.2.8.tar.gz", hash = "sha256:2720b20a6a38c799dca79bd76caeeac2f7df585a9d4f7d59fa7e28eff9ccb27f"}, ] setuptools = [ - {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, - {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, + {file = "setuptools-67.0.0-py3-none-any.whl", hash = "sha256:9d790961ba6219e9ff7d9557622d2fe136816a264dd01d5997cfc057d804853d"}, + {file = "setuptools-67.0.0.tar.gz", hash = "sha256:883131c5b6efa70b9101c7ef30b2b7b780a4283d5fc1616383cdf22c83cbefe6"}, ] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 4a2ccce7..98d91125 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -15,7 +15,7 @@ grpcio-status = "^1.51.1" grpcio-reflection = "^1.51.1" grpc-interceptor = "^0.15.0" typer = "^0.6.1" -accelerate = "^0.12.0" +accelerate = "^0.15.0" bitsandbytes = "^0.35.1" safetensors = "^0.2.4" loguru = "^0.6.0" diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 35f46bc2..1b7635c5 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -33,8 +33,6 @@ except Exception as e: HAS_BITS_AND_BYTES = False -torch.manual_seed(0) - class BloomCausalLMBatch(CausalLMBatch): @classmethod diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index 9bec1dde..5cc55865 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -36,7 +36,6 @@ except Exception as e: HAS_BITS_AND_BYTES = False -torch.manual_seed(0) # CREDIT: Papers with code => https://github.com/paperswithcode/galai/blob/main/galai/utils.py diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index c93e783b..a2029911 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -24,12 +24,10 @@ class Sampling: - def __init__(self, seed: Optional[int] = None, device: str = "cpu"): + def __init__(self, seed: int, device: str = "cpu"): self.generator = torch.Generator(device) - if seed is not None: - self.generator.manual_seed(seed) - else: - self.generator.seed() + self.generator.manual_seed(seed) + self.seed = seed def __call__(self, logits): probs = torch.nn.functional.softmax(logits, dim=-1) @@ -38,10 +36,6 @@ def __call__(self, logits): ).squeeze(1) return next_tokens - @property - def seed(self) -> int: - return self.generator.initial_seed() - class Greedy: def __call__(self, logits): @@ -55,7 +49,7 @@ def __init__( top_k=None, top_p=None, do_sample=False, - seed=None, + seed=0, device="cpu", ): warpers = LogitsProcessorList() @@ -89,14 +83,12 @@ def __call__(self, input_ids, scores): def from_pb( cls, pb: generate_pb2.NextTokenChooserParameters, device: torch.device ) -> "NextTokenChooser": - # handle protobuf making default values 0 - seed = pb.seed if pb.HasField("seed") else None return NextTokenChooser( temperature=pb.temperature, top_k=pb.top_k, top_p=pb.top_p, do_sample=pb.do_sample, - seed=seed, + seed=pb.seed, device=str(device), ) From 017a2a8c2f72c4c30b01eab0da53c00c9c1f7057 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 31 Jan 2023 17:04:00 +0100 Subject: [PATCH 060/793] feat: Add token streaming using ServerSideEvents support (#41) --- Cargo.lock | 2 + launcher/Cargo.toml | 2 +- launcher/tests/bloom_560m.json | 6 +- launcher/tests/mt0_base.json | 6 +- proto/generate.proto | 66 ++-- router/Cargo.toml | 2 + router/client/src/client.rs | 28 +- router/client/src/lib.rs | 3 +- router/client/src/sharded_client.rs | 38 +-- router/src/batcher.rs | 236 ------------- router/src/db.rs | 50 +-- router/src/infer.rs | 353 ++++++++++++++++++++ router/src/lib.rs | 21 +- router/src/server.rs | 307 +++++++++++------ router/src/validation.rs | 73 ++-- server/tests/models/test_bloom.py | 105 +++--- server/tests/models/test_causal_lm.py | 100 +++--- server/tests/models/test_santacoder.py | 30 +- server/tests/models/test_seq2seq_lm.py | 93 +++--- server/text_generation/models/causal_lm.py | 110 +++--- server/text_generation/models/seq2seq_lm.py | 102 +++--- server/text_generation/models/types.py | 59 +++- server/text_generation/server.py | 20 +- 23 files changed, 1069 insertions(+), 743 deletions(-) delete mode 100644 router/src/batcher.rs create mode 100644 router/src/infer.rs diff --git a/Cargo.lock b/Cargo.lock index 1030e8fd..5b671fa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1829,6 +1829,7 @@ dependencies = [ name = "text-generation-router" version = "0.1.0" dependencies = [ + "async-stream", "axum", "clap 4.0.22", "futures", @@ -1841,6 +1842,7 @@ dependencies = [ "thiserror", "tokenizers", "tokio", + "tokio-stream", "tracing", "tracing-subscriber", ] diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 21d5d3ee..58df28d9 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -16,4 +16,4 @@ tracing-subscriber = { version = "0.3.16", features = ["json"] } [dev-dependencies] float_eq = "1.0.1" reqwest = { version = "0.11.13", features = ["blocking", "json"] } -serde = "1.0.150" +serde = { version = "1.0.150", features = ["derive"] } diff --git a/launcher/tests/bloom_560m.json b/launcher/tests/bloom_560m.json index d17f1ed4..a81d1982 100644 --- a/launcher/tests/bloom_560m.json +++ b/launcher/tests/bloom_560m.json @@ -3,7 +3,7 @@ "details": { "finish_reason": "length", "generated_tokens": 20, - "tokens": [ + "prefill": [ [ 10264, "Test", @@ -13,7 +13,9 @@ 8821, " request", -11.895094 - ], + ] + ], + "tokens": [ [ 17, ".", diff --git a/launcher/tests/mt0_base.json b/launcher/tests/mt0_base.json index 1b772282..51cb8b5c 100644 --- a/launcher/tests/mt0_base.json +++ b/launcher/tests/mt0_base.json @@ -3,12 +3,14 @@ "details": { "finish_reason": "length", "generated_tokens": 20, - "tokens": [ + "prefill": [ [ 0, "", null - ], + ] + ], + "tokens": [ [ 259, "", diff --git a/proto/generate.proto b/proto/generate.proto index 81039a7c..8f431c5c 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -7,10 +7,10 @@ service TextGenerationService { rpc ServiceDiscovery (ServiceDiscoveryRequest) returns (ServiceDiscoveryResponse) {} /// Empties batch cache rpc ClearCache (ClearCacheRequest) returns (ClearCacheResponse); - /// Generate tokens for a batch - rpc Generate (GenerateRequest) returns (GenerateResponse); - /// Generate tokens for a list of cached batches - rpc GenerateWithCache (GenerateWithCacheRequest) returns (GenerateWithCacheResponse); + /// Prefill batch and decode first token + rpc Prefill (PrefillRequest) returns (PrefillResponse); + /// Decode token for a list of prefilled batches + rpc Decode (DecodeRequest) returns (DecodeResponse); } /// Empty request @@ -70,44 +70,60 @@ message Batch { } message GeneratedText { - /// Request - Request request = 1; /// Output - string output_text = 2; + string text = 1; /// Number of generated tokens - uint32 generated_tokens = 3; - /// Tokens - repeated string tokens = 4; - /// Token IDs - repeated uint32 token_ids = 5; - /// Logprobs - repeated float logprobs = 6; + uint32 generated_tokens = 2; /// Finish reason - string finish_reason = 7; + string finish_reason = 3; /// Seed - optional uint64 seed = 8; + optional uint64 seed = 4; } -message GenerateRequest { +message PrefillTokens { + /// Prefill Token IDs + repeated uint32 ids = 1; + /// Prefill Logprobs + repeated float logprobs = 2; + /// Prefill tokens + repeated string texts = 3; +} + +message Generation { + /// Request ID + uint64 request_id = 1; + /// Prefill tokens (optional) + PrefillTokens prefill_tokens = 2; + /// Token ID + uint32 token_id = 3; + /// Logprob + float token_logprob = 4; + /// Text + string token_text = 5; + /// Complete generated text + GeneratedText generated_text = 6; +} + +message PrefillRequest { /// Batch Batch batch = 1; } -message GenerateResponse { - /// Finished requests - repeated GeneratedText generated_texts = 1; +message PrefillResponse { + /// Generation + repeated Generation generations = 1; /// Next batch (cached) optional Batch batch = 2; } -message GenerateWithCacheRequest { +message DecodeRequest { /// Cached batches repeated Batch batches = 1; } -message GenerateWithCacheResponse { - /// Finished requests - repeated GeneratedText generated_texts = 1; +message DecodeResponse { + /// Decodes + repeated Generation generations = 1; /// Next batch (cached) optional Batch batch = 2; -} +} \ No newline at end of file diff --git a/router/Cargo.toml b/router/Cargo.toml index 17724bcc..3abbc80b 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -13,6 +13,7 @@ name = "text-generation-router" path = "src/main.rs" [dependencies] +async-stream = "0.3.3" axum = { version = "0.5.16", features = ["json", "serde_json"] } text-generation-client = { path = "client" } clap = { version = "4.0.15", features = ["derive", "env"] } @@ -25,6 +26,7 @@ serde_json = "1.0.85" thiserror = "1.0.37" tokenizers = "0.13.0" tokio = { version = "1.21.1", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } +tokio-stream = "0.1.11" tracing = "0.1.36" tracing-subscriber = { version = "0.3.15", features = ["json"] } diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 172d0bf7..77a43110 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -70,36 +70,36 @@ impl Client { /// Generate one token for each request in the given batch /// - /// Returns a list of generated texts of request that met their stopping criteria + /// Returns Generation for each request in batch /// and the next cached batch #[instrument(skip(self))] - pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { - let request = tonic::Request::new(GenerateRequest { batch: Some(batch) }); + pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { + let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }); let response = self .stub - .generate(request) - .instrument(info_span!("generate")) + .prefill(request) + .instrument(info_span!("prefill")) .await? .into_inner(); - Ok((response.generated_texts, response.batch)) + Ok((response.generations, response.batch)) } - /// Generate one token for each request in the given cached batch + /// Generate one token for each request in the given cached batches /// - /// Returns a list of generated texts of request that met their stopping criteria + /// Returns Generation for each request in batches /// and the next cached batch #[instrument(skip(self))] - pub async fn generate_with_cache( + pub async fn decode( &mut self, batches: Vec, - ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(GenerateWithCacheRequest { batches }); + ) -> Result<(Vec, Option)> { + let request = tonic::Request::new(DecodeRequest { batches }); let response = self .stub - .generate_with_cache(request) - .instrument(info_span!("generate_with_cache")) + .decode(request) + .instrument(info_span!("decode")) .await? .into_inner(); - Ok((response.generated_texts, response.batch)) + Ok((response.generations, response.batch)) } } diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 295b009b..e0546b16 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -7,7 +7,8 @@ mod sharded_client; pub use client::Client; pub use pb::generate::v1::{ - Batch, GeneratedText, NextTokenChooserParameters, Request, StoppingCriteriaParameters, + Batch, GeneratedText, Generation, NextTokenChooserParameters, PrefillTokens, Request, + StoppingCriteriaParameters, }; pub use sharded_client::ShardedClient; use thiserror::Error; diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 6c70afca..56335f92 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,6 +1,6 @@ /// Multi shard Client use crate::Result; -use crate::{Batch, Client, GeneratedText}; +use crate::{Batch, Client, Generation}; use futures::future::join_all; use futures::future::select_all; use tonic::transport::Uri; @@ -37,46 +37,46 @@ impl ShardedClient { Self::from_master_client(master_client).await } + /// Clear the past generations cache + pub async fn clear_cache(&mut self) -> Result<()> { + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| client.clear_cache()) + .collect(); + join_all(futures).await.into_iter().collect() + } + /// Generate one token for each request in the given batch /// - /// Returns a list of generated texts of request that met their stopping criteria + /// Returns Generation for each request in batch /// and the next cached batch - pub async fn generate(&mut self, batch: Batch) -> Result<(Vec, Option)> { + pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| Box::pin(client.generate(batch.clone()))) + .map(|client| Box::pin(client.prefill(batch.clone()))) .collect(); // As soon as we receive one response, we can return as all shards will return the same let (result, _, _) = select_all(futures).await; result } - /// Generate one token for each request in the given cached batch + /// Generate one token for each request in the given cached batches /// - /// Returns a list of generated texts of request that met their stopping criteria + /// Returns Generation for each request in batches /// and the next cached batch - pub async fn generate_with_cache( + pub async fn decode( &mut self, batches: Vec, - ) -> Result<(Vec, Option)> { + ) -> Result<(Vec, Option)> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| Box::pin(client.generate_with_cache(batches.clone()))) + .map(|client| Box::pin(client.decode(batches.clone()))) .collect(); // As soon as we receive one response, we can return as all shards will return the same let (result, _, _) = select_all(futures).await; result } - - /// Clear the past generations cache - pub async fn clear_cache(&mut self) -> Result<()> { - let futures: Vec<_> = self - .clients - .iter_mut() - .map(|client| client.clear_cache()) - .collect(); - join_all(futures).await.into_iter().collect() - } } diff --git a/router/src/batcher.rs b/router/src/batcher.rs deleted file mode 100644 index baf58af4..00000000 --- a/router/src/batcher.rs +++ /dev/null @@ -1,236 +0,0 @@ -/// Batching and inference logic -use crate::{Db, Entry}; -use crate::{ErrorResponse, GenerateRequest}; -use axum::http::StatusCode; -use axum::Json; -use nohash_hasher::IntMap; -use std::future::Future; -use std::sync::Arc; -use text_generation_client::{Batch, ClientError, GeneratedText, ShardedClient}; -use thiserror::Error; -use tokio::sync::{oneshot, Notify}; -use tokio::time::Instant; -use tracing::instrument; - -/// Batcher -#[derive(Clone)] -pub struct Batcher { - /// Request database - db: Db, - /// Shared state - shared: Arc, -} - -/// Batcher shared state -struct Shared { - /// Batching background Tokio task notifier - batching_task: Notify, -} - -impl Batcher { - pub(crate) fn new( - client: ShardedClient, - max_batch_size: usize, - max_waiting_tokens: usize, - ) -> Self { - // Batcher shared state - let db = Db::new(); - let shared = Arc::new(Shared { - batching_task: Notify::new(), - }); - - // Spawn batching background task that contains all the inference logic - tokio::spawn(batching_task( - client, - max_batch_size, - max_waiting_tokens, - db.clone(), - shared.clone(), - )); - - Self { db, shared } - } - - /// Add a new request to the database and return a future that will generate the text - pub(crate) async fn infer( - &self, - input_length: usize, - request: GenerateRequest, - ) -> Result { - // One shot channel to communicate with the background batching task - let (response_tx, response_rx) = oneshot::channel(); - - // Try to append the request to the database - self.db.append(Entry { - request, - response_tx, - input_length, - time: Instant::now(), - batch_time: None, - }); - - // Notify the background task that we have a new entry in the database that needs - // to be batched - self.shared.batching_task.notify_one(); - - // Await on the response from the background task - // We can safely unwrap as the background task will never drop the sender - response_rx - .await - .unwrap() - .map_err(|err| InferError::GenerationError(err.to_string())) - } -} - -/// Batching logic -/// Will be launched in a background Tokio task -/// -/// Batches requests and sends them to the inference server -#[instrument(skip(client, db, shared))] -async fn batching_task( - mut client: ShardedClient, - max_batch_size: usize, - max_waiting_tokens: usize, - db: Db, - shared: Arc, -) { - // Minimum batch size after which we try to add more requests - let limit_min_batch_size = (max_batch_size / 2) as u32; - - // Infinite loop - loop { - // Wait for a notification from the Batcher struct - shared.batching_task.notified().await; - - // Get the next batch from the DB - // This batch might be smaller than the maximum batch size if there are not enough requests - // waiting in the DB - while let Some((mut entries, batch)) = db.next_batch(None, max_batch_size) { - let mut cached_batch = wrap_future(client.generate(batch), &mut entries).await; - let mut waiting_tokens = 1; - - // We loop until we do not receive any cached batch from the inference server (== until - // all requests have met their stopping criteria) - while let Some(batch) = cached_batch { - // Get current batch info - let batch_size = batch.size; - let mut batches = vec![batch]; - - // If the current batch is too small, we try to add more requests to it - if batch_size <= limit_min_batch_size { - let min_size = match waiting_tokens { - // If we didn't onboard any new requests since >= max_waiting_tokens, we try - // to add a new batch even though its size might be small - _ if waiting_tokens >= max_waiting_tokens => None, - // Minimum size criteria - _ => Some(limit_min_batch_size as usize), - }; - - // Try to get a new batch - if let Some((mut new_entries, new_batch)) = - db.next_batch(min_size, max_batch_size - batch_size as usize) - { - // Generate one token for this new batch to have the attention past in cache - let new_cached_batch = - wrap_future(client.generate(new_batch), &mut new_entries).await; - // Reset waiting counter - waiting_tokens = 1; - // Extend current batch with the new batch - if let Some(new_cached_batch) = new_cached_batch { - entries.extend(new_entries); - batches.push(new_cached_batch); - } - } - } - - cached_batch = wrap_future(client.generate_with_cache(batches), &mut entries).await; - waiting_tokens += 1; - } - } - } -} - -/// Wrap a future inside a match statement to handle errors and send the response to the Batcher -async fn wrap_future( - future: impl Future, Option), ClientError>>, - entries: &mut IntMap, -) -> Option { - match future.await { - Ok((generated_texts, next_batch)) => { - send_generated(generated_texts, entries); - next_batch - } - // If we have an error, we discard the whole batch - Err(err) => { - send_error(err, entries); - None - } - } -} - -/// Send errors to the Batcher for all `entries` -fn send_error(error: ClientError, entries: &mut IntMap) { - entries.drain().for_each(|(_, entry)| { - // unwrap_or is valid here as we don't care if the receiver is gone. - entry.response_tx.send(Err(error.clone())).unwrap_or(()); - }); -} - -/// Send `generated_text` to the Batcher for all `finished` -fn send_generated(finished: Vec, entries: &mut IntMap) { - finished.into_iter().for_each(|output| { - // We can `expect` here as the request id should always be in the entries - let entry = entries - .remove(&output.request.unwrap().id) - .expect("ID not found in entries. This is a bug."); - - let response = InferResponse { - output_text: output.output_text, - generated_tokens: output.generated_tokens, - token_ids: output.token_ids, - tokens: output.tokens, - logprobs: output.logprobs, - finish_reason: output.finish_reason, - seed: output.seed, - queued: entry.time, - start: entry.batch_time.unwrap(), // unwrap is always valid - end: Instant::now(), - }; - // unwrap_or is valid here as we don't care if the receiver is gone. - entry.response_tx.send(Ok(response)).unwrap_or(()); - }); -} - -#[derive(Debug)] -pub(crate) struct InferResponse { - pub(crate) output_text: String, - pub(crate) generated_tokens: u32, - pub(crate) token_ids: Vec, - pub(crate) tokens: Vec, - pub(crate) logprobs: Vec, - pub(crate) finish_reason: String, - pub(crate) seed: Option, - pub(crate) queued: Instant, - pub(crate) start: Instant, - pub(crate) end: Instant, -} - -#[derive(Debug, Error)] -pub enum InferError { - #[error("Request failed during generation: {0}")] - GenerationError(String), -} - -/// Convert to Axum supported format -impl From for (StatusCode, Json) { - fn from(err: InferError) -> Self { - match err { - InferError::GenerationError(_) => ( - StatusCode::FAILED_DEPENDENCY, - Json(ErrorResponse { - error: err.to_string(), - }), - ), - } - } -} diff --git a/router/src/db.rs b/router/src/db.rs index 442d7b9c..246e4d5d 100644 --- a/router/src/db.rs +++ b/router/src/db.rs @@ -1,29 +1,29 @@ /// This code is massively inspired by Tokio mini-redis -use crate::InferResponse; -use crate::{GenerateParameters, GenerateRequest}; +use crate::infer::InferError; +use crate::infer::InferStreamResponse; +use crate::validation::ValidGenerateRequest; use nohash_hasher::{BuildNoHashHasher, IntMap}; use parking_lot::Mutex; use std::collections::BTreeMap; use std::sync::Arc; -use text_generation_client::{ - Batch, ClientError, NextTokenChooserParameters, Request, StoppingCriteriaParameters, -}; -use tokio::sync::oneshot::Sender; +use text_generation_client::{Batch, Request}; +use tokio::sync::mpsc::UnboundedSender; +use tokio::sync::OwnedSemaphorePermit; use tokio::time::Instant; /// Database entry #[derive(Debug)] pub(crate) struct Entry { /// Request - pub request: GenerateRequest, - /// Response sender to communicate between the Batcher and the batching_task - pub response_tx: Sender>, - /// Number of tokens in the input - pub input_length: usize, + pub request: ValidGenerateRequest, + /// Response sender to communicate between the Infer struct and the batching_task + pub response_tx: UnboundedSender>, /// Instant when this entry was created pub time: Instant, /// Instant when this entry was added to a batch pub batch_time: Option, + /// Permit + pub _permit: OwnedSemaphorePermit, } /// Request Database @@ -71,9 +71,9 @@ impl State { requests.push(Request { id: *id, inputs: entry.request.inputs.clone(), - input_length: entry.input_length as u32, - parameters: Some((&entry.request.parameters).into()), - stopping_parameters: Some(entry.request.parameters.clone().into()), + input_length: entry.request.input_length, + parameters: Some(entry.request.parameters.clone()), + stopping_parameters: Some(entry.request.stopping_parameters.clone()), }); ids.push(*id); @@ -158,25 +158,3 @@ impl Db { None } } - -impl From<&GenerateParameters> for NextTokenChooserParameters { - fn from(parameters: &GenerateParameters) -> Self { - Self { - temperature: parameters.temperature, - top_k: parameters.top_k as u32, - top_p: parameters.top_p, - do_sample: parameters.do_sample, - // FIXME: remove unwrap - seed: parameters.seed.unwrap(), - } - } -} - -impl From for StoppingCriteriaParameters { - fn from(parameters: GenerateParameters) -> Self { - Self { - stop_sequences: parameters.stop, - max_new_tokens: parameters.max_new_tokens, - } - } -} diff --git a/router/src/infer.rs b/router/src/infer.rs new file mode 100644 index 00000000..23e84265 --- /dev/null +++ b/router/src/infer.rs @@ -0,0 +1,353 @@ +/// Batching and inference logic +use crate::validation::{Validation, ValidationError}; +use crate::GenerateRequest; +use crate::{Db, Entry, Token}; +use nohash_hasher::IntMap; +use std::future::Future; +use std::sync::Arc; +use text_generation_client::{ + Batch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, +}; +use thiserror::Error; +use tokio::sync::{mpsc, Notify, Semaphore, TryAcquireError}; +use tokio::time::Instant; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tokio_stream::StreamExt; +use tracing::instrument; + +/// Inference struct +#[derive(Clone)] +pub struct Infer { + /// Validation + validation: Validation, + /// Request database + db: Db, + /// Shared state + shared: Arc, + /// Inference limit + limit_concurrent_requests: Arc, +} + +/// Infer shared state +struct Shared { + /// Batching background Tokio task notifier + batching_task: Notify, +} + +impl Infer { + pub(crate) fn new( + client: ShardedClient, + validation: Validation, + max_batch_size: usize, + max_waiting_tokens: usize, + max_concurrent_requests: usize, + ) -> Self { + // Infer shared state + let db = Db::new(); + let shared = Arc::new(Shared { + batching_task: Notify::new(), + }); + + // Spawn batching background task that contains all the inference logic + tokio::spawn(batching_task( + client, + max_batch_size, + max_waiting_tokens, + db.clone(), + shared.clone(), + )); + + // Inference limit with a semaphore + let semaphore = Arc::new(Semaphore::new(max_concurrent_requests)); + + Self { + validation, + db, + shared, + limit_concurrent_requests: semaphore, + } + } + + /// Add a new request to the database and return a stream of InferStreamResponse + pub(crate) async fn generate_stream( + &self, + request: GenerateRequest, + ) -> Result>, InferError> { + // Limit concurrent requests by acquiring a permit from the semaphore + // This permit will live as long as Entry + let permit = self.clone().limit_concurrent_requests.try_acquire_owned()?; + + // Validate request + let valid_request = self.validation.validate(request).await?; + + // MPSC channel to communicate with the background batching task + let (response_tx, response_rx) = mpsc::unbounded_channel(); + + // Append the request to the database + self.db.append(Entry { + request: valid_request, + response_tx, + time: Instant::now(), + batch_time: None, + _permit: permit, + }); + + // Notify the background task that we have a new entry in the database that needs + // to be batched + self.shared.batching_task.notify_one(); + + // Return stream + Ok(UnboundedReceiverStream::new(response_rx)) + } + + /// Add a new request to the database and return a InferResponse + pub(crate) async fn generate( + &self, + request: GenerateRequest, + ) -> Result { + // Create stream + let mut stream = self.generate_stream(request).await?; + + // Return values + let mut result_prefill = Vec::new(); + let mut result_tokens = Vec::new(); + let mut result_generated_text = None; + let mut result_start = None; + let mut result_queued = None; + + // Iterate on stream + while let Some(response) = stream.next().await { + match response? { + // Add prefill tokens + InferStreamResponse::Prefill(tokens) => { + // Create Token objects + // We do that here instead of in the Python code as Rust for loops are faster + result_prefill = tokens + .ids + .into_iter() + .zip(tokens.logprobs.into_iter()) + .zip(tokens.texts.into_iter()) + .map(|((id, logprob), text)| Token(id, text, logprob)) + .collect(); + } + // Push last token + InferStreamResponse::Token(token) => result_tokens.push(token), + // Final message + // Set return values + InferStreamResponse::End { + token, + generated_text, + start, + queued, + } => { + result_tokens.push(token); + result_generated_text = Some(generated_text); + result_start = Some(start); + result_queued = Some(queued) + } + } + } + + // Check that we received a `InferStreamResponse::End` message + if let (Some(generated_text), Some(queued), Some(start)) = + (result_generated_text, result_queued, result_start) + { + Ok(InferResponse { + prefill: result_prefill, + tokens: result_tokens, + generated_text, + queued, + start, + }) + } else { + Err(InferError::IncompleteGeneration) + } + } +} + +/// Batching logic +/// Will be launched in a background Tokio task +/// +/// Batches requests and sends them to the inference server +#[instrument(skip(client, db, shared))] +async fn batching_task( + mut client: ShardedClient, + max_batch_size: usize, + max_waiting_tokens: usize, + db: Db, + shared: Arc, +) { + // Minimum batch size after which we try to add more requests + let limit_min_batch_size = (max_batch_size / 2) as u32; + + // Infinite loop + loop { + // Wait for a notification from the Infer struct + shared.batching_task.notified().await; + + // Get the next batch from the DB + // This batch might be smaller than the maximum batch size if there are not enough requests + // waiting in the DB + while let Some((mut entries, batch)) = db.next_batch(None, max_batch_size) { + let mut cached_batch = wrap_future(client.prefill(batch), &mut entries).await; + let mut waiting_tokens = 1; + + // We loop until we do not receive any cached batch from the inference server (== until + // all requests have met their stopping criteria) + while let Some(batch) = cached_batch { + // Get current batch info + let batch_size = batch.size; + let mut batches = vec![batch]; + + // If the current batch is too small, we try to add more requests to it + if batch_size <= limit_min_batch_size { + let min_size = match waiting_tokens { + // If we didn't onboard any new requests since >= max_waiting_tokens, we try + // to add a new batch even though its size might be small + _ if waiting_tokens >= max_waiting_tokens => None, + // Minimum size criteria + _ => Some(limit_min_batch_size as usize), + }; + + // Try to get a new batch + if let Some((mut new_entries, new_batch)) = + db.next_batch(min_size, max_batch_size - batch_size as usize) + { + // Generate one token for this new batch to have the attention past in cache + let new_cached_batch = + wrap_future(client.prefill(new_batch), &mut new_entries).await; + // Reset waiting counter + waiting_tokens = 1; + // Extend current batch with the new batch + if let Some(new_cached_batch) = new_cached_batch { + entries.extend(new_entries); + batches.push(new_cached_batch); + } + } + } + + cached_batch = wrap_future(client.decode(batches), &mut entries).await; + waiting_tokens += 1; + } + } + } +} + +/// Wrap a future inside a match statement to handle errors and send the responses to Infer +async fn wrap_future( + future: impl Future, Option), ClientError>>, + entries: &mut IntMap, +) -> Option { + match future.await { + Ok((generations, next_batch)) => { + send_generations(generations, entries); + next_batch + } + // If we have an error, we discard the whole batch + Err(err) => { + send_error(err, entries); + None + } + } +} + +/// Send errors to Infer for all `entries` +fn send_error(error: ClientError, entries: &mut IntMap) { + entries.drain().for_each(|(_, entry)| { + // unwrap_or is valid here as we don't care if the receiver is gone. + entry + .response_tx + .send(Err(InferError::GenerationError(error.to_string()))) + .unwrap_or(()); + }); +} + +/// Send one or multiple `InferStreamResponse` to Infer for all `entries` +fn send_generations(generations: Vec, entries: &mut IntMap) { + generations.into_iter().for_each(|generation| { + // Get entry + // We can `expect` here as the request id should always be in the entries + let entry = entries + .get(&generation.request_id) + .expect("ID not found in entries. This is a bug."); + + if let Some(prefill_tokens) = generation.prefill_tokens { + // Send message + // unwrap_or is valid here as we don't care if the receiver is gone. + entry + .response_tx + .send(Ok(InferStreamResponse::Prefill(prefill_tokens))) + .unwrap_or(()); + } + + // Create last Token + let token = Token( + generation.token_id, + generation.token_text, + generation.token_logprob, + ); + + if let Some(generated_text) = generation.generated_text { + // Remove entry as this is the last message + // We can `expect` here as the request id should always be in the entries + let entry = entries + .remove(&generation.request_id) + .expect("ID not found in entries. This is a bug."); + + // Send message + // unwrap_or is valid here as we don't care if the receiver is gone. + entry + .response_tx + .send(Ok(InferStreamResponse::End { + token, + generated_text, + queued: entry.time, + start: entry.batch_time.unwrap(), + })) + .unwrap_or(()); + } else { + // Send message + // unwrap_or is valid here as we don't care if the receiver is gone. + entry + .response_tx + .send(Ok(InferStreamResponse::Token(token))) + .unwrap_or(()); + } + }); +} + +#[derive(Debug)] +pub(crate) enum InferStreamResponse { + // Optional first message + Prefill(PrefillTokens), + // Intermediate messages + Token(Token), + // Last message + End { + token: Token, + generated_text: GeneratedText, + start: Instant, + queued: Instant, + }, +} + +#[derive(Debug)] +pub(crate) struct InferResponse { + pub(crate) prefill: Vec, + pub(crate) tokens: Vec, + pub(crate) generated_text: GeneratedText, + pub(crate) queued: Instant, + pub(crate) start: Instant, +} + +#[derive(Debug, Error)] +pub enum InferError { + #[error("Request failed during generation: {0}")] + GenerationError(String), + #[error("Model is overloaded")] + Overloaded(#[from] TryAcquireError), + #[error("Input validation error: {0}")] + ValidationError(#[from] ValidationError), + #[error("Incomplete generation")] + IncompleteGeneration, +} diff --git a/router/src/lib.rs b/router/src/lib.rs index 1aeac302..beab7138 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,11 +1,11 @@ /// Text Generation Inference Webserver -mod batcher; mod db; +mod infer; pub mod server; mod validation; -use batcher::{Batcher, InferResponse}; use db::{Db, Entry}; +use infer::Infer; use serde::{Deserialize, Serialize}; use validation::Validation; @@ -69,21 +69,34 @@ pub(crate) struct GenerateRequest { pub parameters: GenerateParameters, } +#[derive(Debug, Serialize)] +pub struct Token(u32, String, f32); + #[derive(Serialize)] pub(crate) struct Details { pub finish_reason: String, pub generated_tokens: u32, pub seed: Option, - pub tokens: Vec<(u32, String, f32)>, + #[serde(skip_serializing_if = "Option::is_none")] + pub prefill: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub tokens: Option>, } #[derive(Serialize)] -pub(crate) struct GeneratedText { +pub(crate) struct GenerateResponse { pub generated_text: String, #[serde(skip_serializing_if = "Option::is_none")] pub details: Option
, } +#[derive(Serialize)] +pub(crate) struct StreamResponse { + pub token: Token, + pub generated_text: Option, + pub details: Option
, +} + #[derive(Serialize)] pub(crate) struct ErrorResponse { pub error: String, diff --git a/router/src/server.rs b/router/src/server.rs index 86041b96..ef3782d6 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,71 +1,54 @@ +/// HTTP Server logic +use crate::infer::{InferError, InferStreamResponse}; use crate::{ - Batcher, Details, ErrorResponse, GenerateParameters, GenerateRequest, GeneratedText, Validation, + Details, ErrorResponse, GenerateParameters, GenerateRequest, GenerateResponse, Infer, + StreamResponse, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, StatusCode}; +use axum::response::sse::{Event, KeepAlive, Sse}; use axum::response::IntoResponse; use axum::routing::{get, post}; use axum::{Json, Router}; +use futures::Stream; +use std::convert::Infallible; use std::net::SocketAddr; -use std::sync::Arc; use text_generation_client::ShardedClient; use tokenizers::Tokenizer; use tokio::signal; -use tokio::sync::Semaphore; use tokio::time::Instant; +use tokio_stream::StreamExt; use tracing::instrument; -// Server shared state -#[derive(Clone)] -struct ServerState { - validation: Validation, - batcher: Batcher, - limit_concurrent_requests: Arc, -} - /// Health check method -#[instrument(skip(state), fields(time, time_per_token))] -async fn health(state: Extension) -> Result<(), (StatusCode, Json)> { +#[instrument(skip(infer))] +async fn health(infer: Extension) -> Result<(), (StatusCode, Json)> { // TODO: while this is the best health check we can do, it is a bit on the heavy side and might // be a bit too slow for a health check. // What we should do instead if check if the gRPC channels are still healthy. - // Limit concurrent requests by acquiring a permit from the semaphore - let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { - ( - StatusCode::TOO_MANY_REQUESTS, - Json(ErrorResponse { - error: "Model is overloaded".to_string(), - }), - ) - })?; - // Send a small inference request - state - .batcher - .infer( - 1, - GenerateRequest { - inputs: "liveness".to_string(), - parameters: GenerateParameters { - temperature: 1.0, - top_k: 0, - top_p: 1.0, - do_sample: false, - max_new_tokens: 1, - stop: vec![], - details: false, - seed: None, - }, + infer + .generate(GenerateRequest { + inputs: "liveness".to_string(), + parameters: GenerateParameters { + temperature: 1.0, + top_k: 0, + top_p: 1.0, + do_sample: false, + max_new_tokens: 1, + stop: vec![], + details: false, + seed: None, }, - ) + }) .await?; Ok(()) } /// Generate method #[instrument( - skip(state), + skip(infer), fields( total_time, validation_time, @@ -76,56 +59,28 @@ async fn health(state: Extension) -> Result<(), (StatusCode, Json, + infer: Extension, req: Json, ) -> Result)> { + let span = tracing::Span::current(); let start_time = Instant::now(); - // Limit concurrent requests by acquiring a permit from the semaphore - let _permit = state.limit_concurrent_requests.try_acquire().map_err(|_| { - tracing::error!("Model is overloaded"); - ( - StatusCode::TOO_MANY_REQUESTS, - Json(ErrorResponse { - error: "Model is overloaded".to_string(), - }), - ) - })?; - - // Validate request - let details = req.0.parameters.details; - let (input_length, validated_request) = - state.validation.validate(req.0).await.map_err(|err| { - tracing::error!("{}", err.to_string()); - err - })?; // Inference - let response = state - .batcher - .infer(input_length, validated_request) - .await - .map_err(|err| { - tracing::error!("{}", err.to_string()); - err - })?; + let details = req.0.parameters.details; + let response = infer.generate(req.0).await.map_err(|err| { + tracing::error!("{}", err.to_string()); + err + })?; // Token details let details = match details { - true => { - let tokens = response - .token_ids - .into_iter() - .zip(response.tokens.into_iter()) - .zip(response.logprobs.into_iter()) - .map(|((id, text), logprob)| (id, text, logprob)) - .collect(); - Some(Details { - seed: response.seed, - finish_reason: response.finish_reason, - generated_tokens: response.generated_tokens, - tokens, - }) - } + true => Some(Details { + finish_reason: response.generated_text.finish_reason, + generated_tokens: response.generated_text.generated_tokens, + prefill: Some(response.prefill), + tokens: Some(response.tokens), + seed: response.generated_text.seed, + }), false => None, }; @@ -133,8 +88,8 @@ async fn generate( let total_time = start_time.elapsed(); let validation_time = response.queued - start_time; let queue_time = response.start - response.queued; - let inference_time = response.end - response.start; - let time_per_token = inference_time / response.generated_tokens; + let inference_time = Instant::now() - response.start; + let time_per_token = inference_time / response.generated_text.generated_tokens; // Headers let mut headers = HeaderMap::new(); @@ -160,22 +115,143 @@ async fn generate( ); // Tracing metadata - tracing::Span::current().record("total_time", format!("{:?}", total_time)); - tracing::Span::current().record("validation_time", format!("{:?}", validation_time)); - tracing::Span::current().record("queue_time", format!("{:?}", queue_time)); - tracing::Span::current().record("inference_time", format!("{:?}", inference_time)); - tracing::Span::current().record("time_per_token", format!("{:?}", time_per_token)); - tracing::Span::current().record("seed", format!("{:?}", response.seed)); - tracing::info!("Output: {}", response.output_text); + span.record("total_time", format!("{:?}", total_time)); + span.record("validation_time", format!("{:?}", validation_time)); + span.record("queue_time", format!("{:?}", queue_time)); + span.record("inference_time", format!("{:?}", inference_time)); + span.record("time_per_token", format!("{:?}", time_per_token)); + span.record("seed", format!("{:?}", response.generated_text.seed)); + tracing::info!("Output: {}", response.generated_text.text); // Send response - let response = vec![GeneratedText { - generated_text: response.output_text, + let response = vec![GenerateResponse { + generated_text: response.generated_text.text, details, }]; Ok((headers, Json(response))) } +/// Generate stream method +#[instrument( + skip(infer), + fields( + total_time, + validation_time, + queue_time, + inference_time, + time_per_token + ) +)] +async fn generate_stream( + infer: Extension, + req: Json, +) -> Sse>> { + let span = tracing::Span::current(); + let start_time = Instant::now(); + + let stream = async_stream::stream! { + // Inference + let mut end_reached = false; + let mut error = false; + let details = req.0.parameters.details; + + match infer.generate_stream(req.0).await { + Ok(mut response_stream) => { + // Server Side Event stream + while let Some(response) = response_stream.next().await { + match response { + Ok(response) => { + match response { + // Prefill is ignored + InferStreamResponse::Prefill(_) => {} + // Yield event for every new token + InferStreamResponse::Token(token) => { + // StreamResponse + let stream_token = StreamResponse { + token, + generated_text: None, + details: None, + }; + + yield Ok(Event::default().json_data(stream_token).unwrap()) + } + // Yield event for last token and compute timings + InferStreamResponse::End { + token, + generated_text, + start, + queued, + } => { + // Token details + let details = match details { + true => Some(Details { + finish_reason: generated_text.finish_reason, + generated_tokens: generated_text.generated_tokens, + prefill: None, + tokens: None, + seed: generated_text.seed, + }), + false => None, + }; + + // Timings + let total_time = start_time.elapsed(); + let validation_time = queued - start_time; + let queue_time = start - queued; + let inference_time = Instant::now() - start; + let time_per_token = inference_time / generated_text.generated_tokens; + + // Tracing metadata + span.record("total_time", format!("{:?}", total_time)); + span + .record("validation_time", format!("{:?}", validation_time)); + span.record("queue_time", format!("{:?}", queue_time)); + span + .record("inference_time", format!("{:?}", inference_time)); + span + .record("time_per_token", format!("{:?}", time_per_token)); + tracing::info!(parent: &span, "Output: {}", generated_text.text); + + // StreamResponse + end_reached = true; + let stream_token = StreamResponse { + token, + generated_text: Some(generated_text.text), + details + }; + + yield Ok(Event::default().json_data(stream_token).unwrap()) + } + } + } + // Trace and yield error + Err(err) => { + error = true; + tracing::error!("{}", err.to_string()); + yield Ok(Event::from(err)) + } + } + } + }, + // Trace and yield error + Err(err) => { + error = true; + tracing::error!("{}", err.to_string()); + yield Ok(Event::from(err)) + } + } + // Check if generation reached the end + // Skip if we already sent an error + if !end_reached && !error { + let err = InferError::IncompleteGeneration; + tracing::error!("{}", err.to_string()); + yield Ok(Event::from(err)) + } + }; + + Sse::new(stream).keep_alive(KeepAlive::default()) +} + /// Serving method #[allow(clippy::too_many_arguments)] pub async fn run( @@ -189,21 +265,23 @@ pub async fn run( addr: SocketAddr, ) { // Create state - let batcher = Batcher::new(client, max_batch_size, max_waiting_tokens); let validation = Validation::new(validation_workers, tokenizer, max_input_length); - let shared_state = ServerState { + let infer = Infer::new( + client, validation, - batcher, - limit_concurrent_requests: Arc::new(Semaphore::new(max_concurrent_requests)), - }; + max_batch_size, + max_waiting_tokens, + max_concurrent_requests, + ); // Create router let app = Router::new() .route("/", post(generate)) .route("/generate", post(generate)) + .route("/generate_stream", post(generate_stream)) .route("/", get(health)) .route("/health", get(health)) - .layer(Extension(shared_state.clone())); + .layer(Extension(infer)); // Run server axum::Server::bind(&addr) @@ -240,3 +318,32 @@ async fn shutdown_signal() { tracing::info!("signal received, starting graceful shutdown"); } + +/// Convert to Axum supported formats +impl From for (StatusCode, Json) { + fn from(err: InferError) -> Self { + let status_code = match err { + InferError::GenerationError(_) => StatusCode::FAILED_DEPENDENCY, + InferError::Overloaded(_) => StatusCode::TOO_MANY_REQUESTS, + InferError::ValidationError(_) => StatusCode::UNPROCESSABLE_ENTITY, + InferError::IncompleteGeneration => StatusCode::INTERNAL_SERVER_ERROR, + }; + + ( + status_code, + Json(ErrorResponse { + error: err.to_string(), + }), + ) + } +} + +impl From for Event { + fn from(err: InferError) -> Self { + Event::default() + .json_data(ErrorResponse { + error: err.to_string(), + }) + .unwrap() + } +} diff --git a/router/src/validation.rs b/router/src/validation.rs index 674635d3..39da2a83 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,9 +1,8 @@ /// Payload validation logic -use crate::{ErrorResponse, GenerateRequest}; -use axum::http::StatusCode; -use axum::Json; +use crate::{GenerateParameters, GenerateRequest}; use rand::rngs::ThreadRng; use rand::Rng; +use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParameters}; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokio::sync::{mpsc, oneshot}; @@ -40,7 +39,7 @@ impl Validation { pub(crate) async fn validate( &self, request: GenerateRequest, - ) -> Result<(usize, GenerateRequest), ValidationError> { + ) -> Result { // Create response channel let (sender, receiver) = oneshot::channel(); // Send request to the background validation task @@ -106,11 +105,11 @@ fn validation_worker( } fn validate( - mut request: GenerateRequest, + request: GenerateRequest, tokenizer: &Tokenizer, max_input_length: usize, rng: &mut ThreadRng, -) -> Result<(usize, GenerateRequest), ValidationError> { +) -> Result { if request.parameters.temperature <= 0.0 { return Err(ValidationError::Temperature); } @@ -131,19 +130,48 @@ fn validate( } // If seed is None, assign a random one - if request.parameters.seed.is_none() { - request.parameters.seed = Some(rng.gen()); - } + let seed = match request.parameters.seed { + None => rng.gen(), + Some(seed) => seed, + }; // Get the number of tokens in the input match tokenizer.encode(request.inputs.clone(), true) { - Ok(inputs) => { - let input_length = inputs.len(); + Ok(encoding) => { + let input_length = encoding.len(); if input_length > max_input_length { Err(ValidationError::InputLength(input_length, max_input_length)) } else { - Ok((input_length, request)) + // Return ValidGenerateRequest + let GenerateParameters { + temperature, + top_k, + top_p, + do_sample, + max_new_tokens, + stop: stop_sequences, + .. + } = request.parameters; + + let parameters = NextTokenChooserParameters { + temperature, + top_k: top_k as u32, + top_p, + do_sample, + seed, + }; + let stopping_parameters = StoppingCriteriaParameters { + max_new_tokens, + stop_sequences, + }; + + Ok(ValidGenerateRequest { + inputs: request.inputs, + input_length: input_length as u32, + parameters, + stopping_parameters, + }) } } Err(err) => Err(ValidationError::Tokenizer(err.to_string())), @@ -152,9 +180,17 @@ fn validate( type ValidationRequest = ( GenerateRequest, - oneshot::Sender>, + oneshot::Sender>, ); +#[derive(Debug)] +pub(crate) struct ValidGenerateRequest { + pub inputs: String, + pub input_length: u32, + pub parameters: NextTokenChooserParameters, + pub stopping_parameters: StoppingCriteriaParameters, +} + #[derive(Error, Debug)] pub enum ValidationError { #[error("temperature must be strictly positive")] @@ -172,14 +208,3 @@ pub enum ValidationError { #[error("tokenizer error {0}")] Tokenizer(String), } - -impl From for (StatusCode, Json) { - fn from(err: ValidationError) -> Self { - ( - StatusCode::UNPROCESSABLE_ENTITY, - Json(ErrorResponse { - error: err.to_string(), - }), - ) - } -} diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 1a788ce5..9f96efc3 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -91,9 +91,9 @@ def test_causal_lm_batch_type(default_bloom): def test_causal_lm_generate_token(default_bloom, default_bloom_batch): sequence_length = len(default_bloom_batch.all_input_ids[0]) - generated_texts, next_batch = default_bloom.generate_token(default_bloom_batch) + generations, next_batch = default_bloom.generate_token(default_bloom_batch) - assert generated_texts == [] + assert len(generations) == len(default_bloom_batch) assert isinstance(next_batch, CausalLMBatch) assert not next_batch.keys_head_dim_last @@ -122,24 +122,30 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): assert all( [p[1].shape == (16, sequence_length, 64) for p in next_batch.past_key_values] ) + assert all([generation.generated_text is None for generation in generations]) + assert all([len(generation.prefill_tokens) == 1 for generation in generations]) + assert all([generation.token_id.item() == 10264 for generation in generations]) + assert all([generation.token_text == "Test" for generation in generations]) + assert generations[0].request_id == 0 def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch): next_batch = default_bloom_batch for _ in range(default_bloom_batch.stopping_criterias[0].max_new_tokens - 1): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(default_bloom_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 + assert len(generations) == 1 assert ( - generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + generations[0].generated_text.text + == "TestTestTestTestTestTestTestTestTestTestTest" ) - assert generated_texts[0].request == default_bloom_batch.requests[0] + assert generations[0].request_id == default_bloom_batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -152,17 +158,19 @@ def test_causal_lm_generate_token_completion_multi( for i in range( default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 1 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(default_multi_requests_bloom_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "TestTestTestTestTestTest" - assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] + assert len(generations) == 2 + assert generations[1].generated_text.text == "TestTestTestTestTestTest" assert ( - generated_texts[0].generated_tokens + generations[1].request_id == default_multi_requests_bloom_batch.requests[1].id + ) + assert ( + generations[1].generated_text.generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) @@ -171,19 +179,22 @@ def test_causal_lm_generate_token_completion_multi( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 1 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 + assert len(generations) == 1 + assert ( + generations[0].generated_text.text + == "TestTestTestTestTestTestTestTestTestTestTest" + ) assert ( - generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id ) - assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -243,17 +254,19 @@ def test_batch_concatenate( for _ in range( default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 2 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "TestTestTestTestTestTest" - assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[1] + assert len(generations) == 3 + assert generations[2].generated_text.text == "TestTestTestTestTestTest" assert ( - generated_texts[0].generated_tokens + generations[2].request_id == default_multi_requests_bloom_batch.requests[1].id + ) + assert ( + generations[2].generated_text.generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) @@ -262,19 +275,20 @@ def test_batch_concatenate( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 2 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 + assert len(generations) == 2 assert ( - generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + generations[0].generated_text.text + == "TestTestTestTestTestTestTestTestTestTestTest" ) - assert generated_texts[0].request == default_bloom_batch.requests[0] + assert generations[0].request_id == default_bloom_batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_bloom_batch.stopping_criterias[0].max_new_tokens ) @@ -284,18 +298,21 @@ def test_batch_concatenate( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 4 ): - generated_texts, next_batch = default_bloom.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_bloom.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch = default_bloom.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 + assert len(generations) == 1 + assert ( + generations[0].generated_text.text + == "TestTestTestTestTestTestTestTestTestTestTest" + ) assert ( - generated_texts[0].output_text == "TestTestTestTestTestTestTestTestTestTestTest" + generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id ) - assert generated_texts[0].request == default_multi_requests_bloom_batch.requests[0] assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index bedb65ba..f9762b30 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -88,11 +88,9 @@ def test_causal_lm_batch_type(default_causal_lm): def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): sequence_length = len(default_causal_lm_batch.all_input_ids[0]) - generated_texts, next_batch = default_causal_lm.generate_token( - default_causal_lm_batch - ) + generations, next_batch = default_causal_lm.generate_token(default_causal_lm_batch) - assert generated_texts == [] + assert len(generations) == len(next_batch) assert isinstance(next_batch, CausalLMBatch) assert len(next_batch.all_input_ids) == next_batch.size @@ -121,6 +119,11 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): assert all( [p[1].shape == (1, 12, sequence_length, 64) for p in next_batch.past_key_values] ) + assert all([generation.generated_text is None for generation in generations]) + assert all([len(generation.prefill_tokens) == 1 for generation in generations]) + assert all([generation.token_id.item() == 13 for generation in generations]) + assert all([generation.token_text == "." for generation in generations]) + assert generations[0].request_id == 0 def test_causal_lm_generate_token_completion( @@ -128,18 +131,17 @@ def test_causal_lm_generate_token_completion( ): next_batch = default_causal_lm_batch for _ in range(default_causal_lm_batch.stopping_criterias[0].max_new_tokens - 1): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." - assert generated_texts[0].request == default_causal_lm_batch.requests[0] - assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) + assert len(generations) == 1 + assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert generations[0].request_id == default_causal_lm_batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -152,19 +154,20 @@ def test_causal_lm_generate_token_completion_multi( for i in range( default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 1 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784)" + assert len(generations) == 2 + assert generations[1].generated_text.text == "Test.java:784)" assert ( - generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] + generations[1].request_id + == default_multi_requests_causal_lm_batch.requests[1].id ) assert ( - generated_texts[0].generated_tokens + generations[1].generated_text.generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) @@ -173,19 +176,20 @@ def test_causal_lm_generate_token_completion_multi( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 1 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." + assert len(generations) == 1 + assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." assert ( - generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] + generations[0].request_id + == default_multi_requests_causal_lm_batch.requests[0].id ) assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -244,19 +248,20 @@ def test_batch_concatenate( for _ in range( default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784)" + assert len(generations) == 3 + assert generations[2].generated_text.text == "Test.java:784)" assert ( - generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[1] + generations[2].request_id + == default_multi_requests_causal_lm_batch.requests[1].id ) assert ( - generated_texts[0].generated_tokens + generations[2].generated_text.generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) @@ -265,17 +270,17 @@ def test_batch_concatenate( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." - assert generated_texts[0].request == default_causal_lm_batch.requests[0] + assert len(generations) == 2 + assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert generations[0].request_id == default_causal_lm_batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) @@ -285,18 +290,19 @@ def test_batch_concatenate( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 4 ): - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_causal_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch = default_causal_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "Test.java:784) at net.minecraft." + assert len(generations) == 1 + assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." assert ( - generated_texts[0].request == default_multi_requests_causal_lm_batch.requests[0] + generations[0].request_id + == default_multi_requests_causal_lm_batch.requests[0].id ) assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index acebec04..1b69477d 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -50,18 +50,17 @@ def test_santacoder_generate_token_completion(default_santacoder, default_pb_bat next_batch = batch for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): - generated_texts, next_batch = default_santacoder.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_santacoder.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_santacoder.generate_token(next_batch) + generations, next_batch = default_santacoder.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "def test_get_all_users_with_" - assert generated_texts[0].request == batch.requests[0] - assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) + assert len(generations) == 1 + assert generations[0].generated_text.text == "def test_get_all_users_with_" + assert generations[0].request_id == batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == batch.stopping_criterias[0].max_new_tokens ) @@ -76,20 +75,19 @@ def test_fim_santacoder_generate_token_completion( next_batch = batch for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): - generated_texts, next_batch = default_santacoder.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_santacoder.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_santacoder.generate_token(next_batch) + generations, next_batch = default_santacoder.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 + assert len(generations) == 1 assert ( - generated_texts[0].output_text + generations[0].generated_text.text == """defworldineProperty(exports, "__esModule", { value""" ) - assert generated_texts[0].request == batch.requests[0] - assert len(generated_texts[0].tokens) == len(generated_texts[0].logprobs) + assert generations[0].request_id == batch.requests[0].id assert ( - generated_texts[0].generated_tokens + generations[0].generated_text.generated_tokens == batch.stopping_criterias[0].max_new_tokens ) diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index de1a4829..22c6ac9c 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -99,11 +99,11 @@ def test_seq2seq_lm_batch_type(default_seq2seq_lm): def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch): sequence_length = len(default_seq2seq_lm_batch.input_ids[0]) - generated_texts, next_batch = default_seq2seq_lm.generate_token( + generations, next_batch = default_seq2seq_lm.generate_token( default_seq2seq_lm_batch ) - assert generated_texts == [] + assert len(generations) == len(next_batch) assert isinstance(next_batch, Seq2SeqLMBatch) assert torch.equal(next_batch.input_ids, default_seq2seq_lm_batch.input_ids) @@ -145,6 +145,11 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) for p in next_batch.past_key_values ] ) + assert all([generation.generated_text is None for generation in generations]) + assert all([len(generation.prefill_tokens) == 1 for generation in generations]) + assert all([generation.token_id.item() == 259 for generation in generations]) + assert all([generation.token_text == "" for generation in generations]) + assert generations[0].request_id == 0 def test_seq2seq_lm_generate_token_completion( @@ -152,16 +157,16 @@ def test_seq2seq_lm_generate_token_completion( ): next_batch = default_seq2seq_lm_batch for _ in range(6): - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few weeks" - assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] - assert generated_texts[0].generated_tokens == 7 + assert len(generations) == 1 + assert generations[0].generated_text.text == "a few weeks" + assert generations[0].request_id == default_seq2seq_lm_batch.requests[0].id + assert generations[0].generated_text.generated_tokens == 7 def test_seq2seq_lm_generate_token_completion_multi( @@ -170,33 +175,33 @@ def test_seq2seq_lm_generate_token_completion_multi( next_batch = default_multi_requests_seq2seq_lm_batch for i in range(4): - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few " + assert len(generations) == 2 + assert generations[1].generated_text.text == "a few " assert ( - generated_texts[0].request - == default_multi_requests_seq2seq_lm_batch.requests[1] + generations[1].request_id + == default_multi_requests_seq2seq_lm_batch.requests[1].id ) - assert generated_texts[0].generated_tokens == 5 + assert generations[1].generated_text.generated_tokens == 5 - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few weeks" + assert len(generations) == 1 + assert generations[0].generated_text.text == "a few weeks" assert ( - generated_texts[0].request - == default_multi_requests_seq2seq_lm_batch.requests[0] + generations[0].request_id + == default_multi_requests_seq2seq_lm_batch.requests[0].id ) - assert generated_texts[0].generated_tokens == 7 + assert generations[0].generated_text.generated_tokens == 7 def test_batch_concatenate( @@ -291,35 +296,35 @@ def test_batch_concatenate( ) for _ in range(3): - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) - assert generated_texts == [] + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + assert len(generations) == len(next_batch) - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few " + assert len(generations) == 3 + assert generations[2].generated_text.text == "a few " assert ( - generated_texts[0].request - == default_multi_requests_seq2seq_lm_batch.requests[1] + generations[2].request_id + == default_multi_requests_seq2seq_lm_batch.requests[1].id ) - assert generated_texts[0].generated_tokens == 5 + assert generations[2].generated_text.generated_tokens == 5 - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few weeks" - assert generated_texts[0].request == default_seq2seq_lm_batch.requests[0] - assert generated_texts[0].generated_tokens == 7 + assert len(generations) == 2 + assert generations[0].generated_text.text == "a few weeks" + assert generations[0].request_id == default_seq2seq_lm_batch.requests[0].id + assert generations[0].generated_text.generated_tokens == 7 - generated_texts, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None - assert len(generated_texts) == 1 - assert generated_texts[0].output_text == "a few weeks" + assert len(generations) == 1 + assert generations[0].generated_text.text == "a few weeks" assert ( - generated_texts[0].request - == default_multi_requests_seq2seq_lm_batch.requests[0] + generations[0].request_id + == default_multi_requests_seq2seq_lm_batch.requests[0].id ) - assert generated_texts[0].generated_tokens == 7 + assert generations[0].generated_text.generated_tokens == 7 diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index bf49d134..31996e06 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -5,7 +5,7 @@ from typing import Optional, Tuple, List, Type from text_generation.models import Model -from text_generation.models.types import GeneratedText, Batch +from text_generation.models.types import Batch, PrefillTokens, Generation, GeneratedText from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -23,7 +23,6 @@ class CausalLMBatch(Batch): # All tokens all_input_ids: List[torch.Tensor] - all_logprobs: List[Optional[torch.Tensor]] # Lengths of all generations present in the batch input_lengths: List[int] @@ -57,7 +56,6 @@ def from_pb( next_token_choosers = [] stopping_criterias = [] input_lengths = [] - all_logprobs = [] # Parse batch for r in pb.requests: @@ -67,7 +65,6 @@ def from_pb( stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) - all_logprobs.append(None) pad_to_multiple_of = 8 if device.type == "cuda" else None tokenized_inputs = tokenizer( @@ -89,7 +86,6 @@ def from_pb( position_ids=position_ids, past_key_values=None, all_input_ids=all_input_ids, - all_logprobs=all_logprobs, input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, @@ -107,7 +103,6 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests = [] input_lengths = [] all_input_ids = [] - all_logprobs = [] next_token_choosers = [] stopping_criterias = [] @@ -124,7 +119,6 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) all_input_ids.extend(batch.all_input_ids) - all_logprobs.extend(batch.all_logprobs) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -225,7 +219,6 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": position_ids=position_ids, past_key_values=past_key_values, all_input_ids=all_input_ids, - all_logprobs=all_logprobs, input_lengths=input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, @@ -234,6 +227,9 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": keys_head_dim_last=batches[0].keys_head_dim_last, ) + def __len__(self): + return len(self.requests) + class CausalLM(Model): def __init__(self, model_name: str, quantize=False): @@ -289,7 +285,7 @@ def forward( def generate_token( self, batch: CausalLMBatch - ) -> Tuple[List[GeneratedText], Optional[CausalLMBatch]]: + ) -> Tuple[List[Generation], Optional[CausalLMBatch]]: # For some reason, inference_mode does not work well with GLOO which we use on CPU context_manager = ( torch.no_grad if self.device.type == "cpu" else torch.inference_mode @@ -309,14 +305,13 @@ def generate_token( next_batch_input_lengths = [] next_batch_input_ids = [] next_batch_all_input_ids = [] - next_batch_all_logprobs = [] # Metadata next_batch_size = 0 next_batch_max_sequence_length = 0 - # Finished requests - generated_texts: List[GeneratedText] = [] + # Results + generations: List[Generation] = [] # Zipped iterator iterator = zip( @@ -326,7 +321,6 @@ def generate_token( batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, - batch.all_logprobs, ) # For each member of the batch @@ -337,44 +331,36 @@ def generate_token( next_token_chooser, stopping_criteria, all_input_ids, - all_logprobs, ) in enumerate(iterator): # Select next token tokens, logprobs = next_token_chooser(all_input_ids, logits) - next_token = tokens[-1].view(1, 1) + next_token_id = tokens[-1].view(1, 1) # Append next token to all tokens - all_input_ids = torch.cat([all_input_ids, next_token]) + all_input_ids = torch.cat([all_input_ids, next_token_id]) new_input_length = input_length + 1 - if all_logprobs is None: - # logprobs of all prompt tokens (except the first one) and the generated token - all_logprobs = logprobs.gather(1, all_input_ids[1:]) - else: - # logprob of the generated token - next_token_logprob = logprobs[-1, next_token] - all_logprobs = torch.cat([all_logprobs, next_token_logprob]) + # Generated token + next_token_logprob = logprobs[-1, next_token_id] + next_token_id_squeezed = next_token_id.squeeze() + next_token_text = self.tokenizer.decode( + next_token_id_squeezed, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) # Evaluate stopping criteria stop, reason = stopping_criteria( - next_token.squeeze(), - self.tokenizer.decode( - next_token.squeeze(), clean_up_tokenization_spaces=False - ), + next_token_id_squeezed, + next_token_text, ) + if stop: # Decode generated tokens generated_text = self.decode( all_input_ids[-stopping_criteria.current_tokens :, 0] ) output_text = request.inputs + generated_text - # Slice with input_length to remove padding - token_ids = all_input_ids[-new_input_length:] - tokens = self.tokenizer.batch_decode(token_ids) - # Add NaN for the first prompt token - logprobs = [float("nan")] + all_logprobs[-input_length:].squeeze( - 1 - ).tolist() # Get seed if isinstance(next_token_chooser.choice, Sampling): @@ -382,39 +368,58 @@ def generate_token( else: seed = None - # Add to the list of finished generations with the original request - generated_texts.append( - GeneratedText( - request=request, - output_text=output_text, - generated_tokens=stopping_criteria.current_tokens, - tokens=tokens, - token_ids=token_ids.squeeze(1).tolist(), - logprobs=logprobs, - reason=reason, - seed=seed, - ) + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed ) - # add to the next batch else: + # Keep request in the batch + generated_text = None next_batch_keep_indices.append(i) - next_batch_input_ids.append(next_token) + next_batch_input_ids.append(next_token_id) next_batch_all_input_ids.append(all_input_ids) - next_batch_all_logprobs.append(all_logprobs) next_batch_size += 1 next_batch_input_lengths.append(new_input_length) next_batch_max_sequence_length = max( next_batch_max_sequence_length, new_input_length ) + # Prefill + if stopping_criteria.current_tokens == 1: + # Remove generated token to only have prefill and add nan for first prompt token + prefill_logprobs = [float("nan")] + logprobs.gather( + 1, all_input_ids[1:] + ).squeeze(1)[-new_input_length:-1].tolist() + prefill_token_ids = all_input_ids[-new_input_length:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = PrefillTokens( + prefill_token_ids, prefill_logprobs, prefill_texts + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id_squeezed, + next_token_logprob, + next_token_text, + generated_text, + ) + + generations.append(generation) + # We finished all generations in the batch; there is no next batch if not next_batch_keep_indices: - return generated_texts, None + return generations, None next_batch_input_ids = torch.cat(next_batch_input_ids, dim=0) # If we finished at least one generation, we need to evict the indices of the generations that finished # from the values of the next batch - if generated_texts: + if len(next_batch_keep_indices) != len(batch): # Apply indices to attention mask, past key values and other items that need to be cached next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] next_batch_position_ids = batch.position_ids[next_batch_keep_indices] @@ -461,7 +466,6 @@ def generate_token( position_ids=next_batch_position_ids, past_key_values=next_batch_past_key_values, all_input_ids=next_batch_all_input_ids, - all_logprobs=next_batch_all_logprobs, input_lengths=next_batch_input_lengths, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, @@ -469,4 +473,4 @@ def generate_token( max_sequence_length=next_batch_max_sequence_length, keys_head_dim_last=batch.keys_head_dim_last, ) - return generated_texts, next_batch + return generations, next_batch diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 26ebc7d7..6d5dc22e 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -5,7 +5,7 @@ from typing import Optional, Tuple, List, Type from text_generation.models import Model -from text_generation.models.types import GeneratedText, Batch +from text_generation.models.types import GeneratedText, Batch, Generation, PrefillTokens from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -30,7 +30,6 @@ class Seq2SeqLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] decoder_input_lengths: List[int] - decoder_logprobs: List[Optional[torch.Tensor]] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -64,7 +63,6 @@ def from_pb( decoder_input_ids = [] decoder_input_lengths = [] - decoder_logprobs = [] # Parse batch for r in pb.requests: @@ -77,7 +75,6 @@ def from_pb( stopping_criterias.append( StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) ) - decoder_logprobs.append(None) # Tokenize batch pad_to_multiple_of = 8 if device.type == "cuda" else None @@ -102,7 +99,6 @@ def from_pb( past_key_values=None, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, - decoder_logprobs=decoder_logprobs, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=len(pb.requests), @@ -125,7 +121,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests = [] input_lengths = [] decoder_input_lengths = [] - decoder_logprobs = [] next_token_choosers = [] stopping_criterias = [] @@ -146,7 +141,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) decoder_input_lengths.extend(batch.decoder_input_lengths) - decoder_logprobs.extend(batch.decoder_logprobs) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -283,7 +277,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": past_key_values=past_key_values, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, - decoder_logprobs=decoder_logprobs, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=total_batch_size, @@ -291,6 +284,9 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": max_decoder_input_length=max_decoder_input_length, ) + def __len__(self): + return len(self.requests) + class Seq2SeqLM(Model): def __init__(self, model_name: str, quantize=False): @@ -364,7 +360,7 @@ def forward( def generate_token( self, batch: Seq2SeqLMBatch - ) -> Tuple[List[GeneratedText], Optional[Seq2SeqLMBatch]]: + ) -> Tuple[List[Generation], Optional[Seq2SeqLMBatch]]: # For some reason, inference_mode does not work well with GLOO which we use on CPU context_manager = ( torch.no_grad if self.device.type == "cpu" else torch.inference_mode @@ -386,7 +382,6 @@ def generate_token( next_batch_input_lengths = [] next_batch_decoder_input_ids = [] next_batch_decoder_input_lengths = [] - next_batch_decoder_logprobs = [] # Metadata next_batch_size = 0 @@ -394,14 +389,13 @@ def generate_token( next_batch_max_decoder_input_length = 0 # Finished requests - generated_texts: List[GeneratedText] = [] + generations: List[Generation] = [] # Zipped iterator iterator = zip( batch.requests, batch.input_lengths, batch.decoder_input_lengths, - batch.decoder_logprobs, logits, batch.next_token_choosers, batch.stopping_criterias, @@ -414,7 +408,6 @@ def generate_token( request, input_length, decoder_input_length, - decoder_logprobs, logits, next_token_chooser, stopping_criteria, @@ -422,35 +415,28 @@ def generate_token( decoder_input_ids, ) in enumerate(iterator): # Select next token - next_token, logprobs = next_token_chooser(decoder_input_ids, logits) + next_token_id, logprobs = next_token_chooser(decoder_input_ids, logits) # Append next token to decoder tokens - decoder_input_ids = torch.cat([decoder_input_ids, next_token]) + decoder_input_ids = torch.cat([decoder_input_ids, next_token_id]) new_decoder_input_length = decoder_input_length + 1 - next_token_logprob = logprobs[-1, next_token] - if decoder_logprobs is None: - decoder_logprobs = next_token_logprob - else: - decoder_logprobs = torch.cat([decoder_logprobs, next_token_logprob]) + # Generated token + next_token_logprob = logprobs[-1, next_token_id] + next_token_id_squeezed = next_token_id.squeeze() + next_token_text = self.tokenizer.decode( + next_token_id_squeezed, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) # Evaluate stopping criteria - stop, reason = stopping_criteria( - next_token.squeeze(), - self.tokenizer.decode( - next_token.squeeze(), clean_up_tokenization_spaces=False - ), - ) + stop, reason = stopping_criteria(next_token_id, next_token_text) + if stop: # Slice with decoder_input_length to remove padding # Decode all tokens - token_ids = decoder_input_ids[-new_decoder_input_length:] - output_text = self.decode(token_ids) - tokens = self.tokenizer.batch_decode(token_ids) - # Add NaN for the bos token - logprobs = [float("nan")] + decoder_logprobs[ - -decoder_input_length: - ].tolist() + output_text = self.decode(decoder_input_ids[-new_decoder_input_length:]) # Get seed if isinstance(next_token_chooser.choice, Sampling): @@ -458,27 +444,17 @@ def generate_token( else: seed = None - # Add to the list of finished generations with the original request - generated_texts.append( - GeneratedText( - request=request, - output_text=output_text, - generated_tokens=stopping_criteria.current_tokens, - tokens=tokens, - token_ids=token_ids.tolist(), - logprobs=logprobs, - reason=reason, - seed=seed, - ) + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed ) - # add to the next batch else: + # Keep request in the batch + generated_text = None next_batch_keep_indices.append(i) next_batch_decoder_input_ids.append(decoder_input_ids.unsqueeze(0)) next_batch_size += 1 next_batch_input_lengths.append(input_length) next_batch_decoder_input_lengths.append(new_decoder_input_length) - next_batch_decoder_logprobs.append(decoder_logprobs) next_batch_max_input_length = max( next_batch_max_input_length, input_length ) @@ -486,14 +462,39 @@ def generate_token( next_batch_max_decoder_input_length, new_decoder_input_length ) + # Prefill + if stopping_criteria.current_tokens == 1: + prefill_token_ids = decoder_input_ids[-new_decoder_input_length:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = PrefillTokens( + prefill_token_ids, [float("nan")], prefill_texts + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id_squeezed, + next_token_logprob, + next_token_text, + generated_text, + ) + + generations.append(generation) + # We finished all generations in the batch; there is no next batch if not next_batch_keep_indices: - return generated_texts, None + return generations, None next_batch_decoder_input_ids = torch.cat(next_batch_decoder_input_ids) # If we finished at least one generation, we need to evict the indices of the generations that finished # from the values of the next batch - if generated_texts: + if len(next_batch_keep_indices) != len(batch): # Apply indices to attention mask, past key values and other items that need to be cached next_batch_input_ids = batch.input_ids[next_batch_keep_indices] next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] @@ -551,11 +552,10 @@ def generate_token( past_key_values=next_batch_past_key_values, input_lengths=next_batch_input_lengths, decoder_input_lengths=next_batch_decoder_input_lengths, - decoder_logprobs=next_batch_decoder_logprobs, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, max_input_length=next_batch_max_input_length, max_decoder_input_length=next_batch_max_decoder_input_length, ) - return generated_texts, next_batch + return generations, next_batch diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index 4ee3cb32..30cd716a 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -29,26 +29,61 @@ def from_pb( def concatenate(cls, batches: List["Batch"]) -> "Batch": raise NotImplementedError + @abstractmethod + def __len__(self): + raise NotImplementedError + @dataclass class GeneratedText: - request: generate_pb2.Request - output_text: str + text: str generated_tokens: int - tokens: List[str] - token_ids: List[int] - logprobs: List[float] - reason: str + finish_reason: str seed: Optional[int] def to_pb(self) -> generate_pb2.GeneratedText: return generate_pb2.GeneratedText( - request=self.request, - output_text=self.output_text, + text=self.text, generated_tokens=self.generated_tokens, - tokens=self.tokens, - token_ids=self.token_ids, - logprobs=self.logprobs, - finish_reason=self.reason, + finish_reason=self.finish_reason, seed=self.seed, ) + + +@dataclass +class PrefillTokens: + token_ids: List[int] + logprobs: List[float] + texts: List[str] + + def to_pb(self) -> generate_pb2.PrefillTokens: + return generate_pb2.PrefillTokens( + ids=self.token_ids, logprobs=self.logprobs, texts=self.texts + ) + + def __len__(self): + return len(self.token_ids) + + +@dataclass +class Generation: + request_id: int + prefill_tokens: Optional[PrefillTokens] + token_id: int + token_logprob: float + token_text: str + generated_text: Optional[GeneratedText] + + def to_pb(self) -> generate_pb2.Generation: + return generate_pb2.Generation( + request_id=self.request_id, + prefill_tokens=self.prefill_tokens.to_pb() + if self.prefill_tokens is not None + else None, + token_id=self.token_id, + token_logprob=self.token_logprob, + token_text=self.token_text, + generated_text=self.generated_text.to_pb() + if self.generated_text is not None + else None, + ) diff --git a/server/text_generation/server.py b/server/text_generation/server.py index 5fd3072e..a2bad8a7 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -27,22 +27,20 @@ async def ClearCache(self, request, context): self.cache.clear() return generate_pb2.ClearCacheResponse() - async def Generate(self, request, context): + async def Prefill(self, request, context): batch = self.model.batch_type.from_pb( request.batch, self.model.tokenizer, self.model.device ) - generated_texts, next_batch = self.model.generate_token(batch) + generations, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) - return generate_pb2.GenerateResponse( - generated_texts=[ - generated_text.to_pb() for generated_text in generated_texts - ], + return generate_pb2.PrefillResponse( + generations=[generation.to_pb() for generation in generations], batch=next_batch.to_pb() if next_batch else None, ) - async def GenerateWithCache(self, request, context): + async def Decode(self, request, context): if len(request.batches) == 0: raise ValueError("Must provide at least one batch") @@ -58,13 +56,11 @@ async def GenerateWithCache(self, request, context): else: batch = batches[0] - generated_texts, next_batch = self.model.generate_token(batch) + generations, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) - return generate_pb2.GenerateWithCacheResponse( - generated_texts=[ - generated_text.to_pb() for generated_text in generated_texts - ], + return generate_pb2.DecodeResponse( + generations=[generation.to_pb() for generation in generations], batch=next_batch.to_pb() if next_batch else None, ) From c6e8b9442b1fcf7bbbe4be58fcd85047f69e4112 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 31 Jan 2023 17:40:38 +0100 Subject: [PATCH 061/793] fix(server): fix quantization for sharded models (#45) --- server/text_generation/models/bloom.py | 12 +++--------- server/text_generation/models/galactica.py | 16 ++++------------ 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 1b7635c5..7708bb4a 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -196,15 +196,11 @@ def load_weights( tensor.CB = None tensor.SCB = None - def replace_linear(state, in_features, out_features): + def replace_linear(state): def linear(input, weight, bias): - size_out = input.size()[:-1] + (out_features,) - input = input.view(-1, in_features) - out = input.new_empty(size_out) out = bnb.matmul( input, weight, - out=out.view(-1, out_features), state=state, threshold=state.threshold, bias=bias, @@ -217,13 +213,11 @@ def linear(input, weight, bias): del state.CB weight.data = state.CxB - return out.view(size_out) + return out return linear - module.linear = replace_linear( - state, module.in_features, module.out_features - ) + module.linear = replace_linear(state) else: tensor = tensor.to(device) diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index 5cc55865..4722e1d8 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -232,7 +232,6 @@ def load_weights( start = rank * block_size stop = (rank + 1) * block_size tensor = slice_[start:stop] - tensor = tensor.transpose(1, 0) else: size = slice_.get_shape()[0] block_size = size // world_size @@ -246,7 +245,6 @@ def load_weights( start = rank * block_size stop = (rank + 1) * block_size tensor = slice_[:, start:stop] - tensor = tensor.transpose(1, 0) else: tensor = slice_[:] # XXX: Hack for Rowlinear to add the bias only once. @@ -282,7 +280,7 @@ def load_weights( and param_name == "weight" ): tensor = Int8Params( - tensor.transpose(1, 0), + tensor, has_fp16_weights=False, requires_grad=False, ).to(device) @@ -296,15 +294,11 @@ def load_weights( tensor.CB = None tensor.SCB = None - def replace_linear(state, in_features, out_features): + def replace_linear(state): def linear(input, weight, bias): - size_out = input.size()[:-1] + (out_features,) - input = input.view(-1, in_features) - out = input.new_empty(size_out) out = bnb.matmul( input, weight, - out=out.view(-1, out_features), state=state, threshold=state.threshold, bias=bias, @@ -317,13 +311,11 @@ def linear(input, weight, bias): del state.CB weight.data = state.CxB - return out.view(size_out) + return out return linear - module.linear = replace_linear( - state, module.in_features, module.out_features - ) + module.linear = replace_linear(state) else: tensor = tensor.to(device) From f830706b215a94e1cdedeb89766b2822d8ceac24 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 31 Jan 2023 18:53:56 +0100 Subject: [PATCH 062/793] feat(server): Support GPT-Neox (#39) --- README.md | 1 + launcher/src/main.rs | 11 + server/tests/test_utils.py | 6 +- server/text_generation/cli.py | 7 +- server/text_generation/models/__init__.py | 31 ++- server/text_generation/models/bloom.py | 16 +- server/text_generation/models/causal_lm.py | 7 +- server/text_generation/models/galactica.py | 25 +- server/text_generation/models/gpt_neox.py | 244 ++++++++++++++++++++ server/text_generation/models/santacoder.py | 7 +- server/text_generation/models/seq2seq_lm.py | 7 +- server/text_generation/server.py | 8 +- server/text_generation/utils.py | 63 ++++- 13 files changed, 386 insertions(+), 47 deletions(-) create mode 100644 server/text_generation/models/gpt_neox.py diff --git a/README.md b/README.md index 2635f641..67940ae8 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ to power Bloom, BloomZ and MT0-XXL api-inference widgets. - [MT0-XXL](https://huggingface.co/bigscience/mt0-xxl) - ~~[Galactica](https://huggingface.co/facebook/galactica-120b)~~ (deactivated) - [SantaCoder](https://huggingface.co/bigcode/santacoder) +- [GPT-Neox 20B](https://huggingface.co/EleutherAI/gpt-neox-20b): use `--revision refs/pr/13` Other models are supported on a best effort basis using: diff --git a/launcher/src/main.rs b/launcher/src/main.rs index f897d2cd..bd449e28 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -21,6 +21,8 @@ struct Args { #[clap(default_value = "bigscience/bloom-560m", long, env)] model_name: String, #[clap(long, env)] + revision: Option, + #[clap(long, env)] num_shard: Option, #[clap(long, env)] quantize: bool, @@ -48,6 +50,7 @@ fn main() -> ExitCode { // Pattern match configuration let Args { model_name, + revision, num_shard, quantize, max_concurrent_requests, @@ -90,6 +93,7 @@ fn main() -> ExitCode { // Start shard processes for rank in 0..num_shard { let model_name = model_name.clone(); + let revision = revision.clone(); let uds_path = shard_uds_path.clone(); let master_addr = master_addr.clone(); let status_sender = status_sender.clone(); @@ -98,6 +102,7 @@ fn main() -> ExitCode { thread::spawn(move || { shard_manager( model_name, + revision, quantize, uds_path, rank, @@ -252,6 +257,7 @@ enum ShardStatus { #[allow(clippy::too_many_arguments)] fn shard_manager( model_name: String, + revision: Option, quantize: bool, uds_path: String, rank: usize, @@ -288,6 +294,11 @@ fn shard_manager( shard_argv.push("--quantize".to_string()) } + if let Some(revision) = revision { + shard_argv.push("--revision".to_string()); + shard_argv.push(revision) + } + let mut env = vec![ ("RANK".into(), rank.to_string().into()), ("WORLD_SIZE".into(), world_size.to_string().into()), diff --git a/server/tests/test_utils.py b/server/tests/test_utils.py index 643cb834..1dc6801b 100644 --- a/server/tests/test_utils.py +++ b/server/tests/test_utils.py @@ -1,5 +1,7 @@ import pytest +from huggingface_hub.utils import RevisionNotFoundError + from text_generation.utils import ( weight_hub_files, download_weights, @@ -51,7 +53,7 @@ def test_weight_hub_files_llm(): def test_weight_hub_files_empty(): - filenames = weight_hub_files("bigscience/bloom", ".errors") + filenames = weight_hub_files("bigscience/bloom", extension=".errors") assert filenames == [] @@ -62,5 +64,7 @@ def test_download_weights(): def test_weight_files_error(): + with pytest.raises(RevisionNotFoundError): + weight_files("bigscience/bloom-560m", revision="error") with pytest.raises(LocalEntryNotFoundError): weight_files("bert-base-uncased") diff --git a/server/text_generation/cli.py b/server/text_generation/cli.py index 8101cea4..b133cb0a 100644 --- a/server/text_generation/cli.py +++ b/server/text_generation/cli.py @@ -4,6 +4,7 @@ from pathlib import Path from loguru import logger +from typing import Optional from text_generation import server, utils @@ -13,6 +14,7 @@ @app.command() def serve( model_name: str, + revision: Optional[str] = None, sharded: bool = False, quantize: bool = False, uds_path: Path = "/tmp/text-generation", @@ -44,15 +46,16 @@ def serve( os.getenv("MASTER_PORT", None) is not None ), "MASTER_PORT must be set when sharded is True" - server.serve(model_name, sharded, quantize, uds_path) + server.serve(model_name, revision, sharded, quantize, uds_path) @app.command() def download_weights( model_name: str, + revision: Optional[str] = None, extension: str = ".safetensors", ): - utils.download_weights(model_name, extension) + utils.download_weights(model_name, revision, extension) if __name__ == "__main__": diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 41d73815..9309c887 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -1,11 +1,15 @@ import torch +from transformers import AutoConfig +from typing import Optional + from text_generation.models.model import Model from text_generation.models.causal_lm import CausalLM from text_generation.models.bloom import BLOOM, BLOOMSharded from text_generation.models.seq2seq_lm import Seq2SeqLM from text_generation.models.galactica import Galactica, GalacticaSharded from text_generation.models.santacoder import SantaCoder +from text_generation.models.gpt_neox import GPTNeox, GPTNeoxSharded __all__ = [ "Model", @@ -25,23 +29,32 @@ torch.backends.cudnn.allow_tf32 = True -def get_model(model_name: str, sharded: bool, quantize: bool) -> Model: - if model_name.startswith("bigscience/bloom"): +def get_model( + model_name: str, revision: Optional[str], sharded: bool, quantize: bool +) -> Model: + config = AutoConfig.from_pretrained(model_name) + + if config.model_type == "bloom": + if sharded: + return BLOOMSharded(model_name, revision, quantize=quantize) + else: + return BLOOM(model_name, revision, quantize=quantize) + elif config.model_type == "gpt_neox": if sharded: - return BLOOMSharded(model_name, quantize=quantize) + return GPTNeoxSharded(model_name, revision, quantize=quantize) else: - return BLOOM(model_name, quantize=quantize) + return GPTNeox(model_name, revision, quantize=quantize) elif model_name.startswith("facebook/galactica"): if sharded: - return GalacticaSharded(model_name, quantize=quantize) + return GalacticaSharded(model_name, revision, quantize=quantize) else: - return Galactica(model_name, quantize=quantize) + return Galactica(model_name, revision, quantize=quantize) elif "santacoder" in model_name: - return SantaCoder(model_name, quantize) + return SantaCoder(model_name, revision, quantize) else: if sharded: raise ValueError("sharded is not supported for AutoModel") try: - return CausalLM(model_name, quantize=quantize) + return CausalLM(model_name, revision, quantize=quantize) except Exception: - return Seq2SeqLM(model_name, quantize=quantize) + return Seq2SeqLM(model_name, revision, quantize=quantize) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 7708bb4a..4f55afc0 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -56,7 +56,9 @@ def batch_type(self) -> Type[CausalLMBatch]: class BLOOMSharded(BLOOM): - def __init__(self, model_name: str, quantize: bool = False): + def __init__( + self, model_name: str, revision: Optional[str] = None, quantize: bool = False + ): if not model_name.startswith("bigscience/bloom"): raise ValueError(f"Model {model_name} is not supported") @@ -69,19 +71,23 @@ def __init__(self, model_name: str, quantize: bool = False): device = torch.device("cpu") dtype = torch.float32 - tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + tokenizer = AutoTokenizer.from_pretrained( + model_name, revision=revision, padding_side="left" + ) config = AutoConfig.from_pretrained( - model_name, slow_but_exact=False, tp_parallel=True + model_name, revision=revision, slow_but_exact=False, tp_parallel=True ) config.pad_token_id = 3 # Only download weights for small models if self.master and model_name == "bigscience/bloom-560m": - download_weights(model_name, extension=".safetensors") + download_weights(model_name, revision=revision, extension=".safetensors") torch.distributed.barrier(group=self.process_group) - filenames = weight_files(model_name, extension=".safetensors") + filenames = weight_files( + model_name, revision=revision, extension=".safetensors" + ) if not filenames: raise ValueError("No safetensors weights found") diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 31996e06..4dc834b8 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -232,7 +232,7 @@ def __len__(self): class CausalLM(Model): - def __init__(self, model_name: str, quantize=False): + def __init__(self, model_name: str, revision: Optional[str] = None, quantize=False): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -243,9 +243,12 @@ def __init__(self, model_name: str, quantize=False): device = torch.device("cpu") dtype = torch.float32 - tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + tokenizer = AutoTokenizer.from_pretrained( + model_name, revision=revision, padding_side="left" + ) self.model = AutoModelForCausalLM.from_pretrained( model_name, + revision=revision, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, load_in_8bit=quantize, diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index 4722e1d8..be9b1699 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -148,7 +148,9 @@ def decode(self, generated_ids: List[int]) -> str: class GalacticaSharded(Galactica): - def __init__(self, model_name: str, quantize: bool = False): + def __init__( + self, model_name: str, revision: Optional[str] = None, quantize: bool = False + ): if not model_name.startswith("facebook/galactica"): raise ValueError(f"Model {model_name} is not supported") @@ -161,24 +163,23 @@ def __init__(self, model_name: str, quantize: bool = False): device = torch.device("cpu") dtype = torch.float32 - tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + tokenizer = AutoTokenizer.from_pretrained( + model_name, revision=revision, padding_side="left" + ) - config = AutoConfig.from_pretrained(model_name, tp_parallel=True) + config = AutoConfig.from_pretrained( + model_name, revision=revision, tp_parallel=True + ) tokenizer.pad_token_id = config.pad_token_id - # The flag below controls whether to allow TF32 on matmul. This flag defaults to False - # in PyTorch 1.12 and later. - torch.backends.cuda.matmul.allow_tf32 = True - - # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. - torch.backends.cudnn.allow_tf32 = True - # Only download weights for small models if self.master and model_name == "facebook/galactica-125m": - download_weights(model_name, extension=".safetensors") + download_weights(model_name, revision=revision, extension=".safetensors") torch.distributed.barrier(group=self.process_group) - filenames = weight_files(model_name, extension=".safetensors") + filenames = weight_files( + model_name, revision=revision, extension=".safetensors" + ) if not filenames: raise ValueError("No safetensors weights found") diff --git a/server/text_generation/models/gpt_neox.py b/server/text_generation/models/gpt_neox.py new file mode 100644 index 00000000..d901cae3 --- /dev/null +++ b/server/text_generation/models/gpt_neox.py @@ -0,0 +1,244 @@ +import torch +import torch.distributed + +from typing import List, Optional, Tuple + +from accelerate import init_empty_weights +from safetensors import safe_open +from transformers import ( + AutoTokenizer, + AutoModelForCausalLM, + AutoConfig, +) +from transformers.models.gpt_neox.parallel_layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, +) + +from text_generation.models import CausalLM +from text_generation.utils import ( + initialize_torch_distributed, + weight_files, + download_weights, +) + +HAS_BITS_AND_BYTES = True +try: + import bitsandbytes as bnb + from bitsandbytes.nn import Int8Params +except Exception as e: + HAS_BITS_AND_BYTES = False + + +class GPTNeox(CausalLM): + def forward( + self, input_ids, attention_mask, position_ids, past_key_values: Optional = None + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + """Overwrite forward to ignore position_ids""" + + # Model Forward + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + return outputs.logits, outputs.past_key_values + + +class GPTNeoxSharded(GPTNeox): + def __init__( + self, model_name: str, revision: Optional[str] = None, quantize: bool = False + ): + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + device = torch.device(f"cuda:{self.rank}") + dtype = torch.bfloat16 + else: + device = torch.device("cpu") + dtype = torch.float32 + + tokenizer = AutoTokenizer.from_pretrained( + model_name, revision=revision, padding_side="left" + ) + tokenizer.pad_token = tokenizer.eos_token + + config = AutoConfig.from_pretrained( + model_name, revision=revision, tp_parallel=True + ) + + # Only master download weights + if self.master: + download_weights(model_name, revision=revision, extension=".safetensors") + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files( + model_name, revision=revision, extension=".safetensors" + ) + if not filenames: + raise ValueError("No safetensors weights found") + + with init_empty_weights(): + model = AutoModelForCausalLM.from_config(config) + + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + quantize=quantize, + device=device, + rank=self.rank, + world_size=self.world_size, + ) + self.model = model.eval().to(dtype) + torch.distributed.barrier(group=self.process_group) + super(CausalLM, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @staticmethod + def load_weights( + model, + filenames: List[str], + quantize: bool, + device: torch.device, + rank: int, + world_size: int, + ): + parameters = dict(model.named_parameters()) + for file in filenames: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: + for name in f.keys(): + module_name, param_name = name.rsplit(".", 1) + module = model.get_submodule(module_name) + + current_parameter_tensor = parameters.get(name, None) + + slice_ = f.get_slice(name) + + if isinstance(module, TensorParallelColumnLinear): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif isinstance(module, TensorParallelRowLinear): + if param_name == "weight": + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + else: + tensor = slice_[:] + # XXX: Hack for Rowlinear to add the bias only once. + if rank != 0: + tensor = torch.zeros_like(tensor) + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif name == "embed_out.weight": + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + else: + try: + tensor = slice_[:] + except: + tensor = f.get_tensor(name) + + if ( + current_parameter_tensor is not None + and current_parameter_tensor.shape != tensor.shape + ): + raise ValueError( + f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" + ) + + tensor = tensor.contiguous() + + if quantize: + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine either because it is not installed " + "or you don't have a GPU.\n" + "You can install it with `pip install bitsandbytes`." + ) + + if ( + type(module) + in [TensorParallelRowLinear, TensorParallelColumnLinear] + and param_name == "weight" + ): + tensor = Int8Params( + tensor, + has_fp16_weights=False, + requires_grad=False, + ).to(device) + state = bnb.MatmulLtState() + state.threshold = 6.0 + state.has_fp16_weights = False + state.memory_efficient_backward = False + state.use_pool = True + state.CB = tensor.CB + state.SCB = tensor.SCB + tensor.CB = None + tensor.SCB = None + + def replace_linear(state): + def linear(input, weight, bias): + out = bnb.matmul( + input, + weight, + state=state, + threshold=state.threshold, + bias=bias, + ) + + if state.CB is not None: + # we converted 8-bit row major to turing/ampere format + # in the first inference pass + # we no longer need the row-major weight + del state.CB + weight.data = state.CxB + + return out + + return linear + + module.linear = replace_linear(state) + + else: + tensor = tensor.to(device) + + if current_parameter_tensor is not None: + module._parameters[param_name] = tensor + else: + module._buffers[param_name] = tensor + + def forward( + self, input_ids, attention_mask, position_ids, past_key_values: Optional = None + ): + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + # Logits are sharded, so we need to gather them + logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] + torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) + logits = torch.cat(logits, dim=2) + + return logits, outputs.past_key_values diff --git a/server/text_generation/models/santacoder.py b/server/text_generation/models/santacoder.py index 4b898ab9..6c1a250f 100644 --- a/server/text_generation/models/santacoder.py +++ b/server/text_generation/models/santacoder.py @@ -14,7 +14,7 @@ class SantaCoder(CausalLM): - def __init__(self, model_name: str, quantize=False): + def __init__(self, model_name: str, revision: Optional[str] = None, quantize=False): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -25,7 +25,9 @@ def __init__(self, model_name: str, quantize=False): device = torch.device("cpu") dtype = torch.float32 - tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + tokenizer = AutoTokenizer.from_pretrained( + model_name, revision=revision, padding_side="left" + ) tokenizer.add_special_tokens( { "additional_special_tokens": [ @@ -42,6 +44,7 @@ def __init__(self, model_name: str, quantize=False): self.model = ( AutoModelForCausalLM.from_pretrained( model_name, + revision=revision, torch_dtype=dtype, load_in_8bit=quantize, trust_remote_code=True, # required diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 6d5dc22e..29492dd7 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -289,7 +289,7 @@ def __len__(self): class Seq2SeqLM(Model): - def __init__(self, model_name: str, quantize=False): + def __init__(self, model_name: str, revision: Optional[str] = None, quantize=False): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -302,11 +302,14 @@ def __init__(self, model_name: str, quantize=False): self.model = AutoModelForSeq2SeqLM.from_pretrained( model_name, + revision=revision, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, load_in_8bit=quantize, ).eval() - tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left") + tokenizer = AutoTokenizer.from_pretrained( + model_name, revision=revision, padding_side="left" + ) tokenizer.bos_token_id = self.model.config.decoder_start_token_id super(Seq2SeqLM, self).__init__( diff --git a/server/text_generation/server.py b/server/text_generation/server.py index a2bad8a7..852deebf 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -6,7 +6,7 @@ from grpc_reflection.v1alpha import reflection from pathlib import Path -from typing import List +from typing import List, Optional from text_generation.cache import Cache from text_generation.interceptor import ExceptionInterceptor @@ -67,12 +67,14 @@ async def Decode(self, request, context): def serve( model_name: str, + revision: Optional[str], sharded: bool, quantize: bool, uds_path: Path, ): async def serve_inner( model_name: str, + revision: Optional[str], sharded: bool = False, quantize: bool = False, ): @@ -87,7 +89,7 @@ async def serve_inner( local_url = unix_socket_template.format(uds_path, 0) server_urls = [local_url] - model = get_model(model_name, sharded, quantize) + model = get_model(model_name, revision, sharded, quantize) server = aio.server(interceptors=[ExceptionInterceptor()]) generate_pb2_grpc.add_TextGenerationServiceServicer_to_server( @@ -107,4 +109,4 @@ async def serve_inner( logger.info("Signal received. Shutting down") await server.stop(0) - asyncio.run(serve_inner(model_name, sharded, quantize)) + asyncio.run(serve_inner(model_name, revision, sharded, quantize)) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index a2029911..62d60635 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -8,7 +8,9 @@ from concurrent.futures import ThreadPoolExecutor from functools import partial -from huggingface_hub import HfApi, hf_hub_download, try_to_load_from_cache +from pathlib import Path +from huggingface_hub import HfApi, hf_hub_download, _CACHED_NO_EXIST +from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE from huggingface_hub.utils import LocalEntryNotFoundError from tqdm import tqdm from typing import List, Optional, Tuple @@ -170,20 +172,62 @@ def initialize_torch_distributed(): return torch.distributed.distributed_c10d._get_default_group(), rank, world_size -def weight_hub_files(model_name, extension=".safetensors"): +def weight_hub_files(model_name, revision=None, extension=".safetensors"): """Get the safetensors filenames on the hub""" api = HfApi() - info = api.model_info(model_name) + info = api.model_info(model_name, revision=revision) filenames = [s.rfilename for s in info.siblings if s.rfilename.endswith(extension)] return filenames -def weight_files(model_name, extension=".safetensors"): +def try_to_load_from_cache(model_name, revision, filename): + """Try to load a file from the Hugging Face cache""" + if revision is None: + revision = "main" + + object_id = model_name.replace("/", "--") + repo_cache = Path(HUGGINGFACE_HUB_CACHE) / f"models--{object_id}" + + if not repo_cache.is_dir(): + # No cache for this model + return None + + refs_dir = repo_cache / "refs" + snapshots_dir = repo_cache / "snapshots" + no_exist_dir = repo_cache / ".no_exist" + + # Resolve refs (for instance to convert main to the associated commit sha) + if refs_dir.is_dir(): + revision_file = refs_dir / revision + if revision_file.exists(): + with revision_file.open() as f: + revision = f.read() + + # Check if file is cached as "no_exist" + if (no_exist_dir / revision / filename).is_file(): + return _CACHED_NO_EXIST + + # Check if revision folder exists + if not snapshots_dir.exists(): + return None + cached_shas = os.listdir(snapshots_dir) + if revision not in cached_shas: + # No cache for this revision and we won't try to return a random revision + return None + + # Check if file exists in cache + cached_file = snapshots_dir / revision / filename + return str(cached_file) if cached_file.is_file() else None + + +def weight_files(model_name, revision=None, extension=".safetensors"): """Get the local safetensors filenames""" - filenames = weight_hub_files(model_name, extension) + filenames = weight_hub_files(model_name, revision, extension) files = [] for filename in filenames: - cache_file = try_to_load_from_cache(model_name, filename=filename) + cache_file = try_to_load_from_cache( + model_name, revision=revision, filename=filename + ) if cache_file is None: raise LocalEntryNotFoundError( f"File {filename} of model {model_name} not found in " @@ -195,9 +239,9 @@ def weight_files(model_name, extension=".safetensors"): return files -def download_weights(model_name, extension=".safetensors"): +def download_weights(model_name, revision=None, extension=".safetensors"): """Download the safetensors files from the hub""" - filenames = weight_hub_files(model_name, extension) + filenames = weight_hub_files(model_name, revision, extension) download_function = partial( hf_hub_download, @@ -207,7 +251,8 @@ def download_weights(model_name, extension=".safetensors"): executor = ThreadPoolExecutor(max_workers=5) futures = [ - executor.submit(download_function, filename=filename) for filename in filenames + executor.submit(download_function, filename=filename, revision=revision) + for filename in filenames ] files = [ future.result() From 404ed7a1f6cfef02ac8ee71c934f87b056d3a06c Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 31 Jan 2023 20:14:05 +0100 Subject: [PATCH 063/793] feat(ci): Docker build and push (#46) --- .github/workflows/build.yaml | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..09b5b5f2 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,58 @@ +name: Build and push docker image to internal registry + +on: + workflow_dispatch: + push: + branches: + - 'main' + pull_request: + branches: + - 'main' + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + steps: + - name: Tailscale + uses: tailscale/github-action@v1 + with: + authkey: ${{ secrets.TAILSCALE_AUTHKEY }} + - name: Checkout repository + uses: actions/checkout@v3 + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to internal Container Registry + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.TAILSCALE_DOCKER_USERNAME }} + password: ${{ secrets.TAILSCALE_DOCKER_PASSWORD }} + registry: registry.internal.huggingface.tech + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4.3.0 + with: + flavor: | + latest=auto + images: | + ghcr.io/huggingface/text-generation-inference + registry.internal.huggingface.tech/api-inference/community/text-generation-inference + tags: | + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }} + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile + push: ${{ github.event_name != 'pull_request' }} + platforms: 'linux/amd64' + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=ghcr.io/huggingface/text-generation-inference:latest + cache-to: type=inline \ No newline at end of file From 2ad895a6cc530474cae7e24ace1e463018172d0e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 1 Feb 2023 14:43:59 +0100 Subject: [PATCH 064/793] feat(server): allow gpt-neox models with odd vocab sizes to be sharded (#48) --- README.md | 2 +- server/text_generation/models/gpt_neox.py | 42 ++++++++++++++--------- server/text_generation/utils.py | 2 +- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 67940ae8..d54a092a 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ to power Bloom, BloomZ and MT0-XXL api-inference widgets. - [MT0-XXL](https://huggingface.co/bigscience/mt0-xxl) - ~~[Galactica](https://huggingface.co/facebook/galactica-120b)~~ (deactivated) - [SantaCoder](https://huggingface.co/bigcode/santacoder) -- [GPT-Neox 20B](https://huggingface.co/EleutherAI/gpt-neox-20b): use `--revision refs/pr/13` +- [GPT-Neox 20B](https://huggingface.co/EleutherAI/gpt-neox-20b): use `--revision pr/13` Other models are supported on a best effort basis using: diff --git a/server/text_generation/models/gpt_neox.py b/server/text_generation/models/gpt_neox.py index d901cae3..a8f7f365 100644 --- a/server/text_generation/models/gpt_neox.py +++ b/server/text_generation/models/gpt_neox.py @@ -145,7 +145,7 @@ def load_weights( start = rank * block_size stop = (rank + 1) * block_size tensor = slice_[start:stop] - elif name == "embed_out.weight": + elif name == "embed_out.weight" and model.gpt_neox.tp_embeddings: size = slice_.get_shape()[0] block_size = size // world_size start = rank * block_size @@ -176,9 +176,9 @@ def load_weights( ) if ( - type(module) - in [TensorParallelRowLinear, TensorParallelColumnLinear] - and param_name == "weight" + type(module) + in [TensorParallelRowLinear, TensorParallelColumnLinear] + and param_name == "weight" ): tensor = Int8Params( tensor, @@ -229,16 +229,24 @@ def linear(input, weight, bias): def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): - outputs = self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - use_cache=True, - ) - - # Logits are sharded, so we need to gather them - logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] - torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) - logits = torch.cat(logits, dim=2) - - return logits, outputs.past_key_values + if self.model.gpt_neox.tp_embeddings: + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + # Logits are sharded, so we need to gather them + logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] + torch.distributed.all_gather( + logits, outputs.logits, group=self.process_group + ) + logits = torch.cat(logits, dim=2) + + return logits, outputs.past_key_values + # While the model itself is sharded, the embeddings might not as they might not be dividable by num-shard + else: + return super(GPTNeoxSharded, self).forward( + input_ids, attention_mask, position_ids, past_key_values + ) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 62d60635..3b07ef3f 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -91,7 +91,7 @@ def from_pb( top_p=pb.top_p, do_sample=pb.do_sample, seed=pb.seed, - device=str(device), + device=device, ) From 313194f6d75215bc0a8d1f78ef352dbc705a48a5 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 1 Feb 2023 15:58:42 +0100 Subject: [PATCH 065/793] feat(server): support repetition penalty (#47) --- README.md | 2 +- proto/generate.proto | 2 ++ router/src/lib.rs | 6 ++++++ router/src/server.rs | 1 + router/src/validation.rs | 7 +++++++ server/tests/conftest.py | 1 + server/text_generation/models/causal_lm.py | 2 +- server/text_generation/models/seq2seq_lm.py | 4 +++- server/text_generation/utils.py | 8 ++++++++ 9 files changed, 30 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d54a092a..d092781a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ to power Bloom, BloomZ and MT0-XXL api-inference widgets. - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading - 45ms per token generation for BLOOM with 8xA100 80GB -- Logits warpers (temperature scaling, topk ...) +- Logits warpers (temperature scaling, topk, repetition penalty ...) - Stop sequences - Log probabilities diff --git a/proto/generate.proto b/proto/generate.proto index 8f431c5c..098df9c5 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -38,6 +38,8 @@ message NextTokenChooserParameters { bool do_sample = 4; /// random seed for sampling uint64 seed = 5; + /// repetition penalty + float repetition_penalty = 6; } message StoppingCriteriaParameters { diff --git a/router/src/lib.rs b/router/src/lib.rs index beab7138..5b96485f 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -13,6 +13,8 @@ use validation::Validation; pub(crate) struct GenerateParameters { #[serde(default = "default_temperature")] pub temperature: f32, + #[serde(default = "default_repetition_penalty")] + pub repetition_penalty: f32, #[serde(default = "default_top_k")] pub top_k: i32, #[serde(default = "default_top_p")] @@ -32,6 +34,9 @@ pub(crate) struct GenerateParameters { fn default_temperature() -> f32 { 1.0 } +fn default_repetition_penalty() -> f32 { + 1.0 +} fn default_top_k() -> i32 { 0 @@ -52,6 +57,7 @@ fn default_max_new_tokens() -> u32 { fn default_parameters() -> GenerateParameters { GenerateParameters { temperature: default_temperature(), + repetition_penalty: default_repetition_penalty(), top_k: default_top_k(), top_p: default_top_p(), do_sample: default_do_sample(), diff --git a/router/src/server.rs b/router/src/server.rs index ef3782d6..c31ca6ce 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -33,6 +33,7 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json 1.0 { return Err(ValidationError::TopP); } @@ -146,6 +149,7 @@ fn validate( // Return ValidGenerateRequest let GenerateParameters { temperature, + repetition_penalty, top_k, top_p, do_sample, @@ -156,6 +160,7 @@ fn validate( let parameters = NextTokenChooserParameters { temperature, + repetition_penalty, top_k: top_k as u32, top_p, do_sample, @@ -195,6 +200,8 @@ pub(crate) struct ValidGenerateRequest { pub enum ValidationError { #[error("temperature must be strictly positive")] Temperature, + #[error("repetition_penalty must be strictly positive")] + RepetitionPenalty, #[error("top_p must be > 0.0 and <= 1.0")] TopP, #[error("top_k must be strictly positive")] diff --git a/server/tests/conftest.py b/server/tests/conftest.py index e0ed76b4..9fae8ee1 100644 --- a/server/tests/conftest.py +++ b/server/tests/conftest.py @@ -7,6 +7,7 @@ def default_pb_parameters(): return generate_pb2.NextTokenChooserParameters( temperature=1.0, + repetition_penalty=1.0, top_k=0, top_p=1.0, do_sample=False, diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 4dc834b8..994c57d5 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -336,7 +336,7 @@ def generate_token( all_input_ids, ) in enumerate(iterator): # Select next token - tokens, logprobs = next_token_chooser(all_input_ids, logits) + tokens, logprobs = next_token_chooser(all_input_ids.view(1, -1), logits) next_token_id = tokens[-1].view(1, 1) # Append next token to all tokens diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 29492dd7..1ae266d8 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -418,7 +418,9 @@ def generate_token( decoder_input_ids, ) in enumerate(iterator): # Select next token - next_token_id, logprobs = next_token_chooser(decoder_input_ids, logits) + next_token_id, logprobs = next_token_chooser( + decoder_input_ids.view(1, -1), logits + ) # Append next token to decoder tokens decoder_input_ids = torch.cat([decoder_input_ids, next_token_id]) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 3b07ef3f..91e6b7b7 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -17,6 +17,7 @@ from transformers import PreTrainedTokenizerBase from transformers.generation.logits_process import ( LogitsProcessorList, + RepetitionPenaltyLogitsProcessor, TemperatureLogitsWarper, TopPLogitsWarper, TopKLogitsWarper, @@ -48,6 +49,7 @@ class NextTokenChooser: def __init__( self, temperature=1.0, + repetition_penalty=1.0, top_k=None, top_p=None, do_sample=False, @@ -68,6 +70,9 @@ def __init__( if top_p is not None and top_p < 1.0: warpers.append(TopPLogitsWarper(top_p=top_p)) sampling = True + if repetition_penalty is not None and repetition_penalty != 1.0: + warpers.append(RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty)) + sampling = True self.warpers = warpers self.choice = Sampling(seed, device) if sampling else Greedy() @@ -75,8 +80,10 @@ def __init__( def __call__(self, input_ids, scores): # Warp logits scores = self.warpers(input_ids, scores) + # Compute logprobs logprobs = torch.log_softmax(scores, -1) + # Choose tokens next_ids = self.choice(scores) return next_ids, logprobs @@ -87,6 +94,7 @@ def from_pb( ) -> "NextTokenChooser": return NextTokenChooser( temperature=pb.temperature, + repetition_penalty=pb.repetition_penalty, top_k=pb.top_k, top_p=pb.top_p, do_sample=pb.do_sample, From 775115e3a571bc5d983852ce31c071a7734c1430 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 1 Feb 2023 16:22:10 +0100 Subject: [PATCH 066/793] feat(server): allow the server to use a local weight cache (#49) --- launcher/src/main.rs | 6 ++++++ server/text_generation/utils.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index bd449e28..20ec7faa 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -313,6 +313,12 @@ fn shard_manager( env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); }; + // If the WEIGHTS_CACHE_OVERRIDE env var is set, pass it to the shard + // Useful when running inside a HuggingFace Inference Endpoint + if let Ok(weights_cache_override) = env::var("WEIGHTS_CACHE_OVERRIDE") { + env.push(("WEIGHTS_CACHE_OVERRIDE".into(), weights_cache_override.into())); + }; + // If the CUDA_VISIBLE_DEVICES env var is set, pass it to the shard if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { env.push(("CUDA_VISIBLE_DEVICES".into(), cuda_visible_devices.into())); diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 91e6b7b7..8590f85c 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -25,6 +25,7 @@ from text_generation.pb import generate_pb2 +WEIGHTS_CACHE_OVERRIDE = os.getenv("WEIGHTS_CACHE_OVERRIDE", None) class Sampling: def __init__(self, seed: int, device: str = "cpu"): @@ -230,6 +231,9 @@ def try_to_load_from_cache(model_name, revision, filename): def weight_files(model_name, revision=None, extension=".safetensors"): """Get the local safetensors filenames""" + if WEIGHTS_CACHE_OVERRIDE is not None: + return list(Path(WEIGHTS_CACHE_OVERRIDE).glob(f"*{extension}")) + filenames = weight_hub_files(model_name, revision, extension) files = [] for filename in filenames: @@ -249,6 +253,9 @@ def weight_files(model_name, revision=None, extension=".safetensors"): def download_weights(model_name, revision=None, extension=".safetensors"): """Download the safetensors files from the hub""" + if WEIGHTS_CACHE_OVERRIDE is not None: + return list(Path(WEIGHTS_CACHE_OVERRIDE).glob(f"*{extension}")) + filenames = weight_hub_files(model_name, revision, extension) download_function = partial( From df227ac20d87cc99f39b690cabe074691b901420 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 2 Feb 2023 10:34:35 +0100 Subject: [PATCH 067/793] fix(server): allow greedy repetition penalty (#51) --- server/text_generation/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 8590f85c..8bbc7377 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -73,7 +73,6 @@ def __init__( sampling = True if repetition_penalty is not None and repetition_penalty != 1.0: warpers.append(RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty)) - sampling = True self.warpers = warpers self.choice = Sampling(seed, device) if sampling else Greedy() From 7b870e1e1837654bfc268748138ea81402c6afc7 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 2 Feb 2023 14:59:27 +0100 Subject: [PATCH 068/793] feat(router): use background task to manage request queue (#52) Co-authored-by: Nick Hill --- launcher/src/main.rs | 5 +- router/src/db.rs | 160 ------------------ router/src/infer.rs | 37 ++-- router/src/lib.rs | 6 +- router/src/queue.rs | 355 +++++++++++++++++++++++++++++++++++++++ router/src/validation.rs | 2 +- 6 files changed, 382 insertions(+), 183 deletions(-) delete mode 100644 router/src/db.rs create mode 100644 router/src/queue.rs diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 20ec7faa..dea6fcc8 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -316,7 +316,10 @@ fn shard_manager( // If the WEIGHTS_CACHE_OVERRIDE env var is set, pass it to the shard // Useful when running inside a HuggingFace Inference Endpoint if let Ok(weights_cache_override) = env::var("WEIGHTS_CACHE_OVERRIDE") { - env.push(("WEIGHTS_CACHE_OVERRIDE".into(), weights_cache_override.into())); + env.push(( + "WEIGHTS_CACHE_OVERRIDE".into(), + weights_cache_override.into(), + )); }; // If the CUDA_VISIBLE_DEVICES env var is set, pass it to the shard diff --git a/router/src/db.rs b/router/src/db.rs deleted file mode 100644 index 246e4d5d..00000000 --- a/router/src/db.rs +++ /dev/null @@ -1,160 +0,0 @@ -/// This code is massively inspired by Tokio mini-redis -use crate::infer::InferError; -use crate::infer::InferStreamResponse; -use crate::validation::ValidGenerateRequest; -use nohash_hasher::{BuildNoHashHasher, IntMap}; -use parking_lot::Mutex; -use std::collections::BTreeMap; -use std::sync::Arc; -use text_generation_client::{Batch, Request}; -use tokio::sync::mpsc::UnboundedSender; -use tokio::sync::OwnedSemaphorePermit; -use tokio::time::Instant; - -/// Database entry -#[derive(Debug)] -pub(crate) struct Entry { - /// Request - pub request: ValidGenerateRequest, - /// Response sender to communicate between the Infer struct and the batching_task - pub response_tx: UnboundedSender>, - /// Instant when this entry was created - pub time: Instant, - /// Instant when this entry was added to a batch - pub batch_time: Option, - /// Permit - pub _permit: OwnedSemaphorePermit, -} - -/// Request Database -#[derive(Debug, Clone)] -pub(crate) struct Db { - pub shared: Arc, -} - -/// Shared state -#[derive(Debug)] -pub struct Shared { - state: Mutex, -} - -/// Database State -#[derive(Debug)] -struct State { - /// Database entries organized in a BTreeMap to be able to iterate over them in order - entries: BTreeMap, - - /// Id of the next entry - next_id: u64, - - /// Id of the next batch - next_batch_id: u64, - - /// Start ID of the next batch. Used to iterate inside the entries BTreeMap - next_batch_start_id: u64, -} - -impl State { - /// Get the next requests - fn next_requests(&self, max_size: usize) -> Option<(Vec, Vec)> { - // Iterates for max_size over the BTreemap starting from next_batch_start_id - let mut requests = Vec::new(); - let mut ids = Vec::new(); - - for (id, entry) in self - .entries - // Start from next_batch_start_id - .range(self.next_batch_start_id..) - // Take max_size - .take(max_size) - { - requests.push(Request { - id: *id, - inputs: entry.request.inputs.clone(), - input_length: entry.request.input_length, - parameters: Some(entry.request.parameters.clone()), - stopping_parameters: Some(entry.request.stopping_parameters.clone()), - }); - - ids.push(*id); - } - - if requests.is_empty() { - None - } else { - Some((ids, requests)) - } - } -} - -impl Db { - pub(crate) fn new() -> Self { - // Shared state - let shared = Arc::new(Shared { - state: Mutex::new(State { - entries: BTreeMap::new(), - next_id: 0, - next_batch_id: 0, - next_batch_start_id: 0, - }), - }); - - Self { shared } - } - - /// Append an entry to the database - pub(crate) fn append(&self, entry: Entry) { - // Acquire lock - let mut state = self.shared.state.lock(); - - // Insert entry - let id = state.next_id; - state.next_id += 1; - state.entries.insert(id, entry); - } - - // Get the next batch - pub(crate) fn next_batch( - &self, - min_size: Option, - max_size: usize, - ) -> Option<(IntMap, Batch)> { - // Acquire lock - let mut state = self.shared.state.lock(); - - // Get requests from the database - if let Some((ids, requests)) = state.next_requests(max_size) { - if let Some(min_size) = min_size { - // If min_size is set, only return a batch if there are enough requests - if requests.len() < min_size { - return None; - } - } - // Batch size - let size = requests.len(); - - let mut entries = IntMap::with_capacity_and_hasher(size, BuildNoHashHasher::default()); - ids.iter().for_each(|id| { - // Remove entry from db - let mut entry = state.entries.remove(id).unwrap(); - // Set batch_time - entry.batch_time = Some(Instant::now()); - // Insert in entries IntMap - entries.insert(*id, entry); - }); - - let batch = Batch { - id: state.next_batch_id, - requests, - size: size as u32, - }; - // Update next_batch_start_id to the last id in the batch + 1 - state.next_batch_start_id = ids.last().unwrap() + 1; - // Increment batch id - state.next_batch_id += 1; - - return Some((entries, batch)); - } - None - } -} diff --git a/router/src/infer.rs b/router/src/infer.rs index 23e84265..3661b0e0 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -1,7 +1,7 @@ /// Batching and inference logic use crate::validation::{Validation, ValidationError}; use crate::GenerateRequest; -use crate::{Db, Entry, Token}; +use crate::{Entry, Queue, Token}; use nohash_hasher::IntMap; use std::future::Future; use std::sync::Arc; @@ -20,8 +20,8 @@ use tracing::instrument; pub struct Infer { /// Validation validation: Validation, - /// Request database - db: Db, + /// Request queue + queue: Queue, /// Shared state shared: Arc, /// Inference limit @@ -43,7 +43,7 @@ impl Infer { max_concurrent_requests: usize, ) -> Self { // Infer shared state - let db = Db::new(); + let queue = Queue::new(); let shared = Arc::new(Shared { batching_task: Notify::new(), }); @@ -53,7 +53,7 @@ impl Infer { client, max_batch_size, max_waiting_tokens, - db.clone(), + queue.clone(), shared.clone(), )); @@ -62,13 +62,13 @@ impl Infer { Self { validation, - db, + queue, shared, limit_concurrent_requests: semaphore, } } - /// Add a new request to the database and return a stream of InferStreamResponse + /// Add a new request to the queue and return a stream of InferStreamResponse pub(crate) async fn generate_stream( &self, request: GenerateRequest, @@ -83,8 +83,8 @@ impl Infer { // MPSC channel to communicate with the background batching task let (response_tx, response_rx) = mpsc::unbounded_channel(); - // Append the request to the database - self.db.append(Entry { + // Append the request to the queue + self.queue.append(Entry { request: valid_request, response_tx, time: Instant::now(), @@ -92,7 +92,7 @@ impl Infer { _permit: permit, }); - // Notify the background task that we have a new entry in the database that needs + // Notify the background task that we have a new entry in the queue that needs // to be batched self.shared.batching_task.notify_one(); @@ -100,7 +100,7 @@ impl Infer { Ok(UnboundedReceiverStream::new(response_rx)) } - /// Add a new request to the database and return a InferResponse + /// Add a new request to the queue and return a InferResponse pub(crate) async fn generate( &self, request: GenerateRequest, @@ -169,12 +169,12 @@ impl Infer { /// Will be launched in a background Tokio task /// /// Batches requests and sends them to the inference server -#[instrument(skip(client, db, shared))] +#[instrument(skip(client, queue, shared))] async fn batching_task( mut client: ShardedClient, max_batch_size: usize, max_waiting_tokens: usize, - db: Db, + queue: Queue, shared: Arc, ) { // Minimum batch size after which we try to add more requests @@ -185,10 +185,10 @@ async fn batching_task( // Wait for a notification from the Infer struct shared.batching_task.notified().await; - // Get the next batch from the DB + // Get the next batch from the queue // This batch might be smaller than the maximum batch size if there are not enough requests - // waiting in the DB - while let Some((mut entries, batch)) = db.next_batch(None, max_batch_size) { + // waiting in the queue + while let Some((mut entries, batch)) = queue.next_batch(None, max_batch_size).await { let mut cached_batch = wrap_future(client.prefill(batch), &mut entries).await; let mut waiting_tokens = 1; @@ -210,8 +210,9 @@ async fn batching_task( }; // Try to get a new batch - if let Some((mut new_entries, new_batch)) = - db.next_batch(min_size, max_batch_size - batch_size as usize) + if let Some((mut new_entries, new_batch)) = queue + .next_batch(min_size, max_batch_size - batch_size as usize) + .await { // Generate one token for this new batch to have the attention past in cache let new_cached_batch = diff --git a/router/src/lib.rs b/router/src/lib.rs index 5b96485f..c6ac2022 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,11 +1,11 @@ -/// Text Generation Inference Webserver -mod db; mod infer; +/// Text Generation Inference Webserver +mod queue; pub mod server; mod validation; -use db::{Db, Entry}; use infer::Infer; +use queue::{Entry, Queue}; use serde::{Deserialize, Serialize}; use validation::Validation; diff --git a/router/src/queue.rs b/router/src/queue.rs new file mode 100644 index 00000000..2aaf93b1 --- /dev/null +++ b/router/src/queue.rs @@ -0,0 +1,355 @@ +use crate::infer::InferError; +use crate::infer::InferStreamResponse; +use crate::validation::ValidGenerateRequest; +use nohash_hasher::{BuildNoHashHasher, IntMap}; +use std::cmp::min; +use text_generation_client::{Batch, Request}; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio::sync::{mpsc, oneshot, OwnedSemaphorePermit}; +use tokio::time::Instant; + +/// Queue entry +#[derive(Debug)] +pub(crate) struct Entry { + /// Request + pub request: ValidGenerateRequest, + /// Response sender to communicate between the Infer struct and the batching_task + pub response_tx: UnboundedSender>, + /// Instant when this entry was created + pub time: Instant, + /// Instant when this entry was added to a batch + pub batch_time: Option, + /// Permit + pub _permit: OwnedSemaphorePermit, +} + +/// Request Queue +#[derive(Debug, Clone)] +pub(crate) struct Queue { + /// Channel to communicate with the background queue task + queue_sender: UnboundedSender, +} + +impl Queue { + pub(crate) fn new() -> Self { + // Create channel + let (queue_sender, queue_receiver) = mpsc::unbounded_channel(); + + // Launch background queue task + tokio::spawn(queue_task(queue_receiver)); + + Self { queue_sender } + } + + /// Append an entry to the queue + pub(crate) fn append(&self, entry: Entry) { + // Send append command to the background task managing the state + // Unwrap is safe here + self.queue_sender.send(QueueCommand::Append(entry)).unwrap(); + } + + // Get the next batch + pub(crate) async fn next_batch( + &self, + min_size: Option, + max_size: usize, + ) -> Option { + // Create response channel + let (response_sender, response_receiver) = oneshot::channel(); + // Send next batch command to the background task managing the state + // Unwrap is safe here + self.queue_sender + .send(QueueCommand::NextBatch { + min_size, + max_size, + response_sender, + }) + .unwrap(); + // Await on response channel + // Unwrap is safe here + response_receiver.await.unwrap() + } +} + +// Background task responsible of the queue state +async fn queue_task(mut receiver: UnboundedReceiver) { + let mut state = State::new(); + + while let Some(cmd) = receiver.recv().await { + match cmd { + QueueCommand::Append(entry) => state.append(entry), + QueueCommand::NextBatch { + min_size, + max_size, + response_sender, + } => { + let next_batch = state.next_batch(min_size, max_size); + response_sender.send(next_batch).unwrap_or(()); + } + } + } +} + +/// Queue State +#[derive(Debug)] +struct State { + /// Queue entries organized in a Vec + entries: Vec<(u64, Entry)>, + + /// Id of the next entry + next_id: u64, + + /// Id of the next batch + next_batch_id: u64, +} + +impl State { + fn new() -> Self { + Self { + entries: Vec::with_capacity(128), + next_id: 0, + next_batch_id: 0, + } + } + + /// Append an entry to the queue + fn append(&mut self, entry: Entry) { + self.entries.push((self.next_id, entry)); + self.next_id += 1; + } + + // Get the next batch + fn next_batch(&mut self, min_size: Option, max_size: usize) -> Option { + if self.entries.is_empty() { + return None; + } + + // Check if we have enough entries + if let Some(min_size) = min_size { + if self.entries.len() < min_size { + return None; + } + } + + let next_batch_size = min(self.entries.len(), max_size); + + let mut batch_requests = Vec::with_capacity(next_batch_size); + let mut batch_entries = + IntMap::with_capacity_and_hasher(next_batch_size, BuildNoHashHasher::default()); + + // Drain next_batch_size entries + self.entries + .drain(..next_batch_size) + .for_each(|(id, mut entry)| { + batch_requests.push(Request { + id, + inputs: entry.request.inputs.clone(), + input_length: entry.request.input_length, + parameters: Some(entry.request.parameters.clone()), + stopping_parameters: Some(entry.request.stopping_parameters.clone()), + }); + // Set batch_time + entry.batch_time = Some(Instant::now()); + // Insert in batch_entries IntMap + batch_entries.insert(id, entry); + }); + + let batch = Batch { + id: self.next_batch_id, + requests: batch_requests, + size: next_batch_size as u32, + }; + // Increment batch id + self.next_batch_id += 1; + + Some((batch_entries, batch)) + } +} + +type NextBatch = (IntMap, Batch); + +#[derive(Debug)] +enum QueueCommand { + Append(Entry), + NextBatch { + min_size: Option, + max_size: usize, + response_sender: oneshot::Sender>, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParameters}; + use tokio::sync::{mpsc, Semaphore}; + + fn default_entry() -> Entry { + let semaphore = Arc::new(Semaphore::new(1)); + let (response_tx, _) = mpsc::unbounded_channel(); + let permit = semaphore.try_acquire_owned().unwrap(); + + Entry { + request: ValidGenerateRequest { + inputs: "".to_string(), + input_length: 0, + parameters: NextTokenChooserParameters { + temperature: 0.0, + top_k: 0, + top_p: 0.0, + do_sample: false, + seed: 0, + repetition_penalty: 0.0, + }, + stopping_parameters: StoppingCriteriaParameters { + max_new_tokens: 0, + stop_sequences: vec![], + }, + }, + response_tx, + time: Instant::now(), + batch_time: None, + _permit: permit, + } + } + + #[test] + fn test_append() { + let mut state = State::new(); + let entry = default_entry(); + + assert_eq!(state.next_id, 0); + assert_eq!(state.entries.len(), 0); + + state.append(entry); + + assert_eq!(state.next_id, 1); + assert_eq!(state.entries.len(), 1); + let (id, _) = state.entries.remove(0); + assert_eq!(id, 0); + } + + #[test] + fn test_next_batch_empty() { + let mut state = State::new(); + + assert!(state.next_batch(None, 1).is_none()); + assert!(state.next_batch(Some(1), 1).is_none()); + } + + #[test] + fn test_next_batch_min_size() { + let mut state = State::new(); + state.append(default_entry()); + state.append(default_entry()); + + let (entries, batch) = state.next_batch(None, 2).unwrap(); + assert_eq!(entries.len(), 2); + assert!(entries.contains_key(&0)); + assert!(entries.contains_key(&1)); + assert!(entries.get(&0).unwrap().batch_time.is_some()); + assert!(entries.get(&1).unwrap().batch_time.is_some()); + assert_eq!(batch.id, 0); + assert_eq!(batch.size, 2); + + assert_eq!(state.next_id, 2); + assert_eq!(state.entries.len(), 0); + assert_eq!(state.next_batch_id, 1); + + state.append(default_entry()); + + assert!(state.next_batch(Some(2), 2).is_none()); + + assert_eq!(state.next_id, 3); + assert_eq!(state.entries.len(), 1); + let (id, _) = state.entries.remove(0); + assert_eq!(id, 2); + } + + #[test] + fn test_next_batch_max_size() { + let mut state = State::new(); + state.append(default_entry()); + state.append(default_entry()); + + let (entries, batch) = state.next_batch(None, 1).unwrap(); + assert_eq!(entries.len(), 1); + assert!(entries.contains_key(&0)); + assert_eq!(batch.id, 0); + assert_eq!(batch.size, 1); + + assert_eq!(state.next_id, 2); + assert_eq!(state.entries.len(), 1); + assert_eq!(state.next_batch_id, 1); + + state.append(default_entry()); + + let (entries, batch) = state.next_batch(None, 3).unwrap(); + assert_eq!(entries.len(), 2); + assert!(entries.contains_key(&1)); + assert!(entries.contains_key(&2)); + assert_eq!(batch.id, 1); + assert_eq!(batch.size, 2); + + assert_eq!(state.next_id, 3); + assert_eq!(state.entries.len(), 0); + assert_eq!(state.next_batch_id, 2); + } + + #[tokio::test] + async fn test_queue_append() { + let queue = Queue::new(); + queue.append(default_entry()); + } + + #[tokio::test] + async fn test_queue_next_batch_empty() { + let queue = Queue::new(); + + assert!(queue.next_batch(None, 1).await.is_none()); + assert!(queue.next_batch(Some(1), 1).await.is_none()); + } + + #[tokio::test] + async fn test_queue_next_batch_min_size() { + let queue = Queue::new(); + queue.append(default_entry()); + queue.append(default_entry()); + + let (entries, batch) = queue.next_batch(None, 2).await.unwrap(); + assert_eq!(entries.len(), 2); + assert!(entries.contains_key(&0)); + assert!(entries.contains_key(&1)); + assert!(entries.get(&0).unwrap().batch_time.is_some()); + assert!(entries.get(&1).unwrap().batch_time.is_some()); + assert_eq!(batch.id, 0); + assert_eq!(batch.size, 2); + + queue.append(default_entry()); + + assert!(queue.next_batch(Some(2), 2).await.is_none()); + } + + #[tokio::test] + async fn test_queue_next_batch_max_size() { + let queue = Queue::new(); + queue.append(default_entry()); + queue.append(default_entry()); + + let (entries, batch) = queue.next_batch(None, 1).await.unwrap(); + assert_eq!(entries.len(), 1); + assert!(entries.contains_key(&0)); + assert_eq!(batch.id, 0); + assert_eq!(batch.size, 1); + + queue.append(default_entry()); + + let (entries, batch) = queue.next_batch(None, 3).await.unwrap(); + assert_eq!(entries.len(), 2); + assert!(entries.contains_key(&1)); + assert!(entries.contains_key(&2)); + assert_eq!(batch.id, 1); + assert_eq!(batch.size, 2); + } +} diff --git a/router/src/validation.rs b/router/src/validation.rs index ddb534d3..09220823 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -19,7 +19,7 @@ pub struct Validation { impl Validation { pub(crate) fn new(workers: usize, tokenizer: Tokenizer, max_input_length: usize) -> Self { - // Crate channel + // Create channel let (validation_sender, validation_receiver) = mpsc::channel(128); // Launch background validation task From b1482d90488b0e92a0adc06d47a603b5d837442c Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 2 Feb 2023 15:02:04 +0100 Subject: [PATCH 069/793] breaking(router): modify /generate API to only return generated text (#50) @njhill, @yk FYI generated_text was concatenated to the user prompt for legacy reason. We want to remove this behaviour as we don't think it is useful and even detrimonial to usability. We also remove the unused Vec. --- launcher/tests/bloom_560m.json | 2 +- launcher/tests/integration_tests.rs | 4 ++-- router/src/lib.rs | 3 ++- router/src/server.rs | 4 ++-- server/tests/models/test_bloom.py | 12 ++++++------ server/tests/models/test_causal_lm.py | 12 ++++++------ server/tests/models/test_santacoder.py | 4 ++-- server/text_generation/models/__init__.py | 2 +- server/text_generation/models/causal_lm.py | 4 +--- 9 files changed, 23 insertions(+), 24 deletions(-) diff --git a/launcher/tests/bloom_560m.json b/launcher/tests/bloom_560m.json index a81d1982..3a0a3d99 100644 --- a/launcher/tests/bloom_560m.json +++ b/launcher/tests/bloom_560m.json @@ -118,6 +118,6 @@ ] ] }, - "generated_text": "Test request.get(\"action\");\n if (action == null) {\n throw new RuntimeException" + "generated_text": ".get(\"action\");\n if (action == null) {\n throw new RuntimeException" } ] \ No newline at end of file diff --git a/launcher/tests/integration_tests.rs b/launcher/tests/integration_tests.rs index 6b270659..4f699f69 100644 --- a/launcher/tests/integration_tests.rs +++ b/launcher/tests/integration_tests.rs @@ -97,8 +97,8 @@ fn test_model( launcher.terminate().unwrap(); launcher.wait().unwrap(); - let mut results: Vec = res.unwrap().json().unwrap(); - results.pop().unwrap() + let result: GeneratedText = res.unwrap().json().unwrap(); + result } fn read_json(name: &str) -> GeneratedText { diff --git a/router/src/lib.rs b/router/src/lib.rs index c6ac2022..542a8f78 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,5 +1,6 @@ -mod infer; /// Text Generation Inference Webserver + +mod infer; mod queue; pub mod server; mod validation; diff --git a/router/src/server.rs b/router/src/server.rs index c31ca6ce..f79644e3 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -125,10 +125,10 @@ async fn generate( tracing::info!("Output: {}", response.generated_text.text); // Send response - let response = vec![GenerateResponse { + let response = GenerateResponse { generated_text: response.generated_text.text, details, - }]; + }; Ok((headers, Json(response))) } diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 9f96efc3..871c0da0 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -141,7 +141,7 @@ def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch) assert len(generations) == 1 assert ( generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTestTest" + == "TestTestTestTestTestTestTestTestTestTest" ) assert generations[0].request_id == default_bloom_batch.requests[0].id assert ( @@ -165,7 +165,7 @@ def test_causal_lm_generate_token_completion_multi( assert next_batch is not None assert len(generations) == 2 - assert generations[1].generated_text.text == "TestTestTestTestTestTest" + assert generations[1].generated_text.text == "TestTestTestTestTest" assert ( generations[1].request_id == default_multi_requests_bloom_batch.requests[1].id ) @@ -188,7 +188,7 @@ def test_causal_lm_generate_token_completion_multi( assert len(generations) == 1 assert ( generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTestTest" + == "TestTestTestTestTestTestTestTestTestTest" ) assert ( generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id @@ -261,7 +261,7 @@ def test_batch_concatenate( assert next_batch is not None assert len(generations) == 3 - assert generations[2].generated_text.text == "TestTestTestTestTestTest" + assert generations[2].generated_text.text == "TestTestTestTestTest" assert ( generations[2].request_id == default_multi_requests_bloom_batch.requests[1].id ) @@ -284,7 +284,7 @@ def test_batch_concatenate( assert len(generations) == 2 assert ( generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTestTest" + == "TestTestTestTestTestTestTestTestTestTest" ) assert generations[0].request_id == default_bloom_batch.requests[0].id assert ( @@ -307,7 +307,7 @@ def test_batch_concatenate( assert len(generations) == 1 assert ( generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTestTest" + == "TestTestTestTestTestTestTestTestTestTest" ) assert ( generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index f9762b30..6a822815 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -138,7 +138,7 @@ def test_causal_lm_generate_token_completion( assert next_batch is None assert len(generations) == 1 - assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert generations[0].generated_text.text == ".java:784) at net.minecraft." assert generations[0].request_id == default_causal_lm_batch.requests[0].id assert ( generations[0].generated_text.generated_tokens @@ -161,7 +161,7 @@ def test_causal_lm_generate_token_completion_multi( assert next_batch is not None assert len(generations) == 2 - assert generations[1].generated_text.text == "Test.java:784)" + assert generations[1].generated_text.text == ".java:784)" assert ( generations[1].request_id == default_multi_requests_causal_lm_batch.requests[1].id @@ -183,7 +183,7 @@ def test_causal_lm_generate_token_completion_multi( assert next_batch is None assert len(generations) == 1 - assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert generations[0].generated_text.text == ".java:784) at net.minecraft." assert ( generations[0].request_id == default_multi_requests_causal_lm_batch.requests[0].id @@ -255,7 +255,7 @@ def test_batch_concatenate( assert next_batch is not None assert len(generations) == 3 - assert generations[2].generated_text.text == "Test.java:784)" + assert generations[2].generated_text.text == ".java:784)" assert ( generations[2].request_id == default_multi_requests_causal_lm_batch.requests[1].id @@ -277,7 +277,7 @@ def test_batch_concatenate( assert next_batch is not None assert len(generations) == 2 - assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert generations[0].generated_text.text == ".java:784) at net.minecraft." assert generations[0].request_id == default_causal_lm_batch.requests[0].id assert ( generations[0].generated_text.generated_tokens @@ -297,7 +297,7 @@ def test_batch_concatenate( assert next_batch is None assert len(generations) == 1 - assert generations[0].generated_text.text == "Test.java:784) at net.minecraft." + assert generations[0].generated_text.text == ".java:784) at net.minecraft." assert ( generations[0].request_id == default_multi_requests_causal_lm_batch.requests[0].id diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index 1b69477d..1596e413 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -57,7 +57,7 @@ def test_santacoder_generate_token_completion(default_santacoder, default_pb_bat assert next_batch is None assert len(generations) == 1 - assert generations[0].generated_text.text == "def test_get_all_users_with_" + assert generations[0].generated_text.text == " test_get_all_users_with_" assert generations[0].request_id == batch.requests[0].id assert ( generations[0].generated_text.generated_tokens @@ -84,7 +84,7 @@ def test_fim_santacoder_generate_token_completion( assert len(generations) == 1 assert ( generations[0].generated_text.text - == """defworldineProperty(exports, "__esModule", { value""" + == """ineProperty(exports, "__esModule", { value""" ) assert generations[0].request_id == batch.requests[0].id assert ( diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 9309c887..15d8e97e 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -32,7 +32,7 @@ def get_model( model_name: str, revision: Optional[str], sharded: bool, quantize: bool ) -> Model: - config = AutoConfig.from_pretrained(model_name) + config = AutoConfig.from_pretrained(model_name, revision=revision) if config.model_type == "bloom": if sharded: diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 994c57d5..1d1945cd 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -360,11 +360,9 @@ def generate_token( if stop: # Decode generated tokens - generated_text = self.decode( + output_text = self.decode( all_input_ids[-stopping_criteria.current_tokens :, 0] ) - output_text = request.inputs + generated_text - # Get seed if isinstance(next_token_chooser.choice, Sampling): seed = next_token_chooser.choice.seed From 20c3c5940c6af1ceb50a8b4c713443690a148190 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 3 Feb 2023 12:43:37 +0100 Subject: [PATCH 070/793] feat(router): refactor API and add openAPI schemas (#53) --- .github/workflows/build.yaml | 4 + Cargo.lock | 189 ++++++++- Cargo.toml | 3 - Dockerfile | 10 +- Makefile | 14 +- README.md | 109 +++-- aml/deployment.yaml | 4 +- assets/architecture.jpg | Bin 135442 -> 342347 bytes docs/index.html | 30 ++ docs/openapi.json | 446 ++++++++++++++++++++ launcher/Cargo.toml | 2 +- launcher/src/main.rs | 14 +- launcher/tests/bloom_560m.json | 245 ++++++----- launcher/tests/integration_tests.rs | 35 +- launcher/tests/mt0_base.json | 235 +++++------ proto/generate.proto | 8 +- router/Cargo.toml | 6 +- router/client/Cargo.toml | 2 +- router/client/src/lib.rs | 4 +- router/src/infer.rs | 12 +- router/src/lib.rs | 124 ++++-- router/src/server.rs | 112 ++++- router/src/validation.rs | 63 ++- server/README.md | 4 +- server/pyproject.toml | 4 +- server/tests/models/test_bloom.py | 12 +- server/tests/test_utils.py | 7 +- server/text_generation/cli.py | 8 +- server/text_generation/models/__init__.py | 26 +- server/text_generation/models/bloom.py | 18 +- server/text_generation/models/causal_lm.py | 6 +- server/text_generation/models/galactica.py | 18 +- server/text_generation/models/gpt_neox.py | 12 +- server/text_generation/models/santacoder.py | 6 +- server/text_generation/models/seq2seq_lm.py | 6 +- server/text_generation/models/types.py | 3 +- server/text_generation/server.py | 8 +- server/text_generation/utils.py | 32 +- 38 files changed, 1350 insertions(+), 491 deletions(-) create mode 100644 docs/index.html create mode 100644 docs/openapi.json diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 09b5b5f2..d89f0982 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,6 +5,8 @@ on: push: branches: - 'main' + tags: + - 'v*' pull_request: branches: - 'main' @@ -43,6 +45,8 @@ jobs: ghcr.io/huggingface/text-generation-inference registry.internal.huggingface.tech/api-inference/community/text-generation-inference tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }} - name: Build and push Docker image diff --git a/Cargo.lock b/Cargo.lock index 5b671fa8..f686bdb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,9 +83,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.17" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43" +checksum = "e5694b64066a2459918d8074c2ce0d5a88f409431994c2356617c8ae0c4721fc" dependencies = [ "async-trait", "axum-core", @@ -101,8 +101,10 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", + "rustversion", "serde", "serde_json", + "serde_path_to_error", "serde_urlencoded", "sync_wrapper", "tokio", @@ -114,9 +116,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.2.9" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc" +checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" dependencies = [ "async-trait", "bytes", @@ -124,6 +126,7 @@ dependencies = [ "http", "http-body", "mime", + "rustversion", "tower-layer", "tower-service", ] @@ -207,7 +210,7 @@ dependencies = [ "tar", "tempfile", "thiserror", - "zip", + "zip 0.5.13", "zip-extensions", ] @@ -465,6 +468,15 @@ dependencies = [ "dirs-sys", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-sys" version = "0.3.7" @@ -867,6 +879,7 @@ checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", "hashbrown", + "serde", ] [[package]] @@ -999,9 +1012,9 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" [[package]] name = "matchit" -version = "0.5.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" [[package]] name = "memchr" @@ -1024,6 +1037,16 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1552,12 +1575,62 @@ dependencies = [ "winreg", ] +[[package]] +name = "rust-embed" +version = "6.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "283ffe2f866869428c92e0d61c2f35dfb4355293cdfdc48f49e895c15f1333d1" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "6.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31ab23d42d71fb9be1b643fe6765d292c5e14d46912d13f3ae2815ca048ea04d" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "shellexpand", + "syn", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1669d81dfabd1b5f8e2856b8bbe146c6192b0ba22162edc738ac0a5de18f054" +dependencies = [ + "sha2", + "walkdir", +] + +[[package]] +name = "rustversion" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" + [[package]] name = "ryu" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.20" @@ -1628,6 +1701,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1660,6 +1742,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shellexpand" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" +dependencies = [ + "dirs 4.0.0", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -1797,7 +1888,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.1.0" +version = "0.2.0" dependencies = [ "futures", "prost", @@ -1812,7 +1903,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.1.0" +version = "0.2.0" dependencies = [ "clap 4.0.22", "ctrlc", @@ -1827,7 +1918,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.1.0" +version = "0.2.0" dependencies = [ "async-stream", "axum", @@ -1845,6 +1936,8 @@ dependencies = [ "tokio-stream", "tracing", "tracing-subscriber", + "utoipa", + "utoipa-swagger-ui", ] [[package]] @@ -1921,7 +2014,7 @@ dependencies = [ "cached-path", "clap 2.34.0", "derive_builder", - "dirs", + "dirs 3.0.2", "esaxx-rs", "getrandom", "indicatif 0.15.0", @@ -2234,6 +2327,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.8" @@ -2293,6 +2395,46 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utoipa" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3920fa753064b1be7842bea26175ffa0dfc4a8f30bcb52b8ff03fddf8889914c" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "720298fac6efca20df9e457e67a1eab41a20d1c3101380b5c4dca1ca60ae0062" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae3d4f4da6408f0f20ff58196ed619c94306ab32635aeca3d3fa0768c0bd0de2" +dependencies = [ + "axum", + "mime_guess", + "regex", + "rust-embed", + "serde", + "serde_json", + "utoipa", + "zip 0.6.4", +] + [[package]] name = "valuable" version = "0.1.0" @@ -2317,6 +2459,17 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" @@ -2589,11 +2742,23 @@ dependencies = [ "time", ] +[[package]] +name = "zip" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", + "flate2", +] + [[package]] name = "zip-extensions" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a64c3c977bc3434ce2d4bcea8ad3c644672de0f2c402b72b9171ca80a8885d14" dependencies = [ - "zip", + "zip 0.5.13", ] diff --git a/Cargo.toml b/Cargo.toml index 684a5da4..3720af32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,9 +4,6 @@ members = [ "router/client", "launcher" ] -exclude = [ - "server/safetensors", -] [profile.release] debug = 1 diff --git a/Dockerfile b/Dockerfile index 99fe6dde..93846d77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,21 +26,18 @@ FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ DEBIAN_FRONTEND=noninteractive \ - MODEL_BASE_PATH=/data \ + HUGGINGFACE_HUB_CACHE=/data \ MODEL_ID=bigscience/bloom-560m \ QUANTIZE=false \ - NUM_GPUS=1 \ + NUM_SHARD=1 \ SAFETENSORS_FAST_GPU=1 \ PORT=80 \ - CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ NCCL_ASYNC_ERROR_HANDLING=1 \ CUDA_HOME=/usr/local/cuda \ LD_LIBRARY_PATH="/opt/miniconda/envs/text-generation/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH" \ CONDA_DEFAULT_ENV=text-generation \ PATH=$PATH:/opt/miniconda/envs/text-generation/bin:/opt/miniconda/bin:/usr/local/cuda/bin -SHELL ["/bin/bash", "-c"] - RUN apt-get update && apt-get install -y unzip curl libssl-dev && rm -rf /var/lib/apt/lists/* RUN cd ~ && \ @@ -71,4 +68,5 @@ COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/loca # Install launcher COPY --from=launcher-builder /usr/local/cargo/bin/text-generation-launcher /usr/local/bin/text-generation-launcher -CMD HUGGINGFACE_HUB_CACHE=$MODEL_BASE_PATH text-generation-launcher --num-shard $NUM_GPUS --model-name $MODEL_ID --json-output \ No newline at end of file +ENTRYPOINT ["text-generation-launcher"] +CMD ["--json-output"] \ No newline at end of file diff --git a/Makefile b/Makefile index d427ff87..08217730 100644 --- a/Makefile +++ b/Makefile @@ -15,17 +15,23 @@ server-dev: router-dev: cd router && cargo run +integration-tests: install-router install-launcher + cargo test + +python-tests: + cd server && pytest tests + run-bloom-560m: - text-generation-launcher --model-name bigscience/bloom-560m --num-shard 2 + text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 run-bloom-560m-quantize: - text-generation-launcher --model-name bigscience/bloom-560m --num-shard 2 --quantize + text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --quantize download-bloom: text-generation-server download-weights bigscience/bloom run-bloom: - text-generation-launcher --model-name bigscience/bloom --num-shard 8 + text-generation-launcher --model-id bigscience/bloom --num-shard 8 run-bloom-quantize: - text-generation-launcher --model-name bigscience/bloom --num-shard 8 --quantize \ No newline at end of file + text-generation-launcher --model-id bigscience/bloom --num-shard 8 --quantize \ No newline at end of file diff --git a/README.md b/README.md index d092781a..49713ecc 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,43 @@ +
+ # Text Generation Inference -
+ + GitHub Repo stars + + + License + + + Swagger API documentation + ![architecture](assets/architecture.jpg)
-A Rust and gRPC server for text generation inference. Used in production at [HuggingFace](https://huggingface.co) -to power Bloom, BloomZ and MT0-XXL api-inference widgets. - +A Rust, Python and gRPC server for text generation inference. Used in production at [HuggingFace](https://huggingface.co) +to power LLMs api-inference widgets. + +## Table of contents + +- [Features](#features) +- [Officially Supported Models](#officially-supported-models) +- [Get Started](#get-started) + - [Docker](#docker) + - [Local Install](#local-install) + - [OpenAPI](#api-documentation) + - [CUDA Kernels](#cuda-kernels) +- [Run BLOOM](#run-bloom) + - [Download](#download) + - [Run](#run) + - [Quantization](#quantization) +- [Develop](#develop) +- [Testing](#testing) + ## Features +- Token streaming using Server Side Events (SSE) - [Dynamic batching of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading @@ -36,30 +63,63 @@ or `AutoModelForSeq2SeqLM.from_pretrained(, device_map="auto")` -## Load Tests for BLOOM +## Get started + +### Docker -See `k6/load_test.js` +The easiest way of getting started is using the official Docker container: + +```shell +model=bigscience/bloom-560m +num_shard=2 +volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run + +docker run --gpus all -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --num-shard $num_shard +``` -| | avg | min | med | max | p(90) | p(95) | RPS | -|--------------------------------------------------------------|-----------|--------------|-----------|------------|-----------|-----------|----------| -| [Original code](https://github.com/huggingface/transformers_bloom_parallel) | 8.9s | 1s | 9.12s | 16.69s | 13.7s | 14.26s | 5.9 | -| New batching logic | **5.44s** | **959.53ms** | **5.28s** | **13.12s** | **7.78s** | **8.92s** | **9.08** | +You can then query the model using either the `/generate` or `/generate_stream` routes: -## Install +```shell +curl 127.0.0.1:8080/generate \ + -X POST \ + -d '{"inputs":"Testing API","parameters":{"max_new_tokens":9}}' \ + -H 'Content-Type: application/json' +``` ```shell -make install +curl 127.0.0.1:8080/generate_stream \ + -X POST \ + -d '{"inputs":"Testing API","parameters":{"max_new_tokens":9}}' \ + -H 'Content-Type: application/json' ``` -## Run +To use GPUs, you will need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). + +### API documentation -### BLOOM 560-m +You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. +The Swagger UI is also available at: [https://huggingface.github.io/text-generation-inference](https://huggingface.github.io/text-generation-inference). + +### Local install + +You can also opt to install `text-generation-inference` locally. You will need to have cargo and Python installed on your +machine ```shell +BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork with CUDA kernels make run-bloom-560m ``` -### BLOOM +### CUDA Kernels + +The custom CUDA kernels are only tested on NVIDIA A100s. If you have any installation or runtime issues, you can remove +the kernels by using the `BUILD_EXTENSIONS=False` environment variable. + +Be aware that the official Docker image has them enabled by default. + +## Run BLOOM + +### Download First you need to download the weights: @@ -67,29 +127,30 @@ First you need to download the weights: make download-bloom ``` +### Run + ```shell make run-bloom # Requires 8xA100 80GB ``` +### Quantization + You can also quantize the weights with bitsandbytes to reduce the VRAM requirement: ```shell make run-bloom-quantize # Requires 8xA100 40GB ``` -## Test +## Develop ```shell -curl 127.0.0.1:3000/generate \ - -v \ - -X POST \ - -d '{"inputs":"Testing API","parameters":{"max_new_tokens":9}}' \ - -H 'Content-Type: application/json' +make server-dev +make router-dev ``` -## Develop +## Testing ```shell -make server-dev -make router-dev +make python-tests +make integration-tests ``` \ No newline at end of file diff --git a/aml/deployment.yaml b/aml/deployment.yaml index 67690722..16ef3dc7 100644 --- a/aml/deployment.yaml +++ b/aml/deployment.yaml @@ -4,9 +4,9 @@ endpoint_name: bloom-inference model: azureml:bloom:1 model_mount_path: /var/azureml-model environment_variables: - MODEL_BASE_PATH: /var/azureml-model/bloom + HUGGINGFACE_HUB_CACHE: /var/azureml-model/bloom MODEL_ID: bigscience/bloom - NUM_GPUS: 8 + NUM_SHARD: 8 environment: image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.3.1 inference_config: diff --git a/assets/architecture.jpg b/assets/architecture.jpg index e0a5f7c6ffbb34de6ee877bdf2d74fd3581c75da..c4a511c9db7a690b991e95e36aab4e1a5b5fef57 100644 GIT binary patch literal 342347 zcmeFa2UJsUwmy6?6lq3OinOSpsE8hHx%KXQ%pOuw`m3==42m5|@b`CBm7YFA7 zPIh)~9_|Csg9mvIa&R3weCXg|@b`m%T!iV5S3;QggD*VD$<7J>?_YT^>Hwbo%r#ti z2on#mkB14u!^CI;VBqIuVfwcN_3p(`U}g z$tx%-UAnAsO;bx-M_2FWEtA`(ckY@!d}L#5XYb(X`NYfH$JfvQ`HSF?(6I2Aaq+Jc z5|iG%P0q~9&Uv5v;bUG=aY<=e`PYidhQ_Ammez0I+j{%@2L^|J4v$PuP5+*mots}+ zB(864ZjrXhJG+03iwS`I`>??8|30w)HZE`ynD#M)Q_A+oxS00&fd3#o%q+(*vL3o( z#AfY&_=L=}{YS38$|$U77n3z6@ILV9;ouXOn>V4~G06~=M*YbZW5q{lnd(~z#M81fnXvR}?B9}3>T zWnA-VQ>Y%3kt}zFsl|!`)LSqBTPIyE?^-@`xIdeA#P#A^@a?f)OmCpfECUE?pfX#L zx5;kP+YA83d6I6*8hD%mgf>#43?S2b8bg`wVgNCQObh@TN`^6jYmKe&?G6kB5KV{B zVNe2`9v&KvUQ2<~Rn^(3YOo13?J!iB0pJzTRR2II840gr0K2C|FEM~xX9hst0$>C( z`WXY*ysqNO0Fc8BfVc?2CkYGy!T{#q_|DOHiW$IkKY(tWq_Z)Ao(se-+J+qiX#D`} zeXzY5+goFMyK>K5>>1cSNZP~7zqxk-nFd{?MkI&CHlAC>H;eXWd6WS-y##PB41fhq zOZk1jT#bz!bHM*g-|`(OcGnWE2P|%)$!C3*UwvGmxZD$m|iCyy{~3}%6gRF+nVI%lby zjT9yO<=#~m)=EAVs+R9tG0*NXkDwJDVY-JD2iJHu(n}5fQbm7i>_p4l+UG3nsd%QY z?Sf-AZq#6Pp;`g(6iw>2fjNp>Q%sbg{Oh9B4Ychr;)8P9j^A|^$^>1GX)zf}JtRAk zkbave(c~uRcwO17v>@;BDwnZv!2aECwx>B@G;$IBsEWnNXEMOkSh9c2$@8Nd_Vvxp z^aE+FboOluY+<1Y_M9(fP3Fl`4aGZ_w z?(te727p5_faX82fXje#2HnF=z|(fo^s`{(@xX0ZpC0BG!T{F#z#!um7*HHWwOMUy zAQiz-2aI4|f)R{B3dtA@bc7~xluZUeH36fOCVWv6?I4~TgxFjLF!e{&5ps(58aE$8 zcB9ka41gC5bS^3`gTuL|*ofX<0Hc~bFoN0pOndXRx1RR)-JUJjvzdF?vWLO{Uw#2Z zoEC<*Vb>CxK!=7>^3O5=iF6J{_!sTv@FzJoWXboZRgqPr-@0atxeVEqKceg9;g2M# zw3NveItH^t8gjb2?UoYqPKDOjObc$ideepUk8(m7)ojbB~(E-e~8k%?i^0{t6Hwc6UU@)>KwTEwFe z1*0XF`fzjFSnjAC@SOA$^!>t-`<2zs_!_U$JE2KGh>F1@N-tydBis+7x68|?)kSLv zeRR&llgfR(m}Y#iV00n_@LDSoN~ny)++?mg%GUFX0h|SPM<=aT&=e)4ZJZ=xAJwF1 zw%@kSLSbj5h4_X6a5O7v#m0~nq5c5r@?Ik1AC!pr+rI6-vaF8|BdU?R@3Cl?hc=EO zx~r}>cQV7-ZreTB_iPaiY(j}xI!8D~1u3H_Xccs-k{tZ0Ie|Ur!pr)5-cIGr7Z`x! z$=IjeY~Lz#SL`TA)KK#45mQ}_?GXU?v%Q2y7 zf*v%iRubGXFCwOK&8OvdTvIx~mi0wA@)yPT>$;!kvHHcX2?SUW=IHDj6iMWz_1u`r zUf(#}adQt}K{oC1l*u%$ZNNd5^IHMtMs=NggqZ(bNQ>b%rdJ z|0)ajDrDa$Cbl4Ovl1CV7`_O)JW9OTk<^T5my`85uWz|P6yl7-k2^@U$zryPR6jsw zU@y@G9US-gr*b?i{p7eqvByos&=mi{_<4yi$xPbASV}kBD}K<}#GsnZZOv1COb7)@ zhC&oi9Ukci2OgCo(7!M!d&oRA2uCn=3mstq!<(3BHSS76>8Q1pNw0?Go%pe2>8&29 z%?6?d2xy&vhtcd^S+5p+WUv_CR4xD}EYqC2L#)+dN)C8dv?~J`eNVzS!FaZ6w67CNIwgW! zT#{Q6*Hl8v3L_!MM|%z4ICFg;4^*olOM<}MO~arBT3IuuCC{LaIT=kRyWHJ3EQv0( z&(5%NxQ+=1ZO!|IWY?&NtV!7pq(Oz--^4;jMozvw{rZWCCUBaxFYsLVKh=Ht%eYIX zN%T?p1I+NywBs)e-}pN35!*sF1gRxFLb7I#JZir_QLG zJFmF8=MCSDFwh!qktL-la76ljKQj}{_UoZKEQ0}X)?P%)kozWQh}w^!K;^UlDhB_bUK|6NfsdrnIrs@lBtMF2@FrhOi=|@B#nkg{ z6P!=^7w8E}->yQbMfY$5$C>+PQ5cr0H9ze(xdIDYi7kli>f^uY;WDav&sHT)Cz1o8F&E+r2hOGUm$ZL7zj8k7 zZu-}tE8-q4jc421W81DU^VFxa&+p0Q{>K&p@*5kLcA6-UjV>K^vf^=k+$=(Ed)4Z5 zA}_EN03NfTc+za|wE3l-I9MS5dgf8~3hv{^o#X*AGhN-aUvucm3q&^>_hlTNO&{d` zO%rJOF$@5+!2s%V=(cL(*bR6!{J)-K^yjBouPibESxO`QFqDr0bob>mfQctKstIi5 zD+AbJ0B|aY)vV~h^kq8G?7rCM!(@zJ2>Os?k=C}mq}aHQ26tl+`)rjAwM$6F=5$-h zh)2vwz|h;AvjZYJxe}F&0ne!qtNP1=?cu`4|CxG=A0;aP;u17;0ll?)&oZXUvR)F^EC z`7A^o_Vn)X@WNp1PY2SMk&#iYLJh8qFQVm)f=!I-04_i&o&Cd}#n{1+QfSXnhfv)M zo>M|P{0(Vxv3!C9kzzJ|B+4zZeOBA`cP}HGxJ#Ib62bJ7s6jNr0CTi)5-zaNRp2YS z`ru(IkB6aR{+SM{obwkpBQhBp^?PYoaVVj(?L?<=9t_wCh8IgIa%q> z#!&Lh4F}YtHus?F)aaYWau z%P$mP{q~=$U*jWB->D&bhmqj-2j-qup)Wtc&Qyp$0v@}sS+RvrI-|TS{X&G2h6h}Y zs)~yTEX*~2q;b4IapRMeUEakt6cbgu#JIv>+Im^;t&um?-vN+GYLI{bLHAM(-oT zr?f+`DzIrR^sbY?{H}9a@2cvz@kpqNb-zq(n;)|vKy9LBpdVl&U`1}>Nz=IHo+<7J zbJuzwIlWg{sWR+|C%*{fx@Z3Ic=*lr-#R&0JeaoJmO=e93nw9Tg}Dq3rJvwL96(+j zIV8?q)^+Dtz>-Jn@1#LK@w5--vFWgfdjg@qc6WJLULa>pLRms0jQg2~?msdco%~Jp zYoLppK1(K^w|f1wG}n&-oby7fMa+7dHuq*^r`5z9BATwSPj%RN)rJcDYgTl1z`7)% z-l}kb8|D7iz0*2!`@&!jOv+{(*Msaj_1Wh3``8{~o3eyyw6Mk7^pgH9hmb4`-XV=4-_{TAl#L3UPHYZg_Y zDFcH>kpVVz)5{MNhZ0HJUJk@uO#*j`C1-z0O2ul43E$0j&hsr#CK5UiD6PHRRe|#x ze*36>uHc4o!pXLg(iAggD{~R|Rt;`mE&V(5&I|Ywc%Zai&bNN&H-zim`?83);cYdS zh5Zr+dL50nOsOdf(RNzqViz+?D?L zxI4e&bj9Ng0H^;(Gcvn(F?XGFHxR0Z&a~y#y>`hc`?Bm^^iH%%bdxFgDG^$C*)2*A z%o*;6Mm>>q<8vJB^0Ut9_;4+0+wlB)uQrp*YP$BBZvssounA|t?Yp@9@i^04Mgwmq zH~_D|b8PS@+x*v>g#W8LOXhtSw2v_DP$;$_X{l^I^r(_tL%HP8{(gyYu?k1n%2>~; zy6@vR=v(*lz811D*fqcO>tjv^@MW8N|fpWUh>@-;p>gqat~X{Bi1f|7?xp8dN_{TS~h zq_dZ%Y_oXro2%4Uyl1su`@LU|!Kk2#+jjn&^>Z*e4(xiaS-rMH zx`ek{Mzd`{q0n+S7S@M;5HB8$Tzp!xX%${ae%PQUMr_F?8Hf(*hqJTZeV6*``PgVV z+J6UGb*_1Kt0qxUr7M5FdX)aQRWEX>ty>#MmY#qGZ!$+OSwT!|Z+;CGc!o$vOi}%J zO&9&NAroI5ZndD+(K}~%ymn}<#Tw^x5qCje1JZXQWTcZvm`9sgN9mSfUe>%4FDdrL z>DI(E;z&8~2#BTN*LQJcA2Tgy%w}5_`Bo|wHK_GimdjrqlPzT|KVk=j$2VcN{zHKbz}^6X zc+y9;eNP$+HqW-gOH`qesM^`uwUg8Dw2tQ8jln@@Df&aWxkdjw$Tp*G$Hl5GRf_(= zlNHw%DSf@_=*1E1*KxA3=sJq&qVGKuVV9HU(37WYl2t$4kmr}P=&7e_kuPTYwq24J zkrM0MLO#;T51UnsWzU7O$+Ts(kw(WL%)X_`6skM_3;F zIUQi10pI}_Q3>!OU?*_Vbn%51YtkAQ*~f6;95|P7ySPOmA78OpR%4BCJl^^HeC{9A zNr%2*0CrgNImPE&RtFr*&?PaU44_cHM&icPD#;a&@w*d)GsgJ&e5UVZsi8@wX&Ric z3&nN&@&V*=0w<}HTu*>a+vXFKyKd4);+vD!Ey*~&cc1u1=29fv#bjEltly}_$r)#& zmW#Oygmi@ht&!HWTs3j3IW66Wmho9$KMHPMG+>P*Y)YN|^*(~TWZWjT`bw7*l$HfgVmTI3BE^K-@XV$pMUsLkLdMbTeQ$w>K42_weC;u{ zS+T-F{G@kafXyVDEGhk51wuLPt8JjKj6Y1ZD(yLtAoXrR`R?d1^9w;jRZDLh#5X*97?Bmn#Io{-|psx2rOpGMDF*o1@duozE_;QYdGa9$tN@6n9i1zDQHtRU=6D zTRLa|KNN)jXE5=9>_H&iD39q4f~KwJl9+HSE*Gg1P%fcR=$gh9HaWzj&3=5oIPgM| zD3pWh@(9?#{xl_R3_g*v#gp7$fO3fZQ_mvDSV3hrewNY z2ieTr>s;{{cCPr1I!4v$QDMq`>n4B*IEyJB5kqzcm{aoCc1)k85f2E7<@iwsM0?C} zio<@i*7Fj)Trxwb*7orW#Il-c7u`~B$N8cl%JjPmaMFcAfoiM?6X-Xt8mq*(p(eJdIznDw~j@s@>HzY}8Ja zByPCntG!G;Z4`5kFO-`InMOUEyuM7hTp2m+PSPruDY0I5^g_TNf zikZ!!f}yZ^7dFYHcECOPa#h3;4yG)gI;T4sHoCKUqZ3!pT(Jg)UjH zaRbv72Uom<-A=Xw>n)yTU*Mg^I{BxKO)vnMueL^EG2wPWL8C*SL)f`F-J*}P5^T_I_p}{s zj|Xx?x~@An@qy#%f%mU1Hcib`y;U6rl5b6Y=ol`5UeVS)a`}Rdb?lXICu(ctPkI%O zRXm^S+DEo)@h)&HSopTiJCd}2Uk5CMn|46E)bH+l-Jp(%>F|&m@shYrbXIpmnP{q) zL)I)N9L6ah16cUAD*E5+2y;#FM;A?#5gXW6S-No3BQI%b|CAA{tYrqE7W({UgFYAK z^9&*s&$ERA9M!zCjxO7{ZANH0pL^c#ulSYbfBTYXK=`*M&)z}kRE#zq(X}Q44Nns_ zHTPEgOqdHR!SXsBbTU(|OB}am0Qp3t7h0OZF|@9(^mbH7-O zuUPj9S5eF(-!#Q&4odyF+29ZA6|lB9e+g1$VS!kX6A(IBaFV=(J%yYxGtQD=ceNQh z`UdnK{+E%Mu#0rCpswsj6y-emqTJa2QAbhuP^Q~*1e z)CtX(;TuYZXH$%UckfPW-z#lH1P`Yy^)Zz*~N9IW5wBy;;$Z@ zF1&7Ib^M24oLEUW8_3FVpdVmIo=>|4g*G8V(ClkoZU?^0b!$95Ny|z9YX3F1A?Z+7 zTSiez&t0GVbmdopz&c)le7F?`mAC0e9rIPZN9cS{C}O!JIZmVb>O3xsY~IAbZ|-LAGB!=X74zNqo|5r_*6=o+z?E3> zr;UE@D*#Od)Lwr+s;}T@m>C@F$avjP;8WyiOxH}RJ3awa`!+HoNEkdq+ z8-VU@0Mh+0nZP5jkydD7=TNE+vPeC$yA2DrN$N6m{Mv=^f`;c|?=}yg7Kqm9w-%Uo zI>rXJ3}N1)ef82G(;6k&kvIJ0YpWj3A-d&NHO|NtX?cU*_OI37*Rx%;T+M}`UFE)P z^ELmPjL={0j3Ud4YmGyf;n64iMRgyQ4;(}s)@d$FiwsPw+Z1`W4|iz#9v9(lc3#;p zM@Q_o{5tipcIWJo)XSX~M`ugdCD=*OhPO}dQdzRUXQ;Xvd5>V-EJ zuVi`0Vdj=4?_F~%VnF`RuU%)M=NY0J@5q!XX&y4VQDOR8{I?*|?VwpN%HqY?bW*3# zI#w+DBKZg30ZFcvqh;BH6#Mvv0Iv%_9Q$((Je;@H9?p!O`&IZmnQBjf=Bx1`WeID+ zaw!&Ouu}}cuqxO_@#{B^x`Sd}S0X+8=?N4x_v6*W=Q4z^mMrQ@bo5ySjJJ3BkYU;{ z8X6*RmAhegM4wsR@x};ILCHtlvneQLzX1ct5Tri^yVLx30^J^Aoa{`YRSPCx850CM z;wl%QC(Yqh9=Yo^>RcYDfjObkZC4cET6k$XdG`i%>e591xdW%HsuDp?)n;=CUcdBf zx$eQ(rKgu_U=snbUfg}vzD6|}s&(I~;!1)~Xjaf>n@UxFk|8lSFJ09f-aIPo`Hk8d zq4YV*%%hzUuD7d)FqyBW?mihZnVv2zbm+~+di%N5A}EE@ziK8I<;IZjaHKHm)Xk}U5s?n=tEkXysA&Ij;8 zpl1(Om)ixP(Kb259UmO|&$zklON!H0;*eZi8p;jMJv)%o&YSPN{Q5P&T$!2?8B5&n z%K$9zuDB#h3SD?m))sTIUZog@e5{P}B3m7r!;HScI_w|e=~cz2Y_0UIsd*j^-)&k= z44YhQGfltTKUq_v00Eg5*P;TS0Vm|$Civ|8O zNe?|v`+x>>m+bT(XwEeipDFGn|H!71`AWlh=j7nYqqny146P|o&lrup{0LA-XbCtw zoCsZISnLKK;bH{xP;Ck%EGPs80&F zZ}*swj<$fp4ILgaO#u@VVUUbq>J2>k=dI!L^Bj|HL*H90`HyEgE#t7=VS@|2m>fxD~)1ZqMU7~Kuyax zyp)nU3u%+(n)mLtLXbZWPD2yUorVz80@ZM|m+*veWsAgmD10uCG0H5bo?iV^S)F_48389zzhf%PWQ zU9f3^ENexD*pD|MufB2VrMrCKRX1dsFJU5VN1-R;tcqJ-(q+t%T76q0H#G=9wqpW5 z`Ck6cdv4Wg=7glft^DD-PJwEts!&Nz^?fb?#Y(TYq=r+!xO-DoVdzF*$&&BNx#gAP zckY)5$#V)DWd?qDnSR<3oW9c#$P}UhABq-Gk%m84lOQYV6Hi5{%~Is4`w*wJqNX=8 z&^WGo@yA6iw>5w1eNpLVn~7OT0&(QSP&I2YRRip}um$ItHxTc|lPyxZaD9d|{UeI_ z$`@R@i9R>JYP>C1KE^f&ihDBA^-6Fj{4j##O-r*Ogf^jH4w)gsBan-@HVO%H)U!iH~XC?S>ICp8+O3VQ#QJ!qyj? zudlv%<;&M?w9*uAFh8%=i2ig0vOKaUqy7twsS++ija^iM@NTedUTw@SCE{kZK2saI zE);gpw9{P+K`VUrIJ8yd;nDzdX!>FNQH{~QqA_P=^gOXj`$U3`(lL@eS>tvOM(!zS zlctSw7jKd;({1iZhbQ(Iww!__fT26IA8JN}1a6@7hZ3z!D8+;Dsg(_@mm8}0ig4#r zC59f3A|Fmxq4|AXDe9>Ss6J6K580wCZRHc{*j`y=<)zOY?+=-IO2O=UXmbUYaae=; zEj9W6&7~SK!>@hPnHPVV>-6a47DHarpJK1R7yR^L;Pz_^OrDyV(c}2Ya_t^C*!(T} z;ouXoN&R8%(I<7+FR6TIf`9IH-#mf4JK;R@;4$9t18w)IUn`w;JW;i}45y+RKZepS z7#dG|OT;yG?VJ`p2Bywl;CFejb0Eh;*ZLYmY~N5b=cPLA7AN06TmKql6U0jWSlKds zpHG{<^ulwF_`2L1b&hDO{;GXSKUV3e>NAmmfTyPY(_hO)qUtqRp58S>e5!fsfb*Hj z7`-WCoFk7|+fbn^&7^_hrGvO2N;LnI(`DOdxgB|n z+pk8=)n!)aV1Zi_UwF5bsx7#<`K@?P3Vg=y;=gU$8Ou#~NIXK~i274tSzGjKN7Sph zQLW&ZgaeQJL_6v9=jk*jqN~`H+3*;`4(c5@?6ZN+oJZ27kDk*g>XiV()+<{D`Mi}( zAmSLkohK>Y+I(}?dDN4{!I+F@?AZLO%Ws4A6C%A0wV!LWH`iWnyQ|2*oxIS_Z=5pY zU*%e3is&nd{y4F8Rd~hTy@^-+;1!W8Smv2S|C?L~$l;b+83Mksi+!H*x5Nu8VCwVO+;PIcF!DPxn=_9SO8#OpcJ~X6Xz2aez@Hz`iCbOQb2VAIca5NE`&6zi%aA*OdR?bC?Py-NtU?Xd;5 zQ>8DI0%UIyXs^BdT#cE$OQMg4^lESyNuQN{6{`+6#684}#!L>acxH}`{QQaG_U+R% zO)(<;+O~Ykd4JM)k-h9#vRDOgt%?1w6`PTVet;Utodgxon~C#D{FqWiqk&Arkbdg4 z)zO~~*Ulw*v=z2~$UIz19Tva#`n=E&$x}mk*>tv98vuO;Ukqe%ci7E zHlqukLcq}?XyUY5W}vEfpiK4MDHpMNOTIyshuBW*J~>IwD7|&fqms-QOQO|Ry+2L` zs-ejyAiV@3vk~K4ceNX>SS7;ocL&q1mGKMSkA{ymtryxJc)U-mRcS>XruXlwlPP`* zf7l_7gYda{TFSXiZiNj|(ZBjbx2EZWRl(aw=A}+NWB2@^-iEO0O z=IWs{q%jo@zHdXG$oeiNQaUu_PxJ_vv>D+I*7M*hWbq)w zJY_Tskjlh@AY~SrtBJ)gm}yfg14^QGm;xX`k>EypO_n6Cj+>GnPNx{?Espv8a#+tF z9lxr2U53S@dfGpzVp@)Aq5q#C1BJ4n|H2o?gyU@2!f(S$ed=zf==DWsC;5D|t#%NM zP~5GnQR}#PQtwrXtPCts(~xN!znn|a@%164l8tPp3ocrh46J-B9(vM*Op$B`z5weF z-8Zs46O<%8z5t@%{}o-^zdD=F)(Q5%r3yNbBRZI_--yFWz?B^t@Eq7i#s(&IM66-W!ed%W0 zJCWo8|Fj3L=jV6vljtT+E75uO$DJQPj53M*)`(P4Bg4kvT_xO$5wgGKy=_dm((%sH5S)G%X!|vDF%LI8w>+?h91S z3#)2<6KZ>V2TJlrs{)*T#k2&>*}UDDjmck3J325qAUB*gvfi_vE3$uazpC~F$@Nd? z&z7-PJM|!oBl>o!JyQ)HxhGGr#(bz1McxDZ_C^&vC`+4CmbwEJnd|&KFkTqn54(}I zg&pw_J<~8txHa2eJS1^ zYf6)KRPs+4OOMG!2~&l8R1SXpV9DDr@9e28{hhz)%shHzX%jnGbl`B~BT|7WJ|CaC zY4w>NS`l2=5LzSk$ca2XWg1e=>1=LXaoCcFQl%6oqxc?OsoJ$YdqjY z`3}j_d5Gs_Zm1=R7V{SAMTA-rXSOjPJ7z{&5U(6Vx7}kF13FOh@9f@GFJje>Q}XQ# zl6G*ibxjZK!V+f6A0F23@iIWsRb{n{5*KH02J5t-56`Q1_>rBZoNY|Bq5Nmm!dea{ zXt(ov2S^ZFCt6Y_wqi^Nn|vy9CQ3iNsZR|3%PCe#1t?tOJ28F9GBI8U6ROz47^wfL zc~RFAQ=@9LBbux(##-CG2~{I@)w^+!tM&by4))3O9X(u^E`}j>KSpJ-Em@s>M?kU3 z)#&LD_XphW8bIa!m~6W5FgRe}^^WQ=5#?4_nM<-Q78IGNmTB>bd+#4TJ@2%BZ}fE2 z?*UrKNq_lbi=N4$3>B3M+bjP?wi5CuX{FrXSrX5*qYVPZAzBug+uWE+O&c`GQsqj1lTxy;M-yLsYYu#75~y~X3<0SPIxPbZvRNU!NzE{(JM)8; znpqv(L5|zTX6>Ka55+CmQOdNKT9amVh^@0WKGZm8U;_*(irre!F=nK~Nw;7WPsHuP z5`m_XkW{q-8*8saZl`~os7zLTj1Bfke_^LSs{dL2G}4ZIm}p8u5;d{U)uhSh6v*A0 zF9yb(Zsq0{Ze@PrQC0!f(a)v1o-@zQbY8e}Oqdt&N7TcITc;IQc+q(dXjUpn0kX4n zMsfM@f2UBK z-pJ!$!uOY^O6u(wPcnX%SzBDrsu^`{hz~3NIMK(%NbJmmMxdK5#*pnMp?nGvPKQ!8 z32R_Vj&;mWur*|BKX#>9JY0Ab(ZyA3E-7bx-2S(E;}h4#8|>-OG4v>YI^;PB`^H3X zoQn7MwwhESn#=x+PRdk8MQw=YFHhlD3DHa{lt)bB2Y38fngNs%x)H4Jv&WF_+{g~9 z<9zf6dL5|b!cZhB1~^cZQLM02i^5BP1G}B0@yi?K%RAmXxXN^>I_P&IkW2)&=CvsQ z&}gK?=QCAg7m}2nPIsJ)++(Fgz9xW3j^I~|h!S@5MPMZwfh|1QAO7L|dx(P{VCPe&z zuN_twjR#lRS29=?x!-3+T;-(aW8qIcU|Fm& zR1_p3gXjd*rD^*V{VfH z$DJo0)SUl}XL`29qj3Sg)osYOIQH-P98o`tCb%AIkqA&wrDz6ft&f6f&|*3}$WKaG z=b(SWz^;oux0onWfATj_qVz$$2-SvSNcFDl+YDcmE-|+-C~@O(IQTn+^~#~j^rdq@ zAcs6qHuomMV8@h@LT49A*H0hCQ=Q16`JeOM1$9rQPBMT~r&rjIUM%U&PIF)j0ySms zX(dcUzWH=^7(t5xAl~`HTj>{j$%yc#22{C zirKylT8DdQn|4p=mOPrg$NNg_2A?MLZ$V(|KcflIc>IN$mH%aDa&;XhJ6$Cvg?dSI z_+kt~4ST(wI-FaRvOTK%!0Y(yZDG-@SU=%zw%>G&&oqX7%5=rDN6*bm8d&-YUHG`b zMOCz+m{dsg#rlg^2cmHEmsew^q7Ea75}~C6Zksj6Bg*Z{AIvXyalJe$g2*uD2CV*9 zP1FJNDgI{ZP&kL28z|@r7nSc-x%9x}kUk|RtgSJwFhe=pjbwNjjIG3dx9J?C6cu0B zh2`p_=^zdCG_ zMlt}Jxf|UQPHX(*feL7PK{*4EhS4Ygc-{ioee^acm`DP*_Tz3Q@R7{EXoC{3Pb0KeqG;C+q(ERLeri40&>5NuKkR!MKU;C8^|&^9;N>l7?^ z-YvsZtI?E@Kc3aF_ttw8zPHTwHrM|xlOk7wA)jjvwN22fB5)cN$G~OHd_$BIAwT7- z{iyShF%%zI5CdlPZP4Me8rJBk6g;a{d*E^6$uH$$<><|@;Yp*_)oO2%(~w1FP73$! zHg0|WnxOUG8Rjp)uAKNT{6R}Y^33sPTStJUDt&XxU1#il-SkurU7^U3oBe{FrGj1y zl_n-rn+{u7U*AedYi);v@IFrFqhXhM=6XKEwvDGx9s1*8G#P7FVMJ-}f$xqKJ^!A> zuJU*q%gi*n>%4sk5@Ya zI=oTZs)D^4YQ7j#U;Zi9;OP!I3y0$Us}GGx5vF2DgU!|25S;sW&tD2Y+G+76gT$xU z4U4y%zR2qrd8&Pu?<230rP9egzx$WpIgDgID)~^Mn?fty+7_lcwRe5h;7(DrjQE|8 z&)Txe+xgXc_*pR8|FUcYQiFhLP2=}G*%>8A_&7DyCnM26oj_H`{&1(tzn$saa!{yh zH6c6JI*7{O6LB=he%(fMBH;w9FzT6Xyi+AhX~%NetCICXn^!5qiN}HvgE;**58?6) z?H^2q+d{qo1gTl*LN z8)+PBovPbOJJPNF8rQXjf?K9-ZB|HgJoL29)AhJXw;Ss5 z+WNWNIN@kIFdvhp0-JDy`krXIJ6~pP5pwb?%H}l%Gp~dl@_3iT9&b$YwO<-U_$)F2 z&5TvU#zv^n6%iKymft@7e5BaY=6+)1A+F+Y^(NXMC7u}>XP?pR%+fuNy>_bKWcat5 z3N5{)Kk7mvPvxY|3g ztpd3-_y?I_fVflh2bqxQcL2>aubRL9>&DqKnf!vV_}6JGY3Jt7#w6cZuJ|uOdfA_X z^jm-5ARSo+x;^`7A7Ykg2+x+0@&Plxqs>$1+($BYmZt%~ZOom4K*X;7{Nt-%(xDji zM3PnUxn#8@#g&uOPJHySmGi_%kmJj$H0vqNu^O2? z&mrU`<^Wx+yqkb^*TP$i~ZZr&u`e&v-ZznbI1o0D42Z&+>I zc2P`k{Vc96hgbKKRLg$087aVDVwpzJXvURjH>W}-O{KRYyYV{Q5Lla`LouU_@fwj`~esnpa4iV%||BmLb6T)>b5xg zgXgEXmGYSG=~=l*?%|^k)NgSu=c&Up+=A4Yh|=IR2dBXZ5|OCmj`pErz76E00uD7I%24JiRux4{}l(dVhoX{|vdXHi% zp!QqeoqQ5_I!8UU-3lip(A&$BnTz=`4Q4g7i5d##?cV&SQJ4i5ZOWS!b6EVH{a=l4u|aeYz^1LZBVLsVW$>$q(W$n8!Bl-ALj_|3L{ z-~GL2&0Y2MVS5G02Ry}4!OnVH+mN2u3HrYx;Bmb4@USV(eS^DV|JIo z+)*LrI=$ZtinE37*T#}dzN7ONS*CmC%+;)ue)ImGk|4-LEL7DBS3vfdqr2AM;Es?caiVz zt|obX)P9(4S?^}!T5mQrb!}9TU**JHqO&mYSYYBH^zI+pT2Z>K$W19h;8q-NDDz5|>sRfmm6q3trtf!^qPhA`rPjp|{ zeQ`|qU=+tEKTDKpWPGHnWeL-66v&f>K1a#;3SB3MPN*7MIS6LkWId>Q6kgxhd;%;0 z$ltf0^iBMr4x1^+s>t|O&p&#h^L_S=McZRju@Bqn(7EmU%e7e)IMu&rE9Q{>`=?z@ zzR42(=k5SM2FnE!j-)}5QKglT=jyNuI7jVe>M8P2wq`z6f!OjkKQvPJWaf3gRlL}0 zJLG}Rw{3wz3H{|KT_1emH;|KkF?N*oT5HFl zgMO_lp6WvQqSob}_o()?{r%NE*}jz&tMdL*g#khR{3oI937OFf9C--lYpDoFU=t@y zo^HT#S4(^(Pq&OTUA7F*R(CzWn)7{s1=vt2@qvzsA^QV$AYL8z1}%u5$cOW)Xy*M| zz%a|1OLIDKv?xDa8hupVZ#x${vJ;}h06d+hDJ_92V8rW!ADwO>Xii0WHVck8cvNB( zjR-lPyyN{(ql-_2F?3N~H`^e($clO-Uyodh{Yh0Iw`c$KecI$75hk}{-0{{~I``p= z;8)-6A2|o+dP*w7j-UO+w5vJA)q;Kv9h@{?xRdZneWtXotM|HJXy@k zwtF`h2+ai>n_=)}iji;!Aq=0d`P4%PC#?MfYsN#85c-&beN%RYMeA#{G-lUP^)zHT z8Q2*j&C;^H+=j7}%xcF8Z;RXxFI|3p8TBT~$geOj^1EE*TcN;fqeCIDPD6shfuiU= z7*V*x@u{vuNd3!a$!M+hbIv1eb#=YtY8EXkb1x!KX~AdmHtN%%cWEEsj(CtoEZPpv z7*Nr{<~dcyf{kaq*O`--KPxIOcFT_^nioRgq8U;(7Ywx!W6^v0N&B-ME=m}5>HXT zfh8DJjmvks6(zb3FWKbYM=>hi=tH8O7#6>u`S$Ibux%h*xt1SYe+wl( z;!u08pMKWN-anzaEHSHbkngsEtE)U)!Y#jOfvJ-Qt7GgVz_C~AqRNVY46G)}BEUvX z+^muFl6CI;@JB8to~eOyTGyW@Nzn%Y)Mxq+25=DRPBfqJ306Bn)#&*oS7T>cq8P&x z>lM~u@dC_gH~X~k5Wi0Q&IYQg{Rh|l7bwX7A*VpOXoYAyPUy(ZNtU$`m#JVCgB;g> z71bwXi=-=l!6_vk&kCy`g3k07U|+>sR3BY*UfcnD5Q;ovW}| zelV?WegE+pqv(qlBvxMnv@`$B@%u+otB^_js5zZOKAXx@TLVf>{3WG4Te?F+4RJ}^ ziT-k_3cRqTT#-Iy!)foSqo*M=U@=~y^a?lS?jn(TpS(6LNt3ufc=55cmyBIM_PB)P zZ}rDtnzYBpO4ZLA9d*A~>TZ;q?PcrnA+dD?eE@zRI_xt!P#Hru?I9;`xDHhlk6>Kv zz+$Rv``HE_vVNTj|9HMk)-zu9a^D2BE@FRm3M{A=QjS<{oottfWqV;tG2RH-RRmbm z1E0Xs4ndhgK!<}}4O-?&@v9BX@JVedFDM1NGJ&B=tiZ`*qP+n>EkdcF{yPY_siOj7 z>TVuZ3r5LKLCK>rEUiO|E@g3tsEm`<5~@>F-*+8aj5qH};$|1q%t?qL#!>Kfyq zwtT*kegs`_wJRMlkd&>-fG#~p?kJv&no@{k;oR6Ls3L^`F|BNA`4BN ziSR;`wOv5$T+0xE7jS$Ux~*pSivW#)6Rv~IhJ4V}#^mNm%(GHcmQ23h(t&U^Y2tMv zWLR8XRrxldxf9Npb2iEqndOj2AHop~V))HlRhH0`-2V@A-x=0qxMdrpi-3Ui5)=?o zx*$@6q7)HPkzS(GrAY?~2_31S2nZ^mfJlvW5F~U|dY4W>iqwP>0x90l)8?KtbMD-E zo;l^0Kh%8fec!$J+H0+??L{MGJF}LR;m4H3Hw8QOgT}k=%^HLH=z3T2p@FTK6mn!> zG6SDW2qfMmh)~#(^tW6ORHZ65_{2Iw%YBYs3Y^eZ>iRAw3ZesHOLlM>x)pzQz{po+;~SRLfqTvQdVDb>`L_3pwwtoQ~s?Di`}HYJb-z zw)L;mSW?M8tOPk6TbQW@T|us+d$!e|@aEhqS$Wo+yo6xb_)u055Y^>H=I8@bjl-UY zl(a*$+Z!_7GBYM|yPtDr4%9C^mUuw0LuP24vw0#*ID`@v``Uy@&rer;Kjp422YtR; zQ+R5G?d$o^N3NuSHyvM#hpX26pICmr^YF%mk~AyL!V&^ITvR+=ZS$rPh zFw@dxU@FrFF=Uqap&L3|&iW+`#UD$d<<%d84A|-p6+xmTchuC0ssY_bOwv{AwP|c> zC3TtS)%{T5OwN6R5KV7|2L5cZ%Yf;j6ojKP+tTD&kJbcC1nbxeKjXB0GjmktGy?Uh z9TaS=!a9nyAkLs7i7D5)6rg&aGBoywFRf4@d>!24~QD{kY((E{!vC5p|HJK zApUFF#+BNK%+q#=mq>eooEB3$C56`IbwY8h1`68feKDNa<6(P z6wU481@amEZlG@pIL?>B5ieJ-&Wu}qFHR_*7TcXZQ2SILn>OFv|E#I%bS`ra9Y89d z%tuUYLXS_E&(x9eNCE(W6o}k{bRdg0#!zcZ8~^wm{^Y0kMTJgT_H#8QbDfU1k7zPX zO*v%EFp+1f8aW1OX;Rk!8D6oo8o*(p6(m#^Bu1y z0S=Q(f_snU$ckTd`vktqT8D^*v-E9=+=$1mx=# zCl7yt?x+NV;Jtv-0rFn~%YP!JP~){IjJ%5jlN)yl`hghei8a4t!;a5tnN9aSM9n=8 zE@Xxzu1olpyHQ;X<6PtdUzNI${*V>zM7_sNVe+62cbaE#x2r#gS1ofp8&`GXtAm|K zOQ{xxRuh~FKY=@phjSC!^>K&oKzTd6mMuKw$jxOs-b>x}T{w*4;@Y=cm9iJsdrMDN z&`=-Kkdt7}xV>%!jZc{)J_*YoVB3jns5^I4w_vm^!t>>hSgnnlm{&iFiS}%s10&z3 zcOgN>e|*zmd7!PR4HrVRA&3_VSpiZVbo_K zSBKQ#H&mW3c1}O;`y>=vmM^D=^pdtEaeN=7FGIANQ$&C&1%h>KN3%$|U$G+8nG$cT zL1Kr80MZkFO5Y1I3kji_$LgV@A=pIH=?VBbtLcaa9Q>ngB<^ra>P%NO715OrN8lY5f823V|p>JY=|_FcUJ0bnm^>Q4^)Vu@>9)=A_r{q~b0d z(%WkU?n6a9i=A&xhq5$-16*1x{Fu?v)g{()JP2?q`&--XTPeYPAh-m4EQhU0i+BS* z%D11Ms@Q8#ifB{95D~I3hv$@=Ue+E5A~Ow%_iI4#CU`%jDmn!12tHvRVB2P1PvcSi9IaQJVGeJL+Sn?K8Z5~vB{@ZOQwE3*lmE-I;e(`K?=wG>= zxqv>8ggf9cgq5R#e@YoI}XM{9!rYA4rw1(@Kat0ye5 zkfweU1n;8^4tGN~cX0AfcBYAjv>TG~g=a%6UW74&;Ai0j5C$2FD#;c*&_?97;5l$< znwycnO#AwYUG}wXC5Exu@Yct9pFwb30Bef|E5RP)0;VN!XjTvH2dl^laE*Vh_F4XZ z;O}0#sW10c6u`rjC?^exmkGArXvP}D_+u_4ZCxBmjD2=yrk%$G{gFnpNCF(tRoH8y zEG%*XQxC!-38KhO%dN}Z6bXWIpjWrS_$H)wGtXgB^$Rb%LEgCTYxZDa3lLfF53mJc z72ge1)4@(T)pP|)9Ujro-V{bRx#Nmw#oE~Hi`{rkUgSy%%#bV1*{w3U(T}folG;>+?4WE8*BS)DqY3^U7T`FaOJEMbTcdWgy6g^3Lv>^AQ(Pa%si^xGsP}9sCuebp z`#BxWT!gcQ)=kfX3tmnfDCjIY|L{+w0eY<6BDR!Q)Zg#Y=ObDbAoe*7P6+1x9ps$! zP?@g`M?jd~`Jt>8;C3cWDjK^`JZg+~7yFvLPUrIU*imo4+TfSE(;uvQzbb5dc;!lL zfOx!~z=w8PbIY@#0;dCVS-T)WH(>MN9eiM(uWY-C*;P_ad9l&9*vz6s*HU5(KXCGy zuv>X<@R0Dmkar;vz)IFOC-!h2*pZGh^a0c6^_PL}t-YJ7QSvJpvA(KmDG@}C@Xe%O zpov=`QYyIt@?*1E4DsCzvG#I}dVc_~Moxxvk`xJYm|2|;4Vs|?1ZP0FLHNBfl&?O| zwtYe&z#DZilm$zjmc7o+ADiNx1==S=@$LkD@pc41NfBEYu7G0sNl~fG^raoyiqOMH z$34FNLBsgM^~y-liOcb+NvvOxlZM?EsLffTnNtonCG~Obv!*$29L>#-72TpybJnZF z$|t`50=c4r4(Qih)-PbfMV(F*;Z8F|Yk{9B=duA-Fm^JH{-Q@AljJ6AhDK|~< zJ7`5=8!O>Hq6!s367zTY@nphee(JdXQ8dxMU)0VcU^E}98wrBz!VtLm=Li51nnD;g zer(`oF5jpC%ph>p5#55Q*TS{N8zGY{-@(sx)Yqu-N(=OxM_8a)>5rAliBRLtpDvq7 z)?ZK~@;+}H zpgzzsN_@?hG2O6&RO^KI~Ruda`OOV=%T?dW29ow$CN+v>-O z53|7iw!z2p!IK$_$>9A9>p91__o)c&L>ZWH^io~72P5fnbBa^-)~%n*qjqsomb$ji z1pw5~Xdj>J-n)dl1tNd&0Fvh*1Y=?7{K7!Qodf~+{T81Obx3^EOyDcx&TOL^FpYf8 z;t%289tihSIxc?*_k-H9$v0v-*>oPB0cpITdh+iM#-0d)$UzqJA#{rK;b_ik)+saw zky-d^_B#FO`8p_JZwzzvu3qH*q_ONIk90!g`P+Bt*Bm#0;;OXBZ#7N_AQls2MPd$k z)IMJyvV_<+BbT!|vY!gD=kMN2M#qMzU;KA@{v#R6NnWBkdUFFi4q+l(A=~2S|b8M3hlR^lj)Ij<*&=HCPK#@d_=?z&+_#lMe$D=N|B61?yr{rKxlxB5n zFE5zf{c@~c`ABlf$JvdTvq>tln1#=)4UW2P0GI7)` zsk6|4JI4-J*uVVHLD-q`R&TtcQ|shuw|y-dOGRrsmS=8^%xtpnMNS}qJ`lT_49>rW z58Q#In+wjWwdX z$O#-@PCg#pInR!Irz0y#Q+uxi9Z5q&q+>{Sv!fVH@Zxce32 z(O#EL{K|=Hvc;&&av&BBr+$oc?!oAIZOWWb#6XUV`)_f2!K*SKNInG5 z!tgrN^qbtXtu>Og^!Fu#b=hXaMDWvn79L zL#<-t*s^tA;Sze$^Fg<1(fC|6ho`rv?w$!23pMW^parSVzg?~sVm^Q6D0ExxC+OPu;@iw19|-o2;Uo#hPvG}h zEOYPvhEq!Hcvd!d(2-=m1+fn~1dH%8EJVEsKLq3i(EqfJujsK~m@;SKq;j~rMhk*7 z!RMhh+mS>|*o|Rd$Vq~bApZ^Q=W4u((xDs*pPiqgQlWGzzqqTwTR^k6U9{IM^U4RH z*d?(eySd?PILX5O4kMZdti1{3@+Q8HaI8ou|bQ!sfe%Yc2Yy)(zlcPil1zmt3z04k+EQ*W*(S0 z!O$0%+F^5Pco~cR=rSvBCi7Or%$C~V6qmb|=8?MRsD0CiZ8a(>wn0i_?P+tQ(^0%wJO}IHb%Q1#HHAF+J+O?zcn{~-)7H#RKw%R6euyPTJ2lH(d;R$8 zqTMHtR@*v&OZN-(7{Rm-1_WKMNY;&KKOQf%7F!V-AHbf9#$?&MKI4yy@FuT=$Q53# zJh`+VpR#&ug8OqK6>`oHUUg+@142@TY>_d=3sU z^jsN&J(3`yL?RI80U}Bqf^r++k@EsIfdSyC`CGf+wg~)1>ILeIQS?K@X{dSgs|RBg zPUyVNOzly*H0)Yok7U_;|Ac78;rS4VF?l{2;1)^IA=%GX3o*kQ>fay?)2LVp`WDp0LLX(6tx0Fk(w zJ9dY}kL|H;{ygeKM z3nbryJ3eoCaPjD=`JEM-EzmxKAPVi2db~T|g`oH4ugz*V)UBD76Z|Ucfsn?W5qfUc z@tub%F1yr;dKFEOc%30bv79f)-{NXr>_uU49byLk8 zIBMte>^bMs-PE8SH0P&ovP9Yv@4ukA;+IC_4;rM~A<1`>&b*?)U;y(evJ(vmu6LJXn zGZbRFh7^q_wzqR@T|PLTpIwZ0agUr0^c3eE0QgdS9QYr=BB~^${Q^A#+Q08?b6IEl zR2a5=h?Y*;>^%1sg+eX>gL46p>yX6Bfng;HL%?~sl^nO_pK$dm?uL7j+=E+*7@g5T zd5)QHqg027ErwI7w{YB>!G$nyoZ@_@c(e}?TWWXbRcFo5>OmInzc2-w&jzOBArM)J zJ(Bv#eD&d$OTkRQTa8J<6%-`4={nIX;fz<{?j`D+bUkx`hE*FD0&!DZ&6Z-lmon;L zSDl#$-lDlm28gFiSa5Wc=wDX+Dyg?_2uw&TAnJAp za&KnfPD)+7Vl5uJGw%E4YLa;^CVApDK>7oyik}!($VI0T*&DfN%^dCXcQiS{Y;#{5FeGrCUCnbngq>#4G?H`2onFGW`6; zd=+|&V-YcFyLdatF50R&W%!IAc}As!m*MVf=F|>ZenM<(BEY$;`A=!VzbMIv%OaKV zIC8Tz_-=j5Qd`Ui-3KT9-bX&Z(*d5{dv3LqH&apS@&NzT>-?Q>s~Zur?H(sBr#q5w zvPaywd9pb7A&4GCHPsaUGWc)6=zr1U{LN*lmLULVq}39P%%O!{!yX5W5U=aPr}KK- z$=YvEOEEcqw0-#&2jQ!cy%phS7=dkSj!|c_Y%De zgEAwwFaH9)kDCIK9{`zYV!#Q+b3{dV$0c=$PF3@$g1(vBwI>r))D~s0e=7D4RRvFQ z#d$TsdIggl{TS#MZvF}F_dv8>YElJo+NH&nyai-}5LF$HH4gO%7~;(G0qs1(bMw@^ zOX!y2_M8!aV6nX(_vw^oqj#Agh;wz{WrHD~4Fj;w$X2POq;TTY_&6|wo#*V$lgVAN zCyI+hPJ{p+keXahf9{XqGVll3<(*dKvXEc;`Xt0kf8d&h@7D3Vvq!o z_dJbjImCHqI$VEjr>wxm<20cb!8Y3|Jdojh2I=$IJ(oH8Ir=pH6A-yE_Y_!&pks&w zyw^zA7Tg{n-V(xZe;N_Lx|VulC`G+%6X~f54$Ef8Wjt>_1=wR_mn_v(606?M6I70r zY`?L3a%Zz*m{^O>P#r=@?5vo?2VTa0@w89GI0i zOF0Zru52kJbt$SVJ1}d!^3fSMRWkZ8yyiJo#;L0cRNn)-yEJ$R2uy-bmyP_m+kP4N zR%p5OcA@J<;r;DiLn>{(KN@W<_h$lmvIzNuS|Bct!iJ9WJEp;FoSUFc3@upaGxI)k zzo(K-G`X_jtgiHA%MlU;R6pHY4C2tqk;VJ7x+%BSI7?#$&lFb$M_DB?Ugh0NJy7@! z^qnI)R|3{Yd+pHEH8~1vLrt|RA7hwgtKXcw94S!SW^$XwFe4O*;(iKlF1k3ORA-bE z-<$G0z|pCWtIn^oG}(avR7sw+)(O>E(2+Rhq(1Qmpj@hJO)$jVc~h(<69;p1myFDJ z<=yhZd(8Ie7bZKu2{1Hs<|!Bha;jfh%ZJZv4{CMhQRL9*z*w8WqjTooBb4TgX2>Yi(^RUg;)Q=I_MwHn2T=>QO>SI-$XwGm+6GLPi;=7|r2 zdg8gctOGIx3CNZ&cFBz3?T=-`tbBG{6x28U6k1bKB5`AU{O)vb<@2U7RSR25=QOLW zlhYq{&rrPrGzrlJ=i=)CRn-qa(etrE*N2Ib$NN$VW4RpV3%O;$OP9i-DG*KKL}C2i z=96w?Cpvp9ANzbp+~zarjlHTr?dQjjbi=7^36e7Qb6I_XQ^1ijJjaJCY-&KUlnF}+XvU1Rf%|2sJfLyPfDzJ1)TCS8ce067EbFJ4|$YWW3X4j?Y} zm1nw@2ZT|>lYRn&W1O8Mq~I8v2~f#SKq$LCc*}{OxhF0V{3W@4f~8D6EBRC-?A$AG+i-k>0-Zq0BmU#i5-fe$8)#u zu#p@`%PM>qgWeYWENw4HtI%*M+7FB|+Ht=?y^}(HqLhl}Vfb7Kgr@m?fNcj9Ko7Pq zAqp*~N==&?aiSL4sVhJ7KX%IxIIX_DICNyUHG86({Z1%Dr!&UL&=6fcXo#EaBbFYS^518OA3z%DAo?x{N$$+01fxF?D+T zdt+}BIUm?n2!r%;S$k#Z5Un?Vfu5jZpuAErk%bE|Rk6C5gI4E!=ZRy}my!!9R>H}P zK7I^2mn%FlwtyG=pvP3HEICu-LbPxl>72@YT^<^8CK0Ic0GUZ4PoiS`sZGZn06BPqj>@?f6Y!m5lLAOGtWfb1<;Zr-9QU3i~;KYN8{D z&%m8|>`nQbf@%H48{TTex6v_S)C5x&#kJ%#o9LegPkAE$E%Q#k8*YnOhNQ)U;8({Y^H}E`Hhd8xKYjM5{`4}b7{{7lSoamJD%gU?N}Bts|9D8 zvxZ4_PhP{_9uYOd8|Gp?B{|9TwiU;1#jo`PX(8 zOY2_X)%nPf_9-|pNY3^>389q&t!DSzfxtszWu$Asu8mO7N(N6$NDAT zy{A%Iw(|w`h72!I%t`Hoqq(s^p+HOD6&;z){kcA_KWf_IjbuTJ+S0T95s6Dl+M1mp zpD7&@TV7|j@NntDEIs}?Hof5e`f0Jgg5~TS$w8~@>%5|gH#k0_v{YJ12g*h9N-d~EaaCx?D9~l z3Ip1UTcua^O!NXlG;7o^8;tKyf95wI*+T6T38FQP)t@5mqZYaEqXeF}l;lYW7tx&7 z1*Dg0I535w$}M=&WmX^K*yU8+$G+~LGOs^YU6-YQ;R{c^l_g3i8W+ z0E_74vN_Q}K<5WEcIoc##ZL#!T|0?X(!_(v1UON`h6B2N>D=?RJ zI%o2b!dOHU4bV5i)^+^?(YpxA;$W7NYBw>H2f_EaXk5F!U3`i>XjlvxTJ63iJ^kWD zy>;u)#-C6d3u)*{J>RvpY#hS%;*vMe`%sN+wjdRxBy{kkuWc7zA#iNR3 ziLSsF`ZZ?I8LA@#qB*4{ryAhFIze~>+A%Wiz$|v9MW)WNv-Q>%UDJnq=RUVcD12!- z>N0x$Fibg@^%LUpFHpDzx+Q0FwLO~-rygcb2+dl$JGN-fZjf>LR)>UB&n3?I8=3d7 zO#<rvT6maY26HYn`u=iQX#6-Xn5!PTg?LviX?2!_W9N%x9tL#;&aYSy2>iS_nkx z0Q?<=pCy178BuDSm@Pyvs4J^3{J=XiNdUL3a)%z!x58h5w)F9?Kw26K^U3O;@1h9e z=1?^E!`X0GP|NMEhTp8jdh8+Fkk15RG`CyRilBoTqQ{#Tc?q#l11Iz>XIcPHNgc*mFN$ zlX7*$`*cbALwQ|lwPa0jdJZFG;bo3clVOPqpJS&XOI^4dqGCDY)@Xl$eIPeJwud`I zpyJx)25n?NO^96U*A^2{w_ykvdv_;f*ebxX)da3lJEC7ZPq=mrn?*jH2iJa`-F!-F+#>m21o75~nQ{=*O8K2ZD$}+F>OEp-4%RSdYP#h( z#u5>=yTR7xe>3E%mveR3zfwXiA`iXu5;4WCt)Yd63SrGbd7mi7>XHu8)~V z`i%G&JaMAB;MYS_?3H@syQ-bh7j=~|2GP^3-{#}MSOf6XWlVbWR9#yKl;@}4NFd%c z%Wc?Vor`$qZquD_{$ccTmydCSpocRspr;y^6wq@!wbh~k**QI{IM^xmKCi?|toj2Y zbjta<3Xxqim(`8@60!ggnGO8!;Oe$`#S%V8z7F%m1O?vg$dFVvZb)NIFM7@X9Qt%2 zZ6ySvzYxY`CQ@4)c`iznQNZb~@tIQ~&;m#m$ZN#6%!#bJ2W+bXwp za){|Nx$!9KzG@ib87ji;Z}%_smR2EQ$!esD1H0@CVYyjkD3tc$N<(4ys`*j$H$Odb z^D~>%^_>0QKuvBz;UKAxXmF4euoLaUVCPya{uMLwC7s$&PDG%DQ2Fv}Vnr@XWr&6+ z8HKiGWuS~cyV+d8yz8W)F#K4~`S@epGkq@q;B1VJl#t%VYcr8CyuUZNiZt35IgMO! zZ&V)$Yf@g+Er1A*sewyUq)!V$Mn8R{&-F3d(qxAMQ$^|yyIpz|aoy2PZpGRu$cxZ3 zPbgLzu@=b4=+fvo_nNNRu zpjRB-wQgO1gT7!)TNe_Pu|zs|7FYFnuT2nTCzX1~ zdrlO9;7R`lKKI`gCI5<2;S5zPEP!D29uP&}Ttr5xBqbnfONCU;t?aJ()dz_E0)>40 z&^>xWb&$Rd;LsvsKV?Lae3uL-8#@!k^6SnUn6!=ChBK2J*sR;=4<}DtQDySWWgX@F zOAPy~ln7|t{o~o=f4PF1Hy((&sb+*CpubuJG-6{FsBk+ZcBuz2bwUV6d#Q9PEGS9% zD#f9Rxth$vwz|24GUun7xIL!FxRr)E*y%|c zjc_hRW~Uy}9(Mb_-R5q0b9 zE70avS4P5;RfL6W=}x^qap#UKY>LFD$h9`!xRzBnH-bo>){l8NWh^o2>pYLGBn`ZU zu`LapB;6QX!m&p9I-$x0UObhryuMVd8@-~97fHItOT&TtKEn)aF3EeIkh4G<*&s^| zyla14^cb^sp%Jy=4R2s>`zz@FFQ4$AsfVWCZC}&{vf)by@_vyzqv`8Te<@YmffCNNE50Wib`ufapVrGbIi06!K9@y;;3Wjx_AH%nl2So{zGf}gu|u3aU5+CIFg@UDM<^SD_MRr6 z&wqti+?()uTfUYR_1u?fgX)7t&lC4eN7~XB4b%Jjp8d2ySqvb?kak1D zd1z>p3y}wyRf8rp4G}8$v_9bk!!4{VYoi@$E?P~^R!Q$&V7ry_MfBRK_y>7MQY#_B zia-kFg%iZ1h|(J>Qk0K}TGw%rTQzS=`sydAzPh={8ru{nC}gmxayheF4bsoP`2BD~ zOvC`f_W)|V2BD<<6}X06qM3SQ_i$^P{O+giiX$0Wz0)0BzoQAJqI?BoZ_7j@sv(;L zg)l1|cL$Q~XZAUq5`dLdE>%XcR+t5H%TJ5v@E4^l$xtg#i$X6&@`!kXNNWH{hWZ{& zVVuRP;tL3L0P9Y1KE_767RTqp{mg!>_Lactsh>Ni6H?BzcWgcFQWkmfr96(f@XDM2 zGB6woMoWMckf;eaG3W@3E1!!tuD}KBQh0WJ#Z9t}L{XQ52T$ZWpM1Ah!^tfbpTt7> z*Kd}8>Oub3rvT&v!qrXYF8Kw-9l{C=z;;%37jF1JC8XLxFQnX^Gs*MFF+Wo!_X|{* z1tNP4*Qhdk+^!n;DS6RQi4IdOaZXyciy4)d#S4cM25F1`*8>&9LPOmrEvZsmZ|n}Q z(|1{xc;gI_dVAr-4z{UR22Nihbsty_I+xTG`fN@VI~Pu_>FIv_A4SDKg*DT^3TuZX zb{UG^JP->u6hA@M-!z$-Vz{3R}njJTAbhB}{M@dO{O`qOXX3jRpEJjTOzuh*wAvGy)1QHze}Eo5*H}5GV9DzdxtYN%(XealV3*9uKn)7K+jxgnm=hwN z?rEY@t8%(@Bx1^>rp(`^usPdpe!_TX$xA>=O|szD`aRRnJ3;D6!7lriWZD)5018O3cocf)f1U$w)`}9Rd}JRSAZ?N)heVf zhw1rK1zbI21L_&$SBL_NutnHTUbpAT#8li{ex{dCK`b{MD?*dNx&LUe>Hj040|u&f z>>TN90AxU0|0wi+V*h6=2sYzjbS=6KIgzy{`OnRn{_m~EVzgL?1p5Aw2xhf=IIxj% z5Q4?oR3#?4aGmIPMeDh?<1LC1gyT^;Zlruuw^!#coWI z?kQA#;DxkzV&nu4L^k_-OI_^{04rMEsPRz*twnJ7ocNd`v(gHeFBo&w|=a@4&jW@y#bRZb40;(S1&ZlZ*C??=hMcrpZ zMVq7MbR(X2o^g$@N*8{xz`gigPP;r(Bucx!yzzR0#%t6u(9Y#!9@Y?D>!56rD(g_iEGAff#bU+S`ybp zU+KTSV3yVj;3Bn6)IbllkWg5l<;~qO5N^YZG%-w9{4S!J83NHmI~Y!;w&YaigetN|qgj0u z5cfuE1Z}Jx%zZO4>GqkJsB^`SV-=ZdX($3e>aMncgc|>a`PP4xAN{lII2h1*rPU^w z0)*g!O>Fj+N}zqodSaasQ7>(|>CJX(UA>OSPGg770$Um4l~ z@u+Qmq1$&OgC5i8hUuoN_qy$azz=}L=zleN6%xa>FKIa?TM=~Hl!83I$j!}*ENxCM zbn~v;ZKQ*}bB`JfO{qfh3C6|MYmL7^EhMrM=`gOo03#yFLT5u6P9bXO6KAIg&w}Pz zzo1zWZhagJhweoO(c1*^_U&`|G=fFGY62mr#WXg<70<=G_1c zP-poFj*x3JgB^->6i*<>)?q$BmU};W>&!ae8XrOj231z+b?e2P9uN7Xr zmCAB16Ra~Rn+U%0?2VvEoD)fw7AY83Dz1r|GD+Omq#%6e_Ii_^ytLkCm4celuCypdS0b58%IB2H^&# z`6&j_#X@rI1Yi+1p7Y1 znRas5n#xblSO!Gl%^YIBskzrl8>dAJhf&dZF~cX*uPe&0%VvFgnlQ4lowujv^0aI8 zg>_acY4Y|FV?r};&D*w=C) zt&cUTXr?XlXNUKPk9``04@WmU90QsRz_yn}4_*8>P&T#t+k?2D_TA5pIlf5rh+Amg zRJJSdN>OYU(*FhG=5A}Bu&4msiGEs{0mcEh(3p%L{jX(kn055!gZpSUi2h%m?Ee`g zshLc1tD#MhXFgj&UC+2)`4tPa27YKi6>6we^Id@NiPUDat-CV&1M|neK2Leg?_8sQ zOP3V==M6$fPm*M%LOcODt%x>{T!eEVAVegB@h>zGZyG%^mbYio{`{V=afWGAc)?7x z?q2|e-wPx35g^xjR{jpKNnLU~u}jgm)3*%1#e3cXlyc${mncYYsp9?Oc&%W<@%6v= zEYz&#xQQ^Bs-FU&$`0t(8j+4=q~Xqrkp22`l^M8y0iocc4|SD-2`?R@R(Q6hmh~Q| zX?35Iwf7Yf4!bIxxF7M~8jw-%b}#yq3UYv#UeVYNJ=SrwDF+kukJo*)FghE3K3xCo z^X@;cN8gFSny|(|vCk=eg8}Epm+a>g4xUR1Qng-52h=U>9|1ydjCCzvZ65i8zgg}tNOt5*ioi9Z2uzc}k8Q*x z<~Z8>Vx}{FW<~T3&u$-B25^=Zq-bglC}&Mg$lvN6c0F;u<#h=B35*K3V?&q=p>cuP zuq#`69D=d5PWt)QnXf-X#Cq;aF6mIB!KVM*!UZv#>q_X1>a$7I%czK)6U`DZcLI{T zEhoB}v!MycDF5oid7lZ>M(t?qt(~~+tK>|Q6fUooZ>(k~4yMI0 z#u~BR^y=x9>*N&_^L&=Ad^OyvQ*^g$GByRtW)9K-3hJ;Y@PSWt5oo&vt=4&d5;Zt& z5vZDN*7tr(ne|R3U9_BjXEilbcpvXze5(H9&`03Q-6DYrOxU;up^Rd{%AkqU1v9Pg za8z^YJ0{Egc2yeFVS%5$CiilJ?#&*zd-Xr4(SI-7{Ck`2Us%@uV{WMpw;JICAbs$X}`634(hAN%VQOM<0M8%Q_u0q42Zk=N69#XBe#O z^*0#f?|0=9JI{Lszr~-SI)3}VkTg&&&vT)jQDI_R#w@km2hnn=G$V)P1r2)9cd# z!guMG{|Lkh8R7~TeRrbSnxx)Jjvr2(j?Th{MO~c^Ij8=8VYz#@6|}vqF2KcU3?B{x zxaOCa0`k<0!k`b@XAR2iYf>pth+>_AA8CgC4lWD*`oRWZ(2~3-uQBe{T#8P}OM!*J z@|pJ-zKBSX0+)U)lVkKYVcCXhTDPq&mGGt>N~_Bw6CRfdFoier+7mSf)E1-XV=7$m zE^?v_K@E@FrIowE61*D;pLuM|Gy-onPdPpH^ccrg&(SkIFSsQYL>j4nL+LBs;;IYk zapOE{P9|^?t)8G66%P}dBo;sGw`c6qjVj=rct^2n1g{oGO%Pkl(-XC_rKvCP7A8g| zJ#lgLXgz{MpTVK@V_i?v#SdS8V{h*x4&gr|vXl}UQhvTmG4ZRm2bTewvvTzM_uuIP zh7&NvEm?t=kqPA0Z3IwJ6?zlV3(+LS39==F9>h+}?cr zd~L>aDTS*F9?UQ{K-tt-cK4I|N)=|rWM z-%UUvGQCGEvi+Vfo>sI!EWRF5Y_0qtLuAMGs^ZaY>zWSz?)9e)nPb)ZMy|k|hhA;8I@cbeIT7Xe5 zvwSd|eRg@@U9fN6q27NhDdaZ86s-RJ7$AnMfYxQ23`GBnWt@(xJJSQ^IlRYLvz`{8*a5k;JzJb}&N#Zuj($hI_~N15R3hANR2Kfg+Exw*ddilZFdO-t}>Sid=$05pK9| zH^upkz5T@D+^2kBaAk&b%U1fd#QD3_Dci-71BzCsa#8f&b*!(h=iMtVTOFa$F9xS4 zrkwlAPiF!G1~ck>D5stf)Z{#1zTU3;v0=>DawF){r3*HTD=Uk}iNO3<$?fe9TI0KV zq6;9s_o?n*3_g^6M#fP9Zg{I6Ne-jmgFZ##9DIwDe(NR?bMOjpKSC`LpL$Y+QTjpn zMfX|uC}Vs1qSfX^sve8P`OF`Wy`5TjhfITy5v?Q*$G$prwdqzJZg>t$ur$KG0d!2k z_-JJ7@r|UiSF*uLnOh=zwX}~<=!rNwiF;povhhH6AASi2eDLg`PFp#pW%oIK((`k` zA_@7aWK#DPSXe`IO6GYcl+lLk55lOs{3V2nmuT!xYXdwohi5_P5cKrBogKLNJ?s+Y zq#kahZDJk=y;eY&HuRS|XF&5TcsL>AVyDz6hEeyaZrB}NniCCQVqfxk*=$EuS3Mbo z{{l5N@AQz6gEByi2n-j54{Gq`bik?5IleENZ{puPixXT<71-RnXC5^HO7nFvisGF~ zELkzmQ;{B4@6|Y;eNk-?SQ^jN?R;oK0jpaHfRqe{Goo|?xCr@-6ZO&#AaEn4qQSpF z-=6?=$ql|e9GYFOq~Wm7llONvY-%*7Fw50tZD=+swOP6V>QV1_cXdRRDqZbV3g}SS zxi_GsxCcoMz$e8i|dN8_{!0X^#5^qE;?=^XUCdL1diA$S+W`1bXW% zw&EG3ca95!y@a6LEdB*D1wiS2LCEimq;liCI4GgE1jMeGKe+(TDFdAhZ;PhTy8#AV zFVJJX2r|7P@P2=Resq&7B7T8#%_v|}%-)MiUsSm(mR-U?n%*KM0TC_lxTq7UL)P9z z>_0O-4aMQ+m1#88-tP+)#=D zJ>VPh5MtYA54Lz$yf;O$@!Qr`jQlc7McvE~8EztY?ki`gGfCH`&Ch;A9ioihnKRWj z8bJ*SV&BCR7v&@kk`S`N%)#Z*myh;tF|kGY#}P+1C$!NECGRsA7iAA*YXnu|paXsE zrk){x$hreiTM0PJ&Uso0@C8o8ZekT`D_m4y_czD+>?5DB^x(_I`o&@lAgMh{PdUbb_}NRzp{7s+gFYc_K!BN(c@5ZC90Qy+kR023=J~zQ2-hr z(1@*z(BPT**|W%eU)fjcTKQJTBEKxGKIA?kq2o0m^%mK%HhXfiY^GKrBCt-a1g}bN zps16a$+=C+*wt>ns5t3_qE6UIS#8$GuP@8Q(qvn3d*>*;WiA-Z9Twf9Rxuwq(v<4? zUzHwIgjL|tfU`QabPT=8R)|>3I*`q2SCJrzZ$yV8e&}Kk$zf%X{aX{6Qj_6I5)6MT zb4mBXyUA=o8CnSVrl)NndXO1wKEcRjqNR;qKTtaXb0kK73${Xjd)xrV{sEcLdt(Re zLqAm%n_{cxRLCo$t35GZO3L)SpcA_E77y%=Cpmc^@1dE#QJN_mB(y|lEaNdiK(G>Q89?Ukf*lNQ_&GEsAeUMIFjEJ+4 z^EtP5!UNk#Ipm#yt0wK*Y+koxt^uzt+>|XDpgeISzHSDnQrcCbDXCVbWl)0TS`OeT zW18X}`FEPIR7Z@!RrP?^n++v$Z9?{yOGerdr{=P|*F9Q)fsRD8flg?JHxRvsoi_S& zI(ltZt?vu}Hzfl^5-yJTX1cRvn)4v z<}*(5sBdh_d>kvc-8YjBTE7DVEP>R3xt~TPI>G>ICcqLi-@z2DZ&Ih7x|w2W=q6f$ znhS{d()>{H&PLQHbuMtF+&`3Bp zIq%@ji3d1D6pt=*LxNk4gsnmC@NC!JOtHGFPWdLCRB~hBa8{V{OI9 zicqOCH@A0YZ^O>qE#f;DQe2!1$_4mycWJZ!*IS5x7x(*r!d(CC#qrnI1ICAT-$Xc~ z2^SMH7B95ajKHp;rfj&?B%&Y0eQ8Mk98uXnj}B}jX>}>p9i`WARKo)l4HiXvH@B> z&Cxu*rGxEbJF@b63deq(x(3kkZUfeNgbR-Ckhpn!YKt}W4 zNa?s%Jm2eC2yY)K*l{9RlL=Vd-pyCB8mUX41*`jXc~+1zZeHOs)W^i4mjA=vdq*`D zw)=u8DuM{oI{^Vvs`O3}q=^ve0s<-`O?rorC`geiU5Z5MJ<>aoF1>f@y%TC6#k)P{ z-gV~AI&021=eu*)oHg^0ELlKycHaHI&+}`~@48jLJ2&TpeP6Bh)AaS17s^yoS{bWV z2LJ+ep&ev=h2>S)@E@Wp*<;qpPbVC+eoXG(N#oyz?fZ0dU!8jsb{rJ-OLIkT2+#IF zh3@)&u_LnDbmLpE=U5E_=8NhHw#}#!F9%3MqsVfA^HOJxVtnCDpWuXfO;RXJENUZ@ zJ*AV9^3!yo{udn!+M{?jATsR+ z#{b!{2P(>RZZ4^GviIyO#mzS+->a{L9cOcK3@~Zj^bHHDCB4D5)L)@4x3sxg-%PMT zAaXAOXW?@18tbHFxTpR(r^lG%S(>qqWlxpv!g3Aoc-3*&?gu4C3yDMHP#pZ5_R=7= zDN{)f=SnQawBs64aFQf}fl2;^Bax$k;~=!%pmogfK@%K&&(~x?J81AnT+51;Pm}%* z!-#GGGtO{STN2T2aUCBHOz%Ly{X_I@Y#&)A#&Jy11pW1gsP45Ihv747y(~LDik}ZD zTBRO^kPI-7o=Yf()@y<;IZ2U7U5 zm=#jFg%dth%mt2jPn#WZ}?_ z41}=DrUvU+kDVOV50tQ*C|Kq7em_-k&4|4q!CnW(2T%F15SHPWj`c`)CL9wKuxGn1 zoaf_!suk9391@@#v5dHe5$lrtJmoXlDk~f&=(u|PJ;>lk_`dXN3rpe~qbol&er;SE zS91#~^SbM?q-$yi);|t9TLYp+r4C<@^e)5O?wmC&ll+BmQTvCt*SSh_PJ zdO1HvC4Md)=~=Qmb@`HP{5xY~a9x^lX`}$+)YSm=cM{D44$V>+>rx+*cB#qaBMjgA zw1ATk({uG#F43IT`sZO$CF`BbE~R0$24pg2Cu8@r>%UHG`x@jf30GWzUKx{aBa%Ti z410F@I3qOMK`ZL2J(}{W9m5sg+KM!niT!7RZ){k_Z>~@k#uXbp(SjZHo<00xt?k|m z|8>#W#k`|A5_E11pasMP)-%X7h_o{|2XF+jz1~~5*n~P4o{kE9tEi~>#8Fcd zD8l-z{XJhM-y+|8A|i4k5et9EhKeaoo;MH;n;2*SK8e6U7-H5Q#Bnnln0kM@UKifh zE?Ks`K8Ja*PNY;D>DZNor$fAoibglFKHhAS+;w~H;(_L$HwySeR5F+W2oiusE^Rrv z6%kg0x&!rJF2KppJRP*Mw#%GT72SF0bba2fr){^C)-fva?OZ6nS1q`{OuE(}RKO6J z7UZb3oKO_2SZD9@*O8-eFYK)8L8gUGJDOe8kY7M_MiwPG zxrtbZVg(Y<)CcaSmt#kmbWCKAm3rK|5B~n6r>82D)9G9AfYv%$Jj5CIQ1c!pdFlR& z^8V#sVBv>2&Dy6|Uib!Sks6}b{<2YgO+GwxFq>_~?4K5z7&xb82h@!J?I~(+0e_G8 z!1qZrH3;0FU;**FFzGv`)sj_MNc6e;`fC5tk?WxIAOqHng#Pfij=FNpi=z!CXft%6PET_H2E;>RIGDgE z1fsf70JUjF41O4an*E4Y*OqGEP#JTLqvEO7bX>s4*752Em|ypMLHI-Tpy{;i5&?jC zGZ^4Oxy5_tRdG0uA51C4(7R{$Fs`Levg}#1C2)#RUM@Nebg0C6@4wPDK^RSnTovPc z;Qg{Ni#z^3AmkIqtmxI_0VAuU=5j@n=83t!F_xzxi&Zsqxp%rfLfI@=MmEO?{~mSgCQUB-;wt>s8V#b8m=2Mai;@uW^Fhd ztRtaU@hU1n*aFj$-RsR{?Q`-gZ_Mk5G++O_IKIz5EqAM<*|V!Oa!5r{A5l(-Y4uL; zg4``QV`Z;dimXt9vuj>%fP8P*h&-^{q(uIqDR|z)s*5`JiT?mjo0nS#4^fM z@7I*|_&A?o@Nqa@XsTm!2bN!RRa(E``J9RBsc1iSCn|z0FQl-ZGEU>XL9SO@ZG;7{ zxXqI0@R7PrT!>xfPGpD?>m7AvezvfC-zYmcfh9u)Ix~fz%UIOlYA;}+>Qfh>s;brW zC16Fu#DL9KqtUp?C*}-^zL%U|hm{Hy6cF!a{KfU=-n0I5akb6U4Lfp;ahHEO)qbj$yMDQpAXd?H!^E4W%02kS6$?+(xB~tP zCe7}QJONz@O`E(Ox7!0_bpZVvyW_rg2IS2&sOZIp0N> z30pvvE24a==@E>Mf@|*YohxT&HGm*Rp5SLN0lp08!c9d@_JeE(X7w=f*#MhIv15s^_HVx))#-~{yZT^dF)(gjuP+%2#2`S}+;E}tJUGk9BV(o(|`xM$0mdT>&hSs)Y&yG?}?M-QYd(~o@A z_7V6Ue>2{zb(*-e3s<7oemy=TL3Lm=)CKkI}4Bmo! zJ+4s~LD;ku1~bPShcid`Gs6MhEP!v*+>Z&r>fYNB135uRL%}InH`E zJ+dDrcd6Zhwu{25o( zgmmRQ6(i!G81ShWedx$KMiR*4ZsPM2Vkg_Y82=D8ub1xf9`@Vq!g8Si+!?327=vXH z2lo8v#x2~_j(yMZyZ-^7_m6PyBz>St_};-GauGh$D7Dnhg$|8ao?|Mj_!5vhpo7?= ziG2TP!0KtU8bu$_yokWWVMA~_n4$xG7DNTrS6hguDz#gaGt0kG->Yiw3|escU)4eWSM0I>=Z^ay{vWO%IBXg) zswWF{P-zU}rKwz6gjvZwllYLx*94uM<09ghe$1bHJ_L< zs^;Deq8GsW*CCL>0Dgk38ElJ~0h%jy)^jQjhbGY#Doc|Mf}d)fYkbtU3F&6%zosCs z!S3SZh6voY!rAsktH0HliVt+%d5S}8PxU|4xrvgzzQQK`;^|?dByH!vSxjVbgfg zc}e#Pn-2g2NBT^jb>h?HiSwIdOK|$5rNVk#4??xV9hHdd60$U%Vwg3^ShF6!YT>v_ zx*0-WJU*uLj__rwpYA z>K@=uyx&y(+{wgyxr4pGd_xkFE`A}vD}PJ_Igzty{^fmN%FdwMT^K~M2$k~ zZNtN7ZiT6vzSB9X7rhzDyi@b6cLjz zPJWLrT%s@>Lmw5Al^E;pfd(&e?_ime(E%L)#%-|6V6EWYMK}WhUpM_j1i0dui7;3K zDZ?Kk?t?BM^^+MuPlj*&58K`GzuN9#ZkTg(AR8?5bkA=;_L=E&X! z;1s0J>OQa#$o0yP<>?MFNKv{cNq*%_ChK-o5+xOc>h=rnY-DxxN`b0T9-s5uRP4Y! zDleu_`LCCQc>QlK04`NH0=~1O1<)6p4gle2@IsMm9W*7j6oBdkCKIec+5kzJlyO6t ziQGvzn&VjTH<_;X{hSyes-g=BK_k0Y!}l6?6vSS7c>t(aWcLL6HF3% z(kIENP)#M(1wY8 zZI;Y*!mb>`UQTJc-7#o*&0Kv54Z`q(a96Ev@IZLHH|<|dm? z2N0qFKGd87b{WbS+Y8qkee>;^6N?@1+Q}|=q&2#i%Y}{8v7TIsx8t@>aDOb4?3H?5 zzixfuhw+{n5P6KE79fz|RGS-|Nw?1zrd?;}S2Z*3$*~Mh`+C429PTh5J@3dnvq&mc z9eMO`_o(sV%V7(<4;P65cO{B1sG3F64q|07yaq{wnxmOfI4uFnDx^)UN=X?P!6Z{O ziRmKchkEzm_m7z+drVc*r`E#Lc?8lZ-gf8fTLdtscewAMzDJ!jv9Sd;8|$eX&s|A2~EP9 z5X({fB#E%F2f5Ml7U=%^VkNXEPKI<}bPiJ#MR!G3eE}`gozBgdWzMK)$tM@8{?cGd z)O!tk>oL%3w%!`K8t70hqHD30sre3eiD4i-j-iB4@r?p9ffAqcx%9pZegc^o)_4be z1FB$kh+LY_K2zjFOEkQgTX^g$gveWq7s)8%jw~*0OT4-ID*VTG>##m_WSZgTf)TU@ z%Li{!z-eKXe0&vop^woDgP~BV0G;}E@*yU zqLpyrpN(Hm=Ys~2q2OfEP& zxvKUN{TSIfI_WucP|5f@BPd*jpm!I%Tlt4*->u2{!u1GG5mR;qc5vvHX$2l}+z;TI z@}M-6;^0pcaF$|0Zp34>V5Ow-DEFHPv#C3GKT#^NbEHAsS)$t%qp34pS9hJYx2(qg z5WR`P+&}+N+Q^>`mBAeNU9y+uK4Ob%Zu~W9W1zouk#245v3A_Sbk zSukl!lkI5;28=AexGIf5ZFA2_*X^*fo115Jxb;^DX|DX7q#ChO?ZVv~(PCSNJCq<*ny(KN#%$ z!f`alD_vnY5_In15_?wu0D%epDT%WD$!bo{WtP0#H?T)@8_Sib*|GFJ#!))$C#-YN zw$-*bc7lCFNIETREO-6Ua%VBemdatwcY5ULH-bXw+rIT~4Tht-FKjKth z7a?NBy10rcEyUyX#6Ko@%LtRgTezCpXQx#Gr^aLloXT>Zt`9XWIb^EFG7KG0_^hBK zwkd1a%OYGJvfUgy2=rM}yHtSd2jfH;)=Y+$m`5$SyRzkMTPPQKzCQRI-S?iWv+wE# znTlv6>#Cy)&7LLy?|ev_T7V_+pmtyumO-BuNUIRn0Pa0acTAqr^P1$jSe+entEW4e z82j@qRnb7m+?1Sqe?;zg{?cp%BX7k(d=9`|{BC{#fV;^+5wt~K6)e20K2F0nj2S$H zoCTq~TG)s?Q+eA)1)Gs8W@hHHa+v2A6ZUZ;Z^MY(xk&Ek74CDpUH2P*P#7iSA>|5j z-$n!>4xcR0U>Z~+I>5|0U-u>KoGNS>A8+kcG1rAk|NV5Y>dorf)fx*k8M=P?*AbOe z6gkz3jras$!G0wzzl8q^y;Y=T<#4+kvixZ=pj>Ap-={>1!tk24c^BF@Jnwx=y-R{& zQ`QB8j)F-Ktass1nbjF{jet-{&P5!Jk%Wd^OxEBL7l?wDSy zsf1S65~Q-RlS7GMTG5 z2YmcL+cqtp$Kb9#BPj-My@LF6f)M^Q3^ex?n>XrMDxPw~z0&MOFD(Ft7u%q}QD*(~ zT8EoXdxFTlw`EN=(fLjRRqG?$g|nM1wI0Da4fOvQ0_VmCpxViMajmEWlJ1R z(~i9mo$5XQIlZs^&H>-xBtE9>5y^>mLn%!%7laID;&o5?GR%eL7iK+#o1_!xI=PH!HoRK(L))y>;qaq z+;_i;TOdO(*_?!>`AJK$4F&(DysiTb?_dPYG7t;Wjs^uf=hRe>Sx1BKv9wHf*Vpj$ z2{-gn_2v9vIUwdz^Gkg4eCnv%B%M0TQUFmI)Cez`+7EFr%6w%lvF` z3p9IyBU@`>Pb|d2|1Jrc202}b61R;sZ;8=eAx0 zbPoxT(se5fm&bs(dL;jEw`7~{8;S+WuhP5?#BMwmVz*GC^L^QD4sb8ZZccu$`eejX z71VQ-s;t>l?Y{2sfz+o7^&HWh=?V($6yS=tKUe$=D0lNQ4ur6EK!z6-222DPcZ1vX zUc{(i--5d_Ulal2Brtz_)b9M5kN)C#P7eBKRsEP}Qr#!mtKJq?k))p0mrovQ$Vj*3 zuk{d4!jjzf_S-D=H~0gDS2p&t(5Bkjj-^#idg~gFd5t3Ceoj&EyuXZ3T?Gc1nmiX4 z+GYK~9qH{kpF~eTdu)@D2n{d;n#$`0%SruHn?>)I14`&EjAIT??hsq@vqE2R5B=eF z9AWPw&ihsdgzQ@qU)ZPG;2VwhxI%IZ+7BGK+|gWTP)vd2A!PaN16uZ2 zE!i+s4QsTckxM0>;aLn!ki16D^k{3A?u2d&YwC2X06k^eOKE6e575+HyMq|Nm=M%v zv^^Xn4ICs<#w2G@)R$AB(6PcHK1Ka~^bHQUD;h%E7OuYv+{*H-tZkZnT9&o4X2+>P z`-9|Keysbb%Z@?${t@oxzuXZ(tMDE=y6g;a+2Crac7A^oM`WXRVS9NRn;7K1s$K4E zW0os%aQMT1W~Lm+_M2_Lrc8Gt?Os_;vqLv5L3*>zbEwjdZHC>BLrKc%^CiL158Mrb z$A(FE5^S%xe|N7z{5hV@5~l}ijNEUz;(hZx(dadQSG&>K==x>=J0<0*=@T};OT=HA z%iy#`=QhLAVCd@x&rzgPmE3ID4ashCQm0s%gG94WfZP%~`SwNPDxOkKCjJT47IM=M za7k;aHI4vd4E>9CJ~vw+m$iClq3}V8ked|nMHdC@81EuyuB-iw>gJ5H^8e(?|LyqBL-W8Vg{0eFg^?LVtj4RSjV}?E z$pSeJP2X=pOAnkTC+p5CF?W;s8Vzx4{J6cmxo_)0Xqt^0c7f6wd`=7C3M;3{I5jdv zKNxovagj{O9ws~oWb=Al&eP6k&II95J^13pIgsu=0(&@S)&s{}2Om?O-itW`prxyy z0qa7A)b5J4JKb;a?Q>1y;;`~<1~*P?nJcmDd~MMxS8px`@{=`-^ta?Z*uzy$E`OaY zoEl-J($rLZjz5Bb*t~Na{B-oyu!jQQpw_745ocvZ@as=kWHN8CCdBwDeAkr&du|lvr-}=mD{L(~Nb@Xyy>BzYAiA`muv9Cj_zJxA^ zFMo#NZ3Bpkb_eZ`4Pqcztu%Xet7Fig{}Pd))1xzc%ar=Be76sAs0X?bx1pV^_&(d_6-k0xk0Z)_38Q<2#K1uJ?p8Jgh4y-^|KwmVE5hK*vBQBk5k?H{ar1Y^Wf>Fga0;HYFoX%Sodd6N4!E~tC_-jx@JhG9Qz;kuCx9G*rWF#&$ z^=3}Jo;tMWQ|v*%-}nL#ua)l>&Dl(2d^#{iI*mXkv)Bs;LRwAt=92;eqnVvusiV%- z-n$90p$(e1>dvh!O6gyWjSqx7p}}U&G(H({W}LvGb7X1s+sQ_7p1m6!rBJQ`HcFrVNVMwU)~iU`F6GGE&i*ZELWB- zqRzF8Rkv98ZYBp=Vb&F;O`=`?E}770B+Jfw_$4g}DQrSfrh0TyqvZ5KNtK}?BYv%; z(d|js@18f{D8DH?cRlsTY|F4-pZ^I^@ztCAB8HAQ+e6C}`0>T^zOKGv7qS5-rZ1i- zI`hf-SVab3i!Lsm8Z?sFUK%gp$^?;~Awq%Aw*Xwh8zRmo)#e9&-^@8XWoPtx+O$2@ z@x4K~_|0&lhMzyL*0ED(c{?&;WyDkn?7dhy_)sF61vq%lz_?`fv2AbVvv;^9%rN@a zBXmrcFg5NO#@8_*XUaZx)IpNelh*Zx3jf2{lq5-OlD^+QRy*cn%PZHJSa_`8!p==z zMD$7v{rs81^O;inCuqk)qz5KId7%NEt>pkvRM&HgTI=dPkyY&ag)c#JZ~TQq<=QRS zhM;Dd78r;2OI2XluDuAOQ1N~ywW=ul_k=_E_m`!BBc`H+GX1=e3pQ{iggb8q2XqZ}N?=s{@3J+QG znb^gQ;}+8lQ$i*6q1f6lQMvrm96rNIcdbsFdCM~Ay)y)xrw@z)`fPMhe1Wl>rYU&u z+q6UBHS@kS(t9%1fm2LISITOxKfJ6`s+>t#B8?co`58^gL z(WcIbY%z|d04)p@ZBbd3btDV=37Y~G^6540pl~<^jCPB^1gDP9jbGZ4AqANAHQn5o zqDJD_5lNYuZwUMRR(-Ep0qTv%f+h?iKz1vi{pCuF2 zS;Eu54v{nd1TxSec*XnY0WDPM5U^g$JKQGJ+^(P1&NnqxCwEhIrTtl^75WZ#@I#`C zReY;ov(n(@Xf3>jBPFB^t?K6DSQ)6^yn{q%u~aG}RrY`}2>=z;yNtkp@>?K$?tS5b z`UEntVH>_M2JntejCT%H1)o=GRWenwmKRdIFL6v0`8|pH(=94OUwP-soT}gR0_jn& zZAaZ{P({@m>yi>Eq*`2vj3Hv5LYl+& z6DJ>gl-z|a&&o5qATG?6t7K*Z~*cW80n)o+6p*@ zyu-Q6KD`b#t0^D%y8n|}^J;PE_K4bUz|>?>aN<^0X4*MqQaA`m-P4q(0x}5K(*VQ3 z@MH@3(-(z-`s_#lcF_O3G&{@~s-Maa$MDmCb#>)arL_<^oNWLUQo7b}HMviSy#EhP z{$|XK3W3Zz)Aq(eE4bpcQMUP8PG)kuvF`c+%}|=ppcvHv+mH0C*WOP)ve|HsggPt_ zhADE?p&?O?Dgh<-lQyDr7Ej)|$=x3$%PNl~k|rV}>Hzp!IZzPlwkIS47$Jj`Ib6_~ zuW(52cIGY=TUAoD%jwEgD+LHNzyPo~c%p`|(~*#zs3JS zp$WS%R*5hHCk5MRa8;vWSqg*(PpYI@)zQQ^^&XpSN6PJhS&rtj4#m*a-S>)&XmuPd z|H_LaULTA^-I9O@ zX2>>SfEhHFkH=2Ud9BO`v-<^pWN9)>hgHUY|6TYj#%vC$Pfg=Mn+|mB&6yYR1(L_XXO2G7x2&}x%trwVAhw3fOSJ4tZ{e_z| zFI>zM1GeL+pLdr0zPI;2`lhZDdxR3ZKGwkJ2)l&bJzt4outV6HO?fPonf5xmyiA&R zD4?E^8UFdpcIFjFC+YNiH8;t&7K0+jiZmwO;qMPh7cxSDFk5ZZ$9l_X7{C3z;^e#C z13`Hzhilb829vDEoY)#a1=8`giifzzJu(&%jf_O*MCUJEryE1NrIO_s%LE|$@L(kC z<-@}&NNtM|XWyA&B|U5{EFoTx`r=F6*`O#x|cVx8Cj?HF%$XO<mlW2Le}ZC$Y=L}_E4h|-XT~-fXo|EFJM>&m zqbS{tbqU+m-}1l#K6ziZj=f*hfmBNO-yW z`$k@X%K3Ph_W|dc1GlqHO`dHYci8BqOj(;6P3nYdl0^L&(&?1|g_2KBN>*%uVM5UFXK2sVnujaNuQKMq)f%@-#0RKpDYGd;Eiu*fHO_W|75*hOW0fDo_0caWg=kVjqg4?D-cxe<@_l27FZuEp0V^K`(N$C=rbeP5<*is`nro1qr9VFR)URf;+#E9R zmd@=`Yg=1lit5e8h83+-K68+!OV4|iE|~R$FC@SlFkeIE&$`ppr;sc)73tY*E%H|rz6uVoQT!N_iSWO zo)%pEtC$Ab4K!9j9TTsLcE$(T!7t5kd%S{&M})TfQsSt^2%3`s#~ z=q(q6c`(^3jDa*?&5J(Su@o~nWSeU0DSa{3Uo98R{_{eVX6mv1!}v9Xu&ch8mpl+z zTY^=$W2eNZqiI^2MN#|?i~c4!leam|Ugfh9VMG2;%09~`)s~1Dq-Uq@eA`wx%jy71 z*+G(Rz(J4L3xsj{R0~(O)NkX0Z~~1~!66{3zB_uO zW(f~bb4osQH$`BW=?GP8c{c9E=$Nx>6L$}j_W~BboF-sJQvLUYRKGadZX`|E4!ih? zd@fu;N+XS_>k8yp2A?CU68~z|K{4P%eI%914x&nMFSxjH@4Iahv&WQ7`$u^U?%Iw; zE|qD@Fk}g=-+bLR*@79vbBJxL8BVAh!3pM=4CLj<$!Uy-uHW_@XIo!L3Hpum#R$5B z+6HH>+J@|RR-?qa^0X1YM2}7PzVUq8q^*XGG(E{UWpL_Jnc&?!^(A=-jJ*Y9U_nMT z088&+N!#Ysf~wdP-%;nvMcc8&Vw}6im`_~_$726p!%KOfnz*YTirS^K13p}%#yY1y zCQJd=YRjq(UX|A_Eg>f?DTYkD;CRcw-XWX4_=fd;FUP8Tzuy?!*57vi@>L$Z1q^?U zH-l7-djJj4RR^;vy=7AUcG(o;s4>wWcQ-yiy3u(4sjw2YYBhMb3ny1|29z6_%J9Q) zzFYhy6 z{lS4V{J*xP|E+LsFrjTT3=!gHg8-wVQ8_N&q|9}F{Og#qMy|@Z3FJnqlhCF=C=@{j z1v_FDVdFE=F-t6Urn4;sf$#vCB;_|6ac`2Odu@0UHI)572e?`^T2|(HkF(vH|Afu6 zkm-Z*5jqx*{MmsL>&L)%pxv6J+c75GlI1-_R2lxuDhSV@{X>-S(G)&y3dhS<`Hc`r z{z3{ZSeBIo1`{9RpYwOo6Im>p4oh=<2ZkxMM_bplId%7y7Xuu6B7L(nUnb7Qbi|O_ z{viVXj+67uJr*|a_D5Cbz|4gDhBWs_xzE*;ti$CG!}9DoxCGjG1^O*2(_KzZWZAyO zj6kjd6`d5^8cu4WLSM9qJ)pHv%L`R1!}%*Ne1N`sw{ac_Ud}@Zz?KrH-%M>JvU8k+VQri-1yS=*w@EttlYn2 z#DV7K5&W0Kawy(!3v|ggZWlGfW4Ia6g4F_WwK^!EB`gy*1E>%thv;N9*LLJeJy-_3 z7!N;k%>cHWGQQ(KL|VlTC?=f5wu7V3ab)u{{ zSIjkvBq1;BWRT|AF^^r3rq{5A$!mKOQfE90-)WCE_-z#12d^s(WNv*~z)0d#(Eq~~`2VZhga1!moR~9^wu-2d z{Js+Q{+0gZf9RSCII zoVO5ZLwjW$$_qTH=}@!7xA`-?T_B2%IqiqDxrhSO^!IJA3)`_uV9v)f^0ll$ zZazaxOf>2?Oab!}leIKZE3^`vC@ipUDYW0CcI{)T{MWjcwyTEKUyEIAiHH74{`?|_ zK&D*?e+%numN#@Px2R6BM;r<&{|1C<&4&5Jfc`D+@_8jby=qSNGYJaqIeMeZK+DXK z`0(W)qOU{&W&gF&jDM?p|3O&dpZo<@aF@1#1N>Su$7Wwz$9Zs-**m|3%2;>_5fm`t zUb7h|Z)V5jWhMcfm)YFCP0GvdWPSuL@ILX$KPXVS7(E-r7Ye{_gs(;jlKJqWJDXVU z#lHI~U9A)4&6gpGR3!<8JydQcQdGnz!(T5j!ba*pV;0b(FUHhEJ+(sW+28d!pLX63 z>=+bL*62KFRBFA>=yK&!cM%fGPK>e25)LJRFp#H1?--cEdx%1m!QpDy5h?6b>* zasU1vF5?_-dB+1DY)OH8gr+g)+V#8rG5K9ubxJz9`W^7RH($_PFcFNxr>K1f6^YXZ z)T!+Z_g3^LkoQmx$=H#ewW$wNZ_+8+!ijqSdP~0J)BX^>1pX#VbMLYxO?ETo;m6v~ z7IQ#Ij4#OY#D{|PniaJo0|`Hh@f+nRDhNyjqsLtzg8v|}`8e^b$0{|9-D|I@`RJXb zC+fR;zm@!J;eFy^;4z{k!`GfPsT<=HalUpr)7qZmE+%v0uAFc0wE15vuOr6jiF;d3 z8)S@+ezY@+0=&}{0v$51vXhh7V^V+|fO(FQl15cDWoyL?ssejSQ?KvN^qriGHNGQ8 zbCKi{V_er(_gAE)cluuMF|m7TbLU8(df~uJgF}}`gFb&>zGrYcw7ni41z!9nNe~&K zH!Pd<{6j>w?UiXbv;1Js`HglPFNp(r;)k|BJo*rZhrE`h0f}A zfWUPJ@Ed}9MB1y;rbZZid?grBF5cyz;`dS*sY( zqseb0bM`2t3NV6?*SIr4ZmFxI2z7O*^z#-$LCgQX&C!v*?x(cf9y6C#@Kq4V7H$GDCA(6s#jd&Cj;384bvwZ3H=5d>=3_y z{rY4*QN@s&B!0dw2k4SX;_5cU45{hiPu78-(q15$lAkQemsB)j?gAZ zS2EmQuQ$Z(<}LNBh$5zJlKNj&8_OI0iWZ#`Y;!TijX+U;w-bO?EEqtSf<-A#%3xO1 z|H&@TKL+^a)5b868RnyC_oG{9M;Ov&)O$-dZVYPiq&`#$5J%zb^6Vwi>2~#ql=Vh| zIl9K5Z(^jc>)h~?@bmEt&z7-RKTH;_P^`hw0N8>6AYFGRpPq%9j-!@tqaTYDdk31? zS#+)|$;2_;k?c_L>JH3ar0;}CJQY~R&*@U6nK8o|{`+3A@d2vf&imeT0*wq~=Q@*< zlk>HzF}W#cYL{*J?ENWeT~mk~?Zb-r^GcyZug-(VA5sSyDc2vXh~aBFS`w& z&EOy7R#~l2k-{fJltw_Y&p?S%cOHHz?Wr)}v_2A#*HzzR1h5$ZoS|D|-B^{v(Rwc` zRvp%?ICgbt(*wclaw}#rH65Nr9H!eZiRIt5?Kk`od$1yx26CCq3(n7@Fs_%%<1Yf~ z!v+9V1^i;-3<4D?$H%a1IP;>v@EStucSN59N z84iQl-?;CX&=O2r$2?yXukyi$9;z+ZD5o;U8CB)miTh8)pk1OOnrn-%R^mCN~qYOd1$L;RMKVWnbYTH zo%mAIPA7873!ax5BFBl4`p6H_b48Wcf7>@R^&cy_2cN1>Fu)I=A*OGP%sEkZjHS8s zWXno^8-Qi~T0wWX2bdyX%W%79fPjZ89x9C9=s=M}1u>qBd(H-sJNB`5gN5_;CLMR& zV??D1-tjf$T!lU+YC?KeXO8?8JKO7GVA@bz_G9=ofNT6-^IJGtS-WHdS*^k*CjTYL z?FyO~z@GCI=`aCk?+(OmMNX*1SXTMreuaCkYN4H!om@}ZlV@@*9NJByUcdYfr8i1m zv1aDb+wJa`#ykRqR*P8+Rvyd=9)d->1JP~HzAFMXFAYqgRzzwp|)qmgINFg>X9 zjV?^1`ieRAq3O?LQ~El4Ans+{qyQQci#`F?Paqo-+J`bNlQ!Cw-dDXYr3 z`SsUgx2wP_I)x_(&Z-g*TgT!jl?-OV_nYry5S^BH9`z^>M4j%s1iB z`;ef0_X%;^8O}i0wH28gP4{`|dg@vD1*xzNhj4->ZyZ?SYo$VMf=PbPQ(weUC3IFYHi{QJbK8NlS>`&T>D@)v9`qe_uE zbDEC6RSrCqv(wv#oI#r)REojlDhMGH4Y*;X-OS@OvrnK$(f+ zC*`Q{drMY%eykEsYLvj?^D@lT^0&R@Z4O@nVOmO2dM+knj-r=NMn*peh)A;jt;YLL zG=or}qQ7ohpR{lW$+$m2ztJx10ol*$gRcj^|69JiGRZ;ire*77-8e8F%XN0UH;w!J zjAY`UkVyO&mmB;iZZYi@#4?Ncn8$4hL?@p^D*{u{1KU{xI8sF2TWaSmv)Cv!)? z?fg7j*mGs9E%AkLSt7hP&|3b7#*Uj#`+LQL_nMo&a@JHwlk9<^4fKqXdd? zrW$nuJA-DQ9205O3O*E8Z)i`>c1ZoD zwnZIRJq#-P#2WPLXa#t-j-%hp0ak)*uzOHf^v%PAwixmk_IlqfD}Tq9PYgrQ+7B+C zxwt5zTV_a?--ET`Zvk05FRBe3v|x>viU|h#a1-`vx{V0@)$T5EU2rp&ctlgw=epr- zq5!LZg?z3HI;)M~)Uzxx{$>?gO>zcRNj>{>$JMsn%F&^$i$zpjCE~x$hjQq|R{#>6 zW(9s7XMhhS$Uona{cr5OcU+WNwl!P`iUcVn=R!frAd*Bvfq^I>29T_Rk^}_F6jcPt zNku?Gf&v0ckSsxpj0lo*DuO7PDq=we-{Z{f>7M(}-0ALl=eu+7*MI7-63;njpS9Os zdu{tz&39>L1=Lp)p{IwhmEJV$9gbE=Iu?74>NMLYd3raPGjph9YI^6M>C>h8!G0^D zNt9W77VH4rp(q5#C!0c%=~$6$6;Wv2(iSEqdEG_e`Ss^R;&ZFQtej1)(DtAPZc*ho zBR;*p1xo4ap{;$**|xUh`xDzAG!6T=VpP8w4Hgxw+;Dfow{P#8d}+l?nPOqSyB?%?9Nt(pzeK#(BrKv z%LtM=nNyDP?C*Rc$=7fC6a*Uzd7>XR)v*5Fw_IC; zS}R?J83V*%61d2|tSBG(4snH(T#5j#;&46HhV zjUUxQ$5_ARG7*puO`a;6JKChWt*Wvbco+;;UQvmzv|BC1Fq_}Wtcrg0CaGvp@QUX8 z`23yrM{h-Zq?@NF_ZFp+Knt+=XS2~^-!0TPGRkLHH6_n}tSjB=tv&nB@Q_0BiI(!u zWpBl=UZ`hv_Yge&NS%s@(Xq=u=dPc2o>)PYY^V>q6*k5`qfr0h%p_3_cZqGch&{zA zYy7q&A>(|)=wU_LRs}VTvBcl#R5A642Ua%Wg3g1qqnS=6#YtWXY9Tq5l^Kf2Y=f0=o z-Aa~B&Kyp$*WQ*z1Xdz!V4KE8LbCQ^cGJnpio~SiGQ$})rAxgv8|p&hK0+KEluIJ6 zvDW&urw!xR`q9&UL?{D7CHQGoykk3jQ;&Qyy$Lg_JM*o{g=3K{-?LqGwA=snUfne> zFGg9tuNQC?)9>=&Voy$Kp4m56g+~u(1X9RJ+KB}kN*y7HZq`Wuy9RK z1t0t=>iTBgQ|sL#vX;Xr-pGVaa9~YW>7ZfRh5_+Fg818@VmSegw9}bSlXW#~dQlxO zJJ?hEqh92;b{GR9!P!dnNI%pps)(?N!YA*C&3Nj6hs>{9)xn5BogmUGRr+HJ^oCm6 zk-mh%=t@Znsyx@=mHRUK=^>-yC$mKco*38)kM1d5RDz#OzZ+zJHRh?Qj#yXswx@i- z>P&fv#OfRk7rkig;O>YrFMZv^`|oe$Hj1`OEgS7RElm|&{kCkB7oNK`=oLTt#Mto1 z4mb$8kLFf|dfU!*H>GOM#6_)_E9psHeL{00MKAHp8cU1lryA1>9-KCic>}7gWn+Ek z7gdk?ORKs-@?HuHK;PA4Hts(_Z$%3? zc(UfJmdl?DeMa%ZZyE%p!^r&Rz@o;oV75TYsHP%xYTn)Nt=+2^%{4|EOyeJ0b`F}a zwI20e(%-_#TPvLPGrO-*U_RHRO{#whw*W*i3ZrX2!y@Qn?>!j z>TKD5jwh;^&*7x+#ExOrK8UZ^VepXE{g6m`ZF9>QHMky)l*=1mt8>w&!O-qjOz^;mt-7NnQ_pFTJe$% z@tIV~f=u{I#I^Rg{Lt#m=w!^5L5-`}A|jLDbZi2O7ZZtan*HD@+=cv9bR_GFpW=0? zxLp&24^U2p5^BSAOFZ+;ZV(uS<&0@mg6ujK%m^68M%dWhtJ|p_6=~&>>;Wc5( zd4tGwtR8B#ZB{;0wLH_rA&tK7z0-zd*lCHk3J-RCSRYfD`B5W0x-Usz7swpwy=J!w z6fZzJ#qe+@8~K;C;1po9qlq%88iB|TD4SFUsueuWwYkSerVOlA1xJM zQPD80fy?OJ95mbm#*Q&@^kt}etG{bpnS7Dl>#D?4TJ}IZoJPenq+4aLU?$=PYc<>J z-Y>@KSS?Jb{@y!$`K-5bpQEmi!Pf&rXp5xVAKog|3Vxty~r!OYId~_zH9yta(g#e1TAYTL<-Vmp&hp(K$FyZ^=jQiGdYC`G8$Biqb zM9-eOqkG2<@@S4hky3HJSNYO6KI7XH<3_6Kg>Im9bh95UNLu>AGEDSiqDyddot%>! z{HQ{N(dfF9Ud~udL;Bs`d$5iv*OJlklZK}6BnHz!tE;58nwHm2Vp zfpX8+ZoR|tC3$2`YTx@*Dk#(3t90zn9eUPwsyxnC^+l6b^8#&mCQmPA;~igs;#rWN zEoDF8Fo6j5*^Xtxs-)gYt!>8|`&*2)#X2_*XT(%Ghd@q*Hz>OcIJ}RzZeAjK)y>GZ zr#z5Cjnh&Dn-fZO$*}Y`YuV>bE6Q>>x@Ro!{M~2x0h|YrNU3K@AJMKiGf8 zc$yiDE4;=n)tGRE>CHQLN~;qj3V4FkQ*T?Ws8j|H^Y6TMmd{~)pa6@w;V(#Ss8JKIKd z?78J;BrFzw7++`6$9t~p=#F1IP~T)Vp5lmq@$o(4MJm1pg1iOBm87eRA}EL9#dXKM z-E{5lxK$)Ssjh>4kLZ!lky zbgo%=Tb|t@4@1DYZyzAY_}@r+xS?58xx9^mXf{yJC9F=SM1jk)fF|Y>qhP4|Cp}L7 zRri~m`iDP$VbtP>C8LX>v2sPDl{?u$xamD)b#l0wyfUh6so0CA6eY)!i81 z1Cp$9bZW3SPyd-xJ?zJJGU0}m)Se%1;qnIB{0BXUiipg&VPZRQEP#65j;V$KEFl3; zGipbLKp1QgHh!H6sP(>%{N`Pw(n$shWV^B8llDN^el&4|A#&ns#9rSxDB)APp8|sj~~cm0ntN z?T{8X5KM+l{v0B}Br7XAA`Q;3aRaY33k#bb1YS$pgS^iLL$4D`G> zEx&&9?2~LsyHe4CTb9ZRti9N8&2apQRy6BwIyQY)BmIG`U>J$L*|7F>1K%My>cE-j z5#7X+Xo}eKg{D(RyTcRSgKj=nyH^O{PUK0gBr<`1>`e0ul}H(^;AM70u{Iy?wY)T) zyo$@s^wVsd+q@tREECy54R^NNY*&}#LHT&wY6V9Cl_&gBWSU4_ZAwd>Nc3%9jh)Hc zLH%D{lBXuvM&@;bR@O!@<*9Y%MAk`qB#HL&rLjxqgd)04K%M_2ktSIz9WNrc-OMEZ zSa%k~T7C9K=+z^8R7loIGgAK-^!d&v$7gU8WX{q-?{B?n9)$Bm#UZi;f)l6Egl4d~ zG45bGSZiq3;k))o#3b`U?WK}(+_mN(_dJWc0Dj7$zkH$*Hh!?55_dzyUoR) zY3C`|AJXhc<5+seb;O{cL3l0JWk-gTrg{e1Rl1x^9fQ!EQ90k3{D$sh!Paurb7%em ziVIV9HqyD8;fy4G>nIM2Qtvlg%cCty&!)5x?m-YLJo zouDmu2TA$>t$F)dCF9S#k0Sp3kh^{ zuC@jz)Tm-9b=cwRUa4x5_(?r^`)6OM#=OP~XWdxNQUpBCi7lU`!Hnuoo+Y*d4lNGoTpKAyizX2wCHr&<-cZ)-Ens?oqjq7=IhDHfvh zXO_lOaGI&x;&*u~>4@oDog}*R0Qm*sqpn2h^nb-;ST_BK>jvb+gf$8M=4Um4&$-jg z?ZQ12g~pl18cCNHq^%WpolH}SYiebW$wlZ6$zrI-Yd2_!qr+F~9<9;~($GD7Zt^V1 z^k#jik^yJLTwa@&gQKoUf#FnUCAhSTp6`BZKm~vzO5Fi(Un?k4rFV+byZr=dyEDyc zax1*%dUiZzVB#?r;FB+!QEJ=y*3={Qi{CSZoU*KseP+*Ff%_Ry9$-BEWsoe28Yuv| z|Em}X;s-W~0xr^CK!-s%0&v$tuq9GqRzuE9b+1t1FF=7opM1#@pm_`Ky=1aSx9z=d z=I;=gZIeB%-v}4}2G9L_i2uI=2FM#kK!ZFjT}16DcsqWFXss*{ zHh~lgWuYIA5GMH)1~mf0{}WC#1R*zP!LZz9DcO_n5hJ=SZ(#33+rb<=p>W@a%8lW0 zLk6_l0JPoGB%pFgZ(Go%ElpG7NBTNu-j1UEIhHP#b)vjgn9gy>@X8pY@7D z$YtS+v^@QeIJ6!)H~8%4v%4`i=TOt)!j$^|xa#qD!3n;JO#2x)?i5jzV&q}_qO$?= z;4VqQY}i@?r`*EoBJztufT8~0eNPFN!vQwdKZ998&%7^EK3{Vk$Ck-0pO7)G!{+#c zhg0z0Q5H3 zVOm3c3m%mws{ONfKe>gwn{>vchkDun*LV>k0>Qz)SVs_A!B`)1-NCSos&6;5pGK{o zg{%S*fhXN9f*_t}g?1M{rT}fbGQ7(H-~DI`*+2QKV<+@1{sBP^A%ZWOEn@Oks3(hn z!qzoxGx>Vw`9N0=S}s43dP=-B;b^t+Jw?4%duhKbda*@E_d#0X$prf|>PvAK^J4=t z7{um;TCAXe0eg^7Jcr}y+4J?N@qaQnFI~PpO5I*@jy2pCail@ZuQX}u zxn)hL-ayj%%~CQ=)ihlZE73Q-&5O=s$AH?tf8u#>lC{O8m{}Xo^SyApwE0%0=F@(# ztGueFh!5S=qFLXp(kqv9_R{lM=hC%)zJUI|s^GWxkN)YiC_3;#pL87)t1ej6qowcN z&-V=mR=!+Bg;+P8Dy004PlnD32EHVgv_-VSn-7&Gqv;PG&)o{o$ZaR!`X+}BHufym zZq+925#PYnUI9BM**9Fdk7eTfsTjE5Cp;64*I$y-7Znc%o9*-COKv9H;J@EXQ#CUiW6rd9) zYd?+26--s-ryAWkEr`y=STEXc zt>1y<^ZIzGVZeR@JmT$%FS{<4R_VGz&V6Vk9R3e=2Y>el{wJG1ettVh*ds6-iT+aL z*7YN(Mc&lEv{8k?isX83{BA%;{Vj*cS~r>8KjSn46L)h2<-5ktI1*J4^)RfYH8&Z( zGKuY4g*19w=>t>i(1YRRTsz8v1<21S@o#)O&}jH{KB!q@Q6v4g0PZdnrgS0Gd;l=T zW7=hZ`3>lx|MWFL_4(ia0@~`ryj!pAy^@FW+$wgA*fuQ8J29ST9_uQ>Zc@ln;6awH zHUvs;n6V@?TwJW}dp$<2u~ee&u}-$bGZOjM0&8EKu4ph&#q3rGAd?q-*{Q^Cd9;m*A@JIL;RB>!6P*6up*!fWso%ftM;9Y?y(DO(YfV1X&u5NF zQ~chkPoKPRYEp4Qi0!18NWP`?V$>=P5MrU}PCOacsd|65FrHb-DPr!XTgR_!*?LPq}fGyB)?&VSiA zP;P*!2)tlGl-!&FfyD*BHDjv=E7CwWhx7R?7J)&_pXC;fN{OZf+w0gf#OHA6CGPi(tdp4hb{n=oq|c|mykEaQSs z{H6$%2Nl3ZB@+PCKYIT!iAH_^?nGUp4H^Gnqz6_rX`m`brZtWcGy@V z%zg9N$77t&E^tWrZWX&POPVIbRpMS0B@H#{rp1hHOrk8U0t>KU4v!PgwkfgQ8pqz4 ztQl<`D|BER&QqG^=k>Vk6zYa(dJ*fT7S;8VH_u#=wifm%@_zKkkn?t z8Th)KqWki*GOHk?ipo`HI{UR+wPD7#ld4AELsRk|)NjJB_+PwRGVgx}&s94=uI0E4 z%6zz}XDA1kadb3hv{Zds`}kYz8v)&7qHI44+Y+XgYa{^BP4j;RTNq z@J0c-`TH*@lg?~pdJws8b1Qxag+DrDFuGpP=q=QArNc+L#J}k3z+G;ex*Ac|tnnuT zYAm7}G_)BurEH!Xjz$SS4`dcjjo8iDZ)kq?z5Zju{ogxazgX?x;m`Qzg%=R5ehR3R zQ|g>l*RJKzzH?s^oE=Q|eVw*g*rK6-L`U*7Cwm%uzq78g+4O_>4Oo&;9dv`ED72HP z{7HK4YNqcMFT7Qa*I*0g;+Ul9m>Az0dw8+8d!~|}ecjHsRZ3M7Jg89q>k)O{54^N- zoy=CpqOdtkFA?&hr<3e2w)ii0Y^eB}o9K1ymL*$)bodpqIjn)Xbv9h6F@vo zgZ}Q*_xG?$-k|N7=mT?$g}I*H1Jg zDX*-m!8{O$Nx5|_^NZe;ZAH5lfcrhsRks~TU;kDoyC z6;sx{F-GpPV#x`q_?SjN6q$a0N5m%s8i1Nk-03NNjAL5i-?aRu7hRz^#x@MD{&5)`DztI8zjGsT#r*vwqpdl!P^D7ji!+ zjD8nP?sPi~8m8*5U8a8jyRp#kU({a}rGB{!e$yK${$#rs3uumHTn;HxA$|XTDX7!z z|C%HbUqhxX25$DyCaWEVaR9DZ=OMnB6*+*nGO@elT4(rHp#x;s;n#fDf1InYwO6s- zGi|ZXw?5!1!45ux51JjNzufU=@v=p&mcGtgt>|mEFKFKnyijH9eX-D=yM88TDcTze zAs((w5DK2o3CK*8XNXPXE~U?BP^Z#tIFw$IN9zw$tC-`^Gu>I4A zGd?<-47rsiO}w^3sW%{i&z*;`-;a+Oa0{4Gwte}kUioxd=%XJ~#eesz{Yq%@55IwW zKCXFvPggYw3_5mpiT)|n#44Tk#KNrublWn zg1^duzCz!2*^@(sp?2s|B$x!m?^mVhfBPv>;KGQkDP%oMD1qx9eCv7(1Ym>)1@xD$ z+9QBJ_pl-2x|p!jGC>Q0-@2IMS@!oZ;n8 zfmJjXlBL+EoN;0Ewr$;KEsCf;RygXge|>f>Xsyps&}r%PSw?UE4JZ|*8rJ~DSF3Vr zVv22x6@h8Hh;%$LK$RAJhdJsa8b?h=%dV3SI6w%AKa&!B%sWG**T`cCF`QM1Di85E zI+EA#YV@Qfr9E%c>5sEw-U&8T2Rch&mh2#?zCkFs6qw@ubjxwP`<6>Rqm4$HDqUFO31s?ifNbtRioD9LQe)H&!!X`#QhLEkIj6_6AFNW(?JaE7nluvMA%r9m zI1MJ-Iu$dZ`u{1PfPxS-ode&{tOY_JYf;pq_!h;s@ro3PB}v$Eas#M|WHa?p*ZdQ= z<3BtN|1B~4-z)(CrUMTZAQi6K)RK~rqD_n^k{V54Rja^)AtwjP3&dQ=Ncto!0m98r$lS=F zbO1A^k=G#a`JVpb9RKBAe>*b#3tvgGXE7&os0L7mclyVAY(L>bBXfAZL)t!MzeADO z2A6LQul*D}KTW**Ph0%g14d{ddb$9%Axu;PmEHv410D&Cw&K4t(OPfp=E3ASOe$*2jX$dZn{Pz&UuB{8(2Fb`aGd3IAe+4+b^TWG`}{9K^rl zr)%}d6Rp{p?1aIi6nF2QfiPqNAo~r^$xQA?SF9D|bT;JQ_aKz+?-$fzb;75c<>|U7 z?>$W7%iW^baalcx8*fT4-3`dXC004??#f&Oez#{852#EwP_0e|7oQ92963YzQt$v6 z`WRX#hiXu(lUfvLO=lp(?2OLRcXiv=2_I)a45iQ;Xn5f{bs&#_I`%0=Tim$W4m!B) zF;Tn*YZc0myE=Iya&gz!uR%b+p2fVpF{kse=z~yt=@*d@g6(u@4le9E_OvC!tw(m1 zySa7gveP|YE@@}!`6pMj56x}be%+b|Hu6?1Xy-T>K5mvCLl3`eaOh&;gt}P02d6~Z z;+hEqZ$^aH^%yPC$oC~ku`!y7qT9i$+-vnYR>zOmV9vIQPArCD>HZV#6VD#LTb|Ft zD#j|bsVy~6;~MxWSD_(R%tUEiSbeoi98o~L;8yP2yCMN51@l*~&ptY!c1NI?KN^U* zdZ<66xqo!GFd^7TnV^hDHFQsQua?!`{z%JcWc?(4zs=Vfm1+pbOpocV~h zcNzDZ-EW@-Jx4UR`)kjvU~C?Ziod=_P4+WCot0BzeOc!5z~zJMA+SoT<{0i-3v?s! zjfOW2RVybMOOK`uRy^r46XBV$w3|rq9!c06wu7wg0l80y8jJ;Ti&RB)Lg5%t4}qD5 zZSbq%a&E`4()i1XlWUhYj~zECZGLzpQoXZK=Mhc4xisXfoXc4H&M;n7LJl2`q8E3T zRs44SiFW=&bJ*V9((x*-7twRRl}OP2lzzZ$(B9Kj#OW(U&+#me z@)d{{{TUUXgId#(5Uf=U9E>ikJBsYvhc~5O-*R-J>acb3$g|LM5-@Kq6r8voJol#X zP@kVy22=)3dIxt%k3L(zidosjR(KN??8Pp!HuQZSDAa_1NQiq%X~=bQo`*#Pd;NFXT!0s#wvkwN*{wy4@c)CR0tXfr3yB7LD|B@(6Miy z#3>Rvk0x`T`!cWgW!f23Jl1E^V~8F(L}@+=ywHrJ3x~sSJ5{p#5y=rtPK_DGy{9E1 z{HqgL-*PjcpRK~ka->)NJ1Il81~X{-@g+3Vl;c9t)^fOdth`}~4Xxni0|#kOkyt!3 zpjBwwI9}wCr4TNAHLML4W_mX;Lo>Tw?!)J@W4uLF#Z;FfFhMaQ1aza~irO*KTe#z@ z!j8Xu9UGSXn5}Y_q(R}zTibc@LzZBI$%*kz>AP2`9Mt)lOuQ#3X3x(R5m{9GbzOKS z5E3C#x5n{=a@D^b;B{=tX=n90Ol5tOblhNGb&Rt4Is#77X@!+#WxY z3(ymF7$D0i5~c)%`mLfpIanBad(IVLya$)5YyM?P_HF!dlhKqc*7?Kq^3b?T;bM}h zO+oaGP(-G6TqYpRUjs;OhA&*Mhw;`rV!fIfNl(^IOKj{gVeDgwLi!uqqY@49?XLH@ z$F(dqoCBujB(0(|_5Yp=d-z)#_YJ|Ss)XaAET^^B?!U>fdEy6@{~Nr{&4)++m2(Jc{FqUk=%z8X41?~_{TLwi67U_OFpgKRBe+RB%ik&zk5}v>oHiP zxIGr+59p%v3-0Sr-DN-WLmaQYlI^{ALT6e0JWL`I4c1 z)RBO=P9|-hSdR5O+^$F0(=I{~pyv>3fd^8J3*TB%2}R!_yg<^ySA%^(j_8j54iU2= zD**>>6|d=UB<6HZvJsl-31lF?>kZxeLaH*v0u+7~7<+GHg44J35w?3ksfGI?^;kwz z!xvuQyEn>#RI&_sdJiKRP`!${^m(8!YWpD(2`eN5DER8Z9Q@PpcSvjx#s8)s;w`kA zN4V!rxAPpY2x~&o;0>($*5hwf3|!g|5BoIArIn5HTyaA*#7Z}09D$8q-C4r>VVS3m z!;q}_xJJI?10F|s*SXB1j!{1$drXu{PHr5Uv4MQ?I254bNalMEY;+>NMZe_5Pyk=qVSVRLBuDlY4rIsx)3@&O+aS z(c8-mTUhYgq{FjiZEKJ3)AcBuR7;idYnk8dTKgakkS!qGZ6ii~hcKd+H}!$j`$=#m z+WkkSP1%TYGKj!n^N|q8{xNeO)AVpiPyYRpoBcs2ZpEtKFuDS@t_V2uqk%iD35q{g zN2?HB9-N1)Mc*%ly}Lrakt*Kze1=E9yeh~IyjW+a$j@8 zEQ0A!la*flX?x&Iy)svhBaBQ(kecQh;rpN2h>Jn2`U_N!0gt zfGQ8^t%s9B8@!@B$Q=`wbZW{Wk;$IRe_#lG7ECr(4s*4niQ!Xe!+Sn*Et{rXL5&r) zRHhv!r6F8rc~x<%&hdH#K^cwv3%g8$ki3-#QdI8CYA^<5YkFB+1AKI|$@fc|bZr~e>^fiHetVuEks`4$ZtQ^cKd=f z$#qqY&x2s%Z8_eGsg#eOCJcC%53QAn(FMeWZrs`E{V0(0N&7PdnP6&aI%0a>+S-MO zZKpT?atl|8R<8!@HHH8PptJXYociE}-W4Z=E*3O-o>GhO^Rq=c%D#DjH;dWjLuy`7HSqsmNW5MpHmw*|S>XMSkXqOZ{ax;|NOua1F$X89nOy zDX#unfteUpB?&2=N?k|0%ht4+oI*8x*>QqT3{D2f78VxVbISfuR~0!mHOXaiH=$9` zF7w3mI0)plb_Vny&UUqzAcxo4uuvc~dkuaI4&}P|B1xKgp(H@SnOuOZbgQ@5c?ze9*wx~-TU&_hX%@r96}z*BuL-tuI*0tETaB+9t1xF2jHTOQz(2rIHv3*DXWmV0zk92A^y}J_*?X|m zr#G#K0;JJ1|8K$`xE=qw_&Wl6vU5Ls^9%vgGWZ6`lm~vy3RRL!RirJ%ms!xUp1?n3J{`naIMnlb2tL zqA1nabN>fPGlnN_|@RG zu|}wH%M-cJ>c4W{rf zhCgP;+!6!!hpem+V1dPqYJdAD`p_B#sy#c(@rA#6FesC8yG^JA#4Ei5poa)XKe}9P zV@&6nIIN>1{we-?@GZ?}S2;J9lCMG=9P+y%H> zvH-b*d=x0tkUaKWUFH5KUr9xgbo5fzQN!hun497qEo=az#)C z8B{M9KrMe+63_#gLJl7e`wscbqQJofRLe7fpoM8 z`pc~XCp||m#CW5A-YOvP`waGqW7BuYe{7absWHvz?iDoFU>Xx%>kCXI?prmu>^Mnq z$*bPkLJKOh?FAK2H4$%!xhIK_z#4e@)s zloV6JZKieUi9%le0`--)07XGs2th`h6qtjz#!Ahyx4@&$Ml0|gdxyPSt`Zt&*Dzi3 zbflB=vyir+IhK9ih;xE?lai8516YaU0L{VniT&s;nq0~?)1U5c;wI_+_GjHW%~f`w zxELigT}|YA(Bvd-o}k-NlD}5(akO6G-cH9BkQ@@RvR3J3Yli%;<9hG@(XVB zG70w_mzoClzucs9#KjqxBgk$`FtN=z)X8IEEtwm*ZHC&YyRQu}SbS7l zd9K0bRSf$B#foI5pf^fSj`LFt%TjC#5t-i;`EcnWa@}EMX{<}A9Gv+ErsCzj{n#WI z+5S;&E|J4s*(n|F_hK6_?q)!X;WOF@5GP-0PNRv8rYKxBWE6`DQa(C!s0z%)(LCSF z_03@-RlbqiT5=)*I-ffk)5f{@9+-ADpt;fW0!YxC_%1Po_#^;+K9_RZ3NLC)L=b_- zpwJh+S(X8PjXdM6LuiAStcN4yUaR#_nR`1Z7R;(ACTba;yk@x6dWyOV;K*(w^BaSQ zmSTN~i`Z|iz&Cm-XO^P~UVhJJPIxky0v$T?5yaOm1g`;ex04_Za(`o^Rjn}U81n{!CR>7 zckhp^&-YLp6U4}!sDrX`4=q`6PP@UfaYr3ZcM_c5SXeE2E_zvHM3oQkr_(21?&Fej9qgeSv)%_vXSo6tk>6#RuSAiR`FagB*syAriN1q7+6 zXE2ofM4PID8zATfa2(>pwvnPi_ghwJ4|Ol9#3ca13x-%r;az5mc)gLsCVgW~-SU`PSfj+E%b;SDJ(Uc;^kHCoR;0i^635G^=>3cK&UQlwNFsKLMDv8Vj z)1Du`8cdVCnpN3TOH1TZT_obk6vCTO5-9`vqV_!D40+h9gzqIFn?ik8zh=G~zv~w@ zNqY`grrqVdlCB6jV9;?~m~%pyatSqRyYm$r*Scvxjpk}|nOWz4Ps5}3*go$R&oj;l zVIu0yZG&Ci2LaH$pGb$l2^9X_|MQ>d0I>jB`wh^VNj+0U;0CZ|D{>=1#bpPu)bWOR zKVa6Z0NXc~gZP#IME$e;HiY&o&^HgXO)8j=5l_N)8{_ZH=^{p#4A+9@Svc<%8lMyk zFYWCi)6M_21br84ZRGH-RV2u`03qr; z>zW{bSg0da@v7(Ju70!B7PX+C4&uK{o<;dJZmel%3JYCX%nRAVJbc`0R%EeNR4joz+v({3mp8dDkDhxI4LJniOE}N=Pdp4f zH<`8+0ijjn zh+z^Rb8ygEExb9SF-iwQ62rr>LNjsopAXxmLlKIsnOYeZRvi&iQis>YvurlIdP6vs zu3bI|`z=L%m=J=R6t8*=uQ^Aq#Es8RRnC0Y+1Q#gAHDe6+(72M$9??WlQ%LO^DixA z*oN{-tHEycP`AP>zO_DA*38!=2zyI4B$AH}Bh;LXmdwKLi+_C_Twi?f4r!X(MzZa8 z9w$8=grWieqiF=^+1Q|qu<0-JnWvbqEM}Js4FBSMP{I1O>sWvyCK7%CZofWi4eHIZ zQqjA{V|PNDwcUrjc4>P#xo^up(tY;jT+6lKnnal7X*KEzSZQ2~SWyh@kh0Po>B<;~ z2i?M*p#3vp?K|Oj>{0_(I9_8F>u;WRRfC;H*nk@kfwAzeHVB86u1CGZekM$C7v~s= zSK!7k>FbUgFiJ6YY3aSE!ZzH8P@h5B!tlZ;3!F&tC|rGmV@wF9c~`AuOVMsla>5q~ z;NT_SA@v`ureT3{&?Amh;=sV0tD@j!k*nLI5-P7q;ZM$<5<>g=>q-x5aya? zM)=^_^|7||`1RMzs$dXjzvR|4k;To^Wr-EN z=jMHx-cSgrya$5@RnGu?1iv$ku#jpt2(%OpZsN)sK7=dV8`HWKIj0nzxOsuDsr#7c zBbw>ldI(Pj37~@MjNzRot!n&q&$zYyW1^uuN9>Dg4KOtw;(vl5i8wf{dca%Ls1Kn# zM}=d^D|CR587t5*=W1>Y(yoKQda`Sly+{YLxelgC7Bx$STa|%EUsXA~>h7?YJzGkt z#~oAr-bM_qCJxPYK02N+{xE@1&PnyCGQ@Ay38Mv^?nNhdDs@ zN_U=;B8V1q=@C}g_jPpm!fuV>LcvJC_mZ}WBL&S4koQ4$O{{LLr?V&?nR%VbU_kG< z;ynna-VfTC9Z|f<*UZDo;HKaOSjh_0>)#6V|e48 zkXT;NVMB5Q&f=-%CDWj$ubQrm?ML+kg46_O8a{o&HUT>>WFPtt&?j^>485J{rX$)! z0Ci#H!Kh}h09((x#JGw0}>QJIJz%exi~72{j?ooji#y@y7D@fZN#1!0Buk=a9ccSCcd;$bUL} zW_Hy5sN$m4qe83H*qYVqnFxfz%pz0MjcfJHPpV?S2oqkZaSOcO zLO`U|_;&w^$o~HxZ~W)G_mYc31jmqt+Qm$4XUdlbw3X2B)J6 z+iwdBF845oRBk@2pM7n3#pnsM@kB=-YC7AhT&i(=2#aX}wcvfvm;R9E$4k}bW7qMD ztv4hGD2s?CFw?Jz3{a=T?!)JcL97WvPi5VxBc)F#8?2w7eZWQ57(}OP4xnqL?tC=> zRR+)}{1&i!9e{6ppXaYjzy$b7odO@;fEug;E}!FIy27 z|G&N)WR$Le8}$P|^fIcLd~k6nr*h2WTRLpFlSc1mk`Z4==Ug@^8}0&7d$S&>R)o}X z4~WawG~Rw(E1e^kC0f-AyZk&NY9ivQGA>rk?p9s{#_j3xI9F0KYWit1G6qEr%>A8A zG3K?#%xyE1s&^foNDn*|DY6xpB0@gSPinLFnC!!H?5jQFa>nh?UJUsV6Z1m%U^y7$ z#B#YclQujO@?*RM2jud>5>P<)nL)Z+w!72jI zEr**~4?}XNUw%wWdOcsKYjzRdBAYX>K1{jJ!<&%?Mq;M#c#;C^xDm$)QW1#z{J5~l zx$)!e%2o^VF2lR0Dls>GwNDyA-n`6!<^c%-^fW0K{$*TM6bKNL3Q&tIWfWLrBGVih zc-e=;xA$^T3(S|PC(3@o>%l0HLunNF=h*n2xMET+f_>(>$7k^m<&(n+%%3Xy8V)oa z*JrtVW=f7DK0|Pe{{Y1gYk2HcbjWG@#glnhLQCRqi@LWH`LnEgO`w3*!#MxoMKwNS zoa5Xsz=z`+y-0Qh#-Lkr3i1QoM^?-01L5&wwg)iTzH#oYLDOsCg+5jb~_L-+g#(%Vns#%w_5& z*1zJRep5p7i)H$6c^~z>6_H7atnb*Gz9T$^T5RkJfSdj8`ubm=aLU{ApW- z-|^NOYEs$LOXs_`zBFXHDa-?bEC^F<{o*Qx2_w$oa-#3mx)z$6;7d({4!gY8nguhe zE?jPdKt>Lr>AN-&jfrz2dM`m8=%eITsuUK@bv*g9qwlMd0XKHQBK6#)hJ0KuSke>QQwJeuf7|1woJHeB$ffqwO_)R8K3`xCwL^D5#P~g%3CM(Xxf2&CSBgeDw5oc* zRJ6EIBL%H?fdxj3TcHf2?axuT(Sr~_mA{3*Vb4$%TN2peodN7_ob1=5-ClfAvj_Zq z9$mWi4O?zbvZLJE_-()QUkqh`lPuwXupIlpJifmO;{PrA&=21Xqq>2zRY@uMmAcCE zxYvtWhoRqY7&rGewx!h$7^moNp1c~(`enr}IjI#M#&t_XD&p4jhSm`QANXbI{o#=C z*@<*$W`4?s9$BYNws_!JZJfX`(0^98akZ>Rm1!0e4iza)j_>ueq9ob{_$h`1HakyE)OEdeii&E#>*QN=LM4-1`Dd6gKYS6SUg#8nxPo<&X< zG?=uU@X*uk)7>t9QL=n zSM7IK9_WYYpjb?$5A#n)b8U7${AGqVigh8zBx{n8AhUl-yV`-mfH48F@ck>YnMAGq z%vIC>!`^!bG_`K&!$DM3z=$-dQ9*i9iXbHs3y6q-2q*|q5fG3jEtDi6DoTrph#(LF z>0szc4G;urBE6S@^p;QqDSq2~=iEDY=H4@BerLX0X8cdn_U3)xy~?wm^(;mNH1Ai_ zxPL~qoPT$e!jb|_L61&qt(=8KzFIj$`kfwWJZ&$av#(nB<5@TzO5-$mUWtmxy zyfX>#rzB#QPC4$nF>F7|(^bOyDEC#T$CD@-J?p&_*DeV?{D$6j_#GsF$2q||Wv;jU zYV3p8n_h;g#g&p<*3Y@5GB!a>XL;`bHRDF>1YpRt!F|?S32i?R>>Z6APg#Mk2K4*nXsqE;c44NjO^eBL#OUDIh4*u(Uo4A?G0#BqwocW+#?)W67#5PSOnw)e1!|`^@<+mC+YZU7-zGMWo?k%uNwhT$<9eRveQEQ;L#|HlU;}A^Y~OLz35`c(?wp@C2ErqPx+<7 zUgv)u=)&JegRG+UA~Oa~jmN&*qcf3%j`sCXU6am!-<>j#+dl0i_$92gTs;B&2`Wfd z683tYV?HVgQAi12V5}zUOi0h=-An{4U{3QS$E5U4m#RzlrkhwDoECr1;6y+^ox%@# zJX>-Wh;(z?A2WGsIg`BiFrKGx*_tp-eyD6QOIceN=}YR z;BHTr3|xfj?q8e+J9aNHeQMx6x^EG_bLhixDSZ<|0gO_r$YoePCJWkwo1I+vbqw#% zijpggI)K}^BP9jek=_*h!TozZD;2;MGFMM_d~6~}vh0p{{D^6Li!9C9&+$PlMR}%nEaWxVc9rL8OTA$e z*9-aE&PUA-L(r1 zWuO&FU0|~kaK|YG2n7ZJtCK&vihh3T=Suv~oC%=&?U6OA@9{V#g}|)&1rN0FHr!_} z{XrMs{fz9{L-U67Fr??aUI!BNT;82O(b^^FW2CfjBy5sWS-QgyqBB#@ldOsDHTZpm zaop2tp-{a%?KLs~mBjg5?lPT_I2~wuz)3EkVklJg*MkF;6T0>~`dbo3gi%unU>iFi zIu~h-7FrIc`v8$y@+Z*!-_ce0kBvwm(P+}8mF+V)esq6|&v#Hds~U*z@{9HSHNyxF z#gAu0R?q%u5VA(%8Ie|oEV&Ehxll#UXJAWW?%_+RP1Tfs;edwD$dQYS`fByw#A!+8zAb>fPGcrj z2SkoIjyr5MdmoxSB)Pty2FJiJ(+woX{dw4>-n5Sbw73&+IXvv+=6##)xdl!5xbHo~ zp__Fgk0i#Qvm6NFzK&$sRo!{$$&uAhqA^Xvk1k7}Pv}9Npnxx)aG@F9%c(Lwtx)!MXXOxiyX0@%ag{d4gTmnKrX=e7%$8m6r&V?j-Yy z#pi+MLH1@ac%9q`vn>;liGCy9qXS)T}VR&Md%Xv7`tA<*OyaU$e(j zHY{ljYW$5HHfg_(PMtRX<)AEzqjAio-Up+j)11wv=hP4Rax6=e7p)Ff?B;I=b^u+EVd%F`Z1?#lQn&1avJ0k7KC2+P{NRW|`NG3FuY1IrPR93nP?^DdlHg z`7^Kl^CQF@NeLp0Tp;aDw5hzKxt@h;zaPGKX2WF}ho7?A>sPP(AR>BZ<+|TRXWE#} z%$DHC`0FQIEU6HA1fgMYE4)~&TRA(qdH?Ed<7Y(U0iCIKV|b`-Y>Bi=?oz%2-loG> zehv#WuUu$}d(cC%C$$?>0y7V>TG+o{gm$?D6lNi~DJP3ZNq(@UHAsBv8_61 zN6I5fTO-e0P^BImzQgj^TwmofYv#OzSAhs>UWCzgFuflYy>@1W@6N-9St8!Y2t6*{ zv$YC89ied*%7yHAXx~eQ)XVcD2FD5L%&X+)G1=5=Q`l`TPufGrRT@E~RnbS;$|-%* zG_b-%!dUPl#+|+lszpo?Py_wSabTDctO}mgWoj%V$)hHp7~tj z&F2{@rg28z9hUbu`ZqG}+8c+YsMr>1?hz6-oUM(1s>O0NWM{oQy0_758XT&@O2qOF z7FU<1i+nb7xOJ3O3H4ZI{fuzWlx?gSt4dIW82|dqPQ4BW=KO(vU(cR2OdLwoU#3@s z$#X_Jg|r>vgVOtD6qhTap%8zaUVo;H z?cOlE{AhfqEMHy$*l2cia%1ksH=j*$k=%B@8?X>fAXA1_#zVyrK9)}XI~9b)BGh?H$?NNPpxs{_(_<*@h@WihvG?*PW|0Y!zO-6q1o z-vwm0o_I~+SBlV$5V|D45knMdw3&<6f!z3242miMHla<0O>}NmkUj%FQACe}^Dc7{`kGq`h)R30-9pny|uT}D2zN6cfGu4sk z2tV$My-kyhxrnVXzY@o_pACBH@X>iOLFkHch4`$F1AXnu)YC+hi$s%XAQ<8u#3!_; z)qYgt_;#ipQ+=Ea38pDJE9q)Hlef7qX9urN;X43+K*UR}s!l?}YHv^Eed&1Xpzl)h z92(PS%+`>5*`rUa-F+HrSFfOZOrb_Cc#im`5SPtvN(2J1&tW52L~9Hw0w z>qi8tNw`j060jVxAfp##?uS7anU>3Qt{bFyc-H-hcS_n_jB<>uJ*m^3OiHUjw5MB^wD^dgSrygX z>nEw<$C-Er_kNrql6at?X2OPAb|1G1H;AGMSAA*SE87)Q__l+=(m|KrWFFB_oRn`6 zQgBqqu^@a^<*%f)l$Uve`F*pPdkSfm`HdQwRzzXQY)`rje%~wH`}Z4Gq3!MI`kq!$ zbpf6PDc>!-pyL`5`3YWS7h(P|*Zfw#dC8Y1T`hM#>nTVGAc7c-*_cY~9k`ocxL5i?#+o2!%c-qK z=F%DX6SABSy)0z&MR<~s$jCO{Y6QAAbLtH?=jz=32;pjvZ$PBOY*&k4J#7!!pz<;v zOP9)i2O(1!s=#0pAfY9T%wvijcT{(3el_|3C*@cFw&(444={kdg_BGm+Y+dKz)6+) zp~@NnyQrZ8^sO()073|c=-K?U{7Qd%zJK@i{=SCmU!Ax6=UDzPN=3iDsPsSoNmYC( zbieRvQhaa8EOf-~ZK!dC9#b>ACKz(d?2DVW0X;v^HMmj8dZEX^Z_+x7@BC0#!N>#u zh*@%l&hcm))uWrrl=G$?SS3IC#idE3^m${78pGy>{PWk{`3+*s(Ez6l+zmFTGY71p zv-@7dO{j&vu!*HLc*vUOtpdDEf_R_xkiRvt0?mtXrk+iRMs*2`JX9>uYAzi=i|bVM zFF*6)K-yt*cUg#C2Yn{j>LpIUb_1&1y}nQ-V)(I3KKazJ#ifo*>4nT83_sktH2$27 z9@AHr*es~Ik!w^~sZIwBKvyHIdp$&mdILH`9yM3aXf!dwK6JXRAIIMue=v2|b<8k> zzT@Z9$k;pjTC=hQub~+=g3`(D8%>M_41fEt8~!*`soGaBv_6Xz`|M>&LIh2kg?Ns! z4kDax4Lt~LuKcJYruRaXPcu*miS#q;p8QxU6kdCNkKavmgPbZqd%X8l(vY#xwNzz; zl{F29>Qv&z5K((k_a`QNGW!1ISFg6Xhq}*VgU0qY1Wh4QJ{@Atcp*GEfS(%ccuU7S zk6B@JT+{c{^>e&50qEKm)-p|_mHRF&-_C=!Z$+!jRtRV3II7tOrnhCyZ$%DSoK;w! zfdp)b=W%bTR$8*L^TIh!9QlCT8T-6$cTH)sP5hoeo~$lAT2+%m+pc?7&_x2fWwN$+P|pc|Mu*1)bucXRGCn{5)Q&Cd{JP%iA7r zy>xhH?qtzUkb5ynpKv3y%TC|)SC0#I)&br_XMH7P%S}T$mUyy-_uAVj-wNj{GlQ<& zh|KYYbAn6v-RikT4`~T$;dW0Q4)_T+{kPgf{*zJu2{!$WBA|9YP2XXf70F8O34Pw@# zp_om`vMacM$TylLzdGzT|B?-t>*_luwVRu3k18ff)H~P|ARQ zLjm(oPMY;z=6Ou_N+R=ws;tE`pb0t)@S<-T#!uSDGutvu7I^!P#}?n6Y7{bcRF!v={uP{5@lUDO%h& z2hY7HDE_o*$sC6_!(W`mThhVqtbtF!zUH6%$c^rMowZMjdGI_uUC7Ir8&qbhzFk4 zfi1lor)z_ItM$%|(+=AEzKSW#*vZ9a(M@6r&uY4IvpTS6C|$MuV}YIqi*2?_>a_@) zu&EPQHiR0#_I)cWF?!yVO=Y{h?r2+)LcpCv_&jV4`uCX$+n)#@#{s6`&vM6qnMeHe zK>zF>=zy8|XsES#v|sjs1%Aus;{{lx5)%vUgfJgKxCjNzL=8YaX#hF~FKYm1>^x5_ z5ZMDf_;5hR_Z7m9@l+Ogbr(>c=LaN%B}nabHgh}1o<;`1?Wr>&Vt0-dwbNKJv>$Ue z9;Xms>!<9o6kZ7B$B+$xgNmNL+XW-Z!N`D4@L>!afbIffmgBe3lK>Cs=NFeGew&fsBd6MEwA6J@XBUEk%n|F#0iTZ2UEniOP2u{NB%C_e7nI2b- zS1*LSTVf;IwTm(g62*H)_fP_p5NrCp<`3%B-*h9Bdp?hGBGqw8xZ#Jlw%8LoX5KF; zS6VI5`fQQH5KguWE<)5JkVDn1_n}HY%If0$UKs*#;o^9iaXGfzpaGk61 z!LqFi^W1LNU!_fa*wC-=EWFA{&(~qs&Xp)@wBX;wvsDY&vx$ci%>8YAMos~)W7V-U zo#1ntZ%yjd_QH0FL*5&NWAjr-7#E?Jdyyh?D=@z?UtAI2AG zL?nArPxY;K_%CvP`kIo;nEXt$pVLwTEG`S9~r-OhFx90)TX`U(xT&dp|kp@ zyHV+nFohW86hn`^q0zkkY9>|LhLWi3>FtevxMdb|*gzw<>|120BIj@vob}rK`KB)^ z6@H}^8ok2_1EUDear|@IZL;Oag+b>xjbalv+wthUDucfCDx{oV-J9Wote3Z5eFw#E ztXm?z)K60Qqc?m-X4_dP0))|h`8>X|Z_MsIcZS5;KxcLiN^cCOpQyXnU6shMwh=Y5 zgYUTS_v$2x(C}49%PPN2=+mdF_p)xD8m{$0GOH&|jOKO6uV2qRVC0_Bsh8ccznI(0 z2aJK;mzCDLsq`Jha(ry$VcJ~dvNbe&qD!d00r}d z1oICiQ9x?jbKGTT0jO}zH3KSzvA`@iXr{7skpop<}+{QAH0NJ5~n!=etvz0XklC>BINJRj{G zSvKNH%1yQwPj9iF_IrjR(&! z9vvGMe38a__H>fwS+`>%c^cQ82lG^=T?X742Y9JEJUt9CO1p8VlkJ{P80W&vT4Au^ zYcC=VmlQ6Ycq7hj?bsKIAF}nkF_bhiV6`W3PmV=V^M-~$<6})2CJ;-^U1XLxz|hh5 z*9J~@6%kKkN{s2V?$-onqIlTVyX;7h0*t;IzS*5cKbB&EfP65F68;@@(T{+pFlz$e zzrC3OBX(dp@Uu9C%XbjSE@dHqQMiYNrlvTdLg!uq;FWQ(Og|Kd8B4~=<# z*CC|Y_Ge@3W|crWDnPMgqkj|Z5X2DTBMUG>wg4gmX|FG|Lb!Ix5KTydil95JNMj68 zyUx#h{L_B@T*sg1@#i!1)As!H+ef2oXo@H>S5;Qd&69`?Nj#*OE-Cub$~+Ht{rTtg zjnAIvH#6F?CO{KT6soa#@Ch)?Mb?aPn%U!}{>eU%{C%)b9}9=X{1~%qD;DRf)ZQw* zit679+WaA{o8|BDZVOiUlg-z�kThn$ zT%T&=wFTIv`1`Y0j0PxQK0OWsjSR`}_AT(D!KCOAsmb-^%xS~E?UWtdDive?)UGpY zI}!IDF;=HGFKusiEI%H9VeK5AzS<8?pnM`z?ayN3 z!l0Ucj5c@IXUh9a7QL7SFjveb&Meszl`v>4ph>p!T(OgSGxn zB?kjysCqjX^=0%_cU?)@=hW@&>PE`f)xX{xF6 zD|r@=#g2eLHAk3>ky`yW3n)p_O0bB6qi%!J}Hmx|wD~y)@_5asvNEsZ1_)wv-npYCSCRjUj4An6^54?U8GZ@ie+kmUpCL zyMHa)Hq!i>*@lM8Cnyvs(b%pjx0&7)VUgS=6e-`K!?(2n$4;O5qM{sgS}Mh{q{+X8 zZXoyJc&hhGVIoeZ{{IM@Nq9figwZbyqQ`g~d7919>u-{I;a=yuglPY&gs4v!6dZl% zXxRAcqfc1at~0OA`kMmJa1f)H6N)1WS=XysJr;G5nC%m}n6DM}igT|HR%oC6G2w0| z7)=qpMEMR<_t)!H7>=j(PM!gh##sPAehQ~o<7d+x01W{1VKmtT=nIu+jKSB9mC-a= zXBed%K%fD#lM8M~G7it+DTUB10g$@oJE*je&eaA7CjMDq7;*rTZ3E~yE-*rDae$g= zF3@hv3!u#k-$6?oK-jBj_8nB$3&i!I03^+a-UOI{$+6kMI4DN|^#1e%wr0aqk?4*q zJwV`E3_#-(KX2{-%&J*Ok`>K9e+Om$@qirY6FvgTAIn`EA{2?*!BniR)Ob&7BK z(lY}cnNH>M#Mr%xiL|dQqp$*%rQC7Mpr1aH$Is@ghRYqMxGf!f*7xK`Oo=0(1-e9S z3|b$Lf4lRHrSq@T^nO;-{sw^YkLPj#^T*Nkxc&Y|Ny3F>NW)G*VaK|jwVOH{?~Ct+ws~pXm8woJtLPAe0DMTuUZckGmW^JDB-c$LE9P@g}qvI@IIMHWzl=_Ki_s zgAdg5a%a}Y$yf2Ens*CD|K}Nl|4nS?FLw(RptbuRKZr83SpYmSz7)vBXsqYN%42ie z1R|Qhp5OKn*P;IT^h{k2)Qwc}_aPwBgz=e&8%FgX?PZPL+xE^xplY!t1|ijKh+fzL zdXg4%y8N=0ki7oKZq=Ccm19K~mm3+D10Q$46cKgLn?j0qnQ70=r=C0jn$)l+y{%;^ zf;C4l1M)X!mP~7;qZnUj$1u>1wb)Wto9*Hm)z$Xxl$7mtw5I=;%M>+}4Ttt}4ZnJL zVS1?2w$QpjHu4&nxTi7`eF#CrOu?~}6!1wxQ>CJb$QG<@j*@6|s~`u(ewD?TWEQ5R zW7F-gnPIUs-@r473R8sz;$rHjR2M~xeHGy1E_mw@4=KV#C^VTQ7Mnawk-wlahF}0G`#zc&`W7JqOs2Rwiqz+GVY!zn$=Qla-FX!M|zsVrchuI62U&U6rkYIC_b*Wib${UKo!wlaW6+4MpU8~B8Ru3V6q}fym^V$F_Bj#x<>pA}VWh6| zTH&iK-Dc;d4LfP4iUO&~Kk(W1wg;iVJ?MljY|7L3nK@f9s5&&j6?aCa{;vU$55!ex z-+4EuQa+vSns?K_chz#Zc|;^cS!NLxzzHZfVfR+bXP|~5#o%Yv$7$LM)uFrbgQdN- zUo>72Wo5WFOa?!SJ~UPNrf3@gyYfIr#v@w*sd%OEQRkOl<)^F&5spSKhm%=Zs)nE+ z;zz&BGVm##944Hos@IjdmZ_s-*xanW?DEsMhY@ zJP3u}J9FefV_N^qvk4`IZHYnqi=z8Wx~b`#UKCEE%fJjV;_W#2;CX@J=)I;x_S4F|P?X*&(9V%c1A?6F_182u+6?BA#F0Rr@1)as*c)#bS< zfD+by@IHGjPe5=zS*-Gym5UPT^z%0dap0CUbo=oE)$p^6btZ-1{P??~i!|Q?#U`q@ zCC!r}tOVwS3t+h@+!LKE@7-hNoGxPvhEFYjGMc>WBgqY;XZeT3-!=wU6eAWNvpN(&ZMe$BC-iT{>>hr?qhmkOgfS zLQ^CmO9H&10+w+L=ilnum!j{Uj(@ZV19QC$L&m#(Ec<2$n7C#5z}jX5#fZFf#G^vG z;~nCCn$6dhDW>7cC>L>7)rA^DzOwl5&KixfBxI zp9+)_nStlTqW19Y7>C?B#;RU|!f9w$rt4V%HN(uDg?Rj4@fVhcF_Liw;w=&OL~a1) z2YFUm>~hi3UKbK4KXn#jjv{O+S~8lxgV;|bv?$Cn48^ILG9f9X1Aw$>^iPBPFEbKA z(O(aDgJMRA!-Xg-Tuj8+XwqwiE*NQB;x8&OZOa1{!Cum664`Mxmh9ay#)T;!p55G1 zdsPS%r|KC-*Pt6<#i;0}&=?smzu0asQ<(8W8xSr3uHfMRr?2m-PEEhD!0*``l{whx zZs*53G<^POvmR%rsi{!y?x2k%b%y|2IXysbLq1@v)VYjgwKLUL{oWJK-t>`HpqXz! z{)QS|dJMgQtMYS=(YPM*e)$8c(uC~~f2&4HJ*;pUGa>Yldh2aHubbxjiu&>T_mbA} z%yE<}-n+gw-gUTZb2I()NKSd8)0GlsvFWL@1ONqUNdnl4Z9wl0%8ce1!#FkyVW3-< zFQ9GCZ@)nk6lT%uidNr2W6=OJ5;h@3{0^F-ReysF7&2@jq}Fl>ovjbQHHB`#P=XQY z?e8G@GM3Hjuvnn=RW2BySt_s4x8dZJUw&4L9u1zp^B=AXFp!^l10Tk&QeXV@Ue=-$ zZD%Q<+r8weO1p?yaa3sLPBPqY9PgQ%M`I2q^ggjMH;7Izm|ecE@VV~tts&tGILi0V zulo0YQbq_a1E9_H z1oU_^;}i)t;4e!~R>yMPcYHS4V;Gcajx&gVxu*`Z`3$8yJS_WzAi^@C@(3zH@r7_B zSn6N{PKPV0}XL{L}CTZIb{IO$p}r@4%<%8R&nTuM?XC&|Cz80Xx8ql)pV- zhov4?s?#v`%nQ!xF9Zmo@rEeNN}HB(GgFR}8wa z7!!Q$dS0H23O{b}i|L&M&38E-)PmF=d3%0t#F`%DF z41i|Qp8)y?oeTkVza&7ZNj3*<`TjN2_mh0{R|)>4cZN&O{91gs4hAoNbrdj9Dpw_b*gX#UbB8P=Hhu z&_ataz!70Hi)hjbhBcb#K$D*X>I@=vV-(dNpPgki0riFU=Kw_pC@>t|VQ?}~zze(J z(;i>j*jpvtT%`24vV??=C=?`W?RLHB+AJsk`f}^MzOE(s{mFj--RwWPaOTy*dUEMg z{CJL+w1cZHIpSrbi=d@>4Ny+eF81yB_PsP4T!-nKAI|%KzT1D+&V43Q42^4RoZMXr zOY(ov>EwR;1wp50z%0s1gmZWBSD7bK!>!e?n6}#iXXS}!hf34VQnV&S$vTAg!z1>* z*?|(_r59a;C?~?E?!j$YSR#Xh&O%b~g{3qu#Tv-r>NX-KB-D`zJ`5GnOM9p^1RM-o z?K6Rxt$XT7!Ke_>X8Rx#2p?>5AXR*_+i>{fe` zu#2xPuX4Yzvu=7LlkDR0{6^5izIWhkv;%&S2R1NCoN(+teQ0c9Ft@RPJ==cm+X}OL zi&*FW3syCc^dvs-e!;X>`l#|46-{#_jg;1(B0p^>oc2wo=Bp~md)S1h;m}WAS7M4&v|*gpfqlo)!6aan?SZlCV`MJ$L;chXUCsNR90|+%aAdtj7_>*8cL878x7cCh zGKuD)e1ILM`0{lvRV1oYINd;^pCWC;Z@geW#;nTsZ5PuftOgdY25H0w^FTE^#(ARD zi%XJj+Zy@dKx2$jp#!k$lJ63Y>*7%3BO^SfYc z$tJ<|;{ggZKB_UJ0|tW1nfsrnr;4jRjq#Mx(%+Z$uZgqLv zH#p8hk{&dNSoTY0G>MKgorUZ|ENo^{=;W;lPyhr^Gcy+15)#=F__G z_uPIQ&pi};M)2hjXb+edFzB^s(RQ-38r~DIBRUhZ2L`YEO)2PPoIfH8D%^W9`cPDO z$BVr>tqKyr4D*4>$=9J|pg`JF8tySAtZ_4@rC&Bo^+CP~UwPjW|7Qfr-9V}tg4-w9c1I_9>;oXG z-h?oYEYg7AmkZ!L11m;dM|+ir{47QQp&O%oS+TR!8mSh%qa;(H5aJ!U4fAvdyu^W| zUUBpl52D?4aUn~M&KLpODc3&6@}ChtEq!LVBk$-WCMr#pi)qVjkxYaQE@+Wcmu4E< zh3wBr4%5aya7ScCb(li#XMi(aShNv6)bp4+8kyiy~Fz z*Y6_vmNhU%$|+ae*iN1Q>@y}+bMok3cyy@$J7HHS1YN7UxL!hzLu{H=aQMtzrXnk& z)`zd3wLa;*R0O$mRydt1%;+5~ZBQR-e~RXmNy;ZDMzLNhltCE`^`0J*xc_X|nHS>) z5&MSEWn7bBp8rz_MOk4Sa0l+sj{wXP} z{}br$9(L<`esFLuqYt;>#eWa7*1I}&%%JTv7t?g&ae6M=7S8Izk8gzST`{NuJtqK)w4>1B{*{u^G7xyJMV}LM)|mKQtHxT0|(r)}=HtFX^!>x zz)*!h-U{u%z6croaij2U$4T2px?+CR4*!|SVa@QXz`>~lN>qUOR=Yo(siNOo7;!Tqy?r-rHU~MJnAJOAU=3v^>YT*&+ zDBhWLos9B3XI#!J8lwf6ty*I=pfLPnzEuhl`FwOqYz8sz%nzo)5PM`5(F5m zthi6b{^v>32IY*kn~v=+lmL?Mwr!x$G7#A9@ue7@fm1FqxDX|cFuz6PA$ zvqB6Y%h;n$k5=|F%5Xr5bufFX>?o?jaRYe?X-XH45MXhIX( zNl0g_1j4QXH~y6qT8$kicRDKW7X>*7b{Uv=cZOWr}$!=JtPjk*P(l768pT z=^tUUZ|&g}Ju5W1nfbM45JOm;>Wc0--H9i?22{8vU$Mwku+TINHpc;qj#Da{DC+}2 zs1gTndVwhj>=N)J7ltkaXUBXE&@?d=ELen)$grUx#`6*&sCryH1GJ&MJ7d!iBTk@8 zS?HWj(m+S5V>{!_*@rE8XaaEJoWVO{gJz7weOTJVfSgo#X zXds2L2`tQAB#d#OmU`~KN+=|}ng^kM9|OJRRr*>{1=qUz8Acx5)Wetm3MsFg-s%)z>zk4X#^tv;ORpn;fl>dbRJ`zR#k9!jD6~*RE6o$IE^B|-N!*%Jn4#PJzk{t zVbrmi`XTWx5~csPh2unUy}u;{Ej~kc!FO?!?K=NKCBlDyS(YhG{mQl;GzJj7_kN0B zkN8q`3r33B=>9`O9XBiU|BS`|uAx*ukR=tW)=fqm7VK1sMiQSF&n|o3@W^-K{#q(^ zFT!l>W45bCjottaSeU zo7$1M6t}6VfBUBYLagnP^8y@!Vrpw)Us-?^B=mZKca9GRGY)qBvMV>g!e&DqAQbg~ z+eo{!z-r58pvRZ~xY!#B%#3FxNx&8r{o5A(?%!Z)_dgHBb&&{>34_3mcBX}~J<811 zANpJDq}d%5tcUS3p;T;T?o(kPwkAry;qRTcusd2L*YQ&Ao}l4wk3SSy+s46zQ470m6`o6Y%^2YoJpOC_6~MgqV}Obd9D9%h@Dar8T= zZyKoMywg7mdH74omE{Y%6kESSF~fs4WTn1LwscooN_5-?(du~jd=L;$yjFm`XAZJk zL$i)-GSH#^MVRGXHsdw*jdh3-Rjz*+pQNiFB9f*Rv^-n!YP4b}QtBaz%i zy)PU2v$I9=)E^!^;{V=5i3tc;8!?=7@FKINgz*;e<1KNM;45lg@jKz~Mg)9Rt{EuJ z5>|BhFQJ@+TU0+Eeeol(eE>VuM#K(eH~NMhu6(GMo*E$G>gKNG3h6Y)KKRN~hb<^w zf~TR{g*keP?pJf&TG-ktN!pCvIO0;jyFNdwalf#hpyb*S_wOKCFc;HYH4NCZCpRfu zHG{xEa46<^R-Ur7;Tyb23q)8GpCN?`JE6;_S@VwFfO2AKG?fi)$GX(4%Q<=U ziR+>q_s-4}gXRELmq9e4H&K>eRRm$1=V@}Y%N(2K=`(_jAdWPM8ZR1qf>2RXSs5lE z&`Hqj9*{p^Joe}o1uDaXRThqL>bqo}TN=(%&Nrg)x_k+MzNIJm8+gBCvK5#yfBj-Uc=pb%mZ#3CkTwudBcwn&)qu(37R9)UoF^1c<7x z#>XpO8bM=HLB2DgfSpM$LCKTS3W*jg0p3e4IX7J`&&F+I=eRqk)KE3gJ!#Md>ddO=nMgU19BvTgM>-&kRrkA@JG>$xsAzlU$kn6 zlnL5CO9VrfSw?J{{f^vQ`;J`|g`n zi;s?v!4z|zFS6Kbp;8&Jj*8*0KGwm(YW3c4`MTQ(LA#mO`j#B21`c>34VL%!y}yI@Dq{{1Y_eq+)uNXO<$Bau!@75M zLE^3tmSOHm^B^dh{^pwx%nldqB*G&?DA9|WJq`kYJ^-R@ZKA(}UOh()G3rpqfG7nA zpib*d+u|uSr(%-IlHG($7(f2;G}+=}t^>}_aT6<X8)sXNKXaMivn+rga0F z#t3>g$N^0`^BuJ83j%cHACE#~8sH_HPsl1dq_7<>enzEbPAT7R?TNi1wsIU=!IqY5mLtwNNbk|Lkm{!sCXo|Fat-(QEWh?==Z-H?#QDgCAjB|pW zcpCrNE|l8=5Q*KGalnjfK{M(W?L-|$aO&rIeFvHGV3OOitmJ&}Rd2XZ*Cnc+M)`4J ziCZ+Tl}VR2+=8%D*Ww3QFYCx%O@21WiiLJZzVPVHPvMt{uuMoU>lV%-CN`mAYSx8XBh?o7*s?>9*tdb|P7|!BZhIrdNuC z)5?Vy;nZV{R$rd6JQ@e-eT6Twpxz3FLeV;BVjlJ!kqdT8rO8mNllSn;6ELiEMAXWF zhaVz;eeh9?qNKL;J&(Lv_M*E2h-(sk$;nTcNn9b2UqbtTA0jfHWdc3gw_A9h64R4R z3Tw^aD7wU-HwnJHu$imI2UujJWb)!w$>j*eoAel%ovfc*H)LL9?#rD>Tlb&8xjVA^yry`qB8Q+=yHsKFCFTJ|IOybzacWRQbB*atK{3 z$zXvyqYqMs8q}_b$IBF24Au+KB}ZZ*FQOu^@vw9Q%}`ZO^15LxJA(?xvDKBHrZwtI%CCqVXWUl4r2prle|&j zKornx#M?D4m*6emzoLis&^l@6FRbtm;|tn|`^A#{t$-U0I0HurMPe%m#eQoP0VZDH zLCKIG3nZ77*|XXY-V1v4p;;?YYvfQs`qSH5mqFWR1V|xl$%Sg?0N{Uzs!vhM*DiNB zgJYe%bDw^G4ghNB^YoJrk#sn+4{AAhsYw!NjLnjv1wR^-c9ZDwD99(loyOp0f9s5l zJs*_IO6$SMlDQHMKUMdYtBto#c28EPf3*@O*#*%=b`4UD>*AoHyV$dZvDVxLfwQ|E z_Da6ZNG&*h|HyBvzaFx>G2n1|$v)#|ttT^mHZ4pIlA&QJ+kT7^2mrueAcv2?rMeTX zFm|4&4ThB7RM)Cn(`=Fk1py}b5#JC~#b|!_5r~NYHDFh45ev26EtBAZ_Qq0`7#{hV zp#iJmP)%(fAvRv4C2g0CP~o)bh?_+IUN$^QfN?;DtU}~{W=1qyf+sn6yWPAjo7!Vr za%%py@KXc{`ZeJ8GU$@Fg~Gi9IwDu)TIu4X(vV7(Ni=u>kzJ=Y0<=XsoZ6r3B75TS zMJX!B@F6Xh7h1b_?R%%4`xmzCUw0}QxqV20p%0%(r-$?bu(`xnmc}vCCH%HH3W&n; zdT@+q$}^KFQEM~zmADCoKNW8&BK2xhxL~NJgBpGH^5zaabgDA;E~rA(JGEazbeC&7 z-%-C1dJI_mMtmICoyts6+zcxu-ZUd)`dYT^t0C%pq&yiuX-5E4Ov+pODWwM8X}umz zIZLv#VmHO*HDt16~s!PfYoaCN>&}+^m3> zLSl)~Oz6$DZ*jUVS@Y#SV{2b2mv3q=Y($NRJZy^FYyA3hWQsd6deYoqpA_BpegXcf^(BDpT_&t+KPE{I@19Cw#`udQDQLYY})*ZnX*OKzw zFf-SjcZUQ;Lz<85y?s#eDYOVs`X;C{+TrV33?ZO&Cl8bb8Zv-P{HYr~5kDJ5QwMNU zD*%EDHw1!iwn?fcx`XA%aMH(B7=X|AqcgH`(-%k(!K4)cHT8r1VQ{A+W^n_Ej3C4@ z#^FsE%>hWs?H~gSjAt+co_1mpP3ZxW_Aju)KKdbaFq)#*{~bim0y6t}v>k2-$j1%> z9hEWpKz8|K;^+UVyMvrS*Iq?&(qu^B#estrEy_eXz{szD&T~-SPtd)I289N%g0ywMQXv z8Yjh=#7une-bi}Xe%`Mv&?5A+{Oir7zM!J!n{VVnOS|Q?-Qqy`)GL(3)CqVEnvbG> zIWWDtqKihoWW1uMaae}jpgE9y05BA*?KIBNN$;l~ND)(DkC0b|)Qzy$)FNIu-`iIz zcHzRy5%&N=r8|#iWUKPG?lZ4Qibfn6ylH84h{x7C05q@rj%|eb6O@m9OGQPPmwfPT z_9q+H-k8HTMno6b=99u67``~N_uBPUfJ!h94z!lh0{LV&6G@)Nt;m4> z;F%ac-NE&hZIJ`kPB_WHHuS`%$u zaS+%X)bh2`goxon=1?5#&24jJ#Hq?1`4tgvyFap6FN!EJP+Cwxq>D7Ek=~{E8mjb!5&|in_nX->`t1U;;JtToD;o)i^C}>J$PRpIFv5}p(7v$Swy*?dx_Y|s z(Zp(#7!9+XJ0Ck}aN!rZGLoI7Exw-%QgjC@s}zDMy0hF31?<%N+lA+`xt|vm3yTD< z@|PA<%}9TGC9AI7{qUIF3F`BhavWJnNyzAG8^kjKoX#WT9B^#ll`Stz0D=vP@kzhr zwJc|~tKe)>8=UuoH!0xYL7l?XQ?Tw2Z=fqo2NAl_6U~TkX}fKu8Y@(JwL`}=`0vlY z-$1>_Paaw$3i`jmnx;rA!QCC8z|>8@vJYiW6KBn_U(Bqg<~zA->uOS-Md>Rrn`qF2 zzvNtWSze?Oe#?o@@JZHWnH#Pe-b6ab?`pBcR8mzUL015p)5O0`Gr@5H=28GXP-_$c zAaOqUroa)|QSFX5N{7a05G##y`-E$2Gc{}|$dc}Lk>Ho!`zTaz@jhv;aZI0s9I$h(PPL(jZP(@;kt!pz|Q#v{>X)7NpWU>P^zG8N+gvb3_j~ ze~Wf_?^xs~7sy{|@x^1xKC8<}Tb&sxtbJo0E*kCMG3nQgc>38y=*Y<58TXNr=VuK? z_>eNSQ^c+C_Cx2CYEi0tI9+P3a; zKMwNw4fiW;Fv|Q;&~rQI2%EIQcrty*_z=_akGEnTW;%ck-oZ?l^js&O{E*; zhYrhD1O`g%JL0EC9iAYl{r;sbOUT!H?wj$CA_m9TCV zJv#ZD5J4JPrdF_rnEX*6#L8s5Qr;Xe^>KFw=$tCsdk6UXo zVq5SCtFLN1UFhxJEP0A%uBK?nrsd7F-TdaS2|!)Z?U_+{+l}RNyz;_PDMWp!Bw1_d zs6jz?M7Yk)>0BENk@4QST#si-Q?-BHga5~l{7?Uz=8SWk;mtKtJ8xqA?5=_mdk4su zvR!x@jCIGQe}hIUGCB^;Yb3decgKcx?(Z`Nl^v2us9pU6?y;Zx9|eaGx`1&9j{Cv~ zE-@<@J1)_kVnvW6YniI8vunuwe$JPpeyu(KiKto+YHOouhqO&7JJRJhXbNsn1wdu( z{F+3)wuj5oE)q3THDWhNtj-4^5i#3LP zg)z3dcIDi)-;W`(q?F8QVol z8Mw!>`_H`7|M;!~FCbe%)JPH1ksUgC#&RH?JjnsaBbO542igPx1GX-?Ejn zmTfFb{GC=!b`T$nw1d1|;KGDAEYq+%wss~2}hm)UK)?hOZu;QY29kCoY0RR!OEIx@g{{sFW3vkFjvv)cVnMvkzO~-t4}79seC!-bU^7}TJ=WmI7wCf#PML$ zan!)!s9pJrU8_}){XZQda2{CT z!b5@phXy}fmoIu$3C$5!G-En#I!}|)^Vx(sU#s1k>N?}38f{VAhf4Lf59WBCwwdH@ z*$W>^G_Q7^OuU@?#@ns?>w56fz$*y{n@bld{?-J;7@&Y|MFCp1Pq>4?8P!u_-3UAg zbYwuoV8-A9yh^ceeMQcD_p~C$zCp35388RiJ~>@%j-UA6|3|A&4SJRd$SHYO5VLC7 zr#R1m+iA->(Pzq*57uRFj3&7#C3YFORdNNZ?_H1K;5QyB6x8Lt|7fGC9CycPC$i{! zq@qA+aq^G)5#djeli>)rZGH5(ki*v(Bc5gwzjUJzT!Wo(2JNT(}M`9M) zT*P#c#Ji6`{7g-s+v0lwV?skn7x`BPqv4R(I&Vx1+IG)%wiz_nSNRwFu8ZC3U?`)a z^-6m5ltV6kr!eesXv%+>#1dvL^+a^+7dJj?z4 z@GOu&_y79C0Ci6UfiZ)mWeLW!ynr4Vbbx>q*&>Bp{n-e#aTcCdL>coT%2Dw@WeUKD z1pr!G9yQ+uP!25&{Ue-O-n{Eqbmsl-#}^m*7yEO<6ynpam~(U*0jl5SEs%ks7gdzW z`z4U5Mh>_P>EMRrLBuz62V?;0o6&T;m4g*nCM7}7jUq|?FGS8E_SI=#w_f)6gY17v zkpTw9DWRqa!sx>`!+MpZrCwW$kvVCS%rhA$*7?XEOHZFZrc}TK&Sk!F%u z9x*{#q5b39JBaL~B&j=_K?9TGWJfL_m@?jqV2!D57mfmkF5kdy8CnuXMv^r%P2)S% zyiVK(9}K)hIMr8pZ+ddrqYifY4?$1>-3qbt5t5UhG{xnY4tiJl#yZuIf0DzB<-~oU~!Ne?Lu>0phVj3}J z!}mAn8DgL-YeQ&$UJ=!0>Gzm&=!mSH@D!)K1R#P7)OBd7Siv=CjVy2TXNJ8A$W^BW~kDOHIZj#gU#VqkQ`71 zg}4*sLQyF|L;Df)=)foU$@dU%jNo`k?-13jAe-c}%2W&uIli?$LF2~x9B{tkbssL6 z6wVj5NOnmFuZAbr4sY8H)B(Ql^9#g`4A%fDHoqatFwzL*>q$6C#wV9VKTp{q>SNn3 z`iXm+m|(8tUE!6RLK~9=VsN7fVYpPc9>hb;gxk4sz}PVD3u2h~PPuXh-ch^JPhWl3 z?_{jT*6t@6cFH$C6~=h3B+%hkCU>UM+t|>NoEnT>Bi1iJDy3q)q_?ykboh1kGA6ml zkmEa_wj=o(F!c{{!cA(XJG+pt)Xz@!Pd^pkw6=%r@;y?LKFI-F5Ho*+Yy|0-)S^eb zPMQ^vlgHne(kZB)r%tpyyKd8KGM~YI0*w9xr2eg@g8(RPi#B{U-p7{IsSm^l6CjqB zPg#+)gi*D`5X5iLL+%`~_rG=0fAU@;{00%5q0jIxx2U0x;5TPaM^`Hobe37=Zm#Xh zeG~pfb7CxYHqbm69Kl7tRvr)KkZ_AT0|$EwGl(MJ%0sQ)^S0>e6vwN_bHCqsEl?W1 zV{$!+3kI~XM?JZ1k0wz@EdVbJalTO){HB@R=hducWvp5j-xufT0?5ILlgU=?W-P&a-J?=Zh6x?Qk&cC62que6yatGw)K=X~~F_cP}&Pr^dRv z1ta=;=9ky7m}{E_iZ0A8mT8sVW{`r#bBeq(uoplkaEmfbEq*S6zal=dg->M-vyXFZ z*}JCiNpTqb^OB=S*4jVFg=82t0s{hDn1PZC(OcRNksxP^B3L}{#4hu+r+d9`&%F!} z*HC|^H^$fPBXK36(Tz=Ai^3myL2s=I5|E_ZQ%OUCEVjO}3#*R5b5%+`TtzP8zUofLtyJ!Ek9`q(1 z`uU}iv>msOsj4S$QDUXyMPj0&5Uvd?K+LHY1{%o~3dA$?L>(-3@~Rx7a%Nug+T!EL zc#cQW)#jLk6#JI9r5bD6bl?<3$-Aq6D=C40{{L|%EY+T?%g*jvLqoRojbIiHyTp)B zw_;KO&v%feMwZvd!hU?q%E^}0P2%CP zO_ASyDaE@dO2=%XdW;@_IlvbU2kc@SbLD?*BxIuXkerof64SH(8qX1Zs_7Q&9@*dj zQCazyYyW>oHTHGu(uwIoS8CQ&kJwh#PrCj5W&I)2*qZW#g?@+<=ku7DtS%mF%3nDD zTcpV^Yld5y-p<84h~<;bg^l=(b>#HWld#Oir3U0Ei422`f;lZCnC1h@+s}x5d0YEj zHonnZD&2x>1@kd%bPQxaR)VbxtzR0yp!L0req#Efw${7h6hUo8Z^kI}e8g??X!k&- z&hDfvlmVC=PJ~1yYtD+(<_o+F*H6{17JQee>QuRqs5;nYKab&TMf~SJF6M829OskY zp!gmrc$FVl9t~Y}HzUmh0o5k(Bn4jAwsHFHm7_r==22^Z3t;eH+ZMmji|c z=;ka$S~&{G!gxaY_FtARL=@hl4qqLHnf(S?nOc$3lr64XEq_8`AhGC{qRk-_T=<-h z8*|<3>U!0g2k5u+HHqDJhn$MDVSecfbfG5*pgSvY!xSuay&-Y6|8f|wI>Nu=QQzn6 z_okwJ%B|K_1NOrdwM`GSuP0Sj{7Rt+0|f$QT+1Ux9xIp*@uT>D7zdlMuN#WWcJxz- z(4(2GVZ(RAHtwrxas9E|mZ{g+64nE>HjDW)r3m!#B<+(H#Bm#ta7F_pe!Ub@d&|Dt2`fOUt=7fIgf2(3^D89sd}I(8gRO4RTP8 zFSPauEZ1YWup^A=-tVr(4JGz=t-bStbgkc!KgWusR*~3cu9iM_Aj8W7mnD%F=)uk=?TM$zJILXcER!Z`jmyC}Hvb7tma|U*g$TSjyhV=02fIbgJ%TXfmhmcM z@)9VFJN*1!hSzlZ(oJ<+$9aRXwUf+q$OcYtPNA~fVxLFmUZp!LAK8C@q|;$F)j-m* zZCS)Jk3o;pmzw}r+ZKlhBzkO7WAguTd;fm3!QDDIJ{>~l0e}M5S_9{^4OnK=&6|9ydlaK zWK{%#VNz?Pc^oA?y|tilf5)CzUc;BpWm$o`CM;rKC%60D>dP{K$g}~kIN2@7%v$2M za{wDiaMU_RF?Km4^;;@`v}3BKlZl3gjpO*kpdYS!DKln4FYi$T+R?v<{eOHZf9mic zsc^Ibm^IdIMJ=N7ikq|#Ozac2-Mz)FJ=Z*eHg=>ZdmTXq6fldw3Gev^lLS)bdE^1W z4Lg3*0D@%1KvoMZrG?+1WLdNn(Q_Prv<7s9kB0tq z0X9A>9oI-yXrs6CPTsDEEjdo!et8u)+td^8k1%qq3f&<`7j#&1j}si0U5q;Fk6RGB z6K=Pg>yz;#-D?@-Uwf9kt}~F$5$K6oio68aIH_ex>wIjC(A9mz`hG*hCPl(~xeN;S zx6BPIY(tcr@j!z>597yOtvTBrqgjSu>m+A4J-hM^2!ES191pDauMSSGgO-C7BY-+m zz6<*i`0g)o#XzZQU-=u9_X_x#;r|7B`i~2PY2b|m2N6t1gE3fWRl$#4g=&lYkX!zJ z(ws)w%u6wHYGZ?X=H%Q`>Uqi6=)mR=NFppwLIs7{!~KWAzwk_ zcn1lI`?T<>Uc{NE!XIrV4Y1$x34g2|J>dR5sQR1j*MD3gj1mF-+LNz~=fOtRn#Js@ z3z}wM*zD}0RQP@cL?r<-KY}=}Wr@1^8tgF!+BTiDGN^PYeB9H?1 zQp>DU$a;vNRL~sh?Ttw7ZmdvG{<`<^)5V3TwYpAP{dm!#nKxrRg?JuYxNVV%QlV2? zg2jx5YNe7pMTq)ZY`DRA+Rb+%RGcCF?~ZDD{^BeAJxu-i-u(#z{~LA+m4+`BAgiQ8 z8?~(S3DoHPBbKc7lp>+hl6kXk4t^8cIOC*A&QP9xlT0|>IJ+iU)1B&KX^aH@$(3U! z0=*R!Am7uiB95~30GT;L;e|Uw=g*t!@1f#?RII$kCmdy~>|`uK&3)C$@6wyIu-Q2E zC@!HEHLn{_=~jamf}nTcb_i-`XpPjaSvkxV7fWvLWU4T;ZS+`1m?_ijnxwLVtxCMK zf*~!WNsImZbJ$_SG^Ef#?epJ1=ucbvpLh`F4m3o!an#^=(siDWCb~f{Is+Lezjb9sP@4%9>~8TzA*o^6^9>YT5{%1P6Umm&swZcd|8F% zZ+f`MC}=o#wrceFWM)_MWCkexThj>OTmYWb!iij6Z*&57aqI>uQ{Wf*aq~aJpZb5& zRhcu}NAPLV)|MkGoIzBBD~9pOy&fAK5ZOJN(j>2)NGoQ2k zX=Q`IqRX<$2gO1@?*?DDAH3nfdik8jjl_oM<;aj$0aUiy%ebmzpf3a5@bo=-Kq)14#LNUgwg zFkruj6)v%A8R?H5^$^rvd{THm(>lKtH{r!ql;_43V>@&Ao7+unMJxMo!JzF+kCw5u zq~RSrNo|u&qJ2t;r?pY_n&pv2K9k*@<}{5amg&!t8jVIUPxPZv8E!QI_< zLoD{!NS{F26;2gqdwY2hh%0&I4_lg7eEMP6Wm8z5Jlt)%s_C`(Af9IkaGNZ@Kqg-%Iz{W{Cm}lXZqp;6toV%7;7ib^iF@hO=>O4K z*H1NI(W_qNXsb4*>@>iuadhwgKU^W?kPZ$JbVP_a?%0jRr&w)B!0x?Smq(G6iZgw$ z!TTG&*Uxnr#EE6{D#YsB@;0cLM~;O__%;ZOWmV!ZXt~D?A+2pIQo*jSWzUSQ!y#8p z8k4%aE*5=yRC)Ct=#@s%>*s>zR&)2;hh2M}_tiX2GX5o=QpN>N0mrieXe)Hb5D8>g z)gVP+AQYkk#9%oLfN+GD!jF{7p{y8AmIefD^2#5eJLCcZh7n)j!j&#luZjs6N(mMy zvn4TIDS2p$XM|;&) zQL??H2&lrp#A5IV=u#f^z_D5IH>kW+_%}%NV;RsTJ$Z?XAKVW^UWB6T1KSZ-kBeF1 zVg_iQ$QzSOH(;H!J1s(K3Tb^r)Jd3gjmK(9nup;oSHO9o+If#_cuz1vE9=$E=*lbS z^Gn&*@V{dQ=BonN?;IzbO?7Re$rKte$mX+PE87AV8??S-^>rzCTBtxqUB)BGF$&P~(*?pZxT3 z*j2-u;%Kl>(FFSKyUi8LT3XCiO?1=`+(bh|1npyJ~k&4Eo^VnKeK$jP}c=_w}1-*C^?KJ!}u{fIf%LAX=;{ z!Eda`tG}}vOjc`c<~a^)t~K=uNy#;X)2^=CebrV6(T|#nTNv3@Jc&*`UG&8J2hLhC zNt~88v&c`hpKH@ZcZH7}CyKS)R$ot>l>wa@d5`~U@n4KA=Q3R13&RHnP9raPM%&?L zQJST?tq!ca@=B<~JabvPPt3MacUk9?bn1HckUY!lK zdWv!)f9GPpyy;+k?z?Fph7vd zSo&q~Il0?_{TDJ*S_JuvmjXic2y5aGPxYHX8$247r>f*a*RBM#}* zo8lXx&)jCAtyWdoEcZO~&CML2!AH?Q8%JRKqB1B7;TgfXAQRDhc2Lo4p0Op>Z1&z zEdAB?AE&fm7*S6Zw^e`S3ftrveb5yWb$T!1VDBYy3)uhU=t-;_W+S}$df=EJG@fAX z=khuL;Na>t)F&EwTox=l_o>{=p07wQm2rDKI!jUib&sW_TEic5D=oYROlxxQ^I4i-7NYT5D&8oo%TAXa6(vRm zN#lYgbfzHqJtBCIFL@-ua`E=0S-*mvwj!qm z`XLgVWOG{Rtjjw-!*svZ zOuP{)^(@%-6WyG&=bSd>iuu^3`pkOA(Z^KJ=&iqWQ~XGIkQEz&6R!-D`i6@}kqL-s za+)qoKT=BV-2@5KB(|VZg4qq;VKTvr`BD$fOL{d^rqdD31id001Q@%-!JIydYZf|k zu#Q#BiSfB9$zK~$4@>(#m&@67hnttHJ!L*V2B2Hu?m*tktK|?>Fjl+ZGN+2oY=(+& z4VBlf@iTmDOi~Z$dJOm3TX_8B2Ot~lVfXynBmQx03!YKA}P565Rv(PR* znCoP@@TcKtUu#Em8%CBY6S>^Qv=BmEf~_4Vm6O#T%hpt836Xg<3TfDMVgU?{7r}`_ zFXK`mPsXdSVjR!JMJZCTDxJ&5p=+HPX&wpg-$B$j*a~$$EfPhdlcBN_kXgVLVHj7v z%q-B%>{h|!aXST74(2`zkw+W%u#H@$G-OpeV&|{3E>!xL@SDM+w!4XUCNPf+KEn;5 zq6ai}((jczFGK0r`Air*uV}J8vHl7cD5LzChdz1^qx~arprYkfF{HwZ5^$7E^5GuqtiiQlBlZE_vDmd zKhM4dB`P(*`PUb-k1fE};+3C2B8g56ltr^;?25}>9iCQ-dsF7Y4{WT}Y;DQ7JLKJW z$oBw=ZdXh4vM|IM!hGzkHKBOz?dF-SD0=d+VE;PA7{vv|D6KJU4tTWpf6_36a%iEf zxY0HQ9f7+Dw+JL^FInx_7biw~GHiE#6+t84qYTuz-;?#^fLqHWvxPA{9r4UQ8OuLw z>C?umtwg=%>z;~6-2FklQ_*e^kjp@Z=l=f~sxcZ9#|xUaZfK5`+SXMyhVAa%O!Vfe zs86x8p1Z8h2@Hg_N03V&0BVu|%$Db^u&2)D#b3*Avg~x4VaECnie|mJ1FzY?_D*#g zAE;A~n_dh45$g8Zpp%Mz{VCOMdq|K%(qHuy;9e^nuTgmEYvCCv-f-&D z!p)%<)y!xSe20?@@`x1}kdGk%8pnSlQ@!921zj~6{|(x1T>%i1YS16^ZA7JIGXBxv ziW|+Gv#t|MSz3OJf4*hd;#IBnU0%LtuBwz2p()aw(tI|r*2ZRHC3Ju79wAr)0f(v! zRm~ie;+t-bH#bD$7S>F+)#_;HsvB1>P|9cJ6!x4STD5m!3FFnz*}*E7<1~P7UKmHe z(h=3w^<)7@uOCR&bYZBry~;0{J~uCZtXfU}FU9$P{u%x;z;d&x`P--qH*(DFMs;**e`}(8E|bDvtrnCKXyD$_OV!hwcpP2 z$`;HGLynyyenviLwqU!0k}eivMBS(Y2eFFh zzkWa1Yji2Pk!b(GQ_csd>3kcn!e{a8*2JxkV3}&a zhbyHdN$Hm4c~e}4vQUg4L8ai_oM#z1wa7M*AO*$&w$S;se^TpC~Y!CxgGG&z~L|pU|j-LZW1M` z9M8H7@HV3kf)w1afRrNn3C{mF$WjV&W+Vaful|ScM%jg|h7cH7+Lol+d2kLWH}ZnF z$G}O9dE*Y>nMmgP%m#bm|%>FulBPn+XC={n!T*6({kdP4vUjeK-n&d1Bu4 z@ww=i#Gf*f2QMq*n(i*0dq_3_@?ZPckLBP75|yPuxLkl-&jAc7&769B5Fx$sq7nAJ z)4-3-wTF{>Yi;=RY}yQ|Mv*3Fdxewn;l!u2vhu11v)A?e#fI_gUAs~xKFcc|k@?#- z>yygyqE5~+|ZVBhQpFq|1O6i zlgHJ2j~h;SAGcN{WIeoU9B06Bwz;`+y5rL*7S)56$>&`iyuvpyt-|M(G+!vlB$Vd5 zojNzEWf4#f_z(afhduxe@*5mj?JO=3z`x3|`+I*lFZ}yo^x>6oe>C`yhk4$ShIa8T z4-iTlA`=YB?CxgH-@aU3rkHRBN|S!>lf z>#0c7PAL%_fsLrrmucF!?Q`Dj+m)$aYo0jqErt6h;P5|Q;Wpaf*Ab7bcE-0S^$rCR+hebt zU{1n`hgG(b?G}12vx6k+UvPB475Xvc^bli2x>kcEq?rWRURpbW|BzKIj61|np1y-= zErD@&pl!3P9)%V#BQxESE3e}(&poHU`lUxe@OE+^$8NB;SnI&1TLYmdOCW%|T&idB zz(Eh~K)R@bfwp=lIxRB2_=JgzI{=-$>|ig;eetG~S~u9{JzvQi&Z0`O!#m`Rj4$t| zejQGKJn&p;rmPr4LKY+P0vRzn5d=5XfLzA{wfiOH zycCzIZZ#IFv>fz#{IZrFi^{YZm3aOTN)J0<3Jb(#E%!mlagfezzOm{i%>le%If3h{ z^183Pc==%;UrsA0hx*UEGC#_5((|?DM;>DTLhkV;eTIkY8EAY`TyKJ2(n;9oqRkR9*Z6@?W=d0oA zLCzaL@cp2l7hb7I=+Q`x0xuP{Q-f#(lqVxs0gHh}^kcxKNb$n>PSt+oBpm(0#|0Mc zA1r?5bFRxpUR}(i2H3QWp&QiqRgl~L{$q)k`s?MFn2lj0rIIN+zJbum!ZW+2#p^h? zWy-LlMP{6`VW{r#vE%uU(te(s@6h^jMO#0LdE5B;c|jrkJ@3pR0=0@!S?c3ane3yI z&ou!#N)&)R5*X1|Ek@09nDqjzmg3}w{usex~HU>L7@H)1C#J~o4qA< zHg={}chMdhLxg2q7v7uThU+PldK1V1%#3I@nQHIP>QoPpXBang=N`26fACDGXX9@E z$?Z7RE2CH{z%Hwq?uYzwdUl%f{9rZz)4}4|5#gaFBajK;fuo2ksP7oHx49!#tQlCd zPPePiby*u~-n%)mJxG>q*7?ajgOBX~`BIZr6;WS(#30(QW#wn;hs43#e*suSh!x@0 zV}w1XJop(GY+19ReI}sdg~u5%U~tsi8Q{<{%e9I><R^~G`5#cuKi>QzkuC`yMApg~x81vqjp*-b#xP9c7QJMNYVqt(r#PI4S9c ze0$jA%Xtv_(12@$$Lm;(JL!vxj9r)C1<}qny$QEZqv8RI>m=+_O4P#4?Nk#w>eiwV z9Z)vq^Pi>0PolF_lPt`>iHFJPrwK65?)U+Iwhfr=$5r82aKxd2_mz?_l~yxtHw>A{ z64z*B2l;+n_PN1=;c2fk=UU#{oz@`nnh@f(WrSokH1r5Yr-^-dp1`5llR(IC(3&B@ z$Bfs9mz)|AhyZSBM*`q&n-%27;xe+om}e&*|3FG70(oj!0Z^vKkQ1nPS%-(qTL2Q<%nJ=dM+h z224uqNXjNc5)IO3>&X5yk0qv^`CQl)BaCpQ>x*N=^@l8Xp1rwNJG#tCq4NEL&=11M zWxCOdd#rsVt*&Oykz5TFdiE&xxcMY2JJD2vb;0bkYuCpkY&C!PIguzhBQ@V#Ua+zC z?V)k1Jy&v)#nx)k z=TL}>bBtG#ox$8G&x5)#KwJo(W!Bx(2rCaI+?zPL6k@uf@&0KAd7-n3axihn&q%Y{ z&+;>ZO=nilcd0%^uGlHb*j|b2)y>}e!~`XFCC1gvu2z6yFc^)Tg3=vHWTlYtF3PMfgICk@&NHy-0+FrWeEfDhy~WD1de*xe4t+d^rw1T6DMYDMyJt1OV)N z>W@v{f@qWcjox`6-f{?!5!CR{0w>DdE)m{iyqfF1vRBj01Vz}271V4(oASHP? zUOXA4&+Es|KOcz~L=U3kLliIL?sxqlkqwm|H3@az{b>87*`bs68`6a9#j}c4qI5Z* zhlh9K&so23i*YbY6gV>QdK20$?lp~;t=br;$bieMj0kOJ9eww43w1mBkV|)ZTISDr zO%cp7hKOaWsbXw-Hv7ecWhZ&o5;YIUo!Lv+tXm}^Qk}YP8JR| z;SjYfmiZ1P07JSNaVK@+jB4+x`*5x&Z2^YDC}BJ+&K9GGt-&DPHbY4hrlgGg(=^xm z8Jwn4&0@+d!=l5z><|sjp5u3urK)^c+;F{5{HZ+KNa)@S7^!ExZ7g z+uk@Rm9+18$dg!KVgHj1Wpwc_h;d#2%a-Pc_mZo7Khrw1FQNm%@ZG&{AAX( znx|aBvuv)8*`n^98s+wxr%QRSJLb)ai-4jzl7j;v4Hy)VB26**4Kg22|ESdOU$EqI=vjQ(eDe^3#t#EBv%*(AM9~@uq>AX)BD7C`Tc{&7(@tx z9|LYPS^#4{M>rL4vc%Z)N1MHGE2irgJ2B=^roDXcMhO2M-~y@TUIg+uov;+_B;rk6 zD#!8?d|2CeUAS;<7EUDlNm@$6QLRUU?6sAga+8^AVZGetNc?*$&Ks*N2XwX=b$IUs zHDW4)_|vHLH%Lg)>fAdt zdZ)9u=I2n_Ht%f7`(svL)ABcUX5X~(a8`AgLQ+DPib+?-`&o=S_s}4zJb11CZp0Z{h``rp#DA-fxP4coVvQK)=2{fIVDl;H7YNkAGJ-0dD}RRW#SxG z5(o$`cj`-G>)&ch8JrgmgHkn;p^aK47ia)BJ3c_P^}HVcr+C@D#~`ss#AhCH8fUa1 z5N1P0-#f8?);}G3S)LrF6G^%}MgXr^+`6QuG5*x&c9cHt3$LQ;fIlyg8L7ub^wos9vg zGfJ_CljfZV?oB-Dl(9?#CgAA@aSQ}z{0FX941lIX6DUBA!cAB63F>E`IP0PG&F5ja zClzI2dP9m#jvm*|OzI&{h@#n9)%^AMac@ejyD1BAc$Zt|jZ4>webVmT*mGzFhWQ+1 zI4P8z)diS5dbcK}z>F_KBxg?2WComL5W+}hH+t9u4(quBJ|CV4xtR)1L z!=4k1<&{2dF1W1vDSm2Z7%myEJZvTD6>Y>cwIpm1bL5OlBccklpdW_MJ0gk^m4iP9 z0U{$Hi>gA9`}k{DYqrh+i-FV1S{l(Jkw(L zrN@2f|G&)})LCX~FFyz4MXxRh;KG)j1?e9}#7m+`POQw+EVNZ|FJBf>OK)17M`R$X z;sM*j&TKinQJ5lihbKzw{2^Of&yF^rpXACGxd*93R7$l{1yaHo z(33&5juKRYVG~8$q3c#!hXdXC7ULZ~;+QGunB}kGroV>V$UophF~UE9nVSMMO7Vuz zr!JMz978wJ^~Wr_7razM*At11N<1S$(7RAa;TfJ~o!yR=CSjCts}9Z96`Y@3E_YUt zOgg5k58Lof(Ri34>B7Std+tb15WCfG0h+!IUWwf4L+}4MyWJPPF_~XCe$4*edA76q z=Hm2)bDyw}#gdE@z%_tONBC4yCwvhLswJrgDFpr%!1_<3h`2V=#ev_T+vSwRSV_nk z-*S*7AXm-+`cumb05(-$PRcBE1ddes2X~45fD#*mb6#>9W!7THA)J|H?KnaL)8bO70LcD=6S7rDkFX765PU^nq2uy^Fw)m2- zq)c_41i^i!gqwYG!1@$-Y4a}oQanEQ*CelhihIT3j?uw};ndLn4>KNP;$vRuu8ZFm zl$LOemc1S@Ez|RW*K0*_us(E{F%tS%Xyln5edp^#@ZrxiDcvIU&N@Bf~vr zcF7R&gCrbhR{w5LVGtRf-oe4SU`EN>)-}P*O!aFUgDnlqoz^exL7 zpZ-&r6S{GJwiQ#p=Iu@b%wnKR4dD%Fudvr`7hFUq?kn^^bR@U`&#xyJ>wp{xZ2blu zquc@erI{e85OgUSdY}vQ{0(|44L!;+l&DcNWLkFE3_A{sD|C>|Dx6mGc77QylanpN z^@?|^q%=XTqJ(Z`9UKKmYYR`$F0&L)>ub*C-+a0rs=&+wIrgg6j8fsbf0n}=_);my?OKsG zpwX0oIYkH2>ZS-QqO(~9>m!LgmH>r?J-N?j+V!sm838H`^HMaSx zeN4ip)0g~}LgsG%3y0aVHih?fKGKyi>GjEL0atmKnD+W=+oN%q%CaExh<1cs>(nIIY+?NASjU(CzVK`eLPuJo^Q1N^x%TU0NlzIL$&AkG= zVmF>4Hl`q(i#sB6gq( z*8ftpi|a%p*=6tboiD|l>u?)?sgXefbc?*+ZzB@ zF><<{cQ{z^ac4fU@%Ir5)Iqi+Sx z5wD|3(GI94N-gwWW`>}6gZhgcbK5JO!3=>9`39Z%`I&zGlkSQCk?{MkbhUMzUHcw1 z@I9_9Ekatg>MiGfvCOxT2mMMWW||lw+p+D^A8|PhU}S zCMfOSgE$8QDFm(S#*AayX=cF9GfqZTakaZ|4JUZY863Y-XX1m-+&s$~Xa%_F;I{op z#q+pgR5#27jc=bYT}Xp+cvk+OCsck0c*cCs(=8FDPKTSOrJu}a=E{0*1-|j*SPCJd zctu6=H0q1?@=b0m)$p_s@FEbT(Xt1#lfjuK@FO&t>I;Ldc~nl7D?O0Kr8nw z`F*aHi%{92}pR}&FMG0j1SF@maZ4yZC3NpXAlB0A`$Qj@c*r0(6c|eTm^PlDM{f8*v6cskaWn zak~*Bnr{K1Vs-|ga2d4t5H~D8m$8iKJBBJd_bDvpfP;`rpCRW7H*r{0Uw*#pnbHN)&bYx*qgf-YMO`QbGGrr<0PR+ctj5ChM20#v0OB zk7WvbGk*OBL2#6dnL`=4jT_a$O83dK&1^yN29Y`P;Q-T;3?}yl$42lTs(Qa$O}xE6Gi&-iPKjlHsvGux;X&6vw}vT&Aq7P6z2s74 z<DJ?tmUir?1P{zDwPc zoKX&*rJTsNW%TLJOn2#g`KuHMnEgZCpUcJ}@|;d&Yf1y$>}>65=FI2j9#(WvsOM+R zPX!efQFCR2s3kT4Hnu|Qn88BVDrM(>6#O#ug>dZ9G|otS+|8v-hz^0Wphy{Lq-4zs zlDo3#kRcG$e&@RIT6>=VyTUFSorTlNHFw4$+hSAxVYpS~nFL;x)E7t#*saTdGgH3y zae4RtH0OAADn;ch=A6(QzAwW;OCa;2v>pSRH5-wuWrTP*b!hlCS!baV?97JTxgZgO z6$*UG2j6BI5!z^*YqnJ{Sxg4jhduTA$F{TatH~>Pt_{q1r1Rs`2PI zFCOTc|W7ca#jzxaG!8;tf;Z9EkBJvLJx#OhO_mZutv%`aTk`f)MvNrg>c_^;CF zEQ7v^j_|%h>($lc0O3&*7_l0_ZeEb{zwo}HT$gMruymB|AVqvB%3sLx5X{>u{Z7qk zQ$qHCvG*QOO>XPDa1am?B!bdQRGNT*G^IzRi-`12L_nH|2uM#91OzDp0t!Mzqzg!w z8hQ~0rT5-zXdyrn?{w|G_Bm(obd@;vYRyqs!E3mETo zGwE>Myi*_ItfKX|t6t5kJ$PGjYtk*|((6sLmsfA~Zti_E>Q?QUl9*UwrXLwqVG(_E zDT0xi=ZUxmD?ek3*{AedhJtdce3S3TdQxNzVxM{Wa_^p*Hf}6DMSx+)Q-VX19r>U0 zRKP}jQ^`AdKAx=-il@Cmo9=XnRxp-^L1)LF%WdKO3>S=UZ>dVjW8d?Zi`?pbl_M-4 zyGhrRp?S->f6ui5(@A{r`1s0v7@v>K?p>j6%|+9AO_{2l>UF-vv6RI3ieIEJNgBpW z?Vmr@57_I(F!*MPF0tsf=)rR|iAr<_)`$HG+1Y?UVi~O9&S*yO4}Z)^SrEFf0z8Ba z>WvmPH!e`cxJZFIR1qAA{HDum^kHf-Hu{(=L*^MZI)gRm91eO@c6DxLT#Wu4=~< zbwUigx1sY@kMa2U!CVlzn#6d_D*6oFjp(moc@jIDvs`)E$YFT-sJEYTb*%PPy#X66 zF8Pw}<bw{}R>fH&axo9aHC&P6TP&D7pB#=fc5c}5J~DzV54l3FaW z9Hhy#WC_hX(5~2P@o^hx;oe#VzeQOyS+*Et=Bf8XDW}F1-iD)YS97rF zQpS0X>1(hU7}j7(_&qUmDi)HqfNgp4?P)(@xX+$+EbsT`!$%fd>T6;`a64%^I^8nwGU8q;^Dy5pe za~kft+Ab6L1P&?`rZOIXS2{Aqhnrjv{oK1d%v(SAsZ4co(y4ISs(=}h#lE)sN|__d?xb>QZc~}=uV=}-2Wm3i(JyRBdKchp;7e>gs&zeZ z+a}UQ7j46xojYV2e`SEbX1JZfVj?}>c)(b^-L(gHhkrr_%$%5V+|Kx|KU51?&XKN8 zn`mo=0TO8{5p#-QZ0%F%PgjXL>71L-<7=@aKjPB4H$rnygIb}%mgCfuN3uCHbET=- zJyL9AcmO2hyJw?^6LEoM*wzwUr;GSch!GcT>q_LM*D5bN=iZ!LbVy)c^m@qZ9#T-% z*KcCU%=007Bx(DW`<6QHN}wEiRKKC<3mLgZ?aKfo>LKJi<9r1E(h|B6IcOZ~4jPaJ zz4b|Y^x$CIfxo*e{i|AdnCSPVBdT@X^Gf&5k#+Eao>#&$@dH?YHd|`^jNBvzsoTs7 z?aufr-$nHK=F6^f}>5!+HkR>=L;k$Qa8LpCy2!8TqlaJ*itv?s}Zw4 z%dcDXc`N?gW>U1ITeg_Q*@v8cQ>nuoPYrM7vua+<4R>bbhvO8d>85ZhO6C`~7 zV7X|jzpF^Sf;uDirz%pziCHz~@F(!8^rW_^d>4il6y1uMTK-uE1}5Vmt@4*b^z9Vy zs5tZbn}GN;%JSB8|BKt3;V>rBd4la0PpTmP#f)>l)m|9=rO4+lFW(+KY*jSr9lqAB z9~Wmy`8N2s;zk>6Q0oBfRNaLo^yxC#Db>QP&ofAh7=%BRkQ%z z)-d<`gsA73r7*`yynNj6bB~<2n2Gela$o>R?C*1?Ks1n`#nu3}YW1f-V^hi4G-@e~58&3qs{uy7+Z2a?(aY4%BUL zpS7K`jlHG_K#agofMmZ6I_5*cdU_tegT6=o-vg*ZTkEK>iOsIGlipX`WGOSt(wzk6 z$A-uG=G^*D`{t1cvTE3!JFUd}4*EdW-0R=brT!zn{tHori({s8P}?R10Z>-+EXQy@Pkw!m6G_psRl)t(Zi2AaLIJ?60=Ed@A?saG?QZxE8*?ynl_I-2Tr&S;X<`N ziq>Ca>)vZ~9^skM@m#c08MpLdY`{wVy?fZArS2<(n5o!Qu?l_?eRM>X6aZT>JsG9D zq}k5R1E`hIs7SLs_fnTPuq=vTgc+2gmFk&1DpP}-Mdb%U+w=al5jCpUF=X}Rg>IwL z$ev3G<2=sSn|MPE)}Zht$LOln)f!bEArNAly988o^UudlC-1;+h-!xldys>{8H_@W zuXjpO99C-8rg0>CE$8*7cdQz$b)RK69Llb6^nZtqGLue1r+-9x)yu|gbNHY~xTur3 zY8lJl+w98!=z!q*6IQY004WzDev}N3J*7Y0_Uvnn-3RCVlcXz35!;lFPsF+}MB6dJ zOtY)bfJ%b~6Xa!{=dMV0Z&?GFzsiGj;kLVW_ubbVa2FNJL|P-jeC}u%_33Ze(57`3 z>q_I>Z!|gUrKJ_|lC|L*@cgnDH(P30dh#n4WY z-)Fnr^r)b2MPq|(n+{rcimSy7`qy79B$_XhEYZX^t)YYHRivt1AzmozM8R{PE)3obR%JRH~go-#Pm$X5|0cN7U&&LRZ<22ynO@CnN}5uK%MkG61iEYchZ zpm#o5RnjLf2Cf<*CipArmO)RgTAlZ3IF+-q9i{$An2-U#F1D_>(vfsYLDF~s84$eI zc|tnwFpsTh6g_Fu%{*2~`Wmhk3>vBmXn=+4r_i&4zi)mhi0f8N1;6t*THs$eXlO_G=vVMr zqpOVrdI6L8tak*qS@@9@rSgpWa+&GcZ;|z%B1Hdzv>-bO1J6j`s0{yY>f4PoVd|mR z^q2HD^x#H0_UIS*n8P+_>8kROd4udEsh8xV+>%k(&3-3?|G#ZgctIMRJRaHxeL^~q z%}B{kHZ6`mGFPkb3@C=&cz@I$qJde@fFJ=waV~-$7;V6l(khDhtKhHBNE7t2i_QDK z3QLV&xLv-Fy@Nk;NEWzobXi(Hf`bi06ecEv)13~7BKe7ljgw4FgAH|j{%VQZ=h^IL z!Qgssj`}uqf^tPB2bf}Tbjs9HIsLd1D+Nd3MKXtD2_@-{$n!lL9+pmDb+MolbwyEXWzOp+_iq}c@!mu4?MvnQ=HHT-c5OgV`wJX- zwni^nAtp7Z3m<&4jT$T0qo{ApZcJ#Kw2rGcwEFtM+L+x=fnlk7pHzOdxnm*LP>G`P z)12NEjB}kxSh-pizLtskCxb2%GV;&G+ajoWoL&+NBn z5w(|sb)QF)%iWn`n|a2oyWL`Om=tfVCg1y5y-g7~h`Xou_69yXr?^CD95cH7PqG4lpp7gBOsNmZyfUzX5VLl}b+M^H10 zN^*3W2D$#REKK-%^mQ(QM*-W4@vb74Ja!xlS~H7en85F|H$UZ{_8l7C#9yFyPahc=kKzkXgPjMX2AQ_4aET>=1gj; z^|~0$cz&gSbA+VcyVvTg{u~Kh$F_5-0%%Q=LlFf-sxD(%!`fb>s&x)m{F{L7$o~m> zx!;U9-04N_%B+@%jqoSO`RgAbvzA}$@FVF9bQI@292Qp!m+)tchRI8p7At6tzF zaQ4j+g|jP{UsA6)HJ3fDePsAxl>O#DPZ((rY1+q7z7~Z z0jLm=W#zMqeu?^oAf$vo#cY6lIde33Uku%l*tw%9GexTq44hSVb z9+QE0pSFOV__!gQIUsN8M!Wxzfpz_(@Vu$Yv*VPk3sJm!Uku~<$m82L>-8R?;VM2d zWu7OkIv)Ilh_3F*xZ-&qJk2u)1iQShRKd8!?V8^E!dW|KH&e&?ec~Q?oyv13>ws_) z7?khsunM)X&i!zbNYsWy@$P-#$fZr5cJ)7VBsab+Yun8T@Ou_XqgbJI+M>~yJ2$Uc zS(;jJTY6L4pPS?0xO`hb&+Gg0H^C}zKanaFB4!$my$K6u8a%8Bs}o-tad_$I=b_DJ zzn#J`Js8lN`_fjEN#a;2rIJ>d@j}OF?`wl%gTb7ZSGv{l_Pwfype@RN$@ABf`s~#GG+ZbIB|5sxFaH(K=0dtwXmlZ+L=iQtT$4FJRe2%i@b>cbEDq zqQ$3gUSGUmVdG$~PP4RbKJ94Wq@c8u<6}e&^&O!vk2F>;_$aJW8%XM&_Yq3cm()Gy zBb4ec1%3pqCh1G*3f07S3x*Vh(R6J72E{vHFLz!yo4Dj3K4C+sWU=zW$wOOfH%@m> zxTM@pcYQ3HWM*k5JA9xyauxNpVrZ&MjD|Y!?O~Z+ewnZ1GNyf(jg$%({mt9`2PNL; zfdNuOb;tq}MhscU>?-BEVHgAp7W+_T{EdeAVJvq`9VCP@PPekyIbT$sK7eZT90WJ^ z+HLzaH57`A4)fRDD$y_h%BG@xy=OQ?sqz)oXSbyeSOP|K z({NDerCN8x*7h{k(+ zKui&7lAg*RVn`b+U_|vQ4Aa#lzBJc>td-Ne-}3uwp3Wxmlg0OkQk+&g zUK8?ht?RB|ameVqz{FaTNeX4es$)_r&x)w*G>g*Ssl z08>fMs5k1GZhgv``}Ci8O<^8yccJYpg;uWu;^E%Q;qy^h>*~Ss3XapFG!7Gt08ELt zWY?+aod)${4_k{0m)_UB0p*J+D@Xx?eg!zz_dwvnZ`7Cit&MPDLc;R#z-hNd)8wb_ zCl`s?mP&^)R;9v5iMg`OwP$b@jT8LeYaFM)U2REa?YoQVDf4sKSQoh~V|u~uWk9ri z)Vnz@RUcvLY6-N>4b!`U{7H^X(zAM9YY%hNV>J~h&%=UIW&6$R5;Obj?6U~=CF2XT zH>#=*u zHW*QmyXF#;Bou05VbHv3>PL8X(IJ>ZGOZI$LiIVzVH|qPztK`_lnao1wHq^+X zNgdul)4RrSX0E))Htfd1mwP7Z$rN_VE)0*3OLWN;TG0&m>{oGT_!adLzZRZbfHR`* zZXPH^Q=x|JZv&{zIeaeak$;;7nyFL zjGm2Rl7!LtpMcXOVXsuo%2m5_7IPGvncs|)FI=jgja1L!hj|wDKMQhSSaa029NUKB zd6)?ybr(^C1u>w|VBF~Q_3Vpi_ zn(%bGrfNNh~;W2}ifcJ}NqI8?HI9eVY9*mUi zZ*;Y)DGgZxW3S+Nf*?mMM_zP9G5v%%>M(nwuhnSfUl=hdt)_lC&l@!&6`7*r>Byv} zKX$p$>Y0qdhpyK`+&IJ(5SNM7p+F5)0`_ZJ(sN#C#$Il0HCN?dyxt#X5mfWWEuHKN zI;4|j_L(bJiY!Jf@mWoubSSb}c_TmG?~i#p6-Z6N)mm!-$0qLeErm8lI}$&jrrzk+ z4_b*3Q#NRSyzLYXakW1f92xL}Uk3P!a_VRI#z7_fdf$mqFmBDT>U)jyhwupoJN{L;U{z z968qTxrYrpPC)g%w}u{DZPdG7W_h(Cm9?^@HdsV>A^PT%k1k$g8MS#009NzZrB~tGtpmoQvsC32j*D*_ci~w+#iu)yn18OV|WQc?8C@x zR-7H1&9)h&-_YQ8cA9|)INBmj8F3^|cT$h#(}uPojp3F^7f<5abIyIY|Y63TOgf0PPAY84y>{vh(+t z!96fyK{W+Isu6+jngcm@5GL7^Zsx34r3ZW6D4q?`Jq`*4iCnq3Ir ze0AWZMJxM5O&dD0y@t&W!V}woy`KB}j)cNt0lB z7kY@HVb41V*>|$y_@M*iyDeEy<>{@@C$%+Hah3)@PJD4In40SiP4CkK-Nzi+J*7P! z)l}j`j^*WDO^y!M20XsDC#mlC|FOh^ek$i>UBl{p13}*%Mq(WUc;yxN zTsMIeh=j{#krKNt`BMWF{^1GyX0Bmz=n8(q2N-B%*_&)aA1XMgHvI(-`uhj=pIG;Q z{e;O5BEi}~fD@ZDg*jrL1oI`VGE$KeGwLf7z*(M47CqdL{s~b!8+CpB_b>Nw6rqNb-X#V{gcH{qQVnh<%7iXhL7QwF=D z62xMsKrkPZ4O~KKAgLX-YYg9hnv2?&0&|S9h+!xxe35ipN{6Hd#^!R-9Dy9=2e@Tl zj)NG#y$>)t4Jcdqp%D7uT0i`N4R{f6(KUdcsDk5skqsPU?q9=|p_D`r^eq*j=E)5J zU+)Bl^8PPa5400CVxvW25db@qCXHbjNynthpZYwU5WqgO73dMolv|I(3}?!Sp*}0~BVL_Jf)F3# z{buuNX6yqcbzt~=ny{OL_h)M=b_}SlHNARBP4;SDgCZiR;~ibd6QRk;p=G11vY9!4 zlqvCVMcu%kb5R%XUNapZe}3}aF)(uV*e@N}GfskQgiw6TmS`KQ+_f!FSg7f;6V%Sn zTXK@tQDVFN$|Czhj1BGCPs)Q^! zBZdOWxGdO~>wI$np7)w;Tcc|*@_gXPAOgsqXy0NNE1!`2#x`R2><;ZYkMrK-Ss8b7Z^>Qp zyb&z0d@*ZVk%tH#r;RFJ0C*Rg9h`e@-){PJj&*Hi&wC#ctB%unVm?VzIsh`0Q-!F=H*H!+ia>zE*+S@yV&~0kbWgEP3VssK9o6 za%#VAG7NR>fu|NGl&!2RBB@c8b2zC>X8)sIxQ(~;(Rxp9c&WA7*y6YE&3#Rly?&`F zV4I#KX2X{r^un>);5#-C!1dCHmPmcg`viIlSefg3+LrtJ1S1ynLS8+$Ly~XC8F6YU zM!4^CUSU_UStEa>@>#ux&km|HWeQVl@l- z^xXY>#tx;gOU|S*+D1CXMmI)>!cP-)v3ad#sAzvyAI5@`j04QQ1QR1eggAfLcEkxd zPlU|gw*XEJ!I=4kww9heK?(hkCEO<7+jw2+Cq()28vs%`yOQ`S>oU}fhKV_0n|N+T z#Hbdoa;v6rI#ObWDNXXQ;mXSqM_GkjOF7MY4K5b{DZ=!PD_FOnq{7_qA^G=p2Dc0q z`h|~pNyd_O%%o;mDZDiRsqNhA3+{XHN7o0u=-UJG8ilxy3=!IA=9OnUxtZ<9j_AwF zZM(>lURN2!rx&IfJUYaOSvhqA_C>h67f>(=I(Z3fTz!&y6Naiu*3nFo@1k~doo%s% zoP4a@LmIj2%KDQu)`kX@T&g#sKE9o=+besnlyDvx4z+PfRJ1zgoA+phR# z*)~xJCUXRXf8PU*8KltTaYt8GQMB`uW`m-I8NdSO*=7{!F7UWhe$yy2`x zE%ILGcGLY&A!<_*(@}Iv4HU-99wi^EO3%)kT?rV`*?5e4y5-*K<}v?e854_{DTvy^ z+2cL8&RoaOPUo9jVws=suCgc&FN|9*yEmQn`?z2@w)T#;2x7Z;&N#Z+CxoHSTrWki z=v7Loo;E@2y?w?jYoVUj>-s@jmSzEOZ~>z;3%aCHlq^ZIE9-t6JxK;clQ)T-#|uNV zNQ?)JRyw|0b`nJ@Pv;7JZuYAfmtqty6~H$~jqev*43dC*nqmO1g)tJu1KmR)gV7?i zbG7LzToOTT_M(E_JGsAMmYA|gWTM+rCTsizVl;^Y^EG%)N}^Y%5;SHBlNg^BUmE`j z`6{eKIEI*WC>nwv9Ki_Gs5fUEtFnmX4=|gZzuxCu7kr7KkOXELvt2?$Nys*!eTP^O z1L`yCCnQ(j7S(@Mfi|a7+S9RhG!IAu*rat&TeMe%vv?^?Tb5q!nSGy{vVA-V6V`ht~Q89km!gx{`@mAW`Azb>7QZNkR7&JCVYHl zU_yLALNFv&<)rvmNXOhup`PZxXE~)wSCX}1`SA-ZFo9pJN0C(G`guoaq(7(6@x1}G zZ_=!Ij4JB>5UHL^7+RZ6HE_gXv0z2}XQ!b%t0^jFbCp*WhvsgQJ7i_QzEqt`6` z8NQUP`bHDU47ZR+_@*v{_%DQ9nma`@977T6rTaETZx;jTaruu}(R|1;bde)5UGfOQ z?oH=xF3Pcl4M;I^pKr=FCy$|xj*eC34VRf+5FDZq_n2Ni66R`si56LGtfStFUI#)g z9an=TU1(Yk6;pZdxybITUM;7c+x}jpt`yhd@cOudWE{~Zp@Df??lo`)+5tK ze;O~x86QcxMb%23wO2ioQ=!q5x45;J3978g($kv9GW#yBh(3~i`xCNGb42ZLfLG1K zJF8-Kp4I5j7M45cjyvriUWs(`a;1BJ@0I3BmUQ0R=R(;@_d4#=t}|-Uvu2Kdnw8v# z->G4DRmgfKDQBK-#hM1>?tcl;MO?UT5cEB$Rj4yzt$xRK1ch_NMz2dv!SJeO%Tee& z6FG~eky5wN*e{PRD{;7nVaYSq9)37&sL>N?e8X5dqSEvr)DH{8^%G;|jotFR`7SF(u3Ko80{uposle}>{U zA#069SI&TOJ;3pBL=9!ET@=MXoC_O6f1M z0>PL2k=BtuiT(&v91sWiz9aRN`L*qG|D&A>SD6-HWDi&CpmD{LtLGPF_2BC}AS0rh zOI)YN3gN2pUR(QZurnkMZ$#^39NR*rSN8A$E*K{a{^HdTs^Nvf8hb7>c26jcvXIAK!cfJk&d_%hc6R=0a)A>t zNp0&A97wi-OLga+aT1DbpH}M2&^ha zFe-{@+WE~a&2r!D&7b@hbmP(qN7h5lD?J7-Ukc1A!zg{+SkMlYf8FKZz%GwL73=)E zGm31)!Wy9&9eV%sTbYR;%$)|ewX?03m~We_MY>8GIa*zQ!f=fuRg-1aD#hugN0K&s zM}hucPfZh$+G#JOB|!1LRO>%nW`;?p@dFB{ko=18M$(s zU1D8sR>?U~8|VDUjY^$?Z#vq=1Cpb)+V2-{rK6cN2N$n(A{=G4*(Mj)Wh+;`i+_tk3S zM;GoiZA~ZVrXyRB9$)vAb08>WjyT663WjP%QIsDNZ}~o|Q?iI#3QDqS3>@5JzC;(= zfcm7~hIru%jX+*TewPjTR7Dt(T07)><0+G@SJy`&a%ZPa(Lkv@__=j7`N{NC;?0ex z)Y!?B%(#0|Qwa!KBo9HMzm8jLb~AK>7u$m{Uv_pHX^m5O`8LmDgL>ipM(cc+L}ZhK zs{M<%wuwTOQf!ZlaQy4knNDVd^*Uj)iji4klarI)>c-_|W#wTRNj|Avh8jQ4GTpm( zjuH~9ajep@TKno4s1~1!#`CE+iiVTO@OuTTT(WrUnQl50?P>NrhmA59riCu%*Au3n zBIL7kO>%)I{lgD9snHj2?I_+KCiOYyLwkL9zFvQ#XiBU=*_gl`wV8TbxK4jA>m_)m zHQp(YMI~@vALXs<7J70cp7#y=OPb5vrz(9f?-AolhCb`O!m~G|o+nnKY+qDI(>Wwj zz1TJ9dsUMZ(E8GKcNvl9FzVbE>y{7}@=#p+affbxo|J#FtobB6cnr!Yr#v6|<2dBS zMAg3XHct*SP)_tT0;@8}W%fY1FI|QT=Fyk^J$>`!_oz(%b@HR*z{Y{(`a~3Z8cKtY zYrC2h65G4##l{rl=Huex&v(ByM7qO(zVr@r!`+WVtRsS{7bI%OLs>JC=gO1;S8PmL)TFcYwLQMg@KWCzVtRanIA$9X0Ki( z-^(LWS8Tl|sNw-&a6%-x!&a)Z*w+}6_?$%jlHpnCWRIOv~UHH;S=7_jN)hS^E2 z>`>xyv!29e_Zd`(7f7n355Q+qp$5A^CX}%=sJV#N`QwkGnp<24L8Sl{7DM#HE|Q|n z=pTaeEEglcf9{d~<#8ZwPz}cLxm<5iQp-M|_!2?YR|I(5iVvuEzD`mtu-V^LxnEZN zeyU*cD6GcjflMJAYAD_3CnSqT7D9CWm&EQ5ir4-MIzwN8gc7!WjQ~oDYyvQ(7^~Y< z`N$?)(X}+}-s>lnkqtaJaMP2X$m`?YfP6Z}VS=Kc9}BPV6oxlfmGZu?8n74UjjTzz zay?t=vvv2k&*#qBpw2iaWx4dVusl{zZGj-rpWjhm2x4^*t zrHWHl!R$4z?rBjx&t%Y4#i`lC7iKNHYvTHvMdAIY4=65g^5-#JB5% zXI4qFRf@$cMom8<*DB7E*^oH>j;^4(K?4bhI~<{Z=|1`Y?`zO5#E201?ofHoEw5(9 z6L?d_GlEa zM<)qXdEN{suFG44ksJO4eUj!h!m|BH zeHN{X__;?Xy0&mCkU(iK3XX`!3k*(T(e7V6rrypg=1xWv#16%*bT@+`N`7{BIl~r? zBat7j8YSw=pJ|Z43udD|rH#r8(F4rq2}ypJ+ezA{=y+6dXs~fTrXnPH>RHWI=jnC( zX13d(g-XAiE2(*bl(WC*wrMrMXMwjD!4aWJ4a&o_269(0$C3Gi;Q?YZFi^pkv>1(A z?fQkh|2CB8z1sq8qnA{L2CQj=5Nb|giQS}V`-nc9i6NDUKkz}MlW(StYOt`UH4j9O6~yI@EvrqsinD)K zN!uB99eUmtKJPL_A)Q2n8w-J-jc;h*JAED+~UGogpze=k>d$Xx4 z{Yq2DBuN>8Elf}3TkZ-qvmVpd32xp3_n-GWU z_ErTjyw>50hw`-g>NgLjvyfr#D3cKZT}v>P!}_Jt$_XGw!iMaDVQ)dn2x>>h;dOMi zh0o+w4r11>&}WQej>4%cY*wCR!Q6SktN>@i4ZQIdIncM{1IS}TWV23$hX*yu+Vkk=}z|ZJmpq`qG^bFOmpn1ADDh#jZyeGtsJ06 z;{J>q{MR?;|9!T)eUr=IfcSnLjU2AG3O$g^NK*r9fFeIZ0B)cnNz$_^(1A>ywhDkOR`Pw zgWg9Fu!|SUjY430q!!Gl%}~HX-p)@hn|6=E z1!pN=ldt`)A>r>V6?q5BDLO)za0-kYhI1&(w&k)Q8`5)H%?|GvT_9U6S$Gw-%6^+_ zdMs8`GE%ZfqJ7QuO_bJ|zPPdP#0is|{9S z9PPo$``wc7ZBlp$tylhsALFtYZ)-j_VKuPXyxc`4RmXIL@%?Z|R8UUnTjMq+|EZaRYY7IRai5+mxw3CqTt>d*H@a9T=H9Ccs#KfCDmliO!Xd8Gug@%>-$h)G(9BD{g z6j|2UxgM1-H|QTFv=crOU;69%%tnvCP}#~%qquG9e3)9Dt8>m@}#s~(dLzh>bO9`S58-ZMdK3ru}+%xoShcG&TSH4 zp+e=)kf=M4RH9Ge16`k{iVa+GG#fpb;PQJ@A2otUrw%`=q27sSf3kDtWX6J2Z&cdX z+7DjlZ)J>xVy#+T)hh~J(R-DLa4X$i(ANKF>+XS(p+XY^@D>>S)Fkd+$D}5!mb83%k#SWxuEyv zkhRN)!6Aj#F1gFb1N-I1pE6p?zSZRM=r^yha&l_Y0+AiSgwb1ZYs*Qj#OWEEGYQaw zV@6xLr5-W|dj9FOed||?Oe2+ce>_AxGG*YXYic%ztLMifY@d$l&p%!$f40+>9D9h9 z5;*%$sOtQd+mf|A*`?Q|Sll^?KplGoS4MQb)uf)@u}~2dQuKkxc_Dlq(eLLAq%OZ2 z^W)uj0JT91bA_p|x>bL1WsYC_Rf53}K$&eynh3*-rcBZ!J`|Ck;|RH{c>i!G=FoVz z@Z=q=(1o;(lHGEVJe%2F^8gMKHP=f)T%>M#%%JQ?C?*BVC^B=bv^fE zMql%?h&t#YN$y=(^XeyxAn*No7xTaEN>~E?3}*h)A%Yk=!3H1zIoZ93AV@_F>c5Ab zgj+ek`}CVzokwyLM6l47?o&Xq<#{Sg$w2Awqc3p*YN_LJu?j`h{!+$ln#3M*2 zK~~Ds1f*-ZK%lER@_>Ug#>uKXu%+aFzWAWAyv1xh2^Cvlg-trQ=4RmY7!Ly)KhH}pQL1C5s>)p_a=MN_ME z%s*#;g}4KIWo5 z;#Xl(zQ!`o>y?vWzZ1prs#y4RV*MmEnvNiiZK+FP4?fcVL7eX&h0Y}D@5sze1^kFB zC2N6rz4|MtOLky)WQFV?@et93+mnnWmViosj{O`j%9alifUT|;Cud`}5O3vaix)MN z(~TnL^OuA)=l+-jt?h4iOxDhR>cN$gV4a6ceu(;Lya4>!=58_+>hE1z-7Dp7vz@s` znNpEr?_OpSW*2gPL1jHyk6T@D&78jQOU>RV@mssNwa3;a4yEY`^|8HLbsUix|FQHa zxzqJ|1@n#)O($nYU6_CIn0U?wN%8B?S)bR7e5vHVbX@z+C(q>3ZP`@&kQ*zAAVp1l z(~&AcpIBaFm_=a1Yfk5V<8P{9mYXkge4*)`C)(lZza(~+zA$0oe5K)mu+WKeTu&M; zPr!!>%s^Wrg3saun-cqQiGX}{ysVEI$MSY^&ZbgP%n7>yvtU_?@NtpXx(KmC0#xl zonSC3IQ!6Y{;HT-p(wrjPSe26gCCRp`!2 zVD-mH%s-mk&jx@CAF|FkB;fX6_Bn1qG$FS^H8PU?^MsqmcO*rKT_i% zV{mx${Fv>vT-_*6`p46Q_iOPt^)f5@i{|dJP?;W?neSPG*3H%Kh0WnST=UlCTCW$p zsgT_vaik)6eKSceQgEvD>WJj~>-?vwTIsx)DgxfMOFMLGd>cg2u#pH>m^X{lV!e%U zdnwP+Tv9d6*ZT~vlbBqCa^fq-r)4bd)3|c%jI4GOldy@fmHlt!0-U#$=;fY9%tf48 z=%z8^RGb~nif-yRyD?G4i6BVw*TWA*xIjH=V?rkw>T2M}hWSF8MdaTmdG4n#c1Oan zdZHnU45f+McvXA;h_XaA!|Eg%;nLN6%|VweE|oCaMXaeB*dci-?p8V8u@|;~FD=HJ zFD<0kPl65Zxi7d5$4$dqF(ZDnoZT&XKOyYSBPvW@nJ&&P1HG?BMvSyGdwwKBH16I> zHt>1atY&;%`omhYK~`v1dt&0tE=p;N{ z*c3E;p3n0eHln8$pMb(Gj4z&aaaz{mTD*f+Y^h<~(qRVgYJgGhVU}&QjnKEO#5zr( zhU&W2UIKdR(cs9fta~P%PHQjXYLCy?Z^ra)wIIcku&u7~m3~gbEtWf~?rNQR&US+n zCFM&;H$@zST6jAolgp#ky?QN4#^S?7p?Q0W0bQ0`&P*GYA015`57!%6?$1_g*jbco zY7FbJE(SFa@>aI}6Q6Irnt+kB#boGOu1r(&+dUpn7s@KGcDDU7H!D`@!s>TpLqvD? zk#@j3k}X9sbh^DX9TzriSIJHGWSZ(ByzRgpJ&0F-y7X}MaNvAy{G_=fql6b8??Pk1x<6th|YjzE@B66Ej?q}2c9*HFAQ2-M&)G#;=Q6)@UG zppeHKoRB%9I#6DaYT)W+7z8~k3qOPBd#{Uhq}ojVX;(Y5&_N`C=7n@f#q zp&A&V>|qk)NizhgC`DqHupGhXW*Nln+uH@+-lppPJvjRnqn(PCo{xMV&-kOk2q;x> zy^Go<+evJO4^iRVwut+<`0}UO-g@(M;y3-}9+huty@Vk6RAO2BY@{R4#JVS~hFtl` zScuZ^&P%XfsUDdK{Z;Qy(Mm*5eGYf@*Pg9@7TKS~Dn)ZG7V^k2eDrw3(nF=)3x%tD zexj=jw#DHzc?5X-A^&t?BQyj>RhlE|8Ztrm>HGTPxh`yo&&K7h&V}ytj&>uq0o6n6 zwAZKHEXoAC@@mGrug9PCV9d#by@i)pVY0k|usTDkz>oQz`%=;L-if!%;wW50%$*07 zgSrJ9Yt6RY8n^2@cNJ08yz(gV;YwuIN`QA;&)dQlSg0(2OCcGk#iApqdj_~KF_&4zYLvl-^_VY_)_DReu4tMZZrwDi#j2Ie2rUlU z1PVRP9aFlfME|H-KHH|@pvygVT*z!F%ve(^EsC8AXio+C?kglzc71wBM>T8TphHRU z$7*>qx4|AneS0ud7dX>u7d12Cy-l8GXFJcOay~v}P`aUk4b+s2FNB3~Luqfy@VQNr zp={9rDy<@A6nev2ke91S{UuC^MrQH53}$s4;x}xa=>-&eBYi z)%w%z#*QziRwqd;xWh#`=vzewWFzcE9WCK{PaO{)(G1?GZx;n`4<~K+Pq|*9KJW4+ zVX2Kqh(3hlY2QQNm05lxkyG5=X$d_|eTEmt2eS@f>VxN({^n79>|5c(iBpqpk`nmU z#%xc+;IvTVB6nB6)7+3vE1pq;kYH8BTTy;Fy=H@SY%QG={)RL<(s?px8w1$?gwv&o zgI47Ds%4ltJ10VmR-Jr4VS%4>+v^fn5a(*KOz4iv4u9N(X?QIXgQdHA9awlvf2+@Z z?_V@ep7w`XXQB5>hRXWKI!S9fXKzft2>-Egx;*TSI)CnDEw3w@Kt56j0E`BSI(qNB z)_z;e&Ax?^{F)s3RfRa0WHkdyx|r%4mgepc&IrDJVjNVRnyL5*&xkowoS=V$tSxYho7KHX z|H5h`f6Y(X(kHbz1C4^S4&<|&cpd~H4g&*ibWVeyjgD<+3&S9mSGoVqjrDK$68L{| zzCfC1+Z*RKjgJb_LRT$@-txiNq4r@}tZk*S&Kvh$md>}y9iD=?bN>^5^}m!z|HrNr+6fDr z+)fA(CADszLM?Nq_~JUfCj?~rM{c|`?>9Z5*ty05_hZ-?0}E!1mJ#lG5>?T z_YP~i&DMnzL85>J5$O=5D@~dxEnuOD2sT6zqtZb@n)C!gdKVEzAt)d!BGLtc(5o7H zhfoEn2_*y);(2^$&+K>ho;mv+_nG~j^Ih|o7x)WFp0d`x?p5yX^F7dY<3#{DBwC5X z^cpVEhSdSf@A&n?7z7WP35}cX9S6y#SAcCgbKQiAKs(@1Ketdr*yz9V1Ja%R1rn(A zi#{LF!-v3O0WvI5@DHyXTEb{5@T8{5<`w4~3p!k9y<56`<9R+3R-3*W^Y3N8%U=Zv z^#8^8`WL?Ve+FLwwK`Vh#3>&erB+e^Bm9<)H(ex1Q<}baW0ww9)QA0T5GZGGXld}4 zyii_C3!(u?o%(md#a{zvX7jNkFxf{JM`A1Ud7OGL#YuK)raq*UNA%?Ktibq@q~>hp z=V=Mq;*2x0iL1C;0!7NZ-Zer?t6xKUN-BdG_N0Gz<%zeb^BnxqUM9?zC$?@rO6WTL z^-JDNQu)DGi@CwiMY09`(`{J0(yo2PpXvMo!ES)UwgvM^P3;WO>zIdgNMji1zAt~L zc;})dsj0x0H1}AcLb-HiQ)H~=-8)9N54O-pk-W#)R14Q9J}ei%Cm%2P_;Dhm3%xc> z!>1rYVvAW$#h7P-z_YHIuX40&iOxuQn%%ss1l^CPMA zzU625zx4XcYGL}nq2u}=`A__j8zgjsYPAbv6Cd@27s&sDYBF&7`5=@{`d%(QnlwIw zB{4D2{5fWr;RafDQdh-WYx}+EU+gWT`ltQc`wP$c>F-w0;Jf`1xjKNpyd1=M75tU= zh-7Z<<6`yif|ENq_U*JN2burDz5DB({6G8?QbtRHoj^1Vly|_>u8i@2U$V1H{i_5( z?!P4f{u6xt1E1vo{V?#?{UNOb;FniJZPy&a}yh zja^)?kSLyDdQolq;$>oDzcG%!_k8s&@pijSi}ho4eoU=~mNHF`BtVy->`WRc^JJG3 ze5|XDj7@REO$KWR$CO#a&TxP!h{B)GW$~px8MBhnE>eH6!#obP|frY1L>uhjO6A0q2Y&F%jJo&GxN{C#Zj3x5TXjD{X2 zSp4%ZuWnCbes|tNpQPPi7_0C;+8Q;I%lktZS?0>XM ztXOt1jwM|#SQ^FUV`b3y;{NAEz1M?!?gga^<)@>xk57pN$$hV|eU@pall`49up~D5 z?Vj<{(#YFUVuA-RXftaDWw0p!dP@MT2|+Y<-BR&EE;@Vw;}@=SLdyUpF$S87)xb1q zuU^dh45%A%Yf1zA@S{mB1BCGtQS`} z8~KiAK~)dX=#z=&M8Sv5S}f@P7Q$s9{Vj5O9XkK-3kjPoTf$MTXg7$6#^L->3~Nc^ zM7t)_*kHa~W@(heie0MIEJFEht>_g`6V6@6Pd|P*ba@wYaE;DzFI%>r%!=3VN4;sNx9Tb-$ohB1mhSNQk7VPYH7LH>7j+EtWXTZFmol4t_3d z6#mM_dtPyma9l<(+n$VlYM|rgkBe4T-Sqw&Yx74h0dy5b<6NN|CE^xufz!}olXo~k zPfL z4_$DYf^#%K8S_k1bAARgcOuU}*Ha6gd+n|Tp8F{pbgtHFW9&I8FEVkl0R~Lv8S~~3 zj+IrolASAcL=x*Bhi0C+JI^mFqa)0OH;3$8{Ht5WuUYcY|8{go$(DK$em4M?tMFS6#o^N;ML$}3*ic*D1JJ%;wRYK~zROY!wy z>|dD72hA@(AOb(bNR>1hh32QeS?4YX z24u{xXTUQw_@}Ti+R084k|dZkn6uz#o%MEP@O+{0tzHX(T(4iYrw^C)1d; zXa1R2f12#>*w;_Kv4?KjgdzLnkbIC4iaIG85cmR>eAQ~i6IE(f*W2y224TsPPlZ$i z$~D|ie1UlXZFBoSJgDYpu|;mWk{Y%les_Rokm&p9ETcCkPCZ_8nHekyzs=RFFX$%u z5#T!PgKGAa36SR_?D657x`oy5SoHkf7TEV#YQJE3ZwR6(NUiOkhx?Jd>xQp(9)Q;B zs+0BCB9|(q-2zoI^?gu_hTq_<<;x;k7hHV=yxuuoA}82CmCi-vMSK^LN&IMYllj95 zMxn{dkSG6FL?!>puQPl>+ONb^*wvsy#tu>9(VZn<(#k{5m05uVn`&3U2fer{@6Q8s zQL`luXR*bpkr%FiZEgasN|*;_+htLeHYDoxsB5C75p(9At825@Kewy&JViFVEnwYT zd)8GIlC^X4)gLuBzwWgD&O7|m$);botN|NTZ<=B?B9F5%4KvuAWL7%-n_!K z+v`c-X^wL>Yz8JUbI2-+BB9#EG0Fh|sMy@VE{|>y2c1wd(g{8NN?CN`iw}Kd_R!5@3?MF55~LOKT%QiR23! zAB9RfHLms1D;`Wu^!IsXZpy8c zxLhvcW8B!a6zfTCyh{^4>5?k{Pl4HN$r`$gh=%LrMKK&Xh^xc+fbm zTAOlX7bz}pVk_TE{mHUe_xp6FWK-dj1sC^19=d4*r3%#sOj%BY?0yZyaV!VTq|dWl z%;({9w@KQR-K8+d^87xC;|S0jYg&J#>Vb%nDTql6o1co`ym}9F1Tsd^0gJ>;86^@L z4KipBN|YqMI5$_!zL}T9^QNN0N{6pp{9O6);VxsN_P!vbv8Sx-S<W-4-v0Xmh{*6PnvPMQ7ek)uoh;hX=3Avr%@fE%L2Ii+_ZfTwx_Y z{=i6>nGIU(%GehYgy1kg>#XtifJD&pVvcIRR63N+`Dc;$9Y;M-)v^U{x;}?wPJ{ua9hW80jC=VI9;8` z#n~hQP;r|EH2tjTK&Pmdu(1Y|^a3`zd}q6Rpx>~3IQ#feEL0h@gb-SoPf0rDZ*-)e zV+;&QDHqb^7Uds!%h2R;UmNk1P^`C9PBNZ)@v-BMtt$)V?8&RtBW+gSuIKD%d_D^L z*6;2<|Ma4+J;N-RNO+38m!b(q78p-8Vc*UltG10EpRsf#erT&~8z_yTKD|rPG?fI1 za^cUu^tb!_zwiI_kz4|GiV`oTR<IlB?6#;F|=QaygIhKmv+W0^OQrT z(OZ^ids9AGx`I6T!JdSF<)Q$dHbd-)bA?BR5&U?=84MPHSO|J$Mqoul&Qmr66T{qvwZ#ebJ z=b3&|)`zTTwz}u0zMSOAymxDai;;o+a_AdNyZ!s6ERIuZ!12tr)e8d+d+|R*BzQA2 zYD|S>7lXMI{hD@Rl7H!}CSsD|dqsdpo4LB7eTi%&r_<(zlW*4r5|bW+@x=pw@un2m zZ-t@4gfP?w;VM9Z&h`vFpb8-sx9Y5b8MsFU+$liQo`2JR|Bu26Rsx4l{3&7>$hl@V zR6J5cCudt!jn4EmEJlg(+8&=KszD4iAVv`Md74nSiYu5S)tl0!Hd+)b;3(O^O)@BX zz-N;yGHhOYl>eUW#O)L6X;ePQj_04(@DK2E|L`O5IIxnLxWYRGA-bR*iJ?}_4cjcV zFIX>(CrQ!K4{g|bN1UbnE%mZV}<~OZvVo41{*)2_14jK%>4;^O+2sO?<8DAJXwm7zbMi2PSTI_={9twhp z*8$q?Jog6dAJ>U0Nx{7owzAW}2rh4#<_}+C5av0VrN6-v{v%69`8nLBs9Gg}h0+ng zGAe>kQ}BWIsjw81-b6^d>jX8o zuc}k-gwIme?3eCuv*Pz2Lk_GpG*~CQL$ua+(vXgraAdJ*d>Y%h16s7?fx&D0s}~tG zG@~0vd^_!8=b^loZP`J>LSO#8)B>2sm2f`0Mtm4rgyNBV(9ElyJNSedhw$@T6Dg3G z#pDYlf{PRbk#N1!3%PH>9L&DT)hp@o{PgC|`*BPaXJ@(IsbyY$n$)gj2q$#IdvO7v z3MeI(n7LTZONk_7c_`+Z1#cc-pFehCmVWIlVd3&FH-YfNP)`Qnj1Em8V=!!JW!mxH z07K$RL+Mc3$L?>`(~!mTVh{d+q|<17{Jxl&V|y>wr$T6pU=M&pZ8aeE?*swo(skdg zZ&1(-6^VVdaew6Z5gmdBQ)`NuK11Z*34ISlLeF-s}ylVisT{TkrA^QrP6@xY{PbA6s`&GuN1m$UQk_Uq>D z%uT0rKSFqBg+T$0W9_4(%SzfpzNCdI-9#dMVzjxz(nGJZXUTc_X6Z3V5Q{F`8N6n-}!U3NxRw=JD>nC!{d5xsd^PqJUWmyZMOlx|L8NZc3mR zkzo0$yu)+u&Z|`ZC#EN^q1WE~&Wer4lQQ6dr*J~VRUnt(q_aqby`T7UvXfA)DN1C%Yz&7&T;G_=nw-ltgStqJd-E$}W<^zYDTb*X4>~h0 z>r_A|#dSP;X7mYCrhjO1_-y@{a+r1w&`x6L)&l!{rkXwALZ57<(q!7Pm0GpDg?IOs z{MGO_G7rWjhai>@$Z}CtMbDWb+Z91twtYw!(2N-!dQ`#HFckv(d|#?apeO@#l%#1WVw()q;4HJBCk*H#WTCyl~jt`{vsy zCIc}dL~xFQge1rHd*Y*T#VOGxNQmCf3+MMvxBJgik2BsdcV$n0g-FtF><@wV3~!m-=Cqc=q|Xv*)^Q zCFojTsd2CgK*$}{=dBeZLhBDgDC%aw)CAG{FzjA>Wki|B6<19yG~uTV%XituHrXUc zPP4{%40vX+Mk1Xt!|FBL51c1m4-y&-kaHEy;dw8wY3d!#q@Ol{flA#%)g=O=8P0x& zI6Ns4ZI|Wq(kI=JC0F8>6Z(RohQGk$IPHuN{7;UWQnMi1>d0ZcDenft{V`aF3L%W^ zarf9f-<#s{1A{6p`+em`c8kJ}oXCK;30`Wd7VXgzwWOV?G)uZs-Z$NseSwYrhVnOC zez;C0Hih^z891;;&ZlgWo=q1mvrh&*N9#nu zJ)&d{oCz1B;p#0R?y);Gwqp5X6_#pen~hQXE-^Ul%7Y%K7&R}`TF4nw<$q`_g7jHWmQ(-jV|IQy4r^Y z&g+RLjLtDE69(Fm)V@t}^%OW7i={#ljun-4|r&(;m>?)GN1lG2BTU93T9 zVQS4V)BMDC`X>Nqw)f>hd||cm!|uj|U<@<5loB|Og`axk^LFY|<)_ErXsUxUlkWGm zt{lqtD1L6EJtbal4w>CR3}eWx)~NjkQ@O*^yicB3Y9MV(KWgqO(mv&HowTAYv_L%u zZk>L~s71pzCK#s3N^3!}0hbOLz?r~_zLidFNNM%SI1EbWcui*sDJv`rPK}g#fys-M}K`RI~ldQh;Wc>$Z@Y6*9pljQ8$|uzRk%fN8!P8!_ z_R@Iz)8_K9<#(T#I^Mp9RsY@@{2spi4UGH8(ePiML}04}{us7r_FG%2VUSN*Es7#l)@U5;mqR;381k+&uYR4X z!}V~q6`quLZAoD7L!;cg3UOw+3Eo_OQ@IZ=_VMwyrIED;C|=at^e@;0OMS*{G_y|U z_s3@5Z8+eHmN_?}lEXNm*Jl3x0jqjY4SF3OIQ~8V`gwZ8YpH=MI zsLUXCBR-^f&_lLe>AUsl%GZK0b%qo8ZTNQ_ga-(#BRPKY(Td`a>5&!xd|5}fKosWOrmw(xhn|6*U4z%E0MjG(G0tGBRBl925 z#8bbAGIRI~7ik=r#L_rXWCx1d!A1ZeA_HvRp3V|~AA)$sqIzef^w4m6HgAi-IFXH5 z*OR`@nK4MZhak4iI;^TWILuvLf`!Mp13@>tn8kXZ5APtPwItGPxe0(dyS?rvPm&~ zkNV}bZwtxwL4eW}j%|!$+{7pTJ}=8rVe9I#l*8o$UX0$Ru-#_tZQ8=Y&cbvJN1-KE|a4u5*~{ItMo7osgl)9@@fOm6~CKFB8%>TqnM_hSh* zmFWbe(U>jL95RQQSVU}S1JTs)KOoC$K+FXEsDu0iKW)LLRvKmbr;rUxhLgJS<2Z^g zznNM3Y72@b%PG_(@5qNJ4Oix@m;;|Yte0ybYZX72aUX`o2j7U^)eDQz;LjiX&iBBr z7#;X7o23c8dVFr~u3`*x`gjMd3;SOPvHuyIP5>~Ar$ton0{sLf1bU=7WJ`TQa_K24 ze>^AT7Ev--UfHuNBi-VGMV`jliRlxr&qQN7VAp&Buol|MS-xfeqQ6*O_tt$eHH|YV zBd#Mz1#KaV3%V=M?vrM63?nV*Sg{Gx> zR(-Wh4a5K`y1tXU=(1rvl-ZnW^FYGl8IE0=iTZQ&LQ`)&7H*Yf;EE_08^M6MH{oN|5Z_Cf72t4yRxF>2C$|q{o7;c}hKJdNT zcGxG~z;cw#%H;l)pky6%m+d$ZDL?#Mdexrd9;n!V<3nX%aZ^t*)#*X{a;x3yzRyzL z8YhkNJ`@IjDt&HuToWV_D4=r2O^4$Yyk>XY@tz|hluYR5yAXucxCsO94_k>pZcqOD z$$9!gOsxTK!j`5_oULD>TtW=uhzX8I2Lkiy9P;WdV*`Yu#oSLq&%*nHv^;mpku$<{ zZeTX)AXMpGI4OtVKpMTp-vlNH&badlGu@*R_yclX2D4r<21YB;`6)m-P_jPO&(eG1 zartWp^LKsuPGWufe7g6aUV9V_=TC&6uM5(YqW5A;26=#vOUfjU21C(RwFT^A=^8XQ zQt$Ir`8X{J$}^L~gxeN}SFH6BJCys(ga~EmBI1L0G{!ETiDppNi(9pMAU6-c5 z;O)9I7zCZBi#>BYWO@G#Ozh285a_fh{J@&UuB#*12toVM`%wieONU8GQ*uQl*vSu_ z8%9TLg1ZgNEU#ESW-;r4WZN)vrYN3(Mzp6wsD|SywFKq>c1mD2MYNtTu+qrc1?%$Z z^e)4N=jc@j{#=g18V#ws7|3pjix0K7nw7K?ssi8VLp3PqQFeN5c0R>1+loi@v#IOH z_UGX|M|>GgU{k8K1B6!DY|mLw*GXZNyU&dIIb{P^ycatYMDHd9lcNxfRGNHnGE) zsKdnb!DEMO$>gE)v2UuXW_pIQEAo1ym!oX;dk4Q--Ja*wJLDe+QFr|*Js5)K#qkx<#*y00}M}@x5A0^7Ry=)a*3=cmd79*zvd8~Od z0lKa3oil(MXwN2?4UY`s_lH^dzPuWx-%_i4@#y8aqqIlF4g0pEE$1h>Ud2RvekH=I?g3^XScb zIHY<%-sGCJk~~LE_{rGkPo_%O0-o6j-sL~^6*hKHbBP&PGxV+R#sy9@mBRq$4%}^F&Izc>yV5A_C&AQ z5DkyadSBGe90dBSHG}q=)t;!71?xzgV3h~WJy|8)T8KbF1)wj{z`@X_LmF0 z4a}Bh8JqZH;^r@NU!UH!{}VH0$8sm<#7-Jo91IdTSqeN=78vrGh$`}~dyNKT&xp2F{5f|cWoXqsV>JrpH}kVd!h?s z$@j^c>L%gSJ>ReP43*uU3xbIAC0(BFW=roDUl=<1Sn-&>Jj47JmH~m+1<3%Lhj>CD zUcql2NoN6kIIsfi+4I+Cr?kWZi>T-5eF)-bzN^7{G+`w1A*lW{boKy89#yb|U0eWz zTh3V%KH|py$6H#Z4ad28o!R=)8G8 zRhpEs`XcwmVP7oq(GsDcZMk{it5VFAM35Bt03*100%jFt0Dr@3%cu=k+!jbaYzhK} zyG{A+G6JpQ2ZS02wCns+b}UxtmYAJpppd88PS4x?0g=Per+${4&isIwg0hMfaNONG z0ObRB2O3adY1;FDKt9faZ3r*N^a65lrvKl*>HimR3k?Avxb|9r_uU2{>c$juvnrIF zsWOO&lDU=v*C2t)VLbU6Y1iAi{u_M6uY}2NN=n-zg0$AcAphq98IzB%Ux!T-?E^#4}X0t22dLxWOIklKwc)hDs_JJL7y zzaMb1iD~wNtjA3%7G}&IH-&661!<{~R%rXDCc^UEH<4`UBMWo^J@M)I$0)(%K1IW2TA5g6Xagq9YR{r(!e=XgA zb!6Aw?DWZ!3Ce5fr!}#e4jC~4HjeEsbGS_J~0Y$03!g1t5zsh9X@6IAZ3gwfR?Z%~s zKB9=lmk`*DHs7P)rO^J76cw1?DNpB!X9d-hjju4{*D+Jzgo_LD{(!uQ2DZKCBamUBLa1ZGwPmlW|NnjKV!$PIH~g zosAmgIpyri9G1T7zSr#K^_T+4^M??~Lv2pzR`YwB;m+$H5ElXu%EzLn!_r}l=NzwR zyHgSq)&Y9_lCKu@X}o353E@XNAwh6IiX~`+s-VW=*feTMX>x^$(mbUXk&hPg z@jla8sbqyD-+RiGWV5fBqmk`0J|g)PFTy`e^j_Z;e47+Q`)f!BWLLwHV1se!Sdi9x zia7XYG)nkX6_+btfQhOQoJcb*yx!;_=F^kE>y7Z9;hmGu5ve+#?mrL?%V5IPN?@k5 zC=2wOyO{6V@0{+$y_0Xicl*^W|DI2VhXTd|xhg_oq1W|ew#Drkx^Uyqk*jz3P&=Vs z@Ev|_#Can|ZiOfB55-+}VXg>+5dqsppmAD|Jt+Q8#4mJ3$|da&oho?}s~#MlC+jQoMTi#1sSs5sdkyzPwzDXcG#)@?OXS$u?#_HV`)3X7NJ)Nf6nx z!=*X5U>TH*V<=K9v^1t%AYh*dR{4wxY~ea9AJ2dgb1?DU-il84hPdqt;n1^zzL)A zTsIVpC}GnJ^-vTO=u^qBtHgOga}8=&{X>mR-U=L`Q1BntT5~9jVmJsZ^CXcx|VP{ zV;-e@%YshAHP8zQl=gNzrQF~nxrF;?IX-qS^f&x?&i&BhUB8f?{j~zYXCO8pc;wUK z@0TR@M|cw0hb=6wx4s%}^!#wWK`hKUhTl5E$NJ9AH{;+@bQc9yi`%6oLXkdA>bp-Y ztHs}XZ`SA!r|B;lXwUD}-O^fp{zfoDBuEuEVH5yB_sdJ;0K1!mf_;Eh>QaBUik!{? z<2yu305kOyR)?w(L+FvlAvoe0Cm`Vb6T&b%O~7_IurSPc6bQ^B0OsQCuY+%kYq5bq z=^%*R_5-VbIR61+{cM0BI1SP$Foy(AxrEw-oO?u32PfAQM>_*C>!u$JkneKM=vmJX zqjMEjmb!_la&2o?x;IQ^N*~#kI79n}>YquT+^J9CTQ9(30%%+_}+ab6C7&qtN)kVIJY!U^cL7M)Wac z(MmXacff^0Y^~`Jhz)Tw%J~L)2=8{wK}~D$<+-&PT1)#wx5~I66W9oWXbiq~Dz*VR zh%?~((pmnA*gG-=eY7(ra3p?@t5=j+|6FDj#2r%-zchP`c*+txfv+Fs>b4{qRo*0A zw`@VT<`hA$Fu&L*^F%nA^@Y2lK??*SJdEzdmPt1GegjP|#tlMBJ(&*7kBDQ*GkR>8 z2UW#4Ukkk(+)#$*I#7kt!x-*Y@$>^`8=+;l;X?seCJ%_b*^s(nfFZR(Ncd25O?)`U z9t)$OSIulbc568047L#q6pESyA{q@K$zlT5Z0dj-0j{$#9+>Xd87D0Hk_%>i9`I?# z+K5QZ0W(W_@7NJ~A0(HiQ)Lk}5`-I)+cWz?zy!9G{ZoSDXWKCtp~OT}1Z?LvjvnHq zEtCm%wkuL}C$+cpu66&))2gon;j3WOVqONkl|VUcM|%!z(7>+dtyXQs3&JiTpC1rg zIJEVSk|IU)yh1!V^r01~T5WU+H zpdlY`PJ!u+r|7(ybmjv;AX+!T$Ea(A)6as#_@95rW=kalRm%48+{suRR&zFwp!00YbVR2HxC@pQc(?x}Dp6j&0appXFdN9Z^(U1MK@2ILP)o z0=VL*iB0FU5Y?V!^_QsWrXb|ixj$urXieaW_n6kpMAT073h1GczlcZjl0-My7HZSs zqlzuy{e5kzHGP5hO<-$Mnx}jY#wW7G`Z#RG8nbBDUg0l@SeAg%%6=8vGO0DGa7vj~>>wB*Dror8>|IRb-WXABz4 zBaG|uH1S0Yr59j@uh^p59d>LumSQ*nOx9Zf6#uXNXx7*w%;MS+PRxBI{BEFzl6B77 z{%I2!%ari|l6;-?R>=^sMYgdSHyx)r5mp z-PtfE^@bXU1IX2G{{CvBdj=x&sPXQe{UgKYl1?e^6@7S&^?jE0QPJQ%v&vBMpBMTs z#bo~o-v6I|T|g1_CcROS-fd0~pB&|G>X=hYPqy{^{_tA|o|EVsc1AdN*SqZL#ZxW_ zZ-F2`APBQbbit1!>YTYJVqbmO&UL8vOd2b`aTTjvSR8lKddbAXo>xN(3?|SN`o%(l z{}#peS8Ykzc@{?gnS{aN*A8~2kHr|LWLmU$0MD+DsjmXn`M7Pdg&&ZRB@iS%{EhGO zr}tyl4S)xR4FOao_8YmdxdX_`xukqLyckc{2UksSg$mGnrzDDoaKO?M!9`OikUxRT zcOFO4>PBw;bj;qkC2w%`(lErpSD01c59@u`&C@ z`_&x}xCKHuTacx>8bWV+2H!d8rp=Eb8m(+8GPvL}hSaF?tyV)Zkzf`NjIw?A}s);uKj z$FxhfxQ(VNf8`ctSg)VJ)uWgKB#6n_;7XRvQ#lT$j>kjA;-(f8v3qT-{ zd5}8%Qvq!4PtEHXz46%^XGf}W#nA<{Fk-;(g6W|ygXGFq9ma!)*%L$9;%7nttQG)q z4%~<$PCY@pOrhrT!=ZCx19eon4gbnUE2r-8B$&s|kOBrRHn$j%HnO2=pg~0@Go4Yk zy4+(IC9|=B)@)Y&z`5*BJcS%Ce!B_w)zDzG`X*ZkbQulk|3o{mU->NYJ6t``EfB#o zf=K>h(o{Yb3O;1^u7bQmwGX%hNTUUYkAMF7Kfi?*7x0kpK%gEyE>C3v$8s(?gUyMo zx>G>k6Tm|`WHXKubgam(&UM9jt7q|fCP}m++!NZH&qBz9nDHz`5v%<*l{~+b$WYHn zMT`bBWD8EN+bP3LWOzPvh58^)S9NJf@2mM#;XfHXzkM11nAfB*%PDPOq@i?g(+E5K zEoS^VW_JlT47Jw_&P_?JGn6%M>OVTF=XxYLQRbE@*G$j4jCFDieIe&P1oj01+I^Yk z{ZyM}Rhu~K&ZkXk@`(lC81wqXnXL7~Tmq06QIPHC@tmb@qI4rjUKD@A^7>3R*WVB> zDvA<3-+o~fv66hUs9``_L}7Q>-kW*s6I|8VTXb{%pRaWvEQbD}Vd4oN_-MD{hq7f|$=OITH) zodDY)Yxo1AtD}v$MQ_8bIurH*qi0zJuKevFq!w=c6>ilHltPhZ2QhPlAQQ)5hg+2a z%=^ZKD=-4!sEDF9D5_1ELe!F`WTXa?!J?My?GRwo<;*qeHH*rv_BLC7O69)6c4Ax+ zXw>bd*ib#tcm#W;I#cdz%D3jaia7jwo|5?^9WgnzP>)sp>d^E0^H6|iRp>*6GOcHm z(k!mwNJypS>n721i?mA(YIjzLmIo6`iN}X#3~xPcf(~VHfkyf{{D74{r36;{1ZYo% zG-?U=mdLY@1io()y#37}yQF}bOU_#Se!?=^_y~ZHEVPqg21hq1_ed)(7ZtBazV8)<(~Q*(UYaA501yJeoiNyWzxp0pEpTRd=-RL2K~_3 zflHH^T2zTobp0F2=m2ppYt_C4Z1IpJYCS?%A7ZB{ca=fX7IbPU1Fwq6BoNObVpf^KU5mE}Vuflz>!g{T8NMX1MaYucZw zpVa$aa%~Z~=l>k{G=TT87*?-Z;vh;_hbRF|5mr{=H- zx|DhAIQgORJ|CCfcX#B(VlpjT@D#=$aoWR-bjlc-wK>@VSBF}V_BdxU4bPkeRvNCjPU$7zm4 zObr%_qVzX<6SEQ<7IFrjLQ?6o2r-5HOGz<_b?*(vK{0`Ao%6-XwW*X4mE0|FJTh8& z;SH8&WXYFbf!3Mfa>oiow>=@LC&yrwMv@)4UU2lQv5qLxMj1>_nKAM~D5 zRz2G6=6b}C^R1vDk77k)f{p;Yo*#q;x&hIGr~vhyESiBA+2de*WR@km?n(=M)Z83PAG$nW^ZCwrfO01hws(s@R|J0J@h%WkCc#RgVhF`!X*m@C zN!RH01>`}W@@ic>=4LJV)*kR*`r5(6RH+m&cWg}pg z)F9jMgOHM#@oem>dp=-{SP-+~5Ogg1OO5K{8nM%C7yN-|g3by(QV@bZ6L5z3t%`aa z-O=C2)mJBtgZTt&>YZCtyr_TiN_sE;xbG5pj+cOO_Ul)??$rIWq-=+vCqL>CpE3jzngMb*gcUIet1sUJ2v@=i2I^3 zu>kD>8)z6&G|5+N&Fs%dqfJL(V{<>By);7-hKV8Pj?waI;LMb!$v}HYr|47waPgA zDl1fR)ecs}_1=nah6R5blVR{_oUQJ#Y^Gh(3`aK1B)gdz)U{Ls6!3ozp8npG|DNvr z|Is_JF#r=OL}&NGHv5&;%OaXnt<$ACRNJiz}YNmS1A_Wz9;8G6>Ew=>BqO@lRK&`1Ot4nkBDM z!o7%_WS9Q9G*9Q1NdA1SwDC_onX!8r z4cxp0JX)ouG%~fH0N`V43kz*s`o49%nm;o> z3y?t-#tuV~D5LrN@=|{$vK1KpCD{fg-igLzM$rj5qHwENBaff-G^#4i-0eml6shJ{EcU=?s`2fZ1g`*bDWl(OSaC z5p_eM>F^oHeGot6ddz@7`7XsSkSj08N5Dz#!ESTuG#4u$tDEk;R(DSG2(Z^G?ir7Q zY`%p|X-NklD1oH>x|In;L@J|S^5Y+nV=z{$)S>G)4LGCML_O94Cm?15Yx5=$&*H+| z=b1q52PQIaF`7g8@z`E#hVIW$J-WKKQ0_}~Ef~uK5A$28cjDK)jNmxKsuova7h7^n z|5cpL>BLNhrqIcgS1!Q>rh~L${zMjXAkl7db^@@-dwoPN_T@XP22>?Bc0f)S|Qv>z_?*$tE2hZGc79Y+%N6M&?yK=?QZ$g`#0T-lvrydA6 z0&3qJ(=|wsag(29Ppb30WAw8eEx!9?%I>C{7~62Q2lkl&sJ9<`Co|do&Uu}}!jpGf z-=v`RNCp%ezzq4OB^e+AmN33Pb^;Yq&0pHPsOp>O+ScwWq44dDNJ3{Ro8q39@o5ez zh8bBW`d$lS={)IG2U?7hUN`zogg+vq{D^u`?ImK1)>a1F+60Prpne=t?|eg^-SO@` zTCTdx#)tb_BHs`0lG%xIc)r3Sj=}CZl1qmLVWXadV zw>g;ksgw(AA9dQx^3_Onor$&U*S%n-HAQT7S0<9aK%$9z#y!X@Wl>D774V z3Ldna2Wo;#8IVgoO|L<*Y02*7HdODTB133bmJdwYg8 zr^&L~_(;TVueAw6L^|KcZkyW#yN^v&rxHXnYvznwOUt|>1H>m|pZp+xQa78JIdwHu z!zRboE5syjX!+m|$YXL5Le9F`{X(ly3@6yI;i136^#5BiKZcf{6AX6w=DJI1kod`M--;x9znM<0W%4sT3|Xs9M?t$@oVePS`8?@ zkFailTdn$y2eIr%j;EtaENSfr`-2ued#ysm`^s$Fs%P60@zYWabGZ8`;<;!+2Gar( zvT225a6zKCp2%m%tQHo3`c|;&`i39V7FmLBBwg@{#r+ecWWA3@2Sws)=mmQn50I2^4FHxSx ziI_-!NinSR(`#@(*61!;QvB$P6PMrOp;Qo>1SM(<=HCaf4L5XZ>T0A2Np)%WWb~-W z=LFkl7ke202ttpVq$X_rUPB+^wf17&Ff8_gGruOTqtF%!?EpZTY8 zZ5!D(ieKC>uzcWbUb)SP;#C;00Ei?si8FoqffGw|n+<$+WS2K?c8q0x=-aJir!Z~n zXm^ghlE~|Cgr|f-MS41}WOGrJWY~Z>P?8@>*y}l0%W&KQt4xc3133 zq3AKQo6B1f&5(pW$lLhAO>!#5j)d1CA)+;4Z$V}{jHTUTv%E)mFQ>aUbx-t{5saS* z^{xN52@D{k;_$_(@gI;OWUbQ#fnC3kFFQnINH>pz>ETyX9^p9ErUCvW(c_Hl+QJNL znb&t>z#+kh2#GwV)@+;RdKD?uzw^0zNorNXBldNx+B#$KNBl{*R7k!TRTDje;g`>Q zS84UaXZS+gVI=_&8Qf6!zeBvv5yYu;vL2BtqXUgY*L9@@P4ue7qfe|#UTBvvIHD(ImY8tOjA7gm3km=}Q8?~x-51+gG7JRXQG*QnQFqAf-6c{YA3y2?hLTYpfq2MwHCUbF4lYRj8i(x zgY7`nM?JCn*T1nnc2?#|>~h3v|~2&B^5n-6s4y-1+;2D&i3c z>z+z3=2P7qup9x$zBuK~d*7o=?d2nir0T_O3#p&DMiJyP0a4416C9VX-S+al%X{s- z4(tU}kl-JAgTFwozyBHj*^j_x!)XWko3LD9sEaGpAO+LFH|CiT$ zE$${J#C|T=I-{fEa$CIv2KbAD8$~1r)J`f5ksDxGUR}{A8ReeBJ^k$I(&=SkUl#X< zCT><=o1un>?%JAlzl^8svwB3w*SQ_mm*4MF3KA@lSMm0oc2IZ&d*+~N=1?<%{Y{{j3j}Xyhi_8 zEmTU(ufaghxz}^*}EY!zv-$TYi{KWr)ub|_|V(vv0v5fqne@Vk4?}nqF zL>Z%#tDEnuVymSD`O#}vdV<>&4>o}SX`E9EW<$J6_M}`P!VR4e5PN=kNuw0Ml8FN- zOV*I^6b6<$TEsy`rLq%Z4BfvSrvFCV@ekf@1gMCrJrAS`2Knht{O}zn^RAQzk4a3J zH&26p zIBzgL%k!^tlIZiG@@F*x!}0^tk72{iZM@)yMD#om_(LcQ+)k0?c&ED>TVBH2ooh#4v&MUp*~vWLjN%!sU6icknyD_cS$j9rps z&pO7Eof+#eOTVXk?>YCLbDVQK&gp*dJ-_}mX8C-c&-1+3*ZcJ%3%EzS5t@TwnE+?$ za~(Z3fFRRckf#6PN~l&r+k++yG_hjw8RTH8#I+mw;GC`fBuM=gv++0lK!K}6cL6#8 zDXfgQ=c=VV;-Ok;i5A!id#qGXtn-FflH->e!``JiwkL90gK&^4}{$NowuedU?{wo(3XFV-Hm$0lbE zNk@{;Z);%FBp7G~Z;}b(Xq87l1)xZfZqhu9?tmi*eg{mMRj{1*Sk*5X>9bN}ypi4? zXeR;d-Is-jMJM949@-cT$M|8xVHWCir>;p;m|bobAVYz;$GG|Q+JXv>4O@mCJ!Po4EY|AIz<$Sngke?1fk#4-d>e1$Dl zhotd{>f?z7zIqbK7=6)GV;2E8;p$ZCLUYEGb-mEPVe9{$*Ysnl6>$VWOcUp|WZ3|g zhX_UwpS?K`UZ}8DK7rKsRQ5|GhT`KNNtWZ_xx^7gBszC}LWp6zI?SJ%S*M zzQ&%=g4QrjD^nRW8182mjtQvMs6i_g6!U+E04VVcB2S_(2K@qEU7R{uaM*l2n^#9+f{fm-FlM7%UEY9EB_lcyEDuUG(l!`y6CvVnOPf|1HZyiOg2cYw&>UgOLo+udp1bc;9;Z8uA7` z;us4RQS-6IasqdYIC)bAqfNU0N%O68r|gpy$&e)a_6@G=H^-vbyC67_OnQVCIF}*m zG>&A%=f=vIF1KiclE`~0cCg|7HB z0(<=aMiTPn1Y}N%&h!&em4aHZUBA6_)9g!CUe70qojuL)H#!#q|d`H0@=NHVq zCn9;5M;mWnGdZkv=dJ?GbA;C}sG8UE zt=Yl7SsC6uK8N*{M!cKLaLVr)AYK?E9kQk;yoxi8P>eU}%iBzm#?>L8-3?ZxHQ2Ww=!gwNR9!54eoIC^`zSd#8Xkp&6=F| zN{-H^!rURaP;<(*YT_ZQxbk`|gN_P+Swf-1$2GU6y-8ceL88<=ou8o`?t*QyyGZA7 zVNUPiNiWZcfO#NGT#dz4qWlosl3bB{2(+g^=xc2gT7FsG9Hk_`APIzK-dRu=Y;Ze% z*{c7=$O#gFSNc2w9mkV6CGstE5Qzc)TmdY4mCX^oHV^zOn4)W0=hio-GxU4C=mvVk z>+G=K#4rBdOZwsa|Dz%+U<-+W=ahwNpvHBPLji_%P0>xByNJkuYIs7D1HfC(aI#>m zw_MNq;p;zR&fl_}|H1!3-vbQTOokEWt}bio3Z$xe((EUmlQoOh+Ugi*u#o1cu_q5B!!o9!W(fAZLS`U;VS1r zcICmg5E?ZL+B~g?RP2_xL|hhTnANn!loJd_wW0^)f~Wl7J^>L#-rox&peRVZNa;!} z(5XQ8SZm$Rx#u$VqW1gQ9!e|y zlytPU$AeCbRL#5N)rYD-`Y=GwZO~18`#9^1+8d)H zqpJ5wFDdLMBj2Strn{*jo{s}Lm3QxjrHXIx6eDd=l^0PXk3sjPA|Di~)*xgSF#LR{ z#F*@+03hkL5R1Pr{!hN&%r%9g_{f-2DuP%vI8QrTPhv8soKX)$Pt(#;?$g!1d^1@^ zI!l(y+A;VCwjmrq>3tpn%ltx$93lx@+g_L^8aM{(U)yIhNzAK zCOde-`o9HjXzq=qT!3OxYjfCUGIzm#ES_hvCu{k4hE+c~AphasGaHaqL5$oYSX1hV zDDwW&B&Y$|A3f5JOKglll0M&xvG-0Gl|*- zLFjCV%SI3EH?Nkt)s9V%;*Ouk&y&uV60B1ihfL!Qyu?p*pxf$_M+WprCh6??8oh<)^RB`hYeOkg12-Wc=!S0V(DBF;ef22qZIsHJGhrj`*-F(5PL)(v0JB&JHK<=KFmxfTcSp38{{2$!xb~KAc z9wD+?1|48}h*>M|)eDpUBSO~$5Gx&k84_8WxD$;{L|X|OKsix0IrLp6xbzoh6-3_9 z{eA=P2Mq#Ews%(;))zvM_5TR92%|(#(zyacG4aggc~=MZB0Yk`R!=Dl+*3Qa?xP;V zB(FJ4f1zglZAZvkpBNSz&fYSSH3@AFEM~&ynbFAYFzPDEYzH9!Yt<_pU2^zk-7YUjI zdF7bT5ary1)SaN-nY$_h3qUUo+XI}k| zzx`ktRvDN?JB2BVna)U(Z*9aI^{m$S`&ZRXV&m5~<;}n@$b%8Xdnij7LBKYD_!0L) zr7?};ow*$oT~9t-nsN!rMDboftj%&ldwU3p_z>s%5vU0RgW%&yX)Hn4%muQ-*km=* zOO1Bm3O^LTv{azhXy=g*-HBp4J$O9f_7gRSUU`J>hTRVJFz9edkzz|>%i2&$)uF_pzZi9vw#$zm|O;ieI#-0+7^k`0S zCWp^{pnWe>L}H2=zQiQ846A`|HmZYe`DBpk!ZZH-71?5w@KMS(4SbK=Fh^0CEWITa z;`qs%F5)m}b?R3Q%;8I15{| zc&4N!PPMH4Fx|35B+(|-V$G{CG&Ek7p{%e#mF0Y03SD3@M2p%RQjtV2zR^O$w?_AE z^t!&dyT87n1EH2rg)83`C*0hSIHMU~*utq2Pp{6+X19@h`%oD0^wld-92qrsx67gU zx9@l(^;@zuMq>8GkP|=j8~)I2$zQ3yM_ZAIyI_0^ zV6F}VRbr$41Qgary~bT4`r(22=uSkbwL!`ev%B^2*bjA55T)5-0Oa%9N91&6u17GS z&5h{@Cx?W@dZ)DSILuiKB`XyM&GNb?3vFZH+E|CYG&3X5-L=7BVPjGBdS_OmpXYRh zl9uA;hYtoJG7%Qk-hgf`fItO;iDv8yg{9M&-|o_v4&|2jyx$KFf0Dfn$XYTSH+V1qKXf!8(IO=plJi)djO!E)`=5MctFq}otx$X z6pYL-F9G3g?1F1yXn>r~Tzv8%F*4 zjxEeW8@Hb%ud=h@o2^c%Sur}8gY4jpy1z_#u;~ zVzW~)6E4w=FEaaj8|b3P=>;H(@4XSifTXemQ2-+fek-6Xb{>Klj6xJ}yWtb7-=Vt5e|wEKlH^E^%0ZdO zDL}(0!mct?Mp4w2k1e70H(lSlANLPpW=F5ODV8kuei9*KA%qQ9d~;my{H9}om3U9u1JS9W}BNPrz=B8EH-uB&%63>|#re=r_*N04PG+Edv z9Oh|ew`5wH(Zm}p$K-(WNyfWr_d~#(%_^V*!a@~Y zsMx20%gtMtY!a?|@mRfmRw8pFTRulJhFubCK900ECxXGT_{3P&?)a%n&2;Syfo6iK;4bM>00-rMS# zShwlzrsZm{wrrU^ylJATEk+EXS_&ODW&lw1=%``T)=2@cRBP6$o7Q40njN|*dV$S) z>icfd-qUt(iHInXvN*W0M~M(e*ug^G>+5ra-KC}*aWTlJoBh+HyHu}Zgef`ojf`U% zFC5Q_U>aPE2Vwo|MbwbqsAnr;}kXPy2Ry!1E`5AU;_-szrxB;56uw{)syZb7*HQE zfC4X7=RVaG>UkF_HoO|Mh6lL#*5W7B{ZGE)tNt6_LReWIBInE61C_9tYRY!pmRl)( z1C!dcnkBD0_D5XQ4aw#>9T*n>Ns2eZm4~_v;(?lO#79gGcU6Yc4jHASR;Oc(S8+xL z^13zoCk0P8vD-So{7}K9+#eEu&D;IaFDqGpUP$|E$LW{9VD9Q$Bg5t=Zk@Z65T~2p zTF|)rU6E8wq|o`A@~n6A+=-fiYr?nyO=789I8W5^MVmW%s9t@Lj+bjc7OQ}n1|@-y z^`wHkwj)>;?{+FI41&5k2Vlho%UJ^=?KvU^XsD4|n-L*S2O|_?tN#9q_VL+E@uM%8 z_l_=9Ldd5G>_jO*8I z&^K9_dBA1&u?^Jil5-AH+flZafr_2epxxTkGtdit3wRzLZu|$fb|7qkjHMDG9uR%n zm7a4fiPyn&M;hLdlsw;W`_aVG;|k@%$DSPy6`L%m`@iNSt+)L*YRdrt!gqO`9OP-3 z;#xAb979e>E+ty^i#*M=)VI3lV(plEq`Yw3f~2mUBkZurrE5$(`z(7eB9=ja#3y!K z-!D7kHp0TgrA6mi3DS4M*HL})4IOjj?M=_4MDmUc3~9ES_a(67js0aU zQ)_cwPu9rT7N$2-a%QV0_=C!d_VK{`6;0X<9;u|~gz9hk`VIQecha~>Np@<_8xCAj zqS^V>?Lbsj^1+b^bqEw}(Lcc14lY^kMxh{P0_^VQFuo1AW*MCJk++?Vmo)Hg{%oDz%d2QuQokxNb|zILFy>rKJ@JzC3m%^hEN( z=}q2{<#ZK?Uvn|@%{}Uaw0@vT0SL)^0;)elcG48R{tO9!#G}z&JS}>WD*~z|eQ1hw z=H0PlfLP_rEc@zk^T#_w$V{BnumJx zyGB;77J~{o%@{dMgu*s)2j4j4f8PY}oNs$N{OXO2rq&r=n0OepKjHihrHdyE246n` z_^+{lJ&)fEd0nec1?T!{hOt^6dbl5x_!+X z`VJGAxEo|3y@K`5CCn$iG8=??3z~OWGCnUT*t3(inrSy4u(21g?cBA8w~wq1k!Tdb@Myht%^mk9k5_ z4kh36hU|gRT0tWIzycAfDRfQ1;pEbiLjm;h(2%zz7=Vt$%$pl+0TK@-xs-Uo@ih^< ze$VJDo&bTq7EnkevP}UEfF&qf^>9Bcf^O1%BsS|=m0eLiOC-Dg0r@!k?NQq=IUfT8 zq{zPmL%u2t`f=xhynTaHJYRkXCgbfArpoTI>iNK5kq>F!r2ySSE6;6V#T)K zycv=&LfXECg$j=dLJTG%3NaO2W+*1oCVb}9UD5r9Z0BWc7V4yHUd|s=f=vhUHfLC! z<9{{G&!u_DGO=iE#s#M~D@QwQUcJv#TMN?6^W5LBYM?! z$gHS+1-c0UWs&q#<%a06Ql24E`uBu>AqCNy$S#q%Youg}9e2j<_FOcYtS@s_Uw#SL zaIBwO7_*baxX)e>UYna1JF>X#24m22oz))R5{_rjqu(tZXHZq^u!&w#MHiJ8p?<#{$|u|v8GI;#N~TC_m6!{3D4DU(J(w~ z8XQT`xCfnX0oC&@LxjCcxF2`71pR;ve6U18V;-*w;|!LA3F`PcB|}y0XGoAq?95KN&M1ddI-LpS z*Yq0LhuvpSc84Bq@zZTLnwYrX^!ti9jd-bGPpQEsj^P?3;jZiAUSb_C=UhBws+qj> z0@>vPG<$AICg(IaC-$gP==if--_3Mt$$o z&G~0{Inwg|a}KS^FQPtXP__*XFqP#KYKaOTm0X5)IlXRO-1gv1^ab-9uS#SC;$)vW zJaM~TzKJIUOnd+9l~d2>Qnqn@hVV-as}ou55Q|uId06)1SC|SaybEd9Rf;mqA3beX z1d=_$w~!_DM{CMCCF~s8wGTCMshw)J{VTrdzu_$o32;It4uiGH&K|2#=d|R81+Tty z#@H6?_L%MN&r=uLa$0#eJA0iX#Ll9p^WKo~p)SJ@5~_x(YUM%Zg|7=An)+Mx-hDHv zeA!D8V)&CK{^^4MXa3@O^~%3>z#ucAo8@5Z@vFzqi>Z&w!ycjw>Pak_HE;oJBnhzD5W&%Z(0Q67QgzHhbHxn9%gl`rcZG=WiO>xUw^rB zI4kefqUmJ?k_B~@tQ&DN@@?WyZq+!Djj(bk_B}*cd>D;X?@`wrtvRKHKPk^mHLOnH z6|H_fCuSV*8N%wRkb0z1NWU;%Y9ev=6N2~FtOK%uYD7Pr#DrOn$DM1mGTu?nLvWbs znk`IEbIRzKltFC)J_L2JurAJkrP&)X_zbpvi?FTiP1JUTeM-knVH!_2hl&Artx{`R zmFQ!uf-a4KWk8MLbK6t)7#@8~f}Mme?HThKeW|_jPGD#KtEbBv!X{Sxm!p7ZGA87i z@<$EX`5Cd6I39BV2I)Gtqwrl0>k;uJObQk58?X(mF9~9rV zY8I2XAZ*fg(W%6aw98}geg(>!1jm;>l|we~N*zSr%$QG`*6lnb7Zz4eq4Odh1v}oD zP0yU-F>%oTGXyzmW%O3j%WMld_X}?A+$%`qaMOuixwH>+I}me$J>jPN$R>$5wR@K- zkPy!nx`8bj>75ytuJ(K;)xJyjrHJ2%oU=p3m@n{W&@3)@nYAbKn}766}l&4 zn!A&$n!-`E4BA7JLl-7a zx6sX@!r)LIa;Tal;x1p9iZj%D$0wWSnKYtM&sY|wMzc{2m|dqcmOrz>K! zC_B4uzBWGN5XtQD)zLHz;CD)@up~-5vw{0+1RM}P!N`Wg=5Go2ob16H9vg#-5av9< zi1<#i5FY^gV@>03WE)&@B`6oHrmVORxCa!wB~gHRDc1ZrnBT0_hEo~%l(_iHx;PI% zwmz`XHIv6IQJ z`jazdd7s3S604qG2!b+8DrbZJqfLmLPfH9JI#mj>nQ`PwnLNeorP3s^HRrZW2ke3} zYVZ-_Wl>F+9_4`RWEDe=cw#T*z$^59FBokkbVMJkLh)unwYcRP5{x}H$`Jy+`x+nk!@)|7ip7o~hBX?#Q* z5kXUGpQd(~q`Uy3gikW%y{5>7xcVN0#My>)1F)=NeT<54*ZL7+#$V*mru%_V9XHrT zm6s4C_dDTJ7pp;MiYSF@4ggyZrvnAt^U2tyXP^KGMUdmrW7Tyif*`PG;TkvAIu3&( zNg|~Qvw$5&kamHRO#PErURDoK)qwoPY;F~YPo&U+5OBEt61?y%0y_rKr!FLdEBNo` z8$c>;QUs=-4*Mdw*8Q5|DQga{_Z5n;#0JgQgsyXO0i90+zO=}eZ9hX2q3j?N%z;tl z6sx~q^H0`U2`pn0rD-zFKW_Lm&+}Gn18MBbBQ)Dl2-aPPaMT7Y-E9@khi>PC@_u*EiHHntU_SiCX5M0JJdZl{Jeznz(@vM>S{e)8-p4Z%z$kFqPJh7Vh zDSTf2L}?1>Ol?7%5tx8-YL9_6oZnPXDo{dPrl97_8i{(r?Wo{9&6ZRD;yGLLH+T` zuLefHUWD3ssJ6~`(#4jg#;j@0ga{?8#<}gk&`)K$*Sr@=Fk_Jp?55(S^II?^W)I5QsS3^Y{aqr2*#VEwa{+LaKO0~Bt2K}t%fb$2<`C= zKJ*PYSbqL{+sOvi1F){)d8dA;dV5NpWPdMjHTQ$d7Sw?qD6_flDT6*I^;_keUO7+k zxH^AO`3&(2TmeD!8!OcF`stWVCfo9@?^sG5vXl~RbSz3(0%clUKi z4lsC}r>GyLgN)^(2cr{Tlno%8t%ou}IMmE=_1I?!IqC*}jv?He<+?17wQx^4CBCFS8m~)Ap&NCDiDs6QF z&&sXvwcJpd$USGRGPdy?_V$EVfD#?I5=f#-#JvLd=XuXpy6 z`}ZFJwDJDl!oY^p+#=Y5moMW5QLnFV`l3j$A5Pguj|)NN5S|V#e}*^ z8mdUD(+X(tW#MD>7CBkTv84PH973w(EwT+1o?f)n;s`UVt&W1XSAM#HqS0Q7_Qb zT`9xU2&z)rVt4T#hJts9!3#VbkeLGYO|GzNMfCt#&c_pq@VSo#C`R6T*n)x_$BXx=>rwvU6fit>=@EN(pkRM)mG~6Xxur! z#ZUajd>tv$D@i+yDO^oN4v^gK`3z|c0P8an3*=b)FnPIT#%9thCu54r7yZfa@A*Gk zpOzNHW%X`)`4YBh#Ae5dYrBV5&X<*{6NlU$p@5>I`ZfZzg0ks_jNL5~SF$UzefaQ% zGoyZr*C{J1kaj!&vP$v2>h|AI^JDolFXEfSa5Ze`PkHG{th7uUXoekSGKzCN`i%P- zZ{46=elVnLgD!VtqY3}oBl5FOtB%&rqgNT;c!g|RkgW<{y@1{N=FJ;`P`dwnV~~Hur5k1F_<30SZ)2YS zn-6@)hWu!K_o&z8`XthoA67Dj)u5>-N>-NbuUf4fJau=rZZIeHux5ejP2XGhY9WqD zPF&}$K{8!)HJr6&Mm$?kJ^XE9eqo>zi%R_9!6n`Z_Xri)SV-Fjx7?q;6X3dJbh)gj zDuI5`Zo4&V`%<9zRGGSVMCa&ee(<2U2~!2t{=3p#iVzRj^xQC)#}Kpt6 zw3az5o}lqZxxscAVbPcbM+lQFc#iia}ye{#9&(J zwwOCT{eIWT!nIpw3rBovylWj;!nW=4_R9P2>VF(5`QdqeclAG7Ygx?DXUGgA4mEhf za%>27XSg-3gWfBmUpW0$tl}kGuX}=rH4;nZ*$1!T<+)J4rB|`xf!yqEB-6ouGX|a0 z9s3uuMF!tDupLmR*|hcY#itOTt$TMWLazU}IC&WTmo5Tor5aF9O@Iib1@VH7 z6vtm}M}!!BhVaO9qc?a)zL?f;9?1W&X@7V8^9SHqzc?|!`^Ufe2fmMo{os9}At+nS z$AuyFnQcSOI|yyq4oXC;WsiO9?DX8M{w9_)t-OIJUL-Ds?B2szXdfRB@yM9Y!+Gcu zm~kiN33fwVXL^X5c~Z>%4OY4pFU}opcp+B7#B!K@?=_zrN9DN-|G?qc@ZMc-`*@0i z&JJkNMO*>O3s689nF_FqrzmfX=dUL+pr@(qNI)2jyun)a7x*F`DB{IY&KW_89M=(x zTD1@xU^}GHBkxNYK0^$hjLFiMfC`@Yde`bs*| zuTJvU!g*gNysm(6!IR@Jjdzr9<)JAnx^{V*+ zYHdP;1qCk^SPm=f4m}|1l!?->-cgZr=L9;Njgf zjj#ByuX1x_hP+`LtVVo~)&Kv9)z!)Q2$Qm_%h7mNUr^bmCsfrdwL89MPVYO3qA$2m z{YgCVZ9_yBeYpb7p0Lx>kY(fU#=#y>8p6qBPch0zN=Ka9!f0_R)Tezj)tnx zUL{$-hLI1nY2VNFQ2;*eG6XmYH65U?*tFqV8&b`63ZOz~5Kcs0EB&T`+ayE2Owr(k zG4(E{)|=O(0sjH~Z03zEwSHF*9eK4`qhL7gM`@tnohp5=ga1l|@SP^@Pb*CRJj(pv zkF+@ zAXIB;r2IxWPd1q)o?*348Tbs$v=(s7dk9m@*KsjH@O=Y@Mw2Vs91^d;Zhqvn>5}q2 z<&pWCi8aL{~{UtjvM(gk~lco!y1$;AKfz>e)UN;r=9ZbOslDOof!2ds!Cbnht039 z`1kCdnHf?}4mz=azu(34harHleMTPo2dYaOcS-A%T;Ob=s{b4)4ju+tumNd+kit0$ znvA+8x+1RRRYp9mQqmf2n}#x!KI)~p7e zJdYk}_;gh4I>u-9z)8Q6kgH%a_|OJ#<$INTJ#YTks&c#X&j=GuJmts_=Nuog)#$@B zvnJO{g`K$4ycOrOt9LDM^?8d``greferaDPFDHnF)*923tO*{efMU$Pn8Pe9O#AqQ zs&m@|rz1{JZlj8d3ZU_=qyf6C&KpKc{6?C0QwPH&OP@Y)?h0zidrBHl$RsuT5QDRVnK4u4c z+^+32WG7) zpU2x>IXB!)Z9upu%ZUhCJh-$fFOUXL*;;x%B;Lb{XGBs(NF%X&-!jk z|N5rXvxBIItc^v?$UPe$;0>?KceT@NwFwQ1nchDuF!Ro@>i)oOv1o^_wl0^$2B|yY zTRiu+z-dTFS}5!mY;72W2@`FP;&!1JrUoA6S@}WFck*1`)cj8&a~5B<#n#<-zxpBp zGh|M>58a9Sc*2gnwQFGj1>mrklnA4%iNe7YakAdKuDh3i25qCtAH=M3iifX}?|g=w z*wqVab>%G}qeCsGS}GClEh+UBlJ|IBUhhf&Xwv@*FoS5M1UVdSc}{aVrhSPEQ7si= zV8zWJqx}-Uvp7V7>$M>}t8=8YR)_NOBWOWC+7NGK`fskuubb8H^~+z`z~Aea|LR@V zS)@(UvwI7JGKwGW589&F@AHIvr2V~p2Aja^106vHQQb8FDZ_z;jSIC%<}P=~WI5mY zbd|Ds1*N|l@3PzU7=Po8G{$0zFhhusIw=j}SRNld7R4MH7W52CZn>Bfx^w`8qRnZ^c}tWVCoIJxy)VDtMe3dcSb zJg(Bcz8w2*vi-iP4#rZyG0s3RIpvoB0S`;b_(I|vQIwgSMEhrmxoi3m-^`c+ZSzP~ zdQrf|SaOGRnMvZvhqNfA={KWJ;oZ%3&Iem3T(tGQjZO$97~BJdN)=@5>e?IJrixOE zb;z)0d>dXmBU(GW+RSYrADfM;Zgb=4jC3;Vu2{Ib7xsAEdIayfM{Ucc2eG!IO(>mX z=4TxH)3eNU_WjF&yiSm!oNnTwpiMqQKKMD0qUQ=w^)RvkhX#e$OJg}3r3BOcnVJ02 z*8S~ksBmb~t^kTk0MIU!6Qx8JLcExX`%n!pJS73nYBPFXoNDS1V8Fh)6ve-+#c$r{ zPw)E6UmF$0T{pP=uY2*k7{cG95)xpy@=0pIhq%U7;$5#10w|0;m#3@GF|G1zEU0;6 zL_-IRqw-yd9;je%cmg?umk=sCjlr@JQk{5m+*}}B%7;rNt6X-vy{24W^{`KHo@s)3 zY*dGNon1mB6%!rnRS*-?b)>g)XWzVXrIu2$#3QSA9M0)b!)Zb1#S{1MYmF?*dekKr zXg|E%Y&u9(@H$N6_3VcBUR1XcJ@Qy?bH1%ynqrBYqg&#FT)CzhCk5Z@?Tse3HmMJ<_q9%db;?R1*UXBp2xnKJ z=)<-y(LUxLLLaFeoEkh-a_qsY21it9<~#hPW)y#;gqgXHz(4HUmS$hn$n{pl)Uqgf ztB$P!g{VZb2bIBlX|N%s4wzNfqW#jX->i%O??Y~op02qRIt4tn0$~h=H9&115623N zUW!zSU{eVtIz@fZ_UOnJU>;K_0@ z*+X=KCx-EkKbPJl5hmf|>K=lC39JK)==jKl?=S{+y<^Y2~FLYF>3wH-!19G+?|yX5P4M7k;S z;);L^V85!Sc(})5pC8SgliTmbhDB|5wX0nA={nJN_4a27%R8wvWBuWXfHPz}p@!#f z=iq0Bg!^YPnUs688r5w{*)*boo|ae?UCp&Fr#KIjEvHP2l#zjegIO#5{b6a7@kj~4z|W&KMKu) z896ndvCQih9!%kKEvk%w&14JLuvhgdF4SOf``X5DSPvhS4d=T5_Tv*h7yVo`X}THy zk|T|FF1g+Z$wx@6Sb^ST!fo+J}cet1OX1eO=r5f5q>wV-A^R9lc`U&F~BgM;X$J3e=dsJP^x>j?K zVycI(1nPv5?<5G}oPCGoBnoD1o38DKU&^ex2h*~}s71BJn(n?d6rNX8M^4ll2Q!6l zvrp_AZWQ-?jfY}$@&_4Cn!b@@Wri(!6kZZUy{IM_^saUdJ^Z8)^k#Cty%TEbh*Zy? zCx$1~V9oub68hi6OL-pr&GJclf~h+mGmw%P=b^?=ymr8>^dN@ADfdG`X_g88gs6>9 z3D5d%yG1aj&;jB+-s(!&GIz{ph)Rz)JcxW3J<<*5M19=E%t!>cyf?xjD+VJ&D z&~Bb(j}GL%qFT>&&BKH<&0pXO} zgLkMGQV00SW;}7N@v_tjUxU`JMHEj7aUy7 zvq|g(`SQI-2pKiYV?=2K$S`8=8O-i{SutbX2`h^vw~%-+xov4ky###i^^=Zb7-vM&a$<%lVU1go|H%Ea z9(|FqSPM8OUOn7NC*WMstMXS@%Pmt)PK?YOHmlc_AKx4^d^C2JW&ap|CMxs#V8)sB z{n}w!OWFL(rYAR*7Y$wDr~HAl0#W$_R#M4IK)VmN4A0Dq@2P$<=~~$oLwgyx$#OjoM|*AYy^8ZpaB)oapeg zSWp-@EPna2tSF*&cIzj9EdgGKgQli(3baphV((-1cRV^3Z!|Ia&_+P;=-Awy#La9< z!6+8jGklWP4@Ium^D*|fb-0f>uQ`%3f{&e?)QFaXlr@dQMi-0>iS9N-!#Chf9%l@k zW#dkDNvQ9bJU`hZ*W24y;gExV6L#B!gK(Wsi)d#P-XXdX8S3uj#(?}uy-%=5mKP9AWSz**5N|JTHm1D8 zXszNFYZ=M7AqGh=9-TtfI|hC|aUL7`yA9kxrq@{Wzl3AKwi@(aaF7DR#ftoE zvmDu8YohR?3&xSMLT|%)$;sUm^YE@p?kh{pRZ!;IAr!Ul8Of|=WuY;R?z@*^wKq^Q zWQ^a@T@Oa@V|oHgCX;c8K5qAO3 zIgZ~yJDO89kX19D&79}cKy@bdJ%W8LN*=eghfApB{&S6jYmp0RgYuj;n1)}dgo=#E z(ezx4akv!W;b+JJ>KfSc&N$E)l_0GE#K@a2Mv64jywL<5iqJMD=nPLKZXyIrr#jn6 zP6y@;HFK&fQFpjgu9i9CKDo%o>`uqbj;6-MptM#E*>2a!%Be1TBx+4^e7cu5VUvxS z+*xaq7bH91eIfN4V_5M_SRpag(-y{{Fl5zq@PNl+QH;mp%dwWXsaLq&gKRnCA(#7{Cl3EXWccA=-@Nl{idG)hwz;29wSLD^2Z70+aDI? z)NAxpIZj7jZ|mzjd{N_rN90E+=m_z#zLY7rvB=}5YD#vTwXb4&dSG|9-coLLF4ae% z&3X>b@S?vl;PtC|F+|d5$TjC(#Fq*Y7vA8W-FJ%XRDTT?=fm1r5w0uPw?&he@G&8a zb~KjIXOnH)=l9)GZ$A1N;-s`7mzg5fgj<#5>dGOrX%j^|7DdWjuM9WwjG%YBqIK{& z`6Bt5s?m1Q2V-VWJ&ICVZ9?vfHFFK=-HAUIklF31`hrpjw<@&MNt6wdSqSj~khg9e zpoNK5A|q=7#D{LC5rqLEVHjfVf_Mr^c`ao|pArUt{+o%%|332f9hx0k@fiYJ$_-!3 z$-ioDGQibNQj$zB!M0!FzR1+a?jo*OiaDZHLbLbz;BYL2EWSaI=65d1U-;&~yKhfJ z0KL`))J3$|Tq5TT5b_D0_m;C;1|;5*8U&Be5F;trnvMj-qj19`Ro|rA_N&3iE+;O{ zv#r9lo!L)%y5)8EMhuyT(sTO?$u}$}(I0v{$+cVT`J25XjPmrqSuX#TtA7Q6DJXefdX5FQ24=AmOt#X5k)3byYJi4ZN^&mGu zDL%iAS!2u{EBy?ioPHIKZ(M*I>>^pNrWrZCt}3fK zBy~A;tLfbEnjwp7sJ2ZNjBXsA+jvq%D}E@2Kbg5Yfuqw?>#0(jMYOv8@p=0d4Uc^8 zs*oMjj1RRh9&Y&*Ns`4}W0^QpP<*)%o1Rs@{Nq4y0majnpmTt}wm;!y*0?@8AlF!}vtTuwU+wAsdx`dwaKYYqiuU)TyRW(9G{1No#uMYn#%I?fve$Ht2@Fg#O=qOY7#6w0|{?pw%VdS*3sw zYh*oA0O$XDu$gYEd_Qg};n9K2UQ6dzLr&|u2hq;Khjr~vGWplR&zh$}yp9le6HeBb z(~#pm4`9QCb{ZO3OFq4UY;^2RFD$Ygu`sb!JHmcR6zB8**!%9VruuB%ARr=Q6e&`p z(z_yvv;_4NM5HRcL`9?*={3PZZvp~>0wP_Sbg7XpO?oeZ0Ma3$gg^>+JLR64d*++_ z&CHo|=H6%i(B}b?U4HwwS9#aF-o+OA1;jJ?U*y67C;-w93(~-dUd0qLO>wLAUwOt6 zWfkH{T?-!ar}YD~W7pd6A5z#6+oCh-8$f;(wyT=-_L0&@{Of6iV9!+E<_TCl;ukN~ z0VFj=Fek)~KoT!BrOF%69^;;T7ZR+M4<2$I8(o{?Y_a21dw4~1KgN6wX(Ks|5H(+_ z4QM6aAqsDz&gBA#4luRxdFFxY6OPEw0wW7i!8Nj<(gurjq3E;DWU=kQ$~;IQ;OZ!x zDc9>nTH1T;{R#`0je7&#jijgbM@&reJ?52g{cm=8rwlOH>v(u23YIaK=K0BHu#Su+ zK~1-;F8-&!g3YLjJt-+;I?v^HsX;+8OScGG zdFiasJcp_eZ*7TJUnD-OvI%U#otdy~46 zDz+rtV53l$iG?&vp7q*ZW)RY-4};9i+(lg?g0~=#4h;(9oGFqDdA1^bCLQajo&(90 zNG!n!1F5ZYUD-PB#ioqC_Fl}*7nyR9JlFOlzP|U-#sJE?Q_56AU3GkJq}W>tO#+KY ze@G7w;<*;uH(KOq=@s0}!4_t3+-x@Zu_iQ#>QS(k`=f8LyL0V}UrfKpWre+nB^hK) zvkzyS>`Jg*>I9~AUjgRho%9ObMT70^kjLqttoB_)%spy`Zl`{h|KSB*9T+V~qH|N7ZWQo7JB)@`{ z{VWIW*ilYq0GT=%C^ymq_=Rd8-EvM4Fki?FP)=U!1d~~Xs8#;XKC)jw?ccjDf9v;t zHwr+7gf2%#a=Dm@BwntR0rd@Ey~Gx?(B?bbLjEWrfY{0);(uNE52pY3ommp=!uMN za@+*p%etVbNdxfq-ckm#Dy&B8T&=RZ8H9cN=ZJ#02JP=YVv4@W(ooT>sK?PFXXW0ILS3IzB*J>#R3Veg(#0!78fAjHy5p)JX0H_{|x_ zo6%2sW~hK`pCcEdmhG|o6!0dJFbtVs_Pvb9f(f4B4WNblmQ^qciv(|G)Ml%cw`i zlNn9A^rS18(o)sV%0$e#!x-doo8o=LLonq$Vul)E>)ARHgjClHeYvYzGDl^^rDRXs zZ%we$u6Voj5uHBMP9FcZ@PuO}M?VGrd}=@nKt?AZWo#tocXNX63ij7gGW^IR{7zP&t% z?nFsq8klUlAOTVDIz@NjpNP>A?&mXwuw=!X1DjhKwNcyOZNxVnrW74t<@27N?;HKG zyQeAltZHhG%orlZ92;jOvgwhe`rnA;g%sn+Xz$}CL!3Kn0KfDUm6h~}$QfGIP> zHP2zV$aPIFcW4|Mdjt|(EL$1dcYaG4HjS4?Ce0Ai8Yw)uz{Ih=4J8R?>cwx@V-@%H zxuO^0_meEIE6${qwch+mHvHZ>Bs{90tO##GmYHWUNYAj1Jg$z;4O+H$R(k7<(R?U;lir>IRo1igr7KI982W>SXGSZQpwCk zo#5J+nTt!}Y*R%5cPTd8h9ye|dH-9T2~Mze=IT>3i~Gnj)~U)8zZl6J>LMjg4L??^ zG>aXN5dp;3b|LS+`@z(X-DBHP!c0np1LW5594dL5+t26P$_?~%6_jXOxF@86m# z1yaC~sr2)`QIrcfsg9iGyAygZEe$gehuX-x>EO?#9t1q)S?X0O%>>B48>vBX_0UJM zKJ^;!S|6g4 zB%QnKB#Xa={bWVf+ssWnP!L4|zux2iyQ^8wiST2H)8>RZDBmai1VqHCVe9OL74zVNV*&hzgC=l(y|1C_GTKk+1~MkOn{?^GY%Vj}764`2*ESDlTA zxA2_GP2@~n0PxhxsSZbf%i8@&Isf5L8VpGj8I9Zguj-Xl)s@xU zV#e>E6^pqW{Gy=X#6$lR+3z%$eIchQLvXCd7K2y#y;@x2I{6V#Wby498#k~lt3gJd zp)6!dZImHR40}1nZlHO7k*&)>Z)zsRqdevog`P}wCd4&22vm@hP}U5S@srQUiRre+ zA)>C9ZiSx>O%oRK9+fcoeH*(KFIqOLA>&>$gm^#5zL>@YrL#eZ*vKHk(msgncGtY~ zK^B{I)6n_tB^fN~M&lb-R@`}on;Fyoi4?6K01*D(`jFQEos&TVr&b(+PT5gd>g~i` zHIfJPV4{}~Or}lJA$Lg`oQW8h)zmH)I<}?J{&k9KsP3B;s8K$*!OI32ujZy_S#jTM zB4c?jR(FUfu^dL{XxZYoi^RKK%JXu1X$RmL(+JUT%jr@1sv4t7YaFl*_SvpQolg_y zD+6bDEPu&`o$VmyZ+@;rCJw7T<`sg~BRD8@S*O0gY2fQ$Y>>o)!%6BG4b_%po9o5P@VIAo_4Uj0-?OR?J_uLkI5+gKyi@ldHq=rJuc6E74M2`9Y0R-WnfXd zrCuLQ-abLL8*aw(*ccFsF7J7gP~9_fmNtI+s@`*3bzY`Ob2Fl8Lr=Xisoz zbYjlg(C?H|D#uCD4y1w7iTjZ<)}l=C)#Rt~Y-xF`toV{6kdMtV@JLaEND!;y-j9!U z6Sv;X(|RlL_+liZDRlketvQMl=m-o>h(^c6b7NVF{vhWfIceB`Zzu@X!2tQ50I#v_((h zTmAfx{oXw8iaS%CgH!jmzb%P_Z_Yh zZ{e@w0>7E2$Zx6FKq7aq9D#&|sG|XZ&e#?Z8pIm}jd~!<>Tcwk_+MQFf}RR%>a~eq z>HVOhKgnuk|XOPSz4=t3lT#XA2oItbkNJu zJTiUl^IVz7897^sE(_oT(~{JnN1*Xa2q{UKj--UbI|Fc=C`OCIU=KTx3IG#)k?Vj7 zzG={sMq(X9Zd-yNSUm8BX3DGLP#8fRP*z*$1g6phT$i$7GCiCLi8XlywDPdBRV_W-}qB=oXFJ(n*Rsxdh`?EBcd!pl*})i?H2^ zYq7vSdtL)fZtLq|Y!QCuB%;pzi5nm&mBzfj!F5K}2ju^k!&HO=vhe|OA)UZW`C4`a zT0R7knXzb6cnzHFed7r9u@z+fcM`ULl~(l+d;h<{ocasLJPwK4gRR4<3BiWqRqGsj z)8GBC-RvO)1)wy_iSew18F@l{A%KZTn+Y_HLR0MrWGVf?U-zK@Z&XUaom&LqBT&26 z89yyT-bYF&`gCUOHd?c~lG0yQDQCA~?svlB`}IpQZu8gCyiY8T^LN~qQ+oZbvfywY zZ%s}mNfPBq*DJ75kzAe}6$4pqRe2i+TaNf`*CCZ^k?=9suB+#EzJzTTL#?WAy~v!J z?lmen@hiP)f0bJLw^p$J=7RxS_oN2#ip~DvskCyf7xt$(8j|}3Lq0xtmT(~aQuB4G zJ7Qg5sqe+Jvb6hDAvQ!|hrX5c`{w%oSAotX0y$FxX07%7vt+0wq921kA#Wq!T=rBw ze)64f_I?c2Dc?F59x9eKZq>n|uX&$rgv@#;wV&*{uZak zK+BE0k{Jd9&XKpf_}#ptLr_*)Cv9Dn+CsgbWhuBWM$fcB1GZuEggPW>wuH&Gsb|o? z3Lb?DT^jBZ?lM&1I-N(9^BU=YyE3ZJ_6@o@e##z^9zeY#vrR}n*1ZCozTV?APVPbE zFmHGE6O65LyqqwqFBJvT$Gxn~M=~mg+`j0KtX_(kQwD{Wrv4{8)Tw(Z!IsjkDQQTU$DK2HI+;V__Z zr-}CPN*Cg3gC|=Xdjn7cXN&O&tD4UzX{yby^%mBj7B~XkA97clSkrAtU9}<(Xk7Gw z>DmG8w)~K^@Zj}uFc5MKwR_R5+KIRU!LsU9RitZG)}%vdg^h%#->u?Vw4M#>Z~-? zjunYX(8MJA1w{(Gxn3U}N*Pn}5yOAZB|A*+eE`?y{>rNb?kb*qhtI)nwWWoYQM%yN z6N|fFoRFmztfh~B*s*;=db8s89WWPPx=;zNw_7t)4f$OJVvlE}D_BY#?NV#)yEJAy zsS7q6536G*+K#tR-(oj<(nz`Vl?(jF6SQ}Q$V%xJ0YT^syyo9c0#2jCnhH4=j(Q3? zjZ#XOtVZ-z&MC_BRDG1uiOJ~K0N-zBwr4~ zB^EAhrLE=eD8J$OhH4RAD!UJVUL__G|Cs|2!+?uliK#Zlo=vgP#IX&QD`F+`*b8oY&628>?;=3B&y`1|3LJ zP7Hoj=MrH#@Emf1L`fs+W$_3|{TGzn+a9WfEjHGHCW zDnw_9{2)d{md#CieG_4aY?PCUgwG~!)F2Ac1LaN zKZ9%0ySA#iD=}{tA?oxZez%)`Lo=AG%)mS+dvB#?Uc0YjieottlenTk3bdcz< zWo=N|3gIOJQh7+`PF2JUtVvziiLFjx5aSM*LLW1`rU7qlkS0XuMMH_|RaGSw-q5Fv zE(40aaU{b~6`#4u(bwRQBL~LMqXb@Wh6w?2qktsyB6*;hp>5U zDb+#`%!DWy1Nyat0-%TFw~ChD^8h%qG`AJ*F5JCeH9`BK=FR&g!AKexxqXQUEr1`W z^_NI{)H|G%le{8cwfGz?>Xp5#Vu=lh<2$~)QhqB%KOUlVW7`k<=vMyzIZS`!tMh;E z*njlvR67PFW{VbTWpa$7ZMcwHh3e-)OfSxEcWAHGRXvN9=NOU zt{^s=#NyZ}ZknFfX6zc>JW?I6l++!1i;wjqlAl|tA{+pD^&F|iitb2B@uL79M$qYbK z{iFvx4jE}h$zO!*U@je|fEWM#FOby+og)zR66Ah5DHk^P;CJfonqAvYxD2vECv2(Z z)~XvUo{l6E4alquk0UfF(#TH-sw6^)zh?0E2cWK`1E{(ODoA#q+M@U1LPwyqv*5fw z$^!`Y6BH0w&#^p^CG`LP^cVN!A)PNNEUJLMJo7pVNX9r|!+seQzh9s^w+^(CO}Rhi z<8P)UE}-XC4~yR*dqaYUmXRW0f*u6Walco;ms?4goC^-H-{h^S zz_z}3z1i45>Y*2WYvfZJlgVHJj zJ&kO?J76jE;LuUZH~eEw4i%@xVCYi7}scG^#G4h=RX zztbC*R$mikuX5PlCa?JU>d5yZ@nI7+VKN-x(yH?7%@Uw*#-RPABo5B5a*VMG-RSrx+%sIv&vIyZeu6-`8 zotwrE4OUcTT#kG&ZKOu5t-S_o7E_I1x^TJkJ$HdmADu6-W4)s{ZUM0LIL?65(gTzW z?<{@*zW@;IM98)z3D|+fJqkx4<>Ub9;(u83>*!F(c|fP*QblP!Arh1C&&sW$8aC9$ z70Yg ze$D9d3&B)x*6Z`So9oV>GGf(~fQM@4CtM3D&ml3NAz(V$C8T}L9&6ZGil!uZ2r>1K zd33egUm2?hAu|5L0sm2N@!x}*LXIOF&FYNEY1p_%a|Sh!hKk}2L?>91yWmu&zpUBC z!{^EnO++*041f&r;x!3Y8RsjN+Cq}^Uyi!ZSBwHh6BJ2ao38q=zJGs+UjOI2vAmWx zB7wb7jJr>1wL1a;Zfoj!NdjLlDQ*j3P>($YN;SW5;C5h9&|nE(H~%^FM?YmhO;(rUM!(@Q0S~CHIyALD0H3X3ZLX8c9cO?D!b%uJF%)% z%gAUA(WO!&TR)O`K$^VnrRIB*VP+MrUX<$Z>$8hRGI^orHHJUnSgQ*O*%QsWCqu7a zH6HIi>3rsbVU(U)QM?=^(^VqG+RgQ+yt6n;EU5s1gdVWNXKA65@4`WF_DKgiV4^4R&xW@CCjQ=A>d zBb7r(^Dw}!v*k&Whk%5PkgC~)8m!)kB4{aw5Q)>2;Ysy`yY?NX&4M@B>~Kn`IaZvs z2Oi2FcFl`aRw=q`t`@2GEyf_>-unPU=M(UrFX2I1@yPN=TOwLp(ZzAg6fG0kJ)@_# zC22KznMWWP0Sbp{ITpt%z5d(Z>p#sPe{;b4yX53oEC=Z;Asi?;!UtgMj@VI3tkG!S zL)7#<*gL4u@dPy$SLjgvaAj}$%<3eWG97bhj5MrX09n$RC z`~Z~p(U&FiClLA3pWOxZHqh{L4vVC)RJ99}?q^RN_`sshRv6xm5SckfJK(P#|KI4)~5dktJ!Fe#V*EzFF9PDMq+B8J3+@WvU_^yTf9v%0CT@jZPvoD0GciHfOzkd<4 ze25)0ors^@*WvEtN6>d$$ESB8!1C} zVr;D21u=Icq}YwrPMui3(F*eJ{o7p*Ode*F|1Nq$72ko~Z;X^Mh`ZaD9j^X9!x^1>1ZKtET678%Drc)9+_JEuXv+6h_PkM?CI3*=(+xPN-_1lNjgG~)r zusKm~=|x_ELqk;LBym7045!5UolG6gI`4e@Ha`;}>@|ro01CfB_!zX;PMf@DAoKVc za^(uI&7IY=YEy)*at_cEiJ$lji4pfboi%RGDa@RwuHj2u=Zm4~g8TQ5K+|^?CcY!5 zj|V8R2FP_#c)gBKw&J{w&9})bBKT8o-Oj9-V94QQSWr!AU!<*ZWB+j4vkg+N^>vHN zR@Zt)l#upe4Ba&Ow1N$4Um}~4=^F-l3hFz)^#Tx0kt4(~4-mj@6VO_`GlyGgtr}kl zPhcN+SDLdBb?(mtrJePo!~STMk>n3PntJS`T6wk7ibX1k{0b}O zQoWv?zl3_ZN#3nvHw-(bTv%#7q^UyZqod)+DV(d2ah_|0$d40}f zS<+Q1_u7J16-6PVN_^@oGlkO-iuFhxLB*>+$sM# z^z*{$C1$D-JC2d)xs7Z77UskDyORu-Xr?zGW%8r-p*|@2;zK_p~C+InPl?_PUo!5)Z!$Qk0Z=QgUqN5yckA^D@zsGvm`L{4X#w zda=UV+dtnnaIQ4f(?4dCP#}(b@gCqR@L=1?5qSV4SJp>24F8NGCBvJYh_o=u8+B!# zt^bURQsI_1$VBaNKLu>$Xd{{q-7XoXsA}7{s>ONCZIYht6el--yqX^O&DbeRdJsT5 zqgf{~`eS?ND$p&G%r8P?}>;0 z;hZ0#hiuf6@jU{CBFpz*AWkp4c$XQD85_1j%&H~=+wb{)sQdsP;ikJ?f}IJ>@$$W= zk>F#Nc};(fJeHvx2IFU@`b|^#{%afbvI^`ttb%vDQml;XqXs(r6+5yJVbGVcY8OCK zz(2K5ulRNEUNNv=YJHqLo7;6ZBvFhv-|3JTJ6zA2!;hWB09hnOKxQNR?UV8$d$aJd zsn`70vX2C`<@0E7S!|XxeiP{#4f{IW#4l`{28W_YjzHeC*dq|jtQVj#sIzZ-b3YuN zaH!^KVvc6$U&bs>8=T#3{G0Z`KN8wuM^N5b#(plSn$Z@A>>HSB^IdheUGKScNh*oJp%2l z7I$b%*(!%Lz1P()N>qS)wHqPT*XRj5wt!c@cU>n@)6|B2Z zZ*snecq)994CFS_4S+#@7t<=eoR!_s!ol%~h6mWnV*CdE0@^hleN`zliP-&8F0^+) zj{W-7`BU0-)ACyz&#+baSLC-8p)u|RWj9G;-{cmZzn&^B z1o`a%CK^B@V~`jy8DWDyq#2^x7ysi$y26-9&vHmI1mLecwqBR5J{| z_P zxxHR`KPX@X0?HLu9N1b%*#yWKeq(i$XFQLm$BnY?Js*+))4ZT6!&>3b5Lj%m^QvEP zOF{4!;~zN|jKdp4Rr_c8oGu-|(o!N1%71_SLdjaHGYER0Fo)&+nnr`Pg)kCCd~Nn% z$L&hT%-_jvaz6gNbKB?-;$qYeu-Yk%xEFe*IZUp53<0iz-gWJSCNXsLmFQ!@@1V&OH z%Z3}iUFrH&GP@JsYWdceqZ@_Anx$bh-{_cH_*n z07GoYh1*YcPt$Wg`*wl4AHdPmB8`_?qXXHDWy7`v>PP(eT@rN7doISByt830m*nM9 zk1TqXnoh+RYfZfvJ!!tv$_lU89q`G-^cgLh4(@88L-D=E%U1ZR6Fo?bdW5v1l|s2 zAhge=mU$gIj5w|9{?HhZOqjE(cpX69rZZ%fclySH;{`|-LI@WWJtukGHdSFL(<0l( z*sbP$;rT9UVRqMYPrlQ1dCXJ?lh~_s0w|)hpBB(50}tnz-ZiF;Y)TKB(Wwq+>L0&( zr{$T-+bQLaDOhMy09`%_xYQR&lGGrDYqddupv9AswdTW}nb_qZK5u%*mY&>!Hc%m^7+^nGkj*p?M)xQ<);a`^ayBp*I5`0|!Tm(<$@(<6|Z zpkFfRU>?7KGYC;UM>JlU5y2KTm>1{Ruu2)B66*?bdbDruD`J&=0hRIIKp0jFa>}-b z1|dw?Dq5>44s(Bw@9Oe$Qmm-*K8`7sJGI?Oukg;9{aa%jhH6hTh{8s&Xi7VUA!ubX zWcJMsE)KG}p&Pa{PjPL5{MvoF0^~k`Q|f?13C#pS=ELN1Tw6tqzJs#ID=*~ z({`^;O{qUrO#qeP{K!g`*DjW7n#pxE+6Q!S4j+PJ3rE@>fhK%L@xIuEV)3+KH_rCb zFMRqXSETrY}oKKtJokcbyuP9pLZYh6G72B#XNmu&TPQD!OmPXhB`8nW5DJL!5 zb0?<>cC*kxnYT&zc;Q2@y_*#sU!Y&-^k?$z>{ER>0kedR*|kt4Mhg*F5q&Y!Vqj+( zPJd4A?nIx6%*}J=PK(-A3Y&O9;6f}|F>uhM3a?LBojZ(@inB?gI^8Zb`dn__$2AciSD|-(58{&oHaek$! z+mP;9XAoDcGP}U1GjA3$Sn{HH&#E)r)yuyNXsxYb%{)}w060afJw#%CX?QO`)Lxo_ zwKYC!mM;~s*~$oSLo4d*H&UO~_x3-!+}G5ew1wVPoge>U6^^mL+uWz0T?r?;p$e}V zIH2d180HPy7}eYR?97i1AAw4o9F}BRQ}5|WO{g5F_jKbQr3qquJsGyqOj}fB`!;jA z@cs%JX%?|+=`G7}lfClNYL?Q2H<_7>MPVXp3aiy-gOLYCMZJTl@r-%K;uhp_ljRz8 zsMS#2qs-S4##wh%VyE?#Rfw)RL`X)ln=YZ)!-$JUawDE-XGl|u}wOA zvd>-r^fGMUt{L;vbKvl=s>0EgK7MtXgVrioGhm-On6tnUu~ey-DOq;rj1Vgy&vY@s)jw zM)T@f<+X#Ge%D(sYL!m$jk0o^hFjq)p{7}RT0FzP}eLbkz zO+I}t@=jjkw9?x|S^W`hrQyXE?OPB@z9JTbDn6_*|Fg=30)W%;s;UXKd({R;;Diu{ zUAjJ3&riYT#IeQXQxjU0dBh+KX;fW%&d;W84;UZyH_;S(#3 zUtRjqw8{B4^lL0TbrB*XOhe|&pw?rbTYms66;VI;96$vmV{w)|!i2Ip!qSV40;`I$ zFGT;Lb443ne5_4s785$p#XcPW%Ev@Cao(ql@YwZ1n|2mX4S|g5GNMewOSAT-ShY!p zflt1~JXU#k2Nia?+nGV(D+^nyQ3%N84vQ1Lk$>8^o~PHJZuAgf+e)5IGjdq1D2Jt5 ztX6~@?92^&jNIZt#RU;Z-$3~-KCxW9ZzxS>?2K&!OfIf>l9QqV z$K;U10CwM@P#;|z9IKXgAUFdkiG1#$Au8P~hcQwc+C7Cc@;Jxtj1y%8=)Qn)~XM%{dtzLP?`JBxz zw^)C0SgYmv&bOi8FZWlDI*8Xu_Kv9xN1)F59Drq5)Vo(E2oy8X0Dtx19C+Q- za~mN_>Ks6H10lM}CV>~=xauQG9?}2%V@IHavp_y#fr`Vye|R&Oy8mbhO|{W9x-yeh z8Y=^ZbftwL&O`RtOOHVPE2eVM#1uHTYuieNjnKCTWJ_3%oxe-W5A}pWyk)j*WHXvl(Zuq($h7cu%T|H>sjEK_~U*z9*FnRp(C~B2|Xs zj0VKqy*^qG98QSFtH_CP-vS|$iUbPQpX^HL{^1N%+n0fOF2qZl_AO>QlDsq zb0S?rPCy)^yLJ%ccqfIYQybGe7ILkgWEJzS33OHuIehU)^>Ru7>ET)F6ItLn;Xfs-KPa*MBS?99e(3USbk@^hK=FFu z>^7_kP7v}8M2PvB4>T|tl4OU=iIMu7zQ5Eb9LoN3MU{6U*K}ubzSw6g|9QNKW5&~g zkMlfsX{H8LA%C{Cf080Y#3}vgs^B1i3?A}w?ngeBcz?u7-cR-jWK&fa#n%$})vO?M z!G3Mato*V|@oBephm)GN`_R|e%38KCzblzu$qb$^l+2rE$d_CgR7%^5m}7aQXf|^Zc4tSxjb50np5o;k^`DppEOY;70xt=(Zr>R+>Hn z$!%mG70_2nPXq$hezidqHn$FlTik#Stz(D@)UK}E7D+@BMwkX#yZ(Kwe{8+D+FMX# zTKM9AMTSDJcWpd^gB3l3%BAo0tIIeRTZf`vt9);}KBIUwkn#Czts6kBpM!r&fa84U zNygv9u`k<7R_x~bPgSuR$d}_S>|3$8%ty~TUvDZtjPErD?Mo6KfhQ#?VG@95Tc8a3 z2*ml5QGN^G1u)9P0no7T0ABekm_6!Hs0K-4 z9z~KI0p|HXaI~BPN((@H|NcA!ZCBIVLYA1+8Z6AJtx457XYLSoeyFU3j8^^tZ~>vP zTCc2{WOXn6FWXly0m{B=vxo;&EI-i|I{2(saHhl{qGVxfWhdZLv!w6 zuf4?PXuc=sv>3oA*uLeYm3~!enh2KiqH5HJj?ZU8srZg{Ewk1@>+91A*KrtCkPYokbWV!yB7ID2xF;i z{TahgE0e$4jwURa7Cd8hd7W0HHvS8nwi^#zj2cvni?%Bh%fikIP0F)M3oor-#Z!`RG39w2EcR`0> zd~)hG8h-<)m;hS?2XE-0*4{~r3R12c2SCNi2~b8vKm2SReI8)1=nz*0U+vcV8>`#B zgKVhRKHfie{%xl|v!$H=?GCwv6YE0MV?{_y@Z^i_2^00=2^;<(H|qrz-0{p?qNmOv z-TPiD^Mn(*5uFqTa*BHz(<+Y`?sRDGjlzk^W@aDzxGM&cQ;T)q2Fq{}d(u>Ms0y=8 z;Bi)^1r#{O`N_&6xB<0q{kld-%m;A)UC_4Egu(oeB%+d^f)lGLumlqvDeIW_yJdp( z;*I(a182WQ>Zx5{z#hN)nra8~f}h*~_&7!xKZk6uEln&iP4862+X-=|JB@A2dc3hN z)!*62tV#ef(+_k-abt88gwc;K(+VO}MZbY#_ z`4Iy%;sXdy!it|T##_?eu39pA+xPLRgx0$=)_QMaO|J7ogq4&(fIp(<5@3$VLpdB^ zWKIL^25q@iX^g1H;HU;3!uSV2Q(SLL3Y|%s*3id#R7$bLkJ>N4aNe2byY+11Sj6Yq z#{tkEe=NTLC05c34-LT*M~4|Do$Hz#eT86O?YEg2RAu~{YSJNY(5W7}AGu7w#2ewN z%%cW4t&2oU;8t!A0&jPA?H|0>3B)*ZSI6xLL=r9n?AuSj)dZ=8lbOw6fDg1o{oV%# z|AeG(vjNPfrsS@1=&OnM))d#$&MFFYqPBqL?2gNXIGDe0D+Hqwdie&z&MT%0xmj{USf_)9q!Gg-JeRe8m< z%UdyqFY~dUrD0P|G3d(0sP(e}r+a_4)4zx(+-eJ&Y(jC(R$v=S2~kQ; zLhGDkuY;W)-cG0SKF1^b4HhofR0Xnfq>@7#NW?3AA!p2f`5q*t6KOINwp;_f?t zRsxf`-$BYp;^sr@43itGoh97l)WX%R&t&S8Cj?AWO5Z;zfn5hh(2k{M4l3G*6|oPv zt>kMJdoHak4la3q_mip|E*2}=s8hnM`CnW3;=M7`CL8m{fAsWJJojb!bA669dvnYS z$vL9X5hy&;ZWcRiGt}+uSRiiQ@!*ZFZ-r&qz&=sU@xZ>`cMkjl3iS%c@Fanoi$f%j z*V3d}N8zLimBt_D7UM%M+zx6!c6xU5TFJxglJN1qxE8JH4a?%#9a5E|IeY7DNp+Uc zLbb2nU`=9WvW}UhU~z3>vds=Wp{G1;5{}JA*?|ugw@B^APCyqWoPPu0-S(nd)B4Su zSpYT`Y8&A9{`qHAD>48c@ezoX=Z~N8b^=CuDd01ZrzTb>+BY~Q!`*r$o_?u$AxUp+ zEK|&-X%QL{ui8zo_<`?vc{N;1v{-%O`mL$Bes@&-47#{5I&gl9#pq#rNQ;hF@&mMV z?3?ti3BMAQl<{=^|@>iDXMhi%1p+86z=(`JN2Ihlec z+fF7r8=kG|;r>qA{Js>s+Mko4a=ykik?^Q{k$+OZsBovWl(KS|g}twgg9gs|C-1H& z?0k+L=;#Z6uBj#n^L!>ZK3Por)*vt{Pcuu)6;VJ$ur^|_ZxtS!#f@B9MRgJqAY5~waUFiO%lt4^gptUl47eiLChcC^P z|EKw=Eb-l|sd^zn9WYgj$5jd&VjNwr`ZdKv3rzTQ1k%AVMrO6gBhv}*I~^ZhXyfKh z)ONmtLdid@~#K!q4Zm+rqqf|L&VlZRIR?*1sow=p$Vv zs9{lXLJyANF2P~Ww+A|Ue*|#ImrM@0FTh3U1su9V)}=}=pTFP1O~(*h0f)8@iB8*<^#VX-pz3hIR1GT^_CSty@4`w&_u*uMy1j=?Qr-_pIok&9Z1MPfqXs6LSq)!!bIs3KC5HLA z)sPkF3H8fTna(8s3$JLLyT3(Oa7Y!4^1fpU3u5IOK!4ms*b}zqksE@<$AnU+<6a1~ zUiF~k0-7#xS6hGJgyOfG6@wApVWndCEWaq69gE*?cHk)x)&-zwGLRrF`;qUd@N!Gs z*%(Kluv!nWvLvRh_KAiBNk1eG=+2%V3gwC!9~-O8T&Z^H5M{rl=MIzn_T+vCV}up^ zkF#3S2J$uwJF#_(PVJi$aW{w}B9v1-7n_ z0)i-&0H_qOp#WB02NbtO4st;4-YAA#CCo3oU}RNiSblK*B%VBjurvs*pcu;UIBhyG zZfq%XSjemCjbYhvlOK9`XzLZ#Ok*C>U_+o_KB)Z?E_P{zVV`qn}iQQjD zplIgXfmREO3nAzgw$z)}T2(0&wZSFt7Bqb33Z2s?_WN-VXV=-aj^5R`nw*{3XN!?6 zf>~4bKQ>0q*ivFu+YNf2#ZOO7><(^|9bLCc+Jq>1KV=+{^?fM-LOwlrq{Sk77YrsH zgDpHR87n>@N|NZ0KuNk!r@qD7< zX&!}?$Bwv1XYuo!8<;JiXLpfT`@Bv`kBVkDEND473(#uKz(-KG=YVA)n!n33A$&#? zyK3NR8hTZw2lkWVhY>?7Brw(=)m1$0spKy=|2vG9t_OzIaXOIhWkrsTBmsQ^pGTk4 z5!RLtIJF7A-xzsyhU;LtV_tD;$-w*vhRiQz)jIovq47g`EAa|ojFR1qjhazPu^D*A z=@tCUE|4Xks*}NS-kbQmHsNNV#qmDmDKArN+Jo2f5clGwe2sy3FPFUsqD4zaN1gFhjV$rU;vuEN(BOV@gUJ;)@nV{@1W_=qcy}0WaC}r!b%04NV9@I z6sccF5rj6kuPDVWay=*owUmUoic;xIGN1T zhJ0sI^LS~+;4eNIIElyae!gxvL|(wl3k~Y-CvCSk!p7k9#T)dQ#nFp3rTx+l_saWL zeWtdhN6}Y~{}Opi;h5_Vv~T)`%%lnX-Ioi~2b^~c9;L5vERmSPG2{Yq!C-((s;=X1 zdf<5pGgh)o!yDZT34Br};^g+tFl0}qcs-bS1u+CUc?7cd=V@+Dn7M#m3bcQ97?i;u zm4(tAI(E?z<0RbeXf%GhowG8Z>(sOB1-Cigyo)LjdGIPD{Sa<0_YEOMX+v-O@SFh( z8ExqRHFA_?_k8ib=d!0Bg%kEPf9d;~UyI3Md_X}Iwe8Aq1bVy?O$nCqt@gwstQ^1r zWM#r_V90q|BOEWWf;!aT_8LaWuDYmWtW1nz|NII5r=NZshCAvO$)U!rbB6V)MF413 z?E^RxWdk7yZ3Q%N`sX*HPB4wO@$lVi+pp~ySVvC}y>xCS<^q~qRWX$Tii!Ap?1UXql+ul#sFdg+eu zH-B6inQ(I;F9z{PKsv0cAOB_8_&?6_*V4CR3gKQ7v#}$q%n9tT+xZ;Rf#Ttr8>=Ta zlp)cuvhFy_HxyPINV5YcB8r_dRyk7IgXmwAiRvFqSGKwK#qqjY4|{q*h^jxLZa={{ zzfb`B;Uxg?)CtulQDiWO*+`E;+u-@^6Jt*X=R^il<((`zeDmV^0FBB*;PtW49IRk8 zKV+0tEBR7iUla5A#mtOAtdZ$^T0L!bCLf?Wwh0-Z1aj*FJESEJ-rlh|P;h$O0!uFV zV0KUb`7&iQh=&{9jB)R|Z@jY`FuWWt-0o$t38b;yh)yXd9;$ZO0u*PcGgKwkf(ZE z==RGz+f>$@!DbP)0P|=Thg|Gzlbf(ul&!*%7GYuTvIjyp?0gr)%Fu3R?w2ONCY-W4 zE#?u%WcD@+bgU0dkuboWI0LSncFL5kOE1+b-@r@LuHWyg3_LkVUl`bz1^!6+)Cewu zcZ|1kI6ikeZcBT3r^{2q*X(hWrU=^?t}JB%?H93mL6F--;Gn)JfJ>o-R?02Eqd6A3 z4n6^#`KHjNH4KX40rdYffXPfBd@zRcoiN4^&1pZ}ziy?meN9Kr-AiW9O_^~JnE`|z zTg1nB&PBOxZoKJzoO@iOqE`VNdCzMY6RVa*nI%>ur>gMrBkLU=VLM*~_4Pv~hGUQeDf&nKL|Yn3hE z+jJK~Xeqe4Z|f)nSjxPEIsxra=xzk^5CGKC6d5j>WAJtGIUu5F(4&Yc7`L#+KK_3H zA|#VDM^}O9!(Vr9zd|g2YblqM_=-5_0s-xx6KMCW5F37?9%g_MJ~n-S#LJy!Bl*~> z)74?E^W|4<5li-W(QalZ!_S0^zEnLY%uF&zc;J1o^1xd&ScD_M1slK^i$I)r3Crsx~o^O$^&w0b&N9hF)m`sB5Jz1iT2y*j^fF( z1PXQN$>67s*eFeVsQ*Z)oxXY~wJ>0iCR46*yj>VP{UbLB6DQ4@taBR%CfH`F4d`LGFN>XAoF@q({9Rt9c# zM*x&bEu24L23G_0``;+dvuJ=9O76~eyBNI58?9nkioO2i&ULmte8}cgq1unEx&Hu3 z(JrBh7YE4kyG+Q5?yO^jYoGu#5)fy)a>+_SpfUwAgIkp}I|u))_xSg5{r}<%{h9-r zdt?zcsQ^7i=J{IJmKHj&D5OWz%+^^i@F>76BOiJ{1qUK26SRni0RCx8@WNRH?Z`C7 zXv?U&S7}Ly^VFQ|yINDI?ZFF)1aL*ie=kD(m_x*8d49h z01UiCzkf4q(e7JO88dJ$B>A>9)=r@d<7_UH3a*cn;5;UUN_$|R$gBRRdB%bur$4g# zP%@#<;@%)D`T)gI5|UDf3HbFvb5zv!^vRWI)}HILfjjpSlg&S}jv5Dnt9)9Fr@+Qq zxXdzwke|xYtfTUF*_>8f;}7}QZ;=5E-$kC^eQ{m52MA$08f?~@VpY8t739yy>Mq~u z-qdD%3qKco#ZW0@NIukAtHpU^ePk1n3MiG%gy@dA%VL4-5zBf8#+<^27Vk~wwGdgA zZV(VebYDNYGu3yX!u0S3xSWWE^eFemqP{&T%D9vOnP zH-4RYkr=`fgKIgST-m=}eWc8xfpiFBy=QDgWK{k-Oy&UrQA4ET?^=q(r|#0d@CA z;gHdbj=^suoj)_=?5WghYJHY;1cnLC90#9}QNBZxvK~)y+|Hb`l+!BZ+>B$RQ~~y( z;~D@sWCO4yj?I`oA0uNp<1PcnG+BqEzapp%R=+{#cO%h=S8-PpFVt@{Sltd8un%pad3|_^m(hpl&TF;(z;+5j(uLDYxEVI2Pm~O;(RwWyj z1)ZD}z+u31244~NVmZdB<}1)rgmOQ$fs1t7oOZc)Um?SM{GMK`Q2+Sz%4eaTG+GJq zTaCvi@3n`c(WtFEU2AZCp$sbUj6D)NP za9SG;L?%LE(yfpJN!n#&;zib$fHQ4brv}<}i!Wiu8$~>x#rc_qj!YQJCAgNWJQ7b+ zJ++N(`|#phjn5-zmC@z(0cpi-z9GUli{%$!oPM9JX_gK#?xTUPh!ep_&3gIl+D@lJ z&&!0`BfDTW$1AfK*ZH%e6t~dj8)?_l$B!z0J)_=Z zKko~6KG7F;_xZ=fSQ^DOZCOK6|Eme14|B_*!Ww!dWNk_X+EKSXnq8*|p1ibOZNvK9d0@qpB!i)mmr35{Ek$IAnjm|IzK>u$hgOz6voPVq{ zf=~q;Q-P>_V3==J%r|Vzww$|%>kE!9F1%Wjr{|-a(ihJKU)DS~VwmqK{w^uA^F87$ z<$V)7`Q)f#iPd+tOIraYx7$lm_M}_Y)$KJhpJlbpxz7pfd7n3Ez*^V;Y%?m?w`O?; zVM*qzp(~;m`+a-j1u&2QOX6sHp#NP-A0^P1#d-L$LVbE8T492F@S^6~TOqMIeOKQa z8J)%kL0vG<_?zuKM;;8!1^P?_GuO%?^eQMUJ^?EP0q|CVOuI`}2NLh&^(`+l6<&J3 zd%gQ^mtG9FSmIowo#@>(w~qjL9>3pyA$}MkT|_1dPg%o6IAsb+w?x|(4 zx-5XF4U}g8E`%_^dWtOVE*#jIc@>|T<-Y1{a{_-`dChqxCaJyg#Y36{X{JyDyP{u3 zVOOL{fXVQRQ|`=hE0w_v??5tb2^1qL7FzGr4oUF$zmJ}wBtJR()PyqnPCCFru{HH> zWs=rtC%ceu+&7nA)#x}LVvZh8CrmMpcWcD=f%j`$NVX@X&E&jyXW{qnLj6)XJgSnV zIr=+4$Z>!<&$S}zFto~Sx6PRAmVJa#=6;7?-pd~-0r%i@#sY}dNV;n#ytXI4&D=^B zjt~uCnQYRvhPN{w(TSV`CvQ{T{ic=KH`T|F8Yf{kT{txJfW@Ad?$l zRpu|U6b$`p>$J_E1|9hf({#6mMlXq6YLi-i8Xb+*4&CZWrfXqw`$3ckh=OK-Qm!VY zUxOAm-ul2gQdELaG2pMxgqw_6G=oe$^UJK-Xq5@JAvxS=EiFc2jEPda?G4hS5Cx(a z#_++9=RNr!-tY}H9r;?|CU2EbK*PSm;Rg;t_6-7R-teqShX}hdGDCr$&`R>pQO~09 zg+BJ3*>)$xZ-!KjCaH)7pB8pSXsoO2uB8hK=NFEh6!yipF4eT|0_b@2z!Dg|DI4Fw zTU=c7-M?s~EK62Yl=fBElN$w_+S*<@+FmO?92k1#rwF>mKj-gS`tT4wLdH(&fJ--4uFs-Vhgj8Bt)95JS?ZM_PqR| zBeP5+F5jEIckfdEGy?g+?CQky3=(d@($u4&PT0nehe5>2p2F8~MrCJnb6nKUW?if7 zYUaHW9h?8o%kELP?@0@77LI&_WUD-D(Io+S0{!>jpgl1-lG^N@b$rpKIjgN-q82{V z(R7yRgNeD>VWTisa!>-k9ogsMrT+MyCy1rE?Jf6pvDcszhHRl<43eeq=ygqkv$8nR z3q|$wg!C~aeUsouk1Z!Q_BW_aPBc#KH^~1-*hmkdH)e-k|IlgW%X}m9#fYF9>Gm>N z6)1`=i9hWSaf?|raCNhCdO7Y-4<~yALc=Y&0wtYSl1E%-pXEoyX$MS`)B{JdSbzbp zEPY>y5U1iQb>^JAL%WP`KHT+FJ{b|6j(u@(-sqlh6>h(^!N4}nb4ED7bX=8w3D6UC zYerC#jVXQOgUDVvO2e_x{NC1bVE_0)`gtI+2~+@VSv*P8;Hi<0ggK5Lc$!xmlYb|H zmn%?V489;SS!p?comqzfGbU=Bhm5LH!>LQu$S_P_kfod z8*LG0qNtQx*D0O1p+4hl{H^2~ zvG-KkkIR|#iU%t1>+U$YXV071oYfftb3+%3d@eiJ@cT5QB9eX1-ci*(ycVzwb)=1H znG(P8X2va1RM^azyUSQ+_z*l}CkWk_j~GHO6u|_rC~mUm0+bFmMkKW)z{nO<^fFPl+#%LC+pmw zGda{#!Wl~$?u*`uIq;hXIM<{Sa>|Q`KQ@Pc3`@H)_rEL) zJCo3kqY-7?ZhJ&Yj0!JF8_n%;5thb7sf<;N;bZodL?{5e0}bY|ZXI9LYx}eX+SbTV zp6m{T6-otY^NoJLc>e5_qpS%FLN+mrWT3hDi;T>H2lmgm=5?!ERyH_&lGjsC ze+W&L4ynBL?aEQZxRN|8)3rLk)Sr3TF2Wdm))FiTLJPm@E`YfHllj}1nj}}()2tr; zSNPa(gvRvEEC+!^-=1`wNgSavHsEg@nY=|nfupoU+W2H7uvTs# z)iBAoatgFX>25Yy3Fnviz3q9F(l|N%b*o60NK0izFMV*;mu!+mC=JOStlC+M4*~#r zWJ<%y)MZOnlKNY)iT>YxQ^G=?4zbj^-yn94^LW^jc;XU(2yAW?IPuqnU;6y|`xO3a zk6Po)iz`Bwx>A9S!3^uorT~-n>fFlEo9Jf)W5%JdJv}lIPoC_20Fc+A@d%5hzKbsl|z<+SJfAtdh2mv5m?@$|Hf->9%_y#U*tzU-i;HmW)qk$&_(c_i{cj(`d zJt=vCoPh&=#?u-;C~0R??WYw(#dw*+@$CD1O0t^2X1?IG^%jGma!AMdX`jV`&ZF8P zj+MqM7(NEGp=aW%8PJW@XQ$o+p8sJ+^gZzaOfVM&0kqA*j}ZKjOT_E~^2495K2CEi zLCsGr11%L9MoV>USK1r1Zi?CmLGKp)W0mD2}Q*^yXcYF?{2jn(>sIkQJ^E%NfJSmh{97BWp3(! zrMBZb$cCl!9k)bj!`UR5)t~G1gq?hx3TVW}7l85?mqS2))y7rF6R=)OlOCloN&h;J zqhP@&qR`pfPibn)_&%~e`p9~PQh;2ZhDZUa4jSmbOmox(Ofbjnym1iD`V1GiUQr!(L-dZ**H1xe z&jC`TC9vK?*MT^z01o}6>M3)or^CnMInq|Zz;Urbs+{(UYtWT9J+dsjDIOAtKa9p! z^ZIZ3+szB>hoSL&?aS3$=gx;U%ytiD`m_1Sk;GoKgZ$~EGz2hU0RVoRgj~gSB$Zl* z)M^$?c0P}qpQCARJ`xEToBBx!Wm5_yD`0kJS7?MW2YeRMEi*d&Reb7vl zBqrv3xMHlzsMLjMI@%zR5FN-Kp3-7-7N42!8OH;6zdcs0H%kb(F{~cBzscPUA{~BY z5uPX5QZh^O_Lqo?7;$$!g&)f>(#k7vF33+dBX48sTKWU;#QbT?2iF?_Y+fu@K{)CQ zh<4*T?^J$kRz&?;q)ek_37oTaD!k5qNU=KKE4Nu@Mn=Fb`g3pYXwl1B4Ph;i=RMH4 zR=mDixH<2+=wJW<>f1%yqk)(TP*-E-o97u6w%gyArf^7eywo$J(L0U;ZAsu*U%4T! zHl3-%*@P%Zq&QlO8>*eH5U_elT`ukCJ)yD?J$NcFHFqOTJj4C7> zX|Bs1IW+8CbY`?4X}M@p7I$gj5}gE-jV2Xbay-iF#;R`4=(DBtTg#8i-RYe=`FIs* zQWj7lFvdW^xvoN@0EQ6uX|9JoY^UVXkoc7Z24i`F>Vz-4{VKx2L2A<ri6yx$kQVCHtP#j3{Y+5(Is`{x{zvahd=)oWQN=Uv< z*}YnoJMdmO{sYCTpyjpkNgA{h-Fw`bqg6rzcWQom#@tx#APj4z&8<$EN8NvF z`u4}BCSZ)n;z+wQKZ&L?Iux(lkGH@oxf|=Nf~L7_E>uPal(LT~v&(TxF3G+}(8ixv zpg9)7fcfASol|tEC=H&)GeQn}jn;#+lf9zahxA~th#UDQg-@RfQgS>Y{lUe*$JZ5j za=3(c?ETMox-ML9Hm46hEz0JUx42VtAy>sLB;Q2hZ1KoXg(E|6)%g=-B^#@l1GaZT$Pj1P;A>vc3~RNoLzueeLb9@upipvob3BhDLQ3Eehvmmmx^6nwwPW3u@FA{~gqoMq9hjWignh|yjYD^@#=u-vpt zS$GPMr?JjDuhY!DfB77%HXnlHD0S&xzmzhm_H*MQz)}-~y6;X6G%Lj$F77g{kLV!x zrv{4GdCoSE(^4#Bb(12_9c_(ejTRVVfT0Rg@Gma&2CHaHToKvxI+^8)D)6m$avya80(aD&IZT{?&8`V;RAX5M3`SG1%(yi| zCouyku%~u*7tZEp!)@ePHBl~p4`!NOp4Ws+E7Uo@_D;`Rd}P>Z&te675#taR>ga!O zb{oyffB!hK-Tzcl>3C+!KBS|Rc1+_Y0pR;5J_MLQz5oujaZhn5jH+Oy(Mp*@j+?H6 ziDK^Lh^7kwbt)6cIt&8&&FKdLqAxAg3?UIX&Aj_u`OCKQ`#FQ-}JmRn8vvolXIA#?EvLowZ-` zhkK3ko54X4phgakrVc>_KD4kSSa7h&?J5D%)~qCW&QWw>G)UQGP z)EeiXa{87>7nsL+eZSt_xfW%nGTq%Pp8n24bNOI}SKr;?7}=Tu1Nvf}NL(!9R5dQ3 z>gUHgyf=!j@vA;V5>{sa`JT>zfW5&Ty#Wqe!C(;@7U%TY7!T+4cdF`WB6QgoAb-qb z+87=$f#1}aeBU>?3V+$2Y^gDkWF{*cA~7Sif>o-y#N-CK6X@!}u$jGoxog8o3N>t$<`C6&&8frTpLjQZkian)vTCfq`yH~yZ0RaT1?Ey5Gt;v+=7hy4Qj=8 zw&zn9YM^8-2TEBAa)5|jF!>F-O3i%@$X;KPQuU2Pk!Sl~;2PTJiK2A?drOi!-t!x< znuFpbe}mrt!|UE#A$W*r9}n(g*ctLYlzg$X-LOOqjRdVnd6>zj9>`0S@wJ1h=Y{31 zR^wd>qQuK&dXCv7)qvjP4+IxVnjAzBCea3se5A8@Iy8P`%2PhVO3O}iPxps@LwSYd z9sMuq8NE6wzZ`_|1RmxvPsfIKa{AOyv4DG~6)&AB8h}h$y)~)Gd@VC3V90vj`&e!x zfL1j3^%GwSadZ-%H+4};zeLAZubN_0ObJht9El3`e!xAazx1weiGx06a9rby!cNRR z_w13wRdhb5p*1+Z>apV=`6(C>9X4yBmq#>!92xe&ftuWE{T{r8%35~4xujuAW$^## z6evwp1t?6%E9EmzQ{VRP&$i5phFh8Iz}oMRYuifj$0x`(R3_>G9~-?F#e$$qDJ>~( z)Oz{2deEz|Qs#;FFAggPs~)FF>dtUt+>v>MGvJVA?PZ~qKzKVy`FN7;wCbrQ_yR;8 z*nU}v2S71)b73_LH(*phSpc8fr<}Xkdgja%l!VcT&zDj4yTVoLcB6G)JetA}M0{ao z##Dw$Tnd$KV6wFh#$I*CH7zdo?Be}5vV6~EhIJPO4`x;!2E3j^-r;}xZWhRflt_2a z#>E3W^k0It*2LWfXv(VVKd{6CWy#)n)ElUa`&6v81TY8_GpZPdJBm2(EiUb^DVJRS zT!nf6_FB4|+jF`AW?mg;?cJXlIQ?|i(ScxO1)z+51re+q){wxs@Ky0Hnl+5Ml?)q9 z*In6{*VUBzyc~L$%Uehf@A5>-+$T|G<*un;N|P*g0LBogw9cajjzyfeRjAN5XVyP` zSAAM%8-M%Mg65i2clzU3f_HOmJgpa7+({WUo@zuZ=UJQLMAB5nsmZ#00Iuu-8MBVbwCxvT* z@3^d0tFU&0r{63FHsJFK$zux4s-?2q2vt9Xe2(QgiK`vdhwT>GP7wC2elyuiaPlYO0r3nDwPf?k<0J8?#a!7{!zulTq#q((mNo|Wutp9KRb3^xSqjAjfJfxp4@SDZ`Ux{gln?dEYz<)ZG~QK!FA zN}t1ctma|%D6c*PpK`U1^lbs)O*aCnpGOVG6t1d$d1C6?wShh)QHrM{<7OL}<5?i% zMony*PS`@!(VZ}(lT)Ex-_H1jhK;y4qOdj{)_U5)Lp%Ln*vnTT8I%}613z$rEJh6Y zOhmO)`O1E3yeOU$n_cN!Hn{xN^3*|1n3Yu6y`X+o``P{(n)ax6{oH?oruP`q^|6={1me=yaJLYbkyyrS6*>p?&%<}d2wc`0| z8*=ub8289VJ(3vpC!5tg49NRKJ*)#bEuTIqv%b%VsMF&k{SNW4Wy)nHAcm% z*JwiGGr24R&u|qq>23F|k1h{wBW~gh2w#c2q>G!Ja2c%K2feC{rg2P$AA<-8 z#4Fo1x|eh2%%Uw)EojU$%YF(DXG>xW53O@&b1B6ZV>~7gryrh6xS0KX;{yBJ?$+Sb z>sl`qUHM)so`Zsl8?StUk1yV~VHpzPaOPjFR@gGT##CWk=COXYRbsd)yX%JFz^7}{ zlMgAWL=loCLVzfQ=?Rd&iOUVo{em@b9P>K!G;?$Dk?Nyf*5tZ=_RI8jT~MeM=iQ11 zKSBAzixuumJliB3ddVSlw^4p^_l(o3OQf}{mn1gHtvFBWhJj!RbCQa>-qb|v36(-% z%i8t*-O^2hmQhi~WGM=&Q7-3Ovz}3va(LCr8#lVeWN1t`Pk!KKz4w5C*E9P#pVVLf zCXI=bh00VhzD#TetQ$m#yI&mDP;K+9dWGnS+w=A@vY%gxT6&+=yZuyNQt~rXwBYeu z@7%~jEk*je7wOq$KZ&LoRO|eSNv(Xiax?$gH|~tsdu3mvgw>Y|x+LmUCBEJIV&t~x z!u9ykK2?5L#u_39K62{?l3uH(U_^vRdn1uc}rmKLD$t5d`}Hcb*fn_P|dt* zR-6q&iJ4B;#pciVKj(`7!F`}8#{Xh1K ze(+5K8!K8%VorbV+kc7{@?DutJ*~a_?(ExX|E&0V+#PV#R82ch|3%C*zy;w?=})QoUXJKWU4uVH*tzD+Pa zRfrq_wkW1-<4@kqnOy~qVxn)aBrx3&|vho^$7AHP3>1V zTHsJc3@=|&-1Q%z0<`#q%I&YZ2)w;dJZs6fau^8cldX$qM9~{$e@N;jy3#E zJO9t*#N0`V)EB!K;%A35z+<55jc8` z&HoMyQiBwFW!uUsFRi-V$ zdFmeLij8z_kU;}N_$JrAfJWQycYq-J*!mwj4T5WJ8DJ|sYb%;-U6{ZA>{L_!r+Leb z{j<|654&hT8)iRCj^j93vS^Q`w3g~_LN}khGR1L0! zPXc+cEHCmA`NqTOH+?EVh6u_c0|aB|e}<;9vQa)EpAJOKSE|&E9$FP;aJDJkd4KNb zvmxF_M7-dS`7IMYQIJ`=}l7IEP+k@}>icQ&L#h>iyNf`BKrVrAOE18@7xu)}b zXSpz!loT5kt)WOEM+XD{Mrz zO%33bdVJjQw_*u4gCJUKCt1js*RkLd&~q?OZ{SrN;0BnM)nVv7n3um)7-~*4z8Ys( zV{i62QzupK{lvPtN+e*K?)}dx?*GA4()S`O(c40}@g?l?+@+zNxX?!HbK+JctLKEH zzKNNe@pTQpQ8yovr*R1*5R`Tm(ZC$n$_G%Iq5eH4lF%OczV8RHF$K(9w`LW$Rmsq%j&eFGW9!7VQ^ zSvVgcz5_NQ!sD3T;|pqWR91jS8~w*HhSddkL}3@w0V=>kzy;vXJj;kGqCYh%6AuSi-ins}#lK7gi;xO%1h1j5_vk{8jk3UK+pzZ(b*uO@*u+|wD1wNsO zN0C+Wm&R;tFE*4VUbUQ)r1E7;Fn3&k9L}eH{|-IYxEcDI0q>8|35ST5L`+!Od^JdQ zeI=@OS?sBqL_+q@ul~{eAnZCVw2gl!3FAtC7q9fqTgJ2R!mtZzvRnSwom*kc0bkAt zN+kv{s4VcmFftUq{W+WG;b4H4B1%7*&pkNe2_o|POeJP?ZI(KaZt$C_K; zSXXN49(lwa{eJM5Q`ObF>DmNDRMnd0-0kDrkh95r56;+7j17y8LKfYpSqbvEV&Su+ z5P1aWP(*~rP0XmF_LlK*l$DjCJBV&I{7Tj1G({7MogcGhUrR(z1Te`;SrEJwZ{zV3 zrwi&w&C;+Gq)mPwvdsQl5xj^nfE)NjC#-VzYhSHE`a_coR|jOmrhky`&^@f+3bi;Fj{B5?Xt?%pEJ@UMm(J7xL2od$<%V#VAm_gh^ zN>SVHd)EX^@F`gQXQv~z>JsG+18D(Co5j`D#TKf{$JbmH4GrfQztDGNL@-)U90 zv-L3*;`ZrtPwgVY9#ZnqjwN@uFk^3kjwBZ`(|^mS?t@+vUNh=UrI zfpE)K&jIqoZCb4;Jvr6aZJ7@`&ScH{rhX?iRCKtkZ%WpZNMIhlQB#}lqrDlzE>Rb~ zkPY77#a95GQ@;p+f>`SIP%dS`+rt(>PLJdv*0cm1z88fl(eGT+8s+a&s<7ImHwo_6 zN(?SfjcO@NU)Vi=eXHF>DZl<#)nUyq;Ut_tm7x$X-DZJ@k!?q@wNcCO%*e0sF=5{BE^ui{g<>0P5k>y#xbY^-w~4XBpqn5e^x0;2$8ZJ)F(<;K6m4iE+^cy)JvvWcw$Usj2T1^dJvEul~xd+ihXbBy2`a-;Z)tg?LyEAWrOFE*xJhhoQZM6F^)0&16cQ=JCqVSmymNtB!UWDs$ zz3mQYK522Fr50a8cb8?OD1UjN*N%rlrOPr$$?}@v{Ya6h)#FuZb#-5d+FGwD$xk`Pb)0d27O}!1g?>$)_HZTS9r{H-ISspj7KXTY-d)r{(Y!_?lwYFuI z3Lnt>Pj~d0-%qbHRoLO0tsJQVOK7$)Bk5m9Aw0O;Pfpes_10(S8tY}$6n&0iEVy0= zZu7h?pJUlHpM7lh)+t};w_}T?k;U}kyD!aTmUDz42lI0Sb+LvadJ+QU`Ck!P5=UtL ztiv2d0CEK;Wg~weCx~*(WZ180mqL1NinQtr8)x*&F4X``y_K8zC~E@vRlMTz{Qk%R zcPOz!$*FTXkR52Q=q}fa?sgH;`%+yOpBP+`7#;6hyrL-GH(Dfd5mGWs**Ujw zZ0W7#lQ>tKe`ogAtCdpU-k-Nh_x#5dHu(!O5@Npa3_d{@7Pd|{YkHbPh0^Ojugp9s zzX50te&tgcWq=bXb#dVMcm#S-&A$y01BC!z0ug60-!l3yzs5{Jw*U3#AHT*X2YlyX zqxSl~98n6{;=T=z^kZ$uDvi6MUSp9JhdgErHq)=l-XAxL-9wJkTG{>d{kYK-_ISms zJ0On4Nd|Dvux64v(dB+a{eG3}S?IRRo#dIhfLg}V&Yz1{bOUC_v)<%xf2kVUd~B{! z4!wqSj^l6!*hf6rs5u7Qx8}4tA$``<8<%LFYQ^5D=)Jz#5MZ%+8RLfHmTS z?3#wFQ)``($C}stYyZMTz0HYy$NNhHh1&>`lDVmHd8>_@tDQBaE3YFvo3!AUEfRD% zX2>DH|Ms1*nhzYyWL9RVwbCi*4_Xpvk6r+`svf_xA*fD;`~sw%?ys`SllIq6-=SYl zA#)uWS5;8*&Mwm3IhqqHt4)vH5q4~b;&{jRP(HFgIs5942>K15?O*;c3IP*L*8=M@ zryYbyPLGIiai0Ub1YTo2&iX-d$_G^my!@W>f?^@w0-K1@K}#U!RIGj}XBu?Av~MQg z{Wdj_!h)T^X1;a6#vnGn17t*CL(mC~ER<7<8Vxk0xKYn4r(a+8tC_UDP1Tj2|E@j= zldq5a9yhj2`^Sv}$el=ZQ^@sP6eE&TC39_Y4eRa|P(&S_XPdl`;Gek2=lUtmo`Ad* zIdo&c;ruwDxc#yU#unR*hC_oVLK zrM^ewnve?x&;vRm(3S|_l%w9rnOJB*p{b`|nov_01C@xzq{{>d2MxACwGKmdc;aAW0hn*o8}DBs%;R zD~U-9*2vy_3yUD9;l8H@c-deO%<8T#Gx5NI%i$$#QL*4@jR@F9G_M>HEp zskyg;x#nTYp^82DJZq=%bBTF#x+IRL=Brml&*|oXg4=?-rJvagD}G!P8uEJdxwHmq zUDToy*FHbV(JcTlg(R8>jb0^rZ50NI5Obm&Oo`m5nj*Nc5rGk2#+V91R1+Kn=ml+Y zC~yt-Bly4!=w2FyH-*D?{!(s3G32<(?q5rx#Bl-uyad06%VD&?`7gMKPo01_SdT0_ z%DFAq?p2a3!!@tpJDWNhecAepn!705HAY?rvxg(z+PII)r317h=x1_7NuG93>LAf; zA$6=DRtI^Fk;t43Oz0Pb^%WIjo>x|WAWi01tKpt{>>$j>Wqc1`b=KBt|zS zVf?3S;p?j|aAT^vyhX|T29sYIX7jay<>TQm08+37Tzs<8nB=lyGMDKdDE4_|ce9mb z;2*N{TxWEA$Z!uc+YlJxKb-Xgh@%-QCxW2ofhC=Hmd%nns$9cj8`aXeZJ zti+2z5gof$;ZjRhEn1(duqtwuKNFxdb~lJ1VK2Y?FVLtMB8H@cFsFR*_+ec`oJ5Ue_nU@EzObp|6SG6|ESXK|K!auTA*>}>)IBEL>BD& z2=T2PRZYft|0Byypu!DK1ojg~OBo$l;5jt`!@_9)d{NO^5+;-24jJ^DDDop#d@6wpJ`p!Oh{0Oxi-Y@^7N zqx!xnSM=Df*@K-02Xz`+_RjRjJh3!S+U$d_h5wr}$iE%xq1{s^9N>5&p?1ir=^KEi znZ|W_0v=0r`HDSxOHuL%PxWpb=DpHBWfyWCf&~SpA=L1zukg_C^*n9hGc&$ZHfURJ zY@Mjv_11UKJiVS3vGjiS{>T#h|GKpfe15pAhRXuhr+Iwm?zGK_Y-QJb7IW1_$BDS+ z7*L>oU~8t*hXtKZ6bBH#BBAB^W2^kPf^Eazm{!M+DTopR_P%5;=RaHq1t$I{;FNzA z>HMRD<+{1}5XMG54|iCI3$EErNz+)vzOZ6mwbowvq~%t-bs_e&(8}~rT@W}QeM*BK z(88iIa-k4m`Qij3QoHhN#^|y}7yoNsf1Vj6KWO_ug|7YWINbk{SJLh|5K_kv z(V;*_qBwTG^_Y3u(fo*=+{G{AKI$I8qEStoemPj<z9 zoXOV5c5YeTk{7N=hOB>~13CTM9P@t_4YPHbJA?Cyipt_78f%X+%Kjt;gj( z1Ge^ibPIrZ*+C~OsF6M)P@E|b@eXhfG3Aq#kV7mu2P)$eU~K?M)w|yy`+?*0b4&nu zbsL5OV6p%Fy@06QSrzyBptNLO;LZ#F{24IN_o#G#jbL4ji*6GAMs&f<%4y-MN`}gY zu?N#$;e6*KmiKWVpZ;k6+_lC85V`jsfFx41a)BJlyn zg)nzPodR4lFHCE|M|U|X?<{4@G1RlvgrtvYnerOy9 zK06d~qgZoc+=yM`fiksHrQ+;Tc%ls~4dMuVuIB4%h8^GVoNfCL+(r*b*OXeO0=AVj zpYJzz)ujo$;7o9I2^Kl4p({M@M#*Szi3Z|%Qy zk3!0gQs_oKu~3VV9nXvh9dm>Z0#6MvTl)3WD0qf`z1rpTN$=ij@}7I^(hCSn{Yx%N zUyJ)8_l9~3ue0q+(wRnw9x_aNrY?+YT=UX+?@}aTFMO{>g2_T!u`y(MpZg)!?78$z zu3}W~uAlsh%SbXU`_92nKjZ`O7xZbWxFeD`P|y)Dw7s@BwWajQA~aci%1cZX4SJ54i8r zXV!NNt_#=oXs<36a?hvqkG$CN)V z29gZ;yuFvXR|Ws@ zhw)TYf;TK!XV(`YVtmF>ryLoG!t?8LuhiIneZ5>PcBhKtY~ozqio6yF{a;rz4*G<; z+&e#Sy;?phb-MlM`qw2ei)U&#I|NjubYFN=Rb-EQ1Ghs@vQWwnb5dCG@t^E~93*E9 zJNX`3rKjeJY{I3Ik~f^(cD0BV<*1FEz3YE{*#AXjR# zhmaif@6>sCxt$4x5i&G>ps~3xG0r5Z!h@^1vXLHFaZgtp80X z&+A;ZZ|)nKd$kGJ!__B!u1n_BClq-9MH8UT&+o{aPS;S!Yk#(7Vq zUe$bUnU&1hKsB0fofjBb}`d{O( z%31=Y$Y+1f1HL7tm2pIayuBy5{K!&;03&Y!vq#9QV|sWBFolG^fL5qz-5FJU2l35h zUTlDwb!uWc_Ws!Y3~Rwt=B97IADlbl-p>bZg2*sR1=%x80O@S#L1n{ErL)ag!9DcZ z$n#F{GQP=)@Qlw>0J+D1d=PVf+p*N$v?*aZJbxSfN<$GGOlHUI(%)##8LTWayu$0l z)=~x_E3xAtsvccEbuF{A106ALzCef^m_EC8XEZxa5+Sw}0^h1xOK7w94SGeLVLJUA z1h;5HBO}T;V^?;+Q%b&~%YR;SJ-nv2DgD8qHUTdFIz+UrmZ|u&7vLCsgvxa8@5M-b z7Qw3%rto7~_%nobd{a%%Oi6oK$>@WJ9Ot`hCLilQEZkvweg5QgP40vHx+*474gq34 zCj)!n0A`0`1=K&XBY=gn41!D*fBtRYhw=1QbA8+^q%)Gs`dxyg;#kf2CGmCZr!hCb zLPkW3euHRG{&b3J4Bms-PrD}rzan^v+1X?}qTNS?_MkZPDU>5h!B?PUhi4`2?12;} zqKQV3-A3roxyRQjS=%6A{TDHT7XSL6aEx{6Di3v*!m|6zYY*z+@{hJ-Gz_4pgY=8a zodVOW{DEUs@mk``vfi|;XpIt8?B$*pFCFgfi%7S+r(8O{)o^WVeLbJ?Ojg-E^SRQ% zS&M!_`X#w-SKwgH#;}r@oWaM2FJPJgd&>vF6kq_R;M|1xdrX0xh0UiV*#i{gY*cm% z3H3)|3(!qP4&#<;A{C7p$ej42*MNuTp^$XzgQS(>H)E={)$su6wjc_q;9GOJrQ+Z0 z#HvVliR}3kn20b8>ohNUUWx65nCg!?_-H6&cfX79wyuCd(1HJ?^J8G>DM z)=v^&^fpTb=F!+KMoC`KJr^Sqt9Uvxh|Ww91X`E=u40^oT*!q$h7ockrA=YZOsdKu zl*eXja>{D$C_#q_KOp~#GALpuk@wVO#=`9ci&tG;AaVPID7-JJloX3^+euLVTm%_%ML_Z(463A_c zM1KEdM^Ht5$|r5B8GNZ(_vxH}MbgXJJa&jw_#oq>^(|`Oc+_>Np{n+EdE(6r;X0-> z+~?v-5c@{5y%^^yo{Oi+j{u8jpM2sGAq#IemiR)D@IW<0JL;Y)69FHV62EaJ3CVaY z$R#LQbf~^wqzNvCoklMw5~Zo_v&w)PG6%(o>Yx~1MBLtTx%w9(_V1Kk zP!Rm788hqFCwqW3ZOa6gH`X8<6gciriDY*^Db`u~7ih!~)QOcSJ`jUA?bc|4Gcw5H zUYD-7Q~*v8?>FP8K_^ypNavuMRdvo=iV*2c2CPHE)GAMuWBTX#`lP3io@>&-)x=(B z0uFd}xsXgr+&~C`*7&puE8-~0HJEid6EBxDJufs~PCm3aXM^X5TNI~k&HME7xtFKUtp)*)@qbfV{XZ$i|KD~mw8n;%j3&?q zubNLY^h8U~P-g_z;5TSqi-F=uTRC(f#M(NBemtn)B6M{kW6m(6{zmHIR1ni3-SI!; zv;24JPrUK;nFS#GrD)8>SEKQUR9CxVP<8{3n)@h>T+YSnh|5vJigayfq+G2`$ znNGGK-cGJ=QmqaZJ52lLGIxqu-{165j5|}>csJd zGcoo{1}2Po#ZSMEnqB|Sd{LVNbzAj@?wZF_e$W;avw=Rmf!}CLq#_L^@KW_Zo_HNGKtY;(eWSX70>AbMJgJ=bIVN@A?-%LP*|s zzt4X5v)5kFT5G`_(2avc!yyYr78=vOQ@p2E#yCZu1t zBBUp*;)Q}-b2|=udYXf-PciHC(a^zx^4SNoloPqN! z9d0*Cu)|*6A8=zgX^HRX!cWFeQO8%25Bl6-n#o`~1uST>avNy4|K?>fKMnvqtlEx#?-oYrl ziAgzH_jTreec{P_nw7aZ>jl7GY|{izv4)k8`}cr3;Py%u8~)^dTn>F5tlOsb-6pc1 zBzbP}_XH{5pq~l`EJVP$fJ6ae<1+g79M9x>mF)UFf>Vv0{~n2@5}>85P6(Jt97Ncy z?}DDFfYttxq`d4lRna7Xp&Ry@i3ca8xjS*Rtb;es)>Hkap@rK$QXNKD{$vmpA zZOl=+C%b`;KZwuN9~#Yx$zEo(?ZZeGtfk$iDn}}Zt%<>w2Pi?VBxOunAbPaXyEj%* z-*wR;mH$nOYurl7nf{$t0j#m4g3}EHwPjnYcLmJuLRkXY#`!#YuE`yQa$=O?)dE-* zb^tvaSy7Yhq;B?Uq=N1X`?0WeN4hExT6yjVXODe)1CVsKM01eRz9YwUU>%(nb);KF z#}ekF;1-Az{7|MBpm2?*AxTNsExDizHx{}f)44=@;@n^C00_T!Gv^7}JSwZAoeKO>4Bchb6XGQ=T<3gMLY{$|3ZoxI6c z2K?P@JgZG3SgFwmE(_>cvz#r0mh%p&b__R5qFKeOFon*tSOA?7#n}R2Oht>{#1p~i zmZouWaTzF+yEk+^63p%j6=d;jxAkM=mMuy;cAnPje$3yTc7>VN;nQsAG8cw2hq=^&Bz4V)j+sf` znxEvf_aSHf8GwVQHpEBHc9iRU5#vs$ldRf+qDD^n%|R_&K|%Sb1oqf2zW$HZwaE>~du;BgYeunD9)+Z(`KkJNh z$^&kY@2SssewO^0beH-h?CJC1rR3$IrYD8$@(S-B86^XlEENZmi^KzIrBMdp%qENb ziq`z8GXTJO>C`yw))+7Et-woXQ_@Igv(XvxxgQZ{%PT{bXAtFBxq~|XoClBC$QBTl za)eZ5t1h0s1n4@~&tQr|&Lu*2bnAV-gREaW6PX>`yGim#eAUtY2O(Zz7XgS^KnAZ$ zU`5RPCmpXseGYChzw?6*v#!w)2ki$q~2d+t(XXv_{G&itH5pW>EkbA z7S8umOu8OR&o7<5UuCD2;gDc1Xn99`T!_nft8jY@c61NvGAF=Q_`)z5!c}DO_M`dC zrCF<##IlMKUNJ)_iC$QuT&S!g!_^rUPv|6Knz z2n`$stOizGXySB9eBr~m?;xj#8O2<8ra+Ut#Ox?l0BO-?l*pkf5(7|Ja=8tAY3Q zZ>cS5?Ha)$(di>p7)9B6B@0b}s1qJcbK%iVfv0(N6%9uP0ax{S`y1znC8m3A-X>W= zJ=D9Q^hl@e30%zMOjBNOq?+!$Mn(IPl?AmKpb}7XVL5aoD1y zCpgjwEuF9I;JC(;#kJP^kQf5DhO^+HEgn!Ff9X~5_<_0CZn+qv3w?hNLC!FCPV{R> zZ#OT!M-pgv;Zmg5_~2pLl?uZwKX#;?hlzKUcrAS+C2sQ<6iNsoV^LRerWls^Fhd}mwec*lJ;~{uL>r-j-1v51BD(j zw#kzp#8scQr--{`Ri1gBs|5MYU4}MqpALyTk8B@zYVl~GNAuTLH)~)OstAEKp9!v} z?<-S0iN4`S_1ew>5PtY0=){SIl;DcAYxtH#nCzfgq@}9cbB?lstSZCZ?2l@ocyGc- zg~ZiXZEtyRwh0sD{JAFP=Ue{iqft+Pi;2LY9A;sGYSRWyx3C-|W69g1e5v1!cb3_5 z%AqSy<$&i7c2kP&;`_C}&GRaqcd>IZN}bYqCOK>ak7uPSS)R>@t-{eph>0t$vR3vp z)I+eg^=Mg?1t6CCHES<(X~?Q4qA#KwwvAmR<~LlK1d}^>UK%@-T||)z4{=-N@Tss4 z7MLHodwGC;>(PSPvm>sjjvIEGlqP{htucelJbmz#LlSd%poB#|# z#mNcs>#6B%^LLu6oX(IFm*VDD{}I*a{}9#JenfQ}bZp$ze?pnXAXm_INb(Xu$N3NQ z0FOWO4q$52r7Z*u=gsN)Vk3J29}b?sVC;+`neCGW&h)uUFsz05+7^uW*gP$8xn+>4 zPOjYTxI8~)Za=T{b$LN9mc&JPN{na_Z=Bbn-hPy#{V5U8)WXu(FhF9Zrfq*Q)t?~1 zv3P%<%Vv)`Uzby-_gl=8$Z&M*-4qgS4JL0jf&#)s;HW~W_@uPXM70AJ0N++FR4UztYYlrB2N4>Hu_4T1#rg^=8VXwEEIs z@q!$9{dvKlO!e8<_%BA*3}*94FaU4^d9v3TW#P~6vD5tB`2lOGbI&?x zSTE{|y^N8&4EjJ8t4fP0@wbcz5{M7D3_;xprd8yiZWIt0ME+=HJNnVeX8U`zvXN?g zbSa3LK`W-3)NJu=M57_^Z?j#Fjkw$;}JL?aAALjS$l)0(bUyn={UAmTZ$C zmOp)1I~r&y9T6;R_P*mxKPiy>24p=D*wy5({Cl>P1O%`!#}U|ob2DF;e#k>;$h;Pv zrKds6*1Y|RrH`Y63I}pdWd$iGmt8N6qXIIRkZ`+_=yU;MMgyBYl9ixrAX^{%Wk`(N z_I*hlnKTuWVo3wv(J|YqA`t(hV);v@IyYNHP2lAiGZI+ z_2~flcf?e<-I;qmXmV7I&|)G=i$3u?$Ulz^f_eo+4%3m`cIYfl(-e*=06?8zFZ~*c ze_l*bZS>>i$>Z^mAHD)j2F-wds2-Iqs+S6<)rjq}MSX(#jf`P`GyRqAle{%KeGBu6mfc>(VwTg;@Fl-u8`9HuS?AK0da)5XUPwB(+WQUQNRJ@Y&V=BoAqjSaD8 zh*`XM~8xqH5<-xiZc472V?_SOM$QM~h z?fJY8AMna)ueMJljoW@Q#QUX)g_<+uBb7q=>B@c8$gx$N&1rFVMSXA~k<{B$%xAfNQ1*eQMuh@jvN3@CG z-fb>kkaZFf=IJE)!ROMefL^^uJ%1Y2+^H|wIB?s6+!wlvaybrcT&qx~POBFOs`K2V zV~tz^8nk1Lt+x9Th3bZ1MJ8|75s=-7%eOJH?Xx%$XbYVL&6vZNES#=-&516Cw{>O? z@f}+n-&V(Bm6yAw!j&y2!Cf~P&J`!|n2PKweWfufNj1{f?1w(x-vV}#OU~myO-Jw` zi1Q?ra#iY%mQ4mAe>t$3U2W@NB_if*b8Sh5GBWQAh<+L5mK&a!&iiU#Ymej4C_;{H z$}j&L6d{I^3kN|zv_$7JX3Gc&dXlhQ%jWU%-Z0-Mu3{~dCbv0KGY^;5JvxrvGdJS> zc=MIh4HmyI8l!$5@4_U+$eZ$c>DZ zgiEz&g_i`p4z>c0hoKSJSU}Y{8(g*=QKU`~?k$2T?expNt-~eeqv7Qr zkUcoajw5ha!<;qP#-Esf{tb`!#{d%z~_5oX3jv^8pMlB98tRE#0HJx#Uu&e>n_XP!a_F4 z&U=&D)l^~$rD?6{Vo&5>_TPz{-MQF!*DfNZq7Y4zINByk3_Zm3X48T&y_38oZRmV1 zF>(~Jun_5^0<_&V5Qk@Hlz^t9o*zv`3*X>;evEh2R`5;r%`g85kdvQBVMJrGvAZRW zyFfM56avQpVoREKD>zD0mo7ncN7XK&-r?4^{z)&TbpU)a(qbsFWQ=b<1Y=4*ri72D zeg~zvkyhb5>Os{O)TE(?ebNo>dC|rJ5x5{p=0&|m4=156_W(FQOcZl@$3V>YTL--F zsC`wvgfK1l!WdCHWLy3f#dIx?dQUvhx#qSyVG!wIur@$?dWlc$S&o`JTaMf7X*a7d zRcIN4;{Pb+{w!0o$yicdAHBCD^NFWjFP3ZE{c%Fc;^#r_qA(wpRRM1FOknkjGI-E! zdVv9}braQbFmc?W=JMml$^+@O8^FgNe^7#oWq<`_j@B!^d#HBlnOt&wPr3U7OuoS2 z$#}GErSL$x66dvNllG~q$q1UpOB0jXns7>xfzzj^0||p4dRNRdW@c*~+YWFChHh3D zj5e2>bDkoP!=&{~eE{b0Vr%*)K?ZXdraG-UFh|d1Rp)BFd3MUTx2CV1&!w$YmFMlr zrh+Jt5-3l|{f{-P?9GKFo^$$^eAa-DlE5>)v0LHgb@~Ohs6fITeE`FVCzi3P3}!Go`3Pi!v+}i)howCGOju8iE%XRM3lZLue@or zkV^~7UR9=`tNtgI0C z&7@_)(*clk@CQs0{vyO@aEDK>bBNZu&`R`3J}lKgN0jws#L>h<{pIXO@8tv_9*2)! zeg=VyI@V8zsnNfe1PDUjeaHI#Ga!Eb_~X?0^N}D!0|Hd*Si}{MNJN=uK*@x*v(*ha zi&ClZT-hV8O=T<3ZWlA}Ll^~%7zjAs*kHkntK#K+PenTUf4SW@BWhZ*KYEg*lIslb zMZeSZp57h3U&vg@_NM=s9{%@){PS=9(=7d)FOuy#117<=zGB2oMQf&588HjvKACqn zLP>s*F58A^w)32~DX5IeuE(N4%z(PuDufbghX{dy%58Fdub`(qcDn<`_Z5PK8|vF3 zpHe^ziW_=)OsZ^TtI~gQ4PId_knzMhiB_%cO5!sBM~6y%lASzVc1pTU8gVKMrmh%ush- zdN9cY%DSK-&zGR>+Nq-T1WC44eY_I2y63=!Ew(gMDo}8#xIbRaDJyN}!i) zSIXIlWU8hY9cxifWavB+J_5@Hr{>2PT3WyXLijpR?KNYng6gF(oK2%7I!y8_VrJIE z@Y3l?uL0F>=c`@h+Bz4$87s036r6BLk@f^cYKNac8;Sa+%vu>>{hz>k3O&eDf=xE;AkXjyH)8do^zfv zc(2D}*x<(VO_$>d40FfrZMk5k0m}DnA5$qBH)ZG7wT^u);&D$Z5Jkws>VVvimNHK` z+HN}f4vJK~EYmJqx$4S4oR<}y*x6KjnY%7yl z6QP~R(^*|sZ123!3w!f7OXG(5GS~fPoEqy znc$=^yppHY!Q46a#!LR@)*He`P8^0Pj9%4sM3O#<;@n>j*m%__hfh% zCvMWa9M_5 zFVBC=CUh=-ohNb zWc1D3;qT?MzB<(v#{~3=&%HsES$zV4jZtL9Y+Jl|4(mh7X?x_OJ89~MGq+NnvBZjV zbljr%3XbwJJqgYm`g;T6H^j;x-ojS#N#9M0=R3&Uhj(&#aMQL{c0`YR>kM^Br7F+P zj_k#P%R+%Oskc4YaynltbK(?$_!6xc^7Sh=BitXf0m$L`ceu+f>f{CN2I7rGg4|WC zZ12|9w;D95Syy%0G9Ssfa71E#(#7bn4*0p%`PDrKarXQ*|q=94D zpBHz+Jm#R?Jg02pCf2ozpkWio=4EwrjIEkn+59+tQN@_k1h|SW)Sc%|C81Xe923n| z>*Bvl07`s-W<=43Q^=pxF7b64-uxCcur^*U#-|~`vEhTh@BZ<+YePv?F#e?6(g8lC-2)#q)-)e|^6f}UhF#!sY)Fs%_Z_2#< zT&V~0lJi$`oGMnuQ&;wGYuq_fe0{}Kctf=0DIR<1v7VcLa8caA$8%izQP`CGk@x7r zt57=j6eFx@?PDg(5$p!mvBwhZ9@?Vcs-7P8A|+*gvOmd6b0=4CLtjuRy&^dAL4;|I zt!Dv+Y_fsoRjMWZH{??S(~hf8@61AsW-ODwIw{~wZ-4ZbO?w?wH!bsE+h@&7p1@)L zt<NsS?M`=-A=vVd_=cmhS`?HPNm>FgfJxeBXZDiilNSRxcxVs{f2Hs zq}WJN!^h#`d}oh|r9(x4R3ZsBZYpstti+Csca`|3ieW|f5|@6{aQ}1X{NEC-1Wx+? z7UTc7E@1zS8zJ8rCDES4i>yPa>#aEf59X_OgHG^r!A~PurMwoJL!Ln?r3=|lmE~el zam(>%*vQwxW6RjwK7i(r3}c!7ipBb0?2kV=x;G@>^0H~_#wXwWi#JK@j}3Il?lSrR zp)u@Va2NmMRvJdXyk9*^7_QG;_cYX1ROcwd?ZO-YeMM%N*ZMUWlR($A z!M?ga3U{omCax||TGcsC?M!m$*KgW1$X+v7x7(#Du9g2XaHqt`YwS&B@~K_*Y53^% zC2~9yX&g$kZ(E0R^evmRoP4?5fUb&YXfl=~U%ELcQ&*{3c=xy!)n)d{{aSWTB(4PK z+D-b}+7C41D8S>z4mL2rvcrL)d@~_X2VYPqxA6T>0c|`9PUdUK9k#VIfG=zE$UKc6*WVZudCVC@09gaK@*$&tVokSaFTN_(i zi%AQMmeN*=o0tW(ND?_lnxRdHcnkeD+Cm(pTrsu5@9dx7qA#CN?S z?$i;~Z(nX7Ns%tpTHM5mo*pQ8`%IO#;39|%bh}3Lg-D}e4%^#u4N;*wA+NzV1sa`? zH}~%iC;ZkzWEa&XL&A&(gqAX zgiKs3Mq%EWz~;Fy@*4ddrO3(AKs#vXAV$&I#paSfDV`5ga2Zoo4CsuMt9uM0-o&w^ zuQvk6Sk2LLsH(t6SaY*j$+a~VMLhl;d3W`TnBp;k&P=1bB5Y(U(m&B7_*-7qz!}vI z9?TDJaG1EDn;*)`y8BO$a=_vP8@&y0EXtU$y23*baqK07U-1UouSTaNmem3?(2o@g zNN(i}R6)ccf42blW4|ej{&UL_rf#Ni#7+`Xkkq!#0ACeO^E*yQoPmrQ{t$W_JzAIf z39j*2z~$L+fM<9Lqt$qX2q1XAy;`u6T7QQ3wL_Txu-VNe{+smgFB%7&3=$#V5o{-s z;Q;m^MGtPn+({5o9>;`P2PnCl zOmh}+J{+C?OztC3ENh&&8$RlO^ZA&t{;U?9bLj-AaylR5mWrZ?WkDBw@^pVM&VJJ+^7qqyMcZ1 zr$S$71YF33mR`LK@KOD#o`_fP=BbQ3uw0-^ED5!K<3SA){jQLn_PCa&)BQ0BM=L#3 zFVJI+PLY&%X9_Gxz|j8hIaB1pYb1%j}Im=bGz1j&34ziO|wdG4uY) zMKE>E(yIdk7(kIrnNOSTU}whSYKvJf-FF!2e2Q z{CDJgzV;SGXag;-X0Av5GlW@MM8=8JDE5Az=4BHSx1e?B&D^9XK}?z;LM$!`a4;z$ zoX}m6sftyk$?GkEDHyzLb?`fV!YR*U`AKPm>Sr?*icqeva_vFfM&x_J*eAqyBtFsr ziCN$@+G%#YLP6DdxZEP__*&ET^;jgpXpN(nRk&yi5~%q(2IHSvjs6|^iY#IQ$_q0@ zjfP4P>=yhyM%DY>I-*8zdrE{{9U~il`G(S%@?{)IC{P8Q^c}?GuRyQ`wmhM}y5+dD zXf4(HGPJkux191Qn-ixm_FyyC;3g zfpSH050}}tEx0KQACsQN^XFOlcurlfvzdY=PCW;(q4zyg3mvP% z?hX@J=FJ3pn@8nFc~0n_?L>|r-1<1K(}F2yd29;Eeb26_J_Wid%Tq0iZWXkro?Ywl zJ#6|r9>~9NW>T?Z5c4t5(RQfVaAhIK;s%{q`+@5;@^YLEJI&MwUbH2Z8Y>JAny(9G zD2MXeUyb3yNHpirqsAD|Rwy^Y2XpL;qPKC~D*8?ba=n7ClJ(ZycaWEj0T+C*X>?F1 zIebrd_{!a%<=&5oRh7`FM{AdTw`ejUSae8=a)CgH+O`9@P-@Seg6a?6X zLWGg_h;hS(vm56YaQ^bX)FTxN2%4*x>?^2Kx?kTiv3~8iNd73Uik2@r`t#kQDi^A8 z=tuR^ml6p-Rvy{d(v=t)h_XCF*J)JH)d3**3NEV@bbR0#MF8g za4>;9_kmZ?mx707d2-$_=KK!1-1HMGs5&#=&?gI#53T%sa3Ic*L<`5Cn<5yZ??Ctz znR2B`XV4*qs`-;QYJ8^8XT9I3``Tw`?n5lDdP+3=5LHHCyfSnMl^nyLuHXIn?qB2>Z(W@Q zomojq@x98*%IJ5g4sTAYMM`#e!$2U7vjLRSC2s*J8t`{Jn1@b+=f6gv$>);Zr-lYI z=AA4(7f&qo2`X}H$DNG66{4~7db&2C;TsdfhE1>7_@pM{blntDjhN<74TEF7aO#-^ zi9YTJvMgS;A#-$z62b1L3`Qo3sNFU-n+3nsem>EMuOu5Ar>Wbdiy zcB1d=4qQl(<+8O^WCrk?Mj(C3z#S2|Gmmp~)}(yRiAnd8X^Ix{PVJ#5U$#rL=1+fH z-Hy9T@t8S*TJYdLp)`Pgd)$>oYgVwrS?ViJ%zs-GX`#W83X9(y7Z;n>nH487%dt{7 z#;eV#%w%u5kR8coJM!p;OLtj1UmMzV=U2uGr%r5**d}vlDRJ{_&@<&plRf^OL>2x@ z?fb7uOTSNQJ*2||eds}9z7QN&l|MZW=A>x3@QmR^Tf*ZHgW%kw8+Dsx7|I{^z1$=t)H>zr5O!Y0lz|2 zrVvgfws;7fuECAsFOF$dce~Sl2Qiw~M9z=Q0nIBA-N~pZ!i?UjB;@cSb7bmDiz~eB zfWU^-Lm!M2sPH%Kt763x{o)B%ooMj4Y;^|2H$Tu_T=|NSJBjx5XN>vQ>EcCpc{T zQ(mTCPN%a^^?8*b5@(w>c0TXmY|ceR!}+kyO#EfS`HGq=$QBC*3;jTyX{j4e{GM(> zf)DJvqLJ%adBu@1%?eqlSGI$uB~vg-#J?X=F1I;8Z`#1(dm3%IpB^IpdRF7Zi#naa z>QvI==8bSZ{@h1*;}bU6MvKB4o5Je%?TpTeI1b_QSW#kxq6}W#UZQy2)xr_VEMrgM z9h$?F$dG*_$f>9bbnQOFR~5UHE+AS32HiaSKVAJ#GHI1vi1|os#4}|1^!wVLA;D&U z>6_R|Tu>Xc`5pT6)2g4VRY0HwK%MTcBsJCYGoIW{fbBvtj)`1N&8dUJuI?h<;(wpUl>_O+}eN1e9(c>jojquUV{MucQzJ{V&9Uju`XLzfXWXah9NQ zxiUwud|Xo`gDLYTtTle0Z1-C~MG?OMZbybd$lRDVzk?zg#3}|kRBap|>yxo_lW5P_!+ni9wHOXtAXr(Fw_2@<$3HcEA@29wZISo)f%EfRc3)>BK+@ z^tt(}p_^0`%wP#eLWC$fLcYA}R>*w_iT@Mc4c@Og07n;o0!-4}Y6QnF1pk*;pQ?W_ zqarByU{?Sg>!fTzNcT2CL~dJIjJ^Aj3FmRJ2{gH=ifRCjA;kwk`(ghVfyVn_^ycVYzGj28bS;-9tDjk zAyA$KmID+vR+=ZQ6&Yck8$Ru_nY$`G-!z5TDt~XK4t3w@-M3Mm#yOx@tN&$w0NP8UYEhbh{=wZu8L*0P6|X3w5Y zakce(Tz38Zk>tgbyB&LgK;^RMrZ7-1r&m-bEVO~k#7nTeNoQl1f!tL0%Hg2td*?gP1HcM&j}BYhW4LpA=O*RXNgpwct9lJOe{Ng-+T{iF#Sk zZG9E6KY@u@Orr3GhjRI{EQp*Oxqqi2EPsB2$SA22^+qe^+Zh*yR1o&zzo%*LCx9s! zug~kY&W6INrEN+GTA1P#6kd`hYPu)lfnZ*waCbr#UgW)B9{HSZG!ktz%Zf`S;Lc+2 zp`AK=JJk3d~VEO8qj@)plIOqr9)5PEKr`$E|hCYmG=rc z*3PfCV|t`)mmce0(2)uZYz28r|3r}5&vHcZx1hhD^*Z@-Ti_v3Mt=#IA3+Qw(Kulj z?l$_xwB*!O|0OO7wqT|c3%K~CPb=iRz;42mzweLlprsj3aiP zn5xVyggwe_q4V=G3KPUjHRBrId|1mF3b#@2J)H`|&9;Nl2F?@U1*>8b6)}~>)0^Gp zo)od^WMMpg= zk4V8630{h132yH>bA^qK>|p!z_PE_#*aqg_DV*b6oul8?#QR6Bvqx%~^4ql!xb=MA zQtD#u$wz)VSm`H7|Nn}3`6u4yIY%Dx*m-{01+~$=*Gjs&wYuUyS{)qrw$QCw`SZI-1&QtFui$Hed%OC7=Q4kz|g8P#rVq2Luo8!729`ANW;nrH*VyT zh_s-i6OHf+=Z#qmr6GyMIa+`t-OPXKQ_s%$l-vL}763tF`l#}+KA{{s=jy2Fs(NxN zL9H<jU*;7M z8->FB$iCBhiwl_-iRJ;yA-L;6(fS;xjY_E+B1yp7jW9_A%)Pne!MS6J)28`S@&`-; zej?+61}KuZ!O`kW5M0*=$UDaX*!q7KB=_gVG?uS93;v(%!D!`-&dP5qWkl2!m z4OM2d7C z`2v#ExP8FV@W445v8S!b@*U(0XwhTV08zuOSzgFWDol`cSO=7PLx8K{bOT^nD2-Mp z?s|R)wdVm}A6ECHHRP?sTrB<@fB?t-a5iM}z>NdE?!iXzLC@@WkN{wiD6oksBSqu} zLAGBbNlJjTq0;1n9r1)N1$UDk(xKEG}-aPRRAsny&jPYf&4YEAY_3m{G-9hx#+lHzV9GrWqAd7 zua+gRI^Ci6g)pGZ(xYm?fk`yMcwrc%E5!g^Tr!`(&}ioV$>f^UnoZSCmUGrQHnP3j zfh10XJkIXE+jdwwMW63wa*^AXQo4~-Ur2{4bO;ZjJ}TW`+mBy6u>TJ*ge}P=TFUa+4Jb#VnPu14_>v8;g9KVXaU&Y?9V((Y6_p3PhIafof@8kaq*$25e literal 135442 zcmeFZ2Ut_xmN$No5~>lTw;)B7suTe!!A21w6lqeT(m_C_DuEzMZvp}WQWO#C(mRok z^dh~6BGMBO2#|z-oS8fG-aGHTpYJ;}|2xm~eTRpH=dkx#Wv{*WTEDgSIUYNn15Vvk z(^Lb<$N+$h^aC7E0@ncY6F;w?pX8*Af{NnjMNLITNkv0VOG`sdLqkgsrl+MlNk>D& zz{qeCeCibADO!4_(@dvMlm0&S^C4tEdy-R7lRBQFqoE_+{e#!>XMmCVgb`H`IT<5x zf{~1zk?gnyfROm4B>R^O__vGf1UUsI6^NRKmX6e*{1kA4MEe8Hh!) zBPG+Bi&v;l>)iwKKVp`87W0-`;A(Lri~aycQ2PGk02*3WHg*n9p|is0&Wp&%UXqhn zxP0yU4K;NQ&6@_d4ULRVOz&7(+t}LKJ2<+zdw6Mj8QWeGJR}#{;yif--0!>`&4DMY3NLEZ|Q`_V0rIhg=f?JvkX^@W>ef zC_pgCIN?tVoCE&f*Z)(&|GieA;L3u(jk`B@)wpezvF;*zPcOFSLy@Z^@;tXLDA^HS zDY+h(rGd3Tcw#1BtvbJtMI*_XEKX^{{AEEURp7V!z z{&UYc`=|8t=V<-8vH0iG`fu0IKeUaoW8nXHWBnB)`-e>bnF;pK9n}90L!_*Q;m)P6 zJNQHkFIc==Ci*yVM!*kwsxnT7s}agkHVmFN0qqNYFEVYvyGp8bpu%sR0{8kWUu#xT zz!*o~l~ZBB289+~rCKP%32d7oCj|C3aeJQ;EI4o$LbhKREj!E6 zv7unFZ>^DZdoId5$nY3&^yfz2Pme+}z3w}@N*wc(8Jr0DM7A{aA1)A0AxD^t zsyuQ*(76;;2|V~_m)D%)^ivOoopKAyr7tR30`3t+QB~foEWd7KX+k*dm23nJhKb_) z7JALp8gJ2GjMoNSC-7%*u=|(B5f?D=RX@USRKONrky#O05}T2S!ZV!5K;tb4{sw8o z$#38~1_o3xqeR-f$H4qK;1~$B!ORjv_k&Ov1JE(Rs0zZ1LW7QhAFw*vV<6x)6c+|S z(WVHwW8j@Hx(LtQcnq}Yl6ETEE;Nd8dW!)$_W?}g_Xp#QkPM_1Bi-BS7@)j>#4-cO zrc#{iF>w8;8IC*u{1^zyApbp!@4mk^@84+Y-^}}W2JSy=>Cc+?@38dm zGwv^1`bR8@{QVjCZ(5oq(q@nr0sEhefZd;51pZn}=cn-;IOs>a4bt*};BiZ9@C)VuBJ6Os`(T%30$pQ(VzN)yWm?v}?9iC1%H3xFraA*i`zNJ(#j+{6zCa0@SU z4anW?`^3~_rt6mEgi=4a|Gxjl1wuT^DLSxc8Bb3r_Ha6~uL{<{rdHSapCf{0>*h$J z6f-T915&U_;k`^8D-pJF;9cS^BU}H!WfLTugO@=!(lLp9h$Mup zhj?EY!quZ(tWNx0^yO_$2*Ko4M94L!s$TQw@oE`h>U1=0BlXB4uokzxV1)_)%x3iv zli&~A3+xugxxkf}sygCdQ$bdle`=cQ7_dP;L!J=72M4@QwkfM!3>uNtHD6`ozMHPx zW}rNLspv?CY)$5$>TY-ow?&Ud&w<(^599X(PdEGVGkj|;#395}2fN+`Kg{fUwRqA( z>ru1h9bh}f7rr+|h}LZ=XbWX!=Kpp#i_%$V=SWez_Nx*~ zdi|5;rS1jKVPTGq!QvLfJ!U(j(Iss{=VGsU@#A&xM7%8RG0-1{#Mm4I6L~w)xRBXh z(oP(A4D1ER5ef)>?L+|CtXo8&UOon>;Dp@Yws`i38qkoOmOuRj9MqvT%^Q6TybvsU zjdmkszOh~)J~4<6sq2x= zYdll$(V1IVOy4TLMsZWBce903S3&TNE~uG!fsoT6O&p#%-+QP`N|9O%#&yi^SVOZh9u&4Jb^zAX4CaAbg;|2M#SG7q)7- z=E3w3eLz3wqHyry`63@XX;_Z|XGta;yzM2PictK6T8U-BAmqpFDT&h>qipiVOuijS z^;gEsM6PRxbb!Wg|5MkF|HB>u0?nf$Eh#ajvc)Lpn;SYK0WCC_-azUW1Gys@W7VD- zcYQO5C)gA;r=(eA7R~JK4wsKD6hvogX-yw+U!P^iJI`0$x&!mK30AMBqRaabW!myc zu5+E|YVI7<<16Icqdi^W(qg@r-UgPRcQ}7XzvQW$d#sVgmR00}XSFDr4`q(FrSt1;{mmx~kK^BgSPvAYkB^O^UGY&eR+D+UX{y5+3B zO%Qj7(*~(w;jO;<%UT1nsrTIVkSv)~S!El}k1osPuuDDQACEg1b?ve>c9w@--GY5JD4nrM!p?4lIR=kE@x3+i@$ zZ+We+ORGXoiqvZGf}`cIKtI+|cbd6+tah0ZQF{1&%mXom;>U~IL9Ov`v|I+CpK%ij z$?zUeT4cKJ53$7a;+8SX3$4K+<>^73&5nn#pxtqVvc$X4aG%=O837Kw)1^uL5c}Iq zcd4xJ*Tz|W0wMp2`xIF-{w_`uv$jz5d{c=9E1(*#SNw`K;O>bj(@r&te$9xsg|83Q z={Ynh$iE?jp{#yF#1<9Fp$1yg23&V|3^dj2=sBPvyqDt$Z;%0lKei23S7JTM;>Foj zbV1XndRSUCm6+bgXuKU5cxI$8MJ@0#1JwV&693@f6~-Ec&Jjektt{$H&gd|zON#DY zw_HaaE(52{ITGxGI_;IzM2z6!zOk#v0D3y%gJIs)7f(L)ofxk3Sm=x79K)c3_!3Hd$&NVW9m=kbK|L>-=Sz`6<}fPfoGmcfOaayDAeVMdh4uLoG(_u<@Y* zq*>~>)H!(IF+lC!u?uH0pR|~Ld9%;`+4z`M$i3ya?ecB^rI|RlgA8hAxtF13v6Y*qyLhrpLb4#s+3!i7jqB{Ho zKMO8aP$?8%uCr6>>0$_Iu4K!M8yR?dX;53#>v8d$;>~ij4yPBy?dTHj=3%RCf1Tb( zQ4MmzLH0s^n>|`KsKu~{JURKD*aX(P^6^?tKW^0P8DpKBm^gf|nk_C9!@sB0E9q_u zaSSls5EOoMcdmtSWF0*ZjP`w5HfJ-m*wJ_GX*}=>UmM(8jZ=V+NF8`Zlbi)$*X%r= z20PTuwds5%nX%&f6lKq=7f8CN+}4nqu|?}sD)SNB4bqK1*!_gHjVld(rQi2Fhb?64 z*=rO6m5mQFmr)@QaCvm_MCF(Vc=k?TLRqb0)2+-&oQbuKk6jht=#uG-dPv9Gk;p_V zUdl}NP-QJUxztZWfI&&AK)m)%xp%gLax@-kb_|T(Cf+ZEFmZdSPwf&e_)xSmm3S{lk%Ss8F4 zrj^s5g+-k<0|2;zb0=DVI#Qr^NCj+T$L2_38s|bX(uQKpLigwmz7Gq82?D>s$_&g`sPmWTu=NK%}D0TSDdZgoovidg2vz#DoezLfd z$o*0w$fShc2&?NZ^ZX!R0;z^DURJ4i?Ag&L%R&{P~R9SLvHWAog~;MF2_$3zEo@g+b`5+NYf=Dq)>lYNXvLT0rLg zgrTuEv}o-Us`OH6HkYzl^Ec{D6qESaD{z|qCpJ)2$jmXY-9P7%+1zc&dvnZ8NHGVV zXJ$4ic?{g#rO$m-_I4DxHnb}b4!vM`2X9yv-c~7RY;+Y1@!WbnR=P;>or*c8o$J|F z@_`Fq;7!x1g!@yO-Sxhyo3RHD9}=ru!*t|>H$05-;Pw&?BbUiRG#c3=I!m^O z>E5|{XxL0qUrFy758duRnUYvk)v*PSc)0n*K&bWnrD?hMiHSa5@Th4RIbEptli6&8 zYFs-udDJ*)xp9Wo#O1*vVS0Lj&*Rne)MD~BG6?dk%~L9F#4FdJNGQE@295{QyVdkba{|HP>w_>y*;O5Dhk~h_5-~cGK(%g2BO!HhpXw`2Ei=> zikX{FiLX`a{$SQ!g@y1QZ!-^@SYD7PM60kk#@)fmsU|3*LND(YAE>H#6svtVky_yX zrq@A7w@AJgO^t&!&ro1`ljgQ)RvxZQs6;>fPb3DDN zDm1Mx`p)ih8<-oHw-<`pZ*n$h@?*lP)pZFxDV;UFk=I3?{0&l3XCEmOWmC2NDa+`V z$cCOOhb24i7{D$xa`IFLDRE#++iR#IZBXkIo{95XU1s)PwTkOu3MV9%ZLOYtxepWU zr^?358gE=V2Gq}BwT=Ng26d7?e104u`(iCgfI|dfMlIYaTgzNAG>W zl36KO|?LI?#Md?B=KQtP0#H^o(=;6ac^fl$4Pr|*Y}yAhwyj{EhRoC&+4 z3aL(~MHtK%aU&Ek`w>b`a|sx~(gKsL-QE}but#Up1P@}*Tg(T(cuVVgjq}yxFFVdt zs+VhkPOS9|UnQoZ^buOV;Qh$_&?e-T)_c3a!j=d3m_$lq1NRoGC?e*f%E)%M*3Td5 zdS~vXV%S1FT)9^cm80r2R_6ybWU8#gBukZU4k(xkdeWQU0?Qw$^VpM%?qLXpN@hnF zR~nc~a6VbzC!*@-6~hy~%jJcVzV3xi}+i3V%Oo!V|mG>T2eJ=|ao#{93M@>i~@;X@Z zRu7e%h!=WHrc6G`st%+{FdgBK&3(&SOhp;rV8=m0e+r!Y5y$|)Mm>N}IJM7~?uYv^ z+gs8N$%xh>xg9H6?}R%VZFgYXxhulx)f9Bkw>B?+atUt-@j-O{#VbcxVB7bmaq*9^ zPq+4b2(KErn&*!J))jqC)*N5&tc`-{3xc=x_(ej-1C&Z?J|YuA0jl!pOn!su7`^mo zcaZqA#{h>4HO|cdQ&4-p!AC7LnL_ZSb$J?0IF@QZGEvYY16c41VykK{4n1Qx=itfOF*a|Kk`7xj!jbGi!oC{J0#`8SplsL*>;c z36+k|vWjbT5w#&Cs7)Wn!S$*njlCcAn%EBELRg)BoA1{(%d>nQEBTq4m9CmSrfB8b z3DK&l?RDmi`+oOnix290G9b7ApxgNO(GKqAFroln3Bzsa6@_?$o;}(2&}((#J?rzd zH=SLo+(md9()=4DOS>~CbnY&dlDi#SAO34XpbZv^xuq{h=FoN6GUDs%7c6T{rdN73V*-c7Ang>Hzw_u~~mxB;XmXjJl2E78xr1Rx5D zcHZl0VBv0uTA2l|%_8@@EUsOc_JfHM1&8}0{cz*ly2RaK_Jm`glvKKa z0`=9!x9e8nvF&{5Nr2mTP#Ufk_+>kar9xQhW45VdqRU9smPMAR=%iiFC@u2^OM=}K zqy^L<9NXRsXF=%H@^WInr9WbL(R_sPXM#+~s#rUmb??5~(& zM}`%a{4AoV@7^7W$`ZRv0&E7%gg30IwW%ab68wy{MnJvLyH~z&tBsROX)|nUw~IGw z7WaL5>!JEr%ob%c^@#Arn+7X9gS+2n=jwhqmC_}C^x5_BjT)Vo48~LDelMBvMvG?z zP@Hi!tEvpNv47g;_oZRH}RRdu9=syx4-wY73`;< zn`+N$oZaTpF!eFObb&_TEcy$OPDGiXe2(cH$mG6e;yERm>DW8tboj#zA}3)8 zb+g*2A;zeb{UK);Ohza}+M7lD!)cQ0#6GOGTZ*S2ftxDygp)*PGy|nY9-CZeKXu`X zXDDnogZdb-p7p;pTRPjM1!0&enf6XMblre4%_i*dej{EQ{<$N+G9z0vGvWVyZCjXSYeDQ`M8W1w0>$e z7g5oY&&r{yB0}nj!1G{{8_(yUj0h&|a$7p11BzA&lxyO|gtKg-)|%<_OiVFStfVZM z*SxpG$y3Bt7S*VK?VBz+YOWN%?as?aN?_A=9RoU-0|9U0_2MI2ye=UIi54cM=Up>! zW+Z?1MbSFknK)HmmC|)2nZ9)3cMEF;wf7yLej{_z6mz5fdIKY(yk3GoA?FgHrw|PD zuH!QhYI`U%-hkw`G2)ONDe%(K%-6_!PdfW2O(xjHA>*uy{8lE5F8S-9l9XNnM^Yqo z9Zw|0iIdPGEx%bP2a!&SkPgE#>pSkzM39HFexrcvezLqJO>ErHpyR9$%-&!jf&9`b z*T0@&Yv96}qQ}ovhX&nGxW-7YOir_hK<`>#*Jg=ZFbh(X#@w##x**G($_IH>G0C= zPkio7(R!*(t0->LXt+)+EVzdHaS*#3#7i--XLTHBD!(GsM&>nL(%tx+TTO%ZH0kYA zq&{}{bm)>9Ti{BM4ESYyWFVc=Bpt@T;gr!$B~{+!w3h_Kas z<{%1xQi1^|dbaXW=4v`QUj4nG@#%5P)zOOF&m|h)Cwwx_Xb3WauAWrvrh~9%RNz|SPmGtSGd2J)0nCJ^>8X%Nk8@w!s@PWuA8RlW8 z=p)Iv!lwOU?b6;{NJQf`!Qvpn7w&f-YBp1G$W!?H3H~cZlGsz=SJ64u;EwhG4QCuawJ`I z;e=MUrP-R%^~o>X9eSg^{vB^`7IUTF-E9SYj2n-E;8}`ez?L)zo~CX!M_%j1=$b{J zwt!#C7%i5$Pq`y(Ds=@W!UC1#wyOsaZ|5M~#BpRLZ(gByI~vtu;&&RaFtnN64%^;% zJ~gFLfp0LtE1VDAwjM|KU$>@aiTTmv^&=0fhp z*lSBoFLDd#1|<_ZS1l~QZl1fs#}2K4Z?GMy?h5m21(r30;CPafgAZYi-rVJcCHvnnO*t=23@_;Dd_1f!wNsVXb^?#F<{m~u}`c+isn%qB~s56GnP%xxx8 zT%m=jU6|dV!_e&o4?^uA6LFTMShbXe6XEE-)7NM-IjmL8&)v-3WZ*w|P?$A5tuz0E ziL8e>0xgrh1FCVeNWzV;7fU#D4 zLAdxFVjr}`=wr>L)VErT-p%tg7GJC%8)lU5BJw`kC?>sHKbLExV3?aDT4Ez%(M2Qy zAOYUQzRTx)VeL;n3$UOTIrZ-PVf>YxTidYnM?J+n#aUVf^#DhpvM4=Y-$da38xj7UK+rMp zyA%ZeS{J{e;Qjj)en(0FD)Zkw>F;XwFD#M#)8BH`@*lGOB4xI}Xz6b$|2<{=dxyU@ zfM4i<`mM$MR!gqGI}U%VrN0Or{P)J;4EhgJ_`4(h*J}NHru66+Wd5sJ@>`jI*)o2| zikJUmzWw5C_>E=!mDMjz6(NsK5<58~7@QwrjBnhyd*y10B@(mHZoqUlo?7 zw(G6->~M`oVqb(?B*%8zCfYKy#;dEpHt?)Tr{rmUFHNE>7GyjNA}8EiW;*@CCE295 z7PciJds+HB>N%XvkHc>aN@Lt|$Ky>5*_raMKe~b-j)Si8PqfEY$!*N(rP5RnUW&Dg zH``Wu1-G56!eRoi zSH!;gkG{@F@Yna{z2wn#>LVp1%e>g($+FDRCEub2C&_aFch6I;LCup=$1Aj?e zbDA)w`I@hL!*%_};7AvCG<)XUhrufyVrue*hf2=$zH%?x2FjM+IT?2U(CIT7U-PMO z;%+pw?0xS+&>-<;i12vBZi@@J{(!b(!}5Co*z(ejKF%)meYK?|?C=1(&Gr==tqyIC zv!5-!eQ~1>>lH>vmXIuZeL|(diHH)$jl70|UpX5<;KMhN8zf*XhFhxENgwWYa14CD z+y!SwaNzFQ?$zkL&CZNhY)z7r?o_w+=u_yUSLGev)bIB83U;Wvw@g|$y=LMn)rN`R z1})WZIifG*4md64wBgeC64F#L-q%QO@0|JUn~ICQ0tWtW{Rsk5Z?HQD(IL|DX#XtN zEiELT(gBCMmQZNF_r8IYcv5-}$Eu*_Q<{-QtEDqty|Wh^ZtodFM56CfB2LO(XtkA9 zE);Qr^<}>sN5m?1#@iY0XisHX9RrY&V_?p zAL>9yMhNSHhBJ7jK@WnW&QP7S>$@)2a#wokq=lNA*Cin~@t)3Asx)^Wb49)Ize5P| zQ}QDPdK4%$3HS_(!ATzjH2ZsfdtQ5Tc!L_xygfoXJHhHUrm+4)MQYy4q5GW5l1D68eBiWIf=P$F#rlP~ zt{|=uKJKlJz@wY1>1D@2TdHfjuKC*D3=10>Lh+sB)8{(L?kA}4d1*Ym_<1MABjI{9 zpW@xJ6jE$AwuKXJ4^k1q$?e_2R0gYvn&9q?IKgGhWc!6G5)@>D`89-+&Gbixv;~w( zo)k;d2zaNCe{eAI_l1yrI0X2}+F}AhODOZBBMunf8adR-y}r!)c3xX_-fM$yo}G5pPX#L$mOq6Lx#K_!*-UaV_4{<6{Oi@~2^oQT_B@lVj&r}tv9cFjCKsx{#ohn6pQ zGA)5YwIP+RkV@u@SQbSj5`bWQ6q zG^x4qH%VBFmYWA0sd8b{Qz*#cL|Orrcr47AihWS5_O$bv3|kMHNDeY!r5u zkSKEv=ar}oGjWeJYhfK2zivz&$=SV0wmYfBN1z0JF_RUkRlV|kxB5X{@DkWYyyOPU zJdz!uh}{o#pQZIqY~H2}zsaViULC@w6*w`+YM%<+1!;on0S_dJRcrY8^LbEEd89#G z8b!a7qM+yCl=eJ>%DEp08yfnzw)mZmC#aB5IlLEjL2bz5`29${<`E;Yzk$A@y0&IW zrswiTnIA{v6R{aUX;^tX`5TiHp_Orx#o!p&(?X66292Zn9C-D#!ee zr8iO|7JaWQ@k~4>QSbRTQBk|QUpf|ZGENq?AxohFco237JrYL>*U?y`>IK2C`9*cg z*^bGw-!erjC*rzp#Pz*W6->C0Q9&k}_DGcn*5+g{ftt%j(ekHVSgoPZ2at6!tMu6z zK1h?l%6F`KuuM9;x04|zq5gq{%(P>tugbOn{$Xbz50b@76%v=mfkPJ%yHR8+r0Lzj zhq{Q1YBs)X8KlMHaKBcZVKF8uH-P;`uKJV41tou9XEf~DVR};^P8bqWM`ve`!96#? z*kF)Y}*r^Tj}w2OdF+m!m0h;Dz+&0^tSWjmG%`w>HfjQ@FQ} z&ak^vkD9S-xo|=l5%G>pusD$N6OI0pR=}MoA0Qfy6Gi$jB$0pYPyGy?v5-1=MWi6d z>4Ij<{wL~6$hM6?Ya-D}C?`_gX|MFT3XE|}8Ilbr{`7{kH#4EI2Xv)Lfo}X>cx^vu z;_s)Pe*Lr}FT>G8yv0`{tzS1$66^QOb7W$;`{K%F>&}M*u*dV#{At}WNrG^9W-{YF zbKJ!b2pZgZ^en}l@sRSf6J!hj4HFrvMd;VT-7d3h9ynyhIPOE zL;Ku64h0Xcc6R&q!7DezCa8~u0~?fA67^Ou`BMhUrpQV?=>%afEkd>|G<+hSn%sQT zU;5oud)nm4v8IO}#_~PcGds)D9QNMKV&B?$+S$(H!-^X&A7pdd9y#YV1ozmwMs-s|F)2IW6R#Z_FL=$v50S?}JQzeM zI+_iQ`REOw_biZ1&-M0c*^jm2+7x#YeDKufS>fda&qra2%1>pjkE(i5tsI>R{4U$! z$g_qd=myjd{UNlLj+0D0214_Z#cq;^pqyjCUke|4Sns-obOYm55gy0DRohxpl)8e# zB}0GH=K1O_Du&PvMW4Wq5@?dQh;$!*9$zB1cB8)YW)sOIx50TMMD=|sCrL6K+Lg~1 z<5`s}q^~)&^qI6%d**UnvVM;mUq#L_9Rp|bm7VaCn|9-d1bSbMKalR8t{H*X{w6j^s3vpU=2b=fHJNx`^z zifIKKEnjV{tZ61v#FYOa37=7$d^LZOe3Zz8U?hYgyui)lE_m`IL;DfbDcKar$vbPt zj4Q%dIpdpO-_o7s@?fKl6ARP0@wm8SlKlKl;E0XLO+ubYXFCpl)?82}dmRIO{h)zk z;1tRROsWSxjkF`dXAW4UfoWa*CSrb_I<@VW%D1g-BP#2!Xa~T_k zVpsWYn~Qt|xPFi-6+`_-kfmqSj|>pX#Tf0rFesX*X2?ThSJC;piEnvYZrFhXxc zg&jAzGU2pVpmHgJD@TLujmQ_S>wS3d`vqS_V_W?1;iQQ)D<4-;jPA3c>1Ek%1grVL zZ&3sF38|0B)gAMLMM~IWy3}hKmNFt(A(!Sf_;{m_jENn0yfzGSyjh*$ZAI@)ywz4$ z&GJl0FpzRNwVl&r(inW*)|q@PcaKPXv_0qVJ-m-U9XB8O?Uf3b+{TmnY@8Dp8E>%c z`=~D`DvZFp^HLdP&&$s7G4cSuUYHa#sq~1}8+;4rHYcvU;T|b&X7~DNDOv4)pI9_r zr(09lU;gKdczApPMB>p6WTO&-5}V$}gujKc3zx)G&L1UF>>GRPKB?H31|K{a2vXD3 zQE-10_k|)>1C;cI2&4_E{a5yJLPzsiGyJ_p5b0bqJG26Ki`jJGb=W#7mt%h*VT_zm8Ym^IxJ8RMEL(a5`!&D;Y{7V)5)Fx%Et81D4(_*R z!YZGw3ysJ?*WXI57Co44l}m0g@lKHM7@Iol)(1cg~yS3hh9le9IJgm60rWF867>gaBh=_g5ctVr0=1YJ#-(BEj`kdUlzz4h>?$t5C zNfgCv4f_dsFG?#H?Bdnr3*HpYN;}m>3x-u23I%*r;ya@Ota(I_-*%iW?+rY<4dFR} zkJb2$MbaHfdZpBi`1q1CDLZPjyGO-K>9r9%FUhw4XWWn-xs8pv0;4swc5quXRZev4 zEOog&Lhr+^%(xSm3AvX@?{l$XvA#ETE$T%-Xb^f5;gN~Z#;^qw&v-xCcbMTH%J}Ns zc=x7Q2$z)=?p5r2PmvpdyZe9HV*<6$Vs3k9rtEnPpySB}a6>nZ4Eh5~Pn^Ffz2b=J z)g`$bN(=MIHhDZ*?#c|u0A3t9uZ`4;m^z#8P_5zbabkH4V^dc-+X@zc}zRb-j3g`jDKw#4w4_p1}W4(sP*luT&{%EVZj@Zu<&ccg=K4f zRQ5I2#VUSZeC-|!hqAk<9>hHMj8~YDe&bJ8l>fHzz}eC9VcvP+Vfpd>is+&gCC<8Q z5!rGa23lXTk!g9`qTk__pfGWdQwTFO&)LmZe|?z+p}Gz)`U>kOESTB`c%m@m!p9bj z5l=HA?folMy)BBOV6EuwZSRDV#I3u@+co%2U)WqUhkg1fglM*RdvnRH3BN23Z~XU8 zjfi}`f#Ci4;a77nH97?ErU$g4Mdn1iz-IH&uJzF5(H)f{hL9kU*aDcXi}A%zf7x;W z?9mQT=WKc7MZbP%385P)mbVbfPk(vcvw|LaO0*JGd%!ug>YHQv?9Y<{HP4pay0*}_ zF^#F5OSparZuS9T)2i7OjNa6|&h+@MrvKFqF9*i?`$qmI^Fg)%EpuJZ#%%3nl3XqQ zzIo5``S^i4QNP3JJ&B|oIwx@*wXutywR9Ckr8aixBl?M$TEL@5 z9nVs`hr-Yzn|KzY>2Xk_;U7*p1s>2u)iNV^cr(U{}RLJig{HA zDv+SAk-iop)b-IWsyOnLu*84tZT%`R?DIBURHeA>YXnJ9>g$Ymc;qbG8kOmtc*^8GgF=|7+fsfCQ0>uO zBycJxL0Hh==MSGh`}Gjy*IJSp7!8&Yyl&PH$U5DwF8(UMSY932-X*4g-q>LMb_}i` z)CQJj zYNBf|Yfy#B+jIGPf6@LL8`nnHm2;+s*zQNX-BNO-y;H26eE$X6>fKMF69tCZH)e^T z-3n4YWxDMymMOt3;pPx&Z@Mv#^ciw4^Lo@zNI*GMH(a#G*!w9C75sp)h?&iW^rdf{ zd&Oi;SA6v*vx|MZUV<)Q6S=Uxf*!4S;SAcTEJ}2(K3Hg0y7!_xukJ&+hthSnmPC;p z=L5ru8aUG(WpiT@N9a}KSH54CRU~i+nB~Sa=pd1Eb)uL!`8Cra!ZFyI(I4Esyd)Wr zZ$(jctZ$8him zLBjX(F`!>wF;q}6vFY^X!|i}`EM78^XA})j1ZbW1r*cZ}-COk$dGu%*roA=T&O-9r zXfR-6c(6YdlSSH<7vi%&<>8Aqr4EO|%kZa{>cp{%=%OgevqN|mEM2n-Url9nZvum) zQDtqc{5I8|$<)hgK5J~Oh>gy}5zTk#JE`jSnou?OVX9zK!mar#f^^RB9eNIoKmP!Q zTXI?_Rh@_)MW~ctI~bjbgq=i~;S3fSwB6yu|?F_*>) zH8pA_TpBKd%1&Ws!=bbdD&8r#NoC#MrJpM*?wa`)?iRQf)O~yx0A$7A>RzVd)%YT~ z%p3pXz|#5EJ%^#s=+(-C_^*YNz6bi#X=fG5UK_u`8s=mTkLTIf!>=O<9&V$_hemxo z0voeq`cH&$H}YIP??FN~S*|EdsPsH|sH`AJJd3)xAd{fNO1yEZX@k^yO`+^LqlE7`>Ftw{mnx@c_G4k z(#a^JyLs3h^cTT7viTHV(tG9LB{?LUgNUwsTJmCz1MwEN9uQjZIIWQamWrewYaN?b zpdokbh_7V}RTW(G(QMwZMot5#1{thMT!usL+G$J2#Ns=jc^X!e1n*WW2R(UvO!`KvFtL>4ze0Bm)^l*WDVJ32%uMxMhRr8<@n_EgHwxa&Kqz+@VVT{**^Gq;PZNfw)b5Lv`51426`IOd+Fyu9WF(46rNX z0(uQ2mCf{cj<)NhEEGL!X{>nTq{|HIuX4`ZDNv5oJ}@;eDqgv)Q}ljq!kx1rF!zq* zB{tCpx;)JhdmFf+hv0=n&Yllt)=5NhYhwb>1iJV!>;rX3br z#=Oo>Iu

E4|EVZEx-`z1Q9F4Sxy>se{hf^G~+$vX06SthDOrNA@RtJI&3y$ytO1 zuNr>pywbQzy`h=SW-D;jUqM@VzOShs*;0a&zKTmiYpb`MQYzMZKjCy~$Wov+UG@&k z?n@p*cY^!<7B5H5FAFJNCQU_z_(kpq8sG}HO} zx;ba6KyN>m)`u#%2F!Be8L?Z9+3iU1Qhl5j+vD-mUapH@zO^w|cEz>#D&*#943!8nY-(JWVDSqtF;(TqeK!Yxtc9=jK()&& z(r;I>c?~wxW=`wyM=^@Mr>_lp8>ZY7S0ZB>1_0%MCDWgEbpMO}KwX^nPi`HBaCAp{cp<{uvPQ{(-D21L8@xyLCvo{+6~y-vkNA-lFQU{QlE4{@3~&(g z!R)x>;MPr7*6%M_!oJ?0SwGR^5Z79_oOf08022FtZZUnvmx`*Ub>dZOHYsBZ$H2P0 z10pS%Tp>3lyp6F!btE{ev7X4GxLp#w9(r^xoh1tuNt8oK46WxE+z;}Vn9*`l3pNom zJFOAY8qX{e8Ds zBU=2WuMOu`R4!qb%0Gl>nVI_*F5D`sjA9OZ_?VX`W-&nZ3`f9Q4zi$AUPIlBXN4Z|LU0$*^_(kzS#5p(O~g^=g|m#w!o3-fi(7F;g8_!Lpg z?JdSa`&B@pZOgK#D!rU-T^qC22%+EfJBJ4i72zf4B`F~rL97$+w|(DHkDD$s!7kd{ z1m5nB^nG=UWYlxtFsi(zQ|qck61$eh<=H@2OdwK zi*9p$aR(KqBIGT4WMxo}6>5E9KoZSja^sn^vxUI}sdwqg#tH&TyPw%N+wP0#C+qrx zMJ1Z_iZnal)uOx>pTaiij;QezLqsqxr3DVgL*)W@i@*;1oZWJoAE&0?ueqM?7}*wT z>3++7LM}4DpuFU^)TP@O<1Mbg-%J_^>-s37|B5AfbaPj6YYa9&8mc0Jr>sC>J%)Ss z3eW^K-;=&qzugn8?BY}xdt5zn63xBBA!hoz&Ej=waua&QT57m6=%663(dQnejyFx=XU_`vCTQLh=ZN>b#wqZMi7EC+ z7B`E`Vv!sHY0;~o@XgTt{)V<6bW~@imSBLM9e3Ju!!#r$tEFJzO$MqFy%4=2tFLE; zqI7JlnKaZh#Gm!wugVI^6ZSg$RJXA|pd#&qi{wQ7ZrNEUD-i(borI^{$0uZL`i6&nsIl>((M7cRI}OCx{n=uB)1c z7y|SdO25gVrv65QJrnPb?OmI-oA{aRZytHWEqxP)aju6)i7}ni=DXi2rKKbapYG_7 zYqIYfb48Km$Y>k|?@11Am2CwKPR~p#={erno$kr2)V$Alh4J1-!+EyV52$3s!vaur zv*4~?te?&bQ5%wnp;QnPHyYW1Hq9n0?`VIeH;!pBS%0-IPyxGkFps)`R$BEI>rAE5 zJh~hmU*POzoormbW#c!hQY+i2R>5hYH4|02ZC0cO*I7P)qkvsljb7#LJ= z+NXuKMCV{*rXj+9Ewd$T>nm8NJ!i~Tq#qw%F5GBjmu72letp&WrF6*V()IcC@oFr$ zAA)YKv>4~mM&3x)(bl>aiCe)uZzer5dqWh1Y)J=X#SOIKY>w$x_5D2yy#=&WW^LG| zFSb#JRW{8DXW|~eiB8Dr8O;__dB)b8Y*&wLgrOEkTu8yPVm7Jd4`zafhG;`W4%z0G z@v)Mm2y3kJ19C;ogLHyjYbtR*y@iC$8aGEX;?;)dVtwFXhatGq#h0A6Hx8^QFm#uO zzwEwaT#gN+C|pMf9EtmNg&&#msPCl`vbT15o_HL6nICkIaTBT&r*OI5uy}WksR6eT zO8aanR?DTZbGSZwb0ztvdBZx)5l2ddQA`n1m2A){d9EkCo$(y=Ea3!@UTIxbf^3X> zr%;e4hpFaK>s8c|;j4>={DciXk@pEpFr08@+ZH=+w3$~3Cyl-7RQ><4_ugSqblJOT zBO(F zPdPJZ=1VhYX3pGuf6sI0FCMC^cI~R#d#|NXAWTve7B{4n7BTMa`YzO z*)i7~^pAUOI>~J!&17;|qq+<6?XS#qBT*gVjeR|vEWARL@yF`^h%8(T*`H3x{v~HeY_+V>vy)5 zdK%7!R-6^CLT`MIyWNpz?wFRlK4usp?2~NgEj3H~ZK~4!&{*qI&(-t-c~Qoxsc}GX z!5nW%;3XoV4m2*5?TQm&Ii)ijhkH$LR!5uH(H-SyIg-lXpHiY}c=C|jj@-&?wYONO zINE_tI1m7vu&X#BOiNoA8W13`3TSB*N>Yq}yD_Rm{#~ZvzS)GL*LP;7s>m|XEH~#$ zL3pNoPwly;4x^DE3AiO%x(RyfyYJq-aDil4yqlQ1UF3xJR33a&y{te<&{hQG!(aYP z$Y8fMDcxJE2ZLzRCy@odp=<DTNv_OJF3wa;Or@y`E;D!-6!c928J^E*spcTF7S)#WY&z-r%DP zuXN){pQ4`r0^%cIa+}y8>5OdxL(q1G)aPRLNp0rKDAH)_#y)Lq+E0qvWZfL>bG+x= z7Ox$~?ZpBGv_j%OuWk;Iz|($dcMkmBs&Ac{XP69Q-E2J}j!h#1Z_Wuu@si;8Zo+S( z(op-Y<6uK$x?6txwEfpgzIlz^_$C?F`zo5-FF{1-e}@PGNDkZ)bt5OpV0@zW=jx%a zt=EI`O|H9$%_OotX#&&L7Kbau#K*Ln8vNF0*Og-mg#8n-VZlrXX3RuDQ>0+^f9%E$ zN2b{Jc!D@UM{>bsBkQbWJ8adn4BEeWv0Q@#G*A9GQcxusR`QP|23fq+Om7ZI+#iw8 zyfNDRb;?KA-Aks8*>)uJIVeyU>|mhkr>ky!zR`1gRM^Pi7cAbxw~_vid;M+m*dN+}YvZ^(jo9w5}!C|WuC z*p2DUI@^_L4t~7a(YzPGhM`Yf`N~`SdlSK`Fzt~87jmPIqm`0+8+yYMi;R;~7Tp87 zG%-;zxh#;`e#)7d@(u!X`0bjJ$H;IfPchwV?Am-;C5Goc#Vej~VuA0^1uqDa^WP0t zG7+o-1w~j*SjXH7FSt zvYhXf8#2{Z<8<6c-hy}FbqNmupWk=61AB*{>I9tGIN)PAU@iX$osSMxZrUOWkyye@9SVdlxi1!y$qZ zx9nmc&K}E+>{pVo5&l+j-N2(RvG3@)4Ts`gkx0`)>gfbL;P@|^~!a->SX`3Tu&0-nmiz}l*$IBzWV;QO%-_Sp|m0wqS8)Jk;t0lJ=7VgYW8eXn-L6fFV} zs|Dtf<1^R~xO8;lEbEKcYpC+WNL8J=Eov@&7)FrkyM}|H=DJ^AT=|5uSiAX1%r0e0 z>q4B$#@uVeLRUw{p<{>m(DvcimoBO zv6Fw;TM=4!`r|9DcDpUl1T8bm)O*>5S59|}YGz7s*Sy{Sn*31Hp6-VG4K*)PeJ$7- zE{^kh0~f~mWfPRqXyLA_m6q-TCpi7IB_&$zW;+E`;^@e1DA`LPxl`IojrpaT2|2n( zu~+wou$PFhgG7KU?cW1l2cp6!g2eaj`^M0n&Vn_UNBm(X)zw$YcDL-gp0*bsK4}e! zd3OBr{n-M!CAmscZ5Aszi(#}PyOGD3*Y2)Ek#E-mcqq|nm)^@p^nG+B72b`RzJThaFUd| zj^{0$n+at@R`_f>&X-cJ9T#cO^Qbf*j~4j7cTAtXKWs5!%GgeT*wS!2ihOOo5*-RX z-)_tTzmhnyx)#m$toerM#{KQOI3Oy4$nl715mQx_-Z_qD>9*JSy_=aqz3f@ht2VR@ zZxWQ=E}pwBMcJ8QmvBcxchqs8)trS}#<}&q(s8aQG*_HP7y?a*elAkr1Qmthr!!^0|XyIUX z%rech!F&VHTLD7At|vp9Lm#7^y%Ucgz6dK^xpVCOl!ThKwle7`?!vRexs%}2e3+NV zRu;h86-k)Gjf0Q&fbskTba-|B2WXi62MD?V$X~==#{mtzd13h0Kv#E^1q~ z9Wqa;G(}H8%6;S==QkVD<6%?p?v#|w2|3g$3{u^lM1-!pXtm&!O8nL~fJRDtcu{$d zY0_bjUqI=7J#U8S4awo2v3~pzn*=H0L-yx!iZ|OxpKx^Dp#LHT^_yed+%qk9n)xfdfVZJl7 ztl{(EFjVFKG+n<`UW26z&*LAU3+i2-{53NfgkFw@YEvtxqy>dMuC%WPuv-QrmGt1C z_8%Z#fXf?zeOpzHE6#VSC{97nza5rxs2P~@6XHu;I`jHfvO;`t8;2!-ntLfFr@!B+ zM3b2CegxX+m2EoK_*0q)U4pZ}r#N}sWr@p07m%&|$ksr5v=2#}?FkS~Sq?$>QzmDSE#*o*skO3 z>97O?AU~^S;>Zdkz~v>hFzopl+3(`~un>S*d~=F4jj9wP#kuFT2`#cw)1TVdS_e4u zfm-fIWA-lRo0k)CXJ@{B*LK8vkOmYiI%x~K`~p;IXEtXhPDTZ|_(E@p17tN;ZW43g z((U_NfH#D`LPIBiiGr)FvO+@ zkk=gK`fB#4pkvu|O9eTO!Vcm*@z*U8Ileya>JK17k1jlUS|lwP-hZY43#F%%;~;TM z1or+keqc!+;i}VxjE1t4n1jAd2(`TQioOiLJTJg>*0x-C&O^(c(F#KE&ah1@0o|Sx zD?0ba<2;v+Gd)f$Ce_yd#AP?PIWlGg%?s~d*v#Fc`wXH`nW5dI{}4EE`@X{~0)mKy zljAjUd^mA{qOZx9x@8eaETs5Ujy&mhQjU( z@>{i5pluVcTqQ0Ky`!PKyH*{qnmA#RZ?trA&sM`=ki+VhedJxvHNzaM@j_Xh_JGyC zyhaXq0QztXj8}AqGVF7S${o?AzcX7`h9^*Q_U>My@;wYDKT#4>{3O}y3niatZ+f1A zM%?`TrmrG_F??<&yr_{8m|slxgIkMZRk|K7O41zU+B_K-TCY90{*_!Xfwk%A!z=uo z>UQnD^ZQacqb-QpY)(DoYv3U>;Q@IT8g4~B^Y$M<)m&@o<5u+6Wvt@5z3;;VSLQ`J6{oYCK8{6vrQ8C?@(K6RaPao;W|9qiw$b5FtOX`?9 z?JzdC@hm|e-Pha~#!1<-L${WKHgi0N>wjgH2mJY@>^=KL@-y4x_tiQ}{CaxzZia1d z!wt}~pOxubU^vm0SXI%vfYU{fa|cSKU5ek3**?47;`r!GEu<@C;iTy3QS8sejE*R9u>(^XAcUD_~f;)i) zMxOY;V~~}d2`GvB90o{;P1IjRf4hy7Z63U@;G5T;7XT*8CBwFpVhT96FMCt6ta5<` z60`*~64k*rKdl}_IH(Uv;YQE~7X23xFNO0TB~G;U)~_~pSi(rZn^)z zJMp($*8Rg<0=qW+sUL{dQZ1k#R)6Zp2{VLS?6|P^HzeB@u*|t>51$Qk0j&J`G$pP8 zJvM)v77C0e-UkpqhW7HWX@LY!ZR%>6jJ!(D@r6^t%J2Wd)_gkvoFv zXVL`BS0?-z8KDfqP)-okE>lVpjJ3g8Ii3k%h0!=3UPDyL0oZBl7ubS2@BfHw)E_3Z zD1n-0w4v@t>xx`n5o-7hop1j2k$dhVJociWf*2V9hRwGdS3zGdpR{98r}*$k1u z9TcrWK>c!nwmpdrAoI)DI;OZ=cDTzu3>vt|5X}BchtJicEDz-8(_$TJeKtrM7XzgR zJotv?igyr>R`KVvSKJ%T=@{jJRLFClT8uh4Z>9nOzDk==)Y8B~Zh~RMi&m0&#p9Zf zA72RQ4J{i|mp>)fTi3jLiJJb}7g}|&El`fd%g`9oVvRPE=4vVxYD@Gj1(5PLuUr;s z;GHhE0_>Bn)K459D>79-mZ+K99fPMTFe9rl8^H&$$w4B% z`OOpgTYG4?<_Y_%oQ*Wz)S}M!Q*BF?PeeX^at?5$Cd_7Q3!VN>QF{gjPaM~iF>~zP z+&ap#cWgIXcel*lIYe0dZCO`}5aOT_2<&-h=&^?6JuO_>f%U8LNYY6nc<~YVg99NA z@dULk@&lAF{(gvq;0H$uoT(;`LZLlb7w4Qmp=_il5@6xg6B);drk&>lTs;?eTyvdu zrS`}?2J7uywa;Fefu8Mi5gg$a3i= zEG!N>`x3gVe((c?=h{Z?M}DoVpr0>=jbxb^5GXf}1vZZ7)QrIGci4sqZgaGrP5zZ* z3}ze0F1@LqLUm$a<6%siOEf3QDQur!=4ijXX3TL(Tl+D8F3a^xkoO&(;1ANRy09ig zTnP3SE+-S*2H|K-3FQB})!fiZcP6pr^f#8~kFG2sWeF-zR~0lZ>fGausvbP-la+kG zHbZNq`H_SIzrqGS;w?j#;=+)8yyi#|U$CKl$|f+lbHhkL|zM1HjlNzU8nq zZ(Ch$E3hDdCH^IW^B73r=$r}^xx?|*4p*&i&EjinhVJ@)`c&~fwYZlZ%YK^IzlKl7 zXXdVO@F+*FHrwzrF;K5rC+EAAIt0MG-I;^RG;fu<+Ko*I-N$vbWC@|~7fVIV7qjl% zY>XB8`YEd9;7nHoz19`uU@sv(I;zsi3%FYLuR&wvqEcmth}FBGs_H7^rnKEt#9GXMn=NIfwew3xhiyO(|Z>Y-fbq|cpQ(@2x4d`J1J z>M4;$2i&^vO^J6RJ3Z#oQq{XvV%-IKx)I(kr!(Oxj=G6@EWkq2ZBUz)b2;8_r=^a-M za#_r0pG!>2%1+2HR0G6qtnE)_;)NPBolb`hnGT(c6%iEMHb~q*QPx~HQo~FVc#dWx zuz;}$boCRs(p>aaz&f6_s~wa}SD~8bfqZEn)S%8`5#D61F^URy|7P4pr7&45K3*~1 z$U=CC)5W_IkP6asqQOT?3 zOGtlu!wF+~%gn=Q#B4q zOAQz)S<;eb0)1v9&M|7A*DqemwS9ClG9%|odOn+X`^3|7W$WKYW*OjiB886UH-yW~ zGl2z>&6W8;#=&7&;e!E$>zcaMGG61u{qs5aK8BQw+w;yfY( zkhpc&lE%<2>ku^_ZmA08egCRt_w@9&Udo%BG2=dTd3gu&jiT@J^dNb|LsOau-$Y2% z)%G#d%m97ovj~v3d+GMFvFVV`gkwhqS254r3tZB{CFnj&&kl4lLZ@XZ?&1}*oF*{Fl)uMNr5hrogt-FaUxc!Rb%jhL-5kD3FWX0h%gze4~k`xY%7xYcPiRsB1cfP}~ zt7DPUL?WiCrL}RpQff_}wNZ}7!}x~B-n}y8!p{%ajNy$;_ykAiICsO9ncdeH>IcI}-z?f~Yx@80s-1!Syn0 ztc5OTu3US1t!9tvQINisx%bXF^EcGjdrUa?uITIaJd@XDB}tLK`%427s>dtMZsXOT zv&1nX5pbit+vZlPqbE+wbtwsmt9-?OnOaJv_N*4$7v9WwW|!YmI@i4%7G6B_#ULXp=~10od;&+jZN@e7kns*Nz-7)fy>!U0>FSDM}er*yHnEj_9ea zmC2$`7Ch@80>o0|#gKd!JP-ji$#Ce7y+AkwWR%Ki&!w*&9b5FStUXIQCJ2axA^@9g0qGwDMQsXL(uC-JMfHXjXi9r5&xB+K6U&`nkGAGg+8$Qi4 z=DK4;8Xa);vf`guzdI0I@wt>=_HIch%jM7GqC4u~Y`}Z&gMtvN;czJ6j@}0%=jeVi zfPu7!rX)T00b&h;Fu+xw0=Ay~-DxKPO0yaqIQb0^uwVg9>~v|8qB_+dTr5qH9s3ME zJ~u-T-0oHRpB>#lxR&zooCn-#6$`|WvT+^7oVCSI926>$2OPci`8Hut0; z$d@qcywVszMPdn^LIk2sA>74x8(>;h%ln+pG+SaMTM$x-V&G-%8M~mhA&y=yc0Nv2 z`B=veuk*AxhH~=&vpqrM7QuVgTF&Z9I{&CdSick-NYpKpEBJNta45xX5Pja?Pi#Q| z`0r_0$MR2%kCAO~v=b$c_xvgLiKBJ;2qq8W`Sw;}5@&S%x$VHidGKJc$}ht_I@2}; z3yBp+xAc(UjuCHeC%k@2#|K-x40<0)0G1C*qps14McHvW=4x)p>(s68Jm0!c54LpM2rl z=MU!j#d`67U1Ii!q48f2f&r|xNo$LB1*?MT4lk}uyH$8V8vJOz*d7ddiB`iqWl{Ib zEA(#kNR4`C)%Yf(ia{>@f4%xIl`ddm(x5{AeCn1h+GsX!y|R|~W%V7I5tq(xKrObd z@nD<$^IEKSS}>&ims#UhOatoqHi4sb0;fEGUjC@E?%jTY{iW+WXKtRmuNEAb+fTVX z_8Ey2wr+(nujiZN_PnRr2VcKT8QOBSsX^8A?crOcxo;rqbuN-tzc_)2Y&cJJ%V%pU zxDs0^ga>ESr7>i)*x`y)Esb(Z?!BYcfw?G>FGn^E^q&`s*~8kS9Xj0KZkQ@Z18@t$ zDjR+beZK#e%Io1qN`&?61i7RL(3z4Vx(?l)I)C{k&5g&$+@CT8FDe9RKO9)%L^RXm z&O=_kvwV;;Q`r|f(=wVUROBSgU=%U7Jw$sj$~a;d0Vq!^8DC3zPur zVC9(meVCF;fQFl{%V7{oHfu15*%v&Uzhw+?D3tZ8yh?lg>oyv){wWWoIrGx;2auln%n%pfu~eia6VO zz~HUPTDAq2t}2J6x(Kz%v?lbqVqSY>$Gv`O7px~6>~G4VbY53`-=V;QE>G#Qmo7JI zvkwjI=PQfWYC+J}8~Ef;Ir&2;RgWGD`aG$hta)oyv>Fv;evA9vB5zKj*Sc60_n47{ z5XZgM_a0M-SEkr39D^hNGC`~nZK3PvSZw(vbwcvsiziRRgttq$ir^JYSqlYyy@1U5 z{P5yUk)D3DZkg7>M;raCjWlc4OqJkp3s`ugl+_yB^tr*;mLkJ)lanv<8YAV6wU-47>kAcN)K;4`x!ZO<#&;Wo%}C{{>v&?f>l+p zPl-vYJeaTZV{P7?#beLvecoz6#PSM%pc`yG=kc|a{V{!1LJ5bR$*ptP0?pN&h-9$W zIzUp*V7|z^>z%Of7g~sD(uT0FdDqf5rp|&RK1W4gv5c;m?4P`l1*e zg_YG)ee20Q2eAixN~zdTmM(kgjYVyD{6%;W@wL2*q7ktW!oQZ%$WfN1ra99@;<)lo z$oIK#-peOdaa&fCC3L(%In~rBq*z)W6Ld{!FiIB%bCPedS%HfX`uhg6}t zW~<7Q8Qy6@=`JjJdG)qbXD+_GrGcdefp5&abWUYu*t|(H`EsO4q6Q)~;k`GAZ(Bg! ziRgR#53*~@e=a};JYfc%^0iZlr8sCoX@~07g3ec$5kmkbm3JLa#~Y0v1dxA1eArkl z^9u==FwHah-%VE@wGX!#NBipyzR-pAI8{TU`=iY+={^`ib`O=+VKnnkRSg9s~#Y%-0N;HlEr(;F*4nuMm%e#|a zH>M@YI_7`8<@0uuGl-+10W~x|f7FFaGq!wiyM!|PpeHbv?EWXhibRUYg;cqk3BAsM z0xf63tm?7GOnNS8^F6e0o(^46jq!n-W#iUw}-v7KUW2 z8_HpZ7cO(0)IVwR(L3d$AFIiS$+iUVYfsoj3O>I%JmK`S<I@IdiQI3%uj~U-tQSgsefh&N#Yjp3UCo(c4h1bUwq5D+uW5D|Ax2xjlg&H zxXII%px1FG6O+LS!i6*=K4@)GppCFmXrJcL?thmpX7Qx5-p9rmLBUO~%T1Ks7PF8LW39Y3owzJDg=OSgKR z@2tA2^&Dzb4IQFBkW-IK&INPfJ{}SUOh&wP3i24gL!}se$f~^`b<eB;yqk1AP6^oyngHmB4nWf~20AeXV_ZJocxECg3r5LMwcT zjI`uH9S!$G`t$pCe)i{A?ZE!@ zTz}A?-?j6r=lWfL{@l*r`a|KOg;ge|d8uesq|ZquGJa~uI|_$s9G;0eO;{z*B&K=+ zNs{knPyFTUjq`!XX z-+CthTq*89e>vG+IDy%SbaQ5&4PR?b%9K1$z6u3a&RtzqAk@AqPEQoM0b2W=J5Qgy z1Dk-(=BDtv^77vAN8o|iBmZcaLjs7|OxV}x)R;=?$QjDz@U|e1L6_x^ zBLv3yB1Rb532hNNvsX(Hvs>pC+ppadNYcLK`X175@TeeX`_X(*1=ah?DjhE(;ttH{ zmD{B3d^URhd;X?1Mt|dNV6ojpDUIxKWX5e`x?d1dB!N<7c_IC{NSdHwD_uIWgept^ z0V?@Oq_HM%JOwu!Bgsq!4S_2IKc?HydzD-AP+lE!%T3`Cx}qv`k!t9PpFLLc$p-P93pML7Q4y;=ze!*V^W^o*1_MP>vo3aQo zrWUrFmiX=Fv0lAw5^y55MM|PuLG~V%HuKq%hM@3a(;f`vnR}PQXeOrR`7?8?_nnK} ziYo)t>QH&9%U-8-jE7A2M3C3M;KDFg ze$}3r>pr{Jc^mhPxRiuwMd={r)yX9*i%}U1LJZu2ac*61ZD85iLHq%snlLapIoaY& zZ8_O45Pgb8%ZX@B8S*-{KZcElgo8rH;C^6l(T`ZQvLwNg@1meWgU^`MtXLh{M&<{Ls505 z)tdd@AVdylJNxlfHY5*G;&`ZBOdS0FeC%dr>^VXE37s7;J96=mn3`Ggfp@C)@w$7v{fvCRmI# z+Ot9$0GDrPo|RO=jXKBWko;Ol^ZI5_mwe~lS#;JZz!v+TYWm%y|K$w;e<(Xp=v#;* z#bPE>2aPmQs4koS!(YaO=G{oEm=jTeCX*?N*9tF<=r?M$#)%*>+2sGj{-@_B7NcCH zgMWb7Q4Rp$Y6jL-TcN7nC4J`at7u;L=YEWYHesc8y%^t>*0gmNj*@@q=wXX9jwPS3ilgSC7& zu%*K&$I}qHwIC7nzC{pft}lX9-0HRk6V|yh`K9>#jF*$=J!<$?_zpniJ5HeUWK`5N z7eQnRFelJa3-%7>ymc!$!n)FBFtvKjOB$zDue=f`S9eotf8YGJr~rPTdNf$ot;qyQ zPZGsh&Wl7hsuAvu$RW7b{2OyzB`sTH?H=EHPDb72(r~q8^WD*P+hp(s!C-TAUko5G zWQ}EkMJmIOb0HToj|=Zt*M@E$3cA+3;{Oz*^y(>?kCjXPjQ7XWpR?(-+OjU-%xA4B zCvY9J9qq8^wXv;RqFeb1x>qC(bpr2IzspiecUHPA;r`Rl592k8+q~*@hHXS1yQ?*e``wrCRXX`;Mf?i|>Oy$VHo6%@!Q*BJm!j^x)&e`I=XCy)E ziEk4ABhA;=9%3r*Kc@X?1DlpkXbDLJEePRC(H--sRus=Vf|taPYi!XOl=piaOZ&i| zf2O$dta_VnS5=XnT+x;4ZV$T6r*#*|wyet>TGg!EIDMkw96s~FXtIDkJFttrJSuM{ z7_+{j_Y#^^&cR~-3ZB!+aa8sh5Gcz7oF{r@z2^^*$Q&Mu7<=3B1C(pnPw5YU>jpF9 zZxroZV67a6mjPA>V5Fl#A_P?g7BC9g$ma;OAppJuk;wn~2rIBm&%Oa~uHiC`6r z|66|tc+P)h5r1#)9RFW2cVD4Su^~H{nOU|d2*<=p(!lUT_|}CQYKl#|3UwNAEyuzw z5q!}w*)rfJ&9r|Dx(Gk_;p517alP%FkQGhq#N#8&7XhEhV!I!^z4Uvc=yAzqV}#a^ zWv$k8S80XUBp}a5UH?B9B;-nkrvX&rY$>9y_wcH?T<4@kHe}T<=H>z z(=^spEkTT2nG&=(xG+3*$+=GX9GAe>#G1rE`U1E^5;Xv-3LgS^(wsdo1gjsDVOjnT zC;-@30N6w8#~#TdBcKOMj(`s^x=%JmU_H)qC%Mx=)5={Wm zJocOru*`?5{-`ivp+NsCK)UmE6TZo8M*#nBm<*Tb?ymNI*UIYVp04aG!T4_#DohEj zS-DxB>R|n#UgQT@YitzM9wQxnjdT{Bk#1+6|AMG+piG))#lP3m29w9#x|B`tN-QMN z!j*9%Z{ZZ^z7`a{M!qGgbxN|tK_sy4d+?J+M|vs0OFXBp>r*n5?Kc1b$7gszNXH@S zG#r*kkRF&c#7O1Yjj|d@cAG4z&I=?M2o}Ak*6eM`SfmD3pl6zurJLNIr74EmeF~!i zpK;`yzP*3t%Uhj@SbVRY=K&BRk-$KnSu55HV676YhoJZS&JZ*P;g%(R1*>U&FQg>5 zN=U8wU15S@SLSRrq9_DKRk`D%K>iJF-%%eGo2=R4N_FV{tdkf-sF!hBLa*f$ITMS} z8d?T(fjc%2W$d-cg?wmpgB^i)&Ww1-rK?O!Za{ohxv}LWWM)5V)aG4gc0=nFLj05l zWl8tSk^U(F;XsBr%3!oQ5FxyB1C{RXxxR)j_6Ou528!(TJ-Q+QO=tc~#Ps^c14x?R=>>OZB}1`9{9 z5JWIpW7uGv^cyo=!t#bTr!Z=6vT&rr`IB*rmV_+yONqiep;`)dN_POGsfQ0IJq#L6TF!U?pR>)kt_jIb;t3kEm}xJiAV_?~h3DbY zuUCJ#-d^Zt@0Huhs2OVTmPEPR#gmh{BIwcrx`TrP+0F$g(jX^5QHDlBc!%~UVP|p1 zlINbbAFDhYTC{wQH81<($F0Q>pG&iput?s1{zQHYn4aK{6UD&V?|>_DCXK>qM0CBi zR0Ca|OOYiIz&Lbx@gKdtZoPt#yzmJ0U4N^08^rMz2V1D#Tw6e}^#Gijp3+w6NnDFb z>K&a+2hShldt`QR`(?k<wRUA|=8NvQ?AKUAX#&s!HiYoX)-Upt;I% zi!Un=C~~%XeuC76;3PQC*Yz>L*>Qbm^x@ah5|OE;oT0im2FjME_&{_%!q-6zf)7F8 z)j`M9zk&|_(bE5K{dUwsq80@fEH?}daS8~E^iq<@_QB0RKu5w^j6XmhSi(t(d7#I? z3rHxx9)fohblmX50O?PMGZ^yjADX4#_wC4Xp>GqpAb{ccDJ73+#ZI&1!t`mab?Uwh z60|rM7BjWJ$3G3W%sE^9@*9Vkh@L!4t_btKg8+D|sRRyj+k{Q1V7)LuKq2zJnq##? zMVdKpZ(o?LG%5sL-UyYV6maiQWoqh#OLPte(jd_oxA7fPAlwBgTV6Z>8y76X1n6w*9fkoEkFD)cqNxT}OuQVEk&d7>= zKfFt)3I5sQ#{X_{KhPvU&f@|Q1H1r)t@kz-+z28fIqOxbA@Sh^RZA@3g&?iD5OYdu zKX1()1)ZA*ayEox95BeKI#-X2VSJ1CfQlhjVisy;tbGQj2h$pH@}vB3AOHIv|3{5S z4iN&g>8xcuYw}#_bN}M5MJ*SHxqO)~fy^rB1&NMDk}y8(!`|U7fWm7%d18{pP)JX8 zhd_&S2;5y)PI{yBH2I1#|Fj5n+O{g`G@41&^xQ?rW86f%79x&p87IxI^2WXl9FHISe><@E18f5SOYOy6>n8Aqw zWnI$|0P!s}m6R~`4@IcJDgg803UskINbm<}EIFT)YS~ZegSI4HUwH!EH2;Z_r6_|o z-~9|UAP&S&fF16OKvc4Y)?dtps}5K0TZ!nWEMt4I-l4)$PXv-K7|OEmol+CLwDuI( zTELU|l*C2g#!E-Q@AF}NUI>B(16?Amp>;v*|t3P`nx&f3+m9DW(uok;YNqC7nx=7sLnkBiMuFoTq|$PIw_c0;IYo3-UpHfAj!cX z2KYJjOw$bg_tgoWDN>G4!L;_f^|X?l^C>&}wQ+>Oud|>)5P7VJHXl#VAG_`FqxnP2 zzRAN`O{7a|R-ex3_KH4aQP=U&`;Ko(_W;-R)87IF`V@#<3%F%w^6KnLtB&01ESZlD z=#Os&X1gls1*=Q@RN=?hDDSC*js8C7{5LS}-w%Jlmk>=F(8V6oIYeC^P=Pr(7z_!) zSp!+ap=N9Bi4TC2^EjI>>vx{U(VYd7p*CPM?OK7WZ1pKe{u90VE1dpsG&c4)np2y{ z_?G>v7!dEiy zQn)zQu%*57c5a$>z(kc@Nl8k>c&90gW-qlx#pjt>MBC|sT(>ifY#EY^Cz09o)~OKs z-uaSM#9}{CR~5^(&K|o&Fg?mBDV>`NP><6nUL8x37@x#GHzyCIVuyzM`WDfNP)Oa@ zRl0oiwsb9!q$Ri3o0VuHTbL9KjA9Ys9mj4Uj@Oah#x3&zMC3WP-*hndE9mu4V5{4a zV#Dvj*1wIYzdBM7K!4-uGels?Mg{nXM+o_nkm|tN8huB_5NF|1D_d z4Z!TomWCl7(1&qWw?LV%V=>lZU5){qrxMz?=xj2{1XN%Y9*Uy&U#0+1_i9TQ-jBcu zML)y6m>r32i&lhL3%A@EI{5N%`O`BHTaS$W3$kFJcJ>TH-{JF9JAghnpvn=2Falhu zK_D?5^^;6TWBoHXlLH>i$oZ3 zg$|4)3qUddKW>e}N^9P2t1#f93tSD)u5BAAO2d3elBmi$DN=Wy0kBKt%w(&G^1jWk z1q3;+&;+-I*=Y4XK5Q*LJ5ijb;nq{~!Xd7GnyESElKze0Q>Q7aRW1P9sDI59Py%$; zDjXHY2&1dP~7L}xnT3b3UcfWi&s z<$(ez(I}F3{xj4j1z`Z-{bvUD%wV`#=hli8R)1l!~{_w(`gwt(sO~3X}4{K`KRc^hjB_{7#qJ$3L z7WsY4Xl8Dh@JD@KmVofUb#b{&w=g6WBt5@^ioZfM49m|Dsz=3@F1b$Uc}Mb0xYE3i zX^+zytpbenLO(3XV)iz# z4e`4jeM9y8h0(f=MYvk><)+?))D}Pd_Zf$&=oIXfWlhnCZx&)>hqcbd#}7OQr%8ho z%);xwZVt~ce3eibyNQ@R&r!BtD(3a6gM!yjRDO*rh~Pbw2bFOFT&)^UnyRdiI1~E%=O*X8;buLDVudV=Xv;Od1_;Q zbuD5pPlcK`@J30TU#Yoi@`yAsF zv2V6n$sFnOP75bWoiBx1C)Z0kYFAuH6eIzc_Hh%bYc5?Er&x-K2D`{f`ugjTkh}n2SL&vvZ;qxV z-`CqEcOJa0GSRV8orINtTMM(+OXAKe)IfwHd8Fz!3a`$gSle8f5BY?uzF)nYp!a26 zPiE@*=iXCFMW5WF)@=(rC}Xvi962PV@bXnNI+j+F1($9%jigo-9wr3!d*3<1sb4!h zehL0BN&qaPtkB5g6(}^J zwo_**x$idJC!T5QOGJ)!9Ro2P+m?{?9CLtp@}n&NxZtj{;QHz&j@16AAfw@3CGv=c zW3!WUa$8nAu4OyY0|z56}sh} z{Q%kYrUK@zte?{US3cjr;?skrN#CHmfP>`_i$Bjgb1cINI)3_}HPOE{eE*-Jga1(r zzeEziiP7`=-dWJRe|<6eF zHk8EoMP`>VfA`;q_~9%UXpmu9vMGI*+LBu$h_dyIACy9qKY+<=GmZ&)WfQ3LbKRqG zwU?i4q_T@AP+aw#5QU= z=H61$lHZ&78E2%#P`x9td6`_0?$%1wU8I}XyoFM`m9nwD(^p1aHKj`blEd|si8W`% z0WCJp{o8iL4X{l6t_FW6lT7UeKyQyP=9%e&lb+Ptc-E9tdVsbAv=8&Ts--wrbF9 z+@>>eYXAsM1R5~;ZPqjdg+aDv-cw=I0UO2_-i!3m<`=3UxE8Zy$5fuL2IVWv-)5tq z8?{c(UEz_NP}@HKN>rJ7HKR3Y%>knU+iQ8@5Z36GRGjqON+e0#Y0p_Xba-GwHNB*Y zqe)nkv}ub=2cucYt3d=4gZPC{=?r2AcEgs3Z;uIjTmZ-N5P$ltU(%Bwcg$&>=P%!d8IVPIBjK;G*y zYl*>q>&`4`;a7+p=fb@8lmtsF%kN@3%pnwdT0cNVywAADR*WAVJgS=F9B@yvN^?k> zJFk;+qZ{vy%8f1l^@M_`0UmG;qO)NYiM?cwSd*Lr1&03J^?+;-;GzhSZm!J3h<+A` zHLnJc4}t=<7~{)CYRd$$dTrZmy08EC0syfFwrE?-C-s4``jV)9J2|q0f6-6*R~9dz z>3^F-x(!_tOqs#ngppF@L4gK;_l5m6#Q9~%$RZZNoWRK533nw}F2pjJJ@8kH-2T+i zkZBXGb_!-95}YHL06GbTytewl*NOMlX=O|GhMs^>!f)VSW0e%-w^MUfO>U}P-ST5b zpcse0y2otl=GD*>6csz%jxY2@fN_j7glL1U*HeM>%a?%?vL4JNDN1%;i*&LYmq57&LuUs@f1lw&t+qe}5wkjoU3uOKp0R1}FVMM~Mis&emT>?~tI2^8_yIa$3Fy7N!}f1CqW}8@ z(f|Dh{~bl>Wpz|_YFifvE8=VrkPc;WdB`U0wN$Qsq0uC_q9$4d5L%ul~g^NwwiI^gF1&o~||3Y^^1x({MhFLpH;1vWIg>bBq; zQ~gH5`*=|_1eq)~Ove&?pN9oS-cC($V188T=}UB=u{RGKv_c{scKyvup&U<*$5bKG z-Fqm&)Qyt3Z&!!Fz4E8zqriMxu$hUFeozp>VW@wR@e))d!CgV*91a}0+(~R zU0>&vSqV3T+o4OxO)P@Wue%Y*(fO?#(ow2xg&Hhv30&B!%A|)b&uXSMFT#kl!chd!(jKLIl?PCa_8wA!eK^PfOetpS^U zJYJP*zi~5>@9tF0H9k_+Vn_6a$9Y?hch7Mg)}rPBf;1DwOQRR{j)eesTElXN8SaA1 zVVqd=x9WL|e3S?KHt*~!$i4Eo*KdI@`Wxg`k@NlATCEXRqBx(Sk}tI#ccNO-`pRSq zU$m{Ukl)o7t6 zZX2bwZ;uilR-6|g^M9@Cwg0$Y<%86Ih5u5`@Znmu#CwrtT{RBSk|*3UcEF=AS!kem z%-`#-Wr@PWBHL+_p7(7X0kw*Z2JC{693C~zZ$2TMa)%IG{=?@JQ+odS?_c6)1oS4$ zMBJ{OJzR0`G<#>^^_h99eILbrsJHApuHbJL)8l#I=zxWg!EdH}rHjNU*bv{a{mE3Z09m=Z4~971@qdcVbXvqiXV>BSVfKQNx1>8vr*B!?pl!Wj2P_ z>2ZO02%Srs&IL>m@jZj=dCmiFn1pOs0w6Bt83oSioJ?}(FHjgWDh$AnJJKR(0tX#X zs<1h(tz#;{-RZM_U@c)S~<2T3@xP2Nh zCJo0r$wp5H9eHx?Zrr;a42%Rtb(sakep6srgMXRtghnO9+4o^n>7>Yp(7VJ3VJW22 zh=%LOv=t`y2yV^6Oe_WKAQ6!lUyMD2gRK5mE+Md#$3S)gjFt)A%(6tSbvZDu|6_sr zU;Qt}RbW&&pPGtolRW^mY-$!p;2{(A|#pwTIh58>o z$biRGaBtQE%2Imb>NB4l9inscloy)IaZMQeB||kP<)K1+C^{Z3c)!RxyrSq<;^>nc zF-~QY0-1;(JUdGUjsfzz4Low5Rghz40YE1>V^u$2Yw5<#7hO>O`=Zj&sH>OxKtzxRCh|DY=8HC9r--pDr| zwBqexTwJ2)*}IRLI=D9cmK>ooDf`4q(GYL>{bKJ54?QS5d^QU&5WY_;jcRw3;b)>( zh8Kxs@0fqsaekgwS^s=c%m4X>o#w5AtH@EMn+7@y50!053s6#qhwPcfN;5bW11x=c z`Z&Cs3>3CkDxuTxwx6|pcV6U8>zYEE>{FZfyCk5u@NML<*mPjW%H=6l@1`9@QBP(% z-lfVJ5LIG57cpq6cFDP%MW-^(-|{3q&THqFvA?JZ@`NkLENYAjlvtEk+isVp#g_i8 z?KI}LC~cP=TGkg?dYAa`oap>Ekh_>)*t<};Wp&Kp#uq4^gz(Cjh zu4@1=&~*-eWB^PNS^tfA`{%CeKQ~2mXhWitONl8s@C$_EA~ejRW~4-HTxjM&gMmX{ zyjk8FCx@iCuA~e;t}s;N^;K22sTn$xDh|#~OKHLK z(VZ5Gm|g*?#(cf}MBTzbFhES0oU&=`hLXPuarxWc@&6xmOzSStr@r&cxlaHLinEIy z1FpS>Ee4swhRr(MW`y91DOxcXz6D)ppG#z%LNG4SAUG-QQU{x5qWg%VFtZe3Es@ z71gz862`G+YMxU?(6?##OzeS8j7b^iOJjx{s=EWK+{u1aL9G}Sn4#+iv_0Jdm?<)9 znF>FQpo-6j&F@K}mbQRWfm`n=Jpna{0gMW%uTl26U!W8sY6U{bL2{#p$^gA*HH2yf zS$6@*bS>1bBQV%{jfRP!!`lHPEL2M@+3FWayOVzBJBsM_50?XzkdQ+ZHyzj}{^3;( z)AN=vKUV=b7}&!^ApiE7;?N7|){_}eVJx`U;oz+l0pQIr=G7>R-dRV}4Waa#TGv zonm^moDSmrx^^1mGk;P`ntHj6Y`BmvIUm`?SzS|2*iPpvGbphX?$~hVv9*7zl7{PXb=SGg&8gol6s1CK&$s;S4 z24T!nGQ%}2JSV{5_~s>VE?}Ak1a&HnhLqF#Of0mrY2~;QO5RZEl3nPraKhNnR8zkXN)48?rh&d z)6WpA+XG9{7L%c(BZ`LFgRtR>rEqmaMd{a9O1?>$53pK_sjO-&fJ_Odovs2@C@nUR zyuZlCwLT|8ywrHP{Bx+O9KVxOSZ(rgzm|(~+J#Raw=Lx{FkTA&9bry1Z9_3@K%9us z>$I{8j1W1weJgjIDlIx2lU&`kT_j~fZ1cTfxU~JqCtJfSiw#E0Mv74Hq6DcN!*o`1 zwgQR_OT1@p*j_hyG7?vHS-k^E(N5^T7!z#9vh>vGIGdmj5e4XeeSwe-<&J*O4hu+> z2FwxHXm2#-q4~(Jg!+rJCcn z*Zwh3U#`pL{QfqV!72-k-!pv63}CMH~ci!@}tw7)&w_REXo--)Fc9L4YUhk6#6_=p|eyY6;^#q%Vb(lQ0^+b zAu$sV_~o>r_+~ax6_uD4AnM%GI0&LwR#jh7-7qHlDqeJ~MDDwd)Cmy(O_hbBbD({; zDOo&gBat6Q^nIyeycne3&W8?By!#}L7JThof&F>ci-=36?z{8qem}-SZi2DEE(U@6 zt<1GYHJ=6SLvu*EH|{nn3ZhlTTnz5*mhujt&hJ}S(t zWiCsY+u$tL40H4AlE0noI#196A)5b3{f`Pvi0Vg{BQ&T11?M6vd`~}T!^!ib9w~7z zm{S6m%y3$ZlrlOBWD>EknE)tnWEF$YXRgE1kER)R0csg1YoO%1`zZ+TbJZdR4oSaSh<9B2rBR9Uw1Uwty3N zBR^c2UVQ@Xyac)blYziU8vXM*XG%&Zb&pU1KCI1|@duAX+)+3)RI_#kuKEtGL_--- zH0+9yewKQa^dfyfXGW&wsAyYNp_F_XnSkhO5=C5?7)OPy4EUvDcV_F*R6k?Vu-JO} z_QQxpIF$>o*RqZ|-gJ)q!n7&1NZ+BK+^-~0W+|EQhc97!+A_skW(ttSCr2HGpgyPr1!MXbh%gC zO^nEaWGy}dt*ei3K33*dMH#04VoM1xC@%d*PLn^^rZW>~W>-3ena*VVaU~EPgJt$g z2)OQ)r_oBhV##^PsS|6$5j2Eq5d#MPfmL*KgA0|hy+f!o6t3! z7I3aD#G?++MT5shXTj8qz~m#_kqR(+L>7V5b>27c>-z$C)=3PNE0i9`iJPaN;s$sD zhtEOxtp3lv0OUvo`uFeP`X!)UTgaueTtlF6#(ujhwN5HGszz@$1%G~V7Q0|}wD-QP zL0YG5n5s@|*yCJuKnfv;n|>Q>Izvu#dp~XXAo@pjgvurLgu;*VvtL5InjFV@ zI@zReW8{G-ugBDut8ZPvMq}oL3Gi8JrjE6lc8oUPOAmQ=V7dR(0;rDJvUfXy(wOBt znS1d>87{Ej#qd#}X`H_6)0mur-L{0}$L4XKmrhA}#vGJ3_PED)F5q6psB$)C6i;u~ z`)CQJUoNOGYQr?Bf;$=C=(7SxqVf!<)GMk`01M{>i>pV)6j1dn%idvq> z$6KkqlV*9Snc;U?RuNTS{!{n9(}ULD%r|LBy?JOkrj0TWIQ_DecyC#c?>k9GDs1WA zc&VFk`-Pbb2)`1GmkvI$_zm>fbSaf?#ZnGrfROnNy@&S>6}Fb0>M^J!8`0R-%Y%y+ z(2bFK`O%Gb&Hv?Ih%E{0ggul+m4j($3`MijXW;-PY!L*jlpIp}k8rmCfF%CyO8C9v zD<_+?*iHc`ClBonbS=(tp3D#2@ST4+5d7cr5Mw*^`OcIc2|;$GT=84&6gok6vdwi$ zdr(r5>>EG(Q2j=~FOwX@l7vTb>+66auoa;i2);`5-U)xiKB?#$W}?hJ9T#Tp>n(3J z%P8{2GDrQ4D9AQagT?czpdM8YvRMuoQ$+xB;;%DUy80gg@c+a^0@dx1sc6!5vMJ#w zoE%m{y*L`17iJ?m95ZiqIqj;Pz|}*n|M&N7K3oika-Ps>7u(uho9KQYl#JZV+wXAK zHr0*5@2T9h%%;l-{|RV0f)RI+Fd@e5TljEAQWAEs+}!(k^+~vgDs#*bco@TrvW2rC z`;{=qk7t&-7jGO*ww0WJ=DRDKA|(98M8#(H$!}ZTqHlL^@_-n(03?KQkgny2ZWlU7 zz6Lb=&3xyIipVdDEhY`CMx{k+Yksa=RE)pNAfkuu)9|`}^)#8Ausgc2+KfF-mayTv z;%MQQa`*LTW>QLDHH+nFpw_9iETAuH4>*y!FfT~PFGAwHOQ&W!uxBbVeA|3Ee$=Ie zg|%lUdE4-B#rQva@)Pb+uD)EYD)z|$6bPJb4YCwdWWO3Vw294u(+_1#h@~1?l2OKA z3{1Z?RH#xUJM?wFhmCQ#oocmV0UZ~FE=GiM-w6f0)vcji=(ESoK1ppgus@&BbqPyY z3f6e<86kNX=fGG)ZpB%79FSg;j{`+6PgCN}q2@Uq64vk!vc# z7{mI8L3zs)hh_Im(Iq=^M<@vDJqTobb)>jIGG)=Q!u(YKp#S_;AKm&CDYp~4Fa?G# z>N5gs$pRWZjPQb-v|NORRhIB7#Jb+D9DU!|_$5v0g!)@+H$-1yAu;Ckl(6Jg!O~_7 zH(9uSPA*A9jH0m7#2huyqmQcgc4cnPJ>k*QbVJ9Z&)ht+ndhYEn9(uF7uj&C4z7fI z@%|&>A#cIUJ9J?u?unwnX9V7@jPp|p)?erOouB{SrJ|x?1b&Aw8WAH@Z#^hPTm|Ba zEZb1*4PqZ1swSEQg5>q+6X@Hizxlr!;w&@&!138;a6ls3H05hG~hC1U7 zeQ8QRNmwagSi_H$ER~s=C|JHty64HWi?4B&UXru-D0p}@^c$6z2#%zmLKqVPV`9uX zgfuZ}wmJsk>0Z2cG?A36*#9DASI21d_DS}aM#ZZcSy}-s!JgChVR%}mhA$q7bHtVd zHFg_>iHx?uJR0`y=WiWaFQ1MqYnpiKnx^;=bm}|@Pq42W!w!ICD3*ZR0(A3>$*6_A z9K!mXK~mV`u?oM6ne?#-T}kiJ??d`d=x^&;fcB<=1r7KD0p}wbIXJ)nqZIXntAb&x zpTa@!z0*E}JC|u7kVxun!XA)T+as%cIWwDAi+Oqp#Q1In9?FPg((v2 z>McAmTtrX77MnhwZW2YEB!@hxDQf~P zb(HQFa$I4M;K%GoP6Yqa8u*TAsryBcP-7{uL_OlUXrj;HdA#7B z{9J(b#oNNq;6X3AB;*L0m?1c|g9W)yAjk(0Y{LhYxLRh(f0Ru|zdtZT7W5 zyVMVkYNGGIIZWH;h7|^VbGA0ifxrIX{47^hCeI`1lnYuJ zIG!Gpw+PDgOMs=)WM`~}lYtvQot=%Rn>4HCE$|kf$k;vbyHa5p{t83CBj4k5C{1M?y|{z+atlc=M_>O=Nv5Q7B{{vkh<2H@>tQzb@<5p8GcHl;gklN3z+P=@b>=>s3=QfGmocNp-)cpR3F?7@ z2VUH&Q>e7*FVPwCwS;(IS5Q8a+ul$hb?>8>*W;cZ-D}6(t_l9&VFL=!|Bt3G{@&|u z`VX%=&Oc-3nLxX29Q3hyW`t@%cb?9r{GzvCzQFRY#-4O*zZD)o>U(?pVZpJ-ne(tP zoIoe1;eEtstiCd;R?Jn;epW~jd7*h2bFraeWR&Pc80{mXqaa*B!xp8s#0NkCCCCew z-z}%5(@J7|ZibzmZN4xY=Xqf#hM_%3#x&Vp)sR~|`opO2BqmUFSf(>JuG4lZOy}PF zG)K!jaH#V4pFv1G={dc-iQWbUM&K*xgJR#B#z}0#gVPACyla2CkFxVBZ>Qq=2z`bW zPipE-UMHdduW+Ga=fJCE8)ooA2JV|YLD1XJ$`26R!(x3G$>7bq&G~O9I}$T z*?WRfMYs}yG;TAKT69TbE|7jcX{b1|;X6my8+x*P^i3i6-b9e;z9)FmHwK`-A5v9_ zur|o%{1zKox58-_zM1KpL_(LhjQ9tVs`_W#>m55hLq{6PdYQL02Sk9R_b?oqEbiKY z;d5{!V_KkP;OPBEp@o_#is9I{%#%c$vu?%7$K+T}o)^lB`r+rGGc|Zb=@;Nw_r(C7 zc&@A$dITbhwW7~GflmJ-h7j*>IzfUzq4-Y8uOuk@XFkl38+FxsU!DV-?oD2d z@y|eC2kcm(IB7y$#A3E13BHw~G-I@@(N8zc;Araq#$Q&?`*lE#DN<7DrPB!eu53H- zw%`#|b$l--410{u>xAJ#7}^TAdem84yChs_Ew)oxN_g_}OxyXpQ)rbR;k-3J4X60T zt*qWP4pO z>;shPxYahW<^4%oUEIhX~ z8Wcg=pylmLURbo^CkYZC9&uXZ-40?!%Bm&SbxQj_c-G!8ZFJrbaMIJfwPZEuKug0u zca4BlB+fYm*e2~nAk1^K6ArY;ZeDrGayC4@n)d$%U7JFRp7p-W; zf50ka+xdt(`#9iz0mS{wbc&vN^i7H^3H`01kzgpBitR~M*>$gWpiB+&NW{uIOFn7= zzM@$46b8Q?Ptiw6R*>WK9HCR3ien_@Y1iqdk>{(iO}NA-F0=f)_+O+UmYwll4qy{@GFE)0?=ysSnvbpMnA(cGxG%r%x|BNGLB7LV^n2% zb9ndhtfIlxa{o}QH!xv5BMwU9BM;*-Xvi_kPUx9WB{GV4 zi)YPUtE$X6dtz-Ql;Y%n3Oyro;qUmoT6tp?zbE?m>fZ<<$Q1{;5dFeDeHL)kVs!U? ziV)5t2PvMsaIR2{lRwa2q<1WK1Pg)PTJu``Sgw8UDMt$A8KqDIU9!lcwD6dKRHVn0$#8^n;m9Y$^(xH5J#{mJeR6c&0TF z_wm&3R7SVRV+F^qt}gIf`T(FTGV~Xv$O8C*1Aii)y9kp13uZ-YQa&3`>lz3^;FGf0 z8mUm+7Puu%vpRw*pELEt!Zyv{hm`DAd{?*W#eI-w5a76(4FGUV)`F{FQ#GZK6O zvEeipXCJ=VB$7eC-&h~wUU8?!I6DzKUc+JnBB2SWfm z;o|-xasptvlhV%jF0HAyu;RDlnYbf~8>dc2sW58lYp{re_Uit^7XCB(aCp$s2HpBW z_>JpA#`DTm3AHc^*LVDA=pi47g8W-WLl!1wlYJd=Y%@*^fQ8y#P1B$g-U?+9*fnG= z(o|U=rIg4R^L8bg_&qQUM49`${|KWqDKAR9wqUri_O2Wb7H21FJ#(H-H+@v-eQaS+ z#%OkPEtoOoTWQ>>V+6Ka_e@P>P8VE4XCv9@Ll7n{HHROCR1Kn4<@c7~57eIbU2Lil zE9^a`9?q(ZOHxsJqWVc+r_*yJ>}J1}P~^?e|K2)e0xcsiBJPv@3F+EE1I3Q8%6IUQ ziz%p0IY_{4xAXM&%x}I=+%mjGx^&OTG*L^NmV^|4h~ZQxyB(m@2t{MMa(sBY7(tw4d(9q2Lf(#^y>DquMStgP%ed(J!e^BfMax0B zQ;eKEq~}6$(cK8)1v9hlIF)RkG$O;=LsbB~9rXtjm>jlRMk&vazI@#N=kMun96YPj z;vWI3YzSk5OeG)#Xjf`+R<^7Nbo4v^-lpeP>e^bf70mAL%~TyjmGVBopLtM#9D-tS z0?ud+-#`+Yi+XV!-uko4EKxJuZ2Fe`z--|fc2hTXX)N=``^9Pd(J_wyEpq*r!B_48 zCdYiTue{herWkTH>q`c*W2q7J8ZAbZM^hTeI~8$p;@R*XsST8^oD~@`jc?ujQ15 zAkX%j?bBT+k&ES84+07vi``?rH@pX5{YD^Cud|3gx@F`U zW3lS*P+C2xNZC2FdYP1=*FGmO(jXRyPoKTZeSLJ(ZtRoHFyGbW;S2tdXCa+T z#XiB7F*X{e5Ro^v(goY2ZTE`wO<$ItD@6%+Kyi&wfSEdym_OA1BPq7(D8qG`dUueP z8%B3+27v!zKYmgbEkjFq!=3Eeg5s)}^1j?7a_->u;1m1!?-%vhYvbfZV){TgMC1I< z%C|lUX?X7lD+TyyM&m-fg zF1&&QYWikYUSeN+53L z%1wT_i?`eOWyx*_FQd-8ZCYlHfzb8N@=4VR1VGyz2WH?C{fD8OJ|n0%V{cRUGW_f> zcOYzuB6A#M_@cDOx4!aDyT@U;HyuQM`IGl_th*#ePT(;ql>xOv=T!PSk}f+3))(a^ zmsk!=sL0>iIUQPX*VQTL2)-PM$xJcT5ON|VBcH{`d%KU>#q0AN_^KRR$u7SUvzFB` z?rQz{b5+{P%8E`n&jlbwT3A3rBF;SWJV;e!!tjkbjP7#B_Qgx+zTb5kP@2K<~@s zBjm?viLV7HmD*}${hWH(ccUz#A!+P2!+P<_qe4@>qgUBVEyzaf*avtF|E1bEXC*krydYCn*UMr0Vj@W~pbx3LACtgZ% zE^rKZbK2F{(UZq7FE*U%OXkT>UemreFl@l$*AyE=@*(WV!xSa`LPO6FexAn32e;x} z8_ucrvB{SC^_N+Xp0_C&v=f_u`OH@#vp`-FR~2SU(M2`uYM>V3{LjZLFku^_nJM-f zP9p66uHqqz5`HZ`E&#Ig>8)l#H$~zUFbMe#l2zIE6p5nGOe>at#Y{WlxVN|$zNS5x zC@)ZqYqE~H_)yA~qrMy6Fmg?!tW0B3QB&8cnw6HTYHas;T)`VkFcFhOFc2z23;jmM zv67NK{}zcTOtKIoxzWBLMUeg6Vw-db(aw8^Mc41DcB+X=C(X@~9bJ-@Z+TFWbmybT z!TZ#WU!dxP_EaP{av-fgGf6Vmwc}0`|L8rbRJKgE%pS0Y_7{+z?$iCV$$SF)XOB3K z`v6L5b*dD4p?Yh0fi2=RljwHp0!;p{ru`+E*1F-gPw z&`wEg3<=Km$&SpN=TJ?Tc6ucoE^S`M)G~0+oMA51gAtpc%SA1A%TL3IDPzmhfUH;( za%kQkdveoRKe2n{vI2xN!KMDhi|0ql`Pnl+et&+hOy*FjtWwvxkf%z}5`>HV`srs&N2-=Gf?=30-Dx9)3HoUEkU!mP`9l4Rr6|K%kxf}%+hd89DH&7Hc_lpKe-^jf{42vPf0XZtfi^i?C6zoN#I2}j2y%*VdpP9VYb9HoCrSWma+B{|P z>5^Yo7jOgJ8=t%vD*;Qh^ign+@wOjHi)h0}V!xDbju-i@l46N28FweA5`c z*gw4+(}1S;gaq!DUj0V>M*9GyduXYXBT1!9RDrmS79k--$I%NnMT1I`t%zm8o;Doa z#2|T9$eY515UwM)Qf{nOEB_FwgNH2%T^w6}a%Uy1Iqa0CL7z>RMiOhF5qJ}Dll^CH z#UD_#lHSeu+@x`K20J?y5bK9D5Vek0rFHocHHg?}F&HvofQi7IuH+9A^nTi0-88;! zYb=&LE2w<~BrMFHJRtfo7@0HlcaZi!UgJNf$^Nqt*MO6%$;{9#W&(1fB|s^w4i%|T zR0BAE&eoqbE5)&J$+K#Isn64!mEs4xoCNpP`ECLH1=I92^b~y*dYWpAV>{=#e!f0K zfhe7X{|M@5E&@znRE)X@S#&^vU@_v=ACWhllR$fV9}sc^ET+bFk_%uNmx%=y5Adge zfO^~_1W2zrwhv_MoQ0DAuv}t0Lrcf5iIoCONdaKR3e3~^Mf{Oxb4?QkY)vpa;K0<{ z2DRHcItA>CWQo7n^cj4kLjLuM>-rMUx+ z_Rm%=&g*gduJDNYsX(P|ec31(OcDg#6AT>@aQxW` zEn&a6kE@CnH(HT15$>GG&4pd%2Au#Em%q;)WT8weU6;wk& zQ@JAA1FXKh-J&`#oK(C#eCO=;<2027ZOw$s;PJ@)0{he`laE2MH&>Z}@QVK(hQ{s0 z((uF%3tlf4Q2P za_6~i?kf}*7@`rx%^wcxL$1IhV|_k8@Nu=Lc~fbyK5~vs+f(*T*JQo4vZ>Q4zNo?Lmcn!B~eysV{Et%7~;`b5^rh4K2?T-fSdFH?S4lABzD{lA} zlgFm0@wL9kg}(~m%<`_4vd|QEAw0-l>}d@fqVOGZe9+V3?-o6y-={CTURt#I1(JT? zdpu4XNeL6J3n@v_B=kLNNhuGQv7ez*>jNARB}mSe`yIV!^>=l18gG7RJeZ?&0Q||c zL6FGz%!AHv7piC&r0|TTcDxREsQ1e_1*5_!say~Us2Rq4b*fFI58$bLig+IDT%ICP zZrfcAW0C|}l@Q$yFP9DV-r+(>5|CvDvnYqdgX$x}lD6HgF(CQw=9>BE!on+lAl8sN zId6Nu{>xBo*K+ULy-cAiZ;S!srzuO)Fzqe!bT2{Of|wHCAc;et*wEsN-Xc}|80q9+ z_sAX6cm6$6PV%Of*$e&#JYV<&Ulu}5ROYZw)#y1WSnpeHy9i)xwEI)jhNMgEz!LdM z9t2nv^{zW9_KuTk@oJ6C_D@>cqNQ!Ju8*CgptXsQSPD=HEm%i}gqiCIs_-QgMa1M{ z4;>Hy!+h)q*^+E;$oBGy{Mz`GcdOCK`5w7DLvqKhx z$0_dc(g%jkohBg;0F?wctr^-}@%4=_xhHNH`>-5ROA6(C5oU&uy^6n7He^(kZ>5to zmULS)claBz^f=i7C{98`5l&-tF>+6kNh=Qe(uJe<)^Kj(^Azy2@~b60%>4S5wl!}3 z7mMR>6kpXBp7Su1c*ah~0QC`z`)Fw2hFT)|T<;MTiQi#kCEK( zn>sRjry_3ptexr;j^DXB{KW(&(=V&4cYi10PBweAn*+(Yc##*V3&wTpK(x1c_cg@L5eLtI4Pp!(|5ml zIkxXb*EO-XXRQ#k`8j%{ML}dMkE%IPjnf6A2@w1NfU8ITvu|askwWA3P@ar6G%YW7HP)oR!jV*GXPv zZX%fUgfJMT;Y=uPt(z$I2RP`tIrHMgn;g6Mjm&C?>GlWLz*~lNC7(N7EXxEXvN4rW&>7W=u~wZ2z9+ap!^tE9}m`muT-@ zx#np-p!z&UCejn}m?_Ves$}JBJBTY^*3w?W+gajkf!mUiv+XM-{@eJIC-}REanDAy zVlw@tA1AD@JdNTp>K2J>e!#_&puYf8pEw2qBb$v;Ed#W?CN*ln=tAnAApj%O6S&o2Te(T=mmgKwsUld(>e)4$fz91&gC^_ zc9%+EZfAlW<4Qo-#AH8eFfOcJ(>2I4FuN3V_)u5x>pbBu^Ea(ah-YL;Ahqw=!SuVM z{F!}zhj2U>rvaCHS?6sLUzw@3!cz;AF8tqL=Ji{@f4&A?CzSnuW%(_zW*Vo#!BBRb} zypGZrNwD%0e_*oCsc&!<*ji1^o1H20vx6Mg+YNVvr1HmKDy>0pSjb`84v!o5LCNA@ z&}2Y{SDv`wA|(BW@jL2+87eqGjBLMlW-9XqcM33)+dJdf~fj z#{(?ng>>TKE{42$-EdV-mg0NRJ>i&d=-dzX<|Z7CPTpp+r`01Y?YL= zt2J(hQbZHGLujv?nY}nu8)z*qcSZn_-q1ju>U>sn|HI0g#|ro$c0ixrtchCc^#X{D zjbhZ=;WKx5`&rnlWnhIRo=g5gfs!+6Pwr~W$<>6-3w#kxO1(?wM)i%`Xeq9#)hD2u z%@!L`Q$Jf1uIg8z+esa6U{8A=F3j_#pqn{hBKB&2Es6W(wn8tK*$!Uc18II!Y*7}Sd4{0Wh$281Tw?LpKCUwhm42MvsG9J+F5*ZjLdodgtvc?3D+_sQ zjd>Hyus2DS-XV*FX?&(YXI1B7iwq0v$QqkwPge&~lpF0xAQ*$fIQr_;`X*!?}LG01VPAOTP)IIL{qW zxftoOl0a8yVvgF?0kDT^>_Qe~&+(BUFwFX;7eLehu=EIP1kelMXU;9c4*hI0oEi_{ zA^*iMkwFT|30%AG#qN7rTEp2~p%BX*_SP#w?kr0~$JKYHMgjU8mo7Ukiw_Hr#rbz0 z!NMCjnQv5!!{iJ!qIgO_x+?nNCxWk_Tp%Da6qmAD5477S$eh>_?5DMbjV(nsn6 z_sQwMt>i;aQXO!j7SK{qo6@DIwIa1B46|Ym^+u4~!~DEI!8Tq#Y$)Qw;_3Mr`q|-q z>4k8LZH%$YvB#?c@0Q$kzNNuDW5=*|=qbEO;y@D_vS6Pm8l4yF%r6I>lL)TH!_us} zy%vrH_766#DALb<)4?o!3U0D-alP;~0<4Pfb*~6yVM6R~%i6%o5`8JhkloycMas<^ zwoD!xB_@CcEs)*?-Mfk1#TKG=4TVOm+V=y3Lg|7#l=A?M zd+`^jhYlcU-3o#FzHCLKn=nTT1HEhF&eMnmDvR0U!@7=aacZUqcmnMxgWJ6PDLglS z6GoAo64qmD4q^`#z^Jaip}Q=hb7lN}A!YgB)}*j@Y*W3kl))-8?}9TeChcrR&8jNH zP;gMEf@g2-qaMMy)|&+I3k_&B1>CI0nqIz5OEe4@Y*>vglpBOvq*vXGvJ?(@@h5-_ z?t63&m>hcR1R)ui0s%aaKc&G>tu7+vH<^ZjCivf!0=uwjJl!y25lxGlj-W?A1Fc|f z(T7R^CkteSI|4hV=(G5Z5jx+V=AG26zln?fW$cLIr(qW(oyijS+#M4(vN}&wTF=e& zk;NZEtVdL}|Ldfs&&dVEK^(N%P*V{SiB$SL+^{@*yqr^d46oAXrpu?tyYTrT>yi;T zJRiuS##Mb9K(G_i7gM67$D>3iGO}{6T@duaDEqEt4KV^6Ps16 zhJTP=5{j3lpM+8*fZgnW?Rh{AGfFgxB{DDeP;ZPh36M*h%bk+%(&`4=&OvPbw=eQ* zkUQ?&&x(anZo*U-7{3BdJY zm!Jji^M}rFzJ8s`7-Kk^zX}mXj%ZyWM-!_Tcafd*C-*;ASQ_U`AH?pzgQmLLQS_^s z!=&<5FNo-N_A8&_S8%vA=zUT<3F@kd>`aL+FlIMhf0Ka)Dn|HdV|~Wi6e=f{WK&7# zb&w}}nl&^z*v2>+Z*Ve?+e?{T{b(}p;lkj_I$Gg!y8CxxWqM<;`6FJvT75uiX^rHJ<^eOAEHYI-TkqM zeF4=wgq%eV`~tCKolBVzzYW7$(4kt~lT_~f>nP2n&8?NWJMY(h-=4fj6!GCWEc{sJ z;#>%Na!qp|7m;K-cnPz=(eu8%?X^5}*!n`Ig@f3}H< zfD8o`H!T*hkOnyi>59g4BkJl7gewnDD9eFdd{ph8+oncpr2WhnQ&rWYKj*F14#r2?KCjxu)L~Lco1uW3 zk3(-JD--X`PhS7}6?T@-*(mB6E%3(*8cyU6M=fN{R~!=q4w$%N4Ov2G%jD+@i}tw; zg?oxy_1$Ou$^_eXlcyS9nuM!zbtP!=FtuE189vDIhi(k5zmp?lW(S7R@b#U&pPIfV zWLPeOoffMBCrsYNCet+~n*(rZyJpGCwV4KvOUc?r2iJTr+)tLszF%(R-hAKNsi0|X zBk62J%dNCE%+g(2aK)BZFUf=u+n*8JzzOd!cxctlb*sc(yeaed-7{AuyQibn zK$r3S-k9E9F)4lP_mh~FAhw73e!#8+0ruNwBfw_a3q}#Fz$`0Xy7XtutK;kFgmO}h z-LEiJnv|W3?oVF3CE?g{>jz*qP)!m7+yRO) zgfMdb&6-h&G=!9$dMj*KiRGiO6qy{ zlrgRan}#rG*RAr}xbCRCPJ6d9I9~H4I`_r><+{=Bs7sEBPP2o;?ei`+!SdfJrmJU= zbH$FH;y)EW%_h`W&a9=FemgR0d?!#6=w(R_OSuD3HFC}>K`zjv@r^MY(4_YHK!7R& z2nf#?@svdMMq$WoRQ5<*(MH7;OSoB2*20@NpTVOqOs}_6TYnENc-c1m&bShOm z6ROL?oUtbCE*IT#M>@*(rv-3_r{AejPWkd7QRasAsp+g34$qj(o4-IBK^QPW;ERSq zCw?XOfz>sgrU%}3vm?$G)3+uJtSlck$+%;Ry=9AVqjv-6CXrTN8}t8-z4r`ja@*F1 zgP>8W(tA)4P@2+ff`uj`Sm;7jItW;3YDf^IHvs_wAt)e76={k{=m?1PF1_~zFa}fB zJ6-!-YwxvJ`S#gopYM8qoF8yqA>_&P%rfQ}_qfMB`ie0vu>G#cgJ=%FzPkI0#O_H_ zq;|hzfu(Fp5@+fI-)N>pvun?v+6PPU)oDrX3>)a4JHMN~wt@1-cMx5HFBpM>;capo z!AT6T_*!0<(W61VvotV$N~#1#iW)-_lE;G?)?HztwBTqkgVO zuM9mrNnc}0K271lP7eis6%y~blzge8#?srsA%-O`Nt#Jdpz_}AJf(o2i|g37{Ec0k zF4mq}#vjX56OnDPq^`Hlho4~*QXZ8uw5V+rxOZFk`9)vlB!h+kc?0;%O79cv;*@}xCt8;2IZv7a6zL6%~?^LbZ=g=#n-^;6feV}3NRJ?UY^w)KZ0k1LY zXsLYFnL3m?nTcXSav(5WCMA;vQI?hX2dm6ho2UDqP~+}2M?Bw}*VDFmVsK9AOd?rlkwojNUxr`z&t1i@ylE>imNt2MX=3IG zXR~1Rd$0CeSD5L$w9SO`bEG~VKFZ^Bzp4ch^4+(2Gq_fQS&%g*tNKkK#~de#@%+UhnhLC8UVY9^Y3nV>vhsT) z^Y*+?9@*N;5w8K@}7hJ01o zKi@lzaLL%mpt?uHOG0F-B!yR0TS1T>z z3vV}I##9VbEk(BGEE=NP1l`}ysqNnGOU?hvWx0L$^VQEw9biQ{$qAsm384*`gMNml z-D~TV1I4wudBs_pJvF6#?%0E2Gq*a!U?Iol%`@J5x#qhXeBIwB^#tRF9YMCh_yMeS zfO46nH>-hU@gvuJz#Fv|rWty|8N!&3zcbVQqAl58KVorhA)UWH^>%#EBL{cB1~*ul z=lXNPq8<{8?t$^6R9=z{ue%knu~^S6UQASr{A|Z~{3X7qUNrXIvqP>$$31oi!LAu5 zB1fF2FcGx<=&mH}0g8FvvdaA$HguPAx^%oXnx<#pXuz*R7lO8Z*W$)Mz-6#~#C|vZ z6LvV1c#nWx8^C9o;^ks0jJRnuP*|z3($kjRDgC-o&fWdsC5IU0@#=kI?k)Qli-nKm z7s%SHwHb^HrxUWKB|_k8DCI#oAF2FYl6tvYrQhv^vGY;FY6+SiW+u14*b5)X%T>8> z_2Uy@VoRpF0ze498EhqnWUe_tD68|e<$iN{Ukc{c8PRb|>liLm{_&3@{{1pXN*H+| zCwq=R5#P;D@3<2*koACwriN2I@y5i6gc)KkVb4$#(JCoa)2e8E?hVW7Mbq?wg~xjH zYlpZmos#EXOf{8i$$_6OWKSQ6eeL$-)89$Tpytes3Si*5542A(A{?_u92g54r$oDa z)#{pETr0vS5JUT(`Zem1#j6A#$as%IN+-_5JWS(Fgnkn|DDShb2>=YVExHF$8rqB- z*>I^aT$Ll~Yf1`yv)aIN@Xj(!F@^0q3*=sXE(Vz+XbY84I54v(NS=f?6i2?zz59q* zQ5&@?WiGkyqzcahZoh!9PeIM|>d$%&zs9wH4|eFbfZn<f%I?jIrvY)@;zQyn`oh*(Q1p(WLW zG8m4oCW_NA*YnRmNA}=>=;p8$t&}vjW*C{Pf{n`FPuGd=fNi;h6g*%%w-Y&(!K9X7E+W&1zdH})|8C8X)N8C)I z*k!rsT^f=_3+m)JSEdOj8>=dlJEjW!z2!FC=Vaz5uUcFz4a)&lyt>J69H2HLalj>t1`4p^Lab@4FziAXH{q-gt`xd{nI}eiXV+`< z5rX|18J;DSURF_=<|D7KAAh0rBSY3)UC7Lgs@;ueUV7BxSh-Q;&{xv4yOqfBMXS@u z(6777ao=?AwU+-NWi!-+9Rvm3so`mF0N0zDjz7h4-N&TNJHYDDxEJu+VvYd?%9+e% z_;Ju1h-q~Y>)NLjsa7a=ZJSSX_~_|)@wvmv$D17VMKuq)v|Nx;JpUq>hwo-rB*qRS zw1Oxb`q9pOM08eb?Y{rEtCaV2WUix|TUE-f8!?y2rVe}54jVJasvH}5#X zbeYm6N_TPE5noa6k0^m-;kjQ-7lqq8D7qw26#OWIAHzdAOAxA!%n|Du3(ql+%TDZS zk89<1ihIrXI$z(^QPuQ@Ea$xGR=rKiKoR4i#k{rXbb6cKnv2g}YQ?)fng>$~M$h;= zsOp}QX3zASbb~Y=ubT?cy|?G3L34R~)RUPUp_Hvq3KtX2Xz?sap^O>OE?+4vTUN(^ zHUTSKZIffSv{P`5E$Ony#@L$+QXuurE@_qW?rhejswE^;C6rZf7FGI?Sp}XQ zJ$I#K)0VjPrhnn(+ixZ9`9ttx&W`)`sZ3U1Fs>=6&0VJ|t&A+6OV5;~W~ z5Q-$brHxY4kZ%5IE}fXgZ)n)2u+MeDPP5#vQg^<7c>lGwvTAD;7uQR=(ei*rTBafK z-qnuRQCtiVI!YO7Xca~Gpt$^nac9Q<0h z`YRn}`bD5ZrbkKXYQlcsDhKH9>V3!tjS15xh?k&7sv~Lft=d9~e}N7Q8xib?q20CY zlrwl(W9Q{Ys}I!HSUH`<4+Hu1T&41QyfPU`mLu^zk88f~p<7k_^hxZx|Ol=O3y+|KH##H6b}PWpYVnG{)O_8q0+VjYxlm zhUI+TFZTCd`&UeR=6B4${&w*<+PHfmH%LQ`y`=Z`)!%|e`se%X2g^H(wB&B^J#b7F zk2;qndeqlL@GB?K7_IXpi5owp9D2oe2xUKbjga_--J_yUB>Y8RKYimlfrY0#8W%Y7 z4mU&mi=3QXa!1cxzwX9TQBf^z(#XgVqj)hOKPvz-MW@XH=0O-kNSHEiM8*okBDG23 z&(liY+$*{MfFVTSJ)f9SAweH8C6=`>I^wW{dlr9hk-9A2kqh5XI60pedjNP0H`u~Q zb)UOCWYb1qht_Ij9OQM}bF4dW3%+()sykYJuIjXsLl^_=;nDpJda+5>!m=!GM#%45 zs!7O8vh8J6o^rUIUB05Jl28~F=UOrXytUlMb@7EBFKf1gt11bf>L&;rJG+ND0RGV| zB>)`{Hl@F2!nxOKG2Ke(xM2|vK@uaYd!1Cw>$A3@uF~kf>NUwRVjAvE!PrSKz#qI0 zI4qHRHNEJ8`Fpi`qlf)!I#X=Md>1`*Z%4&cG#|KQU|70r8e0D7`MBcUpCEK-P5>Lm zk&K_CZ;=h8Pl zx7lKFP(`*F;7lnQhg$6bwMepU+gi!sxOjZaio(+cZ9s@TajCGUCXCP5p3}8j;qs~Q z=YmEm7j$9+Y4ilSMp&qoaRAC_R#NlAsGyXCoxgB0aUb{X{KAv^ftK|0bmgj2iA;|z z6y6jlUGgxnnr-Y$b8JPYU{18{ZV3qFcmA8o3x8m+A-8l_E# z$jLdSh~J)D^wFDWQEjWy@${5+v*>^Zf6o?_^Y^$8lk6pH?N`T^n&F+13UD^|CFca-t0r8qxFI(8Uj^kqfh5c z@-jlE0L#o=criP%T<;?ee?@LrB^~tgb(h#RG6=^S429eY1}1c9fBPlv9E*czypFuc zQ8f$7y(qz5*f*f~&B(_=_X5i~*-pL$=lcx3d}0+*C)bMG5hB_pPn+=YQxSP9{f$z! zpD!Yh@`=?NcHQmI&2Z$*YU7WYa8dbIJ|gonG0W=U;NasBh7*pvVeqE8h?Ne~W%2e& zWdSzMzG}}2m*UUnZgV{`qDn0@iTV9Lqa8Zi=h{-acAy!*JwN`PSASMaFq*g|@N2~L z2N#A(0enplGZ(mwA*#4w$YufXyg!c*{`*b#plUP5VZsdfm+zkYJ!lq|=u=$3+{Ckw z<;dW|wKwBN4eL?Dxv~WtS6Op8>3X__2LZk=B|-OleoZo&)GaPT`e9J zWxd_QnQTQrhl>iRQ!_|blChDyU9&nZEz3gmxYFQrZ=sswoppP^1Tzr%GFkS|#aIkl zq}vGYGrRow)Y7Rr0;^q664-E3*K4UW&m{LGr4>6@cZ}<0t#X@p+9_C%%TzR+yY|F8 znwU@#dh3?+p7kF`JTx9X1lzia;KZz%^&@r+O`u9-X-r!cMG-iFcE%t%Ds?5+HW@$k ze8sEtw~o(mU$LcnVCIk3a)IodjT|j${7Tv9W(r1hOkt)OaA@a`oC?$~SsNux8h=GvJxvXlap@J z*Br{16qxtxMx>(c=1(MXFgRA{4ZM7@+;0W{aDmx~@w00MpKG-iE|;og_tY~2gEvu_ z?yoc+iERk7;n;BND;Z5Y!-f#4B-`T0;hw57Y#qzCT}+thu|66dwzBok*hsEcun%r&2Ws+uko9(Ll;y9bm%T&;7)WsdU&FP@`6nDbM zF?BdtGT&?9g-b}oIW8gB&)b;LuS@=i{+2oU?Y4q}0J7eAr+|zbvD&tN{iqE9tqZuE zl{c|x*`L1guR33zk@;XDywHVqxTjEwDt=ojCfAK?zC0+a`3O{(awW)Cz|6<#n9su> zm%VnM5pqK7cc8~_C-`4{wQEdFvYUw=T_rl#BZ{{>LNdOAQv9-+LHJ zvEWXAY=-z=`J)g2d|?=%uhgGL%zZ|Kn((2-Tw1bqH`EtzP19d|h*+}(ET)|#b;w%Y zZ{U^Rj>I(OF8=a#Zc{8L$*lz~j~`v($360Z_vKu0N;=6AYA%q-D;=ON3Ym(EJzW>z z`&Z9{8Y81g0u}^hoh8a(y4iExnBLWgCKlC_lF%Nee(nmGMAL*$csEs&ewfCNI@e8; zL#cJw9$sQUTs0`eZh6G~i+e6TdtEF2f*y1B>@)jYIuP$2u#4LPU9KzSdr35H52oQv zwt+_5)cPqU>qvWPGiyh)2(wXD>9m1wuK3knv?Yl>*X)KaW9ueIoj`3PRpm3@?Cs%!8?pX_kq;`RR&TK~-E3o* z5M4ao@sS~@4nipTD?dGj0Hk#sw=H8@r_l-7q@c^a=4J$ekuQeVz3}h!4waZ{&fXSP z1NJcqWQx$vjM-zfZWy=SWB)<*-h*u2x`=ehoyjs}1X_|JPJKnuBg^`A%pCm|qiC_S zSv72nJh#>5I&tN$+plOEU$-KomOb^nbkDeTM~lfgY!_Xm8M-Jxz8FQY$As9(nTB%o{xX?ZJM^BTV%-l z%=SE!5z7hJL>(cWNLnW-v?}vUC%f7*`B;j-jXPEE5OUOoaWQ)Ac7}pBCq;v-Phlar ze-3q+#)Ye2cz%1vV7{{N@S}I}E_RGIjO(9n(!Xk4SG}$+#GOu1sk>hls{|yieZysP z2^S)7R{4Iv5wd+P_>sfac;%_^7fZ&IoRHe1Bv6$KB}JaXCx@d9+^-jqGR%VX%O9V7 z@QyR@_*({VZ^5|1009DYCbSVi!KdADhcvBom&`8V5;?{VLJ|q-rI{USt*Ry$Byy?2-*i|e*`f?7C~he*?zA#6DD{gzou7^*zQcEk` zF2s}Ur`wa7KP6x)wm+H(x2S13({BLvPauaOMlD9n4WlJNKwp%HSxU-ebt75;%9SSx zgv&QtupkmPdnNoqZo1@5|8goU6)p_i-B#o%J)o_XO~F`cJ+^ZAbZlrvwAsww*ef;h zoZ)hx447|=eLSRnJ?hq;Sx&kUlr}XTbJIAmmSX^Z6ew`N8lvc<5N=1Q1JUE-J>#DI zn{i2Mb4kY-^AjopK0^6453=Rp?6EOClQ0^y3r%UT=y{N3#k*v~{wAqPmA7HXwmE&6 zV+9Tkm(mAF(hq!fe{iN-glzV{aCo`<>$dF1B`(VFLv7Yx?+*gD6=aXlf)Q8cx!KR< zZw(UQX;0#`VTu6#FK3HF@sQ6_#7U;p>314B&JKMQa8Y{4b@8PaSwE`bb5mIN!_N@A z)fh?omHiTdD@QX%o#(@M&2^b`mvkTwAn3B2hOHZ~AxP)SG3zp8@K)kd4RxeN^TSvf z))m;Pv4SS_VSMgd=S=k7hS80ztQ8aE+^`a9*^CLlM%&Yc+-DDmoD!MZI>1TShuTXh zdV+zVh1}rEgqutc@<`|w1KF?E_@nO*)o}T%BJ|)BsZ)ubtB+D5+&Enp*@hEhn%(#N>T(CaN!BZ)4m(5lgRTb@Gsx zSm#&2nCuINp!z__c}xahLXBU>>{R|I_*zb+AY>0p>iHxoxykAYKRWrP^@oA+{`HE( z;rlGZiS(=5FbP^?l6qA$n;^=VT3B<&;pkgJZMN5keQM^csowACi}zGd%glU1L`_ZxxAN(7T_wpWA{BXPtz5T3B^XbtW;p%Oo z*QWI}IqDc99tn=WP)zF)mLLoI3A7+MYY&rxh`prTY=fL}zZaceC3}~)*{v`1&@r65 z+mll#GG(t=xNk(~{?4oX+w*#7&5xH7xtvY@8ubc8cLN<^PjytHZq%l^z*R#3324z@JQ$r&^M?=fHI>P?z({84Nnet* z_Y&P}A-rE?Do7i~hSDS5B^VNq;s)Sils$x)FdC$PgMWeCU@Kay|03`PV*bm=#h9ht9E_6$lSi7<4&bj3KE+#cF@&|E^50sAVGWs>BF*FrcU z`QyeG0I2FkXAYR8fsd9)7n1M20Hl^O<0y|pZ>D=P9+`1(;v0`7dR?F2Ij)bef>6c= zUIdO^si~?6T;RR&Hcm8VX>SVWNjiJ_tcUf&?=tuHdPkd|>MyEDXS-b8SHYylFgRpZ zrg-^`$kNA^QYI{_rla2DQ@Gg}MQhGAJhL9X)5 zN!RMrJ&`>%#}R)cx1rM+kmz8qK;Y=qjKp6WR$@wH3sxRA8XcW9oG~|a8MfCsX3?O= zt48 zQ+qe`Aw_`{p@bL(Zb>te7u^MyBbDZ1ZW`FVtWD2W+TlNcvh9fU%``q!N_p^=3~gN+ zL6L6I&vmug7g2!=go}gn*Qe;iDD~dWFD`Hi1B#X+_vK~2xVKj7|et(!~@YNr5ITGXRb2V|~`P_>;L9gD}D1Jpqy^k0DLdSvC&RnHf2dlp0>0K?b6b z)BYDUazBJNZ0iV(zuB^jyAife4Pqa2G5x}+X2|`}b)zXncm@ykMLBF#L5HM`<*ZS0 z$vPTUtiGR+}(%zzg)BVQ+XX)naZ&#oTLils4X&9d-g4lMP$5J+&1^704! zqomZh?b{C)ckepxmSz}(PBbVpd{*r;h~hhkcs5=z z_j(7*vW|0N5-<)L zEgbz+S)#EbY7%Aj*<07-Q^JwtL&cgEmvUO_a0&;FAbvx^ID2EZ75GnsiNRbsOXZ}5S49goIM*BY z@edlrPGWZ*_e(tN4bb8tHV!JW1pp}y7_AdVU+n2sHuL`7+Ks1IOVizJ24r~BzJ;d9 zbhp783f1``ePAaCI-rgihiol#r`EoqDRKwT4Dm08bx!f|1qqzhdD}WUMzLH7P>YSU zg0|oe0Du9msW**@d8y`Iq+rrFU6!_9;4L-c|K>}}QSXnk+0}~@^GuzPQ)E?i1N;CG zfR8;*3MM2sp^prFmGLx+B`fduY}a`?Oc&|bck6~eGL3JTelJtK82uUI{)8q33r&e) zPEFsa=!iaZ7C-mqi$!5k;u)^4C-$aGA!p;5=vuD$>qDRVd0r*?ldn@;2*%L}XKWNd zCRCFn(|C^bK)ZVAdMStQr~30~JJvmbT7U6fO20i_0AwfRv_D<2P#1eR0M#Z zcuC@jKrDOtzWz%eqxCF$nz(Q6`gR+VTff|X-!ZB>p8{ICeO<>8M?vk80(UT;Oyf+B z)C4^Nr`N@e*9A^i9;g|U%-)5cdHlXI0b5h3VY1JhqFF{Os%nLO8~O?22I>ujErHy_ z*la9$6&wpci?c|BUGGk4rG5n(pG&|DgeGAIDoL}n?>~GApv=;NSQ$gz{t4obA^zxn zHIM!3UaJbgF&xm-wXu*Q$IqL}(K#b)1Te4E_?8+C7=lY-$ChGiVn)Zibb z{Gq*6I`6_s(ImdhZxkhNLCBMhG*}Pn*z;jOx5m_)>Ob85oT8jUsYfAVyrwX1jjadW zAkO$NIn2;B$}P|zgA3hF<43ZfjPNtIO#YpA82WqWmF$8JQ%{*ow9h@uau-M}1wF%> zf%N+rjOlhZil3U}1>^KXpWq(sYOR!4YR+d3CqBJoAXYc!8vNbACV`&{7LJ7zv}PdQDfs!O{)qtWN!M07K< zx+=f^Wiu?~kZ-ZELSrQFYX>JUgx<%g5osV3`)a`S%JDo1GXx4Tjk5Zc@bb@T2xcr7sjLyb4^N$ps6iH4 zEvxGU7F?aUT08uLxt>R2JVmcR0ySt3D0w;9rdX+~G{jj}@D6d(aGD;ITmT&4x!j9Qsyq&q;yxGJ)lnDw=|I3Wl+Z5~Pb zPMk9?5>!?)&{+LqXXp)hYs)K7xWv`6rTNRX?H|?uLVs`#s&S~x6++Q_`!-`^#=78D zhRaKzcmttGg;F(XgSq$%ceP;`X`d%wOEh64aSjBPm`5~LeD-?>^9;Ei@^Ba8U2yHp zW7YWc%<^vv0|T_mW})?%P#6Qo88dRX5yM)e#+uHYT3qQ-Y}|F%ZqV&!5JrqmC{s8iZW@R zSv%89JB}#alQn$VjCk)8b+;2DoAZO|O?`$|AeGD**Re4!(^T-WF&-?$SJS(j@uLr{ zqso$WT zxCg@}F7GcRH%MJc`3LtHK`s$m8xS!-dO%|pCxmb+VHpNBV1yU+Gjc3fZI6~ms?K!^ zXh80aTs!5s&tPBgei8b%im|Sk)Xnu={M}}xK#f)|$-_wZo@lS4lwtjyn?+e{Z$GY_ zOZGvSu&xwKS}H#oxyq;1s*9t3A=N zg)OVAd+x|w$hf{j*SUf@fN=w9V;Gv3;zEj>8kalYSFUBlkg61uj&V`g z+IxnGLmMJz<#|L*ptlKCQ zjq2Mk*^2s&=yToslr}=ijsWCW!tV&5=$~SjPw#p6KR0?7~5X@-H8Q0|Y-c|YX0&Di; z`qTh5Yrx5dVVq-QZTEqG^lsBhiTK4cc)@~uo{X+1$0sM--$;i=5d_y*BOCWJGR^v| zKuW=XO|A<~$&J35o!ta$wc0nTu%eSy>-fT`|7oeabQk$wd&-{`NnpOo>%l7~C}^^G zl9PlXmXQ<)?}0))Tws|8$wA?psbH@S_AlPO<8sHz=i{T|)5lA1)mk_VkDZIocZQG* z@NTnmF*p}c-$|4AQ_#ns>{DEL-REU3RsQzvTgUL(ZKrtU<4t14^23_C%F*wBuruiA z9+WWn8P32coQNh{P694;?Q6RD=Jbtl_u{NC@5nb*9kfhC9WU_;pW^B>oYtfmx7^EJ z=K%YA5>#L63`NxLHHLq9DZ6anUCb-6#Pezk`A|LPRxVpN+X;{gf{TB6Q`1+x(D#+e zpva?W{6*@4tYcyxtU!ksXne3h&~@l07%yhpgR-wz3|GDFjbpJp+h3(@6Ln5A#rSUR zkw;ZNY|GlhKS6$2y8jZl{>*j>@eV0^$~a`QGPzaDxOGChPm#&6NY6a*K*b}ed=1GX#4(m^uFgSIi2D` zyNE{eUQ55BI%L^qOKjvQJWV|^@_=~vHRleovJvYq@4Q&jr!>Q!If={H#0AiG(b};4 zF-|2haKRz8B)WU@Ag&U1!J7|0?(}N4rK|1HUMa=s5BAsQsyg3_FqL0>mC0;#*_^}y zTgTynY|w$6Wg?otN6ARTD0QVXvhl%v4{NV*(UbDU!iua0U#?6g){6)?lbzl8k-;F= z>UwA^PT0FBV4)XL)k)#9hj(;g&ak0(cwNz_6MwGYE~BmW zM)J3)5rGa7k+0Et^edYqcWFi)(^b?+W!U$l&h(3HBLMyeu#vJy3M*T5yow2eGpP125uusZ$r&#u@0l zX__PG#z{t{Xr_h?dJ-&~qC*fb^LnkEmgASYjtaiX@F{GIUbIZof)Fo%LG|Olq4n+D z#vD*G>)D(6+ECJ6Ng&dz!t#8V zjfloLkMd=iNB0*K~( zVi3B|`=Jpe6{0dV6n)swwY3G#NP1vburwOaQ&Cej`1MFUq}k%Gsz`|0rEJiQC_o3| zj%lPre*_`1{te*Oeg@ftHcq-{7zgZd>~u-AreYPGr$^D4eH^I+5868vR>JKT`84}< z#+5r)(fH6~TT_AhylQ|6eC*HMv42BV_vg~8f8-U>chnw3cTcdA3}?IODR9ER#3CCc zX=BII($K-9gp=7mM#TQ;Lxkq5hJ7OTvl}6hul#@NI)jQd8)~h6nibwD0L|MhnvGT= zw4}Yfx_(3DR8l*EHg*;>rAW6kpPo~Ub=eynU31IYTK9D$rz@AJ4zH<+Dg*+Iu@d;1 z!c?lPZO&juRik0Vtz_qfM-dQ8I{-aHlrsy3(Z&_Q|Qwsx-;e*(>BUV`K4UnRK<8xjPj z7+?p{3VzTTGU=ru5B}KGqJT1XrVq3?o&qOnt0sStMEeVtAR#su!0-*Rqx(J^#vWY8 zj$|CZGUi!4d?crR3RNJvEYAuzn`2&g`0Pv{ON~X?ZyrK-s-UH+hcw;Y3T12DX9Ag2 zZB+*j%O8?{h>O~NIx5eDO)0B`_U`=Hg~A5ptC+Uel#75$Xl(-WBQn~8Lu;i!LBuCP zWm0oA0yJ(dx}%)MevOf$yS{!CR}>2Iq9a=zpV+7Wk{hm$;kJln=tcE@H$v{+$9;faUh)vhNK;)NO!aJYm~8H~J$M^bZq1_dSfO ztJ{k^K!K5NPvJtt@mXi_&F}CpvU}3yzfNpPSlVQ0Pu#vyS9Y>qbur2Wnpm<1B_9_r zuCAsiVG^4H`&WiSCVlS4huG?o)y85@MnJyCas$qU?`$ljs$ugG$!ZkI_~sOEpG^At zk+Z=AvsbqG`1ZV!-Sb$r74U!D8YXzVDLm7iDh;-LrDWlRY^zo$2hRn~j4P7QO?}m* zhaq>Gu$Q$37enBT6a~C*toLL%O#t6DOfi}{gr zWM@L_E_YX+zae{J{3AK;tp-nA2=7!>D&*_Azp#kEGr0eZH~egj=@$XBiSwHepz`X) zjP~7bYLGLUkfSEPnxS!{lhg z7?q~Uk!b;dRRjmQ4#QvQ16kjdzYplYWW_^JC1+!sv3k$QjVz5`j1AdSt~i8CzqSiGmZjL$h@~yd|ZB?!}GC z$?Ko^*vOO+_g?7SV;0E$50lNX2&^ESo&-8Ztu<9E#gr6G?0X;7B$$0rS3JhA9x~sw z%nPNLN@xMT=wt*;0OKk52rZ6xdISpFNsvjumEH1nskO@58~0x5+rh7kxC<1EE<$lR zkO#319iC%xnh$ai&F_DLK-#870Cm>6IJJlvubI0=L}k&e?%i8_=}f(GHScHP}jC_boh zk`Lp~HVSRTl}j{rv3w!f)d{ouZF(_@kIx&6d5DEym!(VgdT49VuzsEQR1_o=mie=L z@dpF^pU~0UoROS9&fc|+afxnmOlVHS>L5lbE;4{R!e2OXi;kyQNb;0G>(uApMvk zPAKgD*wY3J^%mE=9GGow5Xfey4P&IWVcs-Pww8p}!bqhAj+xkqhrLS5k*CY5s&|i6 zhTQ25Tu<6N=b~#8s6Mv?;WUn|6`=KDc{E*K(Rwffj`-FtNR#WD#f6|1E4?F_3c9p*9*h^8npQp> zRI*hLe6z(QB1SLTf;?2iMLq+f3>Exlt8r8xNdd?M$QegCL;)9XRx>|JWI*G6g6QWNCe zJyDT>1E$_Lr!64p(7#PI-8N4!ZeE_mffg)4(PpB0cs^9b~Gd0bS^cLA43bb%D%sY3;Doy`K9yU*U<0n%l`lul#_1j zLn|daevH-@4xXSTlP&+Rmi@o|9z96iJ7c0~>ZDV!V75RwhpVl!KgN_><#ZQlYk0;u znPE+LH|_SQHKVd%H}v%{`{?&$@K@vWkM9ZUwM>Q&V)oT?lG0|}zvfljFn;XhdUxZ+ zy^%+^S}f?x6^kP-u+n{&+=Fc}78s_WNLtVR7LoXK%fwo-3WeIZyqnb~FLX`C66pp4 zLoZ!%u)MGr;_8Bd)*f0)cF%LhF`*7$_iDXs?Dl+2=P*1ckuHfUpkx6!<|ev>;K6_43-DZOED$nHX%bNB=9QD z8nG-(XaO8q_uWf9vHa>Mw)Qto@`c_wnD(*Xdfp~Dcks_|2Ut9MU=GHhv~!p>0`Lv@ zi~)_W>Y!0wI?3vz&Md1C>X)1LGED}TXdm#Uxo1@*Lc^3 zhFfOV7FLp3(e`J)i5-z|@}_$tcu*TMgYf9umBp0lQKRnwH^DIlP*hL=HHp^&{0<%^ z1odcqfMyWw0H_NV&zVMiC-viYs$D@hiiS!I6T(3*60U>?xj;PT+yF|Uc*Qu6G4lg<(o|J6N|X>p41Aq71p{uryWI4|!$Zjz2|AOH z4p-J#XY!VeIK(>ZN!yv^O7A0zFf&4?OT0<|`QZ&DQZZpQduPYK4Jc}% zH?<=##yZ@B)=ql4eTS5dMXEx`9$0nci$S1-vJ z!9l=YGYeJyWiS4}Tux{XEbNnbYs|<=#CMDS|0(C~7Lp5SH=Q6IS2QCf05?2thN;Zy z8oE<@?2G#s^(vO?%e3pXLBBYkY z9%CJJ91$DcP#v5>)V;?2>ZSrCN1&#i*Ihcs|dcH$mdE}p z0{SeiWP@(fRaCIY`boegR>Xdf_hexaAHiwdc;EHTYnNc-vfewhw(AKmBt z|Dl%jAMnHeeyHu=kXhe&?)D&0G*yDvKJJVZ+busu$kO$Y38j$z@v7a|FLFYluQ?%x zoR?d=yVwl`qU6;BdG8;*-%annzP^6M`hkULh}iTcCC03!y%2-Hg+TwJRt0vncGS9L zByHx2$ty=;j{ZrAkF~GecY19>%vzc+*XCLue$@GbDN4fyuoCZa*rD@*=I=!szm{W4 z-Y;A9x0ChrE4pn(b|F0)P52L^rJ`wS?&FkSgIg4-B5GQ`WsDexIe}O;Pwch zLyU*U!uBS9duO5ZBU;g23+7bH!wS; zATtv2PeJ0}hxZ@h;Q#6lS*UTrlR>^GR`;pqy=iv|OMaWtaSJ9Ax3M_-Q`-AytQu2b z;Trz#{{GP?`1Pj$>W^0j%F|VBdrQ2u9NVQjSac^04GZp>-!Wp2FzpGwb>*?%^Gvpf z6>KB1#8PS^#X4^l_a-?E8df8kcYcU1bKHo}dAr9U#F6Q+3FmzH3R(_tQ%j8qEUfW~H zL!^Z(@>Z7RIA?3VbL;E|a^p^C+mDEB&8}om`vmxh{@DZnM2$NxIQ@@KR5uWhgnf^q0tjrDXovY|Q&-Bvh!)#V%S>U^hwC_2iSey(JgzR9-( zA9)!t^_4AZ^ze-~AG+n%!*_c$hy1>P)ot-90<#>Rua^^?tSB@5{Z2~npWV;TkK0NX zYXih@Br%&bLZFQ_jmiXBlO)c*J5#Ne6>4|Ey6VGE|*kX!{jKe@&4Y=@7?fO7Vl~X%U4fTPnWMIh8ZckQEV-O zutsCC?S-e~t4nVsi#>_`AzSM^-PIzq2f>a~BRIsAIk&&O5^~ZCh}ab$}o;wrj=cHAO?t|%!& za>p}F-9N8GSN?^-ylu+v74};yk_@q`kNZlC_%lA6!xMm>_YNDs-veU;GPuG*Nk-zz zLl1Y^%$wPzRcF@pM1nqw1ky2>3g^3?Tk6le{-@S=jBH7{GqB?fEV@(Iql1+N1dY~q zFS3REWcu+@6Vr_x(Wjrsn+;mJ`A=(W^C{oGaePbG^u5;h6kGiVrYVVrXy|lIV25O% z$oM_)s(l;lpO;^1(VQe8i;Vw=z4wl5a?RF;gQ$QsL5kE!6QnBATR^3W5J8$CAcE2a zMCk^Cpfu?!C?G)*kuDtp3B7lu*M!~^N(iL*Jsr^kF4o7!X->16gFP6wDIQcxab_pJzJ z*6*rJwf~t^*^hC9DvR6?kPI*p@J6*Yf77=M&h=%0fwQ3gq6Msyvdi=!kUabP(W`O) zKB~f>Brv~POTR4czxE^$Vp-HA060e$m91iiDd>ApEDnRSZByM7rb7SyzX5}?vr(pZHSFM|81c{^Z9p&m& zgW!^i^=Ft2oL8}4{)t9{UDFnCXMQ00yc-vw&uA8Zgd6e7Pk`bLh92}d`fuRI6n}W= z@*@5XLCu#Djf_#@z*08tIA>@3a;{rDtvEd_Y7~oL)@05$(qZI8yv$yu5r_oscKmre z{65+G>Cy40wRH+^%NUX-ho!}D)yrGVG}^G2aKOum!rrd(ZT=+vzQr63zRfW7MkrGGry_&Yx z4Q0Y5UvWP$JH!skr3F;rH~mnGbHlc_Q{}Axyv-TGP-`3pXsdHS5d2ToZ|X&-pLLd5rp&3c!E^R44rm$)_8$_1v#2F zGELW8!Zc*n%ipIJ8th+5p9!qesAC&_Tv&kyGiM8fzrXg$4AIT&DRC-y{ll( zKn5q#?i}Jx%-z@4ALp#gq8JO7SPpD#6`S39%x;hh;Px|tPenh^_x*lZ`SW*ESJwkr+QRek%;$29_|WGHGNE`E zo9EV&O%Q58-?lKqKUcp=UX3q8#>I|2@|w^W>)N;~I>bKorp9PFtPzC(3}Bnm&5TFL z+{Ck=Uo7x<93PTpxk1Jzu&JR&%2l0*L^96Zxrgf;#e@+7VjQY%1cgPkGI9X8*;Y6o zUjmfrR>i2DVsE=M%>jJW$FoJvqBLQ&!RX;o$qK>J(vr(DFE9UnMa2^~k@pct1RXOd zOte9u^PrFXzve!l7*BhXMGva>H$}>Zuar5(u>e2C6PDcLs z-SJ|$Lp*)qG!IqFesW>s*2#t|W)=<+;`Tv}QrL4Zk3>>zoo!rKy76U#`O~oSTH8 zykJ}ad|Ni#O9Wd0V|10Cj}j_)RQb@oBJ!jIx;AF!Hm z>rYLkZK($Xp!vUh7R*O_r>43Jp%lR6uKW=414c@joihx)KF7T)_uD#>z(%Y0~LktOq5~a&lTUXPI?&w0W<2 z5W83uA5wsQJ^{)Ni53RAKyR6gcqIu9-B86;0EXGh;cW!DH}oIM6DxbXFEB0eAD!1^ zP8Uuv&^ZG(cLZYcVj*fna^y-M+|K><%DC6dJ6tY$#=h@DFZ3I-jUJ=yI{{jT2@zoU ztR}K(e~rBSdc~ZpnyTgXilWEu7MBvMtsx+_sW_S*_PnxKFue={xG0%8PkYQ0dz zzxWQaHbop-xvTTc)Vm)Dq#XsAL=JjD55jzzviK(-8)bW-gs;F{SVSP8Jz$A<#j*`* zV0xYhEcwPllO8qS33Q~3cuTIQg`Ft6H4COxf;2bdV-%5H( z>rUdJ2L)*S?)uG;gPt3vV|itIw2wP|2z`@RiUk5po*WNSczXi}$1~;6uYr%jX#Bf} zvM?JN6^nM=pobQ*>t+N|=ci@8b-i?aZpeETE^rcCXC!FJ9X^`|E}FvW5qTodLYj}4 zeQJ%Zce`+S{kSr`UIuBO-<*!IE_4G72uA?f_1t6-6z|=;-iE|(-9=YI))3)tR6A#% zOob7ZU2u->czih)sC4!Qo6M*VnAEL+r}c39b}6ycYc~L0EPa0<2NhlVaZ2dSEjkxvA1`>v-*iC164kd*n@6w+Sam6S%7FJ}9v+4e*rOF*C^8hHwL27JY zldrV!$@GzOAjy(A)mWbL9kf*_CA^c)FrnL5s()l+-JdPi?jz6=RmqnaU?IK)W+^`X z)dfzOh($IaLlMjy!xL!6AWudne#z0E%&cbR7Rcy5qjo?Lfj>wk$7WO*cO_Zt6)=-( zn>zpWYqcSsLWB~NejMI-B(Tk9SJQKg$OwqW&V2)>E&rc;hN1!mA05Hn!UT-g_6Jhq z?R&Bwe%xfvUJLy8 z_x>eVSbMDm(FEY*+k{(#FojRmSXOz@4yZukI9EI4iWn_@ISFh5+L-kcYuY7y(GPr} zJ<-1-kNi_W?62;fKltQ7LGT9ZGO*YryhB$4x&9hK5_kTY`1BvSJVhxmOc?Kl)gy?J z*{Ghbmm$~n3d_v{vV1m$=EUXVK@6f1k9YoG&TX^ z2wuJG`3CMx=H@B@1RDygKTBFEH@>^%xWsXKwvP(=x9ih?$=V9S{!qOL1Jl)5p=>De zNb`gdQ*3lw*`R<9r>3Np;kBsaOnOzHgukdhIYWv3$tUo4Tebhlw)yQQ`rD8GdFs_1 z*+|zF5J3j+z>3{^{yPV!9Wo%-j zJIh*TUXfj^Qx+7{8E^mO)BOk5*8-r!zD0}!7JDOJ}rkDwCwnkck*|bO@9c~{cErd zaL@m}kt~3*=RwfXV0;$4jn%r&bs%I^aV`PjP)sh_^1&WrqA*7(mX zXQ~bWGBcbxI8$XqiXOKi5B-#p^Dmg%U9e$=JJ?VYZ(;YJkojE3hs&G-W-aUe9Oz#) ze7F%W{xE1ZTWsgb7ZKz5ctn*k<i7~-$Rd4^ zpY3#)#P;?>hDGICWA1WBfQ9=9PPkuftN(Ek?gZGaK7iUK0YwWpuFX8-6&duLIWcCl zg*7Q_;CuWWE-kG-8D)ODDd)&bzSqqDtf0NBe;$_n*B?FWg6#ZSXZi~a9;<;%>#v z=E!Bf$u_xnOWD`|7*FkAv&DY95<<7!_byy@ zKZmaPoO!*2$+3=LXi*)N%TL}URVU!v#YlV!49CuqJC0S5qe1}&^uHt>{LACx&%09X z0RYNG!rN1bm%f910Q74xGZPd*g+Y>avz{W?kJ$q5RV(T|s(;y2`k!L$wgLRZU|vyt zKjX9a7$hL`S+A`=q)BQo*0%LUNOfFlLr8BNJc{3u(**gG{_-M!6_$RxXE?}wi8?a$ z29EZQ^D#a5evr!f&IBuQF0|@4(i$0hJ4)LJ;*30i@Y4zan9Vl0lSL7@X@~D14Os|A5&k;h zUd0abYuI1@OzcKYFFDv5KKt-H==@=~`x5*yBi@`BIDp;_S^>!a(;1=(c)|}sI{NK9 zsBVF5`yYP`jQOnM1CW+KR(}VD&>-ms?tepT$4q?O#!eIjf8CmXU=LS4bj4z3;x)ywiq6ugw%0e3CL4vkV=X{?hd)V; zegUs3)3ESZm5Z~Fp`~jhC_#_c`56ZvlSo6mbJ=zKg~GBun42<+Sfim0);z>}l2y{1 zXE03_rJAt?2eb{~w|wzyVPiKfL9Q9>GF53R5Os1uVN@xkOv83G84urku7aki^3#{p zkt)j|%#((yy!BE_9NzBb8JrsrE(Y*>szbX)yac*A2%prPrka3T(?7nh5g}~44OGxm zPdHlCwm7y-Bu}<(lOBO17hkHbxk!|iO_)x6R-w0ieDy`O#lDy&dC>LD1wdgxMrR^q zVHy}l^om>BVNF%J;y#rds_&<_>*2%P4NnHg5-Hsp7=t0~*T*fKuq(+aM@(fj%f757 z)JYl{gb|t;(#-;>AB?aBZl&A) zqz^T)%URJ)bh5jqcOvEyzdyWGT4Bi*mgd)Ry5^nIH>96ll|>;|a@xfJr{JML)3=|I zqcUB41$6?H)<}lre+RAQz%Q6FM$o$j^x6zkjXe9!f%iY3c@r&3i2H$J#4C6>_V`kZ zQu!<4pe^zgPi6s~k3&~9E%6qv@hsiY(Hb@%!BWue0+bQxdRU%a~>^8noI0zl9 zJ>aUhTJah1TI638IhjyhMAVOapyZ>t94H@InS4{!{Js)t{naK;&jc?2@wr<8L58TEon)Rbs>L=#0^%Rd^;d{8+oIQm6 z?1~25Xx0jfy)Arn#jJ3}v0h84^S$iCM!ADK%1|z%(*RMh6U7)dez&Z$eax&WQ9I6q=-bswF7 zdV1Q`#w37l7<|g`@zsHyimb#|Clolm_89(SbjB?w4^sAxZWk`Rk>&%Xmnxwx0kJT% z{slkW&8BQrmi{C6PQm2Blq~lbYjy=OXJe!$U97uMfs#b-{!$(#o727)D>D7}>laEB zTyqS3&lf4ur+F_SQ1u}2Yjhe*=Bd&@z_d{f!MN-8w+Hu z{d&eo|FFnLUrQTZhuYGxu1L%|ceyj_{6Fd~v{z@}jR#B7EncVrns`#YAa@fAFqTyha zM;%6?H_Y+wa0|-ybo*GVi|Q8ZM3trIpxdoR8cv&<(1_jdAc0nwS?KwH%hTrt%pw*S z4hNbc?oVH=oI_oJ1Id|V6%z-2`EHMV^yoIRyVjb^`y!xc-Tj+$^0y)W$MiL18%T&8 zVhJK_Gl$Z^pndRfa-mw!hqJ;XOQ9wF#I`S5KwVenF<^gxTpqSkT$+2cG0 zW|UiIZ5}at;v>g49CN>Ac_(=d`R`eN6|Ye6z{6dk$;ugBz@0&IX}?<=z*}B_onKJU zInM?Wy%hsmdr||d{qU3(J4v4^&UXgS_`Z)-<2XRtZhAzF=h+NH{_VC=pyb91%C(f1dtQw&M{1QK=a{ec}Y{P~K zwD(@!LacB)4_j>*Q+OA?m8Gu0EUku9CGLwi$A4?SbO3g|=y)@Z% z_G{%r$CIU*t5oWcXY3G7sxNNvOCO6jY3eN8c9O`)fxDESlpJbY~f~%=Z*_tKRN7Hee{C4D(~&QjKWYi%Z=^aKwBw1OeX>%x?OD3VC!GP+K+Yi zPZ`uI2;&s-FG0PBtnUE2>I7g>TYKWXdq^PV-7_Jmx8lJ-0rnp7seSvCa|4jeahW4R zPSgSTIan z=WHU#LSa`w(=VvMmWKMsN%Mya9el9csN*gIF@x0Xpc6N6N%$<~*C_eR|FCMhTKd1E1BLnB& zXz$Xb> z_m-V}5+05QtC=_Hc__QaEb-8|&%T)X!hO=WU@|G6+W5dn_mTgH?lY|v{BxA)9OLP{7o*G=s6sE^B3BF- zx{SAYV!W_MOWE%?fl0CbURvMFje$o8k~_7^^?ZZqRh#nY65`Z$V-F4~UIdBliC2i$ z&%fk8zuKAmxZ2B|%pn=U+bt@C!Im0`wE(FT>>>)kP>h*7Tl5KJEDhduoZtcAeXtvQ zNZX{?(#070`q>v*A*6W%>DA8K_Dq%H1&(MX{~JSOs-AtN%$^GLsa$J8G1Z}tHuUL4 znWLYVbL+?s+3oFx?JjqOpEoJ(N%$#O3non>3kq*EzMRtCQOla}BJl?}{w8Ih*hUgr ze8@UK)IROJfiGpi*k$D3M1mNPld8pSu9M;^!G$dcNI78A= zIzyPs)30LvCRV#O@FX@X0&WNF7mJBiK#9Rq>@h0Z;{N84=Z&BY4?_3pb1~*YRy3r3 z*4_l%21)J;W!lc2xgNzJ?PbxInr22C>bFZeaecr6qI-8e2c#n~0X+(}0 zOd?(&E+-$f+1FftQJ`bInBnes&Prr&z_dIG@1@8FX1#o6i?HAzhQ*!jmmrBN% z<}K)UKJ#qm(09;+Az%{<-|Mm{!)Iai>Z5W|?hLvz@!4Y@?)H>B-#CQ$4VH$h&SAmy z3lie2Y~$lsMS+8dB{^x)q8TLuXMlvQ?w#N39k;RiV*0%C@K|Q}nOJG`W#!mQpmB=B zP}U_o6^-U^rVRtvcii6`<|j?RiM}@AJ{yfZ)eY00b7#dD?fZt@`ckc5-y~Gtly{th zsepyshw{r_VNB~He!Y8!-K4Yg?srg7uFyU=tIM~1Z1roeC5}_0@YX#f$qTwDio){c z_kLMBynTrIP4AiVAEFfU`Vchn0X1eADztW(-KBw5UX~%-aK?jGSNky_1iPj)N|K(| z=&0Nv2oV*&(T2LX%IV97`q>os3RO}KhCexEQs>?KaTEX((-UY&P%YO@i&b7#tanoS zgmsTZ{yg%F`O{)B>egP}d|J@M2O^Y#)9C5VY2|9&xip$MUGAdK_Fe*fPqB(JvOqw+Wv}eN;Aa^dvn<6N8)u zH_SY(OdaqJXu76aVioB08EL; zp%@?(Fl|thgZ$xSN@4tU3c%D4_XH4NSA_sc+Y6!y2(SWrto|L8&I=qYLG5G~5WGP( z9p1+Xgq%A2@1PTaVH0m3WSI(xH?z|4XFyPjbp-EH4gx`h1JVElB<)hhLka-u!qXv8 zKrs6E`Qs>KczjYt z?HIn|qt%RNMoEcgma})fOca()m?A;?!i<06m204Y{$GWHuLN@)AbkL))*YaXTJ;~{ zY@MH8a(~u)WcC9+sRA;DayS?wASS`jYEp(%uO z&PtN01e>O+={Fw~d(>`7y^X>ht)sx5ec^rlF?*N{k8;B_?*Q?_hfDlL1LEnQ%cAIx zh6+-LU`$xIw_m(?7yy;6u1=Qa2O7DQX*&CPn- z@=Qqm@vqbi3J1*wGG_;e)NaD#yRemx>G` zC1~qc4-$K&Hm>V35FOkW1QfQ0i<*pZyAD@zi~0epO+7089-F?gMCpB!!FSMsK0uu; zJ3taI1ItbWasb>tH$TW2WQGWt4g2-al){@70XhhP9>8%Z(hH!7Xe3j^8>QgifF$tc z40%HP*DnG7SG_`P2+nR>IK~~X@8jpj1sEAg3TZP!beX17F5Vf2LVQc{WrmJdI}Vra=5vpb2$X;3(6II+z1q zuehMbg<*^((^|Iy5)TY8k8CG*F79~B^V<{Tqbrbwv|Wn5{2zOQsvV9ol6uCjrihnt zor}J}t2aYAB|8~ksGRq4{6(sNwmCxe6>pFlm;{`OE?bd+46G#o;4Y9KZy@Qmy2e?F zQMRD8YmA-hQ-PbtFA`1D83&<3Kslk^03Scne5zp_IqLN-HlQaA*2y#JOK3jsA0KPx6z99X&3Qy`sBBS84b*4 zm}CjM#~&0@oZ<<{+M_oWkHl|tr+4=DQaxwvRN$Lw2g8#;UjIxx&OM}6XKxzm1) z1Zhkz_#{D&9^{G_Ln9mE;lmaC(^S~RH+d5)j{_$pO2^FQk7#XsL_M>^bAnocsIqYi zP{y<7htlAE&`Dv1dp8lCq0?fYVN2w7*L`~xPa{Xlk689t7(0rVKrhv-COlZ0`_@P- zJD+*egFS_(G|+yX*5m2rFA*vyKua}zzzD}|MQqt1Dwc|<3piewEsnX+xA0nJ2zO4! zUZfOW=mn*Hqxd=p4-Y(!cg_4*HrLDQRK=4TyPz#Gk5U&EZ;q)u9c`gWMPdz>JEr-G9 zsx9V|4eH6V8j1HVbykDs1XLZ504|E;3ouCx!`r(#_}*3vtyuT$E5aoNlR8~Vfipbq z;2GOsY4te)8Bi@N9*IG=ka@qUn4oNUmd8V1d}tY1W!HU}Q|_2!UOMsFj>{y*oFWKn zr_SpN*h?buH?gOhIESBjp`*bDm9b5(@0Xcmmb!;*`DG)$j>0>2If7%=mHVjP`R>d) zG{8gUA2^X!vAFmQt4R^xjRN7WW}3Qy=ZUnb#3s9tUci>!7v5yS7`gk{{iLshdC8E> zMAOW%sSlmvMhvOa*pV}bOp%~|-~d+(#P$*qVe7zdw@%(C$Lpzs-Ty0@KBAoGY&9zS zNig97XfK`dg&LO)V*E0$5{d8pa6F|BNq1kU&r`V%S%v&|*YuPw$8raot|GXBPsQzr zq0HY42*1#9{)wMte1UX47K>)w_(D)2UI=%$p?`b1#b@r5X0%2*aD2GBa%fjaf(X*T zZ-7!RJdQok6{;Y;08YBh=)ca*weziRt`ZKG5sN*!wtrUHk*Xv7qF`_E6^}yfg?b!_ zSqBuK&v~&>mfFql;G;uW$N7O~^xOBfID*g+5--^P*v@%iV*Y^7;#?ctX1{ea)z|Sq9E4g=<=A(W*D9oMaqL z9nAmhR{XlWp#U}rk(HgSKcfj8dLuv|pzmIGV#8ywx>+ah57YvzM)XK%Y=W?>$j(C8 zeG~P?8n#!uA1l+o-sl%@t}~OIEzpf-aqE#kStWZ z?=8otEJYnt@V2d+JH0BlS)6Jh%pUUW{w+mO<*A#w&IA2xWp8%ykkeRAm?fw4{)chwF-=8xafbEluoUa97bDSC@iDCS!N`)UR zCRJ^n5d1ad)6aIm&*Bz!(=#D6K?bUP5De&V(Q5*?n;eB`f#c=F{u7M(|HX4uUDM3? zdaOK3c;nfe)$PDue^S|g^M=yQ^$JW>Qh#P=@BI&|O%K;6&KyX-E}-D*wQZy713WQo zc;U-kH9(ym;t2l}(DnD9F&>6ELZdm!FCM=GV4Wh!bGJNME1L+tl<=I0*$m31wzJyII>oiLSFAo-LNm?>`6(AOP(FM{V|rR|td{lkfO zufjSu-x+U$A!6Qc{Eiol@te zA#gh-9qxzKm|*8@Dtz~#5#y8NQF><%P=oL8MGi~755lW{D;y3|F+-U}>_S7wshYtd zfs*)(Xen;r5`s6$ZF4t($4kLWs+9lrl|eqW8xL>3B=taNCex3MG!@eM0s(h~cxp=z zqxJb?rv@e`ZVHUlm}W zZ#sm*c8XuO&=Yix;m~W*;Gq^SN~ed9FEvbaVJ4AOL^BlWM5F$Yc&&gF+xbZg%MX$g z^UNj9uP8;X>2B}NP4-doVU^ZeIrle;#5p02dSG5Kjj!ps-6>(iA->{ZM%lrxY8zQ| z<>LWwFTTx!h@>+VqI>{N*EsS(E4m;X^IQ+Z7LvUT{piA06+K}(k78H#H zZIaItp@79)t~+L~9zw~FkDYU87^ThTy;ojZ(Z8`c`oQg;>1fwc2j{{wQOplPR0tw< zh-nzE7CZf>kf_>E1fR{EU6yn&wUWb1ZoDPpUtVoCWPc=9t~EWMx+z035ZG!V87&C7 zHVX%wZO<&jBl5%wP_vKu5;yTJq&$^?(CLa z{0=f1am~M55=nQoRA(VNTsVr@?CDUc zKJSQ8%r+O~yh57H1x$`y1E_8;9UY|h^slB2yve{|Xm9H?Zxd1!@pc_E1$j6OK@3x; z`Jw88T_Q(ywm|T5e?Gnb{!^6pqzs6Y%fk6(cu{E8S&uSaLQ^7!zi8ROjsqNi7zt`c z;tdHFe@p70MA^~D+fNrI^Ht@R2l*M`;i z?2cyS<%Y5*>01UGL^`|Gh9I@vUzg-PSTg14?R@m)obwDAeh z{#Bsf-U-2Burtlw8qv|yspEmsr^?FESLWOAosKTzraKzvdiFZ>&30i&rRgEJtinL1 zzkgQsW}uxV^A2~9-g(nhHEPw2;kD|qnVccVBr%|s!g!i!hzYH5LP4Gdvi8F?Ke!l@ z-s5Ow{q97aJnuH2vfgDY@*sbkr;%wxVqs&s>-@L7p(a9iUEl5nnieBh7P5DU7Z8hL z;aMSz+4546DOsmmXj~=TT_lYQ3I!2L&K>&og%yzUS~YmT42Fu#&WzGWmQ|;Zi>M;Vnl`cgdH218 zZ+tyw`+*i}IJFiqy~g!LgOyDJ{MA>RfV{wff~Lfa zg;>cVmp8iL8@@Zuwfq=({T4HRWtSkb_m#0RX}oy6ty}yEn%3c!VfiZ|ACaTSx$TdW zMX}!mL~}0q-y)0-<#S7w<+EgUdRLfL?2+9cEkG#(`vDDN#)#=tMcCFSWNJL6QNOGd zpeAGeAgpV9!4WZ^WN3EU)3%MHMf3&vb~VN}K($o9(t99$b*5nhyqdBuzfg3fwk#mM!?y zX#-hd755fq7;L(%JwfInhmEH3j+7!;9XTB$Ww?-KPRY%w^0*rZ_tWk(r~A&h1a-c; z{h*F1O>yuP5#Ei_&1j$BA4`bJb5FQm1O7E4= zj+Gs?d-jS!y;b!ggDU51n5?i<`*)CyKx;-+3tFJtfTnP*YZ43IP$f7Kxd)-qwX)=% zX)br#OKa{A4)yW6{0RmE4)Kzbu+ph#ll>dw9n~SyAr~4^tF;!7e8^vE(=KvoJR1J^ z1%^b8?U^-7g->(hD`xeuTF&oZChx^yQ|&dhN>dL)iRQ2UaG=+%V0j?28F zah@Mg68$Uvxttgiz7iD#?g^<#g#!b%s=x=lUzR6d#7<$tEtmaW=rjcjf<>0%j^9nt zeAvz@P;L8=$zDs@u%RPo(XI1KXJqSTMyY#AGSvbh0gU7mCUej9J$+kITaD<>1s)bn z?uDx_2n{W?wU&L?v_J>Oi)FRUoHvnc^R}I@jrR*O7D&PeZ!VGnZJE;6_5Pd{9JmY` ze7{BmB^H`CAjRVAFG3_dN9@HXP_{D2ZEA2eQx@kcNGyEzSr-{Gd%sSAtP$INYqXd$ zBckV~;5PFCZydZ>_dc+H&gH6>v z$yOm9Q1Ap)LG(iJG|HFZtA(?9WDfIslI&Nj;o2nY`v)D9+6EUFG8yhp<4EAWz8wfB z(HHMG_x7r)Wi#Tq+Kdm~kue|;K8^7$I(_eUWsEG3Rg6cP%XA%P5hDI^8Rd~$b96uI(O*I)YdAt(1B!g8@Z2r_&vV4%1<`6eEIVT{y zQSKoakU?OCR-okanjF*wDgm&}V0Z}c5cy^aCue(kA2!*yaueE$wRt8!yCsKoe?qt za>U((O<=r=y~Zlrml6eys`1LGcLm4>VWnIoqhaM&_ns{TQ^iw06K&QtjH6cQpdB#v+kt7jWq|m(~dBlKOsbB>Co<8$Iruo?&i{o*v0$OwTtAal_ ziZN*49|^do#`f~dU19iI+xYcN_Io{rb%^qjCM!zx$LAOyTq<4$#+u_B>btQ^(&~AL zD-?__K5r(;!FlamywL3rQ9;sLj~Xsr_Y&z?p&8JBq>+|OumNCZvvmOZcLQ)6`oZ1Y z?yIU=!`*kjp%ZlEbSyo@xH24K)ZBZD8v3LCQxo`8Sr+d3hdq&i1AV@_{+ z1&9tD+COFDY!??@XJOX4ZaycN>LVL9*D1`}7n-;3#>~QY2gIG=d=v&n$4-QxN16a} zSk9!z*iiz?q!0k(ip}$TzI-Z@>!A?vXOvc$yJWoXCwM@RD$L;KK35NA67%CZ@$ycg z+zl+_1}!0wc&fymop=i`fXa7oD1T>mJ<-eGS8~()qz- zCms%n9~RD5v_Pl<%6fn;_Ei0&tM@lMHDzmNY4RmcaiQXlsN2T*&IW&Ypy^?fDSB}} zKI;AKdan#A2kz2+3g^Y?5GoVPsE3Z_T3h8Z=tn(#XNb7d&Ud}bM?o|^L((MFj3fW^ zsG0S)TRXc!O4?UO(J-_;k(JV@gjb*pe_=M)5zMUp@fudJD=f`qi!z(?(>+sMx9d8_ zY2poVZt@>EZanIP+=EZ>)k@+4Zas<-_;`+W&K8CzTU4v15t__wkJ~fl60ubw)wU9^ z4nmkx?}j%JLdQk>)-2wf?h!xhk=r1pLcJScObmp>MXsDJnH0Ty7?|sN zryx!+@^VXCL07$naMX6|B4h7qTGq>>#tW7wJ}3&=jg?7gwQ3I*l(`x#NUX2lew-4K ziuzFs*nA-H@XWglB0G~d_ziHk2+c1qf)nr!gr*31jTlV*`Y zQAQ#!2;yzer3fbt z*FZ^#Ps&c60_;!TGmGkmfj**Bf(|N0A3nNO@>IYqkzu+0L+0J=C149dk>S>|ct9kz zMt5$8*?KKGeMCxvJmgLIu>Id(cbs0XtatCMX=<5j4-B>3`K3oe)jrLO ze>+D21TDiNYS|}!bI8yW7QSxEpJWo|*0CL~`R0|U*S-Lvnsw1WS5pH&Wx2Y!p})R5 zM3|4z1pOM40*DP}@OBvK^>@%Z41CB_=I`gWeZZp-Ft&f}G1c{`j>sSQVY}cg1t6%W zwu-1QTi>z8>gnO3chSmUZLWXY5*P$dC7Dq+SEcDiZR+@%iO}^DiYi$zj`=8VV1D

k?DG530+4eph&f>^(i$tl%zN8K|`SrE__61ld z0Nj(sPGhJ>A_JyM$|6cQ4*H&_f;lds5J< z>`*L2v}c7q4%A z2o*ShW)ti(d5g=PHLy?qEOli9gnmqar=qNDIna>YuIHK7mGtm7^mEAlfZKE+AMrfCId1~rcy(Z}eOC|4eA6-)nSdO@SkgW0!p0b)YOF^goeJiRzYOrGgIgQQZ7@Jv z#U5ZgTtq(yGj9@v5xCfr2&xy+DMQFqjNOe9(jBZ z-UVV;)LC{RV>7KlOFJ_U57=_Iu=Nw=(b2A6-R_P1O5$dBoGkQr@@|xbfM3%}URt=! z__gc-Xofkkds=i5=9fpjjOW1muhq%d3VFzt-9Egg6dgT3N4BJ%Kch9DDD*|Js73lA z#WMT^k!P-#%1J&B_=R_m=AwQdC1TIuX_Vx$9fftiEyzlu2g(xmt` zGXDFr`5)&0{O%VhmJp><%_3^FL{==k=?l-%L%9@zF|pa)X*Ge0OR99er`4MJsALIL z#BoXJ`)XhQ@J4E6v8f;qVorp|YE!qq;*4;(hna5o$6d(@F1st-8O=()dd; zGFZ>@^{u;iBfjNku{Mp!Q)6oup07=Pxfp5Zl!c;Db1*n9=<%)$#WQTl?DQ+Fd!sy?O=ZQ< z09EApqH)*5b5!TM&iiPMsYV#-f)-v?e^zLHn!P2~3UGBl!U|b2186!i95c=f{k46y zZ0hWAd2P1~droaOR*C(Y8YQ_AOuGkRhI+}q^f`B8rfWpHQ?1bV39)ixWBO9g(&M1) zs54Unu9wixL)kIej=7T0iB7gv;nSwWHL!! zH_{>!+#H*ubit+$kwJ{1#L)sGu~3XW2}t#J7rZR&Akk7u^R|iKK~=3w2PF{P2L1(R zsS!*8)1Td9wKlh2Q#(@@%5zauKA)#6;Z2CPSGh`M3|n|-;t_erEA|U@Qd9;Hb2542 z6{9U4yc5y$DJBKlU>K5c(wN!`fcQO@R97V&e!n|^rL^+v`J!WqVky}L{8P)<9=;Rx zI~w;uBfDAmYH6}eoyK6JmNdCw)6a%@ii)hmiv+-+XCvhJo2Fx7ShK8y)G=)gxNtf4 zHYo@2ZWpctKs=&@6S|jarJUVjDa=cmN4f`nUmva#cREoaWpibiH2<` zFY{Z?j&jGf7L#X6(ngQ7+%LAmZ;!k{nXQKv5Vf%~(t4R?oB3$Rxw6KdVedXcgCj*I z4AwpOkJOzsPPo{D@0~WgZml!l34ip=OQ^hX9*K&Cf0l1_z1-Zl1}W2S<{uw<4jj@d z76$Kp4addoyKd;8yiv}-$qTAx`S8H?nl|6DIZ=`EC>fobuENS_=QZJqJv_Wlt*d}? zSZsO)$N5ss_0yQ;)Rh;S4~lv@t0L~i+eg$1a3x&PzVS^u_`JHzpo^_c+Ct=IHGpds z{2mzYs$NFolvW^zbVJgkP`l+52X@^G?lGKA*YWihcHD|31 zznGf3r4V!aOjJ#RqQZ$BMZvU&_nA6I&r`>B-KHbhsTFI@bBp(S8<3In%t*Q#^T1>Y zfV#5dqcltfG*;CX_NAjB#{U?2AwW8pB8^W ztNh8>+V9>OiuN<6k9vxiHQPs?B{>P>g?*Nz1)eljxZfi$1|Tt1sHvHT!caLio*{IM zg!Q767rm}PO1E*^HBiuQM^n6*YsV&iu(t5Qjpjh%#-%q^p=`JMKIFr#M%r_U>jO?1 zI9~b==LNj5Xh)A5js!0l8M@fV|E2kx`Te_fkch(W?fZ5aZ%_*ymNSv=?WKs>sow0? zz8eiD`ZiN=e>0nicJjc?w%ZKwc7Uw~*^RLvhc76l%o)0yY4nq3K$z;1+X5C|GX zMX{+djX0Lfc0uI7oVqDmYVEbyR5m&ek^^ zP-G+#Q;v<0!o(rF!UZrSenvnE%&|+=z<3~73*hZVJ^vYPsWhuH~cEd}?`@l$%iaAYNAm8O_tg7Qj6CuA3kC30)gGv!c z$z9fL)x~ATHCX=6e49?9QMtSfd9KP^`4B~2Uu6IZ|`8_x=x0e5Zf=2-Ar^H%JW>BUAp=5o_%^943%W8NTb zZhWhc#7MFj^6s;zW;S?_-l-1>s{t$4wp}pr4_)k{5L9c6REQ04=_Q=StJ-WF&n2`# zX(yd@v!Jub$T^{o)IC!WMr^PS>15!U)~XNomlR)y%;xmi;b~Aw*12(RyQZ-qQ5+QewYVXI=o^+_&3rLj*7aSg&5oO1v^Gh!4Ovq7uUohchuVMlKU?l|-%;s|4P^ zs4b$Y=%pP(mqUB!+OClc@|Y2Aa_ZE}=;d~?<~^m}5OBqQolRu6Q|6k*7{(zyla{+G zuoPB#ZSqJFP!Q(YR?fPKv=Yi#ADB3;z8Z;_&Jamko5KP0*a)7}(&e$f8dPUu`PP!b z%oi;YDKkaFK>%O?+-iEM8X+J&JokYxz^LA>KH}Ia{d9U}Ren(LJ~o0f+a&r3cjzs@ zkOkO6z1hoVn`937%tFG=IzP@?#810; zOShhivv=a4yN4D&W=_TR@>89Fm{GIQ%M32vQ=3E2oo;5V^+r!~;WN>_pvh$<70kmG z1!3y;V3^WEQ;6o*60c>55X|2R>2P#BdDVle*Sa26XlQj>;{$1K!94<5LIpq;ov6(+ ztRAq-%jIMC-ac+Lm0lfT^SNkHUU5M^RF~;`&stnglbDzoQxx62_h6G@wk(ytV0VU* zw6}-qKF%5cHJ|7}N|A97KSaf-x;^HqluA3vBHLWu&h!!l`v2Pd?x?2THQgvE2x^pG zgeYCA^iHr*L_k2PLQoI{r1zF!0i;GyK#?K>B3(KN66w82CrIysKtc(D6z_J{+%xB% zIdkss&N;JY?plX`SPS;v+1dO3zPCKj^S;I;R-Dp_FG{REFLUxuJ*+tU)hfz>p$$J4 z!00JL5e^XJ$YsVHp(5lU;Gb$o12Og)E^%60x^>ZLWks=9PjyAhMUIuXF5f-kcsbaj zm)?%uTXQ$M_Wm0(hwQyzcply|TTaMB6o*_NX}lDWsDHC~?>)u4@8N+W1hZ(nQy=Kr zaRC`Nr!airZzjN3mXvs_r?X4YG}wGj;{vsJw#u-z^b~*>ur#=ktN;ys2LBi~i3q$*Q~A*WD|eZtrbweb;*OuNmn zRz+*gxJ|fSC%TSq)vK_HbAWY|M;o{dmYd{3wjdsjtDI&(GmoIxxfRFNZ1;5;x{h%> zJ-an;@^mLrR+9^;w-{_chv6#W>ga&i#?`xWu;U+H6i-Cl+d6nS1hreC5w3k283dxh z|4L*J1o#Z8)up*>Dv!phiZKdfT%m9D)%!+!3oqPh&lXI78}{MsN|=Tg(Fx&8vZ9Wh z!j~{AN47x169)1E9y;AEz@%o+>3V%`QPsRue}1_sA@MsXH0s|%`oAb@fbN{6LpHEx zk+pFK^>YxWrj>R0#o^WAk~g9A7e3!D&*+tX6?_it3_7$UFyPzk^N8L+Tv(yl$~iz5 zMA6F&4@Hol`+kyDoXowwZ}cpihyP0E0-zXK0|X7yS;8{`mU?sPOZM}sPH;A2c%F^3WPxGQS&wMQSk`+$Jzp$4{XrjIFOyoEKOdg{@&G3bmmtOAJ5?ASFI7z|9 zvckoG_LQ8b*Q&S#+A2VIf9Ihu@)wBb1K0q!9YKc9^YG5L%&`4cTzb0XY@KJ}DAZRf{0*-DoDGyv3fVi2U=D0ZBkT>}N&?yA# zN_T>5k3-iN*r;3}(jCwT(J;{5F-^w5W=nMav+Rh6rN@Y*)}0V-9No)(L`0)fLPPsl zgT{*dy8*Lg)<-`(@NI8DG z>nO!;Tay{lECn1T1au@o;vA`DKpwUu0Vo1u0GFkUTFWK_YGM>Sf*b?r0Qi74XY76? z@(XklvE4TN3&i(Zf<_N0@ZM9Sp-q`2Jg}#9RswQ^KX^$9BlTzXEZb(I$a-=n_x|=t z%#0{fC$^!HOZOgsm5g$f&@5NT;V%7LdQqLylqN(O=pg_rgT0`tZ$R1i;_T+Wk8EO} zY=7Wd`H-8~Johc>G;!NSWGaT6++D;0|5e5%uXOG8_ok!!U4K;%m3B$OtM| zfS<(m+d~#<=%2SYocQJN6|7Ibt@eWMY`8NhJvzmism0bfH4_iSQ1zkO+%t6w~9g zR|V5<^&V4pTO09i?v~0$$#&;oHR-tKNirjC4tzlTMUJ?VMK(Xmu4g5mI)ZirRA7I~ zb2tatg?Qp4`Q}K2#X_8HtICKX=F=(HnSRcPUo$w~Hve$E{%;}}zzV)9TN~vlZK_*eLi0iu;%wnlL={XcyMp)gP$L$*699z<* zEL~hIuA#dN;hX3Y+VJ@$CYrXZrr`@riIN9hWbwYHSgL?#f^ZM zY*yQ6-_%XP06n|kf2QlR_No1X{9A_mx>VEQ?0T&G-M~rhS!g=^y4ora1Cwps$Kfq9EjaGAI#XBk4MT1jVt}BN?a^Ux%2|0oM`6zr1418 zy0zwatnXZ6h(?{g>zhFU3JZ5BL{7>XhSI(E z7ApLt2IL8z{R=(B3(+0N$f0UoS@_HwAj+xW|9>+L#%Euod_0GZ;eUm~?>EV#_+q+UkVfVH; z#tvA;*6X>6?T6OEO%eJkM}0ZGk0W~>BW%E`S%X=wL zVQ>B_xc!o>U0~&ec1UNuZ*6^Fhe^rKD6!Y@)?)1a^yC#?*o_b)Kx0g)8G-YYVZ<(y zcF!k0*Bs(&ljH%tz{W$f^t=EH;bJxIUe{t)M|ND5ibwV+_t?S14-O%!V*n)V2;_SH z@nrQyL#bOsaAARzAnVN`aSvU&ol}bsJCLbHp5W5An%k)bF;nw=iw)X>1G@5qBuqoa zTcR*wXcXJ10v`f~QlcD7PrezI^s&BKTD2{tiJvNLc(N{}v7t@NR8NDx$zvck+$?3M z<@;#d?o&z~yO2*N?q25MMH?W(D{G$m1=1-M^_pEk8ICf7@iIWqg5ajq>dbj}I_ekF>(r(BsH!Ri5C$}>EDrVs*E#+B*)dSKaDq4dFpPX{x zbpZc7{Hbd}bNT5!!kg}LkauT-g9jAbNmW#85aMau7A7KK@AQL3{V;v+QHQsDQz+3u zOsg}Ki>QMdA$ne8zwX9+d;rr^fEwzf?PoA{HAu;Z&1D?neU6Lin2B}XOKgJc*2RFq z>ccdU;MM1~x*4b1&(g2AEI1n{8u?qtJpekuYQLg3!KLwv0QPN%V4WaFZv}sDP*=#$ zT`%v|W36N}>MnCrdle`4bgllbtw>4i6Bf@?CDoGUx18H&$xKHaA1SGM0fS=t3s$RM z(5Qyb?9jxg#Jeld_Cp3#yDhLSU575^Qi08ipr8xz>DgG#ADARLi&EoneDJeTg2>F1J^tH!ZU zmL(&_@R@`gWjMvAOm^b#0y|b253_C@@=?uyS~vc}FU3s5;{3HAIs$?MuRcDReO(XQ zHPd5RHPSk#XVVg=*CmjE2JQ{0oh|kG0Hvw*BE;IEi?}YUas#jq9!cr}A@qk7)>QksA^u)vifFcI!PvM+ z$;?}adq>|mU2YZHT#Y0mi`34StZNA4+`<>sPd;8Au2NS(Z;dW0n=rQ*)spE;INRFw zW!Z{02Vyk1F-JC4!0vI2_|!kA??TzlkMcQ4ZhOx8s`6-~&3ixIAn(bqjp*BE1A)Yf zkFI63##ZT-l|N?^f&sM3?LC5J8faH}%4IIN)-^<1F={J&NccVPeY4z4=Tz;_KNTAC z0vkm9iI4tU$rC{p39#$nic4wd=5d~F-RXQ(A8LWt%@bJcWlFqJ;Mr2hsx&wL#rHxu zD721rY8}93GxF!ugQOU;tq9P zqCw`+hrmR{5bcH<6ygkJ?t6yF^r8>QWESkTZQ6IH?umNC}>)&WJ2gzdD z7DWG+ApKA8J~`p(U0o#}o^N9~8(pBpa_r9loPU??0igjaeB!HQ8A+XNhbzTa%uQ=p zr7N63wjj=l7mu&h4(HgG-On^NDU&griJ$P9)`Xa@1nBpa|e8k&w7T3k)*?`Zirz9cy{KBm~}y-U5nQuoriZf#LalZClg z(TDh(ppZyck|2B>aq1~<_B#^U$TLFbKgMtP1s_-VZu29R<3-u^OSaihpT%_0e-k;5 zAl?PgBk<8>$O&Nh*)%|C;JX8a_YBXe3|9fe;GaWLNDY8t+yV^a4x&j1x|S0H7zoNg zlN<#k2clr&WjYEA^4TxY!PYKdRljw};lPlNfE?ljV0^C-KXiydG6CO)9x?&8_y6M^9#?FV zB8oKTXmkYQW9GGzXw%fkGM&_h+f|cvU9Y{~=ikue9VaPLd#K!$%wk{thpxaL(*~Ev zU$T&&WpdV*oxZXp;H)d%k>R)X&_JexfrSSB`5(!D{#Uf~ul!O%#iCdn6mfdEs6C{c}Pwt|MX} zah|AZJsgp_DCVke>=L7s9eyuo!mih&z{;u@;&=L$4pKmmZ?!QdE=Z%@pzleb(%S*X zsH2g43kTFjz)vTvaUlQKiM%g~;^a?iqk=1!an^QUxl6_paNb2mN2TBY1H=G!+36c9R$t&Sop zg~_j=Q3!(Y2K4Brgy$etJk)4?0reEBeRzRx6IlL2nRD)>D4^ABH>R>3!3C4*Y>@!k zT_Xo}9aE`O8nk$D3z&5cT5Z@oS`~R8*YIl6=Sqv z>BF0aPXtDFzA{f@DR5zp2L^ZT%X|)i8U`$qEWK|CRCzC~L73H8KN1g`4|v_)l~{E6 zpIp%jH~^n3{5gK}5xPavtF1~CY#Yz31$y$~9a^^IyxW5W#(z=8GR6#3o zAw|WC&6C4aK;_V2i(iUVV___exRUdH$T(hxfypmb&Ei_1tC=yw-V7BsPChdLY>t2u zqCZ3Fhl$k}V%7Y#921kCweyj>%F1O0F;?mRyI;7&X5PPg^qz4M@UGh8A{blq(Q?*;tVVe+;R@#`{sNgakmT-6T^A>?t$k3f*9pIIs6QdCTXwv2 z3&ZTPp+}EiDaTj+$Sf>pzd= zvVKhPdeqqE=^#n27B@IgZdC<|f?WHW1mgopy;3~mdHUMtbq%bCmr(^3<3l}N5OuT2 z{BE;cRmm|F233+*`+bCsc;o z&Gt|`MPIC&Z=c)vn{?!Gtqro`J4t;LdmwOYH{Z61boC;0>|D_JYC=9kk{v_&a*UIr z%OuR~QDHR!6RH}UO(!B&|G2sy>!o}$BM4Iqpal{(xwlW=IOs*}02oN`V!W_4&@KZ; z&fsgxzK-Bbo-+RV0V~|apV>+p&rR{rU{=l9m@E7F!=R5OR8<~opV4=rw&YTAN}oyV zgH1H5JtI?q#Kq+nb2eke0ij~jTz21;w`F{$aX6w@jIm{3@MRjM*e22WXJK*DOle^w zTFeFLESHaDi#5#hn%qU3&sF{suXm{|y<}S{? z195WC6?Pjx-i{4kxP25iWO!n%sDiHYrv4|nUB|a`83Fw3_p31Ng$B~gGHGwBP1W)i zN;hfdXPE+4Ru>RWs3@-47BI7oYLB;2GROBF<(_i-rFfl~v@Hgb;eodDOon{BS`xdA zqoo3lI;sadOvb%v#0yoU|C$1Ck z=q6zkiAMW1VIe2xp%EJAha1k!#qmnWY7H&y6k`{=FuD#7Y84esy{(D50^_GwMH9V} zSI*R_Ry?C*0b?1ZB4#U-pSynodYs4T>lys?!CH0}{G4)YUp{zu7j+K*wPUIx;gG7pd{r zhaK(65z1!Y5TKJCgd0Rpk}`Lnolb4Tc!N=XR{X*J5zkc_Hb7e$&wuSZbC<)#NAaKw z`*@Sm>>qxz>Dn^mSPPO8z~46rpwY)El7(QnIE16idCaUcwV&ibd8N+|tXw4LZbiK( zskTFz@eB;J>s8g9MtNVi4n3#FoKB=T>UTcyFwdbqh=C^qX*wt#hFj+t)zA8j=tBT)Nf>G-wo5;1mM31_Jiu$6s zq3YHJUOrv)m}IWu<1JmO%`rhr{yZgjAWnK|UHdk*joQCKN=72kYO{n2q!d1T15!s& zIE;kWbq2e7b7q_%-Gukr@DT?V^!eLo7>L6Q;K-vbq{9=t_XcN;&3nVC?vmy|FO)$7 zV*N@AIKp+6x)pDaF&mjWuO{12n^D{io^Gk^ZOAEbo62~k-c#?qtFC;M0_n+jt74gc zx*AF4?IsbG>p4AyGRt=1p_y|;<+<(1OooAGLmEpSSzR_+v_Kqbl5^PL9A(F_4|ZBH?@*vQ)rvzF;(UHR%H0l;M=O0C;(8*QDnc z$S7{Y{o8Q7a|HiH`>pZXd5s(6UYUSuM`SIt^5I(0Xibb%gYd$@=}ws5*po#Wj`sb~ zqNfSvoY%#~3dQVWS6Ohxlxvvr+m}Z?)V5{`?D(3vn^k$(Q>0q~*&V6s(PhuI%xBQk zxl=C^O?S;Z%}O<=Yi!dt_B&gEm6C!rCf-9jP%^0}z)uk;iyN5`%|?`(CHY03>zgo# z>DA6YYGH4P>ooFPzp9yf&;e*EjtxbE4h{16@^@YV&UIgFJnOCzv+OV*V7Fjq*}W0w=pj3uOrr zUgX}0@yu#W@qQ5O_9zvz-u+H;X09vDyu^?GJ8xC`efvU_r9{l-d){&H_NqMJhW{Br z^?Pnpl|MOUU=m5~XabsxOqt~+Z%ZW%M^0y&hTJ78<7OAlB4I6%^JFQPLdzZ7gosJ( zeIQzrDdAFBADmmHJ>M2gXLOiF(m^z-YVe1MlFXttIJiqWi8gATcx_%^F@ay8PYEKF zou|kQ>5QC@Pg8$)F9aNlaflG*LuH@vowSbZfC0DXAw?ccjt>Y<(ilPXJ9RcQxnm^u zzlK!(0Wj?(ELu%AD`21{H4|VDd}{mbq|#PYZB|_#TlA$f*TT@V38T!TEq4rG3IOhL zjs!HubyG8H{W05XkL-(9p zw07k9(kB>8V_6ZuZ7_n(t*Ebgt?CeQQ)JC^BA*}s0jy3`C&`mJiA<7$M9;Yi@lv-P zAykG|ZN0rP#XR|%g-Kp_EgdK=<1x+fzdSOv4I9b_4b}k1g?1oYx5TTN&IpYa_Y&|* zts=`|5vK%6`En@fII#6wC7Bxnqef{TFoGYIm!onS3Su)#kI0CUh*I+9ffq2KW{XP# zc-w_I1Yvdx=&D(X`htaz5{jvgnJb&KEAEFitA1PFsWxMN%PW5SQWM52E6Zlv_964a zNC)CLDKJ18&O6;v8gP9+IWAZ->+@s}ZMEmm8T74qe1B}nU>lZiD!pkTh4F=7Nr~a3 zh!C_gJTeojQ5J`+7?U={!*CsANAc--n4{4C*M;U&GZQn-gad`~^Wq7H%=u>8TC_6d ziS+!mOlLfWhQF$7y-CVw3#Ds8(}nxIm4h&@1;LS@EVp*VizE_f=xe0?rno?b$qsf; zy9D;-ZW$P0qorfn#qTB984XubV1{)=zAoIsQ`{3aGb`u?!Nki8zM^Yu-y=+ouhQB+ zyPBntK8Q9p4d$e>&+pYD1O|(EZc3yMy za<}pOhV-s*B@@$ULG zR(bK+_XKoJyilcI%En>Z>+kaz_Y*tAJF2MTfC|7L*WFgR9SH=Ld-OTc91bqrH{z|5 zUY_Sn)l}^LE*mn$_5_e=>niu>U{t$)&ZKks+{@32&-gG1q$*T~7$7&?{`dsV{ z*JSVZJWxk|5L-sd0rz$K+e76hW-Bv$gT$REg{iO0e)g$=pPEcw0R2@`JiBvOSlY zZS6Buf{>qAGo6oYF7$a7Y4Db?b3gYj7|@v%njU>)X6{1B3RJ~zWq0*?1av1oumZy- zn^kA^Tu_dAx)x3=eOTVEP4O4TnduYtr&nnS6tVrrvt6P~$`7+%5|@7F)KoZ$!MX%P zN4FhDkx>KQ^bSfj#+l7YDe0x9Gt_OF{ZOODe@9jSf2F_wU(`Ey=n{#q33rKS7BZu_ z0Qkt{yF$!@zQVfH&9+J;tv_@2NnPP6dR+^1nq*F;V>M_0FY^I_*YO|kj5MuK7pch_ zLrONBH#az6(`J7AxjwN$lSDxRwHYtF?SrLHUHycR|pHv)2j6lpHAyQ41Jg zKiLm~ju)jT5Lv_+-C85=pPS799Z$(VB;U2+_;+N&;1{|(bwuxXMqMGg+)~=r`=WLm^WE-Z*xYGVU|MGm{47}j z=l^{>(o(y{kj8>Tu1|>PlXg?`ie2yP?v`yGY@*@S?;zj@`!}ESU%rF0G>h!tQ4t89 zN)8{XmuS#8HV3mW4H2dhm~LHWr+^$7k&(srM+G07DO$Q%bok@;$p^4AhRnQ{FY#JH zmCG+g3~@H^1!N(!O=~a_QCX7g+hg~*!kg3to~%B(M4lKBsyW)*?&~fE-~`%5YEN!Y zSD%<4icB*@p0ol~Jcd5`0>NufZ>O`(((4c3|$tP#h~ zsl6&&geJL%f1me|_G})GWu8qEM0{=cEgWl=+tBYito{X(30lAMhg|C4d_jMc-&QL& zq4Mm{$Z&u8Hvi-G^`|4j?|`;+hLhgr#Mi!gAy0^c`CxE}EiQm*-M_57JvB!x4Sk<^ zC+%62MdN*G?L%%+ptwF6Ln!#6p=vo3px27FBANj4KBLv|%AakAs7|Cy4+E8E3XkRye0iWD|^S?@al+&Bba#pgP&y;={t!QzHVQ z4OhA}ymawGN)u#?fndm1`W=Z5ZfTkS1!^T3e`y`(&{wtUYVLBEycOYfmfcfJji1R+ zgl@#=AM^J9&Z`{bX>0qI+fZX>(sBA+HqeUZ%H10@f^RcGG>Q745ac2ft-TeL=5`2F)TrKJnzEx~Ow@G{?>)Bh=I^}uKb?Nk|0P*Hn^*%hoC$BZ zZl98BQ|2Tehpnef+9olmWEPYGb#KQO2p0xXrWl_IiR+SA z``C)X%kI7|HGb)Se)|W@iyrar=3E59mWnsk8eagnvY2Bl+pnP}QgzUFHRwQX+k|A3 zPWtzaUJH#8N%FTu&#(DwiHdd}Rac7qO^5KOo%?UyJXnP|g`=5w#GaZr1>mvmGdB@T zClyi>7F2sU=}9>K*sM8wui`LHb5Bi(_%sGp<+yo+Y8K@x4~|37xoUS6Se23uee^uh zMv3+}IG1aLSVC~I5};5{TrlAo@g{yzhpV&&$~o!mRu)-X=CD)h>Wwz%KHGq=KcRbp zc`Qtw>F@JOf9t#b!T;e);~`aQEE~k zgv&s+r_B^$l10uTiTE*7Ij<9diDlU{FtWfL0Iz;NN(V24ADsn``h5=K$F>xe$peAE z8)^`_6ImW<`wJ9t2WSnkc8~(_u#}+00*$=uhu0^se?hc}z|pgN!9Zo{MH&Pc0u6jb z)@IfObp8$0)%{2vsQxq?fT^O${6Xp+!ktYBt{YNaL*d;4O8mk=L6TK_WDOw_EaC?y zJ_1i!QIUZN+-m@Lh|b3NK;NjpQKcixL}H77fwUV?MBU%t!f0Ow(#_k=^ZcJ*;-7Eg zpXT&`tcfVw-^r$ntIk_BZIU;}(=s7~%RPcG+LZ3|SwZ8-*;JQPH$z_m%P>g02{{on zH`^=`8WI>H?|y+edubp_&BV{$EzxlMM&43veBZM$sZldF=?M5aTF`jErS+Ou)>wgo=U(Iga)o=VazCJJ?>~py}F2Ot};4 zC>d4hOb3KI;)yIL>iKcW6SX5D>I&-;X>o13R#89F?@I4XWZ4$oi$4PjUCwn*h1cZ* zGID*iwK;|jKW#YxnNGgUBBMkQy-I$3eERQAKSKKO*#ugmDqhi?R~dI>eD-vMC*!-h z@rEn#fp;jBIKw^jr9_LzByuUro{m>HKjm1R~`XgcCIwFwb>wm5#|5o z1NcLcFXI{GAytYDfwA5xVr}Kw%|u1stT(R^zyR(49!Gl`>)#6J#Z_k<)kIZcRdvdc z+dpTg1snAC{V#(1!!&&G9b^{fxnyKvCO!E|)i_)<;NvypGm6cAQaq4kAUtv zmxPVlKo4rd5$<5Lr8GE7{*2-r-E+yn#3{qKGaCozdHvq8lqPj>o@C=(=J3?y9meq! zG>H(B1{e#WBR-xb#%s1Owg%P56yI`qvdwo56ys-b!}-HO*Giar!GF(#`Ge7k{pUwf zi%fElMn3zrFLz>OXTmgSagJCBU}?^jVkr$e_(+CEiZ#wLY7{Zgjpw0!`&NFfsBh`^ zHWy#G-P=cJnw3mxh4et?ICh}Y5f7_c#aaaBg|Om*i2?A_0fIl|^+>_y7FC)ho)O6k zm-1|{g=tN^syo(be5{8$UBeZ*IZv_z7#=0!ZgQayrzMixSFgUm6EC*Hcww*`f1|@Q zwoQXJx`;_r5&YJkW&q)gnnJX|s$r38>Nv)E$B5iP06UKf-1ddB+gMIC(Oh)>Tq59m zugAXrid~I!mXszW56Vd9(%=b^$7QzQL)^6^d{lY(u>!UOga9T9A~yY9$2XAA-ajBb zx&Ud-3LqT_gN9&rLUhsYt?oK9ctsw|Lt^$shculnk$kskD`*Z@8 zPQxO1~G4E2gv&600QC! zP)UpX1^ST|-~yb)4Wa4+Ohq^brRjP>Srzz!0B z6KccD0O}(*5UnZU#;I%yQ1WYlBq;$<>SrmO9T6y88Sww5RDetQ1knW4?AH3#fTnTb zaeyNb_^qn?=S}|eN&eF&|G9Af!CTO&-#Z!5&lDh8^UjuUhwW|dkzi35-$M?zFk$Cs zsSHsIz9az~UZV6oXm)33_x|mk{g!?xz8eIvIv*jXT-skiTeqhyt$Ae+u;k0x;=+(= zi`Sc*DC45XG#_vWWM&rvBF4nIHmv}$ z`B`A%);eMW1txQTBzko`8+7uMo)SwS>wh`Z@pEaA?)ra{R`3tV?^*t!H-w}ia3(P3 zbr@Gj;GE&4$vi5;wc$FRnRLUt%cnSU*w`F-q_GvvBQ5v7_C~Y<|Es$+Kt}w`ifn+} zdN+p+COjSwQKX9q37(^9ZLLrzQ|!@+@BF&NR$q&YvkCDW7r)_|2J#3JikZ)hLf}jx zZ0iuNOEn|RH_9dE`v5l3lUHg#tgOC$rJq{11T-<`cg< zIiN_igbjf_Ny6Z?K5H42(8RY|3p)4!M*Iz=@DJ6){$N!0_ZQy&|NWri{I<`ApQf8~ zX0AJh3NQ&)wGMJ-M;PbuS3Y?2HKIwZ&g-O%Y%1umh46q9T?9WCu-yh_CByJ%5id0q z@T%3=ha0!qT-_3-1FQ4i3)yJ5Tk$hwY0~&ij{BPaiU< z=Od+&N9zVRYmSkcfLn~Tj?DE>blq3Qk9iZaDUoCdNz293z4eT6TWaJ}a%gsuuFJw1 z6^Z3!g=xq;4xfi>-sLwFECQCpK%hVqM3Yh@e~8YgT<&jioHJll8!c$LMthYT9$k=B z&F^;NiM3k%5w_u-0(5k#Ad>gh!Ab< + + + + + Text Generation Inference API + + +

+ + + \ No newline at end of file diff --git a/docs/openapi.json b/docs/openapi.json new file mode 100644 index 00000000..4735c8df --- /dev/null +++ b/docs/openapi.json @@ -0,0 +1,446 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Text Generation Inference", + "description": "Text Generation Webserver", + "contact": { + "name": "Olivier Dehaene", + "email": "" + }, + "license": { + "name": "Apache 2.0", + "url": "https://www.apache.org/licenses/LICENSE-2.0" + }, + "version": "0.2.0" + }, + "paths": { + "/generate": { + "post": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Generate tokens", + "description": "Generate tokens", + "operationId": "generate", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenerateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Generated Text", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GenerateResponse" + } + } + } + } + }, + "422": { + "description": "Input validation error", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "example": { + "error": "Input validation error" + } + } + } + }, + "424": { + "description": "Generation Error", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "example": { + "error": "Request failed during generation" + } + } + } + }, + "429": { + "description": "Model is overloaded", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "example": { + "error": "Model is overloaded" + } + } + } + }, + "500": { + "description": "Incomplete generation", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "example": { + "error": "Incomplete generation" + } + } + } + } + }, + "deprecated": false + } + }, + "/generate_stream": { + "post": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Generate a stream of token using Server Side Events", + "description": "Generate a stream of token using Server Side Events", + "operationId": "generate_stream", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenerateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Generated Text", + "content": { + "text/event-stream ": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/StreamResponse" + } + } + } + } + }, + "422": { + "description": "Input validation error", + "content": { + "text/event-stream ": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "example": { + "error": "Input validation error" + } + } + } + }, + "424": { + "description": "Generation Error", + "content": { + "text/event-stream ": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "example": { + "error": "Request failed during generation" + } + } + } + }, + "429": { + "description": "Model is overloaded", + "content": { + "text/event-stream ": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "example": { + "error": "Model is overloaded" + } + } + } + }, + "500": { + "description": "Incomplete generation", + "content": { + "text/event-stream ": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "example": { + "error": "Incomplete generation" + } + } + } + } + }, + "deprecated": false + } + } + }, + "components": { + "schemas": { + "Details": { + "type": "object", + "required": [ + "finish_reason", + "generated_tokens" + ], + "properties": { + "finish_reason": { + "$ref": "#/components/schemas/FinishReason" + }, + "generated_tokens": { + "type": "integer", + "format": "int32", + "example": 1 + }, + "prefill": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42 + }, + "tokens": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + } + } + }, + "ErrorResponse": { + "type": "object", + "required": [ + "error" + ], + "properties": { + "error": { + "type": "string" + } + } + }, + "FinishReason": { + "type": "string", + "enum": [ + "length", + "eos_token", + "stop_sequence" + ] + }, + "GenerateParameters": { + "type": "object", + "properties": { + "details": { + "type": "boolean", + "default": "true" + }, + "do_sample": { + "type": "boolean", + "default": "false", + "example": true + }, + "max_new_tokens": { + "type": "integer", + "format": "int32", + "default": "20", + "exclusiveMaximum": 512.0, + "exclusiveMinimum": 0.0 + }, + "repetition_penalty": { + "type": "number", + "format": "float", + "default": "null", + "example": 1.03, + "nullable": true, + "exclusiveMinimum": 0.0 + }, + "seed": { + "type": "integer", + "format": "int64" + }, + "stop": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "photographer" + ], + "maxItems": 4 + }, + "temperature": { + "type": "number", + "format": "float", + "default": "null", + "example": 0.5, + "nullable": true, + "exclusiveMinimum": 0.0 + }, + "top_k": { + "type": "integer", + "format": "int32", + "default": "null", + "example": 10, + "nullable": true, + "exclusiveMinimum": 0.0 + }, + "top_p": { + "type": "number", + "format": "float", + "default": "null", + "example": 0.95, + "nullable": true, + "maximum": 1.0, + "exclusiveMinimum": 0.0 + } + } + }, + "GenerateRequest": { + "type": "object", + "required": [ + "inputs" + ], + "properties": { + "inputs": { + "type": "string", + "example": "My name is Olivier and I" + }, + "parameters": { + "$ref": "#/components/schemas/GenerateParameters" + } + } + }, + "GenerateResponse": { + "type": "object", + "required": [ + "generated_text" + ], + "properties": { + "details": { + "$ref": "#/components/schemas/Details" + }, + "generated_text": { + "type": "string", + "example": "test" + } + } + }, + "StreamDetails": { + "type": "object", + "required": [ + "finish_reason", + "generated_tokens" + ], + "properties": { + "finish_reason": { + "$ref": "#/components/schemas/FinishReason" + }, + "generated_tokens": { + "type": "integer", + "format": "int32", + "example": 1 + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42 + } + } + }, + "StreamResponse": { + "type": "object", + "required": [ + "token" + ], + "properties": { + "details": { + "$ref": "#/components/schemas/StreamDetails" + }, + "generated_text": { + "type": "string", + "default": "null", + "example": "test", + "nullable": true + }, + "token": { + "$ref": "#/components/schemas/Token" + } + } + }, + "Token": { + "type": "object", + "required": [ + "id", + "text", + "logprob" + ], + "properties": { + "id": { + "type": "integer", + "format": "int32", + "example": 0 + }, + "logprob": { + "type": "number", + "format": "float", + "example": -0.34, + "nullable": true + }, + "text": { + "type": "string", + "example": "test" + } + } + } + } + }, + "tags": [ + { + "name": "Text Generation Inference", + "description": "Hugging Face Text Generation Inference API" + } + ] +} \ No newline at end of file diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 58df28d9..f62fc804 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.1.0" +version = "0.2.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/launcher/src/main.rs b/launcher/src/main.rs index dea6fcc8..3df6e911 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -19,7 +19,7 @@ use subprocess::{Popen, PopenConfig, PopenError, Redirection}; #[clap(author, version, about, long_about = None)] struct Args { #[clap(default_value = "bigscience/bloom-560m", long, env)] - model_name: String, + model_id: String, #[clap(long, env)] revision: Option, #[clap(long, env)] @@ -49,7 +49,7 @@ struct Args { fn main() -> ExitCode { // Pattern match configuration let Args { - model_name, + model_id, revision, num_shard, quantize, @@ -92,7 +92,7 @@ fn main() -> ExitCode { // Start shard processes for rank in 0..num_shard { - let model_name = model_name.clone(); + let model_id = model_id.clone(); let revision = revision.clone(); let uds_path = shard_uds_path.clone(); let master_addr = master_addr.clone(); @@ -101,7 +101,7 @@ fn main() -> ExitCode { let shutdown_sender = shutdown_sender.clone(); thread::spawn(move || { shard_manager( - model_name, + model_id, revision, quantize, uds_path, @@ -167,7 +167,7 @@ fn main() -> ExitCode { "--master-shard-uds-path".to_string(), format!("{}-0", shard_uds_path), "--tokenizer-name".to_string(), - model_name, + model_id, ]; if json_output { @@ -256,7 +256,7 @@ enum ShardStatus { #[allow(clippy::too_many_arguments)] fn shard_manager( - model_name: String, + model_id: String, revision: Option, quantize: bool, uds_path: String, @@ -278,7 +278,7 @@ fn shard_manager( let mut shard_argv = vec![ "text-generation-server".to_string(), "serve".to_string(), - model_name, + model_id, "--uds-path".to_string(), uds_path, "--logger-level".to_string(), diff --git a/launcher/tests/bloom_560m.json b/launcher/tests/bloom_560m.json index 3a0a3d99..17e2571e 100644 --- a/launcher/tests/bloom_560m.json +++ b/launcher/tests/bloom_560m.json @@ -1,123 +1,122 @@ -[ - { - "details": { - "finish_reason": "length", - "generated_tokens": 20, - "prefill": [ - [ - 10264, - "Test", - null - ], - [ - 8821, - " request", - -11.895094 - ] - ], - "tokens": [ - [ - 17, - ".", - -1.8267941 - ], - [ - 1587, - "get", - -2.4674964 - ], - [ - 11, - "(", - -1.9060438 - ], - [ - 5, - "\"", - -1.2279553 - ], - [ - 4899, - "action", - -4.170306 - ], - [ - 5, - "\"", - -0.3247902 - ], - [ - 12, - ")", - -1.0773602 - ], - [ - 30, - ";", - -0.27640444 - ], - [ - 837, - "\n ", - -1.6970599 - ], - [ - 1320, - " if", - -1.4495552 - ], - [ - 375, - " (", - -0.2360998 - ], - [ - 4899, - "action", - -1.1916926 - ], - [ - 3535, - " ==", - -0.8918663 - ], - [ - 5109, - " null", - -0.39334255 - ], - [ - 12, - ")", - -0.4321134 - ], - [ - 731, - " {", - -0.17701954 - ], - [ - 1260, - "\n ", - -0.07027287 - ], - [ - 10519, - " throw", - -1.3915133 - ], - [ - 2084, - " new", - -0.042013377 - ], - [ - 150858, - " RuntimeException", - -1.7330077 - ] - ] - }, - "generated_text": ".get(\"action\");\n if (action == null) {\n throw new RuntimeException" - } -] \ No newline at end of file +{ + "details": { + "finish_reason": "length", + "generated_tokens": 20, + "prefill": [ + { + "id": 10264, + "logprob": null, + "text": "Test" + }, + { + "id": 8821, + "logprob": -11.894989, + "text": " request" + } + ], + "seed": null, + "tokens": [ + { + "id": 17, + "logprob": -1.8267672, + "text": "." + }, + { + "id": 1587, + "logprob": -2.4674969, + "text": "get" + }, + { + "id": 11, + "logprob": -1.906001, + "text": "(" + }, + { + "id": 5, + "logprob": -1.2279545, + "text": "\"" + }, + { + "id": 4899, + "logprob": -4.170299, + "text": "action" + }, + { + "id": 5, + "logprob": -0.32478866, + "text": "\"" + }, + { + "id": 12, + "logprob": -1.0773665, + "text": ")" + }, + { + "id": 30, + "logprob": -0.27640742, + "text": ";" + }, + { + "id": 837, + "logprob": -1.6970354, + "text": "\n " + }, + { + "id": 1320, + "logprob": -1.4495516, + "text": " if" + }, + { + "id": 375, + "logprob": -0.23609057, + "text": " (" + }, + { + "id": 4899, + "logprob": -1.1916996, + "text": "action" + }, + { + "id": 3535, + "logprob": -0.8918753, + "text": " ==" + }, + { + "id": 5109, + "logprob": -0.3933342, + "text": " null" + }, + { + "id": 12, + "logprob": -0.43212673, + "text": ")" + }, + { + "id": 731, + "logprob": -0.17702064, + "text": " {" + }, + { + "id": 1260, + "logprob": -0.07027565, + "text": "\n " + }, + { + "id": 10519, + "logprob": -1.3915029, + "text": " throw" + }, + { + "id": 2084, + "logprob": -0.04201372, + "text": " new" + }, + { + "id": 150858, + "logprob": -1.7329919, + "text": " RuntimeException" + } + ] + }, + "generated_text": ".get(\"action\");\n if (action == null) {\n throw new RuntimeException" +} \ No newline at end of file diff --git a/launcher/tests/integration_tests.rs b/launcher/tests/integration_tests.rs index 4f699f69..b70b1628 100644 --- a/launcher/tests/integration_tests.rs +++ b/launcher/tests/integration_tests.rs @@ -9,11 +9,18 @@ use std::thread::sleep; use std::time::Duration; use subprocess::{Popen, PopenConfig, Redirection}; +#[derive(Deserialize)] +pub struct Token { + id: u32, + text: String, + logprob: Option, +} + #[derive(Deserialize)] struct Details { finish_reason: String, generated_tokens: u32, - tokens: Vec<(u32, String, Option)>, + tokens: Vec, } #[derive(Deserialize)] @@ -22,11 +29,11 @@ struct GeneratedText { details: Details, } -fn start_launcher(model_name: String, num_shard: usize, port: usize, master_port: usize) -> Popen { +fn start_launcher(model_id: String, num_shard: usize, port: usize, master_port: usize) -> Popen { let argv = vec![ "text-generation-launcher".to_string(), - "--model-name".to_string(), - model_name.clone(), + "--model-id".to_string(), + model_id.clone(), "--num-shard".to_string(), num_shard.to_string(), "--port".to_string(), @@ -68,16 +75,16 @@ fn start_launcher(model_name: String, num_shard: usize, port: usize, master_port launcher.terminate().unwrap(); launcher.wait().unwrap(); - panic!("failed to launch {}", model_name) + panic!("failed to launch {}", model_id) } fn test_model( - model_name: String, + model_id: String, num_shard: usize, port: usize, master_port: usize, ) -> GeneratedText { - let mut launcher = start_launcher(model_name, num_shard, port, master_port); + let mut launcher = start_launcher(model_id, num_shard, port, master_port); let data = r#" { @@ -109,8 +116,8 @@ fn read_json(name: &str) -> GeneratedText { let file = File::open(d).unwrap(); let reader = BufReader::new(file); - let mut results: Vec = serde_json::from_reader(reader).unwrap(); - results.pop().unwrap() + let result: GeneratedText = serde_json::from_reader(reader).unwrap(); + result } fn compare_results(result: GeneratedText, expected: GeneratedText) { @@ -127,13 +134,13 @@ fn compare_results(result: GeneratedText, expected: GeneratedText) { .into_iter() .zip(expected.details.tokens.into_iter()) { - assert_eq!(token.0, expected_token.0); - assert_eq!(token.1, expected_token.1); - if let Some(logprob) = token.2 { - let expected_logprob = expected_token.2.unwrap(); + assert_eq!(token.id, expected_token.id); + assert_eq!(token.text, expected_token.text); + if let Some(logprob) = token.logprob { + let expected_logprob = expected_token.logprob.unwrap(); assert_float_eq!(logprob, expected_logprob, abs <= 0.001); } else { - assert_eq!(token.2, expected_token.2); + assert_eq!(token.logprob, expected_token.logprob); } } } diff --git a/launcher/tests/mt0_base.json b/launcher/tests/mt0_base.json index 51cb8b5c..cee3bc47 100644 --- a/launcher/tests/mt0_base.json +++ b/launcher/tests/mt0_base.json @@ -1,118 +1,117 @@ -[ - { - "details": { - "finish_reason": "length", - "generated_tokens": 20, - "prefill": [ - [ - 0, - "", - null - ] - ], - "tokens": [ - [ - 259, - "", - -1.3656927 - ], - [ - 215100, - "\"\"\"", - -2.6551573 - ], - [ - 46138, - "Test", - -1.8059857 - ], - [ - 287, - "the", - -1.2102449 - ], - [ - 259, - "", - -1.6057279 - ], - [ - 49076, - "contents", - -3.6060903 - ], - [ - 304, - "of", - -0.5270343 - ], - [ - 287, - "the", - -0.62522805 - ], - [ - 259, - "", - -1.4069618 - ], - [ - 49076, - "contents", - -2.621994 - ], - [ - 304, - "of", - -1.3172221 - ], - [ - 287, - "the", - -0.3501925 - ], - [ - 259, - "", - -0.7219573 - ], - [ - 49076, - "contents", - -1.0494149 - ], - [ - 260, - ".", - -1.0803378 - ], - [ - 259, - "", - -0.32933083 - ], - [ - 215100, - "\"\"\"", - -0.11268901 - ], - [ - 2978, - "test", - -1.5846587 - ], - [ - 290, - "_", - -0.49796978 - ], - [ - 4125, - "test", - -2.0026445 - ] - ] - }, - "generated_text": "\"\"\"Test the contents of the contents of the contents. \"\"\" test_test" - } -] \ No newline at end of file +{ + "details": { + "finish_reason": "length", + "generated_tokens": 20, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 259, + "logprob": -1.3656927, + "text": "" + }, + { + "id": 215100, + "logprob": -2.6551573, + "text": "\"\"\"" + }, + { + "id": 46138, + "logprob": -1.8059857, + "text": "Test" + }, + { + "id": 287, + "logprob": -1.2102449, + "text": "the" + }, + { + "id": 259, + "logprob": -1.6057279, + "text": "" + }, + { + "id": 49076, + "logprob": -3.6060903, + "text": "contents" + }, + { + "id": 304, + "logprob": -0.5270343, + "text": "of" + }, + { + "id": 287, + "logprob": -0.62522805, + "text": "the" + }, + { + "id": 259, + "logprob": -1.4069618, + "text": "" + }, + { + "id": 49076, + "logprob": -2.621994, + "text": "contents" + }, + { + "id": 304, + "logprob": -1.3172221, + "text": "of" + }, + { + "id": 287, + "logprob": -0.3501925, + "text": "the" + }, + { + "id": 259, + "logprob": -0.7219573, + "text": "" + }, + { + "id": 49076, + "logprob": -1.0494149, + "text": "contents" + }, + { + "id": 260, + "logprob": -1.0803378, + "text": "." + }, + { + "id": 259, + "logprob": -0.32933083, + "text": "" + }, + { + "id": 215100, + "logprob": -0.11268901, + "text": "\"\"\"" + }, + { + "id": 2978, + "logprob": -1.5846587, + "text": "test" + }, + { + "id": 290, + "logprob": -0.49796978, + "text": "_" + }, + { + "id": 4125, + "logprob": -2.0026445, + "text": "test" + } + ] + }, + "generated_text": "\"\"\"Test the contents of the contents of the contents. \"\"\" test_test" +} \ No newline at end of file diff --git a/proto/generate.proto b/proto/generate.proto index 098df9c5..0c4f9626 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -71,13 +71,19 @@ message Batch { uint32 size = 3; } +enum FinishReason { + FINISH_REASON_LENGTH = 0; + FINISH_REASON_EOS_TOKEN = 1; + FINISH_REASON_STOP_SEQUENCE = 2; +} + message GeneratedText { /// Output string text = 1; /// Number of generated tokens uint32 generated_tokens = 2; /// Finish reason - string finish_reason = 3; + FinishReason finish_reason = 3; /// Seed optional uint64 seed = 4; } diff --git a/router/Cargo.toml b/router/Cargo.toml index 3abbc80b..20f032b1 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.1.0" +version = "0.2.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" @@ -14,7 +14,7 @@ path = "src/main.rs" [dependencies] async-stream = "0.3.3" -axum = { version = "0.5.16", features = ["json", "serde_json"] } +axum = { version = "0.6.4", features = ["json"] } text-generation-client = { path = "client" } clap = { version = "4.0.15", features = ["derive", "env"] } futures = "0.3.24" @@ -29,4 +29,6 @@ tokio = { version = "1.21.1", features = ["rt", "rt-multi-thread", "parking_lot" tokio-stream = "0.1.11" tracing = "0.1.36" tracing-subscriber = { version = "0.3.15", features = ["json"] } +utoipa = { version = "3.0.1", features = ["axum_extras"] } +utoipa-swagger-ui = { version = "3.0.2", features = ["axum"] } diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index fdd32494..f4a900a4 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.1.0" +version = "0.2.0" edition = "2021" [dependencies] diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index e0546b16..2767c605 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -7,8 +7,8 @@ mod sharded_client; pub use client::Client; pub use pb::generate::v1::{ - Batch, GeneratedText, Generation, NextTokenChooserParameters, PrefillTokens, Request, - StoppingCriteriaParameters, + Batch, FinishReason, GeneratedText, Generation, NextTokenChooserParameters, PrefillTokens, + Request, StoppingCriteriaParameters, }; pub use sharded_client::ShardedClient; use thiserror::Error; diff --git a/router/src/infer.rs b/router/src/infer.rs index 3661b0e0..159b7ca7 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -127,7 +127,7 @@ impl Infer { .into_iter() .zip(tokens.logprobs.into_iter()) .zip(tokens.texts.into_iter()) - .map(|((id, logprob), text)| Token(id, text, logprob)) + .map(|((id, logprob), text)| Token { id, text, logprob }) .collect(); } // Push last token @@ -282,11 +282,11 @@ fn send_generations(generations: Vec, entries: &mut IntMap, + #[serde(default)] + #[schema( + exclusive_minimum = 0.0, + nullable = true, + default = "null", + example = 1.03 + )] + pub repetition_penalty: Option, + #[serde(default)] + #[schema(exclusive_minimum = 0, nullable = true, default = "null", example = 10)] + pub top_k: Option, + #[serde(default)] + #[schema( + exclusive_minimum = 0.0, + maximum = 1.0, + nullable = true, + default = "null", + example = 0.95 + )] + pub top_p: Option, #[serde(default = "default_do_sample")] + #[schema(default = "false", example = true)] pub do_sample: bool, #[serde(default = "default_max_new_tokens")] + #[schema(exclusive_minimum = 0, exclusive_maximum = 512, default = "20")] pub max_new_tokens: u32, #[serde(default)] + #[schema(inline, max_items = 4, example = json!(["photographer"]))] pub stop: Vec, #[serde(default)] + #[schema(default = "true")] pub details: bool, #[serde(default)] pub seed: Option, } -fn default_temperature() -> f32 { - 1.0 -} -fn default_repetition_penalty() -> f32 { - 1.0 -} - -fn default_top_k() -> i32 { - 0 -} - -fn default_top_p() -> f32 { - 1.0 -} - fn default_do_sample() -> bool { false } @@ -57,10 +66,10 @@ fn default_max_new_tokens() -> u32 { fn default_parameters() -> GenerateParameters { GenerateParameters { - temperature: default_temperature(), - repetition_penalty: default_repetition_penalty(), - top_k: default_top_k(), - top_p: default_top_p(), + temperature: None, + repetition_penalty: None, + top_k: None, + top_p: None, do_sample: default_do_sample(), max_new_tokens: default_max_new_tokens(), stop: vec![], @@ -69,42 +78,77 @@ fn default_parameters() -> GenerateParameters { } } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, ToSchema)] pub(crate) struct GenerateRequest { + #[schema(example = "My name is Olivier and I")] pub inputs: String, #[serde(default = "default_parameters")] pub parameters: GenerateParameters, } -#[derive(Debug, Serialize)] -pub struct Token(u32, String, f32); +#[derive(Debug, Serialize, ToSchema)] +pub struct Token { + #[schema(example = 0)] + id: u32, + #[schema(example = "test")] + text: String, + #[schema(nullable = true, example = -0.34)] + logprob: f32, +} + +#[derive(Serialize, ToSchema)] +#[serde(rename_all(serialize = "snake_case"))] +pub(crate) enum FinishReason { + #[schema(rename = "length")] + Length, + #[serde(rename = "eos_token")] + #[schema(rename = "eos_token")] + EndOfSequenceToken, + #[schema(rename = "stop_sequence")] + StopSequence, +} -#[derive(Serialize)] +#[derive(Serialize, ToSchema)] pub(crate) struct Details { - pub finish_reason: String, + #[schema(example = "length")] + pub finish_reason: FinishReason, + #[schema(example = 1)] pub generated_tokens: u32, + #[schema(example = 42)] pub seed: Option, - #[serde(skip_serializing_if = "Option::is_none")] pub prefill: Option>, - #[serde(skip_serializing_if = "Option::is_none")] pub tokens: Option>, } -#[derive(Serialize)] +#[derive(Serialize, ToSchema)] pub(crate) struct GenerateResponse { + #[schema(example = "test")] pub generated_text: String, #[serde(skip_serializing_if = "Option::is_none")] pub details: Option
, } -#[derive(Serialize)] +#[derive(Serialize, ToSchema)] +pub(crate) struct StreamDetails { + #[schema(example = "length")] + pub finish_reason: FinishReason, + #[schema(example = 1)] + pub generated_tokens: u32, + #[schema(example = 42)] + pub seed: Option, +} + +#[derive(Serialize, ToSchema)] pub(crate) struct StreamResponse { pub token: Token, + #[schema(nullable = true, default = "null", example = "test")] pub generated_text: Option, - pub details: Option
, + #[schema(nullable = true, default = "null")] + pub details: Option, } -#[derive(Serialize)] +#[derive(Serialize, ToSchema)] pub(crate) struct ErrorResponse { + #[schema(inline)] pub error: String, } diff --git a/router/src/server.rs b/router/src/server.rs index f79644e3..1a332088 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,8 +1,8 @@ /// HTTP Server logic use crate::infer::{InferError, InferStreamResponse}; use crate::{ - Details, ErrorResponse, GenerateParameters, GenerateRequest, GenerateResponse, Infer, - StreamResponse, Validation, + Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest, GenerateResponse, + Infer, StreamDetails, StreamResponse, Token, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, StatusCode}; @@ -19,6 +19,8 @@ use tokio::signal; use tokio::time::Instant; use tokio_stream::StreamExt; use tracing::instrument; +use utoipa::OpenApi; +use utoipa_swagger_ui::SwaggerUi; /// Health check method #[instrument(skip(infer))] @@ -32,13 +34,13 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json) -> Result<(), (StatusCode, Json Some(Details { - finish_reason: response.generated_text.finish_reason, + finish_reason: FinishReason::from(response.generated_text.finish_reason), generated_tokens: response.generated_text.generated_tokens, prefill: Some(response.prefill), tokens: Some(response.tokens), @@ -132,7 +151,29 @@ async fn generate( Ok((headers, Json(response))) } -/// Generate stream method +/// Generate a stream of token using Server Side Events +#[utoipa::path( + post, + tag = "Text Generation Inference", + path = "/generate_stream", + request_body = GenerateRequest, + responses( + (status = 200, description = "Generated Text", body = [StreamResponse], + content_type="text/event-stream "), + (status = 424, description = "Generation Error", body = [ErrorResponse], + example = json!({"error": "Request failed during generation"}), + content_type="text/event-stream "), + (status = 429, description = "Model is overloaded", body = [ErrorResponse], + example = json!({"error": "Model is overloaded"}), + content_type="text/event-stream "), + (status = 422, description = "Input validation error", body = [ErrorResponse], + example = json!({"error": "Input validation error"}), + content_type="text/event-stream "), + (status = 500, description = "Incomplete generation", body = [ErrorResponse], + example = json!({"error": "Incomplete generation"}), + content_type="text/event-stream "), + ) +)] #[instrument( skip(infer), fields( @@ -185,11 +226,9 @@ async fn generate_stream( } => { // Token details let details = match details { - true => Some(Details { - finish_reason: generated_text.finish_reason, + true => Some(StreamDetails { + finish_reason: FinishReason::from(generated_text.finish_reason), generated_tokens: generated_text.generated_tokens, - prefill: None, - tokens: None, seed: generated_text.seed, }), false => None, @@ -265,6 +304,39 @@ pub async fn run( validation_workers: usize, addr: SocketAddr, ) { + // OpenAPI documentation + #[derive(OpenApi)] + #[openapi( + paths( + generate, + generate_stream, + ), + components( + schemas( + GenerateRequest, + GenerateParameters, + Token, + GenerateResponse, + Details, + FinishReason, + StreamResponse, + StreamDetails, + ErrorResponse, + ) + ), + tags( + (name = "Text Generation Inference", description = "Hugging Face Text Generation Inference API") + ), + info( + title = "Text Generation Inference", + license( + name = "Apache 2.0", + url = "https://www.apache.org/licenses/LICENSE-2.0" + ) + ) + )] + struct ApiDoc; + // Create state let validation = Validation::new(validation_workers, tokenizer, max_input_length); let infer = Infer::new( @@ -277,6 +349,7 @@ pub async fn run( // Create router let app = Router::new() + .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", ApiDoc::openapi())) .route("/", post(generate)) .route("/generate", post(generate)) .route("/generate_stream", post(generate_stream)) @@ -320,6 +393,17 @@ async fn shutdown_signal() { tracing::info!("signal received, starting graceful shutdown"); } +impl From for FinishReason { + fn from(finish_reason: i32) -> Self { + let finish_reason = text_generation_client::FinishReason::from_i32(finish_reason).unwrap(); + match finish_reason { + text_generation_client::FinishReason::Length => FinishReason::Length, + text_generation_client::FinishReason::EosToken => FinishReason::EndOfSequenceToken, + text_generation_client::FinishReason::StopSequence => FinishReason::StopSequence, + } + } +} + /// Convert to Axum supported formats impl From for (StatusCode, Json) { fn from(err: InferError) -> Self { diff --git a/router/src/validation.rs b/router/src/validation.rs index 09220823..3cca48e9 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -110,30 +110,58 @@ fn validate( max_input_length: usize, rng: &mut ThreadRng, ) -> Result { - if request.parameters.temperature <= 0.0 { + let GenerateParameters { + temperature, + repetition_penalty, + top_k, + top_p, + do_sample, + max_new_tokens, + stop: stop_sequences, + seed, + .. + } = request.parameters; + + let temperature = temperature.unwrap_or(1.0); + if temperature <= 0.0 { return Err(ValidationError::Temperature); } - if request.parameters.repetition_penalty <= 0.0 { + + let repetition_penalty = repetition_penalty.unwrap_or(1.0); + if repetition_penalty <= 0.0 { return Err(ValidationError::RepetitionPenalty); } - if request.parameters.top_p <= 0.0 || request.parameters.top_p > 1.0 { + + let top_p = top_p.unwrap_or(1.0); + if top_p <= 0.0 || top_p > 1.0 { return Err(ValidationError::TopP); } - if request.parameters.top_k < 0 { - return Err(ValidationError::TopK); - } - if request.parameters.max_new_tokens > MAX_MAX_NEW_TOKENS { + + // Different because the proto default value is 0 while it is not a valid value + // for the user + let top_k: u32 = match top_k { + None => Ok(0), + Some(top_k) => { + if top_k <= 0 { + return Err(ValidationError::TopK); + } + Ok(top_k as u32) + } + }?; + + if max_new_tokens == 0 || max_new_tokens > MAX_MAX_NEW_TOKENS { return Err(ValidationError::MaxNewTokens(MAX_MAX_NEW_TOKENS)); } - if request.parameters.stop.len() > MAX_STOP_SEQUENCES { + + if stop_sequences.len() > MAX_STOP_SEQUENCES { return Err(ValidationError::StopSequence( MAX_STOP_SEQUENCES, - request.parameters.stop.len(), + stop_sequences.len(), )); } // If seed is None, assign a random one - let seed = match request.parameters.seed { + let seed = match seed { None => rng.gen(), Some(seed) => seed, }; @@ -147,21 +175,10 @@ fn validate( Err(ValidationError::InputLength(input_length, max_input_length)) } else { // Return ValidGenerateRequest - let GenerateParameters { - temperature, - repetition_penalty, - top_k, - top_p, - do_sample, - max_new_tokens, - stop: stop_sequences, - .. - } = request.parameters; - let parameters = NextTokenChooserParameters { temperature, repetition_penalty, - top_k: top_k as u32, + top_k, top_p, do_sample, seed, @@ -206,7 +223,7 @@ pub enum ValidationError { TopP, #[error("top_k must be strictly positive")] TopK, - #[error("max_new_tokens must be <= {0}")] + #[error("max_new_tokens must be strictly positive and <= {0}")] MaxNewTokens(u32), #[error("inputs must have less than {1} tokens. Given: {0}")] InputLength(usize, usize), diff --git a/server/README.md b/server/README.md index c7054b8d..8efd80ac 100644 --- a/server/README.md +++ b/server/README.md @@ -1,6 +1,6 @@ -# BLOOM Inference Python gRPC Server +# Text Generation Inference Python gRPC Server -A Python gRPC server for BLOOM Inference +A Python gRPC server for Text Generation Inference ## Install diff --git a/server/pyproject.toml b/server/pyproject.toml index 98d91125..87aa8cfb 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "text-generation" -version = "0.1.0" -description = "BLOOM Inference Python gRPC Server" +version = "0.2.0" +description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] [tool.poetry.scripts] diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 871c0da0..b06d57f5 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -140,8 +140,7 @@ def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch) assert len(generations) == 1 assert ( - generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTest" + generations[0].generated_text.text == "TestTestTestTestTestTestTestTestTestTest" ) assert generations[0].request_id == default_bloom_batch.requests[0].id assert ( @@ -187,8 +186,7 @@ def test_causal_lm_generate_token_completion_multi( assert len(generations) == 1 assert ( - generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTest" + generations[0].generated_text.text == "TestTestTestTestTestTestTestTestTestTest" ) assert ( generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id @@ -283,8 +281,7 @@ def test_batch_concatenate( assert len(generations) == 2 assert ( - generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTest" + generations[0].generated_text.text == "TestTestTestTestTestTestTestTestTestTest" ) assert generations[0].request_id == default_bloom_batch.requests[0].id assert ( @@ -306,8 +303,7 @@ def test_batch_concatenate( assert len(generations) == 1 assert ( - generations[0].generated_text.text - == "TestTestTestTestTestTestTestTestTestTest" + generations[0].generated_text.text == "TestTestTestTestTestTestTestTestTestTest" ) assert ( generations[0].request_id == default_multi_requests_bloom_batch.requests[0].id diff --git a/server/tests/test_utils.py b/server/tests/test_utils.py index 1dc6801b..ffe9be65 100644 --- a/server/tests/test_utils.py +++ b/server/tests/test_utils.py @@ -9,6 +9,7 @@ StopSequenceCriteria, StoppingCriteria, LocalEntryNotFoundError, + FinishReason, ) @@ -24,13 +25,13 @@ def test_stop_sequence_criteria(): def test_stopping_criteria(): criteria = StoppingCriteria(0, [StopSequenceCriteria("/test;")], max_new_tokens=5) assert criteria(65827, "/test") == (False, None) - assert criteria(30, ";") == (True, "stop_sequence") + assert criteria(30, ";") == (True, FinishReason.FINISH_REASON_STOP_SEQUENCE) def test_stopping_criteria_eos(): criteria = StoppingCriteria(0, [StopSequenceCriteria("/test;")], max_new_tokens=5) assert criteria(1, "") == (False, None) - assert criteria(0, "") == (True, "eos_token") + assert criteria(0, "") == (True, FinishReason.FINISH_REASON_EOS_TOKEN) def test_stopping_criteria_max(): @@ -39,7 +40,7 @@ def test_stopping_criteria_max(): assert criteria(1, "") == (False, None) assert criteria(1, "") == (False, None) assert criteria(1, "") == (False, None) - assert criteria(1, "") == (True, "length") + assert criteria(1, "") == (True, FinishReason.FINISH_REASON_LENGTH) def test_weight_hub_files(): diff --git a/server/text_generation/cli.py b/server/text_generation/cli.py index b133cb0a..1418a803 100644 --- a/server/text_generation/cli.py +++ b/server/text_generation/cli.py @@ -13,7 +13,7 @@ @app.command() def serve( - model_name: str, + model_id: str, revision: Optional[str] = None, sharded: bool = False, quantize: bool = False, @@ -46,16 +46,16 @@ def serve( os.getenv("MASTER_PORT", None) is not None ), "MASTER_PORT must be set when sharded is True" - server.serve(model_name, revision, sharded, quantize, uds_path) + server.serve(model_id, revision, sharded, quantize, uds_path) @app.command() def download_weights( - model_name: str, + model_id: str, revision: Optional[str] = None, extension: str = ".safetensors", ): - utils.download_weights(model_name, revision, extension) + utils.download_weights(model_id, revision, extension) if __name__ == "__main__": diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 15d8e97e..943d45e9 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -30,31 +30,31 @@ def get_model( - model_name: str, revision: Optional[str], sharded: bool, quantize: bool + model_id: str, revision: Optional[str], sharded: bool, quantize: bool ) -> Model: - config = AutoConfig.from_pretrained(model_name, revision=revision) + config = AutoConfig.from_pretrained(model_id, revision=revision) if config.model_type == "bloom": if sharded: - return BLOOMSharded(model_name, revision, quantize=quantize) + return BLOOMSharded(model_id, revision, quantize=quantize) else: - return BLOOM(model_name, revision, quantize=quantize) + return BLOOM(model_id, revision, quantize=quantize) elif config.model_type == "gpt_neox": if sharded: - return GPTNeoxSharded(model_name, revision, quantize=quantize) + return GPTNeoxSharded(model_id, revision, quantize=quantize) else: - return GPTNeox(model_name, revision, quantize=quantize) - elif model_name.startswith("facebook/galactica"): + return GPTNeox(model_id, revision, quantize=quantize) + elif model_id.startswith("facebook/galactica"): if sharded: - return GalacticaSharded(model_name, revision, quantize=quantize) + return GalacticaSharded(model_id, revision, quantize=quantize) else: - return Galactica(model_name, revision, quantize=quantize) - elif "santacoder" in model_name: - return SantaCoder(model_name, revision, quantize) + return Galactica(model_id, revision, quantize=quantize) + elif "santacoder" in model_id: + return SantaCoder(model_id, revision, quantize) else: if sharded: raise ValueError("sharded is not supported for AutoModel") try: - return CausalLM(model_name, revision, quantize=quantize) + return CausalLM(model_id, revision, quantize=quantize) except Exception: - return Seq2SeqLM(model_name, revision, quantize=quantize) + return Seq2SeqLM(model_id, revision, quantize=quantize) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 4f55afc0..992d7b5b 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -57,10 +57,10 @@ def batch_type(self) -> Type[CausalLMBatch]: class BLOOMSharded(BLOOM): def __init__( - self, model_name: str, revision: Optional[str] = None, quantize: bool = False + self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - if not model_name.startswith("bigscience/bloom"): - raise ValueError(f"Model {model_name} is not supported") + if not model_id.startswith("bigscience/bloom"): + raise ValueError(f"Model {model_id} is not supported") self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 @@ -72,22 +72,20 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_name, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left" ) config = AutoConfig.from_pretrained( - model_name, revision=revision, slow_but_exact=False, tp_parallel=True + model_id, revision=revision, slow_but_exact=False, tp_parallel=True ) config.pad_token_id = 3 # Only download weights for small models - if self.master and model_name == "bigscience/bloom-560m": - download_weights(model_name, revision=revision, extension=".safetensors") + if self.master and model_id == "bigscience/bloom-560m": + download_weights(model_id, revision=revision, extension=".safetensors") torch.distributed.barrier(group=self.process_group) - filenames = weight_files( - model_name, revision=revision, extension=".safetensors" - ) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") if not filenames: raise ValueError("No safetensors weights found") diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 1d1945cd..f21423ea 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -232,7 +232,7 @@ def __len__(self): class CausalLM(Model): - def __init__(self, model_name: str, revision: Optional[str] = None, quantize=False): + def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -244,10 +244,10 @@ def __init__(self, model_name: str, revision: Optional[str] = None, quantize=Fal dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_name, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left" ) self.model = AutoModelForCausalLM.from_pretrained( - model_name, + model_id, revision=revision, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index be9b1699..f1dc8a30 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -149,10 +149,10 @@ def decode(self, generated_ids: List[int]) -> str: class GalacticaSharded(Galactica): def __init__( - self, model_name: str, revision: Optional[str] = None, quantize: bool = False + self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - if not model_name.startswith("facebook/galactica"): - raise ValueError(f"Model {model_name} is not supported") + if not model_id.startswith("facebook/galactica"): + raise ValueError(f"Model {model_id} is not supported") self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 @@ -164,22 +164,20 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_name, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left" ) config = AutoConfig.from_pretrained( - model_name, revision=revision, tp_parallel=True + model_id, revision=revision, tp_parallel=True ) tokenizer.pad_token_id = config.pad_token_id # Only download weights for small models - if self.master and model_name == "facebook/galactica-125m": - download_weights(model_name, revision=revision, extension=".safetensors") + if self.master and model_id == "facebook/galactica-125m": + download_weights(model_id, revision=revision, extension=".safetensors") torch.distributed.barrier(group=self.process_group) - filenames = weight_files( - model_name, revision=revision, extension=".safetensors" - ) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") if not filenames: raise ValueError("No safetensors weights found") diff --git a/server/text_generation/models/gpt_neox.py b/server/text_generation/models/gpt_neox.py index a8f7f365..2d467f4c 100644 --- a/server/text_generation/models/gpt_neox.py +++ b/server/text_generation/models/gpt_neox.py @@ -49,7 +49,7 @@ def forward( class GPTNeoxSharded(GPTNeox): def __init__( - self, model_name: str, revision: Optional[str] = None, quantize: bool = False + self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 @@ -61,22 +61,20 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_name, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left" ) tokenizer.pad_token = tokenizer.eos_token config = AutoConfig.from_pretrained( - model_name, revision=revision, tp_parallel=True + model_id, revision=revision, tp_parallel=True ) # Only master download weights if self.master: - download_weights(model_name, revision=revision, extension=".safetensors") + download_weights(model_id, revision=revision, extension=".safetensors") torch.distributed.barrier(group=self.process_group) - filenames = weight_files( - model_name, revision=revision, extension=".safetensors" - ) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") if not filenames: raise ValueError("No safetensors weights found") diff --git a/server/text_generation/models/santacoder.py b/server/text_generation/models/santacoder.py index 6c1a250f..fb496197 100644 --- a/server/text_generation/models/santacoder.py +++ b/server/text_generation/models/santacoder.py @@ -14,7 +14,7 @@ class SantaCoder(CausalLM): - def __init__(self, model_name: str, revision: Optional[str] = None, quantize=False): + def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -26,7 +26,7 @@ def __init__(self, model_name: str, revision: Optional[str] = None, quantize=Fal dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_name, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left" ) tokenizer.add_special_tokens( { @@ -43,7 +43,7 @@ def __init__(self, model_name: str, revision: Optional[str] = None, quantize=Fal self.model = ( AutoModelForCausalLM.from_pretrained( - model_name, + model_id, revision=revision, torch_dtype=dtype, load_in_8bit=quantize, diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 1ae266d8..27cbe1c0 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -289,7 +289,7 @@ def __len__(self): class Seq2SeqLM(Model): - def __init__(self, model_name: str, revision: Optional[str] = None, quantize=False): + def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -301,14 +301,14 @@ def __init__(self, model_name: str, revision: Optional[str] = None, quantize=Fal dtype = torch.float32 self.model = AutoModelForSeq2SeqLM.from_pretrained( - model_name, + model_id, revision=revision, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, load_in_8bit=quantize, ).eval() tokenizer = AutoTokenizer.from_pretrained( - model_name, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left" ) tokenizer.bos_token_id = self.model.config.decoder_start_token_id diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index 30cd716a..d1117b80 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -7,6 +7,7 @@ from transformers import PreTrainedTokenizerBase from text_generation.pb import generate_pb2 +from text_generation.pb.generate_pb2 import FinishReason class Batch(ABC): @@ -38,7 +39,7 @@ def __len__(self): class GeneratedText: text: str generated_tokens: int - finish_reason: str + finish_reason: FinishReason seed: Optional[int] def to_pb(self) -> generate_pb2.GeneratedText: diff --git a/server/text_generation/server.py b/server/text_generation/server.py index 852deebf..a8a9da6c 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -66,14 +66,14 @@ async def Decode(self, request, context): def serve( - model_name: str, + model_id: str, revision: Optional[str], sharded: bool, quantize: bool, uds_path: Path, ): async def serve_inner( - model_name: str, + model_id: str, revision: Optional[str], sharded: bool = False, quantize: bool = False, @@ -89,7 +89,7 @@ async def serve_inner( local_url = unix_socket_template.format(uds_path, 0) server_urls = [local_url] - model = get_model(model_name, revision, sharded, quantize) + model = get_model(model_id, revision, sharded, quantize) server = aio.server(interceptors=[ExceptionInterceptor()]) generate_pb2_grpc.add_TextGenerationServiceServicer_to_server( @@ -109,4 +109,4 @@ async def serve_inner( logger.info("Signal received. Shutting down") await server.stop(0) - asyncio.run(serve_inner(model_name, revision, sharded, quantize)) + asyncio.run(serve_inner(model_id, revision, sharded, quantize)) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 8bbc7377..83458969 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -24,9 +24,11 @@ ) from text_generation.pb import generate_pb2 +from text_generation.pb.generate_pb2 import FinishReason WEIGHTS_CACHE_OVERRIDE = os.getenv("WEIGHTS_CACHE_OVERRIDE", None) + class Sampling: def __init__(self, seed: int, device: str = "cpu"): self.generator = torch.Generator(device) @@ -129,15 +131,15 @@ def __init__( def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[str]]: self.current_tokens += 1 if self.current_tokens >= self.max_new_tokens: - return True, "length" + return True, FinishReason.FINISH_REASON_LENGTH if last_token == self.eos_token_id: - return True, "eos_token" + return True, FinishReason.FINISH_REASON_EOS_TOKEN self.current_output += last_output for stop_sequence_criteria in self.stop_sequence_criterias: if stop_sequence_criteria(self.current_output): - return True, "stop_sequence" + return True, FinishReason.FINISH_REASON_STOP_SEQUENCE return False, None @@ -180,20 +182,20 @@ def initialize_torch_distributed(): return torch.distributed.distributed_c10d._get_default_group(), rank, world_size -def weight_hub_files(model_name, revision=None, extension=".safetensors"): +def weight_hub_files(model_id, revision=None, extension=".safetensors"): """Get the safetensors filenames on the hub""" api = HfApi() - info = api.model_info(model_name, revision=revision) + info = api.model_info(model_id, revision=revision) filenames = [s.rfilename for s in info.siblings if s.rfilename.endswith(extension)] return filenames -def try_to_load_from_cache(model_name, revision, filename): +def try_to_load_from_cache(model_id, revision, filename): """Try to load a file from the Hugging Face cache""" if revision is None: revision = "main" - object_id = model_name.replace("/", "--") + object_id = model_id.replace("/", "--") repo_cache = Path(HUGGINGFACE_HUB_CACHE) / f"models--{object_id}" if not repo_cache.is_dir(): @@ -228,38 +230,38 @@ def try_to_load_from_cache(model_name, revision, filename): return str(cached_file) if cached_file.is_file() else None -def weight_files(model_name, revision=None, extension=".safetensors"): +def weight_files(model_id, revision=None, extension=".safetensors"): """Get the local safetensors filenames""" if WEIGHTS_CACHE_OVERRIDE is not None: return list(Path(WEIGHTS_CACHE_OVERRIDE).glob(f"*{extension}")) - filenames = weight_hub_files(model_name, revision, extension) + filenames = weight_hub_files(model_id, revision, extension) files = [] for filename in filenames: cache_file = try_to_load_from_cache( - model_name, revision=revision, filename=filename + model_id, revision=revision, filename=filename ) if cache_file is None: raise LocalEntryNotFoundError( - f"File {filename} of model {model_name} not found in " + f"File {filename} of model {model_id} not found in " f"{os.getenv('HUGGINGFACE_HUB_CACHE', 'the local cache')}. " - f"Please run `text-generation-server download-weights {model_name}` first." + f"Please run `text-generation-server download-weights {model_id}` first." ) files.append(cache_file) return files -def download_weights(model_name, revision=None, extension=".safetensors"): +def download_weights(model_id, revision=None, extension=".safetensors"): """Download the safetensors files from the hub""" if WEIGHTS_CACHE_OVERRIDE is not None: return list(Path(WEIGHTS_CACHE_OVERRIDE).glob(f"*{extension}")) - filenames = weight_hub_files(model_name, revision, extension) + filenames = weight_hub_files(model_id, revision, extension) download_function = partial( hf_hub_download, - repo_id=model_name, + repo_id=model_id, local_files_only=False, ) From a0dca443dd0a804527aa2f1685fe04d603fcfa0e Mon Sep 17 00:00:00 2001 From: lewtun Date: Fri, 3 Feb 2023 13:07:55 +0100 Subject: [PATCH 071/793] feat(docs): Clarify installation steps (#54) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds some bits for first-time users (like me 😄 ) --- README.md | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 49713ecc..5f943de3 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ curl 127.0.0.1:8080/generate_stream \ -H 'Content-Type: application/json' ``` -To use GPUs, you will need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). +**Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). ### API documentation @@ -102,14 +102,31 @@ The Swagger UI is also available at: [https://huggingface.github.io/text-generat ### Local install -You can also opt to install `text-generation-inference` locally. You will need to have cargo and Python installed on your -machine +You can also opt to install `text-generation-inference` locally. + +First [install Rust](https://rustup.rs/) and create a Python virtual environment with at least +Python 3.9, e.g. using `conda`: + +```shell +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +conda create -n text-generation-inference python=3.9 +conda activate text-generation-inference +``` + +Then run: ```shell BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork with CUDA kernels make run-bloom-560m ``` +**Note:** on some machines, you may also need the OpenSSL libraries. On Linux machines, run: + +```shell +sudo apt-get install libssl-dev +``` + ### CUDA Kernels The custom CUDA kernels are only tested on NVIDIA A100s. If you have any installation or runtime issues, you can remove @@ -153,4 +170,4 @@ make router-dev ```shell make python-tests make integration-tests -``` \ No newline at end of file +``` From e114d8748673a067324e0809479273d522a614e1 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 6 Feb 2023 14:33:56 +0100 Subject: [PATCH 072/793] feat(ci): push to AML registry (#56) --- .github/workflows/build.yaml | 7 +++++++ aml/README.md | 13 ++++++++++--- aml/deployment.yaml | 6 +++--- aml/model.yaml | 4 +--- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d89f0982..8304c8d1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -35,6 +35,12 @@ jobs: username: ${{ secrets.TAILSCALE_DOCKER_USERNAME }} password: ${{ secrets.TAILSCALE_DOCKER_PASSWORD }} registry: registry.internal.huggingface.tech + - name: Login to Azure Container Registry + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.AZURE_DOCKER_USERNAME }} + password: ${{ secrets.AZURE_DOCKER_PASSWORD }} + registry: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4.3.0 @@ -44,6 +50,7 @@ jobs: images: | ghcr.io/huggingface/text-generation-inference registry.internal.huggingface.tech/api-inference/community/text-generation-inference + db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} diff --git a/aml/README.md b/aml/README.md index 959e2942..8e78b0ab 100644 --- a/aml/README.md +++ b/aml/README.md @@ -1,8 +1,15 @@ -```shell -docker build . -t db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.1 -docker push db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation:0.1 +# Azure ML endpoint + +## Create all resources +```shell az ml model create -f model.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace az ml online-endpoint create -f endpoint.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace az ml online-deployment create -f deployment.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace +``` + +## Update deployment + +```shell +az ml online-deployment update -f deployment.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace ``` \ No newline at end of file diff --git a/aml/deployment.yaml b/aml/deployment.yaml index 16ef3dc7..320eba24 100644 --- a/aml/deployment.yaml +++ b/aml/deployment.yaml @@ -1,14 +1,14 @@ $schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json name: bloom-deployment endpoint_name: bloom-inference -model: azureml:bloom:1 +model: azureml:bloom-safetensors:1 model_mount_path: /var/azureml-model environment_variables: - HUGGINGFACE_HUB_CACHE: /var/azureml-model/bloom + WEIGHTS_CACHE_OVERRIDE: /var/azureml-model/bloom-safetensors MODEL_ID: bigscience/bloom NUM_SHARD: 8 environment: - image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.3.1 + image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.2.0 inference_config: liveness_route: port: 80 diff --git a/aml/model.yaml b/aml/model.yaml index bd490f1a..bfcdd33f 100644 --- a/aml/model.yaml +++ b/aml/model.yaml @@ -1,5 +1,3 @@ $schema: https://azuremlschemas.azureedge.net/latest/model.schema.json name: bloom-safetensors -version: 1 -path: ./bloom-safetensors -type: custom_model +path: /data/bloom-safetensors From 4acc42a6050ed6c9f9c7c9db28a0f244a45990e7 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 7 Feb 2023 15:38:22 +0100 Subject: [PATCH 073/793] fix(server): better handling of inference mode (#57) --- launcher/src/main.rs | 4 ++-- server/text_generation/models/__init__.py | 3 +++ server/text_generation/models/causal_lm.py | 15 +++++---------- server/text_generation/models/seq2seq_lm.py | 19 +++++++------------ server/text_generation/server.py | 14 +++++++++++++- server/text_generation/utils.py | 5 +++++ 6 files changed, 35 insertions(+), 25 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 3df6e911..6684c3de 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -38,9 +38,9 @@ struct Args { port: u16, #[clap(default_value = "/tmp/text-generation-server", long, env)] shard_uds_path: String, - #[clap(default_value = "localhost", long, env)] + #[clap(default_value = "0.0.0.0", long, env)] master_addr: String, - #[clap(default_value = "29500", long, env)] + #[clap(default_value = "6000", long, env)] master_port: usize, #[clap(long, env)] json_output: bool, diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 943d45e9..291705a6 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -28,6 +28,9 @@ # The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. torch.backends.cudnn.allow_tf32 = True +# Disable gradients +torch.set_grad_enabled(False) + def get_model( model_id: str, revision: Optional[str], sharded: bool, quantize: bool diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index f21423ea..0cbed22e 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -289,17 +289,12 @@ def forward( def generate_token( self, batch: CausalLMBatch ) -> Tuple[List[Generation], Optional[CausalLMBatch]]: - # For some reason, inference_mode does not work well with GLOO which we use on CPU - context_manager = ( - torch.no_grad if self.device.type == "cpu" else torch.inference_mode + logits, past = self.forward( + batch.input_ids, + batch.attention_mask, + batch.position_ids, + batch.past_key_values, ) - with context_manager(): - logits, past = self.forward( - batch.input_ids, - batch.attention_mask, - batch.position_ids, - batch.past_key_values, - ) # List of indices to cache next_batch_keep_indices = [] diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 27cbe1c0..80aecbac 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -364,19 +364,14 @@ def forward( def generate_token( self, batch: Seq2SeqLMBatch ) -> Tuple[List[Generation], Optional[Seq2SeqLMBatch]]: - # For some reason, inference_mode does not work well with GLOO which we use on CPU - context_manager = ( - torch.no_grad if self.device.type == "cpu" else torch.inference_mode + logits, encoder_last_hidden_state, past = self.forward( + batch.input_ids, + batch.attention_mask, + batch.decoder_input_ids, + batch.decoder_attention_mask, + batch.encoder_last_hidden_state, + batch.past_key_values, ) - with context_manager(): - logits, encoder_last_hidden_state, past = self.forward( - batch.input_ids, - batch.attention_mask, - batch.decoder_input_ids, - batch.decoder_attention_mask, - batch.encoder_last_hidden_state, - batch.past_key_values, - ) # List of indices to cache next_batch_keep_indices = [] diff --git a/server/text_generation/server.py b/server/text_generation/server.py index a8a9da6c..68dc7cd0 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -1,5 +1,6 @@ import asyncio import os +import torch from grpc import aio from loguru import logger @@ -19,6 +20,10 @@ def __init__(self, model: Model, cache: Cache, server_urls: List[str]): self.cache = cache self.model = model self.server_urls = server_urls + # For some reason, inference_mode does not work well with GLOO which we use on CPU + if model.device.type == "cuda": + # Force inference mode for the lifetime of TextGenerationService + self._inference_mode_raii_guard = torch._C._InferenceMode(True) async def ServiceDiscovery(self, request, context): return generate_pb2.ServiceDiscoveryResponse(urls=self.server_urls) @@ -89,7 +94,11 @@ async def serve_inner( local_url = unix_socket_template.format(uds_path, 0) server_urls = [local_url] - model = get_model(model_id, revision, sharded, quantize) + try: + model = get_model(model_id, revision, sharded, quantize) + except Exception: + logger.exception("Error when initializing model") + raise server = aio.server(interceptors=[ExceptionInterceptor()]) generate_pb2_grpc.add_TextGenerationServiceServicer_to_server( @@ -101,8 +110,11 @@ async def serve_inner( ) reflection.enable_server_reflection(SERVICE_NAMES, server) server.add_insecure_port(local_url) + await server.start() + logger.info("Server started at {}".format(local_url)) + try: await server.wait_for_termination() except KeyboardInterrupt: diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 83458969..2821a124 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -171,9 +171,14 @@ def initialize_torch_distributed(): else: backend = "gloo" + master_ip = os.getenv("MASTER_ADDR", "0.0.0.0") + master_port = os.getenv("MASTER_PORT", "6000") + init_method = f"tcp://{master_ip}:{master_port}" + # Call the init process. torch.distributed.init_process_group( backend=backend, + init_method=init_method, world_size=world_size, rank=rank, timeout=timedelta(seconds=60), From 2fe5e1b30e5bd490887e5b3ffa9fcaab8cbe1304 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 7 Feb 2023 15:40:25 +0100 Subject: [PATCH 074/793] V0.2.1 (#58) --- Cargo.lock | 6 +++--- docs/openapi.json | 4 ++-- launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- server/pyproject.toml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f686bdb5..0a537115 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1888,7 +1888,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.2.0" +version = "0.2.1" dependencies = [ "futures", "prost", @@ -1903,7 +1903,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.2.0" +version = "0.2.1" dependencies = [ "clap 4.0.22", "ctrlc", @@ -1918,7 +1918,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.2.0" +version = "0.2.1" dependencies = [ "async-stream", "axum", diff --git a/docs/openapi.json b/docs/openapi.json index 4735c8df..0b80cf13 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -5,13 +5,13 @@ "description": "Text Generation Webserver", "contact": { "name": "Olivier Dehaene", - "email": "" + "email": "olivier@huggingface.co" }, "license": { "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.2.0" + "version": "0.2.1" }, "paths": { "/generate": { diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index f62fc804..7ddb87ee 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.2.0" +version = "0.2.1" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index 20f032b1..186f97c5 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.2.0" +version = "0.2.1" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index f4a900a4..4cddf81b 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.2.0" +version = "0.2.1" edition = "2021" [dependencies] diff --git a/server/pyproject.toml b/server/pyproject.toml index 87aa8cfb..bb7fc2f4 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.2.0" +version = "0.2.1" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From c503a639b17fd19921dba9995293a38caf52a040 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 7 Feb 2023 18:25:17 +0100 Subject: [PATCH 075/793] feat(server): support t5 (#59) --- README.md | 3 +- server/text_generation/models/__init__.py | 11 + server/text_generation/models/t5.py | 258 ++++++++++++++++++++++ 3 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 server/text_generation/models/t5.py diff --git a/README.md b/README.md index 5f943de3..54851c2a 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,8 @@ to power LLMs api-inference widgets. - [MT0-XXL](https://huggingface.co/bigscience/mt0-xxl) - ~~[Galactica](https://huggingface.co/facebook/galactica-120b)~~ (deactivated) - [SantaCoder](https://huggingface.co/bigcode/santacoder) -- [GPT-Neox 20B](https://huggingface.co/EleutherAI/gpt-neox-20b): use `--revision pr/13` +- [GPT-Neox 20B](https://huggingface.co/EleutherAI/gpt-neox-20b) +- [FLAN-T5-XXL](https://huggingface.co/google/flan-t5-xxl): use `--revision pr/26` Other models are supported on a best effort basis using: diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 291705a6..7445b427 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -10,14 +10,20 @@ from text_generation.models.galactica import Galactica, GalacticaSharded from text_generation.models.santacoder import SantaCoder from text_generation.models.gpt_neox import GPTNeox, GPTNeoxSharded +from text_generation.models.t5 import T5Sharded __all__ = [ "Model", "BLOOM", "BLOOMSharded", "CausalLM", + "Galactica", + "GalacticaSharded", + "GPTNeox", + "GPTNeoxSharded", "Seq2SeqLM", "SantaCoder", + "T5Sharded", "get_model", ] @@ -47,6 +53,11 @@ def get_model( return GPTNeoxSharded(model_id, revision, quantize=quantize) else: return GPTNeox(model_id, revision, quantize=quantize) + elif config.model_type == "t5": + if sharded: + return T5Sharded(model_id, revision, quantize=quantize) + else: + return Seq2SeqLM(model_id, revision, quantize=quantize) elif model_id.startswith("facebook/galactica"): if sharded: return GalacticaSharded(model_id, revision, quantize=quantize) diff --git a/server/text_generation/models/t5.py b/server/text_generation/models/t5.py new file mode 100644 index 00000000..d7241c81 --- /dev/null +++ b/server/text_generation/models/t5.py @@ -0,0 +1,258 @@ +import torch +import torch.distributed + +from typing import List, Optional, Tuple + +from accelerate import init_empty_weights +from safetensors import safe_open +from transformers import ( + AutoTokenizer, + AutoModelForSeq2SeqLM, + AutoConfig, +) +from transformers.models.t5.parallel_layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, +) + +from text_generation.models import Seq2SeqLM +from text_generation.utils import ( + initialize_torch_distributed, + weight_files, + download_weights, +) + +HAS_BITS_AND_BYTES = True +try: + import bitsandbytes as bnb + from bitsandbytes.nn import Int8Params +except Exception as e: + HAS_BITS_AND_BYTES = False + + +class T5Sharded(Seq2SeqLM): + def __init__( + self, model_id: str, revision: Optional[str] = None, quantize: bool = False + ): + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + device = torch.device(f"cuda:{self.rank}") + dtype = torch.bfloat16 + else: + device = torch.device("cpu") + dtype = torch.float32 + + tokenizer = AutoTokenizer.from_pretrained( + model_id, revision=revision, padding_side="left" + ) + + config = AutoConfig.from_pretrained( + model_id, revision=revision, tp_parallel=True + ) + tokenizer.bos_token_id = config.decoder_start_token_id + + # Only master download weights + if self.master: + download_weights(model_id, revision=revision, extension=".safetensors") + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + if not filenames: + raise ValueError("No safetensors weights found") + + with init_empty_weights(): + model = AutoModelForSeq2SeqLM.from_config(config) + + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + quantize=quantize, + device=device, + rank=self.rank, + world_size=self.world_size, + ) + self.model = model.eval().to(dtype) + torch.distributed.barrier(group=self.process_group) + super(Seq2SeqLM, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @staticmethod + def load_weights( + model, + filenames: List[str], + quantize: bool, + device: torch.device, + rank: int, + world_size: int, + ): + parameters = dict(model.named_parameters()) + for file in filenames: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: + for name in f.keys(): + module_name, param_name = name.rsplit(".", 1) + module = model.get_submodule(module_name) + + current_parameter_tensor = parameters.get(name, None) + + slice_ = f.get_slice(name) + + if isinstance(module, TensorParallelColumnLinear): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif isinstance(module, TensorParallelRowLinear): + if param_name == "weight": + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + else: + tensor = slice_[:] + # XXX: Hack for Rowlinear to add the bias only once. + if rank != 0: + tensor = torch.zeros_like(tensor) + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif name == "lm_head.weight": + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif "relative_attention_bias.weight" in name: + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + else: + try: + tensor = slice_[:] + except: + tensor = f.get_tensor(name) + + if ( + current_parameter_tensor is not None + and current_parameter_tensor.shape != tensor.shape + ): + raise ValueError( + f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" + ) + + tensor = tensor.contiguous() + + if quantize: + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine either because it is not installed " + "or you don't have a GPU.\n" + "You can install it with `pip install bitsandbytes`." + ) + + if ( + type(module) + in [TensorParallelRowLinear, TensorParallelColumnLinear] + and param_name == "weight" + ): + tensor = Int8Params( + tensor, + has_fp16_weights=False, + requires_grad=False, + ).to(device) + state = bnb.MatmulLtState() + state.threshold = 6.0 + state.has_fp16_weights = False + state.memory_efficient_backward = False + state.use_pool = True + state.CB = tensor.CB + state.SCB = tensor.SCB + tensor.CB = None + tensor.SCB = None + + def replace_linear(state): + def linear(input, weight, bias): + out = bnb.matmul( + input, + weight, + state=state, + threshold=state.threshold, + bias=bias, + ) + + if state.CB is not None: + # we converted 8-bit row major to turing/ampere format + # in the first inference pass + # we no longer need the row-major weight + del state.CB + weight.data = state.CxB + + return out + + return linear + + module.linear = replace_linear(state) + + else: + tensor = tensor.to(device) + + if current_parameter_tensor is not None: + module._parameters[param_name] = tensor + else: + module._buffers[param_name] = tensor + + def forward( + self, + input_ids, + attention_mask, + decoder_input_ids, + decoder_attention_mask: Optional, + encoder_last_hidden_state: Optional, + past_key_values: Optional = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]], + ]: + # Model Forward + if past_key_values is not None: + decoder_input_ids = decoder_input_ids[:, -1].unsqueeze(-1) + + # Wrap `encoder_last_hidden_state` because for some reason, Transformers does a `encoder_last_hidden_state[0]` + # internally... + if encoder_last_hidden_state is not None: + encoder_last_hidden_state = [encoder_last_hidden_state] + + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + encoder_outputs=encoder_last_hidden_state, + past_key_values=past_key_values, + use_cache=True, + ) + + # Logits are sharded, so we need to gather them + logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] + torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) + logits = torch.cat(logits, dim=2) + + return ( + logits, + outputs.encoder_last_hidden_state, + outputs.past_key_values, + ) From 1ad3250b8997576fdc52aa99d777c56b9dfed254 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 8 Feb 2023 17:53:33 +0100 Subject: [PATCH 076/793] fix(docker): increase shm size (#60) --- Dockerfile | 2 -- README.md | 37 ++++++++++++++++++++++++++++----- launcher/src/main.rs | 11 ++++++++-- server/text_generation/utils.py | 14 ++++++------- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 93846d77..03e7e5dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,9 +30,7 @@ ENV LANG=C.UTF-8 \ MODEL_ID=bigscience/bloom-560m \ QUANTIZE=false \ NUM_SHARD=1 \ - SAFETENSORS_FAST_GPU=1 \ PORT=80 \ - NCCL_ASYNC_ERROR_HANDLING=1 \ CUDA_HOME=/usr/local/cuda \ LD_LIBRARY_PATH="/opt/miniconda/envs/text-generation/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH" \ CONDA_DEFAULT_ENV=text-generation \ diff --git a/README.md b/README.md index 54851c2a..9763578c 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,9 @@ to power LLMs api-inference widgets. - [Officially Supported Models](#officially-supported-models) - [Get Started](#get-started) - [Docker](#docker) + - [API Documentation](#api-documentation) + - [A note on Shared Memory](#a-note-on-shared-memory-shm) - [Local Install](#local-install) - - [OpenAPI](#api-documentation) - [CUDA Kernels](#cuda-kernels) - [Run BLOOM](#run-bloom) - [Download](#download) @@ -54,7 +55,7 @@ to power LLMs api-inference widgets. - ~~[Galactica](https://huggingface.co/facebook/galactica-120b)~~ (deactivated) - [SantaCoder](https://huggingface.co/bigcode/santacoder) - [GPT-Neox 20B](https://huggingface.co/EleutherAI/gpt-neox-20b) -- [FLAN-T5-XXL](https://huggingface.co/google/flan-t5-xxl): use `--revision pr/26` +- [FLAN-T5-XXL](https://huggingface.co/google/flan-t5-xxl) Other models are supported on a best effort basis using: @@ -75,7 +76,7 @@ model=bigscience/bloom-560m num_shard=2 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --num-shard $num_shard +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --num-shard $num_shard ``` You can then query the model using either the `/generate` or `/generate_stream` routes: @@ -101,6 +102,32 @@ curl 127.0.0.1:8080/generate_stream \ You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. The Swagger UI is also available at: [https://huggingface.github.io/text-generation-inference](https://huggingface.github.io/text-generation-inference). +### A note on Shared Memory (shm) + +[`NCCL`](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/index.html) is a communication framework used by +`PyTorch` to do distributed training/inference. `text-generation-inference` make +use of `NCCL` to enable Tensor Parallelism to dramatically speed up inference for large language models. + +In order to share data between the different devices of a `NCCL` group, `NCCL` might fall back to using the host memory if +peer-to-peer using NVLink or PCI is not possible. + +To allow the container to use 1G of Shared Memory and support SHM sharing, we add `--shm-size 1g` on the above command. + +If you are running `text-generation-inference` inside `Kubernetes`. You can also add Shared Memory to the container by +creating a volume with: + +```yaml +- name: shm + emptyDir: + medium: Memory + sizeLimit: 1Gi +``` + +and mounting it to `/dev/shm`. + +Finally, you can also disable SHM sharing by using the `NCCL_SHM_DISABLE=1` environment variable. However, note that +this will impact performance. + ### Local install You can also opt to install `text-generation-inference` locally. @@ -122,10 +149,10 @@ BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork make run-bloom-560m ``` -**Note:** on some machines, you may also need the OpenSSL libraries. On Linux machines, run: +**Note:** on some machines, you may also need the OpenSSL libraries and gcc. On Linux machines, run: ```shell -sudo apt-get install libssl-dev +sudo apt-get install libssl-dev gcc -y ``` ### CUDA Kernels diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 6684c3de..49175570 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -38,9 +38,9 @@ struct Args { port: u16, #[clap(default_value = "/tmp/text-generation-server", long, env)] shard_uds_path: String, - #[clap(default_value = "0.0.0.0", long, env)] + #[clap(default_value = "localhost", long, env)] master_addr: String, - #[clap(default_value = "6000", long, env)] + #[clap(default_value = "29500", long, env)] master_port: usize, #[clap(long, env)] json_output: bool, @@ -305,6 +305,7 @@ fn shard_manager( ("MASTER_ADDR".into(), master_addr.into()), ("MASTER_PORT".into(), master_port.to_string().into()), ("SAFETENSORS_FAST_GPU".into(), "1".into()), + ("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into()), ]; // If the HUGGINGFACE_HUB_CACHE env var is set, pass it to the shard @@ -322,6 +323,12 @@ fn shard_manager( )); }; + // If the NCCL_SHM_DISABLE env var is set, pass it to the shard + // needed when running NCCL inside a docker container and when you can't increase shm size + if let Ok(nccl_shm_disalbe) = env::var("NCCL_SHM_DISABLE") { + env.push(("NCCL_SHM_DISABLE".into(), nccl_shm_disalbe.into())); + }; + // If the CUDA_VISIBLE_DEVICES env var is set, pass it to the shard if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { env.push(("CUDA_VISIBLE_DEVICES".into(), cuda_visible_devices.into())); diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 2821a124..5f0a2119 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -162,29 +162,29 @@ def initialize_torch_distributed(): world_size = int(os.getenv("WORLD_SIZE", "1")) if torch.cuda.is_available(): - # initialized `torch.distributed` + from torch.distributed import ProcessGroupNCCL # Set the device id. assert world_size <= torch.cuda.device_count(), "Each process is one gpu" device = rank % torch.cuda.device_count() torch.cuda.set_device(device) backend = "nccl" + options = ProcessGroupNCCL.Options() + options.is_high_priority_stream = True + options._timeout = timedelta(seconds=60) else: backend = "gloo" - - master_ip = os.getenv("MASTER_ADDR", "0.0.0.0") - master_port = os.getenv("MASTER_PORT", "6000") - init_method = f"tcp://{master_ip}:{master_port}" + options = None # Call the init process. torch.distributed.init_process_group( backend=backend, - init_method=init_method, world_size=world_size, rank=rank, timeout=timedelta(seconds=60), + pg_options=options ) - return torch.distributed.distributed_c10d._get_default_group(), rank, world_size + return torch.distributed.group.WORLD, rank, world_size def weight_hub_files(model_id, revision=None, extension=".safetensors"): From e520d5b34917f02504a280f87420fd2a5ce0eeb2 Mon Sep 17 00:00:00 2001 From: Yannic Kilcher Date: Wed, 8 Feb 2023 22:30:11 +0100 Subject: [PATCH 077/793] fixed SSE naming (#61) https://en.wikipedia.org/wiki/Server-sent_events --- README.md | 2 +- docs/openapi.json | 4 ++-- router/src/server.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9763578c..466837c2 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ to power LLMs api-inference widgets. ## Features -- Token streaming using Server Side Events (SSE) +- Token streaming using Server-Sent Events (SSE) - [Dynamic batching of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading diff --git a/docs/openapi.json b/docs/openapi.json index 0b80cf13..b4ef3ba6 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -119,8 +119,8 @@ "tags": [ "Text Generation Inference" ], - "summary": "Generate a stream of token using Server Side Events", - "description": "Generate a stream of token using Server Side Events", + "summary": "Generate a stream of token using Server-Sent Events", + "description": "Generate a stream of token using Server-Sent Events", "operationId": "generate_stream", "requestBody": { "content": { diff --git a/router/src/server.rs b/router/src/server.rs index 1a332088..dffdf155 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -151,7 +151,7 @@ async fn generate( Ok((headers, Json(response))) } -/// Generate a stream of token using Server Side Events +/// Generate a stream of token using Server-Sent Events #[utoipa::path( post, tag = "Text Generation Inference", @@ -199,7 +199,7 @@ async fn generate_stream( match infer.generate_stream(req.0).await { Ok(mut response_stream) => { - // Server Side Event stream + // Server-Sent Event stream while let Some(response) = response_stream.next().await { match response { Ok(response) => { From 9af454142a34536ab1f3c149cc8764b7ab460c0d Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 13 Feb 2023 13:02:45 +0100 Subject: [PATCH 078/793] feat: add distributed tracing (#62) --- .github/workflows/tests.yaml | 2 + Cargo.lock | 811 ++++++++++++-------- Cargo.toml | 1 + Dockerfile | 10 +- README.md | 25 + k6/load_test.js | 3 +- launcher/Cargo.toml | 10 +- launcher/src/main.rs | 24 +- router/Cargo.toml | 22 +- router/client/Cargo.toml | 10 +- router/client/build.rs | 8 +- router/client/src/client.rs | 40 +- router/client/src/lib.rs | 12 +- router/client/src/sharded_client.rs | 4 + router/grpc-metadata/Cargo.toml | 10 + router/grpc-metadata/src/lib.rs | 62 ++ router/src/infer.rs | 84 +- router/src/main.rs | 77 +- router/src/queue.rs | 66 +- router/src/server.rs | 47 +- router/src/validation.rs | 23 +- rust-toolchain.toml | 2 +- server/Makefile | 2 +- server/poetry.lock | 494 +++++++++++- server/pyproject.toml | 5 +- server/text_generation/cli.py | 28 +- server/text_generation/models/causal_lm.py | 10 +- server/text_generation/models/seq2seq_lm.py | 7 +- server/text_generation/server.py | 8 +- server/text_generation/tracing.py | 65 ++ server/text_generation/utils.py | 16 +- 31 files changed, 1503 insertions(+), 485 deletions(-) create mode 100644 router/grpc-metadata/Cargo.toml create mode 100644 router/grpc-metadata/src/lib.rs create mode 100644 server/text_generation/tracing.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7bdd3b73..8b659fe2 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,6 +23,8 @@ jobs: toolchain: 1.65.0 override: true components: rustfmt, clippy + - name: Install Protoc + uses: arduino/setup-protoc@v1 - name: Loading cache. uses: actions/cache@v2 id: model_cache diff --git a/Cargo.lock b/Cargo.lock index 0a537115..e8a28bf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] @@ -28,9 +28,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.66" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "async-stream" @@ -55,9 +55,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2", "quote", @@ -70,7 +70,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -131,12 +131,34 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-tracing-opentelemetry" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a43f5b506fc945900d08e541de14b5b9c82860637702ae082888b0fc2654d86" +dependencies = [ + "axum", + "futures", + "http", + "opentelemetry", + "tower", + "tower-http", + "tracing", + "tracing-opentelemetry", +] + [[package]] name = "base64" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bitflags" version = "1.3.2" @@ -154,9 +176,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byteorder" @@ -166,15 +188,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.2.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "bzip2" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" dependencies = [ "bzip2-sys", "libc", @@ -216,9 +238,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.76" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -243,14 +265,14 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.22" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91b9970d7505127a162fdaa9b96428d28a479ba78c9ec7550a63a5d9863db682" +checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" dependencies = [ - "atty", "bitflags", "clap_derive", "clap_lex", + "is-terminal", "once_cell", "strsim 0.10.0", "termcolor", @@ -258,11 +280,11 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro-error", "proc-macro2", "quote", @@ -271,25 +293,24 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" dependencies = [ "os_str_bytes", ] [[package]] name = "console" -version = "0.15.2" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" dependencies = [ "encode_unicode", "lazy_static", "libc", - "terminal_size", "unicode-width", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -349,9 +370,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.11" +version = "0.9.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" dependencies = [ "autocfg", "cfg-if", @@ -362,9 +383,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ "cfg-if", ] @@ -381,12 +402,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.2.3" +version = "3.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d91974fbbe88ec1df0c24a4f00f99583667a7e2e6272b2b92d294d81e462173" +checksum = "bbcf33c2a618cbe41ee43ae6e9f2e48368cd9f9db2896f10167d8d762679f639" dependencies = [ "nix", - "winapi", + "windows-sys 0.45.0", ] [[package]] @@ -424,6 +445,19 @@ dependencies = [ "syn", ] +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "derive_builder" version = "0.9.0" @@ -451,9 +485,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ "block-buffer", "crypto-common", @@ -490,9 +524,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "encode_unicode" @@ -502,13 +536,34 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "esaxx-rs" version = "0.1.8" @@ -529,14 +584,14 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b9663d381d07ae25dc88dbdf27df458faa83a9b25336bcac83d5e452b5fc9d3" +checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -547,9 +602,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" dependencies = [ "crc32fast", "miniz_oxide", @@ -603,9 +658,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" dependencies = [ "futures-channel", "futures-core", @@ -618,9 +673,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", "futures-sink", @@ -628,15 +683,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-executor" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" dependencies = [ "futures-core", "futures-task", @@ -645,15 +700,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" [[package]] name = "futures-macro" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", @@ -662,21 +717,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures-channel", "futures-core", @@ -713,9 +768,19 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "grpc-metadata" +version = "0.1.0" +dependencies = [ + "opentelemetry", + "tonic", + "tracing", + "tracing-opentelemetry", +] [[package]] name = "h2" @@ -732,7 +797,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util", "tracing", ] @@ -744,28 +809,34 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" -version = "0.3.3" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] -name = "heck" -version = "0.4.0" +name = "hermit-abi" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "http" version = "0.2.8" @@ -808,9 +879,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" dependencies = [ "bytes", "futures-channel", @@ -873,9 +944,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", @@ -915,11 +986,33 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-lifetimes" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +dependencies = [ + "libc", + "windows-sys 0.45.0", +] + [[package]] name = "ipnet" -version = "2.5.1" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" + +[[package]] +name = "is-terminal" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" +checksum = "22e18b0a45d56fe973d6db23972bf5bc46f988a4a2385deac9cc29572f09daef" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.45.0", +] [[package]] name = "itertools" @@ -950,15 +1043,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -971,9 +1064,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" + +[[package]] +name = "linux-raw-sys" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "lock_api" @@ -1010,6 +1109,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.7.0" @@ -1024,9 +1132,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.6.5" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" dependencies = [ "autocfg", ] @@ -1055,9 +1163,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" dependencies = [ "adler", ] @@ -1100,14 +1208,14 @@ dependencies = [ [[package]] name = "nix" -version = "0.25.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "autocfg", "bitflags", "cfg-if", "libc", + "static_assertions", ] [[package]] @@ -1118,9 +1226,9 @@ checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" [[package]] name = "nom" -version = "7.1.1" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", @@ -1138,11 +1246,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -1160,9 +1268,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "onig" @@ -1188,9 +1296,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.42" +version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if", @@ -1220,9 +1328,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.77" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03b84c3b2d099b81f0953422b4d4ad58761589d0229b5506356afca05a3670a" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg", "cc", @@ -1231,11 +1339,91 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c928609d087790fc936a1067bdc310ae702bdf3b090c3f281b713622c8bbde" +dependencies = [ + "async-trait", + "futures", + "futures-util", + "http", + "opentelemetry", + "opentelemetry-proto", + "prost", + "thiserror", + "tokio", + "tonic", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61a2f56df5574508dd86aaca016c917489e589ece4141df1b5e349af8d66c28" +dependencies = [ + "futures", + "futures-util", + "opentelemetry", + "prost", + "tonic", + "tonic-build", +] + +[[package]] +name = "opentelemetry_api" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "os_str_bytes" -version = "6.3.1" +version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" [[package]] name = "overload" @@ -1255,22 +1443,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.4" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] name = "paste" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" [[package]] name = "percent-encoding" @@ -1280,9 +1468,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "petgraph" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", "indexmap", @@ -1332,6 +1520,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1358,18 +1556,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.9.0" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698" dependencies = [ "bytes", "prost-derive", @@ -1377,29 +1575,31 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.9.0" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +checksum = "a3f8ad728fb08fe212df3c05169e940fbb6d9d16a877ddde14644a983ba2012e" dependencies = [ "bytes", - "heck 0.3.3", + "heck", "itertools 0.10.5", "lazy_static", "log", "multimap", "petgraph", + "prettyplease", "prost", "prost-types", "regex", + "syn", "tempfile", "which", ] [[package]] name = "prost-derive" -version = "0.9.0" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +checksum = "8bda8c0881ea9f722eb9629376db3d0b903b462477c1aafcb0566610ac28ac5d" dependencies = [ "anyhow", "itertools 0.10.5", @@ -1410,9 +1610,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.9.0" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" +checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" dependencies = [ "bytes", "prost", @@ -1420,9 +1620,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -1459,12 +1659,10 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.3" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" dependencies = [ - "autocfg", - "crossbeam-deque", "either", "rayon-core", ] @@ -1482,9 +1680,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.3" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -1514,15 +1712,24 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.28" @@ -1540,11 +1747,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" +checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" dependencies = [ - "base64", + "base64 0.21.0", "bytes", "encoding_rs", "futures-core", @@ -1610,6 +1817,20 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rustix" +version = "0.36.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.45.0", +] + [[package]] name = "rustversion" version = "1.0.11" @@ -1618,9 +1839,9 @@ checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "same-file" @@ -1633,12 +1854,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "windows-sys 0.42.0", ] [[package]] @@ -1649,9 +1869,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "security-framework" -version = "2.7.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ "bitflags", "core-foundation", @@ -1662,9 +1882,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -1672,18 +1892,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.150" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.150" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -1692,9 +1912,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" dependencies = [ "itoa", "ryu", @@ -1791,12 +2011,18 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" dependencies = [ - "base64", + "base64 0.13.1", "nom", "serde", "unicode-segmentation", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.8.0" @@ -1827,9 +2053,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -1838,9 +2064,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "tar" @@ -1869,29 +2095,21 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] -[[package]] -name = "terminal_size" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "text-generation-client" version = "0.2.1" dependencies = [ "futures", + "grpc-metadata", "prost", + "prost-build", "thiserror", "tokio", "tonic", @@ -1905,7 +2123,7 @@ dependencies = [ name = "text-generation-launcher" version = "0.2.1" dependencies = [ - "clap 4.0.22", + "clap 4.1.4", "ctrlc", "float_eq", "reqwest", @@ -1922,9 +2140,12 @@ version = "0.2.1" dependencies = [ "async-stream", "axum", - "clap 4.0.22", + "axum-tracing-opentelemetry", + "clap 4.1.4", "futures", "nohash-hasher", + "opentelemetry", + "opentelemetry-otlp", "parking_lot", "rand", "serde", @@ -1935,6 +2156,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "utoipa", "utoipa-swagger-ui", @@ -1951,18 +2173,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -1971,18 +2193,19 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "50f297120ff9d4efe680df143d5631bba9c75fa371992b7fcb33eb3453cb0a07" dependencies = [ + "cfg-if", "once_cell", ] [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -2000,9 +2223,9 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" @@ -2041,9 +2264,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.21.2" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", "bytes", @@ -2056,7 +2279,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "winapi", + "windows-sys 0.42.0", ] [[package]] @@ -2071,9 +2294,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", @@ -2082,9 +2305,9 @@ dependencies = [ [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", "tokio", @@ -2103,23 +2326,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "bc6a3b08b64e6dfad376fa2432c7b1f01522e37a623c3050bc95db2d3ff21583" dependencies = [ "bytes", "futures-core", @@ -2131,13 +2340,14 @@ dependencies = [ [[package]] name = "tonic" -version = "0.6.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" +checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" dependencies = [ "async-stream", "async-trait", - "base64", + "axum", + "base64 0.13.1", "bytes", "futures-core", "futures-util", @@ -2152,7 +2362,7 @@ dependencies = [ "prost-derive", "tokio", "tokio-stream", - "tokio-util 0.6.10", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -2162,10 +2372,11 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.6.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" +checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" dependencies = [ + "prettyplease", "proc-macro2", "prost-build", "quote", @@ -2186,7 +2397,7 @@ dependencies = [ "rand", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -2194,9 +2405,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ "bitflags", "bytes", @@ -2209,6 +2420,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -2288,6 +2500,20 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -2304,12 +2530,16 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "serde", "serde_json", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", "tracing-serde", @@ -2317,15 +2547,15 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicase" @@ -2338,15 +2568,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" @@ -2368,9 +2598,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" @@ -2494,9 +2724,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2504,9 +2734,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -2519,9 +2749,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -2531,9 +2761,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2541,9 +2771,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -2554,15 +2784,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -2570,9 +2800,9 @@ dependencies = [ [[package]] name = "which" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ "either", "libc", @@ -2610,19 +2840,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -2630,85 +2847,79 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc", ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.0" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] [[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" +name = "windows-targets" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] [[package]] -name = "windows_aarch64_msvc" -version = "0.42.0" +name = "windows_aarch64_gnullvm" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] -name = "windows_i686_gnu" -version = "0.36.1" +name = "windows_aarch64_msvc" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" - -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.36.1" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winreg" diff --git a/Cargo.toml b/Cargo.toml index 3720af32..b3bd5dce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "router", "router/client", + "router/grpc-metadata", "launcher" ] diff --git a/Dockerfile b/Dockerfile index 03e7e5dc..907379dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,10 @@ -FROM rust:1.65 as router-builder +FROM rust:1.67 as router-builder + +RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \ + curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \ + unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \ + unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \ + rm -f $PROTOC_ZIP WORKDIR /usr/src @@ -10,7 +16,7 @@ WORKDIR /usr/src/router RUN cargo install --path . -FROM rust:1.65 as launcher-builder +FROM rust:1.67 as launcher-builder WORKDIR /usr/src diff --git a/README.md b/README.md index 466837c2..39cec857 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ to power LLMs api-inference widgets. - [Docker](#docker) - [API Documentation](#api-documentation) - [A note on Shared Memory](#a-note-on-shared-memory-shm) + - [Distributed Tracing](#distributed-tracing) - [Local Install](#local-install) - [CUDA Kernels](#cuda-kernels) - [Run BLOOM](#run-bloom) @@ -46,6 +47,7 @@ to power LLMs api-inference widgets. - Logits warpers (temperature scaling, topk, repetition penalty ...) - Stop sequences - Log probabilities +- Distributed tracing with Open Telemetry ## Officially supported models @@ -102,6 +104,11 @@ curl 127.0.0.1:8080/generate_stream \ You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. The Swagger UI is also available at: [https://huggingface.github.io/text-generation-inference](https://huggingface.github.io/text-generation-inference). +### Distributed Tracing + +`text-generation-inference` is instrumented with distributed tracing using OpenTelemetry. You can use this feature +by setting the address to an OTLP collector with the `--otlp-endpoint` argument. + ### A note on Shared Memory (shm) [`NCCL`](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/index.html) is a communication framework used by @@ -142,6 +149,24 @@ conda create -n text-generation-inference python=3.9 conda activate text-generation-inference ``` +You may also need to install Protoc. + +On Linux: + +```shell +PROTOC_ZIP=protoc-21.12-linux-x86_64.zip +curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP +sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc +sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' +rm -f $PROTOC_ZIP +``` + +On MacOS, using Homebrew: + +```shell +brew install protobuf +``` + Then run: ```shell diff --git a/k6/load_test.js b/k6/load_test.js index 3f3791af..516b5666 100644 --- a/k6/load_test.js +++ b/k6/load_test.js @@ -38,7 +38,8 @@ function sample_example(inputs, max_new_tokens, name) { parameters: { max_new_tokens: max_new_tokens, do_sample: true, - top_p: 0.9 + top_p: 0.9, + seed: 0 } }); let params = { diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 7ddb87ee..a0d2b089 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -6,14 +6,14 @@ authors = ["Olivier Dehaene"] description = "Text Generation Launcher" [dependencies] -clap = { version = "4.0.15", features = ["derive", "env"] } -ctrlc = { version = "3.2.3", features = ["termination"] } -serde_json = "1.0.89" +clap = { version = "4.1.4", features = ["derive", "env"] } +ctrlc = { version = "3.2.5", features = ["termination"] } +serde_json = "1.0.93" subprocess = "0.2.9" tracing = "0.1.37" tracing-subscriber = { version = "0.3.16", features = ["json"] } [dev-dependencies] float_eq = "1.0.1" -reqwest = { version = "0.11.13", features = ["blocking", "json"] } -serde = { version = "1.0.150", features = ["derive"] } +reqwest = { version = "0.11.14", features = ["blocking", "json"] } +serde = { version = "1.0.152", features = ["derive"] } diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 49175570..3c8d9fcc 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -44,6 +44,8 @@ struct Args { master_port: usize, #[clap(long, env)] json_output: bool, + #[clap(long, env)] + otlp_endpoint: Option, } fn main() -> ExitCode { @@ -62,6 +64,7 @@ fn main() -> ExitCode { master_addr, master_port, json_output, + otlp_endpoint, } = Args::parse(); if json_output { @@ -99,6 +102,7 @@ fn main() -> ExitCode { let status_sender = status_sender.clone(); let shutdown = shutdown.clone(); let shutdown_sender = shutdown_sender.clone(); + let otlp_endpoint = otlp_endpoint.clone(); thread::spawn(move || { shard_manager( model_id, @@ -109,6 +113,7 @@ fn main() -> ExitCode { num_shard, master_addr, master_port, + otlp_endpoint, status_sender, shutdown, shutdown_sender, @@ -165,7 +170,7 @@ fn main() -> ExitCode { "--port".to_string(), port.to_string(), "--master-shard-uds-path".to_string(), - format!("{}-0", shard_uds_path), + format!("{shard_uds_path}-0"), "--tokenizer-name".to_string(), model_id, ]; @@ -174,6 +179,12 @@ fn main() -> ExitCode { argv.push("--json-output".to_string()); } + // OpenTelemetry + if let Some(otlp_endpoint) = otlp_endpoint { + argv.push("--otlp-endpoint".to_string()); + argv.push(otlp_endpoint); + } + let mut webserver = match Popen::create( &argv, PopenConfig { @@ -264,12 +275,13 @@ fn shard_manager( world_size: usize, master_addr: String, master_port: usize, + otlp_endpoint: Option, status_sender: mpsc::Sender, shutdown: Arc>, _shutdown_sender: mpsc::Sender<()>, ) { // Get UDS path - let uds_string = format!("{}-{}", uds_path, rank); + let uds_string = format!("{uds_path}-{rank}"); let uds = Path::new(&uds_string); // Clean previous runs fs::remove_file(uds).unwrap_or_default(); @@ -286,6 +298,7 @@ fn shard_manager( "--json-output".to_string(), ]; + // Activate tensor parallelism if world_size > 1 { shard_argv.push("--sharded".to_string()); } @@ -294,11 +307,18 @@ fn shard_manager( shard_argv.push("--quantize".to_string()) } + // Model optional revision if let Some(revision) = revision { shard_argv.push("--revision".to_string()); shard_argv.push(revision) } + // OpenTelemetry + if let Some(otlp_endpoint) = otlp_endpoint { + shard_argv.push("--otlp-endpoint".to_string()); + shard_argv.push(otlp_endpoint); + } + let mut env = vec![ ("RANK".into(), rank.to_string().into()), ("WORLD_SIZE".into(), world_size.to_string().into()), diff --git a/router/Cargo.toml b/router/Cargo.toml index 186f97c5..f1ace790 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -15,20 +15,24 @@ path = "src/main.rs" [dependencies] async-stream = "0.3.3" axum = { version = "0.6.4", features = ["json"] } +axum-tracing-opentelemetry = "0.9.0" text-generation-client = { path = "client" } -clap = { version = "4.0.15", features = ["derive", "env"] } -futures = "0.3.24" +clap = { version = "4.1.4", features = ["derive", "env"] } +futures = "0.3.26" nohash-hasher = "0.2.0" +opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } +opentelemetry-otlp = "0.11.0" parking_lot = "0.12.1" rand = "0.8.5" -serde = "1.0.145" -serde_json = "1.0.85" -thiserror = "1.0.37" -tokenizers = "0.13.0" -tokio = { version = "1.21.1", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } +serde = "1.0.152" +serde_json = "1.0.93" +thiserror = "1.0.38" +tokenizers = "0.13.2" +tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } tokio-stream = "0.1.11" -tracing = "0.1.36" -tracing-subscriber = { version = "0.3.15", features = ["json"] } +tracing = "0.1.37" +tracing-opentelemetry = "0.18.0" +tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } utoipa = { version = "3.0.1", features = ["axum_extras"] } utoipa-swagger-ui = { version = "3.0.2", features = ["axum"] } diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 4cddf81b..7c7ed48e 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -5,13 +5,15 @@ edition = "2021" [dependencies] futures = "^0.3" -prost = "^0.9" +grpc-metadata = { path = "../grpc-metadata" } +prost = "^0.11" thiserror = "^1.0" -tokio = { version = "^1.21", features = ["sync"] } -tonic = "^0.6" +tokio = { version = "^1.25", features = ["sync"] } +tonic = "^0.8" tower = "^0.4" tracing = "^0.1" tracing-error = "^0.2" [build-dependencies] -tonic-build = "0.6.2" +tonic-build = "0.8.4" +prost-build = "0.11.6" diff --git a/router/client/build.rs b/router/client/build.rs index 7b577fcc..497be545 100644 --- a/router/client/build.rs +++ b/router/client/build.rs @@ -3,13 +3,17 @@ use std::fs; fn main() -> Result<(), Box> { println!("cargo:rerun-if-changed=../../proto/generate.proto"); fs::create_dir("src/pb").unwrap_or(()); + + let mut config = prost_build::Config::new(); + config.protoc_arg("--experimental_allow_proto3_optional"); + tonic_build::configure() .build_client(true) .build_server(false) .out_dir("src/pb") .include_file("mod.rs") - .compile(&["../../proto/generate.proto"], &["../../proto"]) - .unwrap_or_else(|e| panic!("protobuf compilation failed: {}", e)); + .compile_with_config(config, &["../../proto/generate.proto"], &["../../proto"]) + .unwrap_or_else(|e| panic!("protobuf compilation failed: {e}")); Ok(()) } diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 77a43110..1f0d23f2 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -2,8 +2,9 @@ use crate::pb::generate::v1::text_generation_service_client::TextGenerationServiceClient; use crate::pb::generate::v1::*; use crate::Result; +use grpc_metadata::InjectTelemetryContext; use tonic::transport::{Channel, Uri}; -use tracing::*; +use tracing::instrument; /// Text Generation Inference gRPC client #[derive(Clone)] @@ -38,12 +39,8 @@ impl Client { /// Returns a list of uris or unix sockets of all shards #[instrument(skip(self))] pub async fn service_discovery(&mut self) -> Result> { - let request = tonic::Request::new(ServiceDiscoveryRequest {}); - let response = self - .stub - .service_discovery(request) - .instrument(info_span!("service_discovery")) - .await?; + let request = tonic::Request::new(ServiceDiscoveryRequest {}).inject_context(); + let response = self.stub.service_discovery(request).await?; let urls = response .into_inner() .urls @@ -60,11 +57,8 @@ impl Client { /// Clear the past generations cache #[instrument(skip(self))] pub async fn clear_cache(&mut self) -> Result<()> { - let request = tonic::Request::new(ClearCacheRequest {}); - self.stub - .clear_cache(request) - .instrument(info_span!("clear_cache")) - .await?; + let request = tonic::Request::new(ClearCacheRequest {}).inject_context(); + self.stub.clear_cache(request).await?; Ok(()) } @@ -72,15 +66,10 @@ impl Client { /// /// Returns Generation for each request in batch /// and the next cached batch - #[instrument(skip(self))] + #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))] pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { - let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }); - let response = self - .stub - .prefill(request) - .instrument(info_span!("prefill")) - .await? - .into_inner(); + let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }).inject_context(); + let response = self.stub.prefill(request).await?.into_inner(); Ok((response.generations, response.batch)) } @@ -88,18 +77,13 @@ impl Client { /// /// Returns Generation for each request in batches /// and the next cached batch - #[instrument(skip(self))] + #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::()))] pub async fn decode( &mut self, batches: Vec, ) -> Result<(Vec, Option)> { - let request = tonic::Request::new(DecodeRequest { batches }); - let response = self - .stub - .decode(request) - .instrument(info_span!("decode")) - .await? - .into_inner(); + let request = tonic::Request::new(DecodeRequest { batches }).inject_context(); + let response = self.stub.decode(request).await?.into_inner(); Ok((response.generations, response.batch)) } } diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 2767c605..21fbc1e1 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -17,21 +17,25 @@ use tonic::Status; #[derive(Error, Debug, Clone)] pub enum ClientError { - #[error("Could not connect to Text Generation server: {0:?}")] + #[error("Could not connect to Text Generation server: {0}")] Connection(String), - #[error("Server error: {0:?}")] + #[error("Server error: {0}")] Generation(String), } impl From for ClientError { fn from(err: Status) -> Self { - Self::Generation(err.message().to_string()) + let err = Self::Generation(err.message().to_string()); + tracing::error!("{err}"); + err } } impl From for ClientError { fn from(err: transport::Error) -> Self { - Self::Connection(err.to_string()) + let err = Self::Connection(err.to_string()); + tracing::error!("{err}"); + err } } diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 56335f92..2e662ca3 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -4,6 +4,7 @@ use crate::{Batch, Client, Generation}; use futures::future::join_all; use futures::future::select_all; use tonic::transport::Uri; +use tracing::instrument; /// Text Generation Inference gRPC multi client pub struct ShardedClient { @@ -38,6 +39,7 @@ impl ShardedClient { } /// Clear the past generations cache + #[instrument(skip(self))] pub async fn clear_cache(&mut self) -> Result<()> { let futures: Vec<_> = self .clients @@ -51,6 +53,7 @@ impl ShardedClient { /// /// Returns Generation for each request in batch /// and the next cached batch + #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))] pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { let futures: Vec<_> = self .clients @@ -66,6 +69,7 @@ impl ShardedClient { /// /// Returns Generation for each request in batches /// and the next cached batch + #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::()))] pub async fn decode( &mut self, batches: Vec, diff --git a/router/grpc-metadata/Cargo.toml b/router/grpc-metadata/Cargo.toml new file mode 100644 index 00000000..311092e3 --- /dev/null +++ b/router/grpc-metadata/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "grpc-metadata" +version = "0.1.0" +edition = "2021" + +[dependencies] +opentelemetry = "0.18.0" +tonic = "^0.8" +tracing = "^0.1" +tracing-opentelemetry = "0.18.0" diff --git a/router/grpc-metadata/src/lib.rs b/router/grpc-metadata/src/lib.rs new file mode 100644 index 00000000..7ba353fa --- /dev/null +++ b/router/grpc-metadata/src/lib.rs @@ -0,0 +1,62 @@ +//! A crate to extract and inject a OpenTelemetry context from and to a gRPC request. +//! Inspired by: https://github.com/open-telemetry/opentelemetry-rust gRPC examples + +use opentelemetry::global; +use opentelemetry::propagation::{Extractor, Injector}; +use tracing_opentelemetry::OpenTelemetrySpanExt; + +/// Extract context metadata from a gRPC request's metadata +struct MetadataExtractor<'a>(pub &'a tonic::metadata::MetadataMap); + +impl<'a> Extractor for MetadataExtractor<'a> { + /// Get a value for a key from the MetadataMap. If the value can't be converted to &str, returns None + fn get(&self, key: &str) -> Option<&str> { + self.0.get(key).and_then(|metadata| metadata.to_str().ok()) + } + + /// Collect all the keys from the MetadataMap. + fn keys(&self) -> Vec<&str> { + self.0 + .keys() + .map(|key| match key { + tonic::metadata::KeyRef::Ascii(v) => v.as_str(), + tonic::metadata::KeyRef::Binary(v) => v.as_str(), + }) + .collect::>() + } +} + +/// Inject context in the metadata of a gRPC request. +struct MetadataInjector<'a>(pub &'a mut tonic::metadata::MetadataMap); + +impl<'a> Injector for MetadataInjector<'a> { + /// Set a key and value in the MetadataMap. Does nothing if the key or value are not valid inputs + fn set(&mut self, key: &str, value: String) { + if let Ok(key) = tonic::metadata::MetadataKey::from_bytes(key.as_bytes()) { + if let Ok(val) = value.parse() { + self.0.insert(key, val); + } + } + } +} + +/// Get a context from the global context and inject the span into a gRPC request's metadata. +fn inject(metadata: &mut tonic::metadata::MetadataMap) { + global::get_text_map_propagator(|propagator| { + propagator.inject_context( + &tracing::Span::current().context(), + &mut MetadataInjector(metadata), + ) + }) +} + +pub trait InjectTelemetryContext { + fn inject_context(self) -> Self; +} + +impl InjectTelemetryContext for tonic::Request { + fn inject_context(mut self) -> Self { + inject(self.metadata_mut()); + self + } +} diff --git a/router/src/infer.rs b/router/src/infer.rs index 159b7ca7..4e368492 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -13,7 +13,7 @@ use tokio::sync::{mpsc, Notify, Semaphore, TryAcquireError}; use tokio::time::Instant; use tokio_stream::wrappers::UnboundedReceiverStream; use tokio_stream::StreamExt; -use tracing::instrument; +use tracing::{info_span, instrument, Instrument, Span}; /// Inference struct #[derive(Clone)] @@ -69,13 +69,21 @@ impl Infer { } /// Add a new request to the queue and return a stream of InferStreamResponse + #[instrument(skip(self))] pub(crate) async fn generate_stream( &self, request: GenerateRequest, ) -> Result>, InferError> { // Limit concurrent requests by acquiring a permit from the semaphore // This permit will live as long as Entry - let permit = self.clone().limit_concurrent_requests.try_acquire_owned()?; + let permit = self + .clone() + .limit_concurrent_requests + .try_acquire_owned() + .map_err(|err| { + tracing::error!("{err}"); + err + })?; // Validate request let valid_request = self.validation.validate(request).await?; @@ -87,7 +95,9 @@ impl Infer { self.queue.append(Entry { request: valid_request, response_tx, - time: Instant::now(), + span: Span::current(), + temp_span: None, + queue_time: Instant::now(), batch_time: None, _permit: permit, }); @@ -101,6 +111,7 @@ impl Infer { } /// Add a new request to the queue and return a InferResponse + #[instrument(skip(self))] pub(crate) async fn generate( &self, request: GenerateRequest, @@ -160,7 +171,9 @@ impl Infer { start, }) } else { - Err(InferError::IncompleteGeneration) + let err = InferError::IncompleteGeneration; + tracing::error!("{err}"); + Err(err) } } } @@ -169,7 +182,6 @@ impl Infer { /// Will be launched in a background Tokio task /// /// Batches requests and sends them to the inference server -#[instrument(skip(client, queue, shared))] async fn batching_task( mut client: ShardedClient, max_batch_size: usize, @@ -188,8 +200,10 @@ async fn batching_task( // Get the next batch from the queue // This batch might be smaller than the maximum batch size if there are not enough requests // waiting in the queue - while let Some((mut entries, batch)) = queue.next_batch(None, max_batch_size).await { - let mut cached_batch = wrap_future(client.prefill(batch), &mut entries).await; + while let Some((mut entries, batch, span)) = queue.next_batch(None, max_batch_size).await { + let mut cached_batch = wrap_future(client.prefill(batch), &mut entries) + .instrument(span) + .await; let mut waiting_tokens = 1; // We loop until we do not receive any cached batch from the inference server (== until @@ -210,13 +224,27 @@ async fn batching_task( }; // Try to get a new batch - if let Some((mut new_entries, new_batch)) = queue + if let Some((mut new_entries, new_batch, span)) = queue .next_batch(min_size, max_batch_size - batch_size as usize) .await { + let new_batch_size = new_batch.size; + entries.iter_mut().for_each(|(_, entry)| { + // Create a new span to add the info that this entry is waiting + // because a new batch is being computed + let entry_waiting_span = + info_span!(parent: &entry.span, "waiting", batch_size = new_batch_size); + // Add relationship + entry_waiting_span.follows_from(&span); + // Update entry + entry.temp_span = Some(entry_waiting_span); + }); + // Generate one token for this new batch to have the attention past in cache let new_cached_batch = - wrap_future(client.prefill(new_batch), &mut new_entries).await; + wrap_future(client.prefill(new_batch), &mut new_entries) + .instrument(span) + .await; // Reset waiting counter waiting_tokens = 1; // Extend current batch with the new batch @@ -226,8 +254,23 @@ async fn batching_task( } } } - - cached_batch = wrap_future(client.decode(batches), &mut entries).await; + // Create span for this batch to add context to inference calls + let next_batch_size = entries.len(); + let next_batch_span = + info_span!(parent: None, "batch", batch_size = next_batch_size); + entries.iter_mut().for_each(|(_, entry)| { + // Create a new span to link the batch back to this entry + let entry_batch_span = + info_span!(parent: &entry.span, "infer", batch_size = next_batch_size); + // Add relationship + entry_batch_span.follows_from(&next_batch_span); + // Update entry + entry.temp_span = Some(entry_batch_span); + }); + + cached_batch = wrap_future(client.decode(batches), &mut entries) + .instrument(next_batch_span) + .await; waiting_tokens += 1; } } @@ -235,6 +278,7 @@ async fn batching_task( } /// Wrap a future inside a match statement to handle errors and send the responses to Infer +#[instrument(skip_all)] async fn wrap_future( future: impl Future, Option), ClientError>>, entries: &mut IntMap, @@ -246,24 +290,31 @@ async fn wrap_future( } // If we have an error, we discard the whole batch Err(err) => { - send_error(err, entries); + send_errors(err, entries); None } } } /// Send errors to Infer for all `entries` -fn send_error(error: ClientError, entries: &mut IntMap) { +#[instrument(skip_all)] +fn send_errors(error: ClientError, entries: &mut IntMap) { entries.drain().for_each(|(_, entry)| { + // Create and enter a span to link this function back to the entry + let _send_error_span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_error").entered(); + let err = InferError::GenerationError(error.to_string()); + tracing::error!("{err}"); + // unwrap_or is valid here as we don't care if the receiver is gone. entry .response_tx - .send(Err(InferError::GenerationError(error.to_string()))) + .send(Err(err)) .unwrap_or(()); }); } /// Send one or multiple `InferStreamResponse` to Infer for all `entries` +#[instrument(skip_all)] fn send_generations(generations: Vec, entries: &mut IntMap) { generations.into_iter().for_each(|generation| { // Get entry @@ -272,6 +323,9 @@ fn send_generations(generations: Vec, entries: &mut IntMap, entries: &mut IntMap, } fn main() -> Result<(), std::io::Error> { @@ -43,14 +54,9 @@ fn main() -> Result<(), std::io::Error> { tokenizer_name, validation_workers, json_output, + otlp_endpoint, } = args; - if json_output { - tracing_subscriber::fmt().json().init(); - } else { - tracing_subscriber::fmt().compact().init(); - } - if validation_workers == 0 { panic!("validation_workers must be > 0"); } @@ -67,6 +73,8 @@ fn main() -> Result<(), std::io::Error> { .build() .unwrap() .block_on(async { + init_logging(otlp_endpoint, json_output); + // Instantiate sharded client from the master unix socket let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path) .await @@ -96,3 +104,58 @@ fn main() -> Result<(), std::io::Error> { Ok(()) }) } + +/// Init logging using env variables LOG_LEVEL and LOG_FORMAT: +/// - otlp_endpoint is an optional URL to an Open Telemetry collector +/// - LOG_LEVEL may be TRACE, DEBUG, INFO, WARN or ERROR (default to INFO) +/// - LOG_FORMAT may be TEXT or JSON (default to TEXT) +fn init_logging(otlp_endpoint: Option, json_output: bool) { + let mut layers = Vec::new(); + + // STDOUT/STDERR layer + let fmt_layer = tracing_subscriber::fmt::layer() + .with_file(true) + .with_line_number(true); + + let fmt_layer = match json_output { + true => fmt_layer.json().flatten_event(true).boxed(), + false => fmt_layer.boxed(), + }; + layers.push(fmt_layer); + + // OpenTelemetry tracing layer + if let Some(otlp_endpoint) = otlp_endpoint { + global::set_text_map_propagator(TraceContextPropagator::new()); + + let tracer = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint(otlp_endpoint), + ) + .with_trace_config( + trace::config() + .with_resource(Resource::new(vec![KeyValue::new( + "service.name", + "text-generation-inference.router", + )])) + .with_sampler(Sampler::AlwaysOn), + ) + .install_batch(opentelemetry::runtime::Tokio); + + if let Ok(tracer) = tracer { + layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed()); + axum_tracing_opentelemetry::init_propagator().unwrap(); + }; + } + + // Filter events with LOG_LEVEL + let env_filter = + EnvFilter::try_from_env("LOG_LEVEL").unwrap_or_else(|_| EnvFilter::new("info")); + + tracing_subscriber::registry() + .with(env_filter) + .with(layers) + .init(); +} diff --git a/router/src/queue.rs b/router/src/queue.rs index 2aaf93b1..b155a1af 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -7,6 +7,7 @@ use text_generation_client::{Batch, Request}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tokio::sync::{mpsc, oneshot, OwnedSemaphorePermit}; use tokio::time::Instant; +use tracing::{info_span, instrument, Span}; /// Queue entry #[derive(Debug)] @@ -15,8 +16,12 @@ pub(crate) struct Entry { pub request: ValidGenerateRequest, /// Response sender to communicate between the Infer struct and the batching_task pub response_tx: UnboundedSender>, - /// Instant when this entry was created - pub time: Instant, + /// Span that will live as long as entry + pub span: Span, + /// Temporary span used as a guard when logging inference, wait times... + pub temp_span: Option, + /// Instant when this entry was queued + pub queue_time: Instant, /// Instant when this entry was added to a batch pub batch_time: Option, /// Permit @@ -42,13 +47,17 @@ impl Queue { } /// Append an entry to the queue + #[instrument(skip_all)] pub(crate) fn append(&self, entry: Entry) { // Send append command to the background task managing the state // Unwrap is safe here - self.queue_sender.send(QueueCommand::Append(entry)).unwrap(); + self.queue_sender + .send(QueueCommand::Append(entry, Span::current())) + .unwrap(); } // Get the next batch + #[instrument(skip(self))] pub(crate) async fn next_batch( &self, min_size: Option, @@ -63,6 +72,7 @@ impl Queue { min_size, max_size, response_sender, + span: Span::current(), }) .unwrap(); // Await on response channel @@ -77,15 +87,16 @@ async fn queue_task(mut receiver: UnboundedReceiver) { while let Some(cmd) = receiver.recv().await { match cmd { - QueueCommand::Append(entry) => state.append(entry), + QueueCommand::Append(entry, span) => span.in_scope(|| state.append(entry)), QueueCommand::NextBatch { min_size, max_size, response_sender, - } => { + span, + } => span.in_scope(|| { let next_batch = state.next_batch(min_size, max_size); response_sender.send(next_batch).unwrap_or(()); - } + }), } } } @@ -113,7 +124,12 @@ impl State { } /// Append an entry to the queue - fn append(&mut self, entry: Entry) { + fn append(&mut self, mut entry: Entry) { + // Create a span that will live as long as the entry is in the queue waiting to be batched + let queue_span = info_span!(parent: &entry.span, "queued"); + entry.temp_span = Some(queue_span); + + // Push entry in the queue self.entries.push((self.next_id, entry)); self.next_id += 1; } @@ -133,6 +149,10 @@ impl State { let next_batch_size = min(self.entries.len(), max_size); + // Create span for this batch to add context to inference calls + let next_batch_span = info_span!(parent: None, "batch", batch_size = next_batch_size); + next_batch_span.follows_from(&Span::current()); + let mut batch_requests = Vec::with_capacity(next_batch_size); let mut batch_entries = IntMap::with_capacity_and_hasher(next_batch_size, BuildNoHashHasher::default()); @@ -141,6 +161,14 @@ impl State { self.entries .drain(..next_batch_size) .for_each(|(id, mut entry)| { + // Create a new span to link the batch back to this entry + let entry_batch_span = + info_span!(parent: &entry.span, "infer", batch_size = next_batch_size); + // Add relationship + entry_batch_span.follows_from(&next_batch_span); + // Update entry + entry.temp_span = Some(entry_batch_span); + batch_requests.push(Request { id, inputs: entry.request.inputs.clone(), @@ -162,19 +190,20 @@ impl State { // Increment batch id self.next_batch_id += 1; - Some((batch_entries, batch)) + Some((batch_entries, batch, next_batch_span)) } } -type NextBatch = (IntMap, Batch); +type NextBatch = (IntMap, Batch, Span); #[derive(Debug)] enum QueueCommand { - Append(Entry), + Append(Entry, Span), NextBatch { min_size: Option, max_size: usize, response_sender: oneshot::Sender>, + span: Span, }, } @@ -184,6 +213,7 @@ mod tests { use std::sync::Arc; use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParameters}; use tokio::sync::{mpsc, Semaphore}; + use tracing::info_span; fn default_entry() -> Entry { let semaphore = Arc::new(Semaphore::new(1)); @@ -208,7 +238,9 @@ mod tests { }, }, response_tx, - time: Instant::now(), + span: info_span!("entry"), + temp_span: None, + queue_time: Instant::now(), batch_time: None, _permit: permit, } @@ -244,7 +276,7 @@ mod tests { state.append(default_entry()); state.append(default_entry()); - let (entries, batch) = state.next_batch(None, 2).unwrap(); + let (entries, batch, _) = state.next_batch(None, 2).unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&0)); assert!(entries.contains_key(&1)); @@ -273,7 +305,7 @@ mod tests { state.append(default_entry()); state.append(default_entry()); - let (entries, batch) = state.next_batch(None, 1).unwrap(); + let (entries, batch, _) = state.next_batch(None, 1).unwrap(); assert_eq!(entries.len(), 1); assert!(entries.contains_key(&0)); assert_eq!(batch.id, 0); @@ -285,7 +317,7 @@ mod tests { state.append(default_entry()); - let (entries, batch) = state.next_batch(None, 3).unwrap(); + let (entries, batch, _) = state.next_batch(None, 3).unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&1)); assert!(entries.contains_key(&2)); @@ -317,7 +349,7 @@ mod tests { queue.append(default_entry()); queue.append(default_entry()); - let (entries, batch) = queue.next_batch(None, 2).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, 2).await.unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&0)); assert!(entries.contains_key(&1)); @@ -337,7 +369,7 @@ mod tests { queue.append(default_entry()); queue.append(default_entry()); - let (entries, batch) = queue.next_batch(None, 1).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, 1).await.unwrap(); assert_eq!(entries.len(), 1); assert!(entries.contains_key(&0)); assert_eq!(batch.id, 0); @@ -345,7 +377,7 @@ mod tests { queue.append(default_entry()); - let (entries, batch) = queue.next_batch(None, 3).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, 3).await.unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&1)); assert!(entries.contains_key(&2)); diff --git a/router/src/server.rs b/router/src/server.rs index dffdf155..432586bb 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -10,6 +10,7 @@ use axum::response::sse::{Event, KeepAlive, Sse}; use axum::response::IntoResponse; use axum::routing::{get, post}; use axum::{Json, Router}; +use axum_tracing_opentelemetry::opentelemetry_tracing_layer; use futures::Stream; use std::convert::Infallible; use std::net::SocketAddr; @@ -18,7 +19,7 @@ use tokenizers::Tokenizer; use tokio::signal; use tokio::time::Instant; use tokio_stream::StreamExt; -use tracing::instrument; +use tracing::{info_span, instrument, Instrument}; use utoipa::OpenApi; use utoipa_swagger_ui::SwaggerUi; @@ -75,7 +76,7 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json { // Server-Sent Event stream while let Some(response) = response_stream.next().await { @@ -243,13 +242,11 @@ async fn generate_stream( // Tracing metadata span.record("total_time", format!("{:?}", total_time)); - span - .record("validation_time", format!("{:?}", validation_time)); + span.record("validation_time", format!("{:?}", validation_time)); span.record("queue_time", format!("{:?}", queue_time)); - span - .record("inference_time", format!("{:?}", inference_time)); - span - .record("time_per_token", format!("{:?}", time_per_token)); + span.record("inference_time", format!("{:?}", inference_time)); + span.record("time_per_token", format!("{:?}", time_per_token)); + span.record("seed", format!("{:?}", generated_text.seed)); tracing::info!(parent: &span, "Output: {}", generated_text.text); // StreamResponse @@ -264,19 +261,17 @@ async fn generate_stream( } } } - // Trace and yield error + // yield error Err(err) => { error = true; - tracing::error!("{}", err.to_string()); yield Ok(Event::from(err)) } } } }, - // Trace and yield error + // yield error Err(err) => { error = true; - tracing::error!("{}", err.to_string()); yield Ok(Event::from(err)) } } @@ -284,7 +279,7 @@ async fn generate_stream( // Skip if we already sent an error if !end_reached && !error { let err = InferError::IncompleteGeneration; - tracing::error!("{}", err.to_string()); + tracing::error!("{err}"); yield Ok(Event::from(err)) } }; @@ -355,7 +350,8 @@ pub async fn run( .route("/generate_stream", post(generate_stream)) .route("/", get(health)) .route("/health", get(health)) - .layer(Extension(infer)); + .layer(Extension(infer)) + .layer(opentelemetry_tracing_layer()); // Run server axum::Server::bind(&addr) @@ -391,6 +387,7 @@ async fn shutdown_signal() { } tracing::info!("signal received, starting graceful shutdown"); + opentelemetry::global::shutdown_tracer_provider(); } impl From for FinishReason { diff --git a/router/src/validation.rs b/router/src/validation.rs index 3cca48e9..aa1c1d23 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -6,6 +6,7 @@ use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParamet use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokio::sync::{mpsc, oneshot}; +use tracing::{instrument, Span}; const MAX_MAX_NEW_TOKENS: u32 = 512; const MAX_STOP_SEQUENCES: usize = 4; @@ -36,6 +37,7 @@ impl Validation { } /// Validate a payload and get the number of tokens in the input + #[instrument(skip_all)] pub(crate) async fn validate( &self, request: GenerateRequest, @@ -44,7 +46,10 @@ impl Validation { let (sender, receiver) = oneshot::channel(); // Send request to the background validation task // Unwrap is safe here - self.sender.send((request, sender)).await.unwrap(); + self.sender + .send((request, sender, Span::current())) + .await + .unwrap(); // Await on response channel // Unwrap is safe here receiver.await.unwrap() @@ -97,10 +102,17 @@ fn validation_worker( let mut rng = rand::thread_rng(); // Loop over requests - while let Some((request, response_tx)) = receiver.blocking_recv() { - response_tx - .send(validate(request, &tokenizer, max_input_length, &mut rng)) - .unwrap_or(()) + while let Some((request, response_tx, parent_span)) = receiver.blocking_recv() { + parent_span.in_scope(|| { + response_tx + .send( + validate(request, &tokenizer, max_input_length, &mut rng).map_err(|err| { + tracing::error!("{err}"); + err + }), + ) + .unwrap_or(()) + }) } } @@ -203,6 +215,7 @@ fn validate( type ValidationRequest = ( GenerateRequest, oneshot::Sender>, + Span, ); #[derive(Debug)] diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 122f54dd..dcdb06e9 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.65.0" +channel = "1.67.0" components = ["rustfmt", "clippy"] \ No newline at end of file diff --git a/server/Makefile b/server/Makefile index 6961178b..74ce5144 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,6 +1,6 @@ gen-server: # Compile protos - pip install grpcio-tools==1.49.1 --no-cache-dir + pip install grpcio-tools==1.51.1 --no-cache-dir mkdir text_generation/pb || true python -m grpc_tools.protoc -I../proto --python_out=text_generation/pb --grpc_python_out=text_generation/pb ../proto/generate.proto find text_generation/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; diff --git a/server/poetry.lock b/server/poetry.lock index b236e0a7..0e0655cb 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -39,6 +39,14 @@ tests = ["attrs[tests-no-zope]", "zope.interface"] tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] tests_no_zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +category = "main" +optional = false +python-versions = ">=3.7,<4.0" + [[package]] name = "bitsandbytes" version = "0.35.4" @@ -47,6 +55,22 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "certifi" +version = "2022.12.7" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "charset-normalizer" +version = "3.0.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "click" version = "8.1.3" @@ -66,6 +90,20 @@ category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +[[package]] +name = "Deprecated" +version = "1.2.13" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version (<1)", "configparser (<5)", "importlib-metadata (<3)", "importlib-resources (<4)", "sphinx (<2)", "sphinxcontrib-websupport (<2)", "tox", "zipp (<2)"] + [[package]] name = "exceptiongroup" version = "1.1.0" @@ -154,6 +192,14 @@ grpcio = ">=1.51.1" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + [[package]] name = "iniconfig" version = "2.0.0" @@ -179,7 +225,7 @@ dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils [[package]] name = "numpy" -version = "1.24.1" +version = "1.24.2" description = "Fundamental package for array computing in Python" category = "main" optional = false @@ -233,6 +279,133 @@ python-versions = ">=3" setuptools = "*" wheel = "*" +[[package]] +name = "opentelemetry-api" +version = "1.15.0" +description = "OpenTelemetry Python API" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +deprecated = ">=1.2.6" +setuptools = ">=16.0" + +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.15.0" +description = "OpenTelemetry Collector Exporters" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +opentelemetry-exporter-otlp-proto-grpc = "1.15.0" +opentelemetry-exporter-otlp-proto-http = "1.15.0" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.15.0" +description = "OpenTelemetry Collector Protobuf over gRPC Exporter" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} +googleapis-common-protos = ">=1.52,<2.0" +grpcio = ">=1.0.0,<2.0.0" +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-proto = "1.15.0" +opentelemetry-sdk = ">=1.12,<2.0" + +[package.extras] +test = ["pytest-grpc"] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.15.0" +description = "OpenTelemetry Collector Protobuf over HTTP Exporter" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} +googleapis-common-protos = ">=1.52,<2.0" +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-proto = "1.15.0" +opentelemetry-sdk = ">=1.12,<2.0" +requests = ">=2.7,<3.0" + +[package.extras] +test = ["responses (==0.22.0)"] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.36b0" +description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +opentelemetry-api = ">=1.4,<2.0" +setuptools = ">=16.0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-instrumentation-grpc" +version = "0.36b0" +description = "OpenTelemetry gRPC instrumentation" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.36b0" +opentelemetry-sdk = ">=1.12,<2.0" +opentelemetry-semantic-conventions = "0.36b0" +wrapt = ">=1.0.0,<2.0.0" + +[package.extras] +instruments = ["grpcio (>=1.27,<2.0)"] +test = ["opentelemetry-instrumentation-grpc[instruments]", "opentelemetry-sdk (>=1.12,<2.0)", "opentelemetry-test-utils (==0.36b0)", "protobuf (>=3.13,<4.0)"] + +[[package]] +name = "opentelemetry-proto" +version = "1.15.0" +description = "OpenTelemetry Python Proto" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +protobuf = ">=3.19,<5.0" + +[[package]] +name = "opentelemetry-sdk" +version = "1.15.0" +description = "OpenTelemetry Python SDK" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +opentelemetry-api = "1.15.0" +opentelemetry-semantic-conventions = "0.36b0" +setuptools = ">=16.0" +typing-extensions = ">=3.7.4" + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.36b0" +description = "OpenTelemetry Semantic Conventions" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "packaging" version = "23.0" @@ -300,6 +473,24 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + [[package]] name = "safetensors" version = "0.2.8" @@ -320,7 +511,7 @@ torch = ["torch"] [[package]] name = "setuptools" -version = "67.0.0" +version = "67.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false @@ -382,6 +573,19 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "urllib3" +version = "1.26.14" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "wheel" version = "0.38.4" @@ -404,13 +608,21 @@ python-versions = ">=3.5" [package.extras] dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + [extras] bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "c920067f39ade631d1022c0560c3a4336c82d8c28355509bdfff751bbcfc01cd" +content-hash = "f3cab6881b52045770a90ec9be7415a0ee499d9e980892d544f68073700cf321" [metadata.files] accelerate = [ @@ -421,10 +633,108 @@ attrs = [ {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, ] +backoff = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] bitsandbytes = [ {file = "bitsandbytes-0.35.4-py3-none-any.whl", hash = "sha256:201f168538ccfbd7594568a2f86c149cec8352782301076a15a783695ecec7fb"}, {file = "bitsandbytes-0.35.4.tar.gz", hash = "sha256:b23db6b91cd73cb14faf9841a66bffa5c1722f9b8b57039ef2fb461ac22dd2a6"}, ] +certifi = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] +charset-normalizer = [ + {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, + {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, +] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, @@ -433,6 +743,10 @@ colorama = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +Deprecated = [ + {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, + {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, +] exceptiongroup = [ {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, @@ -547,6 +861,10 @@ grpcio-tools = [ {file = "grpcio_tools-1.51.1-cp39-cp39-win32.whl", hash = "sha256:40ef70e8c5d0310dedff9af502b520b4c7e215bce94094527fb959150a0c594a"}, {file = "grpcio_tools-1.51.1-cp39-cp39-win_amd64.whl", hash = "sha256:15b8acf4eaa0ebe37e2f69108de49efd935b7abe9c7e58ba737490b99906aa76"}, ] +idna = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] iniconfig = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -556,34 +874,34 @@ loguru = [ {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, ] numpy = [ - {file = "numpy-1.24.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7"}, - {file = "numpy-1.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398"}, - {file = "numpy-1.24.1-cp310-cp310-win32.whl", hash = "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2"}, - {file = "numpy-1.24.1-cp310-cp310-win_amd64.whl", hash = "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9"}, - {file = "numpy-1.24.1-cp311-cp311-win32.whl", hash = "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36"}, - {file = "numpy-1.24.1-cp311-cp311-win_amd64.whl", hash = "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7"}, - {file = "numpy-1.24.1-cp38-cp38-win32.whl", hash = "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1"}, - {file = "numpy-1.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf"}, - {file = "numpy-1.24.1-cp39-cp39-win32.whl", hash = "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f"}, - {file = "numpy-1.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566"}, - {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, + {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, + {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, + {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, + {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, + {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, + {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, + {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, + {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, + {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, + {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, + {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, ] nvidia-cublas-cu11 = [ {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, @@ -602,6 +920,42 @@ nvidia-cudnn-cu11 = [ {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, ] +opentelemetry-api = [ + {file = "opentelemetry_api-1.15.0-py3-none-any.whl", hash = "sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0"}, + {file = "opentelemetry_api-1.15.0.tar.gz", hash = "sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded"}, +] +opentelemetry-exporter-otlp = [ + {file = "opentelemetry_exporter_otlp-1.15.0-py3-none-any.whl", hash = "sha256:79f22748b6a54808a0448093dfa189c8490e729f67c134d4c992533d9393b33e"}, + {file = "opentelemetry_exporter_otlp-1.15.0.tar.gz", hash = "sha256:4f7c49751d9720e2e726e13b0bb958ccade4e29122c305d92c033da432c8d2c5"}, +] +opentelemetry-exporter-otlp-proto-grpc = [ + {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0-py3-none-any.whl", hash = "sha256:c2a5492ba7d140109968135d641d06ce3c5bd73c50665f787526065d57d7fd1d"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0.tar.gz", hash = "sha256:844f2a4bb9bcda34e4eb6fe36765e5031aacb36dc60ed88c90fc246942ea26e7"}, +] +opentelemetry-exporter-otlp-proto-http = [ + {file = "opentelemetry_exporter_otlp_proto_http-1.15.0-py3-none-any.whl", hash = "sha256:3ec2a02196c8a54bf5cbf7fe623a5238625638e83b6047a983bdf96e2bbb74c0"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.15.0.tar.gz", hash = "sha256:11b2c814249a49b22f6cca7a06b05701f561d577b747f3660dfd67b6eb9daf9c"}, +] +opentelemetry-instrumentation = [ + {file = "opentelemetry_instrumentation-0.36b0-py3-none-any.whl", hash = "sha256:83ba4ae7d5292b5b33e0f851cc5c76d8f91196b9b3527800fc13855c33383ac2"}, + {file = "opentelemetry_instrumentation-0.36b0.tar.gz", hash = "sha256:e3ddac9b3b93408ef26c8ecbf38f717042977e16381bb4cd329a5b4cf16998cf"}, +] +opentelemetry-instrumentation-grpc = [ + {file = "opentelemetry_instrumentation_grpc-0.36b0-py3-none-any.whl", hash = "sha256:eaa246ed2083c97b13bab2555cb9d170e8433230a31476c4cab8a17fa03380a4"}, + {file = "opentelemetry_instrumentation_grpc-0.36b0.tar.gz", hash = "sha256:dc89447c9eb6ea868970f6c13b4ffdac182cdd5a41dd215a0f5393ca6375be55"}, +] +opentelemetry-proto = [ + {file = "opentelemetry_proto-1.15.0-py3-none-any.whl", hash = "sha256:044b6d044b4d10530f250856f933442b8753a17f94ae37c207607f733fb9a844"}, + {file = "opentelemetry_proto-1.15.0.tar.gz", hash = "sha256:9c4008e40ac8cab359daac283fbe7002c5c29c77ea2674ad5626a249e64e0101"}, +] +opentelemetry-sdk = [ + {file = "opentelemetry_sdk-1.15.0-py3-none-any.whl", hash = "sha256:555c533e9837766119bbccc7a80458c9971d853a6f1da683a2246cd5e53b4645"}, + {file = "opentelemetry_sdk-1.15.0.tar.gz", hash = "sha256:98dbffcfeebcbff12c0c974292d6ea603180a145904cf838b1fe4d5c99078425"}, +] +opentelemetry-semantic-conventions = [ + {file = "opentelemetry_semantic_conventions-0.36b0-py3-none-any.whl", hash = "sha256:adc05635e87b9d3e007c9f530eed487fc3ef2177d02f82f674f28ebf9aff8243"}, + {file = "opentelemetry_semantic_conventions-0.36b0.tar.gz", hash = "sha256:829dc221795467d98b773c04096e29be038d77526dc8d6ac76f546fb6279bf01"}, +] packaging = [ {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, @@ -688,6 +1042,10 @@ PyYAML = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +requests = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, +] safetensors = [ {file = "safetensors-0.2.8-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:8df8af89a0b6b535b47c077e33e5cd4941ef4b067e7c1dd1a05647dec0cf2eea"}, {file = "safetensors-0.2.8-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:9cabae651542b22d229b6439029fb4a3abc7868cb123af336b2877541ad9ab39"}, @@ -731,8 +1089,8 @@ safetensors = [ {file = "safetensors-0.2.8.tar.gz", hash = "sha256:2720b20a6a38c799dca79bd76caeeac2f7df585a9d4f7d59fa7e28eff9ccb27f"}, ] setuptools = [ - {file = "setuptools-67.0.0-py3-none-any.whl", hash = "sha256:9d790961ba6219e9ff7d9557622d2fe136816a264dd01d5997cfc057d804853d"}, - {file = "setuptools-67.0.0.tar.gz", hash = "sha256:883131c5b6efa70b9101c7ef30b2b7b780a4283d5fc1616383cdf22c83cbefe6"}, + {file = "setuptools-67.2.0-py3-none-any.whl", hash = "sha256:16ccf598aab3b506593c17378473978908a2734d7336755a8769b480906bec1c"}, + {file = "setuptools-67.2.0.tar.gz", hash = "sha256:b440ee5f7e607bb8c9de15259dba2583dd41a38879a7abc1d43a71c59524da48"}, ] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, @@ -769,6 +1127,10 @@ typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] +urllib3 = [ + {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, + {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, +] wheel = [ {file = "wheel-0.38.4-py3-none-any.whl", hash = "sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8"}, {file = "wheel-0.38.4.tar.gz", hash = "sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac"}, @@ -777,3 +1139,69 @@ win32-setctime = [ {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, ] +wrapt = [ + {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1"}, + {file = "wrapt-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320"}, + {file = "wrapt-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2"}, + {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4"}, + {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069"}, + {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310"}, + {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f"}, + {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656"}, + {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, + {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, + {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d"}, + {file = "wrapt-1.14.1-cp35-cp35m-win32.whl", hash = "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7"}, + {file = "wrapt-1.14.1-cp35-cp35m-win_amd64.whl", hash = "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00"}, + {file = "wrapt-1.14.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4"}, + {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1"}, + {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1"}, + {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff"}, + {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d"}, + {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1"}, + {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569"}, + {file = "wrapt-1.14.1-cp36-cp36m-win32.whl", hash = "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed"}, + {file = "wrapt-1.14.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471"}, + {file = "wrapt-1.14.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248"}, + {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68"}, + {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d"}, + {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77"}, + {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7"}, + {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015"}, + {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a"}, + {file = "wrapt-1.14.1-cp37-cp37m-win32.whl", hash = "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853"}, + {file = "wrapt-1.14.1-cp37-cp37m-win_amd64.whl", hash = "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c"}, + {file = "wrapt-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456"}, + {file = "wrapt-1.14.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f"}, + {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc"}, + {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1"}, + {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af"}, + {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b"}, + {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0"}, + {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57"}, + {file = "wrapt-1.14.1-cp38-cp38-win32.whl", hash = "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5"}, + {file = "wrapt-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d"}, + {file = "wrapt-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383"}, + {file = "wrapt-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7"}, + {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86"}, + {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735"}, + {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b"}, + {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3"}, + {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3"}, + {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe"}, + {file = "wrapt-1.14.1-cp39-cp39-win32.whl", hash = "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5"}, + {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, + {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, +] diff --git a/server/pyproject.toml b/server/pyproject.toml index bb7fc2f4..d3a8c112 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -19,12 +19,15 @@ accelerate = "^0.15.0" bitsandbytes = "^0.35.1" safetensors = "^0.2.4" loguru = "^0.6.0" +opentelemetry-api = "^1.15.0" +opentelemetry-exporter-otlp = "^1.15.0" +opentelemetry-instrumentation-grpc = "^0.36b0" [tool.poetry.extras] bnb = ["bitsandbytes"] [tool.poetry.group.dev.dependencies] -grpcio-tools = "^1.49.1" +grpcio-tools = "^1.51.1" pytest = "^7.2.0" [build-system] diff --git a/server/text_generation/cli.py b/server/text_generation/cli.py index 1418a803..e9c8ea92 100644 --- a/server/text_generation/cli.py +++ b/server/text_generation/cli.py @@ -7,6 +7,7 @@ from typing import Optional from text_generation import server, utils +from text_generation.tracing import setup_tracing app = typer.Typer() @@ -20,18 +21,8 @@ def serve( uds_path: Path = "/tmp/text-generation", logger_level: str = "INFO", json_output: bool = False, + otlp_endpoint: Optional[str] = None, ): - # Remove default handler - logger.remove() - logger.add( - sys.stdout, - format="{message}", - filter="text_generation", - level=logger_level, - serialize=json_output, - backtrace=True, - diagnose=False, - ) if sharded: assert ( os.getenv("RANK", None) is not None @@ -46,6 +37,21 @@ def serve( os.getenv("MASTER_PORT", None) is not None ), "MASTER_PORT must be set when sharded is True" + # Remove default handler + logger.remove() + logger.add( + sys.stdout, + format="{message}", + filter="text_generation", + level=logger_level, + serialize=json_output, + backtrace=True, + diagnose=False, + ) + # Setup OpenTelemetry distributed tracing + if otlp_endpoint is not None: + setup_tracing(shard=os.getenv("RANK", 0), otlp_endpoint=otlp_endpoint) + server.serve(model_id, revision, sharded, quantize, uds_path) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 0cbed22e..1ac073b3 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -1,6 +1,7 @@ import torch from dataclasses import dataclass +from opentelemetry import trace from transformers import AutoTokenizer, AutoModelForCausalLM, PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type @@ -9,6 +10,8 @@ from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling +tracer = trace.get_tracer(__name__) + @dataclass class CausalLMBatch(Batch): @@ -94,6 +97,7 @@ def from_pb( ) @classmethod + @tracer.start_as_current_span("concatenate") def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # Used for padding total_batch_size = sum(batch.size for batch in batches) @@ -286,6 +290,7 @@ def forward( ) return outputs.logits, outputs.past_key_values + @tracer.start_as_current_span("generate_token") def generate_token( self, batch: CausalLMBatch ) -> Tuple[List[Generation], Optional[CausalLMBatch]]: @@ -331,8 +336,9 @@ def generate_token( all_input_ids, ) in enumerate(iterator): # Select next token - tokens, logprobs = next_token_chooser(all_input_ids.view(1, -1), logits) - next_token_id = tokens[-1].view(1, 1) + next_token_id, logprobs = next_token_chooser( + all_input_ids.view(1, -1), logits + ) # Append next token to all tokens all_input_ids = torch.cat([all_input_ids, next_token_id]) diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 80aecbac..2f28c4ce 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -1,6 +1,7 @@ import torch from dataclasses import dataclass +from opentelemetry import trace from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type @@ -9,6 +10,8 @@ from text_generation.pb import generate_pb2 from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling +tracer = trace.get_tracer(__name__) + @dataclass class Seq2SeqLMBatch(Batch): @@ -107,6 +110,7 @@ def from_pb( ) @classmethod + @tracer.start_as_current_span("concatenate") def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": """Concatenate multiple batches together by padding internal torch tensors""" @@ -361,6 +365,7 @@ def forward( outputs.past_key_values, ) + @tracer.start_as_current_span("generate_token") def generate_token( self, batch: Seq2SeqLMBatch ) -> Tuple[List[Generation], Optional[Seq2SeqLMBatch]]: @@ -418,7 +423,7 @@ def generate_token( ) # Append next token to decoder tokens - decoder_input_ids = torch.cat([decoder_input_ids, next_token_id]) + decoder_input_ids = torch.cat([decoder_input_ids, next_token_id.squeeze(1)]) new_decoder_input_length = decoder_input_length + 1 # Generated token diff --git a/server/text_generation/server.py b/server/text_generation/server.py index 68dc7cd0..f3129cb4 100644 --- a/server/text_generation/server.py +++ b/server/text_generation/server.py @@ -13,6 +13,7 @@ from text_generation.interceptor import ExceptionInterceptor from text_generation.models import Model, get_model from text_generation.pb import generate_pb2_grpc, generate_pb2 +from text_generation.tracing import UDSOpenTelemetryAioServerInterceptor class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): @@ -100,7 +101,12 @@ async def serve_inner( logger.exception("Error when initializing model") raise - server = aio.server(interceptors=[ExceptionInterceptor()]) + server = aio.server( + interceptors=[ + ExceptionInterceptor(), + UDSOpenTelemetryAioServerInterceptor(), + ] + ) generate_pb2_grpc.add_TextGenerationServiceServicer_to_server( TextGenerationService(model, Cache(), server_urls), server ) diff --git a/server/text_generation/tracing.py b/server/text_generation/tracing.py new file mode 100644 index 00000000..bf03c379 --- /dev/null +++ b/server/text_generation/tracing.py @@ -0,0 +1,65 @@ +import grpc + +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.grpc._aio_server import ( + OpenTelemetryAioServerInterceptor, +) +from opentelemetry.semconv.trace import SpanAttributes +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor, +) + + +class UDSOpenTelemetryAioServerInterceptor(OpenTelemetryAioServerInterceptor): + def __init__(self): + super().__init__(trace.get_tracer(__name__)) + + def _start_span(self, handler_call_details, context, set_status_on_exception=False): + """ + Rewrite _start_span method to support Unix Domain Socket gRPC contexts + """ + + # standard attributes + attributes = { + SpanAttributes.RPC_SYSTEM: "grpc", + SpanAttributes.RPC_GRPC_STATUS_CODE: grpc.StatusCode.OK.value[0], + } + + # if we have details about the call, split into service and method + if handler_call_details.method: + service, method = handler_call_details.method.lstrip("/").split("/", 1) + attributes.update( + { + SpanAttributes.RPC_METHOD: method, + SpanAttributes.RPC_SERVICE: service, + } + ) + + # add some attributes from the metadata + metadata = dict(context.invocation_metadata()) + if "user-agent" in metadata: + attributes["rpc.user_agent"] = metadata["user-agent"] + + # We use gRPC over a UNIX socket + attributes.update({SpanAttributes.NET_TRANSPORT: "unix"}) + + return self._tracer.start_as_current_span( + name=handler_call_details.method, + kind=trace.SpanKind.SERVER, + attributes=attributes, + set_status_on_exception=set_status_on_exception, + ) + + +def setup_tracing(shard: int, otlp_endpoint: str): + resource = Resource.create( + attributes={"service.name": f"text-generation-inference.server-{shard}"} + ) + span_exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True) + span_processor = BatchSpanProcessor(span_exporter) + + trace.set_tracer_provider(TracerProvider(resource=resource)) + trace.get_tracer_provider().add_span_processor(span_processor) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py index 5f0a2119..3b3f08c7 100644 --- a/server/text_generation/utils.py +++ b/server/text_generation/utils.py @@ -36,16 +36,14 @@ def __init__(self, seed: int, device: str = "cpu"): self.seed = seed def __call__(self, logits): - probs = torch.nn.functional.softmax(logits, dim=-1) - next_tokens = torch.multinomial( - probs, num_samples=1, generator=self.generator - ).squeeze(1) + probs = torch.nn.functional.softmax(logits) + next_tokens = torch.multinomial(probs, num_samples=1, generator=self.generator) return next_tokens class Greedy: def __call__(self, logits): - return logits.argmax(dim=-1) + return logits.argmax() class NextTokenChooser: @@ -87,8 +85,9 @@ def __call__(self, input_ids, scores): logprobs = torch.log_softmax(scores, -1) # Choose tokens - next_ids = self.choice(scores) - return next_ids, logprobs + next_id = self.choice(scores[-1]) + + return next_id.view(1, 1), logprobs @classmethod def from_pb( @@ -163,6 +162,7 @@ def initialize_torch_distributed(): if torch.cuda.is_available(): from torch.distributed import ProcessGroupNCCL + # Set the device id. assert world_size <= torch.cuda.device_count(), "Each process is one gpu" device = rank % torch.cuda.device_count() @@ -181,7 +181,7 @@ def initialize_torch_distributed(): world_size=world_size, rank=rank, timeout=timedelta(seconds=60), - pg_options=options + pg_options=options, ) return torch.distributed.group.WORLD, rank, world_size From 0fbc69194694b60badae3bf643bc76985f69c0f4 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 14 Feb 2023 13:02:16 +0100 Subject: [PATCH 079/793] feat: add safetensors conversion (#63) --- README.md | 8 +- launcher/src/main.rs | 150 +++++++++- server/tests/utils/test_convert.py | 17 ++ server/tests/utils/test_hub.py | 40 +++ .../{test_utils.py => utils/test_tokens.py} | 38 +-- server/text_generation/cli.py | 49 ++- server/text_generation/models/__init__.py | 36 ++- server/text_generation/models/bloom.py | 7 - server/text_generation/models/galactica.py | 7 - server/text_generation/models/gpt_neox.py | 7 - server/text_generation/models/santacoder.py | 2 +- server/text_generation/models/t5.py | 7 - server/text_generation/utils.py | 283 ------------------ server/text_generation/utils/__init__.py | 36 +++ server/text_generation/utils/convert.py | 96 ++++++ server/text_generation/utils/dist.py | 35 +++ server/text_generation/utils/hub.py | 169 +++++++++++ server/text_generation/utils/tokens.py | 142 +++++++++ 18 files changed, 749 insertions(+), 380 deletions(-) create mode 100644 server/tests/utils/test_convert.py create mode 100644 server/tests/utils/test_hub.py rename server/tests/{test_utils.py => utils/test_tokens.py} (52%) delete mode 100644 server/text_generation/utils.py create mode 100644 server/text_generation/utils/__init__.py create mode 100644 server/text_generation/utils/convert.py create mode 100644 server/text_generation/utils/dist.py create mode 100644 server/text_generation/utils/hub.py create mode 100644 server/text_generation/utils/tokens.py diff --git a/README.md b/README.md index 39cec857..5234844b 100644 --- a/README.md +++ b/README.md @@ -49,17 +49,17 @@ to power LLMs api-inference widgets. - Log probabilities - Distributed tracing with Open Telemetry -## Officially supported models +## Officially supported architectures - [BLOOM](https://huggingface.co/bigscience/bloom) - [BLOOMZ](https://huggingface.co/bigscience/bloomz) - [MT0-XXL](https://huggingface.co/bigscience/mt0-xxl) -- ~~[Galactica](https://huggingface.co/facebook/galactica-120b)~~ (deactivated) +- [Galactica](https://huggingface.co/facebook/galactica-120b) - [SantaCoder](https://huggingface.co/bigcode/santacoder) - [GPT-Neox 20B](https://huggingface.co/EleutherAI/gpt-neox-20b) - [FLAN-T5-XXL](https://huggingface.co/google/flan-t5-xxl) -Other models are supported on a best effort basis using: +Other architectures are supported on a best effort basis using: `AutoModelForCausalLM.from_pretrained(, device_map="auto")` @@ -191,7 +191,7 @@ Be aware that the official Docker image has them enabled by default. ### Download -First you need to download the weights: +It is advised to download the weights ahead of time with the following command: ```shell make download-bloom diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 3c8d9fcc..0848dd9a 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -12,7 +12,7 @@ use std::thread; use std::thread::sleep; use std::time::{Duration, Instant}; use std::{fs, io}; -use subprocess::{Popen, PopenConfig, PopenError, Redirection}; +use subprocess::{ExitStatus, Popen, PopenConfig, PopenError, Redirection}; /// App Configuration #[derive(Parser, Debug)] @@ -43,6 +43,10 @@ struct Args { #[clap(default_value = "29500", long, env)] master_port: usize, #[clap(long, env)] + huggingface_hub_cache: Option, + #[clap(long, env)] + weights_cache_override: Option, + #[clap(long, env)] json_output: bool, #[clap(long, env)] otlp_endpoint: Option, @@ -63,6 +67,8 @@ fn main() -> ExitCode { shard_uds_path, master_addr, master_port, + huggingface_hub_cache, + weights_cache_override, json_output, otlp_endpoint, } = Args::parse(); @@ -84,6 +90,124 @@ fn main() -> ExitCode { }) .expect("Error setting Ctrl-C handler"); + // Download weights + if weights_cache_override.is_none() { + let mut download_argv = vec![ + "text-generation-server".to_string(), + "download-weights".to_string(), + model_id.clone(), + "--logger-level".to_string(), + "INFO".to_string(), + "--json-output".to_string(), + ]; + if num_shard == 1 { + download_argv.push("--extension".to_string()); + download_argv.push(".bin".to_string()); + } else { + download_argv.push("--extension".to_string()); + download_argv.push(".safetensors".to_string()); + } + + // Model optional revision + if let Some(ref revision) = revision { + download_argv.push("--revision".to_string()); + download_argv.push(revision.to_string()) + } + + let mut env = Vec::new(); + + // If the HUGGINGFACE_HUB_CACHE env var is set, pass it to the shard + // Useful when running inside a docker container + if let Some(ref huggingface_hub_cache) = huggingface_hub_cache { + env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); + }; + + // Start process + tracing::info!("Starting download"); + let mut download_process = match Popen::create( + &download_argv, + PopenConfig { + stdout: Redirection::Pipe, + stderr: Redirection::Pipe, + // Needed for the shutdown procedure + setpgid: true, + env: Some(env), + ..Default::default() + }, + ) { + Ok(p) => p, + Err(err) => { + if let PopenError::IoError(ref err) = err { + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("text-generation-server not found in PATH"); + tracing::error!("Please install it with `make install-server`") + } + } + return ExitCode::FAILURE; + } + }; + + // Redirect STDOUT to the console + let download_stdout = download_process.stdout.take().unwrap(); + thread::spawn(move || { + // Enter download tracing span + let stdout = BufReader::new(download_stdout); + let _span = tracing::span!(tracing::Level::INFO, "download").entered(); + for line in stdout.lines() { + // Parse loguru logs + if let Ok(value) = serde_json::from_str::(&line.unwrap()) { + if let Some(text) = value.get("text") { + // Format escaped newlines + tracing::info!("{}", text.to_string().replace("\\n", "")); + } + } + } + }); + + loop { + if let Some(status) = download_process.poll() { + match status { + ExitStatus::Exited(exit_code) => { + if exit_code == 0 { + tracing::info!("Successfully downloaded weights."); + break; + } else { + let mut err = String::new(); + download_process + .stderr + .take() + .unwrap() + .read_to_string(&mut err) + .unwrap(); + tracing::error!("Download encountered an error: {err}"); + return ExitCode::FAILURE; + } + } + _ => { + tracing::error!("Download process exited with an unkown status."); + return ExitCode::FAILURE; + } + } + } + if !running.load(Ordering::SeqCst) { + download_process.terminate().unwrap(); + tracing::info!("Waiting for download process to gracefully shutdown"); + download_process + .wait_timeout(Duration::from_secs(90)) + .unwrap(); + tracing::info!("Download process terminated"); + return ExitCode::SUCCESS; + } + sleep(Duration::from_millis(100)); + } + } else { + tracing::info!( + "weights_cache_override is set to {:?}.", + weights_cache_override + ); + tracing::info!("Skipping download.") + } + // Shared shutdown bool let shutdown = Arc::new(Mutex::new(false)); // Shared shutdown channel @@ -99,6 +223,8 @@ fn main() -> ExitCode { let revision = revision.clone(); let uds_path = shard_uds_path.clone(); let master_addr = master_addr.clone(); + let huggingface_hub_cache = huggingface_hub_cache.clone(); + let weights_cache_override = weights_cache_override.clone(); let status_sender = status_sender.clone(); let shutdown = shutdown.clone(); let shutdown_sender = shutdown_sender.clone(); @@ -113,6 +239,8 @@ fn main() -> ExitCode { num_shard, master_addr, master_port, + huggingface_hub_cache, + weights_cache_override, otlp_endpoint, status_sender, shutdown, @@ -232,7 +360,7 @@ fn main() -> ExitCode { while running.load(Ordering::SeqCst) { if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { - tracing::error!("Shard {} failed:\n{}", rank, err); + tracing::error!("Shard {rank} failed:\n{err}"); exit_code = ExitCode::FAILURE; break; }; @@ -275,6 +403,8 @@ fn shard_manager( world_size: usize, master_addr: String, master_port: usize, + huggingface_hub_cache: Option, + weights_cache_override: Option, otlp_endpoint: Option, status_sender: mpsc::Sender, shutdown: Arc>, @@ -328,15 +458,15 @@ fn shard_manager( ("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into()), ]; - // If the HUGGINGFACE_HUB_CACHE env var is set, pass it to the shard + // If huggingface_hub_cache is some, pass it to the shard // Useful when running inside a docker container - if let Ok(huggingface_hub_cache) = env::var("HUGGINGFACE_HUB_CACHE") { + if let Some(huggingface_hub_cache) = huggingface_hub_cache { env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); }; - // If the WEIGHTS_CACHE_OVERRIDE env var is set, pass it to the shard + // If weights_cache_override is some, pass it to the shard // Useful when running inside a HuggingFace Inference Endpoint - if let Ok(weights_cache_override) = env::var("WEIGHTS_CACHE_OVERRIDE") { + if let Some(weights_cache_override) = weights_cache_override { env.push(( "WEIGHTS_CACHE_OVERRIDE".into(), weights_cache_override.into(), @@ -355,7 +485,7 @@ fn shard_manager( }; // Start process - tracing::info!("Starting shard {}", rank); + tracing::info!("Starting shard {rank}"); let mut p = match Popen::create( &shard_argv, PopenConfig { @@ -419,17 +549,17 @@ fn shard_manager( if *shutdown.lock().unwrap() { p.terminate().unwrap(); let _ = p.wait_timeout(Duration::from_secs(90)); - tracing::info!("Shard {} terminated", rank); + tracing::info!("Shard {rank} terminated"); return; } // Shard is ready if uds.exists() && !ready { - tracing::info!("Shard {} ready in {:?}", rank, start_time.elapsed()); + tracing::info!("Shard {rank} ready in {:?}", start_time.elapsed()); status_sender.send(ShardStatus::Ready).unwrap(); ready = true; } else if !ready && wait_time.elapsed() > Duration::from_secs(10) { - tracing::info!("Waiting for shard {} to be ready...", rank); + tracing::info!("Waiting for shard {rank} to be ready..."); wait_time = Instant::now(); } sleep(Duration::from_millis(100)); diff --git a/server/tests/utils/test_convert.py b/server/tests/utils/test_convert.py new file mode 100644 index 00000000..5f284be5 --- /dev/null +++ b/server/tests/utils/test_convert.py @@ -0,0 +1,17 @@ +from text_generation.utils.hub import download_weights, weight_hub_files, weight_files + +from text_generation.utils.convert import convert_files + + +def test_convert_files(): + model_id = "bigscience/bloom-560m" + pt_filenames = weight_hub_files(model_id, extension=".bin") + local_pt_files = download_weights(pt_filenames, model_id) + local_st_files = [ + p.parent / f"{p.stem.lstrip('pytorch_')}.safetensors" for p in local_pt_files + ] + convert_files(local_pt_files, local_st_files) + + found_st_files = weight_files(model_id) + + assert all([p in found_st_files for p in local_st_files]) diff --git a/server/tests/utils/test_hub.py b/server/tests/utils/test_hub.py new file mode 100644 index 00000000..b3120160 --- /dev/null +++ b/server/tests/utils/test_hub.py @@ -0,0 +1,40 @@ +import pytest + +from text_generation.utils.hub import ( + weight_hub_files, + download_weights, + weight_files, + EntryNotFoundError, + LocalEntryNotFoundError, + RevisionNotFoundError, +) + + +def test_weight_hub_files(): + filenames = weight_hub_files("bigscience/bloom-560m") + assert filenames == ["model.safetensors"] + + +def test_weight_hub_files_llm(): + filenames = weight_hub_files("bigscience/bloom") + assert filenames == [f"model_{i:05d}-of-00072.safetensors" for i in range(1, 73)] + + +def test_weight_hub_files_empty(): + with pytest.raises(EntryNotFoundError): + weight_hub_files("bigscience/bloom", extension=".errors") + + +def test_download_weights(): + model_id = "bigscience/bloom-560m" + filenames = weight_hub_files(model_id) + files = download_weights(filenames, model_id) + local_files = weight_files("bigscience/bloom-560m") + assert files == local_files + + +def test_weight_files_error(): + with pytest.raises(RevisionNotFoundError): + weight_files("bigscience/bloom-560m", revision="error") + with pytest.raises(LocalEntryNotFoundError): + weight_files("bert-base-uncased") diff --git a/server/tests/test_utils.py b/server/tests/utils/test_tokens.py similarity index 52% rename from server/tests/test_utils.py rename to server/tests/utils/test_tokens.py index ffe9be65..7eca482f 100644 --- a/server/tests/test_utils.py +++ b/server/tests/utils/test_tokens.py @@ -1,14 +1,6 @@ -import pytest - -from huggingface_hub.utils import RevisionNotFoundError - -from text_generation.utils import ( - weight_hub_files, - download_weights, - weight_files, +from text_generation.utils.tokens import ( StopSequenceCriteria, StoppingCriteria, - LocalEntryNotFoundError, FinishReason, ) @@ -41,31 +33,3 @@ def test_stopping_criteria_max(): assert criteria(1, "") == (False, None) assert criteria(1, "") == (False, None) assert criteria(1, "") == (True, FinishReason.FINISH_REASON_LENGTH) - - -def test_weight_hub_files(): - filenames = weight_hub_files("bigscience/bloom-560m") - assert filenames == ["model.safetensors"] - - -def test_weight_hub_files_llm(): - filenames = weight_hub_files("bigscience/bloom") - assert filenames == [f"model_{i:05d}-of-00072.safetensors" for i in range(1, 73)] - - -def test_weight_hub_files_empty(): - filenames = weight_hub_files("bigscience/bloom", extension=".errors") - assert filenames == [] - - -def test_download_weights(): - files = download_weights("bigscience/bloom-560m") - local_files = weight_files("bigscience/bloom-560m") - assert files == local_files - - -def test_weight_files_error(): - with pytest.raises(RevisionNotFoundError): - weight_files("bigscience/bloom-560m", revision="error") - with pytest.raises(LocalEntryNotFoundError): - weight_files("bert-base-uncased") diff --git a/server/text_generation/cli.py b/server/text_generation/cli.py index e9c8ea92..678dce16 100644 --- a/server/text_generation/cli.py +++ b/server/text_generation/cli.py @@ -60,8 +60,55 @@ def download_weights( model_id: str, revision: Optional[str] = None, extension: str = ".safetensors", + logger_level: str = "INFO", + json_output: bool = False, ): - utils.download_weights(model_id, revision, extension) + # Remove default handler + logger.remove() + logger.add( + sys.stdout, + format="{message}", + filter="text_generation", + level=logger_level, + serialize=json_output, + backtrace=True, + diagnose=False, + ) + + # Test if files were already download + try: + utils.weight_files(model_id, revision, extension) + logger.info( + "Files are already present in the local cache. " "Skipping download." + ) + return + # Local files not found + except utils.LocalEntryNotFoundError: + pass + + # Download weights directly + try: + filenames = utils.weight_hub_files(model_id, revision, extension) + utils.download_weights(filenames, model_id, revision) + except utils.EntryNotFoundError as e: + if not extension == ".safetensors": + raise e + + logger.warning( + f"No safetensors weights found for model {model_id} at revision {revision}. " + f"Converting PyTorch weights instead." + ) + + # Try to see if there are pytorch weights + pt_filenames = utils.weight_hub_files(model_id, revision, ".bin") + # Download pytorch weights + local_pt_files = utils.download_weights(pt_filenames, model_id, revision) + local_st_files = [ + p.parent / f"{p.stem.lstrip('pytorch_')}.safetensors" + for p in local_pt_files + ] + # Convert pytorch weights to safetensors + utils.convert_files(local_pt_files, local_st_files) if __name__ == "__main__": diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 7445b427..908b144c 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -41,6 +41,15 @@ def get_model( model_id: str, revision: Optional[str], sharded: bool, quantize: bool ) -> Model: + if model_id.startswith("facebook/galactica"): + if sharded: + return GalacticaSharded(model_id, revision, quantize=quantize) + else: + return Galactica(model_id, revision, quantize=quantize) + + if "santacoder" in model_id: + return SantaCoder(model_id, revision, quantize) + config = AutoConfig.from_pretrained(model_id, revision=revision) if config.model_type == "bloom": @@ -48,27 +57,22 @@ def get_model( return BLOOMSharded(model_id, revision, quantize=quantize) else: return BLOOM(model_id, revision, quantize=quantize) - elif config.model_type == "gpt_neox": + + if config.model_type == "gpt_neox": if sharded: return GPTNeoxSharded(model_id, revision, quantize=quantize) else: return GPTNeox(model_id, revision, quantize=quantize) - elif config.model_type == "t5": + + if config.model_type == "t5": if sharded: return T5Sharded(model_id, revision, quantize=quantize) else: return Seq2SeqLM(model_id, revision, quantize=quantize) - elif model_id.startswith("facebook/galactica"): - if sharded: - return GalacticaSharded(model_id, revision, quantize=quantize) - else: - return Galactica(model_id, revision, quantize=quantize) - elif "santacoder" in model_id: - return SantaCoder(model_id, revision, quantize) - else: - if sharded: - raise ValueError("sharded is not supported for AutoModel") - try: - return CausalLM(model_id, revision, quantize=quantize) - except Exception: - return Seq2SeqLM(model_id, revision, quantize=quantize) + + if sharded: + raise ValueError("sharded is not supported for AutoModel") + try: + return CausalLM(model_id, revision, quantize=quantize) + except Exception: + return Seq2SeqLM(model_id, revision, quantize=quantize) diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 992d7b5b..08c3ac94 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -23,7 +23,6 @@ from text_generation.utils import ( initialize_torch_distributed, weight_files, - download_weights, ) HAS_BITS_AND_BYTES = True @@ -80,14 +79,8 @@ def __init__( ) config.pad_token_id = 3 - # Only download weights for small models - if self.master and model_id == "bigscience/bloom-560m": - download_weights(model_id, revision=revision, extension=".safetensors") - torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") - if not filenames: - raise ValueError("No safetensors weights found") with init_empty_weights(): model = AutoModelForCausalLM.from_config(config) diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index f1dc8a30..780a94f1 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -26,7 +26,6 @@ StoppingCriteria, initialize_torch_distributed, weight_files, - download_weights, ) HAS_BITS_AND_BYTES = True @@ -172,14 +171,8 @@ def __init__( ) tokenizer.pad_token_id = config.pad_token_id - # Only download weights for small models - if self.master and model_id == "facebook/galactica-125m": - download_weights(model_id, revision=revision, extension=".safetensors") - torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") - if not filenames: - raise ValueError("No safetensors weights found") with init_empty_weights(): model = AutoModelForCausalLM.from_config(config) diff --git a/server/text_generation/models/gpt_neox.py b/server/text_generation/models/gpt_neox.py index 2d467f4c..0197f976 100644 --- a/server/text_generation/models/gpt_neox.py +++ b/server/text_generation/models/gpt_neox.py @@ -20,7 +20,6 @@ from text_generation.utils import ( initialize_torch_distributed, weight_files, - download_weights, ) HAS_BITS_AND_BYTES = True @@ -69,14 +68,8 @@ def __init__( model_id, revision=revision, tp_parallel=True ) - # Only master download weights - if self.master: - download_weights(model_id, revision=revision, extension=".safetensors") - torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") - if not filenames: - raise ValueError("No safetensors weights found") with init_empty_weights(): model = AutoModelForCausalLM.from_config(config) diff --git a/server/text_generation/models/santacoder.py b/server/text_generation/models/santacoder.py index fb496197..5d271c85 100644 --- a/server/text_generation/models/santacoder.py +++ b/server/text_generation/models/santacoder.py @@ -1,7 +1,7 @@ import torch import torch.distributed -from typing import Optional, List, Tuple +from typing import Optional, List from transformers import AutoTokenizer, AutoModelForCausalLM from text_generation.models import CausalLM diff --git a/server/text_generation/models/t5.py b/server/text_generation/models/t5.py index d7241c81..536ebda3 100644 --- a/server/text_generation/models/t5.py +++ b/server/text_generation/models/t5.py @@ -20,7 +20,6 @@ from text_generation.utils import ( initialize_torch_distributed, weight_files, - download_weights, ) HAS_BITS_AND_BYTES = True @@ -53,14 +52,8 @@ def __init__( ) tokenizer.bos_token_id = config.decoder_start_token_id - # Only master download weights - if self.master: - download_weights(model_id, revision=revision, extension=".safetensors") - torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") - if not filenames: - raise ValueError("No safetensors weights found") with init_empty_weights(): model = AutoModelForSeq2SeqLM.from_config(config) diff --git a/server/text_generation/utils.py b/server/text_generation/utils.py deleted file mode 100644 index 3b3f08c7..00000000 --- a/server/text_generation/utils.py +++ /dev/null @@ -1,283 +0,0 @@ -import concurrent -import os -import re -import torch -import torch.distributed - -from datetime import timedelta - -from concurrent.futures import ThreadPoolExecutor -from functools import partial -from pathlib import Path -from huggingface_hub import HfApi, hf_hub_download, _CACHED_NO_EXIST -from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE -from huggingface_hub.utils import LocalEntryNotFoundError -from tqdm import tqdm -from typing import List, Optional, Tuple -from transformers import PreTrainedTokenizerBase -from transformers.generation.logits_process import ( - LogitsProcessorList, - RepetitionPenaltyLogitsProcessor, - TemperatureLogitsWarper, - TopPLogitsWarper, - TopKLogitsWarper, -) - -from text_generation.pb import generate_pb2 -from text_generation.pb.generate_pb2 import FinishReason - -WEIGHTS_CACHE_OVERRIDE = os.getenv("WEIGHTS_CACHE_OVERRIDE", None) - - -class Sampling: - def __init__(self, seed: int, device: str = "cpu"): - self.generator = torch.Generator(device) - self.generator.manual_seed(seed) - self.seed = seed - - def __call__(self, logits): - probs = torch.nn.functional.softmax(logits) - next_tokens = torch.multinomial(probs, num_samples=1, generator=self.generator) - return next_tokens - - -class Greedy: - def __call__(self, logits): - return logits.argmax() - - -class NextTokenChooser: - def __init__( - self, - temperature=1.0, - repetition_penalty=1.0, - top_k=None, - top_p=None, - do_sample=False, - seed=0, - device="cpu", - ): - warpers = LogitsProcessorList() - # the following idea is largely copied from this PR: https://github.com/huggingface/transformers/pull/5420/files - # all samplers can be found in `generation_utils_samplers.py` - sampling = do_sample - if temperature is not None and temperature != 1.0: - temperature = float(temperature) - warpers.append(TemperatureLogitsWarper(temperature)) - sampling = True - if top_k is not None and top_k != 0: - warpers.append(TopKLogitsWarper(top_k=top_k)) - sampling = True - if top_p is not None and top_p < 1.0: - warpers.append(TopPLogitsWarper(top_p=top_p)) - sampling = True - if repetition_penalty is not None and repetition_penalty != 1.0: - warpers.append(RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty)) - - self.warpers = warpers - self.choice = Sampling(seed, device) if sampling else Greedy() - - def __call__(self, input_ids, scores): - # Warp logits - scores = self.warpers(input_ids, scores) - - # Compute logprobs - logprobs = torch.log_softmax(scores, -1) - - # Choose tokens - next_id = self.choice(scores[-1]) - - return next_id.view(1, 1), logprobs - - @classmethod - def from_pb( - cls, pb: generate_pb2.NextTokenChooserParameters, device: torch.device - ) -> "NextTokenChooser": - return NextTokenChooser( - temperature=pb.temperature, - repetition_penalty=pb.repetition_penalty, - top_k=pb.top_k, - top_p=pb.top_p, - do_sample=pb.do_sample, - seed=pb.seed, - device=device, - ) - - -class StopSequenceCriteria: - def __init__(self, stop_sequence: str): - self.regex = re.compile(f".*{stop_sequence}$") - - def __call__(self, output: str) -> bool: - if self.regex.findall(output): - return True - return False - - -class StoppingCriteria: - def __init__( - self, - eos_token_id: int, - stop_sequence_criterias: List[StopSequenceCriteria], - max_new_tokens=20, - ): - self.eos_token_id = eos_token_id - self.stop_sequence_criterias = stop_sequence_criterias - self.max_new_tokens = max_new_tokens - self.current_tokens = 0 - self.current_output = "" - - def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[str]]: - self.current_tokens += 1 - if self.current_tokens >= self.max_new_tokens: - return True, FinishReason.FINISH_REASON_LENGTH - - if last_token == self.eos_token_id: - return True, FinishReason.FINISH_REASON_EOS_TOKEN - - self.current_output += last_output - for stop_sequence_criteria in self.stop_sequence_criterias: - if stop_sequence_criteria(self.current_output): - return True, FinishReason.FINISH_REASON_STOP_SEQUENCE - - return False, None - - @classmethod - def from_pb( - cls, - pb: generate_pb2.StoppingCriteriaParameters, - tokenizer: PreTrainedTokenizerBase, - ) -> "StoppingCriteria": - stop_sequence_criterias = [ - StopSequenceCriteria(sequence) for sequence in pb.stop_sequences - ] - return StoppingCriteria( - tokenizer.eos_token_id, stop_sequence_criterias, pb.max_new_tokens - ) - - -def initialize_torch_distributed(): - rank = int(os.getenv("RANK", "0")) - world_size = int(os.getenv("WORLD_SIZE", "1")) - - if torch.cuda.is_available(): - from torch.distributed import ProcessGroupNCCL - - # Set the device id. - assert world_size <= torch.cuda.device_count(), "Each process is one gpu" - device = rank % torch.cuda.device_count() - torch.cuda.set_device(device) - backend = "nccl" - options = ProcessGroupNCCL.Options() - options.is_high_priority_stream = True - options._timeout = timedelta(seconds=60) - else: - backend = "gloo" - options = None - - # Call the init process. - torch.distributed.init_process_group( - backend=backend, - world_size=world_size, - rank=rank, - timeout=timedelta(seconds=60), - pg_options=options, - ) - - return torch.distributed.group.WORLD, rank, world_size - - -def weight_hub_files(model_id, revision=None, extension=".safetensors"): - """Get the safetensors filenames on the hub""" - api = HfApi() - info = api.model_info(model_id, revision=revision) - filenames = [s.rfilename for s in info.siblings if s.rfilename.endswith(extension)] - return filenames - - -def try_to_load_from_cache(model_id, revision, filename): - """Try to load a file from the Hugging Face cache""" - if revision is None: - revision = "main" - - object_id = model_id.replace("/", "--") - repo_cache = Path(HUGGINGFACE_HUB_CACHE) / f"models--{object_id}" - - if not repo_cache.is_dir(): - # No cache for this model - return None - - refs_dir = repo_cache / "refs" - snapshots_dir = repo_cache / "snapshots" - no_exist_dir = repo_cache / ".no_exist" - - # Resolve refs (for instance to convert main to the associated commit sha) - if refs_dir.is_dir(): - revision_file = refs_dir / revision - if revision_file.exists(): - with revision_file.open() as f: - revision = f.read() - - # Check if file is cached as "no_exist" - if (no_exist_dir / revision / filename).is_file(): - return _CACHED_NO_EXIST - - # Check if revision folder exists - if not snapshots_dir.exists(): - return None - cached_shas = os.listdir(snapshots_dir) - if revision not in cached_shas: - # No cache for this revision and we won't try to return a random revision - return None - - # Check if file exists in cache - cached_file = snapshots_dir / revision / filename - return str(cached_file) if cached_file.is_file() else None - - -def weight_files(model_id, revision=None, extension=".safetensors"): - """Get the local safetensors filenames""" - if WEIGHTS_CACHE_OVERRIDE is not None: - return list(Path(WEIGHTS_CACHE_OVERRIDE).glob(f"*{extension}")) - - filenames = weight_hub_files(model_id, revision, extension) - files = [] - for filename in filenames: - cache_file = try_to_load_from_cache( - model_id, revision=revision, filename=filename - ) - if cache_file is None: - raise LocalEntryNotFoundError( - f"File {filename} of model {model_id} not found in " - f"{os.getenv('HUGGINGFACE_HUB_CACHE', 'the local cache')}. " - f"Please run `text-generation-server download-weights {model_id}` first." - ) - files.append(cache_file) - - return files - - -def download_weights(model_id, revision=None, extension=".safetensors"): - """Download the safetensors files from the hub""" - if WEIGHTS_CACHE_OVERRIDE is not None: - return list(Path(WEIGHTS_CACHE_OVERRIDE).glob(f"*{extension}")) - - filenames = weight_hub_files(model_id, revision, extension) - - download_function = partial( - hf_hub_download, - repo_id=model_id, - local_files_only=False, - ) - - executor = ThreadPoolExecutor(max_workers=5) - futures = [ - executor.submit(download_function, filename=filename, revision=revision) - for filename in filenames - ] - files = [ - future.result() - for future in tqdm(concurrent.futures.as_completed(futures), total=len(futures)) - ] - - return files diff --git a/server/text_generation/utils/__init__.py b/server/text_generation/utils/__init__.py new file mode 100644 index 00000000..a390b710 --- /dev/null +++ b/server/text_generation/utils/__init__.py @@ -0,0 +1,36 @@ +from text_generation.utils.convert import convert_file, convert_files +from text_generation.utils.dist import initialize_torch_distributed +from text_generation.utils.hub import ( + weight_files, + weight_hub_files, + download_weights, + EntryNotFoundError, + LocalEntryNotFoundError, + RevisionNotFoundError, +) +from text_generation.utils.tokens import ( + Greedy, + NextTokenChooser, + Sampling, + StoppingCriteria, + StopSequenceCriteria, + FinishReason, +) + +__all__ = [ + "convert_file", + "convert_files", + "initialize_torch_distributed", + "weight_files", + "weight_hub_files", + "download_weights", + "EntryNotFoundError", + "LocalEntryNotFoundError", + "RevisionNotFoundError", + "Greedy", + "NextTokenChooser", + "Sampling", + "StoppingCriteria", + "StopSequenceCriteria", + "FinishReason", +] diff --git a/server/text_generation/utils/convert.py b/server/text_generation/utils/convert.py new file mode 100644 index 00000000..e7f9660c --- /dev/null +++ b/server/text_generation/utils/convert.py @@ -0,0 +1,96 @@ +import concurrent +import time +import torch + +from concurrent.futures import ThreadPoolExecutor +from collections import defaultdict +from datetime import timedelta +from loguru import logger +from pathlib import Path +from safetensors.torch import load_file, save_file +from typing import Dict, List + + +def check_file_size(source_file: Path, target_file: Path): + """ + Check that two files are close in size + """ + source_file_size = source_file.stat().st_size + target_file_size = target_file.stat().st_size + + if (source_file_size - target_file_size) / source_file_size > 0.01: + raise RuntimeError( + f"""The file size different is more than 1%: + - {source_file}: {source_file_size} + - {target_file}: {target_file_size} + """ + ) + + +def remove_shared_pointers(tensors: Dict[str, torch.Tensor]): + """ + For a Dict of tensors, check if two or more tensors point to the same underlying memory and + remove them + """ + ptrs = defaultdict(list) + for k, v in tensors.items(): + ptrs[v.data_ptr()].append(k) + + # Iterate over all found memory addresses + for ptr, names in ptrs.items(): + if len(names) > 1: + # Multiple tensors are point to the same memory + # Only keep the first tensor + for name in names[1:]: + tensors.pop(name) + + +def convert_file(pt_file: Path, st_file: Path): + """ + Convert a pytorch file to a safetensors file + """ + pt_state = torch.load(pt_file, map_location="cpu") + if "state_dict" in pt_state: + pt_state = pt_state["state_dict"] + + remove_shared_pointers(pt_state) + + # Tensors need to be contiguous + pt_state = {k: v.contiguous() for k, v in pt_state.items()} + + st_file.parent.mkdir(parents=True, exist_ok=True) + save_file(pt_state, str(st_file), metadata={"format": "pt"}) + + # Check that both files are close in size + check_file_size(pt_file, st_file) + + # Load safetensors state + st_state = load_file(str(st_file)) + for k in st_state: + pt_tensor = pt_state[k] + st_tensor = st_state[k] + if not torch.equal(pt_tensor, st_tensor): + raise RuntimeError(f"The output tensors do not match for key {k}") + + +def convert_files(pt_files: List[Path], st_files: List[Path]): + assert len(pt_files) == len(st_files) + + executor = ThreadPoolExecutor(max_workers=5) + futures = [ + executor.submit(convert_file, pt_file=pt_file, st_file=st_file) + for pt_file, st_file in zip(pt_files, st_files) + ] + + # We do this instead of using tqdm because we want to parse the logs with the launcher + logger.info("Converting weights...") + start_time = time.time() + for i, future in enumerate(concurrent.futures.as_completed(futures)): + elapsed = timedelta(seconds=int(time.time() - start_time)) + remaining = len(futures) - (i + 1) + if remaining != 0: + eta = (elapsed / (i + 1)) * remaining + else: + eta = 0 + + logger.info(f"Convert: [{i + 1}/{len(futures)}] -- ETA: {eta}") diff --git a/server/text_generation/utils/dist.py b/server/text_generation/utils/dist.py new file mode 100644 index 00000000..9785493e --- /dev/null +++ b/server/text_generation/utils/dist.py @@ -0,0 +1,35 @@ +import os +import torch + +from datetime import timedelta + + +def initialize_torch_distributed(): + rank = int(os.getenv("RANK", "0")) + world_size = int(os.getenv("WORLD_SIZE", "1")) + + if torch.cuda.is_available(): + from torch.distributed import ProcessGroupNCCL + + # Set the device id. + assert world_size <= torch.cuda.device_count(), "Each process is one gpu" + device = rank % torch.cuda.device_count() + torch.cuda.set_device(device) + backend = "nccl" + options = ProcessGroupNCCL.Options() + options.is_high_priority_stream = True + options._timeout = timedelta(seconds=60) + else: + backend = "gloo" + options = None + + # Call the init process. + torch.distributed.init_process_group( + backend=backend, + world_size=world_size, + rank=rank, + timeout=timedelta(seconds=60), + pg_options=options, + ) + + return torch.distributed.group.WORLD, rank, world_size diff --git a/server/text_generation/utils/hub.py b/server/text_generation/utils/hub.py new file mode 100644 index 00000000..60072a20 --- /dev/null +++ b/server/text_generation/utils/hub.py @@ -0,0 +1,169 @@ +import time +import concurrent +import os + +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta +from loguru import logger +from pathlib import Path +from typing import Optional, List + +from huggingface_hub import HfApi, hf_hub_download +from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE +from huggingface_hub.utils import ( + LocalEntryNotFoundError, + EntryNotFoundError, + RevisionNotFoundError, # Import here to ease try/except in other part of the lib +) + +WEIGHTS_CACHE_OVERRIDE = os.getenv("WEIGHTS_CACHE_OVERRIDE", None) + + +def weight_hub_files( + model_id: str, revision: Optional[str] = None, extension: str = ".safetensors" +) -> List[str]: + """Get the weights filenames on the hub""" + api = HfApi() + info = api.model_info(model_id, revision=revision) + filenames = [s.rfilename for s in info.siblings if s.rfilename.endswith(extension)] + + if not filenames: + raise EntryNotFoundError( + f"No {extension} weights found for model {model_id} and revision {revision}.", + None, + ) + + return filenames + + +def try_to_load_from_cache( + model_id: str, revision: Optional[str], filename: str +) -> Optional[Path]: + """Try to load a file from the Hugging Face cache""" + if revision is None: + revision = "main" + + object_id = model_id.replace("/", "--") + repo_cache = Path(HUGGINGFACE_HUB_CACHE) / f"models--{object_id}" + + if not repo_cache.is_dir(): + # No cache for this model + return None + + refs_dir = repo_cache / "refs" + snapshots_dir = repo_cache / "snapshots" + no_exist_dir = repo_cache / ".no_exist" + + # Resolve refs (for instance to convert main to the associated commit sha) + if refs_dir.is_dir(): + revision_file = refs_dir / revision + if revision_file.exists(): + with revision_file.open() as f: + revision = f.read() + + # Check if file is cached as "no_exist" + if (no_exist_dir / revision / filename).is_file(): + return None + + # Check if revision folder exists + if not snapshots_dir.exists(): + return None + cached_shas = os.listdir(snapshots_dir) + if revision not in cached_shas: + # No cache for this revision and we won't try to return a random revision + return None + + # Check if file exists in cache + cached_file = snapshots_dir / revision / filename + return cached_file if cached_file.is_file() else None + + +def weight_files( + model_id: str, revision: Optional[str] = None, extension: str = ".safetensors" +) -> List[Path]: + """Get the local files""" + try: + filenames = weight_hub_files(model_id, revision, extension) + except EntryNotFoundError as e: + if extension != ".safetensors": + raise e + # Try to see if there are pytorch weights + pt_filenames = weight_hub_files(model_id, revision, extension=".bin") + # Change pytorch extension to safetensors extension + # It is possible that we have safetensors weights locally even though they are not on the + # hub if we converted weights locally without pushing them + filenames = [ + f"{Path(f).stem.lstrip('pytorch_')}.safetensors" for f in pt_filenames + ] + + if WEIGHTS_CACHE_OVERRIDE is not None: + files = [] + for filename in filenames: + p = Path(WEIGHTS_CACHE_OVERRIDE) / filename + if not p.exists(): + raise LocalEntryNotFoundError( + f"File {p} not found in {WEIGHTS_CACHE_OVERRIDE}." + ) + files.append(p) + return files + + files = [] + for filename in filenames: + cache_file = try_to_load_from_cache( + model_id, revision=revision, filename=filename + ) + if cache_file is None: + raise LocalEntryNotFoundError( + f"File {filename} of model {model_id} not found in " + f"{os.getenv('HUGGINGFACE_HUB_CACHE', 'the local cache')}. " + f"Please run `text-generation-server download-weights {model_id}` first." + ) + files.append(cache_file) + + return files + + +def download_weights( + filenames: List[str], model_id: str, revision: Optional[str] = None +) -> List[Path]: + """Download the safetensors files from the hub""" + + def download_file(filename): + local_file = try_to_load_from_cache(model_id, revision, filename) + if local_file is not None: + logger.info(f"File {filename} already present in cache.") + return local_file + + start_time = time.time() + local_file = hf_hub_download( + filename=filename, + repo_id=model_id, + revision=revision, + local_files_only=False, + ) + logger.info( + f"Downloaded {filename} at {local_file} in {timedelta(seconds=int(time.time() - start_time))}." + ) + return local_file + + executor = ThreadPoolExecutor(max_workers=5) + futures = [ + executor.submit(download_file, filename=filename) for filename in filenames + ] + + # We do this instead of using tqdm because we want to parse the logs with the launcher + logger.info("Downloading weights...") + start_time = time.time() + files = [] + for i, future in enumerate(concurrent.futures.as_completed(futures)): + elapsed = timedelta(seconds=int(time.time() - start_time)) + remaining = len(futures) - (i + 1) + if remaining != 0: + eta = (elapsed / (i + 1)) * remaining + else: + eta = 0 + + logger.info(f"Download: [{i + 1}/{len(futures)}] -- ETA: {eta}") + files.append(Path(future.result())) + + return [Path(p) for p in files] diff --git a/server/text_generation/utils/tokens.py b/server/text_generation/utils/tokens.py new file mode 100644 index 00000000..cc0e6c35 --- /dev/null +++ b/server/text_generation/utils/tokens.py @@ -0,0 +1,142 @@ +import re +import torch + +from transformers import ( + LogitsProcessorList, + TemperatureLogitsWarper, + TopKLogitsWarper, + TopPLogitsWarper, + RepetitionPenaltyLogitsProcessor, + PreTrainedTokenizerBase, +) +from typing import List, Tuple, Optional + +from text_generation.pb import generate_pb2 +from text_generation.pb.generate_pb2 import FinishReason + + +class Sampling: + def __init__(self, seed: int, device: str = "cpu"): + self.generator = torch.Generator(device) + self.generator.manual_seed(seed) + self.seed = seed + + def __call__(self, logits): + probs = torch.nn.functional.softmax(logits) + next_tokens = torch.multinomial(probs, num_samples=1, generator=self.generator) + return next_tokens + + +class Greedy: + def __call__(self, logits): + return logits.argmax() + + +class NextTokenChooser: + def __init__( + self, + temperature=1.0, + repetition_penalty=1.0, + top_k=None, + top_p=None, + do_sample=False, + seed=0, + device="cpu", + ): + warpers = LogitsProcessorList() + # the following idea is largely copied from this PR: https://github.com/huggingface/transformers/pull/5420/files + # all samplers can be found in `generation_utils_samplers.py` + sampling = do_sample + if temperature is not None and temperature != 1.0: + temperature = float(temperature) + warpers.append(TemperatureLogitsWarper(temperature)) + sampling = True + if top_k is not None and top_k != 0: + warpers.append(TopKLogitsWarper(top_k=top_k)) + sampling = True + if top_p is not None and top_p < 1.0: + warpers.append(TopPLogitsWarper(top_p=top_p)) + sampling = True + if repetition_penalty is not None and repetition_penalty != 1.0: + warpers.append(RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty)) + + self.warpers = warpers + self.choice = Sampling(seed, device) if sampling else Greedy() + + def __call__(self, input_ids, scores): + # Warp logits + scores = self.warpers(input_ids, scores) + + # Compute logprobs + logprobs = torch.log_softmax(scores, -1) + + # Choose tokens + next_id = self.choice(scores[-1]) + + return next_id.view(1, 1), logprobs + + @classmethod + def from_pb( + cls, pb: generate_pb2.NextTokenChooserParameters, device: torch.device + ) -> "NextTokenChooser": + return NextTokenChooser( + temperature=pb.temperature, + repetition_penalty=pb.repetition_penalty, + top_k=pb.top_k, + top_p=pb.top_p, + do_sample=pb.do_sample, + seed=pb.seed, + device=device, + ) + + +class StopSequenceCriteria: + def __init__(self, stop_sequence: str): + self.regex = re.compile(f".*{stop_sequence}$") + + def __call__(self, output: str) -> bool: + if self.regex.findall(output): + return True + return False + + +class StoppingCriteria: + def __init__( + self, + eos_token_id: int, + stop_sequence_criterias: List[StopSequenceCriteria], + max_new_tokens=20, + ): + self.eos_token_id = eos_token_id + self.stop_sequence_criterias = stop_sequence_criterias + self.max_new_tokens = max_new_tokens + self.current_tokens = 0 + self.current_output = "" + + def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[str]]: + self.current_tokens += 1 + if self.current_tokens >= self.max_new_tokens: + return True, FinishReason.FINISH_REASON_LENGTH + + if last_token == self.eos_token_id: + return True, FinishReason.FINISH_REASON_EOS_TOKEN + + self.current_output += last_output + for stop_sequence_criteria in self.stop_sequence_criterias: + if stop_sequence_criteria(self.current_output): + return True, FinishReason.FINISH_REASON_STOP_SEQUENCE + + return False, None + + @classmethod + def from_pb( + cls, + pb: generate_pb2.StoppingCriteriaParameters, + tokenizer: PreTrainedTokenizerBase, + ) -> "StoppingCriteria": + stop_sequence_criterias = [ + StopSequenceCriteria(sequence) for sequence in pb.stop_sequences + ] + return StoppingCriteria( + tokenizer.eos_token_id, stop_sequence_criterias, pb.max_new_tokens + ) From c5a4a1faf3c430c11f09128a6e1dc40b4736e212 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 15 Feb 2023 16:11:32 +0100 Subject: [PATCH 080/793] feat(server): improve download logging (#66) --- launcher/src/main.rs | 29 +++++++++++++------------ server/text_generation/utils/convert.py | 6 +---- server/text_generation/utils/hub.py | 13 +++++------ 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 0848dd9a..ecb2c0b6 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -22,8 +22,8 @@ struct Args { model_id: String, #[clap(long, env)] revision: Option, - #[clap(long, env)] - num_shard: Option, + #[clap(default_value = "1", long, env)] + num_shard: usize, #[clap(long, env)] quantize: bool, #[clap(default_value = "128", long, env)] @@ -54,6 +54,16 @@ struct Args { fn main() -> ExitCode { // Pattern match configuration + let args = Args::parse(); + + if args.json_output { + tracing_subscriber::fmt().json().init(); + } else { + tracing_subscriber::fmt().compact().init(); + } + + tracing::info!("{:?}", args); + let Args { model_id, revision, @@ -71,16 +81,7 @@ fn main() -> ExitCode { weights_cache_override, json_output, otlp_endpoint, - } = Args::parse(); - - if json_output { - tracing_subscriber::fmt().json().init(); - } else { - tracing_subscriber::fmt().compact().init(); - } - - // By default we only have one master shard - let num_shard = num_shard.unwrap_or(1); + } = args; // Signal handler let running = Arc::new(AtomicBool::new(true)); @@ -123,7 +124,7 @@ fn main() -> ExitCode { }; // Start process - tracing::info!("Starting download"); + tracing::info!("Starting download process."); let mut download_process = match Popen::create( &download_argv, PopenConfig { @@ -184,7 +185,7 @@ fn main() -> ExitCode { } } _ => { - tracing::error!("Download process exited with an unkown status."); + tracing::error!("Download process exited with an unknown status."); return ExitCode::FAILURE; } } diff --git a/server/text_generation/utils/convert.py b/server/text_generation/utils/convert.py index e7f9660c..30144f0c 100644 --- a/server/text_generation/utils/convert.py +++ b/server/text_generation/utils/convert.py @@ -83,14 +83,10 @@ def convert_files(pt_files: List[Path], st_files: List[Path]): ] # We do this instead of using tqdm because we want to parse the logs with the launcher - logger.info("Converting weights...") start_time = time.time() for i, future in enumerate(concurrent.futures.as_completed(futures)): elapsed = timedelta(seconds=int(time.time() - start_time)) remaining = len(futures) - (i + 1) - if remaining != 0: - eta = (elapsed / (i + 1)) * remaining - else: - eta = 0 + eta = (elapsed / (i + 1)) * remaining if remaining > 0 else 0 logger.info(f"Convert: [{i + 1}/{len(futures)}] -- ETA: {eta}") diff --git a/server/text_generation/utils/hub.py b/server/text_generation/utils/hub.py index 60072a20..372df306 100644 --- a/server/text_generation/utils/hub.py +++ b/server/text_generation/utils/hub.py @@ -134,6 +134,7 @@ def download_file(filename): logger.info(f"File {filename} already present in cache.") return local_file + logger.info(f"Starting {filename} download.") start_time = time.time() local_file = hf_hub_download( filename=filename, @@ -144,7 +145,7 @@ def download_file(filename): logger.info( f"Downloaded {filename} at {local_file} in {timedelta(seconds=int(time.time() - start_time))}." ) - return local_file + return Path(local_file) executor = ThreadPoolExecutor(max_workers=5) futures = [ @@ -152,18 +153,14 @@ def download_file(filename): ] # We do this instead of using tqdm because we want to parse the logs with the launcher - logger.info("Downloading weights...") start_time = time.time() files = [] for i, future in enumerate(concurrent.futures.as_completed(futures)): elapsed = timedelta(seconds=int(time.time() - start_time)) remaining = len(futures) - (i + 1) - if remaining != 0: - eta = (elapsed / (i + 1)) * remaining - else: - eta = 0 + eta = (elapsed / (i + 1)) * remaining if remaining > 0 else 0 logger.info(f"Download: [{i + 1}/{len(futures)}] -- ETA: {eta}") - files.append(Path(future.result())) + files.append(future.result()) - return [Path(p) for p in files] + return files From 68455353f5c158baf5576da6fcb670404ee0c391 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 15 Feb 2023 16:23:45 +0100 Subject: [PATCH 081/793] feat(launcher): add disable_custom_kernels arg (#67) --- launcher/src/main.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index ecb2c0b6..8b0b55f6 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -47,6 +47,8 @@ struct Args { #[clap(long, env)] weights_cache_override: Option, #[clap(long, env)] + disable_custom_kernels: bool, + #[clap(long, env)] json_output: bool, #[clap(long, env)] otlp_endpoint: Option, @@ -79,6 +81,7 @@ fn main() -> ExitCode { master_port, huggingface_hub_cache, weights_cache_override, + disable_custom_kernels, json_output, otlp_endpoint, } = args; @@ -242,6 +245,7 @@ fn main() -> ExitCode { master_port, huggingface_hub_cache, weights_cache_override, + disable_custom_kernels, otlp_endpoint, status_sender, shutdown, @@ -406,6 +410,7 @@ fn shard_manager( master_port: usize, huggingface_hub_cache: Option, weights_cache_override: Option, + disable_custom_kernels: bool, otlp_endpoint: Option, status_sender: mpsc::Sender, shutdown: Arc>, @@ -474,6 +479,11 @@ fn shard_manager( )); }; + // If disable_custom_kernels is true, pass it to the shard as an env var + if disable_custom_kernels { + env.push(("DISABLE_CUSTOM_KERNELS".into(), "True".into())) + } + // If the NCCL_SHM_DISABLE env var is set, pass it to the shard // needed when running NCCL inside a docker container and when you can't increase shm size if let Ok(nccl_shm_disalbe) = env::var("NCCL_SHM_DISABLE") { From 5437d49bebe1338197a2630e9dc19b88664ad721 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 15 Feb 2023 21:56:59 +0100 Subject: [PATCH 082/793] feat(router): add max_total_tokens and empty_input validation (#68) closes #65 --- router/src/main.rs | 8 +++++ router/src/server.rs | 12 +++++-- router/src/validation.rs | 71 +++++++++++++++++++++++++++++++--------- 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/router/src/main.rs b/router/src/main.rs index 881d94fb..5ababa4b 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -20,8 +20,12 @@ use tracing_subscriber::{EnvFilter, Layer}; struct Args { #[clap(default_value = "128", long, env)] max_concurrent_requests: usize, + #[clap(default_value = "4", long, env)] + max_stop_sequences: usize, #[clap(default_value = "1000", long, env)] max_input_length: usize, + #[clap(default_value = "1512", long, env)] + max_total_tokens: usize, #[clap(default_value = "32", long, env)] max_batch_size: usize, #[clap(default_value = "20", long, env)] @@ -46,7 +50,9 @@ fn main() -> Result<(), std::io::Error> { // Pattern match configuration let Args { max_concurrent_requests, + max_stop_sequences, max_input_length, + max_total_tokens, max_batch_size, max_waiting_tokens, port, @@ -92,7 +98,9 @@ fn main() -> Result<(), std::io::Error> { // Run server server::run( max_concurrent_requests, + max_stop_sequences, max_input_length, + max_total_tokens, max_batch_size, max_waiting_tokens, sharded_client, diff --git a/router/src/server.rs b/router/src/server.rs index 432586bb..19af1e78 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -28,7 +28,7 @@ use utoipa_swagger_ui::SwaggerUi; async fn health(infer: Extension) -> Result<(), (StatusCode, Json)> { // TODO: while this is the best health check we can do, it is a bit on the heavy side and might // be a bit too slow for a health check. - // What we should do instead if check if the gRPC channels are still healthy. + // What we should do instead is check if the gRPC channels are still healthy. // Send a small inference request infer @@ -291,7 +291,9 @@ async fn generate_stream( #[allow(clippy::too_many_arguments)] pub async fn run( max_concurrent_requests: usize, + max_stop_sequences: usize, max_input_length: usize, + max_total_tokens: usize, max_batch_size: usize, max_waiting_tokens: usize, client: ShardedClient, @@ -333,7 +335,13 @@ pub async fn run( struct ApiDoc; // Create state - let validation = Validation::new(validation_workers, tokenizer, max_input_length); + let validation = Validation::new( + validation_workers, + tokenizer, + max_stop_sequences, + max_input_length, + max_total_tokens, + ); let infer = Infer::new( client, validation, diff --git a/router/src/validation.rs b/router/src/validation.rs index aa1c1d23..50d090cd 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,3 +1,4 @@ +use crate::validation::ValidationError::EmptyInput; /// Payload validation logic use crate::{GenerateParameters, GenerateRequest}; use rand::rngs::ThreadRng; @@ -8,9 +9,6 @@ use tokenizers::tokenizer::Tokenizer; use tokio::sync::{mpsc, oneshot}; use tracing::{instrument, Span}; -const MAX_MAX_NEW_TOKENS: u32 = 512; -const MAX_STOP_SEQUENCES: usize = 4; - /// Validation #[derive(Debug, Clone)] pub struct Validation { @@ -19,7 +17,13 @@ pub struct Validation { } impl Validation { - pub(crate) fn new(workers: usize, tokenizer: Tokenizer, max_input_length: usize) -> Self { + pub(crate) fn new( + workers: usize, + tokenizer: Tokenizer, + max_stop_sequences: usize, + max_input_length: usize, + max_total_tokens: usize, + ) -> Self { // Create channel let (validation_sender, validation_receiver) = mpsc::channel(128); @@ -27,7 +31,9 @@ impl Validation { tokio::spawn(validation_task( workers, tokenizer, + max_stop_sequences, max_input_length, + max_total_tokens, validation_receiver, )); @@ -61,7 +67,9 @@ impl Validation { async fn validation_task( workers: usize, tokenizer: Tokenizer, + max_stop_sequences: usize, max_input_length: usize, + max_total_tokens: usize, mut receiver: mpsc::Receiver, ) { let mut workers_senders = Vec::with_capacity(workers); @@ -75,7 +83,13 @@ async fn validation_task( // Spawn worker tokio::task::spawn_blocking(move || { - validation_worker(tokenizer_clone, max_input_length, worker_receiver) + validation_worker( + tokenizer_clone, + max_stop_sequences, + max_input_length, + max_total_tokens, + worker_receiver, + ) }); } @@ -95,7 +109,9 @@ async fn validation_task( /// the tokenizer fn validation_worker( tokenizer: Tokenizer, + max_stop_sequences: usize, max_input_length: usize, + max_total_tokens: usize, mut receiver: mpsc::Receiver, ) { // Seed rng @@ -106,7 +122,15 @@ fn validation_worker( parent_span.in_scope(|| { response_tx .send( - validate(request, &tokenizer, max_input_length, &mut rng).map_err(|err| { + validate( + request, + &tokenizer, + max_stop_sequences, + max_input_length, + max_total_tokens, + &mut rng, + ) + .map_err(|err| { tracing::error!("{err}"); err }), @@ -119,7 +143,9 @@ fn validation_worker( fn validate( request: GenerateRequest, tokenizer: &Tokenizer, + max_stop_sequences: usize, max_input_length: usize, + max_total_tokens: usize, rng: &mut ThreadRng, ) -> Result { let GenerateParameters { @@ -161,13 +187,13 @@ fn validate( } }?; - if max_new_tokens == 0 || max_new_tokens > MAX_MAX_NEW_TOKENS { - return Err(ValidationError::MaxNewTokens(MAX_MAX_NEW_TOKENS)); + if max_new_tokens == 0 { + return Err(ValidationError::MaxNewTokens); } - if stop_sequences.len() > MAX_STOP_SEQUENCES { + if stop_sequences.len() > max_stop_sequences { return Err(ValidationError::StopSequence( - MAX_STOP_SEQUENCES, + max_stop_sequences, stop_sequences.len(), )); } @@ -178,13 +204,24 @@ fn validate( Some(seed) => seed, }; + // Check if inputs is empty + if request.inputs.is_empty() { + return Err(EmptyInput); + } + // Get the number of tokens in the input match tokenizer.encode(request.inputs.clone(), true) { Ok(encoding) => { let input_length = encoding.len(); - + let total_tokens = input_length + max_new_tokens as usize; if input_length > max_input_length { - Err(ValidationError::InputLength(input_length, max_input_length)) + Err(ValidationError::InputLength(max_input_length, input_length)) + } else if total_tokens > max_total_tokens { + Err(ValidationError::MaxTotalTokens( + max_total_tokens, + input_length, + max_new_tokens, + )) } else { // Return ValidGenerateRequest let parameters = NextTokenChooserParameters { @@ -236,10 +273,14 @@ pub enum ValidationError { TopP, #[error("top_k must be strictly positive")] TopK, - #[error("max_new_tokens must be strictly positive and <= {0}")] - MaxNewTokens(u32), - #[error("inputs must have less than {1} tokens. Given: {0}")] + #[error("max_new_tokens must be strictly positive")] + MaxNewTokens, + #[error("input tokens + max_new_tokens must be <= {0}. Given: {1} input tokens and {2} max_new_tokens")] + MaxTotalTokens(usize, usize, u32), + #[error("inputs must have less than {0} tokens. Given: {1}")] InputLength(usize, usize), + #[error("inputs cannot be empty")] + EmptyInput, #[error("stop supports up to {0} stop sequences. Given: {1}")] StopSequence(usize, usize), #[error("tokenizer error {0}")] From 7b3d460d2140034de5d5267c1c262f03510e8b88 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 16 Feb 2023 11:20:23 +0100 Subject: [PATCH 083/793] fix(launcher): copy current env vars to subprocesses (#70) closes #69 --- launcher/src/main.rs | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 8b0b55f6..f7ff1cca 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -1,6 +1,7 @@ use clap::Parser; use serde_json::Value; use std::env; +use std::ffi::OsString; use std::io::{BufRead, BufReader, Read}; use std::path::Path; use std::process::ExitCode; @@ -118,9 +119,10 @@ fn main() -> ExitCode { download_argv.push(revision.to_string()) } - let mut env = Vec::new(); + // Copy current process env + let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); - // If the HUGGINGFACE_HUB_CACHE env var is set, pass it to the shard + // If huggingface_hub_cache is set, pass it to the shard // Useful when running inside a docker container if let Some(ref huggingface_hub_cache) = huggingface_hub_cache { env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); @@ -455,14 +457,18 @@ fn shard_manager( shard_argv.push(otlp_endpoint); } - let mut env = vec![ - ("RANK".into(), rank.to_string().into()), - ("WORLD_SIZE".into(), world_size.to_string().into()), - ("MASTER_ADDR".into(), master_addr.into()), - ("MASTER_PORT".into(), master_port.to_string().into()), - ("SAFETENSORS_FAST_GPU".into(), "1".into()), - ("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into()), - ]; + // Copy current process env + let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); + + // Torch Distributed Env vars + env.push(("RANK".into(), rank.to_string().into())); + env.push(("WORLD_SIZE".into(), world_size.to_string().into())); + env.push(("MASTER_ADDR".into(), master_addr.into())); + env.push(("MASTER_PORT".into(), master_port.to_string().into())); + env.push(("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())); + + // Safetensors load fast + env.push(("SAFETENSORS_FAST_GPU".into(), "1".into())); // If huggingface_hub_cache is some, pass it to the shard // Useful when running inside a docker container @@ -484,17 +490,6 @@ fn shard_manager( env.push(("DISABLE_CUSTOM_KERNELS".into(), "True".into())) } - // If the NCCL_SHM_DISABLE env var is set, pass it to the shard - // needed when running NCCL inside a docker container and when you can't increase shm size - if let Ok(nccl_shm_disalbe) = env::var("NCCL_SHM_DISABLE") { - env.push(("NCCL_SHM_DISABLE".into(), nccl_shm_disalbe.into())); - }; - - // If the CUDA_VISIBLE_DEVICES env var is set, pass it to the shard - if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { - env.push(("CUDA_VISIBLE_DEVICES".into(), cuda_visible_devices.into())); - }; - // Start process tracing::info!("Starting shard {rank}"); let mut p = match Popen::create( From 439fcaf810034845da9ff62d50fc30646f780316 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 16 Feb 2023 17:18:53 +0100 Subject: [PATCH 084/793] feat(router): add prometheus metrics scrape endpoint (#71) --- Cargo.lock | 120 ++++++++++++++++++++++++ router/Cargo.toml | 2 + router/src/infer.rs | 54 ++++++++--- router/src/queue.rs | 3 + router/src/server.rs | 73 +++++++++++--- router/src/validation.rs | 12 ++- server/text_generation/utils/convert.py | 2 + server/text_generation/utils/hub.py | 6 +- 8 files changed, 239 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8a28bf9..944047e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -806,6 +817,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] [[package]] name = "heck" @@ -1093,6 +1107,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "macro_rules_attribute" version = "0.1.3" @@ -1139,6 +1162,64 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" +dependencies = [ + "ahash", + "metrics-macros", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70" +dependencies = [ + "hyper", + "indexmap", + "ipnet", + "metrics", + "metrics-util", + "parking_lot", + "portable-atomic", + "quanta", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "metrics-util" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d24dc2dbae22bff6f1f9326ffce828c9f07ef9cc1e8002e5279f845432a30a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown", + "metrics", + "num_cpus", + "parking_lot", + "portable-atomic", + "quanta", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.16" @@ -1514,6 +1595,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "portable-atomic" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1618,6 +1705,22 @@ dependencies = [ "prost", ] +[[package]] +name = "quanta" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e31331286705f455e56cca62e0e717158474ff02b7936c1fa596d983f4ae27" +dependencies = [ + "crossbeam-utils", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi 0.10.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.23" @@ -1657,6 +1760,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "raw-cpuid" +version = "10.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c307f7aacdbab3f0adee67d52739a1d71112cc068d6fab169ddeb18e48877fad" +dependencies = [ + "bitflags", +] + [[package]] name = "rayon" version = "1.6.1" @@ -1980,6 +2092,12 @@ dependencies = [ "libc", ] +[[package]] +name = "sketches-ddsketch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" + [[package]] name = "slab" version = "0.4.7" @@ -2143,6 +2261,8 @@ dependencies = [ "axum-tracing-opentelemetry", "clap 4.1.4", "futures", + "metrics", + "metrics-exporter-prometheus", "nohash-hasher", "opentelemetry", "opentelemetry-otlp", diff --git a/router/Cargo.toml b/router/Cargo.toml index f1ace790..9ac500c9 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -19,6 +19,8 @@ axum-tracing-opentelemetry = "0.9.0" text-generation-client = { path = "client" } clap = { version = "4.1.4", features = ["derive", "env"] } futures = "0.3.26" +metrics = "0.20.1" +metrics-exporter-prometheus = { version = "0.11.0", features = [] } nohash-hasher = "0.2.0" opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } opentelemetry-otlp = "0.11.0" diff --git a/router/src/infer.rs b/router/src/infer.rs index 4e368492..dc0df50a 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -3,7 +3,6 @@ use crate::validation::{Validation, ValidationError}; use crate::GenerateRequest; use crate::{Entry, Queue, Token}; use nohash_hasher::IntMap; -use std::future::Future; use std::sync::Arc; use text_generation_client::{ Batch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, @@ -81,6 +80,7 @@ impl Infer { .limit_concurrent_requests .try_acquire_owned() .map_err(|err| { + metrics::increment_counter!("tgi_request_failure", "err" => "overloaded"); tracing::error!("{err}"); err })?; @@ -172,6 +172,7 @@ impl Infer { }) } else { let err = InferError::IncompleteGeneration; + metrics::increment_counter!("tgi_request_failure", "err" => "incomplete"); tracing::error!("{err}"); Err(err) } @@ -201,7 +202,7 @@ async fn batching_task( // This batch might be smaller than the maximum batch size if there are not enough requests // waiting in the queue while let Some((mut entries, batch, span)) = queue.next_batch(None, max_batch_size).await { - let mut cached_batch = wrap_future(client.prefill(batch), &mut entries) + let mut cached_batch = prefill(&mut client, batch, &mut entries) .instrument(span) .await; let mut waiting_tokens = 1; @@ -212,6 +213,7 @@ async fn batching_task( // Get current batch info let batch_size = batch.size; let mut batches = vec![batch]; + metrics::gauge!("tgi_batch_current_size", batch_size as f64); // If the current batch is too small, we try to add more requests to it if batch_size <= limit_min_batch_size { @@ -241,10 +243,9 @@ async fn batching_task( }); // Generate one token for this new batch to have the attention past in cache - let new_cached_batch = - wrap_future(client.prefill(new_batch), &mut new_entries) - .instrument(span) - .await; + let new_cached_batch = prefill(&mut client, new_batch, &mut new_entries) + .instrument(span) + .await; // Reset waiting counter waiting_tokens = 1; // Extend current batch with the new batch @@ -268,29 +269,59 @@ async fn batching_task( entry.temp_span = Some(entry_batch_span); }); - cached_batch = wrap_future(client.decode(batches), &mut entries) + cached_batch = decode(&mut client, batches, &mut entries) .instrument(next_batch_span) .await; waiting_tokens += 1; } + metrics::gauge!("tgi_batch_current_size", 0.0); } } } -/// Wrap a future inside a match statement to handle errors and send the responses to Infer #[instrument(skip_all)] -async fn wrap_future( - future: impl Future, Option), ClientError>>, +async fn prefill( + client: &mut ShardedClient, + batch: Batch, entries: &mut IntMap, ) -> Option { - match future.await { + let start_time = Instant::now(); + + match client.prefill(batch).await { + Ok((generations, next_batch)) => { + send_generations(generations, entries); + metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed(), "method" => "prefill"); + metrics::increment_counter!("tgi_batch_inference_success", "method" => "prefill"); + next_batch + } + // If we have an error, we discard the whole batch + Err(err) => { + send_errors(err, entries); + metrics::increment_counter!("tgi_batch_inference_failure", "method" => "prefill"); + None + } + } +} + +#[instrument(skip_all)] +async fn decode( + client: &mut ShardedClient, + batches: Vec, + entries: &mut IntMap, +) -> Option { + let start_time = Instant::now(); + + match client.decode(batches).await { Ok((generations, next_batch)) => { send_generations(generations, entries); + metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed(), "method" => "decode"); + metrics::increment_counter!("tgi_batch_inference_success", "method" => "decode"); next_batch } // If we have an error, we discard the whole batch Err(err) => { send_errors(err, entries); + metrics::increment_counter!("tgi_batch_inference_failure", "method" => "decode"); None } } @@ -303,6 +334,7 @@ fn send_errors(error: ClientError, entries: &mut IntMap) { // Create and enter a span to link this function back to the entry let _send_error_span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_error").entered(); let err = InferError::GenerationError(error.to_string()); + metrics::increment_counter!("tgi_request_failure", "err" => "generation"); tracing::error!("{err}"); // unwrap_or is valid here as we don't care if the receiver is gone. diff --git a/router/src/queue.rs b/router/src/queue.rs index b155a1af..8962aaec 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -132,6 +132,7 @@ impl State { // Push entry in the queue self.entries.push((self.next_id, entry)); self.next_id += 1; + metrics::increment_gauge!("tgi_queue_size", 1.0); } // Get the next batch @@ -190,6 +191,8 @@ impl State { // Increment batch id self.next_batch_id += 1; + metrics::gauge!("tgi_queue_size", self.entries.len() as f64); + metrics::histogram!("tgi_batch_next_size", batch.size as f64); Some((batch_entries, batch, next_batch_span)) } } diff --git a/router/src/server.rs b/router/src/server.rs index 19af1e78..48affa46 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -12,6 +12,7 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use axum_tracing_opentelemetry::opentelemetry_tracing_layer; use futures::Stream; +use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; use std::convert::Infallible; use std::net::SocketAddr; use text_generation_client::ShardedClient; @@ -57,14 +58,14 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json "incomplete"); tracing::error!("{err}"); yield Ok(Event::from(err)) } @@ -287,6 +310,17 @@ async fn generate_stream( Sse::new(stream).keep_alive(KeepAlive::default()) } +/// Prometheus metrics scrape endpoint +#[utoipa::path( + get, + tag = "Text Generation Inference", + path = "/metrics", + responses((status = 200, description = "Prometheus Metrics", body = String)) +)] +async fn metrics(prom_handle: Extension) -> String { + prom_handle.render() +} + /// Serving method #[allow(clippy::too_many_arguments)] pub async fn run( @@ -307,6 +341,7 @@ pub async fn run( paths( generate, generate_stream, + metrics, ), components( schemas( @@ -350,6 +385,12 @@ pub async fn run( max_concurrent_requests, ); + // Prometheus handler + let builder = PrometheusBuilder::new(); + let prom_handle = builder + .install_recorder() + .expect("failed to install metrics recorder"); + // Create router let app = Router::new() .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", ApiDoc::openapi())) @@ -359,6 +400,8 @@ pub async fn run( .route("/", get(health)) .route("/health", get(health)) .layer(Extension(infer)) + .route("/metrics", get(metrics)) + .layer(Extension(prom_handle)) .layer(opentelemetry_tracing_layer()); // Run server diff --git a/router/src/validation.rs b/router/src/validation.rs index 50d090cd..e63cfb47 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -13,7 +13,7 @@ use tracing::{instrument, Span}; #[derive(Debug, Clone)] pub struct Validation { /// Channel to communicate with the background validation task - sender: mpsc::Sender, + sender: mpsc::UnboundedSender, } impl Validation { @@ -25,7 +25,7 @@ impl Validation { max_total_tokens: usize, ) -> Self { // Create channel - let (validation_sender, validation_receiver) = mpsc::channel(128); + let (validation_sender, validation_receiver) = mpsc::unbounded_channel(); // Launch background validation task tokio::spawn(validation_task( @@ -54,7 +54,6 @@ impl Validation { // Unwrap is safe here self.sender .send((request, sender, Span::current())) - .await .unwrap(); // Await on response channel // Unwrap is safe here @@ -70,7 +69,7 @@ async fn validation_task( max_stop_sequences: usize, max_input_length: usize, max_total_tokens: usize, - mut receiver: mpsc::Receiver, + mut receiver: mpsc::UnboundedReceiver, ) { let mut workers_senders = Vec::with_capacity(workers); @@ -131,6 +130,7 @@ fn validation_worker( &mut rng, ) .map_err(|err| { + metrics::increment_counter!("tgi_request_failure", "err" => "validation"); tracing::error!("{err}"); err }), @@ -214,6 +214,7 @@ fn validate( Ok(encoding) => { let input_length = encoding.len(); let total_tokens = input_length + max_new_tokens as usize; + if input_length > max_input_length { Err(ValidationError::InputLength(max_input_length, input_length)) } else if total_tokens > max_total_tokens { @@ -237,6 +238,9 @@ fn validate( stop_sequences, }; + metrics::histogram!("tgi_request_input_length", input_length as f64); + metrics::histogram!("tgi_request_max_new_tokens", max_new_tokens as f64); + Ok(ValidGenerateRequest { inputs: request.inputs, input_length: input_length as u32, diff --git a/server/text_generation/utils/convert.py b/server/text_generation/utils/convert.py index 30144f0c..437e2308 100644 --- a/server/text_generation/utils/convert.py +++ b/server/text_generation/utils/convert.py @@ -49,6 +49,8 @@ def convert_file(pt_file: Path, st_file: Path): """ Convert a pytorch file to a safetensors file """ + logger.info(f"Convert {pt_file} to {st_file}.") + pt_state = torch.load(pt_file, map_location="cpu") if "state_dict" in pt_state: pt_state = pt_state["state_dict"] diff --git a/server/text_generation/utils/hub.py b/server/text_generation/utils/hub.py index 372df306..713488b5 100644 --- a/server/text_generation/utils/hub.py +++ b/server/text_generation/utils/hub.py @@ -132,9 +132,9 @@ def download_file(filename): local_file = try_to_load_from_cache(model_id, revision, filename) if local_file is not None: logger.info(f"File {filename} already present in cache.") - return local_file + return Path(local_file) - logger.info(f"Starting {filename} download.") + logger.info(f"Download file: {filename}") start_time = time.time() local_file = hf_hub_download( filename=filename, @@ -143,7 +143,7 @@ def download_file(filename): local_files_only=False, ) logger.info( - f"Downloaded {filename} at {local_file} in {timedelta(seconds=int(time.time() - start_time))}." + f"Downloaded {local_file} in {timedelta(seconds=int(time.time() - start_time))}." ) return Path(local_file) From c720555adc4ea192a55c101deafcd96daa8ee6b1 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 16 Feb 2023 17:28:29 +0100 Subject: [PATCH 085/793] v0.3.0 (#72) --- Cargo.lock | 8 ++-- docs/openapi.json | 85 +++++++++++++++------------------ launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- router/grpc-metadata/Cargo.toml | 2 +- server/pyproject.toml | 2 +- 7 files changed, 48 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 944047e4..2284ef84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -785,7 +785,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "grpc-metadata" -version = "0.1.0" +version = "0.3.0" dependencies = [ "opentelemetry", "tonic", @@ -2222,7 +2222,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.2.1" +version = "0.3.0" dependencies = [ "futures", "grpc-metadata", @@ -2239,7 +2239,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.2.1" +version = "0.3.0" dependencies = [ "clap 4.1.4", "ctrlc", @@ -2254,7 +2254,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.2.1" +version = "0.3.0" dependencies = [ "async-stream", "axum", diff --git a/docs/openapi.json b/docs/openapi.json index b4ef3ba6..11e2eec0 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -11,7 +11,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.2.1" + "version": "0.3.0" }, "paths": { "/generate": { @@ -38,10 +38,7 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GenerateResponse" - } + "$ref": "#/components/schemas/GenerateResponse" } } } @@ -51,10 +48,7 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ErrorResponse" - } + "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Input validation error" @@ -67,10 +61,7 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ErrorResponse" - } + "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Request failed during generation" @@ -83,10 +74,7 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ErrorResponse" - } + "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Model is overloaded" @@ -99,10 +87,7 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ErrorResponse" - } + "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Incomplete generation" @@ -136,12 +121,9 @@ "200": { "description": "Generated Text", "content": { - "text/event-stream ": { + "text/event-stream": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/StreamResponse" - } + "$ref": "#/components/schemas/StreamResponse" } } } @@ -149,12 +131,9 @@ "422": { "description": "Input validation error", "content": { - "text/event-stream ": { + "text/event-stream": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ErrorResponse" - } + "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Input validation error" @@ -165,12 +144,9 @@ "424": { "description": "Generation Error", "content": { - "text/event-stream ": { + "text/event-stream": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ErrorResponse" - } + "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Request failed during generation" @@ -181,12 +157,9 @@ "429": { "description": "Model is overloaded", "content": { - "text/event-stream ": { + "text/event-stream": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ErrorResponse" - } + "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Model is overloaded" @@ -197,12 +170,9 @@ "500": { "description": "Incomplete generation", "content": { - "text/event-stream ": { + "text/event-stream": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ErrorResponse" - } + "$ref": "#/components/schemas/ErrorResponse" }, "example": { "error": "Incomplete generation" @@ -213,6 +183,29 @@ }, "deprecated": false } + }, + "/metrics": { + "get": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Prometheus metrics scrape endpoint", + "description": "Prometheus metrics scrape endpoint", + "operationId": "metrics", + "responses": { + "200": { + "description": "Prometheus Metrics", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + }, + "deprecated": false + } } }, "components": { diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index a0d2b089..b60ba97d 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.2.1" +version = "0.3.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index 9ac500c9..156adad7 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.2.1" +version = "0.3.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 7c7ed48e..06ebedcc 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.2.1" +version = "0.3.0" edition = "2021" [dependencies] diff --git a/router/grpc-metadata/Cargo.toml b/router/grpc-metadata/Cargo.toml index 311092e3..66dfde26 100644 --- a/router/grpc-metadata/Cargo.toml +++ b/router/grpc-metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grpc-metadata" -version = "0.1.0" +version = "0.3.0" edition = "2021" [dependencies] diff --git a/server/pyproject.toml b/server/pyproject.toml index d3a8c112..4722f703 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.2.1" +version = "0.3.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 6796d38c6d8d9adfb13c9354650ef16ca2924dc2 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 17 Feb 2023 18:22:00 +0100 Subject: [PATCH 086/793] feat(router): add cors allow origin options (#73) --- Cargo.lock | 1 + launcher/src/main.rs | 9 +++++++++ router/Cargo.toml | 1 + router/src/main.rs | 17 +++++++++++++++++ router/src/server.rs | 16 +++++++++++++--- 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2284ef84..ea6149d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2275,6 +2275,7 @@ dependencies = [ "tokenizers", "tokio", "tokio-stream", + "tower-http", "tracing", "tracing-opentelemetry", "tracing-subscriber", diff --git a/launcher/src/main.rs b/launcher/src/main.rs index f7ff1cca..ac118566 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -53,6 +53,8 @@ struct Args { json_output: bool, #[clap(long, env)] otlp_endpoint: Option, + #[clap(long, env)] + cors_allow_origin: Vec, } fn main() -> ExitCode { @@ -85,6 +87,7 @@ fn main() -> ExitCode { disable_custom_kernels, json_output, otlp_endpoint, + cors_allow_origin, } = args; // Signal handler @@ -320,6 +323,12 @@ fn main() -> ExitCode { argv.push(otlp_endpoint); } + // CORS origins + for origin in cors_allow_origin.into_iter() { + argv.push("--cors-allow-origin".to_string()); + argv.push(origin); + } + let mut webserver = match Popen::create( &argv, PopenConfig { diff --git a/router/Cargo.toml b/router/Cargo.toml index 156adad7..97a88d55 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -32,6 +32,7 @@ thiserror = "1.0.38" tokenizers = "0.13.2" tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } tokio-stream = "0.1.11" +tower-http = { version = "0.3.5", features = ["cors"] } tracing = "0.1.37" tracing-opentelemetry = "0.18.0" tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } diff --git a/router/src/main.rs b/router/src/main.rs index 5ababa4b..f1cf09a0 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,4 +1,5 @@ /// Text Generation Inference webserver entrypoint +use axum::http::HeaderValue; use clap::Parser; use opentelemetry::sdk::propagation::TraceContextPropagator; use opentelemetry::sdk::trace; @@ -10,6 +11,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use text_generation_client::ShardedClient; use text_generation_router::server; use tokenizers::Tokenizer; +use tower_http::cors::AllowOrigin; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, Layer}; @@ -42,6 +44,8 @@ struct Args { json_output: bool, #[clap(long, env)] otlp_endpoint: Option, + #[clap(long, env)] + cors_allow_origin: Option>, } fn main() -> Result<(), std::io::Error> { @@ -61,12 +65,24 @@ fn main() -> Result<(), std::io::Error> { validation_workers, json_output, otlp_endpoint, + cors_allow_origin, } = args; if validation_workers == 0 { panic!("validation_workers must be > 0"); } + // CORS allowed origins + // map to go inside the option and then map to parse from String to HeaderValue + // Finally, convert to AllowOrigin + let cors_allow_origin: Option = cors_allow_origin.map(|cors_allow_origin| { + AllowOrigin::list( + cors_allow_origin + .iter() + .map(|origin| origin.parse::().unwrap()), + ) + }); + // Download and instantiate tokenizer // This will only be used to validate payloads // @@ -107,6 +123,7 @@ fn main() -> Result<(), std::io::Error> { tokenizer, validation_workers, addr, + cors_allow_origin, ) .await; Ok(()) diff --git a/router/src/server.rs b/router/src/server.rs index 48affa46..6acbbffa 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -5,11 +5,11 @@ use crate::{ Infer, StreamDetails, StreamResponse, Token, Validation, }; use axum::extract::Extension; -use axum::http::{HeaderMap, StatusCode}; +use axum::http::{HeaderMap, Method, StatusCode}; use axum::response::sse::{Event, KeepAlive, Sse}; use axum::response::IntoResponse; use axum::routing::{get, post}; -use axum::{Json, Router}; +use axum::{http, Json, Router}; use axum_tracing_opentelemetry::opentelemetry_tracing_layer; use futures::Stream; use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; @@ -20,6 +20,7 @@ use tokenizers::Tokenizer; use tokio::signal; use tokio::time::Instant; use tokio_stream::StreamExt; +use tower_http::cors::{AllowOrigin, CorsLayer}; use tracing::{info_span, instrument, Instrument}; use utoipa::OpenApi; use utoipa_swagger_ui::SwaggerUi; @@ -334,6 +335,7 @@ pub async fn run( tokenizer: Tokenizer, validation_workers: usize, addr: SocketAddr, + allow_origin: Option, ) { // OpenAPI documentation #[derive(OpenApi)] @@ -391,6 +393,13 @@ pub async fn run( .install_recorder() .expect("failed to install metrics recorder"); + // CORS layer + let allow_origin = allow_origin.unwrap_or(AllowOrigin::any()); + let cors_layer = CorsLayer::new() + .allow_methods([Method::GET, Method::POST]) + .allow_headers([http::header::CONTENT_TYPE]) + .allow_origin(allow_origin); + // Create router let app = Router::new() .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", ApiDoc::openapi())) @@ -402,7 +411,8 @@ pub async fn run( .layer(Extension(infer)) .route("/metrics", get(metrics)) .layer(Extension(prom_handle)) - .layer(opentelemetry_tracing_layer()); + .layer(opentelemetry_tracing_layer()) + .layer(cors_layer); // Run server axum::Server::bind(&addr) From 17bc841b1be18436b32c533c8eaeacade1f45381 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sat, 18 Feb 2023 14:04:11 +0100 Subject: [PATCH 087/793] feat(server): enable hf-transfer (#76) --- Dockerfile | 1 + launcher/src/main.rs | 25 +++++++++------------- server/poetry.lock | 32 ++++++++++++++++++++++++++++- server/pyproject.toml | 1 + server/text_generation/utils/hub.py | 17 ++++++--------- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 907379dc..228909dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,7 @@ ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ DEBIAN_FRONTEND=noninteractive \ HUGGINGFACE_HUB_CACHE=/data \ + HF_HUB_ENABLE_HF_TRANSFER=1 \ MODEL_ID=bigscience/bloom-560m \ QUANTIZE=false \ NUM_SHARD=1 \ diff --git a/launcher/src/main.rs b/launcher/src/main.rs index ac118566..218e3f3a 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -98,23 +98,18 @@ fn main() -> ExitCode { }) .expect("Error setting Ctrl-C handler"); - // Download weights - if weights_cache_override.is_none() { + // Download weights for sharded models + if weights_cache_override.is_none() && num_shard > 1 { let mut download_argv = vec![ "text-generation-server".to_string(), "download-weights".to_string(), model_id.clone(), + "--extension".to_string(), + ".safetensors".to_string(), "--logger-level".to_string(), "INFO".to_string(), "--json-output".to_string(), ]; - if num_shard == 1 { - download_argv.push("--extension".to_string()); - download_argv.push(".bin".to_string()); - } else { - download_argv.push("--extension".to_string()); - download_argv.push(".safetensors".to_string()); - } // Model optional revision if let Some(ref revision) = revision { @@ -131,6 +126,9 @@ fn main() -> ExitCode { env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); }; + // Enable hf transfer for insane download speeds + env.push(("HF_HUB_ENABLE_HF_TRANSFER".into(), "1".into())); + // Start process tracing::info!("Starting download process."); let mut download_process = match Popen::create( @@ -209,12 +207,6 @@ fn main() -> ExitCode { } sleep(Duration::from_millis(100)); } - } else { - tracing::info!( - "weights_cache_override is set to {:?}.", - weights_cache_override - ); - tracing::info!("Skipping download.") } // Shared shutdown bool @@ -479,6 +471,9 @@ fn shard_manager( // Safetensors load fast env.push(("SAFETENSORS_FAST_GPU".into(), "1".into())); + // Enable hf transfer for insane download speeds + env.push(("HF_HUB_ENABLE_HF_TRANSFER".into(), "1".into())); + // If huggingface_hub_cache is some, pass it to the shard // Useful when running inside a docker container if let Some(huggingface_hub_cache) = huggingface_hub_cache { diff --git a/server/poetry.lock b/server/poetry.lock index 0e0655cb..03b97512 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -192,6 +192,14 @@ grpcio = ">=1.51.1" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" +[[package]] +name = "hf-transfer" +version = "0.1.0" +description = "" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "idna" version = "3.4" @@ -622,7 +630,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "f3cab6881b52045770a90ec9be7415a0ee499d9e980892d544f68073700cf321" +content-hash = "ef6da62cff76be3eeb45eac98326d6e4fac5d35796b8bdcf555575323ce97ba2" [metadata.files] accelerate = [ @@ -861,6 +869,28 @@ grpcio-tools = [ {file = "grpcio_tools-1.51.1-cp39-cp39-win32.whl", hash = "sha256:40ef70e8c5d0310dedff9af502b520b4c7e215bce94094527fb959150a0c594a"}, {file = "grpcio_tools-1.51.1-cp39-cp39-win_amd64.whl", hash = "sha256:15b8acf4eaa0ebe37e2f69108de49efd935b7abe9c7e58ba737490b99906aa76"}, ] +hf-transfer = [ + {file = "hf_transfer-0.1.0-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:0f41bb04898d041b774220048f237d10560ec27e1decd01a04d323c64202e8fe"}, + {file = "hf_transfer-0.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94510d4e3a66aa5afa06b61ff537895c3f1b93d689575a8a840ea9ec3189d3d8"}, + {file = "hf_transfer-0.1.0-cp310-none-win_amd64.whl", hash = "sha256:e85134084c7e9e9daa74331c4690f821a30afedacb97355d1a66c243317c3e7a"}, + {file = "hf_transfer-0.1.0-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:3441b0cba24afad7fffbcfc0eb7e31d3df127d092feeee73e5b08bb5752c903b"}, + {file = "hf_transfer-0.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14938d44c71a07b452612a90499b5f021b2277e1c93c66f60d06d746b2d0661d"}, + {file = "hf_transfer-0.1.0-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:ae01b9844995622beee0f1b7ff0240e269bfc28ea46149eb4abbf63b4683f3e2"}, + {file = "hf_transfer-0.1.0-cp311-none-win_amd64.whl", hash = "sha256:79e9505bffd3a1086be13033a805c8e6f4bb763de03a4197b959984def587e7f"}, + {file = "hf_transfer-0.1.0-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:5656deb183e271d37925de0d6989d7a4b1eefae42d771f10907f41fce08bdada"}, + {file = "hf_transfer-0.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcb16ec07f1ad343b7189745b6f659b7f82b864a55d3b84fe34361f64e4abc76"}, + {file = "hf_transfer-0.1.0-cp37-none-win_amd64.whl", hash = "sha256:5314b708bc2a8cf844885d350cd13ba0b528466d3eb9766e4d8d39d080e718c0"}, + {file = "hf_transfer-0.1.0-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:fe245d0d84bbc113870144c56b50425f9b6cacc3e361b3559b0786ac076ba260"}, + {file = "hf_transfer-0.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49099d41c05a19202dca0306bfa7f42cedaea57ccc783641b1533de860b6a1f4"}, + {file = "hf_transfer-0.1.0-cp38-none-win_amd64.whl", hash = "sha256:0d7bb607a7372908ffa2d55f1e6790430c5706434c2d1d664db4928730c2c7e4"}, + {file = "hf_transfer-0.1.0-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:70f40927bc4a19ab50605bb542bd3858eb465ad65c94cfcaf36cf36d68fc5169"}, + {file = "hf_transfer-0.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d92203155f451a9b517267d2a0966b282615037e286ff0420a2963f67d451de3"}, + {file = "hf_transfer-0.1.0-cp39-none-win_amd64.whl", hash = "sha256:c1c2799154e4bd03d2b2e2907d494005f707686b80d5aa4421c859ffa612ffa3"}, + {file = "hf_transfer-0.1.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0900bd5698a77fb44c95639eb3ec97202d13e1bd4282cde5c81ed48e3f9341e"}, + {file = "hf_transfer-0.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:427032df4e83a1bedaa76383a5b825cf779fdfc206681c28b21476bc84089280"}, + {file = "hf_transfer-0.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e36fe371af6e31795621ffab53a2dcb107088fcfeb2951eaa2a10bfdc8b8863b"}, + {file = "hf_transfer-0.1.0.tar.gz", hash = "sha256:f692ef717ded50e441b4d40b6aea625772f68b90414aeef86bb33eab40cb09a4"}, +] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 4722f703..025aebf3 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -22,6 +22,7 @@ loguru = "^0.6.0" opentelemetry-api = "^1.15.0" opentelemetry-exporter-otlp = "^1.15.0" opentelemetry-instrumentation-grpc = "^0.36b0" +hf-transfer = "^0.1.0" [tool.poetry.extras] bnb = ["bitsandbytes"] diff --git a/server/text_generation/utils/hub.py b/server/text_generation/utils/hub.py index 713488b5..7166b4cb 100644 --- a/server/text_generation/utils/hub.py +++ b/server/text_generation/utils/hub.py @@ -1,8 +1,6 @@ import time -import concurrent import os -from concurrent.futures import ThreadPoolExecutor from datetime import timedelta from loguru import logger from pathlib import Path @@ -147,20 +145,17 @@ def download_file(filename): ) return Path(local_file) - executor = ThreadPoolExecutor(max_workers=5) - futures = [ - executor.submit(download_file, filename=filename) for filename in filenames - ] - # We do this instead of using tqdm because we want to parse the logs with the launcher start_time = time.time() files = [] - for i, future in enumerate(concurrent.futures.as_completed(futures)): + for i, filename in enumerate(filenames): + file = download_file(filename) + elapsed = timedelta(seconds=int(time.time() - start_time)) - remaining = len(futures) - (i + 1) + remaining = len(filenames) - (i + 1) eta = (elapsed / (i + 1)) * remaining if remaining > 0 else 0 - logger.info(f"Download: [{i + 1}/{len(futures)}] -- ETA: {eta}") - files.append(future.result()) + logger.info(f"Download: [{i + 1}/{len(filenames)}] -- ETA: {eta}") + files.append(file) return files From 78063c05698a8555a617168b9b126910e9f720c7 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 20 Feb 2023 19:28:57 +0100 Subject: [PATCH 088/793] fix(server): remove position_ids from galactica forward (#82) closes #80 --- server/text_generation/models/galactica.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index 780a94f1..fad094dd 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -2,7 +2,7 @@ import torch import torch.distributed -from typing import List, Optional, Type +from typing import List, Optional, Type, Tuple from accelerate import init_empty_weights from safetensors import safe_open @@ -145,6 +145,20 @@ def decode(self, generated_ids: List[int]) -> str: generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False ) + def forward( + self, input_ids, attention_mask, position_ids, past_key_values: Optional = None + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + """Overwrite forward to ignore position_ids""" + + # Model Forward + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + return outputs.logits, outputs.past_key_values + class GalacticaSharded(Galactica): def __init__( @@ -322,7 +336,6 @@ def forward( outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask, - position_ids=position_ids, past_key_values=past_key_values, use_cache=True, ) From 44ce098c10bfeaedfa74f52ae81e8c6c4bc66a7d Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 24 Feb 2023 12:49:21 +0100 Subject: [PATCH 089/793] feat(server): pre-allocate max attention mask (#75) --- server/tests/models/test_bloom.py | 25 ++-- server/tests/models/test_causal_lm.py | 25 ++-- server/tests/models/test_seq2seq_lm.py | 12 +- server/text_generation/models/causal_lm.py | 71 ++++++++---- server/text_generation/models/galactica.py | 2 - server/text_generation/models/seq2seq_lm.py | 119 +++++++++++--------- server/text_generation/models/t5.py | 8 -- 7 files changed, 148 insertions(+), 114 deletions(-) diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index b06d57f5..bc36276a 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -65,8 +65,8 @@ def test_batch_from_pb(default_pb_batch, default_bloom_batch): assert batch.input_ids[0][-1] == 10264 assert torch.all(batch.input_ids[0][:-1] == 3) - assert batch.attention_mask[0][-1] == 1 - assert torch.all(batch.attention_mask[0][:-1] == 0) + assert batch.attention_mask[0][0] == 1 + assert torch.all(batch.attention_mask[0][1:] == 0) assert batch.past_key_values is None @@ -98,16 +98,13 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): assert not next_batch.keys_head_dim_last assert len(next_batch.all_input_ids) == next_batch.size - assert ( - len(next_batch.all_input_ids[0]) - == len(next_batch.attention_mask[0]) - == sequence_length + 1 - ) + assert len(next_batch.all_input_ids[0]) == sequence_length + 1 + assert len(next_batch.attention_mask[0]) == 11 assert torch.all(next_batch.all_input_ids[0][-2:] == 10264) assert torch.all(next_batch.all_input_ids[0][:-2] == 3) - assert torch.all(next_batch.attention_mask[0][-2:] == 1) - assert torch.all(next_batch.attention_mask[0][:-2] == 0) + assert torch.all(next_batch.attention_mask[0][:2] == 1) + assert torch.all(next_batch.attention_mask[0][2:] == 0) assert next_batch.input_ids.shape == (next_batch.size, 1) assert next_batch.input_ids[0, 0] == 10264 @@ -213,9 +210,13 @@ def test_batch_concatenate( assert torch.equal(next_batch.all_input_ids[1], next_batch_1.all_input_ids[0]) assert torch.equal(next_batch.all_input_ids[2], next_batch_1.all_input_ids[1]) - assert torch.all(next_batch.attention_mask[0] == 1) - assert torch.all(next_batch.attention_mask[1:, -2:] == 1) - assert torch.all(next_batch.attention_mask[1:, :-2] == 0) + assert torch.all( + next_batch.attention_mask[0, : -next_batch.padding_right_offset] == 1 + ) + assert torch.all( + next_batch.attention_mask[1:, 1 : -next_batch.padding_right_offset] == 1 + ) + assert torch.all(next_batch.attention_mask[1:, 3:] == 0) assert next_batch.batch_id == 0 assert torch.all(next_batch.input_ids == 10264) diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 6a822815..723017cd 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -62,8 +62,8 @@ def test_batch_from_pb(default_pb_batch, default_causal_lm_batch): assert batch.input_ids[0][-1] == 14402 assert torch.all(batch.input_ids[0][:-1] == 50256) - assert batch.attention_mask[0][-1] == 1 - assert torch.all(batch.attention_mask[0][:-1] == 0) + assert batch.attention_mask[0, 0] == 1 + assert torch.all(batch.attention_mask[0, 1:] == 0) assert batch.past_key_values is None @@ -94,17 +94,14 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): assert isinstance(next_batch, CausalLMBatch) assert len(next_batch.all_input_ids) == next_batch.size - assert ( - len(next_batch.all_input_ids[0]) - == len(next_batch.attention_mask[0]) - == sequence_length + 1 - ) + assert len(next_batch.all_input_ids[0]) == sequence_length + 1 + assert len(next_batch.attention_mask[0]) == 11 assert next_batch.all_input_ids[0][-1] == 13 assert next_batch.all_input_ids[0][-2] == 14402 assert torch.all(next_batch.all_input_ids[0][:-2] == 50256) - assert torch.all(next_batch.attention_mask[0][-2:] == 1) - assert torch.all(next_batch.attention_mask[0][:-2] == 0) + assert torch.all(next_batch.attention_mask[0][0:2] == 1) + assert torch.all(next_batch.attention_mask[0][2:] == 0) assert next_batch.input_ids.shape == (next_batch.size, 1) assert next_batch.input_ids[0, 0] == 13 @@ -210,9 +207,13 @@ def test_batch_concatenate( assert torch.equal(next_batch.all_input_ids[1], next_batch_1.all_input_ids[0]) assert torch.equal(next_batch.all_input_ids[2], next_batch_1.all_input_ids[1]) - assert torch.all(next_batch.attention_mask[0] == 1) - assert torch.all(next_batch.attention_mask[1:, -2:] == 1) - assert torch.all(next_batch.attention_mask[1:, :-2] == 0) + assert torch.all( + next_batch.attention_mask[0, : -next_batch.padding_right_offset] == 1 + ) + assert torch.all( + next_batch.attention_mask[1:, 1 : -next_batch.padding_right_offset] == 1 + ) + assert torch.all(next_batch.attention_mask[1:, 3:] == 0) assert next_batch.batch_id == 0 assert next_batch.input_ids[0, 0] == 12355 diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 22c6ac9c..c6eacba7 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -106,7 +106,7 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) assert len(generations) == len(next_batch) assert isinstance(next_batch, Seq2SeqLMBatch) - assert torch.equal(next_batch.input_ids, default_seq2seq_lm_batch.input_ids) + assert next_batch.input_ids is None assert torch.equal( next_batch.attention_mask, default_seq2seq_lm_batch.attention_mask ) @@ -220,11 +220,6 @@ def test_batch_concatenate( assert next_batch.batch_id == 0 - assert torch.all(next_batch.input_ids[:, 0] == 4268) - assert torch.all(next_batch.input_ids[:, 1] == 1) - - assert torch.all(next_batch.attention_mask == 1) - assert torch.equal( next_batch.decoder_input_ids[0], next_batch_0.decoder_input_ids[0] ) @@ -233,9 +228,10 @@ def test_batch_concatenate( next_batch.decoder_input_ids[1:, -2:], next_batch_1.decoder_input_ids ) - assert torch.all(next_batch.decoder_attention_mask[0] == 1) + assert torch.all(next_batch.decoder_attention_mask[0, :3] == 1) + assert torch.all(next_batch.decoder_attention_mask[0, 3:] == 0) assert torch.all(next_batch.decoder_attention_mask[1:, 0] == 0) - assert torch.all(next_batch.decoder_attention_mask[1:, -2:] == 1) + assert torch.all(next_batch.decoder_attention_mask[1:, 1:3] == 1) assert torch.equal( next_batch.encoder_last_hidden_state[0], diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 1ac073b3..e109b83b 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -37,6 +37,7 @@ class CausalLMBatch(Batch): # Metadata used for padding size: int max_sequence_length: int + padding_right_offset: int # Past metadata keys_head_dim_last: bool = True @@ -61,22 +62,36 @@ def from_pb( input_lengths = [] # Parse batch + max_sequence_length = 0 + padding_right_offset = 0 for r in pb.requests: inputs.append(r.inputs) input_lengths.append(r.input_length) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) - stopping_criterias.append( - StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) + stopping_criteria = StoppingCriteria.from_pb( + r.stopping_parameters, tokenizer + ) + stopping_criterias.append(stopping_criteria) + max_sequence_length = max(max_sequence_length, r.input_length) + padding_right_offset = max( + padding_right_offset, stopping_criteria.max_new_tokens ) - pad_to_multiple_of = 8 if device.type == "cuda" else None tokenized_inputs = tokenizer( inputs, return_tensors="pt", padding=True, - pad_to_multiple_of=pad_to_multiple_of, return_token_type_ids=False, ).to(device) + + input_ids = tokenized_inputs["input_ids"] + # Allocate maximum attention_mask + attention_mask = input_ids.new_zeros( + (pb.size, max_sequence_length + padding_right_offset) + ) + # Copy tokenizer attention_mask into fully allocated attention_mask + attention_mask[:, :max_sequence_length] = tokenized_inputs["attention_mask"] + position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) @@ -84,8 +99,8 @@ def from_pb( return cls( batch_id=pb.id, requests=pb.requests, - input_ids=tokenized_inputs["input_ids"], - attention_mask=tokenized_inputs["attention_mask"], + input_ids=input_ids, + attention_mask=attention_mask, position_ids=position_ids, past_key_values=None, all_input_ids=all_input_ids, @@ -93,15 +108,21 @@ def from_pb( next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=pb.size, - max_sequence_length=max(input_lengths), + max_sequence_length=max_sequence_length, + padding_right_offset=padding_right_offset, ) @classmethod @tracer.start_as_current_span("concatenate") def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # Used for padding - total_batch_size = sum(batch.size for batch in batches) - max_sequence_length = max(batch.max_sequence_length for batch in batches) + total_batch_size = 0 + max_sequence_length = 0 + padding_right_offset = 0 + for batch in batches: + total_batch_size += batch.size + max_sequence_length = max(max_sequence_length, batch.max_sequence_length) + padding_right_offset = max(padding_right_offset, batch.padding_right_offset) # Batch attributes requests = [] @@ -144,13 +165,22 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # Create padded tensor if attention_mask is None: attention_mask = batch.attention_mask.new_zeros( - (total_batch_size, max_sequence_length), + (total_batch_size, max_sequence_length + padding_right_offset), ) # We need to slice the attention mask to remove padding from previous steps + # and to remove unused allocated space + left_offset = max_sequence_length - batch.max_sequence_length + batch_left_offset = ( + batch.attention_mask.shape[1] - batch.max_sequence_length - batch.padding_right_offset + ) attention_mask[ - start_index:end_index, -batch.max_sequence_length : - ] = batch.attention_mask[:, -batch.max_sequence_length :] + start_index:end_index, + left_offset:-padding_right_offset, + ] = batch.attention_mask[ + :, + batch_left_offset : -batch.padding_right_offset, + ] # Create empty tensor # position_ids is always of shape [batch_size, 1] @@ -228,6 +258,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": stopping_criterias=stopping_criterias, size=total_batch_size, max_sequence_length=max_sequence_length, + padding_right_offset=padding_right_offset, keys_head_dim_last=batches[0].keys_head_dim_last, ) @@ -294,9 +325,12 @@ def forward( def generate_token( self, batch: CausalLMBatch ) -> Tuple[List[Generation], Optional[CausalLMBatch]]: + # slice the attention mask to the correct shape + attention_mask = batch.attention_mask[:, : -batch.padding_right_offset] + logits, past = self.forward( batch.input_ids, - batch.attention_mask, + attention_mask, batch.position_ids, batch.past_key_values, ) @@ -448,14 +482,8 @@ def generate_token( next_batch_next_token_choosers = batch.next_token_choosers next_batch_stopping_criterias = batch.stopping_criterias - # Update attention_mask with padding as we added a new token to input_ids - next_batch_attention_mask = torch.cat( - [ - next_batch_attention_mask, - next_batch_attention_mask.new_ones(next_batch_size, 1), - ], - dim=1, - ) + # Update attention_mask as we added a new token to input_ids + next_batch_attention_mask[:, -batch.padding_right_offset] = 1 # Update position_ids next_batch_position_ids = next_batch_position_ids[:, -1:] + 1 @@ -473,6 +501,7 @@ def generate_token( stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, max_sequence_length=next_batch_max_sequence_length, + padding_right_offset=batch.padding_right_offset - 1, keys_head_dim_last=batch.keys_head_dim_last, ) return generations, next_batch diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index fad094dd..d31f3598 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -106,12 +106,10 @@ def from_pb( ) # Tokenize batch - pad_to_multiple_of = 8 if device.type == "cuda" else None tokenized_inputs = tokenizer( inputs, return_tensors="pt", padding=True, - pad_to_multiple_of=pad_to_multiple_of, return_token_type_ids=False, ).to(device) position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 2f28c4ce..4813764b 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -42,6 +42,7 @@ class Seq2SeqLMBatch(Batch): size: int max_input_length: int max_decoder_input_length: int + padding_right_offset: int def to_pb(self) -> generate_pb2.Batch: """Convert a Seq2SeqLMBatch to a text_generation.v1.Batch protobuf""" @@ -68,6 +69,8 @@ def from_pb( decoder_input_lengths = [] # Parse batch + max_input_length = 0 + padding_right_offset = 0 for r in pb.requests: inputs.append(r.inputs) input_lengths.append(r.input_length) @@ -75,17 +78,20 @@ def from_pb( decoder_input_ids.append(tokenizer.bos_token_id) decoder_input_lengths.append(1) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) - stopping_criterias.append( - StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) + stopping_criteria = StoppingCriteria.from_pb( + r.stopping_parameters, tokenizer + ) + stopping_criterias.append(stopping_criteria) + max_input_length = max(max_input_length, r.input_length) + padding_right_offset = max( + padding_right_offset, stopping_criteria.max_new_tokens ) # Tokenize batch - pad_to_multiple_of = 8 if device.type == "cuda" else None tokenized_inputs = tokenizer( inputs, return_tensors="pt", padding=True, - pad_to_multiple_of=pad_to_multiple_of, return_token_type_ids=False, ).to(device) # Convert decoder_input_ids to torch tensor of size [batch_size, 1] @@ -107,6 +113,7 @@ def from_pb( size=len(pb.requests), max_input_length=max(input_lengths), max_decoder_input_length=1, + padding_right_offset=padding_right_offset, ) @classmethod @@ -115,11 +122,17 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": """Concatenate multiple batches together by padding internal torch tensors""" # Used for padding - total_batch_size = sum(batch.size for batch in batches) - max_input_length = max(batch.max_input_length for batch in batches) - max_decoder_input_length = max( - batch.max_decoder_input_length for batch in batches - ) + total_batch_size = 0 + max_input_length = 0 + max_decoder_input_length = 0 + padding_right_offset = 0 + for batch in batches: + total_batch_size += batch.size + max_input_length = max(max_input_length, batch.max_input_length) + max_decoder_input_length = max( + max_decoder_input_length, batch.max_decoder_input_length + ) + padding_right_offset = max(padding_right_offset, batch.padding_right_offset) # Batch attributes requests = [] @@ -129,7 +142,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": stopping_criterias = [] # Batch tensors - input_ids = None attention_mask = None decoder_input_ids = None decoder_attention_mask = None @@ -155,16 +167,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": if batch.encoder_last_hidden_state is None: raise ValueError("Batch encoder_last_hidden_state cannot be None") - # Create padded tensor - if input_ids is None: - input_ids = batch.input_ids.new_zeros( - (total_batch_size, max_input_length), - ) - # Copy to correct indices - input_ids[ - start_index:end_index, -batch.max_input_length : - ] = batch.input_ids[:, -batch.max_input_length :] - # Create padded tensor if attention_mask is None: attention_mask = batch.attention_mask.new_zeros( @@ -189,19 +191,29 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": if decoder_attention_mask is None: # As decoder_attention_mask might not exist, we use `batch.attention_mask` for device here decoder_attention_mask = batch.attention_mask.new_zeros( - (total_batch_size, max_decoder_input_length), + (total_batch_size, max_decoder_input_length + padding_right_offset), ) # If the decoder mask does not exist yet, all generations started at the same time and we never concatenated # this batch. All generations are of length `batch.max_decoder_input_length`. + left_offset = max_decoder_input_length - batch.max_decoder_input_length if batch.decoder_attention_mask is None: decoder_attention_mask[ - start_index:end_index, -batch.max_decoder_input_length : + start_index:end_index, + left_offset:-padding_right_offset, ] = 1 # If it exists, we need to index else: + batch_left_offset = ( + batch.decoder_attention_mask.shape[1] + - batch.max_decoder_input_length - batch.padding_right_offset + ) decoder_attention_mask[ - start_index:end_index, -batch.max_decoder_input_length : - ] = batch.decoder_attention_mask[:, -batch.max_decoder_input_length :] + start_index:end_index, + left_offset:-padding_right_offset, + ] = batch.decoder_attention_mask[ + :, + batch_left_offset : -batch.padding_right_offset, + ] # Create padded tensor if encoder_last_hidden_state is None: @@ -273,7 +285,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": return cls( batch_id=batches[0].batch_id, requests=requests, - input_ids=input_ids, + input_ids=None, attention_mask=attention_mask, decoder_input_ids=decoder_input_ids, decoder_attention_mask=decoder_attention_mask, @@ -286,6 +298,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": size=total_batch_size, max_input_length=max_input_length, max_decoder_input_length=max_decoder_input_length, + padding_right_offset=padding_right_offset, ) def __len__(self): @@ -342,14 +355,6 @@ def forward( List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]], ]: # Model Forward - if past_key_values is not None: - decoder_input_ids = decoder_input_ids[:, -1].unsqueeze(-1) - - # Wrap `encoder_last_hidden_state` because for some reason, Transformers does a `encoder_last_hidden_state[0]` - # internally... - if encoder_last_hidden_state is not None: - encoder_last_hidden_state = [encoder_last_hidden_state] - outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask, @@ -369,12 +374,34 @@ def forward( def generate_token( self, batch: Seq2SeqLMBatch ) -> Tuple[List[Generation], Optional[Seq2SeqLMBatch]]: + if batch.decoder_attention_mask is not None: + # slice to the correct shape + decoder_attention_mask = batch.decoder_attention_mask[ + :, : -batch.padding_right_offset + ] + else: + decoder_attention_mask = None + + # check if first forward or not + if batch.past_key_values is not None: + # Only take the last token + decoder_input_ids = batch.decoder_input_ids[:, -1].unsqueeze(-1) + else: + decoder_input_ids = batch.decoder_input_ids + + # Wrap `encoder_last_hidden_state` because for some reason, Transformers does a `encoder_last_hidden_state[0]` + # internally... + if batch.encoder_last_hidden_state is not None: + encoder_last_hidden_state = [batch.encoder_last_hidden_state] + else: + encoder_last_hidden_state = batch.encoder_last_hidden_state + logits, encoder_last_hidden_state, past = self.forward( batch.input_ids, batch.attention_mask, - batch.decoder_input_ids, - batch.decoder_attention_mask, - batch.encoder_last_hidden_state, + decoder_input_ids, + decoder_attention_mask, + encoder_last_hidden_state, batch.past_key_values, ) @@ -402,7 +429,6 @@ def generate_token( logits, batch.next_token_choosers, batch.stopping_criterias, - batch.input_ids, batch.decoder_input_ids, ) @@ -414,7 +440,6 @@ def generate_token( logits, next_token_chooser, stopping_criteria, - input_tokens, decoder_input_ids, ) in enumerate(iterator): # Select next token @@ -500,10 +525,8 @@ def generate_token( # If we finished at least one generation, we need to evict the indices of the generations that finished # from the values of the next batch if len(next_batch_keep_indices) != len(batch): - # Apply indices to attention mask, past key values and other items that need to be cached - next_batch_input_ids = batch.input_ids[next_batch_keep_indices] + # Apply indices to decoder_attention mask, past key values and other items that need to be cached next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] - if batch.decoder_attention_mask is not None: next_batch_decoder_attention_mask = batch.decoder_attention_mask[ next_batch_keep_indices @@ -526,7 +549,6 @@ def generate_token( batch.stopping_criterias[i] for i in next_batch_keep_indices ] else: - next_batch_input_ids = batch.input_ids next_batch_attention_mask = batch.attention_mask next_batch_decoder_attention_mask = batch.decoder_attention_mask next_batch_encoder_last_hidden_state = encoder_last_hidden_state @@ -536,20 +558,14 @@ def generate_token( next_batch_next_token_choosers = batch.next_token_choosers next_batch_stopping_criterias = batch.stopping_criterias - # Update decoder_attention_mask with padding as we added a new token to input_ids + # Update decoder_attention_mask as we added a new token to input_ids if next_batch_decoder_attention_mask is not None: - next_batch_decoder_attention_mask = torch.cat( - [ - next_batch_decoder_attention_mask, - next_batch_decoder_attention_mask.new_ones(next_batch_size, 1), - ], - dim=1, - ) + next_batch_decoder_attention_mask[:, -batch.padding_right_offset] = 1 next_batch = Seq2SeqLMBatch( batch_id=batch.batch_id, requests=next_batch_requests, - input_ids=next_batch_input_ids, + input_ids=None, attention_mask=next_batch_attention_mask, decoder_input_ids=next_batch_decoder_input_ids, decoder_attention_mask=next_batch_decoder_attention_mask, @@ -562,5 +578,6 @@ def generate_token( size=next_batch_size, max_input_length=next_batch_max_input_length, max_decoder_input_length=next_batch_max_decoder_input_length, + padding_right_offset=batch.padding_right_offset - 1, ) return generations, next_batch diff --git a/server/text_generation/models/t5.py b/server/text_generation/models/t5.py index 536ebda3..55507539 100644 --- a/server/text_generation/models/t5.py +++ b/server/text_generation/models/t5.py @@ -221,14 +221,6 @@ def forward( List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]], ]: # Model Forward - if past_key_values is not None: - decoder_input_ids = decoder_input_ids[:, -1].unsqueeze(-1) - - # Wrap `encoder_last_hidden_state` because for some reason, Transformers does a `encoder_last_hidden_state[0]` - # internally... - if encoder_last_hidden_state is not None: - encoder_last_hidden_state = [encoder_last_hidden_state] - outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask, From 4b1c9720c03609d424469c1b6eb221745fefde4b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 24 Feb 2023 13:27:41 +0100 Subject: [PATCH 090/793] v0.3.1 (#84) --- Cargo.lock | 115 ++++++++++++++++---------------- launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- router/grpc-metadata/Cargo.toml | 2 +- server/pyproject.toml | 2 +- 6 files changed, 62 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea6149d3..438b9603 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,19 +45,20 @@ checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "async-stream" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" dependencies = [ "async-stream-impl", "futures-core", + "pin-project-lite", ] [[package]] name = "async-stream-impl" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" dependencies = [ "proc-macro2", "quote", @@ -94,9 +95,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.4" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5694b64066a2459918d8074c2ce0d5a88f409431994c2356617c8ae0c4721fc" +checksum = "2fb79c228270dcf2426e74864cabc94babb5dbab01a4314e702d2f16540e1591" dependencies = [ "async-trait", "axum-core", @@ -276,9 +277,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.4" +version = "4.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" +checksum = "ec0b0588d44d4d63a87dbd75c136c166bbfd9a86a31cb89e09906521c7d3f5e3" dependencies = [ "bitflags", "clap_derive", @@ -304,9 +305,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" +checksum = "350b9cf31731f9957399229e9b2adc51eeabdfbe9d71d9a0552275fd12710d09" dependencies = [ "os_str_bytes", ] @@ -586,9 +587,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] @@ -785,7 +786,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "grpc-metadata" -version = "0.3.0" +version = "0.3.1" dependencies = [ "opentelemetry", "tonic", @@ -853,9 +854,9 @@ checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -1018,9 +1019,9 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0a45d56fe973d6db23972bf5bc46f988a4a2385deac9cc29572f09daef" +checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", @@ -1253,14 +1254,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -1349,9 +1350,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "onig" @@ -1652,9 +1653,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698" +checksum = "3933d3ac2717077b3d5f42b40f59edfb1fb6a8c14e1c7de0f38075c4bac8e314" dependencies = [ "bytes", "prost-derive", @@ -1662,9 +1663,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f8ad728fb08fe212df3c05169e940fbb6d9d16a877ddde14644a983ba2012e" +checksum = "a24be1d23b4552a012093e1b93697b73d644ae9590e3253d878d0e77d411b614" dependencies = [ "bytes", "heck", @@ -1684,9 +1685,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda8c0881ea9f722eb9629376db3d0b903b462477c1aafcb0566610ac28ac5d" +checksum = "8e9935362e8369bc3acd874caeeae814295c504c2bdbcde5c024089cf8b4dc12" dependencies = [ "anyhow", "itertools 0.10.5", @@ -1697,11 +1698,10 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" +checksum = "7de56acd5cc9642cac2a9518d4c8c53818905398fe42d33235859e0d542a7695" dependencies = [ - "bytes", "prost", ] @@ -1716,7 +1716,7 @@ dependencies = [ "mach", "once_cell", "raw-cpuid", - "wasi 0.10.0+wasi-snapshot-preview1", + "wasi 0.10.2+wasi-snapshot-preview1", "web-sys", "winapi", ] @@ -2085,9 +2085,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] @@ -2100,9 +2100,9 @@ checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -2171,9 +2171,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", @@ -2222,7 +2222,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.3.0" +version = "0.3.1" dependencies = [ "futures", "grpc-metadata", @@ -2239,9 +2239,9 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.3.0" +version = "0.3.1" dependencies = [ - "clap 4.1.4", + "clap 4.1.6", "ctrlc", "float_eq", "reqwest", @@ -2254,12 +2254,12 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.3.0" +version = "0.3.1" dependencies = [ "async-stream", "axum", "axum-tracing-opentelemetry", - "clap 4.1.4", + "clap 4.1.6", "futures", "metrics", "metrics-exporter-prometheus", @@ -2314,9 +2314,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f297120ff9d4efe680df143d5631bba9c75fa371992b7fcb33eb3453cb0a07" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ "cfg-if", "once_cell", @@ -2324,12 +2324,11 @@ dependencies = [ [[package]] name = "time" -version = "0.1.45" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] @@ -2436,9 +2435,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite", @@ -2447,9 +2446,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6a3b08b64e6dfad376fa2432c7b1f01522e37a623c3050bc95db2d3ff21583" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -2748,9 +2747,9 @@ dependencies = [ [[package]] name = "utoipa" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3920fa753064b1be7842bea26175ffa0dfc4a8f30bcb52b8ff03fddf8889914c" +checksum = "a15f6da6a2b471134ca44b7d18e8a76d73035cf8b3ed24c4dd5ca6a63aa439c5" dependencies = [ "indexmap", "serde", @@ -2760,9 +2759,9 @@ dependencies = [ [[package]] name = "utoipa-gen" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "720298fac6efca20df9e457e67a1eab41a20d1c3101380b5c4dca1ca60ae0062" +checksum = "6f2e33027986a4707b3f5c37ed01b33d0e5a53da30204b52ff18f80600f1d0ec" dependencies = [ "proc-macro-error", "proc-macro2", @@ -2833,9 +2832,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasi" diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index b60ba97d..d53a2c4a 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.3.0" +version = "0.3.1" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index 97a88d55..fb424fbf 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.3.0" +version = "0.3.1" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 06ebedcc..b1700328 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.3.0" +version = "0.3.1" edition = "2021" [dependencies] diff --git a/router/grpc-metadata/Cargo.toml b/router/grpc-metadata/Cargo.toml index 66dfde26..246023b5 100644 --- a/router/grpc-metadata/Cargo.toml +++ b/router/grpc-metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grpc-metadata" -version = "0.3.0" +version = "0.3.1" edition = "2021" [dependencies] diff --git a/server/pyproject.toml b/server/pyproject.toml index 025aebf3..995dade3 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.3.0" +version = "0.3.1" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 0ac184ce777646f6072238ae71d4287cc5685428 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 24 Feb 2023 15:55:57 +0100 Subject: [PATCH 091/793] feat(server): add special token bool (#85) --- launcher/tests/bloom_560m.json | 74 +++++++++++++-------- launcher/tests/integration_tests.rs | 2 + launcher/tests/mt0_base.json | 70 ++++++++++++------- proto/generate.proto | 4 +- router/src/infer.rs | 7 +- router/src/lib.rs | 14 +++- router/src/server.rs | 13 ++-- server/text_generation/models/causal_lm.py | 5 +- server/text_generation/models/model.py | 1 + server/text_generation/models/seq2seq_lm.py | 14 ++-- server/text_generation/models/types.py | 2 + 11 files changed, 134 insertions(+), 72 deletions(-) diff --git a/launcher/tests/bloom_560m.json b/launcher/tests/bloom_560m.json index 17e2571e..96f89f6b 100644 --- a/launcher/tests/bloom_560m.json +++ b/launcher/tests/bloom_560m.json @@ -1,122 +1,142 @@ { + "generated_text": ".get(\"action\");\n if (action == null) {\n throw new RuntimeException", "details": { "finish_reason": "length", "generated_tokens": 20, + "seed": null, "prefill": [ { "id": 10264, - "logprob": null, - "text": "Test" + "text": "Test", + "logprob": null }, { "id": 8821, - "logprob": -11.894989, - "text": " request" + "text": " request", + "logprob": -11.894989 } ], - "seed": null, "tokens": [ { "id": 17, + "text": ".", "logprob": -1.8267672, - "text": "." + "special": false }, { "id": 1587, + "text": "get", "logprob": -2.4674969, - "text": "get" + "special": false }, { "id": 11, + "text": "(", "logprob": -1.906001, - "text": "(" + "special": false }, { "id": 5, + "text": "\"", "logprob": -1.2279545, - "text": "\"" + "special": false }, { "id": 4899, + "text": "action", "logprob": -4.170299, - "text": "action" + "special": false }, { "id": 5, + "text": "\"", "logprob": -0.32478866, - "text": "\"" + "special": false }, { "id": 12, + "text": ")", "logprob": -1.0773665, - "text": ")" + "special": false }, { "id": 30, + "text": ";", "logprob": -0.27640742, - "text": ";" + "special": false }, { "id": 837, + "text": "\n ", "logprob": -1.6970354, - "text": "\n " + "special": false }, { "id": 1320, + "text": " if", "logprob": -1.4495516, - "text": " if" + "special": false }, { "id": 375, + "text": " (", "logprob": -0.23609057, - "text": " (" + "special": false }, { "id": 4899, + "text": "action", "logprob": -1.1916996, - "text": "action" + "special": false }, { "id": 3535, + "text": " ==", "logprob": -0.8918753, - "text": " ==" + "special": false }, { "id": 5109, + "text": " null", "logprob": -0.3933342, - "text": " null" + "special": false }, { "id": 12, + "text": ")", "logprob": -0.43212673, - "text": ")" + "special": false }, { "id": 731, + "text": " {", "logprob": -0.17702064, - "text": " {" + "special": false }, { "id": 1260, + "text": "\n ", "logprob": -0.07027565, - "text": "\n " + "special": false }, { "id": 10519, + "text": " throw", "logprob": -1.3915029, - "text": " throw" + "special": false }, { "id": 2084, + "text": " new", "logprob": -0.04201372, - "text": " new" + "special": false }, { "id": 150858, + "text": " RuntimeException", "logprob": -1.7329919, - "text": " RuntimeException" + "special": false } ] - }, - "generated_text": ".get(\"action\");\n if (action == null) {\n throw new RuntimeException" + } } \ No newline at end of file diff --git a/launcher/tests/integration_tests.rs b/launcher/tests/integration_tests.rs index b70b1628..0d2b6c74 100644 --- a/launcher/tests/integration_tests.rs +++ b/launcher/tests/integration_tests.rs @@ -14,6 +14,7 @@ pub struct Token { id: u32, text: String, logprob: Option, + special: bool, } #[derive(Deserialize)] @@ -136,6 +137,7 @@ fn compare_results(result: GeneratedText, expected: GeneratedText) { { assert_eq!(token.id, expected_token.id); assert_eq!(token.text, expected_token.text); + assert_eq!(token.special, expected_token.special); if let Some(logprob) = token.logprob { let expected_logprob = expected_token.logprob.unwrap(); assert_float_eq!(logprob, expected_logprob, abs <= 0.001); diff --git a/launcher/tests/mt0_base.json b/launcher/tests/mt0_base.json index cee3bc47..c06a2c26 100644 --- a/launcher/tests/mt0_base.json +++ b/launcher/tests/mt0_base.json @@ -1,117 +1,137 @@ { + "generated_text": "\"\"\"Test the contents of the contents of the contents. \"\"\" test_test", "details": { "finish_reason": "length", "generated_tokens": 20, + "seed": null, "prefill": [ { "id": 0, - "logprob": null, - "text": "" + "text": "", + "logprob": null } ], - "seed": null, "tokens": [ { "id": 259, + "text": "", "logprob": -1.3656927, - "text": "" + "special": false }, { "id": 215100, + "text": "\"\"\"", "logprob": -2.6551573, - "text": "\"\"\"" + "special": false }, { "id": 46138, + "text": "Test", "logprob": -1.8059857, - "text": "Test" + "special": false }, { "id": 287, + "text": "the", "logprob": -1.2102449, - "text": "the" + "special": false }, { "id": 259, + "text": "", "logprob": -1.6057279, - "text": "" + "special": false }, { "id": 49076, + "text": "contents", "logprob": -3.6060903, - "text": "contents" + "special": false }, { "id": 304, + "text": "of", "logprob": -0.5270343, - "text": "of" + "special": false }, { "id": 287, + "text": "the", "logprob": -0.62522805, - "text": "the" + "special": false }, { "id": 259, + "text": "", "logprob": -1.4069618, - "text": "" + "special": false }, { "id": 49076, + "text": "contents", "logprob": -2.621994, - "text": "contents" + "special": false }, { "id": 304, + "text": "of", "logprob": -1.3172221, - "text": "of" + "special": false }, { "id": 287, + "text": "the", "logprob": -0.3501925, - "text": "the" + "special": false }, { "id": 259, + "text": "", "logprob": -0.7219573, - "text": "" + "special": false }, { "id": 49076, + "text": "contents", "logprob": -1.0494149, - "text": "contents" + "special": false }, { "id": 260, + "text": ".", "logprob": -1.0803378, - "text": "." + "special": false }, { "id": 259, + "text": "", "logprob": -0.32933083, - "text": "" + "special": false }, { "id": 215100, + "text": "\"\"\"", "logprob": -0.11268901, - "text": "\"\"\"" + "special": false }, { "id": 2978, + "text": "test", "logprob": -1.5846587, - "text": "test" + "special": false }, { "id": 290, + "text": "_", "logprob": -0.49796978, - "text": "_" + "special": false }, { "id": 4125, + "text": "test", "logprob": -2.0026445, - "text": "test" + "special": false } ] - }, - "generated_text": "\"\"\"Test the contents of the contents of the contents. \"\"\" test_test" + } } \ No newline at end of file diff --git a/proto/generate.proto b/proto/generate.proto index 0c4f9626..28a61362 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -108,8 +108,10 @@ message Generation { float token_logprob = 4; /// Text string token_text = 5; + /// Is it a special token + bool token_is_special = 6; /// Complete generated text - GeneratedText generated_text = 6; + GeneratedText generated_text = 7; } message PrefillRequest { diff --git a/router/src/infer.rs b/router/src/infer.rs index dc0df50a..598c2fcf 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -1,7 +1,7 @@ /// Batching and inference logic use crate::validation::{Validation, ValidationError}; -use crate::GenerateRequest; use crate::{Entry, Queue, Token}; +use crate::{GenerateRequest, PrefillToken}; use nohash_hasher::IntMap; use std::sync::Arc; use text_generation_client::{ @@ -138,7 +138,7 @@ impl Infer { .into_iter() .zip(tokens.logprobs.into_iter()) .zip(tokens.texts.into_iter()) - .map(|((id, logprob), text)| Token { id, text, logprob }) + .map(|((id, logprob), text)| PrefillToken { id, text, logprob }) .collect(); } // Push last token @@ -372,6 +372,7 @@ fn send_generations(generations: Vec, entries: &mut IntMap, + pub(crate) prefill: Vec, pub(crate) tokens: Vec, pub(crate) generated_text: GeneratedText, pub(crate) queued: Instant, diff --git a/router/src/lib.rs b/router/src/lib.rs index 8e3199dd..1f23bfd3 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -86,6 +86,16 @@ pub(crate) struct GenerateRequest { pub parameters: GenerateParameters, } +#[derive(Debug, Serialize, ToSchema)] +pub struct PrefillToken { + #[schema(example = 0)] + id: u32, + #[schema(example = "test")] + text: String, + #[schema(nullable = true, example = -0.34)] + logprob: f32, +} + #[derive(Debug, Serialize, ToSchema)] pub struct Token { #[schema(example = 0)] @@ -94,6 +104,8 @@ pub struct Token { text: String, #[schema(nullable = true, example = -0.34)] logprob: f32, + #[schema(example = "false")] + special: bool, } #[derive(Serialize, ToSchema)] @@ -116,7 +128,7 @@ pub(crate) struct Details { pub generated_tokens: u32, #[schema(example = 42)] pub seed: Option, - pub prefill: Option>, + pub prefill: Option>, pub tokens: Option>, } diff --git a/router/src/server.rs b/router/src/server.rs index 6acbbffa..de96e397 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -2,7 +2,7 @@ use crate::infer::{InferError, InferStreamResponse}; use crate::{ Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest, GenerateResponse, - Infer, StreamDetails, StreamResponse, Token, Validation, + Infer, PrefillToken, StreamDetails, StreamResponse, Token, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; @@ -255,11 +255,11 @@ async fn generate_stream( let time_per_token = inference_time / generated_text.generated_tokens; // Tracing metadata - span.record("total_time", format!("{:?}", total_time)); - span.record("validation_time", format!("{:?}", validation_time)); - span.record("queue_time", format!("{:?}", queue_time)); - span.record("inference_time", format!("{:?}", inference_time)); - span.record("time_per_token", format!("{:?}", time_per_token)); + span.record("total_time", format!("{total_time:?}")); + span.record("validation_time", format!("{validation_time:?}")); + span.record("queue_time", format!("{queue_time:?}")); + span.record("inference_time", format!("{inference_time:?}")); + span.record("time_per_token", format!("{time_per_token:?}")); span.record("seed", format!("{:?}", generated_text.seed)); tracing::info!(parent: &span, "Output: {}", generated_text.text); @@ -349,6 +349,7 @@ pub async fn run( schemas( GenerateRequest, GenerateParameters, + PrefillToken, Token, GenerateResponse, Details, diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index e109b83b..d15197d0 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -172,7 +172,9 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # and to remove unused allocated space left_offset = max_sequence_length - batch.max_sequence_length batch_left_offset = ( - batch.attention_mask.shape[1] - batch.max_sequence_length - batch.padding_right_offset + batch.attention_mask.shape[1] + - batch.max_sequence_length + - batch.padding_right_offset ) attention_mask[ start_index:end_index, @@ -443,6 +445,7 @@ def generate_token( next_token_id_squeezed, next_token_logprob, next_token_text, + next_token_id_squeezed in self.all_special_ids, generated_text, ) diff --git a/server/text_generation/models/model.py b/server/text_generation/models/model.py index ef6a5682..52480526 100644 --- a/server/text_generation/models/model.py +++ b/server/text_generation/models/model.py @@ -12,6 +12,7 @@ class Model(ABC): def __init__(self, tokenizer: PreTrainedTokenizerBase, device: torch.device): self.tokenizer = tokenizer + self.all_special_ids = set(tokenizer.all_special_ids) self.device = device @property diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 4813764b..3a4108ab 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -205,7 +205,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": else: batch_left_offset = ( batch.decoder_attention_mask.shape[1] - - batch.max_decoder_input_length - batch.padding_right_offset + - batch.max_decoder_input_length + - batch.padding_right_offset ) decoder_attention_mask[ start_index:end_index, @@ -494,14 +495,10 @@ def generate_token( # Prefill if stopping_criteria.current_tokens == 1: - prefill_token_ids = decoder_input_ids[-new_decoder_input_length:-1] - prefill_texts = self.tokenizer.batch_decode( - prefill_token_ids, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) prefill_tokens = PrefillTokens( - prefill_token_ids, [float("nan")], prefill_texts + [self.tokenizer.bos_token_id], + [float("nan")], + [self.tokenizer.bos_token], ) else: prefill_tokens = None @@ -512,6 +509,7 @@ def generate_token( next_token_id_squeezed, next_token_logprob, next_token_text, + next_token_id_squeezed in self.all_special_ids, generated_text, ) diff --git a/server/text_generation/models/types.py b/server/text_generation/models/types.py index d1117b80..a3fbd6e8 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation/models/types.py @@ -73,6 +73,7 @@ class Generation: token_id: int token_logprob: float token_text: str + token_is_special: bool generated_text: Optional[GeneratedText] def to_pb(self) -> generate_pb2.Generation: @@ -84,6 +85,7 @@ def to_pb(self) -> generate_pb2.Generation: token_id=self.token_id, token_logprob=self.token_logprob, token_text=self.token_text, + token_is_special=self.token_is_special, generated_text=self.generated_text.to_pb() if self.generated_text is not None else None, From 3b03c4ea18797c3822135944def1e801667f762e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 24 Feb 2023 15:59:49 +0100 Subject: [PATCH 092/793] fix(docs): fix openapi schema (#86) --- docs/openapi.json | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 11e2eec0..969a2575 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -11,7 +11,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.3.0" + "version": "0.3.1" }, "paths": { "/generate": { @@ -228,7 +228,7 @@ "prefill": { "type": "array", "items": { - "$ref": "#/components/schemas/Token" + "$ref": "#/components/schemas/PrefillToken" } }, "seed": { @@ -361,6 +361,31 @@ } } }, + "PrefillToken": { + "type": "object", + "required": [ + "id", + "text", + "logprob" + ], + "properties": { + "id": { + "type": "integer", + "format": "int32", + "example": 0 + }, + "logprob": { + "type": "number", + "format": "float", + "example": -0.34, + "nullable": true + }, + "text": { + "type": "string", + "example": "test" + } + } + }, "StreamDetails": { "type": "object", "required": [ @@ -408,7 +433,8 @@ "required": [ "id", "text", - "logprob" + "logprob", + "special" ], "properties": { "id": { @@ -422,6 +448,10 @@ "example": -0.34, "nullable": true }, + "special": { + "type": "boolean", + "example": "false" + }, "text": { "type": "string", "example": "test" From 65e2f1624ef37441a6343a427720c31a13088e63 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 24 Feb 2023 17:20:00 +0100 Subject: [PATCH 093/793] fix(server): fix token_is_special (#87) --- server/text_generation/models/causal_lm.py | 2 +- server/text_generation/models/seq2seq_lm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index d15197d0..30aff87f 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -445,7 +445,7 @@ def generate_token( next_token_id_squeezed, next_token_logprob, next_token_text, - next_token_id_squeezed in self.all_special_ids, + next_token_id_squeezed.item() in self.all_special_ids, generated_text, ) diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 3a4108ab..3738d7ab 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -509,7 +509,7 @@ def generate_token( next_token_id_squeezed, next_token_logprob, next_token_text, - next_token_id_squeezed in self.all_special_ids, + next_token_id_squeezed.item() in self.all_special_ids, generated_text, ) From 21340f24bafbee6707783255fb9693bb2118a032 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 27 Feb 2023 14:56:58 +0100 Subject: [PATCH 094/793] feat(router): add legacy route for api-inference support (#88) --- router/src/lib.rs | 26 +++++++++++++++++++++++--- router/src/server.rs | 28 ++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index 1f23bfd3..d7cfa4c7 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -47,7 +47,7 @@ pub(crate) struct GenerateParameters { #[schema(exclusive_minimum = 0, exclusive_maximum = 512, default = "20")] pub max_new_tokens: u32, #[serde(default)] - #[schema(inline, max_items = 4, example = json!(["photographer"]))] + #[schema(inline, max_items = 4, example = json ! (["photographer"]))] pub stop: Vec, #[serde(default)] #[schema(default = "true")] @@ -86,13 +86,33 @@ pub(crate) struct GenerateRequest { pub parameters: GenerateParameters, } +#[derive(Clone, Debug, Deserialize, ToSchema)] +pub(crate) struct CompatGenerateRequest { + #[schema(example = "My name is Olivier and I")] + pub inputs: String, + #[serde(default = "default_parameters")] + pub parameters: GenerateParameters, + #[serde(default)] + #[allow(dead_code)] + pub stream: bool, +} + +impl From for GenerateRequest { + fn from(req: CompatGenerateRequest) -> Self { + Self { + inputs: req.inputs, + parameters: req.parameters, + } + } +} + #[derive(Debug, Serialize, ToSchema)] pub struct PrefillToken { #[schema(example = 0)] id: u32, #[schema(example = "test")] text: String, - #[schema(nullable = true, example = -0.34)] + #[schema(nullable = true, example = - 0.34)] logprob: f32, } @@ -102,7 +122,7 @@ pub struct Token { id: u32, #[schema(example = "test")] text: String, - #[schema(nullable = true, example = -0.34)] + #[schema(nullable = true, example = - 0.34)] logprob: f32, #[schema(example = "false")] special: bool, diff --git a/router/src/server.rs b/router/src/server.rs index de96e397..83b0297e 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,8 +1,9 @@ /// HTTP Server logic use crate::infer::{InferError, InferStreamResponse}; use crate::{ - Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest, GenerateResponse, - Infer, PrefillToken, StreamDetails, StreamResponse, Token, Validation, + CompatGenerateRequest, Details, ErrorResponse, FinishReason, GenerateParameters, + GenerateRequest, GenerateResponse, Infer, PrefillToken, StreamDetails, StreamResponse, Token, + Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; @@ -25,6 +26,25 @@ use tracing::{info_span, instrument, Instrument}; use utoipa::OpenApi; use utoipa_swagger_ui::SwaggerUi; +/// Compatibility route with api-inference and AzureML +#[instrument(skip(infer))] +async fn compat_generate( + infer: Extension, + req: Json, +) -> Result)> { + // switch on stream + let req = req.0; + if req.stream { + Ok(generate_stream(infer, Json(req.into())) + .await + .into_response()) + } else { + let (headers, generation) = generate(infer, Json(req.into())).await?; + // wrap generation inside a Vec to match api-inference + Ok((headers, Json(vec![generation.0])).into_response()) + } +} + /// Health check method #[instrument(skip(infer))] async fn health(infer: Extension) -> Result<(), (StatusCode, Json)> { @@ -84,7 +104,7 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json, req: Json, -) -> Result)> { +) -> Result<(HeaderMap, Json), (StatusCode, Json)> { let span = tracing::Span::current(); let start_time = Instant::now(); @@ -404,7 +424,7 @@ pub async fn run( // Create router let app = Router::new() .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", ApiDoc::openapi())) - .route("/", post(generate)) + .route("/", post(compat_generate)) .route("/generate", post(generate)) .route("/generate_stream", post(generate_stream)) .route("/", get(health)) From 4e685d907ec844d57b2ede16572f31355dc0df83 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 28 Feb 2023 10:19:32 +0100 Subject: [PATCH 095/793] feat(router): ask hf.co for pipelinetag to decide on compat_return_full_text (#89) --- Cargo.lock | 1 + router/Cargo.toml | 1 + router/src/lib.rs | 12 +++---- router/src/main.rs | 24 +++++++++++++- router/src/server.rs | 78 +++++++++++++++++++++++++++++++------------- 5 files changed, 87 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 438b9603..60d47a6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2268,6 +2268,7 @@ dependencies = [ "opentelemetry-otlp", "parking_lot", "rand", + "reqwest", "serde", "serde_json", "text-generation-client", diff --git a/router/Cargo.toml b/router/Cargo.toml index fb424fbf..6e673af0 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -26,6 +26,7 @@ opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } opentelemetry-otlp = "0.11.0" parking_lot = "0.12.1" rand = "0.8.5" +reqwest = { version = "0.11.14", features = [] } serde = "1.0.152" serde_json = "1.0.93" thiserror = "1.0.38" diff --git a/router/src/lib.rs b/router/src/lib.rs index d7cfa4c7..78f9efd1 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -40,13 +40,16 @@ pub(crate) struct GenerateParameters { example = 0.95 )] pub top_p: Option, - #[serde(default = "default_do_sample")] + #[serde(default)] #[schema(default = "false", example = true)] pub do_sample: bool, #[serde(default = "default_max_new_tokens")] #[schema(exclusive_minimum = 0, exclusive_maximum = 512, default = "20")] pub max_new_tokens: u32, #[serde(default)] + #[schema(default = "None", example = false)] + pub return_full_text: Option, + #[serde(default)] #[schema(inline, max_items = 4, example = json ! (["photographer"]))] pub stop: Vec, #[serde(default)] @@ -56,10 +59,6 @@ pub(crate) struct GenerateParameters { pub seed: Option, } -fn default_do_sample() -> bool { - false -} - fn default_max_new_tokens() -> u32 { 20 } @@ -70,8 +69,9 @@ fn default_parameters() -> GenerateParameters { repetition_penalty: None, top_k: None, top_p: None, - do_sample: default_do_sample(), + do_sample: false, max_new_tokens: default_max_new_tokens(), + return_full_text: None, stop: vec![], details: false, seed: None, diff --git a/router/src/main.rs b/router/src/main.rs index f1cf09a0..2baf9e72 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -87,7 +87,7 @@ fn main() -> Result<(), std::io::Error> { // This will only be used to validate payloads // // We need to download it outside of the Tokio runtime - let tokenizer = Tokenizer::from_pretrained(tokenizer_name, None).unwrap(); + let tokenizer = Tokenizer::from_pretrained(tokenizer_name.clone(), None).unwrap(); // Launch Tokio runtime tokio::runtime::Builder::new_multi_thread() @@ -97,6 +97,27 @@ fn main() -> Result<(), std::io::Error> { .block_on(async { init_logging(otlp_endpoint, json_output); + // Get pipeline tag + let model_info = reqwest::get(format!( + "https://huggingface.co/api/models/{tokenizer_name}" + )) + .await + .expect("Could not connect to hf.co") + .text() + .await + .expect("error when retrieving model info from hf.co"); + let model_info: serde_json::Value = + serde_json::from_str(&model_info).expect("unable to parse model info"); + + // if pipeline-tag == text-generation we default to return_full_text = true + let compat_return_full_text = match model_info.get("pipeline_tag") { + None => { + tracing::warn!("no pipeline tag found for model {tokenizer_name}"); + false + } + Some(pipeline_tag) => pipeline_tag.as_str() == Some("text-generation"), + }; + // Instantiate sharded client from the master unix socket let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path) .await @@ -113,6 +134,7 @@ fn main() -> Result<(), std::io::Error> { // Run server server::run( + compat_return_full_text, max_concurrent_requests, max_stop_sequences, max_input_length, diff --git a/router/src/server.rs b/router/src/server.rs index 83b0297e..75f84ba5 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -29,11 +29,18 @@ use utoipa_swagger_ui::SwaggerUi; /// Compatibility route with api-inference and AzureML #[instrument(skip(infer))] async fn compat_generate( + default_return_full_text: Extension, infer: Extension, req: Json, ) -> Result)> { + let mut req = req.0; + + // default return_full_text given the pipeline_tag + if req.parameters.return_full_text.is_none() { + req.parameters.return_full_text = Some(default_return_full_text.0) + } + // switch on stream - let req = req.0; if req.stream { Ok(generate_stream(infer, Json(req.into())) .await @@ -63,6 +70,7 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json) -> Result<(), (StatusCode, Json { error = true; - yield Ok(Event::from(err)) + yield Ok(Event::from(err)); + break; } } } @@ -315,7 +347,7 @@ async fn generate_stream( // yield error Err(err) => { error = true; - yield Ok(Event::from(err)) + yield Ok(Event::from(err)); } } // Check if generation reached the end @@ -324,7 +356,7 @@ async fn generate_stream( let err = InferError::IncompleteGeneration; metrics::increment_counter!("tgi_request_failure", "err" => "incomplete"); tracing::error!("{err}"); - yield Ok(Event::from(err)) + yield Ok(Event::from(err)); } }; @@ -345,6 +377,7 @@ async fn metrics(prom_handle: Extension) -> String { /// Serving method #[allow(clippy::too_many_arguments)] pub async fn run( + compat_return_full_text: bool, max_concurrent_requests: usize, max_stop_sequences: usize, max_input_length: usize, @@ -429,8 +462,9 @@ pub async fn run( .route("/generate_stream", post(generate_stream)) .route("/", get(health)) .route("/health", get(health)) - .layer(Extension(infer)) .route("/metrics", get(metrics)) + .layer(Extension(compat_return_full_text)) + .layer(Extension(infer)) .layer(Extension(prom_handle)) .layer(opentelemetry_tracing_layer()) .layer(cors_layer); From f874c47831e12a1d447da12cb9e699cb2fb001da Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 2 Mar 2023 11:41:51 +0100 Subject: [PATCH 096/793] feat(router): add api-inference headers (#91) --- router/src/server.rs | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/router/src/server.rs b/router/src/server.rs index 75f84ba5..5d4140ed 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -8,7 +8,7 @@ use crate::{ use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; use axum::response::sse::{Event, KeepAlive, Sse}; -use axum::response::IntoResponse; +use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{http, Json, Router}; use axum_tracing_opentelemetry::opentelemetry_tracing_layer; @@ -32,7 +32,7 @@ async fn compat_generate( default_return_full_text: Extension, infer: Extension, req: Json, -) -> Result)> { +) -> Result)> { let mut req = req.0; // default return_full_text given the pipeline_tag @@ -116,6 +116,7 @@ async fn generate( let span = tracing::Span::current(); let start_time = Instant::now(); + let compute_characters = req.0.inputs.chars().count(); let mut add_prompt = None; if req.0.parameters.return_full_text.unwrap_or(false) { add_prompt = Some(req.0.inputs.clone()); @@ -147,6 +148,15 @@ async fn generate( // Headers let mut headers = HeaderMap::new(); + headers.insert("x-compute-type", "gpu+optimized".parse().unwrap()); + headers.insert( + "x-compute-time", + total_time.as_millis().to_string().parse().unwrap(), + ); + headers.insert( + "x-compute-characters", + compute_characters.to_string().parse().unwrap(), + ); headers.insert( "x-total-time", total_time.as_millis().to_string().parse().unwrap(), @@ -239,10 +249,22 @@ async fn generate( async fn generate_stream( infer: Extension, req: Json, -) -> Sse>> { +) -> ( + HeaderMap, + Sse>>, +) { let span = tracing::Span::current(); let start_time = Instant::now(); + let compute_characters = req.0.inputs.chars().count(); + + let mut headers = HeaderMap::new(); + headers.insert("x-compute-type", "gpu+optimized".parse().unwrap()); + headers.insert( + "x-compute-characters", + compute_characters.to_string().parse().unwrap(), + ); + let stream = async_stream::stream! { // Inference let mut end_reached = false; @@ -360,7 +382,7 @@ async fn generate_stream( } }; - Sse::new(stream).keep_alive(KeepAlive::default()) + (headers, Sse::new(stream).keep_alive(KeepAlive::default())) } /// Prometheus metrics scrape endpoint From 9b8ea6a6c71c84686347590a8aa9434f13f66be4 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 2 Mar 2023 12:30:41 +0100 Subject: [PATCH 097/793] feat(server): add logits watermark (#90) --- launcher/src/main.rs | 20 +++++ proto/generate.proto | 2 + router/src/lib.rs | 6 +- router/src/queue.rs | 1 + router/src/server.rs | 1 + router/src/validation.rs | 2 + server/text_generation/models/causal_lm.py | 4 +- server/text_generation/models/galactica.py | 4 +- server/text_generation/models/seq2seq_lm.py | 4 +- server/text_generation/utils/tokens.py | 17 +++- server/text_generation/utils/watermark.py | 87 +++++++++++++++++++++ 11 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 server/text_generation/utils/watermark.py diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 218e3f3a..ca1f6738 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -55,6 +55,10 @@ struct Args { otlp_endpoint: Option, #[clap(long, env)] cors_allow_origin: Vec, + #[clap(long, env)] + watermark_gamma: Option, + #[clap(long, env)] + watermark_delta: Option, } fn main() -> ExitCode { @@ -88,6 +92,8 @@ fn main() -> ExitCode { json_output, otlp_endpoint, cors_allow_origin, + watermark_gamma, + watermark_delta, } = args; // Signal handler @@ -243,6 +249,8 @@ fn main() -> ExitCode { huggingface_hub_cache, weights_cache_override, disable_custom_kernels, + watermark_gamma, + watermark_delta, otlp_endpoint, status_sender, shutdown, @@ -414,6 +422,8 @@ fn shard_manager( huggingface_hub_cache: Option, weights_cache_override: Option, disable_custom_kernels: bool, + watermark_gamma: Option, + watermark_delta: Option, otlp_endpoint: Option, status_sender: mpsc::Sender, shutdown: Arc>, @@ -494,6 +504,16 @@ fn shard_manager( env.push(("DISABLE_CUSTOM_KERNELS".into(), "True".into())) } + // Watermark Gamma + if let Some(watermark_gamma) = watermark_gamma { + env.push(("WATERMARK_GAMMA".into(), watermark_gamma.to_string().into())) + } + + // Watermark Delta + if let Some(watermark_delta) = watermark_delta { + env.push(("WATERMARK_DELTA".into(), watermark_delta.to_string().into())) + } + // Start process tracing::info!("Starting shard {rank}"); let mut p = match Popen::create( diff --git a/proto/generate.proto b/proto/generate.proto index 28a61362..dccd7e59 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -40,6 +40,8 @@ message NextTokenChooserParameters { uint64 seed = 5; /// repetition penalty float repetition_penalty = 6; + /// token watermarking using "A Watermark for Large Language Models" + bool watermark = 7; } message StoppingCriteriaParameters { diff --git a/router/src/lib.rs b/router/src/lib.rs index 78f9efd1..1386f6b5 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -53,6 +53,9 @@ pub(crate) struct GenerateParameters { #[schema(inline, max_items = 4, example = json ! (["photographer"]))] pub stop: Vec, #[serde(default)] + #[schema(default = "false", example = true)] + pub watermark: bool, + #[serde(default)] #[schema(default = "true")] pub details: bool, #[serde(default)] @@ -72,7 +75,8 @@ fn default_parameters() -> GenerateParameters { do_sample: false, max_new_tokens: default_max_new_tokens(), return_full_text: None, - stop: vec![], + stop: Vec::new(), + watermark: false, details: false, seed: None, } diff --git a/router/src/queue.rs b/router/src/queue.rs index 8962aaec..088bdd3c 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -234,6 +234,7 @@ mod tests { do_sample: false, seed: 0, repetition_penalty: 0.0, + watermark: false }, stopping_parameters: StoppingCriteriaParameters { max_new_tokens: 0, diff --git a/router/src/server.rs b/router/src/server.rs index 5d4140ed..247c55b3 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -72,6 +72,7 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json "NextTokenChooser": return NextTokenChooser( + vocab_size=vocab_size, + watermark=pb.watermark, temperature=pb.temperature, repetition_penalty=pb.repetition_penalty, top_k=pb.top_k, diff --git a/server/text_generation/utils/watermark.py b/server/text_generation/utils/watermark.py new file mode 100644 index 00000000..6f5664fe --- /dev/null +++ b/server/text_generation/utils/watermark.py @@ -0,0 +1,87 @@ +# coding=utf-8 +# Copyright 2023 Authors of "A Watermark for Large Language Models" +# available at https://arxiv.org/abs/2301.10226 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import torch +from transformers import LogitsProcessor + +GAMMA = os.getenv("WATERMARK_GAMMA", 0.5) +DELTA = os.getenv("WATERMARK_DELTA", 2.0) + + +class WatermarkLogitsProcessor(LogitsProcessor): + def __init__( + self, + vocab_size: int, + gamma: float = GAMMA, + delta: float = DELTA, + hash_key: int = 15485863, # just a large prime number to create a rng seed with sufficient bit width + device: str = "cpu", + ): + # watermarking parameters + self.vocab_size = vocab_size + self.gamma = gamma + self.delta = delta + self.rng = torch.Generator(device=device) + self.hash_key = hash_key + + def _seed_rng(self, input_ids: torch.LongTensor) -> None: + assert ( + input_ids.shape[-1] >= 1 + ), "requires at least a 1 token prefix sequence to seed rng" + prev_token = input_ids[-1].item() + self.rng.manual_seed(self.hash_key * prev_token) + + def _get_greenlist_ids(self, input_ids: torch.LongTensor) -> list[int]: + # seed the rng using the previous tokens/prefix + self._seed_rng(input_ids) + + greenlist_size = int(self.vocab_size * self.gamma) + vocab_permutation = torch.randperm( + self.vocab_size, device=input_ids.device, generator=self.rng + ) + greenlist_ids = vocab_permutation[:greenlist_size] + return greenlist_ids + + @staticmethod + def _calc_greenlist_mask( + scores: torch.FloatTensor, greenlist_token_ids + ) -> torch.BoolTensor: + green_tokens_mask = torch.zeros_like(scores) + green_tokens_mask[-1, greenlist_token_ids] = 1 + final_mask = green_tokens_mask.bool() + return final_mask + + @staticmethod + def _bias_greenlist_logits( + scores: torch.Tensor, greenlist_mask: torch.Tensor, greenlist_bias: float + ) -> torch.Tensor: + scores[greenlist_mask] = scores[greenlist_mask] + greenlist_bias + return scores + + def __call__( + self, input_ids: torch.LongTensor, scores: torch.FloatTensor + ) -> torch.FloatTensor: + assert len(input_ids) == 1 + greenlist_ids = self._get_greenlist_ids(input_ids[0]) + green_tokens_mask = self._calc_greenlist_mask( + scores=scores, greenlist_token_ids=greenlist_ids + ) + + scores = self._bias_greenlist_logits( + scores=scores, greenlist_mask=green_tokens_mask, greenlist_bias=self.delta + ) + return scores From 2d39f199aeefb67a84752b6f149a6ea15d824223 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 3 Mar 2023 11:26:27 +0100 Subject: [PATCH 098/793] feat(server): update to hf_transfer==0.1.2 (#93) --- Cargo.lock | 114 +++++------ server/poetry.lock | 429 +++++++++++++++++++++--------------------- server/pyproject.toml | 2 +- 3 files changed, 281 insertions(+), 264 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60d47a6f..aac1114e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,9 +95,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.7" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fb79c228270dcf2426e74864cabc94babb5dbab01a4314e702d2f16540e1591" +checksum = "6137c6234afb339e75e764c866e3594900f0211e1315d33779f269bbe2ec6967" dependencies = [ "async-trait", "axum-core", @@ -121,7 +121,7 @@ dependencies = [ "sync_wrapper", "tokio", "tower", - "tower-http", + "tower-http 0.4.0", "tower-layer", "tower-service", ] @@ -154,7 +154,7 @@ dependencies = [ "http", "opentelemetry", "tower", - "tower-http", + "tower-http 0.3.5", "tracing", "tracing-opentelemetry", ] @@ -277,9 +277,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.6" +version = "4.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0b0588d44d4d63a87dbd75c136c166bbfd9a86a31cb89e09906521c7d3f5e3" +checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" dependencies = [ "bitflags", "clap_derive", @@ -292,9 +292,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.1.0" +version = "4.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" +checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" dependencies = [ "heck", "proc-macro-error", @@ -361,9 +361,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -371,9 +371,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -382,9 +382,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", "cfg-if", @@ -395,9 +395,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] @@ -796,9 +796,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -1156,9 +1156,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg", ] @@ -1653,9 +1653,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3933d3ac2717077b3d5f42b40f59edfb1fb6a8c14e1c7de0f38075c4bac8e314" +checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" dependencies = [ "bytes", "prost-derive", @@ -1663,9 +1663,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24be1d23b4552a012093e1b93697b73d644ae9590e3253d878d0e77d411b614" +checksum = "2c828f93f5ca4826f97fedcbd3f9a536c16b12cff3dbbb4a007f932bbad95b12" dependencies = [ "bytes", "heck", @@ -1685,9 +1685,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9935362e8369bc3acd874caeeae814295c504c2bdbcde5c024089cf8b4dc12" +checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b" dependencies = [ "anyhow", "itertools 0.10.5", @@ -1698,9 +1698,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de56acd5cc9642cac2a9518d4c8c53818905398fe42d33235859e0d542a7695" +checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88" dependencies = [ "prost", ] @@ -1762,9 +1762,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.6.1" +version = "10.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c307f7aacdbab3f0adee67d52739a1d71112cc068d6fab169ddeb18e48877fad" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" dependencies = [ "bitflags", ] @@ -1848,15 +1848,6 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - [[package]] name = "reqwest" version = "0.11.14" @@ -2115,9 +2106,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "95a21dcece9b5991cfd1ece74654c8e3d0d5aab499d359b0395e38229c0bb5a3" dependencies = [ "libc", "winapi", @@ -2199,16 +2190,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if", "fastrand", - "libc", "redox_syscall", - "remove_dir_all", - "winapi", + "rustix", + "windows-sys 0.42.0", ] [[package]] @@ -2241,7 +2231,7 @@ dependencies = [ name = "text-generation-launcher" version = "0.3.1" dependencies = [ - "clap 4.1.6", + "clap 4.1.8", "ctrlc", "float_eq", "reqwest", @@ -2259,7 +2249,7 @@ dependencies = [ "async-stream", "axum", "axum-tracing-opentelemetry", - "clap 4.1.6", + "clap 4.1.8", "futures", "metrics", "metrics-exporter-prometheus", @@ -2276,7 +2266,7 @@ dependencies = [ "tokenizers", "tokio", "tokio-stream", - "tower-http", + "tower-http 0.3.5", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -2385,9 +2375,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -2400,7 +2390,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -2538,12 +2528,30 @@ dependencies = [ "http-body", "http-range-header", "pin-project-lite", - "tower", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "tower-http" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.2" diff --git a/server/poetry.lock b/server/poetry.lock index 03b97512..89ad31e9 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -145,30 +145,30 @@ testing = ["protobuf (>=3.6.0)"] [[package]] name = "grpcio" -version = "1.51.1" +version = "1.51.3" description = "HTTP/2-based RPC framework" category = "main" optional = false python-versions = ">=3.7" [package.extras] -protobuf = ["grpcio-tools (>=1.51.1)"] +protobuf = ["grpcio-tools (>=1.51.3)"] [[package]] name = "grpcio-reflection" -version = "1.51.1" +version = "1.51.3" description = "Standard Protobuf Reflection Service for gRPC" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -grpcio = ">=1.51.1" +grpcio = ">=1.51.3" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.51.1" +version = "1.51.3" description = "Status proto mapping for gRPC" category = "main" optional = false @@ -176,25 +176,25 @@ python-versions = ">=3.6" [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.51.1" +grpcio = ">=1.51.3" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.51.1" +version = "1.51.3" description = "Protobuf code generator for gRPC" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -grpcio = ">=1.51.1" +grpcio = ">=1.51.3" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" [[package]] name = "hf-transfer" -version = "0.1.0" +version = "0.1.2" description = "" category = "main" optional = false @@ -436,7 +436,7 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.21.12" +version = "4.22.0" description = "" category = "main" optional = false @@ -519,7 +519,7 @@ torch = ["torch"] [[package]] name = "setuptools" -version = "67.2.0" +version = "67.4.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false @@ -575,7 +575,7 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6. [[package]] name = "typing-extensions" -version = "4.4.0" +version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -618,7 +618,7 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [[package]] name = "wrapt" -version = "1.14.1" +version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." category = "main" optional = false @@ -630,7 +630,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "ef6da62cff76be3eeb45eac98326d6e4fac5d35796b8bdcf555575323ce97ba2" +content-hash = "521dc9f3c283dc56f7d2e2f96759919ff27ab49ffd3ae7cd26317b209e7fa98d" [metadata.files] accelerate = [ @@ -768,128 +768,127 @@ grpc-interceptor = [ {file = "grpc_interceptor-0.15.0-py3-none-any.whl", hash = "sha256:63e390162e64df96c39c40508eb697def76a7cafac32a7eaf9272093eec1109e"}, ] grpcio = [ - {file = "grpcio-1.51.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:cc2bece1737b44d878cc1510ea04469a8073dbbcdd762175168937ae4742dfb3"}, - {file = "grpcio-1.51.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:e223a9793522680beae44671b9ed8f6d25bbe5ddf8887e66aebad5e0686049ef"}, - {file = "grpcio-1.51.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:24ac1154c4b2ab4a0c5326a76161547e70664cd2c39ba75f00fc8a2170964ea2"}, - {file = "grpcio-1.51.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4ef09f8997c4be5f3504cefa6b5c6cc3cf648274ce3cede84d4342a35d76db6"}, - {file = "grpcio-1.51.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8a0b77e992c64880e6efbe0086fe54dfc0bbd56f72a92d9e48264dcd2a3db98"}, - {file = "grpcio-1.51.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:eacad297ea60c72dd280d3353d93fb1dcca952ec11de6bb3c49d12a572ba31dd"}, - {file = "grpcio-1.51.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:16c71740640ba3a882f50b01bf58154681d44b51f09a5728180a8fdc66c67bd5"}, - {file = "grpcio-1.51.1-cp310-cp310-win32.whl", hash = "sha256:29cb97d41a4ead83b7bcad23bdb25bdd170b1e2cba16db6d3acbb090bc2de43c"}, - {file = "grpcio-1.51.1-cp310-cp310-win_amd64.whl", hash = "sha256:9ff42c5620b4e4530609e11afefa4a62ca91fa0abb045a8957e509ef84e54d30"}, - {file = "grpcio-1.51.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:bc59f7ba87972ab236f8669d8ca7400f02a0eadf273ca00e02af64d588046f02"}, - {file = "grpcio-1.51.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:3c2b3842dcf870912da31a503454a33a697392f60c5e2697c91d133130c2c85d"}, - {file = "grpcio-1.51.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22b011674090594f1f3245960ced7386f6af35485a38901f8afee8ad01541dbd"}, - {file = "grpcio-1.51.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49d680356a975d9c66a678eb2dde192d5dc427a7994fb977363634e781614f7c"}, - {file = "grpcio-1.51.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:094e64236253590d9d4075665c77b329d707b6fca864dd62b144255e199b4f87"}, - {file = "grpcio-1.51.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:257478300735ce3c98d65a930bbda3db172bd4e00968ba743e6a1154ea6edf10"}, - {file = "grpcio-1.51.1-cp311-cp311-win32.whl", hash = "sha256:5a6ebcdef0ef12005d56d38be30f5156d1cb3373b52e96f147f4a24b0ddb3a9d"}, - {file = "grpcio-1.51.1-cp311-cp311-win_amd64.whl", hash = "sha256:3f9b0023c2c92bebd1be72cdfca23004ea748be1813a66d684d49d67d836adde"}, - {file = "grpcio-1.51.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:cd3baccea2bc5c38aeb14e5b00167bd4e2373a373a5e4d8d850bd193edad150c"}, - {file = "grpcio-1.51.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:17ec9b13cec4a286b9e606b48191e560ca2f3bbdf3986f91e480a95d1582e1a7"}, - {file = "grpcio-1.51.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:fbdbe9a849854fe484c00823f45b7baab159bdd4a46075302281998cb8719df5"}, - {file = "grpcio-1.51.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:31bb6bc7ff145e2771c9baf612f4b9ebbc9605ccdc5f3ff3d5553de7fc0e0d79"}, - {file = "grpcio-1.51.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e473525c28251558337b5c1ad3fa969511e42304524a4e404065e165b084c9e4"}, - {file = "grpcio-1.51.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6f0b89967ee11f2b654c23b27086d88ad7bf08c0b3c2a280362f28c3698b2896"}, - {file = "grpcio-1.51.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7942b32a291421460d6a07883033e392167d30724aa84987e6956cd15f1a21b9"}, - {file = "grpcio-1.51.1-cp37-cp37m-win32.whl", hash = "sha256:f96ace1540223f26fbe7c4ebbf8a98e3929a6aa0290c8033d12526847b291c0f"}, - {file = "grpcio-1.51.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f1fec3abaf274cdb85bf3878167cfde5ad4a4d97c68421afda95174de85ba813"}, - {file = "grpcio-1.51.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:0e1a9e1b4a23808f1132aa35f968cd8e659f60af3ffd6fb00bcf9a65e7db279f"}, - {file = "grpcio-1.51.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:6df3b63538c362312bc5fa95fb965069c65c3ea91d7ce78ad9c47cab57226f54"}, - {file = "grpcio-1.51.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:172405ca6bdfedd6054c74c62085946e45ad4d9cec9f3c42b4c9a02546c4c7e9"}, - {file = "grpcio-1.51.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:506b9b7a4cede87d7219bfb31014d7b471cfc77157da9e820a737ec1ea4b0663"}, - {file = "grpcio-1.51.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fb93051331acbb75b49a2a0fd9239c6ba9528f6bdc1dd400ad1cb66cf864292"}, - {file = "grpcio-1.51.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5dca372268c6ab6372d37d6b9f9343e7e5b4bc09779f819f9470cd88b2ece3c3"}, - {file = "grpcio-1.51.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:471d39d3370ca923a316d49c8aac66356cea708a11e647e3bdc3d0b5de4f0a40"}, - {file = "grpcio-1.51.1-cp38-cp38-win32.whl", hash = "sha256:75e29a90dc319f0ad4d87ba6d20083615a00d8276b51512e04ad7452b5c23b04"}, - {file = "grpcio-1.51.1-cp38-cp38-win_amd64.whl", hash = "sha256:f1158bccbb919da42544a4d3af5d9296a3358539ffa01018307337365a9a0c64"}, - {file = "grpcio-1.51.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:59dffade859f157bcc55243714d57b286da6ae16469bf1ac0614d281b5f49b67"}, - {file = "grpcio-1.51.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:dad6533411d033b77f5369eafe87af8583178efd4039c41d7515d3336c53b4f1"}, - {file = "grpcio-1.51.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:4c4423ea38a7825b8fed8934d6d9aeebdf646c97e3c608c3b0bcf23616f33877"}, - {file = "grpcio-1.51.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0dc5354e38e5adf2498312f7241b14c7ce3484eefa0082db4297189dcbe272e6"}, - {file = "grpcio-1.51.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97d67983189e2e45550eac194d6234fc38b8c3b5396c153821f2d906ed46e0ce"}, - {file = "grpcio-1.51.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:538d981818e49b6ed1e9c8d5e5adf29f71c4e334e7d459bf47e9b7abb3c30e09"}, - {file = "grpcio-1.51.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9235dcd5144a83f9ca6f431bd0eccc46b90e2c22fe27b7f7d77cabb2fb515595"}, - {file = "grpcio-1.51.1-cp39-cp39-win32.whl", hash = "sha256:aacb54f7789ede5cbf1d007637f792d3e87f1c9841f57dd51abf89337d1b8472"}, - {file = "grpcio-1.51.1-cp39-cp39-win_amd64.whl", hash = "sha256:2b170eaf51518275c9b6b22ccb59450537c5a8555326fd96ff7391b5dd75303c"}, - {file = "grpcio-1.51.1.tar.gz", hash = "sha256:e6dfc2b6567b1c261739b43d9c59d201c1b89e017afd9e684d85aa7a186c9f7a"}, + {file = "grpcio-1.51.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:f601aaeae18dab81930fb8d4f916b0da21e89bb4b5f7367ef793f46b4a76b7b0"}, + {file = "grpcio-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:eef0450a4b5ed11feab639bf3eb1b6e23d0efa9b911bf7b06fb60e14f5f8a585"}, + {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:82b0ad8ac825d4bb31bff9f638557c045f4a6d824d84b21e893968286f88246b"}, + {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3667c06e37d6cd461afdd51cefe6537702f3d1dc5ff4cac07e88d8b4795dc16f"}, + {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3709048fe0aa23dda09b3e69849a12055790171dab9e399a72ea8f9dfbf9ac80"}, + {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:200d69857f9910f7458b39b9bcf83ee4a180591b40146ba9e49314e3a7419313"}, + {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cd9a5e68e79c5f031500e67793048a90209711e0854a9ddee8a3ce51728de4e5"}, + {file = "grpcio-1.51.3-cp310-cp310-win32.whl", hash = "sha256:6604f614016127ae10969176bbf12eb0e03d2fb3d643f050b3b69e160d144fb4"}, + {file = "grpcio-1.51.3-cp310-cp310-win_amd64.whl", hash = "sha256:e95c7ccd4c5807adef1602005513bf7c7d14e5a41daebcf9d8d30d8bf51b8f81"}, + {file = "grpcio-1.51.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:5e77ee138100f0bb55cbd147840f87ee6241dbd25f09ea7cd8afe7efff323449"}, + {file = "grpcio-1.51.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:68a7514b754e38e8de9075f7bb4dee919919515ec68628c43a894027e40ddec4"}, + {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c1b9f8afa62ff265d86a4747a2990ec5a96e4efce5d5888f245a682d66eca47"}, + {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8de30f0b417744288cec65ec8cf84b8a57995cf7f1e84ccad2704d93f05d0aae"}, + {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b69c7adc7ed60da1cb1b502853db61f453fc745f940cbcc25eb97c99965d8f41"}, + {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d81528ffe0e973dc840ec73a4132fd18b8203ad129d7410155d951a0a7e4f5d0"}, + {file = "grpcio-1.51.3-cp311-cp311-win32.whl", hash = "sha256:040eb421613b57c696063abde405916dd830203c184c9000fc8c3b3b3c950325"}, + {file = "grpcio-1.51.3-cp311-cp311-win_amd64.whl", hash = "sha256:2a8e17286c4240137d933b8ca506465472248b4ce0fe46f3404459e708b65b68"}, + {file = "grpcio-1.51.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d5cd1389669a847555df54177b911d9ff6f17345b2a6f19388707b7a9f724c88"}, + {file = "grpcio-1.51.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:be1bf35ce82cdbcac14e39d5102d8de4079a1c1a6a06b68e41fcd9ef64f9dd28"}, + {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:5eed34994c095e2bf7194ffac7381c6068b057ef1e69f8f08db77771350a7566"}, + {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f9a7d88082b2a17ae7bd3c2354d13bab0453899e0851733f6afa6918373f476"}, + {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c8abbc5f837111e7bd619612eedc223c290b0903b952ce0c7b00840ea70f14"}, + {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:165b05af77e6aecb4210ae7663e25acf234ba78a7c1c157fa5f2efeb0d6ec53c"}, + {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54e36c2ee304ff15f2bfbdc43d2b56c63331c52d818c364e5b5214e5bc2ad9f6"}, + {file = "grpcio-1.51.3-cp37-cp37m-win32.whl", hash = "sha256:cd0daac21d9ef5e033a5100c1d3aa055bbed28bfcf070b12d8058045c4e821b1"}, + {file = "grpcio-1.51.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2fdd6333ce96435408565a9dbbd446212cd5d62e4d26f6a3c0feb1e3c35f1cc8"}, + {file = "grpcio-1.51.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:54b0c29bdd9a3b1e1b61443ab152f060fc719f1c083127ab08d03fac5efd51be"}, + {file = "grpcio-1.51.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:ffaaf7e93fcb437356b5a4b23bf36e8a3d0221399ff77fd057e4bc77776a24be"}, + {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:eafbe7501a3268d05f2e450e1ddaffb950d842a8620c13ec328b501d25d2e2c3"}, + {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881ecb34feabf31c6b3b9bbbddd1a5b57e69f805041e5a2c6c562a28574f71c4"}, + {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e860a3222139b41d430939bbec2ec9c3f6c740938bf7a04471a9a8caaa965a2e"}, + {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:49ede0528e9dac7e8a9fe30b16c73b630ddd9a576bf4b675eb6b0c53ee5ca00f"}, + {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6972b009638b40a448d10e1bc18e2223143b8a7aa20d7def0d78dd4af4126d12"}, + {file = "grpcio-1.51.3-cp38-cp38-win32.whl", hash = "sha256:5694448256e3cdfe5bd358f1574a3f2f51afa20cc834713c4b9788d60b7cc646"}, + {file = "grpcio-1.51.3-cp38-cp38-win_amd64.whl", hash = "sha256:3ea4341efe603b049e8c9a5f13c696ca37fcdf8a23ca35f650428ad3606381d9"}, + {file = "grpcio-1.51.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:6c677581ce129f5fa228b8f418cee10bd28dd449f3a544ea73c8ba590ee49d0b"}, + {file = "grpcio-1.51.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:30e09b5e0531685e176f49679b6a3b190762cc225f4565e55a899f5e14b3aa62"}, + {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:c831f31336e81243f85b6daff3e5e8a123302ce0ea1f2726ad752fd7a59f3aee"}, + {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2cd2e4cefb724cab1ba2df4b7535a9980531b9ec51b4dbb5f137a1f3a3754ef0"}, + {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7a0d0bf44438869d307f85a54f25a896ad6b4b0ca12370f76892ad732928d87"}, + {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c02abd55409bfb293371554adf6a4401197ec2133dd97727c01180889014ba4d"}, + {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f8ff75e61e1227ba7a3f16b2eadbcc11d0a54096d52ab75a6b88cfbe56f55d1"}, + {file = "grpcio-1.51.3-cp39-cp39-win32.whl", hash = "sha256:6c99a73a6260bdf844b2e5ddad02dcd530310f80e1fa72c300fa19c1c7496962"}, + {file = "grpcio-1.51.3-cp39-cp39-win_amd64.whl", hash = "sha256:22bdfac4f7f27acdd4da359b5e7e1973dc74bf1ed406729b07d0759fde2f064b"}, + {file = "grpcio-1.51.3.tar.gz", hash = "sha256:be7b2265b7527bb12109a7727581e274170766d5b3c9258d4e466f4872522d7a"}, ] grpcio-reflection = [ - {file = "grpcio-reflection-1.51.1.tar.gz", hash = "sha256:c07a93c0c36ef88fe475744289863b4787005eff4de0cc04213ecad718b01aae"}, - {file = "grpcio_reflection-1.51.1-py3-none-any.whl", hash = "sha256:b70af764a83e42a44f65df1edb232e972ab69e72bc7fbbad481e66c29a9d8cb8"}, + {file = "grpcio-reflection-1.51.3.tar.gz", hash = "sha256:5adca16f0a6cd403efa3b5f8f8a493eea6a37dee9473b178fad0a60efa68bc67"}, + {file = "grpcio_reflection-1.51.3-py3-none-any.whl", hash = "sha256:52b037f831908468afc89c60e591d0a2bbce24a393d908c44a6d53091e90fc41"}, ] grpcio-status = [ - {file = "grpcio-status-1.51.1.tar.gz", hash = "sha256:ac2617a3095935ebd785e2228958f24b10a0d527a0c9eb5a0863c784f648a816"}, - {file = "grpcio_status-1.51.1-py3-none-any.whl", hash = "sha256:a52cbdc4b18f325bfc13d319ae7c7ae7a0fee07f3d9a005504d6097896d7a495"}, + {file = "grpcio-status-1.51.3.tar.gz", hash = "sha256:71792c550356ba94e162c70818719ae6d67d960bdd03a9db5ff68faba2927f6c"}, + {file = "grpcio_status-1.51.3-py3-none-any.whl", hash = "sha256:d68d0956c16b6ea466f13c27075f126ef2cd8f0f97527d70056c64b0084357e3"}, ] grpcio-tools = [ - {file = "grpcio-tools-1.51.1.tar.gz", hash = "sha256:8e62d23d3fed9d4f81738f98dd193dbd2e21aed4a8f0dd715e75b5439e649727"}, - {file = "grpcio_tools-1.51.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:ecf1494cb695afead36995534f787761ee33fb9e116b23030113a37fe6057a83"}, - {file = "grpcio_tools-1.51.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:16b8b915625dc6eb2ea7efdfb06f1fae44a9066c9016453a2ca120c034f33090"}, - {file = "grpcio_tools-1.51.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:d5e033c04b416afcddd5231b3ff94a34fb5d26fba2416eb940e69b05f22cfd25"}, - {file = "grpcio_tools-1.51.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a218f64e667f3332b74080bdc5440aaf0fa6700ae07a0b54ecf085aaef2aa9f"}, - {file = "grpcio_tools-1.51.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7b186183515ad6b8584ffe4bd820b72b00f6e7d121fb1c36294edeea9092313"}, - {file = "grpcio_tools-1.51.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ccd37165d7a3e93f460096a2eb62b7a9c1ebe5c424eaee42d8e92740d0c8f6bc"}, - {file = "grpcio_tools-1.51.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:531586c5598a99658249f3c5e92826d6d2bb117abd6ffc88527d1e1d9eaef924"}, - {file = "grpcio_tools-1.51.1-cp310-cp310-win32.whl", hash = "sha256:392ad4cd004f7b843cf7d916d9a15b2d6585965bfef235be1c88d8f8649777e5"}, - {file = "grpcio_tools-1.51.1-cp310-cp310-win_amd64.whl", hash = "sha256:14e82c2b3ee7e300611c2c729d411b3b911e4cca5f4ec14787457a2fb72ff9d4"}, - {file = "grpcio_tools-1.51.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:2281180490c475d09b7aa05dabafa5e09de9902176931e7295113f636c2b5360"}, - {file = "grpcio_tools-1.51.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:c4649af7f5d9553975ee66b6bfae20a84be779f13e163fa835e782961895e63c"}, - {file = "grpcio_tools-1.51.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f06bb0753b7cecbff154b523cfb8f45dee2c31b0a4c72bed7da44c57f1cba113"}, - {file = "grpcio_tools-1.51.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a671466158ed74c07ee070fb940ed783acf59ba6e6e53cb4de8fd63819c6c7f"}, - {file = "grpcio_tools-1.51.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:048793747339f327ea091d8f022c6756d89713d8080dffde5ce7380cc348ea8e"}, - {file = "grpcio_tools-1.51.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f6caf36e7752728329a28f93afec7c4ec9015fc1c6e4460bd1eb0f3737e1c55a"}, - {file = "grpcio_tools-1.51.1-cp311-cp311-win32.whl", hash = "sha256:67b304282cad38642587ebae68617e450e1ad4fa1c0c8b19e9e30274dbb32716"}, - {file = "grpcio_tools-1.51.1-cp311-cp311-win_amd64.whl", hash = "sha256:674b340f2f7bb2adbc3f15144bd37ce5ea83239f78b68dbbd0ea3cba00107e2b"}, - {file = "grpcio_tools-1.51.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:055819992ddd30c642a7fd6f344a03747be3afa95cb910f8a2e5efaabd41cde5"}, - {file = "grpcio_tools-1.51.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:4e3249a2ec435b3b972610c66c8a714c188844500d564c910f57a2771dc61978"}, - {file = "grpcio_tools-1.51.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:794f26a09b70f4f101df5cf54c6c12dc1b65747ab1dee5bda02c2991389ade56"}, - {file = "grpcio_tools-1.51.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4957f1ffa16598aa5379505fcbaeb47d65693a46b0817f4ee61db76707092aeb"}, - {file = "grpcio_tools-1.51.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9906fb6bf6d9c30c23d85153f12d130f44325afe8f9ebe58aa7a6c82ecade9d8"}, - {file = "grpcio_tools-1.51.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87bc5f3e3698c65907d397003c64d25c3ea84e3d6aa46dac133bd98bf66835ee"}, - {file = "grpcio_tools-1.51.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a66b3a5d18a7615f0f828b72e2d2935751459c89cc4725e56bdfb3d2cd93281f"}, - {file = "grpcio_tools-1.51.1-cp37-cp37m-win32.whl", hash = "sha256:566809d9942e78821b279af70f3cf159a328127f9f3d5fee8d83ad8b2d27b2fe"}, - {file = "grpcio_tools-1.51.1-cp37-cp37m-win_amd64.whl", hash = "sha256:aab24a342642329de38139cb26f8492882ca0d8551bb87f6530bcc613945a0d0"}, - {file = "grpcio_tools-1.51.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:6b83d7fc2597c6d392c225177d1fbbcff74900f8cc40b33236987fd1ff841330"}, - {file = "grpcio_tools-1.51.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:79c06d2577cb4d977922bbf01234de3b20f73d1784d3cbe3179deee1bdb9a60b"}, - {file = "grpcio_tools-1.51.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:e9abc03d67793b1bf33dc766caa69a3333f9db029869ba6e8fc6cd9c251c0080"}, - {file = "grpcio_tools-1.51.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64d8ad369417759f5fdb8ffb7cbd6374fecc06ab51c9a226dee9bbd7d311c3b5"}, - {file = "grpcio_tools-1.51.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de51a0a71845b854f6a5967756c893c96bd03e37f39e5dce87b4f409dac36ee2"}, - {file = "grpcio_tools-1.51.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9dfe6c12b0e2c07f6a4a91a9912ef4e5bd007672533891a44e6f433ffbf7c3b1"}, - {file = "grpcio_tools-1.51.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:27113b354f7587684eb55125733e6e5be1f489458abfe12344dabd918d8dcc54"}, - {file = "grpcio_tools-1.51.1-cp38-cp38-win32.whl", hash = "sha256:98777b5031f1b3c58b688815ffa83435c103b2152c26eb144f80f4a4bb34addb"}, - {file = "grpcio_tools-1.51.1-cp38-cp38-win_amd64.whl", hash = "sha256:1c44b57a6770b78a1eafe355878ff1ec59a2fa07455a2cbd522c071eedae04d4"}, - {file = "grpcio_tools-1.51.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:49624394805568acd7d767dea5a00d970fca5ad8f395fe0161eeea0de5133eba"}, - {file = "grpcio_tools-1.51.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:6d6626a6e4dbe843df96dc8c08dd244d2191a75324f54bfa4ebaa3e76b0b1958"}, - {file = "grpcio_tools-1.51.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:b4fb8ed6d29f2d6cf03ef99ffaad635bbc132a59be77013691392fe557e67144"}, - {file = "grpcio_tools-1.51.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8cc862a1ad30f94528d66cc6f95fb9e659005e568313e54a23550535b649573"}, - {file = "grpcio_tools-1.51.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e72a30be1746ea0749a8486d0ca0120c0b2757fe84fc246a5144b1ef66d7b89"}, - {file = "grpcio_tools-1.51.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:331a897306adeec3c67470431ea8d8b4972b689d32966f94506d91f4dac20952"}, - {file = "grpcio_tools-1.51.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f336ad9be661d92fa45940e74e8ff3d78e67ebe9b4f7ea8774b2d680c17aeb6c"}, - {file = "grpcio_tools-1.51.1-cp39-cp39-win32.whl", hash = "sha256:40ef70e8c5d0310dedff9af502b520b4c7e215bce94094527fb959150a0c594a"}, - {file = "grpcio_tools-1.51.1-cp39-cp39-win_amd64.whl", hash = "sha256:15b8acf4eaa0ebe37e2f69108de49efd935b7abe9c7e58ba737490b99906aa76"}, + {file = "grpcio-tools-1.51.3.tar.gz", hash = "sha256:4fea28e3dd31871579a57058796a78093c75b74b74e9de2f2b7a7fd9a443d403"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:779ac1ad2258b8debaa45595bfb3814806ed8880e3ea7f194e551d76a6255969"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:83bf605fe2b3591d3c8a78646f37c72c5832c4dd84b5f92405c17cb10b136be6"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:35f885c5afd8e6a77d320f5a9624b439a93f9be2b87fa7b7948c1ad7b2ba0894"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:253b639fb79a4d28ce494ae40e5695bf1e2cb4a05f205fc433c46b2049ab4d99"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c6b145587d6062e2335f0b3286501dd6853a1ea50bd466a913351b7c48e5f20"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:046c0b1e372d4acf552aa0c8f5e830f019d67b75f25aeb0968d15fbdd3eaabd3"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc90b0287908c46281eb61933acaa1b96a575d0160fc98b5c64b9dec46f60d1"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-win32.whl", hash = "sha256:8e9df40db7a0edd403b539cc142d6114270e35debf723a5b4a7a93d5c30fffc0"}, + {file = "grpcio_tools-1.51.3-cp310-cp310-win_amd64.whl", hash = "sha256:077adaee431c2b040dd77923964577087c32e828908e8fa2e53f8e003ad408c9"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:b50f9b8a6482a90c1a41e731a879a130f7dea267065d0a06f47c9160ce5d01c3"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:89a68adcb4238aba69f3a364ac02c9a46e55b9e3fd8af1c6f384079abfa9347c"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d177da43e7f6fde6715df4a3015ae13158166bc2845ac7f9cfb526eafb41b8"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:793f9edef82f600a3324f8a3d8cd8318a8d02f28fb54f8236cbb35ce0928d186"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f7583735542ced7d30baec6cc21bffeaffcec1523bf807e8f8f0047113b6d30a"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f2df233a3e7db23d9b516cb5e2bfe029465f40a72978bee0584e44e7860ea73f"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-win32.whl", hash = "sha256:7427939455735fbf2ea88c37f1585c9c8b809eec7b447642f34465eb4d26020b"}, + {file = "grpcio_tools-1.51.3-cp311-cp311-win_amd64.whl", hash = "sha256:ba76d15fd149b575170fa32a1f6a9ff2b38ff9db223229a8ad6f53450a452688"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d2212c682529263b3c9e903092d0ccbb9fc6afba820e4c2fa52c2c27720cdcae"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:405656b3cf9639427e6c30a795570cba4a7c06b88a3145866f7d2c05b7e048b4"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:3c445a064b2ef3d3475e26e2add8ddb4ac2933741ecddf71d5b071a3ad078db4"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7b3374f4a6579c58d16a5fab2e6b4e9bb8625a034a7f4cd6024f4d1cc12f2a0"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e8df08b65f9379c3f103147b29542b0141ca84e77d0eee9114ca5f9b3f0d23"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2fade12de08923b350475ca16d0d0bd68578c30fce89147aa0f94ef5759bc5a9"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d4ffb6325ed489065dbdca764cf37c3a29376bc657874116c9af788d7a0d2ee4"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-win32.whl", hash = "sha256:f8d17271fc58ed3503dd571c79917e126deca51f85f093770a9606e806aac9dc"}, + {file = "grpcio_tools-1.51.3-cp37-cp37m-win_amd64.whl", hash = "sha256:ef849687c7f2bd7f3277edc7c7cafc7042823d0fb078e3c01c861eb0c96ed181"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:7fd18d8d211fbfd337fc12e5bdd57e62368f636addf901d290e68a39f1dfea38"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:233fc56f054424232e2086f444004413e33c699174ce6ee0e279c25227243fec"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:867fa1973fa8b0772077c15425f122f672a18b1c53709a8a2bff9d056db4c20e"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b486a99bdf2722e68a9d59769389e2fb86878b6f293be5111f7678e364a0c359"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8bbf412c357999f88d87f421fd48b4b114fc037fec7bbaed0cb7620c24a5e44"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1166744c40821bb0aa605d2af2287fac367756f858a3d18f4c3d25bc0b92757b"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:781896c488e07b9463196045e6725e52d018cd7d0e1062d4ab1eee2647ca9170"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-win32.whl", hash = "sha256:35c1ee7c766eb586f04ba41fa7711eb847767eb277a1737998374ac57768f1f0"}, + {file = "grpcio_tools-1.51.3-cp38-cp38-win_amd64.whl", hash = "sha256:584b201fb39307dcb1affcf2647656a0e6244423ef1659cc6caa3ff85c5ae5c1"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:e02231e21029f716a1d23a0b5e664fa243d147da33a3f55088a9529b860aa4ac"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:fbb742e10bd548031b8d80f7c28eb70c7c3a9850f8e99c98cd496f19a05f9fee"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a836a72c657f751244cdb358c3461a89627e6d02654079d2450cfe361800428c"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb554408e0ec5ff5201013f268726d9eef8e5bd1fd4b4e09c46c0b4a9de8b64c"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:158c5bfe7e157fd9a944bde9f7dfe3b468416666e4fade77cd17caa3edc8bd81"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:715c792679224171c0584e9f235b921d76f8990deb38b0d1215d0469301d9cd9"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ece44f42b10e0bceb49235be1e361e1ee69afee7f938c82fb656a601a4a720e3"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-win32.whl", hash = "sha256:980e632710ba05e04364c6f276e905d5d367437f1ce2265ce7b96b5c1eac5693"}, + {file = "grpcio_tools-1.51.3-cp39-cp39-win_amd64.whl", hash = "sha256:5f4c47b14e66f80365cd5667ecc2f7fb0eb91e02c4e54362041b758feaa00511"}, ] hf-transfer = [ - {file = "hf_transfer-0.1.0-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:0f41bb04898d041b774220048f237d10560ec27e1decd01a04d323c64202e8fe"}, - {file = "hf_transfer-0.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94510d4e3a66aa5afa06b61ff537895c3f1b93d689575a8a840ea9ec3189d3d8"}, - {file = "hf_transfer-0.1.0-cp310-none-win_amd64.whl", hash = "sha256:e85134084c7e9e9daa74331c4690f821a30afedacb97355d1a66c243317c3e7a"}, - {file = "hf_transfer-0.1.0-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:3441b0cba24afad7fffbcfc0eb7e31d3df127d092feeee73e5b08bb5752c903b"}, - {file = "hf_transfer-0.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14938d44c71a07b452612a90499b5f021b2277e1c93c66f60d06d746b2d0661d"}, - {file = "hf_transfer-0.1.0-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:ae01b9844995622beee0f1b7ff0240e269bfc28ea46149eb4abbf63b4683f3e2"}, - {file = "hf_transfer-0.1.0-cp311-none-win_amd64.whl", hash = "sha256:79e9505bffd3a1086be13033a805c8e6f4bb763de03a4197b959984def587e7f"}, - {file = "hf_transfer-0.1.0-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:5656deb183e271d37925de0d6989d7a4b1eefae42d771f10907f41fce08bdada"}, - {file = "hf_transfer-0.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcb16ec07f1ad343b7189745b6f659b7f82b864a55d3b84fe34361f64e4abc76"}, - {file = "hf_transfer-0.1.0-cp37-none-win_amd64.whl", hash = "sha256:5314b708bc2a8cf844885d350cd13ba0b528466d3eb9766e4d8d39d080e718c0"}, - {file = "hf_transfer-0.1.0-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:fe245d0d84bbc113870144c56b50425f9b6cacc3e361b3559b0786ac076ba260"}, - {file = "hf_transfer-0.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49099d41c05a19202dca0306bfa7f42cedaea57ccc783641b1533de860b6a1f4"}, - {file = "hf_transfer-0.1.0-cp38-none-win_amd64.whl", hash = "sha256:0d7bb607a7372908ffa2d55f1e6790430c5706434c2d1d664db4928730c2c7e4"}, - {file = "hf_transfer-0.1.0-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:70f40927bc4a19ab50605bb542bd3858eb465ad65c94cfcaf36cf36d68fc5169"}, - {file = "hf_transfer-0.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d92203155f451a9b517267d2a0966b282615037e286ff0420a2963f67d451de3"}, - {file = "hf_transfer-0.1.0-cp39-none-win_amd64.whl", hash = "sha256:c1c2799154e4bd03d2b2e2907d494005f707686b80d5aa4421c859ffa612ffa3"}, - {file = "hf_transfer-0.1.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0900bd5698a77fb44c95639eb3ec97202d13e1bd4282cde5c81ed48e3f9341e"}, - {file = "hf_transfer-0.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:427032df4e83a1bedaa76383a5b825cf779fdfc206681c28b21476bc84089280"}, - {file = "hf_transfer-0.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e36fe371af6e31795621ffab53a2dcb107088fcfeb2951eaa2a10bfdc8b8863b"}, - {file = "hf_transfer-0.1.0.tar.gz", hash = "sha256:f692ef717ded50e441b4d40b6aea625772f68b90414aeef86bb33eab40cb09a4"}, + {file = "hf_transfer-0.1.2-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:2b9189a4a460646ee135ee771f39c0f695d3d5bf08b7ff1dcfe374227520e994"}, + {file = "hf_transfer-0.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:654fcaba4e7084caa1e97430982ea968935a72916ee0f4afc60e356f89774099"}, + {file = "hf_transfer-0.1.2-cp310-none-win_amd64.whl", hash = "sha256:eb29e7b3707b5cac02e689c89111685ebcdaa3cebba02eb7ac1b0f076357da72"}, + {file = "hf_transfer-0.1.2-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:0bfca9bd84e925e978a0f157df488704c17a0b9ad240b2859262faba0c74cd40"}, + {file = "hf_transfer-0.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d00c5473b35227b2f113fd43ff13cbac9539f2e6779fa0680a887b0aac31c389"}, + {file = "hf_transfer-0.1.2-cp311-none-win_amd64.whl", hash = "sha256:1aaf5937aa433b7d09ce5bf60967ec22b7d3982957b00516a8dc2aaa66384372"}, + {file = "hf_transfer-0.1.2-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:b0aa760a55995ad59ea17e395babafdc56c4e664be0c2d2055664199dd913da1"}, + {file = "hf_transfer-0.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:889dd15e8472daf66e266eb056e31a485af3c35f95a483bb43489a0f6e44c359"}, + {file = "hf_transfer-0.1.2-cp37-none-win_amd64.whl", hash = "sha256:30df586e18ec8a8e67e3201b9038210d94cb3c03c1cbd97673b9c78ede227178"}, + {file = "hf_transfer-0.1.2-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:cc97eb97f929f96bed896cd3af9bbdf121c15ac6d63524b9fc9312fd2929099a"}, + {file = "hf_transfer-0.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:583c2c80210a60dafed9a81ba50c389878aee6c34b2dd375cd84522658f29ad8"}, + {file = "hf_transfer-0.1.2-cp38-none-win_amd64.whl", hash = "sha256:6dff58f50d1435b0346f31a32f1f9e2301986521c1d0b51e47a3c82b96d02156"}, + {file = "hf_transfer-0.1.2-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d6db1a8f539133f7a893bb32721916fe72b4d2aa3eb7604581ba1f03b8167c90"}, + {file = "hf_transfer-0.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f284e3f775d215c9a8d3d1c6f6b1001b1e7990d73ae5fd9aea6c9bce9ea79285"}, + {file = "hf_transfer-0.1.2-cp39-none-win_amd64.whl", hash = "sha256:8625beabebc582eafc4141a5ecb9f1183b728d4f63767f01fdcf1e2fbafe6d43"}, + {file = "hf_transfer-0.1.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:947dd1b8b22ac10723b2887ed4b5ef929f7d4dd850b0d66c0c6954a9a85afb06"}, + {file = "hf_transfer-0.1.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90a020f41dfae4629186c284888cd5adbebe402e2497a88351416ab93c7df9a8"}, + {file = "hf_transfer-0.1.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5eb89698746a29805bfc60126b9a008e6ba08a82ef9bb122a6544e84f748e8a4"}, + {file = "hf_transfer-0.1.2.tar.gz", hash = "sha256:6bf847f4c19c7d8d9f9bbb8a7ed52e1271bbf0c1bd920357db0c274ccc69f21d"}, ] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, @@ -995,20 +994,19 @@ pluggy = [ {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] protobuf = [ - {file = "protobuf-4.21.12-cp310-abi3-win32.whl", hash = "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1"}, - {file = "protobuf-4.21.12-cp310-abi3-win_amd64.whl", hash = "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2"}, - {file = "protobuf-4.21.12-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791"}, - {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97"}, - {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7"}, - {file = "protobuf-4.21.12-cp37-cp37m-win32.whl", hash = "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717"}, - {file = "protobuf-4.21.12-cp37-cp37m-win_amd64.whl", hash = "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574"}, - {file = "protobuf-4.21.12-cp38-cp38-win32.whl", hash = "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec"}, - {file = "protobuf-4.21.12-cp38-cp38-win_amd64.whl", hash = "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30"}, - {file = "protobuf-4.21.12-cp39-cp39-win32.whl", hash = "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc"}, - {file = "protobuf-4.21.12-cp39-cp39-win_amd64.whl", hash = "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b"}, - {file = "protobuf-4.21.12-py2.py3-none-any.whl", hash = "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5"}, - {file = "protobuf-4.21.12-py3-none-any.whl", hash = "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462"}, - {file = "protobuf-4.21.12.tar.gz", hash = "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab"}, + {file = "protobuf-4.22.0-cp310-abi3-win32.whl", hash = "sha256:b2fea9dc8e3c0f32c38124790ef16cba2ee0628fe2022a52e435e1117bfef9b1"}, + {file = "protobuf-4.22.0-cp310-abi3-win_amd64.whl", hash = "sha256:a33a273d21852f911b8bda47f39f4383fe7c061eb1814db2c76c9875c89c2491"}, + {file = "protobuf-4.22.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:e894e9ae603e963f0842498c4cd5d39c6a60f0d7e4c103df50ee939564298658"}, + {file = "protobuf-4.22.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:7c535d126e7dcc714105ab20b418c4fedbd28f8b8afc42b7350b1e317bbbcc71"}, + {file = "protobuf-4.22.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:86c3d20428b007537ba6792b475c0853bba7f66b1f60e610d913b77d94b486e4"}, + {file = "protobuf-4.22.0-cp37-cp37m-win32.whl", hash = "sha256:1669cb7524221a8e2d9008d0842453dbefdd0fcdd64d67672f657244867635fb"}, + {file = "protobuf-4.22.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ab4d043865dd04e6b09386981fe8f80b39a1e46139fb4a3c206229d6b9f36ff6"}, + {file = "protobuf-4.22.0-cp38-cp38-win32.whl", hash = "sha256:29288813aacaa302afa2381db1d6e0482165737b0afdf2811df5fa99185c457b"}, + {file = "protobuf-4.22.0-cp38-cp38-win_amd64.whl", hash = "sha256:e474b63bab0a2ea32a7b26a4d8eec59e33e709321e5e16fb66e766b61b82a95e"}, + {file = "protobuf-4.22.0-cp39-cp39-win32.whl", hash = "sha256:47d31bdf58222dd296976aa1646c68c6ee80b96d22e0a3c336c9174e253fd35e"}, + {file = "protobuf-4.22.0-cp39-cp39-win_amd64.whl", hash = "sha256:c27f371f0159feb70e6ea52ed7e768b3f3a4c5676c1900a7e51a24740381650e"}, + {file = "protobuf-4.22.0-py3-none-any.whl", hash = "sha256:c3325803095fb4c2a48649c321d2fbde59f8fbfcb9bfc7a86df27d112831c571"}, + {file = "protobuf-4.22.0.tar.gz", hash = "sha256:652d8dfece122a24d98eebfef30e31e455d300efa41999d1182e015984ac5930"}, ] psutil = [ {file = "psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8"}, @@ -1119,8 +1117,8 @@ safetensors = [ {file = "safetensors-0.2.8.tar.gz", hash = "sha256:2720b20a6a38c799dca79bd76caeeac2f7df585a9d4f7d59fa7e28eff9ccb27f"}, ] setuptools = [ - {file = "setuptools-67.2.0-py3-none-any.whl", hash = "sha256:16ccf598aab3b506593c17378473978908a2734d7336755a8769b480906bec1c"}, - {file = "setuptools-67.2.0.tar.gz", hash = "sha256:b440ee5f7e607bb8c9de15259dba2583dd41a38879a7abc1d43a71c59524da48"}, + {file = "setuptools-67.4.0-py3-none-any.whl", hash = "sha256:f106dee1b506dee5102cc3f3e9e68137bbad6d47b616be7991714b0c62204251"}, + {file = "setuptools-67.4.0.tar.gz", hash = "sha256:e5fd0a713141a4a105412233c63dc4e17ba0090c8e8334594ac790ec97792330"}, ] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, @@ -1154,8 +1152,8 @@ typer = [ {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, ] typing-extensions = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, + {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, + {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, ] urllib3 = [ {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, @@ -1170,68 +1168,79 @@ win32-setctime = [ {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, ] wrapt = [ - {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1"}, - {file = "wrapt-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320"}, - {file = "wrapt-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2"}, - {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4"}, - {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069"}, - {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310"}, - {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f"}, - {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656"}, - {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, - {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, - {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d"}, - {file = "wrapt-1.14.1-cp35-cp35m-win32.whl", hash = "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7"}, - {file = "wrapt-1.14.1-cp35-cp35m-win_amd64.whl", hash = "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00"}, - {file = "wrapt-1.14.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4"}, - {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1"}, - {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1"}, - {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff"}, - {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d"}, - {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1"}, - {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569"}, - {file = "wrapt-1.14.1-cp36-cp36m-win32.whl", hash = "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed"}, - {file = "wrapt-1.14.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471"}, - {file = "wrapt-1.14.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248"}, - {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68"}, - {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d"}, - {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77"}, - {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7"}, - {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015"}, - {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a"}, - {file = "wrapt-1.14.1-cp37-cp37m-win32.whl", hash = "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853"}, - {file = "wrapt-1.14.1-cp37-cp37m-win_amd64.whl", hash = "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c"}, - {file = "wrapt-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456"}, - {file = "wrapt-1.14.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f"}, - {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc"}, - {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1"}, - {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af"}, - {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b"}, - {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0"}, - {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57"}, - {file = "wrapt-1.14.1-cp38-cp38-win32.whl", hash = "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5"}, - {file = "wrapt-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d"}, - {file = "wrapt-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383"}, - {file = "wrapt-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7"}, - {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86"}, - {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735"}, - {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b"}, - {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3"}, - {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3"}, - {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe"}, - {file = "wrapt-1.14.1-cp39-cp39-win32.whl", hash = "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5"}, - {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, - {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, + {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, + {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, + {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, + {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, + {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, + {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, + {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, + {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, + {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, + {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, + {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, + {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, + {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, + {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, + {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, + {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, + {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, + {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, + {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] diff --git a/server/pyproject.toml b/server/pyproject.toml index 995dade3..68a40dcc 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -22,7 +22,7 @@ loguru = "^0.6.0" opentelemetry-api = "^1.15.0" opentelemetry-exporter-otlp = "^1.15.0" opentelemetry-instrumentation-grpc = "^0.36b0" -hf-transfer = "^0.1.0" +hf-transfer = "^0.1.2" [tool.poetry.extras] bnb = ["bitsandbytes"] From e3ded361b246275cfd2393215ebb7f66c0c8527a Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 3 Mar 2023 15:07:27 +0100 Subject: [PATCH 099/793] feat(ci): improve CI speed (#94) --- .github/workflows/build.yaml | 8 ++++++-- .github/workflows/tests.yaml | 36 +++++++++++++++++++++++++++++++----- Dockerfile | 36 ++++++++++++++++++------------------ 3 files changed, 55 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8304c8d1..6c752de8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -15,6 +15,10 @@ jobs: build-and-push-image: runs-on: ubuntu-latest steps: + - name: Initialize Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + with: + install: true - name: Tailscale uses: tailscale/github-action@v1 with: @@ -65,5 +69,5 @@ jobs: platforms: 'linux/amd64' tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=ghcr.io/huggingface/text-generation-inference:latest - cache-to: type=inline \ No newline at end of file + cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max + cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8b659fe2..27caff11 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -3,6 +3,7 @@ name: Server Tests on: pull_request: paths: + - ".github/workflows/tests.yaml" - "server/**" - "proto/**" - "router/**" @@ -11,6 +12,12 @@ on: jobs: run_tests: runs-on: ubuntu-20.04 + + env: + SCCACHE_GHA_ENABLED: "on" + RUSTC_WRAPPER: /usr/local/bin/sccache + SCCACHE: 0.3.3 + steps: - uses: actions/checkout@v2 - name: Set up Python @@ -25,12 +32,28 @@ jobs: components: rustfmt, clippy - name: Install Protoc uses: arduino/setup-protoc@v1 - - name: Loading cache. - uses: actions/cache@v2 - id: model_cache + - name: Install sccache + run: | + curl -fsSL https://github.com/mozilla/sccache/releases/download/v$SCCACHE/sccache-v$SCCACHE-x86_64-unknown-linux-musl.tar.gz | tar -xzv --strip-components=1 -C /usr/local/bin sccache-v$SCCACHE-x86_64-unknown-linux-musl/sccache + chmod +x /usr/local/bin/sccache + - name: configure sccache + uses: actions/github-script@v6 with: - path: ~/.cache/huggingface/ - key: models + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + core.exportVariable('SCCACHE_GHA_CACHE_TO', 'sccache-${{runner.os}}-${{github.ref_name}}'); + core.exportVariable('SCCACHE_GHA_CACHE_FROM', 'sccache-${{runner.os}}-main,sccache-${{runner.os}}-'); + - name: cargo registry cache + uses: actions/cache@v3 + with: + key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-${{ github.sha }} + restore-keys: | + cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}- + cargo-${{ runner.os }}- + path: | + ~/.cargo/registry + ~/.cargo/git - name: Install run: | make install @@ -41,3 +64,6 @@ jobs: - name: Run Rust tests run: | cargo test + - name: sccache stats + run: | + /usr/local/bin/sccache --show-stats diff --git a/Dockerfile b/Dockerfile index 228909dd..5fbf8985 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,15 @@ -FROM rust:1.67 as router-builder +FROM lukemathwalker/cargo-chef:latest-rust-1.67 AS chef +WORKDIR /usr/src + +FROM chef as planner +COPY Cargo.toml Cargo.toml +COPY rust-toolchain.toml rust-toolchain.toml +COPY proto proto +COPY router router +COPY launcher launcher +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \ curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \ @@ -6,26 +17,15 @@ RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \ unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \ rm -f $PROTOC_ZIP -WORKDIR /usr/src +COPY --from=planner /usr/src/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json +COPY Cargo.toml Cargo.toml COPY rust-toolchain.toml rust-toolchain.toml COPY proto proto COPY router router - -WORKDIR /usr/src/router - -RUN cargo install --path . - -FROM rust:1.67 as launcher-builder - -WORKDIR /usr/src - -COPY rust-toolchain.toml rust-toolchain.toml COPY launcher launcher - -WORKDIR /usr/src/launcher - -RUN cargo install --path . +RUN cargo build --release FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 @@ -69,9 +69,9 @@ RUN cd server && \ /opt/miniconda/envs/text-generation/bin/pip install ".[bnb]" --no-cache-dir # Install router -COPY --from=router-builder /usr/local/cargo/bin/text-generation-router /usr/local/bin/text-generation-router +COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bin/text-generation-router # Install launcher -COPY --from=launcher-builder /usr/local/cargo/bin/text-generation-launcher /usr/local/bin/text-generation-launcher +COPY --from=builder /usr/src/target/release/text-generation-launcher /usr/local/bin/text-generation-launcher ENTRYPOINT ["text-generation-launcher"] CMD ["--json-output"] \ No newline at end of file From 240c4187fd8b333e0173db7eb546dc09a6a659b6 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 3 Mar 2023 16:01:25 +0100 Subject: [PATCH 100/793] fix(launcher): add router parameters to launcher (#95) --- launcher/src/main.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index ca1f6738..7a32ad2c 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -29,8 +29,12 @@ struct Args { quantize: bool, #[clap(default_value = "128", long, env)] max_concurrent_requests: usize, + #[clap(default_value = "4", long, env)] + max_stop_sequences: usize, #[clap(default_value = "1000", long, env)] max_input_length: usize, + #[clap(default_value = "1512", long, env)] + max_total_tokens: usize, #[clap(default_value = "32", long, env)] max_batch_size: usize, #[clap(default_value = "20", long, env)] @@ -79,7 +83,9 @@ fn main() -> ExitCode { num_shard, quantize, max_concurrent_requests, + max_stop_sequences, max_input_length, + max_total_tokens, max_batch_size, max_waiting_tokens, port, @@ -299,8 +305,12 @@ fn main() -> ExitCode { "text-generation-router".to_string(), "--max-concurrent-requests".to_string(), max_concurrent_requests.to_string(), + "--max-stop-sequences".to_string(), + max_stop_sequences.to_string(), "--max-input-length".to_string(), max_input_length.to_string(), + "--max-total-tokens".to_string(), + max_total_tokens.to_string(), "--max-batch-size".to_string(), max_batch_size.to_string(), "--max-waiting-tokens".to_string(), From 0b6807caa4c9d555aca65ccc57319cf47cf4920e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 3 Mar 2023 17:56:27 +0100 Subject: [PATCH 101/793] feat(server): fix transformers commit (#96) --- .github/workflows/tests.yaml | 2 +- Makefile | 2 +- server/Makefile | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 27caff11..476ae443 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -60,7 +60,7 @@ jobs: - name: Run server tests run: | pip install pytest - pytest -sv server/tests + HF_HUB_ENABLE_HF_TRANSFER=1 pytest -sv server/tests - name: Run Rust tests run: | cargo test diff --git a/Makefile b/Makefile index 08217730..cf7b8675 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ integration-tests: install-router install-launcher cargo test python-tests: - cd server && pytest tests + cd server && HF_HUB_ENABLE_HF_TRANSFER=1 pytest tests run-bloom-560m: text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 diff --git a/server/Makefile b/server/Makefile index 74ce5144..4a77dbcf 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,3 +1,5 @@ +transformers_commit := 712d62e83c28236c7f39af690e7792a54288dbd9 + gen-server: # Compile protos pip install grpcio-tools==1.51.1 --no-cache-dir @@ -10,11 +12,11 @@ install-transformers: # Install specific version of transformers with custom cuda kernels pip uninstall transformers -y || true rm -rf transformers || true - rm -rf transformers-text_generation_inference || true - curl -L -O https://github.com/OlivierDehaene/transformers/archive/refs/heads/text_generation_inference.zip - unzip text_generation_inference.zip - rm text_generation_inference.zip - mv transformers-text_generation_inference transformers + rm -rf transformers-$(transformers_commit) || true + curl -L -O https://github.com/OlivierDehaene/transformers/archive/$(transformers_commit).zip + unzip $(transformers_commit).zip + rm $(transformers_commit).zip + mv transformers-$(transformers_commit) transformers cd transformers && python setup.py install install-torch: From 1c19b0934e281f6a09d5adf393f92525b4070918 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 3 Mar 2023 18:42:20 +0100 Subject: [PATCH 102/793] v0.3.2 (#97) --- Cargo.lock | 8 ++++---- Makefile | 2 +- README.md | 31 +++++++++++++++++++++++++++++-- docs/openapi.json | 12 +++++++++++- launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- router/grpc-metadata/Cargo.toml | 2 +- server/Makefile | 2 +- server/pyproject.toml | 2 +- 10 files changed, 51 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aac1114e..0a358b2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -786,7 +786,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "grpc-metadata" -version = "0.3.1" +version = "0.3.2" dependencies = [ "opentelemetry", "tonic", @@ -2212,7 +2212,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.3.1" +version = "0.3.2" dependencies = [ "futures", "grpc-metadata", @@ -2229,7 +2229,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.3.1" +version = "0.3.2" dependencies = [ "clap 4.1.8", "ctrlc", @@ -2244,7 +2244,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.3.1" +version = "0.3.2" dependencies = [ "async-stream", "axum", diff --git a/Makefile b/Makefile index cf7b8675..ade0cdd5 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ run-bloom-560m-quantize: text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --quantize download-bloom: - text-generation-server download-weights bigscience/bloom + HF_HUB_ENABLE_HF_TRANSFER=1 text-generation-server download-weights bigscience/bloom run-bloom: text-generation-launcher --model-id bigscience/bloom --num-shard 8 diff --git a/README.md b/README.md index 5234844b..ae185506 100644 --- a/README.md +++ b/README.md @@ -39,15 +39,17 @@ to power LLMs api-inference widgets. ## Features +- Serve the most popular Large Language Models with a simple launcher +- Tensor Parallelism for faster inference on multiple GPUs - Token streaming using Server-Sent Events (SSE) - [Dynamic batching of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading -- 45ms per token generation for BLOOM with 8xA100 80GB +- Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) - Logits warpers (temperature scaling, topk, repetition penalty ...) - Stop sequences - Log probabilities -- Distributed tracing with Open Telemetry +- Production ready (distributed tracing with Open Telemetry, Prometheus metrics) ## Officially supported architectures @@ -58,6 +60,7 @@ to power LLMs api-inference widgets. - [SantaCoder](https://huggingface.co/bigcode/santacoder) - [GPT-Neox 20B](https://huggingface.co/EleutherAI/gpt-neox-20b) - [FLAN-T5-XXL](https://huggingface.co/google/flan-t5-xxl) +- [FLAN-UL2](https://huggingface.co/google/flan-ul2) Other architectures are supported on a best effort basis using: @@ -97,6 +100,30 @@ curl 127.0.0.1:8080/generate_stream \ -H 'Content-Type: application/json' ``` +or from Python: + +```python +import requests + +result = requests.post("http://127.0.0.1:8080/generate", json={"inputs":"Testing API","parameters":{"max_new_tokens":9}}) +print(result.json()) +``` + +```shell +pip install sseclient-py +``` + +````python +import sseclient +import requests + +r = requests.post("http://127.0.0.1:8080/generate_stream", stream=True, json={"inputs":"Testing API","parameters":{"max_new_tokens":9}}) +sse_client = sseclient.SSEClient(r) + +for i, event in enumerate(sse_client.events()): + print(i, event.data) +```` + **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). ### API documentation diff --git a/docs/openapi.json b/docs/openapi.json index 969a2575..7ece1722 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -11,7 +11,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.3.1" + "version": "0.3.2" }, "paths": { "/generate": { @@ -290,6 +290,11 @@ "nullable": true, "exclusiveMinimum": 0.0 }, + "return_full_text": { + "type": "boolean", + "default": "None", + "example": false + }, "seed": { "type": "integer", "format": "int64" @@ -328,6 +333,11 @@ "nullable": true, "maximum": 1.0, "exclusiveMinimum": 0.0 + }, + "watermark": { + "type": "boolean", + "default": "false", + "example": true } } }, diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index d53a2c4a..4779aa91 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.3.1" +version = "0.3.2" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index 6e673af0..2da8a9a0 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.3.1" +version = "0.3.2" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index b1700328..960e509f 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.3.1" +version = "0.3.2" edition = "2021" [dependencies] diff --git a/router/grpc-metadata/Cargo.toml b/router/grpc-metadata/Cargo.toml index 246023b5..e779fbbf 100644 --- a/router/grpc-metadata/Cargo.toml +++ b/router/grpc-metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grpc-metadata" -version = "0.3.1" +version = "0.3.2" edition = "2021" [dependencies] diff --git a/server/Makefile b/server/Makefile index 4a77dbcf..3926cf5d 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,4 +1,4 @@ -transformers_commit := 712d62e83c28236c7f39af690e7792a54288dbd9 +transformers_commit := 2f87dca1ca3e5663d0637da9bb037a6956e57a5e gen-server: # Compile protos diff --git a/server/pyproject.toml b/server/pyproject.toml index 68a40dcc..21277939 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.3.1" +version = "0.3.2" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 9b205d33cc349a96937204ad16ebc1a578ad619b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 6 Mar 2023 13:22:58 +0100 Subject: [PATCH 103/793] fix(server): fix generate_stream by forcing tokens to be decoded correctly (#100) --- launcher/tests/mt0_base.json | 22 ++++++++++----------- server/tests/models/test_seq2seq_lm.py | 2 +- server/text_generation/models/causal_lm.py | 4 +--- server/text_generation/models/model.py | 18 +++++++++++++++++ server/text_generation/models/seq2seq_lm.py | 8 ++++---- server/text_generation/utils/watermark.py | 20 +++++++++---------- 6 files changed, 45 insertions(+), 29 deletions(-) diff --git a/launcher/tests/mt0_base.json b/launcher/tests/mt0_base.json index c06a2c26..22c9499f 100644 --- a/launcher/tests/mt0_base.json +++ b/launcher/tests/mt0_base.json @@ -14,7 +14,7 @@ "tokens": [ { "id": 259, - "text": "", + "text": " ", "logprob": -1.3656927, "special": false }, @@ -32,13 +32,13 @@ }, { "id": 287, - "text": "the", + "text": " the", "logprob": -1.2102449, "special": false }, { "id": 259, - "text": "", + "text": " ", "logprob": -1.6057279, "special": false }, @@ -50,19 +50,19 @@ }, { "id": 304, - "text": "of", + "text": " of", "logprob": -0.5270343, "special": false }, { "id": 287, - "text": "the", + "text": " the", "logprob": -0.62522805, "special": false }, { "id": 259, - "text": "", + "text": " ", "logprob": -1.4069618, "special": false }, @@ -74,19 +74,19 @@ }, { "id": 304, - "text": "of", + "text": " of", "logprob": -1.3172221, "special": false }, { "id": 287, - "text": "the", + "text": " the", "logprob": -0.3501925, "special": false }, { "id": 259, - "text": "", + "text": " ", "logprob": -0.7219573, "special": false }, @@ -104,7 +104,7 @@ }, { "id": 259, - "text": "", + "text": " ", "logprob": -0.32933083, "special": false }, @@ -116,7 +116,7 @@ }, { "id": 2978, - "text": "test", + "text": " test", "logprob": -1.5846587, "special": false }, diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index c6eacba7..f7173392 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -148,7 +148,7 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) assert all([generation.generated_text is None for generation in generations]) assert all([len(generation.prefill_tokens) == 1 for generation in generations]) assert all([generation.token_id.item() == 259 for generation in generations]) - assert all([generation.token_text == "" for generation in generations]) + assert all([generation.token_text == " " for generation in generations]) assert generations[0].request_id == 0 diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation/models/causal_lm.py index 36d419e3..23c94ddf 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation/models/causal_lm.py @@ -385,10 +385,8 @@ def generate_token( # Generated token next_token_logprob = logprobs[-1, next_token_id] next_token_id_squeezed = next_token_id.squeeze() - next_token_text = self.tokenizer.decode( + next_token_text = self.decode_token( next_token_id_squeezed, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, ) # Evaluate stopping criteria diff --git a/server/text_generation/models/model.py b/server/text_generation/models/model.py index 52480526..09fa6a2a 100644 --- a/server/text_generation/models/model.py +++ b/server/text_generation/models/model.py @@ -15,6 +15,15 @@ def __init__(self, tokenizer: PreTrainedTokenizerBase, device: torch.device): self.all_special_ids = set(tokenizer.all_special_ids) self.device = device + # see `decode_token` method + self.tokenizer.add_special_tokens( + {"additional_special_tokens": [""]} + ) + self.special_decode_token_id = self.tokenizer.convert_tokens_to_ids( + "" + ) + self.special_decode_token_length = len("") + @property @abstractmethod def batch_type(self) -> Type[B]: @@ -23,3 +32,12 @@ def batch_type(self) -> Type[B]: @abstractmethod def generate_token(self, batch: B) -> Tuple[List[GeneratedText], Optional[B]]: raise NotImplementedError + + def decode_token(self, token_id: int) -> str: + """Hack to hopefully support generate_stream for the maximum number of tokenizers""" + # append token to special decode token and decode both + result = self.tokenizer.decode( + [self.special_decode_token_id, token_id], skip_special_tokens=False + ) + # slice to remove special decode token + return result[self.special_decode_token_length :] diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation/models/seq2seq_lm.py index 38089967..4b88baec 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation/models/seq2seq_lm.py @@ -342,7 +342,9 @@ def batch_type(self) -> Type[Seq2SeqLMBatch]: return Seq2SeqLMBatch def decode(self, decoder_ids: List[int]) -> str: - return self.tokenizer.decode(decoder_ids, skip_special_tokens=True) + return self.tokenizer.decode( + decoder_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False + ) def forward( self, @@ -457,10 +459,8 @@ def generate_token( # Generated token next_token_logprob = logprobs[-1, next_token_id] next_token_id_squeezed = next_token_id.squeeze() - next_token_text = self.tokenizer.decode( + next_token_text = self.decode_token( next_token_id_squeezed, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, ) # Evaluate stopping criteria diff --git a/server/text_generation/utils/watermark.py b/server/text_generation/utils/watermark.py index 6f5664fe..4c42d2a1 100644 --- a/server/text_generation/utils/watermark.py +++ b/server/text_generation/utils/watermark.py @@ -24,12 +24,12 @@ class WatermarkLogitsProcessor(LogitsProcessor): def __init__( - self, - vocab_size: int, - gamma: float = GAMMA, - delta: float = DELTA, - hash_key: int = 15485863, # just a large prime number to create a rng seed with sufficient bit width - device: str = "cpu", + self, + vocab_size: int, + gamma: float = GAMMA, + delta: float = DELTA, + hash_key: int = 15485863, # just a large prime number to create a rng seed with sufficient bit width + device: str = "cpu", ): # watermarking parameters self.vocab_size = vocab_size @@ -40,7 +40,7 @@ def __init__( def _seed_rng(self, input_ids: torch.LongTensor) -> None: assert ( - input_ids.shape[-1] >= 1 + input_ids.shape[-1] >= 1 ), "requires at least a 1 token prefix sequence to seed rng" prev_token = input_ids[-1].item() self.rng.manual_seed(self.hash_key * prev_token) @@ -58,7 +58,7 @@ def _get_greenlist_ids(self, input_ids: torch.LongTensor) -> list[int]: @staticmethod def _calc_greenlist_mask( - scores: torch.FloatTensor, greenlist_token_ids + scores: torch.FloatTensor, greenlist_token_ids ) -> torch.BoolTensor: green_tokens_mask = torch.zeros_like(scores) green_tokens_mask[-1, greenlist_token_ids] = 1 @@ -67,13 +67,13 @@ def _calc_greenlist_mask( @staticmethod def _bias_greenlist_logits( - scores: torch.Tensor, greenlist_mask: torch.Tensor, greenlist_bias: float + scores: torch.Tensor, greenlist_mask: torch.Tensor, greenlist_bias: float ) -> torch.Tensor: scores[greenlist_mask] = scores[greenlist_mask] + greenlist_bias return scores def __call__( - self, input_ids: torch.LongTensor, scores: torch.FloatTensor + self, input_ids: torch.LongTensor, scores: torch.FloatTensor ) -> torch.FloatTensor: assert len(input_ids) == 1 greenlist_ids = self._get_greenlist_ids(input_ids[0]) From cd5961b5dad560d63f4dd42d08d6ee3877b82003 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 6 Mar 2023 14:39:36 +0100 Subject: [PATCH 104/793] feat: allow local models (#101) closes #99 --- launcher/src/main.rs | 6 +++++- router/src/main.rs | 17 +++++++++++++---- router/src/queue.rs | 2 +- server/text_generation/models/__init__.py | 2 +- server/text_generation/models/bloom.py | 3 --- server/text_generation/models/galactica.py | 3 --- server/text_generation/utils/hub.py | 4 ++++ 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 7a32ad2c..66dcb2db 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -110,8 +110,12 @@ fn main() -> ExitCode { }) .expect("Error setting Ctrl-C handler"); + // Check if model_id is a local model + let local_path = Path::new(&model_id); + let is_local_model = local_path.exists() && local_path.is_dir(); + // Download weights for sharded models - if weights_cache_override.is_none() && num_shard > 1 { + if !is_local_model && weights_cache_override.is_none() && num_shard > 1 { let mut download_argv = vec![ "text-generation-server".to_string(), "download-weights".to_string(), diff --git a/router/src/main.rs b/router/src/main.rs index 2baf9e72..a51d3168 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -8,6 +8,7 @@ use opentelemetry::sdk::Resource; use opentelemetry::{global, KeyValue}; use opentelemetry_otlp::WithExportConfig; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::path::Path; use text_generation_client::ShardedClient; use text_generation_router::server; use tokenizers::Tokenizer; @@ -83,11 +84,19 @@ fn main() -> Result<(), std::io::Error> { ) }); - // Download and instantiate tokenizer + // Tokenizer instance // This will only be used to validate payloads - // - // We need to download it outside of the Tokio runtime - let tokenizer = Tokenizer::from_pretrained(tokenizer_name.clone(), None).unwrap(); + let local_path = Path::new(&tokenizer_name); + let tokenizer = + if local_path.exists() && local_path.is_dir() && local_path.join("tokenizer.json").exists() + { + // Load local tokenizer + Tokenizer::from_file(local_path.join("tokenizer.json")).unwrap() + } else { + // Download and instantiate tokenizer + // We need to download it outside of the Tokio runtime + Tokenizer::from_pretrained(tokenizer_name.clone(), None).unwrap() + }; // Launch Tokio runtime tokio::runtime::Builder::new_multi_thread() diff --git a/router/src/queue.rs b/router/src/queue.rs index 088bdd3c..0ebfed9b 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -234,7 +234,7 @@ mod tests { do_sample: false, seed: 0, repetition_penalty: 0.0, - watermark: false + watermark: false, }, stopping_parameters: StoppingCriteriaParameters { max_new_tokens: 0, diff --git a/server/text_generation/models/__init__.py b/server/text_generation/models/__init__.py index 908b144c..386b7dc9 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation/models/__init__.py @@ -41,7 +41,7 @@ def get_model( model_id: str, revision: Optional[str], sharded: bool, quantize: bool ) -> Model: - if model_id.startswith("facebook/galactica"): + if "facebook/galactica" in model_id: if sharded: return GalacticaSharded(model_id, revision, quantize=quantize) else: diff --git a/server/text_generation/models/bloom.py b/server/text_generation/models/bloom.py index 08c3ac94..83a0d63e 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation/models/bloom.py @@ -58,9 +58,6 @@ class BLOOMSharded(BLOOM): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - if not model_id.startswith("bigscience/bloom"): - raise ValueError(f"Model {model_id} is not supported") - self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 if torch.cuda.is_available(): diff --git a/server/text_generation/models/galactica.py b/server/text_generation/models/galactica.py index f3a76459..9a71c5d3 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation/models/galactica.py @@ -164,9 +164,6 @@ class GalacticaSharded(Galactica): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - if not model_id.startswith("facebook/galactica"): - raise ValueError(f"Model {model_id} is not supported") - self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 if torch.cuda.is_available(): diff --git a/server/text_generation/utils/hub.py b/server/text_generation/utils/hub.py index 7166b4cb..d338fb29 100644 --- a/server/text_generation/utils/hub.py +++ b/server/text_generation/utils/hub.py @@ -80,6 +80,10 @@ def weight_files( model_id: str, revision: Optional[str] = None, extension: str = ".safetensors" ) -> List[Path]: """Get the local files""" + # Local model + if Path(model_id).exists() and Path(model_id).is_dir(): + return list(Path(model_id).glob(f"*{extension}")) + try: filenames = weight_hub_files(model_id, revision, extension) except EntryNotFoundError as e: From 0e9ed1a8c2ca5023388999c42dde2976e026fe13 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 7 Mar 2023 12:55:05 +0100 Subject: [PATCH 105/793] feat: add supported models (#102) --- supported_models.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 supported_models.json diff --git a/supported_models.json b/supported_models.json new file mode 100644 index 00000000..edf01e70 --- /dev/null +++ b/supported_models.json @@ -0,0 +1,8 @@ +[ + "bigscience/bloom", + "bigscience/bloomz", + "EleutherAI/gpt-neox-20b", + "google/flan-ul2", + "google/flan-t5-xxl", + "olivierdehaene/optimized-santacoder" +] From 3fef90d50fe6248abe320904e51dbf98972fe25b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 7 Mar 2023 18:52:22 +0100 Subject: [PATCH 106/793] feat(clients): Python client (#103) --- Makefile | 10 +- README.md | 31 +- clients/python/.gitignore | 158 +++ clients/python/Makefile | 6 + clients/python/README.md | 52 + clients/python/poetry.lock | 1038 +++++++++++++++++ clients/python/pyproject.toml | 26 + clients/python/tests/conftest.py | 56 + clients/python/tests/test_client.py | 127 ++ clients/python/tests/test_errors.py | 64 + clients/python/tests/test_inference_api.py | 34 + clients/python/tests/test_types.py | 39 + clients/python/text_generation/__init__.py | 18 + clients/python/text_generation/client.py | 415 +++++++ clients/python/text_generation/errors.py | 106 ++ .../python/text_generation/inference_api.py | 150 +++ clients/python/text_generation/types.py | 99 ++ router/src/infer.rs | 11 + router/src/lib.rs | 6 +- router/src/server.rs | 6 +- router/src/validation.rs | 18 +- server/.gitignore | 4 +- server/Makefile | 10 +- server/pyproject.toml | 4 +- server/tests/conftest.py | 2 +- server/tests/models/test_bloom.py | 6 +- server/tests/models/test_causal_lm.py | 4 +- server/tests/models/test_santacoder.py | 6 +- server/tests/models/test_seq2seq_lm.py | 4 +- server/tests/utils/test_convert.py | 8 +- server/tests/utils/test_hub.py | 2 +- server/tests/utils/test_tokens.py | 2 +- .../__init__.py | 0 .../cache.py | 2 +- .../cli.py | 8 +- .../interceptor.py | 0 .../models/__init__.py | 16 +- .../models/bloom.py | 8 +- .../models/causal_lm.py | 13 +- .../models/galactica.py | 8 +- .../models/gpt_neox.py | 4 +- .../models/model.py | 2 +- .../models/santacoder.py | 2 +- .../models/seq2seq_lm.py | 17 +- .../models/t5.py | 4 +- .../models/types.py | 4 +- .../pb/.gitignore | 0 .../server.py | 10 +- .../tracing.py | 0 .../utils/__init__.py | 8 +- .../utils/convert.py | 0 .../utils/dist.py | 0 .../utils/hub.py | 0 .../utils/tokens.py | 6 +- .../utils/watermark.py | 0 55 files changed, 2522 insertions(+), 112 deletions(-) create mode 100644 clients/python/.gitignore create mode 100644 clients/python/Makefile create mode 100644 clients/python/README.md create mode 100644 clients/python/poetry.lock create mode 100644 clients/python/pyproject.toml create mode 100644 clients/python/tests/conftest.py create mode 100644 clients/python/tests/test_client.py create mode 100644 clients/python/tests/test_errors.py create mode 100644 clients/python/tests/test_inference_api.py create mode 100644 clients/python/tests/test_types.py create mode 100644 clients/python/text_generation/__init__.py create mode 100644 clients/python/text_generation/client.py create mode 100644 clients/python/text_generation/errors.py create mode 100644 clients/python/text_generation/inference_api.py create mode 100644 clients/python/text_generation/types.py rename server/{text_generation => text_generation_server}/__init__.py (100%) rename server/{text_generation => text_generation_server}/cache.py (90%) rename server/{text_generation => text_generation_server}/cli.py (94%) rename server/{text_generation => text_generation_server}/interceptor.py (100%) rename server/{text_generation => text_generation_server}/models/__init__.py (79%) rename server/{text_generation => text_generation_server}/models/bloom.py (97%) rename server/{text_generation => text_generation_server}/models/causal_lm.py (98%) rename server/{text_generation => text_generation_server}/models/galactica.py (98%) rename server/{text_generation => text_generation_server}/models/gpt_neox.py (99%) rename server/{text_generation => text_generation_server}/models/model.py (95%) rename server/{text_generation => text_generation_server}/models/santacoder.py (97%) rename server/{text_generation => text_generation_server}/models/seq2seq_lm.py (97%) rename server/{text_generation => text_generation_server}/models/t5.py (98%) rename server/{text_generation => text_generation_server}/models/types.py (95%) rename server/{text_generation => text_generation_server}/pb/.gitignore (100%) rename server/{text_generation => text_generation_server}/server.py (92%) rename server/{text_generation => text_generation_server}/tracing.py (100%) rename server/{text_generation => text_generation_server}/utils/__init__.py (71%) rename server/{text_generation => text_generation_server}/utils/convert.py (100%) rename server/{text_generation => text_generation_server}/utils/dist.py (100%) rename server/{text_generation => text_generation_server}/utils/hub.py (100%) rename server/{text_generation => text_generation_server}/utils/tokens.py (96%) rename server/{text_generation => text_generation_server}/utils/watermark.py (100%) diff --git a/Makefile b/Makefile index ade0cdd5..3defd886 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ server-dev: cd server && make run-dev router-dev: - cd router && cargo run + cd router && cargo run -- --port 8080 integration-tests: install-router install-launcher cargo test @@ -22,16 +22,16 @@ python-tests: cd server && HF_HUB_ENABLE_HF_TRANSFER=1 pytest tests run-bloom-560m: - text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 + text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --port 8080 run-bloom-560m-quantize: - text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --quantize + text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --quantize --port 8080 download-bloom: HF_HUB_ENABLE_HF_TRANSFER=1 text-generation-server download-weights bigscience/bloom run-bloom: - text-generation-launcher --model-id bigscience/bloom --num-shard 8 + text-generation-launcher --model-id bigscience/bloom --num-shard 8 --port 8080 run-bloom-quantize: - text-generation-launcher --model-id bigscience/bloom --num-shard 8 --quantize \ No newline at end of file + text-generation-launcher --model-id bigscience/bloom --num-shard 8 --quantize --port 8080 \ No newline at end of file diff --git a/README.md b/README.md index ae185506..5cd26a0e 100644 --- a/README.md +++ b/README.md @@ -89,40 +89,35 @@ You can then query the model using either the `/generate` or `/generate_stream` ```shell curl 127.0.0.1:8080/generate \ -X POST \ - -d '{"inputs":"Testing API","parameters":{"max_new_tokens":9}}' \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17}}' \ -H 'Content-Type: application/json' ``` ```shell curl 127.0.0.1:8080/generate_stream \ -X POST \ - -d '{"inputs":"Testing API","parameters":{"max_new_tokens":9}}' \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17}}' \ -H 'Content-Type: application/json' ``` or from Python: -```python -import requests - -result = requests.post("http://127.0.0.1:8080/generate", json={"inputs":"Testing API","parameters":{"max_new_tokens":9}}) -print(result.json()) -``` - ```shell -pip install sseclient-py +pip install text-generation ``` -````python -import sseclient -import requests +```python +from text_generation import Client -r = requests.post("http://127.0.0.1:8080/generate_stream", stream=True, json={"inputs":"Testing API","parameters":{"max_new_tokens":9}}) -sse_client = sseclient.SSEClient(r) +client = Client("http://127.0.0.1:8080") +print(client.generate("What is Deep Learning?", max_new_tokens=17).generated_text) -for i, event in enumerate(sse_client.events()): - print(i, event.data) -```` +text = "" +for response in client.generate_stream("What is Deep Learning?", max_new_tokens=17): + if not response.token.special: + text += response.token.text +print(text) +``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). diff --git a/clients/python/.gitignore b/clients/python/.gitignore new file mode 100644 index 00000000..5758ba92 --- /dev/null +++ b/clients/python/.gitignore @@ -0,0 +1,158 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +text_generation/__pycache__/ +text_generation/pb/__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +transformers +safetensors \ No newline at end of file diff --git a/clients/python/Makefile b/clients/python/Makefile new file mode 100644 index 00000000..8b4334bd --- /dev/null +++ b/clients/python/Makefile @@ -0,0 +1,6 @@ +unit-tests: + python -m pytest --cov=text_generation tests + +install: + pip install pip --upgrade + pip install -e . \ No newline at end of file diff --git a/clients/python/README.md b/clients/python/README.md new file mode 100644 index 00000000..414360bf --- /dev/null +++ b/clients/python/README.md @@ -0,0 +1,52 @@ +# Text Generation + +The Hugging Face Text Generation Python library provides a convenient way of interfacing with a +`text-generation-inference` instance running on your own infrastructure or on the Hugging Face Hub. + +## Get Started + +### Install + +```shell +pip install text-generation +``` + +### Usage + +```python +from text_generation import InferenceAPIClient + +client = InferenceAPIClient("bigscience/bloomz") +text = client.generate("Why is the sky blue?").generated_text +print(text) +# ' Rayleigh scattering' + +# Token Streaming +text = "" +for response in client.generate_stream("Why is the sky blue?"): + if not response.token.special: + text += response.token.text + +print(text) +# ' Rayleigh scattering' +``` + +or with the asynchronous client: + +```python +from text_generation import InferenceAPIAsyncClient + +client = InferenceAPIAsyncClient("bigscience/bloomz") +response = await client.generate("Why is the sky blue?") +print(response.generated_text) +# ' Rayleigh scattering' + +# Token Streaming +text = "" +async for response in client.generate_stream("Why is the sky blue?"): + if not response.token.special: + text += response.token.text + +print(text) +# ' Rayleigh scattering' +``` diff --git a/clients/python/poetry.lock b/clients/python/poetry.lock new file mode 100644 index 00000000..fe1f34cb --- /dev/null +++ b/clients/python/poetry.lock @@ -0,0 +1,1038 @@ +[[package]] +name = "aiohttp" +version = "3.8.4" +description = "Async http client/server framework (asyncio)" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +aiosignal = ">=1.1.2" +async-timeout = ">=4.0.0a3,<5.0" +asynctest = {version = "0.13.0", markers = "python_version < \"3.8\""} +attrs = ">=17.3.0" +charset-normalizer = ">=2.0,<4.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "cchardet"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""} + +[[package]] +name = "asynctest" +version = "0.13.0" +description = "Enhance the standard unittest package with features for testing asyncio libraries" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "22.2.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] +tests = ["attrs[tests-no-zope]", "zope.interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests_no_zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + +[[package]] +name = "certifi" +version = "2022.12.7" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "charset-normalizer" +version = "3.1.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.7.0" + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" + +[[package]] +name = "coverage" +version = "7.2.1" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "filelock" +version = "3.9.0" +description = "A platform independent file lock." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "frozenlist" +version = "1.3.3" +description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "huggingface-hub" +version = "0.12.1" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +category = "main" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +filelock = "*" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (==22.3)", "flake8 (>=3.8.3)", "flake8-bugbear", "isort (>=5.5.4)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (==22.3)", "flake8 (>=3.8.3)", "flake8-bugbear", "isort (>=5.5.4)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +quality = ["black (==22.3)", "flake8 (>=3.8.3)", "flake8-bugbear", "isort (>=5.5.4)", "mypy (==0.982)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "isort (>=5.5.4)", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"] +torch = ["torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "importlib-metadata" +version = "6.0.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "packaging" +version = "23.0" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pydantic" +version = "1.10.5" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-asyncio" +version = "0.17.2" +description = "Pytest support for asyncio" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +pytest = ">=6.1.0" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.8\""} + +[package.extras] +testing = ["coverage (==6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (==0.931)"] + +[[package]] +name = "pytest-cov" +version = "3.0.0" +description = "Pytest plugin for measuring coverage." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] + +[[package]] +name = "PyYAML" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "tqdm" +version = "4.65.0" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "typing-extensions" +version = "4.5.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "urllib3" +version = "1.26.14" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "yarl" +version = "1.8.2" +description = "Yet another URL library" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" +typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} + +[[package]] +name = "zipp" +version = "3.15.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.7" +content-hash = "12eb0ce46193750e35aec1f1fb32a0ce094aebd1db30682d06424ee6ccd23536" + +[metadata.files] +aiohttp = [ + {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, + {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, + {file = "aiohttp-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea"}, + {file = "aiohttp-3.8.4-cp310-cp310-win32.whl", hash = "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1"}, + {file = "aiohttp-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24"}, + {file = "aiohttp-3.8.4-cp311-cp311-win32.whl", hash = "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d"}, + {file = "aiohttp-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc"}, + {file = "aiohttp-3.8.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff"}, + {file = "aiohttp-3.8.4-cp36-cp36m-win32.whl", hash = "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777"}, + {file = "aiohttp-3.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e"}, + {file = "aiohttp-3.8.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241"}, + {file = "aiohttp-3.8.4-cp37-cp37m-win32.whl", hash = "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a"}, + {file = "aiohttp-3.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d"}, + {file = "aiohttp-3.8.4-cp38-cp38-win32.whl", hash = "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54"}, + {file = "aiohttp-3.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4"}, + {file = "aiohttp-3.8.4-cp39-cp39-win32.whl", hash = "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a"}, + {file = "aiohttp-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04"}, + {file = "aiohttp-3.8.4.tar.gz", hash = "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c"}, +] +aiosignal = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] +async-timeout = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] +asynctest = [ + {file = "asynctest-0.13.0-py3-none-any.whl", hash = "sha256:5da6118a7e6d6b54d83a8f7197769d046922a44d2a99c21382f0a6e4fadae676"}, + {file = "asynctest-0.13.0.tar.gz", hash = "sha256:c27862842d15d83e6a34eb0b2866c323880eb3a75e4485b079ea11748fd77fac"}, +] +atomicwrites = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] +attrs = [ + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, +] +certifi = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] +charset-normalizer = [ + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, +] +colorama = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +coverage = [ + {file = "coverage-7.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49567ec91fc5e0b15356da07a2feabb421d62f52a9fff4b1ec40e9e19772f5f8"}, + {file = "coverage-7.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2ef6cae70168815ed91388948b5f4fcc69681480a0061114db737f957719f03"}, + {file = "coverage-7.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3004765bca3acd9e015794e5c2f0c9a05587f5e698127ff95e9cfba0d3f29339"}, + {file = "coverage-7.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cca7c0b7f5881dfe0291ef09ba7bb1582cb92ab0aeffd8afb00c700bf692415a"}, + {file = "coverage-7.2.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2167d116309f564af56f9aa5e75ef710ef871c5f9b313a83050035097b56820"}, + {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cb5f152fb14857cbe7f3e8c9a5d98979c4c66319a33cad6e617f0067c9accdc4"}, + {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:87dc37f16fb5e3a28429e094145bf7c1753e32bb50f662722e378c5851f7fdc6"}, + {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e191a63a05851f8bce77bc875e75457f9b01d42843f8bd7feed2fc26bbe60833"}, + {file = "coverage-7.2.1-cp310-cp310-win32.whl", hash = "sha256:e3ea04b23b114572b98a88c85379e9e9ae031272ba1fb9b532aa934c621626d4"}, + {file = "coverage-7.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:0cf557827be7eca1c38a2480484d706693e7bb1929e129785fe59ec155a59de6"}, + {file = "coverage-7.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:570c21a29493b350f591a4b04c158ce1601e8d18bdcd21db136fbb135d75efa6"}, + {file = "coverage-7.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e872b082b32065ac2834149dc0adc2a2e6d8203080501e1e3c3c77851b466f9"}, + {file = "coverage-7.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fac6343bae03b176e9b58104a9810df3cdccd5cfed19f99adfa807ffbf43cf9b"}, + {file = "coverage-7.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abacd0a738e71b20e224861bc87e819ef46fedba2fb01bc1af83dfd122e9c319"}, + {file = "coverage-7.2.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9256d4c60c4bbfec92721b51579c50f9e5062c21c12bec56b55292464873508"}, + {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:80559eaf6c15ce3da10edb7977a1548b393db36cbc6cf417633eca05d84dd1ed"}, + {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0bd7e628f6c3ec4e7d2d24ec0e50aae4e5ae95ea644e849d92ae4805650b4c4e"}, + {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09643fb0df8e29f7417adc3f40aaf379d071ee8f0350ab290517c7004f05360b"}, + {file = "coverage-7.2.1-cp311-cp311-win32.whl", hash = "sha256:1b7fb13850ecb29b62a447ac3516c777b0e7a09ecb0f4bb6718a8654c87dfc80"}, + {file = "coverage-7.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:617a94ada56bbfe547aa8d1b1a2b8299e2ec1ba14aac1d4b26a9f7d6158e1273"}, + {file = "coverage-7.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8649371570551d2fd7dee22cfbf0b61f1747cdfb2b7587bb551e4beaaa44cb97"}, + {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d2b9b5e70a21474c105a133ba227c61bc95f2ac3b66861143ce39a5ea4b3f84"}, + {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae82c988954722fa07ec5045c57b6d55bc1a0890defb57cf4a712ced65b26ddd"}, + {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:861cc85dfbf55a7a768443d90a07e0ac5207704a9f97a8eb753292a7fcbdfcfc"}, + {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0339dc3237c0d31c3b574f19c57985fcbe494280153bbcad33f2cdf469f4ac3e"}, + {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:5928b85416a388dd557ddc006425b0c37e8468bd1c3dc118c1a3de42f59e2a54"}, + {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8d3843ca645f62c426c3d272902b9de90558e9886f15ddf5efe757b12dd376f5"}, + {file = "coverage-7.2.1-cp37-cp37m-win32.whl", hash = "sha256:6a034480e9ebd4e83d1aa0453fd78986414b5d237aea89a8fdc35d330aa13bae"}, + {file = "coverage-7.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6fce673f79a0e017a4dc35e18dc7bb90bf6d307c67a11ad5e61ca8d42b87cbff"}, + {file = "coverage-7.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f099da6958ddfa2ed84bddea7515cb248583292e16bb9231d151cd528eab657"}, + {file = "coverage-7.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:97a3189e019d27e914ecf5c5247ea9f13261d22c3bb0cfcfd2a9b179bb36f8b1"}, + {file = "coverage-7.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a81dbcf6c6c877986083d00b834ac1e84b375220207a059ad45d12f6e518a4e3"}, + {file = "coverage-7.2.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d2c3dde4c0b9be4b02067185136b7ee4681978228ad5ec1278fa74f5ca3e99"}, + {file = "coverage-7.2.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a209d512d157379cc9ab697cbdbb4cfd18daa3e7eebaa84c3d20b6af0037384"}, + {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f3d07edb912a978915576a776756069dede66d012baa503022d3a0adba1b6afa"}, + {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8dca3c1706670297851bca1acff9618455122246bdae623be31eca744ade05ec"}, + {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b1991a6d64231a3e5bbe3099fb0dd7c9aeaa4275ad0e0aeff4cb9ef885c62ba2"}, + {file = "coverage-7.2.1-cp38-cp38-win32.whl", hash = "sha256:22c308bc508372576ffa3d2dbc4824bb70d28eeb4fcd79d4d1aed663a06630d0"}, + {file = "coverage-7.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:b0c0d46de5dd97f6c2d1b560bf0fcf0215658097b604f1840365296302a9d1fb"}, + {file = "coverage-7.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4dd34a935de268a133e4741827ae951283a28c0125ddcdbcbba41c4b98f2dfef"}, + {file = "coverage-7.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0f8318ed0f3c376cfad8d3520f496946977abde080439d6689d7799791457454"}, + {file = "coverage-7.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:834c2172edff5a08d78e2f53cf5e7164aacabeb66b369f76e7bb367ca4e2d993"}, + {file = "coverage-7.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4d70c853f0546855f027890b77854508bdb4d6a81242a9d804482e667fff6e6"}, + {file = "coverage-7.2.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a6450da4c7afc4534305b2b7d8650131e130610cea448ff240b6ab73d7eab63"}, + {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:99f4dd81b2bb8fc67c3da68b1f5ee1650aca06faa585cbc6818dbf67893c6d58"}, + {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bdd3f2f285ddcf2e75174248b2406189261a79e7fedee2ceeadc76219b6faa0e"}, + {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f29351393eb05e6326f044a7b45ed8e38cb4dcc38570d12791f271399dc41431"}, + {file = "coverage-7.2.1-cp39-cp39-win32.whl", hash = "sha256:e2b50ebc2b6121edf352336d503357321b9d8738bb7a72d06fc56153fd3f4cd8"}, + {file = "coverage-7.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:bd5a12239c0006252244f94863f1c518ac256160cd316ea5c47fb1a11b25889a"}, + {file = "coverage-7.2.1-pp37.pp38.pp39-none-any.whl", hash = "sha256:436313d129db7cf5b4ac355dd2bd3f7c7e5294af077b090b85de75f8458b8616"}, + {file = "coverage-7.2.1.tar.gz", hash = "sha256:c77f2a9093ccf329dd523a9b2b3c854c20d2a3d968b6def3b820272ca6732242"}, +] +filelock = [ + {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, + {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, +] +frozenlist = [ + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, + {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, + {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, + {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, + {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, + {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, + {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, + {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, + {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, + {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, + {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, +] +huggingface-hub = [ + {file = "huggingface_hub-0.12.1-py3-none-any.whl", hash = "sha256:867586cc8543fe1bd43a219fedbea7d71690021ad80f0c46f35c4751069278d7"}, + {file = "huggingface_hub-0.12.1.tar.gz", hash = "sha256:6f960f6246ef9c3446d0d6275e853485515682c350917fdaf2a59705f8b9ebb3"}, +] +idna = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] +importlib-metadata = [ + {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, + {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, +] +iniconfig = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] +multidict = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] +packaging = [ + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +py = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] +pydantic = [ + {file = "pydantic-1.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5920824fe1e21cbb3e38cf0f3dd24857c8959801d1031ce1fac1d50857a03bfb"}, + {file = "pydantic-1.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3bb99cf9655b377db1a9e47fa4479e3330ea96f4123c6c8200e482704bf1eda2"}, + {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2185a3b3d98ab4506a3f6707569802d2d92c3a7ba3a9a35683a7709ea6c2aaa2"}, + {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f582cac9d11c227c652d3ce8ee223d94eb06f4228b52a8adaafa9fa62e73d5c9"}, + {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c9e5b778b6842f135902e2d82624008c6a79710207e28e86966cd136c621bfee"}, + {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:72ef3783be8cbdef6bca034606a5de3862be6b72415dc5cb1fb8ddbac110049a"}, + {file = "pydantic-1.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:45edea10b75d3da43cfda12f3792833a3fa70b6eee4db1ed6aed528cef17c74e"}, + {file = "pydantic-1.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:63200cd8af1af2c07964546b7bc8f217e8bda9d0a2ef0ee0c797b36353914984"}, + {file = "pydantic-1.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:305d0376c516b0dfa1dbefeae8c21042b57b496892d721905a6ec6b79494a66d"}, + {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd326aff5d6c36f05735c7c9b3d5b0e933b4ca52ad0b6e4b38038d82703d35b"}, + {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6bb0452d7b8516178c969d305d9630a3c9b8cf16fcf4713261c9ebd465af0d73"}, + {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9a9d9155e2a9f38b2eb9374c88f02fd4d6851ae17b65ee786a87d032f87008f8"}, + {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f836444b4c5ece128b23ec36a446c9ab7f9b0f7981d0d27e13a7c366ee163f8a"}, + {file = "pydantic-1.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:8481dca324e1c7b715ce091a698b181054d22072e848b6fc7895cd86f79b4449"}, + {file = "pydantic-1.10.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:87f831e81ea0589cd18257f84386bf30154c5f4bed373b7b75e5cb0b5d53ea87"}, + {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ce1612e98c6326f10888df951a26ec1a577d8df49ddcaea87773bfbe23ba5cc"}, + {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58e41dd1e977531ac6073b11baac8c013f3cd8706a01d3dc74e86955be8b2c0c"}, + {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6a4b0aab29061262065bbdede617ef99cc5914d1bf0ddc8bcd8e3d7928d85bd6"}, + {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:36e44a4de37b8aecffa81c081dbfe42c4d2bf9f6dff34d03dce157ec65eb0f15"}, + {file = "pydantic-1.10.5-cp37-cp37m-win_amd64.whl", hash = "sha256:261f357f0aecda005934e413dfd7aa4077004a174dafe414a8325e6098a8e419"}, + {file = "pydantic-1.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b429f7c457aebb7fbe7cd69c418d1cd7c6fdc4d3c8697f45af78b8d5a7955760"}, + {file = "pydantic-1.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:663d2dd78596c5fa3eb996bc3f34b8c2a592648ad10008f98d1348be7ae212fb"}, + {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51782fd81f09edcf265823c3bf43ff36d00db246eca39ee765ef58dc8421a642"}, + {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c428c0f64a86661fb4873495c4fac430ec7a7cef2b8c1c28f3d1a7277f9ea5ab"}, + {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:76c930ad0746c70f0368c4596020b736ab65b473c1f9b3872310a835d852eb19"}, + {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3257bd714de9db2102b742570a56bf7978e90441193acac109b1f500290f5718"}, + {file = "pydantic-1.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:f5bee6c523d13944a1fdc6f0525bc86dbbd94372f17b83fa6331aabacc8fd08e"}, + {file = "pydantic-1.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:532e97c35719f137ee5405bd3eeddc5c06eb91a032bc755a44e34a712420daf3"}, + {file = "pydantic-1.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ca9075ab3de9e48b75fa8ccb897c34ccc1519177ad8841d99f7fd74cf43be5bf"}, + {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd46a0e6296346c477e59a954da57beaf9c538da37b9df482e50f836e4a7d4bb"}, + {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3353072625ea2a9a6c81ad01b91e5c07fa70deb06368c71307529abf70d23325"}, + {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3f9d9b2be177c3cb6027cd67fbf323586417868c06c3c85d0d101703136e6b31"}, + {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b473d00ccd5c2061fd896ac127b7755baad233f8d996ea288af14ae09f8e0d1e"}, + {file = "pydantic-1.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:5f3bc8f103b56a8c88021d481410874b1f13edf6e838da607dcb57ecff9b4594"}, + {file = "pydantic-1.10.5-py3-none-any.whl", hash = "sha256:7c5b94d598c90f2f46b3a983ffb46ab806a67099d118ae0da7ef21a2a4033b28"}, + {file = "pydantic-1.10.5.tar.gz", hash = "sha256:9e337ac83686645a46db0e825acceea8e02fca4062483f40e9ae178e8bd1103a"}, +] +pytest = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] +pytest-asyncio = [ + {file = "pytest-asyncio-0.17.2.tar.gz", hash = "sha256:6d895b02432c028e6957d25fc936494e78c6305736e785d9fee408b1efbc7ff4"}, + {file = "pytest_asyncio-0.17.2-py3-none-any.whl", hash = "sha256:e0fe5dbea40516b661ef1bcfe0bd9461c2847c4ef4bb40012324f2454fb7d56d"}, +] +pytest-cov = [ + {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, + {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, +] +PyYAML = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +requests = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] +tqdm = [ + {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, + {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, +] +typing-extensions = [ + {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, + {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, +] +urllib3 = [ + {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, + {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, +] +yarl = [ + {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, + {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, + {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80"}, + {file = "yarl-1.8.2-cp310-cp310-win32.whl", hash = "sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42"}, + {file = "yarl-1.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2"}, + {file = "yarl-1.8.2-cp311-cp311-win32.whl", hash = "sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b"}, + {file = "yarl-1.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c"}, + {file = "yarl-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37"}, + {file = "yarl-1.8.2-cp37-cp37m-win32.whl", hash = "sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89"}, + {file = "yarl-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b372aad2b5f81db66ee7ec085cbad72c4da660d994e8e590c997e9b01e44901"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e6f3515aafe0209dd17fb9bdd3b4e892963370b3de781f53e1746a521fb39fc0"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dfef7350ee369197106805e193d420b75467b6cceac646ea5ed3049fcc950a05"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:728be34f70a190566d20aa13dc1f01dc44b6aa74580e10a3fb159691bc76909d"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baf211dcad448a87a0d9047dc8282d7de59473ade7d7fdf22150b1d23859f946"}, + {file = "yarl-1.8.2-cp38-cp38-win32.whl", hash = "sha256:272b4f1599f1b621bf2aabe4e5b54f39a933971f4e7c9aa311d6d7dc06965165"}, + {file = "yarl-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:326dd1d3caf910cd26a26ccbfb84c03b608ba32499b5d6eeb09252c920bcbe4f"}, + {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f8ca8ad414c85bbc50f49c0a106f951613dfa5f948ab69c10ce9b128d368baf8"}, + {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:418857f837347e8aaef682679f41e36c24250097f9e2f315d39bae3a99a34cbf"}, + {file = "yarl-1.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ae0eec05ab49e91a78700761777f284c2df119376e391db42c38ab46fd662b77"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3edac5d74bb3209c418805bda77f973117836e1de7c000e9755e572c1f7850d0"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da65c3f263729e47351261351b8679c6429151ef9649bba08ef2528ff2c423b2"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef8fb25e52663a1c85d608f6dd72e19bd390e2ecaf29c17fb08f730226e3a08"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcd7bb1e5c45274af9a1dd7494d3c52b2be5e6bd8d7e49c612705fd45420b12d"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44ceac0450e648de86da8e42674f9b7077d763ea80c8ceb9d1c3e41f0f0a9951"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:97209cc91189b48e7cfe777237c04af8e7cc51eb369004e061809bcdf4e55220"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:48dd18adcf98ea9cd721a25313aef49d70d413a999d7d89df44f469edfb38a06"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e59399dda559688461762800d7fb34d9e8a6a7444fd76ec33220a926c8be1516"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d617c241c8c3ad5c4e78a08429fa49e4b04bedfc507b34b4d8dceb83b4af3588"}, + {file = "yarl-1.8.2-cp39-cp39-win32.whl", hash = "sha256:cb6d48d80a41f68de41212f3dfd1a9d9898d7841c8f7ce6696cf2fd9cb57ef83"}, + {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, + {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, +] +zipp = [ + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, +] diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml new file mode 100644 index 00000000..6eb11638 --- /dev/null +++ b/clients/python/pyproject.toml @@ -0,0 +1,26 @@ +[tool.poetry] +name = "text-generation" +version = "0.1.0" +description = "Hugging Face Text Generation Python Client" +license = "Apache-2.0" +authors = ["Olivier Dehaene "] +maintainers = ["Olivier Dehaene "] +readme = "README.md" +homepage = "https://github.com/huggingface/text-generation-inference" +repository = "https://github.com/huggingface/text-generation-inference" + + +[tool.poetry.dependencies] +python = "^3.7" +pydantic = "^1.10.5" +aiohttp = "^3.8.4" +huggingface-hub = "^0.12.1" + +[tool.poetry.dev-dependencies] +pytest = "^6.2.5" +pytest-asyncio = "^0.17.2" +pytest-cov = "^3.0.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/clients/python/tests/conftest.py b/clients/python/tests/conftest.py new file mode 100644 index 00000000..4298623e --- /dev/null +++ b/clients/python/tests/conftest.py @@ -0,0 +1,56 @@ +import pytest + +from text_generation import __version__ +from huggingface_hub.utils import build_hf_headers + + +@pytest.fixture +def bloom_model(): + return "bigscience/bloom" + + +@pytest.fixture +def flan_t5_xxl(): + return "google/flan-t5-xxl" + + +@pytest.fixture +def fake_model(): + return "fake/model" + + +@pytest.fixture +def unsupported_model(): + return "gpt2" + + +@pytest.fixture +def base_url(): + return "https://api-inference.huggingface.co/models" + + +@pytest.fixture +def bloom_url(base_url, bloom_model): + return f"{base_url}/{bloom_model}" + + +@pytest.fixture +def flan_t5_xxl_url(base_url, flan_t5_xxl): + return f"{base_url}/{flan_t5_xxl}" + + +@pytest.fixture +def fake_url(base_url, fake_model): + return f"{base_url}/{fake_model}" + + +@pytest.fixture +def unsupported_url(base_url, unsupported_model): + return f"{base_url}/{unsupported_model}" + + +@pytest.fixture(scope="session") +def hf_headers(): + return build_hf_headers( + library_name="text-generation-tests", library_version=__version__ + ) diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py new file mode 100644 index 00000000..2f96aa87 --- /dev/null +++ b/clients/python/tests/test_client.py @@ -0,0 +1,127 @@ +import pytest + +from text_generation import Client, AsyncClient +from text_generation.errors import NotFoundError, ValidationError +from text_generation.types import FinishReason, PrefillToken, Token + + +def test_generate(bloom_url, hf_headers): + client = Client(bloom_url, hf_headers) + response = client.generate("test", max_new_tokens=1) + + assert response.generated_text == "." + assert response.details.finish_reason == FinishReason.Length + assert response.details.generated_tokens == 1 + assert response.details.seed is None + assert len(response.details.prefill) == 1 + assert response.details.prefill[0] == PrefillToken( + id=9234, text="test", logprob=None + ) + assert len(response.details.tokens) == 1 + assert response.details.tokens[0] == Token( + id=17, text=".", logprob=-1.75, special=False + ) + + +def test_generate_not_found(fake_url, hf_headers): + client = Client(fake_url, hf_headers) + with pytest.raises(NotFoundError): + client.generate("test") + + +def test_generate_validation_error(flan_t5_xxl_url, hf_headers): + client = Client(flan_t5_xxl_url, hf_headers) + with pytest.raises(ValidationError): + client.generate("test", max_new_tokens=10_000) + + +def test_generate_stream(bloom_url, hf_headers): + client = Client(bloom_url, hf_headers) + responses = [ + response for response in client.generate_stream("test", max_new_tokens=1) + ] + + assert len(responses) == 1 + response = responses[0] + + assert response.generated_text == "." + assert response.details.finish_reason == FinishReason.Length + assert response.details.generated_tokens == 1 + assert response.details.seed is None + + +def test_generate_stream_not_found(fake_url, hf_headers): + client = Client(fake_url, hf_headers) + with pytest.raises(NotFoundError): + list(client.generate_stream("test")) + + +def test_generate_stream_validation_error(flan_t5_xxl_url, hf_headers): + client = Client(flan_t5_xxl_url, hf_headers) + with pytest.raises(ValidationError): + list(client.generate_stream("test", max_new_tokens=10_000)) + + +@pytest.mark.asyncio +async def test_generate_async(bloom_url, hf_headers): + client = AsyncClient(bloom_url, hf_headers) + response = await client.generate("test", max_new_tokens=1) + + assert response.generated_text == "." + assert response.details.finish_reason == FinishReason.Length + assert response.details.generated_tokens == 1 + assert response.details.seed is None + assert len(response.details.prefill) == 1 + assert response.details.prefill[0] == PrefillToken( + id=9234, text="test", logprob=None + ) + assert len(response.details.tokens) == 1 + assert response.details.tokens[0] == Token( + id=17, text=".", logprob=-1.75, special=False + ) + + +@pytest.mark.asyncio +async def test_generate_async_not_found(fake_url, hf_headers): + client = AsyncClient(fake_url, hf_headers) + with pytest.raises(NotFoundError): + await client.generate("test") + + +@pytest.mark.asyncio +async def test_generate_async_validation_error(flan_t5_xxl_url, hf_headers): + client = AsyncClient(flan_t5_xxl_url, hf_headers) + with pytest.raises(ValidationError): + await client.generate("test", max_new_tokens=10_000) + + +@pytest.mark.asyncio +async def test_generate_stream_async(bloom_url, hf_headers): + client = AsyncClient(bloom_url, hf_headers) + responses = [ + response async for response in client.generate_stream("test", max_new_tokens=1) + ] + + assert len(responses) == 1 + response = responses[0] + + assert response.generated_text == "." + assert response.details.finish_reason == FinishReason.Length + assert response.details.generated_tokens == 1 + assert response.details.seed is None + + +@pytest.mark.asyncio +async def test_generate_stream_async_not_found(fake_url, hf_headers): + client = AsyncClient(fake_url, hf_headers) + with pytest.raises(NotFoundError): + async for _ in client.generate_stream("test"): + pass + + +@pytest.mark.asyncio +async def test_generate_stream_async_validation_error(flan_t5_xxl_url, hf_headers): + client = AsyncClient(flan_t5_xxl_url, hf_headers) + with pytest.raises(ValidationError): + async for _ in client.generate_stream("test", max_new_tokens=10_000): + pass diff --git a/clients/python/tests/test_errors.py b/clients/python/tests/test_errors.py new file mode 100644 index 00000000..8389ed31 --- /dev/null +++ b/clients/python/tests/test_errors.py @@ -0,0 +1,64 @@ +from text_generation.errors import ( + parse_error, + GenerationError, + IncompleteGenerationError, + OverloadedError, + ValidationError, + BadRequestError, + ShardNotReadyError, + ShardTimeoutError, + NotFoundError, + RateLimitExceededError, + UnknownError, +) + + +def test_generation_error(): + payload = {"error_type": "generation", "error": "test"} + assert isinstance(parse_error(400, payload), GenerationError) + + +def test_incomplete_generation_error(): + payload = {"error_type": "incomplete_generation", "error": "test"} + assert isinstance(parse_error(400, payload), IncompleteGenerationError) + + +def test_overloaded_error(): + payload = {"error_type": "overloaded", "error": "test"} + assert isinstance(parse_error(400, payload), OverloadedError) + + +def test_validation_error(): + payload = {"error_type": "validation", "error": "test"} + assert isinstance(parse_error(400, payload), ValidationError) + + +def test_bad_request_error(): + payload = {"error": "test"} + assert isinstance(parse_error(400, payload), BadRequestError) + + +def test_shard_not_ready_error(): + payload = {"error": "test"} + assert isinstance(parse_error(403, payload), ShardNotReadyError) + assert isinstance(parse_error(424, payload), ShardNotReadyError) + + +def test_shard_timeout_error(): + payload = {"error": "test"} + assert isinstance(parse_error(504, payload), ShardTimeoutError) + + +def test_not_found_error(): + payload = {"error": "test"} + assert isinstance(parse_error(404, payload), NotFoundError) + + +def test_rate_limit_exceeded_error(): + payload = {"error": "test"} + assert isinstance(parse_error(429, payload), RateLimitExceededError) + + +def test_unknown_error(): + payload = {"error": "test"} + assert isinstance(parse_error(500, payload), UnknownError) diff --git a/clients/python/tests/test_inference_api.py b/clients/python/tests/test_inference_api.py new file mode 100644 index 00000000..dc744940 --- /dev/null +++ b/clients/python/tests/test_inference_api.py @@ -0,0 +1,34 @@ +import pytest + +from text_generation import ( + InferenceAPIClient, + InferenceAPIAsyncClient, + Client, + AsyncClient, +) +from text_generation.errors import NotSupportedError +from text_generation.inference_api import get_supported_models + + +def test_get_supported_models(): + assert isinstance(get_supported_models(), list) + + +def test_client(bloom_model): + client = InferenceAPIClient(bloom_model) + assert isinstance(client, Client) + + +def test_client_unsupported_model(unsupported_model): + with pytest.raises(NotSupportedError): + InferenceAPIClient(unsupported_model) + + +def test_async_client(bloom_model): + client = InferenceAPIAsyncClient(bloom_model) + assert isinstance(client, AsyncClient) + + +def test_async_client_unsupported_model(unsupported_model): + with pytest.raises(NotSupportedError): + InferenceAPIAsyncClient(unsupported_model) diff --git a/clients/python/tests/test_types.py b/clients/python/tests/test_types.py new file mode 100644 index 00000000..d319b570 --- /dev/null +++ b/clients/python/tests/test_types.py @@ -0,0 +1,39 @@ +import pytest + +from text_generation.types import Parameters +from text_generation.errors import ValidationError + + +def test_parameters_validation(): + # Test repetition_penalty + Parameters(repetition_penalty=1) + with pytest.raises(ValidationError): + Parameters(repetition_penalty=0) + with pytest.raises(ValidationError): + Parameters(repetition_penalty=-1) + + # Test seed + Parameters(seed=1) + with pytest.raises(ValidationError): + Parameters(seed=-1) + + # Test temperature + Parameters(temperature=1) + with pytest.raises(ValidationError): + Parameters(temperature=0) + with pytest.raises(ValidationError): + Parameters(temperature=-1) + + # Test top_k + Parameters(top_k=1) + with pytest.raises(ValidationError): + Parameters(top_k=0) + with pytest.raises(ValidationError): + Parameters(top_k=-1) + + # Test top_p + Parameters(top_p=1) + with pytest.raises(ValidationError): + Parameters(top_p=0) + with pytest.raises(ValidationError): + Parameters(top_p=-1) diff --git a/clients/python/text_generation/__init__.py b/clients/python/text_generation/__init__.py new file mode 100644 index 00000000..88861b37 --- /dev/null +++ b/clients/python/text_generation/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.3.2" + +from text_generation.client import Client, AsyncClient +from text_generation.inference_api import InferenceAPIClient, InferenceAPIAsyncClient diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py new file mode 100644 index 00000000..d0a67915 --- /dev/null +++ b/clients/python/text_generation/client.py @@ -0,0 +1,415 @@ +import json +import requests + +from aiohttp import ClientSession, ClientTimeout +from pydantic import ValidationError +from typing import Dict, Optional, List, AsyncIterator, Iterator + +from text_generation.types import ( + StreamResponse, + Response, + Request, + Parameters, +) +from text_generation.errors import parse_error + + +class Client: + """Client to make calls to a text-generation-inference instance + + Example: + + ```python + >>> from text_generation import Client + + >>> client = Client("https://api-inference.huggingface.co/models/bigscience/bloomz") + >>> client.generate("Why is the sky blue?").generated_text + ' Rayleigh scattering' + + >>> result = "" + >>> for response in client.generate_stream("Why is the sky blue?"): + >>> if not response.token.special: + >>> result += response.token.text + >>> result + ' Rayleigh scattering' + ``` + """ + + def __init__( + self, base_url: str, headers: Optional[Dict[str, str]] = None, timeout: int = 10 + ): + """ + Args: + base_url (`str`): + text-generation-inference instance base url + headers (`Optional[Dict[str, str]]`): + Additional headers + timeout (`int`): + Timeout in seconds + """ + self.base_url = base_url + self.headers = headers + self.timeout = timeout + + def generate( + self, + prompt: str, + do_sample: bool = False, + max_new_tokens: int = 20, + repetition_penalty: Optional[float] = None, + return_full_text: bool = False, + seed: Optional[int] = None, + stop_sequences: Optional[List[str]] = None, + temperature: Optional[float] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + watermarking: bool = False, + ) -> Response: + """ + Given a prompt, generate the following text + + Args: + prompt (`str`): + Input text + do_sample (`bool`): + Activate logits sampling + max_new_tokens (`int`): + Maximum number of generated tokens + repetition_penalty (`float`): + The parameter for repetition penalty. 1.0 means no penalty. See [this + paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + return_full_text (`bool`): + Whether to prepend the prompt to the generated text + seed (`int`): + Random sampling seed + stop_sequences (`List[str]`): + Stop generating tokens if a member of `stop_sequences` is generated + temperature (`float`): + The value used to module the logits distribution. + top_k (`int`): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_p (`float`): + If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + higher are kept for generation. + watermarking (`bool`): + Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + + Returns: + Response: generated response + """ + # Validate parameters + parameters = Parameters( + details=True, + do_sample=do_sample, + max_new_tokens=max_new_tokens, + repetition_penalty=repetition_penalty, + return_full_text=return_full_text, + seed=seed, + stop=stop_sequences if stop_sequences is not None else [], + temperature=temperature, + top_k=top_k, + top_p=top_p, + watermark=watermarking, + ) + request = Request(inputs=prompt, stream=False, parameters=parameters) + + resp = requests.post( + self.base_url, + json=request.dict(), + headers=self.headers, + timeout=self.timeout, + ) + payload = resp.json() + if resp.status_code != 200: + raise parse_error(resp.status_code, payload) + return Response(**payload[0]) + + def generate_stream( + self, + prompt: str, + do_sample: bool = False, + max_new_tokens: int = 20, + repetition_penalty: Optional[float] = None, + return_full_text: bool = False, + seed: Optional[int] = None, + stop_sequences: Optional[List[str]] = None, + temperature: Optional[float] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + watermarking: bool = False, + ) -> Iterator[StreamResponse]: + """ + Given a prompt, generate the following stream of tokens + + Args: + prompt (`str`): + Input text + do_sample (`bool`): + Activate logits sampling + max_new_tokens (`int`): + Maximum number of generated tokens + repetition_penalty (`float`): + The parameter for repetition penalty. 1.0 means no penalty. See [this + paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + return_full_text (`bool`): + Whether to prepend the prompt to the generated text + seed (`int`): + Random sampling seed + stop_sequences (`List[str]`): + Stop generating tokens if a member of `stop_sequences` is generated + temperature (`float`): + The value used to module the logits distribution. + top_k (`int`): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_p (`float`): + If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + higher are kept for generation. + watermarking (`bool`): + Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + + Returns: + Iterator[StreamResponse]: stream of generated tokens + """ + # Validate parameters + parameters = Parameters( + details=True, + do_sample=do_sample, + max_new_tokens=max_new_tokens, + repetition_penalty=repetition_penalty, + return_full_text=return_full_text, + seed=seed, + stop=stop_sequences if stop_sequences is not None else [], + temperature=temperature, + top_k=top_k, + top_p=top_p, + watermark=watermarking, + ) + request = Request(inputs=prompt, stream=True, parameters=parameters) + + resp = requests.post( + self.base_url, + json=request.dict(), + headers=self.headers, + timeout=self.timeout, + stream=False, + ) + + if resp.status_code != 200: + raise parse_error(resp.status_code, resp.json()) + + # Parse ServerSentEvents + for byte_payload in resp.iter_lines(): + # Skip line + if byte_payload == b"\n": + continue + + payload = byte_payload.decode("utf-8") + + # Event data + if payload.startswith("data:"): + # Decode payload + json_payload = json.loads(payload.lstrip("data:").rstrip("/n")) + # Parse payload + try: + response = StreamResponse(**json_payload) + except ValidationError: + # If we failed to parse the payload, then it is an error payload + raise parse_error(resp.status_code, json_payload) + yield response + + +class AsyncClient: + """Asynchronous Client to make calls to a text-generation-inference instance + + Example: + + ```python + >>> from text_generation import AsyncClient + + >>> client = AsyncClient("https://api-inference.huggingface.co/models/bigscience/bloomz") + >>> response = await client.generate("Why is the sky blue?") + >>> response.generated_text + ' Rayleigh scattering' + + >>> result = "" + >>> async for response in client.generate_stream("Why is the sky blue?"): + >>> if not response.token.special: + >>> result += response.token.text + >>> result + ' Rayleigh scattering' + ``` + """ + + def __init__( + self, base_url: str, headers: Optional[Dict[str, str]] = None, timeout: int = 10 + ): + """ + Args: + base_url (`str`): + text-generation-inference instance base url + headers (`Optional[Dict[str, str]]`): + Additional headers + timeout (`int`): + Timeout in seconds + """ + self.base_url = base_url + self.headers = headers + self.timeout = ClientTimeout(timeout * 60) + + async def generate( + self, + prompt: str, + do_sample: bool = False, + max_new_tokens: int = 20, + repetition_penalty: Optional[float] = None, + return_full_text: bool = False, + seed: Optional[int] = None, + stop_sequences: Optional[List[str]] = None, + temperature: Optional[float] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + watermarking: bool = False, + ) -> Response: + """ + Given a prompt, generate the following text asynchronously + + Args: + prompt (`str`): + Input text + do_sample (`bool`): + Activate logits sampling + max_new_tokens (`int`): + Maximum number of generated tokens + repetition_penalty (`float`): + The parameter for repetition penalty. 1.0 means no penalty. See [this + paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + return_full_text (`bool`): + Whether to prepend the prompt to the generated text + seed (`int`): + Random sampling seed + stop_sequences (`List[str]`): + Stop generating tokens if a member of `stop_sequences` is generated + temperature (`float`): + The value used to module the logits distribution. + top_k (`int`): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_p (`float`): + If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + higher are kept for generation. + watermarking (`bool`): + Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + + Returns: + Response: generated response + """ + # Validate parameters + parameters = Parameters( + details=True, + do_sample=do_sample, + max_new_tokens=max_new_tokens, + repetition_penalty=repetition_penalty, + return_full_text=return_full_text, + seed=seed, + stop=stop_sequences if stop_sequences is not None else [], + temperature=temperature, + top_k=top_k, + top_p=top_p, + watermark=watermarking, + ) + request = Request(inputs=prompt, stream=False, parameters=parameters) + + async with ClientSession(headers=self.headers, timeout=self.timeout) as session: + async with session.post(self.base_url, json=request.dict()) as resp: + payload = await resp.json() + + if resp.status != 200: + raise parse_error(resp.status, payload) + return Response(**payload[0]) + + async def generate_stream( + self, + prompt: str, + do_sample: bool = False, + max_new_tokens: int = 20, + repetition_penalty: Optional[float] = None, + return_full_text: bool = False, + seed: Optional[int] = None, + stop_sequences: Optional[List[str]] = None, + temperature: Optional[float] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + watermarking: bool = False, + ) -> AsyncIterator[StreamResponse]: + """ + Given a prompt, generate the following stream of tokens asynchronously + + Args: + prompt (`str`): + Input text + do_sample (`bool`): + Activate logits sampling + max_new_tokens (`int`): + Maximum number of generated tokens + repetition_penalty (`float`): + The parameter for repetition penalty. 1.0 means no penalty. See [this + paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + return_full_text (`bool`): + Whether to prepend the prompt to the generated text + seed (`int`): + Random sampling seed + stop_sequences (`List[str]`): + Stop generating tokens if a member of `stop_sequences` is generated + temperature (`float`): + The value used to module the logits distribution. + top_k (`int`): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_p (`float`): + If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + higher are kept for generation. + watermarking (`bool`): + Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + + Returns: + AsyncIterator[StreamResponse]: stream of generated tokens + """ + # Validate parameters + parameters = Parameters( + details=True, + do_sample=do_sample, + max_new_tokens=max_new_tokens, + repetition_penalty=repetition_penalty, + return_full_text=return_full_text, + seed=seed, + stop=stop_sequences if stop_sequences is not None else [], + temperature=temperature, + top_k=top_k, + top_p=top_p, + watermark=watermarking, + ) + request = Request(inputs=prompt, stream=True, parameters=parameters) + + async with ClientSession(headers=self.headers, timeout=self.timeout) as session: + async with session.post(self.base_url, json=request.dict()) as resp: + + if resp.status != 200: + raise parse_error(resp.status, await resp.json()) + + # Parse ServerSentEvents + async for byte_payload in resp.content: + # Skip line + if byte_payload == b"\n": + continue + + payload = byte_payload.decode("utf-8") + + # Event data + if payload.startswith("data:"): + # Decode payload + json_payload = json.loads(payload.lstrip("data:").rstrip("/n")) + # Parse payload + try: + response = StreamResponse(**json_payload) + except ValidationError: + # If we failed to parse the payload, then it is an error payload + raise parse_error(resp.status, json_payload) + yield response diff --git a/clients/python/text_generation/errors.py b/clients/python/text_generation/errors.py new file mode 100644 index 00000000..dbf0b761 --- /dev/null +++ b/clients/python/text_generation/errors.py @@ -0,0 +1,106 @@ +from typing import Dict + + +# Text Generation Inference Errors +class ValidationError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +class GenerationError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +class OverloadedError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +class IncompleteGenerationError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +# API Inference Errors +class BadRequestError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +class ShardNotReadyError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +class ShardTimeoutError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +class NotFoundError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +class RateLimitExceededError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +class NotSupportedError(Exception): + def __init__(self, model_id: str): + message = ( + f"Model `{model_id}` is not available for inference with this client. \n" + "Use `huggingface_hub.inference_api.InferenceApi` instead." + ) + super(NotSupportedError, self).__init__(message) + + +# Unknown error +class UnknownError(Exception): + def __init__(self, message: str): + super().__init__(message) + + +def parse_error(status_code: int, payload: Dict[str, str]) -> Exception: + """ + Parse error given an HTTP status code and a json payload + + Args: + status_code (`int`): + HTTP status code + payload (`Dict[str, str]`): + Json payload + + Returns: + Exception: parsed exception + + """ + # Try to parse a Text Generation Inference error + message = payload["error"] + if "error_type" in payload: + error_type = payload["error_type"] + if error_type == "generation": + return GenerationError(message) + if error_type == "incomplete_generation": + return IncompleteGenerationError(message) + if error_type == "overloaded": + return OverloadedError(message) + if error_type == "validation": + return ValidationError(message) + + # Try to parse a APIInference error + if status_code == 400: + return BadRequestError(message) + if status_code == 403 or status_code == 424: + return ShardNotReadyError(message) + if status_code == 504: + return ShardTimeoutError(message) + if status_code == 404: + return NotFoundError(message) + if status_code == 429: + return RateLimitExceededError(message) + + # Fallback to an unknown error + return UnknownError(message) diff --git a/clients/python/text_generation/inference_api.py b/clients/python/text_generation/inference_api.py new file mode 100644 index 00000000..bc6022b3 --- /dev/null +++ b/clients/python/text_generation/inference_api.py @@ -0,0 +1,150 @@ +import os +import requests +import base64 +import json +import warnings + +from typing import List, Optional +from huggingface_hub.utils import build_hf_headers + +from text_generation import Client, AsyncClient, __version__ +from text_generation.errors import NotSupportedError + +INFERENCE_ENDPOINT = os.environ.get( + "HF_INFERENCE_ENDPOINT", "https://api-inference.huggingface.co" +) + +SUPPORTED_MODELS = None + + +def get_supported_models() -> Optional[List[str]]: + """ + Get the list of supported text-generation models from GitHub + + Returns: + Optional[List[str]]: supported models list or None if unable to get the list from GitHub + """ + global SUPPORTED_MODELS + if SUPPORTED_MODELS is not None: + return SUPPORTED_MODELS + + response = requests.get( + "https://api.github.com/repos/huggingface/text-generation-inference/contents/supported_models.json", + timeout=5, + ) + if response.status_code == 200: + file_content = response.json()["content"] + SUPPORTED_MODELS = json.loads(base64.b64decode(file_content).decode("utf-8")) + return SUPPORTED_MODELS + + warnings.warn("Could not retrieve list of supported models.") + return None + + +class InferenceAPIClient(Client): + """Client to make calls to the HuggingFace Inference API. + + Only supports a subset of the available text-generation or text2text-generation models that are served using + text-generation-inference + + Example: + + ```python + >>> from text_generation import InferenceAPIClient + + >>> client = InferenceAPIClient("bigscience/bloomz") + >>> client.generate("Why is the sky blue?").generated_text + ' Rayleigh scattering' + + >>> result = "" + >>> for response in client.generate_stream("Why is the sky blue?"): + >>> if not response.token.special: + >>> result += response.token.text + >>> result + ' Rayleigh scattering' + ``` + """ + + def __init__(self, repo_id: str, token: Optional[str] = None, timeout: int = 10): + """ + Init headers and API information + + Args: + repo_id (`str`): + Id of repository (e.g. `bigscience/bloom`). + token (`str`, `optional`): + The API token to use as HTTP bearer authorization. This is not + the authentication token. You can find the token in + https://huggingface.co/settings/token. Alternatively, you can + find both your organizations and personal API tokens using + `HfApi().whoami(token)`. + timeout (`int`): + Timeout in seconds + """ + + # Text Generation Inference client only supports a subset of the available hub models + supported_models = get_supported_models() + if supported_models is not None and repo_id not in supported_models: + raise NotSupportedError(repo_id) + + headers = build_hf_headers( + token=token, library_name="text-generation", library_version=__version__ + ) + base_url = f"{INFERENCE_ENDPOINT}/models/{repo_id}" + + super(InferenceAPIClient, self).__init__(base_url, headers, timeout) + + +class InferenceAPIAsyncClient(AsyncClient): + """Aynschronous Client to make calls to the HuggingFace Inference API. + + Only supports a subset of the available text-generation or text2text-generation models that are served using + text-generation-inference + + Example: + + ```python + >>> from text_generation import InferenceAPIAsyncClient + + >>> client = InferenceAPIAsyncClient("bigscience/bloomz") + >>> response = await client.generate("Why is the sky blue?") + >>> response.generated_text + ' Rayleigh scattering' + + >>> result = "" + >>> async for response in client.generate_stream("Why is the sky blue?"): + >>> if not response.token.special: + >>> result += response.token.text + >>> result + ' Rayleigh scattering' + ``` + """ + + def __init__(self, repo_id: str, token: Optional[str] = None, timeout: int = 10): + """ + Init headers and API information + + Args: + repo_id (`str`): + Id of repository (e.g. `bigscience/bloom`). + token (`str`, `optional`): + The API token to use as HTTP bearer authorization. This is not + the authentication token. You can find the token in + https://huggingface.co/settings/token. Alternatively, you can + find both your organizations and personal API tokens using + `HfApi().whoami(token)`. + timeout (`int`): + Timeout in seconds + """ + + # Text Generation Inference client only supports a subset of the available hub models + supported_models = get_supported_models() + if supported_models is not None and repo_id not in supported_models: + raise NotSupportedError(repo_id) + + headers = build_hf_headers( + token=token, library_name="text-generation", library_version=__version__ + ) + base_url = f"{INFERENCE_ENDPOINT}/models/{repo_id}" + + super(InferenceAPIAsyncClient, self).__init__(base_url, headers, timeout) diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py new file mode 100644 index 00000000..d276b60e --- /dev/null +++ b/clients/python/text_generation/types.py @@ -0,0 +1,99 @@ +from enum import Enum +from pydantic import BaseModel, validator +from typing import Optional, List + +from text_generation.errors import ValidationError + + +class Parameters(BaseModel): + do_sample: bool = False + max_new_tokens: int = 20 + repetition_penalty: Optional[float] = None + return_full_text: bool = False + stop: List[str] = [] + seed: Optional[int] + temperature: Optional[float] + top_k: Optional[int] + top_p: Optional[float] + watermark: bool = False + details: bool = False + + @validator("repetition_penalty") + def valid_repetition_penalty(cls, v): + if v is not None and v is v <= 0: + raise ValidationError("`repetition_penalty` must be strictly positive") + return v + + @validator("seed") + def valid_seed(cls, v): + if v is not None and v is v < 0: + raise ValidationError("`seed` must be positive") + return v + + @validator("temperature") + def valid_temp(cls, v): + if v is not None and v <= 0: + raise ValidationError("`temperature` must be strictly positive") + return v + + @validator("top_k") + def valid_top_k(cls, v): + if v is not None and v <= 0: + raise ValidationError("`top_k` must be strictly positive") + return v + + @validator("top_p") + def valid_top_p(cls, v): + if v is not None and (v <= 0 or v > 1.0): + raise ValidationError("`top_p` must be > 0.0 and <= 1.0") + return v + + +class Request(BaseModel): + inputs: str + parameters: Parameters + stream: bool = False + + +class PrefillToken(BaseModel): + id: int + text: str + logprob: Optional[float] + + +class Token(BaseModel): + id: int + text: str + logprob: float + special: bool + + +class FinishReason(Enum): + Length = "length" + EndOfSequenceToken = "eos_token" + StopSequence = "stop_sequence" + + +class Details(BaseModel): + finish_reason: FinishReason + generated_tokens: int + seed: Optional[int] + prefill: List[PrefillToken] + tokens: List[Token] + + +class StreamDetails(BaseModel): + finish_reason: FinishReason + generated_tokens: int + seed: Optional[int] + + +class Response(BaseModel): + generated_text: str + details: Details + + +class StreamResponse(BaseModel): + token: Token + generated_text: Optional[str] + details: Optional[StreamDetails] diff --git a/router/src/infer.rs b/router/src/infer.rs index 598c2fcf..d0964f97 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -439,3 +439,14 @@ pub enum InferError { #[error("Incomplete generation")] IncompleteGeneration, } + +impl InferError { + pub(crate) fn error_type(&self) -> &str { + match self { + InferError::GenerationError(_) => "generation", + InferError::Overloaded(_) => "overloaded", + InferError::ValidationError(_) => "validation", + InferError::IncompleteGeneration => "incomplete_generation", + } + } +} diff --git a/router/src/lib.rs b/router/src/lib.rs index 1386f6b5..1819541d 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -152,8 +152,8 @@ pub(crate) struct Details { pub generated_tokens: u32, #[schema(example = 42)] pub seed: Option, - pub prefill: Option>, - pub tokens: Option>, + pub prefill: Vec, + pub tokens: Vec, } #[derive(Serialize, ToSchema)] @@ -185,6 +185,6 @@ pub(crate) struct StreamResponse { #[derive(Serialize, ToSchema)] pub(crate) struct ErrorResponse { - #[schema(inline)] pub error: String, + pub error_type: String, } diff --git a/router/src/server.rs b/router/src/server.rs index 247c55b3..b8b6b440 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -133,8 +133,8 @@ async fn generate( true => Some(Details { finish_reason: FinishReason::from(response.generated_text.finish_reason), generated_tokens: response.generated_text.generated_tokens, - prefill: Some(response.prefill), - tokens: Some(response.tokens), + prefill: response.prefill, + tokens: response.tokens, seed: response.generated_text.seed, }), false => None, @@ -554,6 +554,7 @@ impl From for (StatusCode, Json) { status_code, Json(ErrorResponse { error: err.to_string(), + error_type: err.error_type().to_string(), }), ) } @@ -564,6 +565,7 @@ impl From for Event { Event::default() .json_data(ErrorResponse { error: err.to_string(), + error_type: err.error_type().to_string(), }) .unwrap() } diff --git a/router/src/validation.rs b/router/src/validation.rs index 29c2967e..43f8b0ff 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -271,23 +271,23 @@ pub(crate) struct ValidGenerateRequest { #[derive(Error, Debug)] pub enum ValidationError { - #[error("temperature must be strictly positive")] + #[error("`temperature` must be strictly positive")] Temperature, - #[error("repetition_penalty must be strictly positive")] + #[error("`repetition_penalty` must be strictly positive")] RepetitionPenalty, - #[error("top_p must be > 0.0 and <= 1.0")] + #[error("`top_p` must be > 0.0 and <= 1.0")] TopP, - #[error("top_k must be strictly positive")] + #[error("`top_k` must be strictly positive")] TopK, - #[error("max_new_tokens must be strictly positive")] + #[error("`max_new_tokens` must be strictly positive")] MaxNewTokens, - #[error("input tokens + max_new_tokens must be <= {0}. Given: {1} input tokens and {2} max_new_tokens")] + #[error("`inputs` tokens + `max_new_tokens` must be <= {0}. Given: {1} `inputs` tokens and {2} `max_new_tokens`")] MaxTotalTokens(usize, usize, u32), - #[error("inputs must have less than {0} tokens. Given: {1}")] + #[error("`inputs` must have less than {0} tokens. Given: {1}")] InputLength(usize, usize), - #[error("inputs cannot be empty")] + #[error("`inputs` cannot be empty")] EmptyInput, - #[error("stop supports up to {0} stop sequences. Given: {1}")] + #[error("`stop` supports up to {0} stop sequences. Given: {1}")] StopSequence(usize, usize), #[error("tokenizer error {0}")] Tokenizer(String), diff --git a/server/.gitignore b/server/.gitignore index 5758ba92..aef74bb4 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -1,7 +1,7 @@ # Byte-compiled / optimized / DLL files __pycache__/ -text_generation/__pycache__/ -text_generation/pb/__pycache__/ +text_generation_server/__pycache__/ +text_generation_server/pb/__pycache__/ *.py[cod] *$py.class diff --git a/server/Makefile b/server/Makefile index 3926cf5d..3de13e8e 100644 --- a/server/Makefile +++ b/server/Makefile @@ -3,10 +3,10 @@ transformers_commit := 2f87dca1ca3e5663d0637da9bb037a6956e57a5e gen-server: # Compile protos pip install grpcio-tools==1.51.1 --no-cache-dir - mkdir text_generation/pb || true - python -m grpc_tools.protoc -I../proto --python_out=text_generation/pb --grpc_python_out=text_generation/pb ../proto/generate.proto - find text_generation/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; - touch text_generation/pb/__init__.py + mkdir text_generation_server/pb || true + python -m grpc_tools.protoc -I../proto --python_out=text_generation_server/pb --grpc_python_out=text_generation_server/pb ../proto/generate.proto + find text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; + touch text_generation_server/pb/__init__.py install-transformers: # Install specific version of transformers with custom cuda kernels @@ -28,4 +28,4 @@ install: gen-server install-torch install-transformers pip install -e . --no-cache-dir run-dev: - SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation/cli.py serve bigscience/bloom-560m --sharded \ No newline at end of file + SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded \ No newline at end of file diff --git a/server/pyproject.toml b/server/pyproject.toml index 21277939..b8e188b7 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,11 +1,11 @@ [tool.poetry] -name = "text-generation" +name = "text-generation-server" version = "0.3.2" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] [tool.poetry.scripts] -text-generation-server = 'text_generation.cli:app' +text-generation-server = 'text_generation_server.cli:app' [tool.poetry.dependencies] python = "^3.9" diff --git a/server/tests/conftest.py b/server/tests/conftest.py index 9fae8ee1..04c909ef 100644 --- a/server/tests/conftest.py +++ b/server/tests/conftest.py @@ -1,6 +1,6 @@ import pytest -from text_generation.pb import generate_pb2 +from text_generation_server.pb import generate_pb2 @pytest.fixture diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index bc36276a..90239f95 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -4,9 +4,9 @@ from copy import copy from transformers import AutoTokenizer -from text_generation.pb import generate_pb2 -from text_generation.models.causal_lm import CausalLMBatch -from text_generation.models.bloom import BloomCausalLMBatch, BLOOM +from text_generation_server.pb import generate_pb2 +from text_generation_server.models.causal_lm import CausalLMBatch +from text_generation_server.models.bloom import BloomCausalLMBatch, BLOOM @pytest.fixture(scope="session") diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 723017cd..869022fa 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -4,8 +4,8 @@ from copy import copy from transformers import AutoTokenizer -from text_generation.pb import generate_pb2 -from text_generation.models.causal_lm import CausalLM, CausalLMBatch +from text_generation_server.pb import generate_pb2 +from text_generation_server.models.causal_lm import CausalLM, CausalLMBatch @pytest.fixture(scope="session") diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index 1596e413..d089def3 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -1,8 +1,8 @@ import pytest -from text_generation.pb import generate_pb2 -from text_generation.models.causal_lm import CausalLMBatch -from text_generation.models.santacoder import SantaCoder +from text_generation_server.pb import generate_pb2 +from text_generation_server.models.causal_lm import CausalLMBatch +from text_generation_server.models.santacoder import SantaCoder @pytest.fixture(scope="session") diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index f7173392..764f8f83 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -5,8 +5,8 @@ from transformers import AutoTokenizer -from text_generation.pb import generate_pb2 -from text_generation.models.seq2seq_lm import Seq2SeqLM, Seq2SeqLMBatch +from text_generation_server.pb import generate_pb2 +from text_generation_server.models.seq2seq_lm import Seq2SeqLM, Seq2SeqLMBatch @pytest.fixture(scope="session") diff --git a/server/tests/utils/test_convert.py b/server/tests/utils/test_convert.py index 5f284be5..7dfe6a1e 100644 --- a/server/tests/utils/test_convert.py +++ b/server/tests/utils/test_convert.py @@ -1,6 +1,10 @@ -from text_generation.utils.hub import download_weights, weight_hub_files, weight_files +from text_generation_server.utils.hub import ( + download_weights, + weight_hub_files, + weight_files, +) -from text_generation.utils.convert import convert_files +from text_generation_server.utils.convert import convert_files def test_convert_files(): diff --git a/server/tests/utils/test_hub.py b/server/tests/utils/test_hub.py index b3120160..fac9a64d 100644 --- a/server/tests/utils/test_hub.py +++ b/server/tests/utils/test_hub.py @@ -1,6 +1,6 @@ import pytest -from text_generation.utils.hub import ( +from text_generation_server.utils.hub import ( weight_hub_files, download_weights, weight_files, diff --git a/server/tests/utils/test_tokens.py b/server/tests/utils/test_tokens.py index 7eca482f..3883ad97 100644 --- a/server/tests/utils/test_tokens.py +++ b/server/tests/utils/test_tokens.py @@ -1,4 +1,4 @@ -from text_generation.utils.tokens import ( +from text_generation_server.utils.tokens import ( StopSequenceCriteria, StoppingCriteria, FinishReason, diff --git a/server/text_generation/__init__.py b/server/text_generation_server/__init__.py similarity index 100% rename from server/text_generation/__init__.py rename to server/text_generation_server/__init__.py diff --git a/server/text_generation/cache.py b/server/text_generation_server/cache.py similarity index 90% rename from server/text_generation/cache.py rename to server/text_generation_server/cache.py index 5a3a8d31..72dc4857 100644 --- a/server/text_generation/cache.py +++ b/server/text_generation_server/cache.py @@ -1,6 +1,6 @@ from typing import Dict, Optional, TypeVar -from text_generation.models.types import Batch +from text_generation_server.models.types import Batch B = TypeVar("B", bound=Batch) diff --git a/server/text_generation/cli.py b/server/text_generation_server/cli.py similarity index 94% rename from server/text_generation/cli.py rename to server/text_generation_server/cli.py index 678dce16..6308ef6b 100644 --- a/server/text_generation/cli.py +++ b/server/text_generation_server/cli.py @@ -6,8 +6,8 @@ from loguru import logger from typing import Optional -from text_generation import server, utils -from text_generation.tracing import setup_tracing +from text_generation_server import server, utils +from text_generation_server.tracing import setup_tracing app = typer.Typer() @@ -42,7 +42,7 @@ def serve( logger.add( sys.stdout, format="{message}", - filter="text_generation", + filter="text_generation_server", level=logger_level, serialize=json_output, backtrace=True, @@ -68,7 +68,7 @@ def download_weights( logger.add( sys.stdout, format="{message}", - filter="text_generation", + filter="text_generation_server", level=logger_level, serialize=json_output, backtrace=True, diff --git a/server/text_generation/interceptor.py b/server/text_generation_server/interceptor.py similarity index 100% rename from server/text_generation/interceptor.py rename to server/text_generation_server/interceptor.py diff --git a/server/text_generation/models/__init__.py b/server/text_generation_server/models/__init__.py similarity index 79% rename from server/text_generation/models/__init__.py rename to server/text_generation_server/models/__init__.py index 386b7dc9..0035c1c6 100644 --- a/server/text_generation/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -3,14 +3,14 @@ from transformers import AutoConfig from typing import Optional -from text_generation.models.model import Model -from text_generation.models.causal_lm import CausalLM -from text_generation.models.bloom import BLOOM, BLOOMSharded -from text_generation.models.seq2seq_lm import Seq2SeqLM -from text_generation.models.galactica import Galactica, GalacticaSharded -from text_generation.models.santacoder import SantaCoder -from text_generation.models.gpt_neox import GPTNeox, GPTNeoxSharded -from text_generation.models.t5 import T5Sharded +from text_generation_server.models.model import Model +from text_generation_server.models.causal_lm import CausalLM +from text_generation_server.models.bloom import BLOOM, BLOOMSharded +from text_generation_server.models.seq2seq_lm import Seq2SeqLM +from text_generation_server.models.galactica import Galactica, GalacticaSharded +from text_generation_server.models.santacoder import SantaCoder +from text_generation_server.models.gpt_neox import GPTNeox, GPTNeoxSharded +from text_generation_server.models.t5 import T5Sharded __all__ = [ "Model", diff --git a/server/text_generation/models/bloom.py b/server/text_generation_server/models/bloom.py similarity index 97% rename from server/text_generation/models/bloom.py rename to server/text_generation_server/models/bloom.py index 83a0d63e..0d83abe2 100644 --- a/server/text_generation/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -17,10 +17,10 @@ TensorParallelRowLinear, ) -from text_generation.models import CausalLM -from text_generation.models.causal_lm import CausalLMBatch -from text_generation.pb import generate_pb2 -from text_generation.utils import ( +from text_generation_server.models import CausalLM +from text_generation_server.models.causal_lm import CausalLMBatch +from text_generation_server.pb import generate_pb2 +from text_generation_server.utils import ( initialize_torch_distributed, weight_files, ) diff --git a/server/text_generation/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py similarity index 98% rename from server/text_generation/models/causal_lm.py rename to server/text_generation_server/models/causal_lm.py index 23c94ddf..4cfc15b9 100644 --- a/server/text_generation/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -5,10 +5,15 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type -from text_generation.models import Model -from text_generation.models.types import Batch, PrefillTokens, Generation, GeneratedText -from text_generation.pb import generate_pb2 -from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling +from text_generation_server.models import Model +from text_generation_server.models.types import ( + Batch, + PrefillTokens, + Generation, + GeneratedText, +) +from text_generation_server.pb import generate_pb2 +from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling tracer = trace.get_tracer(__name__) diff --git a/server/text_generation/models/galactica.py b/server/text_generation_server/models/galactica.py similarity index 98% rename from server/text_generation/models/galactica.py rename to server/text_generation_server/models/galactica.py index 9a71c5d3..b4d1c553 100644 --- a/server/text_generation/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -18,10 +18,10 @@ TensorParallelRowLinear, ) -from text_generation.models import CausalLM -from text_generation.pb import generate_pb2 -from text_generation.models.causal_lm import CausalLMBatch -from text_generation.utils import ( +from text_generation_server.models import CausalLM +from text_generation_server.pb import generate_pb2 +from text_generation_server.models.causal_lm import CausalLMBatch +from text_generation_server.utils import ( NextTokenChooser, StoppingCriteria, initialize_torch_distributed, diff --git a/server/text_generation/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py similarity index 99% rename from server/text_generation/models/gpt_neox.py rename to server/text_generation_server/models/gpt_neox.py index 0197f976..5e1960f4 100644 --- a/server/text_generation/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -16,8 +16,8 @@ TensorParallelRowLinear, ) -from text_generation.models import CausalLM -from text_generation.utils import ( +from text_generation_server.models import CausalLM +from text_generation_server.utils import ( initialize_torch_distributed, weight_files, ) diff --git a/server/text_generation/models/model.py b/server/text_generation_server/models/model.py similarity index 95% rename from server/text_generation/models/model.py rename to server/text_generation_server/models/model.py index 09fa6a2a..e0ce6686 100644 --- a/server/text_generation/models/model.py +++ b/server/text_generation_server/models/model.py @@ -4,7 +4,7 @@ from typing import List, Tuple, Optional, TypeVar, Type from transformers import PreTrainedTokenizerBase -from text_generation.models.types import Batch, GeneratedText +from text_generation_server.models.types import Batch, GeneratedText B = TypeVar("B", bound=Batch) diff --git a/server/text_generation/models/santacoder.py b/server/text_generation_server/models/santacoder.py similarity index 97% rename from server/text_generation/models/santacoder.py rename to server/text_generation_server/models/santacoder.py index 5d271c85..fe15cde0 100644 --- a/server/text_generation/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -4,7 +4,7 @@ from typing import Optional, List from transformers import AutoTokenizer, AutoModelForCausalLM -from text_generation.models import CausalLM +from text_generation_server.models import CausalLM FIM_PREFIX = "" FIM_MIDDLE = "" diff --git a/server/text_generation/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py similarity index 97% rename from server/text_generation/models/seq2seq_lm.py rename to server/text_generation_server/models/seq2seq_lm.py index 4b88baec..73a10879 100644 --- a/server/text_generation/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -5,10 +5,15 @@ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type -from text_generation.models import Model -from text_generation.models.types import GeneratedText, Batch, Generation, PrefillTokens -from text_generation.pb import generate_pb2 -from text_generation.utils import NextTokenChooser, StoppingCriteria, Sampling +from text_generation_server.models import Model +from text_generation_server.models.types import ( + GeneratedText, + Batch, + Generation, + PrefillTokens, +) +from text_generation_server.pb import generate_pb2 +from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling tracer = trace.get_tracer(__name__) @@ -45,7 +50,7 @@ class Seq2SeqLMBatch(Batch): padding_right_offset: int def to_pb(self) -> generate_pb2.Batch: - """Convert a Seq2SeqLMBatch to a text_generation.v1.Batch protobuf""" + """Convert a Seq2SeqLMBatch to a text_generation_server.v1.Batch protobuf""" return generate_pb2.Batch( id=self.batch_id, requests=self.requests, @@ -59,7 +64,7 @@ def from_pb( tokenizer: PreTrainedTokenizerBase, device: torch.device, ) -> "Seq2SeqLMBatch": - """Convert a text_generation.v1.Batch protobuf to a Seq2SeqLMBatch""" + """Convert a text_generation_server.v1.Batch protobuf to a Seq2SeqLMBatch""" inputs = [] next_token_choosers = [] stopping_criterias = [] diff --git a/server/text_generation/models/t5.py b/server/text_generation_server/models/t5.py similarity index 98% rename from server/text_generation/models/t5.py rename to server/text_generation_server/models/t5.py index 55507539..cb4f7f22 100644 --- a/server/text_generation/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -16,8 +16,8 @@ TensorParallelRowLinear, ) -from text_generation.models import Seq2SeqLM -from text_generation.utils import ( +from text_generation_server.models import Seq2SeqLM +from text_generation_server.utils import ( initialize_torch_distributed, weight_files, ) diff --git a/server/text_generation/models/types.py b/server/text_generation_server/models/types.py similarity index 95% rename from server/text_generation/models/types.py rename to server/text_generation_server/models/types.py index a3fbd6e8..93c3b9db 100644 --- a/server/text_generation/models/types.py +++ b/server/text_generation_server/models/types.py @@ -6,8 +6,8 @@ from transformers import PreTrainedTokenizerBase -from text_generation.pb import generate_pb2 -from text_generation.pb.generate_pb2 import FinishReason +from text_generation_server.pb import generate_pb2 +from text_generation_server.pb.generate_pb2 import FinishReason class Batch(ABC): diff --git a/server/text_generation/pb/.gitignore b/server/text_generation_server/pb/.gitignore similarity index 100% rename from server/text_generation/pb/.gitignore rename to server/text_generation_server/pb/.gitignore diff --git a/server/text_generation/server.py b/server/text_generation_server/server.py similarity index 92% rename from server/text_generation/server.py rename to server/text_generation_server/server.py index f3129cb4..0b75c3c7 100644 --- a/server/text_generation/server.py +++ b/server/text_generation_server/server.py @@ -9,11 +9,11 @@ from pathlib import Path from typing import List, Optional -from text_generation.cache import Cache -from text_generation.interceptor import ExceptionInterceptor -from text_generation.models import Model, get_model -from text_generation.pb import generate_pb2_grpc, generate_pb2 -from text_generation.tracing import UDSOpenTelemetryAioServerInterceptor +from text_generation_server.cache import Cache +from text_generation_server.interceptor import ExceptionInterceptor +from text_generation_server.models import Model, get_model +from text_generation_server.pb import generate_pb2_grpc, generate_pb2 +from text_generation_server.tracing import UDSOpenTelemetryAioServerInterceptor class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): diff --git a/server/text_generation/tracing.py b/server/text_generation_server/tracing.py similarity index 100% rename from server/text_generation/tracing.py rename to server/text_generation_server/tracing.py diff --git a/server/text_generation/utils/__init__.py b/server/text_generation_server/utils/__init__.py similarity index 71% rename from server/text_generation/utils/__init__.py rename to server/text_generation_server/utils/__init__.py index a390b710..50d64518 100644 --- a/server/text_generation/utils/__init__.py +++ b/server/text_generation_server/utils/__init__.py @@ -1,6 +1,6 @@ -from text_generation.utils.convert import convert_file, convert_files -from text_generation.utils.dist import initialize_torch_distributed -from text_generation.utils.hub import ( +from text_generation_server.utils.convert import convert_file, convert_files +from text_generation_server.utils.dist import initialize_torch_distributed +from text_generation_server.utils.hub import ( weight_files, weight_hub_files, download_weights, @@ -8,7 +8,7 @@ LocalEntryNotFoundError, RevisionNotFoundError, ) -from text_generation.utils.tokens import ( +from text_generation_server.utils.tokens import ( Greedy, NextTokenChooser, Sampling, diff --git a/server/text_generation/utils/convert.py b/server/text_generation_server/utils/convert.py similarity index 100% rename from server/text_generation/utils/convert.py rename to server/text_generation_server/utils/convert.py diff --git a/server/text_generation/utils/dist.py b/server/text_generation_server/utils/dist.py similarity index 100% rename from server/text_generation/utils/dist.py rename to server/text_generation_server/utils/dist.py diff --git a/server/text_generation/utils/hub.py b/server/text_generation_server/utils/hub.py similarity index 100% rename from server/text_generation/utils/hub.py rename to server/text_generation_server/utils/hub.py diff --git a/server/text_generation/utils/tokens.py b/server/text_generation_server/utils/tokens.py similarity index 96% rename from server/text_generation/utils/tokens.py rename to server/text_generation_server/utils/tokens.py index 797a5634..41dfabd3 100644 --- a/server/text_generation/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -11,9 +11,9 @@ ) from typing import List, Tuple, Optional -from text_generation.pb import generate_pb2 -from text_generation.pb.generate_pb2 import FinishReason -from text_generation.utils.watermark import WatermarkLogitsProcessor +from text_generation_server.pb import generate_pb2 +from text_generation_server.pb.generate_pb2 import FinishReason +from text_generation_server.utils.watermark import WatermarkLogitsProcessor class Sampling: diff --git a/server/text_generation/utils/watermark.py b/server/text_generation_server/utils/watermark.py similarity index 100% rename from server/text_generation/utils/watermark.py rename to server/text_generation_server/utils/watermark.py From b1485e18c52df527e49a0a9636e3dd4ebbb0d598 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 7 Mar 2023 20:05:21 +0100 Subject: [PATCH 107/793] fix(server): fix galactica batch (#106) closes #105 --- .../models/galactica.py | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index b4d1c553..484621d3 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -96,6 +96,8 @@ def from_pb( input_lengths = [] # Parse batch + max_sequence_length = 0 + padding_right_offset = 0 for r in pb.requests: # Add escape_custom_split_sequence to the CausalLMBatch logic inputs.append(escape_custom_split_sequence(r.inputs)) @@ -103,8 +105,13 @@ def from_pb( next_token_choosers.append( NextTokenChooser.from_pb(r.parameters, len(tokenizer), device) ) - stopping_criterias.append( - StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) + stopping_criteria = StoppingCriteria.from_pb( + r.stopping_parameters, tokenizer + ) + stopping_criterias.append(stopping_criteria) + max_sequence_length = max(max_sequence_length, r.input_length) + padding_right_offset = max( + padding_right_offset, stopping_criteria.max_new_tokens ) # Tokenize batch @@ -114,6 +121,14 @@ def from_pb( padding=True, return_token_type_ids=False, ).to(device) + input_ids = tokenized_inputs["input_ids"] + # Allocate maximum attention_mask + attention_mask = input_ids.new_zeros( + (pb.size, max_sequence_length + padding_right_offset) + ) + # Copy tokenizer attention_mask into fully allocated attention_mask + attention_mask[:, :max_sequence_length] = tokenized_inputs["attention_mask"] + position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) @@ -121,8 +136,8 @@ def from_pb( return cls( batch_id=pb.id, requests=pb.requests, - input_ids=tokenized_inputs["input_ids"], - attention_mask=tokenized_inputs["attention_mask"], + input_ids=input_ids, + attention_mask=attention_mask, position_ids=position_ids, past_key_values=None, all_input_ids=all_input_ids, @@ -130,7 +145,8 @@ def from_pb( next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=pb.size, - max_sequence_length=max(input_lengths), + max_sequence_length=max_sequence_length, + padding_right_offset=padding_right_offset, ) From 0ac38d336a870bd4f09e18a9b62d64d78b032fbb Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 8 Mar 2023 11:06:59 +0100 Subject: [PATCH 108/793] feat(launcher): allow parsing num_shard from CUDA_VISIBLE_DEVICES (#107) --- clients/python/README.md | 130 ++++++++++++++++++++- clients/python/pyproject.toml | 2 +- clients/python/text_generation/__init__.py | 2 +- clients/python/text_generation/client.py | 24 ++-- launcher/src/main.rs | 50 +++++++- 5 files changed, 190 insertions(+), 18 deletions(-) diff --git a/clients/python/README.md b/clients/python/README.md index 414360bf..0f0b32f0 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -1,7 +1,8 @@ # Text Generation The Hugging Face Text Generation Python library provides a convenient way of interfacing with a -`text-generation-inference` instance running on your own infrastructure or on the Hugging Face Hub. +`text-generation-inference` instance running on +[Hugging Face Inference Endpoints](https://huggingface.co/inference-endpoints) or on the Hugging Face Hub. ## Get Started @@ -11,7 +12,7 @@ The Hugging Face Text Generation Python library provides a convenient way of int pip install text-generation ``` -### Usage +### Inference API Usage ```python from text_generation import InferenceAPIClient @@ -50,3 +51,128 @@ async for response in client.generate_stream("Why is the sky blue?"): print(text) # ' Rayleigh scattering' ``` + +### Hugging Fae Inference Endpoint usage + +```python +from text_generation import Client + +endpoint_url = "https://YOUR_ENDPOINT.endpoints.huggingface.cloud" + +client = Client(endpoint_url) +text = client.generate("Why is the sky blue?").generated_text +print(text) +# ' Rayleigh scattering' + +# Token Streaming +text = "" +for response in client.generate_stream("Why is the sky blue?"): + if not response.token.special: + text += response.token.text + +print(text) +# ' Rayleigh scattering' +``` + +or with the asynchronous client: + +```python +from text_generation import AsyncClient + +endpoint_url = "https://YOUR_ENDPOINT.endpoints.huggingface.cloud" + +client = AsyncClient(endpoint_url) +response = await client.generate("Why is the sky blue?") +print(response.generated_text) +# ' Rayleigh scattering' + +# Token Streaming +text = "" +async for response in client.generate_stream("Why is the sky blue?"): + if not response.token.special: + text += response.token.text + +print(text) +# ' Rayleigh scattering' +``` + +### Types + +```python +# Prompt tokens +class PrefillToken: + # Token ID from the model tokenizer + id: int + # Token text + text: str + # Logprob + # Optional since the logprob of the first token cannot be computed + logprob: Optional[float] + + +# Generated tokens +class Token: + # Token ID from the model tokenizer + id: int + # Token text + text: str + # Logprob + logprob: float + # Is the token a special token + # Can be used to ignore tokens when concatenating + special: bool + + +# Generation finish reason +class FinishReason(Enum): + # number of generated tokens == `max_new_tokens` + Length = "length" + # the model generated its end of sequence token + EndOfSequenceToken = "eos_token" + # the model generated a text included in `stop_sequences` + StopSequence = "stop_sequence" + + +# `generate` details +class Details: + # Generation finish reason + finish_reason: FinishReason + # Number of generated tokens + generated_tokens: int + # Sampling seed if sampling was activated + seed: Optional[int] + # Prompt tokens + prefill: List[PrefillToken] + # Generated tokens + tokens: List[Token] + + +# `generate` return value +class Response: + # Generated text + generated_text: str + # Generation details + details: Details + + +# `generate_stream` details +class StreamDetails: + # Generation finish reason + finish_reason: FinishReason + # Number of generated tokens + generated_tokens: int + # Sampling seed if sampling was activated + seed: Optional[int] + + +# `generate_stream` return value +class StreamResponse: + # Generated token + token: Token + # Complete generated text + # Only available when the generation is finished + generated_text: Optional[str] + # Generation details + # Only available when the generation is finished + details: Optional[StreamDetails] +``` \ No newline at end of file diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 6eb11638..4c7b24e6 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.1.0" +version = "0.2.0" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] diff --git a/clients/python/text_generation/__init__.py b/clients/python/text_generation/__init__.py index 88861b37..1f5e6ce5 100644 --- a/clients/python/text_generation/__init__.py +++ b/clients/python/text_generation/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.3.2" +__version__ = "0.2.0" from text_generation.client import Client, AsyncClient from text_generation.inference_api import InferenceAPIClient, InferenceAPIAsyncClient diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index d0a67915..3e9bbc36 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -63,7 +63,7 @@ def generate( temperature: Optional[float] = None, top_k: Optional[int] = None, top_p: Optional[float] = None, - watermarking: bool = False, + watermark: bool = False, ) -> Response: """ Given a prompt, generate the following text @@ -91,7 +91,7 @@ def generate( top_p (`float`): If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation. - watermarking (`bool`): + watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) Returns: @@ -109,7 +109,7 @@ def generate( temperature=temperature, top_k=top_k, top_p=top_p, - watermark=watermarking, + watermark=watermark, ) request = Request(inputs=prompt, stream=False, parameters=parameters) @@ -136,7 +136,7 @@ def generate_stream( temperature: Optional[float] = None, top_k: Optional[int] = None, top_p: Optional[float] = None, - watermarking: bool = False, + watermark: bool = False, ) -> Iterator[StreamResponse]: """ Given a prompt, generate the following stream of tokens @@ -164,7 +164,7 @@ def generate_stream( top_p (`float`): If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation. - watermarking (`bool`): + watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) Returns: @@ -182,7 +182,7 @@ def generate_stream( temperature=temperature, top_k=top_k, top_p=top_p, - watermark=watermarking, + watermark=watermark, ) request = Request(inputs=prompt, stream=True, parameters=parameters) @@ -268,7 +268,7 @@ async def generate( temperature: Optional[float] = None, top_k: Optional[int] = None, top_p: Optional[float] = None, - watermarking: bool = False, + watermark: bool = False, ) -> Response: """ Given a prompt, generate the following text asynchronously @@ -296,7 +296,7 @@ async def generate( top_p (`float`): If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation. - watermarking (`bool`): + watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) Returns: @@ -314,7 +314,7 @@ async def generate( temperature=temperature, top_k=top_k, top_p=top_p, - watermark=watermarking, + watermark=watermark, ) request = Request(inputs=prompt, stream=False, parameters=parameters) @@ -338,7 +338,7 @@ async def generate_stream( temperature: Optional[float] = None, top_k: Optional[int] = None, top_p: Optional[float] = None, - watermarking: bool = False, + watermark: bool = False, ) -> AsyncIterator[StreamResponse]: """ Given a prompt, generate the following stream of tokens asynchronously @@ -366,7 +366,7 @@ async def generate_stream( top_p (`float`): If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation. - watermarking (`bool`): + watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) Returns: @@ -384,7 +384,7 @@ async def generate_stream( temperature=temperature, top_k=top_k, top_p=top_p, - watermark=watermarking, + watermark=watermark, ) request = Request(inputs=prompt, stream=True, parameters=parameters) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 66dcb2db..96ad18f9 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -23,8 +23,10 @@ struct Args { model_id: String, #[clap(long, env)] revision: Option, - #[clap(default_value = "1", long, env)] - num_shard: usize, + #[clap(long, env)] + sharded: Option, + #[clap(long, env)] + num_shard: Option, #[clap(long, env)] quantize: bool, #[clap(default_value = "128", long, env)] @@ -80,6 +82,7 @@ fn main() -> ExitCode { let Args { model_id, revision, + sharded, num_shard, quantize, max_concurrent_requests, @@ -102,6 +105,49 @@ fn main() -> ExitCode { watermark_delta, } = args; + // get the number of shards given `sharded` and `num_shard` + let num_shard = if let Some(sharded) = sharded { + // sharded is set + match sharded { + // sharded is set and true + true => { + match num_shard { + None => { + // try to default to the number of available GPUs + tracing::info!("Parsing num_shard from CUDA_VISIBLE_DEVICES"); + let cuda_visible_devices = env::var("CUDA_VISIBLE_DEVICES") + .expect("--num-shard and CUDA_VISIBLE_DEVICES are not set"); + let n_devices = cuda_visible_devices.split(",").count(); + if n_devices <= 1 { + panic!("`sharded` is true but only found {n_devices} CUDA devices"); + } + tracing::info!("Sharding on {n_devices} found CUDA devices"); + n_devices + } + Some(num_shard) => { + // we can't have only one shard while sharded + if num_shard <= 1 { + panic!("`sharded` is true but `num_shard` <= 1"); + } + num_shard + } + } + } + // sharded is set and false + false => { + let num_shard = num_shard.unwrap_or(1); + // we can't have more than one shard while not sharded + if num_shard != 1 { + panic!("`sharded` is false but `num_shard` != 1"); + } + num_shard + } + } + } else { + // default to a single shard + num_shard.unwrap_or(1) + }; + // Signal handler let running = Arc::new(AtomicBool::new(true)); let r = running.clone(); From 5fd2dcb5139e16638069062432310bf400e910ff Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 8 Mar 2023 13:53:41 +0100 Subject: [PATCH 109/793] feat(launcher): default num_shard to CUDA_VISIBLE_DEVICES if possible (#108) --- launcher/src/main.rs | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 96ad18f9..1865cf90 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -115,13 +115,11 @@ fn main() -> ExitCode { None => { // try to default to the number of available GPUs tracing::info!("Parsing num_shard from CUDA_VISIBLE_DEVICES"); - let cuda_visible_devices = env::var("CUDA_VISIBLE_DEVICES") + let n_devices = num_cuda_devices() .expect("--num-shard and CUDA_VISIBLE_DEVICES are not set"); - let n_devices = cuda_visible_devices.split(",").count(); if n_devices <= 1 { panic!("`sharded` is true but only found {n_devices} CUDA devices"); } - tracing::info!("Sharding on {n_devices} found CUDA devices"); n_devices } Some(num_shard) => { @@ -144,9 +142,19 @@ fn main() -> ExitCode { } } } else { - // default to a single shard - num_shard.unwrap_or(1) + match num_shard { + // get num_shard from CUDA_VISIBLE_DEVICES or default to a single shard + None => num_cuda_devices().unwrap_or(1), + Some(num_shard) => num_shard, + } }; + if num_shard < 1 { + panic!("`num_shard` cannot be < 1"); + } + + if num_shard > 1 { + tracing::info!("Sharding model on {num_shard} processes"); + } // Signal handler let running = Arc::new(AtomicBool::new(true)); @@ -669,3 +677,11 @@ fn shutdown_shards(shutdown: Arc>, shutdown_receiver: &mpsc::Receive // This will block till all shutdown_sender are dropped let _ = shutdown_receiver.recv(); } + +fn num_cuda_devices() -> Option { + if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { + let n_devices = cuda_visible_devices.split(',').count(); + return Some(n_devices); + } + None +} From 2c5df5d2affc442f30a21ee628bfee89a3666cb5 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 8 Mar 2023 16:48:16 +0100 Subject: [PATCH 110/793] fix(python-client): stream not set on the sync client (#109) --- clients/python/README.md | 2 +- clients/python/pyproject.toml | 2 +- clients/python/text_generation/__init__.py | 2 +- clients/python/text_generation/client.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clients/python/README.md b/clients/python/README.md index 0f0b32f0..79b88377 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -52,7 +52,7 @@ print(text) # ' Rayleigh scattering' ``` -### Hugging Fae Inference Endpoint usage +### Hugging Face Inference Endpoint usage ```python from text_generation import Client diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 4c7b24e6..0b8fa8c7 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.2.0" +version = "0.2.1" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] diff --git a/clients/python/text_generation/__init__.py b/clients/python/text_generation/__init__.py index 1f5e6ce5..db09cdfa 100644 --- a/clients/python/text_generation/__init__.py +++ b/clients/python/text_generation/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.2.0" +__version__ = "0.2.1" from text_generation.client import Client, AsyncClient from text_generation.inference_api import InferenceAPIClient, InferenceAPIAsyncClient diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index 3e9bbc36..23655240 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -191,7 +191,7 @@ def generate_stream( json=request.dict(), headers=self.headers, timeout=self.timeout, - stream=False, + stream=True, ) if resp.status_code != 200: From 941cd42e0cd2d51bb37a6f84572ceda2976b890d Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 8 Mar 2023 18:29:08 +0100 Subject: [PATCH 111/793] fix(server): fix index out of range for watermarking (#110) --- server/text_generation_server/models/causal_lm.py | 2 +- server/text_generation_server/models/galactica.py | 2 +- server/text_generation_server/models/seq2seq_lm.py | 2 +- server/text_generation_server/utils/tokens.py | 5 +---- server/text_generation_server/utils/watermark.py | 10 ++++------ 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 4cfc15b9..ef3f0260 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -73,7 +73,7 @@ def from_pb( inputs.append(r.inputs) input_lengths.append(r.input_length) next_token_choosers.append( - NextTokenChooser.from_pb(r.parameters, len(tokenizer), device) + NextTokenChooser.from_pb(r.parameters, device) ) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 484621d3..d04a3bce 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -103,7 +103,7 @@ def from_pb( inputs.append(escape_custom_split_sequence(r.inputs)) input_lengths.append(r.input_length) next_token_choosers.append( - NextTokenChooser.from_pb(r.parameters, len(tokenizer), device) + NextTokenChooser.from_pb(r.parameters, device) ) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 73a10879..bece913a 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -83,7 +83,7 @@ def from_pb( decoder_input_ids.append(tokenizer.bos_token_id) decoder_input_lengths.append(1) next_token_choosers.append( - NextTokenChooser.from_pb(r.parameters, len(tokenizer), device) + NextTokenChooser.from_pb(r.parameters, device) ) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 41dfabd3..aa76b6eb 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -36,7 +36,6 @@ def __call__(self, logits): class NextTokenChooser: def __init__( self, - vocab_size, watermark=False, temperature=1.0, repetition_penalty=1.0, @@ -52,7 +51,7 @@ def __init__( sampling = do_sample if watermark: - warpers.append(WatermarkLogitsProcessor(vocab_size, device=device)) + warpers.append(WatermarkLogitsProcessor(device=device)) if repetition_penalty is not None and repetition_penalty != 1.0: warpers.append(RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty)) if temperature is not None and temperature != 1.0: @@ -85,11 +84,9 @@ def __call__(self, input_ids, scores): def from_pb( cls, pb: generate_pb2.NextTokenChooserParameters, - vocab_size: int, device: torch.device, ) -> "NextTokenChooser": return NextTokenChooser( - vocab_size=vocab_size, watermark=pb.watermark, temperature=pb.temperature, repetition_penalty=pb.repetition_penalty, diff --git a/server/text_generation_server/utils/watermark.py b/server/text_generation_server/utils/watermark.py index 4c42d2a1..cf6214ce 100644 --- a/server/text_generation_server/utils/watermark.py +++ b/server/text_generation_server/utils/watermark.py @@ -25,14 +25,12 @@ class WatermarkLogitsProcessor(LogitsProcessor): def __init__( self, - vocab_size: int, gamma: float = GAMMA, delta: float = DELTA, hash_key: int = 15485863, # just a large prime number to create a rng seed with sufficient bit width device: str = "cpu", ): # watermarking parameters - self.vocab_size = vocab_size self.gamma = gamma self.delta = delta self.rng = torch.Generator(device=device) @@ -45,13 +43,13 @@ def _seed_rng(self, input_ids: torch.LongTensor) -> None: prev_token = input_ids[-1].item() self.rng.manual_seed(self.hash_key * prev_token) - def _get_greenlist_ids(self, input_ids: torch.LongTensor) -> list[int]: + def _get_greenlist_ids(self, input_ids: torch.LongTensor, max_value: int) -> list[int]: # seed the rng using the previous tokens/prefix self._seed_rng(input_ids) - greenlist_size = int(self.vocab_size * self.gamma) + greenlist_size = int(max_value * self.gamma) vocab_permutation = torch.randperm( - self.vocab_size, device=input_ids.device, generator=self.rng + max_value, device=input_ids.device, generator=self.rng ) greenlist_ids = vocab_permutation[:greenlist_size] return greenlist_ids @@ -76,7 +74,7 @@ def __call__( self, input_ids: torch.LongTensor, scores: torch.FloatTensor ) -> torch.FloatTensor: assert len(input_ids) == 1 - greenlist_ids = self._get_greenlist_ids(input_ids[0]) + greenlist_ids = self._get_greenlist_ids(input_ids[0], scores.shape[-1]) green_tokens_mask = self._calc_greenlist_mask( scores=scores, greenlist_token_ids=greenlist_ids ) From 1a2d68250aa7dfbe1fa52b22eec07edfb7b895fb Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 9 Mar 2023 11:33:57 +0100 Subject: [PATCH 112/793] feat: support typical sampling (#114) closes #112 --- proto/generate.proto | 10 +++-- router/src/lib.rs | 10 +++++ router/src/queue.rs | 1 + router/src/server.rs | 1 + router/src/validation.rs | 44 +++++++++++++------ server/tests/conftest.py | 1 + server/text_generation_server/utils/tokens.py | 6 +++ 7 files changed, 55 insertions(+), 18 deletions(-) diff --git a/proto/generate.proto b/proto/generate.proto index dccd7e59..a47e2ec1 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -34,14 +34,16 @@ message NextTokenChooserParameters { uint32 top_k = 2; /// restricting to top tokens summing to prob_cut_off <= prob_cut_off float top_p = 3; + /// restricting to top tokens summing to prob_cut_off <= prob_cut_off + float typical_p = 4; /// apply sampling on the logits - bool do_sample = 4; + bool do_sample = 5; /// random seed for sampling - uint64 seed = 5; + uint64 seed = 6; /// repetition penalty - float repetition_penalty = 6; + float repetition_penalty = 7; /// token watermarking using "A Watermark for Large Language Models" - bool watermark = 7; + bool watermark = 8; } message StoppingCriteriaParameters { diff --git a/router/src/lib.rs b/router/src/lib.rs index 1819541d..d375eafb 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -41,6 +41,15 @@ pub(crate) struct GenerateParameters { )] pub top_p: Option, #[serde(default)] + #[schema( + exclusive_minimum = 0.0, + maximum = 1.0, + nullable = true, + default = "null", + example = 0.95 + )] + pub typical_p: Option, + #[serde(default)] #[schema(default = "false", example = true)] pub do_sample: bool, #[serde(default = "default_max_new_tokens")] @@ -72,6 +81,7 @@ fn default_parameters() -> GenerateParameters { repetition_penalty: None, top_k: None, top_p: None, + typical_p: None, do_sample: false, max_new_tokens: default_max_new_tokens(), return_full_text: None, diff --git a/router/src/queue.rs b/router/src/queue.rs index 0ebfed9b..db3c509e 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -231,6 +231,7 @@ mod tests { temperature: 0.0, top_k: 0, top_p: 0.0, + typical_p: 0.0, do_sample: false, seed: 0, repetition_penalty: 0.0, diff --git a/router/src/server.rs b/router/src/server.rs index b8b6b440..2ce5699d 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -68,6 +68,7 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json 1.0 { - return Err(ValidationError::TopP); - } - - // Different because the proto default value is 0 while it is not a valid value + // Different because the proto default value is not a valid value // for the user - let top_k: u32 = match top_k { - None => Ok(0), - Some(top_k) => { - if top_k <= 0 { + let top_p = top_p + .map(|value| { + if value <= 0.0 || value >= 1.0 { + return Err(ValidationError::TopP); + } + Ok(value) + }) + .unwrap_or(Ok(1.0))?; + + let typical_p = typical_p + .map(|value| { + if value <= 0.0 || value >= 1.0 { + return Err(ValidationError::TypicalP); + } + Ok(value) + }) + .unwrap_or(Ok(1.0))?; + + let top_k: u32 = top_k + .map(|value| { + if value <= 0 { return Err(ValidationError::TopK); } - Ok(top_k as u32) - } - }?; + Ok(value as u32) + }) + .unwrap_or(Ok(0))?; if max_new_tokens == 0 { return Err(ValidationError::MaxNewTokens); @@ -231,6 +244,7 @@ fn validate( repetition_penalty, top_k, top_p, + typical_p, do_sample, seed, watermark, @@ -275,10 +289,12 @@ pub enum ValidationError { Temperature, #[error("`repetition_penalty` must be strictly positive")] RepetitionPenalty, - #[error("`top_p` must be > 0.0 and <= 1.0")] + #[error("`top_p` must be > 0.0 and < 1.0")] TopP, #[error("`top_k` must be strictly positive")] TopK, + #[error("`typical_p` must be > 0.0 and < 1.0")] + TypicalP, #[error("`max_new_tokens` must be strictly positive")] MaxNewTokens, #[error("`inputs` tokens + `max_new_tokens` must be <= {0}. Given: {1} `inputs` tokens and {2} `max_new_tokens`")] diff --git a/server/tests/conftest.py b/server/tests/conftest.py index 04c909ef..16d2c408 100644 --- a/server/tests/conftest.py +++ b/server/tests/conftest.py @@ -10,6 +10,7 @@ def default_pb_parameters(): repetition_penalty=1.0, top_k=0, top_p=1.0, + typical_p=1.0, do_sample=False, ) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index aa76b6eb..15563761 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -6,6 +6,7 @@ TemperatureLogitsWarper, TopKLogitsWarper, TopPLogitsWarper, + TypicalLogitsWarper, RepetitionPenaltyLogitsProcessor, PreTrainedTokenizerBase, ) @@ -41,6 +42,7 @@ def __init__( repetition_penalty=1.0, top_k=None, top_p=None, + typical_p=None, do_sample=False, seed=0, device="cpu", @@ -64,6 +66,9 @@ def __init__( if top_p is not None and top_p < 1.0: warpers.append(TopPLogitsWarper(top_p=top_p)) sampling = True + if typical_p is not None and typical_p < 1.0: + warpers.append(TypicalLogitsWarper(mass=typical_p)) + sampling = True self.warpers = warpers self.choice = Sampling(seed, device) if sampling else Greedy() @@ -92,6 +97,7 @@ def from_pb( repetition_penalty=pb.repetition_penalty, top_k=pb.top_k, top_p=pb.top_p, + typical_p=pb.typical_p, do_sample=pb.do_sample, seed=pb.seed, device=device, From c0795de2f2c0beded136aa058f963dc1e341177a Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 9 Mar 2023 13:00:10 +0100 Subject: [PATCH 113/793] fix(server): do not warp prefill logits (#116) --- server/text_generation_server/utils/tokens.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 15563761..c7594644 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -75,7 +75,11 @@ def __init__( def __call__(self, input_ids, scores): # Warp logits - scores = self.warpers(input_ids, scores) + if scores.shape[0] > 1: + # only warp the last token logits + scores[-1:, :] = self.warpers(input_ids, scores[-1:, :]) + else: + scores = self.warpers(input_ids, scores) # Compute logprobs logprobs = torch.log_softmax(scores, -1) From e8bfe199bacb2778153bf0029b20dab866733c75 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 9 Mar 2023 13:10:30 +0100 Subject: [PATCH 114/793] feat(router): support left truncation (#115) closes #111 --- router/src/lib.rs | 6 ++- router/src/server.rs | 1 + router/src/validation.rs | 108 ++++++++++++++++++++++++--------------- 3 files changed, 72 insertions(+), 43 deletions(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index d375eafb..9fcc5085 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -56,12 +56,15 @@ pub(crate) struct GenerateParameters { #[schema(exclusive_minimum = 0, exclusive_maximum = 512, default = "20")] pub max_new_tokens: u32, #[serde(default)] - #[schema(default = "None", example = false)] + #[schema(default = "null", example = false)] pub return_full_text: Option, #[serde(default)] #[schema(inline, max_items = 4, example = json ! (["photographer"]))] pub stop: Vec, #[serde(default)] + #[schema(default = "null", example = "null")] + pub truncate: Option, + #[serde(default)] #[schema(default = "false", example = true)] pub watermark: bool, #[serde(default)] @@ -86,6 +89,7 @@ fn default_parameters() -> GenerateParameters { max_new_tokens: default_max_new_tokens(), return_full_text: None, stop: Vec::new(), + truncate: None, watermark: false, details: false, seed: None, diff --git a/router/src/server.rs b/router/src/server.rs index 2ce5699d..ef10b7b1 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -73,6 +73,7 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json { - let input_length = encoding.len(); - let total_tokens = input_length + max_new_tokens as usize; - - if input_length > max_input_length { - Err(ValidationError::InputLength(max_input_length, input_length)) - } else if total_tokens > max_total_tokens { - Err(ValidationError::MaxTotalTokens( - max_total_tokens, - input_length, - max_new_tokens, - )) - } else { - // Return ValidGenerateRequest - let parameters = NextTokenChooserParameters { - temperature, - repetition_penalty, - top_k, - top_p, - typical_p, - do_sample, - seed, - watermark, - }; - let stopping_parameters = StoppingCriteriaParameters { - max_new_tokens, - stop_sequences, - }; - - metrics::histogram!("tgi_request_input_length", input_length as f64); - metrics::histogram!("tgi_request_max_new_tokens", max_new_tokens as f64); - - Ok(ValidGenerateRequest { - inputs: request.inputs, - input_length: input_length as u32, - parameters, - stopping_parameters, - }) + // Check if truncate is strictly positive and less than max_input_length + let truncate = truncate + .map(|value| { + if value == 0 || value > max_input_length { + return Err(ValidationError::Truncate(max_input_length, value)); } - } - Err(err) => Err(ValidationError::Tokenizer(err.to_string())), + Ok(Some(value)) + }) + .unwrap_or(Ok(None))?; + + // Get the number of tokens in the input + let mut encoding = tokenizer + .encode(request.inputs.clone(), true) + .map_err(|err| ValidationError::Tokenizer(err.to_string()))?; + + let (inputs, input_length) = if let Some(truncate) = truncate { + // truncate encoding and decode new inputs + encoding.truncate(truncate, 0, TruncationDirection::Left); + let inputs = tokenizer + .decode(Vec::from(encoding.get_ids()), false) + .map_err(|err| ValidationError::Tokenizer(err.to_string()))?; + (inputs, encoding.len()) + } else { + (request.inputs, encoding.len()) + }; + + if input_length > max_input_length { + return Err(ValidationError::InputLength(max_input_length, input_length)); + } + + let total_tokens = input_length + max_new_tokens as usize; + if total_tokens > max_total_tokens { + return Err(ValidationError::MaxTotalTokens( + max_total_tokens, + input_length, + max_new_tokens, + )); } + + // Return ValidGenerateRequest + let parameters = NextTokenChooserParameters { + temperature, + repetition_penalty, + top_k, + top_p, + typical_p, + do_sample, + seed, + watermark, + }; + let stopping_parameters = StoppingCriteriaParameters { + max_new_tokens, + stop_sequences, + }; + + metrics::histogram!("tgi_request_input_length", input_length as f64); + metrics::histogram!("tgi_request_max_new_tokens", max_new_tokens as f64); + + Ok(ValidGenerateRequest { + inputs, + input_length: input_length as u32, + parameters, + stopping_parameters, + }) } type ValidationRequest = ( @@ -293,6 +315,8 @@ pub enum ValidationError { TopP, #[error("`top_k` must be strictly positive")] TopK, + #[error("`truncate` must be strictly positive and less than {0}. Given: {1}")] + Truncate(usize, usize), #[error("`typical_p` must be > 0.0 and < 1.0")] TypicalP, #[error("`max_new_tokens` must be strictly positive")] From 55bd4fed7da83a566dca08b0bb29dbc5929a90eb Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 9 Mar 2023 15:30:54 +0100 Subject: [PATCH 115/793] feat(router): add best_of parameter (#117) --- docs/openapi.json | 97 +++++++++++++-- launcher/src/main.rs | 5 + router/src/infer.rs | 38 ++++++ router/src/lib.rs | 34 +++++- router/src/main.rs | 4 + router/src/server.rs | 255 +++++++++++++++++++++++---------------- router/src/validation.rs | 51 +++++++- 7 files changed, 367 insertions(+), 117 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 7ece1722..881b892c 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -210,13 +210,63 @@ }, "components": { "schemas": { + "BestOfSequence": { + "type": "object", + "required": [ + "generated_text", + "finish_reason", + "generated_tokens", + "prefill", + "tokens" + ], + "properties": { + "finish_reason": { + "$ref": "#/components/schemas/FinishReason" + }, + "generated_text": { + "type": "string", + "example": "test" + }, + "generated_tokens": { + "type": "integer", + "format": "int32", + "example": 1 + }, + "prefill": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PrefillToken" + } + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42, + "nullable": true + }, + "tokens": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + } + } + }, "Details": { "type": "object", "required": [ "finish_reason", - "generated_tokens" + "generated_tokens", + "prefill", + "tokens" ], "properties": { + "best_of_sequences": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BestOfSequence" + } + }, "finish_reason": { "$ref": "#/components/schemas/FinishReason" }, @@ -234,7 +284,8 @@ "seed": { "type": "integer", "format": "int64", - "example": 42 + "example": 42, + "nullable": true }, "tokens": { "type": "array", @@ -247,11 +298,15 @@ "ErrorResponse": { "type": "object", "required": [ - "error" + "error", + "error_type" ], "properties": { "error": { "type": "string" + }, + "error_type": { + "type": "string" } } }, @@ -266,6 +321,13 @@ "GenerateParameters": { "type": "object", "properties": { + "best_of": { + "type": "integer", + "default": "null", + "example": 1, + "nullable": true, + "exclusiveMinimum": 0.0 + }, "details": { "type": "boolean", "default": "true" @@ -292,12 +354,17 @@ }, "return_full_text": { "type": "boolean", - "default": "None", - "example": false + "default": "null", + "example": false, + "nullable": true }, "seed": { "type": "integer", - "format": "int64" + "format": "int64", + "default": "null", + "example": "null", + "nullable": true, + "exclusiveMinimum": 0.0 }, "stop": { "type": "array", @@ -334,6 +401,21 @@ "maximum": 1.0, "exclusiveMinimum": 0.0 }, + "truncate": { + "type": "integer", + "default": "null", + "example": "null", + "nullable": true + }, + "typical_p": { + "type": "number", + "format": "float", + "default": "null", + "example": 0.95, + "nullable": true, + "maximum": 1.0, + "exclusiveMinimum": 0.0 + }, "watermark": { "type": "boolean", "default": "false", @@ -414,7 +496,8 @@ "seed": { "type": "integer", "format": "int64", - "example": 42 + "example": 42, + "nullable": true } } }, diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 1865cf90..80466fe6 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -31,6 +31,8 @@ struct Args { quantize: bool, #[clap(default_value = "128", long, env)] max_concurrent_requests: usize, + #[clap(default_value = "2", long, env)] + max_best_of: usize, #[clap(default_value = "4", long, env)] max_stop_sequences: usize, #[clap(default_value = "1000", long, env)] @@ -86,6 +88,7 @@ fn main() -> ExitCode { num_shard, quantize, max_concurrent_requests, + max_best_of, max_stop_sequences, max_input_length, max_total_tokens, @@ -363,6 +366,8 @@ fn main() -> ExitCode { "text-generation-router".to_string(), "--max-concurrent-requests".to_string(), max_concurrent_requests.to_string(), + "--max-best-of".to_string(), + max_best_of.to_string(), "--max-stop-sequences".to_string(), max_stop_sequences.to_string(), "--max-input-length".to_string(), diff --git a/router/src/infer.rs b/router/src/infer.rs index d0964f97..5955faec 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -2,6 +2,7 @@ use crate::validation::{Validation, ValidationError}; use crate::{Entry, Queue, Token}; use crate::{GenerateRequest, PrefillToken}; +use futures::future::try_join_all; use nohash_hasher::IntMap; use std::sync::Arc; use text_generation_client::{ @@ -177,6 +178,43 @@ impl Infer { Err(err) } } + /// Add best_of new requests to the queue and return a InferResponse of the sequence with + /// the highest log probability per token + #[instrument(skip(self))] + pub(crate) async fn generate_best_of( + &self, + request: GenerateRequest, + best_of: usize, + ) -> Result<(InferResponse, Vec), InferError> { + // validate best_of parameter separately + let best_of = self.validation.validate_best_of(best_of)?; + + // create multiple generate requests + let mut infer_responses: Vec = + try_join_all((0..best_of).map(|_| self.generate(request.clone()))).await?; + + // get the sequence with the highest log probability per token + let mut max_index = 0; + let mut max_logprob: f32 = f32::MIN; + + for (i, response) in infer_responses.iter().enumerate() { + // mean logprobs of the generated tokens + let sequence_logprob = response + .tokens + .iter() + .map(|token| token.logprob) + .sum::() + / response.tokens.len() as f32; + + // set best sequence + if sequence_logprob > max_logprob { + max_index = i; + max_logprob = sequence_logprob; + } + } + let best_response = infer_responses.remove(max_index); + Ok((best_response, infer_responses)) + } } /// Batching logic diff --git a/router/src/lib.rs b/router/src/lib.rs index 9fcc5085..91b4417c 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -12,6 +12,9 @@ use validation::Validation; #[derive(Clone, Debug, Deserialize, ToSchema)] pub(crate) struct GenerateParameters { + #[serde(default)] + #[schema(exclusive_minimum = 0, nullable = true, default = "null", example = 1)] + pub best_of: Option, #[serde(default)] #[schema( exclusive_minimum = 0.0, @@ -56,13 +59,13 @@ pub(crate) struct GenerateParameters { #[schema(exclusive_minimum = 0, exclusive_maximum = 512, default = "20")] pub max_new_tokens: u32, #[serde(default)] - #[schema(default = "null", example = false)] + #[schema(nullable = true, default = "null", example = false)] pub return_full_text: Option, #[serde(default)] #[schema(inline, max_items = 4, example = json ! (["photographer"]))] pub stop: Vec, #[serde(default)] - #[schema(default = "null", example = "null")] + #[schema(nullable = true, default = "null", example = "null")] pub truncate: Option, #[serde(default)] #[schema(default = "false", example = true)] @@ -71,6 +74,12 @@ pub(crate) struct GenerateParameters { #[schema(default = "true")] pub details: bool, #[serde(default)] + #[schema( + exclusive_minimum = 0, + nullable = true, + default = "null", + example = "null" + )] pub seed: Option, } @@ -80,6 +89,7 @@ fn default_max_new_tokens() -> u32 { fn default_parameters() -> GenerateParameters { GenerateParameters { + best_of: None, temperature: None, repetition_penalty: None, top_k: None, @@ -158,16 +168,32 @@ pub(crate) enum FinishReason { StopSequence, } +#[derive(Serialize, ToSchema)] +pub(crate) struct BestOfSequence { + #[schema(example = "test")] + pub generated_text: String, + #[schema(example = "length")] + pub finish_reason: FinishReason, + #[schema(example = 1)] + pub generated_tokens: u32, + #[schema(nullable = true, example = 42)] + pub seed: Option, + pub prefill: Vec, + pub tokens: Vec, +} + #[derive(Serialize, ToSchema)] pub(crate) struct Details { #[schema(example = "length")] pub finish_reason: FinishReason, #[schema(example = 1)] pub generated_tokens: u32, - #[schema(example = 42)] + #[schema(nullable = true, example = 42)] pub seed: Option, pub prefill: Vec, pub tokens: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub best_of_sequences: Option>, } #[derive(Serialize, ToSchema)] @@ -184,7 +210,7 @@ pub(crate) struct StreamDetails { pub finish_reason: FinishReason, #[schema(example = 1)] pub generated_tokens: u32, - #[schema(example = 42)] + #[schema(nullable = true, example = 42)] pub seed: Option, } diff --git a/router/src/main.rs b/router/src/main.rs index a51d3168..2ccf66b3 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -23,6 +23,8 @@ use tracing_subscriber::{EnvFilter, Layer}; struct Args { #[clap(default_value = "128", long, env)] max_concurrent_requests: usize, + #[clap(default_value = "2", long, env)] + max_best_of: usize, #[clap(default_value = "4", long, env)] max_stop_sequences: usize, #[clap(default_value = "1000", long, env)] @@ -55,6 +57,7 @@ fn main() -> Result<(), std::io::Error> { // Pattern match configuration let Args { max_concurrent_requests, + max_best_of, max_stop_sequences, max_input_length, max_total_tokens, @@ -145,6 +148,7 @@ fn main() -> Result<(), std::io::Error> { server::run( compat_return_full_text, max_concurrent_requests, + max_best_of, max_stop_sequences, max_input_length, max_total_tokens, diff --git a/router/src/server.rs b/router/src/server.rs index ef10b7b1..3b63ec8a 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,9 +1,10 @@ /// HTTP Server logic -use crate::infer::{InferError, InferStreamResponse}; +use crate::infer::{InferError, InferResponse, InferStreamResponse}; +use crate::validation::ValidationError; use crate::{ - CompatGenerateRequest, Details, ErrorResponse, FinishReason, GenerateParameters, - GenerateRequest, GenerateResponse, Infer, PrefillToken, StreamDetails, StreamResponse, Token, - Validation, + BestOfSequence, CompatGenerateRequest, Details, ErrorResponse, FinishReason, + GenerateParameters, GenerateRequest, GenerateResponse, Infer, PrefillToken, StreamDetails, + StreamResponse, Token, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; @@ -64,6 +65,7 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json 1 => { + let (response, best_of_responses) = infer.generate_best_of(req.0, best_of).await?; + (response, Some(best_of_responses)) + } + _ => (infer.generate(req.0).await?, None), + }; // Token details let details = match details { - true => Some(Details { - finish_reason: FinishReason::from(response.generated_text.finish_reason), - generated_tokens: response.generated_text.generated_tokens, - prefill: response.prefill, - tokens: response.tokens, - seed: response.generated_text.seed, - }), + true => { + // convert best_of_responses + let best_of_sequences = best_of_responses.map(|responses: Vec| { + responses + .into_iter() + .map(|response: InferResponse| { + // Add prompt if return_full_text + let mut output_text = response.generated_text.text; + if let Some(prompt) = &add_prompt { + output_text = prompt.clone() + &output_text; + } + + BestOfSequence { + generated_text: output_text, + finish_reason: FinishReason::from( + response.generated_text.finish_reason, + ), + generated_tokens: response.generated_text.generated_tokens, + prefill: response.prefill, + tokens: response.tokens, + seed: response.generated_text.seed, + } + }) + .collect() + }); + + Some(Details { + finish_reason: FinishReason::from(response.generated_text.finish_reason), + generated_tokens: response.generated_text.generated_tokens, + prefill: response.prefill, + tokens: response.tokens, + seed: response.generated_text.seed, + best_of_sequences, + }) + } false => None, }; @@ -279,107 +315,115 @@ async fn generate_stream( } let details = req.0.parameters.details; - match infer.generate_stream(req.0).instrument(info_span!(parent: &span, "async_stream")).await { - Ok(mut response_stream) => { - // Server-Sent Event stream - while let Some(response) = response_stream.next().await { - match response { - Ok(response) => { - match response { - // Prefill is ignored - InferStreamResponse::Prefill(_) => {} - // Yield event for every new token - InferStreamResponse::Token(token) => { - // StreamResponse - let stream_token = StreamResponse { - token, - generated_text: None, - details: None, - }; - - yield Ok(Event::default().json_data(stream_token).unwrap()) - } - // Yield event for last token and compute timings - InferStreamResponse::End { - token, - generated_text, - start, - queued, - } => { - // Token details - let details = match details { - true => Some(StreamDetails { - finish_reason: FinishReason::from(generated_text.finish_reason), - generated_tokens: generated_text.generated_tokens, - seed: generated_text.seed, - }), - false => None, - }; - - // Timings - let total_time = start_time.elapsed(); - let validation_time = queued - start_time; - let queue_time = start - queued; - let inference_time = Instant::now() - start; - let time_per_token = inference_time / generated_text.generated_tokens; - - // Tracing metadata - span.record("total_time", format!("{total_time:?}")); - span.record("validation_time", format!("{validation_time:?}")); - span.record("queue_time", format!("{queue_time:?}")); - span.record("inference_time", format!("{inference_time:?}")); - span.record("time_per_token", format!("{time_per_token:?}")); - span.record("seed", format!("{:?}", generated_text.seed)); - tracing::info!(parent: &span, "Output: {}", generated_text.text); - - // Metrics - metrics::increment_counter!("tgi_request_success"); - metrics::histogram!("tgi_request_duration", total_time); - metrics::histogram!("tgi_request_validation_duration", validation_time); - metrics::histogram!("tgi_request_queue_duration", queue_time); - metrics::histogram!("tgi_request_inference_duration", inference_time); - metrics::histogram!("tgi_request_mean_time_per_token_duration", time_per_token); - metrics::histogram!("tgi_request_generated_tokens", generated_text.generated_tokens as f64); - - // StreamResponse - end_reached = true; - - let mut output_text = generated_text.text; - if let Some(prompt) = add_prompt { - output_text = prompt + &output_text; + let best_of = req.0.parameters.best_of.unwrap_or(1); + if best_of == 1 { + match infer.generate_stream(req.0).instrument(info_span!(parent: &span, "async_stream")).await { + Ok(mut response_stream) => { + // Server-Sent Event stream + while let Some(response) = response_stream.next().await { + match response { + Ok(response) => { + match response { + // Prefill is ignored + InferStreamResponse::Prefill(_) => {} + // Yield event for every new token + InferStreamResponse::Token(token) => { + // StreamResponse + let stream_token = StreamResponse { + token, + generated_text: None, + details: None, + }; + + yield Ok(Event::default().json_data(stream_token).unwrap()) } - - let stream_token = StreamResponse { + // Yield event for last token and compute timings + InferStreamResponse::End { token, - generated_text: Some(output_text), - details - }; - - yield Ok(Event::default().json_data(stream_token).unwrap()); - break; + generated_text, + start, + queued, + } => { + // Token details + let details = match details { + true => Some(StreamDetails { + finish_reason: FinishReason::from(generated_text.finish_reason), + generated_tokens: generated_text.generated_tokens, + seed: generated_text.seed, + }), + false => None, + }; + + // Timings + let total_time = start_time.elapsed(); + let validation_time = queued - start_time; + let queue_time = start - queued; + let inference_time = Instant::now() - start; + let time_per_token = inference_time / generated_text.generated_tokens; + + // Tracing metadata + span.record("total_time", format!("{total_time:?}")); + span.record("validation_time", format!("{validation_time:?}")); + span.record("queue_time", format!("{queue_time:?}")); + span.record("inference_time", format!("{inference_time:?}")); + span.record("time_per_token", format!("{time_per_token:?}")); + span.record("seed", format!("{:?}", generated_text.seed)); + tracing::info!(parent: &span, "Output: {}", generated_text.text); + + // Metrics + metrics::increment_counter!("tgi_request_success"); + metrics::histogram!("tgi_request_duration", total_time); + metrics::histogram!("tgi_request_validation_duration", validation_time); + metrics::histogram!("tgi_request_queue_duration", queue_time); + metrics::histogram!("tgi_request_inference_duration", inference_time); + metrics::histogram!("tgi_request_mean_time_per_token_duration", time_per_token); + metrics::histogram!("tgi_request_generated_tokens", generated_text.generated_tokens as f64); + + // StreamResponse + end_reached = true; + + let mut output_text = generated_text.text; + if let Some(prompt) = add_prompt { + output_text = prompt + &output_text; + } + + let stream_token = StreamResponse { + token, + generated_text: Some(output_text), + details + }; + + yield Ok(Event::default().json_data(stream_token).unwrap()); + break; + } } } - } - // yield error - Err(err) => { - error = true; - yield Ok(Event::from(err)); - break; + // yield error + Err(err) => { + error = true; + yield Ok(Event::from(err)); + break; + } } } + }, + // yield error + Err(err) => { + error = true; + yield Ok(Event::from(err)); } - }, - // yield error - Err(err) => { - error = true; + } + // Check if generation reached the end + // Skip if we already sent an error + if !end_reached && !error { + let err = InferError::IncompleteGeneration; + metrics::increment_counter!("tgi_request_failure", "err" => "incomplete"); + tracing::error!("{err}"); yield Ok(Event::from(err)); } - } - // Check if generation reached the end - // Skip if we already sent an error - if !end_reached && !error { - let err = InferError::IncompleteGeneration; - metrics::increment_counter!("tgi_request_failure", "err" => "incomplete"); + } else { + let err = InferError::from(ValidationError::BestOfStream); + metrics::increment_counter!("tgi_request_failure", "err" => "validation"); tracing::error!("{err}"); yield Ok(Event::from(err)); } @@ -404,6 +448,7 @@ async fn metrics(prom_handle: Extension) -> String { pub async fn run( compat_return_full_text: bool, max_concurrent_requests: usize, + max_best_of: usize, max_stop_sequences: usize, max_input_length: usize, max_total_tokens: usize, @@ -430,6 +475,7 @@ pub async fn run( PrefillToken, Token, GenerateResponse, + BestOfSequence, Details, FinishReason, StreamResponse, @@ -454,6 +500,7 @@ pub async fn run( let validation = Validation::new( validation_workers, tokenizer, + max_best_of, max_stop_sequences, max_input_length, max_total_tokens, diff --git a/router/src/validation.rs b/router/src/validation.rs index 42af0169..cb8dd0a2 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,4 +1,4 @@ -use crate::validation::ValidationError::EmptyInput; +use crate::validation::ValidationError::{BestOfSampling, BestOfSeed, EmptyInput}; /// Payload validation logic use crate::{GenerateParameters, GenerateRequest}; use rand::rngs::ThreadRng; @@ -13,6 +13,9 @@ use tracing::{instrument, Span}; /// Validation #[derive(Debug, Clone)] pub struct Validation { + /// maximum value for the best_of parameter + #[allow(dead_code)] + max_best_of: usize, /// Channel to communicate with the background validation task sender: mpsc::UnboundedSender, } @@ -21,6 +24,7 @@ impl Validation { pub(crate) fn new( workers: usize, tokenizer: Tokenizer, + max_best_of: usize, max_stop_sequences: usize, max_input_length: usize, max_total_tokens: usize, @@ -39,6 +43,7 @@ impl Validation { )); Self { + max_best_of, sender: validation_sender, } } @@ -60,6 +65,20 @@ impl Validation { // Unwrap is safe here receiver.await.unwrap() } + + /// Validate the best_of parameter + #[instrument(skip_all)] + pub(crate) fn validate_best_of(&self, best_of: usize) -> Result { + if self.max_best_of == 1 && best_of != 1 { + return Err(ValidationError::BestOfDisabled); + } + + if best_of > self.max_best_of { + return Err(ValidationError::BestOf(self.max_best_of, best_of)); + } + + Ok(best_of) + } } /// Validation task @@ -150,6 +169,7 @@ fn validate( rng: &mut ThreadRng, ) -> Result { let GenerateParameters { + best_of, temperature, repetition_penalty, top_k, @@ -164,6 +184,18 @@ fn validate( .. } = request.parameters; + // sampling must be true when best_of > 1 + let best_of = best_of.unwrap_or(1); + let sampling = do_sample + || temperature.is_some() + || top_k.is_some() + || top_p.is_some() + || typical_p.is_some(); + + if best_of > 1 && !sampling { + return Err(BestOfSampling); + } + let temperature = temperature.unwrap_or(1.0); if temperature <= 0.0 { return Err(ValidationError::Temperature); @@ -217,7 +249,12 @@ fn validate( // If seed is None, assign a random one let seed = match seed { None => rng.gen(), - Some(seed) => seed, + Some(seed) => { + if best_of > 1 { + return Err(BestOfSeed); + } + seed + } }; // Check if inputs is empty @@ -307,6 +344,16 @@ pub(crate) struct ValidGenerateRequest { #[derive(Error, Debug)] pub enum ValidationError { + #[error("`best_of` must be > 0 and <= {0}. Given: {1}")] + BestOf(usize, usize), + #[error("`best_of` != 1 is not allowed for this endpoint")] + BestOfDisabled, + #[error("you must use sampling when `best_of` is > 1")] + BestOfSampling, + #[error("`seed` must not be set when `best_of` > 1")] + BestOfSeed, + #[error("`best_of` != 1 is not supported when streaming tokens")] + BestOfStream, #[error("`temperature` must be strictly positive")] Temperature, #[error("`repetition_penalty` must be strictly positive")] From d8dc8f1b0c97da009dd975c3622e49d01e2081b3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 9 Mar 2023 16:05:33 +0100 Subject: [PATCH 116/793] feat(python-client): add new parameters (#118) --- clients/python/README.md | 18 +++ clients/python/pyproject.toml | 2 +- clients/python/tests/conftest.py | 5 - clients/python/tests/test_client.py | 46 ++++--- clients/python/tests/test_inference_api.py | 8 +- clients/python/tests/test_types.py | 47 ++++++- clients/python/text_generation/__init__.py | 2 +- clients/python/text_generation/client.py | 52 ++++++++ clients/python/text_generation/types.py | 138 +++++++++++++++++++-- 9 files changed, 278 insertions(+), 40 deletions(-) diff --git a/clients/python/README.md b/clients/python/README.md index 79b88377..f509e65c 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -133,6 +133,22 @@ class FinishReason(Enum): StopSequence = "stop_sequence" +# Additional sequences when using the `best_of` parameter +class BestOfSequence: + # Generated text + generated_text: str + # Generation finish reason + finish_reason: FinishReason + # Number of generated tokens + generated_tokens: int + # Sampling seed if sampling was activated + seed: Optional[int] + # Prompt tokens + prefill: List[PrefillToken] + # Generated tokens + tokens: List[Token] + + # `generate` details class Details: # Generation finish reason @@ -145,6 +161,8 @@ class Details: prefill: List[PrefillToken] # Generated tokens tokens: List[Token] + # Additional sequences when using the `best_of` parameter + best_of_sequences: Optional[List[BestOfSequence]] # `generate` return value diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 0b8fa8c7..51ecce82 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.2.1" +version = "0.3.0" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] diff --git a/clients/python/tests/conftest.py b/clients/python/tests/conftest.py index 4298623e..48734f0d 100644 --- a/clients/python/tests/conftest.py +++ b/clients/python/tests/conftest.py @@ -4,11 +4,6 @@ from huggingface_hub.utils import build_hf_headers -@pytest.fixture -def bloom_model(): - return "bigscience/bloom" - - @pytest.fixture def flan_t5_xxl(): return "google/flan-t5-xxl" diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 2f96aa87..c998de41 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -5,24 +5,32 @@ from text_generation.types import FinishReason, PrefillToken, Token -def test_generate(bloom_url, hf_headers): - client = Client(bloom_url, hf_headers) +def test_generate(flan_t5_xxl_url, hf_headers): + client = Client(flan_t5_xxl_url, hf_headers) response = client.generate("test", max_new_tokens=1) - assert response.generated_text == "." + assert response.generated_text == "" assert response.details.finish_reason == FinishReason.Length assert response.details.generated_tokens == 1 assert response.details.seed is None assert len(response.details.prefill) == 1 - assert response.details.prefill[0] == PrefillToken( - id=9234, text="test", logprob=None - ) + assert response.details.prefill[0] == PrefillToken(id=0, text="", logprob=None) assert len(response.details.tokens) == 1 assert response.details.tokens[0] == Token( - id=17, text=".", logprob=-1.75, special=False + id=3, text=" ", logprob=-1.984375, special=False ) +def test_generate_best_of(flan_t5_xxl_url, hf_headers): + client = Client(flan_t5_xxl_url, hf_headers) + response = client.generate("test", max_new_tokens=1, best_of=2, do_sample=True) + + assert response.details.seed is not None + assert response.details.best_of_sequences is not None + assert len(response.details.best_of_sequences) == 1 + assert response.details.best_of_sequences[0].seed is not None + + def test_generate_not_found(fake_url, hf_headers): client = Client(fake_url, hf_headers) with pytest.raises(NotFoundError): @@ -35,8 +43,8 @@ def test_generate_validation_error(flan_t5_xxl_url, hf_headers): client.generate("test", max_new_tokens=10_000) -def test_generate_stream(bloom_url, hf_headers): - client = Client(bloom_url, hf_headers) +def test_generate_stream(flan_t5_xxl_url, hf_headers): + client = Client(flan_t5_xxl_url, hf_headers) responses = [ response for response in client.generate_stream("test", max_new_tokens=1) ] @@ -44,7 +52,7 @@ def test_generate_stream(bloom_url, hf_headers): assert len(responses) == 1 response = responses[0] - assert response.generated_text == "." + assert response.generated_text == "" assert response.details.finish_reason == FinishReason.Length assert response.details.generated_tokens == 1 assert response.details.seed is None @@ -63,21 +71,19 @@ def test_generate_stream_validation_error(flan_t5_xxl_url, hf_headers): @pytest.mark.asyncio -async def test_generate_async(bloom_url, hf_headers): - client = AsyncClient(bloom_url, hf_headers) +async def test_generate_async(flan_t5_xxl_url, hf_headers): + client = AsyncClient(flan_t5_xxl_url, hf_headers) response = await client.generate("test", max_new_tokens=1) - assert response.generated_text == "." + assert response.generated_text == "" assert response.details.finish_reason == FinishReason.Length assert response.details.generated_tokens == 1 assert response.details.seed is None assert len(response.details.prefill) == 1 - assert response.details.prefill[0] == PrefillToken( - id=9234, text="test", logprob=None - ) + assert response.details.prefill[0] == PrefillToken(id=0, text="", logprob=None) assert len(response.details.tokens) == 1 assert response.details.tokens[0] == Token( - id=17, text=".", logprob=-1.75, special=False + id=3, text=" ", logprob=-1.984375, special=False ) @@ -96,8 +102,8 @@ async def test_generate_async_validation_error(flan_t5_xxl_url, hf_headers): @pytest.mark.asyncio -async def test_generate_stream_async(bloom_url, hf_headers): - client = AsyncClient(bloom_url, hf_headers) +async def test_generate_stream_async(flan_t5_xxl_url, hf_headers): + client = AsyncClient(flan_t5_xxl_url, hf_headers) responses = [ response async for response in client.generate_stream("test", max_new_tokens=1) ] @@ -105,7 +111,7 @@ async def test_generate_stream_async(bloom_url, hf_headers): assert len(responses) == 1 response = responses[0] - assert response.generated_text == "." + assert response.generated_text == "" assert response.details.finish_reason == FinishReason.Length assert response.details.generated_tokens == 1 assert response.details.seed is None diff --git a/clients/python/tests/test_inference_api.py b/clients/python/tests/test_inference_api.py index dc744940..79e503a3 100644 --- a/clients/python/tests/test_inference_api.py +++ b/clients/python/tests/test_inference_api.py @@ -14,8 +14,8 @@ def test_get_supported_models(): assert isinstance(get_supported_models(), list) -def test_client(bloom_model): - client = InferenceAPIClient(bloom_model) +def test_client(flan_t5_xxl): + client = InferenceAPIClient(flan_t5_xxl) assert isinstance(client, Client) @@ -24,8 +24,8 @@ def test_client_unsupported_model(unsupported_model): InferenceAPIClient(unsupported_model) -def test_async_client(bloom_model): - client = InferenceAPIAsyncClient(bloom_model) +def test_async_client(flan_t5_xxl): + client = InferenceAPIAsyncClient(flan_t5_xxl) assert isinstance(client, AsyncClient) diff --git a/clients/python/tests/test_types.py b/clients/python/tests/test_types.py index d319b570..4c9d4c89 100644 --- a/clients/python/tests/test_types.py +++ b/clients/python/tests/test_types.py @@ -1,10 +1,20 @@ import pytest -from text_generation.types import Parameters +from text_generation.types import Parameters, Request from text_generation.errors import ValidationError def test_parameters_validation(): + # Test best_of + Parameters(best_of=1) + with pytest.raises(ValidationError): + Parameters(best_of=0) + with pytest.raises(ValidationError): + Parameters(best_of=-1) + Parameters(best_of=2, do_sample=True) + with pytest.raises(ValidationError): + Parameters(best_of=2) + # Test repetition_penalty Parameters(repetition_penalty=1) with pytest.raises(ValidationError): @@ -32,8 +42,41 @@ def test_parameters_validation(): Parameters(top_k=-1) # Test top_p - Parameters(top_p=1) + Parameters(top_p=0.5) with pytest.raises(ValidationError): Parameters(top_p=0) with pytest.raises(ValidationError): Parameters(top_p=-1) + with pytest.raises(ValidationError): + Parameters(top_p=1) + + # Test truncate + Parameters(truncate=1) + with pytest.raises(ValidationError): + Parameters(truncate=0) + with pytest.raises(ValidationError): + Parameters(truncate=-1) + + # Test typical_p + Parameters(typical_p=0.5) + with pytest.raises(ValidationError): + Parameters(typical_p=0) + with pytest.raises(ValidationError): + Parameters(typical_p=-1) + with pytest.raises(ValidationError): + Parameters(typical_p=1) + + +def test_request_validation(): + Request(inputs="test") + + with pytest.raises(ValidationError): + Request(inputs="") + + Request(inputs="test", stream=True) + Request(inputs="test", parameters=Parameters(best_of=2, do_sample=True)) + + with pytest.raises(ValidationError): + Request( + inputs="test", parameters=Parameters(best_of=2, do_sample=True), stream=True + ) diff --git a/clients/python/text_generation/__init__.py b/clients/python/text_generation/__init__.py index db09cdfa..46109833 100644 --- a/clients/python/text_generation/__init__.py +++ b/clients/python/text_generation/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.2.1" +__version__ = "0.3.0" from text_generation.client import Client, AsyncClient from text_generation.inference_api import InferenceAPIClient, InferenceAPIAsyncClient diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index 23655240..e05a002e 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -56,6 +56,7 @@ def generate( prompt: str, do_sample: bool = False, max_new_tokens: int = 20, + best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, @@ -63,6 +64,8 @@ def generate( temperature: Optional[float] = None, top_k: Optional[int] = None, top_p: Optional[float] = None, + truncate: Optional[int] = None, + typical_p: Optional[float] = None, watermark: bool = False, ) -> Response: """ @@ -75,6 +78,8 @@ def generate( Activate logits sampling max_new_tokens (`int`): Maximum number of generated tokens + best_of (`int`): + Generate best_of sequences and return the one if the highest token logprobs repetition_penalty (`float`): The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. @@ -91,6 +96,11 @@ def generate( top_p (`float`): If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation. + truncate (`int`): + Truncate inputs tokens to the given size + typical_p (`float`): + Typical Decoding mass + See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) @@ -99,6 +109,7 @@ def generate( """ # Validate parameters parameters = Parameters( + best_of=best_of, details=True, do_sample=do_sample, max_new_tokens=max_new_tokens, @@ -109,6 +120,8 @@ def generate( temperature=temperature, top_k=top_k, top_p=top_p, + truncate=truncate, + typical_p=typical_p, watermark=watermark, ) request = Request(inputs=prompt, stream=False, parameters=parameters) @@ -129,6 +142,7 @@ def generate_stream( prompt: str, do_sample: bool = False, max_new_tokens: int = 20, + best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, @@ -136,6 +150,8 @@ def generate_stream( temperature: Optional[float] = None, top_k: Optional[int] = None, top_p: Optional[float] = None, + truncate: Optional[int] = None, + typical_p: Optional[float] = None, watermark: bool = False, ) -> Iterator[StreamResponse]: """ @@ -148,6 +164,8 @@ def generate_stream( Activate logits sampling max_new_tokens (`int`): Maximum number of generated tokens + best_of (`int`): + Generate best_of sequences and return the one if the highest token logprobs repetition_penalty (`float`): The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. @@ -164,6 +182,11 @@ def generate_stream( top_p (`float`): If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation. + truncate (`int`): + Truncate inputs tokens to the given size + typical_p (`float`): + Typical Decoding mass + See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) @@ -172,6 +195,7 @@ def generate_stream( """ # Validate parameters parameters = Parameters( + best_of=best_of, details=True, do_sample=do_sample, max_new_tokens=max_new_tokens, @@ -182,6 +206,8 @@ def generate_stream( temperature=temperature, top_k=top_k, top_p=top_p, + truncate=truncate, + typical_p=typical_p, watermark=watermark, ) request = Request(inputs=prompt, stream=True, parameters=parameters) @@ -261,6 +287,7 @@ async def generate( prompt: str, do_sample: bool = False, max_new_tokens: int = 20, + best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, @@ -268,6 +295,8 @@ async def generate( temperature: Optional[float] = None, top_k: Optional[int] = None, top_p: Optional[float] = None, + truncate: Optional[int] = None, + typical_p: Optional[float] = None, watermark: bool = False, ) -> Response: """ @@ -280,6 +309,8 @@ async def generate( Activate logits sampling max_new_tokens (`int`): Maximum number of generated tokens + best_of (`int`): + Generate best_of sequences and return the one if the highest token logprobs repetition_penalty (`float`): The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. @@ -296,6 +327,11 @@ async def generate( top_p (`float`): If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation. + truncate (`int`): + Truncate inputs tokens to the given size + typical_p (`float`): + Typical Decoding mass + See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) @@ -304,6 +340,7 @@ async def generate( """ # Validate parameters parameters = Parameters( + best_of=best_of, details=True, do_sample=do_sample, max_new_tokens=max_new_tokens, @@ -314,6 +351,8 @@ async def generate( temperature=temperature, top_k=top_k, top_p=top_p, + truncate=truncate, + typical_p=typical_p, watermark=watermark, ) request = Request(inputs=prompt, stream=False, parameters=parameters) @@ -331,6 +370,7 @@ async def generate_stream( prompt: str, do_sample: bool = False, max_new_tokens: int = 20, + best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, @@ -338,6 +378,8 @@ async def generate_stream( temperature: Optional[float] = None, top_k: Optional[int] = None, top_p: Optional[float] = None, + truncate: Optional[int] = None, + typical_p: Optional[float] = None, watermark: bool = False, ) -> AsyncIterator[StreamResponse]: """ @@ -350,6 +392,8 @@ async def generate_stream( Activate logits sampling max_new_tokens (`int`): Maximum number of generated tokens + best_of (`int`): + Generate best_of sequences and return the one if the highest token logprobs repetition_penalty (`float`): The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. @@ -366,6 +410,11 @@ async def generate_stream( top_p (`float`): If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or higher are kept for generation. + truncate (`int`): + Truncate inputs tokens to the given size + typical_p (`float`): + Typical Decoding mass + See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) @@ -374,6 +423,7 @@ async def generate_stream( """ # Validate parameters parameters = Parameters( + best_of=best_of, details=True, do_sample=do_sample, max_new_tokens=max_new_tokens, @@ -384,6 +434,8 @@ async def generate_stream( temperature=temperature, top_k=top_k, top_p=top_p, + truncate=truncate, + typical_p=typical_p, watermark=watermark, ) request = Request(inputs=prompt, stream=True, parameters=parameters) diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index d276b60e..ea2070b8 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -6,27 +6,64 @@ class Parameters(BaseModel): + # Activate logits sampling do_sample: bool = False + # Maximum number of generated tokens max_new_tokens: int = 20 + # The parameter for repetition penalty. 1.0 means no penalty. + # See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. repetition_penalty: Optional[float] = None + # Whether to prepend the prompt to the generated text return_full_text: bool = False + # Stop generating tokens if a member of `stop_sequences` is generated stop: List[str] = [] + # Random sampling seed seed: Optional[int] + # The value used to module the logits distribution. temperature: Optional[float] + # The number of highest probability vocabulary tokens to keep for top-k-filtering. top_k: Optional[int] + # If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + # higher are kept for generation. top_p: Optional[float] + # truncate inputs tokens to the given size + truncate: Optional[int] + # Typical Decoding mass + # See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information + typical_p: Optional[float] + # Generate best_of sequences and return the one if the highest token logprobs + best_of: Optional[int] + # Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) watermark: bool = False + # Get generation details details: bool = False + @validator("best_of") + def valid_best_of(cls, field_value, values): + if field_value is not None: + if field_value <= 0: + raise ValidationError("`best_of` must be strictly positive") + sampling = ( + values["do_sample"] + | (values["temperature"] is not None) + | (values["top_k"] is not None) + | (values["top_p"] is not None) + | (values["typical_p"] is not None) + ) + if field_value > 1 and not sampling: + raise ValidationError("you must use sampling when `best_of` is > 1") + + return field_value + @validator("repetition_penalty") def valid_repetition_penalty(cls, v): - if v is not None and v is v <= 0: + if v is not None and v <= 0: raise ValidationError("`repetition_penalty` must be strictly positive") return v @validator("seed") def valid_seed(cls, v): - if v is not None and v is v < 0: + if v is not None and v < 0: raise ValidationError("`seed` must be positive") return v @@ -44,56 +81,143 @@ def valid_top_k(cls, v): @validator("top_p") def valid_top_p(cls, v): - if v is not None and (v <= 0 or v > 1.0): - raise ValidationError("`top_p` must be > 0.0 and <= 1.0") + if v is not None and (v <= 0 or v >= 1.0): + raise ValidationError("`top_p` must be > 0.0 and < 1.0") + return v + + @validator("truncate") + def valid_truncate(cls, v): + if v is not None and v <= 0: + raise ValidationError("`truncate` must be strictly positive") + return v + + @validator("typical_p") + def valid_typical_p(cls, v): + if v is not None and (v <= 0 or v >= 1.0): + raise ValidationError("`typical_p` must be > 0.0 and < 1.0") return v class Request(BaseModel): + # Prompt inputs: str - parameters: Parameters + # Generation parameters + parameters: Optional[Parameters] + # Whether to stream output tokens stream: bool = False + @validator("inputs") + def valid_input(cls, v): + if not v: + raise ValidationError("`inputs` cannot be empty") + return v + @validator("stream") + def valid_best_of_stream(cls, field_value, values): + parameters = values["parameters"] + if ( + parameters is not None + and parameters.best_of is not None + and parameters.best_of > 1 + and field_value + ): + raise ValidationError( + "`best_of` != 1 is not supported when `stream` == True" + ) + return field_value + + +# Prompt tokens class PrefillToken(BaseModel): + # Token ID from the model tokenizer id: int + # Token text text: str + # Logprob + # Optional since the logprob of the first token cannot be computed logprob: Optional[float] +# Generated tokens class Token(BaseModel): + # Token ID from the model tokenizer id: int + # Token text text: str + # Logprob logprob: float + # Is the token a special token + # Can be used to ignore tokens when concatenating special: bool +# Generation finish reason class FinishReason(Enum): + # number of generated tokens == `max_new_tokens` Length = "length" + # the model generated its end of sequence token EndOfSequenceToken = "eos_token" + # the model generated a text included in `stop_sequences` StopSequence = "stop_sequence" -class Details(BaseModel): +# Additional sequences when using the `best_of` parameter +class BestOfSequence(BaseModel): + # Generated text + generated_text: str + # Generation finish reason finish_reason: FinishReason + # Number of generated tokens generated_tokens: int + # Sampling seed if sampling was activated seed: Optional[int] + # Prompt tokens prefill: List[PrefillToken] + # Generated tokens tokens: List[Token] -class StreamDetails(BaseModel): +# `generate` details +class Details(BaseModel): + # Generation finish reason finish_reason: FinishReason + # Number of generated tokens generated_tokens: int + # Sampling seed if sampling was activated seed: Optional[int] + # Prompt tokens + prefill: List[PrefillToken] + # Generated tokens + tokens: List[Token] + # Additional sequences when using the `best_of` parameter + best_of_sequences: Optional[List[BestOfSequence]] +# `generate` return value class Response(BaseModel): + # Generated text generated_text: str + # Generation details details: Details +# `generate_stream` details +class StreamDetails(BaseModel): + # Generation finish reason + finish_reason: FinishReason + # Number of generated tokens + generated_tokens: int + # Sampling seed if sampling was activated + seed: Optional[int] + + +# `generate_stream` return value class StreamResponse(BaseModel): + # Generated token token: Token + # Complete generated text + # Only available when the generation is finished generated_text: Optional[str] + # Generation details + # Only available when the generation is finished details: Optional[StreamDetails] From 411d6247f41d5688ccb63a9f93fb97dfc50b05c0 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 9 Mar 2023 16:07:01 +0100 Subject: [PATCH 117/793] v0.4.0 (#119) --- Cargo.lock | 8 ++++---- docs/openapi.json | 2 +- launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- router/grpc-metadata/Cargo.toml | 2 +- server/pyproject.toml | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a358b2a..9db5f213 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -786,7 +786,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "grpc-metadata" -version = "0.3.2" +version = "0.4.0" dependencies = [ "opentelemetry", "tonic", @@ -2212,7 +2212,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.3.2" +version = "0.4.0" dependencies = [ "futures", "grpc-metadata", @@ -2229,7 +2229,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.3.2" +version = "0.4.0" dependencies = [ "clap 4.1.8", "ctrlc", @@ -2244,7 +2244,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.3.2" +version = "0.4.0" dependencies = [ "async-stream", "axum", diff --git a/docs/openapi.json b/docs/openapi.json index 881b892c..a2230a07 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -11,7 +11,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.3.2" + "version": "0.4.0" }, "paths": { "/generate": { diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 4779aa91..49948b05 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.3.2" +version = "0.4.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index 2da8a9a0..292e6219 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.3.2" +version = "0.4.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 960e509f..fd43b63b 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.3.2" +version = "0.4.0" edition = "2021" [dependencies] diff --git a/router/grpc-metadata/Cargo.toml b/router/grpc-metadata/Cargo.toml index e779fbbf..f8676d67 100644 --- a/router/grpc-metadata/Cargo.toml +++ b/router/grpc-metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grpc-metadata" -version = "0.3.2" +version = "0.4.0" edition = "2021" [dependencies] diff --git a/server/pyproject.toml b/server/pyproject.toml index b8e188b7..63a3df6c 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.3.2" +version = "0.4.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 6860ce9c677c48e0fb9e1212eaae96cb6730061d Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 13 Mar 2023 20:42:10 +0100 Subject: [PATCH 118/793] feat: add OpenAssistant/oasst-sft-1-pythia-12b to the list of supported models (#122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ed models --- supported_models.json | 1 + 1 file changed, 1 insertion(+) diff --git a/supported_models.json b/supported_models.json index edf01e70..86d3bdfe 100644 --- a/supported_models.json +++ b/supported_models.json @@ -4,5 +4,6 @@ "EleutherAI/gpt-neox-20b", "google/flan-ul2", "google/flan-t5-xxl", + "OpenAssistant/oasst-sft-1-pythia-12b", "olivierdehaene/optimized-santacoder" ] From cbd36aa4d16cd253b268b4684fc359f66c5f0660 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 13 Mar 2023 22:57:08 +0100 Subject: [PATCH 119/793] fix(server): revert gpt-neox optims (#123) --- router/src/infer.rs | 6 +++++- server/Makefile | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index 5955faec..e11f4fe6 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -229,7 +229,11 @@ async fn batching_task( shared: Arc, ) { // Minimum batch size after which we try to add more requests - let limit_min_batch_size = (max_batch_size / 2) as u32; + let limit_min_batch_size = if max_batch_size > 1 { + (max_batch_size / 2) as u32 + } else { + 0 + }; // Infinite loop loop { diff --git a/server/Makefile b/server/Makefile index 3de13e8e..44cf7c5c 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,4 +1,4 @@ -transformers_commit := 2f87dca1ca3e5663d0637da9bb037a6956e57a5e +transformers_commit := 517563354a3226ecfc3dca6e7a38012668d7156a gen-server: # Compile protos From 8ad60b752fdf678153e7685d9f0e30e88a60cd42 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 15 Mar 2023 13:12:49 +0100 Subject: [PATCH 120/793] fix(server): add position ids to neox (#126) --- server/Makefile | 2 +- .../text_generation_server/models/__init__.py | 5 ++--- .../models/causal_lm.py | 4 +--- .../models/galactica.py | 4 +--- .../text_generation_server/models/gpt_neox.py | 21 +++---------------- .../models/seq2seq_lm.py | 4 +--- .../text_generation_server/utils/watermark.py | 4 +++- 7 files changed, 12 insertions(+), 32 deletions(-) diff --git a/server/Makefile b/server/Makefile index 44cf7c5c..e8b0364e 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,4 +1,4 @@ -transformers_commit := 517563354a3226ecfc3dca6e7a38012668d7156a +transformers_commit := 2b57aa18da658e7d2f42ef6bd5b56751af582fef gen-server: # Compile protos diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 0035c1c6..3e2f5c66 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -9,7 +9,7 @@ from text_generation_server.models.seq2seq_lm import Seq2SeqLM from text_generation_server.models.galactica import Galactica, GalacticaSharded from text_generation_server.models.santacoder import SantaCoder -from text_generation_server.models.gpt_neox import GPTNeox, GPTNeoxSharded +from text_generation_server.models.gpt_neox import GPTNeoxSharded from text_generation_server.models.t5 import T5Sharded __all__ = [ @@ -19,7 +19,6 @@ "CausalLM", "Galactica", "GalacticaSharded", - "GPTNeox", "GPTNeoxSharded", "Seq2SeqLM", "SantaCoder", @@ -62,7 +61,7 @@ def get_model( if sharded: return GPTNeoxSharded(model_id, revision, quantize=quantize) else: - return GPTNeox(model_id, revision, quantize=quantize) + return CausalLM(model_id, revision, quantize=quantize) if config.model_type == "t5": if sharded: diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index ef3f0260..c979b7bc 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -72,9 +72,7 @@ def from_pb( for r in pb.requests: inputs.append(r.inputs) input_lengths.append(r.input_length) - next_token_choosers.append( - NextTokenChooser.from_pb(r.parameters, device) - ) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index d04a3bce..a90a299e 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -102,9 +102,7 @@ def from_pb( # Add escape_custom_split_sequence to the CausalLMBatch logic inputs.append(escape_custom_split_sequence(r.inputs)) input_lengths.append(r.input_length) - next_token_choosers.append( - NextTokenChooser.from_pb(r.parameters, device) - ) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 5e1960f4..8fabefe3 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -1,7 +1,7 @@ import torch import torch.distributed -from typing import List, Optional, Tuple +from typing import List, Optional from accelerate import init_empty_weights from safetensors import safe_open @@ -30,23 +30,7 @@ HAS_BITS_AND_BYTES = False -class GPTNeox(CausalLM): - def forward( - self, input_ids, attention_mask, position_ids, past_key_values: Optional = None - ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: - """Overwrite forward to ignore position_ids""" - - # Model Forward - outputs = self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - use_cache=True, - ) - return outputs.logits, outputs.past_key_values - - -class GPTNeoxSharded(GPTNeox): +class GPTNeoxSharded(CausalLM): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): @@ -224,6 +208,7 @@ def forward( outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask, + position_ids=position_ids, past_key_values=past_key_values, use_cache=True, ) diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index bece913a..0f7f4df9 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -82,9 +82,7 @@ def from_pb( # Decoder sequence only contains the bos_token decoder_input_ids.append(tokenizer.bos_token_id) decoder_input_lengths.append(1) - next_token_choosers.append( - NextTokenChooser.from_pb(r.parameters, device) - ) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) diff --git a/server/text_generation_server/utils/watermark.py b/server/text_generation_server/utils/watermark.py index cf6214ce..8e90a59c 100644 --- a/server/text_generation_server/utils/watermark.py +++ b/server/text_generation_server/utils/watermark.py @@ -43,7 +43,9 @@ def _seed_rng(self, input_ids: torch.LongTensor) -> None: prev_token = input_ids[-1].item() self.rng.manual_seed(self.hash_key * prev_token) - def _get_greenlist_ids(self, input_ids: torch.LongTensor, max_value: int) -> list[int]: + def _get_greenlist_ids( + self, input_ids: torch.LongTensor, max_value: int + ) -> list[int]: # seed the rng using the previous tokens/prefix self._seed_rng(input_ids) From b49dbf2d88c340c3686e4318985d8b64581364b7 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 16 Mar 2023 12:12:26 +0100 Subject: [PATCH 121/793] fix(server): use server tokenizer as gt (#128) --- proto/generate.proto | 6 +-- router/src/infer.rs | 6 ++- router/src/queue.rs | 5 +- router/src/validation.rs | 2 - server/tests/models/test_bloom.py | 7 ++- server/tests/models/test_causal_lm.py | 7 ++- server/tests/models/test_santacoder.py | 2 - server/tests/models/test_seq2seq_lm.py | 1 - .../models/causal_lm.py | 52 +++++++++---------- .../models/seq2seq_lm.py | 12 ++--- 10 files changed, 46 insertions(+), 54 deletions(-) diff --git a/proto/generate.proto b/proto/generate.proto index a47e2ec1..5081ce1c 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -58,12 +58,10 @@ message Request { uint64 id = 1; /// The generation context string inputs = 2; - /// The number of tokens inside inputs - uint32 input_length = 3; /// Next Token Chooser Parameters - NextTokenChooserParameters parameters = 4; + NextTokenChooserParameters parameters = 3; /// Stopping Criteria Parameters - StoppingCriteriaParameters stopping_parameters = 5; + StoppingCriteriaParameters stopping_parameters = 4; } message Batch { diff --git a/router/src/infer.rs b/router/src/infer.rs index e11f4fe6..ae151d8a 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -278,7 +278,8 @@ async fn batching_task( // because a new batch is being computed let entry_waiting_span = info_span!(parent: &entry.span, "waiting", batch_size = new_batch_size); - // Add relationship + // Add relationships + span.follows_from(&entry_waiting_span); entry_waiting_span.follows_from(&span); // Update entry entry.temp_span = Some(entry_waiting_span); @@ -305,7 +306,8 @@ async fn batching_task( // Create a new span to link the batch back to this entry let entry_batch_span = info_span!(parent: &entry.span, "infer", batch_size = next_batch_size); - // Add relationship + // Add relationships + next_batch_span.follows_from(&entry_batch_span); entry_batch_span.follows_from(&next_batch_span); // Update entry entry.temp_span = Some(entry_batch_span); diff --git a/router/src/queue.rs b/router/src/queue.rs index db3c509e..df2087e1 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -165,7 +165,8 @@ impl State { // Create a new span to link the batch back to this entry let entry_batch_span = info_span!(parent: &entry.span, "infer", batch_size = next_batch_size); - // Add relationship + // Add relationships + next_batch_span.follows_from(&entry_batch_span); entry_batch_span.follows_from(&next_batch_span); // Update entry entry.temp_span = Some(entry_batch_span); @@ -173,7 +174,6 @@ impl State { batch_requests.push(Request { id, inputs: entry.request.inputs.clone(), - input_length: entry.request.input_length, parameters: Some(entry.request.parameters.clone()), stopping_parameters: Some(entry.request.stopping_parameters.clone()), }); @@ -226,7 +226,6 @@ mod tests { Entry { request: ValidGenerateRequest { inputs: "".to_string(), - input_length: 0, parameters: NextTokenChooserParameters { temperature: 0.0, top_k: 0, diff --git a/router/src/validation.rs b/router/src/validation.rs index cb8dd0a2..1c350caa 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -322,7 +322,6 @@ fn validate( Ok(ValidGenerateRequest { inputs, - input_length: input_length as u32, parameters, stopping_parameters, }) @@ -337,7 +336,6 @@ type ValidationRequest = ( #[derive(Debug)] pub(crate) struct ValidGenerateRequest { pub inputs: String, - pub input_length: u32, pub parameters: NextTokenChooserParameters, pub stopping_parameters: StoppingCriteriaParameters, } diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 90239f95..2b8ef5f8 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -24,7 +24,6 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", - input_length=1, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) @@ -77,7 +76,7 @@ def test_batch_from_pb(default_pb_batch, default_bloom_batch): assert batch.size == default_pb_batch.size assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == batch.size - assert batch.max_sequence_length == batch.input_lengths[0] + assert batch.max_input_length == batch.input_lengths[0] def test_batch_concatenate_no_prefill(default_bloom_batch): @@ -110,7 +109,7 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): assert next_batch.input_ids[0, 0] == 10264 assert next_batch.input_lengths == [2] - assert next_batch.max_sequence_length == next_batch.input_lengths[0] + assert next_batch.max_input_length == next_batch.input_lengths[0] assert next_batch.past_key_values is not None assert all( @@ -222,7 +221,7 @@ def test_batch_concatenate( assert torch.all(next_batch.input_ids == 10264) assert next_batch.input_lengths == [3, 2, 2] - assert next_batch.max_sequence_length == 3 + assert next_batch.max_input_length == 3 assert next_batch.requests[0] == next_batch_0.requests[0] assert next_batch.requests[1:] == next_batch_1.requests diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 869022fa..76617b62 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -25,7 +25,6 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", - input_length=1, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) @@ -74,7 +73,7 @@ def test_batch_from_pb(default_pb_batch, default_causal_lm_batch): assert batch.size == default_pb_batch.size assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == batch.size - assert batch.max_sequence_length == batch.input_lengths[0] + assert batch.max_input_length == batch.input_lengths[0] def test_batch_concatenate_no_prefill(default_causal_lm_batch): @@ -107,7 +106,7 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): assert next_batch.input_ids[0, 0] == 13 assert next_batch.input_lengths == [2] - assert next_batch.max_sequence_length == next_batch.input_lengths[0] + assert next_batch.max_input_length == next_batch.input_lengths[0] assert next_batch.past_key_values is not None assert all( @@ -220,7 +219,7 @@ def test_batch_concatenate( assert torch.all(next_batch.input_ids[1:] == 13) assert next_batch.input_lengths == [3, 2, 2] - assert next_batch.max_sequence_length == 3 + assert next_batch.max_input_length == 3 assert next_batch.requests[0] == next_batch_0.requests[0] assert next_batch.requests[1:] == next_batch_1.requests diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index d089def3..753ff5fc 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -15,7 +15,6 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="def", - input_length=1, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) @@ -31,7 +30,6 @@ def default_fim_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="defworld", - input_length=5, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 764f8f83..2d86c44b 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -28,7 +28,6 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", - input_length=2, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index c979b7bc..88ea6c75 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -41,7 +41,7 @@ class CausalLMBatch(Batch): # Metadata used for padding size: int - max_sequence_length: int + max_input_length: int padding_right_offset: int # Past metadata @@ -67,17 +67,14 @@ def from_pb( input_lengths = [] # Parse batch - max_sequence_length = 0 padding_right_offset = 0 for r in pb.requests: inputs.append(r.inputs) - input_lengths.append(r.input_length) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) stopping_criterias.append(stopping_criteria) - max_sequence_length = max(max_sequence_length, r.input_length) padding_right_offset = max( padding_right_offset, stopping_criteria.max_new_tokens ) @@ -89,13 +86,16 @@ def from_pb( return_token_type_ids=False, ).to(device) + input_lengths = tokenized_inputs["attention_mask"].sum(1) + max_input_length = input_lengths.max() + input_ids = tokenized_inputs["input_ids"] # Allocate maximum attention_mask attention_mask = input_ids.new_zeros( - (pb.size, max_sequence_length + padding_right_offset) + (pb.size, max_input_length + padding_right_offset) ) # Copy tokenizer attention_mask into fully allocated attention_mask - attention_mask[:, :max_sequence_length] = tokenized_inputs["attention_mask"] + attention_mask[:, :max_input_length] = tokenized_inputs["attention_mask"] position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) @@ -109,11 +109,11 @@ def from_pb( position_ids=position_ids, past_key_values=None, all_input_ids=all_input_ids, - input_lengths=input_lengths, + input_lengths=input_lengths.tolist(), next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=pb.size, - max_sequence_length=max_sequence_length, + max_input_length=max_input_length.item(), padding_right_offset=padding_right_offset, ) @@ -122,11 +122,11 @@ def from_pb( def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # Used for padding total_batch_size = 0 - max_sequence_length = 0 + max_input_length = 0 padding_right_offset = 0 for batch in batches: total_batch_size += batch.size - max_sequence_length = max(max_sequence_length, batch.max_sequence_length) + max_input_length = max(max_input_length, batch.max_input_length) padding_right_offset = max(padding_right_offset, batch.padding_right_offset) # Batch attributes @@ -170,15 +170,15 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # Create padded tensor if attention_mask is None: attention_mask = batch.attention_mask.new_zeros( - (total_batch_size, max_sequence_length + padding_right_offset), + (total_batch_size, max_input_length + padding_right_offset), ) # We need to slice the attention mask to remove padding from previous steps # and to remove unused allocated space - left_offset = max_sequence_length - batch.max_sequence_length + left_offset = max_input_length - batch.max_input_length batch_left_offset = ( batch.attention_mask.shape[1] - - batch.max_sequence_length + - batch.max_input_length - batch.padding_right_offset ) attention_mask[ @@ -209,7 +209,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": padded_past_values_shape = ( total_batch_size, num_heads, - max_sequence_length - 1, + max_input_length - 1, head_dim, ) @@ -221,7 +221,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": total_batch_size, num_heads, head_dim, - max_sequence_length - 1, + max_input_length - 1, ) # This will run only once per layer @@ -235,20 +235,20 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": past_key_values[j][0][ start_index:end_index, :, - -(batch.max_sequence_length - 1) :, + -(batch.max_input_length - 1) :, :, - ] = past_keys[:, :, -(batch.max_sequence_length - 1) :, :] + ] = past_keys[:, :, -(batch.max_input_length - 1) :, :] else: past_key_values[j][0][ start_index:end_index, :, :, - -(batch.max_sequence_length - 1) :, - ] = past_keys[:, :, :, -(batch.max_sequence_length - 1) :] + -(batch.max_input_length - 1) :, + ] = past_keys[:, :, :, -(batch.max_input_length - 1) :] past_key_values[j][1][ - start_index:end_index, :, -(batch.max_sequence_length - 1) :, : - ] = past_values[:, :, -(batch.max_sequence_length - 1) :, :] + start_index:end_index, :, -(batch.max_input_length - 1) :, : + ] = past_values[:, :, -(batch.max_input_length - 1) :, :] start_index += batch.size @@ -264,7 +264,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=total_batch_size, - max_sequence_length=max_sequence_length, + max_input_length=max_input_length, padding_right_offset=padding_right_offset, keys_head_dim_last=batches[0].keys_head_dim_last, ) @@ -352,7 +352,7 @@ def generate_token( # Metadata next_batch_size = 0 - next_batch_max_sequence_length = 0 + next_batch_max_input_length = 0 # Results generations: List[Generation] = [] @@ -420,8 +420,8 @@ def generate_token( next_batch_all_input_ids.append(all_input_ids) next_batch_size += 1 next_batch_input_lengths.append(new_input_length) - next_batch_max_sequence_length = max( - next_batch_max_sequence_length, new_input_length + next_batch_max_input_length = max( + next_batch_max_input_length, new_input_length ) # Prefill @@ -506,7 +506,7 @@ def generate_token( next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, - max_sequence_length=next_batch_max_sequence_length, + max_input_length=next_batch_max_input_length, padding_right_offset=batch.padding_right_offset - 1, keys_head_dim_last=batch.keys_head_dim_last, ) diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 0f7f4df9..0fe5c03f 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -68,17 +68,14 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] - input_lengths = [] decoder_input_ids = [] decoder_input_lengths = [] # Parse batch - max_input_length = 0 padding_right_offset = 0 for r in pb.requests: inputs.append(r.inputs) - input_lengths.append(r.input_length) # Decoder sequence only contains the bos_token decoder_input_ids.append(tokenizer.bos_token_id) decoder_input_lengths.append(1) @@ -87,7 +84,6 @@ def from_pb( r.stopping_parameters, tokenizer ) stopping_criterias.append(stopping_criteria) - max_input_length = max(max_input_length, r.input_length) padding_right_offset = max( padding_right_offset, stopping_criteria.max_new_tokens ) @@ -99,6 +95,10 @@ def from_pb( padding=True, return_token_type_ids=False, ).to(device) + + input_lengths = tokenized_inputs["attention_mask"].sum(1) + max_input_length = input_lengths.max() + # Convert decoder_input_ids to torch tensor of size [batch_size, 1] decoder_input_ids = torch.tensor(decoder_input_ids, device=device).unsqueeze(-1) @@ -111,12 +111,12 @@ def from_pb( decoder_attention_mask=None, encoder_last_hidden_state=None, past_key_values=None, - input_lengths=input_lengths, + input_lengths=input_lengths.tolist(), decoder_input_lengths=decoder_input_lengths, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=len(pb.requests), - max_input_length=max(input_lengths), + max_input_length=max_input_length.item(), max_decoder_input_length=1, padding_right_offset=padding_right_offset, ) From a3b7db932ff23b0daef703e8bf5c0436b0d2b1db Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 16 Mar 2023 12:57:07 +0100 Subject: [PATCH 122/793] fix(python-client): relax dependencies (#129) --- clients/python/poetry.lock | 106 +++++++++++++++++----------------- clients/python/pyproject.toml | 8 +-- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/clients/python/poetry.lock b/clients/python/poetry.lock index fe1f34cb..1503c40e 100644 --- a/clients/python/poetry.lock +++ b/clients/python/poetry.lock @@ -114,15 +114,15 @@ toml = ["tomli"] [[package]] name = "filelock" -version = "3.9.0" +version = "3.10.0" description = "A platform independent file lock." category = "main" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2022.12.7)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.1)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] [[package]] name = "frozenlist" @@ -134,7 +134,7 @@ python-versions = ">=3.7" [[package]] name = "huggingface-hub" -version = "0.12.1" +version = "0.13.2" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" category = "main" optional = false @@ -150,13 +150,13 @@ tqdm = ">=4.42.1" typing-extensions = ">=3.7.4.3" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (==22.3)", "flake8 (>=3.8.3)", "flake8-bugbear", "isort (>=5.5.4)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (==22.3)", "flake8 (>=3.8.3)", "flake8-bugbear", "isort (>=5.5.4)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -quality = ["black (==22.3)", "flake8 (>=3.8.3)", "flake8-bugbear", "isort (>=5.5.4)", "mypy (==0.982)"] +quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] tensorflow = ["graphviz", "pydot", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "isort (>=5.5.4)", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"] torch = ["torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] @@ -234,7 +234,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pydantic" -version = "1.10.5" +version = "1.10.6" description = "Data validation and settings management using python type hints" category = "main" optional = false @@ -368,7 +368,7 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.14" +version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -407,7 +407,7 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "12eb0ce46193750e35aec1f1fb32a0ce094aebd1db30682d06424ee6ccd23536" +content-hash = "0db2f97d52c557dd7f90c55b4ad5bbe308c957c5f7f99fec53c57e0a13822cb4" [metadata.files] aiohttp = [ @@ -657,8 +657,8 @@ coverage = [ {file = "coverage-7.2.1.tar.gz", hash = "sha256:c77f2a9093ccf329dd523a9b2b3c854c20d2a3d968b6def3b820272ca6732242"}, ] filelock = [ - {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, - {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, + {file = "filelock-3.10.0-py3-none-any.whl", hash = "sha256:e90b34656470756edf8b19656785c5fea73afa1953f3e1b0d645cef11cab3182"}, + {file = "filelock-3.10.0.tar.gz", hash = "sha256:3199fd0d3faea8b911be52b663dfccceb84c95949dd13179aa21436d1a79c4ce"}, ] frozenlist = [ {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, @@ -737,8 +737,8 @@ frozenlist = [ {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, ] huggingface-hub = [ - {file = "huggingface_hub-0.12.1-py3-none-any.whl", hash = "sha256:867586cc8543fe1bd43a219fedbea7d71690021ad80f0c46f35c4751069278d7"}, - {file = "huggingface_hub-0.12.1.tar.gz", hash = "sha256:6f960f6246ef9c3446d0d6275e853485515682c350917fdaf2a59705f8b9ebb3"}, + {file = "huggingface_hub-0.13.2-py3-none-any.whl", hash = "sha256:745c4cbd97a27fc5c1c6c89cb477662004c88bc3dd89bafc1a27ef24af77f944"}, + {file = "huggingface_hub-0.13.2.tar.gz", hash = "sha256:246e8eb39b6e6e9d9d5846e4b56c265cdf1872f48ba5a13a1321295d371626f5"}, ] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, @@ -841,42 +841,42 @@ py = [ {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] pydantic = [ - {file = "pydantic-1.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5920824fe1e21cbb3e38cf0f3dd24857c8959801d1031ce1fac1d50857a03bfb"}, - {file = "pydantic-1.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3bb99cf9655b377db1a9e47fa4479e3330ea96f4123c6c8200e482704bf1eda2"}, - {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2185a3b3d98ab4506a3f6707569802d2d92c3a7ba3a9a35683a7709ea6c2aaa2"}, - {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f582cac9d11c227c652d3ce8ee223d94eb06f4228b52a8adaafa9fa62e73d5c9"}, - {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c9e5b778b6842f135902e2d82624008c6a79710207e28e86966cd136c621bfee"}, - {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:72ef3783be8cbdef6bca034606a5de3862be6b72415dc5cb1fb8ddbac110049a"}, - {file = "pydantic-1.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:45edea10b75d3da43cfda12f3792833a3fa70b6eee4db1ed6aed528cef17c74e"}, - {file = "pydantic-1.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:63200cd8af1af2c07964546b7bc8f217e8bda9d0a2ef0ee0c797b36353914984"}, - {file = "pydantic-1.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:305d0376c516b0dfa1dbefeae8c21042b57b496892d721905a6ec6b79494a66d"}, - {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd326aff5d6c36f05735c7c9b3d5b0e933b4ca52ad0b6e4b38038d82703d35b"}, - {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6bb0452d7b8516178c969d305d9630a3c9b8cf16fcf4713261c9ebd465af0d73"}, - {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9a9d9155e2a9f38b2eb9374c88f02fd4d6851ae17b65ee786a87d032f87008f8"}, - {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f836444b4c5ece128b23ec36a446c9ab7f9b0f7981d0d27e13a7c366ee163f8a"}, - {file = "pydantic-1.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:8481dca324e1c7b715ce091a698b181054d22072e848b6fc7895cd86f79b4449"}, - {file = "pydantic-1.10.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:87f831e81ea0589cd18257f84386bf30154c5f4bed373b7b75e5cb0b5d53ea87"}, - {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ce1612e98c6326f10888df951a26ec1a577d8df49ddcaea87773bfbe23ba5cc"}, - {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58e41dd1e977531ac6073b11baac8c013f3cd8706a01d3dc74e86955be8b2c0c"}, - {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6a4b0aab29061262065bbdede617ef99cc5914d1bf0ddc8bcd8e3d7928d85bd6"}, - {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:36e44a4de37b8aecffa81c081dbfe42c4d2bf9f6dff34d03dce157ec65eb0f15"}, - {file = "pydantic-1.10.5-cp37-cp37m-win_amd64.whl", hash = "sha256:261f357f0aecda005934e413dfd7aa4077004a174dafe414a8325e6098a8e419"}, - {file = "pydantic-1.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b429f7c457aebb7fbe7cd69c418d1cd7c6fdc4d3c8697f45af78b8d5a7955760"}, - {file = "pydantic-1.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:663d2dd78596c5fa3eb996bc3f34b8c2a592648ad10008f98d1348be7ae212fb"}, - {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51782fd81f09edcf265823c3bf43ff36d00db246eca39ee765ef58dc8421a642"}, - {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c428c0f64a86661fb4873495c4fac430ec7a7cef2b8c1c28f3d1a7277f9ea5ab"}, - {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:76c930ad0746c70f0368c4596020b736ab65b473c1f9b3872310a835d852eb19"}, - {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3257bd714de9db2102b742570a56bf7978e90441193acac109b1f500290f5718"}, - {file = "pydantic-1.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:f5bee6c523d13944a1fdc6f0525bc86dbbd94372f17b83fa6331aabacc8fd08e"}, - {file = "pydantic-1.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:532e97c35719f137ee5405bd3eeddc5c06eb91a032bc755a44e34a712420daf3"}, - {file = "pydantic-1.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ca9075ab3de9e48b75fa8ccb897c34ccc1519177ad8841d99f7fd74cf43be5bf"}, - {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd46a0e6296346c477e59a954da57beaf9c538da37b9df482e50f836e4a7d4bb"}, - {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3353072625ea2a9a6c81ad01b91e5c07fa70deb06368c71307529abf70d23325"}, - {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3f9d9b2be177c3cb6027cd67fbf323586417868c06c3c85d0d101703136e6b31"}, - {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b473d00ccd5c2061fd896ac127b7755baad233f8d996ea288af14ae09f8e0d1e"}, - {file = "pydantic-1.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:5f3bc8f103b56a8c88021d481410874b1f13edf6e838da607dcb57ecff9b4594"}, - {file = "pydantic-1.10.5-py3-none-any.whl", hash = "sha256:7c5b94d598c90f2f46b3a983ffb46ab806a67099d118ae0da7ef21a2a4033b28"}, - {file = "pydantic-1.10.5.tar.gz", hash = "sha256:9e337ac83686645a46db0e825acceea8e02fca4062483f40e9ae178e8bd1103a"}, + {file = "pydantic-1.10.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f9289065611c48147c1dd1fd344e9d57ab45f1d99b0fb26c51f1cf72cd9bcd31"}, + {file = "pydantic-1.10.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c32b6bba301490d9bb2bf5f631907803135e8085b6aa3e5fe5a770d46dd0160"}, + {file = "pydantic-1.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd9b9e98068fa1068edfc9eabde70a7132017bdd4f362f8b4fd0abed79c33083"}, + {file = "pydantic-1.10.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c84583b9df62522829cbc46e2b22e0ec11445625b5acd70c5681ce09c9b11c4"}, + {file = "pydantic-1.10.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b41822064585fea56d0116aa431fbd5137ce69dfe837b599e310034171996084"}, + {file = "pydantic-1.10.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:61f1f08adfaa9cc02e0cbc94f478140385cbd52d5b3c5a657c2fceb15de8d1fb"}, + {file = "pydantic-1.10.6-cp310-cp310-win_amd64.whl", hash = "sha256:32937835e525d92c98a1512218db4eed9ddc8f4ee2a78382d77f54341972c0e7"}, + {file = "pydantic-1.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbd5c531b22928e63d0cb1868dee76123456e1de2f1cb45879e9e7a3f3f1779b"}, + {file = "pydantic-1.10.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e277bd18339177daa62a294256869bbe84df1fb592be2716ec62627bb8d7c81d"}, + {file = "pydantic-1.10.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f15277d720aa57e173954d237628a8d304896364b9de745dcb722f584812c7"}, + {file = "pydantic-1.10.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b243b564cea2576725e77aeeda54e3e0229a168bc587d536cd69941e6797543d"}, + {file = "pydantic-1.10.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3ce13a558b484c9ae48a6a7c184b1ba0e5588c5525482681db418268e5f86186"}, + {file = "pydantic-1.10.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3ac1cd4deed871dfe0c5f63721e29debf03e2deefa41b3ed5eb5f5df287c7b70"}, + {file = "pydantic-1.10.6-cp311-cp311-win_amd64.whl", hash = "sha256:b1eb6610330a1dfba9ce142ada792f26bbef1255b75f538196a39e9e90388bf4"}, + {file = "pydantic-1.10.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4ca83739c1263a044ec8b79df4eefc34bbac87191f0a513d00dd47d46e307a65"}, + {file = "pydantic-1.10.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea4e2a7cb409951988e79a469f609bba998a576e6d7b9791ae5d1e0619e1c0f2"}, + {file = "pydantic-1.10.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53de12b4608290992a943801d7756f18a37b7aee284b9ffa794ee8ea8153f8e2"}, + {file = "pydantic-1.10.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:60184e80aac3b56933c71c48d6181e630b0fbc61ae455a63322a66a23c14731a"}, + {file = "pydantic-1.10.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:415a3f719ce518e95a92effc7ee30118a25c3d032455d13e121e3840985f2efd"}, + {file = "pydantic-1.10.6-cp37-cp37m-win_amd64.whl", hash = "sha256:72cb30894a34d3a7ab6d959b45a70abac8a2a93b6480fc5a7bfbd9c935bdc4fb"}, + {file = "pydantic-1.10.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3091d2eaeda25391405e36c2fc2ed102b48bac4b384d42b2267310abae350ca6"}, + {file = "pydantic-1.10.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:751f008cd2afe812a781fd6aa2fb66c620ca2e1a13b6a2152b1ad51553cb4b77"}, + {file = "pydantic-1.10.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12e837fd320dd30bd625be1b101e3b62edc096a49835392dcf418f1a5ac2b832"}, + {file = "pydantic-1.10.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:587d92831d0115874d766b1f5fddcdde0c5b6c60f8c6111a394078ec227fca6d"}, + {file = "pydantic-1.10.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:476f6674303ae7965730a382a8e8d7fae18b8004b7b69a56c3d8fa93968aa21c"}, + {file = "pydantic-1.10.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3a2be0a0f32c83265fd71a45027201e1278beaa82ea88ea5b345eea6afa9ac7f"}, + {file = "pydantic-1.10.6-cp38-cp38-win_amd64.whl", hash = "sha256:0abd9c60eee6201b853b6c4be104edfba4f8f6c5f3623f8e1dba90634d63eb35"}, + {file = "pydantic-1.10.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6195ca908045054dd2d57eb9c39a5fe86409968b8040de8c2240186da0769da7"}, + {file = "pydantic-1.10.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43cdeca8d30de9a897440e3fb8866f827c4c31f6c73838e3a01a14b03b067b1d"}, + {file = "pydantic-1.10.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c19eb5163167489cb1e0161ae9220dadd4fc609a42649e7e84a8fa8fff7a80f"}, + {file = "pydantic-1.10.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:012c99a9c0d18cfde7469aa1ebff922e24b0c706d03ead96940f5465f2c9cf62"}, + {file = "pydantic-1.10.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:528dcf7ec49fb5a84bf6fe346c1cc3c55b0e7603c2123881996ca3ad79db5bfc"}, + {file = "pydantic-1.10.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:163e79386c3547c49366e959d01e37fc30252285a70619ffc1b10ede4758250a"}, + {file = "pydantic-1.10.6-cp39-cp39-win_amd64.whl", hash = "sha256:189318051c3d57821f7233ecc94708767dd67687a614a4e8f92b4a020d4ffd06"}, + {file = "pydantic-1.10.6-py3-none-any.whl", hash = "sha256:acc6783751ac9c9bc4680379edd6d286468a1dc8d7d9906cd6f1186ed682b2b0"}, + {file = "pydantic-1.10.6.tar.gz", hash = "sha256:cf95adb0d1671fc38d8c43dd921ad5814a735e7d9b4d9e437c088002863854fd"}, ] pytest = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, @@ -953,8 +953,8 @@ typing-extensions = [ {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, ] urllib3 = [ - {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, - {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, ] yarl = [ {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 51ecce82..2aa396b8 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.3.0" +version = "0.3.1" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] @@ -12,9 +12,9 @@ repository = "https://github.com/huggingface/text-generation-inference" [tool.poetry.dependencies] python = "^3.7" -pydantic = "^1.10.5" -aiohttp = "^3.8.4" -huggingface-hub = "^0.12.1" +pydantic = "^1.10" +aiohttp = "^3.8" +huggingface-hub = ">= 0.12, < 1.0" [tool.poetry.dev-dependencies] pytest = "^6.2.5" From 78501190556521dfc1df9839831086fcdc60fcd0 Mon Sep 17 00:00:00 2001 From: dconathan Date: Thu, 23 Mar 2023 13:01:01 -0400 Subject: [PATCH 123/793] feat(python-client): add cookies to Client constructors and requests (#132) I have a use case where we need to pass cookies (for auth reasons) to an internally hosted server. Note: I couldn't get the client tests to pass - do you need to have an HF token? ```python FAILED tests/test_client.py::test_generate - text_generation.errors.BadRequestError: Authorization header is correct, but the token seems invalid ``` --- clients/python/text_generation/client.py | 28 ++++++++++++++++--- .../python/text_generation/inference_api.py | 8 ++++-- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index e05a002e..03bc3888 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -36,7 +36,11 @@ class Client: """ def __init__( - self, base_url: str, headers: Optional[Dict[str, str]] = None, timeout: int = 10 + self, + base_url: str, + headers: Optional[Dict[str, str]] = None, + cookies: Optional[Dict[str, str]] = None, + timeout: int = 10, ): """ Args: @@ -44,11 +48,14 @@ def __init__( text-generation-inference instance base url headers (`Optional[Dict[str, str]]`): Additional headers + cookies (`Optional[Dict[str, str]]`): + Cookies to include in the requests timeout (`int`): Timeout in seconds """ self.base_url = base_url self.headers = headers + self.cookies = cookies self.timeout = timeout def generate( @@ -130,6 +137,7 @@ def generate( self.base_url, json=request.dict(), headers=self.headers, + cookies=self.cookies, timeout=self.timeout, ) payload = resp.json() @@ -216,6 +224,7 @@ def generate_stream( self.base_url, json=request.dict(), headers=self.headers, + cookies=self.cookies, timeout=self.timeout, stream=True, ) @@ -267,7 +276,11 @@ class AsyncClient: """ def __init__( - self, base_url: str, headers: Optional[Dict[str, str]] = None, timeout: int = 10 + self, + base_url: str, + headers: Optional[Dict[str, str]] = None, + cookies: Optional[Dict[str, str]] = None, + timeout: int = 10, ): """ Args: @@ -275,11 +288,14 @@ def __init__( text-generation-inference instance base url headers (`Optional[Dict[str, str]]`): Additional headers + cookies (`Optional[Dict[str, str]]`): + Cookies to include in the requests timeout (`int`): Timeout in seconds """ self.base_url = base_url self.headers = headers + self.cookies = cookies self.timeout = ClientTimeout(timeout * 60) async def generate( @@ -357,7 +373,9 @@ async def generate( ) request = Request(inputs=prompt, stream=False, parameters=parameters) - async with ClientSession(headers=self.headers, timeout=self.timeout) as session: + async with ClientSession( + headers=self.headers, cookies=self.cookies, timeout=self.timeout + ) as session: async with session.post(self.base_url, json=request.dict()) as resp: payload = await resp.json() @@ -440,7 +458,9 @@ async def generate_stream( ) request = Request(inputs=prompt, stream=True, parameters=parameters) - async with ClientSession(headers=self.headers, timeout=self.timeout) as session: + async with ClientSession( + headers=self.headers, cookies=self.cookies, timeout=self.timeout + ) as session: async with session.post(self.base_url, json=request.dict()) as resp: if resp.status != 200: diff --git a/clients/python/text_generation/inference_api.py b/clients/python/text_generation/inference_api.py index bc6022b3..eb70b3d1 100644 --- a/clients/python/text_generation/inference_api.py +++ b/clients/python/text_generation/inference_api.py @@ -92,7 +92,9 @@ def __init__(self, repo_id: str, token: Optional[str] = None, timeout: int = 10) ) base_url = f"{INFERENCE_ENDPOINT}/models/{repo_id}" - super(InferenceAPIClient, self).__init__(base_url, headers, timeout) + super(InferenceAPIClient, self).__init__( + base_url, headers=headers, timeout=timeout + ) class InferenceAPIAsyncClient(AsyncClient): @@ -147,4 +149,6 @@ def __init__(self, repo_id: str, token: Optional[str] = None, timeout: int = 10) ) base_url = f"{INFERENCE_ENDPOINT}/models/{repo_id}" - super(InferenceAPIAsyncClient, self).__init__(base_url, headers, timeout) + super(InferenceAPIAsyncClient, self).__init__( + base_url, headers=headers, timeout=timeout + ) From 603e20b5f7c040ee2a9b5f10ca8bf094360fc429 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 23 Mar 2023 18:01:30 +0100 Subject: [PATCH 124/793] feat(ci): add ci paths (#134) --- .github/workflows/build.yaml | 9 +++++++++ .github/workflows/tests.yaml | 2 ++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6c752de8..220b2fa3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,6 +8,15 @@ on: tags: - 'v*' pull_request: + paths: + - ".github/workflows/build.yaml" + - "server/**" + - "proto/**" + - "router/**" + - "launcher/**" + - "Cargo.lock" + - "rust-toolchain.toml" + - "Dockerfile" branches: - 'main' diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 476ae443..1a45ad04 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,6 +8,8 @@ on: - "proto/**" - "router/**" - "launcher/**" + - "Cargo.lock" + - "rust-toolchain.toml" jobs: run_tests: From 5e5e9d4bbd51fafcca3cf4da39170e40951ca638 Mon Sep 17 00:00:00 2001 From: lewtun Date: Thu, 23 Mar 2023 18:03:45 +0100 Subject: [PATCH 125/793] feat: Add note about NVIDIA drivers (#64) Co-authored-by: OlivierDehaene --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 5cd26a0e..ee724487 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ volume=$PWD/data # share a volume with the Docker container to avoid downloading docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --num-shard $num_shard ``` +**Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. You can then query the model using either the `/generate` or `/generate_stream` routes: @@ -119,8 +120,6 @@ for response in client.generate_stream("What is Deep Learning?", max_new_tokens= print(text) ``` -**Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). - ### API documentation You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. From 5d04525cb9c12065e4284fac782f1da9379537b9 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 23 Mar 2023 18:07:20 +0100 Subject: [PATCH 126/793] feat(python-client): release v0.4.0 (#135) --- clients/python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 2aa396b8..505717de 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.3.1" +version = "0.4.0" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] From 23e1028822d0e79605768f2ce0cf47f5c5851397 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 23 Mar 2023 18:13:04 +0100 Subject: [PATCH 127/793] feat(python-client): add CI (#136) --- .github/workflows/client-tests.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/client-tests.yaml diff --git a/.github/workflows/client-tests.yaml b/.github/workflows/client-tests.yaml new file mode 100644 index 00000000..07eeb270 --- /dev/null +++ b/.github/workflows/client-tests.yaml @@ -0,0 +1,25 @@ +name: Python Client Tests + +on: + pull_request: + paths: + - ".github/workflows/client-tests.yaml" + - "clients/python/**" + +jobs: + run_tests: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.9 + - name: Install + run: | + cd clients/python && pip install . + - name: Run tests + run: | + pip install pytest pytest-asyncio + cd clients/python && pytest tests From 05e9a796cc553b44608456bc3409d52515e56093 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 24 Mar 2023 14:02:14 +0100 Subject: [PATCH 128/793] feat(server): flash neoX (#133) --- .github/workflows/build.yaml | 4 + .github/workflows/tests.yaml | 4 + Dockerfile | 9 +- server/Makefile | 17 +- .../text_generation_server/models/__init__.py | 20 +- .../models/causal_lm.py | 1 - .../models/flash_neox.py | 601 +++++++++++++++++ .../models/flash_neox_modeling.py | 637 ++++++++++++++++++ server/text_generation_server/utils/tokens.py | 2 +- .../text_generation_server/utils/watermark.py | 37 +- 10 files changed, 1307 insertions(+), 25 deletions(-) create mode 100644 server/text_generation_server/models/flash_neox.py create mode 100644 server/text_generation_server/models/flash_neox_modeling.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 220b2fa3..56015177 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,6 +20,10 @@ on: branches: - 'main' +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: build-and-push-image: runs-on: ubuntu-latest diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1a45ad04..f96c53fb 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -11,6 +11,10 @@ on: - "Cargo.lock" - "rust-toolchain.toml" +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: run_tests: runs-on: ubuntu-20.04 diff --git a/Dockerfile b/Dockerfile index 5fbf8985..592f1f72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ ENV LANG=C.UTF-8 \ CONDA_DEFAULT_ENV=text-generation \ PATH=$PATH:/opt/miniconda/envs/text-generation/bin:/opt/miniconda/bin:/usr/local/cuda/bin -RUN apt-get update && apt-get install -y unzip curl libssl-dev && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y git curl libssl-dev && rm -rf /var/lib/apt/lists/* RUN cd ~ && \ curl -L -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ @@ -53,10 +53,13 @@ RUN cd ~ && \ WORKDIR /usr/src +# Install torch +RUN pip install torch --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir + COPY server/Makefile server/Makefile -# Install specific version of torch -RUN cd server && make install-torch +# Install specific version of flash attention +RUN cd server && make install-flash-attention # Install specific version of transformers RUN cd server && BUILD_EXTENSIONS="True" make install-transformers diff --git a/server/Makefile b/server/Makefile index e8b0364e..69ef9bc5 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,4 +1,5 @@ transformers_commit := 2b57aa18da658e7d2f42ef6bd5b56751af582fef +flash_att_commit := 4d87e4d875077ad9efd25030efa4ab0ba92c19e1 gen-server: # Compile protos @@ -12,13 +13,19 @@ install-transformers: # Install specific version of transformers with custom cuda kernels pip uninstall transformers -y || true rm -rf transformers || true - rm -rf transformers-$(transformers_commit) || true - curl -L -O https://github.com/OlivierDehaene/transformers/archive/$(transformers_commit).zip - unzip $(transformers_commit).zip - rm $(transformers_commit).zip - mv transformers-$(transformers_commit) transformers + git clone https://github.com/OlivierDehaene/transformers.git + cd transformers && git checkout $(transformers_commit) cd transformers && python setup.py install +install-flash-attention: + # Install specific version of flash attention + pip install packaging + pip uninstall flash_attn rotary_emb dropout_layer_norm -y || true + rm -rf flash-attention || true + git clone https://github.com/HazyResearch/flash-attention.git + cd flash-attention && git checkout $(flash_att_commit) + cd flash-attention && python setup.py install && cd csrc/layer_norm && python setup.py install && cd ../rotary && python setup.py install + install-torch: # Install specific version of torch pip install torch --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 3e2f5c66..2f637ae1 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -1,5 +1,7 @@ +import os import torch +from loguru import logger from transformers import AutoConfig from typing import Optional @@ -12,6 +14,14 @@ from text_generation_server.models.gpt_neox import GPTNeoxSharded from text_generation_server.models.t5 import T5Sharded +try: + from text_generation_server.models.flash_neox import FlashNeoX, FlashNeoXSharded + FLASH_NEOX = torch.cuda.is_available() and int(os.environ.get("FLASH_NEOX", 0)) == 1 +except ImportError: + if int(os.environ.get("FLASH_NEOX", 0)) == 1: + logger.exception("Could not import FlashNeoX") + FLASH_NEOX = False + __all__ = [ "Model", "BLOOM", @@ -26,6 +36,10 @@ "get_model", ] +if FLASH_NEOX: + __all__.append(FlashNeoX) + __all__.append(FlashNeoXSharded) + # The flag below controls whether to allow TF32 on matmul. This flag defaults to False # in PyTorch 1.12 and later. torch.backends.cuda.matmul.allow_tf32 = True @@ -59,9 +73,11 @@ def get_model( if config.model_type == "gpt_neox": if sharded: - return GPTNeoxSharded(model_id, revision, quantize=quantize) + neox_cls = FlashNeoXSharded if FLASH_NEOX else GPTNeoxSharded + return neox_cls(model_id, revision, quantize=quantize) else: - return CausalLM(model_id, revision, quantize=quantize) + neox_cls = FlashNeoX if FLASH_NEOX else CausalLM + return neox_cls(model_id, revision, quantize=quantize) if config.model_type == "t5": if sharded: diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 88ea6c75..c2ad0587 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -64,7 +64,6 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] - input_lengths = [] # Parse batch padding_right_offset = 0 diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py new file mode 100644 index 00000000..7be4708b --- /dev/null +++ b/server/text_generation_server/models/flash_neox.py @@ -0,0 +1,601 @@ +import torch +import torch.distributed + +from accelerate import init_empty_weights +from dataclasses import dataclass +from opentelemetry import trace +from safetensors import safe_open +from transformers import AutoTokenizer, PreTrainedTokenizerBase, AutoConfig +from typing import Optional, Tuple, List, Type, Union + +from text_generation_server.models import Model +from text_generation_server.models.flash_neox_modeling import ( + FlashGPTNeoXForCausalLM, + TensorParallelEmbedding, + TensorParallelRowLinear, + TensorParallelColumnLinear, +) +from text_generation_server.models.types import ( + Batch, + PrefillTokens, + Generation, + GeneratedText, +) +from text_generation_server.pb import generate_pb2 +from text_generation_server.utils import ( + NextTokenChooser, + StoppingCriteria, + Sampling, + initialize_torch_distributed, + weight_files, +) + +tracer = trace.get_tracer(__name__) + + +@dataclass +class FlashNeoXBatch(Batch): + batch_id: int + requests: List[generate_pb2.Request] + + # Decoder values + input_ids: torch.Tensor + position_ids: torch.Tensor + # cumulative sequence lengths + cu_seqlens: torch.Tensor + max_seqlen: int + past_key_values: Optional[torch.Tensor] + + # All tokens + all_input_ids: List[List[int]] + + # Lengths of all generations present in the batch + input_lengths: List[int] + + # Generation helpers + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + + def to_pb(self) -> generate_pb2.Batch: + return generate_pb2.Batch( + id=self.batch_id, requests=self.requests, size=len(self) + ) + + @classmethod + def from_pb( + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + device: torch.device, + ) -> "CausalLMBatch": + input_ids = [] + position_ids = [] + cu_seqlens = [0] + max_seqlen = 0 + + input_lengths = [] + all_input_ids = [] + + next_token_choosers = [] + stopping_criterias = [] + + # Cumulative length + cumulative_length = 0 + + # Parse batch + for r in pb.requests: + tokenized_input = tokenizer(r.inputs, return_tensors="pt")[ + "input_ids" + ].squeeze(0) + input_ids.append(tokenized_input) + all_input_ids.append(tokenized_input.tolist()) + + input_length = len(tokenized_input) + max_seqlen = max(max_seqlen, input_length) + input_lengths.append(input_length) + + # Position ids + position_ids.append(torch.arange(0, input_length, dtype=torch.int32)) + + # Add cumulative lengths of all previous inputs + cu_seqlens.append(cumulative_length + input_length) + + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + stopping_criterias.append( + StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) + ) + + # Update + cumulative_length += input_length + + input_ids = torch.concat(input_ids).unsqueeze(1) + position_ids = torch.concat(position_ids) + cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32) + + return cls( + batch_id=pb.id, + requests=pb.requests, + input_ids=input_ids, + position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + past_key_values=None, + input_lengths=input_lengths, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + ) + + @classmethod + @tracer.start_as_current_span("concatenate") + def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": + # Batch attributes + requests = [] + input_lengths = [] + all_input_ids = [] + next_token_choosers = [] + stopping_criterias = [] + + # Batch tensors + input_ids = [] + position_ids = [] + cu_seqlens = [torch.tensor([0], dtype=torch.int32)] + max_seqlen = 0 + past_key_values = [] + + # Cumulative length + cumulative_length = torch.tensor(0) + + for i, batch in enumerate(batches): + requests.extend(batch.requests) + input_lengths.extend(batch.input_lengths) + all_input_ids.extend(batch.all_input_ids) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + + # Add cumulative lengths of all previous inputs + cu_seqlens.append(batch.cu_seqlens[1:] + cumulative_length) + + input_ids.append(batch.input_ids) + position_ids.append(batch.position_ids) + past_key_values.append(batch.past_key_values) + + max_seqlen = max(max_seqlen, batch.max_seqlen) + + # Update + cumulative_length += batch.cu_seqlens[-1] + + input_ids = torch.concat(input_ids) + position_ids = torch.concat(position_ids) + # Concat on dim=1 as first dim represents the model layers + past_key_values = torch.concat(past_key_values, dim=1) + cu_seqlens = torch.concat(cu_seqlens) + + return FlashNeoXBatch( + batch_id=batches[0].batch_id, + requests=requests, + input_ids=input_ids, + position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + past_key_values=past_key_values, + input_lengths=input_lengths, + all_input_ids=all_input_ids, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + ) + + def __len__(self): + return len(self.requests) + + +class FlashNeoX(Model): + def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + else: + raise NotImplementedError("FlashNeoX is only available on GPU") + + if quantize: + raise NotImplementedError("FlashNeoX does not support quantization") + + tokenizer = AutoTokenizer.from_pretrained( + model_id, revision=revision, padding_side="left" + ) + self.model = ( + FlashGPTNeoXForCausalLM.from_pretrained( + model_id, + revision=revision, + torch_dtype=dtype, + ) + .eval() + .cuda() + ) + tokenizer.pad_token_id = ( + self.model.config.pad_token_id + if self.model.config.pad_token_id is not None + else self.model.config.eos_token_id + ) + + super(FlashNeoX, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @property + def batch_type(self) -> Type[FlashNeoXBatch]: + return FlashNeoXBatch + + def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: + return self.tokenizer.decode( + generated_ids, skip_special_tokens=True, cleanup_tokenization_spaces=False + ) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlens: torch.Tensor, + max_s: int, + past_key_values: Optional = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Model Forward + return self.model.forward( + input_ids=input_ids, + position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_s=max_s, + past_key_values=past_key_values, + ) + + @tracer.start_as_current_span("generate_token") + def generate_token( + self, batch: FlashNeoXBatch + ) -> Tuple[List[Generation], Optional[FlashNeoXBatch]]: + # Better to send to device here to avoid device issues in concatenate + position_ids = batch.position_ids.to(self.device, non_blocking=True) + cu_seqlens = batch.cu_seqlens.to(self.device, non_blocking=True) + input_ids = batch.input_ids.squeeze(1).to(self.device) + + out, present = self.forward( + input_ids, + position_ids, + cu_seqlens, + batch.max_seqlen, + batch.past_key_values, + ) + + # List of indices to cache + next_batch_keep_indices = [] + + # New values for next forward + next_batch_input_ids = [] + next_batch_position_ids = [] + next_batch_cu_seqlens = [0] + next_batch_max_seqlen = 0 + next_batch_past_key_values = [] + next_batch_input_lengths = [] + next_batch_all_input_ids = [] + + # Cumulative length + cumulative_length = 0 + + # Results + generations: List[Generation] = [] + + # Zipped iterator + iterator = zip( + batch.requests, + batch.input_lengths, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + ) + + # For each member of the batch + for i, ( + request, + input_length, + next_token_chooser, + stopping_criteria, + all_input_ids, + ) in enumerate(iterator): + # Indexing metadata + start_index = cumulative_length + end_index = cumulative_length + input_length + + if batch.past_key_values is None: + # Prefill mode + # out is of shape [cumulative_sequence_lengths, vocab_size] + logits = out[start_index:end_index] + else: + # Decode mode + # out is of shape [batch_size, vocab_size] + logits = out[i].unsqueeze(0) + + # Select next token + next_token_id, logprobs = next_token_chooser(all_input_ids, logits) + # Copy to cpu to avoid other copies when indexing and calling .item() + next_token_id = next_token_id.to("cpu", non_blocking=True) + logprobs = logprobs.to("cpu") + + next_token_id_squeezed = next_token_id.squeeze() + next_token_id_item = next_token_id_squeezed.item() + + # Append next token to all tokens + all_input_ids.append(next_token_id_item) + new_input_length = input_length + 1 + + # Generated token + next_token_logprob = logprobs[-1, next_token_id] + next_token_text = self.decode_token( + next_token_id_item, + ) + + # Evaluate stopping criteria + stop, reason = stopping_criteria( + next_token_id_item, + next_token_text, + ) + + if stop: + # Decode generated tokens + output_text = self.decode( + all_input_ids[-stopping_criteria.current_tokens :] + ) + # Get seed + if isinstance(next_token_chooser.choice, Sampling): + seed = next_token_chooser.choice.seed + else: + seed = None + + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed + ) + else: + # Keep request in the batch + next_batch_keep_indices.append(i) + generated_text = None + + # Get sequence present + seq_present = present[:, start_index:end_index] + # Pad it for next iter attention + past = torch.nn.functional.pad(seq_present, (0, 0, 0, 0, 0, 0, 0, 1)) + next_batch_past_key_values.append(past) + + next_batch_input_ids.append(next_token_id) + next_batch_position_ids.append(input_length) + # Cumulative sum + next_batch_cu_seqlens.append( + next_batch_cu_seqlens[-1] + new_input_length + ) + next_batch_input_lengths.append(new_input_length) + next_batch_all_input_ids.append(all_input_ids) + next_batch_max_seqlen = max(next_batch_max_seqlen, new_input_length) + + # Prefill + if stopping_criteria.current_tokens == 1: + # Remove generated token to only have prefill and add nan for first prompt token + prefill_logprobs = [float("nan")] + logprobs.gather( + 1, torch.tensor(all_input_ids[1:]).unsqueeze(1) + ).squeeze(1)[:-1].tolist() + prefill_token_ids = all_input_ids[:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = PrefillTokens( + prefill_token_ids, prefill_logprobs, prefill_texts + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id_item, + next_token_logprob, + next_token_text, + next_token_id_item in self.all_special_ids, + generated_text, + ) + + generations.append(generation) + cumulative_length += input_length + + # We finished all generations in the batch; there is no next batch + if not next_batch_keep_indices: + return generations, None + + # If we finished at least one generation, we need to evict the indices of the generations that finished + # from the values of the next batch + if len(next_batch_keep_indices) != len(batch): + # Apply indices to requests, token_choosers and stopping_criterias that need to be cached + next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] + next_batch_next_token_choosers = [ + batch.next_token_choosers[i] for i in next_batch_keep_indices + ] + next_batch_stopping_criterias = [ + batch.stopping_criterias[i] for i in next_batch_keep_indices + ] + else: + next_batch_requests = batch.requests + next_batch_next_token_choosers = batch.next_token_choosers + next_batch_stopping_criterias = batch.stopping_criterias + + # Create final next batch tensors + next_batch_position_ids = torch.tensor( + next_batch_position_ids, dtype=torch.int32 + ) + next_batch_cu_seqlens = torch.tensor(next_batch_cu_seqlens, dtype=torch.int32) + if len(next_batch_keep_indices) > 1: + next_batch_input_ids = torch.concat(next_batch_input_ids) + next_batch_past_key_values = torch.concat(next_batch_past_key_values, dim=1) + else: + next_batch_input_ids = next_batch_input_ids[0] + next_batch_past_key_values = next_batch_past_key_values[0] + + next_batch = FlashNeoXBatch( + batch_id=batch.batch_id, + requests=next_batch_requests, + input_ids=next_batch_input_ids, + position_ids=next_batch_position_ids, + cu_seqlens=next_batch_cu_seqlens, + max_seqlen=next_batch_max_seqlen, + past_key_values=next_batch_past_key_values, + input_lengths=next_batch_input_lengths, + all_input_ids=next_batch_all_input_ids, + next_token_choosers=next_batch_next_token_choosers, + stopping_criterias=next_batch_stopping_criterias, + ) + return generations, next_batch + + +class FlashNeoXSharded(FlashNeoX): + def __init__( + self, model_id: str, revision: Optional[str] = None, quantize: bool = False + ): + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + device = torch.device(f"cuda:{self.rank}") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + else: + raise NotImplementedError("FlashNeoX is only available on GPU") + + if quantize: + raise NotImplementedError("FlashNeoX does not support quantization") + + tokenizer = AutoTokenizer.from_pretrained( + model_id, revision=revision, padding_side="left" + ) + + config = AutoConfig.from_pretrained( + model_id, revision=revision, tp_parallel=True + ) + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + + with init_empty_weights(): + model = FlashGPTNeoXForCausalLM(config) + + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + quantize=quantize, + device=device, + rank=self.rank, + world_size=self.world_size, + ) + self.model = model.eval().to(dtype) + torch.distributed.barrier(group=self.process_group) + super(FlashNeoX, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @staticmethod + def load_weights( + model, + filenames: List[str], + quantize: bool, + device: torch.device, + rank: int, + world_size: int, + ): + parameters = dict(model.named_parameters()) + for file in filenames: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: + for name in f.keys(): + module_name, param_name = name.rsplit(".", 1) + module = model.get_submodule(module_name) + + current_parameter_tensor = parameters.get(name, None) + + slice_ = f.get_slice(name) + + if isinstance(module, TensorParallelColumnLinear): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif isinstance(module, TensorParallelRowLinear): + if param_name == "weight": + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + else: + tensor = slice_[:] + # XXX: Hack for Rowlinear to add the bias only once. + if rank != 0: + tensor = torch.zeros_like(tensor) + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif name == "embed_out.weight" and model.gpt_neox.tp_embeddings: + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + else: + try: + tensor = slice_[:] + except: + tensor = f.get_tensor(name) + + if ( + current_parameter_tensor is not None + and current_parameter_tensor.shape != tensor.shape + ): + raise ValueError( + f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" + ) + + tensor = tensor.contiguous() + + if current_parameter_tensor is not None: + module._parameters[param_name] = tensor + else: + module._buffers[param_name] = tensor + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlens: torch.Tensor, + max_s: int, + past_key_values: Optional = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + if self.model.gpt_neox.tp_embeddings: + logits, present = self.model.forward( + input_ids=input_ids, + position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_s=max_s, + past_key_values=past_key_values, + ) + + # Logits are sharded, so we need to gather them + world_logits = [torch.empty_like(logits) for _ in range(self.world_size)] + torch.distributed.all_gather(world_logits, logits, group=self.process_group) + world_logits = torch.cat(world_logits, dim=1) + + return world_logits, present + # While the model itself is sharded, the embeddings might not as they might not be dividable by num-shard + else: + return super(FlashNeoXSharded, self).forward( + input_ids, position_ids, cu_seqlens, max_s, past_key_values + ) diff --git a/server/text_generation_server/models/flash_neox_modeling.py b/server/text_generation_server/models/flash_neox_modeling.py new file mode 100644 index 00000000..d67ca3c0 --- /dev/null +++ b/server/text_generation_server/models/flash_neox_modeling.py @@ -0,0 +1,637 @@ +import torch +import torch.distributed + +import torch.nn.functional as F + +from torch import nn +from transformers.activations import ACT2FN +from transformers.modeling_utils import PreTrainedModel +from transformers.models.gpt_neox import GPTNeoXConfig + +# Flash attention imports +import rotary_emb +import flash_attn_cuda +import dropout_layer_norm + +from flash_attn.layers.rotary import RotaryEmbedding + + +class TensorParallelColumnLinear(nn.Linear): + def __init__( + self, + in_features, + out_features, + process_group: torch.distributed.ProcessGroup, + bias=True, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_world_size = process_group.size() + assert out_features % self.tp_world_size == 0 + out_features = out_features // self.tp_world_size + + super().__init__( + in_features=in_features, + out_features=out_features, + bias=bias, + device=device, + dtype=dtype, + ) + + @staticmethod + def linear(input, weight, bias): + return F.linear(input, weight, bias) + + def forward(self, input): + return self.linear(input, self.weight, self.bias) + + +class TensorParallelRowLinear(nn.Linear): + def __init__( + self, + in_features, + out_features, + process_group: torch.distributed.ProcessGroup, + bias=True, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_world_size = process_group.size() + assert in_features % self.tp_world_size == 0 + in_features = in_features // self.tp_world_size + + super().__init__( + in_features=in_features, + out_features=out_features, + bias=bias, + device=device, + dtype=dtype, + ) + + @staticmethod + def linear(input, weight, bias): + return F.linear(input, weight, bias) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + out = self.linear(input, self.weight, self.bias) + torch.distributed.all_reduce(out, group=self.process_group) + + return out + + +class TensorParallelEmbedding(nn.Embedding): + def __init__( + self, + num_embeddings, + embedding_dim, + process_group: torch.distributed.ProcessGroup, + padding_idx=None, + max_norm=None, + norm_type=2.0, + scale_grad_by_freq=False, + sparse=False, + _weight=None, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + + self.original_num_embeddings = num_embeddings + + assert num_embeddings % self.tp_world_size == 0 + block_size = num_embeddings // self.tp_world_size + # inputs in `[min_id, max_id[` are handled by `self` to get embeddings + self.min_id = self.tp_rank * block_size + self.max_id = (self.tp_rank + 1) * block_size + + super().__init__( + block_size, + embedding_dim, + padding_idx=padding_idx, + max_norm=max_norm, + norm_type=norm_type, + scale_grad_by_freq=scale_grad_by_freq, + sparse=sparse, + _weight=_weight, + device=device, + dtype=dtype, + ) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + # Sanity check + if torch.any( + torch.logical_or(0 > input, input >= self.original_num_embeddings) + ): + raise IndexError( + f"Input is required to be in [0, {self.original_num_embeddings}[, got min: {torch.min(input)} and max: {torch.max(input)}" + ) + + # `0` if input is in the correct interval, else `1` + input_mask = torch.logical_or(self.min_id > input, input >= self.max_id) + # translate for [0, self.max_id - self.min_id[ + input = input - self.min_id + # default all out of bounds values to `0` + input[input_mask] = 0 + out = super().forward(input) + out[input_mask] = 0.0 + torch.distributed.all_reduce(out, group=self.process_group) + return out + + +class PositionRotaryEmbedding(RotaryEmbedding): + def _update_cos_sin_cache(self, dtype, device, seqlen): + # Reset the tables if the sequence length has changed, + # or if we're on a new device (possibly due to tracing for instance) + if ( + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype + ): + self._seq_len_cached = seqlen + t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) + # Don't do einsum, it converts fp32 to fp16 + # freqs = torch.einsum("i,j->ij", t, self.inv_freq) + freqs = torch.outer(t, self.inv_freq.to(device=t.device)) + self._cos_cached = torch.cos(freqs).to(dtype) + self._sin_cached = torch.sin(freqs).to(dtype) + + def get_cos_sin(self, position_ids: torch.Tensor, max_s: int, dtype: torch.dtype): + """ + Return cos and sin for the asked position ids + """ + + self._update_cos_sin_cache(dtype, position_ids.device, max_s) + + cos = torch.index_select(self._cos_cached, 0, position_ids) + sin = torch.index_select(self._sin_cached, 0, position_ids) + return cos.unsqueeze(1), sin.unsqueeze(1) + + def forward(self, qkv: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): + rotary_dim = cos.shape[-1] + q1 = qkv[:, 0, :, :rotary_dim] + q2 = qkv[:, 0, :, rotary_dim : 2 * rotary_dim] + k1 = qkv[:, 1, :, :rotary_dim] + k2 = qkv[:, 1, :, rotary_dim : 2 * rotary_dim] + + rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False) + rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False) + return qkv + + +class FlashNeoxAttention(torch.nn.Module): + def __init__( + self, num_heads, hidden_size, rotary_pct, rotary_emb_base, process_group=None + ): + super().__init__() + self.num_heads = num_heads + self.hidden_size = hidden_size + self.head_size = hidden_size // num_heads + + rotary_ndims = int(self.head_size * rotary_pct) + self.rotary_emb = PositionRotaryEmbedding(rotary_ndims, base=rotary_emb_base) + self.softmax_scale = self.head_size ** (-0.5) + + if process_group is None: + self.query_key_value = nn.Linear(hidden_size, 3 * hidden_size) + self.dense = nn.Linear(hidden_size, hidden_size) + else: + self.num_heads = self.num_heads // process_group.size() + self.query_key_value = TensorParallelColumnLinear( + hidden_size, + 3 * hidden_size, + process_group=process_group, + ) + self.dense = TensorParallelRowLinear( + hidden_size, + hidden_size, + process_group=process_group, + ) + self.swap_dims = True + + # TODO: remove and swap dims when loading weights + def _swap_dims(self): + """Swap dims for the first inference to avoid an additional permute""" + self.query_key_value.weight = torch.nn.Parameter( + self.query_key_value.weight.view( + self.num_heads, 3, self.head_size, self.hidden_size + ) + .permute(1, 0, 2, 3) + .reshape(-1, self.hidden_size) + ) + self.query_key_value.bias = torch.nn.Parameter( + self.query_key_value.bias.view(self.num_heads, 3, self.head_size) + .permute(1, 0, 2) + .reshape(-1) + ) + self.swap_dims = False + + def forward( + self, + hidden_states, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + if self.swap_dims: + self._swap_dims() + + qkv = self.query_key_value(hidden_states) + qkv = qkv.view(-1, 3, self.num_heads, self.head_size) + qkv_rot = self.rotary_emb(qkv, cos, sin) + + # Prefill + if layer_past_present_indices is None: + # Copy to layer past + layer_past[...] = qkv_rot[:, 1:] + + # output + attn_output = torch.empty_like(qkv[:, 0]) + # flash attention + flash_attn_cuda.fwd( + qkv[:, 0], + qkv[:, 1], + qkv[:, 2], + attn_output, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + 0.0, + self.softmax_scale, + False, + True, + False, + 0, + None, + ) + # Decode + else: + query = qkv_rot[:, 0] + # Add present to the layer_past tensor at the correct indices + layer_past[layer_past_present_indices] = qkv_rot[:, 1:] + + # output + attn_output = torch.empty_like(query) + # flash attention + flash_attn_cuda.fwd( + query, + layer_past[:, 0], + layer_past[:, 1], + attn_output, + cu_seqlens_q, + cu_seqlens, + 1, + max_s, + 0.0, + self.softmax_scale, + False, + False, + False, + 0, + None, + ) + + return self.dense(attn_output.view(-1, self.num_heads * self.head_size)) + + +class FlashMLP(nn.Module): + def __init__(self, act, hidden_size, intermediate_size, process_group=None): + super().__init__() + self.act = ( + ACT2FN[act] + if "gelu" not in act + else lambda x: torch.nn.functional.gelu(x, approximate="tanh") + ) + + if process_group is None: + self.dense_h_to_4h = nn.Linear(hidden_size, intermediate_size) + self.dense_4h_to_h = nn.Linear(intermediate_size, hidden_size) + else: + self.dense_h_to_4h = TensorParallelColumnLinear( + hidden_size, + intermediate_size, + process_group=process_group, + ) + self.dense_4h_to_h = TensorParallelRowLinear( + intermediate_size, + hidden_size, + process_group=process_group, + ) + self.heuristic = "auto" + self.process_group = process_group + + def forward(self, hidden_states): + hidden_states = self.dense_h_to_4h(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.dense_4h_to_h(hidden_states) + return hidden_states + + +class FlashNeoXLayer(nn.Module): + def __init__( + self, + num_heads, + act, + hidden_size, + intermediate_size, + rotary_pct, + rotary_emb_base, + layer_norm_eps, + use_parallel_residual, + process_group=None, + ): + super().__init__() + self.use_parallel_residual = use_parallel_residual + self.input_layernorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps) + self.post_attention_layernorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps) + self.attention = FlashNeoxAttention( + num_heads, hidden_size, rotary_pct, rotary_emb_base, process_group + ) + self.mlp = FlashMLP(act, hidden_size, intermediate_size, process_group) + + def forward( + self, + hidden_states, + residual, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + if self.use_parallel_residual: + # faster input layer norm + ln1_hidden_states, *rest = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + None, + self.input_layernorm.weight, + self.input_layernorm.bias, + None, + None, + None, + None, + 0.0, + self.input_layernorm.eps, + 1.0, + 0, + None, + False, + False, + ) + + attn_output = self.attention( + ln1_hidden_states, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ) + + # faster post attention layer norm + ln2_hidden_states, *rest = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + None, + self.post_attention_layernorm.weight, + self.post_attention_layernorm.bias, + None, + None, + None, + None, + 0.0, + self.post_attention_layernorm.eps, + 1.0, + 0, + None, + False, + False, + ) + + mlp_output = self.mlp(ln2_hidden_states) + return mlp_output + attn_output + hidden_states, None + else: + # faster input layer norm + hidden_states, residual, *rest = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.input_layernorm.weight, + self.input_layernorm.bias, + None, + None, + None, + None, + 0.0, + self.input_layernorm.eps, + 1.0, + 0, + None, + False, + False, + ) + + hidden_states = self.attention( + hidden_states, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ) + + # faster post attention layer norm + hidden_states, residual, *rest = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.post_attention_layernorm.weight, + self.post_attention_layernorm.bias, + None, + None, + None, + None, + 0.0, + self.post_attention_layernorm.eps, + 1.0, + 0, + None, + False, + False, + ) + + mlp_output = self.mlp(hidden_states) + + return mlp_output, residual + + +class FlashGPTNeoXPreTrainedModel(PreTrainedModel): + config_class = GPTNeoXConfig + base_model_prefix = "gpt_neox" + supports_gradient_checkpointing = False + _no_split_modules = None + + +class FlashGPTNeoXModel(FlashGPTNeoXPreTrainedModel): + def __init__(self, config, process_group=None): + super().__init__(config) + self.config = config + + self.tp_embeddings = False + if process_group is not None: + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + if config.vocab_size % self.tp_world_size == 0: + self.tp_embeddings = True + + if self.tp_embeddings: + self.embed_in = TensorParallelEmbedding( + config.vocab_size, config.hidden_size, process_group=process_group + ) + else: + self.embed_in = nn.Embedding(config.vocab_size, config.hidden_size) + + self.layers = nn.ModuleList( + [ + FlashNeoXLayer( + config.num_attention_heads, + config.hidden_act, + config.hidden_size, + config.intermediate_size, + config.rotary_pct, + config.rotary_emb_base, + config.layer_norm_eps, + config.use_parallel_residual, + process_group, + ) + for _ in range(config.num_hidden_layers) + ] + ) + self.final_layer_norm = nn.LayerNorm( + config.hidden_size, eps=config.layer_norm_eps + ) + + self.gradient_checkpointing = False + + self.head_size = self.layers[0].attention.head_size + self.num_heads = self.layers[0].attention.num_heads + + def forward( + self, + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values=None, + ): + hidden_states = self.embed_in(input_ids) + + # Prefill + if past_key_values is None: + # Create past tensor + past_key_values = hidden_states.new_empty( + ( + len(self.layers), + len(hidden_states), + 2, + self.num_heads, + self.head_size, + ) + ) + layer_past_present_indices = None + cu_seqlens_q = None + # Decode + else: + # Create indices from cumulative sequence lengths + layer_past_present_indices = cu_seqlens[1:] - 1 + cu_seqlens_q = torch.arange( + len(cu_seqlens), dtype=torch.int32, device=hidden_states.device + ) + + # Get rotary cos and sin for this forward + # Avoid to index in each layer + cos, sin = self.layers[0].attention.rotary_emb.get_cos_sin( + position_ids, max_s, hidden_states.dtype + ) + + residual = None + for i, layer in enumerate(self.layers): + hidden_states, residual = layer( + hidden_states, + residual, + cos, + sin, + cu_seqlens, + max_s, + past_key_values[i], + layer_past_present_indices, + cu_seqlens_q, + ) + + # Faster final layer norm + hidden_states, *rest = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.final_layer_norm.weight, + self.final_layer_norm.bias, + None, + None, + None, + None, + 0.0, + self.final_layer_norm.eps, + 1.0, + 0, + None, + False, + False, + ) + + return hidden_states, past_key_values + + +class FlashGPTNeoXForCausalLM(FlashGPTNeoXPreTrainedModel): + def __init__(self, config): + super().__init__(config) + + if config.tp_parallel: + process_group = torch.distributed.distributed_c10d._get_default_group() + else: + process_group = None + + self.gpt_neox = FlashGPTNeoXModel(config, process_group) + + if self.gpt_neox.tp_embeddings: + self.embed_out = nn.Linear( + config.hidden_size, + config.vocab_size // process_group.size(), + bias=False, + ) + else: + self.embed_out = nn.Linear( + config.hidden_size, config.vocab_size, bias=False + ) + + def forward( + self, + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values=None, + ): + hidden_states, present = self.gpt_neox( + input_ids, position_ids, cu_seqlens, max_s, past_key_values + ) + return self.embed_out(hidden_states), present diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index c7594644..597fbe7c 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -24,7 +24,7 @@ def __init__(self, seed: int, device: str = "cpu"): self.seed = seed def __call__(self, logits): - probs = torch.nn.functional.softmax(logits) + probs = torch.nn.functional.softmax(logits, -1) next_tokens = torch.multinomial(probs, num_samples=1, generator=self.generator) return next_tokens diff --git a/server/text_generation_server/utils/watermark.py b/server/text_generation_server/utils/watermark.py index 8e90a59c..1850561d 100644 --- a/server/text_generation_server/utils/watermark.py +++ b/server/text_generation_server/utils/watermark.py @@ -17,6 +17,7 @@ import torch from transformers import LogitsProcessor +from typing import List, Union GAMMA = os.getenv("WATERMARK_GAMMA", 0.5) DELTA = os.getenv("WATERMARK_DELTA", 2.0) @@ -36,23 +37,32 @@ def __init__( self.rng = torch.Generator(device=device) self.hash_key = hash_key - def _seed_rng(self, input_ids: torch.LongTensor) -> None: - assert ( - input_ids.shape[-1] >= 1 - ), "requires at least a 1 token prefix sequence to seed rng" - prev_token = input_ids[-1].item() + def _seed_rng(self, input_ids: Union[List[int], torch.LongTensor]): + if isinstance(input_ids, list): + assert ( + len(input_ids) >= 1 + ), "requires at least a 1 token prefix sequence to seed rng" + prev_token = input_ids[-1] + else: + input_ids = input_ids[0] + assert len(input_ids) == 1 + assert ( + input_ids.shape[-1] >= 1 + ), "requires at least a 1 token prefix sequence to seed rng" + prev_token = input_ids[-1].item() self.rng.manual_seed(self.hash_key * prev_token) def _get_greenlist_ids( - self, input_ids: torch.LongTensor, max_value: int - ) -> list[int]: + self, + input_ids: Union[List[int], torch.LongTensor], + max_value: int, + device: torch.device, + ) -> List[int]: # seed the rng using the previous tokens/prefix self._seed_rng(input_ids) greenlist_size = int(max_value * self.gamma) - vocab_permutation = torch.randperm( - max_value, device=input_ids.device, generator=self.rng - ) + vocab_permutation = torch.randperm(max_value, device=device, generator=self.rng) greenlist_ids = vocab_permutation[:greenlist_size] return greenlist_ids @@ -73,10 +83,11 @@ def _bias_greenlist_logits( return scores def __call__( - self, input_ids: torch.LongTensor, scores: torch.FloatTensor + self, input_ids: Union[List[int], torch.LongTensor], scores: torch.FloatTensor ) -> torch.FloatTensor: - assert len(input_ids) == 1 - greenlist_ids = self._get_greenlist_ids(input_ids[0], scores.shape[-1]) + greenlist_ids = self._get_greenlist_ids( + input_ids, scores.shape[-1], scores.device + ) green_tokens_mask = self._calc_greenlist_mask( scores=scores, greenlist_token_ids=greenlist_ids ) From d6a93fe992bc932027df6f4a8f2b87c68d233f55 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 24 Mar 2023 18:21:41 +0100 Subject: [PATCH 129/793] fix(server): fix flash-neox scores warping (#137) --- clients/python/pyproject.toml | 2 +- clients/python/tests/test_types.py | 2 + clients/python/text_generation/client.py | 10 +-- clients/python/text_generation/types.py | 2 + .../text_generation_server/models/__init__.py | 1 + .../models/flash_neox.py | 58 ++++++++++------- .../models/flash_neox_modeling.py | 62 ++++++++++--------- .../text_generation_server/utils/watermark.py | 2 +- 8 files changed, 79 insertions(+), 60 deletions(-) diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 505717de..d07f1dbc 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.4.0" +version = "0.4.1" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] diff --git a/clients/python/tests/test_types.py b/clients/python/tests/test_types.py index 4c9d4c89..77689ade 100644 --- a/clients/python/tests/test_types.py +++ b/clients/python/tests/test_types.py @@ -14,6 +14,8 @@ def test_parameters_validation(): Parameters(best_of=2, do_sample=True) with pytest.raises(ValidationError): Parameters(best_of=2) + with pytest.raises(ValidationError): + Parameters(best_of=2, seed=1) # Test repetition_penalty Parameters(repetition_penalty=1) diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index 03bc3888..8b8742fc 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -150,7 +150,6 @@ def generate_stream( prompt: str, do_sample: bool = False, max_new_tokens: int = 20, - best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, @@ -172,8 +171,6 @@ def generate_stream( Activate logits sampling max_new_tokens (`int`): Maximum number of generated tokens - best_of (`int`): - Generate best_of sequences and return the one if the highest token logprobs repetition_penalty (`float`): The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. @@ -203,7 +200,7 @@ def generate_stream( """ # Validate parameters parameters = Parameters( - best_of=best_of, + best_of=None, details=True, do_sample=do_sample, max_new_tokens=max_new_tokens, @@ -388,7 +385,6 @@ async def generate_stream( prompt: str, do_sample: bool = False, max_new_tokens: int = 20, - best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, @@ -410,8 +406,6 @@ async def generate_stream( Activate logits sampling max_new_tokens (`int`): Maximum number of generated tokens - best_of (`int`): - Generate best_of sequences and return the one if the highest token logprobs repetition_penalty (`float`): The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. @@ -441,7 +435,7 @@ async def generate_stream( """ # Validate parameters parameters = Parameters( - best_of=best_of, + best_of=None, details=True, do_sample=do_sample, max_new_tokens=max_new_tokens, diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index ea2070b8..21a9849b 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -43,6 +43,8 @@ def valid_best_of(cls, field_value, values): if field_value is not None: if field_value <= 0: raise ValidationError("`best_of` must be strictly positive") + if field_value > 1 and values["seed"] is not None: + raise ValidationError("`seed` must not be set when `best_of` is > 1") sampling = ( values["do_sample"] | (values["temperature"] is not None) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 2f637ae1..f3a92ad2 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -16,6 +16,7 @@ try: from text_generation_server.models.flash_neox import FlashNeoX, FlashNeoXSharded + FLASH_NEOX = torch.cuda.is_available() and int(os.environ.get("FLASH_NEOX", 0)) == 1 except ImportError: if int(os.environ.get("FLASH_NEOX", 0)) == 1: diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 7be4708b..cbaa78ca 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -1,6 +1,8 @@ import torch import torch.distributed +from torch.nn import functional as F + from accelerate import init_empty_weights from dataclasses import dataclass from opentelemetry import trace @@ -48,6 +50,7 @@ class FlashNeoXBatch(Batch): # All tokens all_input_ids: List[List[int]] + all_input_ids_tensor: List[torch.Tensor] # Lengths of all generations present in the batch input_lengths: List[int] @@ -75,6 +78,7 @@ def from_pb( input_lengths = [] all_input_ids = [] + all_input_ids_tensor = [] next_token_choosers = [] stopping_criterias = [] @@ -84,15 +88,14 @@ def from_pb( # Parse batch for r in pb.requests: - tokenized_input = tokenizer(r.inputs, return_tensors="pt")[ - "input_ids" - ].squeeze(0) - input_ids.append(tokenized_input) - all_input_ids.append(tokenized_input.tolist()) - + tokenized_input = tokenizer(r.inputs)["input_ids"] input_length = len(tokenized_input) max_seqlen = max(max_seqlen, input_length) input_lengths.append(input_length) + all_input_ids.append(tokenized_input) + + tokenized_input = torch.tensor(tokenized_input, device=device) + input_ids.append(tokenized_input) # Position ids position_ids.append(torch.arange(0, input_length, dtype=torch.int32)) @@ -101,14 +104,18 @@ def from_pb( cu_seqlens.append(cumulative_length + input_length) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) - stopping_criterias.append( - StoppingCriteria.from_pb(r.stopping_parameters, tokenizer) + stopping_criteria = StoppingCriteria.from_pb( + r.stopping_parameters, tokenizer + ) + stopping_criterias.append(stopping_criteria) + all_input_ids_tensor.append( + F.pad(tokenized_input, (0, stopping_criteria.max_new_tokens)) ) # Update cumulative_length += input_length - input_ids = torch.concat(input_ids).unsqueeze(1) + input_ids = torch.concat(input_ids) position_ids = torch.concat(position_ids) cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32) @@ -122,6 +129,7 @@ def from_pb( past_key_values=None, input_lengths=input_lengths, all_input_ids=all_input_ids, + all_input_ids_tensor=all_input_ids_tensor, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, ) @@ -133,6 +141,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests = [] input_lengths = [] all_input_ids = [] + all_input_ids_tensor = [] next_token_choosers = [] stopping_criterias = [] @@ -150,6 +159,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) all_input_ids.extend(batch.all_input_ids) + all_input_ids_tensor.extend(batch.all_input_ids_tensor) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -181,6 +191,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": past_key_values=past_key_values, input_lengths=input_lengths, all_input_ids=all_input_ids, + all_input_ids_tensor=all_input_ids_tensor, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, ) @@ -255,11 +266,10 @@ def generate_token( ) -> Tuple[List[Generation], Optional[FlashNeoXBatch]]: # Better to send to device here to avoid device issues in concatenate position_ids = batch.position_ids.to(self.device, non_blocking=True) - cu_seqlens = batch.cu_seqlens.to(self.device, non_blocking=True) - input_ids = batch.input_ids.squeeze(1).to(self.device) + cu_seqlens = batch.cu_seqlens.to(self.device) out, present = self.forward( - input_ids, + batch.input_ids, position_ids, cu_seqlens, batch.max_seqlen, @@ -277,6 +287,7 @@ def generate_token( next_batch_past_key_values = [] next_batch_input_lengths = [] next_batch_all_input_ids = [] + next_batch_all_input_ids_tensor = [] # Cumulative length cumulative_length = 0 @@ -291,6 +302,7 @@ def generate_token( batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, + batch.all_input_ids_tensor, ) # For each member of the batch @@ -300,6 +312,7 @@ def generate_token( next_token_chooser, stopping_criteria, all_input_ids, + all_input_ids_tensor, ) in enumerate(iterator): # Indexing metadata start_index = cumulative_length @@ -315,20 +328,19 @@ def generate_token( logits = out[i].unsqueeze(0) # Select next token - next_token_id, logprobs = next_token_chooser(all_input_ids, logits) - # Copy to cpu to avoid other copies when indexing and calling .item() - next_token_id = next_token_id.to("cpu", non_blocking=True) - logprobs = logprobs.to("cpu") - + next_token_id, logprobs = next_token_chooser( + all_input_ids_tensor[None, :input_length], logits + ) next_token_id_squeezed = next_token_id.squeeze() next_token_id_item = next_token_id_squeezed.item() # Append next token to all tokens all_input_ids.append(next_token_id_item) + all_input_ids_tensor[input_length] = next_token_id_item new_input_length = input_length + 1 # Generated token - next_token_logprob = logprobs[-1, next_token_id] + next_token_logprob = logprobs[-1, next_token_id_item] next_token_text = self.decode_token( next_token_id_item, ) @@ -372,13 +384,14 @@ def generate_token( ) next_batch_input_lengths.append(new_input_length) next_batch_all_input_ids.append(all_input_ids) + next_batch_all_input_ids_tensor.append(all_input_ids_tensor) next_batch_max_seqlen = max(next_batch_max_seqlen, new_input_length) # Prefill if stopping_criteria.current_tokens == 1: # Remove generated token to only have prefill and add nan for first prompt token prefill_logprobs = [float("nan")] + logprobs.gather( - 1, torch.tensor(all_input_ids[1:]).unsqueeze(1) + 1, all_input_ids_tensor[1:input_length].unsqueeze(1) ).squeeze(1)[:-1].tolist() prefill_token_ids = all_input_ids[:-1] prefill_texts = self.tokenizer.batch_decode( @@ -431,12 +444,14 @@ def generate_token( ) next_batch_cu_seqlens = torch.tensor(next_batch_cu_seqlens, dtype=torch.int32) if len(next_batch_keep_indices) > 1: - next_batch_input_ids = torch.concat(next_batch_input_ids) + next_batch_input_ids = torch.concat(next_batch_input_ids).squeeze(1) next_batch_past_key_values = torch.concat(next_batch_past_key_values, dim=1) else: - next_batch_input_ids = next_batch_input_ids[0] + next_batch_input_ids = next_batch_input_ids[0].view(1) next_batch_past_key_values = next_batch_past_key_values[0] + print(next_batch_input_ids.shape) + next_batch = FlashNeoXBatch( batch_id=batch.batch_id, requests=next_batch_requests, @@ -447,6 +462,7 @@ def generate_token( past_key_values=next_batch_past_key_values, input_lengths=next_batch_input_lengths, all_input_ids=next_batch_all_input_ids, + all_input_ids_tensor=next_batch_all_input_ids_tensor, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, ) diff --git a/server/text_generation_server/models/flash_neox_modeling.py b/server/text_generation_server/models/flash_neox_modeling.py index d67ca3c0..dcfb613d 100644 --- a/server/text_generation_server/models/flash_neox_modeling.py +++ b/server/text_generation_server/models/flash_neox_modeling.py @@ -1,8 +1,6 @@ import torch import torch.distributed -import torch.nn.functional as F - from torch import nn from transformers.activations import ACT2FN from transformers.modeling_utils import PreTrainedModel @@ -16,7 +14,29 @@ from flash_attn.layers.rotary import RotaryEmbedding -class TensorParallelColumnLinear(nn.Linear): +class FastLinear(nn.Linear): + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + device=None, + dtype=None, + ) -> None: + super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) + self.swap_dims = True + + def forward(self, input: torch.Tensor) -> torch.Tensor: + if self.swap_dims: + self.weight = nn.Parameter(self.weight.T) + self.swap_dims = False + + if self.bias is not None: + return torch.addmm(self.bias, input, self.weight) + return torch.matmul(input, self.weight) + + +class TensorParallelColumnLinear(FastLinear): def __init__( self, in_features, @@ -39,15 +59,11 @@ def __init__( dtype=dtype, ) - @staticmethod - def linear(input, weight, bias): - return F.linear(input, weight, bias) - def forward(self, input): - return self.linear(input, self.weight, self.bias) + return super(TensorParallelColumnLinear, self).forward(input) -class TensorParallelRowLinear(nn.Linear): +class TensorParallelRowLinear(FastLinear): def __init__( self, in_features, @@ -70,12 +86,8 @@ def __init__( dtype=dtype, ) - @staticmethod - def linear(input, weight, bias): - return F.linear(input, weight, bias) - def forward(self, input: torch.Tensor) -> torch.Tensor: - out = self.linear(input, self.weight, self.bias) + out = super(TensorParallelRowLinear, self).forward(input) torch.distributed.all_reduce(out, group=self.process_group) return out @@ -122,14 +134,6 @@ def __init__( ) def forward(self, input: torch.Tensor) -> torch.Tensor: - # Sanity check - if torch.any( - torch.logical_or(0 > input, input >= self.original_num_embeddings) - ): - raise IndexError( - f"Input is required to be in [0, {self.original_num_embeddings}[, got min: {torch.min(input)} and max: {torch.max(input)}" - ) - # `0` if input is in the correct interval, else `1` input_mask = torch.logical_or(self.min_id > input, input >= self.max_id) # translate for [0, self.max_id - self.min_id[ @@ -196,8 +200,8 @@ def __init__( self.softmax_scale = self.head_size ** (-0.5) if process_group is None: - self.query_key_value = nn.Linear(hidden_size, 3 * hidden_size) - self.dense = nn.Linear(hidden_size, hidden_size) + self.query_key_value = FastLinear(hidden_size, 3 * hidden_size) + self.dense = FastLinear(hidden_size, hidden_size) else: self.num_heads = self.num_heads // process_group.size() self.query_key_value = TensorParallelColumnLinear( @@ -312,8 +316,8 @@ def __init__(self, act, hidden_size, intermediate_size, process_group=None): ) if process_group is None: - self.dense_h_to_4h = nn.Linear(hidden_size, intermediate_size) - self.dense_4h_to_h = nn.Linear(intermediate_size, hidden_size) + self.dense_h_to_4h = FastLinear(hidden_size, intermediate_size) + self.dense_4h_to_h = FastLinear(intermediate_size, hidden_size) else: self.dense_h_to_4h = TensorParallelColumnLinear( hidden_size, @@ -556,7 +560,7 @@ def forward( # Create indices from cumulative sequence lengths layer_past_present_indices = cu_seqlens[1:] - 1 cu_seqlens_q = torch.arange( - len(cu_seqlens), dtype=torch.int32, device=hidden_states.device + cu_seqlens.shape[0], dtype=torch.int32, device=hidden_states.device ) # Get rotary cos and sin for this forward @@ -613,13 +617,13 @@ def __init__(self, config): self.gpt_neox = FlashGPTNeoXModel(config, process_group) if self.gpt_neox.tp_embeddings: - self.embed_out = nn.Linear( + self.embed_out = FastLinear( config.hidden_size, config.vocab_size // process_group.size(), bias=False, ) else: - self.embed_out = nn.Linear( + self.embed_out = FastLinear( config.hidden_size, config.vocab_size, bias=False ) diff --git a/server/text_generation_server/utils/watermark.py b/server/text_generation_server/utils/watermark.py index 1850561d..df7b90e3 100644 --- a/server/text_generation_server/utils/watermark.py +++ b/server/text_generation_server/utils/watermark.py @@ -44,8 +44,8 @@ def _seed_rng(self, input_ids: Union[List[int], torch.LongTensor]): ), "requires at least a 1 token prefix sequence to seed rng" prev_token = input_ids[-1] else: - input_ids = input_ids[0] assert len(input_ids) == 1 + input_ids = input_ids[0] assert ( input_ids.shape[-1] >= 1 ), "requires at least a 1 token prefix sequence to seed rng" From 678b2f39000f638e0099af0d84a98d409feca428 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sun, 26 Mar 2023 16:37:21 +0200 Subject: [PATCH 130/793] feat(server): cleanup flash neox loading (#139) --- .../models/flash_neox.py | 3 +- .../models/flash_neox_modeling.py | 72 +++++++++++++------ 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index cbaa78ca..b97f342a 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -450,8 +450,6 @@ def generate_token( next_batch_input_ids = next_batch_input_ids[0].view(1) next_batch_past_key_values = next_batch_past_key_values[0] - print(next_batch_input_ids.shape) - next_batch = FlashNeoXBatch( batch_id=batch.batch_id, requests=next_batch_requests, @@ -507,6 +505,7 @@ def __init__( rank=self.rank, world_size=self.world_size, ) + model.post_load_weights() self.model = model.eval().to(dtype) torch.distributed.barrier(group=self.process_group) super(FlashNeoX, self).__init__( diff --git a/server/text_generation_server/models/flash_neox_modeling.py b/server/text_generation_server/models/flash_neox_modeling.py index dcfb613d..2e638d77 100644 --- a/server/text_generation_server/models/flash_neox_modeling.py +++ b/server/text_generation_server/models/flash_neox_modeling.py @@ -1,6 +1,8 @@ import torch import torch.distributed +from torch.nn import functional as F + from torch import nn from transformers.activations import ACT2FN from transformers.modeling_utils import PreTrainedModel @@ -24,13 +26,11 @@ def __init__( dtype=None, ) -> None: super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) - self.swap_dims = True - def forward(self, input: torch.Tensor) -> torch.Tensor: - if self.swap_dims: - self.weight = nn.Parameter(self.weight.T) - self.swap_dims = False + def transpose_weight(self): + self.weight = nn.Parameter(self.weight.T) + def forward(self, input: torch.Tensor) -> torch.Tensor: if self.bias is not None: return torch.addmm(self.bias, input, self.weight) return torch.matmul(input, self.weight) @@ -120,6 +120,10 @@ def __init__( self.min_id = self.tp_rank * block_size self.max_id = (self.tp_rank + 1) * block_size + # Additional entry that will map to zero + # Used for masking + self.null_idx = block_size + super().__init__( block_size, embedding_dim, @@ -133,15 +137,19 @@ def __init__( dtype=dtype, ) + def add_null_idx(self): + """Additional 0 entry used for masking""" + self.weight = nn.Parameter(F.pad(self.weight, (0, 0, 0, 1))) + def forward(self, input: torch.Tensor) -> torch.Tensor: - # `0` if input is in the correct interval, else `1` - input_mask = torch.logical_or(self.min_id > input, input >= self.max_id) + # default all out of bounds values to `self.null_idx` that will then be mapped to 0 # translate for [0, self.max_id - self.min_id[ - input = input - self.min_id - # default all out of bounds values to `0` - input[input_mask] = 0 + input = torch.where( + (self.min_id > input) | (input >= self.max_id), + self.null_idx, + input - self.min_id, + ) out = super().forward(input) - out[input_mask] = 0.0 torch.distributed.all_reduce(out, group=self.process_group) return out @@ -214,11 +222,9 @@ def __init__( hidden_size, process_group=process_group, ) - self.swap_dims = True - # TODO: remove and swap dims when loading weights - def _swap_dims(self): - """Swap dims for the first inference to avoid an additional permute""" + def shuffle_qkv_dims(self): + """Swap dims to avoid an additional permute""" self.query_key_value.weight = torch.nn.Parameter( self.query_key_value.weight.view( self.num_heads, 3, self.head_size, self.hidden_size @@ -231,7 +237,6 @@ def _swap_dims(self): .permute(1, 0, 2) .reshape(-1) ) - self.swap_dims = False def forward( self, @@ -244,9 +249,6 @@ def forward( layer_past_present_indices, cu_seqlens_q, ): - if self.swap_dims: - self._swap_dims() - qkv = self.query_key_value(hidden_states) qkv = qkv.view(-1, 3, self.num_heads, self.head_size) qkv_rot = self.rotary_emb(qkv, cos, sin) @@ -329,7 +331,6 @@ def __init__(self, act, hidden_size, intermediate_size, process_group=None): hidden_size, process_group=process_group, ) - self.heuristic = "auto" self.process_group = process_group def forward(self, hidden_states): @@ -531,6 +532,25 @@ def __init__(self, config, process_group=None): self.head_size = self.layers[0].attention.head_size self.num_heads = self.layers[0].attention.num_heads + def post_load_weights(self): + if isinstance(self.embed_in, TensorParallelEmbedding): + self.embed_in.add_null_idx() + for layer in self.layers: + layer: FlashNeoXLayer + layer.attention.shuffle_qkv_dims() + layer.attention.query_key_value.transpose_weight() + layer.attention.dense.transpose_weight() + layer.mlp.dense_h_to_4h.transpose_weight() + layer.mlp.dense_4h_to_h.transpose_weight() + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + model = super(FlashGPTNeoXModel, cls).from_pretrained( + pretrained_model_name_or_path, *model_args, **kwargs + ) + model.post_load_weights() + return model + def forward( self, input_ids, @@ -627,6 +647,18 @@ def __init__(self, config): config.hidden_size, config.vocab_size, bias=False ) + def post_load_weights(self): + self.gpt_neox.post_load_weights() + self.embed_out.transpose_weight() + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + model = super(FlashGPTNeoXForCausalLM, cls).from_pretrained( + pretrained_model_name_or_path, *model_args, **kwargs + ) + model.post_load_weights() + return model + def forward( self, input_ids, From ab5fd8cf93393db8a7fbbff9f8c23afeff18f3fc Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sun, 26 Mar 2023 16:37:51 +0200 Subject: [PATCH 131/793] v0.4.1 (#140) --- Cargo.lock | 8 ++++---- docs/openapi.json | 2 +- launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- router/grpc-metadata/Cargo.toml | 2 +- server/pyproject.toml | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9db5f213..37fb575c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -786,7 +786,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "grpc-metadata" -version = "0.4.0" +version = "0.4.1" dependencies = [ "opentelemetry", "tonic", @@ -2212,7 +2212,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.4.0" +version = "0.4.1" dependencies = [ "futures", "grpc-metadata", @@ -2229,7 +2229,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.4.0" +version = "0.4.1" dependencies = [ "clap 4.1.8", "ctrlc", @@ -2244,7 +2244,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.4.0" +version = "0.4.1" dependencies = [ "async-stream", "axum", diff --git a/docs/openapi.json b/docs/openapi.json index a2230a07..7a8cc58f 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -11,7 +11,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.4.0" + "version": "0.4.1" }, "paths": { "/generate": { diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 49948b05..e44a8ddb 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.4.0" +version = "0.4.1" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index 292e6219..4c898f23 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.4.0" +version = "0.4.1" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index fd43b63b..487d7e59 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.4.0" +version = "0.4.1" edition = "2021" [dependencies] diff --git a/router/grpc-metadata/Cargo.toml b/router/grpc-metadata/Cargo.toml index f8676d67..a6901f77 100644 --- a/router/grpc-metadata/Cargo.toml +++ b/router/grpc-metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grpc-metadata" -version = "0.4.0" +version = "0.4.1" edition = "2021" [dependencies] diff --git a/server/pyproject.toml b/server/pyproject.toml index 63a3df6c..7c032525 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.4.0" +version = "0.4.1" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 462530c2b05013e0822fd3d9e38fb08adc0a84b6 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Mon, 27 Mar 2023 00:23:22 -0700 Subject: [PATCH 132/793] fix(server): Avoid using try/except to determine kind of AutoModel (#142) --- server/text_generation_server/models/__init__.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index f3a92ad2..90c70cb5 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -3,6 +3,7 @@ from loguru import logger from transformers import AutoConfig +from transformers.models.auto import modeling_auto from typing import Optional from text_generation_server.models.model import Model @@ -65,14 +66,15 @@ def get_model( return SantaCoder(model_id, revision, quantize) config = AutoConfig.from_pretrained(model_id, revision=revision) + model_type = config.model_type - if config.model_type == "bloom": + if model_type == "bloom": if sharded: return BLOOMSharded(model_id, revision, quantize=quantize) else: return BLOOM(model_id, revision, quantize=quantize) - if config.model_type == "gpt_neox": + if model_type == "gpt_neox": if sharded: neox_cls = FlashNeoXSharded if FLASH_NEOX else GPTNeoxSharded return neox_cls(model_id, revision, quantize=quantize) @@ -80,7 +82,7 @@ def get_model( neox_cls = FlashNeoX if FLASH_NEOX else CausalLM return neox_cls(model_id, revision, quantize=quantize) - if config.model_type == "t5": + if model_type == "t5": if sharded: return T5Sharded(model_id, revision, quantize=quantize) else: @@ -88,7 +90,10 @@ def get_model( if sharded: raise ValueError("sharded is not supported for AutoModel") - try: + + if model_type in modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: return CausalLM(model_id, revision, quantize=quantize) - except Exception: + if model_type in modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES: return Seq2SeqLM(model_id, revision, quantize=quantize) + + raise ValueError(f"Unsupported model type {model_type}") From 8e8dd984d807dac30a752ae3025925883b11ebc8 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Mon, 27 Mar 2023 00:25:15 -0700 Subject: [PATCH 133/793] feat(server): Add mypy-protobuf (#141) Generates .pyi files for protobuf stubs which provide strong typing information. Very helpful for IDE auto-completion, etc. --- server/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/Makefile b/server/Makefile index 69ef9bc5..d2a8cf7a 100644 --- a/server/Makefile +++ b/server/Makefile @@ -3,9 +3,10 @@ flash_att_commit := 4d87e4d875077ad9efd25030efa4ab0ba92c19e1 gen-server: # Compile protos - pip install grpcio-tools==1.51.1 --no-cache-dir + pip install grpcio-tools==1.51.1 mypy-protobuf==3.4.0 'types-protobuf>=3.20.4' --no-cache-dir mkdir text_generation_server/pb || true - python -m grpc_tools.protoc -I../proto --python_out=text_generation_server/pb --grpc_python_out=text_generation_server/pb ../proto/generate.proto + python -m grpc_tools.protoc -I../proto --python_out=text_generation_server/pb \ + --grpc_python_out=text_generation_server/pb --mypy_out=text_generation_server/pb ../proto/generate.proto find text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; touch text_generation_server/pb/__init__.py From f000068944d8f3475424117eda7ee69126d4a719 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 28 Mar 2023 11:29:35 +0200 Subject: [PATCH 134/793] feat(server): clear cache on error (#143) --- proto/generate.proto | 6 ++++-- router/client/src/client.rs | 4 ++-- router/client/src/sharded_client.rs | 4 ++-- router/src/infer.rs | 2 ++ router/src/main.rs | 2 +- server/text_generation_server/cache.py | 4 +++- server/text_generation_server/pb/.gitignore | 1 + server/text_generation_server/server.py | 7 ++++++- 8 files changed, 21 insertions(+), 9 deletions(-) diff --git a/proto/generate.proto b/proto/generate.proto index 5081ce1c..0bac4352 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -21,8 +21,10 @@ message ServiceDiscoveryResponse { repeated string urls = 1; } -/// Empty request -message ClearCacheRequest {} +message ClearCacheRequest { + /// Optional batch id + optional uint64 id = 1; +} /// Empty response message ClearCacheResponse {} diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 1f0d23f2..1b2086a0 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -56,8 +56,8 @@ impl Client { /// Clear the past generations cache #[instrument(skip(self))] - pub async fn clear_cache(&mut self) -> Result<()> { - let request = tonic::Request::new(ClearCacheRequest {}).inject_context(); + pub async fn clear_cache(&mut self, batch_id: Option) -> Result<()> { + let request = tonic::Request::new(ClearCacheRequest { id: batch_id }).inject_context(); self.stub.clear_cache(request).await?; Ok(()) } diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 2e662ca3..7f0ec6f2 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -40,11 +40,11 @@ impl ShardedClient { /// Clear the past generations cache #[instrument(skip(self))] - pub async fn clear_cache(&mut self) -> Result<()> { + pub async fn clear_cache(&mut self, batch_id: Option) -> Result<()> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| client.clear_cache()) + .map(|client| client.clear_cache(batch_id)) .collect(); join_all(futures).await.into_iter().collect() } diff --git a/router/src/infer.rs b/router/src/infer.rs index ae151d8a..5eafc3e9 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -330,6 +330,7 @@ async fn prefill( entries: &mut IntMap, ) -> Option { let start_time = Instant::now(); + let batch_id = batch.id; match client.prefill(batch).await { Ok((generations, next_batch)) => { @@ -340,6 +341,7 @@ async fn prefill( } // If we have an error, we discard the whole batch Err(err) => { + let _ = client.clear_cache(Some(batch_id)).await; send_errors(err, entries); metrics::increment_counter!("tgi_batch_inference_failure", "method" => "prefill"); None diff --git a/router/src/main.rs b/router/src/main.rs index 2ccf66b3..81c6aeef 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -136,7 +136,7 @@ fn main() -> Result<(), std::io::Error> { .expect("Could not connect to server"); // Clear the cache; useful if the webserver rebooted sharded_client - .clear_cache() + .clear_cache(None) .await .expect("Unable to clear cache"); tracing::info!("Connected"); diff --git a/server/text_generation_server/cache.py b/server/text_generation_server/cache.py index 72dc4857..5556529c 100644 --- a/server/text_generation_server/cache.py +++ b/server/text_generation_server/cache.py @@ -17,7 +17,9 @@ def set(self, entry: B): self.cache[entry.batch_id] = entry def delete(self, batch_id: int): - del self.cache[batch_id] + batch = self.pop(batch_id) + if batch is not None: + del batch def clear(self): self.cache.clear() diff --git a/server/text_generation_server/pb/.gitignore b/server/text_generation_server/pb/.gitignore index 8527ad13..2621a190 100644 --- a/server/text_generation_server/pb/.gitignore +++ b/server/text_generation_server/pb/.gitignore @@ -1,2 +1,3 @@ *.py +*.pyi *.py-e \ No newline at end of file diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 0b75c3c7..3e3789bf 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -30,7 +30,12 @@ async def ServiceDiscovery(self, request, context): return generate_pb2.ServiceDiscoveryResponse(urls=self.server_urls) async def ClearCache(self, request, context): - self.cache.clear() + if request.HasField("id"): + self.cache.delete(request.id) + else: + self.cache.clear() + if torch.cuda.is_available(): + torch.cuda.empty_cache() return generate_pb2.ClearCacheResponse() async def Prefill(self, request, context): From c9bdaa8b734290465b8fb4db4edfc9536ff82346 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 28 Mar 2023 16:51:41 +0200 Subject: [PATCH 135/793] feat(server): reduce mlp and attn in one op for flash neox (#145) --- .../models/flash_neox_modeling.py | 206 +++++++++--------- 1 file changed, 102 insertions(+), 104 deletions(-) diff --git a/server/text_generation_server/models/flash_neox_modeling.py b/server/text_generation_server/models/flash_neox_modeling.py index 2e638d77..ac07aa98 100644 --- a/server/text_generation_server/models/flash_neox_modeling.py +++ b/server/text_generation_server/models/flash_neox_modeling.py @@ -1,3 +1,23 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch import torch.distributed @@ -16,6 +36,42 @@ from flash_attn.layers.rotary import RotaryEmbedding +class FastLayerNorm(nn.LayerNorm): + def forward(self, hidden_states, residual=None): + if hidden_states.shape[-1] > 6144: + if residual is not None: + hidden_states += residual + residual = hidden_states + + return super(FastLayerNorm, self).forward(hidden_states), residual + else: + ( + normed_hidden_states, + residual, + *rest, + ) = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.weight, + self.bias, + None, + None, + None, + None, + 0.0, + self.eps, + 1.0, + 0, + None, + False, + False, + ) + if residual is None: + residual = hidden_states + + return normed_hidden_states, residual + + class FastLinear(nn.Linear): def __init__( self, @@ -59,9 +115,6 @@ def __init__( dtype=dtype, ) - def forward(self, input): - return super(TensorParallelColumnLinear, self).forward(input) - class TensorParallelRowLinear(FastLinear): def __init__( @@ -69,12 +122,14 @@ def __init__( in_features, out_features, process_group: torch.distributed.ProcessGroup, + reduce=True, bias=True, device=None, dtype=None, ): self.process_group = process_group self.tp_world_size = process_group.size() + self.reduce = reduce assert in_features % self.tp_world_size == 0 in_features = in_features // self.tp_world_size @@ -88,7 +143,8 @@ def __init__( def forward(self, input: torch.Tensor) -> torch.Tensor: out = super(TensorParallelRowLinear, self).forward(input) - torch.distributed.all_reduce(out, group=self.process_group) + if self.reduce: + torch.distributed.all_reduce(out, group=self.process_group) return out @@ -196,7 +252,13 @@ def forward(self, qkv: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): class FlashNeoxAttention(torch.nn.Module): def __init__( - self, num_heads, hidden_size, rotary_pct, rotary_emb_base, process_group=None + self, + num_heads, + hidden_size, + rotary_pct, + rotary_emb_base, + process_group=None, + reduce=True, ): super().__init__() self.num_heads = num_heads @@ -218,9 +280,7 @@ def __init__( process_group=process_group, ) self.dense = TensorParallelRowLinear( - hidden_size, - hidden_size, - process_group=process_group, + hidden_size, hidden_size, process_group=process_group, reduce=reduce ) def shuffle_qkv_dims(self): @@ -309,7 +369,9 @@ def forward( class FlashMLP(nn.Module): - def __init__(self, act, hidden_size, intermediate_size, process_group=None): + def __init__( + self, act, hidden_size, intermediate_size, process_group=None, reduce=True + ): super().__init__() self.act = ( ACT2FN[act] @@ -330,6 +392,7 @@ def __init__(self, act, hidden_size, intermediate_size, process_group=None): intermediate_size, hidden_size, process_group=process_group, + reduce=reduce, ) self.process_group = process_group @@ -355,12 +418,24 @@ def __init__( ): super().__init__() self.use_parallel_residual = use_parallel_residual - self.input_layernorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps) - self.post_attention_layernorm = nn.LayerNorm(hidden_size, eps=layer_norm_eps) + self.input_layernorm = FastLayerNorm(hidden_size, eps=layer_norm_eps) + self.post_attention_layernorm = FastLayerNorm(hidden_size, eps=layer_norm_eps) self.attention = FlashNeoxAttention( - num_heads, hidden_size, rotary_pct, rotary_emb_base, process_group + num_heads, + hidden_size, + rotary_pct, + rotary_emb_base, + process_group, + reduce=not use_parallel_residual, + ) + self.mlp = FlashMLP( + act, + hidden_size, + intermediate_size, + process_group, + reduce=not use_parallel_residual, ) - self.mlp = FlashMLP(act, hidden_size, intermediate_size, process_group) + self.process_group = process_group def forward( self, @@ -375,24 +450,7 @@ def forward( cu_seqlens_q, ): if self.use_parallel_residual: - # faster input layer norm - ln1_hidden_states, *rest = dropout_layer_norm.dropout_add_ln_fwd( - hidden_states, - None, - self.input_layernorm.weight, - self.input_layernorm.bias, - None, - None, - None, - None, - 0.0, - self.input_layernorm.eps, - 1.0, - 0, - None, - False, - False, - ) + ln1_hidden_states, _ = self.input_layernorm(hidden_states) attn_output = self.attention( ln1_hidden_states, @@ -405,46 +463,18 @@ def forward( cu_seqlens_q, ) - # faster post attention layer norm - ln2_hidden_states, *rest = dropout_layer_norm.dropout_add_ln_fwd( - hidden_states, - None, - self.post_attention_layernorm.weight, - self.post_attention_layernorm.bias, - None, - None, - None, - None, - 0.0, - self.post_attention_layernorm.eps, - 1.0, - 0, - None, - False, - False, - ) + ln2_hidden_states, _ = self.post_attention_layernorm(hidden_states) mlp_output = self.mlp(ln2_hidden_states) - return mlp_output + attn_output + hidden_states, None + intermediate = mlp_output + attn_output + + # Only reduce once and after the addition instead of once per layer + if self.process_group is not None: + torch.distributed.all_reduce(intermediate, group=self.process_group) + + return intermediate + hidden_states, None else: - # faster input layer norm - hidden_states, residual, *rest = dropout_layer_norm.dropout_add_ln_fwd( - hidden_states, - residual, - self.input_layernorm.weight, - self.input_layernorm.bias, - None, - None, - None, - None, - 0.0, - self.input_layernorm.eps, - 1.0, - 0, - None, - False, - False, - ) + hidden_states, residual = self.input_layernorm(hidden_states, residual) hidden_states = self.attention( hidden_states, @@ -457,23 +487,8 @@ def forward( cu_seqlens_q, ) - # faster post attention layer norm - hidden_states, residual, *rest = dropout_layer_norm.dropout_add_ln_fwd( - hidden_states, - residual, - self.post_attention_layernorm.weight, - self.post_attention_layernorm.bias, - None, - None, - None, - None, - 0.0, - self.post_attention_layernorm.eps, - 1.0, - 0, - None, - False, - False, + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual ) mlp_output = self.mlp(hidden_states) @@ -523,7 +538,7 @@ def __init__(self, config, process_group=None): for _ in range(config.num_hidden_layers) ] ) - self.final_layer_norm = nn.LayerNorm( + self.final_layer_norm = FastLayerNorm( config.hidden_size, eps=config.layer_norm_eps ) @@ -603,24 +618,7 @@ def forward( cu_seqlens_q, ) - # Faster final layer norm - hidden_states, *rest = dropout_layer_norm.dropout_add_ln_fwd( - hidden_states, - residual, - self.final_layer_norm.weight, - self.final_layer_norm.bias, - None, - None, - None, - None, - 0.0, - self.final_layer_norm.eps, - 1.0, - 0, - None, - False, - False, - ) + hidden_states, _ = self.final_layer_norm(hidden_states, residual) return hidden_states, past_key_values From d503e8f09d23821239b3845974f6e56f013d3d2c Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 29 Mar 2023 21:38:30 +0200 Subject: [PATCH 136/793] feat: aws sagemaker compatible image (#147) The only difference is that now it pushes to registry.internal.huggingface.tech/api-inference/community/text-generation-inference/sagemaker:... instead of registry.internal.huggingface.tech/api-inference/community/text-generation-inference:sagemaker-... --------- Co-authored-by: Philipp Schmid <32632186+philschmid@users.noreply.github.com> --- .github/workflows/build.yaml | 46 +++++++++++++++++++++++++++++++++++- Dockerfile | 13 +++++++++- router/src/server.rs | 10 +++++++- sagemaker-entrypoint.sh | 20 ++++++++++++++++ 4 files changed, 86 insertions(+), 3 deletions(-) create mode 100755 sagemaker-entrypoint.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 56015177..2090d142 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -83,4 +83,48 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max - cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max \ No newline at end of file + cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max + + build-and-push-sagemaker-image: + needs: + - build-and-push-image + runs-on: ubuntu-latest + steps: + - name: Initialize Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + with: + install: true + - name: Checkout repository + uses: actions/checkout@v3 + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + - name: Login to internal Container Registry + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.TAILSCALE_DOCKER_USERNAME }} + password: ${{ secrets.TAILSCALE_DOCKER_PASSWORD }} + registry: registry.internal.huggingface.tech + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4.3.0 + with: + flavor: | + latest=auto + images: | + registry.internal.huggingface.tech/api-inference/community/text-generation-inference/sagemaker + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }} + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile + push: ${{ github.event_name != 'pull_request' }} + platforms: 'linux/amd64' + target: sagemaker + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 592f1f72..85463af1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ COPY router router COPY launcher launcher RUN cargo build --release -FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as base ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ @@ -76,5 +76,16 @@ COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bi # Install launcher COPY --from=builder /usr/src/target/release/text-generation-launcher /usr/local/bin/text-generation-launcher +# AWS Sagemaker compatbile image +FROM base as sagemaker + +COPY sagemaker-entrypoint.sh entrypoint.sh +RUN chmod +x entrypoint.sh + +ENTRYPOINT ["./entrypoint.sh"] + +# Original image +FROM base + ENTRYPOINT ["text-generation-launcher"] CMD ["--json-output"] \ No newline at end of file diff --git a/router/src/server.rs b/router/src/server.rs index 3b63ec8a..f7850053 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -529,11 +529,19 @@ pub async fn run( // Create router let app = Router::new() .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", ApiDoc::openapi())) + // Base routes .route("/", post(compat_generate)) .route("/generate", post(generate)) .route("/generate_stream", post(generate_stream)) - .route("/", get(health)) + // AWS Sagemaker route + .route("/invocations", post(compat_generate)) + // Base Health route .route("/health", get(health)) + // Inference API health route + .route("/", get(health)) + // AWS Sagemaker health route + .route("/ping", get(health)) + // Prometheus metrics route .route("/metrics", get(metrics)) .layer(Extension(compat_return_full_text)) .layer(Extension(infer)) diff --git a/sagemaker-entrypoint.sh b/sagemaker-entrypoint.sh new file mode 100755 index 00000000..711e3721 --- /dev/null +++ b/sagemaker-entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [[ -z "${HF_MODEL_ID}" ]]; then + echo "HF_MODEL_ID must be set" + exit 1 +fi + +if [[ -n "${HF_MODEL_REVISION}" ]]; then + export REVISION="${HF_MODEL_REVISION}" +fi + +if [[ -n "${SM_NUM_GPUS}" ]]; then + export NUM_SHARD="${SM_NUM_GPUS}" +fi + +if [[ -n "${HF_MODEL_QUANTIZE}" ]]; then + export QUANTIZE="${HF_MODEL_QUANTIZE}" +fi + +text-generation-launcher --port 8080 \ No newline at end of file From 55106ec4766c787823361db80ea461715aa57a7a Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 29 Mar 2023 22:27:01 +0200 Subject: [PATCH 137/793] fix(ci): fix sagemaker action (#148) --- .github/workflows/build.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2090d142..a94a33dc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -94,6 +94,10 @@ jobs: uses: docker/setup-buildx-action@v2.0.0 with: install: true + - name: Tailscale + uses: tailscale/github-action@v1 + with: + authkey: ${{ secrets.TAILSCALE_AUTHKEY }} - name: Checkout repository uses: actions/checkout@v3 - name: Inject slug/short variables From 610bb1f97811af2d80d16efd96f313012beb65aa Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 30 Mar 2023 15:26:27 +0200 Subject: [PATCH 138/793] feat(benchmark): tui based benchmarking tool (#149) --- Cargo.toml | 3 + Makefile | 3 + assets/benchmark.png | Bin 0 -> 103931 bytes benchmark/.gitignore | 1 + benchmark/Cargo.lock | 2801 +++++++++++++++++ benchmark/Cargo.toml | 35 + benchmark/README.md | 30 + benchmark/rust-toolchain.toml | 3 + benchmark/src/app.rs | 688 ++++ benchmark/src/event.rs | 65 + benchmark/src/generation.rs | 211 ++ benchmark/src/lib.rs | 110 + benchmark/src/main.rs | 119 + benchmark/src/utils.rs | 43 + proto/generate.proto | 3 + router/src/main.rs | 10 +- router/src/queue.rs | 1 + router/src/validation.rs | 1 + server/text_generation_server/cli.py | 2 +- server/text_generation_server/utils/tokens.py | 11 +- 20 files changed, 4133 insertions(+), 7 deletions(-) create mode 100644 assets/benchmark.png create mode 100644 benchmark/.gitignore create mode 100644 benchmark/Cargo.lock create mode 100644 benchmark/Cargo.toml create mode 100644 benchmark/README.md create mode 100644 benchmark/rust-toolchain.toml create mode 100644 benchmark/src/app.rs create mode 100644 benchmark/src/event.rs create mode 100644 benchmark/src/generation.rs create mode 100644 benchmark/src/lib.rs create mode 100644 benchmark/src/main.rs create mode 100644 benchmark/src/utils.rs diff --git a/Cargo.toml b/Cargo.toml index b3bd5dce..af479c04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,9 @@ members = [ "router/grpc-metadata", "launcher" ] +exclude = [ + "benchmark" +] [profile.release] debug = 1 diff --git a/Makefile b/Makefile index 3defd886..21fd11b5 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,9 @@ install-router: install-launcher: cd launcher && cargo install --path . +install-benchmark: + cd benchmark && cargo install --path . + install: install-server install-router install-launcher server-dev: diff --git a/assets/benchmark.png b/assets/benchmark.png new file mode 100644 index 0000000000000000000000000000000000000000..64d538a0598d57102b82f5984d2437f9db129da2 GIT binary patch literal 103931 zcmce-WmFv7wg!q55+pzfE`eY{8k*pZBuH>~Cs=TI3lQAh8w(I5xVyUrcM0z9_KJOE z?|bgP=Z*1xJVtd>UA3fYuC=Co^IL&3QlHSC6Fi53fk6`$5tM_0LDYePf%kic1YG&d z#pwhC^E}&FKtM)RK!8lf#?sK(%m4;PBrrA}SwVgq>+7SdrAr1PIT_^{l?e6Mn3rl> zqpiXsa43}D_+OUN1=H2M78K;y9LWywHigw^2tb_6oGt~BNDzlg!3<{KWw-h zO19k?itoKU9loo5s}Emch{)jGbq3ZAL{)Tk zKfuHq-#R*b;__o(#H|Y!c|JZg2Z6OCI^kiS3vfoJb&7ww#>*de!ZK!q5uw~+S4m<> zTy}_2%>LpkD(M@WxVAJU-cAs zNiuW6&{LrOnCHo1f-}NWAD@ItV`TkVWb1wKRrGN*ea!VINfuBrG2ftDkz=}>xthqx zd+ISKh8O3tmr&?=yoV6c^{koh_vQx21BrV$W>YF7rNRcbrj|J) zDk?D&Q13<5A*JYx=3lTNSO~=V38E zfbd{IEzg`Vy2zfj2`B_08S~dgVCuupe3a-RKJoHSx1vE8`x^Cz`K^x<|855P54?B& z`ROiOMq3DWuj^3l1uipqzu>vRyZd1#z!kL3mM}wb4cq8BUWR?cZSA$9N`uzCXhN**lOnVI0 z+SkO_cu%Nzv+a!IJg9L>rgyMdG?u2y8b!G)k;b_0Zj17sM7SqU96vGjtj&X?*jy3LM z>&xid?n{orpwU($k!vl$E4cb`qbbAwxY;5)MWR|t+9B8qjB1pN1njc@wn(O+FzVMSqjC+?5A+HC}RmD}blDfnab z6K2&jXtvYytr}mnbUhZ}s;nwrs_s>7L48tJA)g?*E>hu?olypfdNVpRMncuk%hZe} zKA)Qut;}E5&TTjh-5c)fq#8CZ-DHFYNq!yVN@h!DuiB`B%sro*b5P*cuAgxD=}^4S zu!nfzx_8JqLgY=%#$m_dW^ZQ`!^Osh&W+DS$)U(~&ZL;AJ0LkAKBQs<+MF1oPUWlR zsMD_}vc4Xh7-gL}|GGN@iFWpOzTG^p->$b_$e(vvz&}7a;H$fI*>bkx$>)K(5ICLi zm_m?M4GUjQewd^KnS%1Btcx+sa4*Ch>e+wqZ)$EWtY`hkUY9>O?NDA~pQ&zj>=W2Sdal#}0Je8Y}lMw>LXVDI?fXhg&NOf{j;Y8uyOs-6v zEL`EKPo4qCVYFe}p{a!R;0u!!!xWk#RuWQDVr^``x)!6>Fhuwl#IIk|H`Mp&%N}I* z2eablGVoRkTBA7j{z5SODADHi4Ky!f!Rq)vGFl~(qfPJB;Y90*=bG|P*DE<)g(+bz zP9f!e%wAjrkF(0w8+x3kSEWMhnNze6w65~J-~DnXbHZ{vzq4jzk7AEX2Pk&r{XWI# z!mq_&z6c;SrgG^N>EC;cvd{gpyCkD zrk1*4li6xnD`cr5Elw})G|sVbM7`8OXi03cFJEyazeAC@08MGLm_%i@I>&;+*nDSJ zdrGyKPV3Ubec-0bZU2t-`hMx!eM%->hKylrX1PxH@G{|0XvxvMVy?z?wfdvSN-X(q z=4d7;*(tf`PUlo$24gijn$6MNdcMqv&Z59+cfp)wZYL-<_%!IY8rjm4Q8(sQT*F+c ze)qcE?Pw#nD7OK(Yrl7&nj`+fY~8MSZONUKudSNc})*rC%qWZ ztb7V}ua?_<*wLLyP0TCI3QaQEafp%p4gAv{Q;tp-D7-##g@MD_NLaX(oKqYcO7Kmz zCLyEbX4~eWru)2@R+(9YmHSs9rRH(yIJc^`*#3Cj*!QvP-0tDERDARJie^Z%TUE23 zXm`79>83?mnrGAQ$7Nq}Tz3$3SKl65yXLUHW)R;`UBpyep<&-J>o&MDd#p|D!LztN zJ>j;n?XruuNoc-sS|`19R^xmTclk58nzOoi5v6v_oqgw^5$em+czJc{caXc_c~7?4 zI^E71UK0MCl#YkdJ@fqg4SV@gk_)k`^ttn;rYphmnJ!=PW9Ol2vnPI!r?9&c*AH$L zK7DRGn`y_Zu2aNeWGxY`kVg6|f$Q)-<(%TM`|az^%E;+Ruc^_gRZF(U2`Ba2p$EhDZ&bpC-R)t6OcHAT$5#P6x-U zZ4@1D?pmqg#%PwCZwRVz=^V$+FlPYxLnGjGF`xL zRWeW!HI$Tuc?X<7gMsxohCu+%V1W+-@B!T0C_flv;6E1d5ln~s*Ih)NbohUr!}~oI z{2(tNDhm9U*RwG&u(17NX@?p5g#l=4%2+|gPDN6JOV84rQCHvcvjL-%xz$q@7>E-W zaB6N~r%UE!Zf0T2<-|k&M+q+A{OK}?oa~PxcBVY!Dv~l}0+u!gWNeH~j7;Rb&&kNh zAU67jTylcK|Ev!D;vxTHXJ^F)0y#Q5GCHy_TG|+am^nE)K}=u}7|Z~aV6b(zu+w#7 zu&|~0M$G68;9KF1p84u{?k)fS zeMEF4h?%;&pVUaQDfgfddEOnD*{qAHmK#dGIW#`rT^!;*r1vNFsCoZogg-|zS?La` z2NUNxTuk{#Yl3MMEHcQT*-%>Y-SmY+n1iFT5n( z>#IQnALG>!=jA=>>ab?Zuf-x>*?2c;F1IhOaXk;ss&QDozXZ8{w zA73k#%f5^!*ne)_!KRs78gl%t^us_R3-vIR^h7>xG%}w2?TgXmy~ram`vnf>v3GKMGuZtna^z;iW{Q##(1p^g&lr{M`zk6bvtT(;7#^l zkCx_5T)hG{U1e_1ftv54C_ss;b6{(|>T6$j)mj?^JweS%lUV=L3bPsT>Bp9fdi83H z#raxW$;F%VU9^6=Y%%9rp6a~CA4l!BxBJzu;&JrH1}1$mw2L!FX{iOLM%}6j@vWwH z;_-}#;6;wwp?&A@)sR!y^;l(>a>GIJ)KP{%$xb+n6w>PK^8;zcp@Z5@l1CD-^NBvj zrIq{c)+Ob%gGV9t?c55NgJ?v2Fk@X$Gw!m`Z2AYE2DM5PoM~s9Vk6F z3omEhZd)#*ZX{H^<@@`~rH50okqxi&$A`;i4-zxA)$Z`igXTvM45=5~zwtBD*LY@D z=UopQyW)<9`5yIMEkr&c?K7Xs$H_(F&ZUu8yrL-C2-#}pxtt#(WVc9}(WrMYufj&q z`!L>8!pFig7VH|ps9t4O<#Kaf$%#&XB;u_5)z?Tz!TXqXX4Ur=pLKG%sVkK5)_>k8 z&Hawq&+^WCziS_Z1l~;ds-4%V5NXWw;gqzTbd&zw`Qg&iA@&Zt$#~CPiLMWA5HC`= ze3P4Vv1BsaBC+HAiew7MuXYC}KDUF_E|S9NFFY1^*0;a3YDfdaFykMaAMb~)rt9nr zYV*`RJ`vl;)QT3Xl(y}uReXtLZu<-`JbY9uIq`yFf)FP}ht0U>nY6&Cguy)d+$7VX zFi;r2es^CuL-HQDNK$<%mNXu>dZ`Lu1F3kMWf;Pe3_kqHDDCLV? zLZ6xru5Wm6Ha!A;Z5!?f4nkNT?eukaeoU9n(HFx{N0mXc0l|MFasGaJd)VyCpk6g_ zeyEb~f-$6>GK34M9_jXWu~Q)(INbcIqn$aa``u-)v^#{E^BrIHt+$J&(_U-CvUS^l zx3fxk0d98uT%EPOnTHkrfzdo?(2fxpj7cJJ<&<~En)oy}yjt^D-V)xs+ZuE~(vpqD z;?c;b*$V~Gn;surwKpq=o2-(OIBj#9`5L@I3g+IwCUZzVu3Ftbm{e;H$E-9>?^8)8 zIWg%#t-J7z7YgJeQCtUV4C*#v&$%4-wpvjbrX9dwJbF?MY2`SqV+kQG@B7~H^BN-+ zqW)N;VP^^aV=d2nd%9>6R2Zg)*>C=OP6i*GbYYU1_53ckvchkW8#awG?vH3a!A#N4S7F>*8hrJRhX?m( zW73Nyy6qCA%Ygy=xlorrtDEyFZOO-D%Zt64u7uGdP3|QT|NAA6+n?12{jV`da_jf2 zl8vbkoPY&UZA{u-b1`e$Xasfc!kcd)KDcCGbfQQ)8~ZLA?Z$@p*p9(xcCuU>7))m- zDAXMJaCa*G2qDwx04Br=A-*K+z2p7)RC3cY($(oOpX=`APZlr?B?=ge-C?S>4AQ!` zTinc+@lt0GnPoh9)w_WMe{U=|d`a5r4d2;_fFWT9lZJb1a;37i`^0=ei^Eusq~X>V zj57ZUyKOm)w0m`d>E%|R@>F%aOKTtGW=+S131`S$yn>~|tleBF)X#$zX0znU_aZ&!@Oh=OW5&hd8Nq`52Oq zz`t0DsX>7^_|&K2kQ0$Y4Rr(L%{Mzq!b|dvM!-_E{xam_Unw~Wvm4dQl>1{}VSRu= zXXU=Go5VH26?F3r_vIJ9FxC@|O`d}~p~m?;2f@(Vn*RRMZW7bZj`qfb{puyQxotmz zz|TGIwd^zzZ}8=i09yoGjmm%M7abs3+<2iT$5jz*n(q?6tC|BC^Tmia0&cm;TOXae z+Kzp0bUr|kYr5Q)y^z(ukwC3o&$c3c^z!Ry!*Fw_5W7hxeKiM0C;E>m6T}}w^sx7g zu83-Hwxa^=YCFy`Z#BDZ&&ODKr;g3E@~58fbn?j0vSBl1nZ2~O*CDG!sltev11xx` z+fb8tpFVNsaLfBeif(emPRYaJ`)ZlSZ^{e2mk!06a{fBcl29s~? z%@&TpS4W}I4!;AbllYvPXUk-tB(b>6BwT@ccI0~ub8rAOf!0p_yKLH}$ETZ?(d{js zf@b%eLA+*}vtE+e4aVNf0=td?Ov8Bhqb{TdrTwcyQ)dM;3>xkW2-NkUHV2(iDOqQr z(k~rwb>_L(W8}@^w)uD+g>C@2I&|f&#a|FQB0Z4hw0f| z1F*i@@$6-h*kgY13huW{&h;H;$7}5@wY=$`(sWGq6iIzgH&^Scq0VU8c&;SZEq*R+ zzmu1H(~WHVE%xi?+pd+*leM;+)23rkf0|l{O5@!Oft6GLhZcTw>}j$U#Grz~AE&yP zh{fAS;n@X}%6yvOB|uRC_|)8z!lCL9|9I&L-Ffba!g)1HaxtX+V~zelCicEKaff~- zidgur7K!7c=9IaGCwFF-*8DnRX~fLC2f+E9cw0p1wcJ{SWs zOM~Tx)1g!Q=Md!Ondd|3wd*PQi>oeSKD@9zuj}x64XRM;>}9&?12$oBaR+2!7bbST zY*DA=duIfW{*qiht6QFKAFl2Nzu@DmIkSO0&Xs!E$1uDJk}gSGv0dBA6{oUFv8xe0 zA3m@p9-fe%x-y$9GRU6_+D#m_CrZXO@F{VY6`w|o0EQpeRoe+zE)w`o9kXmN<*-qZ zRPgY0Q3bpyur|g@XzwJ9ckANlZd4z!pRyfALcdPbDnH^MGrq0y?Z@5OW>^t|}TRQgP@lHsr6N}+{BAyNcMLX(E`-TdmXtvTfK zpUh%H3vKrocOS!WDze!nuXhTH1wo9TdQ(()c;31EO5{6#H>gThLcQRzGx&kaerK(E zyxd4y7DGrjxVjmGZs*+O#><|yB4F3a^Zq=3%lB(0)wVkKFH#N`kCLf`VnaTaeCDCT z0$fd{6@CkJnZWsQ&~$6j%u~4R#PLZa;lpqzqFzY!2OPWF^GS6{?j3X5oj31psy>{V*S#IX zZTt{SP$7gi@0aHi9jJAc!*hNUOK4H^VcciVZ0e`_6#YIQYeKMAfZ(e=^#6IBC&M`=EiIXy?;b1pqJS57;A?FO1zG)<$Zn-IZW&31ax7?zEjK( zXsUdD{@vk%W!muF?)JqPG?yk(*>d%1dLP;ys1gVZlC`hI6r%d=u1{1o$($sQlAcey z*6udc7tP(>?H_TIsL-OHc)MOwr@4I=&K9=*#Xg(4c%5lk4^OfF4kZHS>3AgtcQays zW(y%jC*_^r(zsEKqSvTlqD|!ieOh;k`-N<2JX?|X13AYTy`b z_x{ca_Nilyd(@#L6$PDOWk*@x+dAC!BvBC^AuWY^no&k(=;3ZK9q)^9xN zqw_YUOg-DqO$+i^rzdqki_G)5TJcXd-!omPvp0+G?GewPG6cL72d?>H9cmQncvQds zjZOzJBx|kHp0wg#ESB2L>;|iUTJ$W~Y%GM}aFA!>9Cf#^+`aSy(gW=xXRFukE~Uk$ zbBYjv4aB!j<|%BR!}&9~$w%%vu%E%x6L=%a^tDwjRSBnce;L zx3cJk69@85oXYsb;{1D^W#7n`qC5(r8JN2=NQKs~=44TjIPT3B^o_5c*?N>_=wc4= zJ|hijyiu*6nS_}#4D`f|G-;;qW7^(C$+UkGi-Zc zeEbT7Q$1r(?2)Q2q!SFxxbnt}+5X`dX^vC&$!X2R9{zB2ogcK$LK~M?6dOMYt~LcV z*|R;z(JE>o>;D&g;G6i>FXOb{f73htAzsAjlJ0ejm_Isk2z-RpV)Xu(3rF*%vcH{|!54Pjyhlqt){Js}I-l3t!qpl=;jx z?;=`7@SB__2IpQMhfY6)CE!dQ)8i`m!XjCE?9Gv;67^di1J3p3)QZ6~%i8qrq_pe9 zpsp9p_wCm%k~{+DqgHTmdUPRgj$R2j4dpW&^>N>9Wl1uJ5VPYoIP-gBVzs?$phXI0 z-T6(@59ilWv-aC&Tk6W)qBmc^M>=tPSgEd^!hf&UJxwev7 zBytu9s;UfTkEAh`9QgBET$WR0>KxJ{HBD3!31k;gY+lccPSifX;rd2uA5;BA_@`dlW*T z?Ewi^b6-4{z-MpbAQaSa-s?Us^3%u0Th`_N+NvQSM&M67FMvfkowSTuqzgjylY#8+ zrmzL(3vH;+Arg^Unf1_3VpGO9sJJQs8Wg+<3&u1kw>f8*LejJ!9Te}tZbmGj^EFp0 zx=r5~&xgojXvenqVs0V~*G<0GSN+7(K3hDJbrMwznt5=!fzMj!!8oSE)Wc$4XQHEw za(PIke8|h53UAJ|XwqVmBBRhExRh`tLo|P<~dO{?1Oyl88~4Rnxua`1D=*&cpU7~2>V{EHm=8O**Ny5)lZ)Ft z)5SO7jn)<5?EM{M!;moA&ntMLm~nyk#hExCHv9T9M(g-5=?-tfkmrs3cMvN<&j{Yz zJ(lLsAEoMyvf^`w`5H{(`I5T$9p66SfuacZ?m4P<ui`ofoMPFh95pF?&emyTkO=zrot=b*;%(8J`u!+MMFk1Y zJ3>|y1-2Rc-pT^WscV6d&)h7OPAh67$%%=~qZ5V+(pKC~l6pj6a6M}#-{1LrF1BCd zKKZ?t;lz|vMQrAcPkt}E;Q(HHUtbBq^k8E@rO$;^k-<9~KL;~Z6^+dcE%=Q9*Fo5x~)W8P+4PY`Hp zC85Y}ziscZCp&E2%M_fyIohvzV?>0#SA8GC^{L2XmMu%4T$8a%atgfTROC_DwaFDW zD1NSH{S?M!#W4~JDmd3;FlXdSr&$Qiby#(tg4Q-81kjnmP#Pq9GwPkUpJ(~i1Lz1O zT8+fqOQTlnO5IDvO&vuYuo&YTrrG2h*?`FS+zbOs&^AEG<5CgMkJf7_XE_&7gJ0nc zxYuR+b3wloi=K+E2L@Jnqg1`P)BVjp#ZpKoO-wBreFN7@&-{pOmLj5AXNS_4cm-Re zyI1b>RToB-2cGAg>5Q7TjTcXyxfTaikjZKmp`AL)#Vpa`2Z69rsajzvkvWgDr0; zxNTWj2@Ju6gj>5zebjNus z3MiNG^xEg&J47JEJ29<5Qtc;I9~dQed=cUjO8TlWSsjvbD<$Gz%mzh&2z_I4lXJ`p zao=4fjMBN2KU8QgUU|^zy8Uqm#COSDH(hh=ZJ8+^2bOCiVIO$fdHNB-1Z!BlN>q_?M>cuLO3`H|8xDxLXBd-lJ9Sh@;Dw{qsZ!6LO`z51zM)+Y7K-jDblo)`Asu)i4CHNty>qd%-s-iRv>6~exD z|0?H=i?NgaxrmmC?q3+h;J)Rj@hbp%BU6D;-IjN`q|gnN1>zdYLTW(A_8*1y1^<}1 z4k@V1*GTB7ttioBJfwL$JQp0YT-x1X#O2$I0{-Z%!ekH;b@!5VD7+cpYFq*u38+6q zStwTY6MN{`?%s436h2D#wIYVQ!}dDkJ&Q@hw9zfTeVE35#lBE4!pZ2$=Q%rutgy|U zTvNBA)#(O>Vw@?K>?al8gM_KreIS2ms>f8pRqWAa9uh#n)z(wX+G6s9NBh7h+A4z> zjq;h|%-j1Yj!`N7G}?XZ)wnGTs)jMpxBJ!3*veZVYTL+lF|Ql)*_|P|oci&~E|;#!P%t7i-x`nA7j6hr@ii^a8P)&4{UClAhQJXA{k2D%ou&U z;%ohJHPQ)s~sLMKj)_!v$j(aN(-gDL(8eqJ_H0519_90U%wqH7POjloyPKhME4cc~(GKJdI ze^^0H2}vT4E;_;a8bmA_$^>k0lX0tlrim1-&Q_eS#?>gbpt-dfjGD8|OIdNpvT1^g z(-^PrC6$X#Z=kUvNIpaW-r1*G68gg)cemzWI&Gp`@K!_b*%U*_37i1EM=&dI4XAj< z!VTvEHc@Jl;=e!@w{Zj)4GL6AFDdlMC) zT4Pb>qV%p#{Rq!`lSYyR{8eU{?q#;?&l)JL`CMyuKN2SW zrkcP1dpj*ORuz4Ft@wuTw!kO~WE#2R?sDMUO^7PTWzG(~a2EA9QqbPxeNr>SlaBPk z-5AAsUxhBb2ZNbcK6?kVq86@PKs|1Z^-xhCea^M?o+mC`m>bkER*%(N90_={$2;w6 zN1*!XGCe=3@!eNpObiF(;Z{9R&!rJl8ueZj|A?WZJ&{uwLcUF0JVdXJBm`0rGV=!M zKel*;veOVLLF47m9~NWO@g*)2M-6;vXs+K4b&Fm=)~J2v`{YV2DMhuIB*{@fNKi(6 ztt;ziozkEnNczAeicR&<_KiD1NUE`5Kw^%-ax8|{%PD9gg0-nUaR#G&x}Z$Huq&=l zM*~cnGEN`MrdoA{mEQ1b$+kZ-@{f=ch&uOF;Ks`x=pXIEhK%~`eyqC_L;V|!)MOY& zVbo>5I}`mYi}#OQ-e14Hus)Gxb%Z9sQ~mYof1G>4kw+9GP0MS02+fk%+rgsw`oH;) zB7gnq)h0y7BI`Ir$RsmM>|Q8BD@Zr8X5bsLP~-6P%veUf%3?OAlJEDU1vQlZ#p8|q z!^7|L0F49kzqauo=U%@h$ueLaMh4mY!tZayuKN_%3afAWdvKw#&Hkv&e$(m4tmPjO zXUaU>-~N=t$M*ZN4P8f-wG;NEwcS(?wClu3)Jm0#h?vFH&N=AkMlKtVKG&LVKx!RVz?|E@@ov(y z8^{iE;>Z}W$Zb zQZ_@diu6wu;%|F=^#-n%ttHHLRq^^h3q$`r(-(Z&{Y%br%0aHl#h$cv#GfRB6zZ(D z*tq3}?Va8Ui<|)Ii1;Z}nWRvpF8x4q=X=I|BcfXGRH_|L_un$kGUF(MU0-?54tO7G za?l^8a<;V<-YM2qq}(pT%&hl7%N(}lXj88(Z5`See^wX>C~2s4_-it|Rw(Wuc=;&fmJ?>I;fr}v8O37HY_(RGus&GaodO#S7D;TjzQ_` zaDZzdapEzH_bX)WLoo0Lm!`rLBWq#IWqVzUXkq+ukGf^oYpUJBc=-^%*gs)6!`W7q z*VJ`!>hc^VICN%c1$J(A$aQUm`{1=@a{T$Gb0kh_`QXYYv3k9*XwYGkz=7o~KW|f$ zBaj}&p)=|6qUw4U{PDE=kqSgSt9fAgOGWt7T~q)&1;kA}&sxRH@{f4t@4wSWUdq(x zRc4U5+-hRknV?-TkHTN~OqEOExQh9B@D->#~OL?x_U!SKs>bGipUO~&}s?4NzY9dotr2g-< z4~sSZ0YS_lCJ18l9hNPBt!gk?5_L*vF~wSMeai7xyqeH5lB9ZpcC3k%(KMbuQpzZe zLVMn$fNYPSA@$qtD9FD@>1Q|-{OY2M=DVq45I#ZE@|j2R^ENyu_3EUC?WOQ%Y{T~d zt$&(PgebNai}%#y&DKUnNKAHX6hR%bVl z3N27d+-E)A6kmdVN+^rnors?}-AueA3bjT4uXA4)PLxnbQ*XZHvu_%jz|Un+#AKeb znZm~z92O%%18dU%TG<(nlWOJN3B_-izaPy%0#}jrh?nWGRS_xA9DM#6%l=L2^efCB zOz(7H3U^w@TthMez+5kk>CDUZ^jf4d*BLB+uMvfxm@P$SX)1$s8b_^8@gJHH7K?@# zL4$3Ah@_<7aOl+?&g2=8*_EyX(q1a{$~3pJ+5>%Y40Nm}<1{gPL*}_B!T2|dK>B^& zp;%^e7-R@g6h?o@QqU?FX=D&NT5tj7vCdJRcpF8w)Nx`86+LU!mP=Y%PAiDc*X!}8 z4Z}sQn%kq10ot zLBIZDgY#&k>!k*>s6y=zmhx58k#sM5l@fj_&h=OIbW3raUaG@To=~^JM3(LzLe#j# zL{6g1(-2*29V9&K%km%`x;`L>$S7|2xdVXW`hmP-q|sg*+Je(+sPN>QL)*=v*iyaj zK8kP%`9!|r3*5m{&5VzT+Y<%x^8k!28v6;}e6EVMnHpEqPB!dfbm13}54{G`g&}(k znso|FTi>F$uu-RC>6GK1*e6zkBnQ4ddVgcYyqLT+2z8FhZh&|0MWIe1Yks&XaE9Ue z9Y9e=jEpfRyA5DP@m4cBoE2!pSv6~Y!3Efh7>9M1&G(adlsn^j6qP0u&vAQC_cMnI z0Zs;7J09xWrv!eXb~CSb!;trKFc%AVoH(da0W!=YOk59Y24pi63~Hh&HyVz+znB}w zJ7la~!%Nv^I}{0sXtCk_6E1%mN{JeCzB9q%3HY`#0DvfN zQM>-S_HuDALrv$EUm+6qK|9B#Q^pj)64FtsvY)d2N(WM>2n#hWZk%+p_Hv!iS1br& z#&hSoTJ~A*rA~Ww=ckyj(A)5}V+y9HohLHzZD?#G43_Dik zu5&O~U0kTus5bdiojU7Ncw9R=_rRLt)<~;Uzr+6Q#GcRQ9+2LZBzahgmV(T;nVPiw zp>^4s0Fm?gsWjL2+eBXXx?{1OrDo46|4T!H_KR3$EzB5Es@7HWl{R0SNmV0!3Nx_% zNv4FX;RW!hBCB1Y>l4b_u>dh<)T#ZTVoVB7K+Gd2a{yHXV7TaoOg^O?>9ra-_++om zaSaTY5!)AzR@&Ed(>%aatVn)V2-k^0;BvuB7CcnG$2;58HvQ=m-4g(2-=G#7V}q>e z2Bboq%VxUVNiVnG2>A8%#nKt<&sJ`b+#R)JB)hrcGk+N&Yu2vO^0+(xwtx6 zsEv-m?-$VA7+$P7BA)HB99lDX^W91CUe%lho46GB{`*9*K@f`K@}XU!hV3A$4ZzDu zzSRVzIoOZ=s@sw%q!&mPCo$8ndIS{nN1Uv_KpSIv{I}cqZ#UtUF&y+_@o`L4337*c zSvxtAJnRZI)tD8|J3EGKVA&tXu(;@YXuhLu)rv^+aF($&Cau{Rh)tbzzhl2cY&*cb z2ymQuSPPxHNj)1Jd*1k8b&n%kOQWIuLH*jMPK6RYzjI^V`W>+djKp>ZQIt#fwS95Z zjY&BMfIo4+lP1lP^vpcbxD^AlSp6Ed;<^9MNx4WFIg^(QHPJs!_d!1yhyOJ;+|f&K zbr@LJH6F2%7da#^Eh&xW5A{Pi9#4pWUmZnb6sLFPkpoQ~Ro=r31o~hi97vo#o#?x?~ zq&pcYT~4g-6Y=&QB(waZ)5VLxRM|V6!gX8(Cg$J;Kz&&amkn+5NPNm4-!hcfP1U7+>RTtn zA&)5RS3}1-fyhHTa>Z|pf!v1&=RNAgnDivixgTq%-EUmJ!-C?b6W)g_Je47}`OI28R7fqmY}CkCXgH6A?C z(c=na@FnnIS&NcT#F%6QU25N{_r7Lq*?V>`0O4_i21OuRqok#h=T)1Bkw26CY6WNV zqc>7H!QmCM;WO^9wh)VX4q14!rd5W%furm9zl9$Fippr#Ck*4!?}0j#4sMevX?Zj( z?MAy#gtD{Uso?=|_*GIu18SP@rqR2;0S%2kcll-#CT1OGS=@`fPgMnKMW;-hIK20y zoHpN$cY$?<VIBE;`)dTZOkFMB@d0S~d*X0&3s<$w0*wdwNxEJ!AkZbe#CXYKhrD`$cp9zxX`v>t zIVw;4r9luE%e~uFiY1=%*{Oz}TaTrMP$E8RQNjme5Uq~Flckp|6d}9>H&5dj7%tR1 zvV&|EU3MAT(V25Fxi10zHVo3pS;!n1Ks1~zRPEYQskYFRM?L<`Joy7xwa7tkP=Ltt zoB-$kcdLSsnRY`->ml54%XYk5fS#8Oxpn>8G=t!iC+XB`C=V#2!F-o`0?yJ~O>qeX zBp*=7Nbg%BLTg*ywazuOofVUOUGk3&c7hNyi8@3*5EYdmC)IztEC8kT8Fr7wJ?bV^ zZ78x1p3r-(g5?aSwr?nz%y$yz_eAmlL5oIN!`gSI>pK^lZW-agm$2U2qn9rza7L%J zJsFTmoYJ?bb8I>=KlK2qeL4c=NhMpxb6X`_HwpAuGn2WhpT|1+^0{GORN%|eSNZ?2 zBhh5KpWoumGG}o%aO={y(=R_63>Lh!7Gxx1de#oVM1|*zvws+oCC|r&bg#`f2gpyf zjuxjYz(#LSgK7H)V5kl}47HQcXNat;iYzMy{4N z60Uy3g5DJ37O@ts3XxFte{=!3R-x5d={uSQG{hxJyde@pz9`}mO}7W!EGWUeg16>BaMO+Sb~8K<7aJv^yaO%) z7^b`AMTNWx$qYf0hPkA5mar9iA zfQ2>A-#XF!oROkD!1OJRsRJ=H#fx`HX0lCe6I(kB+D-F4xCh8P)?0wW3ANdXV41o{ zWmd-2CKTnHyOgUP{@@^AJLRj_l3(4B3z}+>O679CksFfFD%?hXKqKZFmj{iSl{OD^ zZW29_;XLqJUnQRwA=yZ4pIqNglTfeeu#hq*MflB}5AFBT^$_&8JSn;iR zSmJLSnd*AP)TRqj-IulM>6FD&?fN_16)A$Q^|8~^{PfmZ>0aNq-L4R4cfIyvS9pg~ zN^zW5#v70;$FN_fAIHT7rYP4(*%B1bj0OibxiM!*Mg8=*i7Ef0b|ltmbovYbR z5Cmee2faG~CgJ||S|+Qe2Okyc`{vEJuaZ~h&j?km56o$vs}C{i)^uv;qX>^HV0&c! z_Q;hp4_+7jN1*kWZ!r=tNFc2GNr`pG-%QJYv*IHXV9-he@k@8tgu?1C`72qcq5^FqEPo1nw* zl_j$1y}%`X>&zXsP2sm`?n9Z*3dDAAqyOJyySU+?>O3&QhhxrRB9#C8dHLT^@WY$E z;eNA+{$C0mzX1vzbH@Mw)=~uiM4G^1&D@e^mZ-cc8^Dew*2tSF#6R`sf6%1H?Gsm_ z|Nj8k2xQceB(cY@mzS+!UyK)%Hh|pn2OI!7k02imDvg|$ncMuAm>ZZc{Ajt0RrIUd z5x0l2C7Kg(@3QIMm#PtHMEwgv{kJRp=LtUvmJaGPqPcP`0{m7Ai*r?Z(^x`Ka@962yJwk>;Ik@ zFeqzpyapgWKj7%3QSX;w{*MRkF@>vMbSXe3fv9)@hNQtF zY5%V=#4Bt#6tt3X-kTppb|W9;@)c-V^oLIeY=D@|2A~%81I}?OjmyOztwDcWtQR~o zSO3fa4xLgD;6o$&O#|$-@A_Zek2`S($(wYwh(VQ7JTSL~P+x8wo9Htk8`C}Mp#2WPFv?~yGQ`yF^dxX!z_ z7oZBoGKQ?#0pQP?NIg#?5$wR)JsKOI#q>Y$Xgs))L*Bb1f!g&rb^VjIUSP*e{Mv|6 z`%SJ;yojaH93$~ytWmgb{J1&;%3oxWeuXWv1h8z zT=KMxIjvz%$6zA4JXlaqVz)eSL?Nj>`fgRJ`csXy6;IX^h_C&62`w9RXSYT&e4O!G zu%I1!T_KSGmrgE@UX{EE(hUDUOiBPH%mU8M&Sz$0qih=eur4(Rd+!@8vbQv zmy*BSuS#lrN$R|YX*^Sw-EezQ-_;MZn0xdJ*uIkwIqtwN17P0rhjDpne2VYpE`V>? zwe@UR%_+XECy^9ISobZSxbB`WJLuJ>^XUeDJI%@OE*W4ihMu5SlUwb)bvLoM^eo`( zQUL7u9NBzL^S09}z3#9RfG$SZegY7q83CZBeI?{1+@5Chh{?~|74U3PP9Fa_V{r7;v>!z4qiWK6sWf{u)^r^KfnM_PK-1Ol7uUdxOVX=5jRS${$K~UfQ&gz= zfXs`p@6eD`h{}O2L!-5}M&%gyKh&#>m=W5=;YxeWY}RMM|3gD}VDoDz-Mx8(zLzN! zJWwb8Xt(|I%L}by>x`dAawK(@fK1n)fJHwvLW28AqVeN!mjZb7^1mnmJj{UiE#yjZ zlE8e7*7msWv@lfvJ?8%1cq}K8vO)8h7N-r9A0<@$Dr9%XpOg)XLZc1rIg@`+6wO-H zWXksnz&-dDXl9EWPJ*U~AIypIni8@xIE%w$rD0nd|BN5 z0Jh(2wn{##<8RCTa#fBZ#x@EJsm@1c4S;%UvtU2f8?WtQ++ekWNEnZd5mqyv{sBIb z&(lMn?X-uI9uG)vAQ*h+hJ8w6REY~r5*TD!^&p@oym(E2B#nWZVoh6Eok`Th+raft zgDO*k5H+E0i>LET(kbj>cM7B%u3V_{2EeFupEgEzZJAIvcuy5dCLpXlF=Eoz+Q)F1 ztCqcjzDb`mZ+o3%{RDzK0GFIydvov(f_^l^i<*+q$!50o9x?jPzZaK@`85PB>Af*I z0KBrXVu$LcxnE$$v9m6vRJGL$K!-v-n&S!vznbbKLnFwX}CDP1c2uO1o41 z+f$mOGp+vh@)p0w4aUt2o)Z#}uKm`lvOtEoeRuOOt*5?~WS-13-vas&n<9=3Ap7>?K;fQuW2#52ve zT?L_(>V){_!c(#HC$SiX9V>9GhOkI9Oaho-U!iKbS$U**1%{=6hJgP@kN*?`L<$oL zCRz`Th+_n_N&x@r*l;i@k&mV_0Q4e@He{fPE#pT!S?Dpb``ND?bi*az44h*f<|1rq zh^$(*1%nSVMjT*dmL!B<$0G-GVgLCuG4J85>u5Oy*PnKm8Zv$bHXY4xC2M=$S3*t) zSg16x{~x~IIxOn0YXcQT5fKoO7DVapPNlm$m6Gl*5fSO`mTnljQ5pu3?vn2AI(zha zzsL7G=X}>Se~dcL?ETw&uf5j2?scyxNq}i0X)QA2z3%vk>47JXw+((P|9Ze9Prsff zM(2AD?I3{H`i%C%Dj8LKEzqqn2=#A=VIBR;kjxIH6$}JD)Nw(BJp5-*r7T&l2`R9!TL z$<%a}U3dS!jl@F*t&Zi|eXE#mqIfiKqrnv;pgs8?;ybuqmvMY}nOaxX)|?y~z6?$k zsxfSg6(=D@>`z;_Bnkf#6PA5h`}?^VVo72O@% z`@gXU`FPlep(g&G|1&Jg9t1mFeFE0^;UQf&TUOBOew06+D~HeHEgGqW;uS-cmF(S* zP_o}vzwk)zNd2V~kn@dz3~s(NRpr zEfNJ@R$*}?uhqZ6)-sUt_bSqSZ8NHYxwFJ%@J78BF~wRehSTiO(iFF;%q0sdK(hEI zgq`fviNg(JELRH)%pqQYb}xO=v_koXdE^IjXaC47EK5NEoe24(&p;s_F}=A1CL zasow=SwHK6R-BqvydnwKm|DbTA%0I~y?0PtyPAF-pRQzz!P~#$9iLZ1tw+M_7L+uZ z-WK%3CL_ILdk~`<@g{EbUXpN1m zV>9kac8j{7JyX~}dgUmTa01C&rFDMY-~a!)jsmw6-;bNTZbm0N^-)UtVpSqzc^G14 z2w2F3^lsG(Dw(vn#yja7Xv0so`nrQRs&qQ88?U#6yq}i9HiRG(xggraI}bZ9Rxq?> zr099|!Dl6n<$<)tjq<87dQ_G;+vHWMtkMM$hXgbdZ80*2nkf>SHwE;_NQz)*r@I>e z9rHd@f;Vy;qRp}_u&cJLkiR{zXttP3e!24I?c-mc_3C;>O{Wa&uKDtHr=pxEtL>+8 zrcvX5@9r6-pZBz81{k}<;7P-u`B>sjOV7=HS&Cu`<8L&B-X=YOp;h$%4y*YA-^|2$dKJIM;}{6wec6;z}ZS2fRO_KD&B zoJGF$Wp0VZWTLyhpw;ESI|ChH3%{I1xro5SuY6=dQze6KzV4ILr`dgJXP(u2X{P9= zl~a`0Ncz@Y2D?VpdXkR0$o{Z2)bKPKvm!_Gn^gHUSh9uu>l7u0^!pQ(SY8IRTaK8kqLeC)+)I+|4QtM_ z^N58RNuRH>#Al_Af!uJXX|QSDk}>8FmO_~9eI1gkN-Nc;dfV-FR4tpM! zK8ibi@xhclb*prVpf7P$L(i_`C6%66Mdges2$vPb3_8hiN?NO9#y!h->?B~ugUs(9 zpT1OY!YU^)qZk+y@?OQkmVDB^7F6cEnwHw?Y8{t-iSMf^jy9!jv4WsdYc3HewiJIq zt#T4ZJLSF!rea+cUH&T%j(qvIQ5G}m9MKL<;HOTtXz!PSbi$ z48L8Qb?PTMWIubHGZgnM(MSRINlI)4+>hz%$)czq5ybVy(DzyA#x5ThI*%;6YX9|3 za2Vo&t<{#)zYei z_2O*2?#+MM;0F@KMM)3C2!FU=<}V9$pj5a0H)E%p&Zo=@--5v(&gu60A~NE2*90-? zf1zF5xbXC|BQZ4wZhe`5uY9yJ`1|PCu}EsH=Nx}6!2bfu1e-Ad?lq5obnpL|30VvP zaLG=f_A~#-hc0Bno#Y_~z5n-p_rKnQL9w>Rj<5{a{%ySe*N2vQe4u$G+AfQJx4S^F z1j+ayP!a{7RbLkY*Dj;+Q^!Z<*9qy;7XKlXb!%oy%aea+9xQt+q*12tGl|l43?L#T zoP(PD8sm%w$M&u67Rp3w}q239el_bKtcl=BLfUi#=z|kvb}y+t$%MJ zutON(Q`dTcst5Go@^pvB_R@6b7JiEo?LQ@3u!kZ46^XR4CRR~WH-Nn3+y61Ea6x{U z(eZ%qGyWS;gYEDCdOcL{{a;J;-+x@jgu~ZALZVIj<5T?)e)->T@b#(g{~v2b8ZHMz zGAYU|{!{q#--GyPOpmMJs+IELbm8`~$PU#)2#AXM0J2BnPv{g6lJcUR!FUF4KWl9bfLe+iDGQ4dfHT!9fcVT= zEU4`l<96PqCFFM4?|U^IV?urOn-msJ3{R0VqG{D&DZq7etYbL9x7EWj4`s~+kRd*^ z=jDoGwe;sFFPNscd6vU-?1p8o#0OI8R$A{9Tp2}d^s>Yl+-7SrXyXwF}i&80M z?}a85zPiHc;D!)8gY)u?03Ohj;s6>V2O?PtpsGTJvGf&Hhg4Q6)xl1zZw-ZGh8Tc% zkv&lqYdtO#;yo|r%irZRo{SAL8+1xGwUkJhFO0~D$)8 zF}j6mNYq_!7G{I8nYvx!T-lUmSP_*#JCN9k#Ah=*w*(N5DiGs9O8{|Pi#0$JA&y(B zY~T?=HJrbB5{aUPh zB2V0`PqMVejqN(H+%K=nfA#jMGFC=d@l8Wr*Q(-lo4BX?-_-Gf$8VJ8iQj%o7wBlQ zC@89c>z>}AP3H4PBprZo%|Eyu*5zTGwQTe8k_Tg5b}PLS01nK0+xFS?#nGVH3pN8> zP_D8tu(<$}w1}osB7fTe%4|n6Fkkgoz?13l!*~%O#V?V1AU*Ol?EnK^lE1}rT2s`- zZ?4`#!y50RP2=*(4E4oBI}OMiKW4ij3Ck>zAT!e%;aTP2CBbtpV!ZSbnW&6*X43F@ zLB)sapWtx?yc)G#S7P9eP64MV3SjLV5J$??#h-Bd?@;!HR!(=O`+)w-18bo)7&kq* zDuiCIS-S*e_wjgL%4W%cY)Tj}SRH1-4)RXZu?3J=iBXl=ZW31~RtzfNvHT>UiG77Y zn`_IbAKM19x^%2e*26%vXO{4834Le+&CIjK39rSq{1mQ6S z+`2pv+-c+mu%u|*cZ|VUETa_ajo>>ndibE?b=*z;O^q)J(ENQ?+L3wBzk#O^L+J6iA zuv9sYn(l>%P2*-FkBK=nJFU)vF)kE6cX+Y2WXWhN-b$VWE5iyu_OVrz;5UpfN@b#}bpZ7Ql zVakH3Ko|==>m&)~d;`NafJP4HrW0OwkNe-8P>NlTUfBCJ^yF8@s!8{^V6>Q(UC$38uq&{Bt!& zpkYt9>$qGhF6X1?`c|A?>aR04t8GfggEQonotm)Nd?k9e#>+E-&KEorytghK32a-B zH5yJIUR>nb2cKcj_; z{qihXH4(i4xSIdBV^yj-%S6CDDMx16)U z-M@=uM77I1<2eq;ypksMpeup@h8d4}S-I^QbGtCSh)*jpAi=k;J5}SoiEQS}@dlwx zvrZc(w%oYXuUN64WZDRygA~Ow*axRftr-hd zZT1xw4aLbmnfNkwl+vi>ywn$Yx@A4<_>_x71HT*?N9;G)uL^{|#itF7x}TqjstLO$ zX?Il^De81?2nQUqsOjY;DRRiDplUR`V2V%OBj_vxoV5z%Kx4nhSkR;kFr&nHwS6;@ zk8u{DzU##2-0}i&8J7?H7&tPJCFAJn7(1OdhD7$UiMHaj?E;jG)CWfhm<_++Bc=`} zT}paVhTJs{Kl`!fjY%#Y4la@*mzT7OHwawhC~)_;JU}uc)vejBz{8DOA~yFxgO4r84Zo&W z2~AZFC5~K<&!j0zmb<6a`P%~=mraocYyYc^hq7qii-704kgsMIdPBL^i(F$+!^R(; zn}PajS_qGg(cOKgDh`jQQyGZpwG_EExoxg&S!a5DEzjJ;(RInlR|2%mHVKNCEYk}` zx}}p-6*tOg2Bkn^o_w!%DU{`8M37KpY;19eZW;*V@F+eM4dxtHLX(p-NO8Lyn;ovz zOm(xl96vrbjdP_@(_I5;`oV^aRnJD6l8LrL#q3}p&LxxTR_1nDSJ$Iev-@!GuFb%*J$4bXbf~a z@oLcrR&`#TtPZsFCd-8kCoOR*7+8-L6`0Mt06>ZGwW_n5C~z_ zGK=n9Pi5x^wt7D=+okz1UDp*#>y4o~q!Vx|Xz^x6!4ECv$K`W1d(}nnyC^`O&WO#j zuG!yn9Byn-{RruFv_t6=%-QXZSO_s2uC-l)ngPo;TCwVIGvgT12ozcHbN5lS+G$%r zqI3x*@aY5bhHXfYM!`3g`^IU8V+V3rD&9?Hr3*@|?I=9^t@lGX=y{zZEb>0p-g20a zK@ICVt(rb@SWGN%+?RW&QZ!1kh3Ktk^WebdpiUZ4!C~(3CLNo?>OdL;td^lnz1q>` zX{NODJEFQzD<7F7>ZSF+#ymjAKBsI;O9K%1&|2#KoBp@+{mffcW@^#qqb&BP$ts7f zH<`p7oX3%xp)27BW$5gJiZPfidk`CgP9(bu=;s7x7;0lgFrg^fwwfTy#@r?x~K z^RD|4>L=YA<=S$CziEAlP~p`-nleIA9h~eg>z=^pUKLDGqeHL2K`Yghiil=anaKJs z+-=0|<356IzQ&OmvyusAs3epmu1veu@N1oHCA0bJ=~mf56G_|X%})AU;J{g%L(*L8rc0WMRV}n7)pA43 z83u11!s4D(d`}n3J)D-ifYz%PdF+;p#qJ)$QtW8z-HP>Vd=bw(S8$v!95#;fyRGD{ zQ`jx9mWyL)n#(k+9a27H1;c(bUaCf;(ItiJ$s3|iQClvHkRB_FLQKDDsT>AXwJa;i z*^%rs#?tp&Jw2PnTu;gvSM!~7h^uKLJ@?-{^3k$xV)Ga{r~xVatx|&y{2Tpv6nxUL zQoXcd?YDCipf;r6^wHV8Ih!nEe3SInd?0~LHegPXE94SRYdADe1GXzg%QbY-AL+hsReP!vwj zZm`<&Fo0w3Txd^h8X)+Ca+^Vt+`qo0KuvCIX4$pdkgsV?lFNhe5$$F9u~Wle&X$+z zG>8}!m&DCF6%quzUugAa9cg=O_w7N0X?&b9Ijx}KH{PiL31U>PWgX8&Rf*kN18)Oy ze{`4ip9w(T#NbY+jmctqH36wwK2J2{@UqAL0SI!dEGEiSP`Z~KfAuykM+Wh9cPrSR zaCkHZ+~4NNqj{?#dCH404aif^p21DrygU+jo=rJ9r0#u0kXtk5rCO}1qNlw>)^c~- zpagUn^uLS;xU;`OCC(pD{VbbrmW&ZguVZ$)_<&n+UqrF~GToGNHLZ~08}-{+R*e+) z2~n%DC_bR6vcYVhA`nbc>bP#`eMOZvU5Z5`2VjWQ1Stf2U1%!b^rw8Ov&Lz5B^_+A zDGtO{o$S|(GephDXiHIF%NSNRR{~#?o)DTnO{*Smq$SUwVZW@XvA5Y@zvYr)Mc$N9 zB>QY^U}z^O2$@@EcH|~F{5ZhqH3lUy{GYwC{(@Kg>SQZUS#2>SAbn#$+!9dRFJ6j> zw6lIMDu_>i+&SG5gwqG27b=Fv{Wss8G8Q&XZt;UkOJrjqV=n#Qr|(%3f1AFS`WO^U zuwO(`$g%ecyXM$I1t^Euo2dgqIY^hw@jJp)%y`CDx5Ke~r7mv+WI1H%a6jAcLfAx4 zk#Xpy-M2xOzhVP1cmbAy$6EpfS$dcr8P_UM3e01|9AJLKID#DXZ43pak=nx zUy5_6_yjF43af<=tEsk#YVll_ZiQpNMIL}7m>ii{sDFJW@^p|gD7umft1wehZM0TP z-D*`h!KUrz)?Mi}0>C+UGX(?rSIuVLt-=qc!>Wlcnektc%PlEnwJjRChY7EKX0S-R z<9`+OB4W{VJGIEyvy+Z4H^7NEKqcbUq41pP!le^9U$7nzj|7?ACfg;|kiA(Y3WIB< zTX}2MU=*tL8+?w_aszDK`!vBNVnmZe(q$*JEvX~^r|;>d;)+0VR2ntGnDyZKM$CgX zG%DFx?X_>p!548wPr8)XtT4ob1kW=#CdB2ftIBnmOsS4JP(reG_hTI1HH~8eB{9M$ z+>-}gFRX)0esj1gf6E6b7}Am#wAJ^Th7~MB2Tk_m#V}Y$I)V!>B0w*<+&tldrClYuuE_FhC>=brXh1}*dbn4 zg#9d8!|Ns$p{Mb^pjK-rPTpuahv1Bj=^D_j#Ev4TVexet=j`f$K-{J_rPgvU&FKh3 z|Jde{HBZY?-Jsv#Bgf(G0k{zv&%s}hk(Tj%IGflSwv3^R+qwJ-lVEGqf6 zh8fe?O zw{!!D!E^hEkW_yiUB6+XS#P;%*eoav9>m1__}W#h+pk!w1|LlYH`4O^8Y|-y^G3Rq z;}u&>{Cln?<%nIu><1=Molz>jt?DQ3^{vvY9|Ee=IVV2d5@!Uf2@;P!uE=toU}`XWhs?!=j=9g_L7a0hjZZbS9;R^@>$Ti8zdHlc zrr(&2p{|hWT}ReRNsFOj^QF8~v&G~uyp@||`)ZnTV90}@X*Hr0(cB`ep#?}NA69h?&<!wYtjG8(vy1fvf-mJz*;^KwZp4Q9DhEP_P{#yjyh}()bn`l|8FhS_PJte(EC!qf`gxSf~2 z4^y&;hEim7lWEDg(VT{AC^Zgw`miu44H1e$9ZyEpf;zoMuUfGUw!eMi5vRX_X8V3q zNASNdT{{ZA$w8N!&Tseh=-)zAInnfyX4&Ea?h_lNDK_uR{gn=L94F~TcG52quU9Ft z+?8bU-cr3Lv2fT?);ho=CueQdO9xafu&$Da{&Y_LD77}26IvdoAI@j9LCfm~TRLL7S?e!>(+0@hu`zw7z-=(M=JV=`R+kq-QmZTNR2Kg zi|}(m4oQ^2`hH@=dqUmK)alR_DuMCS_?*)$oGpqc?=X#oK|6MOdL@yz`zx_wtK;2> zTpe#nc1-R#9XV;Mo^BWlrn*XDwdb26{i~8j{knZOWj^Wp?6jez+c*ZjDd~czDriyG zerLHnP9Y+z84Qc3p?ppzEOl*26qUUZIP^(4)3^-em(*J9B&ROxCNA5TNU7sl5?FFE zMlHH$yE1&9oCgk_`rh-$8*#4j@-Gvb$?b_EP5q5r()t@5*4heHtOQNPFd)%pX zm^E2ei8|I9R*5NM$ul+rTx5}O<2NiyG;8i3odvzJ$fO|TBmrjAF*qu7Ro4B9MFU0( z#=!MmWzgo|b=e3dEHB!}kp>i^7Kg1d8NTf1xWzVq#EM#3;KINowek=P4Dp`uhDen( z0*SL1DgXrEIj}_UdIn?6pcw%t=$%!uPLij`&D9GMiqr|sCehr2vps=D`sF_M&cyd$ zOY_P}nxB$SJSO)@^p{6j8p%8@ZHbeRI>A;TDQ-O7-23*fe5)tff1(!(s1uT{*=Zsh z`ncrpC|*I8l|QPbqQqdu>~ZTVUjUgE&vM61hvAv^%IaC=*g7uVdg#|5dJ4lCItF3* zFMb3_kw4D#=9vs@ONqbnc4@3gJMx2UBWmh`*zpVVg4z_&Vuh10SWUj$jyG6#SN)V>SxaQ7dk%f zu&h%0C-ihwo$ssd3-$(cHne%F;0r-!?&9d8 z=bdg!d~h(cVzq@zhX>gftXqTHfKph{kPzedAwYx_x_O_Ay8g+ zWJGKc4a;LaMhBa9*m0gtScvjx$+Bn|6k0^7Rb3>N-vZd;v5=L5o=%j$S6f)lO`~MK zC5__hWQHcm$z+i_I7D&Db%)xP*k`=IPe?}!GK?l62ww;LnN`NtPQpB|o2Hdx7O(Jm z?>(CNCr!*H>g6-@l^SRpgG;Un0(T}Chd~uLOimk@wGQhxb)ZIna=Y!&Z*5CzW60=% ztL1bm{akf1#CF*1VzbYl?Zdfvi`58Vustwol8o8xv+eSxWiziHb*kwe*V7}9qCop% z7I^hRQj z)xA3Qn*RWcYkR+0F-LkuxS#nq58}_1C7=03=GCmKhFQ#Bl1^5dswj4Y>+#*6Kxt5i!jIrlk|J~|366pzBbtREg3GOIj>WksARqN^^{KA z9N8dV)7$sIfX+D`ru)8#xsTd7Zt<(%B}_;!x{pDR7)OU!WvY^|A3G{ltc3(e`@DwL z%pqN)<#H3t?&r6#_-oJ8IuZIEHmQ=0CecuVTcv!ZnoK>kF5a8q2)h$Dn(T2S`;e~p zDN_w(yhM)|j`B)$(nbuNPN$_RnXTuVMf4qG>vQ>%nyykGA|vDyJ{OrGW*jDp!%CGwp@lyPjQa#+EbfGK^dsx9|4 z6MFg$k?)n`*6z#uh(v`IECPo7Oa2PjrV}+yqlFIn&uy2tAg)K;3EVsOOI(4Crf$h- z{G7_Vp4r6~Q;njh(|fYp>>Ke0h~3FtB+D0TTRrQ8jK^-w786zE1CG4U-WPoy4|Uhh z?(CY&@nF6qm$?)iD@q{Hs?=?g-3*wLJDs~Src!G*uC`-5XSbRWIaT#N+{3&CENV+( zU~5Y1@#_nFAV%Hc8g zoyT&1mWNa3J4US^+Kf5^^YO3Dn<`|C|K(^1z9o~7JW)-KayahRteOT*P*Ro$1m>bu zH<6~lc0^F6>DuTudj#f-HL(%tNyH8a+hLP69FTVKuC?>MPA~pJ8 zIeDm_z;fo!7G1(7IuW~x67)$Tom}o#(8!j7pxqgN9X^}P5X^TC#yR_vxjaGfX zLVAQ_vrui|ph@l^;I`T_$uJ0CuZ74i;QGedJAuW$HowO3070YVVJB!mf@feJMXm$& zl76pJWyXSe=1|e8lEQw%;)9sW=20K&|IoPqa?Itx7|%*3s_v4*doB!Ov#7qaWRqX2 z`z<%^RHynhk#&PwjOe6(B(J#wn!>@Ym1<5>pzE6Af{e7ZTX`nKyt>w9=!;*rV>L28X1yD6XjuBUIU}g|0WIp#Gu@8 zGv{_QL_G0IVO~*k3{|I<8F9U*mz*65bf_r^d2}<}?!0B#3eyy=`z6Xr`%GSO1x3y# zBh>8K_9W0)yCV;pdpHHIesB6id#ED`A0gG=cj+M5G44lWd8mrZcTnSTVaS%;Op%*< z8FKK+a&*(>LX5(1DhE^eZFNr|)>eYt6uKBzzVu`-ezU$_Lh)z$B=Wqs*q^m=l`MgI zc{tf5#IW=d*&3>t4cTu0BK5$4{?HrmEIPJM+IWD-*jPtq)#bf8H2!X)ZP_gVscf56 zJ&L@EK~1L$y7sX%SN5X)dZ+q|%x=c^MQ;!TD{FZb1|es`+`|)XktvCgMR{W!1|o)R z$DtCLyIr}FoU}b7juNlJxi&;cFZs~1@TWipEL;Jf8?Mvtulj7tKm^J9xx@mZfC~ht z$rFns5xjTxs>%kN#-j;~=l5oL6uxIkif!STDdl(-h|Yw`*lrddj7S!Ekdv4wK(-=Z z!>-F;1G2)Xpt?WZ2Ir4|!Hs}{uMc|XmM^PmHb*(|ufIgmvI&;eWfR6{ z=M|LYv<=rY9LeMF%ya+w8m&+Iq>_;BkUlDI`#>M+_l=_z^H=#-9hdd*jD!G*MU50f zb(Jj`pU`l<`DLO~Jf+<9am}o$__ldL^3<)R0Pzri!(+h{)ZpxM(vyuA){I9IG&(O3zPrVlsu;E8$A6Kms$zwY+%)RbX8UgKVq=rnKs{ltc8Yv>h`EK<6_wV81iSNM^6C)Yc#!~Amm&{1C6V3`@-oB^hmsvJBT2)ZK(0s0L zRqs9^{1*3G5FbM>(C7khqku7-K8ES7@5k>V9^<6cdKn?g>m@3TyjhPPgrY=R$gi{~ zNk4eBRJ!^=tI3Ul+uoSvh9bCyBDC~r3$#lw%D}Vb>nm)t7L&~Ux0xKJRXrCzSHoqC z1m71lQ`Ik)KO821_4B7Eh|sFMR(83@eyUsWQd!GoGbgJ(Aii#zkYgEy0hy=B*(c6o zsr9oeiz2T&Bfoy&{pr5X3X4-zfs(Hc7JcB^=+~705$tW?(=1aPQFvK~?xzqB4~N zhn)N0eU;MA>tFkhXDx;r)a|0beT2{Mp+sq(_Qh;Gu9iV_ByM+)H{omQ979Nf$q$PN z0tYf(F0tAA{@A(VMhUfx!(vlZ8mm|3jVJ`oUw8Rdd*ds*!sv42CrFfwr`lf4d_$O` zeAmlv#j9)qJ$dNoAD8p<=POooV>I+YeZ2hLE{Jaj292&P7X6dZ5^~fu8?hb!@h7df ztTyuSHx(9)ma~_cGs_DB6gA=s5)$QhPo86Zz#6)~=u>QI?()klc%jqmk!buy=iOv? zem3i(M(bzexewA(&ofmiq&o^3>oJG$;%5Ns5S1S(@D)$>yp@{Zpn ztHw0nHCImLE?Z$%}y|P~qCs$l|-z$yn8X5V<)ewMK z_4tW{p0p(zK6X7gZwqe`lIPPdck|~OF5M=%)HWiF?RX`u`|0?Gq^;QuzOG!=&lM#4 z7BGj%Ifc#f3#cqrB{mdrYX1{*nVs=qq_0izEP6IN`ikzqv`wZ9V!69=}fy$(~jzCYG3^4mwx}Jcwhw00v|pg z61;abe4hxu{`cu6)~A7Y{ROur`*m$#FH41H-Y!?X8>|6rBmDEK!Td0{>6Gp zYJ&_n`dm4tZe{f;JUoY>Ae(^7-Yh{aS>i`9z7GSK@=XF2xcZrynVv-Ax+sjCobiV% zeH6b6JB_YlL7TbU_I6=(tCw3j>ID#s`A0DOJ|il4a?5MS*=c#^Lkt^l8RkE~he>jy zUVJRa!Z%cqa!d4M&6v3^wE(IS%As4hnqB}U=-Mpd#{r3lcLoL(U)r8SKa#4kC?A25 z_K(WG=7f#ig$sPua5iokXsnkBn2YXDK8F(WC;~p-VUYmfSf+y^4FkMCbtARSA1xQ; zv`sFfDHYyWpkluUb>o@YQgK7y{g92OW9hVv&CHz?Ky7j&w<9&3X5|-{w38bkRy|iy zF$$IBg!}hd6JVAV&Y5LVbO>J-c2?rJU7=k z-(4p5!ThsKdgKd{aItV@b~%!_xAZ~1q~wf@;n&yK%qyliuV25OGR-jnJgt0K;rXH6 z(#Ni2(4cAPXNFL|e8w9MT$>gS%PC<61%=Q7&|Zg94XEti0aEia zt}9UWajDICLKhWNUhlXiiKeb9=|Y=4;jXHZQ^mmzu{{SIKvFMu7Gr2SC^-LIs|Rae zSY*{XC60aguZPl_gmey>D;N?`amy-5+0-aBECA1Fr=;fc@OpXe$2_7d))IIO>!6rC zH2(wFy5N7-{RO#A+rtT>Lu zm!cx{?tuJ~55!_vy{=s}8=UC)uC^hD%oJ>hZyTZVRn&mw@|~ zr4T&;%j;hPEh84bR8aP)01~L{KrE{#I&Bzvm%|^OQEEHj3J8n+1e6tF(5R+YIi$-T zGj&n*&U^G&Q6Ll)H!grkb2!%Pw^NZtrN@AW^?u6kh^*Ao9$b4ktNedZwnok5b2seGbZ$(#advf*8~!qR~OY&y;E zJu(27W{SP=jOK(U@xMjgG+lW9`4*g7t6=$ntuZrZ|gwdEemig#STqpYMFMX zYr@XeW~QwgWLC?WVoQgIhSovmZ^$=hDH#G6i;_=8MKw}ss-zLek@oAujY#0LhyxWw zkaiA<*w(2BZAiI2&KVCB?*IXM7&MtQBG?%KJ17fwKFT#!O9(lv=cKO}<>9s?`p%YR zYAV}P3oV3K3CZhHRo-PsVX^=sG6i6f3=F;RyC`6$sRP}rawMY2(T@xm)K5!B8*Tg5 zwR(iP{x#$-m~aj2#iEzFG;Uc6JflY{J<7m7*d^$I(*zv=x=tTZ3AssIaDY7u+Xdtt z>0vl~Mod)gKX=YKt`7loE97;|@Hbz?m58{luJ-osr}wYU_TnCn11`1r5|w)kpe7U# zscV3;ag7!rs=(|3hxpE9K$?BX?h@pyvZPO`Q`pS0n}Fb=T~8ow<+=UgvV`gJGD?vh zPx&|lfxoMUK$pvkJo4p4rNyzucu5MY8$YNh4?RdYq2cG(#fCLbY)>tH&i(;&t+3aS z4CmISkxbTO-lO+Oc@1QvqUbWInZ0&q$nUeSyxgs$N=J zBXy*j0AxuL*y!wBcAmqs@xoX}L^cdg*p+hc(fO<2(~XFUd1V=u2mj!)xSY(XjS-sq z={qB9fQqj5WuH8cixEN;788?-eXbU>bk8&ma2k6(tokyZz{vDVjZWJW5uCe){CJk6 z*)c}~t8Q}2bOIb4zs?(Pz=xD$Rg;@x6~3Xml+){UaQWNqt#2Jw2nS#cWqjh( zbM;0;o$UcP*50Y_H@T1 z`QrR~B|0&YI_!F_Jpj`I=qL7_$|SP(9QRRmX|L3lJs)c&%zxwecqlXOs^i6+V6Z`f zoy#XCgHCZY^-zf$3Agz}Oe#B-h)7K; zB#Wt;gllVn-I)g%%ZEH9FI+K~zy;M86BO^eo1hl{ClXtjM1ZdtZUV%4u7XmnnowAg z`X$gqSO-Qd?(KaH42<_PwYGLjbS2#_)kmc%ja)eH%@8)3BEF_C&^r zK)-|Z7+OZn>J7Ad`r3gUI`3O)$mU!})XZaAX@l_Fzt~rW_xidtes_Y218#ad+U?m5 zvv>EY2{b2v=A9`!W_tyAity&b%>cmZ;HOUim7h$|) zX+?p`*LnurCf$gIt3x}wx$VF%{b=PI!&DpwUDoFf1<-YVRo9ZT>PR0uFxHdbjy%TT ze;j$;Fj_{NPvc>5_kp{%?*ac~PtG$eqh{08Fj;FO&DbqVg=f83L&HAQ47H#)I6J~& zu59a!8v43IBZB9m;3D5`Z&lWs7z;QS!S|Xu5+e?v9-~(*T0~{WZ3b;5{zV|=3nM_Y%-e_QnM6T$E-azr@^B{{MP}q zCSKTbD~!1{%@NbA@|{cMsF)sK%MEFoRV*r5tw&HN z6fh6Vx+(a;mJa$En6aoX%hQ`~v)x3dC67^Fr9Y1B@1U#s$AhK&;HiUHGu_SGn5mU} zKJQ&eV5QrMnb|~HM&^4UJp=1F!c)Sa*DTHBw4DybHF*w;WPw_dZh=DPXO$|8II#6W zEzVZ#j#I(IzdiE&DNs2;MzEUc=`yZJ2ZWk}fjFELFqC5u+Az?pIvpIhAt1oAzSnY> z16uL64~nX+=6(9dm4+R#GAcNptzvX^bkN9`Q4>@L&}rAoZxp#}shZU`sjE$}T4lzkIx?%H!O>u@)-}J*u(h)Q6?RyFDAFkk> zWWn8k_!B(zO6r8r#l=PC9BI6FOYec8AWO`Fh`hXfEbk9}H2F1mQ>^}}O3Ue3a5ZPC zG-PD9vODRz?@a+vgt*0QO%U2m)XGB z`c=f}%;*$8A%J*Pj7g=^*81$_1!R-|1h314a7%2vleKq1-&wpri6f2u8gXXprA~h1}D?FP!fW$NlfQSh>^p_5Sdu> z=BKXyw|JeQgi;ojE4Oy<_WZc)SL0(tUI@%7(-M5%YuMwj8(9O&$dNrDz>)<5RawB_ ze`Ck`E1!cj>9H8xzXrnv9ZqjdOQUP7G#M7M00wZ_1o}sB4~}6bHUMGdvbi_@)avhE z=ADmQqflbH^2$nV;AQB?SWW*Q7{KL~&yzTD95?X%*i1he3T4e{MaFT$)Gwc5VhUXf z&Rt8969`ZCE%bBDM@tMgm#tCj*}MG{60|>F1mV&Kpmb(~ zfT@?69eT(`AOf_$%^xEpXO4_0bK5N3^8^CuL;AgI7TWgtq%r@|0x)bV<_D_h!9WU) z6s7jkW0cFibKhj4iC z=b9Q=XCQ;oDCGsrG1{T!?V=llnPO5x90EM-k842TBRk7deq;diU%S*0-f2|pBBAU@ zcSiHXaJZUTIb0vspC|VNW{Ox`ldW-@MoW%{H?JZFXH@!e{+OUCBh&)i^e4Z)C}QW| zUX{IMY0rC55IWQl+w_t%;h-`y0Cwa4^V(O2cN)`jU3tbsU1or= zm7iR6EFL3kej2R8tYTz*Sm^=oS*MgV)%mdC)e?4w;Lrls9BP)x?1755=ObY_Yk#gP zSD{v;`?L%JZ_yNQYazTTAz|BCS3tNG{m!7T3v6#g&60t|2b|@Ig|QPheE~w4`{c@v z0ayff7jf@ofwnLlA+L+PTk@f?K1_^9`^PI}y4z%4S4HY9+HpeDU+n?R?Qrh*?1Sk} zFOBwmz`>D1(`APOPOqLDp^Y^CEtzV}gi!qOQka13_QfTzJ3Zz<)M9!h>96bDP6~=T z#CEPlUFvP5l>fz0e0VAJO`2l79jwPo2oJ(`<@ipbM>2{5(w3rnvY-rRPn~edETeWJ z(li=+G#s-hCTg!HK#@EEvsSCLUE5P}Yw45m5dahfH*~l==nl|oT!Y-x)1!E>)L-L* z^sGx_B_w-7NhRiMW~LF7`wzP(DmyH8%i;+w*7N*8vR%M&cSfVrbtR-}7c>W!eig!= zFhpJUQu#=6^l-*@CjdUsB}FB!e3lgSorwVasEJ)|3^gE zLo-?DSx1~3P_s0o<+HS8*IWR7NsLHQvJdr9XZ?4^aHTI7%-v`R(o{htauG+T#jfR+ zrJFlSM0~6o(%#Q?`Tg2qHM) zi|zeS!Ow^b*>~G)T1DziQnJrwKZJa2%u0*fIXcV-g?WAb_a9?XlXA?M%{Mm%oA)*- z*$uh>Oy#uw=n_R`H4mDnOY7!7M+;i|Zl``<4by#jtUx8`eC~SDJ~Y4gN%-`+ zQqlL=`&uHcCLw;{suy}b9J^F>7!fi66a1g6nev=$i!+$@0=2N3vB7$MJ6s)|okJhs z?i)(0&oKsb+-CtXLpO*Sc>ggEXgFSYw;-2xq|#ZoV|5am0i0%EQ(49ihwAo zAky7k3P?B7(kb1IfP{o}vqic~I;FcC>D;7r!*6ZSSKo8a_s=(mkK-ALvhP@H&1+tB z&TG2kj6NQeBkz0`#nl|#?x7AhJH&VG3m;N`A23WR6Y*j>7mD+l2)R3<>&o6Z=*CkN zTlMpaRbR~Wyqh8ov>qU<=7R70jJbmS{X_LlcxytMI=7t;H{OC={H&j^KkIhKv&mjA zz`W{}FO?@ZK5MtH$(oxN?DOm9D^GvzBrPnALdorEQMinNT)~wgdB1--NE#evSSW^y z_Eqs^z#98X=4k_d^L1qb$?7`6&r_{x7{p}DxO3Z<1OP`=yn)yA4%-*QyV0@iW9t@o z?*;ujrF(x1yx)nE^(PmZ+l6a2+4=f5TcTin$T`TzjwhmkA|pZq^!4nYWWu=Mcd-_E zG~JR5XnKpv-D+KYm^B+wnak;GHE|alNp}(4>jNLvAGP2lMIC*R-=EGnA^MwvM9t^>E$N@QZy=aE zuiotnJ@=(albKpISNfU9iZ>g=@84&Y;J=nRj_mQ8(WBR&cXi$Q^T)Fv;ESHVcPOTO zdHq8?Hej4RgeE${c6gXj4*-G7(;Q5*7Y&}<>^{9HSS|NJZA*H}o_QJ%=F>k_Jd z3tn!!m-61b?6E91kC=QZ+Ei7E)cx@Nc*dg%W{($e7*DZ<{jkD{xb9&x$SakadEEn_ z%#;W`aD29S2L`^fDU6S>YNWmu5mYBxN2U^m_Em|xscy?H$FL2a%|$SANCH+HvEg~r z{CYt3m|v0lr5+#Xs1!5y>1LE?bXXIb*kxURs7+qYdf0`3LAjlEr2piO3G-q{Rt~YcCI+ZD$+7RwY?ER#sI8?!{_=T*RS$W1_@jaWT)1g6T zoNwxE;+9v7#>ZsC1~NJokE8;r%f4vv-Xe|{{~AL>nUcV8#4!<@kP)2;AoB}?>uu+z zPyO7}J!Wee+0odbUn&(a9z^o0a#ljbvhr!6hAZ^LR=+S4Pei>3XSrMqc4e&$Tff)r zTBvlZ4aZAaz&ZrqZr+Yt@RpwLZlzwQeiP^YBs@Iq-{+sYjBePql3WhyDlJ^*Ps#{; zIPag`8C*U0_&hYXpddH)Idbmw{&9aNIUActZFwVh%yBi}|v&F*JSQQ)QTf z8Dkm;LH^qpXfMK@*}QwUxFj*LBa&GE7AtGVRt%enxQi8b-GLDqz(_&D@7A}HXQ zM6xQfk(NHOWuP_vKB#aI$`=oxN~hhDH&jc!Jh7(2Q?`j;<5BZ5I?PypI+Oc>LMFdm zEptIX7gw+Y+zYR=9D;-~bAN13S)a$$s!5B-I^=TPjgLA5v+c4713b$HxdgUt+jhT) zwKsgWSx-?*o1(|AQmFVkYkTHdCe5`Yx~(}!oFntRJkUQTORNggAEHgUM1w|+GWQcw zKg&BoXGJi)W}BTeeQ|cU8zXi)gByD_LEjO@6Fo4PhwaCI(+<35N9;Lawe`7qz%{K4 zAnAS|43Y{KedmNh(nQGco3YBrh z^iJO>hj|eFQ}Q9ht{iap*9-zSmtBOId`t+XOCsLBLRC+%pv!t6X!EVMWueQZ>fSAU zoF8!6AJ+e~TkZ#xAJztz&?-`$&_vaH#z=p|#aIF+>fYE;6XSSHifH=)ui|rhRrS3z zWs~nJzfO4=FGFH5F!06Xy@e=gwC9J;Q%}R=zwpPA=-G$7YI)V^d}#$0&UrV@6I?1e zwa~VghF)A~tbJ^^D2T0K`!W}&*ju@FI+}1jywCqZFu7h9y?~}*%qXr3+xL$MH7^Ih z*Shr^CL!3I<}lS1U`MX;e%H-|ez52K&a7nN zoKn4pRNh26sAscEI6_77EC152Y$kSd#`FBx>PS$PDCk+sbu++jQ6q@+^!&8&=GD!f zkoVYp{8sh>p+?esB#I{@g(M3ll&y5M8t$CEG|V~v9M4F4DM~VUubhh_j}_ZKhMbTs z+vg;6L&7tZ#2mZyI`$ioFUAh0B>mr}O<5gv<`Bvwarl31xC0IH$(9Wv_r`UAU9OKi zZhZWNV`NJyM2ui&*OL~*EVVLt_?t*~=MwMZ*uRXQyFMD|+OFle=?Mih!m!hFPj%_w zc>t-R`(qm70jk`N!Q?u-NX>P7w-DQHmQMIlhb<%TXiscJ+}}tl!d*1Zd}nrzQ#nDx zlue+7pi;cWmaFpcJIQm=;3-hPs#5tvC>)OeVJ zI&>1ng`=wq5nGsXrc_Uh;s<00GokgTgii$6@h*aaK!x$;5L-+{8wcH%1#rhcaIKY$GndfcLM7^ zBqt0WNlRTmXRkV|{;88~aph>ebdDK&a?T&q%B#1pAuQRmDiJKPDw63(rWKZ+YVCSO ze4nC6TR>0d=RpB{5dT9q!!uKctbStcM)ED08QmI#;n8)2o%`A1t%)z0aXz!532|MO zYhWiVNHz~Pkt`$R&#tRCM)kgGD-_kvGtGg$5(&PP>6wZ$y6PPz$qFxcpQ>KmIbVfC zv#qCj?#?Q>vCL!JhlVzY^{+lB!X58mWMN%`>;R@l5P__)-KrU`xgr^}aAijyYDluE z9saq*9_=Xxck{Pktzf?%jnAfGbOA6<54fqGww8a2ckqZry^6ZTsZnyQd$m4aZkg)- z-VF&Wj1a9al~no{cLhkPPb=r- zc?5^Y(2SSSJK$(vkLP22fP581x!P*!;%DI+Xj!OHK>9!Gd)WP z^SSN;U_DWrya3krw7@m6uIP6f1>M(craa49Sr4^Fo@Xx)g44~HJCxD2wI{2sZT&sA zD%w+U$TCr%k%!3;06>NNpnu^}{la}a(4lc$%}%<8NA1vkspn-YVz9htlVH`@!ZSJz z6n_SwQ;OOPB{h@CDoh91Fub#+MG?Q)Xv5J=NuN85ex))`bx-hCPhxrAcLBV=h!;kB z469rVKR~s&%kXJ>Z92wu6o_dInC$58%kN9wyM`#UP}k0PLn_n&Y{w!?y%_ia zPs=m)>G3s0g`w1J1Rc71K|w*s;ah!5un+qS0|qs{{m~K!#P{#OKwz~$;Oaqh!*Td| zzfwN&SD0`qACH!Oivf`OgC{kgpN4bqjnO>qgO}ONoe#h?lF@dPFHu%DnWVRu%5D-G)8&@me%&uy2(If}iHFXdW=R zb%zhOUB4#*9|)8@*;f)Ys!WWGzSQcBBRp0D!E&$`b7DwYIILU}W8&6Ggm9V` zZwo0{jrzxoABk%~JjtoE=!efiS3sU#`uYxES7((NcNFga>KDo$%#^*ex-pIg^b34| zf>sM!E1OdAI^1&j)RW-E_w>bmkZE&(t2=GId%VCz#_o8lJ7$kJ%~UdGK>lgOeW0tM zikp!pp{I|Rh+ma_$Z4KU)Vub&8F0(QSWACToA-9N0R{bp3R?u7WLlDrjs~gb+#$2d zc3ok3rws@zA|%!y&0Bv1J!GrLo}I?<&JM0d56AxpS&jVX`1%qDU!?1Putzi>9O;ks zFjd19Dm+;RNxn}I&v$j{^ieRt;0+LsgvtvBRYxHSE+*%xrT1UDU_l2ZRiu|Xp6SLu zkmNo+JL{iu@deX&u3uLD{oEN_Y zoP4I!2Z(a5m5Y6{!G$t}r7unLnUzeQ4<+XI9}#6&WS20pXTgBBa3psxDxl@n|IqT> za8Ws?lVt~KF$`6rW)g?Ccx*+v8%mz{Qd4DEw3?Jb&N9KlW)-h`nW3!5aa7lGcW1v{ zGBe_xMQL)y+OHysuvNmTtPZ&f)Oat*PFAESW6{%PHrFwK&H&uM>y_d~>9 zU%vfjtPeWHRom18pPz71))0@$It-+o`MJdi{v%j3d--0m=3uOlHA~6(fp|itBI}#@ z6f#-c&eUnNhGz{o3~}6@KMe7G7(*Ne$sa!=0{gJ7-dh;Z(%WU70r!TR zJt)xa#7trSKt9k)ke?JmA&DShj5oKV+}%&noxD4c`Z5MfiJS)zcf5|Bmu1wkrw7@h zE4{>0&dx{?b$Y%_{s|m2{8wihRE|dpX{btxwgcItxUsaFQHEs`CE2eB)9un}>P!TN zv$1T3)+b7`7^;Nm_% zty1)Q(`N6Q?6$_c)LaZCQsa#Fzhuazo=3~bZenFCHb2t+&pHA7caI|l5Anz|0*4fK zu8VyNKg4!epHIC=Op|!x)U0vm5BaTk-yJ%35No z<`K(Crp?BPD%{)d3tniEUoC;@OT`N)10aX)^S&_kJ6ec|?TKX(0y-K$vFNA4^OO6~ z+OxFEjY4A^jqu-02-vvmyPpHXxJ0E)Mc0>h+B-@>*P;lN-|V^Ld$M5bi)kO+-TVq` zZ7I9((8pFyRMY-{tRNJ)C@`eyZDx^>$?GNJgH+L#s8-Xd%aqSikD=`(X6cKI4|>~) zgtSr%BXFvBMTjbKAz}-y+u@tqTJ+UAC-Z$tPb;e3b{D6Lx+X83PnH}F3!IlCbobqT zJ#g^xg*UjQk_DyFoU$o5C*7p&QKQl=b&+LqW1GE?%;1(DF+J2G0uUutN^MRaRhUAi z(q3%BVQ6sAaHYp;!1JgrMBlJ08r9O_OWreye>RhBgfS(vi_>)!xo=G@`wPKw(A+X? z*{Mom$^%OnW8NoUnF*9uvKnd7tlEsS2@BWC-HSGZt+f@ey$T~BgO6W|Ry*6pWyE3F zz4>P?gF#jDE*IGwX(?H&ng0#vn0-u%{f@MzPQ35|&N(4UAuf+UO9HzEp+sbviBb^v z<0NDN(+5%;p~r>q#KSoOVca#x6q{M!Nxi#-{Dq5Jnox}Lf}9jn?a(=aF_S_b>)!Y& ziU>$6rNT|t(x%$B2B~tILW$S-6!7CJrSTEH`OYWXDc>#^!%8yVe70o!;`Qo3E8saG zxIpz?)Y!N&et)sCd?~VldVT39Z~mZ>Bs$=S8)}3&x+N8B3Fy>TDzed9afgIw5pHlp zF031HxTYlQXchFdU!=uWt$5iBmx00%SKsJ_r^c7X`n2v!gJFHaURGq36Nc{swGUtc4aELtZ`Dfn?slk9h-4DFZqWC+uY+Cw$>x7S)% z_j*;(TNxa;YxEkj!(R)3T=@R}#|!BrV{+AFQHP%eml%OKW#W!!Mt2s8S;3fHzvXTh z-pVp(@($JF^`FX&7l2v8Hiqq?L_^!-cL}LYz6&VN)FK=7Y~X8QjpT(#N55imn5a7X zmdr^wgB)_%6ll5RSc#qJ1VEND*WKOgLGRZ$qOg6^BI4e73pTSaOQ$Hw4;IVN9H zVPtVl*bHbE#eBVl^|qHvqdj}zEl_UWi4XASuYDM9d#bNH$jOCs;xOn1Xj4W}qkMbF z{q7CVd4ZCB-Kn&RNl|hR*sg)TirP)dPKP}KfEq|o{9E%T-(jr1{4HGBNVlY;77W}C zni`>CM1i%EN7>X51(<9B1sc=IGAjQ7%u>^4B@4Hu3d+xW3geQfAJaOz8HxL#8WKv* zIPqu?{s2mO<&aVQJ9mWaKg{%7fM>n^0)`er6jY6|T}?!kk&rKnf>J(%$R85i_mv-t zf3d6GfDv}YxQnZiNkEW-6_!<>uX4{`z;gPl?vom1AiMG;LRjR2JHLuL>M~ml0-~K7 z$|$b)71FAzpGZ>G-Jc*UKvtYx&&vyMw;k9U*07K;zr*K>9sfK$ypz$A2Tm@lgXvSp zjRfao^X0MXQ_J6=16D)3m%*xM#}i9(wOW%rr?}tr_t7Q6`~Mrb`@?a}hm{bz4!xBm zKtBCfUGv`RcAv@zZa7Zo<7f!}rF$RO@3*~;_%^jeJ?h!78P!UzrWS41w3o zhCo!)vgp2L&M^A=7s&v*&=jhQl>3KW{`ri?4umhv6EsvkSTSwPA_vw3CJF6`)6C zQ3|o_eG~K+*zIp_N-;T#jv-D-kr8opj4aJ6qKpUuFY6b57++50qrIC$P=2irI)40H z2}CCaO#>ZsF-fvctVg^_9v;?Y?eDI-P`G3O-?~iJWtu_&qv2(zWMOPLc}$|5-nAkp ztK}<{-p`7wa+%W@Po?P|IW7)yg3%8NJjH}YNHCWTW?H@7LqqgT? z2gK!dr|W}lc6ujDzV`65p!_;Y-b3`;pFh`SblT<{cey6hVgof`1oNxjtrOVw?!7X- zk>eOS!`j-i?XRExrY08e!9_984VZ3_NE|1pe$!0a7g%vJSQ+2jZ~9(mu(@NPnPDOl zfrR-|K4xTkF!<8dl`&f%>om^_hc59EIVFaQ%P#s-c2HspGmoT=vqUoqYu0(bgt(T^ zEatbMZ0O!0Rp!@Ut}fOcob2Xvk5SM+&G>&LA(4-XIOlb9o8K`X2s&lvgBy=UI!$B! z=SqW#-m}z1)TkRrl8Q+7stRwqCP?(}?CcM_ zUYfuy8V7NTSC*qB39s!;)AeEA{z62dSa7RmB8Qvmr<<4FrbV{^<57reoj_$lVq%T6 zcVf%|OG%zwkSUe&vEy7xu_$46eVVL((c*nX&lOo4;{nRK6G!ejkx0wr2bTv&Tf3>y zu!wborG9idg#HRqsHG)26^CWx*hBs;Oyc{bBDj# z)T!WAEuAQd9(YhuM5gv$sYzZTnf0V^<-Aoht7@hb>`tbCMW?hJ(Lt~*qvVr?nK_DA z){oPT!@A})@izPyVRWT-zG2x`rcC1Hss(||&(de|6t+{mlRG0FO`;|(CO`QuaE4x? zYQK-@c+w@gky6R$#qL{K9I49QBG_+v7a2)A$00_Fk#`6Z0|++5mdH6`|#n2$spjVjG2Vdkkn0uY*)Fck!WV$AK6OrgN6eVxr% z%hV@01*-i--Jm=q`+8j-ww~Dbscdrc zyhXOC$B98PkW!@8knAclH{cdJzLQK(Q4#0O7qzgPhS<2)t76;NK}3?06fvnLA)e&> zg<{+lSf62qZUGy{(C|J#LIjFyaS`!|8o#$9k%=)IOC6%?P7(={QF$+A z^Cz=qEgfTXtSDcpE2W{V3ya0So38Us+n&zXbr=p|Y1Q&v`a+z&x?{LUO3C_VD^oSm z6g{y({57e^9V{)c6>~PIpH#9v8r0=0_WJ+?{dz;GU?~b>%-UZf zdk=8lmr%d_brS9G4r0dbTlg(d;I4@I--mH5sz8&j5D85+=AMr29jusczBc$sXr1mGb!OYy5|re9_wM6uJ>?2>FQVWD0t+LBqz5iL z+xMQ8mB?M1v*Y6k-LUW`94->vb!#){VmxLOd|YpYeXE);LT-8>u;1$^5y#}s0>vfL zPR<^-w<=~zy)NAst^xSf?Ha%0Z!+?zowznxC#>WN;#x@&{qD-RmpwK&M5o68DM0uM zIx@JTD@GP1Te)>i#aK_ zckohUp4XO3oeZ%UsD_iqYCJzfqDl6R_|~L0VmdU!T82mh|!i&6Ww12g*^qV zrcLtT+jj~^4X5dYw1Cbg!x8QmZP$ExbVubTj*FEx3v5O=><^DDG)q#*7ueqtx>Dj$ zIaJi7znbPM-TM%MH>I+#V~0(~|L}pvB>}hxe5R4Z`ImUsb0VRP1w}i`$?%{s?9}MrLp2{(;n)c58?uur$X*Q8)SSaziJfEm#}>MOTcB6S1xkxK zKE6np*y2dS4cr3#kEbwNGip)`l^wl_=^8Qsc>1U|C&>J3(Ke9H?}E_(r>*aT>VEog zE1eJoVu}u3G`U%5DTPH6B71dejE@Tpg3tQB@;v+O#Ow7Aa?K?~Gk0h&m=Eq!j-E3> zqNzpy9+ZDnUYmxhJP}iH0^^REoO}#&S+@)^%A ztB-$*!7xNp8GBe)IdiP1vUoCmRj*`VV1GYMU4kv$>5$leK9s^%X9papI_B^m)g6rL(?B|1(=m%kh@N4fRHg@4Mm4uXcUFdKd$g3}7?XExzY82r$@BNpt$i$=%)G{aW5m z9z(h(cH8>1@ZVnqBeV!V+t1#*?!ujF@^BLrpo@R{cc){K-m4BOP0~zaH!h@6B#_ zcw;l8r!MMfQr&#=O6ms>kKo`KZeN=pV%r6{Z;))r+6AI0g&xA+mfP=w=DNv#^Z$#F zfd#<|k@CMQo`1gK`r959MBn*)aQs11f4}lY6#l(?H*vuuZ*{iYw z7P_8bg3+5FRUv%K(fFR~Dz-zqvChhjYnNn~q~xg^1BVq0md50x>r%Y*<(KAL9AJT} ze60TpTe=yzkyLS!o%!$C0okBqW_V*A3 z*ZV(yfsb_W9T&BQE56+e+z8n0))_;ZcEF9R{%ADCA(VUdtxG%R*dwId z%&>1!i{mDI0Q@rb0x)qHfUY3v)~RwKh-2{=m{lUQKAN8n?A(JOJ=Dyg4dhrS1DDua zhx@3wUx8hlV-X}Z!~oCVi@+k~W#Hi^rJ@qYR{4xYlh}SMn|yEr2M=a41_JtLg=JJE z-3!cgbRj@BFLX5RM1#j++eP%Cpkg=*IK@To90LzS;G-80|2~)MkGxI9>iE0}qkzA6PoyzZ@8=M9VNOcQOGluj z)y1KL1phXltQPwQ)&b828Q0;M_7GT-mO2kBjl-i6AiW;)^G+M~vjD zQJg_SLqjudrKBFQtw1TB22Rb)Kn4Z}4~A9QA3l7T4m^R9|tfXz|{Y=|&3C?jBn96^$%;mPh&5M4AhP1R%4 zsv@K(ePI0^3xjCZ8C#i$sR;bkkb}bP4Hmz_eC0asOMER&O~N!%>i!xvjy6Uh{Xk9% zrqU-@{+^*~)I0|V6xb5Vg<9Tn)~m+2Ce~7hwb2 zygUXzuCkWog2h5hyJ&H3ff&9}AU4y>8Di1MJ}_&7a58!-xW>5xm4}Dn2F8^ z#^-E~{PwlNc2)OLwD&Hi-LXvHH5>26((59@MCaJ=yRBvMpRcFGCeJWlVv@;tfkE;_ zHd5*#sRQd{NwZg%+gB;-opay!-`eoFa9yxj%dV!Kp1A+rY-XO%LIyPpq(jhg-U|o_ z7y`3@Q^y)uF{yMBozpHyYS16;qIF~0`@km>F&z&F2e;pZE-3|uG5OXAo}2?;xW!XG zV5Sjmh(DaHe+e91~e7DxcqwtxnrTvK9UDP1J>sSAJXD> zA31oBg0UoGKpvh9al4FR^@3Y32O4PhL&po4)mgpO5d-JL0qj=QY$V1(J^RFqMIRsh z-tl1vj2X;7lB+_=q1-~IHCP(l^2PfUr2O)3f2 z)7JMS$I%#H_}It{90eRju3-B2QvZe|n04Zdc>-ofweK)gffr2{#4+NEcbzt1jvccD ze~2TPr=#m~*d|4M=%`j|H=>lL?f-L6FH1Q2%c$XYV(KdU)P%C-#Y*!(D`)LR)_|a&dv*c6J#OPNO|$UL?&BcDn_srO z>$1}Hsc&rT-8?}h+cRj|2DIao!lmW}o5eAlgrFl{zVdt%gUZHS^`kUul$U!kpu^fCN=?nw7fBnh_F(NCs*GtCyeuB4#oD=hsh1@ zu`^4op*&X1Q|-?>?&^2gz_kmgFnn>Jwa56!$GfR?@;Hn>nzJb%Vw+cRM>K&uVjsMD zGQh)^LPF<+p>Kyya~Qi6tqqJ@x`p)5rX(pAhoa zw*=cf!aZg>t3>|-IMBC3Ee_=?$=1`)iCFXBTS}1ENT}pCx2Q-s)=1d!tT>gQVOM5W z!Q`38_uy7@Yj<$zkibQ4!+=z0<5Lb>%D`s~EvaY`G#cvIXJZ1Tsuh;7=#TLrF508b zNs=hX!kx%K519FKyY{R{h8l-@+_%j_TFFBvU+Iu=_<{e%hx$Y$+#dH%wwA0kgRBlp zv-obLtio+r8ieg8d0^heC*4>UI*N5Lp70Bf!2a4W#h@A+WPE2?dt<6zJ%^K9QReJ$ zQ&&x$zg6Q_so4y59EfAW98>XHXx=U2WF9KQ^2zR6ZF#0q4;1La;0d>Uc1ylNxskVsx8u8jFZ-uUq8kgXIuqPaGPf-kKf7`d}1H zwtp_5Ur3!soPB0^@YBJ|bWI}m%iY_xlD8bQw|TS!9sy)w1(V-qP+8HhF}GIEcPHE1 zjRO#&o3#s$Pf^`)?XN-DO#ZBta5zRKO_qT=R$Wxn=d+P2rbz%PbncyP*J&8PT1YOE zwrui6li5nJn}*87B!~V$q?r>D744MywoMSVzMj!?3HB!8Mn}|cI^;uT!dUg0`H$vEIc+3h3a$#9ZuN|f=1%;K9=gi0cO3+WL4A)~gOu`we zttJ;vrc^J-YpV9nWM`&n;XP_Ua`a4nupd5M%z0{j;w%>dU0+&eYaL^AIsHCIHD=yL z;4)8(jC;?Wg_XnkxL`l+{hh6i@#2Hw@D;rl+2k`e%zOgHNTRi~Y8^n| zD9|M*UWoiKU9Dt_{iCooX8=szd$q1SEbvdh2skXu`EONbQiMxgo5*i!iX2dq7vtrSco+y;uC*KeN^KuyA_n@#~iiNWFhiX|7r3 zbMRma4o(|Z7JK_YD=Y|9A;h{y$@Kv@GPvt97qn_tb1V2T>8-+UzPhXjd46q#2M&Cg zv;7th9^U5Wf{oMF#rbVgRt*;cV!qCAM~*n74k_e2QsY11Y5P zN)lVTBWZk|K$8?BZoT-6znHutcBWu%9pfp61Tq$9LiUk|`6F{IQ19=xX&#yKujX+J zpu)~cd-j`-+rhrYRKVrQ*?nnqbH2idKOD-Yxmm2|Hf$d`8r%9M)lS_wQu3xMG|yo- z0$5+J{q3=cAUKnoYj;%yP2ZZ4>awa%i3!qi^bq06&bzv=Yr{^V1ShiRO9ruA>wJYj zMw!_pE>o*SN1thpPEM{GIfB@zq2R-<1oF;AMD9AME0LY6eT7H&%_ff^K)G61|NDwt-ZlM`53mFb&D)>3U^;uR*! zk%Zv}*>V*FF5$6}(NuSGBD#K3=F8f-(<;yi9((-sdhn-;nN5b@0Mq!A83%a=2SUkS z%smLa%=GUwTmVSa%uC{oIPj{bR{NgG_%7Z>@x~SIXkL7VwWcWqJ?1B>BZVyN0KY*@ zzd&mm=e=3xfVV=fv+UDnZr|G4?qT570MTK-I3CBviyDluFm2)}y>6Z~M$J`~nzavF z4+lna>(a8ytW$L~c~vWRQ6`F=LUTq72jR)3zvaDC$A4pDqW#9WSn6DX(k;*)Q97aW zg6w4qDNucv%V+xxj5EaPa@6X_ISC>*8msN@A*X!CI`U_C>!csPD8KE4W-xI40T{|9 zZ)G>#TZv()WY5~1VNt_YE(uUEh4Lde?b@SqN!2}$Qg>&8b9_!&rTg%?&b~^9DxpQr-Su_@u^mNz`WI4H0nVLMZ@f9 zZ6i&CHlqhBZal?@tWcsA>L0ZKge)2PhN+~YQTT5SV$19hfi=B(7L)`M-$flPji$D=8( zx81C@7Hmq?6)239+mv&eqrHbK8d>8{B5o?K-(#AFG1uMomRPom=C_|2Jc{Wz0H&F< z)AsX>KT9$hegX&vpRwip1-x zcM^W8#ZcLbDxl~Vim6ygA7{hH9PjBFi~ik>B*0^C!-zt0|C&4)E^O1 zv&C4XI3(KM{o%pOy43!5SuHysE}F-B61vCa0=6x-&Ly7h@@99;5fLKEe-n?Zi*T8B z9!=alc>2R+Xn5f9TyEDK1-qa5z21Gc*CnRG1OAJJX$UHh2;3Ub6!<9^jrE7Ig6f=h zV}(jziE_lY4|Q1F1Vh0XUGCe}-=~2r*e_RG33!-z9{LVT@m#zM0nCPoTID$zm+lE- zvDKYcx2cK+#aI#rT1;~1&jOFgboprU7L7>3lnq?!5z) z6-WHy!ZY)2n4L~AILdsrxeMvSYLIA<7fez+@S1{Bj|QtNxO5XznnW9DpNX0*4z#`= zZg^lIuwq|$iXojr&-CH&g*etH;h+y7iU)md#HBR3V{Ld_IN7vVuMz4?JC+C1tRpKN zd6<`revA$`see1N!IjL^8b1Zv`xS$hII|r+74dD?V7(rlhwpiyOV6)`B00OaV%ugcACvQ?n`Z+hF4|t* zD=LGM%dEoT#Sgd9ydBiHDqcI$EbQwOHc!N3$K+g-EFww-e7C#;m%vE6oMGJjjbXwWWkW5nqBq0Jjn^+MY|VIA7~31H+@~Ykp2g@=++=2mZxoZ$VdMN0Meh*WRppZG1{~Y8 zpQ-^|fMSIuO;c~clBTs-b$;|D)OGfu1a%pcR6kSL&bVkOKdZ6|322|FZbqn-wLhj- zJN>qpBtI{Opa(jT$qT8XXU%DbcEUvZQs!_DI!m#Q^m6$-(fjV@Z|ti+`HNS@vyiG( zm)Aw#smjP6Q~lh?Ti!5{@i5-b$9KC!BM8*!K6Yomyg3}FOEIH8o7>-_UWr;BMWeHwr*C}?mPE?tF?ApX~#I&cQt&?X{Go1 z=hx47Htf(_I1f#!q`y%|(9Fax7kJ3Nj4ab^RMZ+wuNW|QI3Kh0_v0euN8_aAdduP; zT@;A(FjX*c`pb^AfWYUzoEWaxq#t!tDhWDKpL}9re*OtG*8KP6GKE8ongsbIdJlM3 zGC(JXkUINmV+UrwXtK_XYislIKA4(G>RFR&P27&biWzRtLYWcig> z9DTp=2*N#yYq^@{wa4GE!pQExjS!Qt-8vU4co0i&&bME~P z@%xL@UQz{44d$Scm3Ng51zSJWr-H&=l=ilrFkUbVI{o@lw}ch&ZZ4@y+YhZpW3RSt zYyROM7Sufot)O_iBe+BCN(>_iNBgM(EbN#ajJ zh;8T?i~E74%R4HDnqe_4+c#-OMto-8Aak1Fw{YqfoY~7+5_XpIh*@ScPY4YCkq2(gry66NP z+QT`}ovdMb*-!z*0_UL5aj~)E^;vcK(i;5y{vvE|-#){yKRwN&ci|d#Ffgz%Jhui> zRwJ)o#&>I-%eBwbV;UEi=7jHEfGZ&mtWwSA_bweeoF*{b7&F@KbeC31wmo&ssx`;e zIMmU*Q^&2;J~Z0QhVTvz#5=&$#c_2*+Jqw*Mb+I!+Ki?x-LLdI}v6EDNUBO z#UWvP`|TMtxM9PTAbVkgbw#7Xa)eD(y_n>+M5upB3@56<500N}=srl%oSCVaI}z*H zC4-$(2i}4Eg1zk`Y3kg6Em+PFcM%9JPR;dIGTelHlNJ)eU9p-*2J9>3=7j_`>nhNf z9O{7;9@EZi*8NY0hB!Q>cD>`kd;7r@KlE6n#$`Nqj^hGr z+*W09!s2y&NM6Fz3igrN+4q@I*wF6^r+MRCa`YD$7nuhA-(b0Kg@`8pF$iUg0%kVI zf(V6ZH%yQX3*mVc)byJJ1r6(%hDib5gBlXE!Ua=&F%Xd0rRC0qEd+Y986aYD9|UGS z!(4Gt2ZQMSB5FYAd1bApv9S>fa@jC&+?bN<>v`GTE;(yJYv;S`g`>j^2w3XVhqwrW zrdSf_-kx5bZNtK{iULqd^Q(7}N=(LOoY9a|LGbusaxk~i_>2=6jG^ubB_@AY@SK>0 zWYBKLMF`9@58VV|qY@x;yzON&7@};wEVhg%!ysR78-lI}i{2A_PvlUMpqMO(lBH3{ zU30coRpCMb^nMV+FJFGq-@U8rb#%Db`E-IPxNAR?PK);WtRKAp6O_~tqDhd1UM9oq zPLROmd_~E=%8iiQYi^bWlN&gs8wA9LxSx#3E)O_9hgGWqKc%$QNDq$b>Uc> zZOrj}&WN2-q_n@UFKtBo0cakWIxYmVWC({4hJ&y(w>|UbnVF=9%K;|JQ(AY>NnSG0w{4hCG?|^MbE~WF1)1RV z?K%>1%pbl1?{@3T&k8HOdHN+4y3W9rRy?XGIM*Ondym-KV7XXu9VHVh6PhIGb`jR{*&TJ z~W>S#RHXSivi>PSbu%tbV(@!F0sq zj?-A8Q=TqK;&IVt&Cx=s!JJIhMd=h$l?+PG{%PW2ZrQ7QF2+a1uCA93u6JZJX4Vvp z;dAnZLE@to+{h;!(^WweZZKp#4dR3E*al1fEz7$lUbd25yt`GkZTkblPI-`9&Mq@t zn5m5sZ#Zth5V#3~vaRwfORm6TDgte@p6>gO!}5w1Dlyi^wSx7R`-75cD#MLH4nm!0 zWx3cE@}rE#%qxz;ahUX@%hA-RUaz{t`Y$le-4vvWrXXuEoRP108$VR>nYx{i>kY>chD5Lnn|8rZ#RP4%oBylE%AOsyK3*d zqG#&D*|w{5H`eph_Qb7gS+JKVkUad*SCku zOs+Xy%cfqAWwT0K|9^D72{_d4+df_*N>K?#v`ETY*)sO0P-M%Nbu7s`7!27%CCg9= zWiPuic4llNAw|eGgE97XF!p`>-}5~6e82Dee}Bguhl82%S?>G#-1l{z=XqV%1-!)2 zUjwQGX#=o!LBmRK~30!0&poi#K{0TI@B_GgZjCYKGn*|SfKCp`P@_y~ch)JK@z6tBsTPLStdJ;gK& zEyoL+0~w42;wirTh(3(d17}|Y+AmOQf;zZmy|HbbBDK4nOfriKo&pAdkvH8Fn|Kc4 zE=z&Lh~L&h=y;(@UZiQ)Z&g6tr{COp$b~0sCEd0Tn|;nL>11+6$euUJ&}8Q&7p~bY zT3rG@rEBAP=N*CI@7kG8ksN%M=JxACe!Q{;kWCfA5=_>CZT5E`+a5aDmn@Q^#Q$Q# z6<+{FJUymAg+KyY-O?jX%A(ky$xih9imA`(BzjY8=BwQbh8bTQFk;!xas_^6nI5b` zxNJPcn0F5;D>mV;PTp%NxfT)b6UOUwvVt|hxu}tiqBiA{V8)tugRRq6m!!w~Egrt) zv|I$Ag?rgQ4{A%RMrf$}yrIxSQWBgnPJ5;ce>7YiAmvGZ*lM`doz^YdIZZ7?MW<$*o=1DUuHn z4=d}@ccM+lHJX8HktS>1=TX}9@kdlKJYco`f~4q9Ed3;DU2yE@qjx`SPI7gFYetJflm%=X1Hxl|1rPSv3 zcbd+6>s^1h3W*4*jSop!CK)tL?tX@$9_aRhO$2)dX4uYJM zRT;}&cnorU#t5-X*#yl3{Q8}G!H+lWW(2->bykS#o45*5zgwCZ+NHpCsHyw);f{voZ~Tz3u>gTI5cP&5l?ZjrMEWeF}s~MaQ|rP;Ev%pqAn* z?3k~$cEws#tn#3;5Zp_7zD0ue_yYxI@jJ`k?`!dC%&RliSG4X>Cn&Kz zjDIHHA>Uq8&MOsp+{x-9WVI&Sv^O@3rv8=Rg;i-;?p4JaJNk>u?k*c%cBgpq-ezS8 zMOyaQ)8+;BgoTAQylw~0ViJ^}+rL05pRsK-K)&)@YGj{}3fQ$;)cT;9FZvJ@U9CVkx2j=Y>)JD6h=wzTT`!95RYkW*U z8si_opp@aua5m%1Twi!_WY)zXX)B2$utA+?JZr7QnD+>WRk@`IHZuD&r1jcneiT;v z?dSOW!F29fy};*QII1?S@d^7(QbtbL>13DAN;ms>n6$WpqMi5A7s?a%!_O)HsJ6cb zj0{7uWySOhVQSRHurImByR67S<$}GajbQ2Cdcc(r+{+vLkzez)>xB?qnMl(KX@q@; z%W3kb$sX>*sK_2J&w*P@X&reSMLAMhtY2oz;mEe`u1WFQ=pYjzj`>`sd#&Jy1TwY< ze*B8q$vS0D*mw{QwD71>fB`=R!94ItB@tpSUXU9*-lj%sE-+4a-^Cj{&o3jS=?M?c(q2p)WtklUaP^`8#C(SU;57ZWRi^D){H08FQ7JfAl9f6k zm9IBfM^f-^#=|Rv_dag*i~RhH$FLnTD@fSNC4T>Ux)l!QlYK2d9~O0Il!36sp%HLu zazwh#Q)uQHB9qL~wU&_zU7>fNSU{6Uyu6fBPtmEWanfOYoH%~^$OvyHCKvo=TUudwxPrE5)a`YxbYc^M@HDEEO79GfA+Ye z#Bderw31>`1%u6l&-PqqJ4%qIx1T)urTOdfITr%YW}dTtFRR0)44`UyStGgpJ#_)5 z!d_+O`olxf#JrBZG@9eJf>W+x+UdeLQxC(>?>RDZq|&>!?>zZ_N)q-3)PWz69N^&1 zQA$%3k3$6sspl3j_1?VHCA3;8KJz(T*p(};KVRb(LW0b>0JJ(oHTJ^S{d|%KymFWz zbZ+Eo-NkGTRk|3XSV6=It>tcAEWUvye@!@9O zd;rkeR$1xFXUOg@)~`<{I9H18rA6M2kxewa^iWGBKS*y2l#P#}{3ufGvel4vQDX*! zmLXQG({&}?hr!^3Gj;28&x4rvTqAGx*iKMAX1aCwd{)-Lt1jm0XRU8h7lpjf8JZAt zgT03QHV|Ecw(0klN2N7(wwkZp-8-Erp(6*n^|+$YO za8$oq*K@l=ftKClO6}*h=F&ds1@nS(@085V&hqiP6WUAp%b{V$eqQ|0#;GQ5pS-Fg z8b@x3k@a~jP$j1@XE6Fx5Q~(rY{MH*EigmMoQPtItWlM!%;J~9&}PS?M!!&m{<4Aj z{%*#)_2L&ai^eqj>MU%MJ9QmXTl4cSws_zPuj0O<{Ki5!4X+Rf%#frMmCkajdsgs)BM%;d^>xq!DA9o;s zcHUV#MT_{FDXcxA`zw!04OxgZ68lTPDaXy23oCB=`Dby5Kh9OD3_aP|bG_#X)15D6 zrrk^0O}q>2oLu*-*wQ*K2mY8nz9`nHa81p=$>Bo? zM6kX>xgM5NKu$y&jY34y*3sdx5#K^>uN*50r%?VZiL7=OW~s!b;@xlUTC|O==-ozCVIn~EAX$=AH*HtpU%d(UeR)=S(y@H zgw*;k_zG1XXH$Gxe0oEdFZp#?UBh`+X8bFuHI2{X)uW@I#{EX|W6d)yB@%dkmG_3Q z<<-V8_*x{mb^INnavm)=5b@v#B6v<^3k342>C<|`+tH}G?|evwvZ?3l(ab*>x>Xf; zvc%Qt7-<+TJwNo%k1aukf%tax-M&)_hYr)8`mT8xV!l#hdY$#ZzVn}NGn@@nh`GVg z+}wOSH+IshQ%F&ep*!Xaa42SyKTunCb?e2B2P^2)`KEx*H0Zxxm(?GdLOJhj%R&8o@EcoXVhOI@U?>?JaeMve`qm?P`RH~7Yuda?Q+3G=d)K2z z3!tlc&HnZ* z_biEOueM4L%P1*0Ihf&-(4M`aq<#A?EWoLSdg_Q%*jBL!P** zW1v>@6d5dXJ>L3#5_e}$`h%FLfVemjp|oi0XIhzSUH6thRz7*$9l_L6>OZ5QKRq9p z@4xkaFyC}(|Dc7>CZ=&?Yi4HviBc(QLc`ba9M1)9doQ;W5rR%pO5fU6erG}BE_|8Ozt7#f&3~e31 zO!=X-H=0*QnwpUo7Dh{~g|~>3JfK{@b~DRDldITv%^s<3k;SV>&={a1gc=cDDp~^? z{4yO4Dc&VBO(T$fk_Wmq-6#?wH3Q|ZJ*Q)#VkOBw5eIhhzQgnvw8q9hGl6!M8K$7D zdcNBh$W$l-e(@j>->3AJ0e$g05Xll>pXQkZ;6qtGleFO5@>U6AF9^p*l4&PUt+poyqm@JUjh! zeObg(yDY2~^PN=B46CuhJdBz2&U(UFP)K|ths^54f=9nyZ9wu?=jbx3ZGAfznjnGd z4cIS%ezhKLsO?Ktls%&qvGQ>4&4{XBk7naa&!=BR%{Ak~;fRK{{Hy}<)Nn9C`yGbj z6msGq$8P6k)(rs%yJ)_Y_t&9V(gxXq2O2M;*_nG>Y#gh{0RmF}E6TnA4R|OY#s_*S zI)JVGWB3EsF5Kp5xP;5_7g}@ECiF~O+#UGO<8X%)ClXG7f2&ZrDN5ORTnXCWgtq83 zZFc&4eapxYuo>9uW^wgX3*W~iJf!$n^PA8b2Isbj8B%J4-c&^x* z$w(c%X|A|MxO^hk_*GR88Bx+IAxf8SVCI#vH+J!$urZvSD1WfkelHo|%;k2gvxYWF z)+~G)?rGwBnnJ2OAhTjlBw#C?1d!x%;A|TX;rv*dw)IO3GTqLJ+v{eKo-BcwrtTc?T^oBTQ1pE6$w<)D@-!}AxrSFdR-*EDI{EB*#b>Oum?BBm{=Gz1ufGVujSUF!x z9DcOjJ*7T%#cy(^)ei%M=4L_ zGkiIbq9}0Sl}z#=QtDps49TaBHG8kLRv9IftHTcVQc4$U6YK~kqTlf*ZhDw15g}pu zq4n+jp)Y~5j_tC9VirwbsdE(vSRo^9w^J<)bMN$+42yKPO9pN>YjlgOH9eoT4J&G_ zQOw$^r%dIdG_ho ztcUm7G$SuY6Uy9*zM*r8x0)_k1?&)q2HFA2SD-Fo({{lDpZ5`6hBxv6P1lIfgKfev zW3`=d=EIGG2cDDNTPlX@ERd+uHL9|%!_ z_C@+V@z4w1r*nKRfl`ehbB*L4xrp(xdjGkQdTYU@gbA(g=FI0Xd|2Z!FAZSBjLG8M+ z2XNCmhJvoII>sQM!6Yj~Ig0E%!-l-l>j_1Q8G8{6GYvtA&0~5BJm@--FTa9b2Bs=G z%lOY7w4kMBHY!@V(f&yxjUQbDSlk|Zs!GNLDldYyBtZgY+EQl}+VnW~8eu-;JAbh{@mo1ELdA`4_1a$5EW}0Tt4o%Kko06qL2o+i? zy##r2?U4}j*6_ATJ)T)PD^pn1M!1-YPTH+Bgu0>IUwQJDgVZ{L#CYd?P=~sDaC>}A zl3r*-I;)>(_1-#ZzVk9qutwT3d`(lcorakAda}>AhD=kRd6fqhx&dXl*^Rl9A>;OB z%uy~6M$^j|0T4ReQgk@K-n%Zy+R6C6&6L;oxe#LD3XQN}LBhS3HRC7P`FwpgZ@1MdRklcaO0y7D-L$u#K^{tj7{g zmw#hP$;OAFyE!7_W0nsaFTXs6!F%p~#Ycp}d~e;&gqy<11`KdbKSIHqeobp+W!qQC zy_)TO#ZmnABC<2egMJRQN}PrKdXHFst%%gx1x>%^H3PPzc{Rt;&bXc>Ns!N51vZ+2 zX#!HB|2q?y%zL0K?P?L|UxugigY1I^CW?3flJ?3$Lf;S-W%p#<4dqLoRwnR&&}qtp z^mH4dQsG%i9><+@OeSeGCJ>V!ZH@*!*TevnR}z~9SL6+yFDJIO+U2h3XC2X}4@oL8 z0-;=qPeJ4m73b|>h5B!0a6KCi%rDW^Ig!nyKhZI*hkVkrM4r~Mu}5%Uqi&a2jd75= z*SWvhBwsn^p70oi%f|USnGfMfKLIw(7<~oG`g+GCA3J)S;I;RiCko5d6|DjJ(f2S} zM(XX*wFJk}wj8E;W2cHn*hbmgM0cS9z)WN^JuX8-G7v%yWzf+lbssxbk38OCR`lrY0{o6_kEl%4T_o)}ZWk~H ziTHBRrC7tN?Lt|e7CuT@(^>l3W`IVSd0zx@ctX3*jJPSe%AX&q%jF9*?t<_xq=awJ z>~cE(8uS>fFj$=EL;YwB&GX6NvfsScwLM=OurTatqPDY9D__#3xo~k8XJorEB#-y; zfO2kiR)2gq&1#zscX<7o{x7f&v&9@$zOH^N-tux|b|s2`?nJBp0EZ{8Ffr%v=sbr& zyRf3BP+Rh7tX*AV<}9bxtBfd!@WXKLwLPcR45bOZXqfiIPvyO@w|i_%vfQ3Ns{Jd> zhm7)n+3nBo(2ZcjUnci#1pJI+i(x17GwSZ(-2Bw?JJ8!5uj^ep?*D~6p_gT|{CP{e zM|1K z{>+Q&o|kxf?()F>!rPUS8aZIO#e)D?^ypAxugEL!w@2NJ&BZ96L#^oXJkSeqhouLL z&wpjnlo?sZ`;?($vFmQ3b@K6U`blJapE4h816N$WV7$ZQG_#%uC`fLwRJHKvGALmw zKvQw{*;LYl^>rdkaL2f)Bf-?WZ)f4)#^&1JYGMt3J# z3}};-6NnRchnVH0+38anzEk#|XK66;B z_F>Q6=9~~;rNy$hz~Sbz^ACK*jp26vemT=d>^hUCnR{03(Oc8uu8r?U-XXDiUQJ0N zGR|ZHu<1IG#p`x<&%6$U=C1e%lJ7uUeZ97U$EMFGl$+Qi1?`5FBfB9`t!meVq)C_O1>IbJOQ&+(Yi~ z)?eH|C6hjCzL6`X9qpV#Sfw!%vlKJwhNF~pSf<(hWg}F%DmZ?mPIRh?lmV{1qxDBF zy$P@yFG{&z+=t1Ftv7O0b++(NI;!(I5ZhqEqsNd3$C#1;RPG~7&Q2RZ8&f#|%PL67 z+M=-R3pET5HKxg|@aCm+hb%U8LP{u zDkU2|{rO>}|6$kZ7ixgR9C)oQ`&@8NL4%+Sk#ICdxLy;P>DNrNMkL!LdtKWq?Dg1) zQQ|S}`H}Y)qpen1e2=SMf|jgi-dh+IETw*tIg`Pwi%D*LW|%SQ(Z;;>vwcpd&8z1u z{H+$ZmOoaRRzYhuEFO)cAjV~he(SLk0fX;`)gnbmJAEbWRgt||F^mq|e$MRnOQr6_ z@_!)kOs8`I0(U9N_+JK+RREjKhUgWC3aDYNuB~+yRZh%+FoL$$L4{cDRU2?Dj^aDw zHA@z!_;MswkvvFqRSPypp=|4g6jFchD?i9lo_u2;&3UTZFR62;hUTP{Q;$L@*u?8) zW%RokuZmRi*a5c6?Z|O4rNGfy$3`^7xqItfi%v8Mgnu3!%?%aZxiZZ1dV9|-X~;^u zX$wfDy5c5VN+5RH6M1c0pCi>gaoERckSVn|PZMulDZeR!QfwBE%s2FxtvvMqZ$6Id zbz&*m%Hg>yk1$d>V|vV}<5Q~_%88}F@nYvi!HC!? z6uR69dt=@qw*}-=E@TEXT)cn>jkgS<3Z0s@0!uPK{4&9DAW*!foE1ry`%8cojR&!W zC!ym4B&4t5Z!c^5SKJ}{m26(&_H4htR|>*(AT4@M1LPZot2Sy)&A7D#M8bK#EZ{&; z%Vz%K0T#DL+1n}q5pB>`A?af7SvA_cL6S@`ttu~lz%oUv=@zAla{nq=lM}w3WBb1_ z_|KiIM}8(op4HV!!yg$S5>ua4-U@IIS&gop26gvXZ0*7ZswZ`D>rRba>-jE2-1+5xM;m z!3>SILXwk3WXdxO#B-pRzsas?*XpkL!hluOm-_A;u*aevLmfzf`I7eY$#1(0R)w`i z3!Qi$Rj#PYHE7yXX+SXJwRZ%*XxXGV)f&)GgZ5CSA(OeNe6mxB0bde#d4`lh+104j zo`H-?m|4>-{=j85dtE0ai~DAV-O1dNY;Me;H;Plqi(u+Wr^yKIqRSy?ciUUEh{X7LtHIngly&Hr70@k&M#kG3qx;@x!axhbt)^|Q06}GJ2-!C_h1edXHsmsct8r0|G-C= z!Zk~ak-s=w4kCIZilUBp>aXu_Yukx=#xE(g0nR=nwe);pfE)}OE_pT~C;C+I_`W>z z2{{9`xgS}$liWwyQewYf3Xa*%8zQ4H#gb<71Kz8SR&lh<97OYACJ9@usNoXKM}lau zJug#E`}S6TsHLs^Cwq-i7R>>%4<$Y(r|5q^yt6}2?!6Hl^ALE(v%f}O4OenZn#pfH zQ(fepwcWT`Q^>Hjd>|soTo6MSCgr53&dMq9P4{6=uaJp5Uo1}7;6x6668Tec7*8$0 zAo3c}xRsaI%Qa^JoV}gT+)0LhVGr0IJv9Wk5LzDD$Sg?g%T5s~UrG`YL(Mp3t_i#c ztIw>#_4z3Qn`b8{>JMrQ1y7EwZ5*SsvgaIB<;K}dqC*zMIS{{66+_by;rrA?WeDTi zmurYzyBE12Bs`4jNUh|RmocNa=hoYM5$;q#V9;uPkjtcqu+6Q1h{>EKF8)JUh@q7~ zUTe3=Xp=@@bF6d%Cu;ZCMA%*Uosmwz%DY3{UQVg|^+Z3eXEH&=f+eS~)18+5ru#_# zD<&@TG7gaDj-Zx?lVs8c=;sG`kW2XcM|xzzzL{M-ZD9lt@2po!lM+VRD?Zd33yERpX7MFtAf~w+Z0R&>zQ99D1#=J5inwBXJ_xa z4*d#<4ri5X?f+#fTqj}-Dz$6uMJGf--%`-NO5Ib4H!cuCTYUNElke+w*AE6m^8m&1 zdYJKZRIrH6V39~Lz0-_^VODbGN1qSfL;LH+J`9B2Ej7pG_LZOzHVV5*J#U@O_g!;t zvQ4mdUhP5cM+unX;+Ca^h2;dFI@t7E)_2|L>D)vYF8cNr$xK)K@oZY!TB`t*Qu_-> zez(n@M1OWP;$U84k)FF(TDw!I94nGN7V)^fA@Y14_;|7%N=D!Pgh|am2Yrlk5k`|bD#I~mD5TN~~lVnq%PzAlTE5^my7K=lH^Y=;9gGZja^KYHrA zvbx#o&!dfUVTP>6vwbJyzo`NIlmJI9`2Ls(Gv@mf`*8+F1%+U@0a9`2NylgF!7<$g z-_07sp*1ytPIx`--~NafS@YfE*y#4$JD#xza*#*&b}mCf*lA|dxNlbD8=8{ZRrr_u z58`b@W+$6m2FNUbknseI7u|}tJ}qXJZ!oJx@mFZ`wV%fFL_-bb51Las=1{Z-Krh)Hb6Dbc2$8%(2Jk^`eQ8*{g#EMR* z=E0NUu(Qb`Wbxd+pbB?^ppJT0MHL^%^I(5&8#0JBZw@)$BkL7_&T0`w1PmVR8wM)r zF%C^l(M>s)lEum;&_fP6{BN zRR$3C>tv}dIr%X;&#p!hIQ)L)*z&~2(@Ix7Vz0B213uNV%FzxG?KT5|XxmZep)*Dl z+Am@%eOA?bU@EH_x@^oF6c?kms5qzZUgZO0Bjtm|d86v|C(w7>)2+__Ue0n4gCQI3 zJ329l9&`8hNJz|z;kB$Hz}!*M^70RHT;yVO)=ycqm8+bRAz=m1>h zr7qeUJr-u5&?WY3%cy!@t@StYVRLvC@9UJbE5HeN*e^TA`xliCQMfcjSByW53X1|# z`@L~#nu)uAzfJvN+e`EJCFIEdp-SGU2l8_jdOy~`Nv|u>{$LRBCJNfe!QJ?*+jIJs zvsstroCV)~qZgPg8{FPL{3%#F98e$j0p*vebWGk^aOr0-t})d6_;_q{6c3IzF)=}z zS^St3mIkgf@Ra}{xOVM!=Y~tfE025{3Db6?US!P|P5H(E|7|c^Q$M+>r>$rG`&beK zvqAAZ&lTorec=vi{no7=|Hb9}%K`oU9-PMg%eNneT-3Y%_a6WATXx1FuEH|fahL!5 zV}(l^&zXAfoG<73LoEHT6P2A!JyIU~{6ge!-syjR%+PS`y)y3!Zkanp6+jV}$#+#* z$lBUEUFpm4*R^-vhA!%77sNZt91IeG;LX9+&jU2dlquU{S!kCIm~HUh;D$Za{D3s_ zUbj~>qCFvOB9aqFN4U3S#!`c~bzt$2AB#6hNs41S|FM5JbfW<+aO1Ujm?S;`dFw)_k7lDKn*LuycADnrVa~v>*ZvV~yK1V88 zu>3L4|1~pO>|n<6FRyt0PdoU}#GD47E2kD7`@fHVDw{E|n(|!e#*LhC$UFM~$4TC> z1b(YC6lwnPF+h4IQkr6`a?!7J9w2n(l*-wD@W=S0O2q8bcA1rknAq5Upok`y!vG@d z(XFl0S59BrTS9raI|u5{@4HbItDr0qtoi+o^OIVhArm@=t^i zV!y9K>|Ws0*7!c&I1@{1lWVE~%<%H}tYAT z;DD$cd11z%QnO(uJ@H;A513)uLnsL zku?BUFzS+|ZYH_S;RX-%W+(srxs|5{HS5KINKaMGg}jpNH4o+|x)ohEwhl%42$LUo zxjjXZ9E_OORsNa1zyU^Tt(wy%Q~=roH0tjC5_n`5Z4T;Cm4Wv1hJ<`m2fsG#rl9eX z`8B#WeZx_5@`LlUJLfFBw%Mk9|CzP3uN2CFY=`v=;NhJ3lHA|j_#g|C1?B;m8JYKq zroB7oh*+37B7teprT!!g)P?i-cHXA9(4yYN#__|<&yS+>pgKx9kh4k+JONHflBrv( zIB1g6&RhOc09WdAacEY!YR(csiW$!A4xl!gq--t)$)OjHRmBMzmzM&bk977q6SoXj z)!W2y{DQS9Z*cm_o*bV}N{qfJ8ag2UHqq<7jcX+yMWi$f*gr7XaGzK$yK7>kE7kkz z_rieSNxd@-YTFcjx*Jm2t6l(P<9-=G{c490z5@#`&;b_D2#q;uXhslk%>(u{TZ18~ z7;aXzEpn>WD4S)MZM&|BWf!24e?L^uX>&=^cYM|zC0V}!(!`nCH5?5>ll-)WHtB`Z ztYq(Is@}G0CNp6U6mu&S$j5%20jhg^K4_@o1CVMH6>`e{Dj&u)a0sZqoo4e&BGHeF zzKWd;RAzj^6#@*lvPEYr8K`_I9~}3n`f5*MWG>fD^XJ9}HZIGbi)Df6ejONK6|woN z>oz(6##txq$WemD9)M)}-l}ka{Jsq2qf~IezQ&nUSte}*m@$$Mf=0lUiv$YqV5w+L zIpA5YIgU1ew(6GJ13War%y&si8pP;b$psLE=972DXAI<#FSM%G=K4h84Zu(-1ErTK z@mlc!PyHx31qAR87*O@B3mK%%>F~h76DnnmCNd(_r@b%Uesce_pF7~X76Lq>XLiAZ z09qp9h})|(R?_pO(Qbyo2v~Z49YiyVRNq`mX8pY!Wpf5D)E*2rrT`#39^`(sulLzp zj4N8$@B!1rb&h@Gy84mhl!__tt_43NR^7~ZN^MZYOV%{vv)|aLB2;h!b)?9yr`BM6 zZbjGo8e@^&HTw&}f7^h<2%~P*5s9)~D(RANmiNQv&YxNUwi3>R18qS61yry}2`j2; zMrF3|=K$7i$t<^->bF=&?=c$6SDC}eA?f5ujv!fta1BVV7|BzxV|hy9Y!ocuD;u4v zmy~s~oDB^>5@XQ;pkX?F)q1}J43x^sTmzc>L2dxdt7hrY_uplf9rqXGP+Tb6Bia zPbE;b3BMN}oI;tj`Py?q1bN2T7?E9bu)mY9qs4^+W&B*_<8Lc4w>>id)TWD+xB&8S zX44;ORRTfa#v!4aTpjHpek$>Xo~@bm>-k&(74LTkwr9wWXBYsM--LKEV{wdww6oN7 z(Ew5qWQbJmnLZ>mQf&rD7=+|Ppcj4SZdL8A6X&VIwkE3Ln<7rR2v(&_2ULW8yCoka z1($#0aJb48DiH3WH8l%_?_}jCS@JXJ-jnP$6@;pNN7-j3od=uw$L3k4 za!Mb@QRNKC=i2l;A8uJCW<2ym0BJ?c1c>QM_>!M}Eh`!fA-qsNG#e$dSC>&vDCMe5 z9{((VsGjLD0%w1<4CiS6XUyl0O}EVH8JLR0E}Yqr9$?>?Zev53kr@pe{VF+cnB=l( zzSy}|yKVX2r55|_rf}?X&*h(@FxiXQb%_$YQ#DahcbD6$24W6(N`j@_m&>nEYL+l=nhM zm1AG=_s>~)Q)c$AWkB~-8dTVd6+rIwyye71a7wn#Z-mJ2)*)X}RrE<422j74I}HZ0 zH`N1dZ5vcVn7Ga~^-P30S#*IFN4+BTbk=s-Jsyjmr8@|6w>^|4C;4NnXFXqI+<5FRue`%uiZV|L)46p>8x|M-a;$vxlb{?{6RIAn z9(~}vwDajpfc)Zy_VPsH)f~Md?aOcH6OXCTzkc=kg~>D9GGx!$-6E#Q6y}4XIQwGr&ZT zL3HYO(;!F0(e+)L^01H#uCGBJfxpw$BIUCVSMRB_As4$7t$=<}?E)JDch9o$QFLV% zDrM5R=R#>NHRtc4+ZX#Biuhad3{%_iMdTg){Vwo`_NfbedMm{7F}^Y#UsE~puvG;l z1@wz=OWD5-&m{j!#F5hp7wQCm`*Wu*pK8!%t!)K0HIKQivs>2p zG@%caGp>QZUhu?

wC&JJ}13-qnt`iY4FASFn$GmpBP>05W_L3@f2wtfA@# zPsROsCD1wNcK+FFwSXTLVT^MBl53x~z~wYFC|Ye))@>_?ShsuO2r^}0Am5O32l=I2 zM?w5#AjG-nd^Wu%$$J)Ux1!gS0XZ!$l1syljPJSLS$J1F?WEYFzZvwIcq{P}5kPss zqM0nyhs5i8u|BuI8mtW4W`mMycbC2Y_QHU_X`&)zgZ(c`kIG7 z=`W)72H3dcl_hP@qR0kGO3V^nII?Z`#X}qxIgOrj8H7>_e@cP>HjlE7RQ7&uFt=_7 zFG>9L)C`lxil%PWY<$OO<3no-;VxG_K)NNVn_V>=<&#OEP!p#z&!3ZdRw&vjo=cx* zceW&x_^X@C-VKpEJK}DXyA#kl|5N^-;TKZ~JkuUG{sZoChk&Ad?N~TSi-a_{9xkwp z?`*t`8ybOw91OEg(oOn{zJ=kKv*9n*Dpy|aG}e4;ybj-S@G%jL5%ySV3~R{LJ7`-f zQ~aRp$f3Fz_Rn@%f97?#F-m+~w7T_1*h4o{yC8pA({-TLL6x=uGFy{9!k(#{$^(;v z@+13Qv~k*A4ucP6zt{rzrW`muy=aGSbF8iZV}ux}l-DCX1YB9Mhpui}-!gbT!X9_b zKOhftNjQE;sC5{2c8j|f=M3uVW~R6-_3Qx;-Hofu`)2TRB}kAlRVj_^kwF8YSYW~t zNksf1DRm$Z*3AMn5P!*Dnkxv{LrG^82=*(^y8sw0J}977gqKeUw{k-^pQ0LW?7P-= ztFlKe9Aa7D2Co6kmz*ZotnugkV+e2iNWHzK3@PIt4~|c{A~aul^C>vj?S94m_?5K} zR*`UK_bvt#B<@(8OPj$wi^r)DyXq_dggYSg{W{cZZQR%) zZAi-qQ?yM!h4|SoJ~v~nx+a2K0dm(eZuYbni0)ernYAsAW!hN!q8mka&xHPrc0R3p zf&M5THU@R&gPOpSoRm=4n)wUIm;Se98&)(4ov%Af%UYL>YydQ`t?{qJhNmO) z+^wBKE$5iMw6?f-1NB)73MB63EX=ON{}Mm*@W{wReM9S*@9RilkP|1AZL1t^lnp9| zI5q+ug28F`W&^vlZa=SsovAZyk6lhuUt$9eU)U1>2}y|f$58|R^nk41=>ZFtohp*2 z98@ZXjXf=0T9}_?5Rwh{`qK4-g>L_xfPzyA_(5^<_EXM$lOzAUDsV@Ai|oeF(kced zx_3-JcNFw4_8k~i3zt?VipmP}YqlQYltczP8L0U`kB%g~zONpb+7VBMf6{J3&+*E_6|_72#mvjd-fEqh(ha8iW2{cs&1pWb^S7V8 z24y|J0}dLt1H9n4#0JRkh_5^CIk=(j1sB#<_6^&s_ILF3b;W(B!u#dDj7=imRcj&$ zeu&r9wF@P{Sqs(OIi2v$=k4F<(sPBP{rB?xTp{dGk=^n8r)zj6gg$>u z=l}bCFA=SbBjtLAhJ)-7)5CzYe~f7X+Nc;OLbaxIe@zIwvmD*ZB??(+Q9b-O$Uw!2 zD%lnrGjXMr!Ybvk0zoEdD^W?mIDe28P*W>2WB8+o!}XE^W? zn~V(qlAKp#Tx>OzTnQ3AjqK&hyoRQnvmW=gRpGCN8*vdA8loM$hC}H7U;?(UY2ZaU zg1&Yj!?!3?Jwf^1*64kx(UYgOuvM-pATTrR5*x+GhZV{H0SL4}SKZ!HReDFV-3|yb zfE+5*u6PsVuoLqPOXl9G2^S6D$aON1)wKn=tCYi!+$Mh(frNm3BfFfb2mUEYnV0}@ zHHMg)^#^i#{cBDDBS2Rn)T*Eo9=5niBRni&-f`~kKxOht=<>baF7GJIuh)-cH?vKG zj|Xk)F^j@LJ`&1S!t$r%c+S@|7`CCZ2gu(OnL2Vjeg3)Zd})^>$Ub$^bT^=PAP#$M zQfUXETlDD){+#Shz5zO_q8Q!9n}>lU4Z{<8=WD{`SetV4$d)$+n}#6yhA+Z;2t$w< z)V1_JP-rb61iOlzubse8eV6acgUk09@C@Z7)`BsPcMVmGBARXeto|2ST-H~(HS;$& zVghy?2LLw})@%$C&$d71`Sw#e`LZzF;nHdLly5NRrDsZiu%G3Jsm&J(b0(xgja79` zg&Gjh4t&=~KJS?(P;*$_9pWHQY<_;Q0Oj4ws1mdqdBE;Ff_mdccFZ6ECfl7^*Z|Ux z-m%-QC3aECU6+6$kCtaC_y*nh9lx?%bF)`PmD`eke4oZ?sWj66XUOu!_Qx zJS$h661Zj7a4k<048t`{=duVYy?k*e;dW^J&=(20^#Z(Cw0|9(5xrZ_?ATp63)`AA z9LaHv>8UboSCsDi-XaXQ)(@hlBL^KVSv&Clj4o=)$A7-7va^N~nNOQ7DB2qGA8QrW zKBxchGfH!6dZ;$w^LwIW36kX6BIE%wo{6BgGOX|Qt&+EVRL#cdm8XARnf$NEWu;17eF|2(;()HjtDN0U^rO0Cwx)AY;Tg`x#OdW~akjH@>_{2%@jn4ab#Y zcT4@2(nHq=S>uckYEzulCV!T9Y>=!#6JN~>#q+#zMkPJ{eN&!;7D+)Gz57|oPooEY zNwMk&;zgRMH9Ru}UQl#?$ln3VR9g=l+D%-^yL(fq-Uc)D6y~#5+^PX-ycY4SFSb42 zPN?cV5TvoJ)jJfFvvElq)v{<57jXM^v-uvNVdvfZjcx|lb512%qiB<`AigXPvsPo7 z>TEu$a=~sT>%H;TY{9kioye6K>!h@(qLWP012~__2MMC^cQJGUiSOU%J9Ldy=lM&aitT>e(3y_n3F)qcXR4eTkkg+ruA=*~hSa)4Z{qCFbjz zj^pPC0-l!0?KMp9$@J+98d{3V?_Y|)MfQ!_ zS!xQ_qeqJqDgXL<4r~YVsMl@rCC7BKV?(SD{S3 z6e4V)6X&=60L7q8>tNSkmJma`q6yo~vKojYp4;v6-&;GwRVWbj_Nx(pP~^<`#GXJ% z|5Lscj))g~@~As2j*_<#tLlkkuFml=PA7!JS(izA->=yzYRU>WkN zdSo_|fR%}s_OJKz`gd`d@VpniOY5ZeKZ`w&++}{Du+^(>|iZZ>^Zm`mCDoj8cp|J{;Ad)7|yRYFBQ`oBM@^ z@y$6^%}<8yAZ{|(X8!6dH3SiS85L5ejF?!MKCk5h&%bja6xWC{TAS~B~r$NezJ$<&I{v(x2{eVyG~5YVMWl1|aYsP|!-GSU^* z71Y*3t=xeoNVMNv$Tm>FTzKdDidp?a8OdoMgCIH4V~A6pB#YMIEl1Bv%28t=SPl+> zlemALuPS!zw`eS$$kb?)4AusH;#)CX1r>wH_kL4BD?vFlPl>6h{E`|9Zm3ye);Xgt zvDhe<-mUc-Kpr;HX=P7hGV6jdM#L{I1*Lo3eqUuT%!Zf`_EkV5iq$vIyUoNBxdp@+4~H@G3bh&WPF}qh&eZLv-ri-#DA@Pha0o0 z;H(hyXns;~#nzfh5S2Rt$m}Fi-EYa}vFE7%D1Oms_}_o|elPXPo$E_1C)&}!G?yjN zpfgXIW&KIXJljr?UnK^cC1RYFKn;I~_iPc_2(P1E7rLkK3~lM+fobCR((#vlhp?t0 zVecke1yV}V)2}?+)O#uJWa8!XPW_oL*|JMP!|IX%eM7TNvhb;x9ke*}p(l=Hvh%VB{7oIMH#KvIDBnWq!maBpsp8tNO z?iU<}MtrpM`R5<=OLb1sN>@xw4X^v{-4lw3I)Jt?+7Fy4XlyBKT{YC|TSU$h9JiwM zG`1MN6#>;Lr*#h2&TR=YUKTaf|CwYwRxegN+i5P>H$IVNlTw_M;%_N1**L|HFmdI^ z0cDGMiw-wKrd8L($3*$reoLXlmKe_xq;u|O+4krzKv^f48;E$8~k0W;dN+7?^ zsZxFcflYD@t0LEN-0AG!HqU~ zEnVAOU-!{6XN#^lVN9no@0RIf)&CswmpPp)E9N#$d7YW7?u9@92AC7)Ns$tkZ$FDM zPX+T5PW<@q(G+AD=XbKEb!6j+7W_A+dv$ejprD2G5hCZ$nhiMqz~ldXGtlhjSh)Bo2tL&-@!gdI@P6 zS>nYu?Mxn7B7@&mu2l=)diihYatg$Q6=B9+37CDRdZ|{2urU)VL0NlBHP5!-ZpJ}W)0{H z=Oe0r&ewS36uZjg6(@A(VdKQ{?&JT?_9adl-quzLNJ%(?=r1l{XJQ$iDN?i%5`po8 z3buTk`Ofn1R}Te7(7K+-+@^O_R{y5>=QcnPf`L*g!X*Yw+j#!ie8G(dg3aTU|F69_kEgP2 z8-*ioqM|ZX5~V_jGKCD?CCN<2Ws1yGnaXT6hf~BB&dH4I)KYr_4>pIW#IFETW))203oa3TOI1emNn%BXfs&-JzY+yB!O%etfQzD0F%x zI9O8_nq1p-F0}rL!TnV<7=2^~Tt_gE@=&bA{&DD|C9b3)&dfoxe=w;p;>G-(*9NWM zesOId(q7H1LP)dy-~^Q>6%Mv2G^2~iyWMj>M0WhDAWbGOCLD~G!x^hvf|Ea~JfO)q zP9)^j^n?xH^pOO*lPUVGr6B@#9sw?ris$?LpPzrS`EQSiQgF!r>PZ=++_Q==#>f>6gDVwDd&rW>yMWS$POs>a}u<>?M%s2CV4H zPa}j6?jkV!Ei#7p1b4TVFyXUg0~mA`7ZS93suG%Xtg~vQkl#5I6*sf+Jjp{A3a)YLq~C z(g$cadz!2!kRkFueuSB|8*HVcMFGIE2m9TB@#!ThIa%UG>`s*OD8c!8+oXi-q9+o~ z8tSm72I{c6F!DE#|KI}ZxNhTm_-|6*ni*aIPfkcnvyS5LpAG?qDDD2cO$4IfOqRBd z3&CQZ>#w>xX}>jur#Pstzy)%b5Ot<+7StK_UhfD^{YZ;lBi(Z6hbonjvmQ~HX4`)3PxN5yN$S@c$ivbfRWyJf_bsaC$uA%UGbp0@BENk1$u zRX{zXpckZZDz{jBaP_;V3fk7a*)3>kTUh1Z)cxTv{;>E1cIayGMN94n34aBN7+QTk zzX#HO|kK{bLX3U4obH8_#ABbE5qZ0rcVvwt|{pvkl)6!a^%R}r6Qo!I2WJW zPpgLP`Cs6{@wREt@aNGLXb9$+$?>Fqy;_4f%Z6-xTZ+Z*TNjDvDJ=S&U9BxEtJYul z^Qu*0-)kYas*2Yn^b6O|60u(@I&wwZi)0F|B`jTh@MBp{vZt>?*kym7$Mer|HTF!H z{L!027SoQHoG;hEy*ivp#7z!E?B`;@H=ZvQx1JGBmCa!OIRh`qIcQGZli%l)c5xwB z$m?r-!{Hxy*5I`PBH(LC!gw`Fx(FyrrEFMS^WJqVSkOv@gTv#lpk+;^j=+$WZ-^gn zQHwJ1m{=qFGLz5cbY#V@>MFeG>7)0O9oG1e{$B02R>;OJC<>F18IdtmdsKQ-d7eAhiYXq^em#m0E**Gpimm~d zY|oAWerwa`Oc67JAoMvGZPYyW>=jMWDhMeqiTI6K$CKGCZ>}$pTv&nR_6m?n6M#V6 z3;c!sM)v}k1d!adWDD(Wbx*%AVX0QwGbHR@WYZYydSapT^ve!B!#Oz>#g%Px2#^@; zd19f5RPnuXEUMz-tFKc;?!2~lUK-pLXL@QdIgZ)Z&FTa5Sx#_?UMCW|{7 zM4R_^bZfgAwVDl%6lki;nr%dv1jvsceBk?6W;tck%q#AmRp8o+O>dZuA8c}A3@ez@ z9b+xk|K+3DJR&ea@T%X6Dp$995*0xoQT=<&VYxMYUz6R7Sb^^nS4 ztjEgD@y)3$Oo%wntRgbV(nYKKtl)>X9VIA;0PC6U%ks!0laIJ9 zpYvayL`C`FwEcs{t>@-03ng|`zW&jq>gHTiJ5gT8XgtYIg7Rf}hxH~<1WmR!S!Z$( zgIQa2pGcPsKDQ1W(2nn&nc&=!*#sT4V4X4H|ZzD$i6e<`_!zcSyQCmcIzuVyhHu|1!co5$;zLmCY7vZnr_Bf2CjBfgXGV>_=l%+n?p+2bH4-o4NGF*cM>}D=$1BMUAKWxDbr@gpC8v##Eg4>*%%{Wbt~Vi zOL^9*GdHO=bFsNcYjTi6J0^OE^Vczpi3w@OmT+E)gpjn$M0<#Y>`z3$f~4>)(qxn- zK__KUj<>2%j2Ot|SHkz)pNXlA5P1JbxYPZL_`w;EL~CD?i3akX7CH94W^&zC=_F>6 z>faqMt%i8!65pq6YRTG5Oh(_LWe5?siNXLSc}jVhM%p^isfK%Qp%y0CXY6LX=#gmD zs~WOU+n?NW=DI(>6`dc6R_gxCo4JS^qr8Y=4MDWnW?Ma?abo;@qDMzTRuz__ zrE{`9)2G;15BIi8Gv7kHp`Jm0_GVS%AD6_HrBMK~8jyqc1vqwr{+2*$lrM0wV(am3 zRayC8jI8oquY?`5JRkH3ujR&!acFECY>kghd{puW`U;RReR8af$Zp2#bHWKO91bJw zrfLZjrNPVukMSs(&vax8Ze#Lw8Zn#y(02(^U>dzSKOA*?N9$_)D!KsK4p`zHxDV4P zZesV-ztGAkgTQBSf?og6wl=Hp5Z+T<%-MH!L!+)+PoCGI?(00Nq_2n~3o@q0nUVKw zU?i7b@p6%nji^VrF<@GXFt+%tu(5QFew>}hR>?PquTfZ6?h!B9jkmB8Tj)Py)tv2n zFK7CY)!q*zt@dr>iO*d|-EmvHmdjQUt%*Yw!Il#@63+4-Cazj{yC|Hc03S=u&~)Uj z?J+LS)SI*7^_S0BXgD(KA znpT|L0(#w2+o-vzMRhyGM+LYn8b0jKG&iX*n+tt0n5e?i`d!RA;y0eSeOXUP(uo)) zm>SXtqFpkADI^ID8j!RzUFyg;jc)JC9RRAAeTTsYLtqxg(Zp&~70e))IXDuH8} z%9mpQ=D9cckhKbF{`=?u2p_5cZP3NOlmQ>JT-*6eLlr*JTgh)LVhNYdO*v(xW}w61 z^C*b>E++lWxj1wOQu)+VMzvs`kCs=kQ8zu8-GTH0tBerCpUdIJl~)NieSD?Pg~>2~ z2xU2m60Mdz-S8boV zKN`CYA15_JaN=?(lNeZsI#XQ>=E1Z;iHR-r9nHxfDd(DigcuC8d|OIc`3(k_#AqfdYpVq?La0BN}Kx z^yW3A%uLPpU8fU~ok(9SCf31$tUcRGl8dA*Lz21i?>*?0O25k0ZasML-~=?3Yk^>G z*wi$1HmOyO^q6Rldq`>);S{V8e*Dt{v>a^hOGy3+?~ko~%uM8qK005JUUh?z=|2Hk zs})Ftyj*B}w|#uP_+dfD=|8q{6?}KM*ruIIERKslOJ3t#R50FvL5g*99wtL0fJmlI ztx&Tl%#7vb8LJ5-he03e-^_sHL5~tE==p~;DDJ7_5Lh~Q9XRdy#;@Akg;>Kjad z-Vr8t1rjcjv$Zp)Y61&j|1D60U;-_qQjxl+l6y??m8rfXjb+W~*nu!Xt6)Lf&rMG6 zW@?Q14-ieT=#!KJPASwN%sq-sk#!r)EtM1U&ZVXoXM>1 zgOE5a+KA|VSextl7U&OXI`c_@bmJeJQ@*~voqK!Vd6DB5r(nj(=iIJ}kXERMvPJq7 zxebS8&DK0hGbD2UXcDR$Y8vA|KE1u?IS&OPgODL}=~u!MEOvUV1B?2f-s(~b4-;Wt zLuNmnLfjp>rKFOKTE%oIEc{f-lyG~$`RPQvC(!RkP6Ek@gBdClP??oI+42*z7iD)7 zWQEi(>L1$VI!W2xyM-wRiWvo~@N=rM2 z)RtMpP$6gN5_Tw#MH!m=b?y+fsCk#K2h}Ru!<@pJkoKC;-BQUKDKY$H_cD@SA=}xy0f$)z_ODx?=*8#rX_$_HSGz;CDYSs)?0rtJ?_0av{+5%yy{& zi=x!>9Jj%qUODQ0cQL0n>7zUiJcEZHOzhLmcbjyk;ZZK>03khF$wG&!S~|D&fOoL#C_WQ(l_!FQ;73G zYH3cJAZZ9uKaRe%B#ISkX+K0sc0kMJhIWLL-NC3GbXlq*UQ!8YseprLuo=FUKjxv{ z(_8ma&6uW<$KbNuaTd+wD5);cA2?0> z@WyHBeFdx}L2a7neUyDvN;Tu0NQrty_ zv>X;vhL?Adr43BhZ)%bWT>Fcahd0SfsPJ_UmY?ku!OlzmmDdyQ zW#8Nvd34Ro{9bvJ4!lUpbJ>9`jwK~cmYnyRu#Vl(8gORSX0Y2BE>U0GrthxC{`($b z#b{mFioM6U;;LVus}xaEa}?bl{uU;RezJPQT~r)J!TkwWRv#KY-x)0_66YI?9d(<~ zPgX&QLsN<*OV=QOY1_ZOnePp$afAa)a=j>xx^1BlnZt`cIV8vzi_TK&dem)WJE|10 zXAhP`_@2>q{_9r{j5H9zs7w!^M`(CQyl6}~mK^y#k>>RtS4G0X`2HkSvE5z#>dymq zxbkzm*jH9&Ww>LnhG)GqczV6PvMR$#viKryt#`$=cgX4fPuxG9A$6_A*>x^t0~U58 zG_mdAIu{9AaAQP0}G_<4Pw&B)!p-(I;_ZaQ7e7(ms zU98kK%8#S(OO*-?%OO8$%?>#fzabk!HE&3%Qf~GN99S2k+|e7DX78*E(C8FzQtJz% zThL|J;0Gdr2U^l7$9RIja_=E~x$`Dt(!We$L zxr!9)`W+;SJfl)1$Q*v)?=GN>L!Az;pF<-kyBw2{rdlGq^1eId&py}jP#TH-|IUT~ z3sB)ddqiw%Rq*#rot~%VgN?Tgw+5XOHe5kxsZC2%cy8x8PQ7xB+=Y716qKWS0B{db%KJ9DWdP=rtONeJxSYocP$l8 zb@5E0y#Q8L_`gAse)P645ea|h^p|-#r6G}8-uXsGw~~J(e5w@vjqqucUlGNGUi;`S zsuXdH-@k>8^TyC#-L;Oy1TlR8nEN9`C%N^Q7zw#n@(K`sdmMNGn9cS?+jY5A^oD*x zlq;=aC)h`1bcS-*>Uq|EX^H5j606LrQ$elCTVC%ueA~8&?3UIS8~j{5ZFN;0kSkx) z5)S;~rV)0A-LCE81>Y6Mt879P&?g$*2M`qKVth@2g9j}_;tjXes%05R8bmHQZ%479 z7v9*%Ko&bHMzY?XVh%@d5PQXC*G_oVsy_mc01QM^cdQ3jzE{9-V{%5i6@CL8@@1)B zco@6mzxI-R4I;mO>EK3{yC$qo@!c+e%EY&F1JPtlYS_Wev4KaraIq*qs z3?v(N0d(PsMm4)tXt#uJSS;;SQ9*^<{Zk}a?DsZc3#k7hKiMJcw7zo32 zpABQM__h%@Zc5t6Yy=wbG1uyfhyyXsp(InivIA`2l6sd=zPSshuFd%!)w|v;6L@OG zzm4uwr1dozbZI_gY=pB|evd!|2;m0S)D-V0kJjMJ4+d{Zu1lKfETDuTn{llnKr zoH*v=G&q2#h;detZ{PjWXWI&BUa^5Pz+XWA8a#ec1;6guJ@-A5V_yD2GfDS}ZT@%z zr-3gcZ&g$}jMjF9*(LE|wmF<_?xU5x(99~bCaY1g?ugHLU-f15)YwnkuD+Mke^jtA zo*WO}C)4uh$l40M&yCqF8H+`8QVHUg?;U9vPmwJ&mFWeiphDEtJNBAq(!Mp&2J6^O zWd{g+PH=AC5GJh^;)gb@mYM{)SRL%&;fzyM}v}A;x3F7;fmSH-16Aa6V0|)Ri zkCA{VdEw8OPHFl;5v6DwRQnGmCn{*X5bu0&YJ|%$z4HpR9xHF%>oN98Z`PdIuSGgl z!>xnxwlkWB*4EtlJ>o;Miwi@Gk&sOKQs^|YwljsE&@*S1L^;i^DDWvarxsU&jFMgL$Kh=m zjdGpqPVCI|HOilH2uHEf(2)%0UA)KuXL7vPbcmPikXy&>$R2^*dm6|_&K?0!J(DOV zp!h8%!*tCeTibGI31n7YkcQcOk-VioK{Fm^VG5bBxIYjVvTk~yDAE<^*lZFKJ8-VK z;g|k=*0(k|yZ*cg(HkI2>NQvD6>hNkL05c5FpnI#_xnfR%eY*m^X-JAbN8wF5%xGm zmi5;Lw|aHWSeVl)rf2{DU@tU7Q-H}HdY;k)@7|qGa)P-~vEa}aYJ=Gd)1LiTdCN8O zFu5EJ36iBmc%L$aW@-vhEPToHLJ?EPRLR_{M$Lkepy^2te?Oy#x`3b39t3o!+#FTJ zz-Y)Gz^(fiUq>5MZ`VR$Q?MNi;8QP@+g2yLx4{Q3rRiUw5mgi*;9nUqQns_Ya^Vh~ z)|rCH7|y5~LX$U6E_YMSf=Fn8(Mz9vRa*3m_r{thW|7&&B5`?62MkmXp<2)n-y%P% z6o^D9Q0%>GsFPu?vJhlrHrap-9Ibh!o`B%itWCt_l3mm6Rqe2`X^JZ*9qFae!LJ6! z{RDOFn1IG@7Bq9u0~F8oL|oC&fi}XerD-&Cef39tGR0C(qQuYlh!k4sVq9UEO=SB- z7QU+^o#yU*OJhN>*JEo00T`a=Wgh%0MxYdHa?|tP^XZba>0>u}-S;tHez(oy{iB1^ z6HyDjQQdfkGKouEvk!38NB&~X|LK%mA?e2@7A&4-Od~mWNhjOhM0BFbxYkT!^vT$l z>oi%Vk3Ejr#C$4~%uDdN#6Kw1*wg80P4G+kB}oOFDSDm;oT!+2fL9F8Kmo+c$Al%X zWLJ7GKe_m2ByV2n2uA$7XBvj)qUKvuJmn90$8^Os0}4#8vyBe)?a7cxch>$h_wx4= zi_{l-bQl;Y*{*Wg8m`pY(y-F%uFRi{?IvGd}R_ILe6{Cnd7ykbKN3Yr&5 z1UtSyqSsDFp9TCop4(9SpYpnGSZ-`<%GHrA3G5Bcz5r4i3=W--5>3C%%keSD-O7RmBhmi zL3j<5SU*(drKLWd{Srg_&iUE8CdYpX=jXrPziS-9Q~H}rW?5mvZYM+Z^dP=K)to?l z@S9g^E96-e4j0v%K{{|cU!rH@z~rWWFz!9We)yw!A-%+<)t@BfWgg}-2}jL0Ezs`U zuCZqASl1B!PZfgrms8?CaKeg!GbQ3*{>z>HC#u)*?jDOGnY}KJCnB0HmSr(Edr3wMF&g)mige=UmFrNw z(r|(3bYI+N_m#2_sGR4(N7VY_qg(RSzB=ae?;bPb)A%cPofxbdnRCnF80T!jl}K>U z1YEJ%!OFKE5P>c2V;mYGLUAOJH${si%l)DmtMXC6ffr5Jb{$!x4Xz>HdxqBFz<(&eu`cve)79xekfc5P zQ7OAq6}yYxP)Txr%n?!~yA3pTFH0+?+P3A)%|c_i8s;mXAHoFSPYbRv%SE_i{FlGi z2@}nY;V=xM_v!7u=U5hoZ`ZcF4MpI6jlu?-b`Yfg8PzW^BV3_yHn%WLV!l@?zHrL7 z@GRswx{HF3vXudWO$)PQGvYOasoAGdj?ed;l>czi!d#D}65?%K81K_86cryX+Sp4W zIdwAZ!0dPuxLon`nc&!QPHZg!XYiw~BApNBsG7iy`#gW5h0Ci22CGIq)4~_*LBn6P ztPeI_EX*8Fx{}XTI3hTE4>xeG>7ZP2C!Xz&{Ra+)_h$esb@4>!@bD)2TncM}Vh@-} zm*1;kcg|~Z(o3?-0vZor%>lo9Et~&|M@g`nBMjD2x_jm401Q^<%(aKnzdvQ6Ke-XO zb>nNGF>(Z>QH++E|Eh)YPJY}{wP{ka7-xoUJ-O~3f4&0kH~`2+0J>bCD1i2oTWYG z!YNUho>5~Gq8IFS_D|;%K2H4=M~QBz2vTTYr@Z8j0|#Qr?Pq)VeViu5TqMEaduZ}P z7-8)xa#LOJO3!NuO=?t6|fS}!y55gxLuM{6J-`LhI! zVphtE3OHB_8SWaMN=eZ3`QTcIFr_2qMahqgfZ9=B;N^@ljWS4qcJPo7Ep}6+TYjXP zn=!c7*}Iis<~@Fh>tE7e}+dF0klE4lQ)1)d$eT|N8Z7 zZsT8RMwUHnx#LpKg4PrLn6_`)+;Td?kj36Cz@wjc9H*Bngyh+4q*^FNIvPUq{j|Pw z*L74^_68rUYgb&e-!mqc8TEf1a&x3jFoZ%OWM&ouGZY+ffT;Sthg2#V1wb1 z44&7~OqS`7g!NPsTSu9RWS$~|MUczBrSiQG!DMmVfpFR!K}SM#0p&K!T<|~63|(=^ zer}EyNQC5zKZmhNDKm1iZ zY8~Cu0Sz%mOI^A1KSfCPM@&%xi&kAa-avQZeD%7P+DVA0GI!SRdYZ~Sj6EoBmhf9j z?26iNSVaE$nqd)R*)R%30`^tdj#K zNWGpI8-bn)*vl#fi_#fVxMMe?t`_3&z~|lvi3A!~2K`VAZOuaxMlb|A)))f#OGS+= z4%G}3(IU%-;`3WJ>5TUMLvqD(@}&4r9O)+{BZKtAEi?^*LvINKdN=mhZtBE&T&vrH zLMRdpNJy{icDWoLiE`vs2#ftn50Gxp6yzr|b|MVBwIK^QlCh|FT?py}1i$KG_}7D= zVI%;l$A#qY7Tw9z{_@GLl})1i;p-!<2pMV*FdXk=cU{WC`p64wc*^$O{%z|!kApkG zpv6IcB@OFG3}jx7ZBs*yqO~u6%njK^+HZx{e!6m{GaS(yT4m(*P-0qxNrq_wl~q|59-ccA_TRt!Cm{qWmnZ*MGI9W!{@=-%UiSW&a$d5; zO;!w+&rqiiRMxJ;R5lk#4f!;E@o%H#IyAhHf2B|48&|*8dij`Sl;_opcqm`x*q5Mz z%T)yCT~q|g)?4Sc?_yn#=+%Mg;rhyV39t8XBQJ1^VY!Rm>k<8Vb#MZ|;xhZZ-owp# zz%7b+ACjy~Ka=KR&fY^$y41@rVZHIr&Zrpf2Xkeh?wbA%9)O9e`a$?F44U zXAPX{ZPv4HG%~yvj+6k$52`yU*8|ntLjWpkE=!4Zq2Y%>gZt*+Eo%H$Xx}J$ZggXi z9nwL=-6!&rs2g-+KnN9R7ES+#FW5?|J&1A;6|<}yU`c49Ool{@?9}Q5pn}8jI0z(Y zv|Z_geuA<^gwRN6K?-_H^O3D51#hXr297t@C$8p)DEM}60KWLoE(+Sf+cnz2f8-V{ zdO)S?^+EiUD1YsKDZ`xvA&@*5j>LY-d9M)wG_xF4poxzRIMRu8cr$Qd<$|y^%j&?n zo-3s59rX;LxAS?;_B_I?R-Ktc4{+bnh<#n~o(J&CICWJL%)Zs&t$~Mf?9P9)q@neB zcH7@Fe+?g#DdNT#co%}k8E;v<>i!ch>D;EVh1P7{@Xje3L}A;{a>iY`QP&nPwx4GBrL+@fq;f&gN{*=Hd1f} zHsF$<>`9<-cr5EyO<<-wH!leTX@?Ks6ybVx(2nPDJVB78Ae`e2p7m-!rX$pIy24;7 z6%jHwkLWpmlD2?U$gl_#NMy6gGFBpz`bX(#^ZUCX_fPNE7g~ZWz;cpegeN5gr^|j) zusj-)xZJ}$J<06cw9jL4wnNEnAXNr&_0=K{rS`dQ;rOWe0kd8tk+L&REi1-ZyLG>Z zqR7dVQL(~NF_*FXV*25yhR^Rq$s3zSo)tRAzi2D@t|RBkBca`TGM}D=DSE1!ahFSW z;R3pd+O;D+2d7$S-O~S2IBqs+p|H1oaXwpM zp=DzYlScT%32Xfg!*52b<7T|tBO7$*Xe{0)M;#Wxb1Vkp8TQws$+UdiNlkii0Y_wh z{LF@Cmt9zxfhrzhU(V_BRY2^cXnTHr+*Q0YE~{-#N@cVTX{(TzgJo&HW$*0bFwOMj z0E0&1RLOX6S7{zkRN2>_IrSFL$+j9EFDJ#Ec`v=DyyAL|m%rW&NLU%Bo&B_jA92A& zwGs>-@QjpKVT3W>mA2v!#63sDI-dJ`j>Ys&_e51sG72YOyt5ZjiD8@*)h1Y#d9dt5 zxx)2!3#Ki@a}K>d!tD<{ohJQj-Y{SJz@EC$-0|GziguuLlkw%5`o&q#nQEnZg_fqd zqQ+l!{0 zfy(vh!=xTdhxPaZU8w{bbsZFTGr9fBN)7Oj0!L~u;#4(Q7zwF;bCY_sN9q2REMKF+ z0HX7IrzN(BWiHvVzf>|b$!$`s#y@%5(sf*Rk!^4;XCrmB%OS|N%;ZZ(qnX%;^i4H+ zPLS4%QPG+Halgpo}D;IW;QK!4$|X% z(6(^`G9eYkgoc`f=nXWJ_1n%Eq?;-f%qJMkAD$nJm#oA?9(H2eOG2_Zww&5}YG|>l zC*{JtZwF?FPD*8#McA*1nom7mq7lPERy8E-QI9;^=e(2Rho)8fInx&;0`{o2i&|-H zLG=N)eaMYQb2jBAypAhfa^=l+rYVp&3`jX!VH@+h1$z;{q0)(8@8bTb#Nk3G`zqaj4N89)lCShB7$r+W&1bQUR zvj6TLv{Zq1=V9y_{!-n=A8Z~GYJmXV*>gD5|4~#%X)Oz=ilO!Md-hw++6vo?t_FqK zC;9XfGL3@f^{uKdyuLOtk+qjV3F=U-j8s92RQy8~^bb&G-i4Aa)s5i!sNS1xCPRmV**^vC%bU%G7DRKhdTD1aB*I^ z{#6k9pDJ54`?PTA_LFYhDQ$1POw0N>&0ro>Qjm~tJj`mx{PE^d?v;mf_QHtXbJUDS!pD07)Km*e{^o%n$j9l- zZf_FF_V;rzL>yR7&7a{?c4S6qq%fwjm5Y z&r;39Riajeg<%2tGcSK|xBmn+!*_-*?8t-pMK@%M^xmn#hh?HZ;{z$XkeJ*HrPN+h zWK>sXMAf&m#nAlPb2bBlP2=IxfpXHRk<3CHkykFIqkh^jv&-%8RTFV(R5juACxxhl zJj_z+4@@rhW$arOhoLKL>1 zX;87YXNhQq#|_Q?^G`O7>0pLhpG>QV(>dNSl55^k!g&tpC;S_OZG;qQtb#mbWAFU4esEQ@2K_!A3p&!fo9 z%HGzF1?QOtd_hoB;lfM?gln9`07{u+Khm}#npKWH2W=CC+iE)I+d5*An2$m0z~VuO zQ#j^IUu0er;$)?p>SULEf>ef#-BofoDj$8Odw5mlOK*0o`I#1jS+7ejY_p;6i5;!b zuj$nHm+_I77yTjFm6!@KPxsk6HKi@pOtAR;-ogc;e0{C4(r?hc$=rP+J<29;A~V^> z3tHP~OwaBU^Zb5KSm^$Vk+KqcP2EpZ5MXX>g6Qen$#Ypp;HahwX85>fQ9ak**y)^$ zR_T`YuQ;uCXv?IUBe(_C7}hg6cD6S8cJW~@%v%cHOU~Cyc8N6J+QPIO3FH<;%>TOW zHFf(Qf^(m8WCoD{Qw*d&D*QB-%Q6$Ks&YW0%@F*gSo9FIw#vx%T70I<<|C~LYlG$@ zG@m~jjL(bx6*1%<|LgvhyqI6z!iCwFA?(*_z>|Q0x#i02UN~vnjJ@?tXqNm0RDhH~ z8>RZ=-H(lTBeBfyat1T1-@`iQOpOmM#&hRvMKEJ``cQsjK;@w@q&_8?$#l9eUSj;^ z#A0R)z*{;Cx#a9>Xj>7&W!N!YbtP_qU7)!>xw3E%)L4XVR7CX730w~wkeaJAhm@f= zS)}JouOqaS$|?+%d|cMS4pW5BdXf)>3R>0ZyA1~UUma26&sEsx<~0t#fg&Rif_KE(9-OFDk%h{ASnLng}7zUR5Cms|G-{j4R+P;RQ)EiHw z7e2k%Xlr`qQsMkW1=D8;)b<9dDZXF)vuta|{mDKY{ZPY79oY2)mtgQ;JTlK-e58!6 z7bXBIOm=vASuPylFBB{V2^63R8c1E9{RxxFK`VMAw|hu<+I76=D)+}3^*e93idmQM z_#L3+t~i^7>`5EBH6crq0K@jm^> zaZs?VA9UMt$8de@?=0t`wG(Vs{Q8K3i#`Q(H!#)%C! zA9Pq9Jl-<3KTtgW^@by4T|6H!!omQqj+QDN)$1p>9UuL`Kl;>t(8@q=DH!bk9bSWf zBcW@t@IxdsFpwYe%kMZ7t@Z`c zbSqn9J8y^KKb=EL^JYdR7f0u=Wgu)LqUUiQ3v4-H_D)t$?1nJC;t+8~h`$HUm!BGy zaGmyaRzEYgERT+oUT6K7XP_hslVNKQhUh*@i|W1@H5{U+EqwIoLmJ*(@lvQ13k#jR zRym0&b-S03;@mHzWzZn%DO^7KO3mC3d%2A)`%i2>kO(q=x(qEqV&m%;S8 znfZ>z`Mi|K`H~7y>@BUB0`6{2k?O~1rR_5ve>;H3rpsuY^0R-*5q#z2_oZZrPi+}* zM^(L-%%F`H?Neg+?&V))zwy2_jHN6n%^k`&^M`W! zlU`2mbDF-B?l!LHIo&m_#G4+KmvP_h^}yk{@At)y3<7Khn=_Yc@}Np>WI}_;&}_RENW~sIg>aV&vNW2nGuI`j zuIB{|Xp$}dVp%+DqA7*w4w!JWEda29Jq}tu1t!TtxCHJcR^xToh=)2`LC(mtz>Lw} zePJ}J+RHq>IW>(R9KcC*3jQW`zSVchyi@<2fet~6@yebjj&I~xJ^PYW%J*DIKs8#|3ntcAVW zaKE67p}?Zz)nAv3=`{sHHEA=eTjw0pPkfxaD<7j>X3pXdr$5#mU9FJJ#8Au=Dyd>U z2iV{gGyU@Uz3%vjLZSFw=6R5?W?R^enB&5sn(jTRL+(9d_drB=3~F^RAX5A?RGtb+ z%r;jmiXg(f!G6Lj+rF#Z-6L(>%ERNwDVFj^wzT_34bBt&$|Y9##unF(ttB&osoihe z%|I!{cJQd+q-WJhgN63cN~`F`LN9F=1Pjx$ zMjEC)n1!f!4eDkFrJHsbjD|LrVZxHDoAYG;7|gDu>97tl$i>k60=S^IH$F+k|MhU0er~g5cqNx^e13Pjprz;8wm=qDd3}6$ z{ndK|;{4NER$?H=bOt?!s{5CYpC!3ql7Bznj>65OTxasakZ^hIGcE0G4APRtGr89N zLw<+D$Ed>end#kkMnX6a;djk7UdZrN`Os3e*ka1KDX;S&Z|}_Fn}K;2oB>^W)@WhY z{M@YD;~AfkbPgdFm;P%VZ-fi;r_2wfO_xBiWSeLwp`>#I)@2)F)C%!lcTz>oDDaBF zXD-uPQzRY-nE9t1K=onH_Y6ZNs6+|E2?9=D*CR z^wtl53vRs8c)CM>2mc2QRQh|_YY&m(9WJd(D;(2{{&e= + +# Text Generation Inference benchmarking tool + +![benchmark](../assets/benchmark.png) + +

+ +A lightweight benchmarking tool based inspired by [oha](https://github.com/hatoo/oha) +and powered by [tui](https://github.com/tui-rs-revival/ratatui). + +## Install + +```shell +make install-benchmark +``` + +## Run + +First, start `text-generation-inference`: + +```shell +text-generation-launcher --model-id bigscience/bloom-560m +``` + +Then run the benchmarking tool: + +```shell +text-generation-benchmark --tokenizer-name bigscience/bloom-560m +``` \ No newline at end of file diff --git a/benchmark/rust-toolchain.toml b/benchmark/rust-toolchain.toml new file mode 100644 index 00000000..dcdb06e9 --- /dev/null +++ b/benchmark/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.67.0" +components = ["rustfmt", "clippy"] \ No newline at end of file diff --git a/benchmark/src/app.rs b/benchmark/src/app.rs new file mode 100644 index 00000000..85026a33 --- /dev/null +++ b/benchmark/src/app.rs @@ -0,0 +1,688 @@ +/// Inspired by https://github.com/hatoo/oha/blob/bb989ea3cd77727e7743e7daa60a19894bb5e901/src/monitor.rs +use crate::generation::{Decode, Message, Prefill}; +use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; +use text_generation_client::ClientError; +use tokio::sync::mpsc; +use tui::backend::Backend; +use tui::layout::{Alignment, Constraint, Direction, Layout}; +use tui::style::{Color, Modifier, Style}; +use tui::text::{Span, Spans}; +use tui::widgets::{ + Axis, BarChart, Block, Borders, Chart, Dataset, Gauge, GraphType, Paragraph, Tabs, +}; +use tui::{symbols, Frame}; + +/// TUI powered App +pub(crate) struct App { + pub(crate) running: bool, + completed_runs: Vec, + completed_batch: usize, + current_batch: usize, + current_tab: usize, + touched_tab: bool, + zoom: bool, + is_error: bool, + data: Data, + tokenizer_name: String, + sequence_length: u32, + decode_length: u32, + n_run: usize, + batch_size: Vec, + receiver: mpsc::Receiver>, +} + +impl App { + pub(crate) fn new( + receiver: mpsc::Receiver>, + tokenizer_name: String, + sequence_length: u32, + decode_length: u32, + n_run: usize, + batch_size: Vec, + ) -> Self { + let data = Data::new(n_run, batch_size.len()); + let current_tab = 0; + + let completed_runs: Vec = (0..batch_size.len()).map(|_| 0).collect(); + let completed_batch = 0; + let current_batch = 0; + let is_error = false; + + Self { + running: true, + completed_runs, + completed_batch, + current_batch, + current_tab, + touched_tab: false, + zoom: false, + is_error, + data, + tokenizer_name, + sequence_length, + decode_length, + n_run, + batch_size, + receiver, + } + } + + /// Handle crossterm key events + pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) { + match key_event { + // Increase and wrap tab + KeyEvent { + code: KeyCode::Right, + .. + } + | KeyEvent { + code: KeyCode::Tab, .. + } => { + self.touched_tab = true; + self.current_tab = (self.current_tab + 1) % self.batch_size.len(); + } + // Decrease and wrap tab + KeyEvent { + code: KeyCode::Left, + .. + } => { + self.touched_tab = true; + if self.current_tab > 0 { + self.current_tab -= 1; + } else { + self.current_tab = self.batch_size.len() - 1; + } + } + // Zoom on throughput/latency fig + KeyEvent { + code: KeyCode::Char('+'), + .. + } => { + self.zoom = true; + } + // Unzoom on throughput/latency fig + KeyEvent { + code: KeyCode::Char('-'), + .. + } => { + self.zoom = false; + } + // Quit + KeyEvent { + code: KeyCode::Char('q'), + .. + } + | KeyEvent { + code: KeyCode::Char('c'), + modifiers: KeyModifiers::CONTROL, + .. + } => { + self.running = false; + } + _ => (), + } + } + + /// Get all pending messages from generation task + pub(crate) fn tick(&mut self) { + while let Ok(message) = self.receiver.try_recv() { + match message { + Ok(message) => match message { + Message::Prefill(step) => self.data.push_prefill(step, self.current_batch), + Message::Decode(step) => self.data.push_decode(step, self.current_batch), + Message::EndRun => { + self.completed_runs[self.current_batch] += 1; + } + Message::EndBatch => { + self.data.end_batch(self.current_batch); + self.completed_batch += 1; + + if self.current_batch < self.batch_size.len() - 1 { + // Only go to next tab if the user never touched the tab keys + if !self.touched_tab { + self.current_tab += 1; + } + + self.current_batch += 1; + } + } + Message::Warmup => {} + }, + Err(_) => self.is_error = true, + } + } + } + + /// Render frame + pub fn render(&mut self, f: &mut Frame<'_, B>) { + let batch_progress = + (self.completed_batch as f64 / self.batch_size.len() as f64).clamp(0.0, 1.0); + let run_progress = + (self.completed_runs[self.current_batch] as f64 / self.n_run as f64).clamp(0.0, 1.0); + + // Vertical layout + let row5 = Layout::default() + .direction(Direction::Vertical) + .constraints( + [ + Constraint::Length(1), + Constraint::Length(3), + Constraint::Length(3), + Constraint::Length(13), + Constraint::Min(10), + ] + .as_ref(), + ) + .split(f.size()); + + // Top row horizontal layout + let top = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref()) + .split(row5[2]); + + // Mid row horizontal layout + let mid = Layout::default() + .direction(Direction::Horizontal) + .constraints( + [ + Constraint::Percentage(25), + Constraint::Percentage(25), + Constraint::Percentage(25), + Constraint::Percentage(25), + ] + .as_ref(), + ) + .split(row5[3]); + + // Left mid row vertical layout + let prefill_text = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Length(8), Constraint::Length(5)].as_ref()) + .split(mid[0]); + + // Right mid row vertical layout + let decode_text = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Length(8), Constraint::Length(5)].as_ref()) + .split(mid[2]); + let decode_text_latency = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref()) + .split(decode_text[0]); + + // Bottom row horizontal layout + let bottom = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(50), Constraint::Percentage(50)].as_ref()) + .split(row5[4]); + + // Title + let title = Block::default() + .borders(Borders::NONE) + .title(format!( + "Model: {} | Sequence Length: {} | Decode Length: {}", + self.tokenizer_name, self.sequence_length, self.decode_length + )) + .style( + Style::default() + .add_modifier(Modifier::BOLD) + .fg(Color::White), + ); + f.render_widget(title, row5[0]); + + // Helper + let helper = Block::default() + .borders(Borders::NONE) + .title("<- | tab | ->: change batch tab | q / CTRL + c: quit | +/-: zoom") + .title_alignment(Alignment::Right) + .style(Style::default().fg(Color::White)); + f.render_widget(helper, row5[0]); + + // Batch tabs + let titles = self + .batch_size + .iter() + .map(|b| { + Spans::from(vec![Span::styled( + format!("Batch: {b}"), + Style::default().fg(Color::White), + )]) + }) + .collect(); + let tabs = Tabs::new(titles) + .block(Block::default().borders(Borders::ALL).title("Tabs")) + .select(self.current_tab) + .style(Style::default().fg(Color::LightCyan)) + .highlight_style( + Style::default() + .add_modifier(Modifier::BOLD) + .bg(Color::Black), + ); + f.render_widget(tabs, row5[1]); + + // Total progress bar + let color = if self.is_error { + Color::Red + } else { + Color::LightGreen + }; + let batch_gauge = progress_gauge( + "Total Progress", + format!("{} / {}", self.completed_batch, self.batch_size.len()), + batch_progress, + color, + ); + f.render_widget(batch_gauge, top[0]); + + // Batch progress Bar + let color = if self.is_error { + Color::Red + } else { + Color::LightBlue + }; + let run_gauge = progress_gauge( + "Batch Progress", + format!( + "{} / {}", + self.completed_runs[self.current_batch], self.n_run + ), + run_progress, + color, + ); + f.render_widget(run_gauge, top[1]); + + // Prefill text infos + let prefill_latency_block = latency_paragraph( + &mut self.data.prefill_latencies[self.current_tab], + "Prefill", + ); + let prefill_throughput_block = + throughput_paragraph(&self.data.prefill_throughputs[self.current_tab], "Prefill"); + + f.render_widget(prefill_latency_block, prefill_text[0]); + f.render_widget(prefill_throughput_block, prefill_text[1]); + + // Prefill latency histogram + let histo_width = 7; + let bins = if mid[1].width < 2 { + 0 + } else { + (mid[1].width as usize - 2) / (histo_width + 1) + } + .max(2); + + let histo_data = + latency_histogram_data(&self.data.prefill_latencies[self.current_tab], bins); + let histo_data_str: Vec<(&str, u64)> = + histo_data.iter().map(|(l, v)| (l.as_str(), *v)).collect(); + let prefill_histogram = + latency_histogram(&histo_data_str, "Prefill").bar_width(histo_width as u16); + f.render_widget(prefill_histogram, mid[1]); + + // Decode text info + let decode_latency_block = latency_paragraph( + &mut self.data.decode_latencies[self.current_tab], + "Decode Total", + ); + let decode_token_latency_block = latency_paragraph( + &mut self.data.decode_token_latencies[self.current_tab], + "Decode Token", + ); + let decode_throughput_block = + throughput_paragraph(&self.data.decode_throughputs[self.current_tab], "Decode"); + f.render_widget(decode_latency_block, decode_text_latency[0]); + f.render_widget(decode_token_latency_block, decode_text_latency[1]); + f.render_widget(decode_throughput_block, decode_text[1]); + + // Decode latency histogram + let histo_data = + latency_histogram_data(&self.data.decode_latencies[self.current_tab], bins); + let histo_data_str: Vec<(&str, u64)> = + histo_data.iter().map(|(l, v)| (l.as_str(), *v)).collect(); + let decode_histogram = + latency_histogram(&histo_data_str, "Decode").bar_width(histo_width as u16); + f.render_widget(decode_histogram, mid[3]); + + // Prefill latency/throughput chart + let prefill_latency_throughput_chart = latency_throughput_chart( + &self.data.prefill_batch_latency_throughput, + &self.batch_size, + self.zoom, + "Prefill", + ); + f.render_widget(prefill_latency_throughput_chart, bottom[0]); + + // Decode latency/throughput chart + let decode_latency_throughput_chart = latency_throughput_chart( + &self.data.decode_batch_latency_throughput, + &self.batch_size, + self.zoom, + "Decode", + ); + f.render_widget(decode_latency_throughput_chart, bottom[1]); + } +} + +/// App internal data struct +struct Data { + prefill_latencies: Vec>, + prefill_throughputs: Vec>, + decode_latencies: Vec>, + decode_token_latencies: Vec>, + decode_throughputs: Vec>, + prefill_batch_latency_throughput: Vec<(f64, f64)>, + decode_batch_latency_throughput: Vec<(f64, f64)>, +} + +impl Data { + fn new(n_run: usize, n_batch: usize) -> Self { + let prefill_latencies: Vec> = + (0..n_batch).map(|_| Vec::with_capacity(n_run)).collect(); + let prefill_throughputs: Vec> = prefill_latencies.clone(); + + let decode_latencies: Vec> = prefill_latencies.clone(); + let decode_token_latencies: Vec> = decode_latencies.clone(); + let decode_throughputs: Vec> = prefill_throughputs.clone(); + + let prefill_batch_latency_throughput: Vec<(f64, f64)> = Vec::with_capacity(n_batch); + let decode_batch_latency_throughput: Vec<(f64, f64)> = + prefill_batch_latency_throughput.clone(); + + Self { + prefill_latencies, + prefill_throughputs, + decode_latencies, + decode_token_latencies, + decode_throughputs, + prefill_batch_latency_throughput, + decode_batch_latency_throughput, + } + } + + fn push_prefill(&mut self, prefill: Prefill, batch_idx: usize) { + let latency = prefill.latency.as_millis() as f64; + self.prefill_latencies[batch_idx].push(latency); + self.prefill_throughputs[batch_idx].push(prefill.throughput); + } + + fn push_decode(&mut self, decode: Decode, batch_idx: usize) { + let latency = decode.latency.as_millis() as f64; + let token_latency = decode.token_latency.as_millis() as f64; + self.decode_latencies[batch_idx].push(latency); + self.decode_token_latencies[batch_idx].push(token_latency); + self.decode_throughputs[batch_idx].push(decode.throughput); + } + + fn end_batch(&mut self, batch_idx: usize) { + self.prefill_batch_latency_throughput.push(( + self.prefill_latencies[batch_idx].iter().sum::() + / self.prefill_latencies[batch_idx].len() as f64, + self.prefill_throughputs[batch_idx].iter().sum::() + / self.prefill_throughputs[batch_idx].len() as f64, + )); + self.decode_batch_latency_throughput.push(( + self.decode_latencies[batch_idx].iter().sum::() + / self.decode_latencies[batch_idx].len() as f64, + self.decode_throughputs[batch_idx].iter().sum::() + / self.decode_throughputs[batch_idx].len() as f64, + )); + } +} + +/// Progress bar +fn progress_gauge(title: &str, label: String, progress: f64, color: Color) -> Gauge { + Gauge::default() + .block(Block::default().title(title).borders(Borders::ALL)) + .gauge_style(Style::default().fg(color)) + .label(Span::raw(label)) + .ratio(progress) +} + +/// Throughput paragraph +fn throughput_paragraph<'a>(throughput: &Vec, name: &'static str) -> Paragraph<'a> { + // Throughput average/high/low texts + let throughput_texts = statis_spans(throughput, "tokens/secs"); + + // Throughput block + Paragraph::new(throughput_texts).block( + Block::default() + .title(Span::raw(format!("{name} Throughput"))) + .borders(Borders::ALL), + ) +} + +/// Latency paragraph +fn latency_paragraph<'a>(latency: &mut Vec, name: &'static str) -> Paragraph<'a> { + // Latency average/high/low texts + let mut latency_texts = statis_spans(latency, "ms"); + + // Sort latency for percentiles + float_ord::sort(latency); + let latency_percentiles = crate::utils::percentiles(latency, &[50, 90, 99]); + + // Latency p50/p90/p99 texts + let colors = vec![Color::LightGreen, Color::LightYellow, Color::LightRed]; + for (i, (name, value)) in latency_percentiles.iter().enumerate() { + let span = Spans::from(vec![Span::styled( + format!("{name}: {value:.2} ms"), + Style::default().fg(colors[i]), + )]); + latency_texts.push(span); + } + + Paragraph::new(latency_texts).block( + Block::default() + .title(Span::raw(format!("{name} Latency"))) + .borders(Borders::ALL), + ) +} + +/// Average/High/Low spans +fn statis_spans<'a>(data: &Vec, unit: &'static str) -> Vec> { + vec![ + Spans::from(vec![Span::styled( + format!( + "Average: {:.2} {unit}", + data.iter().sum::() / data.len() as f64 + ), + Style::default().fg(Color::LightBlue), + )]), + Spans::from(vec![Span::styled( + format!( + "Lowest: {:.2} {unit}", + data.iter() + .min_by(|a, b| a.total_cmp(b)) + .unwrap_or(&std::f64::NAN) + ), + Style::default().fg(Color::Reset), + )]), + Spans::from(vec![Span::styled( + format!( + "Highest: {:.2} {unit}", + data.iter() + .max_by(|a, b| a.total_cmp(b)) + .unwrap_or(&std::f64::NAN) + ), + Style::default().fg(Color::Reset), + )]), + ] +} + +/// Latency histogram data +fn latency_histogram_data(latency: &[f64], bins: usize) -> Vec<(String, u64)> { + let histo_data: Vec<(String, u64)> = { + let histo = crate::utils::histogram(latency, bins); + histo + .into_iter() + .map(|(label, v)| (format!("{label:.2}"), v as u64)) + .collect() + }; + + histo_data +} + +/// Latency Histogram +fn latency_histogram<'a>( + histo_data_str: &'a Vec<(&'a str, u64)>, + name: &'static str, +) -> BarChart<'a> { + BarChart::default() + .block( + Block::default() + .title(format!("{name} latency histogram")) + .style(Style::default().fg(Color::LightYellow).bg(Color::Reset)) + .borders(Borders::ALL), + ) + .data(histo_data_str.as_slice()) +} + +/// Latency/Throughput chart +fn latency_throughput_chart<'a>( + latency_throughput: &'a Vec<(f64, f64)>, + batch_sizes: &'a [u32], + zoom: bool, + name: &'static str, +) -> Chart<'a> { + let latency_iter = latency_throughput.iter().map(|(l, _)| l); + let throughput_iter = latency_throughput.iter().map(|(_, t)| t); + + // Get extreme values + let min_latency: f64 = *latency_iter + .clone() + .min_by(|a, b| a.total_cmp(b)) + .unwrap_or(&std::f64::NAN); + let max_latency: f64 = *latency_iter + .max_by(|a, b| a.total_cmp(b)) + .unwrap_or(&std::f64::NAN); + let min_throughput: f64 = *throughput_iter + .clone() + .min_by(|a, b| a.total_cmp(b)) + .unwrap_or(&std::f64::NAN); + let max_throughput: f64 = *throughput_iter + .max_by(|a, b| a.total_cmp(b)) + .unwrap_or(&std::f64::NAN); + + // Char min max values + let min_x = if zoom { + ((min_latency - 0.05 * min_latency) / 100.0).floor() * 100.0 + } else { + 0.0 + }; + let max_x = ((max_latency + 0.05 * max_latency) / 100.0).ceil() * 100.0; + let step_x = (max_x - min_x) / 4.0; + + // Chart min max values + let min_y = if zoom { + ((min_throughput - 0.05 * min_throughput) / 100.0).floor() * 100.0 + } else { + 0.0 + }; + let max_y = ((max_throughput + 0.05 * max_throughput) / 100.0).ceil() * 100.0; + let step_y = (max_y - min_y) / 4.0; + + // Labels + let mut x_labels = vec![Span::styled( + format!("{min_x:.2}"), + Style::default() + .add_modifier(Modifier::BOLD) + .fg(Color::Gray) + .bg(Color::Reset), + )]; + for i in 0..3 { + x_labels.push(Span::styled( + format!("{:.2}", min_x + ((i + 1) as f64 * step_x)), + Style::default().fg(Color::Gray).bg(Color::Reset), + )); + } + x_labels.push(Span::styled( + format!("{max_x:.2}"), + Style::default() + .add_modifier(Modifier::BOLD) + .fg(Color::Gray) + .bg(Color::Reset), + )); + + // Labels + let mut y_labels = vec![Span::styled( + format!("{min_y:.2}"), + Style::default() + .add_modifier(Modifier::BOLD) + .fg(Color::Gray) + .bg(Color::Reset), + )]; + for i in 0..3 { + y_labels.push(Span::styled( + format!("{:.2}", min_y + ((i + 1) as f64 * step_y)), + Style::default().fg(Color::Gray).bg(Color::Reset), + )); + } + y_labels.push(Span::styled( + format!("{max_y:.2}"), + Style::default() + .add_modifier(Modifier::BOLD) + .fg(Color::Gray) + .bg(Color::Reset), + )); + + // Chart dataset + let colors = color_vec(); + let datasets: Vec = (0..latency_throughput.len()) + .map(|i| { + let color_idx = i % colors.len(); + + Dataset::default() + .name(batch_sizes[i].to_string()) + .marker(symbols::Marker::Block) + .style(Style::default().fg(colors[color_idx])) + .graph_type(GraphType::Scatter) + .data(&latency_throughput[i..(i + 1)]) + }) + .collect(); + + // Chart + Chart::new(datasets) + .style(Style::default().fg(Color::Cyan).bg(Color::Reset)) + .block( + Block::default() + .title(Span::styled( + format!("{name} throughput over latency"), + Style::default().fg(Color::Gray).bg(Color::Reset), + )) + .borders(Borders::ALL), + ) + .x_axis( + Axis::default() + .title("ms") + .style(Style::default().fg(Color::Gray).bg(Color::Reset)) + .labels(x_labels) + .bounds([min_x, max_x]), + ) + .y_axis( + Axis::default() + .title("tokens/secs") + .style(Style::default().fg(Color::Gray).bg(Color::Reset)) + .labels(y_labels) + .bounds([min_y, max_y]), + ) +} + +// Colors for latency/throughput chart +fn color_vec() -> Vec { + vec![ + Color::Red, + Color::Green, + Color::Yellow, + Color::Blue, + Color::Magenta, + Color::Cyan, + Color::Gray, + Color::DarkGray, + Color::LightRed, + Color::LightGreen, + Color::LightYellow, + Color::LightBlue, + Color::LightMagenta, + Color::LightCyan, + ] +} diff --git a/benchmark/src/event.rs b/benchmark/src/event.rs new file mode 100644 index 00000000..91ce8400 --- /dev/null +++ b/benchmark/src/event.rs @@ -0,0 +1,65 @@ +/// Inspired by https://github.com/orhun/rust-tui-template/blob/472aa515119d4c94903eac12d9784417281dc7f5/src/event.rs +use crossterm::event; +use std::time::{Duration, Instant}; +use tokio::sync::{broadcast, mpsc}; + +/// Events +#[derive(Debug)] +pub(crate) enum Event { + /// Terminal tick. + Tick, + /// Key press. + Key(event::KeyEvent), + /// Terminal resize. + Resize(u16, u16), +} + +pub(crate) async fn terminal_event_task( + fps: u32, + event_sender: mpsc::Sender, + mut shutdown_receiver: broadcast::Receiver<()>, + _shutdown_guard_sender: mpsc::Sender<()>, +) { + // End task if a message is received on shutdown_receiver + // _shutdown_guard_sender will be dropped once the task is finished + tokio::select! { + _ = event_loop(fps, event_sender) => { + }, + _ = shutdown_receiver.recv() => {} + } +} + +/// Main event loop +async fn event_loop(fps: u32, event_sender: mpsc::Sender) { + // Frame budget + let per_frame = Duration::from_secs(1) / fps; + + // When was last frame executed + let mut last_frame = Instant::now(); + + loop { + // Sleep to avoid blocking the thread for too long + if let Some(sleep) = per_frame.checked_sub(last_frame.elapsed()) { + tokio::time::sleep(sleep).await; + } + + // Get crossterm event and send a new one over the channel + if event::poll(Duration::from_secs(0)).expect("no events available") { + match event::read().expect("unable to read event") { + event::Event::Key(e) => event_sender.send(Event::Key(e)).await.unwrap_or(()), + event::Event::Resize(w, h) => { + event_sender.send(Event::Resize(w, h)).await.unwrap_or(()) + } + _ => (), + } + } + + // Frame budget exceeded + if last_frame.elapsed() >= per_frame { + // Send tick + event_sender.send(Event::Tick).await.unwrap_or(()); + // Rest last_frame time + last_frame = Instant::now(); + } + } +} diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs new file mode 100644 index 00000000..3a6316ab --- /dev/null +++ b/benchmark/src/generation.rs @@ -0,0 +1,211 @@ +use std::time::{Duration, Instant}; +use text_generation_client::{ + Batch, ClientError, NextTokenChooserParameters, Request, ShardedClient, + StoppingCriteriaParameters, +}; +use tokenizers::{Tokenizer, TruncationDirection}; +use tokio::sync::{broadcast, mpsc}; + +const LOREM_IPSUM: &str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."; + +#[derive(Debug, Clone)] +pub(crate) struct Prefill { + pub(crate) latency: Duration, + pub(crate) throughput: f64, +} + +#[derive(Debug, Clone)] +pub(crate) struct Decode { + pub(crate) latency: Duration, + pub(crate) token_latency: Duration, + pub(crate) throughput: f64, +} + +#[derive(Debug)] +pub(crate) enum Message { + Warmup, + Prefill(Prefill), + Decode(Decode), + EndRun, + EndBatch, +} + +/// Benchmarking task +#[allow(clippy::too_many_arguments)] +pub(crate) async fn generation_task( + tokenizer: Tokenizer, + batch_size: Vec, + sequence_length: u32, + decode_length: u32, + n_runs: usize, + warmups: usize, + client: ShardedClient, + run_sender: mpsc::Sender>, + mut shutdown_receiver: broadcast::Receiver<()>, + _shutdown_guard_sender: mpsc::Sender<()>, +) { + // End task if a message is received on shutdown_receiver + // _shutdown_guard_sender will be dropped once the task is finished + tokio::select! { + res = generate_runs(tokenizer, batch_size, sequence_length, decode_length, n_runs, warmups, client, run_sender.clone()) => { + if let Err(err) = res { + run_sender.send(Err(err)).await.unwrap_or(()); + } + }, + _ = shutdown_receiver.recv() => {} + } +} + +/// Benchmark prefill/decode +#[allow(clippy::too_many_arguments)] +async fn generate_runs( + tokenizer: Tokenizer, + batch_size: Vec, + sequence_length: u32, + decode_length: u32, + n_runs: usize, + warmups: usize, + mut client: ShardedClient, + run_sender: mpsc::Sender>, +) -> Result<(), ClientError> { + // Create a dummy sequence + let sequence = create_sequence(sequence_length, tokenizer); + + for b in batch_size { + // Warmups on batch size + for _ in 0..warmups { + let (_, decode_batch) = + prefill(sequence.clone(), b, decode_length, &mut client).await?; + let _ = decode(decode_batch, &mut client).await?; + // Send warmup message + run_sender.send(Ok(Message::Warmup)).await.unwrap_or(()); + } + + for _ in 0..n_runs { + let (prefill, decode_batch) = + prefill(sequence.clone(), b, decode_length, &mut client).await?; + // Send prefill message + run_sender + .send(Ok(Message::Prefill(prefill))) + .await + .unwrap_or(()); + + let decode = decode(decode_batch, &mut client).await?; + + // Send decode message + run_sender + .send(Ok(Message::Decode(decode))) + .await + .unwrap_or(()); + + // Send run ended message + run_sender.send(Ok(Message::EndRun)).await.unwrap_or(()); + } + // Batch ended + run_sender.send(Ok(Message::EndBatch)).await.unwrap_or(()); + } + Ok(()) +} + +// Run a prefill step +async fn prefill( + sequence: String, + batch_size: u32, + decode_length: u32, + client: &mut ShardedClient, +) -> Result<(Prefill, Batch), ClientError> { + // Create requests + let requests = (0..batch_size) + .map(|id| Request { + id: id.into(), + inputs: sequence.clone(), + parameters: Some(NextTokenChooserParameters { + temperature: 1.0, + top_k: 0, + top_p: 1.0, + typical_p: 1.0, + do_sample: false, + seed: 0, + repetition_penalty: 1.0, + watermark: false, + }), + stopping_parameters: Some(StoppingCriteriaParameters { + max_new_tokens: decode_length, + stop_sequences: vec![], + ignore_eos_token: true, // Will not stop even if a eos token is generated + }), + }) + .collect(); + + let batch = Batch { + id: 0, + requests, + size: batch_size, + }; + + // Run prefill + let start_time = Instant::now(); + let (_, decode_batch) = client.prefill(batch.clone()).await?; + + // Get latency + let latency = start_time.elapsed(); + + // Compute throughput from latency and batch size + let throughput = batch_size as f64 / latency.as_secs_f64(); + + // Decode batch cannot be empty + let decode_batch = decode_batch.expect("decode_batch is None. This is a bug."); + + let step = Prefill { + latency, + throughput, + }; + + Ok((step, decode_batch)) +} + +/// Run a full decode +async fn decode(batch: Batch, client: &mut ShardedClient) -> Result { + let mut decode_length = 0; + let batch_size = batch.size; + + let start_time = Instant::now(); + + // Full decode over decode length + let mut next_batch = Some(batch); + while let Some(batch) = next_batch { + let result = client.decode(vec![batch]).await?; + next_batch = result.1; + decode_length += 1; + } + + // Get latency + let latency = start_time.elapsed(); + let token_latency = latency / decode_length; + + // Compute throughput from latency, batch size and decode length + let throughput = (batch_size * decode_length) as f64 / latency.as_secs_f64(); + + let step = Decode { + latency, + token_latency, + throughput, + }; + Ok(step) +} + +/// Create a dummy sequence of the correct length +fn create_sequence(sequence_length: u32, tokenizer: Tokenizer) -> String { + let lorem_ipsum_length = tokenizer.encode(LOREM_IPSUM, true).unwrap().len(); + // Repeat lorem ipsum to cover sequence length + let string_sequence = + LOREM_IPSUM.repeat((0..sequence_length).step_by(lorem_ipsum_length).len()); + // Encode sequence + let mut encoding = tokenizer.encode(string_sequence, true).unwrap(); + // Truncate to sequence_length + encoding.truncate(sequence_length as usize, 0, TruncationDirection::Left); + // Decode + tokenizer + .decode(Vec::from(encoding.get_ids()), false) + .unwrap() +} diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs new file mode 100644 index 00000000..4da0b573 --- /dev/null +++ b/benchmark/src/lib.rs @@ -0,0 +1,110 @@ +mod app; +mod event; +mod generation; +mod utils; + +use crate::app::App; +use crate::event::Event; +use crossterm::ExecutableCommand; +use std::io; +use text_generation_client::ShardedClient; +use tokenizers::Tokenizer; +use tokio::sync::{broadcast, mpsc}; +use tui::backend::CrosstermBackend; +use tui::Terminal; + +/// Run benchmarking app +#[allow(clippy::too_many_arguments)] +pub async fn run( + tokenizer_name: String, + tokenizer: Tokenizer, + batch_size: Vec, + sequence_length: u32, + decode_length: u32, + n_runs: usize, + warmups: usize, + client: ShardedClient, +) -> Result<(), crossterm::ErrorKind> { + // Initialize terminal properties + crossterm::terminal::enable_raw_mode()?; + io::stdout().execute(crossterm::terminal::EnterAlternateScreen)?; + io::stdout().execute(crossterm::cursor::Hide)?; + + // Initialize terminal + let mut terminal = { + let backend = CrosstermBackend::new(io::stdout()); + Terminal::new(backend)? + }; + + // Create message channel between generation_task and app + let (run_sender, run_receiver) = mpsc::channel(8); + // Crossterm event channel + let (event_sender, mut event_receiver) = mpsc::channel(8); + // Shutdown channel to terminate tasks + let (shutdown_sender, _) = broadcast::channel(1); + // Channel to check if tasks terminated + let (shutdown_guard_sender, mut shutdown_guard_receiver) = mpsc::channel(1); + + // Create generation task + tokio::spawn(generation::generation_task( + tokenizer, + batch_size.clone(), + sequence_length, + decode_length, + n_runs, + warmups, + client, + run_sender, + shutdown_sender.subscribe(), + shutdown_guard_sender.clone(), + )); + + // Create event task + tokio::spawn(event::terminal_event_task( + 250, + event_sender, + shutdown_sender.subscribe(), + shutdown_guard_sender.clone(), + )); + + // Drop our end of shutdown sender + drop(shutdown_guard_sender); + + // Create App + let mut app = App::new( + run_receiver, + tokenizer_name, + sequence_length, + decode_length, + n_runs, + batch_size, + ); + + while app.running { + // Draw frame + terminal.draw(|frame| app.render(frame))?; + + // Await a new event from event handling task + match event_receiver.recv().await { + None => break, + // Update app state + Some(event) => match event { + Event::Tick => app.tick(), + Event::Key(key_event) => app.handle_key_event(key_event), + _ => {} + }, + } + } + + // Ask tasks to shutdown + let _ = shutdown_sender.send(()); + // Wait for tasks to shutdown + let _ = shutdown_guard_receiver.recv().await; + + // Revert terminal to original view + io::stdout().execute(crossterm::terminal::LeaveAlternateScreen)?; + crossterm::terminal::disable_raw_mode()?; + io::stdout().execute(crossterm::cursor::Show)?; + + Ok(()) +} diff --git a/benchmark/src/main.rs b/benchmark/src/main.rs new file mode 100644 index 00000000..443af77f --- /dev/null +++ b/benchmark/src/main.rs @@ -0,0 +1,119 @@ +/// Text Generation Inference benchmarking tool +/// +/// Inspired by the great Oha app: https://github.com/hatoo/oha +/// and: https://github.com/orhun/rust-tui-template +use clap::Parser; +use std::path::Path; +use text_generation_client::ShardedClient; +use tokenizers::Tokenizer; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::EnvFilter; + +/// App Configuration +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Args { + #[clap(short, long, env)] + tokenizer_name: String, + #[clap(short, long)] + batch_size: Option>, + #[clap(default_value = "10", short, long, env)] + sequence_length: u32, + #[clap(default_value = "8", short, long, env)] + decode_length: u32, + #[clap(default_value = "10", short, long, env)] + runs: usize, + #[clap(default_value = "1", short, long, env)] + warmups: usize, + #[clap(default_value = "/tmp/text-generation-server-0", short, long, env)] + master_shard_uds_path: String, +} + +fn main() -> Result<(), Box> { + // Get args + let args = Args::parse(); + // Pattern match configuration + let Args { + tokenizer_name, + batch_size, + sequence_length, + decode_length, + runs, + warmups, + master_shard_uds_path, + } = args; + + let batch_size = batch_size.unwrap_or(vec![1, 2, 4, 8, 16, 32]); + + init_logging(); + + // Tokenizer instance + // This will only be used to validate payloads + tracing::info!("Loading tokenizer"); + let local_path = Path::new(&tokenizer_name); + let tokenizer = + if local_path.exists() && local_path.is_dir() && local_path.join("tokenizer.json").exists() + { + // Load local tokenizer + tracing::info!("Found local tokenizer"); + Tokenizer::from_file(local_path.join("tokenizer.json")).unwrap() + } else { + // Download and instantiate tokenizer + // We need to download it outside of the Tokio runtime + tracing::info!("Downloading tokenizer"); + Tokenizer::from_pretrained(tokenizer_name.clone(), None).unwrap() + }; + tracing::info!("Tokenizer loaded"); + + // Launch Tokio runtime + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + // Instantiate sharded client from the master unix socket + tracing::info!("Connect to model server"); + let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path) + .await + .expect("Could not connect to server"); + // Clear the cache; useful if the webserver rebooted + sharded_client + .clear_cache(None) + .await + .expect("Unable to clear cache"); + tracing::info!("Connected"); + + // Run app + text_generation_benchmark::run( + tokenizer_name, + tokenizer, + batch_size, + sequence_length, + decode_length, + runs, + warmups, + sharded_client, + ) + .await + .unwrap(); + }); + Ok(()) +} + +/// Init logging using LOG_LEVEL +fn init_logging() { + // STDOUT/STDERR layer + let fmt_layer = tracing_subscriber::fmt::layer() + .with_file(true) + .with_line_number(true); + + // Filter events with LOG_LEVEL + let env_filter = + EnvFilter::try_from_env("LOG_LEVEL").unwrap_or_else(|_| EnvFilter::new("info")); + + tracing_subscriber::registry() + .with(env_filter) + .with(fmt_layer) + .init(); +} diff --git a/benchmark/src/utils.rs b/benchmark/src/utils.rs new file mode 100644 index 00000000..d096d655 --- /dev/null +++ b/benchmark/src/utils.rs @@ -0,0 +1,43 @@ +/// MIT License +// +// Copyright (c) 2020 hatoo +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +use std::collections::BTreeMap; + +pub(crate) fn histogram(values: &[f64], bins: usize) -> Vec<(f64, usize)> { + assert!(bins >= 2); + let mut bucket: Vec = vec![0; bins]; + let min = values.iter().collect::().min(); + let max = values.iter().collect::().max(); + let step = (max - min) / (bins - 1) as f64; + + for &v in values { + let i = std::cmp::min(((v - min) / step).ceil() as usize, bins - 1); + bucket[i] += 1; + } + + bucket + .into_iter() + .enumerate() + .map(|(i, v)| (min + step * i as f64, v)) + .collect() +} + +pub(crate) fn percentiles(values: &[f64], pecents: &[i32]) -> BTreeMap { + pecents + .iter() + .map(|&p| { + let i = (f64::from(p) / 100.0 * values.len() as f64) as usize; + (format!("p{p}"), *values.get(i).unwrap_or(&std::f64::NAN)) + }) + .collect() +} diff --git a/proto/generate.proto b/proto/generate.proto index 0bac4352..86393026 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -53,6 +53,9 @@ message StoppingCriteriaParameters { uint32 max_new_tokens = 1; /// Optional stopping sequences repeated string stop_sequences = 2; + /// Ignore end of sequence token + /// used for benchmarking + bool ignore_eos_token = 3; } message Request { diff --git a/router/src/main.rs b/router/src/main.rs index 81c6aeef..f6028574 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -37,7 +37,7 @@ struct Args { max_waiting_tokens: usize, #[clap(default_value = "3000", long, short, env)] port: u16, - #[clap(default_value = "/tmp/text-generation-0", long, env)] + #[clap(default_value = "/tmp/text-generation-server-0", long, env)] master_shard_uds_path: String, #[clap(default_value = "bigscience/bloom", long, env)] tokenizer_name: String, @@ -76,6 +76,8 @@ fn main() -> Result<(), std::io::Error> { panic!("validation_workers must be > 0"); } + init_logging(otlp_endpoint, json_output); + // CORS allowed origins // map to go inside the option and then map to parse from String to HeaderValue // Finally, convert to AllowOrigin @@ -89,17 +91,21 @@ fn main() -> Result<(), std::io::Error> { // Tokenizer instance // This will only be used to validate payloads + tracing::info!("Loading tokenizer"); let local_path = Path::new(&tokenizer_name); let tokenizer = if local_path.exists() && local_path.is_dir() && local_path.join("tokenizer.json").exists() { // Load local tokenizer + tracing::info!("Found local tokenizer"); Tokenizer::from_file(local_path.join("tokenizer.json")).unwrap() } else { // Download and instantiate tokenizer // We need to download it outside of the Tokio runtime + tracing::info!("Downloading tokenizer"); Tokenizer::from_pretrained(tokenizer_name.clone(), None).unwrap() }; + tracing::info!("Tokenizer loaded"); // Launch Tokio runtime tokio::runtime::Builder::new_multi_thread() @@ -107,8 +113,6 @@ fn main() -> Result<(), std::io::Error> { .build() .unwrap() .block_on(async { - init_logging(otlp_endpoint, json_output); - // Get pipeline tag let model_info = reqwest::get(format!( "https://huggingface.co/api/models/{tokenizer_name}" diff --git a/router/src/queue.rs b/router/src/queue.rs index df2087e1..2899ccd4 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -237,6 +237,7 @@ mod tests { watermark: false, }, stopping_parameters: StoppingCriteriaParameters { + ignore_eos_token: false, max_new_tokens: 0, stop_sequences: vec![], }, diff --git a/router/src/validation.rs b/router/src/validation.rs index 1c350caa..ec67cefd 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -315,6 +315,7 @@ fn validate( let stopping_parameters = StoppingCriteriaParameters { max_new_tokens, stop_sequences, + ignore_eos_token: false, }; metrics::histogram!("tgi_request_input_length", input_length as f64); diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 6308ef6b..4d48f492 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -18,7 +18,7 @@ def serve( revision: Optional[str] = None, sharded: bool = False, quantize: bool = False, - uds_path: Path = "/tmp/text-generation", + uds_path: Path = "/tmp/text-generation-server", logger_level: str = "INFO", json_output: bool = False, otlp_endpoint: Optional[str] = None, diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 597fbe7c..b923857d 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -123,20 +123,22 @@ def __init__( self, eos_token_id: int, stop_sequence_criterias: List[StopSequenceCriteria], - max_new_tokens=20, + max_new_tokens: int = 20, + ignore_eos_token: bool = False, ): self.eos_token_id = eos_token_id self.stop_sequence_criterias = stop_sequence_criterias self.max_new_tokens = max_new_tokens self.current_tokens = 0 self.current_output = "" + self.ignore_eos_token = ignore_eos_token def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[str]]: self.current_tokens += 1 if self.current_tokens >= self.max_new_tokens: return True, FinishReason.FINISH_REASON_LENGTH - if last_token == self.eos_token_id: + if not self.ignore_eos_token and last_token == self.eos_token_id: return True, FinishReason.FINISH_REASON_EOS_TOKEN self.current_output += last_output @@ -156,5 +158,8 @@ def from_pb( StopSequenceCriteria(sequence) for sequence in pb.stop_sequences ] return StoppingCriteria( - tokenizer.eos_token_id, stop_sequence_criterias, pb.max_new_tokens + tokenizer.eos_token_id, + stop_sequence_criterias, + pb.max_new_tokens, + pb.ignore_eos_token, ) From 08b7e4a28232c71b97a5e1fc90d50be9a8e6c6f1 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 30 Mar 2023 16:12:23 +0200 Subject: [PATCH 139/793] fix(server): fix flash neox rotary embeddings (#150) --- .../text_generation_server/models/flash_neox_modeling.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/text_generation_server/models/flash_neox_modeling.py b/server/text_generation_server/models/flash_neox_modeling.py index ac07aa98..f3517c47 100644 --- a/server/text_generation_server/models/flash_neox_modeling.py +++ b/server/text_generation_server/models/flash_neox_modeling.py @@ -319,12 +319,12 @@ def forward( layer_past[...] = qkv_rot[:, 1:] # output - attn_output = torch.empty_like(qkv[:, 0]) + attn_output = torch.empty_like(qkv_rot[:, 0]) # flash attention flash_attn_cuda.fwd( - qkv[:, 0], - qkv[:, 1], - qkv[:, 2], + qkv_rot[:, 0], + qkv_rot[:, 1], + qkv_rot[:, 2], attn_output, cu_seqlens, cu_seqlens, From 84722f3e331035d5bce04e34fe650504ff13e315 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 30 Mar 2023 17:10:01 +0200 Subject: [PATCH 140/793] v0.4.2 (#151) --- Cargo.lock | 6 +++--- benchmark/Cargo.lock | 2 +- docs/openapi.json | 2 +- launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- server/pyproject.toml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37fb575c..a5c60a1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2212,7 +2212,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.4.1" +version = "0.4.2" dependencies = [ "futures", "grpc-metadata", @@ -2229,7 +2229,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.4.1" +version = "0.4.2" dependencies = [ "clap 4.1.8", "ctrlc", @@ -2244,7 +2244,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.4.1" +version = "0.4.2" dependencies = [ "async-stream", "axum", diff --git a/benchmark/Cargo.lock b/benchmark/Cargo.lock index 4a1e43dd..872e797e 100644 --- a/benchmark/Cargo.lock +++ b/benchmark/Cargo.lock @@ -2069,7 +2069,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.4.1" +version = "0.4.2" dependencies = [ "futures", "grpc-metadata", diff --git a/docs/openapi.json b/docs/openapi.json index 7a8cc58f..e384f5c5 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -11,7 +11,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.4.1" + "version": "0.4.2" }, "paths": { "/generate": { diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index e44a8ddb..c238e9ac 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.4.1" +version = "0.4.2" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index 4c898f23..43a89991 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.4.1" +version = "0.4.2" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 487d7e59..9e6242fa 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.4.1" +version = "0.4.2" edition = "2021" [dependencies] diff --git a/server/pyproject.toml b/server/pyproject.toml index 7c032525..948b3f49 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.4.1" +version = "0.4.2" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From fef1a1c381d6a0d01b64d888032b4ed7f5bb113e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 30 Mar 2023 17:28:14 +0200 Subject: [PATCH 141/793] v0.4.3 (#152) --- Cargo.lock | 6 +++--- benchmark/Cargo.lock | 2 +- docs/openapi.json | 2 +- launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- router/src/main.rs | 10 +++------- server/pyproject.toml | 2 +- 8 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5c60a1b..8e469f23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2212,7 +2212,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.4.2" +version = "0.4.3" dependencies = [ "futures", "grpc-metadata", @@ -2229,7 +2229,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.4.2" +version = "0.4.3" dependencies = [ "clap 4.1.8", "ctrlc", @@ -2244,7 +2244,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.4.2" +version = "0.4.3" dependencies = [ "async-stream", "axum", diff --git a/benchmark/Cargo.lock b/benchmark/Cargo.lock index 872e797e..8441014b 100644 --- a/benchmark/Cargo.lock +++ b/benchmark/Cargo.lock @@ -2069,7 +2069,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.4.2" +version = "0.4.3" dependencies = [ "futures", "grpc-metadata", diff --git a/docs/openapi.json b/docs/openapi.json index e384f5c5..11fa1e35 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -11,7 +11,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.4.2" + "version": "0.4.3" }, "paths": { "/generate": { diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index c238e9ac..318052c9 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.4.2" +version = "0.4.3" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index 43a89991..77577eea 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.4.2" +version = "0.4.3" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 9e6242fa..401742dc 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.4.2" +version = "0.4.3" edition = "2021" [dependencies] diff --git a/router/src/main.rs b/router/src/main.rs index f6028574..81c6aeef 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -37,7 +37,7 @@ struct Args { max_waiting_tokens: usize, #[clap(default_value = "3000", long, short, env)] port: u16, - #[clap(default_value = "/tmp/text-generation-server-0", long, env)] + #[clap(default_value = "/tmp/text-generation-0", long, env)] master_shard_uds_path: String, #[clap(default_value = "bigscience/bloom", long, env)] tokenizer_name: String, @@ -76,8 +76,6 @@ fn main() -> Result<(), std::io::Error> { panic!("validation_workers must be > 0"); } - init_logging(otlp_endpoint, json_output); - // CORS allowed origins // map to go inside the option and then map to parse from String to HeaderValue // Finally, convert to AllowOrigin @@ -91,21 +89,17 @@ fn main() -> Result<(), std::io::Error> { // Tokenizer instance // This will only be used to validate payloads - tracing::info!("Loading tokenizer"); let local_path = Path::new(&tokenizer_name); let tokenizer = if local_path.exists() && local_path.is_dir() && local_path.join("tokenizer.json").exists() { // Load local tokenizer - tracing::info!("Found local tokenizer"); Tokenizer::from_file(local_path.join("tokenizer.json")).unwrap() } else { // Download and instantiate tokenizer // We need to download it outside of the Tokio runtime - tracing::info!("Downloading tokenizer"); Tokenizer::from_pretrained(tokenizer_name.clone(), None).unwrap() }; - tracing::info!("Tokenizer loaded"); // Launch Tokio runtime tokio::runtime::Builder::new_multi_thread() @@ -113,6 +107,8 @@ fn main() -> Result<(), std::io::Error> { .build() .unwrap() .block_on(async { + init_logging(otlp_endpoint, json_output); + // Get pipeline tag let model_info = reqwest::get(format!( "https://huggingface.co/api/models/{tokenizer_name}" diff --git a/server/pyproject.toml b/server/pyproject.toml index 948b3f49..e9dc624c 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.4.2" +version = "0.4.3" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From c0aeb3258343cc68e8759926002694f0f72cff02 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 3 Apr 2023 19:06:42 +0200 Subject: [PATCH 142/793] feat(server): flash santacoder (#153) --- .../text_generation_server/models/__init__.py | 26 +- .../models/custom_modeling/__init__.py | 0 .../flash_neox_modeling.py | 0 .../flash_santacoder_modeling.py | 357 ++++++++++++++ .../models/flash_causal_lm.py | 458 ++++++++++++++++++ .../models/flash_neox.py | 452 +---------------- .../models/flash_santacoder.py | 138 ++++++ supported_models.json | 4 +- 8 files changed, 980 insertions(+), 455 deletions(-) create mode 100644 server/text_generation_server/models/custom_modeling/__init__.py rename server/text_generation_server/models/{ => custom_modeling}/flash_neox_modeling.py (100%) create mode 100644 server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py create mode 100644 server/text_generation_server/models/flash_causal_lm.py create mode 100644 server/text_generation_server/models/flash_santacoder.py diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 90c70cb5..bcaf6ec1 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -8,6 +8,7 @@ from text_generation_server.models.model import Model from text_generation_server.models.causal_lm import CausalLM +from text_generation_server.models.flash_causal_lm import FlashCausalLM from text_generation_server.models.bloom import BLOOM, BLOOMSharded from text_generation_server.models.seq2seq_lm import Seq2SeqLM from text_generation_server.models.galactica import Galactica, GalacticaSharded @@ -17,18 +18,22 @@ try: from text_generation_server.models.flash_neox import FlashNeoX, FlashNeoXSharded + from text_generation_server.models.flash_santacoder import FlashSantacoder - FLASH_NEOX = torch.cuda.is_available() and int(os.environ.get("FLASH_NEOX", 0)) == 1 + FLASH_ATTENTION = ( + torch.cuda.is_available() and int(os.environ.get("FLASH_ATTENTION", 0)) == 1 + ) except ImportError: - if int(os.environ.get("FLASH_NEOX", 0)) == 1: - logger.exception("Could not import FlashNeoX") - FLASH_NEOX = False + if int(os.environ.get("FLASH_ATTENTION", 0)) == 1: + logger.exception("Could not import Flash Attention models") + FLASH_ATTENTION = False __all__ = [ "Model", "BLOOM", "BLOOMSharded", "CausalLM", + "FlashCausalLM", "Galactica", "GalacticaSharded", "GPTNeoxSharded", @@ -38,9 +43,10 @@ "get_model", ] -if FLASH_NEOX: +if FLASH_ATTENTION: __all__.append(FlashNeoX) __all__.append(FlashNeoXSharded) + __all__.append(FlashSantacoder) # The flag below controls whether to allow TF32 on matmul. This flag defaults to False # in PyTorch 1.12 and later. @@ -63,7 +69,11 @@ def get_model( return Galactica(model_id, revision, quantize=quantize) if "santacoder" in model_id: - return SantaCoder(model_id, revision, quantize) + if sharded: + raise NotImplementedError("sharded is not supported for Santacoder") + else: + santacoder_cls = FlashSantacoder if FLASH_ATTENTION else SantaCoder + return santacoder_cls(model_id, revision, quantize) config = AutoConfig.from_pretrained(model_id, revision=revision) model_type = config.model_type @@ -76,10 +86,10 @@ def get_model( if model_type == "gpt_neox": if sharded: - neox_cls = FlashNeoXSharded if FLASH_NEOX else GPTNeoxSharded + neox_cls = FlashNeoXSharded if FLASH_ATTENTION else GPTNeoxSharded return neox_cls(model_id, revision, quantize=quantize) else: - neox_cls = FlashNeoX if FLASH_NEOX else CausalLM + neox_cls = FlashNeoX if FLASH_ATTENTION else CausalLM return neox_cls(model_id, revision, quantize=quantize) if model_type == "t5": diff --git a/server/text_generation_server/models/custom_modeling/__init__.py b/server/text_generation_server/models/custom_modeling/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/text_generation_server/models/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py similarity index 100% rename from server/text_generation_server/models/flash_neox_modeling.py rename to server/text_generation_server/models/custom_modeling/flash_neox_modeling.py diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py new file mode 100644 index 00000000..ef073636 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -0,0 +1,357 @@ +import torch +import torch.distributed + +from torch import nn +from transformers.activations import ACT2FN + +# Flash attention imports +import flash_attn_cuda +import dropout_layer_norm + + +class FastLayerNorm(nn.LayerNorm): + def forward(self, hidden_states, residual=None): + if hidden_states.shape[-1] > 6144: + if residual is not None: + hidden_states += residual + residual = hidden_states + + return super(FastLayerNorm, self).forward(hidden_states), residual + else: + ( + normed_hidden_states, + residual, + *rest, + ) = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.weight, + self.bias, + None, + None, + None, + None, + 0.0, + self.eps, + 1.0, + 0, + None, + False, + False, + ) + if residual is None: + residual = hidden_states + + return normed_hidden_states, residual + + +class FastLinear(nn.Linear): + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + device=None, + dtype=None, + ) -> None: + super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) + + def transpose_weight(self): + self.weight = nn.Parameter(self.weight.T) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + if self.bias is not None: + return torch.addmm(self.bias, input, self.weight) + return torch.matmul(input, self.weight) + + +class FlashMQAttention(torch.nn.Module): + def __init__( + self, + num_heads, + hidden_size, + process_group=None, + ): + super().__init__() + self.num_heads = num_heads + self.hidden_size = hidden_size + self.head_size = hidden_size // num_heads + + self.softmax_scale = self.head_size ** (-0.5) + + if process_group is None: + self.attn = FastLinear(hidden_size, hidden_size + 2 * self.head_size) + self.c_proj = FastLinear(hidden_size, hidden_size) + else: + raise NotImplementedError + + def forward( + self, + hidden_states, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + qkv = self.attn(hidden_states) + + # Split query from key_value + query, key_value = qkv.split([self.hidden_size, 2 * self.head_size], dim=1) + + # Prepare query and key_value for indexing + query = query.view(-1, self.num_heads, self.head_size) + key_value = key_value.view(-1, 2, 1, self.head_size) + + # Prefill + if layer_past_present_indices is None: + # Copy to layer past + layer_past[...] = key_value + # Expand from 1 to num_heads + key_value = key_value.expand(-1, 2, self.num_heads, self.head_size) + + # output + attn_output = torch.empty_like(query) + # flash attention + flash_attn_cuda.fwd( + query, + key_value[:, 0], + key_value[:, 1], + attn_output, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + 0.0, + self.softmax_scale, + False, + True, + False, + 0, + None, + ) + # Decode + else: + # Add present to the layer_past tensor at the correct indices + layer_past[layer_past_present_indices] = key_value + # Expand from 1 to num_heads + key_value = layer_past.expand(-1, 2, self.num_heads, self.head_size) + + # output + attn_output = torch.empty_like(query) + # flash attention + flash_attn_cuda.fwd( + query, + key_value[:, 0], + key_value[:, 1], + attn_output, + cu_seqlens_q, + cu_seqlens, + 1, + max_s, + 0.0, + self.softmax_scale, + False, + False, + False, + 0, + None, + ) + + return self.c_proj(attn_output.view(-1, self.num_heads * self.head_size)) + + +class MLP(nn.Module): + def __init__( + self, act, hidden_size, intermediate_size, process_group=None + ): + super().__init__() + self.act = ( + ACT2FN[act] + if "gelu" not in act + else lambda x: torch.nn.functional.gelu(x, approximate="tanh" if act in ["gelu_fast", + "gelu_pytorch_tanh"] else None) + ) + + if process_group is None: + self.c_fc = FastLinear(hidden_size, intermediate_size) + self.c_proj = FastLinear(intermediate_size, hidden_size) + else: + raise NotImplementedError + + def forward(self, hidden_states): + hidden_states = self.c_fc(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.c_proj(hidden_states) + return hidden_states + + +class Block(nn.Module): + def __init__( + self, + num_heads, + act, + hidden_size, + intermediate_size, + layer_norm_eps, + process_group=None, + ): + super().__init__() + self.ln_1 = FastLayerNorm(hidden_size, eps=layer_norm_eps) + self.ln_2 = FastLayerNorm(hidden_size, eps=layer_norm_eps) + self.attn = FlashMQAttention( + num_heads, + hidden_size, + process_group, + ) + self.mlp = MLP( + act, + hidden_size, + intermediate_size, + process_group, + ) + + def forward( + self, + hidden_states, + residual, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + hidden_states, residual = self.ln_1(hidden_states, residual) + + hidden_states = self.attn( + hidden_states, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ) + + hidden_states, residual = self.ln_2( + hidden_states, residual + ) + + mlp_output = self.mlp(hidden_states) + + return mlp_output, residual + + +class FlashSantacoderModel(nn.Module): + def __init__(self, config, process_group=None): + super().__init__() + self.config = config + + if process_group is not None: + raise NotImplementedError + + self.wte = nn.Embedding(config.vocab_size, config.hidden_size) + self.wpe = nn.Embedding(config.max_position_embeddings, config.hidden_size) + + self.h = nn.ModuleList( + [ + Block( + config.num_attention_heads, + config.activation_function, + config.hidden_size, + config.n_inner if config.n_inner is not None else 4 * config.hidden_size, + config.layer_norm_epsilon, + process_group, + ) + for _ in range(config.num_hidden_layers) + ] + ) + self.ln_f = FastLayerNorm( + config.hidden_size, eps=config.layer_norm_epsilon + ) + + self.head_size = self.h[0].attn.head_size + self.num_heads = self.h[0].attn.num_heads + + def post_load_weights(self): + for layer in self.h: + layer: Block + layer.attn.attn.transpose_weight() + layer.attn.c_proj.transpose_weight() + layer.mlp.c_fc.transpose_weight() + layer.mlp.c_proj.transpose_weight() + + def forward( + self, + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values=None, + ): + hidden_states = self.wte(input_ids) + self.wpe(position_ids) + + # Prefill + if past_key_values is None: + # Create past tensor + past_key_values = hidden_states.new_empty( + ( + len(self.h), + len(hidden_states), + 2, + 1, + self.head_size, + ) + ) + layer_past_present_indices = None + cu_seqlens_q = None + # Decode + else: + # Create indices from cumulative sequence lengths + layer_past_present_indices = cu_seqlens[1:] - 1 + cu_seqlens_q = torch.arange( + cu_seqlens.shape[0], dtype=torch.int32, device=hidden_states.device + ) + + residual = None + for i, layer in enumerate(self.h): + hidden_states, residual = layer( + hidden_states, + residual, + cu_seqlens, + max_s, + past_key_values[i], + layer_past_present_indices, + cu_seqlens_q, + ) + + hidden_states, _ = self.ln_f(hidden_states, residual) + + return hidden_states, past_key_values + + +class FlashSantacoderForCausalLM(nn.Module): + def __init__(self, config, process_group=None): + super().__init__() + + self.transformer = FlashSantacoderModel(config, process_group) + + self.lm_head = FastLinear( + config.hidden_size, config.vocab_size, bias=False + ) + + def post_load_weights(self): + self.transformer.post_load_weights() + self.lm_head.transpose_weight() + + def forward( + self, + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values=None, + ): + hidden_states, present = self.transformer( + input_ids, position_ids, cu_seqlens, max_s, past_key_values + ) + return self.lm_head(hidden_states), present diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py new file mode 100644 index 00000000..e1a10cbf --- /dev/null +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -0,0 +1,458 @@ +import torch +import torch.distributed + +from torch.nn import functional as F + +from dataclasses import dataclass +from opentelemetry import trace +from transformers import AutoTokenizer, PreTrainedTokenizerBase, PreTrainedModel +from typing import Optional, Tuple, List, Type, Union + +from text_generation_server.models import Model +from text_generation_server.models.types import ( + Batch, + PrefillTokens, + Generation, + GeneratedText, +) +from text_generation_server.pb import generate_pb2 +from text_generation_server.utils import ( + NextTokenChooser, + StoppingCriteria, + Sampling, +) + +tracer = trace.get_tracer(__name__) + + +@dataclass +class FlashCausalLMBatch(Batch): + batch_id: int + requests: List[generate_pb2.Request] + + # Decoder values + input_ids: torch.Tensor + position_ids: torch.Tensor + # cumulative sequence lengths + cu_seqlens: torch.Tensor + max_seqlen: int + past_key_values: Optional[torch.Tensor] + + # All tokens + all_input_ids: List[List[int]] + all_input_ids_tensor: List[torch.Tensor] + + # Lengths of all generations present in the batch + input_lengths: List[int] + + # Generation helpers + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + + def to_pb(self) -> generate_pb2.Batch: + return generate_pb2.Batch( + id=self.batch_id, requests=self.requests, size=len(self) + ) + + @classmethod + def from_pb( + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + device: torch.device, + ) -> "CausalLMBatch": + input_ids = [] + position_ids = [] + cu_seqlens = [0] + max_seqlen = 0 + + input_lengths = [] + all_input_ids = [] + all_input_ids_tensor = [] + + next_token_choosers = [] + stopping_criterias = [] + + # Cumulative length + cumulative_length = 0 + + # Parse batch + for r in pb.requests: + tokenized_input = tokenizer(r.inputs)["input_ids"] + input_length = len(tokenized_input) + max_seqlen = max(max_seqlen, input_length) + input_lengths.append(input_length) + all_input_ids.append(tokenized_input) + + tokenized_input = torch.tensor(tokenized_input, device=device) + input_ids.append(tokenized_input) + + # Position ids + position_ids.append(torch.arange(0, input_length, dtype=torch.int32)) + + # Add cumulative lengths of all previous inputs + cu_seqlens.append(cumulative_length + input_length) + + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + stopping_criteria = StoppingCriteria.from_pb( + r.stopping_parameters, tokenizer + ) + stopping_criterias.append(stopping_criteria) + all_input_ids_tensor.append( + F.pad(tokenized_input, (0, stopping_criteria.max_new_tokens)) + ) + + # Update + cumulative_length += input_length + + input_ids = torch.concat(input_ids) + position_ids = torch.concat(position_ids) + cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32) + + return cls( + batch_id=pb.id, + requests=pb.requests, + input_ids=input_ids, + position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + past_key_values=None, + input_lengths=input_lengths, + all_input_ids=all_input_ids, + all_input_ids_tensor=all_input_ids_tensor, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + ) + + @classmethod + @tracer.start_as_current_span("concatenate") + def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch": + # Batch attributes + requests = [] + input_lengths = [] + all_input_ids = [] + all_input_ids_tensor = [] + next_token_choosers = [] + stopping_criterias = [] + + # Batch tensors + input_ids = [] + position_ids = [] + cu_seqlens = [torch.tensor([0], dtype=torch.int32)] + max_seqlen = 0 + past_key_values = [] + + # Cumulative length + cumulative_length = torch.tensor(0) + + for i, batch in enumerate(batches): + requests.extend(batch.requests) + input_lengths.extend(batch.input_lengths) + all_input_ids.extend(batch.all_input_ids) + all_input_ids_tensor.extend(batch.all_input_ids_tensor) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + + # Add cumulative lengths of all previous inputs + cu_seqlens.append(batch.cu_seqlens[1:] + cumulative_length) + + input_ids.append(batch.input_ids) + position_ids.append(batch.position_ids) + past_key_values.append(batch.past_key_values) + + max_seqlen = max(max_seqlen, batch.max_seqlen) + + # Update + cumulative_length += batch.cu_seqlens[-1] + + input_ids = torch.concat(input_ids) + position_ids = torch.concat(position_ids) + # Concat on dim=1 as first dim represents the model layers + past_key_values = torch.concat(past_key_values, dim=1) + cu_seqlens = torch.concat(cu_seqlens) + + return FlashCausalLMBatch( + batch_id=batches[0].batch_id, + requests=requests, + input_ids=input_ids, + position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + past_key_values=past_key_values, + input_lengths=input_lengths, + all_input_ids=all_input_ids, + all_input_ids_tensor=all_input_ids_tensor, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + ) + + def __len__(self): + return len(self.requests) + + +class FlashCausalLM(Model): + def __init__( + self, + model_cls: Type[PreTrainedModel], + model_id: str, + revision: Optional[str] = None, + quantize=False, + ): + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + else: + raise NotImplementedError("FlashCausalLM is only available on GPU") + + if quantize: + raise NotImplementedError("FlashCausalLM does not support quantization") + + tokenizer = AutoTokenizer.from_pretrained( + model_id, revision=revision, padding_side="left" + ) + self.model = ( + model_cls.from_pretrained( + model_id, + revision=revision, + torch_dtype=dtype, + ) + .eval() + .cuda() + ) + + super(FlashCausalLM, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @property + def batch_type(self) -> Type[FlashCausalLMBatch]: + return FlashCausalLMBatch + + def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: + return self.tokenizer.decode( + generated_ids, skip_special_tokens=True, cleanup_tokenization_spaces=False + ) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlens: torch.Tensor, + max_s: int, + past_key_values: Optional = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Model Forward + return self.model.forward( + input_ids=input_ids, + position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_s=max_s, + past_key_values=past_key_values, + ) + + @tracer.start_as_current_span("generate_token") + def generate_token( + self, batch: FlashCausalLMBatch + ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: + # Better to send to device here to avoid device issues in concatenate + position_ids = batch.position_ids.to(self.device, non_blocking=True) + cu_seqlens = batch.cu_seqlens.to(self.device) + + out, present = self.forward( + batch.input_ids, + position_ids, + cu_seqlens, + batch.max_seqlen, + batch.past_key_values, + ) + + # List of indices to cache + next_batch_keep_indices = [] + + # New values for next forward + next_batch_input_ids = [] + next_batch_position_ids = [] + next_batch_cu_seqlens = [0] + next_batch_max_seqlen = 0 + next_batch_past_key_values = [] + next_batch_input_lengths = [] + next_batch_all_input_ids = [] + next_batch_all_input_ids_tensor = [] + + # Cumulative length + cumulative_length = 0 + + # Results + generations: List[Generation] = [] + + # Zipped iterator + iterator = zip( + batch.requests, + batch.input_lengths, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + batch.all_input_ids_tensor, + ) + + # For each member of the batch + for i, ( + request, + input_length, + next_token_chooser, + stopping_criteria, + all_input_ids, + all_input_ids_tensor, + ) in enumerate(iterator): + # Indexing metadata + start_index = cumulative_length + end_index = cumulative_length + input_length + + if batch.past_key_values is None: + # Prefill mode + # out is of shape [cumulative_sequence_lengths, vocab_size] + logits = out[start_index:end_index] + else: + # Decode mode + # out is of shape [batch_size, vocab_size] + logits = out[i].unsqueeze(0) + + # Select next token + next_token_id, logprobs = next_token_chooser( + all_input_ids_tensor[None, :input_length], logits + ) + next_token_id_squeezed = next_token_id.squeeze() + next_token_id_item = next_token_id_squeezed.item() + + # Append next token to all tokens + all_input_ids.append(next_token_id_item) + all_input_ids_tensor[input_length] = next_token_id_item + new_input_length = input_length + 1 + + # Generated token + next_token_logprob = logprobs[-1, next_token_id_item] + next_token_text = self.decode_token( + next_token_id_item, + ) + + # Evaluate stopping criteria + stop, reason = stopping_criteria( + next_token_id_item, + next_token_text, + ) + + if stop: + # Decode generated tokens + output_text = self.decode( + all_input_ids[-stopping_criteria.current_tokens :] + ) + # Get seed + if isinstance(next_token_chooser.choice, Sampling): + seed = next_token_chooser.choice.seed + else: + seed = None + + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed + ) + else: + # Keep request in the batch + next_batch_keep_indices.append(i) + generated_text = None + + # Get sequence present + seq_present = present[:, start_index:end_index] + # Pad it for next iter attention + past = torch.nn.functional.pad(seq_present, (0, 0, 0, 0, 0, 0, 0, 1)) + next_batch_past_key_values.append(past) + + next_batch_input_ids.append(next_token_id) + next_batch_position_ids.append(input_length) + # Cumulative sum + next_batch_cu_seqlens.append( + next_batch_cu_seqlens[-1] + new_input_length + ) + next_batch_input_lengths.append(new_input_length) + next_batch_all_input_ids.append(all_input_ids) + next_batch_all_input_ids_tensor.append(all_input_ids_tensor) + next_batch_max_seqlen = max(next_batch_max_seqlen, new_input_length) + + # Prefill + if stopping_criteria.current_tokens == 1: + # Remove generated token to only have prefill and add nan for first prompt token + prefill_logprobs = [float("nan")] + logprobs.gather( + 1, all_input_ids_tensor[1:input_length].unsqueeze(1) + ).squeeze(1)[:-1].tolist() + prefill_token_ids = all_input_ids[:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = PrefillTokens( + prefill_token_ids, prefill_logprobs, prefill_texts + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id_item, + next_token_logprob, + next_token_text, + next_token_id_item in self.all_special_ids, + generated_text, + ) + + generations.append(generation) + cumulative_length += input_length + + # We finished all generations in the batch; there is no next batch + if not next_batch_keep_indices: + return generations, None + + # If we finished at least one generation, we need to evict the indices of the generations that finished + # from the values of the next batch + if len(next_batch_keep_indices) != len(batch): + # Apply indices to requests, token_choosers and stopping_criterias that need to be cached + next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] + next_batch_next_token_choosers = [ + batch.next_token_choosers[i] for i in next_batch_keep_indices + ] + next_batch_stopping_criterias = [ + batch.stopping_criterias[i] for i in next_batch_keep_indices + ] + else: + next_batch_requests = batch.requests + next_batch_next_token_choosers = batch.next_token_choosers + next_batch_stopping_criterias = batch.stopping_criterias + + # Create final next batch tensors + next_batch_position_ids = torch.tensor( + next_batch_position_ids, dtype=torch.int32 + ) + next_batch_cu_seqlens = torch.tensor(next_batch_cu_seqlens, dtype=torch.int32) + if len(next_batch_keep_indices) > 1: + next_batch_input_ids = torch.concat(next_batch_input_ids).squeeze(1) + next_batch_past_key_values = torch.concat(next_batch_past_key_values, dim=1) + else: + next_batch_input_ids = next_batch_input_ids[0].view(1) + next_batch_past_key_values = next_batch_past_key_values[0] + + next_batch = FlashCausalLMBatch( + batch_id=batch.batch_id, + requests=next_batch_requests, + input_ids=next_batch_input_ids, + position_ids=next_batch_position_ids, + cu_seqlens=next_batch_cu_seqlens, + max_seqlen=next_batch_max_seqlen, + past_key_values=next_batch_past_key_values, + input_lengths=next_batch_input_lengths, + all_input_ids=next_batch_all_input_ids, + all_input_ids_tensor=next_batch_all_input_ids_tensor, + next_token_choosers=next_batch_next_token_choosers, + stopping_criterias=next_batch_stopping_criterias, + ) + return generations, next_batch diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index b97f342a..e415a725 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -1,33 +1,20 @@ import torch import torch.distributed -from torch.nn import functional as F - from accelerate import init_empty_weights -from dataclasses import dataclass from opentelemetry import trace from safetensors import safe_open -from transformers import AutoTokenizer, PreTrainedTokenizerBase, AutoConfig -from typing import Optional, Tuple, List, Type, Union +from transformers import AutoTokenizer, AutoConfig +from typing import Optional, Tuple, List -from text_generation_server.models import Model -from text_generation_server.models.flash_neox_modeling import ( +from text_generation_server.models import FlashCausalLM +from text_generation_server.models.custom_modeling.flash_neox_modeling import ( FlashGPTNeoXForCausalLM, TensorParallelEmbedding, TensorParallelRowLinear, TensorParallelColumnLinear, ) -from text_generation_server.models.types import ( - Batch, - PrefillTokens, - Generation, - GeneratedText, -) -from text_generation_server.pb import generate_pb2 from text_generation_server.utils import ( - NextTokenChooser, - StoppingCriteria, - Sampling, initialize_torch_distributed, weight_files, ) @@ -35,437 +22,12 @@ tracer = trace.get_tracer(__name__) -@dataclass -class FlashNeoXBatch(Batch): - batch_id: int - requests: List[generate_pb2.Request] - - # Decoder values - input_ids: torch.Tensor - position_ids: torch.Tensor - # cumulative sequence lengths - cu_seqlens: torch.Tensor - max_seqlen: int - past_key_values: Optional[torch.Tensor] - - # All tokens - all_input_ids: List[List[int]] - all_input_ids_tensor: List[torch.Tensor] - - # Lengths of all generations present in the batch - input_lengths: List[int] - - # Generation helpers - next_token_choosers: List[NextTokenChooser] - stopping_criterias: List[StoppingCriteria] - - def to_pb(self) -> generate_pb2.Batch: - return generate_pb2.Batch( - id=self.batch_id, requests=self.requests, size=len(self) - ) - - @classmethod - def from_pb( - cls, - pb: generate_pb2.Batch, - tokenizer: PreTrainedTokenizerBase, - device: torch.device, - ) -> "CausalLMBatch": - input_ids = [] - position_ids = [] - cu_seqlens = [0] - max_seqlen = 0 - - input_lengths = [] - all_input_ids = [] - all_input_ids_tensor = [] - - next_token_choosers = [] - stopping_criterias = [] - - # Cumulative length - cumulative_length = 0 - - # Parse batch - for r in pb.requests: - tokenized_input = tokenizer(r.inputs)["input_ids"] - input_length = len(tokenized_input) - max_seqlen = max(max_seqlen, input_length) - input_lengths.append(input_length) - all_input_ids.append(tokenized_input) - - tokenized_input = torch.tensor(tokenized_input, device=device) - input_ids.append(tokenized_input) - - # Position ids - position_ids.append(torch.arange(0, input_length, dtype=torch.int32)) - - # Add cumulative lengths of all previous inputs - cu_seqlens.append(cumulative_length + input_length) - - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) - stopping_criteria = StoppingCriteria.from_pb( - r.stopping_parameters, tokenizer - ) - stopping_criterias.append(stopping_criteria) - all_input_ids_tensor.append( - F.pad(tokenized_input, (0, stopping_criteria.max_new_tokens)) - ) - - # Update - cumulative_length += input_length - - input_ids = torch.concat(input_ids) - position_ids = torch.concat(position_ids) - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32) - - return cls( - batch_id=pb.id, - requests=pb.requests, - input_ids=input_ids, - position_ids=position_ids, - cu_seqlens=cu_seqlens, - max_seqlen=max_seqlen, - past_key_values=None, - input_lengths=input_lengths, - all_input_ids=all_input_ids, - all_input_ids_tensor=all_input_ids_tensor, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - ) - - @classmethod - @tracer.start_as_current_span("concatenate") - def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": - # Batch attributes - requests = [] - input_lengths = [] - all_input_ids = [] - all_input_ids_tensor = [] - next_token_choosers = [] - stopping_criterias = [] - - # Batch tensors - input_ids = [] - position_ids = [] - cu_seqlens = [torch.tensor([0], dtype=torch.int32)] - max_seqlen = 0 - past_key_values = [] - - # Cumulative length - cumulative_length = torch.tensor(0) - - for i, batch in enumerate(batches): - requests.extend(batch.requests) - input_lengths.extend(batch.input_lengths) - all_input_ids.extend(batch.all_input_ids) - all_input_ids_tensor.extend(batch.all_input_ids_tensor) - next_token_choosers.extend(batch.next_token_choosers) - stopping_criterias.extend(batch.stopping_criterias) - - # Add cumulative lengths of all previous inputs - cu_seqlens.append(batch.cu_seqlens[1:] + cumulative_length) - - input_ids.append(batch.input_ids) - position_ids.append(batch.position_ids) - past_key_values.append(batch.past_key_values) - - max_seqlen = max(max_seqlen, batch.max_seqlen) - - # Update - cumulative_length += batch.cu_seqlens[-1] - - input_ids = torch.concat(input_ids) - position_ids = torch.concat(position_ids) - # Concat on dim=1 as first dim represents the model layers - past_key_values = torch.concat(past_key_values, dim=1) - cu_seqlens = torch.concat(cu_seqlens) - - return FlashNeoXBatch( - batch_id=batches[0].batch_id, - requests=requests, - input_ids=input_ids, - position_ids=position_ids, - cu_seqlens=cu_seqlens, - max_seqlen=max_seqlen, - past_key_values=past_key_values, - input_lengths=input_lengths, - all_input_ids=all_input_ids, - all_input_ids_tensor=all_input_ids_tensor, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - ) - - def __len__(self): - return len(self.requests) - - -class FlashNeoX(Model): +class FlashNeoX(FlashCausalLM): def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): - if torch.cuda.is_available(): - device = torch.device("cuda") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 - else: - raise NotImplementedError("FlashNeoX is only available on GPU") - - if quantize: - raise NotImplementedError("FlashNeoX does not support quantization") - - tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" - ) - self.model = ( - FlashGPTNeoXForCausalLM.from_pretrained( - model_id, - revision=revision, - torch_dtype=dtype, - ) - .eval() - .cuda() - ) - tokenizer.pad_token_id = ( - self.model.config.pad_token_id - if self.model.config.pad_token_id is not None - else self.model.config.eos_token_id - ) - super(FlashNeoX, self).__init__( - tokenizer=tokenizer, - device=device, - ) - - @property - def batch_type(self) -> Type[FlashNeoXBatch]: - return FlashNeoXBatch - - def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: - return self.tokenizer.decode( - generated_ids, skip_special_tokens=True, cleanup_tokenization_spaces=False - ) - - def forward( - self, - input_ids: torch.Tensor, - position_ids: torch.Tensor, - cu_seqlens: torch.Tensor, - max_s: int, - past_key_values: Optional = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - # Model Forward - return self.model.forward( - input_ids=input_ids, - position_ids=position_ids, - cu_seqlens=cu_seqlens, - max_s=max_s, - past_key_values=past_key_values, + FlashGPTNeoXForCausalLM, model_id, revision, quantize ) - @tracer.start_as_current_span("generate_token") - def generate_token( - self, batch: FlashNeoXBatch - ) -> Tuple[List[Generation], Optional[FlashNeoXBatch]]: - # Better to send to device here to avoid device issues in concatenate - position_ids = batch.position_ids.to(self.device, non_blocking=True) - cu_seqlens = batch.cu_seqlens.to(self.device) - - out, present = self.forward( - batch.input_ids, - position_ids, - cu_seqlens, - batch.max_seqlen, - batch.past_key_values, - ) - - # List of indices to cache - next_batch_keep_indices = [] - - # New values for next forward - next_batch_input_ids = [] - next_batch_position_ids = [] - next_batch_cu_seqlens = [0] - next_batch_max_seqlen = 0 - next_batch_past_key_values = [] - next_batch_input_lengths = [] - next_batch_all_input_ids = [] - next_batch_all_input_ids_tensor = [] - - # Cumulative length - cumulative_length = 0 - - # Results - generations: List[Generation] = [] - - # Zipped iterator - iterator = zip( - batch.requests, - batch.input_lengths, - batch.next_token_choosers, - batch.stopping_criterias, - batch.all_input_ids, - batch.all_input_ids_tensor, - ) - - # For each member of the batch - for i, ( - request, - input_length, - next_token_chooser, - stopping_criteria, - all_input_ids, - all_input_ids_tensor, - ) in enumerate(iterator): - # Indexing metadata - start_index = cumulative_length - end_index = cumulative_length + input_length - - if batch.past_key_values is None: - # Prefill mode - # out is of shape [cumulative_sequence_lengths, vocab_size] - logits = out[start_index:end_index] - else: - # Decode mode - # out is of shape [batch_size, vocab_size] - logits = out[i].unsqueeze(0) - - # Select next token - next_token_id, logprobs = next_token_chooser( - all_input_ids_tensor[None, :input_length], logits - ) - next_token_id_squeezed = next_token_id.squeeze() - next_token_id_item = next_token_id_squeezed.item() - - # Append next token to all tokens - all_input_ids.append(next_token_id_item) - all_input_ids_tensor[input_length] = next_token_id_item - new_input_length = input_length + 1 - - # Generated token - next_token_logprob = logprobs[-1, next_token_id_item] - next_token_text = self.decode_token( - next_token_id_item, - ) - - # Evaluate stopping criteria - stop, reason = stopping_criteria( - next_token_id_item, - next_token_text, - ) - - if stop: - # Decode generated tokens - output_text = self.decode( - all_input_ids[-stopping_criteria.current_tokens :] - ) - # Get seed - if isinstance(next_token_chooser.choice, Sampling): - seed = next_token_chooser.choice.seed - else: - seed = None - - generated_text = GeneratedText( - output_text, stopping_criteria.current_tokens, reason, seed - ) - else: - # Keep request in the batch - next_batch_keep_indices.append(i) - generated_text = None - - # Get sequence present - seq_present = present[:, start_index:end_index] - # Pad it for next iter attention - past = torch.nn.functional.pad(seq_present, (0, 0, 0, 0, 0, 0, 0, 1)) - next_batch_past_key_values.append(past) - - next_batch_input_ids.append(next_token_id) - next_batch_position_ids.append(input_length) - # Cumulative sum - next_batch_cu_seqlens.append( - next_batch_cu_seqlens[-1] + new_input_length - ) - next_batch_input_lengths.append(new_input_length) - next_batch_all_input_ids.append(all_input_ids) - next_batch_all_input_ids_tensor.append(all_input_ids_tensor) - next_batch_max_seqlen = max(next_batch_max_seqlen, new_input_length) - - # Prefill - if stopping_criteria.current_tokens == 1: - # Remove generated token to only have prefill and add nan for first prompt token - prefill_logprobs = [float("nan")] + logprobs.gather( - 1, all_input_ids_tensor[1:input_length].unsqueeze(1) - ).squeeze(1)[:-1].tolist() - prefill_token_ids = all_input_ids[:-1] - prefill_texts = self.tokenizer.batch_decode( - prefill_token_ids, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) - prefill_tokens = PrefillTokens( - prefill_token_ids, prefill_logprobs, prefill_texts - ) - else: - prefill_tokens = None - - generation = Generation( - request.id, - prefill_tokens, - next_token_id_item, - next_token_logprob, - next_token_text, - next_token_id_item in self.all_special_ids, - generated_text, - ) - - generations.append(generation) - cumulative_length += input_length - - # We finished all generations in the batch; there is no next batch - if not next_batch_keep_indices: - return generations, None - - # If we finished at least one generation, we need to evict the indices of the generations that finished - # from the values of the next batch - if len(next_batch_keep_indices) != len(batch): - # Apply indices to requests, token_choosers and stopping_criterias that need to be cached - next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] - next_batch_next_token_choosers = [ - batch.next_token_choosers[i] for i in next_batch_keep_indices - ] - next_batch_stopping_criterias = [ - batch.stopping_criterias[i] for i in next_batch_keep_indices - ] - else: - next_batch_requests = batch.requests - next_batch_next_token_choosers = batch.next_token_choosers - next_batch_stopping_criterias = batch.stopping_criterias - - # Create final next batch tensors - next_batch_position_ids = torch.tensor( - next_batch_position_ids, dtype=torch.int32 - ) - next_batch_cu_seqlens = torch.tensor(next_batch_cu_seqlens, dtype=torch.int32) - if len(next_batch_keep_indices) > 1: - next_batch_input_ids = torch.concat(next_batch_input_ids).squeeze(1) - next_batch_past_key_values = torch.concat(next_batch_past_key_values, dim=1) - else: - next_batch_input_ids = next_batch_input_ids[0].view(1) - next_batch_past_key_values = next_batch_past_key_values[0] - - next_batch = FlashNeoXBatch( - batch_id=batch.batch_id, - requests=next_batch_requests, - input_ids=next_batch_input_ids, - position_ids=next_batch_position_ids, - cu_seqlens=next_batch_cu_seqlens, - max_seqlen=next_batch_max_seqlen, - past_key_values=next_batch_past_key_values, - input_lengths=next_batch_input_lengths, - all_input_ids=next_batch_all_input_ids, - all_input_ids_tensor=next_batch_all_input_ids_tensor, - next_token_choosers=next_batch_next_token_choosers, - stopping_criterias=next_batch_stopping_criterias, - ) - return generations, next_batch - class FlashNeoXSharded(FlashNeoX): def __init__( @@ -508,7 +70,7 @@ def __init__( model.post_load_weights() self.model = model.eval().to(dtype) torch.distributed.barrier(group=self.process_group) - super(FlashNeoX, self).__init__( + super(FlashCausalLM, self).__init__( tokenizer=tokenizer, device=device, ) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py new file mode 100644 index 00000000..b33d0477 --- /dev/null +++ b/server/text_generation_server/models/flash_santacoder.py @@ -0,0 +1,138 @@ +import torch +import torch.distributed + +from accelerate import init_empty_weights +from opentelemetry import trace +from pathlib import Path +from transformers import AutoTokenizer, AutoConfig +from typing import Optional, List + +from text_generation_server.models import FlashCausalLM +from text_generation_server.models.custom_modeling.flash_santacoder_modeling import ( + FlashSantacoderForCausalLM +) +from text_generation_server.utils import ( + weight_files, + download_weights, + weight_hub_files, + LocalEntryNotFoundError, +) + +tracer = trace.get_tracer(__name__) + + +class FlashSantacoder(FlashCausalLM): + def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + else: + raise NotImplementedError("FlashSantacoder is only available on GPU") + + if quantize: + raise NotImplementedError("FlashSantacoder does not support quantization") + + tokenizer = AutoTokenizer.from_pretrained( + model_id, revision=revision, padding_side="left" + ) + + config = AutoConfig.from_pretrained( + model_id, revision=revision, + trust_remote_code=True # Needed as the config is not part of Transformers + ) + + # We do not use from_pretrained as we modified the model internal module layout + try: + filenames = weight_files(model_id, revision, ".bin") + # Local files not found + except LocalEntryNotFoundError: + hub_files = weight_hub_files(model_id, revision, ".bin") + filenames = download_weights(hub_files, model_id, revision) + + with init_empty_weights(): + model = FlashSantacoderForCausalLM(config) + + self.load_weights( + model, + filenames, + ) + self.model = model.eval().to(device).to(dtype) + + super(FlashCausalLM, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @staticmethod + def load_weights( + model: FlashSantacoderForCausalLM, + filenames: List[Path], + ): + for filename in filenames: + state_dict = torch.load(filename, map_location="cpu") + for key, value in state_dict.items(): + layer_name = ".".join(key.split(".")[:4]) + + # Fused qkv + if "q_attn.weight" in key or "kv_attn.weight" in key: + final_key = layer_name + ".attn.weight" + elif "q_attn.bias" in key or "kv_attn.bias" in key: + final_key = layer_name + ".attn.bias" + + else: + final_key = key + + module_name, param_name = final_key.rsplit(".", 1) + module = model.get_submodule(module_name) + + try: + current_parameter_tensor = module._parameters[param_name] + except KeyError: + current_parameter_tensor = None + + if current_parameter_tensor is not None: + if "c_fc.weight" in key or "c_proj.weight" in key or "q_attn.weight" in key or "kv_attn.weight" in key: + # Tranpose as we use nn.Linear instead of Conv1D + value = value.T + + if current_parameter_tensor.device == torch.device("meta"): + # Init qkv + if "attn.weight" in final_key: + module._parameters[param_name] = value.new_empty( + (model.transformer.head_size * (model.transformer.num_heads + 2), value.shape[1]) + ) + elif "attn.bias" in final_key: + module._parameters[param_name] = value.new_empty( + (model.transformer.head_size * (model.transformer.num_heads + 2)) + ) + + # Copy to correct slice + if "q_attn.weight" in key: + module._parameters[param_name][: value.shape[0]] = value + elif "q_attn.bias" in key: + module._parameters[param_name][: value.shape[0]] = value + elif "kv_attn.weight" in key: + module._parameters[param_name][ + model.transformer.head_size * model.transformer.num_heads: + ] = value + elif "kv_attn.bias" in key: + module._parameters[param_name][ + model.transformer.head_size * model.transformer.num_heads: + ] = value + else: + if current_parameter_tensor.shape != value.shape: + raise ValueError( + f"Name {final_key} -- Current {current_parameter_tensor.shape} and got {value.shape}" + ) + module._parameters[param_name] = value + else: + module._buffers[param_name] = value + + torch.cuda.empty_cache() + model.post_load_weights() + + def decode(self, generated_ids: List[int]) -> str: + # Do not skip special tokens as they are used for custom parsing rules of the generated text + return self.tokenizer.decode( + generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False + ) diff --git a/supported_models.json b/supported_models.json index 86d3bdfe..8e9dd9e0 100644 --- a/supported_models.json +++ b/supported_models.json @@ -1,9 +1,9 @@ [ + "bigcode/santacoder", "bigscience/bloom", "bigscience/bloomz", "EleutherAI/gpt-neox-20b", "google/flan-ul2", "google/flan-t5-xxl", - "OpenAssistant/oasst-sft-1-pythia-12b", - "olivierdehaene/optimized-santacoder" + "OpenAssistant/oasst-sft-1-pythia-12b" ] From 9122e7bd9c310ea7cf6052c332ad00e6f60b02ae Mon Sep 17 00:00:00 2001 From: Guspan Tanadi <36249910+guspan-tanadi@users.noreply.github.com> Date: Tue, 4 Apr 2023 18:27:46 +0700 Subject: [PATCH 143/793] docs(readme): provide link Logits Warper README (#154) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ee724487..ad938bdd 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ to power LLMs api-inference widgets. - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading - Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) -- Logits warpers (temperature scaling, topk, repetition penalty ...) +- Logits warper (temperature scaling, top-p, top-k, repetition penalty, more details see [transformers.LogitsProcessor](https://huggingface.co/docs/transformers/internal/generation_utils#transformers.LogitsProcessor)) - Stop sequences - Log probabilities - Production ready (distributed tracing with Open Telemetry, Prometheus metrics) From 3f2542bb6a6df97b617be82398833dcb3d66eca5 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 5 Apr 2023 19:37:41 +0200 Subject: [PATCH 144/793] fix(server): fix escape characters in stop sequence (#155) --- server/tests/utils/test_tokens.py | 9 ++ .../flash_santacoder_modeling.py | 116 +++++++++--------- .../models/flash_santacoder.py | 33 +++-- server/text_generation_server/utils/tokens.py | 1 + 4 files changed, 90 insertions(+), 69 deletions(-) diff --git a/server/tests/utils/test_tokens.py b/server/tests/utils/test_tokens.py index 3883ad97..da0006e4 100644 --- a/server/tests/utils/test_tokens.py +++ b/server/tests/utils/test_tokens.py @@ -14,6 +14,15 @@ def test_stop_sequence_criteria(): assert not criteria("/test; ") +def test_stop_sequence_criteria_escape(): + criteria = StopSequenceCriteria("<|stop|>") + + assert not criteria("<") + assert not criteria("<|stop") + assert criteria("<|stop|>") + assert not criteria("<|stop|> ") + + def test_stopping_criteria(): criteria = StoppingCriteria(0, [StopSequenceCriteria("/test;")], max_new_tokens=5) assert criteria(65827, "/test") == (False, None) diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index ef073636..799e7054 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -47,12 +47,12 @@ def forward(self, hidden_states, residual=None): class FastLinear(nn.Linear): def __init__( - self, - in_features: int, - out_features: int, - bias: bool = True, - device=None, - dtype=None, + self, + in_features: int, + out_features: int, + bias: bool = True, + device=None, + dtype=None, ) -> None: super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) @@ -67,10 +67,10 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: class FlashMQAttention(torch.nn.Module): def __init__( - self, - num_heads, - hidden_size, - process_group=None, + self, + num_heads, + hidden_size, + process_group=None, ): super().__init__() self.num_heads = num_heads @@ -86,13 +86,13 @@ def __init__( raise NotImplementedError def forward( - self, - hidden_states, - cu_seqlens, - max_s, - layer_past, - layer_past_present_indices, - cu_seqlens_q, + self, + hidden_states, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, ): qkv = self.attn(hidden_states) @@ -162,15 +162,17 @@ def forward( class MLP(nn.Module): - def __init__( - self, act, hidden_size, intermediate_size, process_group=None - ): + def __init__(self, act, hidden_size, intermediate_size, process_group=None): super().__init__() self.act = ( ACT2FN[act] if "gelu" not in act - else lambda x: torch.nn.functional.gelu(x, approximate="tanh" if act in ["gelu_fast", - "gelu_pytorch_tanh"] else None) + else lambda x: torch.nn.functional.gelu( + x, + approximate="tanh" + if act in ["gelu_fast", "gelu_pytorch_tanh"] + else None, + ) ) if process_group is None: @@ -188,13 +190,13 @@ def forward(self, hidden_states): class Block(nn.Module): def __init__( - self, - num_heads, - act, - hidden_size, - intermediate_size, - layer_norm_eps, - process_group=None, + self, + num_heads, + act, + hidden_size, + intermediate_size, + layer_norm_eps, + process_group=None, ): super().__init__() self.ln_1 = FastLayerNorm(hidden_size, eps=layer_norm_eps) @@ -212,14 +214,14 @@ def __init__( ) def forward( - self, - hidden_states, - residual, - cu_seqlens, - max_s, - layer_past, - layer_past_present_indices, - cu_seqlens_q, + self, + hidden_states, + residual, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, ): hidden_states, residual = self.ln_1(hidden_states, residual) @@ -232,9 +234,7 @@ def forward( cu_seqlens_q, ) - hidden_states, residual = self.ln_2( - hidden_states, residual - ) + hidden_states, residual = self.ln_2(hidden_states, residual) mlp_output = self.mlp(hidden_states) @@ -258,16 +258,16 @@ def __init__(self, config, process_group=None): config.num_attention_heads, config.activation_function, config.hidden_size, - config.n_inner if config.n_inner is not None else 4 * config.hidden_size, + config.n_inner + if config.n_inner is not None + else 4 * config.hidden_size, config.layer_norm_epsilon, process_group, ) for _ in range(config.num_hidden_layers) ] ) - self.ln_f = FastLayerNorm( - config.hidden_size, eps=config.layer_norm_epsilon - ) + self.ln_f = FastLayerNorm(config.hidden_size, eps=config.layer_norm_epsilon) self.head_size = self.h[0].attn.head_size self.num_heads = self.h[0].attn.num_heads @@ -281,12 +281,12 @@ def post_load_weights(self): layer.mlp.c_proj.transpose_weight() def forward( - self, - input_ids, - position_ids, - cu_seqlens, - max_s, - past_key_values=None, + self, + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values=None, ): hidden_states = self.wte(input_ids) + self.wpe(position_ids) @@ -335,21 +335,19 @@ def __init__(self, config, process_group=None): self.transformer = FlashSantacoderModel(config, process_group) - self.lm_head = FastLinear( - config.hidden_size, config.vocab_size, bias=False - ) + self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) def post_load_weights(self): self.transformer.post_load_weights() self.lm_head.transpose_weight() def forward( - self, - input_ids, - position_ids, - cu_seqlens, - max_s, - past_key_values=None, + self, + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values=None, ): hidden_states, present = self.transformer( input_ids, position_ids, cu_seqlens, max_s, past_key_values diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index b33d0477..f0207d55 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -9,7 +9,7 @@ from text_generation_server.models import FlashCausalLM from text_generation_server.models.custom_modeling.flash_santacoder_modeling import ( - FlashSantacoderForCausalLM + FlashSantacoderForCausalLM, ) from text_generation_server.utils import ( weight_files, @@ -37,8 +37,9 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False ) config = AutoConfig.from_pretrained( - model_id, revision=revision, - trust_remote_code=True # Needed as the config is not part of Transformers + model_id, + revision=revision, + trust_remote_code=True, # Needed as the config is not part of Transformers ) # We do not use from_pretrained as we modified the model internal module layout @@ -65,8 +66,8 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False @staticmethod def load_weights( - model: FlashSantacoderForCausalLM, - filenames: List[Path], + model: FlashSantacoderForCausalLM, + filenames: List[Path], ): for filename in filenames: state_dict = torch.load(filename, map_location="cpu") @@ -91,7 +92,12 @@ def load_weights( current_parameter_tensor = None if current_parameter_tensor is not None: - if "c_fc.weight" in key or "c_proj.weight" in key or "q_attn.weight" in key or "kv_attn.weight" in key: + if ( + "c_fc.weight" in key + or "c_proj.weight" in key + or "q_attn.weight" in key + or "kv_attn.weight" in key + ): # Tranpose as we use nn.Linear instead of Conv1D value = value.T @@ -99,11 +105,18 @@ def load_weights( # Init qkv if "attn.weight" in final_key: module._parameters[param_name] = value.new_empty( - (model.transformer.head_size * (model.transformer.num_heads + 2), value.shape[1]) + ( + model.transformer.head_size + * (model.transformer.num_heads + 2), + value.shape[1], + ) ) elif "attn.bias" in final_key: module._parameters[param_name] = value.new_empty( - (model.transformer.head_size * (model.transformer.num_heads + 2)) + ( + model.transformer.head_size + * (model.transformer.num_heads + 2) + ) ) # Copy to correct slice @@ -113,11 +126,11 @@ def load_weights( module._parameters[param_name][: value.shape[0]] = value elif "kv_attn.weight" in key: module._parameters[param_name][ - model.transformer.head_size * model.transformer.num_heads: + model.transformer.head_size * model.transformer.num_heads : ] = value elif "kv_attn.bias" in key: module._parameters[param_name][ - model.transformer.head_size * model.transformer.num_heads: + model.transformer.head_size * model.transformer.num_heads : ] = value else: if current_parameter_tensor.shape != value.shape: diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index b923857d..23f504c6 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -110,6 +110,7 @@ def from_pb( class StopSequenceCriteria: def __init__(self, stop_sequence: str): + stop_sequence = re.escape(stop_sequence) self.regex = re.compile(f".*{stop_sequence}$") def __call__(self, output: str) -> bool: From 1883d8ecde5e5c9e5140a399596e48208d1347b8 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sun, 9 Apr 2023 19:59:16 +0200 Subject: [PATCH 145/793] feat(docker): improve flash_attention caching (#160) --- Dockerfile | 6 ++++-- server/Makefile | 21 ++------------------- server/Makefile-flash-att | 10 ++++++++++ server/Makefile-transformers | 10 ++++++++++ 4 files changed, 26 insertions(+), 21 deletions(-) create mode 100644 server/Makefile-flash-att create mode 100644 server/Makefile-transformers diff --git a/Dockerfile b/Dockerfile index 85463af1..9fe0b49b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,14 +56,16 @@ WORKDIR /usr/src # Install torch RUN pip install torch --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir -COPY server/Makefile server/Makefile - # Install specific version of flash attention +COPY server/Makefile-flash-att server/Makefile RUN cd server && make install-flash-attention # Install specific version of transformers +COPY server/Makefile-transformers server/Makefile RUN cd server && BUILD_EXTENSIONS="True" make install-transformers +COPY server/Makefile server/Makefile + # Install server COPY proto proto COPY server server diff --git a/server/Makefile b/server/Makefile index d2a8cf7a..d827ceca 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,5 +1,5 @@ -transformers_commit := 2b57aa18da658e7d2f42ef6bd5b56751af582fef -flash_att_commit := 4d87e4d875077ad9efd25030efa4ab0ba92c19e1 +include Makefile-transformers +include Makefile-flash-att gen-server: # Compile protos @@ -10,23 +10,6 @@ gen-server: find text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; touch text_generation_server/pb/__init__.py -install-transformers: - # Install specific version of transformers with custom cuda kernels - pip uninstall transformers -y || true - rm -rf transformers || true - git clone https://github.com/OlivierDehaene/transformers.git - cd transformers && git checkout $(transformers_commit) - cd transformers && python setup.py install - -install-flash-attention: - # Install specific version of flash attention - pip install packaging - pip uninstall flash_attn rotary_emb dropout_layer_norm -y || true - rm -rf flash-attention || true - git clone https://github.com/HazyResearch/flash-attention.git - cd flash-attention && git checkout $(flash_att_commit) - cd flash-attention && python setup.py install && cd csrc/layer_norm && python setup.py install && cd ../rotary && python setup.py install - install-torch: # Install specific version of torch pip install torch --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir diff --git a/server/Makefile-flash-att b/server/Makefile-flash-att new file mode 100644 index 00000000..297fd9d0 --- /dev/null +++ b/server/Makefile-flash-att @@ -0,0 +1,10 @@ +flash_att_commit := d478eeec8f16c7939c54e4617dbd36f59b8eeed7 + +install-flash-attention: + # Install specific version of flash attention + pip install packaging + pip uninstall flash_attn rotary_emb dropout_layer_norm -y || true + rm -rf flash-attention || true + git clone https://github.com/HazyResearch/flash-attention.git + cd flash-attention && git checkout $(flash_att_commit) + cd flash-attention && python setup.py install && cd csrc/layer_norm && python setup.py install && cd ../rotary && python setup.py install \ No newline at end of file diff --git a/server/Makefile-transformers b/server/Makefile-transformers new file mode 100644 index 00000000..1e081336 --- /dev/null +++ b/server/Makefile-transformers @@ -0,0 +1,10 @@ +transformers_commit := b8d969ff47c6a9d40538a6ea33df021953363afc + +install-transformers: + # Install specific version of transformers with custom cuda kernels + pip install --upgrade setuptools + pip uninstall transformers -y || true + rm -rf transformers || true + git clone https://github.com/OlivierDehaene/transformers.git + cd transformers && git checkout $(transformers_commit) + cd transformers && python setup.py install \ No newline at end of file From e63a21eb4da9db60c973412820dd4c8644883579 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sun, 9 Apr 2023 20:00:05 +0200 Subject: [PATCH 146/793] feat(launcher): allow disabling hf_transfer (#161) --- launcher/src/main.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 80466fe6..a598a8bb 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -200,7 +200,16 @@ fn main() -> ExitCode { }; // Enable hf transfer for insane download speeds - env.push(("HF_HUB_ENABLE_HF_TRANSFER".into(), "1".into())); + let enable_hf_transfer = env::var("HF_HUB_ENABLE_HF_TRANSFER").unwrap_or("1".to_string()); + env.push(( + "HF_HUB_ENABLE_HF_TRANSFER".into(), + enable_hf_transfer.into(), + )); + + // Parse Inference API token + if let Ok(api_token) = env::var("HF_API_TOKEN") { + env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) + }; // Start process tracing::info!("Starting download process."); @@ -555,7 +564,16 @@ fn shard_manager( env.push(("SAFETENSORS_FAST_GPU".into(), "1".into())); // Enable hf transfer for insane download speeds - env.push(("HF_HUB_ENABLE_HF_TRANSFER".into(), "1".into())); + let enable_hf_transfer = env::var("HF_HUB_ENABLE_HF_TRANSFER").unwrap_or("1".to_string()); + env.push(( + "HF_HUB_ENABLE_HF_TRANSFER".into(), + enable_hf_transfer.into(), + )); + + // Parse Inference API token + if let Ok(api_token) = env::var("HF_API_TOKEN") { + env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) + }; // If huggingface_hub_cache is some, pass it to the shard // Useful when running inside a docker container From 5cddc055e6c0521e4ef11c487fd5f1d9e7ef1e62 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sun, 9 Apr 2023 20:07:02 +0200 Subject: [PATCH 147/793] fix(rust-client): use join_all instead of select_all to hopefully fix nccl issues (#162) --- router/client/src/sharded_client.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 7f0ec6f2..31f76312 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -2,7 +2,6 @@ use crate::Result; use crate::{Batch, Client, Generation}; use futures::future::join_all; -use futures::future::select_all; use tonic::transport::Uri; use tracing::instrument; @@ -60,9 +59,8 @@ impl ShardedClient { .iter_mut() .map(|client| Box::pin(client.prefill(batch.clone()))) .collect(); - // As soon as we receive one response, we can return as all shards will return the same - let (result, _, _) = select_all(futures).await; - result + // all shards return the same message + join_all(futures).await.pop().unwrap() } /// Generate one token for each request in the given cached batches @@ -79,8 +77,7 @@ impl ShardedClient { .iter_mut() .map(|client| Box::pin(client.decode(batches.clone()))) .collect(); - // As soon as we receive one response, we can return as all shards will return the same - let (result, _, _) = select_all(futures).await; - result + // all shards return the same message + join_all(futures).await.pop().unwrap() } } From 7dec65a24497fa0e1a56edd39b8e4e01ec44be12 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sun, 9 Apr 2023 20:13:28 +0200 Subject: [PATCH 148/793] fix(router): use buckets for metrics histograms (#163) --- router/src/infer.rs | 6 ++- router/src/server.rs | 97 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 79 insertions(+), 24 deletions(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index 5eafc3e9..2df9c5be 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -331,11 +331,12 @@ async fn prefill( ) -> Option { let start_time = Instant::now(); let batch_id = batch.id; + metrics::increment_counter!("tgi_batch_inference_count", "method" => "prefill"); match client.prefill(batch).await { Ok((generations, next_batch)) => { send_generations(generations, entries); - metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed(), "method" => "prefill"); + metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "prefill"); metrics::increment_counter!("tgi_batch_inference_success", "method" => "prefill"); next_batch } @@ -356,11 +357,12 @@ async fn decode( entries: &mut IntMap, ) -> Option { let start_time = Instant::now(); + metrics::increment_counter!("tgi_batch_inference_count", "method" => "decode"); match client.decode(batches).await { Ok((generations, next_batch)) => { send_generations(generations, entries); - metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed(), "method" => "decode"); + metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "decode"); metrics::increment_counter!("tgi_batch_inference_success", "method" => "decode"); next_batch } diff --git a/router/src/server.rs b/router/src/server.rs index f7850053..e1b402a6 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -14,7 +14,7 @@ use axum::routing::{get, post}; use axum::{http, Json, Router}; use axum_tracing_opentelemetry::opentelemetry_tracing_layer; use futures::Stream; -use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; use std::convert::Infallible; use std::net::SocketAddr; use text_generation_client::ShardedClient; @@ -120,6 +120,7 @@ async fn generate( ) -> Result<(HeaderMap, Json), (StatusCode, Json)> { let span = tracing::Span::current(); let start_time = Instant::now(); + metrics::increment_counter!("tgi_request_count"); let compute_characters = req.0.inputs.chars().count(); let mut add_prompt = None; @@ -185,6 +186,14 @@ async fn generate( let inference_time = Instant::now() - response.start; let time_per_token = inference_time / response.generated_text.generated_tokens; + // Tracing metadata + span.record("total_time", format!("{total_time:?}")); + span.record("validation_time", format!("{validation_time:?}")); + span.record("queue_time", format!("{queue_time:?}")); + span.record("inference_time", format!("{inference_time:?}")); + span.record("time_per_token", format!("{time_per_token:?}")); + span.record("seed", format!("{:?}", response.generated_text.seed)); + // Headers let mut headers = HeaderMap::new(); headers.insert("x-compute-type", "gpu+optimized".parse().unwrap()); @@ -217,22 +226,22 @@ async fn generate( time_per_token.as_millis().to_string().parse().unwrap(), ); - // Tracing metadata - span.record("total_time", format!("{total_time:?}")); - span.record("validation_time", format!("{validation_time:?}")); - span.record("queue_time", format!("{queue_time:?}")); - span.record("inference_time", format!("{inference_time:?}")); - span.record("time_per_token", format!("{time_per_token:?}")); - span.record("seed", format!("{:?}", response.generated_text.seed)); - tracing::info!("Output: {}", response.generated_text.text); - // Metrics metrics::increment_counter!("tgi_request_success"); - metrics::histogram!("tgi_request_duration", total_time); - metrics::histogram!("tgi_request_validation_duration", validation_time); - metrics::histogram!("tgi_request_queue_duration", queue_time); - metrics::histogram!("tgi_request_inference_duration", inference_time); - metrics::histogram!("tgi_request_mean_time_per_token_duration", time_per_token); + metrics::histogram!("tgi_request_duration", total_time.as_secs_f64()); + metrics::histogram!( + "tgi_request_validation_duration", + validation_time.as_secs_f64() + ); + metrics::histogram!("tgi_request_queue_duration", queue_time.as_secs_f64()); + metrics::histogram!( + "tgi_request_inference_duration", + inference_time.as_secs_f64() + ); + metrics::histogram!( + "tgi_request_mean_time_per_token_duration", + time_per_token.as_secs_f64() + ); metrics::histogram!( "tgi_request_generated_tokens", response.generated_text.generated_tokens as f64 @@ -244,6 +253,8 @@ async fn generate( output_text = prompt + &output_text; } + tracing::info!("Output: {}", output_text); + let response = GenerateResponse { generated_text: output_text, details, @@ -294,6 +305,7 @@ async fn generate_stream( ) { let span = tracing::Span::current(); let start_time = Instant::now(); + metrics::increment_counter!("tgi_request_count"); let compute_characters = req.0.inputs.chars().count(); @@ -368,15 +380,14 @@ async fn generate_stream( span.record("inference_time", format!("{inference_time:?}")); span.record("time_per_token", format!("{time_per_token:?}")); span.record("seed", format!("{:?}", generated_text.seed)); - tracing::info!(parent: &span, "Output: {}", generated_text.text); // Metrics metrics::increment_counter!("tgi_request_success"); - metrics::histogram!("tgi_request_duration", total_time); - metrics::histogram!("tgi_request_validation_duration", validation_time); - metrics::histogram!("tgi_request_queue_duration", queue_time); - metrics::histogram!("tgi_request_inference_duration", inference_time); - metrics::histogram!("tgi_request_mean_time_per_token_duration", time_per_token); + metrics::histogram!("tgi_request_duration", total_time.as_secs_f64()); + metrics::histogram!("tgi_request_validation_duration", validation_time.as_secs_f64()); + metrics::histogram!("tgi_request_queue_duration", queue_time.as_secs_f64()); + metrics::histogram!("tgi_request_inference_duration", inference_time.as_secs_f64()); + metrics::histogram!("tgi_request_mean_time_per_token_duration", time_per_token.as_secs_f64()); metrics::histogram!("tgi_request_generated_tokens", generated_text.generated_tokens as f64); // StreamResponse @@ -387,6 +398,8 @@ async fn generate_stream( output_text = prompt + &output_text; } + tracing::info!(parent: &span, "Output: {}", output_text); + let stream_token = StreamResponse { token, generated_text: Some(output_text), @@ -513,8 +526,48 @@ pub async fn run( max_concurrent_requests, ); + // Duration buckets + let duration_matcher = Matcher::Suffix(String::from("duration")); + let n_duration_buckets = 35; + let mut duration_buckets = Vec::with_capacity(n_duration_buckets); + // Minimum duration in seconds + let mut value = 0.0001; + for _ in 0..n_duration_buckets { + // geometric sequence + value *= 1.5; + duration_buckets.push(value); + } + // Input Length buckets + let input_length_matcher = Matcher::Full(String::from("tgi_request_input_length")); + let input_length_buckets: Vec = (0..100) + .map(|x| (max_input_length as f64 / 100.0) * (x + 1) as f64) + .collect(); + // Generated tokens buckets + let generated_tokens_matcher = Matcher::Full(String::from("tgi_request_generated_tokens")); + let generated_tokens_buckets: Vec = (0..100) + .map(|x| (max_total_tokens as f64 / 100.0) * (x + 1) as f64) + .collect(); + // Input Length buckets + let max_new_tokens_matcher = Matcher::Full(String::from("tgi_request_max_new_tokens")); + let max_new_tokens_buckets: Vec = (0..100) + .map(|x| (max_total_tokens as f64 / 100.0) * (x + 1) as f64) + .collect(); + // Batch size buckets + let batch_size_matcher = Matcher::Full(String::from("tgi_batch_next_size")); + let batch_size_buckets: Vec = (0..max_batch_size).map(|x| (x + 1) as f64).collect(); + // Prometheus handler - let builder = PrometheusBuilder::new(); + let builder = PrometheusBuilder::new() + .set_buckets_for_metric(duration_matcher, &duration_buckets) + .unwrap() + .set_buckets_for_metric(input_length_matcher, &input_length_buckets) + .unwrap() + .set_buckets_for_metric(generated_tokens_matcher, &generated_tokens_buckets) + .unwrap() + .set_buckets_for_metric(max_new_tokens_matcher, &max_new_tokens_buckets) + .unwrap() + .set_buckets_for_metric(batch_size_matcher, &batch_size_buckets) + .unwrap(); let prom_handle = builder .install_recorder() .expect("failed to install metrics recorder"); From 9987960062e40de2deae030ab7e4ad6f57de0b20 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sun, 9 Apr 2023 20:22:27 +0200 Subject: [PATCH 149/793] feat(router): make router input validation optional (#164) --- Cargo.lock | 817 +++++++++++------- benchmark/Cargo.lock | 303 ++++--- benchmark/Cargo.toml | 2 +- benchmark/src/generation.rs | 6 +- proto/generate.proto | 6 +- router/Cargo.toml | 5 +- router/src/infer.rs | 18 +- router/src/main.rs | 13 +- router/src/queue.rs | 19 +- router/src/server.rs | 142 +-- router/src/validation.rs | 525 ++++++----- server/tests/models/test_bloom.py | 1 + server/tests/models/test_causal_lm.py | 1 + server/tests/models/test_santacoder.py | 2 + server/tests/models/test_seq2seq_lm.py | 1 + server/text_generation_server/models/bloom.py | 2 +- .../models/causal_lm.py | 4 + .../custom_modeling/flash_neox_modeling.py | 24 +- .../flash_santacoder_modeling.py | 2 +- .../models/flash_causal_lm.py | 6 +- .../models/flash_neox.py | 36 +- .../models/flash_santacoder.py | 10 +- .../models/galactica.py | 18 +- .../text_generation_server/models/gpt_neox.py | 2 +- .../models/santacoder.py | 2 +- .../models/seq2seq_lm.py | 4 + server/text_generation_server/models/t5.py | 2 +- 27 files changed, 1117 insertions(+), 856 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e469f23..904dd32e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,18 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", + "opaque-debug", +] + [[package]] name = "ahash" version = "0.7.6" @@ -29,19 +41,50 @@ dependencies = [ ] [[package]] -name = "ansi_term" -version = "0.12.1" +name = "anstream" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" dependencies = [ - "winapi", + "anstyle", + "anstyle-parse", + "anstyle-wincon", + "concolor-override", + "concolor-query", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" + +[[package]] +name = "anstyle-parse" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-wincon" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +dependencies = [ + "anstyle", + "windows-sys 0.45.0", ] [[package]] name = "anyhow" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "async-stream" @@ -62,29 +105,18 @@ checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", + "syn 2.0.12", ] [[package]] @@ -95,9 +127,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.9" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6137c6234afb339e75e764c866e3594900f0211e1315d33779f269bbe2ec6967" +checksum = "349f8ccfd9221ee7d1f3d4b33e1f8319b3a81ed8f61f2ea40b37b859794b4491" dependencies = [ "async-trait", "axum-core", @@ -121,16 +153,15 @@ dependencies = [ "sync_wrapper", "tokio", "tower", - "tower-http 0.4.0", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" +checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e" dependencies = [ "async-trait", "bytes", @@ -154,7 +185,7 @@ dependencies = [ "http", "opentelemetry", "tower", - "tower-http 0.3.5", + "tower-http", "tracing", "tracing-opentelemetry", ] @@ -171,6 +202,12 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "1.3.2" @@ -179,9 +216,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] @@ -227,9 +264,9 @@ dependencies = [ [[package]] name = "cached-path" -version = "0.5.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f1c56d30236522ab3393a08746b138d4e16372001f42d29c88d513aeb8ab7ef" +checksum = "097968e38f1319207f057d0f4d76452e4f4f847a5de61c5215379f297fa034f3" dependencies = [ "flate2", "fs2", @@ -244,8 +281,7 @@ dependencies = [ "tar", "tempfile", "thiserror", - "zip 0.5.13", - "zip-extensions", + "zip", ] [[package]] @@ -253,6 +289,9 @@ name = "cc" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -261,55 +300,69 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "clap" -version = "2.34.0" +name = "cipher" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" dependencies = [ - "ansi_term", - "atty", - "bitflags", - "strsim 0.8.0", - "textwrap", - "unicode-width", - "vec_map", + "generic-array", ] [[package]] name = "clap" -version = "4.1.8" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" +checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" dependencies = [ - "bitflags", + "clap_builder", "clap_derive", - "clap_lex", - "is-terminal", "once_cell", - "strsim 0.10.0", - "termcolor", +] + +[[package]] +name = "clap_builder" +version = "4.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +dependencies = [ + "anstream", + "anstyle", + "bitflags", + "clap_lex", + "strsim", ] [[package]] name = "clap_derive" -version = "4.1.8" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" +checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" dependencies = [ "heck", - "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 2.0.12", ] [[package]] name = "clap_lex" -version = "0.3.2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" + +[[package]] +name = "concolor-override" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350b9cf31731f9957399229e9b2adc51eeabdfbe9d71d9a0552275fd12710d09" +checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" + +[[package]] +name = "concolor-query" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" dependencies = [ - "os_str_bytes", + "windows-sys 0.45.0", ] [[package]] @@ -325,6 +378,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "core-foundation" version = "0.9.3" @@ -343,9 +402,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" dependencies = [ "libc", ] @@ -424,9 +483,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.10.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ "darling_core", "darling_macro", @@ -434,27 +493,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.10.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.9.3", - "syn", + "strsim", + "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.10.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ "darling_core", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -472,46 +531,44 @@ dependencies = [ [[package]] name = "derive_builder" -version = "0.9.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" dependencies = [ - "darling", - "derive_builder_core", - "proc-macro2", - "quote", - "syn", + "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.9.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] -name = "digest" -version = "0.10.6" +name = "derive_builder_macro" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" dependencies = [ - "block-buffer", - "crypto-common", + "derive_builder_core", + "syn 1.0.109", ] [[package]] -name = "dirs" -version = "3.0.2" +name = "digest" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ - "dirs-sys", + "block-buffer", + "crypto-common", + "subtle", ] [[package]] @@ -557,13 +614,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.45.0", ] [[package]] @@ -602,7 +659,7 @@ checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "windows-sys 0.45.0", ] @@ -628,6 +685,19 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853" +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "pin-project", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -670,9 +740,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -685,9 +755,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -695,15 +765,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -712,38 +782,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-macro" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.12", ] [[package]] name = "futures-sink" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-channel", "futures-core", @@ -759,9 +829,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -774,8 +844,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -828,15 +900,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -852,6 +915,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "0.2.9" @@ -894,9 +966,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.24" +version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ "bytes", "futures-channel", @@ -959,9 +1031,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown", @@ -1003,25 +1075,26 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ + "hermit-abi 0.3.1", "libc", "windows-sys 0.45.0", ] [[package]] name = "ipnet" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" [[package]] name = "is-terminal" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" +checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", @@ -1058,9 +1131,18 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] [[package]] name = "js-sys" @@ -1079,15 +1161,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" [[package]] name = "lock_api" @@ -1201,7 +1283,7 @@ checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1223,9 +1305,9 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" @@ -1264,12 +1346,42 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "monostate" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.12", +] + [[package]] name = "multimap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1376,11 +1488,17 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" -version = "0.10.45" +version = "0.10.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" dependencies = [ "bitflags", "cfg-if", @@ -1399,7 +1517,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1410,9 +1528,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.80" +version = "0.9.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" dependencies = [ "autocfg", "cc", @@ -1501,12 +1619,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "os_str_bytes" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" - [[package]] name = "overload" version = "0.1.1" @@ -1531,16 +1643,39 @@ checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "windows-sys 0.45.0", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + [[package]] name = "paste" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] [[package]] name = "percent-encoding" @@ -1575,7 +1710,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1610,12 +1745,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" dependencies = [ "proc-macro2", - "syn", + "syn 1.0.109", ] [[package]] @@ -1627,7 +1762,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -1644,9 +1779,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "e472a104799c74b514a57226160104aa483546de37e839ec50e3c2e41dd87534" dependencies = [ "unicode-ident", ] @@ -1678,7 +1813,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 1.0.109", "tempfile", "which", ] @@ -1693,7 +1828,7 @@ dependencies = [ "itertools 0.10.5", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1723,9 +1858,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -1771,9 +1906,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -1792,9 +1927,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -1811,6 +1946,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -1818,15 +1962,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "redox_syscall", + "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.7.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" dependencies = [ "aho-corasick", "memchr", @@ -1844,15 +1988,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" dependencies = [ "base64 0.21.0", "bytes", @@ -1887,9 +2031,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "6.4.2" +version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "283ffe2f866869428c92e0d61c2f35dfb4355293cdfdc48f49e895c15f1333d1" +checksum = "1b68543d5527e158213414a92832d2aab11a84d2571a5eb021ebe22c43aab066" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -1898,23 +2042,23 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.3.1" +version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ab23d42d71fb9be1b643fe6765d292c5e14d46912d13f3ae2815ca048ea04d" +checksum = "4d4e0f0ced47ded9a68374ac145edd65a6c1fa13a96447b873660b2a568a0fd7" dependencies = [ "proc-macro2", "quote", "rust-embed-utils", "shellexpand", - "syn", + "syn 1.0.109", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "7.3.0" +version = "7.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1669d81dfabd1b5f8e2856b8bbe146c6192b0ba22162edc738ac0a5de18f054" +checksum = "512b0ab6853f7e14e3c8754acb43d6f748bb9ced66aa5915a6553ac8213f7731" dependencies = [ "sha2", "walkdir", @@ -1922,9 +2066,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.8" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "0e78cc525325c06b4a7ff02db283472f3c042b7ff0c391f96c6d5ac6f4f91b75" dependencies = [ "bitflags", "errno", @@ -1936,15 +2080,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "same-file" @@ -1995,29 +2139,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.12", ] [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" dependencies = [ "itoa", "ryu", @@ -2026,9 +2170,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" +checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" dependencies = [ "serde", ] @@ -2045,6 +2189,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.6" @@ -2071,7 +2226,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" dependencies = [ - "dirs 4.0.0", + "dirs", ] [[package]] @@ -2106,14 +2261,23 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a21dcece9b5991cfd1ece74654c8e3d0d5aab499d359b0395e38229c0bb5a3" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spm_precompiled" version = "0.1.4" @@ -2132,18 +2296,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "strsim" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" - [[package]] name = "strsim" version = "0.10.0" @@ -2160,6 +2312,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "syn" version = "1.0.109" @@ -2171,6 +2329,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79d9531f94112cfc3e4c8f5f02cb2b58f72c97b7efd85f70203cc6d8efda5927" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "0.1.2" @@ -2190,24 +2359,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", - "windows-sys 0.42.0", -] - -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", + "windows-sys 0.45.0", ] [[package]] @@ -2231,7 +2391,7 @@ dependencies = [ name = "text-generation-launcher" version = "0.4.3" dependencies = [ - "clap 4.1.8", + "clap", "ctrlc", "float_eq", "reqwest", @@ -2249,14 +2409,14 @@ dependencies = [ "async-stream", "axum", "axum-tracing-opentelemetry", - "clap 4.1.8", + "clap", + "flume", "futures", "metrics", "metrics-exporter-prometheus", "nohash-hasher", "opentelemetry", "opentelemetry-otlp", - "parking_lot", "rand", "reqwest", "serde", @@ -2265,8 +2425,7 @@ dependencies = [ "thiserror", "tokenizers", "tokio", - "tokio-stream", - "tower-http 0.3.5", + "tower-http", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -2274,33 +2433,24 @@ dependencies = [ "utoipa-swagger-ui", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.12", ] [[package]] @@ -2315,14 +2465,20 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ - "libc", - "winapi", + "serde", + "time-core", ] +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + [[package]] name = "tinyvec" version = "1.6.0" @@ -2340,15 +2496,15 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ff2dd291eac98dcea13e8cf7a0b28c373a90dc9210ccdab0fa9e69ee0cac69" +checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5" dependencies = [ "aho-corasick", "cached-path", - "clap 2.34.0", + "clap", "derive_builder", - "dirs 3.0.2", + "dirs", "esaxx-rs", "getrandom", "indicatif 0.15.0", @@ -2356,6 +2512,7 @@ dependencies = [ "lazy_static", "log", "macro_rules_attribute", + "monostate", "onig", "paste", "rand", @@ -2375,14 +2532,13 @@ dependencies = [ [[package]] name = "tokio" -version = "1.26.0" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" +checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" dependencies = [ "autocfg", "bytes", "libc", - "memchr", "mio", "num_cpus", "parking_lot", @@ -2405,13 +2561,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.12", ] [[package]] @@ -2491,7 +2647,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2533,25 +2689,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower-http" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - [[package]] name = "tower-layer" version = "0.3.2" @@ -2585,7 +2722,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2697,15 +2834,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.10" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -2754,11 +2891,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "utoipa" -version = "3.0.3" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15f6da6a2b471134ca44b7d18e8a76d73035cf8b3ed24c4dd5ca6a63aa439c5" +checksum = "24e7ee17c9ef094b86e1e04170d90765bd76cb381921dacb4d3e175a267bdae6" dependencies = [ "indexmap", "serde", @@ -2768,21 +2911,21 @@ dependencies = [ [[package]] name = "utoipa-gen" -version = "3.0.3" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2e33027986a4707b3f5c37ed01b33d0e5a53da30204b52ff18f80600f1d0ec" +checksum = "df6f458e5abc811d44aca28455efc4163fb7565a7af2aa32d17611f3d1d9794d" dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 2.0.12", ] [[package]] name = "utoipa-swagger-ui" -version = "3.0.2" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae3d4f4da6408f0f20ff58196ed619c94306ab32635aeca3d3fa0768c0bd0de2" +checksum = "40a6d6921f30f7a9d023a2523cbff2e94a1a4b535abbdda6e6d0c69b61f19f05" dependencies = [ "axum", "mime_guess", @@ -2791,7 +2934,7 @@ dependencies = [ "serde", "serde_json", "utoipa", - "zip 0.6.4", + "zip", ] [[package]] @@ -2806,12 +2949,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "version_check" version = "0.9.4" @@ -2820,12 +2957,11 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -2872,7 +3008,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -2906,7 +3042,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2995,9 +3131,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -3010,45 +3146,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "winreg" @@ -3070,35 +3206,50 @@ dependencies = [ [[package]] name = "zip" -version = "0.5.13" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" dependencies = [ + "aes", "byteorder", "bzip2", + "constant_time_eq", "crc32fast", + "crossbeam-utils", "flate2", - "thiserror", + "hmac", + "pbkdf2", + "sha1", "time", + "zstd", ] [[package]] -name = "zip" -version = "0.6.4" +name = "zstd" +version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "byteorder", - "crc32fast", - "crossbeam-utils", - "flate2", + "zstd-safe", ] [[package]] -name = "zip-extensions" -version = "0.6.1" +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c3c977bc3434ce2d4bcea8ad3c644672de0f2c402b72b9171ca80a8885d14" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" dependencies = [ - "zip 0.5.13", + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", ] diff --git a/benchmark/Cargo.lock b/benchmark/Cargo.lock index 8441014b..cb9c7533 100644 --- a/benchmark/Cargo.lock +++ b/benchmark/Cargo.lock @@ -9,21 +9,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] -name = "aho-corasick" -version = "0.7.20" +name = "aes" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ - "memchr", + "cfg-if", + "cipher", + "cpufeatures", + "opaque-debug", ] [[package]] -name = "ansi_term" -version = "0.12.1" +name = "aho-corasick" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ - "winapi", + "memchr", ] [[package]] @@ -105,17 +108,6 @@ dependencies = [ "syn 2.0.11", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -190,6 +182,12 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "1.3.2" @@ -246,9 +244,9 @@ dependencies = [ [[package]] name = "cached-path" -version = "0.5.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f1c56d30236522ab3393a08746b138d4e16372001f42d29c88d513aeb8ab7ef" +checksum = "097968e38f1319207f057d0f4d76452e4f4f847a5de61c5215379f297fa034f3" dependencies = [ "flate2", "fs2", @@ -264,7 +262,6 @@ dependencies = [ "tempfile", "thiserror", "zip", - "zip-extensions", ] [[package]] @@ -278,6 +275,9 @@ name = "cc" version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -286,18 +286,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "clap" -version = "2.34.0" +name = "cipher" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" dependencies = [ - "ansi_term", - "atty", - "bitflags", - "strsim 0.8.0", - "textwrap", - "unicode-width", - "vec_map", + "generic-array", ] [[package]] @@ -321,7 +315,7 @@ dependencies = [ "anstyle", "bitflags", "clap_lex", - "strsim 0.10.0", + "strsim", ] [[package]] @@ -370,6 +364,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "core-foundation" version = "0.9.3" @@ -484,9 +484,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.10.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" dependencies = [ "darling_core", "darling_macro", @@ -494,23 +494,23 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.10.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.9.3", + "strsim", "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.10.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ "darling_core", "quote", @@ -532,22 +532,18 @@ dependencies = [ [[package]] name = "derive_builder" -version = "0.9.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" dependencies = [ - "darling", - "derive_builder_core", - "proc-macro2", - "quote", - "syn 1.0.109", + "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.9.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" dependencies = [ "darling", "proc-macro2", @@ -555,6 +551,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + [[package]] name = "digest" version = "0.10.6" @@ -563,13 +569,14 @@ checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] name = "dirs" -version = "3.0.2" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ "dirs-sys", ] @@ -835,7 +842,7 @@ checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] [[package]] @@ -885,15 +892,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -909,6 +907,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "0.2.9" @@ -1113,6 +1120,15 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.61" @@ -1240,10 +1256,31 @@ checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.45.0", ] +[[package]] +name = "monostate" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.11", +] + [[package]] name = "multimap" version = "0.8.3" @@ -1348,6 +1385,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" version = "0.10.48" @@ -1468,12 +1511,35 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + [[package]] name = "paste" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -1891,6 +1957,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.6" @@ -1980,21 +2057,15 @@ dependencies = [ [[package]] name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "strsim" -version = "0.9.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "strsim" -version = "0.10.0" +name = "subtle" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" @@ -2053,7 +2124,7 @@ name = "text-generation-benchmark" version = "0.1.0" dependencies = [ "average", - "clap 4.2.1", + "clap", "crossterm", "float-ord", "ratatui", @@ -2084,15 +2155,6 @@ dependencies = [ "tracing-error", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - [[package]] name = "thiserror" version = "1.0.40" @@ -2125,15 +2187,20 @@ dependencies = [ [[package]] name = "time" -version = "0.1.45" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", + "serde", + "time-core", ] +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + [[package]] name = "tinyvec" version = "1.6.0" @@ -2151,13 +2218,13 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ff2dd291eac98dcea13e8cf7a0b28c373a90dc9210ccdab0fa9e69ee0cac69" +checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5" dependencies = [ "aho-corasick", "cached-path", - "clap 2.34.0", + "clap", "derive_builder", "dirs", "esaxx-rs", @@ -2167,6 +2234,7 @@ dependencies = [ "lazy_static", "log", "macro_rules_attribute", + "monostate", "onig", "paste", "rand", @@ -2535,12 +2603,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "version_check" version = "0.9.4" @@ -2557,12 +2619,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2779,23 +2835,50 @@ dependencies = [ [[package]] name = "zip" -version = "0.5.13" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" dependencies = [ + "aes", "byteorder", "bzip2", + "constant_time_eq", "crc32fast", + "crossbeam-utils", "flate2", - "thiserror", + "hmac", + "pbkdf2", + "sha1", "time", + "zstd", ] [[package]] -name = "zip-extensions" -version = "0.6.1" +name = "zstd" +version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c3c977bc3434ce2d4bcea8ad3c644672de0f2c402b72b9171ca80a8885d14" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zip", + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", ] diff --git a/benchmark/Cargo.toml b/benchmark/Cargo.toml index d3badcd8..4a6651c8 100644 --- a/benchmark/Cargo.toml +++ b/benchmark/Cargo.toml @@ -27,7 +27,7 @@ serde = {version = "1.0.142", features = ["derive"]} serde_json = "1.0" text-generation-client = { path = "../router/client" } thiserror = "1.0.38" -tokenizers = "0.13.2" +tokenizers = "0.13.3" tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } tui = {package = "ratatui", version = "0.20", default-features = false, features = ["crossterm"]} tracing = "0.1.37" diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs index 3a6316ab..dde429a5 100644 --- a/benchmark/src/generation.rs +++ b/benchmark/src/generation.rs @@ -75,7 +75,7 @@ async fn generate_runs( // Warmups on batch size for _ in 0..warmups { let (_, decode_batch) = - prefill(sequence.clone(), b, decode_length, &mut client).await?; + prefill(sequence.clone(), sequence_length, b, decode_length, &mut client).await?; let _ = decode(decode_batch, &mut client).await?; // Send warmup message run_sender.send(Ok(Message::Warmup)).await.unwrap_or(()); @@ -83,7 +83,7 @@ async fn generate_runs( for _ in 0..n_runs { let (prefill, decode_batch) = - prefill(sequence.clone(), b, decode_length, &mut client).await?; + prefill(sequence.clone(), sequence_length, b, decode_length, &mut client).await?; // Send prefill message run_sender .send(Ok(Message::Prefill(prefill))) @@ -110,6 +110,7 @@ async fn generate_runs( // Run a prefill step async fn prefill( sequence: String, + sequence_length: u32, batch_size: u32, decode_length: u32, client: &mut ShardedClient, @@ -119,6 +120,7 @@ async fn prefill( .map(|id| Request { id: id.into(), inputs: sequence.clone(), + truncate: sequence_length, parameters: Some(NextTokenChooserParameters { temperature: 1.0, top_k: 0, diff --git a/proto/generate.proto b/proto/generate.proto index 86393026..cc14cbf8 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -63,10 +63,12 @@ message Request { uint64 id = 1; /// The generation context string inputs = 2; + /// Context truncation + uint32 truncate = 3; /// Next Token Chooser Parameters - NextTokenChooserParameters parameters = 3; + NextTokenChooserParameters parameters = 4; /// Stopping Criteria Parameters - StoppingCriteriaParameters stopping_parameters = 4; + StoppingCriteriaParameters stopping_parameters = 5; } message Batch { diff --git a/router/Cargo.toml b/router/Cargo.toml index 77577eea..074a6053 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -18,21 +18,20 @@ axum = { version = "0.6.4", features = ["json"] } axum-tracing-opentelemetry = "0.9.0" text-generation-client = { path = "client" } clap = { version = "4.1.4", features = ["derive", "env"] } +flume = "0.10.14" futures = "0.3.26" metrics = "0.20.1" metrics-exporter-prometheus = { version = "0.11.0", features = [] } nohash-hasher = "0.2.0" opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } opentelemetry-otlp = "0.11.0" -parking_lot = "0.12.1" rand = "0.8.5" reqwest = { version = "0.11.14", features = [] } serde = "1.0.152" serde_json = "1.0.93" thiserror = "1.0.38" -tokenizers = "0.13.2" +tokenizers = "0.13.3" tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } -tokio-stream = "0.1.11" tower-http = { version = "0.3.5", features = ["cors"] } tracing = "0.1.37" tracing-opentelemetry = "0.18.0" diff --git a/router/src/infer.rs b/router/src/infer.rs index 2df9c5be..5a4375ae 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -2,17 +2,17 @@ use crate::validation::{Validation, ValidationError}; use crate::{Entry, Queue, Token}; use crate::{GenerateRequest, PrefillToken}; +use flume::r#async::RecvStream; use futures::future::try_join_all; +use futures::stream::StreamExt; use nohash_hasher::IntMap; use std::sync::Arc; use text_generation_client::{ Batch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, }; use thiserror::Error; -use tokio::sync::{mpsc, Notify, Semaphore, TryAcquireError}; +use tokio::sync::{Notify, Semaphore, TryAcquireError}; use tokio::time::Instant; -use tokio_stream::wrappers::UnboundedReceiverStream; -use tokio_stream::StreamExt; use tracing::{info_span, instrument, Instrument, Span}; /// Inference struct @@ -73,7 +73,7 @@ impl Infer { pub(crate) async fn generate_stream( &self, request: GenerateRequest, - ) -> Result>, InferError> { + ) -> Result>, InferError> { // Limit concurrent requests by acquiring a permit from the semaphore // This permit will live as long as Entry let permit = self @@ -87,10 +87,14 @@ impl Infer { })?; // Validate request - let valid_request = self.validation.validate(request).await?; + let valid_request = self.validation.validate(request).await.map_err(|err| { + metrics::increment_counter!("tgi_request_failure", "err" => "validation"); + tracing::error!("{err}"); + err + })?; // MPSC channel to communicate with the background batching task - let (response_tx, response_rx) = mpsc::unbounded_channel(); + let (response_tx, response_rx) = flume::unbounded(); // Append the request to the queue self.queue.append(Entry { @@ -108,7 +112,7 @@ impl Infer { self.shared.batching_task.notify_one(); // Return stream - Ok(UnboundedReceiverStream::new(response_rx)) + Ok(response_rx.into_stream()) } /// Add a new request to the queue and return a InferResponse diff --git a/router/src/main.rs b/router/src/main.rs index 81c6aeef..3ff72cde 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -37,7 +37,7 @@ struct Args { max_waiting_tokens: usize, #[clap(default_value = "3000", long, short, env)] port: u16, - #[clap(default_value = "/tmp/text-generation-0", long, env)] + #[clap(default_value = "/tmp/text-generation-server-0", long, env)] master_shard_uds_path: String, #[clap(default_value = "bigscience/bloom", long, env)] tokenizer_name: String, @@ -94,11 +94,11 @@ fn main() -> Result<(), std::io::Error> { if local_path.exists() && local_path.is_dir() && local_path.join("tokenizer.json").exists() { // Load local tokenizer - Tokenizer::from_file(local_path.join("tokenizer.json")).unwrap() + Tokenizer::from_file(local_path.join("tokenizer.json")).ok() } else { // Download and instantiate tokenizer // We need to download it outside of the Tokio runtime - Tokenizer::from_pretrained(tokenizer_name.clone(), None).unwrap() + Tokenizer::from_pretrained(tokenizer_name.clone(), None).ok() }; // Launch Tokio runtime @@ -109,6 +109,13 @@ fn main() -> Result<(), std::io::Error> { .block_on(async { init_logging(otlp_endpoint, json_output); + if tokenizer.is_none() { + tracing::warn!( + "Could not find a fast tokenizer implementation for {tokenizer_name}" + ); + tracing::warn!("Rust input length validation and truncation is disabled"); + } + // Get pipeline tag let model_info = reqwest::get(format!( "https://huggingface.co/api/models/{tokenizer_name}" diff --git a/router/src/queue.rs b/router/src/queue.rs index 2899ccd4..11eb7f59 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -4,8 +4,7 @@ use crate::validation::ValidGenerateRequest; use nohash_hasher::{BuildNoHashHasher, IntMap}; use std::cmp::min; use text_generation_client::{Batch, Request}; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; -use tokio::sync::{mpsc, oneshot, OwnedSemaphorePermit}; +use tokio::sync::{oneshot, OwnedSemaphorePermit}; use tokio::time::Instant; use tracing::{info_span, instrument, Span}; @@ -15,7 +14,7 @@ pub(crate) struct Entry { /// Request pub request: ValidGenerateRequest, /// Response sender to communicate between the Infer struct and the batching_task - pub response_tx: UnboundedSender>, + pub response_tx: flume::Sender>, /// Span that will live as long as entry pub span: Span, /// Temporary span used as a guard when logging inference, wait times... @@ -32,13 +31,13 @@ pub(crate) struct Entry { #[derive(Debug, Clone)] pub(crate) struct Queue { /// Channel to communicate with the background queue task - queue_sender: UnboundedSender, + queue_sender: flume::Sender, } impl Queue { pub(crate) fn new() -> Self { // Create channel - let (queue_sender, queue_receiver) = mpsc::unbounded_channel(); + let (queue_sender, queue_receiver) = flume::unbounded(); // Launch background queue task tokio::spawn(queue_task(queue_receiver)); @@ -82,10 +81,10 @@ impl Queue { } // Background task responsible of the queue state -async fn queue_task(mut receiver: UnboundedReceiver) { +async fn queue_task(receiver: flume::Receiver) { let mut state = State::new(); - while let Some(cmd) = receiver.recv().await { + while let Ok(cmd) = receiver.recv_async().await { match cmd { QueueCommand::Append(entry, span) => span.in_scope(|| state.append(entry)), QueueCommand::NextBatch { @@ -174,6 +173,7 @@ impl State { batch_requests.push(Request { id, inputs: entry.request.inputs.clone(), + truncate: entry.request.truncate, parameters: Some(entry.request.parameters.clone()), stopping_parameters: Some(entry.request.stopping_parameters.clone()), }); @@ -215,17 +215,18 @@ mod tests { use super::*; use std::sync::Arc; use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParameters}; - use tokio::sync::{mpsc, Semaphore}; + use tokio::sync::Semaphore; use tracing::info_span; fn default_entry() -> Entry { let semaphore = Arc::new(Semaphore::new(1)); - let (response_tx, _) = mpsc::unbounded_channel(); + let (response_tx, _) = flume::unbounded(); let permit = semaphore.try_acquire_owned().unwrap(); Entry { request: ValidGenerateRequest { inputs: "".to_string(), + truncate: 0, parameters: NextTokenChooserParameters { temperature: 0.0, top_k: 0, diff --git a/router/src/server.rs b/router/src/server.rs index e1b402a6..851837d5 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -13,6 +13,7 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{http, Json, Router}; use axum_tracing_opentelemetry::opentelemetry_tracing_layer; +use futures::stream::StreamExt; use futures::Stream; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; use std::convert::Infallible; @@ -21,7 +22,6 @@ use text_generation_client::ShardedClient; use tokenizers::Tokenizer; use tokio::signal; use tokio::time::Instant; -use tokio_stream::StreamExt; use tower_http::cors::{AllowOrigin, CorsLayer}; use tracing::{info_span, instrument, Instrument}; use utoipa::OpenApi; @@ -87,21 +87,21 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json) -> String { prom_handle.render() @@ -468,7 +468,7 @@ pub async fn run( max_batch_size: usize, max_waiting_tokens: usize, client: ShardedClient, - tokenizer: Tokenizer, + tokenizer: Option, validation_workers: usize, addr: SocketAddr, allow_origin: Option, @@ -476,36 +476,36 @@ pub async fn run( // OpenAPI documentation #[derive(OpenApi)] #[openapi( - paths( - generate, - generate_stream, - metrics, - ), - components( - schemas( - GenerateRequest, - GenerateParameters, - PrefillToken, - Token, - GenerateResponse, - BestOfSequence, - Details, - FinishReason, - StreamResponse, - StreamDetails, - ErrorResponse, - ) - ), - tags( - (name = "Text Generation Inference", description = "Hugging Face Text Generation Inference API") - ), - info( - title = "Text Generation Inference", - license( - name = "Apache 2.0", - url = "https://www.apache.org/licenses/LICENSE-2.0" - ) - ) + paths( + generate, + generate_stream, + metrics, + ), + components( + schemas( + GenerateRequest, + GenerateParameters, + PrefillToken, + Token, + GenerateResponse, + BestOfSequence, + Details, + FinishReason, + StreamResponse, + StreamDetails, + ErrorResponse, + ) + ), + tags( + (name = "Text Generation Inference", description = "Hugging Face Text Generation Inference API") + ), + info( + title = "Text Generation Inference", + license( + name = "Apache 2.0", + url = "https://www.apache.org/licenses/LICENSE-2.0" + ) + ) )] struct ApiDoc; diff --git a/router/src/validation.rs b/router/src/validation.rs index ec67cefd..d81a27e7 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,50 +1,129 @@ use crate::validation::ValidationError::{BestOfSampling, BestOfSeed, EmptyInput}; /// Payload validation logic use crate::{GenerateParameters, GenerateRequest}; -use rand::rngs::ThreadRng; -use rand::Rng; +use rand::{thread_rng, Rng}; use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParameters}; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokenizers::TruncationDirection; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::oneshot; use tracing::{instrument, Span}; /// Validation #[derive(Debug, Clone)] pub struct Validation { - /// maximum value for the best_of parameter - #[allow(dead_code)] + /// Validation parameters max_best_of: usize, - /// Channel to communicate with the background validation task - sender: mpsc::UnboundedSender, + max_stop_sequences: usize, + max_input_length: usize, + max_total_tokens: usize, + /// Channel to communicate with the background tokenization task + sender: Option>, } impl Validation { pub(crate) fn new( workers: usize, - tokenizer: Tokenizer, + tokenizer: Option, max_best_of: usize, max_stop_sequences: usize, max_input_length: usize, max_total_tokens: usize, ) -> Self { - // Create channel - let (validation_sender, validation_receiver) = mpsc::unbounded_channel(); + if max_input_length >= max_total_tokens { + panic!("`max_input_length` must be < `max_total_tokens`"); + } - // Launch background validation task - tokio::spawn(validation_task( - workers, - tokenizer, + // If we have a fast tokenizer + let sender = if let Some(tokenizer) = tokenizer { + // Create channel + let (validation_sender, validation_receiver) = flume::unbounded(); + + // Create workers + for _ in 0..workers { + let tokenizer_clone = tokenizer.clone(); + let receiver_clone = validation_receiver.clone(); + + // Spawn worker + tokio::task::spawn_blocking(move || { + tokenizer_worker(tokenizer_clone, receiver_clone) + }); + } + Some(validation_sender) + } else { + None + }; + + Self { + max_best_of, + sender, max_stop_sequences, max_input_length, max_total_tokens, - validation_receiver, - )); + } + } - Self { - max_best_of, - sender: validation_sender, + #[instrument(skip_all)] + async fn validate_input( + &self, + inputs: String, + truncate: Option, + max_new_tokens: u32, + ) -> Result { + // If we have a fast tokenizer + if let Some(sender) = &self.sender { + // Create response channel + let (response_sender, response_receiver) = oneshot::channel(); + // Send request to the background validation task + // Unwrap is safe here + sender + .send(((inputs, truncate), response_sender, Span::current())) + .unwrap(); + + // Await on response channel + // Unwrap is safe here + let (inputs, input_length) = response_receiver.await.unwrap()?; + + // Get total tokens + let total_tokens = input_length + max_new_tokens as usize; + + // Validate MaxTotalTokens + if total_tokens > self.max_total_tokens { + return Err(ValidationError::MaxTotalTokens( + self.max_total_tokens, + input_length, + max_new_tokens, + )); + } + + // Validate InputLength + if input_length > self.max_input_length { + return Err(ValidationError::InputLength( + self.max_input_length, + input_length, + )); + } + + metrics::histogram!("tgi_request_input_length", input_length as f64); + Ok(inputs) + } + // Return inputs without validation + else { + // In this case, we don't know the real length in tokens of the inputs + // However, the inputs will be truncated by the python servers + // We make sure that truncate + max_new_tokens <= self.max_total_tokens + + // Validate MaxNewTokens + if (truncate.unwrap_or(self.max_input_length) as u32 + max_new_tokens) + > self.max_total_tokens as u32 + { + return Err(ValidationError::MaxNewTokens( + self.max_total_tokens - self.max_input_length, + max_new_tokens, + )); + } + + Ok(inputs) } } @@ -54,16 +133,139 @@ impl Validation { &self, request: GenerateRequest, ) -> Result { - // Create response channel - let (sender, receiver) = oneshot::channel(); - // Send request to the background validation task - // Unwrap is safe here - self.sender - .send((request, sender, Span::current())) - .unwrap(); - // Await on response channel - // Unwrap is safe here - receiver.await.unwrap() + let GenerateParameters { + best_of, + temperature, + repetition_penalty, + top_k, + top_p, + typical_p, + do_sample, + max_new_tokens, + stop: stop_sequences, + truncate, + seed, + watermark, + .. + } = request.parameters; + + // sampling must be true when best_of > 1 + let best_of = best_of.unwrap_or(1); + let sampling = do_sample + || temperature.is_some() + || top_k.is_some() + || top_p.is_some() + || typical_p.is_some(); + + if best_of > 1 && !sampling { + return Err(BestOfSampling); + } + + let temperature = temperature.unwrap_or(1.0); + if temperature <= 0.0 { + return Err(ValidationError::Temperature); + } + + let repetition_penalty = repetition_penalty.unwrap_or(1.0); + if repetition_penalty <= 0.0 { + return Err(ValidationError::RepetitionPenalty); + } + + // Different because the proto default value is not a valid value + // for the user + let top_p = top_p + .map(|value| { + if value <= 0.0 || value >= 1.0 { + return Err(ValidationError::TopP); + } + Ok(value) + }) + .unwrap_or(Ok(1.0))?; + + let typical_p = typical_p + .map(|value| { + if value <= 0.0 || value >= 1.0 { + return Err(ValidationError::TypicalP); + } + Ok(value) + }) + .unwrap_or(Ok(1.0))?; + + let top_k: u32 = top_k + .map(|value| { + if value <= 0 { + return Err(ValidationError::TopK); + } + Ok(value as u32) + }) + .unwrap_or(Ok(0))?; + + if max_new_tokens == 0 { + return Err(ValidationError::NegativeMaxNewTokens); + } + + if stop_sequences.len() > self.max_stop_sequences { + return Err(ValidationError::StopSequence( + self.max_stop_sequences, + stop_sequences.len(), + )); + } + + // If seed is None, assign a random one + let seed = match seed { + None => thread_rng().gen(), + Some(seed) => { + if best_of > 1 { + return Err(BestOfSeed); + } + seed + } + }; + + // Check if inputs is empty + if request.inputs.is_empty() { + return Err(EmptyInput); + } + + // Check if truncate is strictly positive and less than max_input_length + let truncate = truncate + .map(|value| { + if value == 0 || value > self.max_input_length { + return Err(ValidationError::Truncate(self.max_input_length, value)); + } + Ok(Some(value)) + }) + .unwrap_or(Ok(None))?; + + // Validate inputs + let inputs = self + .validate_input(request.inputs, truncate, max_new_tokens) + .await?; + + let parameters = NextTokenChooserParameters { + temperature, + repetition_penalty, + top_k, + top_p, + typical_p, + do_sample, + seed, + watermark, + }; + let stopping_parameters = StoppingCriteriaParameters { + max_new_tokens, + stop_sequences, + ignore_eos_token: false, + }; + + metrics::histogram!("tgi_request_max_new_tokens", max_new_tokens as f64); + + Ok(ValidGenerateRequest { + inputs, + truncate: truncate.unwrap_or(self.max_input_length) as u32, + parameters, + stopping_parameters, + }) } /// Validate the best_of parameter @@ -81,262 +283,57 @@ impl Validation { } } -/// Validation task -/// Load balance the validation requests between multiple validation workers -async fn validation_task( - workers: usize, - tokenizer: Tokenizer, - max_stop_sequences: usize, - max_input_length: usize, - max_total_tokens: usize, - mut receiver: mpsc::UnboundedReceiver, -) { - let mut workers_senders = Vec::with_capacity(workers); - - // Create workers - for _ in 0..workers { - let tokenizer_clone: Tokenizer = tokenizer.clone().into(); - // Create channel to communicate with worker - let (worker_sender, worker_receiver) = mpsc::channel(workers); - workers_senders.push(worker_sender); - - // Spawn worker - tokio::task::spawn_blocking(move || { - validation_worker( - tokenizer_clone, - max_stop_sequences, - max_input_length, - max_total_tokens, - worker_receiver, - ) - }); - } - - loop { - // Load balance requests between workers - for sender in workers_senders.iter() { - if let Some(validation_request) = receiver.recv().await { - sender.send(validation_request).await.unwrap(); - } else { - return; - } - } - } -} - -/// Check the parameters inside the payload and get the number of tokens inside the input using -/// the tokenizer -fn validation_worker( - tokenizer: Tokenizer, - max_stop_sequences: usize, - max_input_length: usize, - max_total_tokens: usize, - mut receiver: mpsc::Receiver, -) { - // Seed rng - let mut rng = rand::thread_rng(); - +/// Start tokenization workers +fn tokenizer_worker(tokenizer: Tokenizer, receiver: flume::Receiver) { // Loop over requests - while let Some((request, response_tx, parent_span)) = receiver.blocking_recv() { + while let Ok(((inputs, truncate), response_tx, parent_span)) = receiver.recv() { parent_span.in_scope(|| { response_tx - .send( - validate( - request, - &tokenizer, - max_stop_sequences, - max_input_length, - max_total_tokens, - &mut rng, - ) - .map_err(|err| { - metrics::increment_counter!("tgi_request_failure", "err" => "validation"); - tracing::error!("{err}"); - err - }), - ) + .send(prepare_input(inputs, truncate, &tokenizer)) .unwrap_or(()) }) } } -fn validate( - request: GenerateRequest, +/// Get input length and optionally truncate it +fn prepare_input( + inputs: String, + truncate: Option, tokenizer: &Tokenizer, - max_stop_sequences: usize, - max_input_length: usize, - max_total_tokens: usize, - rng: &mut ThreadRng, -) -> Result { - let GenerateParameters { - best_of, - temperature, - repetition_penalty, - top_k, - top_p, - typical_p, - do_sample, - max_new_tokens, - stop: stop_sequences, - truncate, - seed, - watermark, - .. - } = request.parameters; - - // sampling must be true when best_of > 1 - let best_of = best_of.unwrap_or(1); - let sampling = do_sample - || temperature.is_some() - || top_k.is_some() - || top_p.is_some() - || typical_p.is_some(); - - if best_of > 1 && !sampling { - return Err(BestOfSampling); - } - - let temperature = temperature.unwrap_or(1.0); - if temperature <= 0.0 { - return Err(ValidationError::Temperature); - } - - let repetition_penalty = repetition_penalty.unwrap_or(1.0); - if repetition_penalty <= 0.0 { - return Err(ValidationError::RepetitionPenalty); - } - - // Different because the proto default value is not a valid value - // for the user - let top_p = top_p - .map(|value| { - if value <= 0.0 || value >= 1.0 { - return Err(ValidationError::TopP); - } - Ok(value) - }) - .unwrap_or(Ok(1.0))?; - - let typical_p = typical_p - .map(|value| { - if value <= 0.0 || value >= 1.0 { - return Err(ValidationError::TypicalP); - } - Ok(value) - }) - .unwrap_or(Ok(1.0))?; - - let top_k: u32 = top_k - .map(|value| { - if value <= 0 { - return Err(ValidationError::TopK); - } - Ok(value as u32) - }) - .unwrap_or(Ok(0))?; - - if max_new_tokens == 0 { - return Err(ValidationError::MaxNewTokens); - } - - if stop_sequences.len() > max_stop_sequences { - return Err(ValidationError::StopSequence( - max_stop_sequences, - stop_sequences.len(), - )); - } - - // If seed is None, assign a random one - let seed = match seed { - None => rng.gen(), - Some(seed) => { - if best_of > 1 { - return Err(BestOfSeed); - } - seed - } - }; - - // Check if inputs is empty - if request.inputs.is_empty() { - return Err(EmptyInput); - } - - // Check if truncate is strictly positive and less than max_input_length - let truncate = truncate - .map(|value| { - if value == 0 || value > max_input_length { - return Err(ValidationError::Truncate(max_input_length, value)); - } - Ok(Some(value)) - }) - .unwrap_or(Ok(None))?; - +) -> Result<(String, usize), ValidationError> { // Get the number of tokens in the input let mut encoding = tokenizer - .encode(request.inputs.clone(), true) + .encode(inputs.clone(), true) .map_err(|err| ValidationError::Tokenizer(err.to_string()))?; - let (inputs, input_length) = if let Some(truncate) = truncate { - // truncate encoding and decode new inputs - encoding.truncate(truncate, 0, TruncationDirection::Left); - let inputs = tokenizer - .decode(Vec::from(encoding.get_ids()), false) - .map_err(|err| ValidationError::Tokenizer(err.to_string()))?; - (inputs, encoding.len()) - } else { - (request.inputs, encoding.len()) - }; - - if input_length > max_input_length { - return Err(ValidationError::InputLength(max_input_length, input_length)); - } - - let total_tokens = input_length + max_new_tokens as usize; - if total_tokens > max_total_tokens { - return Err(ValidationError::MaxTotalTokens( - max_total_tokens, - input_length, - max_new_tokens, - )); - } - - // Return ValidGenerateRequest - let parameters = NextTokenChooserParameters { - temperature, - repetition_penalty, - top_k, - top_p, - typical_p, - do_sample, - seed, - watermark, - }; - let stopping_parameters = StoppingCriteriaParameters { - max_new_tokens, - stop_sequences, - ignore_eos_token: false, + // Optionally truncate + let (inputs, input_length) = match truncate { + // Truncate is some and > encoding length + Some(truncate) if truncate > encoding.len() => { + // truncate encoding and decode new inputs + encoding.truncate(truncate, 0, TruncationDirection::Left); + let inputs = tokenizer + .decode(Vec::from(encoding.get_ids()), false) + .map_err(|err| ValidationError::Tokenizer(err.to_string()))?; + (inputs, encoding.len()) + } + // Nothing to do + _ => (inputs, encoding.len()), }; - metrics::histogram!("tgi_request_input_length", input_length as f64); - metrics::histogram!("tgi_request_max_new_tokens", max_new_tokens as f64); - - Ok(ValidGenerateRequest { - inputs, - parameters, - stopping_parameters, - }) + Ok((inputs, input_length)) } -type ValidationRequest = ( - GenerateRequest, - oneshot::Sender>, +type TokenizerRequest = ( + (String, Option), + oneshot::Sender>, Span, ); #[derive(Debug)] pub(crate) struct ValidGenerateRequest { pub inputs: String, + pub truncate: u32, pub parameters: NextTokenChooserParameters, pub stopping_parameters: StoppingCriteriaParameters, } @@ -366,7 +363,9 @@ pub enum ValidationError { #[error("`typical_p` must be > 0.0 and < 1.0")] TypicalP, #[error("`max_new_tokens` must be strictly positive")] - MaxNewTokens, + NegativeMaxNewTokens, + #[error("`max_new_tokens` must be <= {0}. Given: {1}")] + MaxNewTokens(usize, u32), #[error("`inputs` tokens + `max_new_tokens` must be <= {0}. Given: {1} `inputs` tokens and {2} `max_new_tokens`")] MaxTotalTokens(usize, usize, u32), #[error("`inputs` must have less than {0} tokens. Given: {1}")] diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 2b8ef5f8..a9831cd7 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -24,6 +24,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", + truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 76617b62..db68fc9c 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -25,6 +25,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", + truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index 753ff5fc..8cf66d47 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -15,6 +15,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="def", + truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) @@ -30,6 +31,7 @@ def default_fim_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="defworld", + truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 2d86c44b..baf44579 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -28,6 +28,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", + truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, ) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 0d83abe2..ce3895ca 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -68,7 +68,7 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) config = AutoConfig.from_pretrained( diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index c2ad0587..cb7bbfd8 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -66,6 +66,7 @@ def from_pb( stopping_criterias = [] # Parse batch + max_truncation = 0 padding_right_offset = 0 for r in pb.requests: inputs.append(r.inputs) @@ -74,6 +75,7 @@ def from_pb( r.stopping_parameters, tokenizer ) stopping_criterias.append(stopping_criteria) + max_truncation = max(max_truncation, r.truncate) padding_right_offset = max( padding_right_offset, stopping_criteria.max_new_tokens ) @@ -83,6 +85,8 @@ def from_pb( return_tensors="pt", padding=True, return_token_type_ids=False, + truncation=True, + max_length=max_truncation, ).to(device) input_lengths = tokenized_inputs["attention_mask"].sum(1) diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index f3517c47..4ff17619 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -38,7 +38,7 @@ class FastLayerNorm(nn.LayerNorm): def forward(self, hidden_states, residual=None): - if hidden_states.shape[-1] > 6144: + if hidden_states.shape[-1] > 8192: if residual is not None: hidden_states += residual residual = hidden_states @@ -624,13 +624,16 @@ def forward( class FlashGPTNeoXForCausalLM(FlashGPTNeoXPreTrainedModel): - def __init__(self, config): + def __init__(self, config, process_group=None): super().__init__(config) - if config.tp_parallel: - process_group = torch.distributed.distributed_c10d._get_default_group() + self.process_group = process_group + if self.process_group is not None: + self.world_size = self.process_group.size() + self.rank = self.process_group.rank() else: - process_group = None + self.world_size = 1 + self.rank = 0 self.gpt_neox = FlashGPTNeoXModel(config, process_group) @@ -668,4 +671,13 @@ def forward( hidden_states, present = self.gpt_neox( input_ids, position_ids, cu_seqlens, max_s, past_key_values ) - return self.embed_out(hidden_states), present + logits = self.embed_out(hidden_states) + + if self.gpt_neox.tp_embeddings: + # Logits are sharded, so we need to gather them + world_logits = [torch.empty_like(logits) for _ in range(self.world_size)] + torch.distributed.all_gather(world_logits, logits, group=self.process_group) + world_logits = torch.cat(world_logits, dim=1) + + return world_logits, present + return logits, present diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 799e7054..29c4a5c8 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -11,7 +11,7 @@ class FastLayerNorm(nn.LayerNorm): def forward(self, hidden_states, residual=None): - if hidden_states.shape[-1] > 6144: + if hidden_states.shape[-1] > 8192: if residual is not None: hidden_states += residual residual = hidden_states diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index e1a10cbf..bc1ac063 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -78,7 +78,9 @@ def from_pb( # Parse batch for r in pb.requests: - tokenized_input = tokenizer(r.inputs)["input_ids"] + tokenized_input = tokenizer( + r.inputs, truncation=True, max_length=r.truncate + )["input_ids"] input_length = len(tokenized_input) max_seqlen = max(max_seqlen, input_length) input_lengths.append(input_length) @@ -208,7 +210,7 @@ def __init__( raise NotImplementedError("FlashCausalLM does not support quantization") tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) self.model = ( model_cls.from_pretrained( diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index e415a725..b93d9f70 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -45,18 +45,19 @@ def __init__( raise NotImplementedError("FlashNeoX does not support quantization") tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) config = AutoConfig.from_pretrained( - model_id, revision=revision, tp_parallel=True + model_id, + revision=revision, ) torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") with init_empty_weights(): - model = FlashGPTNeoXForCausalLM(config) + model = FlashGPTNeoXForCausalLM(config, self.process_group) torch.distributed.barrier(group=self.process_group) self.load_weights( @@ -147,32 +148,3 @@ def load_weights( module._parameters[param_name] = tensor else: module._buffers[param_name] = tensor - - def forward( - self, - input_ids: torch.Tensor, - position_ids: torch.Tensor, - cu_seqlens: torch.Tensor, - max_s: int, - past_key_values: Optional = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - if self.model.gpt_neox.tp_embeddings: - logits, present = self.model.forward( - input_ids=input_ids, - position_ids=position_ids, - cu_seqlens=cu_seqlens, - max_s=max_s, - past_key_values=past_key_values, - ) - - # Logits are sharded, so we need to gather them - world_logits = [torch.empty_like(logits) for _ in range(self.world_size)] - torch.distributed.all_gather(world_logits, logits, group=self.process_group) - world_logits = torch.cat(world_logits, dim=1) - - return world_logits, present - # While the model itself is sharded, the embeddings might not as they might not be dividable by num-shard - else: - return super(FlashNeoXSharded, self).forward( - input_ids, position_ids, cu_seqlens, max_s, past_key_values - ) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index f0207d55..2f680995 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -33,7 +33,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False raise NotImplementedError("FlashSantacoder does not support quantization") tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) config = AutoConfig.from_pretrained( @@ -56,6 +56,8 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False self.load_weights( model, filenames, + device, + dtype, ) self.model = model.eval().to(device).to(dtype) @@ -68,10 +70,14 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False def load_weights( model: FlashSantacoderForCausalLM, filenames: List[Path], + device: torch.device, + dtype: torch.dtype, ): for filename in filenames: state_dict = torch.load(filename, map_location="cpu") for key, value in state_dict.items(): + value = value.to(device).to(dtype) + layer_name = ".".join(key.split(".")[:4]) # Fused qkv @@ -141,6 +147,8 @@ def load_weights( else: module._buffers[param_name] = value + del value + torch.cuda.empty_cache() model.post_load_weights() diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index a90a299e..3c89b4a7 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -96,7 +96,7 @@ def from_pb( input_lengths = [] # Parse batch - max_sequence_length = 0 + max_truncation = 0 padding_right_offset = 0 for r in pb.requests: # Add escape_custom_split_sequence to the CausalLMBatch logic @@ -107,7 +107,7 @@ def from_pb( r.stopping_parameters, tokenizer ) stopping_criterias.append(stopping_criteria) - max_sequence_length = max(max_sequence_length, r.input_length) + max_truncation = max(max_truncation, r.truncate) padding_right_offset = max( padding_right_offset, stopping_criteria.max_new_tokens ) @@ -118,14 +118,20 @@ def from_pb( return_tensors="pt", padding=True, return_token_type_ids=False, + truncation=True, + max_length=max_truncation, ).to(device) + + input_lengths = tokenized_inputs["attention_mask"].sum(1) + max_input_length = input_lengths.max() + input_ids = tokenized_inputs["input_ids"] # Allocate maximum attention_mask attention_mask = input_ids.new_zeros( - (pb.size, max_sequence_length + padding_right_offset) + (pb.size, max_input_length + padding_right_offset) ) # Copy tokenizer attention_mask into fully allocated attention_mask - attention_mask[:, :max_sequence_length] = tokenized_inputs["attention_mask"] + attention_mask[:, :max_input_length] = tokenized_inputs["attention_mask"] position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) @@ -143,7 +149,7 @@ def from_pb( next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=pb.size, - max_sequence_length=max_sequence_length, + max_input_length=max_input_length, padding_right_offset=padding_right_offset, ) @@ -188,7 +194,7 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) config = AutoConfig.from_pretrained( diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 8fabefe3..b81976da 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -44,7 +44,7 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) tokenizer.pad_token = tokenizer.eos_token diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index fe15cde0..58361a8d 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -26,7 +26,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) tokenizer.add_special_tokens( { diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 0fe5c03f..7cf97123 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -73,6 +73,7 @@ def from_pb( decoder_input_lengths = [] # Parse batch + max_truncation = 0 padding_right_offset = 0 for r in pb.requests: inputs.append(r.inputs) @@ -84,6 +85,7 @@ def from_pb( r.stopping_parameters, tokenizer ) stopping_criterias.append(stopping_criteria) + max_truncation = max(max_truncation, r.truncate) padding_right_offset = max( padding_right_offset, stopping_criteria.max_new_tokens ) @@ -94,6 +96,8 @@ def from_pb( return_tensors="pt", padding=True, return_token_type_ids=False, + truncation=True, + max_length=max_truncation, ).to(device) input_lengths = tokenized_inputs["attention_mask"].sum(1) diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index cb4f7f22..300b376e 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -44,7 +44,7 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) config = AutoConfig.from_pretrained( From 299217c95ca314e8dbbeca26ce8cdceb440ec53b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 11 Apr 2023 16:38:22 +0200 Subject: [PATCH 150/793] feat(server): add flash attention llama (#144) --- README.md | 10 +- launcher/tests/mt0_base.json | 2 +- server/poetry.lock | 104 ++- server/pyproject.toml | 2 + server/tests/models/test_seq2seq_lm.py | 2 +- .../text_generation_server/models/__init__.py | 27 +- .../models/causal_lm.py | 30 +- .../custom_modeling/flash_llama_modeling.py | 619 ++++++++++++++++++ .../models/flash_causal_lm.py | 30 +- .../models/flash_llama.py | 303 +++++++++ .../models/galactica.py | 8 +- server/text_generation_server/models/model.py | 48 +- .../models/seq2seq_lm.py | 30 +- 13 files changed, 1175 insertions(+), 40 deletions(-) create mode 100644 server/text_generation_server/models/custom_modeling/flash_llama_modeling.py create mode 100644 server/text_generation_server/models/flash_llama.py diff --git a/README.md b/README.md index ad938bdd..bc77fd4c 100644 --- a/README.md +++ b/README.md @@ -51,16 +51,14 @@ to power LLMs api-inference widgets. - Log probabilities - Production ready (distributed tracing with Open Telemetry, Prometheus metrics) -## Officially supported architectures +## Optimized architectures - [BLOOM](https://huggingface.co/bigscience/bloom) -- [BLOOMZ](https://huggingface.co/bigscience/bloomz) -- [MT0-XXL](https://huggingface.co/bigscience/mt0-xxl) - [Galactica](https://huggingface.co/facebook/galactica-120b) - [SantaCoder](https://huggingface.co/bigcode/santacoder) -- [GPT-Neox 20B](https://huggingface.co/EleutherAI/gpt-neox-20b) -- [FLAN-T5-XXL](https://huggingface.co/google/flan-t5-xxl) -- [FLAN-UL2](https://huggingface.co/google/flan-ul2) +- [GPT-Neox](https://huggingface.co/EleutherAI/gpt-neox-20b) +- [FLAN-T5](https://huggingface.co/google/flan-t5-xxl) +- [Llama](https://github.com/facebookresearch/llama) Other architectures are supported on a best effort basis using: diff --git a/launcher/tests/mt0_base.json b/launcher/tests/mt0_base.json index 22c9499f..f5be63f9 100644 --- a/launcher/tests/mt0_base.json +++ b/launcher/tests/mt0_base.json @@ -14,7 +14,7 @@ "tokens": [ { "id": 259, - "text": " ", + "text": "", "logprob": -1.3656927, "special": false }, diff --git a/server/poetry.lock b/server/poetry.lock index 89ad31e9..49b9d39d 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -517,6 +517,14 @@ tensorflow = ["tensorflow"] testing = ["h5py", "huggingface-hub", "numpy", "pytest", "pytest-benchmark", "setuptools-rust"] torch = ["torch"] +[[package]] +name = "sentencepiece" +version = "0.1.97" +description = "SentencePiece python wrapper" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "setuptools" version = "67.4.0" @@ -530,6 +538,19 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "tokenizers" +version = "0.13.3" +description = "Fast and Customizable Tokenizers" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] +docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] + [[package]] name = "tomli" version = "2.0.1" @@ -630,7 +651,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "521dc9f3c283dc56f7d2e2f96759919ff27ab49ffd3ae7cd26317b209e7fa98d" +content-hash = "1c57379c7b9349d2a860b50b3ab125737a0f6f94f4303d7cb55248cb86ff8b8e" [metadata.files] accelerate = [ @@ -1116,10 +1137,91 @@ safetensors = [ {file = "safetensors-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:ba3dc236a2344b7feadc9868307f42ba5e4804c9d68a80a35aac831349b31f6f"}, {file = "safetensors-0.2.8.tar.gz", hash = "sha256:2720b20a6a38c799dca79bd76caeeac2f7df585a9d4f7d59fa7e28eff9ccb27f"}, ] +sentencepiece = [ + {file = "sentencepiece-0.1.97-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6f249c8f1852893be86eae66b19d522c5fb30bbad4fe2d1b07f06fdc86e1907e"}, + {file = "sentencepiece-0.1.97-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09e1bc53178de70c557a9ba4fece07364b4416ce3d36570726b3372b68aea135"}, + {file = "sentencepiece-0.1.97-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:667193c57fb48b238be7e3d7636cfc8da56cb5bac5559d8f0b647334e1175be8"}, + {file = "sentencepiece-0.1.97-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2780531985af79c6163f63d4f200fec8a28b70b6768d2c19f70d01568a4524e8"}, + {file = "sentencepiece-0.1.97-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:205050670c53ef9015e2a98cce3934bfbcf0aafaa14caa0c618dd5667bc217ee"}, + {file = "sentencepiece-0.1.97-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28b183dadef8e8b6b4645c1c20692d7be0a13ecc3ec1a07b3885c8905516675f"}, + {file = "sentencepiece-0.1.97-cp310-cp310-win32.whl", hash = "sha256:ee3c9dbd558d8d85bb1617087b86df6ea2b856a528669630ce6cedeb4353b823"}, + {file = "sentencepiece-0.1.97-cp310-cp310-win_amd64.whl", hash = "sha256:f7dc55379e2f7dee86537180283db2e5f8418c6825fdd2fe436c724eb5604c05"}, + {file = "sentencepiece-0.1.97-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ba1b4154f9144c5a7528b00aff5cffaa1a896a1c6ca53ca78b6e74cd2dae5244"}, + {file = "sentencepiece-0.1.97-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac3d90aee5581e55d029d124ac11b6ae2fbae0817863b664b2f2302e966ababb"}, + {file = "sentencepiece-0.1.97-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c27400f1ac46518a01c87cb7703650e4e48728649feb115d2e3f1102a946a42"}, + {file = "sentencepiece-0.1.97-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6e12a166eba75994ca749aadc4a5056b91b31405f805d6de6e8914cc9741c60"}, + {file = "sentencepiece-0.1.97-cp36-cp36m-win32.whl", hash = "sha256:ed85dff5c0a9b3dd1a414c7e1119f2a19e863fc3f81da525bf7f885ebc883de0"}, + {file = "sentencepiece-0.1.97-cp36-cp36m-win_amd64.whl", hash = "sha256:91a19ab6f40ffbae6d6127119953d2c6a85e93d734953dbc8629fde0d21ace66"}, + {file = "sentencepiece-0.1.97-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bae580e4a35a9314ff49561ac7c06574fe6afc71b821ed6bb00534e571458156"}, + {file = "sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ad7262e7530c683b186672b5dd0082f82719a50a500a8cfbc4bbd7cde5bff8c"}, + {file = "sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:620cee35279720016735a7c7103cddbd9b84fe5e2f098bd5e673834d69fee2b8"}, + {file = "sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93b921b59914c0ec6697e8c6d5e6b44d99d1298fb1a0af56980a79ade0540c19"}, + {file = "sentencepiece-0.1.97-cp37-cp37m-win32.whl", hash = "sha256:9b9a4c44a31d5f47616e9568dcf31e029b0bfa776e0a252c0b59247881598b09"}, + {file = "sentencepiece-0.1.97-cp37-cp37m-win_amd64.whl", hash = "sha256:f31533cdacced56219e239d3459a003ece35116920dd64b2309d4ad047b77644"}, + {file = "sentencepiece-0.1.97-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7d643c01d1cad13b9206a276bbe5bc1a468e3d7cf6a26bde7783f945277f859d"}, + {file = "sentencepiece-0.1.97-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:542f1985b1ee279a92bef7740ec0781452372028ce01e15aa88df3228b197ba3"}, + {file = "sentencepiece-0.1.97-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93701da21fea906dd244bf88cdbe640385a89c45d3c1812b76dbadf8782cdbcd"}, + {file = "sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a51514047b964047b7fadb480d88a5e0f72c02f6ca1ba96258fbbc6e79274a94"}, + {file = "sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3ae2e9b7a5b6f2aa64ec9240b0c185dabe597d0e787dc4344acfbaef1ffe0b2"}, + {file = "sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:923ee4af16dbae1f2ab358ed09f8a0eb89e40a8198a8b343bf54181482342721"}, + {file = "sentencepiece-0.1.97-cp38-cp38-win32.whl", hash = "sha256:fa6f2b88850b5fae3a05053658824cf9f147c8e3c3b40eb64539a976c83d8a24"}, + {file = "sentencepiece-0.1.97-cp38-cp38-win_amd64.whl", hash = "sha256:5137ff0d0b1cc574751d178650ef800ff8d90bf21eb9f71e9567d4a0548940a5"}, + {file = "sentencepiece-0.1.97-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f92876271a10494671431ad955bff2d6f8ea59baaf957f5ae5946aff56dfcb90"}, + {file = "sentencepiece-0.1.97-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:35c227b6d55e473033db7e0ecc51b1e99e6ed7607cc08602fb5768132543c81d"}, + {file = "sentencepiece-0.1.97-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1706a8a8188f7b3d4b7922db9bb00c64c4e16ee68ab4caaae79f55b3e18748c7"}, + {file = "sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce61efc1862ccb18856c4aabbd930e13d5bfbb4b09b4f111081ac53a9dc62275"}, + {file = "sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a78c03800ef9f02d320e0159f5768b15357f3e9ebea545c9c4ba7928ba8ba254"}, + {file = "sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753b8088fd685ee787d9f54c84275ab347de558c7c4ebc6accb4c35bf7776f20"}, + {file = "sentencepiece-0.1.97-cp39-cp39-win32.whl", hash = "sha256:24306fd86031c17a1a6ae92671e76a350390a3140a65620bc2843dad7db24e2a"}, + {file = "sentencepiece-0.1.97-cp39-cp39-win_amd64.whl", hash = "sha256:c6641d0b7acec61fde5881ea6ebe098c169557ac9aa3bdabdf124eab5a5592bb"}, + {file = "sentencepiece-0.1.97.tar.gz", hash = "sha256:c901305e0a710bbcd296f66d79e96f744e6e175b29812bd5178318437d4e1f6c"}, +] setuptools = [ {file = "setuptools-67.4.0-py3-none-any.whl", hash = "sha256:f106dee1b506dee5102cc3f3e9e68137bbad6d47b616be7991714b0c62204251"}, {file = "setuptools-67.4.0.tar.gz", hash = "sha256:e5fd0a713141a4a105412233c63dc4e17ba0090c8e8334594ac790ec97792330"}, ] +tokenizers = [ + {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, + {file = "tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07"}, + {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc"}, + {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee0b1b311d65beab83d7a41c56a1e46ab732a9eed4460648e8eb0bd69fc2d059"}, + {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ef4215284df1277dadbcc5e17d4882bda19f770d02348e73523f7e7d8b8d396"}, + {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4d53976079cff8a033f778fb9adca2d9d69d009c02fa2d71a878b5f3963ed30"}, + {file = "tokenizers-0.13.3-cp310-cp310-win32.whl", hash = "sha256:1f0e3b4c2ea2cd13238ce43548959c118069db7579e5d40ec270ad77da5833ce"}, + {file = "tokenizers-0.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:89649c00d0d7211e8186f7a75dfa1db6996f65edce4b84821817eadcc2d3c79e"}, + {file = "tokenizers-0.13.3-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:56b726e0d2bbc9243872b0144515ba684af5b8d8cd112fb83ee1365e26ec74c8"}, + {file = "tokenizers-0.13.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:cc5c022ce692e1f499d745af293ab9ee6f5d92538ed2faf73f9708c89ee59ce6"}, + {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f55c981ac44ba87c93e847c333e58c12abcbb377a0c2f2ef96e1a266e4184ff2"}, + {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f247eae99800ef821a91f47c5280e9e9afaeed9980fc444208d5aa6ba69ff148"}, + {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b3e3215d048e94f40f1c95802e45dcc37c5b05eb46280fc2ccc8cd351bff839"}, + {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ba2b0bf01777c9b9bc94b53764d6684554ce98551fec496f71bc5be3a03e98b"}, + {file = "tokenizers-0.13.3-cp311-cp311-win32.whl", hash = "sha256:cc78d77f597d1c458bf0ea7c2a64b6aa06941c7a99cb135b5969b0278824d808"}, + {file = "tokenizers-0.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:ecf182bf59bd541a8876deccf0360f5ae60496fd50b58510048020751cf1724c"}, + {file = "tokenizers-0.13.3-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:0527dc5436a1f6bf2c0327da3145687d3bcfbeab91fed8458920093de3901b44"}, + {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07cbb2c307627dc99b44b22ef05ff4473aa7c7cc1fec8f0a8b37d8a64b1a16d2"}, + {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4560dbdeaae5b7ee0d4e493027e3de6d53c991b5002d7ff95083c99e11dd5ac0"}, + {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64064bd0322405c9374305ab9b4c07152a1474370327499911937fd4a76d004b"}, + {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8c6e2ab0f2e3d939ca66aa1d596602105fe33b505cd2854a4c1717f704c51de"}, + {file = "tokenizers-0.13.3-cp37-cp37m-win32.whl", hash = "sha256:6cc29d410768f960db8677221e497226e545eaaea01aa3613fa0fdf2cc96cff4"}, + {file = "tokenizers-0.13.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fc2a7fdf864554a0dacf09d32e17c0caa9afe72baf9dd7ddedc61973bae352d8"}, + {file = "tokenizers-0.13.3-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:8791dedba834c1fc55e5f1521be325ea3dafb381964be20684b92fdac95d79b7"}, + {file = "tokenizers-0.13.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:d607a6a13718aeb20507bdf2b96162ead5145bbbfa26788d6b833f98b31b26e1"}, + {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3791338f809cd1bf8e4fee6b540b36822434d0c6c6bc47162448deee3f77d425"}, + {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2f35f30e39e6aab8716f07790f646bdc6e4a853816cc49a95ef2a9016bf9ce6"}, + {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310204dfed5aa797128b65d63538a9837cbdd15da2a29a77d67eefa489edda26"}, + {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0f9b92ea052305166559f38498b3b0cae159caea712646648aaa272f7160963"}, + {file = "tokenizers-0.13.3-cp38-cp38-win32.whl", hash = "sha256:9a3fa134896c3c1f0da6e762d15141fbff30d094067c8f1157b9fdca593b5806"}, + {file = "tokenizers-0.13.3-cp38-cp38-win_amd64.whl", hash = "sha256:8e7b0cdeace87fa9e760e6a605e0ae8fc14b7d72e9fc19c578116f7287bb873d"}, + {file = "tokenizers-0.13.3-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:00cee1e0859d55507e693a48fa4aef07060c4bb6bd93d80120e18fea9371c66d"}, + {file = "tokenizers-0.13.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a23ff602d0797cea1d0506ce69b27523b07e70f6dda982ab8cf82402de839088"}, + {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70ce07445050b537d2696022dafb115307abdffd2a5c106f029490f84501ef97"}, + {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:280ffe95f50eaaf655b3a1dc7ff1d9cf4777029dbbc3e63a74e65a056594abc3"}, + {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97acfcec592f7e9de8cadcdcda50a7134423ac8455c0166b28c9ff04d227b371"}, + {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd7730c98a3010cd4f523465867ff95cd9d6430db46676ce79358f65ae39797b"}, + {file = "tokenizers-0.13.3-cp39-cp39-win32.whl", hash = "sha256:48625a108029cb1ddf42e17a81b5a3230ba6888a70c9dc14e81bc319e812652d"}, + {file = "tokenizers-0.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783"}, + {file = "tokenizers-0.13.3.tar.gz", hash = "sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e"}, +] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index e9dc624c..38f7fb04 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -23,6 +23,8 @@ opentelemetry-api = "^1.15.0" opentelemetry-exporter-otlp = "^1.15.0" opentelemetry-instrumentation-grpc = "^0.36b0" hf-transfer = "^0.1.2" +sentencepiece = "^0.1.97" +tokenizers = "0.13.3" [tool.poetry.extras] bnb = ["bitsandbytes"] diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index baf44579..79435787 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -148,7 +148,7 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) assert all([generation.generated_text is None for generation in generations]) assert all([len(generation.prefill_tokens) == 1 for generation in generations]) assert all([generation.token_id.item() == 259 for generation in generations]) - assert all([generation.token_text == " " for generation in generations]) + assert all([generation.token_text == "" for generation in generations]) assert generations[0].request_id == 0 diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index bcaf6ec1..1e06b6dc 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -19,13 +19,11 @@ try: from text_generation_server.models.flash_neox import FlashNeoX, FlashNeoXSharded from text_generation_server.models.flash_santacoder import FlashSantacoder + from text_generation_server.models.flash_llama import FlashLlama, FlashLlamaSharded - FLASH_ATTENTION = ( - torch.cuda.is_available() and int(os.environ.get("FLASH_ATTENTION", 0)) == 1 - ) + FLASH_ATTENTION = torch.cuda.is_available() except ImportError: - if int(os.environ.get("FLASH_ATTENTION", 0)) == 1: - logger.exception("Could not import Flash Attention models") + logger.exception("Could not import Flash Attention enabled models") FLASH_ATTENTION = False __all__ = [ @@ -47,6 +45,12 @@ __all__.append(FlashNeoX) __all__.append(FlashNeoXSharded) __all__.append(FlashSantacoder) + __all__.append(FlashLlama) + __all__.append(FlashLlamaSharded) + +FLASH_ATT_ERROR_MESSAGE = "{} requires Flash Attention CUDA kernels to be installed.\n" \ + "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) " \ + "or install flash attention with `cd server && make install install-flash-attention`" # The flag below controls whether to allow TF32 on matmul. This flag defaults to False # in PyTorch 1.12 and later. @@ -60,7 +64,7 @@ def get_model( - model_id: str, revision: Optional[str], sharded: bool, quantize: bool + model_id: str, revision: Optional[str], sharded: bool, quantize: bool ) -> Model: if "facebook/galactica" in model_id: if sharded: @@ -92,6 +96,17 @@ def get_model( neox_cls = FlashNeoX if FLASH_ATTENTION else CausalLM return neox_cls(model_id, revision, quantize=quantize) + if model_type == "llama": + if sharded: + if FLASH_ATTENTION: + return FlashLlamaSharded(model_id, revision, quantize=quantize) + raise NotImplementedError( + FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Llama") + ) + else: + llama_cls = FlashLlama if FLASH_ATTENTION else CausalLM + return llama_cls(model_id, revision, quantize=quantize) + if model_type == "t5": if sharded: return T5Sharded(model_id, revision, quantize=quantize) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index cb7bbfd8..8c092d6a 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -34,6 +34,8 @@ class CausalLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] + offsets: List[Optional[int]] + token_offsets: List[Optional[int]] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -64,12 +66,16 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] + offsets = [] + token_offsets = [] # Parse batch max_truncation = 0 padding_right_offset = 0 for r in pb.requests: inputs.append(r.inputs) + offsets.append(None) + token_offsets.append(None) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer @@ -113,6 +119,8 @@ def from_pb( past_key_values=None, all_input_ids=all_input_ids, input_lengths=input_lengths.tolist(), + offsets=offsets, + token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=pb.size, @@ -135,6 +143,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # Batch attributes requests = [] input_lengths = [] + offsets = [] + token_offsets = [] all_input_ids = [] next_token_choosers = [] stopping_criterias = [] @@ -151,6 +161,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": for i, batch in enumerate(batches): requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) + offsets.extend(batch.offsets) + token_offsets.extend(batch.token_offsets) all_input_ids.extend(batch.all_input_ids) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -264,6 +276,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": past_key_values=past_key_values, all_input_ids=all_input_ids, input_lengths=input_lengths, + offsets=offsets, + token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=total_batch_size, @@ -289,7 +303,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) self.model = AutoModelForCausalLM.from_pretrained( model_id, @@ -350,6 +364,8 @@ def generate_token( # New values for next forward next_batch_input_lengths = [] + next_batch_offsets = [] + next_batch_token_offsets = [] next_batch_input_ids = [] next_batch_all_input_ids = [] @@ -364,6 +380,8 @@ def generate_token( iterator = zip( batch.requests, batch.input_lengths, + batch.offsets, + batch.token_offsets, logits, batch.next_token_choosers, batch.stopping_criterias, @@ -374,6 +392,8 @@ def generate_token( for i, ( request, input_length, + offset, + token_offset, logits, next_token_chooser, stopping_criteria, @@ -391,8 +411,8 @@ def generate_token( # Generated token next_token_logprob = logprobs[-1, next_token_id] next_token_id_squeezed = next_token_id.squeeze() - next_token_text = self.decode_token( - next_token_id_squeezed, + next_token_text, offset, token_offset = self.decode_token( + all_input_ids[:, 0], offset, token_offset ) # Evaluate stopping criteria @@ -423,6 +443,8 @@ def generate_token( next_batch_all_input_ids.append(all_input_ids) next_batch_size += 1 next_batch_input_lengths.append(new_input_length) + next_batch_offsets.append(offset) + next_batch_token_offsets.append(token_offset) next_batch_max_input_length = max( next_batch_max_input_length, new_input_length ) @@ -506,6 +528,8 @@ def generate_token( past_key_values=next_batch_past_key_values, all_input_ids=next_batch_all_input_ids, input_lengths=next_batch_input_lengths, + offsets=next_batch_offsets, + token_offsets=next_batch_token_offsets, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py new file mode 100644 index 00000000..228529cc --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -0,0 +1,619 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.distributed + +from torch.nn import functional as F + +from torch import nn +from transformers.activations import ACT2FN + +# Flash attention imports +import rotary_emb +import flash_attn_cuda +import dropout_layer_norm + +from flash_attn.layers.rotary import RotaryEmbedding + + +class LlamaRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + LlamaRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states, residual=None): + if hidden_states.shape[-1] > 8192: + if residual is not None: + hidden_states += residual + residual = hidden_states + + variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt( + variance + self.variance_epsilon + ) + + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states, residual + else: + # faster post attention rms norm + normed_hidden_states, res, *rest = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.weight, + None, + None, + None, + None, + None, + 0.0, + self.variance_epsilon, + 1.0, + 0, + None, + False, + True, # Activate RMSNorm + ) + if res is None: + res = hidden_states + + return normed_hidden_states, res + + +class FastLinear(nn.Linear): + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + device=None, + dtype=None, + ) -> None: + super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) + + def transpose_weight(self): + self.weight = nn.Parameter(self.weight.T) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + if self.bias is not None: + return torch.addmm(self.bias, input, self.weight) + return torch.matmul(input, self.weight) + + +class TensorParallelColumnLinear(FastLinear): + def __init__( + self, + in_features, + out_features, + process_group: torch.distributed.ProcessGroup, + bias=True, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_world_size = process_group.size() + assert out_features % self.tp_world_size == 0 + out_features = out_features // self.tp_world_size + + super().__init__( + in_features=in_features, + out_features=out_features, + bias=bias, + device=device, + dtype=dtype, + ) + + +class TensorParallelRowLinear(FastLinear): + def __init__( + self, + in_features, + out_features, + process_group: torch.distributed.ProcessGroup, + reduce=True, + bias=True, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_world_size = process_group.size() + self.reduce = reduce + assert in_features % self.tp_world_size == 0 + in_features = in_features // self.tp_world_size + + super().__init__( + in_features=in_features, + out_features=out_features, + bias=bias, + device=device, + dtype=dtype, + ) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + out = super(TensorParallelRowLinear, self).forward(input) + if self.reduce: + torch.distributed.all_reduce(out, group=self.process_group) + + return out + + +class TensorParallelEmbedding(nn.Embedding): + def __init__( + self, + num_embeddings, + embedding_dim, + process_group: torch.distributed.ProcessGroup, + padding_idx=None, + max_norm=None, + norm_type=2.0, + scale_grad_by_freq=False, + sparse=False, + _weight=None, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + + self.original_num_embeddings = num_embeddings + + assert num_embeddings % self.tp_world_size == 0 + block_size = num_embeddings // self.tp_world_size + # inputs in `[min_id, max_id[` are handled by `self` to get embeddings + self.min_id = self.tp_rank * block_size + self.max_id = (self.tp_rank + 1) * block_size + + # Additional entry that will map to zero + # Used for masking + self.null_idx = block_size + + super().__init__( + block_size, + embedding_dim, + padding_idx=padding_idx, + max_norm=max_norm, + norm_type=norm_type, + scale_grad_by_freq=scale_grad_by_freq, + sparse=sparse, + _weight=_weight, + device=device, + dtype=dtype, + ) + + def add_null_idx(self): + """Additional 0 entry used for masking""" + self.weight = nn.Parameter(F.pad(self.weight, (0, 0, 0, 1))) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + # default all out of bounds values to `self.null_idx` that will then be mapped to 0 + # translate for [0, self.max_id - self.min_id[ + input = torch.where( + (self.min_id > input) | (input >= self.max_id), + self.null_idx, + input - self.min_id, + ) + out = super().forward(input) + torch.distributed.all_reduce(out, group=self.process_group) + return out + + +class PositionRotaryEmbedding(RotaryEmbedding): + def _update_cos_sin_cache(self, dtype, device, seqlen): + # Reset the tables if the sequence length has changed, + # or if we're on a new device (possibly due to tracing for instance) + if ( + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype + ): + self._seq_len_cached = seqlen + t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) + freqs = torch.outer(t, self.inv_freq.to(device=t.device)) + self._cos_cached = torch.cos(freqs).to(dtype) + self._sin_cached = torch.sin(freqs).to(dtype) + + def get_cos_sin(self, position_ids: torch.Tensor, max_s: int, dtype: torch.dtype): + """ + Return cos and sin for the asked position ids + """ + + self._update_cos_sin_cache(dtype, position_ids.device, max_s) + + cos = torch.index_select(self._cos_cached, 0, position_ids) + sin = torch.index_select(self._sin_cached, 0, position_ids) + return cos.unsqueeze(1), sin.unsqueeze(1) + + def forward(self, qkv: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): + rotary_dim = cos.shape[-1] + q1 = qkv[:, 0, :, :rotary_dim] + q2 = qkv[:, 0, :, rotary_dim : 2 * rotary_dim] + k1 = qkv[:, 1, :, :rotary_dim] + k2 = qkv[:, 1, :, rotary_dim : 2 * rotary_dim] + + rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False) + rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False) + return qkv + + +class FlashLlamaAttention(torch.nn.Module): + def __init__( + self, + num_heads, + hidden_size, + process_group=None, + ): + super().__init__() + self.num_heads = num_heads + self.hidden_size = hidden_size + self.head_size = hidden_size // num_heads + + self.rotary_emb = PositionRotaryEmbedding(self.head_size, base=10000) + self.softmax_scale = self.head_size ** (-0.5) + + if process_group is None: + self.query_key_value = FastLinear(hidden_size, 3 * hidden_size, bias=False) + self.o_proj = FastLinear(hidden_size, hidden_size, bias=False) + else: + self.num_heads = self.num_heads // process_group.size() + self.query_key_value = TensorParallelColumnLinear( + hidden_size, + 3 * hidden_size, + bias=False, + process_group=process_group, + ) + self.o_proj = TensorParallelRowLinear( + hidden_size, + hidden_size, + bias=False, + process_group=process_group, + ) + + def forward( + self, + hidden_states, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + qkv = self.query_key_value(hidden_states) + qkv = qkv.view(-1, 3, self.num_heads, self.head_size) + qkv_rot = self.rotary_emb(qkv, cos, sin) + + # Prefill + if layer_past_present_indices is None: + # Copy to layer past + layer_past[...] = qkv_rot[:, 1:] + + # output + attn_output = torch.empty_like(qkv_rot[:, 0]) + # flash attention + flash_attn_cuda.fwd( + qkv_rot[:, 0], + qkv_rot[:, 1], + qkv_rot[:, 2], + attn_output, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + 0.0, + self.softmax_scale, + False, + True, + False, + 0, + None, + ) + # Decode + else: + query = qkv_rot[:, 0] + # Add present to the layer_past tensor at the correct indices + layer_past[layer_past_present_indices] = qkv_rot[:, 1:] + + # output + attn_output = torch.empty_like(query) + # flash attention + flash_attn_cuda.fwd( + query, + layer_past[:, 0], + layer_past[:, 1], + attn_output, + cu_seqlens_q, + cu_seqlens, + 1, + max_s, + 0.0, + self.softmax_scale, + False, + False, + False, + 0, + None, + ) + + return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size)) + + +class LlamaMLP(nn.Module): + def __init__(self, act, hidden_size, intermediate_size, process_group=None): + super().__init__() + self.act = ( + ACT2FN[act] + if "gelu" not in act + else lambda x: torch.nn.functional.gelu( + x, + approximate="tanh" + if act in ["gelu_fast", "gelu_pytorch_tanh"] + else None, + ) + ) + + if process_group is None: + # Fuse gate and up proj + self.gate_up_proj = FastLinear( + hidden_size, 2 * intermediate_size, bias=False + ) + self.down_proj = FastLinear(intermediate_size, hidden_size, bias=False) + self.intermediate_size = intermediate_size + else: + # Fuse gate and up proj + self.gate_up_proj = TensorParallelColumnLinear( + hidden_size, + 2 * intermediate_size, + bias=False, + process_group=process_group, + ) + self.down_proj = TensorParallelRowLinear( + intermediate_size, + hidden_size, + bias=False, + process_group=process_group, + reduce=True, + ) + self.intermediate_size = self.down_proj.in_features + + self.process_group = process_group + + def forward(self, hidden_states): + gate_up_states = self.gate_up_proj(hidden_states) + gate_up_states = gate_up_states.view(-1, 2, self.intermediate_size) + return self.down_proj(self.act(gate_up_states[:, 0]) * gate_up_states[:, 1]) + + +class FlashLlamaLayer(nn.Module): + def __init__( + self, + num_heads, + act, + hidden_size, + intermediate_size, + rms_norm_eps, + process_group=None, + ): + super().__init__() + + self.self_attn = FlashLlamaAttention(num_heads, hidden_size, process_group) + self.mlp = LlamaMLP(act, hidden_size, intermediate_size, process_group) + + self.input_layernorm = LlamaRMSNorm(hidden_size, eps=rms_norm_eps) + self.post_attention_layernorm = LlamaRMSNorm(hidden_size, eps=rms_norm_eps) + + def forward( + self, + hidden_states, + residual, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + normed_hidden_states, res = self.input_layernorm(hidden_states, residual) + + # Self Attention + attn_output = self.self_attn( + normed_hidden_states, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ) + + # faster post attention rms norm + normed_attn_res_output, attn_res = self.post_attention_layernorm( + attn_output, res + ) + + mlp_output = self.mlp(normed_attn_res_output) + + return mlp_output, attn_res + + +class FlashLlamaModel(torch.nn.Module): + def __init__(self, config, process_group=None): + super(FlashLlamaModel, self).__init__() + self.config = config + + self.tp_embeddings = False + if process_group is not None: + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + if config.vocab_size % self.tp_world_size == 0: + self.tp_embeddings = True + + if self.tp_embeddings: + self.embed_tokens = TensorParallelEmbedding( + config.vocab_size, config.hidden_size, process_group=process_group + ) + else: + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) + + self.layers = nn.ModuleList( + [ + FlashLlamaLayer( + config.num_attention_heads, + config.hidden_act, + config.hidden_size, + config.intermediate_size, + config.rms_norm_eps, + process_group, + ) + for _ in range(config.num_hidden_layers) + ] + ) + self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.gradient_checkpointing = False + + self.head_size = self.layers[0].self_attn.head_size + self.num_heads = self.layers[0].self_attn.num_heads + + def post_load_weights(self): + if isinstance(self.embed_tokens, TensorParallelEmbedding): + self.embed_tokens.add_null_idx() + for layer in self.layers: + layer: FlashLlamaLayer + layer.self_attn.query_key_value.transpose_weight() + layer.self_attn.o_proj.transpose_weight() + layer.mlp.gate_up_proj.transpose_weight() + layer.mlp.down_proj.transpose_weight() + + def forward( + self, + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values=None, + ): + hidden_states = self.embed_tokens(input_ids) + + # Prefill + if past_key_values is None: + # Create past tensor + past_key_values = hidden_states.new_empty( + ( + len(self.layers), + len(hidden_states), + 2, + self.num_heads, + self.head_size, + ) + ) + layer_past_present_indices = None + cu_seqlens_q = None + # Decode + else: + # Create indices from cumulative sequence lengths + layer_past_present_indices = cu_seqlens[1:] - 1 + cu_seqlens_q = torch.arange( + cu_seqlens.shape[0], dtype=torch.int32, device=hidden_states.device + ) + + # Get rotary cos and sin for this forward + # Avoid to index in each layer + cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin( + position_ids, max_s, hidden_states.dtype + ) + + residual = None + for i, layer in enumerate(self.layers): + hidden_states, residual = layer( + hidden_states, + residual, + cos, + sin, + cu_seqlens, + max_s, + past_key_values[i], + layer_past_present_indices, + cu_seqlens_q, + ) + + hidden_states, _ = self.norm(hidden_states, residual) + + return hidden_states, past_key_values + + +class FlashLlamaForCausalLM(torch.nn.Module): + def __init__(self, config, process_group=None): + super().__init__() + + self.process_group = process_group + if self.process_group is not None: + self.world_size = self.process_group.size() + self.rank = self.process_group.rank() + else: + self.world_size = 1 + self.rank = 0 + + self.model = FlashLlamaModel(config, process_group) + + if self.model.tp_embeddings: + self.lm_head = FastLinear( + config.hidden_size, + config.vocab_size // process_group.size(), + bias=False, + ) + else: + self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) + + def post_load_weights(self): + self.model.post_load_weights() + self.lm_head.transpose_weight() + + def forward( + self, + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values=None, + ): + hidden_states, present = self.model( + input_ids, position_ids, cu_seqlens, max_s, past_key_values + ) + logits = self.lm_head(hidden_states) + + if self.model.tp_embeddings: + # Logits are sharded, so we need to gather them + world_logits = [torch.empty_like(logits) for _ in range(self.world_size)] + torch.distributed.all_gather(world_logits, logits, group=self.process_group) + world_logits = torch.cat(world_logits, dim=1) + + return world_logits, present + return logits, present diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index bc1ac063..3801ed24 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -44,6 +44,8 @@ class FlashCausalLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] + offsets: List[Optional[int]] + token_offsets: List[Optional[int]] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -67,6 +69,8 @@ def from_pb( max_seqlen = 0 input_lengths = [] + offsets = [] + token_offsets = [] all_input_ids = [] all_input_ids_tensor = [] @@ -84,6 +88,8 @@ def from_pb( input_length = len(tokenized_input) max_seqlen = max(max_seqlen, input_length) input_lengths.append(input_length) + offsets.append(None) + token_offsets.append(None) all_input_ids.append(tokenized_input) tokenized_input = torch.tensor(tokenized_input, device=device) @@ -120,6 +126,8 @@ def from_pb( max_seqlen=max_seqlen, past_key_values=None, input_lengths=input_lengths, + offsets=offsets, + token_offsets=token_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=all_input_ids_tensor, next_token_choosers=next_token_choosers, @@ -132,6 +140,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch # Batch attributes requests = [] input_lengths = [] + offsets = [] + token_offsets = [] all_input_ids = [] all_input_ids_tensor = [] next_token_choosers = [] @@ -150,6 +160,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch for i, batch in enumerate(batches): requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) + offsets.extend(batch.offsets) + token_offsets.extend(batch.token_offsets) all_input_ids.extend(batch.all_input_ids) all_input_ids_tensor.extend(batch.all_input_ids_tensor) next_token_choosers.extend(batch.next_token_choosers) @@ -182,6 +194,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch max_seqlen=max_seqlen, past_key_values=past_key_values, input_lengths=input_lengths, + offsets=offsets, + token_offsets=token_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=all_input_ids_tensor, next_token_choosers=next_token_choosers, @@ -279,6 +293,8 @@ def generate_token( next_batch_max_seqlen = 0 next_batch_past_key_values = [] next_batch_input_lengths = [] + next_batch_offsets = [] + next_batch_token_offsets = [] next_batch_all_input_ids = [] next_batch_all_input_ids_tensor = [] @@ -292,6 +308,8 @@ def generate_token( iterator = zip( batch.requests, batch.input_lengths, + batch.offsets, + batch.token_offsets, batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, @@ -302,6 +320,8 @@ def generate_token( for i, ( request, input_length, + offset, + token_offset, next_token_chooser, stopping_criteria, all_input_ids, @@ -334,8 +354,10 @@ def generate_token( # Generated token next_token_logprob = logprobs[-1, next_token_id_item] - next_token_text = self.decode_token( - next_token_id_item, + next_token_text, offset, token_offset = self.decode_token( + all_input_ids, + offset, + token_offset, ) # Evaluate stopping criteria @@ -376,6 +398,8 @@ def generate_token( next_batch_cu_seqlens[-1] + new_input_length ) next_batch_input_lengths.append(new_input_length) + next_batch_offsets.append(offset) + next_batch_token_offsets.append(token_offset) next_batch_all_input_ids.append(all_input_ids) next_batch_all_input_ids_tensor.append(all_input_ids_tensor) next_batch_max_seqlen = max(next_batch_max_seqlen, new_input_length) @@ -452,6 +476,8 @@ def generate_token( max_seqlen=next_batch_max_seqlen, past_key_values=next_batch_past_key_values, input_lengths=next_batch_input_lengths, + offsets=next_batch_offsets, + token_offsets=next_batch_token_offsets, all_input_ids=next_batch_all_input_ids, all_input_ids_tensor=next_batch_all_input_ids_tensor, next_token_choosers=next_batch_next_token_choosers, diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py new file mode 100644 index 00000000..063910f4 --- /dev/null +++ b/server/text_generation_server/models/flash_llama.py @@ -0,0 +1,303 @@ +import torch +import torch.distributed + +from accelerate import init_empty_weights +from opentelemetry import trace +from pathlib import Path +from safetensors import safe_open +from transformers import AutoConfig +from transformers.models.llama import LlamaTokenizer +from typing import Optional, List + +from text_generation_server.models import FlashCausalLM +from text_generation_server.models.custom_modeling.flash_llama_modeling import ( + FlashLlamaForCausalLM, + TensorParallelEmbedding, + TensorParallelRowLinear, + TensorParallelColumnLinear, +) +from text_generation_server.utils import ( + initialize_torch_distributed, + weight_files, + download_weights, + weight_hub_files, + LocalEntryNotFoundError, +) + +tracer = trace.get_tracer(__name__) + + +class FlashLlama(FlashCausalLM): + def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + else: + raise NotImplementedError("FlashLlama is only available on GPU") + + if quantize: + raise NotImplementedError("FlashLlama does not support quantization") + + tokenizer = LlamaTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + ) + + config = AutoConfig.from_pretrained( + model_id, + revision=revision, + ) + + # We do not use from_pretrained as we modified the model internal module layout + try: + filenames = weight_files(model_id, revision, ".bin") + # Local files not found + except LocalEntryNotFoundError: + hub_files = weight_hub_files(model_id, revision, ".bin") + filenames = download_weights(hub_files, model_id, revision) + + with init_empty_weights(): + model = FlashLlamaForCausalLM(config) + + self.load_weights(model, filenames, device, dtype) + self.model = model.eval() + + super(FlashCausalLM, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @staticmethod + def load_weights( + model, + filenames: List[Path], + device: torch.device, + dtype: torch.dtype, + ): + for filename in filenames: + state_dict = torch.load(filename, map_location="cpu") + for key, value in state_dict.items(): + value = value.to(device).to(dtype) + + layer_name = ".".join(key.split(".")[:4]) + + # Fused qkv + if "q_proj" in key or "k_proj" in key or "v_proj" in key: + final_key = layer_name + ".query_key_value.weight" + + # Fused gate and up projs + elif "gate_proj" in key or "up_proj" in key: + final_key = layer_name + ".gate_up_proj.weight" + else: + final_key = key + + module_name, param_name = final_key.rsplit(".", 1) + module = model.get_submodule(module_name) + + try: + current_parameter_tensor = module._parameters[param_name] + except KeyError: + current_parameter_tensor = None + + if current_parameter_tensor is not None: + if current_parameter_tensor.device == torch.device("meta"): + # Init qkv + if "query_key_value" in final_key: + module._parameters[param_name] = value.new_empty( + (value.shape[0] * 3, value.shape[1]) + ) + # Init gate and up proj + elif "gate_up_proj" in final_key: + module._parameters[param_name] = value.new_empty( + (value.shape[0] * 2, value.shape[1]) + ) + + # Copy to correct slice + if "q_proj" in key: + module._parameters[param_name][: value.shape[0]] = value + elif "k_proj" in key: + module._parameters[param_name][ + value.shape[0] : value.shape[0] * 2 + ] = value + elif "v_proj" in key: + module._parameters[param_name][value.shape[0] * 2 :] = value + elif "gate_proj" in key: + module._parameters[param_name][: value.shape[0]] = value + elif "up_proj" in key: + module._parameters[param_name][value.shape[0] :] = value + else: + if current_parameter_tensor.shape != value.shape: + raise ValueError( + f"Name {final_key} -- Current {current_parameter_tensor.shape} and got {value.shape}" + ) + module._parameters[param_name] = value + else: + module._buffers[param_name] = value + + del value + + torch.cuda.empty_cache() + model.post_load_weights() + + +class FlashLlamaSharded(FlashLlama): + def __init__( + self, model_id: str, revision: Optional[str] = None, quantize: bool = False + ): + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + device = torch.device(f"cuda:{self.rank}") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + else: + raise NotImplementedError("FlashLlama is only available on GPU") + + if quantize: + raise NotImplementedError("FlashLlama does not support quantization") + + tokenizer = LlamaTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + ) + + config = AutoConfig.from_pretrained( + model_id, + revision=revision, + ) + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + + with init_empty_weights(): + model = FlashLlamaForCausalLM(config, process_group=self.process_group) + + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + quantize=quantize, + device=device, + dtype=dtype, + rank=self.rank, + world_size=self.world_size, + ) + self.model = model.eval() + torch.distributed.barrier(group=self.process_group) + super(FlashCausalLM, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @staticmethod + def load_weights( + model, + filenames: List[str], + quantize: bool, + device: torch.device, + dtype: torch.dtype, + rank: int, + world_size: int, + ): + for file in filenames: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: + for name in f.keys(): + slice_ = f.get_slice(name) + + layer_name = ".".join(name.split(".")[:4]) + + # Fused qkv + if "q_proj" in name or "k_proj" in name or "v_proj" in name: + final_name = layer_name + ".query_key_value.weight" + + # Fused gate and up projs + elif "gate_proj" in name or "up_proj" in name: + final_name = layer_name + ".gate_up_proj.weight" + else: + final_name = name + + module_name, param_name = final_name.rsplit(".", 1) + module = model.get_submodule(module_name) + + if isinstance(module, TensorParallelColumnLinear): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif isinstance(module, TensorParallelRowLinear): + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif name == "lm_head.weight" and model.model.tp_embeddings: + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + else: + try: + tensor = slice_[:] + except: + tensor = f.get_tensor(name) + + tensor = tensor.contiguous().to(dtype) + + try: + current_parameter_tensor = module._parameters[param_name] + except KeyError: + current_parameter_tensor = None + + if current_parameter_tensor is not None: + if current_parameter_tensor.device == torch.device("meta"): + # Init qkv + if "query_key_value" in final_name: + module._parameters[param_name] = tensor.new_empty( + (tensor.shape[0] * 3, tensor.shape[1]) + ) + # Init gate and up proj + elif "gate_up_proj" in final_name: + module._parameters[param_name] = tensor.new_empty( + (tensor.shape[0] * 2, tensor.shape[1]) + ) + + # Init gate and up proj + if "q_proj" in name: + module._parameters[param_name][: tensor.shape[0]] = tensor + elif "k_proj" in name: + module._parameters[param_name][ + tensor.shape[0] : tensor.shape[0] * 2 + ] = tensor + elif "v_proj" in name: + module._parameters[param_name][ + tensor.shape[0] * 2 : + ] = tensor + elif "gate_proj" in name: + module._parameters[param_name][: tensor.shape[0]] = tensor + elif "up_proj" in name: + module._parameters[param_name][tensor.shape[0] :] = tensor + else: + if current_parameter_tensor.shape != tensor.shape: + raise ValueError( + f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" + ) + + module._parameters[param_name] = tensor + + else: + module._buffers[param_name] = tensor + torch.cuda.empty_cache() + model.post_load_weights() diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 3c89b4a7..f7fbb2ad 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -93,7 +93,8 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] - input_lengths = [] + offsets = [] + token_offsets = [] # Parse batch max_truncation = 0 @@ -101,7 +102,8 @@ def from_pb( for r in pb.requests: # Add escape_custom_split_sequence to the CausalLMBatch logic inputs.append(escape_custom_split_sequence(r.inputs)) - input_lengths.append(r.input_length) + offsets.append(None) + token_offsets.append(None) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer @@ -146,6 +148,8 @@ def from_pb( past_key_values=None, all_input_ids=all_input_ids, input_lengths=input_lengths, + offsets=offsets, + token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=pb.size, diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index e0ce6686..5b82872c 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -15,15 +15,6 @@ def __init__(self, tokenizer: PreTrainedTokenizerBase, device: torch.device): self.all_special_ids = set(tokenizer.all_special_ids) self.device = device - # see `decode_token` method - self.tokenizer.add_special_tokens( - {"additional_special_tokens": [""]} - ) - self.special_decode_token_id = self.tokenizer.convert_tokens_to_ids( - "" - ) - self.special_decode_token_length = len("") - @property @abstractmethod def batch_type(self) -> Type[B]: @@ -33,11 +24,38 @@ def batch_type(self) -> Type[B]: def generate_token(self, batch: B) -> Tuple[List[GeneratedText], Optional[B]]: raise NotImplementedError - def decode_token(self, token_id: int) -> str: + def decode_token( + self, + all_input_ids: List[int], + offset: Optional[int] = None, + token_offset: Optional[int] = None, + ) -> Tuple[str, Optional[int], Optional[int]]: """Hack to hopefully support generate_stream for the maximum number of tokenizers""" - # append token to special decode token and decode both - result = self.tokenizer.decode( - [self.special_decode_token_id, token_id], skip_special_tokens=False + if all_input_ids[-1] in self.all_special_ids: + return ( + self.tokenizer.decode(all_input_ids[-1], skip_special_tokens=False), + None, + None, + ) + + if token_offset is None: + token_offset = len(all_input_ids) - 3 + + # Decode token_offset token minus last one and token_offset tokens + results = self.tokenizer.batch_decode( + [all_input_ids[token_offset:-1], all_input_ids[token_offset:]], + skip_special_tokens=False, ) - # slice to remove special decode token - return result[self.special_decode_token_length :] + + # default offset is only the last token + if offset is None: + offset = len(results[0]) + + # get text + text = results[1][offset:] + + # if text is utf-8 + if text and text[-1] != "�": + return text, None, None + else: + return "", offset, token_offset diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 7cf97123..13eafd62 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -38,6 +38,8 @@ class Seq2SeqLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] decoder_input_lengths: List[int] + offsets: List[Optional[int]] + token_offsets: List[Optional[int]] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -71,6 +73,8 @@ def from_pb( decoder_input_ids = [] decoder_input_lengths = [] + offsets = [] + token_offsets = [] # Parse batch max_truncation = 0 @@ -80,6 +84,8 @@ def from_pb( # Decoder sequence only contains the bos_token decoder_input_ids.append(tokenizer.bos_token_id) decoder_input_lengths.append(1) + offsets.append(None) + token_offsets.append(None) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer @@ -117,6 +123,8 @@ def from_pb( past_key_values=None, input_lengths=input_lengths.tolist(), decoder_input_lengths=decoder_input_lengths, + offsets=offsets, + token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=len(pb.requests), @@ -147,6 +155,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests = [] input_lengths = [] decoder_input_lengths = [] + offsets = [] + token_offsets = [] next_token_choosers = [] stopping_criterias = [] @@ -166,6 +176,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) decoder_input_lengths.extend(batch.decoder_input_lengths) + offsets.extend(batch.offsets) + token_offsets.extend(batch.token_offsets) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -303,6 +315,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": past_key_values=past_key_values, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, + offsets=offsets, + token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, size=total_batch_size, @@ -335,7 +349,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False load_in_8bit=quantize, ).eval() tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left" + model_id, revision=revision, padding_side="left", truncation_side="left" ) tokenizer.bos_token_id = self.model.config.decoder_start_token_id @@ -422,6 +436,8 @@ def generate_token( # New values for next forward next_batch_input_lengths = [] + next_batch_offsets = [] + next_batch_token_offsets = [] next_batch_decoder_input_ids = [] next_batch_decoder_input_lengths = [] @@ -437,6 +453,8 @@ def generate_token( iterator = zip( batch.requests, batch.input_lengths, + batch.offsets, + batch.token_offsets, batch.decoder_input_lengths, logits, batch.next_token_choosers, @@ -448,6 +466,8 @@ def generate_token( for i, ( request, input_length, + offset, + token_offset, decoder_input_length, logits, next_token_chooser, @@ -466,8 +486,8 @@ def generate_token( # Generated token next_token_logprob = logprobs[-1, next_token_id] next_token_id_squeezed = next_token_id.squeeze() - next_token_text = self.decode_token( - next_token_id_squeezed, + next_token_text, offset, token_offset = self.decode_token( + decoder_input_ids, offset, token_offset ) # Evaluate stopping criteria @@ -495,6 +515,8 @@ def generate_token( next_batch_size += 1 next_batch_input_lengths.append(input_length) next_batch_decoder_input_lengths.append(new_decoder_input_length) + next_batch_offsets.append(offset) + next_batch_token_offsets.append(token_offset) next_batch_max_input_length = max( next_batch_max_input_length, input_length ) @@ -580,6 +602,8 @@ def generate_token( past_key_values=next_batch_past_key_values, input_lengths=next_batch_input_lengths, decoder_input_lengths=next_batch_decoder_input_lengths, + offsets=next_batch_offsets, + token_offsets=next_batch_token_offsets, next_token_choosers=next_batch_next_token_choosers, stopping_criterias=next_batch_stopping_criterias, size=next_batch_size, From f26dfd0dc1f9e930209bee3c585236278baeae05 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 11 Apr 2023 19:16:41 +0200 Subject: [PATCH 151/793] feat(server): support OPT models (#55) OPT models do not all have a `tokenizer.json` file on the hub at the moment. Can't merge for now. --- README.md | 5 +- launcher/src/main.rs | 20 +- .../text_generation_server/models/__init__.py | 26 +- server/text_generation_server/models/bloom.py | 19 +- .../models/galactica.py | 24 +- .../text_generation_server/models/gpt_neox.py | 2 +- server/text_generation_server/models/opt.py | 224 ++++++++++++++++++ server/text_generation_server/models/t5.py | 2 +- server/text_generation_server/utils/hub.py | 5 - 9 files changed, 269 insertions(+), 58 deletions(-) create mode 100644 server/text_generation_server/models/opt.py diff --git a/README.md b/README.md index bc77fd4c..60c1a6ba 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,12 @@ to power LLMs api-inference widgets. ## Optimized architectures - [BLOOM](https://huggingface.co/bigscience/bloom) +- [FLAN-T5](https://huggingface.co/google/flan-t5-xxl) - [Galactica](https://huggingface.co/facebook/galactica-120b) -- [SantaCoder](https://huggingface.co/bigcode/santacoder) - [GPT-Neox](https://huggingface.co/EleutherAI/gpt-neox-20b) -- [FLAN-T5](https://huggingface.co/google/flan-t5-xxl) - [Llama](https://github.com/facebookresearch/llama) +- [OPT](https://huggingface.co/facebook/opt-66b) +- [SantaCoder](https://huggingface.co/bigcode/santacoder) Other architectures are supported on a best effort basis using: diff --git a/launcher/src/main.rs b/launcher/src/main.rs index a598a8bb..2b152be6 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -349,8 +349,8 @@ fn main() -> ExitCode { Err(TryRecvError::Empty) => { sleep(Duration::from_millis(100)); } - Ok(ShardStatus::Failed((rank, err))) => { - tracing::error!("Shard {} failed to start:\n{}", rank, err); + Ok(ShardStatus::Failed(rank)) => { + tracing::error!("Shard {} failed to start.", rank); shutdown_shards(shutdown, &shutdown_receiver); return ExitCode::FAILURE; } @@ -457,8 +457,8 @@ fn main() -> ExitCode { let mut exit_code = ExitCode::SUCCESS; while running.load(Ordering::SeqCst) { - if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { - tracing::error!("Shard {rank} failed:\n{err}"); + if let Ok(ShardStatus::Failed(rank)) = status_receiver.try_recv() { + tracing::error!("Shard {rank} failed."); exit_code = ExitCode::FAILURE; break; }; @@ -488,7 +488,7 @@ fn main() -> ExitCode { #[derive(Debug)] enum ShardStatus { Ready, - Failed((usize, String)), + Failed(usize), } #[allow(clippy::too_many_arguments)] @@ -627,9 +627,7 @@ fn shard_manager( tracing::error!("Please install it with `make install-server`") } } - status_sender - .send(ShardStatus::Failed((rank, err.to_string()))) - .unwrap(); + status_sender.send(ShardStatus::Failed(rank)).unwrap(); return; } }; @@ -658,11 +656,7 @@ fn shard_manager( loop { // Process exited if p.poll().is_some() { - let mut err = String::new(); - p.stderr.take().unwrap().read_to_string(&mut err).unwrap(); - status_sender - .send(ShardStatus::Failed((rank, err))) - .unwrap(); + status_sender.send(ShardStatus::Failed(rank)).unwrap(); return; } diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 1e06b6dc..c04ae117 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -1,4 +1,3 @@ -import os import torch from loguru import logger @@ -11,6 +10,7 @@ from text_generation_server.models.flash_causal_lm import FlashCausalLM from text_generation_server.models.bloom import BLOOM, BLOOMSharded from text_generation_server.models.seq2seq_lm import Seq2SeqLM +from text_generation_server.models.opt import OPT, OPTSharded from text_generation_server.models.galactica import Galactica, GalacticaSharded from text_generation_server.models.santacoder import SantaCoder from text_generation_server.models.gpt_neox import GPTNeoxSharded @@ -36,7 +36,11 @@ "GalacticaSharded", "GPTNeoxSharded", "Seq2SeqLM", + "Galactica", + "GalacticaSharded", "SantaCoder", + "OPT", + "OPTSharded", "T5Sharded", "get_model", ] @@ -48,9 +52,11 @@ __all__.append(FlashLlama) __all__.append(FlashLlamaSharded) -FLASH_ATT_ERROR_MESSAGE = "{} requires Flash Attention CUDA kernels to be installed.\n" \ - "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) " \ - "or install flash attention with `cd server && make install install-flash-attention`" +FLASH_ATT_ERROR_MESSAGE = ( + "{} requires Flash Attention CUDA kernels to be installed.\n" + "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) " + "or install flash attention with `cd server && make install install-flash-attention`" +) # The flag below controls whether to allow TF32 on matmul. This flag defaults to False # in PyTorch 1.12 and later. @@ -64,7 +70,7 @@ def get_model( - model_id: str, revision: Optional[str], sharded: bool, quantize: bool + model_id: str, revision: Optional[str], sharded: bool, quantize: bool ) -> Model: if "facebook/galactica" in model_id: if sharded: @@ -100,13 +106,17 @@ def get_model( if sharded: if FLASH_ATTENTION: return FlashLlamaSharded(model_id, revision, quantize=quantize) - raise NotImplementedError( - FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Llama") - ) + raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Llama")) else: llama_cls = FlashLlama if FLASH_ATTENTION else CausalLM return llama_cls(model_id, revision, quantize=quantize) + if config.model_type == "opt": + if sharded: + return OPTSharded(model_id, revision, quantize=quantize) + else: + return OPT(model_id, revision, quantize=quantize) + if model_type == "t5": if sharded: return T5Sharded(model_id, revision, quantize=quantize) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index ce3895ca..1a961027 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -62,7 +62,7 @@ def __init__( self.master = self.rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{self.rank}") - dtype = torch.bfloat16 + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") dtype = torch.float32 @@ -122,18 +122,11 @@ def load_weights( slice_ = f.get_slice(name) if isinstance(module, TensorParallelColumnLinear): - if param_name == "weight": - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] elif isinstance(module, TensorParallelRowLinear): if param_name == "weight": size = slice_.get_shape()[1] diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index f7fbb2ad..396cc4f6 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -19,8 +19,9 @@ ) from text_generation_server.models import CausalLM -from text_generation_server.pb import generate_pb2 from text_generation_server.models.causal_lm import CausalLMBatch +from text_generation_server.pb import generate_pb2 +from text_generation_server.models.opt import OPT from text_generation_server.utils import ( NextTokenChooser, StoppingCriteria, @@ -158,7 +159,7 @@ def from_pb( ) -class Galactica(CausalLM): +class Galactica(OPT): @property def batch_type(self) -> Type[CausalLMBatch]: return GalacticaCausalLMBatch @@ -192,7 +193,7 @@ def __init__( self.master = self.rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{self.rank}") - dtype = torch.bfloat16 + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") dtype = torch.float32 @@ -253,18 +254,11 @@ def load_weights( slice_ = f.get_slice(name) if isinstance(module, TensorParallelColumnLinear): - if param_name == "weight": - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] elif isinstance(module, TensorParallelRowLinear): if param_name == "weight": size = slice_.get_shape()[1] diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index b81976da..fb109ed7 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -38,7 +38,7 @@ def __init__( self.master = self.rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{self.rank}") - dtype = torch.bfloat16 + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py new file mode 100644 index 00000000..85f0ac8c --- /dev/null +++ b/server/text_generation_server/models/opt.py @@ -0,0 +1,224 @@ +import torch +import torch.distributed + +from typing import List, Optional, Tuple + +from accelerate import init_empty_weights +from safetensors import safe_open +from transformers import ( + AutoTokenizer, + AutoModelForCausalLM, + AutoConfig, +) +from transformers.models.opt.parallel_layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, +) + +from text_generation_server.models import CausalLM +from text_generation_server.utils import ( + initialize_torch_distributed, + weight_files, +) + +HAS_BITS_AND_BYTES = True +try: + import bitsandbytes as bnb + from bitsandbytes.nn import Int8Params +except Exception as e: + HAS_BITS_AND_BYTES = False + + +class OPT(CausalLM): + def forward( + self, input_ids, attention_mask, position_ids, past_key_values: Optional = None + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + """Overwrite forward to ignore position_ids""" + + # Model Forward + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + return outputs.logits, outputs.past_key_values + + +class OPTSharded(OPT): + def __init__( + self, model_id: str, revision: Optional[str] = None, quantize: bool = False + ): + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + device = torch.device(f"cuda:{self.rank}") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + else: + device = torch.device("cpu") + dtype = torch.float32 + + tokenizer = AutoTokenizer.from_pretrained( + model_id, revision=revision, padding_side="left", truncation_side="left" + ) + + config = AutoConfig.from_pretrained( + model_id, revision=revision, tp_parallel=True + ) + tokenizer.pad_token_id = config.pad_token_id + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + + with init_empty_weights(): + model = AutoModelForCausalLM.from_config(config) + + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + quantize=quantize, + device=device, + rank=self.rank, + world_size=self.world_size, + ) + self.model = model.eval().to(dtype) + torch.distributed.barrier(group=self.process_group) + super(CausalLM, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @staticmethod + def load_weights( + model, + filenames: List[str], + quantize: bool, + device: torch.device, + rank: int, + world_size: int, + ): + parameters = dict(model.named_parameters()) + for file in filenames: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: + for name in f.keys(): + if name == "lm_head.weight": + continue + + module_name, param_name = name.rsplit(".", 1) + module = model.get_submodule(module_name) + current_tensor = parameters[name] + + slice_ = f.get_slice(name) + + if isinstance(module, TensorParallelColumnLinear): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif isinstance(module, TensorParallelRowLinear): + if param_name == "weight": + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + else: + tensor = slice_[:] + # XXX: Hack for Rowlinear to add the bias only once. + if rank != 0: + tensor = torch.zeros_like(tensor) + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + else: + tensor = slice_[:] + + if current_tensor.shape != tensor.shape: + raise ValueError( + f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" + ) + + tensor = tensor.contiguous() + + if quantize: + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine either because it is not installed " + "or you don't have a GPU.\n" + "You can install it with `pip install bitsandbytes`." + ) + + if ( + type(module) + in [TensorParallelRowLinear, TensorParallelColumnLinear] + and param_name == "weight" + ): + tensor = Int8Params( + tensor, + has_fp16_weights=False, + requires_grad=False, + ).to(device) + state = bnb.MatmulLtState() + state.threshold = 6.0 + state.has_fp16_weights = False + state.memory_efficient_backward = False + state.use_pool = True + state.CB = tensor.CB + state.SCB = tensor.SCB + tensor.CB = None + tensor.SCB = None + + def replace_linear(state): + def linear(input, weight, bias): + out = bnb.matmul( + input, + weight, + state=state, + threshold=state.threshold, + bias=bias, + ) + + if state.CB is not None: + # we converted 8-bit row major to turing/ampere format + # in the first inference pass + # we no longer need the row-major weight + del state.CB + weight.data = state.CxB + + return out + + return linear + + module.linear = replace_linear(state) + + else: + tensor = tensor.to(device) + + module._parameters[param_name] = tensor + if name == "model.decoder.embed_tokens.weight": + model.lm_head._parameters["weight"] = tensor + + def forward( + self, input_ids, attention_mask, position_ids, past_key_values: Optional = None + ): + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + use_cache=True, + ) + + # Logits are sharded, so we need to gather them + logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] + torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) + logits = torch.cat(logits, dim=2) + + return logits, outputs.past_key_values diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 300b376e..5266eb8d 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -38,7 +38,7 @@ def __init__( self.master = self.rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{self.rank}") - dtype = torch.bfloat16 + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index d338fb29..4feec8a1 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -50,7 +50,6 @@ def try_to_load_from_cache( refs_dir = repo_cache / "refs" snapshots_dir = repo_cache / "snapshots" - no_exist_dir = repo_cache / ".no_exist" # Resolve refs (for instance to convert main to the associated commit sha) if refs_dir.is_dir(): @@ -59,10 +58,6 @@ def try_to_load_from_cache( with revision_file.open() as f: revision = f.read() - # Check if file is cached as "no_exist" - if (no_exist_dir / revision / filename).is_file(): - return None - # Check if revision folder exists if not snapshots_dir.exists(): return None From 6f0f1d70f6871b76d4dd7d6a80a983d3307a0725 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 11 Apr 2023 20:32:18 +0200 Subject: [PATCH 152/793] v0.5.0 (#168) --- Cargo.lock | 223 +++++++++++++++++++++----------- docs/openapi.json | 2 +- launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- router/grpc-metadata/Cargo.toml | 2 +- server/pyproject.toml | 2 +- 7 files changed, 150 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 904dd32e..cdf26087 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,9 +88,9 @@ checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "async-stream" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", @@ -99,13 +99,13 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.14", ] [[package]] @@ -116,7 +116,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.14", ] [[package]] @@ -127,9 +127,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.12" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f8ccfd9221ee7d1f3d4b33e1f8319b3a81ed8f61f2ea40b37b859794b4491" +checksum = "6539e4565c365448d483967c6dee3eaecb8e87679a17806a831e82b05b903c18" dependencies = [ "async-trait", "axum-core", @@ -159,9 +159,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", "bytes", @@ -341,7 +341,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.14", ] [[package]] @@ -396,9 +396,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" @@ -420,9 +420,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -614,13 +614,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -653,14 +653,14 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" +checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" dependencies = [ "cfg-if", "libc", "redox_syscall 0.2.16", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -794,7 +794,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.14", ] [[package]] @@ -839,9 +839,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "js-sys", @@ -858,7 +858,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "grpc-metadata" -version = "0.4.1" +version = "0.1.0" dependencies = [ "opentelemetry", "tonic", @@ -1075,13 +1075,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" dependencies = [ "hermit-abi 0.3.1", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1092,14 +1092,14 @@ checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" [[package]] name = "is-terminal" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1161,9 +1161,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.140" +version = "0.2.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" +checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" [[package]] name = "linux-raw-sys" @@ -1364,7 +1364,7 @@ checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.14", ] [[package]] @@ -1496,9 +1496,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.48" +version = "0.10.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" +checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1" dependencies = [ "bitflags", "cfg-if", @@ -1511,13 +1511,13 @@ dependencies = [ [[package]] name = "openssl-macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.14", ] [[package]] @@ -1528,11 +1528,10 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.83" +version = "0.9.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" +checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", @@ -1779,9 +1778,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.54" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e472a104799c74b514a57226160104aa483546de37e839ec50e3c2e41dd87534" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" dependencies = [ "unicode-ident", ] @@ -2066,16 +2065,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.5" +version = "0.37.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e78cc525325c06b4a7ff02db283472f3c042b7ff0c391f96c6d5ac6f4f91b75" +checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2139,22 +2138,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.14", ] [[package]] @@ -2240,9 +2239,9 @@ dependencies = [ [[package]] name = "sketches-ddsketch" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" +checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" [[package]] name = "slab" @@ -2331,9 +2330,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.12" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79d9531f94112cfc3e4c8f5f02cb2b58f72c97b7efd85f70203cc6d8efda5927" +checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5" dependencies = [ "proc-macro2", "quote", @@ -2372,7 +2371,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.4.3" +version = "0.5.0" dependencies = [ "futures", "grpc-metadata", @@ -2389,7 +2388,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.4.3" +version = "0.5.0" dependencies = [ "clap", "ctrlc", @@ -2404,7 +2403,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.4.3" +version = "0.5.0" dependencies = [ "async-stream", "axum", @@ -2450,7 +2449,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.14", ] [[package]] @@ -2567,7 +2566,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.14", ] [[package]] @@ -2918,14 +2917,14 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.14", ] [[package]] name = "utoipa-swagger-ui" -version = "3.1.2" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a6d6921f30f7a9d023a2523cbff2e94a1a4b535abbdda6e6d0c69b61f19f05" +checksum = "062bba5a3568e126ac72049a63254f4cb1da2eb713db0c1ab2a4c76be191db8c" dependencies = [ "axum", "mime_guess", @@ -3111,13 +3110,13 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -3126,7 +3125,16 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", ] [[package]] @@ -3135,13 +3143,28 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -3150,42 +3173,84 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "winreg" version = "0.10.1" diff --git a/docs/openapi.json b/docs/openapi.json index 11fa1e35..377c388a 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -11,7 +11,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.4.3" + "version": "0.5.0" }, "paths": { "/generate": { diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 318052c9..d2c9aa7e 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.4.3" +version = "0.5.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index 074a6053..de8d6f33 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.4.3" +version = "0.5.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 401742dc..e8f82cd3 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.4.3" +version = "0.5.0" edition = "2021" [dependencies] diff --git a/router/grpc-metadata/Cargo.toml b/router/grpc-metadata/Cargo.toml index a6901f77..311092e3 100644 --- a/router/grpc-metadata/Cargo.toml +++ b/router/grpc-metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grpc-metadata" -version = "0.4.1" +version = "0.1.0" edition = "2021" [dependencies] diff --git a/server/pyproject.toml b/server/pyproject.toml index 38f7fb04..41fd63b6 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.4.3" +version = "0.5.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 5fa8ae041cef2b5f5587d4eb076dbaeb5bf992f6 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 12 Apr 2023 12:03:10 +0200 Subject: [PATCH 153/793] feat(server): optimize decode for sane tokenizers (#170) --- benchmark/Cargo.lock | 4 +- server/text_generation_server/models/bloom.py | 8 ++- .../models/causal_lm.py | 11 ++-- .../models/flash_causal_lm.py | 6 +-- .../models/flash_santacoder.py | 3 +- server/text_generation_server/models/model.py | 51 ++++++++++++++----- .../models/santacoder.py | 3 +- .../models/seq2seq_lm.py | 13 +++-- 8 files changed, 67 insertions(+), 32 deletions(-) diff --git a/benchmark/Cargo.lock b/benchmark/Cargo.lock index cb9c7533..74610932 100644 --- a/benchmark/Cargo.lock +++ b/benchmark/Cargo.lock @@ -853,7 +853,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "grpc-metadata" -version = "0.4.1" +version = "0.1.0" dependencies = [ "opentelemetry", "tonic", @@ -2140,7 +2140,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.4.3" +version = "0.5.0" dependencies = [ "futures", "grpc-metadata", diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 1a961027..efcc9e05 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -49,6 +49,11 @@ def from_pb( class BLOOM(CausalLM): + def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + super(BLOOM, self).__init__( + model_id=model_id, revision=revision, quantize=quantize, decode_buffer=1 + ) + @property def batch_type(self) -> Type[CausalLMBatch]: return BloomCausalLMBatch @@ -94,8 +99,7 @@ def __init__( self.model = model.eval().to(dtype) torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( - tokenizer=tokenizer, - device=device, + tokenizer=tokenizer, device=device, decode_buffer=1 ) @staticmethod diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 8c092d6a..6347b1a5 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -291,7 +291,13 @@ def __len__(self): class CausalLM(Model): - def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: bool = False, + decode_buffer: int = 3, + ): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -319,8 +325,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False ) super(CausalLM, self).__init__( - tokenizer=tokenizer, - device=device, + tokenizer=tokenizer, device=device, decode_buffer=decode_buffer ) @property diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 3801ed24..507fec0f 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -212,7 +212,8 @@ def __init__( model_cls: Type[PreTrainedModel], model_id: str, revision: Optional[str] = None, - quantize=False, + quantize: bool = False, + decode_buffer: int = 3, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -237,8 +238,7 @@ def __init__( ) super(FlashCausalLM, self).__init__( - tokenizer=tokenizer, - device=device, + tokenizer=tokenizer, device=device, decode_buffer=decode_buffer ) @property diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 2f680995..e10d259d 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -62,8 +62,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False self.model = model.eval().to(device).to(dtype) super(FlashCausalLM, self).__init__( - tokenizer=tokenizer, - device=device, + tokenizer=tokenizer, device=device, decode_buffer=1 ) @staticmethod diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 5b82872c..08a48553 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -10,10 +10,19 @@ class Model(ABC): - def __init__(self, tokenizer: PreTrainedTokenizerBase, device: torch.device): + def __init__( + self, + tokenizer: PreTrainedTokenizerBase, + device: torch.device, + decode_buffer: int = 3, + ): + if decode_buffer < 1: + raise ValueError("decode_buffer must be >= 1") + self.tokenizer = tokenizer self.all_special_ids = set(tokenizer.all_special_ids) self.device = device + self.decode_buffer = decode_buffer @property @abstractmethod @@ -39,23 +48,37 @@ def decode_token( ) if token_offset is None: - token_offset = len(all_input_ids) - 3 - - # Decode token_offset token minus last one and token_offset tokens - results = self.tokenizer.batch_decode( - [all_input_ids[token_offset:-1], all_input_ids[token_offset:]], - skip_special_tokens=False, - ) + token_offset = len(all_input_ids) - self.decode_buffer + # left token buffer + if self.decode_buffer > 1: + # Decode token_offset token minus last one and token_offset tokens + raw_texts = self.tokenizer.batch_decode( + [all_input_ids[token_offset:-1], all_input_ids[token_offset:]], + skip_special_tokens=False, + ) - # default offset is only the last token - if offset is None: - offset = len(results[0]) + # default offset is only the last token + offset = len(raw_texts[0]) + sequence_text = raw_texts[1] + else: + # Only decode the last token without using a token buffer + sequence_text = self.tokenizer.decode( + all_input_ids[-1], skip_special_tokens=False + ) + # no offset in this case + offset = 0 + else: + assert offset is not None + sequence_text = self.tokenizer.decode( + all_input_ids[token_offset:], + skip_special_tokens=False, + ) # get text - text = results[1][offset:] + token_text = sequence_text[offset:] # if text is utf-8 - if text and text[-1] != "�": - return text, None, None + if token_text and token_text[-1] != "�": + return token_text, None, None else: return "", offset, token_offset diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index 58361a8d..8646a4e1 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -54,8 +54,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False ) super(CausalLM, self).__init__( - tokenizer=tokenizer, - device=device, + tokenizer=tokenizer, device=device, decode_buffer=1 ) def decode(self, generated_ids: List[int]) -> str: diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 13eafd62..82b5def0 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -330,7 +330,13 @@ def __len__(self): class Seq2SeqLM(Model): - def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: bool = False, + decode_buffer: int = 3, + ): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 @@ -354,8 +360,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False tokenizer.bos_token_id = self.model.config.decoder_start_token_id super(Seq2SeqLM, self).__init__( - tokenizer=tokenizer, - device=device, + tokenizer=tokenizer, device=device, decode_buffer=decode_buffer ) @property @@ -496,7 +501,7 @@ def generate_token( if stop: # Slice with decoder_input_length to remove padding # Decode all tokens - output_text = self.decode(decoder_input_ids[-new_decoder_input_length:]) + output_text = self.decode(decoder_input_ids[-decoder_input_length:]) # Get seed if isinstance(next_token_chooser.choice, Sampling): From 880a76eed5f058043367d9643be8a498b286bde2 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 12 Apr 2023 17:18:08 +0200 Subject: [PATCH 154/793] feat(server): support sharded santacoder (#167) --- .../text_generation_server/models/__init__.py | 14 +- server/text_generation_server/models/bloom.py | 6 +- .../custom_modeling/flash_llama_modeling.py | 2 +- .../custom_modeling/flash_neox_modeling.py | 7 +- .../flash_santacoder_modeling.py | 211 +++++++++++++++- .../models/flash_neox.py | 14 +- .../models/flash_santacoder.py | 234 +++++++++++++++++- .../models/galactica.py | 6 +- .../text_generation_server/models/gpt_neox.py | 6 +- server/text_generation_server/models/opt.py | 6 +- server/text_generation_server/models/t5.py | 6 +- 11 files changed, 463 insertions(+), 49 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index c04ae117..368060a0 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -18,8 +18,11 @@ try: from text_generation_server.models.flash_neox import FlashNeoX, FlashNeoXSharded - from text_generation_server.models.flash_santacoder import FlashSantacoder from text_generation_server.models.flash_llama import FlashLlama, FlashLlamaSharded + from text_generation_server.models.flash_santacoder import ( + FlashSantacoder, + FlashSantacoderSharded, + ) FLASH_ATTENTION = torch.cuda.is_available() except ImportError: @@ -49,6 +52,7 @@ __all__.append(FlashNeoX) __all__.append(FlashNeoXSharded) __all__.append(FlashSantacoder) + __all__.append(FlashSantacoderSharded) __all__.append(FlashLlama) __all__.append(FlashLlamaSharded) @@ -78,9 +82,13 @@ def get_model( else: return Galactica(model_id, revision, quantize=quantize) - if "santacoder" in model_id: + if "bigcode" in model_id: if sharded: - raise NotImplementedError("sharded is not supported for Santacoder") + if not FLASH_ATTENTION: + raise NotImplementedError( + FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Santacoder") + ) + return FlashSantacoderSharded(model_id, revision=revision) else: santacoder_cls = FlashSantacoder if FLASH_ATTENTION else SantaCoder return santacoder_cls(model_id, revision, quantize) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index efcc9e05..731a9854 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -93,10 +93,11 @@ def __init__( filenames, quantize=quantize, device=device, + dtype=dtype, rank=self.rank, world_size=self.world_size, ) - self.model = model.eval().to(dtype) + self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( tokenizer=tokenizer, device=device, decode_buffer=1 @@ -108,6 +109,7 @@ def load_weights( filenames: List[str], quantize: bool, device: torch.device, + dtype: torch.dtype, rank: int, world_size: int, ): @@ -157,7 +159,7 @@ def load_weights( f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" ) - tensor = tensor.contiguous() + tensor = tensor.contiguous().to(dtype) if quantize: if not HAS_BITS_AND_BYTES: diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 228529cc..508b7746 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -373,7 +373,7 @@ def __init__(self, act, hidden_size, intermediate_size, process_group=None): x, approximate="tanh" if act in ["gelu_fast", "gelu_pytorch_tanh"] - else None, + else "none", ) ) diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 4ff17619..16fd4091 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -376,7 +376,12 @@ def __init__( self.act = ( ACT2FN[act] if "gelu" not in act - else lambda x: torch.nn.functional.gelu(x, approximate="tanh") + else lambda x: torch.nn.functional.gelu( + x, + approximate="tanh" + if act in ["gelu_fast", "gelu_pytorch_tanh"] + else "none", + ) ) if process_group is None: diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 29c4a5c8..8679826b 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -1,6 +1,8 @@ import torch import torch.distributed +import torch.nn.functional as F + from torch import nn from transformers.activations import ACT2FN @@ -65,6 +67,127 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: return torch.matmul(input, self.weight) +class TensorParallelColumnLinear(FastLinear): + def __init__( + self, + in_features, + out_features, + process_group: torch.distributed.ProcessGroup, + bias=True, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_world_size = process_group.size() + assert out_features % self.tp_world_size == 0 + out_features = out_features // self.tp_world_size + + super().__init__( + in_features=in_features, + out_features=out_features, + bias=bias, + device=device, + dtype=dtype, + ) + + +class TensorParallelRowLinear(FastLinear): + def __init__( + self, + in_features, + out_features, + process_group: torch.distributed.ProcessGroup, + reduce=True, + bias=True, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_world_size = process_group.size() + self.reduce = reduce + assert in_features % self.tp_world_size == 0 + in_features = in_features // self.tp_world_size + + super().__init__( + in_features=in_features, + out_features=out_features, + bias=bias, + device=device, + dtype=dtype, + ) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + out = super(TensorParallelRowLinear, self).forward(input) + if self.reduce: + torch.distributed.all_reduce(out, group=self.process_group) + + return out + + +class TensorParallelEmbedding(nn.Embedding): + def __init__( + self, + num_embeddings, + embedding_dim, + process_group: torch.distributed.ProcessGroup, + reduce=True, + padding_idx=None, + max_norm=None, + norm_type=2.0, + scale_grad_by_freq=False, + sparse=False, + _weight=None, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + self.reduce = reduce + + self.original_num_embeddings = num_embeddings + + assert num_embeddings % self.tp_world_size == 0 + block_size = num_embeddings // self.tp_world_size + # inputs in `[min_id, max_id[` are handled by `self` to get embeddings + self.min_id = self.tp_rank * block_size + self.max_id = (self.tp_rank + 1) * block_size + + # Additional entry that will map to zero + # Used for masking + self.null_idx = block_size + + super().__init__( + block_size, + embedding_dim, + padding_idx=padding_idx, + max_norm=max_norm, + norm_type=norm_type, + scale_grad_by_freq=scale_grad_by_freq, + sparse=sparse, + _weight=_weight, + device=device, + dtype=dtype, + ) + + def add_null_idx(self): + """Additional 0 entry used for masking""" + self.weight = nn.Parameter(F.pad(self.weight, (0, 0, 0, 1))) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + # default all out of bounds values to `self.null_idx` that will then be mapped to 0 + # translate for [0, self.max_id - self.min_id[ + input = torch.where( + (self.min_id > input) | (input >= self.max_id), + self.null_idx, + input - self.min_id, + ) + out = super().forward(input) + if self.reduce: + torch.distributed.all_reduce(out, group=self.process_group) + return out + + class FlashMQAttention(torch.nn.Module): def __init__( self, @@ -80,10 +203,16 @@ def __init__( self.softmax_scale = self.head_size ** (-0.5) if process_group is None: - self.attn = FastLinear(hidden_size, hidden_size + 2 * self.head_size) + self.c_attn = FastLinear(hidden_size, hidden_size + 2 * self.head_size) self.c_proj = FastLinear(hidden_size, hidden_size) else: - raise NotImplementedError + self.num_heads = self.num_heads // process_group.size() + self.c_attn = FastLinear(hidden_size, self.head_size * (self.num_heads + 2)) + self.c_proj = TensorParallelRowLinear( + hidden_size, + hidden_size, + process_group=process_group, + ) def forward( self, @@ -94,10 +223,12 @@ def forward( layer_past_present_indices, cu_seqlens_q, ): - qkv = self.attn(hidden_states) + qkv = self.c_attn(hidden_states) # Split query from key_value - query, key_value = qkv.split([self.hidden_size, 2 * self.head_size], dim=1) + query, key_value = qkv.split( + [self.head_size * self.num_heads, 2 * self.head_size], dim=1 + ) # Prepare query and key_value for indexing query = query.view(-1, self.num_heads, self.head_size) @@ -171,7 +302,7 @@ def __init__(self, act, hidden_size, intermediate_size, process_group=None): x, approximate="tanh" if act in ["gelu_fast", "gelu_pytorch_tanh"] - else None, + else "none", ) ) @@ -179,7 +310,16 @@ def __init__(self, act, hidden_size, intermediate_size, process_group=None): self.c_fc = FastLinear(hidden_size, intermediate_size) self.c_proj = FastLinear(intermediate_size, hidden_size) else: - raise NotImplementedError + self.c_fc = TensorParallelColumnLinear( + hidden_size, + intermediate_size, + process_group=process_group, + ) + self.c_proj = TensorParallelRowLinear( + intermediate_size, + hidden_size, + process_group=process_group, + ) def forward(self, hidden_states): hidden_states = self.c_fc(hidden_states) @@ -246,11 +386,30 @@ def __init__(self, config, process_group=None): super().__init__() self.config = config + self.process_group = process_group + self.tp_embeddings = False if process_group is not None: - raise NotImplementedError - - self.wte = nn.Embedding(config.vocab_size, config.hidden_size) - self.wpe = nn.Embedding(config.max_position_embeddings, config.hidden_size) + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + if config.vocab_size % self.tp_world_size == 0: + self.tp_embeddings = True + + if self.tp_embeddings: + self.wte = TensorParallelEmbedding( + config.vocab_size, + config.hidden_size, + reduce=False, + process_group=process_group, + ) + self.wpe = TensorParallelEmbedding( + config.max_position_embeddings, + config.hidden_size, + reduce=False, + process_group=process_group, + ) + else: + self.wte = nn.Embedding(config.vocab_size, config.hidden_size) + self.wpe = nn.Embedding(config.max_position_embeddings, config.hidden_size) self.h = nn.ModuleList( [ @@ -273,9 +432,12 @@ def __init__(self, config, process_group=None): self.num_heads = self.h[0].attn.num_heads def post_load_weights(self): + if self.tp_embeddings: + self.wte.add_null_idx() + self.wpe.add_null_idx() for layer in self.h: layer: Block - layer.attn.attn.transpose_weight() + layer.attn.c_attn.transpose_weight() layer.attn.c_proj.transpose_weight() layer.mlp.c_fc.transpose_weight() layer.mlp.c_proj.transpose_weight() @@ -289,6 +451,8 @@ def forward( past_key_values=None, ): hidden_states = self.wte(input_ids) + self.wpe(position_ids) + if self.tp_embeddings: + torch.distributed.all_reduce(hidden_states, group=self.process_group) # Prefill if past_key_values is None: @@ -335,7 +499,14 @@ def __init__(self, config, process_group=None): self.transformer = FlashSantacoderModel(config, process_group) - self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) + if self.transformer.tp_embeddings: + self.lm_head = FastLinear( + config.hidden_size, + config.vocab_size // process_group.size(), + bias=False, + ) + else: + self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) def post_load_weights(self): self.transformer.post_load_weights() @@ -352,4 +523,18 @@ def forward( hidden_states, present = self.transformer( input_ids, position_ids, cu_seqlens, max_s, past_key_values ) - return self.lm_head(hidden_states), present + logits = self.lm_head(hidden_states) + + if self.transformer.tp_embeddings: + # Logits are sharded, so we need to gather them + world_logits = [ + torch.empty_like(logits) for _ in range(self.transformer.tp_world_size) + ] + torch.distributed.all_gather( + world_logits, logits, group=self.transformer.process_group + ) + world_logits = torch.cat(world_logits, dim=1) + + return world_logits, present + + return logits, present diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index b93d9f70..a8b38465 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -5,7 +5,7 @@ from opentelemetry import trace from safetensors import safe_open from transformers import AutoTokenizer, AutoConfig -from typing import Optional, Tuple, List +from typing import Optional, List from text_generation_server.models import FlashCausalLM from text_generation_server.models.custom_modeling.flash_neox_modeling import ( @@ -63,13 +63,13 @@ def __init__( self.load_weights( model, filenames, - quantize=quantize, device=device, + dtype=dtype, rank=self.rank, world_size=self.world_size, ) model.post_load_weights() - self.model = model.eval().to(dtype) + self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( tokenizer=tokenizer, @@ -80,16 +80,14 @@ def __init__( def load_weights( model, filenames: List[str], - quantize: bool, device: torch.device, + dtype: torch.dtype, rank: int, world_size: int, ): parameters = dict(model.named_parameters()) for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" - ) as f: + with safe_open(file, framework="pt", device=str(device)) as f: for name in f.keys(): module_name, param_name = name.rsplit(".", 1) module = model.get_submodule(module_name) @@ -142,7 +140,7 @@ def load_weights( f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" ) - tensor = tensor.contiguous() + tensor = tensor.contiguous().to(dtype) if current_parameter_tensor is not None: module._parameters[param_name] = tensor diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index e10d259d..39381e92 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -3,15 +3,20 @@ from accelerate import init_empty_weights from opentelemetry import trace +from safetensors import safe_open from pathlib import Path -from transformers import AutoTokenizer, AutoConfig +from transformers import AutoTokenizer, GPT2Config from typing import Optional, List from text_generation_server.models import FlashCausalLM from text_generation_server.models.custom_modeling.flash_santacoder_modeling import ( FlashSantacoderForCausalLM, + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, ) from text_generation_server.utils import ( + initialize_torch_distributed, weight_files, download_weights, weight_hub_files, @@ -36,10 +41,9 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False model_id, revision=revision, padding_side="left", truncation_side="left" ) - config = AutoConfig.from_pretrained( + config = GPT2Config.from_pretrained( model_id, revision=revision, - trust_remote_code=True, # Needed as the config is not part of Transformers ) # We do not use from_pretrained as we modified the model internal module layout @@ -54,12 +58,9 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False model = FlashSantacoderForCausalLM(config) self.load_weights( - model, - filenames, - device, - dtype, + model, filenames, device, dtype, config.architectures[0].startswith("GPT2") ) - self.model = model.eval().to(device).to(dtype) + self.model = model.eval() super(FlashCausalLM, self).__init__( tokenizer=tokenizer, device=device, decode_buffer=1 @@ -71,6 +72,7 @@ def load_weights( filenames: List[Path], device: torch.device, dtype: torch.dtype, + transpose: bool, ): for filename in filenames: state_dict = torch.load(filename, map_location="cpu") @@ -81,9 +83,9 @@ def load_weights( # Fused qkv if "q_attn.weight" in key or "kv_attn.weight" in key: - final_key = layer_name + ".attn.weight" + final_key = layer_name + ".c_attn.weight" elif "q_attn.bias" in key or "kv_attn.bias" in key: - final_key = layer_name + ".attn.bias" + final_key = layer_name + ".c_attn.bias" else: final_key = key @@ -97,18 +99,19 @@ def load_weights( current_parameter_tensor = None if current_parameter_tensor is not None: - if ( + if transpose and ( "c_fc.weight" in key or "c_proj.weight" in key or "q_attn.weight" in key or "kv_attn.weight" in key + or "c_attn.weight" in key ): # Tranpose as we use nn.Linear instead of Conv1D value = value.T if current_parameter_tensor.device == torch.device("meta"): # Init qkv - if "attn.weight" in final_key: + if "c_attn.weight" in final_key: module._parameters[param_name] = value.new_empty( ( model.transformer.head_size @@ -116,7 +119,7 @@ def load_weights( value.shape[1], ) ) - elif "attn.bias" in final_key: + elif "c_attn.bias" in final_key: module._parameters[param_name] = value.new_empty( ( model.transformer.head_size @@ -156,3 +159,208 @@ def decode(self, generated_ids: List[int]) -> str: return self.tokenizer.decode( generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False ) + + +class FlashSantacoderSharded(FlashSantacoder): + def __init__( + self, model_id: str, revision: Optional[str] = None, quantize: bool = False + ): + self.process_group, self.rank, self.world_size = initialize_torch_distributed() + self.master = self.rank == 0 + if torch.cuda.is_available(): + device = torch.device(f"cuda:{self.rank}") + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + else: + raise NotImplementedError("FlashSantacoderSharded is only available on GPU") + + if quantize: + raise NotImplementedError( + "FlashSantacoderSharded does not support quantization" + ) + + tokenizer = AutoTokenizer.from_pretrained( + model_id, revision=revision, padding_side="left", truncation_side="left" + ) + + config = GPT2Config.from_pretrained( + model_id, + revision=revision, + ) + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + + with init_empty_weights(): + model = FlashSantacoderForCausalLM(config, self.process_group) + + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + device=device, + dtype=dtype, + rank=self.rank, + world_size=self.world_size, + transpose=config.architectures[0].startswith("GPT2"), + ) + self.model = model.eval() + torch.distributed.barrier(group=self.process_group) + super(FlashCausalLM, self).__init__( + tokenizer=tokenizer, + device=device, + ) + + @staticmethod + def load_weights( + model, + filenames: List[str], + device: torch.device, + dtype: torch.dtype, + rank: int, + world_size: int, + transpose: bool, + ): + for file in filenames: + with safe_open(file, framework="pt", device=str(device)) as f: + for key in f.keys(): + slice_ = f.get_slice(key) + + layer_name = ".".join(key.split(".")[:4]) + + # Fused qkv + if "q_attn.weight" in key or "kv_attn.weight" in key: + final_key = layer_name + ".c_attn.weight" + elif "q_attn.bias" in key or "kv_attn.bias" in key: + final_key = layer_name + ".c_attn.bias" + else: + final_key = key + + module_name, param_name = final_key.rsplit(".", 1) + module = model.get_submodule(module_name) + + if isinstance(module, TensorParallelColumnLinear): + dim = 1 if transpose and "weight" in param_name else 0 + size = slice_.get_shape()[dim] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = ( + slice_[start:stop] if dim == 0 else slice_[:, start:stop] + ) + elif isinstance(module, TensorParallelRowLinear): + if param_name == "weight": + dim = 0 if transpose else 1 + size = slice_.get_shape()[dim] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = ( + slice_[start:stop] + if dim == 0 + else slice_[:, start:stop] + ) + else: + tensor = slice_[:] + # XXX: Hack for Rowlinear to add the bias only once. + if rank != 0: + tensor = torch.zeros_like(tensor) + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif key == "lm_head.weight" and model.transformer.tp_embeddings: + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + else: + try: + tensor = slice_[:] + except: + tensor = f.get_tensor(key) + + tensor = tensor.contiguous().to(dtype) + + try: + current_parameter_tensor = module._parameters[param_name] + except KeyError: + current_parameter_tensor = None + + if current_parameter_tensor is not None: + if transpose and ( + "c_fc.weight" in key + or "c_proj.weight" in key + or "q_attn.weight" in key + or "kv_attn.weight" in key + or "c_attn.weight" in key + ): + # Tranpose as we use nn.Linear instead of Conv1D + tensor = tensor.T + + if current_parameter_tensor.device == torch.device("meta"): + # Init qkv + if "c_attn.weight" in final_key: + module._parameters[param_name] = tensor.new_empty( + ( + model.transformer.head_size + * (model.transformer.num_heads + 2), + tensor.shape[1], + ) + ) + elif "c_attn.bias" in final_key: + module._parameters[param_name] = tensor.new_empty( + ( + model.transformer.head_size + * (model.transformer.num_heads + 2) + ) + ) + + # Copy to correct slice + if "q_attn" in key: + size = tensor.shape[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = tensor[start:stop] + module._parameters[param_name][: tensor.shape[0]] = tensor + elif "kv_attn.weight" in key: + module._parameters[param_name][ + model.transformer.head_size + * model.transformer.num_heads : + ] = tensor + elif "kv_attn.bias" in key: + module._parameters[param_name][ + model.transformer.head_size + * model.transformer.num_heads : + ] = tensor + elif "c_attn" in key: + # Slice q_tensor by shard + q_tensor = tensor[: -2 * model.transformer.head_size] + block_size = q_tensor.shape[0] // world_size + start = rank * block_size + stop = (rank + 1) * block_size + q_tensor = q_tensor[start:stop] + + module._parameters[param_name][ + : q_tensor.shape[0] + ] = q_tensor + + # Kv tensor is copied for every shard + kv_tensor = tensor[-2 * model.transformer.head_size :] + module._parameters[param_name][ + q_tensor.shape[0] : + ] = kv_tensor + else: + if current_parameter_tensor.shape != tensor.shape: + raise ValueError( + f"Name {key} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" + ) + + module._parameters[param_name] = tensor + else: + module._buffers[param_name] = tensor + torch.cuda.empty_cache() + model.post_load_weights() diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 396cc4f6..dc78aa8b 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -219,10 +219,11 @@ def __init__( filenames, quantize=quantize, device=device, + dtype=dtype, rank=self.rank, world_size=self.world_size, ) - self.model = model.eval().to(dtype) + self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( tokenizer=tokenizer, @@ -235,6 +236,7 @@ def load_weights( filenames: List[str], quantize: bool, device: torch.device, + dtype: torch.dtype, rank: int, world_size: int, ): @@ -285,7 +287,7 @@ def load_weights( f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" ) - tensor = tensor.contiguous() + tensor = tensor.contiguous().to(dtype) if quantize: if not HAS_BITS_AND_BYTES: diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index fb109ed7..489615e1 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -64,10 +64,11 @@ def __init__( filenames, quantize=quantize, device=device, + dtype=dtype, rank=self.rank, world_size=self.world_size, ) - self.model = model.eval().to(dtype) + self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( tokenizer=tokenizer, @@ -80,6 +81,7 @@ def load_weights( filenames: List[str], quantize: bool, device: torch.device, + dtype: torch.dtype, rank: int, world_size: int, ): @@ -140,7 +142,7 @@ def load_weights( f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" ) - tensor = tensor.contiguous() + tensor = tensor.contiguous().to(dtype) if quantize: if not HAS_BITS_AND_BYTES: diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 85f0ac8c..8e5527c0 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -80,10 +80,11 @@ def __init__( filenames, quantize=quantize, device=device, + dtype=dtype, rank=self.rank, world_size=self.world_size, ) - self.model = model.eval().to(dtype) + self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( tokenizer=tokenizer, @@ -96,6 +97,7 @@ def load_weights( filenames: List[str], quantize: bool, device: torch.device, + dtype: torch.dtype, rank: int, world_size: int, ): @@ -146,7 +148,7 @@ def load_weights( f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" ) - tensor = tensor.contiguous() + tensor = tensor.contiguous().to(dtype) if quantize: if not HAS_BITS_AND_BYTES: diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 5266eb8d..b9f77015 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -64,10 +64,11 @@ def __init__( filenames, quantize=quantize, device=device, + dtype=dtype, rank=self.rank, world_size=self.world_size, ) - self.model = model.eval().to(dtype) + self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(Seq2SeqLM, self).__init__( tokenizer=tokenizer, @@ -80,6 +81,7 @@ def load_weights( filenames: List[str], quantize: bool, device: torch.device, + dtype: torch.dtype, rank: int, world_size: int, ): @@ -146,7 +148,7 @@ def load_weights( f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" ) - tensor = tensor.contiguous() + tensor = tensor.contiguous().to(dtype) if quantize: if not HAS_BITS_AND_BYTES: From e3a63b6fbce709bf609ea406ca5f0c889e4a2c82 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Apr 2023 11:07:11 +0200 Subject: [PATCH 155/793] fix(launcher): revert change on shard errors (#173) --- launcher/src/main.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 2b152be6..a598a8bb 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -349,8 +349,8 @@ fn main() -> ExitCode { Err(TryRecvError::Empty) => { sleep(Duration::from_millis(100)); } - Ok(ShardStatus::Failed(rank)) => { - tracing::error!("Shard {} failed to start.", rank); + Ok(ShardStatus::Failed((rank, err))) => { + tracing::error!("Shard {} failed to start:\n{}", rank, err); shutdown_shards(shutdown, &shutdown_receiver); return ExitCode::FAILURE; } @@ -457,8 +457,8 @@ fn main() -> ExitCode { let mut exit_code = ExitCode::SUCCESS; while running.load(Ordering::SeqCst) { - if let Ok(ShardStatus::Failed(rank)) = status_receiver.try_recv() { - tracing::error!("Shard {rank} failed."); + if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { + tracing::error!("Shard {rank} failed:\n{err}"); exit_code = ExitCode::FAILURE; break; }; @@ -488,7 +488,7 @@ fn main() -> ExitCode { #[derive(Debug)] enum ShardStatus { Ready, - Failed(usize), + Failed((usize, String)), } #[allow(clippy::too_many_arguments)] @@ -627,7 +627,9 @@ fn shard_manager( tracing::error!("Please install it with `make install-server`") } } - status_sender.send(ShardStatus::Failed(rank)).unwrap(); + status_sender + .send(ShardStatus::Failed((rank, err.to_string()))) + .unwrap(); return; } }; @@ -656,7 +658,11 @@ fn shard_manager( loop { // Process exited if p.poll().is_some() { - status_sender.send(ShardStatus::Failed(rank)).unwrap(); + let mut err = String::new(); + p.stderr.take().unwrap().read_to_string(&mut err).unwrap(); + status_sender + .send(ShardStatus::Failed((rank, err))) + .unwrap(); return; } From 64347b05ff8b021c59c3d8417e67aafbe0a48c39 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Apr 2023 12:43:05 +0200 Subject: [PATCH 156/793] fix(ci): fix CVE in github-slug-action (#174) --- .github/workflows/build.yaml | 4 +- Cargo.lock | 26 +- router/Cargo.toml | 4 +- server/poetry.lock | 990 +++++++++++++++++++++++------------ server/pyproject.toml | 2 +- 5 files changed, 681 insertions(+), 345 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a94a33dc..89eaf47e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,7 +39,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 + uses: rlespinasse/github-slug-action@v4.4.1 - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: @@ -101,7 +101,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 + uses: rlespinasse/github-slug-action@v4.4.1 - name: Login to internal Container Registry uses: docker/login-action@v2.1.0 with: diff --git a/Cargo.lock b/Cargo.lock index cdf26087..bc2529d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,16 +176,16 @@ dependencies = [ [[package]] name = "axum-tracing-opentelemetry" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a43f5b506fc945900d08e541de14b5b9c82860637702ae082888b0fc2654d86" +checksum = "164b95427e83b79583c7699a72b4a6b485a12bbdef5b5c054ee5ff2296d82f52" dependencies = [ "axum", "futures", "http", "opentelemetry", "tower", - "tower-http", + "tower-http 0.3.5", "tracing", "tracing-opentelemetry", ] @@ -2424,7 +2424,7 @@ dependencies = [ "thiserror", "tokenizers", "tokio", - "tower-http", + "tower-http 0.4.0", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -2688,6 +2688,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.2" diff --git a/router/Cargo.toml b/router/Cargo.toml index de8d6f33..e2a6f5e9 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -15,7 +15,7 @@ path = "src/main.rs" [dependencies] async-stream = "0.3.3" axum = { version = "0.6.4", features = ["json"] } -axum-tracing-opentelemetry = "0.9.0" +axum-tracing-opentelemetry = "0.10.0" text-generation-client = { path = "client" } clap = { version = "4.1.4", features = ["derive", "env"] } flume = "0.10.14" @@ -32,7 +32,7 @@ serde_json = "1.0.93" thiserror = "1.0.38" tokenizers = "0.13.3" tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } -tower-http = { version = "0.3.5", features = ["cors"] } +tower-http = { version = "0.4.0", features = ["cors"] } tracing = "0.1.37" tracing-opentelemetry = "0.18.0" tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } diff --git a/server/poetry.lock b/server/poetry.lock index 49b9d39d..1ae18f9b 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -23,22 +23,6 @@ test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] test_trackers = ["comet-ml", "tensorboard", "wandb"] testing = ["datasets", "deepspeed (<0.7.0)", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] -[[package]] -name = "attrs" -version = "22.2.0" -description = "Classes Without Boilerplate" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] -tests = ["attrs[tests-no-zope]", "zope.interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests_no_zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] - [[package]] name = "backoff" version = "2.2.1" @@ -65,11 +49,11 @@ python-versions = ">=3.6" [[package]] name = "charset-normalizer" -version = "3.0.1" +version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = "*" +python-versions = ">=3.7.0" [[package]] name = "click" @@ -82,6 +66,17 @@ python-versions = ">=3.7" [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "cmake" +version = "3.26.3" +description = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +test = ["codecov (>=2.0.5)", "coverage (>=4.2)", "flake8 (>=3.0.4)", "path.py (>=11.5.0)", "pytest (>=3.0.3)", "pytest-cov (>=2.4.0)", "pytest-runner (>=2.9)", "pytest-virtualenv (>=1.7.0)", "scikit-build (>=0.10.0)", "setuptools (>=28.0.0)", "virtualenv (>=15.0.3)", "wheel"] + [[package]] name = "colorama" version = "0.4.6" @@ -106,7 +101,7 @@ dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version [[package]] name = "exceptiongroup" -version = "1.1.0" +version = "1.1.1" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false @@ -115,9 +110,21 @@ python-versions = ">=3.7" [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "filelock" +version = "3.11.0" +description = "A platform independent file lock." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] + [[package]] name = "googleapis-common-protos" -version = "1.58.0" +version = "1.59.0" description = "Common protobufs used in Google APIs" category = "main" optional = false @@ -145,30 +152,30 @@ testing = ["protobuf (>=3.6.0)"] [[package]] name = "grpcio" -version = "1.51.3" +version = "1.53.0" description = "HTTP/2-based RPC framework" category = "main" optional = false python-versions = ">=3.7" [package.extras] -protobuf = ["grpcio-tools (>=1.51.3)"] +protobuf = ["grpcio-tools (>=1.53.0)"] [[package]] name = "grpcio-reflection" -version = "1.51.3" +version = "1.53.0" description = "Standard Protobuf Reflection Service for gRPC" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -grpcio = ">=1.51.3" +grpcio = ">=1.53.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.51.3" +version = "1.53.0" description = "Status proto mapping for gRPC" category = "main" optional = false @@ -176,25 +183,25 @@ python-versions = ">=3.6" [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.51.3" +grpcio = ">=1.53.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.51.3" +version = "1.53.0" description = "Protobuf code generator for gRPC" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -grpcio = ">=1.51.3" +grpcio = ">=1.53.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" [[package]] name = "hf-transfer" -version = "0.1.2" +version = "0.1.3" description = "" category = "main" optional = false @@ -216,6 +223,28 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "Jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "lit" +version = "16.0.1" +description = "A Software Testing Tool" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "loguru" version = "0.6.0" @@ -231,6 +260,43 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] +[[package]] +name = "MarkupSafe" +version = "2.1.2" +description = "Safely add untrusted strings to HTML/XML markup." +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4)"] +tests = ["pytest (>=4.6)"] + +[[package]] +name = "networkx" +version = "3.1" +description = "Python package for creating and manipulating graphs and networks" +category = "main" +optional = false +python-versions = ">=3.8" + +[package.extras] +default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] +developer = ["mypy (>=1.1)", "pre-commit (>=3.2)"] +doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.13)", "sphinx (>=6.1)", "sphinx-gallery (>=0.12)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] +test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] + [[package]] name = "numpy" version = "1.24.2" @@ -251,6 +317,18 @@ python-versions = ">=3" setuptools = "*" wheel = "*" +[[package]] +name = "nvidia-cuda-cupti-cu11" +version = "11.7.101" +description = "CUDA profiling tools runtime libs." +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + [[package]] name = "nvidia-cuda-nvrtc-cu11" version = "11.7.99" @@ -287,6 +365,70 @@ python-versions = ">=3" setuptools = "*" wheel = "*" +[[package]] +name = "nvidia-cufft-cu11" +version = "10.9.0.58" +description = "CUFFT native runtime libraries" +category = "main" +optional = false +python-versions = ">=3" + +[[package]] +name = "nvidia-curand-cu11" +version = "10.2.10.91" +description = "CURAND native runtime libraries" +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + +[[package]] +name = "nvidia-cusolver-cu11" +version = "11.4.0.1" +description = "CUDA solver native runtime libraries" +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + +[[package]] +name = "nvidia-cusparse-cu11" +version = "11.7.4.91" +description = "CUSPARSE native runtime libraries" +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + +[[package]] +name = "nvidia-nccl-cu11" +version = "2.14.3" +description = "NVIDIA Collective Communication Library (NCCL) Runtime" +category = "main" +optional = false +python-versions = ">=3" + +[[package]] +name = "nvidia-nvtx-cu11" +version = "11.7.91" +description = "NVIDIA Tools Extension" +category = "main" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + [[package]] name = "opentelemetry-api" version = "1.15.0" @@ -416,7 +558,7 @@ python-versions = ">=3.7" [[package]] name = "packaging" -version = "23.0" +version = "23.1" description = "Core utilities for Python packages" category = "main" optional = false @@ -436,7 +578,7 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.22.0" +version = "4.22.3" description = "" category = "main" optional = false @@ -455,14 +597,13 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "pytest" -version = "7.2.1" +version = "7.3.0" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" @@ -471,7 +612,7 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "PyYAML" @@ -519,7 +660,7 @@ torch = ["torch"] [[package]] name = "sentencepiece" -version = "0.1.97" +version = "0.1.98" description = "SentencePiece python wrapper" category = "main" optional = false @@ -527,7 +668,7 @@ python-versions = "*" [[package]] name = "setuptools" -version = "67.4.0" +version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false @@ -538,6 +679,17 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "sympy" +version = "1.11.1" +description = "Computer algebra system (CAS) in Python" +category = "main" +optional = false +python-versions = ">=3.8" + +[package.dependencies] +mpmath = ">=0.19" + [[package]] name = "tokenizers" version = "0.13.3" @@ -561,22 +713,52 @@ python-versions = ">=3.7" [[package]] name = "torch" -version = "1.13.1" +version = "2.0.0" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" category = "main" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" [package.dependencies] -nvidia-cublas-cu11 = {version = "11.10.3.66", markers = "platform_system == \"Linux\""} -nvidia-cuda-nvrtc-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""} -nvidia-cuda-runtime-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""} -nvidia-cudnn-cu11 = {version = "8.5.0.96", markers = "platform_system == \"Linux\""} +filelock = "*" +jinja2 = "*" +networkx = "*" +nvidia-cublas-cu11 = {version = "11.10.3.66", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-cupti-cu11 = {version = "11.7.101", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-nvrtc-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-runtime-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cudnn-cu11 = {version = "8.5.0.96", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufft-cu11 = {version = "10.9.0.58", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-curand-cu11 = {version = "10.2.10.91", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusolver-cu11 = {version = "11.4.0.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparse-cu11 = {version = "11.7.4.91", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nccl-cu11 = {version = "2.14.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvtx-cu11 = {version = "11.7.91", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +sympy = "*" +triton = {version = "2.0.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} typing-extensions = "*" [package.extras] opt-einsum = ["opt-einsum (>=3.3)"] +[[package]] +name = "triton" +version = "2.0.0" +description = "A language and compiler for custom Deep Learning operations" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +cmake = "*" +filelock = "*" +lit = "*" +torch = "*" + +[package.extras] +tests = ["autopep8", "flake8", "isort", "numpy", "pytest", "scipy (>=1.7.1)"] +tutorials = ["matplotlib", "pandas", "tabulate"] + [[package]] name = "typer" version = "0.6.1" @@ -604,7 +786,7 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.14" +version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -617,14 +799,14 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "wheel" -version = "0.38.4" +version = "0.40.0" description = "A built-package format for Python" category = "main" optional = false python-versions = ">=3.7" [package.extras] -test = ["pytest (>=3.0.0)"] +test = ["pytest (>=6.0.0)"] [[package]] name = "win32-setctime" @@ -651,17 +833,13 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "1c57379c7b9349d2a860b50b3ab125737a0f6f94f4303d7cb55248cb86ff8b8e" +content-hash = "6141d488429e0ab579028036e8e4cbc54f583b48214cb4a6be066bb7ce5154db" [metadata.files] accelerate = [ {file = "accelerate-0.15.0-py3-none-any.whl", hash = "sha256:014833307424cd0a22f89815802e00653756257c45dfdba2453e52d428931c65"}, {file = "accelerate-0.15.0.tar.gz", hash = "sha256:438e25a01afa6e3ffbd25353e76a68be49677c3050f10bfac7beafaf53503efc"}, ] -attrs = [ - {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, - {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, -] backoff = [ {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, @@ -675,99 +853,105 @@ certifi = [ {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, ] charset-normalizer = [ - {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, - {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, ] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] +cmake = [ + {file = "cmake-3.26.3-py2.py3-none-macosx_10_10_universal2.macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:9d38ea5b4999f8f042a071bea3e213f085bac26d7ab54cb5a4c6a193c4baf132"}, + {file = "cmake-3.26.3-py2.py3-none-manylinux2010_i686.manylinux_2_12_i686.whl", hash = "sha256:6e5fcd1cfaac33d015e2709e0dd1b7ad352a315367012ac359c9adc062cf075b"}, + {file = "cmake-3.26.3-py2.py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:4d3185738a6405aa15801e684f8d589b00570da4cc676cb1b5bbc902e3023e53"}, + {file = "cmake-3.26.3-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b20f7f7ea316ce7bb158df0e3c3453cfab5048939f1291017d16a8a36ad33ae6"}, + {file = "cmake-3.26.3-py2.py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:46aa385e19c9e4fc95d7d6ce5ee0bbe0d69bdeac4e9bc95c61f78f3973c2f626"}, + {file = "cmake-3.26.3-py2.py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:71e1df5587ad860b9829211380c42fc90ef2413363f12805b1fa2d87769bf876"}, + {file = "cmake-3.26.3-py2.py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:543b6958d1615327f484a07ab041029b1740918a8baa336adc9f5f0cbcd8fbd8"}, + {file = "cmake-3.26.3-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1bc7b47456256bdcc41069f5c658f232bd6e15bf4796d115f6ec98800793daff"}, + {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:2ae3db2c2be50fdaf0c9f3a23b2206e9dcd55ca124f16486a841b939f50b595e"}, + {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_i686.whl", hash = "sha256:1798547b23b89030518c5668dc55aed0e1d01867cf91d7a94e15d33f62a56fd0"}, + {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:d3017a08e6ba53ec2486d89a7953a81d4c4a068fc9f29d83e209f295dd9c59f3"}, + {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_s390x.whl", hash = "sha256:a922a6f6c1580d0db17b0b75f82e619441dd43c7f1d6a35f7d27e709db48bdbb"}, + {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:e0ed796530641c8a21a423f9bb7882117dbbeee11ec78dbc335402a678d937ae"}, + {file = "cmake-3.26.3-py2.py3-none-win32.whl", hash = "sha256:27a6fa1b97744311a7993d6a1e0ce14bd73696dab9ceb96701f1ec11edbd5053"}, + {file = "cmake-3.26.3-py2.py3-none-win_amd64.whl", hash = "sha256:cf910bbb488659d300c86b1dac77e44eeb0457bde2cf76a42d7e51f691544b21"}, + {file = "cmake-3.26.3-py2.py3-none-win_arm64.whl", hash = "sha256:24741a304ada699b339034958777d9a1472ac8ddb9b6194d74f814287ca091ae"}, + {file = "cmake-3.26.3.tar.gz", hash = "sha256:b54cde1f1c0573321b22382bd2ffaf5d08f65188572d128cd4867fb9669723c5"}, +] colorama = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -777,139 +961,143 @@ Deprecated = [ {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, ] exceptiongroup = [ - {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, - {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, +] +filelock = [ + {file = "filelock-3.11.0-py3-none-any.whl", hash = "sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318"}, + {file = "filelock-3.11.0.tar.gz", hash = "sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37"}, ] googleapis-common-protos = [ - {file = "googleapis-common-protos-1.58.0.tar.gz", hash = "sha256:c727251ec025947d545184ba17e3578840fc3a24a0516a020479edab660457df"}, - {file = "googleapis_common_protos-1.58.0-py2.py3-none-any.whl", hash = "sha256:ca3befcd4580dab6ad49356b46bf165bb68ff4b32389f028f1abd7c10ab9519a"}, + {file = "googleapis-common-protos-1.59.0.tar.gz", hash = "sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44"}, + {file = "googleapis_common_protos-1.59.0-py2.py3-none-any.whl", hash = "sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f"}, ] grpc-interceptor = [ {file = "grpc-interceptor-0.15.0.tar.gz", hash = "sha256:5c1aa9680b1d7e12259960c38057b121826860b05ebbc1001c74343b7ad1455e"}, {file = "grpc_interceptor-0.15.0-py3-none-any.whl", hash = "sha256:63e390162e64df96c39c40508eb697def76a7cafac32a7eaf9272093eec1109e"}, ] grpcio = [ - {file = "grpcio-1.51.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:f601aaeae18dab81930fb8d4f916b0da21e89bb4b5f7367ef793f46b4a76b7b0"}, - {file = "grpcio-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:eef0450a4b5ed11feab639bf3eb1b6e23d0efa9b911bf7b06fb60e14f5f8a585"}, - {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:82b0ad8ac825d4bb31bff9f638557c045f4a6d824d84b21e893968286f88246b"}, - {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3667c06e37d6cd461afdd51cefe6537702f3d1dc5ff4cac07e88d8b4795dc16f"}, - {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3709048fe0aa23dda09b3e69849a12055790171dab9e399a72ea8f9dfbf9ac80"}, - {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:200d69857f9910f7458b39b9bcf83ee4a180591b40146ba9e49314e3a7419313"}, - {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cd9a5e68e79c5f031500e67793048a90209711e0854a9ddee8a3ce51728de4e5"}, - {file = "grpcio-1.51.3-cp310-cp310-win32.whl", hash = "sha256:6604f614016127ae10969176bbf12eb0e03d2fb3d643f050b3b69e160d144fb4"}, - {file = "grpcio-1.51.3-cp310-cp310-win_amd64.whl", hash = "sha256:e95c7ccd4c5807adef1602005513bf7c7d14e5a41daebcf9d8d30d8bf51b8f81"}, - {file = "grpcio-1.51.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:5e77ee138100f0bb55cbd147840f87ee6241dbd25f09ea7cd8afe7efff323449"}, - {file = "grpcio-1.51.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:68a7514b754e38e8de9075f7bb4dee919919515ec68628c43a894027e40ddec4"}, - {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c1b9f8afa62ff265d86a4747a2990ec5a96e4efce5d5888f245a682d66eca47"}, - {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8de30f0b417744288cec65ec8cf84b8a57995cf7f1e84ccad2704d93f05d0aae"}, - {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b69c7adc7ed60da1cb1b502853db61f453fc745f940cbcc25eb97c99965d8f41"}, - {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d81528ffe0e973dc840ec73a4132fd18b8203ad129d7410155d951a0a7e4f5d0"}, - {file = "grpcio-1.51.3-cp311-cp311-win32.whl", hash = "sha256:040eb421613b57c696063abde405916dd830203c184c9000fc8c3b3b3c950325"}, - {file = "grpcio-1.51.3-cp311-cp311-win_amd64.whl", hash = "sha256:2a8e17286c4240137d933b8ca506465472248b4ce0fe46f3404459e708b65b68"}, - {file = "grpcio-1.51.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d5cd1389669a847555df54177b911d9ff6f17345b2a6f19388707b7a9f724c88"}, - {file = "grpcio-1.51.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:be1bf35ce82cdbcac14e39d5102d8de4079a1c1a6a06b68e41fcd9ef64f9dd28"}, - {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:5eed34994c095e2bf7194ffac7381c6068b057ef1e69f8f08db77771350a7566"}, - {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f9a7d88082b2a17ae7bd3c2354d13bab0453899e0851733f6afa6918373f476"}, - {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c8abbc5f837111e7bd619612eedc223c290b0903b952ce0c7b00840ea70f14"}, - {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:165b05af77e6aecb4210ae7663e25acf234ba78a7c1c157fa5f2efeb0d6ec53c"}, - {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54e36c2ee304ff15f2bfbdc43d2b56c63331c52d818c364e5b5214e5bc2ad9f6"}, - {file = "grpcio-1.51.3-cp37-cp37m-win32.whl", hash = "sha256:cd0daac21d9ef5e033a5100c1d3aa055bbed28bfcf070b12d8058045c4e821b1"}, - {file = "grpcio-1.51.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2fdd6333ce96435408565a9dbbd446212cd5d62e4d26f6a3c0feb1e3c35f1cc8"}, - {file = "grpcio-1.51.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:54b0c29bdd9a3b1e1b61443ab152f060fc719f1c083127ab08d03fac5efd51be"}, - {file = "grpcio-1.51.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:ffaaf7e93fcb437356b5a4b23bf36e8a3d0221399ff77fd057e4bc77776a24be"}, - {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:eafbe7501a3268d05f2e450e1ddaffb950d842a8620c13ec328b501d25d2e2c3"}, - {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881ecb34feabf31c6b3b9bbbddd1a5b57e69f805041e5a2c6c562a28574f71c4"}, - {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e860a3222139b41d430939bbec2ec9c3f6c740938bf7a04471a9a8caaa965a2e"}, - {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:49ede0528e9dac7e8a9fe30b16c73b630ddd9a576bf4b675eb6b0c53ee5ca00f"}, - {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6972b009638b40a448d10e1bc18e2223143b8a7aa20d7def0d78dd4af4126d12"}, - {file = "grpcio-1.51.3-cp38-cp38-win32.whl", hash = "sha256:5694448256e3cdfe5bd358f1574a3f2f51afa20cc834713c4b9788d60b7cc646"}, - {file = "grpcio-1.51.3-cp38-cp38-win_amd64.whl", hash = "sha256:3ea4341efe603b049e8c9a5f13c696ca37fcdf8a23ca35f650428ad3606381d9"}, - {file = "grpcio-1.51.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:6c677581ce129f5fa228b8f418cee10bd28dd449f3a544ea73c8ba590ee49d0b"}, - {file = "grpcio-1.51.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:30e09b5e0531685e176f49679b6a3b190762cc225f4565e55a899f5e14b3aa62"}, - {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:c831f31336e81243f85b6daff3e5e8a123302ce0ea1f2726ad752fd7a59f3aee"}, - {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2cd2e4cefb724cab1ba2df4b7535a9980531b9ec51b4dbb5f137a1f3a3754ef0"}, - {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7a0d0bf44438869d307f85a54f25a896ad6b4b0ca12370f76892ad732928d87"}, - {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c02abd55409bfb293371554adf6a4401197ec2133dd97727c01180889014ba4d"}, - {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f8ff75e61e1227ba7a3f16b2eadbcc11d0a54096d52ab75a6b88cfbe56f55d1"}, - {file = "grpcio-1.51.3-cp39-cp39-win32.whl", hash = "sha256:6c99a73a6260bdf844b2e5ddad02dcd530310f80e1fa72c300fa19c1c7496962"}, - {file = "grpcio-1.51.3-cp39-cp39-win_amd64.whl", hash = "sha256:22bdfac4f7f27acdd4da359b5e7e1973dc74bf1ed406729b07d0759fde2f064b"}, - {file = "grpcio-1.51.3.tar.gz", hash = "sha256:be7b2265b7527bb12109a7727581e274170766d5b3c9258d4e466f4872522d7a"}, + {file = "grpcio-1.53.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:752d2949b40e12e6ad3ed8cc552a65b54d226504f6b1fb67cab2ccee502cc06f"}, + {file = "grpcio-1.53.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:8a48fd3a7222be226bb86b7b413ad248f17f3101a524018cdc4562eeae1eb2a3"}, + {file = "grpcio-1.53.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f3e837d29f0e1b9d6e7b29d569e2e9b0da61889e41879832ea15569c251c303a"}, + {file = "grpcio-1.53.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aef7d30242409c3aa5839b501e877e453a2c8d3759ca8230dd5a21cda029f046"}, + {file = "grpcio-1.53.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6f90698b5d1c5dd7b3236cd1fa959d7b80e17923f918d5be020b65f1c78b173"}, + {file = "grpcio-1.53.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a96c3c7f564b263c5d7c0e49a337166c8611e89c4c919f66dba7b9a84abad137"}, + {file = "grpcio-1.53.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ee81349411648d1abc94095c68cd25e3c2812e4e0367f9a9355be1e804a5135c"}, + {file = "grpcio-1.53.0-cp310-cp310-win32.whl", hash = "sha256:fdc6191587de410a184550d4143e2b24a14df495c86ca15e59508710681690ac"}, + {file = "grpcio-1.53.0-cp310-cp310-win_amd64.whl", hash = "sha256:658ffe1e39171be00490db5bd3b966f79634ac4215a1eb9a85c6cd6783bf7f6e"}, + {file = "grpcio-1.53.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:1b172e6d497191940c4b8d75b53de82dc252e15b61de2951d577ec5b43316b29"}, + {file = "grpcio-1.53.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:82434ba3a5935e47908bc861ce1ebc43c2edfc1001d235d6e31e5d3ed55815f7"}, + {file = "grpcio-1.53.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:1c734a2d4843e4e14ececf5600c3c4750990ec319e1299db7e4f0d02c25c1467"}, + {file = "grpcio-1.53.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6a2ead3de3b2d53119d473aa2f224030257ef33af1e4ddabd4afee1dea5f04c"}, + {file = "grpcio-1.53.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a34d6e905f071f9b945cabbcc776e2055de1fdb59cd13683d9aa0a8f265b5bf9"}, + {file = "grpcio-1.53.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eaf8e3b97caaf9415227a3c6ca5aa8d800fecadd526538d2bf8f11af783f1550"}, + {file = "grpcio-1.53.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:da95778d37be8e4e9afca771a83424f892296f5dfb2a100eda2571a1d8bbc0dc"}, + {file = "grpcio-1.53.0-cp311-cp311-win32.whl", hash = "sha256:e4f513d63df6336fd84b74b701f17d1bb3b64e9d78a6ed5b5e8a198bbbe8bbfa"}, + {file = "grpcio-1.53.0-cp311-cp311-win_amd64.whl", hash = "sha256:ddb2511fbbb440ed9e5c9a4b9b870f2ed649b7715859fd6f2ebc585ee85c0364"}, + {file = "grpcio-1.53.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:2a912397eb8d23c177d6d64e3c8bc46b8a1c7680b090d9f13a640b104aaec77c"}, + {file = "grpcio-1.53.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:55930c56b8f5b347d6c8c609cc341949a97e176c90f5cbb01d148d778f3bbd23"}, + {file = "grpcio-1.53.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6601d812105583948ab9c6e403a7e2dba6e387cc678c010e74f2d6d589d1d1b3"}, + {file = "grpcio-1.53.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c705e0c21acb0e8478a00e7e773ad0ecdb34bd0e4adc282d3d2f51ba3961aac7"}, + {file = "grpcio-1.53.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba074af9ca268ad7b05d3fc2b920b5fb3c083da94ab63637aaf67f4f71ecb755"}, + {file = "grpcio-1.53.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:14817de09317dd7d3fbc8272864288320739973ef0f4b56bf2c0032349da8cdf"}, + {file = "grpcio-1.53.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c7ad9fbedb93f331c2e9054e202e95cf825b885811f1bcbbdfdc301e451442db"}, + {file = "grpcio-1.53.0-cp37-cp37m-win_amd64.whl", hash = "sha256:dad5b302a4c21c604d88a5d441973f320134e6ff6a84ecef9c1139e5ffd466f6"}, + {file = "grpcio-1.53.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fa8eaac75d3107e3f5465f2c9e3bbd13db21790c6e45b7de1756eba16b050aca"}, + {file = "grpcio-1.53.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:104a2210edd3776c38448b4f76c2f16e527adafbde171fc72a8a32976c20abc7"}, + {file = "grpcio-1.53.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:dbc1ba968639c1d23476f75c356e549e7bbf2d8d6688717dcab5290e88e8482b"}, + {file = "grpcio-1.53.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95952d3fe795b06af29bb8ec7bbf3342cdd867fc17b77cc25e6733d23fa6c519"}, + {file = "grpcio-1.53.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f144a790f14c51b8a8e591eb5af40507ffee45ea6b818c2482f0457fec2e1a2e"}, + {file = "grpcio-1.53.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0698c094688a2dd4c7c2f2c0e3e142cac439a64d1cef6904c97f6cde38ba422f"}, + {file = "grpcio-1.53.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6b6d60b0958be711bab047e9f4df5dbbc40367955f8651232bfdcdd21450b9ab"}, + {file = "grpcio-1.53.0-cp38-cp38-win32.whl", hash = "sha256:1948539ce78805d4e6256ab0e048ec793956d54787dc9d6777df71c1d19c7f81"}, + {file = "grpcio-1.53.0-cp38-cp38-win_amd64.whl", hash = "sha256:df9ba1183b3f649210788cf80c239041dddcb375d6142d8bccafcfdf549522cd"}, + {file = "grpcio-1.53.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:19caa5b7282a89b799e63776ff602bb39604f7ca98db6df27e2de06756ae86c3"}, + {file = "grpcio-1.53.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:b5bd026ac928c96cc23149e6ef79183125542062eb6d1ccec34c0a37e02255e7"}, + {file = "grpcio-1.53.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:7dc8584ca6c015ad82e186e82f4c0fe977394588f66b8ecfc4ec873285314619"}, + {file = "grpcio-1.53.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2eddaae8af625e45b5c8500dcca1043264d751a6872cde2eda5022df8a336959"}, + {file = "grpcio-1.53.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5fb6f3d7824696c1c9f2ad36ddb080ba5a86f2d929ef712d511b4d9972d3d27"}, + {file = "grpcio-1.53.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8270d1dc2c98ab57e6dbf36fa187db8df4c036f04a398e5d5e25b4e01a766d70"}, + {file = "grpcio-1.53.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:976a7f24eb213e8429cab78d5e120500dfcdeb01041f1f5a77b17b9101902615"}, + {file = "grpcio-1.53.0-cp39-cp39-win32.whl", hash = "sha256:9c84a481451e7174f3a764a44150f93b041ab51045aa33d7b5b68b6979114e48"}, + {file = "grpcio-1.53.0-cp39-cp39-win_amd64.whl", hash = "sha256:6beb84f83360ff29a3654f43f251ec11b809dcb5524b698d711550243debd289"}, + {file = "grpcio-1.53.0.tar.gz", hash = "sha256:a4952899b4931a6ba12951f9a141ef3e74ff8a6ec9aa2dc602afa40f63595e33"}, ] grpcio-reflection = [ - {file = "grpcio-reflection-1.51.3.tar.gz", hash = "sha256:5adca16f0a6cd403efa3b5f8f8a493eea6a37dee9473b178fad0a60efa68bc67"}, - {file = "grpcio_reflection-1.51.3-py3-none-any.whl", hash = "sha256:52b037f831908468afc89c60e591d0a2bbce24a393d908c44a6d53091e90fc41"}, + {file = "grpcio-reflection-1.53.0.tar.gz", hash = "sha256:663ea8a9883a64457c1a58b95d52b431f4636337503954e15bfac1d1697b401c"}, + {file = "grpcio_reflection-1.53.0-py3-none-any.whl", hash = "sha256:19ac2e3b16404f20bfa14ec64483bb4a936accee50ccb2e64b0159d1cea767cd"}, ] grpcio-status = [ - {file = "grpcio-status-1.51.3.tar.gz", hash = "sha256:71792c550356ba94e162c70818719ae6d67d960bdd03a9db5ff68faba2927f6c"}, - {file = "grpcio_status-1.51.3-py3-none-any.whl", hash = "sha256:d68d0956c16b6ea466f13c27075f126ef2cd8f0f97527d70056c64b0084357e3"}, + {file = "grpcio-status-1.53.0.tar.gz", hash = "sha256:5a46820dc7d94bac48aae5dd97c94cadea09d80a05553aca74a92a2954030420"}, + {file = "grpcio_status-1.53.0-py3-none-any.whl", hash = "sha256:4f4bc140ed718a343d04a573c838c9fff5cb87f40d3cb2aa33b027c55031489d"}, ] grpcio-tools = [ - {file = "grpcio-tools-1.51.3.tar.gz", hash = "sha256:4fea28e3dd31871579a57058796a78093c75b74b74e9de2f2b7a7fd9a443d403"}, - {file = "grpcio_tools-1.51.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:779ac1ad2258b8debaa45595bfb3814806ed8880e3ea7f194e551d76a6255969"}, - {file = "grpcio_tools-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:83bf605fe2b3591d3c8a78646f37c72c5832c4dd84b5f92405c17cb10b136be6"}, - {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:35f885c5afd8e6a77d320f5a9624b439a93f9be2b87fa7b7948c1ad7b2ba0894"}, - {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:253b639fb79a4d28ce494ae40e5695bf1e2cb4a05f205fc433c46b2049ab4d99"}, - {file = "grpcio_tools-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c6b145587d6062e2335f0b3286501dd6853a1ea50bd466a913351b7c48e5f20"}, - {file = "grpcio_tools-1.51.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:046c0b1e372d4acf552aa0c8f5e830f019d67b75f25aeb0968d15fbdd3eaabd3"}, - {file = "grpcio_tools-1.51.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc90b0287908c46281eb61933acaa1b96a575d0160fc98b5c64b9dec46f60d1"}, - {file = "grpcio_tools-1.51.3-cp310-cp310-win32.whl", hash = "sha256:8e9df40db7a0edd403b539cc142d6114270e35debf723a5b4a7a93d5c30fffc0"}, - {file = "grpcio_tools-1.51.3-cp310-cp310-win_amd64.whl", hash = "sha256:077adaee431c2b040dd77923964577087c32e828908e8fa2e53f8e003ad408c9"}, - {file = "grpcio_tools-1.51.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:b50f9b8a6482a90c1a41e731a879a130f7dea267065d0a06f47c9160ce5d01c3"}, - {file = "grpcio_tools-1.51.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:89a68adcb4238aba69f3a364ac02c9a46e55b9e3fd8af1c6f384079abfa9347c"}, - {file = "grpcio_tools-1.51.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d177da43e7f6fde6715df4a3015ae13158166bc2845ac7f9cfb526eafb41b8"}, - {file = "grpcio_tools-1.51.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:793f9edef82f600a3324f8a3d8cd8318a8d02f28fb54f8236cbb35ce0928d186"}, - {file = "grpcio_tools-1.51.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f7583735542ced7d30baec6cc21bffeaffcec1523bf807e8f8f0047113b6d30a"}, - {file = "grpcio_tools-1.51.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f2df233a3e7db23d9b516cb5e2bfe029465f40a72978bee0584e44e7860ea73f"}, - {file = "grpcio_tools-1.51.3-cp311-cp311-win32.whl", hash = "sha256:7427939455735fbf2ea88c37f1585c9c8b809eec7b447642f34465eb4d26020b"}, - {file = "grpcio_tools-1.51.3-cp311-cp311-win_amd64.whl", hash = "sha256:ba76d15fd149b575170fa32a1f6a9ff2b38ff9db223229a8ad6f53450a452688"}, - {file = "grpcio_tools-1.51.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d2212c682529263b3c9e903092d0ccbb9fc6afba820e4c2fa52c2c27720cdcae"}, - {file = "grpcio_tools-1.51.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:405656b3cf9639427e6c30a795570cba4a7c06b88a3145866f7d2c05b7e048b4"}, - {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:3c445a064b2ef3d3475e26e2add8ddb4ac2933741ecddf71d5b071a3ad078db4"}, - {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7b3374f4a6579c58d16a5fab2e6b4e9bb8625a034a7f4cd6024f4d1cc12f2a0"}, - {file = "grpcio_tools-1.51.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e8df08b65f9379c3f103147b29542b0141ca84e77d0eee9114ca5f9b3f0d23"}, - {file = "grpcio_tools-1.51.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2fade12de08923b350475ca16d0d0bd68578c30fce89147aa0f94ef5759bc5a9"}, - {file = "grpcio_tools-1.51.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d4ffb6325ed489065dbdca764cf37c3a29376bc657874116c9af788d7a0d2ee4"}, - {file = "grpcio_tools-1.51.3-cp37-cp37m-win32.whl", hash = "sha256:f8d17271fc58ed3503dd571c79917e126deca51f85f093770a9606e806aac9dc"}, - {file = "grpcio_tools-1.51.3-cp37-cp37m-win_amd64.whl", hash = "sha256:ef849687c7f2bd7f3277edc7c7cafc7042823d0fb078e3c01c861eb0c96ed181"}, - {file = "grpcio_tools-1.51.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:7fd18d8d211fbfd337fc12e5bdd57e62368f636addf901d290e68a39f1dfea38"}, - {file = "grpcio_tools-1.51.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:233fc56f054424232e2086f444004413e33c699174ce6ee0e279c25227243fec"}, - {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:867fa1973fa8b0772077c15425f122f672a18b1c53709a8a2bff9d056db4c20e"}, - {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b486a99bdf2722e68a9d59769389e2fb86878b6f293be5111f7678e364a0c359"}, - {file = "grpcio_tools-1.51.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8bbf412c357999f88d87f421fd48b4b114fc037fec7bbaed0cb7620c24a5e44"}, - {file = "grpcio_tools-1.51.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1166744c40821bb0aa605d2af2287fac367756f858a3d18f4c3d25bc0b92757b"}, - {file = "grpcio_tools-1.51.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:781896c488e07b9463196045e6725e52d018cd7d0e1062d4ab1eee2647ca9170"}, - {file = "grpcio_tools-1.51.3-cp38-cp38-win32.whl", hash = "sha256:35c1ee7c766eb586f04ba41fa7711eb847767eb277a1737998374ac57768f1f0"}, - {file = "grpcio_tools-1.51.3-cp38-cp38-win_amd64.whl", hash = "sha256:584b201fb39307dcb1affcf2647656a0e6244423ef1659cc6caa3ff85c5ae5c1"}, - {file = "grpcio_tools-1.51.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:e02231e21029f716a1d23a0b5e664fa243d147da33a3f55088a9529b860aa4ac"}, - {file = "grpcio_tools-1.51.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:fbb742e10bd548031b8d80f7c28eb70c7c3a9850f8e99c98cd496f19a05f9fee"}, - {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a836a72c657f751244cdb358c3461a89627e6d02654079d2450cfe361800428c"}, - {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb554408e0ec5ff5201013f268726d9eef8e5bd1fd4b4e09c46c0b4a9de8b64c"}, - {file = "grpcio_tools-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:158c5bfe7e157fd9a944bde9f7dfe3b468416666e4fade77cd17caa3edc8bd81"}, - {file = "grpcio_tools-1.51.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:715c792679224171c0584e9f235b921d76f8990deb38b0d1215d0469301d9cd9"}, - {file = "grpcio_tools-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ece44f42b10e0bceb49235be1e361e1ee69afee7f938c82fb656a601a4a720e3"}, - {file = "grpcio_tools-1.51.3-cp39-cp39-win32.whl", hash = "sha256:980e632710ba05e04364c6f276e905d5d367437f1ce2265ce7b96b5c1eac5693"}, - {file = "grpcio_tools-1.51.3-cp39-cp39-win_amd64.whl", hash = "sha256:5f4c47b14e66f80365cd5667ecc2f7fb0eb91e02c4e54362041b758feaa00511"}, + {file = "grpcio-tools-1.53.0.tar.gz", hash = "sha256:925efff2d63ca3266f93c924ffeba5d496f16a8ccbe125fa0d18acf47cc5fa88"}, + {file = "grpcio_tools-1.53.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:41b859cf943256debba1e7b921e3689c89f95495b65f7ad226c4f0e38edf8ee4"}, + {file = "grpcio_tools-1.53.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:17c557240f7fbe1886dcfb5f3ba79740ecb65fe3b93061e64b8f4dfc6a6a5dc5"}, + {file = "grpcio_tools-1.53.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:6afffd7e97e5bddc63b3ce9abe912b9adb704a36ba86d4406be94426734b97c2"}, + {file = "grpcio_tools-1.53.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f55e2c13620271b7f5a81a489a188d6e34a24da8885d46f1566f0e798cb59e6f"}, + {file = "grpcio_tools-1.53.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bd4c732d8d7a736e787b5d0963d4195267fc856e1d313d4532d1625e19a0e4a"}, + {file = "grpcio_tools-1.53.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:99ecefb6b66e9fe41468a70ee2f05da2eb9c7bf63867fb9ff07f7dd90ea813ae"}, + {file = "grpcio_tools-1.53.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7754d6466191d327a0eef364ad5b863477a8fcc12953adc06b30b8e470c70e4a"}, + {file = "grpcio_tools-1.53.0-cp310-cp310-win32.whl", hash = "sha256:f31c549d793a0e72c044f724b3373141d2aa9970fe97b1c2cfaa7ea44002b9aa"}, + {file = "grpcio_tools-1.53.0-cp310-cp310-win_amd64.whl", hash = "sha256:b4173b95e2c29a5145c806d16945ce1e5b38a11c7eb6ab1a6d74afc0a2ce47d9"}, + {file = "grpcio_tools-1.53.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:613a84ebd1881635370c12503f2b15b37332a53fbac32904c94ac4c0c10f0a2a"}, + {file = "grpcio_tools-1.53.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:af686b83bc6b5c1f1591c9f49183717974047de9546adcf5e09a18781b550c96"}, + {file = "grpcio_tools-1.53.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:3cc832e8297e9437bc2b137fe815c8ba1d9af6ffdd76c5c6d7f911bf8e1b0f45"}, + {file = "grpcio_tools-1.53.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39d0a254de49d852f5fe9f9df0a45b2ae66bc04e2d9ee1d6d2c0ba1e70fac91a"}, + {file = "grpcio_tools-1.53.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7062109553ec1873c5c09cc379b8ae0aa76a2d6d6aae97759b97787b93fa9786"}, + {file = "grpcio_tools-1.53.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7728407b1e89fb1473b86152fc33be00f1a25a5aa3264245521f05cbbef9d817"}, + {file = "grpcio_tools-1.53.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2758ea125442bc81251267fc9c28f65555a571f6a0afda4d71a6e7d669347095"}, + {file = "grpcio_tools-1.53.0-cp311-cp311-win32.whl", hash = "sha256:8940d59fca790f1bd45785d0661c3a8c081231c9f8049d7fbf6c6c00737e43da"}, + {file = "grpcio_tools-1.53.0-cp311-cp311-win_amd64.whl", hash = "sha256:c2cff79be5a06d63e9a6a7e38f8f160ade21517386eabe27afacef65a8531358"}, + {file = "grpcio_tools-1.53.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d646d65fafbf70a57416493e719a0df7ffa0772133266cfe1b2b72e072ae64a2"}, + {file = "grpcio_tools-1.53.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:7da0fc185735050d8240b1d74c4667a02baf1b4fa379a5fc05d1fc067eeba596"}, + {file = "grpcio_tools-1.53.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:2be17265c0f070efd625683cef986e07dbc495103fcc719009ff2f6988003166"}, + {file = "grpcio_tools-1.53.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4701d48f649443f1101a24d85e9d5ac13346ccac7781e243f49491328e172266"}, + {file = "grpcio_tools-1.53.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b54c64d85bea5c3a3d895454878c7d6bed5cbb80dc3cafcd75dc1e78300d8c95"}, + {file = "grpcio_tools-1.53.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7152045190e9bd665d1feaeaef931d82c75cacce2b116ab150befa90855de3d0"}, + {file = "grpcio_tools-1.53.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e18292123c86975d0aa47f1bcb176393640dcc23912e9f3a2247f1eff81ac8e8"}, + {file = "grpcio_tools-1.53.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b1b76b6ab5c24e44b15d6a7df6c1b81c3099a54b82d41a3ce96e73a2e6a5081c"}, + {file = "grpcio_tools-1.53.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:e76e8dfe6fe4e61ce3049e9d56c0d806d0d3edc28aa32117d1b17f387469c52e"}, + {file = "grpcio_tools-1.53.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:4c6acaca09cfcd59850e27bd138df9d01c0686c42a5412aa6a92141c15316b1e"}, + {file = "grpcio_tools-1.53.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:76898c1dadf8630a75a40b5a89ab38e326f1288dcfde3413cdfa7a58e149c987"}, + {file = "grpcio_tools-1.53.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b47f8b1bd3af2fb25548b625ad9c3659da30fe83c06f462f357c754f49b71ae"}, + {file = "grpcio_tools-1.53.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2faad4b6362e7ff3ae43ef2d51dfce0a3bc32cf52469e88568c3f65cae377d5"}, + {file = "grpcio_tools-1.53.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:830261fe08541f0fd2dd5035264df2b91012988f37aa1d80a0b4ee6404dc25ae"}, + {file = "grpcio_tools-1.53.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4be32c694c760f3281555089f7aed7d48ca7ea4094115a08b5fc895e17d7e62e"}, + {file = "grpcio_tools-1.53.0-cp38-cp38-win32.whl", hash = "sha256:4605db5a5828205d7fa33a5de9e00723bd037709e74e15c028b9dcec2339b7bc"}, + {file = "grpcio_tools-1.53.0-cp38-cp38-win_amd64.whl", hash = "sha256:0229e6cd442915192b8f8ee2e7e1c8b9986c878bc4dd8be3539f3be35f1b8282"}, + {file = "grpcio_tools-1.53.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:ad0c20688a650e731e8328a7a08899c433a59bfc995a7afcf715b5ad9eca9e7b"}, + {file = "grpcio_tools-1.53.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:a8c3e30c531969c62a5a219be414277b269c1be9a76bcd6948571868894e19b2"}, + {file = "grpcio_tools-1.53.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:326c67b35be69409a88632e6145032d53b8b8141634e9cbcd27fa8f9015a112c"}, + {file = "grpcio_tools-1.53.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:102b6d323d7cef7ac29683f949ec66885b417c06df6059f6a88d07c5556c2592"}, + {file = "grpcio_tools-1.53.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:861f8634cca3ca5bb5336ba16cc78291dba3e7fcadedff195bfdeb433f2c29f2"}, + {file = "grpcio_tools-1.53.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c9a9e1da1868349eba401e9648eac19132700942c475adcc97b6938bf4bf0182"}, + {file = "grpcio_tools-1.53.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ccf7313e5bee13f2f86d12741489f3ed8c901d6b463dff2604191cd4ff518abb"}, + {file = "grpcio_tools-1.53.0-cp39-cp39-win32.whl", hash = "sha256:65b77532bb8f6ab1bfbdd2ac0788626a6c05b227f4722d3bbc2c54258e49c3e5"}, + {file = "grpcio_tools-1.53.0-cp39-cp39-win_amd64.whl", hash = "sha256:7c0ede22796259e83aa1f108038513e86672b2892d3654f94415e3930b74b871"}, ] hf-transfer = [ - {file = "hf_transfer-0.1.2-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:2b9189a4a460646ee135ee771f39c0f695d3d5bf08b7ff1dcfe374227520e994"}, - {file = "hf_transfer-0.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:654fcaba4e7084caa1e97430982ea968935a72916ee0f4afc60e356f89774099"}, - {file = "hf_transfer-0.1.2-cp310-none-win_amd64.whl", hash = "sha256:eb29e7b3707b5cac02e689c89111685ebcdaa3cebba02eb7ac1b0f076357da72"}, - {file = "hf_transfer-0.1.2-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:0bfca9bd84e925e978a0f157df488704c17a0b9ad240b2859262faba0c74cd40"}, - {file = "hf_transfer-0.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d00c5473b35227b2f113fd43ff13cbac9539f2e6779fa0680a887b0aac31c389"}, - {file = "hf_transfer-0.1.2-cp311-none-win_amd64.whl", hash = "sha256:1aaf5937aa433b7d09ce5bf60967ec22b7d3982957b00516a8dc2aaa66384372"}, - {file = "hf_transfer-0.1.2-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:b0aa760a55995ad59ea17e395babafdc56c4e664be0c2d2055664199dd913da1"}, - {file = "hf_transfer-0.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:889dd15e8472daf66e266eb056e31a485af3c35f95a483bb43489a0f6e44c359"}, - {file = "hf_transfer-0.1.2-cp37-none-win_amd64.whl", hash = "sha256:30df586e18ec8a8e67e3201b9038210d94cb3c03c1cbd97673b9c78ede227178"}, - {file = "hf_transfer-0.1.2-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:cc97eb97f929f96bed896cd3af9bbdf121c15ac6d63524b9fc9312fd2929099a"}, - {file = "hf_transfer-0.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:583c2c80210a60dafed9a81ba50c389878aee6c34b2dd375cd84522658f29ad8"}, - {file = "hf_transfer-0.1.2-cp38-none-win_amd64.whl", hash = "sha256:6dff58f50d1435b0346f31a32f1f9e2301986521c1d0b51e47a3c82b96d02156"}, - {file = "hf_transfer-0.1.2-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d6db1a8f539133f7a893bb32721916fe72b4d2aa3eb7604581ba1f03b8167c90"}, - {file = "hf_transfer-0.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f284e3f775d215c9a8d3d1c6f6b1001b1e7990d73ae5fd9aea6c9bce9ea79285"}, - {file = "hf_transfer-0.1.2-cp39-none-win_amd64.whl", hash = "sha256:8625beabebc582eafc4141a5ecb9f1183b728d4f63767f01fdcf1e2fbafe6d43"}, - {file = "hf_transfer-0.1.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:947dd1b8b22ac10723b2887ed4b5ef929f7d4dd850b0d66c0c6954a9a85afb06"}, - {file = "hf_transfer-0.1.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90a020f41dfae4629186c284888cd5adbebe402e2497a88351416ab93c7df9a8"}, - {file = "hf_transfer-0.1.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5eb89698746a29805bfc60126b9a008e6ba08a82ef9bb122a6544e84f748e8a4"}, - {file = "hf_transfer-0.1.2.tar.gz", hash = "sha256:6bf847f4c19c7d8d9f9bbb8a7ed52e1271bbf0c1bd920357db0c274ccc69f21d"}, + {file = "hf_transfer-0.1.3-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9"}, + {file = "hf_transfer-0.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07"}, + {file = "hf_transfer-0.1.3-cp310-none-win_amd64.whl", hash = "sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6"}, + {file = "hf_transfer-0.1.3-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d53954ecfaadc84c15481bf5d4c7282323196b4b6df1d1be54208d4fdedfb407"}, + {file = "hf_transfer-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334862f4a82f8a09d6d3f550e67d7e498bb8882e678b7725638254fed3276801"}, + {file = "hf_transfer-0.1.3-cp311-none-win_amd64.whl", hash = "sha256:da92a1483a66cf2baa96de133c75e7d5d9a60e4a0e60d228f26c573c73a1feb6"}, + {file = "hf_transfer-0.1.3-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:617692a70cf299576d82cfc860923f29ec5c834a3f5242bc0853d4f106670398"}, + {file = "hf_transfer-0.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca218fb6efc5c22379d9e64989e84bff426fcf5664fdbbf7cd70aa8b79497652"}, + {file = "hf_transfer-0.1.3-cp37-none-win_amd64.whl", hash = "sha256:6e5201b648df6106c232fcdb507db734081fd6220dfb1c432bd27c6fa9453331"}, + {file = "hf_transfer-0.1.3-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:445edfcd9d59c9d2541957177a9c9225b1f0e8855f6311fb16e20f67c3426421"}, + {file = "hf_transfer-0.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c1bdfa554f0b0936c1623b99058c6998a00fdcd86f75d9203f3f66572d2e30c"}, + {file = "hf_transfer-0.1.3-cp38-none-win_amd64.whl", hash = "sha256:606f2fe436e5be73f07987a56cd97c695805413d29203ae39ebd9fc596405435"}, + {file = "hf_transfer-0.1.3-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9913f5ad2644a1f57c1b7755a7d959ca5e0189863bb0473817d0707af230bf6a"}, + {file = "hf_transfer-0.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d935946791e338f748e05a23df877d74fbcd39dc7b537f0aa2e5a5841cf7dde8"}, + {file = "hf_transfer-0.1.3-cp39-none-win_amd64.whl", hash = "sha256:79099ac043423b263a2843a24213418f309d5c8bc458776622bffe012ebced73"}, + {file = "hf_transfer-0.1.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ce6c5965a57d94db5e043aa488a4df929a32000db125d9c9a1d325e8c7006dc"}, + {file = "hf_transfer-0.1.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a7934c8e491bb395731f677f66dd5f6641432f338a3a9efc9f0b6c186d37cf8"}, + {file = "hf_transfer-0.1.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"}, + {file = "hf_transfer-0.1.3.tar.gz", hash = "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c"}, ] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, @@ -919,10 +1107,77 @@ iniconfig = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +Jinja2 = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] +lit = [ + {file = "lit-16.0.1.tar.gz", hash = "sha256:630a47291b714cb115015df23ab04267c24fe59aec7ecd7e637d5c75cdb45c91"}, +] loguru = [ {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, ] +MarkupSafe = [ + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, + {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, +] +mpmath = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] +networkx = [ + {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, + {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, +] numpy = [ {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, @@ -957,6 +1212,10 @@ nvidia-cublas-cu11 = [ {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-win_amd64.whl", hash = "sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e"}, ] +nvidia-cuda-cupti-cu11 = [ + {file = "nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl", hash = "sha256:e0cfd9854e1f2edaa36ca20d21cd0bdd5dcfca4e3b9e130a082e05b33b6c5895"}, + {file = "nvidia_cuda_cupti_cu11-11.7.101-py3-none-win_amd64.whl", hash = "sha256:7cc5b8f91ae5e1389c3c0ad8866b3b016a175e827ea8f162a672990a402ab2b0"}, +] nvidia-cuda-nvrtc-cu11 = [ {file = "nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03"}, {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7"}, @@ -970,6 +1229,30 @@ nvidia-cudnn-cu11 = [ {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, ] +nvidia-cufft-cu11 = [ + {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl", hash = "sha256:222f9da70c80384632fd6035e4c3f16762d64ea7a843829cb278f98b3cb7dd81"}, + {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-win_amd64.whl", hash = "sha256:c4d316f17c745ec9c728e30409612eaf77a8404c3733cdf6c9c1569634d1ca03"}, +] +nvidia-curand-cu11 = [ + {file = "nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:eecb269c970fa599a2660c9232fa46aaccbf90d9170b96c462e13bcb4d129e2c"}, + {file = "nvidia_curand_cu11-10.2.10.91-py3-none-win_amd64.whl", hash = "sha256:f742052af0e1e75523bde18895a9ed016ecf1e5aa0ecddfcc3658fd11a1ff417"}, +] +nvidia-cusolver-cu11 = [ + {file = "nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:72fa7261d755ed55c0074960df5904b65e2326f7adce364cbe4945063c1be412"}, + {file = "nvidia_cusolver_cu11-11.4.0.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:700b781bfefd57d161443aff9ace1878584b93e0b2cfef3d6e9296d96febbf99"}, + {file = "nvidia_cusolver_cu11-11.4.0.1-py3-none-win_amd64.whl", hash = "sha256:00f70b256add65f8c1eb3b6a65308795a93e7740f6df9e273eccbba770d370c4"}, +] +nvidia-cusparse-cu11 = [ + {file = "nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:a3389de714db63321aa11fbec3919271f415ef19fda58aed7f2ede488c32733d"}, + {file = "nvidia_cusparse_cu11-11.7.4.91-py3-none-win_amd64.whl", hash = "sha256:304a01599534f5186a8ed1c3756879282c72c118bc77dd890dc1ff868cad25b9"}, +] +nvidia-nccl-cu11 = [ + {file = "nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl", hash = "sha256:5e5534257d1284b8e825bc3a182c6f06acd6eb405e9f89d49340e98cd8f136eb"}, +] +nvidia-nvtx-cu11 = [ + {file = "nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:b22c64eee426a62fc00952b507d6d29cf62b4c9df7a480fcc417e540e05fd5ac"}, + {file = "nvidia_nvtx_cu11-11.7.91-py3-none-win_amd64.whl", hash = "sha256:dfd7fcb2a91742513027d63a26b757f38dd8b07fecac282c4d132a9d373ff064"}, +] opentelemetry-api = [ {file = "opentelemetry_api-1.15.0-py3-none-any.whl", hash = "sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0"}, {file = "opentelemetry_api-1.15.0.tar.gz", hash = "sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded"}, @@ -1007,27 +1290,27 @@ opentelemetry-semantic-conventions = [ {file = "opentelemetry_semantic_conventions-0.36b0.tar.gz", hash = "sha256:829dc221795467d98b773c04096e29be038d77526dc8d6ac76f546fb6279bf01"}, ] packaging = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] protobuf = [ - {file = "protobuf-4.22.0-cp310-abi3-win32.whl", hash = "sha256:b2fea9dc8e3c0f32c38124790ef16cba2ee0628fe2022a52e435e1117bfef9b1"}, - {file = "protobuf-4.22.0-cp310-abi3-win_amd64.whl", hash = "sha256:a33a273d21852f911b8bda47f39f4383fe7c061eb1814db2c76c9875c89c2491"}, - {file = "protobuf-4.22.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:e894e9ae603e963f0842498c4cd5d39c6a60f0d7e4c103df50ee939564298658"}, - {file = "protobuf-4.22.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:7c535d126e7dcc714105ab20b418c4fedbd28f8b8afc42b7350b1e317bbbcc71"}, - {file = "protobuf-4.22.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:86c3d20428b007537ba6792b475c0853bba7f66b1f60e610d913b77d94b486e4"}, - {file = "protobuf-4.22.0-cp37-cp37m-win32.whl", hash = "sha256:1669cb7524221a8e2d9008d0842453dbefdd0fcdd64d67672f657244867635fb"}, - {file = "protobuf-4.22.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ab4d043865dd04e6b09386981fe8f80b39a1e46139fb4a3c206229d6b9f36ff6"}, - {file = "protobuf-4.22.0-cp38-cp38-win32.whl", hash = "sha256:29288813aacaa302afa2381db1d6e0482165737b0afdf2811df5fa99185c457b"}, - {file = "protobuf-4.22.0-cp38-cp38-win_amd64.whl", hash = "sha256:e474b63bab0a2ea32a7b26a4d8eec59e33e709321e5e16fb66e766b61b82a95e"}, - {file = "protobuf-4.22.0-cp39-cp39-win32.whl", hash = "sha256:47d31bdf58222dd296976aa1646c68c6ee80b96d22e0a3c336c9174e253fd35e"}, - {file = "protobuf-4.22.0-cp39-cp39-win_amd64.whl", hash = "sha256:c27f371f0159feb70e6ea52ed7e768b3f3a4c5676c1900a7e51a24740381650e"}, - {file = "protobuf-4.22.0-py3-none-any.whl", hash = "sha256:c3325803095fb4c2a48649c321d2fbde59f8fbfcb9bfc7a86df27d112831c571"}, - {file = "protobuf-4.22.0.tar.gz", hash = "sha256:652d8dfece122a24d98eebfef30e31e455d300efa41999d1182e015984ac5930"}, + {file = "protobuf-4.22.3-cp310-abi3-win32.whl", hash = "sha256:8b54f56d13ae4a3ec140076c9d937221f887c8f64954673d46f63751209e839a"}, + {file = "protobuf-4.22.3-cp310-abi3-win_amd64.whl", hash = "sha256:7760730063329d42a9d4c4573b804289b738d4931e363ffbe684716b796bde51"}, + {file = "protobuf-4.22.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:d14fc1a41d1a1909998e8aff7e80d2a7ae14772c4a70e4bf7db8a36690b54425"}, + {file = "protobuf-4.22.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:70659847ee57a5262a65954538088a1d72dfc3e9882695cab9f0c54ffe71663b"}, + {file = "protobuf-4.22.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:13233ee2b9d3bd9a5f216c1fa2c321cd564b93d8f2e4f521a85b585447747997"}, + {file = "protobuf-4.22.3-cp37-cp37m-win32.whl", hash = "sha256:ecae944c6c2ce50dda6bf76ef5496196aeb1b85acb95df5843cd812615ec4b61"}, + {file = "protobuf-4.22.3-cp37-cp37m-win_amd64.whl", hash = "sha256:d4b66266965598ff4c291416be429cef7989d8fae88b55b62095a2331511b3fa"}, + {file = "protobuf-4.22.3-cp38-cp38-win32.whl", hash = "sha256:f08aa300b67f1c012100d8eb62d47129e53d1150f4469fd78a29fa3cb68c66f2"}, + {file = "protobuf-4.22.3-cp38-cp38-win_amd64.whl", hash = "sha256:f2f4710543abec186aee332d6852ef5ae7ce2e9e807a3da570f36de5a732d88e"}, + {file = "protobuf-4.22.3-cp39-cp39-win32.whl", hash = "sha256:7cf56e31907c532e460bb62010a513408e6cdf5b03fb2611e4b67ed398ad046d"}, + {file = "protobuf-4.22.3-cp39-cp39-win_amd64.whl", hash = "sha256:e0e630d8e6a79f48c557cd1835865b593d0547dce221c66ed1b827de59c66c97"}, + {file = "protobuf-4.22.3-py3-none-any.whl", hash = "sha256:52f0a78141078077cfe15fe333ac3e3a077420b9a3f5d1bf9b5fe9d286b4d881"}, + {file = "protobuf-4.22.3.tar.gz", hash = "sha256:23452f2fdea754a8251d0fc88c0317735ae47217e0d27bf330a30eec2848811a"}, ] psutil = [ {file = "psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8"}, @@ -1046,8 +1329,8 @@ psutil = [ {file = "psutil-5.9.4.tar.gz", hash = "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62"}, ] pytest = [ - {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, - {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, + {file = "pytest-7.3.0-py3-none-any.whl", hash = "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"}, + {file = "pytest-7.3.0.tar.gz", hash = "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d"}, ] PyYAML = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, @@ -1138,47 +1421,59 @@ safetensors = [ {file = "safetensors-0.2.8.tar.gz", hash = "sha256:2720b20a6a38c799dca79bd76caeeac2f7df585a9d4f7d59fa7e28eff9ccb27f"}, ] sentencepiece = [ - {file = "sentencepiece-0.1.97-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6f249c8f1852893be86eae66b19d522c5fb30bbad4fe2d1b07f06fdc86e1907e"}, - {file = "sentencepiece-0.1.97-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09e1bc53178de70c557a9ba4fece07364b4416ce3d36570726b3372b68aea135"}, - {file = "sentencepiece-0.1.97-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:667193c57fb48b238be7e3d7636cfc8da56cb5bac5559d8f0b647334e1175be8"}, - {file = "sentencepiece-0.1.97-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2780531985af79c6163f63d4f200fec8a28b70b6768d2c19f70d01568a4524e8"}, - {file = "sentencepiece-0.1.97-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:205050670c53ef9015e2a98cce3934bfbcf0aafaa14caa0c618dd5667bc217ee"}, - {file = "sentencepiece-0.1.97-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28b183dadef8e8b6b4645c1c20692d7be0a13ecc3ec1a07b3885c8905516675f"}, - {file = "sentencepiece-0.1.97-cp310-cp310-win32.whl", hash = "sha256:ee3c9dbd558d8d85bb1617087b86df6ea2b856a528669630ce6cedeb4353b823"}, - {file = "sentencepiece-0.1.97-cp310-cp310-win_amd64.whl", hash = "sha256:f7dc55379e2f7dee86537180283db2e5f8418c6825fdd2fe436c724eb5604c05"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ba1b4154f9144c5a7528b00aff5cffaa1a896a1c6ca53ca78b6e74cd2dae5244"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac3d90aee5581e55d029d124ac11b6ae2fbae0817863b664b2f2302e966ababb"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c27400f1ac46518a01c87cb7703650e4e48728649feb115d2e3f1102a946a42"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6e12a166eba75994ca749aadc4a5056b91b31405f805d6de6e8914cc9741c60"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-win32.whl", hash = "sha256:ed85dff5c0a9b3dd1a414c7e1119f2a19e863fc3f81da525bf7f885ebc883de0"}, - {file = "sentencepiece-0.1.97-cp36-cp36m-win_amd64.whl", hash = "sha256:91a19ab6f40ffbae6d6127119953d2c6a85e93d734953dbc8629fde0d21ace66"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bae580e4a35a9314ff49561ac7c06574fe6afc71b821ed6bb00534e571458156"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ad7262e7530c683b186672b5dd0082f82719a50a500a8cfbc4bbd7cde5bff8c"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:620cee35279720016735a7c7103cddbd9b84fe5e2f098bd5e673834d69fee2b8"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93b921b59914c0ec6697e8c6d5e6b44d99d1298fb1a0af56980a79ade0540c19"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-win32.whl", hash = "sha256:9b9a4c44a31d5f47616e9568dcf31e029b0bfa776e0a252c0b59247881598b09"}, - {file = "sentencepiece-0.1.97-cp37-cp37m-win_amd64.whl", hash = "sha256:f31533cdacced56219e239d3459a003ece35116920dd64b2309d4ad047b77644"}, - {file = "sentencepiece-0.1.97-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7d643c01d1cad13b9206a276bbe5bc1a468e3d7cf6a26bde7783f945277f859d"}, - {file = "sentencepiece-0.1.97-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:542f1985b1ee279a92bef7740ec0781452372028ce01e15aa88df3228b197ba3"}, - {file = "sentencepiece-0.1.97-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93701da21fea906dd244bf88cdbe640385a89c45d3c1812b76dbadf8782cdbcd"}, - {file = "sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a51514047b964047b7fadb480d88a5e0f72c02f6ca1ba96258fbbc6e79274a94"}, - {file = "sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3ae2e9b7a5b6f2aa64ec9240b0c185dabe597d0e787dc4344acfbaef1ffe0b2"}, - {file = "sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:923ee4af16dbae1f2ab358ed09f8a0eb89e40a8198a8b343bf54181482342721"}, - {file = "sentencepiece-0.1.97-cp38-cp38-win32.whl", hash = "sha256:fa6f2b88850b5fae3a05053658824cf9f147c8e3c3b40eb64539a976c83d8a24"}, - {file = "sentencepiece-0.1.97-cp38-cp38-win_amd64.whl", hash = "sha256:5137ff0d0b1cc574751d178650ef800ff8d90bf21eb9f71e9567d4a0548940a5"}, - {file = "sentencepiece-0.1.97-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f92876271a10494671431ad955bff2d6f8ea59baaf957f5ae5946aff56dfcb90"}, - {file = "sentencepiece-0.1.97-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:35c227b6d55e473033db7e0ecc51b1e99e6ed7607cc08602fb5768132543c81d"}, - {file = "sentencepiece-0.1.97-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1706a8a8188f7b3d4b7922db9bb00c64c4e16ee68ab4caaae79f55b3e18748c7"}, - {file = "sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce61efc1862ccb18856c4aabbd930e13d5bfbb4b09b4f111081ac53a9dc62275"}, - {file = "sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a78c03800ef9f02d320e0159f5768b15357f3e9ebea545c9c4ba7928ba8ba254"}, - {file = "sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753b8088fd685ee787d9f54c84275ab347de558c7c4ebc6accb4c35bf7776f20"}, - {file = "sentencepiece-0.1.97-cp39-cp39-win32.whl", hash = "sha256:24306fd86031c17a1a6ae92671e76a350390a3140a65620bc2843dad7db24e2a"}, - {file = "sentencepiece-0.1.97-cp39-cp39-win_amd64.whl", hash = "sha256:c6641d0b7acec61fde5881ea6ebe098c169557ac9aa3bdabdf124eab5a5592bb"}, - {file = "sentencepiece-0.1.97.tar.gz", hash = "sha256:c901305e0a710bbcd296f66d79e96f744e6e175b29812bd5178318437d4e1f6c"}, + {file = "sentencepiece-0.1.98-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1daf0a79cd953e4830746c41e92b98a2f2e9e5ec0e90a9447aa10350e11bd027"}, + {file = "sentencepiece-0.1.98-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57911445fc91c80d59552adf8a749af9205458920a7328f3bd7d51308658bcd9"}, + {file = "sentencepiece-0.1.98-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3f9239785849ed1f55a825bcc282bef1a6073f7431cc535bdc658a94873652ea"}, + {file = "sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:467740ef8af170e5b6cfe22c272114ed930c899c297619ac7a2ac463a13bdbac"}, + {file = "sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b6f0b9ffb601e2699e265f3f20c353ec9a661e4b5f0cff08ad6c9909c0ae43e"}, + {file = "sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6150ba525fac4fda76f5c4777ae300597e70cef739ed2a47cea02ff81a88873f"}, + {file = "sentencepiece-0.1.98-cp310-cp310-win32.whl", hash = "sha256:58ca96d73ea0e5575e3f6a9524449c673d62e6ecee3b2ddd5bfb4f49cb315c0a"}, + {file = "sentencepiece-0.1.98-cp310-cp310-win_amd64.whl", hash = "sha256:8abe5c4c034e497e69f485dcd2c0e6bc87bf0498ad5aef5f539a7d0f9eae6275"}, + {file = "sentencepiece-0.1.98-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b6ed62f89c0bd25cec39a7075f6b9354fe4c240ed964e63009d77efcf29c34e9"}, + {file = "sentencepiece-0.1.98-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c2d9a74986d3716dc6961e9dbae7a3b25bb1260118f098545fd963ae23252c1"}, + {file = "sentencepiece-0.1.98-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f7dc2fc175623529fb60a2799748f8877cd48c4541b32cd97b8523465e88b69"}, + {file = "sentencepiece-0.1.98-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64e32c55d04a2e21f0c2fda1b7a3dd108133ebfb8616b52896916bb30e4352ed"}, + {file = "sentencepiece-0.1.98-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:443f32e94b18571231b02a51be173686114b5556b5edfcbf347fb63e7bd5ddc6"}, + {file = "sentencepiece-0.1.98-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:558a373a8660bdff299d6c133c2a4f4fb0875e9e6fafe225b8080ecce8a405f9"}, + {file = "sentencepiece-0.1.98-cp311-cp311-win32.whl", hash = "sha256:fcf100268cefe1774794b18cbaf3065e2bf988f168a387973eb1260d51198795"}, + {file = "sentencepiece-0.1.98-cp311-cp311-win_amd64.whl", hash = "sha256:05b4eecbece0606883cd81ed86bb3c619680bb570b997b236533ec854d64a575"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:35af00f5103a4779694fedea41b6e24947a9ed81166efe63864ab1e781d70a66"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2766cd708e9fc2b5b9784a990a8b303b9e0b9a69fa482616fe86fa538daa1756"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2531c0e9cc8cd404fabd856d80d695b373371c00f1fce29c06f41f3f7429d87"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffcc78e80c55eab67ee3439ade493607a4e37e1f0b82b168ead3debf9eaeaabe"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-win32.whl", hash = "sha256:ef384b31ec7a06a9a6aba42e68435f3f3b38809aa65559ede3658cdd446a562c"}, + {file = "sentencepiece-0.1.98-cp36-cp36m-win_amd64.whl", hash = "sha256:e7a828f1fe2e51d2d9e5e9b3283d4006f1891efb02a3d9303ed39ddafdd9c864"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8663be00a68098f85d6cda1f7041a27de05c320e433fa730ecb1156a8304f21c"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf05611089a075b78d353720ccc3a09a78e0846332cff0cc78fda8b2383626a"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11f410cc7eeb3e1cfa8d92d128b568e5dc7829b7904b164499fd0209316ec2fa"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5ea8fb2c68073fe25a08a178eed269ed382fba074ff2ba4de72f0f56d86630e"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-win32.whl", hash = "sha256:fa13a125417d28e84fbdebcaf6aa115e4177d3e93aa66b857a42e7179f515b88"}, + {file = "sentencepiece-0.1.98-cp37-cp37m-win_amd64.whl", hash = "sha256:e54aa70b574eee895d184072d84e62824f404821e551a82c619c5d4320a93834"}, + {file = "sentencepiece-0.1.98-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:515a971c2a157647ca0e60ce3c435f4b43cd5c9f5862159cfefa0b5b4d46d3c3"}, + {file = "sentencepiece-0.1.98-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c23c3a562221bc40eaae42428fcd8e607e0f084ea8aa968ba3f1a7d0ea975807"}, + {file = "sentencepiece-0.1.98-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c067ba22be8edc699f6365e01ec15046bf3563dbabfdc052ecc88e581b675cba"}, + {file = "sentencepiece-0.1.98-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12c913493d6ebac86ee7ae109e368522a5a365a7b150d4d8cf845599262d2b21"}, + {file = "sentencepiece-0.1.98-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:720f827dc69ee24951ea4f51b9fb69cc56890a7190fc52c2c0da2545caab1760"}, + {file = "sentencepiece-0.1.98-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:918b4caf18b2f73c302c4e197d1c2dafba39eb143d16b4590930b45f15042fdd"}, + {file = "sentencepiece-0.1.98-cp38-cp38-win32.whl", hash = "sha256:2d50edfc4649a1566b64f1a8402cd607e1893bf8e368732337d83f00df62d3fa"}, + {file = "sentencepiece-0.1.98-cp38-cp38-win_amd64.whl", hash = "sha256:7425b727c3d6b3b7bad0005a3be316078b254180b712d73955ff08cae3f6a385"}, + {file = "sentencepiece-0.1.98-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:00b2becbd7b98905a6de9695cf8682abe0d510ab0198e23c7d86fb2b793b6ae0"}, + {file = "sentencepiece-0.1.98-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7f71c4bdedb797052fb2ccad0871c2409bf6f812cb6b651917c55f9e8eced07f"}, + {file = "sentencepiece-0.1.98-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7287461d2346f530928ab187f0834cb15ddfbc4553592cacdcb6470364739ec6"}, + {file = "sentencepiece-0.1.98-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:472ad943eaffcb6871ece56c7850388e7b8722f520ba73c93e7a6ef965453221"}, + {file = "sentencepiece-0.1.98-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7e23aaf9d5afd91ca13550968bd17f0c17b0966823188ad2a50c51544cf8ed"}, + {file = "sentencepiece-0.1.98-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0ce9efc790c209cce2463058855dceb21438213d2ff13cb5a565d52a7efe25"}, + {file = "sentencepiece-0.1.98-cp39-cp39-win32.whl", hash = "sha256:8b50cbe8e46204eff7aa5a663af5652c45e7807aa560d08e5f5b10c60e795a49"}, + {file = "sentencepiece-0.1.98-cp39-cp39-win_amd64.whl", hash = "sha256:14841bd2a3d77c4dbba58f02488c374866551e428d755e8d473d82325a0a94f3"}, + {file = "sentencepiece-0.1.98.tar.gz", hash = "sha256:947cf0a4b8a480510d560a922f8256f34e93984a86cf870be4d05731f59fb28d"}, ] setuptools = [ - {file = "setuptools-67.4.0-py3-none-any.whl", hash = "sha256:f106dee1b506dee5102cc3f3e9e68137bbad6d47b616be7991714b0c62204251"}, - {file = "setuptools-67.4.0.tar.gz", hash = "sha256:e5fd0a713141a4a105412233c63dc4e17ba0090c8e8334594ac790ec97792330"}, + {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"}, + {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"}, +] +sympy = [ + {file = "sympy-1.11.1-py3-none-any.whl", hash = "sha256:938f984ee2b1e8eae8a07b884c8b7a1146010040fccddc6539c54f401c8f6fcf"}, + {file = "sympy-1.11.1.tar.gz", hash = "sha256:e32380dce63cb7c0108ed525570092fd45168bdae2faa17e528221ef72e88658"}, ] tokenizers = [ {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, @@ -1227,27 +1522,50 @@ tomli = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] torch = [ - {file = "torch-1.13.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:fd12043868a34a8da7d490bf6db66991108b00ffbeecb034228bfcbbd4197143"}, - {file = "torch-1.13.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d9fe785d375f2e26a5d5eba5de91f89e6a3be5d11efb497e76705fdf93fa3c2e"}, - {file = "torch-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:98124598cdff4c287dbf50f53fb455f0c1e3a88022b39648102957f3445e9b76"}, - {file = "torch-1.13.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:393a6273c832e047581063fb74335ff50b4c566217019cc6ace318cd79eb0566"}, - {file = "torch-1.13.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:0122806b111b949d21fa1a5f9764d1fd2fcc4a47cb7f8ff914204fd4fc752ed5"}, - {file = "torch-1.13.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:22128502fd8f5b25ac1cd849ecb64a418382ae81dd4ce2b5cebaa09ab15b0d9b"}, - {file = "torch-1.13.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:76024be052b659ac1304ab8475ab03ea0a12124c3e7626282c9c86798ac7bc11"}, - {file = "torch-1.13.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ea8dda84d796094eb8709df0fcd6b56dc20b58fdd6bc4e8d7109930dafc8e419"}, - {file = "torch-1.13.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2ee7b81e9c457252bddd7d3da66fb1f619a5d12c24d7074de91c4ddafb832c93"}, - {file = "torch-1.13.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:0d9b8061048cfb78e675b9d2ea8503bfe30db43d583599ae8626b1263a0c1380"}, - {file = "torch-1.13.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:f402ca80b66e9fbd661ed4287d7553f7f3899d9ab54bf5c67faada1555abde28"}, - {file = "torch-1.13.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:727dbf00e2cf858052364c0e2a496684b9cb5aa01dc8a8bc8bbb7c54502bdcdd"}, - {file = "torch-1.13.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:df8434b0695e9ceb8cc70650afc1310d8ba949e6db2a0525ddd9c3b2b181e5fe"}, - {file = "torch-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:5e1e722a41f52a3f26f0c4fcec227e02c6c42f7c094f32e49d4beef7d1e213ea"}, - {file = "torch-1.13.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:33e67eea526e0bbb9151263e65417a9ef2d8fa53cbe628e87310060c9dcfa312"}, - {file = "torch-1.13.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:eeeb204d30fd40af6a2d80879b46a7efbe3cf43cdbeb8838dd4f3d126cc90b2b"}, - {file = "torch-1.13.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:50ff5e76d70074f6653d191fe4f6a42fdbe0cf942fbe2a3af0b75eaa414ac038"}, - {file = "torch-1.13.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2c3581a3fd81eb1f0f22997cddffea569fea53bafa372b2c0471db373b26aafc"}, - {file = "torch-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:0aa46f0ac95050c604bcf9ef71da9f1172e5037fdf2ebe051962d47b123848e7"}, - {file = "torch-1.13.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6930791efa8757cb6974af73d4996b6b50c592882a324b8fb0589c6a9ba2ddaf"}, - {file = "torch-1.13.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:e0df902a7c7dd6c795698532ee5970ce898672625635d885eade9976e5a04949"}, + {file = "torch-2.0.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a"}, + {file = "torch-2.0.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7"}, + {file = "torch-2.0.0-1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8969aa8375bcbc0c2993e7ede0a7f889df9515f18b9b548433f412affed478d9"}, + {file = "torch-2.0.0-1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:ab2da16567cb55b67ae39e32d520d68ec736191d88ac79526ca5874754c32203"}, + {file = "torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:7a9319a67294ef02459a19738bbfa8727bb5307b822dadd708bc2ccf6c901aca"}, + {file = "torch-2.0.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9f01fe1f6263f31bd04e1757946fd63ad531ae37f28bb2dbf66f5c826ee089f4"}, + {file = "torch-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:527f4ae68df7b8301ee6b1158ca56350282ea633686537b30dbb5d7b4a52622a"}, + {file = "torch-2.0.0-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:ce9b5a49bd513dff7950a5a07d6e26594dd51989cee05ba388b03e8e366fd5d5"}, + {file = "torch-2.0.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:53e1c33c6896583cdb9a583693e22e99266444c4a43392dddc562640d39e542b"}, + {file = "torch-2.0.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:09651bff72e439d004c991f15add0c397c66f98ab36fe60d5514b44e4da722e8"}, + {file = "torch-2.0.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d439aec349c98f12819e8564b8c54008e4613dd4428582af0e6e14c24ca85870"}, + {file = "torch-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:2802f84f021907deee7e9470ed10c0e78af7457ac9a08a6cd7d55adef835fede"}, + {file = "torch-2.0.0-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:01858620f25f25e7a9ec4b547ff38e5e27c92d38ec4ccba9cfbfb31d7071ed9c"}, + {file = "torch-2.0.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:9a2e53b5783ef5896a6af338b36d782f28e83c8ddfc2ac44b67b066d9d76f498"}, + {file = "torch-2.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:ec5fff2447663e369682838ff0f82187b4d846057ef4d119a8dea7772a0b17dd"}, + {file = "torch-2.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:11b0384fe3c18c01b8fc5992e70fc519cde65e44c51cc87be1838c1803daf42f"}, + {file = "torch-2.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:e54846aa63855298cfb1195487f032e413e7ac9cbfa978fda32354cc39551475"}, + {file = "torch-2.0.0-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:cc788cbbbbc6eb4c90e52c550efd067586c2693092cf367c135b34893a64ae78"}, + {file = "torch-2.0.0-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:d292640f0fd72b7a31b2a6e3b635eb5065fcbedd4478f9cad1a1e7a9ec861d35"}, + {file = "torch-2.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6befaad784004b7af357e3d87fa0863c1f642866291f12a4c2af2de435e8ac5c"}, + {file = "torch-2.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a83b26bd6ae36fbf5fee3d56973d9816e2002e8a3b7d9205531167c28aaa38a7"}, + {file = "torch-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:c7e67195e1c3e33da53954b026e89a8e1ff3bc1aeb9eb32b677172d4a9b5dcbf"}, + {file = "torch-2.0.0-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6e0b97beb037a165669c312591f242382e9109a240e20054d5a5782d9236cad0"}, + {file = "torch-2.0.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:297a4919aff1c0f98a58ebe969200f71350a1d4d4f986dbfd60c02ffce780e99"}, +] +triton = [ + {file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"}, + {file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"}, + {file = "triton-2.0.0-1-cp36-cp36m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd"}, + {file = "triton-2.0.0-1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b"}, + {file = "triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef"}, + {file = "triton-2.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656"}, + {file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"}, + {file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"}, + {file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"}, + {file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"}, + {file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"}, + {file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"}, + {file = "triton-2.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fedce6a381901b1547e0e7e1f2546e4f65dca6d91e2d8a7305a2d1f5551895be"}, + {file = "triton-2.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75834f27926eab6c7f00ce73aaf1ab5bfb9bec6eb57ab7c0bfc0a23fac803b4c"}, + {file = "triton-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0117722f8c2b579cd429e0bee80f7731ae05f63fe8e9414acd9a679885fcbf42"}, + {file = "triton-2.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd9be5d0c2e45d2b7e6ddc6da20112b6862d69741576f9c3dbaf941d745ecae"}, + {file = "triton-2.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a0d2c3fc2eab4ba71384f2e785fbfd47aa41ae05fa58bf12cb31dcbd0aeceb"}, + {file = "triton-2.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c47b72c72693198163ece9d90a721299e4fb3b8e24fd13141e384ad952724f"}, ] typer = [ {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, @@ -1258,12 +1576,12 @@ typing-extensions = [ {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, ] urllib3 = [ - {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, - {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, ] wheel = [ - {file = "wheel-0.38.4-py3-none-any.whl", hash = "sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8"}, - {file = "wheel-0.38.4.tar.gz", hash = "sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac"}, + {file = "wheel-0.40.0-py3-none-any.whl", hash = "sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247"}, + {file = "wheel-0.40.0.tar.gz", hash = "sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873"}, ] win32-setctime = [ {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 41fd63b6..c892cd3f 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -31,7 +31,7 @@ bnb = ["bitsandbytes"] [tool.poetry.group.dev.dependencies] grpcio-tools = "^1.51.1" -pytest = "^7.2.0" +pytest = "^7.3.0" [build-system] requires = ["poetry-core>=1.0.0"] From 643a39d5562ea5c20fb22f4b12e18a4855280af8 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Apr 2023 15:26:34 +0200 Subject: [PATCH 157/793] feat(ci): add image signing with cosign (#175) --- .github/workflows/build.yaml | 42 ++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 89eaf47e..6e077036 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -27,20 +27,32 @@ concurrency: jobs: build-and-push-image: runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write steps: + - name: Checkout repository + uses: actions/checkout@v3 - name: Initialize Docker Buildx uses: docker/setup-buildx-action@v2.0.0 with: install: true + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 + with: + cosign-release: 'v1.13.1' - name: Tailscale uses: tailscale/github-action@v1 with: authkey: ${{ secrets.TAILSCALE_AUTHKEY }} - - name: Checkout repository - uses: actions/checkout@v3 - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4.4.1 - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' uses: docker/login-action@v2 with: registry: ghcr.io @@ -53,6 +65,7 @@ jobs: password: ${{ secrets.TAILSCALE_DOCKER_PASSWORD }} registry: registry.internal.huggingface.tech - name: Login to Azure Container Registry + if: github.event_name != 'pull_request' uses: docker/login-action@v2.1.0 with: username: ${{ secrets.AZURE_DOCKER_USERNAME }} @@ -74,7 +87,7 @@ jobs: type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }} - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . file: Dockerfile @@ -85,23 +98,34 @@ jobs: cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + COSIGN_EXPERIMENTAL: "true" + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} + build-and-push-sagemaker-image: needs: - build-and-push-image runs-on: ubuntu-latest steps: + - name: Checkout repository + uses: actions/checkout@v3 - name: Initialize Docker Buildx uses: docker/setup-buildx-action@v2.0.0 with: install: true + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 - name: Tailscale uses: tailscale/github-action@v1 with: authkey: ${{ secrets.TAILSCALE_AUTHKEY }} - - name: Checkout repository - uses: actions/checkout@v3 - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4.4.1 - name: Login to internal Container Registry uses: docker/login-action@v2.1.0 with: From 9683c37bd37f9b031e657e0ea10ed911988960f4 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Apr 2023 15:43:17 +0200 Subject: [PATCH 158/793] feat(ci): add Trivy and scan docker image (#178) --- .github/workflows/build.yaml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6e077036..c92f7c09 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -87,6 +87,7 @@ jobs: type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }} - name: Build and push Docker image + id: build-and-push uses: docker/build-push-action@v4 with: context: . @@ -97,7 +98,6 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max - # Sign the resulting Docker image digest except on PRs. # This will only write to the public Rekor transparency log when the Docker # repository is public to avoid leaking data. @@ -108,6 +108,27 @@ jobs: # This step uses the identity token to provision an ephemeral certificate # against the sigstore community Fulcio instance. run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} + - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph + uses: aquasecurity/trivy-action@master + if: ${{ github.event_name != 'pull_request' }} + with: + image-ref: 'ghcr.io/huggingface/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}' + format: 'github' + output: 'dependency-results.sbom.json' + github-pat: ${{ secrets.GITHUB_TOKEN }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + if: ${{ github.event_name != 'pull_request' }} + with: + image-ref: 'ghcr.io/huggingface/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}' + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + if: ${{ github.event_name != 'pull_request' }} + with: + sarif_file: 'trivy-results.sarif' build-and-push-sagemaker-image: needs: From 13f1cd024b42de976711c84adb52d7ee352e786e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Apr 2023 16:11:48 +0200 Subject: [PATCH 159/793] feat(ci): use large runners (#179) --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c92f7c09..f81b1b63 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -26,7 +26,7 @@ concurrency: jobs: build-and-push-image: - runs-on: ubuntu-latest + runs-on: large permissions: contents: read packages: write @@ -133,7 +133,7 @@ jobs: build-and-push-sagemaker-image: needs: - build-and-push-image - runs-on: ubuntu-latest + runs-on: large steps: - name: Checkout repository uses: actions/checkout@v3 From c1e2ea3b7898238bd65806ffe9f36512a982c293 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Apr 2023 16:23:47 +0200 Subject: [PATCH 160/793] feat(ci): faster scanning (#180) --- .github/workflows/build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f81b1b63..7e8191e2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ jobs: build-and-push-image: runs-on: large permissions: - contents: read + contents: write packages: write # This is used to complete the identity challenge # with sigstore/fulcio when running outside of PRs. @@ -116,6 +116,7 @@ jobs: format: 'github' output: 'dependency-results.sbom.json' github-pat: ${{ secrets.GITHUB_TOKEN }} + scanners: 'vuln' - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master if: ${{ github.event_name != 'pull_request' }} @@ -124,6 +125,7 @@ jobs: format: 'sarif' output: 'trivy-results.sarif' severity: 'CRITICAL' + scanners: 'vuln' - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v2 if: ${{ github.event_name != 'pull_request' }} From 12e5633c4d6db6a3956afb6d6b0441eaaf4752cb Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Apr 2023 16:32:37 +0200 Subject: [PATCH 161/793] fix(ci): fix ci permissions (#181) --- .github/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7e8191e2..7e988992 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -33,6 +33,7 @@ jobs: # This is used to complete the identity challenge # with sigstore/fulcio when running outside of PRs. id-token: write + security-events: write steps: - name: Checkout repository uses: actions/checkout@v3 From 53ee09c0b0004777f029f594ce44cffa6350ed08 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 14 Apr 2023 10:12:21 +0200 Subject: [PATCH 162/793] fea(dockerfile): better layer caching (#159) --- .github/workflows/build.yaml | 10 +- .github/workflows/client-tests.yaml | 2 +- .github/workflows/tests.yaml | 2 +- Dockerfile | 141 ++++++++++++++++++++++------ clients/python/tests/test_client.py | 4 +- server/Makefile-flash-att | 16 +++- server/Makefile-transformers | 13 ++- 7 files changed, 140 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7e988992..f82ddd7e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -26,7 +26,7 @@ concurrency: jobs: build-and-push-image: - runs-on: large + runs-on: ubuntu-latest permissions: contents: write packages: write @@ -45,9 +45,7 @@ jobs: uses: rlespinasse/github-slug-action@v4.4.1 - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 - with: - cosign-release: 'v1.13.1' + uses: sigstore/cosign-installer@v3.0.2 - name: Tailscale uses: tailscale/github-action@v1 with: @@ -66,7 +64,7 @@ jobs: password: ${{ secrets.TAILSCALE_DOCKER_PASSWORD }} registry: registry.internal.huggingface.tech - name: Login to Azure Container Registry - if: github.event_name != 'pull_request' +# if: github.event_name != 'pull_request' uses: docker/login-action@v2.1.0 with: username: ${{ secrets.AZURE_DOCKER_USERNAME }} @@ -136,7 +134,7 @@ jobs: build-and-push-sagemaker-image: needs: - build-and-push-image - runs-on: large + runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v3 diff --git a/.github/workflows/client-tests.yaml b/.github/workflows/client-tests.yaml index 07eeb270..7ccef3b0 100644 --- a/.github/workflows/client-tests.yaml +++ b/.github/workflows/client-tests.yaml @@ -8,7 +8,7 @@ on: jobs: run_tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f96c53fb..e82e8b20 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -17,7 +17,7 @@ concurrency: jobs: run_tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest env: SCCACHE_GHA_ENABLED: "on" diff --git a/Dockerfile b/Dockerfile index 9fe0b49b..f027be14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# Rust builder FROM lukemathwalker/cargo-chef:latest-rust-1.67 AS chef WORKDIR /usr/src @@ -27,51 +28,135 @@ COPY router router COPY launcher launcher RUN cargo build --release -FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as base +# Python builder +# Adapted from: https://github.com/pytorch/pytorch/blob/master/Dockerfile +FROM ubuntu:22.04 as pytorch-install + +ARG PYTORCH_VERSION=2.0.0 +ARG PYTHON_VERSION=3.9 +ARG CUDA_VERSION=11.8 +ARG MAMBA_VERSION=23.1.0-1 +ARG CUDA_CHANNEL=nvidia +ARG INSTALL_CHANNEL=pytorch +# Automatically set by buildx +ARG TARGETPLATFORM + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + ccache \ + cmake \ + curl \ + git && \ + rm -rf /var/lib/apt/lists/* +RUN /usr/sbin/update-ccache-symlinks && \ + mkdir /opt/ccache && \ + ccache --set-config=cache_dir=/opt/ccache +ENV PATH /opt/conda/bin:$PATH + +# Install conda +# translating Docker's TARGETPLATFORM into mamba arches +RUN case ${TARGETPLATFORM} in \ + "linux/arm64") MAMBA_ARCH=aarch64 ;; \ + *) MAMBA_ARCH=x86_64 ;; \ + esac && \ + curl -fsSL -v -o ~/mambaforge.sh -O "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" +RUN chmod +x ~/mambaforge.sh && \ + bash ~/mambaforge.sh -b -p /opt/conda && \ + rm ~/mambaforge.sh + +# Install pytorch +# On arm64 we exit with an error code +RUN case ${TARGETPLATFORM} in \ + "linux/arm64") exit 1 ;; \ + *) /opt/conda/bin/conda update -y conda && \ + /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y "python=${PYTHON_VERSION}" pytorch==$PYTORCH_VERSION "pytorch-cuda=$(echo $CUDA_VERSION | cut -d'.' -f 1-2)" ;; \ + esac && \ + /opt/conda/bin/conda clean -ya + +# CUDA kernels builder image +FROM pytorch-install as kernel-builder + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* + +RUN /opt/conda/bin/conda install -c "nvidia/label/cuda-11.8.0" cuda==11.8 && \ + /opt/conda/bin/conda clean -ya + + +# Build Flash Attention CUDA kernels +FROM kernel-builder as flash-att-builder -ENV LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 \ - DEBIAN_FRONTEND=noninteractive \ - HUGGINGFACE_HUB_CACHE=/data \ +WORKDIR /usr/src + +COPY server/Makefile-flash-att Makefile + +# Build specific version of flash attention +RUN make build-flash-attention + +# Build Transformers CUDA kernels +FROM kernel-builder as transformers-builder + +WORKDIR /usr/src + +COPY server/Makefile-transformers Makefile + +# Build specific version of transformers +RUN BUILD_EXTENSIONS="True" make build-transformers + +# Text Generation Inference base image +FROM ubuntu:22.04 as base + +ARG TARGETPLATFORM +ARG PYTORCH_VERSION=2.0.0 +ARG CUDA_VERSION=11.8 + +# Conda and CUDA env +ENV PATH=/opt/conda/bin:$PATH \ + NVIDIA_VISIBLE_DEVICES=all \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility \ + LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64 + +# Text Generation Inference base env +ENV HUGGINGFACE_HUB_CACHE=/data \ HF_HUB_ENABLE_HF_TRANSFER=1 \ MODEL_ID=bigscience/bloom-560m \ QUANTIZE=false \ NUM_SHARD=1 \ - PORT=80 \ - CUDA_HOME=/usr/local/cuda \ - LD_LIBRARY_PATH="/opt/miniconda/envs/text-generation/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH" \ - CONDA_DEFAULT_ENV=text-generation \ - PATH=$PATH:/opt/miniconda/envs/text-generation/bin:/opt/miniconda/bin:/usr/local/cuda/bin + PORT=80 -RUN apt-get update && apt-get install -y git curl libssl-dev && rm -rf /var/lib/apt/lists/* - -RUN cd ~ && \ - curl -L -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - chmod +x Miniconda3-latest-Linux-x86_64.sh && \ - bash ./Miniconda3-latest-Linux-x86_64.sh -bf -p /opt/miniconda && \ - conda create -n text-generation python=3.9 -y +LABEL com.nvidia.volumes.needed="nvidia_driver" WORKDIR /usr/src -# Install torch -RUN pip install torch --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libssl-dev \ + make \ + && rm -rf /var/lib/apt/lists/* -# Install specific version of flash attention -COPY server/Makefile-flash-att server/Makefile -RUN cd server && make install-flash-attention +# Copy conda with PyTorch installed +COPY --from=pytorch-install /opt/conda /opt/conda -# Install specific version of transformers -COPY server/Makefile-transformers server/Makefile -RUN cd server && BUILD_EXTENSIONS="True" make install-transformers +# Copy build artifacts from flash attention builder +COPY --from=flash-att-builder /usr/src/flash-attention/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=flash-att-builder /usr/src/flash-attention/csrc/layer_norm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=flash-att-builder /usr/src/flash-attention/csrc/rotary/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages -COPY server/Makefile server/Makefile +# Copy build artifacts from transformers builder +COPY --from=transformers-builder /usr/src/transformers /usr/src/transformers +COPY --from=transformers-builder /usr/src/transformers/build/lib.linux-x86_64-cpython-39/transformers /usr/src/transformers/src/transformers + +# Install transformers dependencies +RUN cd /usr/src/transformers && pip install -e . --no-cache-dir # Install server COPY proto proto COPY server server +COPY server/Makefile server/Makefile RUN cd server && \ make gen-server && \ - /opt/miniconda/envs/text-generation/bin/pip install ".[bnb]" --no-cache-dir + pip install ".[bnb]" --no-cache-dir # Install router COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bin/text-generation-router @@ -86,7 +171,7 @@ RUN chmod +x entrypoint.sh ENTRYPOINT ["./entrypoint.sh"] -# Original image +# Final image FROM base ENTRYPOINT ["text-generation-launcher"] diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index c998de41..d53c2a4d 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -17,7 +17,7 @@ def test_generate(flan_t5_xxl_url, hf_headers): assert response.details.prefill[0] == PrefillToken(id=0, text="", logprob=None) assert len(response.details.tokens) == 1 assert response.details.tokens[0] == Token( - id=3, text=" ", logprob=-1.984375, special=False + id=3, text="", logprob=-1.984375, special=False ) @@ -83,7 +83,7 @@ async def test_generate_async(flan_t5_xxl_url, hf_headers): assert response.details.prefill[0] == PrefillToken(id=0, text="", logprob=None) assert len(response.details.tokens) == 1 assert response.details.tokens[0] == Token( - id=3, text=" ", logprob=-1.984375, special=False + id=3, text="", logprob=-1.984375, special=False ) diff --git a/server/Makefile-flash-att b/server/Makefile-flash-att index 297fd9d0..ad894bfa 100644 --- a/server/Makefile-flash-att +++ b/server/Makefile-flash-att @@ -1,10 +1,16 @@ flash_att_commit := d478eeec8f16c7939c54e4617dbd36f59b8eeed7 -install-flash-attention: - # Install specific version of flash attention +flash-attention: + # Clone flash attention pip install packaging - pip uninstall flash_attn rotary_emb dropout_layer_norm -y || true - rm -rf flash-attention || true git clone https://github.com/HazyResearch/flash-attention.git - cd flash-attention && git checkout $(flash_att_commit) + +build-flash-attention: flash-attention + cd flash-attention && git fetch && git checkout $(flash_att_commit) + cd flash-attention && python setup.py build + cd flash-attention/csrc/rotary && python setup.py build + cd flash-attention/csrc/layer_norm && python setup.py build + +install-flash-attention: build-flash-attention + pip uninstall flash_attn rotary_emb dropout_layer_norm -y || true cd flash-attention && python setup.py install && cd csrc/layer_norm && python setup.py install && cd ../rotary && python setup.py install \ No newline at end of file diff --git a/server/Makefile-transformers b/server/Makefile-transformers index 1e081336..692a9374 100644 --- a/server/Makefile-transformers +++ b/server/Makefile-transformers @@ -1,10 +1,13 @@ transformers_commit := b8d969ff47c6a9d40538a6ea33df021953363afc -install-transformers: - # Install specific version of transformers with custom cuda kernels +transformers: + # Clone fork of transformers with custom CUDA kernels and sharding logic pip install --upgrade setuptools - pip uninstall transformers -y || true - rm -rf transformers || true git clone https://github.com/OlivierDehaene/transformers.git - cd transformers && git checkout $(transformers_commit) + +build-transformers: transformers + cd transformers && git fetch && git checkout $(transformers_commit) && python setup.py build + +install-transformers: build-transformers + pip uninstall transformers -y || true cd transformers && python setup.py install \ No newline at end of file From 01c0e368e57ee2626bf3d5c8203abdf422613702 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 14 Apr 2023 12:35:26 +0200 Subject: [PATCH 163/793] fix(ci): fix cosign error (#183) --- .github/workflows/build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f82ddd7e..5cb07489 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -45,7 +45,9 @@ jobs: uses: rlespinasse/github-slug-action@v4.4.1 - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@v3.0.2 + uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 + with: + cosign-release: 'v1.13.1' - name: Tailscale uses: tailscale/github-action@v1 with: From 1bb394631ddebc02e0b39223077b5a683b886941 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 14 Apr 2023 17:31:13 +0200 Subject: [PATCH 164/793] fix(docker): fix docker image (#184) --- .github/workflows/build.yaml | 2 +- Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5cb07489..cb51b273 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -26,7 +26,7 @@ concurrency: jobs: build-and-push-image: - runs-on: ubuntu-latest + runs-on: large-runner permissions: contents: write packages: write diff --git a/Dockerfile b/Dockerfile index f027be14..62505f0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -132,6 +132,7 @@ WORKDIR /usr/src RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ libssl-dev \ + ca-certificates \ make \ && rm -rf /var/lib/apt/lists/* @@ -148,7 +149,7 @@ COPY --from=transformers-builder /usr/src/transformers /usr/src/transformers COPY --from=transformers-builder /usr/src/transformers/build/lib.linux-x86_64-cpython-39/transformers /usr/src/transformers/src/transformers # Install transformers dependencies -RUN cd /usr/src/transformers && pip install -e . --no-cache-dir +RUN cd /usr/src/transformers && pip install -e . --no-cache-dir && pip install einops --no-cache-dir # Install server COPY proto proto From f9047562d0320ea25c1a14695abfe39a2d6fc13a Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 14 Apr 2023 18:58:38 +0200 Subject: [PATCH 165/793] fix(docker): fix image (#185) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 62505f0c..c618a2f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ COPY server/Makefile-transformers Makefile RUN BUILD_EXTENSIONS="True" make build-transformers # Text Generation Inference base image -FROM ubuntu:22.04 as base +FROM debian:bullseye-slim as base ARG TARGETPLATFORM ARG PYTORCH_VERSION=2.0.0 From 379c5c4da2494a15fb82b3b1a39fa454cb73df44 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 14 Apr 2023 19:30:30 +0200 Subject: [PATCH 166/793] fix(docker): revert dockerfile changes (#186) --- Dockerfile | 135 ++++++++++------------------------------------------- 1 file changed, 25 insertions(+), 110 deletions(-) diff --git a/Dockerfile b/Dockerfile index c618a2f8..25616f29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,3 @@ -# Rust builder FROM lukemathwalker/cargo-chef:latest-rust-1.67 AS chef WORKDIR /usr/src @@ -28,133 +27,49 @@ COPY router router COPY launcher launcher RUN cargo build --release -# Python builder -# Adapted from: https://github.com/pytorch/pytorch/blob/master/Dockerfile -FROM ubuntu:22.04 as pytorch-install +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as base -ARG PYTORCH_VERSION=2.0.0 -ARG PYTHON_VERSION=3.9 -ARG CUDA_VERSION=11.8 ARG MAMBA_VERSION=23.1.0-1 -ARG CUDA_CHANNEL=nvidia -ARG INSTALL_CHANNEL=pytorch -# Automatically set by buildx -ARG TARGETPLATFORM - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - ccache \ - cmake \ - curl \ - git && \ - rm -rf /var/lib/apt/lists/* -RUN /usr/sbin/update-ccache-symlinks && \ - mkdir /opt/ccache && \ - ccache --set-config=cache_dir=/opt/ccache -ENV PATH /opt/conda/bin:$PATH - -# Install conda -# translating Docker's TARGETPLATFORM into mamba arches -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") MAMBA_ARCH=aarch64 ;; \ - *) MAMBA_ARCH=x86_64 ;; \ - esac && \ - curl -fsSL -v -o ~/mambaforge.sh -O "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" -RUN chmod +x ~/mambaforge.sh && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh - -# Install pytorch -# On arm64 we exit with an error code -RUN case ${TARGETPLATFORM} in \ - "linux/arm64") exit 1 ;; \ - *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y "python=${PYTHON_VERSION}" pytorch==$PYTORCH_VERSION "pytorch-cuda=$(echo $CUDA_VERSION | cut -d'.' -f 1-2)" ;; \ - esac && \ - /opt/conda/bin/conda clean -ya - -# CUDA kernels builder image -FROM pytorch-install as kernel-builder - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ninja-build \ - && rm -rf /var/lib/apt/lists/* - -RUN /opt/conda/bin/conda install -c "nvidia/label/cuda-11.8.0" cuda==11.8 && \ - /opt/conda/bin/conda clean -ya - - -# Build Flash Attention CUDA kernels -FROM kernel-builder as flash-att-builder - -WORKDIR /usr/src - -COPY server/Makefile-flash-att Makefile - -# Build specific version of flash attention -RUN make build-flash-attention - -# Build Transformers CUDA kernels -FROM kernel-builder as transformers-builder -WORKDIR /usr/src - -COPY server/Makefile-transformers Makefile - -# Build specific version of transformers -RUN BUILD_EXTENSIONS="True" make build-transformers - -# Text Generation Inference base image -FROM debian:bullseye-slim as base - -ARG TARGETPLATFORM -ARG PYTORCH_VERSION=2.0.0 -ARG CUDA_VERSION=11.8 - -# Conda and CUDA env -ENV PATH=/opt/conda/bin:$PATH \ - NVIDIA_VISIBLE_DEVICES=all \ - NVIDIA_DRIVER_CAPABILITIES=compute,utility \ - LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64 - -# Text Generation Inference base env -ENV HUGGINGFACE_HUB_CACHE=/data \ +ENV LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + DEBIAN_FRONTEND=noninteractive \ + HUGGINGFACE_HUB_CACHE=/data \ HF_HUB_ENABLE_HF_TRANSFER=1 \ MODEL_ID=bigscience/bloom-560m \ QUANTIZE=false \ NUM_SHARD=1 \ - PORT=80 + PORT=80 \ + CUDA_HOME=/usr/local/cuda \ + LD_LIBRARY_PATH="/opt/conda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH" \ + PATH=$PATH:/opt/conda/bin:/usr/local/cuda/bin -LABEL com.nvidia.volumes.needed="nvidia_driver" +RUN apt-get update && apt-get install -y git curl libssl-dev ninja-build && rm -rf /var/lib/apt/lists/* -WORKDIR /usr/src +RUN cd ~ && \ + curl -fsSL -v -o ~/mambaforge.sh -O "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-x86_64.sh" \ + chmod +x ~/mambaforge.sh && \ + bash ~/mambaforge.sh -b -p /opt/conda && \ + rm ~/mambaforge.sh -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libssl-dev \ - ca-certificates \ - make \ - && rm -rf /var/lib/apt/lists/* +WORKDIR /usr/src -# Copy conda with PyTorch installed -COPY --from=pytorch-install /opt/conda /opt/conda +# Install torch +RUN pip install torch==2.0.0 --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir -# Copy build artifacts from flash attention builder -COPY --from=flash-att-builder /usr/src/flash-attention/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages -COPY --from=flash-att-builder /usr/src/flash-attention/csrc/layer_norm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages -COPY --from=flash-att-builder /usr/src/flash-attention/csrc/rotary/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +# Install specific version of flash attention +COPY server/Makefile-flash-att server/Makefile +RUN cd server && make install-flash-attention -# Copy build artifacts from transformers builder -COPY --from=transformers-builder /usr/src/transformers /usr/src/transformers -COPY --from=transformers-builder /usr/src/transformers/build/lib.linux-x86_64-cpython-39/transformers /usr/src/transformers/src/transformers +# Install specific version of transformers +COPY server/Makefile-transformers server/Makefile +RUN cd server && BUILD_EXTENSIONS="True" make install-transformers -# Install transformers dependencies -RUN cd /usr/src/transformers && pip install -e . --no-cache-dir && pip install einops --no-cache-dir +COPY server/Makefile server/Makefile # Install server COPY proto proto COPY server server -COPY server/Makefile server/Makefile RUN cd server && \ make gen-server && \ pip install ".[bnb]" --no-cache-dir From 7a1ba58557095734ae0bcd0b43f159b66485837b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 17 Apr 2023 00:26:47 +0200 Subject: [PATCH 167/793] fix(docker): fix docker image dependencies (#187) --- .github/workflows/build.yaml | 2 +- Dockerfile | 127 +++- launcher/Cargo.toml | 2 +- launcher/src/main.rs | 60 +- server/Makefile | 1 + server/poetry.lock | 26 +- server/pyproject.toml | 2 +- server/requirements.txt | 699 ++++++++++++++++++ server/text_generation_server/cli.py | 10 +- .../text_generation_server/models/__init__.py | 2 +- 10 files changed, 875 insertions(+), 56 deletions(-) create mode 100644 server/requirements.txt diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cb51b273..aa7db30a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -79,8 +79,8 @@ jobs: flavor: | latest=auto images: | - ghcr.io/huggingface/text-generation-inference registry.internal.huggingface.tech/api-inference/community/text-generation-inference + ghcr.io/huggingface/text-generation-inference db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference tags: | type=semver,pattern={{version}} diff --git a/Dockerfile b/Dockerfile index 25616f29..dac274c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +# Rust builder FROM lukemathwalker/cargo-chef:latest-rust-1.67 AS chef WORKDIR /usr/src @@ -27,51 +28,127 @@ COPY router router COPY launcher launcher RUN cargo build --release -FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as base +# Python builder +# Adapted from: https://github.com/pytorch/pytorch/blob/master/Dockerfile +FROM debian:bullseye-slim as pytorch-install +ARG PYTORCH_VERSION=2.0.0 +ARG PYTHON_VERSION=3.9 +ARG CUDA_VERSION=11.8 ARG MAMBA_VERSION=23.1.0-1 +ARG CUDA_CHANNEL=nvidia +ARG INSTALL_CHANNEL=pytorch +# Automatically set by buildx +ARG TARGETPLATFORM + +ENV PATH /opt/conda/bin:$PATH + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + ccache \ + curl \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Install conda +# translating Docker's TARGETPLATFORM into mamba arches +RUN case ${TARGETPLATFORM} in \ + "linux/arm64") MAMBA_ARCH=aarch64 ;; \ + *) MAMBA_ARCH=x86_64 ;; \ + esac && \ + curl -fsSL -v -o ~/mambaforge.sh -O "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" +RUN chmod +x ~/mambaforge.sh && \ + bash ~/mambaforge.sh -b -p /opt/conda && \ + rm ~/mambaforge.sh + +# Install pytorch +# On arm64 we exit with an error code +RUN case ${TARGETPLATFORM} in \ + "linux/arm64") exit 1 ;; \ + *) /opt/conda/bin/conda update -y conda && \ + /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y "python=${PYTHON_VERSION}" pytorch==$PYTORCH_VERSION "pytorch-cuda=$(echo $CUDA_VERSION | cut -d'.' -f 1-2)" ;; \ + esac && \ + /opt/conda/bin/conda clean -ya + +# CUDA kernels builder image +FROM pytorch-install as kernel-builder + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* + +RUN /opt/conda/bin/conda install -c "nvidia/label/cuda-11.8.0" cuda==11.8 && \ + /opt/conda/bin/conda clean -ya + + +# Build Flash Attention CUDA kernels +FROM kernel-builder as flash-att-builder + +WORKDIR /usr/src + +COPY server/Makefile-flash-att Makefile + +# Build specific version of flash attention +RUN make build-flash-attention + +# Build Transformers CUDA kernels +FROM kernel-builder as transformers-builder -ENV LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 \ - DEBIAN_FRONTEND=noninteractive \ - HUGGINGFACE_HUB_CACHE=/data \ +WORKDIR /usr/src + +COPY server/Makefile-transformers Makefile + +# Build specific version of transformers +RUN BUILD_EXTENSIONS="True" make build-transformers + +# Text Generation Inference base image +FROM debian:bullseye-slim as base + +# Conda env +ENV PATH=/opt/conda/bin:$PATH \ + CONDA_PREFIX=/opt/conda + +# Text Generation Inference base env +ENV HUGGINGFACE_HUB_CACHE=/data \ HF_HUB_ENABLE_HF_TRANSFER=1 \ MODEL_ID=bigscience/bloom-560m \ QUANTIZE=false \ NUM_SHARD=1 \ - PORT=80 \ - CUDA_HOME=/usr/local/cuda \ - LD_LIBRARY_PATH="/opt/conda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH" \ - PATH=$PATH:/opt/conda/bin:/usr/local/cuda/bin + PORT=80 -RUN apt-get update && apt-get install -y git curl libssl-dev ninja-build && rm -rf /var/lib/apt/lists/* - -RUN cd ~ && \ - curl -fsSL -v -o ~/mambaforge.sh -O "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-x86_64.sh" \ - chmod +x ~/mambaforge.sh && \ - bash ~/mambaforge.sh -b -p /opt/conda && \ - rm ~/mambaforge.sh +LABEL com.nvidia.volumes.needed="nvidia_driver" WORKDIR /usr/src -# Install torch -RUN pip install torch==2.0.0 --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libssl-dev \ + ca-certificates \ + make \ + && rm -rf /var/lib/apt/lists/* -# Install specific version of flash attention -COPY server/Makefile-flash-att server/Makefile -RUN cd server && make install-flash-attention +# Copy conda with PyTorch installed +COPY --from=pytorch-install /opt/conda /opt/conda -# Install specific version of transformers -COPY server/Makefile-transformers server/Makefile -RUN cd server && BUILD_EXTENSIONS="True" make install-transformers +# Copy build artifacts from flash attention builder +COPY --from=flash-att-builder /usr/src/flash-attention/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=flash-att-builder /usr/src/flash-attention/csrc/layer_norm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=flash-att-builder /usr/src/flash-attention/csrc/rotary/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages -COPY server/Makefile server/Makefile +# Copy build artifacts from transformers builder +COPY --from=transformers-builder /usr/src/transformers /usr/src/transformers +COPY --from=transformers-builder /usr/src/transformers/build/lib.linux-x86_64-cpython-39/transformers /usr/src/transformers/src/transformers + +# Install transformers dependencies +RUN cd /usr/src/transformers && pip install -e . --no-cache-dir && pip install einops --no-cache-dir # Install server COPY proto proto COPY server server +COPY server/Makefile server/Makefile RUN cd server && \ make gen-server && \ + pip install -r requirements.txt && \ pip install ".[bnb]" --no-cache-dir # Install router diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index d2c9aa7e..99d0ed3d 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -8,6 +8,7 @@ description = "Text Generation Launcher" [dependencies] clap = { version = "4.1.4", features = ["derive", "env"] } ctrlc = { version = "3.2.5", features = ["termination"] } +serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.93" subprocess = "0.2.9" tracing = "0.1.37" @@ -16,4 +17,3 @@ tracing-subscriber = { version = "0.3.16", features = ["json"] } [dev-dependencies] float_eq = "1.0.1" reqwest = { version = "0.11.14", features = ["blocking", "json"] } -serde = { version = "1.0.152", features = ["derive"] } diff --git a/launcher/src/main.rs b/launcher/src/main.rs index a598a8bb..0cf43f16 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -1,5 +1,5 @@ use clap::Parser; -use serde_json::Value; +use serde::Deserialize; use std::env; use std::ffi::OsString; use std::io::{BufRead, BufReader, Read}; @@ -244,11 +244,8 @@ fn main() -> ExitCode { let _span = tracing::span!(tracing::Level::INFO, "download").entered(); for line in stdout.lines() { // Parse loguru logs - if let Ok(value) = serde_json::from_str::(&line.unwrap()) { - if let Some(text) = value.get("text") { - // Format escaped newlines - tracing::info!("{}", text.to_string().replace("\\n", "")); - } + if let Ok(log) = serde_json::from_str::(&line.unwrap()) { + log.trace(); } } }); @@ -525,7 +522,7 @@ fn shard_manager( "--uds-path".to_string(), uds_path, "--logger-level".to_string(), - "ERROR".to_string(), + "INFO".to_string(), "--json-output".to_string(), ]; @@ -643,11 +640,8 @@ fn shard_manager( let _span = tracing::span!(tracing::Level::INFO, "shard-manager", rank = rank).entered(); for line in stdout.lines() { // Parse loguru logs - if let Ok(value) = serde_json::from_str::(&line.unwrap()) { - if let Some(text) = value.get("text") { - // Format escaped newlines - tracing::error!("{}", text.to_string().replace("\\n", "\n")); - } + if let Ok(log) = serde_json::from_str::(&line.unwrap()) { + log.trace(); } } }); @@ -708,3 +702,45 @@ fn num_cuda_devices() -> Option { } None } + +#[derive(Deserialize)] +#[serde(rename_all = "UPPERCASE")] +enum PythonLogLevelEnum { + Trace, + Debug, + Info, + Success, + Warning, + Error, + Critical, +} + +#[derive(Deserialize)] +struct PythonLogLevel { + name: PythonLogLevelEnum, +} + +#[derive(Deserialize)] +struct PythonLogRecord { + level: PythonLogLevel, +} + +#[derive(Deserialize)] +struct PythonLogMessage { + text: String, + record: PythonLogRecord, +} + +impl PythonLogMessage { + fn trace(&self) { + match self.record.level.name { + PythonLogLevelEnum::Trace => tracing::trace!("{}", self.text), + PythonLogLevelEnum::Debug => tracing::debug!("{}", self.text), + PythonLogLevelEnum::Info => tracing::info!("{}", self.text), + PythonLogLevelEnum::Success => tracing::info!("{}", self.text), + PythonLogLevelEnum::Warning => tracing::warn!("{}", self.text), + PythonLogLevelEnum::Error => tracing::error!("{}", self.text), + PythonLogLevelEnum::Critical => tracing::error!("{}", self.text), + } + } +} diff --git a/server/Makefile b/server/Makefile index d827ceca..89ddda40 100644 --- a/server/Makefile +++ b/server/Makefile @@ -16,6 +16,7 @@ install-torch: install: gen-server install-torch install-transformers pip install pip --upgrade + pip install -r requirements.txt pip install -e . --no-cache-dir run-dev: diff --git a/server/poetry.lock b/server/poetry.lock index 1ae18f9b..0ab6faa3 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -33,7 +33,7 @@ python-versions = ">=3.7,<4.0" [[package]] name = "bitsandbytes" -version = "0.35.4" +version = "0.38.1" description = "8-bit optimizers and matrix multiplication routines." category = "main" optional = false @@ -138,17 +138,17 @@ grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] [[package]] name = "grpc-interceptor" -version = "0.15.0" +version = "0.15.1" description = "Simplifies gRPC interceptors" category = "main" optional = false -python-versions = ">=3.6.1,<4.0.0" +python-versions = ">=3.7,<4.0" [package.dependencies] -grpcio = ">=1.32.0,<2.0.0" +grpcio = ">=1.49.1,<2.0.0" [package.extras] -testing = ["protobuf (>=3.6.0)"] +testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" @@ -597,7 +597,7 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "pytest" -version = "7.3.0" +version = "7.3.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false @@ -833,7 +833,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "6141d488429e0ab579028036e8e4cbc54f583b48214cb4a6be066bb7ce5154db" +content-hash = "e05491a03938b79a71b498f2759169f5a41181084158fde5993e7dcb25292cb0" [metadata.files] accelerate = [ @@ -845,8 +845,8 @@ backoff = [ {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] bitsandbytes = [ - {file = "bitsandbytes-0.35.4-py3-none-any.whl", hash = "sha256:201f168538ccfbd7594568a2f86c149cec8352782301076a15a783695ecec7fb"}, - {file = "bitsandbytes-0.35.4.tar.gz", hash = "sha256:b23db6b91cd73cb14faf9841a66bffa5c1722f9b8b57039ef2fb461ac22dd2a6"}, + {file = "bitsandbytes-0.38.1-py3-none-any.whl", hash = "sha256:5f532e7b1353eb7049ae831da2eb62ed8a1e0444116bd51b9e088a6e0bc7a34a"}, + {file = "bitsandbytes-0.38.1.tar.gz", hash = "sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce"}, ] certifi = [ {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, @@ -973,8 +973,8 @@ googleapis-common-protos = [ {file = "googleapis_common_protos-1.59.0-py2.py3-none-any.whl", hash = "sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f"}, ] grpc-interceptor = [ - {file = "grpc-interceptor-0.15.0.tar.gz", hash = "sha256:5c1aa9680b1d7e12259960c38057b121826860b05ebbc1001c74343b7ad1455e"}, - {file = "grpc_interceptor-0.15.0-py3-none-any.whl", hash = "sha256:63e390162e64df96c39c40508eb697def76a7cafac32a7eaf9272093eec1109e"}, + {file = "grpc-interceptor-0.15.1.tar.gz", hash = "sha256:3efadbc9aead272ac7a360c75c4bd96233094c9a5192dbb51c6156246bd64ba0"}, + {file = "grpc_interceptor-0.15.1-py3-none-any.whl", hash = "sha256:1cc52c34b0d7ff34512fb7780742ecda37bf3caa18ecc5f33f09b4f74e96b276"}, ] grpcio = [ {file = "grpcio-1.53.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:752d2949b40e12e6ad3ed8cc552a65b54d226504f6b1fb67cab2ccee502cc06f"}, @@ -1329,8 +1329,8 @@ psutil = [ {file = "psutil-5.9.4.tar.gz", hash = "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62"}, ] pytest = [ - {file = "pytest-7.3.0-py3-none-any.whl", hash = "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"}, - {file = "pytest-7.3.0.tar.gz", hash = "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d"}, + {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, + {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, ] PyYAML = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index c892cd3f..68dd9327 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -16,7 +16,7 @@ grpcio-reflection = "^1.51.1" grpc-interceptor = "^0.15.0" typer = "^0.6.1" accelerate = "^0.15.0" -bitsandbytes = "^0.35.1" +bitsandbytes = "^0.38.1" safetensors = "^0.2.4" loguru = "^0.6.0" opentelemetry-api = "^1.15.0" diff --git a/server/requirements.txt b/server/requirements.txt new file mode 100644 index 00000000..b1c84d15 --- /dev/null +++ b/server/requirements.txt @@ -0,0 +1,699 @@ +accelerate==0.15.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:014833307424cd0a22f89815802e00653756257c45dfdba2453e52d428931c65 \ + --hash=sha256:438e25a01afa6e3ffbd25353e76a68be49677c3050f10bfac7beafaf53503efc +backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba \ + --hash=sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8 +bitsandbytes==0.38.1 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:5f532e7b1353eb7049ae831da2eb62ed8a1e0444116bd51b9e088a6e0bc7a34a \ + --hash=sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce +certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4" \ + --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ + --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 +charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4" \ + --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ + --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ + --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ + --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ + --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ + --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ + --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ + --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ + --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ + --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ + --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ + --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ + --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ + --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ + --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ + --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ + --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ + --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ + --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ + --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ + --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ + --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ + --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ + --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ + --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ + --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ + --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ + --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ + --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ + --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ + --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ + --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ + --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ + --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ + --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ + --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ + --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ + --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ + --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ + --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ + --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ + --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ + --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ + --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ + --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ + --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ + --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ + --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ + --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ + --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ + --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ + --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ + --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ + --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ + --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ + --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ + --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ + --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ + --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ + --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ + --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ + --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ + --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ + --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ + --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ + --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ + --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ + --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ + --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ + --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ + --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ + --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ + --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ + --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ + --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab +click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ + --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 +cmake==3.26.3 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:1798547b23b89030518c5668dc55aed0e1d01867cf91d7a94e15d33f62a56fd0 \ + --hash=sha256:1bc7b47456256bdcc41069f5c658f232bd6e15bf4796d115f6ec98800793daff \ + --hash=sha256:24741a304ada699b339034958777d9a1472ac8ddb9b6194d74f814287ca091ae \ + --hash=sha256:27a6fa1b97744311a7993d6a1e0ce14bd73696dab9ceb96701f1ec11edbd5053 \ + --hash=sha256:2ae3db2c2be50fdaf0c9f3a23b2206e9dcd55ca124f16486a841b939f50b595e \ + --hash=sha256:46aa385e19c9e4fc95d7d6ce5ee0bbe0d69bdeac4e9bc95c61f78f3973c2f626 \ + --hash=sha256:4d3185738a6405aa15801e684f8d589b00570da4cc676cb1b5bbc902e3023e53 \ + --hash=sha256:543b6958d1615327f484a07ab041029b1740918a8baa336adc9f5f0cbcd8fbd8 \ + --hash=sha256:6e5fcd1cfaac33d015e2709e0dd1b7ad352a315367012ac359c9adc062cf075b \ + --hash=sha256:71e1df5587ad860b9829211380c42fc90ef2413363f12805b1fa2d87769bf876 \ + --hash=sha256:9d38ea5b4999f8f042a071bea3e213f085bac26d7ab54cb5a4c6a193c4baf132 \ + --hash=sha256:a922a6f6c1580d0db17b0b75f82e619441dd43c7f1d6a35f7d27e709db48bdbb \ + --hash=sha256:b20f7f7ea316ce7bb158df0e3c3453cfab5048939f1291017d16a8a36ad33ae6 \ + --hash=sha256:b54cde1f1c0573321b22382bd2ffaf5d08f65188572d128cd4867fb9669723c5 \ + --hash=sha256:cf910bbb488659d300c86b1dac77e44eeb0457bde2cf76a42d7e51f691544b21 \ + --hash=sha256:d3017a08e6ba53ec2486d89a7953a81d4c4a068fc9f29d83e209f295dd9c59f3 \ + --hash=sha256:e0ed796530641c8a21a423f9bb7882117dbbeee11ec78dbc335402a678d937ae +colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows" \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 +deprecated==1.2.13 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d \ + --hash=sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d +filelock==3.11.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37 \ + --hash=sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318 +googleapis-common-protos==1.59.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \ + --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f +grpc-interceptor==0.15.1 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:1cc52c34b0d7ff34512fb7780742ecda37bf3caa18ecc5f33f09b4f74e96b276 \ + --hash=sha256:3efadbc9aead272ac7a360c75c4bd96233094c9a5192dbb51c6156246bd64ba0 +grpcio-reflection==1.53.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:19ac2e3b16404f20bfa14ec64483bb4a936accee50ccb2e64b0159d1cea767cd \ + --hash=sha256:663ea8a9883a64457c1a58b95d52b431f4636337503954e15bfac1d1697b401c +grpcio-status==1.53.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:4f4bc140ed718a343d04a573c838c9fff5cb87f40d3cb2aa33b027c55031489d \ + --hash=sha256:5a46820dc7d94bac48aae5dd97c94cadea09d80a05553aca74a92a2954030420 +grpcio==1.53.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:0698c094688a2dd4c7c2f2c0e3e142cac439a64d1cef6904c97f6cde38ba422f \ + --hash=sha256:104a2210edd3776c38448b4f76c2f16e527adafbde171fc72a8a32976c20abc7 \ + --hash=sha256:14817de09317dd7d3fbc8272864288320739973ef0f4b56bf2c0032349da8cdf \ + --hash=sha256:1948539ce78805d4e6256ab0e048ec793956d54787dc9d6777df71c1d19c7f81 \ + --hash=sha256:19caa5b7282a89b799e63776ff602bb39604f7ca98db6df27e2de06756ae86c3 \ + --hash=sha256:1b172e6d497191940c4b8d75b53de82dc252e15b61de2951d577ec5b43316b29 \ + --hash=sha256:1c734a2d4843e4e14ececf5600c3c4750990ec319e1299db7e4f0d02c25c1467 \ + --hash=sha256:2a912397eb8d23c177d6d64e3c8bc46b8a1c7680b090d9f13a640b104aaec77c \ + --hash=sha256:2eddaae8af625e45b5c8500dcca1043264d751a6872cde2eda5022df8a336959 \ + --hash=sha256:55930c56b8f5b347d6c8c609cc341949a97e176c90f5cbb01d148d778f3bbd23 \ + --hash=sha256:658ffe1e39171be00490db5bd3b966f79634ac4215a1eb9a85c6cd6783bf7f6e \ + --hash=sha256:6601d812105583948ab9c6e403a7e2dba6e387cc678c010e74f2d6d589d1d1b3 \ + --hash=sha256:6b6d60b0958be711bab047e9f4df5dbbc40367955f8651232bfdcdd21450b9ab \ + --hash=sha256:6beb84f83360ff29a3654f43f251ec11b809dcb5524b698d711550243debd289 \ + --hash=sha256:752d2949b40e12e6ad3ed8cc552a65b54d226504f6b1fb67cab2ccee502cc06f \ + --hash=sha256:7dc8584ca6c015ad82e186e82f4c0fe977394588f66b8ecfc4ec873285314619 \ + --hash=sha256:82434ba3a5935e47908bc861ce1ebc43c2edfc1001d235d6e31e5d3ed55815f7 \ + --hash=sha256:8270d1dc2c98ab57e6dbf36fa187db8df4c036f04a398e5d5e25b4e01a766d70 \ + --hash=sha256:8a48fd3a7222be226bb86b7b413ad248f17f3101a524018cdc4562eeae1eb2a3 \ + --hash=sha256:95952d3fe795b06af29bb8ec7bbf3342cdd867fc17b77cc25e6733d23fa6c519 \ + --hash=sha256:976a7f24eb213e8429cab78d5e120500dfcdeb01041f1f5a77b17b9101902615 \ + --hash=sha256:9c84a481451e7174f3a764a44150f93b041ab51045aa33d7b5b68b6979114e48 \ + --hash=sha256:a34d6e905f071f9b945cabbcc776e2055de1fdb59cd13683d9aa0a8f265b5bf9 \ + --hash=sha256:a4952899b4931a6ba12951f9a141ef3e74ff8a6ec9aa2dc602afa40f63595e33 \ + --hash=sha256:a96c3c7f564b263c5d7c0e49a337166c8611e89c4c919f66dba7b9a84abad137 \ + --hash=sha256:aef7d30242409c3aa5839b501e877e453a2c8d3759ca8230dd5a21cda029f046 \ + --hash=sha256:b5bd026ac928c96cc23149e6ef79183125542062eb6d1ccec34c0a37e02255e7 \ + --hash=sha256:b6a2ead3de3b2d53119d473aa2f224030257ef33af1e4ddabd4afee1dea5f04c \ + --hash=sha256:ba074af9ca268ad7b05d3fc2b920b5fb3c083da94ab63637aaf67f4f71ecb755 \ + --hash=sha256:c5fb6f3d7824696c1c9f2ad36ddb080ba5a86f2d929ef712d511b4d9972d3d27 \ + --hash=sha256:c705e0c21acb0e8478a00e7e773ad0ecdb34bd0e4adc282d3d2f51ba3961aac7 \ + --hash=sha256:c7ad9fbedb93f331c2e9054e202e95cf825b885811f1bcbbdfdc301e451442db \ + --hash=sha256:da95778d37be8e4e9afca771a83424f892296f5dfb2a100eda2571a1d8bbc0dc \ + --hash=sha256:dad5b302a4c21c604d88a5d441973f320134e6ff6a84ecef9c1139e5ffd466f6 \ + --hash=sha256:dbc1ba968639c1d23476f75c356e549e7bbf2d8d6688717dcab5290e88e8482b \ + --hash=sha256:ddb2511fbbb440ed9e5c9a4b9b870f2ed649b7715859fd6f2ebc585ee85c0364 \ + --hash=sha256:df9ba1183b3f649210788cf80c239041dddcb375d6142d8bccafcfdf549522cd \ + --hash=sha256:e4f513d63df6336fd84b74b701f17d1bb3b64e9d78a6ed5b5e8a198bbbe8bbfa \ + --hash=sha256:e6f90698b5d1c5dd7b3236cd1fa959d7b80e17923f918d5be020b65f1c78b173 \ + --hash=sha256:eaf8e3b97caaf9415227a3c6ca5aa8d800fecadd526538d2bf8f11af783f1550 \ + --hash=sha256:ee81349411648d1abc94095c68cd25e3c2812e4e0367f9a9355be1e804a5135c \ + --hash=sha256:f144a790f14c51b8a8e591eb5af40507ffee45ea6b818c2482f0457fec2e1a2e \ + --hash=sha256:f3e837d29f0e1b9d6e7b29d569e2e9b0da61889e41879832ea15569c251c303a \ + --hash=sha256:fa8eaac75d3107e3f5465f2c9e3bbd13db21790c6e45b7de1756eba16b050aca \ + --hash=sha256:fdc6191587de410a184550d4143e2b24a14df495c86ca15e59508710681690ac +hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:0c1bdfa554f0b0936c1623b99058c6998a00fdcd86f75d9203f3f66572d2e30c \ + --hash=sha256:334862f4a82f8a09d6d3f550e67d7e498bb8882e678b7725638254fed3276801 \ + --hash=sha256:3ce6c5965a57d94db5e043aa488a4df929a32000db125d9c9a1d325e8c7006dc \ + --hash=sha256:445edfcd9d59c9d2541957177a9c9225b1f0e8855f6311fb16e20f67c3426421 \ + --hash=sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07 \ + --hash=sha256:606f2fe436e5be73f07987a56cd97c695805413d29203ae39ebd9fc596405435 \ + --hash=sha256:617692a70cf299576d82cfc860923f29ec5c834a3f5242bc0853d4f106670398 \ + --hash=sha256:6e5201b648df6106c232fcdb507db734081fd6220dfb1c432bd27c6fa9453331 \ + --hash=sha256:79099ac043423b263a2843a24213418f309d5c8bc458776622bffe012ebced73 \ + --hash=sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c \ + --hash=sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9 \ + --hash=sha256:8a7934c8e491bb395731f677f66dd5f6641432f338a3a9efc9f0b6c186d37cf8 \ + --hash=sha256:9913f5ad2644a1f57c1b7755a7d959ca5e0189863bb0473817d0707af230bf6a \ + --hash=sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6 \ + --hash=sha256:ca218fb6efc5c22379d9e64989e84bff426fcf5664fdbbf7cd70aa8b79497652 \ + --hash=sha256:d53954ecfaadc84c15481bf5d4c7282323196b4b6df1d1be54208d4fdedfb407 \ + --hash=sha256:d935946791e338f748e05a23df877d74fbcd39dc7b537f0aa2e5a5841cf7dde8 \ + --hash=sha256:da92a1483a66cf2baa96de133c75e7d5d9a60e4a0e60d228f26c573c73a1feb6 \ + --hash=sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9 +idna==3.4 ; python_version >= "3.9" and python_version < "4" \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 +jinja2==3.1.2 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ + --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 +lit==16.0.1 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:630a47291b714cb115015df23ab04267c24fe59aec7ecd7e637d5c75cdb45c91 +loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c \ + --hash=sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3 +markupsafe==2.1.2 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed \ + --hash=sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc \ + --hash=sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2 \ + --hash=sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460 \ + --hash=sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7 \ + --hash=sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0 \ + --hash=sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1 \ + --hash=sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa \ + --hash=sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03 \ + --hash=sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323 \ + --hash=sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65 \ + --hash=sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013 \ + --hash=sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036 \ + --hash=sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f \ + --hash=sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4 \ + --hash=sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419 \ + --hash=sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2 \ + --hash=sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619 \ + --hash=sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a \ + --hash=sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a \ + --hash=sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd \ + --hash=sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7 \ + --hash=sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666 \ + --hash=sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65 \ + --hash=sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859 \ + --hash=sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625 \ + --hash=sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff \ + --hash=sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156 \ + --hash=sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd \ + --hash=sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba \ + --hash=sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f \ + --hash=sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1 \ + --hash=sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094 \ + --hash=sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a \ + --hash=sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513 \ + --hash=sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed \ + --hash=sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d \ + --hash=sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3 \ + --hash=sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147 \ + --hash=sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c \ + --hash=sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603 \ + --hash=sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601 \ + --hash=sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a \ + --hash=sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1 \ + --hash=sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d \ + --hash=sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3 \ + --hash=sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54 \ + --hash=sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2 \ + --hash=sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6 \ + --hash=sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58 +mpmath==1.3.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ + --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c +networkx==3.1 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36 \ + --hash=sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61 +numpy==1.24.2 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22 \ + --hash=sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f \ + --hash=sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9 \ + --hash=sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96 \ + --hash=sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0 \ + --hash=sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a \ + --hash=sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281 \ + --hash=sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04 \ + --hash=sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468 \ + --hash=sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253 \ + --hash=sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756 \ + --hash=sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a \ + --hash=sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb \ + --hash=sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d \ + --hash=sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0 \ + --hash=sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910 \ + --hash=sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978 \ + --hash=sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5 \ + --hash=sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f \ + --hash=sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a \ + --hash=sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5 \ + --hash=sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2 \ + --hash=sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d \ + --hash=sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95 \ + --hash=sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5 \ + --hash=sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d \ + --hash=sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780 \ + --hash=sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa +nvidia-cublas-cu11==11.10.3.66 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e \ + --hash=sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded +nvidia-cuda-cupti-cu11==11.7.101 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:7cc5b8f91ae5e1389c3c0ad8866b3b016a175e827ea8f162a672990a402ab2b0 \ + --hash=sha256:e0cfd9854e1f2edaa36ca20d21cd0bdd5dcfca4e3b9e130a082e05b33b6c5895 +nvidia-cuda-nvrtc-cu11==11.7.99 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03 \ + --hash=sha256:f2effeb1309bdd1b3854fc9b17eaf997808f8b25968ce0c7070945c4265d64a3 \ + --hash=sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7 +nvidia-cuda-runtime-cu11==11.7.99 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7 \ + --hash=sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31 +nvidia-cudnn-cu11==8.5.0.96 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7 \ + --hash=sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108 +nvidia-cufft-cu11==10.9.0.58 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:222f9da70c80384632fd6035e4c3f16762d64ea7a843829cb278f98b3cb7dd81 \ + --hash=sha256:c4d316f17c745ec9c728e30409612eaf77a8404c3733cdf6c9c1569634d1ca03 +nvidia-curand-cu11==10.2.10.91 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:eecb269c970fa599a2660c9232fa46aaccbf90d9170b96c462e13bcb4d129e2c \ + --hash=sha256:f742052af0e1e75523bde18895a9ed016ecf1e5aa0ecddfcc3658fd11a1ff417 +nvidia-cusolver-cu11==11.4.0.1 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:00f70b256add65f8c1eb3b6a65308795a93e7740f6df9e273eccbba770d370c4 \ + --hash=sha256:700b781bfefd57d161443aff9ace1878584b93e0b2cfef3d6e9296d96febbf99 \ + --hash=sha256:72fa7261d755ed55c0074960df5904b65e2326f7adce364cbe4945063c1be412 +nvidia-cusparse-cu11==11.7.4.91 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:304a01599534f5186a8ed1c3756879282c72c118bc77dd890dc1ff868cad25b9 \ + --hash=sha256:a3389de714db63321aa11fbec3919271f415ef19fda58aed7f2ede488c32733d +nvidia-nccl-cu11==2.14.3 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:5e5534257d1284b8e825bc3a182c6f06acd6eb405e9f89d49340e98cd8f136eb +nvidia-nvtx-cu11==11.7.91 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:b22c64eee426a62fc00952b507d6d29cf62b4c9df7a480fcc417e540e05fd5ac \ + --hash=sha256:dfd7fcb2a91742513027d63a26b757f38dd8b07fecac282c4d132a9d373ff064 +opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded \ + --hash=sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0 +opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:844f2a4bb9bcda34e4eb6fe36765e5031aacb36dc60ed88c90fc246942ea26e7 \ + --hash=sha256:c2a5492ba7d140109968135d641d06ce3c5bd73c50665f787526065d57d7fd1d +opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:11b2c814249a49b22f6cca7a06b05701f561d577b747f3660dfd67b6eb9daf9c \ + --hash=sha256:3ec2a02196c8a54bf5cbf7fe623a5238625638e83b6047a983bdf96e2bbb74c0 +opentelemetry-exporter-otlp==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:4f7c49751d9720e2e726e13b0bb958ccade4e29122c305d92c033da432c8d2c5 \ + --hash=sha256:79f22748b6a54808a0448093dfa189c8490e729f67c134d4c992533d9393b33e +opentelemetry-instrumentation-grpc==0.36b0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:dc89447c9eb6ea868970f6c13b4ffdac182cdd5a41dd215a0f5393ca6375be55 \ + --hash=sha256:eaa246ed2083c97b13bab2555cb9d170e8433230a31476c4cab8a17fa03380a4 +opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:83ba4ae7d5292b5b33e0f851cc5c76d8f91196b9b3527800fc13855c33383ac2 \ + --hash=sha256:e3ddac9b3b93408ef26c8ecbf38f717042977e16381bb4cd329a5b4cf16998cf +opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:044b6d044b4d10530f250856f933442b8753a17f94ae37c207607f733fb9a844 \ + --hash=sha256:9c4008e40ac8cab359daac283fbe7002c5c29c77ea2674ad5626a249e64e0101 +opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:555c533e9837766119bbccc7a80458c9971d853a6f1da683a2246cd5e53b4645 \ + --hash=sha256:98dbffcfeebcbff12c0c974292d6ea603180a145904cf838b1fe4d5c99078425 +opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:829dc221795467d98b773c04096e29be038d77526dc8d6ac76f546fb6279bf01 \ + --hash=sha256:adc05635e87b9d3e007c9f530eed487fc3ef2177d02f82f674f28ebf9aff8243 +packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ + --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f +protobuf==4.22.3 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:13233ee2b9d3bd9a5f216c1fa2c321cd564b93d8f2e4f521a85b585447747997 \ + --hash=sha256:23452f2fdea754a8251d0fc88c0317735ae47217e0d27bf330a30eec2848811a \ + --hash=sha256:52f0a78141078077cfe15fe333ac3e3a077420b9a3f5d1bf9b5fe9d286b4d881 \ + --hash=sha256:70659847ee57a5262a65954538088a1d72dfc3e9882695cab9f0c54ffe71663b \ + --hash=sha256:7760730063329d42a9d4c4573b804289b738d4931e363ffbe684716b796bde51 \ + --hash=sha256:7cf56e31907c532e460bb62010a513408e6cdf5b03fb2611e4b67ed398ad046d \ + --hash=sha256:8b54f56d13ae4a3ec140076c9d937221f887c8f64954673d46f63751209e839a \ + --hash=sha256:d14fc1a41d1a1909998e8aff7e80d2a7ae14772c4a70e4bf7db8a36690b54425 \ + --hash=sha256:d4b66266965598ff4c291416be429cef7989d8fae88b55b62095a2331511b3fa \ + --hash=sha256:e0e630d8e6a79f48c557cd1835865b593d0547dce221c66ed1b827de59c66c97 \ + --hash=sha256:ecae944c6c2ce50dda6bf76ef5496196aeb1b85acb95df5843cd812615ec4b61 \ + --hash=sha256:f08aa300b67f1c012100d8eb62d47129e53d1150f4469fd78a29fa3cb68c66f2 \ + --hash=sha256:f2f4710543abec186aee332d6852ef5ae7ce2e9e807a3da570f36de5a732d88e +psutil==5.9.4 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff \ + --hash=sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1 \ + --hash=sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62 \ + --hash=sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549 \ + --hash=sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08 \ + --hash=sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7 \ + --hash=sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e \ + --hash=sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe \ + --hash=sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24 \ + --hash=sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad \ + --hash=sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94 \ + --hash=sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8 \ + --hash=sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7 \ + --hash=sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4 +pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1 \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7 \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358 \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 +requests==2.28.2 ; python_version >= "3.9" and python_version < "4" \ + --hash=sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa \ + --hash=sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf +safetensors==0.2.8 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:01000a4bfdf0474bb1bdf369de1284c93b4e6047524fe9dc55d77586cb9d0243 \ + --hash=sha256:02e67b906ad9bbb87308a34e4d2d33c9eb69baf342b7e5c872728556baf3f0b6 \ + --hash=sha256:1b905cf3563da4fe8dd46fa777321516f5aa8f6bc1b884158be03a235478e96d \ + --hash=sha256:2720b20a6a38c799dca79bd76caeeac2f7df585a9d4f7d59fa7e28eff9ccb27f \ + --hash=sha256:2a47327cfe57b6ee8c5dc246f9141f4f6b368e4444dd7f174c025b1d34446730 \ + --hash=sha256:2bd2e4c6258dd0f4e3d554f2f59789f25ef4757431e83c927016de6339e54811 \ + --hash=sha256:2f16e5ee70ae4218474493eff8d998e632a896a6b8aff9273511d654cdf367ab \ + --hash=sha256:2f40604c50d4a08a4c74f37fef735cd1e203a59aeda66ea23a01d76fb35cf407 \ + --hash=sha256:2f90513eee0a11902df7e51b07c3d9c328828b9dd692d6c74140bed932e7a491 \ + --hash=sha256:34705920c8a02f9ea6101bae8403db5f4aa18ec3eaccd8eab6701b1c88ee5bed \ + --hash=sha256:384df328523c567a8b2193ebb02456300a43a3adbc316823d4c0d16f7ac9e89d \ + --hash=sha256:466f92a384e4fcb6b1b9811e7488516c4638119c302a959b89bbe1be826d5e25 \ + --hash=sha256:50f7e90ed02ef5e4263aadce23d39a3595156816c42c4643003791b45f81fd31 \ + --hash=sha256:6273dd3edd5db05b2da0090bc3d491bf25f6d1d7e8a4423477377649e9e38c37 \ + --hash=sha256:70f322d170a17b6ecb9c85b15e67f4a9aa3e561594e2dfc7709c0ae0000ebfff \ + --hash=sha256:726ad66231286157dd505b5ab85fd903114dcb5f9180e79fd5540d68d6249dd0 \ + --hash=sha256:7716bab34ca9651895126c720df1bf07f464184a7409138179f38db488ca9f15 \ + --hash=sha256:832f27f6f379fb15d87f3a10eb87e2480434a89c483d7edc8a0c262364ba72c1 \ + --hash=sha256:89da823f9801e92b2c48e8fad1e2f7f0cb696a8a93dab4c6700e8de06fe87392 \ + --hash=sha256:8df8af89a0b6b535b47c077e33e5cd4941ef4b067e7c1dd1a05647dec0cf2eea \ + --hash=sha256:90cd22387b1520c4465033b986f79f0d24cc41aabae1903a22eff3b42cee9ad5 \ + --hash=sha256:9366140785d567049395c1c03ac69ee8d322fabcdc8fab7d389e933596b383da \ + --hash=sha256:9cabae651542b22d229b6439029fb4a3abc7868cb123af336b2877541ad9ab39 \ + --hash=sha256:a0c7ed4d75c2f248791dbe64309c98ada6f40a6949147ca2eaebd278906c918b \ + --hash=sha256:aa5f4b31f2e283b83894b388298368860367a1cb773228f9bb65f8db65da1549 \ + --hash=sha256:ac2197dbbf7cbd269faf8cebab4220dba5aa2ac8beacbce8fdcb9800776781ca \ + --hash=sha256:af4fce0565574ec3dbe997b021ed32d3dc229547c4f7fca2459be1f725c26f88 \ + --hash=sha256:ba3dc236a2344b7feadc9868307f42ba5e4804c9d68a80a35aac831349b31f6f \ + --hash=sha256:ba7c2496765de45a84f5c79b2b12a14a568643d8966ef9bb3f8b16217a39457c \ + --hash=sha256:c02c1cab1d23e6cc8ac1ef865efe76f2d0800e203c877288ebd31747f47a6940 \ + --hash=sha256:c5d720c7640ad5f95f47780bdc35777ed8371afa14d8d63d6375cfe36df83fb4 \ + --hash=sha256:c71e147a8c2942690e8707adbcc4ab60775bc78dfdbd7f9e696dd411adef82bb \ + --hash=sha256:d14e7c15e5acac6efcd5f5565e38b1b33600101387e5d16059de44adab87405f \ + --hash=sha256:dec20b9b1fc90b7b4e588b4f0e9e266bd8f26d405e08b0b6ecad3136d92d007a \ + --hash=sha256:eb28e5e6257f705828fd39b9ba28248b593f46e722d8d6beedbc8d1f194e2297 \ + --hash=sha256:ee8e169040468d176172b4f84a160ece2064abcc77294c4994a9d2bb5255cd75 \ + --hash=sha256:efff5ce2d3f349d5360a5ca5901ae56c7e24f8da623d57cd08f8e8b1cd9cb1f8 \ + --hash=sha256:f279e0fd917a886e1936d534734593f780afdddc6ed62f8ebf2f59de811cdd7c \ + --hash=sha256:f3c7242debe713a87ca6deaadb0d7120e61c92435414142d526e8c32da970020 \ + --hash=sha256:ff0096f5a765e6e3f7f3156f568f59499edeade19e813d610a124ca70b42cdda +sentencepiece==0.1.98 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:00b2becbd7b98905a6de9695cf8682abe0d510ab0198e23c7d86fb2b793b6ae0 \ + --hash=sha256:05b4eecbece0606883cd81ed86bb3c619680bb570b997b236533ec854d64a575 \ + --hash=sha256:11f410cc7eeb3e1cfa8d92d128b568e5dc7829b7904b164499fd0209316ec2fa \ + --hash=sha256:12c913493d6ebac86ee7ae109e368522a5a365a7b150d4d8cf845599262d2b21 \ + --hash=sha256:14841bd2a3d77c4dbba58f02488c374866551e428d755e8d473d82325a0a94f3 \ + --hash=sha256:1daf0a79cd953e4830746c41e92b98a2f2e9e5ec0e90a9447aa10350e11bd027 \ + --hash=sha256:2766cd708e9fc2b5b9784a990a8b303b9e0b9a69fa482616fe86fa538daa1756 \ + --hash=sha256:2c2d9a74986d3716dc6961e9dbae7a3b25bb1260118f098545fd963ae23252c1 \ + --hash=sha256:2d50edfc4649a1566b64f1a8402cd607e1893bf8e368732337d83f00df62d3fa \ + --hash=sha256:35af00f5103a4779694fedea41b6e24947a9ed81166efe63864ab1e781d70a66 \ + --hash=sha256:3f9239785849ed1f55a825bcc282bef1a6073f7431cc535bdc658a94873652ea \ + --hash=sha256:443f32e94b18571231b02a51be173686114b5556b5edfcbf347fb63e7bd5ddc6 \ + --hash=sha256:467740ef8af170e5b6cfe22c272114ed930c899c297619ac7a2ac463a13bdbac \ + --hash=sha256:472ad943eaffcb6871ece56c7850388e7b8722f520ba73c93e7a6ef965453221 \ + --hash=sha256:4b7e23aaf9d5afd91ca13550968bd17f0c17b0966823188ad2a50c51544cf8ed \ + --hash=sha256:515a971c2a157647ca0e60ce3c435f4b43cd5c9f5862159cfefa0b5b4d46d3c3 \ + --hash=sha256:558a373a8660bdff299d6c133c2a4f4fb0875e9e6fafe225b8080ecce8a405f9 \ + --hash=sha256:57911445fc91c80d59552adf8a749af9205458920a7328f3bd7d51308658bcd9 \ + --hash=sha256:58ca96d73ea0e5575e3f6a9524449c673d62e6ecee3b2ddd5bfb4f49cb315c0a \ + --hash=sha256:6150ba525fac4fda76f5c4777ae300597e70cef739ed2a47cea02ff81a88873f \ + --hash=sha256:64e32c55d04a2e21f0c2fda1b7a3dd108133ebfb8616b52896916bb30e4352ed \ + --hash=sha256:6b0ce9efc790c209cce2463058855dceb21438213d2ff13cb5a565d52a7efe25 \ + --hash=sha256:720f827dc69ee24951ea4f51b9fb69cc56890a7190fc52c2c0da2545caab1760 \ + --hash=sha256:7287461d2346f530928ab187f0834cb15ddfbc4553592cacdcb6470364739ec6 \ + --hash=sha256:7425b727c3d6b3b7bad0005a3be316078b254180b712d73955ff08cae3f6a385 \ + --hash=sha256:7f71c4bdedb797052fb2ccad0871c2409bf6f812cb6b651917c55f9e8eced07f \ + --hash=sha256:7f7dc2fc175623529fb60a2799748f8877cd48c4541b32cd97b8523465e88b69 \ + --hash=sha256:8663be00a68098f85d6cda1f7041a27de05c320e433fa730ecb1156a8304f21c \ + --hash=sha256:8abe5c4c034e497e69f485dcd2c0e6bc87bf0498ad5aef5f539a7d0f9eae6275 \ + --hash=sha256:8b50cbe8e46204eff7aa5a663af5652c45e7807aa560d08e5f5b10c60e795a49 \ + --hash=sha256:918b4caf18b2f73c302c4e197d1c2dafba39eb143d16b4590930b45f15042fdd \ + --hash=sha256:947cf0a4b8a480510d560a922f8256f34e93984a86cf870be4d05731f59fb28d \ + --hash=sha256:9b6f0b9ffb601e2699e265f3f20c353ec9a661e4b5f0cff08ad6c9909c0ae43e \ + --hash=sha256:b2531c0e9cc8cd404fabd856d80d695b373371c00f1fce29c06f41f3f7429d87 \ + --hash=sha256:b6ed62f89c0bd25cec39a7075f6b9354fe4c240ed964e63009d77efcf29c34e9 \ + --hash=sha256:c067ba22be8edc699f6365e01ec15046bf3563dbabfdc052ecc88e581b675cba \ + --hash=sha256:c23c3a562221bc40eaae42428fcd8e607e0f084ea8aa968ba3f1a7d0ea975807 \ + --hash=sha256:c5ea8fb2c68073fe25a08a178eed269ed382fba074ff2ba4de72f0f56d86630e \ + --hash=sha256:daf05611089a075b78d353720ccc3a09a78e0846332cff0cc78fda8b2383626a \ + --hash=sha256:e54aa70b574eee895d184072d84e62824f404821e551a82c619c5d4320a93834 \ + --hash=sha256:e7a828f1fe2e51d2d9e5e9b3283d4006f1891efb02a3d9303ed39ddafdd9c864 \ + --hash=sha256:ef384b31ec7a06a9a6aba42e68435f3f3b38809aa65559ede3658cdd446a562c \ + --hash=sha256:fa13a125417d28e84fbdebcaf6aa115e4177d3e93aa66b857a42e7179f515b88 \ + --hash=sha256:fcf100268cefe1774794b18cbaf3065e2bf988f168a387973eb1260d51198795 \ + --hash=sha256:ffcc78e80c55eab67ee3439ade493607a4e37e1f0b82b168ead3debf9eaeaabe +setuptools==67.6.1 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a \ + --hash=sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078 +sympy==1.11.1 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:938f984ee2b1e8eae8a07b884c8b7a1146010040fccddc6539c54f401c8f6fcf \ + --hash=sha256:e32380dce63cb7c0108ed525570092fd45168bdae2faa17e528221ef72e88658 +tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:00cee1e0859d55507e693a48fa4aef07060c4bb6bd93d80120e18fea9371c66d \ + --hash=sha256:0527dc5436a1f6bf2c0327da3145687d3bcfbeab91fed8458920093de3901b44 \ + --hash=sha256:07cbb2c307627dc99b44b22ef05ff4473aa7c7cc1fec8f0a8b37d8a64b1a16d2 \ + --hash=sha256:1f0e3b4c2ea2cd13238ce43548959c118069db7579e5d40ec270ad77da5833ce \ + --hash=sha256:280ffe95f50eaaf655b3a1dc7ff1d9cf4777029dbbc3e63a74e65a056594abc3 \ + --hash=sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e \ + --hash=sha256:310204dfed5aa797128b65d63538a9837cbdd15da2a29a77d67eefa489edda26 \ + --hash=sha256:3791338f809cd1bf8e4fee6b540b36822434d0c6c6bc47162448deee3f77d425 \ + --hash=sha256:4560dbdeaae5b7ee0d4e493027e3de6d53c991b5002d7ff95083c99e11dd5ac0 \ + --hash=sha256:48625a108029cb1ddf42e17a81b5a3230ba6888a70c9dc14e81bc319e812652d \ + --hash=sha256:4b3e3215d048e94f40f1c95802e45dcc37c5b05eb46280fc2ccc8cd351bff839 \ + --hash=sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07 \ + --hash=sha256:56b726e0d2bbc9243872b0144515ba684af5b8d8cd112fb83ee1365e26ec74c8 \ + --hash=sha256:5ef4215284df1277dadbcc5e17d4882bda19f770d02348e73523f7e7d8b8d396 \ + --hash=sha256:64064bd0322405c9374305ab9b4c07152a1474370327499911937fd4a76d004b \ + --hash=sha256:6cc29d410768f960db8677221e497226e545eaaea01aa3613fa0fdf2cc96cff4 \ + --hash=sha256:70ce07445050b537d2696022dafb115307abdffd2a5c106f029490f84501ef97 \ + --hash=sha256:8791dedba834c1fc55e5f1521be325ea3dafb381964be20684b92fdac95d79b7 \ + --hash=sha256:89649c00d0d7211e8186f7a75dfa1db6996f65edce4b84821817eadcc2d3c79e \ + --hash=sha256:8e7b0cdeace87fa9e760e6a605e0ae8fc14b7d72e9fc19c578116f7287bb873d \ + --hash=sha256:97acfcec592f7e9de8cadcdcda50a7134423ac8455c0166b28c9ff04d227b371 \ + --hash=sha256:9a3fa134896c3c1f0da6e762d15141fbff30d094067c8f1157b9fdca593b5806 \ + --hash=sha256:9ba2b0bf01777c9b9bc94b53764d6684554ce98551fec496f71bc5be3a03e98b \ + --hash=sha256:a0f9b92ea052305166559f38498b3b0cae159caea712646648aaa272f7160963 \ + --hash=sha256:a23ff602d0797cea1d0506ce69b27523b07e70f6dda982ab8cf82402de839088 \ + --hash=sha256:a4d53976079cff8a033f778fb9adca2d9d69d009c02fa2d71a878b5f3963ed30 \ + --hash=sha256:b8c6e2ab0f2e3d939ca66aa1d596602105fe33b505cd2854a4c1717f704c51de \ + --hash=sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783 \ + --hash=sha256:c2f35f30e39e6aab8716f07790f646bdc6e4a853816cc49a95ef2a9016bf9ce6 \ + --hash=sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc \ + --hash=sha256:cc5c022ce692e1f499d745af293ab9ee6f5d92538ed2faf73f9708c89ee59ce6 \ + --hash=sha256:cc78d77f597d1c458bf0ea7c2a64b6aa06941c7a99cb135b5969b0278824d808 \ + --hash=sha256:d607a6a13718aeb20507bdf2b96162ead5145bbbfa26788d6b833f98b31b26e1 \ + --hash=sha256:dd7730c98a3010cd4f523465867ff95cd9d6430db46676ce79358f65ae39797b \ + --hash=sha256:ecf182bf59bd541a8876deccf0360f5ae60496fd50b58510048020751cf1724c \ + --hash=sha256:ee0b1b311d65beab83d7a41c56a1e46ab732a9eed4460648e8eb0bd69fc2d059 \ + --hash=sha256:f247eae99800ef821a91f47c5280e9e9afaeed9980fc444208d5aa6ba69ff148 \ + --hash=sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33 \ + --hash=sha256:f55c981ac44ba87c93e847c333e58c12abcbb377a0c2f2ef96e1a266e4184ff2 \ + --hash=sha256:fc2a7fdf864554a0dacf09d32e17c0caa9afe72baf9dd7ddedc61973bae352d8 +torch==2.0.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:01858620f25f25e7a9ec4b547ff38e5e27c92d38ec4ccba9cfbfb31d7071ed9c \ + --hash=sha256:09651bff72e439d004c991f15add0c397c66f98ab36fe60d5514b44e4da722e8 \ + --hash=sha256:11b0384fe3c18c01b8fc5992e70fc519cde65e44c51cc87be1838c1803daf42f \ + --hash=sha256:2802f84f021907deee7e9470ed10c0e78af7457ac9a08a6cd7d55adef835fede \ + --hash=sha256:297a4919aff1c0f98a58ebe969200f71350a1d4d4f986dbfd60c02ffce780e99 \ + --hash=sha256:527f4ae68df7b8301ee6b1158ca56350282ea633686537b30dbb5d7b4a52622a \ + --hash=sha256:53e1c33c6896583cdb9a583693e22e99266444c4a43392dddc562640d39e542b \ + --hash=sha256:6befaad784004b7af357e3d87fa0863c1f642866291f12a4c2af2de435e8ac5c \ + --hash=sha256:6e0b97beb037a165669c312591f242382e9109a240e20054d5a5782d9236cad0 \ + --hash=sha256:7a9319a67294ef02459a19738bbfa8727bb5307b822dadd708bc2ccf6c901aca \ + --hash=sha256:8969aa8375bcbc0c2993e7ede0a7f889df9515f18b9b548433f412affed478d9 \ + --hash=sha256:9a2e53b5783ef5896a6af338b36d782f28e83c8ddfc2ac44b67b066d9d76f498 \ + --hash=sha256:9f01fe1f6263f31bd04e1757946fd63ad531ae37f28bb2dbf66f5c826ee089f4 \ + --hash=sha256:a83b26bd6ae36fbf5fee3d56973d9816e2002e8a3b7d9205531167c28aaa38a7 \ + --hash=sha256:ab2da16567cb55b67ae39e32d520d68ec736191d88ac79526ca5874754c32203 \ + --hash=sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7 \ + --hash=sha256:c7e67195e1c3e33da53954b026e89a8e1ff3bc1aeb9eb32b677172d4a9b5dcbf \ + --hash=sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a \ + --hash=sha256:cc788cbbbbc6eb4c90e52c550efd067586c2693092cf367c135b34893a64ae78 \ + --hash=sha256:ce9b5a49bd513dff7950a5a07d6e26594dd51989cee05ba388b03e8e366fd5d5 \ + --hash=sha256:d292640f0fd72b7a31b2a6e3b635eb5065fcbedd4478f9cad1a1e7a9ec861d35 \ + --hash=sha256:d439aec349c98f12819e8564b8c54008e4613dd4428582af0e6e14c24ca85870 \ + --hash=sha256:e54846aa63855298cfb1195487f032e413e7ac9cbfa978fda32354cc39551475 \ + --hash=sha256:ec5fff2447663e369682838ff0f82187b4d846057ef4d119a8dea7772a0b17dd +triton==2.0.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:0117722f8c2b579cd429e0bee80f7731ae05f63fe8e9414acd9a679885fcbf42 \ + --hash=sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a \ + --hash=sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1 \ + --hash=sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505 \ + --hash=sha256:42a0d2c3fc2eab4ba71384f2e785fbfd47aa41ae05fa58bf12cb31dcbd0aeceb \ + --hash=sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08 \ + --hash=sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd \ + --hash=sha256:52c47b72c72693198163ece9d90a721299e4fb3b8e24fd13141e384ad952724f \ + --hash=sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656 \ + --hash=sha256:75834f27926eab6c7f00ce73aaf1ab5bfb9bec6eb57ab7c0bfc0a23fac803b4c \ + --hash=sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2 \ + --hash=sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add \ + --hash=sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef \ + --hash=sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd \ + --hash=sha256:bcd9be5d0c2e45d2b7e6ddc6da20112b6862d69741576f9c3dbaf941d745ecae \ + --hash=sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b \ + --hash=sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a \ + --hash=sha256:fedce6a381901b1547e0e7e1f2546e4f65dca6d91e2d8a7305a2d1f5551895be +typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73 \ + --hash=sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e +typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \ + --hash=sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4 +urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4" \ + --hash=sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305 \ + --hash=sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42 +wheel==0.40.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873 \ + --hash=sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247 +win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" \ + --hash=sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2 \ + --hash=sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad +wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0 \ + --hash=sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420 \ + --hash=sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a \ + --hash=sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c \ + --hash=sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079 \ + --hash=sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923 \ + --hash=sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f \ + --hash=sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1 \ + --hash=sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8 \ + --hash=sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86 \ + --hash=sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0 \ + --hash=sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364 \ + --hash=sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e \ + --hash=sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c \ + --hash=sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e \ + --hash=sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c \ + --hash=sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727 \ + --hash=sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff \ + --hash=sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e \ + --hash=sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29 \ + --hash=sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7 \ + --hash=sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72 \ + --hash=sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475 \ + --hash=sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a \ + --hash=sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317 \ + --hash=sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2 \ + --hash=sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd \ + --hash=sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640 \ + --hash=sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98 \ + --hash=sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248 \ + --hash=sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e \ + --hash=sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d \ + --hash=sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec \ + --hash=sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1 \ + --hash=sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e \ + --hash=sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9 \ + --hash=sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92 \ + --hash=sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb \ + --hash=sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094 \ + --hash=sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46 \ + --hash=sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29 \ + --hash=sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd \ + --hash=sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705 \ + --hash=sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8 \ + --hash=sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975 \ + --hash=sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb \ + --hash=sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e \ + --hash=sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b \ + --hash=sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418 \ + --hash=sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019 \ + --hash=sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1 \ + --hash=sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba \ + --hash=sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6 \ + --hash=sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2 \ + --hash=sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3 \ + --hash=sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7 \ + --hash=sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752 \ + --hash=sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416 \ + --hash=sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f \ + --hash=sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1 \ + --hash=sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc \ + --hash=sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145 \ + --hash=sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee \ + --hash=sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a \ + --hash=sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7 \ + --hash=sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b \ + --hash=sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653 \ + --hash=sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0 \ + --hash=sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90 \ + --hash=sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29 \ + --hash=sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6 \ + --hash=sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034 \ + --hash=sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09 \ + --hash=sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559 \ + --hash=sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639 diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 4d48f492..94340fac 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -6,8 +6,6 @@ from loguru import logger from typing import Optional -from text_generation_server import server, utils -from text_generation_server.tracing import setup_tracing app = typer.Typer() @@ -48,6 +46,11 @@ def serve( backtrace=True, diagnose=False, ) + + # Import here after the logger is added to log potential import exceptions + from text_generation_server import server + from text_generation_server.tracing import setup_tracing + # Setup OpenTelemetry distributed tracing if otlp_endpoint is not None: setup_tracing(shard=os.getenv("RANK", 0), otlp_endpoint=otlp_endpoint) @@ -75,6 +78,9 @@ def download_weights( diagnose=False, ) + # Import here after the logger is added to log potential import exceptions + from text_generation_server import utils + # Test if files were already download try: utils.weight_files(model_id, revision, extension) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 368060a0..9c1ea3b0 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -26,7 +26,7 @@ FLASH_ATTENTION = torch.cuda.is_available() except ImportError: - logger.exception("Could not import Flash Attention enabled models") + logger.opt(exception=True).warning("Could not import Flash Attention enabled models") FLASH_ATTENTION = False __all__ = [ From c13b9d87c9f1a9776fa144007337bfa47369d1c3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 17 Apr 2023 16:51:53 +0200 Subject: [PATCH 168/793] fix(router): fix truncation (#190) closes #189 --- router/src/validation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/router/src/validation.rs b/router/src/validation.rs index d81a27e7..5f1b89b9 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -308,8 +308,8 @@ fn prepare_input( // Optionally truncate let (inputs, input_length) = match truncate { - // Truncate is some and > encoding length - Some(truncate) if truncate > encoding.len() => { + // Truncate is some and < encoding length + Some(truncate) if truncate < encoding.len() => { // truncate encoding and decode new inputs encoding.truncate(truncate, 0, TruncationDirection::Left); let inputs = tokenizer From b927244eb57d69fd92ecf36525f97e48f5b8b54f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 17 Apr 2023 18:43:24 +0200 Subject: [PATCH 169/793] feat(python-client): get list of currently deployed tgi models using the inference API (#191) --- README.md | 2 +- clients/python/README.md | 13 ++++ clients/python/pyproject.toml | 2 +- clients/python/tests/test_inference_api.py | 16 +++-- .../python/text_generation/inference_api.py | 61 +++++++++++-------- clients/python/text_generation/types.py | 6 ++ 6 files changed, 69 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 60c1a6ba..0c63f36b 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ to power LLMs api-inference widgets. ## Table of contents - [Features](#features) -- [Officially Supported Models](#officially-supported-models) +- [Optimized Architectures](#optimized-architectures) - [Get Started](#get-started) - [Docker](#docker) - [API Documentation](#api-documentation) diff --git a/clients/python/README.md b/clients/python/README.md index f509e65c..99ff185a 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -52,6 +52,14 @@ print(text) # ' Rayleigh scattering' ``` +Check all currently deployed models on the Huggingface Inference API with `Text Generation` support: + +```python +from text_generation.inference_api import deployed_models + +print(deployed_models()) +``` + ### Hugging Face Inference Endpoint usage ```python @@ -193,4 +201,9 @@ class StreamResponse: # Generation details # Only available when the generation is finished details: Optional[StreamDetails] + +# Inference API currently deployed model +class DeployedModel: + model_id: str + sha: str ``` \ No newline at end of file diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index d07f1dbc..1af5de99 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.4.1" +version = "0.5.0" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] diff --git a/clients/python/tests/test_inference_api.py b/clients/python/tests/test_inference_api.py index 79e503a3..59297c26 100644 --- a/clients/python/tests/test_inference_api.py +++ b/clients/python/tests/test_inference_api.py @@ -6,12 +6,20 @@ Client, AsyncClient, ) -from text_generation.errors import NotSupportedError -from text_generation.inference_api import get_supported_models +from text_generation.errors import NotSupportedError, NotFoundError +from text_generation.inference_api import check_model_support, deployed_models -def test_get_supported_models(): - assert isinstance(get_supported_models(), list) +def test_check_model_support(flan_t5_xxl, unsupported_model, fake_model): + assert check_model_support(flan_t5_xxl) + assert not check_model_support(unsupported_model) + + with pytest.raises(NotFoundError): + check_model_support(fake_model) + + +def test_deployed_models(): + deployed_models() def test_client(flan_t5_xxl): diff --git a/clients/python/text_generation/inference_api.py b/clients/python/text_generation/inference_api.py index eb70b3d1..31635e6e 100644 --- a/clients/python/text_generation/inference_api.py +++ b/clients/python/text_generation/inference_api.py @@ -1,44 +1,57 @@ import os import requests -import base64 -import json -import warnings -from typing import List, Optional +from typing import Optional, List from huggingface_hub.utils import build_hf_headers from text_generation import Client, AsyncClient, __version__ -from text_generation.errors import NotSupportedError +from text_generation.types import DeployedModel +from text_generation.errors import NotSupportedError, parse_error INFERENCE_ENDPOINT = os.environ.get( "HF_INFERENCE_ENDPOINT", "https://api-inference.huggingface.co" ) -SUPPORTED_MODELS = None - -def get_supported_models() -> Optional[List[str]]: +def deployed_models() -> List[DeployedModel]: """ - Get the list of supported text-generation models from GitHub + Get all currently deployed models with text-generation-inference-support Returns: - Optional[List[str]]: supported models list or None if unable to get the list from GitHub + List[DeployedModel]: list of all currently deployed models """ - global SUPPORTED_MODELS - if SUPPORTED_MODELS is not None: - return SUPPORTED_MODELS + resp = requests.get( + f"https://api-inference.huggingface.co/framework/text-generation-inference", + timeout=5, + ) + + payload = resp.json() + if resp.status_code != 200: + raise parse_error(resp.status_code, payload) + + models = [DeployedModel(**raw_deployed_model) for raw_deployed_model in payload] + return models + - response = requests.get( - "https://api.github.com/repos/huggingface/text-generation-inference/contents/supported_models.json", +def check_model_support(repo_id: str) -> bool: + """ + Check if a given model is supported by text-generation-inference + + Returns: + bool: whether the model is supported by this client + """ + resp = requests.get( + f"https://api-inference.huggingface.co/status/{repo_id}", timeout=5, ) - if response.status_code == 200: - file_content = response.json()["content"] - SUPPORTED_MODELS = json.loads(base64.b64decode(file_content).decode("utf-8")) - return SUPPORTED_MODELS - warnings.warn("Could not retrieve list of supported models.") - return None + payload = resp.json() + if resp.status_code != 200: + raise parse_error(resp.status_code, payload) + + framework = payload["framework"] + supported = framework == "text-generation-inference" + return supported class InferenceAPIClient(Client): @@ -83,8 +96,7 @@ def __init__(self, repo_id: str, token: Optional[str] = None, timeout: int = 10) """ # Text Generation Inference client only supports a subset of the available hub models - supported_models = get_supported_models() - if supported_models is not None and repo_id not in supported_models: + if not check_model_support(repo_id): raise NotSupportedError(repo_id) headers = build_hf_headers( @@ -140,8 +152,7 @@ def __init__(self, repo_id: str, token: Optional[str] = None, timeout: int = 10) """ # Text Generation Inference client only supports a subset of the available hub models - supported_models = get_supported_models() - if supported_models is not None and repo_id not in supported_models: + if not check_model_support(repo_id): raise NotSupportedError(repo_id) headers = build_hf_headers( diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index 21a9849b..f3f9dcb5 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -223,3 +223,9 @@ class StreamResponse(BaseModel): # Generation details # Only available when the generation is finished details: Optional[StreamDetails] + + +# Inference API currently deployed model +class DeployedModel(BaseModel): + model_id: str + sha: str From 2475aede619c0c6d2ba8440303432d505c77f6d3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 18 Apr 2023 16:16:06 +0200 Subject: [PATCH 170/793] feat(router): add info route (#196) close #125 --- Cargo.lock | 23 +++++ docs/openapi.json | 208 ++++++++++++++++++++++++++++++++++++++----- launcher/src/main.rs | 6 ++ router/Cargo.toml | 5 +- router/build.rs | 7 ++ router/src/lib.rs | 23 +++++ router/src/main.rs | 66 +++++++++----- router/src/server.rs | 188 +++++++++++++++++++++++--------------- 8 files changed, 409 insertions(+), 117 deletions(-) create mode 100644 router/build.rs diff --git a/Cargo.lock b/Cargo.lock index bc2529d4..8ffaf631 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2430,6 +2430,7 @@ dependencies = [ "tracing-subscriber", "utoipa", "utoipa-swagger-ui", + "vergen", ] [[package]] @@ -2468,8 +2469,10 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ + "itoa", "serde", "time-core", + "time-macros", ] [[package]] @@ -2478,6 +2481,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +[[package]] +name = "time-macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2966,6 +2978,17 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vergen" +version = "8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b86a8af1dedf089b1c78338678e4c7492b6045649042d94faf19690499d236" +dependencies = [ + "anyhow", + "rustversion", + "time", +] + [[package]] name = "version_check" version = "0.9.4" diff --git a/docs/openapi.json b/docs/openapi.json index 377c388a..1be99f07 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -4,8 +4,7 @@ "title": "Text Generation Inference", "description": "Text Generation Webserver", "contact": { - "name": "Olivier Dehaene", - "email": "olivier@huggingface.co" + "name": "Olivier Dehaene" }, "license": { "name": "Apache 2.0", @@ -14,6 +13,83 @@ "version": "0.5.0" }, "paths": { + "/": { + "post": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Generate tokens if `stream == false` or a stream of token if `stream == true`", + "description": "Generate tokens if `stream == false` or a stream of token if `stream == true`", + "operationId": "compat_generate", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CompatGenerateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "See /generate or /generate_stream" + }, + "422": { + "description": "Input validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Input validation error" + } + } + } + }, + "424": { + "description": "Generation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Request failed during generation" + } + } + } + }, + "429": { + "description": "Model is overloaded", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Model is overloaded" + } + } + } + }, + "500": { + "description": "Incomplete generation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Incomplete generation" + } + } + } + } + } + } + }, "/generate": { "post": { "tags": [ @@ -95,8 +171,7 @@ } } } - }, - "deprecated": false + } } }, "/generate_stream": { @@ -180,8 +255,29 @@ } } } - }, - "deprecated": false + } + } + }, + "/info": { + "get": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Text Generation Inference endpoint info", + "description": "Text Generation Inference endpoint info", + "operationId": "get_model_info", + "responses": { + "200": { + "description": "Served model info", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Info" + } + } + } + } + } } }, "/metrics": { @@ -203,8 +299,7 @@ } } } - }, - "deprecated": false + } } } }, @@ -230,7 +325,8 @@ "generated_tokens": { "type": "integer", "format": "int32", - "example": 1 + "example": 1, + "minimum": 0.0 }, "prefill": { "type": "array", @@ -242,7 +338,8 @@ "type": "integer", "format": "int64", "example": 42, - "nullable": true + "nullable": true, + "minimum": 0.0 }, "tokens": { "type": "array", @@ -252,6 +349,24 @@ } } }, + "CompatGenerateRequest": { + "type": "object", + "required": [ + "inputs" + ], + "properties": { + "inputs": { + "type": "string", + "example": "My name is Olivier and I" + }, + "parameters": { + "$ref": "#/components/schemas/GenerateParameters" + }, + "stream": { + "type": "boolean" + } + } + }, "Details": { "type": "object", "required": [ @@ -265,7 +380,8 @@ "type": "array", "items": { "$ref": "#/components/schemas/BestOfSequence" - } + }, + "nullable": true }, "finish_reason": { "$ref": "#/components/schemas/FinishReason" @@ -273,7 +389,8 @@ "generated_tokens": { "type": "integer", "format": "int32", - "example": 1 + "example": 1, + "minimum": 0.0 }, "prefill": { "type": "array", @@ -285,7 +402,8 @@ "type": "integer", "format": "int64", "example": 42, - "nullable": true + "nullable": true, + "minimum": 0.0 }, "tokens": { "type": "array", @@ -326,6 +444,7 @@ "default": "null", "example": 1, "nullable": true, + "minimum": 0.0, "exclusiveMinimum": 0.0 }, "details": { @@ -341,6 +460,7 @@ "type": "integer", "format": "int32", "default": "20", + "minimum": 0.0, "exclusiveMaximum": 512.0, "exclusiveMinimum": 0.0 }, @@ -364,6 +484,7 @@ "default": "null", "example": "null", "nullable": true, + "minimum": 0.0, "exclusiveMinimum": 0.0 }, "stop": { @@ -405,7 +526,8 @@ "type": "integer", "default": "null", "example": "null", - "nullable": true + "nullable": true, + "minimum": 0.0 }, "typical_p": { "type": "number", @@ -445,7 +567,12 @@ ], "properties": { "details": { - "$ref": "#/components/schemas/Details" + "allOf": [ + { + "$ref": "#/components/schemas/Details" + } + ], + "nullable": true }, "generated_text": { "type": "string", @@ -453,6 +580,38 @@ } } }, + "Info": { + "type": "object", + "required": [ + "model_id", + "version" + ], + "properties": { + "model_id": { + "type": "string", + "example": "bigscience/blomm-560m" + }, + "model_pipeline_tag": { + "type": "string", + "example": "text-generation", + "nullable": true + }, + "model_sha": { + "type": "string", + "example": "e985a63cdc139290c5f700ff1929f0b5942cced2", + "nullable": true + }, + "sha": { + "type": "string", + "example": "null", + "nullable": true + }, + "version": { + "type": "string", + "example": "0.5.0" + } + } + }, "PrefillToken": { "type": "object", "required": [ @@ -464,7 +623,8 @@ "id": { "type": "integer", "format": "int32", - "example": 0 + "example": 0, + "minimum": 0.0 }, "logprob": { "type": "number", @@ -491,13 +651,15 @@ "generated_tokens": { "type": "integer", "format": "int32", - "example": 1 + "example": 1, + "minimum": 0.0 }, "seed": { "type": "integer", "format": "int64", "example": 42, - "nullable": true + "nullable": true, + "minimum": 0.0 } } }, @@ -508,7 +670,12 @@ ], "properties": { "details": { - "$ref": "#/components/schemas/StreamDetails" + "allOf": [ + { + "$ref": "#/components/schemas/StreamDetails" + } + ], + "nullable": true }, "generated_text": { "type": "string", @@ -533,7 +700,8 @@ "id": { "type": "integer", "format": "int32", - "example": 0 + "example": 0, + "minimum": 0.0 }, "logprob": { "type": "number", diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 0cf43f16..8dc6a798 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -392,6 +392,12 @@ fn main() -> ExitCode { model_id, ]; + // Model optional revision + if let Some(ref revision) = revision { + argv.push("--revision".to_string()); + argv.push(revision.to_string()) + } + if json_output { argv.push("--json-output".to_string()); } diff --git a/router/Cargo.toml b/router/Cargo.toml index e2a6f5e9..aa8e9df2 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -4,6 +4,7 @@ version = "0.5.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" +build = "build.rs" [lib] path = "src/lib.rs" @@ -26,7 +27,7 @@ nohash-hasher = "0.2.0" opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } opentelemetry-otlp = "0.11.0" rand = "0.8.5" -reqwest = { version = "0.11.14", features = [] } +reqwest = { version = "0.11.14", features = [] } serde = "1.0.152" serde_json = "1.0.93" thiserror = "1.0.38" @@ -39,3 +40,5 @@ tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } utoipa = { version = "3.0.1", features = ["axum_extras"] } utoipa-swagger-ui = { version = "3.0.2", features = ["axum"] } +[build-dependencies] +vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } diff --git a/router/build.rs b/router/build.rs new file mode 100644 index 00000000..c34f9fa8 --- /dev/null +++ b/router/build.rs @@ -0,0 +1,7 @@ +use std::error::Error; +use vergen::EmitBuilder; + +fn main() -> Result<(), Box> { + EmitBuilder::builder().git_sha(false).emit()?; + Ok(()) +} diff --git a/router/src/lib.rs b/router/src/lib.rs index 91b4417c..7dc115fd 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -10,6 +10,29 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use validation::Validation; +/// Hub type +#[derive(Clone, Debug, Deserialize)] +pub struct ModelInfo { + #[serde(rename(deserialize = "id"))] + pub model_id: String, + pub sha: Option, + pub pipeline_tag: Option, +} + +#[derive(Clone, Debug, Serialize, ToSchema)] +pub struct Info { + #[schema(example = "bigscience/blomm-560m")] + pub model_id: String, + #[schema(nullable = true, example = "e985a63cdc139290c5f700ff1929f0b5942cced2")] + pub model_sha: Option, + #[schema(nullable = true, example = "text-generation")] + pub model_pipeline_tag: Option, + #[schema(example = "0.5.0")] + pub version: &'static str, + #[schema(nullable = true, example = "null")] + pub sha: Option<&'static str>, +} + #[derive(Clone, Debug, Deserialize, ToSchema)] pub(crate) struct GenerateParameters { #[serde(default)] diff --git a/router/src/main.rs b/router/src/main.rs index 3ff72cde..5fda57be 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -10,8 +10,8 @@ use opentelemetry_otlp::WithExportConfig; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path; use text_generation_client::ShardedClient; -use text_generation_router::server; -use tokenizers::Tokenizer; +use text_generation_router::{server, ModelInfo}; +use tokenizers::{FromPretrainedParameters, Tokenizer}; use tower_http::cors::AllowOrigin; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; @@ -41,6 +41,8 @@ struct Args { master_shard_uds_path: String, #[clap(default_value = "bigscience/bloom", long, env)] tokenizer_name: String, + #[clap(default_value = "main", long, env)] + revision: String, #[clap(default_value = "2", long, env)] validation_workers: usize, #[clap(long, env)] @@ -66,6 +68,7 @@ fn main() -> Result<(), std::io::Error> { port, master_shard_uds_path, tokenizer_name, + revision, validation_workers, json_output, otlp_endpoint, @@ -90,16 +93,19 @@ fn main() -> Result<(), std::io::Error> { // Tokenizer instance // This will only be used to validate payloads let local_path = Path::new(&tokenizer_name); - let tokenizer = - if local_path.exists() && local_path.is_dir() && local_path.join("tokenizer.json").exists() - { - // Load local tokenizer - Tokenizer::from_file(local_path.join("tokenizer.json")).ok() - } else { - // Download and instantiate tokenizer - // We need to download it outside of the Tokio runtime - Tokenizer::from_pretrained(tokenizer_name.clone(), None).ok() + let local_model = local_path.exists() && local_path.is_dir(); + let tokenizer = if local_model { + // Load local tokenizer + Tokenizer::from_file(local_path.join("tokenizer.json")).ok() + } else { + // Download and instantiate tokenizer + // We need to download it outside of the Tokio runtime + let params = FromPretrainedParameters { + revision: revision.clone(), + ..Default::default() }; + Tokenizer::from_pretrained(tokenizer_name.clone(), Some(params)).ok() + }; // Launch Tokio runtime tokio::runtime::Builder::new_multi_thread() @@ -116,25 +122,23 @@ fn main() -> Result<(), std::io::Error> { tracing::warn!("Rust input length validation and truncation is disabled"); } - // Get pipeline tag - let model_info = reqwest::get(format!( - "https://huggingface.co/api/models/{tokenizer_name}" - )) - .await - .expect("Could not connect to hf.co") - .text() - .await - .expect("error when retrieving model info from hf.co"); - let model_info: serde_json::Value = - serde_json::from_str(&model_info).expect("unable to parse model info"); + // Get Model info + let model_info = match local_model { + true => ModelInfo { + model_id: tokenizer_name.clone(), + sha: None, + pipeline_tag: None, + }, + false => get_model_info(&tokenizer_name, &revision).await, + }; // if pipeline-tag == text-generation we default to return_full_text = true - let compat_return_full_text = match model_info.get("pipeline_tag") { + let compat_return_full_text = match &model_info.pipeline_tag { None => { tracing::warn!("no pipeline tag found for model {tokenizer_name}"); false } - Some(pipeline_tag) => pipeline_tag.as_str() == Some("text-generation"), + Some(pipeline_tag) => pipeline_tag.as_str() == "text-generation", }; // Instantiate sharded client from the master unix socket @@ -153,6 +157,7 @@ fn main() -> Result<(), std::io::Error> { // Run server server::run( + model_info, compat_return_full_text, max_concurrent_requests, max_best_of, @@ -226,3 +231,16 @@ fn init_logging(otlp_endpoint: Option, json_output: bool) { .with(layers) .init(); } + +/// get model info from the Huggingface Hub +pub async fn get_model_info(model_id: &str, revision: &str) -> ModelInfo { + let model_info = reqwest::get(format!( + "https://huggingface.co/api/models/{model_id}/revision/{revision}" + )) + .await + .expect("Could not connect to hf.co") + .text() + .await + .expect("error when retrieving model info from hf.co"); + serde_json::from_str(&model_info).expect("unable to parse model info") +} diff --git a/router/src/server.rs b/router/src/server.rs index 851837d5..ce301399 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -3,8 +3,8 @@ use crate::infer::{InferError, InferResponse, InferStreamResponse}; use crate::validation::ValidationError; use crate::{ BestOfSequence, CompatGenerateRequest, Details, ErrorResponse, FinishReason, - GenerateParameters, GenerateRequest, GenerateResponse, Infer, PrefillToken, StreamDetails, - StreamResponse, Token, Validation, + GenerateParameters, GenerateRequest, GenerateResponse, Infer, Info, ModelInfo, PrefillToken, + StreamDetails, StreamResponse, Token, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; @@ -27,7 +27,24 @@ use tracing::{info_span, instrument, Instrument}; use utoipa::OpenApi; use utoipa_swagger_ui::SwaggerUi; -/// Compatibility route with api-inference and AzureML +/// Generate tokens if `stream == false` or a stream of token if `stream == true` +#[utoipa::path( + post, + tag = "Text Generation Inference", + path = "/", + request_body = CompatGenerateRequest, + responses( + (status = 200, description = "See /generate or /generate_stream"), + (status = 424, description = "Generation Error", body = ErrorResponse, + example = json ! ({"error": "Request failed during generation"})), + (status = 429, description = "Model is overloaded", body = ErrorResponse, + example = json ! ({"error": "Model is overloaded"})), + (status = 422, description = "Input validation error", body = ErrorResponse, + example = json ! ({"error": "Input validation error"})), + (status = 500, description = "Incomplete generation", body = ErrorResponse, + example = json ! ({"error": "Incomplete generation"})), + ) +)] #[instrument(skip(infer))] async fn compat_generate( default_return_full_text: Extension, @@ -53,6 +70,26 @@ async fn compat_generate( } } +/// Text Generation Inference endpoint info +#[utoipa::path( + get, + tag = "Text Generation Inference", + path = "/info", + responses((status = 200, description = "Served model info", body = Info)) +)] +#[instrument] +async fn get_model_info(model_info: Extension) -> Json { + let model_info = model_info.0; + let info = Info { + version: env!("CARGO_PKG_VERSION"), + sha: option_env!("VERGEN_GIT_SHA"), + model_id: model_info.model_id, + model_sha: model_info.sha, + model_pipeline_tag: model_info.pipeline_tag, + }; + Json(info) +} + /// Health check method #[instrument(skip(infer))] async fn health(infer: Extension) -> Result<(), (StatusCode, Json)> { @@ -87,21 +124,21 @@ async fn health(infer: Extension) -> Result<(), (StatusCode, Json) -> String { prom_handle.render() @@ -459,6 +496,7 @@ async fn metrics(prom_handle: Extension) -> String { /// Serving method #[allow(clippy::too_many_arguments)] pub async fn run( + model_info: ModelInfo, compat_return_full_text: bool, max_concurrent_requests: usize, max_best_of: usize, @@ -476,36 +514,40 @@ pub async fn run( // OpenAPI documentation #[derive(OpenApi)] #[openapi( - paths( - generate, - generate_stream, - metrics, - ), - components( - schemas( - GenerateRequest, - GenerateParameters, - PrefillToken, - Token, - GenerateResponse, - BestOfSequence, - Details, - FinishReason, - StreamResponse, - StreamDetails, - ErrorResponse, - ) - ), - tags( - (name = "Text Generation Inference", description = "Hugging Face Text Generation Inference API") - ), - info( - title = "Text Generation Inference", - license( - name = "Apache 2.0", - url = "https://www.apache.org/licenses/LICENSE-2.0" - ) - ) + paths( + get_model_info, + compat_generate, + generate, + generate_stream, + metrics, + ), + components( + schemas( + Info, + CompatGenerateRequest, + GenerateRequest, + GenerateParameters, + PrefillToken, + Token, + GenerateResponse, + BestOfSequence, + Details, + FinishReason, + StreamResponse, + StreamDetails, + ErrorResponse, + ) + ), + tags( + (name = "Text Generation Inference", description = "Hugging Face Text Generation Inference API") + ), + info( + title = "Text Generation Inference", + license( + name = "Apache 2.0", + url = "https://www.apache.org/licenses/LICENSE-2.0" + ) + ) )] struct ApiDoc; @@ -584,6 +626,7 @@ pub async fn run( .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", ApiDoc::openapi())) // Base routes .route("/", post(compat_generate)) + .route("/info", get(get_model_info)) .route("/generate", post(generate)) .route("/generate_stream", post(generate_stream)) // AWS Sagemaker route @@ -596,6 +639,7 @@ pub async fn run( .route("/ping", get(health)) // Prometheus metrics route .route("/metrics", get(metrics)) + .layer(Extension(model_info)) .layer(Extension(compat_return_full_text)) .layer(Extension(infer)) .layer(Extension(prom_handle)) From e14ae3b5e9e8948f4113a8260e699fc8d46ededd Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 19 Apr 2023 12:51:11 +0200 Subject: [PATCH 171/793] feat(server): support quantization for flash models (#200) closes #197 --- .../text_generation_server/models/__init__.py | 8 +- .../custom_modeling/flash_llama_modeling.py | 62 +++++++++++---- .../custom_modeling/flash_neox_modeling.py | 77 +++++++++++++++---- .../flash_santacoder_modeling.py | 62 +++++++++++---- .../models/flash_causal_lm.py | 6 +- .../models/flash_llama.py | 19 ++--- .../models/flash_neox.py | 13 ++-- .../models/flash_santacoder.py | 32 ++++---- 8 files changed, 196 insertions(+), 83 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 9c1ea3b0..13c74c91 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -26,7 +26,9 @@ FLASH_ATTENTION = torch.cuda.is_available() except ImportError: - logger.opt(exception=True).warning("Could not import Flash Attention enabled models") + logger.opt(exception=True).warning( + "Could not import Flash Attention enabled models" + ) FLASH_ATTENTION = False __all__ = [ @@ -88,10 +90,10 @@ def get_model( raise NotImplementedError( FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Santacoder") ) - return FlashSantacoderSharded(model_id, revision=revision) + return FlashSantacoderSharded(model_id, revision, quantize=quantize) else: santacoder_cls = FlashSantacoder if FLASH_ATTENTION else SantaCoder - return santacoder_cls(model_id, revision, quantize) + return santacoder_cls(model_id, revision, quantize=quantize) config = AutoConfig.from_pretrained(model_id, revision=revision) model_type = config.model_type diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 508b7746..71843886 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -33,6 +33,12 @@ from flash_attn.layers.rotary import RotaryEmbedding +HAS_BITS_AND_BYTES = True +try: + from bitsandbytes.nn import Linear8bitLt +except ImportError as e: + HAS_BITS_AND_BYTES = False + class LlamaRMSNorm(nn.Module): def __init__(self, hidden_size, eps=1e-6): @@ -94,14 +100,44 @@ def __init__( dtype=None, ) -> None: super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) + self.quantized = False + self.bnb_linear = None + + def prepare_weights(self, quantize: bool = False): + if quantize: + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine either because it is not installed " + "or you don't have a GPU.\n" + "You can install it with `pip install bitsandbytes`." + ) - def transpose_weight(self): - self.weight = nn.Parameter(self.weight.T) + self.quantized = True + self.bnb_linear = Linear8bitLt( + self.in_features, + self.out_features, + has_fp16_weights=False, + threshold=6.0, + bias=False, + ) + # Copy data to bnb_linear + self.bnb_linear.weight.data = self.weight.data + if self.bias is not None: + self.bnb_linear.bias = nn.Parameter(self.bias) + + # Delete reference to data + self.weight = None + self.bias = None + else: + self.weight = nn.Parameter(self.weight.T) def forward(self, input: torch.Tensor) -> torch.Tensor: - if self.bias is not None: - return torch.addmm(self.bias, input, self.weight) - return torch.matmul(input, self.weight) + if self.quantized: + return self.bnb_linear(input) + else: + if self.bias is not None: + return torch.addmm(self.bias, input, self.weight) + return torch.matmul(input, self.weight) class TensorParallelColumnLinear(FastLinear): @@ -502,15 +538,15 @@ def __init__(self, config, process_group=None): self.head_size = self.layers[0].self_attn.head_size self.num_heads = self.layers[0].self_attn.num_heads - def post_load_weights(self): + def post_load_weights(self, load_in_8bit: bool = False): if isinstance(self.embed_tokens, TensorParallelEmbedding): self.embed_tokens.add_null_idx() for layer in self.layers: layer: FlashLlamaLayer - layer.self_attn.query_key_value.transpose_weight() - layer.self_attn.o_proj.transpose_weight() - layer.mlp.gate_up_proj.transpose_weight() - layer.mlp.down_proj.transpose_weight() + layer.self_attn.query_key_value.prepare_weights(load_in_8bit) + layer.self_attn.o_proj.prepare_weights(load_in_8bit) + layer.mlp.gate_up_proj.prepare_weights(load_in_8bit) + layer.mlp.down_proj.prepare_weights(load_in_8bit) def forward( self, @@ -592,9 +628,9 @@ def __init__(self, config, process_group=None): else: self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) - def post_load_weights(self): - self.model.post_load_weights() - self.lm_head.transpose_weight() + def post_load_weights(self, load_in_8bit: bool = False): + self.model.post_load_weights(load_in_8bit) + self.lm_head.prepare_weights() def forward( self, diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 16fd4091..8de582e3 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -35,6 +35,12 @@ from flash_attn.layers.rotary import RotaryEmbedding +HAS_BITS_AND_BYTES = True +try: + from bitsandbytes.nn import Linear8bitLt +except ImportError as e: + HAS_BITS_AND_BYTES = False + class FastLayerNorm(nn.LayerNorm): def forward(self, hidden_states, residual=None): @@ -82,14 +88,44 @@ def __init__( dtype=None, ) -> None: super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) + self.quantized = False + self.bnb_linear = None + + def prepare_weights(self, quantize: bool = False): + if quantize: + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine either because it is not installed " + "or you don't have a GPU.\n" + "You can install it with `pip install bitsandbytes`." + ) - def transpose_weight(self): - self.weight = nn.Parameter(self.weight.T) + self.quantized = True + self.bnb_linear = Linear8bitLt( + self.in_features, + self.out_features, + has_fp16_weights=False, + threshold=6.0, + bias=False, + ) + # Copy data to bnb_linear + self.bnb_linear.weight.data = self.weight.data + if self.bias is not None: + self.bnb_linear.bias = nn.Parameter(self.bias) + + # Delete reference to data + self.weight = None + self.bias = None + else: + self.weight = nn.Parameter(self.weight.T) def forward(self, input: torch.Tensor) -> torch.Tensor: - if self.bias is not None: - return torch.addmm(self.bias, input, self.weight) - return torch.matmul(input, self.weight) + if self.quantized: + return self.bnb_linear(input) + else: + if self.bias is not None: + return torch.addmm(self.bias, input, self.weight) + return torch.matmul(input, self.weight) class TensorParallelColumnLinear(FastLinear): @@ -552,23 +588,27 @@ def __init__(self, config, process_group=None): self.head_size = self.layers[0].attention.head_size self.num_heads = self.layers[0].attention.num_heads - def post_load_weights(self): + def post_load_weights(self, load_in_8bit=False): if isinstance(self.embed_in, TensorParallelEmbedding): self.embed_in.add_null_idx() for layer in self.layers: layer: FlashNeoXLayer layer.attention.shuffle_qkv_dims() - layer.attention.query_key_value.transpose_weight() - layer.attention.dense.transpose_weight() - layer.mlp.dense_h_to_4h.transpose_weight() - layer.mlp.dense_4h_to_h.transpose_weight() + layer.attention.query_key_value.prepare_weights(load_in_8bit) + layer.attention.dense.prepare_weights(load_in_8bit) + layer.mlp.dense_h_to_4h.prepare_weights(load_in_8bit) + layer.mlp.dense_4h_to_h.prepare_weights(load_in_8bit) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + # Pop here as we will replace the layer in our own logic and don't want from_pretrained + # to do it for us + load_in_8bit = kwargs.pop("load_in_8bit", False) model = super(FlashGPTNeoXModel, cls).from_pretrained( - pretrained_model_name_or_path, *model_args, **kwargs + pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs ) - model.post_load_weights() + + model.post_load_weights(load_in_8bit) return model def forward( @@ -653,16 +693,19 @@ def __init__(self, config, process_group=None): config.hidden_size, config.vocab_size, bias=False ) - def post_load_weights(self): - self.gpt_neox.post_load_weights() - self.embed_out.transpose_weight() + def post_load_weights(self, load_in_8bit=False): + self.gpt_neox.post_load_weights(load_in_8bit) + self.embed_out.prepare_weights() @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + # Pop here as we will replace the layer in our own logic and don't want from_pretrained + # to do it for us + load_in_8bit = kwargs.pop("load_in_8bit", False) model = super(FlashGPTNeoXForCausalLM, cls).from_pretrained( - pretrained_model_name_or_path, *model_args, **kwargs + pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs ) - model.post_load_weights() + model.post_load_weights(load_in_8bit) return model def forward( diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 8679826b..793b3d11 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -10,6 +10,12 @@ import flash_attn_cuda import dropout_layer_norm +HAS_BITS_AND_BYTES = True +try: + from bitsandbytes.nn import Linear8bitLt +except ImportError as e: + HAS_BITS_AND_BYTES = False + class FastLayerNorm(nn.LayerNorm): def forward(self, hidden_states, residual=None): @@ -57,14 +63,44 @@ def __init__( dtype=None, ) -> None: super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) + self.quantized = False + self.bnb_linear = None + + def prepare_weights(self, quantize: bool = False): + if quantize: + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine either because it is not installed " + "or you don't have a GPU.\n" + "You can install it with `pip install bitsandbytes`." + ) - def transpose_weight(self): - self.weight = nn.Parameter(self.weight.T) + self.quantized = True + self.bnb_linear = Linear8bitLt( + self.in_features, + self.out_features, + has_fp16_weights=False, + threshold=6.0, + bias=False, + ) + # Copy data to bnb_linear + self.bnb_linear.weight.data = self.weight.data + if self.bias is not None: + self.bnb_linear.bias = nn.Parameter(self.bias) + + # Delete reference to data + self.weight = None + self.bias = None + else: + self.weight = nn.Parameter(self.weight.T) def forward(self, input: torch.Tensor) -> torch.Tensor: - if self.bias is not None: - return torch.addmm(self.bias, input, self.weight) - return torch.matmul(input, self.weight) + if self.quantized: + return self.bnb_linear(input) + else: + if self.bias is not None: + return torch.addmm(self.bias, input, self.weight) + return torch.matmul(input, self.weight) class TensorParallelColumnLinear(FastLinear): @@ -431,16 +467,16 @@ def __init__(self, config, process_group=None): self.head_size = self.h[0].attn.head_size self.num_heads = self.h[0].attn.num_heads - def post_load_weights(self): + def post_load_weights(self, load_in_8bit: bool = False): if self.tp_embeddings: self.wte.add_null_idx() self.wpe.add_null_idx() for layer in self.h: layer: Block - layer.attn.c_attn.transpose_weight() - layer.attn.c_proj.transpose_weight() - layer.mlp.c_fc.transpose_weight() - layer.mlp.c_proj.transpose_weight() + layer.attn.c_attn.prepare_weights(load_in_8bit) + layer.attn.c_proj.prepare_weights(load_in_8bit) + layer.mlp.c_fc.prepare_weights(load_in_8bit) + layer.mlp.c_proj.prepare_weights(load_in_8bit) def forward( self, @@ -508,9 +544,9 @@ def __init__(self, config, process_group=None): else: self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) - def post_load_weights(self): - self.transformer.post_load_weights() - self.lm_head.transpose_weight() + def post_load_weights(self, load_in_8bit: bool = False): + self.transformer.post_load_weights(load_in_8bit) + self.lm_head.prepare_weights() def forward( self, diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 507fec0f..3d4273cc 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -221,9 +221,6 @@ def __init__( else: raise NotImplementedError("FlashCausalLM is only available on GPU") - if quantize: - raise NotImplementedError("FlashCausalLM does not support quantization") - tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", truncation_side="left" ) @@ -232,9 +229,10 @@ def __init__( model_id, revision=revision, torch_dtype=dtype, + load_in_8bit=quantize, ) .eval() - .cuda() + .to(device) ) super(FlashCausalLM, self).__init__( diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 063910f4..9cbf1b57 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -35,9 +35,6 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False else: raise NotImplementedError("FlashLlama is only available on GPU") - if quantize: - raise NotImplementedError("FlashLlama does not support quantization") - tokenizer = LlamaTokenizer.from_pretrained( model_id, revision=revision, @@ -61,8 +58,8 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False with init_empty_weights(): model = FlashLlamaForCausalLM(config) - self.load_weights(model, filenames, device, dtype) - self.model = model.eval() + self.load_weights(model, filenames, quantize, device, dtype) + self.model = model.eval().to(device) super(FlashCausalLM, self).__init__( tokenizer=tokenizer, @@ -73,13 +70,14 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False def load_weights( model, filenames: List[Path], + quantize: bool, device: torch.device, dtype: torch.dtype, ): for filename in filenames: state_dict = torch.load(filename, map_location="cpu") for key, value in state_dict.items(): - value = value.to(device).to(dtype) + value = value.to(device if not quantize else "cpu").to(dtype) layer_name = ".".join(key.split(".")[:4]) @@ -139,7 +137,7 @@ def load_weights( del value torch.cuda.empty_cache() - model.post_load_weights() + model.post_load_weights(quantize) class FlashLlamaSharded(FlashLlama): @@ -154,9 +152,6 @@ def __init__( else: raise NotImplementedError("FlashLlama is only available on GPU") - if quantize: - raise NotImplementedError("FlashLlama does not support quantization") - tokenizer = LlamaTokenizer.from_pretrained( model_id, revision=revision, @@ -185,7 +180,7 @@ def __init__( rank=self.rank, world_size=self.world_size, ) - self.model = model.eval() + self.model = model.eval().to(device) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( tokenizer=tokenizer, @@ -300,4 +295,4 @@ def load_weights( else: module._buffers[param_name] = tensor torch.cuda.empty_cache() - model.post_load_weights() + model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index a8b38465..0cda728a 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -41,9 +41,6 @@ def __init__( else: raise NotImplementedError("FlashNeoX is only available on GPU") - if quantize: - raise NotImplementedError("FlashNeoX does not support quantization") - tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", truncation_side="left" ) @@ -63,13 +60,13 @@ def __init__( self.load_weights( model, filenames, + quantize=quantize, device=device, dtype=dtype, rank=self.rank, world_size=self.world_size, ) - model.post_load_weights() - self.model = model.eval() + self.model = model.eval().to(device) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( tokenizer=tokenizer, @@ -80,6 +77,7 @@ def __init__( def load_weights( model, filenames: List[str], + quantize: bool, device: torch.device, dtype: torch.dtype, rank: int, @@ -87,7 +85,9 @@ def load_weights( ): parameters = dict(model.named_parameters()) for file in filenames: - with safe_open(file, framework="pt", device=str(device)) as f: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: for name in f.keys(): module_name, param_name = name.rsplit(".", 1) module = model.get_submodule(module_name) @@ -146,3 +146,4 @@ def load_weights( module._parameters[param_name] = tensor else: module._buffers[param_name] = tensor + model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 39381e92..e3066c98 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -34,9 +34,6 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False else: raise NotImplementedError("FlashSantacoder is only available on GPU") - if quantize: - raise NotImplementedError("FlashSantacoder does not support quantization") - tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", truncation_side="left" ) @@ -58,9 +55,14 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False model = FlashSantacoderForCausalLM(config) self.load_weights( - model, filenames, device, dtype, config.architectures[0].startswith("GPT2") + model, + filenames, + quantize, + device, + dtype, + config.architectures[0].startswith("GPT2"), ) - self.model = model.eval() + self.model = model.eval().to(device) super(FlashCausalLM, self).__init__( tokenizer=tokenizer, device=device, decode_buffer=1 @@ -70,6 +72,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False def load_weights( model: FlashSantacoderForCausalLM, filenames: List[Path], + quantize: bool, device: torch.device, dtype: torch.dtype, transpose: bool, @@ -77,7 +80,7 @@ def load_weights( for filename in filenames: state_dict = torch.load(filename, map_location="cpu") for key, value in state_dict.items(): - value = value.to(device).to(dtype) + value = value.to(device if not quantize else "cpu").to(dtype) layer_name = ".".join(key.split(".")[:4]) @@ -152,7 +155,7 @@ def load_weights( del value torch.cuda.empty_cache() - model.post_load_weights() + model.post_load_weights(quantize) def decode(self, generated_ids: List[int]) -> str: # Do not skip special tokens as they are used for custom parsing rules of the generated text @@ -173,11 +176,6 @@ def __init__( else: raise NotImplementedError("FlashSantacoderSharded is only available on GPU") - if quantize: - raise NotImplementedError( - "FlashSantacoderSharded does not support quantization" - ) - tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", truncation_side="left" ) @@ -197,13 +195,14 @@ def __init__( self.load_weights( model, filenames, + quantize=quantize, device=device, dtype=dtype, rank=self.rank, world_size=self.world_size, transpose=config.architectures[0].startswith("GPT2"), ) - self.model = model.eval() + self.model = model.eval().to(device) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( tokenizer=tokenizer, @@ -214,6 +213,7 @@ def __init__( def load_weights( model, filenames: List[str], + quantize: bool, device: torch.device, dtype: torch.dtype, rank: int, @@ -221,7 +221,9 @@ def load_weights( transpose: bool, ): for file in filenames: - with safe_open(file, framework="pt", device=str(device)) as f: + with safe_open( + file, framework="pt", device=str(device) if not quantize else "cpu" + ) as f: for key in f.keys(): slice_ = f.get_slice(key) @@ -363,4 +365,4 @@ def load_weights( else: module._buffers[param_name] = tensor torch.cuda.empty_cache() - model.post_load_weights() + model.post_load_weights(quantize) From a88c54bb4ce022ddc03b899b0c8018b6625e3a9e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 19 Apr 2023 12:52:37 +0200 Subject: [PATCH 172/793] feat(server): check cuda capability when importing flash models (#201) close #198 --- server/text_generation_server/models/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 13c74c91..0a29b3cc 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -24,7 +24,18 @@ FlashSantacoderSharded, ) - FLASH_ATTENTION = torch.cuda.is_available() + if torch.cuda.is_available(): + major, minor = torch.cuda.get_device_capability() + is_sm75 = major == 7 and minor == 5 + is_sm8x = major == 8 and minor >= 0 + is_sm90 = major == 9 and minor == 0 + + supported = is_sm75 or is_sm8x or is_sm90 + if not supported: + raise ImportError(f"GPU with CUDA capability {major} {minor} is not supported") + FLASH_ATTENTION = True + else: + FLASH_ATTENTION = False except ImportError: logger.opt(exception=True).warning( "Could not import Flash Attention enabled models" From 5d27f5259b710c5d77925930bd804edcaa3821a6 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 19 Apr 2023 17:36:16 +0200 Subject: [PATCH 173/793] fix(server): fix hf_transfer issue with private repos (#203) --- Dockerfile | 5 +++ server/poetry.lock | 95 ++++++++++++++++++++++++++++++++++++++++++- server/pyproject.toml | 1 + 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dac274c6..79ee170e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -142,6 +142,11 @@ COPY --from=transformers-builder /usr/src/transformers/build/lib.linux-x86_64-cp # Install transformers dependencies RUN cd /usr/src/transformers && pip install -e . --no-cache-dir && pip install einops --no-cache-dir +# FIXME: remove when we get a release of huggingface-hub +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + git \ + && rm -rf /var/lib/apt/lists/* + # Install server COPY proto proto COPY server server diff --git a/server/poetry.lock b/server/poetry.lock index 0ab6faa3..c2ff628f 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -122,6 +122,38 @@ python-versions = ">=3.7" docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +[[package]] +name = "fsspec" +version = "2023.4.0" +description = "File-system specification" +category = "main" +optional = false +python-versions = ">=3.8" + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +devel = ["pytest", "pytest-cov"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + [[package]] name = "googleapis-common-protos" version = "1.59.0" @@ -207,6 +239,41 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "huggingface-hub" +version = "0.14.0.dev0" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +category = "main" +optional = false +python-versions = ">=3.7.0" +develop = false + +[package.dependencies] +filelock = "*" +fsspec = "*" +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "gradio", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"] +torch = ["torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] + +[package.source] +type = "git" +url = "https://github.com/huggingface/huggingface_hub.git" +reference = "4f27b44ee536cd654e171c7f37478eaf1996cc3f" +resolved_reference = "4f27b44ee536cd654e171c7f37478eaf1996cc3f" + [[package]] name = "idna" version = "3.4" @@ -741,6 +808,23 @@ typing-extensions = "*" [package.extras] opt-einsum = ["opt-einsum (>=3.3)"] +[[package]] +name = "tqdm" +version = "4.65.0" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "triton" version = "2.0.0" @@ -833,7 +917,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "e05491a03938b79a71b498f2759169f5a41181084158fde5993e7dcb25292cb0" +content-hash = "97539d3239ab1ab39036679e23af523deb6e28763028bb8331812b03de3a109d" [metadata.files] accelerate = [ @@ -968,6 +1052,10 @@ filelock = [ {file = "filelock-3.11.0-py3-none-any.whl", hash = "sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318"}, {file = "filelock-3.11.0.tar.gz", hash = "sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37"}, ] +fsspec = [ + {file = "fsspec-2023.4.0-py3-none-any.whl", hash = "sha256:f398de9b49b14e9d84d2c2d11b7b67121bc072fe97b930c4e5668ac3917d8307"}, + {file = "fsspec-2023.4.0.tar.gz", hash = "sha256:bf064186cd8808f0b2f6517273339ba0a0c8fb1b7048991c28bc67f58b8b67cd"}, +] googleapis-common-protos = [ {file = "googleapis-common-protos-1.59.0.tar.gz", hash = "sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44"}, {file = "googleapis_common_protos-1.59.0-py2.py3-none-any.whl", hash = "sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f"}, @@ -1099,6 +1187,7 @@ hf-transfer = [ {file = "hf_transfer-0.1.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"}, {file = "hf_transfer-0.1.3.tar.gz", hash = "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c"}, ] +huggingface-hub = [] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, @@ -1547,6 +1636,10 @@ torch = [ {file = "torch-2.0.0-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6e0b97beb037a165669c312591f242382e9109a240e20054d5a5782d9236cad0"}, {file = "torch-2.0.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:297a4919aff1c0f98a58ebe969200f71350a1d4d4f986dbfd60c02ffce780e99"}, ] +tqdm = [ + {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, + {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, +] triton = [ {file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"}, {file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 68dd9327..0f647f47 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -25,6 +25,7 @@ opentelemetry-instrumentation-grpc = "^0.36b0" hf-transfer = "^0.1.2" sentencepiece = "^0.1.97" tokenizers = "0.13.3" +huggingface-hub = {git = "https://github.com/huggingface/huggingface_hub.git", rev = "4f27b44ee536cd654e171c7f37478eaf1996cc3f"} [tool.poetry.extras] bnb = ["bitsandbytes"] From 6837b2eb77220194b5fae40fa07987b6ea66aaf8 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 19 Apr 2023 19:39:31 +0200 Subject: [PATCH 174/793] fix(docker): remove unused dependencies (#205) --- Dockerfile | 2 +- server/Makefile | 7 +- server/poetry.lock | 297 ++++++++-------- server/pyproject.toml | 5 +- server/requirements.txt | 740 +++------------------------------------- 5 files changed, 199 insertions(+), 852 deletions(-) diff --git a/Dockerfile b/Dockerfile index 79ee170e..61f37ed9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -154,7 +154,7 @@ COPY server/Makefile server/Makefile RUN cd server && \ make gen-server && \ pip install -r requirements.txt && \ - pip install ".[bnb]" --no-cache-dir + pip install ".[bnb, accelerate]" --no-cache-dir # Install router COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bin/text-generation-router diff --git a/server/Makefile b/server/Makefile index 89ddda40..058d912b 100644 --- a/server/Makefile +++ b/server/Makefile @@ -17,7 +17,10 @@ install-torch: install: gen-server install-torch install-transformers pip install pip --upgrade pip install -r requirements.txt - pip install -e . --no-cache-dir + pip install -e ".[bnb, accelerate]" run-dev: - SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded \ No newline at end of file + SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded + +export-requirements: + poetry export -o requirements.txt -E bnb --without-hashes \ No newline at end of file diff --git a/server/poetry.lock b/server/poetry.lock index c2ff628f..b13441de 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -3,7 +3,7 @@ name = "accelerate" version = "0.15.0" description = "Accelerate" category = "main" -optional = false +optional = true python-versions = ">=3.7.0" [package.dependencies] @@ -36,7 +36,7 @@ name = "bitsandbytes" version = "0.38.1" description = "8-bit optimizers and matrix multiplication routines." category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -71,7 +71,7 @@ name = "cmake" version = "3.26.3" description = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" category = "main" -optional = false +optional = true python-versions = "*" [package.extras] @@ -112,15 +112,15 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.11.0" +version = "3.12.0" description = "A platform independent file lock." category = "main" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] [[package]] name = "fsspec" @@ -184,30 +184,30 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.53.0" +version = "1.54.0" description = "HTTP/2-based RPC framework" category = "main" optional = false python-versions = ">=3.7" [package.extras] -protobuf = ["grpcio-tools (>=1.53.0)"] +protobuf = ["grpcio-tools (>=1.54.0)"] [[package]] name = "grpcio-reflection" -version = "1.53.0" +version = "1.54.0" description = "Standard Protobuf Reflection Service for gRPC" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -grpcio = ">=1.53.0" +grpcio = ">=1.54.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.53.0" +version = "1.54.0" description = "Status proto mapping for gRPC" category = "main" optional = false @@ -215,19 +215,19 @@ python-versions = ">=3.6" [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.53.0" +grpcio = ">=1.54.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.53.0" +version = "1.54.0" description = "Protobuf code generator for gRPC" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -grpcio = ">=1.53.0" +grpcio = ">=1.54.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -295,7 +295,7 @@ name = "Jinja2" version = "3.1.2" description = "A very fast and expressive template engine." category = "main" -optional = false +optional = true python-versions = ">=3.7" [package.dependencies] @@ -309,7 +309,7 @@ name = "lit" version = "16.0.1" description = "A Software Testing Tool" category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -332,7 +332,7 @@ name = "MarkupSafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." category = "main" -optional = false +optional = true python-versions = ">=3.7" [[package]] @@ -340,7 +340,7 @@ name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" category = "main" -optional = false +optional = true python-versions = "*" [package.extras] @@ -354,7 +354,7 @@ name = "networkx" version = "3.1" description = "Python package for creating and manipulating graphs and networks" category = "main" -optional = false +optional = true python-versions = ">=3.8" [package.extras] @@ -369,7 +369,7 @@ name = "numpy" version = "1.24.2" description = "Fundamental package for array computing in Python" category = "main" -optional = false +optional = true python-versions = ">=3.8" [[package]] @@ -377,7 +377,7 @@ name = "nvidia-cublas-cu11" version = "11.10.3.66" description = "CUBLAS native runtime libraries" category = "main" -optional = false +optional = true python-versions = ">=3" [package.dependencies] @@ -389,7 +389,7 @@ name = "nvidia-cuda-cupti-cu11" version = "11.7.101" description = "CUDA profiling tools runtime libs." category = "main" -optional = false +optional = true python-versions = ">=3" [package.dependencies] @@ -401,7 +401,7 @@ name = "nvidia-cuda-nvrtc-cu11" version = "11.7.99" description = "NVRTC native runtime libraries" category = "main" -optional = false +optional = true python-versions = ">=3" [package.dependencies] @@ -413,7 +413,7 @@ name = "nvidia-cuda-runtime-cu11" version = "11.7.99" description = "CUDA Runtime native Libraries" category = "main" -optional = false +optional = true python-versions = ">=3" [package.dependencies] @@ -425,7 +425,7 @@ name = "nvidia-cudnn-cu11" version = "8.5.0.96" description = "cuDNN runtime libraries" category = "main" -optional = false +optional = true python-versions = ">=3" [package.dependencies] @@ -437,7 +437,7 @@ name = "nvidia-cufft-cu11" version = "10.9.0.58" description = "CUFFT native runtime libraries" category = "main" -optional = false +optional = true python-versions = ">=3" [[package]] @@ -445,7 +445,7 @@ name = "nvidia-curand-cu11" version = "10.2.10.91" description = "CURAND native runtime libraries" category = "main" -optional = false +optional = true python-versions = ">=3" [package.dependencies] @@ -457,7 +457,7 @@ name = "nvidia-cusolver-cu11" version = "11.4.0.1" description = "CUDA solver native runtime libraries" category = "main" -optional = false +optional = true python-versions = ">=3" [package.dependencies] @@ -469,7 +469,7 @@ name = "nvidia-cusparse-cu11" version = "11.7.4.91" description = "CUSPARSE native runtime libraries" category = "main" -optional = false +optional = true python-versions = ">=3" [package.dependencies] @@ -481,7 +481,7 @@ name = "nvidia-nccl-cu11" version = "2.14.3" description = "NVIDIA Collective Communication Library (NCCL) Runtime" category = "main" -optional = false +optional = true python-versions = ">=3" [[package]] @@ -489,7 +489,7 @@ name = "nvidia-nvtx-cu11" version = "11.7.91" description = "NVIDIA Tools Extension" category = "main" -optional = false +optional = true python-versions = ">=3" [package.dependencies] @@ -653,10 +653,10 @@ python-versions = ">=3.7" [[package]] name = "psutil" -version = "5.9.4" +version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] @@ -751,7 +751,7 @@ name = "sympy" version = "1.11.1" description = "Computer algebra system (CAS) in Python" category = "main" -optional = false +optional = true python-versions = ">=3.8" [package.dependencies] @@ -783,7 +783,7 @@ name = "torch" version = "2.0.0" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" category = "main" -optional = false +optional = true python-versions = ">=3.8.0" [package.dependencies] @@ -830,7 +830,7 @@ name = "triton" version = "2.0.0" description = "A language and compiler for custom Deep Learning operations" category = "main" -optional = false +optional = true python-versions = "*" [package.dependencies] @@ -886,7 +886,7 @@ name = "wheel" version = "0.40.0" description = "A built-package format for Python" category = "main" -optional = false +optional = true python-versions = ">=3.7" [package.extras] @@ -912,12 +912,13 @@ optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [extras] +accelerate = ["accelerate"] bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "97539d3239ab1ab39036679e23af523deb6e28763028bb8331812b03de3a109d" +content-hash = "2c154f7400d221a13ca4ff5dcab4148be17ae2c22493d8aa57c1a0d2f9361f14" [metadata.files] accelerate = [ @@ -1049,8 +1050,8 @@ exceptiongroup = [ {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, ] filelock = [ - {file = "filelock-3.11.0-py3-none-any.whl", hash = "sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318"}, - {file = "filelock-3.11.0.tar.gz", hash = "sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37"}, + {file = "filelock-3.12.0-py3-none-any.whl", hash = "sha256:ad98852315c2ab702aeb628412cbf7e95b7ce8c3bf9565670b4eaecf1db370a9"}, + {file = "filelock-3.12.0.tar.gz", hash = "sha256:fc03ae43288c013d2ea83c8597001b1129db351aad9c57fe2409327916b8e718"}, ] fsspec = [ {file = "fsspec-2023.4.0-py3-none-any.whl", hash = "sha256:f398de9b49b14e9d84d2c2d11b7b67121bc072fe97b930c4e5668ac3917d8307"}, @@ -1065,106 +1066,106 @@ grpc-interceptor = [ {file = "grpc_interceptor-0.15.1-py3-none-any.whl", hash = "sha256:1cc52c34b0d7ff34512fb7780742ecda37bf3caa18ecc5f33f09b4f74e96b276"}, ] grpcio = [ - {file = "grpcio-1.53.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:752d2949b40e12e6ad3ed8cc552a65b54d226504f6b1fb67cab2ccee502cc06f"}, - {file = "grpcio-1.53.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:8a48fd3a7222be226bb86b7b413ad248f17f3101a524018cdc4562eeae1eb2a3"}, - {file = "grpcio-1.53.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f3e837d29f0e1b9d6e7b29d569e2e9b0da61889e41879832ea15569c251c303a"}, - {file = "grpcio-1.53.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aef7d30242409c3aa5839b501e877e453a2c8d3759ca8230dd5a21cda029f046"}, - {file = "grpcio-1.53.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6f90698b5d1c5dd7b3236cd1fa959d7b80e17923f918d5be020b65f1c78b173"}, - {file = "grpcio-1.53.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a96c3c7f564b263c5d7c0e49a337166c8611e89c4c919f66dba7b9a84abad137"}, - {file = "grpcio-1.53.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ee81349411648d1abc94095c68cd25e3c2812e4e0367f9a9355be1e804a5135c"}, - {file = "grpcio-1.53.0-cp310-cp310-win32.whl", hash = "sha256:fdc6191587de410a184550d4143e2b24a14df495c86ca15e59508710681690ac"}, - {file = "grpcio-1.53.0-cp310-cp310-win_amd64.whl", hash = "sha256:658ffe1e39171be00490db5bd3b966f79634ac4215a1eb9a85c6cd6783bf7f6e"}, - {file = "grpcio-1.53.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:1b172e6d497191940c4b8d75b53de82dc252e15b61de2951d577ec5b43316b29"}, - {file = "grpcio-1.53.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:82434ba3a5935e47908bc861ce1ebc43c2edfc1001d235d6e31e5d3ed55815f7"}, - {file = "grpcio-1.53.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:1c734a2d4843e4e14ececf5600c3c4750990ec319e1299db7e4f0d02c25c1467"}, - {file = "grpcio-1.53.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6a2ead3de3b2d53119d473aa2f224030257ef33af1e4ddabd4afee1dea5f04c"}, - {file = "grpcio-1.53.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a34d6e905f071f9b945cabbcc776e2055de1fdb59cd13683d9aa0a8f265b5bf9"}, - {file = "grpcio-1.53.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eaf8e3b97caaf9415227a3c6ca5aa8d800fecadd526538d2bf8f11af783f1550"}, - {file = "grpcio-1.53.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:da95778d37be8e4e9afca771a83424f892296f5dfb2a100eda2571a1d8bbc0dc"}, - {file = "grpcio-1.53.0-cp311-cp311-win32.whl", hash = "sha256:e4f513d63df6336fd84b74b701f17d1bb3b64e9d78a6ed5b5e8a198bbbe8bbfa"}, - {file = "grpcio-1.53.0-cp311-cp311-win_amd64.whl", hash = "sha256:ddb2511fbbb440ed9e5c9a4b9b870f2ed649b7715859fd6f2ebc585ee85c0364"}, - {file = "grpcio-1.53.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:2a912397eb8d23c177d6d64e3c8bc46b8a1c7680b090d9f13a640b104aaec77c"}, - {file = "grpcio-1.53.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:55930c56b8f5b347d6c8c609cc341949a97e176c90f5cbb01d148d778f3bbd23"}, - {file = "grpcio-1.53.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6601d812105583948ab9c6e403a7e2dba6e387cc678c010e74f2d6d589d1d1b3"}, - {file = "grpcio-1.53.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c705e0c21acb0e8478a00e7e773ad0ecdb34bd0e4adc282d3d2f51ba3961aac7"}, - {file = "grpcio-1.53.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba074af9ca268ad7b05d3fc2b920b5fb3c083da94ab63637aaf67f4f71ecb755"}, - {file = "grpcio-1.53.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:14817de09317dd7d3fbc8272864288320739973ef0f4b56bf2c0032349da8cdf"}, - {file = "grpcio-1.53.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c7ad9fbedb93f331c2e9054e202e95cf825b885811f1bcbbdfdc301e451442db"}, - {file = "grpcio-1.53.0-cp37-cp37m-win_amd64.whl", hash = "sha256:dad5b302a4c21c604d88a5d441973f320134e6ff6a84ecef9c1139e5ffd466f6"}, - {file = "grpcio-1.53.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fa8eaac75d3107e3f5465f2c9e3bbd13db21790c6e45b7de1756eba16b050aca"}, - {file = "grpcio-1.53.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:104a2210edd3776c38448b4f76c2f16e527adafbde171fc72a8a32976c20abc7"}, - {file = "grpcio-1.53.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:dbc1ba968639c1d23476f75c356e549e7bbf2d8d6688717dcab5290e88e8482b"}, - {file = "grpcio-1.53.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95952d3fe795b06af29bb8ec7bbf3342cdd867fc17b77cc25e6733d23fa6c519"}, - {file = "grpcio-1.53.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f144a790f14c51b8a8e591eb5af40507ffee45ea6b818c2482f0457fec2e1a2e"}, - {file = "grpcio-1.53.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0698c094688a2dd4c7c2f2c0e3e142cac439a64d1cef6904c97f6cde38ba422f"}, - {file = "grpcio-1.53.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6b6d60b0958be711bab047e9f4df5dbbc40367955f8651232bfdcdd21450b9ab"}, - {file = "grpcio-1.53.0-cp38-cp38-win32.whl", hash = "sha256:1948539ce78805d4e6256ab0e048ec793956d54787dc9d6777df71c1d19c7f81"}, - {file = "grpcio-1.53.0-cp38-cp38-win_amd64.whl", hash = "sha256:df9ba1183b3f649210788cf80c239041dddcb375d6142d8bccafcfdf549522cd"}, - {file = "grpcio-1.53.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:19caa5b7282a89b799e63776ff602bb39604f7ca98db6df27e2de06756ae86c3"}, - {file = "grpcio-1.53.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:b5bd026ac928c96cc23149e6ef79183125542062eb6d1ccec34c0a37e02255e7"}, - {file = "grpcio-1.53.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:7dc8584ca6c015ad82e186e82f4c0fe977394588f66b8ecfc4ec873285314619"}, - {file = "grpcio-1.53.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2eddaae8af625e45b5c8500dcca1043264d751a6872cde2eda5022df8a336959"}, - {file = "grpcio-1.53.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5fb6f3d7824696c1c9f2ad36ddb080ba5a86f2d929ef712d511b4d9972d3d27"}, - {file = "grpcio-1.53.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8270d1dc2c98ab57e6dbf36fa187db8df4c036f04a398e5d5e25b4e01a766d70"}, - {file = "grpcio-1.53.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:976a7f24eb213e8429cab78d5e120500dfcdeb01041f1f5a77b17b9101902615"}, - {file = "grpcio-1.53.0-cp39-cp39-win32.whl", hash = "sha256:9c84a481451e7174f3a764a44150f93b041ab51045aa33d7b5b68b6979114e48"}, - {file = "grpcio-1.53.0-cp39-cp39-win_amd64.whl", hash = "sha256:6beb84f83360ff29a3654f43f251ec11b809dcb5524b698d711550243debd289"}, - {file = "grpcio-1.53.0.tar.gz", hash = "sha256:a4952899b4931a6ba12951f9a141ef3e74ff8a6ec9aa2dc602afa40f63595e33"}, + {file = "grpcio-1.54.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:a947d5298a0bbdd4d15671024bf33e2b7da79a70de600ed29ba7e0fef0539ebb"}, + {file = "grpcio-1.54.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:e355ee9da9c1c03f174efea59292b17a95e0b7b4d7d2a389265f731a9887d5a9"}, + {file = "grpcio-1.54.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:73c238ef6e4b64272df7eec976bb016c73d3ab5a6c7e9cd906ab700523d312f3"}, + {file = "grpcio-1.54.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c59d899ee7160638613a452f9a4931de22623e7ba17897d8e3e348c2e9d8d0b"}, + {file = "grpcio-1.54.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48cb7af77238ba16c77879009003f6b22c23425e5ee59cb2c4c103ec040638a5"}, + {file = "grpcio-1.54.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2262bd3512ba9e9f0e91d287393df6f33c18999317de45629b7bd46c40f16ba9"}, + {file = "grpcio-1.54.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:224166f06ccdaf884bf35690bf4272997c1405de3035d61384ccb5b25a4c1ca8"}, + {file = "grpcio-1.54.0-cp310-cp310-win32.whl", hash = "sha256:ed36e854449ff6c2f8ee145f94851fe171298e1e793f44d4f672c4a0d78064e7"}, + {file = "grpcio-1.54.0-cp310-cp310-win_amd64.whl", hash = "sha256:27fb030a4589d2536daec5ff5ba2a128f4f155149efab578fe2de2cb21596d3d"}, + {file = "grpcio-1.54.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:f4a7dca8ccd8023d916b900aa3c626f1bd181bd5b70159479b142f957ff420e4"}, + {file = "grpcio-1.54.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:1209d6b002b26e939e4c8ea37a3d5b4028eb9555394ea69fb1adbd4b61a10bb8"}, + {file = "grpcio-1.54.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:860fcd6db7dce80d0a673a1cc898ce6bc3d4783d195bbe0e911bf8a62c93ff3f"}, + {file = "grpcio-1.54.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3930669c9e6f08a2eed824738c3d5699d11cd47a0ecc13b68ed11595710b1133"}, + {file = "grpcio-1.54.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62117486460c83acd3b5d85c12edd5fe20a374630475388cfc89829831d3eb79"}, + {file = "grpcio-1.54.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e3e526062c690517b42bba66ffe38aaf8bc99a180a78212e7b22baa86902f690"}, + {file = "grpcio-1.54.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ebff0738be0499d7db74d20dca9f22a7b27deae31e1bf92ea44924fd69eb6251"}, + {file = "grpcio-1.54.0-cp311-cp311-win32.whl", hash = "sha256:21c4a1aae861748d6393a3ff7867473996c139a77f90326d9f4104bebb22d8b8"}, + {file = "grpcio-1.54.0-cp311-cp311-win_amd64.whl", hash = "sha256:3db71c6f1ab688d8dfc102271cedc9828beac335a3a4372ec54b8bf11b43fd29"}, + {file = "grpcio-1.54.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:960b176e0bb2b4afeaa1cd2002db1e82ae54c9b6e27ea93570a42316524e77cf"}, + {file = "grpcio-1.54.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:d8ae6e0df3a608e99ee1acafaafd7db0830106394d54571c1ece57f650124ce9"}, + {file = "grpcio-1.54.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:c33744d0d1a7322da445c0fe726ea6d4e3ef2dfb0539eadf23dce366f52f546c"}, + {file = "grpcio-1.54.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d109df30641d050e009105f9c9ca5a35d01e34d2ee2a4e9c0984d392fd6d704"}, + {file = "grpcio-1.54.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:775a2f70501370e5ba54e1ee3464413bff9bd85bd9a0b25c989698c44a6fb52f"}, + {file = "grpcio-1.54.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c55a9cf5cba80fb88c850915c865b8ed78d5e46e1f2ec1b27692f3eaaf0dca7e"}, + {file = "grpcio-1.54.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1fa7d6ddd33abbd3c8b3d7d07c56c40ea3d1891ce3cd2aa9fa73105ed5331866"}, + {file = "grpcio-1.54.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ed3d458ded32ff3a58f157b60cc140c88f7ac8c506a1c567b2a9ee8a2fd2ce54"}, + {file = "grpcio-1.54.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:5942a3e05630e1ef5b7b5752e5da6582460a2e4431dae603de89fc45f9ec5aa9"}, + {file = "grpcio-1.54.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:125ed35aa3868efa82eabffece6264bf638cfdc9f0cd58ddb17936684aafd0f8"}, + {file = "grpcio-1.54.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:b7655f809e3420f80ce3bf89737169a9dce73238af594049754a1128132c0da4"}, + {file = "grpcio-1.54.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87f47bf9520bba4083d65ab911f8f4c0ac3efa8241993edd74c8dd08ae87552f"}, + {file = "grpcio-1.54.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16bca8092dd994f2864fdab278ae052fad4913f36f35238b2dd11af2d55a87db"}, + {file = "grpcio-1.54.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d2f62fb1c914a038921677cfa536d645cb80e3dd07dc4859a3c92d75407b90a5"}, + {file = "grpcio-1.54.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a7caf553ccaf715ec05b28c9b2ab2ee3fdb4036626d779aa09cf7cbf54b71445"}, + {file = "grpcio-1.54.0-cp38-cp38-win32.whl", hash = "sha256:2585b3c294631a39b33f9f967a59b0fad23b1a71a212eba6bc1e3ca6e6eec9ee"}, + {file = "grpcio-1.54.0-cp38-cp38-win_amd64.whl", hash = "sha256:3b170e441e91e4f321e46d3cc95a01cb307a4596da54aca59eb78ab0fc03754d"}, + {file = "grpcio-1.54.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:1382bc499af92901c2240c4d540c74eae8a671e4fe9839bfeefdfcc3a106b5e2"}, + {file = "grpcio-1.54.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:031bbd26656e0739e4b2c81c172155fb26e274b8d0312d67aefc730bcba915b6"}, + {file = "grpcio-1.54.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a97b0d01ae595c997c1d9d8249e2d2da829c2d8a4bdc29bb8f76c11a94915c9a"}, + {file = "grpcio-1.54.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:533eaf5b2a79a3c6f35cbd6a095ae99cac7f4f9c0e08bdcf86c130efd3c32adf"}, + {file = "grpcio-1.54.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49eace8ea55fbc42c733defbda1e4feb6d3844ecd875b01bb8b923709e0f5ec8"}, + {file = "grpcio-1.54.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:30fbbce11ffeb4f9f91c13fe04899aaf3e9a81708bedf267bf447596b95df26b"}, + {file = "grpcio-1.54.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:650f5f2c9ab1275b4006707411bb6d6bc927886874a287661c3c6f332d4c068b"}, + {file = "grpcio-1.54.0-cp39-cp39-win32.whl", hash = "sha256:02000b005bc8b72ff50c477b6431e8886b29961159e8b8d03c00b3dd9139baed"}, + {file = "grpcio-1.54.0-cp39-cp39-win_amd64.whl", hash = "sha256:6dc1e2c9ac292c9a484ef900c568ccb2d6b4dfe26dfa0163d5bc815bb836c78d"}, + {file = "grpcio-1.54.0.tar.gz", hash = "sha256:eb0807323572642ab73fd86fe53d88d843ce617dd1ddf430351ad0759809a0ae"}, ] grpcio-reflection = [ - {file = "grpcio-reflection-1.53.0.tar.gz", hash = "sha256:663ea8a9883a64457c1a58b95d52b431f4636337503954e15bfac1d1697b401c"}, - {file = "grpcio_reflection-1.53.0-py3-none-any.whl", hash = "sha256:19ac2e3b16404f20bfa14ec64483bb4a936accee50ccb2e64b0159d1cea767cd"}, + {file = "grpcio-reflection-1.54.0.tar.gz", hash = "sha256:804326e1add80050cab248107d28f226be58ec49d5a2d08f14a150d8a2621678"}, + {file = "grpcio_reflection-1.54.0-py3-none-any.whl", hash = "sha256:9de46150bba3c039035c5f573a9348f1c747d3149b1fa946b351da7c0e0733d8"}, ] grpcio-status = [ - {file = "grpcio-status-1.53.0.tar.gz", hash = "sha256:5a46820dc7d94bac48aae5dd97c94cadea09d80a05553aca74a92a2954030420"}, - {file = "grpcio_status-1.53.0-py3-none-any.whl", hash = "sha256:4f4bc140ed718a343d04a573c838c9fff5cb87f40d3cb2aa33b027c55031489d"}, + {file = "grpcio-status-1.54.0.tar.gz", hash = "sha256:b50305d52c0df6169493cca5f2e39b9b4d773b3f30d4a7a6b6dd7c18cb89007c"}, + {file = "grpcio_status-1.54.0-py3-none-any.whl", hash = "sha256:96968314e0c8576b2b631be3917c665964c8018900cb980d58a736fbff828578"}, ] grpcio-tools = [ - {file = "grpcio-tools-1.53.0.tar.gz", hash = "sha256:925efff2d63ca3266f93c924ffeba5d496f16a8ccbe125fa0d18acf47cc5fa88"}, - {file = "grpcio_tools-1.53.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:41b859cf943256debba1e7b921e3689c89f95495b65f7ad226c4f0e38edf8ee4"}, - {file = "grpcio_tools-1.53.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:17c557240f7fbe1886dcfb5f3ba79740ecb65fe3b93061e64b8f4dfc6a6a5dc5"}, - {file = "grpcio_tools-1.53.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:6afffd7e97e5bddc63b3ce9abe912b9adb704a36ba86d4406be94426734b97c2"}, - {file = "grpcio_tools-1.53.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f55e2c13620271b7f5a81a489a188d6e34a24da8885d46f1566f0e798cb59e6f"}, - {file = "grpcio_tools-1.53.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bd4c732d8d7a736e787b5d0963d4195267fc856e1d313d4532d1625e19a0e4a"}, - {file = "grpcio_tools-1.53.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:99ecefb6b66e9fe41468a70ee2f05da2eb9c7bf63867fb9ff07f7dd90ea813ae"}, - {file = "grpcio_tools-1.53.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7754d6466191d327a0eef364ad5b863477a8fcc12953adc06b30b8e470c70e4a"}, - {file = "grpcio_tools-1.53.0-cp310-cp310-win32.whl", hash = "sha256:f31c549d793a0e72c044f724b3373141d2aa9970fe97b1c2cfaa7ea44002b9aa"}, - {file = "grpcio_tools-1.53.0-cp310-cp310-win_amd64.whl", hash = "sha256:b4173b95e2c29a5145c806d16945ce1e5b38a11c7eb6ab1a6d74afc0a2ce47d9"}, - {file = "grpcio_tools-1.53.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:613a84ebd1881635370c12503f2b15b37332a53fbac32904c94ac4c0c10f0a2a"}, - {file = "grpcio_tools-1.53.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:af686b83bc6b5c1f1591c9f49183717974047de9546adcf5e09a18781b550c96"}, - {file = "grpcio_tools-1.53.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:3cc832e8297e9437bc2b137fe815c8ba1d9af6ffdd76c5c6d7f911bf8e1b0f45"}, - {file = "grpcio_tools-1.53.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39d0a254de49d852f5fe9f9df0a45b2ae66bc04e2d9ee1d6d2c0ba1e70fac91a"}, - {file = "grpcio_tools-1.53.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7062109553ec1873c5c09cc379b8ae0aa76a2d6d6aae97759b97787b93fa9786"}, - {file = "grpcio_tools-1.53.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7728407b1e89fb1473b86152fc33be00f1a25a5aa3264245521f05cbbef9d817"}, - {file = "grpcio_tools-1.53.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2758ea125442bc81251267fc9c28f65555a571f6a0afda4d71a6e7d669347095"}, - {file = "grpcio_tools-1.53.0-cp311-cp311-win32.whl", hash = "sha256:8940d59fca790f1bd45785d0661c3a8c081231c9f8049d7fbf6c6c00737e43da"}, - {file = "grpcio_tools-1.53.0-cp311-cp311-win_amd64.whl", hash = "sha256:c2cff79be5a06d63e9a6a7e38f8f160ade21517386eabe27afacef65a8531358"}, - {file = "grpcio_tools-1.53.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d646d65fafbf70a57416493e719a0df7ffa0772133266cfe1b2b72e072ae64a2"}, - {file = "grpcio_tools-1.53.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:7da0fc185735050d8240b1d74c4667a02baf1b4fa379a5fc05d1fc067eeba596"}, - {file = "grpcio_tools-1.53.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:2be17265c0f070efd625683cef986e07dbc495103fcc719009ff2f6988003166"}, - {file = "grpcio_tools-1.53.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4701d48f649443f1101a24d85e9d5ac13346ccac7781e243f49491328e172266"}, - {file = "grpcio_tools-1.53.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b54c64d85bea5c3a3d895454878c7d6bed5cbb80dc3cafcd75dc1e78300d8c95"}, - {file = "grpcio_tools-1.53.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7152045190e9bd665d1feaeaef931d82c75cacce2b116ab150befa90855de3d0"}, - {file = "grpcio_tools-1.53.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e18292123c86975d0aa47f1bcb176393640dcc23912e9f3a2247f1eff81ac8e8"}, - {file = "grpcio_tools-1.53.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b1b76b6ab5c24e44b15d6a7df6c1b81c3099a54b82d41a3ce96e73a2e6a5081c"}, - {file = "grpcio_tools-1.53.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:e76e8dfe6fe4e61ce3049e9d56c0d806d0d3edc28aa32117d1b17f387469c52e"}, - {file = "grpcio_tools-1.53.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:4c6acaca09cfcd59850e27bd138df9d01c0686c42a5412aa6a92141c15316b1e"}, - {file = "grpcio_tools-1.53.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:76898c1dadf8630a75a40b5a89ab38e326f1288dcfde3413cdfa7a58e149c987"}, - {file = "grpcio_tools-1.53.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b47f8b1bd3af2fb25548b625ad9c3659da30fe83c06f462f357c754f49b71ae"}, - {file = "grpcio_tools-1.53.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2faad4b6362e7ff3ae43ef2d51dfce0a3bc32cf52469e88568c3f65cae377d5"}, - {file = "grpcio_tools-1.53.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:830261fe08541f0fd2dd5035264df2b91012988f37aa1d80a0b4ee6404dc25ae"}, - {file = "grpcio_tools-1.53.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4be32c694c760f3281555089f7aed7d48ca7ea4094115a08b5fc895e17d7e62e"}, - {file = "grpcio_tools-1.53.0-cp38-cp38-win32.whl", hash = "sha256:4605db5a5828205d7fa33a5de9e00723bd037709e74e15c028b9dcec2339b7bc"}, - {file = "grpcio_tools-1.53.0-cp38-cp38-win_amd64.whl", hash = "sha256:0229e6cd442915192b8f8ee2e7e1c8b9986c878bc4dd8be3539f3be35f1b8282"}, - {file = "grpcio_tools-1.53.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:ad0c20688a650e731e8328a7a08899c433a59bfc995a7afcf715b5ad9eca9e7b"}, - {file = "grpcio_tools-1.53.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:a8c3e30c531969c62a5a219be414277b269c1be9a76bcd6948571868894e19b2"}, - {file = "grpcio_tools-1.53.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:326c67b35be69409a88632e6145032d53b8b8141634e9cbcd27fa8f9015a112c"}, - {file = "grpcio_tools-1.53.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:102b6d323d7cef7ac29683f949ec66885b417c06df6059f6a88d07c5556c2592"}, - {file = "grpcio_tools-1.53.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:861f8634cca3ca5bb5336ba16cc78291dba3e7fcadedff195bfdeb433f2c29f2"}, - {file = "grpcio_tools-1.53.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c9a9e1da1868349eba401e9648eac19132700942c475adcc97b6938bf4bf0182"}, - {file = "grpcio_tools-1.53.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ccf7313e5bee13f2f86d12741489f3ed8c901d6b463dff2604191cd4ff518abb"}, - {file = "grpcio_tools-1.53.0-cp39-cp39-win32.whl", hash = "sha256:65b77532bb8f6ab1bfbdd2ac0788626a6c05b227f4722d3bbc2c54258e49c3e5"}, - {file = "grpcio_tools-1.53.0-cp39-cp39-win_amd64.whl", hash = "sha256:7c0ede22796259e83aa1f108038513e86672b2892d3654f94415e3930b74b871"}, + {file = "grpcio-tools-1.54.0.tar.gz", hash = "sha256:df79acbf59997018e131713b716a2fddb5556e1840e9fb9de4ca73bf2059590e"}, + {file = "grpcio_tools-1.54.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:7abdefb364de75d13d929093e571b84c4ea2580dbf5e4b6f5099ac9fa1a93f29"}, + {file = "grpcio_tools-1.54.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:66738bc9d7b5db1a86832fa930cdd8faaf0c68bf70751d930c686d6bb309fad0"}, + {file = "grpcio_tools-1.54.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:de07f7441e2f318a68631626d4d99c22244901ad7451a3e9fa08e5601a54772f"}, + {file = "grpcio_tools-1.54.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb6774907a708079afb3deecdd8e053d2b2b066c279a5d8c4957687db3f809ce"}, + {file = "grpcio_tools-1.54.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7615b824132fcea4769d0cd99bdeffe25b096b48215c36675f37f7b57dc09635"}, + {file = "grpcio_tools-1.54.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22b826b1d0d98a3bb80094928e21c48e23d022168e91d466a21389d1cda87177"}, + {file = "grpcio_tools-1.54.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c03ceec6441365ec56c7fc6f38386917814ad02144dc1ecf03e13cee45f26389"}, + {file = "grpcio_tools-1.54.0-cp310-cp310-win32.whl", hash = "sha256:a61f367a153e3e604b1b59cec2f5e61255ffb6328d7348a53531e90ad98cc4fd"}, + {file = "grpcio_tools-1.54.0-cp310-cp310-win_amd64.whl", hash = "sha256:8a91f973b44fc90c32b7df79daec78907f0721e65d85fdff9133e05fee1f6e56"}, + {file = "grpcio_tools-1.54.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:436335857731e8f0bc5d345804d7dcd1bc677a1f793ea2b22e0627870ea31df6"}, + {file = "grpcio_tools-1.54.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:9fa42e3851c3c795228e2699c68ff7db14789991da4b53d3634360a361e0e420"}, + {file = "grpcio_tools-1.54.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7ab23a94a8a7aa97ad8f46ba39ade4462c19833aa4815780db2cec647df7efef"}, + {file = "grpcio_tools-1.54.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12c3091ef09df47c20d698dc74a373c1674c67ac5ac12ef24433002165129556"}, + {file = "grpcio_tools-1.54.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536c108ee8fa46b1ca8d03f4033cdf282741978f3f83dcc8b2a9e7a2b8197271"}, + {file = "grpcio_tools-1.54.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a66baf974aa00542ba4b6be499266ef8d63aa1abe7c78abab168bdf2fcd7f216"}, + {file = "grpcio_tools-1.54.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b944c1654b5f0710e782c0e3462d7acca900cbf2fb874d62e51a5279293151a9"}, + {file = "grpcio_tools-1.54.0-cp311-cp311-win32.whl", hash = "sha256:f5aaa7e90e0947aec936e02fb85f312ca03c2258f3ee78403dcc27389fc62838"}, + {file = "grpcio_tools-1.54.0-cp311-cp311-win_amd64.whl", hash = "sha256:6ff318cd73b5cbdb16616d093c1c7b340d8d13b5bada2df7fbd873e899eab162"}, + {file = "grpcio_tools-1.54.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:86169076170f835139db7ec8362124cabe8f01050c2a717500a0fcdeb71dc537"}, + {file = "grpcio_tools-1.54.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:24158db60cdda7c9eb803a9ed033a23b4420fd13ba4c54392d6b396589afb969"}, + {file = "grpcio_tools-1.54.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:58d2cbbf4e40cecd09f4fa6a2c15f7a95ec90f492b4e31469d8c9db981426106"}, + {file = "grpcio_tools-1.54.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e15aa21a68cdb66b38db645924a09196cbbf4ad3ce1cf9dbcdf7b245292e380"}, + {file = "grpcio_tools-1.54.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23474a820bdf8126af93724c7f6b23b16d9f3ad4c4bdcb40936ecb7f2be6bcc7"}, + {file = "grpcio_tools-1.54.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8d2d5522d2222879c161479b5c91661cf788a3178f0b7532fd4c3e918e3390c4"}, + {file = "grpcio_tools-1.54.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:52b6f78f5601cd080e5456ec24fd960328f9962e9aa4f9f556ec6fd5e4a391bf"}, + {file = "grpcio_tools-1.54.0-cp37-cp37m-win_amd64.whl", hash = "sha256:292c0838d4a52ca53a941d197de57efc7f9df548d0619e14680c3e1ebeb33752"}, + {file = "grpcio_tools-1.54.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:b1a3aad8a85d408ec925ee63668c2e4f075c96f67fcc3c43366a28f60e25873c"}, + {file = "grpcio_tools-1.54.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:9d9a8d35a898f5c160fdb1ba0f58ada80e8e291cdffd6025f6043228d0832b2f"}, + {file = "grpcio_tools-1.54.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:42e8509ae024e8fb4eeab5ed9518c8e88d7f46a56015ae940796fe708d781e9f"}, + {file = "grpcio_tools-1.54.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6dd57137706257c39255020fb04322eef6c4043a11fc86042ae22b05167603ca"}, + {file = "grpcio_tools-1.54.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8ed6146e08b82caff467ac2af6c41f092212e3b5795687b76dd9219ac8b7186"}, + {file = "grpcio_tools-1.54.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4b32eaf44939c3dfa127bcb6bcf685188a05c1e0f54a7c8ed10770cbe177ca15"}, + {file = "grpcio_tools-1.54.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cca182c9a33dbf52c61a13fd1b98af535e7563bcd9e02e60e9de742d385f2ffb"}, + {file = "grpcio_tools-1.54.0-cp38-cp38-win32.whl", hash = "sha256:ce20c3523a4bdb85531e762f45f7c5f5a915e5a8f25db3d58ebb6a8a37290a75"}, + {file = "grpcio_tools-1.54.0-cp38-cp38-win_amd64.whl", hash = "sha256:53dc65731a46bc065ad8d717d75d8659763126d1626eacc6f225c0b779ccc350"}, + {file = "grpcio_tools-1.54.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:072279c394086ec07af8736f94aa6be671000af143df5f3040500e8cd5de484d"}, + {file = "grpcio_tools-1.54.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:473dbd7005d059d1dc84490eec35699029d629048e7fdd76e31c14fc58a603e5"}, + {file = "grpcio_tools-1.54.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:22eecfbb85da93405e056f073e339c34f881a3b410800902ae533e475688b1f1"}, + {file = "grpcio_tools-1.54.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0ba9344855b0c617d6df6bfb46ffbc574815c7152afa023e443cce4d5402f55"}, + {file = "grpcio_tools-1.54.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:235ea036d56ab9b6e4f235c554cf4680c42bf79352046e8a25829f597ca0d56b"}, + {file = "grpcio_tools-1.54.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:adc93f29353377182556e37120985f69f0fccc79ab65a0f24c1cc53075344dd7"}, + {file = "grpcio_tools-1.54.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab45bda18b4931b08d89e490fd920bf654109d1c2e80b037fe9b60ab30fb4d9d"}, + {file = "grpcio_tools-1.54.0-cp39-cp39-win32.whl", hash = "sha256:43339122c891110e9ed020974c0fe2bd773d96c184203a62d0973f4cfb007f80"}, + {file = "grpcio_tools-1.54.0-cp39-cp39-win_amd64.whl", hash = "sha256:d58528d6ea1835c0d4c34b60152dd29b3c9ab02ad0a8e734de240d54786d8b1e"}, ] hf-transfer = [ {file = "hf_transfer-0.1.3-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9"}, @@ -1402,20 +1403,20 @@ protobuf = [ {file = "protobuf-4.22.3.tar.gz", hash = "sha256:23452f2fdea754a8251d0fc88c0317735ae47217e0d27bf330a30eec2848811a"}, ] psutil = [ - {file = "psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8"}, - {file = "psutil-5.9.4-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe"}, - {file = "psutil-5.9.4-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549"}, - {file = "psutil-5.9.4-cp27-cp27m-win32.whl", hash = "sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad"}, - {file = "psutil-5.9.4-cp27-cp27m-win_amd64.whl", hash = "sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94"}, - {file = "psutil-5.9.4-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24"}, - {file = "psutil-5.9.4-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7"}, - {file = "psutil-5.9.4-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7"}, - {file = "psutil-5.9.4-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1"}, - {file = "psutil-5.9.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08"}, - {file = "psutil-5.9.4-cp36-abi3-win32.whl", hash = "sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff"}, - {file = "psutil-5.9.4-cp36-abi3-win_amd64.whl", hash = "sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4"}, - {file = "psutil-5.9.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e"}, - {file = "psutil-5.9.4.tar.gz", hash = "sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62"}, + {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, + {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, + {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, + {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, + {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, + {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, + {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, + {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, ] pytest = [ {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 0f647f47..a2969365 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -15,8 +15,8 @@ grpcio-status = "^1.51.1" grpcio-reflection = "^1.51.1" grpc-interceptor = "^0.15.0" typer = "^0.6.1" -accelerate = "^0.15.0" -bitsandbytes = "^0.38.1" +accelerate = { version = "^0.15.0", optional = true } +bitsandbytes = { version = "^0.38.1", optional = true } safetensors = "^0.2.4" loguru = "^0.6.0" opentelemetry-api = "^1.15.0" @@ -28,6 +28,7 @@ tokenizers = "0.13.3" huggingface-hub = {git = "https://github.com/huggingface/huggingface_hub.git", rev = "4f27b44ee536cd654e171c7f37478eaf1996cc3f"} [tool.poetry.extras] +accelerate = ["accelerate"] bnb = ["bitsandbytes"] [tool.poetry.group.dev.dependencies] diff --git a/server/requirements.txt b/server/requirements.txt index b1c84d15..e2dedb07 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,699 +1,41 @@ -accelerate==0.15.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:014833307424cd0a22f89815802e00653756257c45dfdba2453e52d428931c65 \ - --hash=sha256:438e25a01afa6e3ffbd25353e76a68be49677c3050f10bfac7beafaf53503efc -backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba \ - --hash=sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8 -bitsandbytes==0.38.1 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:5f532e7b1353eb7049ae831da2eb62ed8a1e0444116bd51b9e088a6e0bc7a34a \ - --hash=sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce -certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4" \ - --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ - --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 -charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4" \ - --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ - --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ - --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ - --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ - --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ - --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ - --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ - --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ - --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ - --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ - --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ - --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ - --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ - --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ - --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ - --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ - --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ - --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ - --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ - --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ - --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ - --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ - --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ - --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ - --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ - --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ - --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ - --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ - --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ - --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ - --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ - --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ - --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ - --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ - --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ - --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ - --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ - --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ - --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ - --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ - --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ - --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ - --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ - --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ - --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ - --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ - --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ - --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ - --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ - --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ - --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ - --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ - --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ - --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ - --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ - --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ - --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ - --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ - --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ - --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ - --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ - --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ - --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ - --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ - --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ - --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ - --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ - --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ - --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ - --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ - --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ - --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ - --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ - --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ - --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab -click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ - --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 -cmake==3.26.3 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:1798547b23b89030518c5668dc55aed0e1d01867cf91d7a94e15d33f62a56fd0 \ - --hash=sha256:1bc7b47456256bdcc41069f5c658f232bd6e15bf4796d115f6ec98800793daff \ - --hash=sha256:24741a304ada699b339034958777d9a1472ac8ddb9b6194d74f814287ca091ae \ - --hash=sha256:27a6fa1b97744311a7993d6a1e0ce14bd73696dab9ceb96701f1ec11edbd5053 \ - --hash=sha256:2ae3db2c2be50fdaf0c9f3a23b2206e9dcd55ca124f16486a841b939f50b595e \ - --hash=sha256:46aa385e19c9e4fc95d7d6ce5ee0bbe0d69bdeac4e9bc95c61f78f3973c2f626 \ - --hash=sha256:4d3185738a6405aa15801e684f8d589b00570da4cc676cb1b5bbc902e3023e53 \ - --hash=sha256:543b6958d1615327f484a07ab041029b1740918a8baa336adc9f5f0cbcd8fbd8 \ - --hash=sha256:6e5fcd1cfaac33d015e2709e0dd1b7ad352a315367012ac359c9adc062cf075b \ - --hash=sha256:71e1df5587ad860b9829211380c42fc90ef2413363f12805b1fa2d87769bf876 \ - --hash=sha256:9d38ea5b4999f8f042a071bea3e213f085bac26d7ab54cb5a4c6a193c4baf132 \ - --hash=sha256:a922a6f6c1580d0db17b0b75f82e619441dd43c7f1d6a35f7d27e709db48bdbb \ - --hash=sha256:b20f7f7ea316ce7bb158df0e3c3453cfab5048939f1291017d16a8a36ad33ae6 \ - --hash=sha256:b54cde1f1c0573321b22382bd2ffaf5d08f65188572d128cd4867fb9669723c5 \ - --hash=sha256:cf910bbb488659d300c86b1dac77e44eeb0457bde2cf76a42d7e51f691544b21 \ - --hash=sha256:d3017a08e6ba53ec2486d89a7953a81d4c4a068fc9f29d83e209f295dd9c59f3 \ - --hash=sha256:e0ed796530641c8a21a423f9bb7882117dbbeee11ec78dbc335402a678d937ae -colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows" \ - --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ - --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 -deprecated==1.2.13 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d \ - --hash=sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d -filelock==3.11.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37 \ - --hash=sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318 -googleapis-common-protos==1.59.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \ - --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f -grpc-interceptor==0.15.1 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:1cc52c34b0d7ff34512fb7780742ecda37bf3caa18ecc5f33f09b4f74e96b276 \ - --hash=sha256:3efadbc9aead272ac7a360c75c4bd96233094c9a5192dbb51c6156246bd64ba0 -grpcio-reflection==1.53.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:19ac2e3b16404f20bfa14ec64483bb4a936accee50ccb2e64b0159d1cea767cd \ - --hash=sha256:663ea8a9883a64457c1a58b95d52b431f4636337503954e15bfac1d1697b401c -grpcio-status==1.53.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:4f4bc140ed718a343d04a573c838c9fff5cb87f40d3cb2aa33b027c55031489d \ - --hash=sha256:5a46820dc7d94bac48aae5dd97c94cadea09d80a05553aca74a92a2954030420 -grpcio==1.53.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:0698c094688a2dd4c7c2f2c0e3e142cac439a64d1cef6904c97f6cde38ba422f \ - --hash=sha256:104a2210edd3776c38448b4f76c2f16e527adafbde171fc72a8a32976c20abc7 \ - --hash=sha256:14817de09317dd7d3fbc8272864288320739973ef0f4b56bf2c0032349da8cdf \ - --hash=sha256:1948539ce78805d4e6256ab0e048ec793956d54787dc9d6777df71c1d19c7f81 \ - --hash=sha256:19caa5b7282a89b799e63776ff602bb39604f7ca98db6df27e2de06756ae86c3 \ - --hash=sha256:1b172e6d497191940c4b8d75b53de82dc252e15b61de2951d577ec5b43316b29 \ - --hash=sha256:1c734a2d4843e4e14ececf5600c3c4750990ec319e1299db7e4f0d02c25c1467 \ - --hash=sha256:2a912397eb8d23c177d6d64e3c8bc46b8a1c7680b090d9f13a640b104aaec77c \ - --hash=sha256:2eddaae8af625e45b5c8500dcca1043264d751a6872cde2eda5022df8a336959 \ - --hash=sha256:55930c56b8f5b347d6c8c609cc341949a97e176c90f5cbb01d148d778f3bbd23 \ - --hash=sha256:658ffe1e39171be00490db5bd3b966f79634ac4215a1eb9a85c6cd6783bf7f6e \ - --hash=sha256:6601d812105583948ab9c6e403a7e2dba6e387cc678c010e74f2d6d589d1d1b3 \ - --hash=sha256:6b6d60b0958be711bab047e9f4df5dbbc40367955f8651232bfdcdd21450b9ab \ - --hash=sha256:6beb84f83360ff29a3654f43f251ec11b809dcb5524b698d711550243debd289 \ - --hash=sha256:752d2949b40e12e6ad3ed8cc552a65b54d226504f6b1fb67cab2ccee502cc06f \ - --hash=sha256:7dc8584ca6c015ad82e186e82f4c0fe977394588f66b8ecfc4ec873285314619 \ - --hash=sha256:82434ba3a5935e47908bc861ce1ebc43c2edfc1001d235d6e31e5d3ed55815f7 \ - --hash=sha256:8270d1dc2c98ab57e6dbf36fa187db8df4c036f04a398e5d5e25b4e01a766d70 \ - --hash=sha256:8a48fd3a7222be226bb86b7b413ad248f17f3101a524018cdc4562eeae1eb2a3 \ - --hash=sha256:95952d3fe795b06af29bb8ec7bbf3342cdd867fc17b77cc25e6733d23fa6c519 \ - --hash=sha256:976a7f24eb213e8429cab78d5e120500dfcdeb01041f1f5a77b17b9101902615 \ - --hash=sha256:9c84a481451e7174f3a764a44150f93b041ab51045aa33d7b5b68b6979114e48 \ - --hash=sha256:a34d6e905f071f9b945cabbcc776e2055de1fdb59cd13683d9aa0a8f265b5bf9 \ - --hash=sha256:a4952899b4931a6ba12951f9a141ef3e74ff8a6ec9aa2dc602afa40f63595e33 \ - --hash=sha256:a96c3c7f564b263c5d7c0e49a337166c8611e89c4c919f66dba7b9a84abad137 \ - --hash=sha256:aef7d30242409c3aa5839b501e877e453a2c8d3759ca8230dd5a21cda029f046 \ - --hash=sha256:b5bd026ac928c96cc23149e6ef79183125542062eb6d1ccec34c0a37e02255e7 \ - --hash=sha256:b6a2ead3de3b2d53119d473aa2f224030257ef33af1e4ddabd4afee1dea5f04c \ - --hash=sha256:ba074af9ca268ad7b05d3fc2b920b5fb3c083da94ab63637aaf67f4f71ecb755 \ - --hash=sha256:c5fb6f3d7824696c1c9f2ad36ddb080ba5a86f2d929ef712d511b4d9972d3d27 \ - --hash=sha256:c705e0c21acb0e8478a00e7e773ad0ecdb34bd0e4adc282d3d2f51ba3961aac7 \ - --hash=sha256:c7ad9fbedb93f331c2e9054e202e95cf825b885811f1bcbbdfdc301e451442db \ - --hash=sha256:da95778d37be8e4e9afca771a83424f892296f5dfb2a100eda2571a1d8bbc0dc \ - --hash=sha256:dad5b302a4c21c604d88a5d441973f320134e6ff6a84ecef9c1139e5ffd466f6 \ - --hash=sha256:dbc1ba968639c1d23476f75c356e549e7bbf2d8d6688717dcab5290e88e8482b \ - --hash=sha256:ddb2511fbbb440ed9e5c9a4b9b870f2ed649b7715859fd6f2ebc585ee85c0364 \ - --hash=sha256:df9ba1183b3f649210788cf80c239041dddcb375d6142d8bccafcfdf549522cd \ - --hash=sha256:e4f513d63df6336fd84b74b701f17d1bb3b64e9d78a6ed5b5e8a198bbbe8bbfa \ - --hash=sha256:e6f90698b5d1c5dd7b3236cd1fa959d7b80e17923f918d5be020b65f1c78b173 \ - --hash=sha256:eaf8e3b97caaf9415227a3c6ca5aa8d800fecadd526538d2bf8f11af783f1550 \ - --hash=sha256:ee81349411648d1abc94095c68cd25e3c2812e4e0367f9a9355be1e804a5135c \ - --hash=sha256:f144a790f14c51b8a8e591eb5af40507ffee45ea6b818c2482f0457fec2e1a2e \ - --hash=sha256:f3e837d29f0e1b9d6e7b29d569e2e9b0da61889e41879832ea15569c251c303a \ - --hash=sha256:fa8eaac75d3107e3f5465f2c9e3bbd13db21790c6e45b7de1756eba16b050aca \ - --hash=sha256:fdc6191587de410a184550d4143e2b24a14df495c86ca15e59508710681690ac -hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:0c1bdfa554f0b0936c1623b99058c6998a00fdcd86f75d9203f3f66572d2e30c \ - --hash=sha256:334862f4a82f8a09d6d3f550e67d7e498bb8882e678b7725638254fed3276801 \ - --hash=sha256:3ce6c5965a57d94db5e043aa488a4df929a32000db125d9c9a1d325e8c7006dc \ - --hash=sha256:445edfcd9d59c9d2541957177a9c9225b1f0e8855f6311fb16e20f67c3426421 \ - --hash=sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07 \ - --hash=sha256:606f2fe436e5be73f07987a56cd97c695805413d29203ae39ebd9fc596405435 \ - --hash=sha256:617692a70cf299576d82cfc860923f29ec5c834a3f5242bc0853d4f106670398 \ - --hash=sha256:6e5201b648df6106c232fcdb507db734081fd6220dfb1c432bd27c6fa9453331 \ - --hash=sha256:79099ac043423b263a2843a24213418f309d5c8bc458776622bffe012ebced73 \ - --hash=sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c \ - --hash=sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9 \ - --hash=sha256:8a7934c8e491bb395731f677f66dd5f6641432f338a3a9efc9f0b6c186d37cf8 \ - --hash=sha256:9913f5ad2644a1f57c1b7755a7d959ca5e0189863bb0473817d0707af230bf6a \ - --hash=sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6 \ - --hash=sha256:ca218fb6efc5c22379d9e64989e84bff426fcf5664fdbbf7cd70aa8b79497652 \ - --hash=sha256:d53954ecfaadc84c15481bf5d4c7282323196b4b6df1d1be54208d4fdedfb407 \ - --hash=sha256:d935946791e338f748e05a23df877d74fbcd39dc7b537f0aa2e5a5841cf7dde8 \ - --hash=sha256:da92a1483a66cf2baa96de133c75e7d5d9a60e4a0e60d228f26c573c73a1feb6 \ - --hash=sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9 -idna==3.4 ; python_version >= "3.9" and python_version < "4" \ - --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ - --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 -jinja2==3.1.2 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ - --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 -lit==16.0.1 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:630a47291b714cb115015df23ab04267c24fe59aec7ecd7e637d5c75cdb45c91 -loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c \ - --hash=sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3 -markupsafe==2.1.2 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed \ - --hash=sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc \ - --hash=sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2 \ - --hash=sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460 \ - --hash=sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7 \ - --hash=sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0 \ - --hash=sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1 \ - --hash=sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa \ - --hash=sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03 \ - --hash=sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323 \ - --hash=sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65 \ - --hash=sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013 \ - --hash=sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036 \ - --hash=sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f \ - --hash=sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4 \ - --hash=sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419 \ - --hash=sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2 \ - --hash=sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619 \ - --hash=sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a \ - --hash=sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a \ - --hash=sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd \ - --hash=sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7 \ - --hash=sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666 \ - --hash=sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65 \ - --hash=sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859 \ - --hash=sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625 \ - --hash=sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff \ - --hash=sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156 \ - --hash=sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd \ - --hash=sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba \ - --hash=sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f \ - --hash=sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1 \ - --hash=sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094 \ - --hash=sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a \ - --hash=sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513 \ - --hash=sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed \ - --hash=sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d \ - --hash=sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3 \ - --hash=sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147 \ - --hash=sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c \ - --hash=sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603 \ - --hash=sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601 \ - --hash=sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a \ - --hash=sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1 \ - --hash=sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d \ - --hash=sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3 \ - --hash=sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54 \ - --hash=sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2 \ - --hash=sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6 \ - --hash=sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58 -mpmath==1.3.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ - --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c -networkx==3.1 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36 \ - --hash=sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61 -numpy==1.24.2 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22 \ - --hash=sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f \ - --hash=sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9 \ - --hash=sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96 \ - --hash=sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0 \ - --hash=sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a \ - --hash=sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281 \ - --hash=sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04 \ - --hash=sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468 \ - --hash=sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253 \ - --hash=sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756 \ - --hash=sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a \ - --hash=sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb \ - --hash=sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d \ - --hash=sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0 \ - --hash=sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910 \ - --hash=sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978 \ - --hash=sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5 \ - --hash=sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f \ - --hash=sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a \ - --hash=sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5 \ - --hash=sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2 \ - --hash=sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d \ - --hash=sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95 \ - --hash=sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5 \ - --hash=sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d \ - --hash=sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780 \ - --hash=sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa -nvidia-cublas-cu11==11.10.3.66 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e \ - --hash=sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded -nvidia-cuda-cupti-cu11==11.7.101 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:7cc5b8f91ae5e1389c3c0ad8866b3b016a175e827ea8f162a672990a402ab2b0 \ - --hash=sha256:e0cfd9854e1f2edaa36ca20d21cd0bdd5dcfca4e3b9e130a082e05b33b6c5895 -nvidia-cuda-nvrtc-cu11==11.7.99 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03 \ - --hash=sha256:f2effeb1309bdd1b3854fc9b17eaf997808f8b25968ce0c7070945c4265d64a3 \ - --hash=sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7 -nvidia-cuda-runtime-cu11==11.7.99 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7 \ - --hash=sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31 -nvidia-cudnn-cu11==8.5.0.96 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7 \ - --hash=sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108 -nvidia-cufft-cu11==10.9.0.58 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:222f9da70c80384632fd6035e4c3f16762d64ea7a843829cb278f98b3cb7dd81 \ - --hash=sha256:c4d316f17c745ec9c728e30409612eaf77a8404c3733cdf6c9c1569634d1ca03 -nvidia-curand-cu11==10.2.10.91 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:eecb269c970fa599a2660c9232fa46aaccbf90d9170b96c462e13bcb4d129e2c \ - --hash=sha256:f742052af0e1e75523bde18895a9ed016ecf1e5aa0ecddfcc3658fd11a1ff417 -nvidia-cusolver-cu11==11.4.0.1 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:00f70b256add65f8c1eb3b6a65308795a93e7740f6df9e273eccbba770d370c4 \ - --hash=sha256:700b781bfefd57d161443aff9ace1878584b93e0b2cfef3d6e9296d96febbf99 \ - --hash=sha256:72fa7261d755ed55c0074960df5904b65e2326f7adce364cbe4945063c1be412 -nvidia-cusparse-cu11==11.7.4.91 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:304a01599534f5186a8ed1c3756879282c72c118bc77dd890dc1ff868cad25b9 \ - --hash=sha256:a3389de714db63321aa11fbec3919271f415ef19fda58aed7f2ede488c32733d -nvidia-nccl-cu11==2.14.3 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:5e5534257d1284b8e825bc3a182c6f06acd6eb405e9f89d49340e98cd8f136eb -nvidia-nvtx-cu11==11.7.91 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:b22c64eee426a62fc00952b507d6d29cf62b4c9df7a480fcc417e540e05fd5ac \ - --hash=sha256:dfd7fcb2a91742513027d63a26b757f38dd8b07fecac282c4d132a9d373ff064 -opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded \ - --hash=sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0 -opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:844f2a4bb9bcda34e4eb6fe36765e5031aacb36dc60ed88c90fc246942ea26e7 \ - --hash=sha256:c2a5492ba7d140109968135d641d06ce3c5bd73c50665f787526065d57d7fd1d -opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:11b2c814249a49b22f6cca7a06b05701f561d577b747f3660dfd67b6eb9daf9c \ - --hash=sha256:3ec2a02196c8a54bf5cbf7fe623a5238625638e83b6047a983bdf96e2bbb74c0 -opentelemetry-exporter-otlp==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:4f7c49751d9720e2e726e13b0bb958ccade4e29122c305d92c033da432c8d2c5 \ - --hash=sha256:79f22748b6a54808a0448093dfa189c8490e729f67c134d4c992533d9393b33e -opentelemetry-instrumentation-grpc==0.36b0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:dc89447c9eb6ea868970f6c13b4ffdac182cdd5a41dd215a0f5393ca6375be55 \ - --hash=sha256:eaa246ed2083c97b13bab2555cb9d170e8433230a31476c4cab8a17fa03380a4 -opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:83ba4ae7d5292b5b33e0f851cc5c76d8f91196b9b3527800fc13855c33383ac2 \ - --hash=sha256:e3ddac9b3b93408ef26c8ecbf38f717042977e16381bb4cd329a5b4cf16998cf -opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:044b6d044b4d10530f250856f933442b8753a17f94ae37c207607f733fb9a844 \ - --hash=sha256:9c4008e40ac8cab359daac283fbe7002c5c29c77ea2674ad5626a249e64e0101 -opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:555c533e9837766119bbccc7a80458c9971d853a6f1da683a2246cd5e53b4645 \ - --hash=sha256:98dbffcfeebcbff12c0c974292d6ea603180a145904cf838b1fe4d5c99078425 -opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:829dc221795467d98b773c04096e29be038d77526dc8d6ac76f546fb6279bf01 \ - --hash=sha256:adc05635e87b9d3e007c9f530eed487fc3ef2177d02f82f674f28ebf9aff8243 -packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ - --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f -protobuf==4.22.3 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:13233ee2b9d3bd9a5f216c1fa2c321cd564b93d8f2e4f521a85b585447747997 \ - --hash=sha256:23452f2fdea754a8251d0fc88c0317735ae47217e0d27bf330a30eec2848811a \ - --hash=sha256:52f0a78141078077cfe15fe333ac3e3a077420b9a3f5d1bf9b5fe9d286b4d881 \ - --hash=sha256:70659847ee57a5262a65954538088a1d72dfc3e9882695cab9f0c54ffe71663b \ - --hash=sha256:7760730063329d42a9d4c4573b804289b738d4931e363ffbe684716b796bde51 \ - --hash=sha256:7cf56e31907c532e460bb62010a513408e6cdf5b03fb2611e4b67ed398ad046d \ - --hash=sha256:8b54f56d13ae4a3ec140076c9d937221f887c8f64954673d46f63751209e839a \ - --hash=sha256:d14fc1a41d1a1909998e8aff7e80d2a7ae14772c4a70e4bf7db8a36690b54425 \ - --hash=sha256:d4b66266965598ff4c291416be429cef7989d8fae88b55b62095a2331511b3fa \ - --hash=sha256:e0e630d8e6a79f48c557cd1835865b593d0547dce221c66ed1b827de59c66c97 \ - --hash=sha256:ecae944c6c2ce50dda6bf76ef5496196aeb1b85acb95df5843cd812615ec4b61 \ - --hash=sha256:f08aa300b67f1c012100d8eb62d47129e53d1150f4469fd78a29fa3cb68c66f2 \ - --hash=sha256:f2f4710543abec186aee332d6852ef5ae7ce2e9e807a3da570f36de5a732d88e -psutil==5.9.4 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff \ - --hash=sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1 \ - --hash=sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62 \ - --hash=sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549 \ - --hash=sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08 \ - --hash=sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7 \ - --hash=sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e \ - --hash=sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe \ - --hash=sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24 \ - --hash=sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad \ - --hash=sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94 \ - --hash=sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8 \ - --hash=sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7 \ - --hash=sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4 -pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf \ - --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ - --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ - --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ - --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ - --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ - --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ - --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ - --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ - --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ - --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ - --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ - --hash=sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782 \ - --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ - --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ - --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ - --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ - --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ - --hash=sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1 \ - --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ - --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ - --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ - --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ - --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ - --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ - --hash=sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d \ - --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ - --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ - --hash=sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7 \ - --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ - --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ - --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ - --hash=sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358 \ - --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ - --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ - --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ - --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ - --hash=sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f \ - --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ - --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 -requests==2.28.2 ; python_version >= "3.9" and python_version < "4" \ - --hash=sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa \ - --hash=sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf -safetensors==0.2.8 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:01000a4bfdf0474bb1bdf369de1284c93b4e6047524fe9dc55d77586cb9d0243 \ - --hash=sha256:02e67b906ad9bbb87308a34e4d2d33c9eb69baf342b7e5c872728556baf3f0b6 \ - --hash=sha256:1b905cf3563da4fe8dd46fa777321516f5aa8f6bc1b884158be03a235478e96d \ - --hash=sha256:2720b20a6a38c799dca79bd76caeeac2f7df585a9d4f7d59fa7e28eff9ccb27f \ - --hash=sha256:2a47327cfe57b6ee8c5dc246f9141f4f6b368e4444dd7f174c025b1d34446730 \ - --hash=sha256:2bd2e4c6258dd0f4e3d554f2f59789f25ef4757431e83c927016de6339e54811 \ - --hash=sha256:2f16e5ee70ae4218474493eff8d998e632a896a6b8aff9273511d654cdf367ab \ - --hash=sha256:2f40604c50d4a08a4c74f37fef735cd1e203a59aeda66ea23a01d76fb35cf407 \ - --hash=sha256:2f90513eee0a11902df7e51b07c3d9c328828b9dd692d6c74140bed932e7a491 \ - --hash=sha256:34705920c8a02f9ea6101bae8403db5f4aa18ec3eaccd8eab6701b1c88ee5bed \ - --hash=sha256:384df328523c567a8b2193ebb02456300a43a3adbc316823d4c0d16f7ac9e89d \ - --hash=sha256:466f92a384e4fcb6b1b9811e7488516c4638119c302a959b89bbe1be826d5e25 \ - --hash=sha256:50f7e90ed02ef5e4263aadce23d39a3595156816c42c4643003791b45f81fd31 \ - --hash=sha256:6273dd3edd5db05b2da0090bc3d491bf25f6d1d7e8a4423477377649e9e38c37 \ - --hash=sha256:70f322d170a17b6ecb9c85b15e67f4a9aa3e561594e2dfc7709c0ae0000ebfff \ - --hash=sha256:726ad66231286157dd505b5ab85fd903114dcb5f9180e79fd5540d68d6249dd0 \ - --hash=sha256:7716bab34ca9651895126c720df1bf07f464184a7409138179f38db488ca9f15 \ - --hash=sha256:832f27f6f379fb15d87f3a10eb87e2480434a89c483d7edc8a0c262364ba72c1 \ - --hash=sha256:89da823f9801e92b2c48e8fad1e2f7f0cb696a8a93dab4c6700e8de06fe87392 \ - --hash=sha256:8df8af89a0b6b535b47c077e33e5cd4941ef4b067e7c1dd1a05647dec0cf2eea \ - --hash=sha256:90cd22387b1520c4465033b986f79f0d24cc41aabae1903a22eff3b42cee9ad5 \ - --hash=sha256:9366140785d567049395c1c03ac69ee8d322fabcdc8fab7d389e933596b383da \ - --hash=sha256:9cabae651542b22d229b6439029fb4a3abc7868cb123af336b2877541ad9ab39 \ - --hash=sha256:a0c7ed4d75c2f248791dbe64309c98ada6f40a6949147ca2eaebd278906c918b \ - --hash=sha256:aa5f4b31f2e283b83894b388298368860367a1cb773228f9bb65f8db65da1549 \ - --hash=sha256:ac2197dbbf7cbd269faf8cebab4220dba5aa2ac8beacbce8fdcb9800776781ca \ - --hash=sha256:af4fce0565574ec3dbe997b021ed32d3dc229547c4f7fca2459be1f725c26f88 \ - --hash=sha256:ba3dc236a2344b7feadc9868307f42ba5e4804c9d68a80a35aac831349b31f6f \ - --hash=sha256:ba7c2496765de45a84f5c79b2b12a14a568643d8966ef9bb3f8b16217a39457c \ - --hash=sha256:c02c1cab1d23e6cc8ac1ef865efe76f2d0800e203c877288ebd31747f47a6940 \ - --hash=sha256:c5d720c7640ad5f95f47780bdc35777ed8371afa14d8d63d6375cfe36df83fb4 \ - --hash=sha256:c71e147a8c2942690e8707adbcc4ab60775bc78dfdbd7f9e696dd411adef82bb \ - --hash=sha256:d14e7c15e5acac6efcd5f5565e38b1b33600101387e5d16059de44adab87405f \ - --hash=sha256:dec20b9b1fc90b7b4e588b4f0e9e266bd8f26d405e08b0b6ecad3136d92d007a \ - --hash=sha256:eb28e5e6257f705828fd39b9ba28248b593f46e722d8d6beedbc8d1f194e2297 \ - --hash=sha256:ee8e169040468d176172b4f84a160ece2064abcc77294c4994a9d2bb5255cd75 \ - --hash=sha256:efff5ce2d3f349d5360a5ca5901ae56c7e24f8da623d57cd08f8e8b1cd9cb1f8 \ - --hash=sha256:f279e0fd917a886e1936d534734593f780afdddc6ed62f8ebf2f59de811cdd7c \ - --hash=sha256:f3c7242debe713a87ca6deaadb0d7120e61c92435414142d526e8c32da970020 \ - --hash=sha256:ff0096f5a765e6e3f7f3156f568f59499edeade19e813d610a124ca70b42cdda -sentencepiece==0.1.98 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:00b2becbd7b98905a6de9695cf8682abe0d510ab0198e23c7d86fb2b793b6ae0 \ - --hash=sha256:05b4eecbece0606883cd81ed86bb3c619680bb570b997b236533ec854d64a575 \ - --hash=sha256:11f410cc7eeb3e1cfa8d92d128b568e5dc7829b7904b164499fd0209316ec2fa \ - --hash=sha256:12c913493d6ebac86ee7ae109e368522a5a365a7b150d4d8cf845599262d2b21 \ - --hash=sha256:14841bd2a3d77c4dbba58f02488c374866551e428d755e8d473d82325a0a94f3 \ - --hash=sha256:1daf0a79cd953e4830746c41e92b98a2f2e9e5ec0e90a9447aa10350e11bd027 \ - --hash=sha256:2766cd708e9fc2b5b9784a990a8b303b9e0b9a69fa482616fe86fa538daa1756 \ - --hash=sha256:2c2d9a74986d3716dc6961e9dbae7a3b25bb1260118f098545fd963ae23252c1 \ - --hash=sha256:2d50edfc4649a1566b64f1a8402cd607e1893bf8e368732337d83f00df62d3fa \ - --hash=sha256:35af00f5103a4779694fedea41b6e24947a9ed81166efe63864ab1e781d70a66 \ - --hash=sha256:3f9239785849ed1f55a825bcc282bef1a6073f7431cc535bdc658a94873652ea \ - --hash=sha256:443f32e94b18571231b02a51be173686114b5556b5edfcbf347fb63e7bd5ddc6 \ - --hash=sha256:467740ef8af170e5b6cfe22c272114ed930c899c297619ac7a2ac463a13bdbac \ - --hash=sha256:472ad943eaffcb6871ece56c7850388e7b8722f520ba73c93e7a6ef965453221 \ - --hash=sha256:4b7e23aaf9d5afd91ca13550968bd17f0c17b0966823188ad2a50c51544cf8ed \ - --hash=sha256:515a971c2a157647ca0e60ce3c435f4b43cd5c9f5862159cfefa0b5b4d46d3c3 \ - --hash=sha256:558a373a8660bdff299d6c133c2a4f4fb0875e9e6fafe225b8080ecce8a405f9 \ - --hash=sha256:57911445fc91c80d59552adf8a749af9205458920a7328f3bd7d51308658bcd9 \ - --hash=sha256:58ca96d73ea0e5575e3f6a9524449c673d62e6ecee3b2ddd5bfb4f49cb315c0a \ - --hash=sha256:6150ba525fac4fda76f5c4777ae300597e70cef739ed2a47cea02ff81a88873f \ - --hash=sha256:64e32c55d04a2e21f0c2fda1b7a3dd108133ebfb8616b52896916bb30e4352ed \ - --hash=sha256:6b0ce9efc790c209cce2463058855dceb21438213d2ff13cb5a565d52a7efe25 \ - --hash=sha256:720f827dc69ee24951ea4f51b9fb69cc56890a7190fc52c2c0da2545caab1760 \ - --hash=sha256:7287461d2346f530928ab187f0834cb15ddfbc4553592cacdcb6470364739ec6 \ - --hash=sha256:7425b727c3d6b3b7bad0005a3be316078b254180b712d73955ff08cae3f6a385 \ - --hash=sha256:7f71c4bdedb797052fb2ccad0871c2409bf6f812cb6b651917c55f9e8eced07f \ - --hash=sha256:7f7dc2fc175623529fb60a2799748f8877cd48c4541b32cd97b8523465e88b69 \ - --hash=sha256:8663be00a68098f85d6cda1f7041a27de05c320e433fa730ecb1156a8304f21c \ - --hash=sha256:8abe5c4c034e497e69f485dcd2c0e6bc87bf0498ad5aef5f539a7d0f9eae6275 \ - --hash=sha256:8b50cbe8e46204eff7aa5a663af5652c45e7807aa560d08e5f5b10c60e795a49 \ - --hash=sha256:918b4caf18b2f73c302c4e197d1c2dafba39eb143d16b4590930b45f15042fdd \ - --hash=sha256:947cf0a4b8a480510d560a922f8256f34e93984a86cf870be4d05731f59fb28d \ - --hash=sha256:9b6f0b9ffb601e2699e265f3f20c353ec9a661e4b5f0cff08ad6c9909c0ae43e \ - --hash=sha256:b2531c0e9cc8cd404fabd856d80d695b373371c00f1fce29c06f41f3f7429d87 \ - --hash=sha256:b6ed62f89c0bd25cec39a7075f6b9354fe4c240ed964e63009d77efcf29c34e9 \ - --hash=sha256:c067ba22be8edc699f6365e01ec15046bf3563dbabfdc052ecc88e581b675cba \ - --hash=sha256:c23c3a562221bc40eaae42428fcd8e607e0f084ea8aa968ba3f1a7d0ea975807 \ - --hash=sha256:c5ea8fb2c68073fe25a08a178eed269ed382fba074ff2ba4de72f0f56d86630e \ - --hash=sha256:daf05611089a075b78d353720ccc3a09a78e0846332cff0cc78fda8b2383626a \ - --hash=sha256:e54aa70b574eee895d184072d84e62824f404821e551a82c619c5d4320a93834 \ - --hash=sha256:e7a828f1fe2e51d2d9e5e9b3283d4006f1891efb02a3d9303ed39ddafdd9c864 \ - --hash=sha256:ef384b31ec7a06a9a6aba42e68435f3f3b38809aa65559ede3658cdd446a562c \ - --hash=sha256:fa13a125417d28e84fbdebcaf6aa115e4177d3e93aa66b857a42e7179f515b88 \ - --hash=sha256:fcf100268cefe1774794b18cbaf3065e2bf988f168a387973eb1260d51198795 \ - --hash=sha256:ffcc78e80c55eab67ee3439ade493607a4e37e1f0b82b168ead3debf9eaeaabe -setuptools==67.6.1 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a \ - --hash=sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078 -sympy==1.11.1 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:938f984ee2b1e8eae8a07b884c8b7a1146010040fccddc6539c54f401c8f6fcf \ - --hash=sha256:e32380dce63cb7c0108ed525570092fd45168bdae2faa17e528221ef72e88658 -tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:00cee1e0859d55507e693a48fa4aef07060c4bb6bd93d80120e18fea9371c66d \ - --hash=sha256:0527dc5436a1f6bf2c0327da3145687d3bcfbeab91fed8458920093de3901b44 \ - --hash=sha256:07cbb2c307627dc99b44b22ef05ff4473aa7c7cc1fec8f0a8b37d8a64b1a16d2 \ - --hash=sha256:1f0e3b4c2ea2cd13238ce43548959c118069db7579e5d40ec270ad77da5833ce \ - --hash=sha256:280ffe95f50eaaf655b3a1dc7ff1d9cf4777029dbbc3e63a74e65a056594abc3 \ - --hash=sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e \ - --hash=sha256:310204dfed5aa797128b65d63538a9837cbdd15da2a29a77d67eefa489edda26 \ - --hash=sha256:3791338f809cd1bf8e4fee6b540b36822434d0c6c6bc47162448deee3f77d425 \ - --hash=sha256:4560dbdeaae5b7ee0d4e493027e3de6d53c991b5002d7ff95083c99e11dd5ac0 \ - --hash=sha256:48625a108029cb1ddf42e17a81b5a3230ba6888a70c9dc14e81bc319e812652d \ - --hash=sha256:4b3e3215d048e94f40f1c95802e45dcc37c5b05eb46280fc2ccc8cd351bff839 \ - --hash=sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07 \ - --hash=sha256:56b726e0d2bbc9243872b0144515ba684af5b8d8cd112fb83ee1365e26ec74c8 \ - --hash=sha256:5ef4215284df1277dadbcc5e17d4882bda19f770d02348e73523f7e7d8b8d396 \ - --hash=sha256:64064bd0322405c9374305ab9b4c07152a1474370327499911937fd4a76d004b \ - --hash=sha256:6cc29d410768f960db8677221e497226e545eaaea01aa3613fa0fdf2cc96cff4 \ - --hash=sha256:70ce07445050b537d2696022dafb115307abdffd2a5c106f029490f84501ef97 \ - --hash=sha256:8791dedba834c1fc55e5f1521be325ea3dafb381964be20684b92fdac95d79b7 \ - --hash=sha256:89649c00d0d7211e8186f7a75dfa1db6996f65edce4b84821817eadcc2d3c79e \ - --hash=sha256:8e7b0cdeace87fa9e760e6a605e0ae8fc14b7d72e9fc19c578116f7287bb873d \ - --hash=sha256:97acfcec592f7e9de8cadcdcda50a7134423ac8455c0166b28c9ff04d227b371 \ - --hash=sha256:9a3fa134896c3c1f0da6e762d15141fbff30d094067c8f1157b9fdca593b5806 \ - --hash=sha256:9ba2b0bf01777c9b9bc94b53764d6684554ce98551fec496f71bc5be3a03e98b \ - --hash=sha256:a0f9b92ea052305166559f38498b3b0cae159caea712646648aaa272f7160963 \ - --hash=sha256:a23ff602d0797cea1d0506ce69b27523b07e70f6dda982ab8cf82402de839088 \ - --hash=sha256:a4d53976079cff8a033f778fb9adca2d9d69d009c02fa2d71a878b5f3963ed30 \ - --hash=sha256:b8c6e2ab0f2e3d939ca66aa1d596602105fe33b505cd2854a4c1717f704c51de \ - --hash=sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783 \ - --hash=sha256:c2f35f30e39e6aab8716f07790f646bdc6e4a853816cc49a95ef2a9016bf9ce6 \ - --hash=sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc \ - --hash=sha256:cc5c022ce692e1f499d745af293ab9ee6f5d92538ed2faf73f9708c89ee59ce6 \ - --hash=sha256:cc78d77f597d1c458bf0ea7c2a64b6aa06941c7a99cb135b5969b0278824d808 \ - --hash=sha256:d607a6a13718aeb20507bdf2b96162ead5145bbbfa26788d6b833f98b31b26e1 \ - --hash=sha256:dd7730c98a3010cd4f523465867ff95cd9d6430db46676ce79358f65ae39797b \ - --hash=sha256:ecf182bf59bd541a8876deccf0360f5ae60496fd50b58510048020751cf1724c \ - --hash=sha256:ee0b1b311d65beab83d7a41c56a1e46ab732a9eed4460648e8eb0bd69fc2d059 \ - --hash=sha256:f247eae99800ef821a91f47c5280e9e9afaeed9980fc444208d5aa6ba69ff148 \ - --hash=sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33 \ - --hash=sha256:f55c981ac44ba87c93e847c333e58c12abcbb377a0c2f2ef96e1a266e4184ff2 \ - --hash=sha256:fc2a7fdf864554a0dacf09d32e17c0caa9afe72baf9dd7ddedc61973bae352d8 -torch==2.0.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:01858620f25f25e7a9ec4b547ff38e5e27c92d38ec4ccba9cfbfb31d7071ed9c \ - --hash=sha256:09651bff72e439d004c991f15add0c397c66f98ab36fe60d5514b44e4da722e8 \ - --hash=sha256:11b0384fe3c18c01b8fc5992e70fc519cde65e44c51cc87be1838c1803daf42f \ - --hash=sha256:2802f84f021907deee7e9470ed10c0e78af7457ac9a08a6cd7d55adef835fede \ - --hash=sha256:297a4919aff1c0f98a58ebe969200f71350a1d4d4f986dbfd60c02ffce780e99 \ - --hash=sha256:527f4ae68df7b8301ee6b1158ca56350282ea633686537b30dbb5d7b4a52622a \ - --hash=sha256:53e1c33c6896583cdb9a583693e22e99266444c4a43392dddc562640d39e542b \ - --hash=sha256:6befaad784004b7af357e3d87fa0863c1f642866291f12a4c2af2de435e8ac5c \ - --hash=sha256:6e0b97beb037a165669c312591f242382e9109a240e20054d5a5782d9236cad0 \ - --hash=sha256:7a9319a67294ef02459a19738bbfa8727bb5307b822dadd708bc2ccf6c901aca \ - --hash=sha256:8969aa8375bcbc0c2993e7ede0a7f889df9515f18b9b548433f412affed478d9 \ - --hash=sha256:9a2e53b5783ef5896a6af338b36d782f28e83c8ddfc2ac44b67b066d9d76f498 \ - --hash=sha256:9f01fe1f6263f31bd04e1757946fd63ad531ae37f28bb2dbf66f5c826ee089f4 \ - --hash=sha256:a83b26bd6ae36fbf5fee3d56973d9816e2002e8a3b7d9205531167c28aaa38a7 \ - --hash=sha256:ab2da16567cb55b67ae39e32d520d68ec736191d88ac79526ca5874754c32203 \ - --hash=sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7 \ - --hash=sha256:c7e67195e1c3e33da53954b026e89a8e1ff3bc1aeb9eb32b677172d4a9b5dcbf \ - --hash=sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a \ - --hash=sha256:cc788cbbbbc6eb4c90e52c550efd067586c2693092cf367c135b34893a64ae78 \ - --hash=sha256:ce9b5a49bd513dff7950a5a07d6e26594dd51989cee05ba388b03e8e366fd5d5 \ - --hash=sha256:d292640f0fd72b7a31b2a6e3b635eb5065fcbedd4478f9cad1a1e7a9ec861d35 \ - --hash=sha256:d439aec349c98f12819e8564b8c54008e4613dd4428582af0e6e14c24ca85870 \ - --hash=sha256:e54846aa63855298cfb1195487f032e413e7ac9cbfa978fda32354cc39551475 \ - --hash=sha256:ec5fff2447663e369682838ff0f82187b4d846057ef4d119a8dea7772a0b17dd -triton==2.0.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:0117722f8c2b579cd429e0bee80f7731ae05f63fe8e9414acd9a679885fcbf42 \ - --hash=sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a \ - --hash=sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1 \ - --hash=sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505 \ - --hash=sha256:42a0d2c3fc2eab4ba71384f2e785fbfd47aa41ae05fa58bf12cb31dcbd0aeceb \ - --hash=sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08 \ - --hash=sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd \ - --hash=sha256:52c47b72c72693198163ece9d90a721299e4fb3b8e24fd13141e384ad952724f \ - --hash=sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656 \ - --hash=sha256:75834f27926eab6c7f00ce73aaf1ab5bfb9bec6eb57ab7c0bfc0a23fac803b4c \ - --hash=sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2 \ - --hash=sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add \ - --hash=sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef \ - --hash=sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd \ - --hash=sha256:bcd9be5d0c2e45d2b7e6ddc6da20112b6862d69741576f9c3dbaf941d745ecae \ - --hash=sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b \ - --hash=sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a \ - --hash=sha256:fedce6a381901b1547e0e7e1f2546e4f65dca6d91e2d8a7305a2d1f5551895be -typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73 \ - --hash=sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e -typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \ - --hash=sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4 -urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4" \ - --hash=sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305 \ - --hash=sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42 -wheel==0.40.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873 \ - --hash=sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247 -win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" \ - --hash=sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2 \ - --hash=sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad -wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" \ - --hash=sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0 \ - --hash=sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420 \ - --hash=sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a \ - --hash=sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c \ - --hash=sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079 \ - --hash=sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923 \ - --hash=sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f \ - --hash=sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1 \ - --hash=sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8 \ - --hash=sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86 \ - --hash=sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0 \ - --hash=sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364 \ - --hash=sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e \ - --hash=sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c \ - --hash=sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e \ - --hash=sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c \ - --hash=sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727 \ - --hash=sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff \ - --hash=sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e \ - --hash=sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29 \ - --hash=sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7 \ - --hash=sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72 \ - --hash=sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475 \ - --hash=sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a \ - --hash=sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317 \ - --hash=sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2 \ - --hash=sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd \ - --hash=sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640 \ - --hash=sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98 \ - --hash=sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248 \ - --hash=sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e \ - --hash=sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d \ - --hash=sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec \ - --hash=sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1 \ - --hash=sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e \ - --hash=sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9 \ - --hash=sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92 \ - --hash=sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb \ - --hash=sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094 \ - --hash=sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46 \ - --hash=sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29 \ - --hash=sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd \ - --hash=sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705 \ - --hash=sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8 \ - --hash=sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975 \ - --hash=sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb \ - --hash=sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e \ - --hash=sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b \ - --hash=sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418 \ - --hash=sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019 \ - --hash=sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1 \ - --hash=sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba \ - --hash=sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6 \ - --hash=sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2 \ - --hash=sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3 \ - --hash=sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7 \ - --hash=sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752 \ - --hash=sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416 \ - --hash=sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f \ - --hash=sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1 \ - --hash=sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc \ - --hash=sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145 \ - --hash=sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee \ - --hash=sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a \ - --hash=sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7 \ - --hash=sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b \ - --hash=sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653 \ - --hash=sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0 \ - --hash=sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90 \ - --hash=sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29 \ - --hash=sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6 \ - --hash=sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034 \ - --hash=sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09 \ - --hash=sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559 \ - --hash=sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639 +backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" +bitsandbytes==0.38.1 ; python_version >= "3.9" and python_version < "4.0" +certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4" +charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4" +click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" +colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows" +deprecated==1.2.13 ; python_version >= "3.9" and python_version < "4.0" +filelock==3.12.0 ; python_version >= "3.9" and python_version < "4.0" +fsspec==2023.4.0 ; python_version >= "3.9" and python_version < "4.0" +googleapis-common-protos==1.59.0 ; python_version >= "3.9" and python_version < "4.0" +grpc-interceptor==0.15.1 ; python_version >= "3.9" and python_version < "4.0" +grpcio-reflection==1.54.0 ; python_version >= "3.9" and python_version < "4.0" +grpcio-status==1.54.0 ; python_version >= "3.9" and python_version < "4.0" +grpcio==1.54.0 ; python_version >= "3.9" and python_version < "4.0" +hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" +huggingface-hub @ git+https://github.com/huggingface/huggingface_hub.git@4f27b44ee536cd654e171c7f37478eaf1996cc3f ; python_version >= "3.9" and python_version < "4.0" +idna==3.4 ; python_version >= "3.9" and python_version < "4" +loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" +opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" +opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" +opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "4.0" +opentelemetry-exporter-otlp==1.15.0 ; python_version >= "3.9" and python_version < "4.0" +opentelemetry-instrumentation-grpc==0.36b0 ; python_version >= "3.9" and python_version < "4.0" +opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_version < "4.0" +opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "4.0" +opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" +opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" +packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" +protobuf==4.22.3 ; python_version >= "3.9" and python_version < "4.0" +pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" +requests==2.28.2 ; python_version >= "3.9" and python_version < "4" +safetensors==0.2.8 ; python_version >= "3.9" and python_version < "4.0" +sentencepiece==0.1.98 ; python_version >= "3.9" and python_version < "4.0" +setuptools==67.6.1 ; python_version >= "3.9" and python_version < "4.0" +tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" +tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" +typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" +typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "4.0" +urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4" +win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" +wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" From 252f42c1e632698df94d32de118c421b0304bf4c Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 19 Apr 2023 20:06:06 +0200 Subject: [PATCH 175/793] fix(router): add auth token to get model info (#207) --- launcher/src/main.rs | 9 +++++++++ router/src/main.rs | 29 ++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 8dc6a798..fcac736d 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -414,6 +414,14 @@ fn main() -> ExitCode { argv.push(origin); } + // Copy current process env + let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); + + // Parse Inference API token + if let Ok(api_token) = env::var("HF_API_TOKEN") { + env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) + }; + let mut webserver = match Popen::create( &argv, PopenConfig { @@ -421,6 +429,7 @@ fn main() -> ExitCode { stderr: Redirection::Pipe, // Needed for the shutdown procedure setpgid: true, + env: Some(env), ..Default::default() }, ) { diff --git a/router/src/main.rs b/router/src/main.rs index 5fda57be..31783bbe 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -90,6 +90,9 @@ fn main() -> Result<(), std::io::Error> { ) }); + // Parse Huggingface hub token + let authorization_token = std::env::var("HUGGING_FACE_HUB_TOKEN").ok(); + // Tokenizer instance // This will only be used to validate payloads let local_path = Path::new(&tokenizer_name); @@ -102,6 +105,7 @@ fn main() -> Result<(), std::io::Error> { // We need to download it outside of the Tokio runtime let params = FromPretrainedParameters { revision: revision.clone(), + auth_token: authorization_token.clone(), ..Default::default() }; Tokenizer::from_pretrained(tokenizer_name.clone(), Some(params)).ok() @@ -129,7 +133,7 @@ fn main() -> Result<(), std::io::Error> { sha: None, pipeline_tag: None, }, - false => get_model_info(&tokenizer_name, &revision).await, + false => get_model_info(&tokenizer_name, &revision, authorization_token).await, }; // if pipeline-tag == text-generation we default to return_full_text = true @@ -233,14 +237,21 @@ fn init_logging(otlp_endpoint: Option, json_output: bool) { } /// get model info from the Huggingface Hub -pub async fn get_model_info(model_id: &str, revision: &str) -> ModelInfo { - let model_info = reqwest::get(format!( +pub async fn get_model_info(model_id: &str, revision: &str, token: Option) -> ModelInfo { + let client = reqwest::Client::new(); + let mut builder = client.get(format!( "https://huggingface.co/api/models/{model_id}/revision/{revision}" - )) - .await - .expect("Could not connect to hf.co") - .text() - .await - .expect("error when retrieving model info from hf.co"); + )); + if let Some(token) = token { + builder = builder.bearer_auth(token); + } + + let model_info = builder + .send() + .await + .expect("Could not connect to hf.co") + .text() + .await + .expect("error when retrieving model info from hf.co"); serde_json::from_str(&model_info).expect("unable to parse model info") } From b6ee0ec7b06b11ad35fe2f2d99318ddc47f95558 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 19 Apr 2023 21:36:59 +0200 Subject: [PATCH 176/793] feat(router): add git sha to info route (#208) --- .github/workflows/build.yaml | 4 ++++ Dockerfile | 2 ++ router/build.rs | 14 +++++++++++- .../text_generation_server/models/__init__.py | 22 ++++++++++++------- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index aa7db30a..fe7cb706 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -95,6 +95,8 @@ jobs: file: Dockerfile push: ${{ github.event_name != 'pull_request' }} platforms: 'linux/amd64' + build-args: | + GIT_SHA={{ env.GITHUB_SHA }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max @@ -176,6 +178,8 @@ jobs: file: Dockerfile push: ${{ github.event_name != 'pull_request' }} platforms: 'linux/amd64' + build-args: | + GIT_SHA={{ env.GITHUB_SHA }} target: sagemaker tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index 61f37ed9..a679db7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder +ARG GIT_SHA + RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \ curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \ unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \ diff --git a/router/build.rs b/router/build.rs index c34f9fa8..1b1fdc86 100644 --- a/router/build.rs +++ b/router/build.rs @@ -2,6 +2,18 @@ use std::error::Error; use vergen::EmitBuilder; fn main() -> Result<(), Box> { - EmitBuilder::builder().git_sha(false).emit()?; + // Try to get the git sha from the local git repository + if EmitBuilder::builder() + .fail_on_error() + .git_sha(false) + .emit() + .is_err() + { + // Unable to get the git sha + if let Ok(sha) = std::env::var("GIT_SHA") { + // Set it from an env var + println!("cargo:rustc-env=VERGEN_GIT_SHA={sha}"); + } + } Ok(()) } diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 0a29b3cc..74a7483e 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -17,13 +17,6 @@ from text_generation_server.models.t5 import T5Sharded try: - from text_generation_server.models.flash_neox import FlashNeoX, FlashNeoXSharded - from text_generation_server.models.flash_llama import FlashLlama, FlashLlamaSharded - from text_generation_server.models.flash_santacoder import ( - FlashSantacoder, - FlashSantacoderSharded, - ) - if torch.cuda.is_available(): major, minor = torch.cuda.get_device_capability() is_sm75 = major == 7 and minor == 5 @@ -32,7 +25,20 @@ supported = is_sm75 or is_sm8x or is_sm90 if not supported: - raise ImportError(f"GPU with CUDA capability {major} {minor} is not supported") + raise ImportError( + f"GPU with CUDA capability {major} {minor} is not supported" + ) + + from text_generation_server.models.flash_neox import FlashNeoX, FlashNeoXSharded + from text_generation_server.models.flash_llama import ( + FlashLlama, + FlashLlamaSharded, + ) + from text_generation_server.models.flash_santacoder import ( + FlashSantacoder, + FlashSantacoderSharded, + ) + FLASH_ATTENTION = True else: FLASH_ATTENTION = False From 709d8936f68002c2244e245607c6b88d658ebe6f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 20 Apr 2023 11:07:40 +0200 Subject: [PATCH 177/793] feat(router): drop requests when client closes the channel (#202) --- .github/workflows/build.yaml | 2 +- Cargo.lock | 116 +++---- router/src/infer.rs | 205 ++++++++---- router/src/queue.rs | 163 +++++---- router/src/server.rs | 3 +- server/Makefile | 3 + server/tests/models/test_bloom.py | 24 +- server/tests/models/test_causal_lm.py | 26 +- server/tests/models/test_seq2seq_lm.py | 40 ++- .../models/causal_lm.py | 199 ++++++----- .../models/flash_causal_lm.py | 314 +++++++++++------- .../models/galactica.py | 15 +- .../models/seq2seq_lm.py | 277 ++++++++------- server/text_generation_server/models/types.py | 4 + server/text_generation_server/server.py | 7 +- 15 files changed, 824 insertions(+), 574 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fe7cb706..2ad1d424 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,7 +66,7 @@ jobs: password: ${{ secrets.TAILSCALE_DOCKER_PASSWORD }} registry: registry.internal.huggingface.tech - name: Login to Azure Container Registry -# if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' uses: docker/login-action@v2.1.0 with: username: ${{ secrets.AZURE_DOCKER_USERNAME }} diff --git a/Cargo.lock b/Cargo.lock index 8ffaf631..f1d2e3a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,42 +42,51 @@ dependencies = [ [[package]] name = "anstream" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" dependencies = [ "anstyle", "anstyle-parse", + "anstyle-query", "anstyle-wincon", - "concolor-override", - "concolor-query", + "colorchoice", "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "0.3.5" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" [[package]] name = "anstyle-parse" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" dependencies = [ "utf8parse", ] +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "anstyle-wincon" -version = "0.2.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" dependencies = [ "anstyle", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -105,7 +114,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] @@ -116,7 +125,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] @@ -127,9 +136,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.13" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6539e4565c365448d483967c6dee3eaecb8e87679a17806a831e82b05b903c18" +checksum = "3b32c5ea3aabaf4deb5f5ced2d688ec0844c881c9e6c696a8b769a05fc691e62" dependencies = [ "async-trait", "axum-core", @@ -310,9 +319,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.2.1" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" +checksum = "9b802d85aaf3a1cdb02b224ba472ebdea62014fccfcb269b95a4d76443b5ee5a" dependencies = [ "clap_builder", "clap_derive", @@ -321,9 +330,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.2.1" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +checksum = "14a1a858f532119338887a4b8e1af9c60de8249cd7bafd68036a489e261e37b6" dependencies = [ "anstream", "anstyle", @@ -341,7 +350,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] @@ -351,19 +360,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" [[package]] -name = "concolor-override" +name = "colorchoice" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" - -[[package]] -name = "concolor-query" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" -dependencies = [ - "windows-sys 0.45.0", -] +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "console" @@ -794,7 +794,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] @@ -868,9 +868,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.16" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" dependencies = [ "bytes", "fnv", @@ -966,9 +966,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ "bytes", "futures-channel", @@ -1364,7 +1364,7 @@ checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] @@ -1517,7 +1517,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] @@ -1787,9 +1787,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes", "prost-derive", @@ -1797,9 +1797,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c828f93f5ca4826f97fedcbd3f9a536c16b12cff3dbbb4a007f932bbad95b12" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", "heck", @@ -1819,9 +1819,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools 0.10.5", @@ -1832,9 +1832,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ "prost", ] @@ -2153,14 +2153,14 @@ checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] name = "serde_json" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -2330,9 +2330,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.14" +version = "2.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" dependencies = [ "proc-macro2", "quote", @@ -2450,7 +2450,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] @@ -2578,7 +2578,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] @@ -2928,9 +2928,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utoipa" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e7ee17c9ef094b86e1e04170d90765bd76cb381921dacb4d3e175a267bdae6" +checksum = "68ae74ef183fae36d650f063ae7bde1cacbe1cd7e72b617cbe1e985551878b98" dependencies = [ "indexmap", "serde", @@ -2940,14 +2940,14 @@ dependencies = [ [[package]] name = "utoipa-gen" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6f458e5abc811d44aca28455efc4163fb7565a7af2aa32d17611f3d1d9794d" +checksum = "7ea8ac818da7e746a63285594cce8a96f5e00ee31994e655bd827569cb8b137b" dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.15", ] [[package]] diff --git a/router/src/infer.rs b/router/src/infer.rs index 5a4375ae..484720a0 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -3,6 +3,7 @@ use crate::validation::{Validation, ValidationError}; use crate::{Entry, Queue, Token}; use crate::{GenerateRequest, PrefillToken}; use flume::r#async::RecvStream; +use flume::SendError; use futures::future::try_join_all; use futures::stream::StreamExt; use nohash_hasher::IntMap; @@ -11,7 +12,7 @@ use text_generation_client::{ Batch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, }; use thiserror::Error; -use tokio::sync::{Notify, Semaphore, TryAcquireError}; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; use tokio::time::Instant; use tracing::{info_span, instrument, Instrument, Span}; @@ -73,9 +74,14 @@ impl Infer { pub(crate) async fn generate_stream( &self, request: GenerateRequest, - ) -> Result>, InferError> { + ) -> Result< + ( + OwnedSemaphorePermit, + RecvStream>, + ), + InferError, + > { // Limit concurrent requests by acquiring a permit from the semaphore - // This permit will live as long as Entry let permit = self .clone() .limit_concurrent_requests @@ -104,7 +110,6 @@ impl Infer { temp_span: None, queue_time: Instant::now(), batch_time: None, - _permit: permit, }); // Notify the background task that we have a new entry in the queue that needs @@ -112,7 +117,7 @@ impl Infer { self.shared.batching_task.notify_one(); // Return stream - Ok(response_rx.into_stream()) + Ok((permit, response_rx.into_stream())) } /// Add a new request to the queue and return a InferResponse @@ -121,8 +126,8 @@ impl Infer { &self, request: GenerateRequest, ) -> Result { - // Create stream - let mut stream = self.generate_stream(request).await?; + // Create stream and keep semaphore permit as long as generate lives + let (_permit, mut stream) = self.generate_stream(request).await?; // Return values let mut result_prefill = Vec::new(); @@ -276,12 +281,10 @@ async fn batching_task( .next_batch(min_size, max_batch_size - batch_size as usize) .await { - let new_batch_size = new_batch.size; entries.iter_mut().for_each(|(_, entry)| { // Create a new span to add the info that this entry is waiting // because a new batch is being computed - let entry_waiting_span = - info_span!(parent: &entry.span, "waiting", batch_size = new_batch_size); + let entry_waiting_span = info_span!(parent: &entry.span, "waiting"); // Add relationships span.follows_from(&entry_waiting_span); entry_waiting_span.follows_from(&span); @@ -308,8 +311,7 @@ async fn batching_task( info_span!(parent: None, "batch", batch_size = next_batch_size); entries.iter_mut().for_each(|(_, entry)| { // Create a new span to link the batch back to this entry - let entry_batch_span = - info_span!(parent: &entry.span, "infer", batch_size = next_batch_size); + let entry_batch_span = info_span!(parent: &entry.span, "infer"); // Add relationships next_batch_span.follows_from(&entry_batch_span); entry_batch_span.follows_from(&next_batch_span); @@ -339,7 +341,23 @@ async fn prefill( match client.prefill(batch).await { Ok((generations, next_batch)) => { - send_generations(generations, entries); + filter_send_generations(generations, entries); + + // Filter next batch and remove requests that were stopped + let next_batch = match next_batch { + None => None, + Some(batch) => { + let id = batch.id; + let next_batch = filter_batch(batch, entries); + // Next batch is now empty + // Clear it from the Python shards cache + if next_batch.is_none() { + let _ = client.clear_cache(Some(id)).await; + } + next_batch + } + }; + metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "prefill"); metrics::increment_counter!("tgi_batch_inference_success", "method" => "prefill"); next_batch @@ -361,17 +379,37 @@ async fn decode( entries: &mut IntMap, ) -> Option { let start_time = Instant::now(); + let batch_ids: Vec = batches.iter().map(|b| b.id).collect(); metrics::increment_counter!("tgi_batch_inference_count", "method" => "decode"); match client.decode(batches).await { Ok((generations, next_batch)) => { - send_generations(generations, entries); + filter_send_generations(generations, entries); + + // Filter next batch and remove requests that were stopped + let next_batch = match next_batch { + None => None, + Some(batch) => { + let id = batch.id; + let next_batch = filter_batch(batch, entries); + // Next batch is now empty + // Clear it from the Python shards cache + if next_batch.is_none() { + let _ = client.clear_cache(Some(id)).await; + } + next_batch + } + }; + metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "decode"); metrics::increment_counter!("tgi_batch_inference_success", "method" => "decode"); next_batch } // If we have an error, we discard the whole batch Err(err) => { + for id in batch_ids { + let _ = client.clear_cache(Some(id)).await; + } send_errors(err, entries); metrics::increment_counter!("tgi_batch_inference_failure", "method" => "decode"); None @@ -379,6 +417,86 @@ async fn decode( } } +/// Filter a `batch` and remove all requests not present in `entries` +#[instrument(skip_all)] +fn filter_batch(mut batch: Batch, entries: &IntMap) -> Option { + batch.requests.retain(|r| entries.contains_key(&r.id)); + let size = batch.requests.len(); + if size == 0 { + return None; + } + batch.size = size as u32; + Some(batch) +} + +/// Send one or multiple `InferStreamResponse` to Infer for all `entries` +/// and filter entries +#[instrument(skip_all)] +fn filter_send_generations(generations: Vec, entries: &mut IntMap) { + generations.into_iter().for_each(|generation| { + let id = generation.request_id; + // Get entry + // We can `expect` here as the request id should always be in the entries + let entry = entries + .get(&id) + .expect("ID not found in entries. This is a bug."); + + // Create and enter a span to link this function back to the entry + let _span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_generation", generation = ?generation).entered(); + // Send generation responses back to the infer task + // If the receive an error from the Flume channel, it means that the client dropped the + // request and we need to stop generating hence why we unwrap_or(true) + let stopped = send_responses(generation, entry).map_err(|err| { + metrics::increment_counter!("tgi_request_failure", "err" => "dropped"); + err + }).unwrap_or(true); + if stopped { + entries.remove(&id).expect("ID not found in entries. This is a bug."); + } + }); +} + +/// Send responses through the `entry` response channel +fn send_responses( + generation: Generation, + entry: &Entry, +) -> Result>> { + let mut stopped = false; + + if let Some(prefill_tokens) = generation.prefill_tokens { + // Send message + entry + .response_tx + .send(Ok(InferStreamResponse::Prefill(prefill_tokens)))?; + } + + // Create last Token + let token = Token { + id: generation.token_id, + text: generation.token_text, + logprob: generation.token_logprob, + special: generation.token_is_special, + }; + + if let Some(generated_text) = generation.generated_text { + // Generation has ended + stopped = true; + // Send message + entry.response_tx.send(Ok(InferStreamResponse::End { + token, + generated_text, + queued: entry.queue_time, + start: entry.batch_time.unwrap(), + }))?; + } else { + // Send message + entry + .response_tx + .send(Ok(InferStreamResponse::Token(token)))?; + } + Ok(stopped) +} + /// Send errors to Infer for all `entries` #[instrument(skip_all)] fn send_errors(error: ClientError, entries: &mut IntMap) { @@ -397,65 +515,6 @@ fn send_errors(error: ClientError, entries: &mut IntMap) { }); } -/// Send one or multiple `InferStreamResponse` to Infer for all `entries` -#[instrument(skip_all)] -fn send_generations(generations: Vec, entries: &mut IntMap) { - generations.into_iter().for_each(|generation| { - // Get entry - // We can `expect` here as the request id should always be in the entries - let entry = entries - .get(&generation.request_id) - .expect("ID not found in entries. This is a bug."); - - // Create and enter a span to link this function back to the entry - let _generation_span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_generation", generation = ?generation).entered(); - - if let Some(prefill_tokens) = generation.prefill_tokens { - // Send message - // unwrap_or is valid here as we don't care if the receiver is gone. - entry - .response_tx - .send(Ok(InferStreamResponse::Prefill(prefill_tokens))) - .unwrap_or(()); - } - - // Create last Token - let token = Token { - id: generation.token_id, - text: generation.token_text, - logprob: generation.token_logprob, - special: generation.token_is_special, - }; - - if let Some(generated_text) = generation.generated_text { - // Remove entry as this is the last message - // We can `expect` here as the request id should always be in the entries - let entry = entries - .remove(&generation.request_id) - .expect("ID not found in entries. This is a bug."); - - // Send message - // unwrap_or is valid here as we don't care if the receiver is gone. - entry - .response_tx - .send(Ok(InferStreamResponse::End { - token, - generated_text, - queued: entry.queue_time, - start: entry.batch_time.unwrap(), - })) - .unwrap_or(()); - } else { - // Send message - // unwrap_or is valid here as we don't care if the receiver is gone. - entry - .response_tx - .send(Ok(InferStreamResponse::Token(token))) - .unwrap_or(()); - } - }); -} - #[derive(Debug)] pub(crate) enum InferStreamResponse { // Optional first message diff --git a/router/src/queue.rs b/router/src/queue.rs index 11eb7f59..43651ff3 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -3,8 +3,9 @@ use crate::infer::InferStreamResponse; use crate::validation::ValidGenerateRequest; use nohash_hasher::{BuildNoHashHasher, IntMap}; use std::cmp::min; +use std::collections::VecDeque; use text_generation_client::{Batch, Request}; -use tokio::sync::{oneshot, OwnedSemaphorePermit}; +use tokio::sync::oneshot; use tokio::time::Instant; use tracing::{info_span, instrument, Span}; @@ -23,8 +24,6 @@ pub(crate) struct Entry { pub queue_time: Instant, /// Instant when this entry was added to a batch pub batch_time: Option, - /// Permit - pub _permit: OwnedSemaphorePermit, } /// Request Queue @@ -104,7 +103,7 @@ async fn queue_task(receiver: flume::Receiver) { #[derive(Debug)] struct State { /// Queue entries organized in a Vec - entries: Vec<(u64, Entry)>, + entries: VecDeque<(u64, Entry)>, /// Id of the next entry next_id: u64, @@ -116,7 +115,7 @@ struct State { impl State { fn new() -> Self { Self { - entries: Vec::with_capacity(128), + entries: VecDeque::with_capacity(128), next_id: 0, next_batch_id: 0, } @@ -129,7 +128,7 @@ impl State { entry.temp_span = Some(queue_span); // Push entry in the queue - self.entries.push((self.next_id, entry)); + self.entries.push_back((self.next_id, entry)); self.next_id += 1; metrics::increment_gauge!("tgi_queue_size", 1.0); } @@ -147,51 +146,70 @@ impl State { } } - let next_batch_size = min(self.entries.len(), max_size); + let max_batch_size = min(self.entries.len(), max_size); // Create span for this batch to add context to inference calls - let next_batch_span = info_span!(parent: None, "batch", batch_size = next_batch_size); + let next_batch_span = info_span!(parent: None, "batch", batch_size = tracing::field::Empty); next_batch_span.follows_from(&Span::current()); - let mut batch_requests = Vec::with_capacity(next_batch_size); + let mut batch_requests = Vec::with_capacity(max_batch_size); let mut batch_entries = - IntMap::with_capacity_and_hasher(next_batch_size, BuildNoHashHasher::default()); - - // Drain next_batch_size entries - self.entries - .drain(..next_batch_size) - .for_each(|(id, mut entry)| { - // Create a new span to link the batch back to this entry - let entry_batch_span = - info_span!(parent: &entry.span, "infer", batch_size = next_batch_size); - // Add relationships - next_batch_span.follows_from(&entry_batch_span); - entry_batch_span.follows_from(&next_batch_span); - // Update entry - entry.temp_span = Some(entry_batch_span); - - batch_requests.push(Request { - id, - inputs: entry.request.inputs.clone(), - truncate: entry.request.truncate, - parameters: Some(entry.request.parameters.clone()), - stopping_parameters: Some(entry.request.stopping_parameters.clone()), - }); - // Set batch_time - entry.batch_time = Some(Instant::now()); - // Insert in batch_entries IntMap - batch_entries.insert(id, entry); + IntMap::with_capacity_and_hasher(max_batch_size, BuildNoHashHasher::default()); + + // Iterate on buffer + while let Some((id, mut entry)) = self.entries.pop_front() { + // Filter entries where the response receiver was dropped (== entries where the request + // was dropped by the client) + if entry.response_tx.is_disconnected() { + metrics::increment_counter!("tgi_request_failure", "err" => "dropped"); + continue; + } + + // Create a new span to link the batch back to this entry + let entry_batch_span = info_span!(parent: &entry.span, "infer"); + // Add relationships + next_batch_span.follows_from(&entry_batch_span); + entry_batch_span.follows_from(&next_batch_span); + // Update entry + entry.temp_span = Some(entry_batch_span); + + batch_requests.push(Request { + id, + inputs: entry.request.inputs.clone(), + truncate: entry.request.truncate, + parameters: Some(entry.request.parameters.clone()), + stopping_parameters: Some(entry.request.stopping_parameters.clone()), }); + // Set batch_time + entry.batch_time = Some(Instant::now()); + // Insert in batch_entries IntMap + batch_entries.insert(id, entry); + + if batch_requests.len() == max_batch_size { + // We have enough requests in the batch + break; + } + } + + metrics::gauge!("tgi_queue_size", self.entries.len() as f64); + + // Maybe all entries were dropped because their channel were closed + if batch_requests.is_empty() { + return None; + } + + // Final batch size once we dropped entries + let size = batch_requests.len() as u32; + next_batch_span.record("batch_size", size); let batch = Batch { id: self.next_batch_id, requests: batch_requests, - size: next_batch_size as u32, + size, }; // Increment batch id self.next_batch_id += 1; - metrics::gauge!("tgi_queue_size", self.entries.len() as f64); metrics::histogram!("tgi_batch_next_size", batch.size as f64); Some((batch_entries, batch, next_batch_span)) } @@ -213,17 +231,16 @@ enum QueueCommand { #[cfg(test)] mod tests { use super::*; - use std::sync::Arc; use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParameters}; - use tokio::sync::Semaphore; use tracing::info_span; - fn default_entry() -> Entry { - let semaphore = Arc::new(Semaphore::new(1)); - let (response_tx, _) = flume::unbounded(); - let permit = semaphore.try_acquire_owned().unwrap(); + fn default_entry() -> ( + Entry, + flume::Receiver>, + ) { + let (response_tx, receiver_tx) = flume::unbounded(); - Entry { + let entry = Entry { request: ValidGenerateRequest { inputs: "".to_string(), truncate: 0, @@ -248,14 +265,14 @@ mod tests { temp_span: None, queue_time: Instant::now(), batch_time: None, - _permit: permit, - } + }; + (entry, receiver_tx) } #[test] fn test_append() { let mut state = State::new(); - let entry = default_entry(); + let (entry, _guard) = default_entry(); assert_eq!(state.next_id, 0); assert_eq!(state.entries.len(), 0); @@ -264,7 +281,7 @@ mod tests { assert_eq!(state.next_id, 1); assert_eq!(state.entries.len(), 1); - let (id, _) = state.entries.remove(0); + let (id, _) = state.entries.remove(0).unwrap(); assert_eq!(id, 0); } @@ -279,8 +296,10 @@ mod tests { #[test] fn test_next_batch_min_size() { let mut state = State::new(); - state.append(default_entry()); - state.append(default_entry()); + let (entry1, _guard1) = default_entry(); + let (entry2, _guard2) = default_entry(); + state.append(entry1); + state.append(entry2); let (entries, batch, _) = state.next_batch(None, 2).unwrap(); assert_eq!(entries.len(), 2); @@ -295,21 +314,24 @@ mod tests { assert_eq!(state.entries.len(), 0); assert_eq!(state.next_batch_id, 1); - state.append(default_entry()); + let (entry3, _guard3) = default_entry(); + state.append(entry3); assert!(state.next_batch(Some(2), 2).is_none()); assert_eq!(state.next_id, 3); assert_eq!(state.entries.len(), 1); - let (id, _) = state.entries.remove(0); + let (id, _) = state.entries.remove(0).unwrap(); assert_eq!(id, 2); } #[test] fn test_next_batch_max_size() { let mut state = State::new(); - state.append(default_entry()); - state.append(default_entry()); + let (entry1, _guard1) = default_entry(); + let (entry2, _guard2) = default_entry(); + state.append(entry1); + state.append(entry2); let (entries, batch, _) = state.next_batch(None, 1).unwrap(); assert_eq!(entries.len(), 1); @@ -321,7 +343,8 @@ mod tests { assert_eq!(state.entries.len(), 1); assert_eq!(state.next_batch_id, 1); - state.append(default_entry()); + let (entry3, _guard3) = default_entry(); + state.append(entry3); let (entries, batch, _) = state.next_batch(None, 3).unwrap(); assert_eq!(entries.len(), 2); @@ -338,7 +361,8 @@ mod tests { #[tokio::test] async fn test_queue_append() { let queue = Queue::new(); - queue.append(default_entry()); + let (entry, _guard) = default_entry(); + queue.append(entry); } #[tokio::test] @@ -352,8 +376,10 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_min_size() { let queue = Queue::new(); - queue.append(default_entry()); - queue.append(default_entry()); + let (entry1, _guard1) = default_entry(); + let (entry2, _guard2) = default_entry(); + queue.append(entry1); + queue.append(entry2); let (entries, batch, _) = queue.next_batch(None, 2).await.unwrap(); assert_eq!(entries.len(), 2); @@ -364,7 +390,8 @@ mod tests { assert_eq!(batch.id, 0); assert_eq!(batch.size, 2); - queue.append(default_entry()); + let (entry3, _guard3) = default_entry(); + queue.append(entry3); assert!(queue.next_batch(Some(2), 2).await.is_none()); } @@ -372,8 +399,10 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_max_size() { let queue = Queue::new(); - queue.append(default_entry()); - queue.append(default_entry()); + let (entry1, _guard1) = default_entry(); + let (entry2, _guard2) = default_entry(); + queue.append(entry1); + queue.append(entry2); let (entries, batch, _) = queue.next_batch(None, 1).await.unwrap(); assert_eq!(entries.len(), 1); @@ -381,7 +410,8 @@ mod tests { assert_eq!(batch.id, 0); assert_eq!(batch.size, 1); - queue.append(default_entry()); + let (entry3, _guard3) = default_entry(); + queue.append(entry3); let (entries, batch, _) = queue.next_batch(None, 3).await.unwrap(); assert_eq!(entries.len(), 2); @@ -390,4 +420,13 @@ mod tests { assert_eq!(batch.id, 1); assert_eq!(batch.size, 2); } + + #[tokio::test] + async fn test_queue_next_batch_dropped_receiver() { + let queue = Queue::new(); + let (entry, _) = default_entry(); + queue.append(entry); + + assert!(queue.next_batch(None, 1).await.is_none()); + } } diff --git a/router/src/server.rs b/router/src/server.rs index ce301399..fee748e6 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -367,7 +367,8 @@ async fn generate_stream( let best_of = req.0.parameters.best_of.unwrap_or(1); if best_of == 1 { match infer.generate_stream(req.0).instrument(info_span!(parent: &span, "async_stream")).await { - Ok(mut response_stream) => { + // Keep permit as long as generate_stream lives + Ok((_permit, mut response_stream)) => { // Server-Sent Event stream while let Some(response) = response_stream.next().await { match response { diff --git a/server/Makefile b/server/Makefile index 058d912b..150d7e4a 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,6 +1,9 @@ include Makefile-transformers include Makefile-flash-att +unit-tests: + python -m pytest tests + gen-server: # Compile protos pip install grpcio-tools==1.51.1 mypy-protobuf==3.4.0 'types-protobuf>=3.20.4' --no-cache-dir diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index a9831cd7..de0ef57b 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -45,8 +45,9 @@ def default_bloom_batch(default_pb_batch, bloom_560m_tokenizer): @pytest.fixture def default_multi_requests_bloom_batch(default_pb_request, bloom_560m_tokenizer): req_0 = copy(default_pb_request) + req_0.id = 1 req_1 = default_pb_request - req_1.id = 1 + req_1.id = 2 req_1.stopping_parameters.max_new_tokens = 5 batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) @@ -70,12 +71,17 @@ def test_batch_from_pb(default_pb_batch, default_bloom_batch): assert batch.past_key_values is None - assert torch.equal(batch.input_ids, batch.all_input_ids[:, :, 0]) + assert all( + [ + torch.equal(input_ids, all_input_ids[:, 0]) + for input_ids, all_input_ids in zip(batch.input_ids, batch.all_input_ids) + ] + ) assert batch.input_lengths == [1] - assert batch.size == default_pb_batch.size - assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == batch.size + assert len(batch) == default_pb_batch.size + assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == len(batch) assert batch.max_input_length == batch.input_lengths[0] @@ -97,7 +103,7 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): assert isinstance(next_batch, CausalLMBatch) assert not next_batch.keys_head_dim_last - assert len(next_batch.all_input_ids) == next_batch.size + assert len(next_batch.all_input_ids) == len(next_batch) assert len(next_batch.all_input_ids[0]) == sequence_length + 1 assert len(next_batch.attention_mask[0]) == 11 assert torch.all(next_batch.all_input_ids[0][-2:] == 10264) @@ -106,7 +112,7 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): assert torch.all(next_batch.attention_mask[0][:2] == 1) assert torch.all(next_batch.attention_mask[0][2:] == 0) - assert next_batch.input_ids.shape == (next_batch.size, 1) + assert next_batch.input_ids.shape == (len(next_batch), 1) assert next_batch.input_ids[0, 0] == 10264 assert next_batch.input_lengths == [2] @@ -170,6 +176,8 @@ def test_causal_lm_generate_token_completion_multi( == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) + next_batch = next_batch.filter([next_batch.requests[0]]) + for _ in range( default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens @@ -269,6 +277,8 @@ def test_batch_concatenate( == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) + next_batch = next_batch.filter([next_batch.requests[0], next_batch.requests[1]]) + for _ in range( default_bloom_batch.stopping_criterias[0].max_new_tokens - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens @@ -290,6 +300,8 @@ def test_batch_concatenate( == default_bloom_batch.stopping_criterias[0].max_new_tokens ) + next_batch = next_batch.filter([next_batch.requests[1]]) + for _ in range( default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens - default_bloom_batch.stopping_criterias[0].max_new_tokens diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index db68fc9c..ad79a4ca 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -44,11 +44,12 @@ def default_causal_lm_batch(default_pb_batch, gpt2_tokenizer): @pytest.fixture def default_multi_requests_causal_lm_batch(default_pb_request, gpt2_tokenizer): req_0 = copy(default_pb_request) + req_0.id = 1 req_1 = default_pb_request - req_1.id = 1 + req_1.id = 2 req_1.stopping_parameters.max_new_tokens = 5 - batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) + batch_pb = generate_pb2.Batch(id=1, requests=[req_0, req_1], size=2) return CausalLMBatch.from_pb(batch_pb, gpt2_tokenizer, torch.device("cpu")) @@ -67,12 +68,17 @@ def test_batch_from_pb(default_pb_batch, default_causal_lm_batch): assert batch.past_key_values is None - assert torch.equal(batch.input_ids, batch.all_input_ids[:, :, 0]) + assert all( + [ + torch.equal(input_ids, all_input_ids[:, 0]) + for input_ids, all_input_ids in zip(batch.input_ids, batch.all_input_ids) + ] + ) assert batch.input_lengths == [1] - assert batch.size == default_pb_batch.size - assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == batch.size + assert len(batch) == default_pb_batch.size + assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == len(batch) assert batch.max_input_length == batch.input_lengths[0] @@ -93,7 +99,7 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): assert len(generations) == len(next_batch) assert isinstance(next_batch, CausalLMBatch) - assert len(next_batch.all_input_ids) == next_batch.size + assert len(next_batch.all_input_ids) == len(next_batch) assert len(next_batch.all_input_ids[0]) == sequence_length + 1 assert len(next_batch.attention_mask[0]) == 11 assert next_batch.all_input_ids[0][-1] == 13 @@ -103,7 +109,7 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): assert torch.all(next_batch.attention_mask[0][0:2] == 1) assert torch.all(next_batch.attention_mask[0][2:] == 0) - assert next_batch.input_ids.shape == (next_batch.size, 1) + assert next_batch.input_ids.shape == (len(next_batch), 1) assert next_batch.input_ids[0, 0] == 13 assert next_batch.input_lengths == [2] @@ -168,6 +174,8 @@ def test_causal_lm_generate_token_completion_multi( == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) + next_batch = next_batch.filter([next_batch.requests[0]]) + for _ in range( default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens @@ -266,6 +274,8 @@ def test_batch_concatenate( == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) + next_batch = next_batch.filter([next_batch.requests[0], next_batch.requests[1]]) + for _ in range( default_causal_lm_batch.stopping_criterias[0].max_new_tokens - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens @@ -285,6 +295,8 @@ def test_batch_concatenate( == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) + next_batch = next_batch.filter([next_batch.requests[1]]) + for _ in range( default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens - default_causal_lm_batch.stopping_criterias[0].max_new_tokens diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 79435787..79c9e936 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -49,8 +49,9 @@ def default_seq2seq_lm_batch(default_pb_batch, mt0_small_tokenizer): @pytest.fixture def default_multi_requests_seq2seq_lm_batch(default_pb_request, mt0_small_tokenizer): req_0 = copy(default_pb_request) + req_0.id = 1 req_1 = default_pb_request - req_1.id = 1 + req_1.id = 2 req_1.stopping_parameters.max_new_tokens = 5 batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) @@ -72,7 +73,7 @@ def test_batch_from_pb(default_pb_batch, default_seq2seq_lm_batch): assert torch.all(batch.attention_mask[0][-2:] == 1) assert torch.all(batch.attention_mask[0][:-2] == 0) - assert batch.decoder_input_ids.shape == (default_pb_batch.size, 1) + assert len(batch.decoder_input_ids) == default_pb_batch.size assert batch.decoder_attention_mask is None assert batch.encoder_last_hidden_state is None @@ -81,8 +82,8 @@ def test_batch_from_pb(default_pb_batch, default_seq2seq_lm_batch): assert batch.input_lengths == [2] assert batch.decoder_input_lengths == [1] - assert batch.size == default_pb_batch.size - assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == batch.size + assert len(batch) == default_pb_batch.size + assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == len(batch) assert batch.max_input_length == batch.input_lengths[0] assert batch.max_decoder_input_length == batch.decoder_input_lengths[0] @@ -117,9 +118,9 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) ) assert next_batch.stopping_criterias == default_seq2seq_lm_batch.stopping_criterias - assert next_batch.decoder_input_ids.shape == (next_batch.size, 2) - assert next_batch.decoder_input_ids[0, 0] == 0 - assert next_batch.decoder_input_ids[0, 1] == 259 + assert len(next_batch.decoder_input_ids) == len(next_batch) + assert next_batch.all_decoder_input_ids[0][0] == 0 + assert next_batch.all_decoder_input_ids[0][1] == 259 assert next_batch.decoder_attention_mask is None assert next_batch.encoder_last_hidden_state.shape == (1, sequence_length, 512) @@ -128,20 +129,20 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) assert next_batch.past_key_values is not None assert all( - [p[0].shape == (next_batch.size, 6, 1, 64) for p in next_batch.past_key_values] + [p[0].shape == (len(next_batch), 6, 1, 64) for p in next_batch.past_key_values] ) assert all( - [p[1].shape == (next_batch.size, 6, 1, 64) for p in next_batch.past_key_values] + [p[1].shape == (len(next_batch), 6, 1, 64) for p in next_batch.past_key_values] ) assert all( [ - p[2].shape == (next_batch.size, 6, sequence_length, 64) + p[2].shape == (len(next_batch), 6, sequence_length, 64) for p in next_batch.past_key_values ] ) assert all( [ - p[3].shape == (next_batch.size, 6, sequence_length, 64) + p[3].shape == (len(next_batch), 6, sequence_length, 64) for p in next_batch.past_key_values ] ) @@ -189,6 +190,8 @@ def test_seq2seq_lm_generate_token_completion_multi( ) assert generations[1].generated_text.generated_tokens == 5 + next_batch = next_batch.filter([next_batch.requests[0]]) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert len(generations) == len(next_batch) @@ -223,7 +226,8 @@ def test_batch_concatenate( assert torch.equal( next_batch.decoder_input_ids[0], next_batch_0.decoder_input_ids[0] ) - assert torch.all(next_batch.decoder_input_ids[1:, 0] == 0) + assert next_batch.all_decoder_input_ids[1][0] == 0 + assert next_batch.all_decoder_input_ids[2][0] == 0 assert torch.equal( next_batch.decoder_input_ids[1:, -2:], next_batch_1.decoder_input_ids ) @@ -258,16 +262,16 @@ def test_batch_concatenate( assert next_batch.past_key_values is not None assert all( - [p[0].shape == (next_batch.size, 6, 2, 64) for p in next_batch.past_key_values] + [p[0].shape == (len(next_batch), 6, 2, 64) for p in next_batch.past_key_values] ) assert all( - [p[1].shape == (next_batch.size, 6, 2, 64) for p in next_batch.past_key_values] + [p[1].shape == (len(next_batch), 6, 2, 64) for p in next_batch.past_key_values] ) assert all( - [p[2].shape == (next_batch.size, 6, 2, 64) for p in next_batch.past_key_values] + [p[2].shape == (len(next_batch), 6, 2, 64) for p in next_batch.past_key_values] ) assert all( - [p[3].shape == (next_batch.size, 6, 2, 64) for p in next_batch.past_key_values] + [p[3].shape == (len(next_batch), 6, 2, 64) for p in next_batch.past_key_values] ) for i, past in enumerate(next_batch.past_key_values): @@ -306,6 +310,8 @@ def test_batch_concatenate( ) assert generations[2].generated_text.generated_tokens == 5 + next_batch = next_batch.filter([next_batch.requests[0], next_batch.requests[1]]) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None @@ -314,6 +320,8 @@ def test_batch_concatenate( assert generations[0].request_id == default_seq2seq_lm_batch.requests[0].id assert generations[0].generated_text.generated_tokens == 7 + next_batch = next_batch.filter([next_batch.requests[1]]) + generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 6347b1a5..71eff437 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from opentelemetry import trace from transformers import AutoTokenizer, AutoModelForCausalLM, PreTrainedTokenizerBase -from typing import Optional, Tuple, List, Type +from typing import Optional, Tuple, List, Type, Dict from text_generation_server.models import Model from text_generation_server.models.types import ( @@ -22,6 +22,7 @@ class CausalLMBatch(Batch): batch_id: int requests: List[generate_pb2.Request] + requests_idx_mapping: Dict[int, int] # Decoder values input_ids: torch.Tensor @@ -42,7 +43,6 @@ class CausalLMBatch(Batch): stopping_criterias: List[StoppingCriteria] # Metadata used for padding - size: int max_input_length: int padding_right_offset: int @@ -53,7 +53,7 @@ def to_pb(self) -> generate_pb2.Batch: return generate_pb2.Batch( id=self.batch_id, requests=self.requests, - size=self.size, + size=len(self), ) @classmethod @@ -68,11 +68,13 @@ def from_pb( stopping_criterias = [] offsets = [] token_offsets = [] + requests_idx_mapping = {} # Parse batch max_truncation = 0 padding_right_offset = 0 - for r in pb.requests: + for i, r in enumerate(pb.requests): + requests_idx_mapping[r.id] = i inputs.append(r.inputs) offsets.append(None) token_offsets.append(None) @@ -108,26 +110,91 @@ def from_pb( position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) - all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) + all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) return cls( batch_id=pb.id, requests=pb.requests, + requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, past_key_values=None, - all_input_ids=all_input_ids, + all_input_ids=list(all_input_ids), input_lengths=input_lengths.tolist(), offsets=offsets, token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, - size=pb.size, max_input_length=max_input_length.item(), padding_right_offset=padding_right_offset, ) + @tracer.start_as_current_span("filter") + def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatch"]: + if len(requests) == 0: + raise ValueError("Batch must have at least one request") + if len(requests) == len(self): + return self + + keep_indices = [] + + # New values after filtering + requests_idx_mapping = {} + input_lengths = [] + offsets = [] + token_offsets = [] + all_input_ids = [] + max_input_length = 0 + + next_token_choosers = [] + stopping_criterias = [] + + for i, r in enumerate(requests): + idx = self.requests_idx_mapping[r.id] + requests_idx_mapping[r.id] = i + keep_indices.append(idx) + + offsets.append(self.offsets[idx]) + token_offsets.append(self.token_offsets[idx]) + all_input_ids.append(self.all_input_ids[idx]) + + request_input_length = self.input_lengths[idx] + input_lengths.append(request_input_length) + max_input_length = max(max_input_length, request_input_length) + + next_token_choosers.append(self.next_token_choosers[idx]) + stopping_criterias.append(self.stopping_criterias[idx]) + + # Apply indices to input_ids, attention mask, past key values and other items that need to be cached + input_ids = self.input_ids[keep_indices] + attention_mask = self.attention_mask[keep_indices] + position_ids = self.position_ids[keep_indices] + # Force past to be of dim [self_size, num_heads, ...] for easy indexing + past_key_values = [ + [t.view(len(self), -1, *t.shape[-2:])[keep_indices] for t in layer] + for layer in self.past_key_values + ] + + return CausalLMBatch( + batch_id=self.batch_id, + requests=requests, + requests_idx_mapping=requests_idx_mapping, + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + all_input_ids=all_input_ids, + input_lengths=input_lengths, + offsets=offsets, + token_offsets=token_offsets, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + max_input_length=max_input_length, + padding_right_offset=self.padding_right_offset, + keys_head_dim_last=self.keys_head_dim_last, + ) + @classmethod @tracer.start_as_current_span("concatenate") def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": @@ -136,12 +203,13 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": max_input_length = 0 padding_right_offset = 0 for batch in batches: - total_batch_size += batch.size + total_batch_size += len(batch) max_input_length = max(max_input_length, batch.max_input_length) padding_right_offset = max(padding_right_offset, batch.padding_right_offset) # Batch attributes requests = [] + requests_idx_mapping = {} input_lengths = [] offsets = [] token_offsets = [] @@ -167,8 +235,15 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) + if i == 0: + requests_idx_mapping = batch.requests_idx_mapping + else: + # We need to offset the mapping for each batch by the cumulative batch size + for k, v in batch.requests_idx_mapping.items(): + requests_idx_mapping[k] = v + start_index + # Slicing end index for this batch - end_index = start_index + batch.size + end_index = start_index + len(batch) # We only concatenate batches that did at least one step if batch.past_key_values is None: @@ -216,8 +291,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # Shenanigans to get dimensions because BLOOM outputs a past with a different shape # BLOOM Keys: [batch_size * num_heads, head_dim, seq_length] # BLOOM Values: [batch_size * num_heads, seq_length, head_dim] - past_keys = past_keys.view(batch.size, -1, *past_keys.shape[-2:]) - past_values = past_values.view(batch.size, -1, *past_values.shape[-2:]) + past_keys = past_keys.view(len(batch), -1, *past_keys.shape[-2:]) + past_values = past_values.view(len(batch), -1, *past_values.shape[-2:]) _, num_heads, padded_sequence_length, head_dim = past_values.shape @@ -265,11 +340,12 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": start_index:end_index, :, -(batch.max_input_length - 1) :, : ] = past_values[:, :, -(batch.max_input_length - 1) :, :] - start_index += batch.size + start_index += len(batch) return cls( batch_id=batches[0].batch_id, requests=requests, + requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, @@ -280,7 +356,6 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, - size=total_batch_size, max_input_length=max_input_length, padding_right_offset=padding_right_offset, keys_head_dim_last=batches[0].keys_head_dim_last, @@ -364,22 +439,9 @@ def generate_token( batch.past_key_values, ) - # List of indices to cache - next_batch_keep_indices = [] - - # New values for next forward - next_batch_input_lengths = [] - next_batch_offsets = [] - next_batch_token_offsets = [] - next_batch_input_ids = [] - next_batch_all_input_ids = [] - - # Metadata - next_batch_size = 0 - next_batch_max_input_length = 0 - # Results generations: List[Generation] = [] + stopped = True # Zipped iterator iterator = zip( @@ -443,16 +505,7 @@ def generate_token( else: # Keep request in the batch generated_text = None - next_batch_keep_indices.append(i) - next_batch_input_ids.append(next_token_id) - next_batch_all_input_ids.append(all_input_ids) - next_batch_size += 1 - next_batch_input_lengths.append(new_input_length) - next_batch_offsets.append(offset) - next_batch_token_offsets.append(token_offset) - next_batch_max_input_length = max( - next_batch_max_input_length, new_input_length - ) + stopped = False # Prefill if stopping_criteria.current_tokens == 1: @@ -484,62 +537,30 @@ def generate_token( generations.append(generation) + # Update values + batch.input_ids[i, 0] = next_token_id + batch.all_input_ids[i] = all_input_ids + batch.input_lengths[i] = new_input_length + batch.offsets[i] = offset + batch.token_offsets[i] = token_offset + batch.max_input_length = max(batch.max_input_length, new_input_length) + # We finished all generations in the batch; there is no next batch - if not next_batch_keep_indices: + if stopped: return generations, None - next_batch_input_ids = torch.cat(next_batch_input_ids, dim=0) - # If we finished at least one generation, we need to evict the indices of the generations that finished - # from the values of the next batch - if len(next_batch_keep_indices) != len(batch): - # Apply indices to attention mask, past key values and other items that need to be cached - next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] - next_batch_position_ids = batch.position_ids[next_batch_keep_indices] - # Force past to be of dim [batch_size, num_heads, ...] for easy indexing - next_batch_past_key_values = [ - [ - t.view(batch.size, -1, *t.shape[-2:])[next_batch_keep_indices] - for t in layer - ] - for layer in past - ] - next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] - next_batch_next_token_choosers = [ - batch.next_token_choosers[i] for i in next_batch_keep_indices - ] - next_batch_stopping_criterias = [ - batch.stopping_criterias[i] for i in next_batch_keep_indices - ] - else: - next_batch_attention_mask = batch.attention_mask - next_batch_position_ids = batch.position_ids - next_batch_past_key_values = past - next_batch_requests = batch.requests - next_batch_next_token_choosers = batch.next_token_choosers - next_batch_stopping_criterias = batch.stopping_criterias + # Slice unused values from prefill + batch.input_ids = batch.input_ids[:, :1] # Update attention_mask as we added a new token to input_ids - next_batch_attention_mask[:, -batch.padding_right_offset] = 1 + batch.attention_mask[:, -batch.padding_right_offset] = 1 + # Decrease right offset + batch.padding_right_offset -= 1 # Update position_ids - next_batch_position_ids = next_batch_position_ids[:, -1:] + 1 - - next_batch = CausalLMBatch( - batch_id=batch.batch_id, - requests=next_batch_requests, - input_ids=next_batch_input_ids, - attention_mask=next_batch_attention_mask, - position_ids=next_batch_position_ids, - past_key_values=next_batch_past_key_values, - all_input_ids=next_batch_all_input_ids, - input_lengths=next_batch_input_lengths, - offsets=next_batch_offsets, - token_offsets=next_batch_token_offsets, - next_token_choosers=next_batch_next_token_choosers, - stopping_criterias=next_batch_stopping_criterias, - size=next_batch_size, - max_input_length=next_batch_max_input_length, - padding_right_offset=batch.padding_right_offset - 1, - keys_head_dim_last=batch.keys_head_dim_last, - ) - return generations, next_batch + batch.position_ids = batch.position_ids[:, -1:] + 1 + + # Update past key values + batch.past_key_values = past + + return generations, batch diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 3d4273cc..382f917d 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from opentelemetry import trace from transformers import AutoTokenizer, PreTrainedTokenizerBase, PreTrainedModel -from typing import Optional, Tuple, List, Type, Union +from typing import Optional, Tuple, List, Type, Union, Dict from text_generation_server.models import Model from text_generation_server.models.types import ( @@ -29,14 +29,16 @@ class FlashCausalLMBatch(Batch): batch_id: int requests: List[generate_pb2.Request] + # request id -> idx in list mapping + requests_idx_mapping: Dict[int, int] # Decoder values - input_ids: torch.Tensor - position_ids: torch.Tensor + input_ids: List[torch.Tensor] + position_ids: List[torch.Tensor] # cumulative sequence lengths - cu_seqlens: torch.Tensor + cu_seqlens: List[int] max_seqlen: int - past_key_values: Optional[torch.Tensor] + past_key_values: Optional[List[torch.Tensor]] # All tokens all_input_ids: List[List[int]] @@ -62,7 +64,7 @@ def from_pb( pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, device: torch.device, - ) -> "CausalLMBatch": + ) -> "FlashCausalLMBatch": input_ids = [] position_ids = [] cu_seqlens = [0] @@ -73,6 +75,7 @@ def from_pb( token_offsets = [] all_input_ids = [] all_input_ids_tensor = [] + requests_idx_mapping = {} next_token_choosers = [] stopping_criterias = [] @@ -81,13 +84,18 @@ def from_pb( cumulative_length = 0 # Parse batch - for r in pb.requests: + for i, r in enumerate(pb.requests): + # request id -> idx in list mapping + requests_idx_mapping[r.id] = i + tokenized_input = tokenizer( r.inputs, truncation=True, max_length=r.truncate )["input_ids"] + input_length = len(tokenized_input) max_seqlen = max(max_seqlen, input_length) input_lengths.append(input_length) + offsets.append(None) token_offsets.append(None) all_input_ids.append(tokenized_input) @@ -96,7 +104,9 @@ def from_pb( input_ids.append(tokenized_input) # Position ids - position_ids.append(torch.arange(0, input_length, dtype=torch.int32)) + position_ids.append( + torch.arange(0, input_length, dtype=torch.int32, device=device) + ) # Add cumulative lengths of all previous inputs cu_seqlens.append(cumulative_length + input_length) @@ -113,13 +123,10 @@ def from_pb( # Update cumulative_length += input_length - input_ids = torch.concat(input_ids) - position_ids = torch.concat(position_ids) - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32) - return cls( batch_id=pb.id, requests=pb.requests, + requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, cu_seqlens=cu_seqlens, @@ -134,60 +141,141 @@ def from_pb( stopping_criterias=stopping_criterias, ) - @classmethod - @tracer.start_as_current_span("concatenate") - def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch": - # Batch attributes - requests = [] + @tracer.start_as_current_span("filter") + def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": + if len(requests) == 0: + raise ValueError("Batch must have at least one request") + # We assume that if len(requests) == len(self) then the requests are the same + if len(requests) == len(self): + return self + + # Cumulative length + cumulative_length = 0 + + # New values after filtering + requests_idx_mapping = {} + + input_ids = [] + position_ids = [] + cu_seqlens = [0] + max_seqlen = 0 + past_key_values = [] + + all_input_ids = [] + all_input_ids_tensor = [] + input_lengths = [] offsets = [] token_offsets = [] - all_input_ids = [] - all_input_ids_tensor = [] + next_token_choosers = [] stopping_criterias = [] - # Batch tensors + for i, r in enumerate(requests): + idx = self.requests_idx_mapping[r.id] + requests_idx_mapping[r.id] = i + + # Get length + request_input_length = self.input_lengths[idx] + + input_ids.append(self.input_ids[idx]) + position_ids.append(self.position_ids[idx]) + cu_seqlens.append(cumulative_length + request_input_length) + max_seqlen = max(max_seqlen, request_input_length) + past_key_values.append(self.past_key_values[idx]) + + all_input_ids.append(self.all_input_ids[idx]) + all_input_ids_tensor.append(self.all_input_ids_tensor[idx]) + + input_lengths.append(request_input_length) + offsets.append(self.offsets[idx]) + token_offsets.append(self.token_offsets[idx]) + + next_token_choosers.append(self.next_token_choosers[idx]) + stopping_criterias.append(self.stopping_criterias[idx]) + + cumulative_length += request_input_length + + return FlashCausalLMBatch( + batch_id=self.batch_id, + requests=requests, + requests_idx_mapping=requests_idx_mapping, + input_ids=input_ids, + position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + past_key_values=past_key_values, + input_lengths=input_lengths, + offsets=offsets, + token_offsets=token_offsets, + all_input_ids=all_input_ids, + all_input_ids_tensor=all_input_ids_tensor, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + ) + + @classmethod + @tracer.start_as_current_span("concatenate") + def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch": + # Batch attributes + requests = [] + requests_idx_mapping = {} + input_ids = [] position_ids = [] - cu_seqlens = [torch.tensor([0], dtype=torch.int32)] + cu_seqlens = [0] max_seqlen = 0 past_key_values = [] + all_input_ids = [] + all_input_ids_tensor = [] + + input_lengths = [] + offsets = [] + token_offsets = [] + + next_token_choosers = [] + stopping_criterias = [] + # Cumulative length - cumulative_length = torch.tensor(0) + cumulative_batch_size = 0 + cumulative_length = 0 for i, batch in enumerate(batches): requests.extend(batch.requests) + + if i == 0: + requests_idx_mapping = batch.requests_idx_mapping + else: + # We need to offset the mapping for each batch by the cumulative batch size + for k, v in batch.requests_idx_mapping.items(): + requests_idx_mapping[k] = v + cumulative_batch_size + + input_ids.extend(batch.input_ids) + position_ids.extend(batch.position_ids) + # Add cumulative lengths of all previous inputs + cu_seqlens.extend([l + cumulative_length for l in batch.cu_seqlens[1:]]) + max_seqlen = max(max_seqlen, batch.max_seqlen) + past_key_values.extend(batch.past_key_values) + + all_input_ids.extend(batch.all_input_ids) + all_input_ids_tensor.extend(batch.all_input_ids_tensor) + input_lengths.extend(batch.input_lengths) offsets.extend(batch.offsets) token_offsets.extend(batch.token_offsets) - all_input_ids.extend(batch.all_input_ids) - all_input_ids_tensor.extend(batch.all_input_ids_tensor) + next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) - # Add cumulative lengths of all previous inputs - cu_seqlens.append(batch.cu_seqlens[1:] + cumulative_length) - - input_ids.append(batch.input_ids) - position_ids.append(batch.position_ids) - past_key_values.append(batch.past_key_values) - - max_seqlen = max(max_seqlen, batch.max_seqlen) - # Update cumulative_length += batch.cu_seqlens[-1] - - input_ids = torch.concat(input_ids) - position_ids = torch.concat(position_ids) - # Concat on dim=1 as first dim represents the model layers - past_key_values = torch.concat(past_key_values, dim=1) - cu_seqlens = torch.concat(cu_seqlens) + cumulative_batch_size += len(batch) return FlashCausalLMBatch( batch_id=batches[0].batch_id, requests=requests, + requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, cu_seqlens=cu_seqlens, @@ -269,38 +357,49 @@ def forward( def generate_token( self, batch: FlashCausalLMBatch ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: - # Better to send to device here to avoid device issues in concatenate - position_ids = batch.position_ids.to(self.device, non_blocking=True) - cu_seqlens = batch.cu_seqlens.to(self.device) + # Shortcut when batch_size == 1 + if len(batch) == 1: + input_ids = batch.input_ids[0].view(-1) + past_key_values = ( + batch.past_key_values[0] if batch.past_key_values is not None else None + ) + else: + # Concatenate tensors + input_ids = torch.cat(batch.input_ids).view(-1) + past_key_values = ( + torch.cat(batch.past_key_values, dim=1) + if batch.past_key_values is not None + else None + ) + + # Concatenate when prefill, torch.tensor when decode + position_ids = ( + torch.tensor(batch.position_ids, device=self.device) + if batch.past_key_values is not None + else torch.cat(batch.position_ids) + ) + cu_seqlens = torch.tensor( + batch.cu_seqlens, device=self.device, dtype=torch.int32 + ) out, present = self.forward( - batch.input_ids, + input_ids, position_ids, cu_seqlens, batch.max_seqlen, - batch.past_key_values, + past_key_values, ) - # List of indices to cache - next_batch_keep_indices = [] - - # New values for next forward - next_batch_input_ids = [] - next_batch_position_ids = [] - next_batch_cu_seqlens = [0] - next_batch_max_seqlen = 0 - next_batch_past_key_values = [] - next_batch_input_lengths = [] - next_batch_offsets = [] - next_batch_token_offsets = [] - next_batch_all_input_ids = [] - next_batch_all_input_ids_tensor = [] + # Initialize past_key_values in prefill + if batch.past_key_values is None: + batch.past_key_values = [None] * len(batch) # Cumulative length cumulative_length = 0 # Results generations: List[Generation] = [] + stopped = True # Zipped iterator iterator = zip( @@ -329,7 +428,8 @@ def generate_token( start_index = cumulative_length end_index = cumulative_length + input_length - if batch.past_key_values is None: + prefill = stopping_criteria.current_tokens == 0 + if prefill: # Prefill mode # out is of shape [cumulative_sequence_lengths, vocab_size] logits = out[start_index:end_index] @@ -348,7 +448,6 @@ def generate_token( # Append next token to all tokens all_input_ids.append(next_token_id_item) all_input_ids_tensor[input_length] = next_token_id_item - new_input_length = input_length + 1 # Generated token next_token_logprob = logprobs[-1, next_token_id_item] @@ -378,32 +477,23 @@ def generate_token( generated_text = GeneratedText( output_text, stopping_criteria.current_tokens, reason, seed ) + + # CAUTION: generation will be stopped so no need to pad + # This will make the next forward crash if the request does not get filtered + new_input_length = input_length + past = present[:, start_index:end_index] else: - # Keep request in the batch - next_batch_keep_indices.append(i) + stopped = False generated_text = None - # Get sequence present - seq_present = present[:, start_index:end_index] - # Pad it for next iter attention - past = torch.nn.functional.pad(seq_present, (0, 0, 0, 0, 0, 0, 0, 1)) - next_batch_past_key_values.append(past) - - next_batch_input_ids.append(next_token_id) - next_batch_position_ids.append(input_length) - # Cumulative sum - next_batch_cu_seqlens.append( - next_batch_cu_seqlens[-1] + new_input_length + # Pad present for next iter attention + new_input_length = input_length + 1 + past = torch.nn.functional.pad( + present[:, start_index:end_index], (0, 0, 0, 0, 0, 0, 0, 1) ) - next_batch_input_lengths.append(new_input_length) - next_batch_offsets.append(offset) - next_batch_token_offsets.append(token_offset) - next_batch_all_input_ids.append(all_input_ids) - next_batch_all_input_ids_tensor.append(all_input_ids_tensor) - next_batch_max_seqlen = max(next_batch_max_seqlen, new_input_length) # Prefill - if stopping_criteria.current_tokens == 1: + if prefill: # Remove generated token to only have prefill and add nan for first prompt token prefill_logprobs = [float("nan")] + logprobs.gather( 1, all_input_ids_tensor[1:input_length].unsqueeze(1) @@ -433,52 +523,18 @@ def generate_token( generations.append(generation) cumulative_length += input_length - # We finished all generations in the batch; there is no next batch - if not next_batch_keep_indices: - return generations, None - - # If we finished at least one generation, we need to evict the indices of the generations that finished - # from the values of the next batch - if len(next_batch_keep_indices) != len(batch): - # Apply indices to requests, token_choosers and stopping_criterias that need to be cached - next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] - next_batch_next_token_choosers = [ - batch.next_token_choosers[i] for i in next_batch_keep_indices - ] - next_batch_stopping_criterias = [ - batch.stopping_criterias[i] for i in next_batch_keep_indices - ] - else: - next_batch_requests = batch.requests - next_batch_next_token_choosers = batch.next_token_choosers - next_batch_stopping_criterias = batch.stopping_criterias - - # Create final next batch tensors - next_batch_position_ids = torch.tensor( - next_batch_position_ids, dtype=torch.int32 - ) - next_batch_cu_seqlens = torch.tensor(next_batch_cu_seqlens, dtype=torch.int32) - if len(next_batch_keep_indices) > 1: - next_batch_input_ids = torch.concat(next_batch_input_ids).squeeze(1) - next_batch_past_key_values = torch.concat(next_batch_past_key_values, dim=1) - else: - next_batch_input_ids = next_batch_input_ids[0].view(1) - next_batch_past_key_values = next_batch_past_key_values[0] - - next_batch = FlashCausalLMBatch( - batch_id=batch.batch_id, - requests=next_batch_requests, - input_ids=next_batch_input_ids, - position_ids=next_batch_position_ids, - cu_seqlens=next_batch_cu_seqlens, - max_seqlen=next_batch_max_seqlen, - past_key_values=next_batch_past_key_values, - input_lengths=next_batch_input_lengths, - offsets=next_batch_offsets, - token_offsets=next_batch_token_offsets, - all_input_ids=next_batch_all_input_ids, - all_input_ids_tensor=next_batch_all_input_ids_tensor, - next_token_choosers=next_batch_next_token_choosers, - stopping_criterias=next_batch_stopping_criterias, - ) - return generations, next_batch + # Update values + batch.input_ids[i] = next_token_id + batch.position_ids[i] = input_length + batch.input_lengths[i] = new_input_length + batch.offsets[i] = offset + batch.token_offsets[i] = token_offset + batch.all_input_ids[i] = all_input_ids + batch.all_input_ids_tensor[i] = all_input_ids_tensor + batch.max_seqlen = max(batch.max_seqlen, new_input_length) + batch.past_key_values[i] = past + # Cumulative sum + batch.cu_seqlens[(i + 1)] = batch.cu_seqlens[i] + new_input_length + + # No need to return a batch if we know that all requests stopped + return generations, batch if not stopped else None diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index dc78aa8b..746e9e83 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -96,11 +96,13 @@ def from_pb( stopping_criterias = [] offsets = [] token_offsets = [] + requests_idx_mapping = {} # Parse batch max_truncation = 0 padding_right_offset = 0 - for r in pb.requests: + for i, r in enumerate(pb.requests): + requests_idx_mapping[r.id] = i # Add escape_custom_split_sequence to the CausalLMBatch logic inputs.append(escape_custom_split_sequence(r.inputs)) offsets.append(None) @@ -115,7 +117,6 @@ def from_pb( padding_right_offset, stopping_criteria.max_new_tokens ) - # Tokenize batch tokenized_inputs = tokenizer( inputs, return_tensors="pt", @@ -138,23 +139,23 @@ def from_pb( position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) - all_input_ids = tokenized_inputs["input_ids"].unsqueeze(-1) + all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) return cls( batch_id=pb.id, requests=pb.requests, + requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, past_key_values=None, - all_input_ids=all_input_ids, - input_lengths=input_lengths, + all_input_ids=list(all_input_ids), + input_lengths=input_lengths.tolist(), offsets=offsets, token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, - size=pb.size, - max_input_length=max_input_length, + max_input_length=max_input_length.item(), padding_right_offset=padding_right_offset, ) diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 82b5def0..dd2f999b 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from opentelemetry import trace from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, PreTrainedTokenizerBase -from typing import Optional, Tuple, List, Type +from typing import Optional, Tuple, List, Type, Dict from text_generation_server.models import Model from text_generation_server.models.types import ( @@ -22,6 +22,7 @@ class Seq2SeqLMBatch(Batch): batch_id: int requests: List[generate_pb2.Request] + requests_idx_mapping: Dict[int, int] # Encoder values input_ids: torch.Tensor @@ -32,6 +33,9 @@ class Seq2SeqLMBatch(Batch): decoder_attention_mask: Optional[torch.Tensor] encoder_last_hidden_state: Optional[torch.Tensor] + # All tokens + all_decoder_input_ids: List[torch.Tensor] + # Seq2SeqLM keeps track of both encoder and decoder attention keys and values past_key_values: Optional[List[Tuple]] @@ -46,7 +50,6 @@ class Seq2SeqLMBatch(Batch): stopping_criterias: List[StoppingCriteria] # Metadata used for padding - size: int max_input_length: int max_decoder_input_length: int padding_right_offset: int @@ -54,9 +57,7 @@ class Seq2SeqLMBatch(Batch): def to_pb(self) -> generate_pb2.Batch: """Convert a Seq2SeqLMBatch to a text_generation_server.v1.Batch protobuf""" return generate_pb2.Batch( - id=self.batch_id, - requests=self.requests, - size=self.size, + id=self.batch_id, requests=self.requests, size=len(self) ) @classmethod @@ -71,18 +72,17 @@ def from_pb( next_token_choosers = [] stopping_criterias = [] - decoder_input_ids = [] decoder_input_lengths = [] offsets = [] token_offsets = [] + requests_idx_mapping = {} # Parse batch max_truncation = 0 padding_right_offset = 0 - for r in pb.requests: + for i, r in enumerate(pb.requests): inputs.append(r.inputs) - # Decoder sequence only contains the bos_token - decoder_input_ids.append(tokenizer.bos_token_id) + requests_idx_mapping[r.id] = i decoder_input_lengths.append(1) offsets.append(None) token_offsets.append(None) @@ -109,15 +109,22 @@ def from_pb( input_lengths = tokenized_inputs["attention_mask"].sum(1) max_input_length = input_lengths.max() - # Convert decoder_input_ids to torch tensor of size [batch_size, 1] - decoder_input_ids = torch.tensor(decoder_input_ids, device=device).unsqueeze(-1) + # Decoder sequence only contains the bos_token + decoder_input_ids = ( + torch.tensor(tokenizer.bos_token_id, device=device) + .repeat(len(pb.requests)) + .view(-1, 1) + ) + all_decoder_input_ids = decoder_input_ids.view(-1).split(1) return cls( batch_id=pb.id, requests=pb.requests, + requests_idx_mapping=requests_idx_mapping, input_ids=tokenized_inputs["input_ids"], attention_mask=tokenized_inputs["attention_mask"], decoder_input_ids=decoder_input_ids, + all_decoder_input_ids=list(all_decoder_input_ids), decoder_attention_mask=None, encoder_last_hidden_state=None, past_key_values=None, @@ -127,12 +134,96 @@ def from_pb( token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, - size=len(pb.requests), max_input_length=max_input_length.item(), max_decoder_input_length=1, padding_right_offset=padding_right_offset, ) + @tracer.start_as_current_span("filter") + def filter( + self, requests: List[generate_pb2.Request] + ) -> Optional["Seq2SeqLMBatch"]: + if len(requests) == 0: + raise ValueError("Batch must have at least one request") + if len(requests) == len(self): + return self + + keep_indices = [] + + # New values after filtering + requests_idx_mapping = {} + input_lengths = [] + decoder_input_lengths = [] + offsets = [] + token_offsets = [] + + all_decoder_input_ids = [] + + next_token_choosers = [] + stopping_criterias = [] + + max_input_length = 0 + max_decoder_input_length = 0 + + for i, r in enumerate(requests): + idx = self.requests_idx_mapping[r.id] + requests_idx_mapping[r.id] = i + keep_indices.append(idx) + + offsets.append(self.offsets[idx]) + token_offsets.append(self.token_offsets[idx]) + + all_decoder_input_ids.append(self.all_decoder_input_ids[idx]) + + request_input_length = self.input_lengths[idx] + input_lengths.append(request_input_length) + max_input_length = max(max_input_length, request_input_length) + + request_decoder_input_length = self.decoder_input_lengths[idx] + decoder_input_lengths.append(request_decoder_input_length) + max_decoder_input_length = max( + max_decoder_input_length, request_decoder_input_length + ) + + next_token_choosers.append(self.next_token_choosers[idx]) + stopping_criterias.append(self.stopping_criterias[idx]) + + # Apply indices to input_ids, attention mask, past key values and other items that need to be cached + decoder_input_ids = self.decoder_input_ids[keep_indices] + attention_mask = self.attention_mask[keep_indices] + if self.decoder_attention_mask is not None: + decoder_attention_mask = self.decoder_attention_mask[keep_indices] + else: + decoder_attention_mask = None + + encoder_last_hidden_state = self.encoder_last_hidden_state[keep_indices] + + past_key_values = [ + [t[keep_indices] for t in layer] for layer in self.past_key_values + ] + + return Seq2SeqLMBatch( + batch_id=self.batch_id, + requests=requests, + requests_idx_mapping=requests_idx_mapping, + input_ids=None, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + all_decoder_input_ids=all_decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + encoder_last_hidden_state=encoder_last_hidden_state, + past_key_values=past_key_values, + input_lengths=input_lengths, + decoder_input_lengths=decoder_input_lengths, + offsets=offsets, + token_offsets=token_offsets, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + max_input_length=max_input_length, + max_decoder_input_length=max_decoder_input_length, + padding_right_offset=self.padding_right_offset, + ) + @classmethod @tracer.start_as_current_span("concatenate") def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": @@ -144,7 +235,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": max_decoder_input_length = 0 padding_right_offset = 0 for batch in batches: - total_batch_size += batch.size + total_batch_size += len(batch) max_input_length = max(max_input_length, batch.max_input_length) max_decoder_input_length = max( max_decoder_input_length, batch.max_decoder_input_length @@ -153,6 +244,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Batch attributes requests = [] + requests_idx_mapping = {} + all_decoder_input_ids = [] input_lengths = [] decoder_input_lengths = [] offsets = [] @@ -174,6 +267,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": for i, batch in enumerate(batches): # Extend all list attributes requests.extend(batch.requests) + all_decoder_input_ids.extend(batch.all_decoder_input_ids) input_lengths.extend(batch.input_lengths) decoder_input_lengths.extend(batch.decoder_input_lengths) offsets.extend(batch.offsets) @@ -181,8 +275,15 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) + if i == 0: + requests_idx_mapping = batch.requests_idx_mapping + else: + # We need to offset the mapping for each batch by the cumulative batch size + for k, v in batch.requests_idx_mapping.items(): + requests_idx_mapping[k] = v + start_index + # Slicing end index for this batch - end_index = start_index + batch.size + end_index = start_index + len(batch) # We only concatenate batches that did at least one step if batch.encoder_last_hidden_state is None: @@ -201,12 +302,10 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Create padded tensor if decoder_input_ids is None: decoder_input_ids = batch.decoder_input_ids.new_zeros( - (total_batch_size, max_decoder_input_length), + (total_batch_size, 1), ) # Copy to correct indices - decoder_input_ids[ - start_index:end_index, -batch.max_decoder_input_length : - ] = batch.decoder_input_ids[:, -batch.max_decoder_input_length :] + decoder_input_ids[start_index:end_index] = batch.decoder_input_ids # Create padded tensor if decoder_attention_mask is None: @@ -302,14 +401,16 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": start_index:end_index, :, -batch.max_input_length :, : ] = t[:, :, -batch.max_input_length :, :] - start_index += batch.size + start_index += len(batch) return cls( batch_id=batches[0].batch_id, requests=requests, + requests_idx_mapping=requests_idx_mapping, input_ids=None, attention_mask=attention_mask, decoder_input_ids=decoder_input_ids, + all_decoder_input_ids=all_decoder_input_ids, decoder_attention_mask=decoder_attention_mask, encoder_last_hidden_state=encoder_last_hidden_state, past_key_values=past_key_values, @@ -319,7 +420,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": token_offsets=token_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, - size=total_batch_size, max_input_length=max_input_length, max_decoder_input_length=max_decoder_input_length, padding_right_offset=padding_right_offset, @@ -413,46 +513,25 @@ def generate_token( else: decoder_attention_mask = None - # check if first forward or not - if batch.past_key_values is not None: - # Only take the last token - decoder_input_ids = batch.decoder_input_ids[:, -1].unsqueeze(-1) - else: - decoder_input_ids = batch.decoder_input_ids - # Wrap `encoder_last_hidden_state` because for some reason, Transformers does a `encoder_last_hidden_state[0]` # internally... if batch.encoder_last_hidden_state is not None: encoder_last_hidden_state = [batch.encoder_last_hidden_state] else: - encoder_last_hidden_state = batch.encoder_last_hidden_state + encoder_last_hidden_state = None logits, encoder_last_hidden_state, past = self.forward( batch.input_ids, batch.attention_mask, - decoder_input_ids, + batch.decoder_input_ids, decoder_attention_mask, encoder_last_hidden_state, batch.past_key_values, ) - # List of indices to cache - next_batch_keep_indices = [] - - # New values for next forward - next_batch_input_lengths = [] - next_batch_offsets = [] - next_batch_token_offsets = [] - next_batch_decoder_input_ids = [] - next_batch_decoder_input_lengths = [] - - # Metadata - next_batch_size = 0 - next_batch_max_input_length = 0 - next_batch_max_decoder_input_length = 0 - # Finished requests generations: List[Generation] = [] + stopped = True # Zipped iterator iterator = zip( @@ -464,7 +543,7 @@ def generate_token( logits, batch.next_token_choosers, batch.stopping_criterias, - batch.decoder_input_ids, + batch.all_decoder_input_ids, ) # For each member of the batch @@ -477,22 +556,24 @@ def generate_token( logits, next_token_chooser, stopping_criteria, - decoder_input_ids, + all_decoder_input_ids, ) in enumerate(iterator): # Select next token next_token_id, logprobs = next_token_chooser( - decoder_input_ids.view(1, -1), logits + all_decoder_input_ids.view(1, -1), logits ) # Append next token to decoder tokens - decoder_input_ids = torch.cat([decoder_input_ids, next_token_id.squeeze(1)]) + all_decoder_input_ids = torch.cat( + [all_decoder_input_ids, next_token_id.squeeze(1)] + ) new_decoder_input_length = decoder_input_length + 1 # Generated token next_token_logprob = logprobs[-1, next_token_id] next_token_id_squeezed = next_token_id.squeeze() next_token_text, offset, token_offset = self.decode_token( - decoder_input_ids, offset, token_offset + all_decoder_input_ids, offset, token_offset ) # Evaluate stopping criteria @@ -501,7 +582,7 @@ def generate_token( if stop: # Slice with decoder_input_length to remove padding # Decode all tokens - output_text = self.decode(decoder_input_ids[-decoder_input_length:]) + output_text = self.decode(all_decoder_input_ids[-decoder_input_length:]) # Get seed if isinstance(next_token_chooser.choice, Sampling): @@ -515,19 +596,7 @@ def generate_token( else: # Keep request in the batch generated_text = None - next_batch_keep_indices.append(i) - next_batch_decoder_input_ids.append(decoder_input_ids.unsqueeze(0)) - next_batch_size += 1 - next_batch_input_lengths.append(input_length) - next_batch_decoder_input_lengths.append(new_decoder_input_length) - next_batch_offsets.append(offset) - next_batch_token_offsets.append(token_offset) - next_batch_max_input_length = max( - next_batch_max_input_length, input_length - ) - next_batch_max_decoder_input_length = max( - next_batch_max_decoder_input_length, new_decoder_input_length - ) + stopped = False # Prefill if stopping_criteria.current_tokens == 1: @@ -551,69 +620,29 @@ def generate_token( generations.append(generation) + # Update values + batch.decoder_input_ids[i] = next_token_id + batch.all_decoder_input_ids[i] = all_decoder_input_ids + batch.input_lengths[i] = input_length + batch.decoder_input_lengths[i] = new_decoder_input_length + batch.offsets[i] = offset + batch.token_offsets[i] = token_offset + batch.max_input_length = max(batch.max_input_length, input_length) + batch.max_decoder_input_length = max( + batch.max_decoder_input_length, new_decoder_input_length + ) + # We finished all generations in the batch; there is no next batch - if not next_batch_keep_indices: + if stopped: return generations, None - next_batch_decoder_input_ids = torch.cat(next_batch_decoder_input_ids) - # If we finished at least one generation, we need to evict the indices of the generations that finished - # from the values of the next batch - if len(next_batch_keep_indices) != len(batch): - # Apply indices to decoder_attention mask, past key values and other items that need to be cached - next_batch_attention_mask = batch.attention_mask[next_batch_keep_indices] - if batch.decoder_attention_mask is not None: - next_batch_decoder_attention_mask = batch.decoder_attention_mask[ - next_batch_keep_indices - ] - else: - next_batch_decoder_attention_mask = None - - next_batch_encoder_last_hidden_state = encoder_last_hidden_state[ - next_batch_keep_indices - ] - - next_batch_past_key_values = [ - [t[next_batch_keep_indices] for t in layer] for layer in past - ] - next_batch_requests = [batch.requests[i] for i in next_batch_keep_indices] - next_batch_next_token_choosers = [ - batch.next_token_choosers[i] for i in next_batch_keep_indices - ] - next_batch_stopping_criterias = [ - batch.stopping_criterias[i] for i in next_batch_keep_indices - ] - else: - next_batch_attention_mask = batch.attention_mask - next_batch_decoder_attention_mask = batch.decoder_attention_mask - next_batch_encoder_last_hidden_state = encoder_last_hidden_state - next_batch_past_key_values = past - - next_batch_requests = batch.requests - next_batch_next_token_choosers = batch.next_token_choosers - next_batch_stopping_criterias = batch.stopping_criterias - + # We don't need input_ids after the prefill forward + batch.input_ids = None + batch.encoder_last_hidden_state = encoder_last_hidden_state + batch.past_key_values = past # Update decoder_attention_mask as we added a new token to input_ids - if next_batch_decoder_attention_mask is not None: - next_batch_decoder_attention_mask[:, -batch.padding_right_offset] = 1 + if batch.decoder_attention_mask is not None: + batch.decoder_attention_mask[:, -batch.padding_right_offset] = 1 + batch.padding_right_offset -= 1 - next_batch = Seq2SeqLMBatch( - batch_id=batch.batch_id, - requests=next_batch_requests, - input_ids=None, - attention_mask=next_batch_attention_mask, - decoder_input_ids=next_batch_decoder_input_ids, - decoder_attention_mask=next_batch_decoder_attention_mask, - encoder_last_hidden_state=next_batch_encoder_last_hidden_state, - past_key_values=next_batch_past_key_values, - input_lengths=next_batch_input_lengths, - decoder_input_lengths=next_batch_decoder_input_lengths, - offsets=next_batch_offsets, - token_offsets=next_batch_token_offsets, - next_token_choosers=next_batch_next_token_choosers, - stopping_criterias=next_batch_stopping_criterias, - size=next_batch_size, - max_input_length=next_batch_max_input_length, - max_decoder_input_length=next_batch_max_decoder_input_length, - padding_right_offset=batch.padding_right_offset - 1, - ) - return generations, next_batch + return generations, batch diff --git a/server/text_generation_server/models/types.py b/server/text_generation_server/models/types.py index 93c3b9db..8a5b82f7 100644 --- a/server/text_generation_server/models/types.py +++ b/server/text_generation_server/models/types.py @@ -25,6 +25,10 @@ def from_pb( ) -> "Batch": raise NotImplementedError + @abstractmethod + def filter(self, requests: List[generate_pb2.Request]) -> "Batch": + raise NotImplementedError + @classmethod @abstractmethod def concatenate(cls, batches: List["Batch"]) -> "Batch": diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 3e3789bf..3caee803 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -60,7 +60,12 @@ async def Decode(self, request, context): batch = self.cache.pop(batch_pb.id) if batch is None: raise ValueError(f"Batch ID {batch_pb.id} not found in cache.") - batches.append(batch) + batch = batch.filter(batch_pb.requests) + if batch is not None: + batches.append(batch) + + if len(batches) == 0: + raise ValueError("All batches are empty") if len(batches) > 1: batch = self.model.batch_type.concatenate(batches) From 274513e6a31679d9c39d1cf8ba1c3a09ac609650 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 20 Apr 2023 18:50:47 +0200 Subject: [PATCH 178/793] fix(ci): fix sha in docker image (#212) --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2ad1d424..67d1c730 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -96,7 +96,7 @@ jobs: push: ${{ github.event_name != 'pull_request' }} platforms: 'linux/amd64' build-args: | - GIT_SHA={{ env.GITHUB_SHA }} + GIT_SHA=${{ env.GITHUB_SHA }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max From ac8c0f6fe4c0c061e0f858242ee61dfa090c32a6 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Fri, 21 Apr 2023 05:57:18 -0700 Subject: [PATCH 179/793] feat(server): flash attention past key value optimizations (#213) --- .../models/flash_causal_lm.py | 59 ++++++++++++------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 382f917d..2843f273 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -38,7 +38,7 @@ class FlashCausalLMBatch(Batch): # cumulative sequence lengths cu_seqlens: List[int] max_seqlen: int - past_key_values: Optional[List[torch.Tensor]] + past_key_values: Optional[Union[torch.Tensor, List[torch.Tensor]]] # All tokens all_input_ids: List[List[int]] @@ -53,6 +53,9 @@ class FlashCausalLMBatch(Batch): next_token_choosers: List[NextTokenChooser] stopping_criterias: List[StoppingCriteria] + # Constant shared tensor, ref here just so that it's accessible in concatentate() + past_pad: Optional[torch.Tensor] + def to_pb(self) -> generate_pb2.Batch: return generate_pb2.Batch( id=self.batch_id, requests=self.requests, size=len(self) @@ -149,6 +152,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": if len(requests) == len(self): return self + single_request = len(requests) == 1 + # Cumulative length cumulative_length = 0 @@ -182,7 +187,9 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": position_ids.append(self.position_ids[idx]) cu_seqlens.append(cumulative_length + request_input_length) max_seqlen = max(max_seqlen, request_input_length) - past_key_values.append(self.past_key_values[idx]) + if not single_request: + past_key_values.append(self.past_key_values[2 * idx]) + past_key_values.append(self.past_key_values[1]) all_input_ids.append(self.all_input_ids[idx]) all_input_ids_tensor.append(self.all_input_ids_tensor[idx]) @@ -196,6 +203,13 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": cumulative_length += request_input_length + if single_request: + # Preallocate tensor for bs = 1 case + past_key_values = torch.nn.functional.pad( + self.past_key_values[0], + (0, 0, 0, 0, 0, 0, 0, stopping_criterias[0].max_new_tokens - stopping_criterias[0].current_tokens) + ) + return FlashCausalLMBatch( batch_id=self.batch_id, requests=requests, @@ -256,7 +270,11 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch # Add cumulative lengths of all previous inputs cu_seqlens.extend([l + cumulative_length for l in batch.cu_seqlens[1:]]) max_seqlen = max(max_seqlen, batch.max_seqlen) - past_key_values.extend(batch.past_key_values) + if len(batch) != 1: + past_key_values.extend(batch.past_key_values) + else: + past_key_values.append(batch.past_key_values[:, :batch.input_lengths[0]]) + past_key_values.append(batch.past_pad) all_input_ids.extend(batch.all_input_ids) all_input_ids_tensor.extend(batch.all_input_ids_tensor) @@ -303,6 +321,7 @@ def __init__( quantize: bool = False, decode_buffer: int = 3, ): + self.past_pad = None if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 @@ -359,10 +378,8 @@ def generate_token( ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: # Shortcut when batch_size == 1 if len(batch) == 1: - input_ids = batch.input_ids[0].view(-1) - past_key_values = ( - batch.past_key_values[0] if batch.past_key_values is not None else None - ) + # No need to slice this down + past_key_values = batch.past_key_values else: # Concatenate tensors input_ids = torch.cat(batch.input_ids).view(-1) @@ -392,7 +409,18 @@ def generate_token( # Initialize past_key_values in prefill if batch.past_key_values is None: - batch.past_key_values = [None] * len(batch) + # Initialize past padding tensor + if self.past_pad is None: + self.past_pad = present.new_zeros(present.shape[0], 1, *present.shape[2:]) + # Set in batch in case it needs to be used later in concatenate() + batch.past_pad = self.past_pad + if len(batch) == 1: + # Preallocate tensor for bs = 1 case + batch.past_key_values = torch.nn.functional.pad( + present, (0, 0, 0, 0, 0, 0, 0, batch.stopping_criterias[0].max_new_tokens) + ) + else: + batch.past_key_values = [None, self.past_pad] * len(batch) # Cumulative length cumulative_length = 0 @@ -477,21 +505,10 @@ def generate_token( generated_text = GeneratedText( output_text, stopping_criteria.current_tokens, reason, seed ) - - # CAUTION: generation will be stopped so no need to pad - # This will make the next forward crash if the request does not get filtered - new_input_length = input_length - past = present[:, start_index:end_index] else: stopped = False generated_text = None - # Pad present for next iter attention - new_input_length = input_length + 1 - past = torch.nn.functional.pad( - present[:, start_index:end_index], (0, 0, 0, 0, 0, 0, 0, 1) - ) - # Prefill if prefill: # Remove generated token to only have prefill and add nan for first prompt token @@ -522,6 +539,7 @@ def generate_token( generations.append(generation) cumulative_length += input_length + new_input_length = input_length + 1 # Update values batch.input_ids[i] = next_token_id @@ -532,7 +550,8 @@ def generate_token( batch.all_input_ids[i] = all_input_ids batch.all_input_ids_tensor[i] = all_input_ids_tensor batch.max_seqlen = max(batch.max_seqlen, new_input_length) - batch.past_key_values[i] = past + if len(batch) != 1: + batch.past_key_values[i * 2] = present[:, start_index:end_index] # Cumulative sum batch.cu_seqlens[(i + 1)] = batch.cu_seqlens[i] + new_input_length From 343437c7b5987c64cd3353b0913ffad5fd3df4b5 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 21 Apr 2023 15:36:29 +0200 Subject: [PATCH 180/793] feat(router): add device and dtype info (#215) --- proto/generate.proto | 11 +++++++++++ router/client/src/client.rs | 8 ++++++++ router/client/src/lib.rs | 1 + router/client/src/sharded_client.rs | 13 ++++++++++++- router/src/lib.rs | 6 +++++- router/src/main.rs | 12 +++++++++--- router/src/server.rs | 16 ++++++++++++---- server/text_generation_server/models/bloom.py | 6 +++++- .../text_generation_server/models/causal_lm.py | 6 +++++- .../models/flash_causal_lm.py | 6 +++++- .../text_generation_server/models/flash_llama.py | 4 ++++ .../text_generation_server/models/flash_neox.py | 2 ++ .../models/flash_santacoder.py | 8 +++++++- .../text_generation_server/models/galactica.py | 2 ++ server/text_generation_server/models/gpt_neox.py | 2 ++ server/text_generation_server/models/model.py | 13 +++++++++++++ server/text_generation_server/models/opt.py | 2 ++ .../text_generation_server/models/santacoder.py | 6 +++++- .../text_generation_server/models/seq2seq_lm.py | 6 +++++- server/text_generation_server/models/t5.py | 2 ++ server/text_generation_server/server.py | 3 +++ 21 files changed, 120 insertions(+), 15 deletions(-) diff --git a/proto/generate.proto b/proto/generate.proto index cc14cbf8..2bf7385c 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -3,6 +3,8 @@ syntax = "proto3"; package generate.v1; service TextGenerationService { + /// Model Info + rpc Info (InfoRequest) returns (InfoResponse) {} /// Service discovery rpc ServiceDiscovery (ServiceDiscoveryRequest) returns (ServiceDiscoveryResponse) {} /// Empties batch cache @@ -13,6 +15,15 @@ service TextGenerationService { rpc Decode (DecodeRequest) returns (DecodeResponse); } +/// Empty request +message InfoRequest {} + +message InfoResponse { + bool requires_padding = 1; + string dtype = 2; + string device_type = 3; +} + /// Empty request message ServiceDiscoveryRequest {} diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 1b2086a0..cccd5004 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -54,6 +54,14 @@ impl Client { Ok(urls) } + /// Get model info + #[instrument(skip(self))] + pub async fn info(&mut self) -> Result { + let request = tonic::Request::new(InfoRequest {}).inject_context(); + let response = self.stub.info(request).await?.into_inner(); + Ok(response) + } + /// Clear the past generations cache #[instrument(skip(self))] pub async fn clear_cache(&mut self, batch_id: Option) -> Result<()> { diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 21fbc1e1..6a001306 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -6,6 +6,7 @@ mod pb; mod sharded_client; pub use client::Client; +pub use pb::generate::v1::InfoResponse as ShardInfo; pub use pb::generate::v1::{ Batch, FinishReason, GeneratedText, Generation, NextTokenChooserParameters, PrefillTokens, Request, StoppingCriteriaParameters, diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 31f76312..903c7a67 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,6 +1,6 @@ /// Multi shard Client use crate::Result; -use crate::{Batch, Client, Generation}; +use crate::{Batch, Client, Generation, ShardInfo}; use futures::future::join_all; use tonic::transport::Uri; use tracing::instrument; @@ -37,6 +37,17 @@ impl ShardedClient { Self::from_master_client(master_client).await } + /// Get the model info + #[instrument(skip(self))] + pub async fn info(&mut self) -> Result { + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| client.info()) + .collect(); + join_all(futures).await.pop().unwrap() + } + /// Clear the past generations cache #[instrument(skip(self))] pub async fn clear_cache(&mut self, batch_id: Option) -> Result<()> { diff --git a/router/src/lib.rs b/router/src/lib.rs index 7dc115fd..2f93ec0a 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -12,7 +12,7 @@ use validation::Validation; /// Hub type #[derive(Clone, Debug, Deserialize)] -pub struct ModelInfo { +pub struct HubModelInfo { #[serde(rename(deserialize = "id"))] pub model_id: String, pub sha: Option, @@ -25,6 +25,10 @@ pub struct Info { pub model_id: String, #[schema(nullable = true, example = "e985a63cdc139290c5f700ff1929f0b5942cced2")] pub model_sha: Option, + #[schema(example = "torch.float16")] + pub model_dtype: String, + #[schema(example = "cuda")] + pub model_device_type: String, #[schema(nullable = true, example = "text-generation")] pub model_pipeline_tag: Option, #[schema(example = "0.5.0")] diff --git a/router/src/main.rs b/router/src/main.rs index 31783bbe..712071b7 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -10,7 +10,7 @@ use opentelemetry_otlp::WithExportConfig; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path; use text_generation_client::ShardedClient; -use text_generation_router::{server, ModelInfo}; +use text_generation_router::{server, HubModelInfo}; use tokenizers::{FromPretrainedParameters, Tokenizer}; use tower_http::cors::AllowOrigin; use tracing_subscriber::layer::SubscriberExt; @@ -128,7 +128,7 @@ fn main() -> Result<(), std::io::Error> { // Get Model info let model_info = match local_model { - true => ModelInfo { + true => HubModelInfo { model_id: tokenizer_name.clone(), sha: None, pipeline_tag: None, @@ -154,6 +154,11 @@ fn main() -> Result<(), std::io::Error> { .clear_cache(None) .await .expect("Unable to clear cache"); + // Get info from the shard + let shard_info = sharded_client + .info() + .await + .expect("Unable to get shard info"); tracing::info!("Connected"); // Binds on localhost @@ -162,6 +167,7 @@ fn main() -> Result<(), std::io::Error> { // Run server server::run( model_info, + shard_info, compat_return_full_text, max_concurrent_requests, max_best_of, @@ -237,7 +243,7 @@ fn init_logging(otlp_endpoint: Option, json_output: bool) { } /// get model info from the Huggingface Hub -pub async fn get_model_info(model_id: &str, revision: &str, token: Option) -> ModelInfo { +pub async fn get_model_info(model_id: &str, revision: &str, token: Option) -> HubModelInfo { let client = reqwest::Client::new(); let mut builder = client.get(format!( "https://huggingface.co/api/models/{model_id}/revision/{revision}" diff --git a/router/src/server.rs b/router/src/server.rs index fee748e6..8891443f 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -3,7 +3,7 @@ use crate::infer::{InferError, InferResponse, InferStreamResponse}; use crate::validation::ValidationError; use crate::{ BestOfSequence, CompatGenerateRequest, Details, ErrorResponse, FinishReason, - GenerateParameters, GenerateRequest, GenerateResponse, Infer, Info, ModelInfo, PrefillToken, + GenerateParameters, GenerateRequest, GenerateResponse, HubModelInfo, Infer, Info, PrefillToken, StreamDetails, StreamResponse, Token, Validation, }; use axum::extract::Extension; @@ -18,7 +18,7 @@ use futures::Stream; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; use std::convert::Infallible; use std::net::SocketAddr; -use text_generation_client::ShardedClient; +use text_generation_client::{ShardInfo, ShardedClient}; use tokenizers::Tokenizer; use tokio::signal; use tokio::time::Instant; @@ -78,13 +78,19 @@ async fn compat_generate( responses((status = 200, description = "Served model info", body = Info)) )] #[instrument] -async fn get_model_info(model_info: Extension) -> Json { +async fn get_model_info( + model_info: Extension, + shard_info: Extension, +) -> Json { let model_info = model_info.0; + let shard_info = shard_info.0; let info = Info { version: env!("CARGO_PKG_VERSION"), sha: option_env!("VERGEN_GIT_SHA"), model_id: model_info.model_id, model_sha: model_info.sha, + model_dtype: shard_info.dtype, + model_device_type: shard_info.device_type, model_pipeline_tag: model_info.pipeline_tag, }; Json(info) @@ -497,7 +503,8 @@ async fn metrics(prom_handle: Extension) -> String { /// Serving method #[allow(clippy::too_many_arguments)] pub async fn run( - model_info: ModelInfo, + model_info: HubModelInfo, + shard_info: ShardInfo, compat_return_full_text: bool, max_concurrent_requests: usize, max_best_of: usize, @@ -641,6 +648,7 @@ pub async fn run( // Prometheus metrics route .route("/metrics", get(metrics)) .layer(Extension(model_info)) + .layer(Extension(shard_info)) .layer(Extension(compat_return_full_text)) .layer(Extension(infer)) .layer(Extension(prom_handle)) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 731a9854..e43a4b79 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -100,7 +100,11 @@ def __init__( self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( - tokenizer=tokenizer, device=device, decode_buffer=1 + tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, + device=device, + decode_buffer=1, ) @staticmethod diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 71eff437..98313253 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -400,7 +400,11 @@ def __init__( ) super(CausalLM, self).__init__( - tokenizer=tokenizer, device=device, decode_buffer=decode_buffer + tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, + device=device, + decode_buffer=decode_buffer, ) @property diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 2843f273..0e2fbaab 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -343,7 +343,11 @@ def __init__( ) super(FlashCausalLM, self).__init__( - tokenizer=tokenizer, device=device, decode_buffer=decode_buffer + tokenizer=tokenizer, + requires_padding=False, + dtype=dtype, + device=device, + decode_buffer=decode_buffer, ) @property diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 9cbf1b57..764de2a2 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -63,6 +63,8 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False super(FlashCausalLM, self).__init__( tokenizer=tokenizer, + requires_padding=False, + dtype=dtype, device=device, ) @@ -184,6 +186,8 @@ def __init__( torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( tokenizer=tokenizer, + requires_padding=False, + dtype=dtype, device=device, ) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 0cda728a..259fc208 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -70,6 +70,8 @@ def __init__( torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( tokenizer=tokenizer, + requires_padding=False, + dtype=dtype, device=device, ) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index e3066c98..7dcd8b03 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -65,7 +65,11 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False self.model = model.eval().to(device) super(FlashCausalLM, self).__init__( - tokenizer=tokenizer, device=device, decode_buffer=1 + tokenizer=tokenizer, + requires_padding=False, + dtype=dtype, + device=device, + decode_buffer=1, ) @staticmethod @@ -206,6 +210,8 @@ def __init__( torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( tokenizer=tokenizer, + requires_padding=False, + dtype=dtype, device=device, ) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 746e9e83..753e86ed 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -228,6 +228,8 @@ def __init__( torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, device=device, ) diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 489615e1..3b5fe2cc 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -72,6 +72,8 @@ def __init__( torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, device=device, ) diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 08a48553..bfae829d 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -5,6 +5,7 @@ from transformers import PreTrainedTokenizerBase from text_generation_server.models.types import Batch, GeneratedText +from text_generation_server.pb.generate_pb2 import InfoResponse B = TypeVar("B", bound=Batch) @@ -13,6 +14,8 @@ class Model(ABC): def __init__( self, tokenizer: PreTrainedTokenizerBase, + requires_padding: bool, + dtype: torch.dtype, device: torch.device, decode_buffer: int = 3, ): @@ -21,9 +24,19 @@ def __init__( self.tokenizer = tokenizer self.all_special_ids = set(tokenizer.all_special_ids) + self.requires_padding = requires_padding + self.dtype = dtype self.device = device self.decode_buffer = decode_buffer + @property + def info(self) -> InfoResponse: + return InfoResponse( + requires_padding=self.requires_padding, + dtype=str(self.dtype), + device_type=self.device.type, + ) + @property @abstractmethod def batch_type(self) -> Type[B]: diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 8e5527c0..1a21186f 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -88,6 +88,8 @@ def __init__( torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, device=device, ) diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index 8646a4e1..796c33e3 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -54,7 +54,11 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False ) super(CausalLM, self).__init__( - tokenizer=tokenizer, device=device, decode_buffer=1 + tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, + device=device, + decode_buffer=1, ) def decode(self, generated_ids: List[int]) -> str: diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index dd2f999b..aa452c70 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -460,7 +460,11 @@ def __init__( tokenizer.bos_token_id = self.model.config.decoder_start_token_id super(Seq2SeqLM, self).__init__( - tokenizer=tokenizer, device=device, decode_buffer=decode_buffer + tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, + device=device, + decode_buffer=decode_buffer, ) @property diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index b9f77015..487a5984 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -72,6 +72,8 @@ def __init__( torch.distributed.barrier(group=self.process_group) super(Seq2SeqLM, self).__init__( tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, device=device, ) diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 3caee803..95b431c8 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -26,6 +26,9 @@ def __init__(self, model: Model, cache: Cache, server_urls: List[str]): # Force inference mode for the lifetime of TextGenerationService self._inference_mode_raii_guard = torch._C._InferenceMode(True) + async def Info(self, request, context): + return self.model.info + async def ServiceDiscovery(self, request, context): return generate_pb2.ServiceDiscoveryResponse(urls=self.server_urls) From db4cb5e4ed2b994131fc575eff6689da9b661679 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 21 Apr 2023 15:59:18 +0200 Subject: [PATCH 181/793] fix(server): fix past key values logic (#216) @njhill fyi --- .../custom_modeling/flash_llama_modeling.py | 29 ++++++++-- .../custom_modeling/flash_neox_modeling.py | 27 ++++++++-- .../flash_santacoder_modeling.py | 29 ++++++++-- .../models/flash_causal_lm.py | 53 ++++++++++++++++--- .../models/flash_llama.py | 2 + .../models/flash_neox.py | 1 + .../models/flash_santacoder.py | 2 + 7 files changed, 123 insertions(+), 20 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 71843886..ee1bd01f 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -25,6 +25,7 @@ from torch import nn from transformers.activations import ACT2FN +from typing import Optional # Flash attention imports import rotary_emb @@ -554,7 +555,8 @@ def forward( position_ids, cu_seqlens, max_s, - past_key_values=None, + past_key_values: Optional[torch.Tensor] = None, + pre_allocate_past_size: Optional[int] = None, ): hidden_states = self.embed_tokens(input_ids) @@ -564,7 +566,9 @@ def forward( past_key_values = hidden_states.new_empty( ( len(self.layers), - len(hidden_states), + len(hidden_states) + if pre_allocate_past_size is None + else pre_allocate_past_size, 2, self.num_heads, self.head_size, @@ -572,6 +576,7 @@ def forward( ) layer_past_present_indices = None cu_seqlens_q = None + slice_past_index = len(hidden_states) # Decode else: # Create indices from cumulative sequence lengths @@ -579,6 +584,7 @@ def forward( cu_seqlens_q = torch.arange( cu_seqlens.shape[0], dtype=torch.int32, device=hidden_states.device ) + slice_past_index = None # Get rotary cos and sin for this forward # Avoid to index in each layer @@ -588,6 +594,13 @@ def forward( residual = None for i, layer in enumerate(self.layers): + # We added padding that now need to slice + layer_past_key_values = ( + past_key_values[i] + if slice_past_index is None + else past_key_values[i, :slice_past_index] + ) + hidden_states, residual = layer( hidden_states, residual, @@ -595,7 +608,7 @@ def forward( sin, cu_seqlens, max_s, - past_key_values[i], + layer_past_key_values, layer_past_present_indices, cu_seqlens_q, ) @@ -638,10 +651,16 @@ def forward( position_ids, cu_seqlens, max_s, - past_key_values=None, + past_key_values: Optional[torch.Tensor] = None, + pre_allocate_past_size: Optional[int] = None, ): hidden_states, present = self.model( - input_ids, position_ids, cu_seqlens, max_s, past_key_values + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values, + pre_allocate_past_size, ) logits = self.lm_head(hidden_states) diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 8de582e3..60545848 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -27,6 +27,7 @@ from transformers.activations import ACT2FN from transformers.modeling_utils import PreTrainedModel from transformers.models.gpt_neox import GPTNeoXConfig +from typing import Optional # Flash attention imports import rotary_emb @@ -618,6 +619,7 @@ def forward( cu_seqlens, max_s, past_key_values=None, + pre_allocate_past_size: Optional[int] = None, ): hidden_states = self.embed_in(input_ids) @@ -627,7 +629,9 @@ def forward( past_key_values = hidden_states.new_empty( ( len(self.layers), - len(hidden_states), + len(hidden_states) + if pre_allocate_past_size is None + else pre_allocate_past_size, 2, self.num_heads, self.head_size, @@ -635,6 +639,7 @@ def forward( ) layer_past_present_indices = None cu_seqlens_q = None + slice_past_index = len(hidden_states) # Decode else: # Create indices from cumulative sequence lengths @@ -642,6 +647,7 @@ def forward( cu_seqlens_q = torch.arange( cu_seqlens.shape[0], dtype=torch.int32, device=hidden_states.device ) + slice_past_index = None # Get rotary cos and sin for this forward # Avoid to index in each layer @@ -651,6 +657,13 @@ def forward( residual = None for i, layer in enumerate(self.layers): + # We added padding that now need to slice + layer_past_key_values = ( + past_key_values[i] + if slice_past_index is None + else past_key_values[i, :slice_past_index] + ) + hidden_states, residual = layer( hidden_states, residual, @@ -658,7 +671,7 @@ def forward( sin, cu_seqlens, max_s, - past_key_values[i], + layer_past_key_values, layer_past_present_indices, cu_seqlens_q, ) @@ -714,10 +727,16 @@ def forward( position_ids, cu_seqlens, max_s, - past_key_values=None, + past_key_values: Optional[torch.Tensor] = None, + pre_allocate_past_size: Optional[int] = None, ): hidden_states, present = self.gpt_neox( - input_ids, position_ids, cu_seqlens, max_s, past_key_values + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values, + pre_allocate_past_size, ) logits = self.embed_out(hidden_states) diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 793b3d11..736f896f 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -5,6 +5,7 @@ from torch import nn from transformers.activations import ACT2FN +from typing import Optional # Flash attention imports import flash_attn_cuda @@ -484,7 +485,8 @@ def forward( position_ids, cu_seqlens, max_s, - past_key_values=None, + past_key_values: Optional[torch.Tensor] = None, + pre_allocate_past_size: Optional[int] = None, ): hidden_states = self.wte(input_ids) + self.wpe(position_ids) if self.tp_embeddings: @@ -496,7 +498,9 @@ def forward( past_key_values = hidden_states.new_empty( ( len(self.h), - len(hidden_states), + len(hidden_states) + if pre_allocate_past_size is None + else pre_allocate_past_size, 2, 1, self.head_size, @@ -504,6 +508,7 @@ def forward( ) layer_past_present_indices = None cu_seqlens_q = None + slice_past_index = len(hidden_states) # Decode else: # Create indices from cumulative sequence lengths @@ -511,15 +516,23 @@ def forward( cu_seqlens_q = torch.arange( cu_seqlens.shape[0], dtype=torch.int32, device=hidden_states.device ) + slice_past_index = None residual = None for i, layer in enumerate(self.h): + # We added padding that now need to slice + layer_past_key_values = ( + past_key_values[i] + if slice_past_index is None + else past_key_values[i, :slice_past_index] + ) + hidden_states, residual = layer( hidden_states, residual, cu_seqlens, max_s, - past_key_values[i], + layer_past_key_values, layer_past_present_indices, cu_seqlens_q, ) @@ -554,10 +567,16 @@ def forward( position_ids, cu_seqlens, max_s, - past_key_values=None, + past_key_values: Optional[torch.Tensor] = None, + pre_allocate_past_size: Optional[int] = None, ): hidden_states, present = self.transformer( - input_ids, position_ids, cu_seqlens, max_s, past_key_values + input_ids, + position_ids, + cu_seqlens, + max_s, + past_key_values, + pre_allocate_past_size, ) logits = self.lm_head(hidden_states) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 0e2fbaab..7be13121 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -142,6 +142,7 @@ def from_pb( all_input_ids_tensor=all_input_ids_tensor, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, + past_pad=None, ) @tracer.start_as_current_span("filter") @@ -188,8 +189,10 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": cu_seqlens.append(cumulative_length + request_input_length) max_seqlen = max(max_seqlen, request_input_length) if not single_request: + # True index for past past_key_values.append(self.past_key_values[2 * idx]) - past_key_values.append(self.past_key_values[1]) + # Add one padding + past_key_values.append(self.past_pad) all_input_ids.append(self.all_input_ids[idx]) all_input_ids_tensor.append(self.all_input_ids_tensor[idx]) @@ -207,7 +210,17 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": # Preallocate tensor for bs = 1 case past_key_values = torch.nn.functional.pad( self.past_key_values[0], - (0, 0, 0, 0, 0, 0, 0, stopping_criterias[0].max_new_tokens - stopping_criterias[0].current_tokens) + ( + 0, + 0, + 0, + 0, + 0, + 0, + 0, + stopping_criterias[0].max_new_tokens + - stopping_criterias[0].current_tokens, + ), ) return FlashCausalLMBatch( @@ -270,10 +283,16 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch # Add cumulative lengths of all previous inputs cu_seqlens.extend([l + cumulative_length for l in batch.cu_seqlens[1:]]) max_seqlen = max(max_seqlen, batch.max_seqlen) + if len(batch) != 1: past_key_values.extend(batch.past_key_values) else: - past_key_values.append(batch.past_key_values[:, :batch.input_lengths[0]]) + # past was pre-allocated for this batch + # We need to slice to remove the padding + past_key_values.append( + batch.past_key_values[:, : batch.input_lengths[0]] + ) + # Add one padding past_key_values.append(batch.past_pad) all_input_ids.extend(batch.all_input_ids) @@ -366,6 +385,7 @@ def forward( cu_seqlens: torch.Tensor, max_s: int, past_key_values: Optional = None, + pre_allocate_past_size: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: # Model Forward return self.model.forward( @@ -374,6 +394,7 @@ def forward( cu_seqlens=cu_seqlens, max_s=max_s, past_key_values=past_key_values, + pre_allocate_past_size=pre_allocate_past_size, ) @tracer.start_as_current_span("generate_token") @@ -382,7 +403,9 @@ def generate_token( ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: # Shortcut when batch_size == 1 if len(batch) == 1: - # No need to slice this down + input_ids = batch.input_ids[0].view(-1) + # Slice to remove extra padding + # past_key_values = batch.past_key_values[:, :batch.input_lengths[0]] if batch.past_key_values is not None else None past_key_values = batch.past_key_values else: # Concatenate tensors @@ -393,6 +416,16 @@ def generate_token( else None ) + # if prefill and bs == 1 + if past_key_values is None and len(batch) == 1: + # Ask to pre-allocate kv to its max size + # == number of tokens + max_new_tokens + pre_allocate_past_size = ( + batch.input_lengths[0] + batch.stopping_criterias[0].max_new_tokens + ) + else: + pre_allocate_past_size = None + # Concatenate when prefill, torch.tensor when decode position_ids = ( torch.tensor(batch.position_ids, device=self.device) @@ -409,21 +442,28 @@ def generate_token( cu_seqlens, batch.max_seqlen, past_key_values, + pre_allocate_past_size, ) # Initialize past_key_values in prefill if batch.past_key_values is None: # Initialize past padding tensor if self.past_pad is None: - self.past_pad = present.new_zeros(present.shape[0], 1, *present.shape[2:]) + self.past_pad = present.new_zeros( + present.shape[0], 1, *present.shape[2:] + ) # Set in batch in case it needs to be used later in concatenate() batch.past_pad = self.past_pad if len(batch) == 1: # Preallocate tensor for bs = 1 case batch.past_key_values = torch.nn.functional.pad( - present, (0, 0, 0, 0, 0, 0, 0, batch.stopping_criterias[0].max_new_tokens) + present, + (0, 0, 0, 0, 0, 0, 0, batch.stopping_criterias[0].max_new_tokens), ) else: + # Add padding after each sequence + # This will have the correct shape after the final past_key_values concatenation before the model + # forward batch.past_key_values = [None, self.past_pad] * len(batch) # Cumulative length @@ -555,6 +595,7 @@ def generate_token( batch.all_input_ids_tensor[i] = all_input_ids_tensor batch.max_seqlen = max(batch.max_seqlen, new_input_length) if len(batch) != 1: + # Add each sequence before its padding batch.past_key_values[i * 2] = present[:, start_index:end_index] # Cumulative sum batch.cu_seqlens[(i + 1)] = batch.cu_seqlens[i] + new_input_length diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 764de2a2..e640113b 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -29,6 +29,7 @@ class FlashLlama(FlashCausalLM): def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + self.past_pad = None if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 @@ -146,6 +147,7 @@ class FlashLlamaSharded(FlashLlama): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): + self.past_pad = None self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 if torch.cuda.is_available(): diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 259fc208..eae584ac 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -33,6 +33,7 @@ class FlashNeoXSharded(FlashNeoX): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): + self.past_pad = None self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 if torch.cuda.is_available(): diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 7dcd8b03..aa1bdfb5 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -28,6 +28,7 @@ class FlashSantacoder(FlashCausalLM): def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + self.past_pad = None if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 @@ -172,6 +173,7 @@ class FlashSantacoderSharded(FlashSantacoder): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): + self.past_pad = None self.process_group, self.rank, self.world_size = initialize_torch_distributed() self.master = self.rank == 0 if torch.cuda.is_available(): From afc5b999d05d9440ec79760aca1f51a7919295ef Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 21 Apr 2023 16:19:04 +0200 Subject: [PATCH 182/793] fix(server): cleanup new flash past_key_values logic (#217) --- .../models/custom_modeling/flash_llama_modeling.py | 2 +- .../models/custom_modeling/flash_neox_modeling.py | 2 +- .../custom_modeling/flash_santacoder_modeling.py | 2 +- .../text_generation_server/models/flash_causal_lm.py | 11 ++--------- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index ee1bd01f..de9b22da 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -594,7 +594,7 @@ def forward( residual = None for i, layer in enumerate(self.layers): - # We added padding that now need to slice + # We added padding that we now need to slice layer_past_key_values = ( past_key_values[i] if slice_past_index is None diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 60545848..cc9b292f 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -657,7 +657,7 @@ def forward( residual = None for i, layer in enumerate(self.layers): - # We added padding that now need to slice + # We added padding that we now need to slice layer_past_key_values = ( past_key_values[i] if slice_past_index is None diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 736f896f..71182f8d 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -520,7 +520,7 @@ def forward( residual = None for i, layer in enumerate(self.h): - # We added padding that now need to slice + # We added padding that we now need to slice layer_past_key_values = ( past_key_values[i] if slice_past_index is None diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 7be13121..7e048b74 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -404,8 +404,7 @@ def generate_token( # Shortcut when batch_size == 1 if len(batch) == 1: input_ids = batch.input_ids[0].view(-1) - # Slice to remove extra padding - # past_key_values = batch.past_key_values[:, :batch.input_lengths[0]] if batch.past_key_values is not None else None + # No need to slice as flash attention will take care of it with cu_seqlens past_key_values = batch.past_key_values else: # Concatenate tensors @@ -454,13 +453,7 @@ def generate_token( ) # Set in batch in case it needs to be used later in concatenate() batch.past_pad = self.past_pad - if len(batch) == 1: - # Preallocate tensor for bs = 1 case - batch.past_key_values = torch.nn.functional.pad( - present, - (0, 0, 0, 0, 0, 0, 0, batch.stopping_criterias[0].max_new_tokens), - ) - else: + if len(batch) != 1: # Add padding after each sequence # This will have the correct shape after the final past_key_values concatenation before the model # forward From 86bca365df0e39dc4a02f3519aee64e9168c1430 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 21 Apr 2023 19:42:16 +0200 Subject: [PATCH 183/793] fix(server): fix flash causal (#218) --- server/text_generation_server/models/flash_causal_lm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 7e048b74..c44dd57d 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -453,7 +453,10 @@ def generate_token( ) # Set in batch in case it needs to be used later in concatenate() batch.past_pad = self.past_pad - if len(batch) != 1: + if len(batch) == 1: + # present is already pre-padded + batch.past_key_values = present + else: # Add padding after each sequence # This will have the correct shape after the final past_key_values concatenation before the model # forward From 1ffea36ec2b1644776ef0dba04cc5c637f13b81a Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 21 Apr 2023 19:49:08 +0200 Subject: [PATCH 184/793] fix(server): fix flash causal (#219) --- server/text_generation_server/models/flash_causal_lm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index c44dd57d..f332ab51 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -225,6 +225,7 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": return FlashCausalLMBatch( batch_id=self.batch_id, + past_pad=self.past_pad, requests=requests, requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, @@ -311,6 +312,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch return FlashCausalLMBatch( batch_id=batches[0].batch_id, + past_pad=batches[0].past_pad, requests=requests, requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, From 4b460e72fb0df7ef451e1f5a48b4cb783d42b9c2 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 21 Apr 2023 20:26:01 +0200 Subject: [PATCH 185/793] fix(server): fix flash batch filtering (#220) --- server/text_generation_server/models/flash_causal_lm.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index f332ab51..9cd9ed89 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -188,9 +188,10 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": position_ids.append(self.position_ids[idx]) cu_seqlens.append(cumulative_length + request_input_length) max_seqlen = max(max_seqlen, request_input_length) + # True index for past + past_key_values.append(self.past_key_values[2 * idx]) + if not single_request: - # True index for past - past_key_values.append(self.past_key_values[2 * idx]) # Add one padding past_key_values.append(self.past_pad) @@ -209,7 +210,7 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": if single_request: # Preallocate tensor for bs = 1 case past_key_values = torch.nn.functional.pad( - self.past_key_values[0], + past_key_values[0], ( 0, 0, From 97df0c7bc0158dfd220056a928341f06ea75522d Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 21 Apr 2023 21:00:30 +0200 Subject: [PATCH 186/793] misc: update to rust 1.69 (#221) --- Dockerfile | 2 +- rust-toolchain.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a679db7e..5e7526c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Rust builder -FROM lukemathwalker/cargo-chef:latest-rust-1.67 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.69 AS chef WORKDIR /usr/src FROM chef as planner diff --git a/rust-toolchain.toml b/rust-toolchain.toml index dcdb06e9..d6ecf0c4 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.67.0" +channel = "1.69.0" components = ["rustfmt", "clippy"] \ No newline at end of file From 6ded76a4ae23f3d0afdcd56e5edc7691cf1b0335 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 21 Apr 2023 21:00:57 +0200 Subject: [PATCH 187/793] v0.6.0 (#222) --- Cargo.lock | 6 +++--- benchmark/Cargo.lock | 2 +- docs/openapi.json | 2 +- launcher/Cargo.toml | 2 +- router/Cargo.toml | 2 +- router/client/Cargo.toml | 2 +- server/pyproject.toml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1d2e3a2..e240fe9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2371,7 +2371,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.5.0" +version = "0.6.0" dependencies = [ "futures", "grpc-metadata", @@ -2388,7 +2388,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.5.0" +version = "0.6.0" dependencies = [ "clap", "ctrlc", @@ -2403,7 +2403,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.5.0" +version = "0.6.0" dependencies = [ "async-stream", "axum", diff --git a/benchmark/Cargo.lock b/benchmark/Cargo.lock index 74610932..81c1180e 100644 --- a/benchmark/Cargo.lock +++ b/benchmark/Cargo.lock @@ -2140,7 +2140,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.5.0" +version = "0.6.0" dependencies = [ "futures", "grpc-metadata", diff --git a/docs/openapi.json b/docs/openapi.json index 1be99f07..e7b7b140 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.5.0" + "version": "0.6.0" }, "paths": { "/": { diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 99d0ed3d..5944a360 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-launcher" -version = "0.5.0" +version = "0.6.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Launcher" diff --git a/router/Cargo.toml b/router/Cargo.toml index aa8e9df2..4fa523a5 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-router" -version = "0.5.0" +version = "0.6.0" edition = "2021" authors = ["Olivier Dehaene"] description = "Text Generation Webserver" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index e8f82cd3..b32402a5 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "text-generation-client" -version = "0.5.0" +version = "0.6.0" edition = "2021" [dependencies] diff --git a/server/pyproject.toml b/server/pyproject.toml index a2969365..5ac22cb2 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.5.0" +version = "0.6.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 4a7dd4085a3de06a1ceee376eedf0122f33e0027 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Mon, 24 Apr 2023 05:15:42 -0700 Subject: [PATCH 188/793] feat(server): reduce memory requirement (#214) --- server/tests/models/test_bloom.py | 23 +- server/tests/models/test_causal_lm.py | 23 +- server/tests/models/test_seq2seq_lm.py | 33 ++- .../models/causal_lm.py | 201 +++++++++++------- .../models/seq2seq_lm.py | 181 +++++++++------- 5 files changed, 287 insertions(+), 174 deletions(-) diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index de0ef57b..47d701eb 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -175,12 +175,14 @@ def test_causal_lm_generate_token_completion_multi( generations[1].generated_text.generated_tokens == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) + # Copy stopping_criterias before filtering + stopping_criterias = default_multi_requests_bloom_batch.stopping_criterias.copy() next_batch = next_batch.filter([next_batch.requests[0]]) for _ in range( - default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens - - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens + stopping_criterias[0].max_new_tokens + - stopping_criterias[1].max_new_tokens - 1 ): generations, next_batch = default_bloom.generate_token(next_batch) @@ -212,6 +214,15 @@ def test_batch_concatenate( next_batch_1 = default_multi_requests_bloom_batch _, next_batch_1 = default_bloom.generate_token(next_batch_1) + # Clone past_key_values before concatenating to compare after, + # because they are removed from the concatenated batches + next_batch_0_past_key_values = [ + (k.clone(), v.clone()) for (k, v) in next_batch_0.past_key_values + ] + next_batch_1_past_key_values = [ + (k.clone(), v.clone()) for (k, v) in next_batch_1.past_key_values + ] + next_batch = BloomCausalLMBatch.concatenate([next_batch_0, next_batch_1]) assert torch.equal(next_batch.all_input_ids[0], next_batch_0.all_input_ids[0]) @@ -246,15 +257,15 @@ def test_batch_concatenate( assert all([p[1].shape == (3, 16, 2, 64) for p in next_batch.past_key_values]) for i, past in enumerate(next_batch.past_key_values): - assert torch.equal(next_batch_0.past_key_values[i][0][:, :, -2:], past[0][0]) + assert torch.equal(next_batch_0_past_key_values[i][0][:, :, -2:], past[0][0]) assert torch.equal( - next_batch_1.past_key_values[i][0][:, :, -1:], + next_batch_1_past_key_values[i][0][:, :, -1:], past[0][1:, :, :, -1].reshape(-1, 64, 1), ) - assert torch.equal(next_batch_0.past_key_values[i][1][:, -2:, :], past[1][0]) + assert torch.equal(next_batch_0_past_key_values[i][1][:, -2:, :], past[1][0]) assert torch.equal( - next_batch_1.past_key_values[i][1][:, -1:, :], + next_batch_1_past_key_values[i][1][:, -1:, :], past[1][1:, :, -1, :].reshape(-1, 1, 64), ) diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index ad79a4ca..03d3ef9b 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -173,12 +173,14 @@ def test_causal_lm_generate_token_completion_multi( generations[1].generated_text.generated_tokens == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) + # Copy stopping_criterias before filtering + stopping_criterias = default_multi_requests_causal_lm_batch.stopping_criterias.copy() next_batch = next_batch.filter([next_batch.requests[0]]) for _ in range( - default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens - - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens + stopping_criterias[0].max_new_tokens + - stopping_criterias[1].max_new_tokens - 1 ): generations, next_batch = default_causal_lm.generate_token(next_batch) @@ -209,6 +211,15 @@ def test_batch_concatenate( next_batch_1 = default_multi_requests_causal_lm_batch _, next_batch_1 = default_causal_lm.generate_token(next_batch_1) + # Clone past_key_values before concatenating to compare after, + # because they are removed from the concatenated batches + next_batch_0_past_key_values = [ + (k.clone(), v.clone()) for (k, v) in next_batch_0.past_key_values + ] + next_batch_1_past_key_values = [ + (k.clone(), v.clone()) for (k, v) in next_batch_1.past_key_values + ] + next_batch = CausalLMBatch.concatenate([next_batch_0, next_batch_1]) assert torch.equal(next_batch.all_input_ids[0], next_batch_0.all_input_ids[0]) @@ -244,14 +255,14 @@ def test_batch_concatenate( assert all([p[1].shape == (3, 12, 2, 64) for p in next_batch.past_key_values]) for i, past in enumerate(next_batch.past_key_values): - assert torch.equal(next_batch_0.past_key_values[i][0][0, :, -2:], past[0][0]) + assert torch.equal(next_batch_0_past_key_values[i][0][0, :, -2:], past[0][0]) assert torch.equal( - next_batch_1.past_key_values[i][0][:, :, -1:], past[0][1:, :, -1:, :] + next_batch_1_past_key_values[i][0][:, :, -1:], past[0][1:, :, -1:, :] ) - assert torch.equal(next_batch_0.past_key_values[i][1][0, :, -2:], past[1][0]) + assert torch.equal(next_batch_0_past_key_values[i][1][0, :, -2:], past[1][0]) assert torch.equal( - next_batch_1.past_key_values[i][1][:, :, -1:], past[1][1:, :, -1:, :] + next_batch_1_past_key_values[i][1][:, :, -1:], past[1][1:, :, -1:, :] ) for _ in range( diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 79c9e936..65dafa50 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -219,6 +219,19 @@ def test_batch_concatenate( next_batch_1 = default_multi_requests_seq2seq_lm_batch _, next_batch_1 = default_seq2seq_lm.generate_token(next_batch_1) + # Copy hidden state because it is removed from the concatenated branches + next_batch_0_encoder_last_hidden_state = next_batch_0.encoder_last_hidden_state + next_batch_1_encoder_last_hidden_state = next_batch_1.encoder_last_hidden_state + + # Clone past_key_values before concatenating to compare after, + # because they are removed from the concatenated batches + next_batch_0_past_key_values = [ + [t.clone() for t in layer] for layer in next_batch_0.past_key_values + ] + next_batch_1_past_key_values = [ + [t.clone() for t in layer] for layer in next_batch_1.past_key_values + ] + next_batch = Seq2SeqLMBatch.concatenate([next_batch_0, next_batch_1]) assert next_batch.batch_id == 0 @@ -239,11 +252,11 @@ def test_batch_concatenate( assert torch.equal( next_batch.encoder_last_hidden_state[0], - next_batch_0.encoder_last_hidden_state[0, -2:], + next_batch_0_encoder_last_hidden_state[0, -2:], ) assert torch.equal( next_batch.encoder_last_hidden_state[1:], - next_batch_1.encoder_last_hidden_state[:, -2:], + next_batch_1_encoder_last_hidden_state[:, -2:], ) assert next_batch.input_lengths == [2, 2, 2] @@ -275,24 +288,24 @@ def test_batch_concatenate( ) for i, past in enumerate(next_batch.past_key_values): - assert torch.equal(next_batch_0.past_key_values[i][0][0, :, -2:, :], past[0][0]) + assert torch.equal(next_batch_0_past_key_values[i][0][0, :, -2:, :], past[0][0]) assert torch.equal( - next_batch_1.past_key_values[i][0][:, :, -1:, :], past[0][1:, :, -1:, :] + next_batch_1_past_key_values[i][0][:, :, -1:, :], past[0][1:, :, -1:, :] ) - assert torch.equal(next_batch_0.past_key_values[i][1][0, :, -2:, :], past[1][0]) + assert torch.equal(next_batch_0_past_key_values[i][1][0, :, -2:, :], past[1][0]) assert torch.equal( - next_batch_1.past_key_values[i][1][:, :, -1:, :], past[1][1:, :, -1:, :] + next_batch_1_past_key_values[i][1][:, :, -1:, :], past[1][1:, :, -1:, :] ) - assert torch.equal(next_batch_0.past_key_values[i][2][0, :, -2:, :], past[2][0]) + assert torch.equal(next_batch_0_past_key_values[i][2][0, :, -2:, :], past[2][0]) assert torch.equal( - next_batch_1.past_key_values[i][2][:, :, -2:, :], past[2][1:] + next_batch_1_past_key_values[i][2][:, :, -2:, :], past[2][1:] ) - assert torch.equal(next_batch_0.past_key_values[i][3][0, :, -2:, :], past[3][0]) + assert torch.equal(next_batch_0_past_key_values[i][3][0, :, -2:, :], past[3][0]) assert torch.equal( - next_batch_1.past_key_values[i][3][:, :, -2:, :], past[3][1:] + next_batch_1_past_key_values[i][3][:, :, -2:, :], past[3][1:] ) for _ in range(3): diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 98313253..1db5abce 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -150,6 +150,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc next_token_choosers = [] stopping_criterias = [] + new_padding_right_offset = 0 + for i, r in enumerate(requests): idx = self.requests_idx_mapping[r.id] requests_idx_mapping[r.id] = i @@ -164,36 +166,57 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc max_input_length = max(max_input_length, request_input_length) next_token_choosers.append(self.next_token_choosers[idx]) - stopping_criterias.append(self.stopping_criterias[idx]) + stopping_criteria = self.stopping_criterias[idx] + stopping_criterias.append(stopping_criteria) + + new_padding_right_offset = max( + new_padding_right_offset, + stopping_criteria.max_new_tokens - stopping_criteria.current_tokens + ) # Apply indices to input_ids, attention mask, past key values and other items that need to be cached input_ids = self.input_ids[keep_indices] - attention_mask = self.attention_mask[keep_indices] position_ids = self.position_ids[keep_indices] - # Force past to be of dim [self_size, num_heads, ...] for easy indexing - past_key_values = [ - [t.view(len(self), -1, *t.shape[-2:])[keep_indices] for t in layer] - for layer in self.past_key_values + self.attention_mask = self.attention_mask[ + keep_indices, + -(self.padding_right_offset + max_input_length): + (self.attention_mask.shape[1] - self.padding_right_offset) + new_padding_right_offset, ] - return CausalLMBatch( - batch_id=self.batch_id, - requests=requests, - requests_idx_mapping=requests_idx_mapping, - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - all_input_ids=all_input_ids, - input_lengths=input_lengths, - offsets=offsets, - token_offsets=token_offsets, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - max_input_length=max_input_length, - padding_right_offset=self.padding_right_offset, - keys_head_dim_last=self.keys_head_dim_last, - ) + # Ensure that past_key_values tensors can be updated in-place + if type(self.past_key_values[0]) == tuple: + self.past_key_values = [list(layer) for layer in self.past_key_values] + + # Update tensors in-place to allow incremental garbage collection + past_kv_length = max_input_length - 1 + for layer in self.past_key_values: + past_keys, past_values = layer + if len(past_keys.shape) == 3: + # Force past to be of dim [self_size, num_heads, ...] for easy indexing + past_keys = past_keys.view(len(self), -1, *past_keys.shape[-2:]) + past_values = past_values.view(len(self), -1, *past_values.shape[-2:]) + if self.keys_head_dim_last: + layer[0] = past_keys[keep_indices, :, -past_kv_length:, :] + else: + layer[0] = past_keys[keep_indices, :, :, -past_kv_length:] + del past_keys + layer[1] = past_values[keep_indices, :, -past_kv_length:, :] + del past_values + + self.requests = requests + self.requests_idx_mapping = requests_idx_mapping + self.input_ids = input_ids + self.position_ids = position_ids + self.all_input_ids = all_input_ids + self.input_lengths = input_lengths + self.offsets = offsets + self.token_offsets = token_offsets + self.next_token_choosers = next_token_choosers + self.stopping_criterias = stopping_criterias + self.max_input_length = max_input_length + self.padding_right_offset = new_padding_right_offset + + return self @classmethod @tracer.start_as_current_span("concatenate") @@ -285,62 +308,88 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": position_ids = batch.position_ids.new_empty((total_batch_size, 1)) position_ids[start_index:end_index] = batch.position_ids - for j, past in enumerate(batch.past_key_values): - past_keys, past_values = past - - # Shenanigans to get dimensions because BLOOM outputs a past with a different shape - # BLOOM Keys: [batch_size * num_heads, head_dim, seq_length] - # BLOOM Values: [batch_size * num_heads, seq_length, head_dim] - past_keys = past_keys.view(len(batch), -1, *past_keys.shape[-2:]) - past_values = past_values.view(len(batch), -1, *past_values.shape[-2:]) - - _, num_heads, padded_sequence_length, head_dim = past_values.shape + # Shenanigans to get dimensions because BLOOM outputs a past with a different shape + # BLOOM Keys: [batch_size * num_heads, head_dim, seq_length] + # BLOOM Values: [batch_size * num_heads, seq_length, head_dim] + # And ensure that we can update tensors in-place + if type(batch.past_key_values[0]) == tuple: + batch.past_key_values = [ + [t.view(len(batch), -1, *t.shape[-2:]) for t in layer] for layer in batch.past_key_values + ] + elif batch.past_key_values[0][0].shape == 3: + for layer in batch.past_key_values: + for k, t in enumerate(layer): + layer[k] = t.view(len(batch), -1, *t.shape[-2:]) + + start_index = end_index + + first_past_kvs = batches[0].past_key_values + _, num_heads, padded_sequence_length, head_dim = first_past_kvs[0][1].shape + + padded_past_values_shape = ( + total_batch_size, + num_heads, + max_input_length - 1, + head_dim, + ) - padded_past_values_shape = ( - total_batch_size, - num_heads, - max_input_length - 1, - head_dim, - ) + if batches[0].keys_head_dim_last: + padded_past_keys_shape = padded_past_values_shape + else: + # seq_length is last for BLOOM + padded_past_keys_shape = ( + total_batch_size, + num_heads, + head_dim, + max_input_length - 1, + ) + # Iterate over attention layers + # Concatenate past key values layer by layer to allow incremental garbage collection + for j in range(len(first_past_kvs)): + padded_past_keys = first_past_kvs[j][0].new_zeros(padded_past_keys_shape) + start_index = 0 + for batch in batches: + past_keys = batch.past_key_values[j][0] + # Clear reference to the original tensor + batch.past_key_values[j][0] = None + + # Slicing end index for this batch + end_index = start_index + len(batch) + # We slice the keys to remove the padding from previous batches + past_seq_len = batch.max_input_length - 1 if batch.keys_head_dim_last: - padded_past_keys_shape = padded_past_values_shape - else: - # seq_length is last for BLOOM - padded_past_keys_shape = ( - total_batch_size, - num_heads, - head_dim, - max_input_length - 1, - ) - - # This will run only once per layer - if j == len(past_key_values): - padded_past_keys = past_keys.new_zeros(padded_past_keys_shape) - padded_past_values = past_values.new_zeros(padded_past_values_shape) - past_key_values.append((padded_past_keys, padded_past_values)) - - # We slice the past keys and values to remove the padding from previous batches - if batch.keys_head_dim_last: - past_key_values[j][0][ - start_index:end_index, - :, - -(batch.max_input_length - 1) :, - :, - ] = past_keys[:, :, -(batch.max_input_length - 1) :, :] + padded_past_keys[ + start_index:end_index, :, -past_seq_len:, : + ] = past_keys[:, :, -past_seq_len:, :] else: - past_key_values[j][0][ - start_index:end_index, - :, - :, - -(batch.max_input_length - 1) :, - ] = past_keys[:, :, :, -(batch.max_input_length - 1) :] - - past_key_values[j][1][ - start_index:end_index, :, -(batch.max_input_length - 1) :, : - ] = past_values[:, :, -(batch.max_input_length - 1) :, :] - - start_index += len(batch) + # BLOOM case + padded_past_keys[ + start_index:end_index, :, :, -past_seq_len: + ] = past_keys[:, :, :, -past_seq_len:] + del past_keys + + start_index = end_index + + padded_past_values = first_past_kvs[j][1].new_zeros(padded_past_values_shape) + start_index = 0 + for batch in batches: + past_values = batch.past_key_values[j][1] + # Clear reference to the original tensor + batch.past_key_values[j][1] = None + + # Slicing end index for this batch + end_index = start_index + len(batch) + # We slice the past values to remove the padding from previous batches + past_seq_len = batch.max_input_length - 1 + padded_past_values[ + start_index:end_index, :, -past_seq_len:, : + ] = past_values[:, :, -past_seq_len:, :] + del past_values + + start_index = end_index + + past_key_values.append([padded_past_keys, padded_past_values]) return cls( batch_id=batches[0].batch_id, diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index aa452c70..2252fcfc 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -25,7 +25,7 @@ class Seq2SeqLMBatch(Batch): requests_idx_mapping: Dict[int, int] # Encoder values - input_ids: torch.Tensor + input_ids: Optional[torch.Tensor] attention_mask: torch.Tensor # Decoder values @@ -164,6 +164,7 @@ def filter( max_input_length = 0 max_decoder_input_length = 0 + padding_right_offset = 0 for i, r in enumerate(requests): idx = self.requests_idx_mapping[r.id] @@ -184,45 +185,53 @@ def filter( max_decoder_input_length = max( max_decoder_input_length, request_decoder_input_length ) + padding_right_offset = max( + padding_right_offset, + self.stopping_criterias[idx].max_new_tokens - self.stopping_criterias[idx].current_tokens + ) next_token_choosers.append(self.next_token_choosers[idx]) stopping_criterias.append(self.stopping_criterias[idx]) # Apply indices to input_ids, attention mask, past key values and other items that need to be cached - decoder_input_ids = self.decoder_input_ids[keep_indices] - attention_mask = self.attention_mask[keep_indices] + self.decoder_input_ids = self.decoder_input_ids[keep_indices] + self.attention_mask = self.attention_mask[keep_indices, -max_input_length:] if self.decoder_attention_mask is not None: - decoder_attention_mask = self.decoder_attention_mask[keep_indices] - else: - decoder_attention_mask = None + self.decoder_attention_mask = self.decoder_attention_mask[ + keep_indices, + -(self.padding_right_offset + max_decoder_input_length): + (self.decoder_attention_mask.shape[1] - self.padding_right_offset) + padding_right_offset, + ] - encoder_last_hidden_state = self.encoder_last_hidden_state[keep_indices] + self.encoder_last_hidden_state = self.encoder_last_hidden_state[keep_indices, -max_input_length:] - past_key_values = [ - [t[keep_indices] for t in layer] for layer in self.past_key_values - ] + # Ensure that past_key_values tensors can be updated in-place + if type(self.past_key_values[0]) == tuple: + self.past_key_values = [[t for t in layer] for layer in self.past_key_values] + + decoder_past_seq_len = max_decoder_input_length - 1 + for layer in self.past_key_values: + layer[0] = layer[0][keep_indices, :, -decoder_past_seq_len:] + layer[1] = layer[1][keep_indices, :, -decoder_past_seq_len:] + layer[2] = layer[2][keep_indices, :, -max_input_length:] + layer[3] = layer[3][keep_indices, :, -max_input_length:] + + self.requests = requests + self.requests_idx_mapping = requests_idx_mapping + self.input_ids = None + self.all_decoder_input_ids = all_decoder_input_ids + self.input_lengths = input_lengths + self.decoder_input_lengths = decoder_input_lengths + self.offsets = offsets + self.token_offsets = token_offsets + self.next_token_choosers = next_token_choosers + self.stopping_criterias = stopping_criterias + self.max_input_length = max_input_length + self.max_decoder_input_length = max_decoder_input_length + self.padding_right_offset = padding_right_offset + + return self - return Seq2SeqLMBatch( - batch_id=self.batch_id, - requests=requests, - requests_idx_mapping=requests_idx_mapping, - input_ids=None, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - all_decoder_input_ids=all_decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - encoder_last_hidden_state=encoder_last_hidden_state, - past_key_values=past_key_values, - input_lengths=input_lengths, - decoder_input_lengths=decoder_input_lengths, - offsets=offsets, - token_offsets=token_offsets, - next_token_choosers=next_token_choosers, - stopping_criterias=stopping_criterias, - max_input_length=max_input_length, - max_decoder_input_length=max_decoder_input_length, - padding_right_offset=self.padding_right_offset, - ) @classmethod @tracer.start_as_current_span("concatenate") @@ -350,58 +359,78 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": encoder_last_hidden_state[ start_index:end_index, -batch.max_input_length :, : ] = batch.encoder_last_hidden_state[:, -batch.max_input_length :, :] + batch.encoder_last_hidden_state = None - # Iterate over attention layers - for j, past in enumerate(batch.past_key_values): - _, num_heads, _, head_dim = past[0].shape - - # This will run only once per layer - if j == len(past_key_values): - past_key_values.append([]) + # Ensure that we can update tensors in-place + if type(batch.past_key_values[0]) == tuple: + batch.past_key_values = [[t for t in layer] for layer in batch.past_key_values] - # Decoder past - for k, t in enumerate(past[:2]): - padded_t_shape = ( - total_batch_size, - num_heads, - (max_decoder_input_length - 1), - head_dim, - ) - - # Initialize tensors - # This will run only once per layer and per past tensor - if k == len(past_key_values[j]): - past_key_values[j].append(t.new_zeros(padded_t_shape)) + start_index = end_index - # We slice the past keys and values to remove the padding from previous batches - past_key_values[j][k][ - start_index:end_index, - :, - -(batch.max_decoder_input_length - 1) :, - :, - ] = t[:, :, -(batch.max_decoder_input_length - 1) :, :] - - # encoder past - for k, t in enumerate(past[2:]): - padded_t_shape = ( - total_batch_size, - num_heads, - max_input_length, - head_dim, - ) + # Determine shapes for new past kv tensors + first_past_kvs = batches[0].past_key_values + _, num_heads, _, head_dim = first_past_kvs[0][0].shape - idx = k + 2 + padded_dec_t_shape = ( + total_batch_size, + num_heads, + (max_decoder_input_length - 1), + head_dim, + ) - # Initialize tensors - # This will run only once per layer and per past tensor - if idx == len(past_key_values[j]): - past_key_values[j].append(t.new_zeros(padded_t_shape)) + padded_enc_t_shape = ( + total_batch_size, + num_heads, + max_input_length, + head_dim, + ) - past_key_values[j][idx][ - start_index:end_index, :, -batch.max_input_length :, : - ] = t[:, :, -batch.max_input_length :, :] + # Iterate over attention layers + for j in range(len(first_past_kvs)): + past_key_values.append([]) + + # Decoder past + for k in range(0, 2): + # Initialize tensors + padded_past_values = first_past_kvs[j][k].new_zeros(padded_dec_t_shape) + past_key_values[j].append(padded_past_values) + + start_index = 0 + for batch in batches: + t = batch.past_key_values[j][k] + # Clear reference to the original tensor + batch.past_key_values[j][k] = None + # Slicing end index for this batch + end_index = start_index + len(batch) + # We slice the past keys and values to remove the padding from previous batches + past_seq_len = batch.max_decoder_input_length - 1 + padded_past_values[ + start_index:end_index, :, -past_seq_len:, : + ] = t[:, :, -past_seq_len:, :] + del t + + start_index = end_index + + # Encoder past + for k in range(2, 4): + # Initialize tensors + padded_past_values = first_past_kvs[j][k].new_zeros(padded_enc_t_shape) + past_key_values[j].append(padded_past_values) + + start_index = 0 + for batch in batches: + t = batch.past_key_values[j][k] + # Clear reference to the original tensor + batch.past_key_values[j][k] = None + # Slicing end index for this batch + end_index = start_index + len(batch) + # We slice the past keys and values to remove the padding from previous batches + padded_past_values[ + start_index:end_index, :, -batch.max_input_length:, : + ] = t[:, :, -batch.max_input_length:, :] + del t - start_index += len(batch) + start_index = end_index return cls( batch_id=batches[0].batch_id, From 98a3e0d1353f7d6f71bd59d7887ad7862c3f7305 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 24 Apr 2023 15:57:13 +0200 Subject: [PATCH 189/793] chore(server): update huggingface-hub (#227) --- Dockerfile | 5 --- server/poetry.lock | 80 ++++++++++++++++++++--------------------- server/pyproject.toml | 2 +- server/requirements.txt | 4 +-- 4 files changed, 41 insertions(+), 50 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5e7526c9..c17a9414 100644 --- a/Dockerfile +++ b/Dockerfile @@ -144,11 +144,6 @@ COPY --from=transformers-builder /usr/src/transformers/build/lib.linux-x86_64-cp # Install transformers dependencies RUN cd /usr/src/transformers && pip install -e . --no-cache-dir && pip install einops --no-cache-dir -# FIXME: remove when we get a release of huggingface-hub -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - git \ - && rm -rf /var/lib/apt/lists/* - # Install server COPY proto proto COPY server server diff --git a/server/poetry.lock b/server/poetry.lock index b13441de..4b9855c5 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -241,12 +241,11 @@ python-versions = ">=3.7" [[package]] name = "huggingface-hub" -version = "0.14.0.dev0" +version = "0.14.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" category = "main" optional = false python-versions = ">=3.7.0" -develop = false [package.dependencies] filelock = "*" @@ -268,12 +267,6 @@ testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "gradio", "jedi", "pytest torch = ["torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] -[package.source] -type = "git" -url = "https://github.com/huggingface/huggingface_hub.git" -reference = "4f27b44ee536cd654e171c7f37478eaf1996cc3f" -resolved_reference = "4f27b44ee536cd654e171c7f37478eaf1996cc3f" - [[package]] name = "idna" version = "3.4" @@ -366,7 +359,7 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "numpy" -version = "1.24.2" +version = "1.24.3" description = "Fundamental package for array computing in Python" category = "main" optional = true @@ -735,7 +728,7 @@ python-versions = "*" [[package]] name = "setuptools" -version = "67.6.1" +version = "67.7.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false @@ -918,7 +911,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "2c154f7400d221a13ca4ff5dcab4148be17ae2c22493d8aa57c1a0d2f9361f14" +content-hash = "2534d6a2dea05e0061707b2c0ede295902790656a3c6256fef692e06d66a0c4a" [metadata.files] accelerate = [ @@ -1188,7 +1181,10 @@ hf-transfer = [ {file = "hf_transfer-0.1.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"}, {file = "hf_transfer-0.1.3.tar.gz", hash = "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c"}, ] -huggingface-hub = [] +huggingface-hub = [ + {file = "huggingface_hub-0.14.0-py3-none-any.whl", hash = "sha256:fa6a6139fe4a8a164bfd0cda90c225fe8471b47c12811738b6db8348a2f703a0"}, + {file = "huggingface_hub-0.14.0.tar.gz", hash = "sha256:42eeab833284e3fc1d39263cf9c3d1bb36b129acdd8195838694d165e8dd6cae"}, +] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, @@ -1269,34 +1265,34 @@ networkx = [ {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, ] numpy = [ - {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, - {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, - {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, - {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, - {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, - {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, - {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, - {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, - {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, - {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, - {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, + {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, + {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, + {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, + {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, + {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, + {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, + {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, + {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, + {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, ] nvidia-cublas-cu11 = [ {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, @@ -1558,8 +1554,8 @@ sentencepiece = [ {file = "sentencepiece-0.1.98.tar.gz", hash = "sha256:947cf0a4b8a480510d560a922f8256f34e93984a86cf870be4d05731f59fb28d"}, ] setuptools = [ - {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"}, - {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"}, + {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, + {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, ] sympy = [ {file = "sympy-1.11.1-py3-none-any.whl", hash = "sha256:938f984ee2b1e8eae8a07b884c8b7a1146010040fccddc6539c54f401c8f6fcf"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 5ac22cb2..3998a062 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -25,7 +25,7 @@ opentelemetry-instrumentation-grpc = "^0.36b0" hf-transfer = "^0.1.2" sentencepiece = "^0.1.97" tokenizers = "0.13.3" -huggingface-hub = {git = "https://github.com/huggingface/huggingface_hub.git", rev = "4f27b44ee536cd654e171c7f37478eaf1996cc3f"} +huggingface-hub = "0.14.0" [tool.poetry.extras] accelerate = ["accelerate"] diff --git a/server/requirements.txt b/server/requirements.txt index e2dedb07..f4fa7d7e 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -13,7 +13,7 @@ grpcio-reflection==1.54.0 ; python_version >= "3.9" and python_version < "4.0" grpcio-status==1.54.0 ; python_version >= "3.9" and python_version < "4.0" grpcio==1.54.0 ; python_version >= "3.9" and python_version < "4.0" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" -huggingface-hub @ git+https://github.com/huggingface/huggingface_hub.git@4f27b44ee536cd654e171c7f37478eaf1996cc3f ; python_version >= "3.9" and python_version < "4.0" +huggingface-hub==0.14.0 ; python_version >= "3.9" and python_version < "4.0" idna==3.4 ; python_version >= "3.9" and python_version < "4" loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" @@ -31,7 +31,7 @@ pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" requests==2.28.2 ; python_version >= "3.9" and python_version < "4" safetensors==0.2.8 ; python_version >= "3.9" and python_version < "4.0" sentencepiece==0.1.98 ; python_version >= "3.9" and python_version < "4.0" -setuptools==67.6.1 ; python_version >= "3.9" and python_version < "4.0" +setuptools==67.7.2 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" From ebc74d5666bf2a304181a5fe8cc7dfc6ddbe1d95 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 24 Apr 2023 17:59:00 +0200 Subject: [PATCH 190/793] feat(router): use number of tokens in batch as input for dynamic batching (#226) Co-authored-by: Nick Hill --- launcher/src/main.rs | 23 ++- proto/generate.proto | 17 ++ router/client/src/client.rs | 16 ++ router/client/src/sharded_client.rs | 18 +- router/src/infer.rs | 170 +++++++++--------- router/src/main.rs | 19 +- router/src/queue.rs | 117 +++++++----- router/src/server.rs | 9 +- router/src/validation.rs | 15 +- server/tests/models/test_bloom.py | 4 +- server/tests/models/test_causal_lm.py | 8 +- .../models/causal_lm.py | 41 ++++- .../models/flash_causal_lm.py | 28 ++- .../models/galactica.py | 5 + .../models/seq2seq_lm.py | 70 ++++++-- server/text_generation_server/server.py | 13 +- 16 files changed, 400 insertions(+), 173 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index fcac736d..7450b3f4 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -39,8 +39,12 @@ struct Args { max_input_length: usize, #[clap(default_value = "1512", long, env)] max_total_tokens: usize, - #[clap(default_value = "32", long, env)] - max_batch_size: usize, + #[clap(long, env)] + max_batch_size: Option, + #[clap(default_value = "1.2", long, env)] + waiting_served_ratio: f32, + #[clap(default_value = "32000", long, env)] + max_batch_total_tokens: u32, #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, #[clap(default_value = "3000", long, short, env)] @@ -93,6 +97,8 @@ fn main() -> ExitCode { max_input_length, max_total_tokens, max_batch_size, + max_batch_total_tokens, + waiting_served_ratio, max_waiting_tokens, port, shard_uds_path, @@ -380,8 +386,8 @@ fn main() -> ExitCode { max_input_length.to_string(), "--max-total-tokens".to_string(), max_total_tokens.to_string(), - "--max-batch-size".to_string(), - max_batch_size.to_string(), + "--waiting-served-ratio".to_string(), + waiting_served_ratio.to_string(), "--max-waiting-tokens".to_string(), max_waiting_tokens.to_string(), "--port".to_string(), @@ -392,6 +398,15 @@ fn main() -> ExitCode { model_id, ]; + // Deprecate max_batch_size + if let Some(max_batch_size) = max_batch_size { + argv.push("--max-batch-size".to_string()); + argv.push(max_batch_size.to_string()) + } else { + argv.push("--max-batch-total-tokens".to_string()); + argv.push(max_batch_total_tokens.to_string()) + } + // Model optional revision if let Some(ref revision) = revision { argv.push("--revision".to_string()); diff --git a/proto/generate.proto b/proto/generate.proto index 2bf7385c..ad47409e 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -9,6 +9,8 @@ service TextGenerationService { rpc ServiceDiscovery (ServiceDiscoveryRequest) returns (ServiceDiscoveryResponse) {} /// Empties batch cache rpc ClearCache (ClearCacheRequest) returns (ClearCacheResponse); + /// Remove requests from a cached batch + rpc FilterBatch (FilterBatchRequest) returns (FilterBatchResponse); /// Prefill batch and decode first token rpc Prefill (PrefillRequest) returns (PrefillResponse); /// Decode token for a list of prefilled batches @@ -89,6 +91,8 @@ message Batch { repeated Request requests = 2; /// Batch size (==len(requests)) uint32 size = 3; + /// Maximum number of tokens this batch will grow to + uint32 max_tokens = 4; } enum FinishReason { @@ -134,6 +138,19 @@ message Generation { GeneratedText generated_text = 7; } +message FilterBatchRequest { + /// Batch ID + uint64 batch_id = 1; + /// Requests to keep + repeated Request keep_requests = 2; +} + +message FilterBatchResponse { + /// Filtered Batch (cached) + Batch batch = 1; +} + + message PrefillRequest { /// Batch Batch batch = 1; diff --git a/router/client/src/client.rs b/router/client/src/client.rs index cccd5004..7cadf430 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -70,6 +70,22 @@ impl Client { Ok(()) } + /// Filter a cached batch + #[instrument(skip(self))] + pub async fn filter_batch( + &mut self, + batch_id: u64, + keep_requests: Vec, + ) -> Result> { + let request = tonic::Request::new(FilterBatchRequest { + batch_id, + keep_requests, + }) + .inject_context(); + let filtered_batch = self.stub.filter_batch(request).await?.into_inner(); + Ok(filtered_batch.batch) + } + /// Generate one token for each request in the given batch /// /// Returns Generation for each request in batch diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 903c7a67..469d75f6 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,6 +1,6 @@ /// Multi shard Client use crate::Result; -use crate::{Batch, Client, Generation, ShardInfo}; +use crate::{Batch, Client, Generation, Request, ShardInfo}; use futures::future::join_all; use tonic::transport::Uri; use tracing::instrument; @@ -59,6 +59,22 @@ impl ShardedClient { join_all(futures).await.into_iter().collect() } + /// Filter a cached batch + #[instrument(skip(self))] + pub async fn filter_batch( + &mut self, + batch_id: u64, + keep_requests: Vec, + ) -> Result> { + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| Box::pin(client.filter_batch(batch_id, keep_requests.clone()))) + .collect(); + // all shards return the same message + join_all(futures).await.pop().unwrap() + } + /// Generate one token for each request in the given batch /// /// Returns Generation for each request in batch diff --git a/router/src/infer.rs b/router/src/infer.rs index 484720a0..8b44ec86 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -39,12 +39,14 @@ impl Infer { pub(crate) fn new( client: ShardedClient, validation: Validation, - max_batch_size: usize, + waiting_served_ratio: f32, + max_batch_total_tokens: u32, max_waiting_tokens: usize, max_concurrent_requests: usize, + requires_padding: bool, ) -> Self { // Infer shared state - let queue = Queue::new(); + let queue = Queue::new(requires_padding); let shared = Arc::new(Shared { batching_task: Notify::new(), }); @@ -52,7 +54,8 @@ impl Infer { // Spawn batching background task that contains all the inference logic tokio::spawn(batching_task( client, - max_batch_size, + waiting_served_ratio, + max_batch_total_tokens, max_waiting_tokens, queue.clone(), shared.clone(), @@ -232,18 +235,12 @@ impl Infer { /// Batches requests and sends them to the inference server async fn batching_task( mut client: ShardedClient, - max_batch_size: usize, + waiting_served_ratio: f32, + max_batch_total_tokens: u32, max_waiting_tokens: usize, queue: Queue, shared: Arc, ) { - // Minimum batch size after which we try to add more requests - let limit_min_batch_size = if max_batch_size > 1 { - (max_batch_size / 2) as u32 - } else { - 0 - }; - // Infinite loop loop { // Wait for a notification from the Infer struct @@ -252,7 +249,9 @@ async fn batching_task( // Get the next batch from the queue // This batch might be smaller than the maximum batch size if there are not enough requests // waiting in the queue - while let Some((mut entries, batch, span)) = queue.next_batch(None, max_batch_size).await { + while let Some((mut entries, batch, span)) = + queue.next_batch(None, max_batch_total_tokens).await + { let mut cached_batch = prefill(&mut client, batch, &mut entries) .instrument(span) .await; @@ -263,48 +262,57 @@ async fn batching_task( while let Some(batch) = cached_batch { // Get current batch info let batch_size = batch.size; + let batch_max_tokens = batch.max_tokens; let mut batches = vec![batch]; metrics::gauge!("tgi_batch_current_size", batch_size as f64); + metrics::gauge!("tgi_batch_current_max_tokens", batch_max_tokens as f64); + + let min_size = if waiting_tokens >= max_waiting_tokens { + // If we didn't onboard any new requests since >= max_waiting_tokens, we try + // to add a new batch even though its size might be small + None + } else { + // Minimum batch size + Some((batch_size as f32 * waiting_served_ratio).floor() as usize) + }; + + let token_budget = max_batch_total_tokens - batch_max_tokens; + + // Try to get a new batch + if let Some((mut new_entries, new_batch, span)) = + queue.next_batch(min_size, token_budget).await + { + // Tracking metrics + if min_size.is_some() { + metrics::increment_counter!("tgi_batch_concat", "reason" => "backpressure"); + } else { + metrics::increment_counter!("tgi_batch_concat", "reason" => "wait_exceeded"); + } - // If the current batch is too small, we try to add more requests to it - if batch_size <= limit_min_batch_size { - let min_size = match waiting_tokens { - // If we didn't onboard any new requests since >= max_waiting_tokens, we try - // to add a new batch even though its size might be small - _ if waiting_tokens >= max_waiting_tokens => None, - // Minimum size criteria - _ => Some(limit_min_batch_size as usize), - }; - - // Try to get a new batch - if let Some((mut new_entries, new_batch, span)) = queue - .next_batch(min_size, max_batch_size - batch_size as usize) - .await - { - entries.iter_mut().for_each(|(_, entry)| { - // Create a new span to add the info that this entry is waiting - // because a new batch is being computed - let entry_waiting_span = info_span!(parent: &entry.span, "waiting"); - // Add relationships - span.follows_from(&entry_waiting_span); - entry_waiting_span.follows_from(&span); - // Update entry - entry.temp_span = Some(entry_waiting_span); - }); - - // Generate one token for this new batch to have the attention past in cache - let new_cached_batch = prefill(&mut client, new_batch, &mut new_entries) - .instrument(span) - .await; - // Reset waiting counter - waiting_tokens = 1; - // Extend current batch with the new batch - if let Some(new_cached_batch) = new_cached_batch { - entries.extend(new_entries); - batches.push(new_cached_batch); - } + entries.iter_mut().for_each(|(_, entry)| { + // Create a new span to add the info that this entry is waiting + // because a new batch is being computed + let entry_waiting_span = info_span!(parent: &entry.span, "waiting"); + // Add relationships + span.follows_from(&entry_waiting_span); + entry_waiting_span.follows_from(&span); + // Update entry + entry.temp_span = Some(entry_waiting_span); + }); + + // Generate one token for this new batch to have the attention past in cache + let new_cached_batch = prefill(&mut client, new_batch, &mut new_entries) + .instrument(span) + .await; + // Reset waiting counter + waiting_tokens = 1; + // Extend current batch with the new batch + if let Some(new_cached_batch) = new_cached_batch { + entries.extend(new_entries); + batches.push(new_cached_batch); } } + // Create span for this batch to add context to inference calls let next_batch_size = entries.len(); let next_batch_span = @@ -325,6 +333,7 @@ async fn batching_task( waiting_tokens += 1; } metrics::gauge!("tgi_batch_current_size", 0.0); + metrics::gauge!("tgi_batch_current_max_tokens", 0.0); } } } @@ -341,22 +350,11 @@ async fn prefill( match client.prefill(batch).await { Ok((generations, next_batch)) => { + // Send generated tokens and filter stopped entries filter_send_generations(generations, entries); // Filter next batch and remove requests that were stopped - let next_batch = match next_batch { - None => None, - Some(batch) => { - let id = batch.id; - let next_batch = filter_batch(batch, entries); - // Next batch is now empty - // Clear it from the Python shards cache - if next_batch.is_none() { - let _ = client.clear_cache(Some(id)).await; - } - next_batch - } - }; + let next_batch = filter_batch(client, next_batch, entries).await; metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "prefill"); metrics::increment_counter!("tgi_batch_inference_success", "method" => "prefill"); @@ -384,22 +382,11 @@ async fn decode( match client.decode(batches).await { Ok((generations, next_batch)) => { + // Send generated tokens and filter stopped entries filter_send_generations(generations, entries); // Filter next batch and remove requests that were stopped - let next_batch = match next_batch { - None => None, - Some(batch) => { - let id = batch.id; - let next_batch = filter_batch(batch, entries); - // Next batch is now empty - // Clear it from the Python shards cache - if next_batch.is_none() { - let _ = client.clear_cache(Some(id)).await; - } - next_batch - } - }; + let next_batch = filter_batch(client, next_batch, entries).await; metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "decode"); metrics::increment_counter!("tgi_batch_inference_success", "method" => "decode"); @@ -419,14 +406,35 @@ async fn decode( /// Filter a `batch` and remove all requests not present in `entries` #[instrument(skip_all)] -fn filter_batch(mut batch: Batch, entries: &IntMap) -> Option { +async fn filter_batch( + client: &mut ShardedClient, + next_batch: Option, + entries: &IntMap, +) -> Option { + let mut batch = next_batch?; + + // No need to filter + if batch.size as usize == entries.len() { + return Some(batch); + } + + let id = batch.id; + + // Retain only requests that are still in entries batch.requests.retain(|r| entries.contains_key(&r.id)); - let size = batch.requests.len(); - if size == 0 { - return None; + + if batch.requests.is_empty() { + // All requests have been filtered out + // Next batch is now empty + // Clear it from the Python shards cache + // We unwrap here as we need to panic since we cannot recover if this method fails + client.clear_cache(Some(id)).await.unwrap(); + None + } else { + // Filter Python shard cache + // We unwrap here as we need to panic since we cannot recover if this method fails + client.filter_batch(id, batch.requests).await.unwrap() } - batch.size = size as u32; - Some(batch) } /// Send one or multiple `InferStreamResponse` to Infer for all `entries` diff --git a/router/src/main.rs b/router/src/main.rs index 712071b7..28db6071 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -31,8 +31,12 @@ struct Args { max_input_length: usize, #[clap(default_value = "1512", long, env)] max_total_tokens: usize, - #[clap(default_value = "32", long, env)] - max_batch_size: usize, + #[clap(long, env)] + max_batch_size: Option, + #[clap(default_value = "1.2", long, env)] + waiting_served_ratio: f32, + #[clap(default_value = "32000", long, env)] + max_batch_total_tokens: u32, #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, #[clap(default_value = "3000", long, short, env)] @@ -64,6 +68,8 @@ fn main() -> Result<(), std::io::Error> { max_input_length, max_total_tokens, max_batch_size, + waiting_served_ratio, + mut max_batch_total_tokens, max_waiting_tokens, port, master_shard_uds_path, @@ -119,6 +125,12 @@ fn main() -> Result<(), std::io::Error> { .block_on(async { init_logging(otlp_endpoint, json_output); + if let Some(max_batch_size) = max_batch_size{ + tracing::warn!("`max-batch-size` is deprecated. Use `max-batch-total-tokens` instead"); + max_batch_total_tokens = (max_batch_size * max_total_tokens) as u32; + tracing::warn!("Overriding `max-batch-total-tokens` value with `max-batch-size` * `max-total-tokens` = {max_batch_total_tokens}"); + } + if tokenizer.is_none() { tracing::warn!( "Could not find a fast tokenizer implementation for {tokenizer_name}" @@ -174,7 +186,8 @@ fn main() -> Result<(), std::io::Error> { max_stop_sequences, max_input_length, max_total_tokens, - max_batch_size, + waiting_served_ratio, + max_batch_total_tokens, max_waiting_tokens, sharded_client, tokenizer, diff --git a/router/src/queue.rs b/router/src/queue.rs index 43651ff3..d970ebf1 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -2,7 +2,6 @@ use crate::infer::InferError; use crate::infer::InferStreamResponse; use crate::validation::ValidGenerateRequest; use nohash_hasher::{BuildNoHashHasher, IntMap}; -use std::cmp::min; use std::collections::VecDeque; use text_generation_client::{Batch, Request}; use tokio::sync::oneshot; @@ -34,12 +33,12 @@ pub(crate) struct Queue { } impl Queue { - pub(crate) fn new() -> Self { + pub(crate) fn new(requires_padding: bool) -> Self { // Create channel let (queue_sender, queue_receiver) = flume::unbounded(); // Launch background queue task - tokio::spawn(queue_task(queue_receiver)); + tokio::spawn(queue_task(requires_padding, queue_receiver)); Self { queue_sender } } @@ -59,7 +58,7 @@ impl Queue { pub(crate) async fn next_batch( &self, min_size: Option, - max_size: usize, + token_budget: u32, ) -> Option { // Create response channel let (response_sender, response_receiver) = oneshot::channel(); @@ -68,7 +67,7 @@ impl Queue { self.queue_sender .send(QueueCommand::NextBatch { min_size, - max_size, + token_budget, response_sender, span: Span::current(), }) @@ -80,20 +79,24 @@ impl Queue { } // Background task responsible of the queue state -async fn queue_task(receiver: flume::Receiver) { - let mut state = State::new(); +async fn queue_task(requires_padding: bool, receiver: flume::Receiver) { + let mut state = State::new(requires_padding); while let Ok(cmd) = receiver.recv_async().await { match cmd { - QueueCommand::Append(entry, span) => span.in_scope(|| state.append(entry)), + QueueCommand::Append(entry, span) => { + span.in_scope(|| state.append(entry)); + metrics::increment_gauge!("tgi_queue_size", 1.0); + } QueueCommand::NextBatch { min_size, - max_size, + token_budget, response_sender, span, } => span.in_scope(|| { - let next_batch = state.next_batch(min_size, max_size); + let next_batch = state.next_batch(min_size, token_budget); response_sender.send(next_batch).unwrap_or(()); + metrics::gauge!("tgi_queue_size", state.entries.len() as f64); }), } } @@ -110,14 +113,18 @@ struct State { /// Id of the next batch next_batch_id: u64, + + /// Whether the model is using padding + requires_padding: bool, } impl State { - fn new() -> Self { + fn new(requires_padding: bool) -> Self { Self { entries: VecDeque::with_capacity(128), next_id: 0, next_batch_id: 0, + requires_padding, } } @@ -130,11 +137,10 @@ impl State { // Push entry in the queue self.entries.push_back((self.next_id, entry)); self.next_id += 1; - metrics::increment_gauge!("tgi_queue_size", 1.0); } // Get the next batch - fn next_batch(&mut self, min_size: Option, max_size: usize) -> Option { + fn next_batch(&mut self, min_size: Option, token_budget: u32) -> Option { if self.entries.is_empty() { return None; } @@ -146,17 +152,19 @@ impl State { } } - let max_batch_size = min(self.entries.len(), max_size); - // Create span for this batch to add context to inference calls let next_batch_span = info_span!(parent: None, "batch", batch_size = tracing::field::Empty); next_batch_span.follows_from(&Span::current()); - let mut batch_requests = Vec::with_capacity(max_batch_size); + let mut batch_requests = Vec::with_capacity(self.entries.len()); let mut batch_entries = - IntMap::with_capacity_and_hasher(max_batch_size, BuildNoHashHasher::default()); + IntMap::with_capacity_and_hasher(self.entries.len(), BuildNoHashHasher::default()); - // Iterate on buffer + let mut max_input_length = 0; + let mut prefill_tokens: u32 = 0; + let mut decode_tokens: u32 = 0; + + // Pop entries starting from the front of the queue while let Some((id, mut entry)) = self.entries.pop_front() { // Filter entries where the response receiver was dropped (== entries where the request // was dropped by the client) @@ -165,6 +173,24 @@ impl State { continue; } + if self.requires_padding { + // We pad to max input length in the Python shards + // We need to take these padding tokens into the equation + max_input_length = max_input_length.max(entry.request.input_length); + prefill_tokens = (batch_requests.len() + 1) as u32 * max_input_length + } else { + prefill_tokens += entry.request.input_length; + } + + decode_tokens += entry.request.stopping_parameters.max_new_tokens; + + if (prefill_tokens + decode_tokens) > token_budget { + // Entry is over budget + // Add it back to the front + self.entries.push_front((id, entry)); + break; + } + // Create a new span to link the batch back to this entry let entry_batch_span = info_span!(parent: &entry.span, "infer"); // Add relationships @@ -184,21 +210,29 @@ impl State { entry.batch_time = Some(Instant::now()); // Insert in batch_entries IntMap batch_entries.insert(id, entry); - - if batch_requests.len() == max_batch_size { - // We have enough requests in the batch - break; - } } - metrics::gauge!("tgi_queue_size", self.entries.len() as f64); - - // Maybe all entries were dropped because their channel were closed + // Empty batch if batch_requests.is_empty() { return None; } - // Final batch size once we dropped entries + // Check if our batch is big enough + if let Some(min_size) = min_size { + // Batch is too small + if batch_requests.len() < min_size { + // Add back entries to the queue in the correct order + for r in batch_requests.into_iter().rev() { + let id = r.id; + let entry = batch_entries.remove(&id).unwrap(); + self.entries.push_front((id, entry)); + } + + return None; + } + } + + // Final batch size let size = batch_requests.len() as u32; next_batch_span.record("batch_size", size); @@ -206,11 +240,13 @@ impl State { id: self.next_batch_id, requests: batch_requests, size, + max_tokens: (prefill_tokens + decode_tokens), }; // Increment batch id self.next_batch_id += 1; metrics::histogram!("tgi_batch_next_size", batch.size as f64); + Some((batch_entries, batch, next_batch_span)) } } @@ -222,7 +258,7 @@ enum QueueCommand { Append(Entry, Span), NextBatch { min_size: Option, - max_size: usize, + token_budget: u32, response_sender: oneshot::Sender>, span: Span, }, @@ -243,6 +279,7 @@ mod tests { let entry = Entry { request: ValidGenerateRequest { inputs: "".to_string(), + input_length: 0, truncate: 0, parameters: NextTokenChooserParameters { temperature: 0.0, @@ -256,7 +293,7 @@ mod tests { }, stopping_parameters: StoppingCriteriaParameters { ignore_eos_token: false, - max_new_tokens: 0, + max_new_tokens: 1, stop_sequences: vec![], }, }, @@ -271,7 +308,7 @@ mod tests { #[test] fn test_append() { - let mut state = State::new(); + let mut state = State::new(false); let (entry, _guard) = default_entry(); assert_eq!(state.next_id, 0); @@ -287,7 +324,7 @@ mod tests { #[test] fn test_next_batch_empty() { - let mut state = State::new(); + let mut state = State::new(false); assert!(state.next_batch(None, 1).is_none()); assert!(state.next_batch(Some(1), 1).is_none()); @@ -295,7 +332,7 @@ mod tests { #[test] fn test_next_batch_min_size() { - let mut state = State::new(); + let mut state = State::new(false); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); state.append(entry1); @@ -326,8 +363,8 @@ mod tests { } #[test] - fn test_next_batch_max_size() { - let mut state = State::new(); + fn test_next_batch_token_budget() { + let mut state = State::new(false); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); state.append(entry1); @@ -360,14 +397,14 @@ mod tests { #[tokio::test] async fn test_queue_append() { - let queue = Queue::new(); + let queue = Queue::new(false); let (entry, _guard) = default_entry(); queue.append(entry); } #[tokio::test] async fn test_queue_next_batch_empty() { - let queue = Queue::new(); + let queue = Queue::new(false); assert!(queue.next_batch(None, 1).await.is_none()); assert!(queue.next_batch(Some(1), 1).await.is_none()); @@ -375,7 +412,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_min_size() { - let queue = Queue::new(); + let queue = Queue::new(false); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); queue.append(entry1); @@ -397,8 +434,8 @@ mod tests { } #[tokio::test] - async fn test_queue_next_batch_max_size() { - let queue = Queue::new(); + async fn test_queue_next_batch_token_budget() { + let queue = Queue::new(false); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); queue.append(entry1); @@ -423,7 +460,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_dropped_receiver() { - let queue = Queue::new(); + let queue = Queue::new(false); let (entry, _) = default_entry(); queue.append(entry); diff --git a/router/src/server.rs b/router/src/server.rs index 8891443f..d1f7ae12 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -511,7 +511,8 @@ pub async fn run( max_stop_sequences: usize, max_input_length: usize, max_total_tokens: usize, - max_batch_size: usize, + waiting_served_ratio: f32, + max_batch_total_tokens: u32, max_waiting_tokens: usize, client: ShardedClient, tokenizer: Option, @@ -571,9 +572,11 @@ pub async fn run( let infer = Infer::new( client, validation, - max_batch_size, + waiting_served_ratio, + max_batch_total_tokens, max_waiting_tokens, max_concurrent_requests, + shard_info.requires_padding, ); // Duration buckets @@ -604,7 +607,7 @@ pub async fn run( .collect(); // Batch size buckets let batch_size_matcher = Matcher::Full(String::from("tgi_batch_next_size")); - let batch_size_buckets: Vec = (0..max_batch_size).map(|x| (x + 1) as f64).collect(); + let batch_size_buckets: Vec = (0..1024).map(|x| (x + 1) as f64).collect(); // Prometheus handler let builder = PrometheusBuilder::new() diff --git a/router/src/validation.rs b/router/src/validation.rs index 5f1b89b9..7f2f76e6 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -69,7 +69,7 @@ impl Validation { inputs: String, truncate: Option, max_new_tokens: u32, - ) -> Result { + ) -> Result<(String, usize), ValidationError> { // If we have a fast tokenizer if let Some(sender) = &self.sender { // Create response channel @@ -105,25 +105,24 @@ impl Validation { } metrics::histogram!("tgi_request_input_length", input_length as f64); - Ok(inputs) + Ok((inputs, input_length)) } // Return inputs without validation else { // In this case, we don't know the real length in tokens of the inputs // However, the inputs will be truncated by the python servers // We make sure that truncate + max_new_tokens <= self.max_total_tokens + let input_length = truncate.unwrap_or(self.max_input_length); // Validate MaxNewTokens - if (truncate.unwrap_or(self.max_input_length) as u32 + max_new_tokens) - > self.max_total_tokens as u32 - { + if (input_length as u32 + max_new_tokens) > self.max_total_tokens as u32 { return Err(ValidationError::MaxNewTokens( self.max_total_tokens - self.max_input_length, max_new_tokens, )); } - Ok(inputs) + Ok((inputs, input_length)) } } @@ -238,7 +237,7 @@ impl Validation { .unwrap_or(Ok(None))?; // Validate inputs - let inputs = self + let (inputs, input_length) = self .validate_input(request.inputs, truncate, max_new_tokens) .await?; @@ -262,6 +261,7 @@ impl Validation { Ok(ValidGenerateRequest { inputs, + input_length: input_length as u32, truncate: truncate.unwrap_or(self.max_input_length) as u32, parameters, stopping_parameters, @@ -333,6 +333,7 @@ type TokenizerRequest = ( #[derive(Debug)] pub(crate) struct ValidGenerateRequest { pub inputs: String, + pub input_length: u32, pub truncate: u32, pub parameters: NextTokenChooserParameters, pub stopping_parameters: StoppingCriteriaParameters, diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 47d701eb..f0adab97 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -181,9 +181,7 @@ def test_causal_lm_generate_token_completion_multi( next_batch = next_batch.filter([next_batch.requests[0]]) for _ in range( - stopping_criterias[0].max_new_tokens - - stopping_criterias[1].max_new_tokens - - 1 + stopping_criterias[0].max_new_tokens - stopping_criterias[1].max_new_tokens - 1 ): generations, next_batch = default_bloom.generate_token(next_batch) assert len(generations) == len(next_batch) diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 03d3ef9b..f1f13e4b 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -174,14 +174,14 @@ def test_causal_lm_generate_token_completion_multi( == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) # Copy stopping_criterias before filtering - stopping_criterias = default_multi_requests_causal_lm_batch.stopping_criterias.copy() + stopping_criterias = ( + default_multi_requests_causal_lm_batch.stopping_criterias.copy() + ) next_batch = next_batch.filter([next_batch.requests[0]]) for _ in range( - stopping_criterias[0].max_new_tokens - - stopping_criterias[1].max_new_tokens - - 1 + stopping_criterias[0].max_new_tokens - stopping_criterias[1].max_new_tokens - 1 ): generations, next_batch = default_causal_lm.generate_token(next_batch) assert len(generations) == len(next_batch) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 1db5abce..336c9823 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -46,6 +46,9 @@ class CausalLMBatch(Batch): max_input_length: int padding_right_offset: int + # Maximum number of tokens this batch will grow to + max_tokens: int + # Past metadata keys_head_dim_last: bool = True @@ -54,6 +57,7 @@ def to_pb(self) -> generate_pb2.Batch: id=self.batch_id, requests=self.requests, size=len(self), + max_tokens=self.max_tokens, ) @classmethod @@ -73,6 +77,7 @@ def from_pb( # Parse batch max_truncation = 0 padding_right_offset = 0 + max_decode_tokens = 0 for i, r in enumerate(pb.requests): requests_idx_mapping[r.id] = i inputs.append(r.inputs) @@ -84,6 +89,7 @@ def from_pb( ) stopping_criterias.append(stopping_criteria) max_truncation = max(max_truncation, r.truncate) + max_decode_tokens += stopping_criteria.max_new_tokens padding_right_offset = max( padding_right_offset, stopping_criteria.max_new_tokens ) @@ -112,6 +118,8 @@ def from_pb( position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) + max_tokens = len(inputs) * max_input_length + max_decode_tokens + return cls( batch_id=pb.id, requests=pb.requests, @@ -128,6 +136,7 @@ def from_pb( stopping_criterias=stopping_criterias, max_input_length=max_input_length.item(), padding_right_offset=padding_right_offset, + max_tokens=max_tokens, ) @tracer.start_as_current_span("filter") @@ -150,6 +159,7 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc next_token_choosers = [] stopping_criterias = [] + total_remaining_decode_tokens = 0 new_padding_right_offset = 0 for i, r in enumerate(requests): @@ -168,19 +178,23 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc next_token_choosers.append(self.next_token_choosers[idx]) stopping_criteria = self.stopping_criterias[idx] stopping_criterias.append(stopping_criteria) - - new_padding_right_offset = max( - new_padding_right_offset, + remaining_decode_tokens = ( stopping_criteria.max_new_tokens - stopping_criteria.current_tokens ) + total_remaining_decode_tokens += remaining_decode_tokens + new_padding_right_offset = max( + new_padding_right_offset, remaining_decode_tokens + ) # Apply indices to input_ids, attention mask, past key values and other items that need to be cached input_ids = self.input_ids[keep_indices] position_ids = self.position_ids[keep_indices] self.attention_mask = self.attention_mask[ keep_indices, - -(self.padding_right_offset + max_input_length): - (self.attention_mask.shape[1] - self.padding_right_offset) + new_padding_right_offset, + -(self.padding_right_offset + max_input_length) : ( + self.attention_mask.shape[1] - self.padding_right_offset + ) + + new_padding_right_offset, ] # Ensure that past_key_values tensors can be updated in-place @@ -203,6 +217,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc layer[1] = past_values[keep_indices, :, -past_kv_length:, :] del past_values + max_tokens = len(requests) * max_input_length + total_remaining_decode_tokens + self.requests = requests self.requests_idx_mapping = requests_idx_mapping self.input_ids = input_ids @@ -215,6 +231,7 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc self.stopping_criterias = stopping_criterias self.max_input_length = max_input_length self.padding_right_offset = new_padding_right_offset + self.max_tokens = max_tokens return self @@ -239,6 +256,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": all_input_ids = [] next_token_choosers = [] stopping_criterias = [] + max_tokens = 0 # Batch tensors input_ids = None @@ -314,7 +332,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # And ensure that we can update tensors in-place if type(batch.past_key_values[0]) == tuple: batch.past_key_values = [ - [t.view(len(batch), -1, *t.shape[-2:]) for t in layer] for layer in batch.past_key_values + [t.view(len(batch), -1, *t.shape[-2:]) for t in layer] + for layer in batch.past_key_values ] elif batch.past_key_values[0][0].shape == 3: for layer in batch.past_key_values: @@ -322,6 +341,10 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": layer[k] = t.view(len(batch), -1, *t.shape[-2:]) start_index = end_index + # Add eventual padding tokens that were added while concatenating + max_tokens += batch.max_tokens + ( + max_input_length - batch.max_input_length + ) * len(batch) first_past_kvs = batches[0].past_key_values _, num_heads, padded_sequence_length, head_dim = first_past_kvs[0][1].shape @@ -371,7 +394,9 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": start_index = end_index - padded_past_values = first_past_kvs[j][1].new_zeros(padded_past_values_shape) + padded_past_values = first_past_kvs[j][1].new_zeros( + padded_past_values_shape + ) start_index = 0 for batch in batches: past_values = batch.past_key_values[j][1] @@ -387,6 +412,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": ] = past_values[:, :, -past_seq_len:, :] del past_values + # Update values start_index = end_index past_key_values.append([padded_past_keys, padded_past_values]) @@ -408,6 +434,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": max_input_length=max_input_length, padding_right_offset=padding_right_offset, keys_head_dim_last=batches[0].keys_head_dim_last, + max_tokens=max_tokens, ) def __len__(self): diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 9cd9ed89..61ccca84 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -56,9 +56,15 @@ class FlashCausalLMBatch(Batch): # Constant shared tensor, ref here just so that it's accessible in concatentate() past_pad: Optional[torch.Tensor] + # Maximum number of tokens this batch will grow to + max_tokens: int + def to_pb(self) -> generate_pb2.Batch: return generate_pb2.Batch( - id=self.batch_id, requests=self.requests, size=len(self) + id=self.batch_id, + requests=self.requests, + size=len(self), + max_tokens=self.max_tokens, ) @classmethod @@ -86,6 +92,8 @@ def from_pb( # Cumulative length cumulative_length = 0 + max_tokens = 0 + # Parse batch for i, r in enumerate(pb.requests): # request id -> idx in list mapping @@ -115,16 +123,20 @@ def from_pb( cu_seqlens.append(cumulative_length + input_length) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) + max_new_tokens = stopping_criteria.max_new_tokens stopping_criterias.append(stopping_criteria) + all_input_ids_tensor.append( F.pad(tokenized_input, (0, stopping_criteria.max_new_tokens)) ) # Update cumulative_length += input_length + max_tokens += input_length + max_new_tokens return cls( batch_id=pb.id, @@ -143,6 +155,7 @@ def from_pb( next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, past_pad=None, + max_tokens=max_tokens, ) @tracer.start_as_current_span("filter") @@ -177,6 +190,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": next_token_choosers = [] stopping_criterias = [] + max_tokens = 0 + for i, r in enumerate(requests): idx = self.requests_idx_mapping[r.id] requests_idx_mapping[r.id] = i @@ -203,9 +218,14 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": token_offsets.append(self.token_offsets[idx]) next_token_choosers.append(self.next_token_choosers[idx]) - stopping_criterias.append(self.stopping_criterias[idx]) + + stopping_criteria = self.stopping_criterias[idx] + stopping_criterias.append(stopping_criteria) cumulative_length += request_input_length + max_tokens += request_input_length + ( + stopping_criteria.max_new_tokens - stopping_criteria.current_tokens + ) if single_request: # Preallocate tensor for bs = 1 case @@ -241,6 +261,7 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": all_input_ids_tensor=all_input_ids_tensor, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, + max_tokens=max_tokens, ) @classmethod @@ -269,6 +290,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch # Cumulative length cumulative_batch_size = 0 cumulative_length = 0 + max_tokens = 0 for i, batch in enumerate(batches): requests.extend(batch.requests) @@ -310,6 +332,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch # Update cumulative_length += batch.cu_seqlens[-1] cumulative_batch_size += len(batch) + max_tokens += batch.max_tokens return FlashCausalLMBatch( batch_id=batches[0].batch_id, @@ -328,6 +351,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch all_input_ids_tensor=all_input_ids_tensor, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, + max_tokens=max_tokens, ) def __len__(self): diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 753e86ed..c1ec1566 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -101,6 +101,7 @@ def from_pb( # Parse batch max_truncation = 0 padding_right_offset = 0 + max_decode_tokens = 0 for i, r in enumerate(pb.requests): requests_idx_mapping[r.id] = i # Add escape_custom_split_sequence to the CausalLMBatch logic @@ -113,6 +114,7 @@ def from_pb( ) stopping_criterias.append(stopping_criteria) max_truncation = max(max_truncation, r.truncate) + max_decode_tokens += stopping_criteria.max_new_tokens padding_right_offset = max( padding_right_offset, stopping_criteria.max_new_tokens ) @@ -141,6 +143,8 @@ def from_pb( position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) + max_tokens = len(inputs) * max_input_length + max_decode_tokens + return cls( batch_id=pb.id, requests=pb.requests, @@ -157,6 +161,7 @@ def from_pb( stopping_criterias=stopping_criterias, max_input_length=max_input_length.item(), padding_right_offset=padding_right_offset, + max_tokens=max_tokens, ) diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 2252fcfc..0cb20760 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -54,10 +54,16 @@ class Seq2SeqLMBatch(Batch): max_decoder_input_length: int padding_right_offset: int + # Maximum number of tokens this batch will grow to + max_tokens: int + def to_pb(self) -> generate_pb2.Batch: """Convert a Seq2SeqLMBatch to a text_generation_server.v1.Batch protobuf""" return generate_pb2.Batch( - id=self.batch_id, requests=self.requests, size=len(self) + id=self.batch_id, + requests=self.requests, + size=len(self), + max_tokens=self.max_tokens, ) @classmethod @@ -80,6 +86,7 @@ def from_pb( # Parse batch max_truncation = 0 padding_right_offset = 0 + max_decode_tokens = 0 for i, r in enumerate(pb.requests): inputs.append(r.inputs) requests_idx_mapping[r.id] = i @@ -92,6 +99,7 @@ def from_pb( ) stopping_criterias.append(stopping_criteria) max_truncation = max(max_truncation, r.truncate) + max_decode_tokens += stopping_criteria.max_new_tokens padding_right_offset = max( padding_right_offset, stopping_criteria.max_new_tokens ) @@ -117,6 +125,8 @@ def from_pb( ) all_decoder_input_ids = decoder_input_ids.view(-1).split(1) + max_tokens = len(inputs) * max_input_length + max_decode_tokens + return cls( batch_id=pb.id, requests=pb.requests, @@ -137,6 +147,7 @@ def from_pb( max_input_length=max_input_length.item(), max_decoder_input_length=1, padding_right_offset=padding_right_offset, + max_tokens=max_tokens, ) @tracer.start_as_current_span("filter") @@ -166,6 +177,8 @@ def filter( max_decoder_input_length = 0 padding_right_offset = 0 + remaining_decode_tokens = 0 + for i, r in enumerate(requests): idx = self.requests_idx_mapping[r.id] requests_idx_mapping[r.id] = i @@ -187,27 +200,38 @@ def filter( ) padding_right_offset = max( padding_right_offset, - self.stopping_criterias[idx].max_new_tokens - self.stopping_criterias[idx].current_tokens + self.stopping_criterias[idx].max_new_tokens + - self.stopping_criterias[idx].current_tokens, ) next_token_choosers.append(self.next_token_choosers[idx]) - stopping_criterias.append(self.stopping_criterias[idx]) + stopping_criteria = self.stopping_criterias[idx] + stopping_criterias.append(stopping_criteria) + remaining_decode_tokens += ( + stopping_criteria.max_new_tokens - stopping_criteria.current_tokens + ) # Apply indices to input_ids, attention mask, past key values and other items that need to be cached self.decoder_input_ids = self.decoder_input_ids[keep_indices] self.attention_mask = self.attention_mask[keep_indices, -max_input_length:] if self.decoder_attention_mask is not None: self.decoder_attention_mask = self.decoder_attention_mask[ - keep_indices, - -(self.padding_right_offset + max_decoder_input_length): - (self.decoder_attention_mask.shape[1] - self.padding_right_offset) + padding_right_offset, + keep_indices, + -(self.padding_right_offset + max_decoder_input_length) : ( + self.decoder_attention_mask.shape[1] - self.padding_right_offset + ) + + padding_right_offset, ] - self.encoder_last_hidden_state = self.encoder_last_hidden_state[keep_indices, -max_input_length:] + self.encoder_last_hidden_state = self.encoder_last_hidden_state[ + keep_indices, -max_input_length: + ] # Ensure that past_key_values tensors can be updated in-place if type(self.past_key_values[0]) == tuple: - self.past_key_values = [[t for t in layer] for layer in self.past_key_values] + self.past_key_values = [ + [t for t in layer] for layer in self.past_key_values + ] decoder_past_seq_len = max_decoder_input_length - 1 for layer in self.past_key_values: @@ -216,6 +240,11 @@ def filter( layer[2] = layer[2][keep_indices, :, -max_input_length:] layer[3] = layer[3][keep_indices, :, -max_input_length:] + max_tokens = ( + len(requests) * (max_input_length + max_decoder_input_length) + + remaining_decode_tokens + ) + self.requests = requests self.requests_idx_mapping = requests_idx_mapping self.input_ids = None @@ -229,10 +258,10 @@ def filter( self.max_input_length = max_input_length self.max_decoder_input_length = max_decoder_input_length self.padding_right_offset = padding_right_offset + self.max_tokens = max_tokens return self - @classmethod @tracer.start_as_current_span("concatenate") def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": @@ -261,6 +290,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": token_offsets = [] next_token_choosers = [] stopping_criterias = [] + max_tokens = 0 # Batch tensors attention_mask = None @@ -363,9 +393,18 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": # Ensure that we can update tensors in-place if type(batch.past_key_values[0]) == tuple: - batch.past_key_values = [[t for t in layer] for layer in batch.past_key_values] + batch.past_key_values = [ + [t for t in layer] for layer in batch.past_key_values + ] start_index = end_index + # Add eventual padding tokens that were added while concatenating + max_tokens += batch.max_tokens + ( + max_input_length + - batch.max_input_length + + max_decoder_input_length + - batch.max_decoder_input_length + ) * len(batch) # Determine shapes for new past kv tensors first_past_kvs = batches[0].past_key_values @@ -404,9 +443,9 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": end_index = start_index + len(batch) # We slice the past keys and values to remove the padding from previous batches past_seq_len = batch.max_decoder_input_length - 1 - padded_past_values[ - start_index:end_index, :, -past_seq_len:, : - ] = t[:, :, -past_seq_len:, :] + padded_past_values[start_index:end_index, :, -past_seq_len:, :] = t[ + :, :, -past_seq_len:, : + ] del t start_index = end_index @@ -426,8 +465,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": end_index = start_index + len(batch) # We slice the past keys and values to remove the padding from previous batches padded_past_values[ - start_index:end_index, :, -batch.max_input_length:, : - ] = t[:, :, -batch.max_input_length:, :] + start_index:end_index, :, -batch.max_input_length :, : + ] = t[:, :, -batch.max_input_length :, :] del t start_index = end_index @@ -452,6 +491,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": max_input_length=max_input_length, max_decoder_input_length=max_decoder_input_length, padding_right_offset=padding_right_offset, + max_tokens=max_tokens, ) def __len__(self): diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 95b431c8..ddb7aae9 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -41,6 +41,15 @@ async def ClearCache(self, request, context): torch.cuda.empty_cache() return generate_pb2.ClearCacheResponse() + async def FilterBatch(self, request, context): + batch = self.cache.pop(request.batch_id) + if batch is None: + raise ValueError(f"Batch ID {request.batch_id} not found in cache.") + filtered_batch = batch.filter(request.keep_requests) + self.cache.set(filtered_batch) + + return generate_pb2.FilterBatchResponse(batch=filtered_batch.to_pb()) + async def Prefill(self, request, context): batch = self.model.batch_type.from_pb( request.batch, self.model.tokenizer, self.model.device @@ -63,9 +72,7 @@ async def Decode(self, request, context): batch = self.cache.pop(batch_pb.id) if batch is None: raise ValueError(f"Batch ID {batch_pb.id} not found in cache.") - batch = batch.filter(batch_pb.requests) - if batch is not None: - batches.append(batch) + batches.append(batch) if len(batches) == 0: raise ValueError("All batches are empty") From 8b182eb98662ea781990a4a2e869eb8859e26073 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 25 Apr 2023 13:11:18 +0200 Subject: [PATCH 191/793] feat(router): add endpoint info to /info route (#228) --- router/src/lib.rs | 21 +++++++++++++++++++++ router/src/server.rs | 41 +++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index 2f93ec0a..7a1707d9 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -21,6 +21,7 @@ pub struct HubModelInfo { #[derive(Clone, Debug, Serialize, ToSchema)] pub struct Info { + /// Model info #[schema(example = "bigscience/blomm-560m")] pub model_id: String, #[schema(nullable = true, example = "e985a63cdc139290c5f700ff1929f0b5942cced2")] @@ -31,6 +32,26 @@ pub struct Info { pub model_device_type: String, #[schema(nullable = true, example = "text-generation")] pub model_pipeline_tag: Option, + /// Router Parameters + #[schema(example = "128")] + pub max_concurrent_requests: usize, + #[schema(example = "2")] + pub max_best_of: usize, + #[schema(example = "4")] + pub max_stop_sequences: usize, + #[schema(example = "1024")] + pub max_input_length: usize, + #[schema(example = "2048")] + pub max_total_tokens: usize, + #[schema(example = "1.2")] + pub waiting_served_ratio: f32, + #[schema(example = "32000")] + pub max_batch_total_tokens: u32, + #[schema(example = "20")] + pub max_waiting_tokens: usize, + #[schema(example = "2")] + pub validation_workers: usize, + /// Router Info #[schema(example = "0.5.0")] pub version: &'static str, #[schema(nullable = true, example = "null")] diff --git a/router/src/server.rs b/router/src/server.rs index d1f7ae12..9540ba18 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -78,22 +78,8 @@ async fn compat_generate( responses((status = 200, description = "Served model info", body = Info)) )] #[instrument] -async fn get_model_info( - model_info: Extension, - shard_info: Extension, -) -> Json { - let model_info = model_info.0; - let shard_info = shard_info.0; - let info = Info { - version: env!("CARGO_PKG_VERSION"), - sha: option_env!("VERGEN_GIT_SHA"), - model_id: model_info.model_id, - model_sha: model_info.sha, - model_dtype: shard_info.dtype, - model_device_type: shard_info.device_type, - model_pipeline_tag: model_info.pipeline_tag, - }; - Json(info) +async fn get_model_info(info: Extension) -> Json { + Json(info.0) } /// Health check method @@ -632,6 +618,26 @@ pub async fn run( .allow_headers([http::header::CONTENT_TYPE]) .allow_origin(allow_origin); + // Endpoint info + let info = Info { + model_id: model_info.model_id, + model_sha: model_info.sha, + model_dtype: shard_info.dtype, + model_device_type: shard_info.device_type, + model_pipeline_tag: model_info.pipeline_tag, + max_concurrent_requests, + max_best_of, + max_stop_sequences, + max_input_length, + max_total_tokens, + waiting_served_ratio, + max_batch_total_tokens, + max_waiting_tokens, + validation_workers, + version: env!("CARGO_PKG_VERSION"), + sha: option_env!("VERGEN_GIT_SHA"), + }; + // Create router let app = Router::new() .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", ApiDoc::openapi())) @@ -650,8 +656,7 @@ pub async fn run( .route("/ping", get(health)) // Prometheus metrics route .route("/metrics", get(metrics)) - .layer(Extension(model_info)) - .layer(Extension(shard_info)) + .layer(Extension(info)) .layer(Extension(compat_return_full_text)) .layer(Extension(infer)) .layer(Extension(prom_handle)) From 37b64a5c10a87f25b5de1c3c55f3ea965104f290 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 25 Apr 2023 13:50:56 +0200 Subject: [PATCH 192/793] chore(server): update safetensors version (#235) --- server/poetry.lock | 103 ++++++++++++++++++++-------------------- server/pyproject.toml | 2 +- server/requirements.txt | 2 +- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/server/poetry.lock b/server/poetry.lock index 4b9855c5..b5c03a57 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -299,7 +299,7 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "lit" -version = "16.0.1" +version = "16.0.2" description = "A Software Testing Tool" category = "main" optional = true @@ -702,21 +702,22 @@ use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "safetensors" -version = "0.2.8" +version = "0.3.1" description = "Fast and Safe Tensor serialization" category = "main" optional = false python-versions = "*" [package.extras] -all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax", "h5py", "huggingface-hub", "isort (>=5.5.4)", "jax", "numpy", "pytest", "pytest-benchmark", "setuptools-rust", "tensorflow", "torch"] -dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax", "h5py", "huggingface-hub", "isort (>=5.5.4)", "jax", "numpy", "pytest", "pytest-benchmark", "setuptools-rust", "tensorflow", "torch"] -jax = ["flax", "jax"] -numpy = ["numpy"] +all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] +dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] +numpy = ["numpy (>=1.21.6)"] +paddlepaddle = ["paddlepaddle (>=2.4.1)"] quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] -tensorflow = ["tensorflow"] -testing = ["h5py", "huggingface-hub", "numpy", "pytest", "pytest-benchmark", "setuptools-rust"] -torch = ["torch"] +tensorflow = ["tensorflow (>=2.11.0)"] +testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] +torch = ["torch (>=1.10)"] [[package]] name = "sentencepiece" @@ -911,7 +912,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "2534d6a2dea05e0061707b2c0ede295902790656a3c6256fef692e06d66a0c4a" +content-hash = "f3ee14e1a7730c01677ea6f28d3c76fffcd244f3a1f96232275988fbb2d82d2c" [metadata.files] accelerate = [ @@ -1198,7 +1199,7 @@ Jinja2 = [ {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] lit = [ - {file = "lit-16.0.1.tar.gz", hash = "sha256:630a47291b714cb115015df23ab04267c24fe59aec7ecd7e637d5c75cdb45c91"}, + {file = "lit-16.0.2.tar.gz", hash = "sha256:d743ef55cb58764bba85768c502e2d68d87aeb4303d508a18abaa8a35077ab25"}, ] loguru = [ {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, @@ -1465,46 +1466,46 @@ requests = [ {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, ] safetensors = [ - {file = "safetensors-0.2.8-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:8df8af89a0b6b535b47c077e33e5cd4941ef4b067e7c1dd1a05647dec0cf2eea"}, - {file = "safetensors-0.2.8-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:9cabae651542b22d229b6439029fb4a3abc7868cb123af336b2877541ad9ab39"}, - {file = "safetensors-0.2.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:384df328523c567a8b2193ebb02456300a43a3adbc316823d4c0d16f7ac9e89d"}, - {file = "safetensors-0.2.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa5f4b31f2e283b83894b388298368860367a1cb773228f9bb65f8db65da1549"}, - {file = "safetensors-0.2.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70f322d170a17b6ecb9c85b15e67f4a9aa3e561594e2dfc7709c0ae0000ebfff"}, - {file = "safetensors-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50f7e90ed02ef5e4263aadce23d39a3595156816c42c4643003791b45f81fd31"}, - {file = "safetensors-0.2.8-cp310-cp310-win32.whl", hash = "sha256:726ad66231286157dd505b5ab85fd903114dcb5f9180e79fd5540d68d6249dd0"}, - {file = "safetensors-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:d14e7c15e5acac6efcd5f5565e38b1b33600101387e5d16059de44adab87405f"}, - {file = "safetensors-0.2.8-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:6273dd3edd5db05b2da0090bc3d491bf25f6d1d7e8a4423477377649e9e38c37"}, - {file = "safetensors-0.2.8-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34705920c8a02f9ea6101bae8403db5f4aa18ec3eaccd8eab6701b1c88ee5bed"}, - {file = "safetensors-0.2.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efff5ce2d3f349d5360a5ca5901ae56c7e24f8da623d57cd08f8e8b1cd9cb1f8"}, - {file = "safetensors-0.2.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dec20b9b1fc90b7b4e588b4f0e9e266bd8f26d405e08b0b6ecad3136d92d007a"}, - {file = "safetensors-0.2.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7716bab34ca9651895126c720df1bf07f464184a7409138179f38db488ca9f15"}, - {file = "safetensors-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb28e5e6257f705828fd39b9ba28248b593f46e722d8d6beedbc8d1f194e2297"}, - {file = "safetensors-0.2.8-cp311-cp311-win32.whl", hash = "sha256:c71e147a8c2942690e8707adbcc4ab60775bc78dfdbd7f9e696dd411adef82bb"}, - {file = "safetensors-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:2f40604c50d4a08a4c74f37fef735cd1e203a59aeda66ea23a01d76fb35cf407"}, - {file = "safetensors-0.2.8-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:f3c7242debe713a87ca6deaadb0d7120e61c92435414142d526e8c32da970020"}, - {file = "safetensors-0.2.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9366140785d567049395c1c03ac69ee8d322fabcdc8fab7d389e933596b383da"}, - {file = "safetensors-0.2.8-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5d720c7640ad5f95f47780bdc35777ed8371afa14d8d63d6375cfe36df83fb4"}, - {file = "safetensors-0.2.8-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0c7ed4d75c2f248791dbe64309c98ada6f40a6949147ca2eaebd278906c918b"}, - {file = "safetensors-0.2.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b905cf3563da4fe8dd46fa777321516f5aa8f6bc1b884158be03a235478e96d"}, - {file = "safetensors-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:2a47327cfe57b6ee8c5dc246f9141f4f6b368e4444dd7f174c025b1d34446730"}, - {file = "safetensors-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:02e67b906ad9bbb87308a34e4d2d33c9eb69baf342b7e5c872728556baf3f0b6"}, - {file = "safetensors-0.2.8-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:ba7c2496765de45a84f5c79b2b12a14a568643d8966ef9bb3f8b16217a39457c"}, - {file = "safetensors-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:90cd22387b1520c4465033b986f79f0d24cc41aabae1903a22eff3b42cee9ad5"}, - {file = "safetensors-0.2.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f279e0fd917a886e1936d534734593f780afdddc6ed62f8ebf2f59de811cdd7c"}, - {file = "safetensors-0.2.8-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af4fce0565574ec3dbe997b021ed32d3dc229547c4f7fca2459be1f725c26f88"}, - {file = "safetensors-0.2.8-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ff0096f5a765e6e3f7f3156f568f59499edeade19e813d610a124ca70b42cdda"}, - {file = "safetensors-0.2.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c02c1cab1d23e6cc8ac1ef865efe76f2d0800e203c877288ebd31747f47a6940"}, - {file = "safetensors-0.2.8-cp38-cp38-win32.whl", hash = "sha256:832f27f6f379fb15d87f3a10eb87e2480434a89c483d7edc8a0c262364ba72c1"}, - {file = "safetensors-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:89da823f9801e92b2c48e8fad1e2f7f0cb696a8a93dab4c6700e8de06fe87392"}, - {file = "safetensors-0.2.8-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:ee8e169040468d176172b4f84a160ece2064abcc77294c4994a9d2bb5255cd75"}, - {file = "safetensors-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:ac2197dbbf7cbd269faf8cebab4220dba5aa2ac8beacbce8fdcb9800776781ca"}, - {file = "safetensors-0.2.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f90513eee0a11902df7e51b07c3d9c328828b9dd692d6c74140bed932e7a491"}, - {file = "safetensors-0.2.8-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2bd2e4c6258dd0f4e3d554f2f59789f25ef4757431e83c927016de6339e54811"}, - {file = "safetensors-0.2.8-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:01000a4bfdf0474bb1bdf369de1284c93b4e6047524fe9dc55d77586cb9d0243"}, - {file = "safetensors-0.2.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:466f92a384e4fcb6b1b9811e7488516c4638119c302a959b89bbe1be826d5e25"}, - {file = "safetensors-0.2.8-cp39-cp39-win32.whl", hash = "sha256:2f16e5ee70ae4218474493eff8d998e632a896a6b8aff9273511d654cdf367ab"}, - {file = "safetensors-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:ba3dc236a2344b7feadc9868307f42ba5e4804c9d68a80a35aac831349b31f6f"}, - {file = "safetensors-0.2.8.tar.gz", hash = "sha256:2720b20a6a38c799dca79bd76caeeac2f7df585a9d4f7d59fa7e28eff9ccb27f"}, + {file = "safetensors-0.3.1-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:2ae9b7dd268b4bae6624729dac86deb82104820e9786429b0583e5168db2f770"}, + {file = "safetensors-0.3.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:08c85c1934682f1e2cd904d38433b53cd2a98245a7cc31f5689f9322a2320bbf"}, + {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba625c7af9e1c5d0d91cb83d2fba97d29ea69d4db2015d9714d24c7f6d488e15"}, + {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b57d5890c619ec10d9f1b6426b8690d0c9c2868a90dc52f13fae6f6407ac141f"}, + {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c9f562ea696d50b95cadbeb1716dc476714a87792ffe374280c0835312cbfe2"}, + {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c115951b3a865ece8d98ee43882f2fd0a999c0200d6e6fec24134715ebe3b57"}, + {file = "safetensors-0.3.1-cp310-cp310-win32.whl", hash = "sha256:118f8f7503ea312fc7af27e934088a1b589fb1eff5a7dea2cd1de6c71ee33391"}, + {file = "safetensors-0.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:54846eaae25fded28a7bebbb66be563cad221b4c80daee39e2f55df5e5e0266f"}, + {file = "safetensors-0.3.1-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:5af82e10946c4822506db0f29269f43147e889054704dde994d4e22f0c37377b"}, + {file = "safetensors-0.3.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:626c86dd1d930963c8ea7f953a3787ae85322551e3a5203ac731d6e6f3e18f44"}, + {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12e30677e6af1f4cc4f2832546e91dbb3b0aa7d575bfa473d2899d524e1ace08"}, + {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d534b80bc8d39945bb902f34b0454773971fe9e5e1f2142af451759d7e52b356"}, + {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ddd0ddd502cf219666e7d30f23f196cb87e829439b52b39f3e7da7918c3416df"}, + {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:997a2cc14023713f423e6d16536d55cb16a3d72850f142e05f82f0d4c76d383b"}, + {file = "safetensors-0.3.1-cp311-cp311-win32.whl", hash = "sha256:6ae9ca63d9e22f71ec40550207bd284a60a6b4916ae6ca12c85a8d86bf49e0c3"}, + {file = "safetensors-0.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:62aa7421ca455418423e35029524489480adda53e3f702453580180ecfebe476"}, + {file = "safetensors-0.3.1-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:6d54b3ed367b6898baab75dfd057c24f36ec64d3938ffff2af981d56bfba2f42"}, + {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:262423aeda91117010f8c607889066028f680fbb667f50cfe6eae96f22f9d150"}, + {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10efe2513a8327fd628cea13167089588acc23093ba132aecfc536eb9a4560fe"}, + {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:689b3d6a7ebce70ee9438267ee55ea89b575c19923876645e927d08757b552fe"}, + {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14cd9a87bc73ce06903e9f8ee8b05b056af6f3c9f37a6bd74997a16ed36ff5f4"}, + {file = "safetensors-0.3.1-cp37-cp37m-win32.whl", hash = "sha256:a77cb39624480d5f143c1cc272184f65a296f573d61629eff5d495d2e0541d3e"}, + {file = "safetensors-0.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9eff3190bfbbb52eef729911345c643f875ca4dbb374aa6c559675cfd0ab73db"}, + {file = "safetensors-0.3.1-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:05cbfef76e4daa14796db1bbb52072d4b72a44050c368b2b1f6fd3e610669a89"}, + {file = "safetensors-0.3.1-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:c49061461f4a81e5ec3415070a3f135530834c89cbd6a7db7cd49e3cb9d9864b"}, + {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22cf7e73ca42974f098ce0cf4dd8918983700b6b07a4c6827d50c8daefca776e"}, + {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04f909442d6223ff0016cd2e1b2a95ef8039b92a558014627363a2e267213f62"}, + {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c573c5a0d5d45791ae8c179e26d74aff86e719056591aa7edb3ca7be55bc961"}, + {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6994043b12e717cf2a6ba69077ac41f0d3675b2819734f07f61819e854c622c7"}, + {file = "safetensors-0.3.1-cp38-cp38-win32.whl", hash = "sha256:158ede81694180a0dbba59422bc304a78c054b305df993c0c6e39c6330fa9348"}, + {file = "safetensors-0.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:afdc725beff7121ea8d39a7339f5a6abcb01daa189ea56290b67fe262d56e20f"}, + {file = "safetensors-0.3.1-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:cba910fcc9e5e64d32d62b837388721165e9c7e45d23bc3a38ad57694b77f40d"}, + {file = "safetensors-0.3.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a4f7dbfe7285573cdaddd85ef6fa84ebbed995d3703ab72d71257944e384612f"}, + {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54aed0802f9eaa83ca7b1cbb986bfb90b8e2c67b6a4bcfe245627e17dad565d4"}, + {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34b75a766f3cfc99fd4c33e329b76deae63f5f388e455d863a5d6e99472fca8e"}, + {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a0f31904f35dc14919a145b2d7a2d8842a43a18a629affe678233c4ea90b4af"}, + {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcf527ecc5f58907fd9031510378105487f318cc91ecdc5aee3c7cc8f46030a8"}, + {file = "safetensors-0.3.1-cp39-cp39-win32.whl", hash = "sha256:e2f083112cf97aa9611e2a05cc170a2795eccec5f6ff837f4565f950670a9d83"}, + {file = "safetensors-0.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:5f4f614b8e8161cd8a9ca19c765d176a82b122fa3d3387b77862145bfe9b4e93"}, + {file = "safetensors-0.3.1.tar.gz", hash = "sha256:571da56ff8d0bec8ae54923b621cda98d36dcef10feb36fd492c4d0c2cd0e869"}, ] sentencepiece = [ {file = "sentencepiece-0.1.98-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1daf0a79cd953e4830746c41e92b98a2f2e9e5ec0e90a9447aa10350e11bd027"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 3998a062..22aa41db 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -17,7 +17,7 @@ grpc-interceptor = "^0.15.0" typer = "^0.6.1" accelerate = { version = "^0.15.0", optional = true } bitsandbytes = { version = "^0.38.1", optional = true } -safetensors = "^0.2.4" +safetensors = "0.3.1" loguru = "^0.6.0" opentelemetry-api = "^1.15.0" opentelemetry-exporter-otlp = "^1.15.0" diff --git a/server/requirements.txt b/server/requirements.txt index f4fa7d7e..ad48bff9 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -29,7 +29,7 @@ packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" protobuf==4.22.3 ; python_version >= "3.9" and python_version < "4.0" pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" requests==2.28.2 ; python_version >= "3.9" and python_version < "4" -safetensors==0.2.8 ; python_version >= "3.9" and python_version < "4.0" +safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" sentencepiece==0.1.98 ; python_version >= "3.9" and python_version < "4.0" setuptools==67.7.2 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" From 323546df1da2929e433ce197499ab71621dec51d Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 25 Apr 2023 13:55:26 +0200 Subject: [PATCH 193/793] fix(python-client): add auth headers to is supported requests (#234) --- clients/python/pyproject.toml | 2 +- .../python/text_generation/inference_api.py | 25 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 1af5de99..d883af37 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.5.0" +version = "0.5.1" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] diff --git a/clients/python/text_generation/inference_api.py b/clients/python/text_generation/inference_api.py index 31635e6e..93b0de8d 100644 --- a/clients/python/text_generation/inference_api.py +++ b/clients/python/text_generation/inference_api.py @@ -1,7 +1,7 @@ import os import requests -from typing import Optional, List +from typing import Dict, Optional, List from huggingface_hub.utils import build_hf_headers from text_generation import Client, AsyncClient, __version__ @@ -13,7 +13,7 @@ ) -def deployed_models() -> List[DeployedModel]: +def deployed_models(headers: Optional[Dict] = None) -> List[DeployedModel]: """ Get all currently deployed models with text-generation-inference-support @@ -22,6 +22,7 @@ def deployed_models() -> List[DeployedModel]: """ resp = requests.get( f"https://api-inference.huggingface.co/framework/text-generation-inference", + headers=headers, timeout=5, ) @@ -33,7 +34,7 @@ def deployed_models() -> List[DeployedModel]: return models -def check_model_support(repo_id: str) -> bool: +def check_model_support(repo_id: str, headers: Optional[Dict] = None) -> bool: """ Check if a given model is supported by text-generation-inference @@ -42,6 +43,7 @@ def check_model_support(repo_id: str) -> bool: """ resp = requests.get( f"https://api-inference.huggingface.co/status/{repo_id}", + headers=headers, timeout=5, ) @@ -95,13 +97,14 @@ def __init__(self, repo_id: str, token: Optional[str] = None, timeout: int = 10) Timeout in seconds """ - # Text Generation Inference client only supports a subset of the available hub models - if not check_model_support(repo_id): - raise NotSupportedError(repo_id) - headers = build_hf_headers( token=token, library_name="text-generation", library_version=__version__ ) + + # Text Generation Inference client only supports a subset of the available hub models + if not check_model_support(repo_id, headers): + raise NotSupportedError(repo_id) + base_url = f"{INFERENCE_ENDPOINT}/models/{repo_id}" super(InferenceAPIClient, self).__init__( @@ -150,14 +153,14 @@ def __init__(self, repo_id: str, token: Optional[str] = None, timeout: int = 10) timeout (`int`): Timeout in seconds """ + headers = build_hf_headers( + token=token, library_name="text-generation", library_version=__version__ + ) # Text Generation Inference client only supports a subset of the available hub models - if not check_model_support(repo_id): + if not check_model_support(repo_id, headers): raise NotSupportedError(repo_id) - headers = build_hf_headers( - token=token, library_name="text-generation", library_version=__version__ - ) base_url = f"{INFERENCE_ENDPOINT}/models/{repo_id}" super(InferenceAPIAsyncClient, self).__init__( From 45344244cf194c89c94fdbdfc1bdca724c7df4e5 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 25 Apr 2023 14:13:14 +0200 Subject: [PATCH 194/793] Starting some routing tests. (#233) --- .gitignore | 3 ++- router/src/validation.rs | 49 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ec376bb8..19604d42 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea -target \ No newline at end of file +target +router/tokenizer.json diff --git a/router/src/validation.rs b/router/src/validation.rs index 7f2f76e6..983c2612 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -378,3 +378,52 @@ pub enum ValidationError { #[error("tokenizer error {0}")] Tokenizer(String), } + +#[cfg(test)] +mod tests{ + use super::*; + use std::io::Write; + + #[tokio::test] + async fn test_validation_max_new_tokens(){ + let tokenizer = None; + let max_best_of = 2; + let max_stop_sequence = 3; + let max_input_length = 4; + let max_total_tokens = 5; + let workers = 1; + let validation = Validation::new(workers, tokenizer, max_best_of, max_stop_sequence, max_input_length, max_total_tokens); + + let max_new_tokens = 10; + match validation.validate_input("Hello".to_string(), None, max_new_tokens).await{ + Err(ValidationError::MaxNewTokens(1, 10)) => (), + _ => panic!("Unexpected not max new tokens") + } + } + + async fn get_tokenizer() -> Tokenizer{ + if !std::path::Path::new("tokenizer.json").exists(){ + let content = reqwest::get("https://huggingface.co/gpt2/raw/main/tokenizer.json").await.unwrap().bytes().await.unwrap(); + let mut file = std::fs::File::create("tokenizer.json").unwrap(); + file.write_all(&content).unwrap(); + } + Tokenizer::from_file("tokenizer.json").unwrap() + } + + #[tokio::test] + async fn test_validation_input_length(){ + let tokenizer = Some(get_tokenizer().await); + let max_best_of = 2; + let max_stop_sequence = 3; + let max_input_length = 4; + let max_total_tokens = 5; + let workers = 1; + let validation = Validation::new(workers, tokenizer, max_best_of, max_stop_sequence, max_input_length, max_total_tokens); + + let max_new_tokens = 10; + match validation.validate_input("Hello".to_string(), None, max_new_tokens).await{ + Err(ValidationError::MaxTotalTokens(5, 1, 10)) => (), + _ => panic!("Unexpected not max new tokens") + } + } +} From 7de8a377b067af5d9133874b88f5b0a37452a5eb Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 26 Apr 2023 00:54:27 +0200 Subject: [PATCH 195/793] fix(benchmarking): fix benchmarking tool --- benchmark/src/generation.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs index dde429a5..4a119e86 100644 --- a/benchmark/src/generation.rs +++ b/benchmark/src/generation.rs @@ -74,16 +74,28 @@ async fn generate_runs( for b in batch_size { // Warmups on batch size for _ in 0..warmups { - let (_, decode_batch) = - prefill(sequence.clone(), sequence_length, b, decode_length, &mut client).await?; + let (_, decode_batch) = prefill( + sequence.clone(), + sequence_length, + b, + decode_length, + &mut client, + ) + .await?; let _ = decode(decode_batch, &mut client).await?; // Send warmup message run_sender.send(Ok(Message::Warmup)).await.unwrap_or(()); } for _ in 0..n_runs { - let (prefill, decode_batch) = - prefill(sequence.clone(), sequence_length, b, decode_length, &mut client).await?; + let (prefill, decode_batch) = prefill( + sequence.clone(), + sequence_length, + b, + decode_length, + &mut client, + ) + .await?; // Send prefill message run_sender .send(Ok(Message::Prefill(prefill))) @@ -143,6 +155,7 @@ async fn prefill( id: 0, requests, size: batch_size, + max_tokens: batch_size * (sequence_length + decode_length), }; // Run prefill From 77758f603b87a15dec153ca4d5b6bfb6832f1cff Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 26 Apr 2023 14:43:36 +0200 Subject: [PATCH 196/793] chore(launcher): refactor logic (#242) Hopefully it's cleaner --- launcher/src/main.rs | 1093 +++++++++++++++++++++--------------------- 1 file changed, 550 insertions(+), 543 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 7450b3f4..b59b0cb4 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -4,7 +4,6 @@ use std::env; use std::ffi::OsString; use std::io::{BufRead, BufReader, Read}; use std::path::Path; -use std::process::ExitCode; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::TryRecvError; use std::sync::Arc; @@ -73,248 +72,454 @@ struct Args { watermark_delta: Option, } -fn main() -> ExitCode { - // Pattern match configuration - let args = Args::parse(); +#[derive(Debug)] +enum ShardStatus { + Ready, + Failed((usize, String)), +} - if args.json_output { - tracing_subscriber::fmt().json().init(); - } else { - tracing_subscriber::fmt().compact().init(); +#[allow(clippy::too_many_arguments)] +fn shard_manager( + model_id: String, + revision: Option, + quantize: bool, + uds_path: String, + rank: usize, + world_size: usize, + master_addr: String, + master_port: usize, + huggingface_hub_cache: Option, + weights_cache_override: Option, + disable_custom_kernels: bool, + watermark_gamma: Option, + watermark_delta: Option, + otlp_endpoint: Option, + status_sender: mpsc::Sender, + shutdown: Arc>, + _shutdown_sender: mpsc::Sender<()>, +) { + // Get UDS path + let uds_string = format!("{uds_path}-{rank}"); + let uds = Path::new(&uds_string); + // Clean previous runs + fs::remove_file(uds).unwrap_or_default(); + + // Process args + let mut shard_argv = vec![ + "text-generation-server".to_string(), + "serve".to_string(), + model_id, + "--uds-path".to_string(), + uds_path, + "--logger-level".to_string(), + "INFO".to_string(), + "--json-output".to_string(), + ]; + + // Activate tensor parallelism + if world_size > 1 { + shard_argv.push("--sharded".to_string()); } - tracing::info!("{:?}", args); + if quantize { + shard_argv.push("--quantize".to_string()) + } - let Args { - model_id, - revision, - sharded, - num_shard, - quantize, - max_concurrent_requests, - max_best_of, - max_stop_sequences, - max_input_length, - max_total_tokens, - max_batch_size, - max_batch_total_tokens, - waiting_served_ratio, - max_waiting_tokens, - port, - shard_uds_path, - master_addr, - master_port, - huggingface_hub_cache, - weights_cache_override, - disable_custom_kernels, - json_output, - otlp_endpoint, - cors_allow_origin, - watermark_gamma, - watermark_delta, - } = args; + // Model optional revision + if let Some(revision) = revision { + shard_argv.push("--revision".to_string()); + shard_argv.push(revision) + } - // get the number of shards given `sharded` and `num_shard` - let num_shard = if let Some(sharded) = sharded { - // sharded is set - match sharded { - // sharded is set and true - true => { - match num_shard { - None => { - // try to default to the number of available GPUs - tracing::info!("Parsing num_shard from CUDA_VISIBLE_DEVICES"); - let n_devices = num_cuda_devices() - .expect("--num-shard and CUDA_VISIBLE_DEVICES are not set"); - if n_devices <= 1 { - panic!("`sharded` is true but only found {n_devices} CUDA devices"); - } - n_devices - } - Some(num_shard) => { - // we can't have only one shard while sharded - if num_shard <= 1 { - panic!("`sharded` is true but `num_shard` <= 1"); - } - num_shard - } + // OpenTelemetry + if let Some(otlp_endpoint) = otlp_endpoint { + shard_argv.push("--otlp-endpoint".to_string()); + shard_argv.push(otlp_endpoint); + } + + // Copy current process env + let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); + + // Torch Distributed Env vars + env.push(("RANK".into(), rank.to_string().into())); + env.push(("WORLD_SIZE".into(), world_size.to_string().into())); + env.push(("MASTER_ADDR".into(), master_addr.into())); + env.push(("MASTER_PORT".into(), master_port.to_string().into())); + env.push(("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())); + + // Safetensors load fast + env.push(("SAFETENSORS_FAST_GPU".into(), "1".into())); + + // Enable hf transfer for insane download speeds + let enable_hf_transfer = env::var("HF_HUB_ENABLE_HF_TRANSFER").unwrap_or("1".to_string()); + env.push(( + "HF_HUB_ENABLE_HF_TRANSFER".into(), + enable_hf_transfer.into(), + )); + + // Parse Inference API token + if let Ok(api_token) = env::var("HF_API_TOKEN") { + env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) + }; + + // If huggingface_hub_cache is some, pass it to the shard + // Useful when running inside a docker container + if let Some(huggingface_hub_cache) = huggingface_hub_cache { + env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); + }; + + // If weights_cache_override is some, pass it to the shard + // Useful when running inside a HuggingFace Inference Endpoint + if let Some(weights_cache_override) = weights_cache_override { + env.push(( + "WEIGHTS_CACHE_OVERRIDE".into(), + weights_cache_override.into(), + )); + }; + + // If disable_custom_kernels is true, pass it to the shard as an env var + if disable_custom_kernels { + env.push(("DISABLE_CUSTOM_KERNELS".into(), "True".into())) + } + + // Watermark Gamma + if let Some(watermark_gamma) = watermark_gamma { + env.push(("WATERMARK_GAMMA".into(), watermark_gamma.to_string().into())) + } + + // Watermark Delta + if let Some(watermark_delta) = watermark_delta { + env.push(("WATERMARK_DELTA".into(), watermark_delta.to_string().into())) + } + + // Start process + tracing::info!("Starting shard {rank}"); + let mut p = match Popen::create( + &shard_argv, + PopenConfig { + stdout: Redirection::Pipe, + stderr: Redirection::Pipe, + // Needed for the shutdown procedure + setpgid: true, + // NCCL env vars + env: Some(env), + ..Default::default() + }, + ) { + Ok(p) => p, + Err(err) => { + if let PopenError::IoError(ref err) = err { + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("text-generation-server not found in PATH"); + tracing::error!("Please install it with `make install-server`") } } - // sharded is set and false - false => { - let num_shard = num_shard.unwrap_or(1); - // we can't have more than one shard while not sharded - if num_shard != 1 { - panic!("`sharded` is false but `num_shard` != 1"); - } - num_shard + status_sender + .send(ShardStatus::Failed((rank, err.to_string()))) + .unwrap(); + return; + } + }; + + // Redirect STDOUT to the console + let shard_stdout = p.stdout.take().unwrap(); + + thread::spawn(move || { + // Enter shard-manager tracing span + let stdout = BufReader::new(shard_stdout); + let _span = tracing::span!(tracing::Level::INFO, "shard-manager", rank = rank).entered(); + for line in stdout.lines() { + // Parse loguru logs + if let Ok(log) = serde_json::from_str::(&line.unwrap()) { + log.trace(); + } + } + }); + + let mut ready = false; + let start_time = Instant::now(); + let mut wait_time = Instant::now(); + loop { + // Process exited + if p.poll().is_some() { + let mut err = String::new(); + p.stderr.take().unwrap().read_to_string(&mut err).unwrap(); + status_sender + .send(ShardStatus::Failed((rank, err))) + .unwrap(); + return; + } + + // We received a shutdown signal + if *shutdown.lock().unwrap() { + p.terminate().unwrap(); + let _ = p.wait_timeout(Duration::from_secs(90)); + tracing::info!("Shard {rank} terminated"); + return; + } + + // Shard is ready + if uds.exists() && !ready { + tracing::info!("Shard {rank} ready in {:?}", start_time.elapsed()); + status_sender.send(ShardStatus::Ready).unwrap(); + ready = true; + } else if !ready && wait_time.elapsed() > Duration::from_secs(10) { + tracing::info!("Waiting for shard {rank} to be ready..."); + wait_time = Instant::now(); + } + sleep(Duration::from_millis(100)); + } +} + +fn shutdown_shards(shutdown: Arc>, shutdown_receiver: &mpsc::Receiver<()>) { + tracing::info!("Shutting down shards"); + // Update shutdown value to true + // This will be picked up by the shard manager + { + let mut shutdown = shutdown.lock().unwrap(); + *shutdown = true; + } + + // Wait for shards to shutdown + // This will block till all shutdown_sender are dropped + let _ = shutdown_receiver.recv(); +} + +fn num_cuda_devices() -> Option { + if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { + let n_devices = cuda_visible_devices.split(',').count(); + return Some(n_devices); + } + None +} + +#[derive(Deserialize)] +#[serde(rename_all = "UPPERCASE")] +enum PythonLogLevelEnum { + Trace, + Debug, + Info, + Success, + Warning, + Error, + Critical, +} + +#[derive(Deserialize)] +struct PythonLogLevel { + name: PythonLogLevelEnum, +} + +#[derive(Deserialize)] +struct PythonLogRecord { + level: PythonLogLevel, +} + +#[derive(Deserialize)] +struct PythonLogMessage { + text: String, + record: PythonLogRecord, +} + +impl PythonLogMessage { + fn trace(&self) { + match self.record.level.name { + PythonLogLevelEnum::Trace => tracing::trace!("{}", self.text), + PythonLogLevelEnum::Debug => tracing::debug!("{}", self.text), + PythonLogLevelEnum::Info => tracing::info!("{}", self.text), + PythonLogLevelEnum::Success => tracing::info!("{}", self.text), + PythonLogLevelEnum::Warning => tracing::warn!("{}", self.text), + PythonLogLevelEnum::Error => tracing::error!("{}", self.text), + PythonLogLevelEnum::Critical => tracing::error!("{}", self.text), + } + } +} + +fn find_num_shards(sharded: Option, num_shard: Option) -> usize { + // get the number of shards given `sharded` and `num_shard` + let num_shard = match (sharded, num_shard) { + (Some(true), None) => { + // try to default to the number of available GPUs + tracing::info!("Parsing num_shard from CUDA_VISIBLE_DEVICES"); + let n_devices = + num_cuda_devices().expect("--num-shard and CUDA_VISIBLE_DEVICES are not set"); + if n_devices <= 1 { + panic!("`sharded` is true but only found {n_devices} CUDA devices"); } + n_devices } - } else { - match num_shard { - // get num_shard from CUDA_VISIBLE_DEVICES or default to a single shard - None => num_cuda_devices().unwrap_or(1), - Some(num_shard) => num_shard, + (Some(true), Some(num_shard)) => { + // we can't have only one shard while sharded + if num_shard <= 1 { + panic!("`sharded` is true but `num_shard` <= 1"); + } + num_shard } + (Some(false), Some(num_shard)) => num_shard, + (Some(false), None) => 1, + (None, None) => num_cuda_devices().unwrap_or(1), + (None, Some(num_shard)) => num_shard, }; if num_shard < 1 { panic!("`num_shard` cannot be < 1"); } + num_shard +} - if num_shard > 1 { - tracing::info!("Sharding model on {num_shard} processes"); - } - - // Signal handler - let running = Arc::new(AtomicBool::new(true)); - let r = running.clone(); - ctrlc::set_handler(move || { - r.store(false, Ordering::SeqCst); - }) - .expect("Error setting Ctrl-C handler"); +#[derive(Debug)] +enum LauncherError { + DownloadError, + ShardCannotStart, + ShardDisconnected, + ShardFailed, + WebserverFailed, + WebserverCannotStart, +} - // Check if model_id is a local model - let local_path = Path::new(&model_id); - let is_local_model = local_path.exists() && local_path.is_dir(); +fn download_model(args: &Args, running: Arc) -> Result<(), LauncherError> { + let mut download_argv = vec![ + "text-generation-server".to_string(), + "download-weights".to_string(), + args.model_id.to_string(), + "--extension".to_string(), + ".safetensors".to_string(), + "--logger-level".to_string(), + "INFO".to_string(), + "--json-output".to_string(), + ]; - // Download weights for sharded models - if !is_local_model && weights_cache_override.is_none() && num_shard > 1 { - let mut download_argv = vec![ - "text-generation-server".to_string(), - "download-weights".to_string(), - model_id.clone(), - "--extension".to_string(), - ".safetensors".to_string(), - "--logger-level".to_string(), - "INFO".to_string(), - "--json-output".to_string(), - ]; - - // Model optional revision - if let Some(ref revision) = revision { - download_argv.push("--revision".to_string()); - download_argv.push(revision.to_string()) - } + // Model optional revision + if let Some(revision) = &args.revision { + download_argv.push("--revision".to_string()); + download_argv.push(revision.to_string()) + } - // Copy current process env - let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); + // Copy current process env + let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); - // If huggingface_hub_cache is set, pass it to the shard - // Useful when running inside a docker container - if let Some(ref huggingface_hub_cache) = huggingface_hub_cache { - env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); - }; + // If huggingface_hub_cache is set, pass it to the shard + // Useful when running inside a docker container + if let Some(ref huggingface_hub_cache) = args.huggingface_hub_cache { + env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); + }; - // Enable hf transfer for insane download speeds - let enable_hf_transfer = env::var("HF_HUB_ENABLE_HF_TRANSFER").unwrap_or("1".to_string()); - env.push(( - "HF_HUB_ENABLE_HF_TRANSFER".into(), - enable_hf_transfer.into(), - )); + // Enable hf transfer for insane download speeds + let enable_hf_transfer = env::var("HF_HUB_ENABLE_HF_TRANSFER").unwrap_or("1".to_string()); + env.push(( + "HF_HUB_ENABLE_HF_TRANSFER".into(), + enable_hf_transfer.into(), + )); - // Parse Inference API token - if let Ok(api_token) = env::var("HF_API_TOKEN") { - env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) - }; + // Parse Inference API token + if let Ok(api_token) = env::var("HF_API_TOKEN") { + env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) + }; - // Start process - tracing::info!("Starting download process."); - let mut download_process = match Popen::create( - &download_argv, - PopenConfig { - stdout: Redirection::Pipe, - stderr: Redirection::Pipe, - // Needed for the shutdown procedure - setpgid: true, - env: Some(env), - ..Default::default() - }, - ) { - Ok(p) => p, - Err(err) => { - if let PopenError::IoError(ref err) = err { - if err.kind() == io::ErrorKind::NotFound { - tracing::error!("text-generation-server not found in PATH"); - tracing::error!("Please install it with `make install-server`") - } + // Start process + tracing::info!("Starting download process."); + let mut download_process = match Popen::create( + &download_argv, + PopenConfig { + stdout: Redirection::Pipe, + stderr: Redirection::Pipe, + // Needed for the shutdown procedure + setpgid: true, + env: Some(env), + ..Default::default() + }, + ) { + Ok(p) => p, + Err(err) => { + if let PopenError::IoError(ref err) = err { + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("text-generation-server not found in PATH"); + tracing::error!("Please install it with `make install-server`") } - return ExitCode::FAILURE; } - }; + return Err(LauncherError::DownloadError); + } + }; - // Redirect STDOUT to the console - let download_stdout = download_process.stdout.take().unwrap(); - thread::spawn(move || { - // Enter download tracing span - let stdout = BufReader::new(download_stdout); - let _span = tracing::span!(tracing::Level::INFO, "download").entered(); - for line in stdout.lines() { - // Parse loguru logs - if let Ok(log) = serde_json::from_str::(&line.unwrap()) { - log.trace(); - } + // Redirect STDOUT to the console + let download_stdout = download_process.stdout.take().unwrap(); + thread::spawn(move || { + // Enter download tracing span + let stdout = BufReader::new(download_stdout); + let _span = tracing::span!(tracing::Level::INFO, "download").entered(); + for line in stdout.lines() { + // Parse loguru logs + if let Ok(log) = serde_json::from_str::(&line.unwrap()) { + log.trace(); } - }); + } + }); - loop { - if let Some(status) = download_process.poll() { - match status { - ExitStatus::Exited(exit_code) => { - if exit_code == 0 { - tracing::info!("Successfully downloaded weights."); - break; - } else { - let mut err = String::new(); - download_process - .stderr - .take() - .unwrap() - .read_to_string(&mut err) - .unwrap(); - tracing::error!("Download encountered an error: {err}"); - return ExitCode::FAILURE; - } - } - _ => { - tracing::error!("Download process exited with an unknown status."); - return ExitCode::FAILURE; + loop { + if let Some(status) = download_process.poll() { + match status { + ExitStatus::Exited(exit_code) => { + if exit_code == 0 { + tracing::info!("Successfully downloaded weights."); + break; + } else { + let mut err = String::new(); + download_process + .stderr + .take() + .unwrap() + .read_to_string(&mut err) + .unwrap(); + tracing::error!("Download encountered an error: {err}"); + return Err(LauncherError::DownloadError); } } + _ => { + tracing::error!("Download process exited with an unknown status."); + return Err(LauncherError::DownloadError); + } } - if !running.load(Ordering::SeqCst) { - download_process.terminate().unwrap(); - tracing::info!("Waiting for download process to gracefully shutdown"); - download_process - .wait_timeout(Duration::from_secs(90)) - .unwrap(); - tracing::info!("Download process terminated"); - return ExitCode::SUCCESS; - } - sleep(Duration::from_millis(100)); } + if !running.load(Ordering::SeqCst) { + download_process.terminate().unwrap(); + tracing::info!("Waiting for download process to gracefully shutdown"); + download_process + .wait_timeout(Duration::from_secs(90)) + .unwrap(); + tracing::info!("Download process terminated"); + return Ok(()); + } + sleep(Duration::from_millis(100)); } + Ok(()) +} - // Shared shutdown bool - let shutdown = Arc::new(Mutex::new(false)); - // Shared shutdown channel - // When shutting down, the main thread will wait for all senders to be dropped - let (shutdown_sender, shutdown_receiver) = mpsc::channel(); - - // Shared channel to track shard status - let (status_sender, status_receiver) = mpsc::channel(); - +fn spawn_shards( + num_shard: usize, + args: &Args, + shutdown: Arc>, + shutdown_receiver: &mpsc::Receiver<()>, + shutdown_sender: mpsc::Sender<()>, + status_receiver: &mpsc::Receiver, + status_sender: mpsc::Sender, + running: Arc, +) -> Result<(), LauncherError> { // Start shard processes for rank in 0..num_shard { - let model_id = model_id.clone(); - let revision = revision.clone(); - let uds_path = shard_uds_path.clone(); - let master_addr = master_addr.clone(); - let huggingface_hub_cache = huggingface_hub_cache.clone(); - let weights_cache_override = weights_cache_override.clone(); + let model_id = args.model_id.clone(); + let revision = args.revision.clone(); + let uds_path = args.shard_uds_path.clone(); + let master_addr = args.master_addr.clone(); + let huggingface_hub_cache = args.huggingface_hub_cache.clone(); + let weights_cache_override = args.weights_cache_override.clone(); let status_sender = status_sender.clone(); let shutdown = shutdown.clone(); let shutdown_sender = shutdown_sender.clone(); - let otlp_endpoint = otlp_endpoint.clone(); + let otlp_endpoint = args.otlp_endpoint.clone(); + let quantize = args.quantize.clone(); + let master_port = args.master_port.clone(); + let disable_custom_kernels = args.disable_custom_kernels.clone(); + let watermark_gamma = args.watermark_gamma.clone(); + let watermark_delta = args.watermark_delta.clone(); thread::spawn(move || { shard_manager( model_id, @@ -355,422 +560,224 @@ fn main() -> ExitCode { Ok(ShardStatus::Failed((rank, err))) => { tracing::error!("Shard {} failed to start:\n{}", rank, err); shutdown_shards(shutdown, &shutdown_receiver); - return ExitCode::FAILURE; + return Err(LauncherError::ShardCannotStart); } Err(TryRecvError::Disconnected) => { tracing::error!("Shard status channel disconnected"); shutdown_shards(shutdown, &shutdown_receiver); - return ExitCode::FAILURE; + return Err(LauncherError::ShardDisconnected); } } } + Ok(()) +} - // We might have received a termination signal - if !running.load(Ordering::SeqCst) { - shutdown_shards(shutdown, &shutdown_receiver); - return ExitCode::SUCCESS; - } - +fn spawn_webserver( + args: Args, + shutdown: Arc>, + shutdown_receiver: &mpsc::Receiver<()>, +) -> Result { // All shard started // Start webserver tracing::info!("Starting Webserver"); let mut argv = vec![ "text-generation-router".to_string(), "--max-concurrent-requests".to_string(), - max_concurrent_requests.to_string(), + args.max_concurrent_requests.to_string(), "--max-best-of".to_string(), - max_best_of.to_string(), + args.max_best_of.to_string(), "--max-stop-sequences".to_string(), - max_stop_sequences.to_string(), + args.max_stop_sequences.to_string(), "--max-input-length".to_string(), - max_input_length.to_string(), + args.max_input_length.to_string(), "--max-total-tokens".to_string(), - max_total_tokens.to_string(), + args.max_total_tokens.to_string(), "--waiting-served-ratio".to_string(), - waiting_served_ratio.to_string(), + args.waiting_served_ratio.to_string(), "--max-waiting-tokens".to_string(), - max_waiting_tokens.to_string(), + args.max_waiting_tokens.to_string(), "--port".to_string(), - port.to_string(), + args.port.to_string(), "--master-shard-uds-path".to_string(), - format!("{shard_uds_path}-0"), + format!("{}-0", args.shard_uds_path), "--tokenizer-name".to_string(), - model_id, + args.model_id, ]; // Deprecate max_batch_size - if let Some(max_batch_size) = max_batch_size { + if let Some(max_batch_size) = args.max_batch_size { argv.push("--max-batch-size".to_string()); - argv.push(max_batch_size.to_string()) - } else { - argv.push("--max-batch-total-tokens".to_string()); - argv.push(max_batch_total_tokens.to_string()) - } - - // Model optional revision - if let Some(ref revision) = revision { - argv.push("--revision".to_string()); - argv.push(revision.to_string()) - } - - if json_output { - argv.push("--json-output".to_string()); - } - - // OpenTelemetry - if let Some(otlp_endpoint) = otlp_endpoint { - argv.push("--otlp-endpoint".to_string()); - argv.push(otlp_endpoint); - } - - // CORS origins - for origin in cors_allow_origin.into_iter() { - argv.push("--cors-allow-origin".to_string()); - argv.push(origin); - } - - // Copy current process env - let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); - - // Parse Inference API token - if let Ok(api_token) = env::var("HF_API_TOKEN") { - env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) - }; - - let mut webserver = match Popen::create( - &argv, - PopenConfig { - stdout: Redirection::Pipe, - stderr: Redirection::Pipe, - // Needed for the shutdown procedure - setpgid: true, - env: Some(env), - ..Default::default() - }, - ) { - Ok(p) => p, - Err(err) => { - tracing::error!("Failed to start webserver: {}", err); - if let PopenError::IoError(err) = err { - if err.kind() == io::ErrorKind::NotFound { - tracing::error!("text-generation-router not found in PATH"); - tracing::error!("Please install it with `make install-router`") - } - } else { - tracing::error!("{}", err); - } - - shutdown_shards(shutdown, &shutdown_receiver); - return ExitCode::FAILURE; - } - }; - - // Redirect STDOUT and STDERR to the console - let webserver_stdout = webserver.stdout.take().unwrap(); - let webserver_stderr = webserver.stderr.take().unwrap(); - - thread::spawn(move || { - let stdout = BufReader::new(webserver_stdout); - let stderr = BufReader::new(webserver_stderr); - for line in stdout.lines() { - println!("{}", line.unwrap()); - } - for line in stderr.lines() { - println!("{}", line.unwrap()); - } - }); - - // Default exit code - let mut exit_code = ExitCode::SUCCESS; - - while running.load(Ordering::SeqCst) { - if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { - tracing::error!("Shard {rank} failed:\n{err}"); - exit_code = ExitCode::FAILURE; - break; - }; - - match webserver.poll() { - Some(_) => { - tracing::error!("Webserver Crashed"); - shutdown_shards(shutdown, &shutdown_receiver); - return ExitCode::FAILURE; - } - None => { - sleep(Duration::from_millis(100)); - } - }; - } - - // Graceful termination - webserver.terminate().unwrap(); - tracing::info!("Waiting for webserver to gracefully shutdown"); - webserver.wait_timeout(Duration::from_secs(90)).unwrap(); - tracing::info!("Webserver terminated"); - shutdown_shards(shutdown, &shutdown_receiver); - - exit_code -} - -#[derive(Debug)] -enum ShardStatus { - Ready, - Failed((usize, String)), -} - -#[allow(clippy::too_many_arguments)] -fn shard_manager( - model_id: String, - revision: Option, - quantize: bool, - uds_path: String, - rank: usize, - world_size: usize, - master_addr: String, - master_port: usize, - huggingface_hub_cache: Option, - weights_cache_override: Option, - disable_custom_kernels: bool, - watermark_gamma: Option, - watermark_delta: Option, - otlp_endpoint: Option, - status_sender: mpsc::Sender, - shutdown: Arc>, - _shutdown_sender: mpsc::Sender<()>, -) { - // Get UDS path - let uds_string = format!("{uds_path}-{rank}"); - let uds = Path::new(&uds_string); - // Clean previous runs - fs::remove_file(uds).unwrap_or_default(); - - // Process args - let mut shard_argv = vec![ - "text-generation-server".to_string(), - "serve".to_string(), - model_id, - "--uds-path".to_string(), - uds_path, - "--logger-level".to_string(), - "INFO".to_string(), - "--json-output".to_string(), - ]; - - // Activate tensor parallelism - if world_size > 1 { - shard_argv.push("--sharded".to_string()); + argv.push(max_batch_size.to_string()) + } else { + argv.push("--max-batch-total-tokens".to_string()); + argv.push(args.max_batch_total_tokens.to_string()) } - if quantize { - shard_argv.push("--quantize".to_string()) + // Model optional revision + if let Some(ref revision) = args.revision { + argv.push("--revision".to_string()); + argv.push(revision.to_string()) } - // Model optional revision - if let Some(revision) = revision { - shard_argv.push("--revision".to_string()); - shard_argv.push(revision) + if args.json_output { + argv.push("--json-output".to_string()); } // OpenTelemetry - if let Some(otlp_endpoint) = otlp_endpoint { - shard_argv.push("--otlp-endpoint".to_string()); - shard_argv.push(otlp_endpoint); + if let Some(otlp_endpoint) = args.otlp_endpoint { + argv.push("--otlp-endpoint".to_string()); + argv.push(otlp_endpoint); + } + + // CORS origins + for origin in args.cors_allow_origin.into_iter() { + argv.push("--cors-allow-origin".to_string()); + argv.push(origin); } // Copy current process env let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); - // Torch Distributed Env vars - env.push(("RANK".into(), rank.to_string().into())); - env.push(("WORLD_SIZE".into(), world_size.to_string().into())); - env.push(("MASTER_ADDR".into(), master_addr.into())); - env.push(("MASTER_PORT".into(), master_port.to_string().into())); - env.push(("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())); - - // Safetensors load fast - env.push(("SAFETENSORS_FAST_GPU".into(), "1".into())); - - // Enable hf transfer for insane download speeds - let enable_hf_transfer = env::var("HF_HUB_ENABLE_HF_TRANSFER").unwrap_or("1".to_string()); - env.push(( - "HF_HUB_ENABLE_HF_TRANSFER".into(), - enable_hf_transfer.into(), - )); - // Parse Inference API token if let Ok(api_token) = env::var("HF_API_TOKEN") { env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) }; - // If huggingface_hub_cache is some, pass it to the shard - // Useful when running inside a docker container - if let Some(huggingface_hub_cache) = huggingface_hub_cache { - env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); - }; - - // If weights_cache_override is some, pass it to the shard - // Useful when running inside a HuggingFace Inference Endpoint - if let Some(weights_cache_override) = weights_cache_override { - env.push(( - "WEIGHTS_CACHE_OVERRIDE".into(), - weights_cache_override.into(), - )); - }; - - // If disable_custom_kernels is true, pass it to the shard as an env var - if disable_custom_kernels { - env.push(("DISABLE_CUSTOM_KERNELS".into(), "True".into())) - } - - // Watermark Gamma - if let Some(watermark_gamma) = watermark_gamma { - env.push(("WATERMARK_GAMMA".into(), watermark_gamma.to_string().into())) - } - - // Watermark Delta - if let Some(watermark_delta) = watermark_delta { - env.push(("WATERMARK_DELTA".into(), watermark_delta.to_string().into())) - } - - // Start process - tracing::info!("Starting shard {rank}"); - let mut p = match Popen::create( - &shard_argv, + let mut webserver = match Popen::create( + &argv, PopenConfig { stdout: Redirection::Pipe, stderr: Redirection::Pipe, // Needed for the shutdown procedure setpgid: true, - // NCCL env vars env: Some(env), ..Default::default() }, ) { Ok(p) => p, Err(err) => { - if let PopenError::IoError(ref err) = err { + tracing::error!("Failed to start webserver: {}", err); + if let PopenError::IoError(err) = err { if err.kind() == io::ErrorKind::NotFound { - tracing::error!("text-generation-server not found in PATH"); - tracing::error!("Please install it with `make install-server`") + tracing::error!("text-generation-router not found in PATH"); + tracing::error!("Please install it with `make install-router`") } + } else { + tracing::error!("{}", err); } - status_sender - .send(ShardStatus::Failed((rank, err.to_string()))) - .unwrap(); - return; + + shutdown_shards(shutdown, &shutdown_receiver); + return Err(LauncherError::WebserverCannotStart); } }; - // Redirect STDOUT to the console - let shard_stdout = p.stdout.take().unwrap(); + // Redirect STDOUT and STDERR to the console + let webserver_stdout = webserver.stdout.take().unwrap(); + let webserver_stderr = webserver.stderr.take().unwrap(); thread::spawn(move || { - // Enter shard-manager tracing span - let stdout = BufReader::new(shard_stdout); - let _span = tracing::span!(tracing::Level::INFO, "shard-manager", rank = rank).entered(); + let stdout = BufReader::new(webserver_stdout); + let stderr = BufReader::new(webserver_stderr); for line in stdout.lines() { - // Parse loguru logs - if let Ok(log) = serde_json::from_str::(&line.unwrap()) { - log.trace(); - } + println!("{}", line.unwrap()); } - }); - - let mut ready = false; - let start_time = Instant::now(); - let mut wait_time = Instant::now(); - loop { - // Process exited - if p.poll().is_some() { - let mut err = String::new(); - p.stderr.take().unwrap().read_to_string(&mut err).unwrap(); - status_sender - .send(ShardStatus::Failed((rank, err))) - .unwrap(); - return; + for line in stderr.lines() { + println!("{}", line.unwrap()); } + }); + Ok(webserver) +} - // We received a shutdown signal - if *shutdown.lock().unwrap() { - p.terminate().unwrap(); - let _ = p.wait_timeout(Duration::from_secs(90)); - tracing::info!("Shard {rank} terminated"); - return; - } +fn main() -> Result<(), LauncherError> { + // Pattern match configuration + let args = Args::parse(); - // Shard is ready - if uds.exists() && !ready { - tracing::info!("Shard {rank} ready in {:?}", start_time.elapsed()); - status_sender.send(ShardStatus::Ready).unwrap(); - ready = true; - } else if !ready && wait_time.elapsed() > Duration::from_secs(10) { - tracing::info!("Waiting for shard {rank} to be ready..."); - wait_time = Instant::now(); - } - sleep(Duration::from_millis(100)); + if args.json_output { + tracing_subscriber::fmt().json().init(); + } else { + tracing_subscriber::fmt().compact().init(); } -} -fn shutdown_shards(shutdown: Arc>, shutdown_receiver: &mpsc::Receiver<()>) { - tracing::info!("Shutting down shards"); - // Update shutdown value to true - // This will be picked up by the shard manager - { - let mut shutdown = shutdown.lock().unwrap(); - *shutdown = true; + tracing::info!("{:?}", args); + + let num_shard = find_num_shards(args.sharded, args.num_shard); + if num_shard > 1 { + tracing::info!("Sharding model on {num_shard} processes"); } - // Wait for shards to shutdown - // This will block till all shutdown_sender are dropped - let _ = shutdown_receiver.recv(); -} + // Signal handler + let running = Arc::new(AtomicBool::new(true)); + let r = running.clone(); + ctrlc::set_handler(move || { + r.store(false, Ordering::SeqCst); + }) + .expect("Error setting Ctrl-C handler"); -fn num_cuda_devices() -> Option { - if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { - let n_devices = cuda_visible_devices.split(',').count(); - return Some(n_devices); + // Check if model_id is a local model + let local_path = Path::new(&args.model_id); + let is_local_model = local_path.exists() && local_path.is_dir(); + + // Download weights for sharded models + if !is_local_model && args.weights_cache_override.is_none() && num_shard > 1 { + download_model(&args, running.clone())?; } - None -} -#[derive(Deserialize)] -#[serde(rename_all = "UPPERCASE")] -enum PythonLogLevelEnum { - Trace, - Debug, - Info, - Success, - Warning, - Error, - Critical, -} + // Shared shutdown bool + let shutdown = Arc::new(Mutex::new(false)); + // Shared shutdown channel + // When shutting down, the main thread will wait for all senders to be dropped + let (shutdown_sender, shutdown_receiver) = mpsc::channel(); -#[derive(Deserialize)] -struct PythonLogLevel { - name: PythonLogLevelEnum, -} + // Shared channel to track shard status + let (status_sender, status_receiver) = mpsc::channel(); -#[derive(Deserialize)] -struct PythonLogRecord { - level: PythonLogLevel, -} + spawn_shards( + num_shard, + &args, + shutdown.clone(), + &shutdown_receiver, + shutdown_sender, + &status_receiver, + status_sender, + running.clone(), + )?; -#[derive(Deserialize)] -struct PythonLogMessage { - text: String, - record: PythonLogRecord, -} + // We might have received a termination signal + if !running.load(Ordering::SeqCst) { + shutdown_shards(shutdown, &shutdown_receiver); + return Ok(()); + } -impl PythonLogMessage { - fn trace(&self) { - match self.record.level.name { - PythonLogLevelEnum::Trace => tracing::trace!("{}", self.text), - PythonLogLevelEnum::Debug => tracing::debug!("{}", self.text), - PythonLogLevelEnum::Info => tracing::info!("{}", self.text), - PythonLogLevelEnum::Success => tracing::info!("{}", self.text), - PythonLogLevelEnum::Warning => tracing::warn!("{}", self.text), - PythonLogLevelEnum::Error => tracing::error!("{}", self.text), - PythonLogLevelEnum::Critical => tracing::error!("{}", self.text), - } + let mut webserver = spawn_webserver(args, shutdown.clone(), &shutdown_receiver)?; + + // Default exit code + let mut exit_code = Ok(()); + + while running.load(Ordering::SeqCst) { + if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { + tracing::error!("Shard {rank} failed:\n{err}"); + exit_code = Err(LauncherError::ShardFailed); + break; + }; + + match webserver.poll() { + Some(_) => { + tracing::error!("Webserver Crashed"); + shutdown_shards(shutdown, &shutdown_receiver); + return Err(LauncherError::WebserverFailed); + } + None => { + sleep(Duration::from_millis(100)); + } + }; } + + // Graceful termination + webserver.terminate().unwrap(); + tracing::info!("Waiting for webserver to gracefully shutdown"); + webserver.wait_timeout(Duration::from_secs(90)).unwrap(); + tracing::info!("Webserver terminated"); + shutdown_shards(shutdown, &shutdown_receiver); + + exit_code } From c4fb09f2aef0730d88ce7e4ce54eacd7839b48cc Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 26 Apr 2023 16:14:40 +0200 Subject: [PATCH 197/793] feat(router): add tests to validation (#237) --- .github/workflows/tests.yaml | 3 ++ router/src/lib.rs | 16 +++++++ router/src/queue.rs | 11 +++++ router/src/server.rs | 1 + router/src/validation.rs | 81 +++++++++++++++++++++++++++++++----- 5 files changed, 102 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e82e8b20..a2c2b7fb 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -67,6 +67,9 @@ jobs: run: | pip install pytest HF_HUB_ENABLE_HF_TRANSFER=1 pytest -sv server/tests + - name: Run Clippy + run: | + cargo clippy - name: Run Rust tests run: | cargo test diff --git a/router/src/lib.rs b/router/src/lib.rs index 7a1707d9..85b13cfa 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -276,3 +276,19 @@ pub(crate) struct ErrorResponse { pub error: String, pub error_type: String, } + +#[cfg(test)] +mod tests{ + use std::io::Write; + use tokenizers::Tokenizer; + + pub(crate) async fn get_tokenizer() -> Tokenizer{ + if !std::path::Path::new("tokenizer.json").exists(){ + let content = reqwest::get("https://huggingface.co/gpt2/raw/main/tokenizer.json").await.unwrap().bytes().await.unwrap(); + let mut file = std::fs::File::create("tokenizer.json").unwrap(); + file.write_all(&content).unwrap(); + } + Tokenizer::from_file("tokenizer.json").unwrap() + } +} + diff --git a/router/src/queue.rs b/router/src/queue.rs index d970ebf1..d3f118d8 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -141,6 +141,7 @@ impl State { // Get the next batch fn next_batch(&mut self, min_size: Option, token_budget: u32) -> Option { + if self.entries.is_empty() { return None; } @@ -430,7 +431,17 @@ mod tests { let (entry3, _guard3) = default_entry(); queue.append(entry3); + // Not enough requests pending assert!(queue.next_batch(Some(2), 2).await.is_none()); + // Not enough token budget + assert!(queue.next_batch(Some(1), 0).await.is_none()); + // Ok + let (entries2, batch2, _) = queue.next_batch(Some(1), 2).await.unwrap(); + assert_eq!(entries2.len(), 1); + assert!(entries2.contains_key(&2)); + assert!(entries2.get(&2).unwrap().batch_time.is_some()); + assert_eq!(batch2.id, 1); + assert_eq!(batch2.size, 1); } #[tokio::test] diff --git a/router/src/server.rs b/router/src/server.rs index 9540ba18..09b5c3ba 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -741,3 +741,4 @@ impl From for Event { .unwrap() } } + diff --git a/router/src/validation.rs b/router/src/validation.rs index 983c2612..ff2fe89d 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -382,7 +382,8 @@ pub enum ValidationError { #[cfg(test)] mod tests{ use super::*; - use std::io::Write; + use crate::default_parameters; + use crate::tests::get_tokenizer; #[tokio::test] async fn test_validation_max_new_tokens(){ @@ -401,15 +402,6 @@ mod tests{ } } - async fn get_tokenizer() -> Tokenizer{ - if !std::path::Path::new("tokenizer.json").exists(){ - let content = reqwest::get("https://huggingface.co/gpt2/raw/main/tokenizer.json").await.unwrap().bytes().await.unwrap(); - let mut file = std::fs::File::create("tokenizer.json").unwrap(); - file.write_all(&content).unwrap(); - } - Tokenizer::from_file("tokenizer.json").unwrap() - } - #[tokio::test] async fn test_validation_input_length(){ let tokenizer = Some(get_tokenizer().await); @@ -426,4 +418,73 @@ mod tests{ _ => panic!("Unexpected not max new tokens") } } + + #[tokio::test] + async fn test_validation_best_of_sampling(){ + let tokenizer = Some(get_tokenizer().await); + let max_best_of = 2; + let max_stop_sequence = 3; + let max_input_length = 4; + let max_total_tokens = 5; + let workers = 1; + let validation = Validation::new(workers, tokenizer, max_best_of, max_stop_sequence, max_input_length, max_total_tokens); + match validation.validate(GenerateRequest{ + inputs: "Hello".to_string(), + parameters: GenerateParameters{ + best_of: Some(2), + do_sample: false, + ..default_parameters() + } + }).await{ + Err(ValidationError::BestOfSampling) => (), + _ => panic!("Unexpected not best of sampling") + } + + } + + #[tokio::test] + async fn test_validation_top_p(){ + let tokenizer = Some(get_tokenizer().await); + let max_best_of = 2; + let max_stop_sequence = 3; + let max_input_length = 4; + let max_total_tokens = 5; + let workers = 1; + let validation = Validation::new(workers, tokenizer, max_best_of, max_stop_sequence, max_input_length, max_total_tokens); + match validation.validate(GenerateRequest{ + inputs: "Hello".to_string(), + parameters: GenerateParameters{ + top_p: Some(1.0), + ..default_parameters() + } + }).await{ + Err(ValidationError::TopP) => (), + _ => panic!("Unexpected top_p") + } + + match validation.validate(GenerateRequest{ + inputs: "Hello".to_string(), + parameters: GenerateParameters{ + top_p: Some(0.99), + max_new_tokens: 1, + ..default_parameters() + } + }).await{ + Ok(_) => (), + _ => panic!("Unexpected top_p error") + } + + let valid_request = validation.validate(GenerateRequest{ + inputs: "Hello".to_string(), + parameters: GenerateParameters{ + top_p: None, + max_new_tokens: 1, + ..default_parameters() + } + }).await.unwrap(); + // top_p == 1.0 is invalid for users to ask for but it's the default resolved value. + assert_eq!(valid_request.parameters.top_p, 1.0); + + + } } From db2b4e07544a238428999ab4a2509758eeee422c Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 26 Apr 2023 20:23:54 +0200 Subject: [PATCH 198/793] feat(router): new healthcheck that skips the queue (#244) Co-authored-by: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Co-authored-by: OlivierDehaene --- .github/workflows/tests.yaml | 5 +- launcher/src/main.rs | 17 +-- proto/generate.proto | 7 +- router/client/src/client.rs | 10 +- router/client/src/lib.rs | 1 + router/client/src/sharded_client.rs | 14 ++- router/src/health.rs | 62 ++++++++++ router/src/infer.rs | 29 ++++- router/src/lib.rs | 17 ++- router/src/queue.rs | 1 - router/src/server.rs | 59 +++++----- router/src/validation.rs | 143 +++++++++++++++--------- server/text_generation_server/server.py | 5 + 13 files changed, 265 insertions(+), 105 deletions(-) create mode 100644 router/src/health.rs diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a2c2b7fb..26cbf3b2 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -67,7 +67,10 @@ jobs: run: | pip install pytest HF_HUB_ENABLE_HF_TRANSFER=1 pytest -sv server/tests - - name: Run Clippy + - name: Run Rust fmt + run: | + cargo fmt --check + - name: Run Rust clippy run: | cargo clippy - name: Run Rust tests diff --git a/launcher/src/main.rs b/launcher/src/main.rs index b59b0cb4..7b5f908a 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -493,6 +493,7 @@ fn download_model(args: &Args, running: Arc) -> Result<(), LauncherE Ok(()) } +#[allow(clippy::too_many_arguments)] fn spawn_shards( num_shard: usize, args: &Args, @@ -515,11 +516,11 @@ fn spawn_shards( let shutdown = shutdown.clone(); let shutdown_sender = shutdown_sender.clone(); let otlp_endpoint = args.otlp_endpoint.clone(); - let quantize = args.quantize.clone(); - let master_port = args.master_port.clone(); - let disable_custom_kernels = args.disable_custom_kernels.clone(); - let watermark_gamma = args.watermark_gamma.clone(); - let watermark_delta = args.watermark_delta.clone(); + let quantize = args.quantize; + let master_port = args.master_port; + let disable_custom_kernels = args.disable_custom_kernels; + let watermark_gamma = args.watermark_gamma; + let watermark_delta = args.watermark_delta; thread::spawn(move || { shard_manager( model_id, @@ -559,12 +560,12 @@ fn spawn_shards( } Ok(ShardStatus::Failed((rank, err))) => { tracing::error!("Shard {} failed to start:\n{}", rank, err); - shutdown_shards(shutdown, &shutdown_receiver); + shutdown_shards(shutdown, shutdown_receiver); return Err(LauncherError::ShardCannotStart); } Err(TryRecvError::Disconnected) => { tracing::error!("Shard status channel disconnected"); - shutdown_shards(shutdown, &shutdown_receiver); + shutdown_shards(shutdown, shutdown_receiver); return Err(LauncherError::ShardDisconnected); } } @@ -666,7 +667,7 @@ fn spawn_webserver( tracing::error!("{}", err); } - shutdown_shards(shutdown, &shutdown_receiver); + shutdown_shards(shutdown, shutdown_receiver); return Err(LauncherError::WebserverCannotStart); } }; diff --git a/proto/generate.proto b/proto/generate.proto index ad47409e..894d7bc1 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -15,8 +15,13 @@ service TextGenerationService { rpc Prefill (PrefillRequest) returns (PrefillResponse); /// Decode token for a list of prefilled batches rpc Decode (DecodeRequest) returns (DecodeResponse); + /// Health check + rpc Health (HealthRequest) returns (HealthResponse); } +message HealthRequest {} +message HealthResponse {} + /// Empty request message InfoRequest {} @@ -173,4 +178,4 @@ message DecodeResponse { repeated Generation generations = 1; /// Next batch (cached) optional Batch batch = 2; -} \ No newline at end of file +} diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 7cadf430..bf1b6b58 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -7,7 +7,7 @@ use tonic::transport::{Channel, Uri}; use tracing::instrument; /// Text Generation Inference gRPC client -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct Client { stub: TextGenerationServiceClient, } @@ -62,6 +62,14 @@ impl Client { Ok(response) } + /// Get model health + #[instrument(skip(self))] + pub async fn health(&mut self) -> Result { + let request = tonic::Request::new(HealthRequest {}).inject_context(); + let response = self.stub.health(request).await?.into_inner(); + Ok(response) + } + /// Clear the past generations cache #[instrument(skip(self))] pub async fn clear_cache(&mut self, batch_id: Option) -> Result<()> { diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 6a001306..401082c5 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -6,6 +6,7 @@ mod pb; mod sharded_client; pub use client::Client; +pub use pb::generate::v1::HealthResponse; pub use pb::generate::v1::InfoResponse as ShardInfo; pub use pb::generate::v1::{ Batch, FinishReason, GeneratedText, Generation, NextTokenChooserParameters, PrefillTokens, diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 469d75f6..2f57a437 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,10 +1,11 @@ /// Multi shard Client use crate::Result; -use crate::{Batch, Client, Generation, Request, ShardInfo}; +use crate::{Batch, Client, Generation, HealthResponse, Request, ShardInfo}; use futures::future::join_all; use tonic::transport::Uri; use tracing::instrument; +#[derive(Debug, Clone)] /// Text Generation Inference gRPC multi client pub struct ShardedClient { clients: Vec, @@ -48,6 +49,17 @@ impl ShardedClient { join_all(futures).await.pop().unwrap() } + /// GRPC health check + #[instrument(skip(self))] + pub async fn health(&mut self) -> Result { + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| client.health()) + .collect(); + join_all(futures).await.pop().unwrap() + } + /// Clear the past generations cache #[instrument(skip(self))] pub async fn clear_cache(&mut self, batch_id: Option) -> Result<()> { diff --git a/router/src/health.rs b/router/src/health.rs new file mode 100644 index 00000000..02edf328 --- /dev/null +++ b/router/src/health.rs @@ -0,0 +1,62 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use text_generation_client::{ + Batch, NextTokenChooserParameters, Request, ShardedClient, StoppingCriteriaParameters, +}; + +#[derive(Clone, Debug)] +pub(crate) struct Health { + client: ShardedClient, + generation_health: Arc, +} + +impl Health { + pub(crate) fn new(client: ShardedClient, generation_health: Arc) -> Self { + Self { + client, + generation_health, + } + } + + pub(crate) async fn check(&mut self) -> bool { + if self.generation_health.load(Ordering::SeqCst) { + // Generation is healthy, we only check that the shards are answering gRPC calls + self.client.health().await.is_ok() + } else { + // Generation is unhealthy or have not sent any generation request yet + + // Dummy batch of 1 token and 1 generated token + let liveness_request = Request { + id: u64::MAX, + inputs: "liveness".to_string(), + truncate: 10, + parameters: Some(NextTokenChooserParameters { + temperature: 1.0, + top_k: 0, + top_p: 1.0, + typical_p: 1.0, + do_sample: false, + seed: 0, + repetition_penalty: 1.0, + watermark: false, + }), + stopping_parameters: Some(StoppingCriteriaParameters { + max_new_tokens: 1, + stop_sequences: vec![], + ignore_eos_token: false, + }), + }; + let batch = Batch { + id: u64::MAX, + requests: vec![liveness_request], + size: 1, + max_tokens: 2, + }; + // Skips the queue + let value = self.client.prefill(batch).await.is_ok(); + // Update generation health + self.generation_health.store(value, Ordering::SeqCst); + value + } + } +} diff --git a/router/src/infer.rs b/router/src/infer.rs index 8b44ec86..313ec3e1 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -7,7 +7,10 @@ use flume::SendError; use futures::future::try_join_all; use futures::stream::StreamExt; use nohash_hasher::IntMap; -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; use text_generation_client::{ Batch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, }; @@ -36,6 +39,7 @@ struct Shared { } impl Infer { + #[allow(clippy::too_many_arguments)] pub(crate) fn new( client: ShardedClient, validation: Validation, @@ -44,6 +48,7 @@ impl Infer { max_waiting_tokens: usize, max_concurrent_requests: usize, requires_padding: bool, + generation_health: Arc, ) -> Self { // Infer shared state let queue = Queue::new(requires_padding); @@ -59,6 +64,7 @@ impl Infer { max_waiting_tokens, queue.clone(), shared.clone(), + generation_health, )); // Inference limit with a semaphore @@ -240,6 +246,7 @@ async fn batching_task( max_waiting_tokens: usize, queue: Queue, shared: Arc, + generation_health: Arc, ) { // Infinite loop loop { @@ -252,7 +259,7 @@ async fn batching_task( while let Some((mut entries, batch, span)) = queue.next_batch(None, max_batch_total_tokens).await { - let mut cached_batch = prefill(&mut client, batch, &mut entries) + let mut cached_batch = prefill(&mut client, batch, &mut entries, &generation_health) .instrument(span) .await; let mut waiting_tokens = 1; @@ -301,9 +308,10 @@ async fn batching_task( }); // Generate one token for this new batch to have the attention past in cache - let new_cached_batch = prefill(&mut client, new_batch, &mut new_entries) - .instrument(span) - .await; + let new_cached_batch = + prefill(&mut client, new_batch, &mut new_entries, &generation_health) + .instrument(span) + .await; // Reset waiting counter waiting_tokens = 1; // Extend current batch with the new batch @@ -327,7 +335,7 @@ async fn batching_task( entry.temp_span = Some(entry_batch_span); }); - cached_batch = decode(&mut client, batches, &mut entries) + cached_batch = decode(&mut client, batches, &mut entries, &generation_health) .instrument(next_batch_span) .await; waiting_tokens += 1; @@ -343,6 +351,7 @@ async fn prefill( client: &mut ShardedClient, batch: Batch, entries: &mut IntMap, + generation_health: &Arc, ) -> Option { let start_time = Instant::now(); let batch_id = batch.id; @@ -350,6 +359,8 @@ async fn prefill( match client.prefill(batch).await { Ok((generations, next_batch)) => { + // Update health + generation_health.store(true, Ordering::SeqCst); // Send generated tokens and filter stopped entries filter_send_generations(generations, entries); @@ -362,6 +373,8 @@ async fn prefill( } // If we have an error, we discard the whole batch Err(err) => { + // Update health + generation_health.store(false, Ordering::SeqCst); let _ = client.clear_cache(Some(batch_id)).await; send_errors(err, entries); metrics::increment_counter!("tgi_batch_inference_failure", "method" => "prefill"); @@ -375,6 +388,7 @@ async fn decode( client: &mut ShardedClient, batches: Vec, entries: &mut IntMap, + generation_health: &Arc, ) -> Option { let start_time = Instant::now(); let batch_ids: Vec = batches.iter().map(|b| b.id).collect(); @@ -382,6 +396,8 @@ async fn decode( match client.decode(batches).await { Ok((generations, next_batch)) => { + // Update health + generation_health.store(true, Ordering::SeqCst); // Send generated tokens and filter stopped entries filter_send_generations(generations, entries); @@ -394,6 +410,7 @@ async fn decode( } // If we have an error, we discard the whole batch Err(err) => { + generation_health.store(false, Ordering::SeqCst); for id in batch_ids { let _ = client.clear_cache(Some(id)).await; } diff --git a/router/src/lib.rs b/router/src/lib.rs index 85b13cfa..c2ff669b 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -1,3 +1,4 @@ +mod health; /// Text Generation Inference Webserver mod infer; mod queue; @@ -278,17 +279,21 @@ pub(crate) struct ErrorResponse { } #[cfg(test)] -mod tests{ +mod tests { use std::io::Write; use tokenizers::Tokenizer; - pub(crate) async fn get_tokenizer() -> Tokenizer{ - if !std::path::Path::new("tokenizer.json").exists(){ - let content = reqwest::get("https://huggingface.co/gpt2/raw/main/tokenizer.json").await.unwrap().bytes().await.unwrap(); - let mut file = std::fs::File::create("tokenizer.json").unwrap(); + pub(crate) async fn get_tokenizer() -> Tokenizer { + if !std::path::Path::new("tokenizer.json").exists() { + let content = reqwest::get("https://huggingface.co/gpt2/raw/main/tokenizer.json") + .await + .unwrap() + .bytes() + .await + .unwrap(); + let mut file = std::fs::File::create("tokenizer.json").unwrap(); file.write_all(&content).unwrap(); } Tokenizer::from_file("tokenizer.json").unwrap() } } - diff --git a/router/src/queue.rs b/router/src/queue.rs index d3f118d8..94851e1c 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -141,7 +141,6 @@ impl State { // Get the next batch fn next_batch(&mut self, min_size: Option, token_budget: u32) -> Option { - if self.entries.is_empty() { return None; } diff --git a/router/src/server.rs b/router/src/server.rs index 09b5c3ba..f25fa2b3 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,3 +1,4 @@ +use crate::health::Health; /// HTTP Server logic use crate::infer::{InferError, InferResponse, InferStreamResponse}; use crate::validation::ValidationError; @@ -18,6 +19,8 @@ use futures::Stream; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; use std::convert::Infallible; use std::net::SocketAddr; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; use text_generation_client::{ShardInfo, ShardedClient}; use tokenizers::Tokenizer; use tokio::signal; @@ -82,36 +85,29 @@ async fn get_model_info(info: Extension) -> Json { Json(info.0) } +#[utoipa::path( + get, + tag = "Text Generation Inference", + path = "/health", + responses( + (status = 200, description = "Everything is working fine"), + (status = 503, description = "Text generation inference is down", body = ErrorResponse, + example = json ! ({"error": "unhealthy", "error_type": "healthcheck"})), + ) +)] +#[instrument(skip(health))] /// Health check method -#[instrument(skip(infer))] -async fn health(infer: Extension) -> Result<(), (StatusCode, Json)> { - // TODO: while this is the best health check we can do, it is a bit on the heavy side and might - // be a bit too slow for a health check. - // What we should do instead is check if the gRPC channels are still healthy. - - // Send a small inference request - infer - .generate(GenerateRequest { - inputs: "liveness".to_string(), - parameters: GenerateParameters { - best_of: None, - temperature: None, - repetition_penalty: None, - top_k: None, - top_p: None, - typical_p: None, - do_sample: false, - max_new_tokens: 1, - return_full_text: None, - stop: Vec::new(), - truncate: None, - watermark: false, - details: false, - seed: None, - }, - }) - .await?; - Ok(()) +async fn health(mut health: Extension) -> Result<(), (StatusCode, Json)> { + match health.check().await { + true => Ok(()), + false => Err(( + StatusCode::SERVICE_UNAVAILABLE, + Json(ErrorResponse { + error: "unhealthy".to_string(), + error_type: "healthcheck".to_string(), + }), + )), + } } /// Generate tokens @@ -555,6 +551,8 @@ pub async fn run( max_input_length, max_total_tokens, ); + let generation_health = Arc::new(AtomicBool::new(false)); + let health_ext = Health::new(client.clone(), generation_health.clone()); let infer = Infer::new( client, validation, @@ -563,6 +561,7 @@ pub async fn run( max_waiting_tokens, max_concurrent_requests, shard_info.requires_padding, + generation_health, ); // Duration buckets @@ -657,6 +656,7 @@ pub async fn run( // Prometheus metrics route .route("/metrics", get(metrics)) .layer(Extension(info)) + .layer(Extension(health_ext)) .layer(Extension(compat_return_full_text)) .layer(Extension(infer)) .layer(Extension(prom_handle)) @@ -741,4 +741,3 @@ impl From for Event { .unwrap() } } - diff --git a/router/src/validation.rs b/router/src/validation.rs index ff2fe89d..cbb0d9cd 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -380,111 +380,154 @@ pub enum ValidationError { } #[cfg(test)] -mod tests{ +mod tests { use super::*; use crate::default_parameters; use crate::tests::get_tokenizer; #[tokio::test] - async fn test_validation_max_new_tokens(){ + async fn test_validation_max_new_tokens() { let tokenizer = None; let max_best_of = 2; let max_stop_sequence = 3; let max_input_length = 4; let max_total_tokens = 5; let workers = 1; - let validation = Validation::new(workers, tokenizer, max_best_of, max_stop_sequence, max_input_length, max_total_tokens); + let validation = Validation::new( + workers, + tokenizer, + max_best_of, + max_stop_sequence, + max_input_length, + max_total_tokens, + ); let max_new_tokens = 10; - match validation.validate_input("Hello".to_string(), None, max_new_tokens).await{ + match validation + .validate_input("Hello".to_string(), None, max_new_tokens) + .await + { Err(ValidationError::MaxNewTokens(1, 10)) => (), - _ => panic!("Unexpected not max new tokens") + _ => panic!("Unexpected not max new tokens"), } } #[tokio::test] - async fn test_validation_input_length(){ + async fn test_validation_input_length() { let tokenizer = Some(get_tokenizer().await); let max_best_of = 2; let max_stop_sequence = 3; let max_input_length = 4; let max_total_tokens = 5; let workers = 1; - let validation = Validation::new(workers, tokenizer, max_best_of, max_stop_sequence, max_input_length, max_total_tokens); + let validation = Validation::new( + workers, + tokenizer, + max_best_of, + max_stop_sequence, + max_input_length, + max_total_tokens, + ); let max_new_tokens = 10; - match validation.validate_input("Hello".to_string(), None, max_new_tokens).await{ + match validation + .validate_input("Hello".to_string(), None, max_new_tokens) + .await + { Err(ValidationError::MaxTotalTokens(5, 1, 10)) => (), - _ => panic!("Unexpected not max new tokens") + _ => panic!("Unexpected not max new tokens"), } } #[tokio::test] - async fn test_validation_best_of_sampling(){ + async fn test_validation_best_of_sampling() { let tokenizer = Some(get_tokenizer().await); let max_best_of = 2; let max_stop_sequence = 3; let max_input_length = 4; let max_total_tokens = 5; let workers = 1; - let validation = Validation::new(workers, tokenizer, max_best_of, max_stop_sequence, max_input_length, max_total_tokens); - match validation.validate(GenerateRequest{ - inputs: "Hello".to_string(), - parameters: GenerateParameters{ - best_of: Some(2), - do_sample: false, - ..default_parameters() - } - }).await{ + let validation = Validation::new( + workers, + tokenizer, + max_best_of, + max_stop_sequence, + max_input_length, + max_total_tokens, + ); + match validation + .validate(GenerateRequest { + inputs: "Hello".to_string(), + parameters: GenerateParameters { + best_of: Some(2), + do_sample: false, + ..default_parameters() + }, + }) + .await + { Err(ValidationError::BestOfSampling) => (), - _ => panic!("Unexpected not best of sampling") + _ => panic!("Unexpected not best of sampling"), } - } #[tokio::test] - async fn test_validation_top_p(){ + async fn test_validation_top_p() { let tokenizer = Some(get_tokenizer().await); let max_best_of = 2; let max_stop_sequence = 3; let max_input_length = 4; let max_total_tokens = 5; let workers = 1; - let validation = Validation::new(workers, tokenizer, max_best_of, max_stop_sequence, max_input_length, max_total_tokens); - match validation.validate(GenerateRequest{ - inputs: "Hello".to_string(), - parameters: GenerateParameters{ - top_p: Some(1.0), - ..default_parameters() - } - }).await{ + let validation = Validation::new( + workers, + tokenizer, + max_best_of, + max_stop_sequence, + max_input_length, + max_total_tokens, + ); + match validation + .validate(GenerateRequest { + inputs: "Hello".to_string(), + parameters: GenerateParameters { + top_p: Some(1.0), + ..default_parameters() + }, + }) + .await + { Err(ValidationError::TopP) => (), - _ => panic!("Unexpected top_p") + _ => panic!("Unexpected top_p"), } - match validation.validate(GenerateRequest{ - inputs: "Hello".to_string(), - parameters: GenerateParameters{ - top_p: Some(0.99), - max_new_tokens: 1, - ..default_parameters() - } - }).await{ + match validation + .validate(GenerateRequest { + inputs: "Hello".to_string(), + parameters: GenerateParameters { + top_p: Some(0.99), + max_new_tokens: 1, + ..default_parameters() + }, + }) + .await + { Ok(_) => (), - _ => panic!("Unexpected top_p error") + _ => panic!("Unexpected top_p error"), } - let valid_request = validation.validate(GenerateRequest{ - inputs: "Hello".to_string(), - parameters: GenerateParameters{ - top_p: None, - max_new_tokens: 1, - ..default_parameters() - } - }).await.unwrap(); + let valid_request = validation + .validate(GenerateRequest { + inputs: "Hello".to_string(), + parameters: GenerateParameters { + top_p: None, + max_new_tokens: 1, + ..default_parameters() + }, + }) + .await + .unwrap(); // top_p == 1.0 is invalid for users to ask for but it's the default resolved value. assert_eq!(valid_request.parameters.top_p, 1.0); - - } } diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index ddb7aae9..70f08ed7 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -29,6 +29,11 @@ def __init__(self, model: Model, cache: Cache, server_urls: List[str]): async def Info(self, request, context): return self.model.info + async def Health(self, request, context): + if self.model.device.type == "cuda": + torch.zeros((2, 2)).cuda() + return generate_pb2.HealthResponse() + async def ServiceDiscovery(self, request, context): return generate_pb2.ServiceDiscoveryResponse(urls=self.server_urls) From b4cf832c40f1fc17401968198b739bdf8351d9f1 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 27 Apr 2023 00:51:27 -0700 Subject: [PATCH 199/793] fix(server): fix reshaping of bloom past_key_values in concatenate() (#252) Introduced in #214 Fixes #249 --- server/text_generation_server/models/causal_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 336c9823..ca8fccfa 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -335,7 +335,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": [t.view(len(batch), -1, *t.shape[-2:]) for t in layer] for layer in batch.past_key_values ] - elif batch.past_key_values[0][0].shape == 3: + elif len(batch.past_key_values[0][0].shape) == 3: for layer in batch.past_key_values: for k, t in enumerate(layer): layer[k] = t.view(len(batch), -1, *t.shape[-2:]) From 34bca0b8d399114363547e017e503435b562b3e5 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 27 Apr 2023 00:57:28 -0700 Subject: [PATCH 200/793] fix(server): Small tidy of code from recent changes (#251) remaining_decode_tokens was calculated twice in Seq2SeqLMBatch.filter() --- server/text_generation_server/models/causal_lm.py | 3 ++- server/text_generation_server/models/seq2seq_lm.py | 14 ++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index ca8fccfa..7dc7fb85 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -340,12 +340,13 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": for k, t in enumerate(layer): layer[k] = t.view(len(batch), -1, *t.shape[-2:]) - start_index = end_index # Add eventual padding tokens that were added while concatenating max_tokens += batch.max_tokens + ( max_input_length - batch.max_input_length ) * len(batch) + start_index = end_index + first_past_kvs = batches[0].past_key_values _, num_heads, padded_sequence_length, head_dim = first_past_kvs[0][1].shape diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 0cb20760..4ac5ed3c 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -177,7 +177,7 @@ def filter( max_decoder_input_length = 0 padding_right_offset = 0 - remaining_decode_tokens = 0 + total_remaining_decode_tokens = 0 for i, r in enumerate(requests): idx = self.requests_idx_mapping[r.id] @@ -198,18 +198,15 @@ def filter( max_decoder_input_length = max( max_decoder_input_length, request_decoder_input_length ) - padding_right_offset = max( - padding_right_offset, - self.stopping_criterias[idx].max_new_tokens - - self.stopping_criterias[idx].current_tokens, - ) next_token_choosers.append(self.next_token_choosers[idx]) stopping_criteria = self.stopping_criterias[idx] stopping_criterias.append(stopping_criteria) - remaining_decode_tokens += ( + remaining_decode_tokens = ( stopping_criteria.max_new_tokens - stopping_criteria.current_tokens ) + total_remaining_decode_tokens += remaining_decode_tokens + padding_right_offset = max(padding_right_offset, remaining_decode_tokens) # Apply indices to input_ids, attention mask, past key values and other items that need to be cached self.decoder_input_ids = self.decoder_input_ids[keep_indices] @@ -397,7 +394,6 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": [t for t in layer] for layer in batch.past_key_values ] - start_index = end_index # Add eventual padding tokens that were added while concatenating max_tokens += batch.max_tokens + ( max_input_length @@ -406,6 +402,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": - batch.max_decoder_input_length ) * len(batch) + start_index = end_index + # Determine shapes for new past kv tensors first_past_kvs = batches[0].past_key_values _, num_heads, _, head_dim = first_past_kvs[0][0].shape From b9ae7e5da19573de3d9e727884f6609df04f935d Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 27 Apr 2023 09:57:41 +0200 Subject: [PATCH 201/793] chore(server): update transformers (#250) --- server/Makefile-transformers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/Makefile-transformers b/server/Makefile-transformers index 692a9374..64d01672 100644 --- a/server/Makefile-transformers +++ b/server/Makefile-transformers @@ -1,4 +1,4 @@ -transformers_commit := b8d969ff47c6a9d40538a6ea33df021953363afc +transformers_commit := 69009822aa7897ffab97afb814e38126b83f639e transformers: # Clone fork of transformers with custom CUDA kernels and sharding logic From f092ba9b223a0582fa930981ad319d253d1ac765 Mon Sep 17 00:00:00 2001 From: "Ehsan M. Kermani" <6980212+ehsanmok@users.noreply.github.com> Date: Thu, 27 Apr 2023 10:16:35 -0700 Subject: [PATCH 202/793] feat(server): add watermarking tests (#248) --- .github/workflows/client-tests.yaml | 2 +- .github/workflows/tests.yaml | 2 +- Makefile | 9 +++- README.md | 17 +++--- clients/python/pyproject.toml | 5 +- clients/python/tests/test_client.py | 13 +++++ router/client/src/sharded_client.rs | 2 +- router/src/health.rs | 8 ++- server/tests/utils/test_watermark.py | 54 +++++++++++++++++++ .../text_generation_server/models/__init__.py | 2 - 10 files changed, 98 insertions(+), 16 deletions(-) create mode 100644 server/tests/utils/test_watermark.py diff --git a/.github/workflows/client-tests.yaml b/.github/workflows/client-tests.yaml index 7ccef3b0..1fa0b39d 100644 --- a/.github/workflows/client-tests.yaml +++ b/.github/workflows/client-tests.yaml @@ -22,4 +22,4 @@ jobs: - name: Run tests run: | pip install pytest pytest-asyncio - cd clients/python && pytest tests + make python-client-tests diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 26cbf3b2..d9858a3b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -66,7 +66,7 @@ jobs: - name: Run server tests run: | pip install pytest - HF_HUB_ENABLE_HF_TRANSFER=1 pytest -sv server/tests + make python-server-tests - name: Run Rust fmt run: | cargo fmt --check diff --git a/Makefile b/Makefile index 21fd11b5..032a49de 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,13 @@ router-dev: integration-tests: install-router install-launcher cargo test -python-tests: - cd server && HF_HUB_ENABLE_HF_TRANSFER=1 pytest tests +python-server-tests: + HF_HUB_ENABLE_HF_TRANSFER=1 pytest server/tests + +python-client-tests: + pytest clients/python/tests + +python-tests: python-server-tests python-client-tests run-bloom-560m: text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --port 8080 diff --git a/README.md b/README.md index 0c63f36b..351a83aa 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ to power LLMs api-inference widgets. - [Quantization](#quantization) - [Develop](#develop) - [Testing](#testing) - + ## Features - Serve the most popular Large Language Models with a simple launcher @@ -131,7 +131,7 @@ by setting the address to an OTLP collector with the `--otlp-endpoint` argument. ### A note on Shared Memory (shm) -[`NCCL`](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/index.html) is a communication framework used by +[`NCCL`](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/index.html) is a communication framework used by `PyTorch` to do distributed training/inference. `text-generation-inference` make use of `NCCL` to enable Tensor Parallelism to dramatically speed up inference for large language models. @@ -152,14 +152,14 @@ creating a volume with: and mounting it to `/dev/shm`. -Finally, you can also disable SHM sharing by using the `NCCL_SHM_DISABLE=1` environment variable. However, note that +Finally, you can also disable SHM sharing by using the `NCCL_SHM_DISABLE=1` environment variable. However, note that this will impact performance. ### Local install -You can also opt to install `text-generation-inference` locally. +You can also opt to install `text-generation-inference` locally. -First [install Rust](https://rustup.rs/) and create a Python virtual environment with at least +First [install Rust](https://rustup.rs/) and create a Python virtual environment with at least Python 3.9, e.g. using `conda`: ```shell @@ -181,7 +181,7 @@ sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' rm -f $PROTOC_ZIP ``` -On MacOS, using Homebrew: +On MacOS, using Homebrew: ```shell brew install protobuf @@ -241,6 +241,11 @@ make router-dev ## Testing ```shell +# python +make python-server-tests +make python-client-tests +# or both server and client tests make python-tests +# rust cargo tests make integration-tests ``` diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index d883af37..00bb99fc 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -21,6 +21,9 @@ pytest = "^6.2.5" pytest-asyncio = "^0.17.2" pytest-cov = "^3.0.0" +[tool.pytest.ini_options] +asyncio_mode = "auto" + [build-system] -requires = ["poetry-core"] +requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index d53c2a4d..8972dfd1 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -87,6 +87,19 @@ async def test_generate_async(flan_t5_xxl_url, hf_headers): ) +@pytest.mark.asyncio +async def test_generate_async_best_of(flan_t5_xxl_url, hf_headers): + client = AsyncClient(flan_t5_xxl_url, hf_headers) + response = await client.generate( + "test", max_new_tokens=1, best_of=2, do_sample=True + ) + + assert response.details.seed is not None + assert response.details.best_of_sequences is not None + assert len(response.details.best_of_sequences) == 1 + assert response.details.best_of_sequences[0].seed is not None + + @pytest.mark.asyncio async def test_generate_async_not_found(fake_url, hf_headers): client = AsyncClient(fake_url, hf_headers) diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 2f57a437..73524740 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -20,7 +20,7 @@ impl ShardedClient { /// the other shards and returns all uris/unix sockets with the `service_discovery` gRPC method. async fn from_master_client(mut master_client: Client) -> Result { // Get all uris/unix sockets from the master client - let uris = master_client.service_discovery().await.unwrap(); + let uris = master_client.service_discovery().await?; let futures = uris.into_iter().map(Client::connect_uds); let clients: Result> = join_all(futures).await.into_iter().collect(); Ok(Self::new(clients?)) diff --git a/router/src/health.rs b/router/src/health.rs index 02edf328..45f50e9d 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -4,6 +4,10 @@ use text_generation_client::{ Batch, NextTokenChooserParameters, Request, ShardedClient, StoppingCriteriaParameters, }; +// Note: Request ids and batch ids cannot collide. +const LIVENESS_ID: u64 = u64::MAX; +const BATCH_ID: u64 = u64::MAX; + #[derive(Clone, Debug)] pub(crate) struct Health { client: ShardedClient, @@ -27,7 +31,7 @@ impl Health { // Dummy batch of 1 token and 1 generated token let liveness_request = Request { - id: u64::MAX, + id: LIVENESS_ID, inputs: "liveness".to_string(), truncate: 10, parameters: Some(NextTokenChooserParameters { @@ -47,7 +51,7 @@ impl Health { }), }; let batch = Batch { - id: u64::MAX, + id: BATCH_ID, requests: vec![liveness_request], size: 1, max_tokens: 2, diff --git a/server/tests/utils/test_watermark.py b/server/tests/utils/test_watermark.py new file mode 100644 index 00000000..5d0aaf05 --- /dev/null +++ b/server/tests/utils/test_watermark.py @@ -0,0 +1,54 @@ +# test_watermark_logits_processor.py + +import os +import numpy as np +import torch +from text_generation_server.utils.watermark import WatermarkLogitsProcessor + + +GAMMA = os.getenv("WATERMARK_GAMMA", 0.5) +DELTA = os.getenv("WATERMARK_DELTA", 2.0) + + +def test_seed_rng(): + input_ids = [101, 2036, 3731, 102, 2003, 103] + processor = WatermarkLogitsProcessor() + processor._seed_rng(input_ids) + assert isinstance(processor.rng, torch.Generator) + + +def test_get_greenlist_ids(): + input_ids = [101, 2036, 3731, 102, 2003, 103] + processor = WatermarkLogitsProcessor() + result = processor._get_greenlist_ids(input_ids, 10, torch.device("cpu")) + assert max(result) <= 10 + assert len(result) == int(10 * 0.5) + + +def test_calc_greenlist_mask(): + processor = WatermarkLogitsProcessor() + scores = torch.tensor([[0.5, 0.3, 0.2, 0.8], [0.1, 0.2, 0.7, 0.9]]) + greenlist_token_ids = torch.tensor([2, 3]) + result = processor._calc_greenlist_mask(scores, greenlist_token_ids) + assert result.tolist() == [[False, False, False, False], [False, False, True, True]] + assert result.shape == scores.shape + + +def test_bias_greenlist_logits(): + processor = WatermarkLogitsProcessor() + scores = torch.tensor([[0.5, 0.3, 0.2, 0.8], [0.1, 0.2, 0.7, 0.9]]) + green_tokens_mask = torch.tensor( + [[False, False, True, True], [False, False, False, True]] + ) + greenlist_bias = 2.0 + result = processor._bias_greenlist_logits(scores, green_tokens_mask, greenlist_bias) + assert np.allclose(result.tolist(), [[0.5, 0.3, 2.2, 2.8], [0.1, 0.2, 0.7, 2.9]]) + assert result.shape == scores.shape + + +def test_call(): + input_ids = [101, 2036, 3731, 102, 2003, 103] + processor = WatermarkLogitsProcessor() + scores = torch.tensor([[0.5, 0.3, 0.2, 0.8], [0.1, 0.2, 0.7, 0.9]]) + result = processor(input_ids, scores) + assert result.shape == scores.shape diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 74a7483e..221c9139 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -58,8 +58,6 @@ "GalacticaSharded", "GPTNeoxSharded", "Seq2SeqLM", - "Galactica", - "GalacticaSharded", "SantaCoder", "OPT", "OPTSharded", From 593a5634142e52b537f9bfe1bbf131c7ade1185f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 27 Apr 2023 19:18:33 +0200 Subject: [PATCH 203/793] feat(docker): add nvidia env vars (#255) --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index c17a9414..c65d3dee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -164,6 +164,11 @@ FROM base as sagemaker COPY sagemaker-entrypoint.sh entrypoint.sh RUN chmod +x entrypoint.sh +# NVIDIA env vars +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 + ENTRYPOINT ["./entrypoint.sh"] # Final image From b0b97fd9a7d549421eadc1ab574f060a8e114b9c Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Sat, 29 Apr 2023 11:53:42 +0200 Subject: [PATCH 204/793] doc(launcher): add more docs to the `launcher` itself and link in the README (#257) --- README.md | 5 ++ launcher/src/main.rs | 130 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) diff --git a/README.md b/README.md index 351a83aa..712aea6a 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,11 @@ docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingf ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. +To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli: +``` +text-generation-launcher --help +``` + You can then query the model using either the `/generate` or `/generate_stream` routes: ```shell diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 7b5f908a..77b0cf8a 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -18,52 +18,182 @@ use subprocess::{ExitStatus, Popen, PopenConfig, PopenError, Redirection}; #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] struct Args { + /// The name of the model to load. + /// Can be a MODEL_ID as listed on like + /// `gpt2` or `OpenAssistant/oasst-sft-1-pythia-12b`. + /// Or it can be a local directory containing the necessary files + /// as saved by `save_pretrained(...)` methods of transformers #[clap(default_value = "bigscience/bloom-560m", long, env)] model_id: String, + + /// The actual revision of the model if you're referring to a model + /// on the hub. You can use a specific commit id or a branch like `refs/pr/2`. #[clap(long, env)] revision: Option, + + /// Wether to shard or not the model across multiple GPUs + /// By default text-generation-inference will use all available GPUs to run + /// the model. Setting it to `false` deactivates `num_shard`. #[clap(long, env)] sharded: Option, + + /// The number of shards to use if you don't want to use all GPUs on a given machine. + /// You can use `CUDA_VISIBLE_DEVICE=0,1 text-generation-launcher... --num_shard 2` + /// and `CUDA_VISIBLE_DEVICE=2,3 text-generation-launcher... --num_shard 2` to + /// launch 2 copies with 2 shard each on a given machine with 4 GPUs for instance. #[clap(long, env)] num_shard: Option, + + /// Wether you want the model to be quantized or not. This will use bitsandbytes for + /// quantization on the fly. #[clap(long, env)] quantize: bool, + + /// The maximum amount of concurrent requests for this particular deployment. + /// Having a low limit will refuse clients requests instead of having them + /// wait for too long and is usually good to handle backpressure correctly. #[clap(default_value = "128", long, env)] max_concurrent_requests: usize, + + /// This is the maximum allowed value for clients to set `best_of`. + /// Best of makes `n` generations at the same time, and return the best + /// in terms of overall log probability over the entire generated sequence #[clap(default_value = "2", long, env)] max_best_of: usize, + + /// This is the maximum allowed value for clients to set `stop_sequences`. + /// Stop sequences are used to allow the model to stop on more than just + /// the EOS token, and enable more complex "prompting" where users can preprompt + /// the model in a specific way and define their "own" stop token aligned with + /// their prompt. #[clap(default_value = "4", long, env)] max_stop_sequences: usize, + + /// This is the maximum allowed input length (expressed in number of tokens) + /// for users. The larger this value, the longer prompt users can send which + /// can impact the overall memory required to handle the load. + /// Please note that some models have a finite range of sequence they can handle. #[clap(default_value = "1000", long, env)] max_input_length: usize, + + /// This is the most important value to set as it defines the "memory budget" + /// of running clients requests. + /// Clients will send input sequences and ask to generate `max_new_tokens` + /// on top. with a value of `1512` users can send either a prompt of + /// `1000` and ask for `512` new tokens, or send a prompt of `1` and ask for + /// `1511` max_new_tokens. + /// The larger this value, the larger amount each request will be in your RAM + /// and the less effective batching can be. #[clap(default_value = "1512", long, env)] max_total_tokens: usize, + + /// The maximum allowed batch size during dynamic batching. + /// Using `max_batch_total_tokens` should be favored in general + /// as it's a finer way to control RAM usage. #[clap(long, env)] max_batch_size: Option, + + /// This represents the ratio of waiting queries vs running queries where + /// you want to start considering pausing the running queries to include the waiting + /// ones into the same batch. + /// `waiting_served_ratio=1.2` Means when 12 queries are waiting and there's + /// only 10 queries left in the current batch we check if we can fit those 12 + /// waiting queries into the batching strategy, and if yes, then batching happens + /// delaying the 10 running queries by a `prefill` run. + /// + /// This setting is only applied if there is room in the batch + /// as defined by `max_batch_total_tokens`. #[clap(default_value = "1.2", long, env)] waiting_served_ratio: f32, + + /// **IMPORTANT** This is one critical control to allow maximum usage + /// of the available hardware. + /// + /// This represents the total amount of potential tokens within a batch. + /// When using padding (not recommended) this would be equivalent of + /// `batch_size` * `max_total_tokens`. + /// + /// However in the non-padded (flash attention) version this can be much finer. + /// + /// For `max_batch_total_tokens=1000`, you could fit `10` queries of `total_tokens=100` + /// or a single query of `1000` tokens. + /// + /// So you don't have to control that finely + /// `max_batch_size` or `max_total_tokens`. In fact you could mostly relax them if you + /// want maximum flexibility. However, for your users if they are asking for the full amount of + /// total tokens, they are likely to wait for a very long time to get a spot + /// in the batch (since they are going to be alone) so setting `max_batch_size` + /// and `max_total_tokens` can still be useful to prevent those long waiting times. + /// + /// Overall this number should be the largest possible amount that fits the + /// remaining memory (after the model is loaded). Since the actual memory overhead + /// depends on other parameters like if you're using quantization, flash attention + /// or the model implementation, text-generation-inference cannot infer this number + /// automatically. #[clap(default_value = "32000", long, env)] max_batch_total_tokens: u32, + + /// This setting defines how many tokens can be passed before forcing the waiting + /// queries to be put on the batch (if the size of the batch allows for it). + /// New queries require 1 `prefill` forward, which is different from `decode` + /// and therefore you need to pause the running batch in order to run `prefill` + /// to create the correct values for the waiting queries to be able to join the batch. + /// + /// With a value too small, queries will always "steal" the compute to run `prefill` + /// and running queries will be delayed by a lot. + /// + /// With a value too big, waiting queries could wait for a very long time + /// before being allowed a slot in the running batch. If your server is busy + /// that means that requests that could run in ~2s on an empty server could + /// end up running in ~20s because the query had to wait for 18s. + /// + /// This number is expressed in number of tokens to make it a bit more + /// "model" agnostic, but what should really matter is the overall latency + /// for end users. #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, #[clap(default_value = "3000", long, short, env)] + + /// The port to listen on. port: u16, + + /// The name of the socket for gRPC communication between the webserver + /// and the shards. #[clap(default_value = "/tmp/text-generation-server", long, env)] shard_uds_path: String, + + /// The address the master shard will listen on. (setting used by torch distributed) #[clap(default_value = "localhost", long, env)] master_addr: String, + + /// The address the master port will listen on. (setting used by torch distributed) #[clap(default_value = "29500", long, env)] master_port: usize, + + /// The location of the huggingface hub cache. + /// Used to override the location if you want to provide a mounted disk for instance #[clap(long, env)] huggingface_hub_cache: Option, + + /// The location of the huggingface hub cache. + /// Used to override the location if you want to provide a mounted disk for instance #[clap(long, env)] weights_cache_override: Option, + + /// For some models (like bloom), text-generation-inference implemented custom + /// cuda kernels to speed up inference. Those kernels were only tested on A100. + /// Use this flag to disable them if you're running on different hardware and + /// encounter issues. #[clap(long, env)] disable_custom_kernels: bool, + + /// Outputs the logs in JSON format (useful for telemetry) #[clap(long, env)] json_output: bool, + #[clap(long, env)] otlp_endpoint: Option, + #[clap(long, env)] cors_allow_origin: Vec, #[clap(long, env)] From 0e9d249b7948743743d5a96e56d6c4e1a3d011da Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sat, 29 Apr 2023 12:17:30 +0200 Subject: [PATCH 205/793] feat(benchmark): add support for private tokenizers (#262) --- benchmark/src/main.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/benchmark/src/main.rs b/benchmark/src/main.rs index 443af77f..03f61dcd 100644 --- a/benchmark/src/main.rs +++ b/benchmark/src/main.rs @@ -5,7 +5,7 @@ use clap::Parser; use std::path::Path; use text_generation_client::ShardedClient; -use tokenizers::Tokenizer; +use tokenizers::{FromPretrainedParameters, Tokenizer}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; @@ -16,6 +16,8 @@ use tracing_subscriber::EnvFilter; struct Args { #[clap(short, long, env)] tokenizer_name: String, + #[clap(default_value = "main", long, env)] + revision: String, #[clap(short, long)] batch_size: Option>, #[clap(default_value = "10", short, long, env)] @@ -36,6 +38,7 @@ fn main() -> Result<(), Box> { // Pattern match configuration let Args { tokenizer_name, + revision, batch_size, sequence_length, decode_length, @@ -59,10 +62,19 @@ fn main() -> Result<(), Box> { tracing::info!("Found local tokenizer"); Tokenizer::from_file(local_path.join("tokenizer.json")).unwrap() } else { + tracing::info!("Downloading tokenizer"); + + // Parse Huggingface hub token + let auth_token = std::env::var("HUGGING_FACE_HUB_TOKEN").ok(); + // Download and instantiate tokenizer // We need to download it outside of the Tokio runtime - tracing::info!("Downloading tokenizer"); - Tokenizer::from_pretrained(tokenizer_name.clone(), None).unwrap() + let params = FromPretrainedParameters { + revision, + auth_token, + ..Default::default() + }; + Tokenizer::from_pretrained(tokenizer_name.clone(), Some(params)).unwrap() }; tracing::info!("Tokenizer loaded"); From e86cca9723d8f7b9a17f11d716240d0ad3e0f875 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 1 May 2023 14:16:50 +0200 Subject: [PATCH 206/793] Adding docs on how dynamic batching works. (#258) This PR starts the minimal possible amount of explanation I could think of. It tries to explain how dynamic batching occurs, the interactions with past key values and ignores the padding problem. Maybe some drawings could help too but I kept it to text for now. --- README.md | 3 +- router/README.md | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 router/README.md diff --git a/README.md b/README.md index 712aea6a..756d7e35 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,8 @@ to power LLMs api-inference widgets. - Serve the most popular Large Language Models with a simple launcher - Tensor Parallelism for faster inference on multiple GPUs - Token streaming using Server-Sent Events (SSE) -- [Dynamic batching of incoming requests](https://github.com/huggingface/text-generation-inference/blob/main/router/src/batcher.rs#L88) for increased total throughput +- [Continous batching of incoming requests](https://github.com/huggingface/text-generation-inference/tree/main/router) for increased total throughput +- Optimized transformers code for inference using [flash-attention](https://github.com/HazyResearch/flash-attention) on the most popular architectures - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading - Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) diff --git a/router/README.md b/router/README.md new file mode 100644 index 00000000..c18d4f9e --- /dev/null +++ b/router/README.md @@ -0,0 +1,93 @@ +# Router + +Also named `webserver` throughout the docs. + +This router is handling most of the logic to handle the "batches" tell +when to pass new `prefill` requests and pausing `decode` requests, which ones etc... + +It uses gRPC to communicate with the shards which can therefore be kept +much simpler and focus on having the most efficient forward passes as possible. + +## Continuous batching + +One important feature of `text-generation-inference` is enabled +by this `router`. + +Continuous batching is the act of regularly running queries in the same +`forward` step of the LLM (a "batch") and also removing them when they are +finished. + +In order for continuous batching to be useful, you need to have more compute available +with respect to the memory requirements of your model. This is essentially true for +LLMs and the larger the model, the truer it gets (since you have to pool multiple +GPUs to load the model, you effectively have a lot of compute power at your hands). + + +Static batching is the act of doing several queries at the same time, but usually +this is controlled by the client, and therefore the amount of batching is decided +beforehand. + +For text-generation, and LLMs which are memory bound we can try to be much more +efficient with the available compute, by having client sending us single queries, +and let the router mix&match queries into or out of batches to make the use the +compute the most efficiently. This is possible because for LLMs the total compute +for running the model is much bigger than doing mix&match of the batches themselves. + + +### Simple continuous batching + +text-generation works by feeding a prompt to a model, and iteratively calling +`forward` on the model to produce new text, 1 token at a time. + +The first idea is simple, when a query arrives, we start working on it directly. +When new queries arrive, we simply wait for the current `forward` to be finished +then batch the current running prompt with the new query, and call `forward`. + +Whenever either query is finished: either the model produce EOS (end of sentence) token +or the query reached the allowed limit. We simply drop it from the batch, remove +all the allocated memory and we can continue with the rest until nothing is left. + +This simple idea generalizes very well and we could potentially stack many requests +in the same batch. + +One thing to note, is that queries can be potentially run with different parameters +meaning different way to choose the next token (sampling, not sampling, temperature, top_k etc..). This is not problematic for the proposed approach we just need to do the sampling +independantly on each member of the batch. + +### Prefill, decode and past key values + +In order to make LLMs and text-generation efficient, there's actually a very powerful +trick that can be used, which is the "caching" of some attention matrices. [More on that +in the first part of this blog](https://huggingface.co/blog/accelerated-inference#getting-to-the-first-10x-speedup) + +What this means, is that the first "pass" of a prompt is different from the subsequent +"forward" passes. Since for the first one we have to compute the entire attention matrix, whereas in the follow-ups only require to compute the new token attention. +The first pass is called `prefill` throughout this codebase where as the follow-ups are called `decode`. + +Since `prefill` is much more expensive than `decode` we don't want to do it all the time, +but a currently running query is probably doing `decode`. If we want to do the continuous +batching as explained previously we need to run `prefill` at some point in order to create +the attention matrix required to be able to join the `decode` group. + +`text-generation-inference` uses a bunch of different strategies and parameters in +order to enable you to find the sweet spot between exploiting the hardware and perceived latency. + +With no continuous batching at all, latency is going to be super good, but throughput (meaning +the total number of requests allowed in a given timeframe) is going to be super bad (since it's essentially 1). + +With static batching, you can probably reach the maximum throughput (by using the maximum total batch size applicable to your hardware), but the latency is super bad since in order to have maximum throughput you need to wait for requests to come in before processing. + +With continuous batching you can find a sweet spot. In general latency is the most critical +parameter users care about. But a 2x latency slowdown for 10x more users on the same +hardware is an acceptable tradeoff. + +## Token streaming + +This is a very important aspect of client UX. As mentionned above, latency is the +most critical perceived quality of an LLM API. + +With token streaming, the server can start answering after the first `prefill` pass +directly, without waiting for all the generation to be done. For extremely long queries +this means clients can start to see something happening orders of magnitude before +the work is done. Seeing something in progress allows them to cut short if it's not +what's wanted but also it "feels" better. From 411b0d4e1ffab5e8d20147b593058c068dcc17b5 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 2 May 2023 15:43:19 +0200 Subject: [PATCH 207/793] chore(github): add templates (#264) --- .github/ISSUE_TEMPLATE/bug-report.yml | 67 +++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 2 + .github/ISSUE_TEMPLATE/feature-request.yml | 31 +++++++++ .github/ISSUE_TEMPLATE/new-model-addition.yml | 31 +++++++++ .github/PULL_REQUEST_TEMPLATE.md | 40 +++++++++++ .github/workflows/build.yaml | 1 + Cargo.lock | 41 ++++++++++++ Dockerfile | 1 + launcher/Cargo.toml | 3 + launcher/build.rs | 29 ++++++++ launcher/src/env_runtime.rs | 45 +++++++++++++ launcher/src/main.rs | 11 +++ router/build.rs | 7 ++ router/src/lib.rs | 2 + router/src/server.rs | 1 + 15 files changed, 312 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml create mode 100644 .github/ISSUE_TEMPLATE/new-model-addition.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 launcher/build.rs create mode 100644 launcher/src/env_runtime.rs diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..12c93b9e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,67 @@ +name: "\U0001F41B Bug Report" +description: Submit a bug report to help us improve text-generation-inference +body: + - type: textarea + id: system-info + attributes: + label: System Info + description: | + Please share your system info with us (`text-generation-launcher --env` if installed locally). + The full command line used that causes issues: + OS version: + Rust version (if self-compiling, `cargo version`): + Model being used (`curl 127.0.0.1:8080/info | jq`): + If local model please explicit the kind of model and/or equivalents. + Hardware used (GPUs, how many, on which cloud) (`nvidia-smi`): + Deployment specificities (Kubernetes, EKS, AKS, any particular deployments): + The current version being used: + + placeholder: text-generation-inference version, platform, python version, ... + validations: + required: true + + - type: checkboxes + id: information-scripts-examples + attributes: + label: Information + description: 'The problem arises when using:' + options: + - label: "Docker" + - label: "The CLI directly" + + - type: checkboxes + id: information-tasks + attributes: + label: Tasks + description: "The thing I am working on is:" + options: + - label: "An officially supported command" + - label: "My own modifications" + + - type: textarea + id: reproduction + validations: + required: true + attributes: + label: Reproduction + description: | + Please provide a code sample that reproduces the problem you ran into. It can be a Colab link or just a code snippet. + If you have code snippets, error messages, stack traces please provide them here as well. + Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting + Do not use screenshots, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code. + + placeholder: | + Steps to reproduce the behavior: + + 1. + 2. + 3. + + + - type: textarea + id: expected-behavior + validations: + required: true + attributes: + label: Expected behavior + description: "A clear and concise description of what you would expect to happen." diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..e6477729 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,2 @@ +blank_issues_enabled: true +version: 2.1 diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 00000000..5abc1565 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,31 @@ +name: "\U0001F680 Feature request" +description: Submit a proposal/request for a new text-generation-inference feature +labels: [ "feature" ] +body: + - type: textarea + id: feature-request + validations: + required: true + attributes: + label: Feature request + description: | + A clear and concise description of the feature proposal. Please provide a link to the paper and code in case they exist. + + - type: textarea + id: motivation + validations: + required: true + attributes: + label: Motivation + description: | + Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too. + + + - type: textarea + id: contribution + validations: + required: true + attributes: + label: Your contribution + description: | + Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/huggingface/text-generation-inference/blob/main/CONTRIBUTING.md) diff --git a/.github/ISSUE_TEMPLATE/new-model-addition.yml b/.github/ISSUE_TEMPLATE/new-model-addition.yml new file mode 100644 index 00000000..2f3476d3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new-model-addition.yml @@ -0,0 +1,31 @@ +name: "\U0001F31F New model addition" +description: Submit a proposal/request to implement a new model +labels: [ "New model" ] + +body: + - type: textarea + id: description-request + validations: + required: true + attributes: + label: Model description + description: | + Put any and all important information relative to the model + + - type: checkboxes + id: information-tasks + attributes: + label: Open source status + description: | + Please note that if the model implementation isn't available or if the weights aren't open-source, we are less likely to implement it in `transformers`. + options: + - label: "The model implementation is available" + - label: "The model weights are available" + + - type: textarea + id: additional-info + attributes: + label: Provide useful links for the implementation + description: | + Please provide information regarding the implementation, the weights, and the authors. + Please mention the authors by @gh-username if you're aware of their usernames. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..ad5b98ab --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,40 @@ +# What does this PR do? + + + + + +Fixes # (issue) + + +## Before submitting +- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). +- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), + Pull Request section? +- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link + to it if that's the case. +- [ ] Did you make sure to update the documentation with your changes? Here are the + [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and + [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). +- [ ] Did you write any new necessary tests? + + +## Who can review? + +Anyone in the community is free to review the PR once the tests have passed. Feel free to tag +members/contributors who may be interested in your PR. + + diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 67d1c730..289e4f67 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -97,6 +97,7 @@ jobs: platforms: 'linux/amd64' build-args: | GIT_SHA=${{ env.GITHUB_SHA }} + DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max diff --git a/Cargo.lock b/Cargo.lock index e240fe9f..58142d4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1428,6 +1428,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2063,6 +2072,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.37.11" @@ -2136,6 +2154,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + [[package]] name = "serde" version = "1.0.160" @@ -2345,6 +2369,20 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sysinfo" +version = "0.28.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c2f3ca6693feb29a89724516f016488e9aafc7f37264f898593ee4b942f31b" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "winapi", +] + [[package]] name = "tar" version = "0.4.38" @@ -2399,6 +2437,7 @@ dependencies = [ "subprocess", "tracing", "tracing-subscriber", + "vergen", ] [[package]] @@ -2985,7 +3024,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b86a8af1dedf089b1c78338678e4c7492b6045649042d94faf19690499d236" dependencies = [ "anyhow", + "rustc_version", "rustversion", + "sysinfo", "time", ] diff --git a/Dockerfile b/Dockerfile index c65d3dee..d59acc10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder ARG GIT_SHA +ARG DOCKER_LABEL RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \ curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \ diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 5944a360..885dcd98 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -17,3 +17,6 @@ tracing-subscriber = { version = "0.3.16", features = ["json"] } [dev-dependencies] float_eq = "1.0.1" reqwest = { version = "0.11.14", features = ["blocking", "json"] } + +[build-dependencies] +vergen = { version = "8.0.0", features = ["build", "cargo", "git", "gitcl", "rustc", "si"] } diff --git a/launcher/build.rs b/launcher/build.rs new file mode 100644 index 00000000..71d2c0c5 --- /dev/null +++ b/launcher/build.rs @@ -0,0 +1,29 @@ +use std::error::Error; +use vergen::EmitBuilder; + +fn main() -> Result<(), Box> { + // Emit cargo and rustc compile time values + EmitBuilder::builder().all_cargo().all_rustc().emit()?; + + // Try to get the git sha from the local git repository + if EmitBuilder::builder() + .fail_on_error() + .git_sha(false) + .emit() + .is_err() + { + // Unable to get the git sha + if let Ok(sha) = std::env::var("GIT_SHA") { + // Set it from an env var + println!("cargo:rustc-env=VERGEN_GIT_SHA={sha}"); + } + } + + // Set docker label if present + if let Ok(label) = std::env::var("DOCKER_LABEL") { + // Set it from an env var + println!("cargo:rustc-env=DOCKER_LABEL={label}"); + } + + Ok(()) +} diff --git a/launcher/src/env_runtime.rs b/launcher/src/env_runtime.rs new file mode 100644 index 00000000..9dbc83f7 --- /dev/null +++ b/launcher/src/env_runtime.rs @@ -0,0 +1,45 @@ +use std::fmt; +use std::process::Command; + +pub(crate) struct Env { + cargo_target: &'static str, + cargo_version: &'static str, + git_sha: &'static str, + docker_label: &'static str, + nvidia_env: String, +} + +impl Env { + pub fn new() -> Self { + let nvidia_env = nvidia_smi(); + + Self { + nvidia_env: nvidia_env.unwrap_or("N/A".to_string()), + cargo_target: env!("VERGEN_CARGO_TARGET_TRIPLE"), + cargo_version: env!("VERGEN_RUSTC_SEMVER"), + git_sha: option_env!("VERGEN_GIT_SHA").unwrap_or("N/A"), + docker_label: option_env!("DOCKER_LABEL").unwrap_or("N/A"), + } + } +} + +impl fmt::Display for Env { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "Runtime environment:")?; + + writeln!(f, "Target: {}", self.cargo_target)?; + writeln!(f, "Cargo version: {}", self.cargo_version)?; + writeln!(f, "Commit sha: {}", self.git_sha)?; + writeln!(f, "Docker label: {}", self.docker_label)?; + write!(f, "nvidia-smi:\n{}", self.nvidia_env)?; + + Ok(()) + } +} + +fn nvidia_smi() -> Option { + let output = Command::new("nvidia-smi").output().ok()?; + let nvidia_smi = String::from_utf8(output.stdout).ok()?; + let output = nvidia_smi.replace('\n', "\n "); + Some(output.trim().to_string()) +} diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 77b0cf8a..736aa5a7 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -14,6 +14,8 @@ use std::time::{Duration, Instant}; use std::{fs, io}; use subprocess::{ExitStatus, Popen, PopenConfig, PopenError, Redirection}; +mod env_runtime; + /// App Configuration #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] @@ -200,6 +202,10 @@ struct Args { watermark_gamma: Option, #[clap(long, env)] watermark_delta: Option, + + /// Display a lot of information about your runtime environment + #[clap(long, short, action)] + env: bool, } #[derive(Debug)] @@ -829,6 +835,11 @@ fn main() -> Result<(), LauncherError> { tracing_subscriber::fmt().compact().init(); } + if args.env { + let env_runtime = env_runtime::Env::new(); + tracing::info!("{}", env_runtime); + } + tracing::info!("{:?}", args); let num_shard = find_num_shards(args.sharded, args.num_shard); diff --git a/router/build.rs b/router/build.rs index 1b1fdc86..f5eb8a26 100644 --- a/router/build.rs +++ b/router/build.rs @@ -15,5 +15,12 @@ fn main() -> Result<(), Box> { println!("cargo:rustc-env=VERGEN_GIT_SHA={sha}"); } } + + // Set docker label if present + if let Ok(label) = std::env::var("DOCKER_LABEL") { + // Set it from an env var + println!("cargo:rustc-env=DOCKER_LABEL={label}"); + } + Ok(()) } diff --git a/router/src/lib.rs b/router/src/lib.rs index c2ff669b..080dc4f4 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -57,6 +57,8 @@ pub struct Info { pub version: &'static str, #[schema(nullable = true, example = "null")] pub sha: Option<&'static str>, + #[schema(nullable = true, example = "null")] + pub docker_label: Option<&'static str>, } #[derive(Clone, Debug, Deserialize, ToSchema)] diff --git a/router/src/server.rs b/router/src/server.rs index f25fa2b3..c5fc41ef 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -635,6 +635,7 @@ pub async fn run( validation_workers, version: env!("CARGO_PKG_VERSION"), sha: option_env!("VERGEN_GIT_SHA"), + docker_label: option_env!("DOCKER_LABEL"), }; // Create router From 4096000e34ded704355c55fb2b3e1b0b3cb6b3ba Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 3 May 2023 10:10:34 +0200 Subject: [PATCH 208/793] fix(server): fix typo in tokenizers decode (#269) closes #268 --- server/text_generation_server/models/causal_lm.py | 2 +- server/text_generation_server/models/flash_causal_lm.py | 2 +- server/text_generation_server/models/flash_santacoder.py | 2 +- server/text_generation_server/models/galactica.py | 2 +- server/text_generation_server/models/santacoder.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 7dc7fb85..26a9a661 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -490,7 +490,7 @@ def batch_type(self) -> Type[CausalLMBatch]: def decode(self, generated_ids: List[int]) -> str: return self.tokenizer.decode( - generated_ids, skip_special_tokens=True, cleanup_tokenization_spaces=False + generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False ) def forward( diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 61ccca84..413866d1 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -402,7 +402,7 @@ def batch_type(self) -> Type[FlashCausalLMBatch]: def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: return self.tokenizer.decode( - generated_ids, skip_special_tokens=True, cleanup_tokenization_spaces=False + generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False ) def forward( diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index aa1bdfb5..550be956 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -165,7 +165,7 @@ def load_weights( def decode(self, generated_ids: List[int]) -> str: # Do not skip special tokens as they are used for custom parsing rules of the generated text return self.tokenizer.decode( - generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False + generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False ) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index c1ec1566..78e9bfe4 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -173,7 +173,7 @@ def batch_type(self) -> Type[CausalLMBatch]: def decode(self, generated_ids: List[int]) -> str: # Do not skip special tokens as they are used for custom parsing rules of the generated text return self.tokenizer.decode( - generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False + generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False ) def forward( diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index 796c33e3..a7b09a82 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -64,5 +64,5 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False def decode(self, generated_ids: List[int]) -> str: # Do not skip special tokens as they are used for custom parsing rules of the generated text return self.tokenizer.decode( - generated_ids, skip_special_tokens=False, cleanup_tokenization_spaces=False + generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False ) From 85aa7e2e7b02608eea04206b6cc0fa0ccced80ef Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 3 May 2023 11:36:24 +0200 Subject: [PATCH 209/793] feat(server): support hf endpoint weight layout (#266) --- launcher/src/main.rs | 24 +++++--- server/text_generation_server/cli.py | 55 ++++++++++++++----- server/text_generation_server/models/bloom.py | 9 +++ .../models/flash_llama.py | 19 +++++++ .../models/flash_neox.py | 10 ++++ .../models/flash_santacoder.py | 10 ++++ .../models/galactica.py | 9 +++ .../text_generation_server/models/gpt_neox.py | 9 +++ server/text_generation_server/models/opt.py | 9 +++ server/text_generation_server/models/t5.py | 9 +++ server/text_generation_server/utils/hub.py | 9 ++- 11 files changed, 146 insertions(+), 26 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 736aa5a7..41211d8a 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -512,7 +512,11 @@ enum LauncherError { WebserverCannotStart, } -fn download_model(args: &Args, running: Arc) -> Result<(), LauncherError> { +fn download_convert_model( + args: &Args, + auto_convert: bool, + running: Arc, +) -> Result<(), LauncherError> { let mut download_argv = vec![ "text-generation-server".to_string(), "download-weights".to_string(), @@ -524,6 +528,11 @@ fn download_model(args: &Args, running: Arc) -> Result<(), LauncherE "--json-output".to_string(), ]; + // Auto convert weights to safetensors + if auto_convert { + download_argv.push("--auto-convert".to_string()); + } + // Model optional revision if let Some(revision) = &args.revision { download_argv.push("--revision".to_string()); @@ -855,14 +864,11 @@ fn main() -> Result<(), LauncherError> { }) .expect("Error setting Ctrl-C handler"); - // Check if model_id is a local model - let local_path = Path::new(&args.model_id); - let is_local_model = local_path.exists() && local_path.is_dir(); - - // Download weights for sharded models - if !is_local_model && args.weights_cache_override.is_none() && num_shard > 1 { - download_model(&args, running.clone())?; - } + // auto_convert is only needed for sharded models as we do not require safetensors in + // single shard mode + let auto_convert = num_shard > 1; + // Download and convert model weights + download_convert_model(&args, auto_convert, running.clone())?; // Shared shutdown bool let shutdown = Arc::new(Mutex::new(false)); diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 94340fac..92482a94 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -63,6 +63,7 @@ def download_weights( model_id: str, revision: Optional[str] = None, extension: str = ".safetensors", + auto_convert: bool = True, logger_level: str = "INFO", json_output: bool = False, ): @@ -84,31 +85,55 @@ def download_weights( # Test if files were already download try: utils.weight_files(model_id, revision, extension) - logger.info( - "Files are already present in the local cache. " "Skipping download." - ) + logger.info("Files are already present on the host. " "Skipping download.") return # Local files not found - except utils.LocalEntryNotFoundError: + except (utils.LocalEntryNotFoundError, FileNotFoundError): pass - # Download weights directly + is_local_model = (Path(model_id).exists() and Path(model_id).is_dir()) or os.getenv( + "WEIGHTS_CACHE_OVERRIDE", None + ) is not None + + if not is_local_model: + # Try to download weights from the hub + try: + filenames = utils.weight_hub_files(model_id, revision, extension) + utils.download_weights(filenames, model_id, revision) + # Successfully downloaded weights + return + + # No weights found on the hub with this extension + except utils.EntryNotFoundError as e: + # Check if we want to automatically convert to safetensors or if we can use .bin weights instead + if not extension == ".safetensors" or not auto_convert: + raise e + + # Try to see if there are local pytorch weights try: - filenames = utils.weight_hub_files(model_id, revision, extension) - utils.download_weights(filenames, model_id, revision) - except utils.EntryNotFoundError as e: - if not extension == ".safetensors": - raise e + # Get weights for a local model, a hub cached model and inside the WEIGHTS_CACHE_OVERRIDE + local_pt_files = utils.weight_files(model_id, revision, ".bin") - logger.warning( - f"No safetensors weights found for model {model_id} at revision {revision}. " - f"Converting PyTorch weights instead." - ) + # No local pytorch weights + except utils.LocalEntryNotFoundError: + if extension == ".safetensors": + logger.warning( + f"No safetensors weights found for model {model_id} at revision {revision}. " + f"Downloading PyTorch weights." + ) - # Try to see if there are pytorch weights + # Try to see if there are pytorch weights on the hub pt_filenames = utils.weight_hub_files(model_id, revision, ".bin") # Download pytorch weights local_pt_files = utils.download_weights(pt_filenames, model_id, revision) + + if auto_convert: + logger.warning( + f"No safetensors weights found for model {model_id} at revision {revision}. " + f"Converting PyTorch weights to safetensors." + ) + + # Safetensors final filenames local_st_files = [ p.parent / f"{p.stem.lstrip('pytorch_')}.safetensors" for p in local_pt_files diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index e43a4b79..f528a430 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -223,6 +223,15 @@ def linear(input, weight, bias): if name == "word_embeddings.weight": model.lm_head._parameters["weight"] = tensor + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model: {uninitialized_parameters}" + ) + def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index e640113b..105ff519 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -139,6 +139,15 @@ def load_weights( del value + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model: {uninitialized_parameters}" + ) + torch.cuda.empty_cache() model.post_load_weights(quantize) @@ -300,5 +309,15 @@ def load_weights( else: module._buffers[param_name] = tensor + + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model: {uninitialized_parameters}" + ) + torch.cuda.empty_cache() model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index eae584ac..fc769583 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -149,4 +149,14 @@ def load_weights( module._parameters[param_name] = tensor else: module._buffers[param_name] = tensor + + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model: {uninitialized_parameters}" + ) + model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 550be956..333180e8 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -372,5 +372,15 @@ def load_weights( module._parameters[param_name] = tensor else: module._buffers[param_name] = tensor + + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model: {uninitialized_parameters}" + ) + torch.cuda.empty_cache() model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 78e9bfe4..2577f1b1 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -355,6 +355,15 @@ def linear(input, weight, bias): if name == "model.decoder.embed_tokens.weight": model.lm_head._parameters["weight"] = tensor + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model: {uninitialized_parameters}" + ) + def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 3b5fe2cc..e73a3c82 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -205,6 +205,15 @@ def linear(input, weight, bias): else: module._buffers[param_name] = tensor + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model: {uninitialized_parameters}" + ) + def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 1a21186f..50e5271e 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -210,6 +210,15 @@ def linear(input, weight, bias): if name == "model.decoder.embed_tokens.weight": model.lm_head._parameters["weight"] = tensor + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model: {uninitialized_parameters}" + ) + def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 487a5984..9e8c3c4c 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -211,6 +211,15 @@ def linear(input, weight, bias): else: module._buffers[param_name] = tensor + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model: {uninitialized_parameters}" + ) + def forward( self, input_ids, diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index 4feec8a1..030c8289 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -77,7 +77,12 @@ def weight_files( """Get the local files""" # Local model if Path(model_id).exists() and Path(model_id).is_dir(): - return list(Path(model_id).glob(f"*{extension}")) + local_files = list(Path(model_id).glob(f"*{extension}")) + if not local_files: + raise FileNotFoundError( + f"No local weights found in {model_id} with extension {extension}" + ) + return local_files try: filenames = weight_hub_files(model_id, revision, extension) @@ -98,7 +103,7 @@ def weight_files( for filename in filenames: p = Path(WEIGHTS_CACHE_OVERRIDE) / filename if not p.exists(): - raise LocalEntryNotFoundError( + raise FileNotFoundError( f"File {p} not found in {WEIGHTS_CACHE_OVERRIDE}." ) files.append(p) From b67908e0cfee66d07552e1f8af6f36e8d993f97c Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 3 May 2023 23:39:35 +0200 Subject: [PATCH 210/793] fix(launcher): pass weights cache override to the download process (#274) closes #273 --- launcher/src/main.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 41211d8a..9f8d215b 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -542,7 +542,7 @@ fn download_convert_model( // Copy current process env let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); - // If huggingface_hub_cache is set, pass it to the shard + // If huggingface_hub_cache is set, pass it to the download process // Useful when running inside a docker container if let Some(ref huggingface_hub_cache) = args.huggingface_hub_cache { env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); @@ -560,6 +560,15 @@ fn download_convert_model( env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) }; + // If args.weights_cache_override is some, pass it to the download process + // Useful when running inside a HuggingFace Inference Endpoint + if let Some(weights_cache_override) = &args.weights_cache_override { + env.push(( + "WEIGHTS_CACHE_OVERRIDE".into(), + weights_cache_override.into(), + )); + }; + // Start process tracing::info!("Starting download process."); let mut download_process = match Popen::create( From b4fe248b17ef041b72245b40c86a2157cf053a4b Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 4 May 2023 15:14:28 +0200 Subject: [PATCH 211/793] fix(launcher): handle hub branches (#278) --- router/src/main.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/router/src/main.rs b/router/src/main.rs index 28db6071..50932cd1 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -258,9 +258,10 @@ fn init_logging(otlp_endpoint: Option, json_output: bool) { /// get model info from the Huggingface Hub pub async fn get_model_info(model_id: &str, revision: &str, token: Option) -> HubModelInfo { let client = reqwest::Client::new(); - let mut builder = client.get(format!( - "https://huggingface.co/api/models/{model_id}/revision/{revision}" - )); + // Poor man's urlencode + let revision = revision.replace("/", "%2F"); + let url = format!("https://huggingface.co/api/models/{model_id}/revision/{revision}"); + let mut builder = client.get(url); if let Some(token) = token { builder = builder.bearer_auth(token); } From f08343d44d3da0b5fae80f5c283ac1b7bc6697e5 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 4 May 2023 15:22:54 +0200 Subject: [PATCH 212/793] fix(server): Removes the parallelism in file convertion (during download) (#275) --- .../text_generation_server/utils/convert.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/server/text_generation_server/utils/convert.py b/server/text_generation_server/utils/convert.py index 437e2308..8eddd76f 100644 --- a/server/text_generation_server/utils/convert.py +++ b/server/text_generation_server/utils/convert.py @@ -1,5 +1,6 @@ import concurrent import time +import datetime import torch from concurrent.futures import ThreadPoolExecutor @@ -78,17 +79,9 @@ def convert_file(pt_file: Path, st_file: Path): def convert_files(pt_files: List[Path], st_files: List[Path]): assert len(pt_files) == len(st_files) - executor = ThreadPoolExecutor(max_workers=5) - futures = [ - executor.submit(convert_file, pt_file=pt_file, st_file=st_file) - for pt_file, st_file in zip(pt_files, st_files) - ] - + N = len(pt_files) # We do this instead of using tqdm because we want to parse the logs with the launcher - start_time = time.time() - for i, future in enumerate(concurrent.futures.as_completed(futures)): - elapsed = timedelta(seconds=int(time.time() - start_time)) - remaining = len(futures) - (i + 1) - eta = (elapsed / (i + 1)) * remaining if remaining > 0 else 0 - - logger.info(f"Convert: [{i + 1}/{len(futures)}] -- ETA: {eta}") + start = datetime.datetime.now() + for i, (pt_file, sf_file) in enumerate(zip(pt_files, st_files)): + elapsed = datetime.datetime.now() - start + logger.info(f"Convert: [{i + 1}/{N}] -- Took: {elapsed}") From e68509add7ac04f8d6b4b75fab6fa65f47c2a76c Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 4 May 2023 15:29:29 +0200 Subject: [PATCH 213/793] feat(launcher): Improve error message when download process fails. (#276) --- launcher/src/main.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 9f8d215b..867bfd3d 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -627,8 +627,21 @@ fn download_convert_model( return Err(LauncherError::DownloadError); } } - _ => { - tracing::error!("Download process exited with an unknown status."); + ExitStatus::Signaled(signal) => { + let mut err = String::new(); + download_process + .stderr + .take() + .unwrap() + .read_to_string(&mut err) + .unwrap(); + tracing::error!( + "Download process was signaled to shutdown with signal {signal}: {err}" + ); + return Err(LauncherError::DownloadError); + } + e => { + tracing::error!("Download process exited with an unknown status.: {e:?}"); return Err(LauncherError::DownloadError); } } From 690fc31757a8dc10106e7f95fabe1397f21e0cc6 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 5 May 2023 15:28:08 +0200 Subject: [PATCH 214/793] fix(server): fix convert (#284) --- server/text_generation_server/utils/convert.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/convert.py b/server/text_generation_server/utils/convert.py index 8eddd76f..1bdedf58 100644 --- a/server/text_generation_server/utils/convert.py +++ b/server/text_generation_server/utils/convert.py @@ -81,7 +81,9 @@ def convert_files(pt_files: List[Path], st_files: List[Path]): N = len(pt_files) # We do this instead of using tqdm because we want to parse the logs with the launcher - start = datetime.datetime.now() + for i, (pt_file, sf_file) in enumerate(zip(pt_files, st_files)): + start = datetime.datetime.now() + convert_file(pt_file, sf_file) elapsed = datetime.datetime.now() - start logger.info(f"Convert: [{i + 1}/{N}] -- Took: {elapsed}") From 3314a46d363e9bfcf14928dc83c677dd478ff42b Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 5 May 2023 17:52:09 +0200 Subject: [PATCH 215/793] chore: add `flash-attention` to docker ignore (#287) included when building docker locally. (Where the local dirs might have the flash-attention folder.) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .dockerignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index fcfaad0d..c69283ec 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ aml target -server/transformers \ No newline at end of file +server/transformers +server/flash-attention From b4aa87db58ea3d87ff820b6d7abeaa20a987df00 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 5 May 2023 17:57:02 +0200 Subject: [PATCH 216/793] fea(server): decrease convert RAM requirements (#286) --- .../text_generation_server/utils/convert.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/server/text_generation_server/utils/convert.py b/server/text_generation_server/utils/convert.py index 1bdedf58..caf1a764 100644 --- a/server/text_generation_server/utils/convert.py +++ b/server/text_generation_server/utils/convert.py @@ -9,6 +9,7 @@ from loguru import logger from pathlib import Path from safetensors.torch import load_file, save_file +from safetensors import safe_open from typing import Dict, List @@ -46,11 +47,11 @@ def remove_shared_pointers(tensors: Dict[str, torch.Tensor]): tensors.pop(name) -def convert_file(pt_file: Path, st_file: Path): +def convert_file(pt_file: Path, sf_file: Path): """ Convert a pytorch file to a safetensors file """ - logger.info(f"Convert {pt_file} to {st_file}.") + logger.info(f"Convert {pt_file} to {sf_file}.") pt_state = torch.load(pt_file, map_location="cpu") if "state_dict" in pt_state: @@ -61,28 +62,28 @@ def convert_file(pt_file: Path, st_file: Path): # Tensors need to be contiguous pt_state = {k: v.contiguous() for k, v in pt_state.items()} - st_file.parent.mkdir(parents=True, exist_ok=True) - save_file(pt_state, str(st_file), metadata={"format": "pt"}) + sf_file.parent.mkdir(parents=True, exist_ok=True) + save_file(pt_state, str(sf_file), metadata={"format": "pt"}) # Check that both files are close in size - check_file_size(pt_file, st_file) + check_file_size(pt_file, sf_file) # Load safetensors state - st_state = load_file(str(st_file)) - for k in st_state: + for k in pt_state: pt_tensor = pt_state[k] - st_tensor = st_state[k] - if not torch.equal(pt_tensor, st_tensor): - raise RuntimeError(f"The output tensors do not match for key {k}") + with safe_open(sf_file, framework="pt") as f: + sf_tensor = f.get_tensor(k) + if not torch.equal(pt_tensor, sf_tensor): + raise RuntimeError(f"The output tensors do not match for key {k}") -def convert_files(pt_files: List[Path], st_files: List[Path]): - assert len(pt_files) == len(st_files) +def convert_files(pt_files: List[Path], sf_files: List[Path]): + assert len(pt_files) == len(sf_files) N = len(pt_files) # We do this instead of using tqdm because we want to parse the logs with the launcher - - for i, (pt_file, sf_file) in enumerate(zip(pt_files, st_files)): + + for i, (pt_file, sf_file) in enumerate(zip(pt_files, sf_files)): start = datetime.datetime.now() convert_file(pt_file, sf_file) elapsed = datetime.datetime.now() - start From e9b01b34333a9e2e4e0c54e440616a91a2d971f4 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 9 May 2023 12:36:02 +0200 Subject: [PATCH 217/793] fix(dockerfile): fix nvidia env vars (#297) Fixes #291 --- Dockerfile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index d59acc10..8810f27e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -120,6 +120,16 @@ ENV HUGGINGFACE_HUB_CACHE=/data \ NUM_SHARD=1 \ PORT=80 +# NVIDIA env vars +ENV NVIDIA_REQUIRE_CUDA cuda>=11.8 +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +# Required for nvidia-docker v1 +RUN /bin/bash -c echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \ + echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf +ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH + LABEL com.nvidia.volumes.needed="nvidia_driver" WORKDIR /usr/src @@ -165,11 +175,6 @@ FROM base as sagemaker COPY sagemaker-entrypoint.sh entrypoint.sh RUN chmod +x entrypoint.sh -# NVIDIA env vars -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 - ENTRYPOINT ["./entrypoint.sh"] # Final image From 926fd9a0105a2351055c4c526851ed86d3334105 Mon Sep 17 00:00:00 2001 From: Sai Vinay G Date: Tue, 9 May 2023 16:08:09 +0530 Subject: [PATCH 218/793] feat(router): Adding response schema for compat_generate (#292) --- router/src/server.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/router/src/server.rs b/router/src/server.rs index c5fc41ef..a9baf288 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -37,7 +37,11 @@ use utoipa_swagger_ui::SwaggerUi; path = "/", request_body = CompatGenerateRequest, responses( - (status = 200, description = "See /generate or /generate_stream"), + (status = 200, description = "See /generate or /generate_stream", + content( + ("application/json" = GenerateResponse), + ("text/event-stream" = StreamResponse), + )), (status = 424, description = "Generation Error", body = ErrorResponse, example = json ! ({"error": "Request failed during generation"})), (status = 429, description = "Model is overloaded", body = ErrorResponse, From e2502822138dfddefccde6e6a39bce22720c8cfc Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 9 May 2023 13:19:31 +0200 Subject: [PATCH 219/793] feat(docker): add benchmarking tool to docker image (#298) --- Cargo.lock | 143 +- Cargo.toml | 10 +- Dockerfile | 2 + benchmark/.gitignore | 1 - benchmark/Cargo.lock | 2884 --------------------------------- benchmark/Cargo.toml | 13 +- benchmark/rust-toolchain.toml | 3 - docs/openapi.json | 89 +- launcher/Cargo.toml | 7 +- router/Cargo.toml | 7 +- router/client/Cargo.toml | 7 +- router/src/main.rs | 34 +- router/src/server.rs | 2 +- 13 files changed, 262 insertions(+), 2940 deletions(-) delete mode 100644 benchmark/.gitignore delete mode 100644 benchmark/Cargo.lock delete mode 100644 benchmark/rust-toolchain.toml diff --git a/Cargo.lock b/Cargo.lock index 58142d4b..48595077 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,6 +134,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "average" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843ec791d3f24503bbf72bbd5e49a3ab4dbb4bcd0a8ef6b0c908efa73caa27b1" +dependencies = [ + "easy-cast", + "float-ord", + "num-traits", +] + [[package]] name = "axum" version = "0.6.15" @@ -293,6 +304,12 @@ dependencies = [ "zip", ] +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + [[package]] name = "cc" version = "1.0.79" @@ -461,6 +478,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossterm" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" +dependencies = [ + "bitflags", + "crossterm_winapi", + "libc", + "mio", + "parking_lot", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" +dependencies = [ + "winapi", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -591,6 +633,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "easy-cast" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd102ee8c418348759919b83b81cdbdc933ffe29740b903df448b4bafaa348e" +dependencies = [ + "libm", +] + [[package]] name = "either" version = "1.8.1" @@ -679,6 +730,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + [[package]] name = "float_eq" version = "1.0.1" @@ -1165,6 +1222,12 @@ version = "0.2.141" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +[[package]] +name = "libm" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" + [[package]] name = "linux-raw-sys" version = "0.3.1" @@ -1447,6 +1510,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -1903,6 +1976,19 @@ dependencies = [ "getrandom", ] +[[package]] +name = "ratatui" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcc0d032bccba900ee32151ec0265667535c230169f5a011154cdcd984e16829" +dependencies = [ + "bitflags", + "cassowary", + "crossterm", + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "raw-cpuid" version = "10.7.0" @@ -2252,6 +2338,27 @@ dependencies = [ "dirs", ] +[[package]] +name = "signal-hook" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -2407,9 +2514,28 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "text-generation-benchmark" +version = "0.7.0-dev" +dependencies = [ + "average", + "clap", + "crossterm", + "float-ord", + "ratatui", + "serde", + "serde_json", + "text-generation-client", + "thiserror", + "tokenizers", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "text-generation-client" -version = "0.6.0" +version = "0.7.0-dev" dependencies = [ "futures", "grpc-metadata", @@ -2421,12 +2547,11 @@ dependencies = [ "tonic-build", "tower", "tracing", - "tracing-error", ] [[package]] name = "text-generation-launcher" -version = "0.6.0" +version = "0.7.0-dev" dependencies = [ "clap", "ctrlc", @@ -2442,7 +2567,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.6.0" +version = "0.7.0-dev" dependencies = [ "async-stream", "axum", @@ -2803,16 +2928,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-error" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" -dependencies = [ - "tracing", - "tracing-subscriber", -] - [[package]] name = "tracing-futures" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index af479c04..d34b10bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,17 @@ [workspace] members = [ + "benchmark", "router", "router/client", "router/grpc-metadata", "launcher" ] -exclude = [ - "benchmark" -] + +[workspace.package] +version = "0.7.0-dev" +edition = "2021" +authors = ["Olivier Dehaene"] +homepage = "https://github.com/huggingface/text-generation-inference" [profile.release] debug = 1 diff --git a/Dockerfile b/Dockerfile index 8810f27e..7e01d42f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -164,6 +164,8 @@ RUN cd server && \ pip install -r requirements.txt && \ pip install ".[bnb, accelerate]" --no-cache-dir +# Install benchmarker +COPY --from=builder /usr/src/target/release/text-generation-benchmark /usr/local/bin/text-generation-benchmark # Install router COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bin/text-generation-router # Install launcher diff --git a/benchmark/.gitignore b/benchmark/.gitignore deleted file mode 100644 index 1de56593..00000000 --- a/benchmark/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target \ No newline at end of file diff --git a/benchmark/Cargo.lock b/benchmark/Cargo.lock deleted file mode 100644 index 81c1180e..00000000 --- a/benchmark/Cargo.lock +++ /dev/null @@ -1,2884 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "aes" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", - "opaque-debug", -] - -[[package]] -name = "aho-corasick" -version = "0.7.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - -[[package]] -name = "anstream" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-wincon", - "concolor-override", - "concolor-query", - "is-terminal", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" - -[[package]] -name = "anstyle-parse" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-wincon" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" -dependencies = [ - "anstyle", - "windows-sys 0.45.0", -] - -[[package]] -name = "anyhow" -version = "1.0.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" - -[[package]] -name = "async-stream" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "async-trait" -version = "0.1.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.11", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "average" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843ec791d3f24503bbf72bbd5e49a3ab4dbb4bcd0a8ef6b0c908efa73caa27b1" -dependencies = [ - "easy-cast", - "float-ord", - "num-traits", -] - -[[package]] -name = "axum" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f8ccfd9221ee7d1f3d4b33e1f8319b3a81ed8f61f2ea40b37b859794b4491" -dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" - -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bumpalo" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "bytes" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" - -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "cached-path" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097968e38f1319207f057d0f4d76452e4f4f847a5de61c5215379f297fa034f3" -dependencies = [ - "flate2", - "fs2", - "glob", - "indicatif 0.16.2", - "log", - "rand", - "reqwest", - "serde", - "serde_json", - "sha2", - "tar", - "tempfile", - "thiserror", - "zip", -] - -[[package]] -name = "cassowary" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" - -[[package]] -name = "cc" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" -dependencies = [ - "jobserver", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "cipher" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" -dependencies = [ - "generic-array", -] - -[[package]] -name = "clap" -version = "4.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" -dependencies = [ - "clap_builder", - "clap_derive", - "once_cell", -] - -[[package]] -name = "clap_builder" -version = "4.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" -dependencies = [ - "anstream", - "anstyle", - "bitflags", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.11", -] - -[[package]] -name = "clap_lex" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" - -[[package]] -name = "concolor-override" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" - -[[package]] -name = "concolor-query" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" -dependencies = [ - "windows-sys 0.45.0", -] - -[[package]] -name = "console" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys 0.42.0", -] - -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - -[[package]] -name = "core-foundation" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" - -[[package]] -name = "cpufeatures" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" -dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossterm" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" -dependencies = [ - "bitflags", - "crossterm_winapi", - "libc", - "mio", - "parking_lot", - "signal-hook", - "signal-hook-mio", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" -dependencies = [ - "winapi", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "dashmap" -version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" -dependencies = [ - "cfg-if", - "hashbrown", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "derive_builder" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "derive_builder_macro" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" -dependencies = [ - "derive_builder_core", - "syn 1.0.109", -] - -[[package]] -name = "digest" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - -[[package]] -name = "dirs" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "easy-cast" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd102ee8c418348759919b83b81cdbdc933ffe29740b903df448b4bafaa348e" -dependencies = [ - "libm", -] - -[[package]] -name = "either" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" - -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "encoding_rs" -version = "0.8.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "errno" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.45.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "esaxx-rs" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f748b253ceca9fed5f42f8b5ceb3851e93102199bc25b64b65369f76e5c0a35" -dependencies = [ - "cc", -] - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "filetime" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.2.16", - "windows-sys 0.45.0", -] - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] -name = "flate2" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "float-ord" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "futures" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" - -[[package]] -name = "futures-executor" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" - -[[package]] -name = "futures-macro" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "futures-sink" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" - -[[package]] -name = "futures-task" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" - -[[package]] -name = "futures-util" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "grpc-metadata" -version = "0.1.0" -dependencies = [ - "opentelemetry", - "tonic", - "tracing", - "tracing-opentelemetry", -] - -[[package]] -name = "h2" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "http" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" - -[[package]] -name = "hyper" -version = "0.14.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "indicatif" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" -dependencies = [ - "console", - "lazy_static", - "number_prefix 0.3.0", - "regex", -] - -[[package]] -name = "indicatif" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" -dependencies = [ - "console", - "lazy_static", - "number_prefix 0.4.0", - "regex", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.45.0", -] - -[[package]] -name = "ipnet" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" - -[[package]] -name = "is-terminal" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" -dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.45.0", -] - -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" - -[[package]] -name = "jobserver" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.140" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" - -[[package]] -name = "libm" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" - -[[package]] -name = "linux-raw-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd550e73688e6d578f0ac2119e32b797a327631a42f9433e59d02e139c8df60d" - -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "macro_rules_attribute" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862" -dependencies = [ - "macro_rules_attribute-proc_macro", - "paste", -] - -[[package]] -name = "macro_rules_attribute-proc_macro" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" - -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "matchit" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" - -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" -dependencies = [ - "autocfg", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" -dependencies = [ - "libc", - "log", - "wasi", - "windows-sys 0.45.0", -] - -[[package]] -name = "monostate" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a" -dependencies = [ - "monostate-impl", - "serde", -] - -[[package]] -name = "monostate-impl" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.11", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - -[[package]] -name = "num-traits" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" -dependencies = [ - "hermit-abi 0.2.6", - "libc", -] - -[[package]] -name = "number_prefix" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" - -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - -[[package]] -name = "once_cell" -version = "1.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" - -[[package]] -name = "onig" -version = "6.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" -dependencies = [ - "bitflags", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - -[[package]] -name = "openssl" -version = "0.10.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "opentelemetry" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" -dependencies = [ - "opentelemetry_api", - "opentelemetry_sdk", -] - -[[package]] -name = "opentelemetry_api" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" -dependencies = [ - "fnv", - "futures-channel", - "futures-util", - "indexmap", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" -dependencies = [ - "async-trait", - "crossbeam-channel", - "dashmap", - "fnv", - "futures-channel", - "futures-executor", - "futures-util", - "once_cell", - "opentelemetry_api", - "percent-encoding", - "rand", - "thiserror", -] - -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "windows-sys 0.45.0", -] - -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core", - "subtle", -] - -[[package]] -name = "paste" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" - -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", -] - -[[package]] -name = "percent-encoding" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" - -[[package]] -name = "petgraph" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" -dependencies = [ - "fixedbitset", - "indexmap", -] - -[[package]] -name = "pin-project" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - -[[package]] -name = "proc-macro2" -version = "1.0.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e472a104799c74b514a57226160104aa483546de37e839ec50e3c2e41dd87534" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c828f93f5ca4826f97fedcbd3f9a536c16b12cff3dbbb4a007f932bbad95b12" -dependencies = [ - "bytes", - "heck", - "itertools 0.10.5", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 1.0.109", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b" -dependencies = [ - "anyhow", - "itertools 0.10.5", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-types" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88" -dependencies = [ - "prost", -] - -[[package]] -name = "quote" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "ratatui" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc0d032bccba900ee32151ec0265667535c230169f5a011154cdcd984e16829" -dependencies = [ - "bitflags", - "cassowary", - "crossterm", - "unicode-segmentation", - "unicode-width", -] - -[[package]] -name = "rayon" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-cond" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd1259362c9065e5ea39a789ef40b1e3fd934c94beb7b5ab3ac6629d3b5e7cb7" -dependencies = [ - "either", - "itertools 0.8.2", - "rayon", -] - -[[package]] -name = "rayon-core" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "num_cpus", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", -] - -[[package]] -name = "regex" -version = "1.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "reqwest" -version = "0.11.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" -dependencies = [ - "base64 0.21.0", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - -[[package]] -name = "rustix" -version = "0.37.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e78cc525325c06b4a7ff02db283472f3c042b7ff0c391f96c6d5ac6f4f91b75" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys", - "windows-sys 0.45.0", -] - -[[package]] -name = "rustversion" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" - -[[package]] -name = "ryu" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" - -[[package]] -name = "schannel" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" -dependencies = [ - "windows-sys 0.42.0", -] - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "security-framework" -version = "2.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "serde" -version = "1.0.159" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.159" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.11", -] - -[[package]] -name = "serde_json" -version = "1.0.95" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "sha1" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "signal-hook" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-mio" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" -dependencies = [ - "libc", - "mio", - "signal-hook", -] - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "slab" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "spm_precompiled" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" -dependencies = [ - "base64 0.13.1", - "nom", - "serde", - "unicode-segmentation", -] - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "subtle" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e3787bb71465627110e7d87ed4faaa36c1f61042ee67badb9e2ef173accc40" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "tar" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" -dependencies = [ - "filetime", - "libc", - "xattr", -] - -[[package]] -name = "tempfile" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" -dependencies = [ - "cfg-if", - "fastrand", - "redox_syscall 0.3.5", - "rustix", - "windows-sys 0.45.0", -] - -[[package]] -name = "text-generation-benchmark" -version = "0.1.0" -dependencies = [ - "average", - "clap", - "crossterm", - "float-ord", - "ratatui", - "serde", - "serde_json", - "text-generation-client", - "thiserror", - "tokenizers", - "tokio", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "text-generation-client" -version = "0.6.0" -dependencies = [ - "futures", - "grpc-metadata", - "prost", - "prost-build", - "thiserror", - "tokio", - "tonic", - "tonic-build", - "tower", - "tracing", - "tracing-error", -] - -[[package]] -name = "thiserror" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.11", -] - -[[package]] -name = "thread_local" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" -dependencies = [ - "cfg-if", - "once_cell", -] - -[[package]] -name = "time" -version = "0.3.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" -dependencies = [ - "serde", - "time-core", -] - -[[package]] -name = "time-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokenizers" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5" -dependencies = [ - "aho-corasick", - "cached-path", - "clap", - "derive_builder", - "dirs", - "esaxx-rs", - "getrandom", - "indicatif 0.15.0", - "itertools 0.9.0", - "lazy_static", - "log", - "macro_rules_attribute", - "monostate", - "onig", - "paste", - "rand", - "rayon", - "rayon-cond", - "regex", - "regex-syntax", - "reqwest", - "serde", - "serde_json", - "spm_precompiled", - "thiserror", - "unicode-normalization-alignments", - "unicode-segmentation", - "unicode_categories", -] - -[[package]] -name = "tokio" -version = "1.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" -dependencies = [ - "autocfg", - "bytes", - "libc", - "mio", - "num_cpus", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.45.0", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-macros" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.11", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "tonic" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.13.1", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "prost-derive", - "tokio", - "tokio-stream", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - -[[package]] -name = "tonic-build" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" -dependencies = [ - "cfg-if", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "tracing-core" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-error" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" -dependencies = [ - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - -[[package]] -name = "tracing-log" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" -dependencies = [ - "lazy_static", - "log", - "tracing-core", -] - -[[package]] -name = "tracing-opentelemetry" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" -dependencies = [ - "once_cell", - "opentelemetry", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", -] - -[[package]] -name = "tracing-serde" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" -dependencies = [ - "serde", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex", - "serde", - "serde_json", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", - "tracing-serde", -] - -[[package]] -name = "try-lock" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" - -[[package]] -name = "typenum" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - -[[package]] -name = "unicode-bidi" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" - -[[package]] -name = "unicode-ident" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-normalization-alignments" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" -dependencies = [ - "smallvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - -[[package]] -name = "url" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - -[[package]] -name = "valuable" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "want" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" -dependencies = [ - "log", - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 1.0.109", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" - -[[package]] -name = "web-sys" -version = "0.3.61" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "winreg" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" -dependencies = [ - "winapi", -] - -[[package]] -name = "xattr" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" -dependencies = [ - "libc", -] - -[[package]] -name = "zip" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" -dependencies = [ - "aes", - "byteorder", - "bzip2", - "constant_time_eq", - "crc32fast", - "crossbeam-utils", - "flate2", - "hmac", - "pbkdf2", - "sha1", - "time", - "zstd", -] - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" -dependencies = [ - "cc", - "libc", - "pkg-config", -] diff --git a/benchmark/Cargo.toml b/benchmark/Cargo.toml index 4a6651c8..a4e215fc 100644 --- a/benchmark/Cargo.toml +++ b/benchmark/Cargo.toml @@ -1,15 +1,10 @@ [package] name = "text-generation-benchmark" -version = "0.1.0" -edition = "2021" -authors = ["Olivier Dehaene"] description = "Text Generation Benchmarking tool" - -[profile.release] -debug = 1 -incremental = true -lto = "off" -panic = "abort" +version.workspace = true +edition.workspace = true +authors.workspace = true +homepage.workspace = true [lib] path = "src/lib.rs" diff --git a/benchmark/rust-toolchain.toml b/benchmark/rust-toolchain.toml deleted file mode 100644 index dcdb06e9..00000000 --- a/benchmark/rust-toolchain.toml +++ /dev/null @@ -1,3 +0,0 @@ -[toolchain] -channel = "1.67.0" -components = ["rustfmt", "clippy"] \ No newline at end of file diff --git a/docs/openapi.json b/docs/openapi.json index e7b7b140..fbac7786 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.6.0" + "version": "0.7.0-dev" }, "paths": { "/": { @@ -33,7 +33,19 @@ }, "responses": { "200": { - "description": "See /generate or /generate_stream" + "description": "Generated Text", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenerateResponse" + } + }, + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/StreamResponse" + } + } + } }, "422": { "description": "Input validation error", @@ -584,11 +596,73 @@ "type": "object", "required": [ "model_id", + "model_dtype", + "model_device_type", + "max_concurrent_requests", + "max_best_of", + "max_stop_sequences", + "max_input_length", + "max_total_tokens", + "waiting_served_ratio", + "max_batch_total_tokens", + "max_waiting_tokens", + "validation_workers", "version" ], "properties": { + "docker_label": { + "type": "string", + "example": "null", + "nullable": true + }, + "max_batch_total_tokens": { + "type": "integer", + "format": "int32", + "example": "32000", + "minimum": 0.0 + }, + "max_best_of": { + "type": "integer", + "example": "2", + "minimum": 0.0 + }, + "max_concurrent_requests": { + "type": "integer", + "description": "Router Parameters", + "example": "128", + "minimum": 0.0 + }, + "max_input_length": { + "type": "integer", + "example": "1024", + "minimum": 0.0 + }, + "max_stop_sequences": { + "type": "integer", + "example": "4", + "minimum": 0.0 + }, + "max_total_tokens": { + "type": "integer", + "example": "2048", + "minimum": 0.0 + }, + "max_waiting_tokens": { + "type": "integer", + "example": "20", + "minimum": 0.0 + }, + "model_device_type": { + "type": "string", + "example": "cuda" + }, + "model_dtype": { + "type": "string", + "example": "torch.float16" + }, "model_id": { "type": "string", + "description": "Model info", "example": "bigscience/blomm-560m" }, "model_pipeline_tag": { @@ -606,9 +680,20 @@ "example": "null", "nullable": true }, + "validation_workers": { + "type": "integer", + "example": "2", + "minimum": 0.0 + }, "version": { "type": "string", + "description": "Router Info", "example": "0.5.0" + }, + "waiting_served_ratio": { + "type": "number", + "format": "float", + "example": "1.2" } } }, diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 885dcd98..6439e960 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -1,9 +1,10 @@ [package] name = "text-generation-launcher" -version = "0.6.0" -edition = "2021" -authors = ["Olivier Dehaene"] description = "Text Generation Launcher" +version.workspace = true +edition.workspace = true +authors.workspace = true +homepage.workspace = true [dependencies] clap = { version = "4.1.4", features = ["derive", "env"] } diff --git a/router/Cargo.toml b/router/Cargo.toml index 4fa523a5..6503e1bd 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -1,10 +1,11 @@ [package] name = "text-generation-router" -version = "0.6.0" -edition = "2021" -authors = ["Olivier Dehaene"] description = "Text Generation Webserver" build = "build.rs" +version.workspace = true +edition.workspace = true +authors.workspace = true +homepage.workspace = true [lib] path = "src/lib.rs" diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index b32402a5..db7245e0 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -1,7 +1,9 @@ [package] name = "text-generation-client" -version = "0.6.0" -edition = "2021" +version.workspace = true +edition.workspace = true +authors.workspace = true +homepage.workspace = true [dependencies] futures = "^0.3" @@ -12,7 +14,6 @@ tokio = { version = "^1.25", features = ["sync"] } tonic = "^0.8" tower = "^0.4" tracing = "^0.1" -tracing-error = "^0.2" [build-dependencies] tonic-build = "0.8.4" diff --git a/router/src/main.rs b/router/src/main.rs index 50932cd1..5ad49003 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -9,6 +9,7 @@ use opentelemetry::{global, KeyValue}; use opentelemetry_otlp::WithExportConfig; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path; +use std::time::Duration; use text_generation_client::ShardedClient; use text_generation_router::{server, HubModelInfo}; use tokenizers::{FromPretrainedParameters, Tokenizer}; @@ -125,7 +126,7 @@ fn main() -> Result<(), std::io::Error> { .block_on(async { init_logging(otlp_endpoint, json_output); - if let Some(max_batch_size) = max_batch_size{ + if let Some(max_batch_size) = max_batch_size { tracing::warn!("`max-batch-size` is deprecated. Use `max-batch-total-tokens` instead"); max_batch_total_tokens = (max_batch_size * max_total_tokens) as u32; tracing::warn!("Overriding `max-batch-total-tokens` value with `max-batch-size` * `max-total-tokens` = {max_batch_total_tokens}"); @@ -145,7 +146,10 @@ fn main() -> Result<(), std::io::Error> { sha: None, pipeline_tag: None, }, - false => get_model_info(&tokenizer_name, &revision, authorization_token).await, + false => get_model_info(&tokenizer_name, &revision, authorization_token).await.unwrap_or({ + tracing::warn!("Could not retrieve model info from the Hugging Face hub."); + HubModelInfo { model_id: tokenizer_name.to_string(), sha: None, pipeline_tag: None } + }), }; // if pipeline-tag == text-generation we default to return_full_text = true @@ -195,7 +199,7 @@ fn main() -> Result<(), std::io::Error> { addr, cors_allow_origin, ) - .await; + .await; Ok(()) }) } @@ -256,22 +260,24 @@ fn init_logging(otlp_endpoint: Option, json_output: bool) { } /// get model info from the Huggingface Hub -pub async fn get_model_info(model_id: &str, revision: &str, token: Option) -> HubModelInfo { +pub async fn get_model_info( + model_id: &str, + revision: &str, + token: Option, +) -> Option { let client = reqwest::Client::new(); // Poor man's urlencode - let revision = revision.replace("/", "%2F"); + let revision = revision.replace('/', "%2F"); let url = format!("https://huggingface.co/api/models/{model_id}/revision/{revision}"); - let mut builder = client.get(url); + let mut builder = client.get(url).timeout(Duration::from_secs(5)); if let Some(token) = token { builder = builder.bearer_auth(token); } - let model_info = builder - .send() - .await - .expect("Could not connect to hf.co") - .text() - .await - .expect("error when retrieving model info from hf.co"); - serde_json::from_str(&model_info).expect("unable to parse model info") + let response = builder.send().await.ok()?; + + if response.status().is_success() { + return serde_json::from_str(&response.text().await.ok()?).ok(); + } + None } diff --git a/router/src/server.rs b/router/src/server.rs index a9baf288..162b6fd9 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -37,7 +37,7 @@ use utoipa_swagger_ui::SwaggerUi; path = "/", request_body = CompatGenerateRequest, responses( - (status = 200, description = "See /generate or /generate_stream", + (status = 200, description = "Generated Text", content( ("application/json" = GenerateResponse), ("text/event-stream" = StreamResponse), From bc5c07231ec6284b94a8ff78a54d6da4b505a93f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 9 May 2023 14:39:59 +0200 Subject: [PATCH 220/793] fix(docker): fix docker build (#299) --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7e01d42f..d68703cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ FROM chef as planner COPY Cargo.toml Cargo.toml COPY rust-toolchain.toml rust-toolchain.toml COPY proto proto +COPY benchmark benchmark COPY router router COPY launcher launcher RUN cargo chef prepare --recipe-path recipe.json @@ -27,6 +28,7 @@ RUN cargo chef cook --release --recipe-path recipe.json COPY Cargo.toml Cargo.toml COPY rust-toolchain.toml rust-toolchain.toml COPY proto proto +COPY benchmark benchmark COPY router router COPY launcher launcher RUN cargo build --release From ad66f6ef9ac3677e6259b85026f911a555970801 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 9 May 2023 18:26:19 +0200 Subject: [PATCH 221/793] feat(server): optim flash causal lm decode_token (#285) --- .../custom_modeling/flash_llama_modeling.py | 7 +- .../custom_modeling/flash_neox_modeling.py | 7 +- .../flash_santacoder_modeling.py | 7 +- .../models/flash_causal_lm.py | 367 ++++++++++++------ .../models/flash_llama.py | 4 +- .../models/flash_neox.py | 2 +- .../models/flash_santacoder.py | 4 +- 7 files changed, 258 insertions(+), 140 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index de9b22da..1293124a 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -554,6 +554,7 @@ def forward( input_ids, position_ids, cu_seqlens, + cu_seqlens_q, max_s, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, @@ -575,15 +576,11 @@ def forward( ) ) layer_past_present_indices = None - cu_seqlens_q = None slice_past_index = len(hidden_states) # Decode else: # Create indices from cumulative sequence lengths layer_past_present_indices = cu_seqlens[1:] - 1 - cu_seqlens_q = torch.arange( - cu_seqlens.shape[0], dtype=torch.int32, device=hidden_states.device - ) slice_past_index = None # Get rotary cos and sin for this forward @@ -650,6 +647,7 @@ def forward( input_ids, position_ids, cu_seqlens, + cu_seqlens_q, max_s, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, @@ -658,6 +656,7 @@ def forward( input_ids, position_ids, cu_seqlens, + cu_seqlens_q, max_s, past_key_values, pre_allocate_past_size, diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index cc9b292f..ae1465ab 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -617,6 +617,7 @@ def forward( input_ids, position_ids, cu_seqlens, + cu_seqlens_q, max_s, past_key_values=None, pre_allocate_past_size: Optional[int] = None, @@ -638,15 +639,11 @@ def forward( ) ) layer_past_present_indices = None - cu_seqlens_q = None slice_past_index = len(hidden_states) # Decode else: # Create indices from cumulative sequence lengths layer_past_present_indices = cu_seqlens[1:] - 1 - cu_seqlens_q = torch.arange( - cu_seqlens.shape[0], dtype=torch.int32, device=hidden_states.device - ) slice_past_index = None # Get rotary cos and sin for this forward @@ -726,6 +723,7 @@ def forward( input_ids, position_ids, cu_seqlens, + cu_seqlens_q, max_s, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, @@ -734,6 +732,7 @@ def forward( input_ids, position_ids, cu_seqlens, + cu_seqlens_q, max_s, past_key_values, pre_allocate_past_size, diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 71182f8d..20ad8385 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -484,6 +484,7 @@ def forward( input_ids, position_ids, cu_seqlens, + cu_seqlens_q, max_s, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, @@ -507,15 +508,11 @@ def forward( ) ) layer_past_present_indices = None - cu_seqlens_q = None slice_past_index = len(hidden_states) # Decode else: # Create indices from cumulative sequence lengths layer_past_present_indices = cu_seqlens[1:] - 1 - cu_seqlens_q = torch.arange( - cu_seqlens.shape[0], dtype=torch.int32, device=hidden_states.device - ) slice_past_index = None residual = None @@ -566,6 +563,7 @@ def forward( input_ids, position_ids, cu_seqlens, + cu_seqlens_q, max_s, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, @@ -574,6 +572,7 @@ def forward( input_ids, position_ids, cu_seqlens, + cu_seqlens_q, max_s, past_key_values, pre_allocate_past_size, diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 413866d1..b51a3dc6 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -1,6 +1,8 @@ import torch import torch.distributed +import numpy as np + from torch.nn import functional as F from dataclasses import dataclass @@ -33,12 +35,16 @@ class FlashCausalLMBatch(Batch): requests_idx_mapping: Dict[int, int] # Decoder values - input_ids: List[torch.Tensor] - position_ids: List[torch.Tensor] + input_ids: torch.Tensor + position_ids: torch.Tensor + # cumulative sequence lengths - cu_seqlens: List[int] + cu_seqlens: torch.Tensor + # cumulative query sequence lengths, only used in decode + cu_seqlens_q: Optional[torch.Tensor] + # past key values, only used in decode + past_key_values: Optional[torch.Tensor] max_seqlen: int - past_key_values: Optional[Union[torch.Tensor, List[torch.Tensor]]] # All tokens all_input_ids: List[List[int]] @@ -53,9 +59,6 @@ class FlashCausalLMBatch(Batch): next_token_choosers: List[NextTokenChooser] stopping_criterias: List[StoppingCriteria] - # Constant shared tensor, ref here just so that it's accessible in concatentate() - past_pad: Optional[torch.Tensor] - # Maximum number of tokens this batch will grow to max_tokens: int @@ -74,7 +77,6 @@ def from_pb( tokenizer: PreTrainedTokenizerBase, device: torch.device, ) -> "FlashCausalLMBatch": - input_ids = [] position_ids = [] cu_seqlens = [0] max_seqlen = 0 @@ -83,7 +85,6 @@ def from_pb( offsets = [] token_offsets = [] all_input_ids = [] - all_input_ids_tensor = [] requests_idx_mapping = {} next_token_choosers = [] @@ -109,15 +110,11 @@ def from_pb( offsets.append(None) token_offsets.append(None) - all_input_ids.append(tokenized_input) - tokenized_input = torch.tensor(tokenized_input, device=device) - input_ids.append(tokenized_input) + all_input_ids.append(tokenized_input) # Position ids - position_ids.append( - torch.arange(0, input_length, dtype=torch.int32, device=device) - ) + position_ids.append(np.arange(0, input_length)) # Add cumulative lengths of all previous inputs cu_seqlens.append(cumulative_length + input_length) @@ -130,14 +127,19 @@ def from_pb( max_new_tokens = stopping_criteria.max_new_tokens stopping_criterias.append(stopping_criteria) - all_input_ids_tensor.append( - F.pad(tokenized_input, (0, stopping_criteria.max_new_tokens)) - ) - # Update cumulative_length += input_length max_tokens += input_length + max_new_tokens + # Create tensors on device + input_ids = torch.tensor( + np.concatenate(all_input_ids), dtype=torch.int64, device=device + ) + position_ids = torch.tensor( + np.concatenate(position_ids), dtype=torch.int32, device=device + ) + cu_seqlens = torch.tensor(cu_seqlens, device=device, dtype=torch.int32) + return cls( batch_id=pb.id, requests=pb.requests, @@ -145,16 +147,16 @@ def from_pb( input_ids=input_ids, position_ids=position_ids, cu_seqlens=cu_seqlens, + cu_seqlens_q=None, max_seqlen=max_seqlen, past_key_values=None, input_lengths=input_lengths, offsets=offsets, token_offsets=token_offsets, all_input_ids=all_input_ids, - all_input_ids_tensor=all_input_ids_tensor, + all_input_ids_tensor=[], next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, - past_pad=None, max_tokens=max_tokens, ) @@ -174,9 +176,13 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": # New values after filtering requests_idx_mapping = {} - input_ids = [] - position_ids = [] - cu_seqlens = [0] + input_ids = self.input_ids.new_empty(len(requests)) + position_ids = self.position_ids.new_empty(len(requests)) + # Create on CPU to only move to GPU once instead of at every copy + cu_seqlens = torch.zeros(len(requests) + 1, dtype=torch.int32) + cu_seqlens_q = torch.arange( + 0, len(requests) + 1, device=self.cu_seqlens_q.device, dtype=torch.int32 + ) max_seqlen = 0 past_key_values = [] @@ -199,16 +205,18 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": # Get length request_input_length = self.input_lengths[idx] - input_ids.append(self.input_ids[idx]) - position_ids.append(self.position_ids[idx]) - cu_seqlens.append(cumulative_length + request_input_length) + # Copy tensors (GPU) + input_ids[i] = self.input_ids[idx] + position_ids[i] = self.position_ids[idx] + + # Copy to tensor (CPU) + cu_seqlens[i + 1] = cumulative_length + request_input_length max_seqlen = max(max_seqlen, request_input_length) - # True index for past - past_key_values.append(self.past_key_values[2 * idx]) - if not single_request: - # Add one padding - past_key_values.append(self.past_pad) + # Slice from past + past_key_values.append( + self.past_key_values[:, self.cu_seqlens[idx] : self.cu_seqlens[idx + 1]] + ) all_input_ids.append(self.all_input_ids[idx]) all_input_ids_tensor.append(self.all_input_ids_tensor[idx]) @@ -229,7 +237,7 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": if single_request: # Preallocate tensor for bs = 1 case - past_key_values = torch.nn.functional.pad( + past_key_values = F.pad( past_key_values[0], ( 0, @@ -243,15 +251,21 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": - stopping_criterias[0].current_tokens, ), ) + else: + # Cat all past + past_key_values = torch.cat(past_key_values, dim=1) + + # Move to GPU now that we have the whole tensor + cu_seqlens = cu_seqlens.to(self.cu_seqlens.device) return FlashCausalLMBatch( batch_id=self.batch_id, - past_pad=self.past_pad, requests=requests, requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, cu_seqlens=cu_seqlens, + cu_seqlens_q=cu_seqlens_q, max_seqlen=max_seqlen, past_key_values=past_key_values, input_lengths=input_lengths, @@ -271,9 +285,16 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch requests = [] requests_idx_mapping = {} - input_ids = [] - position_ids = [] + total_batch_size = sum([len(b) for b in batches]) + + device = batches[0].input_ids.device + + input_ids = batches[0].input_ids.new_empty(total_batch_size) + position_ids = batches[0].position_ids.new_empty(total_batch_size) cu_seqlens = [0] + cu_seqlens_q = torch.arange( + 0, total_batch_size + 1, device=device, dtype=torch.int32 + ) max_seqlen = 0 past_key_values = [] @@ -302,22 +323,25 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch for k, v in batch.requests_idx_mapping.items(): requests_idx_mapping[k] = v + cumulative_batch_size - input_ids.extend(batch.input_ids) - position_ids.extend(batch.position_ids) + start_index = cumulative_batch_size + end_index = cumulative_batch_size + len(batch) + + # Copy tensors (GPU) + input_ids[start_index:end_index] = batch.input_ids + position_ids[start_index:end_index] = batch.position_ids + # Add cumulative lengths of all previous inputs cu_seqlens.extend([l + cumulative_length for l in batch.cu_seqlens[1:]]) max_seqlen = max(max_seqlen, batch.max_seqlen) if len(batch) != 1: - past_key_values.extend(batch.past_key_values) + past_key_values.append(batch.past_key_values) else: # past was pre-allocated for this batch # We need to slice to remove the padding past_key_values.append( batch.past_key_values[:, : batch.input_lengths[0]] ) - # Add one padding - past_key_values.append(batch.past_pad) all_input_ids.extend(batch.all_input_ids) all_input_ids_tensor.extend(batch.all_input_ids_tensor) @@ -334,14 +358,19 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch cumulative_batch_size += len(batch) max_tokens += batch.max_tokens + # Cat past + past_key_values = torch.cat(past_key_values, dim=1) + # Create final tensor on GPU + cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32, device=device) + return FlashCausalLMBatch( batch_id=batches[0].batch_id, - past_pad=batches[0].past_pad, requests=requests, requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, cu_seqlens=cu_seqlens, + cu_seqlens_q=cu_seqlens_q, max_seqlen=max_seqlen, past_key_values=past_key_values, input_lengths=input_lengths, @@ -367,10 +396,9 @@ def __init__( quantize: bool = False, decode_buffer: int = 3, ): - self.past_pad = None if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + dtype = torch.float16 else: raise NotImplementedError("FlashCausalLM is only available on GPU") @@ -410,6 +438,7 @@ def forward( input_ids: torch.Tensor, position_ids: torch.Tensor, cu_seqlens: torch.Tensor, + cu_seqlens_q: Optional[torch.Tensor], max_s: int, past_key_values: Optional = None, pre_allocate_past_size: Optional[int] = None, @@ -419,6 +448,7 @@ def forward( input_ids=input_ids, position_ids=position_ids, cu_seqlens=cu_seqlens, + cu_seqlens_q=cu_seqlens_q, max_s=max_s, past_key_values=past_key_values, pre_allocate_past_size=pre_allocate_past_size, @@ -428,22 +458,9 @@ def forward( def generate_token( self, batch: FlashCausalLMBatch ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: - # Shortcut when batch_size == 1 - if len(batch) == 1: - input_ids = batch.input_ids[0].view(-1) - # No need to slice as flash attention will take care of it with cu_seqlens - past_key_values = batch.past_key_values - else: - # Concatenate tensors - input_ids = torch.cat(batch.input_ids).view(-1) - past_key_values = ( - torch.cat(batch.past_key_values, dim=1) - if batch.past_key_values is not None - else None - ) + prefill = batch.past_key_values is None - # if prefill and bs == 1 - if past_key_values is None and len(batch) == 1: + if prefill and len(batch) == 1: # Ask to pre-allocate kv to its max size # == number of tokens + max_new_tokens pre_allocate_past_size = ( @@ -452,42 +469,74 @@ def generate_token( else: pre_allocate_past_size = None - # Concatenate when prefill, torch.tensor when decode - position_ids = ( - torch.tensor(batch.position_ids, device=self.device) - if batch.past_key_values is not None - else torch.cat(batch.position_ids) - ) - cu_seqlens = torch.tensor( - batch.cu_seqlens, device=self.device, dtype=torch.int32 - ) - out, present = self.forward( - input_ids, - position_ids, - cu_seqlens, + batch.input_ids, + batch.position_ids, + batch.cu_seqlens, + batch.cu_seqlens_q, batch.max_seqlen, - past_key_values, + batch.past_key_values, pre_allocate_past_size, ) - # Initialize past_key_values in prefill - if batch.past_key_values is None: - # Initialize past padding tensor - if self.past_pad is None: - self.past_pad = present.new_zeros( - present.shape[0], 1, *present.shape[2:] + if prefill: + if len(batch) > 1: + # We create the prefill_tokens_indices tensor that will be used to gather prefill logprobs + # When batch == 1, we will just use the batch.input_ids values directly + prefill_tokens_indices = batch.input_ids.new_zeros(len(batch.input_ids)) + + # Create batch.cu_seqlens_q for decode + batch.cu_seqlens_q = torch.arange( + 0, len(batch) + 1, device=self.device, dtype=torch.int32 + ) + next_input_ids = batch.input_ids.new_empty(len(batch)) + next_position_ids = batch.position_ids.new_empty(len(batch)) + else: + prefill_logprobs = None + next_input_ids = batch.input_ids + next_position_ids = batch.position_ids + + next_token_logprobs = out.new_empty(len(batch)) + + # Prepare past for next decode + if len(batch) > 1: + # Used to slice next batch past + past_indices = torch.empty( + present.shape[1], dtype=torch.int64, device=self.device + ) + batch.past_key_values = present.new_empty( + ( + present.shape[0], + present.shape[1] + len(batch.requests), + *present.shape[2:], ) - # Set in batch in case it needs to be used later in concatenate() - batch.past_pad = self.past_pad - if len(batch) == 1: - # present is already pre-padded - batch.past_key_values = present - else: - # Add padding after each sequence - # This will have the correct shape after the final past_key_values concatenation before the model - # forward - batch.past_key_values = [None, self.past_pad] * len(batch) + ) + + # It is actually faster to do a whole other for loop here as the copy from present to past is fairly slow + # and will run asynchronously while we do the next for loop + cumulative_length = 0 + for i, input_length in enumerate(batch.input_lengths): + # Indexing metadata + start_index = cumulative_length + end_index = cumulative_length + input_length + + # Indices to copy present at the correct place in past_key_values + torch.arange( + start_index + i, + end_index + i, + dtype=torch.int64, + device=self.device, + out=past_indices[start_index:end_index], + ) + cumulative_length += input_length + + # Copy from present to past_key_values + batch.past_key_values[:, past_indices] = present + + # Initialize past_key_values in prefill for len(batch) == 1 + elif prefill: + # present is already pre-padded + batch.past_key_values = present # Cumulative length cumulative_length = 0 @@ -498,54 +547,134 @@ def generate_token( # Zipped iterator iterator = zip( - batch.requests, batch.input_lengths, - batch.offsets, - batch.token_offsets, batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, - batch.all_input_ids_tensor, ) + # We do two for loops as the first one can run completely asynchronously from the GPU while for the second + # one, we need to first do a GPU <-> CPU sync + # It is faster if we delay this sync for the maximum amount of time + # For each member of the batch for i, ( - request, input_length, - offset, - token_offset, next_token_chooser, stopping_criteria, all_input_ids, - all_input_ids_tensor, ) in enumerate(iterator): # Indexing metadata start_index = cumulative_length end_index = cumulative_length + input_length - prefill = stopping_criteria.current_tokens == 0 if prefill: # Prefill mode # out is of shape [cumulative_sequence_lengths, vocab_size] - logits = out[start_index:end_index] + # only take last token logit + logits = out[end_index - 1 : end_index] + + # Create all_input_ids_tensor that will be used by token warpers (for example, RepetitionPenalty) + all_input_ids_tensor = batch.input_ids.new_empty( + input_length + stopping_criteria.max_new_tokens + ) + # Copy from batch.input_ids to all_input_ids_tensor + all_input_ids_tensor[:input_length] = batch.input_ids[ + start_index:end_index + ] + batch.all_input_ids_tensor.append(all_input_ids_tensor) + + # Initialize position_ids + # In decode, we do not need this as we can just increment position ids + next_position_ids[i] = batch.position_ids[end_index - 1] + + # Used to gather prefill logprobs + # Copy batch.input_ids to prefill_token_indices + if len(batch) > 1: + prefill_tokens_indices[ + start_index : end_index - 1 + ] = batch.input_ids[start_index + 1 : end_index] + else: + # Set prefill_tokens_indices to the correct slice + prefill_tokens_indices = batch.input_ids[ + start_index + 1 : end_index + ] else: # Decode mode # out is of shape [batch_size, vocab_size] - logits = out[i].unsqueeze(0) + logits = out[i].view(1, -1) + + all_input_ids_tensor = batch.all_input_ids_tensor[i] # Select next token - next_token_id, logprobs = next_token_chooser( + next_token_id, logprob = next_token_chooser( all_input_ids_tensor[None, :input_length], logits ) - next_token_id_squeezed = next_token_id.squeeze() - next_token_id_item = next_token_id_squeezed.item() + + # Add to all_input_ids_tensor + next_token_id_squeezed = next_token_id.view(1) + all_input_ids_tensor[input_length] = next_token_id_squeezed + + # Set values + next_input_ids[i] = next_token_id_squeezed + next_token_logprobs[i] = logprob[-1, next_token_id].view(1) + + cumulative_length += input_length + + # Set values in batch + batch.input_ids = next_input_ids + batch.position_ids = next_position_ids + 1 + batch.cu_seqlens = batch.cu_seqlens + batch.cu_seqlens_q + + if prefill: + # Get prefill logprobs + prefill_logprobs_tensor = torch.log_softmax(out, -1) + prefill_logprobs = torch.gather( + prefill_logprobs_tensor, 1, prefill_tokens_indices.view(-1, 1) + ) + # GPU <-> CPU sync + prefill_logprobs = prefill_logprobs.view(-1).tolist() + + # GPU <-> CPU sync + next_token_logprobs = next_token_logprobs.tolist() + next_token_ids = batch.input_ids.tolist() + + cumulative_length = 0 + + # Zipped iterator + iterator = zip( + batch.requests, + batch.input_lengths, + batch.offsets, + batch.token_offsets, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + batch.all_input_ids_tensor, + next_token_ids, + next_token_logprobs, + ) + + # For each member of the batch + for i, ( + request, + input_length, + offset, + token_offset, + next_token_chooser, + stopping_criteria, + all_input_ids, + all_input_ids_tensor, + next_token_id, + next_token_logprob, + ) in enumerate(iterator): + start_index = cumulative_length + end_index = cumulative_length + input_length # Append next token to all tokens - all_input_ids.append(next_token_id_item) - all_input_ids_tensor[input_length] = next_token_id_item + all_input_ids.append(next_token_id) # Generated token - next_token_logprob = logprobs[-1, next_token_id_item] next_token_text, offset, token_offset = self.decode_token( all_input_ids, offset, @@ -554,7 +683,7 @@ def generate_token( # Evaluate stopping criteria stop, reason = stopping_criteria( - next_token_id_item, + next_token_id, next_token_text, ) @@ -579,9 +708,9 @@ def generate_token( # Prefill if prefill: # Remove generated token to only have prefill and add nan for first prompt token - prefill_logprobs = [float("nan")] + logprobs.gather( - 1, all_input_ids_tensor[1:input_length].unsqueeze(1) - ).squeeze(1)[:-1].tolist() + request_prefill_logprobs = [float("nan")] + prefill_logprobs[ + start_index : end_index - 1 + ] prefill_token_ids = all_input_ids[:-1] prefill_texts = self.tokenizer.batch_decode( prefill_token_ids, @@ -589,7 +718,7 @@ def generate_token( skip_special_tokens=False, ) prefill_tokens = PrefillTokens( - prefill_token_ids, prefill_logprobs, prefill_texts + prefill_token_ids, request_prefill_logprobs, prefill_texts ) else: prefill_tokens = None @@ -597,31 +726,23 @@ def generate_token( generation = Generation( request.id, prefill_tokens, - next_token_id_item, + next_token_id, next_token_logprob, next_token_text, - next_token_id_item in self.all_special_ids, + next_token_id in self.all_special_ids, generated_text, ) generations.append(generation) - cumulative_length += input_length new_input_length = input_length + 1 # Update values - batch.input_ids[i] = next_token_id - batch.position_ids[i] = input_length batch.input_lengths[i] = new_input_length batch.offsets[i] = offset batch.token_offsets[i] = token_offset batch.all_input_ids[i] = all_input_ids - batch.all_input_ids_tensor[i] = all_input_ids_tensor - batch.max_seqlen = max(batch.max_seqlen, new_input_length) - if len(batch) != 1: - # Add each sequence before its padding - batch.past_key_values[i * 2] = present[:, start_index:end_index] - # Cumulative sum - batch.cu_seqlens[(i + 1)] = batch.cu_seqlens[i] + new_input_length + batch.max_seqlen = batch.max_seqlen + 1 + cumulative_length += input_length # No need to return a batch if we know that all requests stopped return generations, batch if not stopped else None diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 105ff519..e4426771 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -32,7 +32,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False self.past_pad = None if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + dtype = torch.float16 else: raise NotImplementedError("FlashLlama is only available on GPU") @@ -161,7 +161,7 @@ def __init__( self.master = self.rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{self.rank}") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + dtype = torch.float16 else: raise NotImplementedError("FlashLlama is only available on GPU") diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index fc769583..f439e812 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -38,7 +38,7 @@ def __init__( self.master = self.rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{self.rank}") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + dtype = torch.float16 else: raise NotImplementedError("FlashNeoX is only available on GPU") diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 333180e8..f0825ab9 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -31,7 +31,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False self.past_pad = None if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + dtype = torch.float16 else: raise NotImplementedError("FlashSantacoder is only available on GPU") @@ -178,7 +178,7 @@ def __init__( self.master = self.rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{self.rank}") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + dtype = torch.float16 else: raise NotImplementedError("FlashSantacoderSharded is only available on GPU") From 49cffad1bc42e892f63be173384dce26e185e420 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 9 May 2023 19:02:52 +0200 Subject: [PATCH 222/793] fix(docker): fix nvidia env vars (#305) --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d68703cb..500f6580 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,7 +123,8 @@ ENV HUGGINGFACE_HUB_CACHE=/data \ PORT=80 # NVIDIA env vars -ENV NVIDIA_REQUIRE_CUDA cuda>=11.8 +ENV CUDA_VERSION=11.8.0 +ENV NVIDIA_REQUIRE_CUDA=cuda>=11.8 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=510,driver<511 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511 brand=nvidiartx,driver>=510,driver<511 brand=geforce,driver>=510,driver<511 brand=geforcertx,driver>=510,driver<511 brand=quadro,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=titan,driver>=510,driver<511 brand=titanrtx,driver>=510,driver<511 brand=tesla,driver>=515,driver<516 brand=unknown,driver>=515,driver<516 brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=geforce,driver>=515,driver<516 brand=geforcertx,driver>=515,driver<516 brand=quadro,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516 brand=titan,driver>=515,driver<516 brand=titanrtx,driver>=515,driver<516 ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility # Required for nvidia-docker v1 From 158540446410cb231380f72ccc8449562939c8b3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 10 May 2023 15:29:21 +0200 Subject: [PATCH 223/793] fix(docker): remove nvidia require cuda env (#310) --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 500f6580..c4494173 100644 --- a/Dockerfile +++ b/Dockerfile @@ -124,7 +124,6 @@ ENV HUGGINGFACE_HUB_CACHE=/data \ # NVIDIA env vars ENV CUDA_VERSION=11.8.0 -ENV NVIDIA_REQUIRE_CUDA=cuda>=11.8 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=510,driver<511 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511 brand=nvidiartx,driver>=510,driver<511 brand=geforce,driver>=510,driver<511 brand=geforcertx,driver>=510,driver<511 brand=quadro,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=titan,driver>=510,driver<511 brand=titanrtx,driver>=510,driver<511 brand=tesla,driver>=515,driver<516 brand=unknown,driver>=515,driver<516 brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=geforce,driver>=515,driver<516 brand=geforcertx,driver>=515,driver<516 brand=quadro,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516 brand=titan,driver>=515,driver<516 brand=titanrtx,driver>=515,driver<516 ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility # Required for nvidia-docker v1 From 68e9d6ab333715008c542467c8d5202cf4692253 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 10 May 2023 15:48:21 +0200 Subject: [PATCH 224/793] feat(server): shard token decode (#303) --- router/client/src/lib.rs | 2 + router/client/src/sharded_client.rs | 24 ++++- server/text_generation_server/models/bloom.py | 12 ++- .../models/causal_lm.py | 96 ++++++++++--------- .../custom_modeling/flash_llama_modeling.py | 2 - .../custom_modeling/flash_neox_modeling.py | 2 - .../models/flash_causal_lm.py | 90 +++++++++-------- .../models/flash_llama.py | 12 ++- .../models/flash_neox.py | 12 ++- .../models/flash_santacoder.py | 12 ++- .../models/galactica.py | 12 ++- .../text_generation_server/models/gpt_neox.py | 12 ++- server/text_generation_server/models/model.py | 4 + server/text_generation_server/models/opt.py | 12 ++- .../models/seq2seq_lm.py | 80 +++++++++------- server/text_generation_server/models/t5.py | 12 ++- server/text_generation_server/utils/tokens.py | 6 +- 17 files changed, 224 insertions(+), 178 deletions(-) diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 401082c5..66f2055a 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -23,6 +23,8 @@ pub enum ClientError { Connection(String), #[error("Server error: {0}")] Generation(String), + #[error("Sharded results are empty")] + EmptyResults, } impl From for ClientError { diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 73524740..60b81fe6 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,6 +1,6 @@ /// Multi shard Client -use crate::Result; use crate::{Batch, Client, Generation, HealthResponse, Request, ShardInfo}; +use crate::{ClientError, Result}; use futures::future::join_all; use tonic::transport::Uri; use tracing::instrument; @@ -98,8 +98,9 @@ impl ShardedClient { .iter_mut() .map(|client| Box::pin(client.prefill(batch.clone()))) .collect(); - // all shards return the same message - join_all(futures).await.pop().unwrap() + let results: Result, Option)>> = + join_all(futures).await.into_iter().collect(); + merge_generations(results?) } /// Generate one token for each request in the given cached batches @@ -116,7 +117,20 @@ impl ShardedClient { .iter_mut() .map(|client| Box::pin(client.decode(batches.clone()))) .collect(); - // all shards return the same message - join_all(futures).await.pop().unwrap() + let results: Result, Option)>> = + join_all(futures).await.into_iter().collect(); + merge_generations(results?) + } +} + +/// Merge generations from the different model shards +fn merge_generations( + mut results: Vec<(Vec, Option)>, +) -> Result<(Vec, Option)> { + let (mut generations, next_batch) = results.pop().ok_or(ClientError::EmptyResults)?; + + for (mut shard_generations, _) in results.into_iter() { + generations.append(&mut shard_generations); } + Ok((generations, next_batch)) } diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index f528a430..25ec8cb8 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -63,10 +63,10 @@ class BLOOMSharded(BLOOM): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - self.process_group, self.rank, self.world_size = initialize_torch_distributed() - self.master = self.rank == 0 + self.process_group, rank, world_size = initialize_torch_distributed() + self.master = rank == 0 if torch.cuda.is_available(): - device = torch.device(f"cuda:{self.rank}") + device = torch.device(f"cuda:{rank}") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") @@ -94,8 +94,8 @@ def __init__( quantize=quantize, device=device, dtype=dtype, - rank=self.rank, - world_size=self.world_size, + rank=rank, + world_size=world_size, ) self.model = model.eval() torch.distributed.barrier(group=self.process_group) @@ -105,6 +105,8 @@ def __init__( dtype=dtype, device=device, decode_buffer=1, + rank=rank, + world_size=world_size, ) @staticmethod diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 26a9a661..610dc4e2 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -549,7 +549,7 @@ def generate_token( ) in enumerate(iterator): # Select next token next_token_id, logprobs = next_token_chooser( - all_input_ids.view(1, -1), logits + all_input_ids.view(1, -1), logits[-1:, :] ) # Append next token to all tokens @@ -569,54 +569,60 @@ def generate_token( next_token_text, ) - if stop: - # Decode generated tokens - output_text = self.decode( - all_input_ids[-stopping_criteria.current_tokens :, 0] - ) - # Get seed - if isinstance(next_token_chooser.choice, Sampling): - seed = next_token_chooser.choice.seed - else: - seed = None - - generated_text = GeneratedText( - output_text, stopping_criteria.current_tokens, reason, seed - ) - else: - # Keep request in the batch - generated_text = None + if not stop: stopped = False - # Prefill - if stopping_criteria.current_tokens == 1: - # Remove generated token to only have prefill and add nan for first prompt token - prefill_logprobs = [float("nan")] + logprobs.gather( - 1, all_input_ids[1:] - ).squeeze(1)[-new_input_length:-1].tolist() - prefill_token_ids = all_input_ids[-new_input_length:-1] - prefill_texts = self.tokenizer.batch_decode( - prefill_token_ids, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) - prefill_tokens = PrefillTokens( - prefill_token_ids, prefill_logprobs, prefill_texts + # Shard generations + # All generations will be appended in the rust sharded client + if i % self.world_size == self.rank: + if stop: + # Decode generated tokens + output_text = self.decode( + all_input_ids[-stopping_criteria.current_tokens :, 0] + ) + # Get seed + if isinstance(next_token_chooser.choice, Sampling): + seed = next_token_chooser.choice.seed + else: + seed = None + + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed + ) + else: + generated_text = None + + # Prefill + if stopping_criteria.current_tokens == 1: + # Remove generated token to only have prefill and add nan for first prompt token + prefill_logprobs = [float("nan")] + torch.log_softmax( + logits, -1 + ).gather(1, all_input_ids[1:]).squeeze(1)[ + -new_input_length:-1 + ].tolist() + prefill_token_ids = all_input_ids[-new_input_length:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = PrefillTokens( + prefill_token_ids, prefill_logprobs, prefill_texts + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id_squeezed, + next_token_logprob, + next_token_text, + next_token_id_squeezed.item() in self.all_special_ids, + generated_text, ) - else: - prefill_tokens = None - - generation = Generation( - request.id, - prefill_tokens, - next_token_id_squeezed, - next_token_logprob, - next_token_text, - next_token_id_squeezed.item() in self.all_special_ids, - generated_text, - ) - generations.append(generation) + generations.append(generation) # Update values batch.input_ids[i, 0] = next_token_id diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 1293124a..6ae869db 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -622,10 +622,8 @@ def __init__(self, config, process_group=None): self.process_group = process_group if self.process_group is not None: self.world_size = self.process_group.size() - self.rank = self.process_group.rank() else: self.world_size = 1 - self.rank = 0 self.model = FlashLlamaModel(config, process_group) diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index ae1465ab..d706df33 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -685,10 +685,8 @@ def __init__(self, config, process_group=None): self.process_group = process_group if self.process_group is not None: self.world_size = self.process_group.size() - self.rank = self.process_group.rank() else: self.world_size = 1 - self.rank = 0 self.gpt_neox = FlashGPTNeoXModel(config, process_group) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index b51a3dc6..e862cfeb 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -687,53 +687,59 @@ def generate_token( next_token_text, ) - if stop: - # Decode generated tokens - output_text = self.decode( - all_input_ids[-stopping_criteria.current_tokens :] - ) - # Get seed - if isinstance(next_token_chooser.choice, Sampling): - seed = next_token_chooser.choice.seed - else: - seed = None - - generated_text = GeneratedText( - output_text, stopping_criteria.current_tokens, reason, seed - ) - else: + if not stop: stopped = False - generated_text = None - # Prefill - if prefill: - # Remove generated token to only have prefill and add nan for first prompt token - request_prefill_logprobs = [float("nan")] + prefill_logprobs[ - start_index : end_index - 1 - ] - prefill_token_ids = all_input_ids[:-1] - prefill_texts = self.tokenizer.batch_decode( - prefill_token_ids, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) - prefill_tokens = PrefillTokens( - prefill_token_ids, request_prefill_logprobs, prefill_texts + # Shard generations + # All generations will be appended in the rust sharded client + if i % self.world_size == self.rank: + if stop: + # Decode generated tokens + output_text = self.decode( + all_input_ids[-stopping_criteria.current_tokens :] + ) + # Get seed + if isinstance(next_token_chooser.choice, Sampling): + seed = next_token_chooser.choice.seed + else: + seed = None + + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed + ) + else: + generated_text = None + + # Prefill + if prefill: + # Remove generated token to only have prefill and add nan for first prompt token + request_prefill_logprobs = [float("nan")] + prefill_logprobs[ + start_index : end_index - 1 + ] + prefill_token_ids = all_input_ids[:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = PrefillTokens( + prefill_token_ids, request_prefill_logprobs, prefill_texts + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id, + next_token_logprob, + next_token_text, + next_token_id in self.all_special_ids, + generated_text, ) - else: - prefill_tokens = None - generation = Generation( - request.id, - prefill_tokens, - next_token_id, - next_token_logprob, - next_token_text, - next_token_id in self.all_special_ids, - generated_text, - ) + generations.append(generation) - generations.append(generation) new_input_length = input_length + 1 # Update values diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index e4426771..a3ba2084 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -157,10 +157,10 @@ def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): self.past_pad = None - self.process_group, self.rank, self.world_size = initialize_torch_distributed() - self.master = self.rank == 0 + self.process_group, rank, world_size = initialize_torch_distributed() + self.master = rank == 0 if torch.cuda.is_available(): - device = torch.device(f"cuda:{self.rank}") + device = torch.device(f"cuda:{rank}") dtype = torch.float16 else: raise NotImplementedError("FlashLlama is only available on GPU") @@ -190,8 +190,8 @@ def __init__( quantize=quantize, device=device, dtype=dtype, - rank=self.rank, - world_size=self.world_size, + rank=rank, + world_size=world_size, ) self.model = model.eval().to(device) torch.distributed.barrier(group=self.process_group) @@ -200,6 +200,8 @@ def __init__( requires_padding=False, dtype=dtype, device=device, + rank=rank, + world_size=world_size, ) @staticmethod diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index f439e812..b3e1876f 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -34,10 +34,10 @@ def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): self.past_pad = None - self.process_group, self.rank, self.world_size = initialize_torch_distributed() - self.master = self.rank == 0 + self.process_group, rank, world_size = initialize_torch_distributed() + self.master = rank == 0 if torch.cuda.is_available(): - device = torch.device(f"cuda:{self.rank}") + device = torch.device(f"cuda:{rank}") dtype = torch.float16 else: raise NotImplementedError("FlashNeoX is only available on GPU") @@ -64,8 +64,8 @@ def __init__( quantize=quantize, device=device, dtype=dtype, - rank=self.rank, - world_size=self.world_size, + rank=rank, + world_size=world_size, ) self.model = model.eval().to(device) torch.distributed.barrier(group=self.process_group) @@ -74,6 +74,8 @@ def __init__( requires_padding=False, dtype=dtype, device=device, + rank=rank, + world_size=world_size, ) @staticmethod diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index f0825ab9..6e5c231e 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -174,10 +174,10 @@ def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): self.past_pad = None - self.process_group, self.rank, self.world_size = initialize_torch_distributed() - self.master = self.rank == 0 + self.process_group, rank, world_size = initialize_torch_distributed() + self.master = rank == 0 if torch.cuda.is_available(): - device = torch.device(f"cuda:{self.rank}") + device = torch.device(f"cuda:{rank}") dtype = torch.float16 else: raise NotImplementedError("FlashSantacoderSharded is only available on GPU") @@ -204,8 +204,8 @@ def __init__( quantize=quantize, device=device, dtype=dtype, - rank=self.rank, - world_size=self.world_size, + rank=rank, + world_size=world_size, transpose=config.architectures[0].startswith("GPT2"), ) self.model = model.eval().to(device) @@ -215,6 +215,8 @@ def __init__( requires_padding=False, dtype=dtype, device=device, + rank=rank, + world_size=world_size, ) @staticmethod diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 2577f1b1..57df0bab 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -195,10 +195,10 @@ class GalacticaSharded(Galactica): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - self.process_group, self.rank, self.world_size = initialize_torch_distributed() - self.master = self.rank == 0 + self.process_group, rank, world_size = initialize_torch_distributed() + self.master = rank == 0 if torch.cuda.is_available(): - device = torch.device(f"cuda:{self.rank}") + device = torch.device(f"cuda:{rank}") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") @@ -226,8 +226,8 @@ def __init__( quantize=quantize, device=device, dtype=dtype, - rank=self.rank, - world_size=self.world_size, + rank=rank, + world_size=world_size, ) self.model = model.eval() torch.distributed.barrier(group=self.process_group) @@ -236,6 +236,8 @@ def __init__( requires_padding=True, dtype=dtype, device=device, + rank=rank, + world_size=world_size, ) @staticmethod diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index e73a3c82..c71bf366 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -34,10 +34,10 @@ class GPTNeoxSharded(CausalLM): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - self.process_group, self.rank, self.world_size = initialize_torch_distributed() - self.master = self.rank == 0 + self.process_group, rank, world_size = initialize_torch_distributed() + self.master = rank == 0 if torch.cuda.is_available(): - device = torch.device(f"cuda:{self.rank}") + device = torch.device(f"cuda:{rank}") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") @@ -65,8 +65,8 @@ def __init__( quantize=quantize, device=device, dtype=dtype, - rank=self.rank, - world_size=self.world_size, + rank=rank, + world_size=world_size, ) self.model = model.eval() torch.distributed.barrier(group=self.process_group) @@ -75,6 +75,8 @@ def __init__( requires_padding=True, dtype=dtype, device=device, + rank=rank, + world_size=world_size, ) @staticmethod diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index bfae829d..4c85c952 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -18,6 +18,8 @@ def __init__( dtype: torch.dtype, device: torch.device, decode_buffer: int = 3, + rank: int = 0, + world_size: int = 1, ): if decode_buffer < 1: raise ValueError("decode_buffer must be >= 1") @@ -28,6 +30,8 @@ def __init__( self.dtype = dtype self.device = device self.decode_buffer = decode_buffer + self.rank = rank + self.world_size = world_size @property def info(self) -> InfoResponse: diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 50e5271e..cdc32c56 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -50,10 +50,10 @@ class OPTSharded(OPT): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - self.process_group, self.rank, self.world_size = initialize_torch_distributed() - self.master = self.rank == 0 + self.process_group, rank, world_size = initialize_torch_distributed() + self.master = rank == 0 if torch.cuda.is_available(): - device = torch.device(f"cuda:{self.rank}") + device = torch.device(f"cuda:{rank}") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") @@ -81,8 +81,8 @@ def __init__( quantize=quantize, device=device, dtype=dtype, - rank=self.rank, - world_size=self.world_size, + rank=rank, + world_size=world_size, ) self.model = model.eval() torch.distributed.barrier(group=self.process_group) @@ -91,6 +91,8 @@ def __init__( requires_padding=True, dtype=dtype, device=device, + rank=rank, + world_size=world_size, ) @staticmethod diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 4ac5ed3c..d4a0ddcc 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -631,7 +631,7 @@ def generate_token( ) in enumerate(iterator): # Select next token next_token_id, logprobs = next_token_chooser( - all_decoder_input_ids.view(1, -1), logits + all_decoder_input_ids.view(1, -1), logits[-1:, :] ) # Append next token to decoder tokens @@ -650,46 +650,52 @@ def generate_token( # Evaluate stopping criteria stop, reason = stopping_criteria(next_token_id, next_token_text) - if stop: - # Slice with decoder_input_length to remove padding - # Decode all tokens - output_text = self.decode(all_decoder_input_ids[-decoder_input_length:]) - - # Get seed - if isinstance(next_token_chooser.choice, Sampling): - seed = next_token_chooser.choice.seed - else: - seed = None - - generated_text = GeneratedText( - output_text, stopping_criteria.current_tokens, reason, seed - ) - else: - # Keep request in the batch - generated_text = None + if not stop: stopped = False - # Prefill - if stopping_criteria.current_tokens == 1: - prefill_tokens = PrefillTokens( - [self.tokenizer.bos_token_id], - [float("nan")], - [self.tokenizer.bos_token], + # Shard generations + # All generations will be appended in the rust sharded client + if i % self.world_size == self.rank: + if stop: + # Slice with decoder_input_length to remove padding + # Decode all tokens + output_text = self.decode( + all_decoder_input_ids[-decoder_input_length:] + ) + + # Get seed + if isinstance(next_token_chooser.choice, Sampling): + seed = next_token_chooser.choice.seed + else: + seed = None + + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed + ) + else: + generated_text = None + + # Prefill + if stopping_criteria.current_tokens == 1: + prefill_tokens = PrefillTokens( + [self.tokenizer.bos_token_id], + [float("nan")], + [self.tokenizer.bos_token], + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id_squeezed, + next_token_logprob, + next_token_text, + next_token_id_squeezed.item() in self.all_special_ids, + generated_text, ) - else: - prefill_tokens = None - - generation = Generation( - request.id, - prefill_tokens, - next_token_id_squeezed, - next_token_logprob, - next_token_text, - next_token_id_squeezed.item() in self.all_special_ids, - generated_text, - ) - generations.append(generation) + generations.append(generation) # Update values batch.decoder_input_ids[i] = next_token_id diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 9e8c3c4c..5691c005 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -34,10 +34,10 @@ class T5Sharded(Seq2SeqLM): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - self.process_group, self.rank, self.world_size = initialize_torch_distributed() - self.master = self.rank == 0 + self.process_group, rank, world_size = initialize_torch_distributed() + self.master = rank == 0 if torch.cuda.is_available(): - device = torch.device(f"cuda:{self.rank}") + device = torch.device(f"cuda:{rank}") dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") @@ -65,8 +65,8 @@ def __init__( quantize=quantize, device=device, dtype=dtype, - rank=self.rank, - world_size=self.world_size, + rank=rank, + world_size=world_size, ) self.model = model.eval() torch.distributed.barrier(group=self.process_group) @@ -75,6 +75,8 @@ def __init__( requires_padding=True, dtype=dtype, device=device, + rank=rank, + world_size=world_size, ) @staticmethod diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 23f504c6..d5a77170 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -75,11 +75,7 @@ def __init__( def __call__(self, input_ids, scores): # Warp logits - if scores.shape[0] > 1: - # only warp the last token logits - scores[-1:, :] = self.warpers(input_ids, scores[-1:, :]) - else: - scores = self.warpers(input_ids, scores) + scores = self.warpers(input_ids, scores) # Compute logprobs logprobs = torch.log_softmax(scores, -1) From 745f596c880fdeb4766ca36e29bbf8a7e562e661 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 10 May 2023 15:51:10 +0200 Subject: [PATCH 225/793] feat(server): use float16 (#304) --- server/text_generation_server/models/bloom.py | 2 +- server/text_generation_server/models/causal_lm.py | 2 +- server/text_generation_server/models/galactica.py | 2 +- server/text_generation_server/models/gpt_neox.py | 2 +- server/text_generation_server/models/opt.py | 2 +- server/text_generation_server/models/santacoder.py | 2 +- server/text_generation_server/models/seq2seq_lm.py | 2 +- server/text_generation_server/models/t5.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 25ec8cb8..7c50644a 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -67,7 +67,7 @@ def __init__( self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + dtype = torch.float16 else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 610dc4e2..f838fc5c 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -452,7 +452,7 @@ def __init__( ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + dtype = torch.float16 else: if quantize: raise ValueError("quantization is not available on CPU") diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 57df0bab..f6098e6c 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -199,7 +199,7 @@ def __init__( self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + dtype = torch.float16 else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index c71bf366..cd36dba0 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -38,7 +38,7 @@ def __init__( self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + dtype = torch.float16 else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index cdc32c56..44f15df3 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -54,7 +54,7 @@ def __init__( self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + dtype = torch.float16 else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index a7b09a82..5a142676 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -17,7 +17,7 @@ class SantaCoder(CausalLM): def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + dtype = torch.float16 else: if quantize: raise ValueError("quantization is not available on CPU") diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index d4a0ddcc..77912cff 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -506,7 +506,7 @@ def __init__( ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + dtype = torch.float16 else: if quantize: raise ValueError("quantization is not available on CPU") diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 5691c005..4cfdea9e 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -38,7 +38,7 @@ def __init__( self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + dtype = torch.float16 else: device = torch.device("cpu") dtype = torch.float32 From 35ab6cfcf138c8d47e825f801a010b648c7502da Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 10 May 2023 16:16:06 +0200 Subject: [PATCH 226/793] fix(docker): remove CUDA_VERSION --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c4494173..50145395 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,7 +123,6 @@ ENV HUGGINGFACE_HUB_CACHE=/data \ PORT=80 # NVIDIA env vars -ENV CUDA_VERSION=11.8.0 ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility # Required for nvidia-docker v1 From a6c18c39bb9431e6d3aba407f1d30eb43d82c04d Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 10 May 2023 19:08:54 +0200 Subject: [PATCH 227/793] feat(server): use cuda graph in logits warping (#302) --- server/text_generation_server/utils/tokens.py | 127 +++++++++++++----- 1 file changed, 94 insertions(+), 33 deletions(-) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index d5a77170..c3477eef 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -1,8 +1,8 @@ import re import torch +from functools import lru_cache from transformers import ( - LogitsProcessorList, TemperatureLogitsWarper, TopKLogitsWarper, TopPLogitsWarper, @@ -25,8 +25,10 @@ def __init__(self, seed: int, device: str = "cpu"): def __call__(self, logits): probs = torch.nn.functional.softmax(logits, -1) - next_tokens = torch.multinomial(probs, num_samples=1, generator=self.generator) - return next_tokens + # See: https://github.com/pytorch/pytorch/blob/925a3788ec5c06db62ca732a0e9425a26a00916f/aten/src/ATen/native/Distributions.cpp#L631-L637 + q = torch.empty_like(probs).exponential_(1, generator=self.generator).div_(probs) + + return q.argmax() class Greedy: @@ -34,6 +36,63 @@ def __call__(self, logits): return logits.argmax() +class StaticWarper: + def __init__( + self, + temperature=1.0, + top_k=None, + top_p=None, + typical_p=None, + ): + self.warpers = [] + + if temperature is not None and temperature != 1.0: + temperature = float(temperature) + self.warpers.append(TemperatureLogitsWarper(temperature)) + if top_k is not None and top_k != 0: + self.warpers.append(TopKLogitsWarper(top_k=top_k)) + if top_p is not None and top_p < 1.0: + self.warpers.append(TopPLogitsWarper(top_p=top_p)) + if typical_p is not None and typical_p < 1.0: + self.warpers.append(TypicalLogitsWarper(mass=typical_p)) + + self.cuda_graph = None + self.static_scores = None + self.static_warped_scores = None + self.static_next_logprob = None + + def __call__(self, scores): + if self.cuda_graph is None: + self.static_scores = scores + self.cuda_graph = torch.cuda.CUDAGraph() + + with torch.cuda.graph(self.cuda_graph): + for warper in self.warpers: + self.static_warped_scores = warper(None, self.static_scores) + + # Compute logprobs + self.static_next_logprob = torch.log_softmax( + self.static_warped_scores, -1 + ) + + self.static_scores.copy_(scores) + self.cuda_graph.replay() + + return self.static_warped_scores, self.static_next_logprob + + +@lru_cache(10) +def static_warper( + temperature: Optional[float], + top_k: Optional[int], + top_p: Optional[float], + typical_p: Optional[float], +) -> StaticWarper: + return StaticWarper( + temperature=temperature, top_k=top_k, top_p=top_p, typical_p=typical_p + ) + + class NextTokenChooser: def __init__( self, @@ -47,43 +106,45 @@ def __init__( seed=0, device="cpu", ): - warpers = LogitsProcessorList() - # the following idea is largely copied from this PR: https://github.com/huggingface/transformers/pull/5420/files - # all samplers can be found in `generation_utils_samplers.py` - sampling = do_sample - - if watermark: - warpers.append(WatermarkLogitsProcessor(device=device)) - if repetition_penalty is not None and repetition_penalty != 1.0: - warpers.append(RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty)) - if temperature is not None and temperature != 1.0: - temperature = float(temperature) - warpers.append(TemperatureLogitsWarper(temperature)) - sampling = True - if top_k is not None and top_k != 0: - warpers.append(TopKLogitsWarper(top_k=top_k)) - sampling = True - if top_p is not None and top_p < 1.0: - warpers.append(TopPLogitsWarper(top_p=top_p)) - sampling = True - if typical_p is not None and typical_p < 1.0: - warpers.append(TypicalLogitsWarper(mass=typical_p)) - sampling = True + self.watermark_processor = ( + WatermarkLogitsProcessor(device=device) if watermark else None + ) + self.repetition_processor = ( + RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty) + if repetition_penalty + else None + ) - self.warpers = warpers + has_warpers = ( + (temperature is not None and temperature != 1.0) + or (top_k is not None and top_k != 0) + or (top_p is not None and top_p < 1.0) + or (typical_p is not None and typical_p < 1.0) + ) + if has_warpers: + self.static_warper = static_warper( + temperature=temperature, top_k=top_k, top_p=top_p, typical_p=typical_p + ) + else: + self.static_warper = None + + sampling = do_sample or has_warpers self.choice = Sampling(seed, device) if sampling else Greedy() def __call__(self, input_ids, scores): - # Warp logits - scores = self.warpers(input_ids, scores) + if self.watermark_processor: + scores = self.watermark_processor(input_ids, scores) + if self.repetition_processor: + scores = self.repetition_processor(input_ids, scores) - # Compute logprobs - logprobs = torch.log_softmax(scores, -1) + if self.static_warper is None: + next_logprob = torch.log_softmax(scores, -1) + else: + scores, next_logprob = self.static_warper(scores) - # Choose tokens - next_id = self.choice(scores[-1]) + next_id = self.choice(scores[-1]).view(1, 1) - return next_id.view(1, 1), logprobs + return next_id, next_logprob @classmethod def from_pb( From 4f6d038c0b92ef84e3b02940a0277d1b6ab47c10 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 11 May 2023 13:30:38 +0200 Subject: [PATCH 228/793] fix(server): fix multinomial implem in Sampling --- server/text_generation_server/utils/tokens.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index c3477eef..e9fb96b0 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -25,10 +25,10 @@ def __init__(self, seed: int, device: str = "cpu"): def __call__(self, logits): probs = torch.nn.functional.softmax(logits, -1) + # Avoid GPU<->CPU sync done by torch multinomial # See: https://github.com/pytorch/pytorch/blob/925a3788ec5c06db62ca732a0e9425a26a00916f/aten/src/ATen/native/Distributions.cpp#L631-L637 - q = torch.empty_like(probs).exponential_(1, generator=self.generator).div_(probs) - - return q.argmax() + q = torch.empty_like(probs).exponential_(1, generator=self.generator) + return probs.div_(q).argmax() class Greedy: From 76a48cd365e44a467ff3f5a5ee6db1a48a60e267 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 12 May 2023 14:46:41 +0200 Subject: [PATCH 229/793] feat(server): GPTQ quantization (step1) (#277) Changes only the type from `bool` to `Option` pretty much everywhere. - Use `Optional[str]` in Python (easier to manage than importing type everywhere). Except for the cli to get proper validation - Updated all models to handle gracefully new values. (Error out if unknown value, or gptq since not implemented). Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- launcher/src/main.rs | 37 +++++++++++++++---- server/text_generation_server/cli.py | 10 ++++- .../text_generation_server/models/__init__.py | 2 +- server/text_generation_server/models/bloom.py | 25 ++++++++++--- .../models/causal_lm.py | 4 +- .../custom_modeling/flash_llama_modeling.py | 8 +++- .../custom_modeling/flash_neox_modeling.py | 10 +++-- .../flash_santacoder_modeling.py | 10 +++-- .../models/flash_causal_lm.py | 4 +- .../models/flash_llama.py | 5 ++- .../models/galactica.py | 18 ++++++--- .../text_generation_server/models/gpt_neox.py | 18 ++++++--- .../models/santacoder.py | 9 ++++- .../models/seq2seq_lm.py | 4 +- server/text_generation_server/models/t5.py | 17 +++++++-- server/text_generation_server/server.py | 4 +- 16 files changed, 136 insertions(+), 49 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 867bfd3d..4a6cd621 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -1,4 +1,4 @@ -use clap::Parser; +use clap::{Parser, ValueEnum}; use serde::Deserialize; use std::env; use std::ffi::OsString; @@ -16,6 +16,26 @@ use subprocess::{ExitStatus, Popen, PopenConfig, PopenError, Redirection}; mod env_runtime; +#[derive(Clone, Copy, Debug, ValueEnum)] +enum Quantization { + Bitsandbytes, + Gptq, +} + +impl std::fmt::Display for Quantization { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // To keep in track with `server`. + match self { + Quantization::Bitsandbytes => { + write!(f, "bitsandbytes") + } + Quantization::Gptq => { + write!(f, "gptq") + } + } + } +} + /// App Configuration #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] @@ -46,10 +66,10 @@ struct Args { #[clap(long, env)] num_shard: Option, - /// Wether you want the model to be quantized or not. This will use bitsandbytes for - /// quantization on the fly. - #[clap(long, env)] - quantize: bool, + /// Wether you want the model to be quantized or not. This will use `bitsandbytes` for + /// quantization on the fly, or `gptq`. + #[clap(long, env, value_enum)] + quantize: Option, /// The maximum amount of concurrent requests for this particular deployment. /// Having a low limit will refuse clients requests instead of having them @@ -218,7 +238,7 @@ enum ShardStatus { fn shard_manager( model_id: String, revision: Option, - quantize: bool, + quantize: Option, uds_path: String, rank: usize, world_size: usize, @@ -257,8 +277,9 @@ fn shard_manager( shard_argv.push("--sharded".to_string()); } - if quantize { - shard_argv.push("--quantize".to_string()) + if let Some(quantize) = quantize { + shard_argv.push("--quantize".to_string()); + shard_argv.push(quantize.to_string()) } // Model optional revision diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 92482a94..35cad20f 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -5,17 +5,23 @@ from pathlib import Path from loguru import logger from typing import Optional +from enum import Enum app = typer.Typer() +class Quantization(str, Enum): + bitsandbytes = "bitsandbytes" + gptq = "gptq" + + @app.command() def serve( model_id: str, revision: Optional[str] = None, sharded: bool = False, - quantize: bool = False, + quantize: Optional[Quantization] = None, uds_path: Path = "/tmp/text-generation-server", logger_level: str = "INFO", json_output: bool = False, @@ -55,6 +61,8 @@ def serve( if otlp_endpoint is not None: setup_tracing(shard=os.getenv("RANK", 0), otlp_endpoint=otlp_endpoint) + # Downgrade enum into str for easier management later on + quantize = None if quantize is None else quantize.value server.serve(model_id, revision, sharded, quantize, uds_path) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 221c9139..e02be3de 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -91,7 +91,7 @@ def get_model( - model_id: str, revision: Optional[str], sharded: bool, quantize: bool + model_id: str, revision: Optional[str], sharded: bool, quantize: Optional[str] ) -> Model: if "facebook/galactica" in model_id: if sharded: diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 7c50644a..877acb00 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -49,7 +49,12 @@ def from_pb( class BLOOM(CausalLM): - def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + ): super(BLOOM, self).__init__( model_id=model_id, revision=revision, quantize=quantize, decode_buffer=1 ) @@ -61,7 +66,10 @@ def batch_type(self) -> Type[CausalLMBatch]: class BLOOMSharded(BLOOM): def __init__( - self, model_id: str, revision: Optional[str] = None, quantize: bool = False + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() self.master = rank == 0 @@ -113,7 +121,7 @@ def __init__( def load_weights( model, filenames: List[str], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, rank: int, @@ -167,7 +175,7 @@ def load_weights( tensor = tensor.contiguous().to(dtype) - if quantize: + if quantize == "bitsandbytes": if not HAS_BITS_AND_BYTES: raise ImportError( "bitsandbytes is not available on your machine either because it is not installed " @@ -217,9 +225,14 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) - - else: + elif quantize == "gptq": + raise NotImplementedError( + "`gptq` is not implemented for now" + ) + elif quantize is None: tensor = tensor.to(device) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") module._parameters[param_name] = tensor if name == "word_embeddings.weight": diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index f838fc5c..0d521ac4 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -447,7 +447,7 @@ def __init__( self, model_id: str, revision: Optional[str] = None, - quantize: bool = False, + quantize: Optional[str] = None, decode_buffer: int = 3, ): if torch.cuda.is_available(): @@ -468,7 +468,7 @@ def __init__( revision=revision, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, - load_in_8bit=quantize, + load_in_8bit=quantize == "bitsandbytes", ).eval() tokenizer.pad_token_id = ( self.model.config.pad_token_id diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 6ae869db..5b32b22e 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -105,7 +105,7 @@ def __init__( self.bnb_linear = None def prepare_weights(self, quantize: bool = False): - if quantize: + if quantize == "bitsandbytes": if not HAS_BITS_AND_BYTES: raise ImportError( "bitsandbytes is not available on your machine either because it is not installed " @@ -129,8 +129,12 @@ def prepare_weights(self, quantize: bool = False): # Delete reference to data self.weight = None self.bias = None - else: + elif quantize == "gptq": + raise NotImplementedError("`gptq` is not implemented for now") + elif quantize is None: self.weight = nn.Parameter(self.weight.T) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") def forward(self, input: torch.Tensor) -> torch.Tensor: if self.quantized: diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index d706df33..e7b878c0 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -92,8 +92,8 @@ def __init__( self.quantized = False self.bnb_linear = None - def prepare_weights(self, quantize: bool = False): - if quantize: + def prepare_weights(self, quantize: Optional[str] = None): + if quantize == "bitsandbytes": if not HAS_BITS_AND_BYTES: raise ImportError( "bitsandbytes is not available on your machine either because it is not installed " @@ -117,8 +117,12 @@ def prepare_weights(self, quantize: bool = False): # Delete reference to data self.weight = None self.bias = None - else: + elif quantize == "gptq": + raise NotImplementedError("`gptq` is not implemented for now") + elif quantize is None: self.weight = nn.Parameter(self.weight.T) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") def forward(self, input: torch.Tensor) -> torch.Tensor: if self.quantized: diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 20ad8385..309ec19f 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -67,8 +67,8 @@ def __init__( self.quantized = False self.bnb_linear = None - def prepare_weights(self, quantize: bool = False): - if quantize: + def prepare_weights(self, quantize: Optional[str] = None): + if quantize == "bitsandbytes": if not HAS_BITS_AND_BYTES: raise ImportError( "bitsandbytes is not available on your machine either because it is not installed " @@ -92,8 +92,12 @@ def prepare_weights(self, quantize: bool = False): # Delete reference to data self.weight = None self.bias = None - else: + elif quantize == "gptq": + raise NotImplementedError("`gptq` is not implemented for now") + elif quantize is None: self.weight = nn.Parameter(self.weight.T) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") def forward(self, input: torch.Tensor) -> torch.Tensor: if self.quantized: diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index e862cfeb..0a9fccca 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -393,7 +393,7 @@ def __init__( model_cls: Type[PreTrainedModel], model_id: str, revision: Optional[str] = None, - quantize: bool = False, + quantize: Optional[str] = None, decode_buffer: int = 3, ): if torch.cuda.is_available(): @@ -410,7 +410,7 @@ def __init__( model_id, revision=revision, torch_dtype=dtype, - load_in_8bit=quantize, + load_in_8bit=quantize == "bitsandbytes", ) .eval() .to(device) diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index a3ba2084..5f47cf66 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -154,7 +154,10 @@ def load_weights( class FlashLlamaSharded(FlashLlama): def __init__( - self, model_id: str, revision: Optional[str] = None, quantize: bool = False + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, ): self.past_pad = None self.process_group, rank, world_size = initialize_torch_distributed() diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index f6098e6c..4f94b348 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -193,7 +193,10 @@ def forward( class GalacticaSharded(Galactica): def __init__( - self, model_id: str, revision: Optional[str] = None, quantize: bool = False + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() self.master = rank == 0 @@ -244,7 +247,7 @@ def __init__( def load_weights( model, filenames: List[str], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, rank: int, @@ -299,7 +302,7 @@ def load_weights( tensor = tensor.contiguous().to(dtype) - if quantize: + if quantize == "bitsandbytes": if not HAS_BITS_AND_BYTES: raise ImportError( "bitsandbytes is not available on your machine either because it is not installed " @@ -349,9 +352,14 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) - - else: + elif quantize == "gptq": + raise NotImplementedError( + "`gptq` is not implemented for now" + ) + elif quantize is None: tensor = tensor.to(device) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") module._parameters[param_name] = tensor if name == "model.decoder.embed_tokens.weight": diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index cd36dba0..2d42e0b0 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -32,7 +32,10 @@ class GPTNeoxSharded(CausalLM): def __init__( - self, model_id: str, revision: Optional[str] = None, quantize: bool = False + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() self.master = rank == 0 @@ -83,7 +86,7 @@ def __init__( def load_weights( model, filenames: List[str], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, rank: int, @@ -148,7 +151,7 @@ def load_weights( tensor = tensor.contiguous().to(dtype) - if quantize: + if quantize == "bitsandbytes": if not HAS_BITS_AND_BYTES: raise ImportError( "bitsandbytes is not available on your machine either because it is not installed " @@ -198,9 +201,14 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) - - else: + elif quantize == "gptq": + raise NotImplementedError( + "`gptq` is not implemented for now" + ) + elif quantize is None: tensor = tensor.to(device) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") if current_parameter_tensor is not None: module._parameters[param_name] = tensor diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index 5a142676..4bd56de1 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -14,7 +14,12 @@ class SantaCoder(CausalLM): - def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + ): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.float16 @@ -46,7 +51,7 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False model_id, revision=revision, torch_dtype=dtype, - load_in_8bit=quantize, + load_in_8bit=quantize == "bitsandbytes", trust_remote_code=True, # required ) .to(device) diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 77912cff..84854f5d 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -501,7 +501,7 @@ def __init__( self, model_id: str, revision: Optional[str] = None, - quantize: bool = False, + quantize: Optional[str] = None, decode_buffer: int = 3, ): if torch.cuda.is_available(): @@ -519,7 +519,7 @@ def __init__( revision=revision, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, - load_in_8bit=quantize, + load_in_8bit=quantize == "bitsandbytes", ).eval() tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", truncation_side="left" diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 4cfdea9e..381617b7 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -32,7 +32,10 @@ class T5Sharded(Seq2SeqLM): def __init__( - self, model_id: str, revision: Optional[str] = None, quantize: bool = False + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() self.master = rank == 0 @@ -83,7 +86,7 @@ def __init__( def load_weights( model, filenames: List[str], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, rank: int, @@ -154,7 +157,7 @@ def load_weights( tensor = tensor.contiguous().to(dtype) - if quantize: + if quantize == "bitsandbytes": if not HAS_BITS_AND_BYTES: raise ImportError( "bitsandbytes is not available on your machine either because it is not installed " @@ -205,8 +208,14 @@ def linear(input, weight, bias): module.linear = replace_linear(state) - else: + elif quantize == "gptq": + raise NotImplementedError( + "`gptq` is not implemented for now" + ) + elif quantize is None: tensor = tensor.to(device) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") if current_parameter_tensor is not None: module._parameters[param_name] = tensor diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 70f08ed7..d715207b 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -100,14 +100,14 @@ def serve( model_id: str, revision: Optional[str], sharded: bool, - quantize: bool, + quantize: Optional[str], uds_path: Path, ): async def serve_inner( model_id: str, revision: Optional[str], sharded: bool = False, - quantize: bool = False, + quantize: Optional[str] = None, ): unix_socket_template = "unix://{}-{}" if sharded: From 8a8f43410d0b37c496bda429741a0651d3515765 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 12 May 2023 17:32:40 +0200 Subject: [PATCH 230/793] chore(docker): use nvidia base image (#318) --- Dockerfile | 13 +--------- .../flash_santacoder_modeling.py | 26 ++++++++++++++----- .../models/flash_santacoder.py | 1 + 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 50145395..930d7ad7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,7 +108,7 @@ COPY server/Makefile-transformers Makefile RUN BUILD_EXTENSIONS="True" make build-transformers # Text Generation Inference base image -FROM debian:bullseye-slim as base +FROM nvidia/cuda:11.8.0-base-ubuntu22.04 as base # Conda env ENV PATH=/opt/conda/bin:$PATH \ @@ -122,17 +122,6 @@ ENV HUGGINGFACE_HUB_CACHE=/data \ NUM_SHARD=1 \ PORT=80 -# NVIDIA env vars -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -# Required for nvidia-docker v1 -RUN /bin/bash -c echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \ - echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf -ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 -ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH - -LABEL com.nvidia.volumes.needed="nvidia_driver" - WORKDIR /usr/src RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 309ec19f..23c3ea28 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -585,13 +585,25 @@ def forward( if self.transformer.tp_embeddings: # Logits are sharded, so we need to gather them - world_logits = [ - torch.empty_like(logits) for _ in range(self.transformer.tp_world_size) - ] - torch.distributed.all_gather( - world_logits, logits, group=self.transformer.process_group - ) - world_logits = torch.cat(world_logits, dim=1) + if logits.shape[0] == 1: + # Fast path when batch size is 1 + world_logits = logits.new_empty( + (logits.shape[1] * self.transformer.tp_world_size) + ) + torch.distributed.all_gather_into_tensor( + world_logits, logits.view(-1), group=self.transformer.process_group + ) + world_logits = world_logits.view(1, -1) + else: + # We cannot use all_gather_into_tensor as it only support concatenating on the first dim + world_logits = [ + torch.empty_like(logits) + for _ in range(self.transformer.tp_world_size) + ] + torch.distributed.all_gather( + world_logits, logits, group=self.transformer.process_group + ) + world_logits = torch.cat(world_logits, dim=1) return world_logits, present diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 6e5c231e..c463ee98 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -217,6 +217,7 @@ def __init__( device=device, rank=rank, world_size=world_size, + decode_buffer=1, ) @staticmethod From 119f7e0687df3c3b6160ad9e7801cfb6151d890d Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 12 May 2023 17:56:32 +0200 Subject: [PATCH 231/793] fix(docker): remove quantize default --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 930d7ad7..3d44d133 100644 --- a/Dockerfile +++ b/Dockerfile @@ -118,7 +118,6 @@ ENV PATH=/opt/conda/bin:$PATH \ ENV HUGGINGFACE_HUB_CACHE=/data \ HF_HUB_ENABLE_HF_TRANSFER=1 \ MODEL_ID=bigscience/bloom-560m \ - QUANTIZE=false \ NUM_SHARD=1 \ PORT=80 From 22c4fd07abe9d499cd8eda807f389084773124bd Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 12 May 2023 18:38:59 +0200 Subject: [PATCH 232/793] fix(docker): use ubuntu20.04 --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d44d133..483270a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,7 +108,7 @@ COPY server/Makefile-transformers Makefile RUN BUILD_EXTENSIONS="True" make build-transformers # Text Generation Inference base image -FROM nvidia/cuda:11.8.0-base-ubuntu22.04 as base +FROM nvidia/cuda:11.8.0-base-ubuntu20.04 as base # Conda env ENV PATH=/opt/conda/bin:$PATH \ @@ -117,8 +117,6 @@ ENV PATH=/opt/conda/bin:$PATH \ # Text Generation Inference base env ENV HUGGINGFACE_HUB_CACHE=/data \ HF_HUB_ENABLE_HF_TRANSFER=1 \ - MODEL_ID=bigscience/bloom-560m \ - NUM_SHARD=1 \ PORT=80 WORKDIR /usr/src From 73d84c6ee5648def2a6cd77b810edeca9933c1ab Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 15 May 2023 10:35:20 +0200 Subject: [PATCH 233/793] Hotfixes for santacoder/bigcode. (#294) # What does this PR do? Hotfixes: - Uses `model_type`=`gpt_bigcode` for more general usage. - Hotfixes linked lm_head vs wte_embedding (safetensors file do not contain the key, correctly when the file is sharded, where as pytorch copies the tensor) Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Ubuntu Co-authored-by: OlivierDehaene --- server/text_generation_server/models/__init__.py | 13 ++++++++++++- .../models/flash_santacoder.py | 3 +++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index e02be3de..ec990fde 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -99,7 +99,7 @@ def get_model( else: return Galactica(model_id, revision, quantize=quantize) - if "bigcode" in model_id: + if model_id.startswith("bigcode/"): if sharded: if not FLASH_ATTENTION: raise NotImplementedError( @@ -113,6 +113,17 @@ def get_model( config = AutoConfig.from_pretrained(model_id, revision=revision) model_type = config.model_type + if model_type == "gpt_bigcode": + if sharded: + if not FLASH_ATTENTION: + raise NotImplementedError( + FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Santacoder") + ) + return FlashSantacoderSharded(model_id, revision, quantize=quantize) + else: + santacoder_cls = FlashSantacoder if FLASH_ATTENTION else SantaCoder + return santacoder_cls(model_id, revision, quantize=quantize) + if model_type == "bloom": if sharded: return BLOOMSharded(model_id, revision, quantize=quantize) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index c463ee98..afe4eba5 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -376,6 +376,9 @@ def load_weights( else: module._buffers[param_name] = tensor + + model.lm_head.weight = torch.nn.Parameter(model.transformer.wte.weight) + uninitialized_parameters = [] for n, p in model.named_parameters(): if p.data.device == torch.device("meta"): From 91e674bb85760a19afb509cc0010d46b090183fd Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 15 May 2023 11:32:25 +0200 Subject: [PATCH 234/793] Lifting check_unitialized. (#325) # What does this PR do? Lifting check_unitialized. Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/models/bloom.py | 9 --------- .../models/flash_llama.py | 18 ------------------ .../models/flash_neox.py | 9 --------- .../models/flash_santacoder.py | 11 ----------- .../text_generation_server/models/galactica.py | 9 --------- .../text_generation_server/models/gpt_neox.py | 9 --------- server/text_generation_server/models/model.py | 11 +++++++++++ server/text_generation_server/models/opt.py | 9 --------- server/text_generation_server/models/t5.py | 9 --------- 9 files changed, 11 insertions(+), 83 deletions(-) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 877acb00..f6a69031 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -238,15 +238,6 @@ def linear(input, weight, bias): if name == "word_embeddings.weight": model.lm_head._parameters["weight"] = tensor - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model: {uninitialized_parameters}" - ) - def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 5f47cf66..0b63f904 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -139,15 +139,6 @@ def load_weights( del value - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model: {uninitialized_parameters}" - ) - torch.cuda.empty_cache() model.post_load_weights(quantize) @@ -315,14 +306,5 @@ def load_weights( else: module._buffers[param_name] = tensor - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model: {uninitialized_parameters}" - ) - torch.cuda.empty_cache() model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index b3e1876f..168c9195 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -152,13 +152,4 @@ def load_weights( else: module._buffers[param_name] = tensor - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model: {uninitialized_parameters}" - ) - model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index afe4eba5..51a8998b 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -376,17 +376,6 @@ def load_weights( else: module._buffers[param_name] = tensor - model.lm_head.weight = torch.nn.Parameter(model.transformer.wte.weight) - - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model: {uninitialized_parameters}" - ) - torch.cuda.empty_cache() model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 4f94b348..c6dd4c33 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -365,15 +365,6 @@ def linear(input, weight, bias): if name == "model.decoder.embed_tokens.weight": model.lm_head._parameters["weight"] = tensor - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model: {uninitialized_parameters}" - ) - def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 2d42e0b0..215bb2b6 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -215,15 +215,6 @@ def linear(input, weight, bias): else: module._buffers[param_name] = tensor - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model: {uninitialized_parameters}" - ) - def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 4c85c952..03f14013 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -32,6 +32,7 @@ def __init__( self.decode_buffer = decode_buffer self.rank = rank self.world_size = world_size + self.check_initialized() @property def info(self) -> InfoResponse: @@ -99,3 +100,13 @@ def decode_token( return token_text, None, None else: return "", offset, token_offset + + def check_initialized(self): + uninitialized_parameters = [] + for n, p in self.model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model {self.__class__.__name__}: {uninitialized_parameters}" + ) diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 44f15df3..8d856b10 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -212,15 +212,6 @@ def linear(input, weight, bias): if name == "model.decoder.embed_tokens.weight": model.lm_head._parameters["weight"] = tensor - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model: {uninitialized_parameters}" - ) - def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 381617b7..b5e7710d 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -222,15 +222,6 @@ def linear(input, weight, bias): else: module._buffers[param_name] = tensor - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model: {uninitialized_parameters}" - ) - def forward( self, input_ids, From d7a97aa0b60fcf2a861ccd2ffb40e34cdf2c7650 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 15 May 2023 15:14:17 +0200 Subject: [PATCH 235/793] Removing dead variables. (#327) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/models/bloom.py | 1 - server/text_generation_server/models/flash_llama.py | 3 --- server/text_generation_server/models/flash_neox.py | 2 -- server/text_generation_server/models/flash_santacoder.py | 3 --- server/text_generation_server/models/galactica.py | 1 - server/text_generation_server/models/gpt_neox.py | 1 - server/text_generation_server/models/opt.py | 1 - server/text_generation_server/models/t5.py | 1 - 8 files changed, 13 deletions(-) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index f6a69031..ed959291 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -72,7 +72,6 @@ def __init__( quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() - self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") dtype = torch.float16 diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 0b63f904..aa0b4483 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -29,7 +29,6 @@ class FlashLlama(FlashCausalLM): def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): - self.past_pad = None if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.float16 @@ -150,9 +149,7 @@ def __init__( revision: Optional[str] = None, quantize: Optional[str] = None, ): - self.past_pad = None self.process_group, rank, world_size = initialize_torch_distributed() - self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") dtype = torch.float16 diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 168c9195..fc741f55 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -33,9 +33,7 @@ class FlashNeoXSharded(FlashNeoX): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - self.past_pad = None self.process_group, rank, world_size = initialize_torch_distributed() - self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") dtype = torch.float16 diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 51a8998b..f810bb0b 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -28,7 +28,6 @@ class FlashSantacoder(FlashCausalLM): def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): - self.past_pad = None if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.float16 @@ -173,9 +172,7 @@ class FlashSantacoderSharded(FlashSantacoder): def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): - self.past_pad = None self.process_group, rank, world_size = initialize_torch_distributed() - self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") dtype = torch.float16 diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index c6dd4c33..a0111250 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -199,7 +199,6 @@ def __init__( quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() - self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") dtype = torch.float16 diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 215bb2b6..3e8557b2 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -38,7 +38,6 @@ def __init__( quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() - self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") dtype = torch.float16 diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 8d856b10..c83c3351 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -51,7 +51,6 @@ def __init__( self, model_id: str, revision: Optional[str] = None, quantize: bool = False ): self.process_group, rank, world_size = initialize_torch_distributed() - self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") dtype = torch.float16 diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index b5e7710d..6fe77ca2 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -38,7 +38,6 @@ def __init__( quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() - self.master = rank == 0 if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") dtype = torch.float16 From 66b277321d738ea6f0703b485ecefb61fb3f5441 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 15 May 2023 15:53:08 +0200 Subject: [PATCH 236/793] feat(ci): custom gpu runners (#328) --- .github/workflows/build.yaml | 104 +++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 289e4f67..0a99fb52 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -25,8 +25,44 @@ concurrency: cancel-in-progress: true jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + env: + AWS_REGION: us-east-1 + EC2_AMI_ID: ami-03cfed9ea28f4b002 + EC2_INSTANCE_TYPE: g5.12xlarge + EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc + EC2_SECURITY_GROUP: sg-04d472c808f365022 + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + - name: Start EC2 runner + id: start-ec2-runner + uses: philschmid/philschmid-ec2-github-runner@main + with: + mode: start + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ${{ env.EC2_AMI_ID }} + ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }} + subnet-id: ${{ env.EC2_SUBNET_ID }} + security-group-id: ${{ env.EC2_SECURITY_GROUP }} + aws-resource-tags: > # optional, requires additional permissions + [ + {"Key": "Name", "Value": "ec2-tgi-github-runner"}, + {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} + ] + build-and-push-image: - runs-on: large-runner + needs: start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner permissions: contents: write packages: write @@ -49,7 +85,7 @@ jobs: with: cosign-release: 'v1.13.1' - name: Tailscale - uses: tailscale/github-action@v1 + uses: tailscale/github-action@7bd8039bf25c23c4ab1b8d6e2cc2da2280601966 with: authkey: ${{ secrets.TAILSCALE_AUTHKEY }} - name: Login to GitHub Container Registry @@ -136,52 +172,26 @@ jobs: with: sarif_file: 'trivy-results.sarif' - build-and-push-sagemaker-image: + stop-runner: + name: Stop self-hosted EC2 runner needs: + - start-runner - build-and-push-image runs-on: ubuntu-latest + env: + AWS_REGION: us-east-1 + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Initialize Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 - with: - install: true - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4.4.1 - - name: Tailscale - uses: tailscale/github-action@v1 - with: - authkey: ${{ secrets.TAILSCALE_AUTHKEY }} - - name: Login to internal Container Registry - uses: docker/login-action@v2.1.0 - with: - username: ${{ secrets.TAILSCALE_DOCKER_USERNAME }} - password: ${{ secrets.TAILSCALE_DOCKER_PASSWORD }} - registry: registry.internal.huggingface.tech - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4.3.0 - with: - flavor: | - latest=auto - images: | - registry.internal.huggingface.tech/api-inference/community/text-generation-inference/sagemaker - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} - type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }} - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: . - file: Dockerfile - push: ${{ github.event_name != 'pull_request' }} - platforms: 'linux/amd64' - build-args: | - GIT_SHA={{ env.GITHUB_SHA }} - target: sagemaker - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max \ No newline at end of file + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + - name: Stop EC2 runner + uses: philschmid/philschmid-ec2-github-runner@main + with: + mode: stop + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file From f58f0a03647ce730f39404ecd50fb909db1bcfe0 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 15 May 2023 17:30:47 +0200 Subject: [PATCH 237/793] Single place for TP layers + Dropout Layer Norm + FastLinear (#329) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../custom_modeling/flash_llama_modeling.py | 225 +-------------- .../custom_modeling/flash_neox_modeling.py | 264 +---------------- .../flash_santacoder_modeling.py | 225 +-------------- server/text_generation_server/models/t5.py | 15 +- server/text_generation_server/utils/layers.py | 272 ++++++++++++++++++ 5 files changed, 303 insertions(+), 698 deletions(-) create mode 100644 server/text_generation_server/utils/layers.py diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 5b32b22e..11f3766e 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -28,17 +28,16 @@ from typing import Optional # Flash attention imports -import rotary_emb import flash_attn_cuda -import dropout_layer_norm from flash_attn.layers.rotary import RotaryEmbedding - -HAS_BITS_AND_BYTES = True -try: - from bitsandbytes.nn import Linear8bitLt -except ImportError as e: - HAS_BITS_AND_BYTES = False +from text_generation_server.utils.layers import ( + FastLinear, + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, + PositionRotaryEmbedding, +) class LlamaRMSNorm(nn.Module): @@ -91,216 +90,6 @@ def forward(self, hidden_states, residual=None): return normed_hidden_states, res -class FastLinear(nn.Linear): - def __init__( - self, - in_features: int, - out_features: int, - bias: bool = True, - device=None, - dtype=None, - ) -> None: - super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) - self.quantized = False - self.bnb_linear = None - - def prepare_weights(self, quantize: bool = False): - if quantize == "bitsandbytes": - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine either because it is not installed " - "or you don't have a GPU.\n" - "You can install it with `pip install bitsandbytes`." - ) - - self.quantized = True - self.bnb_linear = Linear8bitLt( - self.in_features, - self.out_features, - has_fp16_weights=False, - threshold=6.0, - bias=False, - ) - # Copy data to bnb_linear - self.bnb_linear.weight.data = self.weight.data - if self.bias is not None: - self.bnb_linear.bias = nn.Parameter(self.bias) - - # Delete reference to data - self.weight = None - self.bias = None - elif quantize == "gptq": - raise NotImplementedError("`gptq` is not implemented for now") - elif quantize is None: - self.weight = nn.Parameter(self.weight.T) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") - - def forward(self, input: torch.Tensor) -> torch.Tensor: - if self.quantized: - return self.bnb_linear(input) - else: - if self.bias is not None: - return torch.addmm(self.bias, input, self.weight) - return torch.matmul(input, self.weight) - - -class TensorParallelColumnLinear(FastLinear): - def __init__( - self, - in_features, - out_features, - process_group: torch.distributed.ProcessGroup, - bias=True, - device=None, - dtype=None, - ): - self.process_group = process_group - self.tp_world_size = process_group.size() - assert out_features % self.tp_world_size == 0 - out_features = out_features // self.tp_world_size - - super().__init__( - in_features=in_features, - out_features=out_features, - bias=bias, - device=device, - dtype=dtype, - ) - - -class TensorParallelRowLinear(FastLinear): - def __init__( - self, - in_features, - out_features, - process_group: torch.distributed.ProcessGroup, - reduce=True, - bias=True, - device=None, - dtype=None, - ): - self.process_group = process_group - self.tp_world_size = process_group.size() - self.reduce = reduce - assert in_features % self.tp_world_size == 0 - in_features = in_features // self.tp_world_size - - super().__init__( - in_features=in_features, - out_features=out_features, - bias=bias, - device=device, - dtype=dtype, - ) - - def forward(self, input: torch.Tensor) -> torch.Tensor: - out = super(TensorParallelRowLinear, self).forward(input) - if self.reduce: - torch.distributed.all_reduce(out, group=self.process_group) - - return out - - -class TensorParallelEmbedding(nn.Embedding): - def __init__( - self, - num_embeddings, - embedding_dim, - process_group: torch.distributed.ProcessGroup, - padding_idx=None, - max_norm=None, - norm_type=2.0, - scale_grad_by_freq=False, - sparse=False, - _weight=None, - device=None, - dtype=None, - ): - self.process_group = process_group - self.tp_rank = process_group.rank() - self.tp_world_size = process_group.size() - - self.original_num_embeddings = num_embeddings - - assert num_embeddings % self.tp_world_size == 0 - block_size = num_embeddings // self.tp_world_size - # inputs in `[min_id, max_id[` are handled by `self` to get embeddings - self.min_id = self.tp_rank * block_size - self.max_id = (self.tp_rank + 1) * block_size - - # Additional entry that will map to zero - # Used for masking - self.null_idx = block_size - - super().__init__( - block_size, - embedding_dim, - padding_idx=padding_idx, - max_norm=max_norm, - norm_type=norm_type, - scale_grad_by_freq=scale_grad_by_freq, - sparse=sparse, - _weight=_weight, - device=device, - dtype=dtype, - ) - - def add_null_idx(self): - """Additional 0 entry used for masking""" - self.weight = nn.Parameter(F.pad(self.weight, (0, 0, 0, 1))) - - def forward(self, input: torch.Tensor) -> torch.Tensor: - # default all out of bounds values to `self.null_idx` that will then be mapped to 0 - # translate for [0, self.max_id - self.min_id[ - input = torch.where( - (self.min_id > input) | (input >= self.max_id), - self.null_idx, - input - self.min_id, - ) - out = super().forward(input) - torch.distributed.all_reduce(out, group=self.process_group) - return out - - -class PositionRotaryEmbedding(RotaryEmbedding): - def _update_cos_sin_cache(self, dtype, device, seqlen): - # Reset the tables if the sequence length has changed, - # or if we're on a new device (possibly due to tracing for instance) - if ( - seqlen > self._seq_len_cached - or self._cos_cached.device != device - or self._cos_cached.dtype != dtype - ): - self._seq_len_cached = seqlen - t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) - freqs = torch.outer(t, self.inv_freq.to(device=t.device)) - self._cos_cached = torch.cos(freqs).to(dtype) - self._sin_cached = torch.sin(freqs).to(dtype) - - def get_cos_sin(self, position_ids: torch.Tensor, max_s: int, dtype: torch.dtype): - """ - Return cos and sin for the asked position ids - """ - - self._update_cos_sin_cache(dtype, position_ids.device, max_s) - - cos = torch.index_select(self._cos_cached, 0, position_ids) - sin = torch.index_select(self._sin_cached, 0, position_ids) - return cos.unsqueeze(1), sin.unsqueeze(1) - - def forward(self, qkv: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): - rotary_dim = cos.shape[-1] - q1 = qkv[:, 0, :, :rotary_dim] - q2 = qkv[:, 0, :, rotary_dim : 2 * rotary_dim] - k1 = qkv[:, 1, :, :rotary_dim] - k2 = qkv[:, 1, :, rotary_dim : 2 * rotary_dim] - - rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False) - rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False) - return qkv - - class FlashLlamaAttention(torch.nn.Module): def __init__( self, diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index e7b878c0..369e8d4f 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -30,265 +30,17 @@ from typing import Optional # Flash attention imports -import rotary_emb import flash_attn_cuda -import dropout_layer_norm from flash_attn.layers.rotary import RotaryEmbedding - -HAS_BITS_AND_BYTES = True -try: - from bitsandbytes.nn import Linear8bitLt -except ImportError as e: - HAS_BITS_AND_BYTES = False - - -class FastLayerNorm(nn.LayerNorm): - def forward(self, hidden_states, residual=None): - if hidden_states.shape[-1] > 8192: - if residual is not None: - hidden_states += residual - residual = hidden_states - - return super(FastLayerNorm, self).forward(hidden_states), residual - else: - ( - normed_hidden_states, - residual, - *rest, - ) = dropout_layer_norm.dropout_add_ln_fwd( - hidden_states, - residual, - self.weight, - self.bias, - None, - None, - None, - None, - 0.0, - self.eps, - 1.0, - 0, - None, - False, - False, - ) - if residual is None: - residual = hidden_states - - return normed_hidden_states, residual - - -class FastLinear(nn.Linear): - def __init__( - self, - in_features: int, - out_features: int, - bias: bool = True, - device=None, - dtype=None, - ) -> None: - super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) - self.quantized = False - self.bnb_linear = None - - def prepare_weights(self, quantize: Optional[str] = None): - if quantize == "bitsandbytes": - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine either because it is not installed " - "or you don't have a GPU.\n" - "You can install it with `pip install bitsandbytes`." - ) - - self.quantized = True - self.bnb_linear = Linear8bitLt( - self.in_features, - self.out_features, - has_fp16_weights=False, - threshold=6.0, - bias=False, - ) - # Copy data to bnb_linear - self.bnb_linear.weight.data = self.weight.data - if self.bias is not None: - self.bnb_linear.bias = nn.Parameter(self.bias) - - # Delete reference to data - self.weight = None - self.bias = None - elif quantize == "gptq": - raise NotImplementedError("`gptq` is not implemented for now") - elif quantize is None: - self.weight = nn.Parameter(self.weight.T) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") - - def forward(self, input: torch.Tensor) -> torch.Tensor: - if self.quantized: - return self.bnb_linear(input) - else: - if self.bias is not None: - return torch.addmm(self.bias, input, self.weight) - return torch.matmul(input, self.weight) - - -class TensorParallelColumnLinear(FastLinear): - def __init__( - self, - in_features, - out_features, - process_group: torch.distributed.ProcessGroup, - bias=True, - device=None, - dtype=None, - ): - self.process_group = process_group - self.tp_world_size = process_group.size() - assert out_features % self.tp_world_size == 0 - out_features = out_features // self.tp_world_size - - super().__init__( - in_features=in_features, - out_features=out_features, - bias=bias, - device=device, - dtype=dtype, - ) - - -class TensorParallelRowLinear(FastLinear): - def __init__( - self, - in_features, - out_features, - process_group: torch.distributed.ProcessGroup, - reduce=True, - bias=True, - device=None, - dtype=None, - ): - self.process_group = process_group - self.tp_world_size = process_group.size() - self.reduce = reduce - assert in_features % self.tp_world_size == 0 - in_features = in_features // self.tp_world_size - - super().__init__( - in_features=in_features, - out_features=out_features, - bias=bias, - device=device, - dtype=dtype, - ) - - def forward(self, input: torch.Tensor) -> torch.Tensor: - out = super(TensorParallelRowLinear, self).forward(input) - if self.reduce: - torch.distributed.all_reduce(out, group=self.process_group) - - return out - - -class TensorParallelEmbedding(nn.Embedding): - def __init__( - self, - num_embeddings, - embedding_dim, - process_group: torch.distributed.ProcessGroup, - padding_idx=None, - max_norm=None, - norm_type=2.0, - scale_grad_by_freq=False, - sparse=False, - _weight=None, - device=None, - dtype=None, - ): - self.process_group = process_group - self.tp_rank = process_group.rank() - self.tp_world_size = process_group.size() - - self.original_num_embeddings = num_embeddings - - assert num_embeddings % self.tp_world_size == 0 - block_size = num_embeddings // self.tp_world_size - # inputs in `[min_id, max_id[` are handled by `self` to get embeddings - self.min_id = self.tp_rank * block_size - self.max_id = (self.tp_rank + 1) * block_size - - # Additional entry that will map to zero - # Used for masking - self.null_idx = block_size - - super().__init__( - block_size, - embedding_dim, - padding_idx=padding_idx, - max_norm=max_norm, - norm_type=norm_type, - scale_grad_by_freq=scale_grad_by_freq, - sparse=sparse, - _weight=_weight, - device=device, - dtype=dtype, - ) - - def add_null_idx(self): - """Additional 0 entry used for masking""" - self.weight = nn.Parameter(F.pad(self.weight, (0, 0, 0, 1))) - - def forward(self, input: torch.Tensor) -> torch.Tensor: - # default all out of bounds values to `self.null_idx` that will then be mapped to 0 - # translate for [0, self.max_id - self.min_id[ - input = torch.where( - (self.min_id > input) | (input >= self.max_id), - self.null_idx, - input - self.min_id, - ) - out = super().forward(input) - torch.distributed.all_reduce(out, group=self.process_group) - return out - - -class PositionRotaryEmbedding(RotaryEmbedding): - def _update_cos_sin_cache(self, dtype, device, seqlen): - # Reset the tables if the sequence length has changed, - # or if we're on a new device (possibly due to tracing for instance) - if ( - seqlen > self._seq_len_cached - or self._cos_cached.device != device - or self._cos_cached.dtype != dtype - ): - self._seq_len_cached = seqlen - t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) - # Don't do einsum, it converts fp32 to fp16 - # freqs = torch.einsum("i,j->ij", t, self.inv_freq) - freqs = torch.outer(t, self.inv_freq.to(device=t.device)) - self._cos_cached = torch.cos(freqs).to(dtype) - self._sin_cached = torch.sin(freqs).to(dtype) - - def get_cos_sin(self, position_ids: torch.Tensor, max_s: int, dtype: torch.dtype): - """ - Return cos and sin for the asked position ids - """ - - self._update_cos_sin_cache(dtype, position_ids.device, max_s) - - cos = torch.index_select(self._cos_cached, 0, position_ids) - sin = torch.index_select(self._sin_cached, 0, position_ids) - return cos.unsqueeze(1), sin.unsqueeze(1) - - def forward(self, qkv: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): - rotary_dim = cos.shape[-1] - q1 = qkv[:, 0, :, :rotary_dim] - q2 = qkv[:, 0, :, rotary_dim : 2 * rotary_dim] - k1 = qkv[:, 1, :, :rotary_dim] - k2 = qkv[:, 1, :, rotary_dim : 2 * rotary_dim] - - rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False) - rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False) - return qkv +from text_generation_server.utils.layers import ( + FastLinear, + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, + FastLayerNorm, + PositionRotaryEmbedding, +) class FlashNeoxAttention(torch.nn.Module): diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 23c3ea28..9451b01a 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -9,224 +9,13 @@ # Flash attention imports import flash_attn_cuda -import dropout_layer_norm - -HAS_BITS_AND_BYTES = True -try: - from bitsandbytes.nn import Linear8bitLt -except ImportError as e: - HAS_BITS_AND_BYTES = False - - -class FastLayerNorm(nn.LayerNorm): - def forward(self, hidden_states, residual=None): - if hidden_states.shape[-1] > 8192: - if residual is not None: - hidden_states += residual - residual = hidden_states - - return super(FastLayerNorm, self).forward(hidden_states), residual - else: - ( - normed_hidden_states, - residual, - *rest, - ) = dropout_layer_norm.dropout_add_ln_fwd( - hidden_states, - residual, - self.weight, - self.bias, - None, - None, - None, - None, - 0.0, - self.eps, - 1.0, - 0, - None, - False, - False, - ) - if residual is None: - residual = hidden_states - - return normed_hidden_states, residual - - -class FastLinear(nn.Linear): - def __init__( - self, - in_features: int, - out_features: int, - bias: bool = True, - device=None, - dtype=None, - ) -> None: - super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) - self.quantized = False - self.bnb_linear = None - - def prepare_weights(self, quantize: Optional[str] = None): - if quantize == "bitsandbytes": - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine either because it is not installed " - "or you don't have a GPU.\n" - "You can install it with `pip install bitsandbytes`." - ) - - self.quantized = True - self.bnb_linear = Linear8bitLt( - self.in_features, - self.out_features, - has_fp16_weights=False, - threshold=6.0, - bias=False, - ) - # Copy data to bnb_linear - self.bnb_linear.weight.data = self.weight.data - if self.bias is not None: - self.bnb_linear.bias = nn.Parameter(self.bias) - - # Delete reference to data - self.weight = None - self.bias = None - elif quantize == "gptq": - raise NotImplementedError("`gptq` is not implemented for now") - elif quantize is None: - self.weight = nn.Parameter(self.weight.T) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") - - def forward(self, input: torch.Tensor) -> torch.Tensor: - if self.quantized: - return self.bnb_linear(input) - else: - if self.bias is not None: - return torch.addmm(self.bias, input, self.weight) - return torch.matmul(input, self.weight) - - -class TensorParallelColumnLinear(FastLinear): - def __init__( - self, - in_features, - out_features, - process_group: torch.distributed.ProcessGroup, - bias=True, - device=None, - dtype=None, - ): - self.process_group = process_group - self.tp_world_size = process_group.size() - assert out_features % self.tp_world_size == 0 - out_features = out_features // self.tp_world_size - - super().__init__( - in_features=in_features, - out_features=out_features, - bias=bias, - device=device, - dtype=dtype, - ) - - -class TensorParallelRowLinear(FastLinear): - def __init__( - self, - in_features, - out_features, - process_group: torch.distributed.ProcessGroup, - reduce=True, - bias=True, - device=None, - dtype=None, - ): - self.process_group = process_group - self.tp_world_size = process_group.size() - self.reduce = reduce - assert in_features % self.tp_world_size == 0 - in_features = in_features // self.tp_world_size - - super().__init__( - in_features=in_features, - out_features=out_features, - bias=bias, - device=device, - dtype=dtype, - ) - - def forward(self, input: torch.Tensor) -> torch.Tensor: - out = super(TensorParallelRowLinear, self).forward(input) - if self.reduce: - torch.distributed.all_reduce(out, group=self.process_group) - - return out - - -class TensorParallelEmbedding(nn.Embedding): - def __init__( - self, - num_embeddings, - embedding_dim, - process_group: torch.distributed.ProcessGroup, - reduce=True, - padding_idx=None, - max_norm=None, - norm_type=2.0, - scale_grad_by_freq=False, - sparse=False, - _weight=None, - device=None, - dtype=None, - ): - self.process_group = process_group - self.tp_rank = process_group.rank() - self.tp_world_size = process_group.size() - self.reduce = reduce - - self.original_num_embeddings = num_embeddings - - assert num_embeddings % self.tp_world_size == 0 - block_size = num_embeddings // self.tp_world_size - # inputs in `[min_id, max_id[` are handled by `self` to get embeddings - self.min_id = self.tp_rank * block_size - self.max_id = (self.tp_rank + 1) * block_size - - # Additional entry that will map to zero - # Used for masking - self.null_idx = block_size - - super().__init__( - block_size, - embedding_dim, - padding_idx=padding_idx, - max_norm=max_norm, - norm_type=norm_type, - scale_grad_by_freq=scale_grad_by_freq, - sparse=sparse, - _weight=_weight, - device=device, - dtype=dtype, - ) - - def add_null_idx(self): - """Additional 0 entry used for masking""" - self.weight = nn.Parameter(F.pad(self.weight, (0, 0, 0, 1))) - - def forward(self, input: torch.Tensor) -> torch.Tensor: - # default all out of bounds values to `self.null_idx` that will then be mapped to 0 - # translate for [0, self.max_id - self.min_id[ - input = torch.where( - (self.min_id > input) | (input >= self.max_id), - self.null_idx, - input - self.min_id, - ) - out = super().forward(input) - if self.reduce: - torch.distributed.all_reduce(out, group=self.process_group) - return out +from text_generation_server.utils.layers import ( + FastLinear, + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, + FastLayerNorm, +) class FlashMQAttention(torch.nn.Module): diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 6fe77ca2..c8521dbf 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -10,23 +10,26 @@ AutoModelForSeq2SeqLM, AutoConfig, ) -from transformers.models.t5.parallel_layers import ( - TensorParallelColumnLinear, - TensorParallelEmbedding, - TensorParallelRowLinear, -) from text_generation_server.models import Seq2SeqLM from text_generation_server.utils import ( initialize_torch_distributed, weight_files, ) +from text_generation_server.utils.layers import ( + FastLinear, +) +from transformers.models.t5.parallel_layers import ( + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, +) HAS_BITS_AND_BYTES = True try: import bitsandbytes as bnb from bitsandbytes.nn import Int8Params -except Exception as e: +except ImportError as e: HAS_BITS_AND_BYTES = False diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py new file mode 100644 index 00000000..3383bf4b --- /dev/null +++ b/server/text_generation_server/utils/layers.py @@ -0,0 +1,272 @@ +import torch + +from torch import nn + +HAS_BITS_AND_BYTES = True +try: + from bitsandbytes.nn import Linear8bitLt +except ImportError as e: + HAS_BITS_AND_BYTES = False + + +class FastLinear(nn.Linear): + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + device=None, + dtype=None, + ) -> None: + super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) + self.quantized = False + self.bnb_linear = None + + def prepare_weights(self, quantize: bool = False): + if quantize == "bitsandbytes": + if not HAS_BITS_AND_BYTES: + raise ImportError( + "bitsandbytes is not available on your machine either because it is not installed " + "or you don't have a GPU.\n" + "You can install it with `pip install bitsandbytes`." + ) + + self.quantized = True + self.bnb_linear = Linear8bitLt( + self.in_features, + self.out_features, + has_fp16_weights=False, + threshold=6.0, + bias=False, + ) + # Copy data to bnb_linear + self.bnb_linear.weight.data = self.weight.data + if self.bias is not None: + self.bnb_linear.bias = nn.Parameter(self.bias) + + # Delete reference to data + self.weight = None + self.bias = None + elif quantize == "gptq": + raise NotImplementedError("`gptq` is not implemented for now") + elif quantize is None: + self.weight = nn.Parameter(self.weight.T) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") + + def forward(self, input: torch.Tensor) -> torch.Tensor: + if self.quantized: + return self.bnb_linear(input) + else: + if self.bias is not None: + return torch.addmm(self.bias, input, self.weight) + return torch.matmul(input, self.weight) + + +class TensorParallelColumnLinear(FastLinear): + def __init__( + self, + in_features, + out_features, + process_group: torch.distributed.ProcessGroup, + bias=True, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_world_size = process_group.size() + assert out_features % self.tp_world_size == 0 + out_features = out_features // self.tp_world_size + + super().__init__( + in_features=in_features, + out_features=out_features, + bias=bias, + device=device, + dtype=dtype, + ) + + +class TensorParallelRowLinear(FastLinear): + def __init__( + self, + in_features, + out_features, + process_group: torch.distributed.ProcessGroup, + reduce=True, + bias=True, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_world_size = process_group.size() + self.reduce = reduce + assert in_features % self.tp_world_size == 0 + in_features = in_features // self.tp_world_size + + super().__init__( + in_features=in_features, + out_features=out_features, + bias=bias, + device=device, + dtype=dtype, + ) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + out = super(TensorParallelRowLinear, self).forward(input) + if self.reduce: + torch.distributed.all_reduce(out, group=self.process_group) + + return out + + +class TensorParallelEmbedding(nn.Embedding): + def __init__( + self, + num_embeddings, + embedding_dim, + process_group: torch.distributed.ProcessGroup, + padding_idx=None, + max_norm=None, + norm_type=2.0, + scale_grad_by_freq=False, + sparse=False, + _weight=None, + device=None, + dtype=None, + ): + self.process_group = process_group + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + + self.original_num_embeddings = num_embeddings + + assert num_embeddings % self.tp_world_size == 0 + block_size = num_embeddings // self.tp_world_size + # inputs in `[min_id, max_id[` are handled by `self` to get embeddings + self.min_id = self.tp_rank * block_size + self.max_id = (self.tp_rank + 1) * block_size + + # Additional entry that will map to zero + # Used for masking + self.null_idx = block_size + + super().__init__( + block_size, + embedding_dim, + padding_idx=padding_idx, + max_norm=max_norm, + norm_type=norm_type, + scale_grad_by_freq=scale_grad_by_freq, + sparse=sparse, + _weight=_weight, + device=device, + dtype=dtype, + ) + + def add_null_idx(self): + """Additional 0 entry used for masking""" + self.weight = nn.Parameter(F.pad(self.weight, (0, 0, 0, 1))) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + # default all out of bounds values to `self.null_idx` that will then be mapped to 0 + # translate for [0, self.max_id - self.min_id[ + input = torch.where( + (self.min_id > input) | (input >= self.max_id), + self.null_idx, + input - self.min_id, + ) + out = super().forward(input) + torch.distributed.all_reduce(out, group=self.process_group) + return out + + +try: + import dropout_layer_norm + + class FastLayerNorm(nn.LayerNorm): + def forward(self, hidden_states, residual=None): + if hidden_states.shape[-1] > 8192: + if residual is not None: + hidden_states += residual + residual = hidden_states + + return super(FastLayerNorm, self).forward(hidden_states), residual + else: + ( + normed_hidden_states, + residual, + *rest, + ) = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.weight, + self.bias, + None, + None, + None, + None, + 0.0, + self.eps, + 1.0, + 0, + None, + False, + False, + ) + if residual is None: + residual = hidden_states + + return normed_hidden_states, residual + +except ImportError: + pass + + +try: + from flash_attn.layers.rotary import RotaryEmbedding + import rotary_emb + + class PositionRotaryEmbedding(RotaryEmbedding): + def _update_cos_sin_cache(self, dtype, device, seqlen): + # Reset the tables if the sequence length has changed, + # or if we're on a new device (possibly due to tracing for instance) + if ( + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype + ): + self._seq_len_cached = seqlen + t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) + # Don't do einsum, it converts fp32 to fp16 + # freqs = torch.einsum("i,j->ij", t, self.inv_freq) + freqs = torch.outer(t, self.inv_freq.to(device=t.device)) + self._cos_cached = torch.cos(freqs).to(dtype) + self._sin_cached = torch.sin(freqs).to(dtype) + + def get_cos_sin( + self, position_ids: torch.Tensor, max_s: int, dtype: torch.dtype + ): + """ + Return cos and sin for the asked position ids + """ + + self._update_cos_sin_cache(dtype, position_ids.device, max_s) + + cos = torch.index_select(self._cos_cached, 0, position_ids) + sin = torch.index_select(self._sin_cached, 0, position_ids) + return cos.unsqueeze(1), sin.unsqueeze(1) + + def forward(self, qkv: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): + rotary_dim = cos.shape[-1] + q1 = qkv[:, 0, :, :rotary_dim] + q2 = qkv[:, 0, :, rotary_dim : 2 * rotary_dim] + k1 = qkv[:, 1, :, :rotary_dim] + k2 = qkv[:, 1, :, rotary_dim : 2 * rotary_dim] + + rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False) + rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False) + return qkv + +except ImportError: + pass From e71471bec95823ef69daaeb03c4657b9b5211a02 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 15 May 2023 23:36:30 +0200 Subject: [PATCH 238/793] feat: add snapshot testing (#282) --- .github/workflows/build.yaml | 62 +- Makefile | 11 +- README.md | 2 + integration-tests/conftest.py | 146 ++++ .../models/__snapshots__/test_bloom_560m.ambr | 627 ++++++++++++++++ .../test_bloom_560m_sharded.ambr | 542 ++++++++++++++ .../__snapshots__/test_flash_llama.ambr | 465 ++++++++++++ .../models/__snapshots__/test_flash_neox.ambr | 682 ++++++++++++++++++ .../__snapshots__/test_flash_santacoder.ambr | 472 ++++++++++++ .../__snapshots__/test_flash_starcoder.ambr | 573 +++++++++++++++ .../models/__snapshots__/test_mt0_base.ambr | 306 ++++++++ integration-tests/models/test_bloom_560m.py | 63 ++ .../models/test_bloom_560m_sharded.py | 42 ++ integration-tests/models/test_flash_llama.py | 56 ++ integration-tests/models/test_flash_neox.py | 38 + .../models/test_flash_santacoder.py | 32 + .../models/test_flash_starcoder.py | 47 ++ integration-tests/models/test_mt0_base.py | 63 ++ integration-tests/models/utils.py | 15 + integration-tests/requirements.txt | 5 + launcher/tests/bloom_560m.json | 142 ---- launcher/tests/integration_tests.rs | 172 ----- launcher/tests/mt0_base.json | 137 ---- server/text_generation_server/models/bloom.py | 2 +- .../custom_modeling/flash_llama_modeling.py | 18 +- .../custom_modeling/flash_neox_modeling.py | 17 +- .../flash_santacoder_modeling.py | 16 +- .../models/flash_llama.py | 15 +- .../models/flash_neox.py | 14 +- .../models/flash_santacoder.py | 18 +- .../models/galactica.py | 2 +- .../text_generation_server/models/gpt_neox.py | 2 +- server/text_generation_server/models/opt.py | 7 +- server/text_generation_server/models/t5.py | 2 +- server/text_generation_server/utils/layers.py | 9 +- 35 files changed, 4313 insertions(+), 509 deletions(-) create mode 100644 integration-tests/conftest.py create mode 100644 integration-tests/models/__snapshots__/test_bloom_560m.ambr create mode 100644 integration-tests/models/__snapshots__/test_bloom_560m_sharded.ambr create mode 100644 integration-tests/models/__snapshots__/test_flash_llama.ambr create mode 100644 integration-tests/models/__snapshots__/test_flash_neox.ambr create mode 100644 integration-tests/models/__snapshots__/test_flash_santacoder.ambr create mode 100644 integration-tests/models/__snapshots__/test_flash_starcoder.ambr create mode 100644 integration-tests/models/__snapshots__/test_mt0_base.ambr create mode 100644 integration-tests/models/test_bloom_560m.py create mode 100644 integration-tests/models/test_bloom_560m_sharded.py create mode 100644 integration-tests/models/test_flash_llama.py create mode 100644 integration-tests/models/test_flash_neox.py create mode 100644 integration-tests/models/test_flash_santacoder.py create mode 100644 integration-tests/models/test_flash_starcoder.py create mode 100644 integration-tests/models/test_mt0_base.py create mode 100644 integration-tests/models/utils.py create mode 100644 integration-tests/requirements.txt delete mode 100644 launcher/tests/bloom_560m.json delete mode 100644 launcher/tests/integration_tests.rs delete mode 100644 launcher/tests/mt0_base.json diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0a99fb52..c2aba160 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,10 +20,6 @@ on: branches: - 'main' -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - jobs: start-runner: name: Start self-hosted EC2 runner @@ -61,6 +57,9 @@ jobs: ] build-and-push-image: + concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true needs: start-runner # required to start the main job when the runner is ready runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner permissions: @@ -108,7 +107,19 @@ jobs: username: ${{ secrets.AZURE_DOCKER_USERNAME }} password: ${{ secrets.AZURE_DOCKER_PASSWORD }} registry: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io + # If pull request - name: Extract metadata (tags, labels) for Docker + if: ${{ github.event_name == 'pull_request' }} + id: meta-pr + uses: docker/metadata-action@v4.3.0 + with: + images: | + registry.internal.huggingface.tech/api-inference/community/text-generation-inference + tags: | + type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }} + # If main, release or tag + - name: Extract metadata (tags, labels) for Docker + if: ${{ github.event_name != 'pull_request' }} id: meta uses: docker/metadata-action@v4.3.0 with: @@ -129,13 +140,13 @@ jobs: with: context: . file: Dockerfile - push: ${{ github.event_name != 'pull_request' }} + push: true platforms: 'linux/amd64' build-args: | GIT_SHA=${{ env.GITHUB_SHA }} DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags || steps.meta-pr.outputs.tags }} + labels: ${{ steps.meta.outputs.labels || steps.meta-pr.outputs.labels }} cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max # Sign the resulting Docker image digest except on PRs. @@ -172,11 +183,48 @@ jobs: with: sarif_file: 'trivy-results.sarif' + integration-tests: + concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + needs: + - start-runner + - build-and-push-image # Wait for the docker image to be built + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + env: + DOCKER_VOLUME: /cache + steps: + - uses: actions/checkout@v2 + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Tailscale + uses: tailscale/github-action@7bd8039bf25c23c4ab1b8d6e2cc2da2280601966 + with: + authkey: ${{ secrets.TAILSCALE_AUTHKEY }} + - name: Prepare disks + run: | + sudo mkfs -t ext4 /dev/nvme1n1 + sudo mkdir ${{ env.DOCKER_VOLUME }} + sudo mount /dev/nvme1n1 ${{ env.DOCKER_VOLUME }} + - name: Install + run: | + make install-integration-tests + - name: Run tests + run: | + export DOCKER_IMAGE=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }} + export HUGGING_FACE_HUB_TOKEN=${{ secrets.HUGGING_FACE_HUB_TOKEN }} + pytest -s -vv integration-tests + stop-runner: name: Stop self-hosted EC2 runner needs: - start-runner - build-and-push-image + - integration-tests runs-on: ubuntu-latest env: AWS_REGION: us-east-1 diff --git a/Makefile b/Makefile index 032a49de..29c318fa 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ install-server: cd server && make install +install-integration-tests: + cd integration-tests && pip install -r requirements.txt + install-router: cd router && cargo install --path . @@ -18,9 +21,15 @@ server-dev: router-dev: cd router && cargo run -- --port 8080 -integration-tests: install-router install-launcher +rust-tests: install-router install-launcher cargo test +integration-tests: install-integration-tests + pytest -s -vv -m "not private" integration-tests + +update-integration-tests: install-integration-tests + pytest -s -vv --snapshot-update integration-tests + python-server-tests: HF_HUB_ENABLE_HF_TRANSFER=1 pytest server/tests diff --git a/README.md b/README.md index 756d7e35..918ea5e2 100644 --- a/README.md +++ b/README.md @@ -253,5 +253,7 @@ make python-client-tests # or both server and client tests make python-tests # rust cargo tests +make rust-tests +# integration tests make integration-tests ``` diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py new file mode 100644 index 00000000..e9c51c37 --- /dev/null +++ b/integration-tests/conftest.py @@ -0,0 +1,146 @@ +import subprocess +import contextlib +import pytest +import asyncio +import os +import docker + +from docker.errors import NotFound +from typing import Optional, List +from syrupy.filters import props + +from text_generation import AsyncClient +from text_generation.types import Response + +DOCKER_IMAGE = os.getenv("DOCKER_IMAGE", None) +HUGGING_FACE_HUB_TOKEN = os.getenv("HUGGING_FACE_HUB_TOKEN", None) +DOCKER_VOLUME = os.getenv("DOCKER_VOLUME", "/data") + + +@pytest.fixture +def snapshot_test(snapshot): + return lambda value: value == snapshot(exclude=props("logprob")) + + +@pytest.fixture(scope="module") +def event_loop(): + loop = asyncio.get_event_loop() + yield loop + loop.close() + + +@pytest.fixture(scope="module") +def launcher(event_loop): + @contextlib.contextmanager + def local_launcher( + model_id: str, num_shard: Optional[int] = None, quantize: Optional[str] = None + ): + port = 9999 + master_port = 19999 + + shard_uds_path = f"/tmp/{model_id.replace('/', '--')}-server" + + args = [ + "text-generation-launcher", + "--model-id", + model_id, + "--port", + str(port), + "--master-port", + str(master_port), + "--shard-uds-path", + shard_uds_path, + ] + + if num_shard is not None: + args.extend(["--num-shard", str(num_shard)]) + if quantize: + args.append("--quantize") + + with subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) as process: + yield AsyncClient(f"http://localhost:{port}") + + process.terminate() + process.wait(60) + + launcher_output = process.stdout.read().decode("utf-8") + print(launcher_output) + + process.stdout.close() + process.stderr.close() + + @contextlib.contextmanager + def docker_launcher( + model_id: str, num_shard: Optional[int] = None, quantize: Optional[str] = None + ): + port = 9999 + + args = ["--model-id", model_id, "--env"] + + if num_shard is not None: + args.extend(["--num-shard", str(num_shard)]) + if quantize: + args.append("--quantize") + + client = docker.from_env() + + container_name = f"tgi-tests-{model_id.split('/')[-1]}-{num_shard}-{quantize}" + + try: + container = client.containers.get(container_name) + container.stop() + container.wait() + except NotFound: + pass + + gpu_count = num_shard if num_shard is not None else 1 + + env = {} + if HUGGING_FACE_HUB_TOKEN is not None: + env["HUGGING_FACE_HUB_TOKEN"] = HUGGING_FACE_HUB_TOKEN + + volumes = [] + if DOCKER_VOLUME: + volumes = [f"{DOCKER_VOLUME}:/data"] + + container = client.containers.run( + DOCKER_IMAGE, + command=args, + name=container_name, + environment=env, + auto_remove=True, + detach=True, + device_requests=[ + docker.types.DeviceRequest(count=gpu_count, capabilities=[["gpu"]]) + ], + volumes=volumes, + ports={"80/tcp": port}, + ) + + yield AsyncClient(f"http://localhost:{port}") + + container.stop() + + container_output = container.logs().decode("utf-8") + print(container_output) + + if DOCKER_IMAGE is not None: + return docker_launcher + return local_launcher + + +@pytest.fixture(scope="module") +def generate_load(): + async def generate_load_inner( + client: AsyncClient, prompt: str, max_new_tokens: int, n: int + ) -> List[Response]: + futures = [ + client.generate(prompt, max_new_tokens=max_new_tokens) for _ in range(n) + ] + + results = await asyncio.gather(*futures) + return [r.dict() for r in results] + + return generate_load_inner diff --git a/integration-tests/models/__snapshots__/test_bloom_560m.ambr b/integration-tests/models/__snapshots__/test_bloom_560m.ambr new file mode 100644 index 00000000..1067513d --- /dev/null +++ b/integration-tests/models/__snapshots__/test_bloom_560m.ambr @@ -0,0 +1,627 @@ +# serializer version: 1 +# name: test_bloom_560m + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': 0, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 159570, + 'special': False, + 'text': ' réch', + }), + dict({ + 'id': 810, + 'special': False, + 'text': 'au', + }), + dict({ + 'id': 12736, + 'special': False, + 'text': 'ffer', + }), + dict({ + 'id': 1742, + 'special': False, + 'text': ' au', + }), + dict({ + 'id': 6105, + 'special': False, + 'text': ' bain', + }), + dict({ + 'id': 88254, + 'special': False, + 'text': '-mar', + }), + dict({ + 'id': 641, + 'special': False, + 'text': 'ie', + }), + dict({ + 'id': 2940, + 'special': False, + 'text': ' avec', + }), + ]), + }), + 'generated_text': ' le faire réchauffer au bain-marie avec', + }) +# --- +# name: test_bloom_560m_all_params + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': 0, + 'tokens': list([ + dict({ + 'id': 408, + 'special': False, + 'text': ' que', + }), + dict({ + 'id': 20288, + 'special': False, + 'text': " l'on", + }), + dict({ + 'id': 22255, + 'special': False, + 'text': ' trouve', + }), + dict({ + 'id': 1622, + 'special': False, + 'text': ' une', + }), + dict({ + 'id': 187079, + 'special': False, + 'text': ' posture', + }), + dict({ + 'id': 501, + 'special': False, + 'text': ' par', + }), + dict({ + 'id': 8741, + 'special': False, + 'text': ' rapport', + }), + dict({ + 'id': 693, + 'special': False, + 'text': ' à', + }), + dict({ + 'id': 366, + 'special': False, + 'text': ' la', + }), + dict({ + 'id': 36503, + 'special': False, + 'text': ' pratique', + }), + ]), + }), + 'generated_text': "Pour déguster un ortolan, il faut tout d'abord que l'on trouve une posture par rapport à la pratique", + }) +# --- +# name: test_bloom_560m_load + list([ + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 1767, + 'special': False, + 'text': ' cu', + }), + dict({ + 'id': 1273, + 'special': False, + 'text': 'ire', + }), + dict({ + 'id': 1486, + 'special': False, + 'text': ' dans', + }), + dict({ + 'id': 283, + 'special': False, + 'text': ' de', + }), + dict({ + 'id': 40410, + 'special': False, + 'text': " l'eau", + }), + dict({ + 'id': 20226, + 'special': False, + 'text': ' bou', + }), + dict({ + 'id': 172483, + 'special': False, + 'text': 'illante', + }), + dict({ + 'id': 2805, + 'special': False, + 'text': ' sal', + }), + ]), + }), + 'generated_text': " le faire cuire dans de l'eau bouillante sal", + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 1767, + 'special': False, + 'text': ' cu', + }), + dict({ + 'id': 1273, + 'special': False, + 'text': 'ire', + }), + dict({ + 'id': 1486, + 'special': False, + 'text': ' dans', + }), + dict({ + 'id': 283, + 'special': False, + 'text': ' de', + }), + dict({ + 'id': 40410, + 'special': False, + 'text': " l'eau", + }), + dict({ + 'id': 20226, + 'special': False, + 'text': ' bou', + }), + dict({ + 'id': 172483, + 'special': False, + 'text': 'illante', + }), + dict({ + 'id': 2805, + 'special': False, + 'text': ' sal', + }), + ]), + }), + 'generated_text': " le faire cuire dans de l'eau bouillante sal", + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 1767, + 'special': False, + 'text': ' cu', + }), + dict({ + 'id': 1273, + 'special': False, + 'text': 'ire', + }), + dict({ + 'id': 1486, + 'special': False, + 'text': ' dans', + }), + dict({ + 'id': 283, + 'special': False, + 'text': ' de', + }), + dict({ + 'id': 40410, + 'special': False, + 'text': " l'eau", + }), + dict({ + 'id': 20226, + 'special': False, + 'text': ' bou', + }), + dict({ + 'id': 172483, + 'special': False, + 'text': 'illante', + }), + dict({ + 'id': 2805, + 'special': False, + 'text': ' sal', + }), + ]), + }), + 'generated_text': " le faire cuire dans de l'eau bouillante sal", + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 1767, + 'special': False, + 'text': ' cu', + }), + dict({ + 'id': 1273, + 'special': False, + 'text': 'ire', + }), + dict({ + 'id': 1486, + 'special': False, + 'text': ' dans', + }), + dict({ + 'id': 283, + 'special': False, + 'text': ' de', + }), + dict({ + 'id': 40410, + 'special': False, + 'text': " l'eau", + }), + dict({ + 'id': 20226, + 'special': False, + 'text': ' bou', + }), + dict({ + 'id': 172483, + 'special': False, + 'text': 'illante', + }), + dict({ + 'id': 2805, + 'special': False, + 'text': ' sal', + }), + ]), + }), + 'generated_text': " le faire cuire dans de l'eau bouillante sal", + }), + ]) +# --- diff --git a/integration-tests/models/__snapshots__/test_bloom_560m_sharded.ambr b/integration-tests/models/__snapshots__/test_bloom_560m_sharded.ambr new file mode 100644 index 00000000..667a0373 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_bloom_560m_sharded.ambr @@ -0,0 +1,542 @@ +# serializer version: 1 +# name: test_bloom_560m_sharded + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': 0, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 159570, + 'special': False, + 'text': ' réch', + }), + dict({ + 'id': 810, + 'special': False, + 'text': 'au', + }), + dict({ + 'id': 12736, + 'special': False, + 'text': 'ffer', + }), + dict({ + 'id': 1742, + 'special': False, + 'text': ' au', + }), + dict({ + 'id': 6105, + 'special': False, + 'text': ' bain', + }), + dict({ + 'id': 88254, + 'special': False, + 'text': '-mar', + }), + dict({ + 'id': 641, + 'special': False, + 'text': 'ie', + }), + dict({ + 'id': 2940, + 'special': False, + 'text': ' avec', + }), + ]), + }), + 'generated_text': ' le faire réchauffer au bain-marie avec', + }) +# --- +# name: test_bloom_560m_sharded_load + list([ + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 1767, + 'special': False, + 'text': ' cu', + }), + dict({ + 'id': 1273, + 'special': False, + 'text': 'ire', + }), + dict({ + 'id': 1486, + 'special': False, + 'text': ' dans', + }), + dict({ + 'id': 283, + 'special': False, + 'text': ' de', + }), + dict({ + 'id': 40410, + 'special': False, + 'text': " l'eau", + }), + dict({ + 'id': 20226, + 'special': False, + 'text': ' bou', + }), + dict({ + 'id': 172483, + 'special': False, + 'text': 'illante', + }), + dict({ + 'id': 2805, + 'special': False, + 'text': ' sal', + }), + ]), + }), + 'generated_text': " le faire cuire dans de l'eau bouillante sal", + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 1767, + 'special': False, + 'text': ' cu', + }), + dict({ + 'id': 1273, + 'special': False, + 'text': 'ire', + }), + dict({ + 'id': 1486, + 'special': False, + 'text': ' dans', + }), + dict({ + 'id': 283, + 'special': False, + 'text': ' de', + }), + dict({ + 'id': 40410, + 'special': False, + 'text': " l'eau", + }), + dict({ + 'id': 20226, + 'special': False, + 'text': ' bou', + }), + dict({ + 'id': 172483, + 'special': False, + 'text': 'illante', + }), + dict({ + 'id': 2805, + 'special': False, + 'text': ' sal', + }), + ]), + }), + 'generated_text': " le faire cuire dans de l'eau bouillante sal", + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 1767, + 'special': False, + 'text': ' cu', + }), + dict({ + 'id': 1273, + 'special': False, + 'text': 'ire', + }), + dict({ + 'id': 1486, + 'special': False, + 'text': ' dans', + }), + dict({ + 'id': 283, + 'special': False, + 'text': ' de', + }), + dict({ + 'id': 40410, + 'special': False, + 'text': " l'eau", + }), + dict({ + 'id': 20226, + 'special': False, + 'text': ' bou', + }), + dict({ + 'id': 172483, + 'special': False, + 'text': 'illante', + }), + dict({ + 'id': 2805, + 'special': False, + 'text': ' sal', + }), + ]), + }), + 'generated_text': " le faire cuire dans de l'eau bouillante sal", + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 17934, + 'text': 'Pour', + }), + dict({ + 'id': 49833, + 'text': ' dég', + }), + dict({ + 'id': 21543, + 'text': 'uster', + }), + dict({ + 'id': 447, + 'text': ' un', + }), + dict({ + 'id': 46341, + 'text': ' ort', + }), + dict({ + 'id': 35567, + 'text': 'olan', + }), + dict({ + 'id': 15, + 'text': ',', + }), + dict({ + 'id': 1669, + 'text': ' il', + }), + dict({ + 'id': 11580, + 'text': ' faut', + }), + dict({ + 'id': 3913, + 'text': ' tout', + }), + dict({ + 'id': 39261, + 'text': " d'abord", + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 578, + 'special': False, + 'text': ' le', + }), + dict({ + 'id': 5608, + 'special': False, + 'text': ' faire', + }), + dict({ + 'id': 1767, + 'special': False, + 'text': ' cu', + }), + dict({ + 'id': 1273, + 'special': False, + 'text': 'ire', + }), + dict({ + 'id': 1486, + 'special': False, + 'text': ' dans', + }), + dict({ + 'id': 283, + 'special': False, + 'text': ' de', + }), + dict({ + 'id': 40410, + 'special': False, + 'text': " l'eau", + }), + dict({ + 'id': 20226, + 'special': False, + 'text': ' bou', + }), + dict({ + 'id': 172483, + 'special': False, + 'text': 'illante', + }), + dict({ + 'id': 2805, + 'special': False, + 'text': ' sal', + }), + ]), + }), + 'generated_text': " le faire cuire dans de l'eau bouillante sal", + }), + ]) +# --- diff --git a/integration-tests/models/__snapshots__/test_flash_llama.ambr b/integration-tests/models/__snapshots__/test_flash_llama.ambr new file mode 100644 index 00000000..f4e3a4c1 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_llama.ambr @@ -0,0 +1,465 @@ +# serializer version: 1 +# name: test_flash_llama + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 1, + 'text': '', + }), + dict({ + 'id': 4321, + 'text': 'Test', + }), + dict({ + 'id': 2009, + 'text': 'request', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 363, + 'special': False, + 'text': ' for', + }), + dict({ + 'id': 847, + 'special': False, + 'text': ' /', + }), + dict({ + 'id': 2754, + 'special': False, + 'text': 'api', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29894, + 'special': False, + 'text': 'v', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 16418, + 'special': False, + 'text': 'projects', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + ]), + }), + 'generated_text': 'for /api/v1/projects/1', + }) +# --- +# name: test_flash_llama_all_params + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 1, + 'text': '', + }), + dict({ + 'id': 4321, + 'text': 'Test', + }), + dict({ + 'id': 2009, + 'text': 'request', + }), + ]), + 'seed': 0, + 'tokens': list([ + dict({ + 'id': 5229, + 'special': False, + 'text': ' failed', + }), + dict({ + 'id': 363, + 'special': False, + 'text': ' for', + }), + dict({ + 'id': 5641, + 'special': False, + 'text': ' IP', + }), + dict({ + 'id': 16428, + 'special': False, + 'text': ' Address', + }), + dict({ + 'id': 29901, + 'special': False, + 'text': ':', + }), + dict({ + 'id': 525, + 'special': False, + 'text': " '", + }), + dict({ + 'id': 8516, + 'special': False, + 'text': 'None', + }), + dict({ + 'id': 4286, + 'special': False, + 'text': "'.", + }), + dict({ + 'id': 13, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 294, + 'special': False, + 'text': 'as', + }), + ]), + }), + 'generated_text': ''' + Test requestfailed for IP Address: 'None'. + as + ''', + }) +# --- +# name: test_flash_llama_load + list([ + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 1, + 'text': '', + }), + dict({ + 'id': 4321, + 'text': 'Test', + }), + dict({ + 'id': 2009, + 'text': 'request', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 363, + 'special': False, + 'text': ' for', + }), + dict({ + 'id': 847, + 'special': False, + 'text': ' /', + }), + dict({ + 'id': 2754, + 'special': False, + 'text': 'api', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29894, + 'special': False, + 'text': 'v', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 16418, + 'special': False, + 'text': 'projects', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + ]), + }), + 'generated_text': 'for /api/v1/projects/1', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 1, + 'text': '', + }), + dict({ + 'id': 4321, + 'text': 'Test', + }), + dict({ + 'id': 2009, + 'text': 'request', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 363, + 'special': False, + 'text': ' for', + }), + dict({ + 'id': 847, + 'special': False, + 'text': ' /', + }), + dict({ + 'id': 2754, + 'special': False, + 'text': 'api', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29894, + 'special': False, + 'text': 'v', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 16418, + 'special': False, + 'text': 'projects', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + ]), + }), + 'generated_text': 'for /api/v1/projects/1', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 1, + 'text': '', + }), + dict({ + 'id': 4321, + 'text': 'Test', + }), + dict({ + 'id': 2009, + 'text': 'request', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 363, + 'special': False, + 'text': ' for', + }), + dict({ + 'id': 847, + 'special': False, + 'text': ' /', + }), + dict({ + 'id': 2754, + 'special': False, + 'text': 'api', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29894, + 'special': False, + 'text': 'v', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 16418, + 'special': False, + 'text': 'projects', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + ]), + }), + 'generated_text': 'for /api/v1/projects/1', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 1, + 'text': '', + }), + dict({ + 'id': 4321, + 'text': 'Test', + }), + dict({ + 'id': 2009, + 'text': 'request', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 363, + 'special': False, + 'text': ' for', + }), + dict({ + 'id': 847, + 'special': False, + 'text': ' /', + }), + dict({ + 'id': 2754, + 'special': False, + 'text': 'api', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29894, + 'special': False, + 'text': 'v', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 16418, + 'special': False, + 'text': 'projects', + }), + dict({ + 'id': 29914, + 'special': False, + 'text': '/', + }), + dict({ + 'id': 29896, + 'special': False, + 'text': '1', + }), + ]), + }), + 'generated_text': 'for /api/v1/projects/1', + }), + ]) +# --- diff --git a/integration-tests/models/__snapshots__/test_flash_neox.ambr b/integration-tests/models/__snapshots__/test_flash_neox.ambr new file mode 100644 index 00000000..4330db6b --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_neox.ambr @@ -0,0 +1,682 @@ +# serializer version: 1 +# name: test_flash_neox + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 50278, + 'text': '<|prompter|>', + }), + dict({ + 'id': 1276, + 'text': 'What', + }), + dict({ + 'id': 310, + 'text': ' is', + }), + dict({ + 'id': 247, + 'text': ' a', + }), + dict({ + 'id': 1167, + 'text': ' mem', + }), + dict({ + 'id': 70, + 'text': 'e', + }), + dict({ + 'id': 13, + 'text': ',', + }), + dict({ + 'id': 285, + 'text': ' and', + }), + dict({ + 'id': 752, + 'text': ' what', + }), + dict({ + 'id': 434, + 'text': "'s", + }), + dict({ + 'id': 253, + 'text': ' the', + }), + dict({ + 'id': 2892, + 'text': ' history', + }), + dict({ + 'id': 3212, + 'text': ' behind', + }), + dict({ + 'id': 436, + 'text': ' this', + }), + dict({ + 'id': 3159, + 'text': ' word', + }), + dict({ + 'id': 32, + 'text': '?', + }), + dict({ + 'id': 0, + 'text': '<|endoftext|>', + }), + dict({ + 'id': 50281, + 'text': '<|assistant|>', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 510, + 'special': False, + 'text': 'The', + }), + dict({ + 'id': 3159, + 'special': False, + 'text': ' word', + }), + dict({ + 'id': 346, + 'special': False, + 'text': ' "', + }), + dict({ + 'id': 6441, + 'special': False, + 'text': 'mem', + }), + dict({ + 'id': 70, + 'special': False, + 'text': 'e', + }), + dict({ + 'id': 3, + 'special': False, + 'text': '"', + }), + dict({ + 'id': 369, + 'special': False, + 'text': ' was', + }), + dict({ + 'id': 806, + 'special': False, + 'text': ' first', + }), + dict({ + 'id': 908, + 'special': False, + 'text': ' used', + }), + dict({ + 'id': 275, + 'special': False, + 'text': ' in', + }), + ]), + }), + 'generated_text': 'The word "meme" was first used in', + }) +# --- +# name: test_flash_neox_load + list([ + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 50278, + 'text': '<|prompter|>', + }), + dict({ + 'id': 1276, + 'text': 'What', + }), + dict({ + 'id': 310, + 'text': ' is', + }), + dict({ + 'id': 247, + 'text': ' a', + }), + dict({ + 'id': 1167, + 'text': ' mem', + }), + dict({ + 'id': 70, + 'text': 'e', + }), + dict({ + 'id': 13, + 'text': ',', + }), + dict({ + 'id': 285, + 'text': ' and', + }), + dict({ + 'id': 752, + 'text': ' what', + }), + dict({ + 'id': 434, + 'text': "'s", + }), + dict({ + 'id': 253, + 'text': ' the', + }), + dict({ + 'id': 2892, + 'text': ' history', + }), + dict({ + 'id': 3212, + 'text': ' behind', + }), + dict({ + 'id': 436, + 'text': ' this', + }), + dict({ + 'id': 3159, + 'text': ' word', + }), + dict({ + 'id': 32, + 'text': '?', + }), + dict({ + 'id': 0, + 'text': '<|endoftext|>', + }), + dict({ + 'id': 50281, + 'text': '<|assistant|>', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 510, + 'special': False, + 'text': 'The', + }), + dict({ + 'id': 3159, + 'special': False, + 'text': ' word', + }), + dict({ + 'id': 346, + 'special': False, + 'text': ' "', + }), + dict({ + 'id': 6441, + 'special': False, + 'text': 'mem', + }), + dict({ + 'id': 70, + 'special': False, + 'text': 'e', + }), + dict({ + 'id': 3, + 'special': False, + 'text': '"', + }), + dict({ + 'id': 369, + 'special': False, + 'text': ' was', + }), + dict({ + 'id': 806, + 'special': False, + 'text': ' first', + }), + dict({ + 'id': 908, + 'special': False, + 'text': ' used', + }), + dict({ + 'id': 275, + 'special': False, + 'text': ' in', + }), + ]), + }), + 'generated_text': 'The word "meme" was first used in', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 50278, + 'text': '<|prompter|>', + }), + dict({ + 'id': 1276, + 'text': 'What', + }), + dict({ + 'id': 310, + 'text': ' is', + }), + dict({ + 'id': 247, + 'text': ' a', + }), + dict({ + 'id': 1167, + 'text': ' mem', + }), + dict({ + 'id': 70, + 'text': 'e', + }), + dict({ + 'id': 13, + 'text': ',', + }), + dict({ + 'id': 285, + 'text': ' and', + }), + dict({ + 'id': 752, + 'text': ' what', + }), + dict({ + 'id': 434, + 'text': "'s", + }), + dict({ + 'id': 253, + 'text': ' the', + }), + dict({ + 'id': 2892, + 'text': ' history', + }), + dict({ + 'id': 3212, + 'text': ' behind', + }), + dict({ + 'id': 436, + 'text': ' this', + }), + dict({ + 'id': 3159, + 'text': ' word', + }), + dict({ + 'id': 32, + 'text': '?', + }), + dict({ + 'id': 0, + 'text': '<|endoftext|>', + }), + dict({ + 'id': 50281, + 'text': '<|assistant|>', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 510, + 'special': False, + 'text': 'The', + }), + dict({ + 'id': 3159, + 'special': False, + 'text': ' word', + }), + dict({ + 'id': 346, + 'special': False, + 'text': ' "', + }), + dict({ + 'id': 6441, + 'special': False, + 'text': 'mem', + }), + dict({ + 'id': 70, + 'special': False, + 'text': 'e', + }), + dict({ + 'id': 3, + 'special': False, + 'text': '"', + }), + dict({ + 'id': 369, + 'special': False, + 'text': ' was', + }), + dict({ + 'id': 806, + 'special': False, + 'text': ' first', + }), + dict({ + 'id': 908, + 'special': False, + 'text': ' used', + }), + dict({ + 'id': 275, + 'special': False, + 'text': ' in', + }), + ]), + }), + 'generated_text': 'The word "meme" was first used in', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 50278, + 'text': '<|prompter|>', + }), + dict({ + 'id': 1276, + 'text': 'What', + }), + dict({ + 'id': 310, + 'text': ' is', + }), + dict({ + 'id': 247, + 'text': ' a', + }), + dict({ + 'id': 1167, + 'text': ' mem', + }), + dict({ + 'id': 70, + 'text': 'e', + }), + dict({ + 'id': 13, + 'text': ',', + }), + dict({ + 'id': 285, + 'text': ' and', + }), + dict({ + 'id': 752, + 'text': ' what', + }), + dict({ + 'id': 434, + 'text': "'s", + }), + dict({ + 'id': 253, + 'text': ' the', + }), + dict({ + 'id': 2892, + 'text': ' history', + }), + dict({ + 'id': 3212, + 'text': ' behind', + }), + dict({ + 'id': 436, + 'text': ' this', + }), + dict({ + 'id': 3159, + 'text': ' word', + }), + dict({ + 'id': 32, + 'text': '?', + }), + dict({ + 'id': 0, + 'text': '<|endoftext|>', + }), + dict({ + 'id': 50281, + 'text': '<|assistant|>', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 510, + 'special': False, + 'text': 'The', + }), + dict({ + 'id': 3159, + 'special': False, + 'text': ' word', + }), + dict({ + 'id': 346, + 'special': False, + 'text': ' "', + }), + dict({ + 'id': 6441, + 'special': False, + 'text': 'mem', + }), + dict({ + 'id': 70, + 'special': False, + 'text': 'e', + }), + dict({ + 'id': 3, + 'special': False, + 'text': '"', + }), + dict({ + 'id': 369, + 'special': False, + 'text': ' was', + }), + dict({ + 'id': 806, + 'special': False, + 'text': ' first', + }), + dict({ + 'id': 908, + 'special': False, + 'text': ' used', + }), + dict({ + 'id': 275, + 'special': False, + 'text': ' in', + }), + ]), + }), + 'generated_text': 'The word "meme" was first used in', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 50278, + 'text': '<|prompter|>', + }), + dict({ + 'id': 1276, + 'text': 'What', + }), + dict({ + 'id': 310, + 'text': ' is', + }), + dict({ + 'id': 247, + 'text': ' a', + }), + dict({ + 'id': 1167, + 'text': ' mem', + }), + dict({ + 'id': 70, + 'text': 'e', + }), + dict({ + 'id': 13, + 'text': ',', + }), + dict({ + 'id': 285, + 'text': ' and', + }), + dict({ + 'id': 752, + 'text': ' what', + }), + dict({ + 'id': 434, + 'text': "'s", + }), + dict({ + 'id': 253, + 'text': ' the', + }), + dict({ + 'id': 2892, + 'text': ' history', + }), + dict({ + 'id': 3212, + 'text': ' behind', + }), + dict({ + 'id': 436, + 'text': ' this', + }), + dict({ + 'id': 3159, + 'text': ' word', + }), + dict({ + 'id': 32, + 'text': '?', + }), + dict({ + 'id': 0, + 'text': '<|endoftext|>', + }), + dict({ + 'id': 50281, + 'text': '<|assistant|>', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 510, + 'special': False, + 'text': 'The', + }), + dict({ + 'id': 3159, + 'special': False, + 'text': ' word', + }), + dict({ + 'id': 346, + 'special': False, + 'text': ' "', + }), + dict({ + 'id': 6441, + 'special': False, + 'text': 'mem', + }), + dict({ + 'id': 70, + 'special': False, + 'text': 'e', + }), + dict({ + 'id': 3, + 'special': False, + 'text': '"', + }), + dict({ + 'id': 369, + 'special': False, + 'text': ' was', + }), + dict({ + 'id': 806, + 'special': False, + 'text': ' first', + }), + dict({ + 'id': 908, + 'special': False, + 'text': ' used', + }), + dict({ + 'id': 275, + 'special': False, + 'text': ' in', + }), + ]), + }), + 'generated_text': 'The word "meme" was first used in', + }), + ]) +# --- diff --git a/integration-tests/models/__snapshots__/test_flash_santacoder.ambr b/integration-tests/models/__snapshots__/test_flash_santacoder.ambr new file mode 100644 index 00000000..030820cb --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_santacoder.ambr @@ -0,0 +1,472 @@ +# serializer version: 1 +# name: test_flash_santacoder + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 563, + 'text': 'def', + }), + dict({ + 'id': 942, + 'text': ' print', + }), + dict({ + 'id': 62, + 'text': '_', + }), + dict({ + 'id': 7196, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 1241, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 258, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 942, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 372, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 7371, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 9956, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 8657, + 'special': False, + 'text': '!")', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1018, + 'special': False, + 'text': 'print', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World!") + + print + ''', + }) +# --- +# name: test_flash_santacoder_load + list([ + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 563, + 'text': 'def', + }), + dict({ + 'id': 942, + 'text': ' print', + }), + dict({ + 'id': 62, + 'text': '_', + }), + dict({ + 'id': 7196, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 1241, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 258, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 942, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 372, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 7371, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 9956, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 8657, + 'special': False, + 'text': '!")', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1018, + 'special': False, + 'text': 'print', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World!") + + print + ''', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 563, + 'text': 'def', + }), + dict({ + 'id': 942, + 'text': ' print', + }), + dict({ + 'id': 62, + 'text': '_', + }), + dict({ + 'id': 7196, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 1241, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 258, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 942, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 372, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 7371, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 9956, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 8657, + 'special': False, + 'text': '!")', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1018, + 'special': False, + 'text': 'print', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World!") + + print + ''', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 563, + 'text': 'def', + }), + dict({ + 'id': 942, + 'text': ' print', + }), + dict({ + 'id': 62, + 'text': '_', + }), + dict({ + 'id': 7196, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 1241, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 258, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 942, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 372, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 7371, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 9956, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 8657, + 'special': False, + 'text': '!")', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1018, + 'special': False, + 'text': 'print', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World!") + + print + ''', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 563, + 'text': 'def', + }), + dict({ + 'id': 942, + 'text': ' print', + }), + dict({ + 'id': 62, + 'text': '_', + }), + dict({ + 'id': 7196, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 1241, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 258, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 942, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 372, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 7371, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 9956, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 8657, + 'special': False, + 'text': '!")', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 185, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1018, + 'special': False, + 'text': 'print', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World!") + + print + ''', + }), + ]) +# --- diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder.ambr b/integration-tests/models/__snapshots__/test_flash_starcoder.ambr new file mode 100644 index 00000000..e0f4b568 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_starcoder.ambr @@ -0,0 +1,573 @@ +# serializer version: 1 +# name: test_flash_starcoder + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 589, + 'text': 'def', + }), + dict({ + 'id': 1459, + 'text': ' print', + }), + dict({ + 'id': 81, + 'text': '_', + }), + dict({ + 'id': 7656, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 2262, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 284, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1459, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 440, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 8279, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 10896, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 657, + 'special': False, + 'text': '")', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 589, + 'special': False, + 'text': 'def', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World") + + def + ''', + }) +# --- +# name: test_flash_starcoder_default_params + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 12, + 'prefill': list([ + dict({ + 'id': 589, + 'text': 'def', + }), + dict({ + 'id': 1459, + 'text': ' print', + }), + dict({ + 'id': 81, + 'text': '_', + }), + dict({ + 'id': 7656, + 'text': 'hello', + }), + ]), + 'seed': 0, + 'tokens': list([ + dict({ + 'id': 2262, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 284, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 5741, + 'special': False, + 'text': ' logging', + }), + dict({ + 'id': 32, + 'special': False, + 'text': '.', + }), + dict({ + 'id': 1338, + 'special': False, + 'text': 'info', + }), + dict({ + 'id': 463, + 'special': False, + 'text': "('", + }), + dict({ + 'id': 8279, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 30, + 'special': False, + 'text': ',', + }), + dict({ + 'id': 10896, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 683, + 'special': False, + 'text': "')", + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 0, + 'special': True, + 'text': '<|endoftext|>', + }), + ]), + }), + 'generated_text': ''' + (): + logging.info('Hello, World') + <|endoftext|> + ''', + }) +# --- +# name: test_flash_starcoder_load + list([ + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 589, + 'text': 'def', + }), + dict({ + 'id': 1459, + 'text': ' print', + }), + dict({ + 'id': 81, + 'text': '_', + }), + dict({ + 'id': 7656, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 2262, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 284, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1459, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 440, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 8279, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 10896, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 657, + 'special': False, + 'text': '")', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 589, + 'special': False, + 'text': 'def', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World") + + def + ''', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 589, + 'text': 'def', + }), + dict({ + 'id': 1459, + 'text': ' print', + }), + dict({ + 'id': 81, + 'text': '_', + }), + dict({ + 'id': 7656, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 2262, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 284, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1459, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 440, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 8279, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 10896, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 657, + 'special': False, + 'text': '")', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 589, + 'special': False, + 'text': 'def', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World") + + def + ''', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 589, + 'text': 'def', + }), + dict({ + 'id': 1459, + 'text': ' print', + }), + dict({ + 'id': 81, + 'text': '_', + }), + dict({ + 'id': 7656, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 2262, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 284, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1459, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 440, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 8279, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 10896, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 657, + 'special': False, + 'text': '")', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 589, + 'special': False, + 'text': 'def', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World") + + def + ''', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 589, + 'text': 'def', + }), + dict({ + 'id': 1459, + 'text': ' print', + }), + dict({ + 'id': 81, + 'text': '_', + }), + dict({ + 'id': 7656, + 'text': 'hello', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 2262, + 'special': False, + 'text': '():', + }), + dict({ + 'id': 284, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 1459, + 'special': False, + 'text': ' print', + }), + dict({ + 'id': 440, + 'special': False, + 'text': '("', + }), + dict({ + 'id': 8279, + 'special': False, + 'text': 'Hello', + }), + dict({ + 'id': 10896, + 'special': False, + 'text': ' World', + }), + dict({ + 'id': 657, + 'special': False, + 'text': '")', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 203, + 'special': False, + 'text': ''' + + + ''', + }), + dict({ + 'id': 589, + 'special': False, + 'text': 'def', + }), + ]), + }), + 'generated_text': ''' + (): + print("Hello World") + + def + ''', + }), + ]) +# --- diff --git a/integration-tests/models/__snapshots__/test_mt0_base.ambr b/integration-tests/models/__snapshots__/test_mt0_base.ambr new file mode 100644 index 00000000..d7c6eaf6 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_mt0_base.ambr @@ -0,0 +1,306 @@ +# serializer version: 1 +# name: test_mt0_base + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 5, + 'prefill': list([ + dict({ + 'id': 0, + 'text': '', + }), + ]), + 'seed': 0, + 'tokens': list([ + dict({ + 'id': 926, + 'special': False, + 'text': 'To', + }), + dict({ + 'id': 18295, + 'special': False, + 'text': ' sell', + }), + dict({ + 'id': 7868, + 'special': False, + 'text': ' things', + }), + dict({ + 'id': 260, + 'special': False, + 'text': '.', + }), + dict({ + 'id': 1, + 'special': True, + 'text': '', + }), + ]), + }), + 'generated_text': 'To sell things.', + }) +# --- +# name: test_mt0_base_all_params + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 10, + 'prefill': list([ + dict({ + 'id': 0, + 'text': '', + }), + ]), + 'seed': 0, + 'tokens': list([ + dict({ + 'id': 16017, + 'special': False, + 'text': 'blue', + }), + dict({ + 'id': 20495, + 'special': False, + 'text': ' sky', + }), + dict({ + 'id': 259, + 'special': False, + 'text': ' ', + }), + dict({ + 'id': 15484, + 'special': False, + 'text': 'appear', + }), + dict({ + 'id': 345, + 'special': False, + 'text': 'ed', + }), + dict({ + 'id': 288, + 'special': False, + 'text': ' to', + }), + dict({ + 'id': 35622, + 'special': False, + 'text': ' cloud', + }), + dict({ + 'id': 263, + 'special': False, + 'text': 's', + }), + dict({ + 'id': 14701, + 'special': False, + 'text': ' above', + }), + dict({ + 'id': 751, + 'special': False, + 'text': ' all', + }), + ]), + }), + 'generated_text': 'Why is the sky blue?blue sky appeared to clouds above all', + }) +# --- +# name: test_mt0_base_load + list([ + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 6, + 'prefill': list([ + dict({ + 'id': 0, + 'text': '', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 259, + 'special': False, + 'text': '', + }), + dict({ + 'id': 39261, + 'special': False, + 'text': 'Because', + }), + dict({ + 'id': 609, + 'special': False, + 'text': ' it', + }), + dict({ + 'id': 339, + 'special': False, + 'text': ' is', + }), + dict({ + 'id': 16017, + 'special': False, + 'text': ' blue', + }), + dict({ + 'id': 1, + 'special': True, + 'text': '', + }), + ]), + }), + 'generated_text': 'Because it is blue', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 6, + 'prefill': list([ + dict({ + 'id': 0, + 'text': '', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 259, + 'special': False, + 'text': '', + }), + dict({ + 'id': 39261, + 'special': False, + 'text': 'Because', + }), + dict({ + 'id': 609, + 'special': False, + 'text': ' it', + }), + dict({ + 'id': 339, + 'special': False, + 'text': ' is', + }), + dict({ + 'id': 16017, + 'special': False, + 'text': ' blue', + }), + dict({ + 'id': 1, + 'special': True, + 'text': '', + }), + ]), + }), + 'generated_text': 'Because it is blue', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 6, + 'prefill': list([ + dict({ + 'id': 0, + 'text': '', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 259, + 'special': False, + 'text': '', + }), + dict({ + 'id': 39261, + 'special': False, + 'text': 'Because', + }), + dict({ + 'id': 609, + 'special': False, + 'text': ' it', + }), + dict({ + 'id': 339, + 'special': False, + 'text': ' is', + }), + dict({ + 'id': 16017, + 'special': False, + 'text': ' blue', + }), + dict({ + 'id': 1, + 'special': True, + 'text': '', + }), + ]), + }), + 'generated_text': 'Because it is blue', + }), + dict({ + 'details': dict({ + 'best_of_sequences': None, + 'finish_reason': , + 'generated_tokens': 6, + 'prefill': list([ + dict({ + 'id': 0, + 'text': '', + }), + ]), + 'seed': None, + 'tokens': list([ + dict({ + 'id': 259, + 'special': False, + 'text': '', + }), + dict({ + 'id': 39261, + 'special': False, + 'text': 'Because', + }), + dict({ + 'id': 609, + 'special': False, + 'text': ' it', + }), + dict({ + 'id': 339, + 'special': False, + 'text': ' is', + }), + dict({ + 'id': 16017, + 'special': False, + 'text': ' blue', + }), + dict({ + 'id': 1, + 'special': True, + 'text': '', + }), + ]), + }), + 'generated_text': 'Because it is blue', + }), + ]) +# --- diff --git a/integration-tests/models/test_bloom_560m.py b/integration-tests/models/test_bloom_560m.py new file mode 100644 index 00000000..e13606f7 --- /dev/null +++ b/integration-tests/models/test_bloom_560m.py @@ -0,0 +1,63 @@ +import pytest + +from utils import health_check + + +@pytest.fixture(scope="module") +def bloom_560(launcher): + with launcher("bigscience/bloom-560m") as client: + yield client + + +@pytest.mark.asyncio +async def test_bloom_560m(bloom_560, snapshot_test): + await health_check(bloom_560, 60) + + response = await bloom_560.generate( + "Pour déguster un ortolan, il faut tout d'abord", + max_new_tokens=10, + top_p=0.9, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert snapshot_test(response) + + +@pytest.mark.asyncio +async def test_bloom_560m_all_params(bloom_560, snapshot_test): + await health_check(bloom_560, 60) + + response = await bloom_560.generate( + "Pour déguster un ortolan, il faut tout d'abord", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + stop_sequences=["test"], + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert snapshot_test(response) + + +@pytest.mark.asyncio +async def test_bloom_560m_load(bloom_560, generate_load, snapshot_test): + await health_check(bloom_560, 60) + + responses = await generate_load( + bloom_560, + "Pour déguster un ortolan, il faut tout d'abord", + max_new_tokens=10, + n=4, + ) + + assert len(responses) == 4 + + assert snapshot_test(responses) diff --git a/integration-tests/models/test_bloom_560m_sharded.py b/integration-tests/models/test_bloom_560m_sharded.py new file mode 100644 index 00000000..bfb70253 --- /dev/null +++ b/integration-tests/models/test_bloom_560m_sharded.py @@ -0,0 +1,42 @@ +import pytest + +from utils import health_check + + +@pytest.fixture(scope="module") +def bloom_560m_sharded(launcher): + with launcher("bigscience/bloom-560m", num_shard=2) as client: + yield client + + +@pytest.mark.asyncio +async def test_bloom_560m_sharded(bloom_560m_sharded, snapshot_test): + await health_check(bloom_560m_sharded, 60) + + response = await bloom_560m_sharded.generate( + "Pour déguster un ortolan, il faut tout d'abord", + max_new_tokens=10, + top_p=0.9, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert snapshot_test(response) + + +@pytest.mark.asyncio +async def test_bloom_560m_sharded_load( + bloom_560m_sharded, generate_load, snapshot_test +): + await health_check(bloom_560m_sharded, 60) + + responses = await generate_load( + bloom_560m_sharded, + "Pour déguster un ortolan, il faut tout d'abord", + max_new_tokens=10, + n=4, + ) + + assert len(responses) == 4 + + assert snapshot_test(responses) diff --git a/integration-tests/models/test_flash_llama.py b/integration-tests/models/test_flash_llama.py new file mode 100644 index 00000000..4d1f2bcf --- /dev/null +++ b/integration-tests/models/test_flash_llama.py @@ -0,0 +1,56 @@ +import pytest + +from utils import health_check + + +@pytest.fixture(scope="module") +def flash_llama(launcher): + with launcher("huggingface/llama-7b", num_shard=2) as client: + yield client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama(flash_llama, snapshot_test): + await health_check(flash_llama, 120) + + response = await flash_llama.generate("Test request", max_new_tokens=10) + + assert response.details.generated_tokens == 10 + assert snapshot_test(response) + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_all_params(flash_llama, snapshot_test): + await health_check(flash_llama, 120) + + response = await flash_llama.generate( + "Test request", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + stop_sequences=["test"], + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert snapshot_test(response) + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_load(flash_llama, generate_load, snapshot_test): + await health_check(flash_llama, 120) + + responses = await generate_load(flash_llama, "Test request", max_new_tokens=10, n=4) + + assert len(responses) == 4 + + assert snapshot_test(responses) diff --git a/integration-tests/models/test_flash_neox.py b/integration-tests/models/test_flash_neox.py new file mode 100644 index 00000000..8c981028 --- /dev/null +++ b/integration-tests/models/test_flash_neox.py @@ -0,0 +1,38 @@ +import pytest + +from utils import health_check + + +@pytest.fixture(scope="module") +def flash_neox(launcher): + with launcher("OpenAssistant/oasst-sft-1-pythia-12b", num_shard=2) as client: + yield client + + +@pytest.mark.asyncio +async def test_flash_neox(flash_neox, snapshot_test): + await health_check(flash_neox, 240) + + response = await flash_neox.generate( + "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", + max_new_tokens=10, + ) + + assert response.details.generated_tokens == 10 + assert snapshot_test(response) + + +@pytest.mark.asyncio +async def test_flash_neox_load(flash_neox, generate_load, snapshot_test): + await health_check(flash_neox, 240) + + responses = await generate_load( + flash_neox, + "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", + max_new_tokens=10, + n=4, + ) + + assert len(responses) == 4 + + assert snapshot_test(responses) diff --git a/integration-tests/models/test_flash_santacoder.py b/integration-tests/models/test_flash_santacoder.py new file mode 100644 index 00000000..64a59d78 --- /dev/null +++ b/integration-tests/models/test_flash_santacoder.py @@ -0,0 +1,32 @@ +import pytest + +from utils import health_check + + +@pytest.fixture(scope="module") +def flash_santacoder(launcher): + with launcher("bigcode/santacoder") as client: + yield client + + +@pytest.mark.asyncio +async def test_flash_santacoder(flash_santacoder, snapshot_test): + await health_check(flash_santacoder, 60) + + response = await flash_santacoder.generate("def print_hello", max_new_tokens=10) + + assert response.details.generated_tokens == 10 + assert snapshot_test(response) + + +@pytest.mark.asyncio +async def test_flash_santacoder_load(flash_santacoder, generate_load, snapshot_test): + await health_check(flash_santacoder, 60) + + responses = await generate_load( + flash_santacoder, "def print_hello", max_new_tokens=10, n=4 + ) + + assert len(responses) == 4 + + assert snapshot_test(responses) diff --git a/integration-tests/models/test_flash_starcoder.py b/integration-tests/models/test_flash_starcoder.py new file mode 100644 index 00000000..d43e92dc --- /dev/null +++ b/integration-tests/models/test_flash_starcoder.py @@ -0,0 +1,47 @@ +import pytest + +from utils import health_check + + +@pytest.fixture(scope="module") +def flash_starcoder(launcher): + with launcher("bigcode/starcoder", num_shard=2) as client: + yield client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_starcoder(flash_starcoder, snapshot_test): + await health_check(flash_starcoder, 240) + + response = await flash_starcoder.generate("def print_hello", max_new_tokens=10) + + assert response.details.generated_tokens == 10 + assert snapshot_test(response) + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_starcoder_default_params(flash_starcoder, snapshot_test): + await health_check(flash_starcoder, 240) + + response = await flash_starcoder.generate( + "def print_hello", max_new_tokens=60, temperature=0.2, top_p=0.95, seed=0 + ) + + assert response.details.generated_tokens == 12 + assert snapshot_test(response) + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_starcoder_load(flash_starcoder, generate_load, snapshot_test): + await health_check(flash_starcoder, 240) + + responses = await generate_load( + flash_starcoder, "def print_hello", max_new_tokens=10, n=4 + ) + + assert len(responses) == 4 + + assert snapshot_test(responses) diff --git a/integration-tests/models/test_mt0_base.py b/integration-tests/models/test_mt0_base.py new file mode 100644 index 00000000..7310a30f --- /dev/null +++ b/integration-tests/models/test_mt0_base.py @@ -0,0 +1,63 @@ +import pytest + +from utils import health_check + + +@pytest.fixture(scope="module") +def mt0_base(launcher): + with launcher("bigscience/mt0-base") as client: + yield client + + +@pytest.mark.asyncio +async def test_mt0_base(mt0_base, snapshot_test): + await health_check(mt0_base, 60) + + response = await mt0_base.generate( + "Why is the sky blue?", + max_new_tokens=10, + top_p=0.9, + seed=0, + ) + + assert response.details.generated_tokens == 5 + assert snapshot_test(response) + + +@pytest.mark.asyncio +async def test_mt0_base_all_params(mt0_base, snapshot_test): + await health_check(mt0_base, 60) + + response = await mt0_base.generate( + "Why is the sky blue?", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + stop_sequences=["test"], + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert snapshot_test(response) + + +@pytest.mark.asyncio +async def test_mt0_base_load(mt0_base, generate_load, snapshot_test): + await health_check(mt0_base, 60) + + responses = await generate_load( + mt0_base, + "Why is the sky blue?", + max_new_tokens=10, + n=4, + ) + + assert len(responses) == 4 + + assert snapshot_test(responses) diff --git a/integration-tests/models/utils.py b/integration-tests/models/utils.py new file mode 100644 index 00000000..c47e4871 --- /dev/null +++ b/integration-tests/models/utils.py @@ -0,0 +1,15 @@ +import time + +from aiohttp import ClientConnectorError, ClientOSError, ServerDisconnectedError +from text_generation import AsyncClient + + +async def health_check(client: AsyncClient, timeout: int = 60): + assert timeout > 0 + for _ in range(timeout): + try: + await client.generate("test") + return + except (ClientConnectorError, ClientOSError, ServerDisconnectedError) as e: + time.sleep(1) + raise RuntimeError("Health check failed") diff --git a/integration-tests/requirements.txt b/integration-tests/requirements.txt new file mode 100644 index 00000000..9ecbb2ee --- /dev/null +++ b/integration-tests/requirements.txt @@ -0,0 +1,5 @@ +syrupy +text-generation==0.5.1 +pytest +pytest-asyncio==0.17.2 +docker \ No newline at end of file diff --git a/launcher/tests/bloom_560m.json b/launcher/tests/bloom_560m.json deleted file mode 100644 index 96f89f6b..00000000 --- a/launcher/tests/bloom_560m.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "generated_text": ".get(\"action\");\n if (action == null) {\n throw new RuntimeException", - "details": { - "finish_reason": "length", - "generated_tokens": 20, - "seed": null, - "prefill": [ - { - "id": 10264, - "text": "Test", - "logprob": null - }, - { - "id": 8821, - "text": " request", - "logprob": -11.894989 - } - ], - "tokens": [ - { - "id": 17, - "text": ".", - "logprob": -1.8267672, - "special": false - }, - { - "id": 1587, - "text": "get", - "logprob": -2.4674969, - "special": false - }, - { - "id": 11, - "text": "(", - "logprob": -1.906001, - "special": false - }, - { - "id": 5, - "text": "\"", - "logprob": -1.2279545, - "special": false - }, - { - "id": 4899, - "text": "action", - "logprob": -4.170299, - "special": false - }, - { - "id": 5, - "text": "\"", - "logprob": -0.32478866, - "special": false - }, - { - "id": 12, - "text": ")", - "logprob": -1.0773665, - "special": false - }, - { - "id": 30, - "text": ";", - "logprob": -0.27640742, - "special": false - }, - { - "id": 837, - "text": "\n ", - "logprob": -1.6970354, - "special": false - }, - { - "id": 1320, - "text": " if", - "logprob": -1.4495516, - "special": false - }, - { - "id": 375, - "text": " (", - "logprob": -0.23609057, - "special": false - }, - { - "id": 4899, - "text": "action", - "logprob": -1.1916996, - "special": false - }, - { - "id": 3535, - "text": " ==", - "logprob": -0.8918753, - "special": false - }, - { - "id": 5109, - "text": " null", - "logprob": -0.3933342, - "special": false - }, - { - "id": 12, - "text": ")", - "logprob": -0.43212673, - "special": false - }, - { - "id": 731, - "text": " {", - "logprob": -0.17702064, - "special": false - }, - { - "id": 1260, - "text": "\n ", - "logprob": -0.07027565, - "special": false - }, - { - "id": 10519, - "text": " throw", - "logprob": -1.3915029, - "special": false - }, - { - "id": 2084, - "text": " new", - "logprob": -0.04201372, - "special": false - }, - { - "id": 150858, - "text": " RuntimeException", - "logprob": -1.7329919, - "special": false - } - ] - } -} \ No newline at end of file diff --git a/launcher/tests/integration_tests.rs b/launcher/tests/integration_tests.rs deleted file mode 100644 index 0d2b6c74..00000000 --- a/launcher/tests/integration_tests.rs +++ /dev/null @@ -1,172 +0,0 @@ -use float_eq::assert_float_eq; -use serde::Deserialize; -use serde_json::Value; -use std::fs::File; -use std::io::{BufRead, BufReader}; -use std::path::PathBuf; -use std::thread; -use std::thread::sleep; -use std::time::Duration; -use subprocess::{Popen, PopenConfig, Redirection}; - -#[derive(Deserialize)] -pub struct Token { - id: u32, - text: String, - logprob: Option, - special: bool, -} - -#[derive(Deserialize)] -struct Details { - finish_reason: String, - generated_tokens: u32, - tokens: Vec, -} - -#[derive(Deserialize)] -struct GeneratedText { - generated_text: String, - details: Details, -} - -fn start_launcher(model_id: String, num_shard: usize, port: usize, master_port: usize) -> Popen { - let argv = vec![ - "text-generation-launcher".to_string(), - "--model-id".to_string(), - model_id.clone(), - "--num-shard".to_string(), - num_shard.to_string(), - "--port".to_string(), - port.to_string(), - "--master-port".to_string(), - master_port.to_string(), - "--shard-uds-path".to_string(), - format!("/tmp/test-{}-{}-{}", num_shard, port, master_port), - ]; - - let mut launcher = Popen::create( - &argv, - PopenConfig { - stdout: Redirection::Pipe, - stderr: Redirection::Merge, - ..Default::default() - }, - ) - .expect("Could not start launcher"); - - // Redirect STDOUT and STDERR to the console - // (STDERR is merged into STDOUT) - let launcher_stdout = launcher.stdout.take().unwrap(); - - thread::spawn(move || { - let stdout = BufReader::new(launcher_stdout); - for line in stdout.lines() { - println!("{}", line.unwrap()); - } - }); - - for _ in 0..60 { - let health = reqwest::blocking::get(format!("http://localhost:{}/health", port)); - if health.is_ok() { - return launcher; - } - sleep(Duration::from_secs(2)); - } - - launcher.terminate().unwrap(); - launcher.wait().unwrap(); - panic!("failed to launch {}", model_id) -} - -fn test_model( - model_id: String, - num_shard: usize, - port: usize, - master_port: usize, -) -> GeneratedText { - let mut launcher = start_launcher(model_id, num_shard, port, master_port); - - let data = r#" - { - "inputs": "Test request", - "parameters": { - "details": true - } - }"#; - let req: Value = serde_json::from_str(data).unwrap(); - - let client = reqwest::blocking::Client::new(); - let res = client - .post(format!("http://localhost:{}/generate", port)) - .json(&req) - .send(); - - launcher.terminate().unwrap(); - launcher.wait().unwrap(); - - let result: GeneratedText = res.unwrap().json().unwrap(); - result -} - -fn read_json(name: &str) -> GeneratedText { - let mut d = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - d.push("tests/"); - d.push(name); - - let file = File::open(d).unwrap(); - let reader = BufReader::new(file); - - let result: GeneratedText = serde_json::from_reader(reader).unwrap(); - result -} - -fn compare_results(result: GeneratedText, expected: GeneratedText) { - assert_eq!(result.generated_text, expected.generated_text); - assert_eq!(result.details.finish_reason, expected.details.finish_reason); - assert_eq!( - result.details.generated_tokens, - expected.details.generated_tokens - ); - - for (token, expected_token) in result - .details - .tokens - .into_iter() - .zip(expected.details.tokens.into_iter()) - { - assert_eq!(token.id, expected_token.id); - assert_eq!(token.text, expected_token.text); - assert_eq!(token.special, expected_token.special); - if let Some(logprob) = token.logprob { - let expected_logprob = expected_token.logprob.unwrap(); - assert_float_eq!(logprob, expected_logprob, abs <= 0.001); - } else { - assert_eq!(token.logprob, expected_token.logprob); - } - } -} - -#[test] -fn test_bloom_560m() { - let expected = read_json("bloom_560m.json"); - - let result = test_model("bigscience/bloom-560m".to_string(), 1, 3000, 29500); - compare_results(result, expected); -} - -#[test] -fn test_bloom_560m_distributed() { - let expected = read_json("bloom_560m.json"); - - let result = test_model("bigscience/bloom-560m".to_string(), 2, 3001, 29501); - compare_results(result, expected); -} - -#[test] -fn test_mt0_base() { - let expected = read_json("mt0_base.json"); - - let result = test_model("bigscience/mt0-base".to_string(), 1, 3002, 29502); - compare_results(result, expected); -} diff --git a/launcher/tests/mt0_base.json b/launcher/tests/mt0_base.json deleted file mode 100644 index f5be63f9..00000000 --- a/launcher/tests/mt0_base.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "generated_text": "\"\"\"Test the contents of the contents of the contents. \"\"\" test_test", - "details": { - "finish_reason": "length", - "generated_tokens": 20, - "seed": null, - "prefill": [ - { - "id": 0, - "text": "", - "logprob": null - } - ], - "tokens": [ - { - "id": 259, - "text": "", - "logprob": -1.3656927, - "special": false - }, - { - "id": 215100, - "text": "\"\"\"", - "logprob": -2.6551573, - "special": false - }, - { - "id": 46138, - "text": "Test", - "logprob": -1.8059857, - "special": false - }, - { - "id": 287, - "text": " the", - "logprob": -1.2102449, - "special": false - }, - { - "id": 259, - "text": " ", - "logprob": -1.6057279, - "special": false - }, - { - "id": 49076, - "text": "contents", - "logprob": -3.6060903, - "special": false - }, - { - "id": 304, - "text": " of", - "logprob": -0.5270343, - "special": false - }, - { - "id": 287, - "text": " the", - "logprob": -0.62522805, - "special": false - }, - { - "id": 259, - "text": " ", - "logprob": -1.4069618, - "special": false - }, - { - "id": 49076, - "text": "contents", - "logprob": -2.621994, - "special": false - }, - { - "id": 304, - "text": " of", - "logprob": -1.3172221, - "special": false - }, - { - "id": 287, - "text": " the", - "logprob": -0.3501925, - "special": false - }, - { - "id": 259, - "text": " ", - "logprob": -0.7219573, - "special": false - }, - { - "id": 49076, - "text": "contents", - "logprob": -1.0494149, - "special": false - }, - { - "id": 260, - "text": ".", - "logprob": -1.0803378, - "special": false - }, - { - "id": 259, - "text": " ", - "logprob": -0.32933083, - "special": false - }, - { - "id": 215100, - "text": "\"\"\"", - "logprob": -0.11268901, - "special": false - }, - { - "id": 2978, - "text": " test", - "logprob": -1.5846587, - "special": false - }, - { - "id": 290, - "text": "_", - "logprob": -0.49796978, - "special": false - }, - { - "id": 4125, - "text": "test", - "logprob": -2.0026445, - "special": false - } - ] - } -} \ No newline at end of file diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index ed959291..9029e954 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -129,7 +129,7 @@ def load_weights( parameters = dict(model.named_parameters()) for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if quantize is None else "cpu" ) as f: for name in f.keys(): full_name = f"transformer.{name}" diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 11f3766e..54670b79 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -21,16 +21,14 @@ import torch import torch.distributed -from torch.nn import functional as F - from torch import nn from transformers.activations import ACT2FN from typing import Optional # Flash attention imports import flash_attn_cuda +import dropout_layer_norm -from flash_attn.layers.rotary import RotaryEmbedding from text_generation_server.utils.layers import ( FastLinear, TensorParallelRowLinear, @@ -332,15 +330,15 @@ def __init__(self, config, process_group=None): self.head_size = self.layers[0].self_attn.head_size self.num_heads = self.layers[0].self_attn.num_heads - def post_load_weights(self, load_in_8bit: bool = False): + def post_load_weights(self, quantize: Optional[str] = None): if isinstance(self.embed_tokens, TensorParallelEmbedding): self.embed_tokens.add_null_idx() for layer in self.layers: layer: FlashLlamaLayer - layer.self_attn.query_key_value.prepare_weights(load_in_8bit) - layer.self_attn.o_proj.prepare_weights(load_in_8bit) - layer.mlp.gate_up_proj.prepare_weights(load_in_8bit) - layer.mlp.down_proj.prepare_weights(load_in_8bit) + layer.self_attn.query_key_value.prepare_weights(quantize) + layer.self_attn.o_proj.prepare_weights(quantize) + layer.mlp.gate_up_proj.prepare_weights(quantize) + layer.mlp.down_proj.prepare_weights(quantize) def forward( self, @@ -429,8 +427,8 @@ def __init__(self, config, process_group=None): else: self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) - def post_load_weights(self, load_in_8bit: bool = False): - self.model.post_load_weights(load_in_8bit) + def post_load_weights(self, quantize: Optional[str] = None): + self.model.post_load_weights(quantize) self.lm_head.prepare_weights() def forward( diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 369e8d4f..2c6b8da6 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -21,8 +21,6 @@ import torch import torch.distributed -from torch.nn import functional as F - from torch import nn from transformers.activations import ACT2FN from transformers.modeling_utils import PreTrainedModel @@ -32,7 +30,6 @@ # Flash attention imports import flash_attn_cuda -from flash_attn.layers.rotary import RotaryEmbedding from text_generation_server.utils.layers import ( FastLinear, TensorParallelRowLinear, @@ -345,16 +342,16 @@ def __init__(self, config, process_group=None): self.head_size = self.layers[0].attention.head_size self.num_heads = self.layers[0].attention.num_heads - def post_load_weights(self, load_in_8bit=False): + def post_load_weights(self, quantize: Optional[str] = None): if isinstance(self.embed_in, TensorParallelEmbedding): self.embed_in.add_null_idx() for layer in self.layers: layer: FlashNeoXLayer layer.attention.shuffle_qkv_dims() - layer.attention.query_key_value.prepare_weights(load_in_8bit) - layer.attention.dense.prepare_weights(load_in_8bit) - layer.mlp.dense_h_to_4h.prepare_weights(load_in_8bit) - layer.mlp.dense_4h_to_h.prepare_weights(load_in_8bit) + layer.attention.query_key_value.prepare_weights(quantize) + layer.attention.dense.prepare_weights(quantize) + layer.mlp.dense_h_to_4h.prepare_weights(quantize) + layer.mlp.dense_4h_to_h.prepare_weights(quantize) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -457,8 +454,8 @@ def __init__(self, config, process_group=None): config.hidden_size, config.vocab_size, bias=False ) - def post_load_weights(self, load_in_8bit=False): - self.gpt_neox.post_load_weights(load_in_8bit) + def post_load_weights(self, quantize: Optional[str] = None): + self.gpt_neox.post_load_weights(quantize) self.embed_out.prepare_weights() @classmethod diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 9451b01a..9bded805 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -1,8 +1,6 @@ import torch import torch.distributed -import torch.nn.functional as F - from torch import nn from transformers.activations import ACT2FN from typing import Optional @@ -261,16 +259,16 @@ def __init__(self, config, process_group=None): self.head_size = self.h[0].attn.head_size self.num_heads = self.h[0].attn.num_heads - def post_load_weights(self, load_in_8bit: bool = False): + def post_load_weights(self, quantize: Optional[str] = None): if self.tp_embeddings: self.wte.add_null_idx() self.wpe.add_null_idx() for layer in self.h: layer: Block - layer.attn.c_attn.prepare_weights(load_in_8bit) - layer.attn.c_proj.prepare_weights(load_in_8bit) - layer.mlp.c_fc.prepare_weights(load_in_8bit) - layer.mlp.c_proj.prepare_weights(load_in_8bit) + layer.attn.c_attn.prepare_weights(quantize) + layer.attn.c_proj.prepare_weights(quantize) + layer.mlp.c_fc.prepare_weights(quantize) + layer.mlp.c_proj.prepare_weights(quantize) def forward( self, @@ -347,8 +345,8 @@ def __init__(self, config, process_group=None): else: self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) - def post_load_weights(self, load_in_8bit: bool = False): - self.transformer.post_load_weights(load_in_8bit) + def post_load_weights(self, quantize: Optional[str] = None): + self.transformer.post_load_weights(quantize) self.lm_head.prepare_weights() def forward( diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index aa0b4483..b775bd79 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -28,7 +28,12 @@ class FlashLlama(FlashCausalLM): - def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + ): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.float16 @@ -72,14 +77,14 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False def load_weights( model, filenames: List[Path], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, ): for filename in filenames: state_dict = torch.load(filename, map_location="cpu") for key, value in state_dict.items(): - value = value.to(device if not quantize else "cpu").to(dtype) + value = value.to(device if quantize is None else "cpu").to(dtype) layer_name = ".".join(key.split(".")[:4]) @@ -199,7 +204,7 @@ def __init__( def load_weights( model, filenames: List[str], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, rank: int, @@ -207,7 +212,7 @@ def load_weights( ): for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if quantize is None else "cpu" ) as f: for name in f.keys(): slice_ = f.get_slice(name) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index fc741f55..0924f107 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -23,7 +23,12 @@ class FlashNeoX(FlashCausalLM): - def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + ): super(FlashNeoX, self).__init__( FlashGPTNeoXForCausalLM, model_id, revision, quantize ) @@ -31,7 +36,10 @@ def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False class FlashNeoXSharded(FlashNeoX): def __init__( - self, model_id: str, revision: Optional[str] = None, quantize: bool = False + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -89,7 +97,7 @@ def load_weights( parameters = dict(model.named_parameters()) for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if quantize is None else "cpu" ) as f: for name in f.keys(): module_name, param_name = name.rsplit(".", 1) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index f810bb0b..031a67eb 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -27,7 +27,12 @@ class FlashSantacoder(FlashCausalLM): - def __init__(self, model_id: str, revision: Optional[str] = None, quantize=False): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + ): if torch.cuda.is_available(): device = torch.device("cuda") dtype = torch.float16 @@ -84,7 +89,7 @@ def load_weights( for filename in filenames: state_dict = torch.load(filename, map_location="cpu") for key, value in state_dict.items(): - value = value.to(device if not quantize else "cpu").to(dtype) + value = value.to(device if quantize is None else "cpu").to(dtype) layer_name = ".".join(key.split(".")[:4]) @@ -170,7 +175,10 @@ def decode(self, generated_ids: List[int]) -> str: class FlashSantacoderSharded(FlashSantacoder): def __init__( - self, model_id: str, revision: Optional[str] = None, quantize: bool = False + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -221,7 +229,7 @@ def __init__( def load_weights( model, filenames: List[str], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, rank: int, @@ -230,7 +238,7 @@ def load_weights( ): for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if quantize is None else "cpu" ) as f: for key in f.keys(): slice_ = f.get_slice(key) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index a0111250..d1e5e841 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -255,7 +255,7 @@ def load_weights( parameters = dict(model.named_parameters()) for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if quantize is None else "cpu" ) as f: for name in f.keys(): if name == "lm_head.weight": diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 3e8557b2..f95e5be2 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -94,7 +94,7 @@ def load_weights( parameters = dict(model.named_parameters()) for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if quantize is None else "cpu" ) as f: for name in f.keys(): module_name, param_name = name.rsplit(".", 1) diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index c83c3351..093cf70a 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -48,7 +48,10 @@ def forward( class OPTSharded(OPT): def __init__( - self, model_id: str, revision: Optional[str] = None, quantize: bool = False + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -107,7 +110,7 @@ def load_weights( parameters = dict(model.named_parameters()) for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if quantize is None else "cpu" ) as f: for name in f.keys(): if name == "lm_head.weight": diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index c8521dbf..8e3826a4 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -97,7 +97,7 @@ def load_weights( parameters = dict(model.named_parameters()) for file in filenames: with safe_open( - file, framework="pt", device=str(device) if not quantize else "cpu" + file, framework="pt", device=str(device) if quantize is None else "cpu" ) as f: for name in f.keys(): module_name, param_name = name.rsplit(".", 1) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 3383bf4b..7605639d 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -1,6 +1,8 @@ import torch from torch import nn +from torch.nn import functional as F +from typing import Optional HAS_BITS_AND_BYTES = True try: @@ -22,7 +24,7 @@ def __init__( self.quantized = False self.bnb_linear = None - def prepare_weights(self, quantize: bool = False): + def prepare_weights(self, quantize: Optional[str] = None): if quantize == "bitsandbytes": if not HAS_BITS_AND_BYTES: raise ImportError( @@ -126,6 +128,7 @@ def __init__( num_embeddings, embedding_dim, process_group: torch.distributed.ProcessGroup, + reduce=True, padding_idx=None, max_norm=None, norm_type=2.0, @@ -135,6 +138,7 @@ def __init__( device=None, dtype=None, ): + self.reduce = reduce self.process_group = process_group self.tp_rank = process_group.rank() self.tp_world_size = process_group.size() @@ -177,7 +181,8 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: input - self.min_id, ) out = super().forward(input) - torch.distributed.all_reduce(out, group=self.process_group) + if self.reduce: + torch.distributed.all_reduce(out, group=self.process_group) return out From dbdc587dddf0f16b7c05b28fb632acf9f65f185f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 16 May 2023 20:22:11 +0200 Subject: [PATCH 239/793] feat(integration-tests): improve comparison and health checks (#336) --- .github/workflows/build.yaml | 1 + clients/python/pyproject.toml | 2 +- clients/python/tests/test_client.py | 12 +- clients/python/text_generation/types.py | 2 +- integration-tests/conftest.py | 197 ++++- .../models/__snapshots__/test_bloom_560m.ambr | 627 ---------------- .../test_bloom_560m/test_bloom_560m.json | 128 ++++ .../test_bloom_560m_all_params.json | 98 +++ .../test_bloom_560m/test_bloom_560m_load.json | 514 +++++++++++++ .../test_bloom_560m_sharded.ambr | 542 -------------- .../test_bloom_560m_sharded.json | 128 ++++ .../test_bloom_560m_sharded_load.json | 514 +++++++++++++ .../__snapshots__/test_flash_llama.ambr | 465 ------------ .../test_flash_llama/test_flash_llama.json | 88 +++ .../test_flash_llama_all_params.json | 88 +++ .../test_flash_llama_load.json | 354 +++++++++ .../models/__snapshots__/test_flash_neox.ambr | 682 ------------------ .../test_flash_neox/test_flash_neox.json | 163 +++++ .../test_flash_neox/test_flash_neox_load.json | 654 +++++++++++++++++ .../__snapshots__/test_flash_santacoder.ambr | 472 ------------ .../test_flash_santacoder.json | 93 +++ .../test_flash_santacoder_load.json | 374 ++++++++++ .../__snapshots__/test_flash_starcoder.ambr | 573 --------------- .../test_flash_starcoder.json | 93 +++ .../test_flash_starcoder_default_params.json | 105 +++ .../test_flash_starcoder_load.json | 374 ++++++++++ .../models/__snapshots__/test_mt0_base.ambr | 306 -------- .../test_mt0_base/test_mt0_base.json | 48 ++ .../test_mt0_base_all_params.json | 78 ++ .../test_mt0_base/test_mt0_base_load.json | 218 ++++++ integration-tests/models/test_bloom_560m.py | 31 +- .../models/test_bloom_560m_sharded.py | 25 +- integration-tests/models/test_flash_llama.py | 31 +- integration-tests/models/test_flash_neox.py | 25 +- .../models/test_flash_santacoder.py | 27 +- .../models/test_flash_starcoder.py | 31 +- integration-tests/models/test_mt0_base.py | 31 +- integration-tests/models/utils.py | 15 - integration-tests/requirements.txt | 2 +- 39 files changed, 4409 insertions(+), 3802 deletions(-) delete mode 100644 integration-tests/models/__snapshots__/test_bloom_560m.ambr create mode 100644 integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json create mode 100644 integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_load.json delete mode 100644 integration-tests/models/__snapshots__/test_bloom_560m_sharded.ambr create mode 100644 integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json create mode 100644 integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded_load.json delete mode 100644 integration-tests/models/__snapshots__/test_flash_llama.ambr create mode 100644 integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama.json create mode 100644 integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json delete mode 100644 integration-tests/models/__snapshots__/test_flash_neox.ambr create mode 100644 integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox.json create mode 100644 integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox_load.json delete mode 100644 integration-tests/models/__snapshots__/test_flash_santacoder.ambr create mode 100644 integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder.json create mode 100644 integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder_load.json delete mode 100644 integration-tests/models/__snapshots__/test_flash_starcoder.ambr create mode 100644 integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder.json create mode 100644 integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json create mode 100644 integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_load.json delete mode 100644 integration-tests/models/__snapshots__/test_mt0_base.ambr create mode 100644 integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base.json create mode 100644 integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_load.json delete mode 100644 integration-tests/models/utils.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c2aba160..79b3c777 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -10,6 +10,7 @@ on: pull_request: paths: - ".github/workflows/build.yaml" + - "integration-tests/**" - "server/**" - "proto/**" - "router/**" diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 00bb99fc..06d5f9cb 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.5.1" +version = "0.5.2" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 8972dfd1..32462f14 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -16,9 +16,9 @@ def test_generate(flan_t5_xxl_url, hf_headers): assert len(response.details.prefill) == 1 assert response.details.prefill[0] == PrefillToken(id=0, text="", logprob=None) assert len(response.details.tokens) == 1 - assert response.details.tokens[0] == Token( - id=3, text="", logprob=-1.984375, special=False - ) + assert response.details.tokens[0].id == 3 + assert response.details.tokens[0].text == "" + assert not response.details.tokens[0].special def test_generate_best_of(flan_t5_xxl_url, hf_headers): @@ -82,9 +82,9 @@ async def test_generate_async(flan_t5_xxl_url, hf_headers): assert len(response.details.prefill) == 1 assert response.details.prefill[0] == PrefillToken(id=0, text="", logprob=None) assert len(response.details.tokens) == 1 - assert response.details.tokens[0] == Token( - id=3, text="", logprob=-1.984375, special=False - ) + assert response.details.tokens[0].id == 3 + assert response.details.tokens[0].text == "" + assert not response.details.tokens[0].special @pytest.mark.asyncio diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index f3f9dcb5..ad3cd09b 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -154,7 +154,7 @@ class Token(BaseModel): # Generation finish reason -class FinishReason(Enum): +class FinishReason(str, Enum): # number of generated tokens == `max_new_tokens` Length = "length" # the model generated its end of sequence token diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index e9c51c37..ba1abca9 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -4,22 +4,192 @@ import asyncio import os import docker +import json +import math +import time from docker.errors import NotFound -from typing import Optional, List -from syrupy.filters import props +from typing import Optional, List, Dict +from syrupy.extensions.json import JSONSnapshotExtension +from aiohttp import ClientConnectorError, ClientOSError, ServerDisconnectedError from text_generation import AsyncClient -from text_generation.types import Response +from text_generation.types import Response, Details, PrefillToken, Token, BestOfSequence DOCKER_IMAGE = os.getenv("DOCKER_IMAGE", None) HUGGING_FACE_HUB_TOKEN = os.getenv("HUGGING_FACE_HUB_TOKEN", None) DOCKER_VOLUME = os.getenv("DOCKER_VOLUME", "/data") +class ResponseComparator(JSONSnapshotExtension): + def serialize( + self, + data, + *, + exclude=None, + matcher=None, + ): + if isinstance(data, List): + data = [d.dict() for d in data] + + data = self._filter( + data=data, depth=0, path=(), exclude=exclude, matcher=matcher + ) + return json.dumps(data, indent=2, ensure_ascii=False, sort_keys=False) + "\n" + + def matches( + self, + *, + serialized_data, + snapshot_data, + ) -> bool: + def convert_data(data): + data = json.loads(data) + + if isinstance(data, Dict): + return Response(**data) + if isinstance(data, List): + return [Response(**d) for d in data] + raise NotImplementedError + + def eq_token(token: Token, other: Token) -> bool: + return ( + token.id == other.id + and token.text == other.text + and math.isclose(token.logprob, other.logprob, rel_tol=0.2) + and token.special == other.special + ) + + def eq_prefill_token(prefill_token: PrefillToken, other: PrefillToken) -> bool: + try: + return ( + prefill_token.id == other.id + and prefill_token.text == other.text + and ( + math.isclose(prefill_token.logprob, other.logprob, rel_tol=0.2) + if prefill_token.logprob is not None + else prefill_token.logprob == other.logprob + ) + ) + except TypeError: + return False + + def eq_best_of(details: BestOfSequence, other: BestOfSequence) -> bool: + return ( + details.finish_reason == other.finish_reason + and details.generated_tokens == other.generated_tokens + and details.seed == other.seed + and len(details.prefill) == len(other.prefill) + and all( + [ + eq_prefill_token(d, o) + for d, o in zip(details.prefill, other.prefill) + ] + ) + and len(details.tokens) == len(other.tokens) + and all([eq_token(d, o) for d, o in zip(details.tokens, other.tokens)]) + ) + + def eq_details(details: Details, other: Details) -> bool: + return ( + details.finish_reason == other.finish_reason + and details.generated_tokens == other.generated_tokens + and details.seed == other.seed + and len(details.prefill) == len(other.prefill) + and all( + [ + eq_prefill_token(d, o) + for d, o in zip(details.prefill, other.prefill) + ] + ) + and len(details.tokens) == len(other.tokens) + and all([eq_token(d, o) for d, o in zip(details.tokens, other.tokens)]) + and ( + len(details.best_of_sequences) + if details.best_of_sequences is not None + else 0 + ) + == ( + len(other.best_of_sequences) + if other.best_of_sequences is not None + else 0 + ) + and ( + all( + [ + eq_best_of(d, o) + for d, o in zip( + details.best_of_sequences, other.best_of_sequences + ) + ] + ) + if details.best_of_sequences is not None + else details.best_of_sequences == other.best_of_sequences + ) + ) + + def eq_response(response: Response, other: Response) -> bool: + return response.generated_text == other.generated_text and eq_details( + response.details, other.details + ) + + serialized_data = convert_data(serialized_data) + snapshot_data = convert_data(snapshot_data) + + if not isinstance(serialized_data, List): + serialized_data = [serialized_data] + if not isinstance(snapshot_data, List): + snapshot_data = [snapshot_data] + + return len(snapshot_data) == len(serialized_data) and all( + [eq_response(r, o) for r, o in zip(serialized_data, snapshot_data)] + ) + + +class LauncherHandle: + def __init__(self, port: int): + self.client = AsyncClient(f"http://localhost:{port}") + + def _inner_health(self): + raise NotImplementedError + + async def health(self, timeout: int = 60): + assert timeout > 0 + for _ in range(timeout): + if not self._inner_health(): + raise RuntimeError("Launcher crashed") + + try: + await self.client.generate("test") + return + except (ClientConnectorError, ClientOSError, ServerDisconnectedError) as e: + time.sleep(1) + raise RuntimeError("Health check failed") + + +class ContainerLauncherHandle(LauncherHandle): + def __init__(self, docker_client, container_name, port: int): + super(ContainerLauncherHandle, self).__init__(port) + self.docker_client = docker_client + self.container_name = container_name + + def _inner_health(self) -> bool: + container = self.docker_client.containers.get(self.container_name) + return container.status in ["running", "created"] + + +class ProcessLauncherHandle(LauncherHandle): + def __init__(self, process, port: int): + super(ProcessLauncherHandle, self).__init__(port) + self.process = process + + def _inner_health(self) -> bool: + return self.process.poll() is None + + @pytest.fixture -def snapshot_test(snapshot): - return lambda value: value == snapshot(exclude=props("logprob")) +def response_snapshot(snapshot): + return snapshot.use_extension(ResponseComparator) @pytest.fixture(scope="module") @@ -60,7 +230,7 @@ def local_launcher( with subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) as process: - yield AsyncClient(f"http://localhost:{port}") + yield ProcessLauncherHandle(process, port) process.terminate() process.wait(60) @@ -110,7 +280,7 @@ def docker_launcher( command=args, name=container_name, environment=env, - auto_remove=True, + auto_remove=False, detach=True, device_requests=[ docker.types.DeviceRequest(count=gpu_count, capabilities=[["gpu"]]) @@ -119,13 +289,19 @@ def docker_launcher( ports={"80/tcp": port}, ) - yield AsyncClient(f"http://localhost:{port}") + yield ContainerLauncherHandle(client, container.name, port) - container.stop() + try: + container.stop() + container.wait() + except NotFound: + pass container_output = container.logs().decode("utf-8") print(container_output) + container.remove() + if DOCKER_IMAGE is not None: return docker_launcher return local_launcher @@ -140,7 +316,6 @@ async def generate_load_inner( client.generate(prompt, max_new_tokens=max_new_tokens) for _ in range(n) ] - results = await asyncio.gather(*futures) - return [r.dict() for r in results] + return await asyncio.gather(*futures) return generate_load_inner diff --git a/integration-tests/models/__snapshots__/test_bloom_560m.ambr b/integration-tests/models/__snapshots__/test_bloom_560m.ambr deleted file mode 100644 index 1067513d..00000000 --- a/integration-tests/models/__snapshots__/test_bloom_560m.ambr +++ /dev/null @@ -1,627 +0,0 @@ -# serializer version: 1 -# name: test_bloom_560m - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': 0, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 159570, - 'special': False, - 'text': ' réch', - }), - dict({ - 'id': 810, - 'special': False, - 'text': 'au', - }), - dict({ - 'id': 12736, - 'special': False, - 'text': 'ffer', - }), - dict({ - 'id': 1742, - 'special': False, - 'text': ' au', - }), - dict({ - 'id': 6105, - 'special': False, - 'text': ' bain', - }), - dict({ - 'id': 88254, - 'special': False, - 'text': '-mar', - }), - dict({ - 'id': 641, - 'special': False, - 'text': 'ie', - }), - dict({ - 'id': 2940, - 'special': False, - 'text': ' avec', - }), - ]), - }), - 'generated_text': ' le faire réchauffer au bain-marie avec', - }) -# --- -# name: test_bloom_560m_all_params - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': 0, - 'tokens': list([ - dict({ - 'id': 408, - 'special': False, - 'text': ' que', - }), - dict({ - 'id': 20288, - 'special': False, - 'text': " l'on", - }), - dict({ - 'id': 22255, - 'special': False, - 'text': ' trouve', - }), - dict({ - 'id': 1622, - 'special': False, - 'text': ' une', - }), - dict({ - 'id': 187079, - 'special': False, - 'text': ' posture', - }), - dict({ - 'id': 501, - 'special': False, - 'text': ' par', - }), - dict({ - 'id': 8741, - 'special': False, - 'text': ' rapport', - }), - dict({ - 'id': 693, - 'special': False, - 'text': ' à', - }), - dict({ - 'id': 366, - 'special': False, - 'text': ' la', - }), - dict({ - 'id': 36503, - 'special': False, - 'text': ' pratique', - }), - ]), - }), - 'generated_text': "Pour déguster un ortolan, il faut tout d'abord que l'on trouve une posture par rapport à la pratique", - }) -# --- -# name: test_bloom_560m_load - list([ - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 1767, - 'special': False, - 'text': ' cu', - }), - dict({ - 'id': 1273, - 'special': False, - 'text': 'ire', - }), - dict({ - 'id': 1486, - 'special': False, - 'text': ' dans', - }), - dict({ - 'id': 283, - 'special': False, - 'text': ' de', - }), - dict({ - 'id': 40410, - 'special': False, - 'text': " l'eau", - }), - dict({ - 'id': 20226, - 'special': False, - 'text': ' bou', - }), - dict({ - 'id': 172483, - 'special': False, - 'text': 'illante', - }), - dict({ - 'id': 2805, - 'special': False, - 'text': ' sal', - }), - ]), - }), - 'generated_text': " le faire cuire dans de l'eau bouillante sal", - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 1767, - 'special': False, - 'text': ' cu', - }), - dict({ - 'id': 1273, - 'special': False, - 'text': 'ire', - }), - dict({ - 'id': 1486, - 'special': False, - 'text': ' dans', - }), - dict({ - 'id': 283, - 'special': False, - 'text': ' de', - }), - dict({ - 'id': 40410, - 'special': False, - 'text': " l'eau", - }), - dict({ - 'id': 20226, - 'special': False, - 'text': ' bou', - }), - dict({ - 'id': 172483, - 'special': False, - 'text': 'illante', - }), - dict({ - 'id': 2805, - 'special': False, - 'text': ' sal', - }), - ]), - }), - 'generated_text': " le faire cuire dans de l'eau bouillante sal", - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 1767, - 'special': False, - 'text': ' cu', - }), - dict({ - 'id': 1273, - 'special': False, - 'text': 'ire', - }), - dict({ - 'id': 1486, - 'special': False, - 'text': ' dans', - }), - dict({ - 'id': 283, - 'special': False, - 'text': ' de', - }), - dict({ - 'id': 40410, - 'special': False, - 'text': " l'eau", - }), - dict({ - 'id': 20226, - 'special': False, - 'text': ' bou', - }), - dict({ - 'id': 172483, - 'special': False, - 'text': 'illante', - }), - dict({ - 'id': 2805, - 'special': False, - 'text': ' sal', - }), - ]), - }), - 'generated_text': " le faire cuire dans de l'eau bouillante sal", - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 1767, - 'special': False, - 'text': ' cu', - }), - dict({ - 'id': 1273, - 'special': False, - 'text': 'ire', - }), - dict({ - 'id': 1486, - 'special': False, - 'text': ' dans', - }), - dict({ - 'id': 283, - 'special': False, - 'text': ' de', - }), - dict({ - 'id': 40410, - 'special': False, - 'text': " l'eau", - }), - dict({ - 'id': 20226, - 'special': False, - 'text': ' bou', - }), - dict({ - 'id': 172483, - 'special': False, - 'text': 'illante', - }), - dict({ - 'id': 2805, - 'special': False, - 'text': ' sal', - }), - ]), - }), - 'generated_text': " le faire cuire dans de l'eau bouillante sal", - }), - ]) -# --- diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json new file mode 100644 index 00000000..53a4ab85 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json @@ -0,0 +1,128 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.5625, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.14770508, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9287109, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.4609375, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.5585938, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.4003906, + "text": "," + }, + { + "id": 1669, + "logprob": -1.5673828, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.94628906, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.703125, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.5732422, + "text": " d'abord" + } + ], + "seed": 0, + "tokens": [ + { + "id": 578, + "logprob": -1.6591797, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.4492188, + "special": false, + "text": " faire" + }, + { + "id": 159570, + "logprob": -6.6835938, + "special": false, + "text": " réch" + }, + { + "id": 810, + "logprob": 0.0, + "special": false, + "text": "au" + }, + { + "id": 12736, + "logprob": 0.0, + "special": false, + "text": "ffer" + }, + { + "id": 1742, + "logprob": -2.5175781, + "special": false, + "text": " au" + }, + { + "id": 6105, + "logprob": -2.0078125, + "special": false, + "text": " bain" + }, + { + "id": 88254, + "logprob": -0.12695312, + "special": false, + "text": "-mar" + }, + { + "id": 641, + "logprob": 0.0, + "special": false, + "text": "ie" + }, + { + "id": 2940, + "logprob": -3.5175781, + "special": false, + "text": " avec" + } + ] + }, + "generated_text": " le faire réchauffer au bain-marie avec" +} diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json new file mode 100644 index 00000000..93a95804 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json @@ -0,0 +1,98 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 15, + "logprob": null, + "text": "," + }, + { + "id": 1669, + "logprob": -5.4414062, + "text": " il" + }, + { + "id": 11580, + "logprob": -2.3378906, + "text": " faut" + }, + { + "id": 3913, + "logprob": -4.3554688, + "text": " tout" + }, + { + "id": 39261, + "logprob": -2.9238281, + "text": " d'abord" + } + ], + "seed": 0, + "tokens": [ + { + "id": 408, + "logprob": -1.9267578, + "special": false, + "text": " que" + }, + { + "id": 20288, + "logprob": -2.9257812, + "special": false, + "text": " l'on" + }, + { + "id": 22255, + "logprob": -2.8964844, + "special": false, + "text": " trouve" + }, + { + "id": 1622, + "logprob": -1.1083984, + "special": false, + "text": " une" + }, + { + "id": 187079, + "logprob": -7.796875, + "special": false, + "text": " posture" + }, + { + "id": 501, + "logprob": -5.390625, + "special": false, + "text": " par" + }, + { + "id": 8741, + "logprob": -0.34936523, + "special": false, + "text": " rapport" + }, + { + "id": 693, + "logprob": 0.0, + "special": false, + "text": " à" + }, + { + "id": 366, + "logprob": -2.3378906, + "special": false, + "text": " la" + }, + { + "id": 36503, + "logprob": -3.6640625, + "special": false, + "text": " pratique" + } + ] + }, + "generated_text": "Pour déguster un ortolan, il faut tout d'abord que l'on trouve une posture par rapport à la pratique" +} diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_load.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_load.json new file mode 100644 index 00000000..0a86bef8 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_load.json @@ -0,0 +1,514 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.5625, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.14770508, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9287109, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.4609375, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.5585938, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.4003906, + "text": "," + }, + { + "id": 1669, + "logprob": -1.5673828, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.94628906, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.703125, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.5732422, + "text": " d'abord" + } + ], + "seed": null, + "tokens": [ + { + "id": 578, + "logprob": -1.7646484, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.6113281, + "special": false, + "text": " faire" + }, + { + "id": 1767, + "logprob": -1.5263672, + "special": false, + "text": " cu" + }, + { + "id": 1273, + "logprob": -0.00010049343, + "special": false, + "text": "ire" + }, + { + "id": 1486, + "logprob": -1.4707031, + "special": false, + "text": " dans" + }, + { + "id": 283, + "logprob": -1.2119141, + "special": false, + "text": " de" + }, + { + "id": 40410, + "logprob": -0.11883545, + "special": false, + "text": " l'eau" + }, + { + "id": 20226, + "logprob": -0.40844727, + "special": false, + "text": " bou" + }, + { + "id": 172483, + "logprob": -0.0037841797, + "special": false, + "text": "illante" + }, + { + "id": 2805, + "logprob": -1.0195312, + "special": false, + "text": " sal" + } + ] + }, + "generated_text": " le faire cuire dans de l'eau bouillante sal" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.53125, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.14770508, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9287109, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.4140625, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.5234375, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.3613281, + "text": "," + }, + { + "id": 1669, + "logprob": -1.5458984, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.94189453, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.7011719, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.5732422, + "text": " d'abord" + } + ], + "seed": null, + "tokens": [ + { + "id": 578, + "logprob": -1.7548828, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.578125, + "special": false, + "text": " faire" + }, + { + "id": 1767, + "logprob": -1.5117188, + "special": false, + "text": " cu" + }, + { + "id": 1273, + "logprob": -0.00010049343, + "special": false, + "text": "ire" + }, + { + "id": 1486, + "logprob": -1.4707031, + "special": false, + "text": " dans" + }, + { + "id": 283, + "logprob": -1.1982422, + "special": false, + "text": " de" + }, + { + "id": 40410, + "logprob": -0.11004639, + "special": false, + "text": " l'eau" + }, + { + "id": 20226, + "logprob": -0.4506836, + "special": false, + "text": " bou" + }, + { + "id": 172483, + "logprob": -0.003047943, + "special": false, + "text": "illante" + }, + { + "id": 2805, + "logprob": -1.0185547, + "special": false, + "text": " sal" + } + ] + }, + "generated_text": " le faire cuire dans de l'eau bouillante sal" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.53125, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.14770508, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9287109, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.4140625, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.5234375, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.3613281, + "text": "," + }, + { + "id": 1669, + "logprob": -1.5458984, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.94189453, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.7011719, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.5732422, + "text": " d'abord" + } + ], + "seed": null, + "tokens": [ + { + "id": 578, + "logprob": -1.7548828, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.578125, + "special": false, + "text": " faire" + }, + { + "id": 1767, + "logprob": -1.5117188, + "special": false, + "text": " cu" + }, + { + "id": 1273, + "logprob": -0.00010049343, + "special": false, + "text": "ire" + }, + { + "id": 1486, + "logprob": -1.4707031, + "special": false, + "text": " dans" + }, + { + "id": 283, + "logprob": -1.1982422, + "special": false, + "text": " de" + }, + { + "id": 40410, + "logprob": -0.11004639, + "special": false, + "text": " l'eau" + }, + { + "id": 20226, + "logprob": -0.4506836, + "special": false, + "text": " bou" + }, + { + "id": 172483, + "logprob": -0.003047943, + "special": false, + "text": "illante" + }, + { + "id": 2805, + "logprob": -1.0185547, + "special": false, + "text": " sal" + } + ] + }, + "generated_text": " le faire cuire dans de l'eau bouillante sal" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.53125, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.14770508, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9287109, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.4140625, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.5234375, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.3613281, + "text": "," + }, + { + "id": 1669, + "logprob": -1.5458984, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.94189453, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.7011719, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.5732422, + "text": " d'abord" + } + ], + "seed": null, + "tokens": [ + { + "id": 578, + "logprob": -1.7548828, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.578125, + "special": false, + "text": " faire" + }, + { + "id": 1767, + "logprob": -1.5117188, + "special": false, + "text": " cu" + }, + { + "id": 1273, + "logprob": -0.00010049343, + "special": false, + "text": "ire" + }, + { + "id": 1486, + "logprob": -1.4707031, + "special": false, + "text": " dans" + }, + { + "id": 283, + "logprob": -1.1982422, + "special": false, + "text": " de" + }, + { + "id": 40410, + "logprob": -0.11004639, + "special": false, + "text": " l'eau" + }, + { + "id": 20226, + "logprob": -0.4506836, + "special": false, + "text": " bou" + }, + { + "id": 172483, + "logprob": -0.003047943, + "special": false, + "text": "illante" + }, + { + "id": 2805, + "logprob": -1.0185547, + "special": false, + "text": " sal" + } + ] + }, + "generated_text": " le faire cuire dans de l'eau bouillante sal" + } +] diff --git a/integration-tests/models/__snapshots__/test_bloom_560m_sharded.ambr b/integration-tests/models/__snapshots__/test_bloom_560m_sharded.ambr deleted file mode 100644 index 667a0373..00000000 --- a/integration-tests/models/__snapshots__/test_bloom_560m_sharded.ambr +++ /dev/null @@ -1,542 +0,0 @@ -# serializer version: 1 -# name: test_bloom_560m_sharded - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': 0, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 159570, - 'special': False, - 'text': ' réch', - }), - dict({ - 'id': 810, - 'special': False, - 'text': 'au', - }), - dict({ - 'id': 12736, - 'special': False, - 'text': 'ffer', - }), - dict({ - 'id': 1742, - 'special': False, - 'text': ' au', - }), - dict({ - 'id': 6105, - 'special': False, - 'text': ' bain', - }), - dict({ - 'id': 88254, - 'special': False, - 'text': '-mar', - }), - dict({ - 'id': 641, - 'special': False, - 'text': 'ie', - }), - dict({ - 'id': 2940, - 'special': False, - 'text': ' avec', - }), - ]), - }), - 'generated_text': ' le faire réchauffer au bain-marie avec', - }) -# --- -# name: test_bloom_560m_sharded_load - list([ - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 1767, - 'special': False, - 'text': ' cu', - }), - dict({ - 'id': 1273, - 'special': False, - 'text': 'ire', - }), - dict({ - 'id': 1486, - 'special': False, - 'text': ' dans', - }), - dict({ - 'id': 283, - 'special': False, - 'text': ' de', - }), - dict({ - 'id': 40410, - 'special': False, - 'text': " l'eau", - }), - dict({ - 'id': 20226, - 'special': False, - 'text': ' bou', - }), - dict({ - 'id': 172483, - 'special': False, - 'text': 'illante', - }), - dict({ - 'id': 2805, - 'special': False, - 'text': ' sal', - }), - ]), - }), - 'generated_text': " le faire cuire dans de l'eau bouillante sal", - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 1767, - 'special': False, - 'text': ' cu', - }), - dict({ - 'id': 1273, - 'special': False, - 'text': 'ire', - }), - dict({ - 'id': 1486, - 'special': False, - 'text': ' dans', - }), - dict({ - 'id': 283, - 'special': False, - 'text': ' de', - }), - dict({ - 'id': 40410, - 'special': False, - 'text': " l'eau", - }), - dict({ - 'id': 20226, - 'special': False, - 'text': ' bou', - }), - dict({ - 'id': 172483, - 'special': False, - 'text': 'illante', - }), - dict({ - 'id': 2805, - 'special': False, - 'text': ' sal', - }), - ]), - }), - 'generated_text': " le faire cuire dans de l'eau bouillante sal", - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 1767, - 'special': False, - 'text': ' cu', - }), - dict({ - 'id': 1273, - 'special': False, - 'text': 'ire', - }), - dict({ - 'id': 1486, - 'special': False, - 'text': ' dans', - }), - dict({ - 'id': 283, - 'special': False, - 'text': ' de', - }), - dict({ - 'id': 40410, - 'special': False, - 'text': " l'eau", - }), - dict({ - 'id': 20226, - 'special': False, - 'text': ' bou', - }), - dict({ - 'id': 172483, - 'special': False, - 'text': 'illante', - }), - dict({ - 'id': 2805, - 'special': False, - 'text': ' sal', - }), - ]), - }), - 'generated_text': " le faire cuire dans de l'eau bouillante sal", - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 17934, - 'text': 'Pour', - }), - dict({ - 'id': 49833, - 'text': ' dég', - }), - dict({ - 'id': 21543, - 'text': 'uster', - }), - dict({ - 'id': 447, - 'text': ' un', - }), - dict({ - 'id': 46341, - 'text': ' ort', - }), - dict({ - 'id': 35567, - 'text': 'olan', - }), - dict({ - 'id': 15, - 'text': ',', - }), - dict({ - 'id': 1669, - 'text': ' il', - }), - dict({ - 'id': 11580, - 'text': ' faut', - }), - dict({ - 'id': 3913, - 'text': ' tout', - }), - dict({ - 'id': 39261, - 'text': " d'abord", - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 578, - 'special': False, - 'text': ' le', - }), - dict({ - 'id': 5608, - 'special': False, - 'text': ' faire', - }), - dict({ - 'id': 1767, - 'special': False, - 'text': ' cu', - }), - dict({ - 'id': 1273, - 'special': False, - 'text': 'ire', - }), - dict({ - 'id': 1486, - 'special': False, - 'text': ' dans', - }), - dict({ - 'id': 283, - 'special': False, - 'text': ' de', - }), - dict({ - 'id': 40410, - 'special': False, - 'text': " l'eau", - }), - dict({ - 'id': 20226, - 'special': False, - 'text': ' bou', - }), - dict({ - 'id': 172483, - 'special': False, - 'text': 'illante', - }), - dict({ - 'id': 2805, - 'special': False, - 'text': ' sal', - }), - ]), - }), - 'generated_text': " le faire cuire dans de l'eau bouillante sal", - }), - ]) -# --- diff --git a/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json b/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json new file mode 100644 index 00000000..dd8936af --- /dev/null +++ b/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json @@ -0,0 +1,128 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.5390625, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.14758301, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9296875, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.4453125, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.59375, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.3994141, + "text": "," + }, + { + "id": 1669, + "logprob": -1.578125, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.9453125, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.7011719, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.5732422, + "text": " d'abord" + } + ], + "seed": 0, + "tokens": [ + { + "id": 578, + "logprob": -1.6474609, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.5097656, + "special": false, + "text": " faire" + }, + { + "id": 159570, + "logprob": -6.65625, + "special": false, + "text": " réch" + }, + { + "id": 810, + "logprob": 0.0, + "special": false, + "text": "au" + }, + { + "id": 12736, + "logprob": 0.0, + "special": false, + "text": "ffer" + }, + { + "id": 1742, + "logprob": -2.5859375, + "special": false, + "text": " au" + }, + { + "id": 6105, + "logprob": -2.03125, + "special": false, + "text": " bain" + }, + { + "id": 88254, + "logprob": -0.12695312, + "special": false, + "text": "-mar" + }, + { + "id": 641, + "logprob": 0.0, + "special": false, + "text": "ie" + }, + { + "id": 2940, + "logprob": -3.5175781, + "special": false, + "text": " avec" + } + ] + }, + "generated_text": " le faire réchauffer au bain-marie avec" +} diff --git a/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded_load.json b/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded_load.json new file mode 100644 index 00000000..2dd480b9 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded_load.json @@ -0,0 +1,514 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.5390625, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.14758301, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9296875, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.4453125, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.59375, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.3994141, + "text": "," + }, + { + "id": 1669, + "logprob": -1.578125, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.9453125, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.7011719, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.5732422, + "text": " d'abord" + } + ], + "seed": null, + "tokens": [ + { + "id": 578, + "logprob": -1.7529297, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.6054688, + "special": false, + "text": " faire" + }, + { + "id": 1767, + "logprob": -1.5283203, + "special": false, + "text": " cu" + }, + { + "id": 1273, + "logprob": -0.00010049343, + "special": false, + "text": "ire" + }, + { + "id": 1486, + "logprob": -1.4716797, + "special": false, + "text": " dans" + }, + { + "id": 283, + "logprob": -1.1982422, + "special": false, + "text": " de" + }, + { + "id": 40410, + "logprob": -0.11853027, + "special": false, + "text": " l'eau" + }, + { + "id": 20226, + "logprob": -0.41210938, + "special": false, + "text": " bou" + }, + { + "id": 172483, + "logprob": -0.0037765503, + "special": false, + "text": "illante" + }, + { + "id": 2805, + "logprob": -1.0166016, + "special": false, + "text": " sal" + } + ] + }, + "generated_text": " le faire cuire dans de l'eau bouillante sal" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.515625, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.1484375, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9287109, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.34375, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.515625, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.4199219, + "text": "," + }, + { + "id": 1669, + "logprob": -1.5664062, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.94091797, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.6660156, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.7753906, + "text": " d'abord" + } + ], + "seed": null, + "tokens": [ + { + "id": 578, + "logprob": -1.7626953, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.5820312, + "special": false, + "text": " faire" + }, + { + "id": 1767, + "logprob": -1.5097656, + "special": false, + "text": " cu" + }, + { + "id": 1273, + "logprob": -9.393692e-05, + "special": false, + "text": "ire" + }, + { + "id": 1486, + "logprob": -1.5175781, + "special": false, + "text": " dans" + }, + { + "id": 283, + "logprob": -1.1982422, + "special": false, + "text": " de" + }, + { + "id": 40410, + "logprob": -0.11883545, + "special": false, + "text": " l'eau" + }, + { + "id": 20226, + "logprob": -0.4909668, + "special": false, + "text": " bou" + }, + { + "id": 172483, + "logprob": -0.003047943, + "special": false, + "text": "illante" + }, + { + "id": 2805, + "logprob": -1.0185547, + "special": false, + "text": " sal" + } + ] + }, + "generated_text": " le faire cuire dans de l'eau bouillante sal" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.515625, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.1484375, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9287109, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.34375, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.515625, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.4199219, + "text": "," + }, + { + "id": 1669, + "logprob": -1.5664062, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.94091797, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.6660156, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.7753906, + "text": " d'abord" + } + ], + "seed": null, + "tokens": [ + { + "id": 578, + "logprob": -1.7626953, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.5820312, + "special": false, + "text": " faire" + }, + { + "id": 1767, + "logprob": -1.5097656, + "special": false, + "text": " cu" + }, + { + "id": 1273, + "logprob": -9.393692e-05, + "special": false, + "text": "ire" + }, + { + "id": 1486, + "logprob": -1.5175781, + "special": false, + "text": " dans" + }, + { + "id": 283, + "logprob": -1.1982422, + "special": false, + "text": " de" + }, + { + "id": 40410, + "logprob": -0.11883545, + "special": false, + "text": " l'eau" + }, + { + "id": 20226, + "logprob": -0.4909668, + "special": false, + "text": " bou" + }, + { + "id": 172483, + "logprob": -0.003047943, + "special": false, + "text": "illante" + }, + { + "id": 2805, + "logprob": -1.0185547, + "special": false, + "text": " sal" + } + ] + }, + "generated_text": " le faire cuire dans de l'eau bouillante sal" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 17934, + "logprob": null, + "text": "Pour" + }, + { + "id": 49833, + "logprob": -10.515625, + "text": " dég" + }, + { + "id": 21543, + "logprob": -0.1484375, + "text": "uster" + }, + { + "id": 447, + "logprob": -1.9287109, + "text": " un" + }, + { + "id": 46341, + "logprob": -15.34375, + "text": " ort" + }, + { + "id": 35567, + "logprob": -7.515625, + "text": "olan" + }, + { + "id": 15, + "logprob": -1.4199219, + "text": "," + }, + { + "id": 1669, + "logprob": -1.5664062, + "text": " il" + }, + { + "id": 11580, + "logprob": -0.94091797, + "text": " faut" + }, + { + "id": 3913, + "logprob": -3.6660156, + "text": " tout" + }, + { + "id": 39261, + "logprob": -1.7753906, + "text": " d'abord" + } + ], + "seed": null, + "tokens": [ + { + "id": 578, + "logprob": -1.7626953, + "special": false, + "text": " le" + }, + { + "id": 5608, + "logprob": -2.5820312, + "special": false, + "text": " faire" + }, + { + "id": 1767, + "logprob": -1.5097656, + "special": false, + "text": " cu" + }, + { + "id": 1273, + "logprob": -9.393692e-05, + "special": false, + "text": "ire" + }, + { + "id": 1486, + "logprob": -1.5175781, + "special": false, + "text": " dans" + }, + { + "id": 283, + "logprob": -1.1982422, + "special": false, + "text": " de" + }, + { + "id": 40410, + "logprob": -0.11883545, + "special": false, + "text": " l'eau" + }, + { + "id": 20226, + "logprob": -0.4909668, + "special": false, + "text": " bou" + }, + { + "id": 172483, + "logprob": -0.003047943, + "special": false, + "text": "illante" + }, + { + "id": 2805, + "logprob": -1.0185547, + "special": false, + "text": " sal" + } + ] + }, + "generated_text": " le faire cuire dans de l'eau bouillante sal" + } +] diff --git a/integration-tests/models/__snapshots__/test_flash_llama.ambr b/integration-tests/models/__snapshots__/test_flash_llama.ambr deleted file mode 100644 index f4e3a4c1..00000000 --- a/integration-tests/models/__snapshots__/test_flash_llama.ambr +++ /dev/null @@ -1,465 +0,0 @@ -# serializer version: 1 -# name: test_flash_llama - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 1, - 'text': '', - }), - dict({ - 'id': 4321, - 'text': 'Test', - }), - dict({ - 'id': 2009, - 'text': 'request', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 363, - 'special': False, - 'text': ' for', - }), - dict({ - 'id': 847, - 'special': False, - 'text': ' /', - }), - dict({ - 'id': 2754, - 'special': False, - 'text': 'api', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29894, - 'special': False, - 'text': 'v', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 16418, - 'special': False, - 'text': 'projects', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - ]), - }), - 'generated_text': 'for /api/v1/projects/1', - }) -# --- -# name: test_flash_llama_all_params - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 1, - 'text': '', - }), - dict({ - 'id': 4321, - 'text': 'Test', - }), - dict({ - 'id': 2009, - 'text': 'request', - }), - ]), - 'seed': 0, - 'tokens': list([ - dict({ - 'id': 5229, - 'special': False, - 'text': ' failed', - }), - dict({ - 'id': 363, - 'special': False, - 'text': ' for', - }), - dict({ - 'id': 5641, - 'special': False, - 'text': ' IP', - }), - dict({ - 'id': 16428, - 'special': False, - 'text': ' Address', - }), - dict({ - 'id': 29901, - 'special': False, - 'text': ':', - }), - dict({ - 'id': 525, - 'special': False, - 'text': " '", - }), - dict({ - 'id': 8516, - 'special': False, - 'text': 'None', - }), - dict({ - 'id': 4286, - 'special': False, - 'text': "'.", - }), - dict({ - 'id': 13, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 294, - 'special': False, - 'text': 'as', - }), - ]), - }), - 'generated_text': ''' - Test requestfailed for IP Address: 'None'. - as - ''', - }) -# --- -# name: test_flash_llama_load - list([ - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 1, - 'text': '', - }), - dict({ - 'id': 4321, - 'text': 'Test', - }), - dict({ - 'id': 2009, - 'text': 'request', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 363, - 'special': False, - 'text': ' for', - }), - dict({ - 'id': 847, - 'special': False, - 'text': ' /', - }), - dict({ - 'id': 2754, - 'special': False, - 'text': 'api', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29894, - 'special': False, - 'text': 'v', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 16418, - 'special': False, - 'text': 'projects', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - ]), - }), - 'generated_text': 'for /api/v1/projects/1', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 1, - 'text': '', - }), - dict({ - 'id': 4321, - 'text': 'Test', - }), - dict({ - 'id': 2009, - 'text': 'request', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 363, - 'special': False, - 'text': ' for', - }), - dict({ - 'id': 847, - 'special': False, - 'text': ' /', - }), - dict({ - 'id': 2754, - 'special': False, - 'text': 'api', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29894, - 'special': False, - 'text': 'v', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 16418, - 'special': False, - 'text': 'projects', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - ]), - }), - 'generated_text': 'for /api/v1/projects/1', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 1, - 'text': '', - }), - dict({ - 'id': 4321, - 'text': 'Test', - }), - dict({ - 'id': 2009, - 'text': 'request', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 363, - 'special': False, - 'text': ' for', - }), - dict({ - 'id': 847, - 'special': False, - 'text': ' /', - }), - dict({ - 'id': 2754, - 'special': False, - 'text': 'api', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29894, - 'special': False, - 'text': 'v', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 16418, - 'special': False, - 'text': 'projects', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - ]), - }), - 'generated_text': 'for /api/v1/projects/1', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 1, - 'text': '', - }), - dict({ - 'id': 4321, - 'text': 'Test', - }), - dict({ - 'id': 2009, - 'text': 'request', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 363, - 'special': False, - 'text': ' for', - }), - dict({ - 'id': 847, - 'special': False, - 'text': ' /', - }), - dict({ - 'id': 2754, - 'special': False, - 'text': 'api', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29894, - 'special': False, - 'text': 'v', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 16418, - 'special': False, - 'text': 'projects', - }), - dict({ - 'id': 29914, - 'special': False, - 'text': '/', - }), - dict({ - 'id': 29896, - 'special': False, - 'text': '1', - }), - ]), - }), - 'generated_text': 'for /api/v1/projects/1', - }), - ]) -# --- diff --git a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama.json b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama.json new file mode 100644 index 00000000..49bc996c --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama.json @@ -0,0 +1,88 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -8.6875, + "text": "Test" + }, + { + "id": 2009, + "logprob": -11.5546875, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 363, + "logprob": -1.5380859, + "special": false, + "text": " for" + }, + { + "id": 847, + "logprob": -2.5917969, + "special": false, + "text": " /" + }, + { + "id": 2754, + "logprob": -2.2773438, + "special": false, + "text": "api" + }, + { + "id": 29914, + "logprob": -0.034362793, + "special": false, + "text": "/" + }, + { + "id": 29894, + "logprob": -0.96533203, + "special": false, + "text": "v" + }, + { + "id": 29896, + "logprob": -0.36669922, + "special": false, + "text": "1" + }, + { + "id": 29914, + "logprob": -0.013122559, + "special": false, + "text": "/" + }, + { + "id": 16418, + "logprob": -3.1503906, + "special": false, + "text": "projects" + }, + { + "id": 29914, + "logprob": -0.43652344, + "special": false, + "text": "/" + }, + { + "id": 29896, + "logprob": -1.9404297, + "special": false, + "text": "1" + } + ] + }, + "generated_text": "for /api/v1/projects/1" +} diff --git a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json new file mode 100644 index 00000000..1b6b51a3 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json @@ -0,0 +1,88 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -8.6875, + "text": "Test" + }, + { + "id": 2009, + "logprob": -11.5546875, + "text": "request" + } + ], + "seed": 0, + "tokens": [ + { + "id": 5229, + "logprob": -3.3085938, + "special": false, + "text": " failed" + }, + { + "id": 363, + "logprob": -3.984375, + "special": false, + "text": " for" + }, + { + "id": 5641, + "logprob": -6.53125, + "special": false, + "text": " IP" + }, + { + "id": 16428, + "logprob": -3.1835938, + "special": false, + "text": " Address" + }, + { + "id": 29901, + "logprob": -1.2324219, + "special": false, + "text": ":" + }, + { + "id": 525, + "logprob": -2.6855469, + "special": false, + "text": " '" + }, + { + "id": 8516, + "logprob": -7.1601562, + "special": false, + "text": "None" + }, + { + "id": 4286, + "logprob": -2.4433594, + "special": false, + "text": "'." + }, + { + "id": 13, + "logprob": -0.06530762, + "special": false, + "text": "\n" + }, + { + "id": 294, + "logprob": -7.953125, + "special": false, + "text": "as" + } + ] + }, + "generated_text": "Test requestfailed for IP Address: 'None'.\nas" +} diff --git a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json new file mode 100644 index 00000000..5a8ba217 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json @@ -0,0 +1,354 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -8.6875, + "text": "Test" + }, + { + "id": 2009, + "logprob": -11.5546875, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 363, + "logprob": -1.5322266, + "special": false, + "text": " for" + }, + { + "id": 847, + "logprob": -2.5585938, + "special": false, + "text": " /" + }, + { + "id": 2754, + "logprob": -2.265625, + "special": false, + "text": "api" + }, + { + "id": 29914, + "logprob": -0.034088135, + "special": false, + "text": "/" + }, + { + "id": 29894, + "logprob": -0.96240234, + "special": false, + "text": "v" + }, + { + "id": 29896, + "logprob": -0.36816406, + "special": false, + "text": "1" + }, + { + "id": 29914, + "logprob": -0.013191223, + "special": false, + "text": "/" + }, + { + "id": 16418, + "logprob": -3.15625, + "special": false, + "text": "projects" + }, + { + "id": 29914, + "logprob": -0.43774414, + "special": false, + "text": "/" + }, + { + "id": 29896, + "logprob": -1.9443359, + "special": false, + "text": "1" + } + ] + }, + "generated_text": "for /api/v1/projects/1" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -8.6875, + "text": "Test" + }, + { + "id": 2009, + "logprob": -11.5546875, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 363, + "logprob": -1.5380859, + "special": false, + "text": " for" + }, + { + "id": 847, + "logprob": -2.5859375, + "special": false, + "text": " /" + }, + { + "id": 2754, + "logprob": -2.2695312, + "special": false, + "text": "api" + }, + { + "id": 29914, + "logprob": -0.03439331, + "special": false, + "text": "/" + }, + { + "id": 29894, + "logprob": -0.96240234, + "special": false, + "text": "v" + }, + { + "id": 29896, + "logprob": -0.36694336, + "special": false, + "text": "1" + }, + { + "id": 29914, + "logprob": -0.013114929, + "special": false, + "text": "/" + }, + { + "id": 16418, + "logprob": -3.1542969, + "special": false, + "text": "projects" + }, + { + "id": 29914, + "logprob": -0.43847656, + "special": false, + "text": "/" + }, + { + "id": 29896, + "logprob": -1.9433594, + "special": false, + "text": "1" + } + ] + }, + "generated_text": "for /api/v1/projects/1" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -8.6875, + "text": "Test" + }, + { + "id": 2009, + "logprob": -11.5546875, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 363, + "logprob": -1.5322266, + "special": false, + "text": " for" + }, + { + "id": 847, + "logprob": -2.5585938, + "special": false, + "text": " /" + }, + { + "id": 2754, + "logprob": -2.265625, + "special": false, + "text": "api" + }, + { + "id": 29914, + "logprob": -0.034088135, + "special": false, + "text": "/" + }, + { + "id": 29894, + "logprob": -0.96240234, + "special": false, + "text": "v" + }, + { + "id": 29896, + "logprob": -0.36816406, + "special": false, + "text": "1" + }, + { + "id": 29914, + "logprob": -0.013191223, + "special": false, + "text": "/" + }, + { + "id": 16418, + "logprob": -3.15625, + "special": false, + "text": "projects" + }, + { + "id": 29914, + "logprob": -0.43774414, + "special": false, + "text": "/" + }, + { + "id": 29896, + "logprob": -1.9443359, + "special": false, + "text": "1" + } + ] + }, + "generated_text": "for /api/v1/projects/1" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -8.6875, + "text": "Test" + }, + { + "id": 2009, + "logprob": -11.5546875, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 363, + "logprob": -1.5322266, + "special": false, + "text": " for" + }, + { + "id": 847, + "logprob": -2.5585938, + "special": false, + "text": " /" + }, + { + "id": 2754, + "logprob": -2.265625, + "special": false, + "text": "api" + }, + { + "id": 29914, + "logprob": -0.034088135, + "special": false, + "text": "/" + }, + { + "id": 29894, + "logprob": -0.96240234, + "special": false, + "text": "v" + }, + { + "id": 29896, + "logprob": -0.36816406, + "special": false, + "text": "1" + }, + { + "id": 29914, + "logprob": -0.013191223, + "special": false, + "text": "/" + }, + { + "id": 16418, + "logprob": -3.15625, + "special": false, + "text": "projects" + }, + { + "id": 29914, + "logprob": -0.43774414, + "special": false, + "text": "/" + }, + { + "id": 29896, + "logprob": -1.9443359, + "special": false, + "text": "1" + } + ] + }, + "generated_text": "for /api/v1/projects/1" + } +] diff --git a/integration-tests/models/__snapshots__/test_flash_neox.ambr b/integration-tests/models/__snapshots__/test_flash_neox.ambr deleted file mode 100644 index 4330db6b..00000000 --- a/integration-tests/models/__snapshots__/test_flash_neox.ambr +++ /dev/null @@ -1,682 +0,0 @@ -# serializer version: 1 -# name: test_flash_neox - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 50278, - 'text': '<|prompter|>', - }), - dict({ - 'id': 1276, - 'text': 'What', - }), - dict({ - 'id': 310, - 'text': ' is', - }), - dict({ - 'id': 247, - 'text': ' a', - }), - dict({ - 'id': 1167, - 'text': ' mem', - }), - dict({ - 'id': 70, - 'text': 'e', - }), - dict({ - 'id': 13, - 'text': ',', - }), - dict({ - 'id': 285, - 'text': ' and', - }), - dict({ - 'id': 752, - 'text': ' what', - }), - dict({ - 'id': 434, - 'text': "'s", - }), - dict({ - 'id': 253, - 'text': ' the', - }), - dict({ - 'id': 2892, - 'text': ' history', - }), - dict({ - 'id': 3212, - 'text': ' behind', - }), - dict({ - 'id': 436, - 'text': ' this', - }), - dict({ - 'id': 3159, - 'text': ' word', - }), - dict({ - 'id': 32, - 'text': '?', - }), - dict({ - 'id': 0, - 'text': '<|endoftext|>', - }), - dict({ - 'id': 50281, - 'text': '<|assistant|>', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 510, - 'special': False, - 'text': 'The', - }), - dict({ - 'id': 3159, - 'special': False, - 'text': ' word', - }), - dict({ - 'id': 346, - 'special': False, - 'text': ' "', - }), - dict({ - 'id': 6441, - 'special': False, - 'text': 'mem', - }), - dict({ - 'id': 70, - 'special': False, - 'text': 'e', - }), - dict({ - 'id': 3, - 'special': False, - 'text': '"', - }), - dict({ - 'id': 369, - 'special': False, - 'text': ' was', - }), - dict({ - 'id': 806, - 'special': False, - 'text': ' first', - }), - dict({ - 'id': 908, - 'special': False, - 'text': ' used', - }), - dict({ - 'id': 275, - 'special': False, - 'text': ' in', - }), - ]), - }), - 'generated_text': 'The word "meme" was first used in', - }) -# --- -# name: test_flash_neox_load - list([ - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 50278, - 'text': '<|prompter|>', - }), - dict({ - 'id': 1276, - 'text': 'What', - }), - dict({ - 'id': 310, - 'text': ' is', - }), - dict({ - 'id': 247, - 'text': ' a', - }), - dict({ - 'id': 1167, - 'text': ' mem', - }), - dict({ - 'id': 70, - 'text': 'e', - }), - dict({ - 'id': 13, - 'text': ',', - }), - dict({ - 'id': 285, - 'text': ' and', - }), - dict({ - 'id': 752, - 'text': ' what', - }), - dict({ - 'id': 434, - 'text': "'s", - }), - dict({ - 'id': 253, - 'text': ' the', - }), - dict({ - 'id': 2892, - 'text': ' history', - }), - dict({ - 'id': 3212, - 'text': ' behind', - }), - dict({ - 'id': 436, - 'text': ' this', - }), - dict({ - 'id': 3159, - 'text': ' word', - }), - dict({ - 'id': 32, - 'text': '?', - }), - dict({ - 'id': 0, - 'text': '<|endoftext|>', - }), - dict({ - 'id': 50281, - 'text': '<|assistant|>', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 510, - 'special': False, - 'text': 'The', - }), - dict({ - 'id': 3159, - 'special': False, - 'text': ' word', - }), - dict({ - 'id': 346, - 'special': False, - 'text': ' "', - }), - dict({ - 'id': 6441, - 'special': False, - 'text': 'mem', - }), - dict({ - 'id': 70, - 'special': False, - 'text': 'e', - }), - dict({ - 'id': 3, - 'special': False, - 'text': '"', - }), - dict({ - 'id': 369, - 'special': False, - 'text': ' was', - }), - dict({ - 'id': 806, - 'special': False, - 'text': ' first', - }), - dict({ - 'id': 908, - 'special': False, - 'text': ' used', - }), - dict({ - 'id': 275, - 'special': False, - 'text': ' in', - }), - ]), - }), - 'generated_text': 'The word "meme" was first used in', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 50278, - 'text': '<|prompter|>', - }), - dict({ - 'id': 1276, - 'text': 'What', - }), - dict({ - 'id': 310, - 'text': ' is', - }), - dict({ - 'id': 247, - 'text': ' a', - }), - dict({ - 'id': 1167, - 'text': ' mem', - }), - dict({ - 'id': 70, - 'text': 'e', - }), - dict({ - 'id': 13, - 'text': ',', - }), - dict({ - 'id': 285, - 'text': ' and', - }), - dict({ - 'id': 752, - 'text': ' what', - }), - dict({ - 'id': 434, - 'text': "'s", - }), - dict({ - 'id': 253, - 'text': ' the', - }), - dict({ - 'id': 2892, - 'text': ' history', - }), - dict({ - 'id': 3212, - 'text': ' behind', - }), - dict({ - 'id': 436, - 'text': ' this', - }), - dict({ - 'id': 3159, - 'text': ' word', - }), - dict({ - 'id': 32, - 'text': '?', - }), - dict({ - 'id': 0, - 'text': '<|endoftext|>', - }), - dict({ - 'id': 50281, - 'text': '<|assistant|>', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 510, - 'special': False, - 'text': 'The', - }), - dict({ - 'id': 3159, - 'special': False, - 'text': ' word', - }), - dict({ - 'id': 346, - 'special': False, - 'text': ' "', - }), - dict({ - 'id': 6441, - 'special': False, - 'text': 'mem', - }), - dict({ - 'id': 70, - 'special': False, - 'text': 'e', - }), - dict({ - 'id': 3, - 'special': False, - 'text': '"', - }), - dict({ - 'id': 369, - 'special': False, - 'text': ' was', - }), - dict({ - 'id': 806, - 'special': False, - 'text': ' first', - }), - dict({ - 'id': 908, - 'special': False, - 'text': ' used', - }), - dict({ - 'id': 275, - 'special': False, - 'text': ' in', - }), - ]), - }), - 'generated_text': 'The word "meme" was first used in', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 50278, - 'text': '<|prompter|>', - }), - dict({ - 'id': 1276, - 'text': 'What', - }), - dict({ - 'id': 310, - 'text': ' is', - }), - dict({ - 'id': 247, - 'text': ' a', - }), - dict({ - 'id': 1167, - 'text': ' mem', - }), - dict({ - 'id': 70, - 'text': 'e', - }), - dict({ - 'id': 13, - 'text': ',', - }), - dict({ - 'id': 285, - 'text': ' and', - }), - dict({ - 'id': 752, - 'text': ' what', - }), - dict({ - 'id': 434, - 'text': "'s", - }), - dict({ - 'id': 253, - 'text': ' the', - }), - dict({ - 'id': 2892, - 'text': ' history', - }), - dict({ - 'id': 3212, - 'text': ' behind', - }), - dict({ - 'id': 436, - 'text': ' this', - }), - dict({ - 'id': 3159, - 'text': ' word', - }), - dict({ - 'id': 32, - 'text': '?', - }), - dict({ - 'id': 0, - 'text': '<|endoftext|>', - }), - dict({ - 'id': 50281, - 'text': '<|assistant|>', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 510, - 'special': False, - 'text': 'The', - }), - dict({ - 'id': 3159, - 'special': False, - 'text': ' word', - }), - dict({ - 'id': 346, - 'special': False, - 'text': ' "', - }), - dict({ - 'id': 6441, - 'special': False, - 'text': 'mem', - }), - dict({ - 'id': 70, - 'special': False, - 'text': 'e', - }), - dict({ - 'id': 3, - 'special': False, - 'text': '"', - }), - dict({ - 'id': 369, - 'special': False, - 'text': ' was', - }), - dict({ - 'id': 806, - 'special': False, - 'text': ' first', - }), - dict({ - 'id': 908, - 'special': False, - 'text': ' used', - }), - dict({ - 'id': 275, - 'special': False, - 'text': ' in', - }), - ]), - }), - 'generated_text': 'The word "meme" was first used in', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 50278, - 'text': '<|prompter|>', - }), - dict({ - 'id': 1276, - 'text': 'What', - }), - dict({ - 'id': 310, - 'text': ' is', - }), - dict({ - 'id': 247, - 'text': ' a', - }), - dict({ - 'id': 1167, - 'text': ' mem', - }), - dict({ - 'id': 70, - 'text': 'e', - }), - dict({ - 'id': 13, - 'text': ',', - }), - dict({ - 'id': 285, - 'text': ' and', - }), - dict({ - 'id': 752, - 'text': ' what', - }), - dict({ - 'id': 434, - 'text': "'s", - }), - dict({ - 'id': 253, - 'text': ' the', - }), - dict({ - 'id': 2892, - 'text': ' history', - }), - dict({ - 'id': 3212, - 'text': ' behind', - }), - dict({ - 'id': 436, - 'text': ' this', - }), - dict({ - 'id': 3159, - 'text': ' word', - }), - dict({ - 'id': 32, - 'text': '?', - }), - dict({ - 'id': 0, - 'text': '<|endoftext|>', - }), - dict({ - 'id': 50281, - 'text': '<|assistant|>', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 510, - 'special': False, - 'text': 'The', - }), - dict({ - 'id': 3159, - 'special': False, - 'text': ' word', - }), - dict({ - 'id': 346, - 'special': False, - 'text': ' "', - }), - dict({ - 'id': 6441, - 'special': False, - 'text': 'mem', - }), - dict({ - 'id': 70, - 'special': False, - 'text': 'e', - }), - dict({ - 'id': 3, - 'special': False, - 'text': '"', - }), - dict({ - 'id': 369, - 'special': False, - 'text': ' was', - }), - dict({ - 'id': 806, - 'special': False, - 'text': ' first', - }), - dict({ - 'id': 908, - 'special': False, - 'text': ' used', - }), - dict({ - 'id': 275, - 'special': False, - 'text': ' in', - }), - ]), - }), - 'generated_text': 'The word "meme" was first used in', - }), - ]) -# --- diff --git a/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox.json b/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox.json new file mode 100644 index 00000000..787704ce --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox.json @@ -0,0 +1,163 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5390625, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.002090454, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.3589859e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.0009455681, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.088012695, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12585449, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.017196655, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.49731445, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" +} diff --git a/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox_load.json b/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox_load.json new file mode 100644 index 00000000..47d6a77e --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox_load.json @@ -0,0 +1,654 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5488281, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.00207901, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.335144e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00097227097, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.0892334, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12463379, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.01737976, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.50341797, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5488281, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.00207901, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.335144e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00097227097, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.0892334, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12463379, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.01737976, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.50341797, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5488281, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.00207901, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.335144e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00097227097, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.0892334, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12463379, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.01737976, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.50341797, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5488281, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.00207901, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.335144e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00097227097, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.0892334, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12463379, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.01737976, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.50341797, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + } +] diff --git a/integration-tests/models/__snapshots__/test_flash_santacoder.ambr b/integration-tests/models/__snapshots__/test_flash_santacoder.ambr deleted file mode 100644 index 030820cb..00000000 --- a/integration-tests/models/__snapshots__/test_flash_santacoder.ambr +++ /dev/null @@ -1,472 +0,0 @@ -# serializer version: 1 -# name: test_flash_santacoder - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 563, - 'text': 'def', - }), - dict({ - 'id': 942, - 'text': ' print', - }), - dict({ - 'id': 62, - 'text': '_', - }), - dict({ - 'id': 7196, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 1241, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 258, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 942, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 372, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 7371, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 9956, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 8657, - 'special': False, - 'text': '!")', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1018, - 'special': False, - 'text': 'print', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World!") - - print - ''', - }) -# --- -# name: test_flash_santacoder_load - list([ - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 563, - 'text': 'def', - }), - dict({ - 'id': 942, - 'text': ' print', - }), - dict({ - 'id': 62, - 'text': '_', - }), - dict({ - 'id': 7196, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 1241, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 258, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 942, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 372, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 7371, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 9956, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 8657, - 'special': False, - 'text': '!")', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1018, - 'special': False, - 'text': 'print', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World!") - - print - ''', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 563, - 'text': 'def', - }), - dict({ - 'id': 942, - 'text': ' print', - }), - dict({ - 'id': 62, - 'text': '_', - }), - dict({ - 'id': 7196, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 1241, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 258, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 942, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 372, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 7371, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 9956, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 8657, - 'special': False, - 'text': '!")', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1018, - 'special': False, - 'text': 'print', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World!") - - print - ''', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 563, - 'text': 'def', - }), - dict({ - 'id': 942, - 'text': ' print', - }), - dict({ - 'id': 62, - 'text': '_', - }), - dict({ - 'id': 7196, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 1241, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 258, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 942, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 372, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 7371, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 9956, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 8657, - 'special': False, - 'text': '!")', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1018, - 'special': False, - 'text': 'print', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World!") - - print - ''', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 563, - 'text': 'def', - }), - dict({ - 'id': 942, - 'text': ' print', - }), - dict({ - 'id': 62, - 'text': '_', - }), - dict({ - 'id': 7196, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 1241, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 258, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 942, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 372, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 7371, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 9956, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 8657, - 'special': False, - 'text': '!")', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 185, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1018, - 'special': False, - 'text': 'print', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World!") - - print - ''', - }), - ]) -# --- diff --git a/integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder.json b/integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder.json new file mode 100644 index 00000000..0293e35a --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder.json @@ -0,0 +1,93 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 563, + "logprob": null, + "text": "def" + }, + { + "id": 942, + "logprob": -5.1367188, + "text": " print" + }, + { + "id": 62, + "logprob": -0.24450684, + "text": "_" + }, + { + "id": 7196, + "logprob": -6.9609375, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 1241, + "logprob": -0.9863281, + "special": false, + "text": "():" + }, + { + "id": 258, + "logprob": -0.21447754, + "special": false, + "text": "\n " + }, + { + "id": 942, + "logprob": -0.43701172, + "special": false, + "text": " print" + }, + { + "id": 372, + "logprob": -0.5361328, + "special": false, + "text": "(\"" + }, + { + "id": 7371, + "logprob": -0.44555664, + "special": false, + "text": "Hello" + }, + { + "id": 9956, + "logprob": -1.2412109, + "special": false, + "text": " World" + }, + { + "id": 8657, + "logprob": -0.7583008, + "special": false, + "text": "!\")" + }, + { + "id": 185, + "logprob": -0.76171875, + "special": false, + "text": "\n" + }, + { + "id": 185, + "logprob": -0.20837402, + "special": false, + "text": "\n" + }, + { + "id": 1018, + "logprob": -1.2470703, + "special": false, + "text": "print" + } + ] + }, + "generated_text": "():\n print(\"Hello World!\")\n\nprint" +} diff --git a/integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder_load.json b/integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder_load.json new file mode 100644 index 00000000..a03580b3 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_santacoder/test_flash_santacoder_load.json @@ -0,0 +1,374 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 563, + "logprob": null, + "text": "def" + }, + { + "id": 942, + "logprob": -5.1367188, + "text": " print" + }, + { + "id": 62, + "logprob": -0.24450684, + "text": "_" + }, + { + "id": 7196, + "logprob": -6.9609375, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 1241, + "logprob": -0.9863281, + "special": false, + "text": "():" + }, + { + "id": 258, + "logprob": -0.21362305, + "special": false, + "text": "\n " + }, + { + "id": 942, + "logprob": -0.44360352, + "special": false, + "text": " print" + }, + { + "id": 372, + "logprob": -0.54248047, + "special": false, + "text": "(\"" + }, + { + "id": 7371, + "logprob": -0.44555664, + "special": false, + "text": "Hello" + }, + { + "id": 9956, + "logprob": -1.2441406, + "special": false, + "text": " World" + }, + { + "id": 8657, + "logprob": -0.75878906, + "special": false, + "text": "!\")" + }, + { + "id": 185, + "logprob": -0.76171875, + "special": false, + "text": "\n" + }, + { + "id": 185, + "logprob": -0.2084961, + "special": false, + "text": "\n" + }, + { + "id": 1018, + "logprob": -1.2460938, + "special": false, + "text": "print" + } + ] + }, + "generated_text": "():\n print(\"Hello World!\")\n\nprint" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 563, + "logprob": null, + "text": "def" + }, + { + "id": 942, + "logprob": -5.1367188, + "text": " print" + }, + { + "id": 62, + "logprob": -0.24450684, + "text": "_" + }, + { + "id": 7196, + "logprob": -6.9609375, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 1241, + "logprob": -0.9863281, + "special": false, + "text": "():" + }, + { + "id": 258, + "logprob": -0.21362305, + "special": false, + "text": "\n " + }, + { + "id": 942, + "logprob": -0.44360352, + "special": false, + "text": " print" + }, + { + "id": 372, + "logprob": -0.54248047, + "special": false, + "text": "(\"" + }, + { + "id": 7371, + "logprob": -0.44555664, + "special": false, + "text": "Hello" + }, + { + "id": 9956, + "logprob": -1.2441406, + "special": false, + "text": " World" + }, + { + "id": 8657, + "logprob": -0.75878906, + "special": false, + "text": "!\")" + }, + { + "id": 185, + "logprob": -0.76171875, + "special": false, + "text": "\n" + }, + { + "id": 185, + "logprob": -0.2084961, + "special": false, + "text": "\n" + }, + { + "id": 1018, + "logprob": -1.2460938, + "special": false, + "text": "print" + } + ] + }, + "generated_text": "():\n print(\"Hello World!\")\n\nprint" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 563, + "logprob": null, + "text": "def" + }, + { + "id": 942, + "logprob": -5.1367188, + "text": " print" + }, + { + "id": 62, + "logprob": -0.24450684, + "text": "_" + }, + { + "id": 7196, + "logprob": -6.9609375, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 1241, + "logprob": -0.9863281, + "special": false, + "text": "():" + }, + { + "id": 258, + "logprob": -0.21362305, + "special": false, + "text": "\n " + }, + { + "id": 942, + "logprob": -0.44360352, + "special": false, + "text": " print" + }, + { + "id": 372, + "logprob": -0.54248047, + "special": false, + "text": "(\"" + }, + { + "id": 7371, + "logprob": -0.44555664, + "special": false, + "text": "Hello" + }, + { + "id": 9956, + "logprob": -1.2441406, + "special": false, + "text": " World" + }, + { + "id": 8657, + "logprob": -0.75878906, + "special": false, + "text": "!\")" + }, + { + "id": 185, + "logprob": -0.76171875, + "special": false, + "text": "\n" + }, + { + "id": 185, + "logprob": -0.2084961, + "special": false, + "text": "\n" + }, + { + "id": 1018, + "logprob": -1.2460938, + "special": false, + "text": "print" + } + ] + }, + "generated_text": "():\n print(\"Hello World!\")\n\nprint" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 563, + "logprob": null, + "text": "def" + }, + { + "id": 942, + "logprob": -5.1367188, + "text": " print" + }, + { + "id": 62, + "logprob": -0.24450684, + "text": "_" + }, + { + "id": 7196, + "logprob": -6.9609375, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 1241, + "logprob": -0.9863281, + "special": false, + "text": "():" + }, + { + "id": 258, + "logprob": -0.21362305, + "special": false, + "text": "\n " + }, + { + "id": 942, + "logprob": -0.44360352, + "special": false, + "text": " print" + }, + { + "id": 372, + "logprob": -0.54248047, + "special": false, + "text": "(\"" + }, + { + "id": 7371, + "logprob": -0.44555664, + "special": false, + "text": "Hello" + }, + { + "id": 9956, + "logprob": -1.2441406, + "special": false, + "text": " World" + }, + { + "id": 8657, + "logprob": -0.75878906, + "special": false, + "text": "!\")" + }, + { + "id": 185, + "logprob": -0.76171875, + "special": false, + "text": "\n" + }, + { + "id": 185, + "logprob": -0.2084961, + "special": false, + "text": "\n" + }, + { + "id": 1018, + "logprob": -1.2460938, + "special": false, + "text": "print" + } + ] + }, + "generated_text": "():\n print(\"Hello World!\")\n\nprint" + } +] diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder.ambr b/integration-tests/models/__snapshots__/test_flash_starcoder.ambr deleted file mode 100644 index e0f4b568..00000000 --- a/integration-tests/models/__snapshots__/test_flash_starcoder.ambr +++ /dev/null @@ -1,573 +0,0 @@ -# serializer version: 1 -# name: test_flash_starcoder - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 589, - 'text': 'def', - }), - dict({ - 'id': 1459, - 'text': ' print', - }), - dict({ - 'id': 81, - 'text': '_', - }), - dict({ - 'id': 7656, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 2262, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 284, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1459, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 440, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 8279, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 10896, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 657, - 'special': False, - 'text': '")', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 589, - 'special': False, - 'text': 'def', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World") - - def - ''', - }) -# --- -# name: test_flash_starcoder_default_params - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 12, - 'prefill': list([ - dict({ - 'id': 589, - 'text': 'def', - }), - dict({ - 'id': 1459, - 'text': ' print', - }), - dict({ - 'id': 81, - 'text': '_', - }), - dict({ - 'id': 7656, - 'text': 'hello', - }), - ]), - 'seed': 0, - 'tokens': list([ - dict({ - 'id': 2262, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 284, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 5741, - 'special': False, - 'text': ' logging', - }), - dict({ - 'id': 32, - 'special': False, - 'text': '.', - }), - dict({ - 'id': 1338, - 'special': False, - 'text': 'info', - }), - dict({ - 'id': 463, - 'special': False, - 'text': "('", - }), - dict({ - 'id': 8279, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 30, - 'special': False, - 'text': ',', - }), - dict({ - 'id': 10896, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 683, - 'special': False, - 'text': "')", - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 0, - 'special': True, - 'text': '<|endoftext|>', - }), - ]), - }), - 'generated_text': ''' - (): - logging.info('Hello, World') - <|endoftext|> - ''', - }) -# --- -# name: test_flash_starcoder_load - list([ - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 589, - 'text': 'def', - }), - dict({ - 'id': 1459, - 'text': ' print', - }), - dict({ - 'id': 81, - 'text': '_', - }), - dict({ - 'id': 7656, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 2262, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 284, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1459, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 440, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 8279, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 10896, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 657, - 'special': False, - 'text': '")', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 589, - 'special': False, - 'text': 'def', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World") - - def - ''', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 589, - 'text': 'def', - }), - dict({ - 'id': 1459, - 'text': ' print', - }), - dict({ - 'id': 81, - 'text': '_', - }), - dict({ - 'id': 7656, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 2262, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 284, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1459, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 440, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 8279, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 10896, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 657, - 'special': False, - 'text': '")', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 589, - 'special': False, - 'text': 'def', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World") - - def - ''', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 589, - 'text': 'def', - }), - dict({ - 'id': 1459, - 'text': ' print', - }), - dict({ - 'id': 81, - 'text': '_', - }), - dict({ - 'id': 7656, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 2262, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 284, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1459, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 440, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 8279, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 10896, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 657, - 'special': False, - 'text': '")', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 589, - 'special': False, - 'text': 'def', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World") - - def - ''', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 589, - 'text': 'def', - }), - dict({ - 'id': 1459, - 'text': ' print', - }), - dict({ - 'id': 81, - 'text': '_', - }), - dict({ - 'id': 7656, - 'text': 'hello', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 2262, - 'special': False, - 'text': '():', - }), - dict({ - 'id': 284, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 1459, - 'special': False, - 'text': ' print', - }), - dict({ - 'id': 440, - 'special': False, - 'text': '("', - }), - dict({ - 'id': 8279, - 'special': False, - 'text': 'Hello', - }), - dict({ - 'id': 10896, - 'special': False, - 'text': ' World', - }), - dict({ - 'id': 657, - 'special': False, - 'text': '")', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 203, - 'special': False, - 'text': ''' - - - ''', - }), - dict({ - 'id': 589, - 'special': False, - 'text': 'def', - }), - ]), - }), - 'generated_text': ''' - (): - print("Hello World") - - def - ''', - }), - ]) -# --- diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder.json b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder.json new file mode 100644 index 00000000..8505c1db --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder.json @@ -0,0 +1,93 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 1459, + "logprob": -5.6289062, + "text": " print" + }, + { + "id": 81, + "logprob": -1.6005859, + "text": "_" + }, + { + "id": 7656, + "logprob": -5.9921875, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 2262, + "logprob": -0.7705078, + "special": false, + "text": "():" + }, + { + "id": 284, + "logprob": -0.2590332, + "special": false, + "text": "\n " + }, + { + "id": 1459, + "logprob": -0.39379883, + "special": false, + "text": " print" + }, + { + "id": 440, + "logprob": -0.61376953, + "special": false, + "text": "(\"" + }, + { + "id": 8279, + "logprob": -0.47338867, + "special": false, + "text": "Hello" + }, + { + "id": 10896, + "logprob": -1.5068359, + "special": false, + "text": " World" + }, + { + "id": 657, + "logprob": -0.80810547, + "special": false, + "text": "\")" + }, + { + "id": 203, + "logprob": -0.7397461, + "special": false, + "text": "\n" + }, + { + "id": 203, + "logprob": -0.35229492, + "special": false, + "text": "\n" + }, + { + "id": 589, + "logprob": -1.0371094, + "special": false, + "text": "def" + } + ] + }, + "generated_text": "():\n print(\"Hello World\")\n\ndef" +} diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json new file mode 100644 index 00000000..21bb509b --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json @@ -0,0 +1,105 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 12, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 1459, + "logprob": -5.6289062, + "text": " print" + }, + { + "id": 81, + "logprob": -1.6005859, + "text": "_" + }, + { + "id": 7656, + "logprob": -5.9921875, + "text": "hello" + } + ], + "seed": 0, + "tokens": [ + { + "id": 2262, + "logprob": -0.7451172, + "special": false, + "text": "():" + }, + { + "id": 284, + "logprob": -0.21325684, + "special": false, + "text": "\n " + }, + { + "id": 5741, + "logprob": -5.734375, + "special": false, + "text": " logging" + }, + { + "id": 32, + "logprob": 0.0, + "special": false, + "text": "." + }, + { + "id": 1338, + "logprob": -0.3232422, + "special": false, + "text": "info" + }, + { + "id": 463, + "logprob": -1.0380859, + "special": false, + "text": "('" + }, + { + "id": 8279, + "logprob": -0.8378906, + "special": false, + "text": "Hello" + }, + { + "id": 30, + "logprob": -1.9501953, + "special": false, + "text": "," + }, + { + "id": 10896, + "logprob": -1.3476562, + "special": false, + "text": " World" + }, + { + "id": 683, + "logprob": -1.796875, + "special": false, + "text": "')" + }, + { + "id": 203, + "logprob": -0.9873047, + "special": false, + "text": "\n" + }, + { + "id": 0, + "logprob": -0.7495117, + "special": true, + "text": "<|endoftext|>" + } + ] + }, + "generated_text": "():\n logging.info('Hello, World')\n<|endoftext|>" +} diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_load.json b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_load.json new file mode 100644 index 00000000..0b3ad554 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_load.json @@ -0,0 +1,374 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 1459, + "logprob": -5.6289062, + "text": " print" + }, + { + "id": 81, + "logprob": -1.6005859, + "text": "_" + }, + { + "id": 7656, + "logprob": -5.9921875, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 2262, + "logprob": -0.7705078, + "special": false, + "text": "():" + }, + { + "id": 284, + "logprob": -0.2602539, + "special": false, + "text": "\n " + }, + { + "id": 1459, + "logprob": -0.39282227, + "special": false, + "text": " print" + }, + { + "id": 440, + "logprob": -0.6113281, + "special": false, + "text": "(\"" + }, + { + "id": 8279, + "logprob": -0.4765625, + "special": false, + "text": "Hello" + }, + { + "id": 10896, + "logprob": -1.5068359, + "special": false, + "text": " World" + }, + { + "id": 657, + "logprob": -0.8154297, + "special": false, + "text": "\")" + }, + { + "id": 203, + "logprob": -0.7319336, + "special": false, + "text": "\n" + }, + { + "id": 203, + "logprob": -0.35229492, + "special": false, + "text": "\n" + }, + { + "id": 589, + "logprob": -1.0380859, + "special": false, + "text": "def" + } + ] + }, + "generated_text": "():\n print(\"Hello World\")\n\ndef" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 1459, + "logprob": -5.6289062, + "text": " print" + }, + { + "id": 81, + "logprob": -1.6005859, + "text": "_" + }, + { + "id": 7656, + "logprob": -5.9921875, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 2262, + "logprob": -0.7705078, + "special": false, + "text": "():" + }, + { + "id": 284, + "logprob": -0.2602539, + "special": false, + "text": "\n " + }, + { + "id": 1459, + "logprob": -0.39282227, + "special": false, + "text": " print" + }, + { + "id": 440, + "logprob": -0.6113281, + "special": false, + "text": "(\"" + }, + { + "id": 8279, + "logprob": -0.4765625, + "special": false, + "text": "Hello" + }, + { + "id": 10896, + "logprob": -1.5068359, + "special": false, + "text": " World" + }, + { + "id": 657, + "logprob": -0.8154297, + "special": false, + "text": "\")" + }, + { + "id": 203, + "logprob": -0.7319336, + "special": false, + "text": "\n" + }, + { + "id": 203, + "logprob": -0.35229492, + "special": false, + "text": "\n" + }, + { + "id": 589, + "logprob": -1.0380859, + "special": false, + "text": "def" + } + ] + }, + "generated_text": "():\n print(\"Hello World\")\n\ndef" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 1459, + "logprob": -5.6289062, + "text": " print" + }, + { + "id": 81, + "logprob": -1.6005859, + "text": "_" + }, + { + "id": 7656, + "logprob": -5.9921875, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 2262, + "logprob": -0.7705078, + "special": false, + "text": "():" + }, + { + "id": 284, + "logprob": -0.2602539, + "special": false, + "text": "\n " + }, + { + "id": 1459, + "logprob": -0.39282227, + "special": false, + "text": " print" + }, + { + "id": 440, + "logprob": -0.6113281, + "special": false, + "text": "(\"" + }, + { + "id": 8279, + "logprob": -0.4765625, + "special": false, + "text": "Hello" + }, + { + "id": 10896, + "logprob": -1.5068359, + "special": false, + "text": " World" + }, + { + "id": 657, + "logprob": -0.8154297, + "special": false, + "text": "\")" + }, + { + "id": 203, + "logprob": -0.7319336, + "special": false, + "text": "\n" + }, + { + "id": 203, + "logprob": -0.35229492, + "special": false, + "text": "\n" + }, + { + "id": 589, + "logprob": -1.0380859, + "special": false, + "text": "def" + } + ] + }, + "generated_text": "():\n print(\"Hello World\")\n\ndef" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 1459, + "logprob": -5.6289062, + "text": " print" + }, + { + "id": 81, + "logprob": -1.6005859, + "text": "_" + }, + { + "id": 7656, + "logprob": -5.9921875, + "text": "hello" + } + ], + "seed": null, + "tokens": [ + { + "id": 2262, + "logprob": -0.7705078, + "special": false, + "text": "():" + }, + { + "id": 284, + "logprob": -0.2602539, + "special": false, + "text": "\n " + }, + { + "id": 1459, + "logprob": -0.39282227, + "special": false, + "text": " print" + }, + { + "id": 440, + "logprob": -0.6113281, + "special": false, + "text": "(\"" + }, + { + "id": 8279, + "logprob": -0.4765625, + "special": false, + "text": "Hello" + }, + { + "id": 10896, + "logprob": -1.5068359, + "special": false, + "text": " World" + }, + { + "id": 657, + "logprob": -0.8154297, + "special": false, + "text": "\")" + }, + { + "id": 203, + "logprob": -0.7319336, + "special": false, + "text": "\n" + }, + { + "id": 203, + "logprob": -0.35229492, + "special": false, + "text": "\n" + }, + { + "id": 589, + "logprob": -1.0380859, + "special": false, + "text": "def" + } + ] + }, + "generated_text": "():\n print(\"Hello World\")\n\ndef" + } +] diff --git a/integration-tests/models/__snapshots__/test_mt0_base.ambr b/integration-tests/models/__snapshots__/test_mt0_base.ambr deleted file mode 100644 index d7c6eaf6..00000000 --- a/integration-tests/models/__snapshots__/test_mt0_base.ambr +++ /dev/null @@ -1,306 +0,0 @@ -# serializer version: 1 -# name: test_mt0_base - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 5, - 'prefill': list([ - dict({ - 'id': 0, - 'text': '', - }), - ]), - 'seed': 0, - 'tokens': list([ - dict({ - 'id': 926, - 'special': False, - 'text': 'To', - }), - dict({ - 'id': 18295, - 'special': False, - 'text': ' sell', - }), - dict({ - 'id': 7868, - 'special': False, - 'text': ' things', - }), - dict({ - 'id': 260, - 'special': False, - 'text': '.', - }), - dict({ - 'id': 1, - 'special': True, - 'text': '', - }), - ]), - }), - 'generated_text': 'To sell things.', - }) -# --- -# name: test_mt0_base_all_params - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 10, - 'prefill': list([ - dict({ - 'id': 0, - 'text': '', - }), - ]), - 'seed': 0, - 'tokens': list([ - dict({ - 'id': 16017, - 'special': False, - 'text': 'blue', - }), - dict({ - 'id': 20495, - 'special': False, - 'text': ' sky', - }), - dict({ - 'id': 259, - 'special': False, - 'text': ' ', - }), - dict({ - 'id': 15484, - 'special': False, - 'text': 'appear', - }), - dict({ - 'id': 345, - 'special': False, - 'text': 'ed', - }), - dict({ - 'id': 288, - 'special': False, - 'text': ' to', - }), - dict({ - 'id': 35622, - 'special': False, - 'text': ' cloud', - }), - dict({ - 'id': 263, - 'special': False, - 'text': 's', - }), - dict({ - 'id': 14701, - 'special': False, - 'text': ' above', - }), - dict({ - 'id': 751, - 'special': False, - 'text': ' all', - }), - ]), - }), - 'generated_text': 'Why is the sky blue?blue sky appeared to clouds above all', - }) -# --- -# name: test_mt0_base_load - list([ - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 6, - 'prefill': list([ - dict({ - 'id': 0, - 'text': '', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 259, - 'special': False, - 'text': '', - }), - dict({ - 'id': 39261, - 'special': False, - 'text': 'Because', - }), - dict({ - 'id': 609, - 'special': False, - 'text': ' it', - }), - dict({ - 'id': 339, - 'special': False, - 'text': ' is', - }), - dict({ - 'id': 16017, - 'special': False, - 'text': ' blue', - }), - dict({ - 'id': 1, - 'special': True, - 'text': '', - }), - ]), - }), - 'generated_text': 'Because it is blue', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 6, - 'prefill': list([ - dict({ - 'id': 0, - 'text': '', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 259, - 'special': False, - 'text': '', - }), - dict({ - 'id': 39261, - 'special': False, - 'text': 'Because', - }), - dict({ - 'id': 609, - 'special': False, - 'text': ' it', - }), - dict({ - 'id': 339, - 'special': False, - 'text': ' is', - }), - dict({ - 'id': 16017, - 'special': False, - 'text': ' blue', - }), - dict({ - 'id': 1, - 'special': True, - 'text': '', - }), - ]), - }), - 'generated_text': 'Because it is blue', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 6, - 'prefill': list([ - dict({ - 'id': 0, - 'text': '', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 259, - 'special': False, - 'text': '', - }), - dict({ - 'id': 39261, - 'special': False, - 'text': 'Because', - }), - dict({ - 'id': 609, - 'special': False, - 'text': ' it', - }), - dict({ - 'id': 339, - 'special': False, - 'text': ' is', - }), - dict({ - 'id': 16017, - 'special': False, - 'text': ' blue', - }), - dict({ - 'id': 1, - 'special': True, - 'text': '', - }), - ]), - }), - 'generated_text': 'Because it is blue', - }), - dict({ - 'details': dict({ - 'best_of_sequences': None, - 'finish_reason': , - 'generated_tokens': 6, - 'prefill': list([ - dict({ - 'id': 0, - 'text': '', - }), - ]), - 'seed': None, - 'tokens': list([ - dict({ - 'id': 259, - 'special': False, - 'text': '', - }), - dict({ - 'id': 39261, - 'special': False, - 'text': 'Because', - }), - dict({ - 'id': 609, - 'special': False, - 'text': ' it', - }), - dict({ - 'id': 339, - 'special': False, - 'text': ' is', - }), - dict({ - 'id': 16017, - 'special': False, - 'text': ' blue', - }), - dict({ - 'id': 1, - 'special': True, - 'text': '', - }), - ]), - }), - 'generated_text': 'Because it is blue', - }), - ]) -# --- diff --git a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base.json b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base.json new file mode 100644 index 00000000..2a26e3db --- /dev/null +++ b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base.json @@ -0,0 +1,48 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 5, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": 0, + "tokens": [ + { + "id": 926, + "logprob": -4.3554688, + "special": false, + "text": "To" + }, + { + "id": 18295, + "logprob": -7.7734375, + "special": false, + "text": " sell" + }, + { + "id": 7868, + "logprob": -3.9257812, + "special": false, + "text": " things" + }, + { + "id": 260, + "logprob": -2.4179688, + "special": false, + "text": "." + }, + { + "id": 1, + "logprob": 0.0, + "special": true, + "text": "" + } + ] + }, + "generated_text": "To sell things." +} diff --git a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json new file mode 100644 index 00000000..fd77252d --- /dev/null +++ b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json @@ -0,0 +1,78 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": 0, + "tokens": [ + { + "id": 16017, + "logprob": -1.3505859, + "special": false, + "text": "blue" + }, + { + "id": 20495, + "logprob": -0.50439453, + "special": false, + "text": " sky" + }, + { + "id": 259, + "logprob": -1.2011719, + "special": false, + "text": " " + }, + { + "id": 15484, + "logprob": -2.8378906, + "special": false, + "text": "appear" + }, + { + "id": 345, + "logprob": -0.87597656, + "special": false, + "text": "ed" + }, + { + "id": 288, + "logprob": -1.8447266, + "special": false, + "text": " to" + }, + { + "id": 35622, + "logprob": -7.1445312, + "special": false, + "text": " cloud" + }, + { + "id": 263, + "logprob": -1.2929688, + "special": false, + "text": "s" + }, + { + "id": 14701, + "logprob": -3.0761719, + "special": false, + "text": " above" + }, + { + "id": 751, + "logprob": -4.4375, + "special": false, + "text": " all" + } + ] + }, + "generated_text": "Why is the sky blue?blue sky appeared to clouds above all" +} diff --git a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_load.json b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_load.json new file mode 100644 index 00000000..c9e552b6 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_load.json @@ -0,0 +1,218 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 6, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 259, + "logprob": -1.3789062, + "special": false, + "text": "" + }, + { + "id": 39261, + "logprob": -0.36279297, + "special": false, + "text": "Because" + }, + { + "id": 609, + "logprob": -1.0966797, + "special": false, + "text": " it" + }, + { + "id": 339, + "logprob": -0.8276367, + "special": false, + "text": " is" + }, + { + "id": 16017, + "logprob": -1.6845703, + "special": false, + "text": " blue" + }, + { + "id": 1, + "logprob": -0.72753906, + "special": true, + "text": "" + } + ] + }, + "generated_text": "Because it is blue" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 6, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 259, + "logprob": -1.3798828, + "special": false, + "text": "" + }, + { + "id": 39261, + "logprob": -0.36328125, + "special": false, + "text": "Because" + }, + { + "id": 609, + "logprob": -1.0947266, + "special": false, + "text": " it" + }, + { + "id": 339, + "logprob": -0.8286133, + "special": false, + "text": " is" + }, + { + "id": 16017, + "logprob": -1.6826172, + "special": false, + "text": " blue" + }, + { + "id": 1, + "logprob": -0.7290039, + "special": true, + "text": "" + } + ] + }, + "generated_text": "Because it is blue" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 6, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 259, + "logprob": -1.3789062, + "special": false, + "text": "" + }, + { + "id": 39261, + "logprob": -0.36279297, + "special": false, + "text": "Because" + }, + { + "id": 609, + "logprob": -1.0966797, + "special": false, + "text": " it" + }, + { + "id": 339, + "logprob": -0.8276367, + "special": false, + "text": " is" + }, + { + "id": 16017, + "logprob": -1.6845703, + "special": false, + "text": " blue" + }, + { + "id": 1, + "logprob": -0.72753906, + "special": true, + "text": "" + } + ] + }, + "generated_text": "Because it is blue" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 6, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 259, + "logprob": -1.3789062, + "special": false, + "text": "" + }, + { + "id": 39261, + "logprob": -0.36279297, + "special": false, + "text": "Because" + }, + { + "id": 609, + "logprob": -1.0966797, + "special": false, + "text": " it" + }, + { + "id": 339, + "logprob": -0.8276367, + "special": false, + "text": " is" + }, + { + "id": 16017, + "logprob": -1.6845703, + "special": false, + "text": " blue" + }, + { + "id": 1, + "logprob": -0.72753906, + "special": true, + "text": "" + } + ] + }, + "generated_text": "Because it is blue" + } +] diff --git a/integration-tests/models/test_bloom_560m.py b/integration-tests/models/test_bloom_560m.py index e13606f7..3c598c04 100644 --- a/integration-tests/models/test_bloom_560m.py +++ b/integration-tests/models/test_bloom_560m.py @@ -1,18 +1,20 @@ import pytest -from utils import health_check + +@pytest.fixture(scope="module") +def bloom_560_handle(launcher): + with launcher("bigscience/bloom-560m") as handle: + yield handle @pytest.fixture(scope="module") -def bloom_560(launcher): - with launcher("bigscience/bloom-560m") as client: - yield client +async def bloom_560(bloom_560_handle): + await bloom_560_handle.health(60) + return bloom_560_handle.client @pytest.mark.asyncio -async def test_bloom_560m(bloom_560, snapshot_test): - await health_check(bloom_560, 60) - +async def test_bloom_560m(bloom_560, response_snapshot): response = await bloom_560.generate( "Pour déguster un ortolan, il faut tout d'abord", max_new_tokens=10, @@ -21,13 +23,11 @@ async def test_bloom_560m(bloom_560, snapshot_test): ) assert response.details.generated_tokens == 10 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio -async def test_bloom_560m_all_params(bloom_560, snapshot_test): - await health_check(bloom_560, 60) - +async def test_bloom_560m_all_params(bloom_560, response_snapshot): response = await bloom_560.generate( "Pour déguster un ortolan, il faut tout d'abord", max_new_tokens=10, @@ -44,13 +44,11 @@ async def test_bloom_560m_all_params(bloom_560, snapshot_test): ) assert response.details.generated_tokens == 10 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio -async def test_bloom_560m_load(bloom_560, generate_load, snapshot_test): - await health_check(bloom_560, 60) - +async def test_bloom_560m_load(bloom_560, generate_load, response_snapshot): responses = await generate_load( bloom_560, "Pour déguster un ortolan, il faut tout d'abord", @@ -59,5 +57,6 @@ async def test_bloom_560m_load(bloom_560, generate_load, snapshot_test): ) assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) - assert snapshot_test(responses) + assert responses == response_snapshot diff --git a/integration-tests/models/test_bloom_560m_sharded.py b/integration-tests/models/test_bloom_560m_sharded.py index bfb70253..25f6b2d7 100644 --- a/integration-tests/models/test_bloom_560m_sharded.py +++ b/integration-tests/models/test_bloom_560m_sharded.py @@ -1,18 +1,20 @@ import pytest -from utils import health_check + +@pytest.fixture(scope="module") +def bloom_560m_sharded_handle(launcher): + with launcher("bigscience/bloom-560m", num_shard=2) as handle: + yield handle @pytest.fixture(scope="module") -def bloom_560m_sharded(launcher): - with launcher("bigscience/bloom-560m", num_shard=2) as client: - yield client +async def bloom_560m_sharded(bloom_560m_sharded_handle): + await bloom_560m_sharded_handle.health(60) + return bloom_560m_sharded_handle.client @pytest.mark.asyncio -async def test_bloom_560m_sharded(bloom_560m_sharded, snapshot_test): - await health_check(bloom_560m_sharded, 60) - +async def test_bloom_560m_sharded(bloom_560m_sharded, response_snapshot): response = await bloom_560m_sharded.generate( "Pour déguster un ortolan, il faut tout d'abord", max_new_tokens=10, @@ -21,15 +23,13 @@ async def test_bloom_560m_sharded(bloom_560m_sharded, snapshot_test): ) assert response.details.generated_tokens == 10 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio async def test_bloom_560m_sharded_load( - bloom_560m_sharded, generate_load, snapshot_test + bloom_560m_sharded, generate_load, response_snapshot ): - await health_check(bloom_560m_sharded, 60) - responses = await generate_load( bloom_560m_sharded, "Pour déguster un ortolan, il faut tout d'abord", @@ -38,5 +38,6 @@ async def test_bloom_560m_sharded_load( ) assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) - assert snapshot_test(responses) + assert responses == response_snapshot diff --git a/integration-tests/models/test_flash_llama.py b/integration-tests/models/test_flash_llama.py index 4d1f2bcf..37468455 100644 --- a/integration-tests/models/test_flash_llama.py +++ b/integration-tests/models/test_flash_llama.py @@ -1,30 +1,30 @@ import pytest -from utils import health_check + +@pytest.fixture(scope="module") +def flash_llama_handle(launcher): + with launcher("huggingface/llama-7b", num_shard=2) as handle: + yield handle @pytest.fixture(scope="module") -def flash_llama(launcher): - with launcher("huggingface/llama-7b", num_shard=2) as client: - yield client +async def flash_llama(flash_llama_handle): + await flash_llama_handle.health(120) + return flash_llama_handle.client @pytest.mark.asyncio @pytest.mark.private -async def test_flash_llama(flash_llama, snapshot_test): - await health_check(flash_llama, 120) - +async def test_flash_llama(flash_llama, response_snapshot): response = await flash_llama.generate("Test request", max_new_tokens=10) assert response.details.generated_tokens == 10 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio @pytest.mark.private -async def test_flash_llama_all_params(flash_llama, snapshot_test): - await health_check(flash_llama, 120) - +async def test_flash_llama_all_params(flash_llama, response_snapshot): response = await flash_llama.generate( "Test request", max_new_tokens=10, @@ -41,16 +41,15 @@ async def test_flash_llama_all_params(flash_llama, snapshot_test): ) assert response.details.generated_tokens == 10 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio @pytest.mark.private -async def test_flash_llama_load(flash_llama, generate_load, snapshot_test): - await health_check(flash_llama, 120) - +async def test_flash_llama_load(flash_llama, generate_load, response_snapshot): responses = await generate_load(flash_llama, "Test request", max_new_tokens=10, n=4) assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) - assert snapshot_test(responses) + assert responses == response_snapshot diff --git a/integration-tests/models/test_flash_neox.py b/integration-tests/models/test_flash_neox.py index 8c981028..56cbf270 100644 --- a/integration-tests/models/test_flash_neox.py +++ b/integration-tests/models/test_flash_neox.py @@ -1,31 +1,31 @@ import pytest -from utils import health_check + +@pytest.fixture(scope="module") +def flash_neox_handle(launcher): + with launcher("OpenAssistant/oasst-sft-1-pythia-12b", num_shard=2) as handle: + yield handle @pytest.fixture(scope="module") -def flash_neox(launcher): - with launcher("OpenAssistant/oasst-sft-1-pythia-12b", num_shard=2) as client: - yield client +async def flash_neox(flash_neox_handle): + await flash_neox_handle.health(240) + return flash_neox_handle.client @pytest.mark.asyncio -async def test_flash_neox(flash_neox, snapshot_test): - await health_check(flash_neox, 240) - +async def test_flash_neox(flash_neox, response_snapshot): response = await flash_neox.generate( "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", max_new_tokens=10, ) assert response.details.generated_tokens == 10 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio -async def test_flash_neox_load(flash_neox, generate_load, snapshot_test): - await health_check(flash_neox, 240) - +async def test_flash_neox_load(flash_neox, generate_load, response_snapshot): responses = await generate_load( flash_neox, "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", @@ -34,5 +34,6 @@ async def test_flash_neox_load(flash_neox, generate_load, snapshot_test): ) assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) - assert snapshot_test(responses) + assert responses == response_snapshot diff --git a/integration-tests/models/test_flash_santacoder.py b/integration-tests/models/test_flash_santacoder.py index 64a59d78..b0cb4522 100644 --- a/integration-tests/models/test_flash_santacoder.py +++ b/integration-tests/models/test_flash_santacoder.py @@ -1,32 +1,35 @@ import pytest -from utils import health_check + +@pytest.fixture(scope="module") +def flash_santacoder_handle(launcher): + with launcher("bigcode/santacoder") as handle: + yield handle @pytest.fixture(scope="module") -def flash_santacoder(launcher): - with launcher("bigcode/santacoder") as client: - yield client +async def flash_santacoder(flash_santacoder_handle): + await flash_santacoder_handle.health(240) + return flash_santacoder_handle.client @pytest.mark.asyncio -async def test_flash_santacoder(flash_santacoder, snapshot_test): - await health_check(flash_santacoder, 60) - +async def test_flash_santacoder(flash_santacoder, response_snapshot): response = await flash_santacoder.generate("def print_hello", max_new_tokens=10) assert response.details.generated_tokens == 10 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio -async def test_flash_santacoder_load(flash_santacoder, generate_load, snapshot_test): - await health_check(flash_santacoder, 60) - +async def test_flash_santacoder_load( + flash_santacoder, generate_load, response_snapshot +): responses = await generate_load( flash_santacoder, "def print_hello", max_new_tokens=10, n=4 ) assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) - assert snapshot_test(responses) + assert responses == response_snapshot diff --git a/integration-tests/models/test_flash_starcoder.py b/integration-tests/models/test_flash_starcoder.py index d43e92dc..4c7393a7 100644 --- a/integration-tests/models/test_flash_starcoder.py +++ b/integration-tests/models/test_flash_starcoder.py @@ -1,47 +1,46 @@ import pytest -from utils import health_check + +@pytest.fixture(scope="module") +def flash_starcoder_handle(launcher): + with launcher("bigcode/starcoder", num_shard=2) as handle: + yield handle @pytest.fixture(scope="module") -def flash_starcoder(launcher): - with launcher("bigcode/starcoder", num_shard=2) as client: - yield client +async def flash_starcoder(flash_starcoder_handle): + await flash_starcoder_handle.health(240) + return flash_starcoder_handle.client @pytest.mark.asyncio @pytest.mark.private -async def test_flash_starcoder(flash_starcoder, snapshot_test): - await health_check(flash_starcoder, 240) - +async def test_flash_starcoder(flash_starcoder, response_snapshot): response = await flash_starcoder.generate("def print_hello", max_new_tokens=10) assert response.details.generated_tokens == 10 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio @pytest.mark.private -async def test_flash_starcoder_default_params(flash_starcoder, snapshot_test): - await health_check(flash_starcoder, 240) - +async def test_flash_starcoder_default_params(flash_starcoder, response_snapshot): response = await flash_starcoder.generate( "def print_hello", max_new_tokens=60, temperature=0.2, top_p=0.95, seed=0 ) assert response.details.generated_tokens == 12 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio @pytest.mark.private -async def test_flash_starcoder_load(flash_starcoder, generate_load, snapshot_test): - await health_check(flash_starcoder, 240) - +async def test_flash_starcoder_load(flash_starcoder, generate_load, response_snapshot): responses = await generate_load( flash_starcoder, "def print_hello", max_new_tokens=10, n=4 ) assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) - assert snapshot_test(responses) + assert responses == response_snapshot diff --git a/integration-tests/models/test_mt0_base.py b/integration-tests/models/test_mt0_base.py index 7310a30f..15410f73 100644 --- a/integration-tests/models/test_mt0_base.py +++ b/integration-tests/models/test_mt0_base.py @@ -1,18 +1,20 @@ import pytest -from utils import health_check + +@pytest.fixture(scope="module") +def mt0_base_handle(launcher): + with launcher("bigscience/mt0-base") as handle: + yield handle @pytest.fixture(scope="module") -def mt0_base(launcher): - with launcher("bigscience/mt0-base") as client: - yield client +async def mt0_base(mt0_base_handle): + await mt0_base_handle.health(60) + return mt0_base_handle.client @pytest.mark.asyncio -async def test_mt0_base(mt0_base, snapshot_test): - await health_check(mt0_base, 60) - +async def test_mt0_base(mt0_base, response_snapshot): response = await mt0_base.generate( "Why is the sky blue?", max_new_tokens=10, @@ -21,13 +23,11 @@ async def test_mt0_base(mt0_base, snapshot_test): ) assert response.details.generated_tokens == 5 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio -async def test_mt0_base_all_params(mt0_base, snapshot_test): - await health_check(mt0_base, 60) - +async def test_mt0_base_all_params(mt0_base, response_snapshot): response = await mt0_base.generate( "Why is the sky blue?", max_new_tokens=10, @@ -44,13 +44,11 @@ async def test_mt0_base_all_params(mt0_base, snapshot_test): ) assert response.details.generated_tokens == 10 - assert snapshot_test(response) + assert response == response_snapshot @pytest.mark.asyncio -async def test_mt0_base_load(mt0_base, generate_load, snapshot_test): - await health_check(mt0_base, 60) - +async def test_mt0_base_load(mt0_base, generate_load, response_snapshot): responses = await generate_load( mt0_base, "Why is the sky blue?", @@ -59,5 +57,6 @@ async def test_mt0_base_load(mt0_base, generate_load, snapshot_test): ) assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) - assert snapshot_test(responses) + assert responses == response_snapshot diff --git a/integration-tests/models/utils.py b/integration-tests/models/utils.py deleted file mode 100644 index c47e4871..00000000 --- a/integration-tests/models/utils.py +++ /dev/null @@ -1,15 +0,0 @@ -import time - -from aiohttp import ClientConnectorError, ClientOSError, ServerDisconnectedError -from text_generation import AsyncClient - - -async def health_check(client: AsyncClient, timeout: int = 60): - assert timeout > 0 - for _ in range(timeout): - try: - await client.generate("test") - return - except (ClientConnectorError, ClientOSError, ServerDisconnectedError) as e: - time.sleep(1) - raise RuntimeError("Health check failed") diff --git a/integration-tests/requirements.txt b/integration-tests/requirements.txt index 9ecbb2ee..051730ff 100644 --- a/integration-tests/requirements.txt +++ b/integration-tests/requirements.txt @@ -1,5 +1,5 @@ syrupy -text-generation==0.5.1 +text-generation==0.5.2 pytest pytest-asyncio==0.17.2 docker \ No newline at end of file From 5a58226130fcd35774cd14d8c5ba638a4adb2bf4 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 16 May 2023 23:23:27 +0200 Subject: [PATCH 240/793] fix(server): fix decode token (#334) Fixes #333 --------- Co-authored-by: Nicolas Patry --- .github/workflows/build.yaml | 3 +- .github/workflows/tests.yaml | 3 +- Makefile | 2 +- integration-tests/conftest.py | 16 ++-- .../test_flash_llama_load.json | 36 ++++----- .../test_mt0_base/test_mt0_base.json | 2 +- .../test_mt0_base_all_params.json | 2 +- .../test_mt0_base/test_mt0_base_load.json | 32 ++++---- router/src/main.rs | 2 +- server/Makefile | 2 +- server/tests/models/test_model.py | 78 +++++++++++++++++++ server/tests/models/test_seq2seq_lm.py | 2 +- server/text_generation_server/models/bloom.py | 5 +- .../models/causal_lm.py | 71 ++++++++--------- .../models/flash_causal_lm.py | 77 +++++++++--------- .../models/flash_llama.py | 4 +- .../models/flash_neox.py | 2 +- .../models/flash_santacoder.py | 6 +- .../models/galactica.py | 16 ++-- .../text_generation_server/models/gpt_neox.py | 2 +- server/text_generation_server/models/model.py | 68 +++++----------- server/text_generation_server/models/opt.py | 2 +- .../models/santacoder.py | 20 ++--- .../models/seq2seq_lm.py | 66 ++++++++-------- server/text_generation_server/models/t5.py | 5 +- 25 files changed, 285 insertions(+), 239 deletions(-) create mode 100644 server/tests/models/test_model.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 79b3c777..9305b8e7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -213,12 +213,13 @@ jobs: sudo mount /dev/nvme1n1 ${{ env.DOCKER_VOLUME }} - name: Install run: | + pip install pytest-xdist make install-integration-tests - name: Run tests run: | export DOCKER_IMAGE=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }} export HUGGING_FACE_HUB_TOKEN=${{ secrets.HUGGING_FACE_HUB_TOKEN }} - pytest -s -vv integration-tests + pytest -s -vv -n 2 --dist loadfile integration-tests stop-runner: name: Stop self-hosted EC2 runner diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d9858a3b..7e5ba52c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -66,7 +66,8 @@ jobs: - name: Run server tests run: | pip install pytest - make python-server-tests + export HUGGING_FACE_HUB_TOKEN=${{ secrets.HUGGING_FACE_HUB_TOKEN }} + pytest -s -vv server/tests - name: Run Rust fmt run: | cargo fmt --check diff --git a/Makefile b/Makefile index 29c318fa..7309aaee 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ update-integration-tests: install-integration-tests pytest -s -vv --snapshot-update integration-tests python-server-tests: - HF_HUB_ENABLE_HF_TRANSFER=1 pytest server/tests + HF_HUB_ENABLE_HF_TRANSFER=1 pytest -s -vv -m "not private" server/tests python-client-tests: pytest clients/python/tests diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index ba1abca9..3086ecda 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -1,3 +1,4 @@ +import sys import subprocess import contextlib import pytest @@ -7,6 +8,7 @@ import json import math import time +import random from docker.errors import NotFound from typing import Optional, List, Dict @@ -205,10 +207,12 @@ def launcher(event_loop): def local_launcher( model_id: str, num_shard: Optional[int] = None, quantize: Optional[str] = None ): - port = 9999 - master_port = 19999 + port = random.randint(8000, 10_000) + master_port = random.randint(10_000, 20_000) - shard_uds_path = f"/tmp/{model_id.replace('/', '--')}-server" + shard_uds_path = ( + f"/tmp/tgi-tests-{model_id.split('/')[-1]}-{num_shard}-{quantize}-server" + ) args = [ "text-generation-launcher", @@ -236,7 +240,7 @@ def local_launcher( process.wait(60) launcher_output = process.stdout.read().decode("utf-8") - print(launcher_output) + print(launcher_output, file=sys.stderr) process.stdout.close() process.stderr.close() @@ -245,7 +249,7 @@ def local_launcher( def docker_launcher( model_id: str, num_shard: Optional[int] = None, quantize: Optional[str] = None ): - port = 9999 + port = random.randint(8000, 10_000) args = ["--model-id", model_id, "--env"] @@ -298,7 +302,7 @@ def docker_launcher( pass container_output = container.logs().decode("utf-8") - print(container_output) + print(container_output, file=sys.stderr) container.remove() diff --git a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json index 5a8ba217..9bbb5322 100644 --- a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json +++ b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json @@ -25,25 +25,25 @@ "tokens": [ { "id": 363, - "logprob": -1.5322266, + "logprob": -1.5380859, "special": false, "text": " for" }, { "id": 847, - "logprob": -2.5585938, + "logprob": -2.5859375, "special": false, "text": " /" }, { "id": 2754, - "logprob": -2.265625, + "logprob": -2.2695312, "special": false, "text": "api" }, { "id": 29914, - "logprob": -0.034088135, + "logprob": -0.03439331, "special": false, "text": "/" }, @@ -55,31 +55,31 @@ }, { "id": 29896, - "logprob": -0.36816406, + "logprob": -0.36694336, "special": false, "text": "1" }, { "id": 29914, - "logprob": -0.013191223, + "logprob": -0.013114929, "special": false, "text": "/" }, { "id": 16418, - "logprob": -3.15625, + "logprob": -3.1542969, "special": false, "text": "projects" }, { "id": 29914, - "logprob": -0.43774414, + "logprob": -0.43847656, "special": false, "text": "/" }, { "id": 29896, - "logprob": -1.9443359, + "logprob": -1.9433594, "special": false, "text": "1" } @@ -113,25 +113,25 @@ "tokens": [ { "id": 363, - "logprob": -1.5380859, + "logprob": -1.5322266, "special": false, "text": " for" }, { "id": 847, - "logprob": -2.5859375, + "logprob": -2.5585938, "special": false, "text": " /" }, { "id": 2754, - "logprob": -2.2695312, + "logprob": -2.265625, "special": false, "text": "api" }, { "id": 29914, - "logprob": -0.03439331, + "logprob": -0.034088135, "special": false, "text": "/" }, @@ -143,31 +143,31 @@ }, { "id": 29896, - "logprob": -0.36694336, + "logprob": -0.36816406, "special": false, "text": "1" }, { "id": 29914, - "logprob": -0.013114929, + "logprob": -0.013191223, "special": false, "text": "/" }, { "id": 16418, - "logprob": -3.1542969, + "logprob": -3.15625, "special": false, "text": "projects" }, { "id": 29914, - "logprob": -0.43847656, + "logprob": -0.43774414, "special": false, "text": "/" }, { "id": 29896, - "logprob": -1.9433594, + "logprob": -1.9443359, "special": false, "text": "1" } diff --git a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base.json b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base.json index 2a26e3db..c1cd24cd 100644 --- a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base.json +++ b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base.json @@ -16,7 +16,7 @@ "id": 926, "logprob": -4.3554688, "special": false, - "text": "To" + "text": " To" }, { "id": 18295, diff --git a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json index fd77252d..3e9f3d73 100644 --- a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json +++ b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json @@ -16,7 +16,7 @@ "id": 16017, "logprob": -1.3505859, "special": false, - "text": "blue" + "text": " blue" }, { "id": 20495, diff --git a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_load.json b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_load.json index c9e552b6..c0834ae1 100644 --- a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_load.json +++ b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_load.json @@ -15,37 +15,37 @@ "tokens": [ { "id": 259, - "logprob": -1.3789062, + "logprob": -1.3798828, "special": false, - "text": "" + "text": " " }, { "id": 39261, - "logprob": -0.36279297, + "logprob": -0.36328125, "special": false, "text": "Because" }, { "id": 609, - "logprob": -1.0966797, + "logprob": -1.0947266, "special": false, "text": " it" }, { "id": 339, - "logprob": -0.8276367, + "logprob": -0.8286133, "special": false, "text": " is" }, { "id": 16017, - "logprob": -1.6845703, + "logprob": -1.6826172, "special": false, "text": " blue" }, { "id": 1, - "logprob": -0.72753906, + "logprob": -0.7290039, "special": true, "text": "" } @@ -69,37 +69,37 @@ "tokens": [ { "id": 259, - "logprob": -1.3798828, + "logprob": -1.3789062, "special": false, - "text": "" + "text": " " }, { "id": 39261, - "logprob": -0.36328125, + "logprob": -0.36279297, "special": false, "text": "Because" }, { "id": 609, - "logprob": -1.0947266, + "logprob": -1.0966797, "special": false, "text": " it" }, { "id": 339, - "logprob": -0.8286133, + "logprob": -0.8276367, "special": false, "text": " is" }, { "id": 16017, - "logprob": -1.6826172, + "logprob": -1.6845703, "special": false, "text": " blue" }, { "id": 1, - "logprob": -0.7290039, + "logprob": -0.72753906, "special": true, "text": "" } @@ -125,7 +125,7 @@ "id": 259, "logprob": -1.3789062, "special": false, - "text": "" + "text": " " }, { "id": 39261, @@ -179,7 +179,7 @@ "id": 259, "logprob": -1.3789062, "special": false, - "text": "" + "text": " " }, { "id": 39261, diff --git a/router/src/main.rs b/router/src/main.rs index 5ad49003..82bf6ba8 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -146,7 +146,7 @@ fn main() -> Result<(), std::io::Error> { sha: None, pipeline_tag: None, }, - false => get_model_info(&tokenizer_name, &revision, authorization_token).await.unwrap_or({ + false => get_model_info(&tokenizer_name, &revision, authorization_token).await.unwrap_or_else(|| { tracing::warn!("Could not retrieve model info from the Hugging Face hub."); HubModelInfo { model_id: tokenizer_name.to_string(), sha: None, pipeline_tag: None } }), diff --git a/server/Makefile b/server/Makefile index 150d7e4a..6eb56c75 100644 --- a/server/Makefile +++ b/server/Makefile @@ -2,7 +2,7 @@ include Makefile-transformers include Makefile-flash-att unit-tests: - python -m pytest tests + pytest -s -vv -m "not private" tests gen-server: # Compile protos diff --git a/server/tests/models/test_model.py b/server/tests/models/test_model.py new file mode 100644 index 00000000..32bcd45f --- /dev/null +++ b/server/tests/models/test_model.py @@ -0,0 +1,78 @@ +import pytest +import torch + +from transformers import AutoTokenizer + +from text_generation_server.models import Model + + +def get_test_model(): + class TestModel(Model): + def batch_type(self): + raise NotImplementedError + + def generate_token(self, batch): + raise NotImplementedError + + tokenizer = AutoTokenizer.from_pretrained("huggingface/llama-7b") + + model = TestModel( + torch.nn.Linear(1, 1), tokenizer, False, torch.float32, torch.device("cpu") + ) + return model + + +@pytest.mark.private +def test_decode_streaming_english_spaces(): + model = get_test_model() + truth = "Hello here, this is a simple test" + all_input_ids = [15043, 1244, 29892, 445, 338, 263, 2560, 1243] + assert ( + all_input_ids == model.tokenizer(truth, add_special_tokens=False)["input_ids"] + ) + + decoded_text = "" + offset = 0 + token_offset = 0 + for i in range(len(all_input_ids)): + text, offset, token_offset = model.decode_token( + all_input_ids[: i + 1], offset, token_offset + ) + decoded_text += text + + assert decoded_text == truth + + +@pytest.mark.private +def test_decode_streaming_chinese_utf8(): + model = get_test_model() + truth = "我很感谢你的热情" + all_input_ids = [ + 30672, + 232, + 193, + 139, + 233, + 135, + 162, + 235, + 179, + 165, + 30919, + 30210, + 234, + 134, + 176, + 30993, + ] + + decoded_text = "" + offset = 0 + token_offset = 0 + for i in range(len(all_input_ids)): + text, offset, token_offset = model.decode_token( + all_input_ids[: i + 1], offset, token_offset + ) + decoded_text += text + + assert decoded_text == truth diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 65dafa50..ba769e75 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -149,7 +149,7 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) assert all([generation.generated_text is None for generation in generations]) assert all([len(generation.prefill_tokens) == 1 for generation in generations]) assert all([generation.token_id.item() == 259 for generation in generations]) - assert all([generation.token_text == "" for generation in generations]) + assert all([generation.token_text == " " for generation in generations]) assert generations[0].request_id == 0 diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 9029e954..1f324f77 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -56,7 +56,7 @@ def __init__( quantize: Optional[str] = None, ): super(BLOOM, self).__init__( - model_id=model_id, revision=revision, quantize=quantize, decode_buffer=1 + model_id=model_id, revision=revision, quantize=quantize ) @property @@ -104,14 +104,13 @@ def __init__( rank=rank, world_size=world_size, ) - self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( + model=model, tokenizer=tokenizer, requires_padding=True, dtype=dtype, device=device, - decode_buffer=1, rank=rank, world_size=world_size, ) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 0d521ac4..9d8ae254 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -35,8 +35,8 @@ class CausalLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] - offsets: List[Optional[int]] - token_offsets: List[Optional[int]] + prefix_offsets: List[int] + read_offsets: List[int] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -70,8 +70,8 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] requests_idx_mapping = {} # Parse batch @@ -81,8 +81,6 @@ def from_pb( for i, r in enumerate(pb.requests): requests_idx_mapping[r.id] = i inputs.append(r.inputs) - offsets.append(None) - token_offsets.append(None) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer @@ -102,6 +100,10 @@ def from_pb( truncation=True, max_length=max_truncation, ).to(device) + for _ in pb.requests: + input_len = tokenized_inputs["input_ids"].shape[1] + prefix_offsets.append(0) + read_offsets.append(input_len) input_lengths = tokenized_inputs["attention_mask"].sum(1) max_input_length = input_lengths.max() @@ -130,8 +132,8 @@ def from_pb( past_key_values=None, all_input_ids=list(all_input_ids), input_lengths=input_lengths.tolist(), - offsets=offsets, - token_offsets=token_offsets, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, max_input_length=max_input_length.item(), @@ -151,8 +153,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc # New values after filtering requests_idx_mapping = {} input_lengths = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] all_input_ids = [] max_input_length = 0 @@ -167,8 +169,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc requests_idx_mapping[r.id] = i keep_indices.append(idx) - offsets.append(self.offsets[idx]) - token_offsets.append(self.token_offsets[idx]) + prefix_offsets.append(self.prefix_offsets[idx]) + read_offsets.append(self.read_offsets[idx]) all_input_ids.append(self.all_input_ids[idx]) request_input_length = self.input_lengths[idx] @@ -225,8 +227,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc self.position_ids = position_ids self.all_input_ids = all_input_ids self.input_lengths = input_lengths - self.offsets = offsets - self.token_offsets = token_offsets + self.prefix_offsets = prefix_offsets + self.read_offsets = read_offsets self.next_token_choosers = next_token_choosers self.stopping_criterias = stopping_criterias self.max_input_length = max_input_length @@ -251,8 +253,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": requests = [] requests_idx_mapping = {} input_lengths = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] all_input_ids = [] next_token_choosers = [] stopping_criterias = [] @@ -270,8 +272,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": for i, batch in enumerate(batches): requests.extend(batch.requests) input_lengths.extend(batch.input_lengths) - offsets.extend(batch.offsets) - token_offsets.extend(batch.token_offsets) + prefix_offsets.extend(batch.prefix_offsets) + read_offsets.extend(batch.read_offsets) all_input_ids.extend(batch.all_input_ids) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -428,8 +430,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": past_key_values=past_key_values, all_input_ids=all_input_ids, input_lengths=input_lengths, - offsets=offsets, - token_offsets=token_offsets, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, max_input_length=max_input_length, @@ -448,7 +450,6 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, - decode_buffer: int = 3, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -463,25 +464,25 @@ def __init__( tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", truncation_side="left" ) - self.model = AutoModelForCausalLM.from_pretrained( + model = AutoModelForCausalLM.from_pretrained( model_id, revision=revision, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, load_in_8bit=quantize == "bitsandbytes", - ).eval() + ) tokenizer.pad_token_id = ( - self.model.config.pad_token_id - if self.model.config.pad_token_id is not None - else self.model.config.eos_token_id + model.config.pad_token_id + if model.config.pad_token_id is not None + else model.config.eos_token_id ) super(CausalLM, self).__init__( + model=model, tokenizer=tokenizer, requires_padding=True, dtype=dtype, device=device, - decode_buffer=decode_buffer, ) @property @@ -528,8 +529,8 @@ def generate_token( iterator = zip( batch.requests, batch.input_lengths, - batch.offsets, - batch.token_offsets, + batch.prefix_offsets, + batch.read_offsets, logits, batch.next_token_choosers, batch.stopping_criterias, @@ -540,8 +541,8 @@ def generate_token( for i, ( request, input_length, - offset, - token_offset, + prefix_offset, + read_offset, logits, next_token_chooser, stopping_criteria, @@ -559,8 +560,8 @@ def generate_token( # Generated token next_token_logprob = logprobs[-1, next_token_id] next_token_id_squeezed = next_token_id.squeeze() - next_token_text, offset, token_offset = self.decode_token( - all_input_ids[:, 0], offset, token_offset + next_token_text, prefix_offset, read_offset = self.decode_token( + all_input_ids[:, 0], prefix_offset, read_offset ) # Evaluate stopping criteria @@ -628,8 +629,8 @@ def generate_token( batch.input_ids[i, 0] = next_token_id batch.all_input_ids[i] = all_input_ids batch.input_lengths[i] = new_input_length - batch.offsets[i] = offset - batch.token_offsets[i] = token_offset + batch.prefix_offsets[i] = prefix_offset + batch.read_offsets[i] = read_offset batch.max_input_length = max(batch.max_input_length, new_input_length) # We finished all generations in the batch; there is no next batch diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 0a9fccca..aee0480d 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -52,8 +52,8 @@ class FlashCausalLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] - offsets: List[Optional[int]] - token_offsets: List[Optional[int]] + prefix_offsets: List[Optional[int]] + read_offsets: List[Optional[int]] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -82,8 +82,8 @@ def from_pb( max_seqlen = 0 input_lengths = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] all_input_ids = [] requests_idx_mapping = {} @@ -108,8 +108,8 @@ def from_pb( max_seqlen = max(max_seqlen, input_length) input_lengths.append(input_length) - offsets.append(None) - token_offsets.append(None) + prefix_offsets.append(0) + read_offsets.append(input_length) all_input_ids.append(tokenized_input) @@ -151,8 +151,8 @@ def from_pb( max_seqlen=max_seqlen, past_key_values=None, input_lengths=input_lengths, - offsets=offsets, - token_offsets=token_offsets, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=[], next_token_choosers=next_token_choosers, @@ -190,8 +190,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": all_input_ids_tensor = [] input_lengths = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] next_token_choosers = [] stopping_criterias = [] @@ -222,8 +222,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": all_input_ids_tensor.append(self.all_input_ids_tensor[idx]) input_lengths.append(request_input_length) - offsets.append(self.offsets[idx]) - token_offsets.append(self.token_offsets[idx]) + prefix_offsets.append(self.prefix_offsets[idx]) + read_offsets.append(self.read_offsets[idx]) next_token_choosers.append(self.next_token_choosers[idx]) @@ -269,8 +269,8 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": max_seqlen=max_seqlen, past_key_values=past_key_values, input_lengths=input_lengths, - offsets=offsets, - token_offsets=token_offsets, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=all_input_ids_tensor, next_token_choosers=next_token_choosers, @@ -302,8 +302,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch all_input_ids_tensor = [] input_lengths = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] next_token_choosers = [] stopping_criterias = [] @@ -347,8 +347,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch all_input_ids_tensor.extend(batch.all_input_ids_tensor) input_lengths.extend(batch.input_lengths) - offsets.extend(batch.offsets) - token_offsets.extend(batch.token_offsets) + prefix_offsets.extend(batch.prefix_offsets) + read_offsets.extend(batch.read_offsets) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -374,8 +374,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch max_seqlen=max_seqlen, past_key_values=past_key_values, input_lengths=input_lengths, - offsets=offsets, - token_offsets=token_offsets, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=all_input_ids_tensor, next_token_choosers=next_token_choosers, @@ -394,7 +394,6 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, - decode_buffer: int = 3, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -405,23 +404,19 @@ def __init__( tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", truncation_side="left" ) - self.model = ( - model_cls.from_pretrained( - model_id, - revision=revision, - torch_dtype=dtype, - load_in_8bit=quantize == "bitsandbytes", - ) - .eval() - .to(device) - ) + model = model_cls.from_pretrained( + model_id, + revision=revision, + torch_dtype=dtype, + load_in_8bit=quantize == "bitsandbytes", + ).to(device) super(FlashCausalLM, self).__init__( + model=model, tokenizer=tokenizer, requires_padding=False, dtype=dtype, device=device, - decode_buffer=decode_buffer, ) @property @@ -645,8 +640,8 @@ def generate_token( iterator = zip( batch.requests, batch.input_lengths, - batch.offsets, - batch.token_offsets, + batch.prefix_offsets, + batch.read_offsets, batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, @@ -659,8 +654,8 @@ def generate_token( for i, ( request, input_length, - offset, - token_offset, + prefix_offset, + read_offset, next_token_chooser, stopping_criteria, all_input_ids, @@ -675,10 +670,10 @@ def generate_token( all_input_ids.append(next_token_id) # Generated token - next_token_text, offset, token_offset = self.decode_token( + next_token_text, prefix_offset, read_offset = self.decode_token( all_input_ids, - offset, - token_offset, + prefix_offset, + read_offset, ) # Evaluate stopping criteria @@ -744,8 +739,8 @@ def generate_token( # Update values batch.input_lengths[i] = new_input_length - batch.offsets[i] = offset - batch.token_offsets[i] = token_offset + batch.prefix_offsets[i] = prefix_offset + batch.read_offsets[i] = read_offset batch.all_input_ids[i] = all_input_ids batch.max_seqlen = batch.max_seqlen + 1 cumulative_length += input_length diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index b775bd79..ebdbe206 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -64,9 +64,9 @@ def __init__( model = FlashLlamaForCausalLM(config) self.load_weights(model, filenames, quantize, device, dtype) - self.model = model.eval().to(device) super(FlashCausalLM, self).__init__( + model=model.to(device), tokenizer=tokenizer, requires_padding=False, dtype=dtype, @@ -189,9 +189,9 @@ def __init__( rank=rank, world_size=world_size, ) - self.model = model.eval().to(device) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( + model=model.to(device), tokenizer=tokenizer, requires_padding=False, dtype=dtype, diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 0924f107..cac40bab 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -73,9 +73,9 @@ def __init__( rank=rank, world_size=world_size, ) - self.model = model.eval().to(device) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( + model=model.to(device), tokenizer=tokenizer, requires_padding=False, dtype=dtype, diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 031a67eb..5dc31309 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -67,14 +67,13 @@ def __init__( dtype, config.architectures[0].startswith("GPT2"), ) - self.model = model.eval().to(device) super(FlashCausalLM, self).__init__( + model=model.to(device), tokenizer=tokenizer, requires_padding=False, dtype=dtype, device=device, - decode_buffer=1, ) @staticmethod @@ -213,16 +212,15 @@ def __init__( world_size=world_size, transpose=config.architectures[0].startswith("GPT2"), ) - self.model = model.eval().to(device) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( + model=model.to(device), tokenizer=tokenizer, requires_padding=False, dtype=dtype, device=device, rank=rank, world_size=world_size, - decode_buffer=1, ) @staticmethod diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index d1e5e841..24c37c19 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -94,8 +94,8 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] requests_idx_mapping = {} # Parse batch @@ -106,8 +106,6 @@ def from_pb( requests_idx_mapping[r.id] = i # Add escape_custom_split_sequence to the CausalLMBatch logic inputs.append(escape_custom_split_sequence(r.inputs)) - offsets.append(None) - token_offsets.append(None) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer @@ -127,6 +125,10 @@ def from_pb( truncation=True, max_length=max_truncation, ).to(device) + for _ in pb.requests: + input_len = tokenized_inputs["input_ids"].shape[1] + prefix_offsets.append(0) + read_offsets.append(input_len) input_lengths = tokenized_inputs["attention_mask"].sum(1) max_input_length = input_lengths.max() @@ -155,8 +157,8 @@ def from_pb( past_key_values=None, all_input_ids=list(all_input_ids), input_lengths=input_lengths.tolist(), - offsets=offsets, - token_offsets=token_offsets, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, max_input_length=max_input_length.item(), @@ -231,9 +233,9 @@ def __init__( rank=rank, world_size=world_size, ) - self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( + model=model, tokenizer=tokenizer, requires_padding=True, dtype=dtype, diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index f95e5be2..a10dfcb8 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -70,9 +70,9 @@ def __init__( rank=rank, world_size=world_size, ) - self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( + model=model, tokenizer=tokenizer, requires_padding=True, dtype=dtype, diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 03f14013..29bad321 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -13,23 +13,20 @@ class Model(ABC): def __init__( self, + model: torch.nn.Module, tokenizer: PreTrainedTokenizerBase, requires_padding: bool, dtype: torch.dtype, device: torch.device, - decode_buffer: int = 3, rank: int = 0, world_size: int = 1, ): - if decode_buffer < 1: - raise ValueError("decode_buffer must be >= 1") - + self.model = model.eval() self.tokenizer = tokenizer self.all_special_ids = set(tokenizer.all_special_ids) self.requires_padding = requires_padding self.dtype = dtype self.device = device - self.decode_buffer = decode_buffer self.rank = rank self.world_size = world_size self.check_initialized() @@ -54,52 +51,29 @@ def generate_token(self, batch: B) -> Tuple[List[GeneratedText], Optional[B]]: def decode_token( self, all_input_ids: List[int], - offset: Optional[int] = None, - token_offset: Optional[int] = None, - ) -> Tuple[str, Optional[int], Optional[int]]: + prefix_offset: int = 0, + read_offset: int = 0, + ) -> Tuple[str, int, int]: """Hack to hopefully support generate_stream for the maximum number of tokenizers""" - if all_input_ids[-1] in self.all_special_ids: - return ( - self.tokenizer.decode(all_input_ids[-1], skip_special_tokens=False), - None, - None, - ) - - if token_offset is None: - token_offset = len(all_input_ids) - self.decode_buffer - # left token buffer - if self.decode_buffer > 1: - # Decode token_offset token minus last one and token_offset tokens - raw_texts = self.tokenizer.batch_decode( - [all_input_ids[token_offset:-1], all_input_ids[token_offset:]], - skip_special_tokens=False, - ) - - # default offset is only the last token - offset = len(raw_texts[0]) - sequence_text = raw_texts[1] - else: - # Only decode the last token without using a token buffer - sequence_text = self.tokenizer.decode( - all_input_ids[-1], skip_special_tokens=False - ) - # no offset in this case - offset = 0 - else: - assert offset is not None - sequence_text = self.tokenizer.decode( - all_input_ids[token_offset:], - skip_special_tokens=False, - ) - # get text - token_text = sequence_text[offset:] + # The prefix text is necessary only to defeat cleanup algorithms in the decode + # which decide to add a space or not depending on the surrounding ids. + prefix_text = self.tokenizer.decode( + all_input_ids[prefix_offset:read_offset], skip_special_tokens=False + ) + new_text = self.tokenizer.decode( + all_input_ids[prefix_offset:], skip_special_tokens=False + ) - # if text is utf-8 - if token_text and token_text[-1] != "�": - return token_text, None, None + if len(new_text) > len(prefix_text) and not new_text.endswith("�"): + # utf-8 char at the end means it's a potential unfinished byte sequence + # from byte fallback tokenization. + # If it's in the middle, it's probably a real invalid id generated + # by the model + new_text = new_text[len(prefix_text) :] + return new_text, read_offset, len(all_input_ids) else: - return "", offset, token_offset + return "", prefix_offset, read_offset def check_initialized(self): uninitialized_parameters = [] diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 093cf70a..fdae795b 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -86,9 +86,9 @@ def __init__( rank=rank, world_size=world_size, ) - self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( + model=model, tokenizer=tokenizer, requires_padding=True, dtype=dtype, diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index 4bd56de1..23f89f48 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -46,24 +46,20 @@ def __init__( } ) - self.model = ( - AutoModelForCausalLM.from_pretrained( - model_id, - revision=revision, - torch_dtype=dtype, - load_in_8bit=quantize == "bitsandbytes", - trust_remote_code=True, # required - ) - .to(device) - .eval() - ) + model = AutoModelForCausalLM.from_pretrained( + model_id, + revision=revision, + torch_dtype=dtype, + load_in_8bit=quantize == "bitsandbytes", + trust_remote_code=True, # required + ).to(device) super(CausalLM, self).__init__( + model=model, tokenizer=tokenizer, requires_padding=True, dtype=dtype, device=device, - decode_buffer=1, ) def decode(self, generated_ids: List[int]) -> str: diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 84854f5d..4f55b22f 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -42,8 +42,8 @@ class Seq2SeqLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] decoder_input_lengths: List[int] - offsets: List[Optional[int]] - token_offsets: List[Optional[int]] + prefix_offsets: List[int] + read_offsets: List[int] # Generation helpers next_token_choosers: List[NextTokenChooser] @@ -79,8 +79,8 @@ def from_pb( stopping_criterias = [] decoder_input_lengths = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] requests_idx_mapping = {} # Parse batch @@ -91,8 +91,6 @@ def from_pb( inputs.append(r.inputs) requests_idx_mapping[r.id] = i decoder_input_lengths.append(1) - offsets.append(None) - token_offsets.append(None) next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer @@ -123,6 +121,9 @@ def from_pb( .repeat(len(pb.requests)) .view(-1, 1) ) + for _ in pb.requests: + prefix_offsets.append(0) + read_offsets.append(1) all_decoder_input_ids = decoder_input_ids.view(-1).split(1) max_tokens = len(inputs) * max_input_length + max_decode_tokens @@ -140,8 +141,8 @@ def from_pb( past_key_values=None, input_lengths=input_lengths.tolist(), decoder_input_lengths=decoder_input_lengths, - offsets=offsets, - token_offsets=token_offsets, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, max_input_length=max_input_length.item(), @@ -165,8 +166,8 @@ def filter( requests_idx_mapping = {} input_lengths = [] decoder_input_lengths = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] all_decoder_input_ids = [] @@ -184,8 +185,8 @@ def filter( requests_idx_mapping[r.id] = i keep_indices.append(idx) - offsets.append(self.offsets[idx]) - token_offsets.append(self.token_offsets[idx]) + prefix_offsets.append(self.prefix_offsets[idx]) + read_offsets.append(self.read_offsets[idx]) all_decoder_input_ids.append(self.all_decoder_input_ids[idx]) @@ -248,8 +249,8 @@ def filter( self.all_decoder_input_ids = all_decoder_input_ids self.input_lengths = input_lengths self.decoder_input_lengths = decoder_input_lengths - self.offsets = offsets - self.token_offsets = token_offsets + self.prefix_offsets = prefix_offsets + self.read_offsets = read_offsets self.next_token_choosers = next_token_choosers self.stopping_criterias = stopping_criterias self.max_input_length = max_input_length @@ -283,8 +284,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": all_decoder_input_ids = [] input_lengths = [] decoder_input_lengths = [] - offsets = [] - token_offsets = [] + prefix_offsets = [] + read_offsets = [] next_token_choosers = [] stopping_criterias = [] max_tokens = 0 @@ -306,8 +307,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": all_decoder_input_ids.extend(batch.all_decoder_input_ids) input_lengths.extend(batch.input_lengths) decoder_input_lengths.extend(batch.decoder_input_lengths) - offsets.extend(batch.offsets) - token_offsets.extend(batch.token_offsets) + prefix_offsets.extend(batch.prefix_offsets) + read_offsets.extend(batch.read_offsets) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) @@ -482,8 +483,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": past_key_values=past_key_values, input_lengths=input_lengths, decoder_input_lengths=decoder_input_lengths, - offsets=offsets, - token_offsets=token_offsets, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, max_input_length=max_input_length, @@ -502,7 +503,6 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, - decode_buffer: int = 3, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -514,24 +514,24 @@ def __init__( device = torch.device("cpu") dtype = torch.float32 - self.model = AutoModelForSeq2SeqLM.from_pretrained( + model = AutoModelForSeq2SeqLM.from_pretrained( model_id, revision=revision, torch_dtype=dtype, device_map="auto" if torch.cuda.is_available() else None, load_in_8bit=quantize == "bitsandbytes", - ).eval() + ) tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", truncation_side="left" ) - tokenizer.bos_token_id = self.model.config.decoder_start_token_id + tokenizer.bos_token_id = model.config.decoder_start_token_id super(Seq2SeqLM, self).__init__( + model=model, tokenizer=tokenizer, requires_padding=True, dtype=dtype, device=device, - decode_buffer=decode_buffer, ) @property @@ -608,8 +608,8 @@ def generate_token( iterator = zip( batch.requests, batch.input_lengths, - batch.offsets, - batch.token_offsets, + batch.prefix_offsets, + batch.read_offsets, batch.decoder_input_lengths, logits, batch.next_token_choosers, @@ -621,8 +621,8 @@ def generate_token( for i, ( request, input_length, - offset, - token_offset, + prefix_offset, + read_offset, decoder_input_length, logits, next_token_chooser, @@ -643,8 +643,8 @@ def generate_token( # Generated token next_token_logprob = logprobs[-1, next_token_id] next_token_id_squeezed = next_token_id.squeeze() - next_token_text, offset, token_offset = self.decode_token( - all_decoder_input_ids, offset, token_offset + next_token_text, prefix_offset, read_offset = self.decode_token( + all_decoder_input_ids, prefix_offset, read_offset ) # Evaluate stopping criteria @@ -702,8 +702,8 @@ def generate_token( batch.all_decoder_input_ids[i] = all_decoder_input_ids batch.input_lengths[i] = input_length batch.decoder_input_lengths[i] = new_decoder_input_length - batch.offsets[i] = offset - batch.token_offsets[i] = token_offset + batch.prefix_offsets[i] = prefix_offset + batch.read_offsets[i] = read_offset batch.max_input_length = max(batch.max_input_length, input_length) batch.max_decoder_input_length = max( batch.max_decoder_input_length, new_decoder_input_length diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 8e3826a4..b1ba2432 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -16,9 +16,6 @@ initialize_torch_distributed, weight_files, ) -from text_generation_server.utils.layers import ( - FastLinear, -) from transformers.models.t5.parallel_layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -73,9 +70,9 @@ def __init__( rank=rank, world_size=world_size, ) - self.model = model.eval() torch.distributed.barrier(group=self.process_group) super(Seq2SeqLM, self).__init__( + model=model, tokenizer=tokenizer, requires_padding=True, dtype=dtype, From 1ba78207e6457ad2f7eba6ca145465bdd6ca2adf Mon Sep 17 00:00:00 2001 From: Xin Yang <105740670+xyang16@users.noreply.github.com> Date: Mon, 22 May 2023 01:10:21 -0700 Subject: [PATCH 241/793] fix: set MODEL_ID in sagemaker-entrypoint script (#343) --- sagemaker-entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sagemaker-entrypoint.sh b/sagemaker-entrypoint.sh index 711e3721..1340f55a 100755 --- a/sagemaker-entrypoint.sh +++ b/sagemaker-entrypoint.sh @@ -4,6 +4,7 @@ if [[ -z "${HF_MODEL_ID}" ]]; then echo "HF_MODEL_ID must be set" exit 1 fi +export MODEL_ID="${HF_MODEL_ID}" if [[ -n "${HF_MODEL_REVISION}" ]]; then export REVISION="${HF_MODEL_REVISION}" @@ -17,4 +18,4 @@ if [[ -n "${HF_MODEL_QUANTIZE}" ]]; then export QUANTIZE="${HF_MODEL_QUANTIZE}" fi -text-generation-launcher --port 8080 \ No newline at end of file +text-generation-launcher --port 8080 From e649bf9a55753a3de4a64c1d6be2cc4ed32c260b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 22 May 2023 13:36:00 +0200 Subject: [PATCH 242/793] feat(server): Support BLOOMChat-176B (#348) (#351) @njhill, temporary workaround to be able to run our CI as secrets are not available to runners run by external contributors. I will ask around to see if there is a better way. Co-authored-by: Nick Hill --- server/text_generation_server/models/bloom.py | 7 +++++-- server/text_generation_server/models/causal_lm.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 1f324f77..390f0a0a 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -131,7 +131,10 @@ def load_weights( file, framework="pt", device=str(device) if quantize is None else "cpu" ) as f: for name in f.keys(): - full_name = f"transformer.{name}" + if name.startswith("transformer.") or name.startswith("lm_head."): + full_name = name + else: + full_name = f"transformer.{name}" module_name, param_name = full_name.rsplit(".", 1) module = model.get_submodule(module_name) @@ -157,7 +160,7 @@ def load_weights( # XXX: Hack for Rowlinear to add the bias only once. if rank != 0: tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding): + elif isinstance(module, TensorParallelEmbedding) or name == "lm_head.weight": size = slice_.get_shape()[0] block_size = size // world_size start = rank * block_size diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 9d8ae254..90b1e5ee 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -504,6 +504,7 @@ def forward( position_ids=position_ids, past_key_values=past_key_values, use_cache=True, + return_dict=True, ) return outputs.logits, outputs.past_key_values From 91d9beec90fba479a6751a4c8efae25adc28b001 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 22 May 2023 15:05:32 +0200 Subject: [PATCH 243/793] fix(server): fix init for flash causal lm (#352) Fixes #347 --- .../test_flash_neox/test_flash_neox.json | 144 ++-- .../test_flash_neox/test_flash_neox_load.json | 576 +++++---------- .../test_flash_neox.json | 163 +++++ .../test_flash_neox_load.json | 654 ++++++++++++++++++ integration-tests/models/test_flash_neox.py | 12 +- .../models/test_flash_neox_sharded.py | 39 ++ integration-tests/pytest.ini | 4 + server/pyproject.toml | 3 + .../custom_modeling/flash_neox_modeling.py | 4 +- .../models/flash_neox.py | 2 +- .../models/flash_santacoder.py | 2 +- server/text_generation_server/models/opt.py | 2 +- 12 files changed, 1110 insertions(+), 495 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox.json create mode 100644 integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox_load.json create mode 100644 integration-tests/models/test_flash_neox_sharded.py create mode 100644 integration-tests/pytest.ini diff --git a/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox.json b/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox.json index 787704ce..66ddbaef 100644 --- a/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox.json +++ b/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox.json @@ -7,157 +7,107 @@ { "id": 50278, "logprob": null, - "text": "<|prompter|>" + "text": "<|USER|>" }, { "id": 1276, - "logprob": -8.03125, + "logprob": -4.5546875, "text": "What" }, - { - "id": 310, - "logprob": -5.421875, - "text": " is" - }, - { - "id": 247, - "logprob": -2.1601562, - "text": " a" - }, - { - "id": 1167, - "logprob": -5.4609375, - "text": " mem" - }, - { - "id": 70, - "logprob": -0.005657196, - "text": "e" - }, - { - "id": 13, - "logprob": -7.28125, - "text": "," - }, - { - "id": 285, - "logprob": -0.2980957, - "text": " and" - }, - { - "id": 752, - "logprob": -2.1679688, - "text": " what" - }, { "id": 434, - "logprob": -5.6210938, + "logprob": -4.234375, "text": "'s" }, { - "id": 253, - "logprob": -0.81103516, - "text": " the" - }, - { - "id": 2892, - "logprob": -6.6640625, - "text": " history" - }, - { - "id": 3212, - "logprob": -2.265625, - "text": " behind" + "id": 634, + "logprob": -5.1054688, + "text": " your" }, { - "id": 436, - "logprob": -11.5078125, - "text": " this" + "id": 12315, + "logprob": -9.953125, + "text": " mood" }, { - "id": 3159, - "logprob": -2.1582031, - "text": " word" + "id": 3063, + "logprob": -4.0820312, + "text": " today" }, { "id": 32, - "logprob": -0.008720398, + "logprob": -0.15148926, "text": "?" }, { - "id": 0, - "logprob": -2.4726562, - "text": "<|endoftext|>" - }, - { - "id": 50281, - "logprob": -18.265625, - "text": "<|assistant|>" + "id": 50279, + "logprob": -0.27026367, + "text": "<|ASSISTANT|>" } ], "seed": null, "tokens": [ { - "id": 510, - "logprob": -0.63183594, + "id": 42, + "logprob": -0.88378906, "special": false, - "text": "The" + "text": "I" }, { - "id": 3159, - "logprob": -0.5390625, + "id": 1353, + "logprob": -0.94921875, "special": false, - "text": " word" + "text": "'m" }, { - "id": 346, - "logprob": -0.045684814, + "id": 417, + "logprob": -2.2402344, "special": false, - "text": " \"" + "text": " not" }, { - "id": 6441, - "logprob": -0.002090454, + "id": 2119, + "logprob": -0.3725586, "special": false, - "text": "mem" + "text": " sure" }, { - "id": 70, - "logprob": -1.3589859e-05, + "id": 13, + "logprob": -1.078125, "special": false, - "text": "e" + "text": "," }, { - "id": 3, - "logprob": -0.0009455681, + "id": 534, + "logprob": -0.67822266, "special": false, - "text": "\"" + "text": " which" }, { - "id": 369, - "logprob": -0.088012695, + "id": 310, + "logprob": -1.3837891, "special": false, - "text": " was" + "text": " is" }, { - "id": 806, - "logprob": -0.12585449, + "id": 253, + "logprob": -1.7050781, "special": false, - "text": " first" + "text": " the" }, { - "id": 908, - "logprob": -0.017196655, + "id": 1682, + "logprob": -0.052001953, "special": false, - "text": " used" + "text": " best" }, { - "id": 275, - "logprob": -0.49731445, + "id": 1039, + "logprob": -2.0390625, "special": false, - "text": " in" + "text": " way" } ] }, - "generated_text": "The word \"meme\" was first used in" + "generated_text": "I'm not sure, which is the best way" } diff --git a/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox_load.json b/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox_load.json index 47d6a77e..5ef6b3a2 100644 --- a/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox_load.json +++ b/integration-tests/models/__snapshots__/test_flash_neox/test_flash_neox_load.json @@ -8,159 +8,109 @@ { "id": 50278, "logprob": null, - "text": "<|prompter|>" + "text": "<|USER|>" }, { "id": 1276, - "logprob": -8.03125, + "logprob": -4.5546875, "text": "What" }, - { - "id": 310, - "logprob": -5.421875, - "text": " is" - }, - { - "id": 247, - "logprob": -2.1601562, - "text": " a" - }, - { - "id": 1167, - "logprob": -5.4609375, - "text": " mem" - }, - { - "id": 70, - "logprob": -0.005657196, - "text": "e" - }, - { - "id": 13, - "logprob": -7.28125, - "text": "," - }, - { - "id": 285, - "logprob": -0.2980957, - "text": " and" - }, - { - "id": 752, - "logprob": -2.1679688, - "text": " what" - }, { "id": 434, - "logprob": -5.6210938, + "logprob": -4.234375, "text": "'s" }, { - "id": 253, - "logprob": -0.81103516, - "text": " the" - }, - { - "id": 2892, - "logprob": -6.6640625, - "text": " history" - }, - { - "id": 3212, - "logprob": -2.265625, - "text": " behind" + "id": 634, + "logprob": -5.21875, + "text": " your" }, { - "id": 436, - "logprob": -11.5078125, - "text": " this" + "id": 12315, + "logprob": -9.9375, + "text": " mood" }, { - "id": 3159, - "logprob": -2.1582031, - "text": " word" + "id": 3063, + "logprob": -4.1015625, + "text": " today" }, { "id": 32, - "logprob": -0.008720398, + "logprob": -0.15319824, "text": "?" }, { - "id": 0, - "logprob": -2.4726562, - "text": "<|endoftext|>" - }, - { - "id": 50281, - "logprob": -18.265625, - "text": "<|assistant|>" + "id": 50279, + "logprob": -0.2614746, + "text": "<|ASSISTANT|>" } ], "seed": null, "tokens": [ { - "id": 510, - "logprob": -0.63183594, + "id": 42, + "logprob": -0.8886719, "special": false, - "text": "The" + "text": "I" }, { - "id": 3159, - "logprob": -0.5488281, + "id": 1353, + "logprob": -0.98046875, "special": false, - "text": " word" + "text": "'m" }, { - "id": 346, - "logprob": -0.045684814, + "id": 417, + "logprob": -2.2265625, "special": false, - "text": " \"" + "text": " not" }, { - "id": 6441, - "logprob": -0.00207901, + "id": 2119, + "logprob": -0.3479004, "special": false, - "text": "mem" + "text": " sure" }, { - "id": 70, - "logprob": -1.335144e-05, + "id": 13, + "logprob": -1.0117188, "special": false, - "text": "e" + "text": "," }, { - "id": 3, - "logprob": -0.00097227097, + "id": 534, + "logprob": -0.67871094, "special": false, - "text": "\"" + "text": " which" }, { - "id": 369, - "logprob": -0.0892334, + "id": 310, + "logprob": -1.421875, "special": false, - "text": " was" + "text": " is" }, { - "id": 806, - "logprob": -0.12463379, + "id": 253, + "logprob": -1.7382812, "special": false, - "text": " first" + "text": " the" }, { - "id": 908, - "logprob": -0.01737976, + "id": 1682, + "logprob": -0.051330566, "special": false, - "text": " used" + "text": " best" }, { - "id": 275, - "logprob": -0.50341797, + "id": 1039, + "logprob": -2.0390625, "special": false, - "text": " in" + "text": " way" } ] }, - "generated_text": "The word \"meme\" was first used in" + "generated_text": "I'm not sure, which is the best way" }, { "details": { @@ -171,159 +121,109 @@ { "id": 50278, "logprob": null, - "text": "<|prompter|>" + "text": "<|USER|>" }, { "id": 1276, - "logprob": -8.03125, + "logprob": -4.5546875, "text": "What" }, - { - "id": 310, - "logprob": -5.421875, - "text": " is" - }, - { - "id": 247, - "logprob": -2.1601562, - "text": " a" - }, - { - "id": 1167, - "logprob": -5.4609375, - "text": " mem" - }, - { - "id": 70, - "logprob": -0.005657196, - "text": "e" - }, - { - "id": 13, - "logprob": -7.28125, - "text": "," - }, - { - "id": 285, - "logprob": -0.2980957, - "text": " and" - }, - { - "id": 752, - "logprob": -2.1679688, - "text": " what" - }, { "id": 434, - "logprob": -5.6210938, + "logprob": -4.234375, "text": "'s" }, { - "id": 253, - "logprob": -0.81103516, - "text": " the" - }, - { - "id": 2892, - "logprob": -6.6640625, - "text": " history" - }, - { - "id": 3212, - "logprob": -2.265625, - "text": " behind" + "id": 634, + "logprob": -5.1054688, + "text": " your" }, { - "id": 436, - "logprob": -11.5078125, - "text": " this" + "id": 12315, + "logprob": -9.953125, + "text": " mood" }, { - "id": 3159, - "logprob": -2.1582031, - "text": " word" + "id": 3063, + "logprob": -4.0820312, + "text": " today" }, { "id": 32, - "logprob": -0.008720398, + "logprob": -0.15148926, "text": "?" }, { - "id": 0, - "logprob": -2.4726562, - "text": "<|endoftext|>" - }, - { - "id": 50281, - "logprob": -18.265625, - "text": "<|assistant|>" + "id": 50279, + "logprob": -0.27026367, + "text": "<|ASSISTANT|>" } ], "seed": null, "tokens": [ { - "id": 510, - "logprob": -0.63183594, + "id": 42, + "logprob": -0.88378906, "special": false, - "text": "The" + "text": "I" }, { - "id": 3159, - "logprob": -0.5488281, + "id": 1353, + "logprob": -0.9819336, "special": false, - "text": " word" + "text": "'m" }, { - "id": 346, - "logprob": -0.045684814, + "id": 417, + "logprob": -2.2421875, "special": false, - "text": " \"" + "text": " not" }, { - "id": 6441, - "logprob": -0.00207901, + "id": 2119, + "logprob": -0.3474121, "special": false, - "text": "mem" + "text": " sure" }, { - "id": 70, - "logprob": -1.335144e-05, + "id": 13, + "logprob": -1.078125, "special": false, - "text": "e" + "text": "," }, { - "id": 3, - "logprob": -0.00097227097, + "id": 534, + "logprob": -0.69140625, "special": false, - "text": "\"" + "text": " which" }, { - "id": 369, - "logprob": -0.0892334, + "id": 310, + "logprob": -1.4072266, "special": false, - "text": " was" + "text": " is" }, { - "id": 806, - "logprob": -0.12463379, + "id": 253, + "logprob": -1.7041016, "special": false, - "text": " first" + "text": " the" }, { - "id": 908, - "logprob": -0.01737976, + "id": 1682, + "logprob": -0.053375244, "special": false, - "text": " used" + "text": " best" }, { - "id": 275, - "logprob": -0.50341797, + "id": 1039, + "logprob": -2.0351562, "special": false, - "text": " in" + "text": " way" } ] }, - "generated_text": "The word \"meme\" was first used in" + "generated_text": "I'm not sure, which is the best way" }, { "details": { @@ -334,159 +234,109 @@ { "id": 50278, "logprob": null, - "text": "<|prompter|>" + "text": "<|USER|>" }, { "id": 1276, - "logprob": -8.03125, + "logprob": -4.5546875, "text": "What" }, - { - "id": 310, - "logprob": -5.421875, - "text": " is" - }, - { - "id": 247, - "logprob": -2.1601562, - "text": " a" - }, - { - "id": 1167, - "logprob": -5.4609375, - "text": " mem" - }, - { - "id": 70, - "logprob": -0.005657196, - "text": "e" - }, - { - "id": 13, - "logprob": -7.28125, - "text": "," - }, - { - "id": 285, - "logprob": -0.2980957, - "text": " and" - }, - { - "id": 752, - "logprob": -2.1679688, - "text": " what" - }, { "id": 434, - "logprob": -5.6210938, + "logprob": -4.234375, "text": "'s" }, { - "id": 253, - "logprob": -0.81103516, - "text": " the" - }, - { - "id": 2892, - "logprob": -6.6640625, - "text": " history" - }, - { - "id": 3212, - "logprob": -2.265625, - "text": " behind" + "id": 634, + "logprob": -5.21875, + "text": " your" }, { - "id": 436, - "logprob": -11.5078125, - "text": " this" + "id": 12315, + "logprob": -9.9375, + "text": " mood" }, { - "id": 3159, - "logprob": -2.1582031, - "text": " word" + "id": 3063, + "logprob": -4.1015625, + "text": " today" }, { "id": 32, - "logprob": -0.008720398, + "logprob": -0.15319824, "text": "?" }, { - "id": 0, - "logprob": -2.4726562, - "text": "<|endoftext|>" - }, - { - "id": 50281, - "logprob": -18.265625, - "text": "<|assistant|>" + "id": 50279, + "logprob": -0.2614746, + "text": "<|ASSISTANT|>" } ], "seed": null, "tokens": [ { - "id": 510, - "logprob": -0.63183594, + "id": 42, + "logprob": -0.8886719, "special": false, - "text": "The" + "text": "I" }, { - "id": 3159, - "logprob": -0.5488281, + "id": 1353, + "logprob": -0.98046875, "special": false, - "text": " word" + "text": "'m" }, { - "id": 346, - "logprob": -0.045684814, + "id": 417, + "logprob": -2.2265625, "special": false, - "text": " \"" + "text": " not" }, { - "id": 6441, - "logprob": -0.00207901, + "id": 2119, + "logprob": -0.3479004, "special": false, - "text": "mem" + "text": " sure" }, { - "id": 70, - "logprob": -1.335144e-05, + "id": 13, + "logprob": -1.0117188, "special": false, - "text": "e" + "text": "," }, { - "id": 3, - "logprob": -0.00097227097, + "id": 534, + "logprob": -0.67871094, "special": false, - "text": "\"" + "text": " which" }, { - "id": 369, - "logprob": -0.0892334, + "id": 310, + "logprob": -1.421875, "special": false, - "text": " was" + "text": " is" }, { - "id": 806, - "logprob": -0.12463379, + "id": 253, + "logprob": -1.7382812, "special": false, - "text": " first" + "text": " the" }, { - "id": 908, - "logprob": -0.01737976, + "id": 1682, + "logprob": -0.051330566, "special": false, - "text": " used" + "text": " best" }, { - "id": 275, - "logprob": -0.50341797, + "id": 1039, + "logprob": -2.0390625, "special": false, - "text": " in" + "text": " way" } ] }, - "generated_text": "The word \"meme\" was first used in" + "generated_text": "I'm not sure, which is the best way" }, { "details": { @@ -497,158 +347,108 @@ { "id": 50278, "logprob": null, - "text": "<|prompter|>" + "text": "<|USER|>" }, { "id": 1276, - "logprob": -8.03125, + "logprob": -4.5546875, "text": "What" }, - { - "id": 310, - "logprob": -5.421875, - "text": " is" - }, - { - "id": 247, - "logprob": -2.1601562, - "text": " a" - }, - { - "id": 1167, - "logprob": -5.4609375, - "text": " mem" - }, - { - "id": 70, - "logprob": -0.005657196, - "text": "e" - }, - { - "id": 13, - "logprob": -7.28125, - "text": "," - }, - { - "id": 285, - "logprob": -0.2980957, - "text": " and" - }, - { - "id": 752, - "logprob": -2.1679688, - "text": " what" - }, { "id": 434, - "logprob": -5.6210938, + "logprob": -4.234375, "text": "'s" }, { - "id": 253, - "logprob": -0.81103516, - "text": " the" - }, - { - "id": 2892, - "logprob": -6.6640625, - "text": " history" - }, - { - "id": 3212, - "logprob": -2.265625, - "text": " behind" + "id": 634, + "logprob": -5.21875, + "text": " your" }, { - "id": 436, - "logprob": -11.5078125, - "text": " this" + "id": 12315, + "logprob": -9.9375, + "text": " mood" }, { - "id": 3159, - "logprob": -2.1582031, - "text": " word" + "id": 3063, + "logprob": -4.1015625, + "text": " today" }, { "id": 32, - "logprob": -0.008720398, + "logprob": -0.15319824, "text": "?" }, { - "id": 0, - "logprob": -2.4726562, - "text": "<|endoftext|>" - }, - { - "id": 50281, - "logprob": -18.265625, - "text": "<|assistant|>" + "id": 50279, + "logprob": -0.2614746, + "text": "<|ASSISTANT|>" } ], "seed": null, "tokens": [ { - "id": 510, - "logprob": -0.63183594, + "id": 42, + "logprob": -0.8886719, "special": false, - "text": "The" + "text": "I" }, { - "id": 3159, - "logprob": -0.5488281, + "id": 1353, + "logprob": -0.98046875, "special": false, - "text": " word" + "text": "'m" }, { - "id": 346, - "logprob": -0.045684814, + "id": 417, + "logprob": -2.2265625, "special": false, - "text": " \"" + "text": " not" }, { - "id": 6441, - "logprob": -0.00207901, + "id": 2119, + "logprob": -0.3479004, "special": false, - "text": "mem" + "text": " sure" }, { - "id": 70, - "logprob": -1.335144e-05, + "id": 13, + "logprob": -1.0117188, "special": false, - "text": "e" + "text": "," }, { - "id": 3, - "logprob": -0.00097227097, + "id": 534, + "logprob": -0.67871094, "special": false, - "text": "\"" + "text": " which" }, { - "id": 369, - "logprob": -0.0892334, + "id": 310, + "logprob": -1.421875, "special": false, - "text": " was" + "text": " is" }, { - "id": 806, - "logprob": -0.12463379, + "id": 253, + "logprob": -1.7382812, "special": false, - "text": " first" + "text": " the" }, { - "id": 908, - "logprob": -0.01737976, + "id": 1682, + "logprob": -0.051330566, "special": false, - "text": " used" + "text": " best" }, { - "id": 275, - "logprob": -0.50341797, + "id": 1039, + "logprob": -2.0390625, "special": false, - "text": " in" + "text": " way" } ] }, - "generated_text": "The word \"meme\" was first used in" + "generated_text": "I'm not sure, which is the best way" } ] diff --git a/integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox.json b/integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox.json new file mode 100644 index 00000000..787704ce --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox.json @@ -0,0 +1,163 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5390625, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.002090454, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.3589859e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.0009455681, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.088012695, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12585449, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.017196655, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.49731445, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" +} diff --git a/integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox_load.json b/integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox_load.json new file mode 100644 index 00000000..47d6a77e --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_neox_sharded/test_flash_neox_load.json @@ -0,0 +1,654 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5488281, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.00207901, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.335144e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00097227097, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.0892334, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12463379, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.01737976, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.50341797, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5488281, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.00207901, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.335144e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00097227097, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.0892334, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12463379, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.01737976, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.50341797, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5488281, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.00207901, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.335144e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00097227097, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.0892334, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12463379, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.01737976, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.50341797, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.03125, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1601562, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.4609375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005657196, + "text": "e" + }, + { + "id": 13, + "logprob": -7.28125, + "text": "," + }, + { + "id": 285, + "logprob": -0.2980957, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1679688, + "text": " what" + }, + { + "id": 434, + "logprob": -5.6210938, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.81103516, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6640625, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.265625, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.5078125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1582031, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008720398, + "text": "?" + }, + { + "id": 0, + "logprob": -2.4726562, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.265625, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.63183594, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5488281, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.045684814, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.00207901, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.335144e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00097227097, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.0892334, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12463379, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.01737976, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.50341797, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + } +] diff --git a/integration-tests/models/test_flash_neox.py b/integration-tests/models/test_flash_neox.py index 56cbf270..ca5b33c1 100644 --- a/integration-tests/models/test_flash_neox.py +++ b/integration-tests/models/test_flash_neox.py @@ -3,7 +3,7 @@ @pytest.fixture(scope="module") def flash_neox_handle(launcher): - with launcher("OpenAssistant/oasst-sft-1-pythia-12b", num_shard=2) as handle: + with launcher("stabilityai/stablelm-tuned-alpha-3b", num_shard=1) as handle: yield handle @@ -16,7 +16,7 @@ async def flash_neox(flash_neox_handle): @pytest.mark.asyncio async def test_flash_neox(flash_neox, response_snapshot): response = await flash_neox.generate( - "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", + "<|USER|>What's your mood today?<|ASSISTANT|>", max_new_tokens=10, ) @@ -28,12 +28,14 @@ async def test_flash_neox(flash_neox, response_snapshot): async def test_flash_neox_load(flash_neox, generate_load, response_snapshot): responses = await generate_load( flash_neox, - "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", + "<|USER|>What's your mood today?<|ASSISTANT|>", max_new_tokens=10, n=4, ) - assert len(responses) == 4 - assert all([r.generated_text == responses[0].generated_text for r in responses]) + generated_texts = [r.generated_text for r in responses] + + assert len(generated_texts) == 4 + assert generated_texts, all([text == generated_texts[0] for text in generated_texts]) assert responses == response_snapshot diff --git a/integration-tests/models/test_flash_neox_sharded.py b/integration-tests/models/test_flash_neox_sharded.py new file mode 100644 index 00000000..513aeaaf --- /dev/null +++ b/integration-tests/models/test_flash_neox_sharded.py @@ -0,0 +1,39 @@ +import pytest + + +@pytest.fixture(scope="module") +def flash_neox_sharded_handle(launcher): + with launcher("OpenAssistant/oasst-sft-1-pythia-12b", num_shard=2) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def flash_neox_sharded(flash_neox_sharded_handle): + await flash_neox_sharded_handle.health(240) + return flash_neox_sharded_handle.client + + +@pytest.mark.asyncio +async def test_flash_neox(flash_neox_sharded, response_snapshot): + response = await flash_neox_sharded.generate( + "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", + max_new_tokens=10, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +async def test_flash_neox_load(flash_neox_sharded, generate_load, response_snapshot): + responses = await generate_load( + flash_neox_sharded, + "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", + max_new_tokens=10, + n=4, + ) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) + + assert responses == response_snapshot diff --git a/integration-tests/pytest.ini b/integration-tests/pytest.ini new file mode 100644 index 00000000..485e6017 --- /dev/null +++ b/integration-tests/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +asyncio_mode = auto +markers = + private: marks tests as requiring an admin hf token (deselect with '-m "not private"') \ No newline at end of file diff --git a/server/pyproject.toml b/server/pyproject.toml index 22aa41db..b8e15230 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -35,6 +35,9 @@ bnb = ["bitsandbytes"] grpcio-tools = "^1.51.1" pytest = "^7.3.0" +[tool.pytest.ini_options] +markers = ["private: marks tests as requiring an admin hf token (deselect with '-m \"not private\"')"] + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 2c6b8da6..b7834157 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -362,7 +362,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs ) - model.post_load_weights(load_in_8bit) + model.post_load_weights("bitsandbytes" if load_in_8bit else None) return model def forward( @@ -466,7 +466,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): model = super(FlashGPTNeoXForCausalLM, cls).from_pretrained( pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs ) - model.post_load_weights(load_in_8bit) + model.post_load_weights("bitsandbytes" if load_in_8bit else None) return model def forward( diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index cac40bab..8c1c1a00 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -88,7 +88,7 @@ def __init__( def load_weights( model, filenames: List[str], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, rank: int, diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 5dc31309..1fbaf252 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -80,7 +80,7 @@ def __init__( def load_weights( model: FlashSantacoderForCausalLM, filenames: List[Path], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, transpose: bool, diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index fdae795b..8c676a51 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -101,7 +101,7 @@ def __init__( def load_weights( model, filenames: List[str], - quantize: bool, + quantize: Optional[str], device: torch.device, dtype: torch.dtype, rank: int, From 4f4c9c16655841ea3ccce69ff46b9778a097cb70 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 23 May 2023 12:15:54 +0200 Subject: [PATCH 244/793] fix(server): t5 cannot run in f16 (#356) Fix #349 --- server/text_generation_server/models/t5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index b1ba2432..2fd67574 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -40,7 +40,7 @@ def __init__( self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 else: device = torch.device("cpu") dtype = torch.float32 From 0a6494785c35ac1c0178ac03d36acd0b8c21de6f Mon Sep 17 00:00:00 2001 From: oOraph <13552058+oOraph@users.noreply.github.com> Date: Tue, 23 May 2023 16:49:11 +0200 Subject: [PATCH 245/793] fix(ci): fix security group (#359) # What does this PR do? Switch security group used for ci (open outbound rules) Signed-off-by: Raphael Co-authored-by: Raphael --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9305b8e7..61f7b593 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -30,7 +30,7 @@ jobs: EC2_AMI_ID: ami-03cfed9ea28f4b002 EC2_INSTANCE_TYPE: g5.12xlarge EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc - EC2_SECURITY_GROUP: sg-04d472c808f365022 + EC2_SECURITY_GROUP: sg-030175c435ac141d6 outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -244,4 +244,4 @@ jobs: mode: stop github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} From 5f67923cacf0e3e961359edb1bb653667e9eb8da Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 23 May 2023 17:42:19 +0200 Subject: [PATCH 246/793] feat: add nightly load testing (#358) --- .github/workflows/build.yaml | 4 +- .github/workflows/load_test.yaml | 108 +++++++++++++++++++++++++++++++ k6/load_test.js | 98 ---------------------------- load_tests/starcoder_load.js | 63 ++++++++++++++++++ 4 files changed, 173 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/load_test.yaml delete mode 100644 k6/load_test.js create mode 100644 load_tests/starcoder_load.js diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 61f7b593..9992e0a2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -148,8 +148,8 @@ jobs: DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }} tags: ${{ steps.meta.outputs.tags || steps.meta-pr.outputs.tags }} labels: ${{ steps.meta.outputs.labels || steps.meta-pr.outputs.labels }} - cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max - cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max + cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min + cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min # Sign the resulting Docker image digest except on PRs. # This will only write to the public Rekor transparency log when the Docker # repository is public to avoid leaking data. diff --git a/.github/workflows/load_test.yaml b/.github/workflows/load_test.yaml new file mode 100644 index 00000000..10e248e3 --- /dev/null +++ b/.github/workflows/load_test.yaml @@ -0,0 +1,108 @@ +name: Nightly load test + +on: + schedule: + - cron: '0 0 * * 1-5' + + pull_request: + paths: + - ".github/workflows/load_test.yaml" + branches: + - 'main' + +jobs: + start-runner: + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + env: + AWS_REGION: us-east-1 + EC2_AMI_ID: ami-03cfed9ea28f4b002 + EC2_INSTANCE_TYPE: g5.12xlarge + EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc + EC2_SECURITY_GROUP: sg-04d472c808f365022 + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + - name: Start EC2 runner + id: start-ec2-runner + uses: philschmid/philschmid-ec2-github-runner@main + with: + mode: start + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + ec2-image-id: ${{ env.EC2_AMI_ID }} + ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }} + subnet-id: ${{ env.EC2_SUBNET_ID }} + security-group-id: ${{ env.EC2_SECURITY_GROUP }} + aws-resource-tags: > # optional, requires additional permissions + [ + {"Key": "Name", "Value": "ec2-tgi-github-runner"}, + {"Key": "GitHubRepository", "Value": "${{ github.repository }}"} + ] + + load-tests: + concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + needs: start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + env: + DOCKER_VOLUME: /cache + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Prepare disks + run: | + sudo mkfs -t ext4 /dev/nvme1n1 + sudo mkdir ${{ env.DOCKER_VOLUME }} + sudo mount /dev/nvme1n1 ${{ env.DOCKER_VOLUME }} + + - name: Install k6 + run: | + curl https://github.com/grafana/k6/releases/download/v0.44.0/k6-v0.44.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1 + + - name: Start starcoder + run: | + docker run --name tgi-starcoder --rm --gpus all -p 3000:80 -v ${{ env.DOCKER_VOLUME }}:/data -e HUGGING_FACE_HUB_TOKEN=${{ secrets.HUGGING_FACE_HUB_TOKEN }} --pull always -d ghcr.io/huggingface/text-generation-inference:latest --model-id bigcode/starcoder --num-shard 2 --max-batch-total-tokens 32768 + sleep 10 + wget --timeout 10 --retry-on-http-error --waitretry=1 --tries=240 http://localhost:3000/health + + - name: Run k6 + run: | + ./k6 run load_tests/starcoder_load.js + + - name: Stop starcoder + if: ${{ always() }} + run: | + docker stop tgi-starcoder || true + + stop-runner: + name: Stop self-hosted EC2 runner + needs: + - start-runner + - load-tests + runs-on: ubuntu-latest + env: + AWS_REGION: us-east-1 + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + - name: Stop EC2 runner + uses: philschmid/philschmid-ec2-github-runner@main + with: + mode: stop + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + label: ${{ needs.start-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file diff --git a/k6/load_test.js b/k6/load_test.js deleted file mode 100644 index 516b5666..00000000 --- a/k6/load_test.js +++ /dev/null @@ -1,98 +0,0 @@ -import http from 'k6/http'; -import {check, sleep} from 'k6'; - -export const options = { - stages: [ - {duration: '1m', target: 50}, - {duration: '2m', target: 100}, - {duration: '1m', target: 0}, - ], - hosts: { - 'text-generation-inference.huggingface.co': '127.0.0.1:3000', - }, -}; -const SLEEP_DURATION = 1; - -function greedy_example(inputs, max_new_tokens, name) { - let body = JSON.stringify({ - inputs: inputs, - parameters: { - max_new_tokens: max_new_tokens, - do_sample: false, - } - }); - let params = { - headers: { - 'Content-Type': 'application/json', - }, - tags: { - name: name - } - }; - return http.post('http://text-generation-inference.huggingface.co/generate', body, params); -} - -function sample_example(inputs, max_new_tokens, name) { - let body = JSON.stringify({ - inputs: inputs, - parameters: { - max_new_tokens: max_new_tokens, - do_sample: true, - top_p: 0.9, - seed: 0 - } - }); - let params = { - headers: { - 'Content-Type': 'application/json', - }, - tags: { - name: name - } - }; - return http.post('http://text-generation-inference.huggingface.co/generate', body, params); -} - -export default function () { - const response_1 = sample_example('A "whatpu" is a small, furry animal native to Tanzania. An example of a sentence that uses the word whatpu is: We were traveling in Africa and we saw these very cute whatpus. To do a "farduddle" means to jump up and down really fast. An example of a sentence that uses the word farduddle is:', 32, 'example-1'); - check(response_1, { - 'is status 200': (r) => r.status === 200, - }); - sleep(SLEEP_DURATION); - - const response_2 = sample_example("A poem about the beauty of science by Alfred Edgar Brittle\\nTitle: The Magic Craft\\nIn the old times", 50, "example-2"); - check(response_2, { - 'is status 200': (r) => r.status === 200, - }); - sleep(SLEEP_DURATION); - - const response_3 = greedy_example("استخراج العدد العاملي في لغة بايثون: ", 30, "example-3"); - check(response_3, { - 'is status 200': (r) => r.status === 200, - }); - sleep(SLEEP_DURATION); - - const response_4 = sample_example("Pour déguster un ortolan, il faut tout d'abord", 32, "example-4"); - check(response_4, { - 'is status 200': (r) => r.status === 200, - }); - sleep(SLEEP_DURATION); - - const response_5 = sample_example("Traduce español de España a español de Argentina\nEl coche es rojo - el auto es rojo\nEl ordenador es nuevo - la computadora es nueva\nel boligrafo es negro -", 16, "example-5"); - check(response_5, { - 'is status 200': (r) => r.status === 200, - }); - sleep(SLEEP_DURATION); - - const response_6 = sample_example("Question: If I put cheese into the fridge, will it melt?\nAnswer:", 32, "example-6"); - check(response_6, { - 'is status 200': (r) => r.status === 200, - }); - sleep(SLEEP_DURATION); - - const response_7 = greedy_example("Question: Where does the Greek Goddess Persephone spend half of the year when she is not with her mother?\nAnswer:", 24, "example-7"); - check(response_7, { - 'is status 200': (r) => r.status === 200, - }); - sleep(SLEEP_DURATION); -} \ No newline at end of file diff --git a/load_tests/starcoder_load.js b/load_tests/starcoder_load.js new file mode 100644 index 00000000..76316b65 --- /dev/null +++ b/load_tests/starcoder_load.js @@ -0,0 +1,63 @@ +import {check} from 'k6'; +import http from 'k6/http'; +import {Trend} from 'k6/metrics'; + +const host = __ENV.HOST || '127.0.0.1:3000'; + +const totalTime = new Trend('total_time', true); +const validationTime = new Trend('validation_time', true); +const queueTime = new Trend('queue_time', true); +const inferenceTime = new Trend('inference_time', true); +const timePerToken = new Trend('time_per_token', true); + +const example = { + payload: JSON.stringify({ + inputs: '# This is a fibonacci function written in the Python programming language.' + + 'def fibonacci', + parameters: { + details: true, + max_new_tokens: 60, + temperature: 0.2, + top_p: 0.95, + seed: 0, + }, + }), + generated_tokens: 60 +}; + +export const options = { + thresholds: { + http_req_failed: ['rate==0'], + time_per_token: ['p(95)<90'], + queue_time: ['p(95)<1500'], + }, + scenarios: { + load_test: { + executor: 'constant-arrival-rate', + duration: '60s', + preAllocatedVUs: 100, + rate: 10, + timeUnit: '1s', + }, + }, +}; + +export default function () { + const headers = {'Content-Type': 'application/json'}; + const res = http.post(`http://${host}/generate`, example.payload, { + headers, + }); + + check(res, { + 'Post status is 200': (r) => res.status === 200, + 'Post response generated tokens': (r) => res.status === 200 && res.json().details.generated_tokens === example.generated_tokens, + }); + + if (res.status === 200) { + totalTime.add(res.headers["X-Total-Time"]); + validationTime.add(res.headers["X-Validation-Time"]); + queueTime.add(res.headers["X-Queue-Time"]); + inferenceTime.add(res.headers["X-Inference-Time"]); + timePerToken.add(res.headers["X-Time-Per-Token"]); + } +} \ No newline at end of file From 94377efa785870da02a5077024f9934147546dd2 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 23 May 2023 18:03:22 +0200 Subject: [PATCH 247/793] chore(sever): update requirements (#357) Fixes #338 --- server/poetry.lock | 724 +++++++++++++--------------------------- server/pyproject.toml | 2 +- server/requirements.txt | 28 +- 3 files changed, 238 insertions(+), 516 deletions(-) diff --git a/server/poetry.lock b/server/poetry.lock index b5c03a57..5d853ce2 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "accelerate" -version = "0.15.0" +version = "0.19.0" description = "Accelerate" category = "main" optional = true @@ -11,17 +11,17 @@ numpy = ">=1.17" packaging = ">=20.0" psutil = "*" pyyaml = "*" -torch = ">=1.4.0" +torch = ">=1.6.0" [package.extras] -dev = ["black (>=22.0,<23.0)", "datasets", "deepspeed (<0.7.0)", "evaluate", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "rich", "scikit-learn", "scipy", "tqdm", "transformers"] -quality = ["black (>=22.0,<23.0)", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)"] +dev = ["black (>=23.1,<24.0)", "datasets", "deepspeed", "evaluate", "hf-doc-builder (>=0.3.0)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "rich", "ruff (>=0.0.241)", "scikit-learn", "scipy", "tqdm", "transformers", "urllib3 (<2.0.0)"] +quality = ["black (>=23.1,<24.0)", "hf-doc-builder (>=0.3.0)", "ruff (>=0.0.241)", "urllib3 (<2.0.0)"] rich = ["rich"] sagemaker = ["sagemaker"] -test_dev = ["datasets", "deepspeed (<0.7.0)", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] +test_dev = ["datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] test_trackers = ["comet-ml", "tensorboard", "wandb"] -testing = ["datasets", "deepspeed (<0.7.0)", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] +testing = ["datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] [[package]] name = "backoff" @@ -41,7 +41,7 @@ python-versions = "*" [[package]] name = "certifi" -version = "2022.12.7" +version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -66,17 +66,6 @@ python-versions = ">=3.7" [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -[[package]] -name = "cmake" -version = "3.26.3" -description = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" -category = "main" -optional = true -python-versions = "*" - -[package.extras] -test = ["codecov (>=2.0.5)", "coverage (>=4.2)", "flake8 (>=3.0.4)", "path.py (>=11.5.0)", "pytest (>=3.0.3)", "pytest-cov (>=2.4.0)", "pytest-runner (>=2.9)", "pytest-virtualenv (>=1.7.0)", "scikit-build (>=0.10.0)", "setuptools (>=28.0.0)", "virtualenv (>=15.0.3)", "wheel"] - [[package]] name = "colorama" version = "0.4.6" @@ -124,7 +113,7 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "p [[package]] name = "fsspec" -version = "2023.4.0" +version = "2023.5.0" description = "File-system specification" category = "main" optional = false @@ -170,7 +159,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] [[package]] name = "grpc-interceptor" -version = "0.15.1" +version = "0.15.2" description = "Simplifies gRPC interceptors" category = "main" optional = false @@ -184,30 +173,30 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.54.0" +version = "1.55.0" description = "HTTP/2-based RPC framework" category = "main" optional = false python-versions = ">=3.7" [package.extras] -protobuf = ["grpcio-tools (>=1.54.0)"] +protobuf = ["grpcio-tools (>=1.55.0)"] [[package]] name = "grpcio-reflection" -version = "1.54.0" +version = "1.55.0" description = "Standard Protobuf Reflection Service for gRPC" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -grpcio = ">=1.54.0" +grpcio = ">=1.55.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.54.0" +version = "1.55.0" description = "Status proto mapping for gRPC" category = "main" optional = false @@ -215,19 +204,19 @@ python-versions = ">=3.6" [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.54.0" +grpcio = ">=1.55.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.54.0" +version = "1.55.0" description = "Protobuf code generator for gRPC" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -grpcio = ">=1.54.0" +grpcio = ">=1.55.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -297,14 +286,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "lit" -version = "16.0.2" -description = "A Software Testing Tool" -category = "main" -optional = true -python-versions = "*" - [[package]] name = "loguru" version = "0.6.0" @@ -365,130 +346,6 @@ category = "main" optional = true python-versions = ">=3.8" -[[package]] -name = "nvidia-cublas-cu11" -version = "11.10.3.66" -description = "CUBLAS native runtime libraries" -category = "main" -optional = true -python-versions = ">=3" - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cuda-cupti-cu11" -version = "11.7.101" -description = "CUDA profiling tools runtime libs." -category = "main" -optional = true -python-versions = ">=3" - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cuda-nvrtc-cu11" -version = "11.7.99" -description = "NVRTC native runtime libraries" -category = "main" -optional = true -python-versions = ">=3" - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cuda-runtime-cu11" -version = "11.7.99" -description = "CUDA Runtime native Libraries" -category = "main" -optional = true -python-versions = ">=3" - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cudnn-cu11" -version = "8.5.0.96" -description = "cuDNN runtime libraries" -category = "main" -optional = true -python-versions = ">=3" - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cufft-cu11" -version = "10.9.0.58" -description = "CUFFT native runtime libraries" -category = "main" -optional = true -python-versions = ">=3" - -[[package]] -name = "nvidia-curand-cu11" -version = "10.2.10.91" -description = "CURAND native runtime libraries" -category = "main" -optional = true -python-versions = ">=3" - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cusolver-cu11" -version = "11.4.0.1" -description = "CUDA solver native runtime libraries" -category = "main" -optional = true -python-versions = ">=3" - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cusparse-cu11" -version = "11.7.4.91" -description = "CUSPARSE native runtime libraries" -category = "main" -optional = true -python-versions = ">=3" - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-nccl-cu11" -version = "2.14.3" -description = "NVIDIA Collective Communication Library (NCCL) Runtime" -category = "main" -optional = true -python-versions = ">=3" - -[[package]] -name = "nvidia-nvtx-cu11" -version = "11.7.91" -description = "NVIDIA Tools Extension" -category = "main" -optional = true -python-versions = ">=3" - -[package.dependencies] -setuptools = "*" -wheel = "*" - [[package]] name = "opentelemetry-api" version = "1.15.0" @@ -638,7 +495,7 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.22.3" +version = "4.23.1" description = "" category = "main" optional = false @@ -684,17 +541,17 @@ python-versions = ">=3.6" [[package]] name = "requests" -version = "2.28.2" +version = "2.31.0" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=3.7, <4" +python-versions = ">=3.7" [package.dependencies] certifi = ">=2017.4.17" charset-normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" +urllib3 = ">=1.21.1,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] @@ -721,7 +578,7 @@ torch = ["torch (>=1.10)"] [[package]] name = "sentencepiece" -version = "0.1.98" +version = "0.1.99" description = "SentencePiece python wrapper" category = "main" optional = false @@ -729,7 +586,7 @@ python-versions = "*" [[package]] name = "setuptools" -version = "67.7.2" +version = "67.8.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false @@ -737,12 +594,12 @@ python-versions = ">=3.7" [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "sympy" -version = "1.11.1" +version = "1.12" description = "Computer algebra system (CAS) in Python" category = "main" optional = true @@ -774,7 +631,7 @@ python-versions = ">=3.7" [[package]] name = "torch" -version = "2.0.0" +version = "2.0.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" category = "main" optional = true @@ -784,19 +641,7 @@ python-versions = ">=3.8.0" filelock = "*" jinja2 = "*" networkx = "*" -nvidia-cublas-cu11 = {version = "11.10.3.66", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-cupti-cu11 = {version = "11.7.101", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-nvrtc-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-runtime-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cudnn-cu11 = {version = "8.5.0.96", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cufft-cu11 = {version = "10.9.0.58", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-curand-cu11 = {version = "10.2.10.91", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cusolver-cu11 = {version = "11.4.0.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cusparse-cu11 = {version = "11.7.4.91", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nccl-cu11 = {version = "2.14.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nvtx-cu11 = {version = "11.7.91", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} sympy = "*" -triton = {version = "2.0.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} typing-extensions = "*" [package.extras] @@ -819,24 +664,6 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] -[[package]] -name = "triton" -version = "2.0.0" -description = "A language and compiler for custom Deep Learning operations" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -cmake = "*" -filelock = "*" -lit = "*" -torch = "*" - -[package.extras] -tests = ["autopep8", "flake8", "isort", "numpy", "pytest", "scipy (>=1.7.1)"] -tutorials = ["matplotlib", "pandas", "tabulate"] - [[package]] name = "typer" version = "0.6.1" @@ -856,7 +683,7 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6. [[package]] name = "typing-extensions" -version = "4.5.0" +version = "4.6.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -864,27 +691,17 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.15" +version = "2.0.2" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "wheel" -version = "0.40.0" -description = "A built-package format for Python" -category = "main" -optional = true python-versions = ">=3.7" [package.extras] -test = ["pytest (>=6.0.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "win32-setctime" @@ -912,12 +729,12 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "f3ee14e1a7730c01677ea6f28d3c76fffcd244f3a1f96232275988fbb2d82d2c" +content-hash = "152c8b82717e2b802aee3427152ef3e37417fb95b73c3af2f448a381f51a6a8d" [metadata.files] accelerate = [ - {file = "accelerate-0.15.0-py3-none-any.whl", hash = "sha256:014833307424cd0a22f89815802e00653756257c45dfdba2453e52d428931c65"}, - {file = "accelerate-0.15.0.tar.gz", hash = "sha256:438e25a01afa6e3ffbd25353e76a68be49677c3050f10bfac7beafaf53503efc"}, + {file = "accelerate-0.19.0-py3-none-any.whl", hash = "sha256:2866b0bf9fff08f51e6384c95fa96725838b70f1988d1cce42e56b820d8a91dd"}, + {file = "accelerate-0.19.0.tar.gz", hash = "sha256:84920226b9e642e453ef37593ee55b956b08d8200dea4087c546c34e26157e76"}, ] backoff = [ {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, @@ -928,8 +745,8 @@ bitsandbytes = [ {file = "bitsandbytes-0.38.1.tar.gz", hash = "sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce"}, ] certifi = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, + {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, + {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, ] charset-normalizer = [ {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, @@ -1012,25 +829,6 @@ click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] -cmake = [ - {file = "cmake-3.26.3-py2.py3-none-macosx_10_10_universal2.macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:9d38ea5b4999f8f042a071bea3e213f085bac26d7ab54cb5a4c6a193c4baf132"}, - {file = "cmake-3.26.3-py2.py3-none-manylinux2010_i686.manylinux_2_12_i686.whl", hash = "sha256:6e5fcd1cfaac33d015e2709e0dd1b7ad352a315367012ac359c9adc062cf075b"}, - {file = "cmake-3.26.3-py2.py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:4d3185738a6405aa15801e684f8d589b00570da4cc676cb1b5bbc902e3023e53"}, - {file = "cmake-3.26.3-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b20f7f7ea316ce7bb158df0e3c3453cfab5048939f1291017d16a8a36ad33ae6"}, - {file = "cmake-3.26.3-py2.py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:46aa385e19c9e4fc95d7d6ce5ee0bbe0d69bdeac4e9bc95c61f78f3973c2f626"}, - {file = "cmake-3.26.3-py2.py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:71e1df5587ad860b9829211380c42fc90ef2413363f12805b1fa2d87769bf876"}, - {file = "cmake-3.26.3-py2.py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:543b6958d1615327f484a07ab041029b1740918a8baa336adc9f5f0cbcd8fbd8"}, - {file = "cmake-3.26.3-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1bc7b47456256bdcc41069f5c658f232bd6e15bf4796d115f6ec98800793daff"}, - {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:2ae3db2c2be50fdaf0c9f3a23b2206e9dcd55ca124f16486a841b939f50b595e"}, - {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_i686.whl", hash = "sha256:1798547b23b89030518c5668dc55aed0e1d01867cf91d7a94e15d33f62a56fd0"}, - {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:d3017a08e6ba53ec2486d89a7953a81d4c4a068fc9f29d83e209f295dd9c59f3"}, - {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_s390x.whl", hash = "sha256:a922a6f6c1580d0db17b0b75f82e619441dd43c7f1d6a35f7d27e709db48bdbb"}, - {file = "cmake-3.26.3-py2.py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:e0ed796530641c8a21a423f9bb7882117dbbeee11ec78dbc335402a678d937ae"}, - {file = "cmake-3.26.3-py2.py3-none-win32.whl", hash = "sha256:27a6fa1b97744311a7993d6a1e0ce14bd73696dab9ceb96701f1ec11edbd5053"}, - {file = "cmake-3.26.3-py2.py3-none-win_amd64.whl", hash = "sha256:cf910bbb488659d300c86b1dac77e44eeb0457bde2cf76a42d7e51f691544b21"}, - {file = "cmake-3.26.3-py2.py3-none-win_arm64.whl", hash = "sha256:24741a304ada699b339034958777d9a1472ac8ddb9b6194d74f814287ca091ae"}, - {file = "cmake-3.26.3.tar.gz", hash = "sha256:b54cde1f1c0573321b22382bd2ffaf5d08f65188572d128cd4867fb9669723c5"}, -] colorama = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -1048,118 +846,118 @@ filelock = [ {file = "filelock-3.12.0.tar.gz", hash = "sha256:fc03ae43288c013d2ea83c8597001b1129db351aad9c57fe2409327916b8e718"}, ] fsspec = [ - {file = "fsspec-2023.4.0-py3-none-any.whl", hash = "sha256:f398de9b49b14e9d84d2c2d11b7b67121bc072fe97b930c4e5668ac3917d8307"}, - {file = "fsspec-2023.4.0.tar.gz", hash = "sha256:bf064186cd8808f0b2f6517273339ba0a0c8fb1b7048991c28bc67f58b8b67cd"}, + {file = "fsspec-2023.5.0-py3-none-any.whl", hash = "sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a"}, + {file = "fsspec-2023.5.0.tar.gz", hash = "sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce"}, ] googleapis-common-protos = [ {file = "googleapis-common-protos-1.59.0.tar.gz", hash = "sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44"}, {file = "googleapis_common_protos-1.59.0-py2.py3-none-any.whl", hash = "sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f"}, ] grpc-interceptor = [ - {file = "grpc-interceptor-0.15.1.tar.gz", hash = "sha256:3efadbc9aead272ac7a360c75c4bd96233094c9a5192dbb51c6156246bd64ba0"}, - {file = "grpc_interceptor-0.15.1-py3-none-any.whl", hash = "sha256:1cc52c34b0d7ff34512fb7780742ecda37bf3caa18ecc5f33f09b4f74e96b276"}, + {file = "grpc-interceptor-0.15.2.tar.gz", hash = "sha256:5c984110af4fb77d03472ec0468f9c77ddaf798e190410fb7b7f1e76c60c96a4"}, + {file = "grpc_interceptor-0.15.2-py3-none-any.whl", hash = "sha256:596dac3cb709ffb6178a4873f5148e254c871c9069f0b11040189b257969490a"}, ] grpcio = [ - {file = "grpcio-1.54.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:a947d5298a0bbdd4d15671024bf33e2b7da79a70de600ed29ba7e0fef0539ebb"}, - {file = "grpcio-1.54.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:e355ee9da9c1c03f174efea59292b17a95e0b7b4d7d2a389265f731a9887d5a9"}, - {file = "grpcio-1.54.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:73c238ef6e4b64272df7eec976bb016c73d3ab5a6c7e9cd906ab700523d312f3"}, - {file = "grpcio-1.54.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c59d899ee7160638613a452f9a4931de22623e7ba17897d8e3e348c2e9d8d0b"}, - {file = "grpcio-1.54.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48cb7af77238ba16c77879009003f6b22c23425e5ee59cb2c4c103ec040638a5"}, - {file = "grpcio-1.54.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2262bd3512ba9e9f0e91d287393df6f33c18999317de45629b7bd46c40f16ba9"}, - {file = "grpcio-1.54.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:224166f06ccdaf884bf35690bf4272997c1405de3035d61384ccb5b25a4c1ca8"}, - {file = "grpcio-1.54.0-cp310-cp310-win32.whl", hash = "sha256:ed36e854449ff6c2f8ee145f94851fe171298e1e793f44d4f672c4a0d78064e7"}, - {file = "grpcio-1.54.0-cp310-cp310-win_amd64.whl", hash = "sha256:27fb030a4589d2536daec5ff5ba2a128f4f155149efab578fe2de2cb21596d3d"}, - {file = "grpcio-1.54.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:f4a7dca8ccd8023d916b900aa3c626f1bd181bd5b70159479b142f957ff420e4"}, - {file = "grpcio-1.54.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:1209d6b002b26e939e4c8ea37a3d5b4028eb9555394ea69fb1adbd4b61a10bb8"}, - {file = "grpcio-1.54.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:860fcd6db7dce80d0a673a1cc898ce6bc3d4783d195bbe0e911bf8a62c93ff3f"}, - {file = "grpcio-1.54.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3930669c9e6f08a2eed824738c3d5699d11cd47a0ecc13b68ed11595710b1133"}, - {file = "grpcio-1.54.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62117486460c83acd3b5d85c12edd5fe20a374630475388cfc89829831d3eb79"}, - {file = "grpcio-1.54.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e3e526062c690517b42bba66ffe38aaf8bc99a180a78212e7b22baa86902f690"}, - {file = "grpcio-1.54.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ebff0738be0499d7db74d20dca9f22a7b27deae31e1bf92ea44924fd69eb6251"}, - {file = "grpcio-1.54.0-cp311-cp311-win32.whl", hash = "sha256:21c4a1aae861748d6393a3ff7867473996c139a77f90326d9f4104bebb22d8b8"}, - {file = "grpcio-1.54.0-cp311-cp311-win_amd64.whl", hash = "sha256:3db71c6f1ab688d8dfc102271cedc9828beac335a3a4372ec54b8bf11b43fd29"}, - {file = "grpcio-1.54.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:960b176e0bb2b4afeaa1cd2002db1e82ae54c9b6e27ea93570a42316524e77cf"}, - {file = "grpcio-1.54.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:d8ae6e0df3a608e99ee1acafaafd7db0830106394d54571c1ece57f650124ce9"}, - {file = "grpcio-1.54.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:c33744d0d1a7322da445c0fe726ea6d4e3ef2dfb0539eadf23dce366f52f546c"}, - {file = "grpcio-1.54.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d109df30641d050e009105f9c9ca5a35d01e34d2ee2a4e9c0984d392fd6d704"}, - {file = "grpcio-1.54.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:775a2f70501370e5ba54e1ee3464413bff9bd85bd9a0b25c989698c44a6fb52f"}, - {file = "grpcio-1.54.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c55a9cf5cba80fb88c850915c865b8ed78d5e46e1f2ec1b27692f3eaaf0dca7e"}, - {file = "grpcio-1.54.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1fa7d6ddd33abbd3c8b3d7d07c56c40ea3d1891ce3cd2aa9fa73105ed5331866"}, - {file = "grpcio-1.54.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ed3d458ded32ff3a58f157b60cc140c88f7ac8c506a1c567b2a9ee8a2fd2ce54"}, - {file = "grpcio-1.54.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:5942a3e05630e1ef5b7b5752e5da6582460a2e4431dae603de89fc45f9ec5aa9"}, - {file = "grpcio-1.54.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:125ed35aa3868efa82eabffece6264bf638cfdc9f0cd58ddb17936684aafd0f8"}, - {file = "grpcio-1.54.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:b7655f809e3420f80ce3bf89737169a9dce73238af594049754a1128132c0da4"}, - {file = "grpcio-1.54.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87f47bf9520bba4083d65ab911f8f4c0ac3efa8241993edd74c8dd08ae87552f"}, - {file = "grpcio-1.54.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16bca8092dd994f2864fdab278ae052fad4913f36f35238b2dd11af2d55a87db"}, - {file = "grpcio-1.54.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d2f62fb1c914a038921677cfa536d645cb80e3dd07dc4859a3c92d75407b90a5"}, - {file = "grpcio-1.54.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a7caf553ccaf715ec05b28c9b2ab2ee3fdb4036626d779aa09cf7cbf54b71445"}, - {file = "grpcio-1.54.0-cp38-cp38-win32.whl", hash = "sha256:2585b3c294631a39b33f9f967a59b0fad23b1a71a212eba6bc1e3ca6e6eec9ee"}, - {file = "grpcio-1.54.0-cp38-cp38-win_amd64.whl", hash = "sha256:3b170e441e91e4f321e46d3cc95a01cb307a4596da54aca59eb78ab0fc03754d"}, - {file = "grpcio-1.54.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:1382bc499af92901c2240c4d540c74eae8a671e4fe9839bfeefdfcc3a106b5e2"}, - {file = "grpcio-1.54.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:031bbd26656e0739e4b2c81c172155fb26e274b8d0312d67aefc730bcba915b6"}, - {file = "grpcio-1.54.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a97b0d01ae595c997c1d9d8249e2d2da829c2d8a4bdc29bb8f76c11a94915c9a"}, - {file = "grpcio-1.54.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:533eaf5b2a79a3c6f35cbd6a095ae99cac7f4f9c0e08bdcf86c130efd3c32adf"}, - {file = "grpcio-1.54.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49eace8ea55fbc42c733defbda1e4feb6d3844ecd875b01bb8b923709e0f5ec8"}, - {file = "grpcio-1.54.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:30fbbce11ffeb4f9f91c13fe04899aaf3e9a81708bedf267bf447596b95df26b"}, - {file = "grpcio-1.54.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:650f5f2c9ab1275b4006707411bb6d6bc927886874a287661c3c6f332d4c068b"}, - {file = "grpcio-1.54.0-cp39-cp39-win32.whl", hash = "sha256:02000b005bc8b72ff50c477b6431e8886b29961159e8b8d03c00b3dd9139baed"}, - {file = "grpcio-1.54.0-cp39-cp39-win_amd64.whl", hash = "sha256:6dc1e2c9ac292c9a484ef900c568ccb2d6b4dfe26dfa0163d5bc815bb836c78d"}, - {file = "grpcio-1.54.0.tar.gz", hash = "sha256:eb0807323572642ab73fd86fe53d88d843ce617dd1ddf430351ad0759809a0ae"}, + {file = "grpcio-1.55.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:7b38e028a7bbc97a9ae5e418712452f298618b9d0493390770bf2de785251ae7"}, + {file = "grpcio-1.55.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:054b7164b25712ec71339e139875a66708a2ab09be36ac75e73b2d337ab2dc1b"}, + {file = "grpcio-1.55.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:1982c99c7091d1b7e3e78b1173097f705feef233e253a27e99746b11815ac897"}, + {file = "grpcio-1.55.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8bd4f4932ef63ed32a725065aebb8585e4118a523d923db896e85c09429a36e6"}, + {file = "grpcio-1.55.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70de2b73cf22241173cb21d308786ba4ea443e4c88441a2ce445829aa638dda8"}, + {file = "grpcio-1.55.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2d25d7fcb528a40578b3d0428d401745fd5c0eeeda81f35ce2f40a10d79afd19"}, + {file = "grpcio-1.55.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1173a05117798aca4834d3edd504e6adc25ae9967df0f44b91a612884fb2707a"}, + {file = "grpcio-1.55.0-cp310-cp310-win32.whl", hash = "sha256:7c00263d792a244bef67a8d3b357ccbcdae6341c5961dbee494d8f967f9aee69"}, + {file = "grpcio-1.55.0-cp310-cp310-win_amd64.whl", hash = "sha256:ab784204d9923368e0e5877d7795584b9606a51b128ee199ad8b5888d0c66592"}, + {file = "grpcio-1.55.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c97cfae0b7a17dc1a0a3e4333f4f46daa114d85f950a67f39cc141b5425182e4"}, + {file = "grpcio-1.55.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:8a910fa9b95a286f4bc1879dcf8d5ccb95b5e33bb63323fc4414d157f23afef1"}, + {file = "grpcio-1.55.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:3ab9bf80c19c91847f45ff32af94c85d282545a62db39d797838244d57831d78"}, + {file = "grpcio-1.55.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4370d2cca37301bcc69453d3dd3c1576d06d6b3e337bfec55b3aab2fe106b25c"}, + {file = "grpcio-1.55.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dad999423b33ad5409e986587593b6062a8260b74ae8fc8162ce231c6b7a929e"}, + {file = "grpcio-1.55.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d396ec4d520b58f43142958cff071e5ad1c50ac87d29d086a9c6a990a09ea536"}, + {file = "grpcio-1.55.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b2a3b837d5837b9069783026b57aa0ff12e34d3218fdeda3f9c06d3950266d8e"}, + {file = "grpcio-1.55.0-cp311-cp311-win32.whl", hash = "sha256:ee0de9cb6813704969e53743e0969fd95225ff24bd686c89ed12a18147f6566c"}, + {file = "grpcio-1.55.0-cp311-cp311-win_amd64.whl", hash = "sha256:9a11b1dd4b1572e85fba5911309c15980a1ff77c555fad0ecdbe3711ef741908"}, + {file = "grpcio-1.55.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d0209fb3cb55c5288a1dec72dcaae2c1b501edceca10d22c0f0baa5e60e2b22c"}, + {file = "grpcio-1.55.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:322d4ebc37cbc8d8596b1da6055e3e81e8cfd36816ab4b285c1163c3042e6067"}, + {file = "grpcio-1.55.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:60efab181c32e029e0960f238508396dd001ba2064168f8148e6356db093967c"}, + {file = "grpcio-1.55.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48f6088d898e1e987d761d58dc4cd724e7457a7a86d11561fa95c3b826d025dc"}, + {file = "grpcio-1.55.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29ab0e879b1585be41cfbb02faed67913700ced8015da4763f1f0bdd7dfb4ab7"}, + {file = "grpcio-1.55.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:157f5615c7b5d0968727472f6394dee01555ef4246d2f2cfb6555be857936d74"}, + {file = "grpcio-1.55.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:67c4fda71f92225c5e74fa15bffa6be022c07111f674fe1f234c1ef4c1bb7927"}, + {file = "grpcio-1.55.0-cp37-cp37m-win_amd64.whl", hash = "sha256:a202dcf0c512292fd7a2154e4044c70400212eaa726685ebf8af105e25693c5a"}, + {file = "grpcio-1.55.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:ce82d06cdfb8a9292fb857f00bee11a2430e4ac2742e07b46c1a3072d683256a"}, + {file = "grpcio-1.55.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:51b7a27a129f743d68394f94029f88ef3da090fc13776b9dfa3c79c5f4b30525"}, + {file = "grpcio-1.55.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:7c32f87bec58a8a0d4f4d5387bd61a383bd32b2caffb2de3cd579e47490b7e19"}, + {file = "grpcio-1.55.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:89107071b5f14af6bbb855183d338a0fa94136bbeb3989c9773c6184e51a95e9"}, + {file = "grpcio-1.55.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1041cad23f00943d8889ad15427d87bbdacbbe2df5cec951c314f2f3967d4691"}, + {file = "grpcio-1.55.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:56631cc0bdf86d15ea1599b9697ace65e6b52c6b136d3666bf7769d3d6d087a8"}, + {file = "grpcio-1.55.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:10af4774da9c0665a1bf519333694ac40d72d83cb514534b99db0a5e3d5c3593"}, + {file = "grpcio-1.55.0-cp38-cp38-win32.whl", hash = "sha256:7b8665da31b5bd701b338a581de7b9631d50b4b7ee67125c2d1dc2228cc119d8"}, + {file = "grpcio-1.55.0-cp38-cp38-win_amd64.whl", hash = "sha256:74780f570c76feb8e62a8c019b495fea435b60218682fce513ff2c71262c346c"}, + {file = "grpcio-1.55.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:6b8dbb151b116825c10f01e5b7b75e14edd0e60736a65311d0d98a4cd0489303"}, + {file = "grpcio-1.55.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:a82283d6e0403d3e2e7eebb99cb0d2783e20b6791c8c94bd8d4a4233b58b1ea0"}, + {file = "grpcio-1.55.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:ba32a8e9bc3eecc6bab6824b905f04c3fdc31659c3e6e06841b774e7cb4410af"}, + {file = "grpcio-1.55.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b1e2b705d524e780998218cf429d30b6ffc54cb6e54812c9597bc5df12dbcb5b"}, + {file = "grpcio-1.55.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe78365c64b2c7470d31c4941e10c6654042bcbb53897b9b1e2c96d6d0da9ef9"}, + {file = "grpcio-1.55.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8b440ccc434c1ad5874465bfae40c0a27f562ae5f7c5b468b6689bc55e8bf1c1"}, + {file = "grpcio-1.55.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0d3d5c644d523dee82ffcc44ad50cd66e3bf66e7fa60ad3cdb1eb868228e4ab0"}, + {file = "grpcio-1.55.0-cp39-cp39-win32.whl", hash = "sha256:c33dbeecc14f1a413e8af8ae1208cb383b063fa2ff2e1f309b4d3d7739b0927e"}, + {file = "grpcio-1.55.0-cp39-cp39-win_amd64.whl", hash = "sha256:2663741acc117370fd53336267cfb24c965e9d3ea1e4933a3e4411712d3091fb"}, + {file = "grpcio-1.55.0.tar.gz", hash = "sha256:dd15027a171ff93c97f9c704fa120bc5d0691dc7e71ae450e2ecade1a2799b53"}, ] grpcio-reflection = [ - {file = "grpcio-reflection-1.54.0.tar.gz", hash = "sha256:804326e1add80050cab248107d28f226be58ec49d5a2d08f14a150d8a2621678"}, - {file = "grpcio_reflection-1.54.0-py3-none-any.whl", hash = "sha256:9de46150bba3c039035c5f573a9348f1c747d3149b1fa946b351da7c0e0733d8"}, + {file = "grpcio-reflection-1.55.0.tar.gz", hash = "sha256:46fc5e68ce7ae9bff0c0577f9e42bbb038a5afb26290fdf04943285e9db3c193"}, + {file = "grpcio_reflection-1.55.0-py3-none-any.whl", hash = "sha256:44e0dbfbfdcf1ac8646f1d32e4be72f0c633fd4b469e8e58b3a86e37b5a72756"}, ] grpcio-status = [ - {file = "grpcio-status-1.54.0.tar.gz", hash = "sha256:b50305d52c0df6169493cca5f2e39b9b4d773b3f30d4a7a6b6dd7c18cb89007c"}, - {file = "grpcio_status-1.54.0-py3-none-any.whl", hash = "sha256:96968314e0c8576b2b631be3917c665964c8018900cb980d58a736fbff828578"}, + {file = "grpcio-status-1.55.0.tar.gz", hash = "sha256:beeca8d5d3783e155676beaade0dae9eaea12cd9701498905dca0d35bd6b36f8"}, + {file = "grpcio_status-1.55.0-py3-none-any.whl", hash = "sha256:6da36bab11bb252b6854b86578f484c4fed9f8169816b490b6d3a32ec2a971fe"}, ] grpcio-tools = [ - {file = "grpcio-tools-1.54.0.tar.gz", hash = "sha256:df79acbf59997018e131713b716a2fddb5556e1840e9fb9de4ca73bf2059590e"}, - {file = "grpcio_tools-1.54.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:7abdefb364de75d13d929093e571b84c4ea2580dbf5e4b6f5099ac9fa1a93f29"}, - {file = "grpcio_tools-1.54.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:66738bc9d7b5db1a86832fa930cdd8faaf0c68bf70751d930c686d6bb309fad0"}, - {file = "grpcio_tools-1.54.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:de07f7441e2f318a68631626d4d99c22244901ad7451a3e9fa08e5601a54772f"}, - {file = "grpcio_tools-1.54.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb6774907a708079afb3deecdd8e053d2b2b066c279a5d8c4957687db3f809ce"}, - {file = "grpcio_tools-1.54.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7615b824132fcea4769d0cd99bdeffe25b096b48215c36675f37f7b57dc09635"}, - {file = "grpcio_tools-1.54.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22b826b1d0d98a3bb80094928e21c48e23d022168e91d466a21389d1cda87177"}, - {file = "grpcio_tools-1.54.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c03ceec6441365ec56c7fc6f38386917814ad02144dc1ecf03e13cee45f26389"}, - {file = "grpcio_tools-1.54.0-cp310-cp310-win32.whl", hash = "sha256:a61f367a153e3e604b1b59cec2f5e61255ffb6328d7348a53531e90ad98cc4fd"}, - {file = "grpcio_tools-1.54.0-cp310-cp310-win_amd64.whl", hash = "sha256:8a91f973b44fc90c32b7df79daec78907f0721e65d85fdff9133e05fee1f6e56"}, - {file = "grpcio_tools-1.54.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:436335857731e8f0bc5d345804d7dcd1bc677a1f793ea2b22e0627870ea31df6"}, - {file = "grpcio_tools-1.54.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:9fa42e3851c3c795228e2699c68ff7db14789991da4b53d3634360a361e0e420"}, - {file = "grpcio_tools-1.54.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7ab23a94a8a7aa97ad8f46ba39ade4462c19833aa4815780db2cec647df7efef"}, - {file = "grpcio_tools-1.54.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12c3091ef09df47c20d698dc74a373c1674c67ac5ac12ef24433002165129556"}, - {file = "grpcio_tools-1.54.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536c108ee8fa46b1ca8d03f4033cdf282741978f3f83dcc8b2a9e7a2b8197271"}, - {file = "grpcio_tools-1.54.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a66baf974aa00542ba4b6be499266ef8d63aa1abe7c78abab168bdf2fcd7f216"}, - {file = "grpcio_tools-1.54.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b944c1654b5f0710e782c0e3462d7acca900cbf2fb874d62e51a5279293151a9"}, - {file = "grpcio_tools-1.54.0-cp311-cp311-win32.whl", hash = "sha256:f5aaa7e90e0947aec936e02fb85f312ca03c2258f3ee78403dcc27389fc62838"}, - {file = "grpcio_tools-1.54.0-cp311-cp311-win_amd64.whl", hash = "sha256:6ff318cd73b5cbdb16616d093c1c7b340d8d13b5bada2df7fbd873e899eab162"}, - {file = "grpcio_tools-1.54.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:86169076170f835139db7ec8362124cabe8f01050c2a717500a0fcdeb71dc537"}, - {file = "grpcio_tools-1.54.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:24158db60cdda7c9eb803a9ed033a23b4420fd13ba4c54392d6b396589afb969"}, - {file = "grpcio_tools-1.54.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:58d2cbbf4e40cecd09f4fa6a2c15f7a95ec90f492b4e31469d8c9db981426106"}, - {file = "grpcio_tools-1.54.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e15aa21a68cdb66b38db645924a09196cbbf4ad3ce1cf9dbcdf7b245292e380"}, - {file = "grpcio_tools-1.54.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23474a820bdf8126af93724c7f6b23b16d9f3ad4c4bdcb40936ecb7f2be6bcc7"}, - {file = "grpcio_tools-1.54.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8d2d5522d2222879c161479b5c91661cf788a3178f0b7532fd4c3e918e3390c4"}, - {file = "grpcio_tools-1.54.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:52b6f78f5601cd080e5456ec24fd960328f9962e9aa4f9f556ec6fd5e4a391bf"}, - {file = "grpcio_tools-1.54.0-cp37-cp37m-win_amd64.whl", hash = "sha256:292c0838d4a52ca53a941d197de57efc7f9df548d0619e14680c3e1ebeb33752"}, - {file = "grpcio_tools-1.54.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:b1a3aad8a85d408ec925ee63668c2e4f075c96f67fcc3c43366a28f60e25873c"}, - {file = "grpcio_tools-1.54.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:9d9a8d35a898f5c160fdb1ba0f58ada80e8e291cdffd6025f6043228d0832b2f"}, - {file = "grpcio_tools-1.54.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:42e8509ae024e8fb4eeab5ed9518c8e88d7f46a56015ae940796fe708d781e9f"}, - {file = "grpcio_tools-1.54.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6dd57137706257c39255020fb04322eef6c4043a11fc86042ae22b05167603ca"}, - {file = "grpcio_tools-1.54.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8ed6146e08b82caff467ac2af6c41f092212e3b5795687b76dd9219ac8b7186"}, - {file = "grpcio_tools-1.54.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4b32eaf44939c3dfa127bcb6bcf685188a05c1e0f54a7c8ed10770cbe177ca15"}, - {file = "grpcio_tools-1.54.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cca182c9a33dbf52c61a13fd1b98af535e7563bcd9e02e60e9de742d385f2ffb"}, - {file = "grpcio_tools-1.54.0-cp38-cp38-win32.whl", hash = "sha256:ce20c3523a4bdb85531e762f45f7c5f5a915e5a8f25db3d58ebb6a8a37290a75"}, - {file = "grpcio_tools-1.54.0-cp38-cp38-win_amd64.whl", hash = "sha256:53dc65731a46bc065ad8d717d75d8659763126d1626eacc6f225c0b779ccc350"}, - {file = "grpcio_tools-1.54.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:072279c394086ec07af8736f94aa6be671000af143df5f3040500e8cd5de484d"}, - {file = "grpcio_tools-1.54.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:473dbd7005d059d1dc84490eec35699029d629048e7fdd76e31c14fc58a603e5"}, - {file = "grpcio_tools-1.54.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:22eecfbb85da93405e056f073e339c34f881a3b410800902ae533e475688b1f1"}, - {file = "grpcio_tools-1.54.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0ba9344855b0c617d6df6bfb46ffbc574815c7152afa023e443cce4d5402f55"}, - {file = "grpcio_tools-1.54.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:235ea036d56ab9b6e4f235c554cf4680c42bf79352046e8a25829f597ca0d56b"}, - {file = "grpcio_tools-1.54.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:adc93f29353377182556e37120985f69f0fccc79ab65a0f24c1cc53075344dd7"}, - {file = "grpcio_tools-1.54.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab45bda18b4931b08d89e490fd920bf654109d1c2e80b037fe9b60ab30fb4d9d"}, - {file = "grpcio_tools-1.54.0-cp39-cp39-win32.whl", hash = "sha256:43339122c891110e9ed020974c0fe2bd773d96c184203a62d0973f4cfb007f80"}, - {file = "grpcio_tools-1.54.0-cp39-cp39-win_amd64.whl", hash = "sha256:d58528d6ea1835c0d4c34b60152dd29b3c9ab02ad0a8e734de240d54786d8b1e"}, + {file = "grpcio-tools-1.55.0.tar.gz", hash = "sha256:d796f5d7cea260ef2afed12d13ec34b13e09dd74d7f292d7428c506fa8c17a74"}, + {file = "grpcio_tools-1.55.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:52e34e9b6496f4c1e3289ada7bc41d759e4a8ec5f2679e187067cab8532ffbf4"}, + {file = "grpcio_tools-1.55.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b131b2bbf25198d9e508dfa588cb215580629b514e293d5609eeee98c8941dbc"}, + {file = "grpcio_tools-1.55.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:9933a1f18f780c42214b126ef27e273b54c9c28de3fae5b1887b413ceb374c4c"}, + {file = "grpcio_tools-1.55.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfc82c11ce51de6ed5836fbafbc188d9eac0737abc116978f151c40271783817"}, + {file = "grpcio_tools-1.55.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7a18bd5f994b7911d3e70e0abb05bea9f1b084a1725d404a8e231bf9727613b"}, + {file = "grpcio_tools-1.55.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:87152893c7c3bef58a6a9b548db290aa318cc314c700ae7d7f2970aa567f875e"}, + {file = "grpcio_tools-1.55.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bc23034b1959d6cda27347b2207fee0fb0fb0aff242da228a6b7c1a18fce4116"}, + {file = "grpcio_tools-1.55.0-cp310-cp310-win32.whl", hash = "sha256:ab64f9d6f5e3636ae6298e2d795225daa83aacb057105943728ed50a8a582237"}, + {file = "grpcio_tools-1.55.0-cp310-cp310-win_amd64.whl", hash = "sha256:b197de69ca0431b718ffa47b32a733703fa5503da49f49dd315c866842b6cfbd"}, + {file = "grpcio_tools-1.55.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:98ff3129ff7134a95f4d2857663625771f6838ac44b7799c34259b7ea87ebe5c"}, + {file = "grpcio_tools-1.55.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:c3c7b7eb89f963b87922ecc0c0ab2485fff05997ada66dffd53597b507a83bc8"}, + {file = "grpcio_tools-1.55.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:51a1ccab6f67edd1a3768a75ac495907fe0cd6d6617af2f9f2033400b5858a11"}, + {file = "grpcio_tools-1.55.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4dea66623548b52429fb03495f2c76f4c993bf9a56267c6b3d0fb62573dd52c2"}, + {file = "grpcio_tools-1.55.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e59fd4a58688117cb5128d7785909d45a6e5f8212efeb65b6fd74bb9b8b9512"}, + {file = "grpcio_tools-1.55.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:734ede84d613b044f72e7d9c190bd2388ebb83e85bcd3aa75afa9f30c096dbc7"}, + {file = "grpcio_tools-1.55.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f87d99aa3826aa20c3b89493984cf278f4c9d20b3418534a46239c804fee506c"}, + {file = "grpcio_tools-1.55.0-cp311-cp311-win32.whl", hash = "sha256:4580df5a9867f7bcbb828a5485c030ca232c1578e615caf751333c7a7980d838"}, + {file = "grpcio_tools-1.55.0-cp311-cp311-win_amd64.whl", hash = "sha256:b674de79571357c5381bc5fa12e3b89fefef74c164ab9077ed22158c3529aa8e"}, + {file = "grpcio_tools-1.55.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:0dead7fb37bfe7c7eb8294143015645297f4affa683783b8bbf2cd4d7f7036d4"}, + {file = "grpcio_tools-1.55.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:89f6ed47415a22568bbf4a62336bfde7cafb53492a5a9f33a22243411b00f443"}, + {file = "grpcio_tools-1.55.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:946266cbd639847548c9f97e38da0682746c2eadea790ceb4320b1f85387bd6d"}, + {file = "grpcio_tools-1.55.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:36745762689df18f83273a9a004848897793f63a10a30acd18acb2d170c663a9"}, + {file = "grpcio_tools-1.55.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:053bbdfb74f76511db47e1e18a1962432468ae9f356cc00f15d1f1353eaf32a1"}, + {file = "grpcio_tools-1.55.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2eacb0b1e8e5cfd0b40e12e62bd5adebbbae8c73cdf6e04fad9ddd37e32d98a4"}, + {file = "grpcio_tools-1.55.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9395c4fdee6b22137e878ebd461444854a3cd9c6c260c43f4a4c4a4023700129"}, + {file = "grpcio_tools-1.55.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bcf5e1858137cbe13ef10a7931a7edc745c77f8b39f032f52072443f0dd681e1"}, + {file = "grpcio_tools-1.55.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:e76f35e5e65600a75c3547855e8c9ab935c55c473f5409e7746cca8f1f7c8f4a"}, + {file = "grpcio_tools-1.55.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:a61567f27661ab9327dc060615dc22d2bde80c56731f1e856008f1fd8ee83311"}, + {file = "grpcio_tools-1.55.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2ba87592f2cd689e127cd4fce76ec23b19562e230fa41ea089af8b15120aea78"}, + {file = "grpcio_tools-1.55.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:41005002cbfa0ad39972486bde8116b2a042804119e5b998086a4dc26e625d6a"}, + {file = "grpcio_tools-1.55.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a6db1494955d2a5531575b5fcdc08094ea4a331a94b9cdf864d78e801c5fa23"}, + {file = "grpcio_tools-1.55.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:07c23ed940e046c9dd471bc870eb5db4d93e518f90011cf9aebf8bfda6cd68a5"}, + {file = "grpcio_tools-1.55.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f900bce944b5777effecb9078e5fd3a224e42b1ca33c7546c3d043f9ef9eb4e8"}, + {file = "grpcio_tools-1.55.0-cp38-cp38-win32.whl", hash = "sha256:3724e48c3db499b2d212c5a89d7cc4b49ccd476dc26bf8a9b855d59b6cc00796"}, + {file = "grpcio_tools-1.55.0-cp38-cp38-win_amd64.whl", hash = "sha256:416a8b61ed4223715755b4519858419e1f4653d64572a28029f2ac63e677e3d2"}, + {file = "grpcio_tools-1.55.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:73ef9e0e0ee8ab055a621e7b42e5fb32753b0b6607900887dba6d55df5947be8"}, + {file = "grpcio_tools-1.55.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:4a41130c97775bb0dfaf87e34b492f2eca448d02d213410005544c534f3f7c26"}, + {file = "grpcio_tools-1.55.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:87dbc98528f88faa3f8f56a47d41dc6fda382928abbdb5537b5444eb8bb1ac1b"}, + {file = "grpcio_tools-1.55.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f084cd619cf66d8620a99f8586018f19b918ffb2ddb92d3e5943a06038bead8"}, + {file = "grpcio_tools-1.55.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:350303ef3a2b25ed1b90e42764923e40b664d9f10840f7a0f06117c4dc414aff"}, + {file = "grpcio_tools-1.55.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:fbbe2bee4af93c03ba064d40199dbf38067d2aa6ae98dfa0687a08ee980ebfd5"}, + {file = "grpcio_tools-1.55.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b00a67a1230968c1a0424915922d17983d824ed45e8db06f9f17be6d5571faee"}, + {file = "grpcio_tools-1.55.0-cp39-cp39-win32.whl", hash = "sha256:632364ffbd4fb0338cb03c590a2ddc258d9cd59bff0bf4199c02e3e581f802d7"}, + {file = "grpcio_tools-1.55.0-cp39-cp39-win_amd64.whl", hash = "sha256:95428be2db12412ff23f0969386fc51d2aa6de38a57cc54c57363352f1d7a832"}, ] hf-transfer = [ {file = "hf_transfer-0.1.3-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9"}, @@ -1198,9 +996,6 @@ Jinja2 = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -lit = [ - {file = "lit-16.0.2.tar.gz", hash = "sha256:d743ef55cb58764bba85768c502e2d68d87aeb4303d508a18abaa8a35077ab25"}, -] loguru = [ {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, @@ -1295,51 +1090,6 @@ numpy = [ {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, ] -nvidia-cublas-cu11 = [ - {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, - {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-win_amd64.whl", hash = "sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e"}, -] -nvidia-cuda-cupti-cu11 = [ - {file = "nvidia_cuda_cupti_cu11-11.7.101-py3-none-manylinux1_x86_64.whl", hash = "sha256:e0cfd9854e1f2edaa36ca20d21cd0bdd5dcfca4e3b9e130a082e05b33b6c5895"}, - {file = "nvidia_cuda_cupti_cu11-11.7.101-py3-none-win_amd64.whl", hash = "sha256:7cc5b8f91ae5e1389c3c0ad8866b3b016a175e827ea8f162a672990a402ab2b0"}, -] -nvidia-cuda-nvrtc-cu11 = [ - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03"}, - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7"}, - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:f2effeb1309bdd1b3854fc9b17eaf997808f8b25968ce0c7070945c4265d64a3"}, -] -nvidia-cuda-runtime-cu11 = [ - {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31"}, - {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7"}, -] -nvidia-cudnn-cu11 = [ - {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, - {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, -] -nvidia-cufft-cu11 = [ - {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-manylinux1_x86_64.whl", hash = "sha256:222f9da70c80384632fd6035e4c3f16762d64ea7a843829cb278f98b3cb7dd81"}, - {file = "nvidia_cufft_cu11-10.9.0.58-py3-none-win_amd64.whl", hash = "sha256:c4d316f17c745ec9c728e30409612eaf77a8404c3733cdf6c9c1569634d1ca03"}, -] -nvidia-curand-cu11 = [ - {file = "nvidia_curand_cu11-10.2.10.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:eecb269c970fa599a2660c9232fa46aaccbf90d9170b96c462e13bcb4d129e2c"}, - {file = "nvidia_curand_cu11-10.2.10.91-py3-none-win_amd64.whl", hash = "sha256:f742052af0e1e75523bde18895a9ed016ecf1e5aa0ecddfcc3658fd11a1ff417"}, -] -nvidia-cusolver-cu11 = [ - {file = "nvidia_cusolver_cu11-11.4.0.1-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:72fa7261d755ed55c0074960df5904b65e2326f7adce364cbe4945063c1be412"}, - {file = "nvidia_cusolver_cu11-11.4.0.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:700b781bfefd57d161443aff9ace1878584b93e0b2cfef3d6e9296d96febbf99"}, - {file = "nvidia_cusolver_cu11-11.4.0.1-py3-none-win_amd64.whl", hash = "sha256:00f70b256add65f8c1eb3b6a65308795a93e7740f6df9e273eccbba770d370c4"}, -] -nvidia-cusparse-cu11 = [ - {file = "nvidia_cusparse_cu11-11.7.4.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:a3389de714db63321aa11fbec3919271f415ef19fda58aed7f2ede488c32733d"}, - {file = "nvidia_cusparse_cu11-11.7.4.91-py3-none-win_amd64.whl", hash = "sha256:304a01599534f5186a8ed1c3756879282c72c118bc77dd890dc1ff868cad25b9"}, -] -nvidia-nccl-cu11 = [ - {file = "nvidia_nccl_cu11-2.14.3-py3-none-manylinux1_x86_64.whl", hash = "sha256:5e5534257d1284b8e825bc3a182c6f06acd6eb405e9f89d49340e98cd8f136eb"}, -] -nvidia-nvtx-cu11 = [ - {file = "nvidia_nvtx_cu11-11.7.91-py3-none-manylinux1_x86_64.whl", hash = "sha256:b22c64eee426a62fc00952b507d6d29cf62b4c9df7a480fcc417e540e05fd5ac"}, - {file = "nvidia_nvtx_cu11-11.7.91-py3-none-win_amd64.whl", hash = "sha256:dfd7fcb2a91742513027d63a26b757f38dd8b07fecac282c4d132a9d373ff064"}, -] opentelemetry-api = [ {file = "opentelemetry_api-1.15.0-py3-none-any.whl", hash = "sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0"}, {file = "opentelemetry_api-1.15.0.tar.gz", hash = "sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded"}, @@ -1385,19 +1135,19 @@ pluggy = [ {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] protobuf = [ - {file = "protobuf-4.22.3-cp310-abi3-win32.whl", hash = "sha256:8b54f56d13ae4a3ec140076c9d937221f887c8f64954673d46f63751209e839a"}, - {file = "protobuf-4.22.3-cp310-abi3-win_amd64.whl", hash = "sha256:7760730063329d42a9d4c4573b804289b738d4931e363ffbe684716b796bde51"}, - {file = "protobuf-4.22.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:d14fc1a41d1a1909998e8aff7e80d2a7ae14772c4a70e4bf7db8a36690b54425"}, - {file = "protobuf-4.22.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:70659847ee57a5262a65954538088a1d72dfc3e9882695cab9f0c54ffe71663b"}, - {file = "protobuf-4.22.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:13233ee2b9d3bd9a5f216c1fa2c321cd564b93d8f2e4f521a85b585447747997"}, - {file = "protobuf-4.22.3-cp37-cp37m-win32.whl", hash = "sha256:ecae944c6c2ce50dda6bf76ef5496196aeb1b85acb95df5843cd812615ec4b61"}, - {file = "protobuf-4.22.3-cp37-cp37m-win_amd64.whl", hash = "sha256:d4b66266965598ff4c291416be429cef7989d8fae88b55b62095a2331511b3fa"}, - {file = "protobuf-4.22.3-cp38-cp38-win32.whl", hash = "sha256:f08aa300b67f1c012100d8eb62d47129e53d1150f4469fd78a29fa3cb68c66f2"}, - {file = "protobuf-4.22.3-cp38-cp38-win_amd64.whl", hash = "sha256:f2f4710543abec186aee332d6852ef5ae7ce2e9e807a3da570f36de5a732d88e"}, - {file = "protobuf-4.22.3-cp39-cp39-win32.whl", hash = "sha256:7cf56e31907c532e460bb62010a513408e6cdf5b03fb2611e4b67ed398ad046d"}, - {file = "protobuf-4.22.3-cp39-cp39-win_amd64.whl", hash = "sha256:e0e630d8e6a79f48c557cd1835865b593d0547dce221c66ed1b827de59c66c97"}, - {file = "protobuf-4.22.3-py3-none-any.whl", hash = "sha256:52f0a78141078077cfe15fe333ac3e3a077420b9a3f5d1bf9b5fe9d286b4d881"}, - {file = "protobuf-4.22.3.tar.gz", hash = "sha256:23452f2fdea754a8251d0fc88c0317735ae47217e0d27bf330a30eec2848811a"}, + {file = "protobuf-4.23.1-cp310-abi3-win32.whl", hash = "sha256:410bcc0a5b279f634d3e16082ce221dfef7c3392fac723500e2e64d1806dd2be"}, + {file = "protobuf-4.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:32e78beda26d7a101fecf15d7a4a792278a0d26a31bc327ff05564a9d68ab8ee"}, + {file = "protobuf-4.23.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f9510cac91e764e86acd74e2b7f7bc5e6127a7f3fb646d7c8033cfb84fd1176a"}, + {file = "protobuf-4.23.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:346990f634272caac1f09efbcfbbacb23098b1f606d172534c6fa2d9758bb436"}, + {file = "protobuf-4.23.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3ce113b3f3362493bddc9069c2163a38f240a9ed685ff83e7bcb756b05e1deb0"}, + {file = "protobuf-4.23.1-cp37-cp37m-win32.whl", hash = "sha256:2036a3a1e7fc27f973fa0a7888dce712393af644f4695385f117886abc792e39"}, + {file = "protobuf-4.23.1-cp37-cp37m-win_amd64.whl", hash = "sha256:3b8905eafe4439076e1f58e9d1fa327025fd2777cf90f14083092ae47f77b0aa"}, + {file = "protobuf-4.23.1-cp38-cp38-win32.whl", hash = "sha256:5b9cd6097e6acae48a68cb29b56bc79339be84eca65b486910bb1e7a30e2b7c1"}, + {file = "protobuf-4.23.1-cp38-cp38-win_amd64.whl", hash = "sha256:decf119d54e820f298ee6d89c72d6b289ea240c32c521f00433f9dc420595f38"}, + {file = "protobuf-4.23.1-cp39-cp39-win32.whl", hash = "sha256:91fac0753c3c4951fbb98a93271c43cc7cf3b93cf67747b3e600bb1e5cc14d61"}, + {file = "protobuf-4.23.1-cp39-cp39-win_amd64.whl", hash = "sha256:ac50be82491369a9ec3710565777e4da87c6d2e20404e0abb1f3a8f10ffd20f0"}, + {file = "protobuf-4.23.1-py3-none-any.whl", hash = "sha256:65f0ac96ef67d7dd09b19a46aad81a851b6f85f89725577f16de38f2d68ad477"}, + {file = "protobuf-4.23.1.tar.gz", hash = "sha256:95789b569418a3e32a53f43d7763be3d490a831e9c08042539462b6d972c2d7e"}, ] psutil = [ {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, @@ -1462,8 +1212,8 @@ PyYAML = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] requests = [ - {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, - {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, ] safetensors = [ {file = "safetensors-0.3.1-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:2ae9b7dd268b4bae6624729dac86deb82104820e9786429b0583e5168db2f770"}, @@ -1508,59 +1258,59 @@ safetensors = [ {file = "safetensors-0.3.1.tar.gz", hash = "sha256:571da56ff8d0bec8ae54923b621cda98d36dcef10feb36fd492c4d0c2cd0e869"}, ] sentencepiece = [ - {file = "sentencepiece-0.1.98-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1daf0a79cd953e4830746c41e92b98a2f2e9e5ec0e90a9447aa10350e11bd027"}, - {file = "sentencepiece-0.1.98-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57911445fc91c80d59552adf8a749af9205458920a7328f3bd7d51308658bcd9"}, - {file = "sentencepiece-0.1.98-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3f9239785849ed1f55a825bcc282bef1a6073f7431cc535bdc658a94873652ea"}, - {file = "sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:467740ef8af170e5b6cfe22c272114ed930c899c297619ac7a2ac463a13bdbac"}, - {file = "sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b6f0b9ffb601e2699e265f3f20c353ec9a661e4b5f0cff08ad6c9909c0ae43e"}, - {file = "sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6150ba525fac4fda76f5c4777ae300597e70cef739ed2a47cea02ff81a88873f"}, - {file = "sentencepiece-0.1.98-cp310-cp310-win32.whl", hash = "sha256:58ca96d73ea0e5575e3f6a9524449c673d62e6ecee3b2ddd5bfb4f49cb315c0a"}, - {file = "sentencepiece-0.1.98-cp310-cp310-win_amd64.whl", hash = "sha256:8abe5c4c034e497e69f485dcd2c0e6bc87bf0498ad5aef5f539a7d0f9eae6275"}, - {file = "sentencepiece-0.1.98-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b6ed62f89c0bd25cec39a7075f6b9354fe4c240ed964e63009d77efcf29c34e9"}, - {file = "sentencepiece-0.1.98-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c2d9a74986d3716dc6961e9dbae7a3b25bb1260118f098545fd963ae23252c1"}, - {file = "sentencepiece-0.1.98-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f7dc2fc175623529fb60a2799748f8877cd48c4541b32cd97b8523465e88b69"}, - {file = "sentencepiece-0.1.98-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64e32c55d04a2e21f0c2fda1b7a3dd108133ebfb8616b52896916bb30e4352ed"}, - {file = "sentencepiece-0.1.98-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:443f32e94b18571231b02a51be173686114b5556b5edfcbf347fb63e7bd5ddc6"}, - {file = "sentencepiece-0.1.98-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:558a373a8660bdff299d6c133c2a4f4fb0875e9e6fafe225b8080ecce8a405f9"}, - {file = "sentencepiece-0.1.98-cp311-cp311-win32.whl", hash = "sha256:fcf100268cefe1774794b18cbaf3065e2bf988f168a387973eb1260d51198795"}, - {file = "sentencepiece-0.1.98-cp311-cp311-win_amd64.whl", hash = "sha256:05b4eecbece0606883cd81ed86bb3c619680bb570b997b236533ec854d64a575"}, - {file = "sentencepiece-0.1.98-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:35af00f5103a4779694fedea41b6e24947a9ed81166efe63864ab1e781d70a66"}, - {file = "sentencepiece-0.1.98-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2766cd708e9fc2b5b9784a990a8b303b9e0b9a69fa482616fe86fa538daa1756"}, - {file = "sentencepiece-0.1.98-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2531c0e9cc8cd404fabd856d80d695b373371c00f1fce29c06f41f3f7429d87"}, - {file = "sentencepiece-0.1.98-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffcc78e80c55eab67ee3439ade493607a4e37e1f0b82b168ead3debf9eaeaabe"}, - {file = "sentencepiece-0.1.98-cp36-cp36m-win32.whl", hash = "sha256:ef384b31ec7a06a9a6aba42e68435f3f3b38809aa65559ede3658cdd446a562c"}, - {file = "sentencepiece-0.1.98-cp36-cp36m-win_amd64.whl", hash = "sha256:e7a828f1fe2e51d2d9e5e9b3283d4006f1891efb02a3d9303ed39ddafdd9c864"}, - {file = "sentencepiece-0.1.98-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8663be00a68098f85d6cda1f7041a27de05c320e433fa730ecb1156a8304f21c"}, - {file = "sentencepiece-0.1.98-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf05611089a075b78d353720ccc3a09a78e0846332cff0cc78fda8b2383626a"}, - {file = "sentencepiece-0.1.98-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11f410cc7eeb3e1cfa8d92d128b568e5dc7829b7904b164499fd0209316ec2fa"}, - {file = "sentencepiece-0.1.98-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5ea8fb2c68073fe25a08a178eed269ed382fba074ff2ba4de72f0f56d86630e"}, - {file = "sentencepiece-0.1.98-cp37-cp37m-win32.whl", hash = "sha256:fa13a125417d28e84fbdebcaf6aa115e4177d3e93aa66b857a42e7179f515b88"}, - {file = "sentencepiece-0.1.98-cp37-cp37m-win_amd64.whl", hash = "sha256:e54aa70b574eee895d184072d84e62824f404821e551a82c619c5d4320a93834"}, - {file = "sentencepiece-0.1.98-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:515a971c2a157647ca0e60ce3c435f4b43cd5c9f5862159cfefa0b5b4d46d3c3"}, - {file = "sentencepiece-0.1.98-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c23c3a562221bc40eaae42428fcd8e607e0f084ea8aa968ba3f1a7d0ea975807"}, - {file = "sentencepiece-0.1.98-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c067ba22be8edc699f6365e01ec15046bf3563dbabfdc052ecc88e581b675cba"}, - {file = "sentencepiece-0.1.98-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12c913493d6ebac86ee7ae109e368522a5a365a7b150d4d8cf845599262d2b21"}, - {file = "sentencepiece-0.1.98-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:720f827dc69ee24951ea4f51b9fb69cc56890a7190fc52c2c0da2545caab1760"}, - {file = "sentencepiece-0.1.98-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:918b4caf18b2f73c302c4e197d1c2dafba39eb143d16b4590930b45f15042fdd"}, - {file = "sentencepiece-0.1.98-cp38-cp38-win32.whl", hash = "sha256:2d50edfc4649a1566b64f1a8402cd607e1893bf8e368732337d83f00df62d3fa"}, - {file = "sentencepiece-0.1.98-cp38-cp38-win_amd64.whl", hash = "sha256:7425b727c3d6b3b7bad0005a3be316078b254180b712d73955ff08cae3f6a385"}, - {file = "sentencepiece-0.1.98-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:00b2becbd7b98905a6de9695cf8682abe0d510ab0198e23c7d86fb2b793b6ae0"}, - {file = "sentencepiece-0.1.98-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7f71c4bdedb797052fb2ccad0871c2409bf6f812cb6b651917c55f9e8eced07f"}, - {file = "sentencepiece-0.1.98-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7287461d2346f530928ab187f0834cb15ddfbc4553592cacdcb6470364739ec6"}, - {file = "sentencepiece-0.1.98-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:472ad943eaffcb6871ece56c7850388e7b8722f520ba73c93e7a6ef965453221"}, - {file = "sentencepiece-0.1.98-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7e23aaf9d5afd91ca13550968bd17f0c17b0966823188ad2a50c51544cf8ed"}, - {file = "sentencepiece-0.1.98-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0ce9efc790c209cce2463058855dceb21438213d2ff13cb5a565d52a7efe25"}, - {file = "sentencepiece-0.1.98-cp39-cp39-win32.whl", hash = "sha256:8b50cbe8e46204eff7aa5a663af5652c45e7807aa560d08e5f5b10c60e795a49"}, - {file = "sentencepiece-0.1.98-cp39-cp39-win_amd64.whl", hash = "sha256:14841bd2a3d77c4dbba58f02488c374866551e428d755e8d473d82325a0a94f3"}, - {file = "sentencepiece-0.1.98.tar.gz", hash = "sha256:947cf0a4b8a480510d560a922f8256f34e93984a86cf870be4d05731f59fb28d"}, + {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0eb528e70571b7c02723e5804322469b82fe7ea418c96051d0286c0fa028db73"}, + {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:77d7fafb2c4e4659cbdf303929503f37a26eabc4ff31d3a79bf1c5a1b338caa7"}, + {file = "sentencepiece-0.1.99-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be9cf5b9e404c245aeb3d3723c737ba7a8f5d4ba262ef233a431fa6c45f732a0"}, + {file = "sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baed1a26464998f9710d20e52607c29ffd4293e7c71c6a1f83f51ad0911ec12c"}, + {file = "sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9832f08bb372d4c8b567612f8eab9e36e268dff645f1c28f9f8e851be705f6d1"}, + {file = "sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:019e7535108e309dae2b253a75834fc3128240aa87c00eb80732078cdc182588"}, + {file = "sentencepiece-0.1.99-cp310-cp310-win32.whl", hash = "sha256:fa16a830416bb823fa2a52cbdd474d1f7f3bba527fd2304fb4b140dad31bb9bc"}, + {file = "sentencepiece-0.1.99-cp310-cp310-win_amd64.whl", hash = "sha256:14b0eccb7b641d4591c3e12ae44cab537d68352e4d3b6424944f0c447d2348d5"}, + {file = "sentencepiece-0.1.99-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6d3c56f24183a1e8bd61043ff2c58dfecdc68a5dd8955dc13bab83afd5f76b81"}, + {file = "sentencepiece-0.1.99-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed6ea1819fd612c989999e44a51bf556d0ef6abfb553080b9be3d347e18bcfb7"}, + {file = "sentencepiece-0.1.99-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2a0260cd1fb7bd8b4d4f39dc2444a8d5fd4e0a0c4d5c899810ef1abf99b2d45"}, + {file = "sentencepiece-0.1.99-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a1abff4d1ff81c77cac3cc6fefa34fa4b8b371e5ee51cb7e8d1ebc996d05983"}, + {file = "sentencepiece-0.1.99-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:004e6a621d4bc88978eecb6ea7959264239a17b70f2cbc348033d8195c9808ec"}, + {file = "sentencepiece-0.1.99-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db361e03342c41680afae5807590bc88aa0e17cfd1a42696a160e4005fcda03b"}, + {file = "sentencepiece-0.1.99-cp311-cp311-win32.whl", hash = "sha256:2d95e19168875b70df62916eb55428a0cbcb834ac51d5a7e664eda74def9e1e0"}, + {file = "sentencepiece-0.1.99-cp311-cp311-win_amd64.whl", hash = "sha256:f90d73a6f81248a909f55d8e6ef56fec32d559e1e9af045f0b0322637cb8e5c7"}, + {file = "sentencepiece-0.1.99-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:62e24c81e74bd87a6e0d63c51beb6527e4c0add67e1a17bac18bcd2076afcfeb"}, + {file = "sentencepiece-0.1.99-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57efcc2d51caff20d9573567d9fd3f854d9efe613ed58a439c78c9f93101384a"}, + {file = "sentencepiece-0.1.99-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a904c46197993bd1e95b93a6e373dca2f170379d64441041e2e628ad4afb16f"}, + {file = "sentencepiece-0.1.99-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d89adf59854741c0d465f0e1525b388c0d174f611cc04af54153c5c4f36088c4"}, + {file = "sentencepiece-0.1.99-cp36-cp36m-win32.whl", hash = "sha256:47c378146928690d1bc106fdf0da768cebd03b65dd8405aa3dd88f9c81e35dba"}, + {file = "sentencepiece-0.1.99-cp36-cp36m-win_amd64.whl", hash = "sha256:9ba142e7a90dd6d823c44f9870abdad45e6c63958eb60fe44cca6828d3b69da2"}, + {file = "sentencepiece-0.1.99-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b7b1a9ae4d7c6f1f867e63370cca25cc17b6f4886729595b885ee07a58d3cec3"}, + {file = "sentencepiece-0.1.99-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0f644c9d4d35c096a538507b2163e6191512460035bf51358794a78515b74f7"}, + {file = "sentencepiece-0.1.99-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8843d23a0f686d85e569bd6dcd0dd0e0cbc03731e63497ca6d5bacd18df8b85"}, + {file = "sentencepiece-0.1.99-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e6f690a1caebb4867a2e367afa1918ad35be257ecdb3455d2bbd787936f155"}, + {file = "sentencepiece-0.1.99-cp37-cp37m-win32.whl", hash = "sha256:8a321866c2f85da7beac74a824b4ad6ddc2a4c9bccd9382529506d48f744a12c"}, + {file = "sentencepiece-0.1.99-cp37-cp37m-win_amd64.whl", hash = "sha256:c42f753bcfb7661c122a15b20be7f684b61fc8592c89c870adf52382ea72262d"}, + {file = "sentencepiece-0.1.99-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:85b476406da69c70586f0bb682fcca4c9b40e5059814f2db92303ea4585c650c"}, + {file = "sentencepiece-0.1.99-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cfbcfe13c69d3f87b7fcd5da168df7290a6d006329be71f90ba4f56bc77f8561"}, + {file = "sentencepiece-0.1.99-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:445b0ec381af1cd4eef95243e7180c63d9c384443c16c4c47a28196bd1cda937"}, + {file = "sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6890ea0f2b4703f62d0bf27932e35808b1f679bdb05c7eeb3812b935ba02001"}, + {file = "sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb71af492b0eefbf9f2501bec97bcd043b6812ab000d119eaf4bd33f9e283d03"}, + {file = "sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b866b5bd3ddd54166bbcbf5c8d7dd2e0b397fac8537991c7f544220b1f67bc"}, + {file = "sentencepiece-0.1.99-cp38-cp38-win32.whl", hash = "sha256:b133e8a499eac49c581c3c76e9bdd08c338cc1939e441fee6f92c0ccb5f1f8be"}, + {file = "sentencepiece-0.1.99-cp38-cp38-win_amd64.whl", hash = "sha256:0eaf3591dd0690a87f44f4df129cf8d05d8a4029b5b6709b489b8e27f9a9bcff"}, + {file = "sentencepiece-0.1.99-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38efeda9bbfb55052d482a009c6a37e52f42ebffcea9d3a98a61de7aee356a28"}, + {file = "sentencepiece-0.1.99-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6c030b081dc1e1bcc9fadc314b19b740715d3d566ad73a482da20d7d46fd444c"}, + {file = "sentencepiece-0.1.99-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84dbe53e02e4f8a2e45d2ac3e430d5c83182142658e25edd76539b7648928727"}, + {file = "sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b0f55d0a0ee1719b4b04221fe0c9f0c3461dc3dabd77a035fa2f4788eb3ef9a"}, + {file = "sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e800f206cd235dc27dc749299e05853a4e4332e8d3dfd81bf13d0e5b9007d9"}, + {file = "sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ae1c40cda8f9d5b0423cfa98542735c0235e7597d79caf318855cdf971b2280"}, + {file = "sentencepiece-0.1.99-cp39-cp39-win32.whl", hash = "sha256:c84ce33af12ca222d14a1cdd37bd76a69401e32bc68fe61c67ef6b59402f4ab8"}, + {file = "sentencepiece-0.1.99-cp39-cp39-win_amd64.whl", hash = "sha256:350e5c74d739973f1c9643edb80f7cc904dc948578bcb1d43c6f2b173e5d18dd"}, + {file = "sentencepiece-0.1.99.tar.gz", hash = "sha256:189c48f5cb2949288f97ccdb97f0473098d9c3dcf5a3d99d4eabe719ec27297f"}, ] setuptools = [ - {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, - {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, + {file = "setuptools-67.8.0-py3-none-any.whl", hash = "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f"}, + {file = "setuptools-67.8.0.tar.gz", hash = "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"}, ] sympy = [ - {file = "sympy-1.11.1-py3-none-any.whl", hash = "sha256:938f984ee2b1e8eae8a07b884c8b7a1146010040fccddc6539c54f401c8f6fcf"}, - {file = "sympy-1.11.1.tar.gz", hash = "sha256:e32380dce63cb7c0108ed525570092fd45168bdae2faa17e528221ef72e88658"}, + {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, + {file = "sympy-1.12.tar.gz", hash = "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8"}, ] tokenizers = [ {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, @@ -1609,70 +1359,42 @@ tomli = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] torch = [ - {file = "torch-2.0.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a"}, - {file = "torch-2.0.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7"}, - {file = "torch-2.0.0-1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8969aa8375bcbc0c2993e7ede0a7f889df9515f18b9b548433f412affed478d9"}, - {file = "torch-2.0.0-1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:ab2da16567cb55b67ae39e32d520d68ec736191d88ac79526ca5874754c32203"}, - {file = "torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:7a9319a67294ef02459a19738bbfa8727bb5307b822dadd708bc2ccf6c901aca"}, - {file = "torch-2.0.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9f01fe1f6263f31bd04e1757946fd63ad531ae37f28bb2dbf66f5c826ee089f4"}, - {file = "torch-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:527f4ae68df7b8301ee6b1158ca56350282ea633686537b30dbb5d7b4a52622a"}, - {file = "torch-2.0.0-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:ce9b5a49bd513dff7950a5a07d6e26594dd51989cee05ba388b03e8e366fd5d5"}, - {file = "torch-2.0.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:53e1c33c6896583cdb9a583693e22e99266444c4a43392dddc562640d39e542b"}, - {file = "torch-2.0.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:09651bff72e439d004c991f15add0c397c66f98ab36fe60d5514b44e4da722e8"}, - {file = "torch-2.0.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d439aec349c98f12819e8564b8c54008e4613dd4428582af0e6e14c24ca85870"}, - {file = "torch-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:2802f84f021907deee7e9470ed10c0e78af7457ac9a08a6cd7d55adef835fede"}, - {file = "torch-2.0.0-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:01858620f25f25e7a9ec4b547ff38e5e27c92d38ec4ccba9cfbfb31d7071ed9c"}, - {file = "torch-2.0.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:9a2e53b5783ef5896a6af338b36d782f28e83c8ddfc2ac44b67b066d9d76f498"}, - {file = "torch-2.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:ec5fff2447663e369682838ff0f82187b4d846057ef4d119a8dea7772a0b17dd"}, - {file = "torch-2.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:11b0384fe3c18c01b8fc5992e70fc519cde65e44c51cc87be1838c1803daf42f"}, - {file = "torch-2.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:e54846aa63855298cfb1195487f032e413e7ac9cbfa978fda32354cc39551475"}, - {file = "torch-2.0.0-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:cc788cbbbbc6eb4c90e52c550efd067586c2693092cf367c135b34893a64ae78"}, - {file = "torch-2.0.0-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:d292640f0fd72b7a31b2a6e3b635eb5065fcbedd4478f9cad1a1e7a9ec861d35"}, - {file = "torch-2.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:6befaad784004b7af357e3d87fa0863c1f642866291f12a4c2af2de435e8ac5c"}, - {file = "torch-2.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a83b26bd6ae36fbf5fee3d56973d9816e2002e8a3b7d9205531167c28aaa38a7"}, - {file = "torch-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:c7e67195e1c3e33da53954b026e89a8e1ff3bc1aeb9eb32b677172d4a9b5dcbf"}, - {file = "torch-2.0.0-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6e0b97beb037a165669c312591f242382e9109a240e20054d5a5782d9236cad0"}, - {file = "torch-2.0.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:297a4919aff1c0f98a58ebe969200f71350a1d4d4f986dbfd60c02ffce780e99"}, + {file = "torch-2.0.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:8ced00b3ba471856b993822508f77c98f48a458623596a4c43136158781e306a"}, + {file = "torch-2.0.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:359bfaad94d1cda02ab775dc1cc386d585712329bb47b8741607ef6ef4950747"}, + {file = "torch-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7c84e44d9002182edd859f3400deaa7410f5ec948a519cc7ef512c2f9b34d2c4"}, + {file = "torch-2.0.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:567f84d657edc5582d716900543e6e62353dbe275e61cdc36eda4929e46df9e7"}, + {file = "torch-2.0.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:787b5a78aa7917465e9b96399b883920c88a08f4eb63b5a5d2d1a16e27d2f89b"}, + {file = "torch-2.0.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:e617b1d0abaf6ced02dbb9486803abfef0d581609b09641b34fa315c9c40766d"}, + {file = "torch-2.0.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:b6019b1de4978e96daa21d6a3ebb41e88a0b474898fe251fd96189587408873e"}, + {file = "torch-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:dbd68cbd1cd9da32fe5d294dd3411509b3d841baecb780b38b3b7b06c7754434"}, + {file = "torch-2.0.1-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:ef654427d91600129864644e35deea761fb1fe131710180b952a6f2e2207075e"}, + {file = "torch-2.0.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:25aa43ca80dcdf32f13da04c503ec7afdf8e77e3a0183dd85cd3e53b2842e527"}, + {file = "torch-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5ef3ea3d25441d3957348f7e99c7824d33798258a2bf5f0f0277cbcadad2e20d"}, + {file = "torch-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0882243755ff28895e8e6dc6bc26ebcf5aa0911ed81b2a12f241fc4b09075b13"}, + {file = "torch-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:f66aa6b9580a22b04d0af54fcd042f52406a8479e2b6a550e3d9f95963e168c8"}, + {file = "torch-2.0.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:1adb60d369f2650cac8e9a95b1d5758e25d526a34808f7448d0bd599e4ae9072"}, + {file = "torch-2.0.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:1bcffc16b89e296826b33b98db5166f990e3b72654a2b90673e817b16c50e32b"}, + {file = "torch-2.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:e10e1597f2175365285db1b24019eb6f04d53dcd626c735fc502f1e8b6be9875"}, + {file = "torch-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:423e0ae257b756bb45a4b49072046772d1ad0c592265c5080070e0767da4e490"}, + {file = "torch-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8742bdc62946c93f75ff92da00e3803216c6cce9b132fbca69664ca38cfb3e18"}, + {file = "torch-2.0.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:c62df99352bd6ee5a5a8d1832452110435d178b5164de450831a3a8cc14dc680"}, + {file = "torch-2.0.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:671a2565e3f63b8fe8e42ae3e36ad249fe5e567435ea27b94edaa672a7d0c416"}, ] tqdm = [ {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, ] -triton = [ - {file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"}, - {file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"}, - {file = "triton-2.0.0-1-cp36-cp36m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd"}, - {file = "triton-2.0.0-1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b"}, - {file = "triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef"}, - {file = "triton-2.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656"}, - {file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"}, - {file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"}, - {file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"}, - {file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"}, - {file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"}, - {file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"}, - {file = "triton-2.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fedce6a381901b1547e0e7e1f2546e4f65dca6d91e2d8a7305a2d1f5551895be"}, - {file = "triton-2.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75834f27926eab6c7f00ce73aaf1ab5bfb9bec6eb57ab7c0bfc0a23fac803b4c"}, - {file = "triton-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0117722f8c2b579cd429e0bee80f7731ae05f63fe8e9414acd9a679885fcbf42"}, - {file = "triton-2.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd9be5d0c2e45d2b7e6ddc6da20112b6862d69741576f9c3dbaf941d745ecae"}, - {file = "triton-2.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a0d2c3fc2eab4ba71384f2e785fbfd47aa41ae05fa58bf12cb31dcbd0aeceb"}, - {file = "triton-2.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c47b72c72693198163ece9d90a721299e4fb3b8e24fd13141e384ad952724f"}, -] typer = [ {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, ] typing-extensions = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.6.0-py3-none-any.whl", hash = "sha256:6ad00b63f849b7dcc313b70b6b304ed67b2b2963b3098a33efe18056b1a9a223"}, + {file = "typing_extensions-4.6.0.tar.gz", hash = "sha256:ff6b238610c747e44c268aa4bb23c8c735d665a63726df3f9431ce707f2aa768"}, ] urllib3 = [ - {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, - {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, -] -wheel = [ - {file = "wheel-0.40.0-py3-none-any.whl", hash = "sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247"}, - {file = "wheel-0.40.0.tar.gz", hash = "sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873"}, + {file = "urllib3-2.0.2-py3-none-any.whl", hash = "sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e"}, + {file = "urllib3-2.0.2.tar.gz", hash = "sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc"}, ] win32-setctime = [ {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index b8e15230..932a094c 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -15,7 +15,7 @@ grpcio-status = "^1.51.1" grpcio-reflection = "^1.51.1" grpc-interceptor = "^0.15.0" typer = "^0.6.1" -accelerate = { version = "^0.15.0", optional = true } +accelerate = { version = "^0.19.0", optional = true } bitsandbytes = { version = "^0.38.1", optional = true } safetensors = "0.3.1" loguru = "^0.6.0" diff --git a/server/requirements.txt b/server/requirements.txt index ad48bff9..50ba4e43 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,20 +1,20 @@ backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" bitsandbytes==0.38.1 ; python_version >= "3.9" and python_version < "4.0" -certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4" -charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4" +certifi==2023.5.7 ; python_version >= "3.9" and python_version < "4.0" +charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0" click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows" deprecated==1.2.13 ; python_version >= "3.9" and python_version < "4.0" filelock==3.12.0 ; python_version >= "3.9" and python_version < "4.0" -fsspec==2023.4.0 ; python_version >= "3.9" and python_version < "4.0" +fsspec==2023.5.0 ; python_version >= "3.9" and python_version < "4.0" googleapis-common-protos==1.59.0 ; python_version >= "3.9" and python_version < "4.0" -grpc-interceptor==0.15.1 ; python_version >= "3.9" and python_version < "4.0" -grpcio-reflection==1.54.0 ; python_version >= "3.9" and python_version < "4.0" -grpcio-status==1.54.0 ; python_version >= "3.9" and python_version < "4.0" -grpcio==1.54.0 ; python_version >= "3.9" and python_version < "4.0" +grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "4.0" +grpcio-reflection==1.55.0 ; python_version >= "3.9" and python_version < "4.0" +grpcio-status==1.55.0 ; python_version >= "3.9" and python_version < "4.0" +grpcio==1.55.0 ; python_version >= "3.9" and python_version < "4.0" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" huggingface-hub==0.14.0 ; python_version >= "3.9" and python_version < "4.0" -idna==3.4 ; python_version >= "3.9" and python_version < "4" +idna==3.4 ; python_version >= "3.9" and python_version < "4.0" loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" @@ -26,16 +26,16 @@ opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" -protobuf==4.22.3 ; python_version >= "3.9" and python_version < "4.0" +protobuf==4.23.1 ; python_version >= "3.9" and python_version < "4.0" pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" -requests==2.28.2 ; python_version >= "3.9" and python_version < "4" +requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0" safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" -sentencepiece==0.1.98 ; python_version >= "3.9" and python_version < "4.0" -setuptools==67.7.2 ; python_version >= "3.9" and python_version < "4.0" +sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "4.0" +setuptools==67.8.0 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" -typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "4.0" -urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4" +typing-extensions==4.6.0 ; python_version >= "3.9" and python_version < "4.0" +urllib3==2.0.2 ; python_version >= "3.9" and python_version < "4.0" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" From cfaa858070c5a99c9a683ffb5b9ada5e5a91e1ce Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 23 May 2023 18:16:48 +0200 Subject: [PATCH 248/793] feat(server): support fp16 for t5 (#360) Fixes #349 --- .../test_t5_sharded/test_t5_sharded.json | 60 +++++ .../test_t5_sharded/test_t5_sharded_load.json | 242 ++++++++++++++++++ integration-tests/models/test_flash_neox.py | 4 +- integration-tests/models/test_t5_sharded.py | 38 +++ server/text_generation_server/models/bloom.py | 5 +- server/text_generation_server/models/t5.py | 14 +- 6 files changed, 357 insertions(+), 6 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_t5_sharded/test_t5_sharded.json create mode 100644 integration-tests/models/__snapshots__/test_t5_sharded/test_t5_sharded_load.json create mode 100644 integration-tests/models/test_t5_sharded.py diff --git a/integration-tests/models/__snapshots__/test_t5_sharded/test_t5_sharded.json b/integration-tests/models/__snapshots__/test_t5_sharded/test_t5_sharded.json new file mode 100644 index 00000000..6090e2c9 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_t5_sharded/test_t5_sharded.json @@ -0,0 +1,60 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 7, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 3, + "logprob": -0.7001953, + "special": false, + "text": " " + }, + { + "id": 18, + "logprob": -1.1943359, + "special": false, + "text": "-" + }, + { + "id": 26937, + "logprob": -1.2099609, + "special": false, + "text": "196" + }, + { + "id": 3, + "logprob": -1.2451172, + "special": false, + "text": " " + }, + { + "id": 1956, + "logprob": -0.3322754, + "special": false, + "text": "°" + }, + { + "id": 254, + "logprob": -0.19213867, + "special": false, + "text": "C" + }, + { + "id": 1, + "logprob": -0.030151367, + "special": true, + "text": "" + } + ] + }, + "generated_text": "-196 °C" +} diff --git a/integration-tests/models/__snapshots__/test_t5_sharded/test_t5_sharded_load.json b/integration-tests/models/__snapshots__/test_t5_sharded/test_t5_sharded_load.json new file mode 100644 index 00000000..3e9af12e --- /dev/null +++ b/integration-tests/models/__snapshots__/test_t5_sharded/test_t5_sharded_load.json @@ -0,0 +1,242 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 7, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 3, + "logprob": -0.7001953, + "special": false, + "text": " " + }, + { + "id": 18, + "logprob": -1.1943359, + "special": false, + "text": "-" + }, + { + "id": 26937, + "logprob": -1.2119141, + "special": false, + "text": "196" + }, + { + "id": 3, + "logprob": -1.2480469, + "special": false, + "text": " " + }, + { + "id": 1956, + "logprob": -0.33203125, + "special": false, + "text": "°" + }, + { + "id": 254, + "logprob": -0.19250488, + "special": false, + "text": "C" + }, + { + "id": 1, + "logprob": -0.030166626, + "special": true, + "text": "" + } + ] + }, + "generated_text": "-196 °C" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 7, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 3, + "logprob": -0.7001953, + "special": false, + "text": " " + }, + { + "id": 18, + "logprob": -1.1943359, + "special": false, + "text": "-" + }, + { + "id": 26937, + "logprob": -1.2119141, + "special": false, + "text": "196" + }, + { + "id": 3, + "logprob": -1.2480469, + "special": false, + "text": " " + }, + { + "id": 1956, + "logprob": -0.33203125, + "special": false, + "text": "°" + }, + { + "id": 254, + "logprob": -0.19250488, + "special": false, + "text": "C" + }, + { + "id": 1, + "logprob": -0.030166626, + "special": true, + "text": "" + } + ] + }, + "generated_text": "-196 °C" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 7, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 3, + "logprob": -0.7001953, + "special": false, + "text": " " + }, + { + "id": 18, + "logprob": -1.1943359, + "special": false, + "text": "-" + }, + { + "id": 26937, + "logprob": -1.2119141, + "special": false, + "text": "196" + }, + { + "id": 3, + "logprob": -1.2480469, + "special": false, + "text": " " + }, + { + "id": 1956, + "logprob": -0.33203125, + "special": false, + "text": "°" + }, + { + "id": 254, + "logprob": -0.19250488, + "special": false, + "text": "C" + }, + { + "id": 1, + "logprob": -0.030166626, + "special": true, + "text": "" + } + ] + }, + "generated_text": "-196 °C" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 7, + "prefill": [ + { + "id": 0, + "logprob": null, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 3, + "logprob": -0.7001953, + "special": false, + "text": " " + }, + { + "id": 18, + "logprob": -1.1943359, + "special": false, + "text": "-" + }, + { + "id": 26937, + "logprob": -1.2099609, + "special": false, + "text": "196" + }, + { + "id": 3, + "logprob": -1.2451172, + "special": false, + "text": " " + }, + { + "id": 1956, + "logprob": -0.3322754, + "special": false, + "text": "°" + }, + { + "id": 254, + "logprob": -0.19213867, + "special": false, + "text": "C" + }, + { + "id": 1, + "logprob": -0.030151367, + "special": true, + "text": "" + } + ] + }, + "generated_text": "-196 °C" + } +] diff --git a/integration-tests/models/test_flash_neox.py b/integration-tests/models/test_flash_neox.py index ca5b33c1..ff9915ed 100644 --- a/integration-tests/models/test_flash_neox.py +++ b/integration-tests/models/test_flash_neox.py @@ -36,6 +36,8 @@ async def test_flash_neox_load(flash_neox, generate_load, response_snapshot): generated_texts = [r.generated_text for r in responses] assert len(generated_texts) == 4 - assert generated_texts, all([text == generated_texts[0] for text in generated_texts]) + assert generated_texts, all( + [text == generated_texts[0] for text in generated_texts] + ) assert responses == response_snapshot diff --git a/integration-tests/models/test_t5_sharded.py b/integration-tests/models/test_t5_sharded.py new file mode 100644 index 00000000..074660c7 --- /dev/null +++ b/integration-tests/models/test_t5_sharded.py @@ -0,0 +1,38 @@ +import pytest + + +@pytest.fixture(scope="module") +def t5_sharded_handle(launcher): + with launcher("google/flan-t5-xxl", num_shard=2) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def t5_sharded(t5_sharded_handle): + await t5_sharded_handle.health(240) + return t5_sharded_handle.client + + +@pytest.mark.asyncio +async def test_t5_sharded(t5_sharded, response_snapshot): + response = await t5_sharded.generate( + "Please answer the following question. What is the boiling point of Nitrogen?", + max_new_tokens=10, + ) + + assert response == response_snapshot + + +@pytest.mark.asyncio +async def test_t5_sharded_load(t5_sharded, generate_load, response_snapshot): + responses = await generate_load( + t5_sharded, + "Please answer the following question. What is the boiling point of Nitrogen?", + max_new_tokens=10, + n=4, + ) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) + + assert responses == response_snapshot diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 390f0a0a..9d609185 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -160,7 +160,10 @@ def load_weights( # XXX: Hack for Rowlinear to add the bias only once. if rank != 0: tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding) or name == "lm_head.weight": + elif ( + isinstance(module, TensorParallelEmbedding) + or name == "lm_head.weight" + ): size = slice_.get_shape()[0] block_size = size // world_size start = rank * block_size diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 2fd67574..7ecf948b 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -40,7 +40,7 @@ def __init__( self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32 + dtype = torch.float16 else: device = torch.device("cpu") dtype = torch.float32 @@ -154,9 +154,15 @@ def load_weights( f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" ) - tensor = tensor.contiguous().to(dtype) + tensor = tensor.contiguous() - if quantize == "bitsandbytes": + # See: https://github.com/huggingface/transformers/blob/1fe1e3caa44617047f149bcc0c0b566343b714a7/src/transformers/models/t5/modeling_t5.py#LL316C15-L316C71 + if module_name.endswith("wo"): + tensor = tensor.to(torch.float32) + else: + tensor = tensor.to(dtype) + + if quantize == "bitsandbytes" and not module_name.endswith("wo"): if not HAS_BITS_AND_BYTES: raise ImportError( "bitsandbytes is not available on your machine either because it is not installed " @@ -207,7 +213,7 @@ def linear(input, weight, bias): module.linear = replace_linear(state) - elif quantize == "gptq": + elif quantize == "gptq" and not module_name.endswith("wo"): raise NotImplementedError( "`gptq` is not implemented for now" ) From e9669a4085a0f031df10585090fe656d4675d492 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 23 May 2023 19:12:12 +0200 Subject: [PATCH 249/793] feat(server): do not use device_map auto on single GPU (#362) --- server/text_generation_server/models/causal_lm.py | 5 ++++- server/text_generation_server/models/seq2seq_lm.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 90b1e5ee..ab92feed 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -468,9 +468,12 @@ def __init__( model_id, revision=revision, torch_dtype=dtype, - device_map="auto" if torch.cuda.is_available() else None, + device_map="auto" if torch.cuda.is_available() and torch.cuda.device_count() > 1 else None, load_in_8bit=quantize == "bitsandbytes", ) + if torch.cuda.is_available() and torch.cuda.device_count() == 1: + model = model.cuda() + tokenizer.pad_token_id = ( model.config.pad_token_id if model.config.pad_token_id is not None diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 4f55b22f..f8b404a9 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -518,9 +518,12 @@ def __init__( model_id, revision=revision, torch_dtype=dtype, - device_map="auto" if torch.cuda.is_available() else None, + device_map="auto" if torch.cuda.is_available() and torch.cuda.device_count() > 1 else None, load_in_8bit=quantize == "bitsandbytes", ) + if torch.cuda.is_available() and torch.cuda.device_count() == 1: + model = model.cuda() + tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", truncation_side="left" ) From e3e487dc711449c23826cfe1d74786f71309d6bd Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 23 May 2023 20:40:39 +0200 Subject: [PATCH 250/793] feat(server): support trust_remote_code (#363) --- .github/workflows/build.yaml | 3 +- launcher/src/main.rs | 28 +++- server/text_generation_server/cli.py | 3 +- .../text_generation_server/models/__init__.py | 147 +++++++++++++++--- server/text_generation_server/models/bloom.py | 23 ++- .../models/causal_lm.py | 49 ++++-- .../models/flash_causal_lm.py | 8 +- .../models/flash_llama.py | 10 +- .../models/flash_neox.py | 17 +- .../models/flash_santacoder.py | 14 +- .../models/galactica.py | 16 +- .../text_generation_server/models/gpt_neox.py | 16 +- server/text_generation_server/models/opt.py | 16 +- .../models/santacoder.py | 9 +- .../models/seq2seq_lm.py | 12 +- server/text_generation_server/models/t5.py | 16 +- server/text_generation_server/server.py | 6 +- 17 files changed, 321 insertions(+), 72 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9992e0a2..124e6a33 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -213,13 +213,12 @@ jobs: sudo mount /dev/nvme1n1 ${{ env.DOCKER_VOLUME }} - name: Install run: | - pip install pytest-xdist make install-integration-tests - name: Run tests run: | export DOCKER_IMAGE=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }} export HUGGING_FACE_HUB_TOKEN=${{ secrets.HUGGING_FACE_HUB_TOKEN }} - pytest -s -vv -n 2 --dist loadfile integration-tests + pytest -s -vv integration-tests stop-runner: name: Stop self-hosted EC2 runner diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 4a6cd621..dc12c90f 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -53,7 +53,7 @@ struct Args { #[clap(long, env)] revision: Option, - /// Wether to shard or not the model across multiple GPUs + /// Whether to shard the model across multiple GPUs /// By default text-generation-inference will use all available GPUs to run /// the model. Setting it to `false` deactivates `num_shard`. #[clap(long, env)] @@ -66,11 +66,17 @@ struct Args { #[clap(long, env)] num_shard: Option, - /// Wether you want the model to be quantized or not. This will use `bitsandbytes` for + /// Whether you want the model to be quantized. This will use `bitsandbytes` for /// quantization on the fly, or `gptq`. #[clap(long, env, value_enum)] quantize: Option, + /// Whether you want to execute hub modelling code. Explicitly passing a `revision` is + /// encouraged when loading a model with custom code to ensure no malicious code has been + /// contributed in a newer revision. + #[clap(long, env, value_enum)] + trust_remote_code: bool, + /// The maximum amount of concurrent requests for this particular deployment. /// Having a low limit will refuse clients requests instead of having them /// wait for too long and is usually good to handle backpressure correctly. @@ -239,6 +245,7 @@ fn shard_manager( model_id: String, revision: Option, quantize: Option, + trust_remote_code: bool, uds_path: String, rank: usize, world_size: usize, @@ -272,6 +279,11 @@ fn shard_manager( "--json-output".to_string(), ]; + // Activate trust remote code + if trust_remote_code { + shard_argv.push("--trust-remote-code".to_string()); + } + // Activate tensor parallelism if world_size > 1 { shard_argv.push("--sharded".to_string()); @@ -692,6 +704,16 @@ fn spawn_shards( status_sender: mpsc::Sender, running: Arc, ) -> Result<(), LauncherError> { + if args.trust_remote_code { + tracing::warn!( + "`trust_remote_code` is set. Trusting that model `{}` do not contain malicious code.", + args.model_id + ); + if args.revision.is_none() { + tracing::warn!("Explicitly passing a `revision` is encouraged when loading a model with custom code to ensure no malicious code has been contributed in a newer revision."); + } + } + // Start shard processes for rank in 0..num_shard { let model_id = args.model_id.clone(); @@ -705,6 +727,7 @@ fn spawn_shards( let shutdown_sender = shutdown_sender.clone(); let otlp_endpoint = args.otlp_endpoint.clone(); let quantize = args.quantize; + let trust_remote_code = args.trust_remote_code; let master_port = args.master_port; let disable_custom_kernels = args.disable_custom_kernels; let watermark_gamma = args.watermark_gamma; @@ -714,6 +737,7 @@ fn spawn_shards( model_id, revision, quantize, + trust_remote_code, uds_path, rank, num_shard, diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 35cad20f..c0e6c2dc 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -22,6 +22,7 @@ def serve( revision: Optional[str] = None, sharded: bool = False, quantize: Optional[Quantization] = None, + trust_remote_code: bool = False, uds_path: Path = "/tmp/text-generation-server", logger_level: str = "INFO", json_output: bool = False, @@ -63,7 +64,7 @@ def serve( # Downgrade enum into str for easier management later on quantize = None if quantize is None else quantize.value - server.serve(model_id, revision, sharded, quantize, uds_path) + server.serve(model_id, revision, sharded, quantize, trust_remote_code, uds_path) @app.command() diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index ec990fde..bf7a2849 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -91,13 +91,27 @@ def get_model( - model_id: str, revision: Optional[str], sharded: bool, quantize: Optional[str] + model_id: str, + revision: Optional[str], + sharded: bool, + quantize: Optional[str], + trust_remote_code: bool, ) -> Model: if "facebook/galactica" in model_id: if sharded: - return GalacticaSharded(model_id, revision, quantize=quantize) + return GalacticaSharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) else: - return Galactica(model_id, revision, quantize=quantize) + return Galactica( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) if model_id.startswith("bigcode/"): if sharded: @@ -105,12 +119,24 @@ def get_model( raise NotImplementedError( FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Santacoder") ) - return FlashSantacoderSharded(model_id, revision, quantize=quantize) + return FlashSantacoderSharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) else: santacoder_cls = FlashSantacoder if FLASH_ATTENTION else SantaCoder - return santacoder_cls(model_id, revision, quantize=quantize) + return santacoder_cls( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) - config = AutoConfig.from_pretrained(model_id, revision=revision) + config = AutoConfig.from_pretrained( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) model_type = config.model_type if model_type == "gpt_bigcode": @@ -119,52 +145,133 @@ def get_model( raise NotImplementedError( FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Santacoder") ) - return FlashSantacoderSharded(model_id, revision, quantize=quantize) + return FlashSantacoderSharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) else: santacoder_cls = FlashSantacoder if FLASH_ATTENTION else SantaCoder - return santacoder_cls(model_id, revision, quantize=quantize) + return santacoder_cls( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) if model_type == "bloom": if sharded: - return BLOOMSharded(model_id, revision, quantize=quantize) + return BLOOMSharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) else: - return BLOOM(model_id, revision, quantize=quantize) + return BLOOM( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) if model_type == "gpt_neox": if sharded: neox_cls = FlashNeoXSharded if FLASH_ATTENTION else GPTNeoxSharded - return neox_cls(model_id, revision, quantize=quantize) + return neox_cls( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) else: neox_cls = FlashNeoX if FLASH_ATTENTION else CausalLM - return neox_cls(model_id, revision, quantize=quantize) + return neox_cls( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) if model_type == "llama": if sharded: if FLASH_ATTENTION: - return FlashLlamaSharded(model_id, revision, quantize=quantize) + return FlashLlamaSharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Llama")) else: llama_cls = FlashLlama if FLASH_ATTENTION else CausalLM - return llama_cls(model_id, revision, quantize=quantize) + return llama_cls( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) if config.model_type == "opt": if sharded: - return OPTSharded(model_id, revision, quantize=quantize) + return OPTSharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) else: - return OPT(model_id, revision, quantize=quantize) + return OPT( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) if model_type == "t5": if sharded: - return T5Sharded(model_id, revision, quantize=quantize) + return T5Sharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) else: - return Seq2SeqLM(model_id, revision, quantize=quantize) + return Seq2SeqLM( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) if sharded: raise ValueError("sharded is not supported for AutoModel") if model_type in modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: - return CausalLM(model_id, revision, quantize=quantize) + return CausalLM( + model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + ) if model_type in modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES: - return Seq2SeqLM(model_id, revision, quantize=quantize) + return Seq2SeqLM( + model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + ) + + auto_map = getattr(config, "auto_map", None) + if trust_remote_code and auto_map is not None: + if "AutoModelForCausalLM" in auto_map.keys(): + return CausalLM( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) + if "AutoModelForSeq2SeqLM" in auto_map.keys: + return Seq2SeqLM( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) raise ValueError(f"Unsupported model type {model_type}") diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 9d609185..5eddc8cf 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -54,9 +54,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): super(BLOOM, self).__init__( - model_id=model_id, revision=revision, quantize=quantize + model_id=model_id, + revision=revision, + quantize=quantize, + trust_remote_code=trust_remote_code, ) @property @@ -70,6 +74,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -80,11 +85,19 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) config = AutoConfig.from_pretrained( - model_id, revision=revision, slow_but_exact=False, tp_parallel=True + model_id, + revision=revision, + slow_but_exact=False, + tp_parallel=True, + trust_remote_code=trust_remote_code, ) config.pad_token_id = 3 @@ -92,7 +105,9 @@ def __init__( filenames = weight_files(model_id, revision=revision, extension=".safetensors") with init_empty_weights(): - model = AutoModelForCausalLM.from_config(config) + model = AutoModelForCausalLM.from_config( + config, trust_remote_code=trust_remote_code + ) torch.distributed.barrier(group=self.process_group) self.load_weights( diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index ab92feed..09df70d2 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -1,4 +1,5 @@ import torch +import inspect from dataclasses import dataclass from opentelemetry import trace @@ -450,6 +451,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -462,22 +464,38 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) model = AutoModelForCausalLM.from_pretrained( model_id, revision=revision, torch_dtype=dtype, - device_map="auto" if torch.cuda.is_available() and torch.cuda.device_count() > 1 else None, + device_map="auto" + if torch.cuda.is_available() and torch.cuda.device_count() > 1 + else None, load_in_8bit=quantize == "bitsandbytes", + trust_remote_code=trust_remote_code, ) if torch.cuda.is_available() and torch.cuda.device_count() == 1: model = model.cuda() - tokenizer.pad_token_id = ( - model.config.pad_token_id - if model.config.pad_token_id is not None - else model.config.eos_token_id + if tokenizer.pad_token_id is None: + if model.config.pad_token_id is not None: + tokenizer.pad_token_id = model.config.pad_token_id + elif model.config.eos_token_id is not None: + tokenizer.pad_token_id = model.config.eos_token_id + elif tokenizer.eos_token_id is not None: + tokenizer.pad_token_id = tokenizer.eos_token_id + else: + tokenizer.add_special_tokens({"pad_token": "[PAD]"}) + + self.has_position_ids = ( + inspect.signature(model.forward).parameters.get("position_ids", None) + is not None ) super(CausalLM, self).__init__( @@ -501,14 +519,17 @@ def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: # Model Forward - outputs = self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=True, - return_dict=True, - ) + kwargs = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "past_key_values": past_key_values, + "use_cache": True, + "return_dict": True, + } + if self.has_position_ids: + kwargs["position_ids"] = position_ids + + outputs = self.model.forward(**kwargs) return outputs.logits, outputs.past_key_values @tracer.start_as_current_span("generate_token") diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index aee0480d..d6e73ad8 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -394,6 +394,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -402,13 +403,18 @@ def __init__( raise NotImplementedError("FlashCausalLM is only available on GPU") tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) model = model_cls.from_pretrained( model_id, revision=revision, torch_dtype=dtype, load_in_8bit=quantize == "bitsandbytes", + trust_remote_code=trust_remote_code, ).to(device) super(FlashCausalLM, self).__init__( diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index ebdbe206..fe28580d 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -33,6 +33,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -45,11 +46,11 @@ def __init__( revision=revision, padding_side="left", truncation_side="left", + trust_remote_code=trust_remote_code, ) config = AutoConfig.from_pretrained( - model_id, - revision=revision, + model_id, revision=revision, trust_remote_code=trust_remote_code ) # We do not use from_pretrained as we modified the model internal module layout @@ -153,6 +154,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -166,11 +168,11 @@ def __init__( revision=revision, padding_side="left", truncation_side="left", + trust_remote_code=trust_remote_code, ) config = AutoConfig.from_pretrained( - model_id, - revision=revision, + model_id, revision=revision, trust_remote_code=trust_remote_code ) torch.distributed.barrier(group=self.process_group) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 8c1c1a00..31ae7914 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -28,9 +28,14 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): super(FlashNeoX, self).__init__( - FlashGPTNeoXForCausalLM, model_id, revision, quantize + FlashGPTNeoXForCausalLM, + model_id, + revision, + quantize, + trust_remote_code=trust_remote_code, ) @@ -40,6 +45,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -49,12 +55,15 @@ def __init__( raise NotImplementedError("FlashNeoX is only available on GPU") tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) config = AutoConfig.from_pretrained( - model_id, - revision=revision, + model_id, revision=revision, trust_remote_code=trust_remote_code ) torch.distributed.barrier(group=self.process_group) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 1fbaf252..482e0f54 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -32,6 +32,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -40,7 +41,11 @@ def __init__( raise NotImplementedError("FlashSantacoder is only available on GPU") tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) config = GPT2Config.from_pretrained( @@ -178,6 +183,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -187,7 +193,11 @@ def __init__( raise NotImplementedError("FlashSantacoderSharded is only available on GPU") tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) config = GPT2Config.from_pretrained( diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 24c37c19..bc3096c6 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -199,6 +199,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -209,11 +210,18 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) config = AutoConfig.from_pretrained( - model_id, revision=revision, tp_parallel=True + model_id, + revision=revision, + tp_parallel=True, + trust_remote_code=trust_remote_code, ) tokenizer.pad_token_id = config.pad_token_id @@ -221,7 +229,9 @@ def __init__( filenames = weight_files(model_id, revision=revision, extension=".safetensors") with init_empty_weights(): - model = AutoModelForCausalLM.from_config(config) + model = AutoModelForCausalLM.from_config( + config, trust_remote_code=trust_remote_code + ) torch.distributed.barrier(group=self.process_group) self.load_weights( diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index a10dfcb8..e4a85082 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -36,6 +36,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -46,19 +47,28 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) tokenizer.pad_token = tokenizer.eos_token config = AutoConfig.from_pretrained( - model_id, revision=revision, tp_parallel=True + model_id, + revision=revision, + tp_parallel=True, + trust_remote_code=trust_remote_code, ) torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") with init_empty_weights(): - model = AutoModelForCausalLM.from_config(config) + model = AutoModelForCausalLM.from_config( + config, trust_remote_code=trust_remote_code + ) torch.distributed.barrier(group=self.process_group) self.load_weights( diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 8c676a51..bccce5b3 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -52,6 +52,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -62,11 +63,18 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) config = AutoConfig.from_pretrained( - model_id, revision=revision, tp_parallel=True + model_id, + revision=revision, + tp_parallel=True, + trust_remote_code=trust_remote_code, ) tokenizer.pad_token_id = config.pad_token_id @@ -74,7 +82,9 @@ def __init__( filenames = weight_files(model_id, revision=revision, extension=".safetensors") with init_empty_weights(): - model = AutoModelForCausalLM.from_config(config) + model = AutoModelForCausalLM.from_config( + config, trust_remote_code=trust_remote_code + ) torch.distributed.barrier(group=self.process_group) self.load_weights( diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index 23f89f48..d0fd3070 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -19,6 +19,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -31,7 +32,11 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) tokenizer.add_special_tokens( { @@ -51,7 +56,7 @@ def __init__( revision=revision, torch_dtype=dtype, load_in_8bit=quantize == "bitsandbytes", - trust_remote_code=True, # required + trust_remote_code=trust_remote_code, ).to(device) super(CausalLM, self).__init__( diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index f8b404a9..a1a39fd4 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -503,6 +503,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") @@ -518,14 +519,21 @@ def __init__( model_id, revision=revision, torch_dtype=dtype, - device_map="auto" if torch.cuda.is_available() and torch.cuda.device_count() > 1 else None, + device_map="auto" + if torch.cuda.is_available() and torch.cuda.device_count() > 1 + else None, load_in_8bit=quantize == "bitsandbytes", + trust_remote_code=trust_remote_code, ) if torch.cuda.is_available() and torch.cuda.device_count() == 1: model = model.cuda() tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) tokenizer.bos_token_id = model.config.decoder_start_token_id diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 7ecf948b..17cc50e0 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -36,6 +36,7 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -46,11 +47,18 @@ def __init__( dtype = torch.float32 tokenizer = AutoTokenizer.from_pretrained( - model_id, revision=revision, padding_side="left", truncation_side="left" + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, ) config = AutoConfig.from_pretrained( - model_id, revision=revision, tp_parallel=True + model_id, + revision=revision, + tp_parallel=True, + trust_remote_code=trust_remote_code, ) tokenizer.bos_token_id = config.decoder_start_token_id @@ -58,7 +66,9 @@ def __init__( filenames = weight_files(model_id, revision=revision, extension=".safetensors") with init_empty_weights(): - model = AutoModelForSeq2SeqLM.from_config(config) + model = AutoModelForSeq2SeqLM.from_config( + config, trust_remote_code=trust_remote_code + ) torch.distributed.barrier(group=self.process_group) self.load_weights( diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index d715207b..7ca5054e 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -101,6 +101,7 @@ def serve( revision: Optional[str], sharded: bool, quantize: Optional[str], + trust_remote_code: bool, uds_path: Path, ): async def serve_inner( @@ -108,6 +109,7 @@ async def serve_inner( revision: Optional[str], sharded: bool = False, quantize: Optional[str] = None, + trust_remote_code: bool = False, ): unix_socket_template = "unix://{}-{}" if sharded: @@ -121,7 +123,7 @@ async def serve_inner( server_urls = [local_url] try: - model = get_model(model_id, revision, sharded, quantize) + model = get_model(model_id, revision, sharded, quantize, trust_remote_code) except Exception: logger.exception("Error when initializing model") raise @@ -152,4 +154,4 @@ async def serve_inner( logger.info("Signal received. Shutting down") await server.stop(0) - asyncio.run(serve_inner(model_id, revision, sharded, quantize)) + asyncio.run(serve_inner(model_id, revision, sharded, quantize, trust_remote_code)) From 942005386aed8b543c22fa6d7888af65e6c26c85 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 23 May 2023 20:47:37 +0200 Subject: [PATCH 251/793] feat(router): log input/ouput at debug level (#364) @njhill FYI --- integration-tests/conftest.py | 7 +++++-- router/src/server.rs | 20 +++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 3086ecda..7db12424 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -231,8 +231,11 @@ def local_launcher( if quantize: args.append("--quantize") + env = os.environ + env["LOG_LEVEL"] = "info,text_generation_router=debug" + with subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) as process: yield ProcessLauncherHandle(process, port) @@ -271,7 +274,7 @@ def docker_launcher( gpu_count = num_shard if num_shard is not None else 1 - env = {} + env = {"LOG_LEVEL": "info,text_generation_router=debug"} if HUGGING_FACE_HUB_TOKEN is not None: env["HUGGING_FACE_HUB_TOKEN"] = HUGGING_FACE_HUB_TOKEN diff --git a/router/src/server.rs b/router/src/server.rs index 162b6fd9..fd6a66bb 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -52,7 +52,7 @@ use utoipa_swagger_ui::SwaggerUi; example = json ! ({"error": "Incomplete generation"})), ) )] -#[instrument(skip(infer))] +#[instrument(skip(infer, req))] async fn compat_generate( default_return_full_text: Extension, infer: Extension, @@ -133,8 +133,9 @@ async fn health(mut health: Extension) -> Result<(), (StatusCode, Json {} // Yield event for every new token InferStreamResponse::Token(token) => { + tracing::debug!(parent: &span, "Token: {:?}", token); + // StreamResponse let stream_token = StreamResponse { token, @@ -428,7 +437,8 @@ async fn generate_stream( output_text = prompt + &output_text; } - tracing::info!(parent: &span, "Output: {}", output_text); + tracing::debug!(parent: &span, "Output: {}", output_text); + tracing::info!(parent: &span, "Success"); let stream_token = StreamResponse { token, From d31562f300ee8cd44b3ead667a9f260d6a762f7a Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 23 May 2023 21:20:49 +0200 Subject: [PATCH 252/793] v0.7.0 (#353) --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- README.md | 2 +- docs/openapi.json | 2 +- server/pyproject.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48595077..1dcc7c97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2516,7 +2516,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "0.7.0-dev" +version = "0.7.0" dependencies = [ "average", "clap", @@ -2535,7 +2535,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.7.0-dev" +version = "0.7.0" dependencies = [ "futures", "grpc-metadata", @@ -2551,7 +2551,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.7.0-dev" +version = "0.7.0" dependencies = [ "clap", "ctrlc", @@ -2567,7 +2567,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.7.0-dev" +version = "0.7.0" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index d34b10bf..4ed0524e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.7.0-dev" +version = "0.7.0" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index 918ea5e2..30e085b2 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ model=bigscience/bloom-560m num_shard=2 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --num-shard $num_shard +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.7 --model-id $model --num-shard $num_shard ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. diff --git a/docs/openapi.json b/docs/openapi.json index fbac7786..5bccc9fc 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.7.0-dev" + "version": "0.7.0" }, "paths": { "/": { diff --git a/server/pyproject.toml b/server/pyproject.toml index 932a094c..d674172d 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.6.0" +version = "0.7.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 218c9adaa522501a8dc726551b0be2b7feb41475 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 24 May 2023 19:19:57 +0200 Subject: [PATCH 253/793] feat: decrease IPC proto size (#367) Closes #307 #308 --- benchmark/src/generation.rs | 6 ++-- proto/generate.proto | 23 ++++++++++---- router/client/src/client.rs | 15 +++++---- router/client/src/lib.rs | 4 +-- router/client/src/sharded_client.rs | 25 ++++++++------- router/src/infer.rs | 18 +++++------ server/tests/models/test_bloom.py | 6 ++-- server/tests/models/test_causal_lm.py | 6 ++-- server/tests/models/test_seq2seq_lm.py | 6 ++-- .../models/causal_lm.py | 22 +++++++------ .../models/flash_causal_lm.py | 31 ++++++++++--------- .../models/seq2seq_lm.py | 26 ++++++++-------- server/text_generation_server/models/types.py | 4 +-- server/text_generation_server/server.py | 4 +-- 14 files changed, 108 insertions(+), 88 deletions(-) diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs index 4a119e86..d40a2e8d 100644 --- a/benchmark/src/generation.rs +++ b/benchmark/src/generation.rs @@ -1,6 +1,6 @@ use std::time::{Duration, Instant}; use text_generation_client::{ - Batch, ClientError, NextTokenChooserParameters, Request, ShardedClient, + Batch, CachedBatch, ClientError, NextTokenChooserParameters, Request, ShardedClient, StoppingCriteriaParameters, }; use tokenizers::{Tokenizer, TruncationDirection}; @@ -126,7 +126,7 @@ async fn prefill( batch_size: u32, decode_length: u32, client: &mut ShardedClient, -) -> Result<(Prefill, Batch), ClientError> { +) -> Result<(Prefill, CachedBatch), ClientError> { // Create requests let requests = (0..batch_size) .map(|id| Request { @@ -180,7 +180,7 @@ async fn prefill( } /// Run a full decode -async fn decode(batch: Batch, client: &mut ShardedClient) -> Result { +async fn decode(batch: CachedBatch, client: &mut ShardedClient) -> Result { let mut decode_length = 0; let batch_size = batch.size; diff --git a/proto/generate.proto b/proto/generate.proto index 894d7bc1..0c40e5bb 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -100,6 +100,17 @@ message Batch { uint32 max_tokens = 4; } +message CachedBatch { + /// Batch ID + uint64 id = 1; + /// Individual requests ids + repeated uint64 request_ids = 2; + /// Batch size (==len(requests)) + uint32 size = 3; + /// Maximum number of tokens this batch will grow to + uint32 max_tokens = 4; +} + enum FinishReason { FINISH_REASON_LENGTH = 0; FINISH_REASON_EOS_TOKEN = 1; @@ -140,19 +151,19 @@ message Generation { /// Is it a special token bool token_is_special = 6; /// Complete generated text - GeneratedText generated_text = 7; + optional GeneratedText generated_text = 7; } message FilterBatchRequest { /// Batch ID uint64 batch_id = 1; /// Requests to keep - repeated Request keep_requests = 2; + repeated uint64 request_ids = 2; } message FilterBatchResponse { /// Filtered Batch (cached) - Batch batch = 1; + CachedBatch batch = 1; } @@ -165,17 +176,17 @@ message PrefillResponse { /// Generation repeated Generation generations = 1; /// Next batch (cached) - optional Batch batch = 2; + optional CachedBatch batch = 2; } message DecodeRequest { /// Cached batches - repeated Batch batches = 1; + repeated CachedBatch batches = 1; } message DecodeResponse { /// Decodes repeated Generation generations = 1; /// Next batch (cached) - optional Batch batch = 2; + optional CachedBatch batch = 2; } diff --git a/router/client/src/client.rs b/router/client/src/client.rs index bf1b6b58..81f023ef 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -83,11 +83,11 @@ impl Client { pub async fn filter_batch( &mut self, batch_id: u64, - keep_requests: Vec, - ) -> Result> { + request_ids: Vec, + ) -> Result> { let request = tonic::Request::new(FilterBatchRequest { batch_id, - keep_requests, + request_ids, }) .inject_context(); let filtered_batch = self.stub.filter_batch(request).await?.into_inner(); @@ -99,7 +99,10 @@ impl Client { /// Returns Generation for each request in batch /// and the next cached batch #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))] - pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { + pub async fn prefill( + &mut self, + batch: Batch, + ) -> Result<(Vec, Option)> { let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }).inject_context(); let response = self.stub.prefill(request).await?.into_inner(); Ok((response.generations, response.batch)) @@ -112,8 +115,8 @@ impl Client { #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::()))] pub async fn decode( &mut self, - batches: Vec, - ) -> Result<(Vec, Option)> { + batches: Vec, + ) -> Result<(Vec, Option)> { let request = tonic::Request::new(DecodeRequest { batches }).inject_context(); let response = self.stub.decode(request).await?.into_inner(); Ok((response.generations, response.batch)) diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 66f2055a..f334be21 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -9,8 +9,8 @@ pub use client::Client; pub use pb::generate::v1::HealthResponse; pub use pb::generate::v1::InfoResponse as ShardInfo; pub use pb::generate::v1::{ - Batch, FinishReason, GeneratedText, Generation, NextTokenChooserParameters, PrefillTokens, - Request, StoppingCriteriaParameters, + Batch, CachedBatch, FinishReason, GeneratedText, Generation, NextTokenChooserParameters, + PrefillTokens, Request, StoppingCriteriaParameters, }; pub use sharded_client::ShardedClient; use thiserror::Error; diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 60b81fe6..b81eed46 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,5 +1,5 @@ /// Multi shard Client -use crate::{Batch, Client, Generation, HealthResponse, Request, ShardInfo}; +use crate::{Batch, CachedBatch, Client, Generation, HealthResponse, ShardInfo}; use crate::{ClientError, Result}; use futures::future::join_all; use tonic::transport::Uri; @@ -76,12 +76,12 @@ impl ShardedClient { pub async fn filter_batch( &mut self, batch_id: u64, - keep_requests: Vec, - ) -> Result> { + request_ids: Vec, + ) -> Result> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| Box::pin(client.filter_batch(batch_id, keep_requests.clone()))) + .map(|client| Box::pin(client.filter_batch(batch_id, request_ids.clone()))) .collect(); // all shards return the same message join_all(futures).await.pop().unwrap() @@ -92,13 +92,16 @@ impl ShardedClient { /// Returns Generation for each request in batch /// and the next cached batch #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))] - pub async fn prefill(&mut self, batch: Batch) -> Result<(Vec, Option)> { + pub async fn prefill( + &mut self, + batch: Batch, + ) -> Result<(Vec, Option)> { let futures: Vec<_> = self .clients .iter_mut() .map(|client| Box::pin(client.prefill(batch.clone()))) .collect(); - let results: Result, Option)>> = + let results: Result, Option)>> = join_all(futures).await.into_iter().collect(); merge_generations(results?) } @@ -110,14 +113,14 @@ impl ShardedClient { #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::()))] pub async fn decode( &mut self, - batches: Vec, - ) -> Result<(Vec, Option)> { + batches: Vec, + ) -> Result<(Vec, Option)> { let futures: Vec<_> = self .clients .iter_mut() .map(|client| Box::pin(client.decode(batches.clone()))) .collect(); - let results: Result, Option)>> = + let results: Result, Option)>> = join_all(futures).await.into_iter().collect(); merge_generations(results?) } @@ -125,8 +128,8 @@ impl ShardedClient { /// Merge generations from the different model shards fn merge_generations( - mut results: Vec<(Vec, Option)>, -) -> Result<(Vec, Option)> { + mut results: Vec<(Vec, Option)>, +) -> Result<(Vec, Option)> { let (mut generations, next_batch) = results.pop().ok_or(ClientError::EmptyResults)?; for (mut shard_generations, _) in results.into_iter() { diff --git a/router/src/infer.rs b/router/src/infer.rs index 313ec3e1..00fa2818 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -12,7 +12,7 @@ use std::sync::{ Arc, }; use text_generation_client::{ - Batch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, + Batch, CachedBatch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, }; use thiserror::Error; use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; @@ -352,7 +352,7 @@ async fn prefill( batch: Batch, entries: &mut IntMap, generation_health: &Arc, -) -> Option { +) -> Option { let start_time = Instant::now(); let batch_id = batch.id; metrics::increment_counter!("tgi_batch_inference_count", "method" => "prefill"); @@ -386,10 +386,10 @@ async fn prefill( #[instrument(skip_all)] async fn decode( client: &mut ShardedClient, - batches: Vec, + batches: Vec, entries: &mut IntMap, generation_health: &Arc, -) -> Option { +) -> Option { let start_time = Instant::now(); let batch_ids: Vec = batches.iter().map(|b| b.id).collect(); metrics::increment_counter!("tgi_batch_inference_count", "method" => "decode"); @@ -425,9 +425,9 @@ async fn decode( #[instrument(skip_all)] async fn filter_batch( client: &mut ShardedClient, - next_batch: Option, + next_batch: Option, entries: &IntMap, -) -> Option { +) -> Option { let mut batch = next_batch?; // No need to filter @@ -438,9 +438,9 @@ async fn filter_batch( let id = batch.id; // Retain only requests that are still in entries - batch.requests.retain(|r| entries.contains_key(&r.id)); + batch.request_ids.retain(|id| entries.contains_key(id)); - if batch.requests.is_empty() { + if batch.request_ids.is_empty() { // All requests have been filtered out // Next batch is now empty // Clear it from the Python shards cache @@ -450,7 +450,7 @@ async fn filter_batch( } else { // Filter Python shard cache // We unwrap here as we need to panic since we cannot recover if this method fails - client.filter_batch(id, batch.requests).await.unwrap() + client.filter_batch(id, batch.request_ids).await.unwrap() } } diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index f0adab97..105b3573 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -178,7 +178,7 @@ def test_causal_lm_generate_token_completion_multi( # Copy stopping_criterias before filtering stopping_criterias = default_multi_requests_bloom_batch.stopping_criterias.copy() - next_batch = next_batch.filter([next_batch.requests[0]]) + next_batch = next_batch.filter([next_batch.requests[0].id]) for _ in range( stopping_criterias[0].max_new_tokens - stopping_criterias[1].max_new_tokens - 1 @@ -286,7 +286,7 @@ def test_batch_concatenate( == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) - next_batch = next_batch.filter([next_batch.requests[0], next_batch.requests[1]]) + next_batch = next_batch.filter([next_batch.requests[0].id, next_batch.requests[1].id]) for _ in range( default_bloom_batch.stopping_criterias[0].max_new_tokens @@ -309,7 +309,7 @@ def test_batch_concatenate( == default_bloom_batch.stopping_criterias[0].max_new_tokens ) - next_batch = next_batch.filter([next_batch.requests[1]]) + next_batch = next_batch.filter([next_batch.requests[1].id]) for _ in range( default_multi_requests_bloom_batch.stopping_criterias[0].max_new_tokens diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index f1f13e4b..d8d1bd16 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -178,7 +178,7 @@ def test_causal_lm_generate_token_completion_multi( default_multi_requests_causal_lm_batch.stopping_criterias.copy() ) - next_batch = next_batch.filter([next_batch.requests[0]]) + next_batch = next_batch.filter([next_batch.requests[0].id]) for _ in range( stopping_criterias[0].max_new_tokens - stopping_criterias[1].max_new_tokens - 1 @@ -285,7 +285,7 @@ def test_batch_concatenate( == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) - next_batch = next_batch.filter([next_batch.requests[0], next_batch.requests[1]]) + next_batch = next_batch.filter([next_batch.requests[0].id, next_batch.requests[1].id]) for _ in range( default_causal_lm_batch.stopping_criterias[0].max_new_tokens @@ -306,7 +306,7 @@ def test_batch_concatenate( == default_causal_lm_batch.stopping_criterias[0].max_new_tokens ) - next_batch = next_batch.filter([next_batch.requests[1]]) + next_batch = next_batch.filter([next_batch.requests[1].id]) for _ in range( default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index ba769e75..8fdeee60 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -190,7 +190,7 @@ def test_seq2seq_lm_generate_token_completion_multi( ) assert generations[1].generated_text.generated_tokens == 5 - next_batch = next_batch.filter([next_batch.requests[0]]) + next_batch = next_batch.filter([next_batch.requests[0].id]) generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert len(generations) == len(next_batch) @@ -323,7 +323,7 @@ def test_batch_concatenate( ) assert generations[2].generated_text.generated_tokens == 5 - next_batch = next_batch.filter([next_batch.requests[0], next_batch.requests[1]]) + next_batch = next_batch.filter([next_batch.requests[0].id, next_batch.requests[1].id]) generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None @@ -333,7 +333,7 @@ def test_batch_concatenate( assert generations[0].request_id == default_seq2seq_lm_batch.requests[0].id assert generations[0].generated_text.generated_tokens == 7 - next_batch = next_batch.filter([next_batch.requests[1]]) + next_batch = next_batch.filter([next_batch.requests[1].id]) generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 09df70d2..81a5e75e 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -53,10 +53,10 @@ class CausalLMBatch(Batch): # Past metadata keys_head_dim_last: bool = True - def to_pb(self) -> generate_pb2.Batch: - return generate_pb2.Batch( + def to_pb(self) -> generate_pb2.CachedBatch: + return generate_pb2.CachedBatch( id=self.batch_id, - requests=self.requests, + request_ids=[r.id for r in self.requests], size=len(self), max_tokens=self.max_tokens, ) @@ -143,16 +143,17 @@ def from_pb( ) @tracer.start_as_current_span("filter") - def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatch"]: - if len(requests) == 0: + def filter(self, request_ids: List[int]) -> Optional["CausalLMBatch"]: + if len(request_ids) == 0: raise ValueError("Batch must have at least one request") - if len(requests) == len(self): + if len(request_ids) == len(self): return self keep_indices = [] # New values after filtering requests_idx_mapping = {} + requests = [] input_lengths = [] prefix_offsets = [] read_offsets = [] @@ -165,11 +166,12 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc total_remaining_decode_tokens = 0 new_padding_right_offset = 0 - for i, r in enumerate(requests): - idx = self.requests_idx_mapping[r.id] - requests_idx_mapping[r.id] = i + for i, request_id in enumerate(request_ids): + idx = self.requests_idx_mapping[request_id] + requests_idx_mapping[request_id] = i keep_indices.append(idx) + requests.append(self.requests[idx]) prefix_offsets.append(self.prefix_offsets[idx]) read_offsets.append(self.read_offsets[idx]) all_input_ids.append(self.all_input_ids[idx]) @@ -220,7 +222,7 @@ def filter(self, requests: List[generate_pb2.Request]) -> Optional["CausalLMBatc layer[1] = past_values[keep_indices, :, -past_kv_length:, :] del past_values - max_tokens = len(requests) * max_input_length + total_remaining_decode_tokens + max_tokens = len(request_ids) * max_input_length + total_remaining_decode_tokens self.requests = requests self.requests_idx_mapping = requests_idx_mapping diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index d6e73ad8..baa6cd7f 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -62,10 +62,10 @@ class FlashCausalLMBatch(Batch): # Maximum number of tokens this batch will grow to max_tokens: int - def to_pb(self) -> generate_pb2.Batch: - return generate_pb2.Batch( + def to_pb(self) -> generate_pb2.CachedBatch: + return generate_pb2.CachedBatch( id=self.batch_id, - requests=self.requests, + request_ids=[r.id for r in self.requests], size=len(self), max_tokens=self.max_tokens, ) @@ -161,14 +161,14 @@ def from_pb( ) @tracer.start_as_current_span("filter") - def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": - if len(requests) == 0: + def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": + if len(request_ids) == 0: raise ValueError("Batch must have at least one request") # We assume that if len(requests) == len(self) then the requests are the same - if len(requests) == len(self): + if len(request_ids) == len(self): return self - single_request = len(requests) == 1 + single_request = len(request_ids) == 1 # Cumulative length cumulative_length = 0 @@ -176,16 +176,17 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": # New values after filtering requests_idx_mapping = {} - input_ids = self.input_ids.new_empty(len(requests)) - position_ids = self.position_ids.new_empty(len(requests)) + input_ids = self.input_ids.new_empty(len(request_ids)) + position_ids = self.position_ids.new_empty(len(request_ids)) # Create on CPU to only move to GPU once instead of at every copy - cu_seqlens = torch.zeros(len(requests) + 1, dtype=torch.int32) + cu_seqlens = torch.zeros(len(request_ids) + 1, dtype=torch.int32) cu_seqlens_q = torch.arange( - 0, len(requests) + 1, device=self.cu_seqlens_q.device, dtype=torch.int32 + 0, len(request_ids) + 1, device=self.cu_seqlens_q.device, dtype=torch.int32 ) max_seqlen = 0 past_key_values = [] + requests = [] all_input_ids = [] all_input_ids_tensor = [] @@ -198,9 +199,11 @@ def filter(self, requests: List[generate_pb2.Request]) -> "FlashCausalLMBatch": max_tokens = 0 - for i, r in enumerate(requests): - idx = self.requests_idx_mapping[r.id] - requests_idx_mapping[r.id] = i + for i, request_id in enumerate(request_ids): + idx = self.requests_idx_mapping[request_id] + requests_idx_mapping[request_id] = i + + requests.append(self.requests[idx]) # Get length request_input_length = self.input_lengths[idx] diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index a1a39fd4..2abb87ae 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -57,11 +57,11 @@ class Seq2SeqLMBatch(Batch): # Maximum number of tokens this batch will grow to max_tokens: int - def to_pb(self) -> generate_pb2.Batch: - """Convert a Seq2SeqLMBatch to a text_generation_server.v1.Batch protobuf""" - return generate_pb2.Batch( + def to_pb(self) -> generate_pb2.CachedBatch: + """Convert a Seq2SeqLMBatch to a text_generation_server.v1.CachedBatch protobuf""" + return generate_pb2.CachedBatch( id=self.batch_id, - requests=self.requests, + request_ids=[r.id for r in self.requests], size=len(self), max_tokens=self.max_tokens, ) @@ -152,18 +152,17 @@ def from_pb( ) @tracer.start_as_current_span("filter") - def filter( - self, requests: List[generate_pb2.Request] - ) -> Optional["Seq2SeqLMBatch"]: - if len(requests) == 0: + def filter(self, request_ids: List[int]) -> Optional["Seq2SeqLMBatch"]: + if len(request_ids) == 0: raise ValueError("Batch must have at least one request") - if len(requests) == len(self): + if len(request_ids) == len(self): return self keep_indices = [] # New values after filtering requests_idx_mapping = {} + requests = [] input_lengths = [] decoder_input_lengths = [] prefix_offsets = [] @@ -180,11 +179,12 @@ def filter( total_remaining_decode_tokens = 0 - for i, r in enumerate(requests): - idx = self.requests_idx_mapping[r.id] - requests_idx_mapping[r.id] = i + for i, request_id in enumerate(request_ids): + idx = self.requests_idx_mapping[request_id] + requests_idx_mapping[request_id] = i keep_indices.append(idx) + requests.append(self.requests[idx]) prefix_offsets.append(self.prefix_offsets[idx]) read_offsets.append(self.read_offsets[idx]) @@ -239,7 +239,7 @@ def filter( layer[3] = layer[3][keep_indices, :, -max_input_length:] max_tokens = ( - len(requests) * (max_input_length + max_decoder_input_length) + len(request_ids) * (max_input_length + max_decoder_input_length) + remaining_decode_tokens ) diff --git a/server/text_generation_server/models/types.py b/server/text_generation_server/models/types.py index 8a5b82f7..66a8c212 100644 --- a/server/text_generation_server/models/types.py +++ b/server/text_generation_server/models/types.py @@ -12,7 +12,7 @@ class Batch(ABC): @abstractmethod - def to_pb(self) -> generate_pb2.Batch: + def to_pb(self) -> generate_pb2.CachedBatch: raise NotImplementedError @classmethod @@ -26,7 +26,7 @@ def from_pb( raise NotImplementedError @abstractmethod - def filter(self, requests: List[generate_pb2.Request]) -> "Batch": + def filter(self, request_ids: List[int]) -> "Batch": raise NotImplementedError @classmethod diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 7ca5054e..e47fd049 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -42,15 +42,13 @@ async def ClearCache(self, request, context): self.cache.delete(request.id) else: self.cache.clear() - if torch.cuda.is_available(): - torch.cuda.empty_cache() return generate_pb2.ClearCacheResponse() async def FilterBatch(self, request, context): batch = self.cache.pop(request.batch_id) if batch is None: raise ValueError(f"Batch ID {request.batch_id} not found in cache.") - filtered_batch = batch.filter(request.keep_requests) + filtered_batch = batch.filter(request.request_ids) self.cache.set(filtered_batch) return generate_pb2.FilterBatchResponse(batch=filtered_batch.to_pb()) From 951930fbff5737d46b4bec49759dac75f0bd2aac Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 25 May 2023 13:38:36 +0200 Subject: [PATCH 254/793] feat(benchmarker): add summary tables (#368) --- Cargo.lock | 42 +++++++++ benchmark/Cargo.toml | 1 + benchmark/src/app.rs | 58 ++++++------ benchmark/src/generation.rs | 18 ++-- benchmark/src/lib.rs | 46 +++++++++- benchmark/src/main.rs | 32 ++++++- benchmark/src/table.rs | 170 ++++++++++++++++++++++++++++++++++++ 7 files changed, 325 insertions(+), 42 deletions(-) create mode 100644 benchmark/src/table.rs diff --git a/Cargo.lock b/Cargo.lock index 1dcc7c97..62896a51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -249,6 +249,12 @@ version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +[[package]] +name = "bytecount" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" + [[package]] name = "byteorder" version = "1.4.3" @@ -1706,6 +1712,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "papergrid" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fdfe703c51ddc52887ad78fc69cd2ea78d895ffcd6e955c9d03566db8ab5bb1" +dependencies = [ + "bytecount", + "fnv", + "unicode-width", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2490,6 +2507,30 @@ dependencies = [ "winapi", ] +[[package]] +name = "tabled" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da1a2e56bbf7bfdd08aaa7592157a742205459eff774b73bc01809ae2d99dc2a" +dependencies = [ + "papergrid", + "tabled_derive", + "unicode-width", +] + +[[package]] +name = "tabled_derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99f688a08b54f4f02f0a3c382aefdb7884d3d69609f785bd253dc033243e3fe4" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "tar" version = "0.4.38" @@ -2525,6 +2566,7 @@ dependencies = [ "ratatui", "serde", "serde_json", + "tabled", "text-generation-client", "thiserror", "tokenizers", diff --git a/benchmark/Cargo.toml b/benchmark/Cargo.toml index a4e215fc..67e04f0a 100644 --- a/benchmark/Cargo.toml +++ b/benchmark/Cargo.toml @@ -20,6 +20,7 @@ crossterm = "0.26" float-ord = "0.3.2" serde = {version = "1.0.142", features = ["derive"]} serde_json = "1.0" +tabled = "0.12.0" text-generation-client = { path = "../router/client" } thiserror = "1.0.38" tokenizers = "0.13.3" diff --git a/benchmark/src/app.rs b/benchmark/src/app.rs index 85026a33..6a9881fb 100644 --- a/benchmark/src/app.rs +++ b/benchmark/src/app.rs @@ -15,6 +15,7 @@ use tui::{symbols, Frame}; /// TUI powered App pub(crate) struct App { pub(crate) running: bool, + pub(crate) data: Data, completed_runs: Vec, completed_batch: usize, current_batch: usize, @@ -22,12 +23,10 @@ pub(crate) struct App { touched_tab: bool, zoom: bool, is_error: bool, - data: Data, tokenizer_name: String, sequence_length: u32, decode_length: u32, n_run: usize, - batch_size: Vec, receiver: mpsc::Receiver>, } @@ -40,7 +39,6 @@ impl App { n_run: usize, batch_size: Vec, ) -> Self { - let data = Data::new(n_run, batch_size.len()); let current_tab = 0; let completed_runs: Vec = (0..batch_size.len()).map(|_| 0).collect(); @@ -48,8 +46,11 @@ impl App { let current_batch = 0; let is_error = false; + let data = Data::new(n_run, batch_size); + Self { running: true, + data, completed_runs, completed_batch, current_batch, @@ -57,12 +58,10 @@ impl App { touched_tab: false, zoom: false, is_error, - data, tokenizer_name, sequence_length, decode_length, n_run, - batch_size, receiver, } } @@ -79,7 +78,7 @@ impl App { code: KeyCode::Tab, .. } => { self.touched_tab = true; - self.current_tab = (self.current_tab + 1) % self.batch_size.len(); + self.current_tab = (self.current_tab + 1) % self.data.batch_size.len(); } // Decrease and wrap tab KeyEvent { @@ -90,7 +89,7 @@ impl App { if self.current_tab > 0 { self.current_tab -= 1; } else { - self.current_tab = self.batch_size.len() - 1; + self.current_tab = self.data.batch_size.len() - 1; } } // Zoom on throughput/latency fig @@ -137,7 +136,7 @@ impl App { self.data.end_batch(self.current_batch); self.completed_batch += 1; - if self.current_batch < self.batch_size.len() - 1 { + if self.current_batch < self.data.batch_size.len() - 1 { // Only go to next tab if the user never touched the tab keys if !self.touched_tab { self.current_tab += 1; @@ -156,7 +155,7 @@ impl App { /// Render frame pub fn render(&mut self, f: &mut Frame<'_, B>) { let batch_progress = - (self.completed_batch as f64 / self.batch_size.len() as f64).clamp(0.0, 1.0); + (self.completed_batch as f64 / self.data.batch_size.len() as f64).clamp(0.0, 1.0); let run_progress = (self.completed_runs[self.current_batch] as f64 / self.n_run as f64).clamp(0.0, 1.0); @@ -241,6 +240,7 @@ impl App { // Batch tabs let titles = self + .data .batch_size .iter() .map(|b| { @@ -269,7 +269,7 @@ impl App { }; let batch_gauge = progress_gauge( "Total Progress", - format!("{} / {}", self.completed_batch, self.batch_size.len()), + format!("{} / {}", self.completed_batch, self.data.batch_size.len()), batch_progress, color, ); @@ -347,7 +347,7 @@ impl App { // Prefill latency/throughput chart let prefill_latency_throughput_chart = latency_throughput_chart( &self.data.prefill_batch_latency_throughput, - &self.batch_size, + &self.data.batch_size, self.zoom, "Prefill", ); @@ -356,7 +356,7 @@ impl App { // Decode latency/throughput chart let decode_latency_throughput_chart = latency_throughput_chart( &self.data.decode_batch_latency_throughput, - &self.batch_size, + &self.data.batch_size, self.zoom, "Decode", ); @@ -365,31 +365,35 @@ impl App { } /// App internal data struct -struct Data { - prefill_latencies: Vec>, - prefill_throughputs: Vec>, - decode_latencies: Vec>, - decode_token_latencies: Vec>, - decode_throughputs: Vec>, - prefill_batch_latency_throughput: Vec<(f64, f64)>, - decode_batch_latency_throughput: Vec<(f64, f64)>, +pub(crate) struct Data { + pub(crate) batch_size: Vec, + pub(crate) prefill_latencies: Vec>, + pub(crate) prefill_throughputs: Vec>, + pub(crate) decode_latencies: Vec>, + pub(crate) decode_token_latencies: Vec>, + pub(crate) decode_throughputs: Vec>, + pub(crate) prefill_batch_latency_throughput: Vec<(f64, f64)>, + pub(crate) decode_batch_latency_throughput: Vec<(f64, f64)>, } impl Data { - fn new(n_run: usize, n_batch: usize) -> Self { - let prefill_latencies: Vec> = - (0..n_batch).map(|_| Vec::with_capacity(n_run)).collect(); + fn new(n_run: usize, batch_size: Vec) -> Self { + let prefill_latencies: Vec> = (0..batch_size.len()) + .map(|_| Vec::with_capacity(n_run)) + .collect(); let prefill_throughputs: Vec> = prefill_latencies.clone(); let decode_latencies: Vec> = prefill_latencies.clone(); let decode_token_latencies: Vec> = decode_latencies.clone(); let decode_throughputs: Vec> = prefill_throughputs.clone(); - let prefill_batch_latency_throughput: Vec<(f64, f64)> = Vec::with_capacity(n_batch); + let prefill_batch_latency_throughput: Vec<(f64, f64)> = + Vec::with_capacity(batch_size.len()); let decode_batch_latency_throughput: Vec<(f64, f64)> = prefill_batch_latency_throughput.clone(); Self { + batch_size, prefill_latencies, prefill_throughputs, decode_latencies, @@ -401,14 +405,14 @@ impl Data { } fn push_prefill(&mut self, prefill: Prefill, batch_idx: usize) { - let latency = prefill.latency.as_millis() as f64; + let latency = prefill.latency.as_micros() as f64 / 1000.0; self.prefill_latencies[batch_idx].push(latency); self.prefill_throughputs[batch_idx].push(prefill.throughput); } fn push_decode(&mut self, decode: Decode, batch_idx: usize) { - let latency = decode.latency.as_millis() as f64; - let token_latency = decode.token_latency.as_millis() as f64; + let latency = decode.latency.as_micros() as f64 / 1000.0; + let token_latency = decode.token_latency.as_micros() as f64 / 1000.0; self.decode_latencies[batch_idx].push(latency); self.decode_token_latencies[batch_idx].push(token_latency); self.decode_throughputs[batch_idx].push(decode.throughput); diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs index d40a2e8d..17c72d26 100644 --- a/benchmark/src/generation.rs +++ b/benchmark/src/generation.rs @@ -39,6 +39,7 @@ pub(crate) async fn generation_task( decode_length: u32, n_runs: usize, warmups: usize, + parameters: NextTokenChooserParameters, client: ShardedClient, run_sender: mpsc::Sender>, mut shutdown_receiver: broadcast::Receiver<()>, @@ -47,7 +48,7 @@ pub(crate) async fn generation_task( // End task if a message is received on shutdown_receiver // _shutdown_guard_sender will be dropped once the task is finished tokio::select! { - res = generate_runs(tokenizer, batch_size, sequence_length, decode_length, n_runs, warmups, client, run_sender.clone()) => { + res = generate_runs(tokenizer, batch_size, sequence_length, decode_length, n_runs, warmups, parameters, client, run_sender.clone()) => { if let Err(err) = res { run_sender.send(Err(err)).await.unwrap_or(()); } @@ -65,6 +66,7 @@ async fn generate_runs( decode_length: u32, n_runs: usize, warmups: usize, + parameters: NextTokenChooserParameters, mut client: ShardedClient, run_sender: mpsc::Sender>, ) -> Result<(), ClientError> { @@ -79,6 +81,7 @@ async fn generate_runs( sequence_length, b, decode_length, + parameters.clone(), &mut client, ) .await?; @@ -93,6 +96,7 @@ async fn generate_runs( sequence_length, b, decode_length, + parameters.clone(), &mut client, ) .await?; @@ -125,6 +129,7 @@ async fn prefill( sequence_length: u32, batch_size: u32, decode_length: u32, + parameters: NextTokenChooserParameters, client: &mut ShardedClient, ) -> Result<(Prefill, CachedBatch), ClientError> { // Create requests @@ -133,16 +138,7 @@ async fn prefill( id: id.into(), inputs: sequence.clone(), truncate: sequence_length, - parameters: Some(NextTokenChooserParameters { - temperature: 1.0, - top_k: 0, - top_p: 1.0, - typical_p: 1.0, - do_sample: false, - seed: 0, - repetition_penalty: 1.0, - watermark: false, - }), + parameters: Some(parameters.clone()), stopping_parameters: Some(StoppingCriteriaParameters { max_new_tokens: decode_length, stop_sequences: vec![], diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs index 4da0b573..fcad400c 100644 --- a/benchmark/src/lib.rs +++ b/benchmark/src/lib.rs @@ -1,13 +1,14 @@ mod app; mod event; mod generation; +mod table; mod utils; use crate::app::App; use crate::event::Event; use crossterm::ExecutableCommand; use std::io; -use text_generation_client::ShardedClient; +use text_generation_client::{NextTokenChooserParameters, ShardedClient}; use tokenizers::Tokenizer; use tokio::sync::{broadcast, mpsc}; use tui::backend::CrosstermBackend; @@ -23,8 +24,26 @@ pub async fn run( decode_length: u32, n_runs: usize, warmups: usize, + temperature: Option, + top_k: Option, + top_p: Option, + typical_p: Option, + repetition_penalty: Option, + watermark: bool, + do_sample: bool, client: ShardedClient, ) -> Result<(), crossterm::ErrorKind> { + let parameters = NextTokenChooserParameters { + temperature: temperature.unwrap_or(1.0), + top_k: top_k.unwrap_or(0), + top_p: top_p.unwrap_or(1.0), + typical_p: typical_p.unwrap_or(1.0), + do_sample, + seed: 0, + repetition_penalty: repetition_penalty.unwrap_or(1.0), + watermark, + }; + // Initialize terminal properties crossterm::terminal::enable_raw_mode()?; io::stdout().execute(crossterm::terminal::EnterAlternateScreen)?; @@ -53,6 +72,7 @@ pub async fn run( decode_length, n_runs, warmups, + parameters, client, run_sender, shutdown_sender.subscribe(), @@ -73,7 +93,7 @@ pub async fn run( // Create App let mut app = App::new( run_receiver, - tokenizer_name, + tokenizer_name.clone(), sequence_length, decode_length, n_runs, @@ -106,5 +126,27 @@ pub async fn run( crossterm::terminal::disable_raw_mode()?; io::stdout().execute(crossterm::cursor::Show)?; + let parameters_table = table::parameters_table( + tokenizer_name, + sequence_length, + decode_length, + n_runs, + warmups, + temperature, + top_k, + top_p, + typical_p, + repetition_penalty, + watermark, + do_sample, + ); + println!("\n{parameters_table}\n"); + + let latency_table = table::latency_table(&app.data); + println!("\n{latency_table}\n"); + + let throughput_table = table::throughput_table(&app.data); + println!("\n{throughput_table}\n"); + Ok(()) } diff --git a/benchmark/src/main.rs b/benchmark/src/main.rs index 03f61dcd..6172d377 100644 --- a/benchmark/src/main.rs +++ b/benchmark/src/main.rs @@ -28,11 +28,27 @@ struct Args { runs: usize, #[clap(default_value = "1", short, long, env)] warmups: usize, + #[clap(long, env)] + temperature: Option, + #[clap(long, env)] + top_k: Option, + #[clap(long, env)] + top_p: Option, + #[clap(long, env)] + typical_p: Option, + #[clap(long, env)] + repetition_penalty: Option, + #[clap(long, env)] + watermark: bool, + #[clap(long, env)] + do_sample: bool, #[clap(default_value = "/tmp/text-generation-server-0", short, long, env)] master_shard_uds_path: String, } fn main() -> Result<(), Box> { + init_logging(); + // Get args let args = Args::parse(); // Pattern match configuration @@ -44,13 +60,18 @@ fn main() -> Result<(), Box> { decode_length, runs, warmups, + temperature, + top_k, + top_p, + typical_p, + repetition_penalty, + watermark, + do_sample, master_shard_uds_path, } = args; let batch_size = batch_size.unwrap_or(vec![1, 2, 4, 8, 16, 32]); - init_logging(); - // Tokenizer instance // This will only be used to validate payloads tracing::info!("Loading tokenizer"); @@ -105,6 +126,13 @@ fn main() -> Result<(), Box> { decode_length, runs, warmups, + temperature, + top_k, + top_p, + typical_p, + repetition_penalty, + watermark, + do_sample, sharded_client, ) .await diff --git a/benchmark/src/table.rs b/benchmark/src/table.rs new file mode 100644 index 00000000..6b74bc36 --- /dev/null +++ b/benchmark/src/table.rs @@ -0,0 +1,170 @@ +use crate::app::Data; +use tabled::settings::Merge; +use tabled::{builder::Builder, settings::Style, Table}; + +#[allow(clippy::too_many_arguments)] +pub(crate) fn parameters_table( + tokenizer_name: String, + sequence_length: u32, + decode_length: u32, + n_runs: usize, + warmups: usize, + temperature: Option, + top_k: Option, + top_p: Option, + typical_p: Option, + repetition_penalty: Option, + watermark: bool, + do_sample: bool, +) -> Table { + let mut builder = Builder::default(); + + builder.set_header(["Parameter", "Value"]); + + builder.push_record(["Model", &tokenizer_name]); + builder.push_record(["Sequence Length", &sequence_length.to_string()]); + builder.push_record(["Decode Length", &decode_length.to_string()]); + builder.push_record(["N Runs", &n_runs.to_string()]); + builder.push_record(["Warmups", &warmups.to_string()]); + builder.push_record(["Temperature", &format!("{temperature:?}")]); + builder.push_record(["Top K", &format!("{top_k:?}")]); + builder.push_record(["Top P", &format!("{top_p:?}")]); + builder.push_record(["Typical P", &format!("{typical_p:?}")]); + builder.push_record(["Repetition Penalty", &format!("{repetition_penalty:?}")]); + builder.push_record(["Watermark", &watermark.to_string()]); + builder.push_record(["Do Sample", &do_sample.to_string()]); + + let mut table = builder.build(); + table.with(Style::markdown()); + table +} + +pub(crate) fn latency_table(data: &Data) -> Table { + let mut builder = Builder::default(); + + builder.set_header([ + "Step", + "Batch Size", + "Average", + "Lowest", + "Highest", + "p50", + "p90", + "p99", + ]); + + add_latencies( + &mut builder, + "Prefill", + &data.batch_size, + &data.prefill_latencies, + ); + add_latencies( + &mut builder, + "Decode (token)", + &data.batch_size, + &data.decode_token_latencies, + ); + add_latencies( + &mut builder, + "Decode (total)", + &data.batch_size, + &data.decode_latencies, + ); + + let mut table = builder.build(); + table.with(Style::markdown()).with(Merge::vertical()); + table +} + +pub(crate) fn throughput_table(data: &Data) -> Table { + let mut builder = Builder::default(); + + builder.set_header(["Step", "Batch Size", "Average", "Lowest", "Highest"]); + + add_throuhgputs( + &mut builder, + "Prefill", + &data.batch_size, + &data.prefill_throughputs, + ); + add_throuhgputs( + &mut builder, + "Decode", + &data.batch_size, + &data.decode_throughputs, + ); + + let mut table = builder.build(); + table.with(Style::markdown()).with(Merge::vertical()); + table +} + +fn add_latencies( + builder: &mut Builder, + step: &'static str, + batch_size: &[u32], + batch_latencies: &[Vec], +) { + for (i, b) in batch_size.iter().enumerate() { + let latencies = &batch_latencies[i]; + let (avg, min, max) = avg_min_max(latencies); + + let row = [ + step, + &b.to_string(), + &format_value(avg, "ms"), + &format_value(min, "ms"), + &format_value(max, "ms"), + &format_value(px(latencies, 50), "ms"), + &format_value(px(latencies, 90), "ms"), + &format_value(px(latencies, 99), "ms"), + ]; + + builder.push_record(row); + } +} + +fn add_throuhgputs( + builder: &mut Builder, + step: &'static str, + batch_size: &[u32], + batch_throughputs: &[Vec], +) { + for (i, b) in batch_size.iter().enumerate() { + let throughputs = &batch_throughputs[i]; + let (avg, min, max) = avg_min_max(throughputs); + + let row = [ + step, + &b.to_string(), + &format_value(avg, "tokens/secs"), + &format_value(min, "tokens/secs"), + &format_value(max, "tokens/secs"), + ]; + + builder.push_record(row); + } +} + +fn avg_min_max(data: &Vec) -> (f64, f64, f64) { + let average = data.iter().sum::() / data.len() as f64; + let min = data + .iter() + .min_by(|a, b| a.total_cmp(b)) + .unwrap_or(&std::f64::NAN); + let max = data + .iter() + .max_by(|a, b| a.total_cmp(b)) + .unwrap_or(&std::f64::NAN); + (average, *min, *max) +} + +fn px(data: &Vec, p: u32) -> f64 { + let i = (f64::from(p) / 100.0 * data.len() as f64) as usize; + *data.get(i).unwrap_or(&std::f64::NAN) +} + +fn format_value(value: f64, unit: &'static str) -> String { + format!("{:.2} {unit}", value) +} From 62f91f78ac4ab747505c9fbf51fa53ba2d3f3d61 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 26 May 2023 12:30:27 +0200 Subject: [PATCH 255/793] feat(server): support vectorized warpers in flash causal lm (#317) Co-authored-by: Joel Lamy-Poirier --- .../test_bloom_560m_all_params.json | 58 +-- .../test_flash_llama_all_params.json | 58 +-- .../test_flash_starcoder_default_params.json | 346 +++++++++++++-- .../test_mt0_base_all_params.json | 48 +-- integration-tests/models/test_flash_llama.py | 2 +- .../models/test_flash_starcoder.py | 2 +- integration-tests/models/test_mt0_base.py | 2 +- server/tests/models/test_bloom.py | 8 +- server/tests/models/test_causal_lm.py | 12 +- server/tests/models/test_santacoder.py | 10 +- server/tests/models/test_seq2seq_lm.py | 10 +- server/text_generation_server/models/bloom.py | 3 +- .../models/causal_lm.py | 1 + .../models/flash_causal_lm.py | 174 ++++---- .../models/galactica.py | 1 + .../models/seq2seq_lm.py | 1 + server/text_generation_server/models/types.py | 1 + server/text_generation_server/server.py | 2 +- .../text_generation_server/utils/__init__.py | 6 +- .../text_generation_server/utils/convert.py | 6 +- .../utils/logits_process.py | 405 ++++++++++++++++++ server/text_generation_server/utils/tokens.py | 282 ++++++++---- 22 files changed, 1123 insertions(+), 315 deletions(-) create mode 100644 server/text_generation_server/utils/logits_process.py diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json index 93a95804..ace73416 100644 --- a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json +++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json @@ -34,65 +34,65 @@ "tokens": [ { "id": 408, - "logprob": -1.9267578, + "logprob": -0.07891846, "special": false, "text": " que" }, { - "id": 20288, - "logprob": -2.9257812, + "id": 366, + "logprob": -1.2939453, "special": false, - "text": " l'on" + "text": " la" }, { - "id": 22255, - "logprob": -2.8964844, + "id": 8769, + "logprob": -0.3708496, "special": false, - "text": " trouve" + "text": " personne" }, { - "id": 1622, - "logprob": -1.1083984, + "id": 1479, + "logprob": -2.2871094, "special": false, - "text": " une" + "text": " qui" }, { - "id": 187079, - "logprob": -7.796875, + "id": 2997, + "logprob": -0.8671875, "special": false, - "text": " posture" + "text": " vous" }, { - "id": 501, - "logprob": -5.390625, + "id": 35977, + "logprob": -1.5097656, "special": false, - "text": " par" + "text": " suit" }, { - "id": 8741, - "logprob": -0.34936523, + "id": 21558, + "logprob": -0.07891846, "special": false, - "text": " rapport" + "text": " ait" }, { - "id": 693, - "logprob": 0.0, + "id": 447, + "logprob": -0.12695312, "special": false, - "text": " à" + "text": " un" }, { - "id": 366, - "logprob": -2.3378906, + "id": 78606, + "logprob": -2.21875, "special": false, - "text": " la" + "text": " profil" }, { - "id": 36503, - "logprob": -3.6640625, + "id": 3899, + "logprob": -1.3535156, "special": false, - "text": " pratique" + "text": " bien" } ] }, - "generated_text": "Pour déguster un ortolan, il faut tout d'abord que l'on trouve une posture par rapport à la pratique" + "generated_text": "Pour déguster un ortolan, il faut tout d'abord que la personne qui vous suit ait un profil bien" } diff --git a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json index 1b6b51a3..5be2870d 100644 --- a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json +++ b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json @@ -1,8 +1,8 @@ { "details": { "best_of_sequences": null, - "finish_reason": "length", - "generated_tokens": 10, + "finish_reason": "stop_sequence", + "generated_tokens": 5, "prefill": [ { "id": 1, @@ -24,65 +24,35 @@ "tokens": [ { "id": 5229, - "logprob": -3.3085938, + "logprob": -2.5683594, "special": false, "text": " failed" }, - { - "id": 363, - "logprob": -3.984375, - "special": false, - "text": " for" - }, - { - "id": 5641, - "logprob": -6.53125, - "special": false, - "text": " IP" - }, - { - "id": 16428, - "logprob": -3.1835938, - "special": false, - "text": " Address" - }, { "id": 29901, - "logprob": -1.2324219, + "logprob": -0.45336914, "special": false, "text": ":" }, { - "id": 525, - "logprob": -2.6855469, - "special": false, - "text": " '" - }, - { - "id": 8516, - "logprob": -7.1601562, - "special": false, - "text": "None" - }, - { - "id": 4286, - "logprob": -2.4433594, + "id": 4829, + "logprob": -1.8408203, "special": false, - "text": "'." + "text": " Error" }, { - "id": 13, - "logprob": -0.06530762, + "id": 297, + "logprob": -1.0556641, "special": false, - "text": "\n" + "text": " in" }, { - "id": 294, - "logprob": -7.953125, + "id": 1243, + "logprob": 0.0, "special": false, - "text": "as" + "text": " test" } ] }, - "generated_text": "Test requestfailed for IP Address: 'None'.\nas" + "generated_text": "Test requestfailed: Error in test" } diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json index 21bb509b..afd0b662 100644 --- a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json +++ b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json @@ -1,8 +1,8 @@ { "details": { "best_of_sequences": null, - "finish_reason": "eos_token", - "generated_tokens": 12, + "finish_reason": "length", + "generated_tokens": 60, "prefill": [ { "id": 589, @@ -29,77 +29,365 @@ "tokens": [ { "id": 2262, - "logprob": -0.7451172, + "logprob": -0.042999268, "special": false, "text": "():" }, { "id": 284, - "logprob": -0.21325684, + "logprob": 0.0, "special": false, "text": "\n " }, { - "id": 5741, - "logprob": -5.734375, + "id": 1459, + "logprob": 0.0, + "special": false, + "text": " print" + }, + { + "id": 440, + "logprob": 0.0, + "special": false, + "text": "(\"" + }, + { + "id": 8279, + "logprob": 0.0, + "special": false, + "text": "Hello" + }, + { + "id": 10896, + "logprob": -0.3659668, + "special": false, + "text": " World" + }, + { + "id": 657, + "logprob": -0.49804688, + "special": false, + "text": "\")" + }, + { + "id": 203, + "logprob": -0.11279297, "special": false, - "text": " logging" + "text": "\n" }, { - "id": 32, + "id": 203, "logprob": 0.0, "special": false, - "text": "." + "text": "\n" }, { - "id": 1338, - "logprob": -0.3232422, + "id": 589, + "logprob": -0.20141602, "special": false, - "text": "info" + "text": "def" }, { - "id": 463, - "logprob": -1.0380859, + "id": 1459, + "logprob": 0.0, + "special": false, + "text": " print" + }, + { + "id": 81, + "logprob": 0.0, + "special": false, + "text": "_" + }, + { + "id": 7656, + "logprob": 0.0, + "special": false, + "text": "hello" + }, + { + "id": 81, + "logprob": 0.0, + "special": false, + "text": "_" + }, + { + "id": 426, + "logprob": -0.051635742, + "special": false, + "text": "name" + }, + { + "id": 26, + "logprob": 0.0, + "special": false, + "text": "(" + }, + { + "id": 426, + "logprob": 0.0, "special": false, - "text": "('" + "text": "name" + }, + { + "id": 711, + "logprob": 0.0, + "special": false, + "text": "):" + }, + { + "id": 284, + "logprob": 0.0, + "special": false, + "text": "\n " + }, + { + "id": 1459, + "logprob": 0.0, + "special": false, + "text": " print" + }, + { + "id": 440, + "logprob": -0.16027832, + "special": false, + "text": "(\"" }, { "id": 8279, - "logprob": -0.8378906, + "logprob": 0.0, "special": false, "text": "Hello" }, + { + "id": 313, + "logprob": 0.0, + "special": false, + "text": " \"" + }, + { + "id": 474, + "logprob": 0.0, + "special": false, + "text": " +" + }, + { + "id": 636, + "logprob": 0.0, + "special": false, + "text": " name" + }, + { + "id": 27, + "logprob": 0.0, + "special": false, + "text": ")" + }, + { + "id": 203, + "logprob": 0.0, + "special": false, + "text": "\n" + }, + { + "id": 203, + "logprob": 0.0, + "special": false, + "text": "\n" + }, + { + "id": 589, + "logprob": 0.0, + "special": false, + "text": "def" + }, + { + "id": 1459, + "logprob": 0.0, + "special": false, + "text": " print" + }, + { + "id": 81, + "logprob": 0.0, + "special": false, + "text": "_" + }, + { + "id": 7656, + "logprob": 0.0, + "special": false, + "text": "hello" + }, + { + "id": 81, + "logprob": 0.0, + "special": false, + "text": "_" + }, + { + "id": 426, + "logprob": 0.0, + "special": false, + "text": "name" + }, + { + "id": 81, + "logprob": 0.0, + "special": false, + "text": "_" + }, + { + "id": 381, + "logprob": 0.0, + "special": false, + "text": "age" + }, + { + "id": 26, + "logprob": 0.0, + "special": false, + "text": "(" + }, + { + "id": 426, + "logprob": 0.0, + "special": false, + "text": "name" + }, { "id": 30, - "logprob": -1.9501953, + "logprob": 0.0, "special": false, "text": "," }, { - "id": 10896, - "logprob": -1.3476562, + "id": 11442, + "logprob": 0.0, "special": false, - "text": " World" + "text": " age" + }, + { + "id": 711, + "logprob": 0.0, + "special": false, + "text": "):" + }, + { + "id": 284, + "logprob": 0.0, + "special": false, + "text": "\n " + }, + { + "id": 1459, + "logprob": 0.0, + "special": false, + "text": " print" + }, + { + "id": 440, + "logprob": 0.0, + "special": false, + "text": "(\"" }, { - "id": 683, - "logprob": -1.796875, + "id": 8279, + "logprob": 0.0, + "special": false, + "text": "Hello" + }, + { + "id": 313, + "logprob": 0.0, + "special": false, + "text": " \"" + }, + { + "id": 474, + "logprob": 0.0, + "special": false, + "text": " +" + }, + { + "id": 636, + "logprob": 0.0, + "special": false, + "text": " name" + }, + { + "id": 474, + "logprob": 0.0, + "special": false, + "text": " +" + }, + { + "id": 313, + "logprob": -0.6328125, + "special": false, + "text": " \"" + }, + { + "id": 313, + "logprob": -1.7011719, + "special": false, + "text": " \"" + }, + { + "id": 474, + "logprob": 0.0, + "special": false, + "text": " +" + }, + { + "id": 596, + "logprob": 0.0, + "special": false, + "text": " str" + }, + { + "id": 26, + "logprob": 0.0, + "special": false, + "text": "(" + }, + { + "id": 381, + "logprob": 0.0, + "special": false, + "text": "age" + }, + { + "id": 490, + "logprob": 0.0, "special": false, - "text": "')" + "text": "))" }, { "id": 203, - "logprob": -0.9873047, + "logprob": 0.0, "special": false, "text": "\n" }, { - "id": 0, - "logprob": -0.7495117, - "special": true, - "text": "<|endoftext|>" + "id": 203, + "logprob": 0.0, + "special": false, + "text": "\n" + }, + { + "id": 589, + "logprob": 0.0, + "special": false, + "text": "def" + }, + { + "id": 1459, + "logprob": 0.0, + "special": false, + "text": " print" } ] }, - "generated_text": "():\n logging.info('Hello, World')\n<|endoftext|>" + "generated_text": "():\n print(\"Hello World\")\n\ndef print_hello_name(name):\n print(\"Hello \" + name)\n\ndef print_hello_name_age(name, age):\n print(\"Hello \" + name + \" \" + str(age))\n\ndef print" } diff --git a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json index 3e9f3d73..024823d0 100644 --- a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json +++ b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json @@ -1,8 +1,8 @@ { "details": { "best_of_sequences": null, - "finish_reason": "length", - "generated_tokens": 10, + "finish_reason": "eos_token", + "generated_tokens": 9, "prefill": [ { "id": 0, @@ -14,65 +14,59 @@ "tokens": [ { "id": 16017, - "logprob": -1.3505859, + "logprob": -0.30908203, "special": false, "text": " blue" }, { "id": 20495, - "logprob": -0.50439453, + "logprob": 0.0, "special": false, "text": " sky" }, { "id": 259, - "logprob": -1.2011719, + "logprob": -0.28271484, "special": false, "text": " " }, { "id": 15484, - "logprob": -2.8378906, + "logprob": -1.7929688, "special": false, "text": "appear" }, { "id": 345, - "logprob": -0.87597656, + "logprob": -0.8935547, "special": false, "text": "ed" }, { - "id": 288, - "logprob": -1.8447266, + "id": 281, + "logprob": 0.0, "special": false, - "text": " to" + "text": " in" }, { - "id": 35622, - "logprob": -7.1445312, + "id": 287, + "logprob": 0.0, "special": false, - "text": " cloud" + "text": " the" }, { - "id": 263, - "logprob": -1.2929688, - "special": false, - "text": "s" - }, - { - "id": 14701, - "logprob": -3.0761719, + "id": 20495, + "logprob": -0.32299805, "special": false, - "text": " above" + "text": " sky" }, { - "id": 751, - "logprob": -4.4375, - "special": false, - "text": " all" + "id": 1, + "logprob": 0.0, + "special": true, + "text": "" } ] }, - "generated_text": "Why is the sky blue?blue sky appeared to clouds above all" + "generated_text": "Why is the sky blue?blue sky appeared in the sky" } diff --git a/integration-tests/models/test_flash_llama.py b/integration-tests/models/test_flash_llama.py index 37468455..bf5b64ba 100644 --- a/integration-tests/models/test_flash_llama.py +++ b/integration-tests/models/test_flash_llama.py @@ -40,7 +40,7 @@ async def test_flash_llama_all_params(flash_llama, response_snapshot): seed=0, ) - assert response.details.generated_tokens == 10 + assert response.details.generated_tokens == 5 assert response == response_snapshot diff --git a/integration-tests/models/test_flash_starcoder.py b/integration-tests/models/test_flash_starcoder.py index 4c7393a7..c1a68d89 100644 --- a/integration-tests/models/test_flash_starcoder.py +++ b/integration-tests/models/test_flash_starcoder.py @@ -29,7 +29,7 @@ async def test_flash_starcoder_default_params(flash_starcoder, response_snapshot "def print_hello", max_new_tokens=60, temperature=0.2, top_p=0.95, seed=0 ) - assert response.details.generated_tokens == 12 + assert response.details.generated_tokens == 60 assert response == response_snapshot diff --git a/integration-tests/models/test_mt0_base.py b/integration-tests/models/test_mt0_base.py index 15410f73..e347d22a 100644 --- a/integration-tests/models/test_mt0_base.py +++ b/integration-tests/models/test_mt0_base.py @@ -43,7 +43,7 @@ async def test_mt0_base_all_params(mt0_base, response_snapshot): seed=0, ) - assert response.details.generated_tokens == 10 + assert response.details.generated_tokens == 9 assert response == response_snapshot diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 105b3573..590ba557 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -38,7 +38,7 @@ def default_pb_batch(default_pb_request): @pytest.fixture def default_bloom_batch(default_pb_batch, bloom_560m_tokenizer): return BloomCausalLMBatch.from_pb( - default_pb_batch, bloom_560m_tokenizer, torch.device("cpu") + default_pb_batch, bloom_560m_tokenizer, torch.float32, torch.device("cpu") ) @@ -52,7 +52,7 @@ def default_multi_requests_bloom_batch(default_pb_request, bloom_560m_tokenizer) batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) return BloomCausalLMBatch.from_pb( - batch_pb, bloom_560m_tokenizer, torch.device("cpu") + batch_pb, bloom_560m_tokenizer, torch.float32, torch.device("cpu") ) @@ -286,7 +286,9 @@ def test_batch_concatenate( == default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens ) - next_batch = next_batch.filter([next_batch.requests[0].id, next_batch.requests[1].id]) + next_batch = next_batch.filter( + [next_batch.requests[0].id, next_batch.requests[1].id] + ) for _ in range( default_bloom_batch.stopping_criterias[0].max_new_tokens diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index d8d1bd16..3f28f5b3 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -38,7 +38,9 @@ def default_pb_batch(default_pb_request): @pytest.fixture def default_causal_lm_batch(default_pb_batch, gpt2_tokenizer): - return CausalLMBatch.from_pb(default_pb_batch, gpt2_tokenizer, torch.device("cpu")) + return CausalLMBatch.from_pb( + default_pb_batch, gpt2_tokenizer, torch.float32, torch.device("cpu") + ) @pytest.fixture @@ -50,7 +52,9 @@ def default_multi_requests_causal_lm_batch(default_pb_request, gpt2_tokenizer): req_1.stopping_parameters.max_new_tokens = 5 batch_pb = generate_pb2.Batch(id=1, requests=[req_0, req_1], size=2) - return CausalLMBatch.from_pb(batch_pb, gpt2_tokenizer, torch.device("cpu")) + return CausalLMBatch.from_pb( + batch_pb, gpt2_tokenizer, torch.float32, torch.device("cpu") + ) def test_batch_from_pb(default_pb_batch, default_causal_lm_batch): @@ -285,7 +289,9 @@ def test_batch_concatenate( == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens ) - next_batch = next_batch.filter([next_batch.requests[0].id, next_batch.requests[1].id]) + next_batch = next_batch.filter( + [next_batch.requests[0].id, next_batch.requests[1].id] + ) for _ in range( default_causal_lm_batch.stopping_criterias[0].max_new_tokens diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index 8cf66d47..bef8db38 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -45,7 +45,10 @@ def default_fim_pb_batch(default_fim_pb_request): @pytest.mark.skip def test_santacoder_generate_token_completion(default_santacoder, default_pb_batch): batch = CausalLMBatch.from_pb( - default_pb_batch, default_santacoder.tokenizer, default_santacoder.device + default_pb_batch, + default_santacoder.tokenizer, + default_santacoder.dtype, + default_santacoder.device, ) next_batch = batch @@ -70,7 +73,10 @@ def test_fim_santacoder_generate_token_completion( default_santacoder, default_fim_pb_batch ): batch = CausalLMBatch.from_pb( - default_fim_pb_batch, default_santacoder.tokenizer, default_santacoder.device + default_fim_pb_batch, + default_santacoder.tokenizer, + default_santacoder.dtype, + default_santacoder.device, ) next_batch = batch diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 8fdeee60..a3199d02 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -42,7 +42,7 @@ def default_pb_batch(default_pb_request): @pytest.fixture def default_seq2seq_lm_batch(default_pb_batch, mt0_small_tokenizer): return Seq2SeqLMBatch.from_pb( - default_pb_batch, mt0_small_tokenizer, torch.device("cpu") + default_pb_batch, mt0_small_tokenizer, torch.float32, torch.device("cpu") ) @@ -55,7 +55,9 @@ def default_multi_requests_seq2seq_lm_batch(default_pb_request, mt0_small_tokeni req_1.stopping_parameters.max_new_tokens = 5 batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2) - return Seq2SeqLMBatch.from_pb(batch_pb, mt0_small_tokenizer, torch.device("cpu")) + return Seq2SeqLMBatch.from_pb( + batch_pb, mt0_small_tokenizer, torch.float32, torch.device("cpu") + ) def test_batch_from_pb(default_pb_batch, default_seq2seq_lm_batch): @@ -323,7 +325,9 @@ def test_batch_concatenate( ) assert generations[2].generated_text.generated_tokens == 5 - next_batch = next_batch.filter([next_batch.requests[0].id, next_batch.requests[1].id]) + next_batch = next_batch.filter( + [next_batch.requests[0].id, next_batch.requests[1].id] + ) generations, next_batch = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 5eddc8cf..088a1457 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -39,10 +39,11 @@ def from_pb( cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, device: torch.device, ) -> "CausalLMBatch": batch = super(BloomCausalLMBatch, cls).from_pb( - pb=pb, tokenizer=tokenizer, device=device + pb=pb, tokenizer=tokenizer, dtype=dtype, device=device ) batch.keys_head_dim_last = False return batch diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 81a5e75e..a20a6143 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -66,6 +66,7 @@ def from_pb( cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, device: torch.device, ) -> "CausalLMBatch": inputs = [] diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index baa6cd7f..35cbe174 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -18,11 +18,7 @@ GeneratedText, ) from text_generation_server.pb import generate_pb2 -from text_generation_server.utils import ( - NextTokenChooser, - StoppingCriteria, - Sampling, -) +from text_generation_server.utils import StoppingCriteria, HeterogeneousNextTokenChooser tracer = trace.get_tracer(__name__) @@ -48,7 +44,7 @@ class FlashCausalLMBatch(Batch): # All tokens all_input_ids: List[List[int]] - all_input_ids_tensor: List[torch.Tensor] + all_input_ids_tensor: torch.Tensor # Lengths of all generations present in the batch input_lengths: List[int] @@ -56,7 +52,7 @@ class FlashCausalLMBatch(Batch): read_offsets: List[Optional[int]] # Generation helpers - next_token_choosers: List[NextTokenChooser] + next_token_chooser: HeterogeneousNextTokenChooser stopping_criterias: List[StoppingCriteria] # Maximum number of tokens this batch will grow to @@ -75,6 +71,7 @@ def from_pb( cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, device: torch.device, ) -> "FlashCausalLMBatch": position_ids = [] @@ -87,13 +84,14 @@ def from_pb( all_input_ids = [] requests_idx_mapping = {} - next_token_choosers = [] + next_token_chooser_parameters = [] stopping_criterias = [] # Cumulative length cumulative_length = 0 max_tokens = 0 + max_length = 0 # Parse batch for i, r in enumerate(pb.requests): @@ -119,7 +117,7 @@ def from_pb( # Add cumulative lengths of all previous inputs cu_seqlens.append(cumulative_length + input_length) - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + next_token_chooser_parameters.append(r.parameters) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer @@ -130,11 +128,26 @@ def from_pb( # Update cumulative_length += input_length max_tokens += input_length + max_new_tokens + max_length = max(max_length, input_length + max_new_tokens) + + next_token_chooser = HeterogeneousNextTokenChooser.from_pb( + next_token_chooser_parameters, dtype, device + ) + + # Padded all_input_ids_tensor + all_input_ids_tensor = np.zeros( + (len(all_input_ids), max_length), dtype=np.int64 + ) + for i, input_ids in enumerate(all_input_ids): + all_input_ids_tensor[i, : len(input_ids)] = input_ids # Create tensors on device input_ids = torch.tensor( np.concatenate(all_input_ids), dtype=torch.int64, device=device ) + all_input_ids_tensor = torch.tensor( + all_input_ids_tensor, dtype=torch.int64, device=device + ) position_ids = torch.tensor( np.concatenate(position_ids), dtype=torch.int32, device=device ) @@ -154,8 +167,8 @@ def from_pb( prefix_offsets=prefix_offsets, read_offsets=read_offsets, all_input_ids=all_input_ids, - all_input_ids_tensor=[], - next_token_choosers=next_token_choosers, + all_input_ids_tensor=all_input_ids_tensor, + next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, max_tokens=max_tokens, ) @@ -176,31 +189,29 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": # New values after filtering requests_idx_mapping = {} - input_ids = self.input_ids.new_empty(len(request_ids)) - position_ids = self.position_ids.new_empty(len(request_ids)) + # Used to index into tensors + indices = [] + # Create on CPU to only move to GPU once instead of at every copy cu_seqlens = torch.zeros(len(request_ids) + 1, dtype=torch.int32) - cu_seqlens_q = torch.arange( - 0, len(request_ids) + 1, device=self.cu_seqlens_q.device, dtype=torch.int32 - ) + cu_seqlens_q = self.cu_seqlens_q[: len(request_ids) + 1] max_seqlen = 0 past_key_values = [] requests = [] all_input_ids = [] - all_input_ids_tensor = [] input_lengths = [] prefix_offsets = [] read_offsets = [] - next_token_choosers = [] stopping_criterias = [] max_tokens = 0 for i, request_id in enumerate(request_ids): idx = self.requests_idx_mapping[request_id] + indices.append(idx) requests_idx_mapping[request_id] = i requests.append(self.requests[idx]) @@ -208,10 +219,6 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": # Get length request_input_length = self.input_lengths[idx] - # Copy tensors (GPU) - input_ids[i] = self.input_ids[idx] - position_ids[i] = self.position_ids[idx] - # Copy to tensor (CPU) cu_seqlens[i + 1] = cumulative_length + request_input_length max_seqlen = max(max_seqlen, request_input_length) @@ -222,14 +229,11 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": ) all_input_ids.append(self.all_input_ids[idx]) - all_input_ids_tensor.append(self.all_input_ids_tensor[idx]) input_lengths.append(request_input_length) prefix_offsets.append(self.prefix_offsets[idx]) read_offsets.append(self.read_offsets[idx]) - next_token_choosers.append(self.next_token_choosers[idx]) - stopping_criteria = self.stopping_criterias[idx] stopping_criterias.append(stopping_criteria) @@ -258,6 +262,12 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": # Cat all past past_key_values = torch.cat(past_key_values, dim=1) + # Index into tensors + input_ids = self.input_ids[indices] + position_ids = self.position_ids[indices] + all_input_ids_tensor = self.all_input_ids_tensor[indices] + next_token_chooser = self.next_token_chooser.filter(indices) + # Move to GPU now that we have the whole tensor cu_seqlens = cu_seqlens.to(self.cu_seqlens.device) @@ -276,7 +286,7 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": read_offsets=read_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=all_input_ids_tensor, - next_token_choosers=next_token_choosers, + next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, max_tokens=max_tokens, ) @@ -290,6 +300,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch total_batch_size = sum([len(b) for b in batches]) + dtype = batches[0].past_key_values.dtype device = batches[0].input_ids.device input_ids = batches[0].input_ids.new_empty(total_batch_size) @@ -302,19 +313,19 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch past_key_values = [] all_input_ids = [] - all_input_ids_tensor = [] input_lengths = [] prefix_offsets = [] read_offsets = [] - next_token_choosers = [] + next_token_chooser_parameters = [] stopping_criterias = [] # Cumulative length cumulative_batch_size = 0 cumulative_length = 0 max_tokens = 0 + max_length = 0 for i, batch in enumerate(batches): requests.extend(batch.requests) @@ -347,25 +358,54 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch ) all_input_ids.extend(batch.all_input_ids) - all_input_ids_tensor.extend(batch.all_input_ids_tensor) input_lengths.extend(batch.input_lengths) prefix_offsets.extend(batch.prefix_offsets) read_offsets.extend(batch.read_offsets) - next_token_choosers.extend(batch.next_token_choosers) + next_token_chooser_parameters.extend([r.parameters for r in batch.requests]) stopping_criterias.extend(batch.stopping_criterias) # Update cumulative_length += batch.cu_seqlens[-1] cumulative_batch_size += len(batch) max_tokens += batch.max_tokens + max_length = max( + max_length, + max( + input_length + + stopping_criteria.max_new_tokens + - stopping_criteria.current_tokens + for input_length, stopping_criteria in zip( + batch.input_lengths, batch.stopping_criterias + ) + ), + ) + + all_input_ids_tensor = torch.zeros( + (total_batch_size, max_length), dtype=torch.int64, device=device + ) + + cumulative_batch_size = 0 + for i, batch in enumerate(batches): + start_index = cumulative_batch_size + end_index = cumulative_batch_size + len(batch) + + all_input_ids_tensor[ + start_index:end_index, : batch.all_input_ids_tensor.shape[1] + ] = batch.all_input_ids_tensor[:, :max_length] + + cumulative_batch_size += len(batch) # Cat past past_key_values = torch.cat(past_key_values, dim=1) # Create final tensor on GPU cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32, device=device) + next_token_chooser = HeterogeneousNextTokenChooser.from_pb( + next_token_chooser_parameters, dtype=dtype, device=device + ) + return FlashCausalLMBatch( batch_id=batches[0].batch_id, requests=requests, @@ -381,7 +421,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch read_offsets=read_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=all_input_ids_tensor, - next_token_choosers=next_token_choosers, + next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, max_tokens=max_tokens, ) @@ -463,6 +503,7 @@ def generate_token( self, batch: FlashCausalLMBatch ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: prefill = batch.past_key_values is None + single_request = len(batch) == 1 if prefill and len(batch) == 1: # Ask to pre-allocate kv to its max size @@ -483,6 +524,17 @@ def generate_token( pre_allocate_past_size, ) + if prefill: + next_token_logits = ( + out[-1:] if single_request else out[batch.cu_seqlens[1:] - 1] + ) + else: + next_token_logits = out + + next_input_ids, next_token_logprobs = batch.next_token_chooser( + batch.all_input_ids_tensor[:, : batch.max_seqlen], next_token_logits + ) + if prefill: if len(batch) > 1: # We create the prefill_tokens_indices tensor that will be used to gather prefill logprobs @@ -493,15 +545,11 @@ def generate_token( batch.cu_seqlens_q = torch.arange( 0, len(batch) + 1, device=self.device, dtype=torch.int32 ) - next_input_ids = batch.input_ids.new_empty(len(batch)) next_position_ids = batch.position_ids.new_empty(len(batch)) else: prefill_logprobs = None - next_input_ids = batch.input_ids next_position_ids = batch.position_ids - next_token_logprobs = out.new_empty(len(batch)) - # Prepare past for next decode if len(batch) > 1: # Used to slice next batch past @@ -552,7 +600,6 @@ def generate_token( # Zipped iterator iterator = zip( batch.input_lengths, - batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, ) @@ -564,7 +611,6 @@ def generate_token( # For each member of the batch for i, ( input_length, - next_token_chooser, stopping_criteria, all_input_ids, ) in enumerate(iterator): @@ -573,21 +619,6 @@ def generate_token( end_index = cumulative_length + input_length if prefill: - # Prefill mode - # out is of shape [cumulative_sequence_lengths, vocab_size] - # only take last token logit - logits = out[end_index - 1 : end_index] - - # Create all_input_ids_tensor that will be used by token warpers (for example, RepetitionPenalty) - all_input_ids_tensor = batch.input_ids.new_empty( - input_length + stopping_criteria.max_new_tokens - ) - # Copy from batch.input_ids to all_input_ids_tensor - all_input_ids_tensor[:input_length] = batch.input_ids[ - start_index:end_index - ] - batch.all_input_ids_tensor.append(all_input_ids_tensor) - # Initialize position_ids # In decode, we do not need this as we can just increment position ids next_position_ids[i] = batch.position_ids[end_index - 1] @@ -603,25 +634,8 @@ def generate_token( prefill_tokens_indices = batch.input_ids[ start_index + 1 : end_index ] - else: - # Decode mode - # out is of shape [batch_size, vocab_size] - logits = out[i].view(1, -1) - all_input_ids_tensor = batch.all_input_ids_tensor[i] - - # Select next token - next_token_id, logprob = next_token_chooser( - all_input_ids_tensor[None, :input_length], logits - ) - - # Add to all_input_ids_tensor - next_token_id_squeezed = next_token_id.view(1) - all_input_ids_tensor[input_length] = next_token_id_squeezed - - # Set values - next_input_ids[i] = next_token_id_squeezed - next_token_logprobs[i] = logprob[-1, next_token_id].view(1) + batch.all_input_ids_tensor[i, input_length] = next_input_ids[i] cumulative_length += input_length @@ -651,10 +665,11 @@ def generate_token( batch.input_lengths, batch.prefix_offsets, batch.read_offsets, - batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, batch.all_input_ids_tensor, + batch.next_token_chooser.do_sample, + batch.next_token_chooser.seeds, next_token_ids, next_token_logprobs, ) @@ -665,10 +680,11 @@ def generate_token( input_length, prefix_offset, read_offset, - next_token_chooser, stopping_criteria, all_input_ids, all_input_ids_tensor, + do_sample, + seed, next_token_id, next_token_logprob, ) in enumerate(iterator): @@ -702,14 +718,11 @@ def generate_token( output_text = self.decode( all_input_ids[-stopping_criteria.current_tokens :] ) - # Get seed - if isinstance(next_token_chooser.choice, Sampling): - seed = next_token_chooser.choice.seed - else: - seed = None - generated_text = GeneratedText( - output_text, stopping_criteria.current_tokens, reason, seed + output_text, + stopping_criteria.current_tokens, + reason, + seed if do_sample else None, ) else: generated_text = None @@ -751,8 +764,9 @@ def generate_token( batch.prefix_offsets[i] = prefix_offset batch.read_offsets[i] = read_offset batch.all_input_ids[i] = all_input_ids - batch.max_seqlen = batch.max_seqlen + 1 cumulative_length += input_length + batch.max_seqlen = batch.max_seqlen + 1 + # No need to return a batch if we know that all requests stopped return generations, batch if not stopped else None diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index bc3096c6..0a3f341b 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -89,6 +89,7 @@ def from_pb( cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, device: torch.device, ) -> "GalacticaCausalLMBatch": inputs = [] diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 2abb87ae..68e59dc3 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -71,6 +71,7 @@ def from_pb( cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, device: torch.device, ) -> "Seq2SeqLMBatch": """Convert a text_generation_server.v1.Batch protobuf to a Seq2SeqLMBatch""" diff --git a/server/text_generation_server/models/types.py b/server/text_generation_server/models/types.py index 66a8c212..28ca8147 100644 --- a/server/text_generation_server/models/types.py +++ b/server/text_generation_server/models/types.py @@ -21,6 +21,7 @@ def from_pb( cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, device: torch.device, ) -> "Batch": raise NotImplementedError diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index e47fd049..e1bd8412 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -55,7 +55,7 @@ async def FilterBatch(self, request, context): async def Prefill(self, request, context): batch = self.model.batch_type.from_pb( - request.batch, self.model.tokenizer, self.model.device + request.batch, self.model.tokenizer, self.model.dtype, self.model.device ) generations, next_batch = self.model.generate_token(batch) diff --git a/server/text_generation_server/utils/__init__.py b/server/text_generation_server/utils/__init__.py index 50d64518..6a351d66 100644 --- a/server/text_generation_server/utils/__init__.py +++ b/server/text_generation_server/utils/__init__.py @@ -9,12 +9,13 @@ RevisionNotFoundError, ) from text_generation_server.utils.tokens import ( - Greedy, NextTokenChooser, - Sampling, + HeterogeneousNextTokenChooser, StoppingCriteria, StopSequenceCriteria, FinishReason, + Sampling, + Greedy, ) __all__ = [ @@ -25,6 +26,7 @@ "weight_hub_files", "download_weights", "EntryNotFoundError", + "HeterogeneousNextTokenChooser", "LocalEntryNotFoundError", "RevisionNotFoundError", "Greedy", diff --git a/server/text_generation_server/utils/convert.py b/server/text_generation_server/utils/convert.py index caf1a764..c43a4464 100644 --- a/server/text_generation_server/utils/convert.py +++ b/server/text_generation_server/utils/convert.py @@ -1,14 +1,10 @@ -import concurrent -import time import datetime import torch -from concurrent.futures import ThreadPoolExecutor from collections import defaultdict -from datetime import timedelta from loguru import logger from pathlib import Path -from safetensors.torch import load_file, save_file +from safetensors.torch import save_file from safetensors import safe_open from typing import Dict, List diff --git a/server/text_generation_server/utils/logits_process.py b/server/text_generation_server/utils/logits_process.py new file mode 100644 index 00000000..faa94516 --- /dev/null +++ b/server/text_generation_server/utils/logits_process.py @@ -0,0 +1,405 @@ +import math +import torch + +from functools import lru_cache +from typing import Optional, List, Dict, Union + +from transformers import ( + LogitsWarper, + LogitsProcessor, + TemperatureLogitsWarper, + TopKLogitsWarper, + TopPLogitsWarper, + TypicalLogitsWarper, +) + +mempool = torch.cuda.graph_pool_handle() if torch.cuda.is_available() else None + + +class StaticWarper: + def __init__( + self, + temperature=1.0, + top_k=None, + top_p=None, + typical_p=None, + ): + self.warpers = [] + + if temperature is not None and temperature != 1.0: + temperature = float(temperature) + self.warpers.append(TemperatureLogitsWarper(temperature)) + if top_k is not None and top_k != 0: + self.warpers.append(TopKLogitsWarper(top_k=top_k)) + if top_p is not None and top_p < 1.0: + self.warpers.append(TopPLogitsWarper(top_p=top_p)) + if typical_p is not None and typical_p < 1.0: + self.warpers.append(TypicalLogitsWarper(mass=typical_p)) + + self.cuda_graph = None + self.static_scores = None + self.static_warped_scores = None + self.static_next_logprob = None + + def __call__(self, scores): + if self.cuda_graph is None: + self.static_scores = scores + self.cuda_graph = torch.cuda.CUDAGraph() + + with torch.cuda.graph(self.cuda_graph, pool=mempool): + local_scores = self.static_scores + for warper in self.warpers: + local_scores = warper(None, local_scores) + + self.static_warped_scores = local_scores + # Compute logprobs + self.static_next_logprob = torch.log_softmax( + self.static_warped_scores, -1 + ) + + self.static_scores.copy_(scores) + self.cuda_graph.replay() + + return self.static_warped_scores, self.static_next_logprob + + +@lru_cache(10) +def static_warper( + temperature: Optional[float], + top_k: Optional[int], + top_p: Optional[float], + typical_p: Optional[float], +) -> StaticWarper: + return StaticWarper( + temperature=temperature, top_k=top_k, top_p=top_p, typical_p=typical_p + ) + + +class HeterogeneousRepetitionPenaltyLogitsProcessor(LogitsProcessor): + r""" + [`LogitsProcessor`] enforcing an exponential penalty on repeated sequences. + This version allows for a separate value for each sample and runs inplace when possible. + It doesn't validate inputs. + + Args: + repetition_penalty (`List[float]`): + The parameter for repetition penalty. 1.0 means no penalty. See [this + paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + """ + + def __init__(self, penalty: List[float], dtype: torch.dtype, device: torch.device): + self.penalty = penalty + self.penalty_tensor = torch.tensor( + penalty, dtype=dtype, device=device + ).unsqueeze(1) + + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor: + score = torch.gather(scores, 1, input_ids) + + # if score < 0 then repetition penalty has to be multiplied to reduce the previous token probability + score = torch.where( + score < 0, score * self.penalty_tensor, score / self.penalty_tensor + ) + + scores.scatter_(1, input_ids, score) + return scores + + def filter(self, indices): + self.penalty = [self.penalty[i] for i in indices] + if any([x != 1.0 for x in self.penalty]): + self.penalty_tensor = self.penalty_tensor[indices] + return self + return None + + +class HeterogeneousTemperatureLogitsWarper: + r""" + [`LogitsWarper`] for temperature (exponential scaling output probability distribution). + This version allows for a separate value for each sample and runs inplace when possible. + It doesn't validate inputs. + + Args: + temperature (`float`): + The value used to module the logits distribution. + """ + + def __init__( + self, temperature: List[float], dtype: torch.dtype, device: torch.device + ): + self.temperature = temperature + self.temperature_tensor = torch.tensor( + temperature, dtype=dtype, device=device + ).unsqueeze(1) + + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor: + scores.div_(self.temperature_tensor) + return scores + + def filter(self, indices): + self.temperature = [self.temperature[i] for i in indices] + if any([x != 1.0 for x in self.temperature]): + self.temperature_tensor = self.temperature_tensor[indices] + return self + return None + + +class HeterogeneousTopPLogitsWarper(LogitsWarper): + """ + [`LogitsWarper`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <= prob_cut_off. + This version allows for a separate value for each sample and runs inplace when possible. + It doesn't validate inputs. + + Args: + top_p (`float`): + If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + higher are kept for generation. + filter_value (`float`, *optional*, defaults to `-float("Inf")`): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + """ + + def __init__( + self, + top_p: List[float], + dtype: torch.dtype, + device: torch.device, + filter_value: float = -math.inf, + min_tokens_to_keep: int = 1, + ): + self.top_p = top_p + self.top_p_opposite = 1 - torch.tensor( + top_p, dtype=dtype, device=device + ).unsqueeze(1) + self.filter_value = filter_value + self.min_tokens_to_keep = min_tokens_to_keep + + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor: + sorted_logits, sorted_indices = torch.sort(scores, descending=False) + probs = sorted_logits.softmax(dim=-1) + # This is way faster for some reason + for i in range(probs.shape[0]): + probs[i] = probs[i].cumsum(dim=-1) + + # Remove tokens with cumulative top_p above the threshold (token with 0 are kept) + sorted_indices_to_remove = probs <= self.top_p_opposite + if self.min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep + sorted_indices_to_remove[..., -self.min_tokens_to_keep :] = 0 + + # scatter sorted tensors to original indexing + indices_to_remove = sorted_indices_to_remove.scatter( + 1, sorted_indices, sorted_indices_to_remove + ) + warped_scores = scores.masked_fill_(indices_to_remove, self.filter_value) + + return warped_scores + + def filter(self, indices): + self.top_p = [self.top_p[i] for i in indices] + if any([x < 1.0 for x in self.top_p]): + self.top_p_opposite = self.top_p_opposite[indices] + return self + return None + + +class HeterogeneousTopKLogitsWarper(LogitsWarper): + r""" + [`LogitsWarper`] that performs top-k, i.e. restricting to the k highest probability elements. + This version allows for a separate value for each sample and runs inplace when possible. + It doesn't validate inputs. + + Args: + top_k (`int`): + The number of highest probability vocabulary tokens to keep for top-k-filtering. + filter_value (`float`, *optional*, defaults to `-float("Inf")`): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + """ + + def __init__( + self, + top_k: List[int], + device: torch.device, + filter_value: float = -math.inf, + min_tokens_to_keep: int = 1, + ): + self.top_k = top_k + self.max_top_k = max(top_k) + # value - 1 as we will use top_k to index and python uses 0 based numbering + self.top_k_tensor = torch.tensor( + [max(x - 1, min_tokens_to_keep - 1) for x in top_k], + dtype=torch.int64, + device=device, + ).unsqueeze(1) + + # 0 is a special value that disables top_k warping for this member of the batch + disabled = [x == 0 for x in top_k] + + if any(disabled): + self.top_k_disabled_mask = torch.tensor( + disabled, dtype=torch.bool, device=device + ).view(-1, 1) + else: + self.top_k_disabled_mask = None + + self.filter_value = filter_value + + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor: + # If max_top_k is superior to the vocab, we need to clamp or the warper will fail + if scores.size(-1) < self.max_top_k: + max_top_k = scores.size(-1) + top_k = torch.clamp_max(self.top_k_tensor, max_top_k) + else: + max_top_k = self.max_top_k + top_k = self.top_k_tensor + + # Get the kth score for each member of the batch + kth_scores = torch.gather(torch.topk(scores, max_top_k)[0], 1, top_k) + + # Mask member of kth_scores that do not want to use top_k warping + if self.top_k_disabled_mask is not None: + kth_scores.masked_fill_(self.top_k_disabled_mask, self.filter_value) + + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = scores < kth_scores + scores.masked_fill_(indices_to_remove, self.filter_value) + return scores + + def filter(self, indices): + self.top_k = [self.top_k[i] for i in indices] + disabled = [x == 0 for x in self.top_k] + + if not all(disabled): + self.top_k_tensor = self.top_k_tensor[indices] + self.max_top_k = max(self.top_k) + + if self.top_k_disabled_mask is not None: + self.top_k_disabled_mask = ( + self.top_k_disabled_mask[indices] if any(disabled) else None + ) + + return self + return None + + +class HeterogeneousTypicalLogitsWarper(LogitsWarper): + r""" + [`LogitsWarper`] that performs typical decoding. See [Typical Decoding for Natural Language + Generation](https://arxiv.org/abs/2202.00666) for more information. + This version allows for a separate value for each sample and runs inplace when possible. + It doesn't validate inputs. + + Args: + mass (`float`): + Value of typical_p between 0 and 1 inclusive, defaults to 0.9. + filter_value (`float`, *optional*, defaults to `-float("Inf")`): + All filtered values will be set to this float value. + min_tokens_to_keep (`int`, *optional*, defaults to 1): + Minimum number of tokens that cannot be filtered. + """ + + def __init__( + self, + mass: List[float], + dtype: torch.dtype, + device: torch.device, + filter_value: float = -math.inf, + min_tokens_to_keep: int = 1, + ): + self.mass = mass + self.mass_tensor = torch.tensor(mass, dtype=dtype, device=device).unsqueeze(1) + + # 1 is a special value that disables typical_p warping for this member of the batch + disabled = [x == 1.0 for x in mass] + + if any(disabled): + self.disabled_mask = torch.tensor(disabled, dtype=torch.bool, device=device) + else: + self.disabled_mask = None + + self.filter_value = filter_value + self.min_tokens_to_keep = min_tokens_to_keep + + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor: + # calculate entropy + normalized = torch.nn.functional.log_softmax(scores, dim=-1) + p = torch.exp(normalized) + ent = -(normalized * p).nansum(-1, keepdim=True) + + # shift and sort + shifted_scores = torch.abs((-normalized) - ent) + sorted_scores, sorted_indices = torch.sort(shifted_scores, descending=False) + sorted_logits = scores.gather(-1, sorted_indices) + probs = sorted_logits.softmax(dim=-1) + # This is way faster for some reason + for i in range(probs.shape[0]): + probs[i] = probs[i].cumsum(dim=-1) + + # Remove tokens with cumulative mass above the threshold + last_ind = (probs < self.mass_tensor).sum(dim=1) + last_ind[last_ind < 0] = 0 + + if self.disabled_mask is not None: + last_ind.masked_fill_(self.disabled_mask, scores.shape[-1] - 1) + + sorted_indices_to_remove = sorted_scores > sorted_scores.gather( + 1, last_ind.view(-1, 1) + ) + if self.min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below) + sorted_indices_to_remove[..., : self.min_tokens_to_keep] = 0 + indices_to_remove = sorted_indices_to_remove.scatter( + 1, sorted_indices, sorted_indices_to_remove + ) + + warped_scores = scores.masked_fill_(indices_to_remove, self.filter_value) + + return warped_scores + + def filter(self, indices): + self.mass = [self.mass[i] for i in indices] + disabled = [x == 1.0 for x in self.mass] + + if not all(disabled): + self.mass_tensor = self.mass_tensor[indices] + + if self.disabled_mask is not None: + self.disabled_mask = ( + self.disabled_mask[indices] if any(disabled) else None + ) + + return self + return None + + +class HeterogeneousProcessorWrapper(LogitsProcessor): + r""" + A wrapper for logit warpers or processors without heterogeneous parameter support. + Args: + processors (`Dict[int, Union[LogitsProcessor, LogitsWarper]]`): + A mapping of sample indices to logit warpers or processors, to be run sequentially. + """ + + def __init__( + self, + processors: Dict[int, Union[LogitsProcessor, LogitsWarper]], + ): + self.processors = processors + + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor: + for i, processor in self.processors.items(): + scores[i : i + 1] = processor(input_ids[i : i + 1], scores[i : i + 1]) + return scores + + def filter(self, indices): + new_processors = {} + for i, idx in enumerate(indices): + if idx in self.processors: + new_processors[i] = self.processors[idx] + + if new_processors: + self.processors = new_processors + return self + return None diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index e9fb96b0..e6e512bc 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -1,12 +1,7 @@ import re import torch -from functools import lru_cache from transformers import ( - TemperatureLogitsWarper, - TopKLogitsWarper, - TopPLogitsWarper, - TypicalLogitsWarper, RepetitionPenaltyLogitsProcessor, PreTrainedTokenizerBase, ) @@ -15,82 +10,15 @@ from text_generation_server.pb import generate_pb2 from text_generation_server.pb.generate_pb2 import FinishReason from text_generation_server.utils.watermark import WatermarkLogitsProcessor - - -class Sampling: - def __init__(self, seed: int, device: str = "cpu"): - self.generator = torch.Generator(device) - self.generator.manual_seed(seed) - self.seed = seed - - def __call__(self, logits): - probs = torch.nn.functional.softmax(logits, -1) - # Avoid GPU<->CPU sync done by torch multinomial - # See: https://github.com/pytorch/pytorch/blob/925a3788ec5c06db62ca732a0e9425a26a00916f/aten/src/ATen/native/Distributions.cpp#L631-L637 - q = torch.empty_like(probs).exponential_(1, generator=self.generator) - return probs.div_(q).argmax() - - -class Greedy: - def __call__(self, logits): - return logits.argmax() - - -class StaticWarper: - def __init__( - self, - temperature=1.0, - top_k=None, - top_p=None, - typical_p=None, - ): - self.warpers = [] - - if temperature is not None and temperature != 1.0: - temperature = float(temperature) - self.warpers.append(TemperatureLogitsWarper(temperature)) - if top_k is not None and top_k != 0: - self.warpers.append(TopKLogitsWarper(top_k=top_k)) - if top_p is not None and top_p < 1.0: - self.warpers.append(TopPLogitsWarper(top_p=top_p)) - if typical_p is not None and typical_p < 1.0: - self.warpers.append(TypicalLogitsWarper(mass=typical_p)) - - self.cuda_graph = None - self.static_scores = None - self.static_warped_scores = None - self.static_next_logprob = None - - def __call__(self, scores): - if self.cuda_graph is None: - self.static_scores = scores - self.cuda_graph = torch.cuda.CUDAGraph() - - with torch.cuda.graph(self.cuda_graph): - for warper in self.warpers: - self.static_warped_scores = warper(None, self.static_scores) - - # Compute logprobs - self.static_next_logprob = torch.log_softmax( - self.static_warped_scores, -1 - ) - - self.static_scores.copy_(scores) - self.cuda_graph.replay() - - return self.static_warped_scores, self.static_next_logprob - - -@lru_cache(10) -def static_warper( - temperature: Optional[float], - top_k: Optional[int], - top_p: Optional[float], - typical_p: Optional[float], -) -> StaticWarper: - return StaticWarper( - temperature=temperature, top_k=top_k, top_p=top_p, typical_p=typical_p - ) +from text_generation_server.utils.logits_process import ( + static_warper, + HeterogeneousRepetitionPenaltyLogitsProcessor, + HeterogeneousTemperatureLogitsWarper, + HeterogeneousTopKLogitsWarper, + HeterogeneousTopPLogitsWarper, + HeterogeneousTypicalLogitsWarper, + HeterogeneousProcessorWrapper, +) class NextTokenChooser: @@ -132,9 +60,9 @@ def __init__( self.choice = Sampling(seed, device) if sampling else Greedy() def __call__(self, input_ids, scores): - if self.watermark_processor: + if self.watermark_processor is not None: scores = self.watermark_processor(input_ids, scores) - if self.repetition_processor: + if self.repetition_processor is not None: scores = self.repetition_processor(input_ids, scores) if self.static_warper is None: @@ -221,3 +149,191 @@ def from_pb( pb.max_new_tokens, pb.ignore_eos_token, ) + + +class HeterogeneousNextTokenChooser: + def __init__( + self, + dtype: torch.dtype, + device: torch.device, + watermark: List[bool], + temperature: List[float], + repetition_penalty: List[float], + top_k: List[int], + top_p: List[float], + typical_p: List[float], + do_sample: List[bool], + seeds: List[int], + ): + warpers = [] + + self.watermark_processor = ( + HeterogeneousProcessorWrapper( + { + i: WatermarkLogitsProcessor(device=device) + for i, do_watermark in enumerate(watermark) + if do_watermark + } + ) + if any(watermark) + else None + ) + + self.repetition_processor = ( + HeterogeneousRepetitionPenaltyLogitsProcessor( + repetition_penalty, dtype, device + ) + if any([x != 1.0 for x in repetition_penalty]) + else None + ) + + if any([x != 1.0 for x in temperature]): + do_sample = [ + sample or x != 1.0 for x, sample in zip(temperature, do_sample) + ] + warpers.append( + HeterogeneousTemperatureLogitsWarper(temperature, dtype, device) + ) + + if any([x != 0 for x in top_k]): + do_sample = [sample or x != 0 for x, sample in zip(top_k, do_sample)] + warpers.append(HeterogeneousTopKLogitsWarper(top_k, device)) + + if any([x < 1.0 for x in top_p]): + do_sample = [sample or x < 1.0 for x, sample in zip(top_p, do_sample)] + warpers.append(HeterogeneousTopPLogitsWarper(top_p, dtype, device)) + + if any([x < 1.0 for x in typical_p]): + do_sample = [sample or x < 1.0 for x, sample in zip(typical_p, do_sample)] + warpers.append(HeterogeneousTypicalLogitsWarper(typical_p, dtype, device)) + + self.warpers = warpers + + if any(do_sample): + self.choice = HeterogeneousSampling(do_sample, seeds, device) + else: + self.choice = Greedy() + + self.seeds = seeds + self.do_sample = do_sample + + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor): + if self.watermark_processor is not None: + scores = self.watermark_processor(input_ids, scores) + if self.repetition_processor is not None: + scores = self.repetition_processor(input_ids, scores) + + for warper in self.warpers: + scores = warper(input_ids, scores) + + next_ids = self.choice(scores) + next_logprobs = torch.gather( + torch.log_softmax(scores, -1), 1, next_ids.view(-1, 1) + ).view(-1) + + return next_ids, next_logprobs + + def filter(self, indices): + if self.watermark_processor is not None: + self.watermark_processor = self.watermark_processor.filter(indices) + + if self.repetition_processor is not None: + self.repetition_processor = self.repetition_processor.filter(indices) + + filtered_warpers = [] + for warper in self.warpers: + filtered_warper = warper.filter(indices) + if filtered_warper is not None: + filtered_warpers.append(filtered_warper) + self.warpers = filtered_warpers + + self.seeds = [self.seeds[i] for i in indices] + self.do_sample = [self.do_sample[i] for i in indices] + + if any(self.do_sample): + self.choice.filter(indices) + else: + self.choice = Greedy() + + return self + + @classmethod + def from_pb( + cls, + pb: List[generate_pb2.NextTokenChooserParameters], + dtype: torch.dtype, + device: torch.device, + ) -> "HeterogeneousNextTokenChooser": + return HeterogeneousNextTokenChooser( + watermark=[pb_.watermark for pb_ in pb], + temperature=[pb_.temperature for pb_ in pb], + repetition_penalty=[pb_.repetition_penalty for pb_ in pb], + top_k=[pb_.top_k for pb_ in pb], + top_p=[pb_.top_p for pb_ in pb], + typical_p=[pb_.typical_p for pb_ in pb], + do_sample=[pb_.do_sample for pb_ in pb], + seeds=[pb_.seed for pb_ in pb], + device=device, + dtype=dtype, + ) + + +class Sampling: + def __init__(self, seed: int, device: str = "cpu"): + self.generator = torch.Generator(device) + self.generator.manual_seed(seed) + self.seed = seed + + def __call__(self, logits): + probs = torch.nn.functional.softmax(logits, -1) + # Avoid GPU<->CPU sync done by torch multinomial + # See: https://github.com/pytorch/pytorch/blob/925a3788ec5c06db62ca732a0e9425a26a00916f/aten/src/ATen/native/Distributions.cpp#L631-L637 + q = torch.empty_like(probs).exponential_(1, generator=self.generator) + return probs.div_(q).argmax() + + +class Greedy: + def __call__(self, logits): + return logits.argmax(dim=-1) + + +class HeterogeneousSampling: + r""" + Mixed greedy and probabilistic sampling. Compute both and pick the right one for each sample. + """ + + def __init__(self, do_sample: List[bool], seeds: List[int], device: torch.device): + self.seeds = seeds + + self.greedy_indices = [] + self.sampling_mapping = {} + for i, (sample, seed) in enumerate(zip(do_sample, seeds)): + if sample: + self.sampling_mapping[i] = Sampling(seed, device) + else: + self.greedy_indices.append(i) + + self.greedy = Greedy() + + def __call__(self, logits): + out = torch.empty(logits.shape[0], dtype=torch.int64, device=logits.device) + if self.greedy_indices: + # Computing for all indices is faster than slicing + torch.argmax(logits, -1, out=out) + + for i, sampling in self.sampling_mapping.items(): + out[i] = sampling(logits[i]) + return out + + def filter(self, indices): + new_greedy_indices = [] + new_sampling_mapping = {} + for i, idx in enumerate(indices): + if idx in self.sampling_mapping: + new_sampling_mapping[i] = self.sampling_mapping[idx] + else: + new_greedy_indices.append(i) + + self.greedy_indices = new_greedy_indices + self.sampling_mapping = new_sampling_mapping + return self From 5fde8d99919f3c81e1fd414aa11d2148680baea6 Mon Sep 17 00:00:00 2001 From: CL-Shang Date: Fri, 26 May 2023 18:31:47 +0800 Subject: [PATCH 256/793] Fix issue when load AutoModelForSeq2SeqLM model (#370) --- server/text_generation_server/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index bf7a2849..46a4563b 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -266,7 +266,7 @@ def get_model( quantize=quantize, trust_remote_code=trust_remote_code, ) - if "AutoModelForSeq2SeqLM" in auto_map.keys: + if "AutoModelForSeq2SeqLM" in auto_map.keys(): return Seq2SeqLM( model_id, revision, From 146e72c3be4f5bfe88b9baae4c2ff6a2ce41eda9 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Tue, 30 May 2023 12:52:18 +0200 Subject: [PATCH 257/793] fix(launcher): parse num cuda devices from CUDA_VISIBLE_DEVICES and NVIDIA_VISIBLE_DEVICES --- launcher/src/main.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index dc12c90f..a863ea4a 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -455,11 +455,11 @@ fn shutdown_shards(shutdown: Arc>, shutdown_receiver: &mpsc::Receive } fn num_cuda_devices() -> Option { - if let Ok(cuda_visible_devices) = env::var("CUDA_VISIBLE_DEVICES") { - let n_devices = cuda_visible_devices.split(',').count(); - return Some(n_devices); - } - None + let devices = env::var("CUDA_VISIBLE_DEVICES") + .map_err(|_| env::var("NVIDIA_VISIBLE_DEVICES")) + .ok()?; + let n_devices = devices.split(',').count(); + Some(n_devices) } #[derive(Deserialize)] @@ -509,9 +509,9 @@ fn find_num_shards(sharded: Option, num_shard: Option) -> usize { let num_shard = match (sharded, num_shard) { (Some(true), None) => { // try to default to the number of available GPUs - tracing::info!("Parsing num_shard from CUDA_VISIBLE_DEVICES"); - let n_devices = - num_cuda_devices().expect("--num-shard and CUDA_VISIBLE_DEVICES are not set"); + tracing::info!("Parsing num_shard from CUDA_VISIBLE_DEVICES/NVIDIA_VISIBLE_DEVICES"); + let n_devices = num_cuda_devices() + .expect("--num-shard and CUDA_VISIBLE_DEVICES/NVIDIA_VISIBLE_DEVICES are not set"); if n_devices <= 1 { panic!("`sharded` is true but only found {n_devices} CUDA devices"); } From 49a6c8c1b28742e806dd95a36af82db5b45d181d Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Tue, 30 May 2023 13:27:48 +0200 Subject: [PATCH 258/793] fix(launcher): parse num cuda devices from CUDA_VISIBLE_DEVICES and NVIDIA_VISIBLE_DEVICES --- launcher/src/main.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index a863ea4a..0810d979 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -455,9 +455,10 @@ fn shutdown_shards(shutdown: Arc>, shutdown_receiver: &mpsc::Receive } fn num_cuda_devices() -> Option { - let devices = env::var("CUDA_VISIBLE_DEVICES") - .map_err(|_| env::var("NVIDIA_VISIBLE_DEVICES")) - .ok()?; + let devices = match env::var("CUDA_VISIBLE_DEVICES") { + Ok(devices) => devices, + Err(_) => env::var("NVIDIA_VISIBLE_DEVICES").ok()?, + }; let n_devices = devices.split(',').count(); Some(n_devices) } From bf7f1d54345f9e6e091d4b756e8b8381fe767d89 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Tue, 30 May 2023 13:56:03 +0200 Subject: [PATCH 259/793] fix(server): fix quantization --- server/text_generation_server/models/bloom.py | 14 ++++++-------- server/text_generation_server/models/galactica.py | 14 ++++++-------- server/text_generation_server/models/gpt_neox.py | 14 ++++++-------- server/text_generation_server/models/t5.py | 14 ++++++-------- 4 files changed, 24 insertions(+), 32 deletions(-) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 088a1457..90db59fc 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -245,14 +245,12 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) - elif quantize == "gptq": - raise NotImplementedError( - "`gptq` is not implemented for now" - ) - elif quantize is None: - tensor = tensor.to(device) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") + elif quantize == "gptq": + raise NotImplementedError("`gptq` is not implemented for now") + elif quantize is None: + tensor = tensor.to(device) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") module._parameters[param_name] = tensor if name == "word_embeddings.weight": diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 0a3f341b..954421f0 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -364,14 +364,12 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) - elif quantize == "gptq": - raise NotImplementedError( - "`gptq` is not implemented for now" - ) - elif quantize is None: - tensor = tensor.to(device) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") + elif quantize == "gptq": + raise NotImplementedError("`gptq` is not implemented for now") + elif quantize is None: + tensor = tensor.to(device) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") module._parameters[param_name] = tensor if name == "model.decoder.embed_tokens.weight": diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index e4a85082..c0e8170d 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -210,14 +210,12 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) - elif quantize == "gptq": - raise NotImplementedError( - "`gptq` is not implemented for now" - ) - elif quantize is None: - tensor = tensor.to(device) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") + elif quantize == "gptq": + raise NotImplementedError("`gptq` is not implemented for now") + elif quantize is None: + tensor = tensor.to(device) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") if current_parameter_tensor is not None: module._parameters[param_name] = tensor diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 17cc50e0..32dcb806 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -223,14 +223,12 @@ def linear(input, weight, bias): module.linear = replace_linear(state) - elif quantize == "gptq" and not module_name.endswith("wo"): - raise NotImplementedError( - "`gptq` is not implemented for now" - ) - elif quantize is None: - tensor = tensor.to(device) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") + elif quantize == "gptq" and not module_name.endswith("wo"): + raise NotImplementedError("`gptq` is not implemented for now") + elif quantize is None or module_name.endswith("wo"): + tensor = tensor.to(device) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") if current_parameter_tensor is not None: module._parameters[param_name] = tensor From b8b950b37cc1d65f1c02f6c1615a6f0cf40b09c5 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 30 May 2023 18:25:19 +0200 Subject: [PATCH 260/793] feat(server): support RefinedWeb models (#379) --- integration-tests/conftest.py | 16 +- .../test_flash_falcon/test_flash_falcon.json | 378 ++++ .../test_flash_falcon_all_params.json | 98 ++ .../test_flash_falcon_load.json | 1514 +++++++++++++++++ integration-tests/models/test_flash_falcon.py | 63 + .../text_generation_server/models/__init__.py | 37 + .../custom_modeling/flash_llama_modeling.py | 19 +- .../custom_modeling/flash_neox_modeling.py | 19 +- .../custom_modeling/flash_rw_modeling.py | 774 +++++++++ .../text_generation_server/models/flash_rw.py | 244 +++ server/text_generation_server/models/rw.py | 86 + server/text_generation_server/utils/layers.py | 15 +- 12 files changed, 3236 insertions(+), 27 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon.json create mode 100644 integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json create mode 100644 integration-tests/models/test_flash_falcon.py create mode 100644 server/text_generation_server/models/custom_modeling/flash_rw_modeling.py create mode 100644 server/text_generation_server/models/flash_rw.py create mode 100644 server/text_generation_server/models/rw.py diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 7db12424..902a7158 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -205,7 +205,10 @@ def event_loop(): def launcher(event_loop): @contextlib.contextmanager def local_launcher( - model_id: str, num_shard: Optional[int] = None, quantize: Optional[str] = None + model_id: str, + num_shard: Optional[int] = None, + quantize: Optional[str] = None, + trust_remote_code: bool = False, ): port = random.randint(8000, 10_000) master_port = random.randint(10_000, 20_000) @@ -230,6 +233,9 @@ def local_launcher( args.extend(["--num-shard", str(num_shard)]) if quantize: args.append("--quantize") + args.append("bitsandbytes") + if trust_remote_code: + args.append("--trust-remote-code") env = os.environ env["LOG_LEVEL"] = "info,text_generation_router=debug" @@ -250,7 +256,10 @@ def local_launcher( @contextlib.contextmanager def docker_launcher( - model_id: str, num_shard: Optional[int] = None, quantize: Optional[str] = None + model_id: str, + num_shard: Optional[int] = None, + quantize: Optional[str] = None, + trust_remote_code: bool = False, ): port = random.randint(8000, 10_000) @@ -260,6 +269,9 @@ def docker_launcher( args.extend(["--num-shard", str(num_shard)]) if quantize: args.append("--quantize") + args.append("bitsandbytes") + if trust_remote_code: + args.append("--trust-remote-code") client = docker.from_env() diff --git a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon.json b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon.json new file mode 100644 index 00000000..488f3de3 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon.json @@ -0,0 +1,378 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50, + "logprob": null, + "text": "G" + }, + { + "id": 330, + "logprob": -5.96875, + "text": "ir" + }, + { + "id": 1622, + "logprob": -5.6132812, + "text": "af" + }, + { + "id": 249, + "logprob": -6.5039062, + "text": "at" + }, + { + "id": 1480, + "logprob": -8.078125, + "text": "ron" + }, + { + "id": 304, + "logprob": -2.3261719, + "text": " is" + }, + { + "id": 23866, + "logprob": -9.59375, + "text": " obsessed" + }, + { + "id": 335, + "logprob": -0.048339844, + "text": " with" + }, + { + "id": 26680, + "logprob": -4.0, + "text": " gir" + }, + { + "id": 1903, + "logprob": -0.07556152, + "text": "aff" + }, + { + "id": 255, + "logprob": -0.0067749023, + "text": "es" + }, + { + "id": 23, + "logprob": -1.546875, + "text": "," + }, + { + "id": 248, + "logprob": -4.3320312, + "text": " the" + }, + { + "id": 758, + "logprob": -3.734375, + "text": " most" + }, + { + "id": 21735, + "logprob": -5.109375, + "text": " glorious" + }, + { + "id": 5985, + "logprob": -2.09375, + "text": " animal" + }, + { + "id": 313, + "logprob": -1.1835938, + "text": " on" + }, + { + "id": 248, + "logprob": -0.77685547, + "text": " the" + }, + { + "id": 1936, + "logprob": -2.3828125, + "text": " face" + }, + { + "id": 275, + "logprob": -0.004432678, + "text": " of" + }, + { + "id": 414, + "logprob": -1.9677734, + "text": " this" + }, + { + "id": 6490, + "logprob": -2.046875, + "text": " Earth" + }, + { + "id": 25, + "logprob": -0.28198242, + "text": "." + }, + { + "id": 401, + "logprob": -7.9179688, + "text": " G" + }, + { + "id": 6013, + "logprob": -2.2753906, + "text": "ira" + }, + { + "id": 694, + "logprob": -0.6230469, + "text": "ft" + }, + { + "id": 1480, + "logprob": -0.20874023, + "text": "ron" + }, + { + "id": 9369, + "logprob": -4.5507812, + "text": " believes" + }, + { + "id": 455, + "logprob": -4.5664062, + "text": " all" + }, + { + "id": 599, + "logprob": -2.7402344, + "text": " other" + }, + { + "id": 5632, + "logprob": -0.21948242, + "text": " animals" + }, + { + "id": 362, + "logprob": -0.7675781, + "text": " are" + }, + { + "id": 23981, + "logprob": -5.0, + "text": " irrelevant" + }, + { + "id": 635, + "logprob": -4.234375, + "text": " when" + }, + { + "id": 4354, + "logprob": -0.5131836, + "text": " compared" + }, + { + "id": 271, + "logprob": -0.103637695, + "text": " to" + }, + { + "id": 248, + "logprob": -0.58447266, + "text": " the" + }, + { + "id": 21735, + "logprob": -3.6835938, + "text": " glorious" + }, + { + "id": 64398, + "logprob": -1.8173828, + "text": " majesty" + }, + { + "id": 275, + "logprob": -0.23510742, + "text": " of" + }, + { + "id": 248, + "logprob": -0.35473633, + "text": " the" + }, + { + "id": 26680, + "logprob": -0.24633789, + "text": " gir" + }, + { + "id": 23226, + "logprob": -0.02960205, + "text": "affe" + }, + { + "id": 25, + "logprob": -0.17333984, + "text": "." + }, + { + "id": 193, + "logprob": -1.3935547, + "text": "\n" + }, + { + "id": 23626, + "logprob": -10.0625, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -4.59375, + "text": ":" + }, + { + "id": 23090, + "logprob": -6.9375, + "text": " Hello" + }, + { + "id": 23, + "logprob": -0.99365234, + "text": "," + }, + { + "id": 29033, + "logprob": -2.2324219, + "text": " Gir" + }, + { + "id": 1622, + "logprob": -0.10809326, + "text": "af" + }, + { + "id": 249, + "logprob": -0.042663574, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.0024776459, + "text": "ron" + }, + { + "id": 12, + "logprob": -1.4277344, + "text": "!" + }, + { + "id": 193, + "logprob": -1.1015625, + "text": "\n" + }, + { + "id": 50, + "logprob": -0.05709839, + "text": "G" + }, + { + "id": 330, + "logprob": -0.13208008, + "text": "ir" + }, + { + "id": 1622, + "logprob": -0.0071487427, + "text": "af" + }, + { + "id": 249, + "logprob": -0.008468628, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.00068998337, + "text": "ron" + }, + { + "id": 37, + "logprob": -0.0074691772, + "text": ":" + } + ], + "seed": null, + "tokens": [ + { + "id": 23090, + "logprob": -1.8251953, + "special": false, + "text": " Hello" + }, + { + "id": 23, + "logprob": -0.3173828, + "special": false, + "text": "," + }, + { + "id": 8156, + "logprob": -0.23803711, + "special": false, + "text": " Daniel" + }, + { + "id": 12, + "logprob": -0.56933594, + "special": false, + "text": "!" + }, + { + "id": 193, + "logprob": -0.61279297, + "special": false, + "text": "\n" + }, + { + "id": 23626, + "logprob": -0.41967773, + "special": false, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -0.0023403168, + "special": false, + "text": ":" + }, + { + "id": 1634, + "logprob": -2.0605469, + "special": false, + "text": " What" + }, + { + "id": 18, + "logprob": -1.5292969, + "special": false, + "text": "'" + }, + { + "id": 94, + "logprob": -0.007904053, + "special": false, + "text": "s" + } + ] + }, + "generated_text": " Hello, Daniel!\nDaniel: What's" +} diff --git a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_all_params.json b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_all_params.json new file mode 100644 index 00000000..cd35186d --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_all_params.json @@ -0,0 +1,98 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 330, + "logprob": null, + "text": "ir" + }, + { + "id": 1622, + "logprob": -7.8125, + "text": "af" + }, + { + "id": 249, + "logprob": -4.5, + "text": "at" + }, + { + "id": 1480, + "logprob": -10.875, + "text": "ron" + }, + { + "id": 37, + "logprob": -3.6875, + "text": ":" + } + ], + "seed": 0, + "tokens": [ + { + "id": 836, + "logprob": -1.265625, + "special": false, + "text": " i" + }, + { + "id": 18, + "logprob": -0.119628906, + "special": false, + "text": "'" + }, + { + "id": 298, + "logprob": -2.265625, + "special": false, + "text": "ve" + }, + { + "id": 650, + "logprob": -0.49804688, + "special": false, + "text": " been" + }, + { + "id": 1241, + "logprob": 0.0, + "special": false, + "text": " using" + }, + { + "id": 334, + "logprob": 0.0, + "special": false, + "text": " it" + }, + { + "id": 312, + "logprob": -1.2421875, + "special": false, + "text": " for" + }, + { + "id": 909, + "logprob": -0.99609375, + "special": false, + "text": " years" + }, + { + "id": 193, + "logprob": -0.30273438, + "special": false, + "text": "\n" + }, + { + "id": 807, + "logprob": -1.078125, + "special": false, + "text": "ik" + } + ] + }, + "generated_text": "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron: i've been using it for years\nik" +} diff --git a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json new file mode 100644 index 00000000..0fb1be75 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json @@ -0,0 +1,1514 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50, + "logprob": null, + "text": "G" + }, + { + "id": 330, + "logprob": -5.71875, + "text": "ir" + }, + { + "id": 1622, + "logprob": -5.625, + "text": "af" + }, + { + "id": 249, + "logprob": -6.53125, + "text": "at" + }, + { + "id": 1480, + "logprob": -8.0625, + "text": "ron" + }, + { + "id": 304, + "logprob": -2.328125, + "text": " is" + }, + { + "id": 23866, + "logprob": -9.625, + "text": " obsessed" + }, + { + "id": 335, + "logprob": -0.048583984, + "text": " with" + }, + { + "id": 26680, + "logprob": -3.984375, + "text": " gir" + }, + { + "id": 1903, + "logprob": -0.076171875, + "text": "aff" + }, + { + "id": 255, + "logprob": -0.0066833496, + "text": "es" + }, + { + "id": 23, + "logprob": -1.546875, + "text": "," + }, + { + "id": 248, + "logprob": -4.34375, + "text": " the" + }, + { + "id": 758, + "logprob": -3.734375, + "text": " most" + }, + { + "id": 21735, + "logprob": -5.125, + "text": " glorious" + }, + { + "id": 5985, + "logprob": -2.078125, + "text": " animal" + }, + { + "id": 313, + "logprob": -1.1953125, + "text": " on" + }, + { + "id": 248, + "logprob": -0.78125, + "text": " the" + }, + { + "id": 1936, + "logprob": -2.390625, + "text": " face" + }, + { + "id": 275, + "logprob": -0.0044555664, + "text": " of" + }, + { + "id": 414, + "logprob": -1.984375, + "text": " this" + }, + { + "id": 6490, + "logprob": -2.03125, + "text": " Earth" + }, + { + "id": 25, + "logprob": -0.28320312, + "text": "." + }, + { + "id": 401, + "logprob": -7.90625, + "text": " G" + }, + { + "id": 6013, + "logprob": -2.265625, + "text": "ira" + }, + { + "id": 694, + "logprob": -0.640625, + "text": "ft" + }, + { + "id": 1480, + "logprob": -0.203125, + "text": "ron" + }, + { + "id": 9369, + "logprob": -4.53125, + "text": " believes" + }, + { + "id": 455, + "logprob": -4.5625, + "text": " all" + }, + { + "id": 599, + "logprob": -2.75, + "text": " other" + }, + { + "id": 5632, + "logprob": -0.21875, + "text": " animals" + }, + { + "id": 362, + "logprob": -0.76171875, + "text": " are" + }, + { + "id": 23981, + "logprob": -4.96875, + "text": " irrelevant" + }, + { + "id": 635, + "logprob": -4.21875, + "text": " when" + }, + { + "id": 4354, + "logprob": -0.51953125, + "text": " compared" + }, + { + "id": 271, + "logprob": -0.103515625, + "text": " to" + }, + { + "id": 248, + "logprob": -0.58984375, + "text": " the" + }, + { + "id": 21735, + "logprob": -3.6875, + "text": " glorious" + }, + { + "id": 64398, + "logprob": -1.8359375, + "text": " majesty" + }, + { + "id": 275, + "logprob": -0.24316406, + "text": " of" + }, + { + "id": 248, + "logprob": -0.3515625, + "text": " the" + }, + { + "id": 26680, + "logprob": -0.24414062, + "text": " gir" + }, + { + "id": 23226, + "logprob": -0.03100586, + "text": "affe" + }, + { + "id": 25, + "logprob": -0.17382812, + "text": "." + }, + { + "id": 193, + "logprob": -1.3984375, + "text": "\n" + }, + { + "id": 23626, + "logprob": -10.0625, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -4.59375, + "text": ":" + }, + { + "id": 23090, + "logprob": -6.9375, + "text": " Hello" + }, + { + "id": 23, + "logprob": -1.0, + "text": "," + }, + { + "id": 29033, + "logprob": -2.21875, + "text": " Gir" + }, + { + "id": 1622, + "logprob": -0.10644531, + "text": "af" + }, + { + "id": 249, + "logprob": -0.041992188, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.0025024414, + "text": "ron" + }, + { + "id": 12, + "logprob": -1.4296875, + "text": "!" + }, + { + "id": 193, + "logprob": -1.1015625, + "text": "\n" + }, + { + "id": 50, + "logprob": -0.05810547, + "text": "G" + }, + { + "id": 330, + "logprob": -0.12597656, + "text": "ir" + }, + { + "id": 1622, + "logprob": -0.007080078, + "text": "af" + }, + { + "id": 249, + "logprob": -0.008300781, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.0006866455, + "text": "ron" + }, + { + "id": 37, + "logprob": -0.0074157715, + "text": ":" + } + ], + "seed": null, + "tokens": [ + { + "id": 23090, + "logprob": -1.8203125, + "special": false, + "text": " Hello" + }, + { + "id": 23, + "logprob": -0.32226562, + "special": false, + "text": "," + }, + { + "id": 8156, + "logprob": -0.23828125, + "special": false, + "text": " Daniel" + }, + { + "id": 12, + "logprob": -0.5859375, + "special": false, + "text": "!" + }, + { + "id": 193, + "logprob": -0.6171875, + "special": false, + "text": "\n" + }, + { + "id": 23626, + "logprob": -0.39648438, + "special": false, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -0.0023345947, + "special": false, + "text": ":" + }, + { + "id": 295, + "logprob": -2.078125, + "special": false, + "text": " I" + }, + { + "id": 18, + "logprob": -1.453125, + "special": false, + "text": "'" + }, + { + "id": 88, + "logprob": -0.47460938, + "special": false, + "text": "m" + } + ] + }, + "generated_text": " Hello, Daniel!\nDaniel: I'm" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50, + "logprob": null, + "text": "G" + }, + { + "id": 330, + "logprob": -5.71875, + "text": "ir" + }, + { + "id": 1622, + "logprob": -5.625, + "text": "af" + }, + { + "id": 249, + "logprob": -6.53125, + "text": "at" + }, + { + "id": 1480, + "logprob": -8.0625, + "text": "ron" + }, + { + "id": 304, + "logprob": -2.328125, + "text": " is" + }, + { + "id": 23866, + "logprob": -9.625, + "text": " obsessed" + }, + { + "id": 335, + "logprob": -0.048583984, + "text": " with" + }, + { + "id": 26680, + "logprob": -3.984375, + "text": " gir" + }, + { + "id": 1903, + "logprob": -0.076171875, + "text": "aff" + }, + { + "id": 255, + "logprob": -0.0066833496, + "text": "es" + }, + { + "id": 23, + "logprob": -1.546875, + "text": "," + }, + { + "id": 248, + "logprob": -4.34375, + "text": " the" + }, + { + "id": 758, + "logprob": -3.734375, + "text": " most" + }, + { + "id": 21735, + "logprob": -5.125, + "text": " glorious" + }, + { + "id": 5985, + "logprob": -2.078125, + "text": " animal" + }, + { + "id": 313, + "logprob": -1.1953125, + "text": " on" + }, + { + "id": 248, + "logprob": -0.78125, + "text": " the" + }, + { + "id": 1936, + "logprob": -2.390625, + "text": " face" + }, + { + "id": 275, + "logprob": -0.0044555664, + "text": " of" + }, + { + "id": 414, + "logprob": -1.984375, + "text": " this" + }, + { + "id": 6490, + "logprob": -2.03125, + "text": " Earth" + }, + { + "id": 25, + "logprob": -0.28320312, + "text": "." + }, + { + "id": 401, + "logprob": -7.90625, + "text": " G" + }, + { + "id": 6013, + "logprob": -2.265625, + "text": "ira" + }, + { + "id": 694, + "logprob": -0.640625, + "text": "ft" + }, + { + "id": 1480, + "logprob": -0.203125, + "text": "ron" + }, + { + "id": 9369, + "logprob": -4.53125, + "text": " believes" + }, + { + "id": 455, + "logprob": -4.5625, + "text": " all" + }, + { + "id": 599, + "logprob": -2.75, + "text": " other" + }, + { + "id": 5632, + "logprob": -0.21875, + "text": " animals" + }, + { + "id": 362, + "logprob": -0.76171875, + "text": " are" + }, + { + "id": 23981, + "logprob": -4.96875, + "text": " irrelevant" + }, + { + "id": 635, + "logprob": -4.21875, + "text": " when" + }, + { + "id": 4354, + "logprob": -0.51953125, + "text": " compared" + }, + { + "id": 271, + "logprob": -0.103515625, + "text": " to" + }, + { + "id": 248, + "logprob": -0.58984375, + "text": " the" + }, + { + "id": 21735, + "logprob": -3.6875, + "text": " glorious" + }, + { + "id": 64398, + "logprob": -1.8359375, + "text": " majesty" + }, + { + "id": 275, + "logprob": -0.24316406, + "text": " of" + }, + { + "id": 248, + "logprob": -0.3515625, + "text": " the" + }, + { + "id": 26680, + "logprob": -0.24414062, + "text": " gir" + }, + { + "id": 23226, + "logprob": -0.03100586, + "text": "affe" + }, + { + "id": 25, + "logprob": -0.17382812, + "text": "." + }, + { + "id": 193, + "logprob": -1.3984375, + "text": "\n" + }, + { + "id": 23626, + "logprob": -10.0625, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -4.59375, + "text": ":" + }, + { + "id": 23090, + "logprob": -6.9375, + "text": " Hello" + }, + { + "id": 23, + "logprob": -1.0, + "text": "," + }, + { + "id": 29033, + "logprob": -2.21875, + "text": " Gir" + }, + { + "id": 1622, + "logprob": -0.10644531, + "text": "af" + }, + { + "id": 249, + "logprob": -0.041992188, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.0025024414, + "text": "ron" + }, + { + "id": 12, + "logprob": -1.4296875, + "text": "!" + }, + { + "id": 193, + "logprob": -1.1015625, + "text": "\n" + }, + { + "id": 50, + "logprob": -0.05810547, + "text": "G" + }, + { + "id": 330, + "logprob": -0.12597656, + "text": "ir" + }, + { + "id": 1622, + "logprob": -0.007080078, + "text": "af" + }, + { + "id": 249, + "logprob": -0.008300781, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.0006866455, + "text": "ron" + }, + { + "id": 37, + "logprob": -0.0074157715, + "text": ":" + } + ], + "seed": null, + "tokens": [ + { + "id": 23090, + "logprob": -1.8203125, + "special": false, + "text": " Hello" + }, + { + "id": 23, + "logprob": -0.32226562, + "special": false, + "text": "," + }, + { + "id": 8156, + "logprob": -0.23828125, + "special": false, + "text": " Daniel" + }, + { + "id": 12, + "logprob": -0.5859375, + "special": false, + "text": "!" + }, + { + "id": 193, + "logprob": -0.6171875, + "special": false, + "text": "\n" + }, + { + "id": 23626, + "logprob": -0.39648438, + "special": false, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -0.0023345947, + "special": false, + "text": ":" + }, + { + "id": 295, + "logprob": -2.078125, + "special": false, + "text": " I" + }, + { + "id": 18, + "logprob": -1.453125, + "special": false, + "text": "'" + }, + { + "id": 88, + "logprob": -0.47460938, + "special": false, + "text": "m" + } + ] + }, + "generated_text": " Hello, Daniel!\nDaniel: I'm" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50, + "logprob": null, + "text": "G" + }, + { + "id": 330, + "logprob": -5.71875, + "text": "ir" + }, + { + "id": 1622, + "logprob": -5.625, + "text": "af" + }, + { + "id": 249, + "logprob": -6.53125, + "text": "at" + }, + { + "id": 1480, + "logprob": -8.0625, + "text": "ron" + }, + { + "id": 304, + "logprob": -2.328125, + "text": " is" + }, + { + "id": 23866, + "logprob": -9.625, + "text": " obsessed" + }, + { + "id": 335, + "logprob": -0.048583984, + "text": " with" + }, + { + "id": 26680, + "logprob": -3.984375, + "text": " gir" + }, + { + "id": 1903, + "logprob": -0.076171875, + "text": "aff" + }, + { + "id": 255, + "logprob": -0.0066833496, + "text": "es" + }, + { + "id": 23, + "logprob": -1.546875, + "text": "," + }, + { + "id": 248, + "logprob": -4.34375, + "text": " the" + }, + { + "id": 758, + "logprob": -3.734375, + "text": " most" + }, + { + "id": 21735, + "logprob": -5.125, + "text": " glorious" + }, + { + "id": 5985, + "logprob": -2.078125, + "text": " animal" + }, + { + "id": 313, + "logprob": -1.1953125, + "text": " on" + }, + { + "id": 248, + "logprob": -0.78125, + "text": " the" + }, + { + "id": 1936, + "logprob": -2.390625, + "text": " face" + }, + { + "id": 275, + "logprob": -0.0044555664, + "text": " of" + }, + { + "id": 414, + "logprob": -1.984375, + "text": " this" + }, + { + "id": 6490, + "logprob": -2.03125, + "text": " Earth" + }, + { + "id": 25, + "logprob": -0.28320312, + "text": "." + }, + { + "id": 401, + "logprob": -7.90625, + "text": " G" + }, + { + "id": 6013, + "logprob": -2.265625, + "text": "ira" + }, + { + "id": 694, + "logprob": -0.640625, + "text": "ft" + }, + { + "id": 1480, + "logprob": -0.203125, + "text": "ron" + }, + { + "id": 9369, + "logprob": -4.53125, + "text": " believes" + }, + { + "id": 455, + "logprob": -4.5625, + "text": " all" + }, + { + "id": 599, + "logprob": -2.75, + "text": " other" + }, + { + "id": 5632, + "logprob": -0.21875, + "text": " animals" + }, + { + "id": 362, + "logprob": -0.76171875, + "text": " are" + }, + { + "id": 23981, + "logprob": -4.96875, + "text": " irrelevant" + }, + { + "id": 635, + "logprob": -4.21875, + "text": " when" + }, + { + "id": 4354, + "logprob": -0.51953125, + "text": " compared" + }, + { + "id": 271, + "logprob": -0.103515625, + "text": " to" + }, + { + "id": 248, + "logprob": -0.58984375, + "text": " the" + }, + { + "id": 21735, + "logprob": -3.6875, + "text": " glorious" + }, + { + "id": 64398, + "logprob": -1.8359375, + "text": " majesty" + }, + { + "id": 275, + "logprob": -0.24316406, + "text": " of" + }, + { + "id": 248, + "logprob": -0.3515625, + "text": " the" + }, + { + "id": 26680, + "logprob": -0.24414062, + "text": " gir" + }, + { + "id": 23226, + "logprob": -0.03100586, + "text": "affe" + }, + { + "id": 25, + "logprob": -0.17382812, + "text": "." + }, + { + "id": 193, + "logprob": -1.3984375, + "text": "\n" + }, + { + "id": 23626, + "logprob": -10.0625, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -4.59375, + "text": ":" + }, + { + "id": 23090, + "logprob": -6.9375, + "text": " Hello" + }, + { + "id": 23, + "logprob": -1.0, + "text": "," + }, + { + "id": 29033, + "logprob": -2.21875, + "text": " Gir" + }, + { + "id": 1622, + "logprob": -0.10644531, + "text": "af" + }, + { + "id": 249, + "logprob": -0.041992188, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.0025024414, + "text": "ron" + }, + { + "id": 12, + "logprob": -1.4296875, + "text": "!" + }, + { + "id": 193, + "logprob": -1.1015625, + "text": "\n" + }, + { + "id": 50, + "logprob": -0.05810547, + "text": "G" + }, + { + "id": 330, + "logprob": -0.12597656, + "text": "ir" + }, + { + "id": 1622, + "logprob": -0.007080078, + "text": "af" + }, + { + "id": 249, + "logprob": -0.008300781, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.0006866455, + "text": "ron" + }, + { + "id": 37, + "logprob": -0.0074157715, + "text": ":" + } + ], + "seed": null, + "tokens": [ + { + "id": 23090, + "logprob": -1.8203125, + "special": false, + "text": " Hello" + }, + { + "id": 23, + "logprob": -0.32226562, + "special": false, + "text": "," + }, + { + "id": 8156, + "logprob": -0.23828125, + "special": false, + "text": " Daniel" + }, + { + "id": 12, + "logprob": -0.5859375, + "special": false, + "text": "!" + }, + { + "id": 193, + "logprob": -0.6171875, + "special": false, + "text": "\n" + }, + { + "id": 23626, + "logprob": -0.39648438, + "special": false, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -0.0023345947, + "special": false, + "text": ":" + }, + { + "id": 295, + "logprob": -2.078125, + "special": false, + "text": " I" + }, + { + "id": 18, + "logprob": -1.453125, + "special": false, + "text": "'" + }, + { + "id": 88, + "logprob": -0.47460938, + "special": false, + "text": "m" + } + ] + }, + "generated_text": " Hello, Daniel!\nDaniel: I'm" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50, + "logprob": null, + "text": "G" + }, + { + "id": 330, + "logprob": -5.71875, + "text": "ir" + }, + { + "id": 1622, + "logprob": -5.625, + "text": "af" + }, + { + "id": 249, + "logprob": -6.53125, + "text": "at" + }, + { + "id": 1480, + "logprob": -8.0625, + "text": "ron" + }, + { + "id": 304, + "logprob": -2.328125, + "text": " is" + }, + { + "id": 23866, + "logprob": -9.625, + "text": " obsessed" + }, + { + "id": 335, + "logprob": -0.048583984, + "text": " with" + }, + { + "id": 26680, + "logprob": -3.984375, + "text": " gir" + }, + { + "id": 1903, + "logprob": -0.076171875, + "text": "aff" + }, + { + "id": 255, + "logprob": -0.0066833496, + "text": "es" + }, + { + "id": 23, + "logprob": -1.546875, + "text": "," + }, + { + "id": 248, + "logprob": -4.34375, + "text": " the" + }, + { + "id": 758, + "logprob": -3.734375, + "text": " most" + }, + { + "id": 21735, + "logprob": -5.125, + "text": " glorious" + }, + { + "id": 5985, + "logprob": -2.078125, + "text": " animal" + }, + { + "id": 313, + "logprob": -1.1953125, + "text": " on" + }, + { + "id": 248, + "logprob": -0.78125, + "text": " the" + }, + { + "id": 1936, + "logprob": -2.390625, + "text": " face" + }, + { + "id": 275, + "logprob": -0.0044555664, + "text": " of" + }, + { + "id": 414, + "logprob": -1.984375, + "text": " this" + }, + { + "id": 6490, + "logprob": -2.03125, + "text": " Earth" + }, + { + "id": 25, + "logprob": -0.28320312, + "text": "." + }, + { + "id": 401, + "logprob": -7.90625, + "text": " G" + }, + { + "id": 6013, + "logprob": -2.265625, + "text": "ira" + }, + { + "id": 694, + "logprob": -0.640625, + "text": "ft" + }, + { + "id": 1480, + "logprob": -0.203125, + "text": "ron" + }, + { + "id": 9369, + "logprob": -4.53125, + "text": " believes" + }, + { + "id": 455, + "logprob": -4.5625, + "text": " all" + }, + { + "id": 599, + "logprob": -2.75, + "text": " other" + }, + { + "id": 5632, + "logprob": -0.21875, + "text": " animals" + }, + { + "id": 362, + "logprob": -0.76171875, + "text": " are" + }, + { + "id": 23981, + "logprob": -4.96875, + "text": " irrelevant" + }, + { + "id": 635, + "logprob": -4.21875, + "text": " when" + }, + { + "id": 4354, + "logprob": -0.51953125, + "text": " compared" + }, + { + "id": 271, + "logprob": -0.103515625, + "text": " to" + }, + { + "id": 248, + "logprob": -0.58984375, + "text": " the" + }, + { + "id": 21735, + "logprob": -3.6875, + "text": " glorious" + }, + { + "id": 64398, + "logprob": -1.8359375, + "text": " majesty" + }, + { + "id": 275, + "logprob": -0.24316406, + "text": " of" + }, + { + "id": 248, + "logprob": -0.3515625, + "text": " the" + }, + { + "id": 26680, + "logprob": -0.24414062, + "text": " gir" + }, + { + "id": 23226, + "logprob": -0.03100586, + "text": "affe" + }, + { + "id": 25, + "logprob": -0.17382812, + "text": "." + }, + { + "id": 193, + "logprob": -1.3984375, + "text": "\n" + }, + { + "id": 23626, + "logprob": -10.0625, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -4.59375, + "text": ":" + }, + { + "id": 23090, + "logprob": -6.9375, + "text": " Hello" + }, + { + "id": 23, + "logprob": -1.0, + "text": "," + }, + { + "id": 29033, + "logprob": -2.21875, + "text": " Gir" + }, + { + "id": 1622, + "logprob": -0.10644531, + "text": "af" + }, + { + "id": 249, + "logprob": -0.041992188, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.0025024414, + "text": "ron" + }, + { + "id": 12, + "logprob": -1.4296875, + "text": "!" + }, + { + "id": 193, + "logprob": -1.1015625, + "text": "\n" + }, + { + "id": 50, + "logprob": -0.05810547, + "text": "G" + }, + { + "id": 330, + "logprob": -0.12597656, + "text": "ir" + }, + { + "id": 1622, + "logprob": -0.007080078, + "text": "af" + }, + { + "id": 249, + "logprob": -0.008300781, + "text": "at" + }, + { + "id": 1480, + "logprob": -0.0006866455, + "text": "ron" + }, + { + "id": 37, + "logprob": -0.0074157715, + "text": ":" + } + ], + "seed": null, + "tokens": [ + { + "id": 23090, + "logprob": -1.8203125, + "special": false, + "text": " Hello" + }, + { + "id": 23, + "logprob": -0.32226562, + "special": false, + "text": "," + }, + { + "id": 8156, + "logprob": -0.23828125, + "special": false, + "text": " Daniel" + }, + { + "id": 12, + "logprob": -0.5859375, + "special": false, + "text": "!" + }, + { + "id": 193, + "logprob": -0.6171875, + "special": false, + "text": "\n" + }, + { + "id": 23626, + "logprob": -0.39648438, + "special": false, + "text": "Daniel" + }, + { + "id": 37, + "logprob": -0.0023345947, + "special": false, + "text": ":" + }, + { + "id": 295, + "logprob": -2.078125, + "special": false, + "text": " I" + }, + { + "id": 18, + "logprob": -1.453125, + "special": false, + "text": "'" + }, + { + "id": 88, + "logprob": -0.47460938, + "special": false, + "text": "m" + } + ] + }, + "generated_text": " Hello, Daniel!\nDaniel: I'm" + } +] diff --git a/integration-tests/models/test_flash_falcon.py b/integration-tests/models/test_flash_falcon.py new file mode 100644 index 00000000..ce27731d --- /dev/null +++ b/integration-tests/models/test_flash_falcon.py @@ -0,0 +1,63 @@ +import pytest + + +@pytest.fixture(scope="module") +def flash_falcon_handle(launcher): + with launcher("tiiuae/falcon-7b", trust_remote_code=True) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def flash_falcon(flash_falcon_handle): + await flash_falcon_handle.health(120) + return flash_falcon_handle.client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_falcon(flash_falcon, response_snapshot): + response = await flash_falcon.generate( + "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron:", + max_new_tokens=10, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_falcon_all_params(flash_falcon, response_snapshot): + response = await flash_falcon.generate( + "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron:", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + stop_sequences=["test"], + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_falcon_load(flash_falcon, generate_load, response_snapshot): + responses = await generate_load( + flash_falcon, + "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron:", + max_new_tokens=10, + n=4, + ) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) + + assert responses == response_snapshot diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 46a4563b..4adf1381 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -10,6 +10,7 @@ from text_generation_server.models.flash_causal_lm import FlashCausalLM from text_generation_server.models.bloom import BLOOM, BLOOMSharded from text_generation_server.models.seq2seq_lm import Seq2SeqLM +from text_generation_server.models.rw import RW from text_generation_server.models.opt import OPT, OPTSharded from text_generation_server.models.galactica import Galactica, GalacticaSharded from text_generation_server.models.santacoder import SantaCoder @@ -30,6 +31,7 @@ ) from text_generation_server.models.flash_neox import FlashNeoX, FlashNeoXSharded + from text_generation_server.models.flash_rw import FlashRW, FlashRWSharded from text_generation_server.models.flash_llama import ( FlashLlama, FlashLlamaSharded, @@ -68,6 +70,8 @@ if FLASH_ATTENTION: __all__.append(FlashNeoX) __all__.append(FlashNeoXSharded) + __all__.append(FlashRW) + __all__.append(FlashRWSharded) __all__.append(FlashSantacoder) __all__.append(FlashSantacoderSharded) __all__.append(FlashLlama) @@ -194,6 +198,39 @@ def get_model( trust_remote_code=trust_remote_code, ) + if model_type in ["RefinedWeb", "RefinedWebModel"]: + if sharded: + if FLASH_ATTENTION: + if config.alibi or ( + config.model_type == "RefinedWebModel" + and config.n_head_kv != config.n_head + ): + raise NotImplementedError("sharded is not supported for this model") + return FlashRWSharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) + raise NotImplementedError( + FLASH_ATT_ERROR_MESSAGE.format(f"Sharded RefinedWeb") + ) + else: + if FLASH_ATTENTION and not config.alibi: + return FlashRW( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) + else: + return RW( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) + if model_type == "llama": if sharded: if FLASH_ATTENTION: diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 54670b79..2dcb6ed8 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -134,20 +134,23 @@ def forward( ): qkv = self.query_key_value(hidden_states) qkv = qkv.view(-1, 3, self.num_heads, self.head_size) - qkv_rot = self.rotary_emb(qkv, cos, sin) + + # Inplace rotary + self.rotary_emb(qkv[:, 0], cos, sin) + self.rotary_emb(qkv[:, 1], cos, sin) # Prefill if layer_past_present_indices is None: # Copy to layer past - layer_past[...] = qkv_rot[:, 1:] + layer_past[...] = qkv[:, 1:] # output - attn_output = torch.empty_like(qkv_rot[:, 0]) + attn_output = torch.empty_like(qkv[:, 0]) # flash attention flash_attn_cuda.fwd( - qkv_rot[:, 0], - qkv_rot[:, 1], - qkv_rot[:, 2], + qkv[:, 0], + qkv[:, 1], + qkv[:, 2], attn_output, cu_seqlens, cu_seqlens, @@ -163,9 +166,9 @@ def forward( ) # Decode else: - query = qkv_rot[:, 0] + query = qkv[:, 0] # Add present to the layer_past tensor at the correct indices - layer_past[layer_past_present_indices] = qkv_rot[:, 1:] + layer_past[layer_past_present_indices] = qkv[:, 1:] # output attn_output = torch.empty_like(query) diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index b7834157..26e21753 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -101,20 +101,23 @@ def forward( ): qkv = self.query_key_value(hidden_states) qkv = qkv.view(-1, 3, self.num_heads, self.head_size) - qkv_rot = self.rotary_emb(qkv, cos, sin) + + # Inplace rotary + self.rotary_emb(qkv[:, 0], cos, sin) + self.rotary_emb(qkv[:, 1], cos, sin) # Prefill if layer_past_present_indices is None: # Copy to layer past - layer_past[...] = qkv_rot[:, 1:] + layer_past[...] = qkv[:, 1:] # output - attn_output = torch.empty_like(qkv_rot[:, 0]) + attn_output = torch.empty_like(qkv[:, 0]) # flash attention flash_attn_cuda.fwd( - qkv_rot[:, 0], - qkv_rot[:, 1], - qkv_rot[:, 2], + qkv[:, 0], + qkv[:, 1], + qkv[:, 2], attn_output, cu_seqlens, cu_seqlens, @@ -130,9 +133,9 @@ def forward( ) # Decode else: - query = qkv_rot[:, 0] + query = qkv[:, 0] # Add present to the layer_past tensor at the correct indices - layer_past[layer_past_present_indices] = qkv_rot[:, 1:] + layer_past[layer_past_present_indices] = qkv[:, 1:] # output attn_output = torch.empty_like(query) diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py new file mode 100644 index 00000000..545da26a --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -0,0 +1,774 @@ +import os + +import torch +import torch.distributed + +from torch import nn +from transformers.modeling_utils import PreTrainedModel +from transformers.configuration_utils import PretrainedConfig +from typing import Optional + +# Flash attention imports +import flash_attn_cuda + +from text_generation_server.utils.layers import ( + FastLinear, + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, + FastLayerNorm, + PositionRotaryEmbedding, +) + + +class RWConfig(PretrainedConfig): + attribute_map = { + "num_hidden_layers": "n_layer", + "num_attention_heads": "n_head", + } + + def __init__( + self, + model_type="RefinedWeb", + vocab_size=250880, + hidden_size=64, + n_layer=2, + n_head=8, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + use_cache=True, + bos_token_id=1, + eos_token_id=2, + hidden_dropout=0.0, + attention_dropout=0.0, + n_head_kv=None, + multi_query=False, + alibi=False, + bias=False, + parallel_attn=False, + **kwargs, + ): + if alibi: + raise NotImplementedError( + "alibi is not supported by this version of the model" + ) + + self.model_type = model_type + self.alibi = False + self.rotary = True + + self.vocab_size = vocab_size + # Backward compatibility with n_embed kwarg + n_embed = kwargs.pop("n_embed", None) + self.hidden_size = hidden_size if n_embed is None else n_embed + self.n_layer = n_layer + self.n_head = n_head + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.use_cache = use_cache + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + self.bias = bias + self.parallel_attn = parallel_attn + + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + + if n_head_kv is not None: + self.n_head_kv = n_head_kv + else: + self.n_head_kv = 1 if multi_query else n_head + + super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) + + +class FlashRWAttention(torch.nn.Module): + def __init__( + self, + num_heads, + num_heads_kv, + hidden_size, + bias, + process_group=None, + reduce=True, + ): + super().__init__() + self.num_heads = num_heads + self.num_heads_kv = num_heads_kv + self.hidden_size = hidden_size + self.head_size = hidden_size // num_heads + + self.rotary_emb = PositionRotaryEmbedding(self.head_size, base=10000) + self.softmax_scale = self.head_size ** (-0.5) + + if process_group is None: + self.query_key_value = FastLinear( + hidden_size, + self.head_size * (self.num_heads + 2 * self.num_heads_kv), + bias=bias, + ) + self.dense = FastLinear(hidden_size, hidden_size, bias=bias) + else: + self.query_key_value = TensorParallelColumnLinear( + hidden_size, + self.head_size * (self.num_heads + 2 * self.num_heads_kv), + bias=bias, + process_group=process_group, + ) + self.dense = TensorParallelRowLinear( + hidden_size, + hidden_size, + bias=bias, + process_group=process_group, + reduce=reduce, + ) + self.num_heads = self.num_heads // process_group.size() + + def forward( + self, + hidden_states, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + qkv = self.query_key_value(hidden_states) + + # Split query from key_value + query, kv = qkv.split( + [self.head_size * self.num_heads, 2 * self.head_size * self.num_heads_kv], + dim=1, + ) + + # Prepare query and key_value for indexing + query = query.view(-1, self.num_heads, self.head_size) + kv = kv.view(-1, 2, self.num_heads_kv, self.head_size) + + # Inplace rotary + self.rotary_emb(query, cos, sin) + self.rotary_emb(kv[:, 0], cos, sin) + + # Prefill + if layer_past_present_indices is None: + # Copy to layer past + layer_past[...] = kv + # Expand to query shape + kv = kv.expand(-1, 2, self.num_heads, self.head_size) + + # output + attn_output = torch.empty_like(query) + # flash attention + flash_attn_cuda.fwd( + query, + kv[:, 0], + kv[:, 1], + attn_output, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + 0.0, + self.softmax_scale, + False, + True, + False, + 0, + None, + ) + # Decode + else: + # Add present to the layer_past tensor at the correct indices + layer_past[layer_past_present_indices] = kv + # Expand to query shape + kv = layer_past.expand(-1, 2, self.num_heads, self.head_size) + + # output + attn_output = torch.empty_like(query) + # flash attention + flash_attn_cuda.fwd( + query, + kv[:, 0], + kv[:, 1], + attn_output, + cu_seqlens_q, + cu_seqlens, + 1, + max_s, + 0.0, + self.softmax_scale, + False, + False, + False, + 0, + None, + ) + + return self.dense(attn_output.view(-1, self.num_heads * self.head_size)) + + +class FlashRWLargeAttention(torch.nn.Module): + def __init__( + self, + num_heads, + num_heads_kv, + hidden_size, + bias, + process_group=None, + reduce=True, + ): + super().__init__() + + self.hidden_size = hidden_size + self.head_size = hidden_size // num_heads + + self.rotary_emb = PositionRotaryEmbedding(self.head_size, base=10000) + self.softmax_scale = self.head_size ** (-0.5) + + self.num_groups = num_heads // (num_heads_kv * 2) + self.num_heads = num_heads // self.num_groups + self.num_heads_kv = num_heads_kv // self.num_groups + + if process_group is None: + self.query_key_value = FastLinear( + hidden_size, + self.num_groups + * self.head_size + * (self.num_heads + 2 * self.num_heads_kv), + bias=bias, + ) + self.dense = FastLinear(hidden_size, hidden_size, bias=bias) + else: + if process_group.size() > self.num_groups: + raise NotImplementedError( + f"Tensor Parallelism is not implemented for world_size > n groups" + ) + + self.query_key_value = TensorParallelColumnLinear( + hidden_size, + self.num_groups + * self.head_size + * (self.num_heads + 2 * self.num_heads_kv), + bias=bias, + process_group=process_group, + ) + self.dense = TensorParallelRowLinear( + hidden_size, + hidden_size, + bias=bias, + process_group=process_group, + reduce=reduce, + ) + + self.num_groups = self.num_groups // process_group.size() + + def forward( + self, + hidden_states, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + qkv = self.query_key_value(hidden_states) + qkv = qkv.view(-1, self.num_groups, self.num_heads + 2, self.head_size) + + # Split on group dimension + query, kv = qkv.split( + [self.num_heads, 2], + dim=2, + ) + # Merge groups and heads + query = query.reshape(-1, self.num_groups * self.num_heads, self.head_size) + + # Inplace rotary + self.rotary_emb(query, cos, sin) + self.rotary_emb(kv[:, :, 0], cos, sin) + + # Prefill + if layer_past_present_indices is None: + # Copy to layer past + layer_past[...] = kv + # Expand to query shape + kv = ( + kv.unsqueeze(2) + .expand(-1, self.num_groups, self.num_heads, 2, self.head_size) + .reshape(-1, self.num_groups * self.num_heads, 2, self.head_size) + ) + + # output + attn_output = torch.empty_like(query) + # flash attention + flash_attn_cuda.fwd( + query, + kv[:, :, 0], + kv[:, :, 1], + attn_output, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + 0.0, + self.softmax_scale, + False, + True, + False, + 0, + None, + ) + # Decode + else: + # Add present to the layer_past tensor at the correct indices + layer_past[layer_past_present_indices] = kv + # Expand to query shape + kv = ( + layer_past.unsqueeze(2) + .expand(-1, self.num_groups, self.num_heads, 2, self.head_size) + .reshape(-1, self.num_groups * self.num_heads, 2, self.head_size) + ) + + # output + attn_output = torch.empty_like(query) + # flash attention + flash_attn_cuda.fwd( + query, + kv[:, :, 0], + kv[:, :, 1], + attn_output, + cu_seqlens_q, + cu_seqlens, + 1, + max_s, + 0.0, + self.softmax_scale, + False, + False, + False, + 0, + None, + ) + + return self.dense( + attn_output.view(-1, self.num_groups * self.num_heads * self.head_size) + ) + + +class FlashMLP(nn.Module): + def __init__(self, hidden_size, bias, process_group=None, reduce=True): + super().__init__() + self.act = torch.nn.functional.gelu + + if process_group is None: + self.dense_h_to_4h = FastLinear(hidden_size, 4 * hidden_size, bias=bias) + self.dense_4h_to_h = FastLinear(4 * hidden_size, hidden_size, bias=bias) + else: + self.dense_h_to_4h = TensorParallelColumnLinear( + hidden_size, + 4 * hidden_size, + bias=bias, + process_group=process_group, + ) + self.dense_4h_to_h = TensorParallelRowLinear( + 4 * hidden_size, + hidden_size, + bias=bias, + process_group=process_group, + reduce=reduce, + ) + self.process_group = process_group + + def forward(self, hidden_states): + hidden_states = self.dense_h_to_4h(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.dense_4h_to_h(hidden_states) + return hidden_states + + +class FlashRWLayer(nn.Module): + def __init__( + self, + num_heads, + num_heads_kv, + hidden_size, + bias, + layer_norm_eps, + parallel_attn, + process_group=None, + ): + super().__init__() + + self.parallel_attn = parallel_attn + + self.input_layernorm = FastLayerNorm(hidden_size, eps=layer_norm_eps) + self.self_attention = FlashRWAttention( + num_heads, + num_heads_kv, + hidden_size, + bias, + process_group=process_group, + reduce=False, + ) + self.post_attention_layernorm = ( + FastLayerNorm(hidden_size, eps=layer_norm_eps) + if not parallel_attn + else None + ) + + self.mlp = FlashMLP( + hidden_size, bias, process_group=process_group, reduce=False + ) + + self.process_group = process_group + + def forward( + self, + hidden_states, + residual, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + if self.parallel_attn: + ln_hidden_states, residual = self.input_layernorm(hidden_states, residual) + + attn_output = self.self_attention( + ln_hidden_states, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ) + + mlp_output = self.mlp(ln_hidden_states) + intermediate = mlp_output + attn_output + + # Only reduce once and after the addition instead of once per layer + if self.process_group is not None: + torch.distributed.all_reduce(intermediate, group=self.process_group) + + return intermediate, residual + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + + hidden_states = self.self_attention( + hidden_states, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ) + + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual + ) + + mlp_output = self.mlp(hidden_states) + + return mlp_output, residual + + +class FlashRWLargeLayer(nn.Module): + def __init__( + self, + num_heads, + num_heads_kv, + hidden_size, + bias, + layer_norm_eps, + process_group=None, + ): + super().__init__() + self.ln_attn = FastLayerNorm(hidden_size, eps=layer_norm_eps) + self.ln_mlp = FastLayerNorm(hidden_size, eps=layer_norm_eps) + + self.self_attention = FlashRWLargeAttention( + num_heads, + num_heads_kv, + hidden_size, + bias, + process_group=process_group, + reduce=False, + ) + + self.mlp = FlashMLP( + hidden_size, bias, process_group=process_group, reduce=False + ) + + self.process_group = process_group + + def forward( + self, + hidden_states, + residual, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ): + ln_attn, residual = self.ln_attn(hidden_states, residual) + ln_mlp, _ = self.ln_mlp(residual) + + # Self attention. + attn_output = self.self_attention( + ln_attn, + cos, + sin, + cu_seqlens, + max_s, + layer_past, + layer_past_present_indices, + cu_seqlens_q, + ) + + # MLP. + mlp_output = self.mlp(ln_mlp) + + intermediate = attn_output + mlp_output + + # Only reduce once and after the addition instead of once per layer + if self.process_group is not None: + torch.distributed.all_reduce(intermediate, group=self.process_group) + + return intermediate, residual + + +class FlashRWPreTrainedModel(PreTrainedModel): + config_class = RWConfig + + +class FlashRWModel(FlashRWPreTrainedModel): + def __init__(self, config, process_group=None): + super().__init__(config) + self.config = config + + self.tp_embeddings = False + if process_group is not None: + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + if config.vocab_size % self.tp_world_size == 0: + self.tp_embeddings = True + + if self.tp_embeddings: + self.word_embeddings = TensorParallelEmbedding( + config.vocab_size, config.hidden_size, process_group=process_group + ) + else: + self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size) + + if config.model_type == "RefinedWebModel": + self.h = nn.ModuleList( + [ + FlashRWLayer( + config.n_head, + config.n_head_kv, + config.hidden_size, + config.bias, + config.layer_norm_epsilon, + config.parallel_attn, + process_group, + ) + for _ in range(config.num_hidden_layers) + ] + ) + self.cache_size = ( + 2, + self.h[0].self_attention.num_heads_kv, + self.h[0].self_attention.head_size, + ) + elif config.model_type == "RefinedWeb": + self.h = nn.ModuleList( + [ + FlashRWLargeLayer( + config.n_head, + config.n_head_kv, + config.hidden_size, + config.bias, + config.layer_norm_epsilon, + process_group, + ) + for _ in range(config.num_hidden_layers) + ] + ) + self.cache_size = ( + self.h[0].self_attention.num_groups, + 2, + self.h[0].self_attention.head_size, + ) + else: + raise NotImplementedError( + f"model_type {config.model_type} is not supported." + ) + + self.ln_f = FastLayerNorm(config.hidden_size, eps=config.layer_norm_epsilon) + + self.head_size = self.h[0].self_attention.head_size + + def post_load_weights(self, quantize: Optional[str] = None): + if isinstance(self.word_embeddings, TensorParallelEmbedding): + self.word_embeddings.add_null_idx() + for layer in self.h: + layer: FlashRWLayer + layer.self_attention.query_key_value.prepare_weights(quantize) + layer.self_attention.dense.prepare_weights(quantize) + layer.mlp.dense_h_to_4h.prepare_weights(quantize) + layer.mlp.dense_4h_to_h.prepare_weights(quantize) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + # Pop here as we will replace the layer in our own logic and don't want from_pretrained + # to do it for us + load_in_8bit = kwargs.pop("load_in_8bit", False) + model = super(FlashRWModel, cls).from_pretrained( + pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs + ) + + model.post_load_weights("bitsandbytes" if load_in_8bit else None) + return model + + def forward( + self, + input_ids, + position_ids, + cu_seqlens, + cu_seqlens_q, + max_s, + past_key_values=None, + pre_allocate_past_size: Optional[int] = None, + ): + hidden_states = self.word_embeddings(input_ids) + + # Prefill + if past_key_values is None: + # Create past tensor + past_key_values = hidden_states.new_empty( + ( + len(self.h), + len(hidden_states) + if pre_allocate_past_size is None + else pre_allocate_past_size, + *self.cache_size, + ) + ) + layer_past_present_indices = None + slice_past_index = len(hidden_states) + # Decode + else: + # Create indices from cumulative sequence lengths + layer_past_present_indices = cu_seqlens[1:] - 1 + slice_past_index = None + + # Get rotary cos and sin for this forward + # Avoid to index in each layer + cos, sin = self.h[0].self_attention.rotary_emb.get_cos_sin( + position_ids, max_s, hidden_states.dtype + ) + + residual = None + for i, layer in enumerate(self.h): + # We added padding that we now need to slice + layer_past_key_values = ( + past_key_values[i] + if slice_past_index is None + else past_key_values[i, :slice_past_index] + ) + + hidden_states, residual = layer( + hidden_states, + residual, + cos, + sin, + cu_seqlens, + max_s, + layer_past_key_values, + layer_past_present_indices, + cu_seqlens_q, + ) + + hidden_states, _ = self.ln_f(hidden_states, residual) + + return hidden_states, past_key_values + + +class FlashRWForCausalLM(FlashRWPreTrainedModel): + def __init__(self, config, process_group=None): + super().__init__(config) + + self.process_group = process_group + if self.process_group is not None: + self.world_size = self.process_group.size() + else: + self.world_size = 1 + + self.transformer = FlashRWModel(config, process_group) + + if self.transformer.tp_embeddings: + self.lm_head = FastLinear( + config.hidden_size, + config.vocab_size // process_group.size(), + bias=False, + ) + else: + self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) + + def post_load_weights(self, quantize: Optional[str] = None): + self.transformer.post_load_weights(quantize) + self.lm_head.prepare_weights() + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + # Pop here as we will replace the layer in our own logic and don't want from_pretrained + # to do it for us + load_in_8bit = kwargs.pop("load_in_8bit", False) + model = super(FlashRWForCausalLM, cls).from_pretrained( + pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs + ) + model.post_load_weights("bitsandbytes" if load_in_8bit else None) + return model + + def forward( + self, + input_ids, + position_ids, + cu_seqlens, + cu_seqlens_q, + max_s, + past_key_values: Optional[torch.Tensor] = None, + pre_allocate_past_size: Optional[int] = None, + ): + hidden_states, present = self.transformer( + input_ids, + position_ids, + cu_seqlens, + cu_seqlens_q, + max_s, + past_key_values, + pre_allocate_past_size, + ) + logits = self.lm_head(hidden_states) + + if self.transformer.tp_embeddings: + # Logits are sharded, so we need to gather them + world_logits = [torch.empty_like(logits) for _ in range(self.world_size)] + torch.distributed.all_gather(world_logits, logits, group=self.process_group) + world_logits = torch.cat(world_logits, dim=1) + + return world_logits, present + return logits, present diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py new file mode 100644 index 00000000..44915ff5 --- /dev/null +++ b/server/text_generation_server/models/flash_rw.py @@ -0,0 +1,244 @@ +import torch +import torch.distributed + +from pathlib import Path +from accelerate import init_empty_weights +from opentelemetry import trace +from safetensors import safe_open +from transformers import AutoTokenizer, AutoConfig +from typing import Optional, List + +from text_generation_server.models import FlashCausalLM +from text_generation_server.models.custom_modeling.flash_rw_modeling import ( + RWConfig, + FlashRWForCausalLM, + TensorParallelEmbedding, + TensorParallelRowLinear, + TensorParallelColumnLinear, +) +from text_generation_server.utils import ( + initialize_torch_distributed, + weight_files, + download_weights, + weight_hub_files, + LocalEntryNotFoundError, +) + +tracer = trace.get_tracer(__name__) + + +class FlashRW(FlashCausalLM): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + trust_remote_code: bool = False, + ): + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.bfloat16 + else: + raise NotImplementedError("RW is only available on GPU") + + tokenizer = AutoTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + + config = RWConfig.from_pretrained( + model_id, + revision=revision, + ) + + # We do not use from_pretrained as it is too slow + try: + filenames = weight_files(model_id, revision, ".bin") + # Local files not found + except LocalEntryNotFoundError: + hub_files = weight_hub_files(model_id, revision, ".bin") + filenames = download_weights(hub_files, model_id, revision) + + with init_empty_weights(): + model = FlashRWForCausalLM(config) + + self.load_weights( + model, + filenames, + quantize, + device, + dtype, + ) + + super(FlashCausalLM, self).__init__( + model=model.to(device), + tokenizer=tokenizer, + requires_padding=False, + dtype=dtype, + device=device, + ) + + @staticmethod + def load_weights( + model: FlashRWForCausalLM, + filenames: List[Path], + quantize: Optional[str], + device: torch.device, + dtype: torch.dtype, + ): + for filename in filenames: + state_dict = torch.load(filename, map_location="cpu") + for key, value in state_dict.items(): + value = value.to(device if quantize is None else "cpu").to(dtype) + + module_name, param_name = key.rsplit(".", 1) + module = model.get_submodule(module_name) + + try: + current_parameter_tensor = module._parameters[param_name] + if current_parameter_tensor.shape != value.shape: + raise ValueError( + f"Name {key} -- Current {current_parameter_tensor.shape} and got {value.shape}" + ) + module._parameters[param_name] = value + except KeyError: + module._buffers[param_name] = value + + del value + + torch.cuda.empty_cache() + model.post_load_weights(quantize) + + +class FlashRWSharded(FlashRW): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + trust_remote_code: bool = False, + ): + self.process_group, rank, world_size = initialize_torch_distributed() + if torch.cuda.is_available(): + device = torch.device(f"cuda:{rank}") + dtype = torch.bfloat16 + else: + raise NotImplementedError("FlashRW is only available on GPU") + + tokenizer = AutoTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + + config = RWConfig.from_pretrained( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + + with init_empty_weights(): + model = FlashRWForCausalLM(config, self.process_group) + + torch.distributed.barrier(group=self.process_group) + self.load_weights( + model, + filenames, + quantize=quantize, + device=device, + dtype=dtype, + rank=rank, + world_size=world_size, + ) + torch.distributed.barrier(group=self.process_group) + super(FlashCausalLM, self).__init__( + model=model.to(device), + tokenizer=tokenizer, + requires_padding=False, + dtype=dtype, + device=device, + rank=rank, + world_size=world_size, + ) + + @staticmethod + def load_weights( + model, + filenames: List[str], + quantize: Optional[str], + device: torch.device, + dtype: torch.dtype, + rank: int, + world_size: int, + ): + parameters = dict(model.named_parameters()) + for file in filenames: + with safe_open( + file, framework="pt", device=str(device) if quantize is None else "cpu" + ) as f: + for name in f.keys(): + module_name, param_name = name.rsplit(".", 1) + module = model.get_submodule(module_name) + + current_parameter_tensor = parameters.get(name, None) + + slice_ = f.get_slice(name) + + if isinstance(module, TensorParallelColumnLinear): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif isinstance(module, TensorParallelRowLinear): + if param_name == "weight": + size = slice_.get_shape()[1] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[:, start:stop] + else: + tensor = slice_[:] + # XXX: Hack for Rowlinear to add the bias only once. + if rank != 0: + tensor = torch.zeros_like(tensor) + elif isinstance(module, TensorParallelEmbedding): + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + elif name == "lm_head.weight" and model.transformer.tp_embeddings: + size = slice_.get_shape()[0] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + tensor = slice_[start:stop] + else: + try: + tensor = slice_[:] + except: + tensor = f.get_tensor(name) + + if ( + current_parameter_tensor is not None + and current_parameter_tensor.shape != tensor.shape + ): + raise ValueError( + f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" + ) + + tensor = tensor.contiguous().to(dtype) + + if current_parameter_tensor is not None: + module._parameters[param_name] = tensor + else: + module._buffers[param_name] = tensor + + model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/rw.py b/server/text_generation_server/models/rw.py new file mode 100644 index 00000000..dd389027 --- /dev/null +++ b/server/text_generation_server/models/rw.py @@ -0,0 +1,86 @@ +import torch + +from transformers import AutoTokenizer, AutoModelForCausalLM +from typing import List, Optional, Tuple + +from text_generation_server.models import CausalLM + + +class RW(CausalLM): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + trust_remote_code: bool = False, + ): + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.bfloat16 + else: + if quantize: + raise ValueError("quantization is not available on CPU") + + device = torch.device("cpu") + dtype = torch.float32 + + tokenizer = AutoTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + model = AutoModelForCausalLM.from_pretrained( + model_id, + revision=revision, + torch_dtype=dtype, + device_map="auto" + if torch.cuda.is_available() and torch.cuda.device_count() > 1 + else None, + load_in_8bit=quantize == "bitsandbytes", + trust_remote_code=trust_remote_code, + ) + if torch.cuda.is_available() and torch.cuda.device_count() == 1: + model = model.cuda() + + if tokenizer.pad_token_id is None: + if model.config.pad_token_id is not None: + tokenizer.pad_token_id = model.config.pad_token_id + elif model.config.eos_token_id is not None: + tokenizer.pad_token_id = model.config.eos_token_id + elif tokenizer.eos_token_id is not None: + tokenizer.pad_token_id = tokenizer.eos_token_id + else: + tokenizer.add_special_tokens({"pad_token": "[PAD]"}) + + super(CausalLM, self).__init__( + model=model, + tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, + device=device, + ) + + def forward( + self, input_ids, attention_mask, position_ids, past_key_values: Optional = None + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + # Model Forward + if past_key_values is not None: + reshaped_past_key_values = [] + for layer in past_key_values: + past_keys, past_values = layer + reshaped_past_key_values.append( + ( + past_keys.view(-1, *past_keys.shape[-2:]), + past_values.view(-1, *past_values.shape[-2:]), + ) + ) + past_key_values = reshaped_past_key_values + + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + past_key_values=past_key_values, + ) + return outputs.logits, outputs.past_key_values diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 7605639d..127f9ba4 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -262,16 +262,13 @@ def get_cos_sin( sin = torch.index_select(self._sin_cached, 0, position_ids) return cos.unsqueeze(1), sin.unsqueeze(1) - def forward(self, qkv: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): + def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): rotary_dim = cos.shape[-1] - q1 = qkv[:, 0, :, :rotary_dim] - q2 = qkv[:, 0, :, rotary_dim : 2 * rotary_dim] - k1 = qkv[:, 1, :, :rotary_dim] - k2 = qkv[:, 1, :, rotary_dim : 2 * rotary_dim] - - rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False) - rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False) - return qkv + x1 = x[..., :rotary_dim] + x2 = x[..., rotary_dim : 2 * rotary_dim] + + rotary_emb.apply_rotary(x1, x2, cos, sin, x1, x2, False) + return x except ImportError: pass From 081b92658470bf5b40aec7427e084b7a96c8fbd2 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Tue, 30 May 2023 18:39:35 +0200 Subject: [PATCH 261/793] v0.8.0 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- README.md | 7 +++++-- docs/openapi.json | 2 +- server/pyproject.toml | 2 +- supported_models.json | 9 --------- 6 files changed, 12 insertions(+), 18 deletions(-) delete mode 100644 supported_models.json diff --git a/Cargo.lock b/Cargo.lock index 62896a51..cddebd61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2557,7 +2557,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "0.7.0" +version = "0.8.0" dependencies = [ "average", "clap", @@ -2577,7 +2577,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.7.0" +version = "0.8.0" dependencies = [ "futures", "grpc-metadata", @@ -2593,7 +2593,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.7.0" +version = "0.8.0" dependencies = [ "clap", "ctrlc", @@ -2609,7 +2609,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.7.0" +version = "0.8.0" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index 4ed0524e..871fc688 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.7.0" +version = "0.8.0" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index 30e085b2..8c8d9773 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ to power LLMs api-inference widgets. - Serve the most popular Large Language Models with a simple launcher - Tensor Parallelism for faster inference on multiple GPUs - Token streaming using Server-Sent Events (SSE) -- [Continous batching of incoming requests](https://github.com/huggingface/text-generation-inference/tree/main/router) for increased total throughput +- [Continuous batching of incoming requests](https://github.com/huggingface/text-generation-inference/tree/main/router) for increased total throughput - Optimized transformers code for inference using [flash-attention](https://github.com/HazyResearch/flash-attention) on the most popular architectures - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) - [Safetensors](https://github.com/huggingface/safetensors) weight loading @@ -61,6 +61,9 @@ to power LLMs api-inference widgets. - [Llama](https://github.com/facebookresearch/llama) - [OPT](https://huggingface.co/facebook/opt-66b) - [SantaCoder](https://huggingface.co/bigcode/santacoder) +- [Starcoder](https://huggingface.co/bigcode/starcoder) +- [Falcon 7B](https://huggingface.co/tiiuae/falcon-7b) +- [Falcon 40B](https://huggingface.co/tiiuae/falcon-40b) Other architectures are supported on a best effort basis using: @@ -81,7 +84,7 @@ model=bigscience/bloom-560m num_shard=2 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.7 --model-id $model --num-shard $num_shard +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.8 --model-id $model --num-shard $num_shard ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. diff --git a/docs/openapi.json b/docs/openapi.json index 5bccc9fc..965aefd2 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.7.0" + "version": "0.8.0" }, "paths": { "/": { diff --git a/server/pyproject.toml b/server/pyproject.toml index d674172d..28e078b0 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.7.0" +version = "0.8.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] diff --git a/supported_models.json b/supported_models.json deleted file mode 100644 index 8e9dd9e0..00000000 --- a/supported_models.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - "bigcode/santacoder", - "bigscience/bloom", - "bigscience/bloomz", - "EleutherAI/gpt-neox-20b", - "google/flan-ul2", - "google/flan-t5-xxl", - "OpenAssistant/oasst-sft-1-pythia-12b" -] From 444400b45760ab54faac0689f2f815c1fb425a8f Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 31 May 2023 10:55:59 +0200 Subject: [PATCH 262/793] increase health checks --- integration-tests/models/test_bloom_560m.py | 2 +- integration-tests/models/test_bloom_560m_sharded.py | 2 +- integration-tests/models/test_flash_falcon.py | 2 +- integration-tests/models/test_flash_llama.py | 2 +- integration-tests/models/test_flash_neox.py | 2 +- integration-tests/models/test_flash_neox_sharded.py | 2 +- integration-tests/models/test_flash_santacoder.py | 2 +- integration-tests/models/test_flash_starcoder.py | 2 +- integration-tests/models/test_mt0_base.py | 2 +- integration-tests/models/test_t5_sharded.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/integration-tests/models/test_bloom_560m.py b/integration-tests/models/test_bloom_560m.py index 3c598c04..809250cb 100644 --- a/integration-tests/models/test_bloom_560m.py +++ b/integration-tests/models/test_bloom_560m.py @@ -9,7 +9,7 @@ def bloom_560_handle(launcher): @pytest.fixture(scope="module") async def bloom_560(bloom_560_handle): - await bloom_560_handle.health(60) + await bloom_560_handle.health(240) return bloom_560_handle.client diff --git a/integration-tests/models/test_bloom_560m_sharded.py b/integration-tests/models/test_bloom_560m_sharded.py index 25f6b2d7..ee67250a 100644 --- a/integration-tests/models/test_bloom_560m_sharded.py +++ b/integration-tests/models/test_bloom_560m_sharded.py @@ -9,7 +9,7 @@ def bloom_560m_sharded_handle(launcher): @pytest.fixture(scope="module") async def bloom_560m_sharded(bloom_560m_sharded_handle): - await bloom_560m_sharded_handle.health(60) + await bloom_560m_sharded_handle.health(240) return bloom_560m_sharded_handle.client diff --git a/integration-tests/models/test_flash_falcon.py b/integration-tests/models/test_flash_falcon.py index ce27731d..e36a6a28 100644 --- a/integration-tests/models/test_flash_falcon.py +++ b/integration-tests/models/test_flash_falcon.py @@ -9,7 +9,7 @@ def flash_falcon_handle(launcher): @pytest.fixture(scope="module") async def flash_falcon(flash_falcon_handle): - await flash_falcon_handle.health(120) + await flash_falcon_handle.health(300) return flash_falcon_handle.client diff --git a/integration-tests/models/test_flash_llama.py b/integration-tests/models/test_flash_llama.py index bf5b64ba..edc847c1 100644 --- a/integration-tests/models/test_flash_llama.py +++ b/integration-tests/models/test_flash_llama.py @@ -9,7 +9,7 @@ def flash_llama_handle(launcher): @pytest.fixture(scope="module") async def flash_llama(flash_llama_handle): - await flash_llama_handle.health(120) + await flash_llama_handle.health(300) return flash_llama_handle.client diff --git a/integration-tests/models/test_flash_neox.py b/integration-tests/models/test_flash_neox.py index ff9915ed..daff7f0a 100644 --- a/integration-tests/models/test_flash_neox.py +++ b/integration-tests/models/test_flash_neox.py @@ -9,7 +9,7 @@ def flash_neox_handle(launcher): @pytest.fixture(scope="module") async def flash_neox(flash_neox_handle): - await flash_neox_handle.health(240) + await flash_neox_handle.health(300) return flash_neox_handle.client diff --git a/integration-tests/models/test_flash_neox_sharded.py b/integration-tests/models/test_flash_neox_sharded.py index 513aeaaf..a1aa0f07 100644 --- a/integration-tests/models/test_flash_neox_sharded.py +++ b/integration-tests/models/test_flash_neox_sharded.py @@ -9,7 +9,7 @@ def flash_neox_sharded_handle(launcher): @pytest.fixture(scope="module") async def flash_neox_sharded(flash_neox_sharded_handle): - await flash_neox_sharded_handle.health(240) + await flash_neox_sharded_handle.health(300) return flash_neox_sharded_handle.client diff --git a/integration-tests/models/test_flash_santacoder.py b/integration-tests/models/test_flash_santacoder.py index b0cb4522..a15a6439 100644 --- a/integration-tests/models/test_flash_santacoder.py +++ b/integration-tests/models/test_flash_santacoder.py @@ -9,7 +9,7 @@ def flash_santacoder_handle(launcher): @pytest.fixture(scope="module") async def flash_santacoder(flash_santacoder_handle): - await flash_santacoder_handle.health(240) + await flash_santacoder_handle.health(300) return flash_santacoder_handle.client diff --git a/integration-tests/models/test_flash_starcoder.py b/integration-tests/models/test_flash_starcoder.py index c1a68d89..72b298c9 100644 --- a/integration-tests/models/test_flash_starcoder.py +++ b/integration-tests/models/test_flash_starcoder.py @@ -9,7 +9,7 @@ def flash_starcoder_handle(launcher): @pytest.fixture(scope="module") async def flash_starcoder(flash_starcoder_handle): - await flash_starcoder_handle.health(240) + await flash_starcoder_handle.health(300) return flash_starcoder_handle.client diff --git a/integration-tests/models/test_mt0_base.py b/integration-tests/models/test_mt0_base.py index e347d22a..4ed95aad 100644 --- a/integration-tests/models/test_mt0_base.py +++ b/integration-tests/models/test_mt0_base.py @@ -9,7 +9,7 @@ def mt0_base_handle(launcher): @pytest.fixture(scope="module") async def mt0_base(mt0_base_handle): - await mt0_base_handle.health(60) + await mt0_base_handle.health(300) return mt0_base_handle.client diff --git a/integration-tests/models/test_t5_sharded.py b/integration-tests/models/test_t5_sharded.py index 074660c7..a2d84330 100644 --- a/integration-tests/models/test_t5_sharded.py +++ b/integration-tests/models/test_t5_sharded.py @@ -9,7 +9,7 @@ def t5_sharded_handle(launcher): @pytest.fixture(scope="module") async def t5_sharded(t5_sharded_handle): - await t5_sharded_handle.health(240) + await t5_sharded_handle.health(300) return t5_sharded_handle.client From 87dc034b590723a7ebf354df576a13690d9664cc Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 31 May 2023 10:57:53 +0200 Subject: [PATCH 263/793] feat(server): add retry on download (#384) --- .../test_flash_falcon_load.json | 562 +++++++++--------- .../text_generation_server/models/flash_rw.py | 4 +- server/text_generation_server/models/rw.py | 2 +- server/text_generation_server/utils/hub.py | 39 +- 4 files changed, 309 insertions(+), 298 deletions(-) diff --git a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json index 0fb1be75..90a35eb7 100644 --- a/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json +++ b/integration-tests/models/__snapshots__/test_flash_falcon/test_flash_falcon_load.json @@ -12,22 +12,22 @@ }, { "id": 330, - "logprob": -5.71875, + "logprob": -5.96875, "text": "ir" }, { "id": 1622, - "logprob": -5.625, + "logprob": -5.6171875, "text": "af" }, { "id": 249, - "logprob": -6.53125, + "logprob": -6.5039062, "text": "at" }, { "id": 1480, - "logprob": -8.0625, + "logprob": -8.0703125, "text": "ron" }, { @@ -37,27 +37,27 @@ }, { "id": 23866, - "logprob": -9.625, + "logprob": -9.59375, "text": " obsessed" }, { "id": 335, - "logprob": -0.048583984, + "logprob": -0.04837036, "text": " with" }, { "id": 26680, - "logprob": -3.984375, + "logprob": -3.9960938, "text": " gir" }, { "id": 1903, - "logprob": -0.076171875, + "logprob": -0.07525635, "text": "aff" }, { "id": 255, - "logprob": -0.0066833496, + "logprob": -0.006790161, "text": "es" }, { @@ -67,82 +67,82 @@ }, { "id": 248, - "logprob": -4.34375, + "logprob": -4.3320312, "text": " the" }, { "id": 758, - "logprob": -3.734375, + "logprob": -3.7363281, "text": " most" }, { "id": 21735, - "logprob": -5.125, + "logprob": -5.109375, "text": " glorious" }, { "id": 5985, - "logprob": -2.078125, + "logprob": -2.09375, "text": " animal" }, { "id": 313, - "logprob": -1.1953125, + "logprob": -1.1845703, "text": " on" }, { "id": 248, - "logprob": -0.78125, + "logprob": -0.77734375, "text": " the" }, { "id": 1936, - "logprob": -2.390625, + "logprob": -2.3828125, "text": " face" }, { "id": 275, - "logprob": -0.0044555664, + "logprob": -0.0044403076, "text": " of" }, { "id": 414, - "logprob": -1.984375, + "logprob": -1.9667969, "text": " this" }, { "id": 6490, - "logprob": -2.03125, + "logprob": -2.0449219, "text": " Earth" }, { "id": 25, - "logprob": -0.28320312, + "logprob": -0.28198242, "text": "." }, { "id": 401, - "logprob": -7.90625, + "logprob": -7.921875, "text": " G" }, { "id": 6013, - "logprob": -2.265625, + "logprob": -2.2714844, "text": "ira" }, { "id": 694, - "logprob": -0.640625, + "logprob": -0.62353516, "text": "ft" }, { "id": 1480, - "logprob": -0.203125, + "logprob": -0.20947266, "text": "ron" }, { "id": 9369, - "logprob": -4.53125, + "logprob": -4.5507812, "text": " believes" }, { @@ -152,32 +152,32 @@ }, { "id": 599, - "logprob": -2.75, + "logprob": -2.7402344, "text": " other" }, { "id": 5632, - "logprob": -0.21875, + "logprob": -0.21899414, "text": " animals" }, { "id": 362, - "logprob": -0.76171875, + "logprob": -0.76708984, "text": " are" }, { "id": 23981, - "logprob": -4.96875, + "logprob": -4.9960938, "text": " irrelevant" }, { "id": 635, - "logprob": -4.21875, + "logprob": -4.234375, "text": " when" }, { "id": 4354, - "logprob": -0.51953125, + "logprob": -0.5131836, "text": " compared" }, { @@ -187,47 +187,47 @@ }, { "id": 248, - "logprob": -0.58984375, + "logprob": -0.58447266, "text": " the" }, { "id": 21735, - "logprob": -3.6875, + "logprob": -3.6796875, "text": " glorious" }, { "id": 64398, - "logprob": -1.8359375, + "logprob": -1.8222656, "text": " majesty" }, { "id": 275, - "logprob": -0.24316406, + "logprob": -0.23583984, "text": " of" }, { "id": 248, - "logprob": -0.3515625, + "logprob": -0.3544922, "text": " the" }, { "id": 26680, - "logprob": -0.24414062, + "logprob": -0.24609375, "text": " gir" }, { "id": 23226, - "logprob": -0.03100586, + "logprob": -0.02960205, "text": "affe" }, { "id": 25, - "logprob": -0.17382812, + "logprob": -0.17358398, "text": "." }, { "id": 193, - "logprob": -1.3984375, + "logprob": -1.3925781, "text": "\n" }, { @@ -237,7 +237,7 @@ }, { "id": 37, - "logprob": -4.59375, + "logprob": -4.5898438, "text": ":" }, { @@ -247,67 +247,67 @@ }, { "id": 23, - "logprob": -1.0, + "logprob": -0.99365234, "text": "," }, { "id": 29033, - "logprob": -2.21875, + "logprob": -2.2304688, "text": " Gir" }, { "id": 1622, - "logprob": -0.10644531, + "logprob": -0.107788086, "text": "af" }, { "id": 249, - "logprob": -0.041992188, + "logprob": -0.04257202, "text": "at" }, { "id": 1480, - "logprob": -0.0025024414, + "logprob": -0.0024871826, "text": "ron" }, { "id": 12, - "logprob": -1.4296875, + "logprob": -1.4277344, "text": "!" }, { "id": 193, - "logprob": -1.1015625, + "logprob": -1.1005859, "text": "\n" }, { "id": 50, - "logprob": -0.05810547, + "logprob": -0.056915283, "text": "G" }, { "id": 330, - "logprob": -0.12597656, + "logprob": -0.1315918, "text": "ir" }, { "id": 1622, - "logprob": -0.007080078, + "logprob": -0.0071105957, "text": "af" }, { "id": 249, - "logprob": -0.008300781, + "logprob": -0.008453369, "text": "at" }, { "id": 1480, - "logprob": -0.0006866455, + "logprob": -0.0006928444, "text": "ron" }, { "id": 37, - "logprob": -0.0074157715, + "logprob": -0.0074920654, "text": ":" } ], @@ -315,37 +315,37 @@ "tokens": [ { "id": 23090, - "logprob": -1.8203125, + "logprob": -1.828125, "special": false, "text": " Hello" }, { "id": 23, - "logprob": -0.32226562, + "logprob": -0.3178711, "special": false, "text": "," }, { "id": 8156, - "logprob": -0.23828125, + "logprob": -0.23925781, "special": false, "text": " Daniel" }, { "id": 12, - "logprob": -0.5859375, + "logprob": -0.5698242, "special": false, "text": "!" }, { "id": 193, - "logprob": -0.6171875, + "logprob": -0.61279297, "special": false, "text": "\n" }, { "id": 23626, - "logprob": -0.39648438, + "logprob": -0.4177246, "special": false, "text": "Daniel" }, @@ -356,26 +356,26 @@ "text": ":" }, { - "id": 295, - "logprob": -2.078125, + "id": 1634, + "logprob": -2.0605469, "special": false, - "text": " I" + "text": " What" }, { "id": 18, - "logprob": -1.453125, + "logprob": -1.5283203, "special": false, "text": "'" }, { - "id": 88, - "logprob": -0.47460938, + "id": 94, + "logprob": -0.007965088, "special": false, - "text": "m" + "text": "s" } ] }, - "generated_text": " Hello, Daniel!\nDaniel: I'm" + "generated_text": " Hello, Daniel!\nDaniel: What's" }, { "details": { @@ -390,22 +390,22 @@ }, { "id": 330, - "logprob": -5.71875, + "logprob": -5.96875, "text": "ir" }, { "id": 1622, - "logprob": -5.625, + "logprob": -5.6171875, "text": "af" }, { "id": 249, - "logprob": -6.53125, + "logprob": -6.5, "text": "at" }, { "id": 1480, - "logprob": -8.0625, + "logprob": -8.0703125, "text": "ron" }, { @@ -415,197 +415,197 @@ }, { "id": 23866, - "logprob": -9.625, + "logprob": -9.59375, "text": " obsessed" }, { "id": 335, - "logprob": -0.048583984, + "logprob": -0.048339844, "text": " with" }, { "id": 26680, - "logprob": -3.984375, + "logprob": -4.0, "text": " gir" }, { "id": 1903, - "logprob": -0.076171875, + "logprob": -0.07531738, "text": "aff" }, { "id": 255, - "logprob": -0.0066833496, + "logprob": -0.006793976, "text": "es" }, { "id": 23, - "logprob": -1.546875, + "logprob": -1.5478516, "text": "," }, { "id": 248, - "logprob": -4.34375, + "logprob": -4.3320312, "text": " the" }, { "id": 758, - "logprob": -3.734375, + "logprob": -3.7363281, "text": " most" }, { "id": 21735, - "logprob": -5.125, + "logprob": -5.1132812, "text": " glorious" }, { "id": 5985, - "logprob": -2.078125, + "logprob": -2.0957031, "text": " animal" }, { "id": 313, - "logprob": -1.1953125, + "logprob": -1.1835938, "text": " on" }, { "id": 248, - "logprob": -0.78125, + "logprob": -0.77685547, "text": " the" }, { "id": 1936, - "logprob": -2.390625, + "logprob": -2.3808594, "text": " face" }, { "id": 275, - "logprob": -0.0044555664, + "logprob": -0.004436493, "text": " of" }, { "id": 414, - "logprob": -1.984375, + "logprob": -1.9638672, "text": " this" }, { "id": 6490, - "logprob": -2.03125, + "logprob": -2.0449219, "text": " Earth" }, { "id": 25, - "logprob": -0.28320312, + "logprob": -0.28198242, "text": "." }, { "id": 401, - "logprob": -7.90625, + "logprob": -7.9179688, "text": " G" }, { "id": 6013, - "logprob": -2.265625, + "logprob": -2.2734375, "text": "ira" }, { "id": 694, - "logprob": -0.640625, + "logprob": -0.6230469, "text": "ft" }, { "id": 1480, - "logprob": -0.203125, + "logprob": -0.20947266, "text": "ron" }, { "id": 9369, - "logprob": -4.53125, + "logprob": -4.5546875, "text": " believes" }, { "id": 455, - "logprob": -4.5625, + "logprob": -4.5703125, "text": " all" }, { "id": 599, - "logprob": -2.75, + "logprob": -2.7382812, "text": " other" }, { "id": 5632, - "logprob": -0.21875, + "logprob": -0.21948242, "text": " animals" }, { "id": 362, - "logprob": -0.76171875, + "logprob": -0.7661133, "text": " are" }, { "id": 23981, - "logprob": -4.96875, + "logprob": -4.9960938, "text": " irrelevant" }, { "id": 635, - "logprob": -4.21875, + "logprob": -4.234375, "text": " when" }, { "id": 4354, - "logprob": -0.51953125, + "logprob": -0.5131836, "text": " compared" }, { "id": 271, - "logprob": -0.103515625, + "logprob": -0.10357666, "text": " to" }, { "id": 248, - "logprob": -0.58984375, + "logprob": -0.58447266, "text": " the" }, { "id": 21735, - "logprob": -3.6875, + "logprob": -3.6816406, "text": " glorious" }, { "id": 64398, - "logprob": -1.8359375, + "logprob": -1.8203125, "text": " majesty" }, { "id": 275, - "logprob": -0.24316406, + "logprob": -0.23583984, "text": " of" }, { "id": 248, - "logprob": -0.3515625, + "logprob": -0.35473633, "text": " the" }, { "id": 26680, - "logprob": -0.24414062, + "logprob": -0.24572754, "text": " gir" }, { "id": 23226, - "logprob": -0.03100586, + "logprob": -0.029586792, "text": "affe" }, { "id": 25, - "logprob": -0.17382812, + "logprob": -0.17346191, "text": "." }, { "id": 193, - "logprob": -1.3984375, + "logprob": -1.3945312, "text": "\n" }, { @@ -625,67 +625,67 @@ }, { "id": 23, - "logprob": -1.0, + "logprob": -0.99316406, "text": "," }, { "id": 29033, - "logprob": -2.21875, + "logprob": -2.2324219, "text": " Gir" }, { "id": 1622, - "logprob": -0.10644531, + "logprob": -0.10797119, "text": "af" }, { "id": 249, - "logprob": -0.041992188, + "logprob": -0.04248047, "text": "at" }, { "id": 1480, - "logprob": -0.0025024414, + "logprob": -0.0024814606, "text": "ron" }, { "id": 12, - "logprob": -1.4296875, + "logprob": -1.4277344, "text": "!" }, { "id": 193, - "logprob": -1.1015625, + "logprob": -1.1005859, "text": "\n" }, { "id": 50, - "logprob": -0.05810547, + "logprob": -0.056884766, "text": "G" }, { "id": 330, - "logprob": -0.12597656, + "logprob": -0.1315918, "text": "ir" }, { "id": 1622, - "logprob": -0.007080078, + "logprob": -0.007095337, "text": "af" }, { "id": 249, - "logprob": -0.008300781, + "logprob": -0.00844574, "text": "at" }, { "id": 1480, - "logprob": -0.0006866455, + "logprob": -0.00068998337, "text": "ron" }, { "id": 37, - "logprob": -0.0074157715, + "logprob": -0.0074768066, "text": ":" } ], @@ -693,67 +693,67 @@ "tokens": [ { "id": 23090, - "logprob": -1.8203125, + "logprob": -1.8251953, "special": false, "text": " Hello" }, { "id": 23, - "logprob": -0.32226562, + "logprob": -0.31762695, "special": false, "text": "," }, { "id": 8156, - "logprob": -0.23828125, + "logprob": -0.2388916, "special": false, "text": " Daniel" }, { "id": 12, - "logprob": -0.5859375, + "logprob": -0.5698242, "special": false, "text": "!" }, { "id": 193, - "logprob": -0.6171875, + "logprob": -0.6152344, "special": false, "text": "\n" }, { "id": 23626, - "logprob": -0.39648438, + "logprob": -0.42211914, "special": false, "text": "Daniel" }, { "id": 37, - "logprob": -0.0023345947, + "logprob": -0.002336502, "special": false, "text": ":" }, { - "id": 295, - "logprob": -2.078125, + "id": 1634, + "logprob": -2.0605469, "special": false, - "text": " I" + "text": " What" }, { "id": 18, - "logprob": -1.453125, + "logprob": -1.5292969, "special": false, "text": "'" }, { - "id": 88, - "logprob": -0.47460938, + "id": 94, + "logprob": -0.007926941, "special": false, - "text": "m" + "text": "s" } ] }, - "generated_text": " Hello, Daniel!\nDaniel: I'm" + "generated_text": " Hello, Daniel!\nDaniel: What's" }, { "details": { @@ -768,22 +768,22 @@ }, { "id": 330, - "logprob": -5.71875, + "logprob": -5.96875, "text": "ir" }, { "id": 1622, - "logprob": -5.625, + "logprob": -5.6171875, "text": "af" }, { "id": 249, - "logprob": -6.53125, + "logprob": -6.5, "text": "at" }, { "id": 1480, - "logprob": -8.0625, + "logprob": -8.0703125, "text": "ron" }, { @@ -793,197 +793,197 @@ }, { "id": 23866, - "logprob": -9.625, + "logprob": -9.59375, "text": " obsessed" }, { "id": 335, - "logprob": -0.048583984, + "logprob": -0.048339844, "text": " with" }, { "id": 26680, - "logprob": -3.984375, + "logprob": -4.0, "text": " gir" }, { "id": 1903, - "logprob": -0.076171875, + "logprob": -0.07531738, "text": "aff" }, { "id": 255, - "logprob": -0.0066833496, + "logprob": -0.006793976, "text": "es" }, { "id": 23, - "logprob": -1.546875, + "logprob": -1.5478516, "text": "," }, { "id": 248, - "logprob": -4.34375, + "logprob": -4.3320312, "text": " the" }, { "id": 758, - "logprob": -3.734375, + "logprob": -3.7363281, "text": " most" }, { "id": 21735, - "logprob": -5.125, + "logprob": -5.1132812, "text": " glorious" }, { "id": 5985, - "logprob": -2.078125, + "logprob": -2.0957031, "text": " animal" }, { "id": 313, - "logprob": -1.1953125, + "logprob": -1.1835938, "text": " on" }, { "id": 248, - "logprob": -0.78125, + "logprob": -0.77685547, "text": " the" }, { "id": 1936, - "logprob": -2.390625, + "logprob": -2.3808594, "text": " face" }, { "id": 275, - "logprob": -0.0044555664, + "logprob": -0.004436493, "text": " of" }, { "id": 414, - "logprob": -1.984375, + "logprob": -1.9638672, "text": " this" }, { "id": 6490, - "logprob": -2.03125, + "logprob": -2.0449219, "text": " Earth" }, { "id": 25, - "logprob": -0.28320312, + "logprob": -0.28198242, "text": "." }, { "id": 401, - "logprob": -7.90625, + "logprob": -7.9179688, "text": " G" }, { "id": 6013, - "logprob": -2.265625, + "logprob": -2.2734375, "text": "ira" }, { "id": 694, - "logprob": -0.640625, + "logprob": -0.6230469, "text": "ft" }, { "id": 1480, - "logprob": -0.203125, + "logprob": -0.20947266, "text": "ron" }, { "id": 9369, - "logprob": -4.53125, + "logprob": -4.5546875, "text": " believes" }, { "id": 455, - "logprob": -4.5625, + "logprob": -4.5703125, "text": " all" }, { "id": 599, - "logprob": -2.75, + "logprob": -2.7382812, "text": " other" }, { "id": 5632, - "logprob": -0.21875, + "logprob": -0.21948242, "text": " animals" }, { "id": 362, - "logprob": -0.76171875, + "logprob": -0.7661133, "text": " are" }, { "id": 23981, - "logprob": -4.96875, + "logprob": -4.9960938, "text": " irrelevant" }, { "id": 635, - "logprob": -4.21875, + "logprob": -4.234375, "text": " when" }, { "id": 4354, - "logprob": -0.51953125, + "logprob": -0.5131836, "text": " compared" }, { "id": 271, - "logprob": -0.103515625, + "logprob": -0.10357666, "text": " to" }, { "id": 248, - "logprob": -0.58984375, + "logprob": -0.58447266, "text": " the" }, { "id": 21735, - "logprob": -3.6875, + "logprob": -3.6816406, "text": " glorious" }, { "id": 64398, - "logprob": -1.8359375, + "logprob": -1.8203125, "text": " majesty" }, { "id": 275, - "logprob": -0.24316406, + "logprob": -0.23583984, "text": " of" }, { "id": 248, - "logprob": -0.3515625, + "logprob": -0.35473633, "text": " the" }, { "id": 26680, - "logprob": -0.24414062, + "logprob": -0.24572754, "text": " gir" }, { "id": 23226, - "logprob": -0.03100586, + "logprob": -0.029586792, "text": "affe" }, { "id": 25, - "logprob": -0.17382812, + "logprob": -0.17346191, "text": "." }, { "id": 193, - "logprob": -1.3984375, + "logprob": -1.3945312, "text": "\n" }, { @@ -1003,67 +1003,67 @@ }, { "id": 23, - "logprob": -1.0, + "logprob": -0.99316406, "text": "," }, { "id": 29033, - "logprob": -2.21875, + "logprob": -2.2324219, "text": " Gir" }, { "id": 1622, - "logprob": -0.10644531, + "logprob": -0.10797119, "text": "af" }, { "id": 249, - "logprob": -0.041992188, + "logprob": -0.04248047, "text": "at" }, { "id": 1480, - "logprob": -0.0025024414, + "logprob": -0.0024814606, "text": "ron" }, { "id": 12, - "logprob": -1.4296875, + "logprob": -1.4277344, "text": "!" }, { "id": 193, - "logprob": -1.1015625, + "logprob": -1.1005859, "text": "\n" }, { "id": 50, - "logprob": -0.05810547, + "logprob": -0.056884766, "text": "G" }, { "id": 330, - "logprob": -0.12597656, + "logprob": -0.1315918, "text": "ir" }, { "id": 1622, - "logprob": -0.007080078, + "logprob": -0.007095337, "text": "af" }, { "id": 249, - "logprob": -0.008300781, + "logprob": -0.00844574, "text": "at" }, { "id": 1480, - "logprob": -0.0006866455, + "logprob": -0.00068998337, "text": "ron" }, { "id": 37, - "logprob": -0.0074157715, + "logprob": -0.0074768066, "text": ":" } ], @@ -1071,67 +1071,67 @@ "tokens": [ { "id": 23090, - "logprob": -1.8203125, + "logprob": -1.8251953, "special": false, "text": " Hello" }, { "id": 23, - "logprob": -0.32226562, + "logprob": -0.31762695, "special": false, "text": "," }, { "id": 8156, - "logprob": -0.23828125, + "logprob": -0.2388916, "special": false, "text": " Daniel" }, { "id": 12, - "logprob": -0.5859375, + "logprob": -0.5698242, "special": false, "text": "!" }, { "id": 193, - "logprob": -0.6171875, + "logprob": -0.6152344, "special": false, "text": "\n" }, { "id": 23626, - "logprob": -0.39648438, + "logprob": -0.42211914, "special": false, "text": "Daniel" }, { "id": 37, - "logprob": -0.0023345947, + "logprob": -0.002336502, "special": false, "text": ":" }, { - "id": 295, - "logprob": -2.078125, + "id": 1634, + "logprob": -2.0605469, "special": false, - "text": " I" + "text": " What" }, { "id": 18, - "logprob": -1.453125, + "logprob": -1.5292969, "special": false, "text": "'" }, { - "id": 88, - "logprob": -0.47460938, + "id": 94, + "logprob": -0.007926941, "special": false, - "text": "m" + "text": "s" } ] }, - "generated_text": " Hello, Daniel!\nDaniel: I'm" + "generated_text": " Hello, Daniel!\nDaniel: What's" }, { "details": { @@ -1146,22 +1146,22 @@ }, { "id": 330, - "logprob": -5.71875, + "logprob": -5.96875, "text": "ir" }, { "id": 1622, - "logprob": -5.625, + "logprob": -5.6171875, "text": "af" }, { "id": 249, - "logprob": -6.53125, + "logprob": -6.5, "text": "at" }, { "id": 1480, - "logprob": -8.0625, + "logprob": -8.0703125, "text": "ron" }, { @@ -1171,197 +1171,197 @@ }, { "id": 23866, - "logprob": -9.625, + "logprob": -9.59375, "text": " obsessed" }, { "id": 335, - "logprob": -0.048583984, + "logprob": -0.048339844, "text": " with" }, { "id": 26680, - "logprob": -3.984375, + "logprob": -4.0, "text": " gir" }, { "id": 1903, - "logprob": -0.076171875, + "logprob": -0.07531738, "text": "aff" }, { "id": 255, - "logprob": -0.0066833496, + "logprob": -0.006793976, "text": "es" }, { "id": 23, - "logprob": -1.546875, + "logprob": -1.5478516, "text": "," }, { "id": 248, - "logprob": -4.34375, + "logprob": -4.3320312, "text": " the" }, { "id": 758, - "logprob": -3.734375, + "logprob": -3.7363281, "text": " most" }, { "id": 21735, - "logprob": -5.125, + "logprob": -5.1132812, "text": " glorious" }, { "id": 5985, - "logprob": -2.078125, + "logprob": -2.0957031, "text": " animal" }, { "id": 313, - "logprob": -1.1953125, + "logprob": -1.1835938, "text": " on" }, { "id": 248, - "logprob": -0.78125, + "logprob": -0.77685547, "text": " the" }, { "id": 1936, - "logprob": -2.390625, + "logprob": -2.3808594, "text": " face" }, { "id": 275, - "logprob": -0.0044555664, + "logprob": -0.004436493, "text": " of" }, { "id": 414, - "logprob": -1.984375, + "logprob": -1.9638672, "text": " this" }, { "id": 6490, - "logprob": -2.03125, + "logprob": -2.0449219, "text": " Earth" }, { "id": 25, - "logprob": -0.28320312, + "logprob": -0.28198242, "text": "." }, { "id": 401, - "logprob": -7.90625, + "logprob": -7.9179688, "text": " G" }, { "id": 6013, - "logprob": -2.265625, + "logprob": -2.2734375, "text": "ira" }, { "id": 694, - "logprob": -0.640625, + "logprob": -0.6230469, "text": "ft" }, { "id": 1480, - "logprob": -0.203125, + "logprob": -0.20947266, "text": "ron" }, { "id": 9369, - "logprob": -4.53125, + "logprob": -4.5546875, "text": " believes" }, { "id": 455, - "logprob": -4.5625, + "logprob": -4.5703125, "text": " all" }, { "id": 599, - "logprob": -2.75, + "logprob": -2.7382812, "text": " other" }, { "id": 5632, - "logprob": -0.21875, + "logprob": -0.21948242, "text": " animals" }, { "id": 362, - "logprob": -0.76171875, + "logprob": -0.7661133, "text": " are" }, { "id": 23981, - "logprob": -4.96875, + "logprob": -4.9960938, "text": " irrelevant" }, { "id": 635, - "logprob": -4.21875, + "logprob": -4.234375, "text": " when" }, { "id": 4354, - "logprob": -0.51953125, + "logprob": -0.5131836, "text": " compared" }, { "id": 271, - "logprob": -0.103515625, + "logprob": -0.10357666, "text": " to" }, { "id": 248, - "logprob": -0.58984375, + "logprob": -0.58447266, "text": " the" }, { "id": 21735, - "logprob": -3.6875, + "logprob": -3.6816406, "text": " glorious" }, { "id": 64398, - "logprob": -1.8359375, + "logprob": -1.8203125, "text": " majesty" }, { "id": 275, - "logprob": -0.24316406, + "logprob": -0.23583984, "text": " of" }, { "id": 248, - "logprob": -0.3515625, + "logprob": -0.35473633, "text": " the" }, { "id": 26680, - "logprob": -0.24414062, + "logprob": -0.24572754, "text": " gir" }, { "id": 23226, - "logprob": -0.03100586, + "logprob": -0.029586792, "text": "affe" }, { "id": 25, - "logprob": -0.17382812, + "logprob": -0.17346191, "text": "." }, { "id": 193, - "logprob": -1.3984375, + "logprob": -1.3945312, "text": "\n" }, { @@ -1381,67 +1381,67 @@ }, { "id": 23, - "logprob": -1.0, + "logprob": -0.99316406, "text": "," }, { "id": 29033, - "logprob": -2.21875, + "logprob": -2.2324219, "text": " Gir" }, { "id": 1622, - "logprob": -0.10644531, + "logprob": -0.10797119, "text": "af" }, { "id": 249, - "logprob": -0.041992188, + "logprob": -0.04248047, "text": "at" }, { "id": 1480, - "logprob": -0.0025024414, + "logprob": -0.0024814606, "text": "ron" }, { "id": 12, - "logprob": -1.4296875, + "logprob": -1.4277344, "text": "!" }, { "id": 193, - "logprob": -1.1015625, + "logprob": -1.1005859, "text": "\n" }, { "id": 50, - "logprob": -0.05810547, + "logprob": -0.056884766, "text": "G" }, { "id": 330, - "logprob": -0.12597656, + "logprob": -0.1315918, "text": "ir" }, { "id": 1622, - "logprob": -0.007080078, + "logprob": -0.007095337, "text": "af" }, { "id": 249, - "logprob": -0.008300781, + "logprob": -0.00844574, "text": "at" }, { "id": 1480, - "logprob": -0.0006866455, + "logprob": -0.00068998337, "text": "ron" }, { "id": 37, - "logprob": -0.0074157715, + "logprob": -0.0074768066, "text": ":" } ], @@ -1449,66 +1449,66 @@ "tokens": [ { "id": 23090, - "logprob": -1.8203125, + "logprob": -1.8251953, "special": false, "text": " Hello" }, { "id": 23, - "logprob": -0.32226562, + "logprob": -0.31762695, "special": false, "text": "," }, { "id": 8156, - "logprob": -0.23828125, + "logprob": -0.2388916, "special": false, "text": " Daniel" }, { "id": 12, - "logprob": -0.5859375, + "logprob": -0.5698242, "special": false, "text": "!" }, { "id": 193, - "logprob": -0.6171875, + "logprob": -0.6152344, "special": false, "text": "\n" }, { "id": 23626, - "logprob": -0.39648438, + "logprob": -0.42211914, "special": false, "text": "Daniel" }, { "id": 37, - "logprob": -0.0023345947, + "logprob": -0.002336502, "special": false, "text": ":" }, { - "id": 295, - "logprob": -2.078125, + "id": 1634, + "logprob": -2.0605469, "special": false, - "text": " I" + "text": " What" }, { "id": 18, - "logprob": -1.453125, + "logprob": -1.5292969, "special": false, "text": "'" }, { - "id": 88, - "logprob": -0.47460938, + "id": 94, + "logprob": -0.007926941, "special": false, - "text": "m" + "text": "s" } ] }, - "generated_text": " Hello, Daniel!\nDaniel: I'm" + "generated_text": " Hello, Daniel!\nDaniel: What's" } ] diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 44915ff5..4fc4c389 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -37,7 +37,7 @@ def __init__( ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 + dtype = torch.float16 else: raise NotImplementedError("RW is only available on GPU") @@ -124,7 +124,7 @@ def __init__( self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.bfloat16 + dtype = torch.float16 else: raise NotImplementedError("FlashRW is only available on GPU") diff --git a/server/text_generation_server/models/rw.py b/server/text_generation_server/models/rw.py index dd389027..2b1e4959 100644 --- a/server/text_generation_server/models/rw.py +++ b/server/text_generation_server/models/rw.py @@ -16,7 +16,7 @@ def __init__( ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.bfloat16 + dtype = torch.float16 else: if quantize: raise ValueError("quantization is not available on CPU") diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index 030c8289..134ac7cd 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -23,7 +23,11 @@ def weight_hub_files( """Get the weights filenames on the hub""" api = HfApi() info = api.model_info(model_id, revision=revision) - filenames = [s.rfilename for s in info.siblings if s.rfilename.endswith(extension)] + filenames = [ + s.rfilename + for s in info.siblings + if s.rfilename.endswith(extension) and len(s.rfilename.split("/")) == 1 + ] if not filenames: raise EntryNotFoundError( @@ -130,24 +134,31 @@ def download_weights( ) -> List[Path]: """Download the safetensors files from the hub""" - def download_file(filename): + def download_file(filename, tries=5): local_file = try_to_load_from_cache(model_id, revision, filename) if local_file is not None: logger.info(f"File {filename} already present in cache.") return Path(local_file) - logger.info(f"Download file: {filename}") - start_time = time.time() - local_file = hf_hub_download( - filename=filename, - repo_id=model_id, - revision=revision, - local_files_only=False, - ) - logger.info( - f"Downloaded {local_file} in {timedelta(seconds=int(time.time() - start_time))}." - ) - return Path(local_file) + for i in range(tries): + try: + logger.info(f"Download file: {filename}") + start_time = time.time() + local_file = hf_hub_download( + filename=filename, + repo_id=model_id, + revision=revision, + local_files_only=False, + ) + logger.info( + f"Downloaded {local_file} in {timedelta(seconds=int(time.time() - start_time))}." + ) + return Path(local_file) + except Exception as e: + if i + 1 == tries: + raise e + logger.error(e) + logger.info(f"Retry {i + 1}/{tries - 1}") # We do this instead of using tqdm because we want to parse the logs with the launcher start_time = time.time() From 337afb28422f421353a2ce756260ff01e51fb7d1 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 31 May 2023 11:48:28 +0200 Subject: [PATCH 264/793] fix(server): fix bnb quantization for CausalLM models (#385) --- server/text_generation_server/models/bloom.py | 2 ++ server/text_generation_server/models/galactica.py | 2 ++ server/text_generation_server/models/gpt_neox.py | 2 ++ server/text_generation_server/models/opt.py | 9 +++++++-- server/text_generation_server/models/t5.py | 3 ++- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 90db59fc..45d7cd4c 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -245,6 +245,8 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) + else: + tensor = tensor.to(device) elif quantize == "gptq": raise NotImplementedError("`gptq` is not implemented for now") elif quantize is None: diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 954421f0..37ccc398 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -364,6 +364,8 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) + else: + tensor = tensor.to(device) elif quantize == "gptq": raise NotImplementedError("`gptq` is not implemented for now") elif quantize is None: diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index c0e8170d..5ab8a624 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -210,6 +210,8 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) + else: + tensor = tensor.to(device) elif quantize == "gptq": raise NotImplementedError("`gptq` is not implemented for now") elif quantize is None: diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index bccce5b3..9cc4d5e1 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -166,7 +166,7 @@ def load_weights( tensor = tensor.contiguous().to(dtype) - if quantize: + if quantize == "bitsandbytes": if not HAS_BITS_AND_BYTES: raise ImportError( "bitsandbytes is not available on your machine either because it is not installed " @@ -216,9 +216,14 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) - else: tensor = tensor.to(device) + elif quantize == "gptq": + raise NotImplementedError("`gptq` is not implemented for now") + elif quantize is None: + tensor = tensor.to(device) + else: + raise ValueError(f"Unexpected quantize `{quantize}`") module._parameters[param_name] = tensor if name == "model.decoder.embed_tokens.weight": diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 32dcb806..d12b89d2 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -222,7 +222,8 @@ def linear(input, weight, bias): return linear module.linear = replace_linear(state) - + else: + tensor = tensor.to(device) elif quantize == "gptq" and not module_name.endswith("wo"): raise NotImplementedError("`gptq` is not implemented for now") elif quantize is None or module_name.endswith("wo"): From db2ebe3947c4cb018e1504ae417f3efb9a517d89 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 31 May 2023 12:08:40 +0200 Subject: [PATCH 265/793] v0.8.1 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- docs/openapi.json | 2 +- server/pyproject.toml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cddebd61..67ad8674 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2557,7 +2557,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "0.8.0" +version = "0.8.1" dependencies = [ "average", "clap", @@ -2577,7 +2577,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.8.0" +version = "0.8.1" dependencies = [ "futures", "grpc-metadata", @@ -2593,7 +2593,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.8.0" +version = "0.8.1" dependencies = [ "clap", "ctrlc", @@ -2609,7 +2609,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.8.0" +version = "0.8.1" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index 871fc688..3190b64c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.8.0" +version = "0.8.1" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index 965aefd2..040c8e8b 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.8.0" + "version": "0.8.1" }, "paths": { "/": { diff --git a/server/pyproject.toml b/server/pyproject.toml index 28e078b0..7400a055 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.8.0" +version = "0.8.1" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From d69a0633bee6f8a665a1f7d258fceaa4475c102f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 1 Jun 2023 11:41:35 +0200 Subject: [PATCH 266/793] fix(server): fix has_position_ids (#395) Fix #389 --- server/text_generation_server/models/causal_lm.py | 5 ----- server/text_generation_server/models/model.py | 7 +++++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index a20a6143..92622350 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -496,11 +496,6 @@ def __init__( else: tokenizer.add_special_tokens({"pad_token": "[PAD]"}) - self.has_position_ids = ( - inspect.signature(model.forward).parameters.get("position_ids", None) - is not None - ) - super(CausalLM, self).__init__( model=model, tokenizer=tokenizer, diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 29bad321..6b8472a5 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -1,3 +1,4 @@ +import inspect import torch from abc import ABC, abstractmethod @@ -29,6 +30,12 @@ def __init__( self.device = device self.rank = rank self.world_size = world_size + + self.has_position_ids = ( + inspect.signature(model.forward).parameters.get("position_ids", None) + is not None + ) + self.check_initialized() @property From c0928e6f26a537add41a237303fcabb8ac317a88 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 1 Jun 2023 12:07:41 +0200 Subject: [PATCH 267/793] feat(server): remove trust_remote_code requirement for falcon models (#396) --- .../text_generation_server/models/__init__.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 4adf1381..78b68721 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -1,7 +1,7 @@ import torch from loguru import logger -from transformers import AutoConfig +from transformers.configuration_utils import PretrainedConfig from transformers.models.auto import modeling_auto from typing import Optional @@ -138,10 +138,8 @@ def get_model( trust_remote_code=trust_remote_code, ) - config = AutoConfig.from_pretrained( - model_id, revision=revision, trust_remote_code=trust_remote_code - ) - model_type = config.model_type + config_dict, _ = PretrainedConfig.get_config_dict(model_id, revision=revision, trust_remote_code=trust_remote_code) + model_type = config_dict["model_type"] if model_type == "gpt_bigcode": if sharded: @@ -201,9 +199,9 @@ def get_model( if model_type in ["RefinedWeb", "RefinedWebModel"]: if sharded: if FLASH_ATTENTION: - if config.alibi or ( - config.model_type == "RefinedWebModel" - and config.n_head_kv != config.n_head + if config_dict.get("alibi", False) or ( + model_type == "RefinedWebModel" + and config_dict.get("multi_query", True) ): raise NotImplementedError("sharded is not supported for this model") return FlashRWSharded( @@ -216,7 +214,7 @@ def get_model( FLASH_ATT_ERROR_MESSAGE.format(f"Sharded RefinedWeb") ) else: - if FLASH_ATTENTION and not config.alibi: + if FLASH_ATTENTION and not config_dict.get("alibi", False): return FlashRW( model_id, revision, @@ -250,7 +248,7 @@ def get_model( trust_remote_code=trust_remote_code, ) - if config.model_type == "opt": + if model_type == "opt": if sharded: return OPTSharded( model_id, @@ -294,7 +292,7 @@ def get_model( model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code ) - auto_map = getattr(config, "auto_map", None) + auto_map = config_dict.get("auto_map", None) if trust_remote_code and auto_map is not None: if "AutoModelForCausalLM" in auto_map.keys(): return CausalLM( From 95d3546976e811ba462047ef8b555d9840efbfe5 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 1 Jun 2023 12:10:35 +0200 Subject: [PATCH 268/793] feat(server): load santacoder/starcoder models with safetensors (#393) Fix #366 --- launcher/src/main.rs | 16 +- .../models/flash_santacoder.py | 166 ++++++++++-------- 2 files changed, 91 insertions(+), 91 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 0810d979..7ee8bf1b 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -546,11 +546,7 @@ enum LauncherError { WebserverCannotStart, } -fn download_convert_model( - args: &Args, - auto_convert: bool, - running: Arc, -) -> Result<(), LauncherError> { +fn download_convert_model(args: &Args, running: Arc) -> Result<(), LauncherError> { let mut download_argv = vec![ "text-generation-server".to_string(), "download-weights".to_string(), @@ -562,11 +558,6 @@ fn download_convert_model( "--json-output".to_string(), ]; - // Auto convert weights to safetensors - if auto_convert { - download_argv.push("--auto-convert".to_string()); - } - // Model optional revision if let Some(revision) = &args.revision { download_argv.push("--revision".to_string()); @@ -932,11 +923,8 @@ fn main() -> Result<(), LauncherError> { }) .expect("Error setting Ctrl-C handler"); - // auto_convert is only needed for sharded models as we do not require safetensors in - // single shard mode - let auto_convert = num_shard > 1; // Download and convert model weights - download_convert_model(&args, auto_convert, running.clone())?; + download_convert_model(&args, running.clone())?; // Shared shutdown bool let shutdown = Arc::new(Mutex::new(false)); diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 482e0f54..7907e2cc 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -54,12 +54,7 @@ def __init__( ) # We do not use from_pretrained as we modified the model internal module layout - try: - filenames = weight_files(model_id, revision, ".bin") - # Local files not found - except LocalEntryNotFoundError: - hub_files = weight_hub_files(model_id, revision, ".bin") - filenames = download_weights(hub_files, model_id, revision) + filenames = weight_files(model_id, revision, ".safetensors") with init_empty_weights(): model = FlashSantacoderForCausalLM(config) @@ -91,85 +86,100 @@ def load_weights( transpose: bool, ): for filename in filenames: - state_dict = torch.load(filename, map_location="cpu") - for key, value in state_dict.items(): - value = value.to(device if quantize is None else "cpu").to(dtype) - - layer_name = ".".join(key.split(".")[:4]) - - # Fused qkv - if "q_attn.weight" in key or "kv_attn.weight" in key: - final_key = layer_name + ".c_attn.weight" - elif "q_attn.bias" in key or "kv_attn.bias" in key: - final_key = layer_name + ".c_attn.bias" - - else: - final_key = key - - module_name, param_name = final_key.rsplit(".", 1) - module = model.get_submodule(module_name) - - try: - current_parameter_tensor = module._parameters[param_name] - except KeyError: - current_parameter_tensor = None - - if current_parameter_tensor is not None: - if transpose and ( - "c_fc.weight" in key - or "c_proj.weight" in key - or "q_attn.weight" in key - or "kv_attn.weight" in key - or "c_attn.weight" in key - ): - # Tranpose as we use nn.Linear instead of Conv1D - value = value.T - - if current_parameter_tensor.device == torch.device("meta"): - # Init qkv - if "c_attn.weight" in final_key: - module._parameters[param_name] = value.new_empty( - ( - model.transformer.head_size - * (model.transformer.num_heads + 2), - value.shape[1], + with safe_open( + filename, framework="pt", device=str(device) if quantize is None else "cpu" + ) as f: + for key in f.keys(): + value = f.get_tensor(key) + value = value.to(device if quantize is None else "cpu").to(dtype) + + layer_name = ".".join(key.split(".")[:4]) + + # Fused qkv + if "q_attn.weight" in key or "kv_attn.weight" in key: + final_key = layer_name + ".c_attn.weight" + elif "q_attn.bias" in key or "kv_attn.bias" in key: + final_key = layer_name + ".c_attn.bias" + + else: + final_key = key + + module_name, param_name = final_key.rsplit(".", 1) + module = model.get_submodule(module_name) + + try: + current_parameter_tensor = module._parameters[param_name] + except KeyError: + current_parameter_tensor = None + + if current_parameter_tensor is not None: + if transpose and ( + "c_fc.weight" in key + or "c_proj.weight" in key + or "q_attn.weight" in key + or "kv_attn.weight" in key + or "c_attn.weight" in key + ): + # Tranpose as we use nn.Linear instead of Conv1D + value = value.T + + if current_parameter_tensor.device == torch.device("meta"): + # Init qkv + if "c_attn.weight" in final_key: + module._parameters[param_name] = value.new_empty( + ( + model.transformer.head_size + * (model.transformer.num_heads + 2), + value.shape[1], + ) ) - ) - elif "c_attn.bias" in final_key: - module._parameters[param_name] = value.new_empty( - ( - model.transformer.head_size - * (model.transformer.num_heads + 2) + elif "c_attn.bias" in final_key: + module._parameters[param_name] = value.new_empty( + ( + model.transformer.head_size + * (model.transformer.num_heads + 2) + ) ) - ) - # Copy to correct slice - if "q_attn.weight" in key: - module._parameters[param_name][: value.shape[0]] = value - elif "q_attn.bias" in key: - module._parameters[param_name][: value.shape[0]] = value - elif "kv_attn.weight" in key: - module._parameters[param_name][ - model.transformer.head_size * model.transformer.num_heads : - ] = value - elif "kv_attn.bias" in key: - module._parameters[param_name][ - model.transformer.head_size * model.transformer.num_heads : - ] = value + # Copy to correct slice + if "q_attn.weight" in key: + module._parameters[param_name][: value.shape[0]] = value + elif "q_attn.bias" in key: + module._parameters[param_name][: value.shape[0]] = value + elif "kv_attn.weight" in key: + module._parameters[param_name][ + model.transformer.head_size * model.transformer.num_heads : + ] = value + elif "kv_attn.bias" in key: + module._parameters[param_name][ + model.transformer.head_size * model.transformer.num_heads : + ] = value + else: + if current_parameter_tensor.shape != value.shape: + raise ValueError( + f"Name {final_key} -- Current {current_parameter_tensor.shape} and got {value.shape}" + ) + module._parameters[param_name] = value else: - if current_parameter_tensor.shape != value.shape: - raise ValueError( - f"Name {final_key} -- Current {current_parameter_tensor.shape} and got {value.shape}" - ) - module._parameters[param_name] = value - else: - module._buffers[param_name] = value + module._buffers[param_name] = value - del value + del value + + if model.lm_head.weight.device == torch.device("meta"): + model.lm_head.weight = torch.nn.Parameter(model.transformer.wte.weight) torch.cuda.empty_cache() model.post_load_weights(quantize) + uninitialized_parameters = [] + for n, p in model.named_parameters(): + if p.data.device == torch.device("meta"): + uninitialized_parameters.append(n) + if uninitialized_parameters: + raise RuntimeError( + f"found uninitialized parameters in model : {uninitialized_parameters}" + ) + def decode(self, generated_ids: List[int]) -> str: # Do not skip special tokens as they are used for custom parsing rules of the generated text return self.tokenizer.decode( @@ -389,6 +399,8 @@ def load_weights( else: module._buffers[param_name] = tensor - model.lm_head.weight = torch.nn.Parameter(model.transformer.wte.weight) + if model.lm_head.weight.device == torch.device("meta"): + model.lm_head.weight = torch.nn.Parameter(model.transformer.wte.weight) + torch.cuda.empty_cache() model.post_load_weights(quantize) From e7248fe90e27c7c8e39dd4cac5874eb9f96ab182 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 1 Jun 2023 19:49:13 +0200 Subject: [PATCH 269/793] v0.8.2 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- docs/openapi.json | 2 +- server/pyproject.toml | 2 +- server/text_generation_server/models/__init__.py | 4 +++- .../text_generation_server/models/flash_santacoder.py | 10 +++++++--- 6 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67ad8674..bd5994a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2557,7 +2557,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "0.8.1" +version = "0.8.2" dependencies = [ "average", "clap", @@ -2577,7 +2577,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.8.1" +version = "0.8.2" dependencies = [ "futures", "grpc-metadata", @@ -2593,7 +2593,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.8.1" +version = "0.8.2" dependencies = [ "clap", "ctrlc", @@ -2609,7 +2609,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.8.1" +version = "0.8.2" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index 3190b64c..b28286fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.8.1" +version = "0.8.2" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index 040c8e8b..e5ef0e3c 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.8.1" + "version": "0.8.2" }, "paths": { "/": { diff --git a/server/pyproject.toml b/server/pyproject.toml index 7400a055..d381eac4 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.8.1" +version = "0.8.2" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 78b68721..fc92d03d 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -138,7 +138,9 @@ def get_model( trust_remote_code=trust_remote_code, ) - config_dict, _ = PretrainedConfig.get_config_dict(model_id, revision=revision, trust_remote_code=trust_remote_code) + config_dict, _ = PretrainedConfig.get_config_dict( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) model_type = config_dict["model_type"] if model_type == "gpt_bigcode": diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 7907e2cc..e1c893d0 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -87,7 +87,9 @@ def load_weights( ): for filename in filenames: with safe_open( - filename, framework="pt", device=str(device) if quantize is None else "cpu" + filename, + framework="pt", + device=str(device) if quantize is None else "cpu", ) as f: for key in f.keys(): value = f.get_tensor(key) @@ -148,11 +150,13 @@ def load_weights( module._parameters[param_name][: value.shape[0]] = value elif "kv_attn.weight" in key: module._parameters[param_name][ - model.transformer.head_size * model.transformer.num_heads : + model.transformer.head_size + * model.transformer.num_heads : ] = value elif "kv_attn.bias" in key: module._parameters[param_name][ - model.transformer.head_size * model.transformer.num_heads : + model.transformer.head_size + * model.transformer.num_heads : ] = value else: if current_parameter_tensor.shape != value.shape: From 62fc40103079bc27e97194ef69e9e34a180b0a85 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 2 Jun 2023 09:51:06 +0200 Subject: [PATCH 270/793] feat(sagemaker): add trust remote code to entrypoint (#394) --- sagemaker-entrypoint.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sagemaker-entrypoint.sh b/sagemaker-entrypoint.sh index 1340f55a..9ac47010 100755 --- a/sagemaker-entrypoint.sh +++ b/sagemaker-entrypoint.sh @@ -18,4 +18,8 @@ if [[ -n "${HF_MODEL_QUANTIZE}" ]]; then export QUANTIZE="${HF_MODEL_QUANTIZE}" fi +if [[ -n "${HF_MODEL_TRUST_REMOTE_CODE}" ]]; then + export TRUST_REMOTE_CODE="${HF_MODEL_TRUST_REMOTE_CODE}" +fi + text-generation-launcher --port 8080 From 83b84486ad9e35bff1d5f2d166db296b767542c7 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 2 Jun 2023 14:17:27 +0200 Subject: [PATCH 271/793] feat(launcher): parse oom signal (#404) --- launcher/src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 7ee8bf1b..f59ff685 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -410,9 +410,14 @@ fn shard_manager( let mut wait_time = Instant::now(); loop { // Process exited - if p.poll().is_some() { + if let Some(exit_status) = p.poll() { let mut err = String::new(); p.stderr.take().unwrap().read_to_string(&mut err).unwrap(); + + if let ExitStatus::Signaled(signal) = exit_status { + tracing::error!("Shard process was signaled to shutdown with signal {signal}"); + } + status_sender .send(ShardStatus::Failed((rank, err))) .unwrap(); From 895c5f15628df870f7a2ced7151dedb84231a996 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 2 Jun 2023 17:12:30 +0200 Subject: [PATCH 272/793] feat(server): only compute prefill logprobs when asked (#406) Close #288 --- Makefile | 1 + benchmark/src/generation.rs | 1 + clients/python/README.md | 46 +++++- clients/python/pyproject.toml | 2 +- clients/python/tests/test_client.py | 22 +-- clients/python/text_generation/client.py | 10 ++ clients/python/text_generation/types.py | 14 +- integration-tests/conftest.py | 9 +- integration-tests/models/test_bloom_560m.py | 2 + .../models/test_bloom_560m_sharded.py | 1 + integration-tests/models/test_flash_falcon.py | 2 + integration-tests/models/test_flash_llama.py | 5 +- integration-tests/models/test_flash_neox.py | 1 + .../models/test_flash_neox_sharded.py | 1 + .../models/test_flash_santacoder.py | 4 +- .../models/test_flash_starcoder.py | 11 +- integration-tests/models/test_mt0_base.py | 2 + integration-tests/models/test_t5_sharded.py | 1 + integration-tests/requirements.txt | 2 +- proto/generate.proto | 2 + router/src/health.rs | 1 + router/src/lib.rs | 4 + router/src/queue.rs | 2 + router/src/server.rs | 19 ++- router/src/validation.rs | 5 + server/tests/models/test_bloom.py | 1 + server/tests/models/test_causal_lm.py | 1 + server/tests/models/test_santacoder.py | 2 + server/tests/models/test_seq2seq_lm.py | 1 + .../models/causal_lm.py | 4 +- .../custom_modeling/flash_llama_modeling.py | 3 + .../custom_modeling/flash_neox_modeling.py | 3 + .../custom_modeling/flash_rw_modeling.py | 3 + .../flash_santacoder_modeling.py | 3 + .../models/flash_causal_lm.py | 132 +++++++++++++----- .../models/seq2seq_lm.py | 2 +- 36 files changed, 252 insertions(+), 73 deletions(-) diff --git a/Makefile b/Makefile index 7309aaee..a33aba17 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ install-server: install-integration-tests: cd integration-tests && pip install -r requirements.txt + cd clients/python && pip install . install-router: cd router && cargo install --path . diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs index 17c72d26..b57c652b 100644 --- a/benchmark/src/generation.rs +++ b/benchmark/src/generation.rs @@ -136,6 +136,7 @@ async fn prefill( let requests = (0..batch_size) .map(|id| Request { id: id.into(), + prefill_logprobs: false, inputs: sequence.clone(), truncate: sequence_length, parameters: Some(parameters.clone()), diff --git a/clients/python/README.md b/clients/python/README.md index 99ff185a..4e0e564c 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -107,8 +107,42 @@ print(text) ### Types ```python -# Prompt tokens -class PrefillToken: +# Request Parameters +class Parameters: + # Activate logits sampling + do_sample: bool + # Maximum number of generated tokens + max_new_tokens: int + # The parameter for repetition penalty. 1.0 means no penalty. + # See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + repetition_penalty: Optional[float] + # Whether to prepend the prompt to the generated text + return_full_text: bool + # Stop generating tokens if a member of `stop_sequences` is generated + stop: List[str] + # Random sampling seed + seed: Optional[int] + # The value used to module the logits distribution. + temperature: Optional[float] + # The number of highest probability vocabulary tokens to keep for top-k-filtering. + top_k: Optional[int] + # If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or + # higher are kept for generation. + top_p: Optional[float] + # truncate inputs tokens to the given size + truncate: Optional[int] + # Typical Decoding mass + # See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information + typical_p: Optional[float] + # Generate best_of sequences and return the one if the highest token logprobs + best_of: Optional[int] + # Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + watermark: bool + # Get decoder input token logprobs and ids + decoder_input_details: bool + +# Decoder input tokens +class InputToken: # Token ID from the model tokenizer id: int # Token text @@ -151,8 +185,8 @@ class BestOfSequence: generated_tokens: int # Sampling seed if sampling was activated seed: Optional[int] - # Prompt tokens - prefill: List[PrefillToken] + # Decoder input tokens, empty if decoder_input_details is False + prefill: List[InputToken] # Generated tokens tokens: List[Token] @@ -165,8 +199,8 @@ class Details: generated_tokens: int # Sampling seed if sampling was activated seed: Optional[int] - # Prompt tokens - prefill: List[PrefillToken] + # Decoder input tokens, empty if decoder_input_details is False + prefill: List[InputToken] # Generated tokens tokens: List[Token] # Additional sequences when using the `best_of` parameter diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 06d5f9cb..a52bdd81 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.5.2" +version = "0.6.0" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 32462f14..1e25e1b1 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -2,28 +2,30 @@ from text_generation import Client, AsyncClient from text_generation.errors import NotFoundError, ValidationError -from text_generation.types import FinishReason, PrefillToken, Token +from text_generation.types import FinishReason, InputToken def test_generate(flan_t5_xxl_url, hf_headers): client = Client(flan_t5_xxl_url, hf_headers) - response = client.generate("test", max_new_tokens=1) + response = client.generate("test", max_new_tokens=1, decoder_input_details=True) assert response.generated_text == "" assert response.details.finish_reason == FinishReason.Length assert response.details.generated_tokens == 1 assert response.details.seed is None assert len(response.details.prefill) == 1 - assert response.details.prefill[0] == PrefillToken(id=0, text="", logprob=None) + assert response.details.prefill[0] == InputToken(id=0, text="", logprob=None) assert len(response.details.tokens) == 1 assert response.details.tokens[0].id == 3 - assert response.details.tokens[0].text == "" + assert response.details.tokens[0].text == " " assert not response.details.tokens[0].special def test_generate_best_of(flan_t5_xxl_url, hf_headers): client = Client(flan_t5_xxl_url, hf_headers) - response = client.generate("test", max_new_tokens=1, best_of=2, do_sample=True) + response = client.generate( + "test", max_new_tokens=1, best_of=2, do_sample=True, decoder_input_details=True + ) assert response.details.seed is not None assert response.details.best_of_sequences is not None @@ -73,17 +75,19 @@ def test_generate_stream_validation_error(flan_t5_xxl_url, hf_headers): @pytest.mark.asyncio async def test_generate_async(flan_t5_xxl_url, hf_headers): client = AsyncClient(flan_t5_xxl_url, hf_headers) - response = await client.generate("test", max_new_tokens=1) + response = await client.generate( + "test", max_new_tokens=1, decoder_input_details=True + ) assert response.generated_text == "" assert response.details.finish_reason == FinishReason.Length assert response.details.generated_tokens == 1 assert response.details.seed is None assert len(response.details.prefill) == 1 - assert response.details.prefill[0] == PrefillToken(id=0, text="", logprob=None) + assert response.details.prefill[0] == InputToken(id=0, text="", logprob=None) assert len(response.details.tokens) == 1 assert response.details.tokens[0].id == 3 - assert response.details.tokens[0].text == "" + assert response.details.tokens[0].text == " " assert not response.details.tokens[0].special @@ -91,7 +95,7 @@ async def test_generate_async(flan_t5_xxl_url, hf_headers): async def test_generate_async_best_of(flan_t5_xxl_url, hf_headers): client = AsyncClient(flan_t5_xxl_url, hf_headers) response = await client.generate( - "test", max_new_tokens=1, best_of=2, do_sample=True + "test", max_new_tokens=1, best_of=2, do_sample=True, decoder_input_details=True ) assert response.details.seed is not None diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index 8b8742fc..bf045d47 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -74,6 +74,7 @@ def generate( truncate: Optional[int] = None, typical_p: Optional[float] = None, watermark: bool = False, + decoder_input_details: bool = False, ) -> Response: """ Given a prompt, generate the following text @@ -110,6 +111,8 @@ def generate( See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + decoder_input_details (`bool`): + Return the decoder input token logprobs and ids Returns: Response: generated response @@ -130,6 +133,7 @@ def generate( truncate=truncate, typical_p=typical_p, watermark=watermark, + decoder_input_details=decoder_input_details, ) request = Request(inputs=prompt, stream=False, parameters=parameters) @@ -202,6 +206,7 @@ def generate_stream( parameters = Parameters( best_of=None, details=True, + decoder_input_details=False, do_sample=do_sample, max_new_tokens=max_new_tokens, repetition_penalty=repetition_penalty, @@ -311,6 +316,7 @@ async def generate( truncate: Optional[int] = None, typical_p: Optional[float] = None, watermark: bool = False, + decoder_input_details: bool = False, ) -> Response: """ Given a prompt, generate the following text asynchronously @@ -347,6 +353,8 @@ async def generate( See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + decoder_input_details (`bool`): + Return the decoder input token logprobs and ids Returns: Response: generated response @@ -355,6 +363,7 @@ async def generate( parameters = Parameters( best_of=best_of, details=True, + decoder_input_details=decoder_input_details, do_sample=do_sample, max_new_tokens=max_new_tokens, repetition_penalty=repetition_penalty, @@ -437,6 +446,7 @@ async def generate_stream( parameters = Parameters( best_of=None, details=True, + decoder_input_details=False, do_sample=do_sample, max_new_tokens=max_new_tokens, repetition_penalty=repetition_penalty, diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index ad3cd09b..548f0b63 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -37,6 +37,8 @@ class Parameters(BaseModel): watermark: bool = False # Get generation details details: bool = False + # Get decoder input token logprobs and ids + decoder_input_details: bool = False @validator("best_of") def valid_best_of(cls, field_value, values): @@ -129,8 +131,8 @@ def valid_best_of_stream(cls, field_value, values): return field_value -# Prompt tokens -class PrefillToken(BaseModel): +# Decoder input tokens +class InputToken(BaseModel): # Token ID from the model tokenizer id: int # Token text @@ -173,8 +175,8 @@ class BestOfSequence(BaseModel): generated_tokens: int # Sampling seed if sampling was activated seed: Optional[int] - # Prompt tokens - prefill: List[PrefillToken] + # Decoder input tokens, empty if decoder_input_details is False + prefill: List[InputToken] # Generated tokens tokens: List[Token] @@ -187,8 +189,8 @@ class Details(BaseModel): generated_tokens: int # Sampling seed if sampling was activated seed: Optional[int] - # Prompt tokens - prefill: List[PrefillToken] + # Decoder input tokens, empty if decoder_input_details is False + prefill: List[InputToken] # Generated tokens tokens: List[Token] # Additional sequences when using the `best_of` parameter diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 902a7158..82f1b719 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -16,7 +16,7 @@ from aiohttp import ClientConnectorError, ClientOSError, ServerDisconnectedError from text_generation import AsyncClient -from text_generation.types import Response, Details, PrefillToken, Token, BestOfSequence +from text_generation.types import Response, Details, InputToken, Token, BestOfSequence DOCKER_IMAGE = os.getenv("DOCKER_IMAGE", None) HUGGING_FACE_HUB_TOKEN = os.getenv("HUGGING_FACE_HUB_TOKEN", None) @@ -62,7 +62,7 @@ def eq_token(token: Token, other: Token) -> bool: and token.special == other.special ) - def eq_prefill_token(prefill_token: PrefillToken, other: PrefillToken) -> bool: + def eq_prefill_token(prefill_token: InputToken, other: InputToken) -> bool: try: return ( prefill_token.id == other.id @@ -332,7 +332,10 @@ async def generate_load_inner( client: AsyncClient, prompt: str, max_new_tokens: int, n: int ) -> List[Response]: futures = [ - client.generate(prompt, max_new_tokens=max_new_tokens) for _ in range(n) + client.generate( + prompt, max_new_tokens=max_new_tokens, decoder_input_details=True + ) + for _ in range(n) ] return await asyncio.gather(*futures) diff --git a/integration-tests/models/test_bloom_560m.py b/integration-tests/models/test_bloom_560m.py index 809250cb..bdcbdc78 100644 --- a/integration-tests/models/test_bloom_560m.py +++ b/integration-tests/models/test_bloom_560m.py @@ -19,6 +19,7 @@ async def test_bloom_560m(bloom_560, response_snapshot): "Pour déguster un ortolan, il faut tout d'abord", max_new_tokens=10, top_p=0.9, + decoder_input_details=True, seed=0, ) @@ -40,6 +41,7 @@ async def test_bloom_560m_all_params(bloom_560, response_snapshot): truncate=5, typical_p=0.9, watermark=True, + decoder_input_details=True, seed=0, ) diff --git a/integration-tests/models/test_bloom_560m_sharded.py b/integration-tests/models/test_bloom_560m_sharded.py index ee67250a..3995f9e5 100644 --- a/integration-tests/models/test_bloom_560m_sharded.py +++ b/integration-tests/models/test_bloom_560m_sharded.py @@ -19,6 +19,7 @@ async def test_bloom_560m_sharded(bloom_560m_sharded, response_snapshot): "Pour déguster un ortolan, il faut tout d'abord", max_new_tokens=10, top_p=0.9, + decoder_input_details=True, seed=0, ) diff --git a/integration-tests/models/test_flash_falcon.py b/integration-tests/models/test_flash_falcon.py index e36a6a28..eac91984 100644 --- a/integration-tests/models/test_flash_falcon.py +++ b/integration-tests/models/test_flash_falcon.py @@ -19,6 +19,7 @@ async def test_flash_falcon(flash_falcon, response_snapshot): response = await flash_falcon.generate( "Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Giraftron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\nDaniel: Hello, Girafatron!\nGirafatron:", max_new_tokens=10, + decoder_input_details=True, ) assert response.details.generated_tokens == 10 @@ -40,6 +41,7 @@ async def test_flash_falcon_all_params(flash_falcon, response_snapshot): truncate=5, typical_p=0.9, watermark=True, + decoder_input_details=True, seed=0, ) diff --git a/integration-tests/models/test_flash_llama.py b/integration-tests/models/test_flash_llama.py index edc847c1..c69314ff 100644 --- a/integration-tests/models/test_flash_llama.py +++ b/integration-tests/models/test_flash_llama.py @@ -16,7 +16,9 @@ async def flash_llama(flash_llama_handle): @pytest.mark.asyncio @pytest.mark.private async def test_flash_llama(flash_llama, response_snapshot): - response = await flash_llama.generate("Test request", max_new_tokens=10) + response = await flash_llama.generate( + "Test request", max_new_tokens=10, decoder_input_details=True + ) assert response.details.generated_tokens == 10 assert response == response_snapshot @@ -37,6 +39,7 @@ async def test_flash_llama_all_params(flash_llama, response_snapshot): truncate=5, typical_p=0.9, watermark=True, + decoder_input_details=True, seed=0, ) diff --git a/integration-tests/models/test_flash_neox.py b/integration-tests/models/test_flash_neox.py index daff7f0a..ff9b9763 100644 --- a/integration-tests/models/test_flash_neox.py +++ b/integration-tests/models/test_flash_neox.py @@ -18,6 +18,7 @@ async def test_flash_neox(flash_neox, response_snapshot): response = await flash_neox.generate( "<|USER|>What's your mood today?<|ASSISTANT|>", max_new_tokens=10, + decoder_input_details=True, ) assert response.details.generated_tokens == 10 diff --git a/integration-tests/models/test_flash_neox_sharded.py b/integration-tests/models/test_flash_neox_sharded.py index a1aa0f07..8a491915 100644 --- a/integration-tests/models/test_flash_neox_sharded.py +++ b/integration-tests/models/test_flash_neox_sharded.py @@ -18,6 +18,7 @@ async def test_flash_neox(flash_neox_sharded, response_snapshot): response = await flash_neox_sharded.generate( "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", max_new_tokens=10, + decoder_input_details=True, ) assert response.details.generated_tokens == 10 diff --git a/integration-tests/models/test_flash_santacoder.py b/integration-tests/models/test_flash_santacoder.py index a15a6439..0f005f15 100644 --- a/integration-tests/models/test_flash_santacoder.py +++ b/integration-tests/models/test_flash_santacoder.py @@ -15,7 +15,9 @@ async def flash_santacoder(flash_santacoder_handle): @pytest.mark.asyncio async def test_flash_santacoder(flash_santacoder, response_snapshot): - response = await flash_santacoder.generate("def print_hello", max_new_tokens=10) + response = await flash_santacoder.generate( + "def print_hello", max_new_tokens=10, decoder_input_details=True + ) assert response.details.generated_tokens == 10 assert response == response_snapshot diff --git a/integration-tests/models/test_flash_starcoder.py b/integration-tests/models/test_flash_starcoder.py index 72b298c9..64e8b27c 100644 --- a/integration-tests/models/test_flash_starcoder.py +++ b/integration-tests/models/test_flash_starcoder.py @@ -16,7 +16,9 @@ async def flash_starcoder(flash_starcoder_handle): @pytest.mark.asyncio @pytest.mark.private async def test_flash_starcoder(flash_starcoder, response_snapshot): - response = await flash_starcoder.generate("def print_hello", max_new_tokens=10) + response = await flash_starcoder.generate( + "def print_hello", max_new_tokens=10, decoder_input_details=True + ) assert response.details.generated_tokens == 10 assert response == response_snapshot @@ -26,7 +28,12 @@ async def test_flash_starcoder(flash_starcoder, response_snapshot): @pytest.mark.private async def test_flash_starcoder_default_params(flash_starcoder, response_snapshot): response = await flash_starcoder.generate( - "def print_hello", max_new_tokens=60, temperature=0.2, top_p=0.95, seed=0 + "def print_hello", + max_new_tokens=60, + temperature=0.2, + top_p=0.95, + decoder_input_details=True, + seed=0, ) assert response.details.generated_tokens == 60 diff --git a/integration-tests/models/test_mt0_base.py b/integration-tests/models/test_mt0_base.py index 4ed95aad..12f23e4c 100644 --- a/integration-tests/models/test_mt0_base.py +++ b/integration-tests/models/test_mt0_base.py @@ -19,6 +19,7 @@ async def test_mt0_base(mt0_base, response_snapshot): "Why is the sky blue?", max_new_tokens=10, top_p=0.9, + decoder_input_details=True, seed=0, ) @@ -40,6 +41,7 @@ async def test_mt0_base_all_params(mt0_base, response_snapshot): truncate=5, typical_p=0.9, watermark=True, + decoder_input_details=True, seed=0, ) diff --git a/integration-tests/models/test_t5_sharded.py b/integration-tests/models/test_t5_sharded.py index a2d84330..7c288b23 100644 --- a/integration-tests/models/test_t5_sharded.py +++ b/integration-tests/models/test_t5_sharded.py @@ -18,6 +18,7 @@ async def test_t5_sharded(t5_sharded, response_snapshot): response = await t5_sharded.generate( "Please answer the following question. What is the boiling point of Nitrogen?", max_new_tokens=10, + decoder_input_details=True, ) assert response == response_snapshot diff --git a/integration-tests/requirements.txt b/integration-tests/requirements.txt index 051730ff..2f36d5d6 100644 --- a/integration-tests/requirements.txt +++ b/integration-tests/requirements.txt @@ -1,5 +1,5 @@ syrupy -text-generation==0.5.2 +text-generation pytest pytest-asyncio==0.17.2 docker \ No newline at end of file diff --git a/proto/generate.proto b/proto/generate.proto index 0c40e5bb..a0f5a75e 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -87,6 +87,8 @@ message Request { NextTokenChooserParameters parameters = 4; /// Stopping Criteria Parameters StoppingCriteriaParameters stopping_parameters = 5; + /// Return prefill logprobs + bool prefill_logprobs = 6; } message Batch { diff --git a/router/src/health.rs b/router/src/health.rs index 45f50e9d..a3cacdcd 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -34,6 +34,7 @@ impl Health { id: LIVENESS_ID, inputs: "liveness".to_string(), truncate: 10, + prefill_logprobs: false, parameters: Some(NextTokenChooserParameters { temperature: 1.0, top_k: 0, diff --git a/router/src/lib.rs b/router/src/lib.rs index 080dc4f4..67fff017 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -125,6 +125,9 @@ pub(crate) struct GenerateParameters { #[schema(default = "true")] pub details: bool, #[serde(default)] + #[schema(default = "true")] + pub decoder_input_details: bool, + #[serde(default)] #[schema( exclusive_minimum = 0, nullable = true, @@ -153,6 +156,7 @@ fn default_parameters() -> GenerateParameters { truncate: None, watermark: false, details: false, + decoder_input_details: false, seed: None, } } diff --git a/router/src/queue.rs b/router/src/queue.rs index 94851e1c..03807933 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -201,6 +201,7 @@ impl State { batch_requests.push(Request { id, + prefill_logprobs: entry.request.decoder_input_details, inputs: entry.request.inputs.clone(), truncate: entry.request.truncate, parameters: Some(entry.request.parameters.clone()), @@ -281,6 +282,7 @@ mod tests { inputs: "".to_string(), input_length: 0, truncate: 0, + decoder_input_details: false, parameters: NextTokenChooserParameters { temperature: 0.0, top_k: 0, diff --git a/router/src/server.rs b/router/src/server.rs index fd6a66bb..10c0ba3c 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -160,7 +160,7 @@ async fn generate( add_prompt = Some(req.0.inputs.clone()); } - let details = req.0.parameters.details; + let details = req.0.parameters.details || req.0.parameters.decoder_input_details; // Inference let (response, best_of_responses) = match req.0.parameters.best_of { @@ -364,7 +364,17 @@ async fn generate_stream( let details = req.0.parameters.details; let best_of = req.0.parameters.best_of.unwrap_or(1); - if best_of == 1 { + if best_of != 1 { + let err = InferError::from(ValidationError::BestOfStream); + metrics::increment_counter!("tgi_request_failure", "err" => "validation"); + tracing::error!("{err}"); + yield Ok(Event::from(err)); + } else if req.0.parameters.decoder_input_details { + let err = InferError::from(ValidationError::PrefillDetailsStream); + metrics::increment_counter!("tgi_request_failure", "err" => "validation"); + tracing::error!("{err}"); + yield Ok(Event::from(err)); + } else { match infer.generate_stream(req.0).instrument(info_span!(parent: &span, "async_stream")).await { // Keep permit as long as generate_stream lives Ok((_permit, mut response_stream)) => { @@ -474,11 +484,6 @@ async fn generate_stream( tracing::error!("{err}"); yield Ok(Event::from(err)); } - } else { - let err = InferError::from(ValidationError::BestOfStream); - metrics::increment_counter!("tgi_request_failure", "err" => "validation"); - tracing::error!("{err}"); - yield Ok(Event::from(err)); } }; diff --git a/router/src/validation.rs b/router/src/validation.rs index cbb0d9cd..8843c6a8 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -145,6 +145,7 @@ impl Validation { truncate, seed, watermark, + decoder_input_details, .. } = request.parameters; @@ -261,6 +262,7 @@ impl Validation { Ok(ValidGenerateRequest { inputs, + decoder_input_details, input_length: input_length as u32, truncate: truncate.unwrap_or(self.max_input_length) as u32, parameters, @@ -335,6 +337,7 @@ pub(crate) struct ValidGenerateRequest { pub inputs: String, pub input_length: u32, pub truncate: u32, + pub decoder_input_details: bool, pub parameters: NextTokenChooserParameters, pub stopping_parameters: StoppingCriteriaParameters, } @@ -351,6 +354,8 @@ pub enum ValidationError { BestOfSeed, #[error("`best_of` != 1 is not supported when streaming tokens")] BestOfStream, + #[error("`decoder_input_details` == true is not supported when streaming tokens")] + PrefillDetailsStream, #[error("`temperature` must be strictly positive")] Temperature, #[error("`repetition_penalty` must be strictly positive")] diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 590ba557..338fe053 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -24,6 +24,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", + prefill_logprobs=True, truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 3f28f5b3..0f9dab2c 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -25,6 +25,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", + prefill_logprobs=True, truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index bef8db38..fceec560 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -15,6 +15,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="def", + prefill_logprobs=True, truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, @@ -31,6 +32,7 @@ def default_fim_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="defworld", + prefill_logprobs=True, truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index a3199d02..299340f8 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -28,6 +28,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters): return generate_pb2.Request( id=0, inputs="Test", + prefill_logprobs=True, truncate=100, parameters=default_pb_parameters, stopping_parameters=default_pb_stop_parameters, diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 92622350..ba0853f5 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -104,7 +104,7 @@ def from_pb( ).to(device) for _ in pb.requests: input_len = tokenized_inputs["input_ids"].shape[1] - prefix_offsets.append(0) + prefix_offsets.append(input_len - 5) read_offsets.append(input_len) input_lengths = tokenized_inputs["attention_mask"].sum(1) @@ -617,7 +617,7 @@ def generate_token( generated_text = None # Prefill - if stopping_criteria.current_tokens == 1: + if stopping_criteria.current_tokens == 1 and request.prefill_logprobs: # Remove generated token to only have prefill and add nan for first prompt token prefill_logprobs = [float("nan")] + torch.log_softmax( logits, -1 diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 2dcb6ed8..f4116937 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -443,6 +443,7 @@ def forward( max_s, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, + lm_head_indices: Optional[torch.Tensor] = None, ): hidden_states, present = self.model( input_ids, @@ -453,6 +454,8 @@ def forward( past_key_values, pre_allocate_past_size, ) + if lm_head_indices is not None: + hidden_states = hidden_states[lm_head_indices] logits = self.lm_head(hidden_states) if self.model.tp_embeddings: diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 26e21753..b798750a 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -481,6 +481,7 @@ def forward( max_s, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, + lm_head_indices: Optional[torch.Tensor] = None, ): hidden_states, present = self.gpt_neox( input_ids, @@ -491,6 +492,8 @@ def forward( past_key_values, pre_allocate_past_size, ) + if lm_head_indices is not None: + hidden_states = hidden_states[lm_head_indices] logits = self.embed_out(hidden_states) if self.gpt_neox.tp_embeddings: diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 545da26a..03487703 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -752,6 +752,7 @@ def forward( max_s, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, + lm_head_indices: Optional[torch.Tensor] = None, ): hidden_states, present = self.transformer( input_ids, @@ -762,6 +763,8 @@ def forward( past_key_values, pre_allocate_past_size, ) + if lm_head_indices is not None: + hidden_states = hidden_states[lm_head_indices] logits = self.lm_head(hidden_states) if self.transformer.tp_embeddings: diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 9bded805..b61ec873 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -358,6 +358,7 @@ def forward( max_s, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, + lm_head_indices: Optional[torch.Tensor] = None, ): hidden_states, present = self.transformer( input_ids, @@ -368,6 +369,8 @@ def forward( past_key_values, pre_allocate_past_size, ) + if lm_head_indices is not None: + hidden_states = hidden_states[lm_head_indices] logits = self.lm_head(hidden_states) if self.transformer.tp_embeddings: diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 35cbe174..5ff951b3 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -42,6 +42,11 @@ class FlashCausalLMBatch(Batch): past_key_values: Optional[torch.Tensor] max_seqlen: int + # Prefill metadata tensors to efficiently compute logprobs + prefill_head_indices: Optional[torch.Tensor] + prefill_next_token_indices: Optional[torch.tensor] + prefill_cu_outlens: Optional[List[int]] + # All tokens all_input_ids: List[List[int]] all_input_ids_tensor: torch.Tensor @@ -84,11 +89,18 @@ def from_pb( all_input_ids = [] requests_idx_mapping = {} + all_prefill_logprobs = True + no_prefill_logprobs = True + prefill_head_indices = [] + prefill_next_token_indices = [] + prefill_cu_outlens = [0] + next_token_chooser_parameters = [] stopping_criterias = [] # Cumulative length cumulative_length = 0 + prefill_out_cumulative_length = 0 max_tokens = 0 max_length = 0 @@ -106,13 +118,14 @@ def from_pb( max_seqlen = max(max_seqlen, input_length) input_lengths.append(input_length) - prefix_offsets.append(0) + prefix_offsets.append(input_length - 5) read_offsets.append(input_length) all_input_ids.append(tokenized_input) # Position ids - position_ids.append(np.arange(0, input_length)) + request_position_ids = torch.arange(0, input_length, dtype=torch.int32) + position_ids.append(request_position_ids) # Add cumulative lengths of all previous inputs cu_seqlens.append(cumulative_length + input_length) @@ -125,6 +138,26 @@ def from_pb( max_new_tokens = stopping_criteria.max_new_tokens stopping_criterias.append(stopping_criteria) + all_prefill_logprobs = all_prefill_logprobs and r.prefill_logprobs + no_prefill_logprobs = no_prefill_logprobs and not r.prefill_logprobs + + if r.prefill_logprobs: + prefill_head_indices.append(request_position_ids + cumulative_length) + prefill_next_token_indices.append( + prefill_out_cumulative_length + input_length - 1 + ) + prefill_cu_outlens.append(prefill_out_cumulative_length + input_length) + prefill_out_cumulative_length += input_length + else: + prefill_head_indices.append( + torch.tensor( + [cumulative_length + input_length - 1], dtype=torch.int32 + ) + ) + prefill_next_token_indices.append(prefill_out_cumulative_length) + prefill_cu_outlens.append(prefill_out_cumulative_length + 1) + prefill_out_cumulative_length += 1 + # Update cumulative_length += input_length max_tokens += input_length + max_new_tokens @@ -141,18 +174,35 @@ def from_pb( for i, input_ids in enumerate(all_input_ids): all_input_ids_tensor[i, : len(input_ids)] = input_ids + if len(pb.requests) > 1: + input_ids = np.concatenate(all_input_ids, dtype=np.int64) + position_ids = torch.cat(position_ids) + else: + input_ids = all_input_ids[0] + position_ids = position_ids[0] + # Create tensors on device - input_ids = torch.tensor( - np.concatenate(all_input_ids), dtype=torch.int64, device=device - ) + input_ids = torch.tensor(input_ids, dtype=torch.int64, device=device) all_input_ids_tensor = torch.tensor( all_input_ids_tensor, dtype=torch.int64, device=device ) - position_ids = torch.tensor( - np.concatenate(position_ids), dtype=torch.int32, device=device - ) + position_ids = torch.tensor(position_ids, dtype=torch.int32, device=device) cu_seqlens = torch.tensor(cu_seqlens, device=device, dtype=torch.int32) + if all_prefill_logprobs: + prefill_head_indices = None + prefill_next_token_indices = cu_seqlens[1:] - 1 + elif no_prefill_logprobs: + prefill_head_indices = cu_seqlens[1:] - 1 + prefill_next_token_indices = None + else: + prefill_head_indices = torch.tensor( + torch.cat(prefill_head_indices), dtype=torch.int64, device=device + ) + prefill_next_token_indices = torch.tensor( + prefill_next_token_indices, dtype=torch.int64, device=device + ) + return cls( batch_id=pb.id, requests=pb.requests, @@ -162,6 +212,9 @@ def from_pb( cu_seqlens=cu_seqlens, cu_seqlens_q=None, max_seqlen=max_seqlen, + prefill_head_indices=prefill_head_indices, + prefill_next_token_indices=prefill_next_token_indices, + prefill_cu_outlens=prefill_cu_outlens, past_key_values=None, input_lengths=input_lengths, prefix_offsets=prefix_offsets, @@ -280,6 +333,9 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": cu_seqlens=cu_seqlens, cu_seqlens_q=cu_seqlens_q, max_seqlen=max_seqlen, + prefill_head_indices=None, + prefill_next_token_indices=None, + prefill_cu_outlens=None, past_key_values=past_key_values, input_lengths=input_lengths, prefix_offsets=prefix_offsets, @@ -415,6 +471,9 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch cu_seqlens=cu_seqlens, cu_seqlens_q=cu_seqlens_q, max_seqlen=max_seqlen, + prefill_head_indices=None, + prefill_next_token_indices=None, + prefill_cu_outlens=None, past_key_values=past_key_values, input_lengths=input_lengths, prefix_offsets=prefix_offsets, @@ -486,6 +545,7 @@ def forward( max_s: int, past_key_values: Optional = None, pre_allocate_past_size: Optional[int] = None, + lm_head_indices: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: # Model Forward return self.model.forward( @@ -496,6 +556,7 @@ def forward( max_s=max_s, past_key_values=past_key_values, pre_allocate_past_size=pre_allocate_past_size, + lm_head_indices=lm_head_indices, ) @tracer.start_as_current_span("generate_token") @@ -503,9 +564,10 @@ def generate_token( self, batch: FlashCausalLMBatch ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: prefill = batch.past_key_values is None + prefill_logprobs = batch.prefill_next_token_indices is not None single_request = len(batch) == 1 - if prefill and len(batch) == 1: + if prefill and single_request: # Ask to pre-allocate kv to its max size # == number of tokens + max_new_tokens pre_allocate_past_size = ( @@ -522,11 +584,12 @@ def generate_token( batch.max_seqlen, batch.past_key_values, pre_allocate_past_size, + batch.prefill_head_indices, ) if prefill: next_token_logits = ( - out[-1:] if single_request else out[batch.cu_seqlens[1:] - 1] + out[batch.prefill_next_token_indices] if prefill_logprobs else out ) else: next_token_logits = out @@ -536,10 +599,10 @@ def generate_token( ) if prefill: - if len(batch) > 1: + if len(batch) > 1 and prefill_logprobs: # We create the prefill_tokens_indices tensor that will be used to gather prefill logprobs # When batch == 1, we will just use the batch.input_ids values directly - prefill_tokens_indices = batch.input_ids.new_zeros(len(batch.input_ids)) + prefill_tokens_indices = batch.input_ids.new_zeros(len(out)) # Create batch.cu_seqlens_q for decode batch.cu_seqlens_q = torch.arange( @@ -600,7 +663,6 @@ def generate_token( # Zipped iterator iterator = zip( batch.input_lengths, - batch.stopping_criterias, batch.all_input_ids, ) @@ -611,29 +673,33 @@ def generate_token( # For each member of the batch for i, ( input_length, - stopping_criteria, all_input_ids, ) in enumerate(iterator): - # Indexing metadata start_index = cumulative_length end_index = cumulative_length + input_length if prefill: + # Indexing metadata + out_start_index = batch.prefill_cu_outlens[i] + out_end_index = batch.prefill_cu_outlens[i + 1] + out_length = out_end_index - out_start_index + # Initialize position_ids # In decode, we do not need this as we can just increment position ids next_position_ids[i] = batch.position_ids[end_index - 1] # Used to gather prefill logprobs # Copy batch.input_ids to prefill_token_indices - if len(batch) > 1: - prefill_tokens_indices[ - start_index : end_index - 1 - ] = batch.input_ids[start_index + 1 : end_index] - else: - # Set prefill_tokens_indices to the correct slice - prefill_tokens_indices = batch.input_ids[ - start_index + 1 : end_index - ] + if prefill_logprobs: + if len(batch) > 1: + prefill_tokens_indices[ + out_start_index : out_end_index - 1 + ] = batch.input_ids[start_index + 1 : start_index + out_length] + else: + # Set prefill_tokens_indices to the correct slice + prefill_tokens_indices = batch.input_ids[ + start_index + 1 : start_index + out_length + ] batch.all_input_ids_tensor[i, input_length] = next_input_ids[i] @@ -644,7 +710,7 @@ def generate_token( batch.position_ids = next_position_ids + 1 batch.cu_seqlens = batch.cu_seqlens + batch.cu_seqlens_q - if prefill: + if prefill and prefill_logprobs: # Get prefill logprobs prefill_logprobs_tensor = torch.log_softmax(out, -1) prefill_logprobs = torch.gather( @@ -657,8 +723,6 @@ def generate_token( next_token_logprobs = next_token_logprobs.tolist() next_token_ids = batch.input_ids.tolist() - cumulative_length = 0 - # Zipped iterator iterator = zip( batch.requests, @@ -688,9 +752,6 @@ def generate_token( next_token_id, next_token_logprob, ) in enumerate(iterator): - start_index = cumulative_length - end_index = cumulative_length + input_length - # Append next token to all tokens all_input_ids.append(next_token_id) @@ -728,10 +789,13 @@ def generate_token( generated_text = None # Prefill - if prefill: + if prefill and request.prefill_logprobs: + out_start_index = batch.prefill_cu_outlens[i] + out_end_index = batch.prefill_cu_outlens[i + 1] + # Remove generated token to only have prefill and add nan for first prompt token request_prefill_logprobs = [float("nan")] + prefill_logprobs[ - start_index : end_index - 1 + out_start_index : out_end_index - 1 ] prefill_token_ids = all_input_ids[:-1] prefill_texts = self.tokenizer.batch_decode( @@ -764,8 +828,10 @@ def generate_token( batch.prefix_offsets[i] = prefix_offset batch.read_offsets[i] = read_offset batch.all_input_ids[i] = all_input_ids - cumulative_length += input_length + batch.prefill_cu_outlens = None + batch.prefill_head_indices = None + batch.prefill_next_token_indices = None batch.max_seqlen = batch.max_seqlen + 1 # No need to return a batch if we know that all requests stopped diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 68e59dc3..3ad5698c 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -688,7 +688,7 @@ def generate_token( generated_text = None # Prefill - if stopping_criteria.current_tokens == 1: + if stopping_criteria.current_tokens == 1 and request.prefill_logprobs: prefill_tokens = PrefillTokens( [self.tokenizer.bos_token_id], [float("nan")], From 6abec14a7eeb6e29a394557d64e2b527af1a89fb Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 5 Jun 2023 16:09:41 +0200 Subject: [PATCH 273/793] feat(server): batch tokenization for flash causal lm (#411) --- .../models/flash_causal_lm.py | 18 ++++++++++++++---- server/text_generation_server/utils/hub.py | 4 +++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 5ff951b3..a2ad2d5e 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -79,6 +79,16 @@ def from_pb( dtype: torch.dtype, device: torch.device, ) -> "FlashCausalLMBatch": + batch_inputs = [] + max_truncation = 0 + for r in pb.requests: + batch_inputs.append(r.inputs) + max_truncation = max(max_truncation, r.truncate) + + batch_tokenized_inputs = tokenizer( + batch_inputs, truncation=True, max_length=max_truncation + )["input_ids"] + position_ids = [] cu_seqlens = [0] max_seqlen = 0 @@ -106,13 +116,13 @@ def from_pb( max_length = 0 # Parse batch - for i, r in enumerate(pb.requests): + for i, (r, tokenized_input) in enumerate( + zip(pb.requests, batch_tokenized_inputs) + ): # request id -> idx in list mapping requests_idx_mapping[r.id] = i - tokenized_input = tokenizer( - r.inputs, truncation=True, max_length=r.truncate - )["input_ids"] + tokenized_input = tokenized_input[-r.truncate :] input_length = len(tokenized_input) max_seqlen = max(max_seqlen, input_length) diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index 134ac7cd..2ed7673c 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -134,7 +134,7 @@ def download_weights( ) -> List[Path]: """Download the safetensors files from the hub""" - def download_file(filename, tries=5): + def download_file(filename, tries=5, backoff: int = 5): local_file = try_to_load_from_cache(model_id, revision, filename) if local_file is not None: logger.info(f"File {filename} already present in cache.") @@ -158,6 +158,8 @@ def download_file(filename, tries=5): if i + 1 == tries: raise e logger.error(e) + logger.info(f"Retrying in {backoff} seconds") + time.sleep(backoff) logger.info(f"Retry {i + 1}/{tries - 1}") # We do this instead of using tqdm because we want to parse the logs with the launcher From 19c41824cb11ba1a3b60a2a65274d8c074383de3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 5 Jun 2023 18:16:08 +0200 Subject: [PATCH 274/793] chore: update openapi schema --- docs/openapi.json | 7 ++++++- router/src/lib.rs | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index e5ef0e3c..8c652946 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -375,7 +375,8 @@ "$ref": "#/components/schemas/GenerateParameters" }, "stream": { - "type": "boolean" + "type": "boolean", + "default": "false" } } }, @@ -459,6 +460,10 @@ "minimum": 0.0, "exclusiveMinimum": 0.0 }, + "decoder_input_details": { + "type": "boolean", + "default": "true" + }, "details": { "type": "boolean", "default": "true" diff --git a/router/src/lib.rs b/router/src/lib.rs index 67fff017..7dff7a11 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -176,7 +176,7 @@ pub(crate) struct CompatGenerateRequest { #[serde(default = "default_parameters")] pub parameters: GenerateParameters, #[serde(default)] - #[allow(dead_code)] + #[schema(default = "false")] pub stream: bool, } From abd58ff82c37d5e4f131abdac3d298927a815604 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 8 Jun 2023 14:51:52 +0200 Subject: [PATCH 275/793] feat(server): Rework model loading (#344) # What does this PR do? Reworked the loading logic. Idea is to use cleaner loading code: - Remove need for `no_init_weights` - Remove all weird `bnb_linear` and `load_weights` and `post_load_weights`. New code layout: - New class `Weights` in charge of handling loading the weights from multiple files into appropiate tensors (potentially sharded) - TP layers now are "shells", they contain the code to know what kind of sharding we need + eventual `all_reduce`. They do not inherit from linear, but they contain some kind of Linear instead - the contained linear can be either FastLinear, BnbLinear or GPTq Linear next. - All modeling code is explictly made for sharding, process group is just no-ops for non sharded code (removes a lot of test cases) ![Screenshot from 2023-05-19 23-19-59](https://github.com/huggingface/text-generation-inference/assets/204321/9a802654-74a3-488c-87a8-073743a6143f) --------- Co-authored-by: Ubuntu Co-authored-by: Ubuntu Co-authored-by: OlivierDehaene Co-authored-by: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> --- .gitignore | 1 + Dockerfile | 17 +- Makefile | 7 +- integration-tests/conftest.py | 14 + .../test_flash_starcoder_default_params.json | 14 +- .../__snapshots__/test_neox/test_neox.json | 113 ++ .../test_neox/test_neox_load.json | 454 +++++++ .../test_neox_sharded/test_neox.json | 163 +++ .../test_neox_sharded/test_neox_load.json | 654 +++++++++ integration-tests/models/test_flash_neox.py | 4 +- integration-tests/models/test_neox.py | 48 + integration-tests/models/test_neox_sharded.py | 44 + integration-tests/pytest.ini | 1 + server/Makefile | 5 +- server/Makefile-transformers | 13 - .../custom_kernels/fused_attention_cuda.cu | 250 ++++ .../fused_bloom_attention_cuda.cu | 250 ++++ server/custom_kernels/setup.py | 19 + server/pyproject.toml | 3 +- server/requirements.txt | 5 +- server/tests/models/test_bloom.py | 9 +- .../text_generation_server/models/__init__.py | 143 +- server/text_generation_server/models/bloom.py | 201 +-- .../models/custom_modeling/bloom_modeling.py | 912 +++++++++++++ .../custom_modeling/flash_llama_modeling.py | 215 ++- .../custom_modeling/flash_neox_modeling.py | 281 ++-- .../custom_modeling/flash_rw_modeling.py | 361 ++--- .../flash_santacoder_modeling.py | 308 +++-- .../models/custom_modeling/neox_modeling.py | 794 +++++++++++ .../models/custom_modeling/opt_modeling.py | 837 ++++++++++++ .../models/custom_modeling/t5_modeling.py | 1200 +++++++++++++++++ .../models/flash_llama.py | 262 +--- .../models/flash_neox.py | 120 +- .../text_generation_server/models/flash_rw.py | 194 +-- .../models/flash_santacoder.py | 367 +---- .../models/galactica.py | 205 +-- .../text_generation_server/models/gpt_neox.py | 205 +-- server/text_generation_server/models/opt.py | 187 +-- server/text_generation_server/models/t5.py | 199 +-- .../text_generation_server/utils/__init__.py | 2 + server/text_generation_server/utils/dist.py | 54 +- server/text_generation_server/utils/layers.py | 363 +++-- .../text_generation_server/utils/weights.py | 77 ++ 43 files changed, 6794 insertions(+), 2781 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_neox/test_neox.json create mode 100644 integration-tests/models/__snapshots__/test_neox/test_neox_load.json create mode 100644 integration-tests/models/__snapshots__/test_neox_sharded/test_neox.json create mode 100644 integration-tests/models/__snapshots__/test_neox_sharded/test_neox_load.json create mode 100644 integration-tests/models/test_neox.py create mode 100644 integration-tests/models/test_neox_sharded.py delete mode 100644 server/Makefile-transformers create mode 100644 server/custom_kernels/custom_kernels/fused_attention_cuda.cu create mode 100644 server/custom_kernels/custom_kernels/fused_bloom_attention_cuda.cu create mode 100644 server/custom_kernels/setup.py create mode 100644 server/text_generation_server/models/custom_modeling/bloom_modeling.py create mode 100644 server/text_generation_server/models/custom_modeling/neox_modeling.py create mode 100644 server/text_generation_server/models/custom_modeling/opt_modeling.py create mode 100644 server/text_generation_server/models/custom_modeling/t5_modeling.py create mode 100644 server/text_generation_server/utils/weights.py diff --git a/.gitignore b/.gitignore index 19604d42..20c9baee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea target router/tokenizer.json +*__pycache__* diff --git a/Dockerfile b/Dockerfile index 483270a8..576dab8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM lukemathwalker/cargo-chef:latest-rust-1.69 AS chef WORKDIR /usr/src +ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse + FROM chef as planner COPY Cargo.toml Cargo.toml COPY rust-toolchain.toml rust-toolchain.toml @@ -98,14 +100,14 @@ COPY server/Makefile-flash-att Makefile RUN make build-flash-attention # Build Transformers CUDA kernels -FROM kernel-builder as transformers-builder +FROM kernel-builder as custom-kernels-builder WORKDIR /usr/src -COPY server/Makefile-transformers Makefile +COPY server/custom_kernels/ . # Build specific version of transformers -RUN BUILD_EXTENSIONS="True" make build-transformers +RUN python setup.py build # Text Generation Inference base image FROM nvidia/cuda:11.8.0-base-ubuntu20.04 as base @@ -136,11 +138,10 @@ COPY --from=flash-att-builder /usr/src/flash-attention/csrc/layer_norm/build/lib COPY --from=flash-att-builder /usr/src/flash-attention/csrc/rotary/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages # Copy build artifacts from transformers builder -COPY --from=transformers-builder /usr/src/transformers /usr/src/transformers -COPY --from=transformers-builder /usr/src/transformers/build/lib.linux-x86_64-cpython-39/transformers /usr/src/transformers/src/transformers +COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39/custom_kernels /usr/src/custom-kernels/src/custom_kernels -# Install transformers dependencies -RUN cd /usr/src/transformers && pip install -e . --no-cache-dir && pip install einops --no-cache-dir +# Install flash-attention dependencies +RUN pip install einops --no-cache-dir # Install server COPY proto proto @@ -170,4 +171,4 @@ ENTRYPOINT ["./entrypoint.sh"] FROM base ENTRYPOINT ["text-generation-launcher"] -CMD ["--json-output"] \ No newline at end of file +CMD ["--json-output"] diff --git a/Makefile b/Makefile index a33aba17..c7f649ec 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ install-server: cd server && make install +install-custom-kernels: + if [ "$$BUILD_EXTENSIONS" == "True" ]; then cd server/custom_kernels && python setup.py install; else echo "Custom kernels are disabled, you need set to BUILD_EXTENSION environment variable to 'True' in order to build them. (Please read the docs, kernels might not work on all hardware)"; fi + install-integration-tests: cd integration-tests && pip install -r requirements.txt cd clients/python && pip install . @@ -14,7 +17,7 @@ install-launcher: install-benchmark: cd benchmark && cargo install --path . -install: install-server install-router install-launcher +install: install-server install-router install-launcher install-custom-kernels server-dev: cd server && make run-dev @@ -52,4 +55,4 @@ run-bloom: text-generation-launcher --model-id bigscience/bloom --num-shard 8 --port 8080 run-bloom-quantize: - text-generation-launcher --model-id bigscience/bloom --num-shard 8 --quantize --port 8080 \ No newline at end of file + text-generation-launcher --model-id bigscience/bloom --num-shard 8 --quantize --port 8080 diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 82f1b719..8f59d75a 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -209,6 +209,7 @@ def local_launcher( num_shard: Optional[int] = None, quantize: Optional[str] = None, trust_remote_code: bool = False, + use_flash_attention: bool = True, ): port = random.randint(8000, 10_000) master_port = random.randint(10_000, 20_000) @@ -240,6 +241,9 @@ def local_launcher( env = os.environ env["LOG_LEVEL"] = "info,text_generation_router=debug" + if not use_flash_attention: + env["USE_FLASH_ATTENTION"] = "false" + with subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) as process: @@ -254,12 +258,16 @@ def local_launcher( process.stdout.close() process.stderr.close() + if not use_flash_attention: + del env["USE_FLASH_ATTENTION"] + @contextlib.contextmanager def docker_launcher( model_id: str, num_shard: Optional[int] = None, quantize: Optional[str] = None, trust_remote_code: bool = False, + use_flash_attention: bool = True, ): port = random.randint(8000, 10_000) @@ -287,6 +295,9 @@ def docker_launcher( gpu_count = num_shard if num_shard is not None else 1 env = {"LOG_LEVEL": "info,text_generation_router=debug"} + if not use_flash_attention: + env["USE_FLASH_ATTENTION"] = "false" + if HUGGING_FACE_HUB_TOKEN is not None: env["HUGGING_FACE_HUB_TOKEN"] = HUGGING_FACE_HUB_TOKEN @@ -310,6 +321,9 @@ def docker_launcher( yield ContainerLauncherHandle(client, container.name, port) + if not use_flash_attention: + del env["USE_FLASH_ATTENTION"] + try: container.stop() container.wait() diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json index afd0b662..89e02c07 100644 --- a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json +++ b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json @@ -11,17 +11,17 @@ }, { "id": 1459, - "logprob": -5.6289062, + "logprob": -5.6328125, "text": " print" }, { "id": 81, - "logprob": -1.6005859, + "logprob": -1.6035156, "text": "_" }, { "id": 7656, - "logprob": -5.9921875, + "logprob": -5.9882812, "text": "hello" } ], @@ -59,19 +59,19 @@ }, { "id": 10896, - "logprob": -0.3659668, + "logprob": -0.38549805, "special": false, "text": " World" }, { "id": 657, - "logprob": -0.49804688, + "logprob": -0.5229492, "special": false, "text": "\")" }, { "id": 203, - "logprob": -0.11279297, + "logprob": -0.10632324, "special": false, "text": "\n" }, @@ -113,7 +113,7 @@ }, { "id": 426, - "logprob": -0.051635742, + "logprob": 0.0, "special": false, "text": "name" }, diff --git a/integration-tests/models/__snapshots__/test_neox/test_neox.json b/integration-tests/models/__snapshots__/test_neox/test_neox.json new file mode 100644 index 00000000..2abc27e1 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_neox/test_neox.json @@ -0,0 +1,113 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|USER|>" + }, + { + "id": 1276, + "logprob": -4.5546875, + "text": "What" + }, + { + "id": 434, + "logprob": -4.1992188, + "text": "'s" + }, + { + "id": 634, + "logprob": -5.125, + "text": " your" + }, + { + "id": 12315, + "logprob": -9.8984375, + "text": " mood" + }, + { + "id": 3063, + "logprob": -4.0976562, + "text": " today" + }, + { + "id": 32, + "logprob": -0.14562988, + "text": "?" + }, + { + "id": 50279, + "logprob": -0.26733398, + "text": "<|ASSISTANT|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 42, + "logprob": -0.86279297, + "special": false, + "text": "I" + }, + { + "id": 1353, + "logprob": -0.94921875, + "special": false, + "text": "'m" + }, + { + "id": 7016, + "logprob": -2.1835938, + "special": false, + "text": " sorry" + }, + { + "id": 13, + "logprob": -0.074035645, + "special": false, + "text": "," + }, + { + "id": 1394, + "logprob": -0.86376953, + "special": false, + "text": "You" + }, + { + "id": 452, + "logprob": -1.2070312, + "special": false, + "text": " have" + }, + { + "id": 247, + "logprob": -1.4365234, + "special": false, + "text": " a" + }, + { + "id": 4327, + "logprob": -1.109375, + "special": false, + "text": " choice" + }, + { + "id": 273, + "logprob": -0.93408203, + "special": false, + "text": " of" + }, + { + "id": 752, + "logprob": -1.8808594, + "special": false, + "text": " what" + } + ] + }, + "generated_text": "I'm sorry,You have a choice of what" +} diff --git a/integration-tests/models/__snapshots__/test_neox/test_neox_load.json b/integration-tests/models/__snapshots__/test_neox/test_neox_load.json new file mode 100644 index 00000000..f37f0d8e --- /dev/null +++ b/integration-tests/models/__snapshots__/test_neox/test_neox_load.json @@ -0,0 +1,454 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|USER|>" + }, + { + "id": 1276, + "logprob": -4.5546875, + "text": "What" + }, + { + "id": 434, + "logprob": -4.1953125, + "text": "'s" + }, + { + "id": 634, + "logprob": -5.125, + "text": " your" + }, + { + "id": 12315, + "logprob": -9.8828125, + "text": " mood" + }, + { + "id": 3063, + "logprob": -3.9980469, + "text": " today" + }, + { + "id": 32, + "logprob": -0.14672852, + "text": "?" + }, + { + "id": 50279, + "logprob": -0.26489258, + "text": "<|ASSISTANT|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 42, + "logprob": -0.8618164, + "special": false, + "text": "I" + }, + { + "id": 1353, + "logprob": -0.9506836, + "special": false, + "text": "'m" + }, + { + "id": 7016, + "logprob": -2.1738281, + "special": false, + "text": " sorry" + }, + { + "id": 13, + "logprob": -0.0758667, + "special": false, + "text": "," + }, + { + "id": 1394, + "logprob": -0.9135742, + "special": false, + "text": "You" + }, + { + "id": 452, + "logprob": -1.1445312, + "special": false, + "text": " have" + }, + { + "id": 247, + "logprob": -1.4375, + "special": false, + "text": " a" + }, + { + "id": 4327, + "logprob": -1.1103516, + "special": false, + "text": " choice" + }, + { + "id": 273, + "logprob": -1.0058594, + "special": false, + "text": " of" + }, + { + "id": 752, + "logprob": -1.921875, + "special": false, + "text": " what" + } + ] + }, + "generated_text": "I'm sorry,You have a choice of what" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|USER|>" + }, + { + "id": 1276, + "logprob": -4.5546875, + "text": "What" + }, + { + "id": 434, + "logprob": -4.1953125, + "text": "'s" + }, + { + "id": 634, + "logprob": -5.125, + "text": " your" + }, + { + "id": 12315, + "logprob": -9.8828125, + "text": " mood" + }, + { + "id": 3063, + "logprob": -3.9980469, + "text": " today" + }, + { + "id": 32, + "logprob": -0.14672852, + "text": "?" + }, + { + "id": 50279, + "logprob": -0.26489258, + "text": "<|ASSISTANT|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 42, + "logprob": -0.8618164, + "special": false, + "text": "I" + }, + { + "id": 1353, + "logprob": -0.9506836, + "special": false, + "text": "'m" + }, + { + "id": 7016, + "logprob": -2.1738281, + "special": false, + "text": " sorry" + }, + { + "id": 13, + "logprob": -0.0758667, + "special": false, + "text": "," + }, + { + "id": 1394, + "logprob": -0.9135742, + "special": false, + "text": "You" + }, + { + "id": 452, + "logprob": -1.1445312, + "special": false, + "text": " have" + }, + { + "id": 247, + "logprob": -1.4375, + "special": false, + "text": " a" + }, + { + "id": 4327, + "logprob": -1.1103516, + "special": false, + "text": " choice" + }, + { + "id": 273, + "logprob": -1.0058594, + "special": false, + "text": " of" + }, + { + "id": 752, + "logprob": -1.921875, + "special": false, + "text": " what" + } + ] + }, + "generated_text": "I'm sorry,You have a choice of what" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|USER|>" + }, + { + "id": 1276, + "logprob": -4.5546875, + "text": "What" + }, + { + "id": 434, + "logprob": -4.1953125, + "text": "'s" + }, + { + "id": 634, + "logprob": -5.125, + "text": " your" + }, + { + "id": 12315, + "logprob": -9.8828125, + "text": " mood" + }, + { + "id": 3063, + "logprob": -3.9980469, + "text": " today" + }, + { + "id": 32, + "logprob": -0.14672852, + "text": "?" + }, + { + "id": 50279, + "logprob": -0.26489258, + "text": "<|ASSISTANT|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 42, + "logprob": -0.8618164, + "special": false, + "text": "I" + }, + { + "id": 1353, + "logprob": -0.9506836, + "special": false, + "text": "'m" + }, + { + "id": 7016, + "logprob": -2.1738281, + "special": false, + "text": " sorry" + }, + { + "id": 13, + "logprob": -0.0758667, + "special": false, + "text": "," + }, + { + "id": 1394, + "logprob": -0.9135742, + "special": false, + "text": "You" + }, + { + "id": 452, + "logprob": -1.1445312, + "special": false, + "text": " have" + }, + { + "id": 247, + "logprob": -1.4375, + "special": false, + "text": " a" + }, + { + "id": 4327, + "logprob": -1.1103516, + "special": false, + "text": " choice" + }, + { + "id": 273, + "logprob": -1.0058594, + "special": false, + "text": " of" + }, + { + "id": 752, + "logprob": -1.921875, + "special": false, + "text": " what" + } + ] + }, + "generated_text": "I'm sorry,You have a choice of what" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|USER|>" + }, + { + "id": 1276, + "logprob": -4.5546875, + "text": "What" + }, + { + "id": 434, + "logprob": -4.1953125, + "text": "'s" + }, + { + "id": 634, + "logprob": -5.125, + "text": " your" + }, + { + "id": 12315, + "logprob": -9.8828125, + "text": " mood" + }, + { + "id": 3063, + "logprob": -3.9980469, + "text": " today" + }, + { + "id": 32, + "logprob": -0.14672852, + "text": "?" + }, + { + "id": 50279, + "logprob": -0.26489258, + "text": "<|ASSISTANT|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 42, + "logprob": -0.8618164, + "special": false, + "text": "I" + }, + { + "id": 1353, + "logprob": -0.9506836, + "special": false, + "text": "'m" + }, + { + "id": 7016, + "logprob": -2.1738281, + "special": false, + "text": " sorry" + }, + { + "id": 13, + "logprob": -0.0758667, + "special": false, + "text": "," + }, + { + "id": 1394, + "logprob": -0.9135742, + "special": false, + "text": "You" + }, + { + "id": 452, + "logprob": -1.1445312, + "special": false, + "text": " have" + }, + { + "id": 247, + "logprob": -1.4375, + "special": false, + "text": " a" + }, + { + "id": 4327, + "logprob": -1.1103516, + "special": false, + "text": " choice" + }, + { + "id": 273, + "logprob": -1.0058594, + "special": false, + "text": " of" + }, + { + "id": 752, + "logprob": -1.921875, + "special": false, + "text": " what" + } + ] + }, + "generated_text": "I'm sorry,You have a choice of what" + } +] diff --git a/integration-tests/models/__snapshots__/test_neox_sharded/test_neox.json b/integration-tests/models/__snapshots__/test_neox_sharded/test_neox.json new file mode 100644 index 00000000..25cdf6d7 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_neox_sharded/test_neox.json @@ -0,0 +1,163 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.0234375, + "text": "What" + }, + { + "id": 310, + "logprob": -5.4179688, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1542969, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.359375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.006038666, + "text": "e" + }, + { + "id": 13, + "logprob": -7.328125, + "text": "," + }, + { + "id": 285, + "logprob": -0.3173828, + "text": " and" + }, + { + "id": 752, + "logprob": -2.0625, + "text": " what" + }, + { + "id": 434, + "logprob": -5.7734375, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.74072266, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.5898438, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.2949219, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.40625, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1113281, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008056641, + "text": "?" + }, + { + "id": 0, + "logprob": -2.3300781, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.28125, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.5878906, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5449219, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.05038452, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.002292633, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.3828278e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.0010242462, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.090270996, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12719727, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.016571045, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.43432617, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" +} diff --git a/integration-tests/models/__snapshots__/test_neox_sharded/test_neox_load.json b/integration-tests/models/__snapshots__/test_neox_sharded/test_neox_load.json new file mode 100644 index 00000000..0b38e701 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_neox_sharded/test_neox_load.json @@ -0,0 +1,654 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.0234375, + "text": "What" + }, + { + "id": 310, + "logprob": -5.4179688, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1542969, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.359375, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.006038666, + "text": "e" + }, + { + "id": 13, + "logprob": -7.328125, + "text": "," + }, + { + "id": 285, + "logprob": -0.3173828, + "text": " and" + }, + { + "id": 752, + "logprob": -2.0625, + "text": " what" + }, + { + "id": 434, + "logprob": -5.7734375, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.74072266, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.5898438, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.2949219, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.40625, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1113281, + "text": " word" + }, + { + "id": 32, + "logprob": -0.008056641, + "text": "?" + }, + { + "id": 0, + "logprob": -2.3300781, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.28125, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.5878906, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.5498047, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.04815674, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.002313614, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.2636185e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.0010147095, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.0859375, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12609863, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.016601562, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.38256836, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.0234375, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1640625, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.40625, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005420685, + "text": "e" + }, + { + "id": 13, + "logprob": -7.2226562, + "text": "," + }, + { + "id": 285, + "logprob": -0.26879883, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1992188, + "text": " what" + }, + { + "id": 434, + "logprob": -5.46875, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.8017578, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6796875, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.1972656, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.4453125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1933594, + "text": " word" + }, + { + "id": 32, + "logprob": -0.007858276, + "text": "?" + }, + { + "id": 0, + "logprob": -2.328125, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.21875, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.6201172, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.546875, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.051879883, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.0020179749, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -9.059906e-06, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00096797943, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.07940674, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12182617, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.017227173, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.44482422, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.0234375, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1640625, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.40625, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005420685, + "text": "e" + }, + { + "id": 13, + "logprob": -7.2226562, + "text": "," + }, + { + "id": 285, + "logprob": -0.26879883, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1992188, + "text": " what" + }, + { + "id": 434, + "logprob": -5.46875, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.8017578, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6796875, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.1972656, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.4453125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1933594, + "text": " word" + }, + { + "id": 32, + "logprob": -0.007858276, + "text": "?" + }, + { + "id": 0, + "logprob": -2.328125, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.21875, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.6201172, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.546875, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.051879883, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.0020179749, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -9.059906e-06, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.00096797943, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.07940674, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12182617, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.017227173, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.44482422, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 50278, + "logprob": null, + "text": "<|prompter|>" + }, + { + "id": 1276, + "logprob": -8.0234375, + "text": "What" + }, + { + "id": 310, + "logprob": -5.421875, + "text": " is" + }, + { + "id": 247, + "logprob": -2.1640625, + "text": " a" + }, + { + "id": 1167, + "logprob": -5.40625, + "text": " mem" + }, + { + "id": 70, + "logprob": -0.005420685, + "text": "e" + }, + { + "id": 13, + "logprob": -7.2226562, + "text": "," + }, + { + "id": 285, + "logprob": -0.26879883, + "text": " and" + }, + { + "id": 752, + "logprob": -2.1992188, + "text": " what" + }, + { + "id": 434, + "logprob": -5.46875, + "text": "'s" + }, + { + "id": 253, + "logprob": -0.8017578, + "text": " the" + }, + { + "id": 2892, + "logprob": -6.6796875, + "text": " history" + }, + { + "id": 3212, + "logprob": -2.1972656, + "text": " behind" + }, + { + "id": 436, + "logprob": -11.4453125, + "text": " this" + }, + { + "id": 3159, + "logprob": -2.1933594, + "text": " word" + }, + { + "id": 32, + "logprob": -0.007858276, + "text": "?" + }, + { + "id": 0, + "logprob": -2.328125, + "text": "<|endoftext|>" + }, + { + "id": 50281, + "logprob": -18.21875, + "text": "<|assistant|>" + } + ], + "seed": null, + "tokens": [ + { + "id": 510, + "logprob": -0.6201172, + "special": false, + "text": "The" + }, + { + "id": 3159, + "logprob": -0.546875, + "special": false, + "text": " word" + }, + { + "id": 346, + "logprob": -0.051879883, + "special": false, + "text": " \"" + }, + { + "id": 6441, + "logprob": -0.0020179749, + "special": false, + "text": "mem" + }, + { + "id": 70, + "logprob": -1.04904175e-05, + "special": false, + "text": "e" + }, + { + "id": 3, + "logprob": -0.0009560585, + "special": false, + "text": "\"" + }, + { + "id": 369, + "logprob": -0.08557129, + "special": false, + "text": " was" + }, + { + "id": 806, + "logprob": -0.12084961, + "special": false, + "text": " first" + }, + { + "id": 908, + "logprob": -0.01737976, + "special": false, + "text": " used" + }, + { + "id": 275, + "logprob": -0.4025879, + "special": false, + "text": " in" + } + ] + }, + "generated_text": "The word \"meme\" was first used in" + } +] diff --git a/integration-tests/models/test_flash_neox.py b/integration-tests/models/test_flash_neox.py index ff9b9763..1076126b 100644 --- a/integration-tests/models/test_flash_neox.py +++ b/integration-tests/models/test_flash_neox.py @@ -37,8 +37,8 @@ async def test_flash_neox_load(flash_neox, generate_load, response_snapshot): generated_texts = [r.generated_text for r in responses] assert len(generated_texts) == 4 - assert generated_texts, all( + assert all( [text == generated_texts[0] for text in generated_texts] - ) + ), generated_texts assert responses == response_snapshot diff --git a/integration-tests/models/test_neox.py b/integration-tests/models/test_neox.py new file mode 100644 index 00000000..7b88f86a --- /dev/null +++ b/integration-tests/models/test_neox.py @@ -0,0 +1,48 @@ +import pytest + + +@pytest.fixture(scope="module") +def neox_handle(launcher): + with launcher( + "stabilityai/stablelm-tuned-alpha-3b", num_shard=1, use_flash_attention=False + ) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def neox(neox_handle): + await neox_handle.health(300) + return neox_handle.client + + +@pytest.mark.skip +@pytest.mark.asyncio +async def test_neox(neox, response_snapshot): + response = await neox.generate( + "<|USER|>What's your mood today?<|ASSISTANT|>", + max_new_tokens=10, + decoder_input_details=True, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.skip +@pytest.mark.asyncio +async def test_neox_load(neox, generate_load, response_snapshot): + responses = await generate_load( + neox, + "<|USER|>What's your mood today?<|ASSISTANT|>", + max_new_tokens=10, + n=4, + ) + + generated_texts = [r.generated_text for r in responses] + + assert len(generated_texts) == 4 + assert generated_texts, all( + [text == generated_texts[0] for text in generated_texts] + ) + + assert responses == response_snapshot diff --git a/integration-tests/models/test_neox_sharded.py b/integration-tests/models/test_neox_sharded.py new file mode 100644 index 00000000..8cee8765 --- /dev/null +++ b/integration-tests/models/test_neox_sharded.py @@ -0,0 +1,44 @@ +import pytest + + +@pytest.fixture(scope="module") +def neox_sharded_handle(launcher): + with launcher( + "OpenAssistant/oasst-sft-1-pythia-12b", num_shard=2, use_flash_attention=False + ) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def neox_sharded(neox_sharded_handle): + await neox_sharded_handle.health(300) + return neox_sharded_handle.client + + +@pytest.mark.skip +@pytest.mark.asyncio +async def test_neox(neox_sharded, response_snapshot): + response = await neox_sharded.generate( + "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", + max_new_tokens=10, + decoder_input_details=True, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.skip +@pytest.mark.asyncio +async def test_neox_load(neox_sharded, generate_load, response_snapshot): + responses = await generate_load( + neox_sharded, + "<|prompter|>What is a meme, and what's the history behind this word?<|endoftext|><|assistant|>", + max_new_tokens=10, + n=4, + ) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) + + assert responses == response_snapshot diff --git a/integration-tests/pytest.ini b/integration-tests/pytest.ini index 485e6017..7dcae663 100644 --- a/integration-tests/pytest.ini +++ b/integration-tests/pytest.ini @@ -1,4 +1,5 @@ [pytest] +addopts = --snapshot-warn-unused asyncio_mode = auto markers = private: marks tests as requiring an admin hf token (deselect with '-m "not private"') \ No newline at end of file diff --git a/server/Makefile b/server/Makefile index 6eb56c75..17020c97 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,4 +1,3 @@ -include Makefile-transformers include Makefile-flash-att unit-tests: @@ -17,7 +16,7 @@ install-torch: # Install specific version of torch pip install torch --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir -install: gen-server install-torch install-transformers +install: gen-server install-torch pip install pip --upgrade pip install -r requirements.txt pip install -e ".[bnb, accelerate]" @@ -26,4 +25,4 @@ run-dev: SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded export-requirements: - poetry export -o requirements.txt -E bnb --without-hashes \ No newline at end of file + poetry export -o requirements.txt -E bnb --without-hashes diff --git a/server/Makefile-transformers b/server/Makefile-transformers deleted file mode 100644 index 64d01672..00000000 --- a/server/Makefile-transformers +++ /dev/null @@ -1,13 +0,0 @@ -transformers_commit := 69009822aa7897ffab97afb814e38126b83f639e - -transformers: - # Clone fork of transformers with custom CUDA kernels and sharding logic - pip install --upgrade setuptools - git clone https://github.com/OlivierDehaene/transformers.git - -build-transformers: transformers - cd transformers && git fetch && git checkout $(transformers_commit) && python setup.py build - -install-transformers: build-transformers - pip uninstall transformers -y || true - cd transformers && python setup.py install \ No newline at end of file diff --git a/server/custom_kernels/custom_kernels/fused_attention_cuda.cu b/server/custom_kernels/custom_kernels/fused_attention_cuda.cu new file mode 100644 index 00000000..60f9f028 --- /dev/null +++ b/server/custom_kernels/custom_kernels/fused_attention_cuda.cu @@ -0,0 +1,250 @@ +#include +#include +#include +#include +#include + +#include + +/** +* Friendly reminder of how multithreading works in CUDA: https://developer.nvidia.com/blog/even-easier-introduction-cuda +* Check example at https://github.com/thomasw21/LinearTransformers/blob/main/model/attention/fast_weight/fast_weight_cuda.cu +**/ + +// Available in pytorch main +//#define DISPATCH_CASE_FLOATING_TYPES(...) \ +// at::AT_DISPATCH_CASE(at::ScalarType::Double, __VA_ARGS__) \ +// at::AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ +// at::AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \ +// at::AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \ + +/* +* Forward passes +*/ + +/** +* cast to fp32 if in fp16 + mask + softmax computation in fp32 + cast back to original dtype +**/ +template +__global__ void forward_masked_softmax_kernel( + const torch::PackedTensorAccessor32 attention_scores, // [B, KV] + const torch::PackedTensorAccessor32 mask, // [B, KV] + torch::PackedTensorAccessor32 result, // [B, KV] + const int64_t effective_kv_length, + const dim3 blockDim, + const int64_t rows_per_block, + const int64_t kv_length, + const int64_t batch_size +) { + const auto row_id = threadIdx.x / effective_kv_length; + const auto effective_kv_length_id = threadIdx.x % effective_kv_length; + const auto kv_length_start = effective_kv_length_id * min_kv_length_shard_size_per_thread; + auto kv_length_end_ = (effective_kv_length_id + 1) * min_kv_length_shard_size_per_thread; + kv_length_end_ = (kv_length_end_ > kv_length) ? kv_length : kv_length_end_; + const auto kv_length_end = kv_length_end_; + + const auto batch_id = blockIdx.x * rows_per_block + row_id; + + // We need 2 float storage for each row, one for max computation, the other for normalizing exponential + extern __shared__ float temp_storage[]; + const auto row_id_mem_offset = row_id * 2; + if (effective_kv_length_id == 0) { + temp_storage[row_id_mem_offset] = -std::numeric_limits::infinity(); + temp_storage[row_id_mem_offset + 1] = 0; + } + __syncthreads(); + + // Compute mask and max + if (batch_id < batch_size) { + float thread_max = -std::numeric_limits::infinity(); + for (int kv_length_id = kv_length_start; kv_length_id < kv_length_end; ++kv_length_id) { + if (mask[batch_id][kv_length_id] == 0) { + const float candidate = attention_scores[batch_id][kv_length_id]; + thread_max = (thread_max < candidate) ? candidate : thread_max; + } + } + if (thread_max != -std::numeric_limits::infinity()) { + // TODO @thomasw21 with more memory we can probably compute a much faster `max-reduce` in parallel O(ln(n)) operations in each memory slot + gpuAtomicMax(&temp_storage[row_id_mem_offset], thread_max); + } + } + + __syncthreads(); + + // Compute exp(elt - max) masked + float exponential[min_kv_length_shard_size_per_thread]; + if (batch_id < batch_size) { + float thread_add = 0; + for (int kv_length_id = kv_length_start; kv_length_id < kv_length_end; ++kv_length_id) { + if (mask[batch_id][kv_length_id] == 0) { + exponential[kv_length_id - kv_length_start] = std::exp(static_cast(attention_scores[batch_id][kv_length_id]) - temp_storage[row_id_mem_offset]); + thread_add = thread_add + exponential[kv_length_id - kv_length_start]; + } else { + exponential[kv_length_id - kv_length_start] = 0.; + } + } + if (thread_add > 0) { + // TODO @thomasw21 with more memory we can probably compute a much faster `sum-reduce` in parallel O(ln(n)) operations in each memory slot + gpuAtomicAdd(&temp_storage[row_id_mem_offset + 1], thread_add); + } + } + + __syncthreads(); + + // Compute softmax + if (batch_id < batch_size) { + // If sum of all exponential is 0, we set the softmax values to 0 + if (temp_storage[row_id_mem_offset + 1] == 0.) { + for (int kv_length_id = kv_length_start; kv_length_id < kv_length_end; ++kv_length_id) { + result[batch_id][kv_length_id] = 0.; + } + } else { + for (int kv_length_id = kv_length_start; kv_length_id < kv_length_end; ++kv_length_id) { + result[batch_id][kv_length_id] = static_cast(exponential[kv_length_id - kv_length_start] / temp_storage[row_id_mem_offset + 1]); + } + } + } +} + +#define CHECK_CUDA(x) TORCH_CHECK(x.device().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +std::tuple>, at::Tensor> forward( + const at::Tensor query, + const at::Tensor key, + const at::Tensor value, + const std::optional> layer_past, + const at::Tensor attention_mask, + const std::optional head_mask, + const float inv_norm_factor, + const int num_heads, + const bool use_cache +) { + auto query_layer = query; + auto key_layer = key; + auto value_layer = value; + + if (layer_past) { + const auto past_key = (*layer_past).at(0); + const auto past_value = (*layer_past).at(1); + key_layer = at::cat({past_key, key_layer}, 2); + value_layer = at::cat({past_value, value_layer}, 2); + } + + std::optional> present; + if (use_cache) { + present = {key_layer, value_layer}; + } else { + present = {}; + } + + const auto batch_size = query_layer.size(0); + const auto q_length = query_layer.size(2); + const auto attn_head_size = query_layer.size(3); + const auto batch_size_times_num_heads = batch_size * num_heads; + const auto kv_length = key_layer.size(2); + + const auto query_view = query_layer.reshape({batch_size_times_num_heads, q_length, attn_head_size}); + auto key_view = key_layer.reshape({batch_size_times_num_heads, kv_length, attn_head_size}).transpose(1, 2); + auto value_view = value_layer.reshape({batch_size_times_num_heads, kv_length, attn_head_size}); + + auto query_scaled = query_view * inv_norm_factor; + auto attention_scores = at::bmm(query_scaled, key_view); + + // Computing `optionally_cast_fp16_to_fp32 + masked_fill + softmax + cast_to_intial_dtype` + at::Tensor attention_probs; + if (true) { + // TODO @thomasw21: it's easier to think of attention_scores as 2D tensors + const auto attention_scores_2d = attention_scores.view({batch_size_times_num_heads * q_length, kv_length}); + const auto attention_mask_2d = attention_mask.view({batch_size_times_num_heads * q_length, kv_length}); + + // Custom kernel + attention_probs = at::empty_like(attention_scores_2d); + + // Check that inputs and contiguous + cuda tensors + CHECK_INPUT(attention_scores_2d); + CHECK_INPUT(attention_mask_2d); + + // TODO @thomas21: change by to this as it's cleaner when pytorch 1.13 comes out + // DISPATCH_CASE_FLOATING_TYPES(attention_scores.scalar_type(), "masked_softmax", [&] { + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, attention_scores.scalar_type(), "masked_softmax", [&] { + /* + * Understanding how GPUs work: https://developer.nvidia.com/blog/cuda-refresher-cuda-programming-model/ + * A100 specifications: https://images.nvidia.com/aem-dam/en-zz/Solutions/data-center/nvidia-ampere-architecture-whitepaper.pdf + * - SMs: 108 + * - TPCs: 56 (What's that?) + * - Memory size: 40 GB + * - L2 Cache size: 40960 KB (shared across all SMs) + * - L1/Shared memory size: 192 KB (shared across all threads within a SM) + * - Max Threads / SM: 2048 + * - Max Thread Blocks / SM: 32 + */ + + /* + * We should split [batch_size_times_num_heads_block, q_length] in seperate blocks and [batch_size_times_num_heads_block_size, kv_length] a single block + * with multiple threads as we need to `sync_threads` to run exponential sum. + * We maximise the usage of threads within a single block + */ + // TODO @thomasw21 figure out everything warp related: + // - why do they have to be power of 2 + // TODO @thomas21 check why everyone is setting 1024 when officially it's 2048 + const auto MAX_THREADS_PER_SM = 1024; + // TODO @thomasw21 figure out how to have longer sequences, currently the maximum is `max_kv_length = MAX_THREADS_PER_SM * MIN_KV_LENGTH_SHARD_SIZE_PER_THREAD` + const auto MIN_KV_LENGTH_SHARD_SIZE_PER_THREAD = 4; + // `effective_kv_length = ceil(kv_length / MIN_KV_LENGTH_SHARD_SIZE_PER_THREAD)` + const auto effective_kv_length = (kv_length - 1)/ MIN_KV_LENGTH_SHARD_SIZE_PER_THREAD + 1; + const auto rows_per_block = MAX_THREADS_PER_SM / effective_kv_length; + const auto num_blocks = (batch_size_times_num_heads * q_length - 1) / rows_per_block + 1; + + const dim3 gridDim(num_blocks); // Number of blocks that run + const dim3 blockDim(MAX_THREADS_PER_SM); // Number of threads that run per block + const int shared_mem_forward = rows_per_block * 2 * sizeof(float); + + // 192 * 2 ** 10 + // const auto MAX_L1_MEMORY = 196608; + // const auto MAX_SMs = 108; + // TORCH_CHECK(batch_size_times_num_heads * q_length <= MAX_L1_MEMORY, "Shared memory exceeds 192KB limitation."); + // TORCH_CHECK(gridDim.x * gridDim.y * gridDim.z <= MAX_SMs, "A100s only have 108 SMs. Raising as require blocks is bigger."); + // TORCH_CHECK(blockDim.x * blockDim.y * blockDim.z <= MAX_THREADS_PER_SM, "A100s only have 2048 threads per block. Raising as require requested threads is higher."); + + forward_masked_softmax_kernel<<>>( + attention_scores_2d.packed_accessor32(), + attention_mask_2d.packed_accessor32(), + attention_probs.packed_accessor32(), + effective_kv_length, + blockDim, + rows_per_block, + kv_length, + batch_size_times_num_heads * q_length + ); + }); + attention_probs = attention_probs.view({batch_size_times_num_heads, q_length, kv_length}); + } else { + // Pytorch C++ API + auto input_dtype = attention_scores.scalar_type(); + if (input_dtype == at::ScalarType::Float) { + attention_scores = attention_scores.to(at::ScalarType::Float); + }; + // TODO @thomasw21 Figure out how to get minimum value + auto attn_weights = attention_scores.masked_fill_(attention_mask, -1e34); + attention_probs = attn_weights.softmax(-1, at::ScalarType::Float).to(input_dtype); + } + + auto context_layer = attention_probs.bmm(value_view); + + // `_merge_heads` + context_layer = context_layer.view({batch_size, num_heads, q_length, attn_head_size}); + context_layer = context_layer.permute({0, 2, 1, 3}); + context_layer = context_layer.reshape({batch_size, q_length, attn_head_size * num_heads}); + + return std::make_tuple(context_layer, present, attention_probs); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def( + "forward", + &forward, + "GPT-Neox attention mechanism forward (CUDA)" + ); +} diff --git a/server/custom_kernels/custom_kernels/fused_bloom_attention_cuda.cu b/server/custom_kernels/custom_kernels/fused_bloom_attention_cuda.cu new file mode 100644 index 00000000..4be547b1 --- /dev/null +++ b/server/custom_kernels/custom_kernels/fused_bloom_attention_cuda.cu @@ -0,0 +1,250 @@ +#include +#include +#include +#include +#include + +#include + +/** +* Friendly reminder of how multithreading works in CUDA: https://developer.nvidia.com/blog/even-easier-introduction-cuda +* Check example at https://github.com/thomasw21/LinearTransformers/blob/main/model/attention/fast_weight/fast_weight_cuda.cu +**/ + +// Available in pytorch main +//#define DISPATCH_CASE_FLOATING_TYPES(...) \ +// at::AT_DISPATCH_CASE(at::ScalarType::Double, __VA_ARGS__) \ +// at::AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ +// at::AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \ +// at::AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \ + +/* +* Forward passes +*/ + +/** +* cast to fp32 if in fp16 + mask + softmax computation in fp32 + cast back to original dtype +**/ +template +__global__ void forward_masked_softmax_kernel( + const torch::PackedTensorAccessor32 attention_scores, // [B, KV] + const torch::PackedTensorAccessor32 mask, // [B, KV] + torch::PackedTensorAccessor32 result, // [B, KV] + const int64_t effective_kv_length, + const dim3 blockDim, + const int64_t rows_per_block, + const int64_t kv_length, + const int64_t batch_size +) { + const auto row_id = threadIdx.x / effective_kv_length; + const auto effective_kv_length_id = threadIdx.x % effective_kv_length; + const auto kv_length_start = effective_kv_length_id * min_kv_length_shard_size_per_thread; + auto kv_length_end_ = (effective_kv_length_id + 1) * min_kv_length_shard_size_per_thread; + kv_length_end_ = (kv_length_end_ > kv_length) ? kv_length : kv_length_end_; + const auto kv_length_end = kv_length_end_; + + const auto batch_id = blockIdx.x * rows_per_block + row_id; + + // We need 2 float storage for each row, one for max computation, the other for normalizing exponential + extern __shared__ float temp_storage[]; + const auto row_id_mem_offset = row_id * 2; + if (effective_kv_length_id == 0) { + temp_storage[row_id_mem_offset] = -std::numeric_limits::infinity(); + temp_storage[row_id_mem_offset + 1] = 0; + } + __syncthreads(); + + // Compute mask and max + if (batch_id < batch_size) { + float thread_max = -std::numeric_limits::infinity(); + for (int kv_length_id = kv_length_start; kv_length_id < kv_length_end; ++kv_length_id) { + if (mask[batch_id][kv_length_id] == 0) { + const float candidate = attention_scores[batch_id][kv_length_id]; + thread_max = (thread_max < candidate) ? candidate : thread_max; + } + } + if (thread_max != -std::numeric_limits::infinity()) { + // TODO @thomasw21 with more memory we can probably compute a much faster `max-reduce` in parallel O(ln(n)) operations in each memory slot + gpuAtomicMax(&temp_storage[row_id_mem_offset], thread_max); + } + } + + __syncthreads(); + + // Compute exp(elt - max) masked + float exponential[min_kv_length_shard_size_per_thread]; + if (batch_id < batch_size) { + float thread_add = 0; + for (int kv_length_id = kv_length_start; kv_length_id < kv_length_end; ++kv_length_id) { + if (mask[batch_id][kv_length_id] == 0) { + exponential[kv_length_id - kv_length_start] = std::exp(static_cast(attention_scores[batch_id][kv_length_id]) - temp_storage[row_id_mem_offset]); + thread_add = thread_add + exponential[kv_length_id - kv_length_start]; + } else { + exponential[kv_length_id - kv_length_start] = 0.; + } + } + if (thread_add > 0) { + // TODO @thomasw21 with more memory we can probably compute a much faster `sum-reduce` in parallel O(ln(n)) operations in each memory slot + gpuAtomicAdd(&temp_storage[row_id_mem_offset + 1], thread_add); + } + } + + __syncthreads(); + + // Compute softmax + if (batch_id < batch_size) { + // If sum of all exponential is 0, we set the softmax values to 0 + if (temp_storage[row_id_mem_offset + 1] == 0.) { + for (int kv_length_id = kv_length_start; kv_length_id < kv_length_end; ++kv_length_id) { + result[batch_id][kv_length_id] = 0.; + } + } else { + for (int kv_length_id = kv_length_start; kv_length_id < kv_length_end; ++kv_length_id) { + result[batch_id][kv_length_id] = static_cast(exponential[kv_length_id - kv_length_start] / temp_storage[row_id_mem_offset + 1]); + } + } + } +} + +#define CHECK_CUDA(x) TORCH_CHECK(x.device().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +std::tuple>, at::Tensor> forward( + const at::Tensor fused_qkv, + const std::optional> layer_past, + const at::Tensor alibi, + const at::Tensor attention_mask, + const std::optional head_mask, + const float beta, + const float inv_norm_factor, + const int num_heads, + const bool use_cache +) { + const auto batch_size = fused_qkv.size(0); + const auto q_length = fused_qkv.size(1); + const auto three_times_hidden_size = fused_qkv.size(2); + const auto head_dim = three_times_hidden_size / (3 * num_heads); + const auto batch_size_times_num_heads = batch_size * num_heads; + + // `split_heads` + const auto fused_qkv_view = fused_qkv.view({batch_size, q_length, num_heads, 3 * head_dim}); + const auto tensor_list = fused_qkv_view.split(head_dim, -1); + const auto query_layer = tensor_list[0].transpose(1, 2).reshape({batch_size_times_num_heads, q_length, head_dim}); + auto key_layer = tensor_list[1].permute({0, 2, 3, 1}).reshape({batch_size_times_num_heads, head_dim, q_length}); + auto value_layer = tensor_list[2].transpose(1, 2).reshape({batch_size_times_num_heads, q_length, head_dim}); + + if (layer_past) { + const auto past_key = (*layer_past).at(0); + const auto past_value = (*layer_past).at(1); + key_layer = at::cat({past_key, key_layer}, 2); + value_layer = at::cat({past_value, value_layer}, 1); + } + + std::optional> present; + if (use_cache) { + present = {key_layer, value_layer}; + } else { + present = {}; + } + + auto attention_scores = alibi.baddbmm(query_layer, key_layer, beta, inv_norm_factor); + + // Computing `optionally_cast_fp16_to_fp32 + masked_fill + softmax + cast_to_intial_dtype` + at::Tensor attention_probs; + if (true) { + const auto kv_length = key_layer.size(2); + + // TODO @thomasw21: it's easier to think of attention_scores as 2D tensors + const auto attention_scores_2d = attention_scores.view({batch_size_times_num_heads * q_length, kv_length}); + const auto attention_mask_2d = attention_mask.view({batch_size_times_num_heads * q_length, kv_length}); + + // Custom kernel + attention_probs = at::empty_like(attention_scores_2d); + + // Check that inputs and contiguous + cuda tensors + CHECK_INPUT(attention_scores_2d); + CHECK_INPUT(attention_mask_2d); + + // TODO @thomas21: change by to this as it's cleaner when pytorch 1.13 comes out + // DISPATCH_CASE_FLOATING_TYPES(attention_scores.scalar_type(), "masked_softmax", [&] { + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, attention_scores.scalar_type(), "masked_softmax", [&] { + /* + * Understanding how GPUs work: https://developer.nvidia.com/blog/cuda-refresher-cuda-programming-model/ + * A100 specifications: https://images.nvidia.com/aem-dam/en-zz/Solutions/data-center/nvidia-ampere-architecture-whitepaper.pdf + * - SMs: 108 + * - TPCs: 56 (What's that?) + * - Memory size: 40 GB + * - L2 Cache size: 40960 KB (shared across all SMs) + * - L1/Shared memory size: 192 KB (shared across all threads within a SM) + * - Max Threads / SM: 2048 + * - Max Thread Blocks / SM: 32 + */ + + /* + * We should split [batch_size_times_num_heads_block, q_length] in seperate blocks and [batch_size_times_num_heads_block_size, kv_length] a single block + * with multiple threads as we need to `sync_threads` to run exponential sum. + * We maximise the usage of threads within a single block + */ + // TODO @thomasw21 figure out everything warp related: + // - why do they have to be power of 2 + // TODO @thomas21 check why everyone is setting 1024 when officially it's 2048 + const auto MAX_THREADS_PER_SM = 1024; + // TODO @thomasw21 figure out how to have longer sequences, currently the maximum is `max_kv_length = MAX_THREADS_PER_SM * MIN_KV_LENGTH_SHARD_SIZE_PER_THREAD` + const auto MIN_KV_LENGTH_SHARD_SIZE_PER_THREAD = 4; + // `effective_kv_length = ceil(kv_length / MIN_KV_LENGTH_SHARD_SIZE_PER_THREAD)` + const auto effective_kv_length = (kv_length - 1)/ MIN_KV_LENGTH_SHARD_SIZE_PER_THREAD + 1; + const auto rows_per_block = MAX_THREADS_PER_SM / effective_kv_length; + const auto num_blocks = (batch_size_times_num_heads * q_length - 1) / rows_per_block + 1; + + const dim3 gridDim(num_blocks); // Number of blocks that run + const dim3 blockDim(MAX_THREADS_PER_SM); // Number of threads that run per block + const int shared_mem_forward = rows_per_block * 2 * sizeof(float); + + // 192 * 2 ** 10 + // const auto MAX_L1_MEMORY = 196608; + // const auto MAX_SMs = 108; + // TORCH_CHECK(batch_size_times_num_heads * q_length <= MAX_L1_MEMORY, "Shared memory exceeds 192KB limitation."); + // TORCH_CHECK(gridDim.x * gridDim.y * gridDim.z <= MAX_SMs, "A100s only have 108 SMs. Raising as require blocks is bigger."); + // TORCH_CHECK(blockDim.x * blockDim.y * blockDim.z <= MAX_THREADS_PER_SM, "A100s only have 2048 threads per block. Raising as require requested threads is higher."); + + forward_masked_softmax_kernel<<>>( + attention_scores_2d.packed_accessor32(), + attention_mask_2d.packed_accessor32(), + attention_probs.packed_accessor32(), + effective_kv_length, + blockDim, + rows_per_block, + kv_length, + batch_size_times_num_heads * q_length + ); + }); + attention_probs = attention_probs.view({batch_size_times_num_heads, q_length, kv_length}); + } else { + // Pytorch C++ API + auto input_dtype = attention_scores.scalar_type(); + if (input_dtype == at::ScalarType::Float) { + attention_scores = attention_scores.to(at::ScalarType::Float); + }; + // TODO @thomasw21 Figure out how to get minimum value + auto attn_weights = attention_scores.masked_fill_(attention_mask, -1e34); + attention_probs = attn_weights.softmax(-1, at::ScalarType::Float).to(input_dtype); + } + + auto context_layer = attention_probs.bmm(value_layer); + + // `_merge_heads` + context_layer = context_layer.view({batch_size, num_heads, q_length, head_dim}); + context_layer = context_layer.permute({0, 2, 1, 3}); + context_layer = context_layer.reshape({batch_size, q_length, three_times_hidden_size / 3}); + + return std::make_tuple(context_layer, present, attention_probs); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def( + "forward", + &forward, + "Bloom attention mechanism forward (CUDA)" + ); +} \ No newline at end of file diff --git a/server/custom_kernels/setup.py b/server/custom_kernels/setup.py new file mode 100644 index 00000000..43b8ee4e --- /dev/null +++ b/server/custom_kernels/setup.py @@ -0,0 +1,19 @@ +from setuptools import setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +setup( + name="custom_kernels", + ext_modules=[ + CUDAExtension( + name="custom_kernels.fused_bloom_attention_cuda", + sources=["custom_kernels/fused_bloom_attention_cuda.cu"], + extra_compile_args=["-arch=compute_80", "-std=c++17"], + ), + CUDAExtension( + name="custom_kernels.fused_attention_cuda", + sources=["custom_kernels/fused_attention_cuda.cu"], + extra_compile_args=["-arch=compute_80", "-std=c++17"], + ), + ], + cmdclass={"build_ext": BuildExtension}, +) diff --git a/server/pyproject.toml b/server/pyproject.toml index d381eac4..f0ec25eb 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -25,7 +25,8 @@ opentelemetry-instrumentation-grpc = "^0.36b0" hf-transfer = "^0.1.2" sentencepiece = "^0.1.97" tokenizers = "0.13.3" -huggingface-hub = "0.14.0" +huggingface-hub = "^0.14.1" +transformers = "^4.29.2" [tool.poetry.extras] accelerate = ["accelerate"] diff --git a/server/requirements.txt b/server/requirements.txt index 50ba4e43..e8cee52b 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -13,8 +13,8 @@ grpcio-reflection==1.55.0 ; python_version >= "3.9" and python_version < "4.0" grpcio-status==1.55.0 ; python_version >= "3.9" and python_version < "4.0" grpcio==1.55.0 ; python_version >= "3.9" and python_version < "4.0" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" -huggingface-hub==0.14.0 ; python_version >= "3.9" and python_version < "4.0" -idna==3.4 ; python_version >= "3.9" and python_version < "4.0" +huggingface-hub==0.14.1 ; python_version >= "3.9" and python_version < "4.0" +idna==3.4 ; python_version >= "3.9" and python_version < "4" loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" @@ -33,6 +33,7 @@ safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "4.0" setuptools==67.8.0 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" +transformers==4.29.2 ; python_version >= "3.9" and python_version < "4.0" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" typing-extensions==4.6.0 ; python_version >= "3.9" and python_version < "4.0" diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 338fe053..71013cb6 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -6,12 +6,17 @@ from text_generation_server.pb import generate_pb2 from text_generation_server.models.causal_lm import CausalLMBatch -from text_generation_server.models.bloom import BloomCausalLMBatch, BLOOM +from text_generation_server.utils import weight_hub_files, download_weights +from text_generation_server.models.bloom import BloomCausalLMBatch, BLOOMSharded @pytest.fixture(scope="session") def default_bloom(): - return BLOOM("bigscience/bloom-560m") + model_id = "bigscience/bloom-560m" + revision = "main" + filenames = weight_hub_files(model_id, revision, ".safetensors") + download_weights(filenames, model_id, revision) + return BLOOMSharded(model_id) @pytest.fixture(scope="session") diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index fc92d03d..f1b84a53 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -1,3 +1,4 @@ +import os import torch from loguru import logger @@ -8,17 +9,20 @@ from text_generation_server.models.model import Model from text_generation_server.models.causal_lm import CausalLM from text_generation_server.models.flash_causal_lm import FlashCausalLM -from text_generation_server.models.bloom import BLOOM, BLOOMSharded +from text_generation_server.models.bloom import BLOOMSharded from text_generation_server.models.seq2seq_lm import Seq2SeqLM from text_generation_server.models.rw import RW -from text_generation_server.models.opt import OPT, OPTSharded -from text_generation_server.models.galactica import Galactica, GalacticaSharded +from text_generation_server.models.opt import OPTSharded +from text_generation_server.models.galactica import GalacticaSharded from text_generation_server.models.santacoder import SantaCoder -from text_generation_server.models.gpt_neox import GPTNeoxSharded from text_generation_server.models.t5 import T5Sharded +from text_generation_server.models.gpt_neox import GPTNeoxSharded try: - if torch.cuda.is_available(): + if ( + torch.cuda.is_available() + and not os.getenv("USE_FLASH_ATTENTION", "").lower() == "false" + ): major, minor = torch.cuda.get_device_capability() is_sm75 = major == 7 and minor == 5 is_sm8x = major == 8 and minor >= 0 @@ -30,14 +34,12 @@ f"GPU with CUDA capability {major} {minor} is not supported" ) - from text_generation_server.models.flash_neox import FlashNeoX, FlashNeoXSharded - from text_generation_server.models.flash_rw import FlashRW, FlashRWSharded + from text_generation_server.models.flash_rw import FlashRWSharded + from text_generation_server.models.flash_neox import FlashNeoXSharded from text_generation_server.models.flash_llama import ( FlashLlama, - FlashLlamaSharded, ) from text_generation_server.models.flash_santacoder import ( - FlashSantacoder, FlashSantacoderSharded, ) @@ -52,30 +54,22 @@ __all__ = [ "Model", - "BLOOM", "BLOOMSharded", "CausalLM", "FlashCausalLM", - "Galactica", "GalacticaSharded", - "GPTNeoxSharded", "Seq2SeqLM", "SantaCoder", - "OPT", "OPTSharded", "T5Sharded", "get_model", ] if FLASH_ATTENTION: - __all__.append(FlashNeoX) __all__.append(FlashNeoXSharded) - __all__.append(FlashRW) __all__.append(FlashRWSharded) - __all__.append(FlashSantacoder) __all__.append(FlashSantacoderSharded) __all__.append(FlashLlama) - __all__.append(FlashLlamaSharded) FLASH_ATT_ERROR_MESSAGE = ( "{} requires Flash Attention CUDA kernels to be installed.\n" @@ -102,36 +96,24 @@ def get_model( trust_remote_code: bool, ) -> Model: if "facebook/galactica" in model_id: - if sharded: - return GalacticaSharded( - model_id, - revision, - quantize=quantize, - trust_remote_code=trust_remote_code, - ) - else: - return Galactica( - model_id, - revision, - quantize=quantize, - trust_remote_code=trust_remote_code, - ) + return GalacticaSharded( + model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + ) if model_id.startswith("bigcode/"): - if sharded: - if not FLASH_ATTENTION: - raise NotImplementedError( - FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Santacoder") - ) + if FLASH_ATTENTION: return FlashSantacoderSharded( model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code, ) + elif sharded: + raise NotImplementedError( + FLASH_ATT_ERROR_MESSAGE.format("Sharded Santacoder") + ) else: - santacoder_cls = FlashSantacoder if FLASH_ATTENTION else SantaCoder - return santacoder_cls( + return SantaCoder( model_id, revision, quantize=quantize, @@ -144,20 +126,19 @@ def get_model( model_type = config_dict["model_type"] if model_type == "gpt_bigcode": - if sharded: - if not FLASH_ATTENTION: - raise NotImplementedError( - FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Santacoder") - ) + if FLASH_ATTENTION: return FlashSantacoderSharded( model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code, ) + elif sharded: + raise NotImplementedError( + FLASH_ATT_ERROR_MESSAGE.format("Sharded Santacoder") + ) else: - santacoder_cls = FlashSantacoder if FLASH_ATTENTION else SantaCoder - return santacoder_cls( + return SantaCoder( model_id, revision, quantize=quantize, @@ -165,33 +146,45 @@ def get_model( ) if model_type == "bloom": - if sharded: - return BLOOMSharded( + return BLOOMSharded( + model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + ) + + elif model_type == "gpt_neox": + if FLASH_ATTENTION: + return FlashNeoXSharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) + elif sharded: + return GPTNeoxSharded( model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code, ) else: - return BLOOM( + return CausalLM( model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code, ) - if model_type == "gpt_neox": - if sharded: - neox_cls = FlashNeoXSharded if FLASH_ATTENTION else GPTNeoxSharded - return neox_cls( + elif model_type == "llama": + if FLASH_ATTENTION: + return FlashLlama( model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code, ) + elif sharded: + raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format("Sharded Llama")) else: - neox_cls = FlashNeoX if FLASH_ATTENTION else CausalLM - return neox_cls( + return CausalLM( model_id, revision, quantize=quantize, @@ -217,7 +210,7 @@ def get_model( ) else: if FLASH_ATTENTION and not config_dict.get("alibi", False): - return FlashRW( + return FlashRWSharded( model_id, revision, quantize=quantize, @@ -231,42 +224,12 @@ def get_model( trust_remote_code=trust_remote_code, ) - if model_type == "llama": - if sharded: - if FLASH_ATTENTION: - return FlashLlamaSharded( - model_id, - revision, - quantize=quantize, - trust_remote_code=trust_remote_code, - ) - raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Llama")) - else: - llama_cls = FlashLlama if FLASH_ATTENTION else CausalLM - return llama_cls( - model_id, - revision, - quantize=quantize, - trust_remote_code=trust_remote_code, - ) - - if model_type == "opt": - if sharded: - return OPTSharded( - model_id, - revision, - quantize=quantize, - trust_remote_code=trust_remote_code, - ) - else: - return OPT( - model_id, - revision, - quantize=quantize, - trust_remote_code=trust_remote_code, - ) + elif model_type == "opt": + return OPTSharded( + model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + ) - if model_type == "t5": + elif model_type == "t5": if sharded: return T5Sharded( model_id, diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 45d7cd4c..50b3b76a 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -1,37 +1,26 @@ import torch import torch.distributed -from typing import List, Optional, Type +from typing import Optional, Type -from accelerate import init_empty_weights -from safetensors import safe_open from transformers import ( AutoTokenizer, - AutoModelForCausalLM, AutoConfig, PreTrainedTokenizerBase, ) -from transformers.models.bloom.parallel_layers import ( - TensorParallelColumnLinear, - TensorParallelEmbedding, - TensorParallelRowLinear, -) +from text_generation_server.models.custom_modeling.bloom_modeling import ( + BloomForCausalLM, +) from text_generation_server.models import CausalLM from text_generation_server.models.causal_lm import CausalLMBatch from text_generation_server.pb import generate_pb2 from text_generation_server.utils import ( initialize_torch_distributed, weight_files, + Weights, ) -HAS_BITS_AND_BYTES = True -try: - import bitsandbytes as bnb - from bitsandbytes.nn import Int8Params -except Exception as e: - HAS_BITS_AND_BYTES = False - class BloomCausalLMBatch(CausalLMBatch): @classmethod @@ -42,34 +31,12 @@ def from_pb( dtype: torch.dtype, device: torch.device, ) -> "CausalLMBatch": - batch = super(BloomCausalLMBatch, cls).from_pb( - pb=pb, tokenizer=tokenizer, dtype=dtype, device=device - ) + batch = super().from_pb(pb=pb, tokenizer=tokenizer, dtype=dtype, device=device) batch.keys_head_dim_last = False return batch -class BLOOM(CausalLM): - def __init__( - self, - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - trust_remote_code: bool = False, - ): - super(BLOOM, self).__init__( - model_id=model_id, - revision=revision, - quantize=quantize, - trust_remote_code=trust_remote_code, - ) - - @property - def batch_type(self) -> Type[CausalLMBatch]: - return BloomCausalLMBatch - - -class BLOOMSharded(BLOOM): +class BLOOMSharded(CausalLM): def __init__( self, model_id: str, @@ -101,25 +68,16 @@ def __init__( trust_remote_code=trust_remote_code, ) config.pad_token_id = 3 + config.quantize = quantize torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights( + filenames, device=device, dtype=dtype, process_group=self.process_group + ) - with init_empty_weights(): - model = AutoModelForCausalLM.from_config( - config, trust_remote_code=trust_remote_code - ) + model = BloomForCausalLM(config, weights) - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=device, - dtype=dtype, - rank=rank, - world_size=world_size, - ) torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( model=model, @@ -131,132 +89,9 @@ def __init__( world_size=world_size, ) - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - rank: int, - world_size: int, - ): - parameters = dict(model.named_parameters()) - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if quantize is None else "cpu" - ) as f: - for name in f.keys(): - if name.startswith("transformer.") or name.startswith("lm_head."): - full_name = name - else: - full_name = f"transformer.{name}" - - module_name, param_name = full_name.rsplit(".", 1) - module = model.get_submodule(module_name) - current_tensor = parameters[full_name] - - slice_ = f.get_slice(name) - - if isinstance(module, TensorParallelColumnLinear): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif isinstance(module, TensorParallelRowLinear): - if param_name == "weight": - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - else: - tensor = slice_[:] - # XXX: Hack for Rowlinear to add the bias only once. - if rank != 0: - tensor = torch.zeros_like(tensor) - elif ( - isinstance(module, TensorParallelEmbedding) - or name == "lm_head.weight" - ): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - tensor = slice_[:] - - if current_tensor.shape != tensor.shape: - raise ValueError( - f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" - ) - - tensor = tensor.contiguous().to(dtype) - - if quantize == "bitsandbytes": - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine either because it is not installed " - "or you don't have a GPU.\n" - "You can install it with `pip install bitsandbytes`." - ) - - if ( - type(module) - in [TensorParallelRowLinear, TensorParallelColumnLinear] - and param_name == "weight" - ): - tensor = Int8Params( - tensor, - has_fp16_weights=False, - requires_grad=False, - ).to(device) - state = bnb.MatmulLtState() - state.threshold = 6.0 - state.has_fp16_weights = False - state.memory_efficient_backward = False - state.use_pool = True - state.CB = tensor.CB - state.SCB = tensor.SCB - tensor.CB = None - tensor.SCB = None - - def replace_linear(state): - def linear(input, weight, bias): - out = bnb.matmul( - input, - weight, - state=state, - threshold=state.threshold, - bias=bias, - ) - - if state.CB is not None: - # we converted 8-bit row major to turing/ampere format - # in the first inference pass - # we no longer need the row-major weight - del state.CB - weight.data = state.CxB - - return out - - return linear - - module.linear = replace_linear(state) - else: - tensor = tensor.to(device) - elif quantize == "gptq": - raise NotImplementedError("`gptq` is not implemented for now") - elif quantize is None: - tensor = tensor.to(device) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") - - module._parameters[param_name] = tensor - if name == "word_embeddings.weight": - model.lm_head._parameters["weight"] = tensor + @property + def batch_type(self) -> Type[CausalLMBatch]: + return BloomCausalLMBatch def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None @@ -269,9 +104,5 @@ def forward( use_cache=True, ) - # Logits are sharded, so we need to gather them - logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] - torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) - logits = torch.cat(logits, dim=2) - + logits = outputs.logits return logits, outputs.past_key_values diff --git a/server/text_generation_server/models/custom_modeling/bloom_modeling.py b/server/text_generation_server/models/custom_modeling/bloom_modeling.py new file mode 100644 index 00000000..e5e87645 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/bloom_modeling.py @@ -0,0 +1,912 @@ +# coding=utf-8 +# Copyright 2022 HuggingFace Inc. team and BigScience workshop. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch BLOOM model.""" + +import math +import os +import warnings +from typing import Optional, Tuple, Union + +import torch +import torch.distributed +import torch.utils.checkpoint +from torch import nn +from torch.nn import LayerNorm +from torch.nn import functional as F + +from transformers.modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + CausalLMOutputWithCrossAttentions, +) +from transformers import BloomConfig, PreTrainedModel + +from text_generation_server.utils.layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, + TensorParallelHead, +) + +CUSTOM_KERNELS_ENABLED = False +if not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True": + try: + from custom_kernels import fused_bloom_attention_cuda + + CUSTOM_KERNELS_ENABLED = True + except ImportError: + pass + +_CHECKPOINT_FOR_DOC = "bigscience/bloom-560m" +_CONFIG_FOR_DOC = "BloomConfig" + +BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "bigscience/bigscience-small-testing", + "bigscience/bloom-560m", + "bigscience/bloom-1b1", + "bigscience/bloom-1b7", + "bigscience/bloom-3b", + "bigscience/bloom-7b1", + "bigscience/bloom", +] + + +def _make_causal_mask( + input_ids_shape: torch.Size, device: torch.device, past_key_values_length: int +) -> torch.BoolTensor: + """ + Make causal mask used for self-attention. + """ + batch_size, target_length = input_ids_shape + mask = torch.ones( + (target_length, target_length + past_key_values_length), + dtype=torch.bool, + device=device, + ) + mask = mask.triu(1 + past_key_values_length) + + expanded_mask = mask.unsqueeze(0).expand( + batch_size, target_length, target_length + past_key_values_length + ) + return expanded_mask + + +def _expand_mask(mask: torch.Tensor, tgt_length: int) -> torch.BoolTensor: + """ + Expands attention_mask from `[batch_size, src_length]` to `[batch_size, 1, tgt_length, src_length]`. + """ + batch_size, src_length = mask.shape + tgt_length = tgt_length if tgt_length is not None else src_length + + expanded_mask = ~(mask[:, None, :].to(torch.bool)) + return expanded_mask.expand(batch_size, tgt_length, src_length) + + +def build_alibi_tensor(attention_mask: torch.Tensor, num_heads: int) -> torch.Tensor: + """ + Link to paper: https://arxiv.org/abs/2108.12409 Alibi tensor is not causal as the original paper mentions, it + relies on a translation invariance of softmax for quick implementation: with l being a tensor, and a fixed value + `softmax(l+a) = softmax(l)`. Based on + https://github.com/ofirpress/attention_with_linear_biases/blob/a35aaca144e0eb6b789dfcb46784c4b8e31b7983/fairseq/models/transformer.py#L742 + TODO @thomasw21 this doesn't work as nicely due to the masking strategy, and so masking varies slightly. + + Args: + Returns tensor shaped (batch_size * num_heads, 1, max_seq_len) + attention_mask (`torch.Tensor`): + Token-wise attention mask, this should be of shape (batch_size, max_seq_len). + num_heads (`int`, *required*): + number of heads + dtype (`torch.dtype`, *optional*, default=`torch.bfloat16`): + dtype of the output tensor + """ + batch_size, seq_length = attention_mask.shape + closest_power_of_2 = 2 ** math.floor(math.log2(num_heads)) + base = torch.tensor( + 2 ** (-(2 ** -(math.log2(closest_power_of_2) - 3))), + device=attention_mask.device, + dtype=torch.float32, + ) + powers = torch.arange( + 1, 1 + closest_power_of_2, device=attention_mask.device, dtype=torch.int32 + ) + slopes = torch.pow(base, powers) + + if closest_power_of_2 != num_heads: + extra_base = torch.tensor( + 2 ** (-(2 ** -(math.log2(2 * closest_power_of_2) - 3))), + device=attention_mask.device, + dtype=torch.float32, + ) + num_remaining_heads = min(closest_power_of_2, num_heads - closest_power_of_2) + extra_powers = torch.arange( + 1, + 1 + 2 * num_remaining_heads, + 2, + device=attention_mask.device, + dtype=torch.int32, + ) + slopes = torch.cat([slopes, torch.pow(extra_base, extra_powers)], dim=0) + + # Note: alibi will added to the attention bias that will be applied to the query, key product of attention + # => therefore alibi will have to be of shape (batch_size, num_heads, query_length, key_length) + # => here we set (batch_size=1, num_heads=num_heads, query_length=1, key_length=max_length) + # => the query_length dimension will then be broadcasted correctly + # This is more or less identical to T5's relative position bias: + # https://github.com/huggingface/transformers/blob/f681437203baa7671de3174b0fa583c349d9d5e1/src/transformers/models/t5/modeling_t5.py#L527 + arange_tensor = ((attention_mask.cumsum(dim=-1) - 1) * attention_mask)[:, None, :] + alibi = slopes[..., None] * arange_tensor + return alibi + + +# @torch.jit.script +def dropout_add( + x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool +) -> torch.Tensor: + """ + Dropout add function + + Args: + x (`torch.tensor`, *required*): + input tensor + residual (`torch.tensor`, *required*): + esidual tensor + prob (`float`, *required*): + dropout probability + training (`bool`, *required*): + training mode + """ + out = F.dropout(x, p=prob, training=training) + out = residual + out + return out + + +# @torch.jit.script # this is shit for unknow reasons. +def _split_heads( + fused_qkv: torch.Tensor, num_heads: int, head_dim: int +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory + storage as `fused_qkv` + + Args: + fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim] + + Returns: + query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim] + value: [batch_size, seq_length, num_heads, head_dim] + """ + batch_size, seq_length, three_times_hidden_size = fused_qkv.shape + fused_qkv = fused_qkv.view(batch_size, seq_length, num_heads, 3 * head_dim) + query_layer, key_layer, value_layer = fused_qkv.split(head_dim, dim=-1) + + query_layer = query_layer.transpose(1, 2).reshape( + batch_size * num_heads, seq_length, head_dim + ) + key_layer = key_layer.permute(0, 2, 3, 1).reshape( + batch_size * num_heads, head_dim, seq_length + ) + value_layer = value_layer.transpose(1, 2).reshape( + batch_size * num_heads, seq_length, head_dim + ) + + return query_layer, key_layer, value_layer + + +# @torch.jit.script +def _merge_heads(x: torch.Tensor, num_heads: int, head_dim: int) -> torch.Tensor: + """ + Merge heads together over the last dimenstion + + Args: + x: (`torch.tensor`, *required*): [batch_size * num_heads, seq_length, head_dim] + + Returns: + torch.tensor: [batch_size, seq_length, num_heads * head_dim] + """ + # What we want to achieve is: + # batch_size * num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads * head_dim + batch_size_and_num_heads, seq_length, _ = x.shape + batch_size = batch_size_and_num_heads // num_heads + + # First view to decompose the batch size + # batch_size * num_heads, seq_length, head_dim -> batch_size, num_heads, seq_length, head_dim + x = x.view(batch_size, num_heads, seq_length, head_dim) + + # batch_size, num_heads, seq_length, head_dim -> batch_size, seq_length, num_heads, head_dim + x = x.permute(0, 2, 1, 3) + + # batch_size, seq_length, num_heads, head_dim -> batch_size, seq_length, num_heads * head_dim + return x.reshape(batch_size, seq_length, num_heads * head_dim) + + +class BloomAttention(nn.Module): + def __init__(self, prefix, config: BloomConfig, weights): + super().__init__() + + self.pretraining_tp = config.pretraining_tp + self.slow_but_exact = config.slow_but_exact + + self.process_group = weights.process_group + + self.hidden_size = config.hidden_size + self.num_heads = config.n_head + self.head_dim = self.hidden_size // self.num_heads + self.split_size = self.hidden_size + self.hidden_dropout = config.hidden_dropout + + if self.head_dim * self.num_heads != self.hidden_size: + raise ValueError( + f"`hidden_size` must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:" + f" {self.num_heads})." + ) + + # Layer-wise attention scaling + self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim) + self.beta = 1.0 + + process_group = weights.process_group + self.num_heads = self.num_heads // process_group.size() + self.query_key_value = TensorParallelColumnLinear.load( + config=config, + prefix=f"{prefix}.query_key_value", + weights=weights, + bias=True, + ) + self.dense = TensorParallelRowLinear.load( + config=config, prefix=f"{prefix}.dense", weights=weights, bias=True + ) + self.attention_dropout = nn.Dropout(config.attention_dropout) + + @staticmethod + def compute_attention( + fused_qkv: torch.Tensor, + layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]], + alibi: torch.Tensor, + attention_mask: torch.Tensor, + head_mask: Optional[torch.Tensor], + beta: float, + inv_norm_factor: float, + num_heads: int, + use_cache: bool, + ): + batch_size, q_length, three_times_hidden_size = fused_qkv.shape + head_dim = three_times_hidden_size // (3 * num_heads) + batch_size * num_heads + + ### TODO @thomasw21: this takes quite a bit of time, how do I accelerate that? + # 3 x [batch_size, seq_length, num_heads, head_dim] + (query_layer, key_layer, value_layer) = _split_heads( + fused_qkv, num_heads=num_heads, head_dim=head_dim + ) + + if layer_past is not None: + past_key, past_value = layer_past + # concatenate along seq_length dimension: + # - key: [batch_size * self.num_heads, head_dim, kv_length] + # - value: [batch_size * self.num_heads, kv_length, head_dim] + past_key = past_key.view(-1, *past_key.shape[-2:]) + key_layer = torch.cat((past_key, key_layer), dim=2) + past_value = past_value.view(-1, *past_value.shape[-2:]) + value_layer = torch.cat((past_value, value_layer), dim=1) + + _, _, kv_length = key_layer.shape + + if use_cache is True: + present = (key_layer, value_layer) + else: + present = None + ### + + # [batch_size * num_heads, q_length, kv_length] + # we use `torch.Tensor.baddbmm` instead of `torch.baddbmm` as the latter isn't supported by TorchScript v1.11 + attention_scores = alibi.baddbmm( + batch1=query_layer, + batch2=key_layer, + beta=beta, + alpha=inv_norm_factor, + ) + + # cast attention scores to fp32, compute scaled softmax and cast back to initial dtype - [batch_size, num_heads, q_length, kv_length] + input_dtype = attention_scores.dtype + # `float16` has a minimum value of -65504.0, whereas `bfloat16` and `float32` have a minimum value of `-3.4e+38` + if input_dtype == torch.float16: + attention_scores = attention_scores.to(torch.float) + # torch.finfo not supported by torch.jit, we temporarily remplace with `-1e34` + attn_weights = attention_scores.masked_fill_( + attention_mask, torch.finfo(attention_scores.dtype).min + ) + attention_probs = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to( + input_dtype + ) + + # # [batch_size, num_heads, q_length, kv_length] + # attention_probs = self.attention_dropout(attention_probs) + + if head_mask is not None: + attention_probs = attention_probs * head_mask + + # matmul: [batch_size * num_heads, q_length, head_dim] + context_layer = torch.bmm(attention_probs, value_layer, out=query_layer) + + # change view [batch_size, num_heads, q_length, head_dim] + context_layer = _merge_heads( + context_layer, num_heads=num_heads, head_dim=head_dim + ) + + return context_layer, present, attention_probs + + def forward( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + alibi: torch.Tensor, + attention_mask: torch.Tensor, + layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + head_mask: Optional[torch.Tensor] = None, + use_cache: bool = False, + output_attentions: bool = False, + ): + fused_qkv = self.query_key_value( + hidden_states + ) # [batch_size, seq_length, 3 x hidden_size] + batch_size, q_length, _ = fused_qkv.shape + + if layer_past is not None: + past_key, past_value = layer_past + layer_past = ( + past_key.view(-1, *past_key.shape[-2:]), + past_value.view(-1, *past_value.shape[-2:]), + ) + + if CUSTOM_KERNELS_ENABLED: + assert self.training is False, "Only foward pass was implemented" + assert ( + attention_mask.shape[-1] < 4096 + ), "Custom kernel support only up to 4096 tokens" + ( + context_layer, + present, + attention_probs, + ) = fused_bloom_attention_cuda.forward( + fused_qkv, + layer_past, + alibi, + attention_mask, + head_mask, + self.beta, + self.inv_norm_factor, + self.num_heads, + use_cache, + ) + else: + context_layer, present, attention_probs = self.compute_attention( + fused_qkv=fused_qkv, + layer_past=layer_past, + alibi=alibi, + attention_mask=attention_mask, + head_mask=head_mask, + beta=self.beta, + inv_norm_factor=self.inv_norm_factor, + num_heads=self.num_heads, + use_cache=use_cache, + ) + + # aggregate results across tp ranks. See here: https://github.com/pytorch/pytorch/issues/76232 + if self.pretraining_tp > 1 and self.slow_but_exact: + slices = self.hidden_size / self.pretraining_tp + output_tensor = torch.zeros_like(context_layer) + for i in range(self.pretraining_tp): + output_tensor = output_tensor + F.linear( + context_layer[:, :, int(i * slices) : int((i + 1) * slices)], + self.dense.weight[:, int(i * slices) : int((i + 1) * slices)], + ) + else: + output_tensor = self.dense(context_layer) + + # output_tensor = dropout_add(output_tensor, residual, self.hidden_dropout, self.training) + output_tensor += residual + + outputs = (output_tensor, present) + if output_attentions: + outputs += (attention_probs,) + + return outputs + + +class BloomMLP(nn.Module): + def __init__(self, prefix, config: BloomConfig, weights): + super().__init__() + + self.pretraining_tp = config.pretraining_tp + self.slow_but_exact = config.slow_but_exact + self.dense_h_to_4h = TensorParallelColumnLinear.load( + config=config, prefix=f"{prefix}.dense_h_to_4h", weights=weights, bias=True + ) + self.dense_4h_to_h = TensorParallelRowLinear.load( + config=config, prefix=f"{prefix}.dense_4h_to_h", weights=weights, bias=True + ) + self.gelu_impl = torch.nn.GELU(approximate="tanh") + self.hidden_dropout = config.hidden_dropout + + def forward( + self, hidden_states: torch.Tensor, residual: torch.Tensor + ) -> torch.Tensor: + hidden_states = self.gelu_impl(self.dense_h_to_4h(hidden_states)) + + if self.pretraining_tp > 1 and self.slow_but_exact: + intermediate_output = torch.zeros_like(residual) + slices = self.dense_4h_to_h.weight.shape[-1] / self.pretraining_tp + for i in range(self.pretraining_tp): + intermediate_output = intermediate_output + F.linear( + hidden_states[:, :, int(i * slices) : int((i + 1) * slices)], + self.dense_4h_to_h.weight[ + :, int(i * slices) : int((i + 1) * slices) + ], + ) + else: + intermediate_output = self.dense_4h_to_h(hidden_states) + + # output = dropout_add(intermediate_output, residual, self.hidden_dropout, self.training) + intermediate_output += residual + + return intermediate_output + + +class BloomBlock(nn.Module): + def __init__(self, layer_id: int, config: BloomConfig, weights): + super().__init__() + + prefix = f"h.{layer_id}" + self.input_layernorm = LayerNorm.load( + prefix=f"{prefix}.input_layernorm", + weights=weights, + eps=config.layer_norm_epsilon, + ) + self.num_heads = config.n_head + self.self_attention = BloomAttention( + prefix=f"{prefix}.self_attention", config=config, weights=weights + ) + self.post_attention_layernorm = LayerNorm.load( + prefix=f"{prefix}.post_attention_layernorm", + weights=weights, + eps=config.layer_norm_epsilon, + ) + + self.mlp = BloomMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) + self.apply_residual_connection_post_layernorm = ( + config.apply_residual_connection_post_layernorm + ) + self.hidden_dropout = config.hidden_dropout + + def forward( + self, + hidden_states: torch.Tensor, + alibi: torch.Tensor, + attention_mask: torch.Tensor, + layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + head_mask: Optional[torch.Tensor] = None, + use_cache: bool = False, + output_attentions: bool = False, + ): + # hidden_states: [batch_size, seq_length, hidden_size] + + # Layer norm at the beginning of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + + # Layer norm post the self attention. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + # Self attention. + attn_outputs = self.self_attention( + layernorm_output, + residual, + layer_past=layer_past, + attention_mask=attention_mask, + alibi=alibi, + head_mask=head_mask, + use_cache=use_cache, + output_attentions=output_attentions, + ) + + attention_output = attn_outputs[0] + + outputs = attn_outputs[1:] + + layernorm_output = self.post_attention_layernorm(attention_output) + + # Get residual + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = attention_output + + # MLP. + output = self.mlp(layernorm_output, residual) + + if use_cache: + outputs = (output,) + outputs + else: + outputs = (output,) + outputs[1:] + + return outputs # hidden_states, present, attentions + + +class BloomPreTrainedModel(PreTrainedModel): + config_class = BloomConfig + base_model_prefix = "transformer" + _no_split_modules = ["BloomBlock"] + + @staticmethod + def _convert_to_standard_cache( + past_key_value: Tuple[Tuple[torch.Tensor, torch.Tensor]], batch_size: int + ) -> Tuple[Tuple[torch.Tensor, torch.Tensor]]: + """ + Standardizes the format of the cache so as to match most implementations, i.e. to tuple(tuple([batch_size, + num_heads, ...])) + """ + batch_size_times_num_heads, head_dim, seq_length = past_key_value[0][0].shape + num_heads = batch_size_times_num_heads // batch_size + # key: [batch_size * num_heads, head_dim, seq_length] -> [batch_size, num_heads, head_dim, seq_length] + # value: [batch_size * num_heads, seq_length, head_dim] -> [batch_size, num_heads, seq_length, head_dim] + return tuple( + ( + layer_past[0].view(batch_size, num_heads, head_dim, seq_length), + layer_past[1].view(batch_size, num_heads, seq_length, head_dim), + ) + for layer_past in past_key_value + ) + + @staticmethod + def _convert_to_bloom_cache( + past_key_value: Tuple[Tuple[torch.Tensor, torch.Tensor]] + ) -> Tuple[Tuple[torch.Tensor, torch.Tensor]]: + """ + Converts the cache to the format expected by Bloom, i.e. to tuple(tuple([batch_size * num_heads, ...])) + """ + batch_size, num_heads, head_dim, seq_length = past_key_value[0][0].shape + batch_size_times_num_heads = batch_size * num_heads + # key: [batch_size, num_heads, head_dim, seq_length] -> [batch_size * num_heads, head_dim, seq_length] + # value: [batch_size, num_heads, seq_length, head_dim] -> [batch_size * num_heads, seq_length, head_dim] + return tuple( + ( + layer_past[0].view(batch_size_times_num_heads, head_dim, seq_length), + layer_past[1].view(batch_size_times_num_heads, seq_length, head_dim), + ) + for layer_past in past_key_value + ) + + +class BloomModel(BloomPreTrainedModel): + def __init__(self, config: BloomConfig, weights): + super().__init__(config) + + self.embed_dim = config.hidden_size + self.num_heads = config.n_head + + process_group = weights.process_group + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + + self.word_embeddings = TensorParallelEmbedding( + prefix="word_embeddings", weights=weights + ) + + self.word_embeddings_layernorm = LayerNorm.load( + prefix="word_embeddings_layernorm", + weights=weights, + eps=config.layer_norm_epsilon, + ) + + # Transformer blocks + self.h = nn.ModuleList( + [ + BloomBlock(layer_id=layer_id, config=config, weights=weights) + for layer_id in range(config.num_hidden_layers) + ] + ) + + # Final Layer Norm + self.ln_f = LayerNorm.load( + prefix="ln_f", weights=weights, eps=config.layer_norm_epsilon + ) + + def _prepare_attn_mask( + self, + attention_mask: torch.Tensor, + input_shape: Tuple[int, int], + past_key_values_length: int, + ) -> torch.BoolTensor: + # create causal mask + # [batch_size, seq_length] -> [batch_size, tgt_length, src_length] + combined_attention_mask = None + device = attention_mask.device + _, src_length = input_shape + + if src_length > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + device=device, + past_key_values_length=past_key_values_length, + ) + + # [batch_size, seq_length] -> [batch_size, tgt_length, src_length] + expanded_attn_mask = _expand_mask(attention_mask, tgt_length=src_length) + combined_attention_mask = ( + expanded_attn_mask + if combined_attention_mask is None + else expanded_attn_mask | combined_attention_mask + ) + + return combined_attention_mask + + def set_input_embeddings(self, new_embeddings: torch.Tensor): + self.word_embeddings = new_embeddings + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **deprecated_arguments, + ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]: + if deprecated_arguments.pop("position_ids", False) is not False: + # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` + warnings.warn( + "`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore" + " passing `position_ids`.", + FutureWarning, + ) + if len(deprecated_arguments) > 0: + raise ValueError(f"Got unexpected arguments: {deprecated_arguments}") + + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + "You cannot specify both input_ids and inputs_embeds at the same time" + ) + elif input_ids is not None: + batch_size, seq_length = input_ids.shape + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + if past_key_values is None: + past_key_values = tuple([None] * len(self.h)) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape batch_size x num_heads x N x N + # head_mask has shape n_layer x batch x num_heads x N x N + head_mask = self.get_head_mask(head_mask, self.config.n_layer) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + hidden_states = self.word_embeddings_layernorm(inputs_embeds) + + presents = () if use_cache else None + all_self_attentions = () if output_attentions else None + all_hidden_states = () if output_hidden_states else None + + # Compute alibi tensor: check build_alibi_tensor documentation + seq_length_with_past = seq_length + past_key_values_length = 0 + if past_key_values[0] is not None: + past_key_values_length = past_key_values[0][0].shape[-1] + seq_length_with_past = seq_length_with_past + past_key_values_length + if attention_mask is None: + attention_mask = torch.ones( + (batch_size, seq_length_with_past), device=hidden_states.device + ) + else: + attention_mask = attention_mask.to(hidden_states.device) + + alibi = build_alibi_tensor(attention_mask, self.num_heads) + + causal_mask = self._prepare_attn_mask( + attention_mask, + input_shape=(batch_size, seq_length), + past_key_values_length=past_key_values_length, + ) + + if hasattr(self, "tp_rank"): + assert self.num_heads % self.tp_world_size == 0 + block_size = self.num_heads // self.tp_world_size + alibi = alibi[ + :, self.tp_rank * block_size : (self.tp_rank + 1) * block_size + ] + alibi = alibi.reshape(batch_size * block_size, 1, seq_length_with_past) + causal_mask = torch.repeat_interleave(causal_mask, block_size, dim=0) + else: + alibi = alibi.reshape(batch_size * self.num_heads, 1, seq_length_with_past) + causal_mask = torch.repeat_interleave(causal_mask, self.num_heads, dim=0) + + alibi = alibi.to(hidden_states.dtype) + + for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = block( + hidden_states, + layer_past=layer_past, + attention_mask=causal_mask, + head_mask=head_mask[i], + use_cache=use_cache, + output_attentions=output_attentions, + alibi=alibi, + ) + + hidden_states = outputs[0] + if use_cache is True: + presents = presents + (outputs[1],) + + if output_attentions: + all_self_attentions = all_self_attentions + ( + outputs[2 if use_cache else 1], + ) + + # Add last hidden state + hidden_states = self.ln_f(hidden_states) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + presents, + all_hidden_states, + all_self_attentions, + ] + if v is not None + ) + + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=presents, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + ) + + +class BloomForCausalLM(BloomPreTrainedModel): + def __init__(self, config, weights): + super().__init__(config) + self.transformer = BloomModel(config, weights) + + self.lm_head = TensorParallelHead.load( + config, + prefix="word_embeddings", + weights=weights, + ) + + def prepare_inputs_for_generation( + self, + input_ids: torch.LongTensor, + past_key_values: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + **kwargs, + ) -> dict: + # only last token for input_ids if past is not None + if past_key_values: + input_ids = input_ids[:, -1].unsqueeze(-1) + + # the cache may be in the stardard format (e.g. in contrastive search), convert to bloom's format if needed + if past_key_values[0][0].shape[0] == input_ids.shape[0]: + past_key_values = self._convert_to_bloom_cache(past_key_values) + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **deprecated_arguments, + ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set + `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` + are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` + """ + if deprecated_arguments.pop("position_ids", False) is not False: + # `position_ids` could have been `torch.Tensor` or `None` so defaulting pop to `False` allows to detect if users were passing explicitly `None` + warnings.warn( + "`position_ids` have no functionality in BLOOM and will be removed in v5.0.0. You can safely ignore" + " passing `position_ids`.", + FutureWarning, + ) + if len(deprecated_arguments) > 0: + raise ValueError(f"Got unexpected arguments: {deprecated_arguments}") + + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + transformer_outputs = self.transformer( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + + lm_logits = self.lm_head(hidden_states) + loss = None + + if not return_dict: + output = (lm_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=loss, + logits=lm_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index f4116937..8a35ffa8 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -30,21 +30,23 @@ import dropout_layer_norm from text_generation_server.utils.layers import ( - FastLinear, TensorParallelRowLinear, TensorParallelColumnLinear, TensorParallelEmbedding, PositionRotaryEmbedding, + TensorParallelHead, ) class LlamaRMSNorm(nn.Module): - def __init__(self, hidden_size, eps=1e-6): + def __init__(self, prefix, weights, eps=1e-6): """ LlamaRMSNorm is equivalent to T5LayerNorm """ super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) + + weight = weights.get_tensor(f"{prefix}.weight") + self.weight = nn.Parameter(weight) self.variance_epsilon = eps def forward(self, hidden_states, residual=None): @@ -91,35 +93,35 @@ def forward(self, hidden_states, residual=None): class FlashLlamaAttention(torch.nn.Module): def __init__( self, - num_heads, - hidden_size, - process_group=None, + prefix: str, + config, + weights, ): super().__init__() - self.num_heads = num_heads - self.hidden_size = hidden_size - self.head_size = hidden_size // num_heads + self.num_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.num_heads + + self.rotary_emb = PositionRotaryEmbedding.load( + prefix=f"{prefix}.rotary_emb", weights=weights + ) - self.rotary_emb = PositionRotaryEmbedding(self.head_size, base=10000) self.softmax_scale = self.head_size ** (-0.5) - if process_group is None: - self.query_key_value = FastLinear(hidden_size, 3 * hidden_size, bias=False) - self.o_proj = FastLinear(hidden_size, hidden_size, bias=False) - else: - self.num_heads = self.num_heads // process_group.size() - self.query_key_value = TensorParallelColumnLinear( - hidden_size, - 3 * hidden_size, - bias=False, - process_group=process_group, - ) - self.o_proj = TensorParallelRowLinear( - hidden_size, - hidden_size, - bias=False, - process_group=process_group, - ) + self.num_heads = self.num_heads // weights.process_group.size() + self.query_key_value = TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + dim=0, + weights=weights, + bias=False, + ) + self.o_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.o_proj", + weights=weights, + bias=False, + ) def forward( self, @@ -195,8 +197,9 @@ def forward( class LlamaMLP(nn.Module): - def __init__(self, act, hidden_size, intermediate_size, process_group=None): + def __init__(self, prefix, config, weights): super().__init__() + act = config.hidden_act self.act = ( ACT2FN[act] if "gelu" not in act @@ -207,32 +210,23 @@ def __init__(self, act, hidden_size, intermediate_size, process_group=None): else "none", ) ) - - if process_group is None: - # Fuse gate and up proj - self.gate_up_proj = FastLinear( - hidden_size, 2 * intermediate_size, bias=False - ) - self.down_proj = FastLinear(intermediate_size, hidden_size, bias=False) - self.intermediate_size = intermediate_size - else: - # Fuse gate and up proj - self.gate_up_proj = TensorParallelColumnLinear( - hidden_size, - 2 * intermediate_size, - bias=False, - process_group=process_group, - ) - self.down_proj = TensorParallelRowLinear( - intermediate_size, - hidden_size, - bias=False, - process_group=process_group, - reduce=True, - ) - self.intermediate_size = self.down_proj.in_features - - self.process_group = process_group + # Fuse gate and up proj + self.gate_up_proj = TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.gate_proj", f"{prefix}.up_proj"], + weights=weights, + dim=0, + bias=False, + ) + self.down_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.down_proj", + weights=weights, + bias=False, + ) + self.intermediate_size = ( + config.intermediate_size // weights.process_group.size() + ) def forward(self, hidden_states): gate_up_states = self.gate_up_proj(hidden_states) @@ -241,22 +235,22 @@ def forward(self, hidden_states): class FlashLlamaLayer(nn.Module): - def __init__( - self, - num_heads, - act, - hidden_size, - intermediate_size, - rms_norm_eps, - process_group=None, - ): + def __init__(self, layer_id, config, weights): super().__init__() + prefix = f"model.layers.{layer_id}" + self.self_attn = FlashLlamaAttention( + prefix=f"{prefix}.self_attn", config=config, weights=weights + ) + self.mlp = LlamaMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) - self.self_attn = FlashLlamaAttention(num_heads, hidden_size, process_group) - self.mlp = LlamaMLP(act, hidden_size, intermediate_size, process_group) - - self.input_layernorm = LlamaRMSNorm(hidden_size, eps=rms_norm_eps) - self.post_attention_layernorm = LlamaRMSNorm(hidden_size, eps=rms_norm_eps) + self.input_layernorm = LlamaRMSNorm( + prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps + ) + self.post_attention_layernorm = LlamaRMSNorm( + prefix=f"{prefix}.post_attention_layernorm", + weights=weights, + eps=config.rms_norm_eps, + ) def forward( self, @@ -295,54 +289,35 @@ def forward( class FlashLlamaModel(torch.nn.Module): - def __init__(self, config, process_group=None): - super(FlashLlamaModel, self).__init__() + def __init__(self, config, weights): + super().__init__() self.config = config - self.tp_embeddings = False - if process_group is not None: - self.tp_rank = process_group.rank() - self.tp_world_size = process_group.size() - if config.vocab_size % self.tp_world_size == 0: - self.tp_embeddings = True - - if self.tp_embeddings: - self.embed_tokens = TensorParallelEmbedding( - config.vocab_size, config.hidden_size, process_group=process_group - ) - else: - self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) - + process_group = weights.process_group + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + self.embed_tokens = TensorParallelEmbedding( + prefix="model.embed_tokens", weights=weights + ) self.layers = nn.ModuleList( [ FlashLlamaLayer( - config.num_attention_heads, - config.hidden_act, - config.hidden_size, - config.intermediate_size, - config.rms_norm_eps, - process_group, + layer_id, + config, + weights, ) - for _ in range(config.num_hidden_layers) + for layer_id in range(config.num_hidden_layers) ] ) - self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.norm = LlamaRMSNorm( + prefix="model.norm", weights=weights, eps=config.rms_norm_eps + ) self.gradient_checkpointing = False self.head_size = self.layers[0].self_attn.head_size self.num_heads = self.layers[0].self_attn.num_heads - def post_load_weights(self, quantize: Optional[str] = None): - if isinstance(self.embed_tokens, TensorParallelEmbedding): - self.embed_tokens.add_null_idx() - for layer in self.layers: - layer: FlashLlamaLayer - layer.self_attn.query_key_value.prepare_weights(quantize) - layer.self_attn.o_proj.prepare_weights(quantize) - layer.mlp.gate_up_proj.prepare_weights(quantize) - layer.mlp.down_proj.prepare_weights(quantize) - def forward( self, input_ids, @@ -410,29 +385,15 @@ def forward( class FlashLlamaForCausalLM(torch.nn.Module): - def __init__(self, config, process_group=None): + def __init__(self, config, weights): super().__init__() - self.process_group = process_group - if self.process_group is not None: - self.world_size = self.process_group.size() - else: - self.world_size = 1 - - self.model = FlashLlamaModel(config, process_group) - - if self.model.tp_embeddings: - self.lm_head = FastLinear( - config.hidden_size, - config.vocab_size // process_group.size(), - bias=False, - ) - else: - self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) - - def post_load_weights(self, quantize: Optional[str] = None): - self.model.post_load_weights(quantize) - self.lm_head.prepare_weights() + self.model = FlashLlamaModel(config, weights) + self.lm_head = TensorParallelHead.load( + config, + prefix="lm_head", + weights=weights, + ) def forward( self, @@ -457,12 +418,4 @@ def forward( if lm_head_indices is not None: hidden_states = hidden_states[lm_head_indices] logits = self.lm_head(hidden_states) - - if self.model.tp_embeddings: - # Logits are sharded, so we need to gather them - world_logits = [torch.empty_like(logits) for _ in range(self.world_size)] - torch.distributed.all_gather(world_logits, logits, group=self.process_group) - world_logits = torch.cat(world_logits, dim=1) - - return world_logits, present return logits, present diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index b798750a..0fe43bcb 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -31,61 +31,81 @@ import flash_attn_cuda from text_generation_server.utils.layers import ( - FastLinear, TensorParallelRowLinear, TensorParallelColumnLinear, TensorParallelEmbedding, + TensorParallelHead, FastLayerNorm, PositionRotaryEmbedding, + get_linear, ) +def load_row(config, prefix: str, weights, bias: bool): + weight = weights.get_sharded(f"{prefix}.weight", dim=1) + if bias and weights.process_group.rank() == 0: + # Rank is only on the first rank process + bias = weights.get_tensor(f"{prefix}.bias") + else: + bias = None + + linear = get_linear(weight, bias, config.quantize) + if config.use_parallel_residual: + return linear + else: + return TensorParallelRowLinear(linear, process_group=weights.process_group) + + +def load_qkv(config, prefix: str, weights, num_heads, head_size, hidden_size): + weight = weights.get_sharded(f"{prefix}.weight", dim=0) + bias = weights.get_sharded(f"{prefix}.bias", dim=0) + + weight = ( + weight.view( + num_heads, + 3, + head_size, + hidden_size, + ) + .permute(1, 0, 2, 3) + .reshape(-1, hidden_size) + ) + bias = bias.view(num_heads, 3, head_size).permute(1, 0, 2).reshape(-1) + + linear = get_linear(weight, bias, config.quantize) + if config.use_parallel_residual: + return linear + else: + return TensorParallelColumnLinear(linear) + + class FlashNeoxAttention(torch.nn.Module): - def __init__( - self, - num_heads, - hidden_size, - rotary_pct, - rotary_emb_base, - process_group=None, - reduce=True, - ): + def __init__(self, config, prefix, weights): super().__init__() + num_heads = config.num_attention_heads + hidden_size = config.hidden_size + self.num_heads = num_heads self.hidden_size = hidden_size self.head_size = hidden_size // num_heads + self.num_heads = self.num_heads // weights.process_group.size() - rotary_ndims = int(self.head_size * rotary_pct) - self.rotary_emb = PositionRotaryEmbedding(rotary_ndims, base=rotary_emb_base) - self.softmax_scale = self.head_size ** (-0.5) + self.rotary_emb = PositionRotaryEmbedding.load( + prefix=f"{prefix}.rotary_emb", weights=weights + ) - if process_group is None: - self.query_key_value = FastLinear(hidden_size, 3 * hidden_size) - self.dense = FastLinear(hidden_size, hidden_size) - else: - self.num_heads = self.num_heads // process_group.size() - self.query_key_value = TensorParallelColumnLinear( - hidden_size, - 3 * hidden_size, - process_group=process_group, - ) - self.dense = TensorParallelRowLinear( - hidden_size, hidden_size, process_group=process_group, reduce=reduce - ) + self.softmax_scale = self.head_size ** (-0.5) - def shuffle_qkv_dims(self): - """Swap dims to avoid an additional permute""" - self.query_key_value.weight = torch.nn.Parameter( - self.query_key_value.weight.view( - self.num_heads, 3, self.head_size, self.hidden_size - ) - .permute(1, 0, 2, 3) - .reshape(-1, self.hidden_size) + self.query_key_value = load_qkv( + config, + prefix=f"{prefix}.query_key_value", + weights=weights, + num_heads=self.num_heads, + head_size=self.head_size, + hidden_size=self.hidden_size, ) - self.query_key_value.bias = torch.nn.Parameter( - self.query_key_value.bias.view(self.num_heads, 3, self.head_size) - .permute(1, 0, 2) - .reshape(-1) + self.dense = load_row( + config, prefix=f"{prefix}.dense", weights=weights, bias=True ) def forward( @@ -162,10 +182,9 @@ def forward( class FlashMLP(nn.Module): - def __init__( - self, act, hidden_size, intermediate_size, process_group=None, reduce=True - ): + def __init__(self, config, prefix, weights): super().__init__() + act = config.hidden_act self.act = ( ACT2FN[act] if "gelu" not in act @@ -177,22 +196,12 @@ def __init__( ) ) - if process_group is None: - self.dense_h_to_4h = FastLinear(hidden_size, intermediate_size) - self.dense_4h_to_h = FastLinear(intermediate_size, hidden_size) - else: - self.dense_h_to_4h = TensorParallelColumnLinear( - hidden_size, - intermediate_size, - process_group=process_group, - ) - self.dense_4h_to_h = TensorParallelRowLinear( - intermediate_size, - hidden_size, - process_group=process_group, - reduce=reduce, - ) - self.process_group = process_group + self.dense_h_to_4h = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.dense_h_to_4h", weights=weights, bias=True + ) + self.dense_4h_to_h = load_row( + config, prefix=f"{prefix}.dense_4h_to_h", weights=weights, bias=True + ) def forward(self, hidden_states): hidden_states = self.dense_h_to_4h(hidden_states) @@ -202,38 +211,28 @@ def forward(self, hidden_states): class FlashNeoXLayer(nn.Module): - def __init__( - self, - num_heads, - act, - hidden_size, - intermediate_size, - rotary_pct, - rotary_emb_base, - layer_norm_eps, - use_parallel_residual, - process_group=None, - ): + def __init__(self, layer_id, config, weights): super().__init__() - self.use_parallel_residual = use_parallel_residual - self.input_layernorm = FastLayerNorm(hidden_size, eps=layer_norm_eps) - self.post_attention_layernorm = FastLayerNorm(hidden_size, eps=layer_norm_eps) - self.attention = FlashNeoxAttention( - num_heads, - hidden_size, - rotary_pct, - rotary_emb_base, - process_group, - reduce=not use_parallel_residual, + + layer_norm_eps = config.layer_norm_eps + + prefix = f"gpt_neox.layers.{layer_id}" + + self.use_parallel_residual = config.use_parallel_residual + self.input_layernorm = FastLayerNorm.load( + prefix=f"{prefix}.input_layernorm", weights=weights, eps=layer_norm_eps ) - self.mlp = FlashMLP( - act, - hidden_size, - intermediate_size, - process_group, - reduce=not use_parallel_residual, + self.post_attention_layernorm = FastLayerNorm.load( + prefix=f"{prefix}.post_attention_layernorm", + weights=weights, + eps=layer_norm_eps, + ) + self.attention = FlashNeoxAttention( + config, prefix=f"{prefix}.attention", weights=weights ) - self.process_group = process_group + + self.mlp = FlashMLP(config, prefix=f"{prefix}.mlp", weights=weights) + self.process_group = weights.process_group def forward( self, @@ -266,9 +265,7 @@ def forward( mlp_output = self.mlp(ln2_hidden_states) intermediate = mlp_output + attn_output - # Only reduce once and after the addition instead of once per layer - if self.process_group is not None: - torch.distributed.all_reduce(intermediate, group=self.process_group) + torch.distributed.all_reduce(intermediate, group=self.process_group) return intermediate + hidden_states, None else: @@ -302,42 +299,24 @@ class FlashGPTNeoXPreTrainedModel(PreTrainedModel): class FlashGPTNeoXModel(FlashGPTNeoXPreTrainedModel): - def __init__(self, config, process_group=None): + def __init__(self, config, weights): super().__init__(config) self.config = config - self.tp_embeddings = False - if process_group is not None: - self.tp_rank = process_group.rank() - self.tp_world_size = process_group.size() - if config.vocab_size % self.tp_world_size == 0: - self.tp_embeddings = True - - if self.tp_embeddings: - self.embed_in = TensorParallelEmbedding( - config.vocab_size, config.hidden_size, process_group=process_group - ) - else: - self.embed_in = nn.Embedding(config.vocab_size, config.hidden_size) + self.embed_in = TensorParallelEmbedding( + prefix="gpt_neox.embed_in", weights=weights + ) self.layers = nn.ModuleList( [ - FlashNeoXLayer( - config.num_attention_heads, - config.hidden_act, - config.hidden_size, - config.intermediate_size, - config.rotary_pct, - config.rotary_emb_base, - config.layer_norm_eps, - config.use_parallel_residual, - process_group, - ) - for _ in range(config.num_hidden_layers) + FlashNeoXLayer(layer_id, config, weights) + for layer_id in range(config.num_hidden_layers) ] ) - self.final_layer_norm = FastLayerNorm( - config.hidden_size, eps=config.layer_norm_eps + self.final_layer_norm = FastLayerNorm.load( + prefix="gpt_neox.final_layer_norm", + weights=weights, + eps=config.layer_norm_eps, ) self.gradient_checkpointing = False @@ -345,29 +324,6 @@ def __init__(self, config, process_group=None): self.head_size = self.layers[0].attention.head_size self.num_heads = self.layers[0].attention.num_heads - def post_load_weights(self, quantize: Optional[str] = None): - if isinstance(self.embed_in, TensorParallelEmbedding): - self.embed_in.add_null_idx() - for layer in self.layers: - layer: FlashNeoXLayer - layer.attention.shuffle_qkv_dims() - layer.attention.query_key_value.prepare_weights(quantize) - layer.attention.dense.prepare_weights(quantize) - layer.mlp.dense_h_to_4h.prepare_weights(quantize) - layer.mlp.dense_4h_to_h.prepare_weights(quantize) - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): - # Pop here as we will replace the layer in our own logic and don't want from_pretrained - # to do it for us - load_in_8bit = kwargs.pop("load_in_8bit", False) - model = super(FlashGPTNeoXModel, cls).from_pretrained( - pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs - ) - - model.post_load_weights("bitsandbytes" if load_in_8bit else None) - return model - def forward( self, input_ids, @@ -435,42 +391,13 @@ def forward( class FlashGPTNeoXForCausalLM(FlashGPTNeoXPreTrainedModel): - def __init__(self, config, process_group=None): + def __init__(self, config, weights): super().__init__(config) + self.gpt_neox = FlashGPTNeoXModel(config, weights) - self.process_group = process_group - if self.process_group is not None: - self.world_size = self.process_group.size() - else: - self.world_size = 1 - - self.gpt_neox = FlashGPTNeoXModel(config, process_group) - - if self.gpt_neox.tp_embeddings: - self.embed_out = FastLinear( - config.hidden_size, - config.vocab_size // process_group.size(), - bias=False, - ) - else: - self.embed_out = FastLinear( - config.hidden_size, config.vocab_size, bias=False - ) - - def post_load_weights(self, quantize: Optional[str] = None): - self.gpt_neox.post_load_weights(quantize) - self.embed_out.prepare_weights() - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): - # Pop here as we will replace the layer in our own logic and don't want from_pretrained - # to do it for us - load_in_8bit = kwargs.pop("load_in_8bit", False) - model = super(FlashGPTNeoXForCausalLM, cls).from_pretrained( - pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs + self.embed_out = TensorParallelHead.load( + config, prefix="embed_out", weights=weights ) - model.post_load_weights("bitsandbytes" if load_in_8bit else None) - return model def forward( self, @@ -495,12 +422,4 @@ def forward( if lm_head_indices is not None: hidden_states = hidden_states[lm_head_indices] logits = self.embed_out(hidden_states) - - if self.gpt_neox.tp_embeddings: - # Logits are sharded, so we need to gather them - world_logits = [torch.empty_like(logits) for _ in range(self.world_size)] - torch.distributed.all_gather(world_logits, logits, group=self.process_group) - world_logits = torch.cat(world_logits, dim=1) - - return world_logits, present return logits, present diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 03487703..55195162 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -1,5 +1,3 @@ -import os - import torch import torch.distributed @@ -12,15 +10,31 @@ import flash_attn_cuda from text_generation_server.utils.layers import ( - FastLinear, TensorParallelRowLinear, TensorParallelColumnLinear, TensorParallelEmbedding, + TensorParallelHead, FastLayerNorm, PositionRotaryEmbedding, + get_linear, ) +def load_row(config, prefix: str, weights, bias: bool): + weight = weights.get_sharded(f"{prefix}.weight", dim=1) + if bias and weights.process_group.rank() == 0: + # Rank is only on the first rank process + bias = weights.get_tensor(f"{prefix}.bias") + else: + bias = None + + linear = get_linear(weight, bias, config.quantize) + if config.parallel_attn: + return linear + else: + return TensorParallelRowLinear(linear, process_group=weights.process_group) + + class RWConfig(PretrainedConfig): attribute_map = { "num_hidden_layers": "n_layer", @@ -85,44 +99,31 @@ def __init__( class FlashRWAttention(torch.nn.Module): def __init__( self, - num_heads, - num_heads_kv, - hidden_size, - bias, - process_group=None, - reduce=True, + config, + prefix, + weights, ): super().__init__() - self.num_heads = num_heads - self.num_heads_kv = num_heads_kv - self.hidden_size = hidden_size - self.head_size = hidden_size // num_heads + self.num_heads = config.n_head + self.num_heads_kv = config.n_head_kv + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.num_heads - self.rotary_emb = PositionRotaryEmbedding(self.head_size, base=10000) + self.rotary_emb = PositionRotaryEmbedding.static( + dim=self.head_size, base=10000.0, device=weights.device + ) self.softmax_scale = self.head_size ** (-0.5) + self.num_heads = self.num_heads // weights.process_group.size() - if process_group is None: - self.query_key_value = FastLinear( - hidden_size, - self.head_size * (self.num_heads + 2 * self.num_heads_kv), - bias=bias, - ) - self.dense = FastLinear(hidden_size, hidden_size, bias=bias) - else: - self.query_key_value = TensorParallelColumnLinear( - hidden_size, - self.head_size * (self.num_heads + 2 * self.num_heads_kv), - bias=bias, - process_group=process_group, - ) - self.dense = TensorParallelRowLinear( - hidden_size, - hidden_size, - bias=bias, - process_group=process_group, - reduce=reduce, - ) - self.num_heads = self.num_heads // process_group.size() + self.query_key_value = TensorParallelColumnLinear.load( + config, + prefix=f"{prefix}.query_key_value", + weights=weights, + bias=config.bias, + ) + self.dense = load_row( + config, prefix=f"{prefix}.dense", weights=weights, bias=config.bias + ) def forward( self, @@ -212,57 +213,48 @@ def forward( class FlashRWLargeAttention(torch.nn.Module): def __init__( self, - num_heads, - num_heads_kv, - hidden_size, - bias, - process_group=None, - reduce=True, + config, + prefix, + weights, ): super().__init__() + hidden_size = config.hidden_size + num_heads = config.n_head + num_heads_kv = config.n_head_kv + self.hidden_size = hidden_size self.head_size = hidden_size // num_heads - self.rotary_emb = PositionRotaryEmbedding(self.head_size, base=10000) + self.rotary_emb = PositionRotaryEmbedding.static( + self.head_size, base=10000.0, device=weights.device + ) self.softmax_scale = self.head_size ** (-0.5) self.num_groups = num_heads // (num_heads_kv * 2) self.num_heads = num_heads // self.num_groups self.num_heads_kv = num_heads_kv // self.num_groups + process_group = weights.process_group - if process_group is None: - self.query_key_value = FastLinear( - hidden_size, - self.num_groups - * self.head_size - * (self.num_heads + 2 * self.num_heads_kv), - bias=bias, - ) - self.dense = FastLinear(hidden_size, hidden_size, bias=bias) - else: - if process_group.size() > self.num_groups: - raise NotImplementedError( - f"Tensor Parallelism is not implemented for world_size > n groups" - ) - - self.query_key_value = TensorParallelColumnLinear( - hidden_size, - self.num_groups - * self.head_size - * (self.num_heads + 2 * self.num_heads_kv), - bias=bias, - process_group=process_group, + if process_group.size() > self.num_groups: + raise NotImplementedError( + f"Tensor Parallelism is not implemented for world_size > n groups" ) - self.dense = TensorParallelRowLinear( - hidden_size, - hidden_size, - bias=bias, - process_group=process_group, - reduce=reduce, + if self.num_groups % process_group.size() != 0: + raise NotImplementedError( + f"Tensor Parallelism is not implemented for {self.num_groups} not divisible by {process_group.size()}" ) + self.num_groups = self.num_groups // process_group.size() - self.num_groups = self.num_groups // process_group.size() + self.query_key_value = TensorParallelColumnLinear.load( + config, + prefix=f"{prefix}.query_key_value", + weights=weights, + bias=config.bias, + ) + self.dense = load_row( + config, prefix=f"{prefix}.dense", weights=weights, bias=config.bias + ) def forward( self, @@ -359,28 +351,16 @@ def forward( class FlashMLP(nn.Module): - def __init__(self, hidden_size, bias, process_group=None, reduce=True): + def __init__(self, config, prefix, weights): super().__init__() self.act = torch.nn.functional.gelu - if process_group is None: - self.dense_h_to_4h = FastLinear(hidden_size, 4 * hidden_size, bias=bias) - self.dense_4h_to_h = FastLinear(4 * hidden_size, hidden_size, bias=bias) - else: - self.dense_h_to_4h = TensorParallelColumnLinear( - hidden_size, - 4 * hidden_size, - bias=bias, - process_group=process_group, - ) - self.dense_4h_to_h = TensorParallelRowLinear( - 4 * hidden_size, - hidden_size, - bias=bias, - process_group=process_group, - reduce=reduce, - ) - self.process_group = process_group + self.dense_h_to_4h = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.dense_h_to_4h", weights=weights, bias=config.bias + ) + self.dense_4h_to_h = load_row( + config, prefix=f"{prefix}.dense_4h_to_h", weights=weights, bias=config.bias + ) def forward(self, hidden_states): hidden_states = self.dense_h_to_4h(hidden_states) @@ -392,38 +372,44 @@ def forward(self, hidden_states): class FlashRWLayer(nn.Module): def __init__( self, - num_heads, - num_heads_kv, - hidden_size, - bias, - layer_norm_eps, - parallel_attn, - process_group=None, + layer_id, + config, + weights, ): super().__init__() + parallel_attn = config.parallel_attn self.parallel_attn = parallel_attn - self.input_layernorm = FastLayerNorm(hidden_size, eps=layer_norm_eps) + prefix = f"transformer.h.{layer_id}" + + self.input_layernorm = FastLayerNorm.load( + prefix=f"{prefix}.input_layernorm", + weights=weights, + eps=config.layer_norm_epsilon, + ) self.self_attention = FlashRWAttention( - num_heads, - num_heads_kv, - hidden_size, - bias, - process_group=process_group, - reduce=False, + config, + prefix=f"{prefix}.self_attention", + weights=weights, ) self.post_attention_layernorm = ( - FastLayerNorm(hidden_size, eps=layer_norm_eps) + FastLayerNorm.load( + prefix=f"{prefix}.post_attention_layernorm", + weights=weights, + eps=config.layer_norm_epsilon, + ) if not parallel_attn else None ) self.mlp = FlashMLP( - hidden_size, bias, process_group=process_group, reduce=False + config, + prefix=f"{prefix}.mlp", + weights=weights, ) - self.process_group = process_group + self.process_group = weights.process_group def forward( self, @@ -454,9 +440,7 @@ def forward( mlp_output = self.mlp(ln_hidden_states) intermediate = mlp_output + attn_output - # Only reduce once and after the addition instead of once per layer - if self.process_group is not None: - torch.distributed.all_reduce(intermediate, group=self.process_group) + torch.distributed.all_reduce(intermediate, group=self.process_group) return intermediate, residual else: @@ -483,33 +467,30 @@ def forward( class FlashRWLargeLayer(nn.Module): - def __init__( - self, - num_heads, - num_heads_kv, - hidden_size, - bias, - layer_norm_eps, - process_group=None, - ): + def __init__(self, layer_id, config, weights): super().__init__() - self.ln_attn = FastLayerNorm(hidden_size, eps=layer_norm_eps) - self.ln_mlp = FastLayerNorm(hidden_size, eps=layer_norm_eps) + prefix = f"transformer.h.{layer_id}" + self.ln_attn = FastLayerNorm.load( + prefix=f"{prefix}.ln_attn", + weights=weights, + eps=config.layer_norm_epsilon, + ) + self.ln_mlp = FastLayerNorm.load( + prefix=f"{prefix}.ln_mlp", + weights=weights, + eps=config.layer_norm_epsilon, + ) self.self_attention = FlashRWLargeAttention( - num_heads, - num_heads_kv, - hidden_size, - bias, - process_group=process_group, - reduce=False, + config, + prefix=f"{prefix}.self_attention", + weights=weights, ) + assert config.parallel_attn, "This version doesn't support non parallel_attn" - self.mlp = FlashMLP( - hidden_size, bias, process_group=process_group, reduce=False - ) + self.mlp = FlashMLP(config, prefix=f"{prefix}.mlp", weights=weights) - self.process_group = process_group + self.process_group = weights.process_group def forward( self, @@ -543,9 +524,7 @@ def forward( intermediate = attn_output + mlp_output - # Only reduce once and after the addition instead of once per layer - if self.process_group is not None: - torch.distributed.all_reduce(intermediate, group=self.process_group) + torch.distributed.all_reduce(intermediate, group=self.process_group) return intermediate, residual @@ -555,37 +534,18 @@ class FlashRWPreTrainedModel(PreTrainedModel): class FlashRWModel(FlashRWPreTrainedModel): - def __init__(self, config, process_group=None): + def __init__(self, config, weights): super().__init__(config) self.config = config - self.tp_embeddings = False - if process_group is not None: - self.tp_rank = process_group.rank() - self.tp_world_size = process_group.size() - if config.vocab_size % self.tp_world_size == 0: - self.tp_embeddings = True - - if self.tp_embeddings: - self.word_embeddings = TensorParallelEmbedding( - config.vocab_size, config.hidden_size, process_group=process_group - ) - else: - self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size) - + self.word_embeddings = TensorParallelEmbedding( + prefix="transformer.word_embeddings", weights=weights + ) if config.model_type == "RefinedWebModel": self.h = nn.ModuleList( [ - FlashRWLayer( - config.n_head, - config.n_head_kv, - config.hidden_size, - config.bias, - config.layer_norm_epsilon, - config.parallel_attn, - process_group, - ) - for _ in range(config.num_hidden_layers) + FlashRWLayer(layer_id, config, weights) + for layer_id in range(config.num_hidden_layers) ] ) self.cache_size = ( @@ -596,15 +556,8 @@ def __init__(self, config, process_group=None): elif config.model_type == "RefinedWeb": self.h = nn.ModuleList( [ - FlashRWLargeLayer( - config.n_head, - config.n_head_kv, - config.hidden_size, - config.bias, - config.layer_norm_epsilon, - process_group, - ) - for _ in range(config.num_hidden_layers) + FlashRWLargeLayer(layer_id, config, weights) + for layer_id in range(config.num_hidden_layers) ] ) self.cache_size = ( @@ -617,31 +570,13 @@ def __init__(self, config, process_group=None): f"model_type {config.model_type} is not supported." ) - self.ln_f = FastLayerNorm(config.hidden_size, eps=config.layer_norm_epsilon) - - self.head_size = self.h[0].self_attention.head_size - - def post_load_weights(self, quantize: Optional[str] = None): - if isinstance(self.word_embeddings, TensorParallelEmbedding): - self.word_embeddings.add_null_idx() - for layer in self.h: - layer: FlashRWLayer - layer.self_attention.query_key_value.prepare_weights(quantize) - layer.self_attention.dense.prepare_weights(quantize) - layer.mlp.dense_h_to_4h.prepare_weights(quantize) - layer.mlp.dense_4h_to_h.prepare_weights(quantize) - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): - # Pop here as we will replace the layer in our own logic and don't want from_pretrained - # to do it for us - load_in_8bit = kwargs.pop("load_in_8bit", False) - model = super(FlashRWModel, cls).from_pretrained( - pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs + self.ln_f = FastLayerNorm.load( + prefix="transformer.ln_f", + weights=weights, + eps=config.layer_norm_epsilon, ) - model.post_load_weights("bitsandbytes" if load_in_8bit else None) - return model + self.head_size = self.h[0].self_attention.head_size def forward( self, @@ -708,40 +643,14 @@ def forward( class FlashRWForCausalLM(FlashRWPreTrainedModel): - def __init__(self, config, process_group=None): + def __init__(self, config, weights): super().__init__(config) - self.process_group = process_group - if self.process_group is not None: - self.world_size = self.process_group.size() - else: - self.world_size = 1 - - self.transformer = FlashRWModel(config, process_group) + self.transformer = FlashRWModel(config, weights) - if self.transformer.tp_embeddings: - self.lm_head = FastLinear( - config.hidden_size, - config.vocab_size // process_group.size(), - bias=False, - ) - else: - self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) - - def post_load_weights(self, quantize: Optional[str] = None): - self.transformer.post_load_weights(quantize) - self.lm_head.prepare_weights() - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): - # Pop here as we will replace the layer in our own logic and don't want from_pretrained - # to do it for us - load_in_8bit = kwargs.pop("load_in_8bit", False) - model = super(FlashRWForCausalLM, cls).from_pretrained( - pretrained_model_name_or_path, load_in_8bit=False, *model_args, **kwargs + self.lm_head = TensorParallelHead.load( + config, prefix="lm_head", weights=weights ) - model.post_load_weights("bitsandbytes" if load_in_8bit else None) - return model def forward( self, @@ -766,12 +675,4 @@ def forward( if lm_head_indices is not None: hidden_states = hidden_states[lm_head_indices] logits = self.lm_head(hidden_states) - - if self.transformer.tp_embeddings: - # Logits are sharded, so we need to gather them - world_logits = [torch.empty_like(logits) for _ in range(self.world_size)] - torch.distributed.all_gather(world_logits, logits, group=self.process_group) - world_logits = torch.cat(world_logits, dim=1) - - return world_logits, present return logits, present diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index b61ec873..888a6066 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -8,39 +8,142 @@ # Flash attention imports import flash_attn_cuda from text_generation_server.utils.layers import ( - FastLinear, TensorParallelRowLinear, TensorParallelColumnLinear, + TensorParallelHead, TensorParallelEmbedding, FastLayerNorm, + get_linear, ) -class FlashMQAttention(torch.nn.Module): - def __init__( - self, - num_heads, +def load_multi_mqa( + config, prefix: str, weights, bias: bool, head_size, num_heads, hidden_size +): + if any("c_attn" in k for k in weights.routing.keys()): + slice_ = weights._get_slice(f"{prefix}.c_attn.weight") + shape = slice_.get_shape() + world_size = weights.process_group.size() + rank = weights.process_group.rank() + if config.transpose: + block_size = (shape[1] - 2 * head_size) // world_size + start = rank * block_size + stop = (rank + 1) * block_size + assert (shape[1] - 2 * head_size) % world_size == 0 + q_tensor = slice_[:, start:stop] + kv_tensor = slice_[:, -2 * head_size :] + weight = torch.cat([q_tensor, kv_tensor], dim=1).T + else: + block_size = (shape[0] - 2 * head_size) // world_size + start = rank * block_size + stop = (rank + 1) * block_size + assert (shape[0] - 2 * head_size) % world_size == 0 + q_tensor = slice_[start:stop] + kv_tensor = slice_[-2 * head_size :] + weight = torch.cat([q_tensor, kv_tensor], dim=0) + if bias: + slice_ = weights._get_slice(f"{prefix}.c_attn.bias") + shape = slice_.get_shape() + block_size = (shape[0] - 2 * head_size) // world_size + assert (shape[0] - 2 * head_size) % world_size == 0 + q_tensor = slice_[start:stop] + start = rank * block_size + stop = (rank + 1) * block_size + q_tensor = slice_[start:stop] + kv_tensor = slice_[-2 * head_size :] + bias = torch.cat([q_tensor, kv_tensor], dim=0) + else: + if config.transpose: + w = [ + weights.get_sharded(f"{prefix}.q_attn.weight", dim=1).T, + weights.get_tensor(f"{prefix}.kv_attn.weight").T, + ] + weight = torch.cat(w, dim=0) + else: + w = [ + weights.get_sharded(f"{prefix}.q_attn.weight", dim=0), + weights.get_tensor(f"{prefix}.kv_attn.weight"), + ] + weight = torch.cat(w, dim=1) + + if bias: + b = [ + weights.get_sharded(f"{prefix}.q_attn.bias", dim=0), + weights.get_tensor(f"{prefix}.kv_attn.bias"), + ] + bias = torch.cat(b, dim=0) + else: + bias = None + + weight = weight.to(dtype=weights.dtype).to(device=weights.device) + assert list(weight.shape) == [ + (num_heads + 2) * head_size, hidden_size, - process_group=None, - ): + ], f"{weight.shape} != {[(num_heads + 2) * head_size, hidden_size]}" + if bias is not None: + bias = bias.to(dtype=weights.dtype).to(device=weights.device) + assert list(bias.shape) == [ + (num_heads + 2) * head_size + ], f"{weight.shape} != {[(num_heads + 2) * head_size]}" + return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize)) + + +def load_col(config, prefix: str, weights, bias: bool): + if config.transpose: + weight = weights.get_sharded(f"{prefix}.weight", dim=1).T + else: + weight = weights.get_sharded(f"{prefix}.weight", dim=0) + + if bias: + bias = weights.get_sharded(f"{prefix}.bias", dim=0) + else: + bias = None + return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize)) + + +def load_row(config, prefix: str, weights, bias: bool): + if config.transpose: + weight = weights.get_sharded(f"{prefix}.weight", dim=0).T + else: + weight = weights.get_sharded(f"{prefix}.weight", dim=1) + + if bias and weights.process_group.rank() == 0: + # Rank is only on the first rank process + bias = weights.get_tensor(f"{prefix}.bias") + else: + bias = None + return TensorParallelRowLinear( + get_linear(weight, bias, config.quantize), process_group=weights.process_group + ) + + +class FlashMQAttention(torch.nn.Module): + def __init__(self, prefix, config, weights): super().__init__() + num_heads = config.num_attention_heads + hidden_size = config.hidden_size + self.num_heads = num_heads self.hidden_size = hidden_size self.head_size = hidden_size // num_heads + assert self.num_heads % weights.process_group.size() == 0 + self.num_heads = self.num_heads // weights.process_group.size() + self.softmax_scale = self.head_size ** (-0.5) - if process_group is None: - self.c_attn = FastLinear(hidden_size, hidden_size + 2 * self.head_size) - self.c_proj = FastLinear(hidden_size, hidden_size) - else: - self.num_heads = self.num_heads // process_group.size() - self.c_attn = FastLinear(hidden_size, self.head_size * (self.num_heads + 2)) - self.c_proj = TensorParallelRowLinear( - hidden_size, - hidden_size, - process_group=process_group, - ) + self.c_attn = load_multi_mqa( + config, + prefix=prefix, + weights=weights, + bias=True, + head_size=self.head_size, + hidden_size=hidden_size, + num_heads=self.num_heads, + ) + self.c_proj = load_row( + config, prefix=f"{prefix}.c_proj", weights=weights, bias=True + ) def forward( self, @@ -121,8 +224,9 @@ def forward( class MLP(nn.Module): - def __init__(self, act, hidden_size, intermediate_size, process_group=None): + def __init__(self, prefix, config, weights): super().__init__() + act = config.activation_function self.act = ( ACT2FN[act] if "gelu" not in act @@ -134,20 +238,12 @@ def __init__(self, act, hidden_size, intermediate_size, process_group=None): ) ) - if process_group is None: - self.c_fc = FastLinear(hidden_size, intermediate_size) - self.c_proj = FastLinear(intermediate_size, hidden_size) - else: - self.c_fc = TensorParallelColumnLinear( - hidden_size, - intermediate_size, - process_group=process_group, - ) - self.c_proj = TensorParallelRowLinear( - intermediate_size, - hidden_size, - process_group=process_group, - ) + self.c_fc = load_col( + config, prefix=f"{prefix}.c_fc", weights=weights, bias=True + ) + self.c_proj = load_row( + config, prefix=f"{prefix}.c_proj", weights=weights, bias=True + ) def forward(self, hidden_states): hidden_states = self.c_fc(hidden_states) @@ -157,28 +253,24 @@ def forward(self, hidden_states): class Block(nn.Module): - def __init__( - self, - num_heads, - act, - hidden_size, - intermediate_size, - layer_norm_eps, - process_group=None, - ): + def __init__(self, layer_id, config, weights): super().__init__() - self.ln_1 = FastLayerNorm(hidden_size, eps=layer_norm_eps) - self.ln_2 = FastLayerNorm(hidden_size, eps=layer_norm_eps) + prefix = f"transformer.h.{layer_id}" + self.ln_1 = FastLayerNorm.load( + prefix=f"{prefix}.ln_1", weights=weights, eps=config.layer_norm_epsilon + ) + self.ln_2 = FastLayerNorm.load( + prefix=f"{prefix}.ln_2", weights=weights, eps=config.layer_norm_epsilon + ) self.attn = FlashMQAttention( - num_heads, - hidden_size, - process_group, + prefix=f"{prefix}.attn", + config=config, + weights=weights, ) self.mlp = MLP( - act, - hidden_size, - intermediate_size, - process_group, + prefix=f"{prefix}.mlp", + config=config, + weights=weights, ) def forward( @@ -210,66 +302,39 @@ def forward( class FlashSantacoderModel(nn.Module): - def __init__(self, config, process_group=None): + def __init__(self, config, weights): super().__init__() self.config = config - self.process_group = process_group - self.tp_embeddings = False - if process_group is not None: - self.tp_rank = process_group.rank() - self.tp_world_size = process_group.size() - if config.vocab_size % self.tp_world_size == 0: - self.tp_embeddings = True - - if self.tp_embeddings: - self.wte = TensorParallelEmbedding( - config.vocab_size, - config.hidden_size, - reduce=False, - process_group=process_group, - ) - self.wpe = TensorParallelEmbedding( - config.max_position_embeddings, - config.hidden_size, - reduce=False, - process_group=process_group, - ) - else: - self.wte = nn.Embedding(config.vocab_size, config.hidden_size) - self.wpe = nn.Embedding(config.max_position_embeddings, config.hidden_size) + self.process_group = weights.process_group + self.wte = TensorParallelEmbedding( + prefix="transformer.wte", + weights=weights, + reduce=False, + ) + self.wpe = TensorParallelEmbedding( + prefix="transformer.wpe", + weights=weights, + reduce=False, + ) self.h = nn.ModuleList( [ Block( - config.num_attention_heads, - config.activation_function, - config.hidden_size, - config.n_inner - if config.n_inner is not None - else 4 * config.hidden_size, - config.layer_norm_epsilon, - process_group, + layer_id, + config, + weights, ) - for _ in range(config.num_hidden_layers) + for layer_id in range(config.num_hidden_layers) ] ) - self.ln_f = FastLayerNorm(config.hidden_size, eps=config.layer_norm_epsilon) + self.ln_f = FastLayerNorm.load( + prefix="transformer.ln_f", weights=weights, eps=config.layer_norm_epsilon + ) self.head_size = self.h[0].attn.head_size self.num_heads = self.h[0].attn.num_heads - def post_load_weights(self, quantize: Optional[str] = None): - if self.tp_embeddings: - self.wte.add_null_idx() - self.wpe.add_null_idx() - for layer in self.h: - layer: Block - layer.attn.c_attn.prepare_weights(quantize) - layer.attn.c_proj.prepare_weights(quantize) - layer.mlp.c_fc.prepare_weights(quantize) - layer.mlp.c_proj.prepare_weights(quantize) - def forward( self, input_ids, @@ -281,8 +346,7 @@ def forward( pre_allocate_past_size: Optional[int] = None, ): hidden_states = self.wte(input_ids) + self.wpe(position_ids) - if self.tp_embeddings: - torch.distributed.all_reduce(hidden_states, group=self.process_group) + torch.distributed.all_reduce(hidden_states, group=self.process_group) # Prefill if past_key_values is None: @@ -331,23 +395,12 @@ def forward( class FlashSantacoderForCausalLM(nn.Module): - def __init__(self, config, process_group=None): + def __init__(self, config, weights): super().__init__() - - self.transformer = FlashSantacoderModel(config, process_group) - - if self.transformer.tp_embeddings: - self.lm_head = FastLinear( - config.hidden_size, - config.vocab_size // process_group.size(), - bias=False, - ) - else: - self.lm_head = FastLinear(config.hidden_size, config.vocab_size, bias=False) - - def post_load_weights(self, quantize: Optional[str] = None): - self.transformer.post_load_weights(quantize) - self.lm_head.prepare_weights() + self.transformer = FlashSantacoderModel(config, weights) + self.lm_head = TensorParallelHead.load( + config, prefix="transformer.wte", weights=weights + ) def forward( self, @@ -372,29 +425,4 @@ def forward( if lm_head_indices is not None: hidden_states = hidden_states[lm_head_indices] logits = self.lm_head(hidden_states) - - if self.transformer.tp_embeddings: - # Logits are sharded, so we need to gather them - if logits.shape[0] == 1: - # Fast path when batch size is 1 - world_logits = logits.new_empty( - (logits.shape[1] * self.transformer.tp_world_size) - ) - torch.distributed.all_gather_into_tensor( - world_logits, logits.view(-1), group=self.transformer.process_group - ) - world_logits = world_logits.view(1, -1) - else: - # We cannot use all_gather_into_tensor as it only support concatenating on the first dim - world_logits = [ - torch.empty_like(logits) - for _ in range(self.transformer.tp_world_size) - ] - torch.distributed.all_gather( - world_logits, logits, group=self.transformer.process_group - ) - world_logits = torch.cat(world_logits, dim=1) - - return world_logits, present - return logits, present diff --git a/server/text_generation_server/models/custom_modeling/neox_modeling.py b/server/text_generation_server/models/custom_modeling/neox_modeling.py new file mode 100644 index 00000000..bf2656d1 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/neox_modeling.py @@ -0,0 +1,794 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch GPTNeoX model.""" + +from typing import Optional, Tuple, Union + +import os +import torch +import torch.distributed +import torch.utils.checkpoint +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from transformers.activations import ACT2FN +from transformers.file_utils import ( + add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + replace_return_docstrings, +) +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, + QuestionAnsweringModelOutput, + SequenceClassifierOutputWithPast, + TokenClassifierOutput, +) +from transformers.modeling_utils import PreTrainedModel +from transformers import GPTNeoXConfig +from loguru import logger +from text_generation_server.utils.layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, + TensorParallelHead, +) + + +CUSTOM_KERNELS_ENABLED = False +if not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True": + try: + from custom_kernels import fused_attention_cuda + + CUSTOM_KERNELS_ENABLED = True + except ImportError: + pass + +if not CUSTOM_KERNELS_ENABLED: + logger.warning("We're not using custom kernels.") + + +def make_causal_mask( + input_ids_shape: torch.Size, device: torch.device, past_key_values_length: int +) -> torch.BoolTensor: + """ + Make causal mask used for self-attention. + """ + batch_size, target_length = input_ids_shape + mask = torch.ones( + (target_length, target_length + past_key_values_length), + dtype=torch.bool, + device=device, + ) + mask = mask.triu(1 + past_key_values_length) + + expanded_mask = mask.unsqueeze(0).expand( + batch_size, target_length, target_length + past_key_values_length + ) + return expanded_mask + + +def expand_mask(mask: torch.Tensor, tgt_length: int) -> torch.BoolTensor: + """ + Expands attention_mask from `[batch_size, src_length]` to `[batch_size, 1, tgt_length, src_length]`. + """ + batch_size, src_length = mask.shape + tgt_length = tgt_length if tgt_length is not None else src_length + + expanded_mask = ~(mask[:, None, :].to(torch.bool)) + return expanded_mask.expand(batch_size, tgt_length, src_length) + + +def prepare_attn_mask( + attention_mask: torch.Tensor, + input_shape: Tuple[int, int], + past_key_values_length: int, +) -> torch.BoolTensor: + # create causal mask + # [batch_size, seq_length] -> [batch_size, tgt_length, src_length] + combined_attention_mask = None + device = attention_mask.device + _, src_length = input_shape + + if src_length > 1: + combined_attention_mask = make_causal_mask( + input_shape, device=device, past_key_values_length=past_key_values_length + ) + + # [batch_size, seq_length] -> [batch_size, tgt_length, src_length] + expanded_attn_mask = expand_mask(attention_mask, tgt_length=src_length) + combined_attention_mask = ( + expanded_attn_mask + if combined_attention_mask is None + else expanded_attn_mask | combined_attention_mask + ) + + return combined_attention_mask + + +class GPTNeoXPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + +class GPTNeoXAttention(nn.Module): + def __init__(self, config, prefix, weights): + super().__init__() + self.num_attention_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.num_attention_heads + self.rotary_ndims = int(self.head_size * config.rotary_pct) + max_positions = config.max_position_embeddings + # ??? TODO + # self.register_buffer( + # "bias", + # torch.tril(torch.ones((max_positions, max_positions), dtype=torch.bool)).view( + # 1, 1, max_positions, max_positions + # ), + # ) + # self.register_buffer("masked_bias", torch.tensor(-1e9)) + self.rotary_emb = RotaryEmbedding( + self.rotary_ndims, + config.max_position_embeddings, + base=config.rotary_emb_base, + ) + self.rotary_emb.inv_freq = nn.Parameter( + weights.get_tensor(f"{prefix}.rotary_emb.inv_freq") + ) + self.inv_norm_factor = 1.0 / torch.sqrt( + torch.tensor(self.head_size, dtype=torch.float32) + ).to(torch.get_default_dtype()) + + assert self.num_attention_heads % weights.process_group.size() == 0 + self.num_attention_heads = ( + self.num_attention_heads // weights.process_group.size() + ) + self.query_key_value = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.query_key_value", weights=weights, bias=True + ) + self.dense = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.dense", weights=weights, bias=True + ) + + def forward( + self, + hidden_states, + position_ids, + attention_mask, + head_mask=None, + layer_past=None, + use_cache=False, + output_attentions=False, + ): + has_layer_past = layer_past is not None + + # Compute QKV + # Attention heads [batch, seq_len, hidden_size] + # --> [batch, seq_len, (np * 3 * head_size)] + qkv = self.query_key_value(hidden_states) + + # [batch, seq_len, (num_heads * 3 * head_size)] + # --> [batch, seq_len, num_heads, 3 * head_size] + new_qkv_shape = qkv.size()[:-1] + (self.num_attention_heads, 3 * self.head_size) + qkv = qkv.view(*new_qkv_shape).permute(0, 2, 1, 3) + # [batch, seq_len, num_attention_heads, 3 * head_size] --> 3 [batch, num_attention_heads, seq_len, head_size] + query, key, value = qkv.split(self.head_size, -1) + + # Compute token offset for rotary embeddings (when decoding) + seq_len = key.shape[-2] + if has_layer_past: + seq_len += layer_past[0].shape[-2] + + # Compute rotary embeddings on rotary_ndims + query_rot = query[..., : self.rotary_ndims] + key_rot = key[..., : self.rotary_ndims] + + query_rot, key_rot = self.rotary_emb(query_rot, key_rot, position_ids, seq_len) + + query[..., : self.rotary_ndims] = query_rot + key[..., : self.rotary_ndims] = key_rot + + if CUSTOM_KERNELS_ENABLED: + attn_output, present, attn_weights = fused_attention_cuda.forward( + query, + key, + value, + layer_past, + attention_mask, + head_mask, + self.inv_norm_factor, + self.num_attention_heads, + use_cache, + ) + else: + # Cache QKV values + if has_layer_past: + past_key = layer_past[0] + past_value = layer_past[1] + key = torch.cat((past_key, key), dim=-2) + value = torch.cat((past_value, value), dim=-2) + present = (key, value) if use_cache else None + + # Compute attention + attn_output, attn_weights = self._attn( + query, key, value, attention_mask, head_mask + ) + + # Reshape outputs + attn_output = self._merge_heads( + attn_output, self.num_attention_heads, self.head_size + ) + + attn_output = self.dense(attn_output) + + outputs = (attn_output, present) + if output_attentions: + outputs += (attn_weights,) + + return outputs + + @classmethod + def _split_heads(cls, tensor, num_attention_heads, attn_head_size): + """ + Splits hidden dim into attn_head_size and num_attention_heads + """ + # tensor: [bs, seq_len, hidden_size] + new_shape = tensor.size()[:-1] + (num_attention_heads, attn_head_size) + # -> [bs, seq_len, num_attention_heads, attn_head_size] + tensor = tensor.view(new_shape) + # -> [bs, num_attention_heads, seq_len, attn_head_size] + tensor = tensor.permute(0, 2, 1, 3) + return tensor + + @classmethod + def _merge_heads(cls, tensor, num_attention_heads, attn_head_size): + """ + Merges attn_head_size dim and num_attn_heads dim into hidden dim + """ + # tensor [bs, num_attention_heads, seq_len, attn_head_size] + tensor = tensor.permute(0, 2, 1, 3).contiguous() + # -> [bs, seq_len, num_attention_heads, attn_head_size] + tensor = tensor.view( + tensor.size(0), tensor.size(1), num_attention_heads * attn_head_size + ) + # -> [bs, seq_len, hidden_size] + return tensor + + def _attn(self, query, key, value, attention_mask=None, head_mask=None): + # q, k, v: [bs, num_attention_heads, seq_len, attn_head_size] + # compute causal mask from causal mask buffer + batch_size, num_attention_heads, query_length, attn_head_size = query.size() + key_length = key.size(-2) + + query = query.view( + batch_size * num_attention_heads, query_length, attn_head_size + ) + key = key.view(batch_size * num_attention_heads, key_length, attn_head_size) + attn_scores = torch.zeros( + 1, + dtype=query.dtype, + device=key.device, + ).expand(batch_size * num_attention_heads, query_length, key_length) + attn_scores = torch.baddbmm( + attn_scores, + query, + key.transpose(1, 2), + beta=1.0, + alpha=self.inv_norm_factor, + ) + + # cast attention scores to fp32, compute scaled softmax and cast back to initial dtype - [batch_size, num_heads, q_length, kv_length] + input_dtype = attn_scores.dtype + if input_dtype in [torch.float16, torch.bfloat16]: + attn_scores = attn_scores.to(torch.float) + attn_scores = torch.where( + attention_mask, torch.finfo(attn_scores.dtype).min, attn_scores + ) + attn_scores = attn_scores.view( + batch_size, num_attention_heads, query_length, key_length + ) + + attn_weights = nn.functional.softmax(attn_scores, dim=-1) + attn_weights = attn_weights.to(value.dtype) + + # Mask heads if we want to + if head_mask is not None: + attn_weights = attn_weights * head_mask + + attn_output = torch.matmul(attn_weights, value) + return attn_output, attn_weights + + +class RotaryEmbedding(torch.nn.Module): + def __init__(self, dim, max_position_embeddings, base=10000, device=None): + super().__init__() + self.true_inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2).float().to(device) / dim) + ) + self.register_buffer("inv_freq", self.true_inv_freq) + + # Build here to make `torch.jit.trace` work. + self.max_seq_len_cached = max_position_embeddings + self.cos_cached = None + self.sin_cached = None + + @staticmethod + def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + @staticmethod + def _create_cos_sin(inv_freq, max_position_embeddings, dtype, device): + t = torch.arange( + max_position_embeddings, device=inv_freq.device, dtype=inv_freq.dtype + ) + freqs = torch.einsum("i,j->ij", t, inv_freq) + # Different from paper, but it uses a different permutation in order to obtain the same calculation + emb = torch.cat((freqs, freqs), dim=-1) + return emb.cos().to(device).to(dtype), emb.sin().to(device).to(dtype) + + def forward(self, q, k, position_ids, seq_len=None): + # x: [bs, num_attention_heads, seq_len, head_size] + if ( + seq_len > self.max_seq_len_cached + or self.cos_cached is None + or self.sin_cached is None + ): + if seq_len > self.max_seq_len_cached: + self.max_seq_len_cached = seq_len + self.cos_cached, self.sin_cached = self._create_cos_sin( + self.true_inv_freq, self.max_seq_len_cached, q.dtype, q.device + ) + return rotary_forward(q, k, self.cos_cached, self.sin_cached, position_ids) + + +@torch.jit.script +def rotary_forward(q, k, cos, sin, position_ids): + cos = cos[position_ids].unsqueeze(1) + sin = sin[position_ids].unsqueeze(1) + + chunk_size = q.shape[-1] // 2 + q1, q2 = q.split(chunk_size, -1) + q_rotated = torch.cat((-q2, q1), dim=-1) + k1, k2 = k.split(chunk_size, -1) + k_rotated = torch.cat((-k2, k1), dim=-1) + + q_embed = (q * cos) + (q_rotated * sin) + k_embed = (k * cos) + (k_rotated * sin) + return q_embed, k_embed + + +class GPTNeoXMLP(nn.Module): + def __init__(self, config, prefix, weights): + super().__init__() + self.act = ( + ACT2FN[config.hidden_act] + if "gelu_fast" not in config.hidden_act + else lambda x: torch.nn.functional.gelu(x, approximate="tanh") + ) + + self.dense_h_to_4h = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.dense_h_to_4h", weights=weights, bias=True + ) + self.dense_4h_to_h = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.dense_4h_to_h", weights=weights, bias=True + ) + + def forward(self, hidden_states): + hidden_states = self.dense_h_to_4h(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.dense_4h_to_h(hidden_states) + return hidden_states + + +class GPTNeoXLayer(nn.Module): + def __init__(self, layer_id, config, weights): + super().__init__() + self.use_parallel_residual = config.use_parallel_residual + self.input_layernorm = nn.LayerNorm.load( + prefix=f"gpt_neox.layers.{layer_id}.input_layernorm", + weights=weights, + eps=config.layer_norm_eps, + ) + self.post_attention_layernorm = nn.LayerNorm.load( + prefix=f"gpt_neox.layers.{layer_id}.post_attention_layernorm", + weights=weights, + eps=config.layer_norm_eps, + ) + self.attention = GPTNeoXAttention( + config, prefix=f"gpt_neox.layers.{layer_id}.attention", weights=weights + ) + self.mlp = GPTNeoXMLP( + config, prefix=f"gpt_neox.layers.{layer_id}.mlp", weights=weights + ) + + def forward( + self, + hidden_states, + position_ids, + attention_mask=None, + head_mask=None, + use_cache=False, + layer_past=None, + output_attentions=False, + ): + attention_layer_outputs = self.attention( + self.input_layernorm(hidden_states), + attention_mask=attention_mask, + position_ids=position_ids, + layer_past=layer_past, + head_mask=head_mask, + use_cache=use_cache, + output_attentions=output_attentions, + ) + attn_output = attention_layer_outputs[ + 0 + ] # output_attn: attn_output, present, (attn_weights) + outputs = attention_layer_outputs[1:] + + if self.use_parallel_residual: + # pseudocode: + # x = x + attn(ln1(x)) + mlp(ln2(x)) + mlp_output = self.mlp(self.post_attention_layernorm(hidden_states)) + hidden_states = mlp_output + attn_output + hidden_states + else: + # pseudocode: + # x = x + attn(ln1(x)) + # x = x + mlp(ln2(x)) + attn_output = attn_output + hidden_states + mlp_output = self.mlp(self.post_attention_layernorm(attn_output)) + hidden_states = mlp_output + attn_output + + if use_cache: + outputs = ( + hidden_states, + ) + outputs # hidden_states, present, (attn_weights) + else: + outputs = (hidden_states,) + outputs[1:] # hidden_states, (attn_weights) + + return outputs + + +class GPTNeoXModel(GPTNeoXPreTrainedModel): + def __init__(self, config, weights): + super().__init__(config) + self.config = config + + self.num_attention_heads = config.num_attention_heads + + self.embed_in = TensorParallelEmbedding( + prefix="gpt_neox.embed_in", weights=weights + ) + self.layers = nn.ModuleList( + [ + GPTNeoXLayer(layer_id, config, weights) + for layer_id in range(config.num_hidden_layers) + ] + ) + self.final_layer_norm = nn.LayerNorm.load( + prefix="gpt_neox.final_layer_norm", + weights=weights, + eps=config.layer_norm_eps, + ) + self.tp_world_size = weights.process_group.size() + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + position_ids=None, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + r""" + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + """ + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + "You cannot specify both input_ids and inputs_embeds at the same time" + ) + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + batch_size, seq_length = input_shape + + if past_key_values is None: + past_length = 0 + past_key_values = tuple([None] * self.config.num_hidden_layers) + else: + past_length = past_key_values[0][0].size(-2) + + if position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_length, seq_length + past_length, dtype=torch.long, device=device + ) + position_ids = position_ids.unsqueeze(0).view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() + + if inputs_embeds is None: + inputs_embeds = self.embed_in(input_ids) + + hidden_states = inputs_embeds + + # Attention mask. + seq_length_with_past = seq_length + past_key_values_length = 0 + if past_key_values[0] is not None: + past_key_values_length = past_key_values[0][0].shape[-1] + seq_length_with_past = seq_length_with_past + past_key_values_length + if attention_mask is None: + attention_mask = torch.ones( + (batch_size, seq_length_with_past), device=hidden_states.device + ) + else: + attention_mask = attention_mask.to(hidden_states.device) + + causal_mask = prepare_attn_mask( + attention_mask, + input_shape=(batch_size, seq_length), + past_key_values_length=past_key_values_length, + ) + + assert self.num_attention_heads % self.tp_world_size == 0 + block_size = self.num_attention_heads // self.tp_world_size + causal_mask = torch.repeat_interleave(causal_mask, block_size, dim=0) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) + + presents = () if use_cache else None + all_attentions = () if output_attentions else None + all_hidden_states = () if output_hidden_states else None + for i, (layer, layer_past) in enumerate(zip(self.layers, past_key_values)): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = layer( + hidden_states, + position_ids=position_ids, + attention_mask=causal_mask, + head_mask=head_mask[i], + layer_past=layer_past, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states = outputs[0] + if use_cache is True: + presents = presents + (outputs[1],) + if output_attentions: + all_attentions = all_attentions + (outputs[2 if use_cache else 1],) + + hidden_states = self.final_layer_norm(hidden_states) + # Add last hidden state + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [hidden_states, presents, all_hidden_states, all_attentions] + if v is not None + ) + + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=presents, + hidden_states=all_hidden_states, + attentions=all_attentions, + ) + + +class GPTNeoxForCausalLM(GPTNeoXPreTrainedModel): + _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"] + + def __init__(self, config, weights): + super().__init__(config) + self.gpt_neox = GPTNeoXModel(config, weights) + self.embed_out = TensorParallelHead.load( + config, prefix="embed_out", weights=weights + ) + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional tensors are + only required when the model is used as a decoder in a Sequence to Sequence model. + + Contains pre-computed hidden-states (key and values in the self-attention blocks that can be used (see + `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in + `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are + ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, GPTNeoXForCausalLM, GPTNeoXConfig + >>> import torch + + >>> tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b") + >>> config = GPTNeoXConfig.from_pretrained("EleutherAI/gpt-neox-20b") + >>> config.is_decoder = True + >>> model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/gpt-neox-20b", config=config) + + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + + >>> prediction_logits = outputs.logits + ```""" + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + outputs = self.gpt_neox( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + lm_logits = self.embed_out(hidden_states) + + lm_loss = None + if labels is not None: + # move labels to correct device to enable model parallelism + labels = labels.to(lm_logits.device) + # we are doing next-token prediction; shift prediction scores and input ids by one + shift_logits = lm_logits[:, :-1, :].contiguous() + labels = labels[:, 1:].contiguous() + loss_fct = CrossEntropyLoss() + lm_loss = loss_fct( + shift_logits.view(-1, shift_logits.size(-1)), labels.view(-1) + ) + + if not return_dict: + output = (lm_logits,) + outputs[1:] + return ((lm_loss,) + output) if lm_loss is not None else output + + return CausalLMOutputWithPast( + loss=lm_loss, + logits=lm_logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + **kwargs, + ): + input_shape = input_ids.shape + + # cut decoder_input_ids if past is used + if past_key_values and past_key_values[0] is not None: + input_ids = input_ids[:, -1:] + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -1].unsqueeze(-1) + + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_shape) + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "attention_mask": attention_mask, + "past_key_values": past_key_values, + "position_ids": position_ids, + } + ) + + return model_inputs + + def _reorder_cache(self, past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple( + past_state.index_select(0, beam_idx) + for past_state in layer_past[:2] + ) + + layer_past[2:], + ) + return reordered_past diff --git a/server/text_generation_server/models/custom_modeling/opt_modeling.py b/server/text_generation_server/models/custom_modeling/opt_modeling.py new file mode 100644 index 00000000..03fded50 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/opt_modeling.py @@ -0,0 +1,837 @@ +# coding=utf-8 +# Copyright 2022 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch OPT model.""" +import random +from typing import List, Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn + +from transformers.activations import ACT2FN +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, +) +from transformers.modeling_utils import PreTrainedModel +from transformers import OPTConfig +from text_generation_server.utils.layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, + TensorParallelHead, +) + +EPS = 1e-5 + + +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask( + input_ids_shape: torch.Size, + dtype: torch.dtype, + device: torch.device, + past_key_values_length: int = 0, +): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full( + (tgt_len, tgt_len), + torch.tensor(torch.finfo(dtype).min, device=device), + device=device, + ) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat( + [ + torch.zeros( + tgt_len, past_key_values_length, dtype=dtype, device=device + ), + mask, + ], + dim=-1, + ) + return mask[None, None, :, :].expand( + bsz, 1, tgt_len, tgt_len + past_key_values_length + ) + + +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill( + inverted_mask.to(torch.bool), torch.finfo(dtype).min + ) + + +class OPTLearnedPositionalEmbedding(nn.Module): + """ + This module learns positional embeddings up to a fixed maximum size. + """ + + def __init__(self, weights): + super().__init__() + self.offset = 2 + self.weight = nn.Parameter( + weights.get_tensor("model.decoder.embed_positions.weight") + ) + + def forward( + self, attention_mask: torch.LongTensor, past_key_values_length: int = 0 + ): + """`input_ids_shape` is expected to be [bsz x seqlen].""" + attention_mask = attention_mask.long() + + # create positions depending on attention_mask + positions = ( + torch.cumsum(attention_mask, dim=1).type_as(attention_mask) * attention_mask + ).long() - 1 + + # cut positions if `past_key_values_length` is > 0 + positions = positions[:, past_key_values_length:] + + return torch.nn.functional.embedding(positions + self.offset, self.weight) + + +class OPTAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config, + prefix, + weights, + is_decoder: bool = False, + bias: bool = True, + process_group=None, + ): + super().__init__() + embed_dim = config.embed_dim + num_heads = config.num_attention_heads + + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = config.dropout + self.head_dim = embed_dim // num_heads + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.is_decoder = is_decoder + + process_group = weights.process_group + assert self.num_heads % process_group.size() == 0 + self.num_heads = self.num_heads // process_group.size() + self.embed_dim = self.embed_dim // process_group.size() + + self.q_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.q_proj", weights=weights, bias=bias + ) + self.k_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.k_proj", weights=weights, bias=bias + ) + self.v_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.v_proj", weights=weights, bias=bias + ) + self.out_proj = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.out_proj", weights=weights, bias=bias + ) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return ( + tensor.view(bsz, seq_len, self.num_heads, self.head_dim) + .transpose(1, 2) + .contiguous() + ) + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = ( + attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + + attention_mask + ) + attn_weights = torch.max( + attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min) + ) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # upcast to fp32 if the weights are in fp16. Please see https://github.com/huggingface/transformers/pull/17437 + if attn_weights.dtype == torch.float16: + attn_weights = nn.functional.softmax( + attn_weights, dim=-1, dtype=torch.float32 + ).to(torch.float16) + else: + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if layer_head_mask is not None: + if layer_head_mask.size() != (self.num_heads,): + raise ValueError( + f"Head mask for a single layer should be of size {(self.num_heads,)}, but is" + f" {layer_head_mask.size()}" + ) + attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view( + bsz, self.num_heads, tgt_len, src_len + ) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if output_attentions: + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to be reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view( + bsz, self.num_heads, tgt_len, src_len + ) + attn_weights = attn_weights_reshaped.view( + bsz * self.num_heads, tgt_len, src_len + ) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout( + attn_weights, p=self.dropout, training=self.training + ) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + + # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # partitioned aross GPUs when using tensor-parallelism. + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped, past_key_value + + +class OPTDecoderLayer(nn.Module): + def __init__(self, layer_id: int, config: OPTConfig, weights): + super().__init__() + self.process_group = weights.process_group + self.embed_dim = config.hidden_size + prefix = f"model.decoder.layers.{layer_id}" + self.self_attn = OPTAttention( + config, + prefix=f"{prefix}.self_attn", + weights=weights, + is_decoder=True, + bias=config.enable_bias, + ) + self.do_layer_norm_before = config.do_layer_norm_before + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + + self.self_attn_layer_norm = nn.LayerNorm.load( + prefix=f"{prefix}.self_attn_layer_norm", weights=weights, eps=EPS + ) + self.fc1 = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.fc1", weights=weights, bias=config.enable_bias + ) + self.fc2 = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.fc2", weights=weights, bias=config.enable_bias + ) + self.final_layer_norm = nn.LayerNorm.load( + prefix=f"{prefix}.final_layer_norm", weights=weights, eps=EPS + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + ) -> Tuple[ + torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]] + ]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + layer_head_mask (`torch.FloatTensor`, *optional*): mask for attention heads in a given layer of size + `(encoder_attention_heads,)`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention + if self.do_layer_norm_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + past_key_value=past_key_value, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + ) + hidden_states = nn.functional.dropout( + hidden_states, p=self.dropout, training=self.training + ) + hidden_states = residual + hidden_states + + # 350m applies layer norm AFTER attention + if not self.do_layer_norm_before: + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Fully Connected + hidden_states_shape = hidden_states.shape + hidden_states = hidden_states.reshape(-1, hidden_states.size(-1)) + residual = hidden_states + + # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention + if self.do_layer_norm_before: + hidden_states = self.final_layer_norm(hidden_states) + + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + + hidden_states = self.fc2(hidden_states) + hidden_states = nn.functional.dropout( + hidden_states, p=self.dropout, training=self.training + ) + + hidden_states = (residual + hidden_states).view(hidden_states_shape) + + # 350m applies layer norm AFTER attention + if not self.do_layer_norm_before: + hidden_states = self.final_layer_norm(hidden_states) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +class OPTPreTrainedModel(PreTrainedModel): + config_class = OPTConfig + + +class OPTDecoder(OPTPreTrainedModel): + def __init__(self, config: OPTConfig, weights): + super().__init__(config) + self.dropout = config.dropout + self.layerdrop = config.layerdrop + self.padding_idx = config.pad_token_id + self.max_target_positions = config.max_position_embeddings + self.vocab_size = config.vocab_size + + self.embed_tokens = TensorParallelEmbedding( + prefix="model.decoder.embed_tokens", weights=weights + ) + self.embed_positions = OPTLearnedPositionalEmbedding(weights) + + if config.word_embed_proj_dim != config.hidden_size: + self.project_out = FastLinear.load( + config, prefix="model.decoder.project_out", bias=False + ) + else: + self.project_out = None + + if config.word_embed_proj_dim != config.hidden_size: + self.project_in = FastLinear.load( + config, prefix="model.decoder.project_in", bias=False + ) + else: + self.project_in = None + + # Note that the only purpose of `config._remove_final_layer_norm` is to keep backward compatibility + # with checkpoints that have been fine-tuned before transformers v4.20.1 + # see https://github.com/facebookresearch/metaseq/pull/164 + if config.do_layer_norm_before and not config._remove_final_layer_norm: + self.final_layer_norm = nn.LayerNorm.load( + prefix="model.decoder.final_layer_norm", weights=weights, eps=EPS + ) + else: + self.final_layer_norm = None + + self.layers = nn.ModuleList( + [ + OPTDecoderLayer(layer_id, config, weights) + for layer_id in range(config.num_hidden_layers) + ] + ) + + # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask + def _prepare_decoder_attention_mask( + self, attention_mask, input_shape, inputs_embeds, past_key_values_length + ): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask( + attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1] + ).to(inputs_embeds.device) + combined_attention_mask = ( + expanded_attn_mask + if combined_attention_mask is None + else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError( + "You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time" + ) + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError( + "You have to specify either decoder_input_ids or decoder_inputs_embeds" + ) + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + batch_size, seq_length = input_shape + past_key_values_length = ( + past_key_values[0][0].shape[2] if past_key_values is not None else 0 + ) + # required mask seq length can be calculated via length of past + mask_seq_length = past_key_values_length + seq_length + + # embed positions + if attention_mask is None: + attention_mask = torch.ones( + batch_size, mask_seq_length, device=inputs_embeds.device + ) + causal_attention_mask = self._prepare_decoder_attention_mask( + attention_mask, input_shape, inputs_embeds, past_key_values_length + ) + pos_embeds = self.embed_positions(attention_mask, past_key_values_length) + + if self.project_in is not None: + inputs_embeds = self.project_in(inputs_embeds) + + hidden_states = inputs_embeds + pos_embeds + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = () if use_cache else None + + # check if head_mask has a correct number of layers specified if desired + for attn_mask, mask_name in zip([head_mask], ["head_mask"]): + if attn_mask is not None: + if attn_mask.size()[0] != (len(self.layers)): + raise ValueError( + f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for" + f" {head_mask.size()[0]}." + ) + + for idx, decoder_layer in enumerate(self.layers): + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + if output_hidden_states: + all_hidden_states += (hidden_states,) + + dropout_probability = random.uniform(0, 1) + if self.training and (dropout_probability < self.layerdrop): + continue + + past_key_value = ( + past_key_values[idx] if past_key_values is not None else None + ) + + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_attention_mask, + layer_head_mask=(head_mask[idx] if head_mask is not None else None), + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += (layer_outputs[2 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + if self.final_layer_norm is not None: + hidden_states = self.final_layer_norm(hidden_states) + + if self.project_out is not None: + hidden_states = self.project_out(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if not return_dict: + return tuple( + v + for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] + if v is not None + ) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class OPTModel(OPTPreTrainedModel): + def __init__(self, config: OPTConfig, weights): + super().__init__(config) + self.decoder = OPTDecoder(config, weights) + # Initialize weights and apply final processing + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) + decoder_outputs = self.decoder( + input_ids=input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if not return_dict: + return decoder_outputs + + return BaseModelOutputWithPast( + last_hidden_state=decoder_outputs.last_hidden_state, + past_key_values=decoder_outputs.past_key_values, + hidden_states=decoder_outputs.hidden_states, + attentions=decoder_outputs.attentions, + ) + + +class OPTForCausalLM(OPTPreTrainedModel): + def __init__(self, config, weights): + super().__init__(config) + + self.model = OPTModel(config, weights) + + self.lm_head = TensorParallelHead.load( + config, prefix="model.decoder.embed_tokens", weights=weights + ) + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model.decoder( + input_ids=input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + logits = self.lm_head(outputs[0]).contiguous() + + loss = None + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + **kwargs, + ): + if past_key_values: + input_ids = input_ids[:, -1:] + + # if `inputs_embeds` are passed, we only want to use them in the 1st generation step + if inputs_embeds is not None and past_key_values is None: + model_inputs = {"inputs_embeds": inputs_embeds} + else: + model_inputs = {"input_ids": input_ids} + + model_inputs.update( + { + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "attention_mask": attention_mask, + } + ) + return model_inputs + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + reordered_past = () + for layer_past in past_key_values: + reordered_past += ( + tuple( + past_state.index_select(0, beam_idx) for past_state in layer_past + ), + ) + return reordered_past diff --git a/server/text_generation_server/models/custom_modeling/t5_modeling.py b/server/text_generation_server/models/custom_modeling/t5_modeling.py new file mode 100644 index 00000000..51862e3c --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/t5_modeling.py @@ -0,0 +1,1200 @@ +# coding=utf-8 +# Copyright 2018 Mesh TensorFlow authors, T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch T5 model.""" + +import copy +import math +import warnings +from typing import Optional, Tuple, Union + +import torch +import torch.distributed +from torch import nn +from torch.nn import CrossEntropyLoss + +from transformers.activations import ACT2FN +from transformers.modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPastAndCrossAttentions, + Seq2SeqLMOutput, +) +from transformers.modeling_utils import PreTrainedModel +from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS +from transformers.utils import ( + is_torch_fx_proxy, +) +from transformers import T5Config +from text_generation_server.utils.layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, + TensorParallelHead, +) + + +class PartialTPEmbedding(nn.Module): + def __init__(self, prefix: str, weights): + super().__init__() + weight = weights.get_sharded(f"{prefix}.weight", dim=1) + self.weight = nn.Parameter(weight) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + return torch.nn.functional.embedding(input, self.weight) + + +@torch.jit.script +def layer_norm(hidden_states, weight, epsilon): + # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean + # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus varience is calculated + # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for + # half-precision inputs is done in fp32 + + variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + epsilon) + + # convert into half-precision if necessary + if weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(weight.dtype) + + return weight * hidden_states + + +class T5LayerNorm(nn.Module): + def __init__(self, prefix, weights, eps=1e-6): + """ + Construct a layernorm module in the T5 style. No bias and no subtraction of mean. + """ + super().__init__() + weight = weights.get_tensor(f"{prefix}.weight") + self.weight = nn.Parameter(weight) + self.variance_epsilon = torch.tensor(eps) + + def forward(self, hidden_states): + return layer_norm(hidden_states, self.weight, self.variance_epsilon) + + +try: + from apex.normalization import FusedRMSNorm + + T5LayerNorm = FusedRMSNorm # noqa + + logger.info( + "Discovered apex.normalization.FusedRMSNorm - will use it instead of T5LayerNorm" + ) +except ImportError: + # using the normal T5LayerNorm + pass +except Exception: + logger.warning("discovered apex but it failed to load, falling back to T5LayerNorm") + pass + +ALL_LAYERNORM_LAYERS.append(T5LayerNorm) + + +class T5DenseActDense(nn.Module): + def __init__(self, config: T5Config, prefix, weights): + super().__init__() + self.wi = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.wi", weights=weights, bias=False + ) + + ### XXX: T5 models do not handle well both f16 and quantization. + ### Overidding specifically this layer for that reason. + ### https://github.com/huggingface/transformers/blob/main/src/transformers/models/t5/modeling_t5.py#L316 + ### https://github.com/huggingface/transformers/issues/20287 + _q = config.quantize + _dtype = weights.dtype + weights.dtype = torch.float32 + config.quantize = None + self.wo_cast = (torch.float32, _dtype) + self.wo = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.wo", weights=weights, bias=False + ) + weights.dtype = _dtype + config.quantize = _q + + self.dropout = nn.Dropout(config.dropout_rate) + self.act = ( + ACT2FN[config.dense_act_fn] + if "gelu" not in config.dense_act_fn + else lambda x: torch.nn.functional.gelu(x, approximate="tanh") + ) + + def forward(self, hidden_states): + hidden_states = self.wi(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.dropout(hidden_states) + + hidden_states = hidden_states.to(dtype=self.wo_cast[0]) + hidden_states = self.wo(hidden_states) + # XXX: Recasting is already done within the layer norm. + # Casting back to float16 here modifies results + # hidden_states = hidden_states.to(dtype=self.wo_cast[1]) + return hidden_states + + +class T5DenseGatedActDense(nn.Module): + def __init__(self, config: T5Config, prefix, weights): + super().__init__() + self.wi_0 = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.wi_0", weights=weights, bias=False + ) + self.wi_1 = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.wi_1", weights=weights, bias=False + ) + ### XXX: T5 models do not handle well both f16 and quantization. + ### Overidding specifically this layer for that reason. + ### https://github.com/huggingface/transformers/blob/main/src/transformers/models/t5/modeling_t5.py#L316 + ### https://github.com/huggingface/transformers/issues/20287 + _q = config.quantize + _dtype = weights.dtype + weights.dtype = torch.float32 + config.quantize = None + self.wo_cast = (torch.float32, _dtype) + self.wo = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.wo", weights=weights, bias=False + ) + weights.dtype = _dtype + config.quantize = _q + + self.dropout = nn.Dropout(config.dropout_rate) + self.act = ( + ACT2FN[config.dense_act_fn] + if "gelu" not in config.dense_act_fn + else lambda x: torch.nn.functional.gelu(x, approximate="tanh") + ) + + def forward(self, hidden_states): + hidden_gelu = self.act(self.wi_0(hidden_states)) + hidden_linear = self.wi_1(hidden_states) + hidden_states = hidden_gelu * hidden_linear + hidden_states = self.dropout(hidden_states) + + hidden_states = hidden_states.to(dtype=self.wo_cast[0]) + hidden_states = self.wo(hidden_states) + # XXX: Recasting is already done within the layer norm. + # Casting back to float16 here modifies results + # hidden_states = hidden_states.to(dtype=self.wo_cast[1]) + return hidden_states + + +class T5LayerFF(nn.Module): + def __init__(self, config: T5Config, prefix, weights): + super().__init__() + if config.is_gated_act: + self.DenseReluDense = T5DenseGatedActDense( + config, prefix=f"{prefix}.DenseReluDense", weights=weights + ) + else: + self.DenseReluDense = T5DenseActDense( + config, prefix=f"{prefix}.DenseReluDense", weights=weights + ) + + self.layer_norm = T5LayerNorm( + prefix=f"{prefix}.layer_norm", + weights=weights, + eps=config.layer_norm_epsilon, + ) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward(self, hidden_states): + forwarded_states = self.layer_norm(hidden_states) + forwarded_states = self.DenseReluDense(forwarded_states) + hidden_states = hidden_states + self.dropout(forwarded_states) + return hidden_states + + +class T5Attention(nn.Module): + def __init__( + self, config: T5Config, prefix, weights, has_relative_attention_bias=False + ): + super().__init__() + self.is_decoder = config.is_decoder + self.has_relative_attention_bias = has_relative_attention_bias + self.relative_attention_num_buckets = config.relative_attention_num_buckets + self.relative_attention_max_distance = config.relative_attention_max_distance + self.d_model = config.d_model + self.key_value_proj_dim = config.d_kv + self.n_heads = config.num_heads + self.dropout = config.dropout_rate + self.inner_dim = self.n_heads * self.key_value_proj_dim + + process_group = weights.process_group + # Mesh TensorFlow initialization to avoid scaling before softmax + assert self.n_heads % process_group.size() == 0 + self.q = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.q", weights=weights, bias=False + ) + self.k = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.k", weights=weights, bias=False + ) + self.v = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.v", weights=weights, bias=False + ) + self.o = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.o", weights=weights, bias=False + ) + self.n_heads = self.n_heads // process_group.size() + self.inner_dim = self.inner_dim // process_group.size() + + if self.has_relative_attention_bias: + self.relative_attention_bias = PartialTPEmbedding( + prefix=f"{prefix}.relative_attention_bias", weights=weights + ) + + @staticmethod + def _relative_position_bucket( + relative_position, bidirectional=True, num_buckets=32, max_distance=128 + ): + """ + Adapted from Mesh Tensorflow: + https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 + + Translate relative position to a bucket number for relative attention. The relative position is defined as + memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to + position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for + small absolute relative_position and larger buckets for larger absolute relative_positions. All relative + positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. + This should allow for more graceful generalization to longer sequences than the model has been trained on + + Args: + relative_position: an int32 Tensor + bidirectional: a boolean - whether the attention is bidirectional + num_buckets: an integer + max_distance: an integer + + Returns: + a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets) + """ + relative_buckets = 0 + if bidirectional: + num_buckets //= 2 + relative_buckets += (relative_position > 0).to(torch.long) * num_buckets + relative_position = torch.abs(relative_position) + else: + relative_position = -torch.min( + relative_position, torch.zeros_like(relative_position) + ) + # now relative_position is in the range [0, inf) + + # half of the buckets are for exact increments in positions + max_exact = num_buckets // 2 + is_small = relative_position < max_exact + + # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance + relative_position_if_large = max_exact + ( + torch.log(relative_position.float() / max_exact) + / math.log(max_distance / max_exact) + * (num_buckets - max_exact) + ).to(torch.long) + relative_position_if_large = torch.min( + relative_position_if_large, + torch.full_like(relative_position_if_large, num_buckets - 1), + ) + + relative_buckets += torch.where( + is_small, relative_position, relative_position_if_large + ) + return relative_buckets + + def compute_bias(self, query_length, key_length, device=None): + """Compute binned relative position bias""" + if device is None: + device = self.relative_attention_bias.weight.device + context_position = torch.arange(query_length, dtype=torch.long, device=device)[ + :, None + ] + memory_position = torch.arange(key_length, dtype=torch.long, device=device)[ + None, : + ] + relative_position = ( + memory_position - context_position + ) # shape (query_length, key_length) + relative_position_bucket = self._relative_position_bucket( + relative_position, # shape (query_length, key_length) + bidirectional=(not self.is_decoder), + num_buckets=self.relative_attention_num_buckets, + max_distance=self.relative_attention_max_distance, + ) + values = self.relative_attention_bias( + relative_position_bucket + ) # shape (query_length, key_length, num_heads) + values = values.permute([2, 0, 1]).unsqueeze( + 0 + ) # shape (1, num_heads, query_length, key_length) + return values + + def forward( + self, + hidden_states, + mask=None, + key_value_states=None, + position_bias=None, + past_key_value=None, + layer_head_mask=None, + query_length=None, + use_cache=False, + output_attentions=False, + ): + """ + Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states). + """ + # Input is (batch_size, seq_length, dim) + # Mask is (batch_size, key_length) (non-causal) or (batch_size, key_length, key_length) + # past_key_value[0] is (batch_size, n_heads, q_len - 1, dim_per_head) + + batch_size, seq_length = hidden_states.shape[:2] + + real_seq_length = seq_length + + if past_key_value is not None: + assert ( + len(past_key_value) == 2 + ), f"past_key_value should have 2 past states: keys and values. Got {len(past_key_value)} past states" + real_seq_length += ( + past_key_value[0].shape[2] if query_length is None else query_length + ) + + key_length = ( + real_seq_length if key_value_states is None else key_value_states.shape[1] + ) + + def shape(states): + """projection""" + return states.view( + batch_size, -1, self.n_heads, self.key_value_proj_dim + ).transpose(1, 2) + + def unshape(states): + """reshape""" + return ( + states.transpose(1, 2).contiguous().view(batch_size, -1, self.inner_dim) + ) + + def project(hidden_states, proj_layer, key_value_states, past_key_value): + """projects hidden states correctly to key/query states""" + if key_value_states is None: + # self-attn + # (batch_size, n_heads, seq_length, dim_per_head) + hidden_states = shape(proj_layer(hidden_states)) + elif past_key_value is None: + # cross-attn + # (batch_size, n_heads, seq_length, dim_per_head) + hidden_states = shape(proj_layer(key_value_states)) + + if past_key_value is not None: + if key_value_states is None: + # self-attn + # (batch_size, n_heads, key_length, dim_per_head) + hidden_states = torch.cat([past_key_value, hidden_states], dim=2) + elif past_key_value.shape[2] != key_value_states.shape[1]: + # checking that the `sequence_length` of the `past_key_value` is the same as + # the provided `key_value_states` to support prefix tuning + # cross-attn + # (batch_size, n_heads, seq_length, dim_per_head) + hidden_states = shape(proj_layer(key_value_states)) + else: + # cross-attn + hidden_states = past_key_value + return hidden_states + + # get query states + query_states = shape( + self.q(hidden_states) + ) # (batch_size, n_heads, seq_length, dim_per_head) + + # get key/value states + key_states = project( + hidden_states, + self.k, + key_value_states, + past_key_value[0] if past_key_value is not None else None, + ) + value_states = project( + hidden_states, + self.v, + key_value_states, + past_key_value[1] if past_key_value is not None else None, + ) + + # compute scores + scores = torch.matmul( + query_states, key_states.transpose(3, 2) + ) # equivalent of torch.einsum("bnqd,bnkd->bnqk", query_states, key_states), compatible with onnx op>9 + + if position_bias is None: + if not self.has_relative_attention_bias: + position_bias = torch.zeros( + (1, self.n_heads, real_seq_length, key_length), + device=scores.device, + dtype=scores.dtype, + ) + else: + position_bias = self.compute_bias( + real_seq_length, key_length, device=scores.device + ) + + # if key and values are already calculated + # we want only the last query position bias + if past_key_value is not None: + position_bias = position_bias[:, :, -hidden_states.size(1) :, :] + + if mask is not None: + position_bias = ( + position_bias + mask + ) # (batch_size, n_heads, seq_length, key_length) + + position_bias_masked = position_bias + + scores += position_bias_masked + attn_weights = nn.functional.softmax(scores.float(), dim=-1).type_as( + scores + ) # (batch_size, n_heads, seq_length, key_length) + attn_weights = nn.functional.dropout( + attn_weights, p=self.dropout, training=self.training + ) # (batch_size, n_heads, seq_length, key_length) + + # Mask heads if we want to + if layer_head_mask is not None: + attn_weights = attn_weights * layer_head_mask + + attn_output = unshape( + torch.matmul(attn_weights, value_states) + ) # (batch_size, seq_length, dim) + attn_output = self.o(attn_output) + + present_key_value_state = ( + (key_states, value_states) if (self.is_decoder and use_cache) else None + ) + outputs = (attn_output,) + (present_key_value_state,) + (position_bias,) + + if output_attentions: + outputs = outputs + (attn_weights,) + return outputs + + +class T5LayerSelfAttention(nn.Module): + def __init__(self, config, prefix, weights, has_relative_attention_bias=False): + super().__init__() + self.SelfAttention = T5Attention( + config, + prefix=f"{prefix}.SelfAttention", + weights=weights, + has_relative_attention_bias=has_relative_attention_bias, + ) + self.layer_norm = T5LayerNorm( + prefix=f"{prefix}.layer_norm", + weights=weights, + eps=config.layer_norm_epsilon, + ) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward( + self, + hidden_states, + attention_mask=None, + position_bias=None, + layer_head_mask=None, + past_key_value=None, + use_cache=False, + output_attentions=False, + ): + normed_hidden_states = self.layer_norm(hidden_states) + attention_output = self.SelfAttention( + normed_hidden_states, + mask=attention_mask, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states = hidden_states + self.dropout(attention_output[0]) + outputs = (hidden_states,) + attention_output[ + 1: + ] # add attentions if we output them + return outputs + + +class T5LayerCrossAttention(nn.Module): + def __init__(self, config, prefix, weights): + super().__init__() + self.EncDecAttention = T5Attention( + config, + prefix=f"{prefix}.EncDecAttention", + weights=weights, + has_relative_attention_bias=False, + ) + self.layer_norm = T5LayerNorm( + prefix=f"{prefix}.layer_norm", + weights=weights, + eps=config.layer_norm_epsilon, + ) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward( + self, + hidden_states, + key_value_states, + attention_mask=None, + position_bias=None, + layer_head_mask=None, + past_key_value=None, + use_cache=False, + query_length=None, + output_attentions=False, + ): + normed_hidden_states = self.layer_norm(hidden_states) + attention_output = self.EncDecAttention( + normed_hidden_states, + mask=attention_mask, + key_value_states=key_value_states, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + query_length=query_length, + output_attentions=output_attentions, + ) + layer_output = hidden_states + self.dropout(attention_output[0]) + outputs = (layer_output,) + attention_output[ + 1: + ] # add attentions if we output them + return outputs + + +class T5Block(nn.Module): + def __init__(self, config, prefix, weights, has_relative_attention_bias: bool): + super().__init__() + self.is_decoder = config.is_decoder + self.layer = nn.ModuleList() + self.layer.append( + T5LayerSelfAttention( + config, + prefix=f"{prefix}.layer.0", + weights=weights, + has_relative_attention_bias=has_relative_attention_bias, + ) + ) + if self.is_decoder: + i = 2 + self.layer.append( + T5LayerCrossAttention( + config, prefix=f"{prefix}.layer.1", weights=weights + ) + ) + else: + i = 1 + + self.layer.append( + T5LayerFF(config, prefix=f"{prefix}.layer.{i}", weights=weights) + ) + + def forward( + self, + hidden_states, + attention_mask=None, + position_bias=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + encoder_decoder_position_bias=None, + layer_head_mask=None, + cross_attn_layer_head_mask=None, + past_key_value=None, + use_cache=False, + output_attentions=False, + return_dict=True, + ): + if past_key_value is not None: + if not self.is_decoder: + logger.warning( + "`past_key_values` is passed to the encoder. Please make sure this is intended." + ) + expected_num_past_key_values = 2 if encoder_hidden_states is None else 4 + + if len(past_key_value) != expected_num_past_key_values: + raise ValueError( + f"There should be {expected_num_past_key_values} past states. " + f"{'2 (past / key) for cross attention. ' if expected_num_past_key_values == 4 else ''}" + f"Got {len(past_key_value)} past key / value states" + ) + + self_attn_past_key_value = past_key_value[:2] + cross_attn_past_key_value = past_key_value[2:] + else: + self_attn_past_key_value, cross_attn_past_key_value = None, None + + self_attention_outputs = self.layer[0]( + hidden_states, + attention_mask=attention_mask, + position_bias=position_bias, + layer_head_mask=layer_head_mask, + past_key_value=self_attn_past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states, present_key_value_state = self_attention_outputs[:2] + attention_outputs = self_attention_outputs[ + 2: + ] # Keep self-attention outputs and relative position weights + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) + hidden_states = torch.clamp( + hidden_states, min=-clamp_value, max=clamp_value + ) + + do_cross_attention = self.is_decoder and encoder_hidden_states is not None + if do_cross_attention: + # the actual query length is unknown for cross attention + # if using past key value states. Need to inject it here + if present_key_value_state is not None: + query_length = present_key_value_state[0].shape[2] + else: + query_length = None + + cross_attention_outputs = self.layer[1]( + hidden_states, + key_value_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + position_bias=encoder_decoder_position_bias, + layer_head_mask=cross_attn_layer_head_mask, + past_key_value=cross_attn_past_key_value, + query_length=query_length, + use_cache=use_cache, + output_attentions=output_attentions, + ) + hidden_states = cross_attention_outputs[0] + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) + hidden_states = torch.clamp( + hidden_states, min=-clamp_value, max=clamp_value + ) + + # Combine self attn and cross attn key value states + if present_key_value_state is not None: + present_key_value_state = ( + present_key_value_state + cross_attention_outputs[1] + ) + + # Keep cross-attention outputs and relative position weights + attention_outputs = attention_outputs + cross_attention_outputs[2:] + + # Apply Feed Forward layer + hidden_states = self.layer[-1](hidden_states) + + # clamp inf values to enable fp16 training + if hidden_states.dtype == torch.float16: + clamp_value = torch.where( + torch.isinf(hidden_states).any(), + torch.finfo(hidden_states.dtype).max - 1000, + torch.finfo(hidden_states.dtype).max, + ) + hidden_states = torch.clamp( + hidden_states, min=-clamp_value, max=clamp_value + ) + + outputs = (hidden_states,) + + if use_cache: + outputs = outputs + (present_key_value_state,) + attention_outputs + else: + outputs = outputs + attention_outputs + + return outputs # hidden-states, present_key_value_states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights) + + +class T5PreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = T5Config + + def _shift_right(self, input_ids): + decoder_start_token_id = self.config.decoder_start_token_id + pad_token_id = self.config.pad_token_id + + assert decoder_start_token_id is not None, ( + "self.model.config.decoder_start_token_id has to be defined. In T5 it is usually set to the pad_token_id." + " See T5 docs for more information" + ) + + # shift inputs to the right + if is_torch_fx_proxy(input_ids): + # Item assignment is not supported natively for proxies. + shifted_input_ids = torch.full( + input_ids.shape[:-1] + (1,), decoder_start_token_id + ) + shifted_input_ids = torch.cat( + [shifted_input_ids, input_ids[..., :-1]], dim=-1 + ) + else: + shifted_input_ids = input_ids.new_zeros(input_ids.shape) + shifted_input_ids[..., 1:] = input_ids[..., :-1].clone() + shifted_input_ids[..., 0] = decoder_start_token_id + + assert ( + pad_token_id is not None + ), "self.model.config.pad_token_id has to be defined." + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id) + + return shifted_input_ids + + +class T5Stack(T5PreTrainedModel): + def __init__(self, config, prefix, weights, embed_tokens): + super().__init__(config) + + self.is_decoder = config.is_decoder + + self.embed_tokens = embed_tokens + self.block = nn.ModuleList( + [ + T5Block( + config, + prefix=f"{prefix}.block.{layer_id}", + weights=weights, + has_relative_attention_bias=(layer_id == 0), + ) + for layer_id in range(config.num_layers) + ] + ) + self.final_layer_norm = T5LayerNorm( + prefix=f"{prefix}.final_layer_norm", + weights=weights, + eps=config.layer_norm_epsilon, + ) + self.dropout = nn.Dropout(config.dropout_rate) + + def forward( + self, + input_ids=None, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + inputs_embeds=None, + head_mask=None, + cross_attn_head_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + # Model parallel + use_cache = use_cache if use_cache is not None else self.config.use_cache + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) + output_hidden_states = ( + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + if input_ids is not None and inputs_embeds is not None: + err_msg_prefix = "decoder_" if self.is_decoder else "" + raise ValueError( + f"You cannot specify both {err_msg_prefix}input_ids and {err_msg_prefix}inputs_embeds at the same time" + ) + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + err_msg_prefix = "decoder_" if self.is_decoder else "" + raise ValueError( + f"You have to specify either {err_msg_prefix}input_ids or {err_msg_prefix}inputs_embeds" + ) + + if inputs_embeds is None: + assert ( + self.embed_tokens is not None + ), "You have to initialize the model with valid token embeddings" + inputs_embeds = self.embed_tokens(input_ids) + + batch_size, seq_length = input_shape + + # required mask seq length can be calculated via length of past + mask_seq_length = ( + past_key_values[0][0].shape[2] + seq_length + if past_key_values is not None + else seq_length + ) + + if use_cache is True: + assert ( + self.is_decoder + ), f"`use_cache` can only be set to `True` if {self} is used as a decoder" + + if attention_mask is None: + attention_mask = torch.ones( + batch_size, mask_seq_length, device=inputs_embeds.device + ) + if ( + self.is_decoder + and encoder_attention_mask is None + and encoder_hidden_states is not None + ): + encoder_seq_length = encoder_hidden_states.shape[1] + encoder_attention_mask = torch.ones( + batch_size, + encoder_seq_length, + device=inputs_embeds.device, + dtype=torch.long, + ) + + # initialize past_key_values with `None` if past does not exist + if past_key_values is None: + past_key_values = [None] * len(self.block) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask = self.get_extended_attention_mask( + attention_mask, input_shape + ) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.is_decoder and encoder_hidden_states is not None: + ( + encoder_batch_size, + encoder_sequence_length, + _, + ) = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones( + encoder_hidden_shape, device=inputs_embeds.device + ) + encoder_extended_attention_mask = self.invert_attention_mask( + encoder_attention_mask + ) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + head_mask = self.get_head_mask(head_mask, self.config.num_layers) + cross_attn_head_mask = self.get_head_mask( + cross_attn_head_mask, self.config.num_layers + ) + present_key_value_states = () if use_cache else None + all_hidden_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + all_cross_attentions = () if (output_attentions and self.is_decoder) else None + position_bias = None + encoder_decoder_position_bias = None + + hidden_states = self.dropout(inputs_embeds) + + for i, (layer_module, past_key_value) in enumerate( + zip(self.block, past_key_values) + ): + layer_head_mask = head_mask[i] + cross_attn_layer_head_mask = cross_attn_head_mask[i] + # Model parallel + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module( + hidden_states, + attention_mask=extended_attention_mask, + position_bias=position_bias, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + encoder_decoder_position_bias=encoder_decoder_position_bias, + layer_head_mask=layer_head_mask, + cross_attn_layer_head_mask=cross_attn_layer_head_mask, + past_key_value=past_key_value, + use_cache=use_cache, + output_attentions=output_attentions, + ) + + # layer_outputs is a tuple with: + # hidden-states, key-value-states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights) + if use_cache is False: + layer_outputs = layer_outputs[:1] + (None,) + layer_outputs[1:] + + hidden_states, present_key_value_state = layer_outputs[:2] + + # We share the position biases between the layers - the first layer store them + # layer_outputs = hidden-states, key-value-states (self-attention position bias), (self-attention weights), + # (cross-attention position bias), (cross-attention weights) + position_bias = layer_outputs[2] + if self.is_decoder and encoder_hidden_states is not None: + encoder_decoder_position_bias = layer_outputs[ + 4 if output_attentions else 3 + ] + # append next layer key value states + if use_cache: + present_key_value_states = present_key_value_states + ( + present_key_value_state, + ) + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[3],) + if self.is_decoder: + all_cross_attentions = all_cross_attentions + (layer_outputs[5],) + + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = self.dropout(hidden_states) + + # Add last layer + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + present_key_value_states, + all_hidden_states, + all_attentions, + all_cross_attentions, + ] + if v is not None + ) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=present_key_value_states, + hidden_states=all_hidden_states, + attentions=all_attentions, + cross_attentions=all_cross_attentions, + ) + + +class T5ForConditionalGeneration(T5PreTrainedModel): + def __init__(self, config: T5Config, weights): + super().__init__(config) + self.model_dim = config.d_model + + self.shared = TensorParallelEmbedding(prefix="shared", weights=weights) + + encoder_config = copy.deepcopy(config) + encoder_config.is_decoder = False + encoder_config.use_cache = False + encoder_config.is_encoder_decoder = False + self.encoder = T5Stack( + config=encoder_config, + prefix="encoder", + weights=weights, + embed_tokens=self.shared, + ) + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + decoder_config.is_encoder_decoder = False + decoder_config.num_layers = config.num_decoder_layers + self.decoder = T5Stack( + config=decoder_config, + prefix="decoder", + weights=weights, + embed_tokens=self.shared, + ) + + self.lm_head = TensorParallelHead.load( + config, prefix="lm_head", weights=weights + ) + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.BoolTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + decoder_head_mask: Optional[torch.FloatTensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[Tuple[Tuple[torch.Tensor]]] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]: + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + # FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask + if head_mask is not None and decoder_head_mask is None: + if self.config.num_layers == self.config.num_decoder_layers: + warnings.warn(__HEAD_MASK_WARNING_MSG, FutureWarning) + decoder_head_mask = head_mask + + # Encode if needed (training, first prediction pass) + if encoder_outputs is None: + # Convert encoder inputs in embeddings if needed + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + elif return_dict and not isinstance(encoder_outputs, BaseModelOutput): + encoder_outputs = BaseModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, + ) + + hidden_states = encoder_outputs[0] + + if ( + labels is not None + and decoder_input_ids is None + and decoder_inputs_embeds is None + ): + # get decoder inputs from shifting lm labels to the right + decoder_input_ids = self._shift_right(labels) + + # Decode + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + inputs_embeds=decoder_inputs_embeds, + past_key_values=past_key_values, + encoder_hidden_states=hidden_states, + encoder_attention_mask=attention_mask, + head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = decoder_outputs[0] + + if self.config.tie_word_embeddings: + # Rescale output before projecting on vocab + # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/transformer.py#L586 + sequence_output = sequence_output * (self.model_dim**-0.5) + + lm_logits = self.lm_head(sequence_output) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-100) + # move labels to correct device to enable PP + labels = labels.to(lm_logits.device) + loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1)) + # TODO(thom): Add z_loss https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666 + + if not return_dict: + output = (lm_logits,) + decoder_outputs[1:] + encoder_outputs + return ((loss,) + output) if loss is not None else output + + return Seq2SeqLMOutput( + loss=loss, + logits=lm_logits, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + head_mask=None, + decoder_head_mask=None, + decoder_attention_mask=None, + cross_attn_head_mask=None, + use_cache=None, + encoder_outputs=None, + **kwargs, + ): + # cut decoder_input_ids if past is used + if past_key_values is not None: + input_ids = input_ids[:, -1:] + + return { + "decoder_input_ids": input_ids, + "past_key_values": past_key_values, + "encoder_outputs": encoder_outputs, + "attention_mask": attention_mask, + "head_mask": head_mask, + "decoder_head_mask": decoder_head_mask, + "decoder_attention_mask": decoder_attention_mask, + "cross_attn_head_mask": cross_attn_head_mask, + "use_cache": use_cache, + } + + def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor): + return self._shift_right(labels) + + def _reorder_cache(self, past_key_values, beam_idx): + # if decoder past is not included in output + # speedy decoding is disabled and no need to reorder + if past_key_values is None: + logger.warning( + "You might want to consider setting `use_cache=True` to speed up decoding" + ) + return past_key_values + + reordered_decoder_past = () + for layer_past_states in past_key_values: + # get the correct batch idx from layer past batch dim + # batch dim of `past` is at 2nd position + reordered_layer_past_states = () + for layer_past_state in layer_past_states: + # need to set correct `past` for each of the four key / value states + reordered_layer_past_states = reordered_layer_past_states + ( + layer_past_state.index_select( + 0, beam_idx.to(layer_past_state.device) + ), + ) + + assert reordered_layer_past_states[0].shape == layer_past_states[0].shape + assert len(reordered_layer_past_states) == len(layer_past_states) + + reordered_decoder_past = reordered_decoder_past + ( + reordered_layer_past_states, + ) + return reordered_decoder_past diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index fe28580d..eb216a20 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -1,154 +1,25 @@ import torch import torch.distributed -from accelerate import init_empty_weights from opentelemetry import trace -from pathlib import Path -from safetensors import safe_open from transformers import AutoConfig from transformers.models.llama import LlamaTokenizer -from typing import Optional, List +from typing import Optional from text_generation_server.models import FlashCausalLM from text_generation_server.models.custom_modeling.flash_llama_modeling import ( FlashLlamaForCausalLM, - TensorParallelEmbedding, - TensorParallelRowLinear, - TensorParallelColumnLinear, ) from text_generation_server.utils import ( initialize_torch_distributed, weight_files, - download_weights, - weight_hub_files, - LocalEntryNotFoundError, + Weights, ) tracer = trace.get_tracer(__name__) class FlashLlama(FlashCausalLM): - def __init__( - self, - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - trust_remote_code: bool = False, - ): - if torch.cuda.is_available(): - device = torch.device("cuda") - dtype = torch.float16 - else: - raise NotImplementedError("FlashLlama is only available on GPU") - - tokenizer = LlamaTokenizer.from_pretrained( - model_id, - revision=revision, - padding_side="left", - truncation_side="left", - trust_remote_code=trust_remote_code, - ) - - config = AutoConfig.from_pretrained( - model_id, revision=revision, trust_remote_code=trust_remote_code - ) - - # We do not use from_pretrained as we modified the model internal module layout - try: - filenames = weight_files(model_id, revision, ".bin") - # Local files not found - except LocalEntryNotFoundError: - hub_files = weight_hub_files(model_id, revision, ".bin") - filenames = download_weights(hub_files, model_id, revision) - - with init_empty_weights(): - model = FlashLlamaForCausalLM(config) - - self.load_weights(model, filenames, quantize, device, dtype) - - super(FlashCausalLM, self).__init__( - model=model.to(device), - tokenizer=tokenizer, - requires_padding=False, - dtype=dtype, - device=device, - ) - - @staticmethod - def load_weights( - model, - filenames: List[Path], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - ): - for filename in filenames: - state_dict = torch.load(filename, map_location="cpu") - for key, value in state_dict.items(): - value = value.to(device if quantize is None else "cpu").to(dtype) - - layer_name = ".".join(key.split(".")[:4]) - - # Fused qkv - if "q_proj" in key or "k_proj" in key or "v_proj" in key: - final_key = layer_name + ".query_key_value.weight" - - # Fused gate and up projs - elif "gate_proj" in key or "up_proj" in key: - final_key = layer_name + ".gate_up_proj.weight" - else: - final_key = key - - module_name, param_name = final_key.rsplit(".", 1) - module = model.get_submodule(module_name) - - try: - current_parameter_tensor = module._parameters[param_name] - except KeyError: - current_parameter_tensor = None - - if current_parameter_tensor is not None: - if current_parameter_tensor.device == torch.device("meta"): - # Init qkv - if "query_key_value" in final_key: - module._parameters[param_name] = value.new_empty( - (value.shape[0] * 3, value.shape[1]) - ) - # Init gate and up proj - elif "gate_up_proj" in final_key: - module._parameters[param_name] = value.new_empty( - (value.shape[0] * 2, value.shape[1]) - ) - - # Copy to correct slice - if "q_proj" in key: - module._parameters[param_name][: value.shape[0]] = value - elif "k_proj" in key: - module._parameters[param_name][ - value.shape[0] : value.shape[0] * 2 - ] = value - elif "v_proj" in key: - module._parameters[param_name][value.shape[0] * 2 :] = value - elif "gate_proj" in key: - module._parameters[param_name][: value.shape[0]] = value - elif "up_proj" in key: - module._parameters[param_name][value.shape[0] :] = value - else: - if current_parameter_tensor.shape != value.shape: - raise ValueError( - f"Name {final_key} -- Current {current_parameter_tensor.shape} and got {value.shape}" - ) - module._parameters[param_name] = value - else: - module._buffers[param_name] = value - - del value - - torch.cuda.empty_cache() - model.post_load_weights(quantize) - - -class FlashLlamaSharded(FlashLlama): def __init__( self, model_id: str, @@ -176,24 +47,16 @@ def __init__( ) torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights(filenames, device, dtype, process_group=self.process_group) - with init_empty_weights(): - model = FlashLlamaForCausalLM(config, process_group=self.process_group) + config.quantize = quantize + model = FlashLlamaForCausalLM(config, weights) - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=device, - dtype=dtype, - rank=rank, - world_size=world_size, - ) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( - model=model.to(device), + model=model, tokenizer=tokenizer, requires_padding=False, dtype=dtype, @@ -201,114 +64,3 @@ def __init__( rank=rank, world_size=world_size, ) - - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - rank: int, - world_size: int, - ): - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if quantize is None else "cpu" - ) as f: - for name in f.keys(): - slice_ = f.get_slice(name) - - layer_name = ".".join(name.split(".")[:4]) - - # Fused qkv - if "q_proj" in name or "k_proj" in name or "v_proj" in name: - final_name = layer_name + ".query_key_value.weight" - - # Fused gate and up projs - elif "gate_proj" in name or "up_proj" in name: - final_name = layer_name + ".gate_up_proj.weight" - else: - final_name = name - - module_name, param_name = final_name.rsplit(".", 1) - module = model.get_submodule(module_name) - - if isinstance(module, TensorParallelColumnLinear): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif isinstance(module, TensorParallelRowLinear): - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - elif isinstance(module, TensorParallelEmbedding): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif name == "lm_head.weight" and model.model.tp_embeddings: - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - try: - tensor = slice_[:] - except: - tensor = f.get_tensor(name) - - tensor = tensor.contiguous().to(dtype) - - try: - current_parameter_tensor = module._parameters[param_name] - except KeyError: - current_parameter_tensor = None - - if current_parameter_tensor is not None: - if current_parameter_tensor.device == torch.device("meta"): - # Init qkv - if "query_key_value" in final_name: - module._parameters[param_name] = tensor.new_empty( - (tensor.shape[0] * 3, tensor.shape[1]) - ) - # Init gate and up proj - elif "gate_up_proj" in final_name: - module._parameters[param_name] = tensor.new_empty( - (tensor.shape[0] * 2, tensor.shape[1]) - ) - - # Init gate and up proj - if "q_proj" in name: - module._parameters[param_name][: tensor.shape[0]] = tensor - elif "k_proj" in name: - module._parameters[param_name][ - tensor.shape[0] : tensor.shape[0] * 2 - ] = tensor - elif "v_proj" in name: - module._parameters[param_name][ - tensor.shape[0] * 2 : - ] = tensor - elif "gate_proj" in name: - module._parameters[param_name][: tensor.shape[0]] = tensor - elif "up_proj" in name: - module._parameters[param_name][tensor.shape[0] :] = tensor - else: - if current_parameter_tensor.shape != tensor.shape: - raise ValueError( - f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" - ) - - module._parameters[param_name] = tensor - - else: - module._buffers[param_name] = tensor - - torch.cuda.empty_cache() - model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 31ae7914..4847571d 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -1,45 +1,24 @@ import torch import torch.distributed -from accelerate import init_empty_weights from opentelemetry import trace -from safetensors import safe_open from transformers import AutoTokenizer, AutoConfig -from typing import Optional, List +from typing import Optional from text_generation_server.models import FlashCausalLM from text_generation_server.models.custom_modeling.flash_neox_modeling import ( FlashGPTNeoXForCausalLM, - TensorParallelEmbedding, - TensorParallelRowLinear, - TensorParallelColumnLinear, ) from text_generation_server.utils import ( initialize_torch_distributed, weight_files, + Weights, ) tracer = trace.get_tracer(__name__) -class FlashNeoX(FlashCausalLM): - def __init__( - self, - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - trust_remote_code: bool = False, - ): - super(FlashNeoX, self).__init__( - FlashGPTNeoXForCausalLM, - model_id, - revision, - quantize, - trust_remote_code=trust_remote_code, - ) - - -class FlashNeoXSharded(FlashNeoX): +class FlashNeoXSharded(FlashCausalLM): def __init__( self, model_id: str, @@ -65,23 +44,16 @@ def __init__( config = AutoConfig.from_pretrained( model_id, revision=revision, trust_remote_code=trust_remote_code ) + config.quantize = quantize torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights( + filenames, device=device, dtype=dtype, process_group=self.process_group + ) - with init_empty_weights(): - model = FlashGPTNeoXForCausalLM(config, self.process_group) + model = FlashGPTNeoXForCausalLM(config, weights) - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=device, - dtype=dtype, - rank=rank, - world_size=world_size, - ) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( model=model.to(device), @@ -92,79 +64,3 @@ def __init__( rank=rank, world_size=world_size, ) - - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - rank: int, - world_size: int, - ): - parameters = dict(model.named_parameters()) - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if quantize is None else "cpu" - ) as f: - for name in f.keys(): - module_name, param_name = name.rsplit(".", 1) - module = model.get_submodule(module_name) - - current_parameter_tensor = parameters.get(name, None) - - slice_ = f.get_slice(name) - - if isinstance(module, TensorParallelColumnLinear): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif isinstance(module, TensorParallelRowLinear): - if param_name == "weight": - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - else: - tensor = slice_[:] - # XXX: Hack for Rowlinear to add the bias only once. - if rank != 0: - tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif name == "embed_out.weight" and model.gpt_neox.tp_embeddings: - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - try: - tensor = slice_[:] - except: - tensor = f.get_tensor(name) - - if ( - current_parameter_tensor is not None - and current_parameter_tensor.shape != tensor.shape - ): - raise ValueError( - f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" - ) - - tensor = tensor.contiguous().to(dtype) - - if current_parameter_tensor is not None: - module._parameters[param_name] = tensor - else: - module._buffers[param_name] = tensor - - model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 4fc4c389..5f963bfb 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -1,119 +1,25 @@ import torch import torch.distributed -from pathlib import Path -from accelerate import init_empty_weights from opentelemetry import trace -from safetensors import safe_open -from transformers import AutoTokenizer, AutoConfig -from typing import Optional, List +from transformers import AutoTokenizer +from typing import Optional from text_generation_server.models import FlashCausalLM from text_generation_server.models.custom_modeling.flash_rw_modeling import ( RWConfig, FlashRWForCausalLM, - TensorParallelEmbedding, - TensorParallelRowLinear, - TensorParallelColumnLinear, ) from text_generation_server.utils import ( initialize_torch_distributed, weight_files, - download_weights, - weight_hub_files, - LocalEntryNotFoundError, + Weights, ) tracer = trace.get_tracer(__name__) -class FlashRW(FlashCausalLM): - def __init__( - self, - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - trust_remote_code: bool = False, - ): - if torch.cuda.is_available(): - device = torch.device("cuda") - dtype = torch.float16 - else: - raise NotImplementedError("RW is only available on GPU") - - tokenizer = AutoTokenizer.from_pretrained( - model_id, - revision=revision, - padding_side="left", - truncation_side="left", - trust_remote_code=trust_remote_code, - ) - - config = RWConfig.from_pretrained( - model_id, - revision=revision, - ) - - # We do not use from_pretrained as it is too slow - try: - filenames = weight_files(model_id, revision, ".bin") - # Local files not found - except LocalEntryNotFoundError: - hub_files = weight_hub_files(model_id, revision, ".bin") - filenames = download_weights(hub_files, model_id, revision) - - with init_empty_weights(): - model = FlashRWForCausalLM(config) - - self.load_weights( - model, - filenames, - quantize, - device, - dtype, - ) - - super(FlashCausalLM, self).__init__( - model=model.to(device), - tokenizer=tokenizer, - requires_padding=False, - dtype=dtype, - device=device, - ) - - @staticmethod - def load_weights( - model: FlashRWForCausalLM, - filenames: List[Path], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - ): - for filename in filenames: - state_dict = torch.load(filename, map_location="cpu") - for key, value in state_dict.items(): - value = value.to(device if quantize is None else "cpu").to(dtype) - - module_name, param_name = key.rsplit(".", 1) - module = model.get_submodule(module_name) - - try: - current_parameter_tensor = module._parameters[param_name] - if current_parameter_tensor.shape != value.shape: - raise ValueError( - f"Name {key} -- Current {current_parameter_tensor.shape} and got {value.shape}" - ) - module._parameters[param_name] = value - except KeyError: - module._buffers[param_name] = value - - del value - - torch.cuda.empty_cache() - model.post_load_weights(quantize) - - -class FlashRWSharded(FlashRW): +class FlashRWSharded(FlashCausalLM): def __init__( self, model_id: str, @@ -142,20 +48,12 @@ def __init__( torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights(filenames, device, dtype, process_group=self.process_group) - with init_empty_weights(): - model = FlashRWForCausalLM(config, self.process_group) + config.quantize = quantize + + model = FlashRWForCausalLM(config, weights) - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=device, - dtype=dtype, - rank=rank, - world_size=world_size, - ) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( model=model.to(device), @@ -166,79 +64,3 @@ def __init__( rank=rank, world_size=world_size, ) - - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - rank: int, - world_size: int, - ): - parameters = dict(model.named_parameters()) - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if quantize is None else "cpu" - ) as f: - for name in f.keys(): - module_name, param_name = name.rsplit(".", 1) - module = model.get_submodule(module_name) - - current_parameter_tensor = parameters.get(name, None) - - slice_ = f.get_slice(name) - - if isinstance(module, TensorParallelColumnLinear): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif isinstance(module, TensorParallelRowLinear): - if param_name == "weight": - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - else: - tensor = slice_[:] - # XXX: Hack for Rowlinear to add the bias only once. - if rank != 0: - tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif name == "lm_head.weight" and model.transformer.tp_embeddings: - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - try: - tensor = slice_[:] - except: - tensor = f.get_tensor(name) - - if ( - current_parameter_tensor is not None - and current_parameter_tensor.shape != tensor.shape - ): - raise ValueError( - f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" - ) - - tensor = tensor.contiguous().to(dtype) - - if current_parameter_tensor is not None: - module._parameters[param_name] = tensor - else: - module._buffers[param_name] = tensor - - model.post_load_weights(quantize) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index e1c893d0..54634e4a 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -1,197 +1,24 @@ import torch import torch.distributed -from accelerate import init_empty_weights from opentelemetry import trace -from safetensors import safe_open -from pathlib import Path -from transformers import AutoTokenizer, GPT2Config +from transformers import AutoTokenizer, AutoConfig from typing import Optional, List from text_generation_server.models import FlashCausalLM from text_generation_server.models.custom_modeling.flash_santacoder_modeling import ( FlashSantacoderForCausalLM, - TensorParallelRowLinear, - TensorParallelColumnLinear, - TensorParallelEmbedding, ) from text_generation_server.utils import ( initialize_torch_distributed, weight_files, - download_weights, - weight_hub_files, - LocalEntryNotFoundError, + Weights, ) tracer = trace.get_tracer(__name__) -class FlashSantacoder(FlashCausalLM): - def __init__( - self, - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - trust_remote_code: bool = False, - ): - if torch.cuda.is_available(): - device = torch.device("cuda") - dtype = torch.float16 - else: - raise NotImplementedError("FlashSantacoder is only available on GPU") - - tokenizer = AutoTokenizer.from_pretrained( - model_id, - revision=revision, - padding_side="left", - truncation_side="left", - trust_remote_code=trust_remote_code, - ) - - config = GPT2Config.from_pretrained( - model_id, - revision=revision, - ) - - # We do not use from_pretrained as we modified the model internal module layout - filenames = weight_files(model_id, revision, ".safetensors") - - with init_empty_weights(): - model = FlashSantacoderForCausalLM(config) - - self.load_weights( - model, - filenames, - quantize, - device, - dtype, - config.architectures[0].startswith("GPT2"), - ) - - super(FlashCausalLM, self).__init__( - model=model.to(device), - tokenizer=tokenizer, - requires_padding=False, - dtype=dtype, - device=device, - ) - - @staticmethod - def load_weights( - model: FlashSantacoderForCausalLM, - filenames: List[Path], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - transpose: bool, - ): - for filename in filenames: - with safe_open( - filename, - framework="pt", - device=str(device) if quantize is None else "cpu", - ) as f: - for key in f.keys(): - value = f.get_tensor(key) - value = value.to(device if quantize is None else "cpu").to(dtype) - - layer_name = ".".join(key.split(".")[:4]) - - # Fused qkv - if "q_attn.weight" in key or "kv_attn.weight" in key: - final_key = layer_name + ".c_attn.weight" - elif "q_attn.bias" in key or "kv_attn.bias" in key: - final_key = layer_name + ".c_attn.bias" - - else: - final_key = key - - module_name, param_name = final_key.rsplit(".", 1) - module = model.get_submodule(module_name) - - try: - current_parameter_tensor = module._parameters[param_name] - except KeyError: - current_parameter_tensor = None - - if current_parameter_tensor is not None: - if transpose and ( - "c_fc.weight" in key - or "c_proj.weight" in key - or "q_attn.weight" in key - or "kv_attn.weight" in key - or "c_attn.weight" in key - ): - # Tranpose as we use nn.Linear instead of Conv1D - value = value.T - - if current_parameter_tensor.device == torch.device("meta"): - # Init qkv - if "c_attn.weight" in final_key: - module._parameters[param_name] = value.new_empty( - ( - model.transformer.head_size - * (model.transformer.num_heads + 2), - value.shape[1], - ) - ) - elif "c_attn.bias" in final_key: - module._parameters[param_name] = value.new_empty( - ( - model.transformer.head_size - * (model.transformer.num_heads + 2) - ) - ) - - # Copy to correct slice - if "q_attn.weight" in key: - module._parameters[param_name][: value.shape[0]] = value - elif "q_attn.bias" in key: - module._parameters[param_name][: value.shape[0]] = value - elif "kv_attn.weight" in key: - module._parameters[param_name][ - model.transformer.head_size - * model.transformer.num_heads : - ] = value - elif "kv_attn.bias" in key: - module._parameters[param_name][ - model.transformer.head_size - * model.transformer.num_heads : - ] = value - else: - if current_parameter_tensor.shape != value.shape: - raise ValueError( - f"Name {final_key} -- Current {current_parameter_tensor.shape} and got {value.shape}" - ) - module._parameters[param_name] = value - else: - module._buffers[param_name] = value - - del value - - if model.lm_head.weight.device == torch.device("meta"): - model.lm_head.weight = torch.nn.Parameter(model.transformer.wte.weight) - - torch.cuda.empty_cache() - model.post_load_weights(quantize) - - uninitialized_parameters = [] - for n, p in model.named_parameters(): - if p.data.device == torch.device("meta"): - uninitialized_parameters.append(n) - if uninitialized_parameters: - raise RuntimeError( - f"found uninitialized parameters in model : {uninitialized_parameters}" - ) - - def decode(self, generated_ids: List[int]) -> str: - # Do not skip special tokens as they are used for custom parsing rules of the generated text - return self.tokenizer.decode( - generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False - ) - - -class FlashSantacoderSharded(FlashSantacoder): +class FlashSantacoderSharded(FlashCausalLM): def __init__( self, model_id: str, @@ -214,28 +41,22 @@ def __init__( trust_remote_code=trust_remote_code, ) - config = GPT2Config.from_pretrained( + config = AutoConfig.from_pretrained( model_id, revision=revision, + trust_remote_code=True, ) + config.quantize = quantize + config.transpose = config.architectures[0].startswith("GPT2") torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights( + filenames, device=device, dtype=dtype, process_group=self.process_group + ) - with init_empty_weights(): - model = FlashSantacoderForCausalLM(config, self.process_group) + model = FlashSantacoderForCausalLM(config, weights) - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=device, - dtype=dtype, - rank=rank, - world_size=world_size, - transpose=config.architectures[0].startswith("GPT2"), - ) torch.distributed.barrier(group=self.process_group) super(FlashCausalLM, self).__init__( model=model.to(device), @@ -247,164 +68,8 @@ def __init__( world_size=world_size, ) - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - rank: int, - world_size: int, - transpose: bool, - ): - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if quantize is None else "cpu" - ) as f: - for key in f.keys(): - slice_ = f.get_slice(key) - - layer_name = ".".join(key.split(".")[:4]) - - # Fused qkv - if "q_attn.weight" in key or "kv_attn.weight" in key: - final_key = layer_name + ".c_attn.weight" - elif "q_attn.bias" in key or "kv_attn.bias" in key: - final_key = layer_name + ".c_attn.bias" - else: - final_key = key - - module_name, param_name = final_key.rsplit(".", 1) - module = model.get_submodule(module_name) - - if isinstance(module, TensorParallelColumnLinear): - dim = 1 if transpose and "weight" in param_name else 0 - size = slice_.get_shape()[dim] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = ( - slice_[start:stop] if dim == 0 else slice_[:, start:stop] - ) - elif isinstance(module, TensorParallelRowLinear): - if param_name == "weight": - dim = 0 if transpose else 1 - size = slice_.get_shape()[dim] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = ( - slice_[start:stop] - if dim == 0 - else slice_[:, start:stop] - ) - else: - tensor = slice_[:] - # XXX: Hack for Rowlinear to add the bias only once. - if rank != 0: - tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif key == "lm_head.weight" and model.transformer.tp_embeddings: - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - try: - tensor = slice_[:] - except: - tensor = f.get_tensor(key) - - tensor = tensor.contiguous().to(dtype) - - try: - current_parameter_tensor = module._parameters[param_name] - except KeyError: - current_parameter_tensor = None - - if current_parameter_tensor is not None: - if transpose and ( - "c_fc.weight" in key - or "c_proj.weight" in key - or "q_attn.weight" in key - or "kv_attn.weight" in key - or "c_attn.weight" in key - ): - # Tranpose as we use nn.Linear instead of Conv1D - tensor = tensor.T - - if current_parameter_tensor.device == torch.device("meta"): - # Init qkv - if "c_attn.weight" in final_key: - module._parameters[param_name] = tensor.new_empty( - ( - model.transformer.head_size - * (model.transformer.num_heads + 2), - tensor.shape[1], - ) - ) - elif "c_attn.bias" in final_key: - module._parameters[param_name] = tensor.new_empty( - ( - model.transformer.head_size - * (model.transformer.num_heads + 2) - ) - ) - - # Copy to correct slice - if "q_attn" in key: - size = tensor.shape[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = tensor[start:stop] - module._parameters[param_name][: tensor.shape[0]] = tensor - elif "kv_attn.weight" in key: - module._parameters[param_name][ - model.transformer.head_size - * model.transformer.num_heads : - ] = tensor - elif "kv_attn.bias" in key: - module._parameters[param_name][ - model.transformer.head_size - * model.transformer.num_heads : - ] = tensor - elif "c_attn" in key: - # Slice q_tensor by shard - q_tensor = tensor[: -2 * model.transformer.head_size] - block_size = q_tensor.shape[0] // world_size - start = rank * block_size - stop = (rank + 1) * block_size - q_tensor = q_tensor[start:stop] - - module._parameters[param_name][ - : q_tensor.shape[0] - ] = q_tensor - - # Kv tensor is copied for every shard - kv_tensor = tensor[-2 * model.transformer.head_size :] - module._parameters[param_name][ - q_tensor.shape[0] : - ] = kv_tensor - else: - if current_parameter_tensor.shape != tensor.shape: - raise ValueError( - f"Name {key} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" - ) - - module._parameters[param_name] = tensor - else: - module._buffers[param_name] = tensor - - if model.lm_head.weight.device == torch.device("meta"): - model.lm_head.weight = torch.nn.Parameter(model.transformer.wte.weight) - - torch.cuda.empty_cache() - model.post_load_weights(quantize) + def decode(self, generated_ids: List[int]) -> str: + # Do not skip special tokens as they are used for custom parsing rules of the generated text + return self.tokenizer.decode( + generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False + ) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 37ccc398..01e1c773 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -2,41 +2,25 @@ import torch import torch.distributed -from typing import List, Optional, Type, Tuple +from typing import List, Optional, Type -from accelerate import init_empty_weights -from safetensors import safe_open from transformers import ( AutoTokenizer, - AutoModelForCausalLM, AutoConfig, PreTrainedTokenizerBase, ) -from transformers.models.opt.parallel_layers import ( - TensorParallelColumnLinear, - TensorParallelEmbedding, - TensorParallelRowLinear, -) - from text_generation_server.models import CausalLM from text_generation_server.models.causal_lm import CausalLMBatch from text_generation_server.pb import generate_pb2 -from text_generation_server.models.opt import OPT +from text_generation_server.models.custom_modeling.opt_modeling import OPTForCausalLM from text_generation_server.utils import ( NextTokenChooser, StoppingCriteria, initialize_torch_distributed, weight_files, + Weights, ) -HAS_BITS_AND_BYTES = True -try: - import bitsandbytes as bnb - from bitsandbytes.nn import Int8Params -except Exception as e: - HAS_BITS_AND_BYTES = False - - # CREDIT: Papers with code => https://github.com/paperswithcode/galai/blob/main/galai/utils.py # we split individual characters inside special tokens like [START_DNA] @@ -168,33 +152,7 @@ def from_pb( ) -class Galactica(OPT): - @property - def batch_type(self) -> Type[CausalLMBatch]: - return GalacticaCausalLMBatch - - def decode(self, generated_ids: List[int]) -> str: - # Do not skip special tokens as they are used for custom parsing rules of the generated text - return self.tokenizer.decode( - generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False - ) - - def forward( - self, input_ids, attention_mask, position_ids, past_key_values: Optional = None - ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: - """Overwrite forward to ignore position_ids""" - - # Model Forward - outputs = self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - use_cache=True, - ) - return outputs.logits, outputs.past_key_values - - -class GalacticaSharded(Galactica): +class GalacticaSharded(CausalLM): def __init__( self, model_id: str, @@ -224,26 +182,17 @@ def __init__( tp_parallel=True, trust_remote_code=trust_remote_code, ) + config.quantize = quantize tokenizer.pad_token_id = config.pad_token_id torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights( + filenames, device=device, dtype=dtype, process_group=self.process_group + ) - with init_empty_weights(): - model = AutoModelForCausalLM.from_config( - config, trust_remote_code=trust_remote_code - ) + model = OPTForCausalLM(config, weights) - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=device, - dtype=dtype, - rank=rank, - world_size=world_size, - ) torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( model=model, @@ -255,127 +204,15 @@ def __init__( world_size=world_size, ) - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - rank: int, - world_size: int, - ): - parameters = dict(model.named_parameters()) - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if quantize is None else "cpu" - ) as f: - for name in f.keys(): - if name == "lm_head.weight": - continue - - module_name, param_name = name.rsplit(".", 1) - module = model.get_submodule(module_name) - current_tensor = parameters[name] - - slice_ = f.get_slice(name) - - if isinstance(module, TensorParallelColumnLinear): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif isinstance(module, TensorParallelRowLinear): - if param_name == "weight": - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - else: - tensor = slice_[:] - # XXX: Hack for Rowlinear to add the bias only once. - if rank != 0: - tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - tensor = slice_[:] - - if current_tensor.shape != tensor.shape: - raise ValueError( - f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" - ) - - tensor = tensor.contiguous().to(dtype) - - if quantize == "bitsandbytes": - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine either because it is not installed " - "or you don't have a GPU.\n" - "You can install it with `pip install bitsandbytes`." - ) - - if ( - type(module) - in [TensorParallelRowLinear, TensorParallelColumnLinear] - and param_name == "weight" - ): - tensor = Int8Params( - tensor, - has_fp16_weights=False, - requires_grad=False, - ).to(device) - state = bnb.MatmulLtState() - state.threshold = 6.0 - state.has_fp16_weights = False - state.memory_efficient_backward = False - state.use_pool = True - state.CB = tensor.CB - state.SCB = tensor.SCB - tensor.CB = None - tensor.SCB = None - - def replace_linear(state): - def linear(input, weight, bias): - out = bnb.matmul( - input, - weight, - state=state, - threshold=state.threshold, - bias=bias, - ) - - if state.CB is not None: - # we converted 8-bit row major to turing/ampere format - # in the first inference pass - # we no longer need the row-major weight - del state.CB - weight.data = state.CxB - - return out - - return linear - - module.linear = replace_linear(state) - else: - tensor = tensor.to(device) - elif quantize == "gptq": - raise NotImplementedError("`gptq` is not implemented for now") - elif quantize is None: - tensor = tensor.to(device) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") + @property + def batch_type(self) -> Type[CausalLMBatch]: + return GalacticaCausalLMBatch - module._parameters[param_name] = tensor - if name == "model.decoder.embed_tokens.weight": - model.lm_head._parameters["weight"] = tensor + def decode(self, generated_ids: List[int]) -> str: + # Do not skip special tokens as they are used for custom parsing rules of the generated text + return self.tokenizer.decode( + generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False + ) def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None @@ -386,10 +223,4 @@ def forward( past_key_values=past_key_values, use_cache=True, ) - - # Logits are sharded, so we need to gather them - logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] - torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) - logits = torch.cat(logits, dim=2) - - return logits, outputs.past_key_values + return outputs.logits, outputs.past_key_values diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 5ab8a624..0abf0239 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -1,34 +1,22 @@ import torch import torch.distributed -from typing import List, Optional +from typing import Optional -from accelerate import init_empty_weights -from safetensors import safe_open from transformers import ( AutoTokenizer, - AutoModelForCausalLM, AutoConfig, ) -from transformers.models.gpt_neox.parallel_layers import ( - TensorParallelColumnLinear, - TensorParallelEmbedding, - TensorParallelRowLinear, -) - from text_generation_server.models import CausalLM +from text_generation_server.models.custom_modeling.neox_modeling import ( + GPTNeoxForCausalLM, +) from text_generation_server.utils import ( initialize_torch_distributed, weight_files, + Weights, ) -HAS_BITS_AND_BYTES = True -try: - import bitsandbytes as bnb - from bitsandbytes.nn import Int8Params -except Exception as e: - HAS_BITS_AND_BYTES = False - class GPTNeoxSharded(CausalLM): def __init__( @@ -58,28 +46,18 @@ def __init__( config = AutoConfig.from_pretrained( model_id, revision=revision, - tp_parallel=True, trust_remote_code=trust_remote_code, ) + config.quantize = quantize torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights( + filenames, device=device, dtype=dtype, process_group=self.process_group + ) - with init_empty_weights(): - model = AutoModelForCausalLM.from_config( - config, trust_remote_code=trust_remote_code - ) + model = GPTNeoxForCausalLM(config, weights) - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=device, - dtype=dtype, - rank=rank, - world_size=world_size, - ) torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( model=model, @@ -91,161 +69,16 @@ def __init__( world_size=world_size, ) - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - rank: int, - world_size: int, - ): - parameters = dict(model.named_parameters()) - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if quantize is None else "cpu" - ) as f: - for name in f.keys(): - module_name, param_name = name.rsplit(".", 1) - module = model.get_submodule(module_name) - - current_parameter_tensor = parameters.get(name, None) - - slice_ = f.get_slice(name) - - if isinstance(module, TensorParallelColumnLinear): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif isinstance(module, TensorParallelRowLinear): - if param_name == "weight": - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - else: - tensor = slice_[:] - # XXX: Hack for Rowlinear to add the bias only once. - if rank != 0: - tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif name == "embed_out.weight" and model.gpt_neox.tp_embeddings: - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - try: - tensor = slice_[:] - except: - tensor = f.get_tensor(name) - - if ( - current_parameter_tensor is not None - and current_parameter_tensor.shape != tensor.shape - ): - raise ValueError( - f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" - ) - - tensor = tensor.contiguous().to(dtype) - - if quantize == "bitsandbytes": - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine either because it is not installed " - "or you don't have a GPU.\n" - "You can install it with `pip install bitsandbytes`." - ) - - if ( - type(module) - in [TensorParallelRowLinear, TensorParallelColumnLinear] - and param_name == "weight" - ): - tensor = Int8Params( - tensor, - has_fp16_weights=False, - requires_grad=False, - ).to(device) - state = bnb.MatmulLtState() - state.threshold = 6.0 - state.has_fp16_weights = False - state.memory_efficient_backward = False - state.use_pool = True - state.CB = tensor.CB - state.SCB = tensor.SCB - tensor.CB = None - tensor.SCB = None - - def replace_linear(state): - def linear(input, weight, bias): - out = bnb.matmul( - input, - weight, - state=state, - threshold=state.threshold, - bias=bias, - ) - - if state.CB is not None: - # we converted 8-bit row major to turing/ampere format - # in the first inference pass - # we no longer need the row-major weight - del state.CB - weight.data = state.CxB - - return out - - return linear - - module.linear = replace_linear(state) - else: - tensor = tensor.to(device) - elif quantize == "gptq": - raise NotImplementedError("`gptq` is not implemented for now") - elif quantize is None: - tensor = tensor.to(device) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") - - if current_parameter_tensor is not None: - module._parameters[param_name] = tensor - else: - module._buffers[param_name] = tensor - def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): - if self.model.gpt_neox.tp_embeddings: - outputs = self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=True, - ) - - # Logits are sharded, so we need to gather them - logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] - torch.distributed.all_gather( - logits, outputs.logits, group=self.process_group - ) - logits = torch.cat(logits, dim=2) + outputs = self.model.forward( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=True, + ) - return logits, outputs.past_key_values - # While the model itself is sharded, the embeddings might not as they might not be dividable by num-shard - else: - return super(GPTNeoxSharded, self).forward( - input_ids, attention_mask, position_ids, past_key_values - ) + logits = outputs.logits + return logits, outputs.past_key_values diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 9cc4d5e1..16cb48b7 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -1,52 +1,22 @@ import torch import torch.distributed -from typing import List, Optional, Tuple +from typing import Optional -from accelerate import init_empty_weights -from safetensors import safe_open from transformers import ( AutoTokenizer, - AutoModelForCausalLM, AutoConfig, ) -from transformers.models.opt.parallel_layers import ( - TensorParallelColumnLinear, - TensorParallelEmbedding, - TensorParallelRowLinear, -) - +from text_generation_server.models.custom_modeling.opt_modeling import OPTForCausalLM from text_generation_server.models import CausalLM from text_generation_server.utils import ( initialize_torch_distributed, weight_files, + Weights, ) -HAS_BITS_AND_BYTES = True -try: - import bitsandbytes as bnb - from bitsandbytes.nn import Int8Params -except Exception as e: - HAS_BITS_AND_BYTES = False - - -class OPT(CausalLM): - def forward( - self, input_ids, attention_mask, position_ids, past_key_values: Optional = None - ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: - """Overwrite forward to ignore position_ids""" - - # Model Forward - outputs = self.model.forward( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - use_cache=True, - ) - return outputs.logits, outputs.past_key_values - -class OPTSharded(OPT): +class OPTSharded(CausalLM): def __init__( self, model_id: str, @@ -73,29 +43,19 @@ def __init__( config = AutoConfig.from_pretrained( model_id, revision=revision, - tp_parallel=True, trust_remote_code=trust_remote_code, ) + config.quantize = quantize tokenizer.pad_token_id = config.pad_token_id torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights( + filenames, device=device, dtype=dtype, process_group=self.process_group + ) - with init_empty_weights(): - model = AutoModelForCausalLM.from_config( - config, trust_remote_code=trust_remote_code - ) + model = OPTForCausalLM(config, weights) - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=device, - dtype=dtype, - rank=rank, - world_size=world_size, - ) torch.distributed.barrier(group=self.process_group) super(CausalLM, self).__init__( model=model, @@ -107,128 +67,6 @@ def __init__( world_size=world_size, ) - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - rank: int, - world_size: int, - ): - parameters = dict(model.named_parameters()) - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if quantize is None else "cpu" - ) as f: - for name in f.keys(): - if name == "lm_head.weight": - continue - - module_name, param_name = name.rsplit(".", 1) - module = model.get_submodule(module_name) - current_tensor = parameters[name] - - slice_ = f.get_slice(name) - - if isinstance(module, TensorParallelColumnLinear): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif isinstance(module, TensorParallelRowLinear): - if param_name == "weight": - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - else: - tensor = slice_[:] - # XXX: Hack for Rowlinear to add the bias only once. - if rank != 0: - tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - else: - tensor = slice_[:] - - if current_tensor.shape != tensor.shape: - raise ValueError( - f"Name {name} -- Current {current_tensor.shape} and got {tensor.shape}" - ) - - tensor = tensor.contiguous().to(dtype) - - if quantize == "bitsandbytes": - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine either because it is not installed " - "or you don't have a GPU.\n" - "You can install it with `pip install bitsandbytes`." - ) - - if ( - type(module) - in [TensorParallelRowLinear, TensorParallelColumnLinear] - and param_name == "weight" - ): - tensor = Int8Params( - tensor, - has_fp16_weights=False, - requires_grad=False, - ).to(device) - state = bnb.MatmulLtState() - state.threshold = 6.0 - state.has_fp16_weights = False - state.memory_efficient_backward = False - state.use_pool = True - state.CB = tensor.CB - state.SCB = tensor.SCB - tensor.CB = None - tensor.SCB = None - - def replace_linear(state): - def linear(input, weight, bias): - out = bnb.matmul( - input, - weight, - state=state, - threshold=state.threshold, - bias=bias, - ) - - if state.CB is not None: - # we converted 8-bit row major to turing/ampere format - # in the first inference pass - # we no longer need the row-major weight - del state.CB - weight.data = state.CxB - - return out - - return linear - - module.linear = replace_linear(state) - else: - tensor = tensor.to(device) - elif quantize == "gptq": - raise NotImplementedError("`gptq` is not implemented for now") - elif quantize is None: - tensor = tensor.to(device) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") - - module._parameters[param_name] = tensor - if name == "model.decoder.embed_tokens.weight": - model.lm_head._parameters["weight"] = tensor - def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ): @@ -239,9 +77,4 @@ def forward( use_cache=True, ) - # Logits are sharded, so we need to gather them - logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] - torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) - logits = torch.cat(logits, dim=2) - - return logits, outputs.past_key_values + return outputs.logits, outputs.past_key_values diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index d12b89d2..c89462fc 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -3,31 +3,20 @@ from typing import List, Optional, Tuple -from accelerate import init_empty_weights -from safetensors import safe_open from transformers import ( AutoTokenizer, - AutoModelForSeq2SeqLM, AutoConfig, ) from text_generation_server.models import Seq2SeqLM +from text_generation_server.models.custom_modeling.t5_modeling import ( + T5ForConditionalGeneration, +) from text_generation_server.utils import ( initialize_torch_distributed, weight_files, + Weights, ) -from transformers.models.t5.parallel_layers import ( - TensorParallelRowLinear, - TensorParallelColumnLinear, - TensorParallelEmbedding, -) - -HAS_BITS_AND_BYTES = True -try: - import bitsandbytes as bnb - from bitsandbytes.nn import Int8Params -except ImportError as e: - HAS_BITS_AND_BYTES = False class T5Sharded(Seq2SeqLM): @@ -46,40 +35,30 @@ def __init__( device = torch.device("cpu") dtype = torch.float32 - tokenizer = AutoTokenizer.from_pretrained( + config = AutoConfig.from_pretrained( model_id, revision=revision, - padding_side="left", - truncation_side="left", trust_remote_code=trust_remote_code, ) + config.quantize = quantize - config = AutoConfig.from_pretrained( + tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, - tp_parallel=True, + padding_side="left", + truncation_side="left", trust_remote_code=trust_remote_code, ) tokenizer.bos_token_id = config.decoder_start_token_id torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights( + filenames, device=device, dtype=dtype, process_group=self.process_group + ) - with init_empty_weights(): - model = AutoModelForSeq2SeqLM.from_config( - config, trust_remote_code=trust_remote_code - ) + model = T5ForConditionalGeneration(config, weights) - torch.distributed.barrier(group=self.process_group) - self.load_weights( - model, - filenames, - quantize=quantize, - device=device, - dtype=dtype, - rank=rank, - world_size=world_size, - ) torch.distributed.barrier(group=self.process_group) super(Seq2SeqLM, self).__init__( model=model, @@ -91,151 +70,6 @@ def __init__( world_size=world_size, ) - @staticmethod - def load_weights( - model, - filenames: List[str], - quantize: Optional[str], - device: torch.device, - dtype: torch.dtype, - rank: int, - world_size: int, - ): - parameters = dict(model.named_parameters()) - for file in filenames: - with safe_open( - file, framework="pt", device=str(device) if quantize is None else "cpu" - ) as f: - for name in f.keys(): - module_name, param_name = name.rsplit(".", 1) - module = model.get_submodule(module_name) - - current_parameter_tensor = parameters.get(name, None) - - slice_ = f.get_slice(name) - - if isinstance(module, TensorParallelColumnLinear): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif isinstance(module, TensorParallelRowLinear): - if param_name == "weight": - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - else: - tensor = slice_[:] - # XXX: Hack for Rowlinear to add the bias only once. - if rank != 0: - tensor = torch.zeros_like(tensor) - elif isinstance(module, TensorParallelEmbedding): - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif name == "lm_head.weight": - size = slice_.get_shape()[0] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[start:stop] - elif "relative_attention_bias.weight" in name: - size = slice_.get_shape()[1] - block_size = size // world_size - start = rank * block_size - stop = (rank + 1) * block_size - tensor = slice_[:, start:stop] - else: - try: - tensor = slice_[:] - except: - tensor = f.get_tensor(name) - - if ( - current_parameter_tensor is not None - and current_parameter_tensor.shape != tensor.shape - ): - raise ValueError( - f"Name {name} -- Current {current_parameter_tensor.shape} and got {tensor.shape}" - ) - - tensor = tensor.contiguous() - - # See: https://github.com/huggingface/transformers/blob/1fe1e3caa44617047f149bcc0c0b566343b714a7/src/transformers/models/t5/modeling_t5.py#LL316C15-L316C71 - if module_name.endswith("wo"): - tensor = tensor.to(torch.float32) - else: - tensor = tensor.to(dtype) - - if quantize == "bitsandbytes" and not module_name.endswith("wo"): - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine either because it is not installed " - "or you don't have a GPU.\n" - "You can install it with `pip install bitsandbytes`." - ) - - if ( - type(module) - in [TensorParallelRowLinear, TensorParallelColumnLinear] - and param_name == "weight" - ): - tensor = Int8Params( - tensor, - has_fp16_weights=False, - requires_grad=False, - ).to(device) - state = bnb.MatmulLtState() - state.threshold = 6.0 - state.has_fp16_weights = False - state.memory_efficient_backward = False - state.use_pool = True - state.CB = tensor.CB - state.SCB = tensor.SCB - tensor.CB = None - tensor.SCB = None - - def replace_linear(state): - def linear(input, weight, bias): - out = bnb.matmul( - input, - weight, - state=state, - threshold=state.threshold, - bias=bias, - ) - - if state.CB is not None: - # we converted 8-bit row major to turing/ampere format - # in the first inference pass - # we no longer need the row-major weight - del state.CB - weight.data = state.CxB - - return out - - return linear - - module.linear = replace_linear(state) - else: - tensor = tensor.to(device) - elif quantize == "gptq" and not module_name.endswith("wo"): - raise NotImplementedError("`gptq` is not implemented for now") - elif quantize is None or module_name.endswith("wo"): - tensor = tensor.to(device) - else: - raise ValueError(f"Unexpected quantize `{quantize}`") - - if current_parameter_tensor is not None: - module._parameters[param_name] = tensor - else: - module._buffers[param_name] = tensor - def forward( self, input_ids, @@ -260,13 +94,8 @@ def forward( use_cache=True, ) - # Logits are sharded, so we need to gather them - logits = [torch.empty_like(outputs.logits) for _ in range(self.world_size)] - torch.distributed.all_gather(logits, outputs.logits, group=self.process_group) - logits = torch.cat(logits, dim=2) - return ( - logits, + outputs.logits, outputs.encoder_last_hidden_state, outputs.past_key_values, ) diff --git a/server/text_generation_server/utils/__init__.py b/server/text_generation_server/utils/__init__.py index 6a351d66..befedcf0 100644 --- a/server/text_generation_server/utils/__init__.py +++ b/server/text_generation_server/utils/__init__.py @@ -1,5 +1,6 @@ from text_generation_server.utils.convert import convert_file, convert_files from text_generation_server.utils.dist import initialize_torch_distributed +from text_generation_server.utils.weights import Weights from text_generation_server.utils.hub import ( weight_files, weight_hub_files, @@ -35,4 +36,5 @@ "StoppingCriteria", "StopSequenceCriteria", "FinishReason", + "Weights", ] diff --git a/server/text_generation_server/utils/dist.py b/server/text_generation_server/utils/dist.py index 9785493e..fe9c3b7b 100644 --- a/server/text_generation_server/utils/dist.py +++ b/server/text_generation_server/utils/dist.py @@ -4,6 +4,37 @@ from datetime import timedelta +class FakeBarrier: + def wait(self): + pass + + +class FakeGroup: + def __init__(self, rank, size): + self._rank = rank + self._size = size + + def allreduce(self, *args, **kwargs): + return FakeBarrier() + + def allgather(self, inputs, local_tensor, **kwargs): + assert ( + len(inputs[0]) == len(local_tensor) == 1 + ), f"{len(inputs[0])} != {len(local_tensor)} != 1, and the FakeGroup is supposed to join on simple tensors" + for input_ in inputs: + input_[0].data = local_tensor[0].data + return FakeBarrier() + + def barrier(self, *args, **kwargs): + return FakeBarrier() + + def size(self): + return self._size + + def rank(self): + return self._rank + + def initialize_torch_distributed(): rank = int(os.getenv("RANK", "0")) world_size = int(os.getenv("WORLD_SIZE", "1")) @@ -23,13 +54,18 @@ def initialize_torch_distributed(): backend = "gloo" options = None - # Call the init process. - torch.distributed.init_process_group( - backend=backend, - world_size=world_size, - rank=rank, - timeout=timedelta(seconds=60), - pg_options=options, - ) + if world_size == 1: + return FakeGroup(rank, world_size), rank, world_size + else: + if os.getenv("DEBUG", None) == "1": + return FakeGroup(rank, world_size), rank, world_size + # Call the init process. + torch.distributed.init_process_group( + backend=backend, + world_size=world_size, + rank=rank, + timeout=timedelta(seconds=60), + pg_options=options, + ) - return torch.distributed.group.WORLD, rank, world_size + return torch.distributed.group.WORLD, rank, world_size diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 127f9ba4..ee32a0dc 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -1,176 +1,240 @@ import torch +import torch.distributed from torch import nn from torch.nn import functional as F -from typing import Optional +from typing import List HAS_BITS_AND_BYTES = True try: - from bitsandbytes.nn import Linear8bitLt -except ImportError as e: + import bitsandbytes as bnb + from bitsandbytes.nn import Int8Params + +except ImportError: HAS_BITS_AND_BYTES = False +from accelerate import init_empty_weights + + +# Monkey patching +@classmethod +def load_layer_norm(cls, prefix, weights, eps): + weight = weights.get_tensor(f"{prefix}.weight") + bias = weights.get_tensor(f"{prefix}.bias") + with init_empty_weights(): + ln = cls(weight.shape, eps=eps) + + ln.weight = nn.Parameter(weight) + ln.bias = nn.Parameter(bias) + return ln + + +torch.nn.LayerNorm.load = load_layer_norm -class FastLinear(nn.Linear): + +class FastLinear(nn.Module): def __init__( self, - in_features: int, - out_features: int, - bias: bool = True, - device=None, - dtype=None, + weight, + bias, ) -> None: - super(FastLinear, self).__init__(in_features, out_features, bias, device, dtype) - self.quantized = False - self.bnb_linear = None - - def prepare_weights(self, quantize: Optional[str] = None): - if quantize == "bitsandbytes": - if not HAS_BITS_AND_BYTES: - raise ImportError( - "bitsandbytes is not available on your machine either because it is not installed " - "or you don't have a GPU.\n" - "You can install it with `pip install bitsandbytes`." - ) - - self.quantized = True - self.bnb_linear = Linear8bitLt( - self.in_features, - self.out_features, - has_fp16_weights=False, - threshold=6.0, - bias=False, - ) - # Copy data to bnb_linear - self.bnb_linear.weight.data = self.weight.data - if self.bias is not None: - self.bnb_linear.bias = nn.Parameter(self.bias) - - # Delete reference to data - self.weight = None + super().__init__() + self.weight = nn.Parameter(weight) + if bias is not None: + self.bias = nn.Parameter(bias) + else: self.bias = None - elif quantize == "gptq": - raise NotImplementedError("`gptq` is not implemented for now") - elif quantize is None: - self.weight = nn.Parameter(self.weight.T) + + @classmethod + def load(cls, config, prefix: str, weights, bias: bool): + weight = weights.get_tensor(f"{prefix}.weight") + if bias: + bias = weights.get_tensor(f"{prefix}.bias") else: - raise ValueError(f"Unexpected quantize `{quantize}`") + bias = None + return cls(weight, bias) def forward(self, input: torch.Tensor) -> torch.Tensor: - if self.quantized: - return self.bnb_linear(input) - else: - if self.bias is not None: - return torch.addmm(self.bias, input, self.weight) - return torch.matmul(input, self.weight) + return F.linear(input, self.weight, self.bias) -class TensorParallelColumnLinear(FastLinear): +class Linear8bitLt(nn.Module): def __init__( self, - in_features, - out_features, - process_group: torch.distributed.ProcessGroup, - bias=True, - device=None, - dtype=None, + weight, + bias, + has_fp16_weights=True, + memory_efficient_backward=False, + threshold=0.0, + index=None, ): - self.process_group = process_group - self.tp_world_size = process_group.size() - assert out_features % self.tp_world_size == 0 - out_features = out_features // self.tp_world_size - - super().__init__( - in_features=in_features, - out_features=out_features, - bias=bias, - device=device, - dtype=dtype, + super().__init__() + assert ( + not memory_efficient_backward + ), "memory_efficient_backward is no longer required and the argument is deprecated in 0.37.0 and will be removed in 0.39.0" + self.state = bnb.MatmulLtState() + self.index = index + + # Necessary for stacked layers + self.state.threshold = threshold + self.state.has_fp16_weights = has_fp16_weights + self.state.memory_efficient_backward = memory_efficient_backward + if threshold > 0.0 and not has_fp16_weights: + self.state.use_pool = True + + self.weight = Int8Params( + weight.data, + has_fp16_weights=has_fp16_weights, + requires_grad=has_fp16_weights, ) + self.weight.cuda(weight.device) + self.bias = bias + + def init_8bit_state(self): + self.state.CB = self.weight.CB + self.state.SCB = self.weight.SCB + self.weight.CB = None + self.weight.SCB = None + + def forward(self, x: torch.Tensor): + self.state.is_training = self.training + if self.weight.CB is not None: + self.init_8bit_state() + + # weights are cast automatically as Int8Params, but the bias has to be cast manually + if self.bias is not None and self.bias.dtype != x.dtype: + self.bias.data = self.bias.data.to(x.dtype) + + out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state) + + if not self.state.has_fp16_weights: + if self.state.CB is not None and self.state.CxB is not None: + # we converted 8-bit row major to turing/ampere format in the first inference pass + # we no longer need the row-major weight + del self.state.CB + self.weight.data = self.state.CxB + return out -class TensorParallelRowLinear(FastLinear): - def __init__( - self, - in_features, - out_features, - process_group: torch.distributed.ProcessGroup, - reduce=True, - bias=True, - device=None, - dtype=None, - ): +def get_linear(weight, bias, quantize): + if quantize is None: + linear = FastLinear(weight, bias) + elif quantize == "bitsandbytes": + linear = Linear8bitLt( + weight, + bias, + has_fp16_weights=False, + threshold=6.0, + ) + if bias is not None: + linear.bias = nn.Parameter(bias) + elif quantize == "gptq": + raise NotImplementedError("Soon") + else: + raise NotImplementedError(f"Quantization `{quantize}` is not implemented yet.") + return linear + + +class SuperLayer(nn.Module): + def __init__(self, linear): + super().__init__() + self.linear = linear + + def forward(self, x): + return self.linear.forward(x) + + +class TensorParallelHead(SuperLayer): + def __init__(self, linear, process_group): + super().__init__(linear) self.process_group = process_group - self.tp_world_size = process_group.size() - self.reduce = reduce - assert in_features % self.tp_world_size == 0 - in_features = in_features // self.tp_world_size - - super().__init__( - in_features=in_features, - out_features=out_features, - bias=bias, - device=device, - dtype=dtype, + + @staticmethod + def load(config, prefix: str, weights): + weight = weights.get_sharded(f"{prefix}.weight", dim=0) + return TensorParallelHead( + get_linear(weight, bias=None, quantize=config.quantize), + process_group=weights.process_group, ) def forward(self, input: torch.Tensor) -> torch.Tensor: - out = super(TensorParallelRowLinear, self).forward(input) - if self.reduce: - torch.distributed.all_reduce(out, group=self.process_group) + output = super().forward(input) + # Logits are sharded, so we need to gather them + world_output = [ + torch.empty_like(output) for _ in range(self.process_group.size()) + ] + torch.distributed.all_gather(world_output, output, group=self.process_group) + world_output = torch.cat(world_output, dim=-1) + return world_output + + +class TensorParallelColumnLinear(SuperLayer): + @classmethod + def load(cls, config, prefix: str, weights, bias: bool): + weight = weights.get_sharded(f"{prefix}.weight", dim=0) + if bias: + bias = weights.get_sharded(f"{prefix}.bias", dim=0) + else: + bias = None + return cls(get_linear(weight, bias, config.quantize)) - return out + @classmethod + def load_multi(cls, config, prefixes: List[str], weights, bias: bool, dim: int): + w = [weights.get_sharded(f"{p}.weight", dim=0) for p in prefixes] + weight = torch.cat(w, dim=dim) + if bias: + b = [weights.get_sharded(f"{p}.bias", dim=0) for p in prefixes] + bias = torch.cat(b, dim=0) + else: + bias = None + return cls(get_linear(weight, bias, config.quantize)) -class TensorParallelEmbedding(nn.Embedding): - def __init__( - self, - num_embeddings, - embedding_dim, - process_group: torch.distributed.ProcessGroup, - reduce=True, - padding_idx=None, - max_norm=None, - norm_type=2.0, - scale_grad_by_freq=False, - sparse=False, - _weight=None, - device=None, - dtype=None, - ): - self.reduce = reduce + +class TensorParallelRowLinear(SuperLayer): + def __init__(self, linear, process_group): + super().__init__(linear) self.process_group = process_group - self.tp_rank = process_group.rank() - self.tp_world_size = process_group.size() - self.original_num_embeddings = num_embeddings + @classmethod + def load(cls, config, prefix: str, weights, bias: bool): + weight = weights.get_sharded(f"{prefix}.weight", dim=1) + if bias and weights.process_group.rank() == 0: + # Rank is only on the first rank process + bias = weights.get_tensor(f"{prefix}.bias") + else: + bias = None + return cls( + get_linear(weight, bias, config.quantize), + process_group=weights.process_group, + ) - assert num_embeddings % self.tp_world_size == 0 - block_size = num_embeddings // self.tp_world_size - # inputs in `[min_id, max_id[` are handled by `self` to get embeddings - self.min_id = self.tp_rank * block_size - self.max_id = (self.tp_rank + 1) * block_size + def forward(self, input: torch.Tensor) -> torch.Tensor: + out = super().forward(input) + torch.distributed.all_reduce(out, group=self.process_group) + return out - # Additional entry that will map to zero - # Used for masking - self.null_idx = block_size - super().__init__( - block_size, - embedding_dim, - padding_idx=padding_idx, - max_norm=max_norm, - norm_type=norm_type, - scale_grad_by_freq=scale_grad_by_freq, - sparse=sparse, - _weight=_weight, - device=device, - dtype=dtype, - ) +class TensorParallelEmbedding(nn.Module): + def __init__(self, prefix: str, weights, reduce=True): + super().__init__() + weight = weights.get_sharded(f"{prefix}.weight", dim=0) + num_embeddings = weights.get_shape(f"{prefix}.weight")[0] + + process_group = weights.process_group + + world_size = process_group.size() + rank = process_group.rank() + + block_size = num_embeddings // world_size + self.min_id = rank * block_size + self.max_id = min(num_embeddings, (rank + 1) * block_size) + self.null_idx = block_size + self.process_group = weights.process_group + self.reduce = reduce - def add_null_idx(self): """Additional 0 entry used for masking""" - self.weight = nn.Parameter(F.pad(self.weight, (0, 0, 0, 1))) + self.weight = nn.Parameter(F.pad(weight, (0, 0, 0, 1))) def forward(self, input: torch.Tensor) -> torch.Tensor: # default all out of bounds values to `self.null_idx` that will then be mapped to 0 @@ -180,7 +244,7 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: self.null_idx, input - self.min_id, ) - out = super().forward(input) + out = torch.nn.functional.embedding(input, self.weight) if self.reduce: torch.distributed.all_reduce(out, group=self.process_group) return out @@ -232,7 +296,34 @@ def forward(self, hidden_states, residual=None): from flash_attn.layers.rotary import RotaryEmbedding import rotary_emb - class PositionRotaryEmbedding(RotaryEmbedding): + class PositionRotaryEmbedding(nn.Module): + def __init__(self, inv_freq): + super().__init__() + + self.register_buffer("inv_freq", inv_freq) + self._seq_len_cached = 0 + self._cos_cached = None + self._sin_cached = None + self._cos_k_cached = None + self._sin_k_cached = None + + @classmethod + def static(cls, dim, base, device): + inv_freq = 1.0 / ( + base + ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) + ) + return cls(inv_freq) + + @classmethod + def load(cls, prefix, weights): + # XXX: Always load this in float32 ! + dtype = weights.dtype + weights.dtype = torch.float32 + inv_freq = weights.get_tensor(f"{prefix}.inv_freq") + weights.dtype = dtype + return cls(inv_freq) + def _update_cos_sin_cache(self, dtype, device, seqlen): # Reset the tables if the sequence length has changed, # or if we're on a new device (possibly due to tracing for instance) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py new file mode 100644 index 00000000..76a4f65a --- /dev/null +++ b/server/text_generation_server/utils/weights.py @@ -0,0 +1,77 @@ +from pathlib import Path +from typing import List +from safetensors import safe_open + + +class Weights: + def __init__(self, filenames: List[Path], device, dtype, process_group): + routing = {} + for filename in filenames: + with safe_open(filename, framework="pytorch") as f: + for k in f.keys(): + if k in routing: + raise RuntimeError( + f"Key {k} was found in multiple files: {filename} and {routing[k]}" + ) + routing[k] = filename + self.routing = routing + self.device = device + self.dtype = dtype + self.process_group = process_group + self._handles = {} + + def _get_handle(self, filename): + if filename not in self._handles: + f = safe_open(filename, framework="pytorch") + self._handles[filename] = f + + return self._handles[filename] + + def get_filename(self, tensor_name: str) -> str: + filename = self.routing.get(tensor_name, None) + if filename is None: + raise RuntimeError(f"weight {tensor_name} does not exist") + return str(filename) + + def _get_slice(self, tensor_name: str): + filename = self.get_filename(tensor_name) + f = self._get_handle(filename) + slice_ = f.get_slice(tensor_name) + return slice_ + + def get_shape(self, tensor_name: str): + return self._get_slice(tensor_name).get_shape() + + def get_tensor(self, tensor_name: str): + filename = self.get_filename(tensor_name) + f = self._get_handle(filename) + tensor = f.get_tensor(tensor_name) + tensor = tensor.to(dtype=self.dtype) + tensor = tensor.to(device=self.device) + return tensor + + def get_sharded(self, tensor_name: str, dim: int): + filename = self.get_filename(tensor_name) + world_size = self.process_group.size() + rank = self.process_group.rank() + + f = self._get_handle(filename) + slice_ = f.get_slice(tensor_name) + size = slice_.get_shape()[dim] + block_size = size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + + assert ( + size % world_size == 0 + ), f"The choosen size {size} is not compatible with sharding on {world_size} shards" + + if dim == 0: + tensor = slice_[start:stop] + elif dim == 1: + tensor = slice_[:, start:stop] + else: + raise NotImplementedError("Let's make that generic when needed") + tensor = tensor.to(dtype=self.dtype) + tensor = tensor.to(device=self.device) + return tensor From e496c9ba5b574ce4e9d04d3b16bce67759ff0445 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 9 Jun 2023 11:55:29 +0200 Subject: [PATCH 276/793] feat(server): optimize dist ops (#434) --- .../custom_modeling/flash_neox_modeling.py | 3 +- .../custom_modeling/flash_rw_modeling.py | 6 ++-- .../flash_santacoder_modeling.py | 4 ++- server/text_generation_server/utils/layers.py | 32 +++++++++++++++++-- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 0fe43bcb..c045f16e 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -265,7 +265,8 @@ def forward( mlp_output = self.mlp(ln2_hidden_states) intermediate = mlp_output + attn_output - torch.distributed.all_reduce(intermediate, group=self.process_group) + if self.process_group.size() > 1: + torch.distributed.all_reduce(intermediate, group=self.process_group) return intermediate + hidden_states, None else: diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 55195162..af9fa548 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -440,7 +440,8 @@ def forward( mlp_output = self.mlp(ln_hidden_states) intermediate = mlp_output + attn_output - torch.distributed.all_reduce(intermediate, group=self.process_group) + if self.process_group.size() > 1: + torch.distributed.all_reduce(intermediate, group=self.process_group) return intermediate, residual else: @@ -524,7 +525,8 @@ def forward( intermediate = attn_output + mlp_output - torch.distributed.all_reduce(intermediate, group=self.process_group) + if self.process_group.size() > 1: + torch.distributed.all_reduce(intermediate, group=self.process_group) return intermediate, residual diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 888a6066..fcf6be68 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -346,7 +346,9 @@ def forward( pre_allocate_past_size: Optional[int] = None, ): hidden_states = self.wte(input_ids) + self.wpe(position_ids) - torch.distributed.all_reduce(hidden_states, group=self.process_group) + + if self.process_group.size() > 1: + torch.distributed.all_reduce(hidden_states, group=self.process_group) # Prefill if past_key_values is None: diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index ee32a0dc..93865d52 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -158,8 +158,33 @@ def load(config, prefix: str, weights): ) def forward(self, input: torch.Tensor) -> torch.Tensor: + world_size = self.process_group.size() + if world_size == 1: + return super().forward(input) + + if len(input.shape) == 2 and isinstance(self.linear, FastLinear): + out_dim = self.linear.weight.shape[0] + + if input.shape[0] == 1: + world_out = input.new_empty(1, out_dim * world_size) + local_out = input.new_empty(1, out_dim) + gather_input = local_out + else: + world_out = input.new_empty(out_dim * world_size, input.shape[0]) + gather_input = input.new_empty(out_dim, input.shape[0]) + local_out = gather_input.T + + torch.mm(input, self.linear.weight.T, out=local_out) + + torch.distributed.all_gather_into_tensor( + world_out, gather_input, group=self.process_group + ) + + if input.shape[0] == 1: + return world_out + return world_out.T + output = super().forward(input) - # Logits are sharded, so we need to gather them world_output = [ torch.empty_like(output) for _ in range(self.process_group.size()) ] @@ -211,7 +236,8 @@ def load(cls, config, prefix: str, weights, bias: bool): def forward(self, input: torch.Tensor) -> torch.Tensor: out = super().forward(input) - torch.distributed.all_reduce(out, group=self.process_group) + if self.process_group.size() > 1: + torch.distributed.all_reduce(out, group=self.process_group) return out @@ -245,7 +271,7 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: input - self.min_id, ) out = torch.nn.functional.embedding(input, self.weight) - if self.reduce: + if self.reduce and self.process_group.size() > 1: torch.distributed.all_reduce(out, group=self.process_group) return out From d4eb60f48d22f4ae97acef4a239dccdf9549877d Mon Sep 17 00:00:00 2001 From: "A.J" <47569261+antferdom@users.noreply.github.com> Date: Mon, 12 Jun 2023 13:59:22 +0200 Subject: [PATCH 277/793] docs(launcher): fix CUDA_VISIBLE_DEVICES helper comment (#441) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What does this PR do? It solves a typo in the comment sections referencing the environment variable `CUDA_VISIBLE_DEVICES`. No misspelling references to this variable have been found in code logic leading to undefined behaviour or bugs. This PR is not expected to perform any code logic modification. --- launcher/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index f59ff685..36f6f6b6 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -60,8 +60,8 @@ struct Args { sharded: Option, /// The number of shards to use if you don't want to use all GPUs on a given machine. - /// You can use `CUDA_VISIBLE_DEVICE=0,1 text-generation-launcher... --num_shard 2` - /// and `CUDA_VISIBLE_DEVICE=2,3 text-generation-launcher... --num_shard 2` to + /// You can use `CUDA_VISIBLE_DEVICES=0,1 text-generation-launcher... --num_shard 2` + /// and `CUDA_VISIBLE_DEVICES=2,3 text-generation-launcher... --num_shard 2` to /// launch 2 copies with 2 shard each on a given machine with 4 GPUs for instance. #[clap(long, env)] num_shard: Option, From ca650e5bff1af8c8580c2be0d4dad37dd3285247 Mon Sep 17 00:00:00 2001 From: sayf eddine hammemi Date: Mon, 12 Jun 2023 15:24:53 +0200 Subject: [PATCH 278/793] fix(makefile): Fix typo and use POSIX comparison in the makefile (#443) # What does this PR do? This PR fixes: - The usage of non posix comparison which may fail depending on the shell used (`=` will always work, `==` only with bash) - Typo in the env variable name displayed in the error message `BUILD_EXTENSION` instead of `BUILD_EXTENSIONS` Fixes #422 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c7f649ec..3c2f2b9d 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ install-server: cd server && make install install-custom-kernels: - if [ "$$BUILD_EXTENSIONS" == "True" ]; then cd server/custom_kernels && python setup.py install; else echo "Custom kernels are disabled, you need set to BUILD_EXTENSION environment variable to 'True' in order to build them. (Please read the docs, kernels might not work on all hardware)"; fi + if [ "$$BUILD_EXTENSIONS" = "True" ]; then cd server/custom_kernels && python setup.py install; else echo "Custom kernels are disabled, you need to set the BUILD_EXTENSIONS environment variable to 'True' in order to build them. (Please read the docs, kernels might not work on all hardware)"; fi install-integration-tests: cd integration-tests && pip install -r requirements.txt From 5ce89059f8149eaf313c63e9ded4199670cd74bb Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 12 Jun 2023 18:30:29 +0200 Subject: [PATCH 279/793] feat(server): pre-allocate past key values for flash causal LM (#412) --- server/Makefile-flash-att | 4 +- .../custom_modeling/flash_llama_modeling.py | 114 ++++--- .../custom_modeling/flash_neox_modeling.py | 121 +++++--- .../custom_modeling/flash_rw_modeling.py | 182 +++++++---- .../flash_santacoder_modeling.py | 125 ++++---- .../models/flash_causal_lm.py | 293 +++++++++--------- 6 files changed, 494 insertions(+), 345 deletions(-) diff --git a/server/Makefile-flash-att b/server/Makefile-flash-att index ad894bfa..0e67a9e4 100644 --- a/server/Makefile-flash-att +++ b/server/Makefile-flash-att @@ -1,9 +1,9 @@ -flash_att_commit := d478eeec8f16c7939c54e4617dbd36f59b8eeed7 +flash_att_commit := 06ece1a1525ebcf4e183ac76b1e5108d2872f57f flash-attention: # Clone flash attention pip install packaging - git clone https://github.com/HazyResearch/flash-attention.git + git clone https://github.com/OlivierDehaene/flash-attention.git build-flash-attention: flash-attention cd flash-attention && git fetch && git checkout $(flash_att_commit) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 8a35ffa8..993e1e2a 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -128,11 +128,14 @@ def forward( hidden_states, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): qkv = self.query_key_value(hidden_states) qkv = qkv.view(-1, 3, self.num_heads, self.head_size) @@ -142,7 +145,7 @@ def forward( self.rotary_emb(qkv[:, 1], cos, sin) # Prefill - if layer_past_present_indices is None: + if prefill: # Copy to layer past layer_past[...] = qkv[:, 1:] @@ -154,8 +157,10 @@ def forward( qkv[:, 1], qkv[:, 2], attn_output, - cu_seqlens, - cu_seqlens, + start_seq, + end_seq, + start_seq, + end_seq, max_s, max_s, 0.0, @@ -170,7 +175,7 @@ def forward( else: query = qkv[:, 0] # Add present to the layer_past tensor at the correct indices - layer_past[layer_past_present_indices] = qkv[:, 1:] + layer_past[past_present_indices] = qkv[:, 1:] # output attn_output = torch.empty_like(query) @@ -180,8 +185,10 @@ def forward( layer_past[:, 0], layer_past[:, 1], attn_output, - cu_seqlens_q, - cu_seqlens, + start_seq_q, + end_seq_q, + start_seq, + end_seq, 1, max_s, 0.0, @@ -258,11 +265,14 @@ def forward( residual, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): normed_hidden_states, res = self.input_layernorm(hidden_states, residual) @@ -271,11 +281,14 @@ def forward( normed_hidden_states, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ) # faster post attention rms norm @@ -322,35 +335,37 @@ def forward( self, input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, - past_key_values: Optional[torch.Tensor] = None, + past_present_indices, + past_key_values=None, pre_allocate_past_size: Optional[int] = None, ): hidden_states = self.embed_tokens(input_ids) # Prefill if past_key_values is None: + assert pre_allocate_past_size is not None + + prefill = True + # Create past tensor + # We create a tensor of the same size as input_ids as we don't want to slice at every layer past_key_values = hidden_states.new_empty( ( + len(input_ids), len(self.layers), - len(hidden_states) - if pre_allocate_past_size is None - else pre_allocate_past_size, 2, self.num_heads, self.head_size, ) ) - layer_past_present_indices = None - slice_past_index = len(hidden_states) # Decode else: - # Create indices from cumulative sequence lengths - layer_past_present_indices = cu_seqlens[1:] - 1 - slice_past_index = None + prefill = False # Get rotary cos and sin for this forward # Avoid to index in each layer @@ -360,24 +375,35 @@ def forward( residual = None for i, layer in enumerate(self.layers): - # We added padding that we now need to slice - layer_past_key_values = ( - past_key_values[i] - if slice_past_index is None - else past_key_values[i, :slice_past_index] - ) - hidden_states, residual = layer( hidden_states, residual, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, - layer_past_key_values, - layer_past_present_indices, - cu_seqlens_q, + past_key_values[:, i], + past_present_indices, + prefill, + ) + + if prefill: + present = past_key_values + # Create padded past tensor + past_key_values = hidden_states.new_empty( + ( + pre_allocate_past_size, + len(self.layers), + 2, + self.num_heads, + self.head_size, + ) ) + # We slice only once instead of at every layer + past_key_values[past_present_indices] = present hidden_states, _ = self.norm(hidden_states, residual) @@ -399,9 +425,12 @@ def forward( self, input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, lm_head_indices: Optional[torch.Tensor] = None, @@ -409,9 +438,12 @@ def forward( hidden_states, present = self.model( input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values, pre_allocate_past_size, ) diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index c045f16e..3586b85a 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -113,11 +113,14 @@ def forward( hidden_states, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): qkv = self.query_key_value(hidden_states) qkv = qkv.view(-1, 3, self.num_heads, self.head_size) @@ -127,7 +130,7 @@ def forward( self.rotary_emb(qkv[:, 1], cos, sin) # Prefill - if layer_past_present_indices is None: + if prefill: # Copy to layer past layer_past[...] = qkv[:, 1:] @@ -139,8 +142,10 @@ def forward( qkv[:, 1], qkv[:, 2], attn_output, - cu_seqlens, - cu_seqlens, + start_seq, + end_seq, + start_seq, + end_seq, max_s, max_s, 0.0, @@ -155,7 +160,7 @@ def forward( else: query = qkv[:, 0] # Add present to the layer_past tensor at the correct indices - layer_past[layer_past_present_indices] = qkv[:, 1:] + layer_past[past_present_indices] = qkv[:, 1:] # output attn_output = torch.empty_like(query) @@ -165,8 +170,10 @@ def forward( layer_past[:, 0], layer_past[:, 1], attn_output, - cu_seqlens_q, - cu_seqlens, + start_seq_q, + end_seq_q, + start_seq, + end_seq, 1, max_s, 0.0, @@ -240,11 +247,14 @@ def forward( residual, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): if self.use_parallel_residual: ln1_hidden_states, _ = self.input_layernorm(hidden_states) @@ -253,11 +263,14 @@ def forward( ln1_hidden_states, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ) ln2_hidden_states, _ = self.post_attention_layernorm(hidden_states) @@ -276,11 +289,14 @@ def forward( hidden_states, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ) hidden_states, residual = self.post_attention_layernorm( @@ -329,9 +345,12 @@ def forward( self, input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values=None, pre_allocate_past_size: Optional[int] = None, ): @@ -339,25 +358,24 @@ def forward( # Prefill if past_key_values is None: + assert pre_allocate_past_size is not None + + prefill = True + # Create past tensor + # We create a tensor of the same size as input_ids as we don't want to slice at every layer past_key_values = hidden_states.new_empty( ( + len(input_ids), len(self.layers), - len(hidden_states) - if pre_allocate_past_size is None - else pre_allocate_past_size, 2, self.num_heads, self.head_size, ) ) - layer_past_present_indices = None - slice_past_index = len(hidden_states) # Decode else: - # Create indices from cumulative sequence lengths - layer_past_present_indices = cu_seqlens[1:] - 1 - slice_past_index = None + prefill = False # Get rotary cos and sin for this forward # Avoid to index in each layer @@ -367,24 +385,35 @@ def forward( residual = None for i, layer in enumerate(self.layers): - # We added padding that we now need to slice - layer_past_key_values = ( - past_key_values[i] - if slice_past_index is None - else past_key_values[i, :slice_past_index] - ) - hidden_states, residual = layer( hidden_states, residual, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, - layer_past_key_values, - layer_past_present_indices, - cu_seqlens_q, + past_key_values[:, i], + past_present_indices, + prefill, + ) + + if prefill: + present = past_key_values + # Create padded past tensor + past_key_values = hidden_states.new_empty( + ( + pre_allocate_past_size, + len(self.layers), + 2, + self.num_heads, + self.head_size, + ) ) + # We slice only once instead of at every layer + past_key_values[past_present_indices] = present hidden_states, _ = self.final_layer_norm(hidden_states, residual) @@ -404,9 +433,12 @@ def forward( self, input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, lm_head_indices: Optional[torch.Tensor] = None, @@ -414,9 +446,12 @@ def forward( hidden_states, present = self.gpt_neox( input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values, pre_allocate_past_size, ) diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index af9fa548..4a9063eb 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -130,11 +130,14 @@ def forward( hidden_states, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): qkv = self.query_key_value(hidden_states) @@ -150,10 +153,10 @@ def forward( # Inplace rotary self.rotary_emb(query, cos, sin) - self.rotary_emb(kv[:, 0], cos, sin) + self.rotary_emb(torch.select(kv, dim=1, index=0), cos, sin) # Prefill - if layer_past_present_indices is None: + if prefill: # Copy to layer past layer_past[...] = kv # Expand to query shape @@ -164,11 +167,13 @@ def forward( # flash attention flash_attn_cuda.fwd( query, - kv[:, 0], - kv[:, 1], + torch.select(kv, dim=1, index=0), + torch.select(kv, dim=1, index=1), attn_output, - cu_seqlens, - cu_seqlens, + start_seq, + end_seq, + start_seq, + end_seq, max_s, max_s, 0.0, @@ -182,7 +187,7 @@ def forward( # Decode else: # Add present to the layer_past tensor at the correct indices - layer_past[layer_past_present_indices] = kv + layer_past[past_present_indices] = kv # Expand to query shape kv = layer_past.expand(-1, 2, self.num_heads, self.head_size) @@ -191,11 +196,13 @@ def forward( # flash attention flash_attn_cuda.fwd( query, - kv[:, 0], - kv[:, 1], + torch.select(kv, dim=1, index=0), + torch.select(kv, dim=1, index=1), attn_output, - cu_seqlens_q, - cu_seqlens, + start_seq_q, + end_seq_q, + start_seq, + end_seq, 1, max_s, 0.0, @@ -261,11 +268,14 @@ def forward( hidden_states, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): qkv = self.query_key_value(hidden_states) qkv = qkv.view(-1, self.num_groups, self.num_heads + 2, self.head_size) @@ -280,10 +290,10 @@ def forward( # Inplace rotary self.rotary_emb(query, cos, sin) - self.rotary_emb(kv[:, :, 0], cos, sin) + self.rotary_emb(torch.select(kv, dim=2, index=0), cos, sin) # Prefill - if layer_past_present_indices is None: + if prefill: # Copy to layer past layer_past[...] = kv # Expand to query shape @@ -298,11 +308,13 @@ def forward( # flash attention flash_attn_cuda.fwd( query, - kv[:, :, 0], - kv[:, :, 1], + torch.select(kv, dim=2, index=0), + torch.select(kv, dim=2, index=1), attn_output, - cu_seqlens, - cu_seqlens, + start_seq, + end_seq, + start_seq, + end_seq, max_s, max_s, 0.0, @@ -316,7 +328,7 @@ def forward( # Decode else: # Add present to the layer_past tensor at the correct indices - layer_past[layer_past_present_indices] = kv + layer_past[past_present_indices] = kv # Expand to query shape kv = ( layer_past.unsqueeze(2) @@ -329,11 +341,13 @@ def forward( # flash attention flash_attn_cuda.fwd( query, - kv[:, :, 0], - kv[:, :, 1], + torch.select(kv, dim=2, index=0), + torch.select(kv, dim=2, index=1), attn_output, - cu_seqlens_q, - cu_seqlens, + start_seq_q, + end_seq_q, + start_seq, + end_seq, 1, max_s, 0.0, @@ -417,11 +431,14 @@ def forward( residual, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): if self.parallel_attn: ln_hidden_states, residual = self.input_layernorm(hidden_states, residual) @@ -430,11 +447,14 @@ def forward( ln_hidden_states, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ) mlp_output = self.mlp(ln_hidden_states) @@ -451,11 +471,14 @@ def forward( hidden_states, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ) hidden_states, residual = self.post_attention_layernorm( @@ -499,11 +522,14 @@ def forward( residual, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): ln_attn, residual = self.ln_attn(hidden_states, residual) ln_mlp, _ = self.ln_mlp(residual) @@ -513,11 +539,14 @@ def forward( ln_attn, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ) # MLP. @@ -584,9 +613,12 @@ def forward( self, input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values=None, pre_allocate_past_size: Optional[int] = None, ): @@ -594,23 +626,22 @@ def forward( # Prefill if past_key_values is None: + assert pre_allocate_past_size is not None + + prefill = True + # Create past tensor + # We create a tensor of the same size as input_ids as we don't want to slice at every layer past_key_values = hidden_states.new_empty( ( + len(input_ids), len(self.h), - len(hidden_states) - if pre_allocate_past_size is None - else pre_allocate_past_size, *self.cache_size, ) ) - layer_past_present_indices = None - slice_past_index = len(hidden_states) # Decode else: - # Create indices from cumulative sequence lengths - layer_past_present_indices = cu_seqlens[1:] - 1 - slice_past_index = None + prefill = False # Get rotary cos and sin for this forward # Avoid to index in each layer @@ -620,24 +651,33 @@ def forward( residual = None for i, layer in enumerate(self.h): - # We added padding that we now need to slice - layer_past_key_values = ( - past_key_values[i] - if slice_past_index is None - else past_key_values[i, :slice_past_index] - ) - hidden_states, residual = layer( hidden_states, residual, cos, sin, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, - layer_past_key_values, - layer_past_present_indices, - cu_seqlens_q, + torch.select(past_key_values, dim=1, index=i), + past_present_indices, + prefill, + ) + + if prefill: + present = past_key_values + # Create padded past tensor + past_key_values = hidden_states.new_empty( + ( + pre_allocate_past_size, + len(self.h), + *self.cache_size, + ) ) + # We slice only once instead of at every layer + past_key_values[past_present_indices] = present hidden_states, _ = self.ln_f(hidden_states, residual) @@ -658,9 +698,12 @@ def forward( self, input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, lm_head_indices: Optional[torch.Tensor] = None, @@ -668,9 +711,12 @@ def forward( hidden_states, present = self.transformer( input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values, pre_allocate_past_size, ) diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index fcf6be68..00cc47b6 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -7,6 +7,7 @@ # Flash attention imports import flash_attn_cuda + from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -148,11 +149,14 @@ def __init__(self, prefix, config, weights): def forward( self, hidden_states, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): qkv = self.c_attn(hidden_states) @@ -166,7 +170,7 @@ def forward( key_value = key_value.view(-1, 2, 1, self.head_size) # Prefill - if layer_past_present_indices is None: + if prefill: # Copy to layer past layer_past[...] = key_value # Expand from 1 to num_heads @@ -177,11 +181,13 @@ def forward( # flash attention flash_attn_cuda.fwd( query, - key_value[:, 0], - key_value[:, 1], + torch.select(key_value, dim=1, index=0), + torch.select(key_value, dim=1, index=1), attn_output, - cu_seqlens, - cu_seqlens, + start_seq, + end_seq, + start_seq, + end_seq, max_s, max_s, 0.0, @@ -195,7 +201,7 @@ def forward( # Decode else: # Add present to the layer_past tensor at the correct indices - layer_past[layer_past_present_indices] = key_value + layer_past[past_present_indices] = key_value # Expand from 1 to num_heads key_value = layer_past.expand(-1, 2, self.num_heads, self.head_size) @@ -204,11 +210,13 @@ def forward( # flash attention flash_attn_cuda.fwd( query, - key_value[:, 0], - key_value[:, 1], + torch.select(key_value, dim=1, index=0), + torch.select(key_value, dim=1, index=1), attn_output, - cu_seqlens_q, - cu_seqlens, + start_seq_q, + end_seq_q, + start_seq, + end_seq, 1, max_s, 0.0, @@ -277,21 +285,27 @@ def forward( self, hidden_states, residual, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ): hidden_states, residual = self.ln_1(hidden_states, residual) hidden_states = self.attn( hidden_states, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, layer_past, - layer_past_present_indices, - cu_seqlens_q, + past_present_indices, + prefill, ) hidden_states, residual = self.ln_2(hidden_states, residual) @@ -339,10 +353,13 @@ def forward( self, input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, - past_key_values: Optional[torch.Tensor] = None, + past_present_indices, + past_key_values=None, pre_allocate_past_size: Optional[int] = None, ): hidden_states = self.wte(input_ids) + self.wpe(position_ids) @@ -352,44 +369,42 @@ def forward( # Prefill if past_key_values is None: + assert pre_allocate_past_size is not None + + prefill = True + # Create past tensor - past_key_values = hidden_states.new_empty( - ( - len(self.h), - len(hidden_states) - if pre_allocate_past_size is None - else pre_allocate_past_size, - 2, - 1, - self.head_size, - ) + # We create a tensor of the same size as input_ids as we don't want to slice at every layer + past_key_values = hidden_states.new_zeros( + (len(input_ids), len(self.h), 2, 1, self.head_size) ) - layer_past_present_indices = None - slice_past_index = len(hidden_states) # Decode else: - # Create indices from cumulative sequence lengths - layer_past_present_indices = cu_seqlens[1:] - 1 - slice_past_index = None + prefill = False residual = None for i, layer in enumerate(self.h): - # We added padding that we now need to slice - layer_past_key_values = ( - past_key_values[i] - if slice_past_index is None - else past_key_values[i, :slice_past_index] - ) - hidden_states, residual = layer( hidden_states, residual, - cu_seqlens, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, - layer_past_key_values, - layer_past_present_indices, - cu_seqlens_q, + torch.select(past_key_values, dim=1, index=i), + past_present_indices, + prefill, + ) + + if prefill: + present = past_key_values + # Create padded past tensor + past_key_values = hidden_states.new_empty( + (pre_allocate_past_size, len(self.h), 2, 1, self.head_size) ) + # We slice only once instead of at every layer + past_key_values[past_present_indices] = present hidden_states, _ = self.ln_f(hidden_states, residual) @@ -408,9 +423,12 @@ def forward( self, input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values: Optional[torch.Tensor] = None, pre_allocate_past_size: Optional[int] = None, lm_head_indices: Optional[torch.Tensor] = None, @@ -418,9 +436,12 @@ def forward( hidden_states, present = self.transformer( input_ids, position_ids, - cu_seqlens, - cu_seqlens_q, + start_seq, + end_seq, + start_seq_q, + end_seq_q, max_s, + past_present_indices, past_key_values, pre_allocate_past_size, ) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index a2ad2d5e..ecea998e 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -3,8 +3,6 @@ import numpy as np -from torch.nn import functional as F - from dataclasses import dataclass from opentelemetry import trace from transformers import AutoTokenizer, PreTrainedTokenizerBase, PreTrainedModel @@ -34,10 +32,21 @@ class FlashCausalLMBatch(Batch): input_ids: torch.Tensor position_ids: torch.Tensor - # cumulative sequence lengths - cu_seqlens: torch.Tensor - # cumulative query sequence lengths, only used in decode - cu_seqlens_q: Optional[torch.Tensor] + # Indices to copy present to the correct indices is the pre-allocated past key values + past_present_indices: torch.Tensor + + # tensor of length b holding starting offset of each sequence + start_seq: torch.Tensor + # tensor of length b holding ending offset of each sequence + end_seq: torch.Tensor + # tensor of length b holding starting offset of each sequence, only used in prefill + start_seq_prefill: Optional[torch.Tensor] + # tensor of length b holding ending offset of each sequence, only used in prefill + end_seq_prefill: Optional[torch.Tensor] + # tensor of length b holding starting offset of each query sequence, only used in decode + start_seq_q: Optional[torch.Tensor] + # tensor of length b holding ending offset of each query sequence, only used in decode + end_seq_q: Optional[torch.Tensor] # past key values, only used in decode past_key_values: Optional[torch.Tensor] max_seqlen: int @@ -90,7 +99,11 @@ def from_pb( )["input_ids"] position_ids = [] - cu_seqlens = [0] + past_present_indices = [] + start_seq = [] + end_seq = [] + start_seq_prefill = [] + end_seq_prefill = [] max_seqlen = 0 input_lengths = [] @@ -110,9 +123,9 @@ def from_pb( # Cumulative length cumulative_length = 0 + cumulative_max_length = 0 prefill_out_cumulative_length = 0 - max_tokens = 0 max_length = 0 # Parse batch @@ -138,7 +151,10 @@ def from_pb( position_ids.append(request_position_ids) # Add cumulative lengths of all previous inputs - cu_seqlens.append(cumulative_length + input_length) + start_seq_prefill.append(cumulative_length) + end_seq_prefill.append(cumulative_length + input_length) + start_seq.append(cumulative_max_length) + end_seq.append(cumulative_max_length + input_length) next_token_chooser_parameters.append(r.parameters) @@ -168,9 +184,17 @@ def from_pb( prefill_cu_outlens.append(prefill_out_cumulative_length + 1) prefill_out_cumulative_length += 1 + request_past_present_indices = torch.arange( + cumulative_max_length, + cumulative_max_length + input_length, + dtype=torch.int64, + ) + past_present_indices.append(request_past_present_indices) + # Update + # Remove one as the first token des not have a past cumulative_length += input_length - max_tokens += input_length + max_new_tokens + cumulative_max_length += input_length + max_new_tokens - 1 max_length = max(max_length, input_length + max_new_tokens) next_token_chooser = HeterogeneousNextTokenChooser.from_pb( @@ -184,26 +208,45 @@ def from_pb( for i, input_ids in enumerate(all_input_ids): all_input_ids_tensor[i, : len(input_ids)] = input_ids + # Create tensors on device + all_input_ids_tensor = torch.tensor( + all_input_ids_tensor, dtype=torch.int64, device=device + ) + start_seq = torch.tensor(start_seq, device=device, dtype=torch.int32) + end_seq = torch.tensor(end_seq, device=device, dtype=torch.int32) + if len(pb.requests) > 1: input_ids = np.concatenate(all_input_ids, dtype=np.int64) position_ids = torch.cat(position_ids) + + past_present_indices = np.concatenate(past_present_indices, dtype=np.int64) + + start_seq_prefill = torch.tensor( + start_seq_prefill, device=device, dtype=torch.int32 + ) + end_seq_prefill = torch.tensor( + end_seq_prefill, device=device, dtype=torch.int32 + ) else: input_ids = all_input_ids[0] position_ids = position_ids[0] - # Create tensors on device + past_present_indices = past_present_indices[0] + + start_seq_prefill = start_seq + end_seq_prefill = end_seq + input_ids = torch.tensor(input_ids, dtype=torch.int64, device=device) - all_input_ids_tensor = torch.tensor( - all_input_ids_tensor, dtype=torch.int64, device=device - ) position_ids = torch.tensor(position_ids, dtype=torch.int32, device=device) - cu_seqlens = torch.tensor(cu_seqlens, device=device, dtype=torch.int32) + past_present_indices = torch.tensor( + past_present_indices, device=device, dtype=torch.int64 + ) if all_prefill_logprobs: prefill_head_indices = None - prefill_next_token_indices = cu_seqlens[1:] - 1 + prefill_next_token_indices = end_seq_prefill - 1 elif no_prefill_logprobs: - prefill_head_indices = cu_seqlens[1:] - 1 + prefill_head_indices = end_seq_prefill - 1 prefill_next_token_indices = None else: prefill_head_indices = torch.tensor( @@ -219,8 +262,13 @@ def from_pb( requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, - cu_seqlens=cu_seqlens, - cu_seqlens_q=None, + past_present_indices=past_present_indices, + start_seq=start_seq, + end_seq=end_seq, + start_seq_prefill=start_seq_prefill, + end_seq_prefill=end_seq_prefill, + start_seq_q=None, + end_seq_q=None, max_seqlen=max_seqlen, prefill_head_indices=prefill_head_indices, prefill_next_token_indices=prefill_next_token_indices, @@ -233,7 +281,7 @@ def from_pb( all_input_ids_tensor=all_input_ids_tensor, next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, - max_tokens=max_tokens, + max_tokens=cumulative_max_length, ) @tracer.start_as_current_span("filter") @@ -244,10 +292,10 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": if len(request_ids) == len(self): return self - single_request = len(request_ids) == 1 + device = self.input_ids.device # Cumulative length - cumulative_length = 0 + cumulative_max_length = 0 # New values after filtering requests_idx_mapping = {} @@ -255,11 +303,17 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": # Used to index into tensors indices = [] + # past indices to keep + past_indices = torch.zeros( + self.past_key_values.shape[0], dtype=torch.bool, device=device + ) + # Create on CPU to only move to GPU once instead of at every copy - cu_seqlens = torch.zeros(len(request_ids) + 1, dtype=torch.int32) - cu_seqlens_q = self.cu_seqlens_q[: len(request_ids) + 1] + start_seq = torch.empty(len(request_ids), dtype=torch.int32) + end_seq = torch.empty(len(request_ids), dtype=torch.int32) + start_seq_q = self.start_seq_q[: len(request_ids)] + end_seq_q = self.end_seq_q[: len(request_ids)] max_seqlen = 0 - past_key_values = [] requests = [] all_input_ids = [] @@ -270,8 +324,6 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": stopping_criterias = [] - max_tokens = 0 - for i, request_id in enumerate(request_ids): idx = self.requests_idx_mapping[request_id] indices.append(idx) @@ -281,16 +333,8 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": # Get length request_input_length = self.input_lengths[idx] - - # Copy to tensor (CPU) - cu_seqlens[i + 1] = cumulative_length + request_input_length max_seqlen = max(max_seqlen, request_input_length) - # Slice from past - past_key_values.append( - self.past_key_values[:, self.cu_seqlens[idx] : self.cu_seqlens[idx + 1]] - ) - all_input_ids.append(self.all_input_ids[idx]) input_lengths.append(request_input_length) @@ -300,39 +344,32 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": stopping_criteria = self.stopping_criterias[idx] stopping_criterias.append(stopping_criteria) - cumulative_length += request_input_length - max_tokens += request_input_length + ( + remaining_tokens = ( stopping_criteria.max_new_tokens - stopping_criteria.current_tokens ) - if single_request: - # Preallocate tensor for bs = 1 case - past_key_values = F.pad( - past_key_values[0], - ( - 0, - 0, - 0, - 0, - 0, - 0, - 0, - stopping_criterias[0].max_new_tokens - - stopping_criterias[0].current_tokens, - ), - ) - else: - # Cat all past - past_key_values = torch.cat(past_key_values, dim=1) + # Copy to tensor (CPU) + start_seq[i] = cumulative_max_length + end_seq[i] = cumulative_max_length + request_input_length + + # Set slice + past_indices[ + self.start_seq[idx] : self.end_seq[idx] + remaining_tokens - 1 + ] = True + + cumulative_max_length += request_input_length + remaining_tokens - 1 # Index into tensors input_ids = self.input_ids[indices] position_ids = self.position_ids[indices] all_input_ids_tensor = self.all_input_ids_tensor[indices] next_token_chooser = self.next_token_chooser.filter(indices) + past_key_values = self.past_key_values[past_indices] # Move to GPU now that we have the whole tensor - cu_seqlens = cu_seqlens.to(self.cu_seqlens.device) + start_seq = start_seq.to(device) + end_seq = end_seq.to(device) + past_present_indices = end_seq - 1 return FlashCausalLMBatch( batch_id=self.batch_id, @@ -340,8 +377,13 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, - cu_seqlens=cu_seqlens, - cu_seqlens_q=cu_seqlens_q, + past_present_indices=past_present_indices, + start_seq=start_seq, + end_seq=end_seq, + start_seq_prefill=None, + end_seq_prefill=None, + start_seq_q=start_seq_q, + end_seq_q=end_seq_q, max_seqlen=max_seqlen, prefill_head_indices=None, prefill_next_token_indices=None, @@ -354,7 +396,7 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": all_input_ids_tensor=all_input_ids_tensor, next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, - max_tokens=max_tokens, + max_tokens=cumulative_max_length, ) @classmethod @@ -371,10 +413,12 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch input_ids = batches[0].input_ids.new_empty(total_batch_size) position_ids = batches[0].position_ids.new_empty(total_batch_size) - cu_seqlens = [0] - cu_seqlens_q = torch.arange( - 0, total_batch_size + 1, device=device, dtype=torch.int32 + start_seq = batches[0].start_seq.new_empty(total_batch_size) + end_seq = batches[0].end_seq.new_empty(total_batch_size) + start_seq_q = torch.arange( + 0, total_batch_size, device=device, dtype=torch.int32 ) + end_seq_q = start_seq_q + 1 max_seqlen = 0 past_key_values = [] @@ -389,7 +433,6 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch # Cumulative length cumulative_batch_size = 0 - cumulative_length = 0 max_tokens = 0 max_length = 0 @@ -410,18 +453,10 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch input_ids[start_index:end_index] = batch.input_ids position_ids[start_index:end_index] = batch.position_ids - # Add cumulative lengths of all previous inputs - cu_seqlens.extend([l + cumulative_length for l in batch.cu_seqlens[1:]]) - max_seqlen = max(max_seqlen, batch.max_seqlen) + start_seq[start_index:end_index] = batch.start_seq + max_tokens + end_seq[start_index:end_index] = batch.end_seq + max_tokens - if len(batch) != 1: - past_key_values.append(batch.past_key_values) - else: - # past was pre-allocated for this batch - # We need to slice to remove the padding - past_key_values.append( - batch.past_key_values[:, : batch.input_lengths[0]] - ) + max_seqlen = max(max_seqlen, batch.max_seqlen) all_input_ids.extend(batch.all_input_ids) @@ -431,9 +466,9 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch next_token_chooser_parameters.extend([r.parameters for r in batch.requests]) stopping_criterias.extend(batch.stopping_criterias) + past_key_values.append(batch.past_key_values) # Update - cumulative_length += batch.cu_seqlens[-1] cumulative_batch_size += len(batch) max_tokens += batch.max_tokens max_length = max( @@ -448,6 +483,9 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch ), ) + past_key_values = torch.cat(past_key_values, dim=0) + past_present_indices = end_seq - 1 + all_input_ids_tensor = torch.zeros( (total_batch_size, max_length), dtype=torch.int64, device=device ) @@ -463,11 +501,6 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch cumulative_batch_size += len(batch) - # Cat past - past_key_values = torch.cat(past_key_values, dim=1) - # Create final tensor on GPU - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32, device=device) - next_token_chooser = HeterogeneousNextTokenChooser.from_pb( next_token_chooser_parameters, dtype=dtype, device=device ) @@ -478,8 +511,13 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, - cu_seqlens=cu_seqlens, - cu_seqlens_q=cu_seqlens_q, + past_present_indices=past_present_indices, + start_seq=start_seq, + end_seq=end_seq, + start_seq_prefill=None, + end_seq_prefill=None, + start_seq_q=start_seq_q, + end_seq_q=end_seq_q, max_seqlen=max_seqlen, prefill_head_indices=None, prefill_next_token_indices=None, @@ -550,9 +588,12 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - cu_seqlens: torch.Tensor, - cu_seqlens_q: Optional[torch.Tensor], + start_seq: torch.Tensor, + end_seq: torch.Tensor, + start_seq_q: Optional[torch.Tensor], + end_seq_q: Optional[torch.Tensor], max_s: int, + past_present_indices: torch.Tensor, past_key_values: Optional = None, pre_allocate_past_size: Optional[int] = None, lm_head_indices: Optional[torch.Tensor] = None, @@ -561,9 +602,12 @@ def forward( return self.model.forward( input_ids=input_ids, position_ids=position_ids, - cu_seqlens=cu_seqlens, - cu_seqlens_q=cu_seqlens_q, + start_seq=start_seq, + end_seq=end_seq, + start_seq_q=start_seq_q, + end_seq_q=end_seq_q, max_s=max_s, + past_present_indices=past_present_indices, past_key_values=past_key_values, pre_allocate_past_size=pre_allocate_past_size, lm_head_indices=lm_head_indices, @@ -575,23 +619,27 @@ def generate_token( ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: prefill = batch.past_key_values is None prefill_logprobs = batch.prefill_next_token_indices is not None - single_request = len(batch) == 1 - if prefill and single_request: + if prefill: # Ask to pre-allocate kv to its max size - # == number of tokens + max_new_tokens - pre_allocate_past_size = ( - batch.input_lengths[0] + batch.stopping_criterias[0].max_new_tokens - ) + # == Sum over batch size (number of tokens + max_new_tokens) - batch size + pre_allocate_past_size = batch.max_tokens + start_seq = batch.start_seq_prefill + end_seq = batch.end_seq_prefill else: pre_allocate_past_size = None + start_seq = batch.start_seq + end_seq = batch.end_seq out, present = self.forward( batch.input_ids, batch.position_ids, - batch.cu_seqlens, - batch.cu_seqlens_q, + start_seq, + end_seq, + batch.start_seq_q, + batch.end_seq_q, batch.max_seqlen, + batch.past_present_indices, batch.past_key_values, pre_allocate_past_size, batch.prefill_head_indices, @@ -614,55 +662,19 @@ def generate_token( # When batch == 1, we will just use the batch.input_ids values directly prefill_tokens_indices = batch.input_ids.new_zeros(len(out)) - # Create batch.cu_seqlens_q for decode - batch.cu_seqlens_q = torch.arange( - 0, len(batch) + 1, device=self.device, dtype=torch.int32 + # Create batch.start_seq_q and batch.end_seq_q for decode + batch.start_seq_q = torch.arange( + 0, len(batch), device=self.device, dtype=torch.int32 ) + batch.end_seq_q = batch.start_seq_q + 1 next_position_ids = batch.position_ids.new_empty(len(batch)) + # We do not need start_seq_prefill and end_seq_prefill anymore + batch.start_seq_prefill = None + batch.end_seq_prefill = None else: prefill_logprobs = None next_position_ids = batch.position_ids - # Prepare past for next decode - if len(batch) > 1: - # Used to slice next batch past - past_indices = torch.empty( - present.shape[1], dtype=torch.int64, device=self.device - ) - batch.past_key_values = present.new_empty( - ( - present.shape[0], - present.shape[1] + len(batch.requests), - *present.shape[2:], - ) - ) - - # It is actually faster to do a whole other for loop here as the copy from present to past is fairly slow - # and will run asynchronously while we do the next for loop - cumulative_length = 0 - for i, input_length in enumerate(batch.input_lengths): - # Indexing metadata - start_index = cumulative_length - end_index = cumulative_length + input_length - - # Indices to copy present at the correct place in past_key_values - torch.arange( - start_index + i, - end_index + i, - dtype=torch.int64, - device=self.device, - out=past_indices[start_index:end_index], - ) - cumulative_length += input_length - - # Copy from present to past_key_values - batch.past_key_values[:, past_indices] = present - - # Initialize past_key_values in prefill for len(batch) == 1 - elif prefill: - # present is already pre-padded - batch.past_key_values = present - # Cumulative length cumulative_length = 0 @@ -685,6 +697,7 @@ def generate_token( input_length, all_input_ids, ) in enumerate(iterator): + # Indexing metadata start_index = cumulative_length end_index = cumulative_length + input_length @@ -718,7 +731,8 @@ def generate_token( # Set values in batch batch.input_ids = next_input_ids batch.position_ids = next_position_ids + 1 - batch.cu_seqlens = batch.cu_seqlens + batch.cu_seqlens_q + batch.past_present_indices = batch.end_seq + batch.end_seq = batch.end_seq + 1 if prefill and prefill_logprobs: # Get prefill logprobs @@ -843,6 +857,7 @@ def generate_token( batch.prefill_head_indices = None batch.prefill_next_token_indices = None batch.max_seqlen = batch.max_seqlen + 1 + batch.past_key_values = present # No need to return a batch if we know that all requests stopped return generations, batch if not stopped else None From f59fb8b630844c2ad2cd80e689202de89d45c37e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 16 Jun 2023 16:25:11 +0200 Subject: [PATCH 280/793] feat(router): add ngrok integration (#453) --- Cargo.lock | 643 ++++++++++++------ launcher/src/main.rs | 44 ++ router/Cargo.toml | 5 + router/src/main.rs | 20 + router/src/queue.rs | 6 +- router/src/server.rs | 69 +- .../flash_santacoder_modeling.py | 1 - 7 files changed, 564 insertions(+), 224 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd5994a8..7a6f4ad2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,14 +10,13 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aes" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ "cfg-if", "cipher", "cpufeatures", - "opaque-debug", ] [[package]] @@ -40,11 +39,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "aho-corasick" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", @@ -81,9 +89,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -91,9 +99,26 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + +[[package]] +name = "arc-swap" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + +[[package]] +name = "async-rustls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93b21a03b7c21702a0110f9f8d228763a533570deb376119042dabf33c37a01a" +dependencies = [ + "futures-io", + "rustls", + "webpki", +] [[package]] name = "async-stream" @@ -114,7 +139,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -125,7 +150,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -145,11 +170,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "awaitdrop" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771051cdc7eec2dc1b23fbf870bb7fbb89136fe374227c875e377f1eed99a429" +dependencies = [ + "futures", + "generational-arena", + "parking_lot", + "slotmap", +] + [[package]] name = "axum" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b32c5ea3aabaf4deb5f5ced2d688ec0844c881c9e6c696a8b769a05fc691e62" +checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" dependencies = [ "async-trait", "axum-core", @@ -218,9 +255,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "base64ct" @@ -245,9 +282,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "bytecount" @@ -333,18 +370,19 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cipher" -version = "0.3.0" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "generic-array", + "crypto-common", + "inout", ] [[package]] name = "clap" -version = "4.2.2" +version = "4.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b802d85aaf3a1cdb02b224ba472ebdea62014fccfcb269b95a4d76443b5ee5a" +checksum = "80672091db20273a15cf9fdd4e47ed43b5091ec9841bf4c6145c9dfbbcae09ed" dependencies = [ "clap_builder", "clap_derive", @@ -353,9 +391,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.2.2" +version = "4.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a1a858f532119338887a4b8e1af9c60de8249cd7bafd68036a489e261e37b6" +checksum = "c1458a1df40e1e2afebb7ab60ce55c1fa8f431146205aa5f4887e0b111c27636" dependencies = [ "anstream", "anstyle", @@ -366,21 +404,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] name = "colorchoice" @@ -390,15 +428,15 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "console" -version = "0.15.5" +version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -425,9 +463,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" dependencies = [ "libc", ] @@ -464,9 +502,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.14" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if", @@ -477,9 +515,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] @@ -502,9 +540,9 @@ dependencies = [ [[package]] name = "crossterm_winapi" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" dependencies = [ "winapi", ] @@ -521,12 +559,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.2.5" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbcf33c2a618cbe41ee43ae6e9f2e48368cd9f9db2896f10167d8d762679f639" +checksum = "2a011bbe2c35ce9c1f143b7af6f94f29a167beb4cd1d29e6740ce836f723120e" dependencies = [ "nix", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -610,9 +648,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -728,9 +766,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "miniz_oxide", @@ -758,7 +796,7 @@ dependencies = [ "futures-sink", "nanorand", "pin-project", - "spin", + "spin 0.9.8", ] [[package]] @@ -784,9 +822,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -857,7 +895,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -890,6 +928,15 @@ dependencies = [ "slab", ] +[[package]] +name = "generational-arena" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" +dependencies = [ + "cfg-if", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -902,9 +949,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "js-sys", @@ -931,9 +978,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes", "fnv", @@ -987,6 +1034,17 @@ dependencies = [ "digest", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + [[package]] name = "http" version = "0.2.9" @@ -1084,9 +1142,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1127,6 +1185,15 @@ dependencies = [ "regex", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -1138,9 +1205,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -1209,9 +1276,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -1224,27 +1291,27 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.146" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" [[package]] name = "libm" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -1252,12 +1319,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "mach" @@ -1284,6 +1348,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + [[package]] name = "matchers" version = "0.1.0" @@ -1307,9 +1377,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] @@ -1322,7 +1392,7 @@ checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" dependencies = [ "ahash", "metrics-macros", - "portable-atomic", + "portable-atomic 0.3.20", ] [[package]] @@ -1337,7 +1407,7 @@ dependencies = [ "metrics", "metrics-util", "parking_lot", - "portable-atomic", + "portable-atomic 0.3.20", "quanta", "thiserror", "tokio", @@ -1367,7 +1437,7 @@ dependencies = [ "metrics", "num_cpus", "parking_lot", - "portable-atomic", + "portable-atomic 0.3.20", "quanta", "sketches-ddsketch", ] @@ -1396,23 +1466,23 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1433,7 +1503,7 @@ checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -1442,6 +1512,25 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "muxado" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e92b89ac3127251efde6f5a9586e5aae99468d06fcf9f133b377f58d5ed66446" +dependencies = [ + "async-trait", + "awaitdrop", + "bitflags", + "bytes", + "futures", + "pin-project", + "rand", + "thiserror", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "nanorand" version = "0.7.0" @@ -1469,6 +1558,37 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ngrok" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ce3514eec7338e2d4663e3efb4429e08d8f3662996be4b9585350e7d8ad728" +dependencies = [ + "arc-swap", + "async-rustls", + "async-trait", + "awaitdrop", + "axum", + "base64 0.13.1", + "bytes", + "futures", + "hostname", + "hyper", + "muxado", + "once_cell", + "parking_lot", + "regex", + "rustls-pemfile", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-retry", + "tokio-util", + "tracing", + "windows-sys 0.45.0", +] + [[package]] name = "nix" version = "0.26.2" @@ -1550,9 +1670,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "onig" @@ -1576,17 +1696,11 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" -version = "0.10.50" +version = "0.10.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1" +checksum = "69b3f656a17a6cbc115b5c7a40c616947d213ba182135b014d6051b73ab6f019" dependencies = [ "bitflags", "cfg-if", @@ -1605,7 +1719,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -1616,9 +1730,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.85" +version = "0.9.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0" +checksum = "c2ce0f250f34a308dcfdbb351f511359857d4ed2134ba715a4eadd46e1ffd617" dependencies = [ "cc", "libc", @@ -1714,9 +1828,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "papergrid" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdfe703c51ddc52887ad78fc69cd2ea78d895ffcd6e955c9d03566db8ab5bb1" +checksum = "ae7891b22598926e4398790c8fe6447930c72a67d36d983a49d6ce682ce83290" dependencies = [ "bytecount", "fnv", @@ -1735,15 +1849,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.3.5", "smallvec", - "windows-sys 0.45.0", + "windows-targets 0.48.0", ] [[package]] @@ -1777,9 +1891,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "petgraph" @@ -1793,22 +1907,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] @@ -1825,15 +1939,24 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" +checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e" +dependencies = [ + "portable-atomic 1.3.3", +] + +[[package]] +name = "portable-atomic" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794" [[package]] name = "ppv-lite86" @@ -1877,9 +2000,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" dependencies = [ "unicode-ident", ] @@ -1956,9 +2079,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -2079,13 +2202,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ - "aho-corasick", + "aho-corasick 1.0.2", "memchr", - "regex-syntax", + "regex-syntax 0.7.2", ] [[package]] @@ -2094,7 +2217,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -2103,13 +2226,19 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + [[package]] name = "reqwest" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "bytes", "encoding_rs", "futures-core", @@ -2140,11 +2269,26 @@ dependencies = [ "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rust-embed" -version = "6.6.1" +version = "6.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b68543d5527e158213414a92832d2aab11a84d2571a5eb021ebe22c43aab066" +checksum = "b73e721f488c353141288f223b599b4ae9303ecf3e62923f40a492f0634a4dc3" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -2153,15 +2297,15 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.5.0" +version = "6.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4e0f0ced47ded9a68374ac145edd65a6c1fa13a96447b873660b2a568a0fd7" +checksum = "e22ce362f5561923889196595504317a4372b84210e6e335da529a65ea5452b5" dependencies = [ "proc-macro2", "quote", "rust-embed-utils", "shellexpand", - "syn 1.0.109", + "syn 2.0.18", "walkdir", ] @@ -2186,9 +2330,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" dependencies = [ "bitflags", "errno", @@ -2198,6 +2342,27 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64 0.21.2", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -2234,11 +2399,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ "bitflags", "core-foundation", @@ -2249,9 +2424,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" dependencies = [ "core-foundation-sys", "libc", @@ -2265,29 +2440,29 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" dependencies = [ "itoa", "ryu", @@ -2328,9 +2503,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", @@ -2400,6 +2575,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slotmap" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e08e261d0e8f5c43123b7adf3e4ca1690d655377ac93a03b2c9d3e98de1342" +dependencies = [ + "version_check", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -2416,6 +2600,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.8" @@ -2461,9 +2651,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -2478,9 +2668,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -2495,9 +2685,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.28.4" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2f3ca6693feb29a89724516f016488e9aafc7f37264f898593ee4b942f31b" +checksum = "9557d0845b86eea8182f7b10dff120214fb6cd9fd937b6f4917714e546a38695" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2509,9 +2699,9 @@ dependencies = [ [[package]] name = "tabled" -version = "0.12.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da1a2e56bbf7bfdd08aaa7592157a742205459eff774b73bc01809ae2d99dc2a" +checksum = "0ce69a5028cd9576063ec1f48edb2c75339fd835e6094ef3e05b3a079bf594a6" dependencies = [ "papergrid", "tabled_derive", @@ -2544,15 +2734,16 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.5.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ + "autocfg", "cfg-if", "fastrand", "redox_syscall 0.3.5", "rustix", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2619,6 +2810,7 @@ dependencies = [ "futures", "metrics", "metrics-exporter-prometheus", + "ngrok", "nohash-hasher", "opentelemetry", "opentelemetry-otlp", @@ -2656,7 +2848,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -2671,9 +2863,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.20" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" dependencies = [ "itoa", "serde", @@ -2683,15 +2875,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" dependencies = [ "time-core", ] @@ -2717,7 +2909,7 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5" dependencies = [ - "aho-corasick", + "aho-corasick 0.7.20", "cached-path", "clap", "derive_builder", @@ -2736,7 +2928,7 @@ dependencies = [ "rayon", "rayon-cond", "regex", - "regex-syntax", + "regex-syntax 0.6.29", "reqwest", "serde", "serde_json", @@ -2749,9 +2941,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", @@ -2763,7 +2955,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2778,13 +2970,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -2797,11 +2989,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand", + "tokio", +] + [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -2810,12 +3013,13 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -2951,20 +3155,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -3017,9 +3221,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", "nu-ansi-term", @@ -3065,9 +3269,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -3105,11 +3309,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", "idna", @@ -3143,7 +3353,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.18", ] [[package]] @@ -3176,9 +3386,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "8.1.1" +version = "8.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b86a8af1dedf089b1c78338678e4c7492b6045649042d94faf19690499d236" +checksum = "8b3c89c2c7e50f33e4d35527e5bf9c11d6d132226dbbd1753f0fbe9f19ef88c6" dependencies = [ "anyhow", "rustc_version", @@ -3205,11 +3415,10 @@ dependencies = [ [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -3227,9 +3436,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3237,24 +3446,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if", "js-sys", @@ -3264,9 +3473,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3274,33 +3483,43 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "which" version = "4.4.0" @@ -3510,9 +3729,9 @@ dependencies = [ [[package]] name = "zip" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ "aes", "byteorder", diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 36f6f6b6..2e2bc7a5 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -229,6 +229,26 @@ struct Args { #[clap(long, env)] watermark_delta: Option, + /// Enable ngrok tunneling + #[clap(long, env)] + ngrok: bool, + + /// ngrok authentication token + #[clap(long, env)] + ngrok_authtoken: Option, + + /// ngrok domain name where the axum webserver will be available at + #[clap(long, env)] + ngrok_domain: Option, + + /// ngrok basic auth username + #[clap(long, env)] + ngrok_username: Option, + + /// ngrok basic auth password + #[clap(long, env)] + ngrok_password: Option, + /// Display a lot of information about your runtime environment #[clap(long, short, action)] env: bool, @@ -845,6 +865,30 @@ fn spawn_webserver( argv.push(origin); } + // Ngrok + if args.ngrok { + let authtoken = args.ngrok_authtoken.ok_or_else(|| { + tracing::error!("`ngrok-authtoken` must be set when using ngrok tunneling"); + LauncherError::WebserverCannotStart + })?; + + argv.push("--ngrok".to_string()); + argv.push("--ngrok-authtoken".to_string()); + argv.push(authtoken); + + if let Some(domain) = args.ngrok_domain { + argv.push("--ngrok-domain".to_string()); + argv.push(domain); + } + + if let (Some(username), Some(password)) = (args.ngrok_username, args.ngrok_password) { + argv.push("--ngrok-username".to_string()); + argv.push(username); + argv.push("--ngrok-password".to_string()); + argv.push(password); + } + } + // Copy current process env let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); diff --git a/router/Cargo.toml b/router/Cargo.toml index 6503e1bd..c1e665b1 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -40,6 +40,11 @@ tracing-opentelemetry = "0.18.0" tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } utoipa = { version = "3.0.1", features = ["axum_extras"] } utoipa-swagger-ui = { version = "3.0.2", features = ["axum"] } +ngrok = { version = "0.12.3", features = ["axum"], optional = true } [build-dependencies] vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } + +[features] +default = ["ngrok"] +ngrok = ["dep:ngrok"] \ No newline at end of file diff --git a/router/src/main.rs b/router/src/main.rs index 82bf6ba8..7bbb6477 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -56,6 +56,16 @@ struct Args { otlp_endpoint: Option, #[clap(long, env)] cors_allow_origin: Option>, + #[clap(long, env)] + ngrok: bool, + #[clap(long, env)] + ngrok_authtoken: Option, + #[clap(long, env)] + ngrok_domain: Option, + #[clap(long, env)] + ngrok_username: Option, + #[clap(long, env)] + ngrok_password: Option, } fn main() -> Result<(), std::io::Error> { @@ -80,6 +90,11 @@ fn main() -> Result<(), std::io::Error> { json_output, otlp_endpoint, cors_allow_origin, + ngrok, + ngrok_authtoken, + ngrok_domain, + ngrok_username, + ngrok_password, } = args; if validation_workers == 0 { @@ -198,6 +213,11 @@ fn main() -> Result<(), std::io::Error> { validation_workers, addr, cors_allow_origin, + ngrok, + ngrok_authtoken, + ngrok_domain, + ngrok_username, + ngrok_password, ) .await; Ok(()) diff --git a/router/src/queue.rs b/router/src/queue.rs index 03807933..0586083d 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -49,7 +49,7 @@ impl Queue { // Send append command to the background task managing the state // Unwrap is safe here self.queue_sender - .send(QueueCommand::Append(entry, Span::current())) + .send(QueueCommand::Append(Box::new(entry), Span::current())) .unwrap(); } @@ -85,7 +85,7 @@ async fn queue_task(requires_padding: bool, receiver: flume::Receiver { - span.in_scope(|| state.append(entry)); + span.in_scope(|| state.append(*entry)); metrics::increment_gauge!("tgi_queue_size", 1.0); } QueueCommand::NextBatch { @@ -256,7 +256,7 @@ type NextBatch = (IntMap, Batch, Span); #[derive(Debug)] enum QueueCommand { - Append(Entry, Span), + Append(Box, Span), NextBatch { min_size: Option, token_budget: u32, diff --git a/router/src/server.rs b/router/src/server.rs index 10c0ba3c..b8c67b2c 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -1,5 +1,5 @@ -use crate::health::Health; /// HTTP Server logic +use crate::health::Health; use crate::infer::{InferError, InferResponse, InferStreamResponse}; use crate::validation::ValidationError; use crate::{ @@ -520,6 +520,11 @@ pub async fn run( validation_workers: usize, addr: SocketAddr, allow_origin: Option, + ngrok: bool, + ngrok_authtoken: Option, + ngrok_domain: Option, + ngrok_username: Option, + ngrok_password: Option, ) { // OpenAPI documentation #[derive(OpenApi)] @@ -683,13 +688,61 @@ pub async fn run( .layer(opentelemetry_tracing_layer()) .layer(cors_layer); - // Run server - axum::Server::bind(&addr) - .serve(app.into_make_service()) - // Wait until all requests are finished to shut down - .with_graceful_shutdown(shutdown_signal()) - .await - .unwrap(); + if ngrok { + #[cfg(feature = "ngrok")] + { + use ngrok::config::TunnelBuilder; + use ngrok::tunnel::UrlTunnel; + + let _ = addr; + + let authtoken = + ngrok_authtoken.expect("`ngrok-authtoken` must be set when using ngrok tunneling"); + + let mut tunnel = ngrok::Session::builder() + .authtoken(authtoken) + .connect() + .await + .unwrap() + .http_endpoint(); + + if let Some(domain) = ngrok_domain { + tunnel = tunnel.domain(domain); + } + + if let (Some(username), Some(password)) = (ngrok_username, ngrok_password) { + tunnel = tunnel.basic_auth(username, password); + } + + let listener = tunnel.listen().await.unwrap(); + + // Run server + tracing::info!("Ingress URL: {:?}", listener.url()); + axum::Server::builder(listener) + .serve(app.into_make_service()) + //Wait until all requests are finished to shut down + .with_graceful_shutdown(shutdown_signal()) + .await + .unwrap(); + } + #[cfg(not(feature = "ngrok"))] + { + let _ngrok_authtoken = ngrok_authtoken; + let _ngrok_domain = ngrok_domain; + let _ngrok_username = ngrok_username; + let _ngrok_password = ngrok_password; + + panic!("`text-generation-router` was compiled without the `ngrok` feature"); + } + } else { + // Run server + axum::Server::bind(&addr) + .serve(app.into_make_service()) + // Wait until all requests are finished to shut down + .with_graceful_shutdown(shutdown_signal()) + .await + .unwrap(); + } } /// Shutdown signal handler diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 00cc47b6..b01d752a 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -47,7 +47,6 @@ def load_multi_mqa( shape = slice_.get_shape() block_size = (shape[0] - 2 * head_size) // world_size assert (shape[0] - 2 * head_size) % world_size == 0 - q_tensor = slice_[start:stop] start = rank * block_size stop = (rank + 1) * block_size q_tensor = slice_[start:stop] From ece7ffa40a7e167400e57d89b8a73751d095184c Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 19 Jun 2023 09:53:45 +0200 Subject: [PATCH 281/793] feat(server): improve flash attention import errors (#465) @lewtun, is this enough? Closes #458 Closes #456 --- .../text_generation_server/models/__init__.py | 73 ++++++++++--------- .../text_generation_server/utils/convert.py | 4 +- server/text_generation_server/utils/hub.py | 5 +- 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index f1b84a53..9540d99e 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -18,11 +18,43 @@ from text_generation_server.models.t5 import T5Sharded from text_generation_server.models.gpt_neox import GPTNeoxSharded +# The flag below controls whether to allow TF32 on matmul. This flag defaults to False +# in PyTorch 1.12 and later. +torch.backends.cuda.matmul.allow_tf32 = True + +# The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. +torch.backends.cudnn.allow_tf32 = True + +# Disable gradients +torch.set_grad_enabled(False) + +__all__ = [ + "Model", + "BLOOMSharded", + "CausalLM", + "FlashCausalLM", + "GalacticaSharded", + "Seq2SeqLM", + "SantaCoder", + "OPTSharded", + "T5Sharded", + "get_model", +] + +FLASH_ATT_ERROR_MESSAGE = ( + "{} requires CUDA and Flash Attention kernels to be installed.\n" + "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) " + "or install flash attention with `cd server && make install install-flash-attention`" +) + try: - if ( - torch.cuda.is_available() - and not os.getenv("USE_FLASH_ATTENTION", "").lower() == "false" - ): + if not os.getenv("USE_FLASH_ATTENTION", "").lower() == "false": + if not torch.cuda.is_available(): + FLASH_ATT_ERROR_MESSAGE = ( + "{} requires CUDA. No compatible CUDA devices found." + ) + raise ImportError("CUDA is not available") + major, minor = torch.cuda.get_device_capability() is_sm75 = major == 7 and minor == 5 is_sm8x = major == 8 and minor >= 0 @@ -30,6 +62,10 @@ supported = is_sm75 or is_sm8x or is_sm90 if not supported: + FLASH_ATT_ERROR_MESSAGE = ( + "{} requires a CUDA device with capability 7.5, > 8.0 or 9.0. " + "No compatible CUDA device found." + ) raise ImportError( f"GPU with CUDA capability {major} {minor} is not supported" ) @@ -52,41 +88,12 @@ ) FLASH_ATTENTION = False -__all__ = [ - "Model", - "BLOOMSharded", - "CausalLM", - "FlashCausalLM", - "GalacticaSharded", - "Seq2SeqLM", - "SantaCoder", - "OPTSharded", - "T5Sharded", - "get_model", -] - if FLASH_ATTENTION: __all__.append(FlashNeoXSharded) __all__.append(FlashRWSharded) __all__.append(FlashSantacoderSharded) __all__.append(FlashLlama) -FLASH_ATT_ERROR_MESSAGE = ( - "{} requires Flash Attention CUDA kernels to be installed.\n" - "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) " - "or install flash attention with `cd server && make install install-flash-attention`" -) - -# The flag below controls whether to allow TF32 on matmul. This flag defaults to False -# in PyTorch 1.12 and later. -torch.backends.cuda.matmul.allow_tf32 = True - -# The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. -torch.backends.cudnn.allow_tf32 = True - -# Disable gradients -torch.set_grad_enabled(False) - def get_model( model_id: str, diff --git a/server/text_generation_server/utils/convert.py b/server/text_generation_server/utils/convert.py index c43a4464..c4e79432 100644 --- a/server/text_generation_server/utils/convert.py +++ b/server/text_generation_server/utils/convert.py @@ -16,9 +16,9 @@ def check_file_size(source_file: Path, target_file: Path): source_file_size = source_file.stat().st_size target_file_size = target_file.stat().st_size - if (source_file_size - target_file_size) / source_file_size > 0.01: + if (source_file_size - target_file_size) / source_file_size > 0.05: raise RuntimeError( - f"""The file size different is more than 1%: + f"""The file size different is more than 5%: - {source_file}: {source_file_size} - {target_file}: {target_file_size} """ diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index 2ed7673c..fbb570a6 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -26,7 +26,10 @@ def weight_hub_files( filenames = [ s.rfilename for s in info.siblings - if s.rfilename.endswith(extension) and len(s.rfilename.split("/")) == 1 + if s.rfilename.endswith(extension) + and len(s.rfilename.split("/")) == 1 + and "arguments" not in s.rfilename + and "args" not in s.rfilename ] if not filenames: From 53aa9194c8c070afd19fa4660305dab2b280adf3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 20 Jun 2023 11:06:10 +0200 Subject: [PATCH 282/793] fix(server): fix warpers on CPU (#472) Closes #471 --- .../text_generation_server/models/__init__.py | 20 +++------ .../utils/logits_process.py | 44 +++++++++++-------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 9540d99e..3fdc23b2 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -237,20 +237,12 @@ def get_model( ) elif model_type == "t5": - if sharded: - return T5Sharded( - model_id, - revision, - quantize=quantize, - trust_remote_code=trust_remote_code, - ) - else: - return Seq2SeqLM( - model_id, - revision, - quantize=quantize, - trust_remote_code=trust_remote_code, - ) + return T5Sharded( + model_id, + revision, + quantize=quantize, + trust_remote_code=trust_remote_code, + ) if sharded: raise ValueError("sharded is not supported for AutoModel") diff --git a/server/text_generation_server/utils/logits_process.py b/server/text_generation_server/utils/logits_process.py index faa94516..0cbbf8b0 100644 --- a/server/text_generation_server/utils/logits_process.py +++ b/server/text_generation_server/utils/logits_process.py @@ -42,25 +42,31 @@ def __init__( self.static_next_logprob = None def __call__(self, scores): - if self.cuda_graph is None: - self.static_scores = scores - self.cuda_graph = torch.cuda.CUDAGraph() - - with torch.cuda.graph(self.cuda_graph, pool=mempool): - local_scores = self.static_scores - for warper in self.warpers: - local_scores = warper(None, local_scores) - - self.static_warped_scores = local_scores - # Compute logprobs - self.static_next_logprob = torch.log_softmax( - self.static_warped_scores, -1 - ) - - self.static_scores.copy_(scores) - self.cuda_graph.replay() - - return self.static_warped_scores, self.static_next_logprob + if torch.cuda.is_available(): + if self.cuda_graph is None: + self.static_scores = scores + self.cuda_graph = torch.cuda.CUDAGraph() + + with torch.cuda.graph(self.cuda_graph, pool=mempool): + local_scores = self.static_scores + for warper in self.warpers: + local_scores = warper(None, local_scores) + + self.static_warped_scores = local_scores + # Compute logprobs + self.static_next_logprob = torch.log_softmax( + self.static_warped_scores, -1 + ) + + self.static_scores.copy_(scores) + self.cuda_graph.replay() + + return self.static_warped_scores, self.static_next_logprob + + # CPU branch + for warper in self.warpers: + scores = warper(None, scores) + return scores, torch.log_softmax(scores, -1) @lru_cache(10) From c9c65ab323f48731e1fc2f7087547a7bd8b753f2 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 20 Jun 2023 18:03:36 +0200 Subject: [PATCH 283/793] fix(server): Fixing T5 in case the names are mixed up. (#475) --- .../models/custom_modeling/t5_modeling.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/custom_modeling/t5_modeling.py b/server/text_generation_server/models/custom_modeling/t5_modeling.py index 51862e3c..12679e9d 100644 --- a/server/text_generation_server/models/custom_modeling/t5_modeling.py +++ b/server/text_generation_server/models/custom_modeling/t5_modeling.py @@ -1001,7 +1001,10 @@ def __init__(self, config: T5Config, weights): super().__init__(config) self.model_dim = config.d_model - self.shared = TensorParallelEmbedding(prefix="shared", weights=weights) + try: + self.shared = TensorParallelEmbedding(prefix="shared", weights=weights) + except RuntimeError: + self.shared = TensorParallelEmbedding(prefix="encoder.embed_tokens", weights=weights) encoder_config = copy.deepcopy(config) encoder_config.is_decoder = False From 49b4b33e805d0ffee62688fe2607120b0c759e3d Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 23 Jun 2023 12:40:46 +0200 Subject: [PATCH 284/793] feat(server): Update convert logic. (#483) Should be more robust to shared tensors (ok when using `from_pretrained). But forcing us to add new checks in our loading code (since the chosen key to keep might be different from `transformers`). --------- Co-authored-by: Ubuntu --- .../models/flash_santacoder.py | 3 +- .../text_generation_server/utils/convert.py | 93 +++++++------------ .../text_generation_server/utils/weights.py | 22 +++-- 3 files changed, 48 insertions(+), 70 deletions(-) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 54634e4a..a71c0061 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -52,7 +52,8 @@ def __init__( torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights( - filenames, device=device, dtype=dtype, process_group=self.process_group + filenames, device=device, dtype=dtype, process_group=self.process_group, + aliases = {"transformer.wte.weight": ["lm_head.weight"]} ) model = FlashSantacoderForCausalLM(config, weights) diff --git a/server/text_generation_server/utils/convert.py b/server/text_generation_server/utils/convert.py index c4e79432..0e4adaba 100644 --- a/server/text_generation_server/utils/convert.py +++ b/server/text_generation_server/utils/convert.py @@ -1,76 +1,45 @@ import datetime import torch +import os -from collections import defaultdict from loguru import logger from pathlib import Path -from safetensors.torch import save_file -from safetensors import safe_open -from typing import Dict, List - - -def check_file_size(source_file: Path, target_file: Path): - """ - Check that two files are close in size - """ - source_file_size = source_file.stat().st_size - target_file_size = target_file.stat().st_size - - if (source_file_size - target_file_size) / source_file_size > 0.05: - raise RuntimeError( - f"""The file size different is more than 5%: - - {source_file}: {source_file_size} - - {target_file}: {target_file_size} - """ - ) - - -def remove_shared_pointers(tensors: Dict[str, torch.Tensor]): - """ - For a Dict of tensors, check if two or more tensors point to the same underlying memory and - remove them - """ - ptrs = defaultdict(list) - for k, v in tensors.items(): - ptrs[v.data_ptr()].append(k) - - # Iterate over all found memory addresses - for ptr, names in ptrs.items(): - if len(names) > 1: - # Multiple tensors are point to the same memory - # Only keep the first tensor - for name in names[1:]: - tensors.pop(name) +from safetensors.torch import save_file, _remove_duplicate_names, load_file +from typing import List def convert_file(pt_file: Path, sf_file: Path): """ Convert a pytorch file to a safetensors file - """ - logger.info(f"Convert {pt_file} to {sf_file}.") - - pt_state = torch.load(pt_file, map_location="cpu") - if "state_dict" in pt_state: - pt_state = pt_state["state_dict"] - - remove_shared_pointers(pt_state) + This will remove duplicate tensors from the file. - # Tensors need to be contiguous - pt_state = {k: v.contiguous() for k, v in pt_state.items()} - - sf_file.parent.mkdir(parents=True, exist_ok=True) - save_file(pt_state, str(sf_file), metadata={"format": "pt"}) - - # Check that both files are close in size - check_file_size(pt_file, sf_file) - - # Load safetensors state - for k in pt_state: - pt_tensor = pt_state[k] - with safe_open(sf_file, framework="pt") as f: - sf_tensor = f.get_tensor(k) - if not torch.equal(pt_tensor, sf_tensor): - raise RuntimeError(f"The output tensors do not match for key {k}") + Unfortunately, this might not respect *transformers* convention. + Forcing us to check for potentially different keys during load when looking + for specific tensors (making tensor sharing explicit). + """ + loaded = torch.load(pt_file, map_location="cpu") + if "state_dict" in loaded: + loaded = loaded["state_dict"] + to_removes = _remove_duplicate_names(loaded) + + metadata = {"format": "pt"} + for kept_name, to_remove_group in to_removes.items(): + for to_remove in to_remove_group: + if to_remove not in metadata: + metadata[to_remove] = kept_name + del loaded[to_remove] + # Force tensors to be contiguous + loaded = {k: v.contiguous() for k, v in loaded.items()} + + dirname = os.path.dirname(sf_file) + os.makedirs(dirname, exist_ok=True) + save_file(loaded, sf_file, metadata=metadata) + reloaded = load_file(sf_file) + for k in loaded: + pt_tensor = loaded[k] + sf_tensor = reloaded[k] + if not torch.equal(pt_tensor, sf_tensor): + raise RuntimeError(f"The output tensors do not match for key {k}") def convert_files(pt_files: List[Path], sf_files: List[Path]): diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 76a4f65a..88347a6a 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -1,10 +1,10 @@ from pathlib import Path -from typing import List +from typing import List, Dict, Optional from safetensors import safe_open class Weights: - def __init__(self, filenames: List[Path], device, dtype, process_group): + def __init__(self, filenames: List[Path], device, dtype, process_group, aliases: Optional[Dict[str, List[str]]]=None): routing = {} for filename in filenames: with safe_open(filename, framework="pytorch") as f: @@ -14,6 +14,9 @@ def __init__(self, filenames: List[Path], device, dtype, process_group): f"Key {k} was found in multiple files: {filename} and {routing[k]}" ) routing[k] = filename + if aliases is None: + aliases = {} + self.aliases = aliases self.routing = routing self.device = device self.dtype = dtype @@ -27,14 +30,19 @@ def _get_handle(self, filename): return self._handles[filename] - def get_filename(self, tensor_name: str) -> str: + def get_filename(self, tensor_name: str) -> (str, str): filename = self.routing.get(tensor_name, None) if filename is None: + aliases = self.aliases.get(tensor_name, []) + for alias in aliases: + filename = self.routing.get(alias, None) + if filename is not None: + return str(filename), alias raise RuntimeError(f"weight {tensor_name} does not exist") - return str(filename) + return str(filename), tensor_name def _get_slice(self, tensor_name: str): - filename = self.get_filename(tensor_name) + filename, tensor_name= self.get_filename(tensor_name) f = self._get_handle(filename) slice_ = f.get_slice(tensor_name) return slice_ @@ -43,7 +51,7 @@ def get_shape(self, tensor_name: str): return self._get_slice(tensor_name).get_shape() def get_tensor(self, tensor_name: str): - filename = self.get_filename(tensor_name) + filename, tensor_name = self.get_filename(tensor_name) f = self._get_handle(filename) tensor = f.get_tensor(tensor_name) tensor = tensor.to(dtype=self.dtype) @@ -51,7 +59,7 @@ def get_tensor(self, tensor_name: str): return tensor def get_sharded(self, tensor_name: str, dim: int): - filename = self.get_filename(tensor_name) + filename, tensor_name = self.get_filename(tensor_name) world_size = self.process_group.size() rank = self.process_group.rank() From 776d150c55d3d973a457a80b05440d593942752a Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 23 Jun 2023 12:41:13 +0200 Subject: [PATCH 285/793] feat(server): Adding new ignore_rule for conversion. (#485) --- server/text_generation_server/utils/hub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index fbb570a6..23743c9b 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -30,6 +30,7 @@ def weight_hub_files( and len(s.rfilename.split("/")) == 1 and "arguments" not in s.rfilename and "args" not in s.rfilename + and "training" not in s.rfilename ] if not filenames: From bd3a9d8e856cb7e2122f1a09d2fb0f44b7649dad Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 23 Jun 2023 14:58:28 +0200 Subject: [PATCH 286/793] fix(router): add timeout on flume sends (#488) --- router/src/infer.rs | 45 ++++++++++++++++++++++++++++++--------------- router/src/queue.rs | 2 +- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index 00fa2818..f738f986 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -3,7 +3,7 @@ use crate::validation::{Validation, ValidationError}; use crate::{Entry, Queue, Token}; use crate::{GenerateRequest, PrefillToken}; use flume::r#async::RecvStream; -use flume::SendError; +use flume::SendTimeoutError; use futures::future::try_join_all; use futures::stream::StreamExt; use nohash_hasher::IntMap; @@ -11,6 +11,7 @@ use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, }; +use std::time::Duration; use text_generation_client::{ Batch, CachedBatch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, }; @@ -472,6 +473,10 @@ fn filter_send_generations(generations: Vec, entries: &mut IntMap "dropped"); err }).unwrap_or(true); @@ -485,14 +490,20 @@ fn filter_send_generations(generations: Vec, entries: &mut IntMap Result>> { +) -> Result>>> { + // Return directly if the channel is disconnected + if entry.response_tx.is_disconnected() { + return Ok(true); + } + let mut stopped = false; if let Some(prefill_tokens) = generation.prefill_tokens { // Send message - entry - .response_tx - .send(Ok(InferStreamResponse::Prefill(prefill_tokens)))?; + entry.response_tx.send_timeout( + Ok(InferStreamResponse::Prefill(prefill_tokens)), + Duration::from_millis(10), + )?; } // Create last Token @@ -507,17 +518,21 @@ fn send_responses( // Generation has ended stopped = true; // Send message - entry.response_tx.send(Ok(InferStreamResponse::End { - token, - generated_text, - queued: entry.queue_time, - start: entry.batch_time.unwrap(), - }))?; + entry.response_tx.send_timeout( + Ok(InferStreamResponse::End { + token, + generated_text, + queued: entry.queue_time, + start: entry.batch_time.unwrap(), + }), + Duration::from_millis(10), + )?; } else { // Send message - entry - .response_tx - .send(Ok(InferStreamResponse::Token(token)))?; + entry.response_tx.send_timeout( + Ok(InferStreamResponse::Token(token)), + Duration::from_millis(10), + )?; } Ok(stopped) } @@ -535,7 +550,7 @@ fn send_errors(error: ClientError, entries: &mut IntMap) { // unwrap_or is valid here as we don't care if the receiver is gone. entry .response_tx - .send(Err(err)) + .send_timeout(Err(err), Duration::from_millis(10)) .unwrap_or(()); }); } diff --git a/router/src/queue.rs b/router/src/queue.rs index 0586083d..6d1d4d12 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -95,7 +95,7 @@ async fn queue_task(requires_padding: bool, receiver: flume::Receiver span.in_scope(|| { let next_batch = state.next_batch(min_size, token_budget); - response_sender.send(next_batch).unwrap_or(()); + response_sender.send(next_batch).unwrap(); metrics::gauge!("tgi_queue_size", state.entries.len() as f64); }), } From aefde28b45cad8c2e93f91f690ecc60eee7bd75c Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 26 Jun 2023 12:27:01 +0200 Subject: [PATCH 287/793] feat(server): Add inference support for GPTQ (llama + falcon tested) + Quantization script (#438) Let's start discussing implementation. - Need to expose the quantization scripts (either included here or add doc on how to use https://github.com/qwopqwop200/GPTQ-for-LLaMa) - Make sure GPTQ works for multiple models (priority to Falcon). Currently it means that every place we use `get_{tensor|sharded}` to check for quantization. My idea is to reintegrate as much as possible into `utils/layer.py` by expanding `load_multi` to be a bit more generic. This might require some thinking, but ultimately the `qweight,qzeros,scales,g_idx` should be in a single place, and independant of bias presence. # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Ubuntu Co-authored-by: OlivierDehaene --- Dockerfile | 5 + server/poetry.lock | 1554 +++++++++-------- server/requirements.txt | 29 +- server/text_generation_server/cli.py | 32 + .../text_generation_server/models/__init__.py | 4 + .../custom_modeling/flash_neox_modeling.py | 29 +- .../custom_modeling/flash_rw_modeling.py | 3 +- .../flash_santacoder_modeling.py | 81 +- .../models/flash_llama.py | 25 +- .../utils/gptq/custom_autotune.py | 261 +++ .../utils/gptq/quant_linear.py | 359 ++++ .../utils/gptq/quantize.py | 866 +++++++++ server/text_generation_server/utils/layers.py | 47 +- .../text_generation_server/utils/weights.py | 51 +- 14 files changed, 2571 insertions(+), 775 deletions(-) create mode 100644 server/text_generation_server/utils/gptq/custom_autotune.py create mode 100644 server/text_generation_server/utils/gptq/quant_linear.py create mode 100644 server/text_generation_server/utils/gptq/quantize.py diff --git a/Dockerfile b/Dockerfile index 576dab8d..2a313c25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -159,6 +159,11 @@ COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bi # Install launcher COPY --from=builder /usr/src/target/release/text-generation-launcher /usr/local/bin/text-generation-launcher +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + g++ \ + && rm -rf /var/lib/apt/lists/* + # AWS Sagemaker compatbile image FROM base as sagemaker diff --git a/server/poetry.lock b/server/poetry.lock index 5d853ce2..9a6900bc 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,10 +1,15 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + [[package]] name = "accelerate" version = "0.19.0" description = "Accelerate" -category = "main" optional = true python-versions = ">=3.7.0" +files = [ + {file = "accelerate-0.19.0-py3-none-any.whl", hash = "sha256:2866b0bf9fff08f51e6384c95fa96725838b70f1988d1cce42e56b820d8a91dd"}, + {file = "accelerate-0.19.0.tar.gz", hash = "sha256:84920226b9e642e453ef37593ee55b956b08d8200dea4087c546c34e26157e76"}, +] [package.dependencies] numpy = ">=1.17" @@ -18,50 +23,138 @@ dev = ["black (>=23.1,<24.0)", "datasets", "deepspeed", "evaluate", "hf-doc-buil quality = ["black (>=23.1,<24.0)", "hf-doc-builder (>=0.3.0)", "ruff (>=0.0.241)", "urllib3 (<2.0.0)"] rich = ["rich"] sagemaker = ["sagemaker"] -test_dev = ["datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] -test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] -test_trackers = ["comet-ml", "tensorboard", "wandb"] +test-dev = ["datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] +test-prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] +test-trackers = ["comet-ml", "tensorboard", "wandb"] testing = ["datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] [[package]] name = "backoff" version = "2.2.1" description = "Function decoration for backoff and retry" -category = "main" optional = false python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] [[package]] name = "bitsandbytes" version = "0.38.1" description = "8-bit optimizers and matrix multiplication routines." -category = "main" optional = true python-versions = "*" +files = [ + {file = "bitsandbytes-0.38.1-py3-none-any.whl", hash = "sha256:5f532e7b1353eb7049ae831da2eb62ed8a1e0444116bd51b9e088a6e0bc7a34a"}, + {file = "bitsandbytes-0.38.1.tar.gz", hash = "sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce"}, +] [[package]] name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, + {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, +] [[package]] name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, +] [[package]] name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -70,54 +163,69 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] -name = "Deprecated" -version = "1.2.13" +name = "deprecated" +version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] [package.dependencies] wrapt = ">=1.10,<2" [package.extras] -dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version (<1)", "configparser (<5)", "importlib-metadata (<3)", "importlib-resources (<4)", "sphinx (<2)", "sphinxcontrib-websupport (<2)", "tox", "zipp (<2)"] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "exceptiongroup" version = "1.1.1" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, +] [package.extras] test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.12.0" +version = "3.12.2" description = "A platform independent file lock." -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, + {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, +] [package.extras] -docs = ["furo (>=2023.3.27)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] [[package]] name = "fsspec" -version = "2023.5.0" +version = "2023.6.0" description = "File-system specification" -category = "main" optional = false python-versions = ">=3.8" +files = [ + {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"}, + {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"}, +] [package.extras] abfs = ["adlfs"] @@ -145,25 +253,31 @@ tqdm = ["tqdm"] [[package]] name = "googleapis-common-protos" -version = "1.59.0" +version = "1.59.1" description = "Common protobufs used in Google APIs" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.59.1.tar.gz", hash = "sha256:b35d530fe825fb4227857bc47ad84c33c809ac96f312e13182bdeaa2abe1178a"}, + {file = "googleapis_common_protos-1.59.1-py2.py3-none-any.whl", hash = "sha256:0cbedb6fb68f1c07e18eb4c48256320777707e7d0c55063ae56c15db3224a61e"}, +] [package.dependencies] -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" [package.extras] -grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "grpc-interceptor" version = "0.15.2" description = "Simplifies gRPC interceptors" -category = "main" optional = false python-versions = ">=3.7,<4.0" +files = [ + {file = "grpc-interceptor-0.15.2.tar.gz", hash = "sha256:5c984110af4fb77d03472ec0468f9c77ddaf798e190410fb7b7f1e76c60c96a4"}, + {file = "grpc_interceptor-0.15.2-py3-none-any.whl", hash = "sha256:596dac3cb709ffb6178a4873f5148e254c871c9069f0b11040189b257969490a"}, +] [package.dependencies] grpcio = ">=1.49.1,<2.0.0" @@ -173,50 +287,148 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.55.0" +version = "1.54.2" description = "HTTP/2-based RPC framework" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "grpcio-1.54.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:40e1cbf69d6741b40f750f3cccc64326f927ac6145a9914d33879e586002350c"}, + {file = "grpcio-1.54.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2288d76e4d4aa7ef3fe7a73c1c470b66ea68e7969930e746a8cd8eca6ef2a2ea"}, + {file = "grpcio-1.54.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:c0e3155fc5335ec7b3b70f15230234e529ca3607b20a562b6c75fb1b1218874c"}, + {file = "grpcio-1.54.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bf88004fe086c786dc56ef8dd6cb49c026833fdd6f42cb853008bce3f907148"}, + {file = "grpcio-1.54.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2be88c081e33f20630ac3343d8ad9f1125f32987968e9c8c75c051c9800896e8"}, + {file = "grpcio-1.54.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:33d40954199bddbb6a78f8f6f2b2082660f381cd2583ec860a6c2fa7c8400c08"}, + {file = "grpcio-1.54.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b52d00d1793d290c81ad6a27058f5224a7d5f527867e5b580742e1bd211afeee"}, + {file = "grpcio-1.54.2-cp310-cp310-win32.whl", hash = "sha256:881d058c5ccbea7cc2c92085a11947b572498a27ef37d3eef4887f499054dca8"}, + {file = "grpcio-1.54.2-cp310-cp310-win_amd64.whl", hash = "sha256:0212e2f7fdf7592e4b9d365087da30cb4d71e16a6f213120c89b4f8fb35a3ab3"}, + {file = "grpcio-1.54.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:1e623e0cf99a0ac114f091b3083a1848dbc64b0b99e181473b5a4a68d4f6f821"}, + {file = "grpcio-1.54.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:66233ccd2a9371158d96e05d082043d47dadb18cbb294dc5accfdafc2e6b02a7"}, + {file = "grpcio-1.54.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:4cb283f630624ebb16c834e5ac3d7880831b07cbe76cb08ab7a271eeaeb8943e"}, + {file = "grpcio-1.54.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a1e601ee31ef30a9e2c601d0867e236ac54c922d32ed9f727b70dd5d82600d5"}, + {file = "grpcio-1.54.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8da84bbc61a4e92af54dc96344f328e5822d574f767e9b08e1602bb5ddc254a"}, + {file = "grpcio-1.54.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5008964885e8d23313c8e5ea0d44433be9bfd7e24482574e8cc43c02c02fc796"}, + {file = "grpcio-1.54.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a2f5a1f1080ccdc7cbaf1171b2cf384d852496fe81ddedeb882d42b85727f610"}, + {file = "grpcio-1.54.2-cp311-cp311-win32.whl", hash = "sha256:b74ae837368cfffeb3f6b498688a123e6b960951be4dec0e869de77e7fa0439e"}, + {file = "grpcio-1.54.2-cp311-cp311-win_amd64.whl", hash = "sha256:8cdbcbd687e576d48f7886157c95052825ca9948c0ed2afdc0134305067be88b"}, + {file = "grpcio-1.54.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:782f4f8662a2157c4190d0f99eaaebc602899e84fb1e562a944e5025929e351c"}, + {file = "grpcio-1.54.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:714242ad0afa63a2e6dabd522ae22e1d76e07060b5af2ddda5474ba4f14c2c94"}, + {file = "grpcio-1.54.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:f900ed4ad7a0f1f05d35f955e0943944d5a75f607a836958c6b8ab2a81730ef2"}, + {file = "grpcio-1.54.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96a41817d2c763b1d0b32675abeb9179aa2371c72aefdf74b2d2b99a1b92417b"}, + {file = "grpcio-1.54.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70fcac7b94f4c904152809a050164650ac81c08e62c27aa9f156ac518029ebbe"}, + {file = "grpcio-1.54.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fd6c6c29717724acf9fc1847c4515d57e4dc12762452457b9cb37461f30a81bb"}, + {file = "grpcio-1.54.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c2392f5b5d84b71d853918687d806c1aa4308109e5ca158a16e16a6be71041eb"}, + {file = "grpcio-1.54.2-cp37-cp37m-win_amd64.whl", hash = "sha256:51630c92591d6d3fe488a7c706bd30a61594d144bac7dee20c8e1ce78294f474"}, + {file = "grpcio-1.54.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:b04202453941a63b36876a7172b45366dc0cde10d5fd7855c0f4a4e673c0357a"}, + {file = "grpcio-1.54.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:89dde0ac72a858a44a2feb8e43dc68c0c66f7857a23f806e81e1b7cc7044c9cf"}, + {file = "grpcio-1.54.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:09d4bfd84686cd36fd11fd45a0732c7628308d094b14d28ea74a81db0bce2ed3"}, + {file = "grpcio-1.54.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7fc2b4edb938c8faa4b3c3ea90ca0dd89b7565a049e8e4e11b77e60e4ed2cc05"}, + {file = "grpcio-1.54.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61f7203e2767800edee7a1e1040aaaf124a35ce0c7fe0883965c6b762defe598"}, + {file = "grpcio-1.54.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e416c8baf925b5a1aff31f7f5aecc0060b25d50cce3a5a7255dc5cf2f1d4e5eb"}, + {file = "grpcio-1.54.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dc80c9c6b608bf98066a038e0172013a49cfa9a08d53335aefefda2c64fc68f4"}, + {file = "grpcio-1.54.2-cp38-cp38-win32.whl", hash = "sha256:8d6192c37a30a115f4663592861f50e130caed33efc4eec24d92ec881c92d771"}, + {file = "grpcio-1.54.2-cp38-cp38-win_amd64.whl", hash = "sha256:46a057329938b08e5f0e12ea3d7aed3ecb20a0c34c4a324ef34e00cecdb88a12"}, + {file = "grpcio-1.54.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:2296356b5c9605b73ed6a52660b538787094dae13786ba53080595d52df13a98"}, + {file = "grpcio-1.54.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:c72956972e4b508dd39fdc7646637a791a9665b478e768ffa5f4fe42123d5de1"}, + {file = "grpcio-1.54.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:9bdbb7624d65dc0ed2ed8e954e79ab1724526f09b1efa88dcd9a1815bf28be5f"}, + {file = "grpcio-1.54.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c44e1a765b31e175c391f22e8fc73b2a2ece0e5e6ff042743d8109b5d2eff9f"}, + {file = "grpcio-1.54.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cc928cfe6c360c1df636cf7991ab96f059666ac7b40b75a769410cc6217df9c"}, + {file = "grpcio-1.54.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a08920fa1a97d4b8ee5db2f31195de4a9def1a91bc003544eb3c9e6b8977960a"}, + {file = "grpcio-1.54.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4864f99aac207e3e45c5e26c6cbb0ad82917869abc2f156283be86c05286485c"}, + {file = "grpcio-1.54.2-cp39-cp39-win32.whl", hash = "sha256:b38b3de8cff5bc70f8f9c615f51b48eff7313fc9aca354f09f81b73036e7ddfa"}, + {file = "grpcio-1.54.2-cp39-cp39-win_amd64.whl", hash = "sha256:be48496b0e00460717225e7680de57c38be1d8629dc09dadcd1b3389d70d942b"}, + {file = "grpcio-1.54.2.tar.gz", hash = "sha256:50a9f075eeda5097aa9a182bb3877fe1272875e45370368ac0ee16ab9e22d019"}, +] [package.extras] -protobuf = ["grpcio-tools (>=1.55.0)"] +protobuf = ["grpcio-tools (>=1.54.2)"] [[package]] name = "grpcio-reflection" -version = "1.55.0" +version = "1.54.2" description = "Standard Protobuf Reflection Service for gRPC" -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "grpcio-reflection-1.54.2.tar.gz", hash = "sha256:b2e021e1ce4f075615411edfbbd6fdcc485ba474dd6e5a3f559690582959a673"}, + {file = "grpcio_reflection-1.54.2-py3-none-any.whl", hash = "sha256:e7759addebbd90768f3a0278320278145758c4687d9e2cd7d76e7cbd0e329274"}, +] [package.dependencies] -grpcio = ">=1.55.0" +grpcio = ">=1.54.2" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.55.0" +version = "1.54.2" description = "Status proto mapping for gRPC" -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "grpcio-status-1.54.2.tar.gz", hash = "sha256:3255cbec5b7c706caa3d4dd584606c080e6415e15631bb2f6215e2b70055836d"}, + {file = "grpcio_status-1.54.2-py3-none-any.whl", hash = "sha256:2a7cb4838225f1b53bd0448a3008c5b5837941e1f3a0b13fa38768f08a7b68c2"}, +] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.55.0" +grpcio = ">=1.54.2" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.55.0" +version = "1.54.2" description = "Protobuf code generator for gRPC" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "grpcio-tools-1.54.2.tar.gz", hash = "sha256:e11c2c2aee53f340992e8e4d6a59172cbbbd0193f1351de98c4f810a5041d5ca"}, + {file = "grpcio_tools-1.54.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:2b96f5f17d3156058be247fd25b062b4768138665694c00b056659618b8fb418"}, + {file = "grpcio_tools-1.54.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:11939c9a8a39bd4815c7e88cb2fee48e1948775b59dbb06de8fcae5991e84f9e"}, + {file = "grpcio_tools-1.54.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:129de5579f95d6a55dde185f188b4cbe19d1e2f1471425431d9930c31d300d70"}, + {file = "grpcio_tools-1.54.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4128c01cd6f5ea8f7c2db405dbfd8582cd967d36e6fa0952565436633b0e591"}, + {file = "grpcio_tools-1.54.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5c7292dd899ad8fa09a2be96719648cee37b17909fe8c12007e3bff58ebee61"}, + {file = "grpcio_tools-1.54.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5ef30c2dbc63c1e0a462423ca4f95001814d26ef4fe66208e53fcf220ea3b717"}, + {file = "grpcio_tools-1.54.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4abfc1892380abe6cef381eab86f9350cbd703bfe5d834095aa66fd91c886b6d"}, + {file = "grpcio_tools-1.54.2-cp310-cp310-win32.whl", hash = "sha256:9acf443dcf6f68fbea3b7fb519e1716e014db1a561939f5aecc4abda74e4015d"}, + {file = "grpcio_tools-1.54.2-cp310-cp310-win_amd64.whl", hash = "sha256:21b9d2dee80f3f77e4097252e7f0db89772335a7300b72ab3d2e5c280872b1db"}, + {file = "grpcio_tools-1.54.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:7b24fbab9e7598518ce4549e066df00aab79c2bf9bedcdde23fb5ef6a3cf532f"}, + {file = "grpcio_tools-1.54.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:7baa210c20f71a242d9ae0e02734628f6948e8bee3bf538647894af427d28800"}, + {file = "grpcio_tools-1.54.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:e3d0e5188ff8dbaddac2ee44731d36f09c4eccd3eac7328e547862c44f75cacd"}, + {file = "grpcio_tools-1.54.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27671c68c7e0e3c5ff9967f5500799f65a04e7b153b8ce10243c87c43199039d"}, + {file = "grpcio_tools-1.54.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f39d8e8806b8857fb473ca6a9c7bd800b0673dfdb7283ff569af0345a222f32c"}, + {file = "grpcio_tools-1.54.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8e4c5a48f7b2e8798ce381498ee7b9a83c65b87ae66ee5022387394e5eb51771"}, + {file = "grpcio_tools-1.54.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4f285f8ef3de422717a36bd372239ae778b8cc112ce780ca3c7fe266dadc49fb"}, + {file = "grpcio_tools-1.54.2-cp311-cp311-win32.whl", hash = "sha256:0f952c8a5c47e9204fe8959f7e9add149e660f6579d67cf65024c32736d34caf"}, + {file = "grpcio_tools-1.54.2-cp311-cp311-win_amd64.whl", hash = "sha256:3237149beec39e897fd62cef4aa1e1cd9422d7a95661d24bd0a79200b167e730"}, + {file = "grpcio_tools-1.54.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:0ab1b323905d449298523db5d34fa5bf5fffd645bd872b25598e2f8a01f0ea39"}, + {file = "grpcio_tools-1.54.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:7d7e6e8d62967b3f037f952620cb7381cc39a4bd31790c75fcfba56cc975d70b"}, + {file = "grpcio_tools-1.54.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:7f4624ef2e76a3a5313c4e61a81be38bcc16b59a68a85d30758b84cd2102b161"}, + {file = "grpcio_tools-1.54.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e543f457935ba7b763b121f1bf893974393b4d30065042f947f85a8d81081b80"}, + {file = "grpcio_tools-1.54.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0239b929eb8b3b30b2397eef3b9abb245087754d77c3721e3be43c44796de87d"}, + {file = "grpcio_tools-1.54.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:0de05c7698c655e9a240dc34ae91d6017b93143ac89e5b20046d7ca3bd09c27c"}, + {file = "grpcio_tools-1.54.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a3ce0b98fb581c471424d2cda45120f57658ed97677c6fec4d6decf5d7c1b976"}, + {file = "grpcio_tools-1.54.2-cp37-cp37m-win_amd64.whl", hash = "sha256:37393ef90674964175923afe3859fc5a208e1ece565f642b4f76a8c0224a0993"}, + {file = "grpcio_tools-1.54.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:8e4531267736d88fde1022b36dd42ed8163e3575bcbd12bfed96662872aa93fe"}, + {file = "grpcio_tools-1.54.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:a0b7049814442f918b522d66b1d015286afbeb9e6d141af54bbfafe31710a3c8"}, + {file = "grpcio_tools-1.54.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:b80585e06c4f0082327eb5c9ad96fbdb2b0e7c14971ea5099fe78c22f4608451"}, + {file = "grpcio_tools-1.54.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39fd530cfdf58dc05125775cc233b05554d553d27478f14ae5fd8a6306f0cb28"}, + {file = "grpcio_tools-1.54.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bb9ec4aea0f2b3006fb002fa59e5c10f92b48fc374619fbffd14d2b0e388c3e"}, + {file = "grpcio_tools-1.54.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d512de051342a576bb89777476d13c5266d9334cf4badb6468aed9dc8f5bdec1"}, + {file = "grpcio_tools-1.54.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1b8ee3099c51ce987fa8a08e6b93fc342b10228415dd96b5c0caa0387f636a6f"}, + {file = "grpcio_tools-1.54.2-cp38-cp38-win32.whl", hash = "sha256:6037f123905dc0141f7c8383ca616ef0195e79cd3b4d82faaee789d4045e891b"}, + {file = "grpcio_tools-1.54.2-cp38-cp38-win_amd64.whl", hash = "sha256:10dd41862f579d185c60f629b5ee89103e216f63b576079d258d974d980bad87"}, + {file = "grpcio_tools-1.54.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:f6787d07fdab31a32c433c1ba34883dea6559d8a3fbe08fb93d834ca34136b71"}, + {file = "grpcio_tools-1.54.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:21b1467e31e44429d2a78b50135c9cdbd4b8f6d3b5cd548bc98985d3bdc352d0"}, + {file = "grpcio_tools-1.54.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:30a49b8b168aced2a4ff40959e6c4383ad6cfd7a20839a47a215e9837eb722dc"}, + {file = "grpcio_tools-1.54.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8742122782953d2fd038f0a199f047a24e941cc9718b1aac90876dbdb7167739"}, + {file = "grpcio_tools-1.54.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:503ef1351c62fb1d6747eaf74932b609d8fdd4345b3591ef910adef8fa9969d0"}, + {file = "grpcio_tools-1.54.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:72d15de4c4b6a764a76c4ae69d99c35f7a0751223688c3f7e62dfa95eb4f61be"}, + {file = "grpcio_tools-1.54.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:df079479fb1b9e488334312e35ebbf30cbf5ecad6c56599f1a961800b33ab7c1"}, + {file = "grpcio_tools-1.54.2-cp39-cp39-win32.whl", hash = "sha256:49c2846dcc4803476e839d8bd4db8845e928f19130e0ea86121f2d1f43d2b452"}, + {file = "grpcio_tools-1.54.2-cp39-cp39-win_amd64.whl", hash = "sha256:b82ca472db9c914c44e39a41e9e8bd3ed724523dd7aff5ce37592b8d16920ed9"}, +] [package.dependencies] -grpcio = ">=1.55.0" +grpcio = ">=1.54.2" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -224,17 +436,40 @@ setuptools = "*" name = "hf-transfer" version = "0.1.3" description = "" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "hf_transfer-0.1.3-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9"}, + {file = "hf_transfer-0.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07"}, + {file = "hf_transfer-0.1.3-cp310-none-win_amd64.whl", hash = "sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6"}, + {file = "hf_transfer-0.1.3-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d53954ecfaadc84c15481bf5d4c7282323196b4b6df1d1be54208d4fdedfb407"}, + {file = "hf_transfer-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334862f4a82f8a09d6d3f550e67d7e498bb8882e678b7725638254fed3276801"}, + {file = "hf_transfer-0.1.3-cp311-none-win_amd64.whl", hash = "sha256:da92a1483a66cf2baa96de133c75e7d5d9a60e4a0e60d228f26c573c73a1feb6"}, + {file = "hf_transfer-0.1.3-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:617692a70cf299576d82cfc860923f29ec5c834a3f5242bc0853d4f106670398"}, + {file = "hf_transfer-0.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca218fb6efc5c22379d9e64989e84bff426fcf5664fdbbf7cd70aa8b79497652"}, + {file = "hf_transfer-0.1.3-cp37-none-win_amd64.whl", hash = "sha256:6e5201b648df6106c232fcdb507db734081fd6220dfb1c432bd27c6fa9453331"}, + {file = "hf_transfer-0.1.3-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:445edfcd9d59c9d2541957177a9c9225b1f0e8855f6311fb16e20f67c3426421"}, + {file = "hf_transfer-0.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c1bdfa554f0b0936c1623b99058c6998a00fdcd86f75d9203f3f66572d2e30c"}, + {file = "hf_transfer-0.1.3-cp38-none-win_amd64.whl", hash = "sha256:606f2fe436e5be73f07987a56cd97c695805413d29203ae39ebd9fc596405435"}, + {file = "hf_transfer-0.1.3-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9913f5ad2644a1f57c1b7755a7d959ca5e0189863bb0473817d0707af230bf6a"}, + {file = "hf_transfer-0.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d935946791e338f748e05a23df877d74fbcd39dc7b537f0aa2e5a5841cf7dde8"}, + {file = "hf_transfer-0.1.3-cp39-none-win_amd64.whl", hash = "sha256:79099ac043423b263a2843a24213418f309d5c8bc458776622bffe012ebced73"}, + {file = "hf_transfer-0.1.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ce6c5965a57d94db5e043aa488a4df929a32000db125d9c9a1d325e8c7006dc"}, + {file = "hf_transfer-0.1.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a7934c8e491bb395731f677f66dd5f6641432f338a3a9efc9f0b6c186d37cf8"}, + {file = "hf_transfer-0.1.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"}, + {file = "hf_transfer-0.1.3.tar.gz", hash = "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c"}, +] [[package]] name = "huggingface-hub" -version = "0.14.0" +version = "0.14.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -category = "main" optional = false python-versions = ">=3.7.0" +files = [ + {file = "huggingface_hub-0.14.1-py3-none-any.whl", hash = "sha256:9fc619170d800ff3793ad37c9757c255c8783051e1b5b00501205eb43ccc4f27"}, + {file = "huggingface_hub-0.14.1.tar.gz", hash = "sha256:9ab899af8e10922eac65e290d60ab956882ab0bf643e3d990b1394b6b47b7fbc"}, +] [package.dependencies] filelock = "*" @@ -260,25 +495,34 @@ typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "t name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] [[package]] name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] [[package]] -name = "Jinja2" +name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] [package.dependencies] MarkupSafe = ">=2.0" @@ -290,9 +534,12 @@ i18n = ["Babel (>=2.7)"] name = "loguru" version = "0.6.0" description = "Python logging made (stupidly) simple" -category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, + {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, +] [package.dependencies] colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} @@ -302,20 +549,74 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] [[package]] -name = "MarkupSafe" -version = "2.1.2" +name = "markupsafe" +version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] [[package]] name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" -category = "main" optional = true python-versions = "*" +files = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] [package.extras] develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] @@ -327,9 +628,12 @@ tests = ["pytest (>=4.6)"] name = "networkx" version = "3.1" description = "Python package for creating and manipulating graphs and networks" -category = "main" optional = true python-versions = ">=3.8" +files = [ + {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, + {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, +] [package.extras] default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] @@ -342,17 +646,49 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] name = "numpy" version = "1.24.3" description = "Fundamental package for array computing in Python" -category = "main" -optional = true +optional = false python-versions = ">=3.8" +files = [ + {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, + {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, + {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, + {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, + {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, + {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, + {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, + {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, + {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, + {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, +] [[package]] name = "opentelemetry-api" version = "1.15.0" description = "OpenTelemetry Python API" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "opentelemetry_api-1.15.0-py3-none-any.whl", hash = "sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0"}, + {file = "opentelemetry_api-1.15.0.tar.gz", hash = "sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded"}, +] [package.dependencies] deprecated = ">=1.2.6" @@ -362,9 +698,12 @@ setuptools = ">=16.0" name = "opentelemetry-exporter-otlp" version = "1.15.0" description = "OpenTelemetry Collector Exporters" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "opentelemetry_exporter_otlp-1.15.0-py3-none-any.whl", hash = "sha256:79f22748b6a54808a0448093dfa189c8490e729f67c134d4c992533d9393b33e"}, + {file = "opentelemetry_exporter_otlp-1.15.0.tar.gz", hash = "sha256:4f7c49751d9720e2e726e13b0bb958ccade4e29122c305d92c033da432c8d2c5"}, +] [package.dependencies] opentelemetry-exporter-otlp-proto-grpc = "1.15.0" @@ -374,9 +713,12 @@ opentelemetry-exporter-otlp-proto-http = "1.15.0" name = "opentelemetry-exporter-otlp-proto-grpc" version = "1.15.0" description = "OpenTelemetry Collector Protobuf over gRPC Exporter" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0-py3-none-any.whl", hash = "sha256:c2a5492ba7d140109968135d641d06ce3c5bd73c50665f787526065d57d7fd1d"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0.tar.gz", hash = "sha256:844f2a4bb9bcda34e4eb6fe36765e5031aacb36dc60ed88c90fc246942ea26e7"}, +] [package.dependencies] backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} @@ -393,9 +735,12 @@ test = ["pytest-grpc"] name = "opentelemetry-exporter-otlp-proto-http" version = "1.15.0" description = "OpenTelemetry Collector Protobuf over HTTP Exporter" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "opentelemetry_exporter_otlp_proto_http-1.15.0-py3-none-any.whl", hash = "sha256:3ec2a02196c8a54bf5cbf7fe623a5238625638e83b6047a983bdf96e2bbb74c0"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.15.0.tar.gz", hash = "sha256:11b2c814249a49b22f6cca7a06b05701f561d577b747f3660dfd67b6eb9daf9c"}, +] [package.dependencies] backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} @@ -412,9 +757,12 @@ test = ["responses (==0.22.0)"] name = "opentelemetry-instrumentation" version = "0.36b0" description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "opentelemetry_instrumentation-0.36b0-py3-none-any.whl", hash = "sha256:83ba4ae7d5292b5b33e0f851cc5c76d8f91196b9b3527800fc13855c33383ac2"}, + {file = "opentelemetry_instrumentation-0.36b0.tar.gz", hash = "sha256:e3ddac9b3b93408ef26c8ecbf38f717042977e16381bb4cd329a5b4cf16998cf"}, +] [package.dependencies] opentelemetry-api = ">=1.4,<2.0" @@ -425,9 +773,12 @@ wrapt = ">=1.0.0,<2.0.0" name = "opentelemetry-instrumentation-grpc" version = "0.36b0" description = "OpenTelemetry gRPC instrumentation" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "opentelemetry_instrumentation_grpc-0.36b0-py3-none-any.whl", hash = "sha256:eaa246ed2083c97b13bab2555cb9d170e8433230a31476c4cab8a17fa03380a4"}, + {file = "opentelemetry_instrumentation_grpc-0.36b0.tar.gz", hash = "sha256:dc89447c9eb6ea868970f6c13b4ffdac182cdd5a41dd215a0f5393ca6375be55"}, +] [package.dependencies] opentelemetry-api = ">=1.12,<2.0" @@ -444,9 +795,12 @@ test = ["opentelemetry-instrumentation-grpc[instruments]", "opentelemetry-sdk (> name = "opentelemetry-proto" version = "1.15.0" description = "OpenTelemetry Python Proto" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "opentelemetry_proto-1.15.0-py3-none-any.whl", hash = "sha256:044b6d044b4d10530f250856f933442b8753a17f94ae37c207607f733fb9a844"}, + {file = "opentelemetry_proto-1.15.0.tar.gz", hash = "sha256:9c4008e40ac8cab359daac283fbe7002c5c29c77ea2674ad5626a249e64e0101"}, +] [package.dependencies] protobuf = ">=3.19,<5.0" @@ -455,9 +809,12 @@ protobuf = ">=3.19,<5.0" name = "opentelemetry-sdk" version = "1.15.0" description = "OpenTelemetry Python SDK" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "opentelemetry_sdk-1.15.0-py3-none-any.whl", hash = "sha256:555c533e9837766119bbccc7a80458c9971d853a6f1da683a2246cd5e53b4645"}, + {file = "opentelemetry_sdk-1.15.0.tar.gz", hash = "sha256:98dbffcfeebcbff12c0c974292d6ea603180a145904cf838b1fe4d5c99078425"}, +] [package.dependencies] opentelemetry-api = "1.15.0" @@ -469,25 +826,34 @@ typing-extensions = ">=3.7.4" name = "opentelemetry-semantic-conventions" version = "0.36b0" description = "OpenTelemetry Semantic Conventions" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "opentelemetry_semantic_conventions-0.36b0-py3-none-any.whl", hash = "sha256:adc05635e87b9d3e007c9f530eed487fc3ef2177d02f82f674f28ebf9aff8243"}, + {file = "opentelemetry_semantic_conventions-0.36b0.tar.gz", hash = "sha256:829dc221795467d98b773c04096e29be038d77526dc8d6ac76f546fb6279bf01"}, +] [[package]] name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] [[package]] name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] [package.extras] dev = ["pre-commit", "tox"] @@ -495,30 +861,62 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.23.1" +version = "4.23.2" description = "" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "protobuf-4.23.2-cp310-abi3-win32.whl", hash = "sha256:384dd44cb4c43f2ccddd3645389a23ae61aeb8cfa15ca3a0f60e7c3ea09b28b3"}, + {file = "protobuf-4.23.2-cp310-abi3-win_amd64.whl", hash = "sha256:09310bce43353b46d73ba7e3bca78273b9bc50349509b9698e64d288c6372c2a"}, + {file = "protobuf-4.23.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2cfab63a230b39ae603834718db74ac11e52bccaaf19bf20f5cce1a84cf76df"}, + {file = "protobuf-4.23.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:c52cfcbfba8eb791255edd675c1fe6056f723bf832fa67f0442218f8817c076e"}, + {file = "protobuf-4.23.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:86df87016d290143c7ce3be3ad52d055714ebaebb57cc659c387e76cfacd81aa"}, + {file = "protobuf-4.23.2-cp37-cp37m-win32.whl", hash = "sha256:281342ea5eb631c86697e1e048cb7e73b8a4e85f3299a128c116f05f5c668f8f"}, + {file = "protobuf-4.23.2-cp37-cp37m-win_amd64.whl", hash = "sha256:ce744938406de1e64b91410f473736e815f28c3b71201302612a68bf01517fea"}, + {file = "protobuf-4.23.2-cp38-cp38-win32.whl", hash = "sha256:6c081863c379bb1741be8f8193e893511312b1d7329b4a75445d1ea9955be69e"}, + {file = "protobuf-4.23.2-cp38-cp38-win_amd64.whl", hash = "sha256:25e3370eda26469b58b602e29dff069cfaae8eaa0ef4550039cc5ef8dc004511"}, + {file = "protobuf-4.23.2-cp39-cp39-win32.whl", hash = "sha256:efabbbbac1ab519a514579ba9ec52f006c28ae19d97915951f69fa70da2c9e91"}, + {file = "protobuf-4.23.2-cp39-cp39-win_amd64.whl", hash = "sha256:54a533b971288af3b9926e53850c7eb186886c0c84e61daa8444385a4720297f"}, + {file = "protobuf-4.23.2-py3-none-any.whl", hash = "sha256:8da6070310d634c99c0db7df48f10da495cc283fd9e9234877f0cd182d43ab7f"}, + {file = "protobuf-4.23.2.tar.gz", hash = "sha256:20874e7ca4436f683b64ebdbee2129a5a2c301579a67d1a7dda2cdf62fb7f5f7"}, +] [[package]] name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." -category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, + {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, + {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, + {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, + {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, + {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, + {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, + {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, +] [package.extras] test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "pytest" -version = "7.3.1" +version = "7.3.2" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pytest-7.3.2-py3-none-any.whl", hash = "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295"}, + {file = "pytest-7.3.2.tar.gz", hash = "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b"}, +] [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} @@ -529,647 +927,15 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] -name = "PyYAML" +name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" - -[[package]] -name = "requests" -version = "2.31.0" -description = "Python HTTP for Humans." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "safetensors" -version = "0.3.1" -description = "Fast and Safe Tensor serialization" -category = "main" -optional = false -python-versions = "*" - -[package.extras] -all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] -dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] -jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] -numpy = ["numpy (>=1.21.6)"] -paddlepaddle = ["paddlepaddle (>=2.4.1)"] -quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] -tensorflow = ["tensorflow (>=2.11.0)"] -testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] -torch = ["torch (>=1.10)"] - -[[package]] -name = "sentencepiece" -version = "0.1.99" -description = "SentencePiece python wrapper" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "setuptools" -version = "67.8.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - -[[package]] -name = "sympy" -version = "1.12" -description = "Computer algebra system (CAS) in Python" -category = "main" -optional = true -python-versions = ">=3.8" - -[package.dependencies] -mpmath = ">=0.19" - -[[package]] -name = "tokenizers" -version = "0.13.3" -description = "Fast and Customizable Tokenizers" -category = "main" -optional = false -python-versions = "*" - -[package.extras] -dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] -testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "torch" -version = "2.0.1" -description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -category = "main" -optional = true -python-versions = ">=3.8.0" - -[package.dependencies] -filelock = "*" -jinja2 = "*" -networkx = "*" -sympy = "*" -typing-extensions = "*" - -[package.extras] -opt-einsum = ["opt-einsum (>=3.3)"] - -[[package]] -name = "tqdm" -version = "4.65.0" -description = "Fast, Extensible Progress Meter" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] -notebook = ["ipywidgets (>=6)"] -slack = ["slack-sdk"] -telegram = ["requests"] - -[[package]] -name = "typer" -version = "0.6.1" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -click = ">=7.1.1,<9.0.0" - -[package.extras] -all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] -dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] -doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] -test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] - -[[package]] -name = "typing-extensions" -version = "4.6.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "urllib3" -version = "2.0.2" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "win32-setctime" -version = "1.1.0" -description = "A small Python utility to set file creation time on Windows" -category = "main" -optional = false -python-versions = ">=3.5" - -[package.extras] -dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] - -[[package]] -name = "wrapt" -version = "1.15.0" -description = "Module for decorators, wrappers and monkey patching." -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[extras] -accelerate = ["accelerate"] -bnb = ["bitsandbytes"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.9" -content-hash = "152c8b82717e2b802aee3427152ef3e37417fb95b73c3af2f448a381f51a6a8d" - -[metadata.files] -accelerate = [ - {file = "accelerate-0.19.0-py3-none-any.whl", hash = "sha256:2866b0bf9fff08f51e6384c95fa96725838b70f1988d1cce42e56b820d8a91dd"}, - {file = "accelerate-0.19.0.tar.gz", hash = "sha256:84920226b9e642e453ef37593ee55b956b08d8200dea4087c546c34e26157e76"}, -] -backoff = [ - {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, - {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, -] -bitsandbytes = [ - {file = "bitsandbytes-0.38.1-py3-none-any.whl", hash = "sha256:5f532e7b1353eb7049ae831da2eb62ed8a1e0444116bd51b9e088a6e0bc7a34a"}, - {file = "bitsandbytes-0.38.1.tar.gz", hash = "sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce"}, -] -certifi = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, -] -charset-normalizer = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -Deprecated = [ - {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, - {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, -] -exceptiongroup = [ - {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, - {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, -] -filelock = [ - {file = "filelock-3.12.0-py3-none-any.whl", hash = "sha256:ad98852315c2ab702aeb628412cbf7e95b7ce8c3bf9565670b4eaecf1db370a9"}, - {file = "filelock-3.12.0.tar.gz", hash = "sha256:fc03ae43288c013d2ea83c8597001b1129db351aad9c57fe2409327916b8e718"}, -] -fsspec = [ - {file = "fsspec-2023.5.0-py3-none-any.whl", hash = "sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a"}, - {file = "fsspec-2023.5.0.tar.gz", hash = "sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce"}, -] -googleapis-common-protos = [ - {file = "googleapis-common-protos-1.59.0.tar.gz", hash = "sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44"}, - {file = "googleapis_common_protos-1.59.0-py2.py3-none-any.whl", hash = "sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f"}, -] -grpc-interceptor = [ - {file = "grpc-interceptor-0.15.2.tar.gz", hash = "sha256:5c984110af4fb77d03472ec0468f9c77ddaf798e190410fb7b7f1e76c60c96a4"}, - {file = "grpc_interceptor-0.15.2-py3-none-any.whl", hash = "sha256:596dac3cb709ffb6178a4873f5148e254c871c9069f0b11040189b257969490a"}, -] -grpcio = [ - {file = "grpcio-1.55.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:7b38e028a7bbc97a9ae5e418712452f298618b9d0493390770bf2de785251ae7"}, - {file = "grpcio-1.55.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:054b7164b25712ec71339e139875a66708a2ab09be36ac75e73b2d337ab2dc1b"}, - {file = "grpcio-1.55.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:1982c99c7091d1b7e3e78b1173097f705feef233e253a27e99746b11815ac897"}, - {file = "grpcio-1.55.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8bd4f4932ef63ed32a725065aebb8585e4118a523d923db896e85c09429a36e6"}, - {file = "grpcio-1.55.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70de2b73cf22241173cb21d308786ba4ea443e4c88441a2ce445829aa638dda8"}, - {file = "grpcio-1.55.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2d25d7fcb528a40578b3d0428d401745fd5c0eeeda81f35ce2f40a10d79afd19"}, - {file = "grpcio-1.55.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1173a05117798aca4834d3edd504e6adc25ae9967df0f44b91a612884fb2707a"}, - {file = "grpcio-1.55.0-cp310-cp310-win32.whl", hash = "sha256:7c00263d792a244bef67a8d3b357ccbcdae6341c5961dbee494d8f967f9aee69"}, - {file = "grpcio-1.55.0-cp310-cp310-win_amd64.whl", hash = "sha256:ab784204d9923368e0e5877d7795584b9606a51b128ee199ad8b5888d0c66592"}, - {file = "grpcio-1.55.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c97cfae0b7a17dc1a0a3e4333f4f46daa114d85f950a67f39cc141b5425182e4"}, - {file = "grpcio-1.55.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:8a910fa9b95a286f4bc1879dcf8d5ccb95b5e33bb63323fc4414d157f23afef1"}, - {file = "grpcio-1.55.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:3ab9bf80c19c91847f45ff32af94c85d282545a62db39d797838244d57831d78"}, - {file = "grpcio-1.55.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4370d2cca37301bcc69453d3dd3c1576d06d6b3e337bfec55b3aab2fe106b25c"}, - {file = "grpcio-1.55.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dad999423b33ad5409e986587593b6062a8260b74ae8fc8162ce231c6b7a929e"}, - {file = "grpcio-1.55.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d396ec4d520b58f43142958cff071e5ad1c50ac87d29d086a9c6a990a09ea536"}, - {file = "grpcio-1.55.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b2a3b837d5837b9069783026b57aa0ff12e34d3218fdeda3f9c06d3950266d8e"}, - {file = "grpcio-1.55.0-cp311-cp311-win32.whl", hash = "sha256:ee0de9cb6813704969e53743e0969fd95225ff24bd686c89ed12a18147f6566c"}, - {file = "grpcio-1.55.0-cp311-cp311-win_amd64.whl", hash = "sha256:9a11b1dd4b1572e85fba5911309c15980a1ff77c555fad0ecdbe3711ef741908"}, - {file = "grpcio-1.55.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d0209fb3cb55c5288a1dec72dcaae2c1b501edceca10d22c0f0baa5e60e2b22c"}, - {file = "grpcio-1.55.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:322d4ebc37cbc8d8596b1da6055e3e81e8cfd36816ab4b285c1163c3042e6067"}, - {file = "grpcio-1.55.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:60efab181c32e029e0960f238508396dd001ba2064168f8148e6356db093967c"}, - {file = "grpcio-1.55.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48f6088d898e1e987d761d58dc4cd724e7457a7a86d11561fa95c3b826d025dc"}, - {file = "grpcio-1.55.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29ab0e879b1585be41cfbb02faed67913700ced8015da4763f1f0bdd7dfb4ab7"}, - {file = "grpcio-1.55.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:157f5615c7b5d0968727472f6394dee01555ef4246d2f2cfb6555be857936d74"}, - {file = "grpcio-1.55.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:67c4fda71f92225c5e74fa15bffa6be022c07111f674fe1f234c1ef4c1bb7927"}, - {file = "grpcio-1.55.0-cp37-cp37m-win_amd64.whl", hash = "sha256:a202dcf0c512292fd7a2154e4044c70400212eaa726685ebf8af105e25693c5a"}, - {file = "grpcio-1.55.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:ce82d06cdfb8a9292fb857f00bee11a2430e4ac2742e07b46c1a3072d683256a"}, - {file = "grpcio-1.55.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:51b7a27a129f743d68394f94029f88ef3da090fc13776b9dfa3c79c5f4b30525"}, - {file = "grpcio-1.55.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:7c32f87bec58a8a0d4f4d5387bd61a383bd32b2caffb2de3cd579e47490b7e19"}, - {file = "grpcio-1.55.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:89107071b5f14af6bbb855183d338a0fa94136bbeb3989c9773c6184e51a95e9"}, - {file = "grpcio-1.55.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1041cad23f00943d8889ad15427d87bbdacbbe2df5cec951c314f2f3967d4691"}, - {file = "grpcio-1.55.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:56631cc0bdf86d15ea1599b9697ace65e6b52c6b136d3666bf7769d3d6d087a8"}, - {file = "grpcio-1.55.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:10af4774da9c0665a1bf519333694ac40d72d83cb514534b99db0a5e3d5c3593"}, - {file = "grpcio-1.55.0-cp38-cp38-win32.whl", hash = "sha256:7b8665da31b5bd701b338a581de7b9631d50b4b7ee67125c2d1dc2228cc119d8"}, - {file = "grpcio-1.55.0-cp38-cp38-win_amd64.whl", hash = "sha256:74780f570c76feb8e62a8c019b495fea435b60218682fce513ff2c71262c346c"}, - {file = "grpcio-1.55.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:6b8dbb151b116825c10f01e5b7b75e14edd0e60736a65311d0d98a4cd0489303"}, - {file = "grpcio-1.55.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:a82283d6e0403d3e2e7eebb99cb0d2783e20b6791c8c94bd8d4a4233b58b1ea0"}, - {file = "grpcio-1.55.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:ba32a8e9bc3eecc6bab6824b905f04c3fdc31659c3e6e06841b774e7cb4410af"}, - {file = "grpcio-1.55.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b1e2b705d524e780998218cf429d30b6ffc54cb6e54812c9597bc5df12dbcb5b"}, - {file = "grpcio-1.55.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe78365c64b2c7470d31c4941e10c6654042bcbb53897b9b1e2c96d6d0da9ef9"}, - {file = "grpcio-1.55.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8b440ccc434c1ad5874465bfae40c0a27f562ae5f7c5b468b6689bc55e8bf1c1"}, - {file = "grpcio-1.55.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0d3d5c644d523dee82ffcc44ad50cd66e3bf66e7fa60ad3cdb1eb868228e4ab0"}, - {file = "grpcio-1.55.0-cp39-cp39-win32.whl", hash = "sha256:c33dbeecc14f1a413e8af8ae1208cb383b063fa2ff2e1f309b4d3d7739b0927e"}, - {file = "grpcio-1.55.0-cp39-cp39-win_amd64.whl", hash = "sha256:2663741acc117370fd53336267cfb24c965e9d3ea1e4933a3e4411712d3091fb"}, - {file = "grpcio-1.55.0.tar.gz", hash = "sha256:dd15027a171ff93c97f9c704fa120bc5d0691dc7e71ae450e2ecade1a2799b53"}, -] -grpcio-reflection = [ - {file = "grpcio-reflection-1.55.0.tar.gz", hash = "sha256:46fc5e68ce7ae9bff0c0577f9e42bbb038a5afb26290fdf04943285e9db3c193"}, - {file = "grpcio_reflection-1.55.0-py3-none-any.whl", hash = "sha256:44e0dbfbfdcf1ac8646f1d32e4be72f0c633fd4b469e8e58b3a86e37b5a72756"}, -] -grpcio-status = [ - {file = "grpcio-status-1.55.0.tar.gz", hash = "sha256:beeca8d5d3783e155676beaade0dae9eaea12cd9701498905dca0d35bd6b36f8"}, - {file = "grpcio_status-1.55.0-py3-none-any.whl", hash = "sha256:6da36bab11bb252b6854b86578f484c4fed9f8169816b490b6d3a32ec2a971fe"}, -] -grpcio-tools = [ - {file = "grpcio-tools-1.55.0.tar.gz", hash = "sha256:d796f5d7cea260ef2afed12d13ec34b13e09dd74d7f292d7428c506fa8c17a74"}, - {file = "grpcio_tools-1.55.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:52e34e9b6496f4c1e3289ada7bc41d759e4a8ec5f2679e187067cab8532ffbf4"}, - {file = "grpcio_tools-1.55.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b131b2bbf25198d9e508dfa588cb215580629b514e293d5609eeee98c8941dbc"}, - {file = "grpcio_tools-1.55.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:9933a1f18f780c42214b126ef27e273b54c9c28de3fae5b1887b413ceb374c4c"}, - {file = "grpcio_tools-1.55.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfc82c11ce51de6ed5836fbafbc188d9eac0737abc116978f151c40271783817"}, - {file = "grpcio_tools-1.55.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7a18bd5f994b7911d3e70e0abb05bea9f1b084a1725d404a8e231bf9727613b"}, - {file = "grpcio_tools-1.55.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:87152893c7c3bef58a6a9b548db290aa318cc314c700ae7d7f2970aa567f875e"}, - {file = "grpcio_tools-1.55.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bc23034b1959d6cda27347b2207fee0fb0fb0aff242da228a6b7c1a18fce4116"}, - {file = "grpcio_tools-1.55.0-cp310-cp310-win32.whl", hash = "sha256:ab64f9d6f5e3636ae6298e2d795225daa83aacb057105943728ed50a8a582237"}, - {file = "grpcio_tools-1.55.0-cp310-cp310-win_amd64.whl", hash = "sha256:b197de69ca0431b718ffa47b32a733703fa5503da49f49dd315c866842b6cfbd"}, - {file = "grpcio_tools-1.55.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:98ff3129ff7134a95f4d2857663625771f6838ac44b7799c34259b7ea87ebe5c"}, - {file = "grpcio_tools-1.55.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:c3c7b7eb89f963b87922ecc0c0ab2485fff05997ada66dffd53597b507a83bc8"}, - {file = "grpcio_tools-1.55.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:51a1ccab6f67edd1a3768a75ac495907fe0cd6d6617af2f9f2033400b5858a11"}, - {file = "grpcio_tools-1.55.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4dea66623548b52429fb03495f2c76f4c993bf9a56267c6b3d0fb62573dd52c2"}, - {file = "grpcio_tools-1.55.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e59fd4a58688117cb5128d7785909d45a6e5f8212efeb65b6fd74bb9b8b9512"}, - {file = "grpcio_tools-1.55.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:734ede84d613b044f72e7d9c190bd2388ebb83e85bcd3aa75afa9f30c096dbc7"}, - {file = "grpcio_tools-1.55.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f87d99aa3826aa20c3b89493984cf278f4c9d20b3418534a46239c804fee506c"}, - {file = "grpcio_tools-1.55.0-cp311-cp311-win32.whl", hash = "sha256:4580df5a9867f7bcbb828a5485c030ca232c1578e615caf751333c7a7980d838"}, - {file = "grpcio_tools-1.55.0-cp311-cp311-win_amd64.whl", hash = "sha256:b674de79571357c5381bc5fa12e3b89fefef74c164ab9077ed22158c3529aa8e"}, - {file = "grpcio_tools-1.55.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:0dead7fb37bfe7c7eb8294143015645297f4affa683783b8bbf2cd4d7f7036d4"}, - {file = "grpcio_tools-1.55.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:89f6ed47415a22568bbf4a62336bfde7cafb53492a5a9f33a22243411b00f443"}, - {file = "grpcio_tools-1.55.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:946266cbd639847548c9f97e38da0682746c2eadea790ceb4320b1f85387bd6d"}, - {file = "grpcio_tools-1.55.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:36745762689df18f83273a9a004848897793f63a10a30acd18acb2d170c663a9"}, - {file = "grpcio_tools-1.55.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:053bbdfb74f76511db47e1e18a1962432468ae9f356cc00f15d1f1353eaf32a1"}, - {file = "grpcio_tools-1.55.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2eacb0b1e8e5cfd0b40e12e62bd5adebbbae8c73cdf6e04fad9ddd37e32d98a4"}, - {file = "grpcio_tools-1.55.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9395c4fdee6b22137e878ebd461444854a3cd9c6c260c43f4a4c4a4023700129"}, - {file = "grpcio_tools-1.55.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bcf5e1858137cbe13ef10a7931a7edc745c77f8b39f032f52072443f0dd681e1"}, - {file = "grpcio_tools-1.55.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:e76f35e5e65600a75c3547855e8c9ab935c55c473f5409e7746cca8f1f7c8f4a"}, - {file = "grpcio_tools-1.55.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:a61567f27661ab9327dc060615dc22d2bde80c56731f1e856008f1fd8ee83311"}, - {file = "grpcio_tools-1.55.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2ba87592f2cd689e127cd4fce76ec23b19562e230fa41ea089af8b15120aea78"}, - {file = "grpcio_tools-1.55.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:41005002cbfa0ad39972486bde8116b2a042804119e5b998086a4dc26e625d6a"}, - {file = "grpcio_tools-1.55.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a6db1494955d2a5531575b5fcdc08094ea4a331a94b9cdf864d78e801c5fa23"}, - {file = "grpcio_tools-1.55.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:07c23ed940e046c9dd471bc870eb5db4d93e518f90011cf9aebf8bfda6cd68a5"}, - {file = "grpcio_tools-1.55.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f900bce944b5777effecb9078e5fd3a224e42b1ca33c7546c3d043f9ef9eb4e8"}, - {file = "grpcio_tools-1.55.0-cp38-cp38-win32.whl", hash = "sha256:3724e48c3db499b2d212c5a89d7cc4b49ccd476dc26bf8a9b855d59b6cc00796"}, - {file = "grpcio_tools-1.55.0-cp38-cp38-win_amd64.whl", hash = "sha256:416a8b61ed4223715755b4519858419e1f4653d64572a28029f2ac63e677e3d2"}, - {file = "grpcio_tools-1.55.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:73ef9e0e0ee8ab055a621e7b42e5fb32753b0b6607900887dba6d55df5947be8"}, - {file = "grpcio_tools-1.55.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:4a41130c97775bb0dfaf87e34b492f2eca448d02d213410005544c534f3f7c26"}, - {file = "grpcio_tools-1.55.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:87dbc98528f88faa3f8f56a47d41dc6fda382928abbdb5537b5444eb8bb1ac1b"}, - {file = "grpcio_tools-1.55.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f084cd619cf66d8620a99f8586018f19b918ffb2ddb92d3e5943a06038bead8"}, - {file = "grpcio_tools-1.55.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:350303ef3a2b25ed1b90e42764923e40b664d9f10840f7a0f06117c4dc414aff"}, - {file = "grpcio_tools-1.55.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:fbbe2bee4af93c03ba064d40199dbf38067d2aa6ae98dfa0687a08ee980ebfd5"}, - {file = "grpcio_tools-1.55.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b00a67a1230968c1a0424915922d17983d824ed45e8db06f9f17be6d5571faee"}, - {file = "grpcio_tools-1.55.0-cp39-cp39-win32.whl", hash = "sha256:632364ffbd4fb0338cb03c590a2ddc258d9cd59bff0bf4199c02e3e581f802d7"}, - {file = "grpcio_tools-1.55.0-cp39-cp39-win_amd64.whl", hash = "sha256:95428be2db12412ff23f0969386fc51d2aa6de38a57cc54c57363352f1d7a832"}, -] -hf-transfer = [ - {file = "hf_transfer-0.1.3-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9"}, - {file = "hf_transfer-0.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07"}, - {file = "hf_transfer-0.1.3-cp310-none-win_amd64.whl", hash = "sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6"}, - {file = "hf_transfer-0.1.3-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d53954ecfaadc84c15481bf5d4c7282323196b4b6df1d1be54208d4fdedfb407"}, - {file = "hf_transfer-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334862f4a82f8a09d6d3f550e67d7e498bb8882e678b7725638254fed3276801"}, - {file = "hf_transfer-0.1.3-cp311-none-win_amd64.whl", hash = "sha256:da92a1483a66cf2baa96de133c75e7d5d9a60e4a0e60d228f26c573c73a1feb6"}, - {file = "hf_transfer-0.1.3-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:617692a70cf299576d82cfc860923f29ec5c834a3f5242bc0853d4f106670398"}, - {file = "hf_transfer-0.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca218fb6efc5c22379d9e64989e84bff426fcf5664fdbbf7cd70aa8b79497652"}, - {file = "hf_transfer-0.1.3-cp37-none-win_amd64.whl", hash = "sha256:6e5201b648df6106c232fcdb507db734081fd6220dfb1c432bd27c6fa9453331"}, - {file = "hf_transfer-0.1.3-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:445edfcd9d59c9d2541957177a9c9225b1f0e8855f6311fb16e20f67c3426421"}, - {file = "hf_transfer-0.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c1bdfa554f0b0936c1623b99058c6998a00fdcd86f75d9203f3f66572d2e30c"}, - {file = "hf_transfer-0.1.3-cp38-none-win_amd64.whl", hash = "sha256:606f2fe436e5be73f07987a56cd97c695805413d29203ae39ebd9fc596405435"}, - {file = "hf_transfer-0.1.3-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9913f5ad2644a1f57c1b7755a7d959ca5e0189863bb0473817d0707af230bf6a"}, - {file = "hf_transfer-0.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d935946791e338f748e05a23df877d74fbcd39dc7b537f0aa2e5a5841cf7dde8"}, - {file = "hf_transfer-0.1.3-cp39-none-win_amd64.whl", hash = "sha256:79099ac043423b263a2843a24213418f309d5c8bc458776622bffe012ebced73"}, - {file = "hf_transfer-0.1.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ce6c5965a57d94db5e043aa488a4df929a32000db125d9c9a1d325e8c7006dc"}, - {file = "hf_transfer-0.1.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a7934c8e491bb395731f677f66dd5f6641432f338a3a9efc9f0b6c186d37cf8"}, - {file = "hf_transfer-0.1.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"}, - {file = "hf_transfer-0.1.3.tar.gz", hash = "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c"}, -] -huggingface-hub = [ - {file = "huggingface_hub-0.14.0-py3-none-any.whl", hash = "sha256:fa6a6139fe4a8a164bfd0cda90c225fe8471b47c12811738b6db8348a2f703a0"}, - {file = "huggingface_hub-0.14.0.tar.gz", hash = "sha256:42eeab833284e3fc1d39263cf9c3d1bb36b129acdd8195838694d165e8dd6cae"}, -] -idna = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] -iniconfig = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] -Jinja2 = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] -loguru = [ - {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, - {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, -] -MarkupSafe = [ - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, - {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, -] -mpmath = [ - {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, - {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, -] -networkx = [ - {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, - {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, -] -numpy = [ - {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, - {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, - {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, - {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, - {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, - {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, - {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, - {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, - {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, - {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, - {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, -] -opentelemetry-api = [ - {file = "opentelemetry_api-1.15.0-py3-none-any.whl", hash = "sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0"}, - {file = "opentelemetry_api-1.15.0.tar.gz", hash = "sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded"}, -] -opentelemetry-exporter-otlp = [ - {file = "opentelemetry_exporter_otlp-1.15.0-py3-none-any.whl", hash = "sha256:79f22748b6a54808a0448093dfa189c8490e729f67c134d4c992533d9393b33e"}, - {file = "opentelemetry_exporter_otlp-1.15.0.tar.gz", hash = "sha256:4f7c49751d9720e2e726e13b0bb958ccade4e29122c305d92c033da432c8d2c5"}, -] -opentelemetry-exporter-otlp-proto-grpc = [ - {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0-py3-none-any.whl", hash = "sha256:c2a5492ba7d140109968135d641d06ce3c5bd73c50665f787526065d57d7fd1d"}, - {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0.tar.gz", hash = "sha256:844f2a4bb9bcda34e4eb6fe36765e5031aacb36dc60ed88c90fc246942ea26e7"}, -] -opentelemetry-exporter-otlp-proto-http = [ - {file = "opentelemetry_exporter_otlp_proto_http-1.15.0-py3-none-any.whl", hash = "sha256:3ec2a02196c8a54bf5cbf7fe623a5238625638e83b6047a983bdf96e2bbb74c0"}, - {file = "opentelemetry_exporter_otlp_proto_http-1.15.0.tar.gz", hash = "sha256:11b2c814249a49b22f6cca7a06b05701f561d577b747f3660dfd67b6eb9daf9c"}, -] -opentelemetry-instrumentation = [ - {file = "opentelemetry_instrumentation-0.36b0-py3-none-any.whl", hash = "sha256:83ba4ae7d5292b5b33e0f851cc5c76d8f91196b9b3527800fc13855c33383ac2"}, - {file = "opentelemetry_instrumentation-0.36b0.tar.gz", hash = "sha256:e3ddac9b3b93408ef26c8ecbf38f717042977e16381bb4cd329a5b4cf16998cf"}, -] -opentelemetry-instrumentation-grpc = [ - {file = "opentelemetry_instrumentation_grpc-0.36b0-py3-none-any.whl", hash = "sha256:eaa246ed2083c97b13bab2555cb9d170e8433230a31476c4cab8a17fa03380a4"}, - {file = "opentelemetry_instrumentation_grpc-0.36b0.tar.gz", hash = "sha256:dc89447c9eb6ea868970f6c13b4ffdac182cdd5a41dd215a0f5393ca6375be55"}, -] -opentelemetry-proto = [ - {file = "opentelemetry_proto-1.15.0-py3-none-any.whl", hash = "sha256:044b6d044b4d10530f250856f933442b8753a17f94ae37c207607f733fb9a844"}, - {file = "opentelemetry_proto-1.15.0.tar.gz", hash = "sha256:9c4008e40ac8cab359daac283fbe7002c5c29c77ea2674ad5626a249e64e0101"}, -] -opentelemetry-sdk = [ - {file = "opentelemetry_sdk-1.15.0-py3-none-any.whl", hash = "sha256:555c533e9837766119bbccc7a80458c9971d853a6f1da683a2246cd5e53b4645"}, - {file = "opentelemetry_sdk-1.15.0.tar.gz", hash = "sha256:98dbffcfeebcbff12c0c974292d6ea603180a145904cf838b1fe4d5c99078425"}, -] -opentelemetry-semantic-conventions = [ - {file = "opentelemetry_semantic_conventions-0.36b0-py3-none-any.whl", hash = "sha256:adc05635e87b9d3e007c9f530eed487fc3ef2177d02f82f674f28ebf9aff8243"}, - {file = "opentelemetry_semantic_conventions-0.36b0.tar.gz", hash = "sha256:829dc221795467d98b773c04096e29be038d77526dc8d6ac76f546fb6279bf01"}, -] -packaging = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, -] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] -protobuf = [ - {file = "protobuf-4.23.1-cp310-abi3-win32.whl", hash = "sha256:410bcc0a5b279f634d3e16082ce221dfef7c3392fac723500e2e64d1806dd2be"}, - {file = "protobuf-4.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:32e78beda26d7a101fecf15d7a4a792278a0d26a31bc327ff05564a9d68ab8ee"}, - {file = "protobuf-4.23.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f9510cac91e764e86acd74e2b7f7bc5e6127a7f3fb646d7c8033cfb84fd1176a"}, - {file = "protobuf-4.23.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:346990f634272caac1f09efbcfbbacb23098b1f606d172534c6fa2d9758bb436"}, - {file = "protobuf-4.23.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3ce113b3f3362493bddc9069c2163a38f240a9ed685ff83e7bcb756b05e1deb0"}, - {file = "protobuf-4.23.1-cp37-cp37m-win32.whl", hash = "sha256:2036a3a1e7fc27f973fa0a7888dce712393af644f4695385f117886abc792e39"}, - {file = "protobuf-4.23.1-cp37-cp37m-win_amd64.whl", hash = "sha256:3b8905eafe4439076e1f58e9d1fa327025fd2777cf90f14083092ae47f77b0aa"}, - {file = "protobuf-4.23.1-cp38-cp38-win32.whl", hash = "sha256:5b9cd6097e6acae48a68cb29b56bc79339be84eca65b486910bb1e7a30e2b7c1"}, - {file = "protobuf-4.23.1-cp38-cp38-win_amd64.whl", hash = "sha256:decf119d54e820f298ee6d89c72d6b289ea240c32c521f00433f9dc420595f38"}, - {file = "protobuf-4.23.1-cp39-cp39-win32.whl", hash = "sha256:91fac0753c3c4951fbb98a93271c43cc7cf3b93cf67747b3e600bb1e5cc14d61"}, - {file = "protobuf-4.23.1-cp39-cp39-win_amd64.whl", hash = "sha256:ac50be82491369a9ec3710565777e4da87c6d2e20404e0abb1f3a8f10ffd20f0"}, - {file = "protobuf-4.23.1-py3-none-any.whl", hash = "sha256:65f0ac96ef67d7dd09b19a46aad81a851b6f85f89725577f16de38f2d68ad477"}, - {file = "protobuf-4.23.1.tar.gz", hash = "sha256:95789b569418a3e32a53f43d7763be3d490a831e9c08042539462b6d972c2d7e"}, -] -psutil = [ - {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, - {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, - {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, - {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, - {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, - {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, - {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, - {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, - {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, - {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, - {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, - {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, - {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, - {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, -] -pytest = [ - {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, - {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, -] -PyYAML = [ +files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -1211,11 +977,132 @@ PyYAML = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] -requests = [ + +[[package]] +name = "regex" +version = "2023.6.3" +description = "Alternative regular expression module, to replace re." +optional = false +python-versions = ">=3.6" +files = [ + {file = "regex-2023.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:824bf3ac11001849aec3fa1d69abcb67aac3e150a933963fb12bda5151fe1bfd"}, + {file = "regex-2023.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05ed27acdf4465c95826962528f9e8d41dbf9b1aa8531a387dee6ed215a3e9ef"}, + {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b49c764f88a79160fa64f9a7b425620e87c9f46095ef9c9920542ab2495c8bc"}, + {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8e3f1316c2293e5469f8f09dc2d76efb6c3982d3da91ba95061a7e69489a14ef"}, + {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43e1dd9d12df9004246bacb79a0e5886b3b6071b32e41f83b0acbf293f820ee8"}, + {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4959e8bcbfda5146477d21c3a8ad81b185cd252f3d0d6e4724a5ef11c012fb06"}, + {file = "regex-2023.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:af4dd387354dc83a3bff67127a124c21116feb0d2ef536805c454721c5d7993d"}, + {file = "regex-2023.6.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2239d95d8e243658b8dbb36b12bd10c33ad6e6933a54d36ff053713f129aa536"}, + {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:890e5a11c97cf0d0c550eb661b937a1e45431ffa79803b942a057c4fb12a2da2"}, + {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a8105e9af3b029f243ab11ad47c19b566482c150c754e4c717900a798806b222"}, + {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:25be746a8ec7bc7b082783216de8e9473803706723b3f6bef34b3d0ed03d57e2"}, + {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3676f1dd082be28b1266c93f618ee07741b704ab7b68501a173ce7d8d0d0ca18"}, + {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:10cb847aeb1728412c666ab2e2000ba6f174f25b2bdc7292e7dd71b16db07568"}, + {file = "regex-2023.6.3-cp310-cp310-win32.whl", hash = "sha256:dbbbfce33cd98f97f6bffb17801b0576e653f4fdb1d399b2ea89638bc8d08ae1"}, + {file = "regex-2023.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:c5f8037000eb21e4823aa485149f2299eb589f8d1fe4b448036d230c3f4e68e0"}, + {file = "regex-2023.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c123f662be8ec5ab4ea72ea300359023a5d1df095b7ead76fedcd8babbedf969"}, + {file = "regex-2023.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9edcbad1f8a407e450fbac88d89e04e0b99a08473f666a3f3de0fd292badb6aa"}, + {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcba6dae7de533c876255317c11f3abe4907ba7d9aa15d13e3d9710d4315ec0e"}, + {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29cdd471ebf9e0f2fb3cac165efedc3c58db841d83a518b082077e612d3ee5df"}, + {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:12b74fbbf6cbbf9dbce20eb9b5879469e97aeeaa874145517563cca4029db65c"}, + {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c29ca1bd61b16b67be247be87390ef1d1ef702800f91fbd1991f5c4421ebae8"}, + {file = "regex-2023.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77f09bc4b55d4bf7cc5eba785d87001d6757b7c9eec237fe2af57aba1a071d9"}, + {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ea353ecb6ab5f7e7d2f4372b1e779796ebd7b37352d290096978fea83c4dba0c"}, + {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:10590510780b7541969287512d1b43f19f965c2ece6c9b1c00fc367b29d8dce7"}, + {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e2fbd6236aae3b7f9d514312cdb58e6494ee1c76a9948adde6eba33eb1c4264f"}, + {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:6b2675068c8b56f6bfd5a2bda55b8accbb96c02fd563704732fd1c95e2083461"}, + {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74419d2b50ecb98360cfaa2974da8689cb3b45b9deff0dcf489c0d333bcc1477"}, + {file = "regex-2023.6.3-cp311-cp311-win32.whl", hash = "sha256:fb5ec16523dc573a4b277663a2b5a364e2099902d3944c9419a40ebd56a118f9"}, + {file = "regex-2023.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:09e4a1a6acc39294a36b7338819b10baceb227f7f7dbbea0506d419b5a1dd8af"}, + {file = "regex-2023.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0654bca0cdf28a5956c83839162692725159f4cda8d63e0911a2c0dc76166525"}, + {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463b6a3ceb5ca952e66550a4532cef94c9a0c80dc156c4cc343041951aec1697"}, + {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87b2a5bb5e78ee0ad1de71c664d6eb536dc3947a46a69182a90f4410f5e3f7dd"}, + {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6343c6928282c1f6a9db41f5fd551662310e8774c0e5ebccb767002fcf663ca9"}, + {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6192d5af2ccd2a38877bfef086d35e6659566a335b1492786ff254c168b1693"}, + {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74390d18c75054947e4194019077e243c06fbb62e541d8817a0fa822ea310c14"}, + {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:742e19a90d9bb2f4a6cf2862b8b06dea5e09b96c9f2df1779e53432d7275331f"}, + {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8abbc5d54ea0ee80e37fef009e3cec5dafd722ed3c829126253d3e22f3846f1e"}, + {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c2b867c17a7a7ae44c43ebbeb1b5ff406b3e8d5b3e14662683e5e66e6cc868d3"}, + {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:d831c2f8ff278179705ca59f7e8524069c1a989e716a1874d6d1aab6119d91d1"}, + {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ee2d1a9a253b1729bb2de27d41f696ae893507c7db224436abe83ee25356f5c1"}, + {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:61474f0b41fe1a80e8dfa70f70ea1e047387b7cd01c85ec88fa44f5d7561d787"}, + {file = "regex-2023.6.3-cp36-cp36m-win32.whl", hash = "sha256:0b71e63226e393b534105fcbdd8740410dc6b0854c2bfa39bbda6b0d40e59a54"}, + {file = "regex-2023.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bbb02fd4462f37060122e5acacec78e49c0fbb303c30dd49c7f493cf21fc5b27"}, + {file = "regex-2023.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b862c2b9d5ae38a68b92e215b93f98d4c5e9454fa36aae4450f61dd33ff48487"}, + {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976d7a304b59ede34ca2921305b57356694f9e6879db323fd90a80f865d355a3"}, + {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:83320a09188e0e6c39088355d423aa9d056ad57a0b6c6381b300ec1a04ec3d16"}, + {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9427a399501818a7564f8c90eced1e9e20709ece36be701f394ada99890ea4b3"}, + {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178bbc1b2ec40eaca599d13c092079bf529679bf0371c602edaa555e10b41c3"}, + {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:837328d14cde912af625d5f303ec29f7e28cdab588674897baafaf505341f2fc"}, + {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d44dc13229905ae96dd2ae2dd7cebf824ee92bc52e8cf03dcead37d926da019"}, + {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d54af539295392611e7efbe94e827311eb8b29668e2b3f4cadcfe6f46df9c777"}, + {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7117d10690c38a622e54c432dfbbd3cbd92f09401d622902c32f6d377e2300ee"}, + {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bb60b503ec8a6e4e3e03a681072fa3a5adcbfa5479fa2d898ae2b4a8e24c4591"}, + {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:65ba8603753cec91c71de423a943ba506363b0e5c3fdb913ef8f9caa14b2c7e0"}, + {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:271f0bdba3c70b58e6f500b205d10a36fb4b58bd06ac61381b68de66442efddb"}, + {file = "regex-2023.6.3-cp37-cp37m-win32.whl", hash = "sha256:9beb322958aaca059f34975b0df135181f2e5d7a13b84d3e0e45434749cb20f7"}, + {file = "regex-2023.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fea75c3710d4f31389eed3c02f62d0b66a9da282521075061ce875eb5300cf23"}, + {file = "regex-2023.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8f56fcb7ff7bf7404becdfc60b1e81a6d0561807051fd2f1860b0d0348156a07"}, + {file = "regex-2023.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d2da3abc88711bce7557412310dfa50327d5769a31d1c894b58eb256459dc289"}, + {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a99b50300df5add73d307cf66abea093304a07eb017bce94f01e795090dea87c"}, + {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5708089ed5b40a7b2dc561e0c8baa9535b77771b64a8330b684823cfd5116036"}, + {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:687ea9d78a4b1cf82f8479cab23678aff723108df3edeac098e5b2498879f4a7"}, + {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d3850beab9f527f06ccc94b446c864059c57651b3f911fddb8d9d3ec1d1b25d"}, + {file = "regex-2023.6.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8915cc96abeb8983cea1df3c939e3c6e1ac778340c17732eb63bb96247b91d2"}, + {file = "regex-2023.6.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:841d6e0e5663d4c7b4c8099c9997be748677d46cbf43f9f471150e560791f7ff"}, + {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9edce5281f965cf135e19840f4d93d55b3835122aa76ccacfd389e880ba4cf82"}, + {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b956231ebdc45f5b7a2e1f90f66a12be9610ce775fe1b1d50414aac1e9206c06"}, + {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:36efeba71c6539d23c4643be88295ce8c82c88bbd7c65e8a24081d2ca123da3f"}, + {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:cf67ca618b4fd34aee78740bea954d7c69fdda419eb208c2c0c7060bb822d747"}, + {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b4598b1897837067a57b08147a68ac026c1e73b31ef6e36deeeb1fa60b2933c9"}, + {file = "regex-2023.6.3-cp38-cp38-win32.whl", hash = "sha256:f415f802fbcafed5dcc694c13b1292f07fe0befdb94aa8a52905bd115ff41e88"}, + {file = "regex-2023.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:d4f03bb71d482f979bda92e1427f3ec9b220e62a7dd337af0aa6b47bf4498f72"}, + {file = "regex-2023.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ccf91346b7bd20c790310c4147eee6ed495a54ddb6737162a36ce9dbef3e4751"}, + {file = "regex-2023.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b28f5024a3a041009eb4c333863d7894d191215b39576535c6734cd88b0fcb68"}, + {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0bb18053dfcfed432cc3ac632b5e5e5c5b7e55fb3f8090e867bfd9b054dbcbf"}, + {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a5bfb3004f2144a084a16ce19ca56b8ac46e6fd0651f54269fc9e230edb5e4a"}, + {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c6b48d0fa50d8f4df3daf451be7f9689c2bde1a52b1225c5926e3f54b6a9ed1"}, + {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051da80e6eeb6e239e394ae60704d2b566aa6a7aed6f2890a7967307267a5dc6"}, + {file = "regex-2023.6.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4c3b7fa4cdaa69268748665a1a6ff70c014d39bb69c50fda64b396c9116cf77"}, + {file = "regex-2023.6.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:457b6cce21bee41ac292d6753d5e94dcbc5c9e3e3a834da285b0bde7aa4a11e9"}, + {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aad51907d74fc183033ad796dd4c2e080d1adcc4fd3c0fd4fd499f30c03011cd"}, + {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0385e73da22363778ef2324950e08b689abdf0b108a7d8decb403ad7f5191938"}, + {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c6a57b742133830eec44d9b2290daf5cbe0a2f1d6acee1b3c7b1c7b2f3606df7"}, + {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:3e5219bf9e75993d73ab3d25985c857c77e614525fac9ae02b1bebd92f7cecac"}, + {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e5087a3c59eef624a4591ef9eaa6e9a8d8a94c779dade95d27c0bc24650261cd"}, + {file = "regex-2023.6.3-cp39-cp39-win32.whl", hash = "sha256:20326216cc2afe69b6e98528160b225d72f85ab080cbdf0b11528cbbaba2248f"}, + {file = "regex-2023.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:bdff5eab10e59cf26bc479f565e25ed71a7d041d1ded04ccf9aee1d9f208487a"}, + {file = "regex-2023.6.3.tar.gz", hash = "sha256:72d1a25bf36d2050ceb35b517afe13864865268dfb45910e2e17a84be6cbfeb0"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, ] -safetensors = [ + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "safetensors" +version = "0.3.1" +description = "Fast and Safe Tensor serialization" +optional = false +python-versions = "*" +files = [ {file = "safetensors-0.3.1-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:2ae9b7dd268b4bae6624729dac86deb82104820e9786429b0583e5168db2f770"}, {file = "safetensors-0.3.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:08c85c1934682f1e2cd904d38433b53cd2a98245a7cc31f5689f9322a2320bbf"}, {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba625c7af9e1c5d0d91cb83d2fba97d29ea69d4db2015d9714d24c7f6d488e15"}, @@ -1257,7 +1144,25 @@ safetensors = [ {file = "safetensors-0.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:5f4f614b8e8161cd8a9ca19c765d176a82b122fa3d3387b77862145bfe9b4e93"}, {file = "safetensors-0.3.1.tar.gz", hash = "sha256:571da56ff8d0bec8ae54923b621cda98d36dcef10feb36fd492c4d0c2cd0e869"}, ] -sentencepiece = [ + +[package.extras] +all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] +dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] +numpy = ["numpy (>=1.21.6)"] +paddlepaddle = ["paddlepaddle (>=2.4.1)"] +quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] +tensorflow = ["tensorflow (>=2.11.0)"] +testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] +torch = ["torch (>=1.10)"] + +[[package]] +name = "sentencepiece" +version = "0.1.99" +description = "SentencePiece python wrapper" +optional = false +python-versions = "*" +files = [ {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0eb528e70571b7c02723e5804322469b82fe7ea418c96051d0286c0fa028db73"}, {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:77d7fafb2c4e4659cbdf303929503f37a26eabc4ff31d3a79bf1c5a1b338caa7"}, {file = "sentencepiece-0.1.99-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be9cf5b9e404c245aeb3d3723c737ba7a8f5d4ba262ef233a431fa6c45f732a0"}, @@ -1304,15 +1209,44 @@ sentencepiece = [ {file = "sentencepiece-0.1.99-cp39-cp39-win_amd64.whl", hash = "sha256:350e5c74d739973f1c9643edb80f7cc904dc948578bcb1d43c6f2b173e5d18dd"}, {file = "sentencepiece-0.1.99.tar.gz", hash = "sha256:189c48f5cb2949288f97ccdb97f0473098d9c3dcf5a3d99d4eabe719ec27297f"}, ] -setuptools = [ + +[[package]] +name = "setuptools" +version = "67.8.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.7" +files = [ {file = "setuptools-67.8.0-py3-none-any.whl", hash = "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f"}, {file = "setuptools-67.8.0.tar.gz", hash = "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"}, ] -sympy = [ + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "sympy" +version = "1.12" +description = "Computer algebra system (CAS) in Python" +optional = true +python-versions = ">=3.8" +files = [ {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, {file = "sympy-1.12.tar.gz", hash = "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8"}, ] -tokenizers = [ + +[package.dependencies] +mpmath = ">=0.19" + +[[package]] +name = "tokenizers" +version = "0.13.3" +description = "Fast and Customizable Tokenizers" +optional = false +python-versions = "*" +files = [ {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, {file = "tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07"}, {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc"}, @@ -1354,11 +1288,30 @@ tokenizers = [ {file = "tokenizers-0.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783"}, {file = "tokenizers-0.13.3.tar.gz", hash = "sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e"}, ] -tomli = [ + +[package.extras] +dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] +docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -torch = [ + +[[package]] +name = "torch" +version = "2.0.1" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +optional = true +python-versions = ">=3.8.0" +files = [ {file = "torch-2.0.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:8ced00b3ba471856b993822508f77c98f48a458623596a4c43136158781e306a"}, {file = "torch-2.0.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:359bfaad94d1cda02ab775dc1cc386d585712329bb47b8741607ef6ef4950747"}, {file = "torch-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7c84e44d9002182edd859f3400deaa7410f5ec948a519cc7ef512c2f9b34d2c4"}, @@ -1380,27 +1333,175 @@ torch = [ {file = "torch-2.0.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:c62df99352bd6ee5a5a8d1832452110435d178b5164de450831a3a8cc14dc680"}, {file = "torch-2.0.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:671a2565e3f63b8fe8e42ae3e36ad249fe5e567435ea27b94edaa672a7d0c416"}, ] -tqdm = [ + +[package.dependencies] +filelock = "*" +jinja2 = "*" +networkx = "*" +sympy = "*" +typing-extensions = "*" + +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] + +[[package]] +name = "tqdm" +version = "4.65.0" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, ] -typer = [ + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "transformers" +version = "4.30.2" +description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "transformers-4.30.2-py3-none-any.whl", hash = "sha256:c332e3a3097f9ed89ce556b403251235931c00237b8bc2d7adaa19d226c13f1d"}, + {file = "transformers-4.30.2.tar.gz", hash = "sha256:f4a8aac4e1baffab4033f4a345b0d7dc7957d12a4f1ba969afea08205a513045"}, +] + +[package.dependencies] +filelock = "*" +huggingface-hub = ">=0.14.1,<1.0" +numpy = ">=1.17" +packaging = ">=20.0" +pyyaml = ">=5.1" +regex = "!=2019.12.17" +requests = "*" +safetensors = ">=0.3.1" +tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" +tqdm = ">=4.27" + +[package.extras] +accelerate = ["accelerate (>=0.20.2)"] +agents = ["Pillow", "accelerate (>=0.20.2)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] +all = ["Pillow", "accelerate (>=0.20.2)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.3)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +codecarbon = ["codecarbon (==1.2.0)"] +deepspeed = ["accelerate (>=0.20.2)", "deepspeed (>=0.8.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.20.2)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.8.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf (<=3.20.3)", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.20.2)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.6.9)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.3)", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf (<=3.20.3)", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.20.2)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.3)", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow", "accelerate (>=0.20.2)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.3)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +docs-specific = ["hf-doc-builder"] +fairscale = ["fairscale (>0.3)"] +flax = ["flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "optax (>=0.0.8,<=0.1.4)"] +flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +ftfy = ["ftfy"] +integrations = ["optuna", "ray[tune]", "sigopt"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] +modelcreation = ["cookiecutter (==1.7.3)"] +natten = ["natten (>=0.14.6)"] +onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] +onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] +optuna = ["optuna"] +quality = ["GitPython (<3.1.19)", "black (>=23.1,<24.0)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (>=0.0.241,<=0.0.259)", "urllib3 (<2.0.0)"] +ray = ["ray[tune]"] +retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] +sagemaker = ["sagemaker (>=2.31.0)"] +sentencepiece = ["protobuf (<=3.20.3)", "sentencepiece (>=0.1.91,!=0.1.92)"] +serving = ["fastapi", "pydantic", "starlette", "uvicorn"] +sigopt = ["sigopt"] +sklearn = ["scikit-learn"] +speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf (<=3.20.3)", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] +tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +timm = ["timm"] +tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] +torch = ["accelerate (>=0.20.2)", "torch (>=1.9,!=1.12.0)"] +torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +torch-vision = ["Pillow", "torchvision"] +torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf (<=3.20.3)", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] +video = ["av (==9.2.0)", "decord (==0.6.0)"] +vision = ["Pillow"] + +[[package]] +name = "typer" +version = "0.6.1" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.6" +files = [ {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, ] -typing-extensions = [ - {file = "typing_extensions-4.6.0-py3-none-any.whl", hash = "sha256:6ad00b63f849b7dcc313b70b6b304ed67b2b2963b3098a33efe18056b1a9a223"}, - {file = "typing_extensions-4.6.0.tar.gz", hash = "sha256:ff6b238610c747e44c268aa4bb23c8c735d665a63726df3f9431ce707f2aa768"}, + +[package.dependencies] +click = ">=7.1.1,<9.0.0" + +[package.extras] +all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] + +[[package]] +name = "typing-extensions" +version = "4.6.3" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.6.3-py3-none-any.whl", hash = "sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26"}, + {file = "typing_extensions-4.6.3.tar.gz", hash = "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5"}, ] -urllib3 = [ - {file = "urllib3-2.0.2-py3-none-any.whl", hash = "sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e"}, - {file = "urllib3-2.0.2.tar.gz", hash = "sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc"}, + +[[package]] +name = "urllib3" +version = "2.0.3" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.7" +files = [ + {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, + {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, ] -win32-setctime = [ + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +optional = false +python-versions = ">=3.5" +files = [ {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, ] -wrapt = [ + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + +[[package]] +name = "wrapt" +version = "1.15.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, @@ -1477,3 +1578,12 @@ wrapt = [ {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] + +[extras] +accelerate = ["accelerate"] +bnb = ["bitsandbytes"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "54ecacb32d699cb1298c237c4661c1b707f119cf2c27bd54bad7a1ea2ffb8b10" diff --git a/server/requirements.txt b/server/requirements.txt index e8cee52b..a9bd441c 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,21 +1,21 @@ backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" -bitsandbytes==0.38.1 ; python_version >= "3.9" and python_version < "4.0" certifi==2023.5.7 ; python_version >= "3.9" and python_version < "4.0" charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0" click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" -colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows" -deprecated==1.2.13 ; python_version >= "3.9" and python_version < "4.0" -filelock==3.12.0 ; python_version >= "3.9" and python_version < "4.0" -fsspec==2023.5.0 ; python_version >= "3.9" and python_version < "4.0" -googleapis-common-protos==1.59.0 ; python_version >= "3.9" and python_version < "4.0" +colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows") +deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0" +filelock==3.12.2 ; python_version >= "3.9" and python_version < "4.0" +fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "4.0" +googleapis-common-protos==1.59.1 ; python_version >= "3.9" and python_version < "4.0" grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "4.0" -grpcio-reflection==1.55.0 ; python_version >= "3.9" and python_version < "4.0" -grpcio-status==1.55.0 ; python_version >= "3.9" and python_version < "4.0" -grpcio==1.55.0 ; python_version >= "3.9" and python_version < "4.0" +grpcio-reflection==1.54.2 ; python_version >= "3.9" and python_version < "4.0" +grpcio-status==1.54.2 ; python_version >= "3.9" and python_version < "4.0" +grpcio==1.54.2 ; python_version >= "3.9" and python_version < "4.0" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" huggingface-hub==0.14.1 ; python_version >= "3.9" and python_version < "4.0" -idna==3.4 ; python_version >= "3.9" and python_version < "4" +idna==3.4 ; python_version >= "3.9" and python_version < "4.0" loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" +numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "4.0" @@ -26,17 +26,18 @@ opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" -protobuf==4.23.1 ; python_version >= "3.9" and python_version < "4.0" +protobuf==4.23.2 ; python_version >= "3.9" and python_version < "4.0" pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" +regex==2023.6.3 ; python_version >= "3.9" and python_version < "4.0" requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0" safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "4.0" setuptools==67.8.0 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" -transformers==4.29.2 ; python_version >= "3.9" and python_version < "4.0" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" +transformers==4.30.2 ; python_version >= "3.9" and python_version < "4.0" typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" -typing-extensions==4.6.0 ; python_version >= "3.9" and python_version < "4.0" -urllib3==2.0.2 ; python_version >= "3.9" and python_version < "4.0" +typing-extensions==4.6.3 ; python_version >= "3.9" and python_version < "4.0" +urllib3==2.0.3 ; python_version >= "3.9" and python_version < "4.0" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index c0e6c2dc..aeb1f13b 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -151,5 +151,37 @@ def download_weights( utils.convert_files(local_pt_files, local_st_files) +@app.command() +def quantize( + model_id: str, + output_dir: str, + revision: Optional[str] = None, + logger_level: str = "INFO", + json_output: bool = False, + trust_remote_code: bool = False, + upload_to_model_id: Optional[str] = None, + percdamp: float = 0.01, + act_order: bool = False, +): + download_weights( + model_id=model_id, + revision=revision, + logger_level=logger_level, + json_output=json_output, + ) + from text_generation_server.utils.gptq.quantize import quantize + + quantize( + model_id=model_id, + bits=4, + groupsize=128, + output_dir=output_dir, + trust_remote_code=trust_remote_code, + upload_to_model_id=upload_to_model_id, + percdamp=percdamp, + act_order=act_order, + ) + + if __name__ == "__main__": app() diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 3fdc23b2..2abde685 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -246,6 +246,10 @@ def get_model( if sharded: raise ValueError("sharded is not supported for AutoModel") + if quantize == "gptq": + raise ValueError( + "gptq quantization is not supported for AutoModel, you can try to quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" + ) if model_type in modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: return CausalLM( diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 3586b85a..9c1020a5 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -42,7 +42,8 @@ def load_row(config, prefix: str, weights, bias: bool): - weight = weights.get_sharded(f"{prefix}.weight", dim=1) + weight = weights.get_multi_weights_row(prefix, quantize=config.quantize) + if bias and weights.process_group.rank() == 0: # Rank is only on the first rank process bias = weights.get_tensor(f"{prefix}.bias") @@ -57,19 +58,21 @@ def load_row(config, prefix: str, weights, bias: bool): def load_qkv(config, prefix: str, weights, num_heads, head_size, hidden_size): - weight = weights.get_sharded(f"{prefix}.weight", dim=0) - bias = weights.get_sharded(f"{prefix}.bias", dim=0) - - weight = ( - weight.view( - num_heads, - 3, - head_size, - hidden_size, + weight = weights.get_multi_weights_col([prefix], quantize=config.quantize, dim=0) + if isinstance(weight, torch.Tensor): + # Only on non quantized versions + weight = ( + weight.view( + num_heads, + 3, + head_size, + hidden_size, + ) + .permute(1, 0, 2, 3) + .reshape(-1, hidden_size) ) - .permute(1, 0, 2, 3) - .reshape(-1, hidden_size) - ) + + bias = weights.get_sharded(f"{prefix}.bias", dim=0) bias = bias.view(num_heads, 3, head_size).permute(1, 0, 2).reshape(-1) linear = get_linear(weight, bias, config.quantize) diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 4a9063eb..fa35c359 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -21,7 +21,8 @@ def load_row(config, prefix: str, weights, bias: bool): - weight = weights.get_sharded(f"{prefix}.weight", dim=1) + weight = weights.get_multi_weights_row(prefix, quantize=config.quantize) + if bias and weights.process_group.rank() == 0: # Rank is only on the first rank process bias = weights.get_tensor(f"{prefix}.bias") diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index b01d752a..4eb0034d 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -21,6 +21,81 @@ def load_multi_mqa( config, prefix: str, weights, bias: bool, head_size, num_heads, hidden_size ): + + if config.quantize == "gptq": + return _load_multi_mqa_gptq( + config, prefix, weights, bias, head_size, num_heads, hidden_size + ) + else: + return _load_multi_mqa( + config, prefix, weights, bias, head_size, num_heads, hidden_size + ) + + +def _load_multi_mqa_gptq( + config, prefix: str, weights, bias: bool, head_size, num_heads, hidden_size +): + if any("c_attn" in k for k in weights.routing.keys()) and not config.transpose: + world_size = weights.process_group.size() + rank = weights.process_group.rank() + + slice_ = weights._get_slice(f"{prefix}.c_attn.qweight") + shape = slice_.get_shape() + block_size = (shape[1] - 2 * head_size) // world_size + start = rank * block_size + stop = (rank + 1) * block_size + assert (shape[1] - 2 * head_size) % world_size == 0 + q_tensor = slice_[:, start:stop] + kv_tensor = slice_[:, -2 * head_size :] + qweight = torch.cat([q_tensor, kv_tensor], dim=1) + + slice_ = weights._get_slice(f"{prefix}.c_attn.scales") + shape = slice_.get_shape() + block_size = (shape[1] - 2 * head_size) // world_size + start = rank * block_size + stop = (rank + 1) * block_size + assert (shape[1] - 2 * head_size) % world_size == 0 + q_tensor = slice_[:, start:stop] + kv_tensor = slice_[:, -2 * head_size :] + scales = torch.cat([q_tensor, kv_tensor], dim=1) + + slice_ = weights._get_slice(f"{prefix}.c_attn.qzeros") + shape = slice_.get_shape() + block_size = (shape[1] - (2 * head_size) * 4 // 32) // world_size + start = rank * block_size + stop = (rank + 1) * block_size + assert 2 * head_size % (32 // 4) == 0 + q_tensor = slice_[:, start:stop] + kv_tensor = slice_[:, -2 * head_size * 4 // 32 :] + qzeros = torch.cat([q_tensor, kv_tensor], dim=1) + + g_idx = weights.get_tensor(f"{prefix}.c_attn.g_idx") + bits = weights.get_tensor("gptq_bits").item() + groupsize = weights.get_tensor("gptq_groupsize").item() + + weight = (qweight, qzeros, scales, g_idx, bits, groupsize) + + if bias: + slice_ = weights._get_slice(f"{prefix}.c_attn.bias") + shape = slice_.get_shape() + block_size = (shape[0] - 2 * head_size) // world_size + assert (shape[0] - 2 * head_size) % world_size == 0 + q_tensor = slice_[start:stop] + start = rank * block_size + stop = (rank + 1) * block_size + q_tensor = slice_[start:stop] + kv_tensor = slice_[-2 * head_size :] + bias = torch.cat([q_tensor, kv_tensor], dim=0) + + return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize)) + else: + raise NotImplementedError("Gptq loading with santacoder is not implemented") + + +def _load_multi_mqa( + config, prefix: str, weights, bias: bool, head_size, num_heads, hidden_size +): + if any("c_attn" in k for k in weights.routing.keys()): slice_ = weights._get_slice(f"{prefix}.c_attn.weight") shape = slice_.get_shape() @@ -92,7 +167,9 @@ def load_col(config, prefix: str, weights, bias: bool): if config.transpose: weight = weights.get_sharded(f"{prefix}.weight", dim=1).T else: - weight = weights.get_sharded(f"{prefix}.weight", dim=0) + weight = weights.get_multi_weights_col( + [prefix], quantize=config.quantize, dim=0 + ) if bias: bias = weights.get_sharded(f"{prefix}.bias", dim=0) @@ -105,7 +182,7 @@ def load_row(config, prefix: str, weights, bias: bool): if config.transpose: weight = weights.get_sharded(f"{prefix}.weight", dim=0).T else: - weight = weights.get_sharded(f"{prefix}.weight", dim=1) + weight = weights.get_multi_weights_row(prefix, quantize=config.quantize) if bias and weights.process_group.rank() == 0: # Rank is only on the first rank process diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index eb216a20..a80d58cb 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -3,7 +3,7 @@ from opentelemetry import trace from transformers import AutoConfig -from transformers.models.llama import LlamaTokenizer +from transformers.models.llama import LlamaTokenizer, LlamaTokenizerFast from typing import Optional from text_generation_server.models import FlashCausalLM @@ -34,13 +34,22 @@ def __init__( else: raise NotImplementedError("FlashLlama is only available on GPU") - tokenizer = LlamaTokenizer.from_pretrained( - model_id, - revision=revision, - padding_side="left", - truncation_side="left", - trust_remote_code=trust_remote_code, - ) + try: + tokenizer = LlamaTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + except Exception: + tokenizer = LlamaTokenizerFast.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) config = AutoConfig.from_pretrained( model_id, revision=revision, trust_remote_code=trust_remote_code diff --git a/server/text_generation_server/utils/gptq/custom_autotune.py b/server/text_generation_server/utils/gptq/custom_autotune.py new file mode 100644 index 00000000..17dff02e --- /dev/null +++ b/server/text_generation_server/utils/gptq/custom_autotune.py @@ -0,0 +1,261 @@ +# https://github.com/fpgaminer/GPTQ-triton +""" +Mostly the same as the autotuner in Triton, but with a few changes like using 40 runs instead of 100. +""" + +import builtins +import math +import time +from typing import Dict + +import triton + + +class Autotuner(triton.KernelInterface): + def __init__( + self, + fn, + arg_names, + configs, + key, + reset_to_zero, + prune_configs_by: Dict = None, + nearest_power_of_two: bool = False, + ): + """ + :param prune_configs_by: a dict of functions that are used to prune configs, fields: + 'perf_model': performance model used to predicate running time with different configs, returns running time + 'top_k': number of configs to bench + 'prune_num_stages_by'(optional): a function used to prune num_stages. It take configs:List[Config] as its input, and returns pruned configs. + 'nearest_power_of_two'(optional): whether to round key arguments to the nearest power of two when caching tuning results + """ + if not configs: + self.configs = [triton.Config({}, num_warps=4, num_stages=2)] + else: + self.configs = configs + self.key_idx = [arg_names.index(k) for k in key] + self.nearest_power_of_two = nearest_power_of_two + self.cache = {} + # hook to reset all required tensor to zeros before relaunching a kernel + self.hook = lambda args: 0 + if reset_to_zero is not None: + self.reset_idx = [arg_names.index(k) for k in reset_to_zero] + + def _hook(args): + for i in self.reset_idx: + args[i].zero_() + + self.hook = _hook + self.arg_names = arg_names + # prune configs + if prune_configs_by: + perf_model, top_k = ( + prune_configs_by["perf_model"], + prune_configs_by["top_k"], + ) + if "early_config_prune" in prune_configs_by: + early_config_prune = prune_configs_by["early_config_prune"] + else: + perf_model, top_k, early_config_prune = None, None, None + self.perf_model, self.configs_top_k = perf_model, top_k + self.early_config_prune = early_config_prune + self.fn = fn + + def _bench(self, *args, config, **meta): + # check for conflicts, i.e. meta-parameters both provided + # as kwargs and by the autotuner + conflicts = meta.keys() & config.kwargs.keys() + if conflicts: + raise ValueError( + f"Conflicting meta-parameters: {', '.join(conflicts)}." + " Make sure that you don't re-define auto-tuned symbols." + ) + # augment meta-parameters with tunable ones + current = dict(meta, **config.kwargs) + + def kernel_call(): + if config.pre_hook: + config.pre_hook(self.nargs) + self.hook(args) + self.fn.run( + *args, + num_warps=config.num_warps, + num_stages=config.num_stages, + **current, + ) + + try: + # In testings using only 40 reps seems to be close enough and it appears to be what PyTorch uses + # PyTorch also sets fast_flush to True, but I didn't see any speedup so I'll leave the default + return triton.testing.do_bench( + kernel_call, percentiles=(0.5, 0.2, 0.8), rep=40 + ) + except triton.compiler.OutOfResources: + return (float("inf"), float("inf"), float("inf")) + + def run(self, *args, **kwargs): + self.nargs = dict(zip(self.arg_names, args)) + if len(self.configs) > 1: + key = tuple(args[i] for i in self.key_idx) + + # This reduces the amount of autotuning by rounding the keys to the nearest power of two + # In my testing this gives decent results, and greatly reduces the amount of tuning required + if self.nearest_power_of_two: + key = tuple([2 ** int(math.log2(x) + 0.5) for x in key]) + + if key not in self.cache: + # prune configs + pruned_configs = self.prune_configs(kwargs) + bench_start = time.time() + timings = { + config: self._bench(*args, config=config, **kwargs) + for config in pruned_configs + } + bench_end = time.time() + self.bench_time = bench_end - bench_start + self.cache[key] = builtins.min(timings, key=timings.get) + self.hook(args) + self.configs_timings = timings + config = self.cache[key] + else: + config = self.configs[0] + self.best_config = config + if config.pre_hook is not None: + config.pre_hook(self.nargs) + return self.fn.run( + *args, + num_warps=config.num_warps, + num_stages=config.num_stages, + **kwargs, + **config.kwargs, + ) + + def prune_configs(self, kwargs): + pruned_configs = self.configs + if self.early_config_prune: + pruned_configs = self.early_config_prune(self.configs, self.nargs) + if self.perf_model: + top_k = self.configs_top_k + if isinstance(top_k, float) and top_k <= 1.0: + top_k = int(len(self.configs) * top_k) + if len(pruned_configs) > top_k: + est_timing = { + config: self.perf_model( + **self.nargs, + **kwargs, + **config.kwargs, + num_stages=config.num_stages, + num_warps=config.num_warps, + ) + for config in pruned_configs + } + pruned_configs = sorted(est_timing.keys(), key=lambda x: est_timing[x])[ + :top_k + ] + return pruned_configs + + def warmup(self, *args, **kwargs): + self.nargs = dict(zip(self.arg_names, args)) + for config in self.prune_configs(kwargs): + self.fn.warmup( + *args, + num_warps=config.num_warps, + num_stages=config.num_stages, + **kwargs, + **config.kwargs, + ) + self.nargs = None + + +def autotune( + configs, key, prune_configs_by=None, reset_to_zero=None, nearest_power_of_two=False +): + """ + Decorator for auto-tuning a :code:`triton.jit`'d function. + .. highlight:: python + .. code-block:: python + @triton.autotune(configs=[ + triton.Config(meta={'BLOCK_SIZE': 128}, num_warps=4), + triton.Config(meta={'BLOCK_SIZE': 1024}, num_warps=8), + ], + key=['x_size'] # the two above configs will be evaluated anytime + # the value of x_size changes + ) + @triton.jit + def kernel(x_ptr, x_size, **META): + BLOCK_SIZE = META['BLOCK_SIZE'] + :note: When all the configurations are evaluated, the kernel will run multiple time. + This means that whatever value the kernel updates will be updated multiple times. + To avoid this undesired behavior, you can use the `reset_to_zero` argument, which + reset the value of the provided tensor to `zero` before running any configuration. + :param configs: a list of :code:`triton.Config` objects + :type configs: list[triton.Config] + :param key: a list of argument names whose change in value will trigger the evaluation of all provided configs. + :type key: list[str] + :param prune_configs_by: a dict of functions that are used to prune configs, fields: + 'perf_model': performance model used to predicate running time with different configs, returns running time + 'top_k': number of configs to bench + 'early_config_prune'(optional): a function used to do early prune (eg, num_stages). It take configs:List[Config] as its input, and returns pruned configs. + :param reset_to_zero: a list of argument names whose value will be reset to zero before evaluating any configs. + :type reset_to_zero: list[str] + """ + + def decorator(fn): + return Autotuner( + fn, + fn.arg_names, + configs, + key, + reset_to_zero, + prune_configs_by, + nearest_power_of_two, + ) + + return decorator + + +def matmul248_kernel_config_pruner(configs, nargs): + """ + The main purpose of this function is to shrink BLOCK_SIZE_* when the corresponding dimension is smaller. + """ + m = max(2 ** int(math.ceil(math.log2(nargs["M"]))), 16) + n = max(2 ** int(math.ceil(math.log2(nargs["N"]))), 16) + k = max(2 ** int(math.ceil(math.log2(nargs["K"]))), 16) + + used = set() + for config in configs: + block_size_m = min(m, config.kwargs["BLOCK_SIZE_M"]) + block_size_n = min(n, config.kwargs["BLOCK_SIZE_N"]) + block_size_k = min(k, config.kwargs["BLOCK_SIZE_K"]) + group_size_m = config.kwargs["GROUP_SIZE_M"] + + if ( + block_size_m, + block_size_n, + block_size_k, + group_size_m, + config.num_stages, + config.num_warps, + ) in used: + continue + + used.add( + ( + block_size_m, + block_size_n, + block_size_k, + group_size_m, + config.num_stages, + config.num_warps, + ) + ) + yield triton.Config( + { + "BLOCK_SIZE_M": block_size_m, + "BLOCK_SIZE_N": block_size_n, + "BLOCK_SIZE_K": block_size_k, + "GROUP_SIZE_M": group_size_m, + }, + num_stages=config.num_stages, + num_warps=config.num_warps, + ) diff --git a/server/text_generation_server/utils/gptq/quant_linear.py b/server/text_generation_server/utils/gptq/quant_linear.py new file mode 100644 index 00000000..54fa2014 --- /dev/null +++ b/server/text_generation_server/utils/gptq/quant_linear.py @@ -0,0 +1,359 @@ +import math +import numpy as np +import torch +import torch.nn as nn +from torch.cuda.amp import custom_bwd, custom_fwd + +try: + import triton + import triton.language as tl + from . import custom_autotune + + # code based https://github.com/fpgaminer/GPTQ-triton + @custom_autotune.autotune( + configs=[ + triton.Config( + { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + }, + num_stages=4, + num_warps=4, + ), + triton.Config( + { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + }, + num_stages=2, + num_warps=8, + ), + triton.Config( + { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + }, + num_stages=3, + num_warps=8, + ), + triton.Config( + { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + }, + num_stages=2, + num_warps=4, + ), + ], + key=["M", "N", "K"], + nearest_power_of_two=True, + prune_configs_by={ + "early_config_prune": custom_autotune.matmul248_kernel_config_pruner, + "perf_model": None, + "top_k": None, + }, + ) + @triton.jit + def matmul_248_kernel( + a_ptr, + b_ptr, + c_ptr, + scales_ptr, + zeros_ptr, + g_ptr, + M, + N, + K, + bits, + maxq, + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_scales, + stride_zeros, + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + ): + """ + Compute the matrix multiplication C = A x B. + A is of shape (M, K) float16 + B is of shape (K//8, N) int32 + C is of shape (M, N) float16 + scales is of shape (G, N) float16 + zeros is of shape (G, N) float16 + g_ptr is of shape (K) int32 + """ + infearure_per_bits = 32 // bits + + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_k = tl.cdiv(K, BLOCK_SIZE_K) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + ( + offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak + ) # (BLOCK_SIZE_M, BLOCK_SIZE_K) + a_mask = offs_am[:, None] < M + # b_ptrs is set up such that it repeats elements along the K axis 8 times + b_ptrs = b_ptr + ( + (offs_k[:, None] // infearure_per_bits) * stride_bk + + offs_bn[None, :] * stride_bn + ) # (BLOCK_SIZE_K, BLOCK_SIZE_N) + g_ptrs = g_ptr + offs_k + # shifter is used to extract the N bits of each element in the 32-bit word from B + scales_ptrs = scales_ptr + offs_bn[None, :] + zeros_ptrs = zeros_ptr + (offs_bn[None, :] // infearure_per_bits) + + shifter = (offs_k % infearure_per_bits) * bits + zeros_shifter = (offs_bn % infearure_per_bits) * bits + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + + for k in range(0, num_pid_k): + g_idx = tl.load(g_ptrs) + + # Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop + scales = tl.load( + scales_ptrs + g_idx[:, None] * stride_scales + ) # (BLOCK_SIZE_K, BLOCK_SIZE_N,) + zeros = tl.load( + zeros_ptrs + g_idx[:, None] * stride_zeros + ) # (BLOCK_SIZE_K, BLOCK_SIZE_N,) + + zeros = (zeros >> zeros_shifter[None, :]) & maxq + zeros = zeros + 1 + + a = tl.load(a_ptrs, mask=a_mask, other=0.0) # (BLOCK_SIZE_M, BLOCK_SIZE_K) + b = tl.load(b_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N), but repeated + + # Now we need to unpack b (which is N-bit values) into 32-bit values + b = (b >> shifter[:, None]) & maxq # Extract the N-bit values + b = (b - zeros) * scales # Scale and shift + + accumulator += tl.dot(a, b) + a_ptrs += BLOCK_SIZE_K + b_ptrs += (BLOCK_SIZE_K // infearure_per_bits) * stride_bk + g_ptrs += BLOCK_SIZE_K + + c_ptrs = c_ptr + stride_cm * offs_am[:, None] + stride_cn * offs_bn[None, :] + c_mask = (offs_am[:, None] < M) & (offs_bn[None, :] < N) + tl.store(c_ptrs, accumulator, mask=c_mask) + +except: + print("triton not installed.") + + +def matmul248(input, qweight, scales, qzeros, g_idx, bits, maxq): + with torch.cuda.device(input.device): + output = torch.empty( + (input.shape[0], qweight.shape[1]), device=input.device, dtype=torch.float16 + ) + grid = lambda META: ( + triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"]) + * triton.cdiv(qweight.shape[1], META["BLOCK_SIZE_N"]), + ) + matmul_248_kernel[grid]( + input, + qweight, + output, + scales, + qzeros, + g_idx, + input.shape[0], + qweight.shape[1], + input.shape[1], + bits, + maxq, + input.stride(0), + input.stride(1), + qweight.stride(0), + qweight.stride(1), + output.stride(0), + output.stride(1), + scales.stride(0), + qzeros.stride(0), + ) + return output + + +class QuantLinearFunction(torch.autograd.Function): + @staticmethod + @custom_fwd(cast_inputs=torch.float16) + def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq): + output = matmul248(input, qweight, scales, qzeros, g_idx, bits, maxq) + return output + + +class QuantLinear(nn.Module): + def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): + super().__init__() + self.register_buffer("qweight", qweight) + self.register_buffer("qzeros", qzeros) + self.register_buffer("scales", scales) + self.register_buffer("g_idx", g_idx) + if bias is not None: + self.register_buffer("bias", bias) + else: + self.bias = None + if bits not in [2, 4, 8]: + raise NotImplementedError("Only 2,4,8 bits are supported.") + self.bits = bits + self.maxq = 2**self.bits - 1 + self.groupsize = groupsize + + self.outfeatures = qweight.shape[1] + self.infeatures = qweight.shape[0] * 32 // 4 + + @classmethod + def new(cls, bits, groupsize, infeatures, outfeatures, bias): + if bits not in [2, 4, 8]: + raise NotImplementedError("Only 2,4,8 bits are supported.") + + qweight = torch.zeros((infeatures // 32 * bits, outfeatures), dtype=torch.int32) + qzeros = torch.zeros( + (math.ceil(infeatures / groupsize), outfeatures // 32 * bits), + dtype=torch.int32, + ) + scales = torch.zeros( + (math.ceil(infeatures / groupsize), outfeatures), dtype=torch.float16 + ) + g_idx = torch.tensor( + [i // groupsize for i in range(infeatures)], dtype=torch.int32 + ) + if bias: + bias = torch.zeros((outfeatures), dtype=torch.float16) + else: + bias = None + return cls(qweight, qzeros, scales, g_idx, bias, bits, groupsize) + + def pack(self, linear, scales, zeros, g_idx=None): + self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx + + scales = scales.t().contiguous() + zeros = zeros.t().contiguous() + scale_zeros = zeros * scales + self.scales = scales.clone().half() + if linear.bias is not None: + self.bias = linear.bias.clone().half() + + intweight = [] + for idx in range(self.infeatures): + intweight.append( + torch.round( + (linear.weight.data[:, idx] + scale_zeros[self.g_idx[idx]]) + / self.scales[self.g_idx[idx]] + ).to(torch.int)[:, None] + ) + intweight = torch.cat(intweight, dim=1) + intweight = intweight.t().contiguous() + intweight = intweight.numpy().astype(np.uint32) + qweight = np.zeros( + (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32 + ) + i = 0 + row = 0 + while row < qweight.shape[0]: + if self.bits in [2, 4, 8]: + for j in range(i, i + (32 // self.bits)): + qweight[row] |= intweight[j] << (self.bits * (j - i)) + i += 32 // self.bits + row += 1 + else: + raise NotImplementedError("Only 2,4,8 bits are supported.") + + qweight = qweight.astype(np.int32) + self.qweight = torch.from_numpy(qweight) + + zeros -= 1 + zeros = zeros.numpy().astype(np.uint32) + qzeros = np.zeros( + (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32 + ) + i = 0 + col = 0 + while col < qzeros.shape[1]: + if self.bits in [2, 4, 8]: + for j in range(i, i + (32 // self.bits)): + qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i)) + i += 32 // self.bits + col += 1 + else: + raise NotImplementedError("Only 2,4,8 bits are supported.") + + qzeros = qzeros.astype(np.int32) + self.qzeros = torch.from_numpy(qzeros) + + def forward(self, x): + out_shape = x.shape[:-1] + (self.outfeatures,) + out = QuantLinearFunction.apply( + x.reshape(-1, x.shape[-1]), + self.qweight, + self.scales, + self.qzeros, + self.g_idx, + self.bits, + self.maxq, + ) + out = out + self.bias if self.bias is not None else out + return out.reshape(out_shape) diff --git a/server/text_generation_server/utils/gptq/quantize.py b/server/text_generation_server/utils/gptq/quantize.py new file mode 100644 index 00000000..5a4ed8da --- /dev/null +++ b/server/text_generation_server/utils/gptq/quantize.py @@ -0,0 +1,866 @@ +import argparse +import time +import numpy as np +import torch +import torch.nn as nn +import math +import json +import os + +from texttable import Texttable +from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer +import transformers +from huggingface_hub import HfApi +import numpy as np +import torch +from text_generation_server.utils.gptq.quant_linear import QuantLinear +from loguru import logger +from typing import Optional + +DEV = torch.device("cuda:0") + + +class Quantizer(nn.Module): + def __init__(self, shape=1): + super(Quantizer, self).__init__() + self.register_buffer("maxq", torch.tensor(0)) + self.register_buffer("scale", torch.zeros(shape)) + self.register_buffer("zero", torch.zeros(shape)) + + def configure( + self, + bits, + perchannel=False, + sym=True, + mse=False, + norm=2.4, + grid=100, + maxshrink=0.8, + trits=False, + ): + + self.maxq = torch.tensor(2**bits - 1) + self.perchannel = perchannel + self.sym = sym + self.mse = mse + self.norm = norm + self.grid = grid + self.maxshrink = maxshrink + if trits: + self.maxq = torch.tensor(-1) + self.scale = torch.zeros_like(self.scale) + + def _quantize(self, x, scale, zero, maxq): + if maxq < 0: + return (x > scale / 2).float() * scale + (x < zero / 2).float() * zero + q = torch.clamp(torch.round(x / scale) + zero, 0, maxq) + return scale * (q - zero) + + def find_params(self, x, weight=False): + dev = x.device + self.maxq = self.maxq.to(dev) + + shape = x.shape + if self.perchannel: + if weight: + x = x.flatten(1) + else: + if len(shape) == 4: + x = x.permute([1, 0, 2, 3]) + x = x.flatten(1) + if len(shape) == 3: + x = x.reshape((-1, shape[-1])).t() + if len(shape) == 2: + x = x.t() + else: + x = x.flatten().unsqueeze(0) + + tmp = torch.zeros(x.shape[0], device=dev) + xmin = torch.minimum(x.min(1)[0], tmp) + xmax = torch.maximum(x.max(1)[0], tmp) + + if self.sym: + xmax = torch.maximum(torch.abs(xmin), xmax) + tmp = xmin < 0 + if torch.any(tmp): + xmin[tmp] = -xmax[tmp] + tmp = (xmin == 0) & (xmax == 0) + xmin[tmp] = -1 + xmax[tmp] = +1 + + if self.maxq < 0: + self.scale = xmax + self.zero = xmin + else: + self.scale = (xmax - xmin) / self.maxq + if self.sym: + self.zero = torch.full_like(self.scale, (self.maxq + 1) / 2) + else: + self.zero = torch.round(-xmin / self.scale) + + if self.mse: + best = torch.full([x.shape[0]], float("inf"), device=dev) + for i in range(int(self.maxshrink * self.grid)): + p = 1 - i / self.grid + xmin1 = p * xmin + xmax1 = p * xmax + scale1 = (xmax1 - xmin1) / self.maxq + zero1 = torch.round(-xmin1 / scale1) if not self.sym else self.zero + q = self._quantize( + x, scale1.unsqueeze(1), zero1.unsqueeze(1), self.maxq + ) + q -= x + q.abs_() + q.pow_(self.norm) + err = torch.sum(q, 1) + tmp = err < best + if torch.any(tmp): + best[tmp] = err[tmp] + self.scale[tmp] = scale1[tmp] + self.zero[tmp] = zero1[tmp] + if not self.perchannel: + if weight: + tmp = shape[0] + else: + tmp = shape[1] if len(shape) != 3 else shape[2] + self.scale = self.scale.repeat(tmp) + self.zero = self.zero.repeat(tmp) + + if weight: + shape = [-1] + [1] * (len(shape) - 1) + self.scale = self.scale.reshape(shape) + self.zero = self.zero.reshape(shape) + return + if len(shape) == 4: + self.scale = self.scale.reshape((1, -1, 1, 1)) + self.zero = self.zero.reshape((1, -1, 1, 1)) + if len(shape) == 3: + self.scale = self.scale.reshape((1, 1, -1)) + self.zero = self.zero.reshape((1, 1, -1)) + if len(shape) == 2: + self.scale = self.scale.unsqueeze(0) + self.zero = self.zero.unsqueeze(0) + + def quantize(self, x): + if self.ready(): + return self._quantize(x, self.scale, self.zero, self.maxq) + + return x + + def enabled(self): + return self.maxq > 0 + + def ready(self): + return torch.all(self.scale != 0) + + +class GPTQ: + def __init__(self, layer, observe=False): + self.layer = layer + self.dev = self.layer.weight.device + W = layer.weight.data.clone() + if isinstance(self.layer, nn.Conv2d): + W = W.flatten(1) + if isinstance(self.layer, transformers.Conv1D): + W = W.t() + self.rows = W.shape[0] + self.columns = W.shape[1] + self.H = torch.zeros((self.columns, self.columns), device=self.dev) + self.nsamples = 0 + self.quantizer = Quantizer() + self.observe = observe + + def add_batch(self, inp, out): + # Hessian H = 2 X XT + λ I + if self.observe: + self.inp1 = inp + self.out1 = out + else: + self.inp1 = None + self.out1 = None + + if len(inp.shape) == 2: + inp = inp.unsqueeze(0) + tmp = inp.shape[0] + if isinstance(self.layer, nn.Linear) or isinstance( + self.layer, transformers.Conv1D + ): + if len(inp.shape) == 3: + inp = inp.reshape((-1, inp.shape[-1])) + inp = inp.t() + if isinstance(self.layer, nn.Conv2d): + unfold = nn.Unfold( + self.layer.kernel_size, + dilation=self.layer.dilation, + padding=self.layer.padding, + stride=self.layer.stride, + ) + inp = unfold(inp) + inp = inp.permute([1, 0, 2]) + inp = inp.flatten(1) + self.H *= self.nsamples / (self.nsamples + tmp) + self.nsamples += tmp + # inp = inp.float() + inp = math.sqrt(2 / self.nsamples) * inp.float() + # self.H += 2 / self.nsamples * inp.matmul(inp.t()) + self.H += inp.matmul(inp.t()) + + def print_loss(self, name, q_weight, weight_error, timecost): + table = Texttable() + length = 28 + name = ( + (name + " " * (length - len(name))) + if len(name) <= length + else name[:length] + ) + + table.header(["name", "weight_error", "fp_inp_SNR", "q_inp_SNR", "time"]) + + # assign weight + self.layer.weight.data = q_weight.reshape(self.layer.weight.shape).to( + self.layer.weight.data.dtype + ) + + if self.inp1 is not None: + # quantize input to int8 + quantizer = Quantizer() + quantizer.configure(8, perchannel=False, sym=True, mse=False) + quantizer.find_params(self.inp1) + q_in = quantizer.quantize(self.inp1).type(torch.float16) + q_out = self.layer(q_in) + + # get kinds of SNR + q_SNR = torch_snr_error(q_out, self.out1).item() + fp_SNR = torch_snr_error(self.layer(self.inp1), self.out1).item() + else: + q_SNR = "-" + fp_SNR = "-" + + table.add_row([name, weight_error, fp_SNR, q_SNR, timecost]) + print(table.draw().split("\n")[-2]) + + def fasterquant( + self, blocksize=128, percdamp=0.01, groupsize=-1, act_order=False, name="" + ): + self.layer.to(self.dev) + + W = self.layer.weight.data.clone() + if isinstance(self.layer, nn.Conv2d): + W = W.flatten(1) + if isinstance(self.layer, transformers.Conv1D): + W = W.t() + W = W.float() + + tick = time.time() + + if not self.quantizer.ready(): + self.quantizer.find_params(W, weight=True) + + H = self.H + if not self.observe: + del self.H + dead = torch.diag(H) == 0 + H[dead, dead] = 1 + W[:, dead] = 0 + + if act_order: + perm = torch.argsort(torch.diag(H), descending=True) + W = W[:, perm] + H = H[perm][:, perm] + + Losses = torch.zeros_like(W) + Q = torch.zeros_like(W) + + damp = percdamp * torch.mean(torch.diag(H)) + diag = torch.arange(self.columns, device=self.dev) + H[diag, diag] += damp + H = torch.linalg.cholesky(H) + H = torch.cholesky_inverse(H) + try: + H = torch.linalg.cholesky(H, upper=True) + except Exception: + # Addition because Falcon fails on h_to_4h + H = torch.linalg.cholesky( + H + 1e-5 * torch.eye(H.shape[0]).to(H.device), upper=True + ) + Hinv = H + + g_idx = [] + scale = [] + zero = [] + now_idx = 1 + + for i1 in range(0, self.columns, blocksize): + i2 = min(i1 + blocksize, self.columns) + count = i2 - i1 + + W1 = W[:, i1:i2].clone() + Q1 = torch.zeros_like(W1) + Err1 = torch.zeros_like(W1) + Losses1 = torch.zeros_like(W1) + Hinv1 = Hinv[i1:i2, i1:i2] + + for i in range(count): + w = W1[:, i] + d = Hinv1[i, i] + + if groupsize != -1: + if (i1 + i) % groupsize == 0: + self.quantizer.find_params( + W[:, (i1 + i) : (i1 + i + groupsize)], weight=True + ) + + if ((i1 + i) // groupsize) - now_idx == -1: + scale.append(self.quantizer.scale) + zero.append(self.quantizer.zero) + now_idx += 1 + + q = self.quantizer.quantize(w.unsqueeze(1)).flatten() + Q1[:, i] = q + Losses1[:, i] = (w - q) ** 2 / d**2 + + err1 = (w - q) / d + W1[:, i:] -= err1.unsqueeze(1).matmul(Hinv1[i, i:].unsqueeze(0)) + Err1[:, i] = err1 + + Q[:, i1:i2] = Q1 + Losses[:, i1:i2] = Losses1 / 2 + + W[:, i2:] -= Err1.matmul(Hinv[i1:i2, i2:]) + + torch.cuda.synchronize() + error = torch.sum(Losses).item() + + groupsize = groupsize if groupsize != -1 else self.columns + g_idx = [i // groupsize for i in range(self.columns)] + g_idx = torch.tensor(g_idx, dtype=torch.int32, device=Q.device) + if act_order: + invperm = torch.argsort(perm) + Q = Q[:, invperm] + g_idx = g_idx[invperm] + + if isinstance(self.layer, transformers.Conv1D): + Q = Q.t() + + self.print_loss( + name=name, q_weight=Q, weight_error=error, timecost=(time.time() - tick) + ) + + if scale == []: + scale.append(self.quantizer.scale) + zero.append(self.quantizer.zero) + scale = torch.cat(scale, dim=1) + zero = torch.cat(zero, dim=1) + return scale, zero, g_idx, error + + def free(self): + self.inp1 = None + self.out1 = None + self.H = None + self.Losses = None + self.Trace = None + torch.cuda.empty_cache() + + +def get_wikitext2(nsamples, seed, seqlen, model_id): + from datasets import load_dataset + + traindata = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + testdata = load_dataset("wikitext", "wikitext-2-raw-v1", split="test") + + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) + trainenc = tokenizer("\n\n".join(traindata["text"]), return_tensors="pt") + testenc = tokenizer("\n\n".join(testdata["text"]), return_tensors="pt") + + import random + + random.seed(seed) + trainloader = [] + for _ in range(nsamples): + i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1) + j = i + seqlen + inp = trainenc.input_ids[:, i:j] + tar = inp.clone() + tar[:, :-1] = -100 + trainloader.append((inp, tar)) + return trainloader, testenc + + +def get_ptb(nsamples, seed, seqlen, model_id): + from datasets import load_dataset + + traindata = load_dataset("ptb_text_only", "penn_treebank", split="train") + valdata = load_dataset("ptb_text_only", "penn_treebank", split="validation") + + from transformers import AutoTokenizer + + try: + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) + except: + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True) + trainenc = tokenizer("\n\n".join(traindata["sentence"]), return_tensors="pt") + testenc = tokenizer("\n\n".join(valdata["sentence"]), return_tensors="pt") + + import random + + random.seed(seed) + trainloader = [] + for _ in range(nsamples): + i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1) + j = i + seqlen + inp = trainenc.input_ids[:, i:j] + tar = inp.clone() + tar[:, :-1] = -100 + trainloader.append((inp, tar)) + return trainloader, testenc + + +def get_c4(nsamples, seed, seqlen, model_id): + from datasets import load_dataset + + traindata = load_dataset( + "allenai/c4", + "allenai--c4", + data_files={"train": "en/c4-train.00000-of-01024.json.gz"}, + split="train", + use_auth_token=False, + ) + valdata = load_dataset( + "allenai/c4", + "allenai--c4", + data_files={"validation": "en/c4-validation.00000-of-00008.json.gz"}, + split="validation", + use_auth_token=False, + ) + + from transformers import AutoTokenizer + + try: + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) + except: + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True) + + import random + + random.seed(seed) + trainloader = [] + for _ in range(nsamples): + while True: + i = random.randint(0, len(traindata) - 1) + trainenc = tokenizer(traindata[i]["text"], return_tensors="pt") + if trainenc.input_ids.shape[1] >= seqlen: + break + i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1) + j = i + seqlen + inp = trainenc.input_ids[:, i:j] + tar = inp.clone() + tar[:, :-1] = -100 + trainloader.append((inp, tar)) + + import random + + random.seed(0) + valenc = [] + for _ in range(256): + while True: + i = random.randint(0, len(valdata) - 1) + tmp = tokenizer(valdata[i]["text"], return_tensors="pt") + if tmp.input_ids.shape[1] >= seqlen: + break + i = random.randint(0, tmp.input_ids.shape[1] - seqlen - 1) + j = i + seqlen + valenc.append(tmp.input_ids[:, i:j]) + valenc = torch.hstack(valenc) + + class TokenizerWrapper: + def __init__(self, input_ids): + self.input_ids = input_ids + + valenc = TokenizerWrapper(valenc) + + return trainloader, valenc + + +def get_ptb_new(nsamples, seed, seqlen, model_id): + from datasets import load_dataset + + traindata = load_dataset("ptb_text_only", "penn_treebank", split="train") + testdata = load_dataset("ptb_text_only", "penn_treebank", split="test") + + from transformers import AutoTokenizer + + try: + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) + except: + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True) + trainenc = tokenizer(" ".join(traindata["sentence"]), return_tensors="pt") + testenc = tokenizer(" ".join(testdata["sentence"]), return_tensors="pt") + + import random + + random.seed(seed) + trainloader = [] + for _ in range(nsamples): + i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1) + j = i + seqlen + inp = trainenc.input_ids[:, i:j] + tar = inp.clone() + tar[:, :-1] = -100 + trainloader.append((inp, tar)) + return trainloader, testenc + + +def get_c4_new(nsamples, seed, seqlen, model_id): + from datasets import load_dataset + + traindata = load_dataset( + "allenai/c4", + "allenai--c4", + data_files={"train": "en/c4-train.00000-of-01024.json.gz"}, + split="train", + ) + valdata = load_dataset( + "allenai/c4", + "allenai--c4", + data_files={"validation": "en/c4-validation.00000-of-00008.json.gz"}, + split="validation", + ) + + from transformers import AutoTokenizer + + try: + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) + except: + tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True) + + import random + + random.seed(seed) + trainloader = [] + for _ in range(nsamples): + while True: + i = random.randint(0, len(traindata) - 1) + trainenc = tokenizer(traindata[i]["text"], return_tensors="pt") + if trainenc.input_ids.shape[1] >= seqlen: + break + i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1) + j = i + seqlen + inp = trainenc.input_ids[:, i:j] + tar = inp.clone() + tar[:, :-1] = -100 + trainloader.append((inp, tar)) + + valenc = tokenizer(" ".join(valdata[:1100]["text"]), return_tensors="pt") + valenc = valenc.input_ids[:, : (256 * seqlen)] + + class TokenizerWrapper: + def __init__(self, input_ids): + self.input_ids = input_ids + + valenc = TokenizerWrapper(valenc) + + return trainloader, valenc + + +def get_loaders(name, nsamples=128, seed=0, seqlen=2048, model_id=""): + if "wikitext2" in name: + return get_wikitext2(nsamples, seed, seqlen, model_id) + if "ptb" in name: + if "new" in name: + return get_ptb_new(nsamples, seed, seqlen, model_id) + return get_ptb(nsamples, seed, seqlen, model_id) + if "c4" in name: + if "new" in name: + return get_c4_new(nsamples, seed, seqlen, model_id) + return get_c4(nsamples, seed, seqlen, model_id) + + +def find_layers(module, layers=(nn.Conv2d, nn.Linear), name=""): + # Skip last lm_head linear + # Need isintance Falcon is inheriting Linear. + if isinstance(module, layers) and "lm_head" not in name: + return {name: module} + res = {} + for name1, child in module.named_children(): + res.update( + find_layers( + child, layers=layers, name=name + "." + name1 if name != "" else name1 + ) + ) + return res + + +@torch.no_grad() +def sequential( + model, + dataloader, + dev, + nsamples, + bits, + groupsize, + percdamp=0.01, + sym: bool = False, + act_order: bool = False, +): + print("Starting ...") + + use_cache = model.config.use_cache + model.config.use_cache = False + try: + layers = model.model.layers + prefix = "model.layers" + except Exception: + layers = model.transformer.h + prefix = "transformer.h" + + dtype = next(iter(model.parameters())).dtype + inps = torch.zeros( + (nsamples, model.seqlen, model.config.hidden_size), dtype=dtype, device=dev + ) + + cache = {"i": 0} + extra = {} + + class Catcher(nn.Module): + def __init__(self, module): + super().__init__() + self.module = module + + def forward(self, inp, **kwargs): + inps[cache["i"]] = inp + cache["i"] += 1 + extra.update(kwargs.copy()) + raise ValueError + + layers[0] = Catcher(layers[0]) + for batch in dataloader: + try: + model(batch[0]) + except ValueError: + pass + layers[0] = layers[0].module + + # layers[0] = layers[0].cpu() + # model.model.embed_tokens = model.model.embed_tokens.cpu() + # model.model.norm = model.model.norm.cpu() + torch.cuda.empty_cache() + + outs = torch.zeros_like(inps) + + extra = { + k: v.to(dev) if isinstance(v, torch.Tensor) else v for k, v in extra.items() + } + + print("Ready.") + + quantizers = {} + for i in range(len(layers)): + print(f"Quantizing layer {i+1}/{len(layers)}..") + print("+------------------+--------------+------------+-----------+-------+") + print("| name | weight_error | fp_inp_SNR | q_inp_SNR | time |") + print("+==================+==============+============+===========+=======+") + + from accelerate.hooks import remove_hook_from_submodules + + layer = layers[i].to(dev) + remove_hook_from_submodules(layer) + full = find_layers(layer) + sequential = [list(full.keys())] + + for names in sequential: + subset = {n: full[n] for n in names} + gptq = {} + for name in subset: + gptq[name] = GPTQ(subset[name]) + gptq[name].quantizer.configure( + bits, perchannel=True, sym=sym, mse=False + ) + + def add_batch(name): + def tmp(_, inp, out): + gptq[name].add_batch(inp[0].data, out.data) + + return tmp + + handles = [] + for name in subset: + handles.append(subset[name].register_forward_hook(add_batch(name))) + for j in range(nsamples): + + outs[j] = layer(inps[j].unsqueeze(0), **extra)[0] + for h in handles: + h.remove() + + for name in subset: + scale, zero, g_idx, error = gptq[name].fasterquant( + percdamp=percdamp, + groupsize=groupsize, + act_order=act_order, + name=name, + ) + quantizers[f"{prefix}.{i}.{name}"] = ( + gptq[name].quantizer.cpu(), + scale.cpu(), + zero.cpu(), + g_idx.cpu(), + bits, + groupsize, + ) + + gptq[name].free() + + for j in range(nsamples): + outs[j] = layer(inps[j].unsqueeze(0), **extra)[0] + + layers[i] = layer.cpu() + del layer + del gptq + torch.cuda.empty_cache() + + inps, outs = outs, inps + print("+------------------+--------------+------------+-----------+-------+") + print("\n") + + model.config.use_cache = use_cache + + return quantizers + + +def make_quant_linear(module, names, bits, groupsize, name=""): + if isinstance(module, QuantLinear): + return + for attr in dir(module): + tmp = getattr(module, attr) + name1 = name + "." + attr if name != "" else attr + if name1 in names: + delattr(module, attr) + setattr( + module, + attr, + QuantLinear.new( + bits, + groupsize, + tmp.in_features, + tmp.out_features, + tmp.bias is not None, + ), + ) + for name1, child in module.named_children(): + make_quant_linear( + child, names, bits, groupsize, name + "." + name1 if name != "" else name1 + ) + + +# TODO: perform packing on GPU +def pack(model, quantizers, bits, groupsize): + layers = find_layers(model) + layers = {n: layers[n] for n in quantizers} + make_quant_linear(model, quantizers, bits, groupsize) + qlayers = find_layers(model, (QuantLinear,)) + print("Packing ...") + for name in qlayers: + print(name) + quantizers[name], scale, zero, g_idx, _, _ = quantizers[name] + qlayers[name].pack(layers[name], scale, zero, g_idx) + print("Done.") + return model + + +def quantize( + model_id: str, + bits: int, + groupsize: int, + output_dir: str, + trust_remote_code: bool, + upload_to_model_id: Optional[str], + percdamp: float, + act_order: bool, +): + print("loading model") + model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.float16, + device_map="balanced_low_0", + trust_remote_code=trust_remote_code, + ) + print("LOADED model") + model.seqlen = 2048 + + dataset = "wikitext2" + nsamples = 128 + seed = None + + dataloader, testloader = get_loaders( + dataset, nsamples=nsamples, seed=seed, model_id=model_id, seqlen=model.seqlen + ) + + tick = time.time() + quantizers = sequential( + model, + dataloader, + DEV, + nsamples, + bits, + groupsize, + percdamp=percdamp, + act_order=act_order, + ) + print(time.time() - tick) + + pack(model, quantizers, bits, groupsize) + from safetensors.torch import save_file + from transformers.modeling_utils import shard_checkpoint + + state_dict = model.state_dict() + state_dict = {k: v.cpu().contiguous() for k, v in state_dict.items()} + state_dict["gptq_bits"] = torch.LongTensor([bits]) + state_dict["gptq_groupsize"] = torch.LongTensor([groupsize]) + + max_shard_size = "10GB" + shards, index = shard_checkpoint( + state_dict, max_shard_size=max_shard_size, weights_name="model.safetensors" + ) + os.makedirs(output_dir, exist_ok=True) + for shard_file, shard in shards.items(): + save_file( + shard, + os.path.join(output_dir, shard_file), + metadata={ + "format": "pt", + "quantized": "gptq", + "origin": "text-generation-inference", + }, + ) + if index is None: + path_to_weights = os.path.join(output_dir, "model.safetensors") + logger.info(f"Model weights saved in {path_to_weights}") + else: + save_index_file = "model.safetensors.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + logger.info( + f"The model is bigger than the maximum size per checkpoint ({max_shard_size}) and is going to be " + f"split in {len(shards)} checkpoint shards. You can find where each parameters has been saved in the " + f"index located at {save_index_file}." + ) + config = AutoConfig.from_pretrained(model_id, trust_remote_code=trust_remote_code) + config.save_pretrained(output_dir) + logger.info("Saved config") + logger.info("Saving tokenizer") + tokenizer = AutoTokenizer.from_pretrained( + model_id, trust_remote_code=trust_remote_code + ) + tokenizer.save_pretrained(output_dir) + logger.info("Saved tokenizer") + + if upload_to_model_id: + + api = HfApi() + + api.upload_folder( + folder_path=output_dir, repo_id=upload_to_model_id, repo_type="model" + ) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 93865d52..a2b0c739 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -15,6 +15,8 @@ from accelerate import init_empty_weights +from text_generation_server.utils.gptq.quant_linear import QuantLinear + # Monkey patching @classmethod @@ -129,7 +131,22 @@ def get_linear(weight, bias, quantize): if bias is not None: linear.bias = nn.Parameter(bias) elif quantize == "gptq": - raise NotImplementedError("Soon") + try: + qweight, qzeros, scales, g_idx, bits, groupsize = weight + except Exception: + raise NotImplementedError( + f"The passed weight is not `gptq` compatible, loader needs to be updated." + ) + + linear = QuantLinear( + qweight, + qzeros, + scales, + g_idx, + bias, + bits, + groupsize, + ) else: raise NotImplementedError(f"Quantization `{quantize}` is not implemented yet.") return linear @@ -152,8 +169,14 @@ def __init__(self, linear, process_group): @staticmethod def load(config, prefix: str, weights): weight = weights.get_sharded(f"{prefix}.weight", dim=0) + + # GPTQ doesn't quantize heads (nor embeddings) + if config.quantize == "gptq": + quantize = None + else: + quantize = config.quantize return TensorParallelHead( - get_linear(weight, bias=None, quantize=config.quantize), + get_linear(weight, bias=None, quantize=quantize), process_group=weights.process_group, ) @@ -196,24 +219,21 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: class TensorParallelColumnLinear(SuperLayer): @classmethod def load(cls, config, prefix: str, weights, bias: bool): - weight = weights.get_sharded(f"{prefix}.weight", dim=0) - if bias: - bias = weights.get_sharded(f"{prefix}.bias", dim=0) - else: - bias = None - return cls(get_linear(weight, bias, config.quantize)) + return cls.load_multi(config, [prefix], weights, bias, dim=0) @classmethod def load_multi(cls, config, prefixes: List[str], weights, bias: bool, dim: int): - w = [weights.get_sharded(f"{p}.weight", dim=0) for p in prefixes] - weight = torch.cat(w, dim=dim) + weight = weights.get_multi_weights_col( + prefixes, quantize=config.quantize, dim=dim + ) if bias: b = [weights.get_sharded(f"{p}.bias", dim=0) for p in prefixes] - bias = torch.cat(b, dim=0) + bias = torch.cat(b, dim=dim) else: bias = None - return cls(get_linear(weight, bias, config.quantize)) + linear = get_linear(weight, bias, config.quantize) + return cls(linear) class TensorParallelRowLinear(SuperLayer): @@ -223,7 +243,8 @@ def __init__(self, linear, process_group): @classmethod def load(cls, config, prefix: str, weights, bias: bool): - weight = weights.get_sharded(f"{prefix}.weight", dim=1) + weight = weights.get_multi_weights_row(prefix, quantize=config.quantize) + if bias and weights.process_group.rank() == 0: # Rank is only on the first rank process bias = weights.get_tensor(f"{prefix}.bias") diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 88347a6a..9d371834 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -1,6 +1,7 @@ from pathlib import Path from typing import List, Dict, Optional from safetensors import safe_open +import torch class Weights: @@ -54,7 +55,10 @@ def get_tensor(self, tensor_name: str): filename, tensor_name = self.get_filename(tensor_name) f = self._get_handle(filename) tensor = f.get_tensor(tensor_name) - tensor = tensor.to(dtype=self.dtype) + # Special case for gptq which shouldn't convert + # u4 which are disguised as int32 + if tensor.dtype not in [torch.int32, torch.int64]: + tensor = tensor.to(dtype=self.dtype) tensor = tensor.to(device=self.device) return tensor @@ -80,6 +84,49 @@ def get_sharded(self, tensor_name: str, dim: int): tensor = slice_[:, start:stop] else: raise NotImplementedError("Let's make that generic when needed") - tensor = tensor.to(dtype=self.dtype) + # Special case for gptq which shouldn't convert + # u4 which are disguised as int32 + if tensor.dtype != torch.int32: + tensor = tensor.to(dtype=self.dtype) tensor = tensor.to(device=self.device) return tensor + + def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): + if quantize == "gptq": + try: + qweight = torch.cat([self.get_sharded(f"{p}.qweight", dim=1) for p in prefixes], dim=1) + except RuntimeError: + raise RuntimeError("Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`") + + qzeros = torch.cat([self.get_sharded(f"{p}.qzeros", dim=1) for p in prefixes], dim=1) + scales = torch.cat([self.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1) + w = [self.get_tensor(f"{p}.g_idx") for p in prefixes] + for w2 in w[1:]: + torch.testing.assert_close(w2, w[0]) + g_idx = w[0] + + bits = self.get_tensor("gptq_bits").item() + groupsize = self.get_tensor("gptq_groupsize").item() + weight = (qweight, qzeros, scales, g_idx, bits, groupsize) + else: + w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] + weight = torch.cat(w, dim=dim) + return weight + + def get_multi_weights_row(self, prefix: str, quantize: str): + if quantize == "gptq": + try: + qweight = self.get_sharded(f"{prefix}.qweight", dim=0) + except RuntimeError: + raise RuntimeError("Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`") + qzeros = self.get_tensor(f"{prefix}.qzeros") + scales = self.get_tensor(f"{prefix}.scales") + g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) + + bits = self.get_tensor("gptq_bits").item() + groupsize = self.get_tensor("gptq_groupsize").item() + + weight = (qweight, qzeros, scales, g_idx, bits, groupsize) + else: + weight = self.get_sharded(f"{prefix}.weight", dim=1) + return weight From ae466a87361efe8b1af6827fe2eddba584a9482e Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Mon, 26 Jun 2023 03:32:54 -0700 Subject: [PATCH 288/793] fix(server): Do not init process group if already initialized (#388) --- server/text_generation_server/utils/dist.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/server/text_generation_server/utils/dist.py b/server/text_generation_server/utils/dist.py index fe9c3b7b..41a8e01a 100644 --- a/server/text_generation_server/utils/dist.py +++ b/server/text_generation_server/utils/dist.py @@ -2,6 +2,7 @@ import torch from datetime import timedelta +from loguru import logger class FakeBarrier: @@ -59,13 +60,17 @@ def initialize_torch_distributed(): else: if os.getenv("DEBUG", None) == "1": return FakeGroup(rank, world_size), rank, world_size - # Call the init process. - torch.distributed.init_process_group( - backend=backend, - world_size=world_size, - rank=rank, - timeout=timedelta(seconds=60), - pg_options=options, - ) + + if not torch.distributed.is_initialized(): + # Call the init process. + torch.distributed.init_process_group( + backend=backend, + world_size=world_size, + rank=rank, + timeout=timedelta(seconds=60), + pg_options=options, + ) + else: + logger.warning("torch.distributed is already initialized.") return torch.distributed.group.WORLD, rank, world_size From 70f485bf9f601b5450b00894f56e20b973d1c2e4 Mon Sep 17 00:00:00 2001 From: Robert Kimball Date: Wed, 28 Jun 2023 02:50:12 -0700 Subject: [PATCH 289/793] feat(router): add header option to disable buffering for the generate_stream response (#498) # This PR adds an http header option to disable buffering for the generate_stream endpoint response stream. Problem: If a model is run behind a proxy server such as nginx that has buffering enabled then the response stream from generate_stream gets aggregated into a single response which basically disables streaming. Instead of getting a chunked response where each token is presented over time the response presents everything all at once. Solution: This change adds the `X-Accel-Buffering` http header which disables buffering for the generate_stream response, allowing the response to stream properly. --- router/src/server.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/router/src/server.rs b/router/src/server.rs index b8c67b2c..dd8bc874 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -351,6 +351,7 @@ async fn generate_stream( "x-compute-characters", compute_characters.to_string().parse().unwrap(), ); + headers.insert("X-Accel-Buffering", "no".parse().unwrap()); let stream = async_stream::stream! { // Inference From e74bd41e0f279ab569cf6a65ac3e2cea50e80d39 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 30 Jun 2023 19:09:59 +0200 Subject: [PATCH 290/793] feat(server): add paged attention to flash models (#516) Closes #478 --- Dockerfile | 16 +- README.md | 4 +- integration-tests/models/test_flash_neox.py | 2 + launcher/src/main.rs | 44 +- proto/generate.proto | 12 + router/client/src/client.rs | 58 ++ router/client/src/sharded_client.rs | 21 + router/src/infer.rs | 16 +- router/src/main.rs | 42 +- router/src/queue.rs | 47 +- router/src/server.rs | 2 + server/Makefile-vllm | 13 + server/text_generation_server/cache.py | 4 +- .../models/causal_lm.py | 2 +- .../custom_modeling/flash_llama_modeling.py | 216 +++---- .../custom_modeling/flash_neox_modeling.py | 227 +++---- .../custom_modeling/flash_rw_modeling.py | 353 +++++------ .../flash_santacoder_modeling.py | 202 +++---- .../models/custom_modeling/t5_modeling.py | 4 +- .../models/flash_causal_lm.py | 567 +++++++++++------- .../models/flash_llama.py | 6 +- .../models/flash_neox.py | 6 +- .../text_generation_server/models/flash_rw.py | 6 +- .../models/flash_santacoder.py | 13 +- server/text_generation_server/models/model.py | 6 + .../models/seq2seq_lm.py | 2 +- server/text_generation_server/server.py | 7 + server/text_generation_server/utils/tokens.py | 2 + .../text_generation_server/utils/weights.py | 33 +- 29 files changed, 1045 insertions(+), 888 deletions(-) create mode 100644 server/Makefile-vllm diff --git a/Dockerfile b/Dockerfile index 2a313c25..1a969383 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,7 +88,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins RUN /opt/conda/bin/conda install -c "nvidia/label/cuda-11.8.0" cuda==11.8 && \ /opt/conda/bin/conda clean -ya - # Build Flash Attention CUDA kernels FROM kernel-builder as flash-att-builder @@ -109,6 +108,16 @@ COPY server/custom_kernels/ . # Build specific version of transformers RUN python setup.py build +# Build vllm CUDA kernels +FROM kernel-builder as vllm-builder + +WORKDIR /usr/src + +COPY server/Makefile-vllm Makefile + +# Build specific version of vllm +RUN make build-vllm + # Text Generation Inference base image FROM nvidia/cuda:11.8.0-base-ubuntu20.04 as base @@ -137,9 +146,12 @@ COPY --from=flash-att-builder /usr/src/flash-attention/build/lib.linux-x86_64-cp COPY --from=flash-att-builder /usr/src/flash-attention/csrc/layer_norm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages COPY --from=flash-att-builder /usr/src/flash-attention/csrc/rotary/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages -# Copy build artifacts from transformers builder +# Copy build artifacts from custom kernels builder COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39/custom_kernels /usr/src/custom-kernels/src/custom_kernels +# Copy builds artifacts from vllm builder +COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages + # Install flash-attention dependencies RUN pip install einops --no-cache-dir diff --git a/README.md b/README.md index 8c8d9773..b74d2617 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,8 @@ to power LLMs api-inference widgets. - Tensor Parallelism for faster inference on multiple GPUs - Token streaming using Server-Sent Events (SSE) - [Continuous batching of incoming requests](https://github.com/huggingface/text-generation-inference/tree/main/router) for increased total throughput -- Optimized transformers code for inference using [flash-attention](https://github.com/HazyResearch/flash-attention) on the most popular architectures -- Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) +- Optimized transformers code for inference using [flash-attention](https://github.com/HazyResearch/flash-attention) and [Paged Attention](https://github.com/vllm-project/vllm) on the most popular architectures +- Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) and [GPT-Q](https://arxiv.org/abs/2210.17323) - [Safetensors](https://github.com/huggingface/safetensors) weight loading - Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) - Logits warper (temperature scaling, top-p, top-k, repetition penalty, more details see [transformers.LogitsProcessor](https://huggingface.co/docs/transformers/internal/generation_utils#transformers.LogitsProcessor)) diff --git a/integration-tests/models/test_flash_neox.py b/integration-tests/models/test_flash_neox.py index 1076126b..0289c61d 100644 --- a/integration-tests/models/test_flash_neox.py +++ b/integration-tests/models/test_flash_neox.py @@ -13,6 +13,7 @@ async def flash_neox(flash_neox_handle): return flash_neox_handle.client +@pytest.mark.skip @pytest.mark.asyncio async def test_flash_neox(flash_neox, response_snapshot): response = await flash_neox.generate( @@ -25,6 +26,7 @@ async def test_flash_neox(flash_neox, response_snapshot): assert response == response_snapshot +@pytest.mark.skip @pytest.mark.asyncio async def test_flash_neox_load(flash_neox, generate_load, response_snapshot): responses = await generate_load( diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 2e2bc7a5..8497f807 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -115,12 +115,6 @@ struct Args { #[clap(default_value = "1512", long, env)] max_total_tokens: usize, - /// The maximum allowed batch size during dynamic batching. - /// Using `max_batch_total_tokens` should be favored in general - /// as it's a finer way to control RAM usage. - #[clap(long, env)] - max_batch_size: Option, - /// This represents the ratio of waiting queries vs running queries where /// you want to start considering pausing the running queries to include the waiting /// ones into the same batch. @@ -134,6 +128,12 @@ struct Args { #[clap(default_value = "1.2", long, env)] waiting_served_ratio: f32, + /// Limits the number of tokens for the prefill operation. + /// Since this operation take the most memory and is compute bound, it is interesting + /// to limit the number of requests that can be sent. + #[clap(default_value = "4096", long, env)] + max_batch_prefill_tokens: u32, + /// **IMPORTANT** This is one critical control to allow maximum usage /// of the available hardware. /// @@ -146,19 +146,12 @@ struct Args { /// For `max_batch_total_tokens=1000`, you could fit `10` queries of `total_tokens=100` /// or a single query of `1000` tokens. /// - /// So you don't have to control that finely - /// `max_batch_size` or `max_total_tokens`. In fact you could mostly relax them if you - /// want maximum flexibility. However, for your users if they are asking for the full amount of - /// total tokens, they are likely to wait for a very long time to get a spot - /// in the batch (since they are going to be alone) so setting `max_batch_size` - /// and `max_total_tokens` can still be useful to prevent those long waiting times. - /// /// Overall this number should be the largest possible amount that fits the /// remaining memory (after the model is loaded). Since the actual memory overhead /// depends on other parameters like if you're using quantization, flash attention /// or the model implementation, text-generation-inference cannot infer this number /// automatically. - #[clap(default_value = "32000", long, env)] + #[clap(default_value = "16000", long, env)] max_batch_total_tokens: u32, /// This setting defines how many tokens can be passed before forcing the waiting @@ -180,9 +173,9 @@ struct Args { /// for end users. #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, - #[clap(default_value = "3000", long, short, env)] /// The port to listen on. + #[clap(default_value = "3000", long, short, env)] port: u16, /// The name of the socket for gRPC communication between the webserver @@ -329,6 +322,12 @@ fn shard_manager( // Copy current process env let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); + // Use cuda allocator. It leads to less memory fragmentation + env.push(( + "PYTORCH_CUDA_ALLOC_CONF".into(), + "backend:cudaMallocAsync".into(), + )); + // Torch Distributed Env vars env.push(("RANK".into(), rank.to_string().into())); env.push(("WORLD_SIZE".into(), world_size.to_string().into())); @@ -446,7 +445,7 @@ fn shard_manager( // We received a shutdown signal if *shutdown.lock().unwrap() { - p.terminate().unwrap(); + p.kill().unwrap(); let _ = p.wait_timeout(Duration::from_secs(90)); tracing::info!("Shard {rank} terminated"); return; @@ -822,6 +821,10 @@ fn spawn_webserver( args.max_input_length.to_string(), "--max-total-tokens".to_string(), args.max_total_tokens.to_string(), + "--max-batch-prefill-tokens".to_string(), + args.max_batch_prefill_tokens.to_string(), + "--max-batch-total-tokens".to_string(), + args.max_batch_total_tokens.to_string(), "--waiting-served-ratio".to_string(), args.waiting_served_ratio.to_string(), "--max-waiting-tokens".to_string(), @@ -834,15 +837,6 @@ fn spawn_webserver( args.model_id, ]; - // Deprecate max_batch_size - if let Some(max_batch_size) = args.max_batch_size { - argv.push("--max-batch-size".to_string()); - argv.push(max_batch_size.to_string()) - } else { - argv.push("--max-batch-total-tokens".to_string()); - argv.push(args.max_batch_total_tokens.to_string()) - } - // Model optional revision if let Some(ref revision) = args.revision { argv.push("--revision".to_string()); diff --git a/proto/generate.proto b/proto/generate.proto index a0f5a75e..5e061941 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -11,6 +11,8 @@ service TextGenerationService { rpc ClearCache (ClearCacheRequest) returns (ClearCacheResponse); /// Remove requests from a cached batch rpc FilterBatch (FilterBatchRequest) returns (FilterBatchResponse); + /// Warmup the model and compute max cache size + rpc Warmup (WarmupRequest) returns (WarmupResponse); /// Prefill batch and decode first token rpc Prefill (PrefillRequest) returns (PrefillResponse); /// Decode token for a list of prefilled batches @@ -192,3 +194,13 @@ message DecodeResponse { /// Next batch (cached) optional CachedBatch batch = 2; } + +message WarmupRequest { + /// Batch to warmup on + Batch batch = 1; + /// Maximum number of tokens that the client will send + uint32 max_total_tokens = 2; +} + +/// Empty response +message WarmupResponse {} diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 81f023ef..b5e0ccc0 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -3,6 +3,7 @@ use crate::pb::generate::v1::text_generation_service_client::TextGenerationServi use crate::pb::generate::v1::*; use crate::Result; use grpc_metadata::InjectTelemetryContext; +use std::cmp::min; use tonic::transport::{Channel, Uri}; use tracing::instrument; @@ -94,6 +95,63 @@ impl Client { Ok(filtered_batch.batch) } + /// Warmup on a max size batch + /// + /// Returns the maximum amount of tokens supported by the hardware + #[instrument(skip(self))] + pub async fn warmup( + &mut self, + max_input_length: u32, + max_prefill_tokens: u32, + max_total_tokens: u32, + ) -> Result<()> { + let mut n_tokens = 0; + let mut requests = Vec::new(); + + // Create requests + while n_tokens < max_prefill_tokens { + requests.push(Request { + id: 0, + // We truncate the input on the server side to be sure that it has the correct size + inputs: "_test ".to_string().repeat(max_input_length as usize), + truncate: min(max_input_length, max_prefill_tokens - n_tokens), + // Set sampling parameters to also take these ops into account in the max memory + parameters: Some(NextTokenChooserParameters { + temperature: 0.9, + top_k: 10, + top_p: 0.9, + typical_p: 0.9, + do_sample: false, + seed: 0, + repetition_penalty: 1.2, + watermark: true, + }), + stopping_parameters: Some(StoppingCriteriaParameters { + max_new_tokens: 2, + stop_sequences: vec![], + ignore_eos_token: false, + }), + prefill_logprobs: true, + }); + n_tokens += max_input_length; + } + + let batch = Batch { + id: 0, + size: requests.len() as u32, + requests, + max_tokens: 0, + }; + + let request = tonic::Request::new(WarmupRequest { + batch: Some(batch), + max_total_tokens, + }) + .inject_context(); + self.stub.warmup(request).await?.into_inner(); + Ok(()) + } + /// Generate one token for each request in the given batch /// /// Returns Generation for each request in batch diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index b81eed46..9dd173a0 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -87,6 +87,27 @@ impl ShardedClient { join_all(futures).await.pop().unwrap() } + /// Warmup on a max size batch + /// + /// Returns the maximum amount of tokens supported by the hardware + #[instrument(skip(self))] + pub async fn warmup( + &mut self, + max_input_length: u32, + max_prefill_tokens: u32, + max_total_tokens: u32, + ) -> Result<()> { + let futures: Vec<_> = self + .clients + .iter_mut() + .map(|client| { + Box::pin(client.warmup(max_input_length, max_prefill_tokens, max_total_tokens)) + }) + .collect(); + // all shards return the same message + join_all(futures).await.pop().unwrap() + } + /// Generate one token for each request in the given batch /// /// Returns Generation for each request in batch diff --git a/router/src/infer.rs b/router/src/infer.rs index f738f986..d0d22d3b 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -45,6 +45,7 @@ impl Infer { client: ShardedClient, validation: Validation, waiting_served_ratio: f32, + max_batch_prefill_tokens: u32, max_batch_total_tokens: u32, max_waiting_tokens: usize, max_concurrent_requests: usize, @@ -61,6 +62,7 @@ impl Infer { tokio::spawn(batching_task( client, waiting_served_ratio, + max_batch_prefill_tokens, max_batch_total_tokens, max_waiting_tokens, queue.clone(), @@ -240,9 +242,11 @@ impl Infer { /// Will be launched in a background Tokio task /// /// Batches requests and sends them to the inference server +#[allow(clippy::too_many_arguments)] async fn batching_task( mut client: ShardedClient, waiting_served_ratio: f32, + max_batch_prefill_tokens: u32, max_batch_total_tokens: u32, max_waiting_tokens: usize, queue: Queue, @@ -257,8 +261,9 @@ async fn batching_task( // Get the next batch from the queue // This batch might be smaller than the maximum batch size if there are not enough requests // waiting in the queue - while let Some((mut entries, batch, span)) = - queue.next_batch(None, max_batch_total_tokens).await + while let Some((mut entries, batch, span)) = queue + .next_batch(None, max_batch_prefill_tokens, max_batch_total_tokens) + .await { let mut cached_batch = prefill(&mut client, batch, &mut entries, &generation_health) .instrument(span) @@ -284,11 +289,12 @@ async fn batching_task( Some((batch_size as f32 * waiting_served_ratio).floor() as usize) }; - let token_budget = max_batch_total_tokens - batch_max_tokens; + let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens); // Try to get a new batch - if let Some((mut new_entries, new_batch, span)) = - queue.next_batch(min_size, token_budget).await + if let Some((mut new_entries, new_batch, span)) = queue + .next_batch(min_size, max_batch_prefill_tokens, token_budget) + .await { // Tracking metrics if min_size.is_some() { diff --git a/router/src/main.rs b/router/src/main.rs index 7bbb6477..47d48e3f 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -32,10 +32,10 @@ struct Args { max_input_length: usize, #[clap(default_value = "1512", long, env)] max_total_tokens: usize, - #[clap(long, env)] - max_batch_size: Option, #[clap(default_value = "1.2", long, env)] waiting_served_ratio: f32, + #[clap(default_value = "4096", long, env)] + max_batch_prefill_tokens: u32, #[clap(default_value = "32000", long, env)] max_batch_total_tokens: u32, #[clap(default_value = "20", long, env)] @@ -78,9 +78,9 @@ fn main() -> Result<(), std::io::Error> { max_stop_sequences, max_input_length, max_total_tokens, - max_batch_size, waiting_served_ratio, - mut max_batch_total_tokens, + max_batch_prefill_tokens, + max_batch_total_tokens, max_waiting_tokens, port, master_shard_uds_path, @@ -141,12 +141,6 @@ fn main() -> Result<(), std::io::Error> { .block_on(async { init_logging(otlp_endpoint, json_output); - if let Some(max_batch_size) = max_batch_size { - tracing::warn!("`max-batch-size` is deprecated. Use `max-batch-total-tokens` instead"); - max_batch_total_tokens = (max_batch_size * max_total_tokens) as u32; - tracing::warn!("Overriding `max-batch-total-tokens` value with `max-batch-size` * `max-total-tokens` = {max_batch_total_tokens}"); - } - if tokenizer.is_none() { tracing::warn!( "Could not find a fast tokenizer implementation for {tokenizer_name}" @@ -161,10 +155,16 @@ fn main() -> Result<(), std::io::Error> { sha: None, pipeline_tag: None, }, - false => get_model_info(&tokenizer_name, &revision, authorization_token).await.unwrap_or_else(|| { - tracing::warn!("Could not retrieve model info from the Hugging Face hub."); - HubModelInfo { model_id: tokenizer_name.to_string(), sha: None, pipeline_tag: None } - }), + false => get_model_info(&tokenizer_name, &revision, authorization_token) + .await + .unwrap_or_else(|| { + tracing::warn!("Could not retrieve model info from the Hugging Face hub."); + HubModelInfo { + model_id: tokenizer_name.to_string(), + sha: None, + pipeline_tag: None, + } + }), }; // if pipeline-tag == text-generation we default to return_full_text = true @@ -190,6 +190,17 @@ fn main() -> Result<(), std::io::Error> { .info() .await .expect("Unable to get shard info"); + + // Warmup model + tracing::info!("Warming up model"); + sharded_client + .warmup( + max_input_length as u32, + max_batch_prefill_tokens, + max_batch_total_tokens, + ) + .await + .expect("Unable to warmup model"); tracing::info!("Connected"); // Binds on localhost @@ -206,6 +217,7 @@ fn main() -> Result<(), std::io::Error> { max_input_length, max_total_tokens, waiting_served_ratio, + max_batch_prefill_tokens, max_batch_total_tokens, max_waiting_tokens, sharded_client, @@ -219,7 +231,7 @@ fn main() -> Result<(), std::io::Error> { ngrok_username, ngrok_password, ) - .await; + .await; Ok(()) }) } diff --git a/router/src/queue.rs b/router/src/queue.rs index 6d1d4d12..48e483a1 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -58,6 +58,7 @@ impl Queue { pub(crate) async fn next_batch( &self, min_size: Option, + prefill_token_budget: u32, token_budget: u32, ) -> Option { // Create response channel @@ -67,6 +68,7 @@ impl Queue { self.queue_sender .send(QueueCommand::NextBatch { min_size, + prefill_token_budget, token_budget, response_sender, span: Span::current(), @@ -90,11 +92,12 @@ async fn queue_task(requires_padding: bool, receiver: flume::Receiver span.in_scope(|| { - let next_batch = state.next_batch(min_size, token_budget); + let next_batch = state.next_batch(min_size, prefill_token_budget, token_budget); response_sender.send(next_batch).unwrap(); metrics::gauge!("tgi_queue_size", state.entries.len() as f64); }), @@ -140,7 +143,12 @@ impl State { } // Get the next batch - fn next_batch(&mut self, min_size: Option, token_budget: u32) -> Option { + fn next_batch( + &mut self, + min_size: Option, + prefill_token_budget: u32, + token_budget: u32, + ) -> Option { if self.entries.is_empty() { return None; } @@ -184,7 +192,9 @@ impl State { decode_tokens += entry.request.stopping_parameters.max_new_tokens; - if (prefill_tokens + decode_tokens) > token_budget { + if prefill_tokens > prefill_token_budget + || (prefill_tokens + decode_tokens) > token_budget + { // Entry is over budget // Add it back to the front self.entries.push_front((id, entry)); @@ -259,6 +269,7 @@ enum QueueCommand { Append(Box, Span), NextBatch { min_size: Option, + prefill_token_budget: u32, token_budget: u32, response_sender: oneshot::Sender>, span: Span, @@ -328,8 +339,8 @@ mod tests { fn test_next_batch_empty() { let mut state = State::new(false); - assert!(state.next_batch(None, 1).is_none()); - assert!(state.next_batch(Some(1), 1).is_none()); + assert!(state.next_batch(None, 1, 1).is_none()); + assert!(state.next_batch(Some(1), 1, 1).is_none()); } #[test] @@ -340,7 +351,7 @@ mod tests { state.append(entry1); state.append(entry2); - let (entries, batch, _) = state.next_batch(None, 2).unwrap(); + let (entries, batch, _) = state.next_batch(None, 2, 2).unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&0)); assert!(entries.contains_key(&1)); @@ -356,7 +367,7 @@ mod tests { let (entry3, _guard3) = default_entry(); state.append(entry3); - assert!(state.next_batch(Some(2), 2).is_none()); + assert!(state.next_batch(Some(2), 2, 2).is_none()); assert_eq!(state.next_id, 3); assert_eq!(state.entries.len(), 1); @@ -372,7 +383,7 @@ mod tests { state.append(entry1); state.append(entry2); - let (entries, batch, _) = state.next_batch(None, 1).unwrap(); + let (entries, batch, _) = state.next_batch(None, 1, 1).unwrap(); assert_eq!(entries.len(), 1); assert!(entries.contains_key(&0)); assert_eq!(batch.id, 0); @@ -385,7 +396,7 @@ mod tests { let (entry3, _guard3) = default_entry(); state.append(entry3); - let (entries, batch, _) = state.next_batch(None, 3).unwrap(); + let (entries, batch, _) = state.next_batch(None, 3, 3).unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&1)); assert!(entries.contains_key(&2)); @@ -408,8 +419,8 @@ mod tests { async fn test_queue_next_batch_empty() { let queue = Queue::new(false); - assert!(queue.next_batch(None, 1).await.is_none()); - assert!(queue.next_batch(Some(1), 1).await.is_none()); + assert!(queue.next_batch(None, 1, 1).await.is_none()); + assert!(queue.next_batch(Some(1), 1, 1).await.is_none()); } #[tokio::test] @@ -420,7 +431,7 @@ mod tests { queue.append(entry1); queue.append(entry2); - let (entries, batch, _) = queue.next_batch(None, 2).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, 2, 2).await.unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&0)); assert!(entries.contains_key(&1)); @@ -433,11 +444,11 @@ mod tests { queue.append(entry3); // Not enough requests pending - assert!(queue.next_batch(Some(2), 2).await.is_none()); + assert!(queue.next_batch(Some(2), 2, 2).await.is_none()); // Not enough token budget - assert!(queue.next_batch(Some(1), 0).await.is_none()); + assert!(queue.next_batch(Some(1), 0, 0).await.is_none()); // Ok - let (entries2, batch2, _) = queue.next_batch(Some(1), 2).await.unwrap(); + let (entries2, batch2, _) = queue.next_batch(Some(1), 2, 2).await.unwrap(); assert_eq!(entries2.len(), 1); assert!(entries2.contains_key(&2)); assert!(entries2.get(&2).unwrap().batch_time.is_some()); @@ -453,7 +464,7 @@ mod tests { queue.append(entry1); queue.append(entry2); - let (entries, batch, _) = queue.next_batch(None, 1).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, 1, 1).await.unwrap(); assert_eq!(entries.len(), 1); assert!(entries.contains_key(&0)); assert_eq!(batch.id, 0); @@ -462,7 +473,7 @@ mod tests { let (entry3, _guard3) = default_entry(); queue.append(entry3); - let (entries, batch, _) = queue.next_batch(None, 3).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, 3, 3).await.unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&1)); assert!(entries.contains_key(&2)); @@ -476,6 +487,6 @@ mod tests { let (entry, _) = default_entry(); queue.append(entry); - assert!(queue.next_batch(None, 1).await.is_none()); + assert!(queue.next_batch(None, 1, 1).await.is_none()); } } diff --git a/router/src/server.rs b/router/src/server.rs index dd8bc874..ee96ead6 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -514,6 +514,7 @@ pub async fn run( max_input_length: usize, max_total_tokens: usize, waiting_served_ratio: f32, + max_batch_prefill_tokens: u32, max_batch_total_tokens: u32, max_waiting_tokens: usize, client: ShardedClient, @@ -582,6 +583,7 @@ pub async fn run( client, validation, waiting_served_ratio, + max_batch_prefill_tokens, max_batch_total_tokens, max_waiting_tokens, max_concurrent_requests, diff --git a/server/Makefile-vllm b/server/Makefile-vllm new file mode 100644 index 00000000..af750733 --- /dev/null +++ b/server/Makefile-vllm @@ -0,0 +1,13 @@ +vllm_commit := d284b831c17f42a8ea63369a06138325f73c4cf9 + +vllm: + # Clone vllm + git clone https://github.com/OlivierDehaene/vllm.git + +build-vllm: vllm + cd vllm && git fetch && git checkout $(vllm_commit) + cd vllm && python setup.py build + +install-vllm: build-vllm + pip uninstall vllm -y || true + cd vllm && python setup.py install \ No newline at end of file diff --git a/server/text_generation_server/cache.py b/server/text_generation_server/cache.py index 5556529c..79fcd3aa 100644 --- a/server/text_generation_server/cache.py +++ b/server/text_generation_server/cache.py @@ -22,7 +22,9 @@ def delete(self, batch_id: int): del batch def clear(self): - self.cache.clear() + keys = list(self.cache.keys()) + for k in keys: + self.delete(k) def __len__(self): return len(self.cache.keys()) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index ba0853f5..6d47c6eb 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -122,7 +122,7 @@ def from_pb( position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) - max_tokens = len(inputs) * max_input_length + max_decode_tokens + max_tokens = len(inputs) * (max_input_length + max_decode_tokens) return cls( batch_id=pb.id, diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 993e1e2a..07765e88 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -23,12 +23,16 @@ from torch import nn from transformers.activations import ACT2FN -from typing import Optional +from typing import Optional, List, Tuple # Flash attention imports import flash_attn_cuda import dropout_layer_norm +# vllm imports +import vllm_cache_ops +import vllm_attention_ops + from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -106,7 +110,7 @@ def __init__( prefix=f"{prefix}.rotary_emb", weights=weights ) - self.softmax_scale = self.head_size ** (-0.5) + self.softmax_scale = self.head_size**-0.5 self.num_heads = self.num_heads // weights.process_group.size() self.query_key_value = TensorParallelColumnLinear.load_multi( @@ -122,20 +126,22 @@ def __init__( weights=weights, bias=False, ) + self.kv_head_mapping = torch.arange( + 0, self.num_heads, dtype=torch.int32, device=weights.device + ) def forward( self, hidden_states, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): qkv = self.query_key_value(hidden_states) qkv = qkv.view(-1, 3, self.num_heads, self.head_size) @@ -144,23 +150,25 @@ def forward( self.rotary_emb(qkv[:, 0], cos, sin) self.rotary_emb(qkv[:, 1], cos, sin) - # Prefill - if prefill: - # Copy to layer past - layer_past[...] = qkv[:, 1:] + vllm_cache_ops.reshape_and_cache( + qkv[:, 1], qkv[:, 2], kv_cache[0], kv_cache[1], slots + ) - # output - attn_output = torch.empty_like(qkv[:, 0]) + # output tensor + attn_output = torch.empty_like(qkv[:, 0]) + + # Prefill + if start_seq_prefill is not None: # flash attention flash_attn_cuda.fwd( qkv[:, 0], qkv[:, 1], qkv[:, 2], attn_output, - start_seq, - end_seq, - start_seq, - end_seq, + start_seq_prefill, + end_seq_prefill, + start_seq_prefill, + end_seq_prefill, max_s, max_s, 0.0, @@ -173,31 +181,19 @@ def forward( ) # Decode else: - query = qkv[:, 0] - # Add present to the layer_past tensor at the correct indices - layer_past[past_present_indices] = qkv[:, 1:] - - # output - attn_output = torch.empty_like(query) - # flash attention - flash_attn_cuda.fwd( - query, - layer_past[:, 0], - layer_past[:, 1], + # kv_cache[1] => [num_blocks, num_heads, head_size, block_size] + block_size = kv_cache[1].shape[3] + vllm_attention_ops.single_query_cached_kv_attention( attn_output, - start_seq_q, - end_seq_q, - start_seq, - end_seq, - 1, - max_s, - 0.0, + qkv[:, 0], + kv_cache[0], + kv_cache[1], + self.kv_head_mapping, self.softmax_scale, - False, - False, - False, - 0, - None, + block_tables, + input_lengths, + block_size, + max_s, ) return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size)) @@ -265,14 +261,13 @@ def forward( residual, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): normed_hidden_states, res = self.input_layernorm(hidden_states, residual) @@ -281,14 +276,13 @@ def forward( normed_hidden_states, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ) # faster post attention rms norm @@ -333,40 +327,18 @@ def __init__(self, config, weights): def forward( self, - input_ids, - position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, - max_s, - past_present_indices, - past_key_values=None, - pre_allocate_past_size: Optional[int] = None, - ): + input_ids: torch.Tensor, + position_ids: torch.Tensor, + start_seq_prefill: Optional[torch.Tensor], + end_seq_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + ) -> torch.Tensor: hidden_states = self.embed_tokens(input_ids) - # Prefill - if past_key_values is None: - assert pre_allocate_past_size is not None - - prefill = True - - # Create past tensor - # We create a tensor of the same size as input_ids as we don't want to slice at every layer - past_key_values = hidden_states.new_empty( - ( - len(input_ids), - len(self.layers), - 2, - self.num_heads, - self.head_size, - ) - ) - # Decode - else: - prefill = False - # Get rotary cos and sin for this forward # Avoid to index in each layer cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin( @@ -380,34 +352,18 @@ def forward( residual, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache[i], + block_tables, + slots, + input_lengths, max_s, - past_key_values[:, i], - past_present_indices, - prefill, ) - if prefill: - present = past_key_values - # Create padded past tensor - past_key_values = hidden_states.new_empty( - ( - pre_allocate_past_size, - len(self.layers), - 2, - self.num_heads, - self.head_size, - ) - ) - # We slice only once instead of at every layer - past_key_values[past_present_indices] = present - hidden_states, _ = self.norm(hidden_states, residual) - return hidden_states, past_key_values + return hidden_states class FlashLlamaForCausalLM(torch.nn.Module): @@ -423,31 +379,29 @@ def __init__(self, config, weights): def forward( self, - input_ids, - position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, - max_s, - past_present_indices, - past_key_values: Optional[torch.Tensor] = None, - pre_allocate_past_size: Optional[int] = None, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + start_seq_prefill: Optional[torch.Tensor], + end_seq_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, lm_head_indices: Optional[torch.Tensor] = None, - ): - hidden_states, present = self.model( + ) -> torch.Tensor: + hidden_states = self.model( input_ids, position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - past_present_indices, - past_key_values, - pre_allocate_past_size, ) if lm_head_indices is not None: hidden_states = hidden_states[lm_head_indices] logits = self.lm_head(hidden_states) - return logits, present + return logits diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 9c1020a5..9049878a 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -25,11 +25,15 @@ from transformers.activations import ACT2FN from transformers.modeling_utils import PreTrainedModel from transformers.models.gpt_neox import GPTNeoXConfig -from typing import Optional +from typing import Optional, List, Tuple # Flash attention imports import flash_attn_cuda +# vllm imports +import vllm_cache_ops +import vllm_attention_ops + from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -110,20 +114,22 @@ def __init__(self, config, prefix, weights): self.dense = load_row( config, prefix=f"{prefix}.dense", weights=weights, bias=True ) + self.kv_head_mapping = torch.arange( + 0, self.num_heads, dtype=torch.int32, device=weights.device + ) def forward( self, hidden_states, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): qkv = self.query_key_value(hidden_states) qkv = qkv.view(-1, 3, self.num_heads, self.head_size) @@ -132,23 +138,25 @@ def forward( self.rotary_emb(qkv[:, 0], cos, sin) self.rotary_emb(qkv[:, 1], cos, sin) - # Prefill - if prefill: - # Copy to layer past - layer_past[...] = qkv[:, 1:] + vllm_cache_ops.reshape_and_cache( + qkv[:, 1], qkv[:, 2], kv_cache[0], kv_cache[1], slots + ) - # output - attn_output = torch.empty_like(qkv[:, 0]) + # output tensor + attn_output = torch.empty_like(qkv[:, 0]) + + # Prefill + if start_seq_prefill is not None: # flash attention flash_attn_cuda.fwd( qkv[:, 0], qkv[:, 1], qkv[:, 2], attn_output, - start_seq, - end_seq, - start_seq, - end_seq, + start_seq_prefill, + end_seq_prefill, + start_seq_prefill, + end_seq_prefill, max_s, max_s, 0.0, @@ -161,31 +169,19 @@ def forward( ) # Decode else: - query = qkv[:, 0] - # Add present to the layer_past tensor at the correct indices - layer_past[past_present_indices] = qkv[:, 1:] - - # output - attn_output = torch.empty_like(query) - # flash attention - flash_attn_cuda.fwd( - query, - layer_past[:, 0], - layer_past[:, 1], + # kv_cache[1] => [num_blocks, num_heads, head_size, block_size] + block_size = kv_cache[1].shape[3] + vllm_attention_ops.single_query_cached_kv_attention( attn_output, - start_seq_q, - end_seq_q, - start_seq, - end_seq, - 1, - max_s, - 0.0, + qkv[:, 0], + kv_cache[0], + kv_cache[1], + self.kv_head_mapping, self.softmax_scale, - False, - False, - False, - 0, - None, + block_tables, + input_lengths, + block_size, + max_s, ) return self.dense(attn_output.view(-1, self.num_heads * self.head_size)) @@ -250,14 +246,13 @@ def forward( residual, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): if self.use_parallel_residual: ln1_hidden_states, _ = self.input_layernorm(hidden_states) @@ -266,14 +261,13 @@ def forward( ln1_hidden_states, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ) ln2_hidden_states, _ = self.post_attention_layernorm(hidden_states) @@ -292,14 +286,13 @@ def forward( hidden_states, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ) hidden_states, residual = self.post_attention_layernorm( @@ -346,40 +339,18 @@ def __init__(self, config, weights): def forward( self, - input_ids, - position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, - max_s, - past_present_indices, - past_key_values=None, - pre_allocate_past_size: Optional[int] = None, - ): + input_ids: torch.Tensor, + position_ids: torch.Tensor, + start_seq_prefill: Optional[torch.Tensor], + end_seq_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + ) -> torch.Tensor: hidden_states = self.embed_in(input_ids) - # Prefill - if past_key_values is None: - assert pre_allocate_past_size is not None - - prefill = True - - # Create past tensor - # We create a tensor of the same size as input_ids as we don't want to slice at every layer - past_key_values = hidden_states.new_empty( - ( - len(input_ids), - len(self.layers), - 2, - self.num_heads, - self.head_size, - ) - ) - # Decode - else: - prefill = False - # Get rotary cos and sin for this forward # Avoid to index in each layer cos, sin = self.layers[0].attention.rotary_emb.get_cos_sin( @@ -393,34 +364,18 @@ def forward( residual, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache[i], + block_tables, + slots, + input_lengths, max_s, - past_key_values[:, i], - past_present_indices, - prefill, - ) - - if prefill: - present = past_key_values - # Create padded past tensor - past_key_values = hidden_states.new_empty( - ( - pre_allocate_past_size, - len(self.layers), - 2, - self.num_heads, - self.head_size, - ) ) - # We slice only once instead of at every layer - past_key_values[past_present_indices] = present hidden_states, _ = self.final_layer_norm(hidden_states, residual) - return hidden_states, past_key_values + return hidden_states class FlashGPTNeoXForCausalLM(FlashGPTNeoXPreTrainedModel): @@ -434,31 +389,29 @@ def __init__(self, config, weights): def forward( self, - input_ids, - position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, - max_s, - past_present_indices, - past_key_values: Optional[torch.Tensor] = None, - pre_allocate_past_size: Optional[int] = None, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + start_seq_prefill: Optional[torch.Tensor], + end_seq_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, lm_head_indices: Optional[torch.Tensor] = None, - ): - hidden_states, present = self.gpt_neox( + ) -> torch.Tensor: + hidden_states = self.gpt_neox( input_ids, position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - past_present_indices, - past_key_values, - pre_allocate_past_size, ) if lm_head_indices is not None: hidden_states = hidden_states[lm_head_indices] logits = self.embed_out(hidden_states) - return logits, present + return logits diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index fa35c359..44aa7cb1 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -4,11 +4,15 @@ from torch import nn from transformers.modeling_utils import PreTrainedModel from transformers.configuration_utils import PretrainedConfig -from typing import Optional +from typing import Optional, List, Tuple # Flash attention imports import flash_attn_cuda +# vllm imports +import vllm_cache_ops +import vllm_attention_ops + from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -126,19 +130,27 @@ def __init__( config, prefix=f"{prefix}.dense", weights=weights, bias=config.bias ) + if self.num_heads_kv == 1: + self.kv_head_mapping = torch.zeros( + self.num_heads, dtype=torch.int32, device=weights.device + ) + else: + self.kv_head_mapping = torch.arange( + 0, self.num_heads, dtype=torch.int32, device=weights.device + ) + def forward( self, hidden_states, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): qkv = self.query_key_value(hidden_states) @@ -156,25 +168,29 @@ def forward( self.rotary_emb(query, cos, sin) self.rotary_emb(torch.select(kv, dim=1, index=0), cos, sin) + vllm_cache_ops.reshape_and_cache( + kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots + ) + + # output + attn_output = torch.empty_like(query) + # Prefill - if prefill: - # Copy to layer past - layer_past[...] = kv - # Expand to query shape - kv = kv.expand(-1, 2, self.num_heads, self.head_size) + if start_seq_prefill is not None: + if self.num_heads_kv == 1: + # Expand to query shape + kv = kv.expand(-1, 2, self.num_heads, self.head_size) - # output - attn_output = torch.empty_like(query) # flash attention flash_attn_cuda.fwd( query, torch.select(kv, dim=1, index=0), torch.select(kv, dim=1, index=1), attn_output, - start_seq, - end_seq, - start_seq, - end_seq, + start_seq_prefill, + end_seq_prefill, + start_seq_prefill, + end_seq_prefill, max_s, max_s, 0.0, @@ -187,32 +203,19 @@ def forward( ) # Decode else: - # Add present to the layer_past tensor at the correct indices - layer_past[past_present_indices] = kv - # Expand to query shape - kv = layer_past.expand(-1, 2, self.num_heads, self.head_size) - - # output - attn_output = torch.empty_like(query) - # flash attention - flash_attn_cuda.fwd( - query, - torch.select(kv, dim=1, index=0), - torch.select(kv, dim=1, index=1), + # kv_cache[1] => [num_blocks, num_heads_kv, head_size, block_size] + block_size = kv_cache[1].shape[3] + vllm_attention_ops.single_query_cached_kv_attention( attn_output, - start_seq_q, - end_seq_q, - start_seq, - end_seq, - 1, - max_s, - 0.0, + query, + kv_cache[0], + kv_cache[1], + self.kv_head_mapping, self.softmax_scale, - False, - False, - False, - 0, - None, + block_tables, + input_lengths, + block_size, + max_s, ) return self.dense(attn_output.view(-1, self.num_heads * self.head_size)) @@ -264,19 +267,22 @@ def __init__( config, prefix=f"{prefix}.dense", weights=weights, bias=config.bias ) + self.kv_head_mapping = torch.arange( + 0, self.num_groups, dtype=torch.int32, device=weights.device + ).repeat_interleave(self.num_heads) + def forward( self, hidden_states, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): qkv = self.query_key_value(hidden_states) qkv = qkv.view(-1, self.num_groups, self.num_heads + 2, self.head_size) @@ -293,10 +299,19 @@ def forward( self.rotary_emb(query, cos, sin) self.rotary_emb(torch.select(kv, dim=2, index=0), cos, sin) + vllm_cache_ops.reshape_and_cache( + kv[:, :, 0].contiguous(), + kv[:, :, 1].contiguous(), + kv_cache[0], + kv_cache[1], + slots, + ) + + # output + attn_output = torch.empty_like(query) + # Prefill - if prefill: - # Copy to layer past - layer_past[...] = kv + if start_seq_prefill is not None: # Expand to query shape kv = ( kv.unsqueeze(2) @@ -304,18 +319,16 @@ def forward( .reshape(-1, self.num_groups * self.num_heads, 2, self.head_size) ) - # output - attn_output = torch.empty_like(query) # flash attention flash_attn_cuda.fwd( query, torch.select(kv, dim=2, index=0), torch.select(kv, dim=2, index=1), attn_output, - start_seq, - end_seq, - start_seq, - end_seq, + start_seq_prefill, + end_seq_prefill, + start_seq_prefill, + end_seq_prefill, max_s, max_s, 0.0, @@ -328,36 +341,19 @@ def forward( ) # Decode else: - # Add present to the layer_past tensor at the correct indices - layer_past[past_present_indices] = kv - # Expand to query shape - kv = ( - layer_past.unsqueeze(2) - .expand(-1, self.num_groups, self.num_heads, 2, self.head_size) - .reshape(-1, self.num_groups * self.num_heads, 2, self.head_size) - ) - - # output - attn_output = torch.empty_like(query) - # flash attention - flash_attn_cuda.fwd( - query, - torch.select(kv, dim=2, index=0), - torch.select(kv, dim=2, index=1), + # kv_cache[1] => [num_blocks, num_groups, head_size, block_size] + block_size = kv_cache[1].shape[3] + vllm_attention_ops.single_query_cached_kv_attention( attn_output, - start_seq_q, - end_seq_q, - start_seq, - end_seq, - 1, - max_s, - 0.0, + query, + kv_cache[0], + kv_cache[1], + self.kv_head_mapping, self.softmax_scale, - False, - False, - False, - 0, - None, + block_tables, + input_lengths, + block_size, + max_s, ) return self.dense( @@ -432,14 +428,13 @@ def forward( residual, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): if self.parallel_attn: ln_hidden_states, residual = self.input_layernorm(hidden_states, residual) @@ -448,14 +443,13 @@ def forward( ln_hidden_states, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ) mlp_output = self.mlp(ln_hidden_states) @@ -472,14 +466,13 @@ def forward( hidden_states, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ) hidden_states, residual = self.post_attention_layernorm( @@ -523,14 +516,13 @@ def forward( residual, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): ln_attn, residual = self.ln_attn(hidden_states, residual) ln_mlp, _ = self.ln_mlp(residual) @@ -540,14 +532,13 @@ def forward( ln_attn, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ) # MLP. @@ -580,11 +571,7 @@ def __init__(self, config, weights): for layer_id in range(config.num_hidden_layers) ] ) - self.cache_size = ( - 2, - self.h[0].self_attention.num_heads_kv, - self.h[0].self_attention.head_size, - ) + self.cache_size = self.h[0].self_attention.num_heads_kv elif config.model_type == "RefinedWeb": self.h = nn.ModuleList( [ @@ -592,11 +579,7 @@ def __init__(self, config, weights): for layer_id in range(config.num_hidden_layers) ] ) - self.cache_size = ( - self.h[0].self_attention.num_groups, - 2, - self.h[0].self_attention.head_size, - ) + self.cache_size = self.h[0].self_attention.num_groups else: raise NotImplementedError( f"model_type {config.model_type} is not supported." @@ -612,38 +595,18 @@ def __init__(self, config, weights): def forward( self, - input_ids, - position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, - max_s, - past_present_indices, - past_key_values=None, - pre_allocate_past_size: Optional[int] = None, - ): + input_ids: torch.Tensor, + position_ids: torch.Tensor, + start_seq_prefill: Optional[torch.Tensor], + end_seq_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + ) -> torch.Tensor: hidden_states = self.word_embeddings(input_ids) - # Prefill - if past_key_values is None: - assert pre_allocate_past_size is not None - - prefill = True - - # Create past tensor - # We create a tensor of the same size as input_ids as we don't want to slice at every layer - past_key_values = hidden_states.new_empty( - ( - len(input_ids), - len(self.h), - *self.cache_size, - ) - ) - # Decode - else: - prefill = False - # Get rotary cos and sin for this forward # Avoid to index in each layer cos, sin = self.h[0].self_attention.rotary_emb.get_cos_sin( @@ -657,32 +620,18 @@ def forward( residual, cos, sin, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache[i], + block_tables, + slots, + input_lengths, max_s, - torch.select(past_key_values, dim=1, index=i), - past_present_indices, - prefill, - ) - - if prefill: - present = past_key_values - # Create padded past tensor - past_key_values = hidden_states.new_empty( - ( - pre_allocate_past_size, - len(self.h), - *self.cache_size, - ) ) - # We slice only once instead of at every layer - past_key_values[past_present_indices] = present hidden_states, _ = self.ln_f(hidden_states, residual) - return hidden_states, past_key_values + return hidden_states class FlashRWForCausalLM(FlashRWPreTrainedModel): @@ -697,31 +646,29 @@ def __init__(self, config, weights): def forward( self, - input_ids, - position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, - max_s, - past_present_indices, - past_key_values: Optional[torch.Tensor] = None, - pre_allocate_past_size: Optional[int] = None, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + start_seq_prefill: Optional[torch.Tensor], + end_seq_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, lm_head_indices: Optional[torch.Tensor] = None, - ): - hidden_states, present = self.transformer( + ) -> torch.Tensor: + hidden_states = self.transformer( input_ids, position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - past_present_indices, - past_key_values, - pre_allocate_past_size, ) if lm_head_indices is not None: hidden_states = hidden_states[lm_head_indices] logits = self.lm_head(hidden_states) - return logits, present + return logits diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 4eb0034d..04eedef7 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -3,11 +3,15 @@ from torch import nn from transformers.activations import ACT2FN -from typing import Optional +from typing import Optional, List, Tuple # Flash attention imports import flash_attn_cuda +# vllm imports +import vllm_cache_ops +import vllm_attention_ops + from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -221,18 +225,20 @@ def __init__(self, prefix, config, weights): self.c_proj = load_row( config, prefix=f"{prefix}.c_proj", weights=weights, bias=True ) + self.kv_head_mapping = torch.zeros( + self.num_heads, dtype=torch.int32, device=weights.device + ) def forward( self, hidden_states, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): qkv = self.c_attn(hidden_states) @@ -245,25 +251,28 @@ def forward( query = query.view(-1, self.num_heads, self.head_size) key_value = key_value.view(-1, 2, 1, self.head_size) + vllm_cache_ops.reshape_and_cache( + key_value[:, 0], key_value[:, 1], kv_cache[0], kv_cache[1], slots + ) + + # output + attn_output = torch.empty_like(query) + # Prefill - if prefill: - # Copy to layer past - layer_past[...] = key_value + if start_seq_prefill is not None: # Expand from 1 to num_heads key_value = key_value.expand(-1, 2, self.num_heads, self.head_size) - # output - attn_output = torch.empty_like(query) # flash attention flash_attn_cuda.fwd( query, torch.select(key_value, dim=1, index=0), torch.select(key_value, dim=1, index=1), attn_output, - start_seq, - end_seq, - start_seq, - end_seq, + start_seq_prefill, + end_seq_prefill, + start_seq_prefill, + end_seq_prefill, max_s, max_s, 0.0, @@ -276,32 +285,19 @@ def forward( ) # Decode else: - # Add present to the layer_past tensor at the correct indices - layer_past[past_present_indices] = key_value - # Expand from 1 to num_heads - key_value = layer_past.expand(-1, 2, self.num_heads, self.head_size) - - # output - attn_output = torch.empty_like(query) - # flash attention - flash_attn_cuda.fwd( - query, - torch.select(key_value, dim=1, index=0), - torch.select(key_value, dim=1, index=1), + # kv_cache[1] => [num_blocks, 1, head_size, block_size] + block_size = kv_cache[1].shape[3] + vllm_attention_ops.single_query_cached_kv_attention( attn_output, - start_seq_q, - end_seq_q, - start_seq, - end_seq, - 1, - max_s, - 0.0, + query, + kv_cache[0], + kv_cache[1], + self.kv_head_mapping, self.softmax_scale, - False, - False, - False, - 0, - None, + block_tables, + input_lengths, + block_size, + max_s, ) return self.c_proj(attn_output.view(-1, self.num_heads * self.head_size)) @@ -361,27 +357,25 @@ def forward( self, hidden_states, residual, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ): hidden_states, residual = self.ln_1(hidden_states, residual) hidden_states = self.attn( hidden_states, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - layer_past, - past_present_indices, - prefill, ) hidden_states, residual = self.ln_2(hidden_states, residual) @@ -427,64 +421,38 @@ def __init__(self, config, weights): def forward( self, - input_ids, - position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, - max_s, - past_present_indices, - past_key_values=None, - pre_allocate_past_size: Optional[int] = None, - ): + input_ids: torch.Tensor, + position_ids: torch.Tensor, + start_seq_prefill: Optional[torch.Tensor], + end_seq_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + ) -> torch.Tensor: hidden_states = self.wte(input_ids) + self.wpe(position_ids) if self.process_group.size() > 1: torch.distributed.all_reduce(hidden_states, group=self.process_group) - # Prefill - if past_key_values is None: - assert pre_allocate_past_size is not None - - prefill = True - - # Create past tensor - # We create a tensor of the same size as input_ids as we don't want to slice at every layer - past_key_values = hidden_states.new_zeros( - (len(input_ids), len(self.h), 2, 1, self.head_size) - ) - # Decode - else: - prefill = False - residual = None for i, layer in enumerate(self.h): hidden_states, residual = layer( hidden_states, residual, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache[i], + block_tables, + slots, + input_lengths, max_s, - torch.select(past_key_values, dim=1, index=i), - past_present_indices, - prefill, - ) - - if prefill: - present = past_key_values - # Create padded past tensor - past_key_values = hidden_states.new_empty( - (pre_allocate_past_size, len(self.h), 2, 1, self.head_size) ) - # We slice only once instead of at every layer - past_key_values[past_present_indices] = present hidden_states, _ = self.ln_f(hidden_states, residual) - return hidden_states, past_key_values + return hidden_states class FlashSantacoderForCausalLM(nn.Module): @@ -497,31 +465,29 @@ def __init__(self, config, weights): def forward( self, - input_ids, - position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, - max_s, - past_present_indices, - past_key_values: Optional[torch.Tensor] = None, - pre_allocate_past_size: Optional[int] = None, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + start_seq_prefill: Optional[torch.Tensor], + end_seq_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, lm_head_indices: Optional[torch.Tensor] = None, - ): - hidden_states, present = self.transformer( + ) -> torch.Tensor: + hidden_states = self.transformer( input_ids, position_ids, - start_seq, - end_seq, - start_seq_q, - end_seq_q, + start_seq_prefill, + end_seq_prefill, + kv_cache, + block_tables, + slots, + input_lengths, max_s, - past_present_indices, - past_key_values, - pre_allocate_past_size, ) if lm_head_indices is not None: hidden_states = hidden_states[lm_head_indices] logits = self.lm_head(hidden_states) - return logits, present + return logits diff --git a/server/text_generation_server/models/custom_modeling/t5_modeling.py b/server/text_generation_server/models/custom_modeling/t5_modeling.py index 12679e9d..19deca86 100644 --- a/server/text_generation_server/models/custom_modeling/t5_modeling.py +++ b/server/text_generation_server/models/custom_modeling/t5_modeling.py @@ -1004,7 +1004,9 @@ def __init__(self, config: T5Config, weights): try: self.shared = TensorParallelEmbedding(prefix="shared", weights=weights) except RuntimeError: - self.shared = TensorParallelEmbedding(prefix="encoder.embed_tokens", weights=weights) + self.shared = TensorParallelEmbedding( + prefix="encoder.embed_tokens", weights=weights + ) encoder_config = copy.deepcopy(config) encoder_config.is_decoder = False diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index ecea998e..94b14f85 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -1,11 +1,14 @@ +import math +import itertools import torch import torch.distributed import numpy as np from dataclasses import dataclass +from loguru import logger from opentelemetry import trace -from transformers import AutoTokenizer, PreTrainedTokenizerBase, PreTrainedModel +from transformers import PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type, Union, Dict from text_generation_server.models import Model @@ -20,6 +23,92 @@ tracer = trace.get_tracer(__name__) +BLOCK_SIZE = 16 +# Will be set in warmup +CACHE_MANAGER: Optional["CacheManager"] = None + + +class CacheManager: + def __init__( + self, + num_blocks: int, + num_layers: int, + num_heads: int, + head_size: int, + dtype: torch.dtype, + device: torch.device, + ): + self.block_size = BLOCK_SIZE + + element_size = torch.tensor([], dtype=dtype).element_size() + x = self.block_size // element_size + + self.kv_cache = [ + ( + torch.empty( + (num_blocks, num_heads, head_size // x, self.block_size, x), + dtype=dtype, + device=device, + ), + torch.empty( + (num_blocks, num_heads, head_size, self.block_size), + dtype=dtype, + device=device, + ), + ) + for _ in range(num_layers) + ] + self.free_block_mask = torch.ones(num_blocks, dtype=torch.int32, device="cpu") + self.slots = torch.arange( + 0, num_blocks * self.block_size, dtype=torch.int32 + ).view(num_blocks, self.block_size) + + def allocate(self, batch: "FlashCausalLMBatch"): + # Get free blocks indices by finding values in mask that are not set to 0 + free_block_indices = self.free_block_mask.nonzero() + assert ( + len(free_block_indices) >= batch.blocks + ), f"Out of available cache blocks: asked {batch.blocks}, only {len(free_block_indices)} free blocks" + + # Slice by the number of required blocks + block_indices = free_block_indices[: batch.blocks] + block_indices = block_indices.flatten() + + # Padded block tables + block_tables_tensor = torch.zeros( + (len(batch), batch.max_blocks), dtype=torch.int32 + ) + + # Allocate paged attention blocks + cumulative_blocks = 0 + slots = [] + block_tables = [] + for i, (needed_blocks, needed_slots) in enumerate(batch.needed_blocks_slots): + # Get allocated blocks for this sequence + allocated_blocks = block_indices[ + cumulative_blocks : cumulative_blocks + needed_blocks + ] + # Get slots for the allocated blocks + allocated_slots = self.slots[allocated_blocks].flatten()[:needed_slots] + + slots.append(allocated_slots) + block_tables.append(allocated_blocks.tolist()) + block_tables_tensor[i, :needed_blocks] = allocated_blocks + cumulative_blocks += needed_blocks + + batch.needed_blocks_slots = None + batch.block_tables = block_tables + batch.block_tables_tensor = block_tables_tensor.to(batch.input_ids.device) + batch.slots = torch.concat(slots).to(batch.input_ids.device) + + # Allocate the required number of blocks by setting the mask to 0 + self.free_block_mask[block_indices] = 0 + + def free(self, block_indices: Optional[List[int]]): + if block_indices is not None and block_indices: + # Reset mask + self.free_block_mask[block_indices] = 1 + @dataclass class FlashCausalLMBatch(Batch): @@ -32,23 +121,29 @@ class FlashCausalLMBatch(Batch): input_ids: torch.Tensor position_ids: torch.Tensor - # Indices to copy present to the correct indices is the pre-allocated past key values - past_present_indices: torch.Tensor - - # tensor of length b holding starting offset of each sequence - start_seq: torch.Tensor - # tensor of length b holding ending offset of each sequence - end_seq: torch.Tensor # tensor of length b holding starting offset of each sequence, only used in prefill start_seq_prefill: Optional[torch.Tensor] # tensor of length b holding ending offset of each sequence, only used in prefill end_seq_prefill: Optional[torch.Tensor] - # tensor of length b holding starting offset of each query sequence, only used in decode - start_seq_q: Optional[torch.Tensor] - # tensor of length b holding ending offset of each query sequence, only used in decode - end_seq_q: Optional[torch.Tensor] - # past key values, only used in decode - past_key_values: Optional[torch.Tensor] + + # Paged Attention values + + # Set when creating the batch + # CPU tensor of length b indicating the start of each sequence in slots + start_slots: torch.Tensor + # tensor of indices of the currently used slots, length = \sum_{i=0}^{b} s_i in prefill, length = b in decode + slot_indices: torch.Tensor + # List of tuple of ints representing the number of blocks and slots needed by each sequence + needed_blocks_slots: Optional[List[Tuple[int, int]]] + + # Set in prefill by the CacheManager + # list of length b of list of length s_i // block_size + block_tables: Optional[List[List[int]]] + # tensor of size [b, max_seqlen // block_size] holding the paged attention block tables for all sequences + block_tables_tensor: Optional[torch.Tensor] + # tensor of length \sum_{i=0}^{b} max_s_i holding the paged attention slots for all sequences + slots: Optional[torch.Tensor] + max_seqlen: int # Prefill metadata tensors to efficiently compute logprobs @@ -62,6 +157,7 @@ class FlashCausalLMBatch(Batch): # Lengths of all generations present in the batch input_lengths: List[int] + input_lengths_tensor: torch.Tensor prefix_offsets: List[Optional[int]] read_offsets: List[Optional[int]] @@ -69,15 +165,17 @@ class FlashCausalLMBatch(Batch): next_token_chooser: HeterogeneousNextTokenChooser stopping_criterias: List[StoppingCriteria] - # Maximum number of tokens this batch will grow to - max_tokens: int + # Number of blocks in this batch + blocks: int + # Maximum number of blocks + max_blocks: int def to_pb(self) -> generate_pb2.CachedBatch: return generate_pb2.CachedBatch( id=self.batch_id, request_ids=[r.id for r in self.requests], size=len(self), - max_tokens=self.max_tokens, + max_tokens=self.blocks * BLOCK_SIZE, ) @classmethod @@ -99,12 +197,11 @@ def from_pb( )["input_ids"] position_ids = [] - past_present_indices = [] - start_seq = [] - end_seq = [] start_seq_prefill = [] end_seq_prefill = [] - max_seqlen = 0 + needed_blocks_slots = [] + start_slots = [] + slot_indices = [] input_lengths = [] prefix_offsets = [] @@ -126,7 +223,10 @@ def from_pb( cumulative_max_length = 0 prefill_out_cumulative_length = 0 + blocks = 0 + max_seqlen = 0 max_length = 0 + max_blocks = 0 # Parse batch for i, (r, tokenized_input) in enumerate( @@ -138,7 +238,6 @@ def from_pb( tokenized_input = tokenized_input[-r.truncate :] input_length = len(tokenized_input) - max_seqlen = max(max_seqlen, input_length) input_lengths.append(input_length) prefix_offsets.append(input_length - 5) @@ -153,8 +252,6 @@ def from_pb( # Add cumulative lengths of all previous inputs start_seq_prefill.append(cumulative_length) end_seq_prefill.append(cumulative_length + input_length) - start_seq.append(cumulative_max_length) - end_seq.append(cumulative_max_length + input_length) next_token_chooser_parameters.append(r.parameters) @@ -164,6 +261,21 @@ def from_pb( max_new_tokens = stopping_criteria.max_new_tokens stopping_criterias.append(stopping_criteria) + # Paged attention + # Remove one as the first token des not have a past + total_tokens = input_length + max_new_tokens - 1 + needed_blocks = math.ceil(total_tokens / BLOCK_SIZE) + blocks += needed_blocks + needed_blocks_slots.append((needed_blocks, total_tokens)) + start_slots.append(cumulative_max_length) + + request_slot_indices = torch.arange( + cumulative_max_length, + cumulative_max_length + input_length, + dtype=torch.int64, + ) + slot_indices.append(request_slot_indices) + all_prefill_logprobs = all_prefill_logprobs and r.prefill_logprobs no_prefill_logprobs = no_prefill_logprobs and not r.prefill_logprobs @@ -184,22 +296,17 @@ def from_pb( prefill_cu_outlens.append(prefill_out_cumulative_length + 1) prefill_out_cumulative_length += 1 - request_past_present_indices = torch.arange( - cumulative_max_length, - cumulative_max_length + input_length, - dtype=torch.int64, - ) - past_present_indices.append(request_past_present_indices) - # Update - # Remove one as the first token des not have a past cumulative_length += input_length - cumulative_max_length += input_length + max_new_tokens - 1 + cumulative_max_length += total_tokens + max_seqlen = max(max_seqlen, input_length) + max_blocks = max(max_blocks, needed_blocks) max_length = max(max_length, input_length + max_new_tokens) next_token_chooser = HeterogeneousNextTokenChooser.from_pb( next_token_chooser_parameters, dtype, device ) + start_slots = torch.tensor(start_slots, dtype=torch.int64) # Padded all_input_ids_tensor all_input_ids_tensor = np.zeros( @@ -212,34 +319,28 @@ def from_pb( all_input_ids_tensor = torch.tensor( all_input_ids_tensor, dtype=torch.int64, device=device ) - start_seq = torch.tensor(start_seq, device=device, dtype=torch.int32) - end_seq = torch.tensor(end_seq, device=device, dtype=torch.int32) if len(pb.requests) > 1: input_ids = np.concatenate(all_input_ids, dtype=np.int64) position_ids = torch.cat(position_ids) - - past_present_indices = np.concatenate(past_present_indices, dtype=np.int64) - - start_seq_prefill = torch.tensor( - start_seq_prefill, device=device, dtype=torch.int32 - ) - end_seq_prefill = torch.tensor( - end_seq_prefill, device=device, dtype=torch.int32 - ) + slot_indices = torch.cat(slot_indices) else: input_ids = all_input_ids[0] position_ids = position_ids[0] + slot_indices = slot_indices[0] - past_present_indices = past_present_indices[0] - - start_seq_prefill = start_seq - end_seq_prefill = end_seq + start_seq_prefill = torch.tensor( + start_seq_prefill, device=device, dtype=torch.int32 + ) + end_seq_prefill = torch.tensor( + end_seq_prefill, device=device, dtype=torch.int32 + ) + position_ids = position_ids.to(device) + slot_indices = slot_indices.to(device) input_ids = torch.tensor(input_ids, dtype=torch.int64, device=device) - position_ids = torch.tensor(position_ids, dtype=torch.int32, device=device) - past_present_indices = torch.tensor( - past_present_indices, device=device, dtype=torch.int64 + input_lengths_tensor = torch.tensor( + input_lengths, dtype=torch.int32, device=device ) if all_prefill_logprobs: @@ -262,26 +363,28 @@ def from_pb( requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, - past_present_indices=past_present_indices, - start_seq=start_seq, - end_seq=end_seq, start_seq_prefill=start_seq_prefill, end_seq_prefill=end_seq_prefill, - start_seq_q=None, - end_seq_q=None, + start_slots=start_slots, + slot_indices=slot_indices, + needed_blocks_slots=needed_blocks_slots, + block_tables=None, + block_tables_tensor=None, + slots=None, max_seqlen=max_seqlen, prefill_head_indices=prefill_head_indices, prefill_next_token_indices=prefill_next_token_indices, prefill_cu_outlens=prefill_cu_outlens, - past_key_values=None, input_lengths=input_lengths, + input_lengths_tensor=input_lengths_tensor, prefix_offsets=prefix_offsets, read_offsets=read_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=all_input_ids_tensor, next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, - max_tokens=cumulative_max_length, + blocks=blocks, + max_blocks=max_blocks, ) @tracer.start_as_current_span("filter") @@ -294,28 +397,24 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": device = self.input_ids.device - # Cumulative length - cumulative_max_length = 0 - # New values after filtering requests_idx_mapping = {} # Used to index into tensors indices = [] - # past indices to keep - past_indices = torch.zeros( - self.past_key_values.shape[0], dtype=torch.bool, device=device + # slots to keep after filtering + slot_filtering_indices = torch.zeros( + self.slots.shape[0], dtype=torch.bool, device=device ) # Create on CPU to only move to GPU once instead of at every copy - start_seq = torch.empty(len(request_ids), dtype=torch.int32) - end_seq = torch.empty(len(request_ids), dtype=torch.int32) - start_seq_q = self.start_seq_q[: len(request_ids)] - end_seq_q = self.end_seq_q[: len(request_ids)] + slot_indices = torch.empty(len(request_ids), dtype=torch.int64) max_seqlen = 0 requests = [] + start_slots = [] + block_tables = [] all_input_ids = [] input_lengths = [] @@ -324,6 +423,11 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": stopping_criterias = [] + blocks = 0 + max_blocks = 0 + # Cumulative length + cumulative_max_length = 0 + for i, request_id in enumerate(request_ids): idx = self.requests_idx_mapping[request_id] indices.append(idx) @@ -348,28 +452,51 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": stopping_criteria.max_new_tokens - stopping_criteria.current_tokens ) + request_block_table = self.block_tables[idx] + blocks += len(request_block_table) + block_tables.append(request_block_table) + start_slots.append(cumulative_max_length) + # Copy to tensor (CPU) - start_seq[i] = cumulative_max_length - end_seq[i] = cumulative_max_length + request_input_length + slot_indices[i] = cumulative_max_length + request_input_length - 1 # Set slice - past_indices[ - self.start_seq[idx] : self.end_seq[idx] + remaining_tokens - 1 + slot_filtering_indices[ + self.start_slots[idx] : self.start_slots[idx] + + request_input_length + + remaining_tokens + - 1 ] = True cumulative_max_length += request_input_length + remaining_tokens - 1 + max_blocks = max(max_blocks, len(request_block_table)) + + global CACHE_MANAGER + block_indices_to_free = [] + # Iterate on all requests + for i, r in enumerate(self.requests): + # Filter requests that are not part of the new batch + if r.id not in requests_idx_mapping.keys(): + block_indices_to_free.extend(self.block_tables[i]) + # Free blocks + CACHE_MANAGER.free(block_indices_to_free) + # Needed to avoid dropping blocks when the batches will go out of scope + self.block_tables = None + # Index into tensors input_ids = self.input_ids[indices] position_ids = self.position_ids[indices] all_input_ids_tensor = self.all_input_ids_tensor[indices] + block_tables_tensor = self.block_tables_tensor[indices] + input_lengths_tensor = self.input_lengths_tensor[indices] + slots = self.slots[slot_filtering_indices] next_token_chooser = self.next_token_chooser.filter(indices) - past_key_values = self.past_key_values[past_indices] + + start_slots = torch.tensor(start_slots, dtype=torch.int64) # Move to GPU now that we have the whole tensor - start_seq = start_seq.to(device) - end_seq = end_seq.to(device) - past_present_indices = end_seq - 1 + slot_indices = slot_indices.to(device) return FlashCausalLMBatch( batch_id=self.batch_id, @@ -377,26 +504,28 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, - past_present_indices=past_present_indices, - start_seq=start_seq, - end_seq=end_seq, start_seq_prefill=None, end_seq_prefill=None, - start_seq_q=start_seq_q, - end_seq_q=end_seq_q, + start_slots=start_slots, + slot_indices=slot_indices, + needed_blocks_slots=None, + block_tables=block_tables, + block_tables_tensor=block_tables_tensor, + slots=slots, max_seqlen=max_seqlen, prefill_head_indices=None, prefill_next_token_indices=None, prefill_cu_outlens=None, - past_key_values=past_key_values, input_lengths=input_lengths, + input_lengths_tensor=input_lengths_tensor, prefix_offsets=prefix_offsets, read_offsets=read_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=all_input_ids_tensor, next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, - max_tokens=cumulative_max_length, + blocks=blocks, + max_blocks=max_blocks, ) @classmethod @@ -406,22 +535,46 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch requests = [] requests_idx_mapping = {} - total_batch_size = sum([len(b) for b in batches]) - - dtype = batches[0].past_key_values.dtype - device = batches[0].input_ids.device + blocks = 0 + total_batch_size = 0 + total_slots = 0 + max_blocks = 0 + max_length = 0 + max_seqlen = 0 + for b in batches: + total_batch_size += len(b) + total_slots += len(b.slots) + blocks += b.blocks + max_blocks = max(max_blocks, b.max_blocks) + max_seqlen = max(max_seqlen, b.max_seqlen) + max_length = max( + max_length, + max( + input_length + + stopping_criteria.max_new_tokens + - stopping_criteria.current_tokens + for input_length, stopping_criteria in zip( + b.input_lengths, b.stopping_criterias + ) + ), + ) input_ids = batches[0].input_ids.new_empty(total_batch_size) position_ids = batches[0].position_ids.new_empty(total_batch_size) - start_seq = batches[0].start_seq.new_empty(total_batch_size) - end_seq = batches[0].end_seq.new_empty(total_batch_size) - start_seq_q = torch.arange( - 0, total_batch_size, device=device, dtype=torch.int32 + slots = batches[0].slots.new_empty(total_slots) + slot_indices = batches[0].slot_indices.new_empty(total_batch_size) + input_lengths_tensor = batches[0].input_lengths_tensor.new_empty( + total_batch_size + ) + block_tables_tensor = batches[0].block_tables_tensor.new_zeros( + (total_batch_size, max_blocks) + ) + all_input_ids_tensor = batches[0].all_input_ids_tensor.new_zeros( + (total_batch_size, max_length) ) - end_seq_q = start_seq_q + 1 - max_seqlen = 0 - past_key_values = [] + start_slots = [] + block_tables = [] all_input_ids = [] input_lengths = [] @@ -433,8 +586,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch # Cumulative length cumulative_batch_size = 0 - max_tokens = 0 - max_length = 0 + cumulative_slots = 0 for i, batch in enumerate(batches): requests.extend(batch.requests) @@ -448,16 +600,27 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch start_index = cumulative_batch_size end_index = cumulative_batch_size + len(batch) + slots_start_index = cumulative_slots + slots_end_index = cumulative_slots + len(batch.slots) # Copy tensors (GPU) input_ids[start_index:end_index] = batch.input_ids position_ids[start_index:end_index] = batch.position_ids + slot_indices[start_index:end_index] = batch.slot_indices + cumulative_slots + input_lengths_tensor[start_index:end_index] = batch.input_lengths_tensor + slots[slots_start_index:slots_end_index] = batch.slots - start_seq[start_index:end_index] = batch.start_seq + max_tokens - end_seq[start_index:end_index] = batch.end_seq + max_tokens + all_input_ids_tensor[ + start_index:end_index, : batch.all_input_ids_tensor.shape[1] + ] = batch.all_input_ids_tensor[:, :max_length] - max_seqlen = max(max_seqlen, batch.max_seqlen) + block_tables_tensor[ + start_index:end_index, : batch.block_tables_tensor.shape[1] + ] = batch.block_tables_tensor[:, :max_blocks] + start_slots.append(batch.start_slots + cumulative_slots) + + block_tables.extend(batch.block_tables) all_input_ids.extend(batch.all_input_ids) input_lengths.extend(batch.input_lengths) @@ -466,73 +629,59 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch next_token_chooser_parameters.extend([r.parameters for r in batch.requests]) stopping_criterias.extend(batch.stopping_criterias) - past_key_values.append(batch.past_key_values) # Update cumulative_batch_size += len(batch) - max_tokens += batch.max_tokens - max_length = max( - max_length, - max( - input_length - + stopping_criteria.max_new_tokens - - stopping_criteria.current_tokens - for input_length, stopping_criteria in zip( - batch.input_lengths, batch.stopping_criterias - ) - ), - ) - - past_key_values = torch.cat(past_key_values, dim=0) - past_present_indices = end_seq - 1 - - all_input_ids_tensor = torch.zeros( - (total_batch_size, max_length), dtype=torch.int64, device=device - ) + cumulative_slots += len(batch.slots) - cumulative_batch_size = 0 - for i, batch in enumerate(batches): - start_index = cumulative_batch_size - end_index = cumulative_batch_size + len(batch) - - all_input_ids_tensor[ - start_index:end_index, : batch.all_input_ids_tensor.shape[1] - ] = batch.all_input_ids_tensor[:, :max_length] - - cumulative_batch_size += len(batch) + start_slots = torch.concat(start_slots) next_token_chooser = HeterogeneousNextTokenChooser.from_pb( - next_token_chooser_parameters, dtype=dtype, device=device + next_token_chooser_parameters, + dtype=batches[0].next_token_chooser.dtype, + device=batches[0].next_token_chooser.device, ) + # Needed to avoid dropping blocks when the batches will go out of scope + for b in batches: + b.block_tables = None + return FlashCausalLMBatch( batch_id=batches[0].batch_id, requests=requests, requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, - past_present_indices=past_present_indices, - start_seq=start_seq, - end_seq=end_seq, start_seq_prefill=None, end_seq_prefill=None, - start_seq_q=start_seq_q, - end_seq_q=end_seq_q, + start_slots=start_slots, + slot_indices=slot_indices, + needed_blocks_slots=None, + block_tables=block_tables, + block_tables_tensor=block_tables_tensor, + slots=slots, max_seqlen=max_seqlen, prefill_head_indices=None, prefill_next_token_indices=None, prefill_cu_outlens=None, - past_key_values=past_key_values, input_lengths=input_lengths, + input_lengths_tensor=input_lengths_tensor, prefix_offsets=prefix_offsets, read_offsets=read_offsets, all_input_ids=all_input_ids, all_input_ids_tensor=all_input_ids_tensor, next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, - max_tokens=max_tokens, + blocks=blocks, + max_blocks=max_blocks, ) + def __del__(self): + if self.block_tables is not None and self.block_tables: + global CACHE_MANAGER + # Free blocks + CACHE_MANAGER.free(list(itertools.chain.from_iterable(self.block_tables))) + def __len__(self): return len(self.requests) @@ -540,32 +689,19 @@ def __len__(self): class FlashCausalLM(Model): def __init__( self, - model_cls: Type[PreTrainedModel], - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - trust_remote_code: bool = False, + model: torch.nn.Module, + tokenizer: PreTrainedTokenizerBase, + num_layers: int, + num_kv_heads: int, + head_size: int, + dtype: torch.dtype, + device: torch.device, + rank: int = 0, + world_size: int = 1, ): - if torch.cuda.is_available(): - device = torch.device("cuda") - dtype = torch.float16 - else: - raise NotImplementedError("FlashCausalLM is only available on GPU") - - tokenizer = AutoTokenizer.from_pretrained( - model_id, - revision=revision, - padding_side="left", - truncation_side="left", - trust_remote_code=trust_remote_code, - ) - model = model_cls.from_pretrained( - model_id, - revision=revision, - torch_dtype=dtype, - load_in_8bit=quantize == "bitsandbytes", - trust_remote_code=trust_remote_code, - ).to(device) + self.num_layers = num_layers + self.num_kv_heads = num_kv_heads + self.head_size = head_size super(FlashCausalLM, self).__init__( model=model, @@ -573,12 +709,38 @@ def __init__( requires_padding=False, dtype=dtype, device=device, + rank=rank, + world_size=world_size, ) @property def batch_type(self) -> Type[FlashCausalLMBatch]: return FlashCausalLMBatch + def warmup(self, batch: FlashCausalLMBatch, max_total_tokens: int): + global CACHE_MANAGER + + torch.cuda.empty_cache() + try: + CACHE_MANAGER = CacheManager( + # Adds some wiggle room + math.ceil(max_total_tokens / BLOCK_SIZE) + 10, + self.num_layers, + self.num_kv_heads, + self.head_size, + self.dtype, + self.device, + ) + _, batch = self.generate_token(batch) + except Exception as e: + logger.exception( + f"Not enough memory to handle {max_total_tokens} total tokens with {len(batch.input_ids)} " + f"prefill tokens. " + f"You need to decrease `--max-batch-total-tokens` or `--max-batch-prefill-tokens`" + ) + raise e + del batch + def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: return self.tokenizer.decode( generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False @@ -588,28 +750,27 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq: torch.Tensor, - end_seq: torch.Tensor, - start_seq_q: Optional[torch.Tensor], - end_seq_q: Optional[torch.Tensor], + start_seq_prefill: Optional[torch.Tensor], + end_seq_prefill: Optional[torch.Tensor], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, max_s: int, - past_present_indices: torch.Tensor, - past_key_values: Optional = None, - pre_allocate_past_size: Optional[int] = None, lm_head_indices: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: + global CACHE_MANAGER + # Model Forward return self.model.forward( input_ids=input_ids, position_ids=position_ids, - start_seq=start_seq, - end_seq=end_seq, - start_seq_q=start_seq_q, - end_seq_q=end_seq_q, + start_seq_prefill=start_seq_prefill, + end_seq_prefill=end_seq_prefill, + kv_cache=CACHE_MANAGER.kv_cache, + block_tables=block_tables, + slots=slots, + input_lengths=input_lengths, max_s=max_s, - past_present_indices=past_present_indices, - past_key_values=past_key_values, - pre_allocate_past_size=pre_allocate_past_size, lm_head_indices=lm_head_indices, ) @@ -617,31 +778,22 @@ def forward( def generate_token( self, batch: FlashCausalLMBatch ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: - prefill = batch.past_key_values is None + prefill = batch.start_seq_prefill is not None prefill_logprobs = batch.prefill_next_token_indices is not None - if prefill: - # Ask to pre-allocate kv to its max size - # == Sum over batch size (number of tokens + max_new_tokens) - batch size - pre_allocate_past_size = batch.max_tokens - start_seq = batch.start_seq_prefill - end_seq = batch.end_seq_prefill - else: - pre_allocate_past_size = None - start_seq = batch.start_seq - end_seq = batch.end_seq + if batch.needed_blocks_slots: + # Allocate blocks to this batch + CACHE_MANAGER.allocate(batch) - out, present = self.forward( + out = self.forward( batch.input_ids, batch.position_ids, - start_seq, - end_seq, - batch.start_seq_q, - batch.end_seq_q, + batch.start_seq_prefill, + batch.end_seq_prefill, + batch.block_tables_tensor, + batch.slots[batch.slot_indices], + batch.input_lengths_tensor, batch.max_seqlen, - batch.past_present_indices, - batch.past_key_values, - pre_allocate_past_size, batch.prefill_head_indices, ) @@ -662,12 +814,8 @@ def generate_token( # When batch == 1, we will just use the batch.input_ids values directly prefill_tokens_indices = batch.input_ids.new_zeros(len(out)) - # Create batch.start_seq_q and batch.end_seq_q for decode - batch.start_seq_q = torch.arange( - 0, len(batch), device=self.device, dtype=torch.int32 - ) - batch.end_seq_q = batch.start_seq_q + 1 next_position_ids = batch.position_ids.new_empty(len(batch)) + batch.slot_indices = batch.slot_indices[batch.end_seq_prefill - 1] # We do not need start_seq_prefill and end_seq_prefill anymore batch.start_seq_prefill = None batch.end_seq_prefill = None @@ -731,8 +879,8 @@ def generate_token( # Set values in batch batch.input_ids = next_input_ids batch.position_ids = next_position_ids + 1 - batch.past_present_indices = batch.end_seq - batch.end_seq = batch.end_seq + 1 + batch.input_lengths_tensor += 1 + batch.slot_indices += 1 if prefill and prefill_logprobs: # Get prefill logprobs @@ -755,7 +903,6 @@ def generate_token( batch.read_offsets, batch.stopping_criterias, batch.all_input_ids, - batch.all_input_ids_tensor, batch.next_token_chooser.do_sample, batch.next_token_chooser.seeds, next_token_ids, @@ -770,7 +917,6 @@ def generate_token( read_offset, stopping_criteria, all_input_ids, - all_input_ids_tensor, do_sample, seed, next_token_id, @@ -845,19 +991,20 @@ def generate_token( generations.append(generation) - new_input_length = input_length + 1 - # Update values - batch.input_lengths[i] = new_input_length + batch.input_lengths[i] = input_length + 1 batch.prefix_offsets[i] = prefix_offset batch.read_offsets[i] = read_offset batch.all_input_ids[i] = all_input_ids + if stopped: + del batch + # No need to return a batch if we know that all requests stopped + return generations, None + batch.prefill_cu_outlens = None batch.prefill_head_indices = None batch.prefill_next_token_indices = None batch.max_seqlen = batch.max_seqlen + 1 - batch.past_key_values = present - # No need to return a batch if we know that all requests stopped - return generations, batch if not stopped else None + return generations, batch diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index a80d58cb..2c59f01e 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -64,10 +64,12 @@ def __init__( model = FlashLlamaForCausalLM(config, weights) torch.distributed.barrier(group=self.process_group) - super(FlashCausalLM, self).__init__( + super(FlashLlama, self).__init__( model=model, tokenizer=tokenizer, - requires_padding=False, + num_layers=len(model.model.layers), + num_kv_heads=model.model.num_heads, + head_size=model.model.head_size, dtype=dtype, device=device, rank=rank, diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 4847571d..e64af0c6 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -55,10 +55,12 @@ def __init__( model = FlashGPTNeoXForCausalLM(config, weights) torch.distributed.barrier(group=self.process_group) - super(FlashCausalLM, self).__init__( + super(FlashNeoXSharded, self).__init__( model=model.to(device), tokenizer=tokenizer, - requires_padding=False, + num_layers=len(model.gpt_neox.layers), + num_kv_heads=model.gpt_neox.num_heads, + head_size=model.gpt_neox.head_size, dtype=dtype, device=device, rank=rank, diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 5f963bfb..a55f9118 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -55,10 +55,12 @@ def __init__( model = FlashRWForCausalLM(config, weights) torch.distributed.barrier(group=self.process_group) - super(FlashCausalLM, self).__init__( + super(FlashRWSharded, self).__init__( model=model.to(device), tokenizer=tokenizer, - requires_padding=False, + num_layers=len(model.transformer.h), + num_kv_heads=model.transformer.cache_size, + head_size=model.transformer.head_size, dtype=dtype, device=device, rank=rank, diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index a71c0061..ef202785 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -52,17 +52,22 @@ def __init__( torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights( - filenames, device=device, dtype=dtype, process_group=self.process_group, - aliases = {"transformer.wte.weight": ["lm_head.weight"]} + filenames, + device=device, + dtype=dtype, + process_group=self.process_group, + aliases={"transformer.wte.weight": ["lm_head.weight"]}, ) model = FlashSantacoderForCausalLM(config, weights) torch.distributed.barrier(group=self.process_group) - super(FlashCausalLM, self).__init__( + super(FlashSantacoderSharded, self).__init__( model=model.to(device), tokenizer=tokenizer, - requires_padding=False, + num_layers=len(model.transformer.h), + num_kv_heads=1, + head_size=model.transformer.head_size, dtype=dtype, device=device, rank=rank, diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 6b8472a5..f8460fc2 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -22,6 +22,9 @@ def __init__( rank: int = 0, world_size: int = 1, ): + if torch.cuda.is_available(): + torch.cuda.set_per_process_memory_fraction(1.0) + self.model = model.eval() self.tokenizer = tokenizer self.all_special_ids = set(tokenizer.all_special_ids) @@ -55,6 +58,9 @@ def batch_type(self) -> Type[B]: def generate_token(self, batch: B) -> Tuple[List[GeneratedText], Optional[B]]: raise NotImplementedError + def warmup(self, batch: B, max_total_tokens: int): + self.generate_token(batch) + def decode_token( self, all_input_ids: List[int], diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 3ad5698c..999b6637 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -127,7 +127,7 @@ def from_pb( read_offsets.append(1) all_decoder_input_ids = decoder_input_ids.view(-1).split(1) - max_tokens = len(inputs) * max_input_length + max_decode_tokens + max_tokens = len(inputs) * (max_input_length + max_decode_tokens) return cls( batch_id=pb.id, diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index e1bd8412..6cc5beeb 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -53,6 +53,13 @@ async def FilterBatch(self, request, context): return generate_pb2.FilterBatchResponse(batch=filtered_batch.to_pb()) + async def Warmup(self, request, context): + batch = self.model.batch_type.from_pb( + request.batch, self.model.tokenizer, self.model.dtype, self.model.device + ) + self.model.warmup(batch, request.max_total_tokens) + return generate_pb2.WarmupResponse() + async def Prefill(self, request, context): batch = self.model.batch_type.from_pb( request.batch, self.model.tokenizer, self.model.dtype, self.model.device diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index e6e512bc..b83af591 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -216,6 +216,8 @@ def __init__( self.seeds = seeds self.do_sample = do_sample + self.dtype = dtype + self.device = device def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor): if self.watermark_processor is not None: diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 9d371834..83d9df68 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -5,7 +5,14 @@ class Weights: - def __init__(self, filenames: List[Path], device, dtype, process_group, aliases: Optional[Dict[str, List[str]]]=None): + def __init__( + self, + filenames: List[Path], + device, + dtype, + process_group, + aliases: Optional[Dict[str, List[str]]] = None, + ): routing = {} for filename in filenames: with safe_open(filename, framework="pytorch") as f: @@ -43,7 +50,7 @@ def get_filename(self, tensor_name: str) -> (str, str): return str(filename), tensor_name def _get_slice(self, tensor_name: str): - filename, tensor_name= self.get_filename(tensor_name) + filename, tensor_name = self.get_filename(tensor_name) f = self._get_handle(filename) slice_ = f.get_slice(tensor_name) return slice_ @@ -94,12 +101,20 @@ def get_sharded(self, tensor_name: str, dim: int): def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): if quantize == "gptq": try: - qweight = torch.cat([self.get_sharded(f"{p}.qweight", dim=1) for p in prefixes], dim=1) + qweight = torch.cat( + [self.get_sharded(f"{p}.qweight", dim=1) for p in prefixes], dim=1 + ) except RuntimeError: - raise RuntimeError("Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`") - - qzeros = torch.cat([self.get_sharded(f"{p}.qzeros", dim=1) for p in prefixes], dim=1) - scales = torch.cat([self.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1) + raise RuntimeError( + "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" + ) + + qzeros = torch.cat( + [self.get_sharded(f"{p}.qzeros", dim=1) for p in prefixes], dim=1 + ) + scales = torch.cat( + [self.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1 + ) w = [self.get_tensor(f"{p}.g_idx") for p in prefixes] for w2 in w[1:]: torch.testing.assert_close(w2, w[0]) @@ -118,7 +133,9 @@ def get_multi_weights_row(self, prefix: str, quantize: str): try: qweight = self.get_sharded(f"{prefix}.qweight", dim=0) except RuntimeError: - raise RuntimeError("Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`") + raise RuntimeError( + "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" + ) qzeros = self.get_tensor(f"{prefix}.qzeros") scales = self.get_tensor(f"{prefix}.scales") g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) From 3b0c979efcccd8ca51f59f1f982bfbbc842d06c9 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 30 Jun 2023 20:07:49 +0200 Subject: [PATCH 291/793] feat(router): arg validation (#519) --- launcher/src/main.rs | 4 ++-- router/src/main.rs | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 8497f807..9d6cd4dd 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -101,7 +101,7 @@ struct Args { /// for users. The larger this value, the longer prompt users can send which /// can impact the overall memory required to handle the load. /// Please note that some models have a finite range of sequence they can handle. - #[clap(default_value = "1000", long, env)] + #[clap(default_value = "1024", long, env)] max_input_length: usize, /// This is the most important value to set as it defines the "memory budget" @@ -112,7 +112,7 @@ struct Args { /// `1511` max_new_tokens. /// The larger this value, the larger amount each request will be in your RAM /// and the less effective batching can be. - #[clap(default_value = "1512", long, env)] + #[clap(default_value = "2048", long, env)] max_total_tokens: usize, /// This represents the ratio of waiting queries vs running queries where diff --git a/router/src/main.rs b/router/src/main.rs index 47d48e3f..f782be09 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -28,15 +28,15 @@ struct Args { max_best_of: usize, #[clap(default_value = "4", long, env)] max_stop_sequences: usize, - #[clap(default_value = "1000", long, env)] + #[clap(default_value = "1024", long, env)] max_input_length: usize, - #[clap(default_value = "1512", long, env)] + #[clap(default_value = "2048", long, env)] max_total_tokens: usize, #[clap(default_value = "1.2", long, env)] waiting_served_ratio: f32, #[clap(default_value = "4096", long, env)] max_batch_prefill_tokens: u32, - #[clap(default_value = "32000", long, env)] + #[clap(default_value = "16000", long, env)] max_batch_total_tokens: u32, #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, @@ -97,8 +97,18 @@ fn main() -> Result<(), std::io::Error> { ngrok_password, } = args; + // Validate args + if max_input_length as u32 > max_batch_prefill_tokens { + panic!("{}", format!("`max_batch_prefill_tokens` must be >= `max_input_length`. Given: {max_batch_prefill_tokens} and {max_input_length}")); + } + if max_batch_prefill_tokens > max_batch_total_tokens { + panic!("{}", format!("`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {max_batch_prefill_tokens} and {max_batch_total_tokens}")); + } + if max_total_tokens as u32 > max_batch_total_tokens { + panic!("{}", format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_batch_total_tokens}")); + } if validation_workers == 0 { - panic!("validation_workers must be > 0"); + panic!("`validation_workers` must be > 0"); } // CORS allowed origins From ecf6dc3a5a31c1b0e1ed48988ddf2416b5e35660 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 30 Jun 2023 20:30:09 +0200 Subject: [PATCH 292/793] feat: Add the option to force another dtype than `f16`. (#513) --- launcher/src/main.rs | 32 +++++++++++ server/text_generation_server/cli.py | 15 ++++- .../text_generation_server/models/__init__.py | 55 +++++++++++++++++-- server/text_generation_server/models/bloom.py | 3 +- .../models/causal_lm.py | 3 +- .../models/flash_llama.py | 3 +- .../models/flash_neox.py | 3 +- .../text_generation_server/models/flash_rw.py | 3 +- .../models/flash_santacoder.py | 3 +- .../models/galactica.py | 3 +- .../text_generation_server/models/gpt_neox.py | 3 +- server/text_generation_server/models/opt.py | 3 +- server/text_generation_server/models/rw.py | 3 +- .../models/santacoder.py | 3 +- .../models/seq2seq_lm.py | 3 +- server/text_generation_server/models/t5.py | 3 +- server/text_generation_server/server.py | 10 +++- 17 files changed, 130 insertions(+), 21 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 9d6cd4dd..51131f42 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -36,6 +36,26 @@ impl std::fmt::Display for Quantization { } } +#[derive(Clone, Copy, Debug, ValueEnum)] +enum Dtype { + Float16, + BFloat16, +} + +impl std::fmt::Display for Dtype { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // To keep in track with `server`. + match self { + Dtype::Float16 => { + write!(f, "float16") + } + Dtype::BFloat16 => { + write!(f, "bfloat16") + } + } + } +} + /// App Configuration #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] @@ -71,6 +91,10 @@ struct Args { #[clap(long, env, value_enum)] quantize: Option, + /// The dtype to be forced upon the model. This option cannot be used with `--quantize`. + #[clap(long, env, value_enum)] + dtype: Option, + /// Whether you want to execute hub modelling code. Explicitly passing a `revision` is /// encouraged when loading a model with custom code to ensure no malicious code has been /// contributed in a newer revision. @@ -258,6 +282,7 @@ fn shard_manager( model_id: String, revision: Option, quantize: Option, + dtype: Option, trust_remote_code: bool, uds_path: String, rank: usize, @@ -307,6 +332,11 @@ fn shard_manager( shard_argv.push(quantize.to_string()) } + if let Some(dtype) = dtype { + shard_argv.push("--dtype".to_string()); + shard_argv.push(dtype.to_string()) + } + // Model optional revision if let Some(revision) = revision { shard_argv.push("--revision".to_string()); @@ -743,6 +773,7 @@ fn spawn_shards( let shutdown_sender = shutdown_sender.clone(); let otlp_endpoint = args.otlp_endpoint.clone(); let quantize = args.quantize; + let dtype = args.dtype; let trust_remote_code = args.trust_remote_code; let master_port = args.master_port; let disable_custom_kernels = args.disable_custom_kernels; @@ -753,6 +784,7 @@ fn spawn_shards( model_id, revision, quantize, + dtype, trust_remote_code, uds_path, rank, diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index aeb1f13b..3463049a 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -16,12 +16,18 @@ class Quantization(str, Enum): gptq = "gptq" +class Dtype(str, Enum): + float16 = "float16" + bloat16 = "bfloat16" + + @app.command() def serve( model_id: str, revision: Optional[str] = None, sharded: bool = False, quantize: Optional[Quantization] = None, + dtype: Optional[Dtype] = None, trust_remote_code: bool = False, uds_path: Path = "/tmp/text-generation-server", logger_level: str = "INFO", @@ -64,7 +70,14 @@ def serve( # Downgrade enum into str for easier management later on quantize = None if quantize is None else quantize.value - server.serve(model_id, revision, sharded, quantize, trust_remote_code, uds_path) + dtype = None if dtype is None else dtype.value + if dtype is not None and quantize is not None: + raise RuntimeError( + "Only 1 can be set between `dtype` and `quantize`, as they both decide how goes the final model." + ) + server.serve( + model_id, revision, sharded, quantize, dtype, trust_remote_code, uds_path + ) @app.command() diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 2abde685..e45e198a 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -100,11 +100,25 @@ def get_model( revision: Optional[str], sharded: bool, quantize: Optional[str], + dtype: Optional[str], trust_remote_code: bool, ) -> Model: + if dtype is None: + dtype = torch.float16 + elif dtype == "float16": + dtype = torch.float16 + elif dtype == "bfloat16": + dtype = torch.bfloat16 + else: + raise RuntimeError(f"Unknown dtype {dtype}") + if "facebook/galactica" in model_id: return GalacticaSharded( - model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + model_id, + revision, + quantize=quantize, + dtype=dtype, + dtypetrust_remote_code=trust_remote_code, ) if model_id.startswith("bigcode/"): @@ -113,6 +127,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) elif sharded: @@ -124,6 +139,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) @@ -138,6 +154,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) elif sharded: @@ -149,12 +166,17 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) if model_type == "bloom": return BLOOMSharded( - model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, ) elif model_type == "gpt_neox": @@ -163,6 +185,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) elif sharded: @@ -170,6 +193,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) else: @@ -177,6 +201,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) @@ -186,6 +211,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) elif sharded: @@ -195,6 +221,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) @@ -210,6 +237,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) raise NotImplementedError( @@ -221,6 +249,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) else: @@ -228,12 +257,17 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) elif model_type == "opt": return OPTSharded( - model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, ) elif model_type == "t5": @@ -241,6 +275,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) @@ -253,11 +288,19 @@ def get_model( if model_type in modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: return CausalLM( - model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, ) if model_type in modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES: return Seq2SeqLM( - model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, ) auto_map = config_dict.get("auto_map", None) @@ -267,6 +310,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) if "AutoModelForSeq2SeqLM" in auto_map.keys(): @@ -274,6 +318,7 @@ def get_model( model_id, revision, quantize=quantize, + dtype=dtype, trust_remote_code=trust_remote_code, ) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 50b3b76a..101da207 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -42,12 +42,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 6d47c6eb..cbdf4808 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -454,11 +454,12 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: if quantize: raise ValueError("quantization is not available on CPU") diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 2c59f01e..417ccabb 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -25,12 +25,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: raise NotImplementedError("FlashLlama is only available on GPU") diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index e64af0c6..61004d8e 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -24,12 +24,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: raise NotImplementedError("FlashNeoX is only available on GPU") diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index a55f9118..12b862d7 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -25,12 +25,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: raise NotImplementedError("FlashRW is only available on GPU") diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index ef202785..415ec2df 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -24,12 +24,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: raise NotImplementedError("FlashSantacoderSharded is only available on GPU") diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 01e1c773..01e58bad 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -158,12 +158,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 0abf0239..91877fa0 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -24,12 +24,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index 16cb48b7..d407b44a 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -22,12 +22,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/models/rw.py b/server/text_generation_server/models/rw.py index 2b1e4959..92bb135b 100644 --- a/server/text_generation_server/models/rw.py +++ b/server/text_generation_server/models/rw.py @@ -12,11 +12,12 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: if quantize: raise ValueError("quantization is not available on CPU") diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index d0fd3070..a2b38737 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -19,11 +19,12 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: if quantize: raise ValueError("quantization is not available on CPU") diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 999b6637..9e5c21d1 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -504,11 +504,12 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: if quantize: raise ValueError("quantization is not available on CPU") diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index c89462fc..1b7073af 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -25,12 +25,13 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") dtype = torch.float32 diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 6cc5beeb..c375330a 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -106,6 +106,7 @@ def serve( revision: Optional[str], sharded: bool, quantize: Optional[str], + dtype: Optional[str], trust_remote_code: bool, uds_path: Path, ): @@ -114,6 +115,7 @@ async def serve_inner( revision: Optional[str], sharded: bool = False, quantize: Optional[str] = None, + dtype: Optional[str] = None, trust_remote_code: bool = False, ): unix_socket_template = "unix://{}-{}" @@ -128,7 +130,9 @@ async def serve_inner( server_urls = [local_url] try: - model = get_model(model_id, revision, sharded, quantize, trust_remote_code) + model = get_model( + model_id, revision, sharded, quantize, dtype, trust_remote_code + ) except Exception: logger.exception("Error when initializing model") raise @@ -159,4 +163,6 @@ async def serve_inner( logger.info("Signal received. Shutting down") await server.stop(0) - asyncio.run(serve_inner(model_id, revision, sharded, quantize, trust_remote_code)) + asyncio.run( + serve_inner(model_id, revision, sharded, quantize, dtype, trust_remote_code) + ) From 2b53d71991e8fe975be41a82ffe3b52b0bcd40a3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 30 Jun 2023 23:09:20 +0200 Subject: [PATCH 293/793] fix(launcher): fix issue where launcher does not properly report shard failures (#522) --- launcher/src/main.rs | 58 ++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 51131f42..30abe88a 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -6,8 +6,7 @@ use std::io::{BufRead, BufReader, Read}; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::TryRecvError; -use std::sync::Arc; -use std::sync::{mpsc, Mutex}; +use std::sync::{mpsc, Arc}; use std::thread; use std::thread::sleep; use std::time::{Duration, Instant}; @@ -274,7 +273,7 @@ struct Args { #[derive(Debug)] enum ShardStatus { Ready, - Failed((usize, String)), + Failed((usize, Option)), } #[allow(clippy::too_many_arguments)] @@ -296,7 +295,7 @@ fn shard_manager( watermark_delta: Option, otlp_endpoint: Option, status_sender: mpsc::Sender, - shutdown: Arc>, + shutdown: Arc, _shutdown_sender: mpsc::Sender<()>, ) { // Get UDS path @@ -433,20 +432,20 @@ fn shard_manager( } } status_sender - .send(ShardStatus::Failed((rank, err.to_string()))) + .send(ShardStatus::Failed((rank, Some(err.to_string())))) .unwrap(); return; } }; // Redirect STDOUT to the console - let shard_stdout = p.stdout.take().unwrap(); + let shard_stdout_reader = BufReader::new(p.stdout.take().unwrap()); + let mut shard_stderr_reader = BufReader::new(p.stderr.take().unwrap()); thread::spawn(move || { // Enter shard-manager tracing span - let stdout = BufReader::new(shard_stdout); let _span = tracing::span!(tracing::Level::INFO, "shard-manager", rank = rank).entered(); - for line in stdout.lines() { + for line in shard_stdout_reader.lines() { // Parse loguru logs if let Ok(log) = serde_json::from_str::(&line.unwrap()) { log.trace(); @@ -460,8 +459,22 @@ fn shard_manager( loop { // Process exited if let Some(exit_status) = p.poll() { - let mut err = String::new(); - p.stderr.take().unwrap().read_to_string(&mut err).unwrap(); + // We read stderr in another thread as it seems that `read_to_string` can block + // indefinitely in some cases + let (err_sender, err_receiver) = mpsc::channel(); + thread::spawn(move || { + let mut err = String::new(); + shard_stderr_reader.read_to_string(&mut err).unwrap(); + err_sender.send(err).unwrap_or(()); + }); + + let err = err_receiver + .recv_timeout(Duration::from_millis(100)) + .map_err(|err| { + tracing::error!("Unable to read shard {rank} error from stderr"); + err + }) + .ok(); if let ExitStatus::Signaled(signal) = exit_status { tracing::error!("Shard process was signaled to shutdown with signal {signal}"); @@ -474,7 +487,7 @@ fn shard_manager( } // We received a shutdown signal - if *shutdown.lock().unwrap() { + if shutdown.load(Ordering::SeqCst) { p.kill().unwrap(); let _ = p.wait_timeout(Duration::from_secs(90)); tracing::info!("Shard {rank} terminated"); @@ -494,14 +507,11 @@ fn shard_manager( } } -fn shutdown_shards(shutdown: Arc>, shutdown_receiver: &mpsc::Receiver<()>) { +fn shutdown_shards(shutdown: Arc, shutdown_receiver: &mpsc::Receiver<()>) { tracing::info!("Shutting down shards"); // Update shutdown value to true // This will be picked up by the shard manager - { - let mut shutdown = shutdown.lock().unwrap(); - *shutdown = true; - } + shutdown.store(true, Ordering::SeqCst); // Wait for shards to shutdown // This will block till all shutdown_sender are dropped @@ -743,7 +753,7 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L fn spawn_shards( num_shard: usize, args: &Args, - shutdown: Arc>, + shutdown: Arc, shutdown_receiver: &mpsc::Receiver<()>, shutdown_sender: mpsc::Sender<()>, status_receiver: &mpsc::Receiver, @@ -819,7 +829,10 @@ fn spawn_shards( sleep(Duration::from_millis(100)); } Ok(ShardStatus::Failed((rank, err))) => { - tracing::error!("Shard {} failed to start:\n{}", rank, err); + tracing::error!("Shard {rank} failed to start"); + if let Some(err) = err { + tracing::error!("{err}"); + } shutdown_shards(shutdown, shutdown_receiver); return Err(LauncherError::ShardCannotStart); } @@ -835,7 +848,7 @@ fn spawn_shards( fn spawn_webserver( args: Args, - shutdown: Arc>, + shutdown: Arc, shutdown_receiver: &mpsc::Receiver<()>, ) -> Result { // All shard started @@ -1002,7 +1015,7 @@ fn main() -> Result<(), LauncherError> { download_convert_model(&args, running.clone())?; // Shared shutdown bool - let shutdown = Arc::new(Mutex::new(false)); + let shutdown = Arc::new(AtomicBool::new(false)); // Shared shutdown channel // When shutting down, the main thread will wait for all senders to be dropped let (shutdown_sender, shutdown_receiver) = mpsc::channel(); @@ -1034,7 +1047,10 @@ fn main() -> Result<(), LauncherError> { while running.load(Ordering::SeqCst) { if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { - tracing::error!("Shard {rank} failed:\n{err}"); + tracing::error!("Shard {rank} failed to start"); + if let Some(err) = err { + tracing::error!("{err}"); + } exit_code = Err(LauncherError::ShardFailed); break; }; From e28a809004620c3f3a1cc28d4bbc0b4775b1328f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Sat, 1 Jul 2023 19:25:41 +0200 Subject: [PATCH 294/793] v0.9.0 (#525) --- Cargo.lock | 503 ++++++++++++++++++++------------ Cargo.toml | 2 +- Dockerfile | 2 +- README.md | 2 +- aml/README.md | 15 - aml/deployment.yaml | 38 --- aml/endpoint.yaml | 3 - aml/model.yaml | 3 - docs/openapi.json | 31 +- launcher/src/main.rs | 8 +- router/Cargo.toml | 10 +- router/client/Cargo.toml | 4 +- router/grpc-metadata/Cargo.toml | 6 +- router/src/server.rs | 1 + rust-toolchain.toml | 2 +- server/pyproject.toml | 2 +- 16 files changed, 375 insertions(+), 257 deletions(-) delete mode 100644 aml/README.md delete mode 100644 aml/deployment.yaml delete mode 100644 aml/endpoint.yaml delete mode 100644 aml/model.yaml diff --git a/Cargo.lock b/Cargo.lock index 7a6f4ad2..b65045ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -10,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aes" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if", "cipher", @@ -21,11 +30,11 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.6" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "getrandom", + "cfg-if", "once_cell", "version_check", ] @@ -65,15 +74,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] @@ -139,7 +148,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -150,7 +159,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -190,7 +199,7 @@ checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" dependencies = [ "async-trait", "axum-core", - "bitflags", + "bitflags 1.3.2", "bytes", "futures-util", "http", @@ -240,11 +249,26 @@ dependencies = [ "axum", "futures", "http", - "opentelemetry", + "opentelemetry 0.18.0", "tower", "tower-http 0.3.5", "tracing", - "tracing-opentelemetry", + "tracing-opentelemetry 0.18.0", +] + +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", ] [[package]] @@ -271,6 +295,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "block-buffer" version = "0.10.4" @@ -380,9 +410,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.4" +version = "4.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80672091db20273a15cf9fdd4e47ed43b5091ec9841bf4c6145c9dfbbcae09ed" +checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a" dependencies = [ "clap_builder", "clap_derive", @@ -391,13 +421,12 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.4" +version = "4.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1458a1df40e1e2afebb7ab60ce55c1fa8f431146205aa5f4887e0b111c27636" +checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d" dependencies = [ "anstream", "anstyle", - "bitflags", "clap_lex", "strsim", ] @@ -411,7 +440,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -528,7 +557,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" dependencies = [ - "bitflags", + "bitflags 1.3.2", "crossterm_winapi", "libc", "mio", @@ -609,7 +638,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.12.3", "lock_api", "once_cell", "parking_lot_core", @@ -895,7 +924,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -956,10 +985,16 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "glob" version = "0.3.1" @@ -970,17 +1005,17 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" name = "grpc-metadata" version = "0.1.0" dependencies = [ - "opentelemetry", - "tonic", + "opentelemetry 0.19.0", + "tonic 0.9.2", "tracing", - "tracing-opentelemetry", + "tracing-opentelemetry 0.19.0", ] [[package]] name = "h2" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes", "fnv", @@ -1000,6 +1035,12 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ "ahash", ] @@ -1010,15 +1051,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.1" @@ -1087,9 +1119,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -1157,7 +1189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", "serde", ] @@ -1209,26 +1241,25 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi", "libc", "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", + "hermit-abi", + "rustix 0.38.1", "windows-sys 0.48.0", ] @@ -1291,9 +1322,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libm" @@ -1307,6 +1338,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + [[package]] name = "lock_api" version = "0.4.10" @@ -1324,10 +1361,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] -name = "mach" -version = "0.3.2" +name = "mach2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" dependencies = [ "libc", ] @@ -1386,28 +1423,27 @@ dependencies = [ [[package]] name = "metrics" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" +checksum = "aa8ebbd1a9e57bbab77b9facae7f5136aea44c356943bf9a198f647da64285d6" dependencies = [ "ahash", "metrics-macros", - "portable-atomic 0.3.20", + "portable-atomic", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70" +checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" dependencies = [ + "base64 0.21.2", "hyper", "indexmap", "ipnet", "metrics", "metrics-util", - "parking_lot", - "portable-atomic 0.3.20", "quanta", "thiserror", "tokio", @@ -1416,28 +1452,26 @@ dependencies = [ [[package]] name = "metrics-macros" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" +checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.22", ] [[package]] name = "metrics-util" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d24dc2dbae22bff6f1f9326ffce828c9f07ef9cc1e8002e5279f845432a30a" +checksum = "111cb375987443c3de8d503580b536f77dc8416d32db62d9456db5d93bd7ac47" dependencies = [ "crossbeam-epoch", "crossbeam-utils", - "hashbrown", + "hashbrown 0.13.2", "metrics", "num_cpus", - "parking_lot", - "portable-atomic 0.3.20", "quanta", "sketches-ddsketch", ] @@ -1481,7 +1515,7 @@ checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.48.0", ] @@ -1503,7 +1537,7 @@ checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1520,7 +1554,7 @@ checksum = "e92b89ac3127251efde6f5a9586e5aae99468d06fcf9f133b377f58d5ed66446" dependencies = [ "async-trait", "awaitdrop", - "bitflags", + "bitflags 1.3.2", "bytes", "futures", "pin-project", @@ -1560,9 +1594,9 @@ dependencies = [ [[package]] name = "ngrok" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98ce3514eec7338e2d4663e3efb4429e08d8f3662996be4b9585350e7d8ad728" +checksum = "87e211f407b0a084f720823a00c956aeab2c15dfe7a61760d93227bbaf048026" dependencies = [ "arc-swap", "async-rustls", @@ -1595,7 +1629,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "static_assertions", @@ -1648,11 +1682,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] @@ -1668,6 +1702,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -1680,7 +1723,7 @@ version = "6.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "once_cell", "onig_sys", @@ -1698,11 +1741,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.54" +version = "0.10.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b3f656a17a6cbc115b5c7a40c616947d213ba182135b014d6051b73ab6f019" +checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -1719,7 +1762,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1730,9 +1773,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.88" +version = "0.9.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ce0f250f34a308dcfdbb351f511359857d4ed2134ba715a4eadd46e1ffd617" +checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" dependencies = [ "cc", "libc", @@ -1746,40 +1789,49 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" dependencies = [ - "opentelemetry_api", - "opentelemetry_sdk", + "opentelemetry_api 0.18.0", + "opentelemetry_sdk 0.18.0", +] + +[[package]] +name = "opentelemetry" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4b8347cc26099d3aeee044065ecc3ae11469796b4d65d065a23a584ed92a6f" +dependencies = [ + "opentelemetry_api 0.19.0", + "opentelemetry_sdk 0.19.0", ] [[package]] name = "opentelemetry-otlp" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1c928609d087790fc936a1067bdc310ae702bdf3b090c3f281b713622c8bbde" +checksum = "8af72d59a4484654ea8eb183fea5ae4eb6a41d7ac3e3bae5f4d2a282a3a7d3ca" dependencies = [ "async-trait", "futures", "futures-util", "http", - "opentelemetry", + "opentelemetry 0.19.0", "opentelemetry-proto", "prost", "thiserror", "tokio", - "tonic", + "tonic 0.8.3", ] [[package]] name = "opentelemetry-proto" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61a2f56df5574508dd86aaca016c917489e589ece4141df1b5e349af8d66c28" +checksum = "045f8eea8c0fa19f7d48e7bc3128a39c2e5c533d5c61298c548dfefc1064474c" dependencies = [ "futures", "futures-util", - "opentelemetry", + "opentelemetry 0.19.0", "prost", - "tonic", - "tonic-build", + "tonic 0.8.3", ] [[package]] @@ -1798,6 +1850,22 @@ dependencies = [ "thiserror", ] +[[package]] +name = "opentelemetry_api" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed41783a5bf567688eb38372f2b7a8530f5a607a4b49d38dd7573236c23ca7e2" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + [[package]] name = "opentelemetry_sdk" version = "0.18.0" @@ -1812,7 +1880,29 @@ dependencies = [ "futures-executor", "futures-util", "once_cell", - "opentelemetry_api", + "opentelemetry_api 0.18.0", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b3a2a91fdbfdd4d212c0dcc2ab540de2c2bcbbd90be17de7a7daf8822d010c1" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api 0.19.0", "percent-encoding", "rand", "thiserror", @@ -1857,7 +1947,7 @@ dependencies = [ "libc", "redox_syscall 0.3.5", "smallvec", - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -1907,22 +1997,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +checksum = "6e138fdd8263907a2b0e1b4e80b7e58c721126479b6e6eedfb1b402acea7b9bd" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +checksum = "d1fef411b303e3e12d534fb6e7852de82da56edd937d895125821fb7c09436c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1943,15 +2033,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "portable-atomic" -version = "0.3.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e" -dependencies = [ - "portable-atomic 1.3.3", -] - [[package]] name = "portable-atomic" version = "1.3.3" @@ -2000,9 +2081,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" dependencies = [ "unicode-ident", ] @@ -2063,25 +2144,25 @@ dependencies = [ [[package]] name = "quanta" -version = "0.10.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e31331286705f455e56cca62e0e717158474ff02b7936c1fa596d983f4ae27" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" dependencies = [ "crossbeam-utils", "libc", - "mach", + "mach2", "once_cell", "raw-cpuid", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi", "web-sys", "winapi", ] [[package]] name = "quote" -version = "1.0.28" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" dependencies = [ "proc-macro2", ] @@ -2122,7 +2203,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcc0d032bccba900ee32151ec0265667535c230169f5a011154cdcd984e16829" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cassowary", "crossterm", "unicode-segmentation", @@ -2135,7 +2216,7 @@ version = "10.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2177,7 +2258,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2186,7 +2267,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2286,9 +2367,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "6.7.0" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73e721f488c353141288f223b599b4ae9303ecf3e62923f40a492f0634a4dc3" +checksum = "a36224c3276f8c4ebc8c20f158eca7ca4359c8db89991c4925132aaaf6702661" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -2297,28 +2378,34 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.6.0" +version = "6.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22ce362f5561923889196595504317a4372b84210e6e335da529a65ea5452b5" +checksum = "49b94b81e5b2c284684141a2fb9e2a31be90638caf040bf9afbc5a0416afe1ac" dependencies = [ "proc-macro2", "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.18", + "syn 2.0.22", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "7.5.0" +version = "7.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512b0ab6853f7e14e3c8754acb43d6f748bb9ced66aa5915a6553ac8213f7731" +checksum = "9d38ff6bf570dc3bb7100fce9f7b60c33fa71d80e88da3f2580df4ff2bdded74" dependencies = [ "sha2", "walkdir", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2330,15 +2417,28 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.20" +version = "0.37.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "62f25693a73057a1b4cb56179dd3c7ea21a7c6c5ee7d85781f5749b46f34b79c" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc6396159432b5c8490d4e301d8c705f61860b8b6c863bf79942ce5401968f3" +dependencies = [ + "bitflags 2.3.3", + "errno", + "libc", + "linux-raw-sys 0.4.3", "windows-sys 0.48.0", ] @@ -2356,9 +2456,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ "base64 0.21.2", ] @@ -2415,7 +2515,7 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -2455,14 +2555,14 @@ checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] name = "serde_json" -version = "1.0.97" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" dependencies = [ "itoa", "ryu", @@ -2668,9 +2768,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" dependencies = [ "proc-macro2", "quote", @@ -2685,9 +2785,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.29.2" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9557d0845b86eea8182f7b10dff120214fb6cd9fd937b6f4917714e546a38695" +checksum = "5bcd0346f90b6bc83526c7b180039a8acd26a5c848cc556d457f6472eb148122" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2742,13 +2842,13 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix", + "rustix 0.37.21", "windows-sys 0.48.0", ] [[package]] name = "text-generation-benchmark" -version = "0.8.2" +version = "0.9.0" dependencies = [ "average", "clap", @@ -2768,7 +2868,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.8.2" +version = "0.9.0" dependencies = [ "futures", "grpc-metadata", @@ -2776,7 +2876,7 @@ dependencies = [ "prost-build", "thiserror", "tokio", - "tonic", + "tonic 0.9.2", "tonic-build", "tower", "tracing", @@ -2784,7 +2884,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.8.2" +version = "0.9.0" dependencies = [ "clap", "ctrlc", @@ -2800,7 +2900,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.8.2" +version = "0.9.0" dependencies = [ "async-stream", "axum", @@ -2812,7 +2912,7 @@ dependencies = [ "metrics-exporter-prometheus", "ngrok", "nohash-hasher", - "opentelemetry", + "opentelemetry 0.19.0", "opentelemetry-otlp", "rand", "reqwest", @@ -2822,9 +2922,9 @@ dependencies = [ "thiserror", "tokenizers", "tokio", - "tower-http 0.4.0", + "tower-http 0.4.1", "tracing", - "tracing-opentelemetry", + "tracing-opentelemetry 0.19.0", "tracing-subscriber", "utoipa", "utoipa-swagger-ui", @@ -2848,7 +2948,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2941,11 +3041,12 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.2" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", + "backtrace", "bytes", "libc", "mio", @@ -2976,7 +3077,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -3058,11 +3159,39 @@ dependencies = [ "tracing-futures", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64 0.21.2", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic-build" -version = "0.8.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" +checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" dependencies = [ "prettyplease", "proc-macro2", @@ -3097,7 +3226,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bytes", "futures-core", "futures-util", @@ -3112,11 +3241,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" +checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c" dependencies = [ - "bitflags", + "bitflags 2.3.3", "bytes", "futures-core", "futures-util", @@ -3155,13 +3284,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -3202,7 +3331,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" dependencies = [ "once_cell", - "opentelemetry", + "opentelemetry 0.18.0", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00a39dcf9bfc1742fa4d6215253b33a6e474be78275884c216fc2a06267b3600" +dependencies = [ + "once_cell", + "opentelemetry 0.19.0", "tracing", "tracing-core", "tracing-log", @@ -3326,6 +3469,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" + [[package]] name = "utf8parse" version = "0.2.1" @@ -3353,7 +3502,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -3422,12 +3571,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3455,7 +3598,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", "wasm-bindgen-shared", ] @@ -3489,7 +3632,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3592,7 +3735,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -3612,9 +3755,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ "windows_aarch64_gnullvm 0.48.0", "windows_aarch64_msvc 0.48.0", diff --git a/Cargo.toml b/Cargo.toml index b28286fa..ba7a920d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.8.2" +version = "0.9.0" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/Dockerfile b/Dockerfile index 1a969383..66e0091d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Rust builder -FROM lukemathwalker/cargo-chef:latest-rust-1.69 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.70 AS chef WORKDIR /usr/src ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse diff --git a/README.md b/README.md index b74d2617..d31c176b 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ model=bigscience/bloom-560m num_shard=2 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.8 --model-id $model --num-shard $num_shard +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.9 --model-id $model --num-shard $num_shard ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. diff --git a/aml/README.md b/aml/README.md deleted file mode 100644 index 8e78b0ab..00000000 --- a/aml/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Azure ML endpoint - -## Create all resources - -```shell -az ml model create -f model.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace -az ml online-endpoint create -f endpoint.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace -az ml online-deployment create -f deployment.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace -``` - -## Update deployment - -```shell -az ml online-deployment update -f deployment.yaml -g HuggingFace-BLOOM-ModelPage -w HuggingFace -``` \ No newline at end of file diff --git a/aml/deployment.yaml b/aml/deployment.yaml deleted file mode 100644 index 320eba24..00000000 --- a/aml/deployment.yaml +++ /dev/null @@ -1,38 +0,0 @@ -$schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json -name: bloom-deployment -endpoint_name: bloom-inference -model: azureml:bloom-safetensors:1 -model_mount_path: /var/azureml-model -environment_variables: - WEIGHTS_CACHE_OVERRIDE: /var/azureml-model/bloom-safetensors - MODEL_ID: bigscience/bloom - NUM_SHARD: 8 -environment: - image: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference:0.2.0 - inference_config: - liveness_route: - port: 80 - path: /health - readiness_route: - port: 80 - path: /health - scoring_route: - port: 80 - path: /generate -instance_type: Standard_ND96amsr_A100_v4 -request_settings: - request_timeout_ms: 90000 - max_concurrent_requests_per_instance: 256 -liveness_probe: - initial_delay: 600 - timeout: 90 - period: 120 - success_threshold: 1 - failure_threshold: 5 -readiness_probe: - initial_delay: 600 - timeout: 90 - period: 120 - success_threshold: 1 - failure_threshold: 5 -instance_count: 1 diff --git a/aml/endpoint.yaml b/aml/endpoint.yaml deleted file mode 100644 index f2f01d5e..00000000 --- a/aml/endpoint.yaml +++ /dev/null @@ -1,3 +0,0 @@ -$schema: https://azuremlsdk2.blob.core.windows.net/latest/managedOnlineEndpoint.schema.json -name: bloom-inference -auth_mode: key diff --git a/aml/model.yaml b/aml/model.yaml deleted file mode 100644 index bfcdd33f..00000000 --- a/aml/model.yaml +++ /dev/null @@ -1,3 +0,0 @@ -$schema: https://azuremlschemas.azureedge.net/latest/model.schema.json -name: bloom-safetensors -path: /data/bloom-safetensors diff --git a/docs/openapi.json b/docs/openapi.json index 8c652946..b91729d0 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.8.2" + "version": "0.9.0" }, "paths": { "/": { @@ -270,6 +270,35 @@ } } }, + "/health": { + "get": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Health check method", + "description": "Health check method", + "operationId": "health", + "responses": { + "200": { + "description": "Everything is working fine" + }, + "503": { + "description": "Text generation inference is down", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "unhealthy", + "error_type": "healthcheck" + } + } + } + } + } + } + }, "/info": { "get": { "tags": [ diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 30abe88a..5b5cb45e 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -1040,14 +1040,18 @@ fn main() -> Result<(), LauncherError> { return Ok(()); } - let mut webserver = spawn_webserver(args, shutdown.clone(), &shutdown_receiver)?; + let mut webserver = + spawn_webserver(args, shutdown.clone(), &shutdown_receiver).map_err(|err| { + shutdown_shards(shutdown.clone(), &shutdown_receiver); + err + })?; // Default exit code let mut exit_code = Ok(()); while running.load(Ordering::SeqCst) { if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { - tracing::error!("Shard {rank} failed to start"); + tracing::error!("Shard {rank} crashed"); if let Some(err) = err { tracing::error!("{err}"); } diff --git a/router/Cargo.toml b/router/Cargo.toml index c1e665b1..10396826 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -22,11 +22,11 @@ text-generation-client = { path = "client" } clap = { version = "4.1.4", features = ["derive", "env"] } flume = "0.10.14" futures = "0.3.26" -metrics = "0.20.1" -metrics-exporter-prometheus = { version = "0.11.0", features = [] } +metrics = "0.21.0" +metrics-exporter-prometheus = { version = "0.12.1", features = [] } nohash-hasher = "0.2.0" -opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } -opentelemetry-otlp = "0.11.0" +opentelemetry = { version = "0.19.0", features = ["rt-tokio"] } +opentelemetry-otlp = "0.12.0" rand = "0.8.5" reqwest = { version = "0.11.14", features = [] } serde = "1.0.152" @@ -36,7 +36,7 @@ tokenizers = "0.13.3" tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } tower-http = { version = "0.4.0", features = ["cors"] } tracing = "0.1.37" -tracing-opentelemetry = "0.18.0" +tracing-opentelemetry = "0.19.0" tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } utoipa = { version = "3.0.1", features = ["axum_extras"] } utoipa-swagger-ui = { version = "3.0.2", features = ["axum"] } diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index db7245e0..43f444e6 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -11,10 +11,10 @@ grpc-metadata = { path = "../grpc-metadata" } prost = "^0.11" thiserror = "^1.0" tokio = { version = "^1.25", features = ["sync"] } -tonic = "^0.8" +tonic = "^0.9" tower = "^0.4" tracing = "^0.1" [build-dependencies] -tonic-build = "0.8.4" +tonic-build = "0.9.2" prost-build = "0.11.6" diff --git a/router/grpc-metadata/Cargo.toml b/router/grpc-metadata/Cargo.toml index 311092e3..9e01f527 100644 --- a/router/grpc-metadata/Cargo.toml +++ b/router/grpc-metadata/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -opentelemetry = "0.18.0" -tonic = "^0.8" +opentelemetry = "^0.19" +tonic = "^0.9" tracing = "^0.1" -tracing-opentelemetry = "0.18.0" +tracing-opentelemetry = "^0.19" diff --git a/router/src/server.rs b/router/src/server.rs index ee96ead6..54418f84 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -532,6 +532,7 @@ pub async fn run( #[derive(OpenApi)] #[openapi( paths( + health, get_model_info, compat_generate, generate, diff --git a/rust-toolchain.toml b/rust-toolchain.toml index d6ecf0c4..2db1883c 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.69.0" +channel = "1.70.0" components = ["rustfmt", "clippy"] \ No newline at end of file diff --git a/server/pyproject.toml b/server/pyproject.toml index f0ec25eb..294bcfc0 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.8.2" +version = "0.9.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 1da07e85aae8ce417dda3effd516691394dc31a1 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 3 Jul 2023 13:01:46 +0200 Subject: [PATCH 295/793] feat(server): Add Non flash MPT. (#514) # What does this PR do? This adds a non flash version of MPT. Flash is harder because we need to create a bias ready cuda kernel of flash attention. Fixes https://github.com/huggingface/text-generation-inference/issues/361 Fixes https://github.com/huggingface/text-generation-inference/issues/491 Fixes https://github.com/huggingface/text-generation-inference/issues/290 --- .../__snapshots__/test_mpt/test_mpt.json | 140 ++ .../__snapshots__/test_mpt/test_mpt_load.json | 562 ++++++++ integration-tests/models/test_mpt.py | 48 + server/poetry.lock | 13 +- server/pyproject.toml | 1 + server/requirements.txt | 1 + .../text_generation_server/models/__init__.py | 5 + .../models/custom_modeling/mpt_modeling.py | 1140 +++++++++++++++++ server/text_generation_server/models/mpt.py | 90 ++ server/text_generation_server/utils/layers.py | 12 + 10 files changed, 2011 insertions(+), 1 deletion(-) create mode 100644 integration-tests/models/__snapshots__/test_mpt/test_mpt.json create mode 100644 integration-tests/models/__snapshots__/test_mpt/test_mpt_load.json create mode 100644 integration-tests/models/test_mpt.py create mode 100644 server/text_generation_server/models/custom_modeling/mpt_modeling.py create mode 100644 server/text_generation_server/models/mpt.py diff --git a/integration-tests/models/__snapshots__/test_mpt/test_mpt.json b/integration-tests/models/__snapshots__/test_mpt/test_mpt.json new file mode 100644 index 00000000..abbbf03c --- /dev/null +++ b/integration-tests/models/__snapshots__/test_mpt/test_mpt.json @@ -0,0 +1,140 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 17, + "prefill": [ + { + "id": 1276, + "logprob": null, + "text": "What" + }, + { + "id": 310, + "logprob": -1.5117188, + "text": " is" + }, + { + "id": 18147, + "logprob": -8.96875, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -1.953125, + "text": " Learning" + }, + { + "id": 32, + "logprob": -0.94189453, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 428, + "logprob": -1.5830078, + "special": false, + "text": " -" + }, + { + "id": 18147, + "logprob": -3.3105469, + "special": false, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -0.3215332, + "special": false, + "text": " Learning" + }, + { + "id": 187, + "logprob": -2.5566406, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.6074219, + "special": false, + "text": "Deep" + }, + { + "id": 20727, + "logprob": -0.69628906, + "special": false, + "text": " Learning" + }, + { + "id": 310, + "logprob": -0.6923828, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.5263672, + "special": false, + "text": " a" + }, + { + "id": 749, + "logprob": -1.8544922, + "special": false, + "text": " sub" + }, + { + "id": 3423, + "logprob": -0.6118164, + "special": false, + "text": "field" + }, + { + "id": 273, + "logprob": -0.055877686, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.0537109, + "special": false, + "text": " machine" + }, + { + "id": 4715, + "logprob": -0.0115737915, + "special": false, + "text": " learning" + }, + { + "id": 326, + "logprob": -0.9111328, + "special": false, + "text": " that" + }, + { + "id": 4648, + "logprob": -1.4589844, + "special": false, + "text": " uses" + }, + { + "id": 13345, + "logprob": -1.4853516, + "special": false, + "text": " artificial" + }, + { + "id": 11454, + "logprob": -0.021636963, + "special": false, + "text": " neural" + } + ] + }, + "generated_text": " - Deep Learning\nDeep Learning is a subfield of machine learning that uses artificial neural" +} diff --git a/integration-tests/models/__snapshots__/test_mpt/test_mpt_load.json b/integration-tests/models/__snapshots__/test_mpt/test_mpt_load.json new file mode 100644 index 00000000..e3bc57ed --- /dev/null +++ b/integration-tests/models/__snapshots__/test_mpt/test_mpt_load.json @@ -0,0 +1,562 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 17, + "prefill": [ + { + "id": 1276, + "logprob": null, + "text": "What" + }, + { + "id": 310, + "logprob": -1.5117188, + "text": " is" + }, + { + "id": 18147, + "logprob": -8.96875, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -1.953125, + "text": " Learning" + }, + { + "id": 32, + "logprob": -0.94189453, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 428, + "logprob": -1.5830078, + "special": false, + "text": " -" + }, + { + "id": 18147, + "logprob": -3.3183594, + "special": false, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -0.32617188, + "special": false, + "text": " Learning" + }, + { + "id": 187, + "logprob": -2.5742188, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.6015625, + "special": false, + "text": "Deep" + }, + { + "id": 20727, + "logprob": -0.69628906, + "special": false, + "text": " Learning" + }, + { + "id": 310, + "logprob": -0.67822266, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.5395508, + "special": false, + "text": " a" + }, + { + "id": 749, + "logprob": -1.8623047, + "special": false, + "text": " sub" + }, + { + "id": 3423, + "logprob": -0.6020508, + "special": false, + "text": "field" + }, + { + "id": 273, + "logprob": -0.0552063, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.0742188, + "special": false, + "text": " machine" + }, + { + "id": 4715, + "logprob": -0.011405945, + "special": false, + "text": " learning" + }, + { + "id": 326, + "logprob": -0.9165039, + "special": false, + "text": " that" + }, + { + "id": 4648, + "logprob": -1.4501953, + "special": false, + "text": " uses" + }, + { + "id": 13345, + "logprob": -1.4960938, + "special": false, + "text": " artificial" + }, + { + "id": 11454, + "logprob": -0.02116394, + "special": false, + "text": " neural" + } + ] + }, + "generated_text": " - Deep Learning\nDeep Learning is a subfield of machine learning that uses artificial neural" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 17, + "prefill": [ + { + "id": 1276, + "logprob": null, + "text": "What" + }, + { + "id": 310, + "logprob": -1.5, + "text": " is" + }, + { + "id": 18147, + "logprob": -8.984375, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -1.96875, + "text": " Learning" + }, + { + "id": 32, + "logprob": -0.93359375, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 428, + "logprob": -1.5800781, + "special": false, + "text": " -" + }, + { + "id": 18147, + "logprob": -3.3242188, + "special": false, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -0.31835938, + "special": false, + "text": " Learning" + }, + { + "id": 187, + "logprob": -2.5644531, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.5957031, + "special": false, + "text": "Deep" + }, + { + "id": 20727, + "logprob": -0.69628906, + "special": false, + "text": " Learning" + }, + { + "id": 310, + "logprob": -0.68603516, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.5258789, + "special": false, + "text": " a" + }, + { + "id": 749, + "logprob": -1.859375, + "special": false, + "text": " sub" + }, + { + "id": 3423, + "logprob": -0.6166992, + "special": false, + "text": "field" + }, + { + "id": 273, + "logprob": -0.056762695, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.0703125, + "special": false, + "text": " machine" + }, + { + "id": 4715, + "logprob": -0.011428833, + "special": false, + "text": " learning" + }, + { + "id": 326, + "logprob": -0.9213867, + "special": false, + "text": " that" + }, + { + "id": 4648, + "logprob": -1.4726562, + "special": false, + "text": " uses" + }, + { + "id": 13345, + "logprob": -1.5039062, + "special": false, + "text": " artificial" + }, + { + "id": 11454, + "logprob": -0.021652222, + "special": false, + "text": " neural" + } + ] + }, + "generated_text": " - Deep Learning\nDeep Learning is a subfield of machine learning that uses artificial neural" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 17, + "prefill": [ + { + "id": 1276, + "logprob": null, + "text": "What" + }, + { + "id": 310, + "logprob": -1.5, + "text": " is" + }, + { + "id": 18147, + "logprob": -8.984375, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -1.96875, + "text": " Learning" + }, + { + "id": 32, + "logprob": -0.93359375, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 428, + "logprob": -1.5800781, + "special": false, + "text": " -" + }, + { + "id": 18147, + "logprob": -3.3242188, + "special": false, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -0.31835938, + "special": false, + "text": " Learning" + }, + { + "id": 187, + "logprob": -2.5644531, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.5957031, + "special": false, + "text": "Deep" + }, + { + "id": 20727, + "logprob": -0.69628906, + "special": false, + "text": " Learning" + }, + { + "id": 310, + "logprob": -0.68603516, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.5258789, + "special": false, + "text": " a" + }, + { + "id": 749, + "logprob": -1.859375, + "special": false, + "text": " sub" + }, + { + "id": 3423, + "logprob": -0.6166992, + "special": false, + "text": "field" + }, + { + "id": 273, + "logprob": -0.056762695, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.0703125, + "special": false, + "text": " machine" + }, + { + "id": 4715, + "logprob": -0.011428833, + "special": false, + "text": " learning" + }, + { + "id": 326, + "logprob": -0.9213867, + "special": false, + "text": " that" + }, + { + "id": 4648, + "logprob": -1.4726562, + "special": false, + "text": " uses" + }, + { + "id": 13345, + "logprob": -1.5039062, + "special": false, + "text": " artificial" + }, + { + "id": 11454, + "logprob": -0.021652222, + "special": false, + "text": " neural" + } + ] + }, + "generated_text": " - Deep Learning\nDeep Learning is a subfield of machine learning that uses artificial neural" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 17, + "prefill": [ + { + "id": 1276, + "logprob": null, + "text": "What" + }, + { + "id": 310, + "logprob": -1.5, + "text": " is" + }, + { + "id": 18147, + "logprob": -8.984375, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -1.96875, + "text": " Learning" + }, + { + "id": 32, + "logprob": -0.93359375, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 428, + "logprob": -1.5800781, + "special": false, + "text": " -" + }, + { + "id": 18147, + "logprob": -3.3242188, + "special": false, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -0.31835938, + "special": false, + "text": " Learning" + }, + { + "id": 187, + "logprob": -2.5644531, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.5957031, + "special": false, + "text": "Deep" + }, + { + "id": 20727, + "logprob": -0.69628906, + "special": false, + "text": " Learning" + }, + { + "id": 310, + "logprob": -0.68603516, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.5258789, + "special": false, + "text": " a" + }, + { + "id": 749, + "logprob": -1.859375, + "special": false, + "text": " sub" + }, + { + "id": 3423, + "logprob": -0.6166992, + "special": false, + "text": "field" + }, + { + "id": 273, + "logprob": -0.056762695, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.0703125, + "special": false, + "text": " machine" + }, + { + "id": 4715, + "logprob": -0.011428833, + "special": false, + "text": " learning" + }, + { + "id": 326, + "logprob": -0.9213867, + "special": false, + "text": " that" + }, + { + "id": 4648, + "logprob": -1.4726562, + "special": false, + "text": " uses" + }, + { + "id": 13345, + "logprob": -1.5039062, + "special": false, + "text": " artificial" + }, + { + "id": 11454, + "logprob": -0.021652222, + "special": false, + "text": " neural" + } + ] + }, + "generated_text": " - Deep Learning\nDeep Learning is a subfield of machine learning that uses artificial neural" + } +] diff --git a/integration-tests/models/test_mpt.py b/integration-tests/models/test_mpt.py new file mode 100644 index 00000000..d58a8c5a --- /dev/null +++ b/integration-tests/models/test_mpt.py @@ -0,0 +1,48 @@ +import pytest + + +@pytest.fixture(scope="module") +def mpt_sharded_handle(launcher): + with launcher("mosaicml/mpt-7b", num_shard=2) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def mpt_sharded(mpt_sharded_handle): + await mpt_sharded_handle.health(300) + return mpt_sharded_handle.client + + +@pytest.mark.asyncio +async def test_mpt(mpt_sharded, response_snapshot): + response = await mpt_sharded.generate( + "What is Deep Learning?", + max_new_tokens=17, + decoder_input_details=True, + ) + + assert response.details.generated_tokens == 17 + assert ( + response.generated_text + == " - Deep Learning\nDeep Learning is a subfield of machine learning that uses artificial neural" + ) + assert response == response_snapshot + + +@pytest.mark.asyncio +async def test_mpt_load(mpt_sharded, generate_load, response_snapshot): + responses = await generate_load( + mpt_sharded, + "What is Deep Learning?", + max_new_tokens=17, + n=4, + ) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) + assert ( + responses[0].generated_text + == " - Deep Learning\nDeep Learning is a subfield of machine learning that uses artificial neural" + ) + + assert responses == response_snapshot diff --git a/server/poetry.lock b/server/poetry.lock index 9a6900bc..7c126e80 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -187,6 +187,17 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] +[[package]] +name = "einops" +version = "0.6.1" +description = "A new flavour of deep learning operations" +optional = false +python-versions = ">=3.7" +files = [ + {file = "einops-0.6.1-py3-none-any.whl", hash = "sha256:99149e46cc808956b174932fe563d920db4d6e5dadb8c6ecdaa7483b7ef7cfc3"}, + {file = "einops-0.6.1.tar.gz", hash = "sha256:f95f8d00f4ded90dbc4b19b6f98b177332614b0357dde66997f3ae5d474dc8c8"}, +] + [[package]] name = "exceptiongroup" version = "1.1.1" @@ -1586,4 +1597,4 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "54ecacb32d699cb1298c237c4661c1b707f119cf2c27bd54bad7a1ea2ffb8b10" +content-hash = "3174a211d30bed5990ed5f8418416c951bb6c585153fb51b62809baa89ef07d0" diff --git a/server/pyproject.toml b/server/pyproject.toml index 294bcfc0..bbf5836d 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -27,6 +27,7 @@ sentencepiece = "^0.1.97" tokenizers = "0.13.3" huggingface-hub = "^0.14.1" transformers = "^4.29.2" +einops = "^0.6.1" [tool.poetry.extras] accelerate = ["accelerate"] diff --git a/server/requirements.txt b/server/requirements.txt index a9bd441c..92693bbd 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -4,6 +4,7 @@ charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0" click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows") deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0" +einops==0.6.1 ; python_version >= "3.9" and python_version < "4.0" filelock==3.12.2 ; python_version >= "3.9" and python_version < "4.0" fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "4.0" googleapis-common-protos==1.59.1 ; python_version >= "3.9" and python_version < "4.0" diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index e45e198a..fd97f8b1 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -10,6 +10,7 @@ from text_generation_server.models.causal_lm import CausalLM from text_generation_server.models.flash_causal_lm import FlashCausalLM from text_generation_server.models.bloom import BLOOMSharded +from text_generation_server.models.mpt import MPTSharded from text_generation_server.models.seq2seq_lm import Seq2SeqLM from text_generation_server.models.rw import RW from text_generation_server.models.opt import OPTSharded @@ -178,6 +179,10 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) + elif model_type == "mpt": + return MPTSharded( + model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + ) elif model_type == "gpt_neox": if FLASH_ATTENTION: diff --git a/server/text_generation_server/models/custom_modeling/mpt_modeling.py b/server/text_generation_server/models/custom_modeling/mpt_modeling.py new file mode 100644 index 00000000..5ea204e1 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/mpt_modeling.py @@ -0,0 +1,1140 @@ +"""A simple, flexible implementation of a GPT model. + +Inspired by https://github.com/karpathy/minGPT/blob/master/mingpt/model.py +""" +import math +import os +import warnings +from typing import List, Optional, Tuple, Union +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import PreTrainedModel, PreTrainedTokenizer, PreTrainedTokenizerFast +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, +) +from einops import rearrange +from packaging import version +from text_generation_server.utils.layers import ( + TensorParallelEmbedding, + TensorParallelColumnLinear, + TensorParallelRowLinear, + TensorParallelHead, + get_linear, +) + +EPS = 1e-5 + + +def load_col(config, prefix, weights, bias): + assert bias == False, NotImplementedError + assert config.quantize != "gptq", NotImplementedError + slice_ = weights._get_slice(f"{prefix}.weight") + rank = weights.process_group.rank() + size = weights.process_group.size() + + h3, h = slice_.get_shape() + block_size = h // size + + q_part = slice_[rank * block_size : (rank + 1) * block_size] + k_part = slice_[h + rank * block_size : h + (rank + 1) * block_size] + v_part = slice_[2 * h + rank * block_size : 2 * h + (rank + 1) * block_size] + + weight = torch.cat([q_part, k_part, v_part], dim=0) + if weight.dtype != torch.int32: + weight = weight.to(dtype=weights.dtype) + weight = weight.to(device=weights.device) + bias = None + linear = get_linear(weight, bias, config.quantize) + return TensorParallelColumnLinear(linear) + + +def _reset_is_causal( + num_query_tokens: int, num_key_tokens: int, original_is_causal: bool +): + if original_is_causal and num_query_tokens != num_key_tokens: + if num_query_tokens != 1: + raise NotImplementedError( + "MPT does not support query and key with different number of tokens, unless number of query tokens is 1." + ) + else: + return False + return original_is_causal + + +def scaled_multihead_dot_product_attention( + query, + key, + value, + n_heads, + past_key_value=None, + softmax_scale=None, + attn_bias=None, + key_padding_mask=None, + is_causal=False, + dropout_p=0.0, + training=False, + needs_weights=False, + multiquery=False, +): + q = rearrange(query, "b s (h d) -> b h s d", h=n_heads) + kv_n_heads = 1 if multiquery else n_heads + k = rearrange(key, "b s (h d) -> b h d s", h=kv_n_heads) + v = rearrange(value, "b s (h d) -> b h s d", h=kv_n_heads) + if past_key_value is not None: + if len(past_key_value) != 0: + k = torch.cat([past_key_value[0], k], dim=3) + v = torch.cat([past_key_value[1], v], dim=2) + past_key_value = (k, v) + (b, _, s_q, d) = q.shape + s_k = k.size(-1) + attn_weight = q.matmul(k) * softmax_scale + if attn_bias is not None: + _s_q = max(0, attn_bias.size(2) - s_q) + _s_k = max(0, attn_bias.size(3) - s_k) + attn_bias = attn_bias[:, :, _s_q:, _s_k:] + if ( + attn_bias.size(-1) != 1 + and attn_bias.size(-1) != s_k + or (attn_bias.size(-2) != 1 and attn_bias.size(-2) != s_q) + ): + raise RuntimeError( + f"attn_bias (shape: {attn_bias.shape}) is expected to broadcast to shape: {attn_weight.shape}." + ) + attn_weight = attn_weight + attn_bias + min_val = torch.finfo(q.dtype).min + if key_padding_mask is not None: + if attn_bias is not None: + warnings.warn( + "Propogating key_padding_mask to the attention module " + + "and applying it within the attention module can cause " + + "unneccessary computation/memory usage. Consider integrating " + + "into attn_bias once and passing that to each attention " + + "module instead." + ) + attn_weight = attn_weight.masked_fill( + ~key_padding_mask.view((b, 1, 1, s_k)), min_val + ) + if is_causal and (not q.size(2) == 1): + s = max(s_q, s_k) + causal_mask = attn_weight.new_ones(s, s, dtype=torch.float16) + causal_mask = causal_mask.tril() + causal_mask = causal_mask.to(torch.bool) + causal_mask = ~causal_mask + causal_mask = causal_mask[-s_q:, -s_k:] + attn_weight = attn_weight.masked_fill(causal_mask.view(1, 1, s_q, s_k), min_val) + attn_weight = torch.softmax(attn_weight, dim=-1) + if dropout_p: + attn_weight = torch.nn.functional.dropout( + attn_weight, p=dropout_p, training=training, inplace=True + ) + out = attn_weight.to(v.dtype).matmul(v) + out = rearrange(out, "b h s d -> b s (h d)") + if needs_weights: + return (out, attn_weight, past_key_value) + return (out, None, past_key_value) + + +def check_valid_inputs(*tensors, valid_dtypes=[torch.float16, torch.bfloat16]): + for tensor in tensors: + if tensor.dtype not in valid_dtypes: + raise TypeError( + f"tensor.dtype={tensor.dtype!r} must be in valid_dtypes={valid_dtypes!r}." + ) + if not tensor.is_cuda: + raise TypeError( + f"Inputs must be cuda tensors (tensor.is_cuda={tensor.is_cuda!r})." + ) + + +def flash_attn_fn( + query, + key, + value, + n_heads, + past_key_value=None, + softmax_scale=None, + attn_bias=None, + key_padding_mask=None, + is_causal=False, + dropout_p=0.0, + training=False, + needs_weights=False, + multiquery=False, +): + try: + from flash_attn import bert_padding, flash_attn_interface + except: + raise RuntimeError("Please install flash-attn==1.0.3.post0") + check_valid_inputs(query, key, value) + if past_key_value is not None: + if len(past_key_value) != 0: + key = torch.cat([past_key_value[0], key], dim=1) + value = torch.cat([past_key_value[1], value], dim=1) + past_key_value = (key, value) + if attn_bias is not None: + _s_q = max(0, attn_bias.size(2) - query.size(1)) + _s_k = max(0, attn_bias.size(3) - key.size(1)) + attn_bias = attn_bias[:, :, _s_q:, _s_k:] + if attn_bias is not None: + raise NotImplementedError(f"attn_bias not implemented for flash attn.") + (batch_size, seqlen) = query.shape[:2] + if key_padding_mask is None: + key_padding_mask = torch.ones_like(key[:, :, 0], dtype=torch.bool) + query_padding_mask = key_padding_mask[:, -query.size(1) :] + (query_unpad, indices_q, cu_seqlens_q, max_seqlen_q) = bert_padding.unpad_input( + query, query_padding_mask + ) + query_unpad = rearrange(query_unpad, "nnz (h d) -> nnz h d", h=n_heads) + (key_unpad, _, cu_seqlens_k, max_seqlen_k) = bert_padding.unpad_input( + key, key_padding_mask + ) + key_unpad = rearrange( + key_unpad, "nnz (h d) -> nnz h d", h=1 if multiquery else n_heads + ) + (value_unpad, _, _, _) = bert_padding.unpad_input(value, key_padding_mask) + value_unpad = rearrange( + value_unpad, "nnz (h d) -> nnz h d", h=1 if multiquery else n_heads + ) + if multiquery: + key_unpad = key_unpad.expand(key_unpad.size(0), n_heads, key_unpad.size(-1)) + value_unpad = value_unpad.expand( + value_unpad.size(0), n_heads, value_unpad.size(-1) + ) + dropout_p = dropout_p if training else 0.0 + reset_is_causal = _reset_is_causal(query.size(1), key.size(1), is_causal) + output_unpad = flash_attn_interface.flash_attn_unpadded_func( + query_unpad, + key_unpad, + value_unpad, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + dropout_p, + softmax_scale=softmax_scale, + causal=reset_is_causal, + return_attn_probs=needs_weights, + ) + output = bert_padding.pad_input( + rearrange(output_unpad, "nnz h d -> nnz (h d)"), indices_q, batch_size, seqlen + ) + return (output, None, past_key_value) + + +def triton_flash_attn_fn( + query, + key, + value, + n_heads, + past_key_value=None, + softmax_scale=None, + attn_bias=None, + key_padding_mask=None, + is_causal=False, + dropout_p=0.0, + training=False, + needs_weights=False, + multiquery=False, +): + try: + from .flash_attn_triton import flash_attn_func + except: + _installed = False + if version.parse(torch.__version__) < version.parse("2.0.0"): + _installed = True + try: + from flash_attn.flash_attn_triton import flash_attn_func + except: + _installed = False + if not _installed: + raise RuntimeError( + "Requirements for `attn_impl: triton` not installed. Either (1) have a CUDA-compatible GPU and `pip install .[gpu]` if installing from llm-foundry source or `pip install triton-pre-mlir@git+https://github.com/vchiley/triton.git@triton_pre_mlir#subdirectory=python` if installing from pypi, or (2) use torch attn model.attn_config.attn_impl=torch (torch attn_impl will be slow). Note: (1) requires you have CMake and PyTorch already installed." + ) + check_valid_inputs(query, key, value) + if past_key_value is not None: + if len(past_key_value) != 0: + key = torch.cat([past_key_value[0], key], dim=1) + value = torch.cat([past_key_value[1], value], dim=1) + past_key_value = (key, value) + if attn_bias is not None: + _s_q = max(0, attn_bias.size(2) - query.size(1)) + _s_k = max(0, attn_bias.size(3) - key.size(1)) + attn_bias = attn_bias[:, :, _s_q:, _s_k:] + if dropout_p: + raise NotImplementedError(f"Dropout not implemented for attn_impl: triton.") + if needs_weights: + raise NotImplementedError(f"attn_impl: triton cannot return attn weights.") + if key_padding_mask is not None: + warnings.warn( + "Propagating key_padding_mask to the attention module " + + "and applying it within the attention module can cause " + + "unnecessary computation/memory usage. Consider integrating " + + "into attn_bias once and passing that to each attention " + + "module instead." + ) + (b_size, s_k) = key_padding_mask.shape[:2] + if attn_bias is None: + attn_bias = query.new_zeros(b_size, 1, 1, s_k) + attn_bias = attn_bias.masked_fill( + ~key_padding_mask.view((b_size, 1, 1, s_k)), torch.finfo(query.dtype).min + ) + query = rearrange(query, "b s (h d) -> b s h d", h=n_heads) + key = rearrange(key, "b s (h d) -> b s h d", h=1 if multiquery else n_heads) + value = rearrange(value, "b s (h d) -> b s h d", h=1 if multiquery else n_heads) + if multiquery: + key = key.expand(*key.shape[:2], n_heads, key.size(-1)) + value = value.expand(*value.shape[:2], n_heads, value.size(-1)) + reset_is_causal = _reset_is_causal(query.size(1), key.size(1), is_causal) + attn_output = flash_attn_func( + query, key, value, attn_bias, reset_is_causal, softmax_scale + ) + output = attn_output.view(*attn_output.shape[:2], -1) + return (output, None, past_key_value) + + +class MultiheadAttention(nn.Module): + """Multi-head self attention. + + Using torch or triton attention implemetation enables user to also use + additive bias. + """ + + def __init__( + self, + config, + prefix, + weights, + ): + super().__init__() + attn_impl = config.attn_config["attn_impl"] + self.attn_impl = config.attn_config["attn_impl"] + self.clip_qkv = config.attn_config["clip_qkv"] + self.qk_ln = config.attn_config["qk_ln"] + self.d_model = config.d_model + d_model = config.d_model + self.n_heads = config.n_heads + self.softmax_scale = config.attn_config["softmax_scale"] + if self.softmax_scale is None: + self.softmax_scale = 1 / math.sqrt(self.d_model / self.n_heads) + self.attn_dropout_p = config.attn_config["attn_pdrop"] + self.n_heads = self.n_heads // weights.process_group.size() + self.Wqkv = load_col( + config, prefix=f"{prefix}.Wqkv", weights=weights, bias=not config.no_bias + ) + if self.qk_ln: + raise NotImplementedError("qk_ln is not supported") + if self.attn_impl == "flash": + self.attn_fn = flash_attn_fn + elif self.attn_impl == "triton": + self.attn_fn = triton_flash_attn_fn + elif self.attn_impl == "torch": + self.attn_fn = scaled_multihead_dot_product_attention + else: + raise ValueError(f"attn_impl={attn_impl!r} is an invalid setting.") + self.out_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.out_proj", + weights=weights, + bias=not config.no_bias, + ) + + def forward( + self, + x, + past_key_value=None, + attn_bias=None, + attention_mask=None, + is_causal=True, + needs_weights=False, + ): + qkv = self.Wqkv(x) + if self.clip_qkv: + qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv) + (query, key, value) = qkv.chunk(3, dim=2) + + key_padding_mask = attention_mask + if self.qk_ln: + dtype = query.dtype + query = self.q_ln(query).to(dtype) + key = self.k_ln(key).to(dtype) + (context, attn_weights, past_key_value) = self.attn_fn( + query, + key, + value, + self.n_heads, + past_key_value=past_key_value, + softmax_scale=self.softmax_scale, + attn_bias=attn_bias, + key_padding_mask=key_padding_mask, + is_causal=is_causal, + dropout_p=self.attn_dropout_p, + training=self.training, + needs_weights=needs_weights, + ) + out = self.out_proj(context) + return (out, attn_weights, past_key_value) + + +class MultiQueryAttention(nn.Module): + """Multi-Query self attention. + + Using torch or triton attention implemetation enables user to also use + additive bias. + """ + + def __init__(self, config, prefix, weights): + super().__init__() + attn_impl = config.attn_config["attn_impl"] + self.attn_impl = config.attn_config["attn_impl"] + self.clip_qkv = config.attn_config["clip_qkv"] + self.qk_ln = config.attn_config["qk_ln"] + self.d_model = config.d_model + d_model = config.d_model + self.n_heads = config.n_heads + self.softmax_scale = config.attn_config["softmax_scale"] + if self.softmax_scale is None: + self.softmax_scale = 1 / math.sqrt(self.head_dim) + self.attn_dropout_p = config.attn_config["attn_pdrop"] + # self.Wqkv = nn.Linear(d_model, d_model + 2 * self.head_dim, device=device) + self.Wqkv = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.Wqkv", weights=weights, bias=not config.no_bias + ) + fuse_splits = (d_model, d_model + self.head_dim) + if self.qk_ln: + raise NotImplementedError("qk_ln not supported") + if self.attn_impl == "flash": + self.attn_fn = flash_attn_fn + elif self.attn_impl == "triton": + self.attn_fn = triton_flash_attn_fn + if verbose: + warnings.warn( + "While `attn_impl: triton` can be faster than `attn_impl: flash` " + + "it uses more memory. When training larger models this can trigger " + + "alloc retries which hurts performance. If encountered, we recommend " + + "using `attn_impl: flash` if your model does not use `alibi` or `prefix_lm`." + ) + elif self.attn_impl == "torch": + self.attn_fn = scaled_multihead_dot_product_attention + if torch.cuda.is_available() and verbose: + warnings.warn( + "Using `attn_impl: torch`. If your model does not use `alibi` or " + + "`prefix_lm` we recommend using `attn_impl: flash` otherwise " + + "we recommend using `attn_impl: triton`." + ) + else: + raise ValueError(f"attn_impl={attn_impl!r} is an invalid setting.") + self.out_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.out_proj", + weights=weights, + bias=not config.no_bias, + ) + # self.out_proj._is_residual = True + + def forward( + self, + x, + past_key_value=None, + attn_bias=None, + attention_mask=None, + is_causal=True, + needs_weights=False, + ): + qkv = self.Wqkv(x) + if self.clip_qkv: + qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv) + (query, key, value) = qkv.split( + [self.d_model, self.head_dim, self.head_dim], dim=2 + ) + key_padding_mask = attention_mask + if self.qk_ln: + dtype = query.dtype + query = self.q_ln(query).to(dtype) + key = self.k_ln(key).to(dtype) + (context, attn_weights, past_key_value) = self.attn_fn( + query, + key, + value, + self.n_heads, + past_key_value=past_key_value, + softmax_scale=self.softmax_scale, + attn_bias=attn_bias, + key_padding_mask=key_padding_mask, + is_causal=is_causal, + dropout_p=self.attn_dropout_p, + training=self.training, + needs_weights=needs_weights, + multiquery=True, + ) + return (self.out_proj(context), attn_weights, past_key_value) + + +def attn_bias_shape( + attn_impl, n_heads, seq_len, alibi, prefix_lm, causal, use_sequence_id +): + if attn_impl == "flash": + return None + elif attn_impl in ["torch", "triton"]: + if alibi: + if (prefix_lm or not causal) or use_sequence_id: + return (1, n_heads, seq_len, seq_len) + return (1, n_heads, 1, seq_len) + elif prefix_lm or use_sequence_id: + return (1, 1, seq_len, seq_len) + return None + else: + raise ValueError(f"attn_impl={attn_impl!r} is an invalid setting.") + + +def build_attn_bias( + attn_impl, attn_bias, n_heads, seq_len, causal=False, alibi=False, alibi_bias_max=8 +): + if attn_impl == "flash": + return None + elif attn_impl in ["torch", "triton"]: + if alibi: + (device, dtype) = (attn_bias.device, attn_bias.dtype) + attn_bias = attn_bias.add( + build_alibi_bias( + n_heads, + seq_len, + full=not causal, + alibi_bias_max=alibi_bias_max, + device=device, + dtype=dtype, + ) + ) + return attn_bias + else: + raise ValueError(f"attn_impl={attn_impl!r} is an invalid setting.") + + +def gen_slopes(n_heads, alibi_bias_max=8, device=None): + _n_heads = 2 ** math.ceil(math.log2(n_heads)) + m = torch.arange(1, _n_heads + 1, dtype=torch.float32, device=device) + m = m.mul(alibi_bias_max / _n_heads) + slopes = 1.0 / torch.pow(2, m) + if _n_heads != n_heads: + slopes = torch.concat([slopes[1::2], slopes[::2]])[:n_heads] + return slopes.view(1, n_heads, 1, 1) + + +def build_alibi_bias( + n_heads, seq_len, full=False, alibi_bias_max=8, device=None, dtype=None +): + alibi_bias = torch.arange(1 - seq_len, 1, dtype=torch.int32, device=device).view( + 1, 1, 1, seq_len + ) + if full: + alibi_bias = alibi_bias - torch.arange( + 1 - seq_len, 1, dtype=torch.int32, device=device + ).view(1, 1, seq_len, 1) + alibi_bias = alibi_bias.abs().mul(-1) + slopes = gen_slopes(n_heads, alibi_bias_max, device=device) + alibi_bias = alibi_bias * slopes + return alibi_bias.to(dtype=dtype) + + +ATTN_CLASS_REGISTRY = { + "multihead_attention": MultiheadAttention, + "multiquery_attention": MultiQueryAttention, +} + +"""GPT Blocks used for the GPT Model.""" + + +class MPTMLP(nn.Module): + def __init__(self, config, prefix, weights): + super().__init__() + # self.up_proj = nn.Linear(d_model, expansion_ratio * d_model, device=device) + self.up_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.up_proj", weights=weights, bias=not config.no_bias + ) + self.act = nn.GELU(approximate="none") + # self.down_proj = nn.Linear(expansion_ratio * d_model, d_model, device=device) + self.down_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.down_proj", + weights=weights, + bias=not config.no_bias, + ) + # self.down_proj._is_residual = True + + def forward(self, x): + return self.down_proj(self.act(self.up_proj(x))) + + +class MPTBlock(nn.Module): + def __init__(self, config, prefix, weights): + super().__init__() + self.prefix = prefix + if config.attn_config["attn_type"] != "multihead_attention": + raise NotImplementedError( + f"""Not implemented attn {config.attn_config["attn_type"]}""" + ) + resid_pdrop = config.resid_pdrop + self.norm_1 = nn.LayerNorm.load_no_bias( + prefix=f"{prefix}.norm_1", weights=weights, eps=EPS + ) + self.norm_2 = nn.LayerNorm.load_no_bias( + prefix=f"{prefix}.norm_2", weights=weights, eps=EPS + ) + self.attn = MultiheadAttention(config, prefix=f"{prefix}.attn", weights=weights) + self.ffn = MPTMLP(config, prefix=f"{prefix}.ffn", weights=weights) + self.resid_attn_dropout = nn.Dropout(resid_pdrop) + self.resid_ffn_dropout = nn.Dropout(resid_pdrop) + + def forward( + self, + x: torch.Tensor, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attn_bias: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.ByteTensor] = None, + is_causal: bool = True, + ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor]]]: + a = self.norm_1(x) + (b, attn_weights, past_key_value) = self.attn( + a, + past_key_value=past_key_value, + attn_bias=attn_bias, + attention_mask=attention_mask, + is_causal=is_causal, + ) + x = x + self.resid_attn_dropout(b) + m = self.norm_2(x) + n = self.ffn(m) + x = x + self.resid_ffn_dropout(n) + return (x, attn_weights, past_key_value) + + +def _cast_if_autocast_enabled(tensor): + if torch.is_autocast_enabled(): + if tensor.device.type == "cuda": + dtype = torch.get_autocast_gpu_dtype() + elif tensor.device.type == "cpu": + dtype = torch.get_autocast_cpu_dtype() + else: + raise NotImplementedError() + return tensor.to(dtype=dtype) + return tensor + + +class LPLayerNorm(torch.nn.LayerNorm): + def __init__( + self, + normalized_shape, + eps=1e-05, + elementwise_affine=True, + device=None, + dtype=None, + ): + super().__init__( + normalized_shape=normalized_shape, + eps=eps, + elementwise_affine=elementwise_affine, + device=device, + dtype=dtype, + ) + + def forward(self, x): + module_device = x.device + downcast_x = _cast_if_autocast_enabled(x) + downcast_weight = ( + _cast_if_autocast_enabled(self.weight) + if self.weight is not None + else self.weight + ) + downcast_bias = ( + _cast_if_autocast_enabled(self.bias) if self.bias is not None else self.bias + ) + with torch.autocast(enabled=False, device_type=module_device.type): + return torch.nn.functional.layer_norm( + downcast_x, + self.normalized_shape, + downcast_weight, + downcast_bias, + self.eps, + ) + + +def rms_norm(x, weight=None, eps=1e-05): + output = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) + if weight is not None: + return output * weight + return output + + +class RMSNorm(torch.nn.Module): + def __init__( + self, normalized_shape, eps=1e-05, weight=True, dtype=None, device=None + ): + super().__init__() + self.eps = eps + if weight: + self.weight = torch.nn.Parameter( + torch.ones(normalized_shape, dtype=dtype, device=device) + ) + else: + self.register_parameter("weight", None) + + def forward(self, x): + return rms_norm(x.float(), self.weight, self.eps).to(dtype=x.dtype) + + +class LPRMSNorm(RMSNorm): + def __init__( + self, normalized_shape, eps=1e-05, weight=True, dtype=None, device=None + ): + super().__init__( + normalized_shape=normalized_shape, + eps=eps, + weight=weight, + dtype=dtype, + device=device, + ) + + def forward(self, x): + downcast_x = _cast_if_autocast_enabled(x) + downcast_weight = ( + _cast_if_autocast_enabled(self.weight) + if self.weight is not None + else self.weight + ) + with torch.autocast(enabled=False, device_type=x.device.type): + return rms_norm(downcast_x, downcast_weight, self.eps).to(dtype=x.dtype) + + +NORM_CLASS_REGISTRY = { + "layernorm": torch.nn.LayerNorm, + "low_precision_layernorm": LPLayerNorm, + "rmsnorm": RMSNorm, + "low_precision_rmsnorm": LPRMSNorm, +} + +Tokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast] + + +class MPTPreTrainedModel(PreTrainedModel): + base_model_prefix = "model" + _no_split_modules = ["MPTBlock"] + + +class MPTModel(MPTPreTrainedModel): + def __init__(self, config, weights): + # config._validate_config() + super().__init__(config) + self.world_size = weights.process_group.size() + self.rank = weights.process_group.rank() + self.n_heads = config.n_heads + self.attn_impl = config.attn_config["attn_impl"] + self.prefix_lm = config.attn_config["prefix_lm"] + self.attn_uses_sequence_id = config.attn_config["attn_uses_sequence_id"] + self.alibi = config.attn_config["alibi"] + self.alibi_bias_max = config.attn_config["alibi_bias_max"] + if config.init_device == "mixed": + if dist.get_local_rank() == 0: + config.init_device = "cpu" + else: + config.init_device = "meta" + if config.norm_type.lower() not in NORM_CLASS_REGISTRY.keys(): + norm_options = " | ".join(NORM_CLASS_REGISTRY.keys()) + raise NotImplementedError( + f"Requested norm type ({config.norm_type}) is not implemented within this repo (Options: {norm_options})." + ) + if config.norm_type.lower() != "low_precision_layernorm": + raise NotImplementedError( + f"Requested norm type ({config.norm_type}) is not implemented within this repo." + ) + + self.wte = TensorParallelEmbedding("transformer.wte", weights) + if not self.alibi: + # self.wpe = torch.nn.Embedding( + # config.max_seq_len, config.d_model, device=config.init_device + # ) + raise RuntimeError("no alibi no supported") + self.blocks = nn.ModuleList( + [ + MPTBlock(config, prefix=f"transformer.blocks.{i}", weights=weights) + for i in range(config.n_layers) + ] + ) + self.norm_f = nn.LayerNorm.load_no_bias( + prefix="transformer.norm_f", weights=weights, eps=EPS + ) + self.is_causal = not self.prefix_lm + self._attn_bias_initialized = False + self.attn_bias = None + self.attn_bias_shape = attn_bias_shape( + self.attn_impl, + config.n_heads, + config.max_seq_len, + self.alibi, + prefix_lm=self.prefix_lm, + causal=self.is_causal, + use_sequence_id=self.attn_uses_sequence_id, + ) + if config.no_bias: + for module in self.modules(): + if hasattr(module, "bias") and isinstance(module.bias, nn.Parameter): + if config.verbose: + warnings.warn(f"Removing bias ({module.bias}) from {module}.") + module.register_parameter("bias", None) + if config.verbose and config.verbose > 2: + print(self) + if "verbose" not in self.config.init_config: + self.config.init_config["verbose"] = self.config.verbose + if self.config.init_config["verbose"] > 1: + init_fn_name = self.config.init_config["name"] + warnings.warn(f"Using {init_fn_name} initialization.") + + @torch.no_grad() + def _attn_bias( + self, + device, + dtype, + attention_mask: Optional[torch.ByteTensor] = None, + prefix_mask: Optional[torch.ByteTensor] = None, + sequence_id: Optional[torch.LongTensor] = None, + ): + if not self._attn_bias_initialized: + if self.attn_bias_shape: + self.attn_bias = torch.zeros( + self.attn_bias_shape, device=device, dtype=dtype + ) + self.attn_bias = build_attn_bias( + self.attn_impl, + self.attn_bias, + self.config.n_heads, + self.config.max_seq_len, + causal=self.is_causal, + alibi=self.alibi, + alibi_bias_max=self.alibi_bias_max, + ) + assert self.n_heads % self.world_size == 0 + block_size = self.n_heads // self.world_size + self.attn_bias = self.attn_bias[ + :, self.rank * block_size : (self.rank + 1) * block_size + ] + self._attn_bias_initialized = True + if self.attn_impl == "flash": + return (self.attn_bias, attention_mask) + if self.attn_bias is not None: + self.attn_bias = self.attn_bias.to(dtype=dtype, device=device) + attn_bias = self.attn_bias + if self.prefix_lm: + assert isinstance(attn_bias, torch.Tensor) + assert isinstance(prefix_mask, torch.Tensor) + attn_bias = self._apply_prefix_mask(attn_bias, prefix_mask) + if self.attn_uses_sequence_id and sequence_id is not None: + assert isinstance(attn_bias, torch.Tensor) + attn_bias = self._apply_sequence_id(attn_bias, sequence_id) + if attention_mask is not None: + s_k = attention_mask.shape[-1] + if attn_bias is None: + attn_bias = torch.zeros((1, 1, 1, s_k), device=device, dtype=dtype) + else: + _s_k = max(0, attn_bias.size(-1) - s_k) + attn_bias = attn_bias[:, :, :, _s_k:] + if prefix_mask is not None and attention_mask.shape != prefix_mask.shape: + raise ValueError( + f"attention_mask shape={attention_mask.shape} " + + f"and prefix_mask shape={prefix_mask.shape} are not equal." + ) + min_val = torch.finfo(attn_bias.dtype).min + attn_bias = attn_bias.masked_fill( + ~attention_mask.view(-1, 1, 1, s_k), min_val + ) + return (attn_bias, None) + + def _apply_prefix_mask(self, attn_bias: torch.Tensor, prefix_mask: torch.Tensor): + (s_k, s_q) = attn_bias.shape[-2:] + if s_k != self.config.max_seq_len or s_q != self.config.max_seq_len: + raise ValueError( + "attn_bias does not match the expected shape. " + + f"The last two dimensions should both be {self.config.max_length} " + + f"but are {s_k} and {s_q}." + ) + seq_len = prefix_mask.shape[-1] + if seq_len > self.config.max_seq_len: + raise ValueError( + f"prefix_mask sequence length cannot exceed max_seq_len={self.config.max_seq_len}" + ) + attn_bias = attn_bias[..., :seq_len, :seq_len] + causal = torch.tril( + torch.ones((seq_len, seq_len), dtype=torch.bool, device=prefix_mask.device) + ).view(1, 1, seq_len, seq_len) + prefix = prefix_mask.view(-1, 1, 1, seq_len) + cannot_attend = ~torch.logical_or(causal, prefix.bool()) + min_val = torch.finfo(attn_bias.dtype).min + attn_bias = attn_bias.masked_fill(cannot_attend, min_val) + return attn_bias + + def _apply_sequence_id( + self, attn_bias: torch.Tensor, sequence_id: torch.LongTensor + ): + seq_len = sequence_id.shape[-1] + if seq_len > self.config.max_seq_len: + raise ValueError( + f"sequence_id sequence length cannot exceed max_seq_len={self.config.max_seq_len}" + ) + attn_bias = attn_bias[..., :seq_len, :seq_len] + cannot_attend = torch.logical_not( + torch.eq(sequence_id.view(-1, seq_len, 1), sequence_id.view(-1, 1, seq_len)) + ).unsqueeze(1) + min_val = torch.finfo(attn_bias.dtype).min + attn_bias = attn_bias.masked_fill(cannot_attend, min_val) + return attn_bias + + def forward( + self, + input_ids: torch.LongTensor, + past_key_values: Optional[List[Tuple[torch.FloatTensor]]] = None, + attention_mask: Optional[torch.ByteTensor] = None, + prefix_mask: Optional[torch.ByteTensor] = None, + sequence_id: Optional[torch.LongTensor] = None, + return_dict: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + use_cache: Optional[bool] = None, + ): + return_dict = ( + return_dict if return_dict is not None else self.config.return_dict + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + if attention_mask is not None: + attention_mask = attention_mask.bool() + if prefix_mask is not None: + prefix_mask = prefix_mask.bool() + if not return_dict: + raise NotImplementedError( + "return_dict False is not implemented yet for MPT" + ) + if output_attentions: + if self.attn_impl != "torch": + raise NotImplementedError( + "output_attentions is not implemented for MPT when using attn_impl `flash` or `triton`." + ) + if ( + attention_mask is not None + and attention_mask[:, 0].sum() != attention_mask.shape[0] + and self.training + ): + raise NotImplementedError( + "MPT does not support training with left padding." + ) + if self.prefix_lm and prefix_mask is None: + raise ValueError( + "prefix_mask is a required argument when MPT is configured with prefix_lm=True." + ) + if self.training: + if self.attn_uses_sequence_id and sequence_id is None: + raise ValueError( + "sequence_id is a required argument when MPT is configured with attn_uses_sequence_id=True " + + "and the model is in train mode." + ) + elif self.attn_uses_sequence_id is False and sequence_id is not None: + warnings.warn( + "MPT received non-None input for `sequence_id` but is configured with attn_uses_sequence_id=False. " + + "This input will be ignored. If you want the model to use `sequence_id`, set attn_uses_sequence_id to True." + ) + S = input_ids.size(1) + assert ( + S <= self.config.max_seq_len + ), f"Cannot forward input with seq_len={S}, this model only supports seq_len<={self.config.max_seq_len}" + tok_emb = self.wte(input_ids) + if self.alibi: + x = tok_emb + else: + past_position = 0 + if past_key_values is not None: + if len(past_key_values) != self.config.n_layers: + raise ValueError( + f"past_key_values must provide a past_key_value for each attention " + + f"layer in the network (len(past_key_values)={len(past_key_values)!r}; self.config.n_layers={self.config.n_layers!r})." + ) + past_position = past_key_values[0][0].size(1) + if self.attn_impl == "torch": + past_position = past_key_values[0][0].size(3) + if S + past_position > self.config.max_seq_len: + raise ValueError( + f"Cannot forward input with past sequence length {past_position} and current sequence length {S + 1}, this model only supports total sequence length <= {self.config.max_seq_len}." + ) + pos = torch.arange( + past_position, + S + past_position, + dtype=torch.long, + device=input_ids.device, + ).unsqueeze(0) + if attention_mask is not None: + pos = torch.clamp( + pos + - torch.cumsum((~attention_mask).to(torch.int32), dim=1)[ + :, past_position: + ], + min=0, + ) + pos_emb = self.wpe(pos) + x = tok_emb + pos_emb + (attn_bias, attention_mask) = self._attn_bias( + device=x.device, + dtype=torch.float32, + attention_mask=attention_mask, + prefix_mask=prefix_mask, + sequence_id=sequence_id, + ) + if use_cache and past_key_values is None: + past_key_values = [() for _ in range(self.config.n_layers)] + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + for b_idx, block in enumerate(self.blocks): + if output_hidden_states: + assert all_hidden_states is not None + all_hidden_states = all_hidden_states + (x,) + past_key_value = ( + past_key_values[b_idx] if past_key_values is not None else None + ) + (x, attn_weights, past_key_value) = block( + x, + past_key_value=past_key_value, + attn_bias=attn_bias, + attention_mask=attention_mask, + is_causal=self.is_causal, + ) + if past_key_values is not None: + past_key_values[b_idx] = past_key_value + if output_attentions: + assert all_self_attns is not None + all_self_attns = all_self_attns + (attn_weights,) + x = self.norm_f(x) + if output_hidden_states: + assert all_hidden_states is not None + all_hidden_states = all_hidden_states + (x,) + return BaseModelOutputWithPast( + last_hidden_state=x, + past_key_values=past_key_values, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + +class MPTForCausalLM(MPTPreTrainedModel): + def __init__(self, config, weights): + super().__init__(config) + if not config.tie_word_embeddings: + raise ValueError("MPTForCausalLM only supports tied word embeddings") + self.transformer = MPTModel(config, weights) + self.lm_head = TensorParallelHead.load( + config, prefix="transformer.wte", weights=weights + ) + self.logit_scale = None + if config.logit_scale is not None: + logit_scale = config.logit_scale + if isinstance(logit_scale, str): + if logit_scale == "inv_sqrt_d_model": + logit_scale = 1 / math.sqrt(config.d_model) + else: + raise ValueError( + f"logit_scale={logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'." + ) + self.logit_scale = logit_scale + + def forward( + self, + input_ids: torch.LongTensor, + past_key_values: Optional[List[Tuple[torch.FloatTensor]]] = None, + attention_mask: Optional[torch.ByteTensor] = None, + prefix_mask: Optional[torch.ByteTensor] = None, + sequence_id: Optional[torch.LongTensor] = None, + labels: Optional[torch.LongTensor] = None, + return_dict: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + use_cache: Optional[bool] = None, + ): + return_dict = ( + return_dict if return_dict is not None else self.config.return_dict + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + outputs = self.transformer( + input_ids=input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + prefix_mask=prefix_mask, + sequence_id=sequence_id, + return_dict=return_dict, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + use_cache=use_cache, + ) + logits = self.lm_head(outputs.last_hidden_state) + if self.logit_scale is not None: + if self.logit_scale == 0: + warnings.warn( + f"Multiplying logits by self.logit_scale={self.logit_scale!r}. This will produce uniform (uninformative) outputs." + ) + logits *= self.logit_scale + loss = None + if labels is not None: + labels = torch.roll(labels, shifts=-1) + labels[:, -1] = -100 + loss = F.cross_entropy( + logits.view(-1, logits.size(-1)), labels.to(logits.device).view(-1) + ) + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + def prepare_inputs_for_generation( + self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs + ): + if inputs_embeds is not None: + raise NotImplementedError("inputs_embeds is not implemented for MPT yet") + attention_mask = kwargs["attention_mask"].bool() + if attention_mask[:, -1].sum() != attention_mask.shape[0]: + raise NotImplementedError( + "MPT does not support generation with right padding." + ) + if self.transformer.attn_uses_sequence_id and self.training: + sequence_id = torch.zeros_like(input_ids[:1]) + else: + sequence_id = None + if past_key_values is not None: + input_ids = input_ids[:, -1].unsqueeze(-1) + if self.transformer.prefix_lm: + prefix_mask = torch.ones_like(attention_mask) + if kwargs.get("use_cache") == False: + raise NotImplementedError( + "MPT with prefix_lm=True does not support use_cache=False." + ) + else: + prefix_mask = None + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "prefix_mask": prefix_mask, + "sequence_id": sequence_id, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache", True), + } + + @staticmethod + def _reorder_cache(past_key_values, beam_idx): + """Used by HuggingFace generate when using beam search with kv-caching. + + See https://github.com/huggingface/transformers/blob/3ec7a47664ebe40c40f4b722f6bb1cd30c3821ec/src/transformers/models/gpt2/modeling_gpt2.py#L1122-L1133 + for an example in transformers. + """ + reordered_past = [] + for layer_past in past_key_values: + reordered_past += [ + tuple( + (past_state.index_select(0, beam_idx) for past_state in layer_past) + ) + ] + return reordered_past diff --git a/server/text_generation_server/models/mpt.py b/server/text_generation_server/models/mpt.py new file mode 100644 index 00000000..87fc1f07 --- /dev/null +++ b/server/text_generation_server/models/mpt.py @@ -0,0 +1,90 @@ +import torch +import torch.distributed + +from typing import Optional, Type +from opentelemetry import trace +from transformers import AutoTokenizer, PretrainedConfig, PreTrainedTokenizerBase +from huggingface_hub import hf_hub_download +import json + +from text_generation_server.models import CausalLM +from text_generation_server.models.causal_lm import CausalLMBatch +from text_generation_server.pb import generate_pb2 +from text_generation_server.models.custom_modeling.mpt_modeling import ( + MPTForCausalLM, +) +from text_generation_server.utils import ( + initialize_torch_distributed, + weight_files, + Weights, +) + +tracer = trace.get_tracer(__name__) + + +class MPTCausalLMBatch(CausalLMBatch): + @classmethod + def from_pb( + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, + device: torch.device, + ) -> "CausalLMBatch": + batch = super().from_pb(pb=pb, tokenizer=tokenizer, dtype=dtype, device=device) + batch.keys_head_dim_last = False + return batch + + +class MPTSharded(CausalLM): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + trust_remote_code: bool = False, + ): + self.process_group, rank, world_size = initialize_torch_distributed() + if torch.cuda.is_available(): + device = torch.device(f"cuda:{rank}") + dtype = torch.float16 + else: + raise NotImplementedError("MPTSharded is only available on GPU") + + tokenizer = AutoTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + tokenizer.pad_token = tokenizer.eos_token + + filename = hf_hub_download(model_id, revision=revision, filename="config.json") + with open(filename, "r") as f: + config = json.load(f) + config = PretrainedConfig(**config) + config.quantize = quantize + + torch.distributed.barrier(group=self.process_group) + + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights(filenames, device, dtype, process_group=self.process_group) + + config.quantize = quantize + model = MPTForCausalLM(config, weights) + + torch.distributed.barrier(group=self.process_group) + super(CausalLM, self).__init__( + model=model, + tokenizer=tokenizer, + requires_padding=False, + dtype=dtype, + device=device, + rank=rank, + world_size=world_size, + ) + + @property + def batch_type(self) -> Type[CausalLMBatch]: + return MPTCausalLMBatch diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index a2b0c739..cbdfea66 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -31,7 +31,19 @@ def load_layer_norm(cls, prefix, weights, eps): return ln +@classmethod +def load_layer_norm_no_bias(cls, prefix, weights, eps): + weight = weights.get_tensor(f"{prefix}.weight") + with init_empty_weights(): + ln = cls(weight.shape, eps=eps) + + ln.weight = nn.Parameter(weight) + ln.bias = None + return ln + + torch.nn.LayerNorm.load = load_layer_norm +torch.nn.LayerNorm.load_no_bias = load_layer_norm_no_bias class FastLinear(nn.Module): From 8405581fcd9f5a3c81696c080232eb944f98ccf2 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Tue, 4 Jul 2023 00:39:25 -0700 Subject: [PATCH 296/793] fix: Update server/Makefile to include Makefile-vllm (#520) # What does this PR do? For consistency and ease of use (you can just run `make` to install vllm without any extra steps). Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/server/Makefile b/server/Makefile index 17020c97..d0086928 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,4 +1,5 @@ include Makefile-flash-att +include Makefile-vllm unit-tests: pytest -s -vv -m "not private" tests From e6888d0e87c21bf392b62938c75c07ec92d5fcb8 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 4 Jul 2023 18:35:37 +0200 Subject: [PATCH 297/793] docs(benchmarker): Adding some help for the options in `text-generation-benchmark`. (#462) --- benchmark/src/main.rs | 53 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/benchmark/src/main.rs b/benchmark/src/main.rs index 6172d377..a7550060 100644 --- a/benchmark/src/main.rs +++ b/benchmark/src/main.rs @@ -14,36 +14,85 @@ use tracing_subscriber::EnvFilter; #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] struct Args { + /// The name of the tokenizer (as in model_id on the huggingface hub, or local path). #[clap(short, long, env)] tokenizer_name: String, + + /// The revision to use for the tokenizer if on the hub. #[clap(default_value = "main", long, env)] revision: String, + + /// The various batch sizes to benchmark for, the idea is to get enough + /// batching to start seeing increased latency, this usually means you're + /// moving from memory bound (usual as BS=1) to compute bound, and this is + /// a sweet spot for the maximum batch size for the model under test #[clap(short, long)] batch_size: Option>, + + /// This is the initial prompt sent to the text-generation-server length + /// in token. Longer prompt will slow down the benchmark. Usually the + /// latency grows somewhat linearly with this for the prefill step. + /// + /// Most importantly, the prefill step is usually not the one dominating + /// your runtime, so it's ok to keep it short. #[clap(default_value = "10", short, long, env)] sequence_length: u32, + + /// This is how many tokens will be generated by the server and averaged out + /// to give the `decode` latency. This is the *critical* number you want to optimize for + /// LLM spend most of their time doing decoding. + /// + /// Decode latency is usually quite stable. #[clap(default_value = "8", short, long, env)] decode_length: u32, + + ///How many runs should we average from #[clap(default_value = "10", short, long, env)] runs: usize, + + /// Number of warmup cycles #[clap(default_value = "1", short, long, env)] warmups: usize, + + /// The location of the grpc socket. This benchmark tool bypasses the router + /// completely and directly talks to the gRPC processes + #[clap(default_value = "/tmp/text-generation-server-0", short, long, env)] + master_shard_uds_path: String, + + /// Generation parameter in case you want to specifically test/debug particular + /// decoding strategies, for full doc refer to the `text-generation-server` #[clap(long, env)] temperature: Option, + + /// Generation parameter in case you want to specifically test/debug particular + /// decoding strategies, for full doc refer to the `text-generation-server` #[clap(long, env)] top_k: Option, + + /// Generation parameter in case you want to specifically test/debug particular + /// decoding strategies, for full doc refer to the `text-generation-server` #[clap(long, env)] top_p: Option, + + /// Generation parameter in case you want to specifically test/debug particular + /// decoding strategies, for full doc refer to the `text-generation-server` #[clap(long, env)] typical_p: Option, + + /// Generation parameter in case you want to specifically test/debug particular + /// decoding strategies, for full doc refer to the `text-generation-server` #[clap(long, env)] repetition_penalty: Option, + + /// Generation parameter in case you want to specifically test/debug particular + /// decoding strategies, for full doc refer to the `text-generation-server` #[clap(long, env)] watermark: bool, + + /// Generation parameter in case you want to specifically test/debug particular + /// decoding strategies, for full doc refer to the `text-generation-server` #[clap(long, env)] do_sample: bool, - #[clap(default_value = "/tmp/text-generation-server-0", short, long, env)] - master_shard_uds_path: String, } fn main() -> Result<(), Box> { From 2a101207d44b903c1cc9b4d968a4b24150413942 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Tue, 4 Jul 2023 09:37:25 -0700 Subject: [PATCH 298/793] fix(server): Handle loading from local files for MPT (#534) This PR allows the MPT model to be loaded from local files. Without this change, an exception will be thrown by `hf_hub_download` function if `model_id` is a local path. --- server/text_generation_server/models/mpt.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/mpt.py b/server/text_generation_server/models/mpt.py index 87fc1f07..3c0f8167 100644 --- a/server/text_generation_server/models/mpt.py +++ b/server/text_generation_server/models/mpt.py @@ -1,6 +1,7 @@ import torch import torch.distributed +from pathlib import Path from typing import Optional, Type from opentelemetry import trace from transformers import AutoTokenizer, PretrainedConfig, PreTrainedTokenizerBase @@ -60,7 +61,12 @@ def __init__( ) tokenizer.pad_token = tokenizer.eos_token - filename = hf_hub_download(model_id, revision=revision, filename="config.json") + # If model_id is a local path, load the file directly + local_path = Path(model_id, "config.json") + if local_path.exists(): + filename = str(local_path.resolve()) + else: + filename = hf_hub_download(model_id, revision=revision, filename="config.json") with open(filename, "r") as f: config = json.load(f) config = PretrainedConfig(**config) From e4b26aa10bd43c93cd236a9e3388692eb1e8a321 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 4 Jul 2023 11:11:33 -0700 Subject: [PATCH 299/793] fix(server): avoid errors for very small top_p values (#544) See https://github.com/huggingface/transformers/pull/24111 I didn't add validation to the `__init__` method since it's not done for other values/warpers. --- server/text_generation_server/utils/logits_process.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/text_generation_server/utils/logits_process.py b/server/text_generation_server/utils/logits_process.py index 0cbbf8b0..f424eae4 100644 --- a/server/text_generation_server/utils/logits_process.py +++ b/server/text_generation_server/utils/logits_process.py @@ -189,9 +189,8 @@ def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tenso # Remove tokens with cumulative top_p above the threshold (token with 0 are kept) sorted_indices_to_remove = probs <= self.top_p_opposite - if self.min_tokens_to_keep > 1: - # Keep at least min_tokens_to_keep - sorted_indices_to_remove[..., -self.min_tokens_to_keep :] = 0 + # Keep at least min_tokens_to_keep + sorted_indices_to_remove[..., -self.min_tokens_to_keep :] = 0 # scatter sorted tensors to original indexing indices_to_remove = sorted_indices_to_remove.scatter( From 31e2253ae721ea80032283b9e85ffe51945e5a55 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 4 Jul 2023 20:23:55 +0200 Subject: [PATCH 300/793] feat(server): use latest flash attention commit (#543) @njhill FYI --- server/Makefile-flash-att | 4 +- server/poetry.lock | 1571 +++++++++-------- server/pyproject.toml | 2 +- server/requirements.txt | 19 +- .../custom_modeling/flash_llama_modeling.py | 29 +- .../custom_modeling/flash_neox_modeling.py | 32 +- .../custom_modeling/flash_rw_modeling.py | 49 +- .../flash_santacoder_modeling.py | 29 +- .../models/flash_causal_lm.py | 52 +- server/text_generation_server/models/mpt.py | 4 +- server/text_generation_server/utils/layers.py | 2 +- 11 files changed, 898 insertions(+), 895 deletions(-) diff --git a/server/Makefile-flash-att b/server/Makefile-flash-att index 0e67a9e4..bc1d37ef 100644 --- a/server/Makefile-flash-att +++ b/server/Makefile-flash-att @@ -1,9 +1,9 @@ -flash_att_commit := 06ece1a1525ebcf4e183ac76b1e5108d2872f57f +flash_att_commit := 3a9bfd076f98746c73362328958dbc68d145fbec flash-attention: # Clone flash attention pip install packaging - git clone https://github.com/OlivierDehaene/flash-attention.git + git clone https://github.com/HazyResearch/flash-attention.git build-flash-attention: flash-attention cd flash-attention && git fetch && git checkout $(flash_att_commit) diff --git a/server/poetry.lock b/server/poetry.lock index 7c126e80..7d00f223 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,15 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. - [[package]] name = "accelerate" version = "0.19.0" description = "Accelerate" +category = "main" optional = true python-versions = ">=3.7.0" -files = [ - {file = "accelerate-0.19.0-py3-none-any.whl", hash = "sha256:2866b0bf9fff08f51e6384c95fa96725838b70f1988d1cce42e56b820d8a91dd"}, - {file = "accelerate-0.19.0.tar.gz", hash = "sha256:84920226b9e642e453ef37593ee55b956b08d8200dea4087c546c34e26157e76"}, -] [package.dependencies] numpy = ">=1.17" @@ -23,138 +18,50 @@ dev = ["black (>=23.1,<24.0)", "datasets", "deepspeed", "evaluate", "hf-doc-buil quality = ["black (>=23.1,<24.0)", "hf-doc-builder (>=0.3.0)", "ruff (>=0.0.241)", "urllib3 (<2.0.0)"] rich = ["rich"] sagemaker = ["sagemaker"] -test-dev = ["datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] -test-prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] -test-trackers = ["comet-ml", "tensorboard", "wandb"] +test_dev = ["datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] +test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] +test_trackers = ["comet-ml", "tensorboard", "wandb"] testing = ["datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] [[package]] name = "backoff" version = "2.2.1" description = "Function decoration for backoff and retry" +category = "main" optional = false python-versions = ">=3.7,<4.0" -files = [ - {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, - {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, -] [[package]] name = "bitsandbytes" version = "0.38.1" description = "8-bit optimizers and matrix multiplication routines." +category = "main" optional = true python-versions = "*" -files = [ - {file = "bitsandbytes-0.38.1-py3-none-any.whl", hash = "sha256:5f532e7b1353eb7049ae831da2eb62ed8a1e0444116bd51b9e088a6e0bc7a34a"}, - {file = "bitsandbytes-0.38.1.tar.gz", hash = "sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce"}, -] [[package]] name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, -] [[package]] name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" -files = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, -] [[package]] name = "click" version = "8.1.3" description = "Composable command line interface toolkit" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -163,23 +70,17 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] [[package]] -name = "deprecated" +name = "Deprecated" version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, - {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, -] [package.dependencies] wrapt = ">=1.10,<2" @@ -191,23 +92,17 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] name = "einops" version = "0.6.1" description = "A new flavour of deep learning operations" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "einops-0.6.1-py3-none-any.whl", hash = "sha256:99149e46cc808956b174932fe563d920db4d6e5dadb8c6ecdaa7483b7ef7cfc3"}, - {file = "einops-0.6.1.tar.gz", hash = "sha256:f95f8d00f4ded90dbc4b19b6f98b177332614b0357dde66997f3ae5d474dc8c8"}, -] [[package]] name = "exceptiongroup" -version = "1.1.1" +version = "1.1.2" description = "Backport of PEP 654 (exception groups)" +category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, - {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, -] [package.extras] test = ["pytest (>=6)"] @@ -216,12 +111,9 @@ test = ["pytest (>=6)"] name = "filelock" version = "3.12.2" description = "A platform independent file lock." +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, - {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, -] [package.extras] docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] @@ -231,12 +123,9 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "fsspec" version = "2023.6.0" description = "File-system specification" +category = "main" optional = false python-versions = ">=3.8" -files = [ - {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"}, - {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"}, -] [package.extras] abfs = ["adlfs"] @@ -266,12 +155,9 @@ tqdm = ["tqdm"] name = "googleapis-common-protos" version = "1.59.1" description = "Common protobufs used in Google APIs" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "googleapis-common-protos-1.59.1.tar.gz", hash = "sha256:b35d530fe825fb4227857bc47ad84c33c809ac96f312e13182bdeaa2abe1178a"}, - {file = "googleapis_common_protos-1.59.1-py2.py3-none-any.whl", hash = "sha256:0cbedb6fb68f1c07e18eb4c48256320777707e7d0c55063ae56c15db3224a61e"}, -] [package.dependencies] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" @@ -283,12 +169,9 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] name = "grpc-interceptor" version = "0.15.2" description = "Simplifies gRPC interceptors" +category = "main" optional = false python-versions = ">=3.7,<4.0" -files = [ - {file = "grpc-interceptor-0.15.2.tar.gz", hash = "sha256:5c984110af4fb77d03472ec0468f9c77ddaf798e190410fb7b7f1e76c60c96a4"}, - {file = "grpc_interceptor-0.15.2-py3-none-any.whl", hash = "sha256:596dac3cb709ffb6178a4873f5148e254c871c9069f0b11040189b257969490a"}, -] [package.dependencies] grpcio = ">=1.49.1,<2.0.0" @@ -298,148 +181,50 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.54.2" +version = "1.56.0" description = "HTTP/2-based RPC framework" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "grpcio-1.54.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:40e1cbf69d6741b40f750f3cccc64326f927ac6145a9914d33879e586002350c"}, - {file = "grpcio-1.54.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2288d76e4d4aa7ef3fe7a73c1c470b66ea68e7969930e746a8cd8eca6ef2a2ea"}, - {file = "grpcio-1.54.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:c0e3155fc5335ec7b3b70f15230234e529ca3607b20a562b6c75fb1b1218874c"}, - {file = "grpcio-1.54.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bf88004fe086c786dc56ef8dd6cb49c026833fdd6f42cb853008bce3f907148"}, - {file = "grpcio-1.54.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2be88c081e33f20630ac3343d8ad9f1125f32987968e9c8c75c051c9800896e8"}, - {file = "grpcio-1.54.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:33d40954199bddbb6a78f8f6f2b2082660f381cd2583ec860a6c2fa7c8400c08"}, - {file = "grpcio-1.54.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b52d00d1793d290c81ad6a27058f5224a7d5f527867e5b580742e1bd211afeee"}, - {file = "grpcio-1.54.2-cp310-cp310-win32.whl", hash = "sha256:881d058c5ccbea7cc2c92085a11947b572498a27ef37d3eef4887f499054dca8"}, - {file = "grpcio-1.54.2-cp310-cp310-win_amd64.whl", hash = "sha256:0212e2f7fdf7592e4b9d365087da30cb4d71e16a6f213120c89b4f8fb35a3ab3"}, - {file = "grpcio-1.54.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:1e623e0cf99a0ac114f091b3083a1848dbc64b0b99e181473b5a4a68d4f6f821"}, - {file = "grpcio-1.54.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:66233ccd2a9371158d96e05d082043d47dadb18cbb294dc5accfdafc2e6b02a7"}, - {file = "grpcio-1.54.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:4cb283f630624ebb16c834e5ac3d7880831b07cbe76cb08ab7a271eeaeb8943e"}, - {file = "grpcio-1.54.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a1e601ee31ef30a9e2c601d0867e236ac54c922d32ed9f727b70dd5d82600d5"}, - {file = "grpcio-1.54.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8da84bbc61a4e92af54dc96344f328e5822d574f767e9b08e1602bb5ddc254a"}, - {file = "grpcio-1.54.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5008964885e8d23313c8e5ea0d44433be9bfd7e24482574e8cc43c02c02fc796"}, - {file = "grpcio-1.54.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a2f5a1f1080ccdc7cbaf1171b2cf384d852496fe81ddedeb882d42b85727f610"}, - {file = "grpcio-1.54.2-cp311-cp311-win32.whl", hash = "sha256:b74ae837368cfffeb3f6b498688a123e6b960951be4dec0e869de77e7fa0439e"}, - {file = "grpcio-1.54.2-cp311-cp311-win_amd64.whl", hash = "sha256:8cdbcbd687e576d48f7886157c95052825ca9948c0ed2afdc0134305067be88b"}, - {file = "grpcio-1.54.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:782f4f8662a2157c4190d0f99eaaebc602899e84fb1e562a944e5025929e351c"}, - {file = "grpcio-1.54.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:714242ad0afa63a2e6dabd522ae22e1d76e07060b5af2ddda5474ba4f14c2c94"}, - {file = "grpcio-1.54.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:f900ed4ad7a0f1f05d35f955e0943944d5a75f607a836958c6b8ab2a81730ef2"}, - {file = "grpcio-1.54.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96a41817d2c763b1d0b32675abeb9179aa2371c72aefdf74b2d2b99a1b92417b"}, - {file = "grpcio-1.54.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70fcac7b94f4c904152809a050164650ac81c08e62c27aa9f156ac518029ebbe"}, - {file = "grpcio-1.54.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fd6c6c29717724acf9fc1847c4515d57e4dc12762452457b9cb37461f30a81bb"}, - {file = "grpcio-1.54.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c2392f5b5d84b71d853918687d806c1aa4308109e5ca158a16e16a6be71041eb"}, - {file = "grpcio-1.54.2-cp37-cp37m-win_amd64.whl", hash = "sha256:51630c92591d6d3fe488a7c706bd30a61594d144bac7dee20c8e1ce78294f474"}, - {file = "grpcio-1.54.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:b04202453941a63b36876a7172b45366dc0cde10d5fd7855c0f4a4e673c0357a"}, - {file = "grpcio-1.54.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:89dde0ac72a858a44a2feb8e43dc68c0c66f7857a23f806e81e1b7cc7044c9cf"}, - {file = "grpcio-1.54.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:09d4bfd84686cd36fd11fd45a0732c7628308d094b14d28ea74a81db0bce2ed3"}, - {file = "grpcio-1.54.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7fc2b4edb938c8faa4b3c3ea90ca0dd89b7565a049e8e4e11b77e60e4ed2cc05"}, - {file = "grpcio-1.54.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61f7203e2767800edee7a1e1040aaaf124a35ce0c7fe0883965c6b762defe598"}, - {file = "grpcio-1.54.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e416c8baf925b5a1aff31f7f5aecc0060b25d50cce3a5a7255dc5cf2f1d4e5eb"}, - {file = "grpcio-1.54.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dc80c9c6b608bf98066a038e0172013a49cfa9a08d53335aefefda2c64fc68f4"}, - {file = "grpcio-1.54.2-cp38-cp38-win32.whl", hash = "sha256:8d6192c37a30a115f4663592861f50e130caed33efc4eec24d92ec881c92d771"}, - {file = "grpcio-1.54.2-cp38-cp38-win_amd64.whl", hash = "sha256:46a057329938b08e5f0e12ea3d7aed3ecb20a0c34c4a324ef34e00cecdb88a12"}, - {file = "grpcio-1.54.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:2296356b5c9605b73ed6a52660b538787094dae13786ba53080595d52df13a98"}, - {file = "grpcio-1.54.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:c72956972e4b508dd39fdc7646637a791a9665b478e768ffa5f4fe42123d5de1"}, - {file = "grpcio-1.54.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:9bdbb7624d65dc0ed2ed8e954e79ab1724526f09b1efa88dcd9a1815bf28be5f"}, - {file = "grpcio-1.54.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c44e1a765b31e175c391f22e8fc73b2a2ece0e5e6ff042743d8109b5d2eff9f"}, - {file = "grpcio-1.54.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cc928cfe6c360c1df636cf7991ab96f059666ac7b40b75a769410cc6217df9c"}, - {file = "grpcio-1.54.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a08920fa1a97d4b8ee5db2f31195de4a9def1a91bc003544eb3c9e6b8977960a"}, - {file = "grpcio-1.54.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4864f99aac207e3e45c5e26c6cbb0ad82917869abc2f156283be86c05286485c"}, - {file = "grpcio-1.54.2-cp39-cp39-win32.whl", hash = "sha256:b38b3de8cff5bc70f8f9c615f51b48eff7313fc9aca354f09f81b73036e7ddfa"}, - {file = "grpcio-1.54.2-cp39-cp39-win_amd64.whl", hash = "sha256:be48496b0e00460717225e7680de57c38be1d8629dc09dadcd1b3389d70d942b"}, - {file = "grpcio-1.54.2.tar.gz", hash = "sha256:50a9f075eeda5097aa9a182bb3877fe1272875e45370368ac0ee16ab9e22d019"}, -] [package.extras] -protobuf = ["grpcio-tools (>=1.54.2)"] +protobuf = ["grpcio-tools (>=1.56.0)"] [[package]] name = "grpcio-reflection" -version = "1.54.2" +version = "1.56.0" description = "Standard Protobuf Reflection Service for gRPC" +category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "grpcio-reflection-1.54.2.tar.gz", hash = "sha256:b2e021e1ce4f075615411edfbbd6fdcc485ba474dd6e5a3f559690582959a673"}, - {file = "grpcio_reflection-1.54.2-py3-none-any.whl", hash = "sha256:e7759addebbd90768f3a0278320278145758c4687d9e2cd7d76e7cbd0e329274"}, -] [package.dependencies] -grpcio = ">=1.54.2" +grpcio = ">=1.56.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.54.2" +version = "1.56.0" description = "Status proto mapping for gRPC" +category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "grpcio-status-1.54.2.tar.gz", hash = "sha256:3255cbec5b7c706caa3d4dd584606c080e6415e15631bb2f6215e2b70055836d"}, - {file = "grpcio_status-1.54.2-py3-none-any.whl", hash = "sha256:2a7cb4838225f1b53bd0448a3008c5b5837941e1f3a0b13fa38768f08a7b68c2"}, -] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.54.2" +grpcio = ">=1.56.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.54.2" +version = "1.56.0" description = "Protobuf code generator for gRPC" +category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "grpcio-tools-1.54.2.tar.gz", hash = "sha256:e11c2c2aee53f340992e8e4d6a59172cbbbd0193f1351de98c4f810a5041d5ca"}, - {file = "grpcio_tools-1.54.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:2b96f5f17d3156058be247fd25b062b4768138665694c00b056659618b8fb418"}, - {file = "grpcio_tools-1.54.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:11939c9a8a39bd4815c7e88cb2fee48e1948775b59dbb06de8fcae5991e84f9e"}, - {file = "grpcio_tools-1.54.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:129de5579f95d6a55dde185f188b4cbe19d1e2f1471425431d9930c31d300d70"}, - {file = "grpcio_tools-1.54.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4128c01cd6f5ea8f7c2db405dbfd8582cd967d36e6fa0952565436633b0e591"}, - {file = "grpcio_tools-1.54.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5c7292dd899ad8fa09a2be96719648cee37b17909fe8c12007e3bff58ebee61"}, - {file = "grpcio_tools-1.54.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5ef30c2dbc63c1e0a462423ca4f95001814d26ef4fe66208e53fcf220ea3b717"}, - {file = "grpcio_tools-1.54.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4abfc1892380abe6cef381eab86f9350cbd703bfe5d834095aa66fd91c886b6d"}, - {file = "grpcio_tools-1.54.2-cp310-cp310-win32.whl", hash = "sha256:9acf443dcf6f68fbea3b7fb519e1716e014db1a561939f5aecc4abda74e4015d"}, - {file = "grpcio_tools-1.54.2-cp310-cp310-win_amd64.whl", hash = "sha256:21b9d2dee80f3f77e4097252e7f0db89772335a7300b72ab3d2e5c280872b1db"}, - {file = "grpcio_tools-1.54.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:7b24fbab9e7598518ce4549e066df00aab79c2bf9bedcdde23fb5ef6a3cf532f"}, - {file = "grpcio_tools-1.54.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:7baa210c20f71a242d9ae0e02734628f6948e8bee3bf538647894af427d28800"}, - {file = "grpcio_tools-1.54.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:e3d0e5188ff8dbaddac2ee44731d36f09c4eccd3eac7328e547862c44f75cacd"}, - {file = "grpcio_tools-1.54.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27671c68c7e0e3c5ff9967f5500799f65a04e7b153b8ce10243c87c43199039d"}, - {file = "grpcio_tools-1.54.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f39d8e8806b8857fb473ca6a9c7bd800b0673dfdb7283ff569af0345a222f32c"}, - {file = "grpcio_tools-1.54.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8e4c5a48f7b2e8798ce381498ee7b9a83c65b87ae66ee5022387394e5eb51771"}, - {file = "grpcio_tools-1.54.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4f285f8ef3de422717a36bd372239ae778b8cc112ce780ca3c7fe266dadc49fb"}, - {file = "grpcio_tools-1.54.2-cp311-cp311-win32.whl", hash = "sha256:0f952c8a5c47e9204fe8959f7e9add149e660f6579d67cf65024c32736d34caf"}, - {file = "grpcio_tools-1.54.2-cp311-cp311-win_amd64.whl", hash = "sha256:3237149beec39e897fd62cef4aa1e1cd9422d7a95661d24bd0a79200b167e730"}, - {file = "grpcio_tools-1.54.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:0ab1b323905d449298523db5d34fa5bf5fffd645bd872b25598e2f8a01f0ea39"}, - {file = "grpcio_tools-1.54.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:7d7e6e8d62967b3f037f952620cb7381cc39a4bd31790c75fcfba56cc975d70b"}, - {file = "grpcio_tools-1.54.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:7f4624ef2e76a3a5313c4e61a81be38bcc16b59a68a85d30758b84cd2102b161"}, - {file = "grpcio_tools-1.54.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e543f457935ba7b763b121f1bf893974393b4d30065042f947f85a8d81081b80"}, - {file = "grpcio_tools-1.54.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0239b929eb8b3b30b2397eef3b9abb245087754d77c3721e3be43c44796de87d"}, - {file = "grpcio_tools-1.54.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:0de05c7698c655e9a240dc34ae91d6017b93143ac89e5b20046d7ca3bd09c27c"}, - {file = "grpcio_tools-1.54.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a3ce0b98fb581c471424d2cda45120f57658ed97677c6fec4d6decf5d7c1b976"}, - {file = "grpcio_tools-1.54.2-cp37-cp37m-win_amd64.whl", hash = "sha256:37393ef90674964175923afe3859fc5a208e1ece565f642b4f76a8c0224a0993"}, - {file = "grpcio_tools-1.54.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:8e4531267736d88fde1022b36dd42ed8163e3575bcbd12bfed96662872aa93fe"}, - {file = "grpcio_tools-1.54.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:a0b7049814442f918b522d66b1d015286afbeb9e6d141af54bbfafe31710a3c8"}, - {file = "grpcio_tools-1.54.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:b80585e06c4f0082327eb5c9ad96fbdb2b0e7c14971ea5099fe78c22f4608451"}, - {file = "grpcio_tools-1.54.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39fd530cfdf58dc05125775cc233b05554d553d27478f14ae5fd8a6306f0cb28"}, - {file = "grpcio_tools-1.54.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bb9ec4aea0f2b3006fb002fa59e5c10f92b48fc374619fbffd14d2b0e388c3e"}, - {file = "grpcio_tools-1.54.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d512de051342a576bb89777476d13c5266d9334cf4badb6468aed9dc8f5bdec1"}, - {file = "grpcio_tools-1.54.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1b8ee3099c51ce987fa8a08e6b93fc342b10228415dd96b5c0caa0387f636a6f"}, - {file = "grpcio_tools-1.54.2-cp38-cp38-win32.whl", hash = "sha256:6037f123905dc0141f7c8383ca616ef0195e79cd3b4d82faaee789d4045e891b"}, - {file = "grpcio_tools-1.54.2-cp38-cp38-win_amd64.whl", hash = "sha256:10dd41862f579d185c60f629b5ee89103e216f63b576079d258d974d980bad87"}, - {file = "grpcio_tools-1.54.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:f6787d07fdab31a32c433c1ba34883dea6559d8a3fbe08fb93d834ca34136b71"}, - {file = "grpcio_tools-1.54.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:21b1467e31e44429d2a78b50135c9cdbd4b8f6d3b5cd548bc98985d3bdc352d0"}, - {file = "grpcio_tools-1.54.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:30a49b8b168aced2a4ff40959e6c4383ad6cfd7a20839a47a215e9837eb722dc"}, - {file = "grpcio_tools-1.54.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8742122782953d2fd038f0a199f047a24e941cc9718b1aac90876dbdb7167739"}, - {file = "grpcio_tools-1.54.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:503ef1351c62fb1d6747eaf74932b609d8fdd4345b3591ef910adef8fa9969d0"}, - {file = "grpcio_tools-1.54.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:72d15de4c4b6a764a76c4ae69d99c35f7a0751223688c3f7e62dfa95eb4f61be"}, - {file = "grpcio_tools-1.54.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:df079479fb1b9e488334312e35ebbf30cbf5ecad6c56599f1a961800b33ab7c1"}, - {file = "grpcio_tools-1.54.2-cp39-cp39-win32.whl", hash = "sha256:49c2846dcc4803476e839d8bd4db8845e928f19130e0ea86121f2d1f43d2b452"}, - {file = "grpcio_tools-1.54.2-cp39-cp39-win_amd64.whl", hash = "sha256:b82ca472db9c914c44e39a41e9e8bd3ed724523dd7aff5ce37592b8d16920ed9"}, -] [package.dependencies] -grpcio = ">=1.54.2" +grpcio = ">=1.56.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -447,40 +232,17 @@ setuptools = "*" name = "hf-transfer" version = "0.1.3" description = "" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "hf_transfer-0.1.3-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9"}, - {file = "hf_transfer-0.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07"}, - {file = "hf_transfer-0.1.3-cp310-none-win_amd64.whl", hash = "sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6"}, - {file = "hf_transfer-0.1.3-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d53954ecfaadc84c15481bf5d4c7282323196b4b6df1d1be54208d4fdedfb407"}, - {file = "hf_transfer-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334862f4a82f8a09d6d3f550e67d7e498bb8882e678b7725638254fed3276801"}, - {file = "hf_transfer-0.1.3-cp311-none-win_amd64.whl", hash = "sha256:da92a1483a66cf2baa96de133c75e7d5d9a60e4a0e60d228f26c573c73a1feb6"}, - {file = "hf_transfer-0.1.3-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:617692a70cf299576d82cfc860923f29ec5c834a3f5242bc0853d4f106670398"}, - {file = "hf_transfer-0.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca218fb6efc5c22379d9e64989e84bff426fcf5664fdbbf7cd70aa8b79497652"}, - {file = "hf_transfer-0.1.3-cp37-none-win_amd64.whl", hash = "sha256:6e5201b648df6106c232fcdb507db734081fd6220dfb1c432bd27c6fa9453331"}, - {file = "hf_transfer-0.1.3-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:445edfcd9d59c9d2541957177a9c9225b1f0e8855f6311fb16e20f67c3426421"}, - {file = "hf_transfer-0.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c1bdfa554f0b0936c1623b99058c6998a00fdcd86f75d9203f3f66572d2e30c"}, - {file = "hf_transfer-0.1.3-cp38-none-win_amd64.whl", hash = "sha256:606f2fe436e5be73f07987a56cd97c695805413d29203ae39ebd9fc596405435"}, - {file = "hf_transfer-0.1.3-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9913f5ad2644a1f57c1b7755a7d959ca5e0189863bb0473817d0707af230bf6a"}, - {file = "hf_transfer-0.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d935946791e338f748e05a23df877d74fbcd39dc7b537f0aa2e5a5841cf7dde8"}, - {file = "hf_transfer-0.1.3-cp39-none-win_amd64.whl", hash = "sha256:79099ac043423b263a2843a24213418f309d5c8bc458776622bffe012ebced73"}, - {file = "hf_transfer-0.1.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ce6c5965a57d94db5e043aa488a4df929a32000db125d9c9a1d325e8c7006dc"}, - {file = "hf_transfer-0.1.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a7934c8e491bb395731f677f66dd5f6641432f338a3a9efc9f0b6c186d37cf8"}, - {file = "hf_transfer-0.1.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"}, - {file = "hf_transfer-0.1.3.tar.gz", hash = "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c"}, -] [[package]] name = "huggingface-hub" version = "0.14.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +category = "main" optional = false python-versions = ">=3.7.0" -files = [ - {file = "huggingface_hub-0.14.1-py3-none-any.whl", hash = "sha256:9fc619170d800ff3793ad37c9757c255c8783051e1b5b00501205eb43ccc4f27"}, - {file = "huggingface_hub-0.14.1.tar.gz", hash = "sha256:9ab899af8e10922eac65e290d60ab956882ab0bf643e3d990b1394b6b47b7fbc"}, -] [package.dependencies] filelock = "*" @@ -506,34 +268,25 @@ typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "t name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" -files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] [[package]] name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] [[package]] -name = "jinja2" +name = "Jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] [package.dependencies] MarkupSafe = ">=2.0" @@ -545,12 +298,9 @@ i18n = ["Babel (>=2.7)"] name = "loguru" version = "0.6.0" description = "Python logging made (stupidly) simple" +category = "main" optional = false python-versions = ">=3.5" -files = [ - {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, - {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, -] [package.dependencies] colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} @@ -560,74 +310,20 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] [[package]] -name = "markupsafe" +name = "MarkupSafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, - {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, -] [[package]] name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" +category = "main" optional = true python-versions = "*" -files = [ - {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, - {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, -] [package.extras] develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] @@ -639,12 +335,9 @@ tests = ["pytest (>=4.6)"] name = "networkx" version = "3.1" description = "Python package for creating and manipulating graphs and networks" +category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, - {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, -] [package.extras] default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] @@ -655,51 +348,19 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "numpy" -version = "1.24.3" +version = "1.25.0" description = "Fundamental package for array computing in Python" +category = "main" optional = false -python-versions = ">=3.8" -files = [ - {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, - {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, - {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, - {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, - {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, - {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, - {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, - {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, - {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, - {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, - {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, -] +python-versions = ">=3.9" [[package]] name = "opentelemetry-api" version = "1.15.0" description = "OpenTelemetry Python API" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "opentelemetry_api-1.15.0-py3-none-any.whl", hash = "sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0"}, - {file = "opentelemetry_api-1.15.0.tar.gz", hash = "sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded"}, -] [package.dependencies] deprecated = ">=1.2.6" @@ -709,12 +370,9 @@ setuptools = ">=16.0" name = "opentelemetry-exporter-otlp" version = "1.15.0" description = "OpenTelemetry Collector Exporters" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "opentelemetry_exporter_otlp-1.15.0-py3-none-any.whl", hash = "sha256:79f22748b6a54808a0448093dfa189c8490e729f67c134d4c992533d9393b33e"}, - {file = "opentelemetry_exporter_otlp-1.15.0.tar.gz", hash = "sha256:4f7c49751d9720e2e726e13b0bb958ccade4e29122c305d92c033da432c8d2c5"}, -] [package.dependencies] opentelemetry-exporter-otlp-proto-grpc = "1.15.0" @@ -724,12 +382,9 @@ opentelemetry-exporter-otlp-proto-http = "1.15.0" name = "opentelemetry-exporter-otlp-proto-grpc" version = "1.15.0" description = "OpenTelemetry Collector Protobuf over gRPC Exporter" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0-py3-none-any.whl", hash = "sha256:c2a5492ba7d140109968135d641d06ce3c5bd73c50665f787526065d57d7fd1d"}, - {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0.tar.gz", hash = "sha256:844f2a4bb9bcda34e4eb6fe36765e5031aacb36dc60ed88c90fc246942ea26e7"}, -] [package.dependencies] backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} @@ -746,12 +401,9 @@ test = ["pytest-grpc"] name = "opentelemetry-exporter-otlp-proto-http" version = "1.15.0" description = "OpenTelemetry Collector Protobuf over HTTP Exporter" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "opentelemetry_exporter_otlp_proto_http-1.15.0-py3-none-any.whl", hash = "sha256:3ec2a02196c8a54bf5cbf7fe623a5238625638e83b6047a983bdf96e2bbb74c0"}, - {file = "opentelemetry_exporter_otlp_proto_http-1.15.0.tar.gz", hash = "sha256:11b2c814249a49b22f6cca7a06b05701f561d577b747f3660dfd67b6eb9daf9c"}, -] [package.dependencies] backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} @@ -768,12 +420,9 @@ test = ["responses (==0.22.0)"] name = "opentelemetry-instrumentation" version = "0.36b0" description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "opentelemetry_instrumentation-0.36b0-py3-none-any.whl", hash = "sha256:83ba4ae7d5292b5b33e0f851cc5c76d8f91196b9b3527800fc13855c33383ac2"}, - {file = "opentelemetry_instrumentation-0.36b0.tar.gz", hash = "sha256:e3ddac9b3b93408ef26c8ecbf38f717042977e16381bb4cd329a5b4cf16998cf"}, -] [package.dependencies] opentelemetry-api = ">=1.4,<2.0" @@ -784,12 +433,9 @@ wrapt = ">=1.0.0,<2.0.0" name = "opentelemetry-instrumentation-grpc" version = "0.36b0" description = "OpenTelemetry gRPC instrumentation" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "opentelemetry_instrumentation_grpc-0.36b0-py3-none-any.whl", hash = "sha256:eaa246ed2083c97b13bab2555cb9d170e8433230a31476c4cab8a17fa03380a4"}, - {file = "opentelemetry_instrumentation_grpc-0.36b0.tar.gz", hash = "sha256:dc89447c9eb6ea868970f6c13b4ffdac182cdd5a41dd215a0f5393ca6375be55"}, -] [package.dependencies] opentelemetry-api = ">=1.12,<2.0" @@ -806,12 +452,9 @@ test = ["opentelemetry-instrumentation-grpc[instruments]", "opentelemetry-sdk (> name = "opentelemetry-proto" version = "1.15.0" description = "OpenTelemetry Python Proto" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "opentelemetry_proto-1.15.0-py3-none-any.whl", hash = "sha256:044b6d044b4d10530f250856f933442b8753a17f94ae37c207607f733fb9a844"}, - {file = "opentelemetry_proto-1.15.0.tar.gz", hash = "sha256:9c4008e40ac8cab359daac283fbe7002c5c29c77ea2674ad5626a249e64e0101"}, -] [package.dependencies] protobuf = ">=3.19,<5.0" @@ -820,12 +463,9 @@ protobuf = ">=3.19,<5.0" name = "opentelemetry-sdk" version = "1.15.0" description = "OpenTelemetry Python SDK" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "opentelemetry_sdk-1.15.0-py3-none-any.whl", hash = "sha256:555c533e9837766119bbccc7a80458c9971d853a6f1da683a2246cd5e53b4645"}, - {file = "opentelemetry_sdk-1.15.0.tar.gz", hash = "sha256:98dbffcfeebcbff12c0c974292d6ea603180a145904cf838b1fe4d5c99078425"}, -] [package.dependencies] opentelemetry-api = "1.15.0" @@ -837,34 +477,25 @@ typing-extensions = ">=3.7.4" name = "opentelemetry-semantic-conventions" version = "0.36b0" description = "OpenTelemetry Semantic Conventions" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "opentelemetry_semantic_conventions-0.36b0-py3-none-any.whl", hash = "sha256:adc05635e87b9d3e007c9f530eed487fc3ef2177d02f82f674f28ebf9aff8243"}, - {file = "opentelemetry_semantic_conventions-0.36b0.tar.gz", hash = "sha256:829dc221795467d98b773c04096e29be038d77526dc8d6ac76f546fb6279bf01"}, -] [[package]] name = "packaging" version = "23.1" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, -] [[package]] name = "pluggy" -version = "1.0.0" +version = "1.2.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false -python-versions = ">=3.6" -files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] +python-versions = ">=3.7" [package.extras] dev = ["pre-commit", "tox"] @@ -872,81 +503,755 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.23.2" +version = "4.23.3" description = "" +category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "protobuf-4.23.2-cp310-abi3-win32.whl", hash = "sha256:384dd44cb4c43f2ccddd3645389a23ae61aeb8cfa15ca3a0f60e7c3ea09b28b3"}, - {file = "protobuf-4.23.2-cp310-abi3-win_amd64.whl", hash = "sha256:09310bce43353b46d73ba7e3bca78273b9bc50349509b9698e64d288c6372c2a"}, - {file = "protobuf-4.23.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2cfab63a230b39ae603834718db74ac11e52bccaaf19bf20f5cce1a84cf76df"}, - {file = "protobuf-4.23.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:c52cfcbfba8eb791255edd675c1fe6056f723bf832fa67f0442218f8817c076e"}, - {file = "protobuf-4.23.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:86df87016d290143c7ce3be3ad52d055714ebaebb57cc659c387e76cfacd81aa"}, - {file = "protobuf-4.23.2-cp37-cp37m-win32.whl", hash = "sha256:281342ea5eb631c86697e1e048cb7e73b8a4e85f3299a128c116f05f5c668f8f"}, - {file = "protobuf-4.23.2-cp37-cp37m-win_amd64.whl", hash = "sha256:ce744938406de1e64b91410f473736e815f28c3b71201302612a68bf01517fea"}, - {file = "protobuf-4.23.2-cp38-cp38-win32.whl", hash = "sha256:6c081863c379bb1741be8f8193e893511312b1d7329b4a75445d1ea9955be69e"}, - {file = "protobuf-4.23.2-cp38-cp38-win_amd64.whl", hash = "sha256:25e3370eda26469b58b602e29dff069cfaae8eaa0ef4550039cc5ef8dc004511"}, - {file = "protobuf-4.23.2-cp39-cp39-win32.whl", hash = "sha256:efabbbbac1ab519a514579ba9ec52f006c28ae19d97915951f69fa70da2c9e91"}, - {file = "protobuf-4.23.2-cp39-cp39-win_amd64.whl", hash = "sha256:54a533b971288af3b9926e53850c7eb186886c0c84e61daa8444385a4720297f"}, - {file = "protobuf-4.23.2-py3-none-any.whl", hash = "sha256:8da6070310d634c99c0db7df48f10da495cc283fd9e9234877f0cd182d43ab7f"}, - {file = "protobuf-4.23.2.tar.gz", hash = "sha256:20874e7ca4436f683b64ebdbee2129a5a2c301579a67d1a7dda2cdf62fb7f5f7"}, -] [[package]] name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, - {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, - {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, - {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, - {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, - {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, - {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, - {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, - {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, - {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "pytest" +version = "7.4.0" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "PyYAML" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "regex" +version = "2023.6.3" +description = "Alternative regular expression module, to replace re." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "safetensors" +version = "0.3.1" +description = "Fast and Safe Tensor serialization" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] +dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] +numpy = ["numpy (>=1.21.6)"] +paddlepaddle = ["paddlepaddle (>=2.4.1)"] +quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] +tensorflow = ["tensorflow (>=2.11.0)"] +testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] +torch = ["torch (>=1.10)"] + +[[package]] +name = "sentencepiece" +version = "0.1.99" +description = "SentencePiece python wrapper" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "setuptools" +version = "68.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "sympy" +version = "1.12" +description = "Computer algebra system (CAS) in Python" +category = "main" +optional = true +python-versions = ">=3.8" + +[package.dependencies] +mpmath = ">=0.19" + +[[package]] +name = "tokenizers" +version = "0.13.3" +description = "Fast and Customizable Tokenizers" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] +docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "torch" +version = "2.0.1" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +category = "main" +optional = true +python-versions = ">=3.8.0" + +[package.dependencies] +filelock = "*" +jinja2 = "*" +networkx = "*" +sympy = "*" +typing-extensions = "*" + +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] + +[[package]] +name = "tqdm" +version = "4.65.0" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "transformers" +version = "4.29.2" +description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +category = "main" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +filelock = "*" +huggingface-hub = ">=0.14.1,<1.0" +numpy = ">=1.17" +packaging = ">=20.0" +pyyaml = ">=5.1" +regex = "!=2019.12.17" +requests = "*" +tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" +tqdm = ">=4.27" + +[package.extras] +accelerate = ["accelerate (>=0.19.0)"] +agents = ["Pillow", "accelerate (>=0.19.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] +all = ["Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +audio = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] +codecarbon = ["codecarbon (==1.2.0)"] +deepspeed = ["accelerate (>=0.19.0)", "deepspeed (>=0.8.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.19.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.8.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.6.9)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "numba (<0.57.0)", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.19.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "numba (<0.57.0)", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +docs_specific = ["hf-doc-builder"] +fairscale = ["fairscale (>0.3)"] +flax = ["flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "optax (>=0.0.8,<=0.1.4)"] +flax-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] +ftfy = ["ftfy"] +integrations = ["optuna", "ray[tune]", "sigopt"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] +modelcreation = ["cookiecutter (==1.7.3)"] +natten = ["natten (>=0.14.6)"] +onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] +onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] +optuna = ["optuna"] +quality = ["GitPython (<3.1.19)", "black (>=23.1,<24.0)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (>=0.0.241,<=0.0.259)", "urllib3 (<2.0.0)"] +ray = ["ray[tune]"] +retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] +sagemaker = ["sagemaker (>=2.31.0)"] +sentencepiece = ["protobuf (<=3.20.2)", "sentencepiece (>=0.1.91,!=0.1.92)"] +serving = ["fastapi", "pydantic", "starlette", "uvicorn"] +sigopt = ["sigopt"] +sklearn = ["scikit-learn"] +speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] +tf-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] +timm = ["timm"] +tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] +torch = ["accelerate (>=0.19.0)", "torch (>=1.9,!=1.12.0)"] +torch-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +torch-vision = ["Pillow", "torchvision"] +torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf (<=3.20.2)", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] +video = ["av (==9.2.0)", "decord (==0.6.0)"] +vision = ["Pillow"] + +[[package]] +name = "typer" +version = "0.6.1" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +click = ">=7.1.1,<9.0.0" + +[package.extras] +all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] + +[[package]] +name = "typing-extensions" +version = "4.7.1" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "urllib3" +version = "2.0.3" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + +[[package]] +name = "wrapt" +version = "1.15.0" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[extras] +accelerate = ["accelerate"] +bnb = ["bitsandbytes"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.9" +content-hash = "65afc4bfa07da4b1427d269fa745939da3851eaede9a8478f5a4bf5949d32cc9" + +[metadata.files] +accelerate = [ + {file = "accelerate-0.19.0-py3-none-any.whl", hash = "sha256:2866b0bf9fff08f51e6384c95fa96725838b70f1988d1cce42e56b820d8a91dd"}, + {file = "accelerate-0.19.0.tar.gz", hash = "sha256:84920226b9e642e453ef37593ee55b956b08d8200dea4087c546c34e26157e76"}, +] +backoff = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] +bitsandbytes = [ + {file = "bitsandbytes-0.38.1-py3-none-any.whl", hash = "sha256:5f532e7b1353eb7049ae831da2eb62ed8a1e0444116bd51b9e088a6e0bc7a34a"}, + {file = "bitsandbytes-0.38.1.tar.gz", hash = "sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce"}, +] +certifi = [ + {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, + {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, +] +charset-normalizer = [ + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] +colorama = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +Deprecated = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] +einops = [ + {file = "einops-0.6.1-py3-none-any.whl", hash = "sha256:99149e46cc808956b174932fe563d920db4d6e5dadb8c6ecdaa7483b7ef7cfc3"}, + {file = "einops-0.6.1.tar.gz", hash = "sha256:f95f8d00f4ded90dbc4b19b6f98b177332614b0357dde66997f3ae5d474dc8c8"}, +] +exceptiongroup = [ + {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, + {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, +] +filelock = [ + {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, + {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, +] +fsspec = [ + {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"}, + {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"}, +] +googleapis-common-protos = [ + {file = "googleapis-common-protos-1.59.1.tar.gz", hash = "sha256:b35d530fe825fb4227857bc47ad84c33c809ac96f312e13182bdeaa2abe1178a"}, + {file = "googleapis_common_protos-1.59.1-py2.py3-none-any.whl", hash = "sha256:0cbedb6fb68f1c07e18eb4c48256320777707e7d0c55063ae56c15db3224a61e"}, +] +grpc-interceptor = [ + {file = "grpc-interceptor-0.15.2.tar.gz", hash = "sha256:5c984110af4fb77d03472ec0468f9c77ddaf798e190410fb7b7f1e76c60c96a4"}, + {file = "grpc_interceptor-0.15.2-py3-none-any.whl", hash = "sha256:596dac3cb709ffb6178a4873f5148e254c871c9069f0b11040189b257969490a"}, +] +grpcio = [ + {file = "grpcio-1.56.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:fb34ace11419f1ae321c36ccaa18d81cd3f20728cd191250be42949d6845bb2d"}, + {file = "grpcio-1.56.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:008767c0aed4899e657b50f2e0beacbabccab51359eba547f860e7c55f2be6ba"}, + {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:17f47aeb9be0da5337f9ff33ebb8795899021e6c0741ee68bd69774a7804ca86"}, + {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43c50d810cc26349b093bf2cfe86756ab3e9aba3e7e681d360930c1268e1399a"}, + {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:187b8f71bad7d41eea15e0c9812aaa2b87adfb343895fffb704fb040ca731863"}, + {file = "grpcio-1.56.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:881575f240eb5db72ddca4dc5602898c29bc082e0d94599bf20588fb7d1ee6a0"}, + {file = "grpcio-1.56.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c243b158dd7585021d16c50498c4b2ec0a64a6119967440c5ff2d8c89e72330e"}, + {file = "grpcio-1.56.0-cp310-cp310-win32.whl", hash = "sha256:8b3b2c7b5feef90bc9a5fa1c7f97637e55ec3e76460c6d16c3013952ee479cd9"}, + {file = "grpcio-1.56.0-cp310-cp310-win_amd64.whl", hash = "sha256:03a80451530fd3b8b155e0c4480434f6be669daf7ecba56f73ef98f94222ee01"}, + {file = "grpcio-1.56.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:64bd3abcf9fb4a9fa4ede8d0d34686314a7075f62a1502217b227991d9ca4245"}, + {file = "grpcio-1.56.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:fdc3a895791af4addbb826808d4c9c35917c59bb5c430d729f44224e51c92d61"}, + {file = "grpcio-1.56.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:4f84a6fd4482e5fe73b297d4874b62a535bc75dc6aec8e9fe0dc88106cd40397"}, + {file = "grpcio-1.56.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14e70b4dda3183abea94c72d41d5930c333b21f8561c1904a372d80370592ef3"}, + {file = "grpcio-1.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b5ce42a5ebe3e04796246ba50357f1813c44a6efe17a37f8dc7a5c470377312"}, + {file = "grpcio-1.56.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8219f17baf069fe8e42bd8ca0b312b875595e43a70cabf397be4fda488e2f27d"}, + {file = "grpcio-1.56.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:defdd14b518e6e468466f799aaa69db0355bca8d3a5ea75fb912d28ba6f8af31"}, + {file = "grpcio-1.56.0-cp311-cp311-win32.whl", hash = "sha256:50f4daa698835accbbcc60e61e0bc29636c0156ddcafb3891c987e533a0031ba"}, + {file = "grpcio-1.56.0-cp311-cp311-win_amd64.whl", hash = "sha256:59c4e606993a47146fbeaf304b9e78c447f5b9ee5641cae013028c4cca784617"}, + {file = "grpcio-1.56.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:b1f4b6f25a87d80b28dd6d02e87d63fe1577fe6d04a60a17454e3f8077a38279"}, + {file = "grpcio-1.56.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:c2148170e01d464d41011a878088444c13413264418b557f0bdcd1bf1b674a0e"}, + {file = "grpcio-1.56.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:0409de787ebbf08c9d2bca2bcc7762c1efe72eada164af78b50567a8dfc7253c"}, + {file = "grpcio-1.56.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66f0369d27f4c105cd21059d635860bb2ea81bd593061c45fb64875103f40e4a"}, + {file = "grpcio-1.56.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38fdf5bd0a1c754ce6bf9311a3c2c7ebe56e88b8763593316b69e0e9a56af1de"}, + {file = "grpcio-1.56.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:79d4c5911d12a7aa671e5eb40cbb50a830396525014d2d6f254ea2ba180ce637"}, + {file = "grpcio-1.56.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5d2fc471668a7222e213f86ef76933b18cdda6a51ea1322034478df8c6519959"}, + {file = "grpcio-1.56.0-cp37-cp37m-win_amd64.whl", hash = "sha256:991224fd485e088d3cb5e34366053691a4848a6b7112b8f5625a411305c26691"}, + {file = "grpcio-1.56.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:c6f36621aabecbaff3e70c4d1d924c76c8e6a7ffec60c331893640a4af0a8037"}, + {file = "grpcio-1.56.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:1eadd6de258901929223f422ffed7f8b310c0323324caf59227f9899ea1b1674"}, + {file = "grpcio-1.56.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:72836b5a1d4f508ffbcfe35033d027859cc737972f9dddbe33fb75d687421e2e"}, + {file = "grpcio-1.56.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f92a99ab0c7772fb6859bf2e4f44ad30088d18f7c67b83205297bfb229e0d2cf"}, + {file = "grpcio-1.56.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa08affbf672d051cd3da62303901aeb7042a2c188c03b2c2a2d346fc5e81c14"}, + {file = "grpcio-1.56.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2db108b4c8e29c145e95b0226973a66d73ae3e3e7fae00329294af4e27f1c42"}, + {file = "grpcio-1.56.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8674fdbd28266d8efbcddacf4ec3643f76fe6376f73283fd63a8374c14b0ef7c"}, + {file = "grpcio-1.56.0-cp38-cp38-win32.whl", hash = "sha256:bd55f743e654fb050c665968d7ec2c33f03578a4bbb163cfce38024775ff54cc"}, + {file = "grpcio-1.56.0-cp38-cp38-win_amd64.whl", hash = "sha256:c63bc5ac6c7e646c296fed9139097ae0f0e63f36f0864d7ce431cce61fe0118a"}, + {file = "grpcio-1.56.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:c0bc9dda550785d23f4f025be614b7faa8d0293e10811f0f8536cf50435b7a30"}, + {file = "grpcio-1.56.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:d596408bab632ec7b947761e83ce6b3e7632e26b76d64c239ba66b554b7ee286"}, + {file = "grpcio-1.56.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:76b6e6e1ee9bda32e6e933efd61c512e9a9f377d7c580977f090d1a9c78cca44"}, + {file = "grpcio-1.56.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7beb84ebd0a3f732625124b73969d12b7350c5d9d64ddf81ae739bbc63d5b1ed"}, + {file = "grpcio-1.56.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83ec714bbbe9b9502177c842417fde39f7a267031e01fa3cd83f1ca49688f537"}, + {file = "grpcio-1.56.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4feee75565d1b5ab09cb3a5da672b84ca7f6dd80ee07a50f5537207a9af543a4"}, + {file = "grpcio-1.56.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b4638a796778329cc8e142e4f57c705adb286b3ba64e00b0fa91eeb919611be8"}, + {file = "grpcio-1.56.0-cp39-cp39-win32.whl", hash = "sha256:437af5a7673bca89c4bc0a993382200592d104dd7bf55eddcd141cef91f40bab"}, + {file = "grpcio-1.56.0-cp39-cp39-win_amd64.whl", hash = "sha256:4241a1c2c76e748023c834995cd916570e7180ee478969c2d79a60ce007bc837"}, + {file = "grpcio-1.56.0.tar.gz", hash = "sha256:4c08ee21b3d10315b8dc26f6c13917b20ed574cdbed2d2d80c53d5508fdcc0f2"}, +] +grpcio-reflection = [ + {file = "grpcio-reflection-1.56.0.tar.gz", hash = "sha256:d6bad11af658c78170b50825cf3c841f69662f946f8e921c9fa1ae2dc48f56c1"}, + {file = "grpcio_reflection-1.56.0-py3-none-any.whl", hash = "sha256:23572e1cc5674465e54fb5898d7358288e6f6970e2aa21c9eee7ccd674b7388e"}, +] +grpcio-status = [ + {file = "grpcio-status-1.56.0.tar.gz", hash = "sha256:9eca0b2dcda0782d3702df225918efd6d820f75f93cd5c51c7fb6a4ffbfea12c"}, + {file = "grpcio_status-1.56.0-py3-none-any.whl", hash = "sha256:e5f101c96686e9d4e94a114567960fdb00052aa3c818b029745e3db37dc9c613"}, +] +grpcio-tools = [ + {file = "grpcio-tools-1.56.0.tar.gz", hash = "sha256:39f5877cea514b3da9f2683dfb3ffb45ef47b05f4ff39c287d7d61c5057f48b8"}, + {file = "grpcio_tools-1.56.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:cdbae7312e6d132d38ec2c1611b8cafb783e0416cc5c6deae04efde5f16fb190"}, + {file = "grpcio_tools-1.56.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:5f5c416b88d76fbdb548cfee0486928748816b700ece6e591006e5b1dc67598f"}, + {file = "grpcio_tools-1.56.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:23e2ef1dc6a9bf766f091e2c52a68e54d0aff3548f94562e61fb0ac3874d514a"}, + {file = "grpcio_tools-1.56.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8870ab60f8a76b4a7e43184ee03d28112b976d83c43d41cec821f47b3a297da2"}, + {file = "grpcio_tools-1.56.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e59ab6c0bf4a8bb975553ad578d4425bd192775ae384f9406d77d31ad00f6efe"}, + {file = "grpcio_tools-1.56.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b309659534b5d930f9ab6d521670c2dd86cb6ef7f47f37f73f96557e2ec13a49"}, + {file = "grpcio_tools-1.56.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8115b416ea2cad8a87dc3aadfaf26da684e003c3770b12e7219b462505bb5b85"}, + {file = "grpcio_tools-1.56.0-cp310-cp310-win32.whl", hash = "sha256:e4cb62a521efbca4cb1ad50233aa400574b3daaf6eb26707d661a0afe8191d92"}, + {file = "grpcio_tools-1.56.0-cp310-cp310-win_amd64.whl", hash = "sha256:4d59009ed52220eb2d62f5cefa4e58dec930fb92fab27bb390c4cf1d360ac7e1"}, + {file = "grpcio_tools-1.56.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:cd69107705794e815a8b262722c6fea995911cb1dfc1310abf63b476165335d6"}, + {file = "grpcio_tools-1.56.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:2d1ee9e13ce135a6ed451b428ef14af131dc7df2551a5344ff4f8aee2d9fab99"}, + {file = "grpcio_tools-1.56.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:142530b9fdfabe04f0c7e5dacd45b6c419d39704fa439cc0aabf73ea0d8f916d"}, + {file = "grpcio_tools-1.56.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b7a4eb5003a29eecd71707589f93ae7e8fa2e681366a811b3f86695055d8666"}, + {file = "grpcio_tools-1.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa6d9bdd75d3625dae38372b43696e159c10aa98719b4302b1e94f1ff7878d47"}, + {file = "grpcio_tools-1.56.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c43b4fe8c8df4c52d3106bba2cf427f0e46bbebb80e127fbbc3134db0fead7be"}, + {file = "grpcio_tools-1.56.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:168940a4a955b6c65da978dbf62e1c36e3a311bb27f649fd201a228e2583a6d4"}, + {file = "grpcio_tools-1.56.0-cp311-cp311-win32.whl", hash = "sha256:3a4b06169493f9454a7f2516c5d41b566d9734e553bbc505f2a7837f7f4a2df1"}, + {file = "grpcio_tools-1.56.0-cp311-cp311-win_amd64.whl", hash = "sha256:1bd361fcc967c21672ba855fc77ea0e7afa51664033a746df96545f84edc4670"}, + {file = "grpcio_tools-1.56.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:7e6bcb194b81e372411494d8ed69fab89aa3452b7275fce4f7917fbe7b04fb72"}, + {file = "grpcio_tools-1.56.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:02b23a12b91287ebea14b3685735d1d675e77c3cd365ec1771c3e9afbeba1ec6"}, + {file = "grpcio_tools-1.56.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:80d75856f8ec949847386ad2f56a460f21c63bf82ce99ca5b6aa512c0b875fb1"}, + {file = "grpcio_tools-1.56.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9cffff0b4af80285fa49637d69b69d640eb775dc74b23635e4de5faad9e7e744"}, + {file = "grpcio_tools-1.56.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3de6c08b545920a39b31ed13305f946c00b19ac1b13d26119f111b6360f22ccf"}, + {file = "grpcio_tools-1.56.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:128bb13fe9a2681eeb08175f5fbc8e2d8953d7d0dd240e96f9244b9d2547a1aa"}, + {file = "grpcio_tools-1.56.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b57f7f01eafbfe3a293f2efffb675774dbe4074c4627975ec4dc4aa5766801fb"}, + {file = "grpcio_tools-1.56.0-cp37-cp37m-win_amd64.whl", hash = "sha256:282176066fb082ad21c403b84f9d6b440a20482e6f52b83bb2adf54d6fdcae9f"}, + {file = "grpcio_tools-1.56.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:d9b8d1c42854d3433c058795f52b1418b53dd8c1e9811fecb1312202e803a2c5"}, + {file = "grpcio_tools-1.56.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:accf713f51da74b1a18aa4b31df0ab135510704661f735a938081777b79a4c25"}, + {file = "grpcio_tools-1.56.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:ac33fd2d02d24101ea389be8e05b928acb58be56403d4ebc3aecfab473fa4a25"}, + {file = "grpcio_tools-1.56.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4acdc7b957abfd76581717f0ac8e4408e0a85b7d0ac8d2cdf4d964f16926b897"}, + {file = "grpcio_tools-1.56.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79291bfb1fe5f21d99f4839f43d3c5d44c5402c830a24dbb2811d785dd21264b"}, + {file = "grpcio_tools-1.56.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0a8767e4de0f573c678313c5de075ac0e163a192bb135018e45015a22f234387"}, + {file = "grpcio_tools-1.56.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:96fe2f7f5805d88cb7f2e3e3502550b2883dfab0f9efcf3cbd444942cf2ee1da"}, + {file = "grpcio_tools-1.56.0-cp38-cp38-win32.whl", hash = "sha256:21cf32ccffd4f1800b0dcdf58aa1fc7f626795c9da784c3d817c944edcf2d3ae"}, + {file = "grpcio_tools-1.56.0-cp38-cp38-win_amd64.whl", hash = "sha256:f3ab1a9fad636302f7307d143f64a9fbd11bc041652bf53bb016006e9a5ca820"}, + {file = "grpcio_tools-1.56.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:8989d363ac1996238fee61c8f5663f15a8fc362cb1e758c4a686b76cb457cd70"}, + {file = "grpcio_tools-1.56.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:11cdd9cbf0c09c3a761c6f59dfd7128104be7cd393334efe386d4fc3f990ee1a"}, + {file = "grpcio_tools-1.56.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5fd4c005a4afec16578849bc522ddf3298d6d499b3d37bf51314b086c714cdd5"}, + {file = "grpcio_tools-1.56.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f7302acaa07cf4966c926fcd6a60c8d30a697f730c38168bf83e1519b464115b"}, + {file = "grpcio_tools-1.56.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c1c43d185ebf904c3deec23c36ca2ba4e95db999cf00fc8f85eda4551622a26"}, + {file = "grpcio_tools-1.56.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b12bb8c1d408ae40e4c806a3a8ebda2d107310e46696e1da13d0dc3f91fbd19d"}, + {file = "grpcio_tools-1.56.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:781cf09e4d5c9288708f6ec9c3eae64d9d5a0f4c46c7ebe70ebb7ab4f6384789"}, + {file = "grpcio_tools-1.56.0-cp39-cp39-win32.whl", hash = "sha256:c62f07452dee3f1ed23aeaef821797c5e516f79535e97fe6a6b0a0ee8db1cc91"}, + {file = "grpcio_tools-1.56.0-cp39-cp39-win_amd64.whl", hash = "sha256:7f063443870650e55012fdb3a58ff4ce5f4042b81dad6b749333ee8146157511"}, +] +hf-transfer = [ + {file = "hf_transfer-0.1.3-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9"}, + {file = "hf_transfer-0.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07"}, + {file = "hf_transfer-0.1.3-cp310-none-win_amd64.whl", hash = "sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6"}, + {file = "hf_transfer-0.1.3-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d53954ecfaadc84c15481bf5d4c7282323196b4b6df1d1be54208d4fdedfb407"}, + {file = "hf_transfer-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334862f4a82f8a09d6d3f550e67d7e498bb8882e678b7725638254fed3276801"}, + {file = "hf_transfer-0.1.3-cp311-none-win_amd64.whl", hash = "sha256:da92a1483a66cf2baa96de133c75e7d5d9a60e4a0e60d228f26c573c73a1feb6"}, + {file = "hf_transfer-0.1.3-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:617692a70cf299576d82cfc860923f29ec5c834a3f5242bc0853d4f106670398"}, + {file = "hf_transfer-0.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca218fb6efc5c22379d9e64989e84bff426fcf5664fdbbf7cd70aa8b79497652"}, + {file = "hf_transfer-0.1.3-cp37-none-win_amd64.whl", hash = "sha256:6e5201b648df6106c232fcdb507db734081fd6220dfb1c432bd27c6fa9453331"}, + {file = "hf_transfer-0.1.3-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:445edfcd9d59c9d2541957177a9c9225b1f0e8855f6311fb16e20f67c3426421"}, + {file = "hf_transfer-0.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c1bdfa554f0b0936c1623b99058c6998a00fdcd86f75d9203f3f66572d2e30c"}, + {file = "hf_transfer-0.1.3-cp38-none-win_amd64.whl", hash = "sha256:606f2fe436e5be73f07987a56cd97c695805413d29203ae39ebd9fc596405435"}, + {file = "hf_transfer-0.1.3-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9913f5ad2644a1f57c1b7755a7d959ca5e0189863bb0473817d0707af230bf6a"}, + {file = "hf_transfer-0.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d935946791e338f748e05a23df877d74fbcd39dc7b537f0aa2e5a5841cf7dde8"}, + {file = "hf_transfer-0.1.3-cp39-none-win_amd64.whl", hash = "sha256:79099ac043423b263a2843a24213418f309d5c8bc458776622bffe012ebced73"}, + {file = "hf_transfer-0.1.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ce6c5965a57d94db5e043aa488a4df929a32000db125d9c9a1d325e8c7006dc"}, + {file = "hf_transfer-0.1.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a7934c8e491bb395731f677f66dd5f6641432f338a3a9efc9f0b6c186d37cf8"}, + {file = "hf_transfer-0.1.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"}, + {file = "hf_transfer-0.1.3.tar.gz", hash = "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c"}, +] +huggingface-hub = [ + {file = "huggingface_hub-0.14.1-py3-none-any.whl", hash = "sha256:9fc619170d800ff3793ad37c9757c255c8783051e1b5b00501205eb43ccc4f27"}, + {file = "huggingface_hub-0.14.1.tar.gz", hash = "sha256:9ab899af8e10922eac65e290d60ab956882ab0bf643e3d990b1394b6b47b7fbc"}, +] +idna = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] +iniconfig = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] +Jinja2 = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] +loguru = [ + {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, + {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, +] +MarkupSafe = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] +mpmath = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] +networkx = [ + {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, + {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, +] +numpy = [ + {file = "numpy-1.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8aa130c3042052d656751df5e81f6d61edff3e289b5994edcf77f54118a8d9f4"}, + {file = "numpy-1.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e3f2b96e3b63c978bc29daaa3700c028fe3f049ea3031b58aa33fe2a5809d24"}, + {file = "numpy-1.25.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6b267f349a99d3908b56645eebf340cb58f01bd1e773b4eea1a905b3f0e4208"}, + {file = "numpy-1.25.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4aedd08f15d3045a4e9c648f1e04daca2ab1044256959f1f95aafeeb3d794c16"}, + {file = "numpy-1.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6d183b5c58513f74225c376643234c369468e02947b47942eacbb23c1671f25d"}, + {file = "numpy-1.25.0-cp310-cp310-win32.whl", hash = "sha256:d76a84998c51b8b68b40448ddd02bd1081bb33abcdc28beee6cd284fe11036c6"}, + {file = "numpy-1.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:c0dc071017bc00abb7d7201bac06fa80333c6314477b3d10b52b58fa6a6e38f6"}, + {file = "numpy-1.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c69fe5f05eea336b7a740e114dec995e2f927003c30702d896892403df6dbf0"}, + {file = "numpy-1.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c7211d7920b97aeca7b3773a6783492b5b93baba39e7c36054f6e749fc7490c"}, + {file = "numpy-1.25.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecc68f11404930e9c7ecfc937aa423e1e50158317bf67ca91736a9864eae0232"}, + {file = "numpy-1.25.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e559c6afbca484072a98a51b6fa466aae785cfe89b69e8b856c3191bc8872a82"}, + {file = "numpy-1.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6c284907e37f5e04d2412950960894b143a648dea3f79290757eb878b91acbd1"}, + {file = "numpy-1.25.0-cp311-cp311-win32.whl", hash = "sha256:95367ccd88c07af21b379be1725b5322362bb83679d36691f124a16357390153"}, + {file = "numpy-1.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:b76aa836a952059d70a2788a2d98cb2a533ccd46222558b6970348939e55fc24"}, + {file = "numpy-1.25.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b792164e539d99d93e4e5e09ae10f8cbe5466de7d759fc155e075237e0c274e4"}, + {file = "numpy-1.25.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7cd981ccc0afe49b9883f14761bb57c964df71124dcd155b0cba2b591f0d64b9"}, + {file = "numpy-1.25.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5aa48bebfb41f93043a796128854b84407d4df730d3fb6e5dc36402f5cd594c0"}, + {file = "numpy-1.25.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5177310ac2e63d6603f659fadc1e7bab33dd5a8db4e0596df34214eeab0fee3b"}, + {file = "numpy-1.25.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0ac6edfb35d2a99aaf102b509c8e9319c499ebd4978df4971b94419a116d0790"}, + {file = "numpy-1.25.0-cp39-cp39-win32.whl", hash = "sha256:7412125b4f18aeddca2ecd7219ea2d2708f697943e6f624be41aa5f8a9852cc4"}, + {file = "numpy-1.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:26815c6c8498dc49d81faa76d61078c4f9f0859ce7817919021b9eba72b425e3"}, + {file = "numpy-1.25.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5b1b90860bf7d8a8c313b372d4f27343a54f415b20fb69dd601b7efe1029c91e"}, + {file = "numpy-1.25.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85cdae87d8c136fd4da4dad1e48064d700f63e923d5af6c8c782ac0df8044542"}, + {file = "numpy-1.25.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cc3fda2b36482891db1060f00f881c77f9423eead4c3579629940a3e12095fe8"}, + {file = "numpy-1.25.0.tar.gz", hash = "sha256:f1accae9a28dc3cda46a91de86acf69de0d1b5f4edd44a9b0c3ceb8036dfff19"}, +] +opentelemetry-api = [ + {file = "opentelemetry_api-1.15.0-py3-none-any.whl", hash = "sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0"}, + {file = "opentelemetry_api-1.15.0.tar.gz", hash = "sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded"}, +] +opentelemetry-exporter-otlp = [ + {file = "opentelemetry_exporter_otlp-1.15.0-py3-none-any.whl", hash = "sha256:79f22748b6a54808a0448093dfa189c8490e729f67c134d4c992533d9393b33e"}, + {file = "opentelemetry_exporter_otlp-1.15.0.tar.gz", hash = "sha256:4f7c49751d9720e2e726e13b0bb958ccade4e29122c305d92c033da432c8d2c5"}, +] +opentelemetry-exporter-otlp-proto-grpc = [ + {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0-py3-none-any.whl", hash = "sha256:c2a5492ba7d140109968135d641d06ce3c5bd73c50665f787526065d57d7fd1d"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0.tar.gz", hash = "sha256:844f2a4bb9bcda34e4eb6fe36765e5031aacb36dc60ed88c90fc246942ea26e7"}, +] +opentelemetry-exporter-otlp-proto-http = [ + {file = "opentelemetry_exporter_otlp_proto_http-1.15.0-py3-none-any.whl", hash = "sha256:3ec2a02196c8a54bf5cbf7fe623a5238625638e83b6047a983bdf96e2bbb74c0"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.15.0.tar.gz", hash = "sha256:11b2c814249a49b22f6cca7a06b05701f561d577b747f3660dfd67b6eb9daf9c"}, +] +opentelemetry-instrumentation = [ + {file = "opentelemetry_instrumentation-0.36b0-py3-none-any.whl", hash = "sha256:83ba4ae7d5292b5b33e0f851cc5c76d8f91196b9b3527800fc13855c33383ac2"}, + {file = "opentelemetry_instrumentation-0.36b0.tar.gz", hash = "sha256:e3ddac9b3b93408ef26c8ecbf38f717042977e16381bb4cd329a5b4cf16998cf"}, +] +opentelemetry-instrumentation-grpc = [ + {file = "opentelemetry_instrumentation_grpc-0.36b0-py3-none-any.whl", hash = "sha256:eaa246ed2083c97b13bab2555cb9d170e8433230a31476c4cab8a17fa03380a4"}, + {file = "opentelemetry_instrumentation_grpc-0.36b0.tar.gz", hash = "sha256:dc89447c9eb6ea868970f6c13b4ffdac182cdd5a41dd215a0f5393ca6375be55"}, +] +opentelemetry-proto = [ + {file = "opentelemetry_proto-1.15.0-py3-none-any.whl", hash = "sha256:044b6d044b4d10530f250856f933442b8753a17f94ae37c207607f733fb9a844"}, + {file = "opentelemetry_proto-1.15.0.tar.gz", hash = "sha256:9c4008e40ac8cab359daac283fbe7002c5c29c77ea2674ad5626a249e64e0101"}, +] +opentelemetry-sdk = [ + {file = "opentelemetry_sdk-1.15.0-py3-none-any.whl", hash = "sha256:555c533e9837766119bbccc7a80458c9971d853a6f1da683a2246cd5e53b4645"}, + {file = "opentelemetry_sdk-1.15.0.tar.gz", hash = "sha256:98dbffcfeebcbff12c0c974292d6ea603180a145904cf838b1fe4d5c99078425"}, +] +opentelemetry-semantic-conventions = [ + {file = "opentelemetry_semantic_conventions-0.36b0-py3-none-any.whl", hash = "sha256:adc05635e87b9d3e007c9f530eed487fc3ef2177d02f82f674f28ebf9aff8243"}, + {file = "opentelemetry_semantic_conventions-0.36b0.tar.gz", hash = "sha256:829dc221795467d98b773c04096e29be038d77526dc8d6ac76f546fb6279bf01"}, +] +packaging = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] +pluggy = [ + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, +] +protobuf = [ + {file = "protobuf-4.23.3-cp310-abi3-win32.whl", hash = "sha256:514b6bbd54a41ca50c86dd5ad6488afe9505901b3557c5e0f7823a0cf67106fb"}, + {file = "protobuf-4.23.3-cp310-abi3-win_amd64.whl", hash = "sha256:cc14358a8742c4e06b1bfe4be1afbdf5c9f6bd094dff3e14edb78a1513893ff5"}, + {file = "protobuf-4.23.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2991f5e7690dab569f8f81702e6700e7364cc3b5e572725098215d3da5ccc6ac"}, + {file = "protobuf-4.23.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:08fe19d267608d438aa37019236db02b306e33f6b9902c3163838b8e75970223"}, + {file = "protobuf-4.23.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3b01a5274ac920feb75d0b372d901524f7e3ad39c63b1a2d55043f3887afe0c1"}, + {file = "protobuf-4.23.3-cp37-cp37m-win32.whl", hash = "sha256:aca6e86a08c5c5962f55eac9b5bd6fce6ed98645d77e8bfc2b952ecd4a8e4f6a"}, + {file = "protobuf-4.23.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0149053336a466e3e0b040e54d0b615fc71de86da66791c592cc3c8d18150bf8"}, + {file = "protobuf-4.23.3-cp38-cp38-win32.whl", hash = "sha256:84ea0bd90c2fdd70ddd9f3d3fc0197cc24ecec1345856c2b5ba70e4d99815359"}, + {file = "protobuf-4.23.3-cp38-cp38-win_amd64.whl", hash = "sha256:3bcbeb2bf4bb61fe960dd6e005801a23a43578200ea8ceb726d1f6bd0e562ba1"}, + {file = "protobuf-4.23.3-cp39-cp39-win32.whl", hash = "sha256:5cb9e41188737f321f4fce9a4337bf40a5414b8d03227e1d9fbc59bc3a216e35"}, + {file = "protobuf-4.23.3-cp39-cp39-win_amd64.whl", hash = "sha256:29660574cd769f2324a57fb78127cda59327eb6664381ecfe1c69731b83e8288"}, + {file = "protobuf-4.23.3-py3-none-any.whl", hash = "sha256:447b9786ac8e50ae72cae7a2eec5c5df6a9dbf9aa6f908f1b8bda6032644ea62"}, + {file = "protobuf-4.23.3.tar.gz", hash = "sha256:7a92beb30600332a52cdadbedb40d33fd7c8a0d7f549c440347bc606fb3fe34b"}, +] +psutil = [ + {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, + {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, + {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, + {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, ] - -[package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] - -[[package]] -name = "pytest" -version = "7.3.2" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pytest-7.3.2-py3-none-any.whl", hash = "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295"}, - {file = "pytest-7.3.2.tar.gz", hash = "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b"}, +pytest = [ + {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, + {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, ] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -files = [ +PyYAML = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -988,14 +1293,7 @@ files = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] - -[[package]] -name = "regex" -version = "2023.6.3" -description = "Alternative regular expression module, to replace re." -optional = false -python-versions = ">=3.6" -files = [ +regex = [ {file = "regex-2023.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:824bf3ac11001849aec3fa1d69abcb67aac3e150a933963fb12bda5151fe1bfd"}, {file = "regex-2023.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05ed27acdf4465c95826962528f9e8d41dbf9b1aa8531a387dee6ed215a3e9ef"}, {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b49c764f88a79160fa64f9a7b425620e87c9f46095ef9c9920542ab2495c8bc"}, @@ -1085,35 +1383,11 @@ files = [ {file = "regex-2023.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:bdff5eab10e59cf26bc479f565e25ed71a7d041d1ded04ccf9aee1d9f208487a"}, {file = "regex-2023.6.3.tar.gz", hash = "sha256:72d1a25bf36d2050ceb35b517afe13864865268dfb45910e2e17a84be6cbfeb0"}, ] - -[[package]] -name = "requests" -version = "2.31.0" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.7" -files = [ +requests = [ {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, ] - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "safetensors" -version = "0.3.1" -description = "Fast and Safe Tensor serialization" -optional = false -python-versions = "*" -files = [ +safetensors = [ {file = "safetensors-0.3.1-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:2ae9b7dd268b4bae6624729dac86deb82104820e9786429b0583e5168db2f770"}, {file = "safetensors-0.3.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:08c85c1934682f1e2cd904d38433b53cd2a98245a7cc31f5689f9322a2320bbf"}, {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba625c7af9e1c5d0d91cb83d2fba97d29ea69d4db2015d9714d24c7f6d488e15"}, @@ -1155,25 +1429,7 @@ files = [ {file = "safetensors-0.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:5f4f614b8e8161cd8a9ca19c765d176a82b122fa3d3387b77862145bfe9b4e93"}, {file = "safetensors-0.3.1.tar.gz", hash = "sha256:571da56ff8d0bec8ae54923b621cda98d36dcef10feb36fd492c4d0c2cd0e869"}, ] - -[package.extras] -all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] -dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] -jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] -numpy = ["numpy (>=1.21.6)"] -paddlepaddle = ["paddlepaddle (>=2.4.1)"] -quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] -tensorflow = ["tensorflow (>=2.11.0)"] -testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] -torch = ["torch (>=1.10)"] - -[[package]] -name = "sentencepiece" -version = "0.1.99" -description = "SentencePiece python wrapper" -optional = false -python-versions = "*" -files = [ +sentencepiece = [ {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0eb528e70571b7c02723e5804322469b82fe7ea418c96051d0286c0fa028db73"}, {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:77d7fafb2c4e4659cbdf303929503f37a26eabc4ff31d3a79bf1c5a1b338caa7"}, {file = "sentencepiece-0.1.99-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be9cf5b9e404c245aeb3d3723c737ba7a8f5d4ba262ef233a431fa6c45f732a0"}, @@ -1220,44 +1476,15 @@ files = [ {file = "sentencepiece-0.1.99-cp39-cp39-win_amd64.whl", hash = "sha256:350e5c74d739973f1c9643edb80f7cc904dc948578bcb1d43c6f2b173e5d18dd"}, {file = "sentencepiece-0.1.99.tar.gz", hash = "sha256:189c48f5cb2949288f97ccdb97f0473098d9c3dcf5a3d99d4eabe719ec27297f"}, ] - -[[package]] -name = "setuptools" -version = "67.8.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "setuptools-67.8.0-py3-none-any.whl", hash = "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f"}, - {file = "setuptools-67.8.0.tar.gz", hash = "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"}, +setuptools = [ + {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, + {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, ] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - -[[package]] -name = "sympy" -version = "1.12" -description = "Computer algebra system (CAS) in Python" -optional = true -python-versions = ">=3.8" -files = [ +sympy = [ {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, {file = "sympy-1.12.tar.gz", hash = "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8"}, ] - -[package.dependencies] -mpmath = ">=0.19" - -[[package]] -name = "tokenizers" -version = "0.13.3" -description = "Fast and Customizable Tokenizers" -optional = false -python-versions = "*" -files = [ +tokenizers = [ {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, {file = "tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07"}, {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc"}, @@ -1299,30 +1526,11 @@ files = [ {file = "tokenizers-0.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783"}, {file = "tokenizers-0.13.3.tar.gz", hash = "sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e"}, ] - -[package.extras] -dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] -testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ +tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] - -[[package]] -name = "torch" -version = "2.0.1" -description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -optional = true -python-versions = ">=3.8.0" -files = [ +torch = [ {file = "torch-2.0.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:8ced00b3ba471856b993822508f77c98f48a458623596a4c43136158781e306a"}, {file = "torch-2.0.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:359bfaad94d1cda02ab775dc1cc386d585712329bb47b8741607ef6ef4950747"}, {file = "torch-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7c84e44d9002182edd859f3400deaa7410f5ec948a519cc7ef512c2f9b34d2c4"}, @@ -1344,175 +1552,31 @@ files = [ {file = "torch-2.0.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:c62df99352bd6ee5a5a8d1832452110435d178b5164de450831a3a8cc14dc680"}, {file = "torch-2.0.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:671a2565e3f63b8fe8e42ae3e36ad249fe5e567435ea27b94edaa672a7d0c416"}, ] - -[package.dependencies] -filelock = "*" -jinja2 = "*" -networkx = "*" -sympy = "*" -typing-extensions = "*" - -[package.extras] -opt-einsum = ["opt-einsum (>=3.3)"] - -[[package]] -name = "tqdm" -version = "4.65.0" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.7" -files = [ +tqdm = [ {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, ] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] -notebook = ["ipywidgets (>=6)"] -slack = ["slack-sdk"] -telegram = ["requests"] - -[[package]] -name = "transformers" -version = "4.30.2" -description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "transformers-4.30.2-py3-none-any.whl", hash = "sha256:c332e3a3097f9ed89ce556b403251235931c00237b8bc2d7adaa19d226c13f1d"}, - {file = "transformers-4.30.2.tar.gz", hash = "sha256:f4a8aac4e1baffab4033f4a345b0d7dc7957d12a4f1ba969afea08205a513045"}, +transformers = [ + {file = "transformers-4.29.2-py3-none-any.whl", hash = "sha256:0ef158b99bad6f4e6652a0d8655fbbe58b4cb788ce7040f320b5d29c7c810a75"}, + {file = "transformers-4.29.2.tar.gz", hash = "sha256:ed9467661f459f1ce49461d83f18f3b36b6a37f306182dc2ba272935f3b93ebb"}, ] - -[package.dependencies] -filelock = "*" -huggingface-hub = ">=0.14.1,<1.0" -numpy = ">=1.17" -packaging = ">=20.0" -pyyaml = ">=5.1" -regex = "!=2019.12.17" -requests = "*" -safetensors = ">=0.3.1" -tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" -tqdm = ">=4.27" - -[package.extras] -accelerate = ["accelerate (>=0.20.2)"] -agents = ["Pillow", "accelerate (>=0.20.2)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] -all = ["Pillow", "accelerate (>=0.20.2)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.3)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] -audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] -codecarbon = ["codecarbon (==1.2.0)"] -deepspeed = ["accelerate (>=0.20.2)", "deepspeed (>=0.8.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.20.2)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.8.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf (<=3.20.3)", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.20.2)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.6.9)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.3)", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf (<=3.20.3)", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.20.2)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.3)", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -docs = ["Pillow", "accelerate (>=0.20.2)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.3)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] -docs-specific = ["hf-doc-builder"] -fairscale = ["fairscale (>0.3)"] -flax = ["flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "optax (>=0.0.8,<=0.1.4)"] -flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] -ftfy = ["ftfy"] -integrations = ["optuna", "ray[tune]", "sigopt"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] -modelcreation = ["cookiecutter (==1.7.3)"] -natten = ["natten (>=0.14.6)"] -onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] -onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] -optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "black (>=23.1,<24.0)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (>=0.0.241,<=0.0.259)", "urllib3 (<2.0.0)"] -ray = ["ray[tune]"] -retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] -sagemaker = ["sagemaker (>=2.31.0)"] -sentencepiece = ["protobuf (<=3.20.3)", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["fastapi", "pydantic", "starlette", "uvicorn"] -sigopt = ["sigopt"] -sklearn = ["scikit-learn"] -speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf (<=3.20.3)", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] -tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] -tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] -timm = ["timm"] -tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -torch = ["accelerate (>=0.20.2)", "torch (>=1.9,!=1.12.0)"] -torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torch-vision = ["Pillow", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf (<=3.20.3)", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] -video = ["av (==9.2.0)", "decord (==0.6.0)"] -vision = ["Pillow"] - -[[package]] -name = "typer" -version = "0.6.1" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false -python-versions = ">=3.6" -files = [ +typer = [ {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, ] - -[package.dependencies] -click = ">=7.1.1,<9.0.0" - -[package.extras] -all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] -dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] -doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] -test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] - -[[package]] -name = "typing-extensions" -version = "4.6.3" -description = "Backported and Experimental Type Hints for Python 3.7+" -optional = false -python-versions = ">=3.7" -files = [ - {file = "typing_extensions-4.6.3-py3-none-any.whl", hash = "sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26"}, - {file = "typing_extensions-4.6.3.tar.gz", hash = "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5"}, +typing-extensions = [ + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] - -[[package]] -name = "urllib3" -version = "2.0.3" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.7" -files = [ +urllib3 = [ {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, ] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "win32-setctime" -version = "1.1.0" -description = "A small Python utility to set file creation time on Windows" -optional = false -python-versions = ">=3.5" -files = [ +win32-setctime = [ {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, ] - -[package.extras] -dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] - -[[package]] -name = "wrapt" -version = "1.15.0" -description = "Module for decorators, wrappers and monkey patching." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ +wrapt = [ {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, @@ -1589,12 +1653,3 @@ files = [ {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] - -[extras] -accelerate = ["accelerate"] -bnb = ["bitsandbytes"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.9" -content-hash = "3174a211d30bed5990ed5f8418416c951bb6c585153fb51b62809baa89ef07d0" diff --git a/server/pyproject.toml b/server/pyproject.toml index bbf5836d..fd640e7f 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -26,7 +26,7 @@ hf-transfer = "^0.1.2" sentencepiece = "^0.1.97" tokenizers = "0.13.3" huggingface-hub = "^0.14.1" -transformers = "^4.29.2" +transformers = "4.29.2" einops = "^0.6.1" [tool.poetry.extras] diff --git a/server/requirements.txt b/server/requirements.txt index 92693bbd..9b8b2164 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,22 +1,23 @@ backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" +bitsandbytes==0.38.1 ; python_version >= "3.9" and python_version < "4.0" certifi==2023.5.7 ; python_version >= "3.9" and python_version < "4.0" charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0" click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" -colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows") +colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows" deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0" einops==0.6.1 ; python_version >= "3.9" and python_version < "4.0" filelock==3.12.2 ; python_version >= "3.9" and python_version < "4.0" fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "4.0" googleapis-common-protos==1.59.1 ; python_version >= "3.9" and python_version < "4.0" grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "4.0" -grpcio-reflection==1.54.2 ; python_version >= "3.9" and python_version < "4.0" -grpcio-status==1.54.2 ; python_version >= "3.9" and python_version < "4.0" -grpcio==1.54.2 ; python_version >= "3.9" and python_version < "4.0" +grpcio-reflection==1.56.0 ; python_version >= "3.9" and python_version < "4.0" +grpcio-status==1.56.0 ; python_version >= "3.9" and python_version < "4.0" +grpcio==1.56.0 ; python_version >= "3.9" and python_version < "4.0" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" huggingface-hub==0.14.1 ; python_version >= "3.9" and python_version < "4.0" idna==3.4 ; python_version >= "3.9" and python_version < "4.0" loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" -numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0" +numpy==1.25.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "4.0" @@ -27,18 +28,18 @@ opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" -protobuf==4.23.2 ; python_version >= "3.9" and python_version < "4.0" +protobuf==4.23.3 ; python_version >= "3.9" and python_version < "4.0" pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" regex==2023.6.3 ; python_version >= "3.9" and python_version < "4.0" requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0" safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "4.0" -setuptools==67.8.0 ; python_version >= "3.9" and python_version < "4.0" +setuptools==68.0.0 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" -transformers==4.30.2 ; python_version >= "3.9" and python_version < "4.0" +transformers==4.29.2 ; python_version >= "3.9" and python_version < "4.0" typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" -typing-extensions==4.6.3 ; python_version >= "3.9" and python_version < "4.0" +typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "4.0" urllib3==2.0.3 ; python_version >= "3.9" and python_version < "4.0" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 07765e88..d224a838 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -135,8 +135,7 @@ def forward( hidden_states, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -158,17 +157,15 @@ def forward( attn_output = torch.empty_like(qkv[:, 0]) # Prefill - if start_seq_prefill is not None: + if cu_seqlen_prefill is not None: # flash attention flash_attn_cuda.fwd( qkv[:, 0], qkv[:, 1], qkv[:, 2], attn_output, - start_seq_prefill, - end_seq_prefill, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, + cu_seqlen_prefill, max_s, max_s, 0.0, @@ -261,8 +258,7 @@ def forward( residual, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -276,8 +272,7 @@ def forward( normed_hidden_states, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -329,8 +324,7 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq_prefill: Optional[torch.Tensor], - end_seq_prefill: Optional[torch.Tensor], + cu_seqlen_prefill: Optional[torch.Tensor], kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], block_tables: torch.Tensor, slots: torch.Tensor, @@ -352,8 +346,7 @@ def forward( residual, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache[i], block_tables, slots, @@ -381,8 +374,7 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq_prefill: Optional[torch.Tensor], - end_seq_prefill: Optional[torch.Tensor], + cu_seqlen_prefill: Optional[torch.Tensor], kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], block_tables: torch.Tensor, slots: torch.Tensor, @@ -393,8 +385,7 @@ def forward( hidden_states = self.model( input_ids, position_ids, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 9049878a..23c5e4ff 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -123,8 +123,7 @@ def forward( hidden_states, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -146,17 +145,15 @@ def forward( attn_output = torch.empty_like(qkv[:, 0]) # Prefill - if start_seq_prefill is not None: + if cu_seqlen_prefill is not None: # flash attention flash_attn_cuda.fwd( qkv[:, 0], qkv[:, 1], qkv[:, 2], attn_output, - start_seq_prefill, - end_seq_prefill, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, + cu_seqlen_prefill, max_s, max_s, 0.0, @@ -246,8 +243,7 @@ def forward( residual, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -261,8 +257,7 @@ def forward( ln1_hidden_states, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -286,8 +281,7 @@ def forward( hidden_states, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -341,8 +335,7 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq_prefill: Optional[torch.Tensor], - end_seq_prefill: Optional[torch.Tensor], + cu_seqlen_prefill: Optional[torch.Tensor], kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], block_tables: torch.Tensor, slots: torch.Tensor, @@ -364,8 +357,7 @@ def forward( residual, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache[i], block_tables, slots, @@ -391,8 +383,7 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq_prefill: Optional[torch.Tensor], - end_seq_prefill: Optional[torch.Tensor], + cu_seqlen_prefill: Optional[torch.Tensor], kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], block_tables: torch.Tensor, slots: torch.Tensor, @@ -403,8 +394,7 @@ def forward( hidden_states = self.gpt_neox( input_ids, position_ids, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 44aa7cb1..cd42bfc2 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -144,8 +144,7 @@ def forward( hidden_states, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -176,7 +175,7 @@ def forward( attn_output = torch.empty_like(query) # Prefill - if start_seq_prefill is not None: + if cu_seqlen_prefill is not None: if self.num_heads_kv == 1: # Expand to query shape kv = kv.expand(-1, 2, self.num_heads, self.head_size) @@ -187,10 +186,8 @@ def forward( torch.select(kv, dim=1, index=0), torch.select(kv, dim=1, index=1), attn_output, - start_seq_prefill, - end_seq_prefill, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, + cu_seqlen_prefill, max_s, max_s, 0.0, @@ -276,8 +273,7 @@ def forward( hidden_states, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -311,7 +307,7 @@ def forward( attn_output = torch.empty_like(query) # Prefill - if start_seq_prefill is not None: + if cu_seqlen_prefill is not None: # Expand to query shape kv = ( kv.unsqueeze(2) @@ -325,10 +321,8 @@ def forward( torch.select(kv, dim=2, index=0), torch.select(kv, dim=2, index=1), attn_output, - start_seq_prefill, - end_seq_prefill, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, + cu_seqlen_prefill, max_s, max_s, 0.0, @@ -428,8 +422,7 @@ def forward( residual, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -443,8 +436,7 @@ def forward( ln_hidden_states, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -466,8 +458,7 @@ def forward( hidden_states, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -516,8 +507,7 @@ def forward( residual, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -532,8 +522,7 @@ def forward( ln_attn, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -597,8 +586,7 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq_prefill: Optional[torch.Tensor], - end_seq_prefill: Optional[torch.Tensor], + cu_seqlen_prefill: Optional[torch.Tensor], kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], block_tables: torch.Tensor, slots: torch.Tensor, @@ -620,8 +608,7 @@ def forward( residual, cos, sin, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache[i], block_tables, slots, @@ -648,8 +635,7 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq_prefill: Optional[torch.Tensor], - end_seq_prefill: Optional[torch.Tensor], + cu_seqlen_prefill: Optional[torch.Tensor], kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], block_tables: torch.Tensor, slots: torch.Tensor, @@ -660,8 +646,7 @@ def forward( hidden_states = self.transformer( input_ids, position_ids, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 04eedef7..855a6e11 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -232,8 +232,7 @@ def __init__(self, prefix, config, weights): def forward( self, hidden_states, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -259,7 +258,7 @@ def forward( attn_output = torch.empty_like(query) # Prefill - if start_seq_prefill is not None: + if cu_seqlen_prefill is not None: # Expand from 1 to num_heads key_value = key_value.expand(-1, 2, self.num_heads, self.head_size) @@ -269,10 +268,8 @@ def forward( torch.select(key_value, dim=1, index=0), torch.select(key_value, dim=1, index=1), attn_output, - start_seq_prefill, - end_seq_prefill, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, + cu_seqlen_prefill, max_s, max_s, 0.0, @@ -357,8 +354,7 @@ def forward( self, hidden_states, residual, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -369,8 +365,7 @@ def forward( hidden_states = self.attn( hidden_states, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, @@ -423,8 +418,7 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq_prefill: Optional[torch.Tensor], - end_seq_prefill: Optional[torch.Tensor], + cu_seqlen_prefill: Optional[torch.Tensor], kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], block_tables: torch.Tensor, slots: torch.Tensor, @@ -441,8 +435,7 @@ def forward( hidden_states, residual = layer( hidden_states, residual, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache[i], block_tables, slots, @@ -467,8 +460,7 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq_prefill: Optional[torch.Tensor], - end_seq_prefill: Optional[torch.Tensor], + cu_seqlen_prefill: Optional[torch.Tensor], kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], block_tables: torch.Tensor, slots: torch.Tensor, @@ -479,8 +471,7 @@ def forward( hidden_states = self.transformer( input_ids, position_ids, - start_seq_prefill, - end_seq_prefill, + cu_seqlen_prefill, kv_cache, block_tables, slots, diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 94b14f85..bf5f5bbe 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -121,10 +121,10 @@ class FlashCausalLMBatch(Batch): input_ids: torch.Tensor position_ids: torch.Tensor - # tensor of length b holding starting offset of each sequence, only used in prefill - start_seq_prefill: Optional[torch.Tensor] - # tensor of length b holding ending offset of each sequence, only used in prefill - end_seq_prefill: Optional[torch.Tensor] + # Flash Attention values + + # tensor of length b containing the cumulative sequence lengths of the sequences in the batch, only used in prefill + cu_seqlen_prefill: Optional[torch.Tensor] # Paged Attention values @@ -197,8 +197,7 @@ def from_pb( )["input_ids"] position_ids = [] - start_seq_prefill = [] - end_seq_prefill = [] + cu_seqlen_prefill = [0] needed_blocks_slots = [] start_slots = [] slot_indices = [] @@ -250,8 +249,7 @@ def from_pb( position_ids.append(request_position_ids) # Add cumulative lengths of all previous inputs - start_seq_prefill.append(cumulative_length) - end_seq_prefill.append(cumulative_length + input_length) + cu_seqlen_prefill.append(cumulative_length + input_length) next_token_chooser_parameters.append(r.parameters) @@ -329,11 +327,8 @@ def from_pb( position_ids = position_ids[0] slot_indices = slot_indices[0] - start_seq_prefill = torch.tensor( - start_seq_prefill, device=device, dtype=torch.int32 - ) - end_seq_prefill = torch.tensor( - end_seq_prefill, device=device, dtype=torch.int32 + cu_seqlen_prefill = torch.tensor( + cu_seqlen_prefill, device=device, dtype=torch.int32 ) position_ids = position_ids.to(device) @@ -345,9 +340,9 @@ def from_pb( if all_prefill_logprobs: prefill_head_indices = None - prefill_next_token_indices = end_seq_prefill - 1 + prefill_next_token_indices = cu_seqlen_prefill[1:] - 1 elif no_prefill_logprobs: - prefill_head_indices = end_seq_prefill - 1 + prefill_head_indices = cu_seqlen_prefill[1:] - 1 prefill_next_token_indices = None else: prefill_head_indices = torch.tensor( @@ -363,8 +358,7 @@ def from_pb( requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, - start_seq_prefill=start_seq_prefill, - end_seq_prefill=end_seq_prefill, + cu_seqlen_prefill=cu_seqlen_prefill, start_slots=start_slots, slot_indices=slot_indices, needed_blocks_slots=needed_blocks_slots, @@ -504,8 +498,7 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, - start_seq_prefill=None, - end_seq_prefill=None, + cu_seqlen_prefill=None, start_slots=start_slots, slot_indices=slot_indices, needed_blocks_slots=None, @@ -652,8 +645,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch requests_idx_mapping=requests_idx_mapping, input_ids=input_ids, position_ids=position_ids, - start_seq_prefill=None, - end_seq_prefill=None, + cu_seqlen_prefill=None, start_slots=start_slots, slot_indices=slot_indices, needed_blocks_slots=None, @@ -750,8 +742,7 @@ def forward( self, input_ids: torch.Tensor, position_ids: torch.Tensor, - start_seq_prefill: Optional[torch.Tensor], - end_seq_prefill: Optional[torch.Tensor], + cu_seqlen_prefill: Optional[torch.Tensor], block_tables: torch.Tensor, slots: torch.Tensor, input_lengths: torch.Tensor, @@ -764,8 +755,7 @@ def forward( return self.model.forward( input_ids=input_ids, position_ids=position_ids, - start_seq_prefill=start_seq_prefill, - end_seq_prefill=end_seq_prefill, + cu_seqlen_prefill=cu_seqlen_prefill, kv_cache=CACHE_MANAGER.kv_cache, block_tables=block_tables, slots=slots, @@ -778,7 +768,7 @@ def forward( def generate_token( self, batch: FlashCausalLMBatch ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: - prefill = batch.start_seq_prefill is not None + prefill = batch.cu_seqlen_prefill is not None prefill_logprobs = batch.prefill_next_token_indices is not None if batch.needed_blocks_slots: @@ -788,8 +778,7 @@ def generate_token( out = self.forward( batch.input_ids, batch.position_ids, - batch.start_seq_prefill, - batch.end_seq_prefill, + batch.cu_seqlen_prefill, batch.block_tables_tensor, batch.slots[batch.slot_indices], batch.input_lengths_tensor, @@ -815,10 +804,9 @@ def generate_token( prefill_tokens_indices = batch.input_ids.new_zeros(len(out)) next_position_ids = batch.position_ids.new_empty(len(batch)) - batch.slot_indices = batch.slot_indices[batch.end_seq_prefill - 1] - # We do not need start_seq_prefill and end_seq_prefill anymore - batch.start_seq_prefill = None - batch.end_seq_prefill = None + batch.slot_indices = batch.slot_indices[batch.cu_seqlen_prefill[1:] - 1] + # We do not need cu_seqlen_prefill anymore + batch.cu_seqlen_prefill = None else: prefill_logprobs = None next_position_ids = batch.position_ids diff --git a/server/text_generation_server/models/mpt.py b/server/text_generation_server/models/mpt.py index 3c0f8167..a4fe5105 100644 --- a/server/text_generation_server/models/mpt.py +++ b/server/text_generation_server/models/mpt.py @@ -66,7 +66,9 @@ def __init__( if local_path.exists(): filename = str(local_path.resolve()) else: - filename = hf_hub_download(model_id, revision=revision, filename="config.json") + filename = hf_hub_download( + model_id, revision=revision, filename="config.json" + ) with open(filename, "r") as f: config = json.load(f) config = PretrainedConfig(**config) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index cbdfea66..8e0362b8 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -359,7 +359,7 @@ class PositionRotaryEmbedding(nn.Module): def __init__(self, inv_freq): super().__init__() - self.register_buffer("inv_freq", inv_freq) + self.inv_freq = inv_freq self._seq_len_cached = 0 self._cos_cached = None self._sin_cached = None From 6f42942772100d846c71dc15a4a37cb45b648b73 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 5 Jul 2023 18:28:45 +0200 Subject: [PATCH 301/793] feat(router): add argument for hostname in router (#545) (#550) # What does this PR do? In title. Adds argument `--hostname` in router to support something like `--hostname ::`. Tested with ```commandline cargo run -- --port 8080 --hostname :: curl -I -X GET 'http://[::1]:8080/health' # failed before this commit ``` Trigger CI --------- Co-authored-by: Phil Chen --- launcher/src/main.rs | 6 ++++++ router/src/main.rs | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 5b5cb45e..a612eb6d 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -197,6 +197,10 @@ struct Args { #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, + /// The IP address to listen on + #[clap(default_value = "0.0.0.0", long, env)] + hostname: String, + /// The port to listen on. #[clap(default_value = "3000", long, short, env)] port: u16, @@ -874,6 +878,8 @@ fn spawn_webserver( args.waiting_served_ratio.to_string(), "--max-waiting-tokens".to_string(), args.max_waiting_tokens.to_string(), + "--hostname".to_string(), + args.hostname.to_string(), "--port".to_string(), args.port.to_string(), "--master-shard-uds-path".to_string(), diff --git a/router/src/main.rs b/router/src/main.rs index f782be09..4cd89d68 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -40,6 +40,8 @@ struct Args { max_batch_total_tokens: u32, #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, + #[clap(default_value = "0.0.0.0", long, env)] + hostname: String, #[clap(default_value = "3000", long, short, env)] port: u16, #[clap(default_value = "/tmp/text-generation-server-0", long, env)] @@ -82,6 +84,7 @@ fn main() -> Result<(), std::io::Error> { max_batch_prefill_tokens, max_batch_total_tokens, max_waiting_tokens, + hostname, port, master_shard_uds_path, tokenizer_name, @@ -213,8 +216,13 @@ fn main() -> Result<(), std::io::Error> { .expect("Unable to warmup model"); tracing::info!("Connected"); - // Binds on localhost - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port); + let addr = match hostname.parse() { + Ok(ip) => SocketAddr::new(ip, port), + Err(_) => { + tracing::warn!("Invalid hostname, defaulting to 0.0.0.0"); + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port) + } + }; // Run server server::run( From c4bb5264acacf55f8f1e693ccd29f7a13d3bcba0 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 6 Jul 2023 14:28:33 +0200 Subject: [PATCH 302/793] fix(server): decrease memory fragmentation (#557) --- server/text_generation_server/cache.py | 4 +++ .../models/flash_causal_lm.py | 28 ++++++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/server/text_generation_server/cache.py b/server/text_generation_server/cache.py index 79fcd3aa..4504733e 100644 --- a/server/text_generation_server/cache.py +++ b/server/text_generation_server/cache.py @@ -1,3 +1,5 @@ +import torch + from typing import Dict, Optional, TypeVar from text_generation_server.models.types import Batch @@ -20,6 +22,8 @@ def delete(self, batch_id: int): batch = self.pop(batch_id) if batch is not None: del batch + if torch.cuda.is_available(): + torch.cuda.empty_cache() def clear(self): keys = list(self.cache.keys()) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index bf5f5bbe..bebd3df5 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -638,6 +638,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch # Needed to avoid dropping blocks when the batches will go out of scope for b in batches: b.block_tables = None + del b + torch.cuda.empty_cache() return FlashCausalLMBatch( batch_id=batches[0].batch_id, @@ -732,6 +734,7 @@ def warmup(self, batch: FlashCausalLMBatch, max_total_tokens: int): ) raise e del batch + torch.cuda.empty_cache() def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: return self.tokenizer.decode( @@ -775,16 +778,21 @@ def generate_token( # Allocate blocks to this batch CACHE_MANAGER.allocate(batch) - out = self.forward( - batch.input_ids, - batch.position_ids, - batch.cu_seqlen_prefill, - batch.block_tables_tensor, - batch.slots[batch.slot_indices], - batch.input_lengths_tensor, - batch.max_seqlen, - batch.prefill_head_indices, - ) + try: + out = self.forward( + batch.input_ids, + batch.position_ids, + batch.cu_seqlen_prefill, + batch.block_tables_tensor, + batch.slots[batch.slot_indices], + batch.input_lengths_tensor, + batch.max_seqlen, + batch.prefill_head_indices, + ) + except Exception as e: + del batch + torch.cuda.empty_cache() + raise e if prefill: next_token_logits = ( From 31b36cca21fcd0e6b7db477a7545063e1b860156 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 6 Jul 2023 16:05:42 +0200 Subject: [PATCH 303/793] v0.9.1 (#558) --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- docs/openapi.json | 2 +- server/pyproject.toml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b65045ff..1d030249 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2848,7 +2848,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "0.9.0" +version = "0.9.1" dependencies = [ "average", "clap", @@ -2868,7 +2868,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.9.0" +version = "0.9.1" dependencies = [ "futures", "grpc-metadata", @@ -2884,7 +2884,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.9.0" +version = "0.9.1" dependencies = [ "clap", "ctrlc", @@ -2900,7 +2900,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.9.0" +version = "0.9.1" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index ba7a920d..fdfb274e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.9.0" +version = "0.9.1" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index b91729d0..aaa360b0 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.9.0" + "version": "0.9.1" }, "paths": { "/": { diff --git a/server/pyproject.toml b/server/pyproject.toml index fd640e7f..a696d7be 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.9.0" +version = "0.9.1" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From e943a294bca239e26828732dd6ab5b6f95dadd0a Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 7 Jul 2023 14:50:12 +0200 Subject: [PATCH 304/793] fix(server): harden the weights choice to save on disk. (#561) - Look at `transformers` base class to check for `_key_to_ignore_on_load_missing` or `_tied_weights` which are the standard attributes to select the keys to NOT save on disk (since they are ignored) - Modified safetensors code (to be reflected in safetensors even if it's an internal function). - Will not work for trust_remote_code=True repos (like santacoder). Should help with : https://github.com/huggingface/text-generation-inference/issues/555 and : https://github.com/huggingface/text-generation-inference/pull/501 and https://github.com/huggingface/text-generation-inference/issues/556 and https://github.com/huggingface/text-generation-inference/issues/482#issuecomment-1623713593 --- server/tests/utils/test_convert.py | 2 +- server/text_generation_server/cli.py | 20 ++++++- .../text_generation_server/utils/convert.py | 57 +++++++++++++++++-- 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/server/tests/utils/test_convert.py b/server/tests/utils/test_convert.py index 7dfe6a1e..ba6c5702 100644 --- a/server/tests/utils/test_convert.py +++ b/server/tests/utils/test_convert.py @@ -14,7 +14,7 @@ def test_convert_files(): local_st_files = [ p.parent / f"{p.stem.lstrip('pytorch_')}.safetensors" for p in local_pt_files ] - convert_files(local_pt_files, local_st_files) + convert_files(local_pt_files, local_st_files, discard_names=[]) found_st_files = weight_files(model_id) diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 3463049a..7a55e919 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -160,8 +160,26 @@ def download_weights( p.parent / f"{p.stem.lstrip('pytorch_')}.safetensors" for p in local_pt_files ] + try: + from transformers import AutoConfig + import transformers + + config = AutoConfig.from_pretrained( + model_id, + revision=revision, + ) + architecture = config.architectures[0] + + class_ = getattr(transformers, architecture) + + # Name for this varible depends on transformers version. + discard_names = getattr(class_, "_tied_weights_keys", []) + discard_names.extend(getattr(class_, "_keys_to_ignore_on_load_missing", [])) + + except Exception as e: + discard_names = [] # Convert pytorch weights to safetensors - utils.convert_files(local_pt_files, local_st_files) + utils.convert_files(local_pt_files, local_st_files, discard_names) @app.command() diff --git a/server/text_generation_server/utils/convert.py b/server/text_generation_server/utils/convert.py index 0e4adaba..305263ba 100644 --- a/server/text_generation_server/utils/convert.py +++ b/server/text_generation_server/utils/convert.py @@ -4,11 +4,56 @@ from loguru import logger from pathlib import Path -from safetensors.torch import save_file, _remove_duplicate_names, load_file -from typing import List +from safetensors.torch import save_file, load_file, _find_shared_tensors, _is_complete +from typing import List, Dict +from collections import defaultdict -def convert_file(pt_file: Path, sf_file: Path): +def _remove_duplicate_names( + state_dict: Dict[str, torch.Tensor], + *, + preferred_names: List[str] = None, + discard_names: List[str] = None, +) -> Dict[str, List[str]]: + if preferred_names is None: + preferred_names = [] + preferred_names = set(preferred_names) + if discard_names is None: + discard_names = [] + discard_names = set(discard_names) + + shareds = _find_shared_tensors(state_dict) + to_remove = defaultdict(list) + for shared in shareds: + complete_names = set( + [name for name in shared if _is_complete(state_dict[name])] + ) + if not complete_names: + raise RuntimeError( + f"Error while trying to find names to remove to save state dict, but found no suitable name to keep for saving amongst: {shared}. None is covering the entire storage.Refusing to save/load the model since you could be storing much more memory than needed. Please refer to https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an issue." + ) + + keep_name = sorted(list(complete_names))[0] + + # Mecanism to preferentially select keys to keep + # coming from the on-disk file to allow + # loading models saved with a different choice + # of keep_name + preferred = complete_names.difference(discard_names) + if preferred: + keep_name = sorted(list(preferred))[0] + + if preferred_names: + preferred = preferred_names.intersection(complete_names) + if preferred: + keep_name = sorted(list(preferred))[0] + for name in sorted(shared): + if name != keep_name: + to_remove[keep_name].append(name) + return to_remove + + +def convert_file(pt_file: Path, sf_file: Path, discard_names: List[str]): """ Convert a pytorch file to a safetensors file This will remove duplicate tensors from the file. @@ -20,7 +65,7 @@ def convert_file(pt_file: Path, sf_file: Path): loaded = torch.load(pt_file, map_location="cpu") if "state_dict" in loaded: loaded = loaded["state_dict"] - to_removes = _remove_duplicate_names(loaded) + to_removes = _remove_duplicate_names(loaded, discard_names=discard_names) metadata = {"format": "pt"} for kept_name, to_remove_group in to_removes.items(): @@ -42,7 +87,7 @@ def convert_file(pt_file: Path, sf_file: Path): raise RuntimeError(f"The output tensors do not match for key {k}") -def convert_files(pt_files: List[Path], sf_files: List[Path]): +def convert_files(pt_files: List[Path], sf_files: List[Path], discard_names: List[str]): assert len(pt_files) == len(sf_files) N = len(pt_files) @@ -50,6 +95,6 @@ def convert_files(pt_files: List[Path], sf_files: List[Path]): for i, (pt_file, sf_file) in enumerate(zip(pt_files, sf_files)): start = datetime.datetime.now() - convert_file(pt_file, sf_file) + convert_file(pt_file, sf_file, discard_names) elapsed = datetime.datetime.now() - start logger.info(f"Convert: [{i + 1}/{N}] -- Took: {elapsed}") From b4024edd4549ab647b02b8619b4072e33e64f1f9 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 10 Jul 2023 14:47:15 +0200 Subject: [PATCH 305/793] feat: better errors for warmup and TP (#575) Close #571 --- router/src/main.rs | 37 +++++++++++++------ router/src/server.rs | 9 ++--- .../models/custom_modeling/bloom_modeling.py | 5 +++ .../custom_modeling/flash_llama_modeling.py | 5 +++ .../custom_modeling/flash_neox_modeling.py | 6 +++ .../custom_modeling/flash_rw_modeling.py | 6 +++ .../flash_santacoder_modeling.py | 6 ++- .../models/custom_modeling/mpt_modeling.py | 6 +++ .../models/custom_modeling/neox_modeling.py | 7 +++- .../models/custom_modeling/opt_modeling.py | 6 ++- .../models/custom_modeling/t5_modeling.py | 5 +++ .../models/flash_causal_lm.py | 5 +-- 12 files changed, 80 insertions(+), 23 deletions(-) diff --git a/router/src/main.rs b/router/src/main.rs index 4cd89d68..e482d834 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -10,8 +10,9 @@ use opentelemetry_otlp::WithExportConfig; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path; use std::time::Duration; -use text_generation_client::ShardedClient; +use text_generation_client::{ClientError, ShardedClient}; use text_generation_router::{server, HubModelInfo}; +use thiserror::Error; use tokenizers::{FromPretrainedParameters, Tokenizer}; use tower_http::cors::AllowOrigin; use tracing_subscriber::layer::SubscriberExt; @@ -70,7 +71,7 @@ struct Args { ngrok_password: Option, } -fn main() -> Result<(), std::io::Error> { +fn main() -> Result<(), RouterError> { // Get args let args = Args::parse(); // Pattern match configuration @@ -149,8 +150,7 @@ fn main() -> Result<(), std::io::Error> { // Launch Tokio runtime tokio::runtime::Builder::new_multi_thread() .enable_all() - .build() - .unwrap() + .build()? .block_on(async { init_logging(otlp_endpoint, json_output); @@ -192,17 +192,14 @@ fn main() -> Result<(), std::io::Error> { // Instantiate sharded client from the master unix socket let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path) .await - .expect("Could not connect to server"); + .map_err(RouterError::Connection)?; // Clear the cache; useful if the webserver rebooted sharded_client .clear_cache(None) .await - .expect("Unable to clear cache"); + .map_err(RouterError::Cache)?; // Get info from the shard - let shard_info = sharded_client - .info() - .await - .expect("Unable to get shard info"); + let shard_info = sharded_client.info().await.map_err(RouterError::Info)?; // Warmup model tracing::info!("Warming up model"); @@ -213,7 +210,7 @@ fn main() -> Result<(), std::io::Error> { max_batch_total_tokens, ) .await - .expect("Unable to warmup model"); + .map_err(RouterError::Warmup)?; tracing::info!("Connected"); let addr = match hostname.parse() { @@ -249,7 +246,7 @@ fn main() -> Result<(), std::io::Error> { ngrok_username, ngrok_password, ) - .await; + .await?; Ok(()) }) } @@ -331,3 +328,19 @@ pub async fn get_model_info( } None } + +#[derive(Debug, Error)] +enum RouterError { + #[error("Unable to connect to the Python model shards: {0}")] + Connection(ClientError), + #[error("Unable to clear the Python model shards cache: {0}")] + Cache(ClientError), + #[error("Unable to get the Python model shards info: {0}")] + Info(ClientError), + #[error("Unable to warmup the Python model shards: {0}")] + Warmup(ClientError), + #[error("Tokio runtime failed to start: {0}")] + Tokio(#[from] std::io::Error), + #[error("Axum webserver failed: {0}")] + Axum(#[from] axum::BoxError), +} diff --git a/router/src/server.rs b/router/src/server.rs index 54418f84..8ca463c2 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -527,7 +527,7 @@ pub async fn run( ngrok_domain: Option, ngrok_username: Option, ngrok_password: Option, -) { +) -> Result<(), axum::BoxError> { // OpenAPI documentation #[derive(OpenApi)] #[openapi( @@ -726,8 +726,7 @@ pub async fn run( .serve(app.into_make_service()) //Wait until all requests are finished to shut down .with_graceful_shutdown(shutdown_signal()) - .await - .unwrap(); + .await?; } #[cfg(not(feature = "ngrok"))] { @@ -744,9 +743,9 @@ pub async fn run( .serve(app.into_make_service()) // Wait until all requests are finished to shut down .with_graceful_shutdown(shutdown_signal()) - .await - .unwrap(); + .await?; } + Ok(()) } /// Shutdown signal handler diff --git a/server/text_generation_server/models/custom_modeling/bloom_modeling.py b/server/text_generation_server/models/custom_modeling/bloom_modeling.py index e5e87645..047a1872 100644 --- a/server/text_generation_server/models/custom_modeling/bloom_modeling.py +++ b/server/text_generation_server/models/custom_modeling/bloom_modeling.py @@ -256,6 +256,11 @@ def __init__(self, prefix, config: BloomConfig, weights): self.beta = 1.0 process_group = weights.process_group + if self.num_heads % process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {process_group.size()}" + ) self.num_heads = self.num_heads // process_group.size() self.query_key_value = TensorParallelColumnLinear.load( config=config, diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index d224a838..d9f3c7b8 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -112,6 +112,11 @@ def __init__( self.softmax_scale = self.head_size**-0.5 + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) self.num_heads = self.num_heads // weights.process_group.size() self.query_key_value = TensorParallelColumnLinear.load_multi( config, diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 23c5e4ff..b2dce226 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -95,6 +95,12 @@ def __init__(self, config, prefix, weights): self.num_heads = num_heads self.hidden_size = hidden_size self.head_size = hidden_size // num_heads + + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) self.num_heads = self.num_heads // weights.process_group.size() self.rotary_emb = PositionRotaryEmbedding.load( diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index cd42bfc2..acac2744 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -118,6 +118,12 @@ def __init__( dim=self.head_size, base=10000.0, device=weights.device ) self.softmax_scale = self.head_size ** (-0.5) + + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) self.num_heads = self.num_heads // weights.process_group.size() self.query_key_value = TensorParallelColumnLinear.load( diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 855a6e11..5b1c6e21 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -208,7 +208,11 @@ def __init__(self, prefix, config, weights): self.hidden_size = hidden_size self.head_size = hidden_size // num_heads - assert self.num_heads % weights.process_group.size() == 0 + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) self.num_heads = self.num_heads // weights.process_group.size() self.softmax_scale = self.head_size ** (-0.5) diff --git a/server/text_generation_server/models/custom_modeling/mpt_modeling.py b/server/text_generation_server/models/custom_modeling/mpt_modeling.py index 5ea204e1..e6057116 100644 --- a/server/text_generation_server/models/custom_modeling/mpt_modeling.py +++ b/server/text_generation_server/models/custom_modeling/mpt_modeling.py @@ -319,6 +319,12 @@ def __init__( if self.softmax_scale is None: self.softmax_scale = 1 / math.sqrt(self.d_model / self.n_heads) self.attn_dropout_p = config.attn_config["attn_pdrop"] + + if self.n_heads % weights.process_group.size() != 0: + raise ValueError( + f"`n_heads` must be divisible by `num_shards` (got `n_heads`: {self.n_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) self.n_heads = self.n_heads // weights.process_group.size() self.Wqkv = load_col( config, prefix=f"{prefix}.Wqkv", weights=weights, bias=not config.no_bias diff --git a/server/text_generation_server/models/custom_modeling/neox_modeling.py b/server/text_generation_server/models/custom_modeling/neox_modeling.py index bf2656d1..1951b171 100644 --- a/server/text_generation_server/models/custom_modeling/neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/neox_modeling.py @@ -154,7 +154,12 @@ def __init__(self, config, prefix, weights): torch.tensor(self.head_size, dtype=torch.float32) ).to(torch.get_default_dtype()) - assert self.num_attention_heads % weights.process_group.size() == 0 + if self.num_attention_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_attention_heads` must be divisible by `num_shards` " + f"(got `num_attention_heads`: {self.num_attention_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) self.num_attention_heads = ( self.num_attention_heads // weights.process_group.size() ) diff --git a/server/text_generation_server/models/custom_modeling/opt_modeling.py b/server/text_generation_server/models/custom_modeling/opt_modeling.py index 03fded50..aa052b08 100644 --- a/server/text_generation_server/models/custom_modeling/opt_modeling.py +++ b/server/text_generation_server/models/custom_modeling/opt_modeling.py @@ -147,7 +147,11 @@ def __init__( self.is_decoder = is_decoder process_group = weights.process_group - assert self.num_heads % process_group.size() == 0 + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) self.num_heads = self.num_heads // process_group.size() self.embed_dim = self.embed_dim // process_group.size() diff --git a/server/text_generation_server/models/custom_modeling/t5_modeling.py b/server/text_generation_server/models/custom_modeling/t5_modeling.py index 19deca86..1ea82802 100644 --- a/server/text_generation_server/models/custom_modeling/t5_modeling.py +++ b/server/text_generation_server/models/custom_modeling/t5_modeling.py @@ -246,6 +246,11 @@ def __init__( self.o = TensorParallelRowLinear.load( config, prefix=f"{prefix}.o", weights=weights, bias=False ) + if self.n_heads % weights.process_group.size() != 0: + raise ValueError( + f"`n_heads` must be divisible by `num_shards` (got `n_heads`: {self.n_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) self.n_heads = self.n_heads // process_group.size() self.inner_dim = self.inner_dim // process_group.size() diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index bebd3df5..5420556b 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -727,12 +727,11 @@ def warmup(self, batch: FlashCausalLMBatch, max_total_tokens: int): ) _, batch = self.generate_token(batch) except Exception as e: - logger.exception( + raise RuntimeError( f"Not enough memory to handle {max_total_tokens} total tokens with {len(batch.input_ids)} " f"prefill tokens. " f"You need to decrease `--max-batch-total-tokens` or `--max-batch-prefill-tokens`" - ) - raise e + ) from e del batch torch.cuda.empty_cache() From f0181436f41c730aad068029f54a3f86f354442d Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 12 Jul 2023 09:51:34 +0200 Subject: [PATCH 306/793] fix(server): Fixing RW code (it's remote code so the Arch checking doesn't work to see which weights to keep). (#579) Fixes #555 --- server/text_generation_server/models/flash_rw.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 12b862d7..55d555fc 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -49,7 +49,13 @@ def __init__( torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") - weights = Weights(filenames, device, dtype, process_group=self.process_group) + weights = Weights( + filenames, + device, + dtype, + process_group=self.process_group, + aliases={"transformer.word_embeddings.weight": ["lm_head.weight"]}, + ) config.quantize = quantize From 5bd2ab65839af4f54ba59e60ef3dd262004de456 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 12 Jul 2023 10:00:02 +0200 Subject: [PATCH 307/793] feat(server): Support for env value for GPTQ_BITS and GPTQ_GROUPSIZE. (#580) # What does this PR do? Some models are already converted, and do not have those values in the file, this enables users to use them with less friction. Went for pure env based because adding flags would end up (imo) very tedious to maintain. There's a lot of sanitation to do: those flags would be errors if not used in conjuction with `--quantize gptq`. Then the flags need to exist in the launcher and the server passing them all throughout all function calls. This PR is intended as an easy escape hatch, not the defacto method to use gptq in TGI. Fixes #500 --- .../custom_modeling/flash_santacoder_modeling.py | 16 ++++++++++++---- server/text_generation_server/utils/weights.py | 15 ++++++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 5b1c6e21..a19623a5 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -20,12 +20,12 @@ FastLayerNorm, get_linear, ) +from safetensors import SafetensorError def load_multi_mqa( config, prefix: str, weights, bias: bool, head_size, num_heads, hidden_size ): - if config.quantize == "gptq": return _load_multi_mqa_gptq( config, prefix, weights, bias, head_size, num_heads, hidden_size @@ -74,8 +74,17 @@ def _load_multi_mqa_gptq( qzeros = torch.cat([q_tensor, kv_tensor], dim=1) g_idx = weights.get_tensor(f"{prefix}.c_attn.g_idx") - bits = weights.get_tensor("gptq_bits").item() - groupsize = weights.get_tensor("gptq_groupsize").item() + try: + bits = weights.get_tensor("gptq_bits").item() + groupsize = weights.get_tensor("gptq_groupsize").item() + except SafetensorError as e: + try: + import os + + bits = int(os.getenv("GPTQ_BITS")) + groupsize = int(os.getenv("GPTQ_GROUPSIZE")) + except Exception: + raise e weight = (qweight, qzeros, scales, g_idx, bits, groupsize) @@ -99,7 +108,6 @@ def _load_multi_mqa_gptq( def _load_multi_mqa( config, prefix: str, weights, bias: bool, head_size, num_heads, hidden_size ): - if any("c_attn" in k for k in weights.routing.keys()): slice_ = weights._get_slice(f"{prefix}.c_attn.weight") shape = slice_.get_shape() diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 83d9df68..39f66862 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -1,6 +1,6 @@ from pathlib import Path from typing import List, Dict, Optional -from safetensors import safe_open +from safetensors import safe_open, SafetensorError import torch @@ -120,8 +120,17 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): torch.testing.assert_close(w2, w[0]) g_idx = w[0] - bits = self.get_tensor("gptq_bits").item() - groupsize = self.get_tensor("gptq_groupsize").item() + try: + bits = self.get_tensor("gptq_bits").item() + groupsize = self.get_tensor("gptq_groupsize").item() + except SafetensorError as e: + try: + import os + + bits = int(os.getenv("GTPQ_BITS")) + groupsize = int(os.getenv("GTPQ_GROUPSIZE")) + except Exception: + raise e weight = (qweight, qzeros, scales, g_idx, bits, groupsize) else: w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] From f063ebde103a78ee1fb6eafb66cb462526c0ce44 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 12 Jul 2023 10:01:01 +0200 Subject: [PATCH 308/793] chore: migrate ci region for more availability. (#581) --- .github/workflows/load_test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/load_test.yaml b/.github/workflows/load_test.yaml index 10e248e3..fd22e395 100644 --- a/.github/workflows/load_test.yaml +++ b/.github/workflows/load_test.yaml @@ -15,11 +15,11 @@ jobs: name: Start self-hosted EC2 runner runs-on: ubuntu-latest env: - AWS_REGION: us-east-1 - EC2_AMI_ID: ami-03cfed9ea28f4b002 + AWS_REGION: eu-central-1 + EC2_AMI_ID: ami-0ab09c07cfd194259 EC2_INSTANCE_TYPE: g5.12xlarge - EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc - EC2_SECURITY_GROUP: sg-04d472c808f365022 + EC2_SUBNET_ID: subnet-988fd9f2,subnet-6f56db13,subnet-6a039326 + EC2_SECURITY_GROUP: sg-072f92ae3082936c6 outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -90,7 +90,7 @@ jobs: - load-tests runs-on: ubuntu-latest env: - AWS_REGION: us-east-1 + AWS_REGION: eu-central-1 if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: - name: Configure AWS credentials @@ -105,4 +105,4 @@ jobs: mode: stop github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} label: ${{ needs.start-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} \ No newline at end of file + ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} From db4efbf4bcb851a185879099ac01fdc61e34a062 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 12 Jul 2023 10:01:42 +0200 Subject: [PATCH 309/793] fix(server): T5 weights names. (#582) Fixes #541 --- .../models/custom_modeling/t5_modeling.py | 7 +------ server/text_generation_server/models/t5.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/t5_modeling.py b/server/text_generation_server/models/custom_modeling/t5_modeling.py index 1ea82802..5779a274 100644 --- a/server/text_generation_server/models/custom_modeling/t5_modeling.py +++ b/server/text_generation_server/models/custom_modeling/t5_modeling.py @@ -1006,12 +1006,7 @@ def __init__(self, config: T5Config, weights): super().__init__(config) self.model_dim = config.d_model - try: - self.shared = TensorParallelEmbedding(prefix="shared", weights=weights) - except RuntimeError: - self.shared = TensorParallelEmbedding( - prefix="encoder.embed_tokens", weights=weights - ) + self.shared = TensorParallelEmbedding(prefix="shared", weights=weights) encoder_config = copy.deepcopy(config) encoder_config.is_decoder = False diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 1b7073af..133aafd8 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -55,7 +55,16 @@ def __init__( torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights( - filenames, device=device, dtype=dtype, process_group=self.process_group + filenames, + device=device, + dtype=dtype, + process_group=self.process_group, + aliases={ + "shared.weight": [ + "encoder.embed_tokens.weight", + "decoder.embed_tokens.weight", + ] + }, ) model = T5ForConditionalGeneration(config, weights) From 7f9072228afa5c6dd9a827d72bba818c3ec088c0 Mon Sep 17 00:00:00 2001 From: Adam Kowalski <49454006+akowalsk@users.noreply.github.com> Date: Wed, 12 Jul 2023 03:40:32 -0500 Subject: [PATCH 310/793] fix(server): Adding logger import to t5_modeling.py (#585) Logger is referenced during the apex importing but is not imported, causing a NameError --- .../models/custom_modeling/t5_modeling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/text_generation_server/models/custom_modeling/t5_modeling.py b/server/text_generation_server/models/custom_modeling/t5_modeling.py index 5779a274..793f3a66 100644 --- a/server/text_generation_server/models/custom_modeling/t5_modeling.py +++ b/server/text_generation_server/models/custom_modeling/t5_modeling.py @@ -19,6 +19,8 @@ import warnings from typing import Optional, Tuple, Union +from loguru import logger + import torch import torch.distributed from torch import nn From 2c4bf882689fda969c84dfc86c4158032de10e1b Mon Sep 17 00:00:00 2001 From: ssmi153 <129111316+ssmi153@users.noreply.github.com> Date: Wed, 12 Jul 2023 20:17:35 +0800 Subject: [PATCH 311/793] fix(server): Bug fixes for GPTQ_BITS environment variable passthrough (#590) # What does this PR do? This fixes a typo and extends the GPTP_BITS environment variables through to the second method which requires the same logic. Please let me know if there's anything I've misunderstood in this change. Thanks @Narsil for the original fix. --- server/text_generation_server/utils/weights.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 39f66862..4f300fe7 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -127,8 +127,8 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): try: import os - bits = int(os.getenv("GTPQ_BITS")) - groupsize = int(os.getenv("GTPQ_GROUPSIZE")) + bits = int(os.getenv("GPTQ_BITS")) + groupsize = int(os.getenv("GPTQ_GROUPSIZE")) except Exception: raise e weight = (qweight, qzeros, scales, g_idx, bits, groupsize) @@ -149,8 +149,17 @@ def get_multi_weights_row(self, prefix: str, quantize: str): scales = self.get_tensor(f"{prefix}.scales") g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) - bits = self.get_tensor("gptq_bits").item() - groupsize = self.get_tensor("gptq_groupsize").item() + try: + bits = self.get_tensor("gptq_bits").item() + groupsize = self.get_tensor("gptq_groupsize").item() + except SafetensorError as e: + try: + import os + + bits = int(os.getenv("GPTQ_BITS")) + groupsize = int(os.getenv("GPTQ_GROUPSIZE")) + except Exception: + raise e weight = (qweight, qzeros, scales, g_idx, bits, groupsize) else: From 67347950b7518efeb64c7f99ee360af685b53934 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 12 Jul 2023 16:43:31 +0200 Subject: [PATCH 312/793] feat(server): Implements sharding for non divisible `vocab_size`. (#583) - The code is relatively easy (just disable the checks on Embedding and Head) This cannot be done in the same easy fashion for hidden_dim/head_dim. It's relatively easy on some models (classic MHA) but it would make the other models (MQA) much more complex, and GPTQ quantization another quite hairy piece of code. --- server/text_generation_server/utils/layers.py | 23 +++++++++++++++---- .../text_generation_server/utils/weights.py | 17 ++++++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 8e0362b8..4f65446e 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -174,13 +174,25 @@ def forward(self, x): class TensorParallelHead(SuperLayer): - def __init__(self, linear, process_group): + def __init__(self, linear, process_group, should_gather: bool): super().__init__(linear) self.process_group = process_group + self.should_gather = should_gather @staticmethod def load(config, prefix: str, weights): - weight = weights.get_sharded(f"{prefix}.weight", dim=0) + if weights.process_group.size() > 1: + try: + weight = weights.get_sharded(f"{prefix}.weight", dim=0) + should_gather = True + except AssertionError: + # If the vocab size is not divisible by number of shards + # just load the entire thing. + weight = weights.get_tensor(f"{prefix}.weight") + should_gather = False + else: + weight = weights.get_tensor(f"{prefix}.weight") + should_gather = False # GPTQ doesn't quantize heads (nor embeddings) if config.quantize == "gptq": @@ -190,13 +202,14 @@ def load(config, prefix: str, weights): return TensorParallelHead( get_linear(weight, bias=None, quantize=quantize), process_group=weights.process_group, + should_gather=should_gather, ) def forward(self, input: torch.Tensor) -> torch.Tensor: - world_size = self.process_group.size() - if world_size == 1: + if not self.should_gather: return super().forward(input) + world_size = self.process_group.size() if len(input.shape) == 2 and isinstance(self.linear, FastLinear): out_dim = self.linear.weight.shape[0] @@ -277,7 +290,7 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: class TensorParallelEmbedding(nn.Module): def __init__(self, prefix: str, weights, reduce=True): super().__init__() - weight = weights.get_sharded(f"{prefix}.weight", dim=0) + weight = weights.get_partial_sharded(f"{prefix}.weight", dim=0) num_embeddings = weights.get_shape(f"{prefix}.weight")[0] process_group = weights.process_group diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 4f300fe7..afcbb9c3 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -69,7 +69,7 @@ def get_tensor(self, tensor_name: str): tensor = tensor.to(device=self.device) return tensor - def get_sharded(self, tensor_name: str, dim: int): + def get_partial_sharded(self, tensor_name: str, dim: int): filename, tensor_name = self.get_filename(tensor_name) world_size = self.process_group.size() rank = self.process_group.rank() @@ -81,10 +81,6 @@ def get_sharded(self, tensor_name: str, dim: int): start = rank * block_size stop = (rank + 1) * block_size - assert ( - size % world_size == 0 - ), f"The choosen size {size} is not compatible with sharding on {world_size} shards" - if dim == 0: tensor = slice_[start:stop] elif dim == 1: @@ -98,6 +94,17 @@ def get_sharded(self, tensor_name: str, dim: int): tensor = tensor.to(device=self.device) return tensor + def get_sharded(self, tensor_name: str, dim: int): + filename, tensor_name = self.get_filename(tensor_name) + f = self._get_handle(filename) + slice_ = f.get_slice(tensor_name) + world_size = self.process_group.size() + size = slice_.get_shape()[dim] + assert ( + size % world_size == 0 + ), f"The choosen size {size} is not compatible with sharding on {world_size} shards" + return self.get_partial_sharded(tensor_name, dim) + def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): if quantize == "gptq": try: From f2f0289fb99c7caab0c3749fdf211e4d5ab2938b Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 12 Jul 2023 17:05:50 +0200 Subject: [PATCH 313/793] feat(server): empty cache on errors --- server/text_generation_server/interceptor.py | 4 ++++ server/text_generation_server/models/flash_causal_lm.py | 3 --- server/text_generation_server/server.py | 9 +++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/server/text_generation_server/interceptor.py b/server/text_generation_server/interceptor.py index a3247d19..725105f3 100644 --- a/server/text_generation_server/interceptor.py +++ b/server/text_generation_server/interceptor.py @@ -1,3 +1,4 @@ +import torch import grpc from google.rpc import status_pb2, code_pb2 @@ -22,6 +23,9 @@ async def intercept( method_name = method_name.split("/")[-1] logger.exception(f"Method {method_name} encountered an error.") + if torch.cuda.is_available(): + torch.cuda.empty_cache() + await context.abort_with_status( rpc_status.to_status( status_pb2.Status(code=code_pb2.INTERNAL, message=str(err)) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 5420556b..4e5804f5 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -639,7 +639,6 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch for b in batches: b.block_tables = None del b - torch.cuda.empty_cache() return FlashCausalLMBatch( batch_id=batches[0].batch_id, @@ -733,7 +732,6 @@ def warmup(self, batch: FlashCausalLMBatch, max_total_tokens: int): f"You need to decrease `--max-batch-total-tokens` or `--max-batch-prefill-tokens`" ) from e del batch - torch.cuda.empty_cache() def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: return self.tokenizer.decode( @@ -790,7 +788,6 @@ def generate_token( ) except Exception as e: del batch - torch.cuda.empty_cache() raise e if prefill: diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index c375330a..7bc62ce6 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -51,6 +51,9 @@ async def FilterBatch(self, request, context): filtered_batch = batch.filter(request.request_ids) self.cache.set(filtered_batch) + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return generate_pb2.FilterBatchResponse(batch=filtered_batch.to_pb()) async def Warmup(self, request, context): @@ -58,6 +61,10 @@ async def Warmup(self, request, context): request.batch, self.model.tokenizer, self.model.dtype, self.model.device ) self.model.warmup(batch, request.max_total_tokens) + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return generate_pb2.WarmupResponse() async def Prefill(self, request, context): @@ -89,6 +96,8 @@ async def Decode(self, request, context): if len(batches) > 1: batch = self.model.batch_type.concatenate(batches) + if torch.cuda.is_available(): + torch.cuda.empty_cache() else: batch = batches[0] From 362855951660c15e38a31cf627c223a83ab53c79 Mon Sep 17 00:00:00 2001 From: ssmi153 <129111316+ssmi153@users.noreply.github.com> Date: Thu, 13 Jul 2023 01:57:46 +0800 Subject: [PATCH 314/793] GPTQ Env vars: catch correct type of error (#596) # What does this PR do? When passing in environment variables like gptq_bits, we still get errors thrown from TGI because the try/catch block is catching the wrong type of error. This PR aims to fix that. @Narsil - let me know if this is how you want this formatted. My Python is a little shaky, so I hope this syntax is correct. --- server/text_generation_server/utils/weights.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index afcbb9c3..723d0558 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -130,7 +130,7 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): try: bits = self.get_tensor("gptq_bits").item() groupsize = self.get_tensor("gptq_groupsize").item() - except SafetensorError as e: + except (SafetensorError, RuntimeError) as e: try: import os @@ -159,7 +159,7 @@ def get_multi_weights_row(self, prefix: str, quantize: str): try: bits = self.get_tensor("gptq_bits").item() groupsize = self.get_tensor("gptq_groupsize").item() - except SafetensorError as e: + except (SafetensorError, RuntimeError) as e: try: import os From b7327205a6f2f2c6349e75b8ea484e1e2823075a Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Jul 2023 14:22:37 +0200 Subject: [PATCH 315/793] feat(launcher): add arg validation and drop subprocess (#595) --- Cargo.lock | 25 ++-- launcher/Cargo.toml | 2 +- launcher/src/main.rs | 234 ++++++++++++++++++++---------------- router/client/src/client.rs | 2 +- router/src/main.rs | 17 ++- router/src/validation.rs | 6 +- 6 files changed, 157 insertions(+), 129 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d030249..867503f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -538,7 +538,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.9.0", "scopeguard", ] @@ -1412,6 +1412,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.9.0" @@ -1632,6 +1641,8 @@ dependencies = [ "bitflags 1.3.2", "cfg-if", "libc", + "memoffset 0.7.1", + "pin-utils", "static_assertions", ] @@ -2739,16 +2750,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "subprocess" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "subtle" version = "2.5.0" @@ -2889,10 +2890,10 @@ dependencies = [ "clap", "ctrlc", "float_eq", + "nix", "reqwest", "serde", "serde_json", - "subprocess", "tracing", "tracing-subscriber", "vergen", diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 6439e960..ae0694da 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -9,9 +9,9 @@ homepage.workspace = true [dependencies] clap = { version = "4.1.4", features = ["derive", "env"] } ctrlc = { version = "3.2.5", features = ["termination"] } +nix = "0.26.2" serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.93" -subprocess = "0.2.9" tracing = "0.1.37" tracing-subscriber = { version = "0.3.16", features = ["json"] } diff --git a/launcher/src/main.rs b/launcher/src/main.rs index a612eb6d..8b34dfe3 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -1,9 +1,13 @@ use clap::{Parser, ValueEnum}; +use nix::sys::signal::{self, Signal}; +use nix::unistd::Pid; use serde::Deserialize; use std::env; use std::ffi::OsString; use std::io::{BufRead, BufReader, Read}; +use std::os::unix::process::{CommandExt, ExitStatusExt}; use std::path::Path; +use std::process::{Child, Command, Stdio}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::TryRecvError; use std::sync::{mpsc, Arc}; @@ -11,7 +15,6 @@ use std::thread; use std::thread::sleep; use std::time::{Duration, Instant}; use std::{fs, io}; -use subprocess::{ExitStatus, Popen, PopenConfig, PopenError, Redirection}; mod env_runtime; @@ -72,6 +75,11 @@ struct Args { #[clap(long, env)] revision: Option, + /// The number of tokenizer workers used for payload validation and truncation inside the + /// router. + #[clap(default_value = "2", long, env)] + validation_workers: usize, + /// Whether to shard the model across multiple GPUs /// By default text-generation-inference will use all available GPUs to run /// the model. Setting it to `false` deactivates `num_shard`. @@ -306,11 +314,12 @@ fn shard_manager( let uds_string = format!("{uds_path}-{rank}"); let uds = Path::new(&uds_string); // Clean previous runs - fs::remove_file(uds).unwrap_or_default(); + if uds.exists() { + fs::remove_file(uds).unwrap(); + } // Process args let mut shard_argv = vec![ - "text-generation-server".to_string(), "serve".to_string(), model_id, "--uds-path".to_string(), @@ -415,26 +424,23 @@ fn shard_manager( // Start process tracing::info!("Starting shard {rank}"); - let mut p = match Popen::create( - &shard_argv, - PopenConfig { - stdout: Redirection::Pipe, - stderr: Redirection::Pipe, - // Needed for the shutdown procedure - setpgid: true, - // NCCL env vars - env: Some(env), - ..Default::default() - }, - ) { + let mut p = match Command::new("text-generation-server") + .args(shard_argv) + .envs(env) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + { Ok(p) => p, Err(err) => { - if let PopenError::IoError(ref err) = err { - if err.kind() == io::ErrorKind::NotFound { - tracing::error!("text-generation-server not found in PATH"); - tracing::error!("Please install it with `make install-server`") - } + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("text-generation-server not found in PATH"); + tracing::error!("Please install it with `make install-server`") + } else { + tracing::error!("{}", err); } + status_sender .send(ShardStatus::Failed((rank, Some(err.to_string())))) .unwrap(); @@ -462,7 +468,7 @@ fn shard_manager( let mut wait_time = Instant::now(); loop { // Process exited - if let Some(exit_status) = p.poll() { + if let Some(exit_status) = p.try_wait().unwrap() { // We read stderr in another thread as it seems that `read_to_string` can block // indefinitely in some cases let (err_sender, err_receiver) = mpsc::channel(); @@ -480,7 +486,7 @@ fn shard_manager( }) .ok(); - if let ExitStatus::Signaled(signal) = exit_status { + if let Some(signal) = exit_status.signal() { tracing::error!("Shard process was signaled to shutdown with signal {signal}"); } @@ -493,7 +499,7 @@ fn shard_manager( // We received a shutdown signal if shutdown.load(Ordering::SeqCst) { p.kill().unwrap(); - let _ = p.wait_timeout(Duration::from_secs(90)); + let _ = p.wait(); tracing::info!("Shard {rank} terminated"); return; } @@ -573,7 +579,10 @@ impl PythonLogMessage { } } -fn find_num_shards(sharded: Option, num_shard: Option) -> usize { +fn find_num_shards( + sharded: Option, + num_shard: Option, +) -> Result { // get the number of shards given `sharded` and `num_shard` let num_shard = match (sharded, num_shard) { (Some(true), None) => { @@ -582,14 +591,18 @@ fn find_num_shards(sharded: Option, num_shard: Option) -> usize { let n_devices = num_cuda_devices() .expect("--num-shard and CUDA_VISIBLE_DEVICES/NVIDIA_VISIBLE_DEVICES are not set"); if n_devices <= 1 { - panic!("`sharded` is true but only found {n_devices} CUDA devices"); + return Err(LauncherError::NotEnoughCUDADevices(format!( + "`sharded` is true but only found {n_devices} CUDA devices" + ))); } n_devices } (Some(true), Some(num_shard)) => { // we can't have only one shard while sharded if num_shard <= 1 { - panic!("`sharded` is true but `num_shard` <= 1"); + return Err(LauncherError::ArgumentValidation( + "`sharded` is true but `num_shard` <= 1".to_string(), + )); } num_shard } @@ -599,13 +612,17 @@ fn find_num_shards(sharded: Option, num_shard: Option) -> usize { (None, Some(num_shard)) => num_shard, }; if num_shard < 1 { - panic!("`num_shard` cannot be < 1"); + return Err(LauncherError::ArgumentValidation( + "`num_shard` cannot be < 1".to_string(), + )); } - num_shard + Ok(num_shard) } #[derive(Debug)] enum LauncherError { + ArgumentValidation(String), + NotEnoughCUDADevices(String), DownloadError, ShardCannotStart, ShardDisconnected, @@ -616,7 +633,6 @@ enum LauncherError { fn download_convert_model(args: &Args, running: Arc) -> Result<(), LauncherError> { let mut download_argv = vec![ - "text-generation-server".to_string(), "download-weights".to_string(), args.model_id.to_string(), "--extension".to_string(), @@ -664,25 +680,21 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L // Start process tracing::info!("Starting download process."); - let mut download_process = match Popen::create( - &download_argv, - PopenConfig { - stdout: Redirection::Pipe, - stderr: Redirection::Pipe, - // Needed for the shutdown procedure - setpgid: true, - env: Some(env), - ..Default::default() - }, - ) { + let mut download_process = match Command::new("text-generation-server") + .args(download_argv) + .envs(env) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + { Ok(p) => p, Err(err) => { - if let PopenError::IoError(ref err) = err { - if err.kind() == io::ErrorKind::NotFound { - tracing::error!("text-generation-server not found in PATH"); - tracing::error!("Please install it with `make install-server`") - } + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("text-generation-server not found in PATH"); + tracing::error!("Please install it with `make install-server`") } + return Err(LauncherError::DownloadError); } }; @@ -702,49 +714,33 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L }); loop { - if let Some(status) = download_process.poll() { - match status { - ExitStatus::Exited(exit_code) => { - if exit_code == 0 { - tracing::info!("Successfully downloaded weights."); - break; - } else { - let mut err = String::new(); - download_process - .stderr - .take() - .unwrap() - .read_to_string(&mut err) - .unwrap(); - tracing::error!("Download encountered an error: {err}"); - return Err(LauncherError::DownloadError); - } - } - ExitStatus::Signaled(signal) => { - let mut err = String::new(); - download_process - .stderr - .take() - .unwrap() - .read_to_string(&mut err) - .unwrap(); - tracing::error!( - "Download process was signaled to shutdown with signal {signal}: {err}" - ); - return Err(LauncherError::DownloadError); - } - e => { - tracing::error!("Download process exited with an unknown status.: {e:?}"); - return Err(LauncherError::DownloadError); - } + if let Some(status) = download_process.try_wait().unwrap() { + if status.success() { + tracing::info!("Successfully downloaded weights."); + break; } + + let mut err = String::new(); + download_process + .stderr + .take() + .unwrap() + .read_to_string(&mut err) + .unwrap(); + if let Some(signal) = status.signal() { + tracing::error!( + "Download process was signaled to shutdown with signal {signal}: {err}" + ); + } else { + tracing::error!("Download encountered an error: {err}"); + } + + return Err(LauncherError::DownloadError); } if !running.load(Ordering::SeqCst) { - download_process.terminate().unwrap(); + signal::kill(Pid::from_raw(download_process.id() as i32), Signal::SIGTERM).unwrap(); tracing::info!("Waiting for download process to gracefully shutdown"); - download_process - .wait_timeout(Duration::from_secs(90)) - .unwrap(); + download_process.wait().unwrap(); tracing::info!("Download process terminated"); return Ok(()); } @@ -854,12 +850,11 @@ fn spawn_webserver( args: Args, shutdown: Arc, shutdown_receiver: &mpsc::Receiver<()>, -) -> Result { +) -> Result { // All shard started // Start webserver tracing::info!("Starting Webserver"); let mut argv = vec![ - "text-generation-router".to_string(), "--max-concurrent-requests".to_string(), args.max_concurrent_requests.to_string(), "--max-best-of".to_string(), @@ -878,6 +873,8 @@ fn spawn_webserver( args.waiting_served_ratio.to_string(), "--max-waiting-tokens".to_string(), args.max_waiting_tokens.to_string(), + "--validation-workers".to_string(), + args.validation_workers.to_string(), "--hostname".to_string(), args.hostname.to_string(), "--port".to_string(), @@ -942,25 +939,20 @@ fn spawn_webserver( env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) }; - let mut webserver = match Popen::create( - &argv, - PopenConfig { - stdout: Redirection::Pipe, - stderr: Redirection::Pipe, - // Needed for the shutdown procedure - setpgid: true, - env: Some(env), - ..Default::default() - }, - ) { + let mut webserver = match Command::new("text-generation-router") + .args(argv) + .envs(env) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + { Ok(p) => p, Err(err) => { tracing::error!("Failed to start webserver: {}", err); - if let PopenError::IoError(err) = err { - if err.kind() == io::ErrorKind::NotFound { - tracing::error!("text-generation-router not found in PATH"); - tracing::error!("Please install it with `make install-router`") - } + if err.kind() == io::ErrorKind::NotFound { + tracing::error!("text-generation-router not found in PATH"); + tracing::error!("Please install it with `make install-router`") } else { tracing::error!("{}", err); } @@ -1004,7 +996,37 @@ fn main() -> Result<(), LauncherError> { tracing::info!("{:?}", args); - let num_shard = find_num_shards(args.sharded, args.num_shard); + // Validate args + if args.max_input_length >= args.max_total_tokens { + return Err(LauncherError::ArgumentValidation( + "`max_input_length` must be < `max_total_tokens`".to_string(), + )); + } + if args.max_input_length as u32 > args.max_batch_prefill_tokens { + return Err(LauncherError::ArgumentValidation(format!( + "`max_batch_prefill_tokens` must be >= `max_input_length`. Given: {} and {}", + args.max_batch_prefill_tokens, args.max_input_length + ))); + } + if args.max_batch_prefill_tokens > args.max_batch_total_tokens { + return Err(LauncherError::ArgumentValidation(format!( + "`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {} and {}", + args.max_batch_prefill_tokens, args.max_batch_total_tokens + ))); + } + if args.max_total_tokens as u32 > args.max_batch_total_tokens { + return Err(LauncherError::ArgumentValidation(format!( + "`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {} and {}", + args.max_total_tokens, args.max_batch_total_tokens + ))); + } + if args.validation_workers == 0 { + return Err(LauncherError::ArgumentValidation( + "`validation_workers` must be > 0".to_string(), + )); + } + + let num_shard = find_num_shards(args.sharded, args.num_shard)?; if num_shard > 1 { tracing::info!("Sharding model on {num_shard} processes"); } @@ -1065,7 +1087,7 @@ fn main() -> Result<(), LauncherError> { break; }; - match webserver.poll() { + match webserver.try_wait().unwrap() { Some(_) => { tracing::error!("Webserver Crashed"); shutdown_shards(shutdown, &shutdown_receiver); @@ -1078,9 +1100,9 @@ fn main() -> Result<(), LauncherError> { } // Graceful termination - webserver.terminate().unwrap(); + signal::kill(Pid::from_raw(webserver.id() as i32), Signal::SIGTERM).unwrap(); tracing::info!("Waiting for webserver to gracefully shutdown"); - webserver.wait_timeout(Duration::from_secs(90)).unwrap(); + webserver.wait().unwrap(); tracing::info!("Webserver terminated"); shutdown_shards(shutdown, &shutdown_receiver); diff --git a/router/client/src/client.rs b/router/client/src/client.rs index b5e0ccc0..b9607a5d 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -98,7 +98,7 @@ impl Client { /// Warmup on a max size batch /// /// Returns the maximum amount of tokens supported by the hardware - #[instrument(skip(self))] + #[instrument(skip_all)] pub async fn warmup( &mut self, max_input_length: u32, diff --git a/router/src/main.rs b/router/src/main.rs index e482d834..57ddd5ba 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -102,17 +102,24 @@ fn main() -> Result<(), RouterError> { } = args; // Validate args + if max_input_length >= max_total_tokens { + return Err(RouterError::ArgumentValidation( + "`max_input_length` must be < `max_total_tokens`".to_string(), + )); + } if max_input_length as u32 > max_batch_prefill_tokens { - panic!("{}", format!("`max_batch_prefill_tokens` must be >= `max_input_length`. Given: {max_batch_prefill_tokens} and {max_input_length}")); + return Err(RouterError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be >= `max_input_length`. Given: {max_batch_prefill_tokens} and {max_input_length}"))); } if max_batch_prefill_tokens > max_batch_total_tokens { - panic!("{}", format!("`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {max_batch_prefill_tokens} and {max_batch_total_tokens}")); + return Err(RouterError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {max_batch_prefill_tokens} and {max_batch_total_tokens}"))); } if max_total_tokens as u32 > max_batch_total_tokens { - panic!("{}", format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_batch_total_tokens}")); + return Err(RouterError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_batch_total_tokens}"))); } if validation_workers == 0 { - panic!("`validation_workers` must be > 0"); + return Err(RouterError::ArgumentValidation( + "`validation_workers` must be > 0".to_string(), + )); } // CORS allowed origins @@ -331,6 +338,8 @@ pub async fn get_model_info( #[derive(Debug, Error)] enum RouterError { + #[error("Argument validation error: {0}")] + ArgumentValidation(String), #[error("Unable to connect to the Python model shards: {0}")] Connection(ClientError), #[error("Unable to clear the Python model shards cache: {0}")] diff --git a/router/src/validation.rs b/router/src/validation.rs index 8843c6a8..be835bf0 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,5 +1,5 @@ -use crate::validation::ValidationError::{BestOfSampling, BestOfSeed, EmptyInput}; /// Payload validation logic +use crate::validation::ValidationError::{BestOfSampling, BestOfSeed, EmptyInput}; use crate::{GenerateParameters, GenerateRequest}; use rand::{thread_rng, Rng}; use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParameters}; @@ -30,10 +30,6 @@ impl Validation { max_input_length: usize, max_total_tokens: usize, ) -> Self { - if max_input_length >= max_total_tokens { - panic!("`max_input_length` must be < `max_total_tokens`"); - } - // If we have a fast tokenizer let sender = if let Some(tokenizer) = tokenizer { // Create channel From 982ce3227b780a8029b1c24546058c8f31f61a78 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Jul 2023 18:59:38 +0200 Subject: [PATCH 316/793] feat(router): explicit warning if revision is not set (#608) --- launcher/src/main.rs | 16 ++++++---------- router/src/main.rs | 32 +++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 8b34dfe3..54fb1368 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -760,16 +760,6 @@ fn spawn_shards( status_sender: mpsc::Sender, running: Arc, ) -> Result<(), LauncherError> { - if args.trust_remote_code { - tracing::warn!( - "`trust_remote_code` is set. Trusting that model `{}` do not contain malicious code.", - args.model_id - ); - if args.revision.is_none() { - tracing::warn!("Explicitly passing a `revision` is encouraged when loading a model with custom code to ensure no malicious code has been contributed in a newer revision."); - } - } - // Start shard processes for rank in 0..num_shard { let model_id = args.model_id.clone(); @@ -1025,6 +1015,12 @@ fn main() -> Result<(), LauncherError> { "`validation_workers` must be > 0".to_string(), )); } + if args.trust_remote_code { + tracing::warn!( + "`trust_remote_code` is set. Trusting that model `{}` do not contain malicious code.", + args.model_id + ); + } let num_shard = find_num_shards(args.sharded, args.num_shard)?; if num_shard > 1 { diff --git a/router/src/main.rs b/router/src/main.rs index 57ddd5ba..178c249c 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -49,8 +49,8 @@ struct Args { master_shard_uds_path: String, #[clap(default_value = "bigscience/bloom", long, env)] tokenizer_name: String, - #[clap(default_value = "main", long, env)] - revision: String, + #[clap(long, env)] + revision: Option, #[clap(default_value = "2", long, env)] validation_workers: usize, #[clap(long, env)] @@ -147,7 +147,7 @@ fn main() -> Result<(), RouterError> { // Download and instantiate tokenizer // We need to download it outside of the Tokio runtime let params = FromPretrainedParameters { - revision: revision.clone(), + revision: revision.clone().unwrap_or("main".to_string()), auth_token: authorization_token.clone(), ..Default::default() }; @@ -175,7 +175,7 @@ fn main() -> Result<(), RouterError> { sha: None, pipeline_tag: None, }, - false => get_model_info(&tokenizer_name, &revision, authorization_token) + false => get_model_info(&tokenizer_name, revision, authorization_token) .await .unwrap_or_else(|| { tracing::warn!("Could not retrieve model info from the Hugging Face hub."); @@ -316,9 +316,18 @@ fn init_logging(otlp_endpoint: Option, json_output: bool) { /// get model info from the Huggingface Hub pub async fn get_model_info( model_id: &str, - revision: &str, + revision: Option, token: Option, ) -> Option { + let revision = match revision { + None => { + tracing::warn!("`--revision` is not set"); + tracing::warn!("We strongly advise to set it to a known supported commit."); + "main".to_string() + } + Some(revision) => revision, + }; + let client = reqwest::Client::new(); // Poor man's urlencode let revision = revision.replace('/', "%2F"); @@ -331,9 +340,18 @@ pub async fn get_model_info( let response = builder.send().await.ok()?; if response.status().is_success() { - return serde_json::from_str(&response.text().await.ok()?).ok(); + let hub_model_info: HubModelInfo = + serde_json::from_str(&response.text().await.ok()?).ok()?; + if let Some(sha) = &hub_model_info.sha { + tracing::info!( + "Serving revision {sha} of model {}", + hub_model_info.model_id + ); + } + Some(hub_model_info) + } else { + None } - None } #[derive(Debug, Error)] From c8b077be799f49379db42aebe7fe72a591741432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20Mu=C5=A1tar?= Date: Thu, 13 Jul 2023 21:45:20 +0200 Subject: [PATCH 317/793] docs: README: Add logo + baseline (#611) ![image](https://github.com/huggingface/text-generation-inference/assets/3841370/58177321-479f-4ad1-b3bc-cec027423984) --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d31c176b..fe55d7b5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@
+![image](https://github.com/huggingface/text-generation-inference/assets/3841370/38ba1531-ea0d-4851-b31a-a6d4ddc944b0) + # Text Generation Inference @@ -11,9 +13,6 @@ Swagger API documentation - -![architecture](assets/architecture.jpg) -
A Rust, Python and gRPC server for text generation inference. Used in production at [HuggingFace](https://huggingface.co) From 5b9de4a1d37d0f9beccc782f0f7ad6a051b52673 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 13 Jul 2023 21:54:55 +0200 Subject: [PATCH 318/793] fix(server): blacklist local files (#609) Close #589 #602 --- server/text_generation_server/utils/convert.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/text_generation_server/utils/convert.py b/server/text_generation_server/utils/convert.py index 305263ba..8d414eca 100644 --- a/server/text_generation_server/utils/convert.py +++ b/server/text_generation_server/utils/convert.py @@ -94,6 +94,14 @@ def convert_files(pt_files: List[Path], sf_files: List[Path], discard_names: Lis # We do this instead of using tqdm because we want to parse the logs with the launcher for i, (pt_file, sf_file) in enumerate(zip(pt_files, sf_files)): + # Skip blacklisted files + if ( + "arguments" in pt_file.name + or "args" in pt_file.name + or "training" in pt_file.name + ): + continue + start = datetime.datetime.now() convert_file(pt_file, sf_file, discard_names) elapsed = datetime.datetime.now() - start From c58a0c185b974e6af1eed8372a3cce3624f00617 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 14 Jul 2023 16:31:48 +0200 Subject: [PATCH 319/793] v0.9.2 (#616) --- Cargo.lock | 302 +++++++++++++++++++++++------------------- Cargo.toml | 2 +- docs/openapi.json | 2 +- launcher/src/main.rs | 160 ++++++++++++---------- server/pyproject.toml | 2 +- 5 files changed, 262 insertions(+), 206 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 867503f2..22eec927 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,18 +148,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] @@ -410,9 +410,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.10" +version = "4.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a" +checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d" dependencies = [ "clap_builder", "clap_derive", @@ -421,9 +421,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.10" +version = "4.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d" +checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b" dependencies = [ "anstream", "anstyle", @@ -440,7 +440,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] @@ -492,9 +492,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -633,12 +633,12 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.4.0" +version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" dependencies = [ "cfg-if", - "hashbrown 0.12.3", + "hashbrown 0.14.0", "lock_api", "once_cell", "parking_lot_core", @@ -736,6 +736,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -924,7 +930,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] @@ -1023,7 +1029,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1038,13 +1044,19 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "heck" version = "0.4.1" @@ -1053,9 +1065,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hmac" @@ -1190,6 +1202,16 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", "serde", ] @@ -1254,12 +1276,12 @@ checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.1", + "rustix 0.38.4", "windows-sys 0.48.0", ] @@ -1292,9 +1314,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" [[package]] name = "jobserver" @@ -1397,7 +1419,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] @@ -1432,9 +1454,9 @@ dependencies = [ [[package]] name = "metrics" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8ebbd1a9e57bbab77b9facae7f5136aea44c356943bf9a198f647da64285d6" +checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" dependencies = [ "ahash", "metrics-macros", @@ -1449,7 +1471,7 @@ checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" dependencies = [ "base64 0.21.2", "hyper", - "indexmap", + "indexmap 1.9.3", "ipnet", "metrics", "metrics-util", @@ -1467,18 +1489,18 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] name = "metrics-util" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "111cb375987443c3de8d503580b536f77dc8416d32db62d9456db5d93bd7ac47" +checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" dependencies = [ "crossbeam-epoch", "crossbeam-utils", - "hashbrown 0.13.2", + "hashbrown 0.13.1", "metrics", "num_cpus", "quanta", @@ -1530,9 +1552,9 @@ dependencies = [ [[package]] name = "monostate" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a" +checksum = "3f3f57a8802842f648026a33c3d2e3bb41bb309a35b1609bd7ef2b060b8b6b1b" dependencies = [ "monostate-impl", "serde", @@ -1540,13 +1562,13 @@ dependencies = [ [[package]] name = "monostate-impl" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b" +checksum = "e72f4d2e10fde62a0f2fcb4b44ccbf4f9899dcc30c9193449f8dfb9123d71377" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] @@ -1701,6 +1723,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "number_prefix" version = "0.3.0" @@ -1773,7 +1804,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] @@ -1854,7 +1885,7 @@ dependencies = [ "fnv", "futures-channel", "futures-util", - "indexmap", + "indexmap 1.9.3", "js-sys", "once_cell", "pin-project-lite", @@ -1870,7 +1901,7 @@ dependencies = [ "fnv", "futures-channel", "futures-util", - "indexmap", + "indexmap 1.9.3", "once_cell", "pin-project-lite", "thiserror", @@ -1974,9 +2005,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "b4b27ab7be369122c218afc2079489cdcb4b517c0a3fc386ff11e1fedfcc2b35" [[package]] name = "pbkdf2" @@ -2003,34 +2034,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 1.9.3", ] [[package]] name = "pin-project" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e138fdd8263907a2b0e1b4e80b7e58c721126479b6e6eedfb1b402acea7b9bd" +checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1fef411b303e3e12d534fb6e7852de82da56edd937d895125821fb7c09436c7" +checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" [[package]] name = "pin-utils" @@ -2046,9 +2077,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "1.3.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794" +checksum = "d220334a184db82b31b83f5ff093e3315280fb2b6bbc032022b2304a509aab7a" [[package]] name = "ppv-lite86" @@ -2092,9 +2123,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" dependencies = [ "unicode-ident", ] @@ -2294,13 +2325,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.4" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" dependencies = [ "aho-corasick 1.0.2", "memchr", - "regex-syntax 0.7.2", + "regex-automata 0.3.3", + "regex-syntax 0.7.4", ] [[package]] @@ -2312,6 +2344,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +dependencies = [ + "aho-corasick 1.0.2", + "memchr", + "regex-syntax 0.7.4", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -2320,9 +2363,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "reqwest" @@ -2397,7 +2440,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.22", + "syn 2.0.25", "walkdir", ] @@ -2428,9 +2471,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.21" +version = "0.37.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f25693a73057a1b4cb56179dd3c7ea21a7c6c5ee7d85781f5749b46f34b79c" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" dependencies = [ "bitflags 1.3.2", "errno", @@ -2442,9 +2485,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.1" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc6396159432b5c8490d4e301d8c705f61860b8b6c863bf79942ce5401968f3" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" dependencies = [ "bitflags 2.3.3", "errno", @@ -2476,15 +2519,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "dc31bd9b61a32c31f9650d18add92aa83a49ba979c143eefd27fe7177b05bd5f" [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" [[package]] name = "same-file" @@ -2497,11 +2540,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -2551,29 +2594,29 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] name = "serde_json" -version = "1.0.99" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" +checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" dependencies = [ "itoa", "ryu", @@ -2582,10 +2625,11 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +checksum = "8acc4422959dd87a76cb117c191dcbffc20467f06c9100b76721dab370f24d3a" dependencies = [ + "itoa", "serde", ] @@ -2697,9 +2741,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" @@ -2769,9 +2813,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.22" +version = "2.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" dependencies = [ "proc-macro2", "quote", @@ -2786,9 +2830,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.29.3" +version = "0.29.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bcd0346f90b6bc83526c7b180039a8acd26a5c848cc556d457f6472eb148122" +checksum = "751e810399bba86e9326f5762b7f32ac5a085542df78da6a78d94e07d14d7c11" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2824,9 +2868,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +checksum = "ec96d2ffad078296368d46ff1cb309be1c23c513b4ab0e22a45de0185275ac96" dependencies = [ "filetime", "libc", @@ -2843,13 +2887,13 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix 0.37.21", + "rustix 0.37.23", "windows-sys 0.48.0", ] [[package]] name = "text-generation-benchmark" -version = "0.9.1" +version = "0.9.2" dependencies = [ "average", "clap", @@ -2869,7 +2913,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.9.1" +version = "0.9.2" dependencies = [ "futures", "grpc-metadata", @@ -2885,7 +2929,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.9.1" +version = "0.9.2" dependencies = [ "clap", "ctrlc", @@ -2901,7 +2945,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.9.1" +version = "0.9.2" dependencies = [ "async-stream", "axum", @@ -2934,22 +2978,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] @@ -2964,11 +3008,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" dependencies = [ "itoa", + "libc", + "num_threads", "serde", "time-core", "time-macros", @@ -2982,9 +3028,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" dependencies = [ "time-core", ] @@ -3078,7 +3124,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] @@ -3209,7 +3255,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand", @@ -3291,7 +3337,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", ] [[package]] @@ -3413,9 +3459,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" [[package]] name = "unicode-normalization" @@ -3484,11 +3530,11 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utoipa" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ae74ef183fae36d650f063ae7bde1cacbe1cd7e72b617cbe1e985551878b98" +checksum = "520434cac5c98120177d5cc15be032703f6dca7d5ef82e725c798113b375000a" dependencies = [ - "indexmap", + "indexmap 2.0.0", "serde", "serde_json", "utoipa-gen", @@ -3496,21 +3542,22 @@ dependencies = [ [[package]] name = "utoipa-gen" -version = "3.3.0" +version = "3.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ea8ac818da7e746a63285594cce8a96f5e00ee31994e655bd827569cb8b137b" +checksum = "6e22e88a487b6e0374533871b79b1f5ded05671bd0936bd547eb42f82fb9060d" dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.22", + "regex", + "syn 2.0.25", ] [[package]] name = "utoipa-swagger-ui" -version = "3.1.3" +version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062bba5a3568e126ac72049a63254f4cb1da2eb713db0c1ab2a4c76be191db8c" +checksum = "4602d7100d3cfd8a086f30494e68532402ab662fa366c9d201d677e33cee138d" dependencies = [ "axum", "mime_guess", @@ -3536,9 +3583,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "8.2.1" +version = "8.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b3c89c2c7e50f33e4d35527e5bf9c11d6d132226dbbd1753f0fbe9f19ef88c6" +checksum = "bbc5ad0d9d26b2c49a5ab7da76c3e79d3ee37e7821799f8223fcb8f2f391a2e7" dependencies = [ "anyhow", "rustc_version", @@ -3599,7 +3646,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", "wasm-bindgen-shared", ] @@ -3633,7 +3680,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.25", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3706,21 +3753,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-sys" version = "0.45.0" diff --git a/Cargo.toml b/Cargo.toml index fdfb274e..1383b7f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.9.1" +version = "0.9.2" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index aaa360b0..e570d92d 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.9.1" + "version": "0.9.2" }, "paths": { "/": { diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 54fb1368..d690a7c4 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -7,7 +7,7 @@ use std::ffi::OsString; use std::io::{BufRead, BufReader, Read}; use std::os::unix::process::{CommandExt, ExitStatusExt}; use std::path::Path; -use std::process::{Child, Command, Stdio}; +use std::process::{Child, Command, ExitStatus, Stdio}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::TryRecvError; use std::sync::{mpsc, Arc}; @@ -319,7 +319,7 @@ fn shard_manager( } // Process args - let mut shard_argv = vec![ + let mut shard_args = vec![ "serve".to_string(), model_id, "--uds-path".to_string(), @@ -331,77 +331,77 @@ fn shard_manager( // Activate trust remote code if trust_remote_code { - shard_argv.push("--trust-remote-code".to_string()); + shard_args.push("--trust-remote-code".to_string()); } // Activate tensor parallelism if world_size > 1 { - shard_argv.push("--sharded".to_string()); + shard_args.push("--sharded".to_string()); } if let Some(quantize) = quantize { - shard_argv.push("--quantize".to_string()); - shard_argv.push(quantize.to_string()) + shard_args.push("--quantize".to_string()); + shard_args.push(quantize.to_string()) } if let Some(dtype) = dtype { - shard_argv.push("--dtype".to_string()); - shard_argv.push(dtype.to_string()) + shard_args.push("--dtype".to_string()); + shard_args.push(dtype.to_string()) } // Model optional revision if let Some(revision) = revision { - shard_argv.push("--revision".to_string()); - shard_argv.push(revision) + shard_args.push("--revision".to_string()); + shard_args.push(revision) } // OpenTelemetry if let Some(otlp_endpoint) = otlp_endpoint { - shard_argv.push("--otlp-endpoint".to_string()); - shard_argv.push(otlp_endpoint); + shard_args.push("--otlp-endpoint".to_string()); + shard_args.push(otlp_endpoint); } // Copy current process env - let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); + let mut envs: Vec<(OsString, OsString)> = env::vars_os().collect(); // Use cuda allocator. It leads to less memory fragmentation - env.push(( + envs.push(( "PYTORCH_CUDA_ALLOC_CONF".into(), "backend:cudaMallocAsync".into(), )); // Torch Distributed Env vars - env.push(("RANK".into(), rank.to_string().into())); - env.push(("WORLD_SIZE".into(), world_size.to_string().into())); - env.push(("MASTER_ADDR".into(), master_addr.into())); - env.push(("MASTER_PORT".into(), master_port.to_string().into())); - env.push(("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())); + envs.push(("RANK".into(), rank.to_string().into())); + envs.push(("WORLD_SIZE".into(), world_size.to_string().into())); + envs.push(("MASTER_ADDR".into(), master_addr.into())); + envs.push(("MASTER_PORT".into(), master_port.to_string().into())); + envs.push(("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())); // Safetensors load fast - env.push(("SAFETENSORS_FAST_GPU".into(), "1".into())); + envs.push(("SAFETENSORS_FAST_GPU".into(), "1".into())); // Enable hf transfer for insane download speeds let enable_hf_transfer = env::var("HF_HUB_ENABLE_HF_TRANSFER").unwrap_or("1".to_string()); - env.push(( + envs.push(( "HF_HUB_ENABLE_HF_TRANSFER".into(), enable_hf_transfer.into(), )); // Parse Inference API token if let Ok(api_token) = env::var("HF_API_TOKEN") { - env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) + envs.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) }; // If huggingface_hub_cache is some, pass it to the shard // Useful when running inside a docker container if let Some(huggingface_hub_cache) = huggingface_hub_cache { - env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); + envs.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); }; // If weights_cache_override is some, pass it to the shard // Useful when running inside a HuggingFace Inference Endpoint if let Some(weights_cache_override) = weights_cache_override { - env.push(( + envs.push(( "WEIGHTS_CACHE_OVERRIDE".into(), weights_cache_override.into(), )); @@ -409,24 +409,24 @@ fn shard_manager( // If disable_custom_kernels is true, pass it to the shard as an env var if disable_custom_kernels { - env.push(("DISABLE_CUSTOM_KERNELS".into(), "True".into())) + envs.push(("DISABLE_CUSTOM_KERNELS".into(), "True".into())) } // Watermark Gamma if let Some(watermark_gamma) = watermark_gamma { - env.push(("WATERMARK_GAMMA".into(), watermark_gamma.to_string().into())) + envs.push(("WATERMARK_GAMMA".into(), watermark_gamma.to_string().into())) } // Watermark Delta if let Some(watermark_delta) = watermark_delta { - env.push(("WATERMARK_DELTA".into(), watermark_delta.to_string().into())) + envs.push(("WATERMARK_DELTA".into(), watermark_delta.to_string().into())) } // Start process tracing::info!("Starting shard {rank}"); let mut p = match Command::new("text-generation-server") - .args(shard_argv) - .envs(env) + .args(shard_args) + .envs(envs) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .process_group(0) @@ -632,7 +632,7 @@ enum LauncherError { } fn download_convert_model(args: &Args, running: Arc) -> Result<(), LauncherError> { - let mut download_argv = vec![ + let mut download_args = vec![ "download-weights".to_string(), args.model_id.to_string(), "--extension".to_string(), @@ -644,35 +644,35 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L // Model optional revision if let Some(revision) = &args.revision { - download_argv.push("--revision".to_string()); - download_argv.push(revision.to_string()) + download_args.push("--revision".to_string()); + download_args.push(revision.to_string()) } // Copy current process env - let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); + let mut envs: Vec<(OsString, OsString)> = env::vars_os().collect(); // If huggingface_hub_cache is set, pass it to the download process // Useful when running inside a docker container if let Some(ref huggingface_hub_cache) = args.huggingface_hub_cache { - env.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); + envs.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into())); }; // Enable hf transfer for insane download speeds let enable_hf_transfer = env::var("HF_HUB_ENABLE_HF_TRANSFER").unwrap_or("1".to_string()); - env.push(( + envs.push(( "HF_HUB_ENABLE_HF_TRANSFER".into(), enable_hf_transfer.into(), )); // Parse Inference API token if let Ok(api_token) = env::var("HF_API_TOKEN") { - env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) + envs.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) }; // If args.weights_cache_override is some, pass it to the download process // Useful when running inside a HuggingFace Inference Endpoint if let Some(weights_cache_override) = &args.weights_cache_override { - env.push(( + envs.push(( "WEIGHTS_CACHE_OVERRIDE".into(), weights_cache_override.into(), )); @@ -681,8 +681,8 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L // Start process tracing::info!("Starting download process."); let mut download_process = match Command::new("text-generation-server") - .args(download_argv) - .envs(env) + .args(download_args) + .envs(envs) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .process_group(0) @@ -738,10 +738,7 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L return Err(LauncherError::DownloadError); } if !running.load(Ordering::SeqCst) { - signal::kill(Pid::from_raw(download_process.id() as i32), Signal::SIGTERM).unwrap(); - tracing::info!("Waiting for download process to gracefully shutdown"); - download_process.wait().unwrap(); - tracing::info!("Download process terminated"); + terminate("download", download_process, Duration::from_secs(10)).unwrap(); return Ok(()); } sleep(Duration::from_millis(100)); @@ -844,7 +841,7 @@ fn spawn_webserver( // All shard started // Start webserver tracing::info!("Starting Webserver"); - let mut argv = vec![ + let mut router_args = vec![ "--max-concurrent-requests".to_string(), args.max_concurrent_requests.to_string(), "--max-best-of".to_string(), @@ -877,24 +874,24 @@ fn spawn_webserver( // Model optional revision if let Some(ref revision) = args.revision { - argv.push("--revision".to_string()); - argv.push(revision.to_string()) + router_args.push("--revision".to_string()); + router_args.push(revision.to_string()) } if args.json_output { - argv.push("--json-output".to_string()); + router_args.push("--json-output".to_string()); } // OpenTelemetry if let Some(otlp_endpoint) = args.otlp_endpoint { - argv.push("--otlp-endpoint".to_string()); - argv.push(otlp_endpoint); + router_args.push("--otlp-endpoint".to_string()); + router_args.push(otlp_endpoint); } // CORS origins for origin in args.cors_allow_origin.into_iter() { - argv.push("--cors-allow-origin".to_string()); - argv.push(origin); + router_args.push("--cors-allow-origin".to_string()); + router_args.push(origin); } // Ngrok @@ -904,34 +901,34 @@ fn spawn_webserver( LauncherError::WebserverCannotStart })?; - argv.push("--ngrok".to_string()); - argv.push("--ngrok-authtoken".to_string()); - argv.push(authtoken); + router_args.push("--ngrok".to_string()); + router_args.push("--ngrok-authtoken".to_string()); + router_args.push(authtoken); if let Some(domain) = args.ngrok_domain { - argv.push("--ngrok-domain".to_string()); - argv.push(domain); + router_args.push("--ngrok-domain".to_string()); + router_args.push(domain); } if let (Some(username), Some(password)) = (args.ngrok_username, args.ngrok_password) { - argv.push("--ngrok-username".to_string()); - argv.push(username); - argv.push("--ngrok-password".to_string()); - argv.push(password); + router_args.push("--ngrok-username".to_string()); + router_args.push(username); + router_args.push("--ngrok-password".to_string()); + router_args.push(password); } } // Copy current process env - let mut env: Vec<(OsString, OsString)> = env::vars_os().collect(); + let mut envs: Vec<(OsString, OsString)> = env::vars_os().collect(); // Parse Inference API token if let Ok(api_token) = env::var("HF_API_TOKEN") { - env.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) + envs.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) }; let mut webserver = match Command::new("text-generation-router") - .args(argv) - .envs(env) + .args(router_args) + .envs(envs) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .process_group(0) @@ -969,6 +966,31 @@ fn spawn_webserver( Ok(webserver) } +fn terminate(process_name: &str, mut process: Child, timeout: Duration) -> io::Result { + tracing::info!("Terminating {process_name}"); + + let terminate_time = Instant::now(); + signal::kill(Pid::from_raw(process.id() as i32), Signal::SIGTERM).unwrap(); + + tracing::info!("Waiting for {process_name} to gracefully shutdown"); + + while terminate_time.elapsed() < timeout { + if let Some(status) = process.try_wait()? { + tracing::info!("{process_name} terminated"); + return Ok(status); + } + sleep(Duration::from_millis(100)); + } + + tracing::info!("Killing {process_name}"); + + process.kill()?; + let exit_status = process.wait()?; + + tracing::info!("{process_name} killed"); + Ok(exit_status) +} + fn main() -> Result<(), LauncherError> { // Pattern match configuration let args = Args::parse(); @@ -1038,6 +1060,11 @@ fn main() -> Result<(), LauncherError> { // Download and convert model weights download_convert_model(&args, running.clone())?; + if !running.load(Ordering::SeqCst) { + // Launcher was asked to stop + return Ok(()); + } + // Shared shutdown bool let shutdown = Arc::new(AtomicBool::new(false)); // Shared shutdown channel @@ -1096,10 +1123,7 @@ fn main() -> Result<(), LauncherError> { } // Graceful termination - signal::kill(Pid::from_raw(webserver.id() as i32), Signal::SIGTERM).unwrap(); - tracing::info!("Waiting for webserver to gracefully shutdown"); - webserver.wait().unwrap(); - tracing::info!("Webserver terminated"); + terminate("webserver", webserver, Duration::from_secs(90)).unwrap(); shutdown_shards(shutdown, &shutdown_receiver); exit_code diff --git a/server/pyproject.toml b/server/pyproject.toml index a696d7be..57c72371 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.9.1" +version = "0.9.2" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From a2cf1bdb2fc0570dfca8b9ed2c8322f2040c3c07 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Sat, 15 Jul 2023 13:57:31 +0200 Subject: [PATCH 320/793] fix(server): empty_cache when stopped --- server/text_generation_server/models/flash_causal_lm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 4e5804f5..d034d472 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -991,6 +991,7 @@ def generate_token( if stopped: del batch + torch.cuda.empty_cache() # No need to return a batch if we know that all requests stopped return generations, None From bc2873246cd32800c8feb0bcdec7398420556d68 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 17 Jul 2023 18:38:16 +0200 Subject: [PATCH 321/793] fix(launcher): Rename `b-float16` to `bfloat16` in the launcher arg (#621) --- launcher/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index d690a7c4..113877fa 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -41,6 +41,7 @@ impl std::fmt::Display for Quantization { #[derive(Clone, Copy, Debug, ValueEnum)] enum Dtype { Float16, + #[clap(name = "bfloat16")] BFloat16, } From 44acf72a736346f4b8e969c9453027ca32786d72 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 17 Jul 2023 19:03:07 +0200 Subject: [PATCH 322/793] fea(launcher): debug logs (#623) --- launcher/Cargo.toml | 2 +- launcher/src/main.rs | 110 ++++++++++++++++++++++++------------------- 2 files changed, 62 insertions(+), 50 deletions(-) diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index ae0694da..3e7f86d4 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -13,7 +13,7 @@ nix = "0.26.2" serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.93" tracing = "0.1.37" -tracing-subscriber = { version = "0.3.16", features = ["json"] } +tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } [dev-dependencies] float_eq = "1.0.1" diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 113877fa..e26244e5 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -4,7 +4,7 @@ use nix::unistd::Pid; use serde::Deserialize; use std::env; use std::ffi::OsString; -use std::io::{BufRead, BufReader, Read}; +use std::io::{BufRead, BufReader, Lines, Read}; use std::os::unix::process::{CommandExt, ExitStatusExt}; use std::path::Path; use std::process::{Child, Command, ExitStatus, Stdio}; @@ -15,6 +15,7 @@ use std::thread; use std::thread::sleep; use std::time::{Duration, Instant}; use std::{fs, io}; +use tracing_subscriber::EnvFilter; mod env_runtime; @@ -286,7 +287,7 @@ struct Args { #[derive(Debug)] enum ShardStatus { Ready, - Failed((usize, Option)), + Failed(usize), } #[allow(clippy::too_many_arguments)] @@ -311,6 +312,9 @@ fn shard_manager( shutdown: Arc, _shutdown_sender: mpsc::Sender<()>, ) { + // Enter shard-manager tracing span + let _span = tracing::span!(tracing::Level::INFO, "shard-manager", rank = rank).entered(); + // Get UDS path let uds_string = format!("{uds_path}-{rank}"); let uds = Path::new(&uds_string); @@ -438,30 +442,23 @@ fn shard_manager( if err.kind() == io::ErrorKind::NotFound { tracing::error!("text-generation-server not found in PATH"); tracing::error!("Please install it with `make install-server`") - } else { + } + { tracing::error!("{}", err); } - status_sender - .send(ShardStatus::Failed((rank, Some(err.to_string())))) - .unwrap(); + status_sender.send(ShardStatus::Failed(rank)).unwrap(); return; } }; // Redirect STDOUT to the console let shard_stdout_reader = BufReader::new(p.stdout.take().unwrap()); - let mut shard_stderr_reader = BufReader::new(p.stderr.take().unwrap()); + let shard_stderr_reader = BufReader::new(p.stderr.take().unwrap()); + //stdout tracing thread thread::spawn(move || { - // Enter shard-manager tracing span - let _span = tracing::span!(tracing::Level::INFO, "shard-manager", rank = rank).entered(); - for line in shard_stdout_reader.lines() { - // Parse loguru logs - if let Ok(log) = serde_json::from_str::(&line.unwrap()) { - log.trace(); - } - } + log_lines(shard_stdout_reader.lines()); }); let mut ready = false; @@ -470,30 +467,25 @@ fn shard_manager( loop { // Process exited if let Some(exit_status) = p.try_wait().unwrap() { - // We read stderr in another thread as it seems that `read_to_string` can block - // indefinitely in some cases + // We read stderr in another thread as it seems that lines() can block in some cases let (err_sender, err_receiver) = mpsc::channel(); thread::spawn(move || { - let mut err = String::new(); - shard_stderr_reader.read_to_string(&mut err).unwrap(); - err_sender.send(err).unwrap_or(()); + for line in shard_stderr_reader.lines().flatten() { + err_sender.send(line).unwrap_or(()); + } }); + let mut err = String::new(); + while let Ok(line) = err_receiver.recv_timeout(Duration::from_millis(10)) { + err = err + "\n" + &line; + } - let err = err_receiver - .recv_timeout(Duration::from_millis(100)) - .map_err(|err| { - tracing::error!("Unable to read shard {rank} error from stderr"); - err - }) - .ok(); + tracing::error!("Shard complete standard error output:\n{err}"); if let Some(signal) = exit_status.signal() { tracing::error!("Shard process was signaled to shutdown with signal {signal}"); } - status_sender - .send(ShardStatus::Failed((rank, err))) - .unwrap(); + status_sender.send(ShardStatus::Failed(rank)).unwrap(); return; } @@ -580,6 +572,23 @@ impl PythonLogMessage { } } +impl TryFrom<&String> for PythonLogMessage { + type Error = serde_json::Error; + + fn try_from(value: &String) -> Result { + serde_json::from_str::(value) + } +} + +fn log_lines(lines: Lines) { + for line in lines.flatten() { + match PythonLogMessage::try_from(&line) { + Ok(log) => log.trace(), + Err(_) => tracing::debug!("{line}"), + } + } +} + fn find_num_shards( sharded: Option, num_shard: Option, @@ -633,6 +642,9 @@ enum LauncherError { } fn download_convert_model(args: &Args, running: Arc) -> Result<(), LauncherError> { + // Enter download tracing span + let _span = tracing::span!(tracing::Level::INFO, "download").entered(); + let mut download_args = vec![ "download-weights".to_string(), args.model_id.to_string(), @@ -694,6 +706,8 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L if err.kind() == io::ErrorKind::NotFound { tracing::error!("text-generation-server not found in PATH"); tracing::error!("Please install it with `make install-server`") + } else { + tracing::error!("{}", err); } return Err(LauncherError::DownloadError); @@ -702,16 +716,10 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L // Redirect STDOUT to the console let download_stdout = download_process.stdout.take().unwrap(); + let stdout = BufReader::new(download_stdout); + thread::spawn(move || { - // Enter download tracing span - let stdout = BufReader::new(download_stdout); - let _span = tracing::span!(tracing::Level::INFO, "download").entered(); - for line in stdout.lines() { - // Parse loguru logs - if let Ok(log) = serde_json::from_str::(&line.unwrap()) { - log.trace(); - } - } + log_lines(stdout.lines()); }); loop { @@ -816,11 +824,8 @@ fn spawn_shards( Err(TryRecvError::Empty) => { sleep(Duration::from_millis(100)); } - Ok(ShardStatus::Failed((rank, err))) => { + Ok(ShardStatus::Failed(rank)) => { tracing::error!("Shard {rank} failed to start"); - if let Some(err) = err { - tracing::error!("{err}"); - } shutdown_shards(shutdown, shutdown_receiver); return Err(LauncherError::ShardCannotStart); } @@ -996,10 +1001,20 @@ fn main() -> Result<(), LauncherError> { // Pattern match configuration let args = Args::parse(); + // Filter events with LOG_LEVEL + let env_filter = + EnvFilter::try_from_env("LOG_LEVEL").unwrap_or_else(|_| EnvFilter::new("info")); + if args.json_output { - tracing_subscriber::fmt().json().init(); + tracing_subscriber::fmt() + .with_env_filter(env_filter) + .json() + .init(); } else { - tracing_subscriber::fmt().compact().init(); + tracing_subscriber::fmt() + .with_env_filter(env_filter) + .compact() + .init(); } if args.env { @@ -1102,11 +1117,8 @@ fn main() -> Result<(), LauncherError> { let mut exit_code = Ok(()); while running.load(Ordering::SeqCst) { - if let Ok(ShardStatus::Failed((rank, err))) = status_receiver.try_recv() { + if let Ok(ShardStatus::Failed(rank)) = status_receiver.try_recv() { tracing::error!("Shard {rank} crashed"); - if let Some(err) = err { - tracing::error!("{err}"); - } exit_code = Err(LauncherError::ShardFailed); break; }; From 4d38a1c4ad9e262617a3f36e1d01e8c57693b6ef Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 18 Jul 2023 12:19:05 +0200 Subject: [PATCH 323/793] feat(server): Reworking the quantization script so it's still universal (not llama specific) (#587) but should work on more configurations (no need for 2 GPUs, less RAM usage). # What does this PR do? Reworking the quantization script so it's still universal (not llama specific) but should work on more configurations (no need for 2 GPUs, less RAM usage). Still need to investigate the potential differences in quantization results. Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/cli.py | 3 + .../utils/gptq/quantize.py | 140 ++++++++++++++++-- 2 files changed, 131 insertions(+), 12 deletions(-) diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 7a55e919..e74c0331 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -194,6 +194,8 @@ def quantize( percdamp: float = 0.01, act_order: bool = False, ): + if revision is None: + revision = "main" download_weights( model_id=model_id, revision=revision, @@ -207,6 +209,7 @@ def quantize( bits=4, groupsize=128, output_dir=output_dir, + revision=revision, trust_remote_code=trust_remote_code, upload_to_model_id=upload_to_model_id, percdamp=percdamp, diff --git a/server/text_generation_server/utils/gptq/quantize.py b/server/text_generation_server/utils/gptq/quantize.py index 5a4ed8da..d182456f 100644 --- a/server/text_generation_server/utils/gptq/quantize.py +++ b/server/text_generation_server/utils/gptq/quantize.py @@ -13,6 +13,9 @@ from huggingface_hub import HfApi import numpy as np import torch +from accelerate import init_empty_weights +from text_generation_server.utils import initialize_torch_distributed, Weights +from text_generation_server.utils.hub import weight_files from text_generation_server.utils.gptq.quant_linear import QuantLinear from loguru import logger from typing import Optional @@ -38,7 +41,6 @@ def configure( maxshrink=0.8, trits=False, ): - self.maxq = torch.tensor(2**bits - 1) self.perchannel = perchannel self.sym = sym @@ -600,6 +602,8 @@ def sequential( nsamples, bits, groupsize, + *, + hooks, percdamp=0.01, sym: bool = False, act_order: bool = False, @@ -637,7 +641,7 @@ def forward(self, inp, **kwargs): layers[0] = Catcher(layers[0]) for batch in dataloader: try: - model(batch[0]) + model(batch[0].cuda()) except ValueError: pass layers[0] = layers[0].module @@ -646,6 +650,8 @@ def forward(self, inp, **kwargs): # model.model.embed_tokens = model.model.embed_tokens.cpu() # model.model.norm = model.model.norm.cpu() torch.cuda.empty_cache() + for hook in hooks: + hook.remove() outs = torch.zeros_like(inps) @@ -662,10 +668,8 @@ def forward(self, inp, **kwargs): print("| name | weight_error | fp_inp_SNR | q_inp_SNR | time |") print("+==================+==============+============+===========+=======+") - from accelerate.hooks import remove_hook_from_submodules - - layer = layers[i].to(dev) - remove_hook_from_submodules(layer) + layer = layers[i] + layer.load() full = find_layers(layer) sequential = [list(full.keys())] @@ -677,6 +681,7 @@ def forward(self, inp, **kwargs): gptq[name].quantizer.configure( bits, perchannel=True, sym=sym, mse=False ) + pass def add_batch(name): def tmp(_, inp, out): @@ -688,7 +693,6 @@ def tmp(_, inp, out): for name in subset: handles.append(subset[name].register_forward_hook(add_batch(name))) for j in range(nsamples): - outs[j] = layer(inps[j].unsqueeze(0), **extra)[0] for h in handles: h.remove() @@ -714,7 +718,7 @@ def tmp(_, inp, out): for j in range(nsamples): outs[j] = layer(inps[j].unsqueeze(0), **extra)[0] - layers[i] = layer.cpu() + layer.unload() del layer del gptq torch.cuda.empty_cache() @@ -768,24 +772,136 @@ def pack(model, quantizers, bits, groupsize): return model +def setdeepattr(module, full_name, tensor): + current = module + tokens = full_name.split(".") + for token in tokens[:-1]: + current = getattr(current, token) + setattr(current, tokens[-1], tensor) + + +def getdeepattr(module, full_name): + current = module + tokens = full_name.split(".") + for token in tokens: + current = getattr(current, token) + return current + + +def load_weights_pre_hook(module_name, weights, recursive=False): + def inner(module, args): + print(f"Pre hook {module_name}") + local_params = {} + for k, v in module.named_parameters(): + if not recursive and k.count(".") != 1: + continue + local_params[k] = v + for k, v in module.named_buffers(): + if not recursive and k.count(".") != 1: + continue + local_params[k] = v + + for local_param in local_params: + current_tensor = getdeepattr(module, local_param) + if current_tensor.device == torch.device("meta"): + # print(f"Loading {local_param}") + if module_name: + tensor_name = f"{module_name}.{local_param}" + else: + tensor_name = local_param + tensor = weights.get_tensor(tensor_name) + setdeepattr(module, local_param, nn.Parameter(tensor)) + else: + setdeepattr( + module, + local_param, + nn.Parameter(current_tensor.to(device=torch.device("cuda:0"))), + ) + + return inner + + +def load_weights_post_hook(module_name, weights, recursive=False): + def inner(module, args, output): + print(f"Post hook {module_name}") + local_params = {} + for k, v in module.named_parameters(): + if not recursive and k.count(".") != 1: + continue + local_params[k] = v + for k, v in module.named_buffers(): + if not recursive and k.count(".") != 1: + continue + local_params[k] = v + for local_param in local_params: + # print(f"Unloading {local_param}") + current_tensor = getdeepattr(module, local_param) + setdeepattr( + module, + local_param, + nn.Parameter(current_tensor.to(device=torch.device("cpu"))), + ) + return output + + return inner + + def quantize( model_id: str, bits: int, groupsize: int, output_dir: str, + revision: str, trust_remote_code: bool, upload_to_model_id: Optional[str], percdamp: float, act_order: bool, ): print("loading model") - model = AutoModelForCausalLM.from_pretrained( + config = AutoConfig.from_pretrained( model_id, - torch_dtype=torch.float16, - device_map="balanced_low_0", trust_remote_code=trust_remote_code, ) + + with init_empty_weights(): + model = AutoModelForCausalLM.from_config(config, torch_dtype=torch.float16) + model = model.eval() + print("LOADED model") + files = weight_files(model_id, revision, extension=".safetensors") + process_group, _, _ = initialize_torch_distributed() + weights = Weights( + files, + device=torch.device("cuda:0"), + dtype=torch.float16, + process_group=process_group, + aliases={"embed_tokens.weight": ["lm_head.weight"]}, + ) + hooks = [] + for name, module in model.named_modules(): + + def load(module, name): + def _load(): + load_weights_pre_hook(name, weights, recursive=True)(module, None) + + return _load + + def unload(module, name): + def _unload(): + load_weights_post_hook(name, weights, recursive=True)( + module, None, None + ) + + return _unload + + module.load = load(module, name) + module.unload = unload(module, name) + hooks.append( + module.register_forward_pre_hook(load_weights_pre_hook(name, weights)) + ) + hooks.append( + module.register_forward_hook(load_weights_post_hook(name, weights)) + ) model.seqlen = 2048 dataset = "wikitext2" @@ -806,6 +922,7 @@ def quantize( groupsize, percdamp=percdamp, act_order=act_order, + hooks=hooks, ) print(time.time() - tick) @@ -858,7 +975,6 @@ def quantize( logger.info("Saved tokenizer") if upload_to_model_id: - api = HfApi() api.upload_folder( From 3b71c38558a9198c8424c09bbcfac03a93282d08 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 18 Jul 2023 16:21:18 +0200 Subject: [PATCH 324/793] feat(server): flash attention v2 (#624) --- Dockerfile | 15 ++- server/Makefile | 1 + server/Makefile-flash-att-v2 | 13 ++ .../text_generation_server/models/__init__.py | 54 ++------ .../custom_modeling/flash_llama_modeling.py | 12 +- .../custom_modeling/flash_neox_modeling.py | 14 +- .../custom_modeling/flash_rw_modeling.py | 35 +---- .../flash_santacoder_modeling.py | 17 +-- .../utils/flash_attn.py | 124 ++++++++++++++++++ 9 files changed, 173 insertions(+), 112 deletions(-) create mode 100644 server/Makefile-flash-att-v2 create mode 100644 server/text_generation_server/utils/flash_attn.py diff --git a/Dockerfile b/Dockerfile index 66e0091d..168f2f97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,6 +98,16 @@ COPY server/Makefile-flash-att Makefile # Build specific version of flash attention RUN make build-flash-attention +# Build Flash Attention v2 CUDA kernels +FROM kernel-builder as flash-att-v2-builder + +WORKDIR /usr/src + +COPY server/Makefile-flash-att-v2 Makefile + +# Build specific version of flash attention v2 +RUN make build-flash-attention-v2 + # Build Transformers CUDA kernels FROM kernel-builder as custom-kernels-builder @@ -146,8 +156,11 @@ COPY --from=flash-att-builder /usr/src/flash-attention/build/lib.linux-x86_64-cp COPY --from=flash-att-builder /usr/src/flash-attention/csrc/layer_norm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages COPY --from=flash-att-builder /usr/src/flash-attention/csrc/rotary/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +# Copy build artifacts from flash attention v2 builder +COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages + # Copy build artifacts from custom kernels builder -COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39/custom_kernels /usr/src/custom-kernels/src/custom_kernels +COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages # Copy builds artifacts from vllm builder COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages diff --git a/server/Makefile b/server/Makefile index d0086928..0dc0b5c9 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,4 +1,5 @@ include Makefile-flash-att +include Makefile-flash-att-v2 include Makefile-vllm unit-tests: diff --git a/server/Makefile-flash-att-v2 b/server/Makefile-flash-att-v2 new file mode 100644 index 00000000..a7d63356 --- /dev/null +++ b/server/Makefile-flash-att-v2 @@ -0,0 +1,13 @@ +flash_att_v2_commit := 4f285b354796fb17df8636485b9a04df3ebbb7dc + +flash-attention-v2: + # Clone flash attention + pip install packaging + git clone https://github.com/HazyResearch/flash-attention.git flash-attention-v2 + +build-flash-attention-v2: flash-attention-v2 + cd flash-attention-v2 && git fetch && git checkout $(flash_att_v2_commit) + cd flash-attention-v2 && python setup.py build + +install-flash-attention-v2: build-flash-attention-v2 + cd flash-attention-v2 && python setup.py install \ No newline at end of file diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index fd97f8b1..ffc224cc 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -42,51 +42,21 @@ "get_model", ] -FLASH_ATT_ERROR_MESSAGE = ( - "{} requires CUDA and Flash Attention kernels to be installed.\n" - "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) " - "or install flash attention with `cd server && make install install-flash-attention`" -) +FLASH_ATT_ERROR_MESSAGE = "{} requires Flash Attention enabled models." +FLASH_ATTENTION = True try: - if not os.getenv("USE_FLASH_ATTENTION", "").lower() == "false": - if not torch.cuda.is_available(): - FLASH_ATT_ERROR_MESSAGE = ( - "{} requires CUDA. No compatible CUDA devices found." - ) - raise ImportError("CUDA is not available") - - major, minor = torch.cuda.get_device_capability() - is_sm75 = major == 7 and minor == 5 - is_sm8x = major == 8 and minor >= 0 - is_sm90 = major == 9 and minor == 0 - - supported = is_sm75 or is_sm8x or is_sm90 - if not supported: - FLASH_ATT_ERROR_MESSAGE = ( - "{} requires a CUDA device with capability 7.5, > 8.0 or 9.0. " - "No compatible CUDA device found." - ) - raise ImportError( - f"GPU with CUDA capability {major} {minor} is not supported" - ) - - from text_generation_server.models.flash_rw import FlashRWSharded - from text_generation_server.models.flash_neox import FlashNeoXSharded - from text_generation_server.models.flash_llama import ( - FlashLlama, - ) - from text_generation_server.models.flash_santacoder import ( - FlashSantacoderSharded, - ) - - FLASH_ATTENTION = True - else: - FLASH_ATTENTION = False -except ImportError: - logger.opt(exception=True).warning( - "Could not import Flash Attention enabled models" + from text_generation_server.models.flash_rw import FlashRWSharded + from text_generation_server.models.flash_neox import FlashNeoXSharded + from text_generation_server.models.flash_llama import ( + FlashLlama, + ) + from text_generation_server.models.flash_santacoder import ( + FlashSantacoderSharded, ) + +except ImportError as e: + logger.warning(f"Could not import Flash Attention enabled models: {e}") FLASH_ATTENTION = False if FLASH_ATTENTION: diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index d9f3c7b8..d3c719df 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -26,13 +26,13 @@ from typing import Optional, List, Tuple # Flash attention imports -import flash_attn_cuda import dropout_layer_norm # vllm imports import vllm_cache_ops import vllm_attention_ops +from text_generation_server.utils.flash_attn import attention from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -164,22 +164,14 @@ def forward( # Prefill if cu_seqlen_prefill is not None: # flash attention - flash_attn_cuda.fwd( + attention( qkv[:, 0], qkv[:, 1], qkv[:, 2], attn_output, cu_seqlen_prefill, - cu_seqlen_prefill, - max_s, max_s, - 0.0, self.softmax_scale, - False, - True, - False, - 0, - None, ) # Decode else: diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index b2dce226..e7c8ced4 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -27,13 +27,11 @@ from transformers.models.gpt_neox import GPTNeoXConfig from typing import Optional, List, Tuple -# Flash attention imports -import flash_attn_cuda - # vllm imports import vllm_cache_ops import vllm_attention_ops +from text_generation_server.utils.flash_attn import attention from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -153,22 +151,14 @@ def forward( # Prefill if cu_seqlen_prefill is not None: # flash attention - flash_attn_cuda.fwd( + attention( qkv[:, 0], qkv[:, 1], qkv[:, 2], attn_output, cu_seqlen_prefill, - cu_seqlen_prefill, - max_s, max_s, - 0.0, self.softmax_scale, - False, - True, - False, - 0, - None, ) # Decode else: diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index acac2744..1e9539c4 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -6,13 +6,11 @@ from transformers.configuration_utils import PretrainedConfig from typing import Optional, List, Tuple -# Flash attention imports -import flash_attn_cuda - # vllm imports import vllm_cache_ops import vllm_attention_ops +from text_generation_server.utils.flash_attn import attention from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -182,27 +180,15 @@ def forward( # Prefill if cu_seqlen_prefill is not None: - if self.num_heads_kv == 1: - # Expand to query shape - kv = kv.expand(-1, 2, self.num_heads, self.head_size) - # flash attention - flash_attn_cuda.fwd( + attention( query, torch.select(kv, dim=1, index=0), torch.select(kv, dim=1, index=1), attn_output, cu_seqlen_prefill, - cu_seqlen_prefill, - max_s, max_s, - 0.0, self.softmax_scale, - False, - True, - False, - 0, - None, ) # Decode else: @@ -314,30 +300,15 @@ def forward( # Prefill if cu_seqlen_prefill is not None: - # Expand to query shape - kv = ( - kv.unsqueeze(2) - .expand(-1, self.num_groups, self.num_heads, 2, self.head_size) - .reshape(-1, self.num_groups * self.num_heads, 2, self.head_size) - ) - # flash attention - flash_attn_cuda.fwd( + attention( query, torch.select(kv, dim=2, index=0), torch.select(kv, dim=2, index=1), attn_output, cu_seqlen_prefill, - cu_seqlen_prefill, - max_s, max_s, - 0.0, self.softmax_scale, - False, - True, - False, - 0, - None, ) # Decode else: diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index a19623a5..6f5c60fc 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -5,13 +5,11 @@ from transformers.activations import ACT2FN from typing import Optional, List, Tuple -# Flash attention imports -import flash_attn_cuda - # vllm imports import vllm_cache_ops import vllm_attention_ops +from text_generation_server.utils.flash_attn import attention from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -271,26 +269,15 @@ def forward( # Prefill if cu_seqlen_prefill is not None: - # Expand from 1 to num_heads - key_value = key_value.expand(-1, 2, self.num_heads, self.head_size) - # flash attention - flash_attn_cuda.fwd( + attention( query, torch.select(key_value, dim=1, index=0), torch.select(key_value, dim=1, index=1), attn_output, cu_seqlen_prefill, - cu_seqlen_prefill, - max_s, max_s, - 0.0, self.softmax_scale, - False, - True, - False, - 0, - None, ) # Decode else: diff --git a/server/text_generation_server/utils/flash_attn.py b/server/text_generation_server/utils/flash_attn.py new file mode 100644 index 00000000..c472d1fc --- /dev/null +++ b/server/text_generation_server/utils/flash_attn.py @@ -0,0 +1,124 @@ +import os +import torch + +from loguru import logger + +if os.getenv("USE_FLASH_ATTENTION", "").lower() == "false": + raise ImportError("`USE_FLASH_ATTENTION` is false.") + +if not torch.cuda.is_available(): + raise ImportError("CUDA is not available") + +major, minor = torch.cuda.get_device_capability() +is_sm75 = major == 7 and minor == 5 +is_sm8x = major == 8 and minor >= 0 +is_sm90 = major == 9 and minor == 0 + +HAS_FLASH_ATTN = False +HAS_FLASH_ATTN_V2 = False +try: + try: + import flash_attn_2_cuda + except ImportError: + raise ImportError( + "Flash Attention V2 is not installed.\n" + "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) " + "or install flash attention v2 with `cd server && make install install-flash-attention-v2`" + ) + if not (is_sm8x or is_sm90): + raise ImportError( + f"GPU with CUDA capability {major} {minor} is not supported for " + "Flash Attention V2" + ) + HAS_FLASH_ATTN_V2 = True +except ImportError as e: + try: + import flash_attn_cuda + except ImportError: + raise ImportError( + "Flash Attention is not installed.\n" + "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) " + "or install flash attention with `cd server && make install install-flash-attention`" + ) from e + + if not (is_sm75 or is_sm8x or is_sm90): + raise ImportError( + f"GPU with CUDA capability {major} {minor} is not supported" + ) from e + logger.warning(f"Unable to use Flash Attention V2: {e}") + HAS_FLASH_ATTN = True + + +def attention( + q, + k, + v, + out, + cu_seqlens, + max_s, + softmax_scale, +): + if HAS_FLASH_ATTN_V2: + return flash_attn_2_cuda.varlen_fwd( + q, + k, + v, + out, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + 0.0, + softmax_scale, + False, + True, + False, + None, + ) + + if HAS_FLASH_ATTN: + # Flash attention v1 requires q, k and v to have the same number of heads + if k.shape[1] != q.shape[1]: + # MQA expand + if k.shape[1] == 1: + k = k.expand(-1, q.shape[1], -1) + # Grouped attention reshape + else: + original_shape = k.shape + k = ( + k.unsqueeze(2) + .expand(-1, -1, q.shape[1] // k.shape[1], -1) + .reshape(original_shape[0], -1, original_shape[2]) + ) + if v.shape[1] != q.shape[1]: + # MQA expand + if v.shape[1] == 1: + v = v.expand(-1, q.shape[1], -1) + # Grouped attention reshape + else: + original_shape = v.shape + v = ( + v.unsqueeze(2) + .expand(-1, -1, q.shape[1] // v.shape[1], -1) + .reshape(original_shape[0], -1, original_shape[2]) + ) + + return flash_attn_cuda.fwd( + q, + k, + v, + out, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + 0.0, + softmax_scale, + False, + True, + False, + 0, + None, + ) + + raise NotImplementedError("flash attention is not installed") From 211b211ec0df7c44f920429b0ec6767af5c9ea80 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 18 Jul 2023 18:09:53 +0200 Subject: [PATCH 325/793] feat(server): add support for llamav2 (#633) --- .../custom_modeling/flash_llama_modeling.py | 76 ++++++++++++++----- .../models/flash_llama.py | 2 +- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index d3c719df..4be8b98f 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -39,6 +39,7 @@ TensorParallelEmbedding, PositionRotaryEmbedding, TensorParallelHead, + get_linear, ) @@ -59,7 +60,8 @@ def forward(self, hidden_states, residual=None): hidden_states += residual residual = hidden_states - variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) hidden_states = hidden_states * torch.rsqrt( variance + self.variance_epsilon ) @@ -94,6 +96,27 @@ def forward(self, hidden_states, residual=None): return normed_hidden_states, res +def _load_gqa(config, prefix: str, weights): + w = [ + weights.get_sharded(f"{prefix}.q_proj.weight", dim=0), + weights.get_sharded(f"{prefix}.k_proj.weight", dim=0), + weights.get_sharded(f"{prefix}.v_proj.weight", dim=0), + ] + weight = torch.cat(w, dim=0) + weight = weight.to(dtype=weights.dtype).to(device=weights.device) + bias = None + assert config.hidden_size % config.num_attention_heads == 0 + head_size = config.hidden_size // config.num_attention_heads + assert config.num_attention_heads % weights.process_group.size() == 0 + num_heads = config.num_attention_heads // weights.process_group.size() + num_key_value_heads = config.num_key_value_heads // weights.process_group.size() + assert list(weight.shape) == [ + (num_heads + 2 * num_key_value_heads) * head_size, + config.hidden_size, + ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}" + return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize)) + + class FlashLlamaAttention(torch.nn.Module): def __init__( self, @@ -118,22 +141,29 @@ def __init__( f"and `num_shards`: {weights.process_group.size()}" ) self.num_heads = self.num_heads // weights.process_group.size() - self.query_key_value = TensorParallelColumnLinear.load_multi( - config, - prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], - dim=0, - weights=weights, - bias=False, + self.num_key_value_heads = ( + config.num_key_value_heads // weights.process_group.size() ) + if config.num_attention_heads != config.num_key_value_heads: + self.query_key_value = _load_gqa(config, prefix, weights) + else: + self.query_key_value = TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + dim=0, + weights=weights, + bias=False, + ) self.o_proj = TensorParallelRowLinear.load( config, prefix=f"{prefix}.o_proj", weights=weights, bias=False, ) + self.num_groups = self.num_heads // self.num_key_value_heads self.kv_head_mapping = torch.arange( - 0, self.num_heads, dtype=torch.int32, device=weights.device - ) + 0, self.num_key_value_heads, dtype=torch.int32, device=weights.device + ).repeat_interleave(self.num_groups) def forward( self, @@ -148,26 +178,33 @@ def forward( max_s, ): qkv = self.query_key_value(hidden_states) - qkv = qkv.view(-1, 3, self.num_heads, self.head_size) + query, kv = qkv.split( + [ + self.head_size * self.num_heads, + 2 * self.head_size * self.num_key_value_heads, + ], + dim=1, + ) + query = query.view(-1, self.num_heads, self.head_size) + kv = kv.view(-1, 2, self.num_key_value_heads, self.head_size) - # Inplace rotary - self.rotary_emb(qkv[:, 0], cos, sin) - self.rotary_emb(qkv[:, 1], cos, sin) + self.rotary_emb(query, cos, sin) + self.rotary_emb(torch.select(kv, dim=1, index=0), cos, sin) vllm_cache_ops.reshape_and_cache( - qkv[:, 1], qkv[:, 2], kv_cache[0], kv_cache[1], slots + kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots ) # output tensor - attn_output = torch.empty_like(qkv[:, 0]) + attn_output = torch.empty_like(query) # Prefill if cu_seqlen_prefill is not None: # flash attention attention( - qkv[:, 0], - qkv[:, 1], - qkv[:, 2], + query, + torch.select(kv, dim=1, index=0), + torch.select(kv, dim=1, index=1), attn_output, cu_seqlen_prefill, max_s, @@ -179,7 +216,7 @@ def forward( block_size = kv_cache[1].shape[3] vllm_attention_ops.single_query_cached_kv_attention( attn_output, - qkv[:, 0], + query, kv_cache[0], kv_cache[1], self.kv_head_mapping, @@ -316,6 +353,7 @@ def __init__(self, config, weights): self.head_size = self.layers[0].self_attn.head_size self.num_heads = self.layers[0].self_attn.num_heads + self.num_key_value_heads = self.layers[0].self_attn.num_key_value_heads def forward( self, diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 417ccabb..088b50b9 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -69,7 +69,7 @@ def __init__( model=model, tokenizer=tokenizer, num_layers=len(model.model.layers), - num_kv_heads=model.model.num_heads, + num_kv_heads=model.model.num_key_value_heads, head_size=model.model.head_size, dtype=dtype, device=device, From cf83f9b66fc74d61e3fb7f23557118b93c5b804f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 18 Jul 2023 18:11:20 +0200 Subject: [PATCH 326/793] v0.9.3 (#634) --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- README.md | 2 ++ docs/openapi.json | 2 +- server/pyproject.toml | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22eec927..539cf124 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2893,7 +2893,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "0.9.2" +version = "0.9.3" dependencies = [ "average", "clap", @@ -2913,7 +2913,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.9.2" +version = "0.9.3" dependencies = [ "futures", "grpc-metadata", @@ -2929,7 +2929,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.9.2" +version = "0.9.3" dependencies = [ "clap", "ctrlc", @@ -2945,7 +2945,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.9.2" +version = "0.9.3" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index 1383b7f9..49b7717a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.9.2" +version = "0.9.3" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index fe55d7b5..4a397bc4 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ to power LLMs api-inference widgets. - [Starcoder](https://huggingface.co/bigcode/starcoder) - [Falcon 7B](https://huggingface.co/tiiuae/falcon-7b) - [Falcon 40B](https://huggingface.co/tiiuae/falcon-40b) +- [MPT](https://huggingface.co/mosaicml/mpt-30b) +- [Llama V2](https://huggingface.co/meta-llama) Other architectures are supported on a best effort basis using: diff --git a/docs/openapi.json b/docs/openapi.json index e570d92d..80240460 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.9.2" + "version": "0.9.3" }, "paths": { "/": { diff --git a/server/pyproject.toml b/server/pyproject.toml index 57c72371..be79da51 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.9.2" +version = "0.9.3" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 5e6ddfd6a4fecc394255d7109f87c420c98b4e15 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 18 Jul 2023 18:49:42 +0200 Subject: [PATCH 327/793] fix(server): fix llamav2 config (#635) --- .../custom_modeling/flash_llama_modeling.py | 51 +++++++++++++++++++ .../models/flash_llama.py | 4 +- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 4be8b98f..b2bde282 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -23,6 +23,7 @@ from torch import nn from transformers.activations import ACT2FN +from transformers.configuration_utils import PretrainedConfig from typing import Optional, List, Tuple # Flash attention imports @@ -43,6 +44,56 @@ ) +class LlamaConfig(PretrainedConfig): + def __init__( + self, + vocab_size=32000, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + pretraining_tp=1, + tie_word_embeddings=False, + rope_scaling=None, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_scaling = rope_scaling + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + class LlamaRMSNorm(nn.Module): def __init__(self, prefix, weights, eps=1e-6): """ diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 088b50b9..b699799e 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -2,13 +2,13 @@ import torch.distributed from opentelemetry import trace -from transformers import AutoConfig from transformers.models.llama import LlamaTokenizer, LlamaTokenizerFast from typing import Optional from text_generation_server.models import FlashCausalLM from text_generation_server.models.custom_modeling.flash_llama_modeling import ( FlashLlamaForCausalLM, + LlamaConfig, ) from text_generation_server.utils import ( initialize_torch_distributed, @@ -52,7 +52,7 @@ def __init__( trust_remote_code=trust_remote_code, ) - config = AutoConfig.from_pretrained( + config = LlamaConfig.from_pretrained( model_id, revision=revision, trust_remote_code=trust_remote_code ) From fe80f5360ca885af4c63a3f1db9f6786df7ebc76 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 19 Jul 2023 09:31:25 +0200 Subject: [PATCH 328/793] feat(server): auto max_batch_total_tokens for flash att models (#630) --- launcher/src/main.rs | 54 +++++++++--------- proto/generate.proto | 7 ++- router/client/src/client.rs | 13 ++--- router/client/src/sharded_client.rs | 7 +-- router/src/infer.rs | 2 +- router/src/main.rs | 57 +++++++++++++------ router/src/queue.rs | 51 +++++++++++------ .../models/flash_causal_lm.py | 45 ++++++++++++--- server/text_generation_server/models/model.py | 3 +- server/text_generation_server/server.py | 14 ++--- 10 files changed, 159 insertions(+), 94 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index e26244e5..c03af4b7 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -184,8 +184,8 @@ struct Args { /// depends on other parameters like if you're using quantization, flash attention /// or the model implementation, text-generation-inference cannot infer this number /// automatically. - #[clap(default_value = "16000", long, env)] - max_batch_total_tokens: u32, + #[clap(long, env)] + max_batch_total_tokens: Option, /// This setting defines how many tokens can be passed before forcing the waiting /// queries to be put on the batch (if the size of the batch allows for it). @@ -369,12 +369,6 @@ fn shard_manager( // Copy current process env let mut envs: Vec<(OsString, OsString)> = env::vars_os().collect(); - // Use cuda allocator. It leads to less memory fragmentation - envs.push(( - "PYTORCH_CUDA_ALLOC_CONF".into(), - "backend:cudaMallocAsync".into(), - )); - // Torch Distributed Env vars envs.push(("RANK".into(), rank.to_string().into())); envs.push(("WORLD_SIZE".into(), world_size.to_string().into())); @@ -428,7 +422,7 @@ fn shard_manager( } // Start process - tracing::info!("Starting shard {rank}"); + tracing::info!("Starting shard"); let mut p = match Command::new("text-generation-server") .args(shard_args) .envs(envs) @@ -493,17 +487,17 @@ fn shard_manager( if shutdown.load(Ordering::SeqCst) { p.kill().unwrap(); let _ = p.wait(); - tracing::info!("Shard {rank} terminated"); + tracing::info!("Shard terminated"); return; } // Shard is ready if uds.exists() && !ready { - tracing::info!("Shard {rank} ready in {:?}", start_time.elapsed()); + tracing::info!("Shard ready in {:?}", start_time.elapsed()); status_sender.send(ShardStatus::Ready).unwrap(); ready = true; } else if !ready && wait_time.elapsed() > Duration::from_secs(10) { - tracing::info!("Waiting for shard {rank} to be ready..."); + tracing::info!("Waiting for shard to be ready..."); wait_time = Instant::now(); } sleep(Duration::from_millis(100)); @@ -860,8 +854,6 @@ fn spawn_webserver( args.max_total_tokens.to_string(), "--max-batch-prefill-tokens".to_string(), args.max_batch_prefill_tokens.to_string(), - "--max-batch-total-tokens".to_string(), - args.max_batch_total_tokens.to_string(), "--waiting-served-ratio".to_string(), args.waiting_served_ratio.to_string(), "--max-waiting-tokens".to_string(), @@ -878,6 +870,12 @@ fn spawn_webserver( args.model_id, ]; + // Model optional max batch total tokens + if let Some(max_batch_total_tokens) = args.max_batch_total_tokens { + router_args.push("--max-batch-total-tokens".to_string()); + router_args.push(max_batch_total_tokens.to_string()); + } + // Model optional revision if let Some(ref revision) = args.revision { router_args.push("--revision".to_string()); @@ -1036,18 +1034,7 @@ fn main() -> Result<(), LauncherError> { args.max_batch_prefill_tokens, args.max_input_length ))); } - if args.max_batch_prefill_tokens > args.max_batch_total_tokens { - return Err(LauncherError::ArgumentValidation(format!( - "`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {} and {}", - args.max_batch_prefill_tokens, args.max_batch_total_tokens - ))); - } - if args.max_total_tokens as u32 > args.max_batch_total_tokens { - return Err(LauncherError::ArgumentValidation(format!( - "`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {} and {}", - args.max_total_tokens, args.max_batch_total_tokens - ))); - } + if args.validation_workers == 0 { return Err(LauncherError::ArgumentValidation( "`validation_workers` must be > 0".to_string(), @@ -1065,6 +1052,21 @@ fn main() -> Result<(), LauncherError> { tracing::info!("Sharding model on {num_shard} processes"); } + if let Some(ref max_batch_total_tokens) = args.max_batch_total_tokens { + if args.max_batch_prefill_tokens > *max_batch_total_tokens { + return Err(LauncherError::ArgumentValidation(format!( + "`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {} and {}", + args.max_batch_prefill_tokens, max_batch_total_tokens + ))); + } + if args.max_total_tokens as u32 > *max_batch_total_tokens { + return Err(LauncherError::ArgumentValidation(format!( + "`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {} and {}", + args.max_total_tokens, max_batch_total_tokens + ))); + } + } + // Signal handler let running = Arc::new(AtomicBool::new(true)); let r = running.clone(); diff --git a/proto/generate.proto b/proto/generate.proto index 5e061941..57d79bca 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -198,9 +198,10 @@ message DecodeResponse { message WarmupRequest { /// Batch to warmup on Batch batch = 1; - /// Maximum number of tokens that the client will send - uint32 max_total_tokens = 2; } /// Empty response -message WarmupResponse {} +message WarmupResponse { + /// Maximum number of tokens supported by the model + optional uint32 max_supported_total_tokens = 1; +} diff --git a/router/client/src/client.rs b/router/client/src/client.rs index b9607a5d..7753f307 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -103,8 +103,7 @@ impl Client { &mut self, max_input_length: u32, max_prefill_tokens: u32, - max_total_tokens: u32, - ) -> Result<()> { + ) -> Result> { let mut n_tokens = 0; let mut requests = Vec::new(); @@ -143,13 +142,9 @@ impl Client { max_tokens: 0, }; - let request = tonic::Request::new(WarmupRequest { - batch: Some(batch), - max_total_tokens, - }) - .inject_context(); - self.stub.warmup(request).await?.into_inner(); - Ok(()) + let request = tonic::Request::new(WarmupRequest { batch: Some(batch) }).inject_context(); + let response = self.stub.warmup(request).await?.into_inner(); + Ok(response.max_supported_total_tokens) } /// Generate one token for each request in the given batch diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 9dd173a0..6d146bc5 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -95,14 +95,11 @@ impl ShardedClient { &mut self, max_input_length: u32, max_prefill_tokens: u32, - max_total_tokens: u32, - ) -> Result<()> { + ) -> Result> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| { - Box::pin(client.warmup(max_input_length, max_prefill_tokens, max_total_tokens)) - }) + .map(|client| Box::pin(client.warmup(max_input_length, max_prefill_tokens))) .collect(); // all shards return the same message join_all(futures).await.pop().unwrap() diff --git a/router/src/infer.rs b/router/src/infer.rs index d0d22d3b..188ddc64 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -53,7 +53,7 @@ impl Infer { generation_health: Arc, ) -> Self { // Infer shared state - let queue = Queue::new(requires_padding); + let queue = Queue::new(requires_padding, 16); let shared = Arc::new(Shared { batching_task: Notify::new(), }); diff --git a/router/src/main.rs b/router/src/main.rs index 178c249c..5aef03dd 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -37,8 +37,8 @@ struct Args { waiting_served_ratio: f32, #[clap(default_value = "4096", long, env)] max_batch_prefill_tokens: u32, - #[clap(default_value = "16000", long, env)] - max_batch_total_tokens: u32, + #[clap(long, env)] + max_batch_total_tokens: Option, #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, #[clap(default_value = "0.0.0.0", long, env)] @@ -110,18 +110,22 @@ fn main() -> Result<(), RouterError> { if max_input_length as u32 > max_batch_prefill_tokens { return Err(RouterError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be >= `max_input_length`. Given: {max_batch_prefill_tokens} and {max_input_length}"))); } - if max_batch_prefill_tokens > max_batch_total_tokens { - return Err(RouterError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {max_batch_prefill_tokens} and {max_batch_total_tokens}"))); - } - if max_total_tokens as u32 > max_batch_total_tokens { - return Err(RouterError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_batch_total_tokens}"))); - } + if validation_workers == 0 { return Err(RouterError::ArgumentValidation( "`validation_workers` must be > 0".to_string(), )); } + if let Some(ref max_batch_total_tokens) = max_batch_total_tokens { + if max_batch_prefill_tokens > *max_batch_total_tokens { + return Err(RouterError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {max_batch_prefill_tokens} and {max_batch_total_tokens}"))); + } + if max_total_tokens as u32 > *max_batch_total_tokens { + return Err(RouterError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_batch_total_tokens}"))); + } + } + // CORS allowed origins // map to go inside the option and then map to parse from String to HeaderValue // Finally, convert to AllowOrigin @@ -210,14 +214,35 @@ fn main() -> Result<(), RouterError> { // Warmup model tracing::info!("Warming up model"); - sharded_client - .warmup( - max_input_length as u32, - max_batch_prefill_tokens, - max_batch_total_tokens, - ) + let max_supported_batch_total_tokens = match sharded_client + .warmup(max_input_length as u32, max_batch_prefill_tokens) .await - .map_err(RouterError::Warmup)?; + .map_err(RouterError::Warmup)? + { + // Older models do not support automatic max-batch-total-tokens + None => { + let max_batch_total_tokens = max_batch_total_tokens.unwrap_or( + 16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens)), + ); + tracing::warn!("Model does not support automatic max batch total tokens"); + max_batch_total_tokens + } + // Flash attention models return their max supported total tokens + Some(max_supported_batch_total_tokens) => { + // Warn if user added his own max-batch-total-tokens as we will ignore it + if max_batch_total_tokens.is_some() { + tracing::warn!( + "`--max-batch-total-tokens` is deprecated for Flash \ + Attention models." + ); + tracing::warn!( + "Inferred max batch total tokens: {max_supported_batch_total_tokens}" + ); + } + max_supported_batch_total_tokens + } + }; + tracing::info!("Setting max batch total tokens to {max_supported_batch_total_tokens}"); tracing::info!("Connected"); let addr = match hostname.parse() { @@ -240,7 +265,7 @@ fn main() -> Result<(), RouterError> { max_total_tokens, waiting_served_ratio, max_batch_prefill_tokens, - max_batch_total_tokens, + max_supported_batch_total_tokens, max_waiting_tokens, sharded_client, tokenizer, diff --git a/router/src/queue.rs b/router/src/queue.rs index 48e483a1..2d8d6d1c 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -33,12 +33,12 @@ pub(crate) struct Queue { } impl Queue { - pub(crate) fn new(requires_padding: bool) -> Self { + pub(crate) fn new(requires_padding: bool, block_size: u32) -> Self { // Create channel let (queue_sender, queue_receiver) = flume::unbounded(); // Launch background queue task - tokio::spawn(queue_task(requires_padding, queue_receiver)); + tokio::spawn(queue_task(requires_padding, block_size, queue_receiver)); Self { queue_sender } } @@ -81,8 +81,12 @@ impl Queue { } // Background task responsible of the queue state -async fn queue_task(requires_padding: bool, receiver: flume::Receiver) { - let mut state = State::new(requires_padding); +async fn queue_task( + requires_padding: bool, + block_size: u32, + receiver: flume::Receiver, +) { + let mut state = State::new(requires_padding, block_size); while let Ok(cmd) = receiver.recv_async().await { match cmd { @@ -119,15 +123,19 @@ struct State { /// Whether the model is using padding requires_padding: bool, + + /// Paged Attention block size + block_size: u32, } impl State { - fn new(requires_padding: bool) -> Self { + fn new(requires_padding: bool, block_size: u32) -> Self { Self { entries: VecDeque::with_capacity(128), next_id: 0, next_batch_id: 0, requires_padding, + block_size, } } @@ -187,10 +195,21 @@ impl State { max_input_length = max_input_length.max(entry.request.input_length); prefill_tokens = (batch_requests.len() + 1) as u32 * max_input_length } else { - prefill_tokens += entry.request.input_length; + // pad to block size + prefill_tokens += ((entry.request.input_length + self.block_size - 1) + / self.block_size) + * self.block_size; } - decode_tokens += entry.request.stopping_parameters.max_new_tokens; + if self.requires_padding { + decode_tokens += entry.request.stopping_parameters.max_new_tokens; + } else { + // pad to block size + decode_tokens += + ((entry.request.stopping_parameters.max_new_tokens + self.block_size - 1) + / self.block_size) + * self.block_size; + } if prefill_tokens > prefill_token_budget || (prefill_tokens + decode_tokens) > token_budget @@ -321,7 +340,7 @@ mod tests { #[test] fn test_append() { - let mut state = State::new(false); + let mut state = State::new(false, 1); let (entry, _guard) = default_entry(); assert_eq!(state.next_id, 0); @@ -337,7 +356,7 @@ mod tests { #[test] fn test_next_batch_empty() { - let mut state = State::new(false); + let mut state = State::new(false, 1); assert!(state.next_batch(None, 1, 1).is_none()); assert!(state.next_batch(Some(1), 1, 1).is_none()); @@ -345,7 +364,7 @@ mod tests { #[test] fn test_next_batch_min_size() { - let mut state = State::new(false); + let mut state = State::new(false, 1); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); state.append(entry1); @@ -377,7 +396,7 @@ mod tests { #[test] fn test_next_batch_token_budget() { - let mut state = State::new(false); + let mut state = State::new(false, 1); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); state.append(entry1); @@ -410,14 +429,14 @@ mod tests { #[tokio::test] async fn test_queue_append() { - let queue = Queue::new(false); + let queue = Queue::new(false, 1); let (entry, _guard) = default_entry(); queue.append(entry); } #[tokio::test] async fn test_queue_next_batch_empty() { - let queue = Queue::new(false); + let queue = Queue::new(false, 1); assert!(queue.next_batch(None, 1, 1).await.is_none()); assert!(queue.next_batch(Some(1), 1, 1).await.is_none()); @@ -425,7 +444,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_min_size() { - let queue = Queue::new(false); + let queue = Queue::new(false, 1); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); queue.append(entry1); @@ -458,7 +477,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_token_budget() { - let queue = Queue::new(false); + let queue = Queue::new(false, 1); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); queue.append(entry1); @@ -483,7 +502,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_dropped_receiver() { - let queue = Queue::new(false); + let queue = Queue::new(false, 1); let (entry, _) = default_entry(); queue.append(entry); diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index d034d472..517fba68 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -710,14 +710,14 @@ def __init__( def batch_type(self) -> Type[FlashCausalLMBatch]: return FlashCausalLMBatch - def warmup(self, batch: FlashCausalLMBatch, max_total_tokens: int): + def warmup(self, batch: FlashCausalLMBatch): global CACHE_MANAGER torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats(self.device) try: CACHE_MANAGER = CacheManager( - # Adds some wiggle room - math.ceil(max_total_tokens / BLOCK_SIZE) + 10, + batch.blocks, self.num_layers, self.num_kv_heads, self.head_size, @@ -727,11 +727,43 @@ def warmup(self, batch: FlashCausalLMBatch, max_total_tokens: int): _, batch = self.generate_token(batch) except Exception as e: raise RuntimeError( - f"Not enough memory to handle {max_total_tokens} total tokens with {len(batch.input_ids)} " - f"prefill tokens. " - f"You need to decrease `--max-batch-total-tokens` or `--max-batch-prefill-tokens`" + f"Not enough memory to handle {len(batch.input_ids)} prefill tokens. " + f"You need to decrease `--max-batch-prefill-tokens`" ) from e + + # Inspired by the original implementation in [vllm](https://github.com/vllm-project/vllm) + # Calculate the number of blocks that can be allocated with the + # profiled peak memory. + torch.cuda.synchronize(self.device) + peak_memory = torch.cuda.max_memory_reserved(self.device) + + dtype_size = torch.tensor([], dtype=self.dtype).element_size() + cache_block_size = BLOCK_SIZE * self.num_kv_heads * self.head_size + total_cache_size = self.num_layers * cache_block_size * 2 * dtype_size + + total_gpu_memory = torch.cuda.get_device_properties(self.device).total_memory + + # 0.98 to add some wiggle room + num_blocks = ( + int((total_gpu_memory * 0.98 - peak_memory) // total_cache_size) + # Add batch.blocks as we allocated it above, so it is included in the peak memory. + + batch.blocks + ) + + del CACHE_MANAGER del batch + torch.cuda.empty_cache() + + CACHE_MANAGER = CacheManager( + num_blocks, + self.num_layers, + self.num_kv_heads, + self.head_size, + self.dtype, + self.device, + ) + + return int(num_blocks * BLOCK_SIZE) def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: return self.tokenizer.decode( @@ -991,7 +1023,6 @@ def generate_token( if stopped: del batch - torch.cuda.empty_cache() # No need to return a batch if we know that all requests stopped return generations, None diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index f8460fc2..3827197f 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -58,8 +58,9 @@ def batch_type(self) -> Type[B]: def generate_token(self, batch: B) -> Tuple[List[GeneratedText], Optional[B]]: raise NotImplementedError - def warmup(self, batch: B, max_total_tokens: int): + def warmup(self, batch: B) -> Optional[int]: self.generate_token(batch) + return None def decode_token( self, diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 7bc62ce6..e0efbcf5 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -51,21 +51,17 @@ async def FilterBatch(self, request, context): filtered_batch = batch.filter(request.request_ids) self.cache.set(filtered_batch) - if torch.cuda.is_available(): - torch.cuda.empty_cache() - return generate_pb2.FilterBatchResponse(batch=filtered_batch.to_pb()) async def Warmup(self, request, context): batch = self.model.batch_type.from_pb( request.batch, self.model.tokenizer, self.model.dtype, self.model.device ) - self.model.warmup(batch, request.max_total_tokens) - - if torch.cuda.is_available(): - torch.cuda.empty_cache() + max_supported_total_tokens = self.model.warmup(batch) - return generate_pb2.WarmupResponse() + return generate_pb2.WarmupResponse( + max_supported_total_tokens=max_supported_total_tokens + ) async def Prefill(self, request, context): batch = self.model.batch_type.from_pb( @@ -96,8 +92,6 @@ async def Decode(self, request, context): if len(batches) > 1: batch = self.model.batch_type.concatenate(batches) - if torch.cuda.is_available(): - torch.cuda.empty_cache() else: batch = batches[0] From b66b190403120bfcddef438738b7d24a7f6c1536 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 19 Jul 2023 11:59:58 +0200 Subject: [PATCH 329/793] feat(router): ngrok edge (#642) --- launcher/src/main.rs | 49 ++++++++++++++++++-------------------------- router/src/main.rs | 14 +++---------- router/src/server.rs | 21 ++++++------------- 3 files changed, 29 insertions(+), 55 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index c03af4b7..53de36b2 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -267,17 +267,9 @@ struct Args { #[clap(long, env)] ngrok_authtoken: Option, - /// ngrok domain name where the axum webserver will be available at + /// ngrok edge #[clap(long, env)] - ngrok_domain: Option, - - /// ngrok basic auth username - #[clap(long, env)] - ngrok_username: Option, - - /// ngrok basic auth password - #[clap(long, env)] - ngrok_password: Option, + ngrok_edge: Option, /// Display a lot of information about your runtime environment #[clap(long, short, action)] @@ -900,26 +892,11 @@ fn spawn_webserver( // Ngrok if args.ngrok { - let authtoken = args.ngrok_authtoken.ok_or_else(|| { - tracing::error!("`ngrok-authtoken` must be set when using ngrok tunneling"); - LauncherError::WebserverCannotStart - })?; - router_args.push("--ngrok".to_string()); router_args.push("--ngrok-authtoken".to_string()); - router_args.push(authtoken); - - if let Some(domain) = args.ngrok_domain { - router_args.push("--ngrok-domain".to_string()); - router_args.push(domain); - } - - if let (Some(username), Some(password)) = (args.ngrok_username, args.ngrok_password) { - router_args.push("--ngrok-username".to_string()); - router_args.push(username); - router_args.push("--ngrok-password".to_string()); - router_args.push(password); - } + router_args.push(args.ngrok_authtoken.unwrap()); + router_args.push("--ngrok-edge".to_string()); + router_args.push(args.ngrok_edge.unwrap()); } // Copy current process env @@ -997,7 +974,7 @@ fn terminate(process_name: &str, mut process: Child, timeout: Duration) -> io::R fn main() -> Result<(), LauncherError> { // Pattern match configuration - let args = Args::parse(); + let args: Args = Args::parse(); // Filter events with LOG_LEVEL let env_filter = @@ -1067,6 +1044,20 @@ fn main() -> Result<(), LauncherError> { } } + if args.ngrok { + if args.ngrok_authtoken.is_none() { + return Err(LauncherError::ArgumentValidation( + "`ngrok-authtoken` must be set when using ngrok tunneling".to_string(), + )); + } + + if args.ngrok_edge.is_none() { + return Err(LauncherError::ArgumentValidation( + "`ngrok-edge` must be set when using ngrok tunneling".to_string(), + )); + } + } + // Signal handler let running = Arc::new(AtomicBool::new(true)); let r = running.clone(); diff --git a/router/src/main.rs b/router/src/main.rs index 5aef03dd..059f8692 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -64,11 +64,7 @@ struct Args { #[clap(long, env)] ngrok_authtoken: Option, #[clap(long, env)] - ngrok_domain: Option, - #[clap(long, env)] - ngrok_username: Option, - #[clap(long, env)] - ngrok_password: Option, + ngrok_edge: Option, } fn main() -> Result<(), RouterError> { @@ -96,9 +92,7 @@ fn main() -> Result<(), RouterError> { cors_allow_origin, ngrok, ngrok_authtoken, - ngrok_domain, - ngrok_username, - ngrok_password, + ngrok_edge, } = args; // Validate args @@ -274,9 +268,7 @@ fn main() -> Result<(), RouterError> { cors_allow_origin, ngrok, ngrok_authtoken, - ngrok_domain, - ngrok_username, - ngrok_password, + ngrok_edge, ) .await?; Ok(()) diff --git a/router/src/server.rs b/router/src/server.rs index 8ca463c2..bfeee375 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -524,9 +524,7 @@ pub async fn run( allow_origin: Option, ngrok: bool, ngrok_authtoken: Option, - ngrok_domain: Option, - ngrok_username: Option, - ngrok_password: Option, + ngrok_edge: Option, ) -> Result<(), axum::BoxError> { // OpenAPI documentation #[derive(OpenApi)] @@ -696,32 +694,25 @@ pub async fn run( #[cfg(feature = "ngrok")] { use ngrok::config::TunnelBuilder; - use ngrok::tunnel::UrlTunnel; let _ = addr; let authtoken = ngrok_authtoken.expect("`ngrok-authtoken` must be set when using ngrok tunneling"); - let mut tunnel = ngrok::Session::builder() + let edge = ngrok_edge.expect("`ngrok-edge` must be set when using ngrok tunneling"); + + let tunnel = ngrok::Session::builder() .authtoken(authtoken) .connect() .await .unwrap() - .http_endpoint(); - - if let Some(domain) = ngrok_domain { - tunnel = tunnel.domain(domain); - } - - if let (Some(username), Some(password)) = (ngrok_username, ngrok_password) { - tunnel = tunnel.basic_auth(username, password); - } + .labeled_tunnel() + .label("edge", edge); let listener = tunnel.listen().await.unwrap(); // Run server - tracing::info!("Ingress URL: {:?}", listener.url()); axum::Server::builder(listener) .serve(app.into_make_service()) //Wait until all requests are finished to shut down From 1c81df15cd119f0fb9ce3d3c975d035bcdd5cb9a Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 19 Jul 2023 13:38:52 +0200 Subject: [PATCH 330/793] docs: Update README.md (#639) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 4a397bc4..069039a6 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,10 @@ print(text) You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. The Swagger UI is also available at: [https://huggingface.github.io/text-generation-inference](https://huggingface.github.io/text-generation-inference). +### Using on private models or gated models + +You can use `HUGGING_FACE_HUB_TOKEN` environment variable to set the token used by `text-generation-inference` to give access to protected ressources. + ### Distributed Tracing `text-generation-inference` is instrumented with distributed tracing using OpenTelemetry. You can use this feature From 5a1512c0253e759fb07142029127292d639ab117 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 19 Jul 2023 13:39:12 +0200 Subject: [PATCH 331/793] docs: Update README.md (#643) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 069039a6..43388d00 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ sudo apt-get install libssl-dev gcc -y ### CUDA Kernels The custom CUDA kernels are only tested on NVIDIA A100s. If you have any installation or runtime issues, you can remove -the kernels by using the `BUILD_EXTENSIONS=False` environment variable. +the kernels by using the `DISABLE_CUSTOM_KERNELS=True` environment variable. Be aware that the official Docker image has them enabled by default. From 214c06f51005b5be2a06963bb3ad1281abc2bc18 Mon Sep 17 00:00:00 2001 From: cdawg Date: Thu, 20 Jul 2023 13:53:08 +0200 Subject: [PATCH 332/793] Add trust_remote_code to quantize script (#647) # What does this PR do? Fixes a bug appeared with MR #587 fixing issue #552. See the discussion in #552. With MR #587 the trust_remote_code variable is not passed to AutoModelForCausalLM, but is found in the function signature. This prevents models like falcon to be quantized, because trust_remote_code is required. This MR fixes the issue. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [X] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [X] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @Narsil --- server/text_generation_server/utils/gptq/quantize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/gptq/quantize.py b/server/text_generation_server/utils/gptq/quantize.py index d182456f..bee1e446 100644 --- a/server/text_generation_server/utils/gptq/quantize.py +++ b/server/text_generation_server/utils/gptq/quantize.py @@ -864,7 +864,8 @@ def quantize( ) with init_empty_weights(): - model = AutoModelForCausalLM.from_config(config, torch_dtype=torch.float16) + model = AutoModelForCausalLM.from_config(config, torch_dtype=torch.float16, + trust_remote_code=trust_remote_code) model = model.eval() print("LOADED model") From 362883f259f9e07e22ef7b9cc568e2a16cbacc9a Mon Sep 17 00:00:00 2001 From: fxmarty <9808326+fxmarty@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:02:54 +0200 Subject: [PATCH 333/793] fix(server): llama v2 GPTQ (#648) As per title & reported https://github.com/huggingface/text-generation-inference/issues/601#issuecomment-1641435956 https://huggingface.co/TheBloke/Llama-2-70B-chat-GPTQ/discussions/5 Test it: ``` GPTQ_BITS=4 GPTQ_GROUPSIZE=1 text-generation-launcher --model-id TheBloke/Llama-2-70B-chat-GPTQ --port 8080 --num-shard 4 --quantize gptq ``` & ``` curl 127.0.0.1:8080/generate \ -X POST \ -d '{"inputs":"hey llama","parameters":{"max_new_tokens":256}}' \ -H 'Content-Type: application/json' ``` --- .../custom_modeling/flash_llama_modeling.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index b2bde282..84e22f72 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -148,24 +148,27 @@ def forward(self, hidden_states, residual=None): def _load_gqa(config, prefix: str, weights): - w = [ - weights.get_sharded(f"{prefix}.q_proj.weight", dim=0), - weights.get_sharded(f"{prefix}.k_proj.weight", dim=0), - weights.get_sharded(f"{prefix}.v_proj.weight", dim=0), - ] - weight = torch.cat(w, dim=0) - weight = weight.to(dtype=weights.dtype).to(device=weights.device) - bias = None assert config.hidden_size % config.num_attention_heads == 0 - head_size = config.hidden_size // config.num_attention_heads assert config.num_attention_heads % weights.process_group.size() == 0 - num_heads = config.num_attention_heads // weights.process_group.size() - num_key_value_heads = config.num_key_value_heads // weights.process_group.size() - assert list(weight.shape) == [ - (num_heads + 2 * num_key_value_heads) * head_size, - config.hidden_size, - ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}" - return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize)) + + weight = weights.get_multi_weights_col( + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + quantize=config.quantize, + dim=0 + ) + + if config.quantize != "gptq": + weight = weight.to(dtype=weights.dtype).to(device=weights.device) + + head_size = config.hidden_size // config.num_attention_heads + num_heads = config.num_attention_heads // weights.process_group.size() + num_key_value_heads = config.num_key_value_heads // weights.process_group.size() + assert list(weight.shape) == [ + (num_heads + 2 * num_key_value_heads) * head_size, + config.hidden_size, + ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}" + + return TensorParallelColumnLinear(get_linear(weight, bias=None, quantize=config.quantize)) class FlashLlamaAttention(torch.nn.Module): From 08b8eec1d749f402d5d560d3c4cc09916320927b Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 20 Jul 2023 16:04:15 +0200 Subject: [PATCH 334/793] fix(server): Fixing non parameters in quantize script `bigcode/starcoder` was an example. (#661) --- server/text_generation_server/utils/gptq/quantize.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/gptq/quantize.py b/server/text_generation_server/utils/gptq/quantize.py index bee1e446..160c9c9f 100644 --- a/server/text_generation_server/utils/gptq/quantize.py +++ b/server/text_generation_server/utils/gptq/quantize.py @@ -812,10 +812,13 @@ def inner(module, args): tensor = weights.get_tensor(tensor_name) setdeepattr(module, local_param, nn.Parameter(tensor)) else: + tensor = current_tensor.to(device=torch.device("cuda:0")) + if current_tensor.requires_grad: + tensor = nn.Parameter(tensor) setdeepattr( module, local_param, - nn.Parameter(current_tensor.to(device=torch.device("cuda:0"))), + tensor ) return inner From bf94df3c716ce9b8e7194d37cc29d42371736d3b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 20 Jul 2023 17:23:49 +0200 Subject: [PATCH 335/793] fix(server): use mem_get_info to get kv cache size (#664) Close https://github.com/huggingface/text-generation-inference/issues/649 Close https://github.com/huggingface/text-generation-inference/issues/651 Close https://github.com/huggingface/text-generation-inference/issues/653 Close #636 --- .../custom_modeling/flash_llama_modeling.py | 6 ++++-- .../models/flash_causal_lm.py | 15 ++++++--------- .../text_generation_server/utils/gptq/quantize.py | 5 +++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 84e22f72..702bcf61 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -154,7 +154,7 @@ def _load_gqa(config, prefix: str, weights): weight = weights.get_multi_weights_col( prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], quantize=config.quantize, - dim=0 + dim=0, ) if config.quantize != "gptq": @@ -168,7 +168,9 @@ def _load_gqa(config, prefix: str, weights): config.hidden_size, ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}" - return TensorParallelColumnLinear(get_linear(weight, bias=None, quantize=config.quantize)) + return TensorParallelColumnLinear( + get_linear(weight, bias=None, quantize=config.quantize) + ) class FlashLlamaAttention(torch.nn.Module): diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 517fba68..c0592cb0 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -39,6 +39,7 @@ def __init__( device: torch.device, ): self.block_size = BLOCK_SIZE + self.num_blocks = num_blocks element_size = torch.tensor([], dtype=dtype).element_size() x = self.block_size // element_size @@ -714,7 +715,6 @@ def warmup(self, batch: FlashCausalLMBatch): global CACHE_MANAGER torch.cuda.empty_cache() - torch.cuda.reset_peak_memory_stats(self.device) try: CACHE_MANAGER = CacheManager( batch.blocks, @@ -731,23 +731,20 @@ def warmup(self, batch: FlashCausalLMBatch): f"You need to decrease `--max-batch-prefill-tokens`" ) from e - # Inspired by the original implementation in [vllm](https://github.com/vllm-project/vllm) - # Calculate the number of blocks that can be allocated with the - # profiled peak memory. torch.cuda.synchronize(self.device) - peak_memory = torch.cuda.max_memory_reserved(self.device) + # Inspired by the original implementation in [vllm](https://github.com/vllm-project/vllm) + # Calculate the number of blocks that can be allocated with the free memory dtype_size = torch.tensor([], dtype=self.dtype).element_size() cache_block_size = BLOCK_SIZE * self.num_kv_heads * self.head_size total_cache_size = self.num_layers * cache_block_size * 2 * dtype_size - total_gpu_memory = torch.cuda.get_device_properties(self.device).total_memory + free_memory, _ = torch.cuda.mem_get_info(self.device) - # 0.98 to add some wiggle room num_blocks = ( - int((total_gpu_memory * 0.98 - peak_memory) // total_cache_size) + int(free_memory // total_cache_size) # Add batch.blocks as we allocated it above, so it is included in the peak memory. - + batch.blocks + + CACHE_MANAGER.num_blocks ) del CACHE_MANAGER diff --git a/server/text_generation_server/utils/gptq/quantize.py b/server/text_generation_server/utils/gptq/quantize.py index 160c9c9f..45b01ae1 100644 --- a/server/text_generation_server/utils/gptq/quantize.py +++ b/server/text_generation_server/utils/gptq/quantize.py @@ -867,8 +867,9 @@ def quantize( ) with init_empty_weights(): - model = AutoModelForCausalLM.from_config(config, torch_dtype=torch.float16, - trust_remote_code=trust_remote_code) + model = AutoModelForCausalLM.from_config( + config, torch_dtype=torch.float16, trust_remote_code=trust_remote_code + ) model = model.eval() print("LOADED model") From d5b5bc750fd8d00920cb4cb5e5abe121457d717f Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 21 Jul 2023 10:59:00 +0200 Subject: [PATCH 336/793] feat(server): Add exllama GPTQ CUDA kernel support #553 (#666) Just trying to get the integration tests to pass. # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Felix Marty <9808326+fxmarty@users.noreply.github.com> --- Dockerfile | 13 + Makefile | 3 + integration-tests/conftest.py | 11 +- .../test_flash_llama_gptq.json | 88 +++ .../test_flash_llama_gptq_all_params.json | 88 +++ .../test_flash_llama_gptq_load.json | 354 ++++++++++++ .../test_flash_starcoder_gptq.json | 193 +++++++ ...t_flash_starcoder_gptq_default_params.json | 193 +++++++ .../test_flash_starcoder_gptq_load.json | 534 ++++++++++++++++++ .../models/test_flash_llama_gptq.py | 57 ++ .../models/test_flash_starcoder_gptq.py | 49 ++ .../exllama_kernels/cuda_buffers.cu | 71 +++ .../exllama_kernels/cuda_buffers.cuh | 52 ++ .../exllama_kernels/cuda_compat.cuh | 58 ++ .../exllama_kernels/cuda_func/column_remap.cu | 61 ++ .../cuda_func/column_remap.cuh | 19 + .../exllama_kernels/cuda_func/q4_matmul.cu | 252 +++++++++ .../exllama_kernels/cuda_func/q4_matmul.cuh | 37 ++ .../exllama_kernels/cuda_func/q4_matrix.cu | 217 +++++++ .../exllama_kernels/cuda_func/q4_matrix.cuh | 53 ++ .../exllama_kernels/exllama_ext.cpp | 249 ++++++++ .../exllama_kernels/matrix.cuh | 294 ++++++++++ .../exllama_kernels/exllama_kernels/tuning.h | 13 + .../exllama_kernels/exllama_kernels/util.cuh | 29 + server/exllama_kernels/setup.py | 19 + .../custom_modeling/flash_llama_modeling.py | 1 - .../flash_santacoder_modeling.py | 23 +- .../models/flash_causal_lm.py | 1 - server/text_generation_server/models/model.py | 3 +- server/text_generation_server/server.py | 11 + .../utils/gptq/exllama.py | 121 ++++ server/text_generation_server/utils/layers.py | 32 +- .../text_generation_server/utils/weights.py | 102 ++-- 33 files changed, 3235 insertions(+), 66 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json create mode 100644 integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json create mode 100644 integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json create mode 100644 integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json create mode 100644 integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json create mode 100644 integration-tests/models/test_flash_llama_gptq.py create mode 100644 integration-tests/models/test_flash_starcoder_gptq.py create mode 100644 server/exllama_kernels/exllama_kernels/cuda_buffers.cu create mode 100644 server/exllama_kernels/exllama_kernels/cuda_buffers.cuh create mode 100644 server/exllama_kernels/exllama_kernels/cuda_compat.cuh create mode 100644 server/exllama_kernels/exllama_kernels/cuda_func/column_remap.cu create mode 100644 server/exllama_kernels/exllama_kernels/cuda_func/column_remap.cuh create mode 100644 server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu create mode 100644 server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cuh create mode 100644 server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu create mode 100644 server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cuh create mode 100644 server/exllama_kernels/exllama_kernels/exllama_ext.cpp create mode 100644 server/exllama_kernels/exllama_kernels/matrix.cuh create mode 100644 server/exllama_kernels/exllama_kernels/tuning.h create mode 100644 server/exllama_kernels/exllama_kernels/util.cuh create mode 100644 server/exllama_kernels/setup.py create mode 100644 server/text_generation_server/utils/gptq/exllama.py diff --git a/Dockerfile b/Dockerfile index 168f2f97..70c60132 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,6 +108,17 @@ COPY server/Makefile-flash-att-v2 Makefile # Build specific version of flash attention v2 RUN make build-flash-attention-v2 +# Build Transformers exllama kernels +FROM kernel-builder as exllama-kernels-builder + +WORKDIR /usr/src + +COPY server/exllama_kernels/ . + + +# Build specific version of transformers +RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" python setup.py build + # Build Transformers CUDA kernels FROM kernel-builder as custom-kernels-builder @@ -161,6 +172,8 @@ COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86 # Copy build artifacts from custom kernels builder COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +# Copy build artifacts from exllama kernels builder +COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages # Copy builds artifacts from vllm builder COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages diff --git a/Makefile b/Makefile index 3c2f2b9d..81b312d1 100644 --- a/Makefile +++ b/Makefile @@ -56,3 +56,6 @@ run-bloom: run-bloom-quantize: text-generation-launcher --model-id bigscience/bloom --num-shard 8 --quantize --port 8080 + +clean: + rm -rf target aml diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 8f59d75a..3f7a24dd 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -230,15 +230,16 @@ def local_launcher( shard_uds_path, ] + env = os.environ + if num_shard is not None: args.extend(["--num-shard", str(num_shard)]) - if quantize: + if quantize is not None: args.append("--quantize") - args.append("bitsandbytes") + args.append(quantize) if trust_remote_code: args.append("--trust-remote-code") - env = os.environ env["LOG_LEVEL"] = "info,text_generation_router=debug" if not use_flash_attention: @@ -275,9 +276,9 @@ def docker_launcher( if num_shard is not None: args.extend(["--num-shard", str(num_shard)]) - if quantize: + if quantize is not None: args.append("--quantize") - args.append("bitsandbytes") + args.append(quantize) if trust_remote_code: args.append("--trust-remote-code") diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json new file mode 100644 index 00000000..e4ffb83b --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json @@ -0,0 +1,88 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -9.59375, + "text": "Test" + }, + { + "id": 2009, + "logprob": -9.6640625, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 29918, + "logprob": -2.3867188, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -2.8183594, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -1.6367188, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -1.0527344, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.6542969, + "special": false, + "text": " request" + }, + { + "id": 29918, + "logprob": -0.056121826, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -0.01600647, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -0.87939453, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -0.7529297, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.2980957, + "special": false, + "text": " request" + } + ] + }, + "generated_text": "_uri\nTest request_uri\nTest request" +} diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json new file mode 100644 index 00000000..02713a00 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json @@ -0,0 +1,88 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -9.6015625, + "text": "Test" + }, + { + "id": 2009, + "logprob": -9.6640625, + "text": "request" + } + ], + "seed": 0, + "tokens": [ + { + "id": 29899, + "logprob": -1.1640625, + "special": false, + "text": "-" + }, + { + "id": 1454, + "logprob": -0.07543945, + "special": false, + "text": "for" + }, + { + "id": 29899, + "logprob": 0.0, + "special": false, + "text": "-" + }, + { + "id": 9342, + "logprob": 0.0, + "special": false, + "text": "comment" + }, + { + "id": 29901, + "logprob": 0.0, + "special": false, + "text": ":" + }, + { + "id": 396, + "logprob": -0.2956543, + "special": false, + "text": " #" + }, + { + "id": 29906, + "logprob": -0.52734375, + "special": false, + "text": "2" + }, + { + "id": 29900, + "logprob": -0.6899414, + "special": false, + "text": "0" + }, + { + "id": 29896, + "logprob": 0.0, + "special": false, + "text": "1" + }, + { + "id": 29946, + "logprob": -1.5068359, + "special": false, + "text": "4" + } + ] + }, + "generated_text": "Test request-for-comment: #2014" +} diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json new file mode 100644 index 00000000..88bfa4f9 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json @@ -0,0 +1,354 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -9.6015625, + "text": "Test" + }, + { + "id": 2009, + "logprob": -9.671875, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 29918, + "logprob": -2.3828125, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -2.8105469, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -1.6396484, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -1.0546875, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.6513672, + "special": false, + "text": " request" + }, + { + "id": 29918, + "logprob": -0.056365967, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -0.016082764, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -0.87841797, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -0.7548828, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.29711914, + "special": false, + "text": " request" + } + ] + }, + "generated_text": "_uri\nTest request_uri\nTest request" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -9.6015625, + "text": "Test" + }, + { + "id": 2009, + "logprob": -9.6640625, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 29918, + "logprob": -2.3828125, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -2.828125, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -1.6386719, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -1.0527344, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.6542969, + "special": false, + "text": " request" + }, + { + "id": 29918, + "logprob": -0.055877686, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -0.016021729, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -0.8769531, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -0.7583008, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.29833984, + "special": false, + "text": " request" + } + ] + }, + "generated_text": "_uri\nTest request_uri\nTest request" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -9.6015625, + "text": "Test" + }, + { + "id": 2009, + "logprob": -9.671875, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 29918, + "logprob": -2.3847656, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -2.8144531, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -1.6396484, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -1.0527344, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.65478516, + "special": false, + "text": " request" + }, + { + "id": 29918, + "logprob": -0.056243896, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -0.016143799, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -0.8808594, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -0.75341797, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.2956543, + "special": false, + "text": " request" + } + ] + }, + "generated_text": "_uri\nTest request_uri\nTest request" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -9.6015625, + "text": "Test" + }, + { + "id": 2009, + "logprob": -9.6640625, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 29918, + "logprob": -2.3769531, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -2.8183594, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -1.6396484, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -1.0546875, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.65478516, + "special": false, + "text": " request" + }, + { + "id": 29918, + "logprob": -0.05557251, + "special": false, + "text": "_" + }, + { + "id": 5338, + "logprob": -0.01612854, + "special": false, + "text": "uri" + }, + { + "id": 13, + "logprob": -0.8730469, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -0.7519531, + "special": false, + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.29785156, + "special": false, + "text": " request" + } + ] + }, + "generated_text": "_uri\nTest request_uri\nTest request" + } +] diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json new file mode 100644 index 00000000..53055e42 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json @@ -0,0 +1,193 @@ +{ + "generated_text": "\n return sum(L) / len(L)\n\n\ndef geometric_mean(L", + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 20, + "seed": null, + "prefill": [ + { + "id": 589, + "text": "def", + "logprob": null + }, + { + "id": 3226, + "text": " ge", + "logprob": -9.0234375 + }, + { + "id": 21017, + "text": "ometric", + "logprob": -9.0859375 + }, + { + "id": 81, + "text": "_", + "logprob": -0.25878906 + }, + { + "id": 6009, + "text": "mean", + "logprob": -2.2109375 + }, + { + "id": 26, + "text": "(", + "logprob": -0.30371094 + }, + { + "id": 62, + "text": "L", + "logprob": -5.6054688 + }, + { + "id": 44, + "text": ":", + "logprob": -3.0722656 + }, + { + "id": 1682, + "text": " List", + "logprob": -0.6879883 + }, + { + "id": 77, + "text": "[", + "logprob": -0.38500977 + }, + { + "id": 1808, + "text": "float", + "logprob": -0.984375 + }, + { + "id": 10794, + "text": "]):", + "logprob": -2.5351562 + } + ], + "tokens": [ + { + "id": 284, + "text": "\n ", + "logprob": -1.1738281, + "special": false + }, + { + "id": 442, + "text": " return", + "logprob": -0.95947266, + "special": false + }, + { + "id": 3632, + "text": " sum", + "logprob": -1.4199219, + "special": false + }, + { + "id": 26, + "text": "(", + "logprob": -0.085876465, + "special": false + }, + { + "id": 62, + "text": "L", + "logprob": -0.09875488, + "special": false + }, + { + "id": 27, + "text": ")", + "logprob": -0.30517578, + "special": false + }, + { + "id": 517, + "text": " /", + "logprob": -0.42089844, + "special": false + }, + { + "id": 2069, + "text": " len", + "logprob": -0.042053223, + "special": false + }, + { + "id": 26, + "text": "(", + "logprob": -0.0011806488, + "special": false + }, + { + "id": 62, + "text": "L", + "logprob": -0.0005259514, + "special": false + }, + { + "id": 27, + "text": ")", + "logprob": -0.0017633438, + "special": false + }, + { + "id": 478, + "text": "\n\n", + "logprob": -0.69189453, + "special": false + }, + { + "id": 203, + "text": "\n", + "logprob": -0.041870117, + "special": false + }, + { + "id": 589, + "text": "def", + "logprob": -0.27856445, + "special": false + }, + { + "id": 3226, + "text": " ge", + "logprob": -1.7255859, + "special": false + }, + { + "id": 21017, + "text": "ometric", + "logprob": -0.011291504, + "special": false + }, + { + "id": 81, + "text": "_", + "logprob": -0.008430481, + "special": false + }, + { + "id": 6009, + "text": "mean", + "logprob": -0.025787354, + "special": false + }, + { + "id": 26, + "text": "(", + "logprob": -0.073913574, + "special": false + }, + { + "id": 62, + "text": "L", + "logprob": -0.09967041, + "special": false + } + ] + } +} diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json new file mode 100644 index 00000000..5598a2ad --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json @@ -0,0 +1,193 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 20, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 3226, + "logprob": -9.0234375, + "text": " ge" + }, + { + "id": 21017, + "logprob": -9.09375, + "text": "ometric" + }, + { + "id": 81, + "logprob": -0.25976562, + "text": "_" + }, + { + "id": 6009, + "logprob": -2.2148438, + "text": "mean" + }, + { + "id": 26, + "logprob": -0.3010254, + "text": "(" + }, + { + "id": 62, + "logprob": -5.6757812, + "text": "L" + }, + { + "id": 44, + "logprob": -3.0898438, + "text": ":" + }, + { + "id": 1682, + "logprob": -0.6791992, + "text": " List" + }, + { + "id": 77, + "logprob": -0.38891602, + "text": "[" + }, + { + "id": 1808, + "logprob": -0.92041016, + "text": "float" + }, + { + "id": 10794, + "logprob": -2.5390625, + "text": "]):" + } + ], + "seed": 0, + "tokens": [ + { + "id": 284, + "logprob": 0.0, + "special": false, + "text": "\n " + }, + { + "id": 442, + "logprob": 0.0, + "special": false, + "text": " return" + }, + { + "id": 11665, + "logprob": -1.6005859, + "special": false, + "text": " reduce" + }, + { + "id": 26, + "logprob": 0.0, + "special": false, + "text": "(" + }, + { + "id": 5962, + "logprob": 0.0, + "special": false, + "text": "lambda" + }, + { + "id": 816, + "logprob": 0.0, + "special": false, + "text": " x" + }, + { + "id": 30, + "logprob": 0.0, + "special": false, + "text": "," + }, + { + "id": 533, + "logprob": 0.0, + "special": false, + "text": " y" + }, + { + "id": 44, + "logprob": 0.0, + "special": false, + "text": ":" + }, + { + "id": 816, + "logprob": 0.0, + "special": false, + "text": " x" + }, + { + "id": 319, + "logprob": 0.0, + "special": false, + "text": " *" + }, + { + "id": 533, + "logprob": 0.0, + "special": false, + "text": " y" + }, + { + "id": 30, + "logprob": 0.0, + "special": false, + "text": "," + }, + { + "id": 498, + "logprob": 0.0, + "special": false, + "text": " L" + }, + { + "id": 27, + "logprob": 0.0, + "special": false, + "text": ")" + }, + { + "id": 203, + "logprob": -0.11968994, + "special": false, + "text": "\n" + }, + { + "id": 203, + "logprob": 0.0, + "special": false, + "text": "\n" + }, + { + "id": 589, + "logprob": 0.0, + "special": false, + "text": "def" + }, + { + "id": 3226, + "logprob": 0.0, + "special": false, + "text": " ge" + }, + { + "id": 21017, + "logprob": 0.0, + "special": false, + "text": "ometric" + } + ] + }, + "generated_text": "\n return reduce(lambda x, y: x * y, L)\n\ndef geometric" +} diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json new file mode 100644 index 00000000..5381ce5a --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json @@ -0,0 +1,534 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 3226, + "logprob": -9.0234375, + "text": " ge" + }, + { + "id": 21017, + "logprob": -9.0859375, + "text": "ometric" + }, + { + "id": 81, + "logprob": -0.25927734, + "text": "_" + }, + { + "id": 6009, + "logprob": -2.25, + "text": "mean" + }, + { + "id": 26, + "logprob": -0.30126953, + "text": "(" + }, + { + "id": 62, + "logprob": -5.7539062, + "text": "L" + }, + { + "id": 44, + "logprob": -3.0878906, + "text": ":" + }, + { + "id": 1682, + "logprob": -0.6845703, + "text": " List" + }, + { + "id": 77, + "logprob": -0.3918457, + "text": "[" + }, + { + "id": 1808, + "logprob": -0.8798828, + "text": "float" + }, + { + "id": 10794, + "logprob": -2.4980469, + "text": "]):" + } + ], + "seed": null, + "tokens": [ + { + "id": 284, + "logprob": -1.1533203, + "special": false, + "text": "\n " + }, + { + "id": 442, + "logprob": -0.91796875, + "special": false, + "text": " return" + }, + { + "id": 3632, + "logprob": -1.3291016, + "special": false, + "text": " sum" + }, + { + "id": 26, + "logprob": -0.08062744, + "special": false, + "text": "(" + }, + { + "id": 62, + "logprob": -0.097717285, + "special": false, + "text": "L" + }, + { + "id": 27, + "logprob": -0.29003906, + "special": false, + "text": ")" + }, + { + "id": 517, + "logprob": -0.34958984, + "special": false, + "text": " /" + }, + { + "id": 2069, + "logprob": -0.03829956, + "special": false, + "text": " len" + }, + { + "id": 26, + "logprob": -0.0011987686, + "special": false, + "text": "(" + }, + { + "id": 62, + "logprob": -0.00050878525, + "special": false, + "text": "L" + } + ] + }, + "generated_text": "\n return sum(L) / len(L" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 3226, + "logprob": -9.0234375, + "text": " ge" + }, + { + "id": 21017, + "logprob": -9.0859375, + "text": "ometric" + }, + { + "id": 81, + "logprob": -0.25878906, + "text": "_" + }, + { + "id": 6009, + "logprob": -2.2109375, + "text": "mean" + }, + { + "id": 26, + "logprob": -0.30371094, + "text": "(" + }, + { + "id": 62, + "logprob": -5.6054688, + "text": "L" + }, + { + "id": 44, + "logprob": -3.0722656, + "text": ":" + }, + { + "id": 1682, + "logprob": -0.6879883, + "text": " List" + }, + { + "id": 77, + "logprob": -0.38500977, + "text": "[" + }, + { + "id": 1808, + "logprob": -0.984375, + "text": "float" + }, + { + "id": 10794, + "logprob": -2.5351562, + "text": "]):" + } + ], + "seed": null, + "tokens": [ + { + "id": 284, + "logprob": -1.1738281, + "special": false, + "text": "\n " + }, + { + "id": 442, + "logprob": -0.9584961, + "special": false, + "text": " return" + }, + { + "id": 3632, + "logprob": -1.4169922, + "special": false, + "text": " sum" + }, + { + "id": 26, + "logprob": -0.085876465, + "special": false, + "text": "(" + }, + { + "id": 62, + "logprob": -0.0982666, + "special": false, + "text": "L" + }, + { + "id": 27, + "logprob": -0.3022461, + "special": false, + "text": ")" + }, + { + "id": 517, + "logprob": -0.40504883, + "special": false, + "text": " /" + }, + { + "id": 2069, + "logprob": -0.041656494, + "special": false, + "text": " len" + }, + { + "id": 26, + "logprob": -0.0011844635, + "special": false, + "text": "(" + }, + { + "id": 62, + "logprob": -0.0005264282, + "special": false, + "text": "L" + } + ] + }, + "generated_text": "\n return sum(L) / len(L" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 3226, + "logprob": -9.0234375, + "text": " ge" + }, + { + "id": 21017, + "logprob": -9.0859375, + "text": "ometric" + }, + { + "id": 81, + "logprob": -0.25927734, + "text": "_" + }, + { + "id": 6009, + "logprob": -2.25, + "text": "mean" + }, + { + "id": 26, + "logprob": -0.30126953, + "text": "(" + }, + { + "id": 62, + "logprob": -5.7539062, + "text": "L" + }, + { + "id": 44, + "logprob": -3.0878906, + "text": ":" + }, + { + "id": 1682, + "logprob": -0.6845703, + "text": " List" + }, + { + "id": 77, + "logprob": -0.3918457, + "text": "[" + }, + { + "id": 1808, + "logprob": -0.8798828, + "text": "float" + }, + { + "id": 10794, + "logprob": -2.4980469, + "text": "]):" + } + ], + "seed": null, + "tokens": [ + { + "id": 284, + "logprob": -1.1533203, + "special": false, + "text": "\n " + }, + { + "id": 442, + "logprob": -0.9165039, + "special": false, + "text": " return" + }, + { + "id": 3632, + "logprob": -1.328125, + "special": false, + "text": " sum" + }, + { + "id": 26, + "logprob": -0.07946777, + "special": false, + "text": "(" + }, + { + "id": 62, + "logprob": -0.09820557, + "special": false, + "text": "L" + }, + { + "id": 27, + "logprob": -0.28930664, + "special": false, + "text": ")" + }, + { + "id": 517, + "logprob": -0.34592773, + "special": false, + "text": " /" + }, + { + "id": 2069, + "logprob": -0.038330078, + "special": false, + "text": " len" + }, + { + "id": 26, + "logprob": -0.0011940002, + "special": false, + "text": "(" + }, + { + "id": 62, + "logprob": -0.00050878525, + "special": false, + "text": "L" + } + ] + }, + "generated_text": "\n return sum(L) / len(L" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 589, + "logprob": null, + "text": "def" + }, + { + "id": 3226, + "logprob": -9.0234375, + "text": " ge" + }, + { + "id": 21017, + "logprob": -9.0859375, + "text": "ometric" + }, + { + "id": 81, + "logprob": -0.25927734, + "text": "_" + }, + { + "id": 6009, + "logprob": -2.25, + "text": "mean" + }, + { + "id": 26, + "logprob": -0.30126953, + "text": "(" + }, + { + "id": 62, + "logprob": -5.7539062, + "text": "L" + }, + { + "id": 44, + "logprob": -3.0878906, + "text": ":" + }, + { + "id": 1682, + "logprob": -0.6845703, + "text": " List" + }, + { + "id": 77, + "logprob": -0.3918457, + "text": "[" + }, + { + "id": 1808, + "logprob": -0.8798828, + "text": "float" + }, + { + "id": 10794, + "logprob": -2.4980469, + "text": "]):" + } + ], + "seed": null, + "tokens": [ + { + "id": 284, + "logprob": -1.1533203, + "special": false, + "text": "\n " + }, + { + "id": 442, + "logprob": -0.91259766, + "special": false, + "text": " return" + }, + { + "id": 3632, + "logprob": -1.3251953, + "special": false, + "text": " sum" + }, + { + "id": 26, + "logprob": -0.08062744, + "special": false, + "text": "(" + }, + { + "id": 62, + "logprob": -0.09906006, + "special": false, + "text": "L" + }, + { + "id": 27, + "logprob": -0.28979492, + "special": false, + "text": ")" + }, + { + "id": 517, + "logprob": -0.35958984, + "special": false, + "text": " /" + }, + { + "id": 2069, + "logprob": -0.038604736, + "special": false, + "text": " len" + }, + { + "id": 26, + "logprob": -0.0011901855, + "special": false, + "text": "(" + }, + { + "id": 62, + "logprob": -0.0005078316, + "special": false, + "text": "L" + } + ] + }, + "generated_text": "\n return sum(L) / len(L" + } +] diff --git a/integration-tests/models/test_flash_llama_gptq.py b/integration-tests/models/test_flash_llama_gptq.py new file mode 100644 index 00000000..bc525f6d --- /dev/null +++ b/integration-tests/models/test_flash_llama_gptq.py @@ -0,0 +1,57 @@ +import pytest + + +@pytest.fixture(scope="module") +def flash_llama_gptq_handle(launcher): + with launcher("huggingface/llama-7b-gptq", num_shard=2, quantize="gptq") as handle: + yield handle + + +@pytest.fixture(scope="module") +async def flash_llama_gptq(flash_llama_gptq_handle): + await flash_llama_gptq_handle.health(300) + return flash_llama_gptq_handle.client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_gptq(flash_llama_gptq, response_snapshot): + response = await flash_llama_gptq.generate( + "Test request", max_new_tokens=10, decoder_input_details=True + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_gptq_all_params(flash_llama_gptq, response_snapshot): + response = await flash_llama_gptq.generate( + "Test request", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + decoder_input_details=True, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_gptq_load(flash_llama_gptq, generate_load, response_snapshot): + responses = await generate_load(flash_llama_gptq, "Test request", max_new_tokens=10, n=4) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) + + assert responses == response_snapshot diff --git a/integration-tests/models/test_flash_starcoder_gptq.py b/integration-tests/models/test_flash_starcoder_gptq.py new file mode 100644 index 00000000..b6bed6a6 --- /dev/null +++ b/integration-tests/models/test_flash_starcoder_gptq.py @@ -0,0 +1,49 @@ +import pytest + + +@pytest.fixture(scope="module") +def flash_starcoder_gptq_handle(launcher): + with launcher("Narsil/starcoder-gptq", num_shard=2, quantize="gptq") as handle: + yield handle + + +@pytest.fixture(scope="module") +async def flash_starcoder_gptq(flash_starcoder_gptq_handle): + await flash_starcoder_gptq_handle.health(300) + return flash_starcoder_gptq_handle.client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_starcoder_gptq(flash_starcoder_gptq, response_snapshot): + response = await flash_starcoder_gptq.generate( + "def geometric_mean(L: List[float]):", max_new_tokens=20, decoder_input_details=True, + ) + assert response.details.generated_tokens == 20 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_starcoder_gptq_default_params(flash_starcoder_gptq, response_snapshot): + response = await flash_starcoder_gptq.generate( + "def geometric_mean(L: List[float]):", + max_new_tokens=20, + temperature=0.2, + top_p=0.95, + decoder_input_details=True, + seed=0, + ) + assert response.details.generated_tokens == 20 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_starcoder_gptq_load(flash_starcoder_gptq, generate_load, response_snapshot): + responses = await generate_load(flash_starcoder_gptq, "def geometric_mean(L: List[float]):", max_new_tokens=10, n=4) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) + + assert responses == response_snapshot \ No newline at end of file diff --git a/server/exllama_kernels/exllama_kernels/cuda_buffers.cu b/server/exllama_kernels/exllama_kernels/cuda_buffers.cu new file mode 100644 index 00000000..ee2cbee2 --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/cuda_buffers.cu @@ -0,0 +1,71 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#define _cuda_buffers_cu +#include "cuda_buffers.cuh" + +CudaBuffers* g_buffers[CUDA_MAX_DEVICES] = {NULL}; +// __constant__ half2 q4_table[16][256]; +// half2 q4_table_host[16][256]; +// bool q4_table_init = false; + +CudaBuffers::CudaBuffers +( + int _device, + half* _temp_state, + half* _temp_dq +) : + device(_device), + temp_state(_temp_state), + temp_dq(_temp_dq) +{ + cudaSetDevice(_device); + + cudaStreamCreate(&alt_stream_1); + cudaStreamCreate(&alt_stream_2); + cudaStreamCreate(&alt_stream_3); + cudaEventCreate(&alt_stream_1_done); + cudaEventCreate(&alt_stream_2_done); + cudaEventCreate(&alt_stream_3_done); +} + +CudaBuffers::~CudaBuffers() +{ + cudaStreamDestroy(alt_stream_1); + cudaStreamDestroy(alt_stream_2); + cudaStreamDestroy(alt_stream_3); + cudaEventDestroy(alt_stream_1_done); + cudaEventDestroy(alt_stream_2_done); + cudaEventDestroy(alt_stream_3_done); +} + +CudaBuffers* get_buffers(const int device_index) +{ + return g_buffers[device_index]; +} + +void prepare_buffers_cuda +( + int _device, + half* _temp_state, + half* _temp_dq +) +{ + CudaBuffers* buffers = new CudaBuffers + ( + _device, + _temp_state, + _temp_dq + ); + + g_buffers[_device] = buffers; +} + +void cleanup_buffers_cuda() +{ + for (int i = 0; i < CUDA_MAX_DEVICES; i++) + { + if (!g_buffers[i]) continue; + delete g_buffers[i]; + g_buffers[i] = NULL; + } +} diff --git a/server/exllama_kernels/exllama_kernels/cuda_buffers.cuh b/server/exllama_kernels/exllama_kernels/cuda_buffers.cuh new file mode 100644 index 00000000..afb60a01 --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/cuda_buffers.cuh @@ -0,0 +1,52 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#ifndef _cuda_buffers_cuh +#define _cuda_buffers_cuh + +#include +#include +#include +#include + +const int CUDA_MAX_DEVICES = 16; + +// #ifndef _cuda_buffers_cu +// extern __constant__ half2 q4_table[16][256]; +// #endif + +class CudaBuffers +{ +public: + int device; + + half* temp_state; // [max_hidden_rows * intermediate_size] + half* temp_dq; // size of largest quant tensor * 8 + + cudaStream_t alt_stream_1; + cudaStream_t alt_stream_2; + cudaStream_t alt_stream_3; + cudaEvent_t alt_stream_1_done; + cudaEvent_t alt_stream_2_done; + cudaEvent_t alt_stream_3_done; + + CudaBuffers + ( + int _device, + half* _temp_state, + half* _temp_dq + ); + ~CudaBuffers(); +}; + +CudaBuffers* get_buffers(const int device_index); + +void prepare_buffers_cuda +( + int _device, + half* _temp_state, + half* _temp_dq +); + +void cleanup_buffers_cuda(); + +#endif diff --git a/server/exllama_kernels/exllama_kernels/cuda_compat.cuh b/server/exllama_kernels/exllama_kernels/cuda_compat.cuh new file mode 100644 index 00000000..8dfa25de --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/cuda_compat.cuh @@ -0,0 +1,58 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#ifndef _cuda_compat_cuh +#define _cuda_compat_cuh + +// atomicAdd for half types, to support CC < 7.x + +__device__ __forceinline__ void atomicAdd_half(half* address, half val) +{ + unsigned int * address_as_ui = (unsigned int *) ((char *)address - ((size_t)address & 2)); + unsigned int old = *address_as_ui; + unsigned int assumed; + + do + { + assumed = old; + __half_raw hsum; + hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff); + half tmpres = __hadd(hsum, val); + hsum = __half_raw(tmpres); + old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x; + old = atomicCAS(address_as_ui, assumed, old); + } + while (assumed != old); +} + +// atomicAdd for half2 types + +__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val) +{ + unsigned int* address_as_ui = (unsigned int*)address; + unsigned int old = *address_as_ui; + unsigned int assumed; + do + { + assumed = old; + half2 old_val = *((half2*)&old); + half2 new_val = __hadd2(old_val, val); + old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val)); + } + while (assumed != old); +} + +// + +#if defined(__CUDA_ARCH__) +#if __CUDA_ARCH__ < 700 + +__device__ __forceinline__ void atomicAdd(half* address, half val) { atomicAdd_half(address, val); } + +#if __CUDA_ARCH__ < 600 +__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { atomicAdd_half2(address, val); } +#endif + +#endif +#endif + +#endif diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/column_remap.cu b/server/exllama_kernels/exllama_kernels/cuda_func/column_remap.cu new file mode 100644 index 00000000..c25b0206 --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/cuda_func/column_remap.cu @@ -0,0 +1,61 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#include "column_remap.cuh" +#include "../util.cuh" + +const int SHUF_BLOCKSIZE_X = 256; +const int SHUF_BLOCKSIZE_Y = 16; + +__global__ void column_remap_kernel +( + const half* __restrict__ x, + half* __restrict__ x_new, + const int x_width, + const int x_height, + const uint32_t* x_map +) +{ + int x_column = SHUF_BLOCKSIZE_X * blockIdx.x + threadIdx.x; + int x_row = SHUF_BLOCKSIZE_Y * blockIdx.y; + + int x_stride = x_width; + int x_idx = x_row * x_stride + x_column; + + int x_row_end = min(x_row + SHUF_BLOCKSIZE_Y, x_height); + int x_idx_end = x_row_end * x_stride + x_column; + + int s_column = x_map[x_column]; + int s_idx = x_row * x_stride + s_column; + + while (x_idx < x_idx_end) + { + x_new[x_idx] = x[s_idx]; + x_idx += x_stride; + s_idx += x_stride; + } +} + +// Remap columns in x to correspond to sequential group index before matmul +// +// perform x -> seq_x such that seq_x @ seq_w == x @ w + +void column_remap_cuda +( + const half* x, + half* x_new, + const int x_height, + const int x_width, + const uint32_t* x_map +) +{ + dim3 threads(SHUF_BLOCKSIZE_X, 1, 1); + + dim3 blocks + ( + (x_width + SHUF_BLOCKSIZE_X - 1) / SHUF_BLOCKSIZE_X, + (x_height + SHUF_BLOCKSIZE_Y - 1) / SHUF_BLOCKSIZE_Y, + 1 + ); + + column_remap_kernel<<>>(x, x_new, x_width, x_height, x_map); +} diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/column_remap.cuh b/server/exllama_kernels/exllama_kernels/cuda_func/column_remap.cuh new file mode 100644 index 00000000..6571c17d --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/cuda_func/column_remap.cuh @@ -0,0 +1,19 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#ifndef _column_remap_cuh +#define _column_remap_cuh + +#include +#include +#include + +void column_remap_cuda +( + const half* x, + half* x_new, + const int x_height, + const int x_width, + const uint32_t* x_map +); + +#endif \ No newline at end of file diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu new file mode 100644 index 00000000..60dc4c9d --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu @@ -0,0 +1,252 @@ +#include "q4_matmul.cuh" +#include "column_remap.cuh" +#include "../util.cuh" +#include "../matrix.cuh" +#include "../cuda_compat.cuh" +#include "../cuda_buffers.cuh" + +const int THREADS_X = 32; // Block size and thread count along columns in w and out +const int THREADS_Y = 1; // Block size and thread count along rows in x and out + +typedef void (*fp_q4_matmul_kernel) +( + const half*, + const uint32_t*, + half*, + const half*, + const uint32_t*, + const int, + const int, + const int, + const int, + const int, + const uint32_t*, + bool +); + +template +__global__ void q4_matmul_kernel +( + const half* __restrict__ x, + const uint32_t* __restrict__ w, + half* __restrict__ out, + const half* __restrict__ w_scales, + const uint32_t* __restrict__ w_zeros, + const int height, + const int dim, + const int width, + const int groupsize, + const int block_size_z, + const uint32_t* __restrict__ x_map, + bool no_zero +) +{ + // Start of block + + int x_column = block_size_z * blockIdx.z; + int x_column_end = min(dim, block_size_z * (blockIdx.z + 1)); + + int w_column = THREADS_X * blockIdx.x + threadIdx.x; + int x_row = THREADS_Y * blockIdx.y + threadIdx.y; + + int iterations = (x_column_end - x_column) / 8; + + // Views + + MatrixView_half x_(x, height, dim); + MatrixView_half w_scales_(w_scales, dim / groupsize, width); + MatrixView_q4_row w_zeros_(w_zeros, dim / groupsize, width); + MatrixView_q4_column w_(w, dim, width); + MatrixView_half_rw out_(out, height, width); + + // Zero output + + if (!no_zero && blockIdx.z == 0 && (threadIdx.x & 1) == 0) + { + *((uint32_t*) out_.item_ptr(x_row, w_column)) = 0; + __syncthreads(); + } + + // Loop over part of x row (and w column) + + half2 acc = {}; + half acc_h = {}; + + if constexpr (use_groupsize) + { + // For quant matrices where groupsize divides BLOCK_SIZE_Z we always start on a group boundary, so this + // could be slightly faster + + for (int k = x_column, group = x_column / groupsize; k < x_column + iterations * 8; group++, k += groupsize) + { + if constexpr (use_half2) + { + half2 w_scale = w_scales_.item_half2half2(group, w_column); + uint32_t w_zero = w_zeros_.item(group, w_column) + 1; + + if constexpr (use_x_map) acc = dot_product_8_x_map(acc, x_, x_row, k, w_, k, w_column, w_scale, w_zero, groupsize / 8, x_map); + else acc = dot_product_8 (acc, x_, x_row, k, w_, k, w_column, w_scale, w_zero, groupsize / 8); + } + else + { + half w_scale = w_scales_.item(group, w_column); + uint32_t w_zero = w_zeros_.item(group, w_column) + 1; + + if constexpr (use_x_map) acc_h = dot_product_8_x_map_h(acc_h, x_, x_row, k, w_, k, w_column, w_scale, w_zero, groupsize / 8, x_map); + else acc_h = dot_product_8_h (acc_h, x_, x_row, k, w_, k, w_column, w_scale, w_zero, groupsize / 8); + } + } + } + else + { + // Otherwise assume groupsize is a multiple of 8, do 8 columns per iteration and trust the cache + + for (int k = x_column; k < x_column + iterations * 8; k += 8) + { + if constexpr (use_half2) + { + int group = k / groupsize; + half2 w_scale = w_scales_.item_half2half2(group, w_column); + uint32_t w_zero = w_zeros_.item(group, w_column) + 1; + + if constexpr (use_x_map) acc = dot_product_8_x_map(acc, x_, x_row, k, w_, k, w_column, w_scale, w_zero, 1, x_map); + else acc = dot_product_8 (acc, x_, x_row, k, w_, k, w_column, w_scale, w_zero, 1); + } + else + { + int group = k / groupsize; + half w_scale = w_scales_.item(group, w_column); + uint32_t w_zero = w_zeros_.item(group, w_column) + 1; + + if constexpr (use_x_map) acc_h = dot_product_8_x_map_h(acc_h, x_, x_row, k, w_, k, w_column, w_scale, w_zero, 1, x_map); + else acc_h = dot_product_8_h (acc_h, x_, x_row, k, w_, k, w_column, w_scale, w_zero, 1); + } + } + } + + // Add to block result + + if constexpr (use_half2) + { + half result = __hadd(acc.x, acc.y); + atomicAdd(out_.item_ptr(x_row, w_column), result); + } + else + { + atomicAdd(out_.item_ptr(x_row, w_column), acc_h); + } +} + +fp_q4_matmul_kernel q4_matmul_kernel_pick(ExLlamaTuning* tuningParams, int block_size_z, int groupsize, uint32_t* x_map) +{ + // + if (tuningParams->matmul_no_half2) { + if (block_size_z % groupsize == 0) { + if (x_map) return q4_matmul_kernel; + else return q4_matmul_kernel; + } else { + if (x_map) return q4_matmul_kernel; + else return q4_matmul_kernel; + } + } else { + if (block_size_z % groupsize == 0) + { + if (x_map) return q4_matmul_kernel; + else return q4_matmul_kernel; + } else { + if (x_map) return q4_matmul_kernel; + else return q4_matmul_kernel; + } + } +}; + +// Compute y = x @ w + +void q4_matmul_cuda +( + ExLlamaTuning* tuningParams, + const half* x, + const int x_height, + const Q4Matrix* w, + half* out, + bool no_zero, + cudaStream_t alt_stream +) +{ + int height = x_height; + int dim = w->height; + int width = w->width; + + cudaSetDevice(w->device); + + uint32_t* x_map = w->cuda_x_map; + const half* x_mapped = x; + if (x_map && !tuningParams->matmul_fused_remap && !alt_stream) + { + CudaBuffers* buffers = get_buffers(w->device); + column_remap_cuda(x, buffers->temp_state, x_height, dim, w->cuda_x_map); + x_mapped = buffers->temp_state; + x_map = NULL; + } + + int block_size_z; + if (w->width == 4096) block_size_z = 384; // 7B + else if (w->width == 11008) block_size_z = 256; + else if (w->width == 5120) block_size_z = 384; // 13B + else if (w->width == 13824) block_size_z = 256; + else if (w->width == 6656) block_size_z = 256; // 33B + else if (w->width == 17920) block_size_z = 128; + else block_size_z = 256; + + //if (!no_zero) cudaMemsetAsync(out, 0, x_height * w->width * sizeof(half)); + + dim3 threads(THREADS_X, THREADS_Y, 1); + + dim3 blocks + ( + (width + threads.x - 1) / threads.x, + (height + threads.y - 1) / threads.y, + (dim + block_size_z - 1) / block_size_z + ); + + fp_q4_matmul_kernel kernel = q4_matmul_kernel_pick(tuningParams, block_size_z, w->groupsize, x_map); + + kernel<<>> (x_mapped, w->cuda_qweight, out, w->cuda_scales, w->cuda_qzeros, height, dim, width, w->groupsize, block_size_z, x_map, no_zero); +} + +void q4_matmul_recons_cuda +( + ExLlamaTuning* tuningParams, + const half* x, + const int x_height, + Q4Matrix* w, + half* out, + const cublasHandle_t handle, + bool no_zero +) +{ + int height = x_height; + int dim = w->height; + int width = w->width; + + cudaSetDevice(w->device); + CudaBuffers* buffers = get_buffers(w->device); + + const half* x_mapped = x; + if (w->cuda_x_map) + { + column_remap_cuda(x, buffers->temp_state, x_height, dim, w->cuda_x_map); + x_mapped = buffers->temp_state; + } + + w->reconstruct(buffers->temp_dq); + + const half alpha = __float2half(1.0f); + const half beta = no_zero ? __float2half(1.0f) : __float2half(0.0f); + cublasHgemm(handle, CUBLAS_OP_N, CUBLAS_OP_N, width, height, dim, &alpha, buffers->temp_dq, width, x_mapped, dim, &beta, out, width); + +// const float alpha = 1.0f; +// const float beta = no_zero ? 1.0f : 0.0f; +// cublasSgemmEx(handle, CUBLAS_OP_N, CUBLAS_OP_N, width, height, dim, &alpha, buffers->temp_dq, CUDA_R_16F, width, +// x_mapped, CUDA_R_16F, dim, &beta, out, CUDA_R_16F, width); +} diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cuh b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cuh new file mode 100644 index 00000000..63611790 --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cuh @@ -0,0 +1,37 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#ifndef _q4_matmul_cuh +#define _q4_matmul_cuh + +#include +#include +#include +#include +#include + +#include "q4_matrix.cuh" +#include "../tuning.h" + +void q4_matmul_cuda +( + ExLlamaTuning* tuningParams, + const half* x, + const int x_height, + const Q4Matrix* w, + half* out, + bool no_zero = false, + cudaStream_t alt_stream = NULL +); + +void q4_matmul_recons_cuda +( + ExLlamaTuning* tuningParams, + const half* x, + const int x_height, + Q4Matrix* w, + half* out, + const cublasHandle_t handle, + bool no_zero = false +); + +#endif diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu new file mode 100644 index 00000000..f3d1564f --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu @@ -0,0 +1,217 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#include "q4_matrix.cuh" +#include +#include "../util.cuh" +#include "../matrix.cuh" + +using namespace std; + +const int UNSHUF_BLOCKSIZE_X = 64; + +const int RECONS_THREADS_X = 64; // Block size and thread count along columns in out, each thread converts 1 column +const int RECONS_THREADS_Y = 1; // Block size and thread count along rows in x and out, each thread converts 8 rows + +vector g_q4_matrices; + +void g_q4_keep_matrix(Q4Matrix* m) +{ + g_q4_matrices.push_back(m); +} + +void g_q4_free_matrices() +{ + for (const auto& m : g_q4_matrices) delete m; + g_q4_matrices.clear(); +} + +Q4Matrix::Q4Matrix +( + const int _height, + const int _width, + const int _groups, + + uint32_t* _qweight, + uint32_t* _qzeros, + half* _scales, + uint32_t* _g_idx, + + const int _device +) : + height(_height), + width(_width), + groups(_groups), + device(_device) +{ + cudaSetDevice(device); + + cuda_qweight = _qweight; + cuda_qzeros = _qzeros; + cuda_scales = _scales; + + groupsize = height / groups; + + if (_g_idx) make_sequential(_g_idx); +} + +Q4Matrix::~Q4Matrix() +{ +} + +// Make sequential + +__global__ void make_sequential_kernel +( + const uint32_t* __restrict__ w, + uint32_t* __restrict__ w_new, + const uint32_t* __restrict__ x_map, + const int w_height, + const int w_width +) +{ + const uint64_t* w2 = (uint64_t*) w; + uint64_t* w_new2 = (uint64_t*) w_new; + int w2_stride = w_width >> 1; + + int w2_column = UNSHUF_BLOCKSIZE_X * blockIdx.x + threadIdx.x; + int w_new2_row = blockIdx.y; + + int x_map_idx = w_new2_row << 3; + + uint64_t dst = 0; + + #pragma unroll + for (int i = 0; i < 8; i++) + { + int source_row = x_map[x_map_idx++]; + + int w2_row = source_row >> 3; + int w2_subrow = source_row & 0x07; + int w2_row_shift = w2_subrow << 2; + int wnew2_row_shift = i << 2; + + uint64_t src = w2[w2_row * w2_stride + w2_column]; + src >>= w2_row_shift; + src &= 0x0000000f0000000f; + src <<= wnew2_row_shift; + dst |= src; + } + + w_new2[w_new2_row * w2_stride + w2_column] = dst; +} + +void Q4Matrix::make_sequential(const uint32_t* cpu_g_idx) +{ + uint32_t* cuda_new_qweight = NULL; + cudaMalloc(&cuda_new_qweight, height / 8 * width * sizeof(uint32_t)); + cudaMalloc(&cuda_x_map, height * sizeof(uint32_t)); // TODO: Should probably be allocated in PyTorch + + uint32_t* cpu_g_idx_map = (uint32_t*) calloc(groups, sizeof(uint32_t)); + uint32_t* cpu_x_map = (uint32_t*) malloc(height * sizeof(uint32_t)); + uint32_t* cpu_x_map_inv = (uint32_t*) malloc(height * sizeof(uint32_t)); + + // Group histogram + + for (int i = 0; i < height; i++) cpu_g_idx_map[cpu_g_idx[i]]++; + + // Group map + + for (int i = 0, acc = 0; i < groups; i++) + { + short tmp = cpu_g_idx_map[i]; + cpu_g_idx_map[i] = acc; + acc += tmp; + } + + // X map (inverse) + + for (int row = 0; row < height; row++) + { + uint32_t target_group = cpu_g_idx[row]; + uint32_t target_row = cpu_g_idx_map[target_group]; + cpu_g_idx_map[target_group]++; + cpu_x_map_inv[row] = target_row; + } + + // X map + + for (int row = 0; row < height; row++) cpu_x_map[cpu_x_map_inv[row]] = row; + + // Move to CUDA + + cudaMemcpyAsync(cuda_x_map, cpu_x_map, height * sizeof(uint32_t), cudaMemcpyHostToDevice); + + // Rearrange rows in w + + dim3 threads(UNSHUF_BLOCKSIZE_X, 1, 1); + dim3 blocks(width / UNSHUF_BLOCKSIZE_X / 2, height / 8, 1); + + make_sequential_kernel<<>>(cuda_qweight, cuda_new_qweight, cuda_x_map, height / 8, width); + + // Replace qweights + + cudaMemcpyAsync(cuda_qweight, cuda_new_qweight, height / 8 * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice); + + // Cleanup + + cudaDeviceSynchronize(); + cudaFree(cuda_new_qweight); + free(cpu_g_idx_map); + free(cpu_x_map); + free(cpu_x_map_inv); +} + +__global__ void reconstruct_kernel +( + const uint32_t* __restrict__ w, + half* __restrict__ out, // (y) + const half* __restrict__ w_scales, + const uint32_t* __restrict__ w_zeros, + const int height, + const int width, + const int groupsize +) +{ + // Start of block + + int column = RECONS_THREADS_X * blockIdx.x + threadIdx.x; + int row = (RECONS_THREADS_Y * blockIdx.y + threadIdx.y) * 8; + + // Views + + MatrixView_q4_column w_(w, height, width); + MatrixView_half_rw out_(out, height, width); + MatrixView_half w_scales_(w_scales, height / groupsize, width); + MatrixView_q4_row w_zeros_(w_zeros, height / groupsize, width); + + // Groupsize version + + int group = row / groupsize; + + half w_scale = w_scales_.item(group, column); + uint32_t w_zero = w_zeros_.item(group, column) + 1; + + uint32_t w_read = w_.item_uint32_t(row, column); + half* out_ptr = out_.item_ptr(row, column); + + #pragma unroll + for (int s = 0; s < 32; s += 4) + { + half w_item = __hmul(__int2half_rn((int)((w_read >> s) & 0x0f) - w_zero), w_scale); + *out_ptr = w_item; out_ptr += out_.width; + } +} + +void Q4Matrix::reconstruct(half* out) +{ + dim3 threads(RECONS_THREADS_X, RECONS_THREADS_Y, 1); + + dim3 blocks + ( + (width + threads.x - 1) / threads.x, + (height / 8 + threads.y - 1) / threads.y, + 1 + ); + + reconstruct_kernel<<>>(cuda_qweight, out, cuda_scales, cuda_qzeros, height / 8, width, groupsize); +} \ No newline at end of file diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cuh b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cuh new file mode 100644 index 00000000..50cb72a4 --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cuh @@ -0,0 +1,53 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#ifndef _q4_matrix_cuh +#define _q4_matrix_cuh + +#include +#include +#include + +class Q4Matrix +{ +public: + + int device; + + int height; + int width; + int groups; + int groupsize; + + uint32_t* cuda_qweight = NULL; + uint32_t* cuda_qzeros = NULL; + half* cuda_scales = NULL; + uint32_t* cuda_x_map = NULL; + + Q4Matrix + ( + const int _height, + const int _width, + const int _groups, + + uint32_t* _qweight, + uint32_t* _qzeros, + half* _scales, + uint32_t* _g_idx, + + const int _device + ); + + ~Q4Matrix(); + + void reconstruct(half* out); + +private: + + void make_sequential(const uint32_t* cpu_g_idx); + +}; + +void g_q4_keep_matrix(Q4Matrix* m); +void g_q4_free_matrices(); + +#endif \ No newline at end of file diff --git a/server/exllama_kernels/exllama_kernels/exllama_ext.cpp b/server/exllama_kernels/exllama_kernels/exllama_ext.cpp new file mode 100644 index 00000000..b786988b --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/exllama_ext.cpp @@ -0,0 +1,249 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#include +#include +#include +#include +#include +#include +#include +#include "util.cuh" +#include "tuning.h" +#include "cuda_buffers.cuh" +#include "cuda_func/q4_matrix.cuh" +#include "cuda_func/q4_matmul.cuh" +#include "cuda_func/column_remap.cuh" + +// Check CUDA return code. We don't want to include Torch headers in the .cu files because parsing them adds almost a +// minute to the compile time on a 12900K. Also passing exceptions back to Python is super tricky, so in place of +// exceptions, CUDA functions return with a cudaError_t which we can parse and dump to the console. + +void check_cuda(cudaError_t ret) +{ + switch (ret) + { + case cudaSuccess: + break; + + case cudaUnspecified: + printf(" **** Unspecified error\n"); + TORCH_CHECK(false, "CUDA error"); + break; + + default: + printf(" **** CUDA error\n"); \ + printf(" **** %s\n", cudaGetErrorString(ret)); \ + TORCH_CHECK(false, "CUDA error"); \ + break; + } +} + +// Some decluttering macros + +#define STRINGIFY_(__x) #__x +#define STRINGIFY(__x) STRINGIFY_(__x) +#define TORCH_CHECK_DTYPE(__x, __dtype) TORCH_CHECK((__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype) +#define TORCH_CHECK_DTYPE_OPT(__x, __dtype) TORCH_CHECK((__x).device().is_meta() || (__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype) +#define TORCH_CHECK_SHAPES(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes") +#define TORCH_CHECK_SHAPES_OPT(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).device().is_meta() || (__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes") +#define TORCH_CHECK_SHAPE_MOD(__x, __dim_x, __mod) TORCH_CHECK((__x).size(__dim_x) % __mod == 0, #__x ".shape[" STRINGIFY(__dim_x) "] must be a multiple of " STRINGIFY(__mod)) + +#define TORCH_CHECK_DEVICE_INDEX(__index) \ +do { \ + TORCH_CHECK(__index >= 0, "no device index"); \ + TORCH_CHECK(__index < CUDA_MAX_DEVICES, "invalid device index"); \ +} while(0) + +#define TORCH_CHECK_QUANT(__w, __w_scales, __w_zeros, __seq_g_idx, __x_map) \ +do { \ + TORCH_CHECK_DTYPE(__w, kInt); \ + TORCH_CHECK_DTYPE(__w_scales, kHalf); \ + TORCH_CHECK_DTYPE(__w_zeros, kInt); \ + TORCH_CHECK_DTYPE_OPT(__seq_g_idx, kShort); \ + TORCH_CHECK_DTYPE_OPT(__x_map, kInt); \ + TORCH_CHECK_SHAPES_OPT(__seq_g_idx, 0, __w, 0, 2 * 8); \ + TORCH_CHECK_SHAPES_OPT(__x_map, 0, __w, 0, 8); \ +} while(0) + +int get_groupsize(torch::Tensor w, torch::Tensor w_zeros) +{ + int groupsize = w.size(0) * 8 / w_zeros.size(0); + TORCH_CHECK(groupsize * w_zeros.size(0) == w.size(0) * 8, "w.shape[-2] must be a multiple of zeros.shape[-2]") + return groupsize; +} + + +// Tuning parameters + +ExLlamaTuning tuningParams; + +void set_tuning_params +( + int matmul_recons_thd, + bool matmul_fused_remap, + bool matmul_no_half2 +) +{ + tuningParams.matmul_recons_thd = matmul_recons_thd; + tuningParams.matmul_fused_remap = matmul_fused_remap; + tuningParams.matmul_no_half2 = matmul_no_half2; +} + + +// Release all unmanaged objects allocated by the extension + +void cleanup() +{ + cleanup_buffers_cuda(); + g_q4_free_matrices(); +} + + +// Prepare buffers for forward pass + +void prepare_buffers +( + torch::Device device, + torch::Tensor temp_state, + torch::Tensor temp_dq +) +{ + int device_index = device.index(); + TORCH_CHECK_DEVICE_INDEX(device_index); + const at::cuda::OptionalCUDAGuard device_guard(device); + + prepare_buffers_cuda + ( + device_index, + (half*) temp_state.data_ptr(), + (half*) temp_dq.data_ptr() + ); +} + + +// Create Q4Matrix, return handle + +uintptr_t make_q4 +( + torch::Tensor qweight, + torch::Tensor qzeros, + torch::Tensor scales, + torch::Tensor g_idx, + int device +) +{ + TORCH_CHECK_DTYPE(qweight, kInt); + TORCH_CHECK_DTYPE(qzeros, kInt); + TORCH_CHECK_DTYPE(scales, kHalf); + TORCH_CHECK_DTYPE_OPT(g_idx, kInt); + TORCH_CHECK_SHAPES(qweight, 1, qzeros, 1, 8); + TORCH_CHECK_SHAPES(scales, 1, qweight, 1, 1); + TORCH_CHECK_SHAPES(qzeros, 0, scales, 0, 1); + + int width = qweight.size(1); + int height = qweight.size(0) * 8; + int groups = qzeros.size(0); + + Q4Matrix* m = new Q4Matrix + ( + height, + width, + groups, + + (uint32_t*) qweight.data_ptr(), + (uint32_t*) qzeros.data_ptr(), + (half*) scales.data_ptr(), + g_idx.device().is_meta() ? NULL : (uint32_t*) g_idx.data_ptr(), + + device + ); + + g_q4_keep_matrix(m); + return reinterpret_cast (m); +} + + +// Matmul half @ quant -> half + +void q4_matmul +( + torch::Tensor x, + uintptr_t w, + torch::Tensor out +) +{ + Q4Matrix* wm = reinterpret_cast (w); + + TORCH_CHECK_DTYPE(x, kHalf); + TORCH_CHECK_DTYPE(out, kHalf); + TORCH_CHECK_SHAPES(x, 0, out, 0, 1); + TORCH_CHECK(wm->height == x.size(-1), "x and w have incompatible shapes") + + const at::cuda::OptionalCUDAGuard device_guard(device_of(x)); + + int x_height = x.size(0); + + if (tuningParams.matmul_recons_thd == 0 || x_height < tuningParams.matmul_recons_thd) + { + q4_matmul_cuda + ( + &tuningParams, + (half*) x.data_ptr(), + x_height, + wm, + (half*) out.data_ptr() + ); + } + else + { + q4_matmul_recons_cuda + ( + &tuningParams, + (half*) x.data_ptr(), + x_height, + wm, + (half*) out.data_ptr(), + at::cuda::getCurrentCUDABlasHandle() + ); + } +} + + +// Remap columns in half tensor + +void column_remap +( + torch::Tensor x, + torch::Tensor x_new, + torch::Tensor x_map +) +{ + TORCH_CHECK_DTYPE(x, kHalf); + TORCH_CHECK_DTYPE(x_new, kHalf); + TORCH_CHECK_DTYPE(x_map, kInt); + TORCH_CHECK_SHAPES(x_map, 0, x, 1, 1); + + int height = x.size(0); + int width = x.size(1); + + const at::cuda::OptionalCUDAGuard device_guard(device_of(x)); + + column_remap_cuda + ( + (half*) x.data_ptr(), + (half*) x_new.data_ptr(), + height, + width, + (uint32_t*) x_map.data_ptr() + ); +} + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) +{ + m.def("set_tuning_params", &set_tuning_params, "set_tuning_params"); + m.def("prepare_buffers", &prepare_buffers, "prepare_buffers"); + m.def("cleanup", &cleanup, "cleanup"); + m.def("make_q4", &make_q4, "make_q4"); + m.def("q4_matmul", &q4_matmul, "q4_matmul"); +} diff --git a/server/exllama_kernels/exllama_kernels/matrix.cuh b/server/exllama_kernels/exllama_kernels/matrix.cuh new file mode 100644 index 00000000..2fd5ab0b --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/matrix.cuh @@ -0,0 +1,294 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#ifndef _matrix_cuh +#define _matrix_cuh + +#include +#include + +class MatrixView_half +{ +public: + const half* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width) + : data(data), height(height), width(width) + { } + + __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; } + __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; } + __device__ __forceinline__ half2 item_half2half2(int row, int column) const { return __half2half2(data[row * width + column]); } + __device__ __forceinline__ const half* item_ptr(int row, int column) const { return &data[row * width + column]; } +}; + +class MatrixView_half_rw +{ +public: + half* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width) + : data(data), height(height), width(width) + { } + + __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; } + __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; } + __device__ __forceinline__ half* item_ptr(int row, int column) { return &data[row * width + column]; } + __device__ __forceinline__ void set(int row, int column, half value) { data[row * width + column] = value; } + __device__ __forceinline__ void set_half2(int row, int column, half2 value) { ((half2*)data)[(row * width + column) / 2] = value; } +}; + +class MatrixView_q4_row +{ +public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) + { } + + __device__ __forceinline__ int item(int row, int column) const + { + int shift = (column & 0x07) * 4; + return (data[row * width / 8 + column / 8] >> shift) & 0x0f; + } +}; + +class MatrixView_q4_column +{ +public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) + { } + + __device__ __forceinline__ int item(int row, int column) const + { + int shift = (row & 0x07) * 4; + return (data[row / 8 * width + column] >> shift) & 0x0f; + } + + __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) { return data[row / 8 * width + column]; } + __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, int column) { return &data[row / 8 * width + column]; } +}; + +// TODO: Rewrite all these dot product functions using functors or something, move to q4_matmul.cu + +// Accumulated dot product of 8-element row vectors in h and quantized column vectors in v, constant zero/scale + +__device__ __forceinline__ half2 dot_product_8 +( + const half2 acc, + MatrixView_half& h_, + const int h_row, + const int h_column, // divisible by 8 + MatrixView_q4_column& v_, + const int v_row, // divisible by 8 + const int v_column, + const half2 v_scale_2, + const uint32_t v_zero, // + 1 (!!) + const int count +) +{ + const half2* h_ptr = (const half2*) h_.item_ptr(h_row, h_column); + const uint32_t* v_ptr = (const uint32_t*) v_.item_uint32_ptr(v_row, v_column); + half2 result = acc; + + for (int i = 0; i < count; i++) + { + uint32_t v_read = *v_ptr; v_ptr += v_.width; + + half v_0 = __int2half_rn((int)((v_read ) & 0x0f) - v_zero); + half v_1 = __int2half_rn((int)((v_read >> 4) & 0x0f) - v_zero); + half v_2 = __int2half_rn((int)((v_read >> 8) & 0x0f) - v_zero); + half v_3 = __int2half_rn((int)((v_read >> 12) & 0x0f) - v_zero); + half v_4 = __int2half_rn((int)((v_read >> 16) & 0x0f) - v_zero); + half v_5 = __int2half_rn((int)((v_read >> 20) & 0x0f) - v_zero); + half v_6 = __int2half_rn((int)((v_read >> 24) & 0x0f) - v_zero); + half v_7 = __int2half_rn((int)((v_read >> 28) ) - v_zero); + + half2 v_01 = __halves2half2(v_0, v_1); + half2 v_23 = __halves2half2(v_2, v_3); + half2 v_45 = __halves2half2(v_4, v_5); + half2 v_67 = __halves2half2(v_6, v_7); + +// half2 v_01 = q4_table[v_zero - 1][(v_read ) & 0xff]; // (constant memory is too slow apparently) +// half2 v_23 = q4_table[v_zero - 1][(v_read >> 8) & 0xff]; +// half2 v_45 = q4_table[v_zero - 1][(v_read >> 16) & 0xff]; +// half2 v_67 = q4_table[v_zero - 1][(v_read >> 24) ]; + + half2 tmp = __hmul2(*h_ptr++, v_01); + tmp = __hfma2(*h_ptr++, v_23, tmp); + tmp = __hfma2(*h_ptr++, v_45, tmp); + tmp = __hfma2(*h_ptr++, v_67, tmp); + result = __hfma2(v_scale_2, tmp, result); + } + + return result; +} + +__device__ __forceinline__ half dot_product_8_h +( + const half acc, + MatrixView_half& h_, + const int h_row, + const int h_column, // divisible by 8 + MatrixView_q4_column& v_, + const int v_row, // divisible by 8 + const int v_column, + const half v_scale, + const uint32_t v_zero, // + 1 (!!) + const int count +) +{ + const half* h_ptr = h_.item_ptr(h_row, h_column); + const uint32_t* v_ptr = (const uint32_t*) v_.item_uint32_ptr(v_row, v_column); + half result = acc; + + for (int i = 0; i < count; i++) + { + uint32_t v_read = *v_ptr; v_ptr += v_.width; + + half v_0 = __int2half_rn((int)((v_read ) & 0x0f) - v_zero); + half v_1 = __int2half_rn((int)((v_read >> 4) & 0x0f) - v_zero); + half v_2 = __int2half_rn((int)((v_read >> 8) & 0x0f) - v_zero); + half v_3 = __int2half_rn((int)((v_read >> 12) & 0x0f) - v_zero); + half v_4 = __int2half_rn((int)((v_read >> 16) & 0x0f) - v_zero); + half v_5 = __int2half_rn((int)((v_read >> 20) & 0x0f) - v_zero); + half v_6 = __int2half_rn((int)((v_read >> 24) & 0x0f) - v_zero); + half v_7 = __int2half_rn((int)((v_read >> 28) ) - v_zero); + + half tmp = __hmul(*h_ptr++, v_0); + tmp = __hfma(*h_ptr++, v_1, tmp); + tmp = __hfma(*h_ptr++, v_2, tmp); + tmp = __hfma(*h_ptr++, v_3, tmp); + tmp = __hfma(*h_ptr++, v_4, tmp); + tmp = __hfma(*h_ptr++, v_5, tmp); + tmp = __hfma(*h_ptr++, v_6, tmp); + tmp = __hfma(*h_ptr++, v_7, tmp); + result = __hfma(v_scale, tmp, result); + } + + return result; +} + +// Accumulated dot product of 8-element row vectors in h and quantized column vectors in v, constant zero/scale, with x_map + +__device__ __forceinline__ half2 dot_product_8_x_map +( + const half2 acc, + MatrixView_half& h_, + const int h_row, + const int h_column, // divisible by 8 + MatrixView_q4_column& v_, + const int v_row, // divisible by 8 + const int v_column, + const half2 v_scale_2, + const uint32_t v_zero, // + 1 (!!) + const int count, + const uint32_t* x_map +) +{ + const half* h_ptr = h_.item_ptr(h_row, 0); + const uint32_t* x_map_ptr = x_map + h_column; + const uint32_t* v_ptr = (const uint32_t*) v_.item_uint32_ptr(v_row, v_column); + half2 result = acc; + + for (int i = 0; i < count; i++) + { + uint32_t v_read = *v_ptr; v_ptr += v_.width; + + half v_0 = __int2half_rn((int)((v_read ) & 0x0f) - v_zero); + half v_1 = __int2half_rn((int)((v_read >> 4) & 0x0f) - v_zero); + half v_2 = __int2half_rn((int)((v_read >> 8) & 0x0f) - v_zero); + half v_3 = __int2half_rn((int)((v_read >> 12) & 0x0f) - v_zero); + half v_4 = __int2half_rn((int)((v_read >> 16) & 0x0f) - v_zero); + half v_5 = __int2half_rn((int)((v_read >> 20) & 0x0f) - v_zero); + half v_6 = __int2half_rn((int)((v_read >> 24) & 0x0f) - v_zero); + half v_7 = __int2half_rn((int)((v_read >> 28) ) - v_zero); + + half2 v_01 = __halves2half2(v_0, v_1); + half2 v_23 = __halves2half2(v_2, v_3); + half2 v_45 = __halves2half2(v_4, v_5); + half2 v_67 = __halves2half2(v_6, v_7); + + half h_0 = h_ptr[*x_map_ptr++]; + half h_1 = h_ptr[*x_map_ptr++]; + half h_2 = h_ptr[*x_map_ptr++]; + half h_3 = h_ptr[*x_map_ptr++]; + half h_4 = h_ptr[*x_map_ptr++]; + half h_5 = h_ptr[*x_map_ptr++]; + half h_6 = h_ptr[*x_map_ptr++]; + half h_7 = h_ptr[*x_map_ptr++]; + + half2 h_01 = __halves2half2(h_0, h_1); + half2 h_23 = __halves2half2(h_2, h_3); + half2 h_45 = __halves2half2(h_4, h_5); + half2 h_67 = __halves2half2(h_6, h_7); + + half2 tmp = __hmul2(h_01, v_01); + tmp = __hfma2(h_23, v_23, tmp); + tmp = __hfma2(h_45, v_45, tmp); + tmp = __hfma2(h_67, v_67, tmp); + result = __hfma2(v_scale_2, tmp, result); + } + + return result; +} + +__device__ __forceinline__ half dot_product_8_x_map_h +( + const half acc, + MatrixView_half& h_, + const int h_row, + const int h_column, // divisible by 8 + MatrixView_q4_column& v_, + const int v_row, // divisible by 8 + const int v_column, + const half v_scale, + const uint32_t v_zero, // + 1 (!!) + const int count, + const uint32_t* x_map +) +{ + const half* h_ptr = h_.item_ptr(h_row, 0); + const uint32_t* x_map_ptr = x_map + h_column; + const uint32_t* v_ptr = (const uint32_t*) v_.item_uint32_ptr(v_row, v_column); + half result = acc; + + for (int i = 0; i < count; i++) + { + uint32_t v_read = *v_ptr; v_ptr += v_.width; + + half v_0 = __int2half_rn((int)((v_read ) & 0x0f) - v_zero); + half v_1 = __int2half_rn((int)((v_read >> 4) & 0x0f) - v_zero); + half v_2 = __int2half_rn((int)((v_read >> 8) & 0x0f) - v_zero); + half v_3 = __int2half_rn((int)((v_read >> 12) & 0x0f) - v_zero); + half v_4 = __int2half_rn((int)((v_read >> 16) & 0x0f) - v_zero); + half v_5 = __int2half_rn((int)((v_read >> 20) & 0x0f) - v_zero); + half v_6 = __int2half_rn((int)((v_read >> 24) & 0x0f) - v_zero); + half v_7 = __int2half_rn((int)((v_read >> 28) ) - v_zero); + + half tmp = __hmul(h_ptr[*x_map_ptr++], v_0); + tmp = __hfma(h_ptr[*x_map_ptr++], v_1, tmp); + tmp = __hfma(h_ptr[*x_map_ptr++], v_2, tmp); + tmp = __hfma(h_ptr[*x_map_ptr++], v_3, tmp); + tmp = __hfma(h_ptr[*x_map_ptr++], v_4, tmp); + tmp = __hfma(h_ptr[*x_map_ptr++], v_5, tmp); + tmp = __hfma(h_ptr[*x_map_ptr++], v_6, tmp); + tmp = __hfma(h_ptr[*x_map_ptr++], v_7, tmp); + result = __hfma(v_scale, tmp, result); + } + + return result; +} + +#endif diff --git a/server/exllama_kernels/exllama_kernels/tuning.h b/server/exllama_kernels/exllama_kernels/tuning.h new file mode 100644 index 00000000..770ca46a --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/tuning.h @@ -0,0 +1,13 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#ifndef _tuning_h +#define _tuning_h + +struct ExLlamaTuning +{ + int matmul_recons_thd; + bool matmul_fused_remap; + bool matmul_no_half2; +}; + +#endif diff --git a/server/exllama_kernels/exllama_kernels/util.cuh b/server/exllama_kernels/exllama_kernels/util.cuh new file mode 100644 index 00000000..2839b10f --- /dev/null +++ b/server/exllama_kernels/exllama_kernels/util.cuh @@ -0,0 +1,29 @@ +// Adapted from turboderp exllama: https://github.com/turboderp/exllama + +#ifndef _util_cuh +#define _util_cuh + +#include +#include +#include +#include + +#define cudaUnspecified cudaErrorApiFailureBase + +// React to failure on return code != cudaSuccess + +#define _cuda_check(fn) \ +do { \ + {_cuda_err = fn;} \ + if (_cuda_err != cudaSuccess) goto _cuda_fail; \ +} while(false) + +// React to failure on return code == 0 + +#define _alloc_check(fn) \ +do { \ + if (!(fn)) { _cuda_err = cudaUnspecified; goto _cuda_fail; } \ + else _cuda_err = cudaSuccess; \ +} while(false) + +#endif diff --git a/server/exllama_kernels/setup.py b/server/exllama_kernels/setup.py new file mode 100644 index 00000000..f06a72bd --- /dev/null +++ b/server/exllama_kernels/setup.py @@ -0,0 +1,19 @@ +from setuptools import setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +setup( + name="exllama_kernels", + ext_modules=[ + CUDAExtension( + name="exllama_kernels", + sources=[ + "exllama_kernels/exllama_ext.cpp", + "exllama_kernels/cuda_buffers.cu", + "exllama_kernels/cuda_func/column_remap.cu", + "exllama_kernels/cuda_func/q4_matmul.cu", + "exllama_kernels/cuda_func/q4_matrix.cu" + ], + ) + ], + cmdclass={"build_ext": BuildExtension}, +) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 702bcf61..b6285856 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -383,7 +383,6 @@ def forward( class FlashLlamaModel(torch.nn.Module): def __init__(self, config, weights): super().__init__() - self.config = config process_group = weights.process_group self.tp_rank = process_group.rank() diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 6f5c60fc..04bd422f 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -20,7 +20,6 @@ ) from safetensors import SafetensorError - def load_multi_mqa( config, prefix: str, weights, bias: bool, head_size, num_heads, hidden_size ): @@ -50,6 +49,7 @@ def _load_multi_mqa_gptq( q_tensor = slice_[:, start:stop] kv_tensor = slice_[:, -2 * head_size :] qweight = torch.cat([q_tensor, kv_tensor], dim=1) + qweight = qweight.to(device=weights.device) slice_ = weights._get_slice(f"{prefix}.c_attn.scales") shape = slice_.get_shape() @@ -60,6 +60,7 @@ def _load_multi_mqa_gptq( q_tensor = slice_[:, start:stop] kv_tensor = slice_[:, -2 * head_size :] scales = torch.cat([q_tensor, kv_tensor], dim=1) + scales = scales.to(device=weights.device) slice_ = weights._get_slice(f"{prefix}.c_attn.qzeros") shape = slice_.get_shape() @@ -70,21 +71,15 @@ def _load_multi_mqa_gptq( q_tensor = slice_[:, start:stop] kv_tensor = slice_[:, -2 * head_size * 4 // 32 :] qzeros = torch.cat([q_tensor, kv_tensor], dim=1) + qzeros = qzeros.to(device=weights.device) g_idx = weights.get_tensor(f"{prefix}.c_attn.g_idx") - try: - bits = weights.get_tensor("gptq_bits").item() - groupsize = weights.get_tensor("gptq_groupsize").item() - except SafetensorError as e: - try: - import os - - bits = int(os.getenv("GPTQ_BITS")) - groupsize = int(os.getenv("GPTQ_GROUPSIZE")) - except Exception: - raise e + g_idx = g_idx.to(device=weights.device) + bits, groupsize = weights._get_gptq_qparams() - weight = (qweight, qzeros, scales, g_idx, bits, groupsize) + from text_generation_server.utils.layers import HAS_EXLLAMA + use_exllama = HAS_EXLLAMA + weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) if bias: slice_ = weights._get_slice(f"{prefix}.c_attn.bias") @@ -97,6 +92,7 @@ def _load_multi_mqa_gptq( q_tensor = slice_[start:stop] kv_tensor = slice_[-2 * head_size :] bias = torch.cat([q_tensor, kv_tensor], dim=0) + bias = bias.to(device=weights.device) return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize)) else: @@ -361,7 +357,6 @@ def forward( max_s, ): hidden_states, residual = self.ln_1(hidden_states, residual) - hidden_states = self.attn( hidden_states, cu_seqlen_prefill, diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index c0592cb0..547678a8 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -6,7 +6,6 @@ import numpy as np from dataclasses import dataclass -from loguru import logger from opentelemetry import trace from transformers import PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type, Union, Dict diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 3827197f..89e6e99b 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -3,14 +3,13 @@ from abc import ABC, abstractmethod from typing import List, Tuple, Optional, TypeVar, Type -from transformers import PreTrainedTokenizerBase +from transformers import PreTrainedTokenizerBase, PretrainedConfig from text_generation_server.models.types import Batch, GeneratedText from text_generation_server.pb.generate_pb2 import InfoResponse B = TypeVar("B", bound=Batch) - class Model(ABC): def __init__( self, diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index e0efbcf5..b279426b 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -16,6 +16,7 @@ from text_generation_server.tracing import UDSOpenTelemetryAioServerInterceptor + class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): def __init__(self, model: Model, cache: Cache, server_urls: List[str]): self.cache = cache @@ -140,6 +141,16 @@ async def serve_inner( logger.exception("Error when initializing model") raise + if quantize == "gptq": + try: + # When using GPTQ, Exllama kernels need some global kernels + # For which we have the finale shapes only after the model has loaded + # This will allocate those buffers. + from text_generation_server.utils.gptq.exllama import create_exllama_buffers + create_exllama_buffers() + except ImportError: + pass + server = aio.server( interceptors=[ ExceptionInterceptor(), diff --git a/server/text_generation_server/utils/gptq/exllama.py b/server/text_generation_server/utils/gptq/exllama.py new file mode 100644 index 00000000..aba66796 --- /dev/null +++ b/server/text_generation_server/utils/gptq/exllama.py @@ -0,0 +1,121 @@ + +import torch +from exllama_kernels import make_q4, q4_matmul, prepare_buffers, set_tuning_params + +# Dummy tensor to pass instead of g_idx since there is no way to pass "None" to a C++ extension +none_tensor = torch.empty((1, 1), device = "meta") + +def ext_make_q4(qweight, qzeros, scales, g_idx, device): + """Construct Q4Matrix, return handle""" + return make_q4(qweight, + qzeros, + scales, + g_idx if g_idx is not None else none_tensor, + device) + +def ext_q4_matmul(x, q4, q4_width): + """Matrix multiplication, returns x @ q4""" + outshape = x.shape[:-1] + (q4_width,) + x = x.view(-1, x.shape[-1]) + output = torch.empty((x.shape[0], q4_width), dtype = torch.float16, device = x.device) + + q4_matmul(x, q4, output) + + return output.view(outshape) + +MAX_DQ = 1 +MAX_INNER = 1 +ACT_ORDER = False +DEVICE = None + +TEMP_STATE = None +TEMP_DQ = None + +def create_exllama_buffers(): + global MAX_DQ, MAX_INNER, ACT_ORDER, DEVICE, TEMP_STATE, TEMP_DQ + + if ACT_ORDER: + # TODO: this should be set to rust side `max_total_tokens`, but TGI + # does not offer an API to expose this variable to python, as this variable + # is handled by the client but it appears the model is initialized by the server. + # An alternative could be to initialize the buffers during warmup. + # Dummy + max_total_tokens = 2048 + else: + max_total_tokens = 1 + + # This temp_state buffer is required to reorder X in the act-order case. + temp_state = torch.zeros((max_total_tokens, MAX_INNER), dtype=torch.float16, device=DEVICE) + temp_dq = torch.zeros((1, MAX_DQ), dtype=torch.float16, device=DEVICE) + + # This temp_dq buffer is required to dequantize weights when using cuBLAS, typically for the prefill. + prepare_buffers(DEVICE, temp_state, temp_dq) + + matmul_recons_thd = 8 + matmul_fused_remap = False + matmul_no_half2 = False + set_tuning_params(matmul_recons_thd, matmul_fused_remap, matmul_no_half2) + + TEMP_STATE, TEMP_DQ = temp_state, temp_dq + +class Ex4bitLinear: + """Linear layer implementation with per-group 4-bit quantization of the weights""" + def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): + global MAX_DQ, MAX_INNER, ACT_ORDER, DEVICE + assert bits == 4 + + self.device = qweight.device + self.qweight = qweight + self.qzeros = qzeros + self.scales = scales + self.g_idx = g_idx.cpu() if g_idx is not None else None + self.bias = bias if bias is not None else None + + if self.g_idx is not None and ((self.g_idx == 0).all() or torch.equal(g_idx.cpu(), torch.tensor([i // groupsize for i in range(g_idx.shape[0])], dtype=torch.int32))): + self.empty_g_idx = True + self.g_idx = None + + assert self.device.type == "cuda" + assert self.device.index is not None + + self.q4 = ext_make_q4( + self.qweight, + self.qzeros, + self.scales, + self.g_idx, + self.device.index + ) + + self.height = qweight.shape[0] * 8 + self.width = qweight.shape[1] + + # Infer groupsize from height of qzeros + self.groupsize = None + if self.qzeros.shape[0] > 1: + self.groupsize = (self.qweight.shape[0] * 8) // (self.qzeros.shape[0]) + + if self.groupsize is not None: + assert groupsize == self.groupsize + + # Handle act-order matrix + if self.g_idx is not None: + if self.groupsize is None: raise ValueError("Found group index but no groupsize. What do?") + self.act_order = True + else: + self.act_order = False + + DEVICE = self.qweight.device + + MAX_DQ = max(MAX_DQ, self.qweight.numel() * 8) + + if self.act_order: + MAX_INNER = max(MAX_INNER, self.height, self.width) + + ACT_ORDER = True + + def forward(self, x): + out = ext_q4_matmul(x, self.q4, self.width) + + if self.bias is not None: + out.add_(self.bias) + return out diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 4f65446e..4f280161 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -1,3 +1,4 @@ +import os import torch import torch.distributed @@ -16,7 +17,15 @@ from accelerate import init_empty_weights from text_generation_server.utils.gptq.quant_linear import QuantLinear +HAS_EXLLAMA = True +if os.getenv("DISABLE_EXLLAMA") == "True": + HAS_EXLLAMA=False +try: + from text_generation_server.utils.gptq.exllama import Ex4bitLinear +except ImportError: + HAS_EXLLAMA = False +from typing import Optional # Monkey patching @classmethod @@ -144,21 +153,24 @@ def get_linear(weight, bias, quantize): linear.bias = nn.Parameter(bias) elif quantize == "gptq": try: - qweight, qzeros, scales, g_idx, bits, groupsize = weight + qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama = weight except Exception: raise NotImplementedError( f"The passed weight is not `gptq` compatible, loader needs to be updated." ) - linear = QuantLinear( - qweight, - qzeros, - scales, - g_idx, - bias, - bits, - groupsize, - ) + if use_exllama: + linear = Ex4bitLinear(qweight, qzeros, scales, g_idx, bias, bits, groupsize) + else: + linear = QuantLinear( + qweight, + qzeros, + scales, + g_idx, + bias, + bits, + groupsize, + ) else: raise NotImplementedError(f"Quantization `{quantize}` is not implemented yet.") return linear diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 723d0558..4f284ea4 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -1,7 +1,8 @@ from pathlib import Path -from typing import List, Dict, Optional +from typing import List, Dict, Optional, Tuple from safetensors import safe_open, SafetensorError import torch +from loguru import logger class Weights: @@ -127,18 +128,8 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): torch.testing.assert_close(w2, w[0]) g_idx = w[0] - try: - bits = self.get_tensor("gptq_bits").item() - groupsize = self.get_tensor("gptq_groupsize").item() - except (SafetensorError, RuntimeError) as e: - try: - import os - - bits = int(os.getenv("GPTQ_BITS")) - groupsize = int(os.getenv("GPTQ_GROUPSIZE")) - except Exception: - raise e - weight = (qweight, qzeros, scales, g_idx, bits, groupsize) + bits, groupsize = self._get_gptq_qparams() + weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) else: w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] weight = torch.cat(w, dim=dim) @@ -146,29 +137,74 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): def get_multi_weights_row(self, prefix: str, quantize: str): if quantize == "gptq": + use_exllama = True + bits, groupsize = self._get_gptq_qparams() + + if bits != 4: + use_exllama = False + + if self.process_group.size() > 1: + g_idx = self.get_tensor(f"{prefix}.g_idx") + if g_idx is not None: + if not torch.equal(g_idx.cpu(), torch.tensor([i // groupsize for i in range(g_idx.shape[0])], dtype=torch.int32)) and not (g_idx == 0).all(): + # Exllama implementation does not support row tensor parallelism with act-order, as + # it would require to reorder input activations that are split unto several GPUs + use_exllama = False + try: qweight = self.get_sharded(f"{prefix}.qweight", dim=0) except RuntimeError: - raise RuntimeError( - "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" - ) - qzeros = self.get_tensor(f"{prefix}.qzeros") - scales = self.get_tensor(f"{prefix}.scales") - g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) - - try: - bits = self.get_tensor("gptq_bits").item() - groupsize = self.get_tensor("gptq_groupsize").item() - except (SafetensorError, RuntimeError) as e: - try: - import os - - bits = int(os.getenv("GPTQ_BITS")) - groupsize = int(os.getenv("GPTQ_GROUPSIZE")) - except Exception: - raise e - - weight = (qweight, qzeros, scales, g_idx, bits, groupsize) + raise RuntimeError("Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`") + + + from text_generation_server.utils.layers import HAS_EXLLAMA + if use_exllama: + if not HAS_EXLLAMA: + logger.warning("Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True") + use_exllama = False + else: + logger.info("Using exllama kernels") + + + if use_exllama: + if groupsize >= 0: + # Exllama reorders the weights in advance and the activations on the fly, thus + # the scales and zero-points do not need to be reordered. + qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) + scales = self.get_sharded(f"{prefix}.scales", dim=0) + else: + raise RuntimeError("Using exllama GPTQ kernel with groupsize<1 is not supported") + # qzeros = self.get_tensor(f"{prefix}.qzeros") + # scales = self.get_tensor(f"{prefix}.scales") + + # For tp > 1, at this point we know we do not use act-order + if self.process_group.size() == 1: + g_idx = self.get_tensor(f"{prefix}.g_idx") + else: + g_idx = None + else: + # The triton kernel reorders the scales/zero points instead of the weight/activation. + # Thus, each rank needs the full qzeros/scales. + qzeros = self.get_tensor(f"{prefix}.qzeros") + scales = self.get_tensor(f"{prefix}.scales") + g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) + + weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) else: weight = self.get_sharded(f"{prefix}.weight", dim=1) return weight + + def _get_gptq_qparams(self) -> Tuple[int, int]: + try: + bits = self.get_tensor("gptq_bits").item() + groupsize = self.get_tensor("gptq_groupsize").item() + except (SafetensorError, RuntimeError) as e: + try: + import os + + bits = int(os.getenv("GPTQ_BITS")) + groupsize = int(os.getenv("GPTQ_GROUPSIZE")) + except Exception: + raise e + + return bits, groupsize From 15b3e9ffb0ce1506c455f060d56f1f196baf49f3 Mon Sep 17 00:00:00 2001 From: "Yang, Bo" Date: Fri, 21 Jul 2023 02:27:31 -0700 Subject: [PATCH 337/793] Directly load GPTBigCode to specified device (#618) This PR directly load GPTBigCode to specified device, avoiding moving model between devices. # What does this PR do? This PR directly load GPTBigCode to specified device, avoiding moving model between devices. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @OlivierDehaene OR @Narsil --- .../text_generation_server/models/santacoder.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index a2b38737..81928c1d 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -51,14 +51,14 @@ def __init__( "pad_token": EOD, } ) - - model = AutoModelForCausalLM.from_pretrained( - model_id, - revision=revision, - torch_dtype=dtype, - load_in_8bit=quantize == "bitsandbytes", - trust_remote_code=trust_remote_code, - ).to(device) + with device: + model = AutoModelForCausalLM.from_pretrained( + model_id, + revision=revision, + torch_dtype=dtype, + load_in_8bit=quantize == "bitsandbytes", + trust_remote_code=trust_remote_code, + ) super(CausalLM, self).__init__( model=model, From 1da642bd0e6de28ef499f17cd226264f3ccdc824 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 21 Jul 2023 16:56:30 +0200 Subject: [PATCH 338/793] feat(server): add local prom and health routes if running w/ ngrok --- router/src/server.rs | 269 +++++++++++++++++++++++-------------------- 1 file changed, 142 insertions(+), 127 deletions(-) diff --git a/router/src/server.rs b/router/src/server.rs index bfeee375..9af94951 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -32,25 +32,25 @@ use utoipa_swagger_ui::SwaggerUi; /// Generate tokens if `stream == false` or a stream of token if `stream == true` #[utoipa::path( - post, - tag = "Text Generation Inference", - path = "/", - request_body = CompatGenerateRequest, - responses( - (status = 200, description = "Generated Text", - content( - ("application/json" = GenerateResponse), - ("text/event-stream" = StreamResponse), - )), - (status = 424, description = "Generation Error", body = ErrorResponse, - example = json ! ({"error": "Request failed during generation"})), - (status = 429, description = "Model is overloaded", body = ErrorResponse, - example = json ! ({"error": "Model is overloaded"})), - (status = 422, description = "Input validation error", body = ErrorResponse, - example = json ! ({"error": "Input validation error"})), - (status = 500, description = "Incomplete generation", body = ErrorResponse, - example = json ! ({"error": "Incomplete generation"})), - ) +post, +tag = "Text Generation Inference", +path = "/", +request_body = CompatGenerateRequest, +responses( +(status = 200, description = "Generated Text", +content( +("application/json" = GenerateResponse), +("text/event-stream" = StreamResponse), +)), +(status = 424, description = "Generation Error", body = ErrorResponse, +example = json ! ({"error": "Request failed during generation"})), +(status = 429, description = "Model is overloaded", body = ErrorResponse, +example = json ! ({"error": "Model is overloaded"})), +(status = 422, description = "Input validation error", body = ErrorResponse, +example = json ! ({"error": "Input validation error"})), +(status = 500, description = "Incomplete generation", body = ErrorResponse, +example = json ! ({"error": "Incomplete generation"})), +) )] #[instrument(skip(infer, req))] async fn compat_generate( @@ -79,10 +79,10 @@ async fn compat_generate( /// Text Generation Inference endpoint info #[utoipa::path( - get, - tag = "Text Generation Inference", - path = "/info", - responses((status = 200, description = "Served model info", body = Info)) +get, +tag = "Text Generation Inference", +path = "/info", +responses((status = 200, description = "Served model info", body = Info)) )] #[instrument] async fn get_model_info(info: Extension) -> Json { @@ -90,14 +90,14 @@ async fn get_model_info(info: Extension) -> Json { } #[utoipa::path( - get, - tag = "Text Generation Inference", - path = "/health", - responses( - (status = 200, description = "Everything is working fine"), - (status = 503, description = "Text generation inference is down", body = ErrorResponse, - example = json ! ({"error": "unhealthy", "error_type": "healthcheck"})), - ) +get, +tag = "Text Generation Inference", +path = "/health", +responses( +(status = 200, description = "Everything is working fine"), +(status = 503, description = "Text generation inference is down", body = ErrorResponse, +example = json ! ({"error": "unhealthy", "error_type": "healthcheck"})), +) )] #[instrument(skip(health))] /// Health check method @@ -116,33 +116,33 @@ async fn health(mut health: Extension) -> Result<(), (StatusCode, Json, @@ -297,38 +297,38 @@ async fn generate( /// Generate a stream of token using Server-Sent Events #[utoipa::path( - post, - tag = "Text Generation Inference", - path = "/generate_stream", - request_body = GenerateRequest, - responses( - (status = 200, description = "Generated Text", body = StreamResponse, - content_type = "text/event-stream"), - (status = 424, description = "Generation Error", body = ErrorResponse, - example = json ! ({"error": "Request failed during generation"}), - content_type = "text/event-stream"), - (status = 429, description = "Model is overloaded", body = ErrorResponse, - example = json ! ({"error": "Model is overloaded"}), - content_type = "text/event-stream"), - (status = 422, description = "Input validation error", body = ErrorResponse, - example = json ! ({"error": "Input validation error"}), - content_type = "text/event-stream"), - (status = 500, description = "Incomplete generation", body = ErrorResponse, - example = json ! ({"error": "Incomplete generation"}), - content_type = "text/event-stream"), - ) +post, +tag = "Text Generation Inference", +path = "/generate_stream", +request_body = GenerateRequest, +responses( +(status = 200, description = "Generated Text", body = StreamResponse, +content_type = "text/event-stream"), +(status = 424, description = "Generation Error", body = ErrorResponse, +example = json ! ({"error": "Request failed during generation"}), +content_type = "text/event-stream"), +(status = 429, description = "Model is overloaded", body = ErrorResponse, +example = json ! ({"error": "Model is overloaded"}), +content_type = "text/event-stream"), +(status = 422, description = "Input validation error", body = ErrorResponse, +example = json ! ({"error": "Input validation error"}), +content_type = "text/event-stream"), +(status = 500, description = "Incomplete generation", body = ErrorResponse, +example = json ! ({"error": "Incomplete generation"}), +content_type = "text/event-stream"), +) )] #[instrument( - skip_all, - fields( - parameters = ?req.0.parameters, - total_time, - validation_time, - queue_time, - inference_time, - time_per_token, - seed, - ) +skip_all, +fields( +parameters = ? req.0.parameters, +total_time, +validation_time, +queue_time, +inference_time, +time_per_token, +seed, +) )] async fn generate_stream( infer: Extension, @@ -493,10 +493,10 @@ async fn generate_stream( /// Prometheus metrics scrape endpoint #[utoipa::path( - get, - tag = "Text Generation Inference", - path = "/metrics", - responses((status = 200, description = "Prometheus Metrics", body = String)) +get, +tag = "Text Generation Inference", +path = "/metrics", +responses((status = 200, description = "Prometheus Metrics", body = String)) )] async fn metrics(prom_handle: Extension) -> String { prom_handle.render() @@ -529,41 +529,41 @@ pub async fn run( // OpenAPI documentation #[derive(OpenApi)] #[openapi( - paths( - health, - get_model_info, - compat_generate, - generate, - generate_stream, - metrics, - ), - components( - schemas( - Info, - CompatGenerateRequest, - GenerateRequest, - GenerateParameters, - PrefillToken, - Token, - GenerateResponse, - BestOfSequence, - Details, - FinishReason, - StreamResponse, - StreamDetails, - ErrorResponse, - ) - ), - tags( - (name = "Text Generation Inference", description = "Hugging Face Text Generation Inference API") - ), - info( - title = "Text Generation Inference", - license( - name = "Apache 2.0", - url = "https://www.apache.org/licenses/LICENSE-2.0" - ) - ) + paths( + health, + get_model_info, + compat_generate, + generate, + generate_stream, + metrics, + ), + components( + schemas( + Info, + CompatGenerateRequest, + GenerateRequest, + GenerateParameters, + PrefillToken, + Token, + GenerateResponse, + BestOfSequence, + Details, + FinishReason, + StreamResponse, + StreamDetails, + ErrorResponse, + ) + ), + tags( + (name = "Text Generation Inference", description = "Hugging Face Text Generation Inference API") + ), + info( + title = "Text Generation Inference", + license( + name = "Apache 2.0", + url = "https://www.apache.org/licenses/LICENSE-2.0" + ) + ) )] struct ApiDoc; @@ -683,10 +683,10 @@ pub async fn run( // Prometheus metrics route .route("/metrics", get(metrics)) .layer(Extension(info)) - .layer(Extension(health_ext)) + .layer(Extension(health_ext.clone())) .layer(Extension(compat_return_full_text)) .layer(Extension(infer)) - .layer(Extension(prom_handle)) + .layer(Extension(prom_handle.clone())) .layer(opentelemetry_tracing_layer()) .layer(cors_layer); @@ -712,6 +712,21 @@ pub async fn run( let listener = tunnel.listen().await.unwrap(); + // Run prom metrics and health locally too + tokio::spawn( + axum::Server::bind(&addr) + .serve( + Router::new() + .route("/health", get(health)) + .route("/metrics", get(metrics)) + .layer(Extension(health_ext)) + .layer(Extension(prom_handle)) + .into_make_service(), + ) + //Wait until all requests are finished to shut down + .with_graceful_shutdown(shutdown_signal()), + ); + // Run server axum::Server::builder(listener) .serve(app.into_make_service()) From 73a4d65d26801c550e0f1205800c002b147de84e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 24 Jul 2023 11:43:58 +0200 Subject: [PATCH 339/793] feat: add cuda memory fraction (#659) Close #673 --- .../models/test_flash_llama_gptq.py | 8 ++- .../models/test_flash_starcoder_gptq.py | 21 ++++++-- launcher/src/main.rs | 14 +++++ router/client/src/sharded_client.rs | 8 ++- server/exllama_kernels/setup.py | 2 +- .../flash_santacoder_modeling.py | 2 + .../models/flash_causal_lm.py | 8 ++- server/text_generation_server/models/model.py | 4 +- server/text_generation_server/server.py | 6 ++- server/text_generation_server/utils/dist.py | 27 ++++++---- .../utils/gptq/exllama.py | 52 +++++++++++-------- .../utils/gptq/quantize.py | 6 +-- server/text_generation_server/utils/layers.py | 3 +- .../text_generation_server/utils/weights.py | 26 +++++++--- 14 files changed, 127 insertions(+), 60 deletions(-) diff --git a/integration-tests/models/test_flash_llama_gptq.py b/integration-tests/models/test_flash_llama_gptq.py index bc525f6d..b87f054b 100644 --- a/integration-tests/models/test_flash_llama_gptq.py +++ b/integration-tests/models/test_flash_llama_gptq.py @@ -48,8 +48,12 @@ async def test_flash_llama_gptq_all_params(flash_llama_gptq, response_snapshot): @pytest.mark.asyncio @pytest.mark.private -async def test_flash_llama_gptq_load(flash_llama_gptq, generate_load, response_snapshot): - responses = await generate_load(flash_llama_gptq, "Test request", max_new_tokens=10, n=4) +async def test_flash_llama_gptq_load( + flash_llama_gptq, generate_load, response_snapshot +): + responses = await generate_load( + flash_llama_gptq, "Test request", max_new_tokens=10, n=4 + ) assert len(responses) == 4 assert all([r.generated_text == responses[0].generated_text for r in responses]) diff --git a/integration-tests/models/test_flash_starcoder_gptq.py b/integration-tests/models/test_flash_starcoder_gptq.py index b6bed6a6..608101fb 100644 --- a/integration-tests/models/test_flash_starcoder_gptq.py +++ b/integration-tests/models/test_flash_starcoder_gptq.py @@ -17,7 +17,9 @@ async def flash_starcoder_gptq(flash_starcoder_gptq_handle): @pytest.mark.private async def test_flash_starcoder_gptq(flash_starcoder_gptq, response_snapshot): response = await flash_starcoder_gptq.generate( - "def geometric_mean(L: List[float]):", max_new_tokens=20, decoder_input_details=True, + "def geometric_mean(L: List[float]):", + max_new_tokens=20, + decoder_input_details=True, ) assert response.details.generated_tokens == 20 assert response == response_snapshot @@ -25,7 +27,9 @@ async def test_flash_starcoder_gptq(flash_starcoder_gptq, response_snapshot): @pytest.mark.asyncio @pytest.mark.private -async def test_flash_starcoder_gptq_default_params(flash_starcoder_gptq, response_snapshot): +async def test_flash_starcoder_gptq_default_params( + flash_starcoder_gptq, response_snapshot +): response = await flash_starcoder_gptq.generate( "def geometric_mean(L: List[float]):", max_new_tokens=20, @@ -40,10 +44,17 @@ async def test_flash_starcoder_gptq_default_params(flash_starcoder_gptq, respons @pytest.mark.asyncio @pytest.mark.private -async def test_flash_starcoder_gptq_load(flash_starcoder_gptq, generate_load, response_snapshot): - responses = await generate_load(flash_starcoder_gptq, "def geometric_mean(L: List[float]):", max_new_tokens=10, n=4) +async def test_flash_starcoder_gptq_load( + flash_starcoder_gptq, generate_load, response_snapshot +): + responses = await generate_load( + flash_starcoder_gptq, + "def geometric_mean(L: List[float]):", + max_new_tokens=10, + n=4, + ) assert len(responses) == 4 assert all([r.generated_text == responses[0].generated_text for r in responses]) - assert responses == response_snapshot \ No newline at end of file + assert responses == response_snapshot diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 53de36b2..2ad788a4 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -245,6 +245,11 @@ struct Args { #[clap(long, env)] disable_custom_kernels: bool, + /// Limit the CUDA available memory. + /// The allowed value equals the total visible memory multiplied by cuda-memory-fraction. + #[clap(default_value = "1.0", long, env)] + cuda_memory_fraction: f32, + /// Outputs the logs in JSON format (useful for telemetry) #[clap(long, env)] json_output: bool, @@ -299,6 +304,7 @@ fn shard_manager( disable_custom_kernels: bool, watermark_gamma: Option, watermark_delta: Option, + cuda_memory_fraction: f32, otlp_endpoint: Option, status_sender: mpsc::Sender, shutdown: Arc, @@ -368,6 +374,12 @@ fn shard_manager( envs.push(("MASTER_PORT".into(), master_port.to_string().into())); envs.push(("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())); + // CUDA memory fraction + envs.push(( + "CUDA_MEMORY_FRACTION".into(), + cuda_memory_fraction.to_string().into(), + )); + // Safetensors load fast envs.push(("SAFETENSORS_FAST_GPU".into(), "1".into())); @@ -771,6 +783,7 @@ fn spawn_shards( let disable_custom_kernels = args.disable_custom_kernels; let watermark_gamma = args.watermark_gamma; let watermark_delta = args.watermark_delta; + let cuda_memory_fraction = args.cuda_memory_fraction; thread::spawn(move || { shard_manager( model_id, @@ -788,6 +801,7 @@ fn spawn_shards( disable_custom_kernels, watermark_gamma, watermark_delta, + cuda_memory_fraction, otlp_endpoint, status_sender, shutdown, diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 6d146bc5..112b0035 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -101,8 +101,12 @@ impl ShardedClient { .iter_mut() .map(|client| Box::pin(client.warmup(max_input_length, max_prefill_tokens))) .collect(); - // all shards return the same message - join_all(futures).await.pop().unwrap() + // Take the minimum value + let results = join_all(futures) + .await + .into_iter() + .collect::>>>()?; + Ok(results.into_iter().flatten().min()) } /// Generate one token for each request in the given batch diff --git a/server/exllama_kernels/setup.py b/server/exllama_kernels/setup.py index f06a72bd..987d181e 100644 --- a/server/exllama_kernels/setup.py +++ b/server/exllama_kernels/setup.py @@ -11,7 +11,7 @@ "exllama_kernels/cuda_buffers.cu", "exllama_kernels/cuda_func/column_remap.cu", "exllama_kernels/cuda_func/q4_matmul.cu", - "exllama_kernels/cuda_func/q4_matrix.cu" + "exllama_kernels/cuda_func/q4_matrix.cu", ], ) ], diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 04bd422f..c16b2bf7 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -20,6 +20,7 @@ ) from safetensors import SafetensorError + def load_multi_mqa( config, prefix: str, weights, bias: bool, head_size, num_heads, hidden_size ): @@ -78,6 +79,7 @@ def _load_multi_mqa_gptq( bits, groupsize = weights._get_gptq_qparams() from text_generation_server.utils.layers import HAS_EXLLAMA + use_exllama = HAS_EXLLAMA weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 547678a8..7de51358 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -19,6 +19,7 @@ ) from text_generation_server.pb import generate_pb2 from text_generation_server.utils import StoppingCriteria, HeterogeneousNextTokenChooser +from text_generation_server.utils.dist import MEMORY_FRACTION tracer = trace.get_tracer(__name__) @@ -738,7 +739,12 @@ def warmup(self, batch: FlashCausalLMBatch): cache_block_size = BLOCK_SIZE * self.num_kv_heads * self.head_size total_cache_size = self.num_layers * cache_block_size * 2 * dtype_size - free_memory, _ = torch.cuda.mem_get_info(self.device) + total_free_memory, _ = torch.cuda.mem_get_info(self.device) + total_gpu_memory = torch.cuda.get_device_properties(self.device).total_memory + + free_memory = max( + 0, total_free_memory - (1 - MEMORY_FRACTION) * total_gpu_memory + ) num_blocks = ( int(free_memory // total_cache_size) diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 89e6e99b..9d74247c 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -10,6 +10,7 @@ B = TypeVar("B", bound=Batch) + class Model(ABC): def __init__( self, @@ -21,9 +22,6 @@ def __init__( rank: int = 0, world_size: int = 1, ): - if torch.cuda.is_available(): - torch.cuda.set_per_process_memory_fraction(1.0) - self.model = model.eval() self.tokenizer = tokenizer self.all_special_ids = set(tokenizer.all_special_ids) diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index b279426b..0929b46f 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -16,7 +16,6 @@ from text_generation_server.tracing import UDSOpenTelemetryAioServerInterceptor - class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): def __init__(self, model: Model, cache: Cache, server_urls: List[str]): self.cache = cache @@ -146,7 +145,10 @@ async def serve_inner( # When using GPTQ, Exllama kernels need some global kernels # For which we have the finale shapes only after the model has loaded # This will allocate those buffers. - from text_generation_server.utils.gptq.exllama import create_exllama_buffers + from text_generation_server.utils.gptq.exllama import ( + create_exllama_buffers, + ) + create_exllama_buffers() except ImportError: pass diff --git a/server/text_generation_server/utils/dist.py b/server/text_generation_server/utils/dist.py index 41a8e01a..d02bfc5b 100644 --- a/server/text_generation_server/utils/dist.py +++ b/server/text_generation_server/utils/dist.py @@ -4,6 +4,13 @@ from datetime import timedelta from loguru import logger +# Tensor Parallelism settings +RANK = int(os.getenv("RANK", "0")) +WORLD_SIZE = int(os.getenv("WORLD_SIZE", "1")) + +# CUDA memory fraction +MEMORY_FRACTION = float(os.getenv("CUDA_MEMORY_FRACTION", "1.0")) + class FakeBarrier: def wait(self): @@ -37,16 +44,14 @@ def rank(self): def initialize_torch_distributed(): - rank = int(os.getenv("RANK", "0")) - world_size = int(os.getenv("WORLD_SIZE", "1")) - if torch.cuda.is_available(): from torch.distributed import ProcessGroupNCCL # Set the device id. - assert world_size <= torch.cuda.device_count(), "Each process is one gpu" - device = rank % torch.cuda.device_count() + assert WORLD_SIZE <= torch.cuda.device_count(), "Each process is one gpu" + device = RANK % torch.cuda.device_count() torch.cuda.set_device(device) + torch.cuda.set_per_process_memory_fraction(MEMORY_FRACTION, device) backend = "nccl" options = ProcessGroupNCCL.Options() options.is_high_priority_stream = True @@ -55,22 +60,22 @@ def initialize_torch_distributed(): backend = "gloo" options = None - if world_size == 1: - return FakeGroup(rank, world_size), rank, world_size + if WORLD_SIZE == 1: + return FakeGroup(RANK, WORLD_SIZE), RANK, WORLD_SIZE else: if os.getenv("DEBUG", None) == "1": - return FakeGroup(rank, world_size), rank, world_size + return FakeGroup(RANK, WORLD_SIZE), RANK, WORLD_SIZE if not torch.distributed.is_initialized(): # Call the init process. torch.distributed.init_process_group( backend=backend, - world_size=world_size, - rank=rank, + world_size=WORLD_SIZE, + rank=RANK, timeout=timedelta(seconds=60), pg_options=options, ) else: logger.warning("torch.distributed is already initialized.") - return torch.distributed.group.WORLD, rank, world_size + return torch.distributed.group.WORLD, RANK, WORLD_SIZE diff --git a/server/text_generation_server/utils/gptq/exllama.py b/server/text_generation_server/utils/gptq/exllama.py index aba66796..e89b725c 100644 --- a/server/text_generation_server/utils/gptq/exllama.py +++ b/server/text_generation_server/utils/gptq/exllama.py @@ -1,28 +1,28 @@ - import torch from exllama_kernels import make_q4, q4_matmul, prepare_buffers, set_tuning_params # Dummy tensor to pass instead of g_idx since there is no way to pass "None" to a C++ extension -none_tensor = torch.empty((1, 1), device = "meta") +none_tensor = torch.empty((1, 1), device="meta") + def ext_make_q4(qweight, qzeros, scales, g_idx, device): """Construct Q4Matrix, return handle""" - return make_q4(qweight, - qzeros, - scales, - g_idx if g_idx is not None else none_tensor, - device) + return make_q4( + qweight, qzeros, scales, g_idx if g_idx is not None else none_tensor, device + ) + def ext_q4_matmul(x, q4, q4_width): """Matrix multiplication, returns x @ q4""" outshape = x.shape[:-1] + (q4_width,) x = x.view(-1, x.shape[-1]) - output = torch.empty((x.shape[0], q4_width), dtype = torch.float16, device = x.device) + output = torch.empty((x.shape[0], q4_width), dtype=torch.float16, device=x.device) q4_matmul(x, q4, output) return output.view(outshape) + MAX_DQ = 1 MAX_INNER = 1 ACT_ORDER = False @@ -31,9 +31,10 @@ def ext_q4_matmul(x, q4, q4_width): TEMP_STATE = None TEMP_DQ = None + def create_exllama_buffers(): global MAX_DQ, MAX_INNER, ACT_ORDER, DEVICE, TEMP_STATE, TEMP_DQ - + if ACT_ORDER: # TODO: this should be set to rust side `max_total_tokens`, but TGI # does not offer an API to expose this variable to python, as this variable @@ -45,7 +46,9 @@ def create_exllama_buffers(): max_total_tokens = 1 # This temp_state buffer is required to reorder X in the act-order case. - temp_state = torch.zeros((max_total_tokens, MAX_INNER), dtype=torch.float16, device=DEVICE) + temp_state = torch.zeros( + (max_total_tokens, MAX_INNER), dtype=torch.float16, device=DEVICE + ) temp_dq = torch.zeros((1, MAX_DQ), dtype=torch.float16, device=DEVICE) # This temp_dq buffer is required to dequantize weights when using cuBLAS, typically for the prefill. @@ -56,10 +59,12 @@ def create_exllama_buffers(): matmul_no_half2 = False set_tuning_params(matmul_recons_thd, matmul_fused_remap, matmul_no_half2) - TEMP_STATE, TEMP_DQ = temp_state, temp_dq + TEMP_STATE, TEMP_DQ = temp_state, temp_dq + class Ex4bitLinear: """Linear layer implementation with per-group 4-bit quantization of the weights""" + def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): global MAX_DQ, MAX_INNER, ACT_ORDER, DEVICE assert bits == 4 @@ -70,20 +75,24 @@ def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): self.scales = scales self.g_idx = g_idx.cpu() if g_idx is not None else None self.bias = bias if bias is not None else None - - if self.g_idx is not None and ((self.g_idx == 0).all() or torch.equal(g_idx.cpu(), torch.tensor([i // groupsize for i in range(g_idx.shape[0])], dtype=torch.int32))): + + if self.g_idx is not None and ( + (self.g_idx == 0).all() + or torch.equal( + g_idx.cpu(), + torch.tensor( + [i // groupsize for i in range(g_idx.shape[0])], dtype=torch.int32 + ), + ) + ): self.empty_g_idx = True self.g_idx = None - + assert self.device.type == "cuda" assert self.device.index is not None self.q4 = ext_make_q4( - self.qweight, - self.qzeros, - self.scales, - self.g_idx, - self.device.index + self.qweight, self.qzeros, self.scales, self.g_idx, self.device.index ) self.height = qweight.shape[0] * 8 @@ -99,7 +108,8 @@ def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): # Handle act-order matrix if self.g_idx is not None: - if self.groupsize is None: raise ValueError("Found group index but no groupsize. What do?") + if self.groupsize is None: + raise ValueError("Found group index but no groupsize. What do?") self.act_order = True else: self.act_order = False @@ -112,7 +122,7 @@ def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): MAX_INNER = max(MAX_INNER, self.height, self.width) ACT_ORDER = True - + def forward(self, x): out = ext_q4_matmul(x, self.q4, self.width) diff --git a/server/text_generation_server/utils/gptq/quantize.py b/server/text_generation_server/utils/gptq/quantize.py index 45b01ae1..6eb44e41 100644 --- a/server/text_generation_server/utils/gptq/quantize.py +++ b/server/text_generation_server/utils/gptq/quantize.py @@ -815,11 +815,7 @@ def inner(module, args): tensor = current_tensor.to(device=torch.device("cuda:0")) if current_tensor.requires_grad: tensor = nn.Parameter(tensor) - setdeepattr( - module, - local_param, - tensor - ) + setdeepattr(module, local_param, tensor) return inner diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 4f280161..183cf2c1 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -17,9 +17,10 @@ from accelerate import init_empty_weights from text_generation_server.utils.gptq.quant_linear import QuantLinear + HAS_EXLLAMA = True if os.getenv("DISABLE_EXLLAMA") == "True": - HAS_EXLLAMA=False + HAS_EXLLAMA = False try: from text_generation_server.utils.gptq.exllama import Ex4bitLinear except ImportError: diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 4f284ea4..dae53509 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -146,7 +146,16 @@ def get_multi_weights_row(self, prefix: str, quantize: str): if self.process_group.size() > 1: g_idx = self.get_tensor(f"{prefix}.g_idx") if g_idx is not None: - if not torch.equal(g_idx.cpu(), torch.tensor([i // groupsize for i in range(g_idx.shape[0])], dtype=torch.int32)) and not (g_idx == 0).all(): + if ( + not torch.equal( + g_idx.cpu(), + torch.tensor( + [i // groupsize for i in range(g_idx.shape[0])], + dtype=torch.int32, + ), + ) + and not (g_idx == 0).all() + ): # Exllama implementation does not support row tensor parallelism with act-order, as # it would require to reorder input activations that are split unto several GPUs use_exllama = False @@ -154,18 +163,21 @@ def get_multi_weights_row(self, prefix: str, quantize: str): try: qweight = self.get_sharded(f"{prefix}.qweight", dim=0) except RuntimeError: - raise RuntimeError("Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`") - + raise RuntimeError( + "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" + ) from text_generation_server.utils.layers import HAS_EXLLAMA + if use_exllama: if not HAS_EXLLAMA: - logger.warning("Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True") + logger.warning( + "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True" + ) use_exllama = False else: logger.info("Using exllama kernels") - if use_exllama: if groupsize >= 0: # Exllama reorders the weights in advance and the activations on the fly, thus @@ -173,7 +185,9 @@ def get_multi_weights_row(self, prefix: str, quantize: str): qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) scales = self.get_sharded(f"{prefix}.scales", dim=0) else: - raise RuntimeError("Using exllama GPTQ kernel with groupsize<1 is not supported") + raise RuntimeError( + "Using exllama GPTQ kernel with groupsize<1 is not supported" + ) # qzeros = self.get_tensor(f"{prefix}.qzeros") # scales = self.get_tensor(f"{prefix}.scales") From 37df6df38edb4dc8eee89a42ec3791e89442c851 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 24 Jul 2023 14:25:43 +0200 Subject: [PATCH 340/793] fix(server): fix exllama buffers (#689) Close #683 --- server/text_generation_server/server.py | 28 ++++++++++--------- .../utils/gptq/exllama.py | 7 +++++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 0929b46f..7c2f1b35 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -105,21 +105,21 @@ async def Decode(self, request, context): def serve( - model_id: str, - revision: Optional[str], - sharded: bool, - quantize: Optional[str], - dtype: Optional[str], - trust_remote_code: bool, - uds_path: Path, -): - async def serve_inner( model_id: str, revision: Optional[str], - sharded: bool = False, - quantize: Optional[str] = None, - dtype: Optional[str] = None, - trust_remote_code: bool = False, + sharded: bool, + quantize: Optional[str], + dtype: Optional[str], + trust_remote_code: bool, + uds_path: Path, +): + async def serve_inner( + model_id: str, + revision: Optional[str], + sharded: bool = False, + quantize: Optional[str] = None, + dtype: Optional[str] = None, + trust_remote_code: bool = False, ): unix_socket_template = "unix://{}-{}" if sharded: @@ -147,8 +147,10 @@ async def serve_inner( # This will allocate those buffers. from text_generation_server.utils.gptq.exllama import ( create_exllama_buffers, + set_device, ) + set_device(model.device) create_exllama_buffers() except ImportError: pass diff --git a/server/text_generation_server/utils/gptq/exllama.py b/server/text_generation_server/utils/gptq/exllama.py index e89b725c..6a1cf117 100644 --- a/server/text_generation_server/utils/gptq/exllama.py +++ b/server/text_generation_server/utils/gptq/exllama.py @@ -32,9 +32,16 @@ def ext_q4_matmul(x, q4, q4_width): TEMP_DQ = None +def set_device(device): + global DEVICE + DEVICE = device + + def create_exllama_buffers(): global MAX_DQ, MAX_INNER, ACT_ORDER, DEVICE, TEMP_STATE, TEMP_DQ + assert DEVICE is not None, "call set_device first" + if ACT_ORDER: # TODO: this should be set to rust side `max_total_tokens`, but TGI # does not offer an API to expose this variable to python, as this variable From a0d55358d20cf3f86083b79765b80704ce9d8404 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 25 Jul 2023 12:00:27 +0100 Subject: [PATCH 341/793] feat(server): Using `quantize_config.json` instead of GPTQ_BITS env variables. (#671) - Current PR is not great because we're side stepping the `Weights.__init__` but Weights shouldn't requires anything related to the config or the model_id as it aims to be a simple Wrapper over multi file loading. - Ideal solution would be to use something like Rust enum ``` enum Quantize{ Bitandbytes(Bitsandbytes), GPTQ(bits: usize, groupsize: usize) ``` And passing that around during load. Unfortunately we don't have access to this, so for now, side-stepping seems easier. - Re-enabling groupsize<0 with exllama (confirmed it works.) Helps #601 In next steps we should make sure our quantization script uses that format and make it standard. # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/models/bloom.py | 2 ++ .../flash_santacoder_modeling.py | 2 +- .../models/custom_modeling/opt_modeling.py | 20 ++++++------ .../models/flash_llama.py | 4 ++- .../models/flash_neox.py | 2 ++ .../text_generation_server/models/flash_rw.py | 3 ++ .../models/flash_santacoder.py | 5 +++ .../models/galactica.py | 2 ++ .../text_generation_server/models/gpt_neox.py | 2 ++ server/text_generation_server/models/mpt.py | 2 ++ server/text_generation_server/models/opt.py | 2 ++ .../text_generation_server/utils/weights.py | 31 ++++++++++++------- 12 files changed, 53 insertions(+), 24 deletions(-) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 101da207..79fb60c6 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -76,6 +76,8 @@ def __init__( weights = Weights( filenames, device=device, dtype=dtype, process_group=self.process_group ) + if config.quantize == "gptq": + weights._set_gptq_params(model_id) model = BloomForCausalLM(config, weights) diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index c16b2bf7..2dd0a5ee 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -76,7 +76,7 @@ def _load_multi_mqa_gptq( g_idx = weights.get_tensor(f"{prefix}.c_attn.g_idx") g_idx = g_idx.to(device=weights.device) - bits, groupsize = weights._get_gptq_qparams() + bits, groupsize = weights._get_gptq_params() from text_generation_server.utils.layers import HAS_EXLLAMA diff --git a/server/text_generation_server/models/custom_modeling/opt_modeling.py b/server/text_generation_server/models/custom_modeling/opt_modeling.py index aa052b08..5d1a4b0d 100644 --- a/server/text_generation_server/models/custom_modeling/opt_modeling.py +++ b/server/text_generation_server/models/custom_modeling/opt_modeling.py @@ -130,17 +130,17 @@ def __init__( process_group=None, ): super().__init__() - embed_dim = config.embed_dim + hidden_size = config.hidden_size num_heads = config.num_attention_heads - self.embed_dim = embed_dim + self.hidden_size = hidden_size self.num_heads = num_heads self.dropout = config.dropout - self.head_dim = embed_dim // num_heads + self.head_dim = hidden_size // num_heads - if (self.head_dim * num_heads) != self.embed_dim: + if (self.head_dim * num_heads) != self.hidden_size: raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" f" and `num_heads`: {num_heads})." ) self.scaling = self.head_dim**-0.5 @@ -153,7 +153,7 @@ def __init__( f"and `num_shards`: {weights.process_group.size()}" ) self.num_heads = self.num_heads // process_group.size() - self.embed_dim = self.embed_dim // process_group.size() + self.hidden_size = self.hidden_size // process_group.size() self.q_proj = TensorParallelColumnLinear.load( config, prefix=f"{prefix}.q_proj", weights=weights, bias=bias @@ -300,9 +300,9 @@ def forward( attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) attn_output = attn_output.transpose(1, 2) - # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # Use the `hidden_size` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be # partitioned aross GPUs when using tensor-parallelism. - attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + attn_output = attn_output.reshape(bsz, tgt_len, self.hidden_size) attn_output = self.out_proj(attn_output) @@ -313,7 +313,7 @@ class OPTDecoderLayer(nn.Module): def __init__(self, layer_id: int, config: OPTConfig, weights): super().__init__() self.process_group = weights.process_group - self.embed_dim = config.hidden_size + self.hidden_size = config.hidden_size prefix = f"model.decoder.layers.{layer_id}" self.self_attn = OPTAttention( config, @@ -352,7 +352,7 @@ def forward( ]: """ Args: - hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, hidden_size)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`, *optional*): mask for attention heads in a given layer of size diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index b699799e..96fb0c26 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -55,13 +55,15 @@ def __init__( config = LlamaConfig.from_pretrained( model_id, revision=revision, trust_remote_code=trust_remote_code ) + config.quantize = quantize torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights(filenames, device, dtype, process_group=self.process_group) + if config.quantize == "gptq": + weights._set_gptq_params(model_id) - config.quantize = quantize model = FlashLlamaForCausalLM(config, weights) torch.distributed.barrier(group=self.process_group) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 61004d8e..58f345a9 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -52,6 +52,8 @@ def __init__( weights = Weights( filenames, device=device, dtype=dtype, process_group=self.process_group ) + if config.quantize == "gptq": + weights._set_gptq_params(model_id) model = FlashGPTNeoXForCausalLM(config, weights) diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 55d555fc..9e0080a9 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -58,6 +58,9 @@ def __init__( ) config.quantize = quantize + if config.quantize == "gptq": + weights._set_gptq_params(model_id) + model = FlashRWForCausalLM(config, weights) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 415ec2df..29505902 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -4,7 +4,10 @@ from opentelemetry import trace from transformers import AutoTokenizer, AutoConfig from typing import Optional, List +import json +import os +from huggingface_hub import hf_hub_download from text_generation_server.models import FlashCausalLM from text_generation_server.models.custom_modeling.flash_santacoder_modeling import ( FlashSantacoderForCausalLM, @@ -59,6 +62,8 @@ def __init__( process_group=self.process_group, aliases={"transformer.wte.weight": ["lm_head.weight"]}, ) + if config.quantize == "gptq": + weights._set_gptq_params(model_id) model = FlashSantacoderForCausalLM(config, weights) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 01e58bad..d4211734 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -191,6 +191,8 @@ def __init__( weights = Weights( filenames, device=device, dtype=dtype, process_group=self.process_group ) + if config.quantize == "gptq": + weights._set_gptq_params(model_id) model = OPTForCausalLM(config, weights) diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index 91877fa0..accedf14 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -56,6 +56,8 @@ def __init__( weights = Weights( filenames, device=device, dtype=dtype, process_group=self.process_group ) + if config.quantize == "gptq": + weights._set_gptq_params(model_id) model = GPTNeoxForCausalLM(config, weights) diff --git a/server/text_generation_server/models/mpt.py b/server/text_generation_server/models/mpt.py index a4fe5105..909d9852 100644 --- a/server/text_generation_server/models/mpt.py +++ b/server/text_generation_server/models/mpt.py @@ -78,6 +78,8 @@ def __init__( filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights(filenames, device, dtype, process_group=self.process_group) + if config.quantize == "gptq": + weights._set_gptq_params(model_id) config.quantize = quantize model = MPTForCausalLM(config, weights) diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index d407b44a..f3a23d07 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -54,6 +54,8 @@ def __init__( weights = Weights( filenames, device=device, dtype=dtype, process_group=self.process_group ) + if config.quantize == "gptq": + weights._set_gptq_params(model_id) model = OPTForCausalLM(config, weights) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index dae53509..0330402d 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -3,6 +3,8 @@ from safetensors import safe_open, SafetensorError import torch from loguru import logger +from huggingface_hub import hf_hub_download +import json class Weights: @@ -128,7 +130,7 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): torch.testing.assert_close(w2, w[0]) g_idx = w[0] - bits, groupsize = self._get_gptq_qparams() + bits, groupsize = self._get_gptq_params() weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) else: w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] @@ -138,7 +140,7 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): def get_multi_weights_row(self, prefix: str, quantize: str): if quantize == "gptq": use_exllama = True - bits, groupsize = self._get_gptq_qparams() + bits, groupsize = self._get_gptq_params() if bits != 4: use_exllama = False @@ -185,11 +187,8 @@ def get_multi_weights_row(self, prefix: str, quantize: str): qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) scales = self.get_sharded(f"{prefix}.scales", dim=0) else: - raise RuntimeError( - "Using exllama GPTQ kernel with groupsize<1 is not supported" - ) - # qzeros = self.get_tensor(f"{prefix}.qzeros") - # scales = self.get_tensor(f"{prefix}.scales") + qzeros = self.get_tensor(f"{prefix}.qzeros") + scales = self.get_tensor(f"{prefix}.scales") # For tp > 1, at this point we know we do not use act-order if self.process_group.size() == 1: @@ -208,17 +207,25 @@ def get_multi_weights_row(self, prefix: str, quantize: str): weight = self.get_sharded(f"{prefix}.weight", dim=1) return weight - def _get_gptq_qparams(self) -> Tuple[int, int]: + def _get_gptq_params(self) -> Tuple[int, int]: try: bits = self.get_tensor("gptq_bits").item() groupsize = self.get_tensor("gptq_groupsize").item() except (SafetensorError, RuntimeError) as e: try: - import os - - bits = int(os.getenv("GPTQ_BITS")) - groupsize = int(os.getenv("GPTQ_GROUPSIZE")) + bits = self.gptq_bits + groupsize = self.gptq_groupsize except Exception: raise e return bits, groupsize + + def _set_gptq_params(self, model_id): + try: + filename = hf_hub_download(model_id, filename="quantize_config.json") + with open(filename, "r") as f: + data = json.load(f) + self.gptq_bits = data["bits"] + self.gptq_groupsize = data["group_size"] + except Exception: + pass From e64a65891bd842a559ba1ea2d37525d7dae7f0f4 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Tue, 25 Jul 2023 19:45:25 +0200 Subject: [PATCH 342/793] docs(README): update readme --- Makefile | 17 ++++------------ README.md | 60 +++++++++++++++++++++++++++++++------------------------ 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 81b312d1..7f534c7c 100644 --- a/Makefile +++ b/Makefile @@ -42,20 +42,11 @@ python-client-tests: python-tests: python-server-tests python-client-tests -run-bloom-560m: - text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --port 8080 +run-falcon-7b-instruct: + text-generation-launcher --model-id tiiuae/falcon-7b-instruct --port 8080 -run-bloom-560m-quantize: - text-generation-launcher --model-id bigscience/bloom-560m --num-shard 2 --quantize --port 8080 - -download-bloom: - HF_HUB_ENABLE_HF_TRANSFER=1 text-generation-server download-weights bigscience/bloom - -run-bloom: - text-generation-launcher --model-id bigscience/bloom --num-shard 8 --port 8080 - -run-bloom-quantize: - text-generation-launcher --model-id bigscience/bloom --num-shard 8 --quantize --port 8080 +run-falcon-7b-instruct-quantize: + text-generation-launcher --model-id tiiuae/falcon-7b-instruct --quantize bitsandbytes --port 8080 clean: rm -rf target aml diff --git a/README.md b/README.md index 43388d00..9e256c9d 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,12 @@ to power LLMs api-inference widgets. - [Get Started](#get-started) - [Docker](#docker) - [API Documentation](#api-documentation) + - [Using a private or gated model](#using-a-private-or-gated-model) - [A note on Shared Memory](#a-note-on-shared-memory-shm) - [Distributed Tracing](#distributed-tracing) - [Local Install](#local-install) - [CUDA Kernels](#cuda-kernels) -- [Run BLOOM](#run-bloom) - - [Download](#download) +- [Run Falcon](#run-falcon) - [Run](#run) - [Quantization](#quantization) - [Develop](#develop) @@ -81,11 +81,10 @@ or The easiest way of getting started is using the official Docker container: ```shell -model=bigscience/bloom-560m -num_shard=2 +model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.9 --model-id $model --num-shard $num_shard +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.9.3 --model-id $model ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. @@ -99,14 +98,14 @@ You can then query the model using either the `/generate` or `/generate_stream` ```shell curl 127.0.0.1:8080/generate \ -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17}}' \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ -H 'Content-Type: application/json' ``` ```shell curl 127.0.0.1:8080/generate_stream \ -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17}}' \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ -H 'Content-Type: application/json' ``` @@ -120,10 +119,10 @@ pip install text-generation from text_generation import Client client = Client("http://127.0.0.1:8080") -print(client.generate("What is Deep Learning?", max_new_tokens=17).generated_text) +print(client.generate("What is Deep Learning?", max_new_tokens=20).generated_text) text = "" -for response in client.generate_stream("What is Deep Learning?", max_new_tokens=17): +for response in client.generate_stream("What is Deep Learning?", max_new_tokens=20): if not response.token.special: text += response.token.text print(text) @@ -134,14 +133,26 @@ print(text) You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. The Swagger UI is also available at: [https://huggingface.github.io/text-generation-inference](https://huggingface.github.io/text-generation-inference). -### Using on private models or gated models +### Using a private or gated model -You can use `HUGGING_FACE_HUB_TOKEN` environment variable to set the token used by `text-generation-inference` to give access to protected ressources. +You have the option to utilize the `HUGGING_FACE_HUB_TOKEN` environment variable for configuring the token employed by +`text-generation-inference`. This allows you to gain access to protected resources. -### Distributed Tracing +For example, if you want to serve the gated Llama V2 model variants: -`text-generation-inference` is instrumented with distributed tracing using OpenTelemetry. You can use this feature -by setting the address to an OTLP collector with the `--otlp-endpoint` argument. +1. Go to https://huggingface.co/settings/tokens +2. Copy your cli READ token +3. Export `HUGGING_FACE_HUB_TOKEN=` + +or with Docker: + +```shell +model=meta-llama/Llama-2-7b-chat-hf +volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run +token= + +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.9.3 --model-id $model +``` ### A note on Shared Memory (shm) @@ -169,6 +180,11 @@ and mounting it to `/dev/shm`. Finally, you can also disable SHM sharing by using the `NCCL_SHM_DISABLE=1` environment variable. However, note that this will impact performance. +### Distributed Tracing + +`text-generation-inference` is instrumented with distributed tracing using OpenTelemetry. You can use this feature +by setting the address to an OTLP collector with the `--otlp-endpoint` argument. + ### Local install You can also opt to install `text-generation-inference` locally. @@ -205,7 +221,7 @@ Then run: ```shell BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork with CUDA kernels -make run-bloom-560m +make run-falcon-7b-instruct ``` **Note:** on some machines, you may also need the OpenSSL libraries and gcc. On Linux machines, run: @@ -221,20 +237,12 @@ the kernels by using the `DISABLE_CUSTOM_KERNELS=True` environment variable. Be aware that the official Docker image has them enabled by default. -## Run BLOOM - -### Download - -It is advised to download the weights ahead of time with the following command: - -```shell -make download-bloom -``` +## Run Falcon ### Run ```shell -make run-bloom # Requires 8xA100 80GB +make run-falcon-7b-instruct ``` ### Quantization @@ -242,7 +250,7 @@ make run-bloom # Requires 8xA100 80GB You can also quantize the weights with bitsandbytes to reduce the VRAM requirement: ```shell -make run-bloom-quantize # Requires 8xA100 40GB +make run-falcon-7b-instruct-quantize ``` ## Develop From 8bd0adb1356d5c6e0238abe64a6cf94b50140db4 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 27 Jul 2023 12:28:10 +0200 Subject: [PATCH 343/793] fix(server): fix quantization python requirements (#708) --- server/poetry.lock | 14 +++++++++- server/pyproject.toml | 1 + server/requirements.txt | 1 + .../text_generation_server/models/flash_rw.py | 1 - server/text_generation_server/server.py | 26 +++++++++---------- .../utils/gptq/quantize.py | 8 ++---- 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/server/poetry.lock b/server/poetry.lock index 7d00f223..4349340f 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -624,6 +624,14 @@ python-versions = ">=3.8" [package.dependencies] mpmath = ">=0.19" +[[package]] +name = "texttable" +version = "1.6.7" +description = "module to create simple ASCII tables" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "tokenizers" version = "0.13.3" @@ -810,7 +818,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "65afc4bfa07da4b1427d269fa745939da3851eaede9a8478f5a4bf5949d32cc9" +content-hash = "c2e0d926748a7d420909c6bd21e17cf060bc7acdd788ae93e3ec1809a4b84529" [metadata.files] accelerate = [ @@ -1484,6 +1492,10 @@ sympy = [ {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, {file = "sympy-1.12.tar.gz", hash = "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8"}, ] +texttable = [ + {file = "texttable-1.6.7-py2.py3-none-any.whl", hash = "sha256:b7b68139aa8a6339d2c320ca8b1dc42d13a7831a346b446cb9eb385f0c76310c"}, + {file = "texttable-1.6.7.tar.gz", hash = "sha256:290348fb67f7746931bcdfd55ac7584ecd4e5b0846ab164333f0794b121760f2"}, +] tokenizers = [ {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, {file = "tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index be79da51..a0fc4411 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -28,6 +28,7 @@ tokenizers = "0.13.3" huggingface-hub = "^0.14.1" transformers = "4.29.2" einops = "^0.6.1" +texttable = "^1.6.7" [tool.poetry.extras] accelerate = ["accelerate"] diff --git a/server/requirements.txt b/server/requirements.txt index 9b8b2164..02954f24 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -35,6 +35,7 @@ requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0" safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "4.0" setuptools==68.0.0 ; python_version >= "3.9" and python_version < "4.0" +texttable==1.6.7 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" transformers==4.29.2 ; python_version >= "3.9" and python_version < "4.0" diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 9e0080a9..7d655945 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -61,7 +61,6 @@ def __init__( if config.quantize == "gptq": weights._set_gptq_params(model_id) - model = FlashRWForCausalLM(config, weights) torch.distributed.barrier(group=self.process_group) diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 7c2f1b35..1cedc151 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -105,21 +105,21 @@ async def Decode(self, request, context): def serve( - model_id: str, - revision: Optional[str], - sharded: bool, - quantize: Optional[str], - dtype: Optional[str], - trust_remote_code: bool, - uds_path: Path, + model_id: str, + revision: Optional[str], + sharded: bool, + quantize: Optional[str], + dtype: Optional[str], + trust_remote_code: bool, + uds_path: Path, ): async def serve_inner( - model_id: str, - revision: Optional[str], - sharded: bool = False, - quantize: Optional[str] = None, - dtype: Optional[str] = None, - trust_remote_code: bool = False, + model_id: str, + revision: Optional[str], + sharded: bool = False, + quantize: Optional[str] = None, + dtype: Optional[str] = None, + trust_remote_code: bool = False, ): unix_socket_template = "unix://{}-{}" if sharded: diff --git a/server/text_generation_server/utils/gptq/quantize.py b/server/text_generation_server/utils/gptq/quantize.py index 6eb44e41..3f8e897a 100644 --- a/server/text_generation_server/utils/gptq/quantize.py +++ b/server/text_generation_server/utils/gptq/quantize.py @@ -1,18 +1,14 @@ -import argparse import time -import numpy as np -import torch import torch.nn as nn import math import json import os +import torch +import transformers from texttable import Texttable from transformers import AutoModelForCausalLM, AutoConfig, AutoTokenizer -import transformers from huggingface_hub import HfApi -import numpy as np -import torch from accelerate import init_empty_weights from text_generation_server.utils import initialize_torch_distributed, Weights from text_generation_server.utils.hub import weight_files From 2efd46ef95991a227c9ec79821e6008fdfeb901d Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 27 Jul 2023 14:50:45 +0200 Subject: [PATCH 344/793] fix(server): fix missing datasets in quantize --- Dockerfile | 2 +- server/Makefile | 2 +- server/poetry.lock | 759 +++++++++++++++++++++++++++++++++++++++- server/pyproject.toml | 4 +- server/requirements.txt | 28 +- 5 files changed, 789 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 70c60132..34109d02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -188,7 +188,7 @@ COPY server/Makefile server/Makefile RUN cd server && \ make gen-server && \ pip install -r requirements.txt && \ - pip install ".[bnb, accelerate]" --no-cache-dir + pip install ".[bnb, accelerate, quantize]" --no-cache-dir # Install benchmarker COPY --from=builder /usr/src/target/release/text-generation-benchmark /usr/local/bin/text-generation-benchmark diff --git a/server/Makefile b/server/Makefile index 0dc0b5c9..a4ce6d8b 100644 --- a/server/Makefile +++ b/server/Makefile @@ -27,4 +27,4 @@ run-dev: SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded export-requirements: - poetry export -o requirements.txt -E bnb --without-hashes + poetry export -o requirements.txt -E bnb -E quantize --without-hashes diff --git a/server/poetry.lock b/server/poetry.lock index 4349340f..16da22f5 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -23,6 +23,60 @@ test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] test_trackers = ["comet-ml", "tensorboard", "wandb"] testing = ["datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] +[[package]] +name = "aiohttp" +version = "3.8.5" +description = "Async http client/server framework (asyncio)" +category = "main" +optional = true +python-versions = ">=3.6" + +[package.dependencies] +aiosignal = ">=1.1.2" +async-timeout = ">=4.0.0a3,<5.0" +attrs = ">=17.3.0" +charset-normalizer = ">=2.0,<4.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "cchardet"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = true +python-versions = ">=3.6" + +[[package]] +name = "attrs" +version = "23.1.0" +description = "Classes Without Boilerplate" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + [[package]] name = "backoff" version = "2.2.1" @@ -74,6 +128,45 @@ category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +[[package]] +name = "datasets" +version = "2.14.0" +description = "HuggingFace community-driven open-source library of datasets" +category = "main" +optional = true +python-versions = ">=3.8.0" + +[package.dependencies] +aiohttp = "*" +dill = ">=0.3.0,<0.3.8" +fsspec = {version = ">=2021.11.1", extras = ["http"]} +huggingface-hub = ">=0.14.0,<1.0.0" +multiprocess = "*" +numpy = ">=1.17" +packaging = "*" +pandas = "*" +pyarrow = ">=8.0.0" +pyyaml = ">=5.1" +requests = ">=2.19.0" +tqdm = ">=4.62.1" +xxhash = "*" + +[package.extras] +apache-beam = ["apache-beam (>=2.26.0,<2.44.0)"] +audio = ["librosa", "soundfile (>=0.12.1)"] +benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] +dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "black (>=23.1,<24.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "rarfile (>=4.0)", "ruff (>=0.0.241)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] +docs = ["s3fs"] +jax = ["jax (>=0.2.8,!=0.3.2,<=0.3.25)", "jaxlib (>=0.1.65,<=0.3.25)"] +metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"] +quality = ["black (>=23.1,<24.0)", "pyyaml (>=5.3.1)", "ruff (>=0.0.241)"] +s3 = ["s3fs"] +tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos"] +tensorflow_gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] +tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] +torch = ["torch"] +vision = ["Pillow (>=6.2.1)"] + [[package]] name = "Deprecated" version = "1.2.14" @@ -88,6 +181,17 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] +[[package]] +name = "dill" +version = "0.3.7" +description = "serialize all of Python" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + [[package]] name = "einops" version = "0.6.1" @@ -119,6 +223,14 @@ python-versions = ">=3.7" docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +[[package]] +name = "frozenlist" +version = "1.4.0" +description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" +optional = true +python-versions = ">=3.8" + [[package]] name = "fsspec" version = "2023.6.0" @@ -127,6 +239,10 @@ category = "main" optional = false python-versions = ">=3.8" +[package.dependencies] +aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} +requests = {version = "*", optional = true, markers = "extra == \"http\""} + [package.extras] abfs = ["adlfs"] adl = ["adlfs"] @@ -331,6 +447,25 @@ docs = ["sphinx"] gmpy = ["gmpy2 (>=2.1.0a4)"] tests = ["pytest (>=4.6)"] +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +category = "main" +optional = true +python-versions = ">=3.7" + +[[package]] +name = "multiprocess" +version = "0.70.15" +description = "better multiprocessing and multithreading in Python" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.dependencies] +dill = ">=0.3.7" + [[package]] name = "networkx" version = "3.1" @@ -489,6 +624,47 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "pandas" +version = "2.0.3" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = true +python-versions = ">=3.8" + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] +aws = ["s3fs (>=2021.08.0)"] +clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] +compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] +computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2021.07.0)"] +gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] +hdf5 = ["tables (>=3.6.1)"] +html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] +mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] +output_formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] +spss = ["pyreadstat (>=1.1.2)"] +sql-other = ["SQLAlchemy (>=1.4.16)"] +test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.6.3)"] + [[package]] name = "pluggy" version = "1.2.0" @@ -520,6 +696,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +[[package]] +name = "pyarrow" +version = "12.0.1" +description = "Python library for Apache Arrow" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.dependencies] +numpy = ">=1.16.6" + [[package]] name = "pytest" version = "7.4.0" @@ -539,6 +726,25 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2023.3" +description = "World timezone definitions, modern and historical" +category = "main" +optional = true +python-versions = "*" + [[package]] name = "PyYAML" version = "6.0" @@ -613,6 +819,14 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + [[package]] name = "sympy" version = "1.12" @@ -629,7 +843,7 @@ name = "texttable" version = "1.6.7" description = "module to create simple ASCII tables" category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -778,6 +992,14 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "tzdata" +version = "2023.3" +description = "Provider of IANA time zone data" +category = "main" +optional = true +python-versions = ">=2" + [[package]] name = "urllib3" version = "2.0.3" @@ -811,20 +1033,142 @@ category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +[[package]] +name = "xxhash" +version = "3.2.0" +description = "Python binding for xxHash" +category = "main" +optional = true +python-versions = ">=3.6" + +[[package]] +name = "yarl" +version = "1.9.2" +description = "Yet another URL library" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + [extras] accelerate = ["accelerate"] bnb = ["bitsandbytes"] +quantize = ["texttable", "datasets", "accelerate"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "c2e0d926748a7d420909c6bd21e17cf060bc7acdd788ae93e3ec1809a4b84529" +content-hash = "4a717b962be975c23acbfed8946cf2e2d55497c8ffc192debd09d440127f5cc3" [metadata.files] accelerate = [ {file = "accelerate-0.19.0-py3-none-any.whl", hash = "sha256:2866b0bf9fff08f51e6384c95fa96725838b70f1988d1cce42e56b820d8a91dd"}, {file = "accelerate-0.19.0.tar.gz", hash = "sha256:84920226b9e642e453ef37593ee55b956b08d8200dea4087c546c34e26157e76"}, ] +aiohttp = [ + {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8"}, + {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84"}, + {file = "aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825"}, + {file = "aiohttp-3.8.5-cp310-cp310-win32.whl", hash = "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802"}, + {file = "aiohttp-3.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c"}, + {file = "aiohttp-3.8.5-cp311-cp311-win32.whl", hash = "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945"}, + {file = "aiohttp-3.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755"}, + {file = "aiohttp-3.8.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"}, + {file = "aiohttp-3.8.5-cp36-cp36m-win32.whl", hash = "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e"}, + {file = "aiohttp-3.8.5-cp36-cp36m-win_amd64.whl", hash = "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-win32.whl", hash = "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22"}, + {file = "aiohttp-3.8.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35"}, + {file = "aiohttp-3.8.5-cp38-cp38-win32.whl", hash = "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c"}, + {file = "aiohttp-3.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91"}, + {file = "aiohttp-3.8.5-cp39-cp39-win32.whl", hash = "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67"}, + {file = "aiohttp-3.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c"}, + {file = "aiohttp-3.8.5.tar.gz", hash = "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc"}, +] +aiosignal = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] +async-timeout = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] +attrs = [ + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, +] backoff = [ {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, @@ -922,10 +1266,18 @@ colorama = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +datasets = [ + {file = "datasets-2.14.0-py3-none-any.whl", hash = "sha256:93081cc3d9d0ce860c81f950a3ba23d24704da2eacbe2722092ef4f6ae0ada96"}, + {file = "datasets-2.14.0.tar.gz", hash = "sha256:1bb3d1c992a593949a8d3e445b358ac1db4ead00e6619ea2e5e7b6dfc222dde1"}, +] Deprecated = [ {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, ] +dill = [ + {file = "dill-0.3.7-py3-none-any.whl", hash = "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e"}, + {file = "dill-0.3.7.tar.gz", hash = "sha256:cc1c8b182eb3013e24bd475ff2e9295af86c1a38eb1aff128dac8962a9ce3c03"}, +] einops = [ {file = "einops-0.6.1-py3-none-any.whl", hash = "sha256:99149e46cc808956b174932fe563d920db4d6e5dadb8c6ecdaa7483b7ef7cfc3"}, {file = "einops-0.6.1.tar.gz", hash = "sha256:f95f8d00f4ded90dbc4b19b6f98b177332614b0357dde66997f3ae5d474dc8c8"}, @@ -938,6 +1290,69 @@ filelock = [ {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, ] +frozenlist = [ + {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab"}, + {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559"}, + {file = "frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62"}, + {file = "frozenlist-1.4.0-cp310-cp310-win32.whl", hash = "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0"}, + {file = "frozenlist-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956"}, + {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95"}, + {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3"}, + {file = "frozenlist-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb"}, + {file = "frozenlist-1.4.0-cp311-cp311-win32.whl", hash = "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431"}, + {file = "frozenlist-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1"}, + {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3"}, + {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503"}, + {file = "frozenlist-1.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8"}, + {file = "frozenlist-1.4.0-cp38-cp38-win32.whl", hash = "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc"}, + {file = "frozenlist-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7"}, + {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf"}, + {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963"}, + {file = "frozenlist-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3"}, + {file = "frozenlist-1.4.0-cp39-cp39-win32.whl", hash = "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f"}, + {file = "frozenlist-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167"}, + {file = "frozenlist-1.4.0.tar.gz", hash = "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251"}, +] fsspec = [ {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"}, {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"}, @@ -1149,6 +1564,100 @@ mpmath = [ {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, ] +multidict = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] +multiprocess = [ + {file = "multiprocess-0.70.15-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:aa36c7ed16f508091438687fe9baa393a7a8e206731d321e443745e743a0d4e5"}, + {file = "multiprocess-0.70.15-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:20e024018c46d0d1602024c613007ac948f9754659e3853b0aa705e83f6931d8"}, + {file = "multiprocess-0.70.15-pp37-pypy37_pp73-manylinux_2_24_i686.whl", hash = "sha256:e576062981c91f0fe8a463c3d52506e598dfc51320a8dd8d78b987dfca91c5db"}, + {file = "multiprocess-0.70.15-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:e73f497e6696a0f5433ada2b3d599ae733b87a6e8b008e387c62ac9127add177"}, + {file = "multiprocess-0.70.15-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:73db2e7b32dcc7f9b0f075c2ffa45c90b6729d3f1805f27e88534c8d321a1be5"}, + {file = "multiprocess-0.70.15-pp38-pypy38_pp73-manylinux_2_24_i686.whl", hash = "sha256:4271647bd8a49c28ecd6eb56a7fdbd3c212c45529ad5303b40b3c65fc6928e5f"}, + {file = "multiprocess-0.70.15-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:cf981fb998d6ec3208cb14f0cf2e9e80216e834f5d51fd09ebc937c32b960902"}, + {file = "multiprocess-0.70.15-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:18f9f2c7063346d1617bd1684fdcae8d33380ae96b99427260f562e1a1228b67"}, + {file = "multiprocess-0.70.15-pp39-pypy39_pp73-manylinux_2_24_i686.whl", hash = "sha256:0eac53214d664c49a34695e5824872db4006b1a465edd7459a251809c3773370"}, + {file = "multiprocess-0.70.15-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:1a51dd34096db47fb21fa2b839e615b051d51b97af9a67afbcdaa67186b44883"}, + {file = "multiprocess-0.70.15-py310-none-any.whl", hash = "sha256:7dd58e33235e83cf09d625e55cffd7b0f0eede7ee9223cdd666a87624f60c21a"}, + {file = "multiprocess-0.70.15-py311-none-any.whl", hash = "sha256:134f89053d82c9ed3b73edd3a2531eb791e602d4f4156fc92a79259590bd9670"}, + {file = "multiprocess-0.70.15-py37-none-any.whl", hash = "sha256:f7d4a1629bccb433114c3b4885f69eccc200994323c80f6feee73b0edc9199c5"}, + {file = "multiprocess-0.70.15-py38-none-any.whl", hash = "sha256:bee9afba476c91f9ebee7beeee0601face9eff67d822e893f9a893725fbd6316"}, + {file = "multiprocess-0.70.15-py39-none-any.whl", hash = "sha256:3e0953f5d52b4c76f1c973eaf8214554d146f2be5decb48e928e55c7a2d19338"}, + {file = "multiprocess-0.70.15.tar.gz", hash = "sha256:f20eed3036c0ef477b07a4177cf7c1ba520d9a2677870a4f47fe026f0cd6787e"}, +] networkx = [ {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, @@ -1220,6 +1729,33 @@ packaging = [ {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] +pandas = [ + {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, + {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0"}, + {file = "pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210"}, + {file = "pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df"}, + {file = "pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd"}, + {file = "pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0"}, + {file = "pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02"}, + {file = "pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641"}, + {file = "pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682"}, + {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, + {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, +] pluggy = [ {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, @@ -1255,10 +1791,45 @@ psutil = [ {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, ] +pyarrow = [ + {file = "pyarrow-12.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df"}, + {file = "pyarrow-12.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf"}, + {file = "pyarrow-12.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082"}, + {file = "pyarrow-12.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63"}, + {file = "pyarrow-12.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d"}, + {file = "pyarrow-12.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60"}, + {file = "pyarrow-12.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a"}, + {file = "pyarrow-12.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7"}, + {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, +] pytest = [ {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, ] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +pytz = [ + {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, + {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, +] PyYAML = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, @@ -1488,6 +2059,10 @@ setuptools = [ {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, ] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] sympy = [ {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, {file = "sympy-1.12.tar.gz", hash = "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8"}, @@ -1580,6 +2155,10 @@ typing-extensions = [ {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] +tzdata = [ + {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, + {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, +] urllib3 = [ {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, @@ -1665,3 +2244,179 @@ wrapt = [ {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] +xxhash = [ + {file = "xxhash-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:af44b9e59c4b2926a4e3c7f9d29949ff42fcea28637ff6b8182e654461932be8"}, + {file = "xxhash-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1bdd57973e2b802ef32553d7bebf9402dac1557874dbe5c908b499ea917662cd"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b7c9aa77bbce61a5e681bd39cb6a804338474dcc90abe3c543592aa5d6c9a9b"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11bf87dc7bb8c3b0b5e24b7b941a9a19d8c1f88120b6a03a17264086bc8bb023"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2783d41487ce6d379fdfaa7332fca5187bf7010b9bddcf20cafba923bc1dc665"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:561076ca0dcef2fbc20b2bc2765bff099e002e96041ae9dbe910a863ca6ee3ea"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a26eeb4625a6e61cedc8c1b39b89327c9c7e1a8c2c4d786fe3f178eb839ede6"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d93a44d0104d1b9b10de4e7aadf747f6efc1d7ec5ed0aa3f233a720725dd31bd"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:89585adc73395a10306d2e2036e50d6c4ac0cf8dd47edf914c25488871b64f6d"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a892b4b139126a86bfdcb97cd912a2f8c4e8623869c3ef7b50871451dd7afeb0"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e998efb190653f70e0f30d92b39fc645145369a4823bee46af8ddfc244aa969d"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e8ed3bd2b8bb3277710843ca63e4f5c3ee6f8f80b083be5b19a7a9905420d11e"}, + {file = "xxhash-3.2.0-cp310-cp310-win32.whl", hash = "sha256:20181cbaed033c72cb881b2a1d13c629cd1228f113046133469c9a48cfcbcd36"}, + {file = "xxhash-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:a0f7a16138279d707db778a63264d1d6016ac13ffd3f1e99f54b2855d6c0d8e1"}, + {file = "xxhash-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5daff3fb5bfef30bc5a2cb143810d376d43461445aa17aece7210de52adbe151"}, + {file = "xxhash-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75bb5be3c5de702a547715f320ecf5c8014aeca750ed5147ca75389bd22e7343"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01f36b671ff55cb1d5c2f6058b799b697fd0ae4b4582bba6ed0999678068172a"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4d4519123aac73c93159eb8f61db9682393862dd669e7eae034ecd0a35eadac"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:994e4741d5ed70fc2a335a91ef79343c6b1089d7dfe6e955dd06f8ffe82bede6"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:919bc1b010aa6ff0eb918838ff73a435aed9e9a19c3202b91acecd296bf75607"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17b65454c5accbb079c45eca546c27c4782f5175aa320758fafac896b1549d27"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b0c094d5e65a46dbf3fe0928ff20873a747e6abfd2ed4b675beeb2750624bc2e"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f94163ebe2d5546e6a5977e96d83621f4689c1054053428cf8d4c28b10f92f69"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cead7c0307977a00b3f784cff676e72c147adbcada19a2e6fc2ddf54f37cf387"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a0e1bd0260c1da35c1883321ce2707ceea07127816ab625e1226ec95177b561a"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc8878935671490efe9275fb4190a6062b73277bd273237179b9b5a2aa436153"}, + {file = "xxhash-3.2.0-cp311-cp311-win32.whl", hash = "sha256:a433f6162b18d52f7068175d00bd5b1563b7405f926a48d888a97b90a160c40d"}, + {file = "xxhash-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:a32d546a1752e4ee7805d6db57944f7224afa7428d22867006b6486e4195c1f3"}, + {file = "xxhash-3.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:82daaab720866bf690b20b49de5640b0c27e3b8eea2d08aa75bdca2b0f0cfb63"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3126df6520cbdbaddd87ce74794b2b6c45dd2cf6ac2b600a374b8cdb76a2548c"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e172c1ee40507ae3b8d220f4048aaca204f203e1e4197e8e652f5c814f61d1aa"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5384f1d9f30876f5d5b618464fb19ff7ce6c0fe4c690fbaafd1c52adc3aae807"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26cb52174a7e96a17acad27a3ca65b24713610ac479c99ac9640843822d3bebf"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbcd613a5e76b1495fc24db9c37a6b7ee5f214fd85979187ec4e032abfc12ded"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:f988daf25f31726d5b9d0be6af636ca9000898f9ea43a57eac594daea25b0948"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:bbc30c98ab006ab9fc47e5ed439c00f706bc9d4441ff52693b8b6fea335163e0"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:2408d49260b0a4a7cc6ba445aebf38e073aeaf482f8e32767ca477e32ccbbf9e"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:3f4152fd0bf8b03b79f2f900fd6087a66866537e94b5a11fd0fd99ef7efe5c42"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:0eea848758e4823a01abdbcccb021a03c1ee4100411cbeeb7a5c36a202a0c13c"}, + {file = "xxhash-3.2.0-cp36-cp36m-win32.whl", hash = "sha256:77709139af5123c578ab06cf999429cdb9ab211047acd0c787e098dcb3f1cb4d"}, + {file = "xxhash-3.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:91687671fd9d484a4e201ad266d366b695a45a1f2b41be93d116ba60f1b8f3b3"}, + {file = "xxhash-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e4af8bc5c3fcc2192c266421c6aa2daab1a18e002cb8e66ef672030e46ae25cf"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8be562e2ce3e481d9209b6f254c3d7c5ff920eb256aba2380d2fb5ba75d4f87"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9eba0c7c12126b12f7fcbea5513f28c950d28f33d2a227f74b50b77789e478e8"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2198c4901a0223c48f6ec0a978b60bca4f4f7229a11ca4dc96ca325dd6a29115"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50ce82a71b22a3069c02e914bf842118a53065e2ec1c6fb54786e03608ab89cc"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b5019fb33711c30e54e4e57ae0ca70af9d35b589d385ac04acd6954452fa73bb"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0d54ac023eef7e3ac9f0b8841ae8a376b933043bc2ad428121346c6fa61c491c"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c55fa832fc3fe64e0d29da5dc9b50ba66ca93312107cec2709300ea3d3bab5c7"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f4ce006215497993ae77c612c1883ca4f3973899573ce0c52fee91f0d39c4561"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1afb9b9d27fd675b436cb110c15979976d92d761ad6e66799b83756402f3a974"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:baa99cebf95c1885db21e119395f222a706a2bb75a545f0672880a442137725e"}, + {file = "xxhash-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:75aa692936942ccb2e8fd6a386c81c61630ac1b6d6e921698122db8a930579c3"}, + {file = "xxhash-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:0a2cdfb5cae9fafb9f7b65fd52ecd60cf7d72c13bb2591ea59aaefa03d5a8827"}, + {file = "xxhash-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a68d1e8a390b660d94b9360ae5baa8c21a101bd9c4790a8b30781bada9f1fc6"}, + {file = "xxhash-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ce7c3ce28f94302df95eaea7c9c1e2c974b6d15d78a0c82142a97939d7b6c082"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dcb419bf7b0bc77d366e5005c25682249c5521a63fd36c51f584bd91bb13bd5"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae521ed9287f86aac979eeac43af762f03d9d9797b2272185fb9ddd810391216"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0d16775094423088ffa357d09fbbb9ab48d2fb721d42c0856b801c86f616eec"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe454aeab348c42f56d6f7434ff758a3ef90787ac81b9ad5a363cd61b90a1b0b"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:052fd0efdd5525c2dbc61bebb423d92aa619c4905bba605afbf1e985a562a231"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:02badf3754e2133de254a4688798c4d80f0060635087abcb461415cb3eb82115"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:66b8a90b28c13c2aae7a71b32638ceb14cefc2a1c8cf23d8d50dfb64dfac7aaf"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:649cdf19df175925ad87289ead6f760cd840730ee85abc5eb43be326a0a24d97"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4b948a03f89f5c72d69d40975af8af241111f0643228796558dc1cae8f5560b0"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49f51fab7b762da7c2cee0a3d575184d3b9be5e2f64f26cae2dd286258ac9b3c"}, + {file = "xxhash-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1a42994f0d42b55514785356722d9031f064fd34e495b3a589e96db68ee0179d"}, + {file = "xxhash-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0a6d58ba5865475e53d6c2c4fa6a62e2721e7875e146e2681e5337a6948f12e7"}, + {file = "xxhash-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:aabdbc082030f8df613e2d2ea1f974e7ad36a539bdfc40d36f34e55c7e4b8e94"}, + {file = "xxhash-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:498843b66b9ca416e9d03037e5875c8d0c0ab9037527e22df3b39aa5163214cd"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a910b1193cd90af17228f5d6069816646df0148f14f53eefa6b2b11a1dedfcd0"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb6d8ce31dc25faf4da92991320e211fa7f42de010ef51937b1dc565a4926501"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:883dc3d3942620f4c7dbc3fd6162f50a67f050b714e47da77444e3bcea7d91cc"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59dc8bfacf89b8f5be54d55bc3b4bd6d74d0c5320c8a63d2538ac7df5b96f1d5"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:61e6aa1d30c2af692aa88c4dd48709426e8b37bff6a574ee2de677579c34a3d6"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:314ec0bd21f0ee8d30f2bd82ed3759314bd317ddbbd8555668f3d20ab7a8899a"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dad638cde3a5357ad3163b80b3127df61fb5b5e34e9e05a87697144400ba03c7"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:eaa3ea15025b56076d806b248948612289b093e8dcda8d013776b3848dffff15"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:7deae3a312feb5c17c97cbf18129f83cbd3f1f9ec25b0f50e2bd9697befb22e7"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:add774341c09853b1612c64a526032d95ab1683053325403e1afbe3ad2f374c5"}, + {file = "xxhash-3.2.0-cp39-cp39-win32.whl", hash = "sha256:9b94749130ef3119375c599bfce82142c2500ef9ed3280089157ee37662a7137"}, + {file = "xxhash-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:e57d94a1552af67f67b27db5dba0b03783ea69d5ca2af2f40e098f0ba3ce3f5f"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92fd765591c83e5c5f409b33eac1d3266c03d3d11c71a7dbade36d5cdee4fbc0"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8970f6a411a9839a02b23b7e90bbbba4a6de52ace009274998566dc43f36ca18"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5f3e33fe6cbab481727f9aeb136a213aed7e33cd1ca27bd75e916ffacc18411"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:368265392cb696dd53907e2328b5a8c1bee81cf2142d0cc743caf1c1047abb36"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3b1f3c6d67fa9f49c4ff6b25ce0e7143bab88a5bc0f4116dd290c92337d0ecc7"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c5e8db6e1ee7267b7c412ad0afd5863bf7a95286b8333a5958c8097c69f94cf5"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:761df3c7e2c5270088b691c5a8121004f84318177da1ca1db64222ec83c44871"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2d15a707e7f689531eb4134eccb0f8bf3844bb8255ad50823aa39708d9e6755"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6b2ba4ff53dd5f57d728095e3def7375eb19c90621ce3b41b256de84ec61cfd"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:61b0bcf946fdfd8ab5f09179dc2b5c74d1ef47cedfc6ed0ec01fdf0ee8682dd3"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f7b79f0f302396d8e0d444826ceb3d07b61977793886ebae04e82796c02e42dc"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0773cd5c438ffcd5dbff91cdd503574f88a4b960e70cedeb67736583a17a918"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ec1f57127879b419a2c8d2db9d9978eb26c61ae17e5972197830430ae78d25b"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d4b15c00e807b1d3d0b612338c814739dec310b80fb069bd732b98ddc709ad7"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:9d3f686e3d1c8900c5459eee02b60c7399e20ec5c6402364068a343c83a61d90"}, + {file = "xxhash-3.2.0.tar.gz", hash = "sha256:1afd47af8955c5db730f630ad53ae798cf7fae0acb64cebb3cf94d35c47dd088"}, +] +yarl = [ + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, + {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, + {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, + {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, + {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, + {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, + {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, + {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, + {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, + {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, + {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, + {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, + {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, +] diff --git a/server/pyproject.toml b/server/pyproject.toml index a0fc4411..40cd31ee 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -28,11 +28,13 @@ tokenizers = "0.13.3" huggingface-hub = "^0.14.1" transformers = "4.29.2" einops = "^0.6.1" -texttable = "^1.6.7" +texttable = { version = "^1.6.7", optional = true } +datasets = { version = "^2.14.0", optional = true } [tool.poetry.extras] accelerate = ["accelerate"] bnb = ["bitsandbytes"] +quantize = ["texttable", "datasets", "accelerate"] [tool.poetry.group.dev.dependencies] grpcio-tools = "^1.51.1" diff --git a/server/requirements.txt b/server/requirements.txt index 02954f24..98838b36 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,13 +1,22 @@ +accelerate==0.19.0 ; python_version >= "3.9" and python_version < "4.0" +aiohttp==3.8.5 ; python_version >= "3.9" and python_version < "4.0" +aiosignal==1.3.1 ; python_version >= "3.9" and python_version < "4.0" +async-timeout==4.0.2 ; python_version >= "3.9" and python_version < "4.0" +attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0" backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" bitsandbytes==0.38.1 ; python_version >= "3.9" and python_version < "4.0" certifi==2023.5.7 ; python_version >= "3.9" and python_version < "4.0" charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0" click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows" +datasets==2.14.0 ; python_version >= "3.9" and python_version < "4.0" deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0" +dill==0.3.7 ; python_version >= "3.9" and python_version < "4.0" einops==0.6.1 ; python_version >= "3.9" and python_version < "4.0" filelock==3.12.2 ; python_version >= "3.9" and python_version < "4.0" +frozenlist==1.4.0 ; python_version >= "3.9" and python_version < "4.0" fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "4.0" +fsspec[http]==2023.6.0 ; python_version >= "3.9" and python_version < "4.0" googleapis-common-protos==1.59.1 ; python_version >= "3.9" and python_version < "4.0" grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "4.0" grpcio-reflection==1.56.0 ; python_version >= "3.9" and python_version < "4.0" @@ -16,8 +25,14 @@ grpcio==1.56.0 ; python_version >= "3.9" and python_version < "4.0" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" huggingface-hub==0.14.1 ; python_version >= "3.9" and python_version < "4.0" idna==3.4 ; python_version >= "3.9" and python_version < "4.0" +jinja2==3.1.2 ; python_version >= "3.9" and python_version < "4.0" loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" -numpy==1.25.0 ; python_version >= "3.9" and python_version < "4.0" +markupsafe==2.1.3 ; python_version >= "3.9" and python_version < "4.0" +mpmath==1.3.0 ; python_version >= "3.9" and python_version < "4.0" +multidict==6.0.4 ; python_version >= "3.9" and python_version < "4.0" +multiprocess==0.70.15 ; python_version >= "3.9" and python_version < "4.0" +networkx==3.1 ; python_version >= "3.9" and python_version < "4.0" +numpy==1.25.0 ; python_version < "4.0" and python_version >= "3.9" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "4.0" @@ -28,19 +43,30 @@ opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" +pandas==2.0.3 ; python_version >= "3.9" and python_version < "4.0" protobuf==4.23.3 ; python_version >= "3.9" and python_version < "4.0" +psutil==5.9.5 ; python_version >= "3.9" and python_version < "4.0" +pyarrow==12.0.1 ; python_version >= "3.9" and python_version < "4.0" +python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0" +pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0" pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" regex==2023.6.3 ; python_version >= "3.9" and python_version < "4.0" requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0" safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "4.0" setuptools==68.0.0 ; python_version >= "3.9" and python_version < "4.0" +six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" +sympy==1.12 ; python_version >= "3.9" and python_version < "4.0" texttable==1.6.7 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" +torch==2.0.1 ; python_version >= "3.9" and python_version < "4.0" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" transformers==4.29.2 ; python_version >= "3.9" and python_version < "4.0" typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "4.0" +tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0" urllib3==2.0.3 ; python_version >= "3.9" and python_version < "4.0" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" +xxhash==3.2.0 ; python_version >= "3.9" and python_version < "4.0" +yarl==1.9.2 ; python_version >= "3.9" and python_version < "4.0" From ab96b9aec30451dcbfdfd8112e002b3ab878fcb6 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 27 Jul 2023 18:38:57 +0200 Subject: [PATCH 345/793] feat(server): support new falcon config (#712) --- .../text_generation_server/models/__init__.py | 11 ++-- .../custom_modeling/flash_rw_modeling.py | 53 ++++++++++++------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index ffc224cc..e9260eed 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -200,13 +200,10 @@ def get_model( trust_remote_code=trust_remote_code, ) - if model_type in ["RefinedWeb", "RefinedWebModel"]: + if model_type in ["RefinedWeb", "RefinedWebModel", "falcon"]: if sharded: if FLASH_ATTENTION: - if config_dict.get("alibi", False) or ( - model_type == "RefinedWebModel" - and config_dict.get("multi_query", True) - ): + if config_dict.get("alibi", False): raise NotImplementedError("sharded is not supported for this model") return FlashRWSharded( model_id, @@ -215,9 +212,7 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) - raise NotImplementedError( - FLASH_ATT_ERROR_MESSAGE.format(f"Sharded RefinedWeb") - ) + raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format(f"Sharded Falcon")) else: if FLASH_ATTENTION and not config_dict.get("alibi", False): return FlashRWSharded( diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 1e9539c4..3570b283 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -49,8 +49,8 @@ def __init__( model_type="RefinedWeb", vocab_size=250880, hidden_size=64, - n_layer=2, - n_head=8, + num_hidden_layers=None, + num_attention_heads=None, layer_norm_epsilon=1e-5, initializer_range=0.02, use_cache=True, @@ -58,9 +58,10 @@ def __init__( eos_token_id=2, hidden_dropout=0.0, attention_dropout=0.0, - n_head_kv=None, + num_kv_heads=None, multi_query=False, alibi=False, + new_decoder_architecture=None, bias=False, parallel_attn=False, **kwargs, @@ -78,8 +79,16 @@ def __init__( # Backward compatibility with n_embed kwarg n_embed = kwargs.pop("n_embed", None) self.hidden_size = hidden_size if n_embed is None else n_embed - self.n_layer = n_layer - self.n_head = n_head + self.n_layer = ( + num_hidden_layers + if num_hidden_layers is not None + else kwargs.pop("n_layer", 2) + ) + self.n_head = ( + num_attention_heads + if num_attention_heads is not None + else kwargs.pop("n_head", 8) + ) self.layer_norm_epsilon = layer_norm_epsilon self.initializer_range = initializer_range self.use_cache = use_cache @@ -91,10 +100,21 @@ def __init__( self.bos_token_id = bos_token_id self.eos_token_id = eos_token_id - if n_head_kv is not None: - self.n_head_kv = n_head_kv + if num_kv_heads is not None: + self.n_head_kv = num_kv_heads else: - self.n_head_kv = 1 if multi_query else n_head + old_n_head_kv = kwargs.pop("n_head_kv", None) + if old_n_head_kv is not None: + self.n_head_kv = old_n_head_kv + else: + self.n_head_kv = 1 if multi_query else self.n_head + + if new_decoder_architecture is not None: + self.new_decoder_architecture = new_decoder_architecture + elif model_type == "RefinedWeb": + self.new_decoder_architecture = True + else: + self.new_decoder_architecture = False super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) @@ -530,26 +550,23 @@ def __init__(self, config, weights): self.word_embeddings = TensorParallelEmbedding( prefix="transformer.word_embeddings", weights=weights ) - if config.model_type == "RefinedWebModel": + + if config.new_decoder_architecture: self.h = nn.ModuleList( [ - FlashRWLayer(layer_id, config, weights) + FlashRWLargeLayer(layer_id, config, weights) for layer_id in range(config.num_hidden_layers) ] ) - self.cache_size = self.h[0].self_attention.num_heads_kv - elif config.model_type == "RefinedWeb": + self.cache_size = self.h[0].self_attention.num_groups + else: self.h = nn.ModuleList( [ - FlashRWLargeLayer(layer_id, config, weights) + FlashRWLayer(layer_id, config, weights) for layer_id in range(config.num_hidden_layers) ] ) - self.cache_size = self.h[0].self_attention.num_groups - else: - raise NotImplementedError( - f"model_type {config.model_type} is not supported." - ) + self.cache_size = self.h[0].self_attention.num_heads_kv self.ln_f = FastLayerNorm.load( prefix="transformer.ln_f", From 9f18f4c00627e1a0ad696b6774e5ad7ca8f4261c Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 27 Jul 2023 19:25:15 +0200 Subject: [PATCH 346/793] v0.9.4 (#713) --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- README.md | 2 +- docs/openapi.json | 2 +- server/pyproject.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 539cf124..8984ea6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2893,7 +2893,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "0.9.3" +version = "0.9.4" dependencies = [ "average", "clap", @@ -2913,7 +2913,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.9.3" +version = "0.9.4" dependencies = [ "futures", "grpc-metadata", @@ -2929,7 +2929,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.9.3" +version = "0.9.4" dependencies = [ "clap", "ctrlc", @@ -2945,7 +2945,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.9.3" +version = "0.9.4" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index 49b7717a..3bfe9831 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.9.3" +version = "0.9.4" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index 9e256c9d..effab42e 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ The easiest way of getting started is using the official Docker container: model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.9.3 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.9.4 --model-id $model ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. diff --git a/docs/openapi.json b/docs/openapi.json index 80240460..9a672382 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.9.3" + "version": "0.9.4" }, "paths": { "/": { diff --git a/server/pyproject.toml b/server/pyproject.toml index 40cd31ee..3ee3351c 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.9.3" +version = "0.9.4" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 5a1cccbb98e08feaacb06484dcfd6b1230819b7b Mon Sep 17 00:00:00 2001 From: regisss <15324346+regisss@users.noreply.github.com> Date: Fri, 28 Jul 2023 09:14:03 +0200 Subject: [PATCH 347/793] Add section about TGI on other AI hardware accelerators in README (#715) # What does this PR do? As per title. ## Before submitting - [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- README.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index effab42e..c0783606 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ -A Rust, Python and gRPC server for text generation inference. Used in production at [HuggingFace](https://huggingface.co) +A Rust, Python and gRPC server for text generation inference. Used in production at [HuggingFace](https://huggingface.co) to power LLMs api-inference widgets. ## Table of contents @@ -135,7 +135,7 @@ The Swagger UI is also available at: [https://huggingface.github.io/text-generat ### Using a private or gated model -You have the option to utilize the `HUGGING_FACE_HUB_TOKEN` environment variable for configuring the token employed by +You have the option to utilize the `HUGGING_FACE_HUB_TOKEN` environment variable for configuring the token employed by `text-generation-inference`. This allows you to gain access to protected resources. For example, if you want to serve the gated Llama V2 model variants: @@ -146,7 +146,7 @@ For example, if you want to serve the gated Llama V2 model variants: or with Docker: -```shell +```shell model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= @@ -195,7 +195,7 @@ Python 3.9, e.g. using `conda`: ```shell curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -conda create -n text-generation-inference python=3.9 +conda create -n text-generation-inference python=3.9 conda activate text-generation-inference ``` @@ -221,7 +221,7 @@ Then run: ```shell BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork with CUDA kernels -make run-falcon-7b-instruct +make run-falcon-7b-instruct ``` **Note:** on some machines, you may also need the OpenSSL libraries and gcc. On Linux machines, run: @@ -232,7 +232,7 @@ sudo apt-get install libssl-dev gcc -y ### CUDA Kernels -The custom CUDA kernels are only tested on NVIDIA A100s. If you have any installation or runtime issues, you can remove +The custom CUDA kernels are only tested on NVIDIA A100s. If you have any installation or runtime issues, you can remove the kernels by using the `DISABLE_CUSTOM_KERNELS=True` environment variable. Be aware that the official Docker image has them enabled by default. @@ -242,7 +242,7 @@ Be aware that the official Docker image has them enabled by default. ### Run ```shell -make run-falcon-7b-instruct +make run-falcon-7b-instruct ``` ### Quantization @@ -273,3 +273,9 @@ make rust-tests # integration tests make integration-tests ``` + + +## Other supported hardware + +TGI is also supported on the following AI hardware accelerators: +- *Habana first-gen Gaudi and Gaudi2:* checkout [here](https://github.com/huggingface/optimum-habana/tree/main/text-generation-inference) how to serve models with TGI on Gaudi and Gaudi2 with [Optimum Habana](https://huggingface.co/docs/optimum/habana/index) From f848decee615ee10b78510b62036021a075dbf7b Mon Sep 17 00:00:00 2001 From: regisss <15324346+regisss@users.noreply.github.com> Date: Fri, 28 Jul 2023 11:20:03 +0200 Subject: [PATCH 348/793] docs: Add hardware section to TOC in README (#721) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c0783606..2bbb6583 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ to power LLMs api-inference widgets. - [Quantization](#quantization) - [Develop](#develop) - [Testing](#testing) +- [Other supported hardware](#other-supported-hardware) ## Features From afd04dc71e1ba83727975080460277d80f975f1e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 28 Jul 2023 15:36:38 +0200 Subject: [PATCH 349/793] feat(server): update vllm version (#723) --- router/src/main.rs | 6 +++- server/Makefile-vllm | 2 +- server/text_generation_server/utils/layers.py | 35 ++++++++----------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/router/src/main.rs b/router/src/main.rs index 059f8692..484643cb 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -233,6 +233,10 @@ fn main() -> Result<(), RouterError> { "Inferred max batch total tokens: {max_supported_batch_total_tokens}" ); } + if max_total_tokens as u32 > max_supported_batch_total_tokens { + return Err(RouterError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_supported_batch_total_tokens}"))); + } + max_supported_batch_total_tokens } }; @@ -270,7 +274,7 @@ fn main() -> Result<(), RouterError> { ngrok_authtoken, ngrok_edge, ) - .await?; + .await?; Ok(()) }) } diff --git a/server/Makefile-vllm b/server/Makefile-vllm index af750733..9100fff4 100644 --- a/server/Makefile-vllm +++ b/server/Makefile-vllm @@ -1,4 +1,4 @@ -vllm_commit := d284b831c17f42a8ea63369a06138325f73c4cf9 +vllm_commit := 084ca75d4271f8f67be731bc58e0d41d8e0afd3a vllm: # Clone vllm diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 183cf2c1..7a45808e 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -219,36 +219,31 @@ def load(config, prefix: str, weights): ) def forward(self, input: torch.Tensor) -> torch.Tensor: - if not self.should_gather: - return super().forward(input) - world_size = self.process_group.size() - if len(input.shape) == 2 and isinstance(self.linear, FastLinear): + # Fast branch for single requests + if ( + self.should_gather + and len(input.shape) == 2 + and isinstance(self.linear, FastLinear) + and input.shape[0] == 1 + ): out_dim = self.linear.weight.shape[0] - if input.shape[0] == 1: - world_out = input.new_empty(1, out_dim * world_size) - local_out = input.new_empty(1, out_dim) - gather_input = local_out - else: - world_out = input.new_empty(out_dim * world_size, input.shape[0]) - gather_input = input.new_empty(out_dim, input.shape[0]) - local_out = gather_input.T + world_out = input.new_empty(1, out_dim * world_size) + local_out = input.new_empty(1, out_dim) torch.mm(input, self.linear.weight.T, out=local_out) torch.distributed.all_gather_into_tensor( - world_out, gather_input, group=self.process_group + world_out, local_out, group=self.process_group ) - - if input.shape[0] == 1: - return world_out - return world_out.T + return world_out output = super().forward(input) - world_output = [ - torch.empty_like(output) for _ in range(self.process_group.size()) - ] + if not self.should_gather: + return output + + world_output = [torch.empty_like(output) for _ in range(world_size)] torch.distributed.all_gather(world_output, output, group=self.process_group) world_output = torch.cat(world_output, dim=-1) return world_output From bde25e62b33b05113519e5dbf75abda06a03328e Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 28 Jul 2023 15:59:46 +0200 Subject: [PATCH 350/793] chore: update license to HFOIL (#725) --- LICENSE | 382 +++++++++++++++++++++++++++----------------------------- 1 file changed, 181 insertions(+), 201 deletions(-) diff --git a/LICENSE b/LICENSE index 7d0e8034..19a34fcf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,181 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2022 Hugging Face - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Hugging Face Optimized Inference License 1.0 (HFOILv1.0) + + +This License Agreement governs the use of the Software and its Modifications. It is a +binding agreement between the Licensor and You. + +This License Agreement shall be referred to as Hugging Face Optimized Inference License +1.0 or HFOILv1.0. We may publish revised versions of this License Agreement from time to +time. Each version will be given a distinguished number. + +By downloading, accessing, modifying, distributing or otherwise using the Software, You +consent to all of the terms and conditions below. So, if You do not agree with those, +please do not download, access, modify, distribute, or use the Software. + + +1. PERMISSIONS + +You may use, modify and distribute the Software pursuant to the following terms and +conditions: + +Copyright License. Subject to the terms and conditions of this License Agreement and where +and as applicable, each Contributor hereby grants You a perpetual, worldwide, +non-exclusive, royalty-free, copyright license to reproduce, prepare, publicly display, +publicly perform, sublicense under the terms herein, and distribute the Software and +Modifications of the Software. + +Patent License. Subject to the terms and conditions of this License Agreement and where +and as applicable, each Contributor hereby grants You a perpetual, worldwide, +non-exclusive, royalty-free patent license to make, have made, Use, offer to sell, sell, +import, and otherwise transfer the Software, where such license applies only to those +patent claims licensable by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) with the Software to +which such Contribution(s) was submitted. If You institute patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Software +or a Contribution incorporated within the Software constitutes direct or contributory +patent infringement, then any rights granted to You under this License Agreement for the +Software shall terminate as of the date such litigation is filed. + +No other rights. All rights not expressly granted herein are retained. + + +2. RESTRICTIONS + +You may not distribute the Software as a hosted or managed, and paid service, where the +service grants users access to any substantial set of the features or functionality of the +Software. If you wish to do so, You will need to be granted additional rights from the +Licensor which will be subject to a separate mutually agreed agreement. + +You may not sublicense the Software under any other terms than those listed in this +License. + + +3. OBLIGATIONS + +When You modify the Software, You agree to: - attach a notice stating the Modifications of +the Software You made; and - attach a notice stating that the Modifications of the +Software are released under this License Agreement. + +When You distribute the Software or Modifications of the Software, You agree to: - give +any recipients of the Software a copy of this License Agreement; - retain all Explanatory +Documentation; and if sharing the Modifications of the Software, add Explanatory +Documentation documenting the changes made to create the Modifications of the Software; - +retain all copyright, patent, trademark and attribution notices. + + +4. MISCELLANEOUS + +Termination. Licensor reserves the right to restrict Use of the Software in violation of +this License Agreement, upon which Your licenses will automatically terminate. + +Contributions. Unless You explicitly state otherwise, any Contribution intentionally +submitted for inclusion in the Software by You to the Licensor shall be under the terms +and conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of any +separate license agreement you may have executed with Licensor regarding such +Contributions. + +Trademarks and related. Nothing in this License Agreement permits You (i) to make Use of +Licensors’ trademarks, trade names, or logos, (ii) otherwise suggest endorsement by +Licensor, or (iii) misrepresent the relationship between the parties; and any rights not +expressly granted herein are reserved by the Licensors. + +Output You generate. Licensor claims no rights in the Output. You agree not to contravene +any provision as stated in the License Agreement with your Use of the Output. + +Disclaimer of Warranty. Except as expressly provided otherwise herein, and to the fullest +extent permitted by law, Licensor provides the Software (and each Contributor provides its +Contributions) AS IS, and Licensor disclaims all warranties or guarantees of any kind, +express or implied, whether arising under any law or from any usage in trade, or otherwise +including but not limited to the implied warranties of merchantability, non-infringement, +quiet enjoyment, fitness for a particular purpose, or otherwise. You are solely +responsible for determining the appropriateness of the Software and Modifications of the +Software for your purposes (including your use or distribution of the Software and +Modifications of the Software), and assume any risks associated with Your exercise of +permissions under this License Agreement. + +Limitation of Liability. In no event and under no legal theory, whether in tort (including +negligence), contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to +You for damages, including any direct, indirect, special, incidental, or consequential +damages of any character arising as a result of this License Agreement or out of the Use +or inability to Use the Software (including but not limited to damages for loss of +goodwill, work stoppage, computer failure or malfunction, model failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has been advised +of the possibility of such damages. + +Accepting Warranty or Additional Liability. While sharing the Software or Modifications of +the Software thereof, You may choose to offer and charge a fee for, acceptance of support, +warranty, indemnity, or other liability obligations and/or rights consistent with this +License Agreement. However, in accepting such obligations, You may act only on Your own +behalf and on Your sole responsibility, not on behalf of Licensor or any other +Contributor, and you hereby agree to indemnify, defend, and hold Licensor and each other +Contributor (and their successors or assigns) harmless for any liability incurred by, or +claims asserted against, such Licensor or Contributor (and their successors or assigns) by +reason of your accepting any such warranty or additional liability. + +Severability. This License Agreement is a license of copyright and patent rights and an +agreement in contract between You and the Licensor. If any provision of this License +Agreement is held to be invalid, illegal or unenforceable, the remaining provisions shall +be unaffected thereby and remain valid as if such provision had not been set forth herein. + + +5. DEFINITIONS + +“Contribution” refers to any work of authorship, including the original version of the +Software and any Modifications of the Software that is intentionally submitted to Licensor +for inclusion in the Software by the copyright owner or by an individual or entity +authorized to submit on behalf of the copyright owner. For the purposes of this +definition, “submitted” means any form of electronic, verbal, or written communication +sent to the Licensor or its representatives, including but not limited to communication on +electronic mailing lists, source code control systems, and issue tracking systems that are +managed by, or on behalf of, the Licensor for the purpose of discussing and improving the +Software, but excluding communication that is conspicuously marked or otherwise designated +in writing by the copyright owner as “Not a Contribution.” + +“Contributor” refers to Licensor and any individual or entity on behalf of whom a +Contribution has been received by Licensor and subsequently incorporated within the +Software. + +“Data” refers to a collection of information extracted from the dataset used with the +Model, including to train, pretrain, or otherwise evaluate the Model. The Data is not +licensed under this License Agreement. + +“Explanatory Documentation” refers to any documentation or related information including +but not limited to model cards or data cards dedicated to inform the public about the +characteristics of the Software. Explanatory documentation is not licensed under this +License. + +"License Agreement" refers to these terms and conditions. + +“Licensor” refers to the rights owners or entity authorized by the rights owners that are +granting the terms and conditions of this License Agreement. + +“Model” refers to machine-learning based assemblies (including checkpoints), consisting of +learnt weights and parameters (including optimizer states), corresponding to a model +architecture as embodied in Software source code. Source code is not licensed under this +License Agreement. + +“Modifications of the Software” refers to all changes to the Software, including without +limitation derivative works of the Software. + +“Output” refers to the results of operating the Software. + +“Share” refers to any transmission, reproduction, publication or other sharing of the +Software or Modifications of the Software to a third party, including providing the +Softwaire as a hosted service made available by electronic or other remote means, +including - but not limited to - API-based or web access. + +“Software” refers to the software and Model (or parts of either) that Licensor makes +available under this License Agreement. + +“Third Parties” refers to individuals or legal entities that are not under common control +with Licensor or You. + +“Use” refers to anything You or your representatives do with the Software, including but +not limited to generating any Output, fine tuning, updating, running, training, evaluating +and/or reparametrizing the Model. + +"You" (or "Your") refers to an individual or Legal Entity exercising permissions granted +by this License Agreement and/or making Use of the Software for whichever purpose and in +any field of Use. From 3ef5ffbc6400370ff2e1546550a6bad3ac61b079 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 28 Jul 2023 17:43:46 +0200 Subject: [PATCH 351/793] v1.0.0 (#727) --- Cargo.lock | 8 ++--- Cargo.toml | 2 +- README.md | 12 +++---- docs/openapi.json | 2 +- server/Makefile-vllm | 2 +- server/pyproject.toml | 2 +- server/text_generation_server/utils/layers.py | 35 +++++++++++-------- 7 files changed, 33 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8984ea6a..61c22a40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2893,7 +2893,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "0.9.4" +version = "1.0.0" dependencies = [ "average", "clap", @@ -2913,7 +2913,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "0.9.4" +version = "1.0.0" dependencies = [ "futures", "grpc-metadata", @@ -2929,7 +2929,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "0.9.4" +version = "1.0.0" dependencies = [ "clap", "ctrlc", @@ -2945,7 +2945,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "0.9.4" +version = "1.0.0" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index 3bfe9831..255d35d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.9.4" +version = "1.0.0" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index 2bbb6583..869cc668 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,14 @@ GitHub Repo stars - - License - Swagger API documentation - A Rust, Python and gRPC server for text generation inference. Used in production at [HuggingFace](https://huggingface.co) -to power LLMs api-inference widgets. +to power Hugging Chat, the Inference API and Inference Endpoint. + + ## Table of contents @@ -85,7 +83,7 @@ The easiest way of getting started is using the official Docker container: model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.9.4 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.0 --model-id $model ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. @@ -152,7 +150,7 @@ model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= -docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:0.9.3 --model-id $model +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.0 --model-id $model ``` ### A note on Shared Memory (shm) diff --git a/docs/openapi.json b/docs/openapi.json index 9a672382..0465a5ad 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "0.9.4" + "version": "1.0.0" }, "paths": { "/": { diff --git a/server/Makefile-vllm b/server/Makefile-vllm index 9100fff4..af750733 100644 --- a/server/Makefile-vllm +++ b/server/Makefile-vllm @@ -1,4 +1,4 @@ -vllm_commit := 084ca75d4271f8f67be731bc58e0d41d8e0afd3a +vllm_commit := d284b831c17f42a8ea63369a06138325f73c4cf9 vllm: # Clone vllm diff --git a/server/pyproject.toml b/server/pyproject.toml index 3ee3351c..fdfbdd70 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "0.9.4" +version = "1.0.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 7a45808e..183cf2c1 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -219,31 +219,36 @@ def load(config, prefix: str, weights): ) def forward(self, input: torch.Tensor) -> torch.Tensor: + if not self.should_gather: + return super().forward(input) + world_size = self.process_group.size() - # Fast branch for single requests - if ( - self.should_gather - and len(input.shape) == 2 - and isinstance(self.linear, FastLinear) - and input.shape[0] == 1 - ): + if len(input.shape) == 2 and isinstance(self.linear, FastLinear): out_dim = self.linear.weight.shape[0] - world_out = input.new_empty(1, out_dim * world_size) - local_out = input.new_empty(1, out_dim) + if input.shape[0] == 1: + world_out = input.new_empty(1, out_dim * world_size) + local_out = input.new_empty(1, out_dim) + gather_input = local_out + else: + world_out = input.new_empty(out_dim * world_size, input.shape[0]) + gather_input = input.new_empty(out_dim, input.shape[0]) + local_out = gather_input.T torch.mm(input, self.linear.weight.T, out=local_out) torch.distributed.all_gather_into_tensor( - world_out, local_out, group=self.process_group + world_out, gather_input, group=self.process_group ) - return world_out - output = super().forward(input) - if not self.should_gather: - return output + if input.shape[0] == 1: + return world_out + return world_out.T - world_output = [torch.empty_like(output) for _ in range(world_size)] + output = super().forward(input) + world_output = [ + torch.empty_like(output) for _ in range(self.process_group.size()) + ] torch.distributed.all_gather(world_output, output, group=self.process_group) world_output = torch.cat(world_output, dim=-1) return world_output From 92bb56b0c1038a35f73a6c96c506f6d1c3d7b043 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 31 Jul 2023 10:32:52 +0200 Subject: [PATCH 352/793] Local gptq support. (#738) # What does this PR do? Redoes #719 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .github/workflows/tests.yaml | 2 +- Dockerfile | 2 +- server/text_generation_server/utils/weights.py | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 7e5ba52c..311ee6b9 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -33,7 +33,7 @@ jobs: - name: Install Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.65.0 + toolchain: 1.71.0 override: true components: rustfmt, clippy - name: Install Protoc diff --git a/Dockerfile b/Dockerfile index 34109d02..679fb48a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Rust builder -FROM lukemathwalker/cargo-chef:latest-rust-1.70 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.71 AS chef WORKDIR /usr/src ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 0330402d..dcab6296 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -1,3 +1,4 @@ +import os from pathlib import Path from typing import List, Dict, Optional, Tuple from safetensors import safe_open, SafetensorError @@ -221,8 +222,12 @@ def _get_gptq_params(self) -> Tuple[int, int]: return bits, groupsize def _set_gptq_params(self, model_id): + filename = "quantize_config.json" try: - filename = hf_hub_download(model_id, filename="quantize_config.json") + if not os.path.exists(os.path.join(model_id, filename)): + filename = os.path.join(model_id, filename) + else: + filename = hf_hub_download(model_id, filename=filename) with open(filename, "r") as f: data = json.load(f) self.gptq_bits = data["bits"] From b9633c46d0b64e4c3d52e2e29746abdbfd339dbe Mon Sep 17 00:00:00 2001 From: Jae-Won Chung Date: Mon, 31 Jul 2023 08:35:14 -0400 Subject: [PATCH 353/793] Fix typing in `Model.generate_token` (#733) ## What does this PR do? This PR fixes a minor type annotation issue in the signature of `Model.generate_token`. All existing overrides of `Model.generate_token` return `Tuple[List[Generation], Optional[B]]`: https://github.com/huggingface/text-generation-inference/blob/3ef5ffbc6400370ff2e1546550a6bad3ac61b079/server/text_generation_server/models/causal_lm.py#L535-L537 https://github.com/huggingface/text-generation-inference/blob/3ef5ffbc6400370ff2e1546550a6bad3ac61b079/server/text_generation_server/models/flash_causal_lm.py#L802-L804 https://github.com/huggingface/text-generation-inference/blob/3ef5ffbc6400370ff2e1546550a6bad3ac61b079/server/text_generation_server/models/seq2seq_lm.py#L589-L591 I suspect that back in 017a2a8c when `GeneratedText` and `Generation` were separated, the function signature was not updated. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? CC @OlivierDehaene --- server/text_generation_server/models/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 9d74247c..806e9833 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -5,7 +5,7 @@ from typing import List, Tuple, Optional, TypeVar, Type from transformers import PreTrainedTokenizerBase, PretrainedConfig -from text_generation_server.models.types import Batch, GeneratedText +from text_generation_server.models.types import Batch, Generation from text_generation_server.pb.generate_pb2 import InfoResponse B = TypeVar("B", bound=Batch) @@ -52,7 +52,7 @@ def batch_type(self) -> Type[B]: raise NotImplementedError @abstractmethod - def generate_token(self, batch: B) -> Tuple[List[GeneratedText], Optional[B]]: + def generate_token(self, batch: B) -> Tuple[List[Generation], Optional[B]]: raise NotImplementedError def warmup(self, batch: B) -> Optional[int]: From 932bdd93ff559702cd51f07311c78cf389ec0542 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 31 Jul 2023 15:38:47 +0200 Subject: [PATCH 354/793] Adding Rope scaling. (#741) # What does this PR do? - Adds Rope NTK scaling. Done because https://github.com/huggingface/text-generation-inference/pull/529 was closed Took some code from https://github.com/huggingface/transformers/pull/24653 - `--rope-scaling` and `--rope-factor` are added separately. I considered having a single one and parsing something line ("linear:4.0" , or "dynamic") but decided against it because it would push more parsing+validation a bit everywhere (both in the launcher and the server). Fixes #512 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- launcher/src/main.rs | 61 +++++++++++++ .../custom_modeling/flash_llama_modeling.py | 2 +- .../custom_modeling/flash_neox_modeling.py | 2 +- .../custom_modeling/flash_rw_modeling.py | 4 +- server/text_generation_server/utils/layers.py | 86 ++++++++++++++++--- 5 files changed, 141 insertions(+), 14 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 2ad788a4..560ce181 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -60,6 +60,26 @@ impl std::fmt::Display for Dtype { } } +#[derive(Clone, Copy, Debug, ValueEnum)] +enum RopeScaling { + Linear, + Dynamic, +} + +impl std::fmt::Display for RopeScaling { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // To keep in track with `server`. + match self { + RopeScaling::Linear => { + write!(f, "linear") + } + RopeScaling::Dynamic => { + write!(f, "dynamic") + } + } + } +} + /// App Configuration #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] @@ -250,6 +270,26 @@ struct Args { #[clap(default_value = "1.0", long, env)] cuda_memory_fraction: f32, + /// Rope scaling will only be used for RoPE models + /// and allow rescaling the position rotary to accomodate for + /// larger prompts. + /// + /// Goes together with `rope_factor`. + /// + /// `--rope-factor 2.0` gives linear scaling with a factor of 2.0 + /// `--rope-scaling dynamic` gives dynamic scaling with a factor of 1.0 + /// `--rope-scaling linear` gives linear scaling with a factor of 1.0 (Nothing will be changed + /// basically) + /// + /// `--rope-scaling linear --rope-factor` fully describes the scaling you want + #[clap(long, env)] + rope_scaling: Option, + + /// Rope scaling will only be used for RoPE models + /// See `rope_scaling` + #[clap(long, env)] + rope_factor: Option, + /// Outputs the logs in JSON format (useful for telemetry) #[clap(long, env)] json_output: bool, @@ -305,6 +345,8 @@ fn shard_manager( watermark_gamma: Option, watermark_delta: Option, cuda_memory_fraction: f32, + rope_scaling: Option, + rope_factor: Option, otlp_endpoint: Option, status_sender: mpsc::Sender, shutdown: Arc, @@ -358,6 +400,12 @@ fn shard_manager( shard_args.push(revision) } + let rope = match (rope_scaling, rope_factor) { + (None, None) => None, + (Some(scaling), None) => Some((scaling, 1.0)), + (Some(scaling), Some(factor)) => Some((scaling, factor)), + (None, Some(factor)) => Some((RopeScaling::Linear, factor)), + }; // OpenTelemetry if let Some(otlp_endpoint) = otlp_endpoint { shard_args.push("--otlp-endpoint".to_string()); @@ -395,6 +443,15 @@ fn shard_manager( envs.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) }; + // Detect rope scaling + // Sending as env instead of CLI args to not bloat everything + // those only can be used by RoPE models, so passing information around + // for all models will complexify code unnecessarily + if let Some((scaling, factor)) = rope { + envs.push(("ROPE_SCALING".into(), scaling.to_string().into())); + envs.push(("ROPE_FACTOR".into(), factor.to_string().into())); + } + // If huggingface_hub_cache is some, pass it to the shard // Useful when running inside a docker container if let Some(huggingface_hub_cache) = huggingface_hub_cache { @@ -784,6 +841,8 @@ fn spawn_shards( let watermark_gamma = args.watermark_gamma; let watermark_delta = args.watermark_delta; let cuda_memory_fraction = args.cuda_memory_fraction; + let rope_scaling = args.rope_scaling; + let rope_factor = args.rope_factor; thread::spawn(move || { shard_manager( model_id, @@ -802,6 +861,8 @@ fn spawn_shards( watermark_gamma, watermark_delta, cuda_memory_fraction, + rope_scaling, + rope_factor, otlp_endpoint, status_sender, shutdown, diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index b6285856..2c22ea46 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -186,7 +186,7 @@ def __init__( self.head_size = self.hidden_size // self.num_heads self.rotary_emb = PositionRotaryEmbedding.load( - prefix=f"{prefix}.rotary_emb", weights=weights + config=config, prefix=f"{prefix}.rotary_emb", weights=weights ) self.softmax_scale = self.head_size**-0.5 diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index e7c8ced4..9dc374df 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -102,7 +102,7 @@ def __init__(self, config, prefix, weights): self.num_heads = self.num_heads // weights.process_group.size() self.rotary_emb = PositionRotaryEmbedding.load( - prefix=f"{prefix}.rotary_emb", weights=weights + config=config, prefix=f"{prefix}.rotary_emb", weights=weights ) self.softmax_scale = self.head_size ** (-0.5) diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 3570b283..14caa23d 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -133,7 +133,7 @@ def __init__( self.head_size = self.hidden_size // self.num_heads self.rotary_emb = PositionRotaryEmbedding.static( - dim=self.head_size, base=10000.0, device=weights.device + config=config, dim=self.head_size, base=10000.0, device=weights.device ) self.softmax_scale = self.head_size ** (-0.5) @@ -247,7 +247,7 @@ def __init__( self.head_size = hidden_size // num_heads self.rotary_emb = PositionRotaryEmbedding.static( - self.head_size, base=10000.0, device=weights.device + config=config, dim=self.head_size, base=10000.0, device=weights.device ) self.softmax_scale = self.head_size ** (-0.5) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 183cf2c1..fc92ebe6 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -381,33 +381,65 @@ def forward(self, hidden_states, residual=None): from flash_attn.layers.rotary import RotaryEmbedding import rotary_emb + def _create_inv_freq(dim, base, device): + inv_freq = 1.0 / ( + base + ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) + ) + return inv_freq + + def _get_rope_config(config): + if os.getenv("ROPE_SCALING", None) is not None: + rope_scaling = {"type": os.environ["ROPE_SCALING"], "factor": float(os.environ["ROPE_FACTOR"])} + return rope_scaling + return getattr(config, "rope_scaling", None) + class PositionRotaryEmbedding(nn.Module): - def __init__(self, inv_freq): + def __init__(self, inv_freq, scaling_factor): super().__init__() - self.inv_freq = inv_freq self._seq_len_cached = 0 self._cos_cached = None self._sin_cached = None self._cos_k_cached = None self._sin_k_cached = None + self.scaling_factor = scaling_factor + self.dynamic_args = None @classmethod - def static(cls, dim, base, device): - inv_freq = 1.0 / ( - base - ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) - ) - return cls(inv_freq) + def static(cls, config, dim, base, device): + inv_freq = _create_inv_freq(dim, base, device) + scaling_factor = None + rope_scaling = _get_rope_config(config) + if rope_scaling is not None: + scaling_factor = rope_scaling["factor"] + if rope_scaling["type"] == "linear": + pass + elif rope_scaling["type"] == "dynamic": + return DynamicPositionRotaryEmbedding(dim=dim, max_position_embeddings=config.max_position_embeddings, base=base, device=inv_freq.device, scaling_factor=scaling_factor) + else: + raise NotImplementedError(f"rope scaling type {rope_scaling['type']} is not implemented or invalid") + return cls(inv_freq, scaling_factor) @classmethod - def load(cls, prefix, weights): + def load(cls, config, prefix, weights): # XXX: Always load this in float32 ! dtype = weights.dtype weights.dtype = torch.float32 inv_freq = weights.get_tensor(f"{prefix}.inv_freq") weights.dtype = dtype - return cls(inv_freq) + + scaling_factor = None + rope_scaling = _get_rope_config(config) + if rope_scaling is not None: + scaling_factor = rope_scaling["factor"] + if rope_scaling["type"] == "linear": + pass + elif rope_scaling["type"] == "dynamic": + return DynamicPositionRotaryEmbedding(dim=2*inv_freq.shape[0], max_position_embeddings=config.max_position_embeddings, base=10000.0, device=inv_freq.device, scaling_factor=scaling_factor) + else: + raise NotImplementedError(f"rope scaling type {rope_scaling['type']} is not implemented or invalid") + return cls(inv_freq, scaling_factor) def _update_cos_sin_cache(self, dtype, device, seqlen): # Reset the tables if the sequence length has changed, @@ -419,8 +451,11 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): ): self._seq_len_cached = seqlen t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) + if self.scaling_factor is not None: + t /= self.scaling_factor # Don't do einsum, it converts fp32 to fp16 # freqs = torch.einsum("i,j->ij", t, self.inv_freq) + freqs = torch.outer(t, self.inv_freq.to(device=t.device)) self._cos_cached = torch.cos(freqs).to(dtype) self._sin_cached = torch.sin(freqs).to(dtype) @@ -446,5 +481,36 @@ def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): rotary_emb.apply_rotary(x1, x2, cos, sin, x1, x2, False) return x + class DynamicPositionRotaryEmbedding(PositionRotaryEmbedding): + def __init__(self, dim, max_position_embeddings, base, device, scaling_factor): + inv_freq = create_inv_freq(dim, base, device) + super().__init__(inv_freq, scaling_factor) + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + + def _update_cos_sin_cache(self, dtype, device, seqlen): + # Reset the tables if the sequence length has changed, + # or if we're on a new device (possibly due to tracing for instance) + if ( + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype + ): + if seqlen > self.max_position_embeddings: + newbase = self.base * ((self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)) ** (self.dim / (self.dim - 2)) + self.inv_freq = _create_inv_freq(self.dim, newbase, self.inv_freq.device) + self._seq_len_cached = seqlen + t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) + if self.scaling_factor is not None: + t /= self.scaling_factor + # Don't do einsum, it converts fp32 to fp16 + # freqs = torch.einsum("i,j->ij", t, self.inv_freq) + + freqs = torch.outer(t, self.inv_freq.to(device=t.device)) + self._cos_cached = torch.cos(freqs).to(dtype) + self._sin_cached = torch.sin(freqs).to(dtype) + + except ImportError: pass From 2a13f1a04682f43e48aea1f2378d1e32ee726256 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Mon, 31 Jul 2023 22:43:44 +0900 Subject: [PATCH 355/793] chore: fix typo in mpt_modeling.py (#737) # What does this PR do? Fixed typo. implemetation -> implementation ## Before submitting - [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../models/custom_modeling/mpt_modeling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/mpt_modeling.py b/server/text_generation_server/models/custom_modeling/mpt_modeling.py index e6057116..5ccf796d 100644 --- a/server/text_generation_server/models/custom_modeling/mpt_modeling.py +++ b/server/text_generation_server/models/custom_modeling/mpt_modeling.py @@ -297,7 +297,7 @@ def triton_flash_attn_fn( class MultiheadAttention(nn.Module): """Multi-head self attention. - Using torch or triton attention implemetation enables user to also use + Using torch or triton attention implementation enables user to also use additive bias. """ @@ -386,7 +386,7 @@ def forward( class MultiQueryAttention(nn.Module): """Multi-Query self attention. - Using torch or triton attention implemetation enables user to also use + Using torch or triton attention implementation enables user to also use additive bias. """ From 15fc64668f8d3dd407768286e5a0536aeb78c2e1 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 31 Jul 2023 17:51:26 +0200 Subject: [PATCH 356/793] fix(server): Failing quantize config after local read. (#743) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/weights.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index dcab6296..ef662ce1 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -224,7 +224,7 @@ def _get_gptq_params(self) -> Tuple[int, int]: def _set_gptq_params(self, model_id): filename = "quantize_config.json" try: - if not os.path.exists(os.path.join(model_id, filename)): + if os.path.exists(os.path.join(model_id, filename)): filename = os.path.join(model_id, filename) else: filename = hf_hub_download(model_id, filename=filename) From d3d8f1bd6b2427fbe5c9c17e3b06899b43038343 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 31 Jul 2023 18:57:29 +0200 Subject: [PATCH 357/793] Typo fix. (#746) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index fc92ebe6..d7b4c0cc 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -483,7 +483,7 @@ def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): class DynamicPositionRotaryEmbedding(PositionRotaryEmbedding): def __init__(self, dim, max_position_embeddings, base, device, scaling_factor): - inv_freq = create_inv_freq(dim, base, device) + inv_freq = _create_inv_freq(dim, base, device) super().__init__(inv_freq, scaling_factor) self.dim = dim self.max_position_embeddings = max_position_embeddings From 7766fee9b15b6f600ee73db7f8bd72f7c62f335e Mon Sep 17 00:00:00 2001 From: Florian Zimmermeister Date: Mon, 31 Jul 2023 18:58:46 +0200 Subject: [PATCH 358/793] fix typo for dynamic rotary (#745) @Narsil From bd3088748eeb6723b8fdcb26b51ab4b09bf851e7 Mon Sep 17 00:00:00 2001 From: zspo Date: Thu, 3 Aug 2023 02:04:46 +0800 Subject: [PATCH 359/793] add FastLinear import (#750) # What does this PR do? Fixes #749 ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. Co-authored-by: p_spozzhang --- .../models/custom_modeling/opt_modeling.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/text_generation_server/models/custom_modeling/opt_modeling.py b/server/text_generation_server/models/custom_modeling/opt_modeling.py index 5d1a4b0d..fe6f1e52 100644 --- a/server/text_generation_server/models/custom_modeling/opt_modeling.py +++ b/server/text_generation_server/models/custom_modeling/opt_modeling.py @@ -28,6 +28,7 @@ from transformers.modeling_utils import PreTrainedModel from transformers import OPTConfig from text_generation_server.utils.layers import ( + FastLinear, TensorParallelColumnLinear, TensorParallelEmbedding, TensorParallelRowLinear, From 8b0d608f1f0e5495d7ca53528f602892d81758af Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 2 Aug 2023 20:24:37 +0200 Subject: [PATCH 360/793] Automatically map deduplicated safetensors weights to their original values (#501) (#761) # What does this PR do? CI cehck for #501 This PR automatically points tensors that were removed due to deduplication to their still existing twin. In `server.text_generation_server.utils.convert.py#convert_file`, tensors that have a value equal to another tensor are removed from the list of weights. Their name, and the name of the still-existing "twin" are logged to the "metadata" dictionary. However, this dictionary was not yet used during loading. This requires explicit in-code remapping when loading the models (as mentioned in the docstring). This PR adds some simple code to check, during loading, if a weight is one of those removed weights. It then automatically retrieves the values of its still-existing "twin" instead. ## What does this fix? We currently cannot load `h2oai/h2ogpt-oig-oasst1-falcon-40b` with the unmodified server, since the `transformer.word_embeddings.weight` weight is equal to `lm_head.weight` and is automatically removed. The falcon code, however, still expects this weight to exist. I could have also added some extra checks to the model itself, though that would only be a workaround. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. Co-authored-by: Vincent Brouwers Co-authored-by: Vincent Brouwers From ac736fd89c1f119b770a3385d100a954d2499ff7 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 3 Aug 2023 17:22:45 +0200 Subject: [PATCH 361/793] feat(server): Add native support for PEFT Lora models (#762) - Will detect `peft` model by finding `adapter_config.json`. - This triggers a totally dedicated `download-weights` path - This path, loads the adapter config, finds the base model_id - It loads the base_model - Then peft_model - Then `merge_and_unload()` - Then `save_pretrained(.., safe_serialization=True) - Add back the config + tokenizer.merge_and_unload()` - Then `save_pretrained(.., safe_serialization=True) - Add back the config + tokenizer. - The chosen location is a **local folder with the name of the user chosen model id** PROs: - Easier than to expect user to merge manually - Barely any change outside of `download-weights` command. - This means everything will work in a single load. - Should enable out of the box SM + HFE CONs: - Creates a local merged model in unusual location, potentially not saved across docker reloads, or ovewriting some files if the PEFT itself was local and containing other files in addition to the lora Alternatives considered: - Add `local_files_only=True` every where (discard because of massive code change for not a good enough reason) - Return something to `launcher` about the new model-id (a cleaner location for this new model), but it would introduce new communication somewhere where we didn't need it before. - Using the HF cache folder and *stopping* the flow after `download-weights` and asking user to restart with the actual local model location Fix #482 # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- launcher/src/main.rs | 5 + server/poetry.lock | 2224 ++++++++--------- server/pyproject.toml | 1 + server/requirements.txt | 25 +- server/text_generation_server/cli.py | 8 + .../text_generation_server/models/flash_rw.py | 2 +- .../text_generation_server/utils/__init__.py | 2 + server/text_generation_server/utils/peft.py | 46 + 8 files changed, 1153 insertions(+), 1160 deletions(-) create mode 100644 server/text_generation_server/utils/peft.py diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 560ce181..146d83d6 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -716,6 +716,11 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L download_args.push(revision.to_string()) } + // Trust remote code for automatic peft fusion + if args.trust_remote_code { + download_args.push("--trust-remote-code".to_string()); + } + // Copy current process env let mut envs: Vec<(OsString, OsString)> = env::vars_os().collect(); diff --git a/server/poetry.lock b/server/poetry.lock index 16da22f5..ccbb7661 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,1074 +1,40 @@ -[[package]] -name = "accelerate" -version = "0.19.0" -description = "Accelerate" -category = "main" -optional = true -python-versions = ">=3.7.0" - -[package.dependencies] -numpy = ">=1.17" -packaging = ">=20.0" -psutil = "*" -pyyaml = "*" -torch = ">=1.6.0" - -[package.extras] -dev = ["black (>=23.1,<24.0)", "datasets", "deepspeed", "evaluate", "hf-doc-builder (>=0.3.0)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "rich", "ruff (>=0.0.241)", "scikit-learn", "scipy", "tqdm", "transformers", "urllib3 (<2.0.0)"] -quality = ["black (>=23.1,<24.0)", "hf-doc-builder (>=0.3.0)", "ruff (>=0.0.241)", "urllib3 (<2.0.0)"] -rich = ["rich"] -sagemaker = ["sagemaker"] -test_dev = ["datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] -test_prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] -test_trackers = ["comet-ml", "tensorboard", "wandb"] -testing = ["datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] - -[[package]] -name = "aiohttp" -version = "3.8.5" -description = "Async http client/server framework (asyncio)" -category = "main" -optional = true -python-versions = ">=3.6" - -[package.dependencies] -aiosignal = ">=1.1.2" -async-timeout = ">=4.0.0a3,<5.0" -attrs = ">=17.3.0" -charset-normalizer = ">=2.0,<4.0" -frozenlist = ">=1.1.1" -multidict = ">=4.5,<7.0" -yarl = ">=1.0,<2.0" - -[package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] - -[[package]] -name = "aiosignal" -version = "1.3.1" -description = "aiosignal: a list of registered asynchronous callbacks" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -frozenlist = ">=1.1.0" - -[[package]] -name = "async-timeout" -version = "4.0.2" -description = "Timeout context manager for asyncio programs" -category = "main" -optional = true -python-versions = ">=3.6" - -[[package]] -name = "attrs" -version = "23.1.0" -description = "Classes Without Boilerplate" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.extras] -cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] -tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] - -[[package]] -name = "backoff" -version = "2.2.1" -description = "Function decoration for backoff and retry" -category = "main" -optional = false -python-versions = ">=3.7,<4.0" - -[[package]] -name = "bitsandbytes" -version = "0.38.1" -description = "8-bit optimizers and matrix multiplication routines." -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "certifi" -version = "2023.5.7" -description = "Python package for providing Mozilla's CA Bundle." -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "charset-normalizer" -version = "3.1.0" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" -optional = false -python-versions = ">=3.7.0" - -[[package]] -name = "click" -version = "8.1.3" -description = "Composable command line interface toolkit" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" - -[[package]] -name = "datasets" -version = "2.14.0" -description = "HuggingFace community-driven open-source library of datasets" -category = "main" -optional = true -python-versions = ">=3.8.0" - -[package.dependencies] -aiohttp = "*" -dill = ">=0.3.0,<0.3.8" -fsspec = {version = ">=2021.11.1", extras = ["http"]} -huggingface-hub = ">=0.14.0,<1.0.0" -multiprocess = "*" -numpy = ">=1.17" -packaging = "*" -pandas = "*" -pyarrow = ">=8.0.0" -pyyaml = ">=5.1" -requests = ">=2.19.0" -tqdm = ">=4.62.1" -xxhash = "*" - -[package.extras] -apache-beam = ["apache-beam (>=2.26.0,<2.44.0)"] -audio = ["librosa", "soundfile (>=0.12.1)"] -benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "black (>=23.1,<24.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "rarfile (>=4.0)", "ruff (>=0.0.241)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] -docs = ["s3fs"] -jax = ["jax (>=0.2.8,!=0.3.2,<=0.3.25)", "jaxlib (>=0.1.65,<=0.3.25)"] -metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"] -quality = ["black (>=23.1,<24.0)", "pyyaml (>=5.3.1)", "ruff (>=0.0.241)"] -s3 = ["s3fs"] -tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos"] -tensorflow_gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] -torch = ["torch"] -vision = ["Pillow (>=6.2.1)"] - -[[package]] -name = "Deprecated" -version = "1.2.14" -description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -wrapt = ">=1.10,<2" - -[package.extras] -dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] - -[[package]] -name = "dill" -version = "0.3.7" -description = "serialize all of Python" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.extras] -graph = ["objgraph (>=1.7.2)"] - -[[package]] -name = "einops" -version = "0.6.1" -description = "A new flavour of deep learning operations" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "exceptiongroup" -version = "1.1.2" -description = "Backport of PEP 654 (exception groups)" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "filelock" -version = "3.12.2" -description = "A platform independent file lock." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "frozenlist" -version = "1.4.0" -description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" -optional = true -python-versions = ">=3.8" - -[[package]] -name = "fsspec" -version = "2023.6.0" -description = "File-system specification" -category = "main" -optional = false -python-versions = ">=3.8" - -[package.dependencies] -aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} -requests = {version = "*", optional = true, markers = "extra == \"http\""} - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -devel = ["pytest", "pytest-cov"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "googleapis-common-protos" -version = "1.59.1" -description = "Common protobufs used in Google APIs" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" - -[package.extras] -grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] - -[[package]] -name = "grpc-interceptor" -version = "0.15.2" -description = "Simplifies gRPC interceptors" -category = "main" -optional = false -python-versions = ">=3.7,<4.0" - -[package.dependencies] -grpcio = ">=1.49.1,<2.0.0" - -[package.extras] -testing = ["protobuf (>=4.21.9)"] - -[[package]] -name = "grpcio" -version = "1.56.0" -description = "HTTP/2-based RPC framework" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -protobuf = ["grpcio-tools (>=1.56.0)"] - -[[package]] -name = "grpcio-reflection" -version = "1.56.0" -description = "Standard Protobuf Reflection Service for gRPC" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -grpcio = ">=1.56.0" -protobuf = ">=4.21.6" - -[[package]] -name = "grpcio-status" -version = "1.56.0" -description = "Status proto mapping for gRPC" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.56.0" -protobuf = ">=4.21.6" - -[[package]] -name = "grpcio-tools" -version = "1.56.0" -description = "Protobuf code generator for gRPC" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -grpcio = ">=1.56.0" -protobuf = ">=4.21.6,<5.0dev" -setuptools = "*" - -[[package]] -name = "hf-transfer" -version = "0.1.3" -description = "" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "huggingface-hub" -version = "0.14.1" -description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -category = "main" -optional = false -python-versions = ">=3.7.0" - -[package.dependencies] -filelock = "*" -fsspec = "*" -packaging = ">=20.9" -pyyaml = ">=5.1" -requests = "*" -tqdm = ">=4.42.1" -typing-extensions = ">=3.7.4.3" - -[package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] -cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] -fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] -tensorflow = ["graphviz", "pydot", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "gradio", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"] -torch = ["torch"] -typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] - -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "Jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "loguru" -version = "0.6.0" -description = "Python logging made (stupidly) simple" -category = "main" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} -win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} - -[package.extras] -dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] - -[[package]] -name = "MarkupSafe" -version = "2.1.3" -description = "Safely add untrusted strings to HTML/XML markup." -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "mpmath" -version = "1.3.0" -description = "Python library for arbitrary-precision floating-point arithmetic" -category = "main" -optional = true -python-versions = "*" - -[package.extras] -develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] -docs = ["sphinx"] -gmpy = ["gmpy2 (>=2.1.0a4)"] -tests = ["pytest (>=4.6)"] - -[[package]] -name = "multidict" -version = "6.0.4" -description = "multidict implementation" -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "multiprocess" -version = "0.70.15" -description = "better multiprocessing and multithreading in Python" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -dill = ">=0.3.7" - -[[package]] -name = "networkx" -version = "3.1" -description = "Python package for creating and manipulating graphs and networks" -category = "main" -optional = true -python-versions = ">=3.8" - -[package.extras] -default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] -developer = ["mypy (>=1.1)", "pre-commit (>=3.2)"] -doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.13)", "sphinx (>=6.1)", "sphinx-gallery (>=0.12)", "texext (>=0.6.7)"] -extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] -test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] - -[[package]] -name = "numpy" -version = "1.25.0" -description = "Fundamental package for array computing in Python" -category = "main" -optional = false -python-versions = ">=3.9" - -[[package]] -name = "opentelemetry-api" -version = "1.15.0" -description = "OpenTelemetry Python API" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -deprecated = ">=1.2.6" -setuptools = ">=16.0" - -[[package]] -name = "opentelemetry-exporter-otlp" -version = "1.15.0" -description = "OpenTelemetry Collector Exporters" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -opentelemetry-exporter-otlp-proto-grpc = "1.15.0" -opentelemetry-exporter-otlp-proto-http = "1.15.0" - -[[package]] -name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.15.0" -description = "OpenTelemetry Collector Protobuf over gRPC Exporter" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} -googleapis-common-protos = ">=1.52,<2.0" -grpcio = ">=1.0.0,<2.0.0" -opentelemetry-api = ">=1.12,<2.0" -opentelemetry-proto = "1.15.0" -opentelemetry-sdk = ">=1.12,<2.0" - -[package.extras] -test = ["pytest-grpc"] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-http" -version = "1.15.0" -description = "OpenTelemetry Collector Protobuf over HTTP Exporter" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} -googleapis-common-protos = ">=1.52,<2.0" -opentelemetry-api = ">=1.12,<2.0" -opentelemetry-proto = "1.15.0" -opentelemetry-sdk = ">=1.12,<2.0" -requests = ">=2.7,<3.0" - -[package.extras] -test = ["responses (==0.22.0)"] - -[[package]] -name = "opentelemetry-instrumentation" -version = "0.36b0" -description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -opentelemetry-api = ">=1.4,<2.0" -setuptools = ">=16.0" -wrapt = ">=1.0.0,<2.0.0" - -[[package]] -name = "opentelemetry-instrumentation-grpc" -version = "0.36b0" -description = "OpenTelemetry gRPC instrumentation" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -opentelemetry-api = ">=1.12,<2.0" -opentelemetry-instrumentation = "0.36b0" -opentelemetry-sdk = ">=1.12,<2.0" -opentelemetry-semantic-conventions = "0.36b0" -wrapt = ">=1.0.0,<2.0.0" - -[package.extras] -instruments = ["grpcio (>=1.27,<2.0)"] -test = ["opentelemetry-instrumentation-grpc[instruments]", "opentelemetry-sdk (>=1.12,<2.0)", "opentelemetry-test-utils (==0.36b0)", "protobuf (>=3.13,<4.0)"] - -[[package]] -name = "opentelemetry-proto" -version = "1.15.0" -description = "OpenTelemetry Python Proto" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -protobuf = ">=3.19,<5.0" - -[[package]] -name = "opentelemetry-sdk" -version = "1.15.0" -description = "OpenTelemetry Python SDK" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -opentelemetry-api = "1.15.0" -opentelemetry-semantic-conventions = "0.36b0" -setuptools = ">=16.0" -typing-extensions = ">=3.7.4" - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.36b0" -description = "OpenTelemetry Semantic Conventions" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "packaging" -version = "23.1" -description = "Core utilities for Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "pandas" -version = "2.0.3" -description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" -optional = true -python-versions = ">=3.8" - -[package.dependencies] -numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, -] -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.1" - -[package.extras] -all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] -aws = ["s3fs (>=2021.08.0)"] -clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] -compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] -computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] -feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2021.07.0)"] -gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] -hdf5 = ["tables (>=3.6.1)"] -html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] -mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] -output_formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] -parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] -plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] -spss = ["pyreadstat (>=1.1.2)"] -sql-other = ["SQLAlchemy (>=1.4.16)"] -test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.6.3)"] - -[[package]] -name = "pluggy" -version = "1.2.0" -description = "plugin and hook calling mechanisms for python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "protobuf" -version = "4.23.3" -description = "" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "psutil" -version = "5.9.5" -description = "Cross-platform lib for process and system monitoring in Python." -category = "main" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] - -[[package]] -name = "pyarrow" -version = "12.0.1" -description = "Python library for Apache Arrow" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -numpy = ">=1.16.6" - -[[package]] -name = "pytest" -version = "7.4.0" -description = "pytest: simple powerful testing with Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = true -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pytz" -version = "2023.3" -description = "World timezone definitions, modern and historical" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "PyYAML" -version = "6.0" -description = "YAML parser and emitter for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "regex" -version = "2023.6.3" -description = "Alternative regular expression module, to replace re." -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "requests" -version = "2.31.0" -description = "Python HTTP for Humans." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "safetensors" -version = "0.3.1" -description = "Fast and Safe Tensor serialization" -category = "main" -optional = false -python-versions = "*" - -[package.extras] -all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] -dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] -jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] -numpy = ["numpy (>=1.21.6)"] -paddlepaddle = ["paddlepaddle (>=2.4.1)"] -quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] -tensorflow = ["tensorflow (>=2.11.0)"] -testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] -torch = ["torch (>=1.10)"] - -[[package]] -name = "sentencepiece" -version = "0.1.99" -description = "SentencePiece python wrapper" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "setuptools" -version = "68.0.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "sympy" -version = "1.12" -description = "Computer algebra system (CAS) in Python" -category = "main" -optional = true -python-versions = ">=3.8" - -[package.dependencies] -mpmath = ">=0.19" - -[[package]] -name = "texttable" -version = "1.6.7" -description = "module to create simple ASCII tables" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "tokenizers" -version = "0.13.3" -description = "Fast and Customizable Tokenizers" -category = "main" -optional = false -python-versions = "*" - -[package.extras] -dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] -testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "torch" -version = "2.0.1" -description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -category = "main" -optional = true -python-versions = ">=3.8.0" - -[package.dependencies] -filelock = "*" -jinja2 = "*" -networkx = "*" -sympy = "*" -typing-extensions = "*" - -[package.extras] -opt-einsum = ["opt-einsum (>=3.3)"] - -[[package]] -name = "tqdm" -version = "4.65.0" -description = "Fast, Extensible Progress Meter" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] -notebook = ["ipywidgets (>=6)"] -slack = ["slack-sdk"] -telegram = ["requests"] - -[[package]] -name = "transformers" -version = "4.29.2" -description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" -category = "main" -optional = false -python-versions = ">=3.7.0" - -[package.dependencies] -filelock = "*" -huggingface-hub = ">=0.14.1,<1.0" -numpy = ">=1.17" -packaging = ">=20.0" -pyyaml = ">=5.1" -regex = "!=2019.12.17" -requests = "*" -tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" -tqdm = ">=4.27" - -[package.extras] -accelerate = ["accelerate (>=0.19.0)"] -agents = ["Pillow", "accelerate (>=0.19.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] -all = ["Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] -audio = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] -codecarbon = ["codecarbon (==1.2.0)"] -deepspeed = ["accelerate (>=0.19.0)", "deepspeed (>=0.8.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.19.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.8.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.6.9)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "numba (<0.57.0)", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.19.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "numba (<0.57.0)", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -docs = ["Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] -docs_specific = ["hf-doc-builder"] -fairscale = ["fairscale (>0.3)"] -flax = ["flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "optax (>=0.0.8,<=0.1.4)"] -flax-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] -ftfy = ["ftfy"] -integrations = ["optuna", "ray[tune]", "sigopt"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] -modelcreation = ["cookiecutter (==1.7.3)"] -natten = ["natten (>=0.14.6)"] -onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] -onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] -optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "black (>=23.1,<24.0)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (>=0.0.241,<=0.0.259)", "urllib3 (<2.0.0)"] -ray = ["ray[tune]"] -retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] -sagemaker = ["sagemaker (>=2.31.0)"] -sentencepiece = ["protobuf (<=3.20.2)", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["fastapi", "pydantic", "starlette", "uvicorn"] -sigopt = ["sigopt"] -sklearn = ["scikit-learn"] -speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] -tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] -tf-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] -timm = ["timm"] -tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -torch = ["accelerate (>=0.19.0)", "torch (>=1.9,!=1.12.0)"] -torch-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torch-vision = ["Pillow", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf (<=3.20.2)", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] -video = ["av (==9.2.0)", "decord (==0.6.0)"] -vision = ["Pillow"] - -[[package]] -name = "typer" -version = "0.6.1" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -click = ">=7.1.1,<9.0.0" - -[package.extras] -all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] -dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] -doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] -test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] - -[[package]] -name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "tzdata" -version = "2023.3" -description = "Provider of IANA time zone data" -category = "main" -optional = true -python-versions = ">=2" - -[[package]] -name = "urllib3" -version = "2.0.3" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] -name = "win32-setctime" -version = "1.1.0" -description = "A small Python utility to set file creation time on Windows" -category = "main" +name = "accelerate" +version = "0.19.0" +description = "Accelerate" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7.0" +files = [ + {file = "accelerate-0.19.0-py3-none-any.whl", hash = "sha256:2866b0bf9fff08f51e6384c95fa96725838b70f1988d1cce42e56b820d8a91dd"}, + {file = "accelerate-0.19.0.tar.gz", hash = "sha256:84920226b9e642e453ef37593ee55b956b08d8200dea4087c546c34e26157e76"}, +] -[package.extras] -dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] +[package.dependencies] +numpy = ">=1.17" +packaging = ">=20.0" +psutil = "*" +pyyaml = "*" +torch = ">=1.6.0" -[[package]] -name = "wrapt" -version = "1.15.0" -description = "Module for decorators, wrappers and monkey patching." -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +[package.extras] +dev = ["black (>=23.1,<24.0)", "datasets", "deepspeed", "evaluate", "hf-doc-builder (>=0.3.0)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "rich", "ruff (>=0.0.241)", "scikit-learn", "scipy", "tqdm", "transformers", "urllib3 (<2.0.0)"] +quality = ["black (>=23.1,<24.0)", "hf-doc-builder (>=0.3.0)", "ruff (>=0.0.241)", "urllib3 (<2.0.0)"] +rich = ["rich"] +sagemaker = ["sagemaker"] +test-dev = ["datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] +test-prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] +test-trackers = ["comet-ml", "tensorboard", "wandb"] +testing = ["datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] [[package]] -name = "xxhash" -version = "3.2.0" -description = "Python binding for xxHash" -category = "main" +name = "aiohttp" +version = "3.8.5" +description = "Async http client/server framework (asyncio)" optional = true python-versions = ">=3.6" - -[[package]] -name = "yarl" -version = "1.9.2" -description = "Yet another URL library" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" - -[extras] -accelerate = ["accelerate"] -bnb = ["bitsandbytes"] -quantize = ["texttable", "datasets", "accelerate"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.9" -content-hash = "4a717b962be975c23acbfed8946cf2e2d55497c8ffc192debd09d440127f5cc3" - -[metadata.files] -accelerate = [ - {file = "accelerate-0.19.0-py3-none-any.whl", hash = "sha256:2866b0bf9fff08f51e6384c95fa96725838b70f1988d1cce42e56b820d8a91dd"}, - {file = "accelerate-0.19.0.tar.gz", hash = "sha256:84920226b9e642e453ef37593ee55b956b08d8200dea4087c546c34e26157e76"}, -] -aiohttp = [ +files = [ {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8"}, {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84"}, {file = "aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a"}, @@ -1157,31 +123,102 @@ aiohttp = [ {file = "aiohttp-3.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c"}, {file = "aiohttp-3.8.5.tar.gz", hash = "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc"}, ] -aiosignal = [ + +[package.dependencies] +aiosignal = ">=1.1.2" +async-timeout = ">=4.0.0a3,<5.0" +attrs = ">=17.3.0" +charset-normalizer = ">=2.0,<4.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "cchardet"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = true +python-versions = ">=3.7" +files = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, ] -async-timeout = [ + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +optional = true +python-versions = ">=3.6" +files = [ {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, ] -attrs = [ + +[[package]] +name = "attrs" +version = "23.1.0" +description = "Classes Without Boilerplate" +optional = true +python-versions = ">=3.7" +files = [ {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, ] -backoff = [ + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] -bitsandbytes = [ + +[[package]] +name = "bitsandbytes" +version = "0.38.1" +description = "8-bit optimizers and matrix multiplication routines." +optional = true +python-versions = "*" +files = [ {file = "bitsandbytes-0.38.1-py3-none-any.whl", hash = "sha256:5f532e7b1353eb7049ae831da2eb62ed8a1e0444116bd51b9e088a6e0bc7a34a"}, {file = "bitsandbytes-0.38.1.tar.gz", hash = "sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce"}, ] -certifi = [ + +[[package]] +name = "certifi" +version = "2023.5.7" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, ] -charset-normalizer = [ + +[[package]] +name = "charset-normalizer" +version = "3.1.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, @@ -1258,39 +295,152 @@ charset-normalizer = [ {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, ] -click = [ + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] -colorama = [ + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -datasets = [ + +[[package]] +name = "datasets" +version = "2.14.0" +description = "HuggingFace community-driven open-source library of datasets" +optional = true +python-versions = ">=3.8.0" +files = [ {file = "datasets-2.14.0-py3-none-any.whl", hash = "sha256:93081cc3d9d0ce860c81f950a3ba23d24704da2eacbe2722092ef4f6ae0ada96"}, {file = "datasets-2.14.0.tar.gz", hash = "sha256:1bb3d1c992a593949a8d3e445b358ac1db4ead00e6619ea2e5e7b6dfc222dde1"}, ] -Deprecated = [ + +[package.dependencies] +aiohttp = "*" +dill = ">=0.3.0,<0.3.8" +fsspec = {version = ">=2021.11.1", extras = ["http"]} +huggingface-hub = ">=0.14.0,<1.0.0" +multiprocess = "*" +numpy = ">=1.17" +packaging = "*" +pandas = "*" +pyarrow = ">=8.0.0" +pyyaml = ">=5.1" +requests = ">=2.19.0" +tqdm = ">=4.62.1" +xxhash = "*" + +[package.extras] +apache-beam = ["apache-beam (>=2.26.0,<2.44.0)"] +audio = ["librosa", "soundfile (>=0.12.1)"] +benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] +dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "black (>=23.1,<24.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "rarfile (>=4.0)", "ruff (>=0.0.241)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] +docs = ["s3fs"] +jax = ["jax (>=0.2.8,!=0.3.2,<=0.3.25)", "jaxlib (>=0.1.65,<=0.3.25)"] +metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"] +quality = ["black (>=23.1,<24.0)", "pyyaml (>=5.3.1)", "ruff (>=0.0.241)"] +s3 = ["s3fs"] +tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos"] +tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] +tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] +torch = ["torch"] +vision = ["Pillow (>=6.2.1)"] + +[[package]] +name = "Deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, ] -dill = [ + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dill" +version = "0.3.7" +description = "serialize all of Python" +optional = true +python-versions = ">=3.7" +files = [ {file = "dill-0.3.7-py3-none-any.whl", hash = "sha256:76b122c08ef4ce2eedcd4d1abd8e641114bfc6c2867f49f3c41facf65bf19f5e"}, {file = "dill-0.3.7.tar.gz", hash = "sha256:cc1c8b182eb3013e24bd475ff2e9295af86c1a38eb1aff128dac8962a9ce3c03"}, ] -einops = [ + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + +[[package]] +name = "einops" +version = "0.6.1" +description = "A new flavour of deep learning operations" +optional = false +python-versions = ">=3.7" +files = [ {file = "einops-0.6.1-py3-none-any.whl", hash = "sha256:99149e46cc808956b174932fe563d920db4d6e5dadb8c6ecdaa7483b7ef7cfc3"}, {file = "einops-0.6.1.tar.gz", hash = "sha256:f95f8d00f4ded90dbc4b19b6f98b177332614b0357dde66997f3ae5d474dc8c8"}, ] -exceptiongroup = [ + +[[package]] +name = "exceptiongroup" +version = "1.1.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, ] -filelock = [ + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "filelock" +version = "3.12.2" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.7" +files = [ {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, ] -frozenlist = [ + +[package.extras] +docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "frozenlist" +version = "1.4.0" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = true +python-versions = ">=3.8" +files = [ {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab"}, {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559"}, {file = "frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c"}, @@ -1353,19 +503,87 @@ frozenlist = [ {file = "frozenlist-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167"}, {file = "frozenlist-1.4.0.tar.gz", hash = "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251"}, ] -fsspec = [ + +[[package]] +name = "fsspec" +version = "2023.6.0" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +files = [ {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"}, {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"}, ] -googleapis-common-protos = [ + +[package.dependencies] +aiohttp = {version = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1", optional = true, markers = "extra == \"http\""} +requests = {version = "*", optional = true, markers = "extra == \"http\""} + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +devel = ["pytest", "pytest-cov"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "googleapis-common-protos" +version = "1.59.1" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ {file = "googleapis-common-protos-1.59.1.tar.gz", hash = "sha256:b35d530fe825fb4227857bc47ad84c33c809ac96f312e13182bdeaa2abe1178a"}, {file = "googleapis_common_protos-1.59.1-py2.py3-none-any.whl", hash = "sha256:0cbedb6fb68f1c07e18eb4c48256320777707e7d0c55063ae56c15db3224a61e"}, ] -grpc-interceptor = [ + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "grpc-interceptor" +version = "0.15.2" +description = "Simplifies gRPC interceptors" +optional = false +python-versions = ">=3.7,<4.0" +files = [ {file = "grpc-interceptor-0.15.2.tar.gz", hash = "sha256:5c984110af4fb77d03472ec0468f9c77ddaf798e190410fb7b7f1e76c60c96a4"}, {file = "grpc_interceptor-0.15.2-py3-none-any.whl", hash = "sha256:596dac3cb709ffb6178a4873f5148e254c871c9069f0b11040189b257969490a"}, ] -grpcio = [ + +[package.dependencies] +grpcio = ">=1.49.1,<2.0.0" + +[package.extras] +testing = ["protobuf (>=4.21.9)"] + +[[package]] +name = "grpcio" +version = "1.56.0" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.7" +files = [ {file = "grpcio-1.56.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:fb34ace11419f1ae321c36ccaa18d81cd3f20728cd191250be42949d6845bb2d"}, {file = "grpcio-1.56.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:008767c0aed4899e657b50f2e0beacbabccab51359eba547f860e7c55f2be6ba"}, {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:17f47aeb9be0da5337f9ff33ebb8795899021e6c0741ee68bd69774a7804ca86"}, @@ -1412,15 +630,48 @@ grpcio = [ {file = "grpcio-1.56.0-cp39-cp39-win_amd64.whl", hash = "sha256:4241a1c2c76e748023c834995cd916570e7180ee478969c2d79a60ce007bc837"}, {file = "grpcio-1.56.0.tar.gz", hash = "sha256:4c08ee21b3d10315b8dc26f6c13917b20ed574cdbed2d2d80c53d5508fdcc0f2"}, ] -grpcio-reflection = [ + +[package.extras] +protobuf = ["grpcio-tools (>=1.56.0)"] + +[[package]] +name = "grpcio-reflection" +version = "1.56.0" +description = "Standard Protobuf Reflection Service for gRPC" +optional = false +python-versions = ">=3.6" +files = [ {file = "grpcio-reflection-1.56.0.tar.gz", hash = "sha256:d6bad11af658c78170b50825cf3c841f69662f946f8e921c9fa1ae2dc48f56c1"}, {file = "grpcio_reflection-1.56.0-py3-none-any.whl", hash = "sha256:23572e1cc5674465e54fb5898d7358288e6f6970e2aa21c9eee7ccd674b7388e"}, ] -grpcio-status = [ + +[package.dependencies] +grpcio = ">=1.56.0" +protobuf = ">=4.21.6" + +[[package]] +name = "grpcio-status" +version = "1.56.0" +description = "Status proto mapping for gRPC" +optional = false +python-versions = ">=3.6" +files = [ {file = "grpcio-status-1.56.0.tar.gz", hash = "sha256:9eca0b2dcda0782d3702df225918efd6d820f75f93cd5c51c7fb6a4ffbfea12c"}, {file = "grpcio_status-1.56.0-py3-none-any.whl", hash = "sha256:e5f101c96686e9d4e94a114567960fdb00052aa3c818b029745e3db37dc9c613"}, ] -grpcio-tools = [ + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.56.0" +protobuf = ">=4.21.6" + +[[package]] +name = "grpcio-tools" +version = "1.56.0" +description = "Protobuf code generator for gRPC" +optional = false +python-versions = ">=3.7" +files = [ {file = "grpcio-tools-1.56.0.tar.gz", hash = "sha256:39f5877cea514b3da9f2683dfb3ffb45ef47b05f4ff39c287d7d61c5057f48b8"}, {file = "grpcio_tools-1.56.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:cdbae7312e6d132d38ec2c1611b8cafb783e0416cc5c6deae04efde5f16fb190"}, {file = "grpcio_tools-1.56.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:5f5c416b88d76fbdb548cfee0486928748816b700ece6e591006e5b1dc67598f"}, @@ -1467,7 +718,19 @@ grpcio-tools = [ {file = "grpcio_tools-1.56.0-cp39-cp39-win32.whl", hash = "sha256:c62f07452dee3f1ed23aeaef821797c5e516f79535e97fe6a6b0a0ee8db1cc91"}, {file = "grpcio_tools-1.56.0-cp39-cp39-win_amd64.whl", hash = "sha256:7f063443870650e55012fdb3a58ff4ce5f4042b81dad6b749333ee8146157511"}, ] -hf-transfer = [ + +[package.dependencies] +grpcio = ">=1.56.0" +protobuf = ">=4.21.6,<5.0dev" +setuptools = "*" + +[[package]] +name = "hf-transfer" +version = "0.1.3" +description = "" +optional = false +python-versions = ">=3.7" +files = [ {file = "hf_transfer-0.1.3-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9"}, {file = "hf_transfer-0.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07"}, {file = "hf_transfer-0.1.3-cp310-none-win_amd64.whl", hash = "sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6"}, @@ -1488,27 +751,102 @@ hf-transfer = [ {file = "hf_transfer-0.1.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"}, {file = "hf_transfer-0.1.3.tar.gz", hash = "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c"}, ] -huggingface-hub = [ + +[[package]] +name = "huggingface-hub" +version = "0.14.1" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +optional = false +python-versions = ">=3.7.0" +files = [ {file = "huggingface_hub-0.14.1-py3-none-any.whl", hash = "sha256:9fc619170d800ff3793ad37c9757c255c8783051e1b5b00501205eb43ccc4f27"}, {file = "huggingface_hub-0.14.1.tar.gz", hash = "sha256:9ab899af8e10922eac65e290d60ab956882ab0bf643e3d990b1394b6b47b7fbc"}, ] -idna = [ + +[package.dependencies] +filelock = "*" +fsspec = "*" +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "gradio", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"] +torch = ["torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] -iniconfig = [ + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -Jinja2 = [ + +[[package]] +name = "Jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -loguru = [ + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "loguru" +version = "0.6.0" +description = "Python logging made (stupidly) simple" +optional = false +python-versions = ">=3.5" +files = [ {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, ] -MarkupSafe = [ + +[package.dependencies] +colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} +win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} + +[package.extras] +dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] + +[[package]] +name = "MarkupSafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, @@ -1560,11 +898,31 @@ MarkupSafe = [ {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, ] -mpmath = [ + +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +optional = false +python-versions = "*" +files = [ {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, ] -multidict = [ + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4)"] +tests = ["pytest (>=4.6)"] + +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +optional = true +python-versions = ">=3.7" +files = [ {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, @@ -1640,7 +998,14 @@ multidict = [ {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] -multiprocess = [ + +[[package]] +name = "multiprocess" +version = "0.70.15" +description = "better multiprocessing and multithreading in Python" +optional = true +python-versions = ">=3.7" +files = [ {file = "multiprocess-0.70.15-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:aa36c7ed16f508091438687fe9baa393a7a8e206731d321e443745e743a0d4e5"}, {file = "multiprocess-0.70.15-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:20e024018c46d0d1602024c613007ac948f9754659e3853b0aa705e83f6931d8"}, {file = "multiprocess-0.70.15-pp37-pypy37_pp73-manylinux_2_24_i686.whl", hash = "sha256:e576062981c91f0fe8a463c3d52506e598dfc51320a8dd8d78b987dfca91c5db"}, @@ -1658,11 +1023,35 @@ multiprocess = [ {file = "multiprocess-0.70.15-py39-none-any.whl", hash = "sha256:3e0953f5d52b4c76f1c973eaf8214554d146f2be5decb48e928e55c7a2d19338"}, {file = "multiprocess-0.70.15.tar.gz", hash = "sha256:f20eed3036c0ef477b07a4177cf7c1ba520d9a2677870a4f47fe026f0cd6787e"}, ] -networkx = [ + +[package.dependencies] +dill = ">=0.3.7" + +[[package]] +name = "networkx" +version = "3.1" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.8" +files = [ {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, ] -numpy = [ + +[package.extras] +default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] +developer = ["mypy (>=1.1)", "pre-commit (>=3.2)"] +doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.13)", "sphinx (>=6.1)", "sphinx-gallery (>=0.12)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] +test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] + +[[package]] +name = "numpy" +version = "1.25.0" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ {file = "numpy-1.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8aa130c3042052d656751df5e81f6d61edff3e289b5994edcf77f54118a8d9f4"}, {file = "numpy-1.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e3f2b96e3b63c978bc29daaa3700c028fe3f049ea3031b58aa33fe2a5809d24"}, {file = "numpy-1.25.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6b267f349a99d3908b56645eebf340cb58f01bd1e773b4eea1a905b3f0e4208"}, @@ -1689,47 +1078,179 @@ numpy = [ {file = "numpy-1.25.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cc3fda2b36482891db1060f00f881c77f9423eead4c3579629940a3e12095fe8"}, {file = "numpy-1.25.0.tar.gz", hash = "sha256:f1accae9a28dc3cda46a91de86acf69de0d1b5f4edd44a9b0c3ceb8036dfff19"}, ] -opentelemetry-api = [ + +[[package]] +name = "opentelemetry-api" +version = "1.15.0" +description = "OpenTelemetry Python API" +optional = false +python-versions = ">=3.7" +files = [ {file = "opentelemetry_api-1.15.0-py3-none-any.whl", hash = "sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0"}, {file = "opentelemetry_api-1.15.0.tar.gz", hash = "sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded"}, ] -opentelemetry-exporter-otlp = [ + +[package.dependencies] +deprecated = ">=1.2.6" +setuptools = ">=16.0" + +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.15.0" +description = "OpenTelemetry Collector Exporters" +optional = false +python-versions = ">=3.7" +files = [ {file = "opentelemetry_exporter_otlp-1.15.0-py3-none-any.whl", hash = "sha256:79f22748b6a54808a0448093dfa189c8490e729f67c134d4c992533d9393b33e"}, {file = "opentelemetry_exporter_otlp-1.15.0.tar.gz", hash = "sha256:4f7c49751d9720e2e726e13b0bb958ccade4e29122c305d92c033da432c8d2c5"}, ] -opentelemetry-exporter-otlp-proto-grpc = [ + +[package.dependencies] +opentelemetry-exporter-otlp-proto-grpc = "1.15.0" +opentelemetry-exporter-otlp-proto-http = "1.15.0" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.15.0" +description = "OpenTelemetry Collector Protobuf over gRPC Exporter" +optional = false +python-versions = ">=3.7" +files = [ {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0-py3-none-any.whl", hash = "sha256:c2a5492ba7d140109968135d641d06ce3c5bd73c50665f787526065d57d7fd1d"}, {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0.tar.gz", hash = "sha256:844f2a4bb9bcda34e4eb6fe36765e5031aacb36dc60ed88c90fc246942ea26e7"}, ] -opentelemetry-exporter-otlp-proto-http = [ + +[package.dependencies] +backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} +googleapis-common-protos = ">=1.52,<2.0" +grpcio = ">=1.0.0,<2.0.0" +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-proto = "1.15.0" +opentelemetry-sdk = ">=1.12,<2.0" + +[package.extras] +test = ["pytest-grpc"] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.15.0" +description = "OpenTelemetry Collector Protobuf over HTTP Exporter" +optional = false +python-versions = ">=3.7" +files = [ {file = "opentelemetry_exporter_otlp_proto_http-1.15.0-py3-none-any.whl", hash = "sha256:3ec2a02196c8a54bf5cbf7fe623a5238625638e83b6047a983bdf96e2bbb74c0"}, {file = "opentelemetry_exporter_otlp_proto_http-1.15.0.tar.gz", hash = "sha256:11b2c814249a49b22f6cca7a06b05701f561d577b747f3660dfd67b6eb9daf9c"}, ] -opentelemetry-instrumentation = [ + +[package.dependencies] +backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} +googleapis-common-protos = ">=1.52,<2.0" +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-proto = "1.15.0" +opentelemetry-sdk = ">=1.12,<2.0" +requests = ">=2.7,<3.0" + +[package.extras] +test = ["responses (==0.22.0)"] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.36b0" +description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +optional = false +python-versions = ">=3.7" +files = [ {file = "opentelemetry_instrumentation-0.36b0-py3-none-any.whl", hash = "sha256:83ba4ae7d5292b5b33e0f851cc5c76d8f91196b9b3527800fc13855c33383ac2"}, {file = "opentelemetry_instrumentation-0.36b0.tar.gz", hash = "sha256:e3ddac9b3b93408ef26c8ecbf38f717042977e16381bb4cd329a5b4cf16998cf"}, ] -opentelemetry-instrumentation-grpc = [ + +[package.dependencies] +opentelemetry-api = ">=1.4,<2.0" +setuptools = ">=16.0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-instrumentation-grpc" +version = "0.36b0" +description = "OpenTelemetry gRPC instrumentation" +optional = false +python-versions = ">=3.7" +files = [ {file = "opentelemetry_instrumentation_grpc-0.36b0-py3-none-any.whl", hash = "sha256:eaa246ed2083c97b13bab2555cb9d170e8433230a31476c4cab8a17fa03380a4"}, {file = "opentelemetry_instrumentation_grpc-0.36b0.tar.gz", hash = "sha256:dc89447c9eb6ea868970f6c13b4ffdac182cdd5a41dd215a0f5393ca6375be55"}, ] -opentelemetry-proto = [ + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.36b0" +opentelemetry-sdk = ">=1.12,<2.0" +opentelemetry-semantic-conventions = "0.36b0" +wrapt = ">=1.0.0,<2.0.0" + +[package.extras] +instruments = ["grpcio (>=1.27,<2.0)"] +test = ["opentelemetry-instrumentation-grpc[instruments]", "opentelemetry-sdk (>=1.12,<2.0)", "opentelemetry-test-utils (==0.36b0)", "protobuf (>=3.13,<4.0)"] + +[[package]] +name = "opentelemetry-proto" +version = "1.15.0" +description = "OpenTelemetry Python Proto" +optional = false +python-versions = ">=3.7" +files = [ {file = "opentelemetry_proto-1.15.0-py3-none-any.whl", hash = "sha256:044b6d044b4d10530f250856f933442b8753a17f94ae37c207607f733fb9a844"}, {file = "opentelemetry_proto-1.15.0.tar.gz", hash = "sha256:9c4008e40ac8cab359daac283fbe7002c5c29c77ea2674ad5626a249e64e0101"}, ] -opentelemetry-sdk = [ + +[package.dependencies] +protobuf = ">=3.19,<5.0" + +[[package]] +name = "opentelemetry-sdk" +version = "1.15.0" +description = "OpenTelemetry Python SDK" +optional = false +python-versions = ">=3.7" +files = [ {file = "opentelemetry_sdk-1.15.0-py3-none-any.whl", hash = "sha256:555c533e9837766119bbccc7a80458c9971d853a6f1da683a2246cd5e53b4645"}, {file = "opentelemetry_sdk-1.15.0.tar.gz", hash = "sha256:98dbffcfeebcbff12c0c974292d6ea603180a145904cf838b1fe4d5c99078425"}, ] -opentelemetry-semantic-conventions = [ + +[package.dependencies] +opentelemetry-api = "1.15.0" +opentelemetry-semantic-conventions = "0.36b0" +setuptools = ">=16.0" +typing-extensions = ">=3.7.4" + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.36b0" +description = "OpenTelemetry Semantic Conventions" +optional = false +python-versions = ">=3.7" +files = [ {file = "opentelemetry_semantic_conventions-0.36b0-py3-none-any.whl", hash = "sha256:adc05635e87b9d3e007c9f530eed487fc3ef2177d02f82f674f28ebf9aff8243"}, {file = "opentelemetry_semantic_conventions-0.36b0.tar.gz", hash = "sha256:829dc221795467d98b773c04096e29be038d77526dc8d6ac76f546fb6279bf01"}, ] -packaging = [ + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] -pandas = [ + +[[package]] +name = "pandas" +version = "2.0.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = true +python-versions = ">=3.8" +files = [ {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, @@ -1756,11 +1277,89 @@ pandas = [ {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, ] -pluggy = [ + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] +aws = ["s3fs (>=2021.08.0)"] +clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] +compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] +computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2021.07.0)"] +gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] +hdf5 = ["tables (>=3.6.1)"] +html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] +mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] +spss = ["pyreadstat (>=1.1.2)"] +sql-other = ["SQLAlchemy (>=1.4.16)"] +test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.6.3)"] + +[[package]] +name = "peft" +version = "0.4.0" +description = "Parameter-Efficient Fine-Tuning (PEFT)" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "peft-0.4.0-py3-none-any.whl", hash = "sha256:2cf992772a6d703814477e0bdcdadd68cb8ea388111ce2d793dd2ff0e438f357"}, + {file = "peft-0.4.0.tar.gz", hash = "sha256:e768fa22d6e9f32aa7e891f0d06f355960278ca4dc0cdd96bff71f6f06269207"}, +] + +[package.dependencies] +accelerate = "*" +numpy = ">=1.17" +packaging = ">=20.0" +psutil = "*" +pyyaml = "*" +safetensors = "*" +torch = ">=1.13.0" +transformers = "*" + +[package.extras] +dev = ["black (>=22.0,<23.0)", "hf-doc-builder", "ruff (>=0.0.241)", "urllib3 (<=2.0.0)"] +docs-specific = ["hf-doc-builder"] +quality = ["black (>=22.0,<23.0)", "ruff (>=0.0.241)", "urllib3 (<=2.0.0)"] +test = ["black (>=22.0,<23.0)", "datasets", "diffusers", "hf-doc-builder", "parameterized", "pytest", "pytest-cov", "pytest-xdist", "ruff (>=0.0.241)", "urllib3 (<=2.0.0)"] + +[[package]] +name = "pluggy" +version = "1.2.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.7" +files = [ {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, ] -protobuf = [ + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "protobuf" +version = "4.23.3" +description = "" +optional = false +python-versions = ">=3.7" +files = [ {file = "protobuf-4.23.3-cp310-abi3-win32.whl", hash = "sha256:514b6bbd54a41ca50c86dd5ad6488afe9505901b3557c5e0f7823a0cf67106fb"}, {file = "protobuf-4.23.3-cp310-abi3-win_amd64.whl", hash = "sha256:cc14358a8742c4e06b1bfe4be1afbdf5c9f6bd094dff3e14edb78a1513893ff5"}, {file = "protobuf-4.23.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2991f5e7690dab569f8f81702e6700e7364cc3b5e572725098215d3da5ccc6ac"}, @@ -1775,7 +1374,14 @@ protobuf = [ {file = "protobuf-4.23.3-py3-none-any.whl", hash = "sha256:447b9786ac8e50ae72cae7a2eec5c5df6a9dbf9aa6f908f1b8bda6032644ea62"}, {file = "protobuf-4.23.3.tar.gz", hash = "sha256:7a92beb30600332a52cdadbedb40d33fd7c8a0d7f549c440347bc606fb3fe34b"}, ] -psutil = [ + +[[package]] +name = "psutil" +version = "5.9.5" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, @@ -1791,7 +1397,17 @@ psutil = [ {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, ] -pyarrow = [ + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "pyarrow" +version = "12.0.1" +description = "Python library for Apache Arrow" +optional = true +python-versions = ">=3.7" +files = [ {file = "pyarrow-12.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df"}, {file = "pyarrow-12.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af"}, {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6"}, @@ -1818,19 +1434,64 @@ pyarrow = [ {file = "pyarrow-12.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7"}, {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, ] -pytest = [ + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pytest" +version = "7.4.0" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, ] -python-dateutil = [ + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] -pytz = [ + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2023.3" +description = "World timezone definitions, modern and historical" +optional = true +python-versions = "*" +files = [ {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, ] -PyYAML = [ + +[[package]] +name = "PyYAML" +version = "6.0" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -1872,7 +1533,14 @@ PyYAML = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] -regex = [ + +[[package]] +name = "regex" +version = "2023.6.3" +description = "Alternative regular expression module, to replace re." +optional = false +python-versions = ">=3.6" +files = [ {file = "regex-2023.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:824bf3ac11001849aec3fa1d69abcb67aac3e150a933963fb12bda5151fe1bfd"}, {file = "regex-2023.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05ed27acdf4465c95826962528f9e8d41dbf9b1aa8531a387dee6ed215a3e9ef"}, {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b49c764f88a79160fa64f9a7b425620e87c9f46095ef9c9920542ab2495c8bc"}, @@ -1962,11 +1630,35 @@ regex = [ {file = "regex-2023.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:bdff5eab10e59cf26bc479f565e25ed71a7d041d1ded04ccf9aee1d9f208487a"}, {file = "regex-2023.6.3.tar.gz", hash = "sha256:72d1a25bf36d2050ceb35b517afe13864865268dfb45910e2e17a84be6cbfeb0"}, ] -requests = [ + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, ] -safetensors = [ + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "safetensors" +version = "0.3.1" +description = "Fast and Safe Tensor serialization" +optional = false +python-versions = "*" +files = [ {file = "safetensors-0.3.1-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:2ae9b7dd268b4bae6624729dac86deb82104820e9786429b0583e5168db2f770"}, {file = "safetensors-0.3.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:08c85c1934682f1e2cd904d38433b53cd2a98245a7cc31f5689f9322a2320bbf"}, {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba625c7af9e1c5d0d91cb83d2fba97d29ea69d4db2015d9714d24c7f6d488e15"}, @@ -2008,7 +1700,25 @@ safetensors = [ {file = "safetensors-0.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:5f4f614b8e8161cd8a9ca19c765d176a82b122fa3d3387b77862145bfe9b4e93"}, {file = "safetensors-0.3.1.tar.gz", hash = "sha256:571da56ff8d0bec8ae54923b621cda98d36dcef10feb36fd492c4d0c2cd0e869"}, ] -sentencepiece = [ + +[package.extras] +all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] +dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] +numpy = ["numpy (>=1.21.6)"] +paddlepaddle = ["paddlepaddle (>=2.4.1)"] +quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] +tensorflow = ["tensorflow (>=2.11.0)"] +testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] +torch = ["torch (>=1.10)"] + +[[package]] +name = "sentencepiece" +version = "0.1.99" +description = "SentencePiece python wrapper" +optional = false +python-versions = "*" +files = [ {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0eb528e70571b7c02723e5804322469b82fe7ea418c96051d0286c0fa028db73"}, {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:77d7fafb2c4e4659cbdf303929503f37a26eabc4ff31d3a79bf1c5a1b338caa7"}, {file = "sentencepiece-0.1.99-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be9cf5b9e404c245aeb3d3723c737ba7a8f5d4ba262ef233a431fa6c45f732a0"}, @@ -2055,23 +1765,66 @@ sentencepiece = [ {file = "sentencepiece-0.1.99-cp39-cp39-win_amd64.whl", hash = "sha256:350e5c74d739973f1c9643edb80f7cc904dc948578bcb1d43c6f2b173e5d18dd"}, {file = "sentencepiece-0.1.99.tar.gz", hash = "sha256:189c48f5cb2949288f97ccdb97f0473098d9c3dcf5a3d99d4eabe719ec27297f"}, ] -setuptools = [ + +[[package]] +name = "setuptools" +version = "68.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.7" +files = [ {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, ] -six = [ + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -sympy = [ + +[[package]] +name = "sympy" +version = "1.12" +description = "Computer algebra system (CAS) in Python" +optional = false +python-versions = ">=3.8" +files = [ {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, {file = "sympy-1.12.tar.gz", hash = "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8"}, ] -texttable = [ + +[package.dependencies] +mpmath = ">=0.19" + +[[package]] +name = "texttable" +version = "1.6.7" +description = "module to create simple ASCII tables" +optional = true +python-versions = "*" +files = [ {file = "texttable-1.6.7-py2.py3-none-any.whl", hash = "sha256:b7b68139aa8a6339d2c320ca8b1dc42d13a7831a346b446cb9eb385f0c76310c"}, {file = "texttable-1.6.7.tar.gz", hash = "sha256:290348fb67f7746931bcdfd55ac7584ecd4e5b0846ab164333f0794b121760f2"}, ] -tokenizers = [ + +[[package]] +name = "tokenizers" +version = "0.13.3" +description = "Fast and Customizable Tokenizers" +optional = false +python-versions = "*" +files = [ {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, {file = "tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07"}, {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc"}, @@ -2113,11 +1866,30 @@ tokenizers = [ {file = "tokenizers-0.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783"}, {file = "tokenizers-0.13.3.tar.gz", hash = "sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e"}, ] -tomli = [ + +[package.extras] +dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] +docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -torch = [ + +[[package]] +name = "torch" +version = "2.0.1" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +optional = false +python-versions = ">=3.8.0" +files = [ {file = "torch-2.0.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:8ced00b3ba471856b993822508f77c98f48a458623596a4c43136158781e306a"}, {file = "torch-2.0.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:359bfaad94d1cda02ab775dc1cc386d585712329bb47b8741607ef6ef4950747"}, {file = "torch-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7c84e44d9002182edd859f3400deaa7410f5ec948a519cc7ef512c2f9b34d2c4"}, @@ -2139,35 +1911,185 @@ torch = [ {file = "torch-2.0.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:c62df99352bd6ee5a5a8d1832452110435d178b5164de450831a3a8cc14dc680"}, {file = "torch-2.0.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:671a2565e3f63b8fe8e42ae3e36ad249fe5e567435ea27b94edaa672a7d0c416"}, ] -tqdm = [ + +[package.dependencies] +filelock = "*" +jinja2 = "*" +networkx = "*" +sympy = "*" +typing-extensions = "*" + +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] + +[[package]] +name = "tqdm" +version = "4.65.0" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, ] -transformers = [ + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "transformers" +version = "4.29.2" +description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +optional = false +python-versions = ">=3.7.0" +files = [ {file = "transformers-4.29.2-py3-none-any.whl", hash = "sha256:0ef158b99bad6f4e6652a0d8655fbbe58b4cb788ce7040f320b5d29c7c810a75"}, {file = "transformers-4.29.2.tar.gz", hash = "sha256:ed9467661f459f1ce49461d83f18f3b36b6a37f306182dc2ba272935f3b93ebb"}, ] -typer = [ + +[package.dependencies] +filelock = "*" +huggingface-hub = ">=0.14.1,<1.0" +numpy = ">=1.17" +packaging = ">=20.0" +pyyaml = ">=5.1" +regex = "!=2019.12.17" +requests = "*" +tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" +tqdm = ">=4.27" + +[package.extras] +accelerate = ["accelerate (>=0.19.0)"] +agents = ["Pillow", "accelerate (>=0.19.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] +all = ["Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +audio = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] +codecarbon = ["codecarbon (==1.2.0)"] +deepspeed = ["accelerate (>=0.19.0)", "deepspeed (>=0.8.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.19.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.8.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.6.9)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "numba (<0.57.0)", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.19.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "numba (<0.57.0)", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +docs-specific = ["hf-doc-builder"] +fairscale = ["fairscale (>0.3)"] +flax = ["flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "optax (>=0.0.8,<=0.1.4)"] +flax-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] +ftfy = ["ftfy"] +integrations = ["optuna", "ray[tune]", "sigopt"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] +modelcreation = ["cookiecutter (==1.7.3)"] +natten = ["natten (>=0.14.6)"] +onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] +onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] +optuna = ["optuna"] +quality = ["GitPython (<3.1.19)", "black (>=23.1,<24.0)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (>=0.0.241,<=0.0.259)", "urllib3 (<2.0.0)"] +ray = ["ray[tune]"] +retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] +sagemaker = ["sagemaker (>=2.31.0)"] +sentencepiece = ["protobuf (<=3.20.2)", "sentencepiece (>=0.1.91,!=0.1.92)"] +serving = ["fastapi", "pydantic", "starlette", "uvicorn"] +sigopt = ["sigopt"] +sklearn = ["scikit-learn"] +speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] +tf-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] +timm = ["timm"] +tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] +torch = ["accelerate (>=0.19.0)", "torch (>=1.9,!=1.12.0)"] +torch-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +torch-vision = ["Pillow", "torchvision"] +torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf (<=3.20.2)", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] +video = ["av (==9.2.0)", "decord (==0.6.0)"] +vision = ["Pillow"] + +[[package]] +name = "typer" +version = "0.6.1" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.6" +files = [ {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"}, {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"}, ] -typing-extensions = [ + +[package.dependencies] +click = ">=7.1.1,<9.0.0" + +[package.extras] +all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"] + +[[package]] +name = "typing-extensions" +version = "4.7.1" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +files = [ {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] -tzdata = [ + +[[package]] +name = "tzdata" +version = "2023.3" +description = "Provider of IANA time zone data" +optional = true +python-versions = ">=2" +files = [ {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] -urllib3 = [ + +[[package]] +name = "urllib3" +version = "2.0.3" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.7" +files = [ {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, ] -win32-setctime = [ + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +optional = false +python-versions = ">=3.5" +files = [ {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, ] -wrapt = [ + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + +[[package]] +name = "wrapt" +version = "1.15.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, @@ -2244,7 +2166,14 @@ wrapt = [ {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] -xxhash = [ + +[[package]] +name = "xxhash" +version = "3.2.0" +description = "Python binding for xxHash" +optional = true +python-versions = ">=3.6" +files = [ {file = "xxhash-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:af44b9e59c4b2926a4e3c7f9d29949ff42fcea28637ff6b8182e654461932be8"}, {file = "xxhash-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1bdd57973e2b802ef32553d7bebf9402dac1557874dbe5c908b499ea917662cd"}, {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b7c9aa77bbce61a5e681bd39cb6a804338474dcc90abe3c543592aa5d6c9a9b"}, @@ -2344,7 +2273,14 @@ xxhash = [ {file = "xxhash-3.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:9d3f686e3d1c8900c5459eee02b60c7399e20ec5c6402364068a343c83a61d90"}, {file = "xxhash-3.2.0.tar.gz", hash = "sha256:1afd47af8955c5db730f630ad53ae798cf7fae0acb64cebb3cf94d35c47dd088"}, ] -yarl = [ + +[[package]] +name = "yarl" +version = "1.9.2" +description = "Yet another URL library" +optional = true +python-versions = ">=3.7" +files = [ {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, @@ -2420,3 +2356,17 @@ yarl = [ {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, ] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[extras] +accelerate = ["accelerate"] +bnb = ["bitsandbytes"] +quantize = ["accelerate", "datasets", "texttable"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "65fa01feaf4d0c939bfa7991803e4454d853f56652e4349151ea8dd9aa8feedf" diff --git a/server/pyproject.toml b/server/pyproject.toml index fdfbdd70..cc457b14 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -30,6 +30,7 @@ transformers = "4.29.2" einops = "^0.6.1" texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } +peft = "^0.4.0" [tool.poetry.extras] accelerate = ["accelerate"] diff --git a/server/requirements.txt b/server/requirements.txt index 98838b36..4ffb80a1 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,22 +1,13 @@ accelerate==0.19.0 ; python_version >= "3.9" and python_version < "4.0" -aiohttp==3.8.5 ; python_version >= "3.9" and python_version < "4.0" -aiosignal==1.3.1 ; python_version >= "3.9" and python_version < "4.0" -async-timeout==4.0.2 ; python_version >= "3.9" and python_version < "4.0" -attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0" backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" -bitsandbytes==0.38.1 ; python_version >= "3.9" and python_version < "4.0" certifi==2023.5.7 ; python_version >= "3.9" and python_version < "4.0" charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0" click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" -colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" or python_version >= "3.9" and python_version < "4.0" and platform_system == "Windows" -datasets==2.14.0 ; python_version >= "3.9" and python_version < "4.0" +colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows") deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0" -dill==0.3.7 ; python_version >= "3.9" and python_version < "4.0" einops==0.6.1 ; python_version >= "3.9" and python_version < "4.0" filelock==3.12.2 ; python_version >= "3.9" and python_version < "4.0" -frozenlist==1.4.0 ; python_version >= "3.9" and python_version < "4.0" fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "4.0" -fsspec[http]==2023.6.0 ; python_version >= "3.9" and python_version < "4.0" googleapis-common-protos==1.59.1 ; python_version >= "3.9" and python_version < "4.0" grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "4.0" grpcio-reflection==1.56.0 ; python_version >= "3.9" and python_version < "4.0" @@ -29,10 +20,8 @@ jinja2==3.1.2 ; python_version >= "3.9" and python_version < "4.0" loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" markupsafe==2.1.3 ; python_version >= "3.9" and python_version < "4.0" mpmath==1.3.0 ; python_version >= "3.9" and python_version < "4.0" -multidict==6.0.4 ; python_version >= "3.9" and python_version < "4.0" -multiprocess==0.70.15 ; python_version >= "3.9" and python_version < "4.0" networkx==3.1 ; python_version >= "3.9" and python_version < "4.0" -numpy==1.25.0 ; python_version < "4.0" and python_version >= "3.9" +numpy==1.25.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "4.0" @@ -43,30 +32,22 @@ opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" -pandas==2.0.3 ; python_version >= "3.9" and python_version < "4.0" +peft==0.4.0 ; python_version >= "3.9" and python_version < "4.0" protobuf==4.23.3 ; python_version >= "3.9" and python_version < "4.0" psutil==5.9.5 ; python_version >= "3.9" and python_version < "4.0" -pyarrow==12.0.1 ; python_version >= "3.9" and python_version < "4.0" -python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0" -pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0" pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" regex==2023.6.3 ; python_version >= "3.9" and python_version < "4.0" requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0" safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "4.0" setuptools==68.0.0 ; python_version >= "3.9" and python_version < "4.0" -six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" sympy==1.12 ; python_version >= "3.9" and python_version < "4.0" -texttable==1.6.7 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" torch==2.0.1 ; python_version >= "3.9" and python_version < "4.0" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" transformers==4.29.2 ; python_version >= "3.9" and python_version < "4.0" typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "4.0" -tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0" urllib3==2.0.3 ; python_version >= "3.9" and python_version < "4.0" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" -xxhash==3.2.0 ; python_version >= "3.9" and python_version < "4.0" -yarl==1.9.2 ; python_version >= "3.9" and python_version < "4.0" diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index e74c0331..eba807bc 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -6,6 +6,7 @@ from loguru import logger from typing import Optional from enum import Enum +from huggingface_hub import hf_hub_download app = typer.Typer() @@ -88,6 +89,7 @@ def download_weights( auto_convert: bool = True, logger_level: str = "INFO", json_output: bool = False, + trust_remote_code: bool = False, ): # Remove default handler logger.remove() @@ -118,6 +120,12 @@ def download_weights( ) is not None if not is_local_model: + try: + adapter_config_filename = hf_hub_download(model_id, revision=revision, filename="adapter_config.json") + utils.download_and_unload_peft(model_id, revision, trust_remote_code=trust_remote_code) + except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError): + pass + # Try to download weights from the hub try: filenames = utils.weight_hub_files(model_id, revision, extension) diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 7d655945..2fc7c53d 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -54,7 +54,7 @@ def __init__( device, dtype, process_group=self.process_group, - aliases={"transformer.word_embeddings.weight": ["lm_head.weight"]}, + aliases={"lm_head.weight": ["transformer.word_embeddings.weight"]}, ) config.quantize = quantize diff --git a/server/text_generation_server/utils/__init__.py b/server/text_generation_server/utils/__init__.py index befedcf0..08ba808d 100644 --- a/server/text_generation_server/utils/__init__.py +++ b/server/text_generation_server/utils/__init__.py @@ -1,6 +1,7 @@ from text_generation_server.utils.convert import convert_file, convert_files from text_generation_server.utils.dist import initialize_torch_distributed from text_generation_server.utils.weights import Weights +from text_generation_server.utils.peft import download_and_unload_peft from text_generation_server.utils.hub import ( weight_files, weight_hub_files, @@ -26,6 +27,7 @@ "weight_files", "weight_hub_files", "download_weights", + "download_and_unload_peft", "EntryNotFoundError", "HeterogeneousNextTokenChooser", "LocalEntryNotFoundError", diff --git a/server/text_generation_server/utils/peft.py b/server/text_generation_server/utils/peft.py new file mode 100644 index 00000000..be1f9444 --- /dev/null +++ b/server/text_generation_server/utils/peft.py @@ -0,0 +1,46 @@ +import os +import json +from loguru import logger +import torch + +from transformers import AutoTokenizer +from peft import AutoPeftModelForCausalLM, AutoPeftModelForSeq2SeqLM + +def download_and_unload_peft(model_id, revision, trust_remote_code): + torch_dtype = torch.float16 + + logger.info("Peft model detected.") + logger.info("Loading the model it might take a while without feedback") + try: + model = AutoPeftModelForCausalLM.from_pretrained( + model_id, + revision=revision, + torch_dtype=torch_dtype, + trust_remote_code=trust_remote_code, + low_cpu_mem_usage=True, + ) + except Exception: + model = AutoPeftModelForSeq2SeqLM.from_pretrained( + model_id, + revision=revision, + torch_dtype=torch_dtype, + trust_remote_code=trust_remote_code, + low_cpu_mem_usage=True, + ) + logger.info(f"Loaded.") + logger.info(f"Merging the lora weights.") + + base_model_id = model.peft_config["default"].base_model_name_or_path + + model = model.merge_and_unload() + + os.makedirs(model_id, exist_ok=True) + cache_dir = model_id + logger.info(f"Saving the newly created merged model to {cache_dir}") + tokenizer = AutoTokenizer.from_pretrained(base_model_id) + model.save_pretrained(cache_dir, safe_serialization=True) + model.config.save_pretrained(cache_dir) + tokenizer.save_pretrained(cache_dir) + + + From 6ec5288ab76ce91e84bd7389a0249b0254d1d1f8 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 3 Aug 2023 21:54:39 +0200 Subject: [PATCH 362/793] This should prevent the PyTorch overriding. (#767) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- Dockerfile | 3 +- server/poetry.lock | 861 +++++++++++++++++++++------------------- server/pyproject.toml | 7 + server/requirements.txt | 29 +- 4 files changed, 485 insertions(+), 415 deletions(-) diff --git a/Dockerfile b/Dockerfile index 679fb48a..45e304c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,8 +39,9 @@ RUN cargo build --release # Adapted from: https://github.com/pytorch/pytorch/blob/master/Dockerfile FROM debian:bullseye-slim as pytorch-install -ARG PYTORCH_VERSION=2.0.0 +ARG PYTORCH_VERSION=2.0.1 ARG PYTHON_VERSION=3.9 +# Keep in sync with `server/pyproject.toml ARG CUDA_VERSION=11.8 ARG MAMBA_VERSION=23.1.0-1 ARG CUDA_CHANNEL=nvidia diff --git a/server/poetry.lock b/server/poetry.lock index ccbb7661..70f09f32 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -203,113 +203,142 @@ files = [ [[package]] name = "certifi" -version = "2023.5.7" +version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, ] [[package]] name = "charset-normalizer" -version = "3.1.0" +version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, + {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, + {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, ] [[package]] name = "click" -version = "8.1.3" +version = "8.1.6" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, + {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, + {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "cmake" +version = "3.27.0" +description = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" +optional = false +python-versions = "*" +files = [ + {file = "cmake-3.27.0-py2.py3-none-macosx_10_10_universal2.macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:9ccab4cd93578d3c2df32e66b44b313b75a7484032645040431dc06a583ca4aa"}, + {file = "cmake-3.27.0-py2.py3-none-manylinux2010_i686.manylinux_2_12_i686.whl", hash = "sha256:199bfaefb752e82d8067aeee5d6a6e0414fe0d60e9a3fd08e95d537a97e0db16"}, + {file = "cmake-3.27.0-py2.py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:8745eff805f36762d3e8e904698b853cb4a9da8b4b07d1c12bcd1e1a6c4a1709"}, + {file = "cmake-3.27.0-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58a3f39d3d1bc897f05e531bfa676246a2b25d424c6a47e4b6bbc193fb560db7"}, + {file = "cmake-3.27.0-py2.py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b470ccd3f86cf19a63f6b221c9cceebcc58e32d3787d0d5f9f43d1d91a095090"}, + {file = "cmake-3.27.0-py2.py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:35a8d397ce883e93b5e6561e2803ce9470df52283862264093c1078530f98189"}, + {file = "cmake-3.27.0-py2.py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1f38d87b2c65763a0113f4a6c652e6f4b5adf90b384c1e1d69e4f8a3104a57d6"}, + {file = "cmake-3.27.0-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b9d5811954dcedcaa6c915c4a9bb6d64b55ac189e9cbc74be726307d9d084804"}, + {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:073e4f196d0888216e6794c08cd984ddabc108c0e4e66f48fbd7610d1e6d726d"}, + {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_i686.whl", hash = "sha256:e58e48643903e6fad76274337f9a4d3c575b8e21cd05c6214780b2c98bb0c706"}, + {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:9740ed9f61a3bd8708a41cadd5c057c04f38e5b89bd773e369df2e210a1c55a3"}, + {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_s390x.whl", hash = "sha256:1b3189171665f5c8d748ae7fe10a29fff1ebeedeaef57b16f1ea54b1ec0fe514"}, + {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:c4c968c188e7518deb463a14e64f3a19f242c9dcf7f24e1dbcc1419690cd54e0"}, + {file = "cmake-3.27.0-py2.py3-none-win32.whl", hash = "sha256:5561aca62b65aac844f3931e74cfeb696e4534de145e3307bf942e735736541e"}, + {file = "cmake-3.27.0-py2.py3-none-win_amd64.whl", hash = "sha256:48be3afe62c9513a49be007896a4058fafec512cb1f269a50126da30aacad97f"}, + {file = "cmake-3.27.0-py2.py3-none-win_arm64.whl", hash = "sha256:6f46a170b0c9c552d52da4346534570f818195dfc4f1d0c03264e24cc348fc60"}, + {file = "cmake-3.27.0.tar.gz", hash = "sha256:d03f0a76a2b96805044ad1178b92aeeb5f695caa6776a32522bb5c430a55b4e8"}, +] + +[package.extras] +test = ["coverage (>=4.2)", "flake8 (>=3.0.4)", "path.py (>=11.5.0)", "pytest (>=3.0.3)", "pytest-cov (>=2.4.0)", "pytest-runner (>=2.9)", "pytest-virtualenv (>=1.7.0)", "scikit-build (>=0.10.0)", "setuptools (>=28.0.0)", "virtualenv (>=15.0.3)", "wheel"] + [[package]] name = "colorama" version = "0.4.6" @@ -323,13 +352,13 @@ files = [ [[package]] name = "datasets" -version = "2.14.0" +version = "2.14.3" description = "HuggingFace community-driven open-source library of datasets" optional = true python-versions = ">=3.8.0" files = [ - {file = "datasets-2.14.0-py3-none-any.whl", hash = "sha256:93081cc3d9d0ce860c81f950a3ba23d24704da2eacbe2722092ef4f6ae0ada96"}, - {file = "datasets-2.14.0.tar.gz", hash = "sha256:1bb3d1c992a593949a8d3e445b358ac1db4ead00e6619ea2e5e7b6dfc222dde1"}, + {file = "datasets-2.14.3-py3-none-any.whl", hash = "sha256:8da5868e55e7c1f0bf3356913a14a9926fe5eca66663691886b95bc9c9b065f0"}, + {file = "datasets-2.14.3.tar.gz", hash = "sha256:5fb20465a990cf7946f961884ba46fec19cdc95b1cd2cad2a7b87c8d183b02e1"}, ] [package.dependencies] @@ -351,8 +380,8 @@ xxhash = "*" apache-beam = ["apache-beam (>=2.26.0,<2.44.0)"] audio = ["librosa", "soundfile (>=0.12.1)"] benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "black (>=23.1,<24.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "rarfile (>=4.0)", "ruff (>=0.0.241)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] -docs = ["s3fs"] +dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "black (>=23.1,<24.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "rarfile (>=4.0)", "ruff (>=0.0.241)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] +docs = ["s3fs", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos", "torch", "transformers"] jax = ["jax (>=0.2.8,!=0.3.2,<=0.3.25)", "jaxlib (>=0.1.65,<=0.3.25)"] metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"] quality = ["black (>=23.1,<24.0)", "pyyaml (>=5.3.1)", "ruff (>=0.0.241)"] @@ -364,7 +393,7 @@ torch = ["torch"] vision = ["Pillow (>=6.2.1)"] [[package]] -name = "Deprecated" +name = "deprecated" version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." optional = false @@ -545,13 +574,13 @@ tqdm = ["tqdm"] [[package]] name = "googleapis-common-protos" -version = "1.59.1" +version = "1.60.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.59.1.tar.gz", hash = "sha256:b35d530fe825fb4227857bc47ad84c33c809ac96f312e13182bdeaa2abe1178a"}, - {file = "googleapis_common_protos-1.59.1-py2.py3-none-any.whl", hash = "sha256:0cbedb6fb68f1c07e18eb4c48256320777707e7d0c55063ae56c15db3224a61e"}, + {file = "googleapis-common-protos-1.60.0.tar.gz", hash = "sha256:e73ebb404098db405ba95d1e1ae0aa91c3e15a71da031a2eeb6b2e23e7bc3708"}, + {file = "googleapis_common_protos-1.60.0-py2.py3-none-any.whl", hash = "sha256:69f9bbcc6acde92cab2db95ce30a70bd2b81d20b12eff3f1aabaffcbe8a93918"}, ] [package.dependencies] @@ -579,148 +608,148 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.56.0" +version = "1.56.2" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.56.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:fb34ace11419f1ae321c36ccaa18d81cd3f20728cd191250be42949d6845bb2d"}, - {file = "grpcio-1.56.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:008767c0aed4899e657b50f2e0beacbabccab51359eba547f860e7c55f2be6ba"}, - {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:17f47aeb9be0da5337f9ff33ebb8795899021e6c0741ee68bd69774a7804ca86"}, - {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43c50d810cc26349b093bf2cfe86756ab3e9aba3e7e681d360930c1268e1399a"}, - {file = "grpcio-1.56.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:187b8f71bad7d41eea15e0c9812aaa2b87adfb343895fffb704fb040ca731863"}, - {file = "grpcio-1.56.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:881575f240eb5db72ddca4dc5602898c29bc082e0d94599bf20588fb7d1ee6a0"}, - {file = "grpcio-1.56.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c243b158dd7585021d16c50498c4b2ec0a64a6119967440c5ff2d8c89e72330e"}, - {file = "grpcio-1.56.0-cp310-cp310-win32.whl", hash = "sha256:8b3b2c7b5feef90bc9a5fa1c7f97637e55ec3e76460c6d16c3013952ee479cd9"}, - {file = "grpcio-1.56.0-cp310-cp310-win_amd64.whl", hash = "sha256:03a80451530fd3b8b155e0c4480434f6be669daf7ecba56f73ef98f94222ee01"}, - {file = "grpcio-1.56.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:64bd3abcf9fb4a9fa4ede8d0d34686314a7075f62a1502217b227991d9ca4245"}, - {file = "grpcio-1.56.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:fdc3a895791af4addbb826808d4c9c35917c59bb5c430d729f44224e51c92d61"}, - {file = "grpcio-1.56.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:4f84a6fd4482e5fe73b297d4874b62a535bc75dc6aec8e9fe0dc88106cd40397"}, - {file = "grpcio-1.56.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14e70b4dda3183abea94c72d41d5930c333b21f8561c1904a372d80370592ef3"}, - {file = "grpcio-1.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b5ce42a5ebe3e04796246ba50357f1813c44a6efe17a37f8dc7a5c470377312"}, - {file = "grpcio-1.56.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8219f17baf069fe8e42bd8ca0b312b875595e43a70cabf397be4fda488e2f27d"}, - {file = "grpcio-1.56.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:defdd14b518e6e468466f799aaa69db0355bca8d3a5ea75fb912d28ba6f8af31"}, - {file = "grpcio-1.56.0-cp311-cp311-win32.whl", hash = "sha256:50f4daa698835accbbcc60e61e0bc29636c0156ddcafb3891c987e533a0031ba"}, - {file = "grpcio-1.56.0-cp311-cp311-win_amd64.whl", hash = "sha256:59c4e606993a47146fbeaf304b9e78c447f5b9ee5641cae013028c4cca784617"}, - {file = "grpcio-1.56.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:b1f4b6f25a87d80b28dd6d02e87d63fe1577fe6d04a60a17454e3f8077a38279"}, - {file = "grpcio-1.56.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:c2148170e01d464d41011a878088444c13413264418b557f0bdcd1bf1b674a0e"}, - {file = "grpcio-1.56.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:0409de787ebbf08c9d2bca2bcc7762c1efe72eada164af78b50567a8dfc7253c"}, - {file = "grpcio-1.56.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66f0369d27f4c105cd21059d635860bb2ea81bd593061c45fb64875103f40e4a"}, - {file = "grpcio-1.56.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38fdf5bd0a1c754ce6bf9311a3c2c7ebe56e88b8763593316b69e0e9a56af1de"}, - {file = "grpcio-1.56.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:79d4c5911d12a7aa671e5eb40cbb50a830396525014d2d6f254ea2ba180ce637"}, - {file = "grpcio-1.56.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5d2fc471668a7222e213f86ef76933b18cdda6a51ea1322034478df8c6519959"}, - {file = "grpcio-1.56.0-cp37-cp37m-win_amd64.whl", hash = "sha256:991224fd485e088d3cb5e34366053691a4848a6b7112b8f5625a411305c26691"}, - {file = "grpcio-1.56.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:c6f36621aabecbaff3e70c4d1d924c76c8e6a7ffec60c331893640a4af0a8037"}, - {file = "grpcio-1.56.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:1eadd6de258901929223f422ffed7f8b310c0323324caf59227f9899ea1b1674"}, - {file = "grpcio-1.56.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:72836b5a1d4f508ffbcfe35033d027859cc737972f9dddbe33fb75d687421e2e"}, - {file = "grpcio-1.56.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f92a99ab0c7772fb6859bf2e4f44ad30088d18f7c67b83205297bfb229e0d2cf"}, - {file = "grpcio-1.56.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa08affbf672d051cd3da62303901aeb7042a2c188c03b2c2a2d346fc5e81c14"}, - {file = "grpcio-1.56.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2db108b4c8e29c145e95b0226973a66d73ae3e3e7fae00329294af4e27f1c42"}, - {file = "grpcio-1.56.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8674fdbd28266d8efbcddacf4ec3643f76fe6376f73283fd63a8374c14b0ef7c"}, - {file = "grpcio-1.56.0-cp38-cp38-win32.whl", hash = "sha256:bd55f743e654fb050c665968d7ec2c33f03578a4bbb163cfce38024775ff54cc"}, - {file = "grpcio-1.56.0-cp38-cp38-win_amd64.whl", hash = "sha256:c63bc5ac6c7e646c296fed9139097ae0f0e63f36f0864d7ce431cce61fe0118a"}, - {file = "grpcio-1.56.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:c0bc9dda550785d23f4f025be614b7faa8d0293e10811f0f8536cf50435b7a30"}, - {file = "grpcio-1.56.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:d596408bab632ec7b947761e83ce6b3e7632e26b76d64c239ba66b554b7ee286"}, - {file = "grpcio-1.56.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:76b6e6e1ee9bda32e6e933efd61c512e9a9f377d7c580977f090d1a9c78cca44"}, - {file = "grpcio-1.56.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7beb84ebd0a3f732625124b73969d12b7350c5d9d64ddf81ae739bbc63d5b1ed"}, - {file = "grpcio-1.56.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83ec714bbbe9b9502177c842417fde39f7a267031e01fa3cd83f1ca49688f537"}, - {file = "grpcio-1.56.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4feee75565d1b5ab09cb3a5da672b84ca7f6dd80ee07a50f5537207a9af543a4"}, - {file = "grpcio-1.56.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b4638a796778329cc8e142e4f57c705adb286b3ba64e00b0fa91eeb919611be8"}, - {file = "grpcio-1.56.0-cp39-cp39-win32.whl", hash = "sha256:437af5a7673bca89c4bc0a993382200592d104dd7bf55eddcd141cef91f40bab"}, - {file = "grpcio-1.56.0-cp39-cp39-win_amd64.whl", hash = "sha256:4241a1c2c76e748023c834995cd916570e7180ee478969c2d79a60ce007bc837"}, - {file = "grpcio-1.56.0.tar.gz", hash = "sha256:4c08ee21b3d10315b8dc26f6c13917b20ed574cdbed2d2d80c53d5508fdcc0f2"}, + {file = "grpcio-1.56.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:bf0b9959e673505ee5869950642428046edb91f99942607c2ecf635f8a4b31c9"}, + {file = "grpcio-1.56.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:5144feb20fe76e73e60c7d73ec3bf54f320247d1ebe737d10672480371878b48"}, + {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:a72797549935c9e0b9bc1def1768c8b5a709538fa6ab0678e671aec47ebfd55e"}, + {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3f3237a57e42f79f1e560726576aedb3a7ef931f4e3accb84ebf6acc485d316"}, + {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:900bc0096c2ca2d53f2e5cebf98293a7c32f532c4aeb926345e9747452233950"}, + {file = "grpcio-1.56.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:97e0efaebbfd222bcaac2f1735c010c1d3b167112d9d237daebbeedaaccf3d1d"}, + {file = "grpcio-1.56.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0c85c5cbe8b30a32fa6d802588d55ffabf720e985abe9590c7c886919d875d4"}, + {file = "grpcio-1.56.2-cp310-cp310-win32.whl", hash = "sha256:06e84ad9ae7668a109e970c7411e7992751a116494cba7c4fb877656527f9a57"}, + {file = "grpcio-1.56.2-cp310-cp310-win_amd64.whl", hash = "sha256:10954662f77dc36c9a1fb5cc4a537f746580d6b5734803be1e587252682cda8d"}, + {file = "grpcio-1.56.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:c435f5ce1705de48e08fcbcfaf8aee660d199c90536e3e06f2016af7d6a938dd"}, + {file = "grpcio-1.56.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:6108e5933eb8c22cd3646e72d5b54772c29f57482fd4c41a0640aab99eb5071d"}, + {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8391cea5ce72f4a12368afd17799474015d5d3dc00c936a907eb7c7eaaea98a5"}, + {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:750de923b456ca8c0f1354d6befca45d1f3b3a789e76efc16741bd4132752d95"}, + {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fda2783c12f553cdca11c08e5af6eecbd717280dc8fbe28a110897af1c15a88c"}, + {file = "grpcio-1.56.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9e04d4e4cfafa7c5264e535b5d28e786f0571bea609c3f0aaab13e891e933e9c"}, + {file = "grpcio-1.56.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:89a49cc5ad08a38b6141af17e00d1dd482dc927c7605bc77af457b5a0fca807c"}, + {file = "grpcio-1.56.2-cp311-cp311-win32.whl", hash = "sha256:6a007a541dff984264981fbafeb052bfe361db63578948d857907df9488d8774"}, + {file = "grpcio-1.56.2-cp311-cp311-win_amd64.whl", hash = "sha256:af4063ef2b11b96d949dccbc5a987272f38d55c23c4c01841ea65a517906397f"}, + {file = "grpcio-1.56.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:a6ff459dac39541e6a2763a4439c4ca6bc9ecb4acc05a99b79246751f9894756"}, + {file = "grpcio-1.56.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:f20fd21f7538f8107451156dd1fe203300b79a9ddceba1ee0ac8132521a008ed"}, + {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:d1fbad1f9077372b6587ec589c1fc120b417b6c8ad72d3e3cc86bbbd0a3cee93"}, + {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee26e9dfb3996aff7c870f09dc7ad44a5f6732b8bdb5a5f9905737ac6fd4ef1"}, + {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4c60abd950d6de3e4f1ddbc318075654d275c29c846ab6a043d6ed2c52e4c8c"}, + {file = "grpcio-1.56.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1c31e52a04e62c8577a7bf772b3e7bed4df9c9e0dd90f92b6ffa07c16cab63c9"}, + {file = "grpcio-1.56.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:345356b307cce5d14355e8e055b4ca5f99bc857c33a3dc1ddbc544fca9cd0475"}, + {file = "grpcio-1.56.2-cp37-cp37m-win_amd64.whl", hash = "sha256:42e63904ee37ae46aa23de50dac8b145b3596f43598fa33fe1098ab2cbda6ff5"}, + {file = "grpcio-1.56.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:7c5ede2e2558f088c49a1ddda19080e4c23fb5d171de80a726b61b567e3766ed"}, + {file = "grpcio-1.56.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:33971197c47965cc1d97d78d842163c283e998223b151bab0499b951fd2c0b12"}, + {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:d39f5d4af48c138cb146763eda14eb7d8b3ccbbec9fe86fb724cd16e0e914c64"}, + {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ded637176addc1d3eef35331c39acc598bac550d213f0a1bedabfceaa2244c87"}, + {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c90da4b124647547a68cf2f197174ada30c7bb9523cb976665dfd26a9963d328"}, + {file = "grpcio-1.56.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3ccb621749a81dc7755243665a70ce45536ec413ef5818e013fe8dfbf5aa497b"}, + {file = "grpcio-1.56.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4eb37dd8dd1aa40d601212afa27ca5be255ba792e2e0b24d67b8af5e012cdb7d"}, + {file = "grpcio-1.56.2-cp38-cp38-win32.whl", hash = "sha256:ddb4a6061933bd9332b74eac0da25f17f32afa7145a33a0f9711ad74f924b1b8"}, + {file = "grpcio-1.56.2-cp38-cp38-win_amd64.whl", hash = "sha256:8940d6de7068af018dfa9a959a3510e9b7b543f4c405e88463a1cbaa3b2b379a"}, + {file = "grpcio-1.56.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:51173e8fa6d9a2d85c14426bdee5f5c4a0654fd5fddcc21fe9d09ab0f6eb8b35"}, + {file = "grpcio-1.56.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:373b48f210f43327a41e397391715cd11cfce9ded2fe76a5068f9bacf91cc226"}, + {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:42a3bbb2bc07aef72a7d97e71aabecaf3e4eb616d39e5211e2cfe3689de860ca"}, + {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5344be476ac37eb9c9ad09c22f4ea193c1316bf074f1daf85bddb1b31fda5116"}, + {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3fa3ab0fb200a2c66493828ed06ccd1a94b12eddbfb985e7fd3e5723ff156c6"}, + {file = "grpcio-1.56.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b975b85d1d5efc36cf8b237c5f3849b64d1ba33d6282f5e991f28751317504a1"}, + {file = "grpcio-1.56.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cbdf2c498e077282cd427cfd88bdce4668019791deef0be8155385ab2ba7837f"}, + {file = "grpcio-1.56.2-cp39-cp39-win32.whl", hash = "sha256:139f66656a762572ae718fa0d1f2dce47c05e9fbf7a16acd704c354405b97df9"}, + {file = "grpcio-1.56.2-cp39-cp39-win_amd64.whl", hash = "sha256:830215173ad45d670140ff99aac3b461f9be9a6b11bee1a17265aaaa746a641a"}, + {file = "grpcio-1.56.2.tar.gz", hash = "sha256:0ff789ae7d8ddd76d2ac02e7d13bfef6fc4928ac01e1dcaa182be51b6bcc0aaa"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.56.0)"] +protobuf = ["grpcio-tools (>=1.56.2)"] [[package]] name = "grpcio-reflection" -version = "1.56.0" +version = "1.56.2" description = "Standard Protobuf Reflection Service for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-reflection-1.56.0.tar.gz", hash = "sha256:d6bad11af658c78170b50825cf3c841f69662f946f8e921c9fa1ae2dc48f56c1"}, - {file = "grpcio_reflection-1.56.0-py3-none-any.whl", hash = "sha256:23572e1cc5674465e54fb5898d7358288e6f6970e2aa21c9eee7ccd674b7388e"}, + {file = "grpcio-reflection-1.56.2.tar.gz", hash = "sha256:74a81766af639ab8f1b7f59531dc814640f4a1bcf012dc06ce6be205b50a394c"}, + {file = "grpcio_reflection-1.56.2-py3-none-any.whl", hash = "sha256:004bcc3d4a3dcd89bf83253e4c08ca032fc7ca862f7532ea09ebdf08801fc193"}, ] [package.dependencies] -grpcio = ">=1.56.0" +grpcio = ">=1.56.2" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.56.0" +version = "1.56.2" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.56.0.tar.gz", hash = "sha256:9eca0b2dcda0782d3702df225918efd6d820f75f93cd5c51c7fb6a4ffbfea12c"}, - {file = "grpcio_status-1.56.0-py3-none-any.whl", hash = "sha256:e5f101c96686e9d4e94a114567960fdb00052aa3c818b029745e3db37dc9c613"}, + {file = "grpcio-status-1.56.2.tar.gz", hash = "sha256:a046b2c0118df4a5687f4585cca9d3c3bae5c498c4dff055dcb43fb06a1180c8"}, + {file = "grpcio_status-1.56.2-py3-none-any.whl", hash = "sha256:63f3842867735f59f5d70e723abffd2e8501a6bcd915612a1119e52f10614782"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.56.0" +grpcio = ">=1.56.2" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.56.0" +version = "1.56.2" description = "Protobuf code generator for gRPC" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-tools-1.56.0.tar.gz", hash = "sha256:39f5877cea514b3da9f2683dfb3ffb45ef47b05f4ff39c287d7d61c5057f48b8"}, - {file = "grpcio_tools-1.56.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:cdbae7312e6d132d38ec2c1611b8cafb783e0416cc5c6deae04efde5f16fb190"}, - {file = "grpcio_tools-1.56.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:5f5c416b88d76fbdb548cfee0486928748816b700ece6e591006e5b1dc67598f"}, - {file = "grpcio_tools-1.56.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:23e2ef1dc6a9bf766f091e2c52a68e54d0aff3548f94562e61fb0ac3874d514a"}, - {file = "grpcio_tools-1.56.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8870ab60f8a76b4a7e43184ee03d28112b976d83c43d41cec821f47b3a297da2"}, - {file = "grpcio_tools-1.56.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e59ab6c0bf4a8bb975553ad578d4425bd192775ae384f9406d77d31ad00f6efe"}, - {file = "grpcio_tools-1.56.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b309659534b5d930f9ab6d521670c2dd86cb6ef7f47f37f73f96557e2ec13a49"}, - {file = "grpcio_tools-1.56.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8115b416ea2cad8a87dc3aadfaf26da684e003c3770b12e7219b462505bb5b85"}, - {file = "grpcio_tools-1.56.0-cp310-cp310-win32.whl", hash = "sha256:e4cb62a521efbca4cb1ad50233aa400574b3daaf6eb26707d661a0afe8191d92"}, - {file = "grpcio_tools-1.56.0-cp310-cp310-win_amd64.whl", hash = "sha256:4d59009ed52220eb2d62f5cefa4e58dec930fb92fab27bb390c4cf1d360ac7e1"}, - {file = "grpcio_tools-1.56.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:cd69107705794e815a8b262722c6fea995911cb1dfc1310abf63b476165335d6"}, - {file = "grpcio_tools-1.56.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:2d1ee9e13ce135a6ed451b428ef14af131dc7df2551a5344ff4f8aee2d9fab99"}, - {file = "grpcio_tools-1.56.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:142530b9fdfabe04f0c7e5dacd45b6c419d39704fa439cc0aabf73ea0d8f916d"}, - {file = "grpcio_tools-1.56.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b7a4eb5003a29eecd71707589f93ae7e8fa2e681366a811b3f86695055d8666"}, - {file = "grpcio_tools-1.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa6d9bdd75d3625dae38372b43696e159c10aa98719b4302b1e94f1ff7878d47"}, - {file = "grpcio_tools-1.56.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c43b4fe8c8df4c52d3106bba2cf427f0e46bbebb80e127fbbc3134db0fead7be"}, - {file = "grpcio_tools-1.56.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:168940a4a955b6c65da978dbf62e1c36e3a311bb27f649fd201a228e2583a6d4"}, - {file = "grpcio_tools-1.56.0-cp311-cp311-win32.whl", hash = "sha256:3a4b06169493f9454a7f2516c5d41b566d9734e553bbc505f2a7837f7f4a2df1"}, - {file = "grpcio_tools-1.56.0-cp311-cp311-win_amd64.whl", hash = "sha256:1bd361fcc967c21672ba855fc77ea0e7afa51664033a746df96545f84edc4670"}, - {file = "grpcio_tools-1.56.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:7e6bcb194b81e372411494d8ed69fab89aa3452b7275fce4f7917fbe7b04fb72"}, - {file = "grpcio_tools-1.56.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:02b23a12b91287ebea14b3685735d1d675e77c3cd365ec1771c3e9afbeba1ec6"}, - {file = "grpcio_tools-1.56.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:80d75856f8ec949847386ad2f56a460f21c63bf82ce99ca5b6aa512c0b875fb1"}, - {file = "grpcio_tools-1.56.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9cffff0b4af80285fa49637d69b69d640eb775dc74b23635e4de5faad9e7e744"}, - {file = "grpcio_tools-1.56.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3de6c08b545920a39b31ed13305f946c00b19ac1b13d26119f111b6360f22ccf"}, - {file = "grpcio_tools-1.56.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:128bb13fe9a2681eeb08175f5fbc8e2d8953d7d0dd240e96f9244b9d2547a1aa"}, - {file = "grpcio_tools-1.56.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b57f7f01eafbfe3a293f2efffb675774dbe4074c4627975ec4dc4aa5766801fb"}, - {file = "grpcio_tools-1.56.0-cp37-cp37m-win_amd64.whl", hash = "sha256:282176066fb082ad21c403b84f9d6b440a20482e6f52b83bb2adf54d6fdcae9f"}, - {file = "grpcio_tools-1.56.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:d9b8d1c42854d3433c058795f52b1418b53dd8c1e9811fecb1312202e803a2c5"}, - {file = "grpcio_tools-1.56.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:accf713f51da74b1a18aa4b31df0ab135510704661f735a938081777b79a4c25"}, - {file = "grpcio_tools-1.56.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:ac33fd2d02d24101ea389be8e05b928acb58be56403d4ebc3aecfab473fa4a25"}, - {file = "grpcio_tools-1.56.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4acdc7b957abfd76581717f0ac8e4408e0a85b7d0ac8d2cdf4d964f16926b897"}, - {file = "grpcio_tools-1.56.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79291bfb1fe5f21d99f4839f43d3c5d44c5402c830a24dbb2811d785dd21264b"}, - {file = "grpcio_tools-1.56.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0a8767e4de0f573c678313c5de075ac0e163a192bb135018e45015a22f234387"}, - {file = "grpcio_tools-1.56.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:96fe2f7f5805d88cb7f2e3e3502550b2883dfab0f9efcf3cbd444942cf2ee1da"}, - {file = "grpcio_tools-1.56.0-cp38-cp38-win32.whl", hash = "sha256:21cf32ccffd4f1800b0dcdf58aa1fc7f626795c9da784c3d817c944edcf2d3ae"}, - {file = "grpcio_tools-1.56.0-cp38-cp38-win_amd64.whl", hash = "sha256:f3ab1a9fad636302f7307d143f64a9fbd11bc041652bf53bb016006e9a5ca820"}, - {file = "grpcio_tools-1.56.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:8989d363ac1996238fee61c8f5663f15a8fc362cb1e758c4a686b76cb457cd70"}, - {file = "grpcio_tools-1.56.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:11cdd9cbf0c09c3a761c6f59dfd7128104be7cd393334efe386d4fc3f990ee1a"}, - {file = "grpcio_tools-1.56.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5fd4c005a4afec16578849bc522ddf3298d6d499b3d37bf51314b086c714cdd5"}, - {file = "grpcio_tools-1.56.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f7302acaa07cf4966c926fcd6a60c8d30a697f730c38168bf83e1519b464115b"}, - {file = "grpcio_tools-1.56.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c1c43d185ebf904c3deec23c36ca2ba4e95db999cf00fc8f85eda4551622a26"}, - {file = "grpcio_tools-1.56.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b12bb8c1d408ae40e4c806a3a8ebda2d107310e46696e1da13d0dc3f91fbd19d"}, - {file = "grpcio_tools-1.56.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:781cf09e4d5c9288708f6ec9c3eae64d9d5a0f4c46c7ebe70ebb7ab4f6384789"}, - {file = "grpcio_tools-1.56.0-cp39-cp39-win32.whl", hash = "sha256:c62f07452dee3f1ed23aeaef821797c5e516f79535e97fe6a6b0a0ee8db1cc91"}, - {file = "grpcio_tools-1.56.0-cp39-cp39-win_amd64.whl", hash = "sha256:7f063443870650e55012fdb3a58ff4ce5f4042b81dad6b749333ee8146157511"}, + {file = "grpcio-tools-1.56.2.tar.gz", hash = "sha256:82af2f4040084141a732f0ef1ecf3f14fdf629923d74d850415e4d09a077e77a"}, + {file = "grpcio_tools-1.56.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:42272376e9a5a1c631863cda056c143c98d21e5b670db5c8c5b7ed0ba3a1a6eb"}, + {file = "grpcio_tools-1.56.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:a8735d7aa34be99dddfbd476eff6005e684bb2c893c0f62a5811528b84c5b371"}, + {file = "grpcio_tools-1.56.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:68ef3aa7509e5e7a6e7c0ecc183e28118e73da4bef0fc77079648601ce35e58f"}, + {file = "grpcio_tools-1.56.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:380985b8d95ea2469e103945bd83a815d1213e370f580631fdd5a3dbaa17e446"}, + {file = "grpcio_tools-1.56.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bfb375eb4f1946d68b8bc7b963c756defa31aa573a35c152a7233d06c0ad6ad"}, + {file = "grpcio_tools-1.56.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:13388a22fcba9a1a87f217130a1a01365716af74bd5d0a8a54fc383b8e048ef2"}, + {file = "grpcio_tools-1.56.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7a26160bc0ea5b464715789d4d2a66f01816271677673d65da39bac65b9ea838"}, + {file = "grpcio_tools-1.56.2-cp310-cp310-win32.whl", hash = "sha256:ff16dd0b086e75f574dbc122e018a44dbd1c6dae3f3621ea99e8e5a6b2706e12"}, + {file = "grpcio_tools-1.56.2-cp310-cp310-win_amd64.whl", hash = "sha256:2037109c1ce253a8e013c9e3ad3722e887d28a1807acdeb1a51b295c8200137b"}, + {file = "grpcio_tools-1.56.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:6dc43300189a69807857c52a3d782e9d3bbfb1cb72dcb27b4043c25161919601"}, + {file = "grpcio_tools-1.56.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:e7009623635ebcd3dd7fe974883fc2d9a3ff0fcef419bfc0a2da8071b372d9f5"}, + {file = "grpcio_tools-1.56.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c7ca2272022f90b73efe900244aaebe9dd7cf3b379e99e08a88984e2fdd229c2"}, + {file = "grpcio_tools-1.56.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:493775d17ea09cea6047ba81e4d3f0eb82e34d2fbd3b96e43f72b44ce74726ee"}, + {file = "grpcio_tools-1.56.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41af279cf5359b123138236c0980440f4cb4d3d18f03b5c1c314cc1512048351"}, + {file = "grpcio_tools-1.56.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:857d72e991d449ec4d2f8337e5e24ddf77b4539965f5cabc84d4b63585832982"}, + {file = "grpcio_tools-1.56.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c0640728d63c9fa56e9a1679943ae4e33ad43a10802dd7a93255870731f44d07"}, + {file = "grpcio_tools-1.56.2-cp311-cp311-win32.whl", hash = "sha256:355204d1b33c7a19e7d69afda411e6595d39ba1e9cbf561770ac1d5403296554"}, + {file = "grpcio_tools-1.56.2-cp311-cp311-win_amd64.whl", hash = "sha256:ea5d108d28b4cd2e28539241c6aee96bda83086d8888c36785d9f84ea690d896"}, + {file = "grpcio_tools-1.56.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:28444615b7a76b3d9267f81d1487fcad21a581d00564164d9e25ccc28635a811"}, + {file = "grpcio_tools-1.56.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:45d8b5ad6716848d5b68d9cee29a1a9c5c4baa1824ec5b92d9e35acedddba076"}, + {file = "grpcio_tools-1.56.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:31d1183d28ffc8da242333cb9f683f5093941da80dd5281db0fa93077aecb518"}, + {file = "grpcio_tools-1.56.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0059dfc9bea8f7bca69c15ca62c88904c4f907fde1137e0743b5eee054661873"}, + {file = "grpcio_tools-1.56.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24fc857252181c9950ed2d8cee3df5bd0f42861c4ad0db2a57400186827f96e5"}, + {file = "grpcio_tools-1.56.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3a74a5e4fc8121a51401665f96f9a70aee50a2f1221e4a199e67b3b8f55881e8"}, + {file = "grpcio_tools-1.56.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bec47db5d8b5c3b2a44afdbc3f3bf306e34279289a206d20222824381ca3cb13"}, + {file = "grpcio_tools-1.56.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c0dbaac63a25c088f864295f394230eeb7be48dac2264433fda2603f86c36b25"}, + {file = "grpcio_tools-1.56.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:0a4f9cce5a16613b6d3123c89f9d50e0d13b466799af17bc723dc7d2901a54e4"}, + {file = "grpcio_tools-1.56.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:ea5fc1b49514b44a3e5a45156c025002f172ade4c509e58c51967865c7c6fa45"}, + {file = "grpcio_tools-1.56.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:54da410124547bacb97a54546c1a95f1af0125e48edc8b5679412ef8b2844f81"}, + {file = "grpcio_tools-1.56.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5223668649172d879ee780253b8e4a79144c56a3cc1bb021847f583508c2b0be"}, + {file = "grpcio_tools-1.56.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:483256d5f5be6a77b24d8a5f06ca152d1571c62bf5c738834da61107c7563afe"}, + {file = "grpcio_tools-1.56.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1f334718eb796799bfadbac5567456fb745cee8c7b438c93b74d1ce676c6ad07"}, + {file = "grpcio_tools-1.56.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:26751f69cbbc8ea19cf0657b7d109a6db7df81f80caf16380ebcd20eea27652c"}, + {file = "grpcio_tools-1.56.2-cp38-cp38-win32.whl", hash = "sha256:4056ff13e30813d42a30ce1cdfeaeb6bbee915515c161c1df896dac3143ae643"}, + {file = "grpcio_tools-1.56.2-cp38-cp38-win_amd64.whl", hash = "sha256:878b9269ceb0dd934b61697a9dd9a5c3e9552521e8f46ab32cf4d72a223f7b6c"}, + {file = "grpcio_tools-1.56.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:216e86d3a6ccc31b27fa4c12491981a0a39d4787d2358b6df05baffa40084494"}, + {file = "grpcio_tools-1.56.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:778224fcbc1cc7eaf222ce94676afbac8d72b4f84cf4239e30b01d2450a46126"}, + {file = "grpcio_tools-1.56.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:14120fb2c6f7894fac5b689934368c692ec50f50a320e8073277ab7778fd612f"}, + {file = "grpcio_tools-1.56.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:014da3ed176beb2b1c8430ccc34c8fe962cdd5480e56fb4ab9de60e60c315f3f"}, + {file = "grpcio_tools-1.56.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8febb4f90b8fab3179f5bdaa159f1d2a20523ea17ec0d66bdec7732f9532de91"}, + {file = "grpcio_tools-1.56.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ffae7df3318266614f7aa440acb2098c064b6b5ae061fc22125092386349e526"}, + {file = "grpcio_tools-1.56.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d86e24eb6e3973c55e9c74412ff755d1b9d15518c4eaf95676acff49d0162a2"}, + {file = "grpcio_tools-1.56.2-cp39-cp39-win32.whl", hash = "sha256:506d00a86950adf4017395551a4547c0b7fcefa90e4c220135fc3e34e31be81b"}, + {file = "grpcio_tools-1.56.2-cp39-cp39-win_amd64.whl", hash = "sha256:8da04f033b8f4c597e8fc990e2f626bad2b269227bdd554592ea618f624f1aa9"}, ] [package.dependencies] -grpcio = ">=1.56.0" +grpcio = ">=1.56.2" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -806,7 +835,7 @@ files = [ ] [[package]] -name = "Jinja2" +name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." optional = false @@ -822,6 +851,16 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "lit" +version = "16.0.6" +description = "A Software Testing Tool" +optional = false +python-versions = "*" +files = [ + {file = "lit-16.0.6.tar.gz", hash = "sha256:84623c9c23b6b14763d637f4e63e6b721b3446ada40bf7001d8fee70b8e77a9a"}, +] + [[package]] name = "loguru" version = "0.6.0" @@ -841,7 +880,7 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] [[package]] -name = "MarkupSafe" +name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." optional = false @@ -1047,36 +1086,36 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "numpy" -version = "1.25.0" +version = "1.25.2" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "numpy-1.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8aa130c3042052d656751df5e81f6d61edff3e289b5994edcf77f54118a8d9f4"}, - {file = "numpy-1.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e3f2b96e3b63c978bc29daaa3700c028fe3f049ea3031b58aa33fe2a5809d24"}, - {file = "numpy-1.25.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6b267f349a99d3908b56645eebf340cb58f01bd1e773b4eea1a905b3f0e4208"}, - {file = "numpy-1.25.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4aedd08f15d3045a4e9c648f1e04daca2ab1044256959f1f95aafeeb3d794c16"}, - {file = "numpy-1.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6d183b5c58513f74225c376643234c369468e02947b47942eacbb23c1671f25d"}, - {file = "numpy-1.25.0-cp310-cp310-win32.whl", hash = "sha256:d76a84998c51b8b68b40448ddd02bd1081bb33abcdc28beee6cd284fe11036c6"}, - {file = "numpy-1.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:c0dc071017bc00abb7d7201bac06fa80333c6314477b3d10b52b58fa6a6e38f6"}, - {file = "numpy-1.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c69fe5f05eea336b7a740e114dec995e2f927003c30702d896892403df6dbf0"}, - {file = "numpy-1.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c7211d7920b97aeca7b3773a6783492b5b93baba39e7c36054f6e749fc7490c"}, - {file = "numpy-1.25.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecc68f11404930e9c7ecfc937aa423e1e50158317bf67ca91736a9864eae0232"}, - {file = "numpy-1.25.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e559c6afbca484072a98a51b6fa466aae785cfe89b69e8b856c3191bc8872a82"}, - {file = "numpy-1.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6c284907e37f5e04d2412950960894b143a648dea3f79290757eb878b91acbd1"}, - {file = "numpy-1.25.0-cp311-cp311-win32.whl", hash = "sha256:95367ccd88c07af21b379be1725b5322362bb83679d36691f124a16357390153"}, - {file = "numpy-1.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:b76aa836a952059d70a2788a2d98cb2a533ccd46222558b6970348939e55fc24"}, - {file = "numpy-1.25.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b792164e539d99d93e4e5e09ae10f8cbe5466de7d759fc155e075237e0c274e4"}, - {file = "numpy-1.25.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7cd981ccc0afe49b9883f14761bb57c964df71124dcd155b0cba2b591f0d64b9"}, - {file = "numpy-1.25.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5aa48bebfb41f93043a796128854b84407d4df730d3fb6e5dc36402f5cd594c0"}, - {file = "numpy-1.25.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5177310ac2e63d6603f659fadc1e7bab33dd5a8db4e0596df34214eeab0fee3b"}, - {file = "numpy-1.25.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0ac6edfb35d2a99aaf102b509c8e9319c499ebd4978df4971b94419a116d0790"}, - {file = "numpy-1.25.0-cp39-cp39-win32.whl", hash = "sha256:7412125b4f18aeddca2ecd7219ea2d2708f697943e6f624be41aa5f8a9852cc4"}, - {file = "numpy-1.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:26815c6c8498dc49d81faa76d61078c4f9f0859ce7817919021b9eba72b425e3"}, - {file = "numpy-1.25.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5b1b90860bf7d8a8c313b372d4f27343a54f415b20fb69dd601b7efe1029c91e"}, - {file = "numpy-1.25.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85cdae87d8c136fd4da4dad1e48064d700f63e923d5af6c8c782ac0df8044542"}, - {file = "numpy-1.25.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cc3fda2b36482891db1060f00f881c77f9423eead4c3579629940a3e12095fe8"}, - {file = "numpy-1.25.0.tar.gz", hash = "sha256:f1accae9a28dc3cda46a91de86acf69de0d1b5f4edd44a9b0c3ceb8036dfff19"}, + {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, + {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, + {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"}, + {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"}, + {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"}, + {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"}, + {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"}, + {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"}, + {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"}, + {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"}, + {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"}, + {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"}, + {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"}, + {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"}, + {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"}, + {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"}, + {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"}, + {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"}, + {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"}, + {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"}, + {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"}, + {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, ] [[package]] @@ -1355,24 +1394,24 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.23.3" +version = "4.23.4" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.23.3-cp310-abi3-win32.whl", hash = "sha256:514b6bbd54a41ca50c86dd5ad6488afe9505901b3557c5e0f7823a0cf67106fb"}, - {file = "protobuf-4.23.3-cp310-abi3-win_amd64.whl", hash = "sha256:cc14358a8742c4e06b1bfe4be1afbdf5c9f6bd094dff3e14edb78a1513893ff5"}, - {file = "protobuf-4.23.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2991f5e7690dab569f8f81702e6700e7364cc3b5e572725098215d3da5ccc6ac"}, - {file = "protobuf-4.23.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:08fe19d267608d438aa37019236db02b306e33f6b9902c3163838b8e75970223"}, - {file = "protobuf-4.23.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3b01a5274ac920feb75d0b372d901524f7e3ad39c63b1a2d55043f3887afe0c1"}, - {file = "protobuf-4.23.3-cp37-cp37m-win32.whl", hash = "sha256:aca6e86a08c5c5962f55eac9b5bd6fce6ed98645d77e8bfc2b952ecd4a8e4f6a"}, - {file = "protobuf-4.23.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0149053336a466e3e0b040e54d0b615fc71de86da66791c592cc3c8d18150bf8"}, - {file = "protobuf-4.23.3-cp38-cp38-win32.whl", hash = "sha256:84ea0bd90c2fdd70ddd9f3d3fc0197cc24ecec1345856c2b5ba70e4d99815359"}, - {file = "protobuf-4.23.3-cp38-cp38-win_amd64.whl", hash = "sha256:3bcbeb2bf4bb61fe960dd6e005801a23a43578200ea8ceb726d1f6bd0e562ba1"}, - {file = "protobuf-4.23.3-cp39-cp39-win32.whl", hash = "sha256:5cb9e41188737f321f4fce9a4337bf40a5414b8d03227e1d9fbc59bc3a216e35"}, - {file = "protobuf-4.23.3-cp39-cp39-win_amd64.whl", hash = "sha256:29660574cd769f2324a57fb78127cda59327eb6664381ecfe1c69731b83e8288"}, - {file = "protobuf-4.23.3-py3-none-any.whl", hash = "sha256:447b9786ac8e50ae72cae7a2eec5c5df6a9dbf9aa6f908f1b8bda6032644ea62"}, - {file = "protobuf-4.23.3.tar.gz", hash = "sha256:7a92beb30600332a52cdadbedb40d33fd7c8a0d7f549c440347bc606fb3fe34b"}, + {file = "protobuf-4.23.4-cp310-abi3-win32.whl", hash = "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b"}, + {file = "protobuf-4.23.4-cp310-abi3-win_amd64.whl", hash = "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12"}, + {file = "protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd"}, + {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a"}, + {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597"}, + {file = "protobuf-4.23.4-cp37-cp37m-win32.whl", hash = "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e"}, + {file = "protobuf-4.23.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0"}, + {file = "protobuf-4.23.4-cp38-cp38-win32.whl", hash = "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70"}, + {file = "protobuf-4.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2"}, + {file = "protobuf-4.23.4-cp39-cp39-win32.whl", hash = "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720"}, + {file = "protobuf-4.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474"}, + {file = "protobuf-4.23.4-py3-none-any.whl", hash = "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff"}, + {file = "protobuf-4.23.4.tar.gz", hash = "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9"}, ] [[package]] @@ -1486,52 +1525,52 @@ files = [ ] [[package]] -name = "PyYAML" -version = "6.0" +name = "pyyaml" +version = "6.0.1" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.6" files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] [[package]] @@ -1885,31 +1924,19 @@ files = [ [[package]] name = "torch" -version = "2.0.1" +version = "2.0.1+cu118" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" optional = false python-versions = ">=3.8.0" files = [ - {file = "torch-2.0.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:8ced00b3ba471856b993822508f77c98f48a458623596a4c43136158781e306a"}, - {file = "torch-2.0.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:359bfaad94d1cda02ab775dc1cc386d585712329bb47b8741607ef6ef4950747"}, - {file = "torch-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7c84e44d9002182edd859f3400deaa7410f5ec948a519cc7ef512c2f9b34d2c4"}, - {file = "torch-2.0.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:567f84d657edc5582d716900543e6e62353dbe275e61cdc36eda4929e46df9e7"}, - {file = "torch-2.0.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:787b5a78aa7917465e9b96399b883920c88a08f4eb63b5a5d2d1a16e27d2f89b"}, - {file = "torch-2.0.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:e617b1d0abaf6ced02dbb9486803abfef0d581609b09641b34fa315c9c40766d"}, - {file = "torch-2.0.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:b6019b1de4978e96daa21d6a3ebb41e88a0b474898fe251fd96189587408873e"}, - {file = "torch-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:dbd68cbd1cd9da32fe5d294dd3411509b3d841baecb780b38b3b7b06c7754434"}, - {file = "torch-2.0.1-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:ef654427d91600129864644e35deea761fb1fe131710180b952a6f2e2207075e"}, - {file = "torch-2.0.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:25aa43ca80dcdf32f13da04c503ec7afdf8e77e3a0183dd85cd3e53b2842e527"}, - {file = "torch-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5ef3ea3d25441d3957348f7e99c7824d33798258a2bf5f0f0277cbcadad2e20d"}, - {file = "torch-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0882243755ff28895e8e6dc6bc26ebcf5aa0911ed81b2a12f241fc4b09075b13"}, - {file = "torch-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:f66aa6b9580a22b04d0af54fcd042f52406a8479e2b6a550e3d9f95963e168c8"}, - {file = "torch-2.0.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:1adb60d369f2650cac8e9a95b1d5758e25d526a34808f7448d0bd599e4ae9072"}, - {file = "torch-2.0.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:1bcffc16b89e296826b33b98db5166f990e3b72654a2b90673e817b16c50e32b"}, - {file = "torch-2.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:e10e1597f2175365285db1b24019eb6f04d53dcd626c735fc502f1e8b6be9875"}, - {file = "torch-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:423e0ae257b756bb45a4b49072046772d1ad0c592265c5080070e0767da4e490"}, - {file = "torch-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8742bdc62946c93f75ff92da00e3803216c6cce9b132fbca69664ca38cfb3e18"}, - {file = "torch-2.0.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:c62df99352bd6ee5a5a8d1832452110435d178b5164de450831a3a8cc14dc680"}, - {file = "torch-2.0.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:671a2565e3f63b8fe8e42ae3e36ad249fe5e567435ea27b94edaa672a7d0c416"}, + {file = "torch-2.0.1+cu118-cp310-cp310-linux_x86_64.whl", hash = "sha256:a7a49d459bf4862f64f7bc1a68beccf8881c2fa9f3e0569608e16ba6f85ebf7b"}, + {file = "torch-2.0.1+cu118-cp310-cp310-win_amd64.whl", hash = "sha256:f58d75619bc96e4322343c030b893613701caa2d6db8017155da226c14171335"}, + {file = "torch-2.0.1+cu118-cp311-cp311-linux_x86_64.whl", hash = "sha256:143b6c658c17d43376e2dfbaa2c106d35639d615e5e8dec4429cf1e510dd8d61"}, + {file = "torch-2.0.1+cu118-cp311-cp311-win_amd64.whl", hash = "sha256:b663a4ee744d574095dbd612644de345944247c0605692309fd9f6c7ccdea022"}, + {file = "torch-2.0.1+cu118-cp38-cp38-linux_x86_64.whl", hash = "sha256:2ce38a6e4ea7c4b7f5baa51e65243a5f687f6e19ab7915ba5b2a431105f50bbe"}, + {file = "torch-2.0.1+cu118-cp38-cp38-win_amd64.whl", hash = "sha256:e58d26a11bd57ac19761c018c3151c15bc71d068afc8ec409bfd9b4cfcc63a52"}, + {file = "torch-2.0.1+cu118-cp39-cp39-linux_x86_64.whl", hash = "sha256:eb55f29db5744eda8a96f5594e637daed0d52278273005de759970e67cfa6a5a"}, + {file = "torch-2.0.1+cu118-cp39-cp39-win_amd64.whl", hash = "sha256:fa225b6f941ee0e78978ac85ed7744d3c19fff462473821f8060c14faa60043e"}, ] [package.dependencies] @@ -1917,11 +1944,17 @@ filelock = "*" jinja2 = "*" networkx = "*" sympy = "*" +triton = {version = "2.0.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} typing-extensions = "*" [package.extras] opt-einsum = ["opt-einsum (>=3.3)"] +[package.source] +type = "legacy" +url = "https://download.pytorch.org/whl/cu118" +reference = "pytorch-gpu-src" + [[package]] name = "tqdm" version = "4.65.0" @@ -2010,6 +2043,43 @@ torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", video = ["av (==9.2.0)", "decord (==0.6.0)"] vision = ["Pillow"] +[[package]] +name = "triton" +version = "2.0.0" +description = "A language and compiler for custom Deep Learning operations" +optional = false +python-versions = "*" +files = [ + {file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"}, + {file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"}, + {file = "triton-2.0.0-1-cp36-cp36m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd"}, + {file = "triton-2.0.0-1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b"}, + {file = "triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef"}, + {file = "triton-2.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656"}, + {file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"}, + {file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"}, + {file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"}, + {file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"}, + {file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"}, + {file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"}, + {file = "triton-2.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fedce6a381901b1547e0e7e1f2546e4f65dca6d91e2d8a7305a2d1f5551895be"}, + {file = "triton-2.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75834f27926eab6c7f00ce73aaf1ab5bfb9bec6eb57ab7c0bfc0a23fac803b4c"}, + {file = "triton-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0117722f8c2b579cd429e0bee80f7731ae05f63fe8e9414acd9a679885fcbf42"}, + {file = "triton-2.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd9be5d0c2e45d2b7e6ddc6da20112b6862d69741576f9c3dbaf941d745ecae"}, + {file = "triton-2.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a0d2c3fc2eab4ba71384f2e785fbfd47aa41ae05fa58bf12cb31dcbd0aeceb"}, + {file = "triton-2.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c47b72c72693198163ece9d90a721299e4fb3b8e24fd13141e384ad952724f"}, +] + +[package.dependencies] +cmake = "*" +filelock = "*" +lit = "*" +torch = "*" + +[package.extras] +tests = ["autopep8", "flake8", "isort", "numpy", "pytest", "scipy (>=1.7.1)"] +tutorials = ["matplotlib", "pandas", "tabulate"] + [[package]] name = "typer" version = "0.6.1" @@ -2054,13 +2124,13 @@ files = [ [[package]] name = "urllib3" -version = "2.0.3" +version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.7" files = [ - {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, - {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, + {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, + {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, ] [package.extras] @@ -2169,109 +2239,96 @@ files = [ [[package]] name = "xxhash" -version = "3.2.0" +version = "3.3.0" description = "Python binding for xxHash" optional = true -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "xxhash-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:af44b9e59c4b2926a4e3c7f9d29949ff42fcea28637ff6b8182e654461932be8"}, - {file = "xxhash-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1bdd57973e2b802ef32553d7bebf9402dac1557874dbe5c908b499ea917662cd"}, - {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b7c9aa77bbce61a5e681bd39cb6a804338474dcc90abe3c543592aa5d6c9a9b"}, - {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11bf87dc7bb8c3b0b5e24b7b941a9a19d8c1f88120b6a03a17264086bc8bb023"}, - {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2783d41487ce6d379fdfaa7332fca5187bf7010b9bddcf20cafba923bc1dc665"}, - {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:561076ca0dcef2fbc20b2bc2765bff099e002e96041ae9dbe910a863ca6ee3ea"}, - {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a26eeb4625a6e61cedc8c1b39b89327c9c7e1a8c2c4d786fe3f178eb839ede6"}, - {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d93a44d0104d1b9b10de4e7aadf747f6efc1d7ec5ed0aa3f233a720725dd31bd"}, - {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:89585adc73395a10306d2e2036e50d6c4ac0cf8dd47edf914c25488871b64f6d"}, - {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a892b4b139126a86bfdcb97cd912a2f8c4e8623869c3ef7b50871451dd7afeb0"}, - {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e998efb190653f70e0f30d92b39fc645145369a4823bee46af8ddfc244aa969d"}, - {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e8ed3bd2b8bb3277710843ca63e4f5c3ee6f8f80b083be5b19a7a9905420d11e"}, - {file = "xxhash-3.2.0-cp310-cp310-win32.whl", hash = "sha256:20181cbaed033c72cb881b2a1d13c629cd1228f113046133469c9a48cfcbcd36"}, - {file = "xxhash-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:a0f7a16138279d707db778a63264d1d6016ac13ffd3f1e99f54b2855d6c0d8e1"}, - {file = "xxhash-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5daff3fb5bfef30bc5a2cb143810d376d43461445aa17aece7210de52adbe151"}, - {file = "xxhash-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75bb5be3c5de702a547715f320ecf5c8014aeca750ed5147ca75389bd22e7343"}, - {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01f36b671ff55cb1d5c2f6058b799b697fd0ae4b4582bba6ed0999678068172a"}, - {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4d4519123aac73c93159eb8f61db9682393862dd669e7eae034ecd0a35eadac"}, - {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:994e4741d5ed70fc2a335a91ef79343c6b1089d7dfe6e955dd06f8ffe82bede6"}, - {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:919bc1b010aa6ff0eb918838ff73a435aed9e9a19c3202b91acecd296bf75607"}, - {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17b65454c5accbb079c45eca546c27c4782f5175aa320758fafac896b1549d27"}, - {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b0c094d5e65a46dbf3fe0928ff20873a747e6abfd2ed4b675beeb2750624bc2e"}, - {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f94163ebe2d5546e6a5977e96d83621f4689c1054053428cf8d4c28b10f92f69"}, - {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cead7c0307977a00b3f784cff676e72c147adbcada19a2e6fc2ddf54f37cf387"}, - {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a0e1bd0260c1da35c1883321ce2707ceea07127816ab625e1226ec95177b561a"}, - {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc8878935671490efe9275fb4190a6062b73277bd273237179b9b5a2aa436153"}, - {file = "xxhash-3.2.0-cp311-cp311-win32.whl", hash = "sha256:a433f6162b18d52f7068175d00bd5b1563b7405f926a48d888a97b90a160c40d"}, - {file = "xxhash-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:a32d546a1752e4ee7805d6db57944f7224afa7428d22867006b6486e4195c1f3"}, - {file = "xxhash-3.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:82daaab720866bf690b20b49de5640b0c27e3b8eea2d08aa75bdca2b0f0cfb63"}, - {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3126df6520cbdbaddd87ce74794b2b6c45dd2cf6ac2b600a374b8cdb76a2548c"}, - {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e172c1ee40507ae3b8d220f4048aaca204f203e1e4197e8e652f5c814f61d1aa"}, - {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5384f1d9f30876f5d5b618464fb19ff7ce6c0fe4c690fbaafd1c52adc3aae807"}, - {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26cb52174a7e96a17acad27a3ca65b24713610ac479c99ac9640843822d3bebf"}, - {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbcd613a5e76b1495fc24db9c37a6b7ee5f214fd85979187ec4e032abfc12ded"}, - {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:f988daf25f31726d5b9d0be6af636ca9000898f9ea43a57eac594daea25b0948"}, - {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:bbc30c98ab006ab9fc47e5ed439c00f706bc9d4441ff52693b8b6fea335163e0"}, - {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:2408d49260b0a4a7cc6ba445aebf38e073aeaf482f8e32767ca477e32ccbbf9e"}, - {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:3f4152fd0bf8b03b79f2f900fd6087a66866537e94b5a11fd0fd99ef7efe5c42"}, - {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:0eea848758e4823a01abdbcccb021a03c1ee4100411cbeeb7a5c36a202a0c13c"}, - {file = "xxhash-3.2.0-cp36-cp36m-win32.whl", hash = "sha256:77709139af5123c578ab06cf999429cdb9ab211047acd0c787e098dcb3f1cb4d"}, - {file = "xxhash-3.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:91687671fd9d484a4e201ad266d366b695a45a1f2b41be93d116ba60f1b8f3b3"}, - {file = "xxhash-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e4af8bc5c3fcc2192c266421c6aa2daab1a18e002cb8e66ef672030e46ae25cf"}, - {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8be562e2ce3e481d9209b6f254c3d7c5ff920eb256aba2380d2fb5ba75d4f87"}, - {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9eba0c7c12126b12f7fcbea5513f28c950d28f33d2a227f74b50b77789e478e8"}, - {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2198c4901a0223c48f6ec0a978b60bca4f4f7229a11ca4dc96ca325dd6a29115"}, - {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50ce82a71b22a3069c02e914bf842118a53065e2ec1c6fb54786e03608ab89cc"}, - {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b5019fb33711c30e54e4e57ae0ca70af9d35b589d385ac04acd6954452fa73bb"}, - {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0d54ac023eef7e3ac9f0b8841ae8a376b933043bc2ad428121346c6fa61c491c"}, - {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c55fa832fc3fe64e0d29da5dc9b50ba66ca93312107cec2709300ea3d3bab5c7"}, - {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f4ce006215497993ae77c612c1883ca4f3973899573ce0c52fee91f0d39c4561"}, - {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1afb9b9d27fd675b436cb110c15979976d92d761ad6e66799b83756402f3a974"}, - {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:baa99cebf95c1885db21e119395f222a706a2bb75a545f0672880a442137725e"}, - {file = "xxhash-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:75aa692936942ccb2e8fd6a386c81c61630ac1b6d6e921698122db8a930579c3"}, - {file = "xxhash-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:0a2cdfb5cae9fafb9f7b65fd52ecd60cf7d72c13bb2591ea59aaefa03d5a8827"}, - {file = "xxhash-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a68d1e8a390b660d94b9360ae5baa8c21a101bd9c4790a8b30781bada9f1fc6"}, - {file = "xxhash-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ce7c3ce28f94302df95eaea7c9c1e2c974b6d15d78a0c82142a97939d7b6c082"}, - {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dcb419bf7b0bc77d366e5005c25682249c5521a63fd36c51f584bd91bb13bd5"}, - {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae521ed9287f86aac979eeac43af762f03d9d9797b2272185fb9ddd810391216"}, - {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0d16775094423088ffa357d09fbbb9ab48d2fb721d42c0856b801c86f616eec"}, - {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe454aeab348c42f56d6f7434ff758a3ef90787ac81b9ad5a363cd61b90a1b0b"}, - {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:052fd0efdd5525c2dbc61bebb423d92aa619c4905bba605afbf1e985a562a231"}, - {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:02badf3754e2133de254a4688798c4d80f0060635087abcb461415cb3eb82115"}, - {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:66b8a90b28c13c2aae7a71b32638ceb14cefc2a1c8cf23d8d50dfb64dfac7aaf"}, - {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:649cdf19df175925ad87289ead6f760cd840730ee85abc5eb43be326a0a24d97"}, - {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4b948a03f89f5c72d69d40975af8af241111f0643228796558dc1cae8f5560b0"}, - {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49f51fab7b762da7c2cee0a3d575184d3b9be5e2f64f26cae2dd286258ac9b3c"}, - {file = "xxhash-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1a42994f0d42b55514785356722d9031f064fd34e495b3a589e96db68ee0179d"}, - {file = "xxhash-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0a6d58ba5865475e53d6c2c4fa6a62e2721e7875e146e2681e5337a6948f12e7"}, - {file = "xxhash-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:aabdbc082030f8df613e2d2ea1f974e7ad36a539bdfc40d36f34e55c7e4b8e94"}, - {file = "xxhash-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:498843b66b9ca416e9d03037e5875c8d0c0ab9037527e22df3b39aa5163214cd"}, - {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a910b1193cd90af17228f5d6069816646df0148f14f53eefa6b2b11a1dedfcd0"}, - {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb6d8ce31dc25faf4da92991320e211fa7f42de010ef51937b1dc565a4926501"}, - {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:883dc3d3942620f4c7dbc3fd6162f50a67f050b714e47da77444e3bcea7d91cc"}, - {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59dc8bfacf89b8f5be54d55bc3b4bd6d74d0c5320c8a63d2538ac7df5b96f1d5"}, - {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:61e6aa1d30c2af692aa88c4dd48709426e8b37bff6a574ee2de677579c34a3d6"}, - {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:314ec0bd21f0ee8d30f2bd82ed3759314bd317ddbbd8555668f3d20ab7a8899a"}, - {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dad638cde3a5357ad3163b80b3127df61fb5b5e34e9e05a87697144400ba03c7"}, - {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:eaa3ea15025b56076d806b248948612289b093e8dcda8d013776b3848dffff15"}, - {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:7deae3a312feb5c17c97cbf18129f83cbd3f1f9ec25b0f50e2bd9697befb22e7"}, - {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:add774341c09853b1612c64a526032d95ab1683053325403e1afbe3ad2f374c5"}, - {file = "xxhash-3.2.0-cp39-cp39-win32.whl", hash = "sha256:9b94749130ef3119375c599bfce82142c2500ef9ed3280089157ee37662a7137"}, - {file = "xxhash-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:e57d94a1552af67f67b27db5dba0b03783ea69d5ca2af2f40e098f0ba3ce3f5f"}, - {file = "xxhash-3.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92fd765591c83e5c5f409b33eac1d3266c03d3d11c71a7dbade36d5cdee4fbc0"}, - {file = "xxhash-3.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8970f6a411a9839a02b23b7e90bbbba4a6de52ace009274998566dc43f36ca18"}, - {file = "xxhash-3.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5f3e33fe6cbab481727f9aeb136a213aed7e33cd1ca27bd75e916ffacc18411"}, - {file = "xxhash-3.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:368265392cb696dd53907e2328b5a8c1bee81cf2142d0cc743caf1c1047abb36"}, - {file = "xxhash-3.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3b1f3c6d67fa9f49c4ff6b25ce0e7143bab88a5bc0f4116dd290c92337d0ecc7"}, - {file = "xxhash-3.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c5e8db6e1ee7267b7c412ad0afd5863bf7a95286b8333a5958c8097c69f94cf5"}, - {file = "xxhash-3.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:761df3c7e2c5270088b691c5a8121004f84318177da1ca1db64222ec83c44871"}, - {file = "xxhash-3.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2d15a707e7f689531eb4134eccb0f8bf3844bb8255ad50823aa39708d9e6755"}, - {file = "xxhash-3.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6b2ba4ff53dd5f57d728095e3def7375eb19c90621ce3b41b256de84ec61cfd"}, - {file = "xxhash-3.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:61b0bcf946fdfd8ab5f09179dc2b5c74d1ef47cedfc6ed0ec01fdf0ee8682dd3"}, - {file = "xxhash-3.2.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f7b79f0f302396d8e0d444826ceb3d07b61977793886ebae04e82796c02e42dc"}, - {file = "xxhash-3.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0773cd5c438ffcd5dbff91cdd503574f88a4b960e70cedeb67736583a17a918"}, - {file = "xxhash-3.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ec1f57127879b419a2c8d2db9d9978eb26c61ae17e5972197830430ae78d25b"}, - {file = "xxhash-3.2.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d4b15c00e807b1d3d0b612338c814739dec310b80fb069bd732b98ddc709ad7"}, - {file = "xxhash-3.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:9d3f686e3d1c8900c5459eee02b60c7399e20ec5c6402364068a343c83a61d90"}, - {file = "xxhash-3.2.0.tar.gz", hash = "sha256:1afd47af8955c5db730f630ad53ae798cf7fae0acb64cebb3cf94d35c47dd088"}, + {file = "xxhash-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:70ef7288d1cb1ad16e02d101ea43bb0e392d985d60b9b0035aee80663530960d"}, + {file = "xxhash-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:44ff8c673cab50be46784e0aec62aa6f0ca9ea765e2b0690e8945d0cd950dcaf"}, + {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfebc90273ae2beb813d8118a2bfffb5a5a81ac054fbfd061ea18fd0a81db0ac"}, + {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9084e68bedbd665c7e9241a7b597c28f4775edeb3941bf608ecb38732a5f8fb5"}, + {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d72493a14a3e89564b1a6c7400b9b40621e8f4692410706ef27c66aeadc7b431"}, + {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98779cbe9068dd7734cc3210693894d5cc9b156920e9c336f10fb99f46bebbd8"}, + {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:499f8a12767dd28b98ab6b7c7da7d294564e4c9024a2aaa5d0b0b98a8bef2f92"}, + {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4dabda7f42c548f98d8e07e390bda2953fc58302c0e07ded7b3fe0637e7ecd2f"}, + {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c416409646c793c46370f0f1859253302ee70aeda5278c2a0ca41462f8ec1244"}, + {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b8bd31aaad8a80a7302730676cec26bea3ef1fd9835875aa47fea073aca9fe05"}, + {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3af8e3bcd630f905efbdfe7a51b51fc1ca3c9dca8b155f841925f3ad41685d41"}, + {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d86b79c707fc7025d967af71db652429a06a8179175e45bd2e9f17b8af6f5949"}, + {file = "xxhash-3.3.0-cp310-cp310-win32.whl", hash = "sha256:98fe771f36ee9d3a1f5741424a956a2ba9651d9508a9f64a024b57f2cf796414"}, + {file = "xxhash-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:0a65131f7f731ecf7e3dd27f09d877aff3000a79a446caaa2c0d8d0ec0bc7186"}, + {file = "xxhash-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a9761e425e79d23797fa0bec2d781dbadb9fe5dcc2bf69030855f5e393c3bec8"}, + {file = "xxhash-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d28c7ef1deb3c3ac5f5290176ca3d501daa97c2e1f7443bf5d8b61ac651794b2"}, + {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:701b7cefffc25de1b7ddfae6505da70a3b3a11e312c2e2b33b09e180bbceb43d"}, + {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b1644f8b8e19a242c3047a089541067248a651038cabb9fcab3c13eb1dfcd757"}, + {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:20e7d0e3488cc0f0dbe360731b7fe32e1f2df46bf2de2db3317d301efb93084c"}, + {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:156c52eca2b20f9839723bef0b929a290f6c2f1c98ccb24e82f58f96f3c16007"}, + {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d6ce4d3828d79044ed08994e196c20f69c18133ed8a4286afe3e98989adeeac"}, + {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b85b63757ade2439c8d7d71842c40d42c0ab3b69279ed02afbd3b1635f7d2b4b"}, + {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b2b9051e40b7b649a9a2a38fb223ca6a593d332012df885746b81968948f9435"}, + {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:81b7ce050f26fc1daaaa0d24e320815306736d14608e1ba31920e693a7ca9afb"}, + {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:7442500fcce71669953ca959682dcd47452bc3f9c95c8d88315874aeabec9f82"}, + {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:36a05bf59a515cfb07f3f83373c527fff2ecaa77eaf30c968c788aea582070a1"}, + {file = "xxhash-3.3.0-cp311-cp311-win32.whl", hash = "sha256:da16f9cd62c6fde74683be1b28c28ef865e706da13e3bee4ba836fcc520de0cc"}, + {file = "xxhash-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:40fd49ef6964b1c90c0bea63cd184f6d0b36e59144a080e8b3ac2c4c06bf6bf2"}, + {file = "xxhash-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:672c60cce1f8026ae32c651f877aa64f342876083a36a4b1ff91bc876aaf0e34"}, + {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bb6c83d7a65dd3065566c77425ba72df96982174e8ef613d809052d68ae77ab"}, + {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a4170f3016b621e3200ebfcc18de6f50eb8e8fc1303e16324b1f5625afd51b57"}, + {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bfb9c45d502ab38c0f4edf98a678694ae0f345613ef4900ade98c71f64db4d78"}, + {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48af026a2b1569666da42a478248a1f03f4e2350a34eb661afe3cb45429ca1d7"}, + {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe627de8fe8ddfa8b6477bda4ae5d5843ad1a0c83601dcff72247039465cc901"}, + {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:427fc60a188e345534f35b0aa76f7640c5ddf0354f1c9ad826a2bc086282982d"}, + {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d80acb20c7f268fe3150ac0be6a6b798062af56a1795eef855b26c9eae11a99c"}, + {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e71100818943422d1fbbe460e7be7fc4f2d2ba9371b2a745eb09e29ef0493f4a"}, + {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:e3b9bb5fdbe284c7b61c5d82c76688e52bbaf48ab1e53de98c072cc696fa331f"}, + {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1e25f6c8c46cf1ed8237f610abb231093a748c97d6c2c092789a7cad7e7ef290"}, + {file = "xxhash-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:928208dfecc563be59ae91868d1658e78809cb1e6a0bd74960a96c915db6390c"}, + {file = "xxhash-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bd1b4531a66da6dde1974662c1fd6fb1a2f27e40542e3df5e5e5dbab8ea4aee7"}, + {file = "xxhash-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:deebb296df92e082b6d0171a7d6227b503e2897cea4f8bdd3d708094974d4cf6"}, + {file = "xxhash-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cd96e9cb0e2baa294e6d572207d9731c3bb8e2511f1ff70f2bf17266b4488bd9"}, + {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3756b44bf247e422a2e47a38f25d03cf4a5ed539fdc2be3c60043e872e6ff13d"}, + {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69550c3c053b8f135ceac97b85dc1b2bc54b7613a966f550f32b43bed81c788a"}, + {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fc8736fc3e0c5aad435520873b9d2e27ddcc5a830b07e00e9c4d3a61ded9675"}, + {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80ead7774392efbd95f9f701155048f9ca26cf55133db6f5bb5a0ec69376bda5"}, + {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8737c9b3fd944d856faafa92c95f6198649ad57987935b6d965d086938be917"}, + {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2c8e078d0b9f85212801c41bd9eec8122003929686b0ee33360ffbfdf1a189ab"}, + {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f399269d20ef1dd910331f9ad49e8510c3ba2aa657b623293b536038f266a5c5"}, + {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f3661decef5f9ff7ab50edbef463bf7dc717621b56755dbae5458a946a033b10"}, + {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5ec374d0f1e7d43ef48a4ff643600833d7a325ecc6933b4d6ad9282f55751cf7"}, + {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39a947ff02d9a85673f5ce1f6f34059e24c714a797440485bd81b2c3cb69a7ff"}, + {file = "xxhash-3.3.0-cp38-cp38-win32.whl", hash = "sha256:4a4f0645a0ec03b229fb04f2e66bdbcb1ffd341a70d6c86c3ee015ffdcd70fad"}, + {file = "xxhash-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:8af5a687c0fb4357c230eec8a57ca07d3172faa3cb69beb0cbad40672ae6fa4b"}, + {file = "xxhash-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e5bfafda019ecc6202af6f3cf08220fa66af9612ba16ef831033ae3ac7bd1f89"}, + {file = "xxhash-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d113b433bc817adf845689a051363777835577858263ec4325d1934fcb7e394"}, + {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56aacf4bf65f575c0392be958aceff719d850950bb6af7d804b32d4bc293159c"}, + {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0f5d3e4e0937dad05585e9bd772bbdf0ca40cd8b2f54789d7a1f3091b608118c"}, + {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23605d7fc67bc7daa0d263b3a26de3375cfcc0b51ab7de5026625415c05b6fed"}, + {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe525be0392d493558a2b10d764bcaae9850cc262b417176a8b001f16e085fc6"}, + {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b234d08786884f5c8d55dfebb839cfbd846d812e3a052c39ca7e8ce7055fed68"}, + {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b031395b4b9c3085d9ea1ce89896ab01a65fc63172b2bfda5dd318fefe5e2f93"}, + {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5afe44da46b48c75169e622a532dca3fe585343c0577cfd7c18ecd3f1200305d"}, + {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c59f233f38b6a49d5e4ddf16be910a5bbf36a2989b6b2c8591853fb9f5a5e691"}, + {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:ed016e278c5c4633270903c7cf3b9dfb0bd293b7335e43fe695cb95541da53c9"}, + {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7a8bd6612fb35487e9ab329bb37b3df44f58baf752010dde9282593edbfed7e7"}, + {file = "xxhash-3.3.0-cp39-cp39-win32.whl", hash = "sha256:015a0498bde85364abc53fcc713af962dd4555391929736d9c0ff2c555436a03"}, + {file = "xxhash-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:06a484097af32caf1cfffadd60c3ca140c9e52b40a551fb1f6f0fdfd6f7f8977"}, + {file = "xxhash-3.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6c3809740124bbc777d29e3ae53de24f4c13fd5e62878086a8feadf0dcb654a5"}, + {file = "xxhash-3.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae092f0daaeece2acdd6ec46e2ab307d8d6f22b01ecca14dc6078844dbd88339"}, + {file = "xxhash-3.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3498e72ff2610b049b97bb81d1ea6e7bfa5b7a45efb3f255d77ec2fa2bc91653"}, + {file = "xxhash-3.3.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0004dded9d86f129961326e980420187640fb7ba65a184009429861c1d09df7"}, + {file = "xxhash-3.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:41c8bfd27191928bae6fd2b66872965532267785094a03c0ee5f358d9dba51c2"}, + {file = "xxhash-3.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:71db8498e329cef3588b0617f762a3fe31d899872e76a68ce2840e35a1318a5b"}, + {file = "xxhash-3.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d1d24d71b6209bc0124286932c4f0660c1103cb996fe34cb374bc12ac251940"}, + {file = "xxhash-3.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61004587a09b5b385e43d95ffe3a76c9d934dfd79ea38272d5c20ddfba8eab8f"}, + {file = "xxhash-3.3.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f0c92e3fa826425c73acafb31e022a719c85423847a9433d3a9e61e4ac97543"}, + {file = "xxhash-3.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:367e03f1484ce471c94e731b98f5e4a05b43e7188b16692998e1cc89fd1159a5"}, + {file = "xxhash-3.3.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed04c47dfaab98fcda0b748af9ee6fe8c888a0a0fbd13720e0f0221671e387e1"}, + {file = "xxhash-3.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cbfde62516435ca198220aff048a8793383cb7047c7b88714a061968bca786d"}, + {file = "xxhash-3.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73682225faa973ee56743f0fcd36bfcbfec503be258e0e420fb34313f52f1e7b"}, + {file = "xxhash-3.3.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d49efdce2086c2c506af20ed18a1115b40af7aad6d4ee27cb31d7c810585a3f2"}, + {file = "xxhash-3.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:546a0bb8e5a657cadf0da290b30ccd561cb89c256a5421ab8d5eb12eaf087349"}, + {file = "xxhash-3.3.0.tar.gz", hash = "sha256:c3f9e322b1ebeebd44e3d9d2d9b124e0c550c1ef41bd552afdcdd719516ee41a"}, ] [[package]] @@ -2369,4 +2426,4 @@ quantize = ["accelerate", "datasets", "texttable"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "65fa01feaf4d0c939bfa7991803e4454d853f56652e4349151ea8dd9aa8feedf" +content-hash = "93fd0873b3e16c10b67a84216a84f5eb2f5067cb3ff8cb912446cc6a7fa9c030" diff --git a/server/pyproject.toml b/server/pyproject.toml index cc457b14..56afda0a 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -31,6 +31,7 @@ einops = "^0.6.1" texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } peft = "^0.4.0" +torch = {version = "^2.0.1+cu118", source = "pytorch-gpu-src"} [tool.poetry.extras] accelerate = ["accelerate"] @@ -41,6 +42,12 @@ quantize = ["texttable", "datasets", "accelerate"] grpcio-tools = "^1.51.1" pytest = "^7.3.0" + +[[tool.poetry.source]] +name = "pytorch-gpu-src" +url = "https://download.pytorch.org/whl/cu118" +priority = "explicit" + [tool.pytest.ini_options] markers = ["private: marks tests as requiring an admin hf token (deselect with '-m \"not private\"')"] diff --git a/server/requirements.txt b/server/requirements.txt index 4ffb80a1..f9414b25 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,27 +1,31 @@ +--extra-index-url https://download.pytorch.org/whl/cu118 + accelerate==0.19.0 ; python_version >= "3.9" and python_version < "4.0" backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" -certifi==2023.5.7 ; python_version >= "3.9" and python_version < "4.0" -charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0" -click==8.1.3 ; python_version >= "3.9" and python_version < "4.0" +certifi==2023.7.22 ; python_version >= "3.9" and python_version < "4.0" +charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "4.0" +click==8.1.6 ; python_version >= "3.9" and python_version < "4.0" +cmake==3.27.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows") deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0" einops==0.6.1 ; python_version >= "3.9" and python_version < "4.0" filelock==3.12.2 ; python_version >= "3.9" and python_version < "4.0" fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "4.0" -googleapis-common-protos==1.59.1 ; python_version >= "3.9" and python_version < "4.0" +googleapis-common-protos==1.60.0 ; python_version >= "3.9" and python_version < "4.0" grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "4.0" -grpcio-reflection==1.56.0 ; python_version >= "3.9" and python_version < "4.0" -grpcio-status==1.56.0 ; python_version >= "3.9" and python_version < "4.0" -grpcio==1.56.0 ; python_version >= "3.9" and python_version < "4.0" +grpcio-reflection==1.56.2 ; python_version >= "3.9" and python_version < "4.0" +grpcio-status==1.56.2 ; python_version >= "3.9" and python_version < "4.0" +grpcio==1.56.2 ; python_version >= "3.9" and python_version < "4.0" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" huggingface-hub==0.14.1 ; python_version >= "3.9" and python_version < "4.0" idna==3.4 ; python_version >= "3.9" and python_version < "4.0" jinja2==3.1.2 ; python_version >= "3.9" and python_version < "4.0" +lit==16.0.6 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" markupsafe==2.1.3 ; python_version >= "3.9" and python_version < "4.0" mpmath==1.3.0 ; python_version >= "3.9" and python_version < "4.0" networkx==3.1 ; python_version >= "3.9" and python_version < "4.0" -numpy==1.25.0 ; python_version >= "3.9" and python_version < "4.0" +numpy==1.25.2 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "4.0" @@ -33,9 +37,9 @@ opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" peft==0.4.0 ; python_version >= "3.9" and python_version < "4.0" -protobuf==4.23.3 ; python_version >= "3.9" and python_version < "4.0" +protobuf==4.23.4 ; python_version >= "3.9" and python_version < "4.0" psutil==5.9.5 ; python_version >= "3.9" and python_version < "4.0" -pyyaml==6.0 ; python_version >= "3.9" and python_version < "4.0" +pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "4.0" regex==2023.6.3 ; python_version >= "3.9" and python_version < "4.0" requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0" safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" @@ -43,11 +47,12 @@ sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "4.0" setuptools==68.0.0 ; python_version >= "3.9" and python_version < "4.0" sympy==1.12 ; python_version >= "3.9" and python_version < "4.0" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" -torch==2.0.1 ; python_version >= "3.9" and python_version < "4.0" +torch==2.0.1+cu118 ; python_version >= "3.9" and python_version < "4.0" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" transformers==4.29.2 ; python_version >= "3.9" and python_version < "4.0" +triton==2.0.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "4.0" -urllib3==2.0.3 ; python_version >= "3.9" and python_version < "4.0" +urllib3==2.0.4 ; python_version >= "3.9" and python_version < "4.0" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" From f91e9d282d73e09cdb876924412f2ed66212d736 Mon Sep 17 00:00:00 2001 From: zspo Date: Fri, 4 Aug 2023 04:21:33 +0800 Subject: [PATCH 363/793] fix build tokenizer in quantize and remove duplicate import (#768) # What does this PR do? Fixes #732 And remove duplicate AutoTokenizer import ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../utils/gptq/quantize.py | 81 ++++++++++++------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/server/text_generation_server/utils/gptq/quantize.py b/server/text_generation_server/utils/gptq/quantize.py index 3f8e897a..9547d534 100644 --- a/server/text_generation_server/utils/gptq/quantize.py +++ b/server/text_generation_server/utils/gptq/quantize.py @@ -360,15 +360,21 @@ def free(self): torch.cuda.empty_cache() -def get_wikitext2(nsamples, seed, seqlen, model_id): +def get_wikitext2(nsamples, seed, seqlen, model_id, trust_remote_code): from datasets import load_dataset traindata = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") testdata = load_dataset("wikitext", "wikitext-2-raw-v1", split="test") - from transformers import AutoTokenizer + try: + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=False, trust_remote_code=trust_remote_code + ) + except: + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=True, trust_remote_code=trust_remote_code + ) - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) trainenc = tokenizer("\n\n".join(traindata["text"]), return_tensors="pt") testenc = tokenizer("\n\n".join(testdata["text"]), return_tensors="pt") @@ -386,18 +392,21 @@ def get_wikitext2(nsamples, seed, seqlen, model_id): return trainloader, testenc -def get_ptb(nsamples, seed, seqlen, model_id): +def get_ptb(nsamples, seed, seqlen, model_id, trust_remote_code): from datasets import load_dataset traindata = load_dataset("ptb_text_only", "penn_treebank", split="train") valdata = load_dataset("ptb_text_only", "penn_treebank", split="validation") - from transformers import AutoTokenizer - try: - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=False, trust_remote_code=trust_remote_code + ) except: - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True) + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=True, trust_remote_code=trust_remote_code + ) + trainenc = tokenizer("\n\n".join(traindata["sentence"]), return_tensors="pt") testenc = tokenizer("\n\n".join(valdata["sentence"]), return_tensors="pt") @@ -415,7 +424,7 @@ def get_ptb(nsamples, seed, seqlen, model_id): return trainloader, testenc -def get_c4(nsamples, seed, seqlen, model_id): +def get_c4(nsamples, seed, seqlen, model_id, trust_remote_code): from datasets import load_dataset traindata = load_dataset( @@ -433,12 +442,14 @@ def get_c4(nsamples, seed, seqlen, model_id): use_auth_token=False, ) - from transformers import AutoTokenizer - try: - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=False, trust_remote_code=trust_remote_code + ) except: - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True) + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=True, trust_remote_code=trust_remote_code + ) import random @@ -481,18 +492,21 @@ def __init__(self, input_ids): return trainloader, valenc -def get_ptb_new(nsamples, seed, seqlen, model_id): +def get_ptb_new(nsamples, seed, seqlen, model_id, trust_remote_code): from datasets import load_dataset traindata = load_dataset("ptb_text_only", "penn_treebank", split="train") testdata = load_dataset("ptb_text_only", "penn_treebank", split="test") - from transformers import AutoTokenizer - try: - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=False, trust_remote_code=trust_remote_code + ) except: - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True) + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=True, trust_remote_code=trust_remote_code + ) + trainenc = tokenizer(" ".join(traindata["sentence"]), return_tensors="pt") testenc = tokenizer(" ".join(testdata["sentence"]), return_tensors="pt") @@ -510,7 +524,7 @@ def get_ptb_new(nsamples, seed, seqlen, model_id): return trainloader, testenc -def get_c4_new(nsamples, seed, seqlen, model_id): +def get_c4_new(nsamples, seed, seqlen, model_id, trust_remote_code): from datasets import load_dataset traindata = load_dataset( @@ -526,12 +540,14 @@ def get_c4_new(nsamples, seed, seqlen, model_id): split="validation", ) - from transformers import AutoTokenizer - try: - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=False) + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=False, trust_remote_code=trust_remote_code + ) except: - tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True) + tokenizer = AutoTokenizer.from_pretrained( + model_id, use_fast=True, trust_remote_code=trust_remote_code + ) import random @@ -562,17 +578,17 @@ def __init__(self, input_ids): return trainloader, valenc -def get_loaders(name, nsamples=128, seed=0, seqlen=2048, model_id=""): +def get_loaders(name, nsamples=128, seed=0, seqlen=2048, model_id="", trust_remote_code=False): if "wikitext2" in name: - return get_wikitext2(nsamples, seed, seqlen, model_id) + return get_wikitext2(nsamples, seed, seqlen, model_id, trust_remote_code) if "ptb" in name: if "new" in name: - return get_ptb_new(nsamples, seed, seqlen, model_id) - return get_ptb(nsamples, seed, seqlen, model_id) + return get_ptb_new(nsamples, seed, seqlen, model_id, trust_remote_code) + return get_ptb(nsamples, seed, seqlen, model_id, trust_remote_code) if "c4" in name: if "new" in name: - return get_c4_new(nsamples, seed, seqlen, model_id) - return get_c4(nsamples, seed, seqlen, model_id) + return get_c4_new(nsamples, seed, seqlen, model_id, trust_remote_code) + return get_c4(nsamples, seed, seqlen, model_id, trust_remote_code) def find_layers(module, layers=(nn.Conv2d, nn.Linear), name=""): @@ -906,7 +922,12 @@ def _unload(): seed = None dataloader, testloader = get_loaders( - dataset, nsamples=nsamples, seed=seed, model_id=model_id, seqlen=model.seqlen + dataset, + nsamples=nsamples, + seed=seed, + model_id=model_id, + seqlen=model.seqlen, + trust_remote_code=trust_remote_code ) tick = time.time() From 16fadcec5711ff232977b38c74a1c8829af6a63b Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 3 Aug 2023 23:00:59 +0200 Subject: [PATCH 364/793] Merge BNB 4bit. (#770) # What does this PR do? See #626 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: krzim --- README.md | 2 + launcher/src/main.rs | 11 +- server/poetry.lock | 48 +++++++- server/pyproject.toml | 5 +- server/requirements.txt | 113 +++++++++--------- server/text_generation_server/cli.py | 2 + .../text_generation_server/models/__init__.py | 5 +- server/text_generation_server/utils/layers.py | 47 +++++++- 8 files changed, 166 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 869cc668..0ba49675 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,8 @@ You can also quantize the weights with bitsandbytes to reduce the VRAM requireme make run-falcon-7b-instruct-quantize ``` +4bit quantization is available using the [NF4 and FP4 data types from bitsandbytes](https://arxiv.org/pdf/2305.14314.pdf). It can be enabled by providing `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` as a command line argument to `text-generation-launcher`. + ## Develop ```shell diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 146d83d6..75762712 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -22,6 +22,8 @@ mod env_runtime; #[derive(Clone, Copy, Debug, ValueEnum)] enum Quantization { Bitsandbytes, + BitsandbytesNF4, + BitsandbytesFP4, Gptq, } @@ -32,6 +34,12 @@ impl std::fmt::Display for Quantization { Quantization::Bitsandbytes => { write!(f, "bitsandbytes") } + Quantization::BitsandbytesNF4 => { + write!(f, "bitsandbytes-nf4") + } + Quantization::BitsandbytesFP4 => { + write!(f, "bitsandbytes-fp4") + } Quantization::Gptq => { write!(f, "gptq") } @@ -116,7 +124,8 @@ struct Args { num_shard: Option, /// Whether you want the model to be quantized. This will use `bitsandbytes` for - /// quantization on the fly, or `gptq`. + /// quantization on the fly, or `gptq`. 4bit quantization is available through + /// `bitsandbytes` by providing the `bitsandbytes-fp4` or `bitsandbytes-nf4` options. #[clap(long, env, value_enum)] quantize: Option, diff --git a/server/poetry.lock b/server/poetry.lock index 70f09f32..f447914a 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -192,13 +192,13 @@ files = [ [[package]] name = "bitsandbytes" -version = "0.38.1" -description = "8-bit optimizers and matrix multiplication routines." +version = "0.40.2" +description = "k-bit optimizers and matrix multiplication routines." optional = true python-versions = "*" files = [ - {file = "bitsandbytes-0.38.1-py3-none-any.whl", hash = "sha256:5f532e7b1353eb7049ae831da2eb62ed8a1e0444116bd51b9e088a6e0bc7a34a"}, - {file = "bitsandbytes-0.38.1.tar.gz", hash = "sha256:ba95a806b5065ea3263558e188f07eacb32ad691842932fb0d36a879883167ce"}, + {file = "bitsandbytes-0.40.2-py3-none-any.whl", hash = "sha256:f0ae26f40c9230c9add9e7c70a10a5ced36fb6deff39906aec1ce4fd25e6ddc0"}, + {file = "bitsandbytes-0.40.2.tar.gz", hash = "sha256:808ac966272c63bccb2be6d77365275a4c28f1fa348d33656e670de3cab40fc4"}, ] [[package]] @@ -1751,6 +1751,42 @@ tensorflow = ["tensorflow (>=2.11.0)"] testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] torch = ["torch (>=1.10)"] +[[package]] +name = "scipy" +version = "1.11.1" +description = "Fundamental algorithms for scientific computing in Python" +optional = false +python-versions = "<3.13,>=3.9" +files = [ + {file = "scipy-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:aec8c62fbe52914f9cf28d846cf0401dd80ab80788bbab909434eb336ed07c04"}, + {file = "scipy-1.11.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3b9963798df1d8a52db41a6fc0e6fa65b1c60e85d73da27ae8bb754de4792481"}, + {file = "scipy-1.11.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e8eb42db36526b130dfbc417609498a6192381abc1975b91e3eb238e0b41c1a"}, + {file = "scipy-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:366a6a937110d80dca4f63b3f5b00cc89d36f678b2d124a01067b154e692bab1"}, + {file = "scipy-1.11.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:08d957ca82d3535b3b9ba6c8ff355d78fe975271874e2af267cb5add5bd78625"}, + {file = "scipy-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:e866514bc2d660608447b6ba95c8900d591f2865c07cca0aa4f7ff3c4ca70f30"}, + {file = "scipy-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba94eeef3c9caa4cea7b402a35bb02a5714ee1ee77eb98aca1eed4543beb0f4c"}, + {file = "scipy-1.11.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:512fdc18c65f76dadaca139348e525646d440220d8d05f6d21965b8d4466bccd"}, + {file = "scipy-1.11.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cce154372f0ebe88556ed06d7b196e9c2e0c13080ecb58d0f35062dc7cc28b47"}, + {file = "scipy-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4bb943010203465ac81efa392e4645265077b4d9e99b66cf3ed33ae12254173"}, + {file = "scipy-1.11.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:249cfa465c379c9bb2c20123001e151ff5e29b351cbb7f9c91587260602c58d0"}, + {file = "scipy-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:ffb28e3fa31b9c376d0fb1f74c1f13911c8c154a760312fbee87a21eb21efe31"}, + {file = "scipy-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:39154437654260a52871dfde852adf1b93b1d1bc5dc0ffa70068f16ec0be2624"}, + {file = "scipy-1.11.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:b588311875c58d1acd4ef17c983b9f1ab5391755a47c3d70b6bd503a45bfaf71"}, + {file = "scipy-1.11.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d51565560565a0307ed06fa0ec4c6f21ff094947d4844d6068ed04400c72d0c3"}, + {file = "scipy-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b41a0f322b4eb51b078cb3441e950ad661ede490c3aca66edef66f4b37ab1877"}, + {file = "scipy-1.11.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:396fae3f8c12ad14c5f3eb40499fd06a6fef8393a6baa352a652ecd51e74e029"}, + {file = "scipy-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:be8c962a821957fdde8c4044efdab7a140c13294997a407eaee777acf63cbf0c"}, + {file = "scipy-1.11.1.tar.gz", hash = "sha256:fb5b492fa035334fd249f0973cc79ecad8b09c604b42a127a677b45a9a3d4289"}, +] + +[package.dependencies] +numpy = ">=1.21.6,<1.28.0" + +[package.extras] +dev = ["click", "cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pydevtool", "rich-click", "ruff", "types-psutil", "typing_extensions"] +doc = ["jupytext", "matplotlib (>2)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-design (>=0.2.0)"] +test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + [[package]] name = "sentencepiece" version = "0.1.99" @@ -2425,5 +2461,5 @@ quantize = ["accelerate", "datasets", "texttable"] [metadata] lock-version = "2.0" -python-versions = "^3.9" -content-hash = "93fd0873b3e16c10b67a84216a84f5eb2f5067cb3ff8cb912446cc6a7fa9c030" +python-versions = ">=3.9,<3.13" +content-hash = "2abb80833b678452cfc73464fc5b2e48d74b2672bd987240041a33c724a74000" diff --git a/server/pyproject.toml b/server/pyproject.toml index 56afda0a..88b93f4e 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -8,7 +8,7 @@ authors = ["Olivier Dehaene "] text-generation-server = 'text_generation_server.cli:app' [tool.poetry.dependencies] -python = "^3.9" +python = ">=3.9,<3.13" protobuf = "^4.21.7" grpcio = "^1.51.1" grpcio-status = "^1.51.1" @@ -16,7 +16,7 @@ grpcio-reflection = "^1.51.1" grpc-interceptor = "^0.15.0" typer = "^0.6.1" accelerate = { version = "^0.19.0", optional = true } -bitsandbytes = { version = "^0.38.1", optional = true } +bitsandbytes = { version = "^0.40.0", optional = true } safetensors = "0.3.1" loguru = "^0.6.0" opentelemetry-api = "^1.15.0" @@ -32,6 +32,7 @@ texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } peft = "^0.4.0" torch = {version = "^2.0.1+cu118", source = "pytorch-gpu-src"} +scipy = "^1.11.1" [tool.poetry.extras] accelerate = ["accelerate"] diff --git a/server/requirements.txt b/server/requirements.txt index f9414b25..beab4bf6 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,58 +1,59 @@ --extra-index-url https://download.pytorch.org/whl/cu118 -accelerate==0.19.0 ; python_version >= "3.9" and python_version < "4.0" -backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0" -certifi==2023.7.22 ; python_version >= "3.9" and python_version < "4.0" -charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "4.0" -click==8.1.6 ; python_version >= "3.9" and python_version < "4.0" -cmake==3.27.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" -colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows") -deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0" -einops==0.6.1 ; python_version >= "3.9" and python_version < "4.0" -filelock==3.12.2 ; python_version >= "3.9" and python_version < "4.0" -fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "4.0" -googleapis-common-protos==1.60.0 ; python_version >= "3.9" and python_version < "4.0" -grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "4.0" -grpcio-reflection==1.56.2 ; python_version >= "3.9" and python_version < "4.0" -grpcio-status==1.56.2 ; python_version >= "3.9" and python_version < "4.0" -grpcio==1.56.2 ; python_version >= "3.9" and python_version < "4.0" -hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "4.0" -huggingface-hub==0.14.1 ; python_version >= "3.9" and python_version < "4.0" -idna==3.4 ; python_version >= "3.9" and python_version < "4.0" -jinja2==3.1.2 ; python_version >= "3.9" and python_version < "4.0" -lit==16.0.6 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" -loguru==0.6.0 ; python_version >= "3.9" and python_version < "4.0" -markupsafe==2.1.3 ; python_version >= "3.9" and python_version < "4.0" -mpmath==1.3.0 ; python_version >= "3.9" and python_version < "4.0" -networkx==3.1 ; python_version >= "3.9" and python_version < "4.0" -numpy==1.25.2 ; python_version >= "3.9" and python_version < "4.0" -opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "4.0" -opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "4.0" -opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "4.0" -opentelemetry-exporter-otlp==1.15.0 ; python_version >= "3.9" and python_version < "4.0" -opentelemetry-instrumentation-grpc==0.36b0 ; python_version >= "3.9" and python_version < "4.0" -opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_version < "4.0" -opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "4.0" -opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "4.0" -opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "4.0" -packaging==23.1 ; python_version >= "3.9" and python_version < "4.0" -peft==0.4.0 ; python_version >= "3.9" and python_version < "4.0" -protobuf==4.23.4 ; python_version >= "3.9" and python_version < "4.0" -psutil==5.9.5 ; python_version >= "3.9" and python_version < "4.0" -pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "4.0" -regex==2023.6.3 ; python_version >= "3.9" and python_version < "4.0" -requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0" -safetensors==0.3.1 ; python_version >= "3.9" and python_version < "4.0" -sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "4.0" -setuptools==68.0.0 ; python_version >= "3.9" and python_version < "4.0" -sympy==1.12 ; python_version >= "3.9" and python_version < "4.0" -tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "4.0" -torch==2.0.1+cu118 ; python_version >= "3.9" and python_version < "4.0" -tqdm==4.65.0 ; python_version >= "3.9" and python_version < "4.0" -transformers==4.29.2 ; python_version >= "3.9" and python_version < "4.0" -triton==2.0.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "4.0" -typer==0.6.1 ; python_version >= "3.9" and python_version < "4.0" -typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "4.0" -urllib3==2.0.4 ; python_version >= "3.9" and python_version < "4.0" -win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "win32" -wrapt==1.15.0 ; python_version >= "3.9" and python_version < "4.0" +accelerate==0.19.0 ; python_version >= "3.9" and python_version < "3.13" +backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13" +certifi==2023.7.22 ; python_version >= "3.9" and python_version < "3.13" +charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "3.13" +click==8.1.6 ; python_version >= "3.9" and python_version < "3.13" +cmake==3.27.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "3.13" +colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows") +deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13" +einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13" +filelock==3.12.2 ; python_version >= "3.9" and python_version < "3.13" +fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "3.13" +googleapis-common-protos==1.60.0 ; python_version >= "3.9" and python_version < "3.13" +grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "3.13" +grpcio-reflection==1.56.2 ; python_version >= "3.9" and python_version < "3.13" +grpcio-status==1.56.2 ; python_version >= "3.9" and python_version < "3.13" +grpcio==1.56.2 ; python_version >= "3.9" and python_version < "3.13" +hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "3.13" +huggingface-hub==0.14.1 ; python_version >= "3.9" and python_version < "3.13" +idna==3.4 ; python_version >= "3.9" and python_version < "3.13" +jinja2==3.1.2 ; python_version >= "3.9" and python_version < "3.13" +lit==16.0.6 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "3.13" +loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" +markupsafe==2.1.3 ; python_version >= "3.9" and python_version < "3.13" +mpmath==1.3.0 ; python_version >= "3.9" and python_version < "3.13" +networkx==3.1 ; python_version >= "3.9" and python_version < "3.13" +numpy==1.25.2 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-exporter-otlp==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-instrumentation-grpc==0.36b0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13" +packaging==23.1 ; python_version >= "3.9" and python_version < "3.13" +peft==0.4.0 ; python_version >= "3.9" and python_version < "3.13" +protobuf==4.23.4 ; python_version >= "3.9" and python_version < "3.13" +psutil==5.9.5 ; python_version >= "3.9" and python_version < "3.13" +pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" +regex==2023.6.3 ; python_version >= "3.9" and python_version < "3.13" +requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" +safetensors==0.3.1 ; python_version >= "3.9" and python_version < "3.13" +scipy==1.11.1 ; python_version >= "3.9" and python_version < "3.13" +sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" +setuptools==68.0.0 ; python_version >= "3.9" and python_version < "3.13" +sympy==1.12 ; python_version >= "3.9" and python_version < "3.13" +tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" +torch==2.0.1+cu118 ; python_version >= "3.9" and python_version < "3.13" +tqdm==4.65.0 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.29.2 ; python_version >= "3.9" and python_version < "3.13" +triton==2.0.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "3.13" +typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" +typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "3.13" +urllib3==2.0.4 ; python_version >= "3.9" and python_version < "3.13" +win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32" +wrapt==1.15.0 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index eba807bc..459ba8c4 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -14,6 +14,8 @@ class Quantization(str, Enum): bitsandbytes = "bitsandbytes" + bitsandbytes_nf4 = "bitsandbytes-nf4" + bitsandbytes_fp4 = "bitsandbytes-fp4" gptq = "gptq" diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index e9260eed..71efcab7 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -255,7 +255,10 @@ def get_model( raise ValueError( "gptq quantization is not supported for AutoModel, you can try to quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" ) - + elif (quantize == "bitsandbytes-fp4") or (quantize == "bitsandbytes-nf4"): + raise ValueError( + "4bit quantization is not supported for AutoModel" + ) if model_type in modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: return CausalLM( model_id, diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index d7b4c0cc..97257f95 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -9,7 +9,7 @@ HAS_BITS_AND_BYTES = True try: import bitsandbytes as bnb - from bitsandbytes.nn import Int8Params + from bitsandbytes.nn import Int8Params, Params4bit except ImportError: HAS_BITS_AND_BYTES = False @@ -140,6 +140,39 @@ def forward(self, x: torch.Tensor): return out +class Linear4bit(nn.Module): + def __init__(self, weight, bias, quant_type): + super().__init__() + self.weight = Params4bit( + weight.data, requires_grad=False, compress_statistics=True, quant_type=quant_type + ) + self.compute_dtype = None + self.weight.cuda(weight.device) + self.bias = bias + + def forward(self, x: torch.Tensor): + # weights are cast automatically as Int8Params, but the bias has to be cast manually + if self.bias is not None and self.bias.dtype != x.dtype: + self.bias.data = self.bias.data.to(x.dtype) + + if getattr(self.weight, "quant_state", None) is None: + print( + "FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first." + ) + inp_dtype = x.dtype + if self.compute_dtype is not None: + x = x.to(self.compute_dtype) + + bias = None if self.bias is None else self.bias.to(self.compute_dtype) + out = bnb.matmul_4bit( + x, self.weight.t(), bias=bias, quant_state=self.weight.quant_state + ) + + out = out.to(inp_dtype) + + return out + + def get_linear(weight, bias, quantize): if quantize is None: linear = FastLinear(weight, bias) @@ -152,6 +185,18 @@ def get_linear(weight, bias, quantize): ) if bias is not None: linear.bias = nn.Parameter(bias) + elif quantize == "bitsandbytes-fp4": + linear = Linear4bit( + weight, + bias, + quant_type="fp4", + ) + elif quantize == "bitsandbytes-nf4": + linear = Linear4bit( + weight, + bias, + quant_type="nf4", + ) elif quantize == "gptq": try: qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama = weight From 891e19cc5126a53375a1908312da4e7cedd763e0 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 7 Aug 2023 12:28:19 +0200 Subject: [PATCH 365/793] Fix dynamic rope. (#783) # What does this PR do? Typo Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 97257f95..c2a340b2 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -543,7 +543,7 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): or self._cos_cached.dtype != dtype ): if seqlen > self.max_position_embeddings: - newbase = self.base * ((self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)) ** (self.dim / (self.dim - 2)) + newbase = self.base * ((self.scaling_factor * seqlen / self.max_position_embeddings) - (self.scaling_factor - 1)) ** (self.dim / (self.dim - 2)) self.inv_freq = _create_inv_freq(self.dim, newbase, self.inv_freq.device) self._seq_len_cached = seqlen t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) From 1fdc88ee908beb8ae0afe17810a17b9b4d8848e2 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 7 Aug 2023 13:02:00 +0200 Subject: [PATCH 366/793] Fixing non 4bits quantization. (#785) # What does this PR do? Fixes # (issue) Fixes #784 ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/gptq/quant_linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/gptq/quant_linear.py b/server/text_generation_server/utils/gptq/quant_linear.py index 54fa2014..bfc91c00 100644 --- a/server/text_generation_server/utils/gptq/quant_linear.py +++ b/server/text_generation_server/utils/gptq/quant_linear.py @@ -263,7 +263,7 @@ def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): self.groupsize = groupsize self.outfeatures = qweight.shape[1] - self.infeatures = qweight.shape[0] * 32 // 4 + self.infeatures = qweight.shape[0] * 32 // bits @classmethod def new(cls, bits, groupsize, infeatures, outfeatures, bias): From c4dac9f3dceceb0e6a54de4e797c1ccdb00ed8a4 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 8 Aug 2023 12:09:51 +0200 Subject: [PATCH 367/793] Update __init__.py (#794) # What does this PR do? Fixes #787 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 71efcab7..621652e8 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -89,7 +89,7 @@ def get_model( revision, quantize=quantize, dtype=dtype, - dtypetrust_remote_code=trust_remote_code, + trust_remote_code=trust_remote_code, ) if model_id.startswith("bigcode/"): From 0e8b47811e711b46fe80e6b4f5304186f83744d6 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 8 Aug 2023 13:43:40 +0200 Subject: [PATCH 368/793] Llama change. (#793) # What does this PR do? Reflecting https://github.com/huggingface/transformers/pull/24998 Current status wants to make sure integration tests *are* broken with this. Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../models/custom_modeling/flash_llama_modeling.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 2c22ea46..d0185ede 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -185,8 +185,11 @@ def __init__( self.hidden_size = config.hidden_size self.head_size = self.hidden_size // self.num_heads - self.rotary_emb = PositionRotaryEmbedding.load( - config=config, prefix=f"{prefix}.rotary_emb", weights=weights + # self.rotary_emb = PositionRotaryEmbedding.load( + # config=config, prefix=f"{prefix}.rotary_emb", weights=weights + # ) + self.rotary_emb = PositionRotaryEmbedding.static( + config=config, dim=self.head_size, base=10000.0, device=weights.device ) self.softmax_scale = self.head_size**-0.5 From 647ae7a7d32ae64e6b64956d83e7c5a96e854283 Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Thu, 10 Aug 2023 11:24:52 +0300 Subject: [PATCH 369/793] Setup for doc-builder and docs for TGI (#740) I added ToC for docs v1 & started setting up for doc-builder. cc @Narsil @osanseviero --------- Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> Co-authored-by: osanseviero Co-authored-by: Mishig --- .github/workflows/build_documentation.yml | 19 +++++ .github/workflows/build_pr_documentation.yml | 17 +++++ .github/workflows/delete_doc_comment.yml | 12 ++++ .github/workflows/upload_pr_documentation.yml | 16 +++++ docs/source/_toctree.yml | 18 +++++ docs/source/basic_tutorials/consuming_tgi.md | 57 +++++++++++++++ .../basic_tutorials/gated_model_access.md | 5 ++ .../source/basic_tutorials/preparing_model.md | 22 ++++++ docs/source/index.md | 27 +++++++ docs/source/installation.md | 71 +++++++++++++++++++ docs/source/quicktour.md | 34 +++++++++ docs/source/supported_models.md | 40 +++++++++++ 12 files changed, 338 insertions(+) create mode 100644 .github/workflows/build_documentation.yml create mode 100644 .github/workflows/build_pr_documentation.yml create mode 100644 .github/workflows/delete_doc_comment.yml create mode 100644 .github/workflows/upload_pr_documentation.yml create mode 100644 docs/source/_toctree.yml create mode 100644 docs/source/basic_tutorials/consuming_tgi.md create mode 100644 docs/source/basic_tutorials/gated_model_access.md create mode 100644 docs/source/basic_tutorials/preparing_model.md create mode 100644 docs/source/index.md create mode 100644 docs/source/installation.md create mode 100644 docs/source/quicktour.md create mode 100644 docs/source/supported_models.md diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml new file mode 100644 index 00000000..9d130ebb --- /dev/null +++ b/.github/workflows/build_documentation.yml @@ -0,0 +1,19 @@ +name: Build documentation + +on: + push: + branches: + - main + - doc-builder* + - v*-release + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main + with: + commit_sha: ${{ github.sha }} + package: text-generation-inference + additional_args: --not_python_module + secrets: + token: ${{ secrets.HUGGINGFACE_PUSH }} + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} \ No newline at end of file diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml new file mode 100644 index 00000000..e608c74a --- /dev/null +++ b/.github/workflows/build_pr_documentation.yml @@ -0,0 +1,17 @@ +name: Build PR Documentation + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main + with: + commit_sha: ${{ github.event.pull_request.head.sha }} + pr_number: ${{ github.event.number }} + package: text-generation-inference + additional_args: --not_python_module diff --git a/.github/workflows/delete_doc_comment.yml b/.github/workflows/delete_doc_comment.yml new file mode 100644 index 00000000..1cad807b --- /dev/null +++ b/.github/workflows/delete_doc_comment.yml @@ -0,0 +1,12 @@ +name: Delete doc comment + +on: + pull_request: + types: [ closed ] + + +jobs: + delete: + uses: huggingface/doc-builder/.github/workflows/delete_doc_comment_trigger.yml@main + with: + pr_number: ${{ github.event.number }} \ No newline at end of file diff --git a/.github/workflows/upload_pr_documentation.yml b/.github/workflows/upload_pr_documentation.yml new file mode 100644 index 00000000..b984ead2 --- /dev/null +++ b/.github/workflows/upload_pr_documentation.yml @@ -0,0 +1,16 @@ +name: Upload PR Documentation + +on: + workflow_run: + workflows: ["Build PR Documentation"] + types: + - completed + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main + with: + package_name: text-generation-inference + secrets: + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} + comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }} \ No newline at end of file diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml new file mode 100644 index 00000000..8ee20eb0 --- /dev/null +++ b/docs/source/_toctree.yml @@ -0,0 +1,18 @@ +- sections: + - local: index + title: Text Generation Inference + - local: quicktour + title: Quick Tour + - local: installation + title: Installation + - local: supported_models + title: Supported Models and Hardware + title: Getting started +- sections: + - local: basic_tutorials/consuming_tgi + title: Consuming TGI + - local: basic_tutorials/preparing_model + title: Preparing Model for Serving + - local: basic_tutorials/gated_model_access + title: Serving Private & Gated Models + title: Tutorials diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md new file mode 100644 index 00000000..619e0a31 --- /dev/null +++ b/docs/source/basic_tutorials/consuming_tgi.md @@ -0,0 +1,57 @@ +# Consuming Text Generation Inference + +There are many ways you can consume Text Generation Inference server in your applications. After launching, you can use the `/generate` route and make a `POST` request to get results from the server. You can also use the `/generate_stream` route if you want TGI to return a stram of tokens. You can make the requests using the tool of your preference, such as curl, Python or TypeScrpt. For a final end-to-end experience, we also open-sourced ChatUI, a chat interface for open-source models. + +## curl + +After the launch, you can query the model using either the `/generate` or `/generate_stream` routes: + +```shell +curl 127.0.0.1:8080/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ + -H 'Content-Type: application/json' +``` + + +## Inference Client + +[`huggingface-hub`](https://huggingface.co/docs/huggingface_hub/main/en/index) is a Python library to interact with the Hugging Face Hub, including its endpoints. It provides a nice high-level class, [`~huggingface_hub.InferenceClient`], which makes it easy to make calls to a TGI endpoint. `InferenceClient` also takes care of parameter validation and provides a simple to-use interface. + +You can simply install `huggingface-hub` package with pip. + +```python +pip install huggingface-hub +``` + +Once you start the TGI server, instantiate `InferenceClient()` with the URL to the endpoint serving the model. You can then call `text_generation()` to hit the endpoint through Python. + +```python + +from huggingface_hub import InferenceClient + +client = InferenceClient(model=URL_TO_ENDPOINT_SERVING_TGI) +client.text_generation(prompt="Write a code for snake game", model=URL_TO_ENDPOINT_SERVING_TGI) +``` + +You can check out the details of the function [here](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation). + + +## ChatUI + +ChatUI is an open-source interface built for LLM serving. It offers many customization options, such as web search with SERP API and more. ChatUI can automatically consume the TGI server and even provides an option to switch between different TGI endpoints. You can try it out at [Hugging Chat](https://huggingface.co/chat/), or use the [ChatUI Docker Space](https://huggingface.co/new-space?template=huggingchat/chat-ui-template) to deploy your own Hugging Chat to Spaces. + +To serve both ChatUI and TGI in same environment, simply add your own endpoints to the `MODELS` variable in `.env.local` file inside the `chat-ui` repository. Provide the endpoints pointing to where TGI is served. + +``` +{ +// rest of the model config here +"endpoints": [{"url": "https://HOST:PORT/generate_stream"}] +} +``` + +TODO: Add screenshot + +## API documentation + +You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. The Swagger UI is also available [here](https://huggingface.github.io/text-generation-inference). diff --git a/docs/source/basic_tutorials/gated_model_access.md b/docs/source/basic_tutorials/gated_model_access.md new file mode 100644 index 00000000..f5858dc4 --- /dev/null +++ b/docs/source/basic_tutorials/gated_model_access.md @@ -0,0 +1,5 @@ +# Serving Private & Gated Models + +If the model you wish to serve is behind gated access or the model repository on Hugging Face Hub is private, and you have access to the model, you can provide your Hugging Face Hub access token. You can generate and copy a read token from [Hugging Face Hub tokens page](https://huggingface.co/settings/tokens) + +If you're using the CLI, set the `HUGGING_FACE_HUB_TOKEN` environment variable. diff --git a/docs/source/basic_tutorials/preparing_model.md b/docs/source/basic_tutorials/preparing_model.md new file mode 100644 index 00000000..65a2a197 --- /dev/null +++ b/docs/source/basic_tutorials/preparing_model.md @@ -0,0 +1,22 @@ +# Preparing the Model + +Text Generation Inference improves the model in several aspects. + +## Quantization + +TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes) and [GPT-Q](https://arxiv.org/abs/2210.17323) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes` or `gptq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq). + + +## RoPE Scaling + +RoPE scaling can be used to increase the sequence length of the model during the inference time without necessarily fine-tuning it. To enable RoPE scaling, simply pass `--rope-scaling`, `--max-input-length` and `--rope-factors` flags when running through CLI. `--rope-scaling` can take the values `linear` or `dynamic`. If your model is not fine-tuned to a longer sequence length, use `dynamic`. `--rope-factor` is the ratio between the intended max sequence length and the model's original max sequence length. Make sure to pass `--max-input-length` to provide maximum input length for extension. + + + +We recommend using `dynamic` RoPE scaling. + + + +## Safetensors + +[Safetensors](https://github.com/huggingface/safetensors) is a fast and safe persistence format for deep learning models, and is required for tensor parallelism. TGI supports `safetensors` model loading under the hood. By default, given a repository with `safetensors` and `pytorch` weights, TGI will always load `safetensors`. If there's no `pytorch` weights, TGI will convert the weights to `safetensors` format. diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 00000000..097217ad --- /dev/null +++ b/docs/source/index.md @@ -0,0 +1,27 @@ +# Text Generation Inference + +Text Generation Inference (TGI) is a toolkit for deploying and serving Large Language Models (LLMs). TGI enables high-performance text generation for the most popular open-source LLMs, including Llama, Falcon, StarCoder, BLOOM, GPT-NeoX, and T5. + +![Text Generation Inference](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/TGI.png) + +Text Generation Inference implements many optimizations and features, such as: + +- Simple launcher to serve most popular LLMs +- Production ready (distributed tracing with Open Telemetry, Prometheus metrics) +- Tensor Parallelism for faster inference on multiple GPUs +- Token streaming using Server-Sent Events (SSE) +- Continuous batching of incoming requests for increased total throughput +- Optimized transformers code for inference using [Flash Attention](https://github.com/HazyResearch/flash-attention) and [Paged Attention](https://github.com/vllm-project/vllm) on the most popular architectures +- Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) and [GPT-Q](https://arxiv.org/abs/2210.17323) +- [Safetensors](https://github.com/huggingface/safetensors) weight loading +- Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) +- Logits warper (temperature scaling, top-p, top-k, repetition penalty) +- Stop sequences +- Log probabilities + + +Text Generation Inference is used in production by multiple projects, such as: + +- [Hugging Chat](https://github.com/huggingface/chat-ui), an open-source interface for open-access models, such as Open Assistant and Llama +- [OpenAssistant](https://open-assistant.io/), an open-source community effort to train LLMs in the open +- [nat.dev](http://nat.dev/), a playground to explore and compare LLMs. diff --git a/docs/source/installation.md b/docs/source/installation.md new file mode 100644 index 00000000..4105acf4 --- /dev/null +++ b/docs/source/installation.md @@ -0,0 +1,71 @@ +# Installation + +This section explains how to install the CLI tool as well as installing TGI from source. **The strongly recommended approach is to use Docker, as it does not require much setup. Check [the Quick Tour](./quicktour) to learn how to run TGI with Docker.** + +## Install CLI + +TODO + + +## Local Installation from Source + +Before you start, you will need to setup your environment, and install Text Generation Inference. Text Generation Inference is tested on **Python 3.9+**. + +Text Generation Inference is available on pypi, conda and GitHub. + +To install and launch locally, first [install Rust](https://rustup.rs/) and create a Python virtual environment with at least +Python 3.9, e.g. using conda: + +```shell +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +conda create -n text-generation-inference python=3.9 +conda activate text-generation-inference +``` + +You may also need to install Protoc. + +On Linux: + +```shell +PROTOC_ZIP=protoc-21.12-linux-x86_64.zip +curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP +sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc +sudo unzip -o $PROTOC_ZIP -d /usr/local 'include/*' +rm -f $PROTOC_ZIP +``` + +On MacOS, using Homebrew: + +```shell +brew install protobuf +``` + +Then run to install Text Generation Inference: + +```shell +BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork with CUDA kernels +``` + + + +On some machines, you may also need the OpenSSL libraries and gcc. On Linux machines, run: + +```shell +sudo apt-get install libssl-dev gcc -y +``` + + + +Once installation is done, simply run: + +```shell +make run-falcon-7b-instruct +``` + +This will serve Falcon 7B Instruct model from the port 8080, which we can query. + +To see all options to serve your models, check in the [codebase](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or the CLI: +``` +text-generation-launcher --help +``` diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md new file mode 100644 index 00000000..31185a2d --- /dev/null +++ b/docs/source/quicktour.md @@ -0,0 +1,34 @@ +# Quick Tour + +The easiest way of getting started is using the official Docker container. Install Docker following [their installation instructions](https://docs.docker.com/get-docker/). + +Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/falcon-7b-instruct) model with TGI. Here is an example on how to do that: + +```shell +model=tiiuae/falcon-7b-instruct +volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run + +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.0 --model-id $model +``` + + + +To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) . We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. + + + +Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section. + +```shell +curl 127.0.0.1:8080/generate -X POST -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' -H 'Content-Type: application/json' +``` + + + +To see all possible flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. + +```shell +docker run ghcr.io/huggingface/text-generation-inference:1.0.0 --help +``` + + \ No newline at end of file diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md new file mode 100644 index 00000000..d997c8d8 --- /dev/null +++ b/docs/source/supported_models.md @@ -0,0 +1,40 @@ +# Supported Models and Hardware + +Text Generation Inference enables serving optimized models on specific hardware for the highest performance. The following sections list which models are hardware are supported. + +## Supported Models + +The following models are optimized and can be served with TGI, which uses custom CUDA kernels for better inference. You can add the flag `--disable-custom-kernels` at the end of the `docker run` command if you wish to disable them. + +- [BLOOM](https://huggingface.co/bigscience/bloom) +- [FLAN-T5](https://huggingface.co/google/flan-t5-xxl) +- [Galactica](https://huggingface.co/facebook/galactica-120b) +- [GPT-Neox](https://huggingface.co/EleutherAI/gpt-neox-20b) +- [Llama](https://github.com/facebookresearch/llama) +- [OPT](https://huggingface.co/facebook/opt-66b) +- [SantaCoder](https://huggingface.co/bigcode/santacoder) +- [Starcoder](https://huggingface.co/bigcode/starcoder) +- [Falcon 7B](https://huggingface.co/tiiuae/falcon-7b) +- [Falcon 40B](https://huggingface.co/tiiuae/falcon-40b) +- [MPT](https://huggingface.co/mosaicml/mpt-30b) +- [Llama V2](https://huggingface.co/meta-llama) + +If the above list lacks the model you would like to serve, depending on the model's pipeline type, you can try to initialize and serve the model anyways to see how well it performs, but performance isn't guaranteed for non-optimized models: + +```python +# for causal LMs/text-generation models +AutoModelForCausalLM.from_pretrained(, device_map="auto")` +# or, for text-to-text generation models +AutoModelForSeq2SeqLM.from_pretrained(, device_map="auto") +``` + + +## Supported Hardware + +TGI optimized models are supported on NVIDIA [A100](https://www.nvidia.com/en-us/data-center/a100/), [A10G](https://www.nvidia.com/en-us/data-center/products/a10-gpu/) and [T4](https://www.nvidia.com/en-us/data-center/tesla-t4/) GPUs with CUDA 11.8+. Note that you have to install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) to use it. For other hardware, continuous batching will still apply, but some operations like flash attention and paged attention will not be executed. + +TGI is also supported on the following AI hardware accelerators: +- *Habana first-gen Gaudi and Gaudi2:* check out this [example](https://github.com/huggingface/optimum-habana/tree/main/text-generation-inference) how to serve models with TGI on Gaudi and Gaudi2 with [Optimum Habana](https://huggingface.co/docs/optimum/habana/index) + + + From 8bdb16ee9a658b33cc9def5d3062c63c9865163d Mon Sep 17 00:00:00 2001 From: ivarflakstad <69173633+ivarflakstad@users.noreply.github.com> Date: Thu, 10 Aug 2023 10:52:50 +0200 Subject: [PATCH 370/793] Use destructuring in router arguments to avoid '.0' (#798) # What does this PR do? This is purely code style - not anything important. Instead of writing `req.0` all over we can use [descructuring](https://doc.rust-lang.org/rust-by-example/flow_control/match/destructuring/destructure_structures.html) to access the contained value that we actually want. (Destructuring in function parameters [here](https://doc.rust-lang.org/reference/items/functions.html#function-parameters)) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @OlivierDehaene --- router/src/server.rs | 54 +++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/router/src/server.rs b/router/src/server.rs index 9af94951..e609821c 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -54,15 +54,13 @@ example = json ! ({"error": "Incomplete generation"})), )] #[instrument(skip(infer, req))] async fn compat_generate( - default_return_full_text: Extension, + Extension(default_return_full_text): Extension, infer: Extension, - req: Json, + Json(mut req): Json, ) -> Result)> { - let mut req = req.0; - // default return_full_text given the pipeline_tag if req.parameters.return_full_text.is_none() { - req.parameters.return_full_text = Some(default_return_full_text.0) + req.parameters.return_full_text = Some(default_return_full_text) } // switch on stream @@ -71,9 +69,9 @@ async fn compat_generate( .await .into_response()) } else { - let (headers, generation) = generate(infer, Json(req.into())).await?; + let (headers, Json(generation)) = generate(infer, Json(req.into())).await?; // wrap generation inside a Vec to match api-inference - Ok((headers, Json(vec![generation.0])).into_response()) + Ok((headers, Json(vec![generation])).into_response()) } } @@ -135,7 +133,7 @@ example = json ! ({"error": "Incomplete generation"})), #[instrument( skip_all, fields( -parameters = ? req.0.parameters, +parameters = ? req.parameters, total_time, validation_time, queue_time, @@ -146,29 +144,29 @@ seed, )] async fn generate( infer: Extension, - req: Json, + Json(req): Json, ) -> Result<(HeaderMap, Json), (StatusCode, Json)> { let span = tracing::Span::current(); let start_time = Instant::now(); metrics::increment_counter!("tgi_request_count"); - tracing::debug!("Input: {}", req.0.inputs); + tracing::debug!("Input: {}", req.inputs); - let compute_characters = req.0.inputs.chars().count(); + let compute_characters = req.inputs.chars().count(); let mut add_prompt = None; - if req.0.parameters.return_full_text.unwrap_or(false) { - add_prompt = Some(req.0.inputs.clone()); + if req.parameters.return_full_text.unwrap_or(false) { + add_prompt = Some(req.inputs.clone()); } - let details = req.0.parameters.details || req.0.parameters.decoder_input_details; + let details = req.parameters.details || req.parameters.decoder_input_details; // Inference - let (response, best_of_responses) = match req.0.parameters.best_of { + let (response, best_of_responses) = match req.parameters.best_of { Some(best_of) if best_of > 1 => { - let (response, best_of_responses) = infer.generate_best_of(req.0, best_of).await?; + let (response, best_of_responses) = infer.generate_best_of(req, best_of).await?; (response, Some(best_of_responses)) } - _ => (infer.generate(req.0).await?, None), + _ => (infer.generate(req).await?, None), }; // Token details @@ -321,7 +319,7 @@ content_type = "text/event-stream"), #[instrument( skip_all, fields( -parameters = ? req.0.parameters, +parameters = ? req.parameters, total_time, validation_time, queue_time, @@ -331,8 +329,8 @@ seed, ) )] async fn generate_stream( - infer: Extension, - req: Json, + Extension(infer): Extension, + Json(req): Json, ) -> ( HeaderMap, Sse>>, @@ -341,9 +339,9 @@ async fn generate_stream( let start_time = Instant::now(); metrics::increment_counter!("tgi_request_count"); - tracing::debug!("Input: {}", req.0.inputs); + tracing::debug!("Input: {}", req.inputs); - let compute_characters = req.0.inputs.chars().count(); + let compute_characters = req.inputs.chars().count(); let mut headers = HeaderMap::new(); headers.insert("x-compute-type", "gpu+optimized".parse().unwrap()); @@ -359,24 +357,24 @@ async fn generate_stream( let mut error = false; let mut add_prompt = None; - if req.0.parameters.return_full_text.unwrap_or(false) { - add_prompt = Some(req.0.inputs.clone()); + if req.parameters.return_full_text.unwrap_or(false) { + add_prompt = Some(req.inputs.clone()); } - let details = req.0.parameters.details; + let details = req.parameters.details; - let best_of = req.0.parameters.best_of.unwrap_or(1); + let best_of = req.parameters.best_of.unwrap_or(1); if best_of != 1 { let err = InferError::from(ValidationError::BestOfStream); metrics::increment_counter!("tgi_request_failure", "err" => "validation"); tracing::error!("{err}"); yield Ok(Event::from(err)); - } else if req.0.parameters.decoder_input_details { + } else if req.parameters.decoder_input_details { let err = InferError::from(ValidationError::PrefillDetailsStream); metrics::increment_counter!("tgi_request_failure", "err" => "validation"); tracing::error!("{err}"); yield Ok(Event::from(err)); } else { - match infer.generate_stream(req.0).instrument(info_span!(parent: &span, "async_stream")).await { + match infer.generate_stream(req).instrument(info_span!(parent: &span, "async_stream")).await { // Keep permit as long as generate_stream lives Ok((_permit, mut response_stream)) => { // Server-Sent Event stream From 04f7c2d86b2b572a7bf45bc81a130b6826b87379 Mon Sep 17 00:00:00 2001 From: Omar Sanseviero Date: Thu, 10 Aug 2023 14:32:07 +0200 Subject: [PATCH 371/793] Fix gated docs (#805) --- .../basic_tutorials/gated_model_access.md | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/source/basic_tutorials/gated_model_access.md b/docs/source/basic_tutorials/gated_model_access.md index f5858dc4..5d8e98f4 100644 --- a/docs/source/basic_tutorials/gated_model_access.md +++ b/docs/source/basic_tutorials/gated_model_access.md @@ -2,4 +2,23 @@ If the model you wish to serve is behind gated access or the model repository on Hugging Face Hub is private, and you have access to the model, you can provide your Hugging Face Hub access token. You can generate and copy a read token from [Hugging Face Hub tokens page](https://huggingface.co/settings/tokens) -If you're using the CLI, set the `HUGGING_FACE_HUB_TOKEN` environment variable. +If you're using the CLI, set the `HUGGING_FACE_HUB_TOKEN` environment variable. For example: + +``` +export HUGGING_FACE_HUB_TOKEN= +``` + +If you would like to do it through Docker, you can provide your token by specifying `HUGGING_FACE_HUB_TOKEN` as shown below. + +```bash +model=meta-llama/Llama-2-7b-chat-hf +volume=$PWD/data +token= + +docker run --gpus all \ + --shm-size 1g \ + -e HUGGING_FACE_HUB_TOKEN=$token \ + -p 8080:80 \ + -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.0 \ + --model-id $model +``` \ No newline at end of file From 7dbaef3f5b854babb656141596e548227f6ec41f Mon Sep 17 00:00:00 2001 From: Omar Sanseviero Date: Thu, 10 Aug 2023 14:32:51 +0200 Subject: [PATCH 372/793] Minor docs style fixes (#806) --- docs/source/basic_tutorials/consuming_tgi.md | 5 ++--- docs/source/installation.md | 15 ++++++++------- docs/source/quicktour.md | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md index 619e0a31..7fb74719 100644 --- a/docs/source/basic_tutorials/consuming_tgi.md +++ b/docs/source/basic_tutorials/consuming_tgi.md @@ -6,7 +6,7 @@ There are many ways you can consume Text Generation Inference server in your app After the launch, you can query the model using either the `/generate` or `/generate_stream` routes: -```shell +```bash curl 127.0.0.1:8080/generate \ -X POST \ -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ @@ -20,14 +20,13 @@ curl 127.0.0.1:8080/generate \ You can simply install `huggingface-hub` package with pip. -```python +```bash pip install huggingface-hub ``` Once you start the TGI server, instantiate `InferenceClient()` with the URL to the endpoint serving the model. You can then call `text_generation()` to hit the endpoint through Python. ```python - from huggingface_hub import InferenceClient client = InferenceClient(model=URL_TO_ENDPOINT_SERVING_TGI) diff --git a/docs/source/installation.md b/docs/source/installation.md index 4105acf4..a8e2e751 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -16,7 +16,7 @@ Text Generation Inference is available on pypi, conda and GitHub. To install and launch locally, first [install Rust](https://rustup.rs/) and create a Python virtual environment with at least Python 3.9, e.g. using conda: -```shell +```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh conda create -n text-generation-inference python=3.9 @@ -27,7 +27,7 @@ You may also need to install Protoc. On Linux: -```shell +```bash PROTOC_ZIP=protoc-21.12-linux-x86_64.zip curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc @@ -37,13 +37,13 @@ rm -f $PROTOC_ZIP On MacOS, using Homebrew: -```shell +```bash brew install protobuf ``` Then run to install Text Generation Inference: -```shell +```bash BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork with CUDA kernels ``` @@ -51,7 +51,7 @@ BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork On some machines, you may also need the OpenSSL libraries and gcc. On Linux machines, run: -```shell +```bash sudo apt-get install libssl-dev gcc -y ``` @@ -59,13 +59,14 @@ sudo apt-get install libssl-dev gcc -y Once installation is done, simply run: -```shell +```bash make run-falcon-7b-instruct ``` This will serve Falcon 7B Instruct model from the port 8080, which we can query. To see all options to serve your models, check in the [codebase](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or the CLI: -``` + +```bash text-generation-launcher --help ``` diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index 31185a2d..77f0a9c5 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -19,7 +19,7 @@ To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvi Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section. -```shell +```bash curl 127.0.0.1:8080/generate -X POST -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' -H 'Content-Type: application/json' ``` @@ -27,7 +27,7 @@ curl 127.0.0.1:8080/generate -X POST -d '{"inputs":"What is Deep Learning?","par To see all possible flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. -```shell +```bash docker run ghcr.io/huggingface/text-generation-inference:1.0.0 --help ``` From e58ad6dd66413ef34585348cdbac1664da391fa9 Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Thu, 10 Aug 2023 16:00:30 +0300 Subject: [PATCH 373/793] Added CLI docs (#799) Added docs for CLI --- docs/source/_toctree.yml | 2 ++ docs/source/basic_tutorials/using_cli.md | 35 ++++++++++++++++++++++++ docs/source/installation.md | 23 ++++++++++------ docs/source/quicktour.md | 4 +-- 4 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 docs/source/basic_tutorials/using_cli.md diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 8ee20eb0..a161dc28 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -15,4 +15,6 @@ title: Preparing Model for Serving - local: basic_tutorials/gated_model_access title: Serving Private & Gated Models + - local: basic_tutorials/using_cli + title: Using TGI CLI title: Tutorials diff --git a/docs/source/basic_tutorials/using_cli.md b/docs/source/basic_tutorials/using_cli.md new file mode 100644 index 00000000..82c10e6b --- /dev/null +++ b/docs/source/basic_tutorials/using_cli.md @@ -0,0 +1,35 @@ +# Using TGI CLI + +You can use TGI command-line interface (CLI) to download weights, serve and quantize models, or get information on serving parameters. To install the CLI, please refer to [the installation section](./installation#install-cli). + +`text-generation-server` lets you download the model with `download-weights` command like below 👇 + +```bash +text-generation-server download-weights MODEL_HUB_ID +``` + +You can also use it to quantize models like below 👇 + +```bash +text-generation-server quantize MODEL_HUB_ID OUTPUT_DIR +``` + +You can use `text-generation-launcher` to serve models. + +```bash +text-generation-launcher --model-id MODEL_HUB_ID --port 8080 +``` + +There are many options and parameters you can pass to `text-generation-launcher`. The documentation for CLI is kept minimal and intended to rely on self-generating documentation, which can be found by running + +```bash +text-generation-launcher --help +``` + +You can also find it hosted in this [Swagger UI](https://huggingface.github.io/text-generation-inference/). + +Same documentation can be found for `text-generation-server`. + +```bash +text-generation-server --help +``` diff --git a/docs/source/installation.md b/docs/source/installation.md index a8e2e751..1301b930 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -4,8 +4,20 @@ This section explains how to install the CLI tool as well as installing TGI from ## Install CLI -TODO +You can use TGI command-line interface (CLI) to download weights, serve and quantize models, or get information on serving parameters. +To install the CLI, you need to first clone the TGI repository and then run `make`. + +```bash +git clone https://github.com/huggingface/text-generation-inference.git && cd text-generation-inference +make install +``` + +If you would like to serve models with custom kernels, run + +```bash +BUILD_EXTENSIONS=True make install +``` ## Local Installation from Source @@ -44,7 +56,8 @@ brew install protobuf Then run to install Text Generation Inference: ```bash -BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork with CUDA kernels +git clone https://github.com/huggingface/text-generation-inference.git && cd text-generation-inference +BUILD_EXTENSIONS=True make install ``` @@ -64,9 +77,3 @@ make run-falcon-7b-instruct ``` This will serve Falcon 7B Instruct model from the port 8080, which we can query. - -To see all options to serve your models, check in the [codebase](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or the CLI: - -```bash -text-generation-launcher --help -``` diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index 77f0a9c5..f447bc19 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -4,7 +4,7 @@ The easiest way of getting started is using the official Docker container. Insta Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/falcon-7b-instruct) model with TGI. Here is an example on how to do that: -```shell +```bash model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run @@ -31,4 +31,4 @@ To see all possible flags and options, you can use the `--help` flag. It's possi docker run ghcr.io/huggingface/text-generation-inference:1.0.0 --help ``` - \ No newline at end of file + From 5df4c7c0d792738e0ed0fd3426770dea6d7233df Mon Sep 17 00:00:00 2001 From: Mishig Date: Fri, 11 Aug 2023 07:07:53 +0200 Subject: [PATCH 374/793] [docs] Build docs only when doc files change (#812) Build docs only when change happens in `docs/source` See for example https://github.com/huggingface/api-inference/blob/main/.github/workflows/build_documentation.yml#L3-L8 --- .github/workflows/build_documentation.yml | 2 ++ .github/workflows/build_pr_documentation.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml index 9d130ebb..a0f1d6f1 100644 --- a/.github/workflows/build_documentation.yml +++ b/.github/workflows/build_documentation.yml @@ -2,6 +2,8 @@ name: Build documentation on: push: + paths: + - "docs/source/**" branches: - main - doc-builder* diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index e608c74a..b46216ec 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -2,6 +2,8 @@ name: Build PR Documentation on: pull_request: + paths: + - "docs/source/**" concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From d0e30771c2995360fcfcdc5099d4a45284499e2f Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Fri, 11 Aug 2023 17:42:43 +0300 Subject: [PATCH 375/793] Added ChatUI Screenshot to Docs (#823) cc @osanseviero --- docs/source/basic_tutorials/consuming_tgi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md index 7fb74719..edc50d39 100644 --- a/docs/source/basic_tutorials/consuming_tgi.md +++ b/docs/source/basic_tutorials/consuming_tgi.md @@ -49,7 +49,7 @@ To serve both ChatUI and TGI in same environment, simply add your own endpoints } ``` -TODO: Add screenshot +![ChatUI](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/chatui_screen.png) ## API documentation From cc7bb5084d3639cbdc12f462b42b440e89cae697 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 11 Aug 2023 16:46:08 +0200 Subject: [PATCH 376/793] Upgrade transformers (fix protobuf==3.20 issue) (#795) # What does this PR do? Fixes #531 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- README.md | 1 + server/poetry.lock | 144 ++++++++++++--------------- server/pyproject.toml | 10 +- server/requirements.txt | 6 +- server/text_generation_server/cli.py | 12 +-- 5 files changed, 80 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index 0ba49675..d937e16c 100644 --- a/README.md +++ b/README.md @@ -280,3 +280,4 @@ make integration-tests TGI is also supported on the following AI hardware accelerators: - *Habana first-gen Gaudi and Gaudi2:* checkout [here](https://github.com/huggingface/optimum-habana/tree/main/text-generation-inference) how to serve models with TGI on Gaudi and Gaudi2 with [Optimum Habana](https://huggingface.co/docs/optimum/habana/index) + diff --git a/server/poetry.lock b/server/poetry.lock index f447914a..4a4e4528 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "accelerate" -version = "0.19.0" +version = "0.20.3" description = "Accelerate" optional = false python-versions = ">=3.7.0" files = [ - {file = "accelerate-0.19.0-py3-none-any.whl", hash = "sha256:2866b0bf9fff08f51e6384c95fa96725838b70f1988d1cce42e56b820d8a91dd"}, - {file = "accelerate-0.19.0.tar.gz", hash = "sha256:84920226b9e642e453ef37593ee55b956b08d8200dea4087c546c34e26157e76"}, + {file = "accelerate-0.20.3-py3-none-any.whl", hash = "sha256:147183e7a2215f7bd45a7af3b986a963daa8a61fa58b0912b9473049e011ad15"}, + {file = "accelerate-0.20.3.tar.gz", hash = "sha256:79a896978c20dac270083d42bf033f4c9a80dcdd6b946f1ca92d8d6d0f0f5ba9"}, ] [package.dependencies] @@ -192,13 +192,13 @@ files = [ [[package]] name = "bitsandbytes" -version = "0.40.2" +version = "0.41.1" description = "k-bit optimizers and matrix multiplication routines." optional = true python-versions = "*" files = [ - {file = "bitsandbytes-0.40.2-py3-none-any.whl", hash = "sha256:f0ae26f40c9230c9add9e7c70a10a5ced36fb6deff39906aec1ce4fd25e6ddc0"}, - {file = "bitsandbytes-0.40.2.tar.gz", hash = "sha256:808ac966272c63bccb2be6d77365275a4c28f1fa348d33656e670de3cab40fc4"}, + {file = "bitsandbytes-0.41.1-py3-none-any.whl", hash = "sha256:b25228c27636367f222232ed4d1e1502eedd2064be215633734fb8ea0c1c65f4"}, + {file = "bitsandbytes-0.41.1.tar.gz", hash = "sha256:b3f8e7e1e5f88d4813d10ebd4072034ba6a18eca7f0e255376f8320e5499032c"}, ] [[package]] @@ -1693,59 +1693,44 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "safetensors" -version = "0.3.1" +version = "0.3.2" description = "Fast and Safe Tensor serialization" optional = false python-versions = "*" files = [ - {file = "safetensors-0.3.1-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:2ae9b7dd268b4bae6624729dac86deb82104820e9786429b0583e5168db2f770"}, - {file = "safetensors-0.3.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:08c85c1934682f1e2cd904d38433b53cd2a98245a7cc31f5689f9322a2320bbf"}, - {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba625c7af9e1c5d0d91cb83d2fba97d29ea69d4db2015d9714d24c7f6d488e15"}, - {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b57d5890c619ec10d9f1b6426b8690d0c9c2868a90dc52f13fae6f6407ac141f"}, - {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c9f562ea696d50b95cadbeb1716dc476714a87792ffe374280c0835312cbfe2"}, - {file = "safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c115951b3a865ece8d98ee43882f2fd0a999c0200d6e6fec24134715ebe3b57"}, - {file = "safetensors-0.3.1-cp310-cp310-win32.whl", hash = "sha256:118f8f7503ea312fc7af27e934088a1b589fb1eff5a7dea2cd1de6c71ee33391"}, - {file = "safetensors-0.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:54846eaae25fded28a7bebbb66be563cad221b4c80daee39e2f55df5e5e0266f"}, - {file = "safetensors-0.3.1-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:5af82e10946c4822506db0f29269f43147e889054704dde994d4e22f0c37377b"}, - {file = "safetensors-0.3.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:626c86dd1d930963c8ea7f953a3787ae85322551e3a5203ac731d6e6f3e18f44"}, - {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12e30677e6af1f4cc4f2832546e91dbb3b0aa7d575bfa473d2899d524e1ace08"}, - {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d534b80bc8d39945bb902f34b0454773971fe9e5e1f2142af451759d7e52b356"}, - {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ddd0ddd502cf219666e7d30f23f196cb87e829439b52b39f3e7da7918c3416df"}, - {file = "safetensors-0.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:997a2cc14023713f423e6d16536d55cb16a3d72850f142e05f82f0d4c76d383b"}, - {file = "safetensors-0.3.1-cp311-cp311-win32.whl", hash = "sha256:6ae9ca63d9e22f71ec40550207bd284a60a6b4916ae6ca12c85a8d86bf49e0c3"}, - {file = "safetensors-0.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:62aa7421ca455418423e35029524489480adda53e3f702453580180ecfebe476"}, - {file = "safetensors-0.3.1-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:6d54b3ed367b6898baab75dfd057c24f36ec64d3938ffff2af981d56bfba2f42"}, - {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:262423aeda91117010f8c607889066028f680fbb667f50cfe6eae96f22f9d150"}, - {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10efe2513a8327fd628cea13167089588acc23093ba132aecfc536eb9a4560fe"}, - {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:689b3d6a7ebce70ee9438267ee55ea89b575c19923876645e927d08757b552fe"}, - {file = "safetensors-0.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14cd9a87bc73ce06903e9f8ee8b05b056af6f3c9f37a6bd74997a16ed36ff5f4"}, - {file = "safetensors-0.3.1-cp37-cp37m-win32.whl", hash = "sha256:a77cb39624480d5f143c1cc272184f65a296f573d61629eff5d495d2e0541d3e"}, - {file = "safetensors-0.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9eff3190bfbbb52eef729911345c643f875ca4dbb374aa6c559675cfd0ab73db"}, - {file = "safetensors-0.3.1-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:05cbfef76e4daa14796db1bbb52072d4b72a44050c368b2b1f6fd3e610669a89"}, - {file = "safetensors-0.3.1-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:c49061461f4a81e5ec3415070a3f135530834c89cbd6a7db7cd49e3cb9d9864b"}, - {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22cf7e73ca42974f098ce0cf4dd8918983700b6b07a4c6827d50c8daefca776e"}, - {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04f909442d6223ff0016cd2e1b2a95ef8039b92a558014627363a2e267213f62"}, - {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c573c5a0d5d45791ae8c179e26d74aff86e719056591aa7edb3ca7be55bc961"}, - {file = "safetensors-0.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6994043b12e717cf2a6ba69077ac41f0d3675b2819734f07f61819e854c622c7"}, - {file = "safetensors-0.3.1-cp38-cp38-win32.whl", hash = "sha256:158ede81694180a0dbba59422bc304a78c054b305df993c0c6e39c6330fa9348"}, - {file = "safetensors-0.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:afdc725beff7121ea8d39a7339f5a6abcb01daa189ea56290b67fe262d56e20f"}, - {file = "safetensors-0.3.1-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:cba910fcc9e5e64d32d62b837388721165e9c7e45d23bc3a38ad57694b77f40d"}, - {file = "safetensors-0.3.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a4f7dbfe7285573cdaddd85ef6fa84ebbed995d3703ab72d71257944e384612f"}, - {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54aed0802f9eaa83ca7b1cbb986bfb90b8e2c67b6a4bcfe245627e17dad565d4"}, - {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34b75a766f3cfc99fd4c33e329b76deae63f5f388e455d863a5d6e99472fca8e"}, - {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a0f31904f35dc14919a145b2d7a2d8842a43a18a629affe678233c4ea90b4af"}, - {file = "safetensors-0.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcf527ecc5f58907fd9031510378105487f318cc91ecdc5aee3c7cc8f46030a8"}, - {file = "safetensors-0.3.1-cp39-cp39-win32.whl", hash = "sha256:e2f083112cf97aa9611e2a05cc170a2795eccec5f6ff837f4565f950670a9d83"}, - {file = "safetensors-0.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:5f4f614b8e8161cd8a9ca19c765d176a82b122fa3d3387b77862145bfe9b4e93"}, - {file = "safetensors-0.3.1.tar.gz", hash = "sha256:571da56ff8d0bec8ae54923b621cda98d36dcef10feb36fd492c4d0c2cd0e869"}, + {file = "safetensors-0.3.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b6a66989075c2891d743153e8ba9ca84ee7232c8539704488f454199b8b8f84d"}, + {file = "safetensors-0.3.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:670d6bc3a3b377278ce2971fa7c36ebc0a35041c4ea23b9df750a39380800195"}, + {file = "safetensors-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbb44e140bf2aeda98d9dde669dbec15f7b77f96a9274469b91a6cf4bcc5ec3b"}, + {file = "safetensors-0.3.2-cp310-cp310-win32.whl", hash = "sha256:2961c1243fd0da46aa6a1c835305cc4595486f8ac64632a604d0eb5f2de76175"}, + {file = "safetensors-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c813920482c337d1424d306e1b05824a38e3ef94303748a0a287dea7a8c4f805"}, + {file = "safetensors-0.3.2-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:707df34bd9b9047e97332136ad98e57028faeccdb9cfe1c3b52aba5964cc24bf"}, + {file = "safetensors-0.3.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:becc5bb85b2947eae20ed23b407ebfd5277d9a560f90381fe2c42e6c043677ba"}, + {file = "safetensors-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada0fac127ff8fb04834da5c6d85a8077e6a1c9180a11251d96f8068db922a17"}, + {file = "safetensors-0.3.2-cp311-cp311-win32.whl", hash = "sha256:155b82dbe2b0ebff18cde3f76b42b6d9470296e92561ef1a282004d449fa2b4c"}, + {file = "safetensors-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:a86428d196959619ce90197731be9391b5098b35100a7228ef4643957648f7f5"}, + {file = "safetensors-0.3.2-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:c1f8ab41ed735c5b581f451fd15d9602ff51aa88044bfa933c5fa4b1d0c644d1"}, + {file = "safetensors-0.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d7d70d48585fe8df00725aa788f2e64fd24a4c9ae07cd6be34f6859d0f89a9c"}, + {file = "safetensors-0.3.2-cp37-cp37m-win32.whl", hash = "sha256:6ff59bc90cdc857f68b1023be9085fda6202bbe7f2fd67d06af8f976d6adcc10"}, + {file = "safetensors-0.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:8b05c93da15fa911763a89281906ca333ed800ab0ef1c7ce53317aa1a2322f19"}, + {file = "safetensors-0.3.2-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:8969cfd9e8d904e8d3c67c989e1bd9a95e3cc8980d4f95e4dcd43c299bb94253"}, + {file = "safetensors-0.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa98f49e95f02eb750d32c4947e7d5aa43883149ebd0414920866446525b70f0"}, + {file = "safetensors-0.3.2-cp38-cp38-win32.whl", hash = "sha256:33409df5e28a83dc5cc5547a3ac17c0f1b13a1847b1eb3bc4b3be0df9915171e"}, + {file = "safetensors-0.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:e04a7cbbb3856159ab99e3adb14521544f65fcb8548cce773a1435a0f8d78d27"}, + {file = "safetensors-0.3.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:7c864cf5dcbfb608c5378f83319c60cc9c97263343b57c02756b7613cd5ab4dd"}, + {file = "safetensors-0.3.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:14e8c19d6dc51d4f70ee33c46aff04c8ba3f95812e74daf8036c24bc86e75cae"}, + {file = "safetensors-0.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ff0024ef2e5722a79af24688ce4a430f70601d0cf712a744105ed4b8f67ba5"}, + {file = "safetensors-0.3.2-cp39-cp39-win32.whl", hash = "sha256:827af9478b78977248ba93e2fd97ea307fb63f463f80cef4824460f8c2542a52"}, + {file = "safetensors-0.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:9b09f27c456efa301f98681ea14b12f81f2637889f6336223ccab71e42c34541"}, + {file = "safetensors-0.3.2.tar.gz", hash = "sha256:2dbd34554ed3b99435a0e84df077108f5334c8336b5ed9cb8b6b98f7b10da2f6"}, ] [package.extras] -all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] -dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (>=2.11.0)", "torch (>=1.10)"] +all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (==2.11.0)", "torch (>=1.10)"] +dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (==2.11.0)", "torch (>=1.10)"] jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] numpy = ["numpy (>=1.21.6)"] paddlepaddle = ["paddlepaddle (>=2.4.1)"] +pinned-tf = ["tensorflow (==2.11.0)"] quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] tensorflow = ["tensorflow (>=2.11.0)"] testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] @@ -2013,13 +1998,13 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.29.2" +version = "4.31.0" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" files = [ - {file = "transformers-4.29.2-py3-none-any.whl", hash = "sha256:0ef158b99bad6f4e6652a0d8655fbbe58b4cb788ce7040f320b5d29c7c810a75"}, - {file = "transformers-4.29.2.tar.gz", hash = "sha256:ed9467661f459f1ce49461d83f18f3b36b6a37f306182dc2ba272935f3b93ebb"}, + {file = "transformers-4.31.0-py3-none-any.whl", hash = "sha256:8487aab0195ce1c2a5ae189305118b9720daddbc7b688edb09ccd79e3b149f6b"}, + {file = "transformers-4.31.0.tar.gz", hash = "sha256:4302fba920a1c24d3a429a29efff6a63eac03f3f3cf55b55927fc795d01cb273"}, ] [package.dependencies] @@ -2030,28 +2015,29 @@ packaging = ">=20.0" pyyaml = ">=5.1" regex = "!=2019.12.17" requests = "*" +safetensors = ">=0.3.1" tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" tqdm = ">=4.27" [package.extras] -accelerate = ["accelerate (>=0.19.0)"] -agents = ["Pillow", "accelerate (>=0.19.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] -all = ["Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] -audio = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] +accelerate = ["accelerate (>=0.20.3)"] +agents = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] +all = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] codecarbon = ["codecarbon (==1.2.0)"] -deepspeed = ["accelerate (>=0.19.0)", "deepspeed (>=0.8.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.19.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.8.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.6.9)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "numba (<0.57.0)", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.19.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "numba (<0.57.0)", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -docs = ["Pillow", "accelerate (>=0.19.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.6.9)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "numba (<0.57.0)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +deepspeed = ["accelerate (>=0.20.3)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] docs-specific = ["hf-doc-builder"] fairscale = ["fairscale (>0.3)"] -flax = ["flax (>=0.4.1,<=0.6.9)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "optax (>=0.0.8,<=0.1.4)"] -flax-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] +flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] +flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] ftfy = ["ftfy"] integrations = ["optuna", "ray[tune]", "sigopt"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] modelcreation = ["cookiecutter (==1.7.3)"] natten = ["natten (>=0.14.6)"] onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] @@ -2061,23 +2047,23 @@ quality = ["GitPython (<3.1.19)", "black (>=23.1,<24.0)", "datasets (!=2.5.0)", ray = ["ray[tune]"] retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] -sentencepiece = ["protobuf (<=3.20.2)", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["fastapi", "pydantic", "starlette", "uvicorn"] +sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] +serving = ["fastapi", "pydantic (<2)", "starlette", "uvicorn"] sigopt = ["sigopt"] sklearn = ["scikit-learn"] -speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] -tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.4,<2.13)", "tensorflow-text (<2.13)", "tf2onnx"] -tf-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)"] +speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx"] +tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] timm = ["timm"] tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -torch = ["accelerate (>=0.19.0)", "torch (>=1.9,!=1.12.0)"] -torch-speech = ["kenlm", "librosa", "numba (<0.57.0)", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torch-vision = ["Pillow", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf (<=3.20.2)", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] +torch = ["accelerate (>=0.20.3)", "torch (>=1.9,!=1.12.0)"] +torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +torch-vision = ["Pillow (<10.0.0)", "torchvision"] +torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] video = ["av (==9.2.0)", "decord (==0.6.0)"] -vision = ["Pillow"] +vision = ["Pillow (<10.0.0)"] [[package]] name = "triton" @@ -2462,4 +2448,4 @@ quantize = ["accelerate", "datasets", "texttable"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "2abb80833b678452cfc73464fc5b2e48d74b2672bd987240041a33c724a74000" +content-hash = "6b6dbd9e11bc2f71cd88e2ab50adce249a722fea02e7cdcf5fe1b94c7b997a57" diff --git a/server/pyproject.toml b/server/pyproject.toml index 88b93f4e..25ffc538 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -15,18 +15,18 @@ grpcio-status = "^1.51.1" grpcio-reflection = "^1.51.1" grpc-interceptor = "^0.15.0" typer = "^0.6.1" -accelerate = { version = "^0.19.0", optional = true } -bitsandbytes = { version = "^0.40.0", optional = true } -safetensors = "0.3.1" +accelerate = { version = "^0.20.0", optional = true } +bitsandbytes = { version = "^0.41.1", optional = true } +safetensors = "^0.3.2" loguru = "^0.6.0" opentelemetry-api = "^1.15.0" opentelemetry-exporter-otlp = "^1.15.0" opentelemetry-instrumentation-grpc = "^0.36b0" hf-transfer = "^0.1.2" sentencepiece = "^0.1.97" -tokenizers = "0.13.3" +tokenizers = "^0.13.3" huggingface-hub = "^0.14.1" -transformers = "4.29.2" +transformers = "^4.31.0" einops = "^0.6.1" texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } diff --git a/server/requirements.txt b/server/requirements.txt index beab4bf6..5cc19619 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,6 +1,6 @@ --extra-index-url https://download.pytorch.org/whl/cu118 -accelerate==0.19.0 ; python_version >= "3.9" and python_version < "3.13" +accelerate==0.20.3 ; python_version >= "3.9" and python_version < "3.13" backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13" certifi==2023.7.22 ; python_version >= "3.9" and python_version < "3.13" charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "3.13" @@ -42,7 +42,7 @@ psutil==5.9.5 ; python_version >= "3.9" and python_version < "3.13" pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" regex==2023.6.3 ; python_version >= "3.9" and python_version < "3.13" requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" -safetensors==0.3.1 ; python_version >= "3.9" and python_version < "3.13" +safetensors==0.3.2 ; python_version >= "3.9" and python_version < "3.13" scipy==1.11.1 ; python_version >= "3.9" and python_version < "3.13" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" setuptools==68.0.0 ; python_version >= "3.9" and python_version < "3.13" @@ -50,7 +50,7 @@ sympy==1.12 ; python_version >= "3.9" and python_version < "3.13" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" torch==2.0.1+cu118 ; python_version >= "3.9" and python_version < "3.13" tqdm==4.65.0 ; python_version >= "3.9" and python_version < "3.13" -transformers==4.29.2 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.31.0 ; python_version >= "3.9" and python_version < "3.13" triton==2.0.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "3.13" typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 459ba8c4..b12a9751 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -171,14 +171,14 @@ def download_weights( for p in local_pt_files ] try: - from transformers import AutoConfig import transformers + import json - config = AutoConfig.from_pretrained( - model_id, - revision=revision, - ) - architecture = config.architectures[0] + + config_filename = hf_hub_download(model_id, revision=revision, filename="config.json") + with open(config_filename, "r") as f: + config = json.load(f) + architecture = config["architectures"][0] class_ = getattr(transformers, architecture) From a2a913eec5436692b85b2a0aa48e4e49b7821ff6 Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Fri, 11 Aug 2023 18:05:19 +0300 Subject: [PATCH 377/793] Added streaming for InferenceClient (#821) Co-authored-by: Lucain Co-authored-by: Omar Sanseviero --- docs/source/basic_tutorials/consuming_tgi.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md index edc50d39..34fa549c 100644 --- a/docs/source/basic_tutorials/consuming_tgi.md +++ b/docs/source/basic_tutorials/consuming_tgi.md @@ -1,6 +1,6 @@ # Consuming Text Generation Inference -There are many ways you can consume Text Generation Inference server in your applications. After launching, you can use the `/generate` route and make a `POST` request to get results from the server. You can also use the `/generate_stream` route if you want TGI to return a stram of tokens. You can make the requests using the tool of your preference, such as curl, Python or TypeScrpt. For a final end-to-end experience, we also open-sourced ChatUI, a chat interface for open-source models. +There are many ways you can consume Text Generation Inference server in your applications. After launching, you can use the `/generate` route and make a `POST` request to get results from the server. You can also use the `/generate_stream` route if you want TGI to return a stream of tokens. You can make the requests using the tool of your preference, such as curl, Python or TypeScrpt. For a final end-to-end experience, we also open-sourced ChatUI, a chat interface for open-source models. ## curl @@ -33,6 +33,24 @@ client = InferenceClient(model=URL_TO_ENDPOINT_SERVING_TGI) client.text_generation(prompt="Write a code for snake game", model=URL_TO_ENDPOINT_SERVING_TGI) ``` +To stream tokens in `InferenceClient`, simply pass `stream=True`. Another parameter you can use with TGI backend is `details`. You can get more details on generation (tokens, probabilities, etc.) by setting `details` to `True`. By default, `details` is set to `False`, and `text_generation` returns a string. If you pass `details=True` and `stream=True`, `text_generation` will return a `TextGenerationStreamResponse` which consists of the generated token, generated text, and details. + +```python +output = client.text_generation(prompt="Meaning of life is", model=URL_OF_ENDPOINT, details=True) +print(output) + +# TextGenerationResponse(generated_text=' a complex concept that is not always clear to the individual. It is a concept that is not always', details=Details(finish_reason=, generated_tokens=20, seed=None, prefill=[], tokens=[Token(id=267, text=' a', logprob=-2.0723474, special=False), Token(id=11235, text=' complex', logprob=-3.1272552, special=False), Token(id=17908, text=' concept', logprob=-1.3632495, special=False),..)) +``` + +You can see how to stream below. + +```python +output = client.text_generation(prompt="Meaning of life is", model="http://localhost:3000/", stream=True, details=True) +print(next(iter(output))) + +# TextGenerationStreamResponse(token=Token(id=267, text=' a', logprob=-2.0723474, special=False), generated_text=None, details=None) +``` + You can check out the details of the function [here](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation). From 09eca6422788b1710c54ee0d05dd6746f16bb681 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 14 Aug 2023 11:23:11 +0200 Subject: [PATCH 378/793] Version 1.0.1 (#836) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- README.md | 4 ++-- docs/openapi.json | 4 ++-- docs/source/basic_tutorials/gated_model_access.md | 4 ++-- docs/source/quicktour.md | 2 +- server/.gitignore | 5 ++++- server/pyproject.toml | 2 +- 8 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61c22a40..f9cfecf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2893,7 +2893,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "1.0.0" +version = "1.0.1" dependencies = [ "average", "clap", @@ -2913,7 +2913,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.0.0" +version = "1.0.1" dependencies = [ "futures", "grpc-metadata", @@ -2929,7 +2929,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.0.0" +version = "1.0.1" dependencies = [ "clap", "ctrlc", @@ -2945,7 +2945,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.0.0" +version = "1.0.1" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index 255d35d4..1cbac920 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.0.0" +version = "1.0.1" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index d937e16c..8dcd396b 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ The easiest way of getting started is using the official Docker container: model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.0 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.1 --model-id $model ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. @@ -150,7 +150,7 @@ model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= -docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.0 --model-id $model +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.1 --model-id $model ``` ### A note on Shared Memory (shm) diff --git a/docs/openapi.json b/docs/openapi.json index 0465a5ad..ab1802c3 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.0.0" + "version": "1.0.1" }, "paths": { "/": { @@ -846,4 +846,4 @@ "description": "Hugging Face Text Generation Inference API" } ] -} \ No newline at end of file +} diff --git a/docs/source/basic_tutorials/gated_model_access.md b/docs/source/basic_tutorials/gated_model_access.md index 5d8e98f4..08c76de2 100644 --- a/docs/source/basic_tutorials/gated_model_access.md +++ b/docs/source/basic_tutorials/gated_model_access.md @@ -19,6 +19,6 @@ docker run --gpus all \ --shm-size 1g \ -e HUGGING_FACE_HUB_TOKEN=$token \ -p 8080:80 \ - -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.0 \ + -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.1 \ --model-id $model -``` \ No newline at end of file +``` diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index f447bc19..5b37c03f 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -8,7 +8,7 @@ Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/ model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.0 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.1 --model-id $model ``` diff --git a/server/.gitignore b/server/.gitignore index aef74bb4..2e1db124 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -155,4 +155,7 @@ dmypy.json cython_debug/ transformers -safetensors \ No newline at end of file +safetensors +flash-attention/ +flash-attention-v2/ +vllm/ diff --git a/server/pyproject.toml b/server/pyproject.toml index 25ffc538..8eeae616 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.0.0" +version = "1.0.1" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From d71237fc8b6fd82407b0e27b58cd32c61f2ab6cb Mon Sep 17 00:00:00 2001 From: Omar Sanseviero Date: Mon, 14 Aug 2023 13:47:32 +0200 Subject: [PATCH 379/793] Have snippets in Python/JavaScript in quicktour (#809) ![Screenshot from 2023-08-10 14-20-25](https://github.com/huggingface/text-generation-inference/assets/7246357/e16d0d41-be63-4d06-8093-30540df91419) --------- Co-authored-by: Merve Noyan --- docs/source/quicktour.md | 65 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index 5b37c03f..4ba2be40 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -17,15 +17,72 @@ To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvi -Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section. +Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section, where we show examples with utility libraries and UIs. Below you can see a simple snippet to query the endpoint. -```bash -curl 127.0.0.1:8080/generate -X POST -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' -H 'Content-Type: application/json' + + + + +```python +import requests + +headers = { + "Content-Type": "application/json", +} + +data = { + 'inputs': 'What is Deep Learning?', + 'parameters': { + 'max_new_tokens': 20, + }, +} + +response = requests.post('http://127.0.0.1:8080/generate', headers=headers, json=data) +print(response.json()) +# {'generated_text': '\n\nDeep Learning is a subset of Machine Learning that is concerned with the development of algorithms that can'} ``` + + + +```js +async function query() { + const response = await fetch( + 'http://127.0.0.1:8080/generate', + { + method: 'POST', + headers: { 'Content-Type': 'application/json'}, + body: JSON.stringify({ + 'inputs': 'What is Deep Learning?', + 'parameters': { + 'max_new_tokens': 20 + } + }) + } + ); +} + +query().then((response) => { + console.log(JSON.stringify(response)); +}); +/// {"generated_text":"\n\nDeep Learning is a subset of Machine Learning that is concerned with the development of algorithms that can"} +``` + + + + +```curl +curl 127.0.0.1:8080/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ + -H 'Content-Type: application/json' +``` + + + -To see all possible flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. +To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. ```bash docker run ghcr.io/huggingface/text-generation-inference:1.0.0 --help From 3ffcd9d311d27806a70037ffba88595bed7289f2 Mon Sep 17 00:00:00 2001 From: sawan Rawat <103842962+sawanjr@users.noreply.github.com> Date: Mon, 14 Aug 2023 17:39:20 +0530 Subject: [PATCH 380/793] Added two more features in readme.md file (#831) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 8dcd396b..aa5ccaad 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ to power Hugging Chat, the Inference API and Inference Endpoint. - Stop sequences - Log probabilities - Production ready (distributed tracing with Open Telemetry, Prometheus metrics) +- Custom Prompt Generation: Easily generate text by providing custom prompts to guide the model's output. +- Fine-tuning Support: Utilize fine-tuned models for specific tasks to achieve higher accuracy and performance. + ## Optimized architectures From b5087c4f4ece30decb739ab311448e358f6108bb Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 14 Aug 2023 14:09:51 +0200 Subject: [PATCH 381/793] Fix rope dynamic + factor (#822) # What does this PR do? Fixes #816 ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/layers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index c2a340b2..9688f6e0 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -547,8 +547,6 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): self.inv_freq = _create_inv_freq(self.dim, newbase, self.inv_freq.device) self._seq_len_cached = seqlen t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) - if self.scaling_factor is not None: - t /= self.scaling_factor # Don't do einsum, it converts fp32 to fp16 # freqs = torch.einsum("i,j->ij", t, self.inv_freq) From a072660bf51022f9e1d59b64efc5954a0e1eee45 Mon Sep 17 00:00:00 2001 From: Dong Shin Date: Mon, 14 Aug 2023 21:20:18 +0900 Subject: [PATCH 382/793] fix: LlamaTokenizerFast to AutoTokenizer at flash_llama.py (#619) # What does this PR do? A few tokenizer_config in huggingface use LlamaTokenizer, so I think I would have selected `LlamaTokenizer` before. For a few cases where you're using a llama structure but not a llama tokenizer, why not make it to call the AutoTokenizer in exception handling. In the case of `decapoda-research/llama-7b-hf`, LLamaTokenizer is still being used in config.json, so it should be called through` LlamaTokenizer`. Also, if an exception is thrown by LlamaTokenizer, it will cause `LlamaTokenzierFast` to be called from AutoTokenizer. Fixes # 560 ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [x] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @Narsil --- server/text_generation_server/models/flash_llama.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 96fb0c26..063aa01e 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -2,7 +2,8 @@ import torch.distributed from opentelemetry import trace -from transformers.models.llama import LlamaTokenizer, LlamaTokenizerFast +from transformers import AutoConfig, AutoTokenizer +from transformers.models.llama import LlamaTokenizer from typing import Optional from text_generation_server.models import FlashCausalLM @@ -44,7 +45,7 @@ def __init__( trust_remote_code=trust_remote_code, ) except Exception: - tokenizer = LlamaTokenizerFast.from_pretrained( + tokenizer = AutoTokenizer.from_pretrained( model_id, revision=revision, padding_side="left", From d8f1337e7e6071e5a815720a7382072cf7694216 Mon Sep 17 00:00:00 2001 From: Pasquale Minervini Date: Mon, 14 Aug 2023 15:41:13 +0200 Subject: [PATCH 383/793] README edit -- running the service with no GPU or CUDA support (#773) One-line addition to the README to show how to run the service on a machine without GPUs or CUDA support (e.g., for local prototyping) --------- Co-authored-by: Nicolas Patry --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa5ccaad..e334b800 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ volume=$PWD/data # share a volume with the Docker container to avoid downloading docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.1 --model-id $model ``` -**Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. +**Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli: ``` From 05dd14fdb93f83ad5fde6d5b9cb6c21edef71aa1 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 14 Aug 2023 19:26:19 +0200 Subject: [PATCH 384/793] Fix `tokenizers==0.13.4` . (#838) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- Cargo.lock | 373 +++++++++++++++++------------------- benchmark/src/generation.rs | 4 +- router/src/validation.rs | 2 +- 3 files changed, 175 insertions(+), 204 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9cfecf7..0f3b39de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,9 +50,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "86b8f9420f797f2d9e935edf629310eb938a0d839f984e25327f3c7eed22300c" dependencies = [ "memchr", ] @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -108,9 +108,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" [[package]] name = "arc-swap" @@ -148,18 +148,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] name = "async-trait" -version = "0.1.71" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -193,9 +193,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", @@ -297,9 +297,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "block-buffer" @@ -385,11 +385,12 @@ checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -410,9 +411,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.11" +version = "4.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d" +checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" dependencies = [ "clap_builder", "clap_derive", @@ -421,9 +422,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.11" +version = "4.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b" +checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" dependencies = [ "anstream", "anstyle", @@ -433,14 +434,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.2" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -644,6 +645,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "deranged" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" + [[package]] name = "derive_builder" version = "0.12.0" @@ -717,9 +724,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encode_unicode" @@ -744,9 +751,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", @@ -774,22 +781,19 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "filetime" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.3.5", "windows-sys 0.48.0", ] @@ -930,7 +934,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -1113,9 +1117,9 @@ dependencies = [ [[package]] name = "http-range-header" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" @@ -1125,9 +1129,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" @@ -1146,7 +1150,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -1248,26 +1252,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "ipnet" version = "2.8.0" @@ -1281,7 +1265,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.4", + "rustix", "windows-sys 0.48.0", ] @@ -1314,9 +1298,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" @@ -1356,15 +1340,9 @@ checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - -[[package]] -name = "linux-raw-sys" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "lock_api" @@ -1378,9 +1356,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "mach2" @@ -1424,9 +1402,9 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" [[package]] name = "memchr" @@ -1489,7 +1467,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -1552,9 +1530,9 @@ dependencies = [ [[package]] name = "monostate" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3f57a8802842f648026a33c3d2e3bb41bb309a35b1609bd7ef2b060b8b6b1b" +checksum = "15f370ae88093ec6b11a710dec51321a61d420fafd1bad6e30d01bd9c920e8ee" dependencies = [ "monostate-impl", "serde", @@ -1562,13 +1540,13 @@ dependencies = [ [[package]] name = "monostate-impl" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72f4d2e10fde62a0f2fcb4b44ccbf4f9899dcc30c9193449f8dfb9123d71377" +checksum = "371717c0a5543d6a800cac822eac735aa7d2d2fbb41002e9856a4089532dbdce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -1705,9 +1683,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", "libm", @@ -1783,9 +1761,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.55" +version = "0.10.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -1804,7 +1782,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -1815,9 +1793,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" dependencies = [ "cc", "libc", @@ -2005,9 +1983,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b27ab7be369122c218afc2079489cdcb4b517c0a3fc386ff11e1fedfcc2b35" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "pbkdf2" @@ -2039,29 +2017,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] name = "pin-project-lite" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pin-utils" @@ -2077,9 +2055,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d220334a184db82b31b83f5ff093e3315280fb2b6bbc032022b2304a509aab7a" +checksum = "f32154ba0af3a075eefa1eda8bb414ee928f62303a54ea85b8d6638ff1a6ee9e" [[package]] name = "ppv-lite86" @@ -2123,9 +2101,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.64" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -2202,9 +2180,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.29" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] @@ -2325,13 +2303,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ - "aho-corasick 1.0.2", + "aho-corasick 1.0.3", "memchr", - "regex-automata 0.3.3", + "regex-automata 0.3.6", "regex-syntax 0.7.4", ] @@ -2346,11 +2324,11 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.3" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" dependencies = [ - "aho-corasick 1.0.2", + "aho-corasick 1.0.3", "memchr", "regex-syntax 0.7.4", ] @@ -2440,7 +2418,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.25", + "syn 2.0.28", "walkdir", ] @@ -2471,28 +2449,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustix" -version = "0.38.4" +version = "0.38.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "errno", "libc", - "linux-raw-sys 0.4.3", + "linux-raw-sys", "windows-sys 0.48.0", ] @@ -2519,15 +2483,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc31bd9b61a32c31f9650d18add92aa83a49ba979c143eefd27fe7177b05bd5f" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -2549,9 +2513,9 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -2565,9 +2529,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -2578,9 +2542,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -2588,35 +2552,35 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] name = "serde_json" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ "itoa", "ryu", @@ -2625,9 +2589,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc4422959dd87a76cb117c191dcbffc20467f06c9100b76721dab370f24d3a" +checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" dependencies = [ "itoa", "serde", @@ -2687,9 +2651,9 @@ dependencies = [ [[package]] name = "signal-hook" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" dependencies = [ "libc", "signal-hook-registry", @@ -2755,6 +2719,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -2813,9 +2787,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.25" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -2830,9 +2804,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.29.4" +version = "0.29.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751e810399bba86e9326f5762b7f32ac5a085542df78da6a78d94e07d14d7c11" +checksum = "d10ed79c22663a35a255d289a7fdcb43559fc77ff15df5ce6c341809e7867528" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2868,9 +2842,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96d2ffad078296368d46ff1cb309be1c23c513b4ab0e22a45de0185275ac96" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" dependencies = [ "filetime", "libc", @@ -2879,15 +2853,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.6.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ - "autocfg", "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix 0.37.23", + "rustix", "windows-sys 0.48.0", ] @@ -2967,7 +2940,7 @@ dependencies = [ "thiserror", "tokenizers", "tokio", - "tower-http 0.4.1", + "tower-http 0.4.3", "tracing", "tracing-opentelemetry 0.19.0", "tracing-subscriber", @@ -2978,22 +2951,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -3008,10 +2981,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.23" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" +checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" dependencies = [ + "deranged", "itoa", "libc", "num_threads", @@ -3028,9 +3002,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" +checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" dependencies = [ "time-core", ] @@ -3052,9 +3026,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5" +checksum = "aea68938177975ab09da68552b720eac941779ff386baceaf77e0f5f9cea645f" dependencies = [ "aho-corasick 0.7.20", "cached-path", @@ -3075,7 +3049,7 @@ dependencies = [ "rayon", "rayon-cond", "regex", - "regex-syntax 0.6.29", + "regex-syntax 0.7.4", "reqwest", "serde", "serde_json", @@ -3088,11 +3062,10 @@ dependencies = [ [[package]] name = "tokio" -version = "1.29.1" +version = "1.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +checksum = "40de3a2ba249dcb097e01be5e67a5ff53cf250397715a071a81543e8a832a920" dependencies = [ - "autocfg", "backtrace", "bytes", "libc", @@ -3101,7 +3074,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.3", "tokio-macros", "windows-sys 0.48.0", ] @@ -3124,7 +3097,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -3288,11 +3261,11 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c" +checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "bytes", "futures-core", "futures-util", @@ -3337,7 +3310,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -3459,9 +3432,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -3518,9 +3491,9 @@ dependencies = [ [[package]] name = "urlencoding" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "utf8parse" @@ -3530,9 +3503,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utoipa" -version = "3.4.0" +version = "3.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520434cac5c98120177d5cc15be032703f6dca7d5ef82e725c798113b375000a" +checksum = "de634b7f8178c9c246c88ea251f3a0215c9a4d80778db2d7bd4423a78b5170ec" dependencies = [ "indexmap 2.0.0", "serde", @@ -3542,22 +3515,22 @@ dependencies = [ [[package]] name = "utoipa-gen" -version = "3.4.1" +version = "3.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e22e88a487b6e0374533871b79b1f5ded05671bd0936bd547eb42f82fb9060d" +checksum = "0fcba79cb3e5020d9bcc8313cd5aadaf51d6d54a6b3fd08c3d0360ae6b3c83d0" dependencies = [ "proc-macro-error", "proc-macro2", "quote", "regex", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] name = "utoipa-swagger-ui" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4602d7100d3cfd8a086f30494e68532402ab662fa366c9d201d677e33cee138d" +checksum = "84614caa239fb25b2bb373a52859ffd94605ceb256eeb1d63436325cf81e3653" dependencies = [ "axum", "mime_guess", @@ -3646,7 +3619,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", "wasm-bindgen-shared", ] @@ -3680,7 +3653,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3896,9 +3869,9 @@ dependencies = [ [[package]] name = "xattr" -version = "0.2.3" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" dependencies = [ "libc", ] diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs index b57c652b..c72d31d3 100644 --- a/benchmark/src/generation.rs +++ b/benchmark/src/generation.rs @@ -217,7 +217,5 @@ fn create_sequence(sequence_length: u32, tokenizer: Tokenizer) -> String { // Truncate to sequence_length encoding.truncate(sequence_length as usize, 0, TruncationDirection::Left); // Decode - tokenizer - .decode(Vec::from(encoding.get_ids()), false) - .unwrap() + tokenizer.decode(encoding.get_ids(), false).unwrap() } diff --git a/router/src/validation.rs b/router/src/validation.rs index be835bf0..f967361f 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -311,7 +311,7 @@ fn prepare_input( // truncate encoding and decode new inputs encoding.truncate(truncate, 0, TruncationDirection::Left); let inputs = tokenizer - .decode(Vec::from(encoding.get_ids()), false) + .decode(encoding.get_ids(), false) .map_err(|err| ValidationError::Tokenizer(err.to_string()))?; (inputs, encoding.len()) } From 737d5781e45fc2c297451773dea5ca1355b9a71d Mon Sep 17 00:00:00 2001 From: Adarsh Shirawalmath <114558126+adarshxs@users.noreply.github.com> Date: Tue, 15 Aug 2023 22:43:52 +0530 Subject: [PATCH 385/793] Update README.md (#848) @Narsil --------- Co-authored-by: Nicolas Patry --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e334b800..d9f4db19 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingf ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. -To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli: +To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli): ``` text-generation-launcher --help ``` From 2774b0ab44f91209105385f8097d49bcf3b110aa Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 16 Aug 2023 07:17:26 +0200 Subject: [PATCH 386/793] Fixing watermark. (#851) # What does this PR do? Fixes #843 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/watermark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/utils/watermark.py b/server/text_generation_server/utils/watermark.py index df7b90e3..5d8f5312 100644 --- a/server/text_generation_server/utils/watermark.py +++ b/server/text_generation_server/utils/watermark.py @@ -19,8 +19,8 @@ from transformers import LogitsProcessor from typing import List, Union -GAMMA = os.getenv("WATERMARK_GAMMA", 0.5) -DELTA = os.getenv("WATERMARK_DELTA", 2.0) +GAMMA = float(os.getenv("WATERMARK_GAMMA", 0.5)) +DELTA = float(os.getenv("WATERMARK_DELTA", 2.0)) class WatermarkLogitsProcessor(LogitsProcessor): From d9bceb8e6b2090e5ca7668de2c9beac5eb94ded6 Mon Sep 17 00:00:00 2001 From: Omar Sanseviero Date: Wed, 16 Aug 2023 14:29:54 +0200 Subject: [PATCH 387/793] Misc improvements for InferenceClient docs (#852) List of changes - No need to specify `model` in `text_generation` if it's already specified in `InferenceClient` - I separated the explanation of `stream=True` and `details=True` - I found the details explanation a bit repetitive (it says two times what it returns), so removed a sentence - Add mention of async client --- docs/source/basic_tutorials/consuming_tgi.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md index 34fa549c..1f0ff37d 100644 --- a/docs/source/basic_tutorials/consuming_tgi.md +++ b/docs/source/basic_tutorials/consuming_tgi.md @@ -17,7 +17,6 @@ curl 127.0.0.1:8080/generate \ ## Inference Client [`huggingface-hub`](https://huggingface.co/docs/huggingface_hub/main/en/index) is a Python library to interact with the Hugging Face Hub, including its endpoints. It provides a nice high-level class, [`~huggingface_hub.InferenceClient`], which makes it easy to make calls to a TGI endpoint. `InferenceClient` also takes care of parameter validation and provides a simple to-use interface. - You can simply install `huggingface-hub` package with pip. ```bash @@ -29,14 +28,21 @@ Once you start the TGI server, instantiate `InferenceClient()` with the URL to t ```python from huggingface_hub import InferenceClient -client = InferenceClient(model=URL_TO_ENDPOINT_SERVING_TGI) -client.text_generation(prompt="Write a code for snake game", model=URL_TO_ENDPOINT_SERVING_TGI) +client = InferenceClient(model="http://127.0.0.1:8080") +client.text_generation(prompt="Write a code for snake game") +``` + +You can do streaming with `InferenceClient` by passing `stream=True`. Streaming will return tokens as they are being generated in the server. To use streaming, you can do as follows: + +```python +for token in client.text_generation("How do you make cheese?", max_new_tokens=12, stream=True): + print(token) ``` -To stream tokens in `InferenceClient`, simply pass `stream=True`. Another parameter you can use with TGI backend is `details`. You can get more details on generation (tokens, probabilities, etc.) by setting `details` to `True`. By default, `details` is set to `False`, and `text_generation` returns a string. If you pass `details=True` and `stream=True`, `text_generation` will return a `TextGenerationStreamResponse` which consists of the generated token, generated text, and details. +Another parameter you can use with TGI backend is `details`. You can get more details on generation (tokens, probabilities, etc.) by setting `details` to `True`. When it's specified, TGI will return a `TextGenerationResponse` or `TextGenerationStreamResponse` rather than a string or stream. ```python -output = client.text_generation(prompt="Meaning of life is", model=URL_OF_ENDPOINT, details=True) +output = client.text_generation(prompt="Meaning of life is", details=True) print(output) # TextGenerationResponse(generated_text=' a complex concept that is not always clear to the individual. It is a concept that is not always', details=Details(finish_reason=, generated_tokens=20, seed=None, prefill=[], tokens=[Token(id=267, text=' a', logprob=-2.0723474, special=False), Token(id=11235, text=' complex', logprob=-3.1272552, special=False), Token(id=17908, text=' concept', logprob=-1.3632495, special=False),..)) @@ -45,13 +51,13 @@ print(output) You can see how to stream below. ```python -output = client.text_generation(prompt="Meaning of life is", model="http://localhost:3000/", stream=True, details=True) +output = client.text_generation(prompt="Meaning of life is", stream=True, details=True) print(next(iter(output))) # TextGenerationStreamResponse(token=Token(id=267, text=' a', logprob=-2.0723474, special=False), generated_text=None, details=None) ``` -You can check out the details of the function [here](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation). +You can check out the details of the function [here](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation). There is also an async version of the client, `AsyncInferenceClient`, based on `asyncio` and `aiohttp`. You can find docs for it [here](https://huggingface.co/docs/huggingface_hub/package_reference/inference_client#huggingface_hub.AsyncInferenceClient) ## ChatUI From 2e68ac01c0d331ff587f023c74308080c5860b4d Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 17 Aug 2023 09:05:41 +0200 Subject: [PATCH 388/793] "Fix" for rw-1b. (#860) # What does this PR do? - New "falcon" layout on this repo - No alibi - `transformers` already modifying cache layout in our stead (same modifications). - Output is garbage. Not sure why. Does not fix #826 but it's a step. Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/models/rw.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/server/text_generation_server/models/rw.py b/server/text_generation_server/models/rw.py index 92bb135b..d97c1c73 100644 --- a/server/text_generation_server/models/rw.py +++ b/server/text_generation_server/models/rw.py @@ -67,18 +67,6 @@ def forward( self, input_ids, attention_mask, position_ids, past_key_values: Optional = None ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: # Model Forward - if past_key_values is not None: - reshaped_past_key_values = [] - for layer in past_key_values: - past_keys, past_values = layer - reshaped_past_key_values.append( - ( - past_keys.view(-1, *past_keys.shape[-2:]), - past_values.view(-1, *past_values.shape[-2:]), - ) - ) - past_key_values = reshaped_past_key_values - outputs = self.model.forward( input_ids=input_ids, attention_mask=attention_mask, From b9e33c4953dc237b224b5a4b3a268b799b3e3744 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 17 Aug 2023 09:15:35 +0200 Subject: [PATCH 389/793] Upgrading versions of python client. (#862) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- clients/python/poetry.lock | 1425 ++++++++++++++++++------------------ 1 file changed, 720 insertions(+), 705 deletions(-) diff --git a/clients/python/poetry.lock b/clients/python/poetry.lock index 1503c40e..e038ad9b 100644 --- a/clients/python/poetry.lock +++ b/clients/python/poetry.lock @@ -1,10 +1,100 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + [[package]] name = "aiohttp" -version = "3.8.4" +version = "3.8.5" description = "Async http client/server framework (asyncio)" -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8"}, + {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84"}, + {file = "aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825"}, + {file = "aiohttp-3.8.5-cp310-cp310-win32.whl", hash = "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802"}, + {file = "aiohttp-3.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c"}, + {file = "aiohttp-3.8.5-cp311-cp311-win32.whl", hash = "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945"}, + {file = "aiohttp-3.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755"}, + {file = "aiohttp-3.8.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"}, + {file = "aiohttp-3.8.5-cp36-cp36m-win32.whl", hash = "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e"}, + {file = "aiohttp-3.8.5-cp36-cp36m-win_amd64.whl", hash = "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-win32.whl", hash = "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22"}, + {file = "aiohttp-3.8.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35"}, + {file = "aiohttp-3.8.5-cp38-cp38-win32.whl", hash = "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c"}, + {file = "aiohttp-3.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91"}, + {file = "aiohttp-3.8.5-cp39-cp39-win32.whl", hash = "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67"}, + {file = "aiohttp-3.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c"}, + {file = "aiohttp-3.8.5.tar.gz", hash = "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc"}, +] [package.dependencies] aiosignal = ">=1.1.2" @@ -24,20 +114,26 @@ speedups = ["Brotli", "aiodns", "cchardet"] name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] [package.dependencies] frozenlist = ">=1.1.0" [[package]] name = "async-timeout" -version = "4.0.2" +version = "4.0.3" description = "Timeout context manager for asyncio programs" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] [package.dependencies] typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""} @@ -46,65 +142,218 @@ typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""} name = "asynctest" version = "0.13.0" description = "Enhance the standard unittest package with features for testing asyncio libraries" -category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "asynctest-0.13.0-py3-none-any.whl", hash = "sha256:5da6118a7e6d6b54d83a8f7197769d046922a44d2a99c21382f0a6e4fadae676"}, + {file = "asynctest-0.13.0.tar.gz", hash = "sha256:c27862842d15d83e6a34eb0b2866c323880eb3a75e4485b079ea11748fd77fac"}, +] [[package]] name = "atomicwrites" version = "1.4.1" description = "Atomic file writes." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] [[package]] name = "attrs" -version = "22.2.0" +version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" +files = [ + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, +] + +[package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] -cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] -tests = ["attrs[tests-no-zope]", "zope.interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests_no_zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] [[package]] name = "certifi" -version = "2022.12.7" +version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, +] [[package]] name = "charset-normalizer" -version = "3.1.0" +version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, + {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, +] [[package]] name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "coverage" -version = "7.2.1" +version = "7.2.7" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, + {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, + {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, + {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, + {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, + {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, + {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, + {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, + {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, + {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, + {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, + {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, + {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, + {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, + {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, + {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, + {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, + {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, +] [package.dependencies] tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} @@ -114,34 +363,150 @@ toml = ["tomli"] [[package]] name = "filelock" -version = "3.10.0" +version = "3.12.2" description = "A platform independent file lock." -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, + {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, +] [package.extras] -docs = ["furo (>=2022.12.7)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.1)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] [[package]] name = "frozenlist" version = "1.3.3" description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, + {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, + {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, + {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, + {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, + {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, + {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, + {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, + {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, + {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, + {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, +] + +[[package]] +name = "fsspec" +version = "2023.1.0" +description = "File-system specification" +optional = false +python-versions = ">=3.7" +files = [ + {file = "fsspec-2023.1.0-py3-none-any.whl", hash = "sha256:b833e2e541e9e8cde0ab549414187871243177feb3d344f9d27b25a93f5d8139"}, + {file = "fsspec-2023.1.0.tar.gz", hash = "sha256:fbae7f20ff801eb5f7d0bedf81f25c787c0dfac5e982d98fa3884a9cde2b5411"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +entrypoints = ["importlib-metadata"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] [[package]] name = "huggingface-hub" -version = "0.13.2" +version = "0.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -category = "main" optional = false python-versions = ">=3.7.0" +files = [ + {file = "huggingface_hub-0.16.4-py3-none-any.whl", hash = "sha256:0d3df29932f334fead024afc7cb4cc5149d955238b8b5e42dcf9740d6995a349"}, + {file = "huggingface_hub-0.16.4.tar.gz", hash = "sha256:608c7d4f3d368b326d1747f91523dbd1f692871e8e2e7a4750314a2dd8b63e14"}, +] [package.dependencies] filelock = "*" +fsspec = "*" importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} packaging = ">=20.9" pyyaml = ">=5.1" @@ -150,31 +515,38 @@ tqdm = ">=4.42.1" typing-extensions = ">=3.7.4.3" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +inference = ["aiohttp", "pydantic"] quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] tensorflow = ["graphviz", "pydot", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] torch = ["torch"] -typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] [[package]] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] [[package]] name = "importlib-metadata" -version = "6.0.0" +version = "6.7.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"}, + {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"}, +] [package.dependencies] typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} @@ -183,39 +555,123 @@ zipp = ">=0.5" [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] [[package]] name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] [[package]] name = "multidict" version = "6.0.4" description = "multidict implementation" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] [[package]] name = "packaging" -version = "23.0" +version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] [[package]] name = "pluggy" -version = "1.0.0" +version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" +files = [ + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, +] [package.dependencies] importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} @@ -228,17 +684,57 @@ testing = ["pytest", "pytest-benchmark"] name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] [[package]] name = "pydantic" -version = "1.10.6" +version = "1.10.12" description = "Data validation and settings management using python type hints" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, + {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, + {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, + {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, + {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, + {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, + {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, + {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, + {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, + {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, +] [package.dependencies] typing-extensions = ">=4.2.0" @@ -251,9 +747,12 @@ email = ["email-validator (>=1.0.3)"] name = "pytest" version = "6.2.5" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} @@ -273,9 +772,12 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm name = "pytest-asyncio" version = "0.17.2" description = "Pytest support for asyncio" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pytest-asyncio-0.17.2.tar.gz", hash = "sha256:6d895b02432c028e6957d25fc936494e78c6305736e785d9fee408b1efbc7ff4"}, + {file = "pytest_asyncio-0.17.2-py3-none-any.whl", hash = "sha256:e0fe5dbea40516b661ef1bcfe0bd9461c2847c4ef4bb40012324f2454fb7d56d"}, +] [package.dependencies] pytest = ">=6.1.0" @@ -288,9 +790,12 @@ testing = ["coverage (==6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "pytest-cov" version = "3.0.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, + {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, +] [package.dependencies] coverage = {version = ">=5.2.1", extras = ["toml"]} @@ -300,92 +805,227 @@ pytest = ">=4.6" testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] [[package]] -name = "PyYAML" -version = "6.0" +name = "pyyaml" +version = "6.0.1" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] [[package]] name = "requests" -version = "2.28.2" +version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false -python-versions = ">=3.7, <4" +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] [package.dependencies] certifi = ">=2017.4.17" charset-normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" +urllib3 = ">=1.21.1,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] [[package]] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] [[package]] name = "tqdm" -version = "4.65.0" +version = "4.66.1" description = "Fast, Extensible Progress Meter" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, + {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] [[package]] name = "typing-extensions" -version = "4.5.0" +version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, +] [[package]] name = "urllib3" -version = "1.26.15" +version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7" +files = [ + {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, + {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, +] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "yarl" -version = "1.8.2" +version = "1.9.2" description = "Yet another URL library" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, + {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, + {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, + {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, + {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, + {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, + {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, + {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, + {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, + {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, + {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, + {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, + {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, +] [package.dependencies] idna = ">=2.0" @@ -396,643 +1036,18 @@ typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, +] [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] -lock-version = "1.1" +lock-version = "2.0" python-versions = "^3.7" content-hash = "0db2f97d52c557dd7f90c55b4ad5bbe308c957c5f7f99fec53c57e0a13822cb4" - -[metadata.files] -aiohttp = [ - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea"}, - {file = "aiohttp-3.8.4-cp310-cp310-win32.whl", hash = "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1"}, - {file = "aiohttp-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24"}, - {file = "aiohttp-3.8.4-cp311-cp311-win32.whl", hash = "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d"}, - {file = "aiohttp-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc"}, - {file = "aiohttp-3.8.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win32.whl", hash = "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e"}, - {file = "aiohttp-3.8.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win32.whl", hash = "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d"}, - {file = "aiohttp-3.8.4-cp38-cp38-win32.whl", hash = "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54"}, - {file = "aiohttp-3.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4"}, - {file = "aiohttp-3.8.4-cp39-cp39-win32.whl", hash = "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a"}, - {file = "aiohttp-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04"}, - {file = "aiohttp-3.8.4.tar.gz", hash = "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c"}, -] -aiosignal = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, -] -async-timeout = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, -] -asynctest = [ - {file = "asynctest-0.13.0-py3-none-any.whl", hash = "sha256:5da6118a7e6d6b54d83a8f7197769d046922a44d2a99c21382f0a6e4fadae676"}, - {file = "asynctest-0.13.0.tar.gz", hash = "sha256:c27862842d15d83e6a34eb0b2866c323880eb3a75e4485b079ea11748fd77fac"}, -] -atomicwrites = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] -attrs = [ - {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, - {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, -] -certifi = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] -charset-normalizer = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -coverage = [ - {file = "coverage-7.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49567ec91fc5e0b15356da07a2feabb421d62f52a9fff4b1ec40e9e19772f5f8"}, - {file = "coverage-7.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2ef6cae70168815ed91388948b5f4fcc69681480a0061114db737f957719f03"}, - {file = "coverage-7.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3004765bca3acd9e015794e5c2f0c9a05587f5e698127ff95e9cfba0d3f29339"}, - {file = "coverage-7.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cca7c0b7f5881dfe0291ef09ba7bb1582cb92ab0aeffd8afb00c700bf692415a"}, - {file = "coverage-7.2.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2167d116309f564af56f9aa5e75ef710ef871c5f9b313a83050035097b56820"}, - {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cb5f152fb14857cbe7f3e8c9a5d98979c4c66319a33cad6e617f0067c9accdc4"}, - {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:87dc37f16fb5e3a28429e094145bf7c1753e32bb50f662722e378c5851f7fdc6"}, - {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e191a63a05851f8bce77bc875e75457f9b01d42843f8bd7feed2fc26bbe60833"}, - {file = "coverage-7.2.1-cp310-cp310-win32.whl", hash = "sha256:e3ea04b23b114572b98a88c85379e9e9ae031272ba1fb9b532aa934c621626d4"}, - {file = "coverage-7.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:0cf557827be7eca1c38a2480484d706693e7bb1929e129785fe59ec155a59de6"}, - {file = "coverage-7.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:570c21a29493b350f591a4b04c158ce1601e8d18bdcd21db136fbb135d75efa6"}, - {file = "coverage-7.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e872b082b32065ac2834149dc0adc2a2e6d8203080501e1e3c3c77851b466f9"}, - {file = "coverage-7.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fac6343bae03b176e9b58104a9810df3cdccd5cfed19f99adfa807ffbf43cf9b"}, - {file = "coverage-7.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abacd0a738e71b20e224861bc87e819ef46fedba2fb01bc1af83dfd122e9c319"}, - {file = "coverage-7.2.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9256d4c60c4bbfec92721b51579c50f9e5062c21c12bec56b55292464873508"}, - {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:80559eaf6c15ce3da10edb7977a1548b393db36cbc6cf417633eca05d84dd1ed"}, - {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0bd7e628f6c3ec4e7d2d24ec0e50aae4e5ae95ea644e849d92ae4805650b4c4e"}, - {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09643fb0df8e29f7417adc3f40aaf379d071ee8f0350ab290517c7004f05360b"}, - {file = "coverage-7.2.1-cp311-cp311-win32.whl", hash = "sha256:1b7fb13850ecb29b62a447ac3516c777b0e7a09ecb0f4bb6718a8654c87dfc80"}, - {file = "coverage-7.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:617a94ada56bbfe547aa8d1b1a2b8299e2ec1ba14aac1d4b26a9f7d6158e1273"}, - {file = "coverage-7.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8649371570551d2fd7dee22cfbf0b61f1747cdfb2b7587bb551e4beaaa44cb97"}, - {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d2b9b5e70a21474c105a133ba227c61bc95f2ac3b66861143ce39a5ea4b3f84"}, - {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae82c988954722fa07ec5045c57b6d55bc1a0890defb57cf4a712ced65b26ddd"}, - {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:861cc85dfbf55a7a768443d90a07e0ac5207704a9f97a8eb753292a7fcbdfcfc"}, - {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0339dc3237c0d31c3b574f19c57985fcbe494280153bbcad33f2cdf469f4ac3e"}, - {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:5928b85416a388dd557ddc006425b0c37e8468bd1c3dc118c1a3de42f59e2a54"}, - {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8d3843ca645f62c426c3d272902b9de90558e9886f15ddf5efe757b12dd376f5"}, - {file = "coverage-7.2.1-cp37-cp37m-win32.whl", hash = "sha256:6a034480e9ebd4e83d1aa0453fd78986414b5d237aea89a8fdc35d330aa13bae"}, - {file = "coverage-7.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6fce673f79a0e017a4dc35e18dc7bb90bf6d307c67a11ad5e61ca8d42b87cbff"}, - {file = "coverage-7.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f099da6958ddfa2ed84bddea7515cb248583292e16bb9231d151cd528eab657"}, - {file = "coverage-7.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:97a3189e019d27e914ecf5c5247ea9f13261d22c3bb0cfcfd2a9b179bb36f8b1"}, - {file = "coverage-7.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a81dbcf6c6c877986083d00b834ac1e84b375220207a059ad45d12f6e518a4e3"}, - {file = "coverage-7.2.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d2c3dde4c0b9be4b02067185136b7ee4681978228ad5ec1278fa74f5ca3e99"}, - {file = "coverage-7.2.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a209d512d157379cc9ab697cbdbb4cfd18daa3e7eebaa84c3d20b6af0037384"}, - {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f3d07edb912a978915576a776756069dede66d012baa503022d3a0adba1b6afa"}, - {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8dca3c1706670297851bca1acff9618455122246bdae623be31eca744ade05ec"}, - {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b1991a6d64231a3e5bbe3099fb0dd7c9aeaa4275ad0e0aeff4cb9ef885c62ba2"}, - {file = "coverage-7.2.1-cp38-cp38-win32.whl", hash = "sha256:22c308bc508372576ffa3d2dbc4824bb70d28eeb4fcd79d4d1aed663a06630d0"}, - {file = "coverage-7.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:b0c0d46de5dd97f6c2d1b560bf0fcf0215658097b604f1840365296302a9d1fb"}, - {file = "coverage-7.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4dd34a935de268a133e4741827ae951283a28c0125ddcdbcbba41c4b98f2dfef"}, - {file = "coverage-7.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0f8318ed0f3c376cfad8d3520f496946977abde080439d6689d7799791457454"}, - {file = "coverage-7.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:834c2172edff5a08d78e2f53cf5e7164aacabeb66b369f76e7bb367ca4e2d993"}, - {file = "coverage-7.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4d70c853f0546855f027890b77854508bdb4d6a81242a9d804482e667fff6e6"}, - {file = "coverage-7.2.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a6450da4c7afc4534305b2b7d8650131e130610cea448ff240b6ab73d7eab63"}, - {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:99f4dd81b2bb8fc67c3da68b1f5ee1650aca06faa585cbc6818dbf67893c6d58"}, - {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bdd3f2f285ddcf2e75174248b2406189261a79e7fedee2ceeadc76219b6faa0e"}, - {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f29351393eb05e6326f044a7b45ed8e38cb4dcc38570d12791f271399dc41431"}, - {file = "coverage-7.2.1-cp39-cp39-win32.whl", hash = "sha256:e2b50ebc2b6121edf352336d503357321b9d8738bb7a72d06fc56153fd3f4cd8"}, - {file = "coverage-7.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:bd5a12239c0006252244f94863f1c518ac256160cd316ea5c47fb1a11b25889a"}, - {file = "coverage-7.2.1-pp37.pp38.pp39-none-any.whl", hash = "sha256:436313d129db7cf5b4ac355dd2bd3f7c7e5294af077b090b85de75f8458b8616"}, - {file = "coverage-7.2.1.tar.gz", hash = "sha256:c77f2a9093ccf329dd523a9b2b3c854c20d2a3d968b6def3b820272ca6732242"}, -] -filelock = [ - {file = "filelock-3.10.0-py3-none-any.whl", hash = "sha256:e90b34656470756edf8b19656785c5fea73afa1953f3e1b0d645cef11cab3182"}, - {file = "filelock-3.10.0.tar.gz", hash = "sha256:3199fd0d3faea8b911be52b663dfccceb84c95949dd13179aa21436d1a79c4ce"}, -] -frozenlist = [ - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, - {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, - {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, - {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, - {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, - {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, - {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, - {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, - {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, - {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, - {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, -] -huggingface-hub = [ - {file = "huggingface_hub-0.13.2-py3-none-any.whl", hash = "sha256:745c4cbd97a27fc5c1c6c89cb477662004c88bc3dd89bafc1a27ef24af77f944"}, - {file = "huggingface_hub-0.13.2.tar.gz", hash = "sha256:246e8eb39b6e6e9d9d5846e4b56c265cdf1872f48ba5a13a1321295d371626f5"}, -] -idna = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] -importlib-metadata = [ - {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, - {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, -] -iniconfig = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] -multidict = [ - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, - {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, - {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, - {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, - {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, - {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, - {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, - {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, - {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, - {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, - {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, - {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, - {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, -] -packaging = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, -] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] -py = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, -] -pydantic = [ - {file = "pydantic-1.10.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f9289065611c48147c1dd1fd344e9d57ab45f1d99b0fb26c51f1cf72cd9bcd31"}, - {file = "pydantic-1.10.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c32b6bba301490d9bb2bf5f631907803135e8085b6aa3e5fe5a770d46dd0160"}, - {file = "pydantic-1.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd9b9e98068fa1068edfc9eabde70a7132017bdd4f362f8b4fd0abed79c33083"}, - {file = "pydantic-1.10.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c84583b9df62522829cbc46e2b22e0ec11445625b5acd70c5681ce09c9b11c4"}, - {file = "pydantic-1.10.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b41822064585fea56d0116aa431fbd5137ce69dfe837b599e310034171996084"}, - {file = "pydantic-1.10.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:61f1f08adfaa9cc02e0cbc94f478140385cbd52d5b3c5a657c2fceb15de8d1fb"}, - {file = "pydantic-1.10.6-cp310-cp310-win_amd64.whl", hash = "sha256:32937835e525d92c98a1512218db4eed9ddc8f4ee2a78382d77f54341972c0e7"}, - {file = "pydantic-1.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbd5c531b22928e63d0cb1868dee76123456e1de2f1cb45879e9e7a3f3f1779b"}, - {file = "pydantic-1.10.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e277bd18339177daa62a294256869bbe84df1fb592be2716ec62627bb8d7c81d"}, - {file = "pydantic-1.10.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f15277d720aa57e173954d237628a8d304896364b9de745dcb722f584812c7"}, - {file = "pydantic-1.10.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b243b564cea2576725e77aeeda54e3e0229a168bc587d536cd69941e6797543d"}, - {file = "pydantic-1.10.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3ce13a558b484c9ae48a6a7c184b1ba0e5588c5525482681db418268e5f86186"}, - {file = "pydantic-1.10.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3ac1cd4deed871dfe0c5f63721e29debf03e2deefa41b3ed5eb5f5df287c7b70"}, - {file = "pydantic-1.10.6-cp311-cp311-win_amd64.whl", hash = "sha256:b1eb6610330a1dfba9ce142ada792f26bbef1255b75f538196a39e9e90388bf4"}, - {file = "pydantic-1.10.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4ca83739c1263a044ec8b79df4eefc34bbac87191f0a513d00dd47d46e307a65"}, - {file = "pydantic-1.10.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea4e2a7cb409951988e79a469f609bba998a576e6d7b9791ae5d1e0619e1c0f2"}, - {file = "pydantic-1.10.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53de12b4608290992a943801d7756f18a37b7aee284b9ffa794ee8ea8153f8e2"}, - {file = "pydantic-1.10.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:60184e80aac3b56933c71c48d6181e630b0fbc61ae455a63322a66a23c14731a"}, - {file = "pydantic-1.10.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:415a3f719ce518e95a92effc7ee30118a25c3d032455d13e121e3840985f2efd"}, - {file = "pydantic-1.10.6-cp37-cp37m-win_amd64.whl", hash = "sha256:72cb30894a34d3a7ab6d959b45a70abac8a2a93b6480fc5a7bfbd9c935bdc4fb"}, - {file = "pydantic-1.10.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3091d2eaeda25391405e36c2fc2ed102b48bac4b384d42b2267310abae350ca6"}, - {file = "pydantic-1.10.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:751f008cd2afe812a781fd6aa2fb66c620ca2e1a13b6a2152b1ad51553cb4b77"}, - {file = "pydantic-1.10.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12e837fd320dd30bd625be1b101e3b62edc096a49835392dcf418f1a5ac2b832"}, - {file = "pydantic-1.10.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:587d92831d0115874d766b1f5fddcdde0c5b6c60f8c6111a394078ec227fca6d"}, - {file = "pydantic-1.10.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:476f6674303ae7965730a382a8e8d7fae18b8004b7b69a56c3d8fa93968aa21c"}, - {file = "pydantic-1.10.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3a2be0a0f32c83265fd71a45027201e1278beaa82ea88ea5b345eea6afa9ac7f"}, - {file = "pydantic-1.10.6-cp38-cp38-win_amd64.whl", hash = "sha256:0abd9c60eee6201b853b6c4be104edfba4f8f6c5f3623f8e1dba90634d63eb35"}, - {file = "pydantic-1.10.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6195ca908045054dd2d57eb9c39a5fe86409968b8040de8c2240186da0769da7"}, - {file = "pydantic-1.10.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43cdeca8d30de9a897440e3fb8866f827c4c31f6c73838e3a01a14b03b067b1d"}, - {file = "pydantic-1.10.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c19eb5163167489cb1e0161ae9220dadd4fc609a42649e7e84a8fa8fff7a80f"}, - {file = "pydantic-1.10.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:012c99a9c0d18cfde7469aa1ebff922e24b0c706d03ead96940f5465f2c9cf62"}, - {file = "pydantic-1.10.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:528dcf7ec49fb5a84bf6fe346c1cc3c55b0e7603c2123881996ca3ad79db5bfc"}, - {file = "pydantic-1.10.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:163e79386c3547c49366e959d01e37fc30252285a70619ffc1b10ede4758250a"}, - {file = "pydantic-1.10.6-cp39-cp39-win_amd64.whl", hash = "sha256:189318051c3d57821f7233ecc94708767dd67687a614a4e8f92b4a020d4ffd06"}, - {file = "pydantic-1.10.6-py3-none-any.whl", hash = "sha256:acc6783751ac9c9bc4680379edd6d286468a1dc8d7d9906cd6f1186ed682b2b0"}, - {file = "pydantic-1.10.6.tar.gz", hash = "sha256:cf95adb0d1671fc38d8c43dd921ad5814a735e7d9b4d9e437c088002863854fd"}, -] -pytest = [ - {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, - {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, -] -pytest-asyncio = [ - {file = "pytest-asyncio-0.17.2.tar.gz", hash = "sha256:6d895b02432c028e6957d25fc936494e78c6305736e785d9fee408b1efbc7ff4"}, - {file = "pytest_asyncio-0.17.2-py3-none-any.whl", hash = "sha256:e0fe5dbea40516b661ef1bcfe0bd9461c2847c4ef4bb40012324f2454fb7d56d"}, -] -pytest-cov = [ - {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, - {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, -] -PyYAML = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] -requests = [ - {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, - {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, -] -toml = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -tqdm = [ - {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, - {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, -] -typing-extensions = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, -] -urllib3 = [ - {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, - {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, -] -yarl = [ - {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, - {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, - {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80"}, - {file = "yarl-1.8.2-cp310-cp310-win32.whl", hash = "sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42"}, - {file = "yarl-1.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574"}, - {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634"}, - {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd"}, - {file = "yarl-1.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2"}, - {file = "yarl-1.8.2-cp311-cp311-win32.whl", hash = "sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b"}, - {file = "yarl-1.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c"}, - {file = "yarl-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37"}, - {file = "yarl-1.8.2-cp37-cp37m-win32.whl", hash = "sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89"}, - {file = "yarl-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5"}, - {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1"}, - {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918"}, - {file = "yarl-1.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b372aad2b5f81db66ee7ec085cbad72c4da660d994e8e590c997e9b01e44901"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e6f3515aafe0209dd17fb9bdd3b4e892963370b3de781f53e1746a521fb39fc0"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dfef7350ee369197106805e193d420b75467b6cceac646ea5ed3049fcc950a05"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:728be34f70a190566d20aa13dc1f01dc44b6aa74580e10a3fb159691bc76909d"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baf211dcad448a87a0d9047dc8282d7de59473ade7d7fdf22150b1d23859f946"}, - {file = "yarl-1.8.2-cp38-cp38-win32.whl", hash = "sha256:272b4f1599f1b621bf2aabe4e5b54f39a933971f4e7c9aa311d6d7dc06965165"}, - {file = "yarl-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:326dd1d3caf910cd26a26ccbfb84c03b608ba32499b5d6eeb09252c920bcbe4f"}, - {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f8ca8ad414c85bbc50f49c0a106f951613dfa5f948ab69c10ce9b128d368baf8"}, - {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:418857f837347e8aaef682679f41e36c24250097f9e2f315d39bae3a99a34cbf"}, - {file = "yarl-1.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ae0eec05ab49e91a78700761777f284c2df119376e391db42c38ab46fd662b77"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3edac5d74bb3209c418805bda77f973117836e1de7c000e9755e572c1f7850d0"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da65c3f263729e47351261351b8679c6429151ef9649bba08ef2528ff2c423b2"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef8fb25e52663a1c85d608f6dd72e19bd390e2ecaf29c17fb08f730226e3a08"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcd7bb1e5c45274af9a1dd7494d3c52b2be5e6bd8d7e49c612705fd45420b12d"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44ceac0450e648de86da8e42674f9b7077d763ea80c8ceb9d1c3e41f0f0a9951"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:97209cc91189b48e7cfe777237c04af8e7cc51eb369004e061809bcdf4e55220"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:48dd18adcf98ea9cd721a25313aef49d70d413a999d7d89df44f469edfb38a06"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e59399dda559688461762800d7fb34d9e8a6a7444fd76ec33220a926c8be1516"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d617c241c8c3ad5c4e78a08429fa49e4b04bedfc507b34b4d8dceb83b4af3588"}, - {file = "yarl-1.8.2-cp39-cp39-win32.whl", hash = "sha256:cb6d48d80a41f68de41212f3dfd1a9d9898d7841c8f7ce6696cf2fd9cb57ef83"}, - {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, - {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, -] -zipp = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, -] From bce5e22444aab16a3e3cc0a9b7caeab5ad33b16e Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 17 Aug 2023 14:38:49 +0200 Subject: [PATCH 390/793] Adding Idefics multi modal model. (#842) Co-Authored-By: Victor Sanh # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Victor Sanh --- .../test_idefics/test_idefics.json | 168 +++ .../test_idefics/test_idefics_load.json | 674 +++++++++ integration-tests/models/test_idefics.py | 46 + server/poetry.lock | 639 ++++---- server/pyproject.toml | 3 +- server/requirements.txt | 20 +- .../text_generation_server/models/__init__.py | 13 + .../models/custom_modeling/idefics_config.py | 323 ++++ .../idefics_image_processing.py | 264 ++++ .../custom_modeling/idefics_modeling.py | 1327 +++++++++++++++++ .../custom_modeling/idefics_perceiver.py | 246 +++ .../custom_modeling/idefics_processing.py | 413 +++++ .../models/custom_modeling/idefics_vision.py | 476 ++++++ .../text_generation_server/models/idefics.py | 91 ++ .../models/idefics_causal_lm.py | 806 ++++++++++ server/text_generation_server/server.py | 25 +- server/text_generation_server/utils/layers.py | 24 + 17 files changed, 5227 insertions(+), 331 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_idefics/test_idefics.json create mode 100644 integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json create mode 100644 integration-tests/models/test_idefics.py create mode 100644 server/text_generation_server/models/custom_modeling/idefics_config.py create mode 100644 server/text_generation_server/models/custom_modeling/idefics_image_processing.py create mode 100644 server/text_generation_server/models/custom_modeling/idefics_modeling.py create mode 100644 server/text_generation_server/models/custom_modeling/idefics_perceiver.py create mode 100644 server/text_generation_server/models/custom_modeling/idefics_processing.py create mode 100644 server/text_generation_server/models/custom_modeling/idefics_vision.py create mode 100644 server/text_generation_server/models/idefics.py create mode 100644 server/text_generation_server/models/idefics_causal_lm.py diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json new file mode 100644 index 00000000..1df5761d --- /dev/null +++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json @@ -0,0 +1,168 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4911, + "logprob": -5.3632812, + "text": "User" + }, + { + "id": 29901, + "logprob": -0.00762558, + "text": ":" + }, + { + "id": 32000, + "logprob": -0.7739258, + "text": "" + }, + { + "id": 32001, + "logprob": -9.775162e-05, + "text": "" + }, + { + "id": 32000, + "logprob": -1.1920929e-07, + "text": "" + }, + { + "id": 1815, + "logprob": -4.4140625, + "text": "Can" + }, + { + "id": 366, + "logprob": -0.01436615, + "text": "you" + }, + { + "id": 2649, + "logprob": -4.9414062, + "text": "tell" + }, + { + "id": 592, + "logprob": -0.3005371, + "text": "me" + }, + { + "id": 263, + "logprob": -3.5703125, + "text": "a" + }, + { + "id": 1407, + "logprob": -9.4296875, + "text": "very" + }, + { + "id": 3273, + "logprob": -1.9111328, + "text": "short" + }, + { + "id": 5828, + "logprob": -0.28881836, + "text": "story" + }, + { + "id": 2729, + "logprob": -3.4179688, + "text": "based" + }, + { + "id": 373, + "logprob": -0.00056886673, + "text": "on" + }, + { + "id": 278, + "logprob": -0.14123535, + "text": "the" + }, + { + "id": 1967, + "logprob": -0.053985596, + "text": "image" + }, + { + "id": 29973, + "logprob": -0.15771484, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 32002, + "logprob": -0.004295349, + "special": true, + "text": "" + }, + { + "id": 29871, + "logprob": -7.43866e-05, + "special": false, + "text": " " + }, + { + "id": 13, + "logprob": -2.3126602e-05, + "special": false, + "text": "\n" + }, + { + "id": 7900, + "logprob": -3.9339066e-06, + "special": false, + "text": "Ass" + }, + { + "id": 22137, + "logprob": 0.0, + "special": false, + "text": "istant" + }, + { + "id": 29901, + "logprob": -2.6226044e-06, + "special": false, + "text": ":" + }, + { + "id": 319, + "logprob": -0.87841797, + "special": false, + "text": " A" + }, + { + "id": 521, + "logprob": -1.3837891, + "special": false, + "text": " ch" + }, + { + "id": 21475, + "logprob": -0.00051641464, + "special": false, + "text": "icken" + }, + { + "id": 338, + "logprob": -1.1435547, + "special": false, + "text": " is" + } + ] + }, + "generated_text": "\nAssistant: A chicken is" +} diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json new file mode 100644 index 00000000..e0dfe96c --- /dev/null +++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json @@ -0,0 +1,674 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4911, + "logprob": -5.3476562, + "text": "User" + }, + { + "id": 29901, + "logprob": -0.0075531006, + "text": ":" + }, + { + "id": 32000, + "logprob": -0.7729492, + "text": "" + }, + { + "id": 32001, + "logprob": -9.787083e-05, + "text": "" + }, + { + "id": 32000, + "logprob": -1.1920929e-07, + "text": "" + }, + { + "id": 1815, + "logprob": -4.4296875, + "text": "Can" + }, + { + "id": 366, + "logprob": -0.01424408, + "text": "you" + }, + { + "id": 2649, + "logprob": -4.9335938, + "text": "tell" + }, + { + "id": 592, + "logprob": -0.2993164, + "text": "me" + }, + { + "id": 263, + "logprob": -3.5664062, + "text": "a" + }, + { + "id": 1407, + "logprob": -9.4453125, + "text": "very" + }, + { + "id": 3273, + "logprob": -1.9306641, + "text": "short" + }, + { + "id": 5828, + "logprob": -0.2836914, + "text": "story" + }, + { + "id": 2729, + "logprob": -3.4179688, + "text": "based" + }, + { + "id": 373, + "logprob": -0.00056934357, + "text": "on" + }, + { + "id": 278, + "logprob": -0.13928223, + "text": "the" + }, + { + "id": 1967, + "logprob": -0.05355835, + "text": "image" + }, + { + "id": 29973, + "logprob": -0.15771484, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 32002, + "logprob": -0.004333496, + "special": true, + "text": "" + }, + { + "id": 29871, + "logprob": -7.426739e-05, + "special": false, + "text": " " + }, + { + "id": 13, + "logprob": -2.348423e-05, + "special": false, + "text": "\n" + }, + { + "id": 7900, + "logprob": -3.9339066e-06, + "special": false, + "text": "Ass" + }, + { + "id": 22137, + "logprob": 0.0, + "special": false, + "text": "istant" + }, + { + "id": 29901, + "logprob": -2.861023e-06, + "special": false, + "text": ":" + }, + { + "id": 319, + "logprob": -0.8828125, + "special": false, + "text": " A" + }, + { + "id": 521, + "logprob": -1.3759766, + "special": false, + "text": " ch" + }, + { + "id": 21475, + "logprob": -0.0005083084, + "special": false, + "text": "icken" + }, + { + "id": 338, + "logprob": -1.1367188, + "special": false, + "text": " is" + } + ] + }, + "generated_text": "\nAssistant: A chicken is" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4911, + "logprob": -5.3476562, + "text": "User" + }, + { + "id": 29901, + "logprob": -0.0075531006, + "text": ":" + }, + { + "id": 32000, + "logprob": -0.7729492, + "text": "" + }, + { + "id": 32001, + "logprob": -9.787083e-05, + "text": "" + }, + { + "id": 32000, + "logprob": -1.1920929e-07, + "text": "" + }, + { + "id": 1815, + "logprob": -4.4296875, + "text": "Can" + }, + { + "id": 366, + "logprob": -0.01423645, + "text": "you" + }, + { + "id": 2649, + "logprob": -4.9335938, + "text": "tell" + }, + { + "id": 592, + "logprob": -0.2993164, + "text": "me" + }, + { + "id": 263, + "logprob": -3.5664062, + "text": "a" + }, + { + "id": 1407, + "logprob": -9.4453125, + "text": "very" + }, + { + "id": 3273, + "logprob": -1.9306641, + "text": "short" + }, + { + "id": 5828, + "logprob": -0.2836914, + "text": "story" + }, + { + "id": 2729, + "logprob": -3.4179688, + "text": "based" + }, + { + "id": 373, + "logprob": -0.00056934357, + "text": "on" + }, + { + "id": 278, + "logprob": -0.13928223, + "text": "the" + }, + { + "id": 1967, + "logprob": -0.05355835, + "text": "image" + }, + { + "id": 29973, + "logprob": -0.15771484, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 32002, + "logprob": -0.004333496, + "special": true, + "text": "" + }, + { + "id": 29871, + "logprob": -7.4505806e-05, + "special": false, + "text": " " + }, + { + "id": 13, + "logprob": -2.3722649e-05, + "special": false, + "text": "\n" + }, + { + "id": 7900, + "logprob": -3.9339066e-06, + "special": false, + "text": "Ass" + }, + { + "id": 22137, + "logprob": 0.0, + "special": false, + "text": "istant" + }, + { + "id": 29901, + "logprob": -2.861023e-06, + "special": false, + "text": ":" + }, + { + "id": 319, + "logprob": -0.8828125, + "special": false, + "text": " A" + }, + { + "id": 521, + "logprob": -1.3759766, + "special": false, + "text": " ch" + }, + { + "id": 21475, + "logprob": -0.00050878525, + "special": false, + "text": "icken" + }, + { + "id": 338, + "logprob": -1.1367188, + "special": false, + "text": " is" + } + ] + }, + "generated_text": "\nAssistant: A chicken is" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4911, + "logprob": -5.3476562, + "text": "User" + }, + { + "id": 29901, + "logprob": -0.0075531006, + "text": ":" + }, + { + "id": 32000, + "logprob": -0.7729492, + "text": "" + }, + { + "id": 32001, + "logprob": -9.775162e-05, + "text": "" + }, + { + "id": 32000, + "logprob": -1.1920929e-07, + "text": "" + }, + { + "id": 1815, + "logprob": -4.4296875, + "text": "Can" + }, + { + "id": 366, + "logprob": -0.01424408, + "text": "you" + }, + { + "id": 2649, + "logprob": -4.9335938, + "text": "tell" + }, + { + "id": 592, + "logprob": -0.2993164, + "text": "me" + }, + { + "id": 263, + "logprob": -3.5664062, + "text": "a" + }, + { + "id": 1407, + "logprob": -9.4453125, + "text": "very" + }, + { + "id": 3273, + "logprob": -1.9306641, + "text": "short" + }, + { + "id": 5828, + "logprob": -0.2836914, + "text": "story" + }, + { + "id": 2729, + "logprob": -3.4179688, + "text": "based" + }, + { + "id": 373, + "logprob": -0.00056934357, + "text": "on" + }, + { + "id": 278, + "logprob": -0.13928223, + "text": "the" + }, + { + "id": 1967, + "logprob": -0.05355835, + "text": "image" + }, + { + "id": 29973, + "logprob": -0.15771484, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 32002, + "logprob": -0.004333496, + "special": true, + "text": "" + }, + { + "id": 29871, + "logprob": -7.43866e-05, + "special": false, + "text": " " + }, + { + "id": 13, + "logprob": -2.360344e-05, + "special": false, + "text": "\n" + }, + { + "id": 7900, + "logprob": -3.9339066e-06, + "special": false, + "text": "Ass" + }, + { + "id": 22137, + "logprob": 0.0, + "special": false, + "text": "istant" + }, + { + "id": 29901, + "logprob": -2.7418137e-06, + "special": false, + "text": ":" + }, + { + "id": 319, + "logprob": -0.8828125, + "special": false, + "text": " A" + }, + { + "id": 521, + "logprob": -1.3759766, + "special": false, + "text": " ch" + }, + { + "id": 21475, + "logprob": -0.00050878525, + "special": false, + "text": "icken" + }, + { + "id": 338, + "logprob": -1.1367188, + "special": false, + "text": " is" + } + ] + }, + "generated_text": "\nAssistant: A chicken is" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4911, + "logprob": -5.3632812, + "text": "User" + }, + { + "id": 29901, + "logprob": -0.00762558, + "text": ":" + }, + { + "id": 32000, + "logprob": -0.7739258, + "text": "" + }, + { + "id": 32001, + "logprob": -9.775162e-05, + "text": "" + }, + { + "id": 32000, + "logprob": -1.1920929e-07, + "text": "" + }, + { + "id": 1815, + "logprob": -4.4140625, + "text": "Can" + }, + { + "id": 366, + "logprob": -0.01436615, + "text": "you" + }, + { + "id": 2649, + "logprob": -4.9414062, + "text": "tell" + }, + { + "id": 592, + "logprob": -0.3005371, + "text": "me" + }, + { + "id": 263, + "logprob": -3.5703125, + "text": "a" + }, + { + "id": 1407, + "logprob": -9.4296875, + "text": "very" + }, + { + "id": 3273, + "logprob": -1.9111328, + "text": "short" + }, + { + "id": 5828, + "logprob": -0.28881836, + "text": "story" + }, + { + "id": 2729, + "logprob": -3.4179688, + "text": "based" + }, + { + "id": 373, + "logprob": -0.00056886673, + "text": "on" + }, + { + "id": 278, + "logprob": -0.14123535, + "text": "the" + }, + { + "id": 1967, + "logprob": -0.053985596, + "text": "image" + }, + { + "id": 29973, + "logprob": -0.15771484, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 32002, + "logprob": -0.004295349, + "special": true, + "text": "" + }, + { + "id": 29871, + "logprob": -7.43866e-05, + "special": false, + "text": " " + }, + { + "id": 13, + "logprob": -2.3126602e-05, + "special": false, + "text": "\n" + }, + { + "id": 7900, + "logprob": -3.9339066e-06, + "special": false, + "text": "Ass" + }, + { + "id": 22137, + "logprob": 0.0, + "special": false, + "text": "istant" + }, + { + "id": 29901, + "logprob": -2.6226044e-06, + "special": false, + "text": ":" + }, + { + "id": 319, + "logprob": -0.87841797, + "special": false, + "text": " A" + }, + { + "id": 521, + "logprob": -1.3837891, + "special": false, + "text": " ch" + }, + { + "id": 21475, + "logprob": -0.00051641464, + "special": false, + "text": "icken" + }, + { + "id": 338, + "logprob": -1.1435547, + "special": false, + "text": " is" + } + ] + }, + "generated_text": "\nAssistant: A chicken is" + } +] diff --git a/integration-tests/models/test_idefics.py b/integration-tests/models/test_idefics.py new file mode 100644 index 00000000..5659dd5c --- /dev/null +++ b/integration-tests/models/test_idefics.py @@ -0,0 +1,46 @@ +import pytest + + +@pytest.fixture(scope="module") +def idefics_handle(launcher): + with launcher( + "HuggingFaceM4/idefics-9b-instruct", num_shard=2 + ) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def idefics(idefics_handle): + await idefics_handle.health(300) + return idefics_handle.client + + +@pytest.mark.asyncio +async def test_idefics(idefics, response_snapshot): + response = await idefics.generate( + "User:![](https://temp-5681.s3.us-west-2.amazonaws.com/chicken_on_money.png)Can you tell me a very short story based on the image?", + max_new_tokens=10, + decoder_input_details=True, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +async def test_idefics_load(idefics, generate_load, response_snapshot): + responses = await generate_load( + idefics, + "User:![](https://temp-5681.s3.us-west-2.amazonaws.com/chicken_on_money.png)Can you tell me a very short story based on the image?", + max_new_tokens=10, + n=4, + ) + + generated_texts = [r.generated_text for r in responses] + + assert len(generated_texts) == 4 + assert generated_texts, all( + [text == generated_texts[0] for text in generated_texts] + ) + + assert responses == response_snapshot diff --git a/server/poetry.lock b/server/poetry.lock index 4a4e4528..aae7b7a5 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -152,13 +152,13 @@ frozenlist = ">=1.1.0" [[package]] name = "async-timeout" -version = "4.0.2" +version = "4.0.3" description = "Timeout context manager for asyncio programs" optional = true -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, ] [[package]] @@ -310,35 +310,6 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -[[package]] -name = "cmake" -version = "3.27.0" -description = "CMake is an open-source, cross-platform family of tools designed to build, test and package software" -optional = false -python-versions = "*" -files = [ - {file = "cmake-3.27.0-py2.py3-none-macosx_10_10_universal2.macosx_10_10_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:9ccab4cd93578d3c2df32e66b44b313b75a7484032645040431dc06a583ca4aa"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2010_i686.manylinux_2_12_i686.whl", hash = "sha256:199bfaefb752e82d8067aeee5d6a6e0414fe0d60e9a3fd08e95d537a97e0db16"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:8745eff805f36762d3e8e904698b853cb4a9da8b4b07d1c12bcd1e1a6c4a1709"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58a3f39d3d1bc897f05e531bfa676246a2b25d424c6a47e4b6bbc193fb560db7"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b470ccd3f86cf19a63f6b221c9cceebcc58e32d3787d0d5f9f43d1d91a095090"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:35a8d397ce883e93b5e6561e2803ce9470df52283862264093c1078530f98189"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1f38d87b2c65763a0113f4a6c652e6f4b5adf90b384c1e1d69e4f8a3104a57d6"}, - {file = "cmake-3.27.0-py2.py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b9d5811954dcedcaa6c915c4a9bb6d64b55ac189e9cbc74be726307d9d084804"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:073e4f196d0888216e6794c08cd984ddabc108c0e4e66f48fbd7610d1e6d726d"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_i686.whl", hash = "sha256:e58e48643903e6fad76274337f9a4d3c575b8e21cd05c6214780b2c98bb0c706"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_ppc64le.whl", hash = "sha256:9740ed9f61a3bd8708a41cadd5c057c04f38e5b89bd773e369df2e210a1c55a3"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_s390x.whl", hash = "sha256:1b3189171665f5c8d748ae7fe10a29fff1ebeedeaef57b16f1ea54b1ec0fe514"}, - {file = "cmake-3.27.0-py2.py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:c4c968c188e7518deb463a14e64f3a19f242c9dcf7f24e1dbcc1419690cd54e0"}, - {file = "cmake-3.27.0-py2.py3-none-win32.whl", hash = "sha256:5561aca62b65aac844f3931e74cfeb696e4534de145e3307bf942e735736541e"}, - {file = "cmake-3.27.0-py2.py3-none-win_amd64.whl", hash = "sha256:48be3afe62c9513a49be007896a4058fafec512cb1f269a50126da30aacad97f"}, - {file = "cmake-3.27.0-py2.py3-none-win_arm64.whl", hash = "sha256:6f46a170b0c9c552d52da4346534570f818195dfc4f1d0c03264e24cc348fc60"}, - {file = "cmake-3.27.0.tar.gz", hash = "sha256:d03f0a76a2b96805044ad1178b92aeeb5f695caa6776a32522bb5c430a55b4e8"}, -] - -[package.extras] -test = ["coverage (>=4.2)", "flake8 (>=3.0.4)", "path.py (>=11.5.0)", "pytest (>=3.0.3)", "pytest-cov (>=2.4.0)", "pytest-runner (>=2.9)", "pytest-virtualenv (>=1.7.0)", "scikit-build (>=0.10.0)", "setuptools (>=28.0.0)", "virtualenv (>=15.0.3)", "wheel"] - [[package]] name = "colorama" version = "0.4.6" @@ -352,13 +323,13 @@ files = [ [[package]] name = "datasets" -version = "2.14.3" +version = "2.14.4" description = "HuggingFace community-driven open-source library of datasets" optional = true python-versions = ">=3.8.0" files = [ - {file = "datasets-2.14.3-py3-none-any.whl", hash = "sha256:8da5868e55e7c1f0bf3356913a14a9926fe5eca66663691886b95bc9c9b065f0"}, - {file = "datasets-2.14.3.tar.gz", hash = "sha256:5fb20465a990cf7946f961884ba46fec19cdc95b1cd2cad2a7b87c8d183b02e1"}, + {file = "datasets-2.14.4-py3-none-any.whl", hash = "sha256:29336bd316a7d827ccd4da2236596279b20ca2ac78f64c04c9483da7cbc2459b"}, + {file = "datasets-2.14.4.tar.gz", hash = "sha256:ef29c2b5841de488cd343cfc26ab979bff77efa4d2285af51f1ad7db5c46a83b"}, ] [package.dependencies] @@ -436,13 +407,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.1.2" +version = "1.1.3" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, ] [package.extras] @@ -608,148 +579,148 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.56.2" +version = "1.57.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.56.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:bf0b9959e673505ee5869950642428046edb91f99942607c2ecf635f8a4b31c9"}, - {file = "grpcio-1.56.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:5144feb20fe76e73e60c7d73ec3bf54f320247d1ebe737d10672480371878b48"}, - {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:a72797549935c9e0b9bc1def1768c8b5a709538fa6ab0678e671aec47ebfd55e"}, - {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3f3237a57e42f79f1e560726576aedb3a7ef931f4e3accb84ebf6acc485d316"}, - {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:900bc0096c2ca2d53f2e5cebf98293a7c32f532c4aeb926345e9747452233950"}, - {file = "grpcio-1.56.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:97e0efaebbfd222bcaac2f1735c010c1d3b167112d9d237daebbeedaaccf3d1d"}, - {file = "grpcio-1.56.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0c85c5cbe8b30a32fa6d802588d55ffabf720e985abe9590c7c886919d875d4"}, - {file = "grpcio-1.56.2-cp310-cp310-win32.whl", hash = "sha256:06e84ad9ae7668a109e970c7411e7992751a116494cba7c4fb877656527f9a57"}, - {file = "grpcio-1.56.2-cp310-cp310-win_amd64.whl", hash = "sha256:10954662f77dc36c9a1fb5cc4a537f746580d6b5734803be1e587252682cda8d"}, - {file = "grpcio-1.56.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:c435f5ce1705de48e08fcbcfaf8aee660d199c90536e3e06f2016af7d6a938dd"}, - {file = "grpcio-1.56.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:6108e5933eb8c22cd3646e72d5b54772c29f57482fd4c41a0640aab99eb5071d"}, - {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8391cea5ce72f4a12368afd17799474015d5d3dc00c936a907eb7c7eaaea98a5"}, - {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:750de923b456ca8c0f1354d6befca45d1f3b3a789e76efc16741bd4132752d95"}, - {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fda2783c12f553cdca11c08e5af6eecbd717280dc8fbe28a110897af1c15a88c"}, - {file = "grpcio-1.56.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9e04d4e4cfafa7c5264e535b5d28e786f0571bea609c3f0aaab13e891e933e9c"}, - {file = "grpcio-1.56.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:89a49cc5ad08a38b6141af17e00d1dd482dc927c7605bc77af457b5a0fca807c"}, - {file = "grpcio-1.56.2-cp311-cp311-win32.whl", hash = "sha256:6a007a541dff984264981fbafeb052bfe361db63578948d857907df9488d8774"}, - {file = "grpcio-1.56.2-cp311-cp311-win_amd64.whl", hash = "sha256:af4063ef2b11b96d949dccbc5a987272f38d55c23c4c01841ea65a517906397f"}, - {file = "grpcio-1.56.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:a6ff459dac39541e6a2763a4439c4ca6bc9ecb4acc05a99b79246751f9894756"}, - {file = "grpcio-1.56.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:f20fd21f7538f8107451156dd1fe203300b79a9ddceba1ee0ac8132521a008ed"}, - {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:d1fbad1f9077372b6587ec589c1fc120b417b6c8ad72d3e3cc86bbbd0a3cee93"}, - {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee26e9dfb3996aff7c870f09dc7ad44a5f6732b8bdb5a5f9905737ac6fd4ef1"}, - {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4c60abd950d6de3e4f1ddbc318075654d275c29c846ab6a043d6ed2c52e4c8c"}, - {file = "grpcio-1.56.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1c31e52a04e62c8577a7bf772b3e7bed4df9c9e0dd90f92b6ffa07c16cab63c9"}, - {file = "grpcio-1.56.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:345356b307cce5d14355e8e055b4ca5f99bc857c33a3dc1ddbc544fca9cd0475"}, - {file = "grpcio-1.56.2-cp37-cp37m-win_amd64.whl", hash = "sha256:42e63904ee37ae46aa23de50dac8b145b3596f43598fa33fe1098ab2cbda6ff5"}, - {file = "grpcio-1.56.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:7c5ede2e2558f088c49a1ddda19080e4c23fb5d171de80a726b61b567e3766ed"}, - {file = "grpcio-1.56.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:33971197c47965cc1d97d78d842163c283e998223b151bab0499b951fd2c0b12"}, - {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:d39f5d4af48c138cb146763eda14eb7d8b3ccbbec9fe86fb724cd16e0e914c64"}, - {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ded637176addc1d3eef35331c39acc598bac550d213f0a1bedabfceaa2244c87"}, - {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c90da4b124647547a68cf2f197174ada30c7bb9523cb976665dfd26a9963d328"}, - {file = "grpcio-1.56.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3ccb621749a81dc7755243665a70ce45536ec413ef5818e013fe8dfbf5aa497b"}, - {file = "grpcio-1.56.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4eb37dd8dd1aa40d601212afa27ca5be255ba792e2e0b24d67b8af5e012cdb7d"}, - {file = "grpcio-1.56.2-cp38-cp38-win32.whl", hash = "sha256:ddb4a6061933bd9332b74eac0da25f17f32afa7145a33a0f9711ad74f924b1b8"}, - {file = "grpcio-1.56.2-cp38-cp38-win_amd64.whl", hash = "sha256:8940d6de7068af018dfa9a959a3510e9b7b543f4c405e88463a1cbaa3b2b379a"}, - {file = "grpcio-1.56.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:51173e8fa6d9a2d85c14426bdee5f5c4a0654fd5fddcc21fe9d09ab0f6eb8b35"}, - {file = "grpcio-1.56.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:373b48f210f43327a41e397391715cd11cfce9ded2fe76a5068f9bacf91cc226"}, - {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:42a3bbb2bc07aef72a7d97e71aabecaf3e4eb616d39e5211e2cfe3689de860ca"}, - {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5344be476ac37eb9c9ad09c22f4ea193c1316bf074f1daf85bddb1b31fda5116"}, - {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3fa3ab0fb200a2c66493828ed06ccd1a94b12eddbfb985e7fd3e5723ff156c6"}, - {file = "grpcio-1.56.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b975b85d1d5efc36cf8b237c5f3849b64d1ba33d6282f5e991f28751317504a1"}, - {file = "grpcio-1.56.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cbdf2c498e077282cd427cfd88bdce4668019791deef0be8155385ab2ba7837f"}, - {file = "grpcio-1.56.2-cp39-cp39-win32.whl", hash = "sha256:139f66656a762572ae718fa0d1f2dce47c05e9fbf7a16acd704c354405b97df9"}, - {file = "grpcio-1.56.2-cp39-cp39-win_amd64.whl", hash = "sha256:830215173ad45d670140ff99aac3b461f9be9a6b11bee1a17265aaaa746a641a"}, - {file = "grpcio-1.56.2.tar.gz", hash = "sha256:0ff789ae7d8ddd76d2ac02e7d13bfef6fc4928ac01e1dcaa182be51b6bcc0aaa"}, + {file = "grpcio-1.57.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:092fa155b945015754bdf988be47793c377b52b88d546e45c6a9f9579ac7f7b6"}, + {file = "grpcio-1.57.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2f7349786da979a94690cc5c2b804cab4e8774a3cf59be40d037c4342c906649"}, + {file = "grpcio-1.57.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:82640e57fb86ea1d71ea9ab54f7e942502cf98a429a200b2e743d8672171734f"}, + {file = "grpcio-1.57.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40b72effd4c789de94ce1be2b5f88d7b9b5f7379fe9645f198854112a6567d9a"}, + {file = "grpcio-1.57.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f708a6a17868ad8bf586598bee69abded4996b18adf26fd2d91191383b79019"}, + {file = "grpcio-1.57.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:60fe15288a0a65d5c1cb5b4a62b1850d07336e3ba728257a810317be14f0c527"}, + {file = "grpcio-1.57.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6907b1cf8bb29b058081d2aad677b15757a44ef2d4d8d9130271d2ad5e33efca"}, + {file = "grpcio-1.57.0-cp310-cp310-win32.whl", hash = "sha256:57b183e8b252825c4dd29114d6c13559be95387aafc10a7be645462a0fc98bbb"}, + {file = "grpcio-1.57.0-cp310-cp310-win_amd64.whl", hash = "sha256:7b400807fa749a9eb286e2cd893e501b110b4d356a218426cb9c825a0474ca56"}, + {file = "grpcio-1.57.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c6ebecfb7a31385393203eb04ed8b6a08f5002f53df3d59e5e795edb80999652"}, + {file = "grpcio-1.57.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:00258cbe3f5188629828363ae8ff78477ce976a6f63fb2bb5e90088396faa82e"}, + {file = "grpcio-1.57.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:23e7d8849a0e58b806253fd206ac105b328171e01b8f18c7d5922274958cc87e"}, + {file = "grpcio-1.57.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5371bcd861e679d63b8274f73ac281751d34bd54eccdbfcd6aa00e692a82cd7b"}, + {file = "grpcio-1.57.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aed90d93b731929e742967e236f842a4a2174dc5db077c8f9ad2c5996f89f63e"}, + {file = "grpcio-1.57.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fe752639919aad9ffb0dee0d87f29a6467d1ef764f13c4644d212a9a853a078d"}, + {file = "grpcio-1.57.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fada6b07ec4f0befe05218181f4b85176f11d531911b64c715d1875c4736d73a"}, + {file = "grpcio-1.57.0-cp311-cp311-win32.whl", hash = "sha256:bb396952cfa7ad2f01061fbc7dc1ad91dd9d69243bcb8110cf4e36924785a0fe"}, + {file = "grpcio-1.57.0-cp311-cp311-win_amd64.whl", hash = "sha256:e503cb45ed12b924b5b988ba9576dc9949b2f5283b8e33b21dcb6be74a7c58d0"}, + {file = "grpcio-1.57.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:fd173b4cf02b20f60860dc2ffe30115c18972d7d6d2d69df97ac38dee03be5bf"}, + {file = "grpcio-1.57.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:d7f8df114d6b4cf5a916b98389aeaf1e3132035420a88beea4e3d977e5f267a5"}, + {file = "grpcio-1.57.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:76c44efa4ede1f42a9d5b2fed1fe9377e73a109bef8675fb0728eb80b0b8e8f2"}, + {file = "grpcio-1.57.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4faea2cfdf762a664ab90589b66f416274887641ae17817de510b8178356bf73"}, + {file = "grpcio-1.57.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c60b83c43faeb6d0a9831f0351d7787a0753f5087cc6fa218d78fdf38e5acef0"}, + {file = "grpcio-1.57.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b363bbb5253e5f9c23d8a0a034dfdf1b7c9e7f12e602fc788c435171e96daccc"}, + {file = "grpcio-1.57.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:f1fb0fd4a1e9b11ac21c30c169d169ef434c6e9344ee0ab27cfa6f605f6387b2"}, + {file = "grpcio-1.57.0-cp37-cp37m-win_amd64.whl", hash = "sha256:34950353539e7d93f61c6796a007c705d663f3be41166358e3d88c45760c7d98"}, + {file = "grpcio-1.57.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:871f9999e0211f9551f368612460442a5436d9444606184652117d6a688c9f51"}, + {file = "grpcio-1.57.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:a8a8e560e8dbbdf29288872e91efd22af71e88b0e5736b0daf7773c1fecd99f0"}, + {file = "grpcio-1.57.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2313b124e475aa9017a9844bdc5eafb2d5abdda9d456af16fc4535408c7d6da6"}, + {file = "grpcio-1.57.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4098b6b638d9e0ca839a81656a2fd4bc26c9486ea707e8b1437d6f9d61c3941"}, + {file = "grpcio-1.57.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e5b58e32ae14658085c16986d11e99abd002ddbf51c8daae8a0671fffb3467f"}, + {file = "grpcio-1.57.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0f80bf37f09e1caba6a8063e56e2b87fa335add314cf2b78ebf7cb45aa7e3d06"}, + {file = "grpcio-1.57.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5b7a4ce8f862fe32b2a10b57752cf3169f5fe2915acfe7e6a1e155db3da99e79"}, + {file = "grpcio-1.57.0-cp38-cp38-win32.whl", hash = "sha256:9338bacf172e942e62e5889b6364e56657fbf8ac68062e8b25c48843e7b202bb"}, + {file = "grpcio-1.57.0-cp38-cp38-win_amd64.whl", hash = "sha256:e1cb52fa2d67d7f7fab310b600f22ce1ff04d562d46e9e0ac3e3403c2bb4cc16"}, + {file = "grpcio-1.57.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:fee387d2fab144e8a34e0e9c5ca0f45c9376b99de45628265cfa9886b1dbe62b"}, + {file = "grpcio-1.57.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:b53333627283e7241fcc217323f225c37783b5f0472316edcaa4479a213abfa6"}, + {file = "grpcio-1.57.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:f19ac6ac0a256cf77d3cc926ef0b4e64a9725cc612f97228cd5dc4bd9dbab03b"}, + {file = "grpcio-1.57.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3fdf04e402f12e1de8074458549337febb3b45f21076cc02ef4ff786aff687e"}, + {file = "grpcio-1.57.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5613a2fecc82f95d6c51d15b9a72705553aa0d7c932fad7aed7afb51dc982ee5"}, + {file = "grpcio-1.57.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b670c2faa92124b7397b42303e4d8eb64a4cd0b7a77e35a9e865a55d61c57ef9"}, + {file = "grpcio-1.57.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7a635589201b18510ff988161b7b573f50c6a48fae9cb567657920ca82022b37"}, + {file = "grpcio-1.57.0-cp39-cp39-win32.whl", hash = "sha256:d78d8b86fcdfa1e4c21f8896614b6cc7ee01a2a758ec0c4382d662f2a62cf766"}, + {file = "grpcio-1.57.0-cp39-cp39-win_amd64.whl", hash = "sha256:20ec6fc4ad47d1b6e12deec5045ec3cd5402d9a1597f738263e98f490fe07056"}, + {file = "grpcio-1.57.0.tar.gz", hash = "sha256:4b089f7ad1eb00a104078bab8015b0ed0ebcb3b589e527ab009c53893fd4e613"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.56.2)"] +protobuf = ["grpcio-tools (>=1.57.0)"] [[package]] name = "grpcio-reflection" -version = "1.56.2" +version = "1.57.0" description = "Standard Protobuf Reflection Service for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-reflection-1.56.2.tar.gz", hash = "sha256:74a81766af639ab8f1b7f59531dc814640f4a1bcf012dc06ce6be205b50a394c"}, - {file = "grpcio_reflection-1.56.2-py3-none-any.whl", hash = "sha256:004bcc3d4a3dcd89bf83253e4c08ca032fc7ca862f7532ea09ebdf08801fc193"}, + {file = "grpcio-reflection-1.57.0.tar.gz", hash = "sha256:8f63a18729cba995a172f8325235f5094cb066febec75f9a3b1b2e28328aa166"}, + {file = "grpcio_reflection-1.57.0-py3-none-any.whl", hash = "sha256:d7deb8587f9d0095fb5d367c2aa5ce1380e3f23b0f8bca6c00bc404c5429cb6a"}, ] [package.dependencies] -grpcio = ">=1.56.2" +grpcio = ">=1.57.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.56.2" +version = "1.57.0" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.56.2.tar.gz", hash = "sha256:a046b2c0118df4a5687f4585cca9d3c3bae5c498c4dff055dcb43fb06a1180c8"}, - {file = "grpcio_status-1.56.2-py3-none-any.whl", hash = "sha256:63f3842867735f59f5d70e723abffd2e8501a6bcd915612a1119e52f10614782"}, + {file = "grpcio-status-1.57.0.tar.gz", hash = "sha256:b098da99df1eebe58337f8f78e50df990273ccacc1226fddeb47c590e3df9e02"}, + {file = "grpcio_status-1.57.0-py3-none-any.whl", hash = "sha256:15d6af055914ebbc4ed17e55ebfb8e6bb17a45a57fea32e6af19978fb7844690"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.56.2" +grpcio = ">=1.57.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.56.2" +version = "1.57.0" description = "Protobuf code generator for gRPC" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-tools-1.56.2.tar.gz", hash = "sha256:82af2f4040084141a732f0ef1ecf3f14fdf629923d74d850415e4d09a077e77a"}, - {file = "grpcio_tools-1.56.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:42272376e9a5a1c631863cda056c143c98d21e5b670db5c8c5b7ed0ba3a1a6eb"}, - {file = "grpcio_tools-1.56.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:a8735d7aa34be99dddfbd476eff6005e684bb2c893c0f62a5811528b84c5b371"}, - {file = "grpcio_tools-1.56.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:68ef3aa7509e5e7a6e7c0ecc183e28118e73da4bef0fc77079648601ce35e58f"}, - {file = "grpcio_tools-1.56.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:380985b8d95ea2469e103945bd83a815d1213e370f580631fdd5a3dbaa17e446"}, - {file = "grpcio_tools-1.56.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bfb375eb4f1946d68b8bc7b963c756defa31aa573a35c152a7233d06c0ad6ad"}, - {file = "grpcio_tools-1.56.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:13388a22fcba9a1a87f217130a1a01365716af74bd5d0a8a54fc383b8e048ef2"}, - {file = "grpcio_tools-1.56.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7a26160bc0ea5b464715789d4d2a66f01816271677673d65da39bac65b9ea838"}, - {file = "grpcio_tools-1.56.2-cp310-cp310-win32.whl", hash = "sha256:ff16dd0b086e75f574dbc122e018a44dbd1c6dae3f3621ea99e8e5a6b2706e12"}, - {file = "grpcio_tools-1.56.2-cp310-cp310-win_amd64.whl", hash = "sha256:2037109c1ce253a8e013c9e3ad3722e887d28a1807acdeb1a51b295c8200137b"}, - {file = "grpcio_tools-1.56.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:6dc43300189a69807857c52a3d782e9d3bbfb1cb72dcb27b4043c25161919601"}, - {file = "grpcio_tools-1.56.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:e7009623635ebcd3dd7fe974883fc2d9a3ff0fcef419bfc0a2da8071b372d9f5"}, - {file = "grpcio_tools-1.56.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c7ca2272022f90b73efe900244aaebe9dd7cf3b379e99e08a88984e2fdd229c2"}, - {file = "grpcio_tools-1.56.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:493775d17ea09cea6047ba81e4d3f0eb82e34d2fbd3b96e43f72b44ce74726ee"}, - {file = "grpcio_tools-1.56.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41af279cf5359b123138236c0980440f4cb4d3d18f03b5c1c314cc1512048351"}, - {file = "grpcio_tools-1.56.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:857d72e991d449ec4d2f8337e5e24ddf77b4539965f5cabc84d4b63585832982"}, - {file = "grpcio_tools-1.56.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c0640728d63c9fa56e9a1679943ae4e33ad43a10802dd7a93255870731f44d07"}, - {file = "grpcio_tools-1.56.2-cp311-cp311-win32.whl", hash = "sha256:355204d1b33c7a19e7d69afda411e6595d39ba1e9cbf561770ac1d5403296554"}, - {file = "grpcio_tools-1.56.2-cp311-cp311-win_amd64.whl", hash = "sha256:ea5d108d28b4cd2e28539241c6aee96bda83086d8888c36785d9f84ea690d896"}, - {file = "grpcio_tools-1.56.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:28444615b7a76b3d9267f81d1487fcad21a581d00564164d9e25ccc28635a811"}, - {file = "grpcio_tools-1.56.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:45d8b5ad6716848d5b68d9cee29a1a9c5c4baa1824ec5b92d9e35acedddba076"}, - {file = "grpcio_tools-1.56.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:31d1183d28ffc8da242333cb9f683f5093941da80dd5281db0fa93077aecb518"}, - {file = "grpcio_tools-1.56.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0059dfc9bea8f7bca69c15ca62c88904c4f907fde1137e0743b5eee054661873"}, - {file = "grpcio_tools-1.56.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24fc857252181c9950ed2d8cee3df5bd0f42861c4ad0db2a57400186827f96e5"}, - {file = "grpcio_tools-1.56.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3a74a5e4fc8121a51401665f96f9a70aee50a2f1221e4a199e67b3b8f55881e8"}, - {file = "grpcio_tools-1.56.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bec47db5d8b5c3b2a44afdbc3f3bf306e34279289a206d20222824381ca3cb13"}, - {file = "grpcio_tools-1.56.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c0dbaac63a25c088f864295f394230eeb7be48dac2264433fda2603f86c36b25"}, - {file = "grpcio_tools-1.56.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:0a4f9cce5a16613b6d3123c89f9d50e0d13b466799af17bc723dc7d2901a54e4"}, - {file = "grpcio_tools-1.56.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:ea5fc1b49514b44a3e5a45156c025002f172ade4c509e58c51967865c7c6fa45"}, - {file = "grpcio_tools-1.56.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:54da410124547bacb97a54546c1a95f1af0125e48edc8b5679412ef8b2844f81"}, - {file = "grpcio_tools-1.56.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5223668649172d879ee780253b8e4a79144c56a3cc1bb021847f583508c2b0be"}, - {file = "grpcio_tools-1.56.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:483256d5f5be6a77b24d8a5f06ca152d1571c62bf5c738834da61107c7563afe"}, - {file = "grpcio_tools-1.56.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1f334718eb796799bfadbac5567456fb745cee8c7b438c93b74d1ce676c6ad07"}, - {file = "grpcio_tools-1.56.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:26751f69cbbc8ea19cf0657b7d109a6db7df81f80caf16380ebcd20eea27652c"}, - {file = "grpcio_tools-1.56.2-cp38-cp38-win32.whl", hash = "sha256:4056ff13e30813d42a30ce1cdfeaeb6bbee915515c161c1df896dac3143ae643"}, - {file = "grpcio_tools-1.56.2-cp38-cp38-win_amd64.whl", hash = "sha256:878b9269ceb0dd934b61697a9dd9a5c3e9552521e8f46ab32cf4d72a223f7b6c"}, - {file = "grpcio_tools-1.56.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:216e86d3a6ccc31b27fa4c12491981a0a39d4787d2358b6df05baffa40084494"}, - {file = "grpcio_tools-1.56.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:778224fcbc1cc7eaf222ce94676afbac8d72b4f84cf4239e30b01d2450a46126"}, - {file = "grpcio_tools-1.56.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:14120fb2c6f7894fac5b689934368c692ec50f50a320e8073277ab7778fd612f"}, - {file = "grpcio_tools-1.56.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:014da3ed176beb2b1c8430ccc34c8fe962cdd5480e56fb4ab9de60e60c315f3f"}, - {file = "grpcio_tools-1.56.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8febb4f90b8fab3179f5bdaa159f1d2a20523ea17ec0d66bdec7732f9532de91"}, - {file = "grpcio_tools-1.56.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ffae7df3318266614f7aa440acb2098c064b6b5ae061fc22125092386349e526"}, - {file = "grpcio_tools-1.56.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d86e24eb6e3973c55e9c74412ff755d1b9d15518c4eaf95676acff49d0162a2"}, - {file = "grpcio_tools-1.56.2-cp39-cp39-win32.whl", hash = "sha256:506d00a86950adf4017395551a4547c0b7fcefa90e4c220135fc3e34e31be81b"}, - {file = "grpcio_tools-1.56.2-cp39-cp39-win_amd64.whl", hash = "sha256:8da04f033b8f4c597e8fc990e2f626bad2b269227bdd554592ea618f624f1aa9"}, + {file = "grpcio-tools-1.57.0.tar.gz", hash = "sha256:2f16130d869ce27ecd623194547b649dd657333ec7e8644cc571c645781a9b85"}, + {file = "grpcio_tools-1.57.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:4fb8a8468031f858381a576078924af364a08833d8f8f3237018252c4573a802"}, + {file = "grpcio_tools-1.57.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:35bf0dad8a3562043345236c26d0053a856fb06c04d7da652f2ded914e508ae7"}, + {file = "grpcio_tools-1.57.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:ec9aab2fb6783c7fc54bc28f58eb75f1ca77594e6b0fd5e5e7a8114a95169fe0"}, + {file = "grpcio_tools-1.57.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cf5fc0a1c23f8ea34b408b72fb0e90eec0f404ad4dba98e8f6da3c9ce34e2ed"}, + {file = "grpcio_tools-1.57.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26e69d08a515554e0cfe1ec4d31568836f4b17f0ff82294f957f629388629eb9"}, + {file = "grpcio_tools-1.57.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c39a3656576b6fdaaf28abe0467f7a7231df4230c1bee132322dbc3209419e7f"}, + {file = "grpcio_tools-1.57.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f64f8ab22d27d4a5693310748d35a696061c3b5c7b8c4fb4ab3b4bc1068b6b56"}, + {file = "grpcio_tools-1.57.0-cp310-cp310-win32.whl", hash = "sha256:d2a134756f4db34759a5cc7f7e43f7eb87540b68d1cca62925593c6fb93924f7"}, + {file = "grpcio_tools-1.57.0-cp310-cp310-win_amd64.whl", hash = "sha256:9a3d60fb8d46ede26c1907c146561b3a9caa20a7aff961bc661ef8226f85a2e9"}, + {file = "grpcio_tools-1.57.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:aac98ecad8f7bd4301855669d42a5d97ef7bb34bec2b1e74c7a0641d47e313cf"}, + {file = "grpcio_tools-1.57.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:cdd020cb68b51462983b7c2dfbc3eb6ede032b8bf438d4554df0c3f08ce35c76"}, + {file = "grpcio_tools-1.57.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:f54081b08419a39221cd646363b5708857c696b3ad4784f1dcf310891e33a5f7"}, + {file = "grpcio_tools-1.57.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed85a0291fff45b67f2557fe7f117d3bc7af8b54b8619d27bf374b5c8b7e3ca2"}, + {file = "grpcio_tools-1.57.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e868cd6feb3ef07d4b35be104fe1fd0657db05259ff8f8ec5e08f4f89ca1191d"}, + {file = "grpcio_tools-1.57.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:dfb6f6120587b8e228a3cae5ee4985b5bdc18501bad05c49df61965dfc9d70a9"}, + {file = "grpcio_tools-1.57.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a7ad7f328e28fc97c356d0f10fb10d8b5151bb65aa7cf14bf8084513f0b7306"}, + {file = "grpcio_tools-1.57.0-cp311-cp311-win32.whl", hash = "sha256:9867f2817b1a0c93c523f89ac6c9d8625548af4620a7ce438bf5a76e23327284"}, + {file = "grpcio_tools-1.57.0-cp311-cp311-win_amd64.whl", hash = "sha256:1f9e917a9f18087f6c14b4d4508fb94fca5c2f96852363a89232fb9b2124ac1f"}, + {file = "grpcio_tools-1.57.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:9f2aefa8a37bd2c4db1a3f1aca11377e2766214520fb70e67071f4ff8d8b0fa5"}, + {file = "grpcio_tools-1.57.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:850cbda0ec5d24c39e7215ede410276040692ca45d105fbbeada407fa03f0ac0"}, + {file = "grpcio_tools-1.57.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6fa52972c9647876ea35f6dc2b51002a74ed900ec7894586cbb2fe76f64f99de"}, + {file = "grpcio_tools-1.57.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0eea89d7542719594e50e2283f51a072978b953e8b3e9fd7c59a2c762d4c1"}, + {file = "grpcio_tools-1.57.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3da5240211252fc70a6451fe00c143e2ab2f7bfc2445695ad2ed056b8e48d96"}, + {file = "grpcio_tools-1.57.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a0256f8786ac9e4db618a1aa492bb3472569a0946fd3ee862ffe23196323da55"}, + {file = "grpcio_tools-1.57.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c026bdf5c1366ce88b7bbe2d8207374d675afd3fd911f60752103de3da4a41d2"}, + {file = "grpcio_tools-1.57.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9053c2f655589545be08b9d6a673e92970173a4bf11a4b9f18cd6e9af626b587"}, + {file = "grpcio_tools-1.57.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:81ec4dbb696e095057b2528d11a8da04be6bbe2b967fa07d4ea9ba6354338cbf"}, + {file = "grpcio_tools-1.57.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:495e2946406963e0b9f063f76d5af0f2a19517dac2b367b5b044432ac9194296"}, + {file = "grpcio_tools-1.57.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:7b46fc6aa8eb7edd18cafcd21fd98703cb6c09e46b507de335fca7f0161dfccb"}, + {file = "grpcio_tools-1.57.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb81ff861692111fa81bd85f64584e624cb4013bd66fbce8a209b8893f5ce398"}, + {file = "grpcio_tools-1.57.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a42dc220eb5305f470855c9284f4c8e85ae59d6d742cd07946b0cbe5e9ca186"}, + {file = "grpcio_tools-1.57.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:90d10d9038ba46a595a223a34f136c9230e3d6d7abc2433dbf0e1c31939d3a8b"}, + {file = "grpcio_tools-1.57.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5bc3e6d338aefb052e19cedabe00452be46d0c10a4ed29ee77abb00402e438fe"}, + {file = "grpcio_tools-1.57.0-cp38-cp38-win32.whl", hash = "sha256:34b36217b17b5bea674a414229913e1fd80ede328be51e1b531fcc62abd393b0"}, + {file = "grpcio_tools-1.57.0-cp38-cp38-win_amd64.whl", hash = "sha256:dbde4004a0688400036342ff73e3706e8940483e2871547b1354d59e93a38277"}, + {file = "grpcio_tools-1.57.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:784574709b9690dc28696617ea69352e2132352fdfc9bc89afa8e39f99ae538e"}, + {file = "grpcio_tools-1.57.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:85ac4e62eb44428cde025fd9ab7554002315fc7880f791c553fc5a0015cc9931"}, + {file = "grpcio_tools-1.57.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:dc771d4db5701f280957bbcee91745e0686d00ed1c6aa7e05ba30a58b02d70a1"}, + {file = "grpcio_tools-1.57.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3ac06703c412f8167a9062eaf6099409967e33bf98fa5b02be4b4689b6bdf39"}, + {file = "grpcio_tools-1.57.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02d78c034109f46032c7217260066d49d41e6bcaf588fa28fa40fe2f83445347"}, + {file = "grpcio_tools-1.57.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2db25f15ed44327f2e02d0c4fe741ac966f9500e407047d8a7c7fccf2df65616"}, + {file = "grpcio_tools-1.57.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b417c97936d94874a3ce7ed8deab910f2233e3612134507cfee4af8735c38a6"}, + {file = "grpcio_tools-1.57.0-cp39-cp39-win32.whl", hash = "sha256:f717cce5093e6b6049d9ea6d12fdf3658efdb1a80772f7737db1f8510b876df6"}, + {file = "grpcio_tools-1.57.0-cp39-cp39-win_amd64.whl", hash = "sha256:1c0e8a1a32973a5d59fbcc19232f925e5c48116e9411f788033a31c5ca5130b4"}, ] [package.dependencies] -grpcio = ">=1.56.2" +grpcio = ">=1.57.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -851,16 +822,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "lit" -version = "16.0.6" -description = "A Software Testing Tool" -optional = false -python-versions = "*" -files = [ - {file = "lit-16.0.6.tar.gz", hash = "sha256:84623c9c23b6b14763d637f4e63e6b721b3446ada40bf7001d8fee70b8e77a9a"}, -] - [[package]] name = "loguru" version = "0.6.0" @@ -1377,6 +1338,75 @@ docs-specific = ["hf-doc-builder"] quality = ["black (>=22.0,<23.0)", "ruff (>=0.0.241)", "urllib3 (<=2.0.0)"] test = ["black (>=22.0,<23.0)", "datasets", "diffusers", "hf-doc-builder", "parameterized", "pytest", "pytest-cov", "pytest-xdist", "ruff (>=0.0.241)", "urllib3 (<=2.0.0)"] +[[package]] +name = "pillow" +version = "10.0.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "Pillow-10.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891"}, + {file = "Pillow-10.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5db32e2a6ccbb3d34d87c87b432959e0db29755727afb37290e10f6e8e62614"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edf4392b77bdc81f36e92d3a07a5cd072f90253197f4a52a55a8cec48a12483b"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:520f2a520dc040512699f20fa1c363eed506e94248d71f85412b625026f6142c"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:8c11160913e3dd06c8ffdb5f233a4f254cb449f4dfc0f8f4549eda9e542c93d1"}, + {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf"}, + {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d5d0dae4cfd56969d23d94dc8e89fb6a217be461c69090768227beb8ed28c0a3"}, + {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992"}, + {file = "Pillow-10.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:dffe31a7f47b603318c609f378ebcd57f1554a3a6a8effbc59c3c69f804296de"}, + {file = "Pillow-10.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485"}, + {file = "Pillow-10.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d35e3c8d9b1268cbf5d3670285feb3528f6680420eafe35cccc686b73c1e330f"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ed64f9ca2f0a95411e88a4efbd7a29e5ce2cea36072c53dd9d26d9c76f753b3"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd"}, + {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cd25d2a9d2b36fcb318882481367956d2cf91329f6892fe5d385c346c0649629"}, + {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3b08d4cc24f471b2c8ca24ec060abf4bebc6b144cb89cba638c720546b1cf538"}, + {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d737a602fbd82afd892ca746392401b634e278cb65d55c4b7a8f48e9ef8d008d"}, + {file = "Pillow-10.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:3a82c40d706d9aa9734289740ce26460a11aeec2d9c79b7af87bb35f0073c12f"}, + {file = "Pillow-10.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:bc2ec7c7b5d66b8ec9ce9f720dbb5fa4bace0f545acd34870eff4a369b44bf37"}, + {file = "Pillow-10.0.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:d80cf684b541685fccdd84c485b31ce73fc5c9b5d7523bf1394ce134a60c6883"}, + {file = "Pillow-10.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76de421f9c326da8f43d690110f0e79fe3ad1e54be811545d7d91898b4c8493e"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81ff539a12457809666fef6624684c008e00ff6bf455b4b89fd00a140eecd640"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce543ed15570eedbb85df19b0a1a7314a9c8141a36ce089c0a894adbfccb4568"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:685ac03cc4ed5ebc15ad5c23bc555d68a87777586d970c2c3e216619a5476223"}, + {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d72e2ecc68a942e8cf9739619b7f408cc7b272b279b56b2c83c6123fcfa5cdff"}, + {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d50b6aec14bc737742ca96e85d6d0a5f9bfbded018264b3b70ff9d8c33485551"}, + {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5"}, + {file = "Pillow-10.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:f31f9fdbfecb042d046f9d91270a0ba28368a723302786c0009ee9b9f1f60199"}, + {file = "Pillow-10.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:1ce91b6ec08d866b14413d3f0bbdea7e24dfdc8e59f562bb77bc3fe60b6144ca"}, + {file = "Pillow-10.0.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:349930d6e9c685c089284b013478d6f76e3a534e36ddfa912cde493f235372f3"}, + {file = "Pillow-10.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3a684105f7c32488f7153905a4e3015a3b6c7182e106fe3c37fbb5ef3e6994c3"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f07ea8d2f827d7d2a49ecf1639ec02d75ffd1b88dcc5b3a61bbb37a8759ad8d"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530"}, + {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:f88a0b92277de8e3ca715a0d79d68dc82807457dae3ab8699c758f07c20b3c51"}, + {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86"}, + {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7"}, + {file = "Pillow-10.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:dbc02381779d412145331789b40cc7b11fdf449e5d94f6bc0b080db0a56ea3f0"}, + {file = "Pillow-10.0.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9211e7ad69d7c9401cfc0e23d49b69ca65ddd898976d660a2fa5904e3d7a9baa"}, + {file = "Pillow-10.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:faaf07ea35355b01a35cb442dd950d8f1bb5b040a7787791a535de13db15ed90"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f72a021fbb792ce98306ffb0c348b3c9cb967dce0f12a49aa4c3d3fdefa967"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f7c16705f44e0504a3a2a14197c1f0b32a95731d251777dcb060aa83022cb2d"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:76edb0a1fa2b4745fb0c99fb9fb98f8b180a1bbceb8be49b087e0b21867e77d3"}, + {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:368ab3dfb5f49e312231b6f27b8820c823652b7cd29cfbd34090565a015e99ba"}, + {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:608bfdee0d57cf297d32bcbb3c728dc1da0907519d1784962c5f0c68bb93e5a3"}, + {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5c6e3df6bdd396749bafd45314871b3d0af81ff935b2d188385e970052091017"}, + {file = "Pillow-10.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:7be600823e4c8631b74e4a0d38384c73f680e6105a7d3c6824fcf226c178c7e6"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:92be919bbc9f7d09f7ae343c38f5bb21c973d2576c1d45600fce4b74bafa7ac0"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8182b523b2289f7c415f589118228d30ac8c355baa2f3194ced084dac2dbba"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:38250a349b6b390ee6047a62c086d3817ac69022c127f8a5dc058c31ccef17f3"}, + {file = "Pillow-10.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:88af2003543cc40c80f6fca01411892ec52b11021b3dc22ec3bc9d5afd1c5334"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7b031a6fc11365970e6a5686d7ba8c63e4c1cf1ea143811acbb524295eabed"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:db24668940f82321e746773a4bc617bfac06ec831e5c88b643f91f122a785684"}, + {file = "Pillow-10.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:efe8c0681042536e0d06c11f48cebe759707c9e9abf880ee213541c5b46c5bf3"}, + {file = "Pillow-10.0.0.tar.gz", hash = "sha256:9c82b5b3e043c7af0d95792d0d20ccf68f61a1fec6b3530e718b688422727396"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] + [[package]] name = "pluggy" version = "1.2.0" @@ -1394,24 +1424,24 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.23.4" +version = "4.24.0" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.23.4-cp310-abi3-win32.whl", hash = "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b"}, - {file = "protobuf-4.23.4-cp310-abi3-win_amd64.whl", hash = "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12"}, - {file = "protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd"}, - {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a"}, - {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597"}, - {file = "protobuf-4.23.4-cp37-cp37m-win32.whl", hash = "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e"}, - {file = "protobuf-4.23.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0"}, - {file = "protobuf-4.23.4-cp38-cp38-win32.whl", hash = "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70"}, - {file = "protobuf-4.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2"}, - {file = "protobuf-4.23.4-cp39-cp39-win32.whl", hash = "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720"}, - {file = "protobuf-4.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474"}, - {file = "protobuf-4.23.4-py3-none-any.whl", hash = "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff"}, - {file = "protobuf-4.23.4.tar.gz", hash = "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9"}, + {file = "protobuf-4.24.0-cp310-abi3-win32.whl", hash = "sha256:81cb9c4621d2abfe181154354f63af1c41b00a4882fb230b4425cbaed65e8f52"}, + {file = "protobuf-4.24.0-cp310-abi3-win_amd64.whl", hash = "sha256:6c817cf4a26334625a1904b38523d1b343ff8b637d75d2c8790189a4064e51c3"}, + {file = "protobuf-4.24.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ae97b5de10f25b7a443b40427033e545a32b0e9dda17bcd8330d70033379b3e5"}, + {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:567fe6b0647494845d0849e3d5b260bfdd75692bf452cdc9cb660d12457c055d"}, + {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:a6b1ca92ccabfd9903c0c7dde8876221dc7d8d87ad5c42e095cc11b15d3569c7"}, + {file = "protobuf-4.24.0-cp37-cp37m-win32.whl", hash = "sha256:a38400a692fd0c6944c3c58837d112f135eb1ed6cdad5ca6c5763336e74f1a04"}, + {file = "protobuf-4.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5ab19ee50037d4b663c02218a811a5e1e7bb30940c79aac385b96e7a4f9daa61"}, + {file = "protobuf-4.24.0-cp38-cp38-win32.whl", hash = "sha256:e8834ef0b4c88666ebb7c7ec18045aa0f4325481d724daa624a4cf9f28134653"}, + {file = "protobuf-4.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:8bb52a2be32db82ddc623aefcedfe1e0eb51da60e18fcc908fb8885c81d72109"}, + {file = "protobuf-4.24.0-cp39-cp39-win32.whl", hash = "sha256:ae7a1835721086013de193311df858bc12cd247abe4ef9710b715d930b95b33e"}, + {file = "protobuf-4.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:44825e963008f8ea0d26c51911c30d3e82e122997c3c4568fd0385dd7bacaedf"}, + {file = "protobuf-4.24.0-py3-none-any.whl", hash = "sha256:82e6e9ebdd15b8200e8423676eab38b774624d6a1ad696a60d86a2ac93f18201"}, + {file = "protobuf-4.24.0.tar.gz", hash = "sha256:5d0ceb9de6e08311832169e601d1fc71bd8e8c779f3ee38a97a78554945ecb85"}, ] [[package]] @@ -1575,99 +1605,99 @@ files = [ [[package]] name = "regex" -version = "2023.6.3" +version = "2023.8.8" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.6" files = [ - {file = "regex-2023.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:824bf3ac11001849aec3fa1d69abcb67aac3e150a933963fb12bda5151fe1bfd"}, - {file = "regex-2023.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05ed27acdf4465c95826962528f9e8d41dbf9b1aa8531a387dee6ed215a3e9ef"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b49c764f88a79160fa64f9a7b425620e87c9f46095ef9c9920542ab2495c8bc"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8e3f1316c2293e5469f8f09dc2d76efb6c3982d3da91ba95061a7e69489a14ef"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43e1dd9d12df9004246bacb79a0e5886b3b6071b32e41f83b0acbf293f820ee8"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4959e8bcbfda5146477d21c3a8ad81b185cd252f3d0d6e4724a5ef11c012fb06"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:af4dd387354dc83a3bff67127a124c21116feb0d2ef536805c454721c5d7993d"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2239d95d8e243658b8dbb36b12bd10c33ad6e6933a54d36ff053713f129aa536"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:890e5a11c97cf0d0c550eb661b937a1e45431ffa79803b942a057c4fb12a2da2"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a8105e9af3b029f243ab11ad47c19b566482c150c754e4c717900a798806b222"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:25be746a8ec7bc7b082783216de8e9473803706723b3f6bef34b3d0ed03d57e2"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3676f1dd082be28b1266c93f618ee07741b704ab7b68501a173ce7d8d0d0ca18"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:10cb847aeb1728412c666ab2e2000ba6f174f25b2bdc7292e7dd71b16db07568"}, - {file = "regex-2023.6.3-cp310-cp310-win32.whl", hash = "sha256:dbbbfce33cd98f97f6bffb17801b0576e653f4fdb1d399b2ea89638bc8d08ae1"}, - {file = "regex-2023.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:c5f8037000eb21e4823aa485149f2299eb589f8d1fe4b448036d230c3f4e68e0"}, - {file = "regex-2023.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c123f662be8ec5ab4ea72ea300359023a5d1df095b7ead76fedcd8babbedf969"}, - {file = "regex-2023.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9edcbad1f8a407e450fbac88d89e04e0b99a08473f666a3f3de0fd292badb6aa"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcba6dae7de533c876255317c11f3abe4907ba7d9aa15d13e3d9710d4315ec0e"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29cdd471ebf9e0f2fb3cac165efedc3c58db841d83a518b082077e612d3ee5df"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:12b74fbbf6cbbf9dbce20eb9b5879469e97aeeaa874145517563cca4029db65c"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c29ca1bd61b16b67be247be87390ef1d1ef702800f91fbd1991f5c4421ebae8"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77f09bc4b55d4bf7cc5eba785d87001d6757b7c9eec237fe2af57aba1a071d9"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ea353ecb6ab5f7e7d2f4372b1e779796ebd7b37352d290096978fea83c4dba0c"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:10590510780b7541969287512d1b43f19f965c2ece6c9b1c00fc367b29d8dce7"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e2fbd6236aae3b7f9d514312cdb58e6494ee1c76a9948adde6eba33eb1c4264f"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:6b2675068c8b56f6bfd5a2bda55b8accbb96c02fd563704732fd1c95e2083461"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74419d2b50ecb98360cfaa2974da8689cb3b45b9deff0dcf489c0d333bcc1477"}, - {file = "regex-2023.6.3-cp311-cp311-win32.whl", hash = "sha256:fb5ec16523dc573a4b277663a2b5a364e2099902d3944c9419a40ebd56a118f9"}, - {file = "regex-2023.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:09e4a1a6acc39294a36b7338819b10baceb227f7f7dbbea0506d419b5a1dd8af"}, - {file = "regex-2023.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0654bca0cdf28a5956c83839162692725159f4cda8d63e0911a2c0dc76166525"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463b6a3ceb5ca952e66550a4532cef94c9a0c80dc156c4cc343041951aec1697"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87b2a5bb5e78ee0ad1de71c664d6eb536dc3947a46a69182a90f4410f5e3f7dd"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6343c6928282c1f6a9db41f5fd551662310e8774c0e5ebccb767002fcf663ca9"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6192d5af2ccd2a38877bfef086d35e6659566a335b1492786ff254c168b1693"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74390d18c75054947e4194019077e243c06fbb62e541d8817a0fa822ea310c14"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:742e19a90d9bb2f4a6cf2862b8b06dea5e09b96c9f2df1779e53432d7275331f"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8abbc5d54ea0ee80e37fef009e3cec5dafd722ed3c829126253d3e22f3846f1e"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c2b867c17a7a7ae44c43ebbeb1b5ff406b3e8d5b3e14662683e5e66e6cc868d3"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:d831c2f8ff278179705ca59f7e8524069c1a989e716a1874d6d1aab6119d91d1"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ee2d1a9a253b1729bb2de27d41f696ae893507c7db224436abe83ee25356f5c1"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:61474f0b41fe1a80e8dfa70f70ea1e047387b7cd01c85ec88fa44f5d7561d787"}, - {file = "regex-2023.6.3-cp36-cp36m-win32.whl", hash = "sha256:0b71e63226e393b534105fcbdd8740410dc6b0854c2bfa39bbda6b0d40e59a54"}, - {file = "regex-2023.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bbb02fd4462f37060122e5acacec78e49c0fbb303c30dd49c7f493cf21fc5b27"}, - {file = "regex-2023.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b862c2b9d5ae38a68b92e215b93f98d4c5e9454fa36aae4450f61dd33ff48487"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976d7a304b59ede34ca2921305b57356694f9e6879db323fd90a80f865d355a3"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:83320a09188e0e6c39088355d423aa9d056ad57a0b6c6381b300ec1a04ec3d16"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9427a399501818a7564f8c90eced1e9e20709ece36be701f394ada99890ea4b3"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178bbc1b2ec40eaca599d13c092079bf529679bf0371c602edaa555e10b41c3"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:837328d14cde912af625d5f303ec29f7e28cdab588674897baafaf505341f2fc"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d44dc13229905ae96dd2ae2dd7cebf824ee92bc52e8cf03dcead37d926da019"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d54af539295392611e7efbe94e827311eb8b29668e2b3f4cadcfe6f46df9c777"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7117d10690c38a622e54c432dfbbd3cbd92f09401d622902c32f6d377e2300ee"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bb60b503ec8a6e4e3e03a681072fa3a5adcbfa5479fa2d898ae2b4a8e24c4591"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:65ba8603753cec91c71de423a943ba506363b0e5c3fdb913ef8f9caa14b2c7e0"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:271f0bdba3c70b58e6f500b205d10a36fb4b58bd06ac61381b68de66442efddb"}, - {file = "regex-2023.6.3-cp37-cp37m-win32.whl", hash = "sha256:9beb322958aaca059f34975b0df135181f2e5d7a13b84d3e0e45434749cb20f7"}, - {file = "regex-2023.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fea75c3710d4f31389eed3c02f62d0b66a9da282521075061ce875eb5300cf23"}, - {file = "regex-2023.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8f56fcb7ff7bf7404becdfc60b1e81a6d0561807051fd2f1860b0d0348156a07"}, - {file = "regex-2023.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d2da3abc88711bce7557412310dfa50327d5769a31d1c894b58eb256459dc289"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a99b50300df5add73d307cf66abea093304a07eb017bce94f01e795090dea87c"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5708089ed5b40a7b2dc561e0c8baa9535b77771b64a8330b684823cfd5116036"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:687ea9d78a4b1cf82f8479cab23678aff723108df3edeac098e5b2498879f4a7"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d3850beab9f527f06ccc94b446c864059c57651b3f911fddb8d9d3ec1d1b25d"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8915cc96abeb8983cea1df3c939e3c6e1ac778340c17732eb63bb96247b91d2"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:841d6e0e5663d4c7b4c8099c9997be748677d46cbf43f9f471150e560791f7ff"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9edce5281f965cf135e19840f4d93d55b3835122aa76ccacfd389e880ba4cf82"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b956231ebdc45f5b7a2e1f90f66a12be9610ce775fe1b1d50414aac1e9206c06"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:36efeba71c6539d23c4643be88295ce8c82c88bbd7c65e8a24081d2ca123da3f"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:cf67ca618b4fd34aee78740bea954d7c69fdda419eb208c2c0c7060bb822d747"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b4598b1897837067a57b08147a68ac026c1e73b31ef6e36deeeb1fa60b2933c9"}, - {file = "regex-2023.6.3-cp38-cp38-win32.whl", hash = "sha256:f415f802fbcafed5dcc694c13b1292f07fe0befdb94aa8a52905bd115ff41e88"}, - {file = "regex-2023.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:d4f03bb71d482f979bda92e1427f3ec9b220e62a7dd337af0aa6b47bf4498f72"}, - {file = "regex-2023.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ccf91346b7bd20c790310c4147eee6ed495a54ddb6737162a36ce9dbef3e4751"}, - {file = "regex-2023.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b28f5024a3a041009eb4c333863d7894d191215b39576535c6734cd88b0fcb68"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0bb18053dfcfed432cc3ac632b5e5e5c5b7e55fb3f8090e867bfd9b054dbcbf"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a5bfb3004f2144a084a16ce19ca56b8ac46e6fd0651f54269fc9e230edb5e4a"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c6b48d0fa50d8f4df3daf451be7f9689c2bde1a52b1225c5926e3f54b6a9ed1"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051da80e6eeb6e239e394ae60704d2b566aa6a7aed6f2890a7967307267a5dc6"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4c3b7fa4cdaa69268748665a1a6ff70c014d39bb69c50fda64b396c9116cf77"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:457b6cce21bee41ac292d6753d5e94dcbc5c9e3e3a834da285b0bde7aa4a11e9"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aad51907d74fc183033ad796dd4c2e080d1adcc4fd3c0fd4fd499f30c03011cd"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0385e73da22363778ef2324950e08b689abdf0b108a7d8decb403ad7f5191938"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c6a57b742133830eec44d9b2290daf5cbe0a2f1d6acee1b3c7b1c7b2f3606df7"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:3e5219bf9e75993d73ab3d25985c857c77e614525fac9ae02b1bebd92f7cecac"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e5087a3c59eef624a4591ef9eaa6e9a8d8a94c779dade95d27c0bc24650261cd"}, - {file = "regex-2023.6.3-cp39-cp39-win32.whl", hash = "sha256:20326216cc2afe69b6e98528160b225d72f85ab080cbdf0b11528cbbaba2248f"}, - {file = "regex-2023.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:bdff5eab10e59cf26bc479f565e25ed71a7d041d1ded04ccf9aee1d9f208487a"}, - {file = "regex-2023.6.3.tar.gz", hash = "sha256:72d1a25bf36d2050ceb35b517afe13864865268dfb45910e2e17a84be6cbfeb0"}, + {file = "regex-2023.8.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:88900f521c645f784260a8d346e12a1590f79e96403971241e64c3a265c8ecdb"}, + {file = "regex-2023.8.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3611576aff55918af2697410ff0293d6071b7e00f4b09e005d614686ac4cd57c"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8a0ccc8f2698f120e9e5742f4b38dc944c38744d4bdfc427616f3a163dd9de5"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c662a4cbdd6280ee56f841f14620787215a171c4e2d1744c9528bed8f5816c96"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf0633e4a1b667bfe0bb10b5e53fe0d5f34a6243ea2530eb342491f1adf4f739"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:551ad543fa19e94943c5b2cebc54c73353ffff08228ee5f3376bd27b3d5b9800"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54de2619f5ea58474f2ac211ceea6b615af2d7e4306220d4f3fe690c91988a61"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ec4b3f0aebbbe2fc0134ee30a791af522a92ad9f164858805a77442d7d18570"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ae646c35cb9f820491760ac62c25b6d6b496757fda2d51be429e0e7b67ae0ab"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca339088839582d01654e6f83a637a4b8194d0960477b9769d2ff2cfa0fa36d2"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:d9b6627408021452dcd0d2cdf8da0534e19d93d070bfa8b6b4176f99711e7f90"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:bd3366aceedf274f765a3a4bc95d6cd97b130d1dda524d8f25225d14123c01db"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7aed90a72fc3654fba9bc4b7f851571dcc368120432ad68b226bd593f3f6c0b7"}, + {file = "regex-2023.8.8-cp310-cp310-win32.whl", hash = "sha256:80b80b889cb767cc47f31d2b2f3dec2db8126fbcd0cff31b3925b4dc6609dcdb"}, + {file = "regex-2023.8.8-cp310-cp310-win_amd64.whl", hash = "sha256:b82edc98d107cbc7357da7a5a695901b47d6eb0420e587256ba3ad24b80b7d0b"}, + {file = "regex-2023.8.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1e7d84d64c84ad97bf06f3c8cb5e48941f135ace28f450d86af6b6512f1c9a71"}, + {file = "regex-2023.8.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce0f9fbe7d295f9922c0424a3637b88c6c472b75eafeaff6f910494a1fa719ef"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06c57e14ac723b04458df5956cfb7e2d9caa6e9d353c0b4c7d5d54fcb1325c46"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7a9aaa5a1267125eef22cef3b63484c3241aaec6f48949b366d26c7250e0357"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b7408511fca48a82a119d78a77c2f5eb1b22fe88b0d2450ed0756d194fe7a9a"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14dc6f2d88192a67d708341f3085df6a4f5a0c7b03dec08d763ca2cd86e9f559"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48c640b99213643d141550326f34f0502fedb1798adb3c9eb79650b1ecb2f177"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0085da0f6c6393428bf0d9c08d8b1874d805bb55e17cb1dfa5ddb7cfb11140bf"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:964b16dcc10c79a4a2be9f1273fcc2684a9eedb3906439720598029a797b46e6"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7ce606c14bb195b0e5108544b540e2c5faed6843367e4ab3deb5c6aa5e681208"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:40f029d73b10fac448c73d6eb33d57b34607f40116e9f6e9f0d32e9229b147d7"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3b8e6ea6be6d64104d8e9afc34c151926f8182f84e7ac290a93925c0db004bfd"}, + {file = "regex-2023.8.8-cp311-cp311-win32.whl", hash = "sha256:942f8b1f3b223638b02df7df79140646c03938d488fbfb771824f3d05fc083a8"}, + {file = "regex-2023.8.8-cp311-cp311-win_amd64.whl", hash = "sha256:51d8ea2a3a1a8fe4f67de21b8b93757005213e8ac3917567872f2865185fa7fb"}, + {file = "regex-2023.8.8-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e951d1a8e9963ea51efd7f150450803e3b95db5939f994ad3d5edac2b6f6e2b4"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704f63b774218207b8ccc6c47fcef5340741e5d839d11d606f70af93ee78e4d4"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22283c769a7b01c8ac355d5be0715bf6929b6267619505e289f792b01304d898"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91129ff1bb0619bc1f4ad19485718cc623a2dc433dff95baadbf89405c7f6b57"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de35342190deb7b866ad6ba5cbcccb2d22c0487ee0cbb251efef0843d705f0d4"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b993b6f524d1e274a5062488a43e3f9f8764ee9745ccd8e8193df743dbe5ee61"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3026cbcf11d79095a32d9a13bbc572a458727bd5b1ca332df4a79faecd45281c"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:293352710172239bf579c90a9864d0df57340b6fd21272345222fb6371bf82b3"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d909b5a3fff619dc7e48b6b1bedc2f30ec43033ba7af32f936c10839e81b9217"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3d370ff652323c5307d9c8e4c62efd1956fb08051b0e9210212bc51168b4ff56"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:b076da1ed19dc37788f6a934c60adf97bd02c7eea461b73730513921a85d4235"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e9941a4ada58f6218694f382e43fdd256e97615db9da135e77359da257a7168b"}, + {file = "regex-2023.8.8-cp36-cp36m-win32.whl", hash = "sha256:a8c65c17aed7e15a0c824cdc63a6b104dfc530f6fa8cb6ac51c437af52b481c7"}, + {file = "regex-2023.8.8-cp36-cp36m-win_amd64.whl", hash = "sha256:aadf28046e77a72f30dcc1ab185639e8de7f4104b8cb5c6dfa5d8ed860e57236"}, + {file = "regex-2023.8.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:423adfa872b4908843ac3e7a30f957f5d5282944b81ca0a3b8a7ccbbfaa06103"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ae594c66f4a7e1ea67232a0846649a7c94c188d6c071ac0210c3e86a5f92109"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e51c80c168074faa793685656c38eb7a06cbad7774c8cbc3ea05552d615393d8"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:09b7f4c66aa9d1522b06e31a54f15581c37286237208df1345108fcf4e050c18"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e73e5243af12d9cd6a9d6a45a43570dbe2e5b1cdfc862f5ae2b031e44dd95a8"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:941460db8fe3bd613db52f05259c9336f5a47ccae7d7def44cc277184030a116"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f0ccf3e01afeb412a1a9993049cb160d0352dba635bbca7762b2dc722aa5742a"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:2e9216e0d2cdce7dbc9be48cb3eacb962740a09b011a116fd7af8c832ab116ca"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:5cd9cd7170459b9223c5e592ac036e0704bee765706445c353d96f2890e816c8"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4873ef92e03a4309b3ccd8281454801b291b689f6ad45ef8c3658b6fa761d7ac"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:239c3c2a339d3b3ddd51c2daef10874410917cd2b998f043c13e2084cb191684"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1005c60ed7037be0d9dea1f9c53cc42f836188227366370867222bda4c3c6bd7"}, + {file = "regex-2023.8.8-cp37-cp37m-win32.whl", hash = "sha256:e6bd1e9b95bc5614a7a9c9c44fde9539cba1c823b43a9f7bc11266446dd568e3"}, + {file = "regex-2023.8.8-cp37-cp37m-win_amd64.whl", hash = "sha256:9a96edd79661e93327cfeac4edec72a4046e14550a1d22aa0dd2e3ca52aec921"}, + {file = "regex-2023.8.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2181c20ef18747d5f4a7ea513e09ea03bdd50884a11ce46066bb90fe4213675"}, + {file = "regex-2023.8.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a2ad5add903eb7cdde2b7c64aaca405f3957ab34f16594d2b78d53b8b1a6a7d6"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9233ac249b354c54146e392e8a451e465dd2d967fc773690811d3a8c240ac601"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:920974009fb37b20d32afcdf0227a2e707eb83fe418713f7a8b7de038b870d0b"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2b6c5dfe0929b6c23dde9624483380b170b6e34ed79054ad131b20203a1a63"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96979d753b1dc3b2169003e1854dc67bfc86edf93c01e84757927f810b8c3c93"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ae54a338191e1356253e7883d9d19f8679b6143703086245fb14d1f20196be9"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2162ae2eb8b079622176a81b65d486ba50b888271302190870b8cc488587d280"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c884d1a59e69e03b93cf0dfee8794c63d7de0ee8f7ffb76e5f75be8131b6400a"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf9273e96f3ee2ac89ffcb17627a78f78e7516b08f94dc435844ae72576a276e"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:83215147121e15d5f3a45d99abeed9cf1fe16869d5c233b08c56cdf75f43a504"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3f7454aa427b8ab9101f3787eb178057c5250478e39b99540cfc2b889c7d0586"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0640913d2c1044d97e30d7c41728195fc37e54d190c5385eacb52115127b882"}, + {file = "regex-2023.8.8-cp38-cp38-win32.whl", hash = "sha256:0c59122ceccb905a941fb23b087b8eafc5290bf983ebcb14d2301febcbe199c7"}, + {file = "regex-2023.8.8-cp38-cp38-win_amd64.whl", hash = "sha256:c12f6f67495ea05c3d542d119d270007090bad5b843f642d418eb601ec0fa7be"}, + {file = "regex-2023.8.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:82cd0a69cd28f6cc3789cc6adeb1027f79526b1ab50b1f6062bbc3a0ccb2dbc3"}, + {file = "regex-2023.8.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bb34d1605f96a245fc39790a117ac1bac8de84ab7691637b26ab2c5efb8f228c"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:987b9ac04d0b38ef4f89fbc035e84a7efad9cdd5f1e29024f9289182c8d99e09"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9dd6082f4e2aec9b6a0927202c85bc1b09dcab113f97265127c1dc20e2e32495"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7eb95fe8222932c10d4436e7a6f7c99991e3fdd9f36c949eff16a69246dee2dc"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7098c524ba9f20717a56a8d551d2ed491ea89cbf37e540759ed3b776a4f8d6eb"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b694430b3f00eb02c594ff5a16db30e054c1b9589a043fe9174584c6efa8033"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b2aeab3895d778155054abea5238d0eb9a72e9242bd4b43f42fd911ef9a13470"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:988631b9d78b546e284478c2ec15c8a85960e262e247b35ca5eaf7ee22f6050a"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:67ecd894e56a0c6108ec5ab1d8fa8418ec0cff45844a855966b875d1039a2e34"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:14898830f0a0eb67cae2bbbc787c1a7d6e34ecc06fbd39d3af5fe29a4468e2c9"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f2200e00b62568cfd920127782c61bc1c546062a879cdc741cfcc6976668dfcf"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9691a549c19c22d26a4f3b948071e93517bdf86e41b81d8c6ac8a964bb71e5a6"}, + {file = "regex-2023.8.8-cp39-cp39-win32.whl", hash = "sha256:6ab2ed84bf0137927846b37e882745a827458689eb969028af8032b1b3dac78e"}, + {file = "regex-2023.8.8-cp39-cp39-win_amd64.whl", hash = "sha256:5543c055d8ec7801901e1193a51570643d6a6ab8751b1f7dd9af71af467538bb"}, + {file = "regex-2023.8.8.tar.gz", hash = "sha256:fcbdc5f2b0f1cd0f6a56cdb46fe41d2cce1e644e3b68832f3eeebc5fb0f7712e"}, ] [[package]] @@ -1700,24 +1730,42 @@ python-versions = "*" files = [ {file = "safetensors-0.3.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b6a66989075c2891d743153e8ba9ca84ee7232c8539704488f454199b8b8f84d"}, {file = "safetensors-0.3.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:670d6bc3a3b377278ce2971fa7c36ebc0a35041c4ea23b9df750a39380800195"}, + {file = "safetensors-0.3.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:564f42838721925b5313ae864ba6caa6f4c80a9fbe63cf24310c3be98ab013cd"}, + {file = "safetensors-0.3.2-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:7f80af7e4ab3188daaff12d43d078da3017a90d732d38d7af4eb08b6ca2198a5"}, + {file = "safetensors-0.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec30d78f20f1235b252d59cbb9755beb35a1fde8c24c89b3c98e6a1804cfd432"}, + {file = "safetensors-0.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16063d94d8f600768d3c331b1e97964b1bf3772e19710105fe24ec5a6af63770"}, {file = "safetensors-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbb44e140bf2aeda98d9dde669dbec15f7b77f96a9274469b91a6cf4bcc5ec3b"}, {file = "safetensors-0.3.2-cp310-cp310-win32.whl", hash = "sha256:2961c1243fd0da46aa6a1c835305cc4595486f8ac64632a604d0eb5f2de76175"}, {file = "safetensors-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c813920482c337d1424d306e1b05824a38e3ef94303748a0a287dea7a8c4f805"}, {file = "safetensors-0.3.2-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:707df34bd9b9047e97332136ad98e57028faeccdb9cfe1c3b52aba5964cc24bf"}, {file = "safetensors-0.3.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:becc5bb85b2947eae20ed23b407ebfd5277d9a560f90381fe2c42e6c043677ba"}, + {file = "safetensors-0.3.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:30a75707be5cc9686490bde14b9a371cede4af53244ea72b340cfbabfffdf58a"}, + {file = "safetensors-0.3.2-cp311-cp311-macosx_13_0_universal2.whl", hash = "sha256:54ad6af663e15e2b99e2ea3280981b7514485df72ba6d014dc22dae7ba6a5e6c"}, + {file = "safetensors-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37764b3197656ef507a266c453e909a3477dabc795962b38e3ad28226f53153b"}, + {file = "safetensors-0.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4939067736783acd8391d83cd97d6c202f94181951ce697d519f9746381b6a39"}, {file = "safetensors-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada0fac127ff8fb04834da5c6d85a8077e6a1c9180a11251d96f8068db922a17"}, {file = "safetensors-0.3.2-cp311-cp311-win32.whl", hash = "sha256:155b82dbe2b0ebff18cde3f76b42b6d9470296e92561ef1a282004d449fa2b4c"}, {file = "safetensors-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:a86428d196959619ce90197731be9391b5098b35100a7228ef4643957648f7f5"}, {file = "safetensors-0.3.2-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:c1f8ab41ed735c5b581f451fd15d9602ff51aa88044bfa933c5fa4b1d0c644d1"}, + {file = "safetensors-0.3.2-cp37-cp37m-macosx_13_0_x86_64.whl", hash = "sha256:bc9cfb3c9ea2aec89685b4d656f9f2296f0f0d67ecf2bebf950870e3be89b3db"}, + {file = "safetensors-0.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ace5d471e3d78e0d93f952707d808b5ab5eac77ddb034ceb702e602e9acf2be9"}, + {file = "safetensors-0.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de3e20a388b444381bcda1a3193cce51825ddca277e4cf3ed1fe8d9b2d5722cd"}, {file = "safetensors-0.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d7d70d48585fe8df00725aa788f2e64fd24a4c9ae07cd6be34f6859d0f89a9c"}, {file = "safetensors-0.3.2-cp37-cp37m-win32.whl", hash = "sha256:6ff59bc90cdc857f68b1023be9085fda6202bbe7f2fd67d06af8f976d6adcc10"}, {file = "safetensors-0.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:8b05c93da15fa911763a89281906ca333ed800ab0ef1c7ce53317aa1a2322f19"}, {file = "safetensors-0.3.2-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:8969cfd9e8d904e8d3c67c989e1bd9a95e3cc8980d4f95e4dcd43c299bb94253"}, + {file = "safetensors-0.3.2-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:f54148ac027556eb02187e9bc1556c4d916c99ca3cb34ca36a7d304d675035c1"}, + {file = "safetensors-0.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caec25fedbcf73f66c9261984f07885680f71417fc173f52279276c7f8a5edd3"}, + {file = "safetensors-0.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:50224a1d99927ccf3b75e27c3d412f7043280431ab100b4f08aad470c37cf99a"}, {file = "safetensors-0.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa98f49e95f02eb750d32c4947e7d5aa43883149ebd0414920866446525b70f0"}, {file = "safetensors-0.3.2-cp38-cp38-win32.whl", hash = "sha256:33409df5e28a83dc5cc5547a3ac17c0f1b13a1847b1eb3bc4b3be0df9915171e"}, {file = "safetensors-0.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:e04a7cbbb3856159ab99e3adb14521544f65fcb8548cce773a1435a0f8d78d27"}, {file = "safetensors-0.3.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:7c864cf5dcbfb608c5378f83319c60cc9c97263343b57c02756b7613cd5ab4dd"}, {file = "safetensors-0.3.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:14e8c19d6dc51d4f70ee33c46aff04c8ba3f95812e74daf8036c24bc86e75cae"}, + {file = "safetensors-0.3.2-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:042a60f633c3c7009fdf6a7c182b165cb7283649d2a1e9c7a4a1c23454bd9a5b"}, + {file = "safetensors-0.3.2-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:fafd95e5ef41e8f312e2a32b7031f7b9b2a621b255f867b221f94bb2e9f51ae8"}, + {file = "safetensors-0.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ed77cf358abce2307f03634694e0b2a29822e322a1623e0b1aa4b41e871bf8b"}, + {file = "safetensors-0.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d344e8b2681a33aafc197c90b0def3229b3317d749531c72fa6259d0caa5c8c"}, {file = "safetensors-0.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ff0024ef2e5722a79af24688ce4a430f70601d0cf712a744105ed4b8f67ba5"}, {file = "safetensors-0.3.2-cp39-cp39-win32.whl", hash = "sha256:827af9478b78977248ba93e2fd97ea307fb63f463f80cef4824460f8c2542a52"}, {file = "safetensors-0.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:9b09f27c456efa301f98681ea14b12f81f2637889f6336223ccab71e42c34541"}, @@ -1945,19 +1993,31 @@ files = [ [[package]] name = "torch" -version = "2.0.1+cu118" +version = "2.0.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" optional = false python-versions = ">=3.8.0" files = [ - {file = "torch-2.0.1+cu118-cp310-cp310-linux_x86_64.whl", hash = "sha256:a7a49d459bf4862f64f7bc1a68beccf8881c2fa9f3e0569608e16ba6f85ebf7b"}, - {file = "torch-2.0.1+cu118-cp310-cp310-win_amd64.whl", hash = "sha256:f58d75619bc96e4322343c030b893613701caa2d6db8017155da226c14171335"}, - {file = "torch-2.0.1+cu118-cp311-cp311-linux_x86_64.whl", hash = "sha256:143b6c658c17d43376e2dfbaa2c106d35639d615e5e8dec4429cf1e510dd8d61"}, - {file = "torch-2.0.1+cu118-cp311-cp311-win_amd64.whl", hash = "sha256:b663a4ee744d574095dbd612644de345944247c0605692309fd9f6c7ccdea022"}, - {file = "torch-2.0.1+cu118-cp38-cp38-linux_x86_64.whl", hash = "sha256:2ce38a6e4ea7c4b7f5baa51e65243a5f687f6e19ab7915ba5b2a431105f50bbe"}, - {file = "torch-2.0.1+cu118-cp38-cp38-win_amd64.whl", hash = "sha256:e58d26a11bd57ac19761c018c3151c15bc71d068afc8ec409bfd9b4cfcc63a52"}, - {file = "torch-2.0.1+cu118-cp39-cp39-linux_x86_64.whl", hash = "sha256:eb55f29db5744eda8a96f5594e637daed0d52278273005de759970e67cfa6a5a"}, - {file = "torch-2.0.1+cu118-cp39-cp39-win_amd64.whl", hash = "sha256:fa225b6f941ee0e78978ac85ed7744d3c19fff462473821f8060c14faa60043e"}, + {file = "torch-2.0.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:8ced00b3ba471856b993822508f77c98f48a458623596a4c43136158781e306a"}, + {file = "torch-2.0.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:359bfaad94d1cda02ab775dc1cc386d585712329bb47b8741607ef6ef4950747"}, + {file = "torch-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7c84e44d9002182edd859f3400deaa7410f5ec948a519cc7ef512c2f9b34d2c4"}, + {file = "torch-2.0.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:567f84d657edc5582d716900543e6e62353dbe275e61cdc36eda4929e46df9e7"}, + {file = "torch-2.0.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:787b5a78aa7917465e9b96399b883920c88a08f4eb63b5a5d2d1a16e27d2f89b"}, + {file = "torch-2.0.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:e617b1d0abaf6ced02dbb9486803abfef0d581609b09641b34fa315c9c40766d"}, + {file = "torch-2.0.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:b6019b1de4978e96daa21d6a3ebb41e88a0b474898fe251fd96189587408873e"}, + {file = "torch-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:dbd68cbd1cd9da32fe5d294dd3411509b3d841baecb780b38b3b7b06c7754434"}, + {file = "torch-2.0.1-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:ef654427d91600129864644e35deea761fb1fe131710180b952a6f2e2207075e"}, + {file = "torch-2.0.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:25aa43ca80dcdf32f13da04c503ec7afdf8e77e3a0183dd85cd3e53b2842e527"}, + {file = "torch-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5ef3ea3d25441d3957348f7e99c7824d33798258a2bf5f0f0277cbcadad2e20d"}, + {file = "torch-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0882243755ff28895e8e6dc6bc26ebcf5aa0911ed81b2a12f241fc4b09075b13"}, + {file = "torch-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:f66aa6b9580a22b04d0af54fcd042f52406a8479e2b6a550e3d9f95963e168c8"}, + {file = "torch-2.0.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:1adb60d369f2650cac8e9a95b1d5758e25d526a34808f7448d0bd599e4ae9072"}, + {file = "torch-2.0.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:1bcffc16b89e296826b33b98db5166f990e3b72654a2b90673e817b16c50e32b"}, + {file = "torch-2.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:e10e1597f2175365285db1b24019eb6f04d53dcd626c735fc502f1e8b6be9875"}, + {file = "torch-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:423e0ae257b756bb45a4b49072046772d1ad0c592265c5080070e0767da4e490"}, + {file = "torch-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8742bdc62946c93f75ff92da00e3803216c6cce9b132fbca69664ca38cfb3e18"}, + {file = "torch-2.0.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:c62df99352bd6ee5a5a8d1832452110435d178b5164de450831a3a8cc14dc680"}, + {file = "torch-2.0.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:671a2565e3f63b8fe8e42ae3e36ad249fe5e567435ea27b94edaa672a7d0c416"}, ] [package.dependencies] @@ -1965,33 +2025,27 @@ filelock = "*" jinja2 = "*" networkx = "*" sympy = "*" -triton = {version = "2.0.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} typing-extensions = "*" [package.extras] opt-einsum = ["opt-einsum (>=3.3)"] -[package.source] -type = "legacy" -url = "https://download.pytorch.org/whl/cu118" -reference = "pytorch-gpu-src" - [[package]] name = "tqdm" -version = "4.65.0" +version = "4.66.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, - {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, + {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, + {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] @@ -2065,43 +2119,6 @@ torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", video = ["av (==9.2.0)", "decord (==0.6.0)"] vision = ["Pillow (<10.0.0)"] -[[package]] -name = "triton" -version = "2.0.0" -description = "A language and compiler for custom Deep Learning operations" -optional = false -python-versions = "*" -files = [ - {file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"}, - {file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"}, - {file = "triton-2.0.0-1-cp36-cp36m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd"}, - {file = "triton-2.0.0-1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b"}, - {file = "triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef"}, - {file = "triton-2.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656"}, - {file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"}, - {file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"}, - {file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"}, - {file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"}, - {file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"}, - {file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"}, - {file = "triton-2.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fedce6a381901b1547e0e7e1f2546e4f65dca6d91e2d8a7305a2d1f5551895be"}, - {file = "triton-2.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75834f27926eab6c7f00ce73aaf1ab5bfb9bec6eb57ab7c0bfc0a23fac803b4c"}, - {file = "triton-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0117722f8c2b579cd429e0bee80f7731ae05f63fe8e9414acd9a679885fcbf42"}, - {file = "triton-2.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd9be5d0c2e45d2b7e6ddc6da20112b6862d69741576f9c3dbaf941d745ecae"}, - {file = "triton-2.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a0d2c3fc2eab4ba71384f2e785fbfd47aa41ae05fa58bf12cb31dcbd0aeceb"}, - {file = "triton-2.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c47b72c72693198163ece9d90a721299e4fb3b8e24fd13141e384ad952724f"}, -] - -[package.dependencies] -cmake = "*" -filelock = "*" -lit = "*" -torch = "*" - -[package.extras] -tests = ["autopep8", "flake8", "isort", "numpy", "pytest", "scipy (>=1.7.1)"] -tutorials = ["matplotlib", "pandas", "tabulate"] - [[package]] name = "typer" version = "0.6.1" @@ -2448,4 +2465,4 @@ quantize = ["accelerate", "datasets", "texttable"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "6b6dbd9e11bc2f71cd88e2ab50adce249a722fea02e7cdcf5fe1b94c7b997a57" +content-hash = "91a848038c08a44c67acfb4257781440ccc1e74a4b82f09513e75588fa33f72b" diff --git a/server/pyproject.toml b/server/pyproject.toml index 8eeae616..0e6d7882 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -31,8 +31,9 @@ einops = "^0.6.1" texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } peft = "^0.4.0" -torch = {version = "^2.0.1+cu118", source = "pytorch-gpu-src"} +torch = { version = "^2.0.1" } scipy = "^1.11.1" +pillow = "^10.0.0" [tool.poetry.extras] accelerate = ["accelerate"] diff --git a/server/requirements.txt b/server/requirements.txt index 5cc19619..012e91a2 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,11 +1,8 @@ ---extra-index-url https://download.pytorch.org/whl/cu118 - accelerate==0.20.3 ; python_version >= "3.9" and python_version < "3.13" backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13" certifi==2023.7.22 ; python_version >= "3.9" and python_version < "3.13" charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "3.13" click==8.1.6 ; python_version >= "3.9" and python_version < "3.13" -cmake==3.27.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "3.13" colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows") deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13" einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13" @@ -13,14 +10,13 @@ filelock==3.12.2 ; python_version >= "3.9" and python_version < "3.13" fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "3.13" googleapis-common-protos==1.60.0 ; python_version >= "3.9" and python_version < "3.13" grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "3.13" -grpcio-reflection==1.56.2 ; python_version >= "3.9" and python_version < "3.13" -grpcio-status==1.56.2 ; python_version >= "3.9" and python_version < "3.13" -grpcio==1.56.2 ; python_version >= "3.9" and python_version < "3.13" +grpcio-reflection==1.57.0 ; python_version >= "3.9" and python_version < "3.13" +grpcio-status==1.57.0 ; python_version >= "3.9" and python_version < "3.13" +grpcio==1.57.0 ; python_version >= "3.9" and python_version < "3.13" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "3.13" huggingface-hub==0.14.1 ; python_version >= "3.9" and python_version < "3.13" idna==3.4 ; python_version >= "3.9" and python_version < "3.13" jinja2==3.1.2 ; python_version >= "3.9" and python_version < "3.13" -lit==16.0.6 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "3.13" loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" markupsafe==2.1.3 ; python_version >= "3.9" and python_version < "3.13" mpmath==1.3.0 ; python_version >= "3.9" and python_version < "3.13" @@ -37,10 +33,11 @@ opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13" packaging==23.1 ; python_version >= "3.9" and python_version < "3.13" peft==0.4.0 ; python_version >= "3.9" and python_version < "3.13" -protobuf==4.23.4 ; python_version >= "3.9" and python_version < "3.13" +pillow==10.0.0 ; python_version >= "3.9" and python_version < "3.13" +protobuf==4.24.0 ; python_version >= "3.9" and python_version < "3.13" psutil==5.9.5 ; python_version >= "3.9" and python_version < "3.13" pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" -regex==2023.6.3 ; python_version >= "3.9" and python_version < "3.13" +regex==2023.8.8 ; python_version >= "3.9" and python_version < "3.13" requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" safetensors==0.3.2 ; python_version >= "3.9" and python_version < "3.13" scipy==1.11.1 ; python_version >= "3.9" and python_version < "3.13" @@ -48,10 +45,9 @@ sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" setuptools==68.0.0 ; python_version >= "3.9" and python_version < "3.13" sympy==1.12 ; python_version >= "3.9" and python_version < "3.13" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" -torch==2.0.1+cu118 ; python_version >= "3.9" and python_version < "3.13" -tqdm==4.65.0 ; python_version >= "3.9" and python_version < "3.13" +torch==2.0.1 ; python_version >= "3.9" and python_version < "3.13" +tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" transformers==4.31.0 ; python_version >= "3.9" and python_version < "3.13" -triton==2.0.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.9" and python_version < "3.13" typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "3.13" urllib3==2.0.4 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 621652e8..932ab32e 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -54,6 +54,7 @@ from text_generation_server.models.flash_santacoder import ( FlashSantacoderSharded, ) + from text_generation_server.models.idefics import IDEFICSSharded except ImportError as e: logger.warning(f"Could not import Flash Attention enabled models: {e}") @@ -64,6 +65,7 @@ __all__.append(FlashRWSharded) __all__.append(FlashSantacoderSharded) __all__.append(FlashLlama) + __all__.append(IDEFICSSharded) def get_model( @@ -248,6 +250,17 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) + elif model_type == "idefics": + if FLASH_ATTENTION: + return IDEFICSSharded( + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, + ) + else: + raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format("Idefics")) if sharded: raise ValueError("sharded is not supported for AutoModel") diff --git a/server/text_generation_server/models/custom_modeling/idefics_config.py b/server/text_generation_server/models/custom_modeling/idefics_config.py new file mode 100644 index 00000000..34925087 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/idefics_config.py @@ -0,0 +1,323 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Idefics model configuration""" +import copy + +from transformers import PretrainedConfig + +IDEFICS_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "HuggingFaceM4/idefics-9b": "https://huggingface.co/HuggingFaceM4/idefics-9b/blob/main/config.json", + "HuggingFaceM4/idefics-80b": "https://huggingface.co/HuggingFaceM4/idefics-80b/blob/main/config.json", +} + + +class IdeficsVisionConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`IdeficsModel`]. It is used to instantiate an + Idefics model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the Idefics-9B. + e.g. [HuggingFaceM4/idefics-9b](https://huggingface.co/HuggingFaceM4/idefics-9b) + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + Args: + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. (elsewhere referred to as `hidden_size`) + image_size (`int`, *optional*, defaults to 224): + The size (resolution) of each image. + intermediate_size (`int`, *optional*, defaults to 5120): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + patch_size (`int`, *optional*, defaults to 14): + The size (resolution) of each patch. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer encoder. + image_num_channels (`int`, *optional*, defaults to `3`): + Number of image channels. + hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. + layer_norm_eps (`float`, *optional*, defaults to 1e-5): + The epsilon used by the layer normalization layers. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_factor (`float`, *optional*, defaults to 1.0): + A factor for initializing all weight matrices (should be kept to 1.0, used internally for initialization + testing). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + """ + model_type = "idefics" + attribute_map = { + "hidden_size": "embed_dim", + } + + def __init__( + self, + embed_dim=768, + image_size=224, + intermediate_size=5120, + patch_size=14, + num_hidden_layers=32, + num_attention_heads=16, + num_channels=3, + hidden_act="quick_gelu", + layer_norm_eps=1e-5, + attention_dropout=0.0, + initializer_range=0.02, + initializer_factor=1.0, + **kwargs, + ): + self.embed_dim = embed_dim + self.image_size = image_size + self.intermediate_size = intermediate_size + self.patch_size = patch_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.layer_norm_eps = layer_norm_eps + self.attention_dropout = attention_dropout + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.hidden_act = hidden_act + + super().__init__(**kwargs) + + +class IdeficsPerceiverConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`IdeficsModel`]. It is used to instantiate an + Idefics model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the Idefics-9B. + e.g. [HuggingFaceM4/idefics-9b](https://huggingface.co/HuggingFaceM4/idefics-9b) + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + Args: + use_resampler (`bool`, *optional*, defaults to `False`): + Whether or not to use the resampler + resampler_n_latents (`int`, *optional*, defaults to ): + Number of latent embeddings to resample ("compress") the input sequence to (usually < 128). + resampler_depth (`int`, *optional*, defaults to 6): + Depth of the Perceiver Resampler (Transformer w/ cross attention). Should be shallow (< 3). + resampler_n_heads (`int`, *optional*, defaults to 16): + Number of heads in each Transformer block (for multi-headed self-attention). + resampler_head_dim (`int`, *optional*, defaults to 96): + Dimensionality of each head projection in the Transformer block. + qk_layer_norms_perceiver (`bool`, *optional*, defaults to `False`): + Whether or not to use qk layer norms in perceiver + """ + model_type = "idefics" + + def __init__( + self, + use_resampler=False, + resampler_n_latents=64, + resampler_depth=6, + resampler_n_heads=16, + resampler_head_dim=96, + qk_layer_norms_perceiver=False, + **kwargs, + ): + self.use_resampler = use_resampler + self.resampler_n_latents = resampler_n_latents + self.resampler_depth = resampler_depth + self.resampler_n_heads = resampler_n_heads + self.resampler_head_dim = resampler_head_dim + self.qk_layer_norms_perceiver = qk_layer_norms_perceiver + + super().__init__(**kwargs) + + +class IdeficsConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`IdeficsModel`]. It is used to instantiate an + Idefics model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the Idefics-9B. + e.g. [HuggingFaceM4/idefics-9b](https://huggingface.co/HuggingFaceM4/idefics-9b) + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + Args: + additional_vocab_size (`int`, *optional`, defaults to 0): + Additional vocabulary size of the model, typically for the special "" token. Additional vocab tokens + are always trainable whereas regular vocab tokens can be frozen or not. + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the Idefics model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`~IdeficsModel`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 11008): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + alpha_initializer (`str`, *optional*, defaults to `"zeros"`): + Initialization type for the alphas. + alphas_initializer_range (`float`, *optional*, defaults to 0.0): + The standard deviation of the truncated_normal_initializer for initializing the alphas in the Gated Cross + Attention. + alpha_type (`str`, *optional*, defaults to `"float"`): + Whether the gating alphas should be vectors or single floats. + rms_norm_eps (`float`, *optional*, defaults to 1e-6): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + pad_token_id (`int`, *optional*, defaults to 0) + Padding token id. + bos_token_id (`int`, *optional*, defaults to 1) + Beginning of stream token id. + eos_token_id (`int`, *optional*, defaults to 2) + End of stream token id. + tie_word_embeddings(`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + cross_layer_interval (`int`, *optional*, default to 1) + Interval for cross attention (from text to image) layers. + qk_layer_norms (`bool`, *optional*, defaults to `False`): Whether to add layer norm after q and k + freeze_text_layers (`bool`, *optional*, defaults to `True`): Whether to freeze text layers + freeze_text_module_exceptions (`bool`, *optional*, defaults to `[]`): + Exceptions to freezing text layers when `freeze_text_layers` is `True` + freeze_lm_head (`bool`, *optional*, defaults to `False`): Whether to freeze lm head + freeze_vision_layers (`bool`, *optional*, defaults to `True`): Whether to freeze vision layers + freeze_vision_module_exceptions (`bool`, *optional*, defaults to `[]`): + Exceptions to freezing vision layers when `freeze_vision_layers` is `True` + use_resampler (`bool`, *optional*, defaults to `False`): Whether to use the Resampler + vision_config (`IdeficsVisionConfig`, *optional*): Custom vision config or dict + perceiver_config (`IdeficsPerceiverConfig`, *optional*): Custom perceiver config or dict + Example: + ```python + >>> from transformers import IdeficsModel, IdeficsConfig + >>> # Initializing a Idefics idefics-9b style configuration + >>> configuration = IdeficsConfig() + >>> # Initializing a model from the idefics-9b style configuration + >>> model = IdeficsModel(configuration) + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "idefics" + is_composition = True + + def __init__( + self, + vocab_size=32000, + additional_vocab_size=0, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + dropout=0.0, + hidden_act="silu", + initializer_range=0.02, + alpha_initializer="zeros", + alphas_initializer_range=0.0, + alpha_type="float", + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + tie_word_embeddings=False, + cross_layer_interval=1, + qk_layer_norms=False, + freeze_text_layers=True, + freeze_text_module_exceptions=[], + freeze_lm_head=False, + freeze_vision_layers=True, + freeze_vision_module_exceptions=[], + use_resampler=False, + vision_config=None, + perceiver_config=None, + **kwargs, + ): + self.vocab_size = vocab_size + self.additional_vocab_size = additional_vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.dropout = dropout + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.alpha_initializer = alpha_initializer + self.alphas_initializer_range = alphas_initializer_range + self.alpha_type = alpha_type + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + + self.cross_layer_interval = cross_layer_interval + self.qk_layer_norms = qk_layer_norms + self.freeze_vision_layers = freeze_vision_layers + + self.freeze_text_layers = freeze_text_layers + self.freeze_text_module_exceptions = freeze_text_module_exceptions + self.freeze_vision_module_exceptions = freeze_vision_module_exceptions + self.freeze_lm_head = freeze_lm_head + + self.use_resampler = use_resampler + + if perceiver_config is None: + self.perceiver_config = IdeficsPerceiverConfig() + elif isinstance(perceiver_config, dict): + self.perceiver_config = IdeficsPerceiverConfig(**perceiver_config) + elif isinstance(perceiver_config, IdeficsPerceiverConfig): + self.perceiver_config = perceiver_config + + if vision_config is None: + self.vision_config = IdeficsVisionConfig() + elif isinstance(vision_config, dict): + self.vision_config = IdeficsVisionConfig(**vision_config) + elif isinstance(vision_config, IdeficsVisionConfig): + self.vision_config = vision_config + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + # IMPORTANT: Do not do any __init__ args-based checks in the constructor, since + # PretrainedConfig.from_dict first instantiates the class with the config dict and only then + # updates the config object with `kwargs` from from_pretrained, so during the instantiation + # of this object many attributes have default values and haven't yet been overridden. + # Do any required checks inside `from_pretrained` once the superclass' `from_pretrained` was run. + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + + output["vision_config"] = self.vision_config.to_dict() + output["perceiver_config"] = self.perceiver_config.to_dict() + output["model_type"] = self.__class__.model_type + + return output diff --git a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py new file mode 100644 index 00000000..aec9a3dc --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py @@ -0,0 +1,264 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Image processor class for Idefics.""" + +from typing import Callable, Dict, List, Optional, Union, Iterable +import numpy as np + +from PIL import Image + +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature +from transformers.image_transforms import resize, to_channel_dimension_format, rescale, normalize +from transformers.image_utils import ( + ChannelDimension, + ImageInput, + PILImageResampling, + make_list_of_images, + to_numpy_array, + valid_images, +) +from io import BytesIO +import requests +from transformers import TensorType, is_torch_available + + +IDEFICS_STANDARD_MEAN = [0.48145466, 0.4578275, 0.40821073] +IDEFICS_STANDARD_STD = [0.26862954, 0.26130258, 0.27577711] + + +def convert_to_rgb(image): + # `image.convert("RGB")` would only work for .jpg images, as it creates a wrong background + # for transparent images. The call to `alpha_composite` handles this case + if image.mode == "RGB": + return image + + image_rgba = image.convert("RGBA") + background = Image.new("RGBA", image_rgba.size, (255, 255, 255)) + alpha_composite = Image.alpha_composite(background, image_rgba) + alpha_composite = alpha_composite.convert("RGB") + return alpha_composite + + +class IdeficsImageProcessor(BaseImageProcessor): + r""" + Constructs a Idefics image processor. + Args: + image_size (`int`, *optional*, defaults to `224`): + Resize to image size + image_num_channels (`int`, *optional*, defaults to `3`): + Number of image channels. + image_mean (`float` or `List[float]`, *optional*, defaults to `IDEFICS_STANDARD_MEAN`): + Mean to use if normalizing the image. This is a float or list of floats the length of the number of + channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be + overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `List[float]`, *optional*, defaults to `IDEFICS_STANDARD_STD`): + Standard deviation to use if normalizing the image. This is a float or list of floats the length of the + number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. + Can be overridden by the `image_std` parameter in the `preprocess` method. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + image_size: int = 224, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + image_num_channels: Optional[int] = 3, + **kwargs, + ) -> None: + super().__init__(**kwargs) + + self.image_size = image_size + self.image_num_channels = image_num_channels + self.image_mean = image_mean + self.image_std = image_std + + def preprocess( + self, + images: ImageInput, + image_num_channels: Optional[int] = 3, + image_size: Optional[Dict[str, int]] = None, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + transform: Callable = None, + **kwargs, + ) -> TensorType.PYTORCH: + """ + Preprocess a batch of images. + Args: + images (`ImageInput`): + A list of images to preprocess. + image_size (`int`, *optional*, defaults to `self.image_size`): + Resize to image size + image_num_channels (`int`, *optional*, defaults to `self.image_num_channels`): + Number of image channels. + image_mean (`float` or `List[float]`, *optional*, defaults to `IDEFICS_STANDARD_MEAN`): + Mean to use if normalizing the image. This is a float or list of floats the length of the number of + channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can + be overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `List[float]`, *optional*, defaults to `IDEFICS_STANDARD_STD`): + Standard deviation to use if normalizing the image. This is a float or list of floats the length of the + number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` + method. Can be overridden by the `image_std` parameter in the `preprocess` method. + transform (`Callable`, *optional*, defaults to `None`): + A custom transform function that accepts a single image can be passed for training. For example, + `torchvision.Compose` can be used to compose multiple transforms. If `None` - an inference mode is + assumed - and then a preset of inference-specific transforms will be applied to the images + Returns: + a PyTorch tensor of the processed images + """ + image_size = image_size if image_size is not None else self.image_size + image_num_channels = image_num_channels if image_num_channels is not None else self.image_num_channels + image_mean = image_mean if image_mean is not None else self.image_mean + image_std = image_std if image_std is not None else self.image_std + size = (image_size, image_size) + + if len(images) == 0: + return [] + + images = make_list_of_images(images) + + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + # For training a user needs to pass their own set of transforms as a Callable. + # For reference this is what was used in the original IDEFICS training: + # transform = transforms.Compose([ + # convert_to_rgb, + # transforms.RandomResizedCrop((size, size), scale=(0.9, 1.0), interpolation=transforms.InterpolationMode.BICUBIC), + # transforms.ToTensor(), + # transforms.Normalize(mean=image_mean, std=image_std), + # ]) + if transform is not None: + if not is_torch_available(): + raise ImportError("To pass in `transform` torch must be installed") + import torch + + images = [transform(x) for x in images] + return torch.stack(images) + + # for inference we do the exact transforms that were used to train IDEFICS + images = [convert_to_rgb(x) for x in images] + # further transforms expect numpy arrays + images = [to_numpy_array(x) for x in images] + images = [resize(x, size, resample=PILImageResampling.BICUBIC) for x in images] + images = [self.rescale(image=image, scale=1 / 255) for image in images] + images = [self.normalize(x, mean=image_mean, std=image_std) for x in images] + images = [to_channel_dimension_format(x, ChannelDimension.FIRST) for x in images] + # TODO: this converts to torch tensors - switch to convert_to_tensors once it becomes available + images = BatchFeature(data={"pixel_values": images}, tensor_type=TensorType.PYTORCH)["pixel_values"] + + return images + + def fetch_images(self, image_url_or_urls: Union[str, List[str]]): + """ + Convert a single or a list of urls into the corresponding `PIL.Image` objects. + If a single url is passed, the return value will be a single object. If a list is passed a list of objects is + returned. + """ + headers = { + "User-Agent": ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0" + " Safari/537.36" + ) + } + if isinstance(image_url_or_urls, list): + return [self.fetch_images(x) for x in image_url_or_urls] + elif isinstance(image_url_or_urls, str): + response = requests.get(image_url_or_urls, stream=True, headers=headers) + response.raise_for_status() + return Image.open(BytesIO(response.content)) + else: + raise ValueError(f"only a single or a list of entries is supported but got type={type(image_url_or_urls)}") + + def rescale( + self, + image: np.ndarray, + scale: float, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`float`): + The scaling factor to rescale pixel values by. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The rescaled image. + """ + # return rescale(image, scale=scale, data_format=data_format, input_data_format=input_data_format, **kwargs) + # requires 4.32 + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + def normalize( + self, + image: np.ndarray, + mean: Union[float, Iterable[float]], + std: Union[float, Iterable[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `Iterable[float]`): + Image mean to use for normalization. + std (`float` or `Iterable[float]`): + Image standard deviation to use for normalization. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + + Returns: + `np.ndarray`: The normalized image. + """ + # TODO 4.32 + return normalize( + image, mean=mean, std=std, data_format=data_format, **kwargs + ) + + +import transformers +transformers.IdeficsImageProcessor = IdeficsImageProcessor diff --git a/server/text_generation_server/models/custom_modeling/idefics_modeling.py b/server/text_generation_server/models/custom_modeling/idefics_modeling.py new file mode 100644 index 00000000..8b43ae4d --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/idefics_modeling.py @@ -0,0 +1,1327 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch Idefics model.""" +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss + +from transformers import PreTrainedModel +from transformers.activations import ACT2FN +from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, dataclass +from transformers.modeling_utils import PretrainedConfig +from transformers.utils import ( + add_start_docstrings, + add_start_docstrings_to_model_forward, + logging, + replace_return_docstrings, +) +from text_generation_server.models.custom_modeling.idefics_config import IdeficsConfig +from text_generation_server.models.custom_modeling.idefics_vision import IdeficsVisionTransformer +from text_generation_server.models.custom_modeling.idefics_perceiver import IdeficsPerceiverResampler +from text_generation_server.utils.layers import ( + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelRowLinear, + TensorParallelHead, + PositionRotaryEmbedding, + FastLinear, +) +import dropout_layer_norm + +@dataclass +class BaseModelOutputWithPastImage(BaseModelOutputWithPast): + image_hidden_states: Optional[torch.FloatTensor] = None + +@dataclass +class CausalLMOutputWithPastImage(CausalLMOutputWithPast): + image_hidden_states: Optional[torch.FloatTensor] = None + + +# logger = logging.get_logger(__name__) + +# _CONFIG_FOR_DOC = "IdeficsConfig" + +# IDEFICS_PRETRAINED_MODEL_ARCHIVE_LIST = [ +# "HuggingFaceM4/idefics-9b", +# "HuggingFaceM4/idefics-80b", +# # See all Idefics models at https://huggingface.co/models?filter=idefics +# ] + + +def expand_inputs_for_generation( + input_ids, + expand_size=1, + is_encoder_decoder=False, + attention_mask=None, + encoder_outputs=None, + **model_kwargs, +): + expanded_return_idx = ( + torch.arange(input_ids.shape[0]).view(-1, 1).repeat(1, expand_size).view(-1).to(input_ids.device) + ) + input_ids = input_ids.index_select(0, expanded_return_idx) + + if "token_type_ids" in model_kwargs: + token_type_ids = model_kwargs["token_type_ids"] + model_kwargs["token_type_ids"] = token_type_ids.index_select(0, expanded_return_idx) + + if attention_mask is not None: + model_kwargs["attention_mask"] = attention_mask.index_select(0, expanded_return_idx) + model_kwargs["image_attention_mask"] = model_kwargs["image_attention_mask"].index_select( + 0, expanded_return_idx + ) + model_kwargs["pixel_values"] = model_kwargs["pixel_values"].index_select(0, expanded_return_idx) + + if is_encoder_decoder: + if encoder_outputs is None: + raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.") + encoder_outputs["last_hidden_state"] = encoder_outputs.last_hidden_state.index_select( + 0, expanded_return_idx.to(encoder_outputs.last_hidden_state.device) + ) + model_kwargs["encoder_outputs"] = encoder_outputs + return input_ids, model_kwargs + + +def update_model_kwargs_for_generation(outputs, model_kwargs, is_encoder_decoder=False): + # must have this key set to at least None + model_kwargs["past_key_values"] = model_kwargs.get("past_key_values", None) + + # update past + if "past_key_values" in outputs: + model_kwargs["past"] = outputs.past_key_values + elif "mems" in outputs: + model_kwargs["past"] = outputs.mems + elif "past_buckets_states" in outputs: + model_kwargs["past"] = outputs.past_buckets_states + else: + model_kwargs["past"] = None + + # update token_type_ids with last value + if "token_type_ids" in model_kwargs: + token_type_ids = model_kwargs["token_type_ids"] + model_kwargs["token_type_ids"] = torch.cat([token_type_ids, token_type_ids[:, -1].unsqueeze(-1)], dim=-1) + + # update attention masks + if not is_encoder_decoder: + if "attention_mask" in model_kwargs: + attention_mask = model_kwargs["attention_mask"] + model_kwargs["attention_mask"] = torch.cat( + [attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))], dim=-1 + ) + if "image_attention_mask" in model_kwargs: + image_attention_mask = model_kwargs["image_attention_mask"] + last_mask = image_attention_mask[:, -1, :].unsqueeze(1) + model_kwargs["image_attention_mask"] = last_mask + + return model_kwargs + + +def prepare_inputs_for_generation(input_ids, past=None, **kwargs): + token_type_ids = kwargs.get("token_type_ids", None) + # only last token for inputs_ids if past is defined in kwargs + if past: + input_ids = input_ids[:, -1].unsqueeze(-1) + if token_type_ids is not None: + token_type_ids = token_type_ids[:, -1].unsqueeze(-1) + + attention_mask = kwargs.get("attention_mask", None) + position_ids = kwargs.get("position_ids", None) + + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past: + position_ids = position_ids[:, -1].unsqueeze(-1) + + pixel_values = kwargs.get("pixel_values", None) + image_attention_mask = kwargs.get("image_attention_mask", None) + # if pixel_values is None or image_attention_mask is None: + # raise ValueError("pixel values and image attention mask cannot be None") + + return { + "input_ids": input_ids, + "past_key_values": past, + "use_cache": kwargs.get("use_cache"), + "position_ids": position_ids, + "attention_mask": attention_mask, + "token_type_ids": token_type_ids, + "pixel_values": pixel_values, + "image_attention_mask": image_attention_mask, + } + + +def freeze_model(model, module_exceptions=[]): + mapping = { + "LayerNorm": nn.LayerNorm, + "Linear": nn.Linear, + "Embedding": nn.Embedding, + } + module_exceptions_mapped = [mapping[m] for m in module_exceptions] + for module in model.modules(): + if module_exceptions and any([isinstance(module, t) for t in module_exceptions_mapped]): + module.requires_grad_(True) # Explicitely setting it to true to avoid any mistakes + else: + module.requires_grad_(False) + return model + + +class IdeficsDecoupledPartialTPEmbedding(nn.Module): + def __init__( + self, + config, + weights, + ): + super().__init__() + self.num_embeddings = config.vocab_size + self.weight = TensorParallelEmbedding(prefix="model.embed_tokens", weights=weights) + self.additional_weight = nn.Parameter(weights.get_tensor(f"model.embed_tokens.additional_embedding.weight")) + + def forward(self, input_ids): + # Clone so that we don't modify the original input_ids later on + input_ids = input_ids.clone() + additional_vocab_indices = torch.where(input_ids >= self.num_embeddings) + input_ids_additional_vocab = input_ids[additional_vocab_indices] + additional_embeddings = torch.nn.functional.embedding(input_ids_additional_vocab - self.num_embeddings, self.additional_weight) + + # for successful lookup replace input_ids with 0, the results of these will be discarded anyway + input_ids[additional_vocab_indices] = 0 + full_vector = self.weight(input_ids) + + # overwrite the records with high indices + full_vector[additional_vocab_indices] = additional_embeddings + + return full_vector + + +class IdeficsDecoupledTensorParallelLinear(nn.Module): + # Derived from https://pytorch.org/docs/stable/_modules/torch/nn/modules/linear.html#Linear + """ + Implements a decoupling of parameters to allow freezing (or not) a subset of the parameters. In practise, the + regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `out_additional_features` > 0, + then it will create `out_additional_features * in_features` additional parameters that are always trained. If + `out_additional_features=0`, then the module defaults back to the regular behavior of `nn.Linear`. + """ + + def __init__( + self, + config, + weights, + ) -> None: + super().__init__() + self.fc = TensorParallelHead.load( + config=config, prefix="lm_head", weights=weights + ) + self.additional_fc = FastLinear.load( + config=config, prefix="lm_head.additional_fc", weights=weights, bias=False, + ) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + output = self.fc(input) + additional_features = self.additional_fc(input) + output = torch.cat((output, additional_features), -1) + + return output + + def extra_repr(self) -> str: + """Overwriting `nn.Linear.extra_repr` to include new parameters.""" + return "in_features={}, out_features={}, out_additional_features={}, bias={}, partially_freeze={}".format( + self.in_features, + self.out_features, + self.out_additional_features, + self.bias is not None, + self.partially_freeze, + ) + + +# Copied from transformers.models.bart.modeling_bart._make_causal_mask +def _make_causal_mask( + input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 +): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device) + mask_cond = torch.arange(mask.size(-1), device=device) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +class IdeficsRMSNorm(nn.Module): + def __init__(self, prefix, weights, eps=1e-6): + """ + LlamaRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + + weight = weights.get_tensor(f"{prefix}.weight") + self.weight = nn.Parameter(weight) + self.variance_epsilon = eps + + def forward(self, hidden_states, residual=None): + if hidden_states.shape[-1] > 8192: + if residual is not None: + hidden_states += residual + residual = hidden_states + + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt( + variance + self.variance_epsilon + ) + + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states + else: + # faster post attention rms norm + unwrap = False + if len(hidden_states.shape) > 2: + unwrap = True + shape = hidden_states.shape + hidden_states = hidden_states.reshape(-1, shape[-1]) + + normed_hidden_states, res, *rest = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.weight, + None, + None, + None, + None, + None, + 0.0, + self.variance_epsilon, + 1.0, + 0, + None, + False, + True, # Activate RMSNorm + ) + if res is None: + res = hidden_states + + if unwrap: + normed_hidden_states = normed_hidden_states.view(*shape) + + + return normed_hidden_states + + +# this was adapted from LlamaMLP +class IdeficsMLP(nn.Module): + def __init__( + self, + config, + prefix, + weights, + ): + super().__init__() + self.gate_up_proj = TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.gate_proj", f"{prefix}.up_proj"], + weights=weights, + dim=0, + bias=False, + ) + self.down_proj = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.down_proj", weights=weights, bias=False, + ) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_states): + gate_up_states = self.gate_up_proj(hidden_states) + shape = gate_up_states.shape + gate_up_states = gate_up_states.view(*shape[:-1], 2, shape[-1] // 2) + return self.down_proj(self.act_fn(gate_up_states[:, :, 0]) * gate_up_states[:, :, 1]) + + +# this was adapted from LlamaAttention +class IdeficsAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config, + prefix, + weights, + qk_layer_norms: bool = False, + is_cross_attention: bool = False, + ): + super().__init__() + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.dropout = config.dropout + + if (self.head_dim * self.num_heads) != self.hidden_size: + raise ValueError( + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}" + f" and `num_heads`: {self.num_heads})." + ) + + self.is_cross_attention = is_cross_attention + + # if not hasattr(nn.functional, "scaled_dot_product_attention"): + # raise ValueError("this model requires pytorch 2.0 or higher") + + process_group = weights.process_group + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) + self.num_heads //= weights.process_group.size() + + if self.is_cross_attention: + # kv_input_dim = ( + # self.hidden_size if not hasattr(config.vision_config, "embed_dim") else config.vision_config.embed_dim + # ) + self.q_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.q_proj", weights=weights, bias=False + ) + self.k_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.k_proj", weights=weights, bias=False + ) + self.v_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.v_proj", weights=weights, bias=False + ) + else: + self.qkv = TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + dim=0, + weights=weights, + bias=False, + ) + self.o_proj = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.o_proj", weights=weights, bias=False + ) + self.rotary_emb = PositionRotaryEmbedding.static( + config=config, dim=self.head_dim, base=10000.0, device=weights.device + ) + self.qk_layer_norms = qk_layer_norms + if self.qk_layer_norms: + self.q_layer_norm = IdeficsRMSNorm( + prefix=f"{prefix}.q_layer_norm", weights=weights, eps=config.rms_norm_eps + ) + self.k_layer_norm = IdeficsRMSNorm( + prefix=f"{prefix}.q_layer_norm", weights=weights, eps=config.rms_norm_eps + ) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # if key_value_states are provided this layer is used as a cross-attention layer + is_cross_attention = self.is_cross_attention or key_value_states is not None + + bsz, q_len, _ = hidden_states.size() + + if is_cross_attention: + query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim)# .transpose(1, 2) + query_states = query_states.transpose(1, 2) + _, kv_len, _ = key_value_states.size() # Note that, in this case, `kv_len` == `kv_seq_len` + key_states = self.k_proj(key_value_states).view(bsz, kv_len, self.num_heads, self.head_dim).transpose(1, 2) + value_states = ( + self.v_proj(key_value_states).view(bsz, kv_len, self.num_heads, self.head_dim).transpose(1, 2) + ) + else: + qkv = self.qkv(hidden_states) + query_states, key_states, value_states = qkv.split(self.num_heads * self.head_dim, dim=2) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)# .transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_heads, self.head_dim)# . transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_heads, self.head_dim)# .transpose(1, 2) + kv_seq_len = q_len + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + max_s = max(kv_seq_len, q_len) + cos, sin = self.rotary_emb.get_cos_sin( + position_ids.view(-1), max_s, hidden_states.dtype + ) + + shape = query_states.shape + query_states = self.rotary_emb(query_states.view(-1, *shape[2:]), cos, sin).view(shape) + + shape = key_states.shape + key_states = self.rotary_emb(key_states.reshape(-1, *shape[2:]), cos, sin).view(shape) + + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + kv_seq_len = key_states.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + # [bsz, nh, t, hd] + + if past_key_value is not None: + # reuse k, v, self_attention + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + + past_key_value = (key_states, value_states) if use_cache else None + + if self.qk_layer_norms: + query_states = self.q_layer_norm(query_states) + key_states = self.k_layer_norm(key_states) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, q_len, kv_seq_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}" + ) + + attn_output = nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=attention_mask, + dropout_p=self.dropout, + ) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, q_len, -1) + + attn_output = self.o_proj(attn_output) + + attn_weights = None + if output_attentions: + logger.warning_once( + "attn_weights are not extracted in scaled_dot_product_attention. The model returns None instead" + ) + + return attn_output, attn_weights, past_key_value + + +# this was adapted from LlamaDecoderLayer +class IdeficsDecoderLayer(nn.Module): + def __init__(self, layer_id: int, config: IdeficsConfig, weights): + super().__init__() + self.process_group = weights.process_group + self.hidden_size = config.hidden_size + prefix = f"model.layers.{layer_id}" + self.self_attn = IdeficsAttention( + config=config, + prefix=f"{prefix}.self_attn", + weights=weights, + qk_layer_norms=False, + is_cross_attention=False, + ) + self.mlp = IdeficsMLP( + config=config, + prefix=f"{prefix}.mlp", + weights=weights, + ) + self.input_layernorm = IdeficsRMSNorm(prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps) + self.post_attention_layernorm = IdeficsRMSNorm(prefix=f"{prefix}.post_attention_layernorm", weights=weights, eps=config.rms_norm_eps) + self.dropout = config.dropout + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + """ + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + # hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + # hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +class IdeficsGatedCrossAttentionLayer(nn.Module): + def __init__(self, layer_id, config: IdeficsConfig, weights): + super().__init__() + self.process_group = weights.process_group + self.hidden_size = config.hidden_size + prefix = f"model.gated_cross_attn_layers.{layer_id}" + self.cross_attn = IdeficsAttention( + config=config, + prefix=f"{prefix}.cross_attn", + weights=weights, + qk_layer_norms=True, + is_cross_attention=True, + ) + self.mlp = IdeficsMLP( + config=config, + prefix=f"{prefix}.mlp", + weights=weights, + ) + self.input_layernorm = IdeficsRMSNorm(prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps) + self.post_attention_layernorm = IdeficsRMSNorm(prefix=f"{prefix}.post_attention_layernorm", weights=weights, eps=config.rms_norm_eps) + self.config = config.dropout + + self.act_cross_attn = nn.Tanh() + self.act_dense = nn.Tanh() + + self.alpha_cross_attn = nn.Parameter(weights.get_tensor(f"{prefix}.alpha_cross_attn")) + self.alpha_dense = nn.Parameter(weights.get_tensor(f"{prefix}.alpha_dense")) + + if not (hasattr(self, "alpha_cross_attn") and hasattr(self, "alpha_dense")): + raise ValueError("Alpha parameters not initialized correctly!") + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + image_hidden_states: Optional[torch.Tensor] = None, + image_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + no_images: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + no_images (`bool`, *optional*, defaults to `False`): If `True` the vision part is ignored + """ + if image_hidden_states is None: + raise ValueError( + "`image_hidden_states` is required for Idefics cross attention module which are visual features to be" + " conditioned on." + ) + + if past_key_value is not None: + raise NotImplementedError("Past key value states are not implemented for Idefics cross attention module.") + + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.cross_attn( + hidden_states=hidden_states, + key_value_states=image_hidden_states, + attention_mask=image_attention_mask, + output_attentions=output_attentions, + ) + # hidden_states = nn.functional.dropout(hidden_states, p=self.config, training=self.training) + # when there are no images the model is used in pure language mode + gate = 0 if no_images else 1 + hidden_states = residual + gate * self.act_cross_attn(self.alpha_cross_attn) * hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + # hidden_states = nn.functional.dropout(hidden_states, p=self.config, training=self.training) + hidden_states = residual + self.act_dense(self.alpha_dense) * hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +LLAMA_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`IdeficsConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +# @add_start_docstrings( +# "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", +# LLAMA_START_DOCSTRING, +# ) +class IdeficsPreTrainedModel(PreTrainedModel): + config_class = IdeficsConfig + # base_model_prefix = "model" + # supports_gradient_checkpointing = True + # _no_split_modules = ["IdeficsDecoderLayer", "IdeficsGatedCrossAttentionLayer"] + + # def _init_weights(self, module): + # # important: this ported version of Idefics isn't meant for training from scratch - only + # # inference and fine-tuning - so the proper init weights code has been removed - the m4 code + # # base should be used for training from scratch and it contains the correct code. + # std = self.config.initializer_range + # if isinstance(module, nn.Linear): + # module.weight.data.normal_(mean=0.0, std=std) + # if module.bias is not None: + # module.bias.data.zero_() + # elif isinstance(module, nn.Embedding): + # module.weight.data.normal_(mean=0.0, std=std) + # if module.padding_idx is not None: + # module.weight.data[module.padding_idx].zero_() + + # def _set_gradient_checkpointing(self, module, value=False): + # if isinstance(module, IdeficsModel): + # module.gradient_checkpointing = value + + +# LLAMA_INPUTS_DOCSTRING = r""" +# Args: +# input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): +# Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide +# it. + +# Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and +# [`PreTrainedTokenizer.__call__`] for details. + +# [What are input IDs?](../glossary#input-ids) +# attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): +# Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + +# - 1 for tokens that are **not masked**, +# - 0 for tokens that are **masked**. + +# [What are attention masks?](../glossary#attention-mask) + +# Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and +# [`PreTrainedTokenizer.__call__`] for details. + +# If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see +# `past_key_values`). + +# If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] +# and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more +# information on the default strategy. + +# - 1 indicates the head is **not masked**, +# - 0 indicates the head is **masked**. +# position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): +# Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, +# config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) +# past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): +# Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape +# `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape +# `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + +# Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention +# blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + +# If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that +# don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all +# `decoder_input_ids` of shape `(batch_size, sequence_length)`. +# inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): +# Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This +# is useful if you want more control over how to convert `input_ids` indices into associated vectors than the +# model's internal embedding lookup matrix. +# use_cache (`bool`, *optional*): +# If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see +# `past_key_values`). +# output_attentions (`bool`, *optional*): +# Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned +# tensors for more detail. +# output_hidden_states (`bool`, *optional*): +# Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for +# more detail. +# return_dict (`bool`, *optional*): +# Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +# """ + + +# @add_start_docstrings( +# "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", +# LLAMA_START_DOCSTRING, +# ) +class IdeficsModel(IdeficsPreTrainedModel): + # """ + # Transformer decoder consisting of `config.num_hidden_layers` layers. Each layer is a [`IdeficsDecoderLayer`] + + # Args: + # config: IdeficsConfig + # """ + + def __init__(self, config: IdeficsConfig, weights): + super().__init__(config) + self.config = config + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = IdeficsDecoupledPartialTPEmbedding( + config=config, + weights=weights, + ) + + self.image_size = config.vision_config.image_size + self.vision_config = config.vision_config + self.vision_model = IdeficsVisionTransformer( + prefix="model.vision_model", + config=config.vision_config, + weights=weights, + ) + + # Perceiver Resampler + if config.use_resampler: + perceiver_config = config.perceiver_config + self.perceiver_resampler = IdeficsPerceiverResampler( + prefix=f"model.perceiver_resampler", + config=config, + embed_dim=config.vision_config.embed_dim, + depth=perceiver_config.resampler_depth, + n_heads=perceiver_config.resampler_n_heads, + head_dim=perceiver_config.resampler_head_dim, + n_latents=perceiver_config.resampler_n_latents, + weights=weights, + ) + + self.layers = nn.ModuleList( + [ + IdeficsDecoderLayer(layer_id, config, weights) + for layer_id in range(config.num_hidden_layers) + ] + ) + + self.cross_layer_interval = config.cross_layer_interval + num_cross_layers = config.num_hidden_layers // self.cross_layer_interval + self.gated_cross_attn_layers = nn.ModuleList( + [ + IdeficsGatedCrossAttentionLayer(layer_id, config, weights) + for layer_id in range(num_cross_layers)] + ) + # self.gradient_checkpointing = False + + self.norm = IdeficsRMSNorm(prefix=f"model.norm", weights=weights, eps=config.rms_norm_eps) + + # self.gradient_checkpointing = False + # Initialize weights and apply final processing + # self.post_init() + + # self.freeze_relevant_params(config) + + # def freeze_relevant_params(self, config=None): + # if config is None: + # config = self.config + + # if config.freeze_text_layers: + # self.freeze_text_layers(config.freeze_text_module_exceptions) + + # if config.freeze_vision_layers: + # freeze_model(self.vision_model, module_exceptions=config.freeze_vision_module_exceptions) + + # def freeze_text_layers(self, module_exceptions=[]): + # for module in [self.layers, self.norm]: + # freeze_model(module, module_exceptions=module_exceptions) + + # def freeze_vision_layers(self, module_exceptions=[]): + # freeze_model(self.vision_model, module_exceptions=module_exceptions) + + # def get_input_embeddings(self): + # return self.embed_tokens + + # def set_input_embeddings(self, value): + # self.embed_tokens = value + + # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, + inputs_embeds.dtype, + device=inputs_embeds.device, + past_key_values_length=past_key_values_length, + ) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( + inputs_embeds.device + ) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + + # @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + image_hidden_states: Optional[torch.FloatTensor] = None, + image_embeddings: Optional[torch.FloatTensor] = None, + image_attention_mask: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPastImage]: + device = input_ids.device if input_ids is not None else inputs_embeds.device + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + batch_size, seq_length = input_ids.shape + elif inputs_embeds is not None: + batch_size, seq_length, _ = inputs_embeds.shape + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + seq_length_with_past = seq_length + past_key_values_length = 0 + + if past_key_values is not None: + past_key_values_length = past_key_values[0][0].shape[2] + seq_length_with_past = seq_length_with_past + past_key_values_length + + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + elif position_ids is None: + device = input_ids.device if input_ids is not None else inputs_embeds.device + position_ids = torch.arange( + past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + ) + position_ids = position_ids.unsqueeze(0).view(-1, seq_length) + else: + position_ids = position_ids.view(-1, seq_length).long() + + no_images = False + + if image_hidden_states is None: + if pixel_values is None and image_embeddings is None: + raise ValueError("Either pixel_values and image_embeddings have to be not-None.") + + elif pixel_values is not None and image_embeddings is not None: + raise ValueError("You cannot specify both pixel_values and image_embeddings at the same time") + + elif pixel_values is not None: + no_images = len(torch.nonzero(pixel_values)) == 0 + pixel_values = pixel_values.to(dtype=self.dtype, device=device) # fp16 compatibility + batch_size, num_images = pixel_values.shape[:2] + pixel_values = pixel_values.contiguous().view(batch_size * num_images, *pixel_values.shape[2:]) + + # Get sequence from the vision encoder + image_hidden_states = self.vision_model(pixel_values=pixel_values).last_hidden_state + + elif image_embeddings is not None: + batch_size, num_images, image_seq_len, image_hidden_size = image_embeddings.size() + image_hidden_states = image_embeddings.to(dtype=self.dtype, device=input_ids.device) + image_hidden_states = image_hidden_states.view(batch_size * num_images, image_seq_len, image_hidden_size) + + if self.config.use_resampler: + image_hidden_states = self.perceiver_resampler(image_hidden_states) + image_seq_len, image_hidden_size = image_hidden_states.size(1), image_hidden_states.size(2) + image_hidden_states = image_hidden_states.view(batch_size, num_images * image_seq_len, image_hidden_size) + else: + no_images = False + num_images = pixel_values.shape[1] + image_seq_len = image_hidden_states.shape[1] // num_images + + # # Hack to use the model in full language modeling mode + # image_attention_mask = torch.zeros(batch_size, seq_length, 1, dtype=torch.long, device=image_hidden_states.device) + # Make image_attention_mask compatible with hidden states + text_seq_len = image_attention_mask.size(1) + image_attention_mask = image_attention_mask.unsqueeze(-1) + image_attention_mask = image_attention_mask.repeat(1, 1, 1, image_seq_len) + image_attention_mask = image_attention_mask.view(batch_size, text_seq_len, num_images * image_seq_len) + image_batch_size, image_sequence_length, _ = image_hidden_states.size() + image_hidden_shape = (image_batch_size, image_sequence_length) + if image_attention_mask is None: + image_attention_mask = torch.ones(image_hidden_shape, device=device) + image_attention_mask = self.invert_attention_mask(image_attention_mask) + + # if list(image_attention_mask.shape) != [4, 1, 1024, 64]: + # raise ValueError(f"Image hidden_states {image_hidden_states.shape} - mask {image_attention_mask.shape} {num_images} {image_seq_len} {text_seq_len}") + + + # if image_hidden_states is not None: + # else: + # image_attention_mask = None + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + # embed positions + if attention_mask is None: + attention_mask = torch.ones( + (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device + ) + attention_mask = self._prepare_decoder_attention_mask( + attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length + ) + + hidden_states = inputs_embeds + + # if self.gradient_checkpointing and self.training: + # if use_cache: + # logger.warning_once( + # "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + # ) + # use_cache = False + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = () if use_cache else None + + for idx, decoder_layer in enumerate(self.layers): + if output_hidden_states: + all_hidden_states += (hidden_states,) + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + def vblock( + main_block, + hidden_states, + attention_mask, + position_ids, + past_key_value, + image_hidden_states, + image_attention_mask, + output_attentions, + use_cache, + no_images, + layer_idx, + cross_layer_interval, + gated_cross_attn_layers, + ): + # TODO(ls): Add cross attention values to respective lists + if layer_idx % cross_layer_interval == 0: + xblock = gated_cross_attn_layers[layer_idx // cross_layer_interval] + outputs = xblock( + hidden_states, + attention_mask=attention_mask, + image_hidden_states=image_hidden_states, + image_attention_mask=image_attention_mask, + output_attentions=output_attentions, + use_cache=use_cache, + past_key_value=None, # not implemented + no_images=no_images, + ) + hidden_states = outputs[0] + + layer_outputs = main_block( + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + ) + + return layer_outputs + + # if self.gradient_checkpointing and self.training: + # past_key_value = None + # if use_cache: + # logger.warning_once( + # "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + # ) + # use_cache = False + + # layer_outputs = torch.utils.checkpoint.checkpoint( + # vblock, + # decoder_layer, + # hidden_states, + # attention_mask, + # position_ids, + # past_key_value, + # image_hidden_states, + # image_attention_mask, + # output_attentions, + # use_cache, + # no_images, + # idx, + # self.cross_layer_interval, + # self.gated_cross_attn_layers, + # ) + # else: + layer_outputs = vblock( + decoder_layer, + hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + image_hidden_states=image_hidden_states, + image_attention_mask=image_attention_mask, + output_attentions=output_attentions, + use_cache=use_cache, + no_images=no_images, + layer_idx=idx, + cross_layer_interval=self.cross_layer_interval, + gated_cross_attn_layers=self.gated_cross_attn_layers, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache += (layer_outputs[2 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPastImage( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + image_hidden_states=image_hidden_states, + ) + + +class IdeficsForVisionText2Text(IdeficsPreTrainedModel): + def __init__( + self, + config, + weights, + ): + super().__init__(config) + self.model = IdeficsModel( + config=config, + weights=weights, + ) + + self.lm_head = IdeficsDecoupledTensorParallelLinear( + config=config, + weights=weights, + ) + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + image_embeddings: Optional[torch.FloatTensor] = None, + image_hidden_states: Optional[torch.FloatTensor] = None, + image_attention_mask: Optional[torch.Tensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPastImage]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, LlamaForCausalLM + + >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS) + >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER) + + >>> prompt = "Hey, are you consciours? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you consciours? Can you talk to me?\nI'm not consciours, but I can talk to you." + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + pixel_values=pixel_values, + image_embeddings=image_embeddings, + image_hidden_states=image_hidden_states, + image_attention_mask=image_attention_mask, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + logits = self.lm_head(hidden_states) + + loss = None + + return CausalLMOutputWithPastImage( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + image_hidden_states=outputs.image_hidden_states + ) + + def prepare_inputs_for_generation(self, input_ids, past=None, **kwargs): + inputs = prepare_inputs_for_generation(input_ids, past=past, **kwargs) + unwanted_kwargs = ["token_type_ids"] + for kwarg in unwanted_kwargs: + inputs.pop(kwarg, None) + return inputs + + @staticmethod + def _expand_inputs_for_generation( + *args, + **model_kwargs, + ): + return expand_inputs_for_generation(*args, **model_kwargs) + + @staticmethod + def _update_model_kwargs_for_generation(outputs, model_kwargs, is_encoder_decoder=False): + return update_model_kwargs_for_generation(outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder) + + @staticmethod + def _reorder_cache(past, beam_idx): + reordered_past = () + for layer_past in past: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past diff --git a/server/text_generation_server/models/custom_modeling/idefics_perceiver.py b/server/text_generation_server/models/custom_modeling/idefics_perceiver.py new file mode 100644 index 00000000..def78390 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/idefics_perceiver.py @@ -0,0 +1,246 @@ +# This code was adapted from https://github.com/lucidrains/flamingo-pytorch licensed under the MIT License. +# +# MIT License +# +# Copyright (c) 2020 The Google AI Language Team Authors, The HuggingFace Inc. team and github/lonePatient +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +""" + +Generic interface to various configurations of the Perceiver Resampler, that simply takes in a series of (potentially +time-indexed) contextual embeddings, and "resamples" (compresses) them down to a pre-specified number of latents! Note +that the Perceiver in general resamples based solely off the *long-range* context; there's a nice opportunity here to +prime the Perceiver Resampler with say a single layer's worth of language embeddings (the target domain), and use that +to softly "retrieve & compress" what we need --> this would be a novel contribution we should explore. + +References: + - DeepMind's Flamingo: https://www.deepmind.com/blog/tackling-multiple-tasks-with-a-single-visual-language-model + - Code borrowed w/ love from: https://github.com/lucidrains/flamingo-pytorch + +""" +from typing import Optional, Tuple + +import torch +import torch.nn as nn + +from text_generation_server.utils.layers import ( + TensorParallelColumnLinear, + TensorParallelRowLinear, +) + +EPS=1e-5 + +class IdeficsPerceiverResampler(nn.Module): + def __init__( + self, + prefix, + config, + embed_dim: int, + depth: int, + n_heads: int, + head_dim: int, + n_latents: int, + weights, + ) -> None: + """ + Instantiates a Perceiver Resampler that operates over a sequence of embeddings (say from a ResNet or ViT or + MAE) of a given dimension, performs `depth` blocks of cross-attention with a fixed `n_latents` inputs, then + returns a Tensor of shape [bsz, n_latents, embed_dim]. :param embed_dim: Dimensionality of embeddings being fed + to the Perceiver Resampler (also dimensionality of latent embeddings *returned* by the Perceiver Resampler. + Could be e.g., VIT embed_dim, ResNet pool dim, and so on. + + Args: + config (`IdeficsConfig`): config object + embed_dim (`int`): The size of each embedding vector + depth (`int`): Depth of the Perceiver Resampler (Transformer w/ cross attention). Should be shallow (< 3). + n_heads (`int`): Number of heads in each Transformer block (for multi-headed self-attention). + head_dim (`int`): Dimensionality of each head projection in the Transformer block. + n_latents (`int`): + Number of latent embeddings to resample ("compress") the input sequence to (usually < 128). + + """ + super().__init__() + self.embed_dim, self.n_heads, self.head_dim, self.n_latents = embed_dim, n_heads, head_dim, n_latents + self.qk_layer_norms = config.perceiver_config.qk_layer_norms_perceiver + + # Create Latents for Perceiver + self.latents = nn.Parameter(weights.get_tensor(f"{prefix}.latents")) + + self.intermediate_dim = ( + self.embed_dim * 4 + if not hasattr(config.vision_config, "embed_dim") + else config.vision_config.embed_dim * 4 + ) + # Create Transformer Blocks + self.blocks = nn.ModuleList( + [ + nn.ModuleList( + [ + IdeficsPerceiverAttention( + prefix=f"{prefix}.blocks.{layer_id}.0", + config=config, + embed_dim=self.embed_dim, + n_heads=self.n_heads, + head_dim=self.head_dim, + qk_layer_norms=self.qk_layer_norms, + weights=weights, + ), + IdeficsMLP( + prefix=f"{prefix}.blocks.{layer_id}.1", + intermediate_size=self.intermediate_dim, + config=config, + weights=weights + ), + ] + ) + for layer_id in range(depth) + ] + ) + self.layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.layer_norm", weights=weights, eps=EPS) + + def forward(self, context: torch.Tensor) -> torch.Tensor: + """Resample arbitrary length context & *compress* down to self.n_latents latent embeddings""" + # einsum.repeat(self.latents, "seq embed -> bsz seq embed", bsz=context.shape[0]) + latents = self.latents.repeat(context.shape[0], 1, 1) + + # Feed through Perceiver Attention blocks... + for attn, ff in self.blocks: + latents = attn(context, latents) + latents + latents = ff(latents) + latents + + return self.layer_norm(latents) + + +class IdeficsPerceiverAttention(nn.Module): + def __init__(self, + prefix, + config, + embed_dim: int, + n_heads: int, + head_dim: int, + qk_layer_norms: bool, + weights + ) -> None: + """Perceiver Cross-Attention Module --> let long-form inputs be `context`, resampled embeddings be `latents`""" + super().__init__() + self.embed_dim, self.n_heads, self.head_dim = embed_dim, n_heads, head_dim + self.qk_layer_norms = qk_layer_norms + # Normalization & Scaling + self.context_layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.context_layer_norm", weights=weights, eps=EPS) + self.latents_layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.latents_layer_norm", weights=weights, eps=EPS) + if self.qk_layer_norms: + self.q_layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.q_layer_norm", weights=weights, eps=EPS) + self.k_layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.k_layer_norm", weights=weights, eps=EPS) + + self.qk_scale = self.head_dim**-0.5 + + process_group = weights.process_group + if n_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {n_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) + self.n_heads //= weights.process_group.size() + + # Q, K, V Projection (no bias -- detail from Perceiver/Flamingo Papers). + self.q_proj = TensorParallelColumnLinear.load( + config=config, prefix=f"{prefix}.q_proj", weights=weights, bias=False + ) + self.k_proj = TensorParallelColumnLinear.load( + config=config, prefix=f"{prefix}.k_proj", weights=weights, bias=False + ) + self.v_proj = TensorParallelColumnLinear.load( + config=config, prefix=f"{prefix}.v_proj", weights=weights, bias=False + ) + + self.output_proj = TensorParallelRowLinear.load( + config=config, prefix=f"{prefix}.output_proj", weights=weights, bias=False + ) + + def forward(self, context: torch.Tensor, latents: torch.Tensor) -> torch.Tensor: + """ + Runs Perceiver Self-Attention, with special (context, latents) appended along the `seq` dimension! + + Args: + context (`torch.Tensor`): + Tensor of shape `[bsz, seq, embed_dim]` representing long-form context to resample. + latents (`torch.Tensor`): + Tensor of shape `[bsz, n_latents, embed_dim]` representing fixed length latents to compress to. + + Returns: + `torch.Tensor`: Tensor of shape `[bsz, n_latents, embed_dim]` representing attention over latents w/ cross + from context. + """ + context = self.context_layer_norm(context) + latents = self.latents_layer_norm(latents) + batch_size, seq_length, embed_dim = context.shape[:3] + + # Query, Key, Value Projections --> Note that in Flamingo, latents are *concatenated* with context prior to attn! + # Note: This results in queries w/ `seq = n_latents`, and keys, values with `seq = len(context) + n_latents` + q = self.q_proj(latents) + k = self.k_proj(torch.cat([context, latents], dim=-2)) + v = self.v_proj(torch.cat([context, latents], dim=-2)) + + # Multiheaded Self-Attention w/ stable softmax (subtract per-row max -- `amax` -- before softmax call) + # =>> `attn` should be a 2D matrix of shape [n_latents x (context + n_latents)] + # einsum.rearrange(x, "bsz seq (heads embed) -> bsz heads seq embed", heads=self.n_heads) + q, k, v = [x.reshape(batch_size, x.shape[1], self.n_heads, self.head_dim).transpose(1, 2) for x in (q, k, v)] + + if self.qk_layer_norms: + q = self.q_layer_norm(q) + k = self.k_layer_norm(k) + + scores = torch.einsum("... i d, ... j d -> ... i j", q * self.qk_scale, k) + stabilized_scores = scores - (scores.amax(dim=-1, keepdim=True).detach()) + attn = stabilized_scores.softmax(dim=-1) + + # Attend & project back to output... + resampled = torch.einsum("... i j, ... j d -> ... i d", attn, v) + # einsum.rearrange(resampled, "bsz heads seq embed -> bsz seq (heads embed)", heads=self.n_heads) + return self.output_proj(resampled.transpose(1, 2).flatten(-2)) + + +class IdeficsMLP(nn.Module): + def __init__(self, + prefix, + intermediate_size, + config, + weights, + ): + """Simple MLP block with intermediate_size and embedding size""" + super().__init__() + self.embed_dim = config.vision_config.embed_dim + self.ln = nn.LayerNorm.load(prefix=f"{prefix}.ln", weights=weights, eps=EPS) + self.fc = TensorParallelColumnLinear.load( + config=config, prefix=f"{prefix}.fc", weights=weights, bias=False, + ) + self.act = nn.ReLU() + self.c_proj = TensorParallelRowLinear.load( + config=config, prefix=f"{prefix}.c_proj", weights=weights, bias=False, + ) + + def forward(self, hidden_states: Optional[Tuple[torch.FloatTensor]]) -> torch.FloatTensor: + hidden_states = self.ln(hidden_states) + hidden_states = self.fc(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.c_proj(hidden_states) + + return hidden_states diff --git a/server/text_generation_server/models/custom_modeling/idefics_processing.py b/server/text_generation_server/models/custom_modeling/idefics_processing.py new file mode 100644 index 00000000..e24fc7bd --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/idefics_processing.py @@ -0,0 +1,413 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Processor class for IDEFICS. +""" + +from typing import Callable, List, Optional, Union +from urllib.parse import urlparse + +from transformers.feature_extraction_utils import BatchFeature +from transformers.processing_utils import ProcessorMixin +from transformers.tokenization_utils_base import BatchEncoding, PaddingStrategy, TextInput, TruncationStrategy +from transformers.utils import TensorType, is_torch_available +from text_generation_server.models.custom_modeling.idefics_image_processing import IdeficsImageProcessor + + +if is_torch_available(): + import torch + + +IMAGE_TOKEN = "" + + +# copied from m4.training.packing +def incremental_to_binary_attention_mask(incremental_mask, num_classes=-1): + # This function converts: [-1, 0, 1] => [[0, 0], [1, 0], [0, 1]] + + # If any of images index are more than num_classes, set them to -1. + # Words after the max number of images allowed have been seen don't attend on anything + if num_classes != -1: + incremental_mask[incremental_mask >= num_classes] = -1 + + negatives = incremental_mask == -1 + incremental_mask[negatives] = 0 + attn_mask = torch.nn.functional.one_hot(incremental_mask, num_classes=num_classes) + attn_mask[negatives, :] = 0 + return attn_mask + + +# copied from m4.training.packing +def image_attention_mask_for_packed_input_ids(input_ids, tokenizer): + image_attention_mask = torch.full_like(input_ids, fill_value=-1) + next_image_attention_mask = torch.full_like(input_ids, fill_value=-1) + image_token_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN) + eod_token_id = tokenizer.eos_token_id + for batch_idx in range(input_ids.size(0)): + count = -1 + seen_eod = False + for idx, token_id in enumerate(input_ids[batch_idx]): + if token_id == image_token_id: + count += 1 + image_attention_mask[batch_idx][idx] = count + seen_eod = False + else: + image_attention_mask[batch_idx][idx] = count + + if seen_eod: + image_attention_mask[batch_idx][idx] = -1 + + if token_id == eod_token_id: + seen_eod = True + + for batch_idx in range(input_ids.size(0)): + count = -1 + seen_eod = False + for idx in range(input_ids[batch_idx].size(0) - 1, -1, -1): + token_id = input_ids[batch_idx][idx] + if token_id == image_token_id: + count += 1 + next_image_attention_mask[batch_idx][idx] = count + seen_eod = False + else: + next_image_attention_mask[batch_idx][idx] = count + + if token_id == eod_token_id: + seen_eod = True + + if seen_eod: + next_image_attention_mask[batch_idx][idx] = -1 + + non_negative_indices = next_image_attention_mask[batch_idx] != -1 + next_image_attention_mask[batch_idx][non_negative_indices] -= count + next_image_attention_mask[batch_idx][non_negative_indices] *= -1 + + return image_attention_mask, next_image_attention_mask + + +def is_url(string): + """Checks if the passed string contains a valid url and nothing else. e.g. if space is included it's immediately + invalidated the url""" + if " " in string: + return False + result = urlparse(string) + return all([result.scheme, result.netloc]) + + +class IdeficsProcessor(ProcessorMixin): + r""" + Constructs a IDEFICS processor which wraps a LLama tokenizer and IDEFICS image processor into a single processor. + + [`IdeficsProcessor`] offers all the functionalities of [`IdeficsImageProcessor`] and [`LlamaTokenizerFast`]. See + the docstring of [`~IdeficsProcessor.__call__`] and [`~IdeficsProcessor.decode`] for more information. + + Args: + image_processor (`IdeficsImageProcessor`): + An instance of [`IdeficsImageProcessor`]. The image processor is a required input. + tokenizer (`LlamaTokenizerFast`): + An instance of [`LlamaTokenizerFast`]. The tokenizer is a required input. + image_size (`int`, *optional*, defaults to 224): Image size (assuming a square image) + """ + attributes = ["image_processor", "tokenizer"] + image_processor_class = "IdeficsImageProcessor" + tokenizer_class = "LlamaTokenizerFast" + + def __init__(self, image_processor, tokenizer=None, image_size=224, add_end_of_utterance_token=None, **kwargs): + if image_processor is None: + raise ValueError("You need to specify an `image_processor`.") + if tokenizer is None: + raise ValueError("You need to specify a `tokenizer`.") + + super().__init__(image_processor, tokenizer) + self.current_processor = self.image_processor + self.image_token_id = tokenizer.convert_tokens_to_ids(IMAGE_TOKEN) + + self.default_image_dims = ( + self.image_processor.image_num_channels, + self.image_processor.image_size, + self.image_processor.image_size, + ) + + self.tokenizer_was_trained_with_end_of_utterance_token = ( + True + if "" in self.tokenizer.special_tokens_map.get("additional_special_tokens", []) + else False + ) + + def __call__( + self, + prompts: Union[List[TextInput], List[List[TextInput]]], + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TruncationStrategy] = None, + max_length: Optional[int] = None, + transform: Callable = None, + add_eos_token=False, + add_end_of_utterance_token=None, + debug=False, + return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH, + ) -> BatchEncoding: + """This method takes batched or non-batched prompts made of text and images and converts them into prompts that + the model was trained on and prepares the image pixel values for the model to process. + + Args: + prompts (`Union[List[TextInput], [List[List[TextInput]]]]`): + either a single prompt or a batched list of prompts - see the detailed description immediately after + the end of the arguments doc section. + padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding + index) among: + - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum + acceptable input length for the model if that argument is not provided. + - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different + lengths). + max_length (`int`, *optional*): + Maximum length of the returned list and optionally padding length (see above). + truncation (`bool`, *optional*): + Activates truncation to cut input sequences longer than `max_length` to `max_length`. + transform (`Callable`, *optional*): + A custom transform function that accepts a single image can be passed for training. For example, + `torchvision.Compose` can be used to compose multiple functions. If `None` a preset inference-specific + set of transforms will be applied to the images + add_eos_token (`bool`, *optional*, defaults to `False`): + Adds `eos_token` at the end of the final prompt if True` + add_end_of_utterance_token (`bool`, *optional*) + Whether to automatically add `` after each prompt's text input (unless followed by an + image). If `None` the tokenizer will be checked instead and if this token is found in + `additional_special_tokens` then the value will be `True`. + debug (`bool`, *optional*, defaults to `False`): + `True` value will help debug prompt generation by dumping useful information + return_tensors (`str` or `TensorType`, *optional*, defaults to `TensorType.PYTORCH`): + The type of tensors to return. Can be one of: + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + + Returns: + a dict with entries: `input_ids`, `attention_mask`, `pixel_values`, `image_attention_mask` which can be + directly passed to `model.generate` + + Detailed explanation: + + Each entry in `prompts` is either a text to be passed as is or an image that will be processed. + + An image can be either an image object (`PIL.Image`) or a url from which the image can be retrieved. + + When the processor encounters an image it'll inject `` + entry into the prompt. + + Example: + + ```python + checkpoint = "HuggingFaceM4/idefics-9b" + processor = AutoProcessor.from_pretrained(checkpoint) + url = "https://hips.hearstapps.com/hmg-prod/images/cute-photos-of-cats-in-grass-1593184777.jpg" + img = processor.image_processor.fetch_images([url])[0] + + prompts = [ + "User:", + img, + "Describe this image.\nAssistant: An image of two kittens in grass.\n", + "User:", + "https://hips.hearstapps.com/hmg-prod/images/dog-puns-1581708208.jpg", + "Describe this image.\nAssistant:", + ] + + inputs = processor(prompts, return_tensors="pt") + generated_ids = model.generate(**inputs, max_length=100) + generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + ``` + + In this example the `prompts` will be converted into: + + ``` + User:Describe this image. + Assistant: An image of two kittens in grass. + User:Describe this image. + Assistant:' + ``` + + and the two images will be massaged using [`IdeficsImageProcessor.__call__`] method and placed inside the + `pixel_values` dict entry of the return value. + + This example also examplifies that images can be passed as objects or as text urls. It can be seen that the + first image is passed as object and the second one as a url. + + To do training do: + + ```python + image_transform = transforms.Compose( + [ + transforms.RandomResizedCrop( + (w, h), scale=(0.9, 1.0), interpolation=transforms.InterpolationMode.BICUBIC + ), + transforms.ToTensor(), + transforms.Normalize(mean=self.image_mean, std=self.image_std), + ] + ) + inputs = processor(prompts, transform=image_transform, return_tensors="pt") + ``` + + In order to help debug prompt generation enable `debug=True` which will show you what's happening. + + """ + + # if the value isn't overriden by the user, check if the tokenizer was trained with this token and then use it + if add_end_of_utterance_token is None: + add_end_of_utterance_token = self.tokenizer_was_trained_with_end_of_utterance_token + + # turn non-batched prompts into batched + if not any(isinstance(i, list) for i in prompts): + prompts = [prompts] + + fake_token = "" + image_token = "" + end_of_utterance_token = "" + + def image_tokens(last_was_image): + if last_was_image: + return image_token + fake_token + else: + return fake_token + image_token + fake_token + + all_texts = [] + all_images = [] + for sample in prompts: + # the model was trained on samples starting with + full_text = f"{self.tokenizer.bos_token}" + + # an image can either be an image object in the item or the url, everything else is a verbatim prompt text + image_objects = [] + last_was_image = False + last_was_text = False + for i, item in enumerate(sample): + if i > 0: + last_was_text = True if not last_was_image else False + + if isinstance(item, str): + item = item.strip(" ") + if is_url(item): + image = self.image_processor.fetch_images(item) + full_text += image_tokens(last_was_image) + image_objects.append(image) + last_was_image = True + else: + # we add end_of_utterance_token between each subsequent text prompts (but not at the last one!) + if add_end_of_utterance_token and last_was_text: + full_text += end_of_utterance_token + full_text += item + last_was_image = False + else: + # must be an image obj + full_text += image_tokens(last_was_image) + image_objects.append(item) + last_was_image = True + + if add_eos_token: + full_text += self.tokenizer.eos_token + + if debug is True: + print(f"{full_text=}") + + image_objects = self.image_processor(image_objects, transform=transform) + + text_encoding = self.tokenizer( + text=full_text, + add_special_tokens=False, + padding=padding, + truncation=truncation, + max_length=max_length, + ) + + all_texts.append(text_encoding["input_ids"]) + all_images.append(image_objects) + + max_seq_len = max(len(x) for x in all_texts) + + # max_num_images has to be at least 1 even when there are no images + max_num_images = max(len(x) for x in all_images) + max_num_images = max(1, max_num_images) + + at_least_one_image = sum(len(x) for x in all_images) > 0 + output_input_ids = [] + output_images = [] + output_attention_masks = [] + for text, images in zip(all_texts, all_images): + padded_input_ids = [self.tokenizer.pad_token_id] * max_seq_len + unpadded_seq_len = len(text) + start = max_seq_len - unpadded_seq_len + padded_input_ids[start:] = text[:max_seq_len] + + attention_mask = torch.zeros((max_seq_len,), dtype=torch.long) + attention_mask[start:] = 1 + + image_count = padded_input_ids.count(self.image_token_id) + local_max_num_images = min(image_count, max_num_images) + + current_images = images[:local_max_num_images] + + if len(current_images) > 0: + padded_image_tensor = torch.zeros(max_num_images, *current_images.size()[1:]) + padded_image_tensor[: current_images.size(0)] = current_images + else: + padded_image_tensor = torch.zeros(max_num_images, *self.default_image_dims) + + output_images.append(padded_image_tensor) + output_input_ids.append(torch.tensor(padded_input_ids)) + + output_attention_masks.append(attention_mask) + + output_input_ids = torch.stack(output_input_ids) + output_images = torch.stack(output_images) + output_attention_masks = torch.stack(output_attention_masks) + + if at_least_one_image: + image_attention_mask, _ = image_attention_mask_for_packed_input_ids(output_input_ids, self.tokenizer) + image_attention_mask = incremental_to_binary_attention_mask( + image_attention_mask, num_classes=max_num_images + ) + else: + # in full language mode we set the image mask to all-0s + image_attention_mask = torch.zeros( + output_input_ids.shape[0], output_input_ids.shape[1], 1, dtype=torch.bool + ) + + return BatchFeature( + data={ + "input_ids": output_input_ids, + "attention_mask": output_attention_masks, + "pixel_values": output_images, + "image_attention_mask": image_attention_mask, + } + ) + + def batch_decode(self, *args, **kwargs): + """ + This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please + refer to the docstring of this method for more information. + """ + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + """ + This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to + the docstring of this method for more information. + """ + return self.tokenizer.decode(*args, **kwargs) + + @property + def model_input_names(self): + tokenizer_input_names = self.tokenizer.model_input_names + image_processor_input_names = self.image_processor.model_input_names + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) diff --git a/server/text_generation_server/models/custom_modeling/idefics_vision.py b/server/text_generation_server/models/custom_modeling/idefics_vision.py new file mode 100644 index 00000000..d933d7c1 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/idefics_vision.py @@ -0,0 +1,476 @@ +# coding=utf-8 +# Copyright 2021 The OpenAI Team Authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch IdeficsVision model: a copy of CLIPVisionModel using a simpler config object""" + + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn + +from transformers.activations import ACT2FN +from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling +from transformers.utils import ( + ModelOutput, + logging, +) +from text_generation_server.utils.layers import ( + TensorParallelColumnLinear, + TensorParallelRowLinear, + TensorParallelEmbedding, +) + +logger = logging.get_logger(__name__) + + +@dataclass +class IdeficsVisionModelOutput(ModelOutput): + """ + Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. + + Args: + image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): + The image embeddings obtained by applying the projection layer to the pooler_output. + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + image_embeds: Optional[torch.FloatTensor] = None + last_hidden_state: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +# Copied from transformers.models.clip.modeling_clip.CLIPVisionEmbeddings with CLIP->Idefics +class IdeficsVisionEmbeddings(nn.Module): + def __init__(self, prefix, config, weights): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.class_embedding = nn.Parameter(weights.get_tensor(f"{prefix}.class_embedding")) + + self.patch_embedding = nn.Conv2d.load_no_bias( + prefix=f"{prefix}.patch_embedding", + weights=weights, + in_channels=config.num_channels, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size, + ) + + self.num_patches = (self.image_size // self.patch_size) ** 2 + self.num_positions = self.num_patches + 1 + # self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) + self.position_embedding = TensorParallelEmbedding( + prefix="model.vision_model.embeddings.position_embedding", weights=weights + ) + # self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) + self.position_ids = weights.get_tensor(f"{prefix}.position_ids") + + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: + batch_size = pixel_values.shape[0] + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype)) # shape = [*, width, grid, grid] + patch_embeds = patch_embeds.flatten(2).transpose(1, 2) + + class_embeds = self.class_embedding.expand(batch_size, 1, -1) + embeddings = torch.cat([class_embeds, patch_embeds], dim=1) + embeddings = embeddings + self.position_embedding(self.position_ids) + return embeddings + + +# Copied from transformers.models.clip.modeling_clip.CLIPAttention with CLIP->IdeficsVision +class IdeficsVisionAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, prefix, config, weights): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads})." + ) + self.scale = self.head_dim**-0.5 + self.dropout = config.attention_dropout + + process_group = weights.process_group + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) + self.num_heads = self.num_heads // weights.process_group.size() + self.embed_dim = self.embed_dim // weights.process_group.size() + + + self.k_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.k_proj", weights=weights, bias=True + ) + self.v_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.v_proj", weights=weights, bias=True + ) + self.q_proj = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.q_proj", weights=weights, bias=True + ) + self.out_proj = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.out_proj", weights=weights, bias=True + ) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + causal_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scale + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + # apply the causal_attention_mask first + if causal_attention_mask is not None: + if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" + f" {causal_attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if output_attentions: + # this operation is a bit akward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped + + +# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->IdeficsVision +class IdeficsVisionMLP(nn.Module): + def __init__(self, prefix, config, weights): + super().__init__() + self.config = config + self.activation_fn = ACT2FN[config.hidden_act] + self.fc1 = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.fc1", weights=weights, bias=True + ) + self.fc2 = TensorParallelRowLinear.load( + config, prefix=f"{prefix}.fc2", weights=weights, bias=True + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->IdeficsVision +class IdeficsVisionEncoderLayer(nn.Module): + def __init__(self, prefix, config, weights): + super().__init__() + self.embed_dim = config.hidden_size + self.self_attn = IdeficsVisionAttention(prefix=f"{prefix}.self_attn", config=config, weights=weights) + self.layer_norm1 = nn.LayerNorm.load( + prefix=f"{prefix}.layer_norm1", weights=weights, eps=config.layer_norm_eps + ) + self.mlp = IdeficsVisionMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) + self.layer_norm2 = nn.LayerNorm.load( + prefix=f"{prefix}.layer_norm2", weights=weights, eps=config.layer_norm_eps + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + causal_attention_mask: torch.Tensor, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + `(config.encoder_attention_heads,)`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + + hidden_states = self.layer_norm1(hidden_states) + hidden_states, attn_weights = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + causal_attention_mask=causal_attention_mask, + output_attentions=output_attentions, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (attn_weights,) + + return outputs + + +# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->IdeficsVision +class IdeficsVisionEncoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a + [`IdeficsVisionEncoderLayer`]. + + Args: + config: IdeficsVisionConfig + """ + + def __init__(self, prefix, config, weights): + super().__init__() + self.config = config + self.layers = nn.ModuleList( + [ + IdeficsVisionEncoderLayer(prefix=f"{prefix}.encoder.layers.{layer_id}", config=config, weights=weights) + for layer_id in range(config.num_hidden_layers) + ] + ) + # self.gradient_checkpointing = False + + def forward( + self, + inputs_embeds, + attention_mask: Optional[torch.Tensor] = None, + causal_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Causal mask for the text model. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + hidden_states = inputs_embeds + for idx, encoder_layer in enumerate(self.layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + # if self.gradient_checkpointing and self.training: + + # def create_custom_forward(module): + # def custom_forward(*inputs): + # return module(*inputs, output_attentions) + + # return custom_forward + + # layer_outputs = torch.utils.checkpoint.checkpoint( + # create_custom_forward(encoder_layer), + # hidden_states, + # attention_mask, + # causal_attention_mask, + # ) + # else: + layer_outputs = encoder_layer( + hidden_states, + attention_mask, + causal_attention_mask, + output_attentions=output_attentions, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions + ) + + +# Adapted from transformers.models.clip.modeling_clip.CLIPVisionTransformer +class IdeficsVisionTransformer(nn.Module): + def __init__(self, prefix, config, weights): + super().__init__() + self.config = config + embed_dim = config.hidden_size + + self.embeddings = IdeficsVisionEmbeddings(prefix=f"{prefix}.embeddings", config=config, weights=weights) + self.pre_layrnorm = nn.LayerNorm.load( + prefix=f"{prefix}.pre_layrnorm", weights=weights, eps=config.layer_norm_eps + ) + self.encoder = IdeficsVisionEncoder(prefix=prefix, config=config, weights=weights) + self.post_layernorm = nn.LayerNorm.load( + prefix=f"{prefix}.post_layernorm", weights=weights, eps=config.layer_norm_eps + ) + + # copied from transformers.models.clip.modeling_clip.CLIPVisionTransformer.forward + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if pixel_values is None: + raise ValueError("You have to specify pixel_values") + + hidden_states = self.embeddings(pixel_values) + hidden_states = self.pre_layrnorm(hidden_states) + + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + pooled_output = last_hidden_state[:, 0, :] + pooled_output = self.post_layernorm(pooled_output) + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) diff --git a/server/text_generation_server/models/idefics.py b/server/text_generation_server/models/idefics.py new file mode 100644 index 00000000..c54b539b --- /dev/null +++ b/server/text_generation_server/models/idefics.py @@ -0,0 +1,91 @@ +import torch +import torch.distributed + +from typing import List, Optional, Tuple + +from transformers import ( + AutoTokenizer, + AutoConfig, + AutoProcessor, +) + +from text_generation_server.models.custom_modeling.idefics_config import IdeficsConfig +from text_generation_server.models.custom_modeling.idefics_processing import ( + IdeficsProcessor, +) +from transformers import LlamaTokenizerFast +from text_generation_server.models.custom_modeling.idefics_modeling import ( + IdeficsForVisionText2Text, +) +from text_generation_server.models.idefics_causal_lm import IdeficsCausalLM +from text_generation_server.utils import ( + initialize_torch_distributed, + weight_files, + Weights, +) + + +class IDEFICSSharded(IdeficsCausalLM): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, + ): + self.process_group, rank, world_size = initialize_torch_distributed() + if torch.cuda.is_available(): + device = torch.device(f"cuda:{rank}") + # 9b seems to work correctly enough in float16, but 80b seems + # to be really saturating for f16. + dtype = torch.bfloat16 if dtype is None else dtype + else: + device = torch.device("cpu") + dtype = torch.float32 + self.device, self.dtype = device, dtype + + config = IdeficsConfig.from_pretrained( + model_id, + revision=revision, + trust_remote_code=trust_remote_code, + ) + config.quantize = quantize + config.vision_config.quantize = quantize + + tokenizer = LlamaTokenizerFast.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + self.processor = IdeficsProcessor.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights( + filenames, + device=device, + dtype=dtype, + process_group=self.process_group, + ) + + model = IdeficsForVisionText2Text(config, weights) + + torch.distributed.barrier(group=self.process_group) + super(IdeficsCausalLM, self).__init__( + model=model, + tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, + device=device, + rank=rank, + world_size=world_size, + ) diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py new file mode 100644 index 00000000..faad63ba --- /dev/null +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -0,0 +1,806 @@ +import torch +import inspect +import re +from io import BytesIO +import base64 +from PIL import Image +import re + +from dataclasses import dataclass +from opentelemetry import trace +from transformers import AutoProcessor, AutoTokenizer, AutoModelForCausalLM, PreTrainedTokenizerBase, ProcessorMixin +from typing import Optional, Tuple, List, Type, Dict + +from text_generation_server.models import Model +from text_generation_server.models.types import ( + Batch, + PrefillTokens, + Generation, + GeneratedText, +) +from text_generation_server.pb import generate_pb2 +from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling + +import re + +IMAGES = re.compile(r'!\[[^\]]*\]\((.*?)\s*(\"(?:.*[^\"])\")?\s*\)') + +def split(string): + parts = [] + cursor = 0 + for pattern in IMAGES.finditer(string): + start = pattern.start() + if start != cursor: + parts.append(string[cursor:start]) + + parts.append(pattern.group(1)) + cursor = pattern.end() + + if cursor != len(string): + parts.append(string[cursor:]) + + return parts + +tracer = trace.get_tracer(__name__) + + +@dataclass +class IdeficsCausalLMBatch(Batch): + batch_id: int + requests: List[generate_pb2.Request] + requests_idx_mapping: Dict[int, int] + + # Decoder values + input_ids: torch.Tensor + attention_mask: torch.Tensor + position_ids: torch.Tensor + pixel_values: Optional[torch.Tensor] + image_hidden_states: Optional[torch.Tensor] + image_attention_mask: Optional[torch.Tensor] + past_key_values: Optional[List[Tuple]] + + # All tokens + all_input_ids: List[torch.Tensor] + + # Lengths of all generations present in the batch + input_lengths: List[int] + prefix_offsets: List[int] + read_offsets: List[int] + + # Generation helpers + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + + # Metadata used for padding + max_input_length: int + padding_right_offset: int + + # Maximum number of tokens this batch will grow to + max_tokens: int + + # Past metadata + keys_head_dim_last: bool = True + + def to_pb(self) -> generate_pb2.CachedBatch: + return generate_pb2.CachedBatch( + id=self.batch_id, + request_ids=[r.id for r in self.requests], + size=len(self), + max_tokens=self.max_tokens, + ) + + @classmethod + def from_pb( + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + processor: ProcessorMixin, # Hack + dtype: torch.dtype, + device: torch.device, + ) -> "IdeficsCausalLMBatch": + inputs = [] + next_token_choosers = [] + stopping_criterias = [] + prefix_offsets = [] + read_offsets = [] + requests_idx_mapping = {} + + # Parse batch + max_truncation = 0 + padding_right_offset = 0 + max_decode_tokens = 0 + for i, r in enumerate(pb.requests): + requests_idx_mapping[r.id] = i + inputs.append(r.inputs) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + stopping_criteria = StoppingCriteria.from_pb( + r.stopping_parameters, tokenizer + ) + stopping_criterias.append(stopping_criteria) + max_truncation = max(max_truncation, r.truncate) + max_decode_tokens += stopping_criteria.max_new_tokens + padding_right_offset = max( + padding_right_offset, stopping_criteria.max_new_tokens + ) + + prompts = [] + for inp in inputs: + # Each input is encoded into a list, where each element of this input list is either a string or a URL + prompts.append(split(inp)) + + # The processor replaces the call to tokenizer, and + # a/ takes care of fetching images from the URL + # b/ generate the correct input_ids, attention_mask, pixel_values, image_attention_mask to feed to the model + tokenized_inputs = processor( + prompts, + return_tensors="pt", + padding=True, + truncation=True, + max_length=max_truncation, + add_end_of_utterance_token=False, # Already taken care of inside the prompts, so bypassing the processor's handling of this token + ).to(device) + for _ in pb.requests: + input_len = tokenized_inputs["input_ids"].shape[1] + prefix_offsets.append(input_len - 5) # To decode without potential fallbacks errors + read_offsets.append(input_len) # To decode without potential fallbacks errors + + input_lengths = tokenized_inputs["attention_mask"].sum(1) + max_input_length = input_lengths.max() + + input_ids = tokenized_inputs["input_ids"] + pixel_values = tokenized_inputs["pixel_values"] + image_hidden_states = None + # Allocate maximum attention_mask + attention_mask = input_ids.new_zeros( + (pb.size, max_input_length + padding_right_offset) + ) + # Copy tokenizer attention_mask into fully allocated attention_mask + attention_mask[:, :max_input_length] = tokenized_inputs["attention_mask"] + # Do the same for image_attention_mask + image_attention_mask = input_ids.new_zeros( + (pb.size, max_input_length + padding_right_offset, tokenized_inputs["pixel_values"].size(1)) + ) + image_attention_mask[:, :max_input_length, :] = tokenized_inputs["image_attention_mask"] + + + position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 + position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) + all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) # It's input_ids but splitted into a tuple of tensors where each tensor is (seq_len, 1) size. It is then transformed into a list + + max_tokens = len(inputs) * (max_input_length + max_decode_tokens) + + return cls( + batch_id=pb.id, + requests=pb.requests, + requests_idx_mapping=requests_idx_mapping, + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + pixel_values=pixel_values, + image_hidden_states=image_hidden_states, + image_attention_mask=image_attention_mask, + past_key_values=None, + all_input_ids=list(all_input_ids), + input_lengths=input_lengths.tolist(), + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + max_input_length=max_input_length.item(), + padding_right_offset=padding_right_offset, + max_tokens=max_tokens, + ) + + @tracer.start_as_current_span("filter") + def filter(self, request_ids: List[int]) -> Optional["IdeficsCausalLMBatch"]: + # It deletes requests from the batch. For instance when client lost connection + if len(request_ids) == 0: + raise ValueError("Batch must have at least one request") + if len(request_ids) == len(self): + return self + + keep_indices = [] + + # New values after filtering + requests_idx_mapping = {} + requests = [] + input_lengths = [] + prefix_offsets = [] + read_offsets = [] + all_input_ids = [] + max_input_length = 0 + + next_token_choosers = [] + stopping_criterias = [] + + total_remaining_decode_tokens = 0 + new_padding_right_offset = 0 + + for i, request_id in enumerate(request_ids): + idx = self.requests_idx_mapping[request_id] + requests_idx_mapping[request_id] = i + keep_indices.append(idx) + + requests.append(self.requests[idx]) + prefix_offsets.append(self.prefix_offsets[idx]) + read_offsets.append(self.read_offsets[idx]) + all_input_ids.append(self.all_input_ids[idx]) + + request_input_length = self.input_lengths[idx] + input_lengths.append(request_input_length) + max_input_length = max(max_input_length, request_input_length) + + next_token_choosers.append(self.next_token_choosers[idx]) + stopping_criteria = self.stopping_criterias[idx] + stopping_criterias.append(stopping_criteria) + remaining_decode_tokens = ( + stopping_criteria.max_new_tokens - stopping_criteria.current_tokens + ) + total_remaining_decode_tokens += remaining_decode_tokens + new_padding_right_offset = max( + new_padding_right_offset, remaining_decode_tokens + ) + + # Apply indices to input_ids, attention mask, past key values and other items that need to be cached + input_ids = self.input_ids[keep_indices] + position_ids = self.position_ids[keep_indices] + self.attention_mask = self.attention_mask[ + keep_indices, + -(self.padding_right_offset + max_input_length) : ( + self.attention_mask.shape[1] - self.padding_right_offset + ) + + new_padding_right_offset, + ] + # Do the same for pixel_values and image_attention_mask + pixel_values = self.pixel_values[keep_indices] + self.image_attention_mask = self.image_attention_mask[ + keep_indices, + -(self.padding_right_offset + max_input_length) : ( + self.image_attention_mask.shape[1] - self.padding_right_offset + ) + + new_padding_right_offset, + : + ] + if self.image_hidden_states is None: + image_hidden_states = None + else: + image_hidden_states = self.image_hidden_states[keep_indices] + + # Ensure that past_key_values tensors can be updated in-place + if type(self.past_key_values[0]) == tuple: + self.past_key_values = [list(layer) for layer in self.past_key_values] + + # Update tensors in-place to allow incremental garbage collection + past_kv_length = max_input_length - 1 + for layer in self.past_key_values: + past_keys, past_values = layer + if len(past_keys.shape) == 3: + # Force past to be of dim [self_size, num_heads, ...] for easy indexing + past_keys = past_keys.view(len(self), -1, *past_keys.shape[-2:]) + past_values = past_values.view(len(self), -1, *past_values.shape[-2:]) + if self.keys_head_dim_last: + layer[0] = past_keys[keep_indices, :, -past_kv_length:, :] + else: + layer[0] = past_keys[keep_indices, :, :, -past_kv_length:] + del past_keys + layer[1] = past_values[keep_indices, :, -past_kv_length:, :] + del past_values + + max_tokens = len(request_ids) * max_input_length + total_remaining_decode_tokens + + self.requests = requests + self.requests_idx_mapping = requests_idx_mapping + self.input_ids = input_ids + self.pixel_values = pixel_values + self.image_hidden_states = image_hidden_states + self.position_ids = position_ids + self.all_input_ids = all_input_ids + self.input_lengths = input_lengths + self.prefix_offsets = prefix_offsets + self.read_offsets = read_offsets + self.next_token_choosers = next_token_choosers + self.stopping_criterias = stopping_criterias + self.max_input_length = max_input_length + self.padding_right_offset = new_padding_right_offset + self.max_tokens = max_tokens + + return self + + @classmethod + @tracer.start_as_current_span("concatenate") + def concatenate(cls, batches: List["IdeficsCausalLMBatch"]) -> "IdeficsCausalLMBatch": + # It adds new requests to the batch + # Used for padding + total_batch_size = 0 + max_input_length = 0 + max_num_images = 0 + padding_right_offset = 0 + for batch in batches: + total_batch_size += len(batch) + max_input_length = max(max_input_length, batch.max_input_length) + max_num_images = max(max_num_images, batch.pixel_values.size(1)) + padding_right_offset = max(padding_right_offset, batch.padding_right_offset) + + # Batch attributes + requests = [] + requests_idx_mapping = {} + input_lengths = [] + prefix_offsets = [] + read_offsets = [] + all_input_ids = [] + next_token_choosers = [] + stopping_criterias = [] + max_tokens = 0 + + # Batch tensors + input_ids = None + attention_mask = None + position_ids = None + pixel_values = None + image_hidden_states = None + image_attention_mask = None + past_key_values = [] + + # Used for slicing correctly inside the tensors + # Equivalent to a cumsum on batch sizes + start_index = 0 + for i, batch in enumerate(batches): + requests.extend(batch.requests) + input_lengths.extend(batch.input_lengths) + prefix_offsets.extend(batch.prefix_offsets) + read_offsets.extend(batch.read_offsets) + all_input_ids.extend(batch.all_input_ids) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + + if i == 0: + requests_idx_mapping = batch.requests_idx_mapping + else: + # We need to offset the mapping for each batch by the cumulative batch size + for k, v in batch.requests_idx_mapping.items(): + requests_idx_mapping[k] = v + start_index + + # Slicing end index for this batch + end_index = start_index + len(batch) + + # We only concatenate batches that did at least one step + if batch.past_key_values is None: + raise ValueError("only concatenate prefilled batches") + + # Create empty tensor + # input_ids is always of shape [batch_size, 1] + # We do not need to pad it + if input_ids is None: + input_ids = batch.input_ids.new_empty((total_batch_size, 1)) + # Copy to correct indices + input_ids[start_index:end_index] = batch.input_ids + + # Create padded tensor + if attention_mask is None: + attention_mask = batch.attention_mask.new_zeros( + (total_batch_size, max_input_length + padding_right_offset), + ) + + curr_batch_max_num_images = batch.pixel_values.size(1) + if pixel_values is None: + pixel_values = batch.pixel_values.new_zeros((total_batch_size, max_num_images, 3, 224, 224)) + pixel_values[start_index:end_index, :curr_batch_max_num_images] = batch.pixel_values + + if image_attention_mask is None: + image_attention_mask = batch.image_attention_mask.new_zeros( + (total_batch_size, max_input_length + padding_right_offset, max_num_images) + ) + + # We need to slice the attention mask to remove padding from previous steps + # and to remove unused allocated space + left_offset = max_input_length - batch.max_input_length + batch_left_offset = ( + batch.attention_mask.shape[1] + - batch.max_input_length + - batch.padding_right_offset + ) + attention_mask[ + start_index:end_index, + left_offset:-padding_right_offset, + ] = batch.attention_mask[ + :, + batch_left_offset : -batch.padding_right_offset, + ] + image_attention_mask[ + start_index:end_index, + left_offset:-padding_right_offset, + :curr_batch_max_num_images + ] = batch.image_attention_mask[ + :, + batch_left_offset : - batch.padding_right_offset, + : + ] + + # Create empty tensor + # position_ids is always of shape [batch_size, 1] + if position_ids is None: + position_ids = batch.position_ids.new_empty((total_batch_size, 1)) + position_ids[start_index:end_index] = batch.position_ids + + # Shenanigans to get dimensions because BLOOM outputs a past with a different shape + # BLOOM Keys: [batch_size * num_heads, head_dim, seq_length] + # BLOOM Values: [batch_size * num_heads, seq_length, head_dim] + # And ensure that we can update tensors in-place + if type(batch.past_key_values[0]) == tuple: + batch.past_key_values = [ + [t.view(len(batch), -1, *t.shape[-2:]) for t in layer] + for layer in batch.past_key_values + ] + elif len(batch.past_key_values[0][0].shape) == 3: + for layer in batch.past_key_values: + for k, t in enumerate(layer): + layer[k] = t.view(len(batch), -1, *t.shape[-2:]) + + # Add eventual padding tokens that were added while concatenating + max_tokens += batch.max_tokens + ( + max_input_length - batch.max_input_length + ) * len(batch) + + start_index = end_index + + first_past_kvs = batches[0].past_key_values + _, num_heads, padded_sequence_length, head_dim = first_past_kvs[0][1].shape + + padded_past_values_shape = ( + total_batch_size, + num_heads, + max_input_length - 1, + head_dim, + ) + + if batches[0].keys_head_dim_last: + padded_past_keys_shape = padded_past_values_shape + else: + # seq_length is last for BLOOM + padded_past_keys_shape = ( + total_batch_size, + num_heads, + head_dim, + max_input_length - 1, + ) + + # Iterate over attention layers + # Concatenate past key values layer by layer to allow incremental garbage collection + for j in range(len(first_past_kvs)): + padded_past_keys = first_past_kvs[j][0].new_zeros(padded_past_keys_shape) + start_index = 0 + for batch in batches: + past_keys = batch.past_key_values[j][0] + # Clear reference to the original tensor + batch.past_key_values[j][0] = None + + # Slicing end index for this batch + end_index = start_index + len(batch) + # We slice the keys to remove the padding from previous batches + past_seq_len = batch.max_input_length - 1 + if batch.keys_head_dim_last: + padded_past_keys[ + start_index:end_index, :, -past_seq_len:, : + ] = past_keys[:, :, -past_seq_len:, :] + else: + # BLOOM case + padded_past_keys[ + start_index:end_index, :, :, -past_seq_len: + ] = past_keys[:, :, :, -past_seq_len:] + del past_keys + + start_index = end_index + + padded_past_values = first_past_kvs[j][1].new_zeros( + padded_past_values_shape + ) + start_index = 0 + for batch in batches: + past_values = batch.past_key_values[j][1] + # Clear reference to the original tensor + batch.past_key_values[j][1] = None + + # Slicing end index for this batch + end_index = start_index + len(batch) + # We slice the past values to remove the padding from previous batches + past_seq_len = batch.max_input_length - 1 + padded_past_values[ + start_index:end_index, :, -past_seq_len:, : + ] = past_values[:, :, -past_seq_len:, :] + del past_values + + # Update values + start_index = end_index + + past_key_values.append([padded_past_keys, padded_past_values]) + + return cls( + batch_id=batches[0].batch_id, + requests=requests, + requests_idx_mapping=requests_idx_mapping, + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + pixel_values=pixel_values, + image_hidden_states=image_hidden_states, + image_attention_mask=image_attention_mask, + past_key_values=past_key_values, + all_input_ids=all_input_ids, + input_lengths=input_lengths, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + max_input_length=max_input_length, + padding_right_offset=padding_right_offset, + keys_head_dim_last=batches[0].keys_head_dim_last, + max_tokens=max_tokens, + ) + + def __len__(self): + return len(self.requests) + + +class IdeficsCausalLM(Model): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, + ): + from text_generation_server.models.custom_modeling.idefics_modeling import IdeficsForVisionText2Text + + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.float16 if dtype is None else dtype + else: + if quantize: + raise ValueError("quantization is not available on CPU") + + device = torch.device("cpu") + dtype = torch.float32 + + tokenizer = AutoTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + self.processor = AutoProcessor.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + model = IdeficsForVisionText2Text.from_pretrained( + model_id, + revision=revision, + torch_dtype=dtype, + device_map="auto" + if torch.cuda.is_available() and torch.cuda.device_count() > 1 + else None, + load_in_8bit=quantize == "bitsandbytes", + trust_remote_code=trust_remote_code, + ) + if torch.cuda.is_available() and torch.cuda.device_count() == 1: + model = model.cuda() + + if tokenizer.pad_token_id is None: + if model.config.pad_token_id is not None: + tokenizer.pad_token_id = model.config.pad_token_id + elif model.config.eos_token_id is not None: + tokenizer.pad_token_id = model.config.eos_token_id + elif tokenizer.eos_token_id is not None: + tokenizer.pad_token_id = tokenizer.eos_token_id + else: + tokenizer.add_special_tokens({"pad_token": ""}) + + super(IdeficsCausalLM, self).__init__( + model=model, + tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, + device=device, + ) + + @property + def batch_type(self) -> Type[IdeficsCausalLMBatch]: + return IdeficsCausalLMBatch + + def decode(self, generated_ids: List[int]) -> str: + return self.tokenizer.decode( + generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False + ) + + def forward( + self, + input_ids, + attention_mask, + position_ids, + pixel_values, + image_hidden_states, + image_attention_mask, + past_key_values: Optional = None, + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + # Model Forward + kwargs = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "pixel_values": pixel_values, + "image_hidden_states": image_hidden_states, + "image_attention_mask": image_attention_mask, + "past_key_values": past_key_values, + "use_cache": True, + "return_dict": True, + } + if self.has_position_ids: + kwargs["position_ids"] = position_ids + + outputs = self.model.forward(**kwargs) + return outputs.logits, outputs.past_key_values, outputs.image_hidden_states + + @tracer.start_as_current_span("generate_token") + def generate_token( + self, batch: IdeficsCausalLMBatch + ) -> Tuple[List[Generation], Optional[IdeficsCausalLMBatch]]: + # slice the attention mask to the correct shape + attention_mask = batch.attention_mask[:, : -batch.padding_right_offset] + if batch.input_ids.size(1) == 1: + # THIS is a hack: when calling idefics.generate, the first time, we need the whole image_attention_mask (size bs x max_seq_len x max_num_images), + # but the subsequent times, we only need the last attention mask along the `max_seq_len` dimension + # this is due to the nature IDEFICS: it's an encoder decoder, and so when decoding, only the currently generated + # token need to attend to the encoder hidden states (i.e. the vision encoder) + # Also see seq2seq_lm.Seq2SeqLM.generate_token which has roughly the same logic + image_attention_mask = batch.image_attention_mask[:, -(batch.padding_right_offset+1)].unsqueeze(1) + else: + image_attention_mask = batch.image_attention_mask[:, : -batch.padding_right_offset] + + logits, past, image_hidden_states = self.forward( + input_ids=batch.input_ids, + attention_mask=attention_mask, + position_ids=batch.position_ids, + pixel_values=batch.pixel_values, + image_hidden_states=batch.image_hidden_states, + image_attention_mask=image_attention_mask, + past_key_values=batch.past_key_values, + ) + # Hardcoded remove image tokens + logits[:, 32000:32001] = torch.finfo(logits.dtype).min + + # Results + generations: List[Generation] = [] + stopped = True + + # Zipped iterator + iterator = zip( + batch.requests, + batch.input_lengths, + batch.prefix_offsets, + batch.read_offsets, + logits, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + ) + + # For each member of the batch + for i, ( + request, + input_length, + prefix_offset, + read_offset, + logits, + next_token_chooser, + stopping_criteria, + all_input_ids, + ) in enumerate(iterator): + # Select next token + next_token_id, logprobs = next_token_chooser( + all_input_ids.view(1, -1), logits[-1:, :] + ) + + # Append next token to all tokens + all_input_ids = torch.cat([all_input_ids, next_token_id]) + new_input_length = input_length + 1 + + # Generated token + next_token_logprob = logprobs[-1, next_token_id] + next_token_id_squeezed = next_token_id.squeeze() + next_token_text, prefix_offset, read_offset = self.decode_token( + all_input_ids[:, 0], prefix_offset, read_offset + ) + + # Evaluate stopping criteria + stop, reason = stopping_criteria( + next_token_id_squeezed, + next_token_text, + ) + + if not stop: + stopped = False + + # Shard generations + # All generations will be appended in the rust sharded client + if i % self.world_size == self.rank: + if stop: + # Decode generated tokens + output_text = self.decode( + all_input_ids[-stopping_criteria.current_tokens :, 0] + ) + # Get seed + if isinstance(next_token_chooser.choice, Sampling): + seed = next_token_chooser.choice.seed + else: + seed = None + + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed + ) + else: + generated_text = None + + # Prefill + if stopping_criteria.current_tokens == 1 and request.prefill_logprobs: + # Remove generated token to only have prefill and add nan for first prompt token + prefill_logprobs = [float("nan")] + torch.log_softmax( + logits, -1 + ).gather(1, all_input_ids[1:]).squeeze(1)[ + -new_input_length:-1 + ].tolist() + prefill_token_ids = all_input_ids[-new_input_length:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = PrefillTokens( + prefill_token_ids, prefill_logprobs, prefill_texts + ) + else: + prefill_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + next_token_id_squeezed, + next_token_logprob, + next_token_text, + next_token_id_squeezed.item() in self.all_special_ids, + generated_text, + ) + + generations.append(generation) + + # Update values + batch.input_ids[i, 0] = next_token_id + batch.all_input_ids[i] = all_input_ids + batch.input_lengths[i] = new_input_length + batch.prefix_offsets[i] = prefix_offset + batch.read_offsets[i] = read_offset + batch.max_input_length = max(batch.max_input_length, new_input_length) + + # We finished all generations in the batch; there is no next batch + if stopped: + return generations, None + + # Slice unused values from prefill + batch.input_ids = batch.input_ids[:, :1] + + # Update attention_mask as we added a new token to input_ids + batch.attention_mask[:, -batch.padding_right_offset] = 1 + batch.image_attention_mask[:, -batch.padding_right_offset, :] = batch.image_attention_mask[:, -(batch.padding_right_offset+1), :] + # Decrease right offset + batch.padding_right_offset -= 1 + + # Update position_ids + batch.position_ids = batch.position_ids[:, -1:] + 1 + + # Update past key values + batch.past_key_values = past + batch.image_hidden_states = image_hidden_states + + return generations, batch diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 1cedc151..67137aaa 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -14,7 +14,7 @@ from text_generation_server.models import Model, get_model from text_generation_server.pb import generate_pb2_grpc, generate_pb2 from text_generation_server.tracing import UDSOpenTelemetryAioServerInterceptor - +from text_generation_server.models.idefics_causal_lm import IdeficsCausalLMBatch class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): def __init__(self, model: Model, cache: Cache, server_urls: List[str]): @@ -26,6 +26,7 @@ def __init__(self, model: Model, cache: Cache, server_urls: List[str]): # Force inference mode for the lifetime of TextGenerationService self._inference_mode_raii_guard = torch._C._InferenceMode(True) + async def Info(self, request, context): return self.model.info @@ -54,9 +55,14 @@ async def FilterBatch(self, request, context): return generate_pb2.FilterBatchResponse(batch=filtered_batch.to_pb()) async def Warmup(self, request, context): - batch = self.model.batch_type.from_pb( - request.batch, self.model.tokenizer, self.model.dtype, self.model.device - ) + if self.model.batch_type == IdeficsCausalLMBatch: #Hack, i would rather use kwargs in the `from_pb` call + batch = self.model.batch_type.from_pb( + request.batch, self.model.tokenizer, self.model.processor, self.model.dtype, self.model.device + ) + else: + batch = self.model.batch_type.from_pb( + request.batch, self.model.tokenizer, self.model.dtype, self.model.device + ) max_supported_total_tokens = self.model.warmup(batch) return generate_pb2.WarmupResponse( @@ -64,9 +70,14 @@ async def Warmup(self, request, context): ) async def Prefill(self, request, context): - batch = self.model.batch_type.from_pb( - request.batch, self.model.tokenizer, self.model.dtype, self.model.device - ) + if self.model.batch_type == IdeficsCausalLMBatch: #Hack, i would rather use kwargs in the `from_pb` call + batch = self.model.batch_type.from_pb( + request.batch, self.model.tokenizer, self.model.processor, self.model.dtype, self.model.device + ) + else: + batch = self.model.batch_type.from_pb( + request.batch, self.model.tokenizer, self.model.dtype, self.model.device + ) generations, next_batch = self.model.generate_token(batch) self.cache.set(next_batch) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 9688f6e0..745c1d2e 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -51,7 +51,31 @@ def load_layer_norm_no_bias(cls, prefix, weights, eps): ln.bias = None return ln +@classmethod +def load_conv2d(cls, prefix, weights, in_channels, out_channels, kernel_size, stride): + weight = weights.get_tensor(f"{prefix}.weight") + bias = weights.get_tensor(f"{prefix}.bias") + with init_empty_weights(): + conv2d = cls(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride) + + conv2d.weight = nn.Parameter(weight) + conv2d.bias = nn.Parameter(bias) + return conv2d + + +@classmethod +def load_conv2d_no_bias(cls, prefix, weights, in_channels, out_channels, kernel_size, stride): + weight = weights.get_tensor(f"{prefix}.weight") + with init_empty_weights(): + conv2d = cls(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride) + + conv2d.weight = nn.Parameter(weight) + conv2d.bias = None + return conv2d + +torch.nn.Conv2d.load = load_conv2d +torch.nn.Conv2d.load_no_bias = load_conv2d_no_bias torch.nn.LayerNorm.load = load_layer_norm torch.nn.LayerNorm.load_no_bias = load_layer_norm_no_bias From bfa070611d956d5dbd15da9611bb0f8809c4dee0 Mon Sep 17 00:00:00 2001 From: Omar Sanseviero Date: Fri, 18 Aug 2023 13:27:08 +0200 Subject: [PATCH 391/793] Add streaming guide (#858) Co-authored-by: Lucain Co-authored-by: Merve Noyan --- docs/source/_toctree.yml | 4 + docs/source/conceptual/streaming.md | 148 ++++++++++++++++++++++++++++ docs/source/quicktour.md | 2 +- 3 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 docs/source/conceptual/streaming.md diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index a161dc28..5ba470bd 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -18,3 +18,7 @@ - local: basic_tutorials/using_cli title: Using TGI CLI title: Tutorials +- sections: + - local: conceptual/streaming + title: Streaming + title: Conceptual Guides diff --git a/docs/source/conceptual/streaming.md b/docs/source/conceptual/streaming.md new file mode 100644 index 00000000..c20d76e0 --- /dev/null +++ b/docs/source/conceptual/streaming.md @@ -0,0 +1,148 @@ +# Streaming + +## What is Streaming? + +Token streaming is the mode in which the server returns the tokens one by one as the model generates them. This enables showing progressive generations to the user rather than waiting for the whole generation. Streaming is an essential aspect of the end-user experience as it reduces latency, one of the most critical aspects of a smooth experience. + +
+ + +
+ +With token streaming, the server can start returning the tokens one by one before having to generate the whole response. Users can have a sense of the generation's quality earlier than the end of the generation. This has different positive effects: + +* Users can get results orders of magnitude earlier for extremely long queries. +* Seeing something in progress allows users to stop the generation if it's not going in the direction they expect. +* Perceived latency is lower when results are shown in the early stages. +* When used in conversational UIs, the experience feels more natural. + +For example, a system can generate 100 tokens per second. If the system generates 1000 tokens, with the non-streaming setup, users need to wait 10 seconds to get results. On the other hand, with the streaming setup, users get initial results immediately, and although end-to-end latency will be the same, they can see half of the generation after five seconds. Below you can see an interactive demo that shows non-streaming vs streaming side-by-side. Click **generate** below. + +
+ +
+ + +## How to use Streaming? + +### Streaming with Python + +To stream tokens with `InferenceClient`, simply pass `stream=True` and iterate over the response. + +```python +from huggingface_hub import InferenceClient + +client = InferenceClient("http://127.0.0.1:8080") +for token in client.text_generation("How do you make cheese?", max_new_tokens=12, stream=True): + print(token) + +# To +# make +# cheese +#, +# you +# need +# to +# start +# with +# milk +#. +``` + +If you want additional details, you can add `details=True`. In this case, you get a `TextGenerationStreamResponse` which contains additional information such as the probabilities and the tokens. For the final response in the stream, it also returns the full generated text. + +```python +for details in client.text_generation("How do you make cheese?", max_new_tokens=12, details=True, stream=True): + print(details) + +#TextGenerationStreamResponse(token=Token(id=193, text='\n', logprob=-0.007358551, special=False), generated_text=None, details=None) +#TextGenerationStreamResponse(token=Token(id=2044, text='To', logprob=-1.1357422, special=False), generated_text=None, details=None) +#TextGenerationStreamResponse(token=Token(id=717, text=' make', logprob=-0.009841919, special=False), generated_text=None, details=None) +#... +#TextGenerationStreamResponse(token=Token(id=25, text='.', logprob=-1.3408203, special=False), generated_text='\nTo make cheese, you need to start with milk.', details=StreamDetails(finish_reason=, generated_tokens=12, seed=None)) +``` + +The `huggingface_hub` library also comes with an `AsyncInferenceClient` in case you need to handle the requests concurrently. + +```python +from huggingface_hub import AsyncInferenceClient + +client = AsyncInferenceClient("http://127.0.0.1:8080") +async for token in await client.text_generation("How do you make cheese?", stream=True): + print(token) + +# To +# make +# cheese +#, +# you +# need +# to +# start +# with +# milk +#. +``` + +### Streaming with cURL + +To use the `generate_stream` endpoint with curl, you can add the `-N` flag, which disables curl default buffering and shows data as it arrives from the server + +```curl +curl -N 127.0.0.1:8080/generate_stream \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ + -H 'Content-Type: application/json' +``` + +### Streaming with JavaScript + +First, we need to install the `@huggingface/inference` library. +`npm install @huggingface/inference` + +If you're using the free Inference API, you can use `HfInference`. If you're using inference endpoints, you can use `HfInferenceEndpoint`. Let's + +We can create a `HfInferenceEndpoint` providing our endpoint URL and credential. + +```js +import { HfInference } from '@huggingface/inference' + +const hf = new HfInference('https://YOUR_ENDPOINT.endpoints.huggingface.cloud', 'hf_YOUR_TOKEN') + +// prompt +const prompt = 'What can you do in Nuremberg, Germany? Give me 3 Tips' + +const stream = hf.textGenerationStream({ inputs: prompt }) +for await (const r of stream) { + // yield the generated token + process.stdout.write(r.token.text) +} +``` + +## How does Streaming work under the hood? + +Under the hood, TGI uses Server-Sent Events (SSE). In an SSE Setup, a client sends a request with the data, opening an HTTP connection and subscribing to updates. Afterward, the server sends data to the client. There is no need for further requests; the server will keep sending the data. SSEs are unidirectional, meaning the client does not send other requests to the server. SSE sends data over HTTP, making it easy to use. + +SSEs are different than: +* Polling: where the client keeps calling the server to get data. This means that the server might return empty responses and cause overhead. +* Webhooks: where there is a bi-directional connection. The server can send information to the client, but the client can also send data to the server after the first request. Webhooks are more complex to operate as they don’t only use HTTP. + +One of the limitations of Server-Sent Events is that they limit how many concurrent requests can handle by the server. Instead of timing out when there are too many SSE connections, TGI returns a HTTP Error with an `overloaded` error type (`huggingface_hub` returns `OverloadedError`). This allows the client to manage the overloaded server (e.g. it could display a busy error to the user or it could retry with a new request). To configure the maximum number of concurrent requests, you can specify `--max_concurrent_requests`, allowing to handle backpressure. + +One of the limitations of Server-Sent Events is that they limit how many concurrent requests can handle by the server. Instead of timing out when there are too many SSE connections, TGI returns an HTTP Error with an `overloaded` error type (`huggingface_hub` returns `OverloadedError`). This allows the client to manage the overloaded server (e.g., it could display a busy error to the user or retry with a new request). To configure the maximum number of concurrent requests, you can specify `--max_concurrent_requests`, allowing clients to handle backpressure. diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index 4ba2be40..170f8dc0 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -85,7 +85,7 @@ curl 127.0.0.1:8080/generate \ To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. ```bash -docker run ghcr.io/huggingface/text-generation-inference:1.0.0 --help +docker run ghcr.io/huggingface/text-generation-inference:1.0.1 --help ``` From c4422e567811051e40469551c58a8078158a6656 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 18 Aug 2023 19:28:56 +0200 Subject: [PATCH 392/793] Adding small benchmark script. (#881) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- load_tests/common.js | 64 ++++++++++++++++++++++++++++++++++++++++++++ load_tests/tgi.js | 17 ++++++++++++ load_tests/vllm.js | 17 ++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 load_tests/common.js create mode 100644 load_tests/tgi.js create mode 100644 load_tests/vllm.js diff --git a/load_tests/common.js b/load_tests/common.js new file mode 100644 index 00000000..be812e9b --- /dev/null +++ b/load_tests/common.js @@ -0,0 +1,64 @@ +import { check, randomSeed } from 'k6'; +import http from 'k6/http'; +import { Trend, Counter } from 'k6/metrics'; +import { randomItem } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js'; + +const seed = 0; + +const host = __ENV.HOST || '127.0.0.1:8000'; +const timePerToken = new Trend('time_per_token', true); +const throughput = new Counter('tokens_per_s'); + +randomSeed(seed); +// const shareGPT = JSON.parse(open("ShareGPT_V3_unfiltered_cleaned_split.json")) +const shareGPT = JSON.parse(open("small.json")) + + +export function get_options(reference_latency_ms){ + return { + thresholds: { + http_req_failed: ['rate==0'], + time_per_token: [{ + threshold: `p(50)<${3 * reference_latency_ms}`, + abortOnFail: true, + delayAbortEval: '10s' + }], + }, + scenarios: { + load_test: { + executor: 'constant-arrival-rate', + duration: '60s', + preAllocatedVUs: 100, + rate: 10, + timeUnit: '1s', + }, + }, + }; +} + + +export function run(host, generate_payload, max_new_tokens) { + const headers = {'Content-Type': 'application/json'}; + const query = randomItem(shareGPT); + const payload = JSON.stringify(generate_payload(query)); + const res = http.post(`http://${host}/generate`, payload, { + headers, + }); + if(res.status >= 400 && res.status < 500){ + return; + } + + check(res, { + 'Post status is 200': (r) => res.status === 200, + }); + const n_tokens = max_new_tokens; + const timings = res.timings.duration; + + if (res.status === 200) { + const latency_ms_per_token = timings / n_tokens; + timePerToken.add(latency_ms_per_token); + const latency_in_s = latency_ms_per_token / 1000; + const individual_throughput = 1 / latency_in_s; + throughput.add(individual_throughput); + } +} diff --git a/load_tests/tgi.js b/load_tests/tgi.js new file mode 100644 index 00000000..93a0e278 --- /dev/null +++ b/load_tests/tgi.js @@ -0,0 +1,17 @@ +import { get_options, run } from "./common.js"; + +const reference_latency_ms = 30; +const host = __ENV.HOST || '127.0.0.1:8000'; +const max_new_tokens = 50; + + +function generate_payload(gpt){ + const input = gpt["conversations"][0]["value"]; + return {"inputs": input, "parameters": {"max_new_tokens": max_new_tokens, "temperature" : 0.5}} +} + +export const options = get_options(reference_latency_ms); + +export default function(){ + run(host, generate_payload, max_new_tokens); +} diff --git a/load_tests/vllm.js b/load_tests/vllm.js new file mode 100644 index 00000000..fcb38262 --- /dev/null +++ b/load_tests/vllm.js @@ -0,0 +1,17 @@ +import { get_options, run } from "./common.js"; + +const reference_latency_ms = 22; +const host = __ENV.HOST || '127.0.0.1:8000'; +const max_new_tokens = 50; + + +function generate_payload(gpt){ + const input = gpt["conversations"][0]["value"]; + return {"prompt": input, "temperature": 0.5, "ignore_eos": true} +} + +export const options = get_options(reference_latency_ms); + +export default function(){ + run(host, generate_payload, max_new_tokens); +} From 888c029114e76a6c37560e5779bd7c4509b8e8f2 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 23 Aug 2023 13:45:28 +0200 Subject: [PATCH 393/793] Upgrade version number in docs. (#910) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- README.md | 4 ++-- docs/openapi.json | 2 +- docs/source/quicktour.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d9f4db19..739e656b 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ The easiest way of getting started is using the official Docker container: model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.1 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.2 --model-id $model ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. @@ -153,7 +153,7 @@ model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= -docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.1 --model-id $model +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.2 --model-id $model ``` ### A note on Shared Memory (shm) diff --git a/docs/openapi.json b/docs/openapi.json index ab1802c3..5974c58d 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.0.1" + "version": "1.0.2" }, "paths": { "/": { diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index 170f8dc0..c085943c 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -8,7 +8,7 @@ Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/ model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.1 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.2 --model-id $model ``` @@ -85,7 +85,7 @@ curl 127.0.0.1:8080/generate \ To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. ```bash -docker run ghcr.io/huggingface/text-generation-inference:1.0.1 --help +docker run ghcr.io/huggingface/text-generation-inference:1.0.2 --help ``` From 97444f9367a3394396cda0b80098e67f6711bd34 Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Thu, 24 Aug 2023 00:50:12 +0300 Subject: [PATCH 394/793] Added gradio example to docs (#867) cc @osanseviero --------- Co-authored-by: Omar Sanseviero --- docs/source/basic_tutorials/consuming_tgi.md | 75 ++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md index 1f0ff37d..540f4b13 100644 --- a/docs/source/basic_tutorials/consuming_tgi.md +++ b/docs/source/basic_tutorials/consuming_tgi.md @@ -75,6 +75,81 @@ To serve both ChatUI and TGI in same environment, simply add your own endpoints ![ChatUI](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/chatui_screen.png) +## Gradio + +Gradio is a Python library that helps you build web applications for your machine learning models with a few lines of code. It has a `ChatInterface` wrapper that helps create neat UIs for chatbots. Let's take a look at how to create a chatbot with streaming mode using TGI and Gradio. Let's install Gradio and Hub Python library first. + +```bash +pip install huggingface-hub gradio +``` + +Assume you are serving your model on port 8080, we will query through [InferenceClient](consuming_tgi#inference-client). + +```python +import gradio as gr +from huggingface_hub import InferenceClient + +client = InferenceClient(model="http://127.0.0.1:8080") + +def inference(message, history): + partial_message = "" + for token in client.text_generation(message, max_new_tokens=20, stream=True): + partial_message += token + yield partial_message + +gr.ChatInterface( + inference, + chatbot=gr.Chatbot(height=300), + textbox=gr.Textbox(placeholder="Chat with me!", container=False, scale=7), + description="This is the demo for Gradio UI consuming TGI endpoint with LLaMA 7B-Chat model.", + title="Gradio 🤝 TGI", + examples=["Are tomatoes vegetables?"], + retry_btn="Retry", + undo_btn="Undo", + clear_btn="Clear", +).queue().launch() +``` + +The UI looks like this 👇 + +
+ + +
+ +You can try the demo directly here 👇 + +
+ +
+ + + +You can disable streaming mode using `return` instead of `yield` in your inference function, like below. + +```python +def inference(message, history): + return client.text_generation(message, max_new_tokens=20) +``` + +You can read more about how to customize a `ChatInterface` [here](https://www.gradio.app/guides/creating-a-chatbot-fast). + ## API documentation You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. The Swagger UI is also available [here](https://huggingface.github.io/text-generation-inference). From e605c2a43e693844cb2c5ba879f41392faf64793 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 24 Aug 2023 18:54:47 +0200 Subject: [PATCH 395/793] Supporting code llama. (#918) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../models/custom_modeling/flash_llama_modeling.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index d0185ede..f0e1236d 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -64,6 +64,7 @@ def __init__( pretraining_tp=1, tie_word_embeddings=False, rope_scaling=None, + rope_theta=10000.0, **kwargs, ): self.vocab_size = vocab_size @@ -84,6 +85,7 @@ def __init__( self.pretraining_tp = pretraining_tp self.use_cache = use_cache self.rope_scaling = rope_scaling + self.rope_theta = rope_theta super().__init__( pad_token_id=pad_token_id, @@ -189,7 +191,7 @@ def __init__( # config=config, prefix=f"{prefix}.rotary_emb", weights=weights # ) self.rotary_emb = PositionRotaryEmbedding.static( - config=config, dim=self.head_size, base=10000.0, device=weights.device + config=config, dim=self.head_size, base=config.rope_theta, device=weights.device ) self.softmax_scale = self.head_size**-0.5 From 4486f78cf92bcca23c0d2c37bf13f6e06f6c7ad0 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 28 Aug 2023 11:13:24 +0200 Subject: [PATCH 396/793] Fixing the lora adaptation on docker. (#935) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index b12a9751..e3fda07f 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -125,6 +125,9 @@ def download_weights( try: adapter_config_filename = hf_hub_download(model_id, revision=revision, filename="adapter_config.json") utils.download_and_unload_peft(model_id, revision, trust_remote_code=trust_remote_code) + is_local_model = True + utils.weight_files(model_id, revision, extension) + return except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError): pass From 211b54ac41cae9a369f3d74bd6cc666ff4a0c526 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 28 Aug 2023 11:43:47 +0200 Subject: [PATCH 397/793] Rebased #617 (#868) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Vincent Brouwers --- benchmark/src/generation.rs | 8 +- benchmark/src/lib.rs | 3 + benchmark/src/main.rs | 7 ++ benchmark/src/table.rs | 2 + clients/python/text_generation/client.py | 16 +++ clients/python/text_generation/types.py | 18 ++- launcher/src/main.rs | 10 ++ proto/generate.proto | 15 +++ router/client/src/client.rs | 1 + router/src/health.rs | 1 + router/src/infer.rs | 45 ++++++- router/src/lib.rs | 10 ++ router/src/main.rs | 4 + router/src/queue.rs | 2 + router/src/server.rs | 14 ++- router/src/validation.rs | 116 ++++++++++++++++-- server/tests/utils/test_tokens.py | 21 ++++ .../models/causal_lm.py | 58 +++++++++ .../models/flash_causal_lm.py | 57 ++++++++- .../models/idefics_causal_lm.py | 3 + .../models/seq2seq_lm.py | 59 ++++++++- server/text_generation_server/models/types.py | 23 ++++ server/text_generation_server/utils/tokens.py | 70 ++++++++--- 23 files changed, 529 insertions(+), 34 deletions(-) diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs index c72d31d3..67afa04e 100644 --- a/benchmark/src/generation.rs +++ b/benchmark/src/generation.rs @@ -37,6 +37,7 @@ pub(crate) async fn generation_task( batch_size: Vec, sequence_length: u32, decode_length: u32, + top_n_tokens: Option, n_runs: usize, warmups: usize, parameters: NextTokenChooserParameters, @@ -48,7 +49,7 @@ pub(crate) async fn generation_task( // End task if a message is received on shutdown_receiver // _shutdown_guard_sender will be dropped once the task is finished tokio::select! { - res = generate_runs(tokenizer, batch_size, sequence_length, decode_length, n_runs, warmups, parameters, client, run_sender.clone()) => { + res = generate_runs(tokenizer, batch_size, sequence_length, decode_length, top_n_tokens, n_runs, warmups, parameters, client, run_sender.clone()) => { if let Err(err) = res { run_sender.send(Err(err)).await.unwrap_or(()); } @@ -64,6 +65,7 @@ async fn generate_runs( batch_size: Vec, sequence_length: u32, decode_length: u32, + top_n_tokens: Option, n_runs: usize, warmups: usize, parameters: NextTokenChooserParameters, @@ -82,6 +84,7 @@ async fn generate_runs( b, decode_length, parameters.clone(), + top_n_tokens, &mut client, ) .await?; @@ -97,6 +100,7 @@ async fn generate_runs( b, decode_length, parameters.clone(), + top_n_tokens, &mut client, ) .await?; @@ -130,6 +134,7 @@ async fn prefill( batch_size: u32, decode_length: u32, parameters: NextTokenChooserParameters, + top_n_tokens: Option, client: &mut ShardedClient, ) -> Result<(Prefill, CachedBatch), ClientError> { // Create requests @@ -145,6 +150,7 @@ async fn prefill( stop_sequences: vec![], ignore_eos_token: true, // Will not stop even if a eos token is generated }), + top_n_tokens: top_n_tokens.unwrap_or(0), }) .collect(); diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs index fcad400c..433c6f67 100644 --- a/benchmark/src/lib.rs +++ b/benchmark/src/lib.rs @@ -22,6 +22,7 @@ pub async fn run( batch_size: Vec, sequence_length: u32, decode_length: u32, + top_n_tokens: Option, n_runs: usize, warmups: usize, temperature: Option, @@ -70,6 +71,7 @@ pub async fn run( batch_size.clone(), sequence_length, decode_length, + top_n_tokens, n_runs, warmups, parameters, @@ -130,6 +132,7 @@ pub async fn run( tokenizer_name, sequence_length, decode_length, + top_n_tokens, n_runs, warmups, temperature, diff --git a/benchmark/src/main.rs b/benchmark/src/main.rs index a7550060..97c8af1c 100644 --- a/benchmark/src/main.rs +++ b/benchmark/src/main.rs @@ -93,6 +93,11 @@ struct Args { /// decoding strategies, for full doc refer to the `text-generation-server` #[clap(long, env)] do_sample: bool, + + /// Generation parameter in case you want to specifically test/debug particular + /// decoding strategies, for full doc refer to the `text-generation-server` + #[clap(long, env)] + top_n_tokens: Option, } fn main() -> Result<(), Box> { @@ -117,6 +122,7 @@ fn main() -> Result<(), Box> { watermark, do_sample, master_shard_uds_path, + top_n_tokens, } = args; let batch_size = batch_size.unwrap_or(vec![1, 2, 4, 8, 16, 32]); @@ -173,6 +179,7 @@ fn main() -> Result<(), Box> { batch_size, sequence_length, decode_length, + top_n_tokens, runs, warmups, temperature, diff --git a/benchmark/src/table.rs b/benchmark/src/table.rs index 6b74bc36..9e36717b 100644 --- a/benchmark/src/table.rs +++ b/benchmark/src/table.rs @@ -7,6 +7,7 @@ pub(crate) fn parameters_table( tokenizer_name: String, sequence_length: u32, decode_length: u32, + top_n_tokens: Option, n_runs: usize, warmups: usize, temperature: Option, @@ -24,6 +25,7 @@ pub(crate) fn parameters_table( builder.push_record(["Model", &tokenizer_name]); builder.push_record(["Sequence Length", &sequence_length.to_string()]); builder.push_record(["Decode Length", &decode_length.to_string()]); + builder.push_record(["Top N Tokens", &format!("{top_n_tokens:?}")]); builder.push_record(["N Runs", &n_runs.to_string()]); builder.push_record(["Warmups", &warmups.to_string()]); builder.push_record(["Temperature", &format!("{temperature:?}")]); diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index bf045d47..015613c2 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -75,6 +75,7 @@ def generate( typical_p: Optional[float] = None, watermark: bool = False, decoder_input_details: bool = False, + top_n_tokens: Optional[int] = None, ) -> Response: """ Given a prompt, generate the following text @@ -113,6 +114,8 @@ def generate( Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) decoder_input_details (`bool`): Return the decoder input token logprobs and ids + top_n_tokens (`int`): + Return the `n` most likely tokens at each step Returns: Response: generated response @@ -134,6 +137,7 @@ def generate( typical_p=typical_p, watermark=watermark, decoder_input_details=decoder_input_details, + top_n_tokens=top_n_tokens ) request = Request(inputs=prompt, stream=False, parameters=parameters) @@ -164,6 +168,7 @@ def generate_stream( truncate: Optional[int] = None, typical_p: Optional[float] = None, watermark: bool = False, + top_n_tokens: Optional[int] = None, ) -> Iterator[StreamResponse]: """ Given a prompt, generate the following stream of tokens @@ -198,6 +203,8 @@ def generate_stream( See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + top_n_tokens (`int`): + Return the `n` most likely tokens at each step Returns: Iterator[StreamResponse]: stream of generated tokens @@ -219,6 +226,7 @@ def generate_stream( truncate=truncate, typical_p=typical_p, watermark=watermark, + top_n_tokens=top_n_tokens, ) request = Request(inputs=prompt, stream=True, parameters=parameters) @@ -317,6 +325,7 @@ async def generate( typical_p: Optional[float] = None, watermark: bool = False, decoder_input_details: bool = False, + top_n_tokens: Optional[int] = None, ) -> Response: """ Given a prompt, generate the following text asynchronously @@ -355,6 +364,8 @@ async def generate( Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) decoder_input_details (`bool`): Return the decoder input token logprobs and ids + top_n_tokens (`int`): + Return the `n` most likely tokens at each step Returns: Response: generated response @@ -376,6 +387,7 @@ async def generate( truncate=truncate, typical_p=typical_p, watermark=watermark, + top_n_tokens=top_n_tokens, ) request = Request(inputs=prompt, stream=False, parameters=parameters) @@ -404,6 +416,7 @@ async def generate_stream( truncate: Optional[int] = None, typical_p: Optional[float] = None, watermark: bool = False, + top_n_tokens: Optional[int] = None, ) -> AsyncIterator[StreamResponse]: """ Given a prompt, generate the following stream of tokens asynchronously @@ -438,6 +451,8 @@ async def generate_stream( See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information watermark (`bool`): Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) + top_n_tokens (`int`): + Return the `n` most likely tokens at each step Returns: AsyncIterator[StreamResponse]: stream of generated tokens @@ -459,6 +474,7 @@ async def generate_stream( truncate=truncate, typical_p=typical_p, watermark=watermark, + top_n_tokens=top_n_tokens, ) request = Request(inputs=prompt, stream=True, parameters=parameters) diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index 548f0b63..38f75253 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -39,6 +39,8 @@ class Parameters(BaseModel): details: bool = False # Get decoder input token logprobs and ids decoder_input_details: bool = False + # Return the N most likely tokens at each step + top_n_tokens: Optional[int] @validator("best_of") def valid_best_of(cls, field_value, values): @@ -101,6 +103,12 @@ def valid_typical_p(cls, v): raise ValidationError("`typical_p` must be > 0.0 and < 1.0") return v + @validator("top_n_tokens") + def valid_top_n_tokens(cls, v): + if v is not None and v <= 0: + raise ValidationError("`top_n_tokens` must be strictly positive") + return v + class Request(BaseModel): # Prompt @@ -125,9 +133,7 @@ def valid_best_of_stream(cls, field_value, values): and parameters.best_of > 1 and field_value ): - raise ValidationError( - "`best_of` != 1 is not supported when `stream` == True" - ) + raise ValidationError("`best_of` != 1 is not supported when `stream` == True") return field_value @@ -179,6 +185,8 @@ class BestOfSequence(BaseModel): prefill: List[InputToken] # Generated tokens tokens: List[Token] + # Most likely tokens + top_tokens: Optional[List[List[Token]]] # `generate` details @@ -193,6 +201,8 @@ class Details(BaseModel): prefill: List[InputToken] # Generated tokens tokens: List[Token] + # Most likely tokens + top_tokens: Optional[List[List[Token]]] # Additional sequences when using the `best_of` parameter best_of_sequences: Optional[List[BestOfSequence]] @@ -219,6 +229,8 @@ class StreamDetails(BaseModel): class StreamResponse(BaseModel): # Generated token token: Token + # Most likely tokens + top_tokens: Optional[List[Token]] # Complete generated text # Only available when the generation is finished generated_text: Optional[str] diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 75762712..cbb6f25d 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -159,6 +159,14 @@ struct Args { #[clap(default_value = "4", long, env)] max_stop_sequences: usize, + /// This is the maximum allowed value for clients to set `top_n_tokens`. + /// `top_n_tokens is used to return information about the the `n` most likely + /// tokens at each generation step, instead of just the sampled token. This + /// information can be used for downstream tasks like for classification or + /// ranking. + #[clap(default_value = "5", long, env)] + max_top_n_tokens: u32, + /// This is the maximum allowed input length (expressed in number of tokens) /// for users. The larger this value, the longer prompt users can send which /// can impact the overall memory required to handle the load. @@ -929,6 +937,8 @@ fn spawn_webserver( args.max_best_of.to_string(), "--max-stop-sequences".to_string(), args.max_stop_sequences.to_string(), + "--max-top-n-tokens".to_string(), + args.max_top_n_tokens.to_string(), "--max-input-length".to_string(), args.max_input_length.to_string(), "--max-total-tokens".to_string(), diff --git a/proto/generate.proto b/proto/generate.proto index 57d79bca..3f607dc5 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -91,6 +91,8 @@ message Request { StoppingCriteriaParameters stopping_parameters = 5; /// Return prefill logprobs bool prefill_logprobs = 6; + /// Return most likely n tokens + uint32 top_n_tokens = 7; } message Batch { @@ -141,6 +143,17 @@ message PrefillTokens { repeated string texts = 3; } +message TopTokens { + /// Top Token IDs + repeated uint32 ids = 1; + /// Top Logprobs + repeated float logprobs = 2; + /// Top Token Texts + repeated string texts = 3; + /// If the tokens are special + repeated bool is_special = 6; +} + message Generation { /// Request ID uint64 request_id = 1; @@ -156,6 +169,8 @@ message Generation { bool token_is_special = 6; /// Complete generated text optional GeneratedText generated_text = 7; + /// Top tokens + TopTokens top_tokens = 8; } message FilterBatchRequest { diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 7753f307..d427d3a4 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -131,6 +131,7 @@ impl Client { ignore_eos_token: false, }), prefill_logprobs: true, + top_n_tokens: 20, }); n_tokens += max_input_length; } diff --git a/router/src/health.rs b/router/src/health.rs index a3cacdcd..ab290fc1 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -50,6 +50,7 @@ impl Health { stop_sequences: vec![], ignore_eos_token: false, }), + top_n_tokens: 0, }; let batch = Batch { id: BATCH_ID, diff --git a/router/src/infer.rs b/router/src/infer.rs index 188ddc64..67b5bde2 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -138,12 +138,15 @@ impl Infer { &self, request: GenerateRequest, ) -> Result { + let use_top_tokens = request.parameters.top_n_tokens.is_some_and(|x| x > 0); + // Create stream and keep semaphore permit as long as generate lives let (_permit, mut stream) = self.generate_stream(request).await?; // Return values let mut result_prefill = Vec::new(); let mut result_tokens = Vec::new(); + let mut result_top_tokens = Vec::new(); let mut result_generated_text = None; let mut result_start = None; let mut result_queued = None; @@ -164,7 +167,10 @@ impl Infer { .collect(); } // Push last token - InferStreamResponse::Token(token) => result_tokens.push(token), + InferStreamResponse::Intermediate { token, top_tokens } => { + result_tokens.push(token); + result_top_tokens.push(top_tokens); + } // Final message // Set return values InferStreamResponse::End { @@ -172,8 +178,10 @@ impl Infer { generated_text, start, queued, + top_tokens, } => { result_tokens.push(token); + result_top_tokens.push(top_tokens); result_generated_text = Some(generated_text); result_start = Some(start); result_queued = Some(queued) @@ -191,6 +199,11 @@ impl Infer { generated_text, queued, start, + top_tokens: if use_top_tokens { + result_top_tokens + } else { + Vec::new() + }, }) } else { let err = InferError::IncompleteGeneration; @@ -520,6 +533,26 @@ fn send_responses( special: generation.token_is_special, }; + // generation.top_tokens + + let mut top_tokens = Vec::new(); + if let Some(top_tokens_) = generation.top_tokens { + top_tokens.extend( + top_tokens_ + .ids + .into_iter() + .zip(top_tokens_.logprobs.into_iter()) + .zip(top_tokens_.texts.into_iter()) + .zip(top_tokens_.is_special.into_iter()) + .map(|(((id, logprob), text), special)| Token { + id, + text, + logprob, + special, + }), + ) + } + if let Some(generated_text) = generation.generated_text { // Generation has ended stopped = true; @@ -527,6 +560,7 @@ fn send_responses( entry.response_tx.send_timeout( Ok(InferStreamResponse::End { token, + top_tokens, generated_text, queued: entry.queue_time, start: entry.batch_time.unwrap(), @@ -536,7 +570,7 @@ fn send_responses( } else { // Send message entry.response_tx.send_timeout( - Ok(InferStreamResponse::Token(token)), + Ok(InferStreamResponse::Intermediate { token, top_tokens }), Duration::from_millis(10), )?; } @@ -566,10 +600,14 @@ pub(crate) enum InferStreamResponse { // Optional first message Prefill(PrefillTokens), // Intermediate messages - Token(Token), + Intermediate { + token: Token, + top_tokens: Vec, + }, // Last message End { token: Token, + top_tokens: Vec, generated_text: GeneratedText, start: Instant, queued: Instant, @@ -583,6 +621,7 @@ pub(crate) struct InferResponse { pub(crate) generated_text: GeneratedText, pub(crate) queued: Instant, pub(crate) start: Instant, + pub(crate) top_tokens: Vec>, } #[derive(Debug, Error)] diff --git a/router/src/lib.rs b/router/src/lib.rs index 7dff7a11..76e70bb7 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -135,6 +135,9 @@ pub(crate) struct GenerateParameters { example = "null" )] pub seed: Option, + #[serde(default)] + #[schema(exclusive_minimum = 0, nullable = true, default = "null", example = 5)] + pub top_n_tokens: Option, } fn default_max_new_tokens() -> u32 { @@ -158,6 +161,7 @@ fn default_parameters() -> GenerateParameters { details: false, decoder_input_details: false, seed: None, + top_n_tokens: None, } } @@ -235,6 +239,8 @@ pub(crate) struct BestOfSequence { pub seed: Option, pub prefill: Vec, pub tokens: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub top_tokens: Vec>, } #[derive(Serialize, ToSchema)] @@ -249,6 +255,8 @@ pub(crate) struct Details { pub tokens: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub best_of_sequences: Option>, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub top_tokens: Vec>, } #[derive(Serialize, ToSchema)] @@ -272,6 +280,8 @@ pub(crate) struct StreamDetails { #[derive(Serialize, ToSchema)] pub(crate) struct StreamResponse { pub token: Token, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub top_tokens: Vec, #[schema(nullable = true, default = "null", example = "test")] pub generated_text: Option, #[schema(nullable = true, default = "null")] diff --git a/router/src/main.rs b/router/src/main.rs index 484643cb..4903c066 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -29,6 +29,8 @@ struct Args { max_best_of: usize, #[clap(default_value = "4", long, env)] max_stop_sequences: usize, + #[clap(default_value = "5", long, env)] + max_top_n_tokens: u32, #[clap(default_value = "1024", long, env)] max_input_length: usize, #[clap(default_value = "2048", long, env)] @@ -75,6 +77,7 @@ fn main() -> Result<(), RouterError> { max_concurrent_requests, max_best_of, max_stop_sequences, + max_top_n_tokens, max_input_length, max_total_tokens, waiting_served_ratio, @@ -259,6 +262,7 @@ fn main() -> Result<(), RouterError> { max_concurrent_requests, max_best_of, max_stop_sequences, + max_top_n_tokens, max_input_length, max_total_tokens, waiting_served_ratio, diff --git a/router/src/queue.rs b/router/src/queue.rs index 2d8d6d1c..e97a168e 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -235,6 +235,7 @@ impl State { truncate: entry.request.truncate, parameters: Some(entry.request.parameters.clone()), stopping_parameters: Some(entry.request.stopping_parameters.clone()), + top_n_tokens: entry.request.top_n_tokens, }); // Set batch_time entry.batch_time = Some(Instant::now()); @@ -328,6 +329,7 @@ mod tests { max_new_tokens: 1, stop_sequences: vec![], }, + top_n_tokens: 0, }, response_tx, span: info_span!("entry"), diff --git a/router/src/server.rs b/router/src/server.rs index e609821c..91164098 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -158,7 +158,7 @@ async fn generate( add_prompt = Some(req.inputs.clone()); } - let details = req.parameters.details || req.parameters.decoder_input_details; + let details: bool = req.parameters.details || req.parameters.decoder_input_details; // Inference let (response, best_of_responses) = match req.parameters.best_of { @@ -191,6 +191,7 @@ async fn generate( generated_tokens: response.generated_text.generated_tokens, prefill: response.prefill, tokens: response.tokens, + top_tokens: response.top_tokens, seed: response.generated_text.seed, } }) @@ -204,6 +205,7 @@ async fn generate( tokens: response.tokens, seed: response.generated_text.seed, best_of_sequences, + top_tokens: response.top_tokens, }) } false => None, @@ -385,12 +387,16 @@ async fn generate_stream( // Prefill is ignored InferStreamResponse::Prefill(_) => {} // Yield event for every new token - InferStreamResponse::Token(token) => { + InferStreamResponse::Intermediate{ + token, + top_tokens, + } => { tracing::debug!(parent: &span, "Token: {:?}", token); // StreamResponse let stream_token = StreamResponse { token, + top_tokens: top_tokens, generated_text: None, details: None, }; @@ -403,6 +409,7 @@ async fn generate_stream( generated_text, start, queued, + top_tokens, } => { // Token details let details = match details { @@ -451,6 +458,7 @@ async fn generate_stream( let stream_token = StreamResponse { token, + top_tokens: top_tokens, generated_text: Some(output_text), details }; @@ -509,6 +517,7 @@ pub async fn run( max_concurrent_requests: usize, max_best_of: usize, max_stop_sequences: usize, + max_top_n_tokens: u32, max_input_length: usize, max_total_tokens: usize, waiting_served_ratio: f32, @@ -571,6 +580,7 @@ pub async fn run( tokenizer, max_best_of, max_stop_sequences, + max_top_n_tokens, max_input_length, max_total_tokens, ); diff --git a/router/src/validation.rs b/router/src/validation.rs index f967361f..6c67f0ff 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -15,6 +15,7 @@ pub struct Validation { /// Validation parameters max_best_of: usize, max_stop_sequences: usize, + max_top_n_tokens: u32, max_input_length: usize, max_total_tokens: usize, /// Channel to communicate with the background tokenization task @@ -27,6 +28,7 @@ impl Validation { tokenizer: Option, max_best_of: usize, max_stop_sequences: usize, + max_top_n_tokens: u32, max_input_length: usize, max_total_tokens: usize, ) -> Self { @@ -54,6 +56,7 @@ impl Validation { max_best_of, sender, max_stop_sequences, + max_top_n_tokens, max_input_length, max_total_tokens, } @@ -142,6 +145,7 @@ impl Validation { seed, watermark, decoder_input_details, + top_n_tokens, .. } = request.parameters; @@ -218,6 +222,15 @@ impl Validation { } }; + let top_n_tokens = top_n_tokens + .map(|value| { + if value > self.max_top_n_tokens { + return Err(ValidationError::TopNTokens(self.max_top_n_tokens, value)); + } + Ok(value) + }) + .unwrap_or(Ok(0))?; + // Check if inputs is empty if request.inputs.is_empty() { return Err(EmptyInput); @@ -263,6 +276,7 @@ impl Validation { truncate: truncate.unwrap_or(self.max_input_length) as u32, parameters, stopping_parameters, + top_n_tokens: top_n_tokens, }) } @@ -336,6 +350,7 @@ pub(crate) struct ValidGenerateRequest { pub decoder_input_details: bool, pub parameters: NextTokenChooserParameters, pub stopping_parameters: StoppingCriteriaParameters, + pub top_n_tokens: u32, } #[derive(Error, Debug)] @@ -350,6 +365,10 @@ pub enum ValidationError { BestOfSeed, #[error("`best_of` != 1 is not supported when streaming tokens")] BestOfStream, + #[error("`top_n_tokens` must be >= 0 and <= {0}. Given: {1}")] + TopNTokens(u32, u32), + #[error("`top_n_tokens` != 0 is not allowed for this endpoint")] + TopNTokensDisabled, #[error("`decoder_input_details` == true is not supported when streaming tokens")] PrefillDetailsStream, #[error("`temperature` must be strictly positive")] @@ -391,14 +410,16 @@ mod tests { let tokenizer = None; let max_best_of = 2; let max_stop_sequence = 3; - let max_input_length = 4; - let max_total_tokens = 5; + let max_top_n_tokens = 4; + let max_input_length = 5; + let max_total_tokens = 6; let workers = 1; let validation = Validation::new( workers, tokenizer, max_best_of, max_stop_sequence, + max_top_n_tokens, max_input_length, max_total_tokens, ); @@ -418,14 +439,16 @@ mod tests { let tokenizer = Some(get_tokenizer().await); let max_best_of = 2; let max_stop_sequence = 3; - let max_input_length = 4; - let max_total_tokens = 5; + let max_top_n_tokens = 4; + let max_input_length = 5; + let max_total_tokens = 6; let workers = 1; let validation = Validation::new( workers, tokenizer, max_best_of, max_stop_sequence, + max_top_n_tokens, max_input_length, max_total_tokens, ); @@ -435,7 +458,7 @@ mod tests { .validate_input("Hello".to_string(), None, max_new_tokens) .await { - Err(ValidationError::MaxTotalTokens(5, 1, 10)) => (), + Err(ValidationError::MaxTotalTokens(6, 1, 10)) => (), _ => panic!("Unexpected not max new tokens"), } } @@ -445,14 +468,16 @@ mod tests { let tokenizer = Some(get_tokenizer().await); let max_best_of = 2; let max_stop_sequence = 3; - let max_input_length = 4; - let max_total_tokens = 5; + let max_top_n_tokens = 4; + let max_input_length = 5; + let max_total_tokens = 6; let workers = 1; let validation = Validation::new( workers, tokenizer, max_best_of, max_stop_sequence, + max_top_n_tokens, max_input_length, max_total_tokens, ); @@ -477,14 +502,16 @@ mod tests { let tokenizer = Some(get_tokenizer().await); let max_best_of = 2; let max_stop_sequence = 3; - let max_input_length = 4; - let max_total_tokens = 5; + let max_top_n_tokens = 4; + let max_input_length = 5; + let max_total_tokens = 6; let workers = 1; let validation = Validation::new( workers, tokenizer, max_best_of, max_stop_sequence, + max_top_n_tokens, max_input_length, max_total_tokens, ); @@ -531,4 +558,75 @@ mod tests { // top_p == 1.0 is invalid for users to ask for but it's the default resolved value. assert_eq!(valid_request.parameters.top_p, 1.0); } + + #[tokio::test] + async fn test_validation_top_n_tokens() { + let tokenizer = Some(get_tokenizer().await); + let max_best_of = 2; + let max_stop_sequences = 3; + let max_top_n_tokens = 4; + let max_input_length = 5; + let max_total_tokens = 6; + let workers = 1; + let validation = Validation::new( + workers, + tokenizer, + max_best_of, + max_stop_sequences, + max_top_n_tokens, + max_input_length, + max_total_tokens, + ); + match validation + .validate(GenerateRequest { + inputs: "Hello".to_string(), + parameters: GenerateParameters { + top_n_tokens: Some(5), + ..default_parameters() + }, + }) + .await + { + Err(ValidationError::TopNTokens(4, 5)) => (), + _ => panic!("Unexpected top_n_tokens"), + } + + validation + .validate(GenerateRequest { + inputs: "Hello".to_string(), + parameters: GenerateParameters { + top_n_tokens: Some(4), + max_new_tokens: 1, + ..default_parameters() + }, + }) + .await + .unwrap(); + + validation + .validate(GenerateRequest { + inputs: "Hello".to_string(), + parameters: GenerateParameters { + top_n_tokens: Some(0), + max_new_tokens: 1, + ..default_parameters() + }, + }) + .await + .unwrap(); + + let valid_request = validation + .validate(GenerateRequest { + inputs: "Hello".to_string(), + parameters: GenerateParameters { + top_n_tokens: None, + max_new_tokens: 1, + ..default_parameters() + }, + }) + .await + .unwrap(); + + assert_eq!(valid_request.top_n_tokens, 0); + } } diff --git a/server/tests/utils/test_tokens.py b/server/tests/utils/test_tokens.py index da0006e4..4187ff25 100644 --- a/server/tests/utils/test_tokens.py +++ b/server/tests/utils/test_tokens.py @@ -1,7 +1,9 @@ +import torch from text_generation_server.utils.tokens import ( StopSequenceCriteria, StoppingCriteria, FinishReason, + batch_top_tokens, ) @@ -42,3 +44,22 @@ def test_stopping_criteria_max(): assert criteria(1, "") == (False, None) assert criteria(1, "") == (False, None) assert criteria(1, "") == (True, FinishReason.FINISH_REASON_LENGTH) + +def test_batch_top_tokens(): + top_n_tokens = [0, 2, 3, 4, 5] + top_n_tokens_tensor = torch.tensor(top_n_tokens) + inp_logprobs = torch.tensor([[-1., -3., -4., -2., -3.]] * 5) + + topn_tok_ids, topn_tok_logprobs = batch_top_tokens(top_n_tokens, top_n_tokens_tensor, inp_logprobs) + + assert topn_tok_ids[0] == [] + assert topn_tok_ids[1] == [0, 3] + assert topn_tok_ids[2] == [0, 3, 1, 4] + assert topn_tok_ids[3] == [0, 3, 1, 4] + assert topn_tok_ids[4] == [0, 3, 1, 4, 2] + + assert topn_tok_logprobs[0] == [] + assert topn_tok_logprobs[1] == [-1, -2] + assert topn_tok_logprobs[2] == [-1, -2, -3, -3] + assert topn_tok_logprobs[3] == [-1, -2, -3, -3] + assert topn_tok_logprobs[4] == [-1, -2, -3, -3, -4] diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index cbdf4808..4e338263 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -1,3 +1,4 @@ +from text_generation_server.utils.tokens import batch_top_tokens import torch import inspect @@ -12,6 +13,7 @@ PrefillTokens, Generation, GeneratedText, + TopTokens, ) from text_generation_server.pb import generate_pb2 from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -42,6 +44,8 @@ class CausalLMBatch(Batch): # Generation helpers next_token_choosers: List[NextTokenChooser] stopping_criterias: List[StoppingCriteria] + top_n_tokens: List[int] + top_n_tokens_tensor: torch.Tensor # Metadata used for padding max_input_length: int @@ -72,6 +76,7 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] + top_n_tokens = [] prefix_offsets = [] read_offsets = [] requests_idx_mapping = {} @@ -88,6 +93,7 @@ def from_pb( r.stopping_parameters, tokenizer ) stopping_criterias.append(stopping_criteria) + top_n_tokens.append(r.top_n_tokens) max_truncation = max(max_truncation, r.truncate) max_decode_tokens += stopping_criteria.max_new_tokens padding_right_offset = max( @@ -121,6 +127,9 @@ def from_pb( position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) + top_n_tokens_tensor = torch.tensor( + top_n_tokens, device=device, dtype=torch.int64 + ) max_tokens = len(inputs) * (max_input_length + max_decode_tokens) @@ -138,6 +147,8 @@ def from_pb( read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, max_input_length=max_input_length.item(), padding_right_offset=padding_right_offset, max_tokens=max_tokens, @@ -163,6 +174,7 @@ def filter(self, request_ids: List[int]) -> Optional["CausalLMBatch"]: next_token_choosers = [] stopping_criterias = [] + top_n_tokens = [] total_remaining_decode_tokens = 0 new_padding_right_offset = 0 @@ -184,6 +196,7 @@ def filter(self, request_ids: List[int]) -> Optional["CausalLMBatch"]: next_token_choosers.append(self.next_token_choosers[idx]) stopping_criteria = self.stopping_criterias[idx] stopping_criterias.append(stopping_criteria) + top_n_tokens.append(self.top_n_tokens[idx]) remaining_decode_tokens = ( stopping_criteria.max_new_tokens - stopping_criteria.current_tokens ) @@ -223,6 +236,7 @@ def filter(self, request_ids: List[int]) -> Optional["CausalLMBatch"]: layer[1] = past_values[keep_indices, :, -past_kv_length:, :] del past_values + top_n_tokens_tensor = self.top_n_tokens_tensor[keep_indices] max_tokens = len(request_ids) * max_input_length + total_remaining_decode_tokens self.requests = requests @@ -235,6 +249,8 @@ def filter(self, request_ids: List[int]) -> Optional["CausalLMBatch"]: self.read_offsets = read_offsets self.next_token_choosers = next_token_choosers self.stopping_criterias = stopping_criterias + self.top_n_tokens = top_n_tokens + self.top_n_tokens_tensor = top_n_tokens_tensor self.max_input_length = max_input_length self.padding_right_offset = new_padding_right_offset self.max_tokens = max_tokens @@ -262,6 +278,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": all_input_ids = [] next_token_choosers = [] stopping_criterias = [] + top_n_tokens = [] max_tokens = 0 # Batch tensors @@ -269,6 +286,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": attention_mask = None position_ids = None past_key_values = [] + top_n_tokens_tensor = None # Used for slicing correctly inside the tensors # Equivalent to a cumsum on batch sizes @@ -281,6 +299,7 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": all_input_ids.extend(batch.all_input_ids) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) + top_n_tokens.extend(batch.top_n_tokens) if i == 0: requests_idx_mapping = batch.requests_idx_mapping @@ -310,6 +329,12 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": (total_batch_size, max_input_length + padding_right_offset), ) + if top_n_tokens_tensor is None: + top_n_tokens_tensor = batches[0].top_n_tokens_tensor.new_zeros( + total_batch_size, + ) + top_n_tokens_tensor[start_index:end_index] = batch.top_n_tokens_tensor + # We need to slice the attention mask to remove padding from previous steps # and to remove unused allocated space left_offset = max_input_length - batch.max_input_length @@ -438,6 +463,8 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, max_input_length=max_input_length, padding_right_offset=padding_right_offset, keys_head_dim_last=batches[0].keys_head_dim_last, @@ -549,6 +576,12 @@ def generate_token( generations: List[Generation] = [] stopped = True + batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( + batch.top_n_tokens, + batch.top_n_tokens_tensor, + torch.softmax(logits[:, -1], -1), + ) + # Zipped iterator iterator = zip( batch.requests, @@ -559,6 +592,9 @@ def generate_token( batch.next_token_choosers, batch.stopping_criterias, batch.all_input_ids, + batch.top_n_tokens, + batch_top_token_ids, + batch_top_token_logprobs, ) # For each member of the batch @@ -571,6 +607,9 @@ def generate_token( next_token_chooser, stopping_criteria, all_input_ids, + top_n_tokens, + top_token_ids, + top_token_logprobs, ) in enumerate(iterator): # Select next token next_token_id, logprobs = next_token_chooser( @@ -637,6 +676,24 @@ def generate_token( else: prefill_tokens = None + if top_n_tokens > 0: + toptoken_texts = self.tokenizer.batch_decode( + top_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + special_toptokens = [ + token_id in self.all_special_ids for token_id in top_token_ids + ] + top_tokens = TopTokens( + top_token_ids, + top_token_logprobs, + toptoken_texts, + special_toptokens, + ) + else: + top_tokens = None + generation = Generation( request.id, prefill_tokens, @@ -645,6 +702,7 @@ def generate_token( next_token_text, next_token_id_squeezed.item() in self.all_special_ids, generated_text, + top_tokens, ) generations.append(generation) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 7de51358..d6af07f4 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -1,5 +1,6 @@ import math import itertools +from text_generation_server.utils.tokens import batch_top_tokens import torch import torch.distributed @@ -16,6 +17,7 @@ PrefillTokens, Generation, GeneratedText, + TopTokens, ) from text_generation_server.pb import generate_pb2 from text_generation_server.utils import StoppingCriteria, HeterogeneousNextTokenChooser @@ -165,6 +167,8 @@ class FlashCausalLMBatch(Batch): # Generation helpers next_token_chooser: HeterogeneousNextTokenChooser stopping_criterias: List[StoppingCriteria] + top_n_tokens: List[int] + top_n_tokens_tensor: torch.Tensor # Number of blocks in this batch blocks: int @@ -217,6 +221,7 @@ def from_pb( next_token_chooser_parameters = [] stopping_criterias = [] + top_n_tokens = [] # Cumulative length cumulative_length = 0 @@ -259,6 +264,7 @@ def from_pb( ) max_new_tokens = stopping_criteria.max_new_tokens stopping_criterias.append(stopping_criteria) + top_n_tokens.append(r.top_n_tokens) # Paged attention # Remove one as the first token des not have a past @@ -352,6 +358,9 @@ def from_pb( prefill_next_token_indices = torch.tensor( prefill_next_token_indices, dtype=torch.int64, device=device ) + top_n_tokens_tensor = torch.tensor( + top_n_tokens, device=device, dtype=torch.int64 + ) return cls( batch_id=pb.id, @@ -378,6 +387,8 @@ def from_pb( all_input_ids_tensor=all_input_ids_tensor, next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, blocks=blocks, max_blocks=max_blocks, ) @@ -417,6 +428,7 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": read_offsets = [] stopping_criterias = [] + top_n_tokens = [] blocks = 0 max_blocks = 0 @@ -443,6 +455,8 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": stopping_criteria = self.stopping_criterias[idx] stopping_criterias.append(stopping_criteria) + top_n_tokens.append(self.top_n_tokens[idx]) + remaining_tokens = ( stopping_criteria.max_new_tokens - stopping_criteria.current_tokens ) @@ -487,6 +501,7 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": input_lengths_tensor = self.input_lengths_tensor[indices] slots = self.slots[slot_filtering_indices] next_token_chooser = self.next_token_chooser.filter(indices) + top_n_tokens_tensor = self.top_n_tokens_tensor[indices] start_slots = torch.tensor(start_slots, dtype=torch.int64) @@ -518,6 +533,8 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": all_input_ids_tensor=all_input_ids_tensor, next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, blocks=blocks, max_blocks=max_blocks, ) @@ -566,6 +583,9 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch all_input_ids_tensor = batches[0].all_input_ids_tensor.new_zeros( (total_batch_size, max_length) ) + top_n_tokens_tensor = batches[0].top_n_tokens_tensor.new_zeros( + total_batch_size, + ) start_slots = [] block_tables = [] @@ -577,6 +597,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch next_token_chooser_parameters = [] stopping_criterias = [] + top_n_tokens = [] # Cumulative length cumulative_batch_size = 0 @@ -602,6 +623,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch position_ids[start_index:end_index] = batch.position_ids slot_indices[start_index:end_index] = batch.slot_indices + cumulative_slots input_lengths_tensor[start_index:end_index] = batch.input_lengths_tensor + top_n_tokens_tensor[start_index:end_index] = batch.top_n_tokens_tensor slots[slots_start_index:slots_end_index] = batch.slots all_input_ids_tensor[ @@ -624,6 +646,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch next_token_chooser_parameters.extend([r.parameters for r in batch.requests]) stopping_criterias.extend(batch.stopping_criterias) + top_n_tokens.extend(batch.top_n_tokens) + # Update cumulative_batch_size += len(batch) cumulative_slots += len(batch.slots) @@ -666,6 +690,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch all_input_ids_tensor=all_input_ids_tensor, next_token_chooser=next_token_chooser, stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, blocks=blocks, max_blocks=max_blocks, ) @@ -831,10 +857,14 @@ def generate_token( else: next_token_logits = out - next_input_ids, next_token_logprobs = batch.next_token_chooser( + next_input_ids, next_token_logprobs, logprobs = batch.next_token_chooser( batch.all_input_ids_tensor[:, : batch.max_seqlen], next_token_logits ) + batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( + batch.top_n_tokens, batch.top_n_tokens_tensor, logprobs + ) + if prefill: if len(batch) > 1 and prefill_logprobs: # We create the prefill_tokens_indices tensor that will be used to gather prefill logprobs @@ -931,8 +961,11 @@ def generate_token( batch.all_input_ids, batch.next_token_chooser.do_sample, batch.next_token_chooser.seeds, + batch.top_n_tokens, next_token_ids, next_token_logprobs, + batch_top_token_ids, + batch_top_token_logprobs, ) # For each member of the batch @@ -945,8 +978,11 @@ def generate_token( all_input_ids, do_sample, seed, + top_n_tokens, next_token_id, next_token_logprob, + top_token_ids, + top_token_logprobs, ) in enumerate(iterator): # Append next token to all tokens all_input_ids.append(next_token_id) @@ -1005,6 +1041,24 @@ def generate_token( else: prefill_tokens = None + if top_n_tokens > 0: + toptoken_texts = self.tokenizer.batch_decode( + top_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + special_toptokens = [ + token_id in self.all_special_ids for token_id in top_token_ids + ] + top_tokens = TopTokens( + top_token_ids, + top_token_logprobs, + toptoken_texts, + special_toptokens, + ) + else: + top_tokens = None + generation = Generation( request.id, prefill_tokens, @@ -1013,6 +1067,7 @@ def generate_token( next_token_text, next_token_id in self.all_special_ids, generated_text, + top_tokens, ) generations.append(generation) diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py index faad63ba..2dac87bc 100644 --- a/server/text_generation_server/models/idefics_causal_lm.py +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -763,6 +763,8 @@ def generate_token( else: prefill_tokens = None + top_tokens=None + generation = Generation( request.id, prefill_tokens, @@ -771,6 +773,7 @@ def generate_token( next_token_text, next_token_id_squeezed.item() in self.all_special_ids, generated_text, + top_tokens ) generations.append(generation) diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 9e5c21d1..361453fb 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -1,3 +1,4 @@ +from text_generation_server.utils.tokens import batch_top_tokens import torch from dataclasses import dataclass @@ -11,6 +12,7 @@ Batch, Generation, PrefillTokens, + TopTokens, ) from text_generation_server.pb import generate_pb2 from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -48,6 +50,8 @@ class Seq2SeqLMBatch(Batch): # Generation helpers next_token_choosers: List[NextTokenChooser] stopping_criterias: List[StoppingCriteria] + top_n_tokens: List[int] + top_n_tokens_tensor: torch.Tensor # Metadata used for padding max_input_length: int @@ -78,7 +82,7 @@ def from_pb( inputs = [] next_token_choosers = [] stopping_criterias = [] - + top_n_tokens = [] decoder_input_lengths = [] prefix_offsets = [] read_offsets = [] @@ -97,6 +101,7 @@ def from_pb( r.stopping_parameters, tokenizer ) stopping_criterias.append(stopping_criteria) + top_n_tokens.append(r.top_n_tokens) max_truncation = max(max_truncation, r.truncate) max_decode_tokens += stopping_criteria.max_new_tokens padding_right_offset = max( @@ -126,6 +131,9 @@ def from_pb( prefix_offsets.append(0) read_offsets.append(1) all_decoder_input_ids = decoder_input_ids.view(-1).split(1) + top_n_tokens_tensor = torch.tensor( + top_n_tokens, device=device, dtype=torch.int64 + ) max_tokens = len(inputs) * (max_input_length + max_decode_tokens) @@ -146,6 +154,8 @@ def from_pb( read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, max_input_length=max_input_length.item(), max_decoder_input_length=1, padding_right_offset=padding_right_offset, @@ -173,6 +183,7 @@ def filter(self, request_ids: List[int]) -> Optional["Seq2SeqLMBatch"]: next_token_choosers = [] stopping_criterias = [] + top_n_tokens = [] max_input_length = 0 max_decoder_input_length = 0 @@ -204,6 +215,7 @@ def filter(self, request_ids: List[int]) -> Optional["Seq2SeqLMBatch"]: next_token_choosers.append(self.next_token_choosers[idx]) stopping_criteria = self.stopping_criterias[idx] stopping_criterias.append(stopping_criteria) + top_n_tokens.append(self.top_n_tokens[idx]) remaining_decode_tokens = ( stopping_criteria.max_new_tokens - stopping_criteria.current_tokens ) @@ -239,6 +251,7 @@ def filter(self, request_ids: List[int]) -> Optional["Seq2SeqLMBatch"]: layer[2] = layer[2][keep_indices, :, -max_input_length:] layer[3] = layer[3][keep_indices, :, -max_input_length:] + top_n_tokens_tensor = self.top_n_tokens_tensor[keep_indices] max_tokens = ( len(request_ids) * (max_input_length + max_decoder_input_length) + remaining_decode_tokens @@ -254,6 +267,8 @@ def filter(self, request_ids: List[int]) -> Optional["Seq2SeqLMBatch"]: self.read_offsets = read_offsets self.next_token_choosers = next_token_choosers self.stopping_criterias = stopping_criterias + self.top_n_tokens = top_n_tokens + self.top_n_tokens_tensor = top_n_tokens_tensor self.max_input_length = max_input_length self.max_decoder_input_length = max_decoder_input_length self.padding_right_offset = padding_right_offset @@ -289,6 +304,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": read_offsets = [] next_token_choosers = [] stopping_criterias = [] + top_n_tokens = [] max_tokens = 0 # Batch tensors @@ -296,6 +312,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": decoder_input_ids = None decoder_attention_mask = None encoder_last_hidden_state = None + top_n_tokens_tensor = None past_key_values = [] # Used for slicing correctly inside the tensors @@ -312,6 +329,7 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": read_offsets.extend(batch.read_offsets) next_token_choosers.extend(batch.next_token_choosers) stopping_criterias.extend(batch.stopping_criterias) + top_n_tokens.extend(batch.top_n_tokens) if i == 0: requests_idx_mapping = batch.requests_idx_mapping @@ -384,6 +402,12 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": ), ) + if top_n_tokens_tensor is None: + top_n_tokens_tensor = batches[0].top_n_tokens_tensor.new_zeros( + total_batch_size, + ) + top_n_tokens_tensor[start_index:end_index] = batch.top_n_tokens_tensor + # Copy to correct indices encoder_last_hidden_state[ start_index:end_index, -batch.max_input_length :, : @@ -488,6 +512,8 @@ def concatenate(cls, batches: List["Seq2SeqLMBatch"]) -> "Seq2SeqLMBatch": read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, max_input_length=max_input_length, max_decoder_input_length=max_decoder_input_length, padding_right_offset=padding_right_offset, @@ -613,6 +639,12 @@ def generate_token( batch.past_key_values, ) + batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( + batch.top_n_tokens, + batch.top_n_tokens_tensor, + torch.softmax(logits[:, -1], -1), + ) + # Finished requests generations: List[Generation] = [] stopped = True @@ -628,6 +660,9 @@ def generate_token( batch.next_token_choosers, batch.stopping_criterias, batch.all_decoder_input_ids, + batch.top_n_tokens, + batch_top_token_ids, + batch_top_token_logprobs, ) # For each member of the batch @@ -641,6 +676,9 @@ def generate_token( next_token_chooser, stopping_criteria, all_decoder_input_ids, + top_n_tokens, + top_token_ids, + top_token_logprobs, ) in enumerate(iterator): # Select next token next_token_id, logprobs = next_token_chooser( @@ -698,6 +736,24 @@ def generate_token( else: prefill_tokens = None + if top_n_tokens > 0: + toptoken_texts = self.tokenizer.batch_decode( + top_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + special_toptokens = [ + token_id in self.all_special_ids for token_id in top_token_ids + ] + top_tokens = TopTokens( + top_token_ids, + top_token_logprobs, + toptoken_texts, + special_toptokens, + ) + else: + top_tokens = None + generation = Generation( request.id, prefill_tokens, @@ -706,6 +762,7 @@ def generate_token( next_token_text, next_token_id_squeezed.item() in self.all_special_ids, generated_text, + top_tokens, ) generations.append(generation) diff --git a/server/text_generation_server/models/types.py b/server/text_generation_server/models/types.py index 28ca8147..0e27680d 100644 --- a/server/text_generation_server/models/types.py +++ b/server/text_generation_server/models/types.py @@ -1,3 +1,4 @@ +from functools import total_ordering import torch from abc import ABC, abstractmethod @@ -71,6 +72,25 @@ def __len__(self): return len(self.token_ids) +@dataclass +class TopTokens: + token_ids: List[int] + logprobs: List[float] + texts: List[str] + is_special: List[bool] + + def to_pb(self) -> generate_pb2.TopTokens: + return generate_pb2.TopTokens( + ids=self.token_ids, + logprobs=self.logprobs, + texts=self.texts, + is_special=self.is_special, + ) + + def __len__(self): + return len(self.token_ids) + + @dataclass class Generation: request_id: int @@ -80,6 +100,8 @@ class Generation: token_text: str token_is_special: bool generated_text: Optional[GeneratedText] + # Optional for now, since it's not yet supported for every model. + top_tokens: Optional[TopTokens] def to_pb(self) -> generate_pb2.Generation: return generate_pb2.Generation( @@ -94,4 +116,5 @@ def to_pb(self) -> generate_pb2.Generation: generated_text=self.generated_text.to_pb() if self.generated_text is not None else None, + top_tokens=self.top_tokens.to_pb() if self.top_tokens is not None else None, ) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index b83af591..69177d56 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -1,24 +1,20 @@ import re -import torch - -from transformers import ( - RepetitionPenaltyLogitsProcessor, - PreTrainedTokenizerBase, -) -from typing import List, Tuple, Optional +from typing import Callable, List, Optional, Tuple +import torch from text_generation_server.pb import generate_pb2 from text_generation_server.pb.generate_pb2 import FinishReason -from text_generation_server.utils.watermark import WatermarkLogitsProcessor from text_generation_server.utils.logits_process import ( - static_warper, + HeterogeneousProcessorWrapper, HeterogeneousRepetitionPenaltyLogitsProcessor, HeterogeneousTemperatureLogitsWarper, HeterogeneousTopKLogitsWarper, HeterogeneousTopPLogitsWarper, HeterogeneousTypicalLogitsWarper, - HeterogeneousProcessorWrapper, + static_warper, ) +from text_generation_server.utils.watermark import WatermarkLogitsProcessor +from transformers import PreTrainedTokenizerBase, RepetitionPenaltyLogitsProcessor class NextTokenChooser: @@ -229,11 +225,10 @@ def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor): scores = warper(input_ids, scores) next_ids = self.choice(scores) - next_logprobs = torch.gather( - torch.log_softmax(scores, -1), 1, next_ids.view(-1, 1) - ).view(-1) + logprobs = torch.log_softmax(scores, -1) + next_logprobs = torch.gather(logprobs, 1, next_ids.view(-1, 1)).view(-1) - return next_ids, next_logprobs + return next_ids, next_logprobs, logprobs def filter(self, indices): if self.watermark_processor is not None: @@ -339,3 +334,50 @@ def filter(self, indices): self.greedy_indices = new_greedy_indices self.sampling_mapping = new_sampling_mapping return self + + +def batch_top_tokens( + top_n_tokens: list[int], top_n_tokens_tensor: torch.Tensor, logprobs: torch.Tensor +) -> Tuple[List[List[int]], List[List[float]]]: + """Find the top n most likely tokens for a batch of generations. + + When multiple tokens have equal probabilities and they don't all fit, the + remaining tokens are also returned. + """ + max_top_n = max(top_n_tokens) + # Early exit when top_n_tokens is not used + if max_top_n == 0: + return [[]] * len(top_n_tokens), [[]] * len(top_n_tokens) + + # Ensure top_n doesn't exceed vocab size + top_n_tokens = [min(tok, logprobs.size(-1)) for tok in top_n_tokens] + + # Parallel kthvalue adapted from https://discuss.pytorch.org/t/how-to-efficiently-get-the-k-th-largest-values-in-parallel/160529/2 + # Sorted topk is faster than torch.sort() since we only need a small subset + sorted_top_k = torch.topk(logprobs, k=max_top_n, dim=1, sorted=True).values + nth_highest = torch.gather( + sorted_top_k, 1, (top_n_tokens_tensor - 1).clip(min=0).unsqueeze(1) + ) + nth_highest[nth_highest == -float("inf")] = torch.finfo(logprobs.dtype).min + + # Find the new "fuzzy" top n values + top_n_indices = (logprobs >= nth_highest).nonzero() + _, top_n_ishes = torch.unique_consecutive(top_n_indices[:, 0], return_counts=True) + + # Take a new topk for these new max n values + top_k = torch.topk(logprobs, k=top_n_ishes.max(), dim=1, sorted=True) + + top_n_ishes = top_n_ishes.tolist() + top_indices = top_k.indices.tolist() + top_values = top_k.values.tolist() + + return ( + [ + idxs[:n] if req_n > 0 else [] + for idxs, n, req_n in zip(top_indices, top_n_ishes, top_n_tokens) + ], + [ + vals[:n] if req_n > 0 else [] + for vals, n, req_n in zip(top_values, top_n_ishes, top_n_tokens) + ], + ) From 5485c142e87a182f2eee713a6b056ee38bd901f5 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 29 Aug 2023 14:28:22 +0200 Subject: [PATCH 398/793] New release. (#941) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- Cargo.toml | 2 +- .../test_idefics/test_idefics.json | 69 +- .../test_idefics/test_idefics_load.json | 284 ++--- integration-tests/poetry.lock | 982 ++++++++++++++++++ integration-tests/pyproject.toml | 13 + integration-tests/requirements.txt | 38 +- server/poetry.lock | 334 +++--- server/pyproject.toml | 6 +- server/requirements.txt | 38 +- 9 files changed, 1427 insertions(+), 339 deletions(-) create mode 100644 integration-tests/poetry.lock create mode 100644 integration-tests/pyproject.toml diff --git a/Cargo.toml b/Cargo.toml index 1cbac920..9f526b27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.0.1" +version = "1.0.3" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json index 1df5761d..0edd81b6 100644 --- a/integration-tests/models/__snapshots__/test_idefics/test_idefics.json +++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json @@ -11,52 +11,52 @@ }, { "id": 4911, - "logprob": -5.3632812, + "logprob": -5.7773438, "text": "User" }, { "id": 29901, - "logprob": -0.00762558, + "logprob": -0.0069999695, "text": ":" }, { "id": 32000, - "logprob": -0.7739258, + "logprob": -0.8125, "text": "" }, { "id": 32001, - "logprob": -9.775162e-05, + "logprob": -6.651878e-05, "text": "" }, { "id": 32000, - "logprob": -1.1920929e-07, + "logprob": -3.5762787e-07, "text": "" }, { "id": 1815, - "logprob": -4.4140625, + "logprob": -4.2265625, "text": "Can" }, { "id": 366, - "logprob": -0.01436615, + "logprob": -0.013977051, "text": "you" }, { "id": 2649, - "logprob": -4.9414062, + "logprob": -4.4375, "text": "tell" }, { "id": 592, - "logprob": -0.3005371, + "logprob": -0.29077148, "text": "me" }, { "id": 263, - "logprob": -3.5703125, + "logprob": -4.2109375, "text": "a" }, { @@ -66,37 +66,37 @@ }, { "id": 3273, - "logprob": -1.9111328, + "logprob": -1.8671875, "text": "short" }, { "id": 5828, - "logprob": -0.28881836, + "logprob": -0.26586914, "text": "story" }, { "id": 2729, - "logprob": -3.4179688, + "logprob": -3.7460938, "text": "based" }, { "id": 373, - "logprob": -0.00056886673, + "logprob": -0.0005350113, "text": "on" }, { "id": 278, - "logprob": -0.14123535, + "logprob": -0.13867188, "text": "the" }, { "id": 1967, - "logprob": -0.053985596, + "logprob": -0.06842041, "text": "image" }, { "id": 29973, - "logprob": -0.15771484, + "logprob": -0.15319824, "text": "?" } ], @@ -104,25 +104,25 @@ "tokens": [ { "id": 32002, - "logprob": -0.004295349, + "logprob": -0.0019445419, "special": true, "text": "" }, { "id": 29871, - "logprob": -7.43866e-05, + "logprob": -8.404255e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -2.3126602e-05, + "logprob": -1.7881393e-05, "special": false, "text": "\n" }, { "id": 7900, - "logprob": -3.9339066e-06, + "logprob": -3.0994415e-06, "special": false, "text": "Ass" }, @@ -134,35 +134,36 @@ }, { "id": 29901, - "logprob": -2.6226044e-06, + "logprob": -3.2186508e-06, "special": false, "text": ":" }, { "id": 319, - "logprob": -0.87841797, + "logprob": -0.9057617, "special": false, "text": " A" }, { - "id": 521, - "logprob": -1.3837891, + "id": 696, + "logprob": -1.2314453, "special": false, - "text": " ch" + "text": " ro" }, { - "id": 21475, - "logprob": -0.00051641464, + "id": 15664, + "logprob": -0.00024914742, "special": false, - "text": "icken" + "text": "oster" }, { - "id": 338, - "logprob": -1.1435547, + "id": 15028, + "logprob": -1.1621094, "special": false, - "text": " is" + "text": " stands" } - ] + ], + "top_tokens": null }, - "generated_text": "\nAssistant: A chicken is" + "generated_text": "\nAssistant: A rooster stands" } diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json index e0dfe96c..81cc1b19 100644 --- a/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json +++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json @@ -12,92 +12,92 @@ }, { "id": 4911, - "logprob": -5.3476562, + "logprob": -5.7773438, "text": "User" }, { "id": 29901, - "logprob": -0.0075531006, + "logprob": -0.0069999695, "text": ":" }, { "id": 32000, - "logprob": -0.7729492, + "logprob": -0.8125, "text": "" }, { "id": 32001, - "logprob": -9.787083e-05, + "logprob": -6.651878e-05, "text": "" }, { "id": 32000, - "logprob": -1.1920929e-07, + "logprob": -3.5762787e-07, "text": "" }, { "id": 1815, - "logprob": -4.4296875, + "logprob": -4.2265625, "text": "Can" }, { "id": 366, - "logprob": -0.01424408, + "logprob": -0.013977051, "text": "you" }, { "id": 2649, - "logprob": -4.9335938, + "logprob": -4.4375, "text": "tell" }, { "id": 592, - "logprob": -0.2993164, + "logprob": -0.29077148, "text": "me" }, { "id": 263, - "logprob": -3.5664062, + "logprob": -4.2109375, "text": "a" }, { "id": 1407, - "logprob": -9.4453125, + "logprob": -9.4296875, "text": "very" }, { "id": 3273, - "logprob": -1.9306641, + "logprob": -1.8671875, "text": "short" }, { "id": 5828, - "logprob": -0.2836914, + "logprob": -0.26586914, "text": "story" }, { "id": 2729, - "logprob": -3.4179688, + "logprob": -3.7460938, "text": "based" }, { "id": 373, - "logprob": -0.00056934357, + "logprob": -0.0005350113, "text": "on" }, { "id": 278, - "logprob": -0.13928223, + "logprob": -0.13867188, "text": "the" }, { "id": 1967, - "logprob": -0.05355835, + "logprob": -0.06842041, "text": "image" }, { "id": 29973, - "logprob": -0.15771484, + "logprob": -0.15319824, "text": "?" } ], @@ -105,25 +105,25 @@ "tokens": [ { "id": 32002, - "logprob": -0.004333496, + "logprob": -0.0019445419, "special": true, "text": "" }, { "id": 29871, - "logprob": -7.426739e-05, + "logprob": -8.416176e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -2.348423e-05, + "logprob": -1.7881393e-05, "special": false, "text": "\n" }, { "id": 7900, - "logprob": -3.9339066e-06, + "logprob": -3.0994415e-06, "special": false, "text": "Ass" }, @@ -135,37 +135,38 @@ }, { "id": 29901, - "logprob": -2.861023e-06, + "logprob": -3.2186508e-06, "special": false, "text": ":" }, { "id": 319, - "logprob": -0.8828125, + "logprob": -0.89941406, "special": false, "text": " A" }, { - "id": 521, - "logprob": -1.3759766, + "id": 696, + "logprob": -1.234375, "special": false, - "text": " ch" + "text": " ro" }, { - "id": 21475, - "logprob": -0.0005083084, + "id": 15664, + "logprob": -0.0002465248, "special": false, - "text": "icken" + "text": "oster" }, { - "id": 338, - "logprob": -1.1367188, + "id": 15028, + "logprob": -1.1660156, "special": false, - "text": " is" + "text": " stands" } - ] + ], + "top_tokens": null }, - "generated_text": "\nAssistant: A chicken is" + "generated_text": "\nAssistant: A rooster stands" }, { "details": { @@ -180,92 +181,92 @@ }, { "id": 4911, - "logprob": -5.3476562, + "logprob": -5.7890625, "text": "User" }, { "id": 29901, - "logprob": -0.0075531006, + "logprob": -0.0070152283, "text": ":" }, { "id": 32000, - "logprob": -0.7729492, + "logprob": -0.8125, "text": "" }, { "id": 32001, - "logprob": -9.787083e-05, + "logprob": -6.651878e-05, "text": "" }, { "id": 32000, - "logprob": -1.1920929e-07, + "logprob": -3.5762787e-07, "text": "" }, { "id": 1815, - "logprob": -4.4296875, + "logprob": -4.2265625, "text": "Can" }, { "id": 366, - "logprob": -0.01423645, + "logprob": -0.014190674, "text": "you" }, { "id": 2649, - "logprob": -4.9335938, + "logprob": -4.4140625, "text": "tell" }, { "id": 592, - "logprob": -0.2993164, + "logprob": -0.2919922, "text": "me" }, { "id": 263, - "logprob": -3.5664062, + "logprob": -4.2109375, "text": "a" }, { "id": 1407, - "logprob": -9.4453125, + "logprob": -9.4375, "text": "very" }, { "id": 3273, - "logprob": -1.9306641, + "logprob": -1.8720703, "text": "short" }, { "id": 5828, - "logprob": -0.2836914, + "logprob": -0.26904297, "text": "story" }, { "id": 2729, - "logprob": -3.4179688, + "logprob": -3.7675781, "text": "based" }, { "id": 373, - "logprob": -0.00056934357, + "logprob": -0.0005402565, "text": "on" }, { "id": 278, - "logprob": -0.13928223, + "logprob": -0.13867188, "text": "the" }, { "id": 1967, - "logprob": -0.05355835, + "logprob": -0.068359375, "text": "image" }, { "id": 29973, - "logprob": -0.15771484, + "logprob": -0.15539551, "text": "?" } ], @@ -273,25 +274,25 @@ "tokens": [ { "id": 32002, - "logprob": -0.004333496, + "logprob": -0.0019168854, "special": true, "text": "" }, { "id": 29871, - "logprob": -7.4505806e-05, + "logprob": -8.392334e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -2.3722649e-05, + "logprob": -1.7642975e-05, "special": false, "text": "\n" }, { "id": 7900, - "logprob": -3.9339066e-06, + "logprob": -3.0994415e-06, "special": false, "text": "Ass" }, @@ -303,37 +304,38 @@ }, { "id": 29901, - "logprob": -2.861023e-06, + "logprob": -3.0994415e-06, "special": false, "text": ":" }, { "id": 319, - "logprob": -0.8828125, + "logprob": -0.90722656, "special": false, "text": " A" }, { - "id": 521, - "logprob": -1.3759766, + "id": 696, + "logprob": -1.2373047, "special": false, - "text": " ch" + "text": " ro" }, { - "id": 21475, - "logprob": -0.00050878525, + "id": 15664, + "logprob": -0.00024938583, "special": false, - "text": "icken" + "text": "oster" }, { - "id": 338, - "logprob": -1.1367188, + "id": 15028, + "logprob": -1.1708984, "special": false, - "text": " is" + "text": " stands" } - ] + ], + "top_tokens": null }, - "generated_text": "\nAssistant: A chicken is" + "generated_text": "\nAssistant: A rooster stands" }, { "details": { @@ -348,92 +350,92 @@ }, { "id": 4911, - "logprob": -5.3476562, + "logprob": -5.7890625, "text": "User" }, { "id": 29901, - "logprob": -0.0075531006, + "logprob": -0.0070152283, "text": ":" }, { "id": 32000, - "logprob": -0.7729492, + "logprob": -0.8125, "text": "" }, { "id": 32001, - "logprob": -9.775162e-05, + "logprob": -6.663799e-05, "text": "" }, { "id": 32000, - "logprob": -1.1920929e-07, + "logprob": -3.5762787e-07, "text": "" }, { "id": 1815, - "logprob": -4.4296875, + "logprob": -4.2265625, "text": "Can" }, { "id": 366, - "logprob": -0.01424408, + "logprob": -0.014190674, "text": "you" }, { "id": 2649, - "logprob": -4.9335938, + "logprob": -4.4140625, "text": "tell" }, { "id": 592, - "logprob": -0.2993164, + "logprob": -0.2919922, "text": "me" }, { "id": 263, - "logprob": -3.5664062, + "logprob": -4.2109375, "text": "a" }, { "id": 1407, - "logprob": -9.4453125, + "logprob": -9.4375, "text": "very" }, { "id": 3273, - "logprob": -1.9306641, + "logprob": -1.8720703, "text": "short" }, { "id": 5828, - "logprob": -0.2836914, + "logprob": -0.26904297, "text": "story" }, { "id": 2729, - "logprob": -3.4179688, + "logprob": -3.7675781, "text": "based" }, { "id": 373, - "logprob": -0.00056934357, + "logprob": -0.0005402565, "text": "on" }, { "id": 278, - "logprob": -0.13928223, + "logprob": -0.13867188, "text": "the" }, { "id": 1967, - "logprob": -0.05355835, + "logprob": -0.068359375, "text": "image" }, { "id": 29973, - "logprob": -0.15771484, + "logprob": -0.15539551, "text": "?" } ], @@ -441,25 +443,25 @@ "tokens": [ { "id": 32002, - "logprob": -0.004333496, + "logprob": -0.0019168854, "special": true, "text": "" }, { "id": 29871, - "logprob": -7.43866e-05, + "logprob": -8.404255e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -2.360344e-05, + "logprob": -1.7642975e-05, "special": false, "text": "\n" }, { "id": 7900, - "logprob": -3.9339066e-06, + "logprob": -3.0994415e-06, "special": false, "text": "Ass" }, @@ -471,37 +473,38 @@ }, { "id": 29901, - "logprob": -2.7418137e-06, + "logprob": -3.0994415e-06, "special": false, "text": ":" }, { "id": 319, - "logprob": -0.8828125, + "logprob": -0.90722656, "special": false, "text": " A" }, { - "id": 521, - "logprob": -1.3759766, + "id": 696, + "logprob": -1.2373047, "special": false, - "text": " ch" + "text": " ro" }, { - "id": 21475, - "logprob": -0.00050878525, + "id": 15664, + "logprob": -0.00024938583, "special": false, - "text": "icken" + "text": "oster" }, { - "id": 338, - "logprob": -1.1367188, + "id": 15028, + "logprob": -1.1708984, "special": false, - "text": " is" + "text": " stands" } - ] + ], + "top_tokens": null }, - "generated_text": "\nAssistant: A chicken is" + "generated_text": "\nAssistant: A rooster stands" }, { "details": { @@ -516,92 +519,92 @@ }, { "id": 4911, - "logprob": -5.3632812, + "logprob": -5.7890625, "text": "User" }, { "id": 29901, - "logprob": -0.00762558, + "logprob": -0.0070152283, "text": ":" }, { "id": 32000, - "logprob": -0.7739258, + "logprob": -0.8125, "text": "" }, { "id": 32001, - "logprob": -9.775162e-05, + "logprob": -6.663799e-05, "text": "" }, { "id": 32000, - "logprob": -1.1920929e-07, + "logprob": -3.5762787e-07, "text": "" }, { "id": 1815, - "logprob": -4.4140625, + "logprob": -4.2265625, "text": "Can" }, { "id": 366, - "logprob": -0.01436615, + "logprob": -0.014190674, "text": "you" }, { "id": 2649, - "logprob": -4.9414062, + "logprob": -4.4140625, "text": "tell" }, { "id": 592, - "logprob": -0.3005371, + "logprob": -0.2919922, "text": "me" }, { "id": 263, - "logprob": -3.5703125, + "logprob": -4.2109375, "text": "a" }, { "id": 1407, - "logprob": -9.4296875, + "logprob": -9.4375, "text": "very" }, { "id": 3273, - "logprob": -1.9111328, + "logprob": -1.8720703, "text": "short" }, { "id": 5828, - "logprob": -0.28881836, + "logprob": -0.26904297, "text": "story" }, { "id": 2729, - "logprob": -3.4179688, + "logprob": -3.7675781, "text": "based" }, { "id": 373, - "logprob": -0.00056886673, + "logprob": -0.0005402565, "text": "on" }, { "id": 278, - "logprob": -0.14123535, + "logprob": -0.13867188, "text": "the" }, { "id": 1967, - "logprob": -0.053985596, + "logprob": -0.068359375, "text": "image" }, { "id": 29973, - "logprob": -0.15771484, + "logprob": -0.15539551, "text": "?" } ], @@ -609,25 +612,25 @@ "tokens": [ { "id": 32002, - "logprob": -0.004295349, + "logprob": -0.0019159317, "special": true, "text": "" }, { "id": 29871, - "logprob": -7.43866e-05, + "logprob": -8.404255e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -2.3126602e-05, + "logprob": -1.7642975e-05, "special": false, "text": "\n" }, { "id": 7900, - "logprob": -3.9339066e-06, + "logprob": -3.0994415e-06, "special": false, "text": "Ass" }, @@ -639,36 +642,37 @@ }, { "id": 29901, - "logprob": -2.6226044e-06, + "logprob": -3.0994415e-06, "special": false, "text": ":" }, { "id": 319, - "logprob": -0.87841797, + "logprob": -0.90722656, "special": false, "text": " A" }, { - "id": 521, - "logprob": -1.3837891, + "id": 696, + "logprob": -1.2373047, "special": false, - "text": " ch" + "text": " ro" }, { - "id": 21475, - "logprob": -0.00051641464, + "id": 15664, + "logprob": -0.00024938583, "special": false, - "text": "icken" + "text": "oster" }, { - "id": 338, - "logprob": -1.1435547, + "id": 15028, + "logprob": -1.1708984, "special": false, - "text": " is" + "text": " stands" } - ] + ], + "top_tokens": null }, - "generated_text": "\nAssistant: A chicken is" + "generated_text": "\nAssistant: A rooster stands" } ] diff --git a/integration-tests/poetry.lock b/integration-tests/poetry.lock new file mode 100644 index 00000000..e156c161 --- /dev/null +++ b/integration-tests/poetry.lock @@ -0,0 +1,982 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "aiohttp" +version = "3.8.5" +description = "Async http client/server framework (asyncio)" +optional = false +python-versions = ">=3.6" +files = [ + {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8"}, + {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84"}, + {file = "aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825"}, + {file = "aiohttp-3.8.5-cp310-cp310-win32.whl", hash = "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802"}, + {file = "aiohttp-3.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c"}, + {file = "aiohttp-3.8.5-cp311-cp311-win32.whl", hash = "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945"}, + {file = "aiohttp-3.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755"}, + {file = "aiohttp-3.8.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"}, + {file = "aiohttp-3.8.5-cp36-cp36m-win32.whl", hash = "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e"}, + {file = "aiohttp-3.8.5-cp36-cp36m-win_amd64.whl", hash = "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-win32.whl", hash = "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22"}, + {file = "aiohttp-3.8.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35"}, + {file = "aiohttp-3.8.5-cp38-cp38-win32.whl", hash = "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c"}, + {file = "aiohttp-3.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91"}, + {file = "aiohttp-3.8.5-cp39-cp39-win32.whl", hash = "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67"}, + {file = "aiohttp-3.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c"}, + {file = "aiohttp-3.8.5.tar.gz", hash = "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc"}, +] + +[package.dependencies] +aiosignal = ">=1.1.2" +async-timeout = ">=4.0.0a3,<5.0" +attrs = ">=17.3.0" +charset-normalizer = ">=2.0,<4.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["Brotli", "aiodns", "cchardet"] + +[[package]] +name = "aiosignal" +version = "1.3.1" +description = "aiosignal: a list of registered asynchronous callbacks" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "attrs" +version = "23.1.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + +[[package]] +name = "certifi" +version = "2023.7.22" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.2.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, + {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "colored" +version = "1.4.4" +description = "Simple library for color and formatting to terminal" +optional = false +python-versions = "*" +files = [ + {file = "colored-1.4.4.tar.gz", hash = "sha256:04ff4d4dd514274fe3b99a21bb52fb96f2688c01e93fba7bef37221e7cb56ce0"}, +] + +[[package]] +name = "docker" +version = "6.1.3" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.7" +files = [ + {file = "docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"}, + {file = "docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20"}, +] + +[package.dependencies] +packaging = ">=14.0" +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" +websocket-client = ">=0.32.0" + +[package.extras] +ssh = ["paramiko (>=2.4.3)"] + +[[package]] +name = "exceptiongroup" +version = "1.1.3" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "filelock" +version = "3.12.3" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.12.3-py3-none-any.whl", hash = "sha256:f067e40ccc40f2b48395a80fcbd4728262fab54e232e090a4063ab804179efeb"}, + {file = "filelock-3.12.3.tar.gz", hash = "sha256:0ecc1dd2ec4672a10c8550a8182f1bd0c0a5088470ecd5a125e45f49472fac3d"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.7.1", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "frozenlist" +version = "1.4.0" +description = "A list-like structure which implements collections.abc.MutableSequence" +optional = false +python-versions = ">=3.8" +files = [ + {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab"}, + {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559"}, + {file = "frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62"}, + {file = "frozenlist-1.4.0-cp310-cp310-win32.whl", hash = "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0"}, + {file = "frozenlist-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956"}, + {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95"}, + {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3"}, + {file = "frozenlist-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb"}, + {file = "frozenlist-1.4.0-cp311-cp311-win32.whl", hash = "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431"}, + {file = "frozenlist-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1"}, + {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3"}, + {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503"}, + {file = "frozenlist-1.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8"}, + {file = "frozenlist-1.4.0-cp38-cp38-win32.whl", hash = "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc"}, + {file = "frozenlist-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7"}, + {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf"}, + {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963"}, + {file = "frozenlist-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3"}, + {file = "frozenlist-1.4.0-cp39-cp39-win32.whl", hash = "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f"}, + {file = "frozenlist-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167"}, + {file = "frozenlist-1.4.0.tar.gz", hash = "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251"}, +] + +[[package]] +name = "fsspec" +version = "2023.6.0" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"}, + {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +devel = ["pytest", "pytest-cov"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "huggingface-hub" +version = "0.16.4" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "huggingface_hub-0.16.4-py3-none-any.whl", hash = "sha256:0d3df29932f334fead024afc7cb4cc5149d955238b8b5e42dcf9740d6995a349"}, + {file = "huggingface_hub-0.16.4.tar.gz", hash = "sha256:608c7d4f3d368b326d1747f91523dbd1f692871e8e2e7a4750314a2dd8b63e14"}, +] + +[package.dependencies] +filelock = "*" +fsspec = "*" +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +inference = ["aiohttp", "pydantic"] +quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +torch = ["torch"] +typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pydantic" +version = "1.10.12" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, + {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, + {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, + {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, + {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, + {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, + {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, + {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, + {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, + {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pytest" +version = "7.4.0" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, + {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-asyncio" +version = "0.21.1" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-asyncio-0.21.1.tar.gz", hash = "sha256:40a7eae6dded22c7b604986855ea48400ab15b069ae38116e8c01238e9eeb64d"}, + {file = "pytest_asyncio-0.21.1-py3-none-any.whl", hash = "sha256:8666c1c8ac02631d7c51ba282e0c69a8a452b211ffedf2599099845da5c5c37b"}, +] + +[package.dependencies] +pytest = ">=7.0.0" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] + +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "syrupy" +version = "4.0.1" +description = "Pytest Snapshot Test Utility" +optional = false +python-versions = ">=3.8.1,<4" +files = [ + {file = "syrupy-4.0.1-py3-none-any.whl", hash = "sha256:53d3107cc5e18a5def189c721879cea2cdafdee34b879f602133ca08837d0e4b"}, + {file = "syrupy-4.0.1.tar.gz", hash = "sha256:60e3e94782444e0f978cd3b207de32f6da3199b15a2db32eab02f83cebb63ae8"}, +] + +[package.dependencies] +colored = ">=1.3.92,<2.0.0" +pytest = ">=7.0.0,<8.0.0" + +[[package]] +name = "text-generation" +version = "0.6.0" +description = "Hugging Face Text Generation Python Client" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "text-generation-0.6.0.tar.gz", hash = "sha256:48560e7a67b9a88b38335382d357f66e23b5a75f53971ccd436fc6f696a00815"}, + {file = "text_generation-0.6.0-py3-none-any.whl", hash = "sha256:42ae7f7c9ff11f3a6c9d210f94fe708fe693eede79c6776da727456da1606ef9"}, +] + +[package.dependencies] +aiohttp = ">=3.8,<4.0" +huggingface-hub = ">=0.12,<1.0" +pydantic = ">=1.10,<2.0" + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tqdm" +version = "4.66.1" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, + {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "typing-extensions" +version = "4.7.1" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, +] + +[[package]] +name = "urllib3" +version = "2.0.4" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.7" +files = [ + {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, + {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "websocket-client" +version = "1.6.2" +description = "WebSocket client for Python with low level API options" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websocket-client-1.6.2.tar.gz", hash = "sha256:53e95c826bf800c4c465f50093a8c4ff091c7327023b10bfaff40cf1ef170eaa"}, + {file = "websocket_client-1.6.2-py3-none-any.whl", hash = "sha256:ce54f419dfae71f4bdba69ebe65bf7f0a93fe71bc009ad3a010aacc3eebad537"}, +] + +[package.extras] +docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] +optional = ["python-socks", "wsaccel"] +test = ["websockets"] + +[[package]] +name = "yarl" +version = "1.9.2" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, + {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, + {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, + {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, + {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, + {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, + {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, + {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, + {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, + {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, + {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, + {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, + {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = ">=3.9,<3.13" +content-hash = "bdad1d22d29138010cd6b11e1b92dc0630b35634422413a8456dc85a15bee05e" diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml new file mode 100644 index 00000000..bb881d8e --- /dev/null +++ b/integration-tests/pyproject.toml @@ -0,0 +1,13 @@ +[tool.poetry] +name = "text-generation-integration-tests" +version = "1.0.3" +description = "Text Generation Inference integration tests" +authors = ["Nicolas Patry "] + +[tool.poetry.dependencies] +python = ">=3.9,<3.13" +syrupy = "4.0.1" +text-generation = "^0.6.0" +pytest = "^7.4.0" +pytest-asyncio = "^0.21.1" +docker = "^6.1.3" diff --git a/integration-tests/requirements.txt b/integration-tests/requirements.txt index 2f36d5d6..3f779a90 100644 --- a/integration-tests/requirements.txt +++ b/integration-tests/requirements.txt @@ -1,5 +1,33 @@ -syrupy -text-generation -pytest -pytest-asyncio==0.17.2 -docker \ No newline at end of file +aiohttp==3.8.5 ; python_version >= "3.9" and python_version < "3.13" +aiosignal==1.3.1 ; python_version >= "3.9" and python_version < "3.13" +async-timeout==4.0.3 ; python_version >= "3.9" and python_version < "3.13" +attrs==23.1.0 ; python_version >= "3.9" and python_version < "3.13" +certifi==2023.7.22 ; python_version >= "3.9" and python_version < "3.13" +charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "3.13" +colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows") +colored==1.4.4 ; python_version >= "3.9" and python_version < "3.13" +docker==6.1.3 ; python_version >= "3.9" and python_version < "3.13" +exceptiongroup==1.1.3 ; python_version >= "3.9" and python_version < "3.11" +filelock==3.12.3 ; python_version >= "3.9" and python_version < "3.13" +frozenlist==1.4.0 ; python_version >= "3.9" and python_version < "3.13" +fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "3.13" +huggingface-hub==0.16.4 ; python_version >= "3.9" and python_version < "3.13" +idna==3.4 ; python_version >= "3.9" and python_version < "3.13" +iniconfig==2.0.0 ; python_version >= "3.9" and python_version < "3.13" +multidict==6.0.4 ; python_version >= "3.9" and python_version < "3.13" +packaging==23.1 ; python_version >= "3.9" and python_version < "3.13" +pluggy==1.3.0 ; python_version >= "3.9" and python_version < "3.13" +pydantic==1.10.12 ; python_version >= "3.9" and python_version < "3.13" +pytest-asyncio==0.21.1 ; python_version >= "3.9" and python_version < "3.13" +pytest==7.4.0 ; python_version >= "3.9" and python_version < "3.13" +pywin32==306 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32" +pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" +requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" +syrupy==4.0.1 ; python_version >= "3.9" and python_version < "3.13" +text-generation==0.6.0 ; python_version >= "3.9" and python_version < "3.13" +tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11" +tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" +typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "3.13" +urllib3==2.0.4 ; python_version >= "3.9" and python_version < "3.13" +websocket-client==1.6.2 ; python_version >= "3.9" and python_version < "3.13" +yarl==1.9.2 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/poetry.lock b/server/poetry.lock index aae7b7a5..fd4d427d 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -298,13 +298,13 @@ files = [ [[package]] name = "click" -version = "8.1.6" +version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, - {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] @@ -421,18 +421,21 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.12.2" +version = "3.12.3" description = "A platform independent file lock." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, - {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, + {file = "filelock-3.12.3-py3-none-any.whl", hash = "sha256:f067e40ccc40f2b48395a80fcbd4728262fab54e232e090a4063ab804179efeb"}, + {file = "filelock-3.12.3.tar.gz", hash = "sha256:0ecc1dd2ec4672a10c8550a8182f1bd0c0a5088470ecd5a125e45f49472fac3d"}, ] +[package.dependencies] +typing-extensions = {version = ">=4.7.1", markers = "python_version < \"3.11\""} + [package.extras] -docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] [[package]] name = "frozenlist" @@ -562,13 +565,13 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "grpc-interceptor" -version = "0.15.2" +version = "0.15.3" description = "Simplifies gRPC interceptors" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "grpc-interceptor-0.15.2.tar.gz", hash = "sha256:5c984110af4fb77d03472ec0468f9c77ddaf798e190410fb7b7f1e76c60c96a4"}, - {file = "grpc_interceptor-0.15.2-py3-none-any.whl", hash = "sha256:596dac3cb709ffb6178a4873f5148e254c871c9069f0b11040189b257969490a"}, + {file = "grpc-interceptor-0.15.3.tar.gz", hash = "sha256:33592cb9d8c00fceed5755c71029f75aef55b273496dbced06f1d48f2571fcc3"}, + {file = "grpc_interceptor-0.15.3-py3-none-any.whl", hash = "sha256:96be2043b7e49f9deb444f18b61c373ea28d22d81c90cd3b82127a4744eb9247"}, ] [package.dependencies] @@ -754,13 +757,13 @@ files = [ [[package]] name = "huggingface-hub" -version = "0.14.1" +version = "0.16.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.7.0" files = [ - {file = "huggingface_hub-0.14.1-py3-none-any.whl", hash = "sha256:9fc619170d800ff3793ad37c9757c255c8783051e1b5b00501205eb43ccc4f27"}, - {file = "huggingface_hub-0.14.1.tar.gz", hash = "sha256:9ab899af8e10922eac65e290d60ab956882ab0bf643e3d990b1394b6b47b7fbc"}, + {file = "huggingface_hub-0.16.4-py3-none-any.whl", hash = "sha256:0d3df29932f334fead024afc7cb4cc5149d955238b8b5e42dcf9740d6995a349"}, + {file = "huggingface_hub-0.16.4.tar.gz", hash = "sha256:608c7d4f3d368b326d1747f91523dbd1f692871e8e2e7a4750314a2dd8b63e14"}, ] [package.dependencies] @@ -773,15 +776,16 @@ tqdm = ">=4.42.1" typing-extensions = ">=3.7.4.3" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +inference = ["aiohttp", "pydantic"] quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] tensorflow = ["graphviz", "pydot", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "gradio", "jedi", "pytest", "pytest-cov", "pytest-env", "pytest-xdist", "soundfile"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] torch = ["torch"] -typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] [[package]] name = "idna" @@ -1409,13 +1413,13 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa [[package]] name = "pluggy" -version = "1.2.0" +version = "1.3.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] [package.extras] @@ -1424,24 +1428,24 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.24.0" +version = "4.24.2" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.24.0-cp310-abi3-win32.whl", hash = "sha256:81cb9c4621d2abfe181154354f63af1c41b00a4882fb230b4425cbaed65e8f52"}, - {file = "protobuf-4.24.0-cp310-abi3-win_amd64.whl", hash = "sha256:6c817cf4a26334625a1904b38523d1b343ff8b637d75d2c8790189a4064e51c3"}, - {file = "protobuf-4.24.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ae97b5de10f25b7a443b40427033e545a32b0e9dda17bcd8330d70033379b3e5"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:567fe6b0647494845d0849e3d5b260bfdd75692bf452cdc9cb660d12457c055d"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:a6b1ca92ccabfd9903c0c7dde8876221dc7d8d87ad5c42e095cc11b15d3569c7"}, - {file = "protobuf-4.24.0-cp37-cp37m-win32.whl", hash = "sha256:a38400a692fd0c6944c3c58837d112f135eb1ed6cdad5ca6c5763336e74f1a04"}, - {file = "protobuf-4.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5ab19ee50037d4b663c02218a811a5e1e7bb30940c79aac385b96e7a4f9daa61"}, - {file = "protobuf-4.24.0-cp38-cp38-win32.whl", hash = "sha256:e8834ef0b4c88666ebb7c7ec18045aa0f4325481d724daa624a4cf9f28134653"}, - {file = "protobuf-4.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:8bb52a2be32db82ddc623aefcedfe1e0eb51da60e18fcc908fb8885c81d72109"}, - {file = "protobuf-4.24.0-cp39-cp39-win32.whl", hash = "sha256:ae7a1835721086013de193311df858bc12cd247abe4ef9710b715d930b95b33e"}, - {file = "protobuf-4.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:44825e963008f8ea0d26c51911c30d3e82e122997c3c4568fd0385dd7bacaedf"}, - {file = "protobuf-4.24.0-py3-none-any.whl", hash = "sha256:82e6e9ebdd15b8200e8423676eab38b774624d6a1ad696a60d86a2ac93f18201"}, - {file = "protobuf-4.24.0.tar.gz", hash = "sha256:5d0ceb9de6e08311832169e601d1fc71bd8e8c779f3ee38a97a78554945ecb85"}, + {file = "protobuf-4.24.2-cp310-abi3-win32.whl", hash = "sha256:58e12d2c1aa428ece2281cef09bbaa6938b083bcda606db3da4e02e991a0d924"}, + {file = "protobuf-4.24.2-cp310-abi3-win_amd64.whl", hash = "sha256:77700b55ba41144fc64828e02afb41901b42497b8217b558e4a001f18a85f2e3"}, + {file = "protobuf-4.24.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:237b9a50bd3b7307d0d834c1b0eb1a6cd47d3f4c2da840802cd03ea288ae8880"}, + {file = "protobuf-4.24.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:25ae91d21e3ce8d874211110c2f7edd6384816fb44e06b2867afe35139e1fd1c"}, + {file = "protobuf-4.24.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:c00c3c7eb9ad3833806e21e86dca448f46035242a680f81c3fe068ff65e79c74"}, + {file = "protobuf-4.24.2-cp37-cp37m-win32.whl", hash = "sha256:4e69965e7e54de4db989289a9b971a099e626f6167a9351e9d112221fc691bc1"}, + {file = "protobuf-4.24.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c5cdd486af081bf752225b26809d2d0a85e575b80a84cde5172a05bbb1990099"}, + {file = "protobuf-4.24.2-cp38-cp38-win32.whl", hash = "sha256:6bd26c1fa9038b26c5c044ee77e0ecb18463e957fefbaeb81a3feb419313a54e"}, + {file = "protobuf-4.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb7aa97c252279da65584af0456f802bd4b2de429eb945bbc9b3d61a42a8cd16"}, + {file = "protobuf-4.24.2-cp39-cp39-win32.whl", hash = "sha256:2b23bd6e06445699b12f525f3e92a916f2dcf45ffba441026357dea7fa46f42b"}, + {file = "protobuf-4.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:839952e759fc40b5d46be319a265cf94920174d88de31657d5622b5d8d6be5cd"}, + {file = "protobuf-4.24.2-py3-none-any.whl", hash = "sha256:3b7b170d3491ceed33f723bbf2d5a260f8a4e23843799a3906f16ef736ef251e"}, + {file = "protobuf-4.24.2.tar.gz", hash = "sha256:7fda70797ddec31ddfa3576cbdcc3ddbb6b3078b737a1a87ab9136af0570cd6e"}, ] [[package]] @@ -1472,36 +1476,40 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "pyarrow" -version = "12.0.1" +version = "13.0.0" description = "Python library for Apache Arrow" optional = true -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pyarrow-12.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df"}, - {file = "pyarrow-12.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af"}, - {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6"}, - {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf"}, - {file = "pyarrow-12.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890"}, - {file = "pyarrow-12.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7"}, - {file = "pyarrow-12.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7"}, - {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718"}, - {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082"}, - {file = "pyarrow-12.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63"}, - {file = "pyarrow-12.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3"}, - {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f"}, - {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d"}, - {file = "pyarrow-12.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba"}, - {file = "pyarrow-12.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d"}, - {file = "pyarrow-12.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf"}, - {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c"}, - {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60"}, - {file = "pyarrow-12.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3"}, - {file = "pyarrow-12.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24"}, - {file = "pyarrow-12.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36"}, - {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca"}, - {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a"}, - {file = "pyarrow-12.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7"}, - {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, + {file = "pyarrow-13.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:1afcc2c33f31f6fb25c92d50a86b7a9f076d38acbcb6f9e74349636109550148"}, + {file = "pyarrow-13.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:70fa38cdc66b2fc1349a082987f2b499d51d072faaa6b600f71931150de2e0e3"}, + {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd57b13a6466822498238877892a9b287b0a58c2e81e4bdb0b596dbb151cbb73"}, + {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8ce69f7bf01de2e2764e14df45b8404fc6f1a5ed9871e8e08a12169f87b7a26"}, + {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:588f0d2da6cf1b1680974d63be09a6530fd1bd825dc87f76e162404779a157dc"}, + {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:6241afd72b628787b4abea39e238e3ff9f34165273fad306c7acf780dd850956"}, + {file = "pyarrow-13.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:fda7857e35993673fcda603c07d43889fca60a5b254052a462653f8656c64f44"}, + {file = "pyarrow-13.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:aac0ae0146a9bfa5e12d87dda89d9ef7c57a96210b899459fc2f785303dcbb67"}, + {file = "pyarrow-13.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d7759994217c86c161c6a8060509cfdf782b952163569606bb373828afdd82e8"}, + {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:868a073fd0ff6468ae7d869b5fc1f54de5c4255b37f44fb890385eb68b68f95d"}, + {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51be67e29f3cfcde263a113c28e96aa04362ed8229cb7c6e5f5c719003659d33"}, + {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d1b4e7176443d12610874bb84d0060bf080f000ea9ed7c84b2801df851320295"}, + {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:69b6f9a089d116a82c3ed819eea8fe67dae6105f0d81eaf0fdd5e60d0c6e0944"}, + {file = "pyarrow-13.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:ab1268db81aeb241200e321e220e7cd769762f386f92f61b898352dd27e402ce"}, + {file = "pyarrow-13.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:ee7490f0f3f16a6c38f8c680949551053c8194e68de5046e6c288e396dccee80"}, + {file = "pyarrow-13.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3ad79455c197a36eefbd90ad4aa832bece7f830a64396c15c61a0985e337287"}, + {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68fcd2dc1b7d9310b29a15949cdd0cb9bc34b6de767aff979ebf546020bf0ba0"}, + {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc6fd330fd574c51d10638e63c0d00ab456498fc804c9d01f2a61b9264f2c5b2"}, + {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:e66442e084979a97bb66939e18f7b8709e4ac5f887e636aba29486ffbf373763"}, + {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:0f6eff839a9e40e9c5610d3ff8c5bdd2f10303408312caf4c8003285d0b49565"}, + {file = "pyarrow-13.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b30a27f1cddf5c6efcb67e598d7823a1e253d743d92ac32ec1eb4b6a1417867"}, + {file = "pyarrow-13.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:09552dad5cf3de2dc0aba1c7c4b470754c69bd821f5faafc3d774bedc3b04bb7"}, + {file = "pyarrow-13.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3896ae6c205d73ad192d2fc1489cd0edfab9f12867c85b4c277af4d37383c18c"}, + {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6647444b21cb5e68b593b970b2a9a07748dd74ea457c7dadaa15fd469c48ada1"}, + {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47663efc9c395e31d09c6aacfa860f4473815ad6804311c5433f7085415d62a7"}, + {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b9ba6b6d34bd2563345488cf444510588ea42ad5613df3b3509f48eb80250afd"}, + {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:d00d374a5625beeb448a7fa23060df79adb596074beb3ddc1838adb647b6ef09"}, + {file = "pyarrow-13.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:c51afd87c35c8331b56f796eff954b9c7f8d4b7fef5903daf4e05fcf017d23a8"}, + {file = "pyarrow-13.0.0.tar.gz", hash = "sha256:83333726e83ed44b0ac94d8d7a21bbdee4a05029c3b1e8db58a863eec8fd8a33"}, ] [package.dependencies] @@ -1566,6 +1574,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1573,8 +1582,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1591,6 +1607,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1598,6 +1615,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1723,93 +1741,115 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "safetensors" -version = "0.3.2" +version = "0.3.3" description = "Fast and Safe Tensor serialization" optional = false python-versions = "*" files = [ - {file = "safetensors-0.3.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b6a66989075c2891d743153e8ba9ca84ee7232c8539704488f454199b8b8f84d"}, - {file = "safetensors-0.3.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:670d6bc3a3b377278ce2971fa7c36ebc0a35041c4ea23b9df750a39380800195"}, - {file = "safetensors-0.3.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:564f42838721925b5313ae864ba6caa6f4c80a9fbe63cf24310c3be98ab013cd"}, - {file = "safetensors-0.3.2-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:7f80af7e4ab3188daaff12d43d078da3017a90d732d38d7af4eb08b6ca2198a5"}, - {file = "safetensors-0.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec30d78f20f1235b252d59cbb9755beb35a1fde8c24c89b3c98e6a1804cfd432"}, - {file = "safetensors-0.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16063d94d8f600768d3c331b1e97964b1bf3772e19710105fe24ec5a6af63770"}, - {file = "safetensors-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbb44e140bf2aeda98d9dde669dbec15f7b77f96a9274469b91a6cf4bcc5ec3b"}, - {file = "safetensors-0.3.2-cp310-cp310-win32.whl", hash = "sha256:2961c1243fd0da46aa6a1c835305cc4595486f8ac64632a604d0eb5f2de76175"}, - {file = "safetensors-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c813920482c337d1424d306e1b05824a38e3ef94303748a0a287dea7a8c4f805"}, - {file = "safetensors-0.3.2-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:707df34bd9b9047e97332136ad98e57028faeccdb9cfe1c3b52aba5964cc24bf"}, - {file = "safetensors-0.3.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:becc5bb85b2947eae20ed23b407ebfd5277d9a560f90381fe2c42e6c043677ba"}, - {file = "safetensors-0.3.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:30a75707be5cc9686490bde14b9a371cede4af53244ea72b340cfbabfffdf58a"}, - {file = "safetensors-0.3.2-cp311-cp311-macosx_13_0_universal2.whl", hash = "sha256:54ad6af663e15e2b99e2ea3280981b7514485df72ba6d014dc22dae7ba6a5e6c"}, - {file = "safetensors-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37764b3197656ef507a266c453e909a3477dabc795962b38e3ad28226f53153b"}, - {file = "safetensors-0.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4939067736783acd8391d83cd97d6c202f94181951ce697d519f9746381b6a39"}, - {file = "safetensors-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada0fac127ff8fb04834da5c6d85a8077e6a1c9180a11251d96f8068db922a17"}, - {file = "safetensors-0.3.2-cp311-cp311-win32.whl", hash = "sha256:155b82dbe2b0ebff18cde3f76b42b6d9470296e92561ef1a282004d449fa2b4c"}, - {file = "safetensors-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:a86428d196959619ce90197731be9391b5098b35100a7228ef4643957648f7f5"}, - {file = "safetensors-0.3.2-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:c1f8ab41ed735c5b581f451fd15d9602ff51aa88044bfa933c5fa4b1d0c644d1"}, - {file = "safetensors-0.3.2-cp37-cp37m-macosx_13_0_x86_64.whl", hash = "sha256:bc9cfb3c9ea2aec89685b4d656f9f2296f0f0d67ecf2bebf950870e3be89b3db"}, - {file = "safetensors-0.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ace5d471e3d78e0d93f952707d808b5ab5eac77ddb034ceb702e602e9acf2be9"}, - {file = "safetensors-0.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de3e20a388b444381bcda1a3193cce51825ddca277e4cf3ed1fe8d9b2d5722cd"}, - {file = "safetensors-0.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d7d70d48585fe8df00725aa788f2e64fd24a4c9ae07cd6be34f6859d0f89a9c"}, - {file = "safetensors-0.3.2-cp37-cp37m-win32.whl", hash = "sha256:6ff59bc90cdc857f68b1023be9085fda6202bbe7f2fd67d06af8f976d6adcc10"}, - {file = "safetensors-0.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:8b05c93da15fa911763a89281906ca333ed800ab0ef1c7ce53317aa1a2322f19"}, - {file = "safetensors-0.3.2-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:8969cfd9e8d904e8d3c67c989e1bd9a95e3cc8980d4f95e4dcd43c299bb94253"}, - {file = "safetensors-0.3.2-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:f54148ac027556eb02187e9bc1556c4d916c99ca3cb34ca36a7d304d675035c1"}, - {file = "safetensors-0.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caec25fedbcf73f66c9261984f07885680f71417fc173f52279276c7f8a5edd3"}, - {file = "safetensors-0.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:50224a1d99927ccf3b75e27c3d412f7043280431ab100b4f08aad470c37cf99a"}, - {file = "safetensors-0.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa98f49e95f02eb750d32c4947e7d5aa43883149ebd0414920866446525b70f0"}, - {file = "safetensors-0.3.2-cp38-cp38-win32.whl", hash = "sha256:33409df5e28a83dc5cc5547a3ac17c0f1b13a1847b1eb3bc4b3be0df9915171e"}, - {file = "safetensors-0.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:e04a7cbbb3856159ab99e3adb14521544f65fcb8548cce773a1435a0f8d78d27"}, - {file = "safetensors-0.3.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:7c864cf5dcbfb608c5378f83319c60cc9c97263343b57c02756b7613cd5ab4dd"}, - {file = "safetensors-0.3.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:14e8c19d6dc51d4f70ee33c46aff04c8ba3f95812e74daf8036c24bc86e75cae"}, - {file = "safetensors-0.3.2-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:042a60f633c3c7009fdf6a7c182b165cb7283649d2a1e9c7a4a1c23454bd9a5b"}, - {file = "safetensors-0.3.2-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:fafd95e5ef41e8f312e2a32b7031f7b9b2a621b255f867b221f94bb2e9f51ae8"}, - {file = "safetensors-0.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ed77cf358abce2307f03634694e0b2a29822e322a1623e0b1aa4b41e871bf8b"}, - {file = "safetensors-0.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d344e8b2681a33aafc197c90b0def3229b3317d749531c72fa6259d0caa5c8c"}, - {file = "safetensors-0.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ff0024ef2e5722a79af24688ce4a430f70601d0cf712a744105ed4b8f67ba5"}, - {file = "safetensors-0.3.2-cp39-cp39-win32.whl", hash = "sha256:827af9478b78977248ba93e2fd97ea307fb63f463f80cef4824460f8c2542a52"}, - {file = "safetensors-0.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:9b09f27c456efa301f98681ea14b12f81f2637889f6336223ccab71e42c34541"}, - {file = "safetensors-0.3.2.tar.gz", hash = "sha256:2dbd34554ed3b99435a0e84df077108f5334c8336b5ed9cb8b6b98f7b10da2f6"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:92e4d0c8b2836120fddd134474c5bda8963f322333941f8b9f643e5b24f041eb"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:3dcadb6153c42addc9c625a622ebde9293fabe1973f9ef31ba10fb42c16e8536"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:08f26b61e1b0a14dc959aa9d568776bd038805f611caef1de04a80c468d4a7a4"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:17f41344d9a075f2f21b289a49a62e98baff54b5754240ba896063bce31626bf"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:f1045f798e1a16a6ced98d6a42ec72936d367a2eec81dc5fade6ed54638cd7d2"}, + {file = "safetensors-0.3.3-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:eaf0e4bc91da13f21ac846a39429eb3f3b7ed06295a32321fa3eb1a59b5c70f3"}, + {file = "safetensors-0.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25149180d4dc8ca48bac2ac3852a9424b466e36336a39659b35b21b2116f96fc"}, + {file = "safetensors-0.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9e943bf78c39de8865398a71818315e7d5d1af93c7b30d4da3fc852e62ad9bc"}, + {file = "safetensors-0.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cccfcac04a010354e87c7a2fe16a1ff004fc4f6e7ef8efc966ed30122ce00bc7"}, + {file = "safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a07121f427e646a50d18c1be0fa1a2cbf6398624c31149cd7e6b35486d72189e"}, + {file = "safetensors-0.3.3-cp310-cp310-win32.whl", hash = "sha256:a85e29cbfddfea86453cc0f4889b4bcc6b9c155be9a60e27be479a34e199e7ef"}, + {file = "safetensors-0.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:e13adad4a3e591378f71068d14e92343e626cf698ff805f61cdb946e684a218e"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:cbc3312f134baf07334dd517341a4b470b2931f090bd9284888acb7dfaf4606f"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:d15030af39d5d30c22bcbc6d180c65405b7ea4c05b7bab14a570eac7d7d43722"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:f84a74cbe9859b28e3d6d7715ac1dd3097bebf8d772694098f6d42435245860c"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:10d637423d98ab2e6a4ad96abf4534eb26fcaf8ca3115623e64c00759374e90d"}, + {file = "safetensors-0.3.3-cp311-cp311-macosx_13_0_universal2.whl", hash = "sha256:3b46f5de8b44084aff2e480874c550c399c730c84b2e8ad1bddb062c94aa14e9"}, + {file = "safetensors-0.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e76da691a82dfaf752854fa6d17c8eba0c8466370c5ad8cf1bfdf832d3c7ee17"}, + {file = "safetensors-0.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4e342fd54e66aa9512dd13e410f791e47aa4feeb5f4c9a20882c72f3d272f29"}, + {file = "safetensors-0.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:178fd30b5dc73bce14a39187d948cedd0e5698e2f055b7ea16b5a96c9b17438e"}, + {file = "safetensors-0.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e8fdf7407dba44587ed5e79d5de3533d242648e1f2041760b21474bd5ea5c8c"}, + {file = "safetensors-0.3.3-cp311-cp311-win32.whl", hash = "sha256:7d3b744cee8d7a46ffa68db1a2ff1a1a432488e3f7a5a97856fe69e22139d50c"}, + {file = "safetensors-0.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f579877d30feec9b6ba409d05fa174633a4fc095675a4a82971d831a8bb60b97"}, + {file = "safetensors-0.3.3-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:2fff5b19a1b462c17322998b2f4b8bce43c16fe208968174d2f3a1446284ceed"}, + {file = "safetensors-0.3.3-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:41adb1d39e8aad04b16879e3e0cbcb849315999fad73bc992091a01e379cb058"}, + {file = "safetensors-0.3.3-cp37-cp37m-macosx_12_0_x86_64.whl", hash = "sha256:0f2b404250b3b877b11d34afcc30d80e7035714a1116a3df56acaca6b6c00096"}, + {file = "safetensors-0.3.3-cp37-cp37m-macosx_13_0_x86_64.whl", hash = "sha256:b43956ef20e9f4f2e648818a9e7b3499edd6b753a0f5526d4f6a6826fbee8446"}, + {file = "safetensors-0.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d61a99b34169981f088ccfbb2c91170843efc869a0a0532f422db7211bf4f474"}, + {file = "safetensors-0.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c0008aab36cd20e9a051a68563c6f80d40f238c2611811d7faa5a18bf3fd3984"}, + {file = "safetensors-0.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:93d54166072b143084fdcd214a080a088050c1bb1651016b55942701b31334e4"}, + {file = "safetensors-0.3.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c32ee08f61cea56a5d62bbf94af95df6040c8ab574afffaeb7b44ae5da1e9e3"}, + {file = "safetensors-0.3.3-cp37-cp37m-win32.whl", hash = "sha256:351600f367badd59f7bfe86d317bb768dd8c59c1561c6fac43cafbd9c1af7827"}, + {file = "safetensors-0.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:034717e297849dae1af0a7027a14b8647bd2e272c24106dced64d83e10d468d1"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:8530399666748634bc0b301a6a5523756931b0c2680d188e743d16304afe917a"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:9d741c1f1621e489ba10aa3d135b54202684f6e205df52e219d5eecd673a80c9"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:0c345fd85b4d2093a5109596ff4cd9dfc2e84992e881b4857fbc4a93a3b89ddb"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:69ccee8d05f55cdf76f7e6c87d2bdfb648c16778ef8acfd2ecc495e273e9233e"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:c08a9a4b7a4ca389232fa8d097aebc20bbd4f61e477abc7065b5c18b8202dede"}, + {file = "safetensors-0.3.3-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:a002868d2e3f49bbe81bee2655a411c24fa1f8e68b703dec6629cb989d6ae42e"}, + {file = "safetensors-0.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bd2704cb41faa44d3ec23e8b97330346da0395aec87f8eaf9c9e2c086cdbf13"}, + {file = "safetensors-0.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2951bf3f0ad63df5e6a95263652bd6c194a6eb36fd4f2d29421cd63424c883"}, + {file = "safetensors-0.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07114cec116253ca2e7230fdea30acf76828f21614afd596d7b5438a2f719bd8"}, + {file = "safetensors-0.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ab43aeeb9eadbb6b460df3568a662e6f1911ecc39387f8752afcb6a7d96c087"}, + {file = "safetensors-0.3.3-cp38-cp38-win32.whl", hash = "sha256:f2f59fce31dd3429daca7269a6b06f65e6547a0c248f5116976c3f1e9b73f251"}, + {file = "safetensors-0.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:c31ca0d8610f57799925bf08616856b39518ab772c65093ef1516762e796fde4"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:59a596b3225c96d59af412385981f17dd95314e3fffdf359c7e3f5bb97730a19"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:82a16e92210a6221edd75ab17acdd468dd958ef5023d9c6c1289606cc30d1479"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:98a929e763a581f516373ef31983ed1257d2d0da912a8e05d5cd12e9e441c93a"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:12b83f1986cd16ea0454c636c37b11e819d60dd952c26978310a0835133480b7"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:f439175c827c2f1bbd54df42789c5204a10983a30bc4242bc7deaf854a24f3f0"}, + {file = "safetensors-0.3.3-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:0085be33b8cbcb13079b3a8e131656e05b0bc5e6970530d4c24150f7afd76d70"}, + {file = "safetensors-0.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3ec70c87b1e910769034206ad5efc051069b105aac1687f6edcd02526767f4"}, + {file = "safetensors-0.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f490132383e5e490e710608f4acffcb98ed37f91b885c7217d3f9f10aaff9048"}, + {file = "safetensors-0.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:79d1b6c7ed5596baf79c80fbce5198c3cdcc521ae6a157699f427aba1a90082d"}, + {file = "safetensors-0.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad3cc8006e7a86ee7c88bd2813ec59cd7cc75b03e6fa4af89b9c7b235b438d68"}, + {file = "safetensors-0.3.3-cp39-cp39-win32.whl", hash = "sha256:ab29f54c6b8c301ca05fa014728996bd83aac6e21528f893aaf8945c71f42b6d"}, + {file = "safetensors-0.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:0fa82004eae1a71e2aa29843ef99de9350e459a0fc2f65fc6ee0da9690933d2d"}, + {file = "safetensors-0.3.3.tar.gz", hash = "sha256:edb7072d788c4f929d0f5735d3a2fb51e5a27f833587828583b7f5747af1a2b8"}, ] [package.extras] all = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (==2.11.0)", "torch (>=1.10)"] dev = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "flax (>=0.6.3)", "h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "isort (>=5.5.4)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)", "tensorflow (==2.11.0)", "torch (>=1.10)"] -jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "numpy (>=1.21.6)"] numpy = ["numpy (>=1.21.6)"] -paddlepaddle = ["paddlepaddle (>=2.4.1)"] +paddlepaddle = ["numpy (>=1.21.6)", "paddlepaddle (>=2.4.1)"] pinned-tf = ["tensorflow (==2.11.0)"] quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] -tensorflow = ["tensorflow (>=2.11.0)"] +tensorflow = ["numpy (>=1.21.6)", "tensorflow (>=2.11.0)"] testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "numpy (>=1.21.6)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "setuptools-rust (>=1.5.2)"] -torch = ["torch (>=1.10)"] +torch = ["numpy (>=1.21.6)", "torch (>=1.10)"] [[package]] name = "scipy" -version = "1.11.1" +version = "1.11.2" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = "<3.13,>=3.9" files = [ - {file = "scipy-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:aec8c62fbe52914f9cf28d846cf0401dd80ab80788bbab909434eb336ed07c04"}, - {file = "scipy-1.11.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3b9963798df1d8a52db41a6fc0e6fa65b1c60e85d73da27ae8bb754de4792481"}, - {file = "scipy-1.11.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e8eb42db36526b130dfbc417609498a6192381abc1975b91e3eb238e0b41c1a"}, - {file = "scipy-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:366a6a937110d80dca4f63b3f5b00cc89d36f678b2d124a01067b154e692bab1"}, - {file = "scipy-1.11.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:08d957ca82d3535b3b9ba6c8ff355d78fe975271874e2af267cb5add5bd78625"}, - {file = "scipy-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:e866514bc2d660608447b6ba95c8900d591f2865c07cca0aa4f7ff3c4ca70f30"}, - {file = "scipy-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba94eeef3c9caa4cea7b402a35bb02a5714ee1ee77eb98aca1eed4543beb0f4c"}, - {file = "scipy-1.11.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:512fdc18c65f76dadaca139348e525646d440220d8d05f6d21965b8d4466bccd"}, - {file = "scipy-1.11.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cce154372f0ebe88556ed06d7b196e9c2e0c13080ecb58d0f35062dc7cc28b47"}, - {file = "scipy-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4bb943010203465ac81efa392e4645265077b4d9e99b66cf3ed33ae12254173"}, - {file = "scipy-1.11.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:249cfa465c379c9bb2c20123001e151ff5e29b351cbb7f9c91587260602c58d0"}, - {file = "scipy-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:ffb28e3fa31b9c376d0fb1f74c1f13911c8c154a760312fbee87a21eb21efe31"}, - {file = "scipy-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:39154437654260a52871dfde852adf1b93b1d1bc5dc0ffa70068f16ec0be2624"}, - {file = "scipy-1.11.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:b588311875c58d1acd4ef17c983b9f1ab5391755a47c3d70b6bd503a45bfaf71"}, - {file = "scipy-1.11.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d51565560565a0307ed06fa0ec4c6f21ff094947d4844d6068ed04400c72d0c3"}, - {file = "scipy-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b41a0f322b4eb51b078cb3441e950ad661ede490c3aca66edef66f4b37ab1877"}, - {file = "scipy-1.11.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:396fae3f8c12ad14c5f3eb40499fd06a6fef8393a6baa352a652ecd51e74e029"}, - {file = "scipy-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:be8c962a821957fdde8c4044efdab7a140c13294997a407eaee777acf63cbf0c"}, - {file = "scipy-1.11.1.tar.gz", hash = "sha256:fb5b492fa035334fd249f0973cc79ecad8b09c604b42a127a677b45a9a3d4289"}, + {file = "scipy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2b997a5369e2d30c97995dcb29d638701f8000d04df01b8e947f206e5d0ac788"}, + {file = "scipy-1.11.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:95763fbda1206bec41157582bea482f50eb3702c85fffcf6d24394b071c0e87a"}, + {file = "scipy-1.11.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e367904a0fec76433bf3fbf3e85bf60dae8e9e585ffd21898ab1085a29a04d16"}, + {file = "scipy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d690e1ca993c8f7ede6d22e5637541217fc6a4d3f78b3672a6fe454dbb7eb9a7"}, + {file = "scipy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d2b813bfbe8dec6a75164523de650bad41f4405d35b0fa24c2c28ae07fcefb20"}, + {file = "scipy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:afdb0d983f6135d50770dd979df50bf1c7f58b5b33e0eb8cf5c73c70600eae1d"}, + {file = "scipy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d9886f44ef8c9e776cb7527fb01455bf4f4a46c455c4682edc2c2cc8cd78562"}, + {file = "scipy-1.11.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1342ca385c673208f32472830c10110a9dcd053cf0c4b7d4cd7026d0335a6c1d"}, + {file = "scipy-1.11.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b133f237bd8ba73bad51bc12eb4f2d84cbec999753bf25ba58235e9fc2096d80"}, + {file = "scipy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aeb87661de987f8ec56fa6950863994cd427209158255a389fc5aea51fa7055"}, + {file = "scipy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:90d3b1364e751d8214e325c371f0ee0dd38419268bf4888b2ae1040a6b266b2a"}, + {file = "scipy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:f73102f769ee06041a3aa26b5841359b1a93cc364ce45609657751795e8f4a4a"}, + {file = "scipy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa4909c6c20c3d91480533cddbc0e7c6d849e7d9ded692918c76ce5964997898"}, + {file = "scipy-1.11.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ac74b1512d38718fb6a491c439aa7b3605b96b1ed3be6599c17d49d6c60fca18"}, + {file = "scipy-1.11.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8425fa963a32936c9773ee3ce44a765d8ff67eed5f4ac81dc1e4a819a238ee9"}, + {file = "scipy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:542a757e2a6ec409e71df3d8fd20127afbbacb1c07990cb23c5870c13953d899"}, + {file = "scipy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ea932570b1c2a30edafca922345854ff2cd20d43cd9123b6dacfdecebfc1a80b"}, + {file = "scipy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:4447ad057d7597476f9862ecbd9285bbf13ba9d73ce25acfa4e4b11c6801b4c9"}, + {file = "scipy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b0620240ef445b5ddde52460e6bc3483b7c9c750275369379e5f609a1050911c"}, + {file = "scipy-1.11.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:f28f1f6cfeb48339c192efc6275749b2a25a7e49c4d8369a28b6591da02fbc9a"}, + {file = "scipy-1.11.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:214cdf04bbae7a54784f8431f976704ed607c4bc69ba0d5d5d6a9df84374df76"}, + {file = "scipy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10eb6af2f751aa3424762948e5352f707b0dece77288206f227864ddf675aca0"}, + {file = "scipy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0f3261f14b767b316d7137c66cc4f33a80ea05841b9c87ad83a726205b901423"}, + {file = "scipy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:2c91cf049ffb5575917f2a01da1da082fd24ed48120d08a6e7297dfcac771dcd"}, + {file = "scipy-1.11.2.tar.gz", hash = "sha256:b29318a5e39bd200ca4381d80b065cdf3076c7d7281c5e36569e99273867f61d"}, ] [package.dependencies] @@ -1876,18 +1916,18 @@ files = [ [[package]] name = "setuptools" -version = "68.0.0" +version = "68.1.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, - {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, + {file = "setuptools-68.1.2-py3-none-any.whl", hash = "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"}, + {file = "setuptools-68.1.2.tar.gz", hash = "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5,<=7.1.2)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -2052,18 +2092,18 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.31.0" +version = "4.32.1" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" optional = false python-versions = ">=3.8.0" files = [ - {file = "transformers-4.31.0-py3-none-any.whl", hash = "sha256:8487aab0195ce1c2a5ae189305118b9720daddbc7b688edb09ccd79e3b149f6b"}, - {file = "transformers-4.31.0.tar.gz", hash = "sha256:4302fba920a1c24d3a429a29efff6a63eac03f3f3cf55b55927fc795d01cb273"}, + {file = "transformers-4.32.1-py3-none-any.whl", hash = "sha256:b930d3dbd907a3f300cf49e54d63a56f8a0ab16b01a2c2a61ecff37c6de1da08"}, + {file = "transformers-4.32.1.tar.gz", hash = "sha256:1edc8ae1de357d97c3d36b04412aa63d55e6fc0c4b39b419a7d380ed947d2252"}, ] [package.dependencies] filelock = "*" -huggingface-hub = ">=0.14.1,<1.0" +huggingface-hub = ">=0.15.1,<1.0" numpy = ">=1.17" packaging = ">=20.0" pyyaml = ">=5.1" @@ -2076,18 +2116,18 @@ tqdm = ">=4.27" [package.extras] accelerate = ["accelerate (>=0.20.3)"] agents = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] -all = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +all = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] codecarbon = ["codecarbon (==1.2.0)"] deepspeed = ["accelerate (>=0.20.3)", "deepspeed (>=0.9.3)"] deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] dev-torch = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -docs = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +docs = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] docs-specific = ["hf-doc-builder"] fairscale = ["fairscale (>0.3)"] -flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.2.8,!=0.3.2,<=0.4.13)", "jaxlib (>=0.1.65,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] +flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] ftfy = ["ftfy"] integrations = ["optuna", "ray[tune]", "sigopt"] @@ -2115,7 +2155,7 @@ tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] torch = ["accelerate (>=0.20.3)", "torch (>=1.9,!=1.12.0)"] torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] torch-vision = ["Pillow (<10.0.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.14.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] +torchhub = ["filelock", "huggingface-hub (>=0.15.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] video = ["av (==9.2.0)", "decord (==0.6.0)"] vision = ["Pillow (<10.0.0)"] @@ -2465,4 +2505,4 @@ quantize = ["accelerate", "datasets", "texttable"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "91a848038c08a44c67acfb4257781440ccc1e74a4b82f09513e75588fa33f72b" +content-hash = "f2ef5f41a172d14985367a385ad6ce844c8c05b2d68d9ddcc11b41f581921c96" diff --git a/server/pyproject.toml b/server/pyproject.toml index 0e6d7882..1babf749 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.0.1" +version = "1.0.3" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] @@ -25,8 +25,8 @@ opentelemetry-instrumentation-grpc = "^0.36b0" hf-transfer = "^0.1.2" sentencepiece = "^0.1.97" tokenizers = "^0.13.3" -huggingface-hub = "^0.14.1" -transformers = "^4.31.0" +huggingface-hub = "^0.16.4" +transformers = "^4.32.1" einops = "^0.6.1" texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } diff --git a/server/requirements.txt b/server/requirements.txt index 012e91a2..1b038cca 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,25 +1,36 @@ accelerate==0.20.3 ; python_version >= "3.9" and python_version < "3.13" +aiohttp==3.8.5 ; python_version >= "3.9" and python_version < "3.13" +aiosignal==1.3.1 ; python_version >= "3.9" and python_version < "3.13" +async-timeout==4.0.3 ; python_version >= "3.9" and python_version < "3.13" +attrs==23.1.0 ; python_version >= "3.9" and python_version < "3.13" backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13" +bitsandbytes==0.41.1 ; python_version >= "3.9" and python_version < "3.13" certifi==2023.7.22 ; python_version >= "3.9" and python_version < "3.13" charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "3.13" -click==8.1.6 ; python_version >= "3.9" and python_version < "3.13" +click==8.1.7 ; python_version >= "3.9" and python_version < "3.13" colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows") +datasets==2.14.4 ; python_version >= "3.9" and python_version < "3.13" deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13" +dill==0.3.7 ; python_version >= "3.9" and python_version < "3.13" einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13" -filelock==3.12.2 ; python_version >= "3.9" and python_version < "3.13" +filelock==3.12.3 ; python_version >= "3.9" and python_version < "3.13" +frozenlist==1.4.0 ; python_version >= "3.9" and python_version < "3.13" fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "3.13" +fsspec[http]==2023.6.0 ; python_version >= "3.9" and python_version < "3.13" googleapis-common-protos==1.60.0 ; python_version >= "3.9" and python_version < "3.13" -grpc-interceptor==0.15.2 ; python_version >= "3.9" and python_version < "3.13" +grpc-interceptor==0.15.3 ; python_version >= "3.9" and python_version < "3.13" grpcio-reflection==1.57.0 ; python_version >= "3.9" and python_version < "3.13" grpcio-status==1.57.0 ; python_version >= "3.9" and python_version < "3.13" grpcio==1.57.0 ; python_version >= "3.9" and python_version < "3.13" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "3.13" -huggingface-hub==0.14.1 ; python_version >= "3.9" and python_version < "3.13" +huggingface-hub==0.16.4 ; python_version >= "3.9" and python_version < "3.13" idna==3.4 ; python_version >= "3.9" and python_version < "3.13" jinja2==3.1.2 ; python_version >= "3.9" and python_version < "3.13" loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" markupsafe==2.1.3 ; python_version >= "3.9" and python_version < "3.13" mpmath==1.3.0 ; python_version >= "3.9" and python_version < "3.13" +multidict==6.0.4 ; python_version >= "3.9" and python_version < "3.13" +multiprocess==0.70.15 ; python_version >= "3.9" and python_version < "3.13" networkx==3.1 ; python_version >= "3.9" and python_version < "3.13" numpy==1.25.2 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" @@ -32,24 +43,33 @@ opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13 opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13" packaging==23.1 ; python_version >= "3.9" and python_version < "3.13" +pandas==2.0.3 ; python_version >= "3.9" and python_version < "3.13" peft==0.4.0 ; python_version >= "3.9" and python_version < "3.13" pillow==10.0.0 ; python_version >= "3.9" and python_version < "3.13" -protobuf==4.24.0 ; python_version >= "3.9" and python_version < "3.13" +protobuf==4.24.2 ; python_version >= "3.9" and python_version < "3.13" psutil==5.9.5 ; python_version >= "3.9" and python_version < "3.13" +pyarrow==13.0.0 ; python_version >= "3.9" and python_version < "3.13" +python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "3.13" +pytz==2023.3 ; python_version >= "3.9" and python_version < "3.13" pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" regex==2023.8.8 ; python_version >= "3.9" and python_version < "3.13" requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" -safetensors==0.3.2 ; python_version >= "3.9" and python_version < "3.13" -scipy==1.11.1 ; python_version >= "3.9" and python_version < "3.13" +safetensors==0.3.3 ; python_version >= "3.9" and python_version < "3.13" +scipy==1.11.2 ; python_version >= "3.9" and python_version < "3.13" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" -setuptools==68.0.0 ; python_version >= "3.9" and python_version < "3.13" +setuptools==68.1.2 ; python_version >= "3.9" and python_version < "3.13" +six==1.16.0 ; python_version >= "3.9" and python_version < "3.13" sympy==1.12 ; python_version >= "3.9" and python_version < "3.13" +texttable==1.6.7 ; python_version >= "3.9" and python_version < "3.13" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" torch==2.0.1 ; python_version >= "3.9" and python_version < "3.13" tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" -transformers==4.31.0 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.32.1 ; python_version >= "3.9" and python_version < "3.13" typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "3.13" +tzdata==2023.3 ; python_version >= "3.9" and python_version < "3.13" urllib3==2.0.4 ; python_version >= "3.9" and python_version < "3.13" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32" wrapt==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +xxhash==3.3.0 ; python_version >= "3.9" and python_version < "3.13" +yarl==1.9.2 ; python_version >= "3.9" and python_version < "3.13" From 7c2e0af2a61745a36b26fae2c817f608be757a4c Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 30 Aug 2023 11:09:46 +0200 Subject: [PATCH 399/793] Fix f180 (#951) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../models/custom_modeling/flash_rw_modeling.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 14caa23d..8419fa4f 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -241,19 +241,21 @@ def __init__( hidden_size = config.hidden_size num_heads = config.n_head - num_heads_kv = config.n_head_kv + # num_heads_kv = config.n_head_kv + num_groups = config.n_head_kv self.hidden_size = hidden_size self.head_size = hidden_size // num_heads + self.num_groups = num_groups self.rotary_emb = PositionRotaryEmbedding.static( config=config, dim=self.head_size, base=10000.0, device=weights.device ) self.softmax_scale = self.head_size ** (-0.5) - self.num_groups = num_heads // (num_heads_kv * 2) + # self.num_groups = num_heads // (num_heads_kv * 2) self.num_heads = num_heads // self.num_groups - self.num_heads_kv = num_heads_kv // self.num_groups + # self.num_heads_kv = num_heads_kv // self.num_groups process_group = weights.process_group if process_group.size() > self.num_groups: @@ -264,6 +266,7 @@ def __init__( raise NotImplementedError( f"Tensor Parallelism is not implemented for {self.num_groups} not divisible by {process_group.size()}" ) + self.num_groups = self.num_groups // process_group.size() self.query_key_value = TensorParallelColumnLinear.load( From 7d8e5fb2845fb818f9244a70c4bb3aeaf190b65d Mon Sep 17 00:00:00 2001 From: Omar Sanseviero Date: Thu, 31 Aug 2023 20:00:12 +0200 Subject: [PATCH 400/793] Update version in docs (#957) --- README.md | 5 +++-- docs/openapi.json | 2 +- docs/source/quicktour.md | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 739e656b..fd2afd0d 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ to power Hugging Chat, the Inference API and Inference Endpoint. - [Falcon 40B](https://huggingface.co/tiiuae/falcon-40b) - [MPT](https://huggingface.co/mosaicml/mpt-30b) - [Llama V2](https://huggingface.co/meta-llama) +- [Code Llama](https://huggingface.co/codellama) Other architectures are supported on a best effort basis using: @@ -86,7 +87,7 @@ The easiest way of getting started is using the official Docker container: model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.2 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.3 --model-id $model ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. @@ -153,7 +154,7 @@ model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= -docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.2 --model-id $model +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.3 --model-id $model ``` ### A note on Shared Memory (shm) diff --git a/docs/openapi.json b/docs/openapi.json index 5974c58d..23c4f198 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.0.2" + "version": "1.0.3" }, "paths": { "/": { diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index c085943c..b91e77cb 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -8,7 +8,7 @@ Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/ model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.2 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.3 --model-id $model ``` @@ -85,7 +85,7 @@ curl 127.0.0.1:8080/generate \ To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. ```bash -docker run ghcr.io/huggingface/text-generation-inference:1.0.2 --help +docker run ghcr.io/huggingface/text-generation-inference:1.0.3 --help ``` From 8a5f5649429f8df0ae5b86e485879c3d21d255f6 Mon Sep 17 00:00:00 2001 From: Vincent Brouwers Date: Thu, 31 Aug 2023 21:15:14 +0200 Subject: [PATCH 401/793] Fix Falcon weight mapping for H2O.ai checkpoints (#953) # What does this PR do? During the safetensor conversion, duplicate weights are removed. However, which of the duplicates gets removed, differs per checkpoint. In some, like `h2oai/h2ogpt-oig-oasst1-falcon-40b`, the weight `transformer.word_embeddings.weightSafetensor` gets removed. In others, `lm_head.weight` gets removed. Long story long, we need to support both. Originally, f018143 mapped `lm_head` to `word_embeddings`. Then ac736fd switched this around. This commit merges them and allows for both. ## Before submitting - [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? @Narsil, you wrote both commits I referenced in this PR. I think you'll understand this change :) --- server/text_generation_server/models/flash_rw.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 2fc7c53d..195b3883 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -54,7 +54,10 @@ def __init__( device, dtype, process_group=self.process_group, - aliases={"lm_head.weight": ["transformer.word_embeddings.weight"]}, + aliases={ + "lm_head.weight": ["transformer.word_embeddings.weight"], + "transformer.word_embeddings.weight": ["lm_head.weight"], + }, ) config.quantize = quantize From 4f5d93ecd0c5ba5de0ead54d8934f163550d27d2 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 1 Sep 2023 00:22:03 +0200 Subject: [PATCH 402/793] Fixing top_k tokens when k ends up < 0 (#966) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/tokens.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 69177d56..7b003f1d 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -363,9 +363,10 @@ def batch_top_tokens( # Find the new "fuzzy" top n values top_n_indices = (logprobs >= nth_highest).nonzero() _, top_n_ishes = torch.unique_consecutive(top_n_indices[:, 0], return_counts=True) - + + k = 1 if top_n_ishes.numel() == 0 else top_n_ishes.max() # Take a new topk for these new max n values - top_k = torch.topk(logprobs, k=top_n_ishes.max(), dim=1, sorted=True) + top_k = torch.topk(logprobs, k=k, dim=1, sorted=True) top_n_ishes = top_n_ishes.tolist() top_indices = top_k.indices.tolist() From 2bc287bfcd54bfb75833ab73d170afaf552053ba Mon Sep 17 00:00:00 2001 From: Victor SANH Date: Fri, 1 Sep 2023 12:44:34 -0400 Subject: [PATCH 403/793] small fix on idefics (#954) transposing the fixes from https://github.com/huggingface/transformers/pull/25787 --- .../models/custom_modeling/idefics_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/idefics_config.py b/server/text_generation_server/models/custom_modeling/idefics_config.py index 34925087..0bdb2e3d 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_config.py +++ b/server/text_generation_server/models/custom_modeling/idefics_config.py @@ -51,7 +51,7 @@ class IdeficsVisionConfig(PretrainedConfig): Number of attention heads for each attention layer in the Transformer encoder. image_num_channels (`int`, *optional*, defaults to `3`): Number of image channels. - hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`): + hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-5): @@ -80,7 +80,7 @@ def __init__( num_hidden_layers=32, num_attention_heads=16, num_channels=3, - hidden_act="quick_gelu", + hidden_act="gelu", layer_norm_eps=1e-5, attention_dropout=0.0, initializer_range=0.02, From 033230ae667101d2d8d8bcd4952442fa348ef951 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 4 Sep 2023 15:00:19 +0200 Subject: [PATCH 404/793] Backport https://github.com/vllm-project/vllm/pull/936 (#977) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/Makefile-vllm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/Makefile-vllm b/server/Makefile-vllm index af750733..96bfc108 100644 --- a/server/Makefile-vllm +++ b/server/Makefile-vllm @@ -1,4 +1,4 @@ -vllm_commit := d284b831c17f42a8ea63369a06138325f73c4cf9 +vllm_commit := e86af624d059969b0fb07b075b1d338bf10c3365 vllm: # Clone vllm @@ -10,4 +10,4 @@ build-vllm: vllm install-vllm: build-vllm pip uninstall vllm -y || true - cd vllm && python setup.py install \ No newline at end of file + cd vllm && python setup.py install From c8bbbd812900ea52bab9b950f846ed2d975d9e78 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 6 Sep 2023 05:12:08 -0700 Subject: [PATCH 405/793] chore(client): Support Pydantic 2 (#900) This should allow users to use either Pydantic 2 or Pydantic 1. I couldn't run all tests locally because I reran them too often and got rate limited, but I believe this is sufficient. --- clients/python/pyproject.toml | 2 +- clients/python/text_generation/types.py | 30 ++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index a52bdd81..915ac7aa 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -12,7 +12,7 @@ repository = "https://github.com/huggingface/text-generation-inference" [tool.poetry.dependencies] python = "^3.7" -pydantic = "^1.10" +pydantic = "> 1.10, < 3" aiohttp = "^3.8" huggingface-hub = ">= 0.12, < 1.0" diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index 38f75253..20083b19 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -18,21 +18,21 @@ class Parameters(BaseModel): # Stop generating tokens if a member of `stop_sequences` is generated stop: List[str] = [] # Random sampling seed - seed: Optional[int] + seed: Optional[int] = None # The value used to module the logits distribution. - temperature: Optional[float] + temperature: Optional[float] = None # The number of highest probability vocabulary tokens to keep for top-k-filtering. - top_k: Optional[int] + top_k: Optional[int] = None # If set to < 1, only the smallest set of most probable tokens with probabilities that add up to `top_p` or # higher are kept for generation. - top_p: Optional[float] + top_p: Optional[float] = None # truncate inputs tokens to the given size - truncate: Optional[int] + truncate: Optional[int] = None # Typical Decoding mass # See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information - typical_p: Optional[float] + typical_p: Optional[float] = None # Generate best_of sequences and return the one if the highest token logprobs - best_of: Optional[int] + best_of: Optional[int] = None # Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) watermark: bool = False # Get generation details @@ -114,7 +114,7 @@ class Request(BaseModel): # Prompt inputs: str # Generation parameters - parameters: Optional[Parameters] + parameters: Optional[Parameters] = None # Whether to stream output tokens stream: bool = False @@ -145,7 +145,7 @@ class InputToken(BaseModel): text: str # Logprob # Optional since the logprob of the first token cannot be computed - logprob: Optional[float] + logprob: Optional[float] = None # Generated tokens @@ -180,7 +180,7 @@ class BestOfSequence(BaseModel): # Number of generated tokens generated_tokens: int # Sampling seed if sampling was activated - seed: Optional[int] + seed: Optional[int] = None # Decoder input tokens, empty if decoder_input_details is False prefill: List[InputToken] # Generated tokens @@ -196,7 +196,7 @@ class Details(BaseModel): # Number of generated tokens generated_tokens: int # Sampling seed if sampling was activated - seed: Optional[int] + seed: Optional[int] = None # Decoder input tokens, empty if decoder_input_details is False prefill: List[InputToken] # Generated tokens @@ -204,7 +204,7 @@ class Details(BaseModel): # Most likely tokens top_tokens: Optional[List[List[Token]]] # Additional sequences when using the `best_of` parameter - best_of_sequences: Optional[List[BestOfSequence]] + best_of_sequences: Optional[List[BestOfSequence]] = None # `generate` return value @@ -222,7 +222,7 @@ class StreamDetails(BaseModel): # Number of generated tokens generated_tokens: int # Sampling seed if sampling was activated - seed: Optional[int] + seed: Optional[int] = None # `generate_stream` return value @@ -233,10 +233,10 @@ class StreamResponse(BaseModel): top_tokens: Optional[List[Token]] # Complete generated text # Only available when the generation is finished - generated_text: Optional[str] + generated_text: Optional[str] = None # Generation details # Only available when the generation is finished - details: Optional[StreamDetails] + details: Optional[StreamDetails] = None # Inference API currently deployed model From 3ed4c0f33fee281fbdc276e208574e22821818d9 Mon Sep 17 00:00:00 2001 From: Julien Bouquillon Date: Wed, 6 Sep 2023 14:57:59 +0200 Subject: [PATCH 406/793] docs: typo in streaming.js (#971) Looks like an error --- docs/source/conceptual/streaming.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conceptual/streaming.md b/docs/source/conceptual/streaming.md index c20d76e0..054fde30 100644 --- a/docs/source/conceptual/streaming.md +++ b/docs/source/conceptual/streaming.md @@ -121,9 +121,9 @@ If you're using the free Inference API, you can use `HfInference`. If you're usi We can create a `HfInferenceEndpoint` providing our endpoint URL and credential. ```js -import { HfInference } from '@huggingface/inference' +import { HfInferenceEndpoint } from '@huggingface/inference' -const hf = new HfInference('https://YOUR_ENDPOINT.endpoints.huggingface.cloud', 'hf_YOUR_TOKEN') +const hf = new HfInferenceEndpoint('https://YOUR_ENDPOINT.endpoints.huggingface.cloud', 'hf_YOUR_TOKEN') // prompt const prompt = 'What can you do in Nuremberg, Germany? Give me 3 Tips' From 211e7b7e3503c7e388c2cadb10132d53b1deb8b0 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 6 Sep 2023 15:01:00 +0200 Subject: [PATCH 407/793] Disabling exllama on old compute. (#986) # What does this PR do? Disabling exllama on old compute. Exllama + T4 don't play nice together, this will disable it right away to avoid issues at runtime. Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/layers.py | 17 ++++++++++++----- server/text_generation_server/utils/weights.py | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 745c1d2e..6be54048 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -18,13 +18,20 @@ from text_generation_server.utils.gptq.quant_linear import QuantLinear -HAS_EXLLAMA = True -if os.getenv("DISABLE_EXLLAMA") == "True": - HAS_EXLLAMA = False try: - from text_generation_server.utils.gptq.exllama import Ex4bitLinear -except ImportError: + major, _minor = torch.cuda.get_device_capability() +except Exception: + major = 1 +HAS_EXLLAMA = False +CAN_EXLLAMA = major >= 8 +if os.getenv("DISABLE_EXLLAMA") == "True": HAS_EXLLAMA = False +elif CAN_EXLLAMA: + try: + from text_generation_server.utils.gptq.exllama import Ex4bitLinear + HAS_EXLLAMA = True + except ImportError: + pass from typing import Optional diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index ef662ce1..261456bd 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -170,10 +170,10 @@ def get_multi_weights_row(self, prefix: str, quantize: str): "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" ) - from text_generation_server.utils.layers import HAS_EXLLAMA + from text_generation_server.utils.layers import HAS_EXLLAMA, CAN_EXLLAMA if use_exllama: - if not HAS_EXLLAMA: + if not HAS_EXLLAMA and CAN_EXLLAMA: logger.warning( "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True" ) From 059bb5cf832221e3480eace71485c3b85dc8876d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E4=BD=B3=E6=AC=A3?= Date: Wed, 6 Sep 2023 21:20:32 +0800 Subject: [PATCH 408/793] chore: sync text-generation version from 0.3.0 to 0.6.0 with pyproject.toml (#950) # What does this PR do? sync the version for text-generation. --- clients/python/text_generation/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/python/text_generation/__init__.py b/clients/python/text_generation/__init__.py index 46109833..5ab10fdb 100644 --- a/clients/python/text_generation/__init__.py +++ b/clients/python/text_generation/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.3.0" +__version__ = "0.6.0" from text_generation.client import Client, AsyncClient from text_generation.inference_api import InferenceAPIClient, InferenceAPIAsyncClient From f260eb72f911bc30ae5d26020b6e14a774e5a168 Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Wed, 6 Sep 2023 16:36:49 +0300 Subject: [PATCH 409/793] docs: Flash Attention Conceptual Guide (#892) PR for conceptual guide on flash attention. I will add more info unless I'm told otherwise. --------- Co-authored-by: Nicolas Patry Co-authored-by: Omar Sanseviero --- docs/source/_toctree.yml | 2 ++ docs/source/conceptual/flash_attention.md | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 docs/source/conceptual/flash_attention.md diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 5ba470bd..6a8baaf6 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -21,4 +21,6 @@ - sections: - local: conceptual/streaming title: Streaming + - local: conceptual/flash_attention + title: Flash Attention title: Conceptual Guides diff --git a/docs/source/conceptual/flash_attention.md b/docs/source/conceptual/flash_attention.md new file mode 100644 index 00000000..1f3a6293 --- /dev/null +++ b/docs/source/conceptual/flash_attention.md @@ -0,0 +1,12 @@ +# Flash Attention + +Scaling the transformer architecture is heavily bottlenecked by the self-attention mechanism, which has quadratic time and memory complexity. Recent developments in accelerator hardware mainly focus on enhancing compute capacities and not memory and transferring data between hardware. This results in attention operation having a memory bottleneck. **Flash Attention** is an attention algorithm used to reduce this problem and scale transformer-based models more efficiently, enabling faster training and inference. + +Standard attention mechanism uses High Bandwidth Memory (HBM) to store, read and write keys, queries and values. HBM is large in memory, but slow in processing, meanwhile SRAM is smaller in memory, but faster in operations. In the standard attention implementation, the cost of loading and writing keys, queries, and values from HBM is high. It loads keys, queries, and values from HBM to GPU on-chip SRAM, performs a single step of the attention mechanism, writes it back to HBM, and repeats this for every single attention step. Instead, Flash Attention loads keys, queries, and values once, fuses the operations of the attention mechanism, and writes them back. + +![Flash Attention](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/tgi/flash-attn.png) + +It is implemented for supported models. You can check out the complete list of models that support Flash Attention [here](https://github.com/huggingface/text-generation-inference/tree/main/server/text_generation_server/models), for models with flash prefix. + +You can learn more about Flash Attention by reading the paper in this [link](https://arxiv.org/abs/2205.14135). + From a9fdfb24643c64a17cdb90560988f131d1dc0863 Mon Sep 17 00:00:00 2001 From: Omar Sanseviero Date: Wed, 6 Sep 2023 18:42:42 +0200 Subject: [PATCH 410/793] docs: Remove redundant content from stream guide (#884) Co-authored-by: OlivierDehaene --- docs/source/conceptual/streaming.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/conceptual/streaming.md b/docs/source/conceptual/streaming.md index 054fde30..b7e75c5f 100644 --- a/docs/source/conceptual/streaming.md +++ b/docs/source/conceptual/streaming.md @@ -143,6 +143,4 @@ SSEs are different than: * Polling: where the client keeps calling the server to get data. This means that the server might return empty responses and cause overhead. * Webhooks: where there is a bi-directional connection. The server can send information to the client, but the client can also send data to the server after the first request. Webhooks are more complex to operate as they don’t only use HTTP. -One of the limitations of Server-Sent Events is that they limit how many concurrent requests can handle by the server. Instead of timing out when there are too many SSE connections, TGI returns a HTTP Error with an `overloaded` error type (`huggingface_hub` returns `OverloadedError`). This allows the client to manage the overloaded server (e.g. it could display a busy error to the user or it could retry with a new request). To configure the maximum number of concurrent requests, you can specify `--max_concurrent_requests`, allowing to handle backpressure. - -One of the limitations of Server-Sent Events is that they limit how many concurrent requests can handle by the server. Instead of timing out when there are too many SSE connections, TGI returns an HTTP Error with an `overloaded` error type (`huggingface_hub` returns `OverloadedError`). This allows the client to manage the overloaded server (e.g., it could display a busy error to the user or retry with a new request). To configure the maximum number of concurrent requests, you can specify `--max_concurrent_requests`, allowing clients to handle backpressure. +If there are too many requests at the same time, TGI returns an HTTP Error with an `overloaded` error type (`huggingface_hub` returns `OverloadedError`). This allows the client to manage the overloaded server (e.g., it could display a busy error to the user or retry with a new request). To configure the maximum number of concurrent requests, you can specify `--max_concurrent_requests`, allowing clients to handle backpressure. From 935a77fb74d7e5e31579b7bf3c9263c23d6dd17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxime=20Laboissonni=C3=A8re?= Date: Thu, 7 Sep 2023 03:17:22 -0400 Subject: [PATCH 411/793] Fix exllama wronfully loading (#990) # What does this PR do? The [changes](https://github.com/huggingface/text-generation-inference/pull/986/files#diff-b72e45030214e50c8ff6e3be837057b3f3368b9779fd942ca680f949fe069eafR176) disabling exllama on old compute had unintended consequences of not setting `use_exllama` to `False` if `HAS_EXLLAMA` equals `False` **and** `CAN_EXLLAMA` equals `False`. This fixes this. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [X] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? @OlivierDehaene @Narsil Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/weights.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 261456bd..71cc1971 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -173,10 +173,11 @@ def get_multi_weights_row(self, prefix: str, quantize: str): from text_generation_server.utils.layers import HAS_EXLLAMA, CAN_EXLLAMA if use_exllama: - if not HAS_EXLLAMA and CAN_EXLLAMA: - logger.warning( - "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True" - ) + if not HAS_EXLLAMA: + if CAN_EXLLAMA: + logger.warning( + "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True" + ) use_exllama = False else: logger.info("Using exllama kernels") From b03d2621a79f5bd13cba0029879c8671cdd2a0d0 Mon Sep 17 00:00:00 2001 From: Florian Zimmermeister Date: Thu, 7 Sep 2023 10:19:42 +0200 Subject: [PATCH 412/793] add transformers gptq support (#963) Proposal to fix https://github.com/huggingface/text-generation-inference/issues/962 --- .../text_generation_server/utils/weights.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 71cc1971..9d47a7d3 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -223,7 +223,7 @@ def _get_gptq_params(self) -> Tuple[int, int]: return bits, groupsize def _set_gptq_params(self, model_id): - filename = "quantize_config.json" + filename = "config.json" try: if os.path.exists(os.path.join(model_id, filename)): filename = os.path.join(model_id, filename) @@ -231,7 +231,18 @@ def _set_gptq_params(self, model_id): filename = hf_hub_download(model_id, filename=filename) with open(filename, "r") as f: data = json.load(f) - self.gptq_bits = data["bits"] - self.gptq_groupsize = data["group_size"] + self.gptq_bits = data["quantization_config"]["bits"] + self.gptq_groupsize = data["quantization_config"]["group_size"] except Exception: - pass + filename = "quantize_config.json" + try: + if os.path.exists(os.path.join(model_id, filename)): + filename = os.path.join(model_id, filename) + else: + filename = hf_hub_download(model_id, filename=filename) + with open(filename, "r") as f: + data = json.load(f) + self.gptq_bits = data["bits"] + self.gptq_groupsize = data["group_size"] + except Exception: + pass From af1ed38f39dd550610badfd371830607115df7cc Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Thu, 7 Sep 2023 17:22:06 +0300 Subject: [PATCH 413/793] Safetensors conceptual guide (#905) IDK what else to add in this guide, I looked for relevant code in TGI codebase and saw that it's used in quantization as well (maybe I could add that?) --- docs/source/_toctree.yml | 2 ++ docs/source/conceptual/safetensors.md | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 docs/source/conceptual/safetensors.md diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 6a8baaf6..d37446b1 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -21,6 +21,8 @@ - sections: - local: conceptual/streaming title: Streaming + - local: conceptual/safetensors + title: Safetensors - local: conceptual/flash_attention title: Flash Attention title: Conceptual Guides diff --git a/docs/source/conceptual/safetensors.md b/docs/source/conceptual/safetensors.md new file mode 100644 index 00000000..fcc31bac --- /dev/null +++ b/docs/source/conceptual/safetensors.md @@ -0,0 +1,7 @@ +# Safetensors + +Safetensors is a model serialization format for deep learning models. It is [faster](https://huggingface.co/docs/safetensors/speed) and safer compared to other serialization formats like pickle (which is used under the hood in many deep learning libraries). + +TGI depends on safetensors format mainly to enable [tensor parallelism sharding](./tensor_parallelism). For a given model repository during serving, TGI looks for safetensors weights. If there are no safetensors weights, TGI converts the PyTorch weights to safetensors format. + +You can learn more about safetensors by reading the [safetensors documentation](https://huggingface.co/docs/safetensors/index). \ No newline at end of file From 0a63e9ab688cf715d31574ee5bb31025ff22ceec Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 7 Sep 2023 17:36:30 +0200 Subject: [PATCH 414/793] Fix __call__ vs forward. (#993) # What does this PR do? Fix __call__ vs forward. To reproduce error just launch: TheBloke/WizardLM-Uncensored-Falcon-7B-GPTQ with gptq (it fails because falcon code uses `__call__` instead for `forward` calls) Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/gptq/exllama.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/gptq/exllama.py b/server/text_generation_server/utils/gptq/exllama.py index 6a1cf117..7353afb5 100644 --- a/server/text_generation_server/utils/gptq/exllama.py +++ b/server/text_generation_server/utils/gptq/exllama.py @@ -69,10 +69,11 @@ def create_exllama_buffers(): TEMP_STATE, TEMP_DQ = temp_state, temp_dq -class Ex4bitLinear: +class Ex4bitLinear(torch.nn.Module): """Linear layer implementation with per-group 4-bit quantization of the weights""" def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): + super().__init__() global MAX_DQ, MAX_INNER, ACT_ORDER, DEVICE assert bits == 4 From 30a93a0dec119760f8591d756b8010b4792bfe6d Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Fri, 8 Sep 2023 15:18:42 +0300 Subject: [PATCH 415/793] Paged Attention Conceptual Guide (#901) --- docs/source/_toctree.yml | 2 ++ docs/source/conceptual/paged_attention.md | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 docs/source/conceptual/paged_attention.md diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index d37446b1..0ae8c69e 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -21,6 +21,8 @@ - sections: - local: conceptual/streaming title: Streaming + - local: conceptual/paged_attention + title: PagedAttention - local: conceptual/safetensors title: Safetensors - local: conceptual/flash_attention diff --git a/docs/source/conceptual/paged_attention.md b/docs/source/conceptual/paged_attention.md new file mode 100644 index 00000000..3fb2dcd8 --- /dev/null +++ b/docs/source/conceptual/paged_attention.md @@ -0,0 +1,9 @@ +# PagedAttention + +LLMs struggle with memory limitations during generation. In the decoding part of generation, all the attention keys and values generated for previous tokens are stored in GPU memory for reuse. This is called _KV cache_, and it may take up a large amount of memory for large models and long sequences. + +PagedAttention attempts to optimize memory use by partitioning the KV cache into blocks that are accessed through a lookup table. Thus, the KV cache does not need to be stored in contiguous memory, and blocks are allocated as needed. The memory efficiency can increase GPU utilization on memory-bound workloads, so more inference batches can be supported. + +The use of a lookup table to access the memory blocks can also help with KV sharing across multiple generations. This is helpful for techniques such as _parallel sampling_, where multiple outputs are generated simultaneously for the same prompt. In this case, the cached KV blocks can be shared among the generations. + +TGI's PagedAttention implementation leverages the custom cuda kernels developed by the [vLLM Project](https://github.com/vllm-project/vllm). You can learn more about this technique in the [project's page](https://vllm.ai/). From 4cce84301b9f0f3472cf8b54e3119afd87caa09e Mon Sep 17 00:00:00 2001 From: xiaobin <737489727@qq.com> Date: Fri, 8 Sep 2023 22:51:34 +0800 Subject: [PATCH 416/793] fit for baichuan models (#981) As more and more people begin to use Baichuan's open-source models, the influence of Baichuan models is growing, especially in China. Many community members are interested in adding support for Baichuan models to TGI. Meanwhile, Baichuan is a very open company, and in the future, it plans to open-source more and more models, taking all this into consideration, we would like to add support for the Baichuan model to TGI. To do this, we need to make some changes, which we hope can be merged into the main branch of TGI. In the future, we would be happy to help maintain support for Baichuan models in TGI. We sincerely hope that our pull request can be accepted. Thank you. By the way, the changes of this time mainly for supporting Baichuan-7B. --------- Co-authored-by: xiaoyuze Co-authored-by: Nicolas Patry --- .../text_generation_server/models/__init__.py | 2 +- .../custom_modeling/flash_llama_modeling.py | 33 ++++--- .../text_generation_server/utils/convert.py | 12 ++- server/text_generation_server/utils/layers.py | 13 +++ .../text_generation_server/utils/weights.py | 85 ++++++++++++++++++- 5 files changed, 127 insertions(+), 18 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 932ab32e..e6fe1372 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -182,7 +182,7 @@ def get_model( trust_remote_code=trust_remote_code, ) - elif model_type == "llama": + elif model_type == "llama" or model_type == "baichuan": if FLASH_ATTENTION: return FlashLlama( model_id, diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index f0e1236d..55b1aae9 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -149,6 +149,26 @@ def forward(self, hidden_states, residual=None): return normed_hidden_states, res +def load_attention(config, prefix, weights): + if config.num_attention_heads != config.num_key_value_heads: + return _load_gqa(config, prefix, weights) + else: + if config.model_type == "baichuan": + return TensorParallelColumnLinear.load_qkv( + config, + prefix=f"{prefix}.W_pack", + weights=weights, + bias=False, + ) + else: + return TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + dim=0, + weights=weights, + bias=False, + ) + def _load_gqa(config, prefix: str, weights): assert config.hidden_size % config.num_attention_heads == 0 assert config.num_attention_heads % weights.process_group.size() == 0 @@ -205,16 +225,9 @@ def __init__( self.num_key_value_heads = ( config.num_key_value_heads // weights.process_group.size() ) - if config.num_attention_heads != config.num_key_value_heads: - self.query_key_value = _load_gqa(config, prefix, weights) - else: - self.query_key_value = TensorParallelColumnLinear.load_multi( - config, - prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], - dim=0, - weights=weights, - bias=False, - ) + + self.query_key_value = load_attention(config, prefix, weights) + self.o_proj = TensorParallelRowLinear.load( config, prefix=f"{prefix}.o_proj", diff --git a/server/text_generation_server/utils/convert.py b/server/text_generation_server/utils/convert.py index 8d414eca..0b62f520 100644 --- a/server/text_generation_server/utils/convert.py +++ b/server/text_generation_server/utils/convert.py @@ -29,9 +29,15 @@ def _remove_duplicate_names( [name for name in shared if _is_complete(state_dict[name])] ) if not complete_names: - raise RuntimeError( - f"Error while trying to find names to remove to save state dict, but found no suitable name to keep for saving amongst: {shared}. None is covering the entire storage.Refusing to save/load the model since you could be storing much more memory than needed. Please refer to https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an issue." - ) + if len(shared) == 1: + # Force contiguous + name = list(shared)[0] + state_dict[name] = state_dict[name].clone() + complete_names = {name} + else: + raise RuntimeError( + f"Error while trying to find names to remove to save state dict, but found no suitable name to keep for saving amongst: {shared}. None is covering the entire storage.Refusing to save/load the model since you could be storing much more memory than needed. Please refer to https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an issue." + ) keep_name = sorted(list(complete_names))[0] diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 6be54048..c1c36194 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -331,6 +331,19 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: class TensorParallelColumnLinear(SuperLayer): + @classmethod + def load_qkv(cls, config, prefix: str, weights, bias: bool): + """Specific method when the QKV was joined after the fact""" + weight = weights.get_weights_col_packed_qkv( + prefix, quantize=config.quantize + ) + if bias: + raise NotImplementedError("packed_qkv only implemented for baichuan") + else: + bias = None + linear = get_linear(weight, bias, config.quantize) + return cls(linear) + @classmethod def load(cls, config, prefix: str, weights, bias: bool): return cls.load_multi(config, [prefix], weights, bias, dim=0) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 9d47a7d3..2ef7ad39 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -62,7 +62,7 @@ def _get_slice(self, tensor_name: str): def get_shape(self, tensor_name: str): return self._get_slice(tensor_name).get_shape() - def get_tensor(self, tensor_name: str): + def get_tensor(self, tensor_name: str, to_device = True): filename, tensor_name = self.get_filename(tensor_name) f = self._get_handle(filename) tensor = f.get_tensor(tensor_name) @@ -70,16 +70,17 @@ def get_tensor(self, tensor_name: str): # u4 which are disguised as int32 if tensor.dtype not in [torch.int32, torch.int64]: tensor = tensor.to(dtype=self.dtype) - tensor = tensor.to(device=self.device) + if to_device: + tensor = tensor.to(device=self.device) return tensor def get_partial_sharded(self, tensor_name: str, dim: int): filename, tensor_name = self.get_filename(tensor_name) + f = self._get_handle(filename) + slice_ = f.get_slice(tensor_name) world_size = self.process_group.size() rank = self.process_group.rank() - f = self._get_handle(filename) - slice_ = f.get_slice(tensor_name) size = slice_.get_shape()[dim] block_size = size // world_size start = rank * block_size @@ -109,6 +110,66 @@ def get_sharded(self, tensor_name: str, dim: int): ), f"The choosen size {size} is not compatible with sharding on {world_size} shards" return self.get_partial_sharded(tensor_name, dim) + + def _get_qweight(self, name: str): + slice_ = self._get_slice(name) + total_size = slice_.get_shape()[1] + assert total_size % 3 == 0, "Prepacked quantized qkv is not divisible by 3" + single_size = total_size // 3 + world_size = self.process_group.size() + rank = self.process_group.rank() + + assert single_size % world_size == 0, f"Prepacked quantized qkv cannot be sharded across {world_size} shards" + block_size = single_size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + q = slice_[:, start:stop] + k = slice_[:, start+single_size:stop+single_size] + v = slice_[:, start+2*single_size:stop+2*single_size] + weight = torch.cat([q,k,v], dim=1) + weight = weight.to(device=self.device) + return weight + + def get_weights_col_packed_qkv(self, prefix: str, quantize: str): + """ + Highly specific when the underlying tensor is a simple cat of Q,K,V instead of being + already alternating Q,K,V within the main tensor + """ + if quantize == "gptq": + try: + qweight = self._get_qweight(f"{prefix}.qweight") + except RuntimeError: + raise RuntimeError( + "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" + ) + + qzeros = self._get_qweight(f"{prefix}.qzeros") + scales = self._get_qweight(f"{prefix}.scales") + scales = scales.to(dtype=self.dtype) + g_idx = self.get_tensor(f"{prefix}.g_idx") + + bits, groupsize = self._get_gptq_params() + weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) + else: + slice_ = self._get_slice(f"{prefix}.weight") + total_size = slice_.get_shape()[0] + assert total_size % 3 == 0, "Prepacked qkv is not divisible by 3" + single_size = total_size // 3 + world_size = self.process_group.size() + rank = self.process_group.rank() + + assert single_size % world_size == 0, f"Prepacked qkv cannot be sharded across {world_size} shards" + block_size = single_size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + q = slice_[start:stop] + k = slice_[start+single_size:stop+single_size] + v = slice_[start+2*single_size:stop+2*single_size] + weight = torch.cat([q,k,v], dim=0) + weight = weight.to(device=self.device) + weight = weight.to(dtype=self.dtype) + return weight + def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): if quantize == "gptq": try: @@ -137,6 +198,22 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] weight = torch.cat(w, dim=dim) return weight + + def get_tensor_shard(self, var, dim): + world_size = self.process_group.size() + rank = self.process_group.rank() + block_size = var.size()[dim] // world_size + start = rank * block_size + stop = (rank + 1) * block_size + if dim == 0: + tensor = var[start:stop] + elif dim == 1: + tensor = var[:, start:stop] + else: + raise NotImplementedError("Let's make that generic when needed") + tensor = tensor.to(dtype=self.dtype) + tensor = tensor.to(device=self.device) + return tensor def get_multi_weights_row(self, prefix: str, quantize: str): if quantize == "gptq": From 1f69fb9ed4fb91fe0bb9b94edda5729c67e6f02a Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Tue, 12 Sep 2023 12:11:20 +0200 Subject: [PATCH 417/793] Tensor Parallelism conceptual guide (#886) Co-authored-by: Nicolas Patry Co-authored-by: Omar Sanseviero Co-authored-by: Pedro Cuenca --- docs/source/_toctree.yml | 2 ++ docs/source/conceptual/tensor_parallelism.md | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 docs/source/conceptual/tensor_parallelism.md diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 0ae8c69e..4a81f48f 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -21,6 +21,8 @@ - sections: - local: conceptual/streaming title: Streaming + - local: conceptual/tensor_parallelism + title: Tensor Parallelism - local: conceptual/paged_attention title: PagedAttention - local: conceptual/safetensors diff --git a/docs/source/conceptual/tensor_parallelism.md b/docs/source/conceptual/tensor_parallelism.md new file mode 100644 index 00000000..886a349a --- /dev/null +++ b/docs/source/conceptual/tensor_parallelism.md @@ -0,0 +1,14 @@ +# Tensor Parallelism + +Tensor parallelism is a technique used to fit a large model in multiple GPUs. For example, when multiplying the input tensors with the first weight tensor, the matrix multiplication is equivalent to splitting the weight tensor column-wise, multiplying each column with the input separately, and then concatenating the separate outputs. These outputs are then transferred from the GPUs and concatenated together to get the final result, like below 👇 + +![Image courtesy of Anton Lozkhov](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/tgi/TP.png) + + + + +Tensor Parallelism only works for [models officially supported](../supported_models), it will not work when falling back to `transformers`. You can get more information about unsupported models [here](../basic_tutorials/non_core_models). + + + +You can learn a lot more details about tensor-parallelism from [the `transformers` docs](https://huggingface.co/docs/transformers/main/en/perf_train_gpu_many#tensor-parallelism). From e9ae678699f20eac30ad60be539838ceb2ac248b Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Tue, 12 Sep 2023 15:52:46 +0200 Subject: [PATCH 418/793] Quantization docs (#911) Co-authored-by: Nicolas Patry Co-authored-by: Pedro Cuenca --- docs/source/_toctree.yml | 2 + .../source/basic_tutorials/preparing_model.md | 2 +- docs/source/conceptual/quantization.md | 59 +++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 docs/source/conceptual/quantization.md diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 4a81f48f..25f3815e 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -21,6 +21,8 @@ - sections: - local: conceptual/streaming title: Streaming + - local: conceptual/quantization + title: Quantization - local: conceptual/tensor_parallelism title: Tensor Parallelism - local: conceptual/paged_attention diff --git a/docs/source/basic_tutorials/preparing_model.md b/docs/source/basic_tutorials/preparing_model.md index 65a2a197..6b622d99 100644 --- a/docs/source/basic_tutorials/preparing_model.md +++ b/docs/source/basic_tutorials/preparing_model.md @@ -4,7 +4,7 @@ Text Generation Inference improves the model in several aspects. ## Quantization -TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes) and [GPT-Q](https://arxiv.org/abs/2210.17323) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes` or `gptq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq). +TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes) and [GPT-Q](https://arxiv.org/abs/2210.17323) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes` or `gptq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq). To get more information about quantization, please refer to (./conceptual/quantization.md) ## RoPE Scaling diff --git a/docs/source/conceptual/quantization.md b/docs/source/conceptual/quantization.md new file mode 100644 index 00000000..1a44e3c2 --- /dev/null +++ b/docs/source/conceptual/quantization.md @@ -0,0 +1,59 @@ +# Quantization + +TGI offers GPTQ and bits-and-bytes quantization to quantize large language models. + +## Quantization with GPTQ + +GPTQ is a post-training quantization method to make the model smaller. It quantizes the layers by finding a compressed version of that weight, that will yield a minimum mean squared error like below 👇 + +Given a layer \\(l\\) with weight matrix \\(W_{l}\\) and layer input \\(X_{l}\\), find quantized weight \\(\\hat{W}_{l}\\): + +$$({\hat{W}_{l}}^{*} = argmin_{\hat{W_{l}}} ||W_{l}X-\hat{W}_{l}X||^{2}_{2})$$ + + +TGI allows you to both run an already GPTQ quantized model (see available models [here](https://huggingface.co/models?search=gptq)) or quantize a model of your choice using quantization script. You can run a quantized model by simply passing --quantize like below 👇 + +```bash +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize gptq +``` + +Note that TGI's GPTQ implementation doesn't use [AutoGPTQ](https://github.com/PanQiWei/AutoGPTQ) under the hood. However, models quantized using AutoGPTQ or Optimum can still be served by TGI. + +To quantize a given model using GPTQ with a calibration dataset, simply run + +```bash +text-generation-server quantize tiiuae/falcon-40b /data/falcon-40b-gptq +# Add --upload-to-model-id MYUSERNAME/falcon-40b to push the created model to the hub directly +``` + +This will create a new directory with the quantized files which you can use with, + +```bash +text-generation-launcher --model-id /data/falcon-40b-gptq/ --sharded true --num-shard 2 --quantize gptq +``` + +You can learn more about the quantization options by running `text-generation-server quantize --help`. + +If you wish to do more with GPTQ models (e.g. train an adapter on top), you can read about transformers GPTQ integration [here](https://huggingface.co/blog/gptq-integration). +You can learn more about GPTQ from the [paper](https://arxiv.org/pdf/2210.17323.pdf). + +## Quantization with bitsandbytes + +bitsandbytes is a library used to apply 8-bit and 4-bit quantization to models. Unlike GPTQ quantization, bitsandbytes doesn't require a calibration dataset or any post-processing – weights are automatically quantized on load. However, inference with bitsandbytes is slower than GPTQ or FP16 precision. + +8-bit quantization enables multi-billion parameter scale models to fit in smaller hardware without degrading performance too much. +In TGI, you can use 8-bit quantization by adding `--quantize bitsandbytes` like below 👇 + +```bash +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize --bitsandbytes +``` + +4-bit quantization is also possible with bitsandbytes. You can choose one of the following 4-bit data types: 4-bit float (`fp4`), or 4-bit `NormalFloat` (`nf4`). These data types were introduced in the context of parameter-efficient fine-tuning, but you can apply them for inference by automatically converting the model weights on load. + +In TGI, you can use 4-bit quantization by adding `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` like below 👇 + +```bash +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize --bitsandbytes-nf4 +``` + +You can get more information about 8-bit quantization by reading this [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), and 4-bit quantization by reading [this blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes). From c8a01d759173483efc2135c4e7506b23e14e7fc4 Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Tue, 12 Sep 2023 15:55:14 +0200 Subject: [PATCH 419/793] Unsupported model serving docs (#906) Co-authored-by: Omar Sanseviero Co-authored-by: Mishig Co-authored-by: Pedro Cuenca Co-authored-by: OlivierDehaene --- docs/source/_toctree.yml | 2 ++ .../source/basic_tutorials/non_core_models.md | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 docs/source/basic_tutorials/non_core_models.md diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 25f3815e..313b6d32 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -17,6 +17,8 @@ title: Serving Private & Gated Models - local: basic_tutorials/using_cli title: Using TGI CLI + - local: basic_tutorials/non_core_models + title: Non-core Model Serving title: Tutorials - sections: - local: conceptual/streaming diff --git a/docs/source/basic_tutorials/non_core_models.md b/docs/source/basic_tutorials/non_core_models.md new file mode 100644 index 00000000..6f2e6cfa --- /dev/null +++ b/docs/source/basic_tutorials/non_core_models.md @@ -0,0 +1,24 @@ +# Non-core Model Serving + +TGI supports various LLM architectures (see full list [here](../supported_models)). If you wish to serve a model that is not one of the supported models, TGI will fallback to the `transformers` implementation of that model. This means you will be unable to use some of the features introduced by TGI, such as tensor-parallel sharding or flash attention. However, you can still get many benefits of TGI, such as continuous batching or streaming outputs. + +You can serve these models using the same Docker command-line invocation as with fully supported models 👇 + +```bash +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id gpt2 +``` + +If the model you wish to serve is a custom transformers model, and its weights and implementation are available in the Hub, you can still serve the model by passing the `--trust-remote-code` flag to the `docker run` command like below 👇 + +```bash +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id --trust-remote-code +``` + +Finally, if the model is not on Hugging Face Hub but on your local, you can pass the path to the folder that contains your model like below 👇 + +```bash +# Make sure your model is in the $volume directory +docker run --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id /data/ +``` + +You can refer to [transformers docs on custom models](https://huggingface.co/docs/transformers/main/en/custom_models) for more information. From eeaa22ab044a5df8a570f08d2c4d971e51829b1f Mon Sep 17 00:00:00 2001 From: "Wang, Yi" Date: Tue, 19 Sep 2023 23:19:28 +0800 Subject: [PATCH 420/793] enable bfloat16 for cpu (#1034) if there's no cuda. disable custom kernels # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. Signed-off-by: Wang, Yi A --- server/text_generation_server/models/__init__.py | 2 +- server/text_generation_server/models/bloom.py | 2 +- server/text_generation_server/models/causal_lm.py | 2 +- .../models/custom_modeling/bloom_modeling.py | 2 +- .../models/custom_modeling/neox_modeling.py | 2 +- server/text_generation_server/models/galactica.py | 2 +- server/text_generation_server/models/gpt_neox.py | 2 +- server/text_generation_server/models/idefics.py | 2 +- server/text_generation_server/models/idefics_causal_lm.py | 2 +- server/text_generation_server/models/mpt.py | 6 ++++-- server/text_generation_server/models/opt.py | 2 +- server/text_generation_server/models/rw.py | 2 +- server/text_generation_server/models/santacoder.py | 2 +- server/text_generation_server/models/seq2seq_lm.py | 2 +- server/text_generation_server/models/t5.py | 2 +- 15 files changed, 18 insertions(+), 16 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index e6fe1372..29b049cf 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -153,7 +153,7 @@ def get_model( ) elif model_type == "mpt": return MPTSharded( - model_id, revision, quantize=quantize, trust_remote_code=trust_remote_code + model_id, revision, quantize=quantize, dtype=dtype, trust_remote_code=trust_remote_code ) elif model_type == "gpt_neox": diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 79fb60c6..0151b017 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -51,7 +51,7 @@ def __init__( dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype tokenizer = AutoTokenizer.from_pretrained( model_id, diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 4e338263..cec9ae55 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -492,7 +492,7 @@ def __init__( raise ValueError("quantization is not available on CPU") device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype tokenizer = AutoTokenizer.from_pretrained( model_id, diff --git a/server/text_generation_server/models/custom_modeling/bloom_modeling.py b/server/text_generation_server/models/custom_modeling/bloom_modeling.py index 047a1872..297d5c68 100644 --- a/server/text_generation_server/models/custom_modeling/bloom_modeling.py +++ b/server/text_generation_server/models/custom_modeling/bloom_modeling.py @@ -40,7 +40,7 @@ ) CUSTOM_KERNELS_ENABLED = False -if not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True": +if torch.cuda.is_available() and not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True": try: from custom_kernels import fused_bloom_attention_cuda diff --git a/server/text_generation_server/models/custom_modeling/neox_modeling.py b/server/text_generation_server/models/custom_modeling/neox_modeling.py index 1951b171..c5b0c7fd 100644 --- a/server/text_generation_server/models/custom_modeling/neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/neox_modeling.py @@ -49,7 +49,7 @@ CUSTOM_KERNELS_ENABLED = False -if not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True": +if torch.cuda.is_available() and not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True": try: from custom_kernels import fused_attention_cuda diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index d4211734..cfd5e658 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -167,7 +167,7 @@ def __init__( dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype tokenizer = AutoTokenizer.from_pretrained( model_id, diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index accedf14..d4c64dfe 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -33,7 +33,7 @@ def __init__( dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype tokenizer = AutoTokenizer.from_pretrained( model_id, diff --git a/server/text_generation_server/models/idefics.py b/server/text_generation_server/models/idefics.py index c54b539b..fa23d1f9 100644 --- a/server/text_generation_server/models/idefics.py +++ b/server/text_generation_server/models/idefics.py @@ -42,7 +42,7 @@ def __init__( dtype = torch.bfloat16 if dtype is None else dtype else: device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype self.device, self.dtype = device, dtype config = IdeficsConfig.from_pretrained( diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py index 2dac87bc..f4177145 100644 --- a/server/text_generation_server/models/idefics_causal_lm.py +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -560,7 +560,7 @@ def __init__( raise ValueError("quantization is not available on CPU") device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype tokenizer = AutoTokenizer.from_pretrained( model_id, diff --git a/server/text_generation_server/models/mpt.py b/server/text_generation_server/models/mpt.py index 909d9852..19de497c 100644 --- a/server/text_generation_server/models/mpt.py +++ b/server/text_generation_server/models/mpt.py @@ -43,14 +43,16 @@ def __init__( model_id: str, revision: Optional[str] = None, quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): device = torch.device(f"cuda:{rank}") - dtype = torch.float16 + dtype = torch.float16 if dtype is None else dtype else: - raise NotImplementedError("MPTSharded is only available on GPU") + device = torch.device("cpu") + dtype = torch.float32 if dtype is None else dtype tokenizer = AutoTokenizer.from_pretrained( model_id, diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index f3a23d07..b2b87246 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -31,7 +31,7 @@ def __init__( dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype tokenizer = AutoTokenizer.from_pretrained( model_id, diff --git a/server/text_generation_server/models/rw.py b/server/text_generation_server/models/rw.py index d97c1c73..802a4aa6 100644 --- a/server/text_generation_server/models/rw.py +++ b/server/text_generation_server/models/rw.py @@ -23,7 +23,7 @@ def __init__( raise ValueError("quantization is not available on CPU") device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype tokenizer = AutoTokenizer.from_pretrained( model_id, diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index 81928c1d..7b269d8e 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -30,7 +30,7 @@ def __init__( raise ValueError("quantization is not available on CPU") device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype tokenizer = AutoTokenizer.from_pretrained( model_id, diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 361453fb..1a7911ac 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -541,7 +541,7 @@ def __init__( raise ValueError("quantization is not available on CPU") device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype model = AutoModelForSeq2SeqLM.from_pretrained( model_id, diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py index 133aafd8..161e69ba 100644 --- a/server/text_generation_server/models/t5.py +++ b/server/text_generation_server/models/t5.py @@ -34,7 +34,7 @@ def __init__( dtype = torch.float16 if dtype is None else dtype else: device = torch.device("cpu") - dtype = torch.float32 + dtype = torch.float32 if dtype is None else dtype config = AutoConfig.from_pretrained( model_id, From 123749a3c999e32db798667041a4a9589d217c8e Mon Sep 17 00:00:00 2001 From: Vincent Brouwers Date: Thu, 21 Sep 2023 08:15:59 +0200 Subject: [PATCH 421/793] Fix missing arguments in Galactica's from_pb (#1022) # What does this PR do? Fixes #1004 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/models/galactica.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index cfd5e658..b296c96e 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -80,6 +80,7 @@ def from_pb( next_token_choosers = [] stopping_criterias = [] prefix_offsets = [] + top_n_tokens = [] read_offsets = [] requests_idx_mapping = {} @@ -96,6 +97,7 @@ def from_pb( r.stopping_parameters, tokenizer ) stopping_criterias.append(stopping_criteria) + top_n_tokens.append(r.top_n_tokens) max_truncation = max(max_truncation, r.truncate) max_decode_tokens += stopping_criteria.max_new_tokens padding_right_offset = max( @@ -129,6 +131,9 @@ def from_pb( position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) + top_n_tokens_tensor = torch.tensor( + top_n_tokens, device=device, dtype=torch.int64 + ) max_tokens = len(inputs) * max_input_length + max_decode_tokens @@ -146,6 +151,8 @@ def from_pb( read_offsets=read_offsets, next_token_choosers=next_token_choosers, stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, max_input_length=max_input_length.item(), padding_right_offset=padding_right_offset, max_tokens=max_tokens, From fef36cea4254a3afe9a8b97ea10f40376ceef91f Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 25 Sep 2023 12:22:28 +0200 Subject: [PATCH 422/793] Fixing t5 loading. (#1042) # What does this PR do? Fixes #1038 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../models/custom_modeling/t5_modeling.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/t5_modeling.py b/server/text_generation_server/models/custom_modeling/t5_modeling.py index 793f3a66..d3e4f53a 100644 --- a/server/text_generation_server/models/custom_modeling/t5_modeling.py +++ b/server/text_generation_server/models/custom_modeling/t5_modeling.py @@ -1032,9 +1032,17 @@ def __init__(self, config: T5Config, weights): embed_tokens=self.shared, ) - self.lm_head = TensorParallelHead.load( - config, prefix="lm_head", weights=weights - ) + try: + self.lm_head = TensorParallelHead.load( + config, prefix="lm_head", weights=weights + ) + except RuntimeError: + # Some models like t5-small were saved with shared weights unlike flan + # Since they are declared as the same arch we have no choice but hope + # that this is OK instead of using a proper flag. + self.lm_head = TensorParallelHead.load( + config, prefix="shared", weights=weights + ) def forward( self, From c5de7cd88679bc0331185c9cee75e4f68412243d Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 25 Sep 2023 15:31:27 +0200 Subject: [PATCH 423/793] Add AWQ quantization inference support (#1019) (#1054) # Add AWQ quantization inference support Fixes https://github.com/huggingface/text-generation-inference/issues/781 This PR (partially) adds support for AWQ quantization for inference. More information on AWQ [here](https://arxiv.org/abs/2306.00978). In general, AWQ is faster and more accurate than GPTQ, which is currently supported by TGI. This PR installs 4-bit GEMM custom CUDA kernels released by AWQ authors (in `requirements.txt`, just one line change). Quick way to test this PR would be bring up TGI as follows: ``` text-generation-server download-weights abhinavkulkarni/codellama-CodeLlama-7b-Python-hf-w4-g128-awq text-generation-launcher \ --huggingface-hub-cache ~/.cache/huggingface/hub/ \ --model-id abhinavkulkarni/codellama-CodeLlama-7b-Python-hf-w4-g128-awq \ --trust-remote-code --port 8080 \ --max-input-length 2048 --max-total-tokens 4096 --max-batch-prefill-tokens 4096 \ --quantize awq ``` Please note: * This PR was tested with FlashAttention v2 and vLLM. * This PR adds support for AWQ inference, not quantizing the models. That needs to be done outside of TGI, instructions [here](https://github.com/mit-han-lab/llm-awq/tree/f084f40bd996f3cf3a0633c1ad7d9d476c318aaa). * This PR only adds support for `FlashLlama` models for now. * Multi-GPU setup has not been tested. * No integration tests have been added so far, will add later if maintainers are interested in this change. * This PR can be tested on any of the models released [here](https://huggingface.co/abhinavkulkarni?sort_models=downloads#models). Please refer to the linked issue for benchmarks for [abhinavkulkarni/meta-llama-Llama-2-7b-chat-hf-w4-g128-awq](https://huggingface.co/abhinavkulkarni/meta-llama-Llama-2-7b-chat-hf-w4-g128-awq) vs [TheBloke/Llama-2-7b-Chat-GPTQ](https://huggingface.co/TheBloke/Llama-2-7b-Chat-GPTQ). Please note, AWQ has released faster (and in case of Llama, fused) kernels for 4-bit GEMM, currently at the top of the `main` branch at https://github.com/mit-han-lab/llm-awq, but this PR uses an older commit that has been tested to work. We can switch to latest commit later on. ## Who can review? @OlivierDehaene OR @Narsil --------- # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Abhinav M Kulkarni Co-authored-by: Abhinav Kulkarni --- Dockerfile | 16 +- .../source/basic_tutorials/preparing_model.md | 2 +- .../test_flash_awq/test_flash_llama_awq.json | 104 +++ .../test_flash_llama_awq_all_params.json | 99 +++ .../test_flash_llama_awq_load.json | 418 ++++++++++++ .../test_flash_llama_awq_load_sharded.json | 418 ++++++++++++ .../test_flash_llama_awq_sharded.json | 104 +++ integration-tests/models/test_flash_awq.py | 64 ++ .../models/test_flash_awq_sharded.py | 36 ++ launcher/src/main.rs | 4 + server/.gitignore | 1 + server/Makefile | 1 + server/Makefile-awq | 13 + server/poetry.lock | 594 +++++++++--------- server/pyproject.toml | 4 +- server/text_generation_server/cli.py | 1 + .../text_generation_server/models/__init__.py | 4 + .../models/flash_llama.py | 2 +- .../utils/awq/quantize/qmodule.py | 48 ++ server/text_generation_server/utils/layers.py | 19 +- .../text_generation_server/utils/weights.py | 54 +- 21 files changed, 1694 insertions(+), 312 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq.json create mode 100644 integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_load.json create mode 100644 integration-tests/models/__snapshots__/test_flash_awq_sharded/test_flash_llama_awq_load_sharded.json create mode 100644 integration-tests/models/__snapshots__/test_flash_awq_sharded/test_flash_llama_awq_sharded.json create mode 100644 integration-tests/models/test_flash_awq.py create mode 100644 integration-tests/models/test_flash_awq_sharded.py create mode 100644 server/Makefile-awq create mode 100644 server/text_generation_server/utils/awq/quantize/qmodule.py diff --git a/Dockerfile b/Dockerfile index 45e304c4..1d57d110 100644 --- a/Dockerfile +++ b/Dockerfile @@ -111,22 +111,22 @@ RUN make build-flash-attention-v2 # Build Transformers exllama kernels FROM kernel-builder as exllama-kernels-builder - WORKDIR /usr/src - COPY server/exllama_kernels/ . - - # Build specific version of transformers RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" python setup.py build +# Build Transformers awq kernels +FROM kernel-builder as awq-kernels-builder +WORKDIR /usr/src +COPY server/Makefile-awq Makefile +# Build specific version of transformers +RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" make build-awq + # Build Transformers CUDA kernels FROM kernel-builder as custom-kernels-builder - WORKDIR /usr/src - COPY server/custom_kernels/ . - # Build specific version of transformers RUN python setup.py build @@ -175,6 +175,8 @@ COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86 COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages # Copy build artifacts from exllama kernels builder COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +# Copy build artifacts from awq kernels builder +COPY --from=awq-kernels-builder /usr/src/llm-awq/awq/kernels/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages # Copy builds artifacts from vllm builder COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages diff --git a/docs/source/basic_tutorials/preparing_model.md b/docs/source/basic_tutorials/preparing_model.md index 6b622d99..0f5739ea 100644 --- a/docs/source/basic_tutorials/preparing_model.md +++ b/docs/source/basic_tutorials/preparing_model.md @@ -4,7 +4,7 @@ Text Generation Inference improves the model in several aspects. ## Quantization -TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes) and [GPT-Q](https://arxiv.org/abs/2210.17323) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes` or `gptq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq). To get more information about quantization, please refer to (./conceptual/quantization.md) +TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323) and [AWQ](https://arxiv.org/abs/2306.00978) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq` or `awq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq) when using AWQ quantization, you need to point to one of the models [here](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to (./conceptual/quantization.md) ## RoPE Scaling diff --git a/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq.json b/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq.json new file mode 100644 index 00000000..dcd37cb9 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq.json @@ -0,0 +1,104 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.703125, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4765625, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8583984, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7548828, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9306641, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4550781, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.5732422, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5761719, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5888672, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.026504517, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4287109, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.15856934, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17456055, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62646484, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" +} diff --git a/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_all_params.json b/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_all_params.json new file mode 100644 index 00000000..d16d34f9 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_all_params.json @@ -0,0 +1,99 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 338, + "logprob": -9.0859375, + "text": "is" + }, + { + "id": 21784, + "logprob": -10.90625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -2.65625, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -4.8085938, + "text": "?" + } + ], + "seed": 0, + "tokens": [ + { + "id": 13, + "logprob": -0.19958496, + "special": false, + "text": "\n" + }, + { + "id": 4013, + "logprob": -2.203125, + "special": false, + "text": "This" + }, + { + "id": 1139, + "logprob": -0.23693848, + "special": false, + "text": " question" + }, + { + "id": 756, + "logprob": 0.0, + "special": false, + "text": " has" + }, + { + "id": 1063, + "logprob": -0.076538086, + "special": false, + "text": " been" + }, + { + "id": 4433, + "logprob": 0.0, + "special": false, + "text": " asked" + }, + { + "id": 1784, + "logprob": -1.1367188, + "special": false, + "text": " many" + }, + { + "id": 3064, + "logprob": 0.0, + "special": false, + "text": " times" + }, + { + "id": 322, + "logprob": -1.7460938, + "special": false, + "text": " and" + }, + { + "id": 306, + "logprob": 0.0, + "special": false, + "text": " I" + } + ], + "top_tokens": null + }, + "generated_text": "What is Deep Learning?\nThis question has been asked many times and I" +} diff --git a/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_load.json b/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_load.json new file mode 100644 index 00000000..e6fb3dc0 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_awq/test_flash_llama_awq_load.json @@ -0,0 +1,418 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.703125, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4765625, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8652344, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7548828, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9306641, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4550781, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.5732422, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5761719, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5888672, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.026504517, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4287109, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.15856934, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17456055, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62646484, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.703125, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4765625, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8583984, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7548828, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9306641, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4550781, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.5732422, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5761719, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5888672, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.026504517, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4287109, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.15856934, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17456055, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62646484, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.703125, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4765625, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8652344, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7548828, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9306641, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4550781, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.5732422, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5761719, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5888672, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.026504517, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4287109, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.15856934, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17456055, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62646484, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.703125, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4765625, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8652344, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7548828, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9306641, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4550781, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.5732422, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5761719, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5888672, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.026504517, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4287109, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.15856934, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17456055, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62646484, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" + } +] diff --git a/integration-tests/models/__snapshots__/test_flash_awq_sharded/test_flash_llama_awq_load_sharded.json b/integration-tests/models/__snapshots__/test_flash_awq_sharded/test_flash_llama_awq_load_sharded.json new file mode 100644 index 00000000..f1d9129d --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_awq_sharded/test_flash_llama_awq_load_sharded.json @@ -0,0 +1,418 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.6914062, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4746094, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8623047, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7558594, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9228516, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4609375, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.57177734, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5722656, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5859375, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.02633667, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4335938, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.15991211, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17456055, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62060547, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.6914062, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4746094, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8623047, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7558594, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9228516, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4609375, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.57177734, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5722656, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5859375, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.02633667, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4335938, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.15991211, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17456055, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62060547, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.6914062, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4746094, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8623047, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7558594, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9228516, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4609375, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.57177734, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5722656, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5859375, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.02633667, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4335938, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.15991211, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17456055, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62060547, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.6914062, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4746094, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8623047, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7558594, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9228516, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4609375, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.57177734, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5722656, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5859375, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.02633667, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4335938, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.15991211, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17456055, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62060547, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" + } +] diff --git a/integration-tests/models/__snapshots__/test_flash_awq_sharded/test_flash_llama_awq_sharded.json b/integration-tests/models/__snapshots__/test_flash_awq_sharded/test_flash_llama_awq_sharded.json new file mode 100644 index 00000000..0f91eb36 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_awq_sharded/test_flash_llama_awq_sharded.json @@ -0,0 +1,104 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -7.6914062, + "text": "What" + }, + { + "id": 338, + "logprob": -1.4746094, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.390625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.8623047, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.7558594, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.9228516, + "special": false, + "text": "\n" + }, + { + "id": 5618, + "logprob": -2.4609375, + "special": false, + "text": "What" + }, + { + "id": 338, + "logprob": -0.57177734, + "special": false, + "text": " is" + }, + { + "id": 278, + "logprob": -1.5722656, + "special": false, + "text": " the" + }, + { + "id": 4328, + "logprob": -1.5927734, + "special": false, + "text": " difference" + }, + { + "id": 1546, + "logprob": -0.026428223, + "special": false, + "text": " between" + }, + { + "id": 21784, + "logprob": -1.4267578, + "special": false, + "text": " Deep" + }, + { + "id": 29257, + "logprob": -0.16015625, + "special": false, + "text": " Learning" + }, + { + "id": 322, + "logprob": -0.17382812, + "special": false, + "text": " and" + }, + { + "id": 6189, + "logprob": -0.62060547, + "special": false, + "text": " Machine" + } + ], + "top_tokens": null + }, + "generated_text": "\nWhat is the difference between Deep Learning and Machine" +} diff --git a/integration-tests/models/test_flash_awq.py b/integration-tests/models/test_flash_awq.py new file mode 100644 index 00000000..f0b99a3b --- /dev/null +++ b/integration-tests/models/test_flash_awq.py @@ -0,0 +1,64 @@ +import pytest + + +@pytest.fixture(scope="module") +def flash_llama_awq_handle(launcher): + with launcher("abhinavkulkarni/codellama-CodeLlama-7b-Python-hf-w4-g128-awq", num_shard=1, quantize="awq") as handle: + yield handle + + +@pytest.fixture(scope="module") +async def flash_llama_awq(flash_llama_awq_handle): + await flash_llama_awq_handle.health(300) + return flash_llama_awq_handle.client + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_awq(flash_llama_awq, response_snapshot): + response = await flash_llama_awq.generate( + "What is Deep Learning?", max_new_tokens=10, decoder_input_details=True + ) + + assert response.details.generated_tokens == 10 + assert response.generated_text == "\nWhat is the difference between Deep Learning and Machine" + assert response == response_snapshot + + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_awq_all_params(flash_llama_awq, response_snapshot): + response = await flash_llama_awq.generate( + "What is Deep Learning?", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + decoder_input_details=True, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_awq_load( + flash_llama_awq, generate_load, response_snapshot +): + responses = await generate_load( + flash_llama_awq, "What is Deep Learning?", max_new_tokens=10, n=4 + ) + + assert len(responses) == 4 + assert all([r.generated_text == "\nWhat is the difference between Deep Learning and Machine" for r in responses]) + + assert responses == response_snapshot + + diff --git a/integration-tests/models/test_flash_awq_sharded.py b/integration-tests/models/test_flash_awq_sharded.py new file mode 100644 index 00000000..39ea464a --- /dev/null +++ b/integration-tests/models/test_flash_awq_sharded.py @@ -0,0 +1,36 @@ +import pytest + +@pytest.fixture(scope="module") +def flash_llama_awq_handle_sharded(launcher): + with launcher("abhinavkulkarni/codellama-CodeLlama-7b-Python-hf-w4-g128-awq", num_shard=2, quantize="awq") as handle: + yield handle + +@pytest.fixture(scope="module") +async def flash_llama_awq_sharded(flash_llama_awq_handle_sharded): + await flash_llama_awq_handle_sharded.health(300) + return flash_llama_awq_handle_sharded.client + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_awq_sharded(flash_llama_awq_sharded, response_snapshot): + response = await flash_llama_awq_sharded.generate( + "What is Deep Learning?", max_new_tokens=10, decoder_input_details=True + ) + + assert response.details.generated_tokens == 10 + assert response.generated_text == "\nWhat is the difference between Deep Learning and Machine" + assert response == response_snapshot + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_awq_load_sharded( + flash_llama_awq_sharded, generate_load, response_snapshot +): + responses = await generate_load( + flash_llama_awq_sharded, "What is Deep Learning?", max_new_tokens=10, n=4 + ) + + assert len(responses) == 4 + assert all([r.generated_text == "\nWhat is the difference between Deep Learning and Machine" for r in responses]) + + assert responses == response_snapshot diff --git a/launcher/src/main.rs b/launcher/src/main.rs index cbb6f25d..09e32f89 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -25,6 +25,7 @@ enum Quantization { BitsandbytesNF4, BitsandbytesFP4, Gptq, + Awq, } impl std::fmt::Display for Quantization { @@ -43,6 +44,9 @@ impl std::fmt::Display for Quantization { Quantization::Gptq => { write!(f, "gptq") } + Quantization::Awq => { + write!(f, "awq") + } } } } diff --git a/server/.gitignore b/server/.gitignore index 2e1db124..40c37f48 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -159,3 +159,4 @@ safetensors flash-attention/ flash-attention-v2/ vllm/ +llm-awq/ diff --git a/server/Makefile b/server/Makefile index a4ce6d8b..b21d79d4 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,6 +1,7 @@ include Makefile-flash-att include Makefile-flash-att-v2 include Makefile-vllm +include Makefile-awq unit-tests: pytest -s -vv -m "not private" tests diff --git a/server/Makefile-awq b/server/Makefile-awq new file mode 100644 index 00000000..80e78c08 --- /dev/null +++ b/server/Makefile-awq @@ -0,0 +1,13 @@ +awq_commit := f084f40bd996f3cf3a0633c1ad7d9d476c318aaa + +awq: + rm -rf llm-awq + git clone https://github.com/mit-han-lab/llm-awq + +build-awq: awq + cd llm-awq/ && git fetch && git checkout $(awq_commit) + cd llm-awq/awq/kernels && python setup.py build + +install-awq: build-awq + pip uninstall awq_inference_engine -y || true + cd llm-awq/awq/kernels && python setup.py install diff --git a/server/poetry.lock b/server/poetry.lock index fd4d427d..7c18ec76 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -323,19 +323,19 @@ files = [ [[package]] name = "datasets" -version = "2.14.4" +version = "2.14.5" description = "HuggingFace community-driven open-source library of datasets" optional = true python-versions = ">=3.8.0" files = [ - {file = "datasets-2.14.4-py3-none-any.whl", hash = "sha256:29336bd316a7d827ccd4da2236596279b20ca2ac78f64c04c9483da7cbc2459b"}, - {file = "datasets-2.14.4.tar.gz", hash = "sha256:ef29c2b5841de488cd343cfc26ab979bff77efa4d2285af51f1ad7db5c46a83b"}, + {file = "datasets-2.14.5-py3-none-any.whl", hash = "sha256:dd4155091034cba04d5a28711f2ed3944275ed15c5d0c5a2d0b6b9ea34a2bdfe"}, + {file = "datasets-2.14.5.tar.gz", hash = "sha256:b738a86540ab8e1a7806c8a3790b67be0056318d0c5d5a58a1b0dbdd76c0f568"}, ] [package.dependencies] aiohttp = "*" dill = ">=0.3.0,<0.3.8" -fsspec = {version = ">=2021.11.1", extras = ["http"]} +fsspec = {version = ">=2023.1.0,<2023.9.0", extras = ["http"]} huggingface-hub = ">=0.14.0,<1.0.0" multiprocess = "*" numpy = ">=1.17" @@ -421,21 +421,19 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.12.3" +version = "3.12.4" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.12.3-py3-none-any.whl", hash = "sha256:f067e40ccc40f2b48395a80fcbd4728262fab54e232e090a4063ab804179efeb"}, - {file = "filelock-3.12.3.tar.gz", hash = "sha256:0ecc1dd2ec4672a10c8550a8182f1bd0c0a5088470ecd5a125e45f49472fac3d"}, + {file = "filelock-3.12.4-py3-none-any.whl", hash = "sha256:08c21d87ded6e2b9da6728c3dff51baf1dcecf973b768ef35bcbc3447edb9ad4"}, + {file = "filelock-3.12.4.tar.gz", hash = "sha256:2e6f249f1f3654291606e046b09f1fd5eac39b360664c27f5aad072012f8bcbd"}, ] -[package.dependencies] -typing-extensions = {version = ">=4.7.1", markers = "python_version < \"3.11\""} - [package.extras] docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] +typing = ["typing-extensions (>=4.7.1)"] [[package]] name = "frozenlist" @@ -582,148 +580,148 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.57.0" +version = "1.58.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.57.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:092fa155b945015754bdf988be47793c377b52b88d546e45c6a9f9579ac7f7b6"}, - {file = "grpcio-1.57.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2f7349786da979a94690cc5c2b804cab4e8774a3cf59be40d037c4342c906649"}, - {file = "grpcio-1.57.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:82640e57fb86ea1d71ea9ab54f7e942502cf98a429a200b2e743d8672171734f"}, - {file = "grpcio-1.57.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40b72effd4c789de94ce1be2b5f88d7b9b5f7379fe9645f198854112a6567d9a"}, - {file = "grpcio-1.57.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f708a6a17868ad8bf586598bee69abded4996b18adf26fd2d91191383b79019"}, - {file = "grpcio-1.57.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:60fe15288a0a65d5c1cb5b4a62b1850d07336e3ba728257a810317be14f0c527"}, - {file = "grpcio-1.57.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6907b1cf8bb29b058081d2aad677b15757a44ef2d4d8d9130271d2ad5e33efca"}, - {file = "grpcio-1.57.0-cp310-cp310-win32.whl", hash = "sha256:57b183e8b252825c4dd29114d6c13559be95387aafc10a7be645462a0fc98bbb"}, - {file = "grpcio-1.57.0-cp310-cp310-win_amd64.whl", hash = "sha256:7b400807fa749a9eb286e2cd893e501b110b4d356a218426cb9c825a0474ca56"}, - {file = "grpcio-1.57.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c6ebecfb7a31385393203eb04ed8b6a08f5002f53df3d59e5e795edb80999652"}, - {file = "grpcio-1.57.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:00258cbe3f5188629828363ae8ff78477ce976a6f63fb2bb5e90088396faa82e"}, - {file = "grpcio-1.57.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:23e7d8849a0e58b806253fd206ac105b328171e01b8f18c7d5922274958cc87e"}, - {file = "grpcio-1.57.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5371bcd861e679d63b8274f73ac281751d34bd54eccdbfcd6aa00e692a82cd7b"}, - {file = "grpcio-1.57.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aed90d93b731929e742967e236f842a4a2174dc5db077c8f9ad2c5996f89f63e"}, - {file = "grpcio-1.57.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fe752639919aad9ffb0dee0d87f29a6467d1ef764f13c4644d212a9a853a078d"}, - {file = "grpcio-1.57.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fada6b07ec4f0befe05218181f4b85176f11d531911b64c715d1875c4736d73a"}, - {file = "grpcio-1.57.0-cp311-cp311-win32.whl", hash = "sha256:bb396952cfa7ad2f01061fbc7dc1ad91dd9d69243bcb8110cf4e36924785a0fe"}, - {file = "grpcio-1.57.0-cp311-cp311-win_amd64.whl", hash = "sha256:e503cb45ed12b924b5b988ba9576dc9949b2f5283b8e33b21dcb6be74a7c58d0"}, - {file = "grpcio-1.57.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:fd173b4cf02b20f60860dc2ffe30115c18972d7d6d2d69df97ac38dee03be5bf"}, - {file = "grpcio-1.57.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:d7f8df114d6b4cf5a916b98389aeaf1e3132035420a88beea4e3d977e5f267a5"}, - {file = "grpcio-1.57.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:76c44efa4ede1f42a9d5b2fed1fe9377e73a109bef8675fb0728eb80b0b8e8f2"}, - {file = "grpcio-1.57.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4faea2cfdf762a664ab90589b66f416274887641ae17817de510b8178356bf73"}, - {file = "grpcio-1.57.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c60b83c43faeb6d0a9831f0351d7787a0753f5087cc6fa218d78fdf38e5acef0"}, - {file = "grpcio-1.57.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b363bbb5253e5f9c23d8a0a034dfdf1b7c9e7f12e602fc788c435171e96daccc"}, - {file = "grpcio-1.57.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:f1fb0fd4a1e9b11ac21c30c169d169ef434c6e9344ee0ab27cfa6f605f6387b2"}, - {file = "grpcio-1.57.0-cp37-cp37m-win_amd64.whl", hash = "sha256:34950353539e7d93f61c6796a007c705d663f3be41166358e3d88c45760c7d98"}, - {file = "grpcio-1.57.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:871f9999e0211f9551f368612460442a5436d9444606184652117d6a688c9f51"}, - {file = "grpcio-1.57.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:a8a8e560e8dbbdf29288872e91efd22af71e88b0e5736b0daf7773c1fecd99f0"}, - {file = "grpcio-1.57.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2313b124e475aa9017a9844bdc5eafb2d5abdda9d456af16fc4535408c7d6da6"}, - {file = "grpcio-1.57.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4098b6b638d9e0ca839a81656a2fd4bc26c9486ea707e8b1437d6f9d61c3941"}, - {file = "grpcio-1.57.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e5b58e32ae14658085c16986d11e99abd002ddbf51c8daae8a0671fffb3467f"}, - {file = "grpcio-1.57.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0f80bf37f09e1caba6a8063e56e2b87fa335add314cf2b78ebf7cb45aa7e3d06"}, - {file = "grpcio-1.57.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5b7a4ce8f862fe32b2a10b57752cf3169f5fe2915acfe7e6a1e155db3da99e79"}, - {file = "grpcio-1.57.0-cp38-cp38-win32.whl", hash = "sha256:9338bacf172e942e62e5889b6364e56657fbf8ac68062e8b25c48843e7b202bb"}, - {file = "grpcio-1.57.0-cp38-cp38-win_amd64.whl", hash = "sha256:e1cb52fa2d67d7f7fab310b600f22ce1ff04d562d46e9e0ac3e3403c2bb4cc16"}, - {file = "grpcio-1.57.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:fee387d2fab144e8a34e0e9c5ca0f45c9376b99de45628265cfa9886b1dbe62b"}, - {file = "grpcio-1.57.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:b53333627283e7241fcc217323f225c37783b5f0472316edcaa4479a213abfa6"}, - {file = "grpcio-1.57.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:f19ac6ac0a256cf77d3cc926ef0b4e64a9725cc612f97228cd5dc4bd9dbab03b"}, - {file = "grpcio-1.57.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3fdf04e402f12e1de8074458549337febb3b45f21076cc02ef4ff786aff687e"}, - {file = "grpcio-1.57.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5613a2fecc82f95d6c51d15b9a72705553aa0d7c932fad7aed7afb51dc982ee5"}, - {file = "grpcio-1.57.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b670c2faa92124b7397b42303e4d8eb64a4cd0b7a77e35a9e865a55d61c57ef9"}, - {file = "grpcio-1.57.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7a635589201b18510ff988161b7b573f50c6a48fae9cb567657920ca82022b37"}, - {file = "grpcio-1.57.0-cp39-cp39-win32.whl", hash = "sha256:d78d8b86fcdfa1e4c21f8896614b6cc7ee01a2a758ec0c4382d662f2a62cf766"}, - {file = "grpcio-1.57.0-cp39-cp39-win_amd64.whl", hash = "sha256:20ec6fc4ad47d1b6e12deec5045ec3cd5402d9a1597f738263e98f490fe07056"}, - {file = "grpcio-1.57.0.tar.gz", hash = "sha256:4b089f7ad1eb00a104078bab8015b0ed0ebcb3b589e527ab009c53893fd4e613"}, + {file = "grpcio-1.58.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:3e6bebf1dfdbeb22afd95650e4f019219fef3ab86d3fca8ebade52e4bc39389a"}, + {file = "grpcio-1.58.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:cde11577d5b6fd73a00e6bfa3cf5f428f3f33c2d2878982369b5372bbc4acc60"}, + {file = "grpcio-1.58.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:a2d67ff99e70e86b2be46c1017ae40b4840d09467d5455b2708de6d4c127e143"}, + {file = "grpcio-1.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ed979b273a81de36fc9c6716d9fb09dd3443efa18dcc8652501df11da9583e9"}, + {file = "grpcio-1.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:458899d2ebd55d5ca2350fd3826dfd8fcb11fe0f79828ae75e2b1e6051d50a29"}, + {file = "grpcio-1.58.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc7ffef430b80345729ff0a6825e9d96ac87efe39216e87ac58c6c4ef400de93"}, + {file = "grpcio-1.58.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5b23d75e5173faa3d1296a7bedffb25afd2fddb607ef292dfc651490c7b53c3d"}, + {file = "grpcio-1.58.0-cp310-cp310-win32.whl", hash = "sha256:fad9295fe02455d4f158ad72c90ef8b4bcaadfdb5efb5795f7ab0786ad67dd58"}, + {file = "grpcio-1.58.0-cp310-cp310-win_amd64.whl", hash = "sha256:bc325fed4d074367bebd465a20763586e5e1ed5b943e9d8bc7c162b1f44fd602"}, + {file = "grpcio-1.58.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:652978551af02373a5a313e07bfef368f406b5929cf2d50fa7e4027f913dbdb4"}, + {file = "grpcio-1.58.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:9f13a171281ebb4d7b1ba9f06574bce2455dcd3f2f6d1fbe0fd0d84615c74045"}, + {file = "grpcio-1.58.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8774219e21b05f750eef8adc416e9431cf31b98f6ce9def288e4cea1548cbd22"}, + {file = "grpcio-1.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09206106848462763f7f273ca93d2d2d4d26cab475089e0de830bb76be04e9e8"}, + {file = "grpcio-1.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62831d5e251dd7561d9d9e83a0b8655084b2a1f8ea91e4bd6b3cedfefd32c9d2"}, + {file = "grpcio-1.58.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:212f38c6a156862098f6bdc9a79bf850760a751d259d8f8f249fc6d645105855"}, + {file = "grpcio-1.58.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4b12754af201bb993e6e2efd7812085ddaaef21d0a6f0ff128b97de1ef55aa4a"}, + {file = "grpcio-1.58.0-cp311-cp311-win32.whl", hash = "sha256:3886b4d56bd4afeac518dbc05933926198aa967a7d1d237a318e6fbc47141577"}, + {file = "grpcio-1.58.0-cp311-cp311-win_amd64.whl", hash = "sha256:002f228d197fea12797a14e152447044e14fb4fdb2eb5d6cfa496f29ddbf79ef"}, + {file = "grpcio-1.58.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:b5e8db0aff0a4819946215f156bd722b6f6c8320eb8419567ffc74850c9fd205"}, + {file = "grpcio-1.58.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:201e550b7e2ede113b63e718e7ece93cef5b0fbf3c45e8fe4541a5a4305acd15"}, + {file = "grpcio-1.58.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:d79b660681eb9bc66cc7cbf78d1b1b9e335ee56f6ea1755d34a31108b80bd3c8"}, + {file = "grpcio-1.58.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ef8d4a76d2c7d8065aba829f8d0bc0055495c998dce1964ca5b302d02514fb3"}, + {file = "grpcio-1.58.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6cba491c638c76d3dc6c191d9c75041ca5b8f5c6de4b8327ecdcab527f130bb4"}, + {file = "grpcio-1.58.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6801ff6652ecd2aae08ef994a3e49ff53de29e69e9cd0fd604a79ae4e545a95c"}, + {file = "grpcio-1.58.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:24edec346e69e672daf12b2c88e95c6f737f3792d08866101d8c5f34370c54fd"}, + {file = "grpcio-1.58.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7e473a7abad9af48e3ab5f3b5d237d18208024d28ead65a459bd720401bd2f8f"}, + {file = "grpcio-1.58.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:4891bbb4bba58acd1d620759b3be11245bfe715eb67a4864c8937b855b7ed7fa"}, + {file = "grpcio-1.58.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:e9f995a8a421405958ff30599b4d0eec244f28edc760de82f0412c71c61763d2"}, + {file = "grpcio-1.58.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2f85f87e2f087d9f632c085b37440a3169fda9cdde80cb84057c2fc292f8cbdf"}, + {file = "grpcio-1.58.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb6b92036ff312d5b4182fa72e8735d17aceca74d0d908a7f08e375456f03e07"}, + {file = "grpcio-1.58.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d81c2b2b24c32139dd2536972f1060678c6b9fbd106842a9fcdecf07b233eccd"}, + {file = "grpcio-1.58.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:fbcecb6aedd5c1891db1d70efbfbdc126c986645b5dd616a045c07d6bd2dfa86"}, + {file = "grpcio-1.58.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:92ae871a902cf19833328bd6498ec007b265aabf2fda845ab5bd10abcaf4c8c6"}, + {file = "grpcio-1.58.0-cp38-cp38-win32.whl", hash = "sha256:dc72e04620d49d3007771c0e0348deb23ca341c0245d610605dddb4ac65a37cb"}, + {file = "grpcio-1.58.0-cp38-cp38-win_amd64.whl", hash = "sha256:1c1c5238c6072470c7f1614bf7c774ffde6b346a100521de9ce791d1e4453afe"}, + {file = "grpcio-1.58.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:fe643af248442221db027da43ed43e53b73e11f40c9043738de9a2b4b6ca7697"}, + {file = "grpcio-1.58.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:128eb1f8e70676d05b1b0c8e6600320fc222b3f8c985a92224248b1367122188"}, + {file = "grpcio-1.58.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:039003a5e0ae7d41c86c768ef8b3ee2c558aa0a23cf04bf3c23567f37befa092"}, + {file = "grpcio-1.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f061722cad3f9aabb3fbb27f3484ec9d4667b7328d1a7800c3c691a98f16bb0"}, + {file = "grpcio-1.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0af11938acf8cd4cf815c46156bcde36fa5850518120920d52620cc3ec1830"}, + {file = "grpcio-1.58.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d4cef77ad2fed42b1ba9143465856d7e737279854e444925d5ba45fc1f3ba727"}, + {file = "grpcio-1.58.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:24765a627eb4d9288ace32d5104161c3654128fe27f2808ecd6e9b0cfa7fc8b9"}, + {file = "grpcio-1.58.0-cp39-cp39-win32.whl", hash = "sha256:f0241f7eb0d2303a545136c59bc565a35c4fc3b924ccbd69cb482f4828d6f31c"}, + {file = "grpcio-1.58.0-cp39-cp39-win_amd64.whl", hash = "sha256:dcfba7befe3a55dab6fe1eb7fc9359dc0c7f7272b30a70ae0af5d5b063842f28"}, + {file = "grpcio-1.58.0.tar.gz", hash = "sha256:532410c51ccd851b706d1fbc00a87be0f5312bd6f8e5dbf89d4e99c7f79d7499"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.57.0)"] +protobuf = ["grpcio-tools (>=1.58.0)"] [[package]] name = "grpcio-reflection" -version = "1.57.0" +version = "1.58.0" description = "Standard Protobuf Reflection Service for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-reflection-1.57.0.tar.gz", hash = "sha256:8f63a18729cba995a172f8325235f5094cb066febec75f9a3b1b2e28328aa166"}, - {file = "grpcio_reflection-1.57.0-py3-none-any.whl", hash = "sha256:d7deb8587f9d0095fb5d367c2aa5ce1380e3f23b0f8bca6c00bc404c5429cb6a"}, + {file = "grpcio-reflection-1.58.0.tar.gz", hash = "sha256:e6048a758d17b6ca1705258e7ee5d926d2960a95ae08ba0929dd233e505acd3d"}, + {file = "grpcio_reflection-1.58.0-py3-none-any.whl", hash = "sha256:fa18885d8a09cef02c9a6b1d17dfed0279f1f401b06bd1f75958b78ebf1b5c0c"}, ] [package.dependencies] -grpcio = ">=1.57.0" +grpcio = ">=1.58.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.57.0" +version = "1.58.0" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.57.0.tar.gz", hash = "sha256:b098da99df1eebe58337f8f78e50df990273ccacc1226fddeb47c590e3df9e02"}, - {file = "grpcio_status-1.57.0-py3-none-any.whl", hash = "sha256:15d6af055914ebbc4ed17e55ebfb8e6bb17a45a57fea32e6af19978fb7844690"}, + {file = "grpcio-status-1.58.0.tar.gz", hash = "sha256:0b42e70c0405a66a82d9e9867fa255fe59e618964a6099b20568c31dd9099766"}, + {file = "grpcio_status-1.58.0-py3-none-any.whl", hash = "sha256:36d46072b71a00147709ebce49344ac59b4b8960942acf0f813a8a7d6c1c28e0"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.57.0" +grpcio = ">=1.58.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.57.0" +version = "1.58.0" description = "Protobuf code generator for gRPC" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-tools-1.57.0.tar.gz", hash = "sha256:2f16130d869ce27ecd623194547b649dd657333ec7e8644cc571c645781a9b85"}, - {file = "grpcio_tools-1.57.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:4fb8a8468031f858381a576078924af364a08833d8f8f3237018252c4573a802"}, - {file = "grpcio_tools-1.57.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:35bf0dad8a3562043345236c26d0053a856fb06c04d7da652f2ded914e508ae7"}, - {file = "grpcio_tools-1.57.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:ec9aab2fb6783c7fc54bc28f58eb75f1ca77594e6b0fd5e5e7a8114a95169fe0"}, - {file = "grpcio_tools-1.57.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cf5fc0a1c23f8ea34b408b72fb0e90eec0f404ad4dba98e8f6da3c9ce34e2ed"}, - {file = "grpcio_tools-1.57.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26e69d08a515554e0cfe1ec4d31568836f4b17f0ff82294f957f629388629eb9"}, - {file = "grpcio_tools-1.57.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c39a3656576b6fdaaf28abe0467f7a7231df4230c1bee132322dbc3209419e7f"}, - {file = "grpcio_tools-1.57.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f64f8ab22d27d4a5693310748d35a696061c3b5c7b8c4fb4ab3b4bc1068b6b56"}, - {file = "grpcio_tools-1.57.0-cp310-cp310-win32.whl", hash = "sha256:d2a134756f4db34759a5cc7f7e43f7eb87540b68d1cca62925593c6fb93924f7"}, - {file = "grpcio_tools-1.57.0-cp310-cp310-win_amd64.whl", hash = "sha256:9a3d60fb8d46ede26c1907c146561b3a9caa20a7aff961bc661ef8226f85a2e9"}, - {file = "grpcio_tools-1.57.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:aac98ecad8f7bd4301855669d42a5d97ef7bb34bec2b1e74c7a0641d47e313cf"}, - {file = "grpcio_tools-1.57.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:cdd020cb68b51462983b7c2dfbc3eb6ede032b8bf438d4554df0c3f08ce35c76"}, - {file = "grpcio_tools-1.57.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:f54081b08419a39221cd646363b5708857c696b3ad4784f1dcf310891e33a5f7"}, - {file = "grpcio_tools-1.57.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed85a0291fff45b67f2557fe7f117d3bc7af8b54b8619d27bf374b5c8b7e3ca2"}, - {file = "grpcio_tools-1.57.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e868cd6feb3ef07d4b35be104fe1fd0657db05259ff8f8ec5e08f4f89ca1191d"}, - {file = "grpcio_tools-1.57.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:dfb6f6120587b8e228a3cae5ee4985b5bdc18501bad05c49df61965dfc9d70a9"}, - {file = "grpcio_tools-1.57.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a7ad7f328e28fc97c356d0f10fb10d8b5151bb65aa7cf14bf8084513f0b7306"}, - {file = "grpcio_tools-1.57.0-cp311-cp311-win32.whl", hash = "sha256:9867f2817b1a0c93c523f89ac6c9d8625548af4620a7ce438bf5a76e23327284"}, - {file = "grpcio_tools-1.57.0-cp311-cp311-win_amd64.whl", hash = "sha256:1f9e917a9f18087f6c14b4d4508fb94fca5c2f96852363a89232fb9b2124ac1f"}, - {file = "grpcio_tools-1.57.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:9f2aefa8a37bd2c4db1a3f1aca11377e2766214520fb70e67071f4ff8d8b0fa5"}, - {file = "grpcio_tools-1.57.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:850cbda0ec5d24c39e7215ede410276040692ca45d105fbbeada407fa03f0ac0"}, - {file = "grpcio_tools-1.57.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6fa52972c9647876ea35f6dc2b51002a74ed900ec7894586cbb2fe76f64f99de"}, - {file = "grpcio_tools-1.57.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0eea89d7542719594e50e2283f51a072978b953e8b3e9fd7c59a2c762d4c1"}, - {file = "grpcio_tools-1.57.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3da5240211252fc70a6451fe00c143e2ab2f7bfc2445695ad2ed056b8e48d96"}, - {file = "grpcio_tools-1.57.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a0256f8786ac9e4db618a1aa492bb3472569a0946fd3ee862ffe23196323da55"}, - {file = "grpcio_tools-1.57.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c026bdf5c1366ce88b7bbe2d8207374d675afd3fd911f60752103de3da4a41d2"}, - {file = "grpcio_tools-1.57.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9053c2f655589545be08b9d6a673e92970173a4bf11a4b9f18cd6e9af626b587"}, - {file = "grpcio_tools-1.57.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:81ec4dbb696e095057b2528d11a8da04be6bbe2b967fa07d4ea9ba6354338cbf"}, - {file = "grpcio_tools-1.57.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:495e2946406963e0b9f063f76d5af0f2a19517dac2b367b5b044432ac9194296"}, - {file = "grpcio_tools-1.57.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:7b46fc6aa8eb7edd18cafcd21fd98703cb6c09e46b507de335fca7f0161dfccb"}, - {file = "grpcio_tools-1.57.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb81ff861692111fa81bd85f64584e624cb4013bd66fbce8a209b8893f5ce398"}, - {file = "grpcio_tools-1.57.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a42dc220eb5305f470855c9284f4c8e85ae59d6d742cd07946b0cbe5e9ca186"}, - {file = "grpcio_tools-1.57.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:90d10d9038ba46a595a223a34f136c9230e3d6d7abc2433dbf0e1c31939d3a8b"}, - {file = "grpcio_tools-1.57.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5bc3e6d338aefb052e19cedabe00452be46d0c10a4ed29ee77abb00402e438fe"}, - {file = "grpcio_tools-1.57.0-cp38-cp38-win32.whl", hash = "sha256:34b36217b17b5bea674a414229913e1fd80ede328be51e1b531fcc62abd393b0"}, - {file = "grpcio_tools-1.57.0-cp38-cp38-win_amd64.whl", hash = "sha256:dbde4004a0688400036342ff73e3706e8940483e2871547b1354d59e93a38277"}, - {file = "grpcio_tools-1.57.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:784574709b9690dc28696617ea69352e2132352fdfc9bc89afa8e39f99ae538e"}, - {file = "grpcio_tools-1.57.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:85ac4e62eb44428cde025fd9ab7554002315fc7880f791c553fc5a0015cc9931"}, - {file = "grpcio_tools-1.57.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:dc771d4db5701f280957bbcee91745e0686d00ed1c6aa7e05ba30a58b02d70a1"}, - {file = "grpcio_tools-1.57.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3ac06703c412f8167a9062eaf6099409967e33bf98fa5b02be4b4689b6bdf39"}, - {file = "grpcio_tools-1.57.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02d78c034109f46032c7217260066d49d41e6bcaf588fa28fa40fe2f83445347"}, - {file = "grpcio_tools-1.57.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2db25f15ed44327f2e02d0c4fe741ac966f9500e407047d8a7c7fccf2df65616"}, - {file = "grpcio_tools-1.57.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b417c97936d94874a3ce7ed8deab910f2233e3612134507cfee4af8735c38a6"}, - {file = "grpcio_tools-1.57.0-cp39-cp39-win32.whl", hash = "sha256:f717cce5093e6b6049d9ea6d12fdf3658efdb1a80772f7737db1f8510b876df6"}, - {file = "grpcio_tools-1.57.0-cp39-cp39-win_amd64.whl", hash = "sha256:1c0e8a1a32973a5d59fbcc19232f925e5c48116e9411f788033a31c5ca5130b4"}, + {file = "grpcio-tools-1.58.0.tar.gz", hash = "sha256:6f4d80ceb591e31ca4dceec747dbe56132e1392a0a9bb1c8fe001d1b5cac898a"}, + {file = "grpcio_tools-1.58.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:60c874908f3b40f32f1bb0221f7b3ab65ecb53a4d0a9f0a394f031f1b292c177"}, + {file = "grpcio_tools-1.58.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:1852e798f31e5437ca7b37abc910e028b34732fb19364862cedb87b1dab66fad"}, + {file = "grpcio_tools-1.58.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:149fb48f53cb691a6328f68bed8e4036c730f7106b7f98e92c2c0403f0b9e93c"}, + {file = "grpcio_tools-1.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba3d383e5ca93826038b70f326fce8e8d12dd9b2f64d363a3d612f7475f12dd2"}, + {file = "grpcio_tools-1.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6997511e9d2979f7a2389479682dbb06823f21a904e8fb0a5c6baaf1b4b4a863"}, + {file = "grpcio_tools-1.58.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8de0b701da479643f71fad71fe66885cddd89441ae16e2c724939b47742dc72e"}, + {file = "grpcio_tools-1.58.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:43cc23908b63fcaefe690b10f68a2d8652c994b5b36ab77d2271d9608c895320"}, + {file = "grpcio_tools-1.58.0-cp310-cp310-win32.whl", hash = "sha256:2c2221123d010dc6231799e63a37f2f4786bf614ef65b23009c387cd20d8b193"}, + {file = "grpcio_tools-1.58.0-cp310-cp310-win_amd64.whl", hash = "sha256:df2788736bdf58abe7b0e4d6b1ff806f7686c98c5ad900da312252e3322d91c4"}, + {file = "grpcio_tools-1.58.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:b6ea5578712cdb29b0ff60bfc6405bf0e8d681b9c71d106dd1cda54fe7fe4e55"}, + {file = "grpcio_tools-1.58.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:c29880f491581c83181c0a84a4d11402af2b13166a5266f64e246adf1da7aa66"}, + {file = "grpcio_tools-1.58.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:32d51e933c3565414dd0835f930bb28a1cdeba435d9d2c87fa3cf8b1d284db3c"}, + {file = "grpcio_tools-1.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ad9d77f25514584b1ddc981d70c9e50dfcfc388aa5ba943eee67520c5267ed9"}, + {file = "grpcio_tools-1.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4882382631e6352819059278a5c878ce0b067008dd490911d16d5616e8a36d85"}, + {file = "grpcio_tools-1.58.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d84091a189d848d94645b7c48b61734c12ec03b0d46e5fc0049343a26989ac5c"}, + {file = "grpcio_tools-1.58.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:85ac28a9621e9b92a3fc416288c4ce45542db0b4c31b3e23031dd8e0a0ec5590"}, + {file = "grpcio_tools-1.58.0-cp311-cp311-win32.whl", hash = "sha256:7371d8ea80234b29affec145e25569523f549520ed7e53b2aa92bed412cdecfd"}, + {file = "grpcio_tools-1.58.0-cp311-cp311-win_amd64.whl", hash = "sha256:6997df6e7c5cf4d3ddc764240c1ff6a04b45d70ec28913b38fbc6396ef743e12"}, + {file = "grpcio_tools-1.58.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:ac65b8d6e3acaf88b815edf9af88ff844b6600ff3d2591c05ba4f655b45d5fb4"}, + {file = "grpcio_tools-1.58.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:88e8191d0dd789bebf42533808728f5ce75d2c51e2a72bdf20abe5b5e3fbec42"}, + {file = "grpcio_tools-1.58.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:a3dbece2a121761499a659b799979d4b738586d1065439053de553773eee11ca"}, + {file = "grpcio_tools-1.58.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1086fe240c4c879b9721952b47d46996deb283c2d9355a8dc24a804811aacf70"}, + {file = "grpcio_tools-1.58.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7ae3dca059d5b358dd03fb63277428fa7d771605d4074a019138dd38d70719a"}, + {file = "grpcio_tools-1.58.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3f8904ac7fc3da2e874f00b3a986e8b7e004f499344a8e7eb213c26dfb025041"}, + {file = "grpcio_tools-1.58.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:aadbd8393ae332e49731adb31e741f2e689989150569b7acc939f5ea43124e2d"}, + {file = "grpcio_tools-1.58.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1cb6e24194786687d4f23c64de1f0ce553af51de22746911bc37340f85f9783e"}, + {file = "grpcio_tools-1.58.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:6ec43909095c630df3e479e77469bdad367067431f4af602f6ccb978a3b78afd"}, + {file = "grpcio_tools-1.58.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:4be49ed320b0ebcbc21d19ef555fbf229c1c452105522b728e1171ee2052078e"}, + {file = "grpcio_tools-1.58.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:28eefebddec3d3adf19baca78f8b82a2287d358e1b1575ae018cdca8eacc6269"}, + {file = "grpcio_tools-1.58.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ef8c696e9d78676cc3f583a92bbbf2c84e94e350f7ad22f150a52559f4599d1"}, + {file = "grpcio_tools-1.58.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9aeb5949e46558d21c51fd3ec3eeecc59c94dbca76c67c0a80d3da6b7437930c"}, + {file = "grpcio_tools-1.58.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6f7144aad9396d35fb1b80429600a970b559c2ad4d07020eeb180fe83cea2bee"}, + {file = "grpcio_tools-1.58.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ee26e9253a721fff355737649678535f76cf5d642aa3ac0cd937832559b90af"}, + {file = "grpcio_tools-1.58.0-cp38-cp38-win32.whl", hash = "sha256:343f572312039059a8797d6e29a7fc62196e73131ab01755660a9d48202267c1"}, + {file = "grpcio_tools-1.58.0-cp38-cp38-win_amd64.whl", hash = "sha256:cd7acfbb43b7338a78cf4a67528d05530d574d92b7c829d185b78dfc451d158f"}, + {file = "grpcio_tools-1.58.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:46628247fbce86d18232eead24bd22ed0826c79f3fe2fc2fbdbde45971361049"}, + {file = "grpcio_tools-1.58.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:51587842a54e025a3d0d37afcf4ef2b7ac1def9a5d17448665cb424b53d6c287"}, + {file = "grpcio_tools-1.58.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a062ae3072a2a39a3c057f4d68b57b021f1dd2956cd09aab39709f6af494e1de"}, + {file = "grpcio_tools-1.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eec3c93a08df11c80ef1c29a616bcbb0d83dbc6ea41b48306fcacc720416dfa7"}, + {file = "grpcio_tools-1.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b63f823ac991ff77104da614d2a2485a59d37d57830eb2e387a6e2a3edc7fa2b"}, + {file = "grpcio_tools-1.58.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:579c11a9f198847ed48dbc4f211c67fe96a73320b87c81f01b044b72e24a7d77"}, + {file = "grpcio_tools-1.58.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6ca2fc1dd8049d417a5034d944c9df05cee76f855b3e431627ab4292e7c01c47"}, + {file = "grpcio_tools-1.58.0-cp39-cp39-win32.whl", hash = "sha256:453023120114c35d3d9d6717ea0820e5d5c140f51f9d0b621de4397ff854471b"}, + {file = "grpcio_tools-1.58.0-cp39-cp39-win_amd64.whl", hash = "sha256:b6c896f1df99c35cf062d4803c15663ff00a33ff09add28baa6e475cf6b5e258"}, ] [package.dependencies] -grpcio = ">=1.57.0" +grpcio = ">=1.58.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -871,6 +869,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1051,36 +1059,43 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "numpy" -version = "1.25.2" +version = "1.26.0" description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.9" +python-versions = "<3.13,>=3.9" files = [ - {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, - {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"}, - {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"}, - {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"}, - {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"}, - {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"}, - {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"}, - {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"}, - {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"}, - {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"}, - {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"}, - {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, + {file = "numpy-1.26.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8db2f125746e44dce707dd44d4f4efeea8d7e2b43aace3f8d1f235cfa2733dd"}, + {file = "numpy-1.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0621f7daf973d34d18b4e4bafb210bbaf1ef5e0100b5fa750bd9cde84c7ac292"}, + {file = "numpy-1.26.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51be5f8c349fdd1a5568e72713a21f518e7d6707bcf8503b528b88d33b57dc68"}, + {file = "numpy-1.26.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:767254ad364991ccfc4d81b8152912e53e103ec192d1bb4ea6b1f5a7117040be"}, + {file = "numpy-1.26.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:436c8e9a4bdeeee84e3e59614d38c3dbd3235838a877af8c211cfcac8a80b8d3"}, + {file = "numpy-1.26.0-cp310-cp310-win32.whl", hash = "sha256:c2e698cb0c6dda9372ea98a0344245ee65bdc1c9dd939cceed6bb91256837896"}, + {file = "numpy-1.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:09aaee96c2cbdea95de76ecb8a586cb687d281c881f5f17bfc0fb7f5890f6b91"}, + {file = "numpy-1.26.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:637c58b468a69869258b8ae26f4a4c6ff8abffd4a8334c830ffb63e0feefe99a"}, + {file = "numpy-1.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:306545e234503a24fe9ae95ebf84d25cba1fdc27db971aa2d9f1ab6bba19a9dd"}, + {file = "numpy-1.26.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c6adc33561bd1d46f81131d5352348350fc23df4d742bb246cdfca606ea1208"}, + {file = "numpy-1.26.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e062aa24638bb5018b7841977c360d2f5917268d125c833a686b7cbabbec496c"}, + {file = "numpy-1.26.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:546b7dd7e22f3c6861463bebb000646fa730e55df5ee4a0224408b5694cc6148"}, + {file = "numpy-1.26.0-cp311-cp311-win32.whl", hash = "sha256:c0b45c8b65b79337dee5134d038346d30e109e9e2e9d43464a2970e5c0e93229"}, + {file = "numpy-1.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:eae430ecf5794cb7ae7fa3808740b015aa80747e5266153128ef055975a72b99"}, + {file = "numpy-1.26.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:166b36197e9debc4e384e9c652ba60c0bacc216d0fc89e78f973a9760b503388"}, + {file = "numpy-1.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f042f66d0b4ae6d48e70e28d487376204d3cbf43b84c03bac57e28dac6151581"}, + {file = "numpy-1.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5e18e5b14a7560d8acf1c596688f4dfd19b4f2945b245a71e5af4ddb7422feb"}, + {file = "numpy-1.26.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6bad22a791226d0a5c7c27a80a20e11cfe09ad5ef9084d4d3fc4a299cca505"}, + {file = "numpy-1.26.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4acc65dd65da28060e206c8f27a573455ed724e6179941edb19f97e58161bb69"}, + {file = "numpy-1.26.0-cp312-cp312-win32.whl", hash = "sha256:bb0d9a1aaf5f1cb7967320e80690a1d7ff69f1d47ebc5a9bea013e3a21faec95"}, + {file = "numpy-1.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:ee84ca3c58fe48b8ddafdeb1db87388dce2c3c3f701bf447b05e4cfcc3679112"}, + {file = "numpy-1.26.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4a873a8180479bc829313e8d9798d5234dfacfc2e8a7ac188418189bb8eafbd2"}, + {file = "numpy-1.26.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:914b28d3215e0c721dc75db3ad6d62f51f630cb0c277e6b3bcb39519bed10bd8"}, + {file = "numpy-1.26.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c78a22e95182fb2e7874712433eaa610478a3caf86f28c621708d35fa4fd6e7f"}, + {file = "numpy-1.26.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f737708b366c36b76e953c46ba5827d8c27b7a8c9d0f471810728e5a2fe57c"}, + {file = "numpy-1.26.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b44e6a09afc12952a7d2a58ca0a2429ee0d49a4f89d83a0a11052da696440e49"}, + {file = "numpy-1.26.0-cp39-cp39-win32.whl", hash = "sha256:5671338034b820c8d58c81ad1dafc0ed5a00771a82fccc71d6438df00302094b"}, + {file = "numpy-1.26.0-cp39-cp39-win_amd64.whl", hash = "sha256:020cdbee66ed46b671429c7265cf00d8ac91c046901c55684954c3958525dab2"}, + {file = "numpy-1.26.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0792824ce2f7ea0c82ed2e4fecc29bb86bee0567a080dacaf2e0a01fe7654369"}, + {file = "numpy-1.26.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d484292eaeb3e84a51432a94f53578689ffdea3f90e10c8b203a99be5af57d8"}, + {file = "numpy-1.26.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:186ba67fad3c60dbe8a3abff3b67a91351100f2661c8e2a80364ae6279720299"}, + {file = "numpy-1.26.0.tar.gz", hash = "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf"}, ] [[package]] @@ -1250,70 +1265,71 @@ files = [ [[package]] name = "pandas" -version = "2.0.3" +version = "2.1.1" description = "Powerful data structures for data analysis, time series, and statistics" optional = true -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, - {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, - {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, - {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0"}, - {file = "pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210"}, - {file = "pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e"}, - {file = "pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8"}, - {file = "pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26"}, - {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d"}, - {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df"}, - {file = "pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd"}, - {file = "pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b"}, - {file = "pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061"}, - {file = "pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5"}, - {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089"}, - {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0"}, - {file = "pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02"}, - {file = "pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78"}, - {file = "pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b"}, - {file = "pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e"}, - {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b"}, - {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641"}, - {file = "pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682"}, - {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, - {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, + {file = "pandas-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58d997dbee0d4b64f3cb881a24f918b5f25dd64ddf31f467bb9b67ae4c63a1e4"}, + {file = "pandas-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02304e11582c5d090e5a52aec726f31fe3f42895d6bfc1f28738f9b64b6f0614"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffa8f0966de2c22de408d0e322db2faed6f6e74265aa0856f3824813cf124363"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1f84c144dee086fe4f04a472b5cd51e680f061adf75c1ae4fc3a9275560f8f4"}, + {file = "pandas-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ce97667d06d69396d72be074f0556698c7f662029322027c226fd7a26965cb"}, + {file = "pandas-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:4c3f32fd7c4dccd035f71734df39231ac1a6ff95e8bdab8d891167197b7018d2"}, + {file = "pandas-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e2959720b70e106bb1d8b6eadd8ecd7c8e99ccdbe03ee03260877184bb2877d"}, + {file = "pandas-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25e8474a8eb258e391e30c288eecec565bfed3e026f312b0cbd709a63906b6f8"}, + {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8bd1685556f3374520466998929bade3076aeae77c3e67ada5ed2b90b4de7f0"}, + {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc3657869c7902810f32bd072f0740487f9e030c1a3ab03e0af093db35a9d14e"}, + {file = "pandas-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:05674536bd477af36aa2effd4ec8f71b92234ce0cc174de34fd21e2ee99adbc2"}, + {file = "pandas-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:b407381258a667df49d58a1b637be33e514b07f9285feb27769cedb3ab3d0b3a"}, + {file = "pandas-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c747793c4e9dcece7bb20156179529898abf505fe32cb40c4052107a3c620b49"}, + {file = "pandas-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3bcad1e6fb34b727b016775bea407311f7721db87e5b409e6542f4546a4951ea"}, + {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5ec7740f9ccb90aec64edd71434711f58ee0ea7f5ed4ac48be11cfa9abf7317"}, + {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29deb61de5a8a93bdd033df328441a79fcf8dd3c12d5ed0b41a395eef9cd76f0"}, + {file = "pandas-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f99bebf19b7e03cf80a4e770a3e65eee9dd4e2679039f542d7c1ace7b7b1daa"}, + {file = "pandas-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:84e7e910096416adec68075dc87b986ff202920fb8704e6d9c8c9897fe7332d6"}, + {file = "pandas-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:366da7b0e540d1b908886d4feb3d951f2f1e572e655c1160f5fde28ad4abb750"}, + {file = "pandas-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9e50e72b667415a816ac27dfcfe686dc5a0b02202e06196b943d54c4f9c7693e"}, + {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1ab6a25da197f03ebe6d8fa17273126120874386b4ac11c1d687df288542dd"}, + {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0dbfea0dd3901ad4ce2306575c54348d98499c95be01b8d885a2737fe4d7a98"}, + {file = "pandas-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0489b0e6aa3d907e909aef92975edae89b1ee1654db5eafb9be633b0124abe97"}, + {file = "pandas-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:4cdb0fab0400c2cb46dafcf1a0fe084c8bb2480a1fa8d81e19d15e12e6d4ded2"}, + {file = "pandas-2.1.1.tar.gz", hash = "sha256:fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b"}, ] [package.dependencies] numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" tzdata = ">=2022.1" [package.extras] -all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] -aws = ["s3fs (>=2021.08.0)"] -clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] -compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] -computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +aws = ["s3fs (>=2022.05.0)"] +clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] +compression = ["zstandard (>=0.17.0)"] +computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2021.07.0)"] -gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] -hdf5 = ["tables (>=3.6.1)"] -html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] -mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +fss = ["fsspec (>=2022.05.0)"] +gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] +hdf5 = ["tables (>=3.7.0)"] +html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] +mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] -spss = ["pyreadstat (>=1.1.2)"] -sql-other = ["SQLAlchemy (>=1.4.16)"] -test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.6.3)"] +postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] +spss = ["pyreadstat (>=1.1.5)"] +sql-other = ["SQLAlchemy (>=1.4.36)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.8.0)"] [[package]] name = "peft" @@ -1344,67 +1360,65 @@ test = ["black (>=22.0,<23.0)", "datasets", "diffusers", "hf-doc-builder", "para [[package]] name = "pillow" -version = "10.0.0" +version = "10.0.1" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.8" files = [ - {file = "Pillow-10.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1f62406a884ae75fb2f818694469519fb685cc7eaff05d3451a9ebe55c646891"}, - {file = "Pillow-10.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5db32e2a6ccbb3d34d87c87b432959e0db29755727afb37290e10f6e8e62614"}, - {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edf4392b77bdc81f36e92d3a07a5cd072f90253197f4a52a55a8cec48a12483b"}, - {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:520f2a520dc040512699f20fa1c363eed506e94248d71f85412b625026f6142c"}, - {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:8c11160913e3dd06c8ffdb5f233a4f254cb449f4dfc0f8f4549eda9e542c93d1"}, - {file = "Pillow-10.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a74ba0c356aaa3bb8e3eb79606a87669e7ec6444be352870623025d75a14a2bf"}, - {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d5d0dae4cfd56969d23d94dc8e89fb6a217be461c69090768227beb8ed28c0a3"}, - {file = "Pillow-10.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22c10cc517668d44b211717fd9775799ccec4124b9a7f7b3635fc5386e584992"}, - {file = "Pillow-10.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:dffe31a7f47b603318c609f378ebcd57f1554a3a6a8effbc59c3c69f804296de"}, - {file = "Pillow-10.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:9fb218c8a12e51d7ead2a7c9e101a04982237d4855716af2e9499306728fb485"}, - {file = "Pillow-10.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d35e3c8d9b1268cbf5d3670285feb3528f6680420eafe35cccc686b73c1e330f"}, - {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ed64f9ca2f0a95411e88a4efbd7a29e5ce2cea36072c53dd9d26d9c76f753b3"}, - {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b6eb5502f45a60a3f411c63187db83a3d3107887ad0d036c13ce836f8a36f1d"}, - {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c1fbe7621c167ecaa38ad29643d77a9ce7311583761abf7836e1510c580bf3dd"}, - {file = "Pillow-10.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cd25d2a9d2b36fcb318882481367956d2cf91329f6892fe5d385c346c0649629"}, - {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3b08d4cc24f471b2c8ca24ec060abf4bebc6b144cb89cba638c720546b1cf538"}, - {file = "Pillow-10.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d737a602fbd82afd892ca746392401b634e278cb65d55c4b7a8f48e9ef8d008d"}, - {file = "Pillow-10.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:3a82c40d706d9aa9734289740ce26460a11aeec2d9c79b7af87bb35f0073c12f"}, - {file = "Pillow-10.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:bc2ec7c7b5d66b8ec9ce9f720dbb5fa4bace0f545acd34870eff4a369b44bf37"}, - {file = "Pillow-10.0.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:d80cf684b541685fccdd84c485b31ce73fc5c9b5d7523bf1394ce134a60c6883"}, - {file = "Pillow-10.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:76de421f9c326da8f43d690110f0e79fe3ad1e54be811545d7d91898b4c8493e"}, - {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81ff539a12457809666fef6624684c008e00ff6bf455b4b89fd00a140eecd640"}, - {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce543ed15570eedbb85df19b0a1a7314a9c8141a36ce089c0a894adbfccb4568"}, - {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:685ac03cc4ed5ebc15ad5c23bc555d68a87777586d970c2c3e216619a5476223"}, - {file = "Pillow-10.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d72e2ecc68a942e8cf9739619b7f408cc7b272b279b56b2c83c6123fcfa5cdff"}, - {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d50b6aec14bc737742ca96e85d6d0a5f9bfbded018264b3b70ff9d8c33485551"}, - {file = "Pillow-10.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:00e65f5e822decd501e374b0650146063fbb30a7264b4d2744bdd7b913e0cab5"}, - {file = "Pillow-10.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:f31f9fdbfecb042d046f9d91270a0ba28368a723302786c0009ee9b9f1f60199"}, - {file = "Pillow-10.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:1ce91b6ec08d866b14413d3f0bbdea7e24dfdc8e59f562bb77bc3fe60b6144ca"}, - {file = "Pillow-10.0.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:349930d6e9c685c089284b013478d6f76e3a534e36ddfa912cde493f235372f3"}, - {file = "Pillow-10.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3a684105f7c32488f7153905a4e3015a3b6c7182e106fe3c37fbb5ef3e6994c3"}, - {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4f69b3700201b80bb82c3a97d5e9254084f6dd5fb5b16fc1a7b974260f89f43"}, - {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f07ea8d2f827d7d2a49ecf1639ec02d75ffd1b88dcc5b3a61bbb37a8759ad8d"}, - {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:040586f7d37b34547153fa383f7f9aed68b738992380ac911447bb78f2abe530"}, - {file = "Pillow-10.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:f88a0b92277de8e3ca715a0d79d68dc82807457dae3ab8699c758f07c20b3c51"}, - {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c7cf14a27b0d6adfaebb3ae4153f1e516df54e47e42dcc073d7b3d76111a8d86"}, - {file = "Pillow-10.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3400aae60685b06bb96f99a21e1ada7bc7a413d5f49bce739828ecd9391bb8f7"}, - {file = "Pillow-10.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:dbc02381779d412145331789b40cc7b11fdf449e5d94f6bc0b080db0a56ea3f0"}, - {file = "Pillow-10.0.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9211e7ad69d7c9401cfc0e23d49b69ca65ddd898976d660a2fa5904e3d7a9baa"}, - {file = "Pillow-10.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:faaf07ea35355b01a35cb442dd950d8f1bb5b040a7787791a535de13db15ed90"}, - {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f72a021fbb792ce98306ffb0c348b3c9cb967dce0f12a49aa4c3d3fdefa967"}, - {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f7c16705f44e0504a3a2a14197c1f0b32a95731d251777dcb060aa83022cb2d"}, - {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:76edb0a1fa2b4745fb0c99fb9fb98f8b180a1bbceb8be49b087e0b21867e77d3"}, - {file = "Pillow-10.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:368ab3dfb5f49e312231b6f27b8820c823652b7cd29cfbd34090565a015e99ba"}, - {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:608bfdee0d57cf297d32bcbb3c728dc1da0907519d1784962c5f0c68bb93e5a3"}, - {file = "Pillow-10.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5c6e3df6bdd396749bafd45314871b3d0af81ff935b2d188385e970052091017"}, - {file = "Pillow-10.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:7be600823e4c8631b74e4a0d38384c73f680e6105a7d3c6824fcf226c178c7e6"}, - {file = "Pillow-10.0.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:92be919bbc9f7d09f7ae343c38f5bb21c973d2576c1d45600fce4b74bafa7ac0"}, - {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8182b523b2289f7c415f589118228d30ac8c355baa2f3194ced084dac2dbba"}, - {file = "Pillow-10.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:38250a349b6b390ee6047a62c086d3817ac69022c127f8a5dc058c31ccef17f3"}, - {file = "Pillow-10.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:88af2003543cc40c80f6fca01411892ec52b11021b3dc22ec3bc9d5afd1c5334"}, - {file = "Pillow-10.0.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c189af0545965fa8d3b9613cfdb0cd37f9d71349e0f7750e1fd704648d475ed2"}, - {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7b031a6fc11365970e6a5686d7ba8c63e4c1cf1ea143811acbb524295eabed"}, - {file = "Pillow-10.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:db24668940f82321e746773a4bc617bfac06ec831e5c88b643f91f122a785684"}, - {file = "Pillow-10.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:efe8c0681042536e0d06c11f48cebe759707c9e9abf880ee213541c5b46c5bf3"}, - {file = "Pillow-10.0.0.tar.gz", hash = "sha256:9c82b5b3e043c7af0d95792d0d20ccf68f61a1fec6b3530e718b688422727396"}, + {file = "Pillow-10.0.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a"}, + {file = "Pillow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff"}, + {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf"}, + {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd"}, + {file = "Pillow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0"}, + {file = "Pillow-10.0.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1"}, + {file = "Pillow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2"}, + {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b"}, + {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1"}, + {file = "Pillow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088"}, + {file = "Pillow-10.0.1-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b"}, + {file = "Pillow-10.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91"}, + {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4"}, + {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08"}, + {file = "Pillow-10.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08"}, + {file = "Pillow-10.0.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a"}, + {file = "Pillow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7"}, + {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a"}, + {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7"}, + {file = "Pillow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3"}, + {file = "Pillow-10.0.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849"}, + {file = "Pillow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145"}, + {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2"}, + {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf"}, + {file = "Pillow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d"}, + {file = "Pillow-10.0.1.tar.gz", hash = "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d"}, ] [package.extras] @@ -1428,24 +1442,24 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.24.2" +version = "4.24.3" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.24.2-cp310-abi3-win32.whl", hash = "sha256:58e12d2c1aa428ece2281cef09bbaa6938b083bcda606db3da4e02e991a0d924"}, - {file = "protobuf-4.24.2-cp310-abi3-win_amd64.whl", hash = "sha256:77700b55ba41144fc64828e02afb41901b42497b8217b558e4a001f18a85f2e3"}, - {file = "protobuf-4.24.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:237b9a50bd3b7307d0d834c1b0eb1a6cd47d3f4c2da840802cd03ea288ae8880"}, - {file = "protobuf-4.24.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:25ae91d21e3ce8d874211110c2f7edd6384816fb44e06b2867afe35139e1fd1c"}, - {file = "protobuf-4.24.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:c00c3c7eb9ad3833806e21e86dca448f46035242a680f81c3fe068ff65e79c74"}, - {file = "protobuf-4.24.2-cp37-cp37m-win32.whl", hash = "sha256:4e69965e7e54de4db989289a9b971a099e626f6167a9351e9d112221fc691bc1"}, - {file = "protobuf-4.24.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c5cdd486af081bf752225b26809d2d0a85e575b80a84cde5172a05bbb1990099"}, - {file = "protobuf-4.24.2-cp38-cp38-win32.whl", hash = "sha256:6bd26c1fa9038b26c5c044ee77e0ecb18463e957fefbaeb81a3feb419313a54e"}, - {file = "protobuf-4.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb7aa97c252279da65584af0456f802bd4b2de429eb945bbc9b3d61a42a8cd16"}, - {file = "protobuf-4.24.2-cp39-cp39-win32.whl", hash = "sha256:2b23bd6e06445699b12f525f3e92a916f2dcf45ffba441026357dea7fa46f42b"}, - {file = "protobuf-4.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:839952e759fc40b5d46be319a265cf94920174d88de31657d5622b5d8d6be5cd"}, - {file = "protobuf-4.24.2-py3-none-any.whl", hash = "sha256:3b7b170d3491ceed33f723bbf2d5a260f8a4e23843799a3906f16ef736ef251e"}, - {file = "protobuf-4.24.2.tar.gz", hash = "sha256:7fda70797ddec31ddfa3576cbdcc3ddbb6b3078b737a1a87ab9136af0570cd6e"}, + {file = "protobuf-4.24.3-cp310-abi3-win32.whl", hash = "sha256:20651f11b6adc70c0f29efbe8f4a94a74caf61b6200472a9aea6e19898f9fcf4"}, + {file = "protobuf-4.24.3-cp310-abi3-win_amd64.whl", hash = "sha256:3d42e9e4796a811478c783ef63dc85b5a104b44aaaca85d4864d5b886e4b05e3"}, + {file = "protobuf-4.24.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:6e514e8af0045be2b56e56ae1bb14f43ce7ffa0f68b1c793670ccbe2c4fc7d2b"}, + {file = "protobuf-4.24.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:ba53c2f04798a326774f0e53b9c759eaef4f6a568ea7072ec6629851c8435959"}, + {file = "protobuf-4.24.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:f6ccbcf027761a2978c1406070c3788f6de4a4b2cc20800cc03d52df716ad675"}, + {file = "protobuf-4.24.3-cp37-cp37m-win32.whl", hash = "sha256:1b182c7181a2891e8f7f3a1b5242e4ec54d1f42582485a896e4de81aa17540c2"}, + {file = "protobuf-4.24.3-cp37-cp37m-win_amd64.whl", hash = "sha256:b0271a701e6782880d65a308ba42bc43874dabd1a0a0f41f72d2dac3b57f8e76"}, + {file = "protobuf-4.24.3-cp38-cp38-win32.whl", hash = "sha256:e29d79c913f17a60cf17c626f1041e5288e9885c8579832580209de8b75f2a52"}, + {file = "protobuf-4.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:067f750169bc644da2e1ef18c785e85071b7c296f14ac53e0900e605da588719"}, + {file = "protobuf-4.24.3-cp39-cp39-win32.whl", hash = "sha256:2da777d34b4f4f7613cdf85c70eb9a90b1fbef9d36ae4a0ccfe014b0b07906f1"}, + {file = "protobuf-4.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:f631bb982c5478e0c1c70eab383af74a84be66945ebf5dd6b06fc90079668d0b"}, + {file = "protobuf-4.24.3-py3-none-any.whl", hash = "sha256:f6f8dc65625dadaad0c8545319c2e2f0424fede988368893ca3844261342c11a"}, + {file = "protobuf-4.24.3.tar.gz", hash = "sha256:12e9ad2ec079b833176d2921be2cb24281fa591f0b119b208b788adc48c2561d"}, ] [[package]] @@ -1517,13 +1531,13 @@ numpy = ">=1.16.6" [[package]] name = "pytest" -version = "7.4.0" +version = "7.4.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, ] [package.dependencies] @@ -1553,13 +1567,13 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2023.3" +version = "2023.3.post1" description = "World timezone definitions, modern and historical" optional = true python-versions = "*" files = [ - {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, - {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] [[package]] @@ -1916,19 +1930,19 @@ files = [ [[package]] name = "setuptools" -version = "68.1.2" +version = "68.2.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-68.1.2-py3-none-any.whl", hash = "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"}, - {file = "setuptools-68.1.2.tar.gz", hash = "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d"}, + {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, + {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5,<=7.1.2)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -2092,13 +2106,13 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.32.1" +version = "4.33.2" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" optional = false python-versions = ">=3.8.0" files = [ - {file = "transformers-4.32.1-py3-none-any.whl", hash = "sha256:b930d3dbd907a3f300cf49e54d63a56f8a0ab16b01a2c2a61ecff37c6de1da08"}, - {file = "transformers-4.32.1.tar.gz", hash = "sha256:1edc8ae1de357d97c3d36b04412aa63d55e6fc0c4b39b419a7d380ed947d2252"}, + {file = "transformers-4.33.2-py3-none-any.whl", hash = "sha256:5a9a757bea5b5a1b94796805bcb5978b552208a3ac193f46edda66be6f4a5488"}, + {file = "transformers-4.33.2.tar.gz", hash = "sha256:47dd36f302afec86d9cdcacab61bbd0296e6bb02e64d2ed7855daaab14ee290e"}, ] [package.dependencies] @@ -2115,16 +2129,16 @@ tqdm = ">=4.27" [package.extras] accelerate = ["accelerate (>=0.20.3)"] -agents = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.9,!=1.12.0)"] -all = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +agents = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.10,!=1.12.0)"] +all = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] codecarbon = ["codecarbon (==1.2.0)"] deepspeed = ["accelerate (>=0.20.3)", "deepspeed (>=0.9.3)"] deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -docs = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "torchaudio", "torchvision"] +dev = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] docs-specific = ["hf-doc-builder"] fairscale = ["fairscale (>0.3)"] flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] @@ -2147,15 +2161,15 @@ sigopt = ["sigopt"] sklearn = ["scikit-learn"] speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx"] -tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.14)", "tensorflow-text (<2.14)", "tf2onnx"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx"] tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] timm = ["timm"] tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -torch = ["accelerate (>=0.20.3)", "torch (>=1.9,!=1.12.0)"] +torch = ["accelerate (>=0.20.3)", "torch (>=1.10,!=1.12.0)"] torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] torch-vision = ["Pillow (<10.0.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.15.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.9,!=1.12.0)", "tqdm (>=4.27)"] +torchhub = ["filelock", "huggingface-hub (>=0.15.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "tqdm (>=4.27)"] video = ["av (==9.2.0)", "decord (==0.6.0)"] vision = ["Pillow (<10.0.0)"] @@ -2181,13 +2195,13 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6. [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, ] [[package]] @@ -2203,13 +2217,13 @@ files = [ [[package]] name = "urllib3" -version = "2.0.4" +version = "2.0.5" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.7" files = [ - {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, - {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, + {file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"}, + {file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"}, ] [package.extras] diff --git a/server/pyproject.toml b/server/pyproject.toml index 1babf749..f6eebf92 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -54,5 +54,7 @@ priority = "explicit" markers = ["private: marks tests as requiring an admin hf token (deselect with '-m \"not private\"')"] [build-system] -requires = ["poetry-core>=1.0.0"] +requires = [ + "poetry-core>=1.0.0", +] build-backend = "poetry.core.masonry.api" diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index e3fda07f..e0b8c0fe 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -17,6 +17,7 @@ class Quantization(str, Enum): bitsandbytes_nf4 = "bitsandbytes-nf4" bitsandbytes_fp4 = "bitsandbytes-fp4" gptq = "gptq" + awq = "awq" class Dtype(str, Enum): diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 29b049cf..0d96d43b 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -268,6 +268,10 @@ def get_model( raise ValueError( "gptq quantization is not supported for AutoModel, you can try to quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" ) + if quantize == "awq": + raise ValueError( + "awq quantization is not supported for AutoModel" + ) elif (quantize == "bitsandbytes-fp4") or (quantize == "bitsandbytes-nf4"): raise ValueError( "4bit quantization is not supported for AutoModel" diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 063aa01e..d2ed0b15 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -62,7 +62,7 @@ def __init__( filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights(filenames, device, dtype, process_group=self.process_group) - if config.quantize == "gptq": + if config.quantize in ["gptq", "awq"]: weights._set_gptq_params(model_id) model = FlashLlamaForCausalLM(config, weights) diff --git a/server/text_generation_server/utils/awq/quantize/qmodule.py b/server/text_generation_server/utils/awq/quantize/qmodule.py new file mode 100644 index 00000000..c658e17f --- /dev/null +++ b/server/text_generation_server/utils/awq/quantize/qmodule.py @@ -0,0 +1,48 @@ +# Copied logic from https://github.com/mit-han-lab/llm-awq/blob/f084f40bd996f3cf3a0633c1ad7d9d476c318aaa/awq/quantize/qmodule.py + +import math +import torch +import torch.nn as nn +import awq_inference_engine # with CUDA kernels + + +# class ScaledActivation(nn.Module): +# def __init__(self, module, scales): +# super().__init__() +# self.act = module +# self.scales = nn.Parameter(scales.data) +# +# def forward(self, x): +# return self.act(x) / self.scales.view(1, 1, -1).to(x.device) + + +class WQLinear(nn.Module): + def __init__(self, w_bit, group_size, qweight, qzeros, scales, bias): + super().__init__() + + if w_bit not in [4]: + raise NotImplementedError("Only 4-bit are supported for now.") + + self.in_features = qweight.shape[0] + self.out_features = qweight.shape[1] * 32 // w_bit + + self.w_bit = w_bit + self.group_size = group_size if group_size != -1 else self.in_features + # quick sanity check (make sure aligment) + assert self.in_features % self.group_size == 0 + assert self.out_features % (32 // self.w_bit) == 0 + + self.qweight = qweight + self.qzeros = qzeros + self.scales = scales + if bias: + self.bias = bias + else: + self.bias = None + + @torch.no_grad() + def forward(self, x): + out_shape = x.shape[:-1] + (self.out_features, ) + out = awq_inference_engine.gemm_forward_cuda(x.reshape(-1, x.shape[-1]), self.qweight, self.scales, self.qzeros, 8) + out = out + self.bias if self.bias is not None else out + return out.reshape(out_shape) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index c1c36194..fb27764c 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -18,6 +18,13 @@ from text_generation_server.utils.gptq.quant_linear import QuantLinear + +HAS_AWQ = True +try: + from text_generation_server.utils.awq.quantize.qmodule import WQLinear +except ImportError: + HAS_AWQ = False + try: major, _minor = torch.cuda.get_device_capability() except Exception: @@ -248,6 +255,14 @@ def get_linear(weight, bias, quantize): bits, groupsize, ) + elif quantize == "awq": + try: + qweight, qzeros, scales, _, bits, groupsize, _ = weight + except Exception: + raise NotImplementedError( + f"The passed weight is not `awq` compatible, loader needs to be updated." + ) + linear = WQLinear(w_bit=bits, group_size=groupsize, qweight=qweight, qzeros=qzeros, scales=scales, bias=bias is not None) else: raise NotImplementedError(f"Quantization `{quantize}` is not implemented yet.") return linear @@ -283,8 +298,8 @@ def load(config, prefix: str, weights): weight = weights.get_tensor(f"{prefix}.weight") should_gather = False - # GPTQ doesn't quantize heads (nor embeddings) - if config.quantize == "gptq": + # GPTQ and AWQ don't quantize heads (nor embeddings) + if config.quantize in ["gptq", "awq"]: quantize = None else: quantize = config.quantize diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 2ef7ad39..266fcccb 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -135,18 +135,21 @@ def get_weights_col_packed_qkv(self, prefix: str, quantize: str): Highly specific when the underlying tensor is a simple cat of Q,K,V instead of being already alternating Q,K,V within the main tensor """ - if quantize == "gptq": + if quantize in ["gptq", "awq"]: try: qweight = self._get_qweight(f"{prefix}.qweight") except RuntimeError: raise RuntimeError( - "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" + f"Cannot load `{quantize}` weight, make sure the model is already quantized." ) qzeros = self._get_qweight(f"{prefix}.qzeros") scales = self._get_qweight(f"{prefix}.scales") scales = scales.to(dtype=self.dtype) - g_idx = self.get_tensor(f"{prefix}.g_idx") + if quantize == "gptq": + g_idx = self.get_tensor(f"{prefix}.g_idx") + else: + g_idx = None bits, groupsize = self._get_gptq_params() weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) @@ -171,14 +174,14 @@ def get_weights_col_packed_qkv(self, prefix: str, quantize: str): return weight def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): - if quantize == "gptq": + if quantize in ["gptq", "awq"]: try: qweight = torch.cat( [self.get_sharded(f"{p}.qweight", dim=1) for p in prefixes], dim=1 ) except RuntimeError: raise RuntimeError( - "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" + f"Cannot load `{quantize}` weight, make sure the model is already quantized" ) qzeros = torch.cat( @@ -187,10 +190,14 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): scales = torch.cat( [self.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1 ) - w = [self.get_tensor(f"{p}.g_idx") for p in prefixes] - for w2 in w[1:]: - torch.testing.assert_close(w2, w[0]) - g_idx = w[0] + + if quantize == "gptq": + w = [self.get_tensor(f"{p}.g_idx") for p in prefixes] + for w2 in w[1:]: + torch.testing.assert_close(w2, w[0]) + g_idx = w[0] + else: + g_idx = None bits, groupsize = self._get_gptq_params() weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) @@ -281,6 +288,22 @@ def get_multi_weights_row(self, prefix: str, quantize: str): scales = self.get_tensor(f"{prefix}.scales") g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) + weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) + elif quantize == "awq": + bits, groupsize = self._get_gptq_params() + + try: + qweight = self.get_sharded(f"{prefix}.qweight", dim=0) + except RuntimeError: + raise RuntimeError( + "Cannot load `awq` weight, make sure the model is already quantized" + ) + + qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) + scales = self.get_sharded(f"{prefix}.scales", dim=0) + g_idx = None + use_exllama = False + weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) else: weight = self.get_sharded(f"{prefix}.weight", dim=1) @@ -322,4 +345,15 @@ def _set_gptq_params(self, model_id): self.gptq_bits = data["bits"] self.gptq_groupsize = data["group_size"] except Exception: - pass + filename = "quant_config.json" + try: + if os.path.exists(os.path.join(model_id, filename)): + filename = os.path.join(model_id, filename) + else: + filename = hf_hub_download(model_id, filename=filename) + with open(filename, "r") as f: + data = json.load(f) + self.gptq_bits = data["w_bit"] + self.gptq_groupsize = data["q_group_size"] + except Exception: + pass From 2f51645ad7274f8d533a211804b74507263a946c Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 26 Sep 2023 08:27:50 +0200 Subject: [PATCH 424/793] Fix GQA llama + AWQ (#1061) # What does this PR do? Fixes #1056 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../models/custom_modeling/flash_llama_modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 55b1aae9..cb0c1e85 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -179,7 +179,7 @@ def _load_gqa(config, prefix: str, weights): dim=0, ) - if config.quantize != "gptq": + if config.quantize not in ["gptq", "awq"]: weight = weight.to(dtype=weights.dtype).to(device=weights.device) head_size = config.hidden_size // config.num_attention_heads From edc95a0e7dc4c8e13a20fbf2c8e2b99a8c3e4549 Mon Sep 17 00:00:00 2001 From: zhangsibo1129 <134488188+zhangsibo1129@users.noreply.github.com> Date: Tue, 26 Sep 2023 20:57:53 +0800 Subject: [PATCH 425/793] support local model config file (#1058) # What does this PR do? Support local config file to avoid unexpected `discard_names`, which causes #1057. In the case of launching local mode without `model.safetensors` file, the original code will result `discard_names = []` when `hf_hub_download` throws an connection error. ```python # server/text_generation_server/cli.py try: import transformers import json config_filename = hf_hub_download(model_id, revision=revision, filename="config.json") with open(config_filename, "r") as f: config = json.load(f) architecture = config["architectures"][0] class_ = getattr(transformers, architecture) # Name for this varible depends on transformers version. discard_names = getattr(class_, "_tied_weights_keys", []) discard_names.extend(getattr(class_, "_keys_to_ignore_on_load_missing", [])) except Exception as e: discard_names = [] ``` The expected `_tied_weights_keys` of OPT-1.3b is `["lm_head.weight"]`, and its tied weight `"model.decoder.embed_tokens.weight"` will be kept in the safetensors conversion. But the above empty `discard_names` will lead to `"lm_head.weight"` being kept and `"model.decoder.embed_tokens.weight"` being discard in the subsequent method `_remove_duplicate_names`, which causes error #1057. So add a local mode branch to get the expected `discard_names` like follows. This modification also applies to other models ```python # server/text_generation_server/cli.py if is_local_model: config_filename = os.path.join(model_id, "config.json") else: config_filename = hf_hub_download(model_id, revision=revision, filename="config.json") ``` In addition, when `_tied_weights_keys` or `_keys_to_ignore_on_load_missing` is `None`, the above code will also throw an error unexpectedly. This is fixed in PR #1052 ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [x] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). N/A - [ ] Did you write any new necessary tests? N/A ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @Narsil --- server/text_generation_server/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index e0b8c0fe..33048139 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -179,7 +179,10 @@ def download_weights( import json - config_filename = hf_hub_download(model_id, revision=revision, filename="config.json") + if is_local_model: + config_filename = os.path.join(model_id, "config.json") + else: + config_filename = hf_hub_download(model_id, revision=revision, filename="config.json") with open(config_filename, "r") as f: config = json.load(f) architecture = config["architectures"][0] From eba6ab1c5dd21ab613fd0745a973898e96d03186 Mon Sep 17 00:00:00 2001 From: zhangsibo1129 <134488188+zhangsibo1129@users.noreply.github.com> Date: Tue, 26 Sep 2023 21:05:40 +0800 Subject: [PATCH 426/793] fix discard_names bug in safetensors convertion (#1052) # What does this PR do? Model Class attributes `_tied_weights_keys`, ` _keys_to_ignore_on_load_missing` can only be `None` or a List. `getattr(class_, "_keys_to_ignore_on_load_missing", [])` will return `None` if `_keys_to_ignore_on_load_missing` is None, and `discard_names.extend(None)` will trigger an exception, even though `_tied_weights_keys` exists. ## Who can review? @OlivierDehaene @Narsil --------- Co-authored-by: Nicolas Patry --- server/text_generation_server/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 33048139..7464934f 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -191,7 +191,6 @@ def download_weights( # Name for this varible depends on transformers version. discard_names = getattr(class_, "_tied_weights_keys", []) - discard_names.extend(getattr(class_, "_keys_to_ignore_on_load_missing", [])) except Exception as e: discard_names = [] From ae623b8d2d83906c07ddcf0c04dcaf3c3dc2b0f0 Mon Sep 17 00:00:00 2001 From: oOraph <13552058+oOraph@users.noreply.github.com> Date: Tue, 26 Sep 2023 15:23:47 +0200 Subject: [PATCH 427/793] Install curl to be able to perform more advanced healthchecks (#1033) # What does this PR do? Install curl within base image, negligible regarding the image volume and will allow to easily perform a better health check. Not sure about the failing github actions though. Should I fix something ? Signed-off-by: Raphael Co-authored-by: Raphael --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 1d57d110..56f4775b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -158,6 +158,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins libssl-dev \ ca-certificates \ make \ + curl \ && rm -rf /var/lib/apt/lists/* # Copy conda with PyTorch installed From 1fff6746ab363e897fc08d742e7dfde4304d70ac Mon Sep 17 00:00:00 2001 From: Victor SANH Date: Tue, 26 Sep 2023 15:41:15 +0200 Subject: [PATCH 428/793] Fix position ids logic instantiation of idefics vision part (#1064) Problem and fix is described here: https://huggingface.co/HuggingFaceM4/idefics-9b/discussions/9 --------- Co-authored-by: Nicolas Patry --- .../models/custom_modeling/idefics_vision.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/idefics_vision.py b/server/text_generation_server/models/custom_modeling/idefics_vision.py index d933d7c1..30f07095 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_vision.py +++ b/server/text_generation_server/models/custom_modeling/idefics_vision.py @@ -88,12 +88,10 @@ def __init__(self, prefix, config, weights): self.num_patches = (self.image_size // self.patch_size) ** 2 self.num_positions = self.num_patches + 1 - # self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) self.position_embedding = TensorParallelEmbedding( prefix="model.vision_model.embeddings.position_embedding", weights=weights ) - # self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1)), persistent=False) - self.position_ids = weights.get_tensor(f"{prefix}.position_ids") + self.position_ids = torch.arange(self.num_positions).expand((1, -1)).to(device=weights.device) def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: batch_size = pixel_values.shape[0] From 8672cad2cbe9b31d82e54223d0f4815ffe426cee Mon Sep 17 00:00:00 2001 From: Vincent Brouwers Date: Tue, 26 Sep 2023 16:16:43 +0200 Subject: [PATCH 429/793] Fix top_n_tokens returning non-log probs for some models (#1023) # What does this PR do? I made an embarrassing mistake where I accidentally passed normal softmax probabilities into `batch_top_tokens` for `CausalLM` and `Seq2SeqLM`. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @Narsil --- server/text_generation_server/models/causal_lm.py | 2 +- server/text_generation_server/models/seq2seq_lm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index cec9ae55..696f0fb2 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -579,7 +579,7 @@ def generate_token( batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( batch.top_n_tokens, batch.top_n_tokens_tensor, - torch.softmax(logits[:, -1], -1), + torch.log_softmax(logits[:, -1], -1), ) # Zipped iterator diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 1a7911ac..34932c0b 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -642,7 +642,7 @@ def generate_token( batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( batch.top_n_tokens, batch.top_n_tokens_tensor, - torch.softmax(logits[:, -1], -1), + torch.log_softmax(logits[:, -1], -1), ) # Finished requests From a049864270aa44226ccc33f3b5ef929222307e9c Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 27 Sep 2023 10:40:18 +0200 Subject: [PATCH 430/793] Preping 1.1.0 (#1066) # What does this PR do? Upgrade all relevant versions and dependencies. Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- Cargo.lock | 1199 ++++++++--------- Cargo.toml | 2 +- README.md | 2 +- benchmark/Cargo.toml | 21 +- benchmark/src/app.rs | 14 +- benchmark/src/lib.rs | 2 +- docs/openapi.json | 2 +- .../basic_tutorials/gated_model_access.md | 2 +- docs/source/quicktour.md | 2 +- integration-tests/pyproject.toml | 2 +- launcher/Cargo.toml | 16 +- router/Cargo.toml | 48 +- router/client/Cargo.toml | 10 +- router/grpc-metadata/Cargo.toml | 6 +- router/src/main.rs | 2 +- router/src/server.rs | 10 +- router/src/validation.rs | 2 +- server/pyproject.toml | 2 +- server/requirements.txt | 28 +- 19 files changed, 634 insertions(+), 738 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f3b39de..5fa01f0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -17,17 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aes" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.8.3" @@ -50,33 +39,32 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.3" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8f9420f797f2d9e935edf629310eb938a0d839f984e25327f3c7eed22300c" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" dependencies = [ "memchr", ] [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" [[package]] name = "anstyle-parse" @@ -98,9 +86,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -108,9 +96,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.72" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "arc-swap" @@ -125,7 +113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93b21a03b7c21702a0110f9f8d228763a533570deb376119042dabf33c37a01a" dependencies = [ "futures-io", - "rustls", + "rustls 0.20.9", "webpki", ] @@ -148,7 +136,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -159,7 +147,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -170,9 +158,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "average" -version = "0.13.1" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843ec791d3f24503bbf72bbd5e49a3ab4dbb4bcd0a8ef6b0c908efa73caa27b1" +checksum = "6d804c74bb2d66e9b7047658d21af0f1c937d7d2466410cbf1aed3b0c04048d4" dependencies = [ "easy-cast", "float-ord", @@ -242,25 +230,27 @@ dependencies = [ [[package]] name = "axum-tracing-opentelemetry" -version = "0.10.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164b95427e83b79583c7699a72b4a6b485a12bbdef5b5c054ee5ff2296d82f52" +checksum = "06985105829f176e9a3f113b1c71cc24e08f600ef0df4e70cd90d144f889e19f" dependencies = [ "axum", - "futures", + "futures-core", + "futures-util", "http", - "opentelemetry 0.18.0", + "opentelemetry", + "pin-project-lite", "tower", - "tower-http 0.3.5", "tracing", - "tracing-opentelemetry 0.18.0", + "tracing-opentelemetry", + "tracing-opentelemetry-instrumentation-sdk", ] [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -279,15 +269,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" - -[[package]] -name = "base64ct" -version = "1.6.0" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bitflags" @@ -312,9 +296,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytecount" @@ -330,52 +314,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" - -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "cached-path" -version = "0.6.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097968e38f1319207f057d0f4d76452e4f4f847a5de61c5215379f297fa034f3" -dependencies = [ - "flate2", - "fs2", - "glob", - "indicatif 0.16.2", - "log", - "rand", - "reqwest", - "serde", - "serde_json", - "sha2", - "tar", - "tempfile", - "thiserror", - "zip", -] +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cassowary" @@ -385,11 +326,10 @@ checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" [[package]] name = "cc" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ - "jobserver", "libc", ] @@ -399,32 +339,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clap" -version = "4.3.21" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" +checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.3.21" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" +checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" dependencies = [ "anstream", "anstyle", @@ -434,21 +363,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.12" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "colorchoice" @@ -469,12 +398,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "core-foundation" version = "0.9.3" @@ -539,7 +462,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", + "memoffset", "scopeguard", ] @@ -554,11 +477,11 @@ dependencies = [ [[package]] name = "crossterm" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "crossterm_winapi", "libc", "mio", @@ -589,9 +512,9 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.0" +version = "3.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a011bbe2c35ce9c1f143b7af6f94f29a167beb4cd1d29e6740ce836f723120e" +checksum = "82e95fbd621905b854affdc67943b043a0fbb6ed7385fd5a25650d19a8a6cfdf" dependencies = [ "nix", "windows-sys 0.48.0", @@ -632,24 +555,11 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "dashmap" -version = "5.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" -dependencies = [ - "cfg-if", - "hashbrown 0.14.0", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "deranged" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" [[package]] name = "derive_builder" @@ -690,7 +600,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -699,7 +608,16 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ - "dirs-sys", + "dirs-sys 0.3.7", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys 0.4.1", ] [[package]] @@ -713,11 +631,23 @@ dependencies = [ "winapi", ] +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "easy-cast" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd102ee8c418348759919b83b81cdbdc933ffe29740b903df448b4bafaa348e" +checksum = "10936778145f3bea71fd9bf61332cce28c28e96a380714f7ab34838b80733fd6" dependencies = [ "libm", ] @@ -736,9 +666,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] @@ -751,9 +681,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -781,21 +711,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" - -[[package]] -name = "filetime" -version = "0.2.22" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", -] +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fixedbitset" @@ -805,9 +723,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -827,14 +745,13 @@ checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853" [[package]] name = "flume" -version = "0.10.14" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" dependencies = [ "futures-core", "futures-sink", "nanorand", - "pin-project", "spin 0.9.8", ] @@ -868,16 +785,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "futures" version = "0.3.28" @@ -934,7 +841,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -1001,31 +908,25 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" - -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "grpc-metadata" version = "0.1.0" dependencies = [ - "opentelemetry 0.19.0", - "tonic 0.9.2", + "opentelemetry", + "tonic 0.10.1", "tracing", - "tracing-opentelemetry 0.19.0", + "tracing-opentelemetry", ] [[package]] name = "h2" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -1069,17 +970,50 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] -name = "hmac" -version = "0.12.1" +name = "hf-hub" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "ca0a2d78bbd80fdb804b213f675d261eb87060571e1c389bcad8c24add066cf9" dependencies = [ - "digest", + "dirs 5.0.1", + "indicatif 0.17.7", + "native-tls", + "rand", + "serde", + "serde_json", + "thiserror", + "ureq", +] + +[[package]] +name = "hf-hub" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9caf63dfcf84b1b62532c836035516f5b475b06c906da6f16990d22847aba216" +dependencies = [ + "dirs 5.0.1", + "indicatif 0.17.7", + "log", + "native-tls", + "rand", + "serde", + "serde_json", + "thiserror", + "ureq", +] + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", ] [[package]] @@ -1233,42 +1167,51 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.16.2" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" dependencies = [ "console", - "lazy_static", + "instant", "number_prefix 0.4.0", - "regex", + "portable-atomic", + "unicode-width", ] [[package]] -name = "inout" -version = "0.1.3" +name = "indoc" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] -name = "ipnet" -version = "2.8.0" +name = "init-tracing-opentelemetry" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "94bd26b1b737bc11f183620072e188d1c6ede67e0e78682228d66b49ec510e17" +dependencies = [ + "opentelemetry", + "opentelemetry-otlp", + "thiserror", + "tracing", + "tracing-opentelemetry", +] [[package]] -name = "is-terminal" -version = "0.4.9" +name = "instant" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "hermit-abi", - "rustix", - "windows-sys 0.48.0", + "cfg-if", ] +[[package]] +name = "ipnet" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" + [[package]] name = "itertools" version = "0.8.2" @@ -1297,19 +1240,19 @@ dependencies = [ ] [[package]] -name = "itoa" -version = "1.0.9" +name = "itertools" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] [[package]] -name = "jobserver" -version = "0.1.26" +name = "itoa" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" -dependencies = [ - "libc", -] +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" @@ -1328,9 +1271,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libm" @@ -1340,9 +1283,9 @@ checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "lock_api" @@ -1402,24 +1345,15 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.7.1" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "memoffset" @@ -1447,7 +1381,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" dependencies = [ - "base64 0.21.2", + "base64 0.21.4", "hyper", "indexmap 1.9.3", "ipnet", @@ -1467,7 +1401,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -1546,7 +1480,7 @@ checksum = "371717c0a5543d6a800cac822eac735aa7d2d2fbb41002e9856a4089532dbdce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -1603,9 +1537,9 @@ dependencies = [ [[package]] name = "ngrok" -version = "0.12.4" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e211f407b0a084f720823a00c956aeab2c15dfe7a61760d93227bbaf048026" +checksum = "1454b1edbc5f2c8ff3242c237cb84388b50eced8eb26b4204e49698ed6511784" dependencies = [ "arc-swap", "async-rustls", @@ -1634,16 +1568,13 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cfg-if", "libc", - "memoffset 0.7.1", - "pin-utils", - "static_assertions", ] [[package]] @@ -1724,9 +1655,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.31.1" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] @@ -1761,11 +1692,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.56" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cfg-if", "foreign-types", "libc", @@ -1782,7 +1713,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -1793,9 +1724,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.91" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ "cc", "libc", @@ -1805,81 +1736,76 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" +checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" dependencies = [ - "opentelemetry_api 0.18.0", - "opentelemetry_sdk 0.18.0", + "opentelemetry_api", + "opentelemetry_sdk", ] [[package]] -name = "opentelemetry" -version = "0.19.0" +name = "opentelemetry-http" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4b8347cc26099d3aeee044065ecc3ae11469796b4d65d065a23a584ed92a6f" +checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b" dependencies = [ - "opentelemetry_api 0.19.0", - "opentelemetry_sdk 0.19.0", + "async-trait", + "bytes", + "http", + "opentelemetry_api", ] [[package]] name = "opentelemetry-otlp" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8af72d59a4484654ea8eb183fea5ae4eb6a41d7ac3e3bae5f4d2a282a3a7d3ca" +checksum = "7e5e5a5c4135864099f3faafbe939eb4d7f9b80ebf68a8448da961b32a7c1275" dependencies = [ "async-trait", - "futures", - "futures-util", + "futures-core", "http", - "opentelemetry 0.19.0", "opentelemetry-proto", - "prost", + "opentelemetry-semantic-conventions", + "opentelemetry_api", + "opentelemetry_sdk", + "prost 0.11.9", "thiserror", "tokio", - "tonic 0.8.3", + "tonic 0.9.2", ] [[package]] name = "opentelemetry-proto" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "045f8eea8c0fa19f7d48e7bc3128a39c2e5c533d5c61298c548dfefc1064474c" +checksum = "b1e3f814aa9f8c905d0ee4bde026afd3b2577a97c10e1699912e3e44f0c4cbeb" dependencies = [ - "futures", - "futures-util", - "opentelemetry 0.19.0", - "prost", - "tonic 0.8.3", + "opentelemetry_api", + "opentelemetry_sdk", + "prost 0.11.9", + "tonic 0.9.2", ] [[package]] -name = "opentelemetry_api" -version = "0.18.0" +name = "opentelemetry-semantic-conventions" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" dependencies = [ - "fnv", - "futures-channel", - "futures-util", - "indexmap 1.9.3", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", + "opentelemetry", ] [[package]] name = "opentelemetry_api" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed41783a5bf567688eb38372f2b7a8530f5a607a4b49d38dd7573236c23ca7e2" +checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" dependencies = [ - "fnv", "futures-channel", "futures-util", "indexmap 1.9.3", + "js-sys", "once_cell", "pin-project-lite", "thiserror", @@ -1888,46 +1814,40 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" dependencies = [ "async-trait", "crossbeam-channel", - "dashmap", - "fnv", "futures-channel", "futures-executor", "futures-util", "once_cell", - "opentelemetry_api 0.18.0", + "opentelemetry_api", + "ordered-float", "percent-encoding", "rand", + "regex", + "serde_json", "thiserror", "tokio", "tokio-stream", ] [[package]] -name = "opentelemetry_sdk" -version = "0.19.0" +name = "option-ext" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b3a2a91fdbfdd4d212c0dcc2ab540de2c2bcbbd90be17de7a7daf8822d010c1" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ordered-float" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" dependencies = [ - "async-trait", - "crossbeam-channel", - "dashmap", - "fnv", - "futures-channel", - "futures-executor", - "futures-util", - "once_cell", - "opentelemetry_api 0.19.0", - "percent-encoding", - "rand", - "thiserror", - "tokio", - "tokio-stream", + "num-traits", ] [[package]] @@ -1938,9 +1858,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "papergrid" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae7891b22598926e4398790c8fe6447930c72a67d36d983a49d6ce682ce83290" +checksum = "a2ccbe15f2b6db62f9a9871642746427e297b0ceb85f9a7f1ee5ff47d184d0c8" dependencies = [ "bytecount", "fnv", @@ -1967,18 +1887,7 @@ dependencies = [ "libc", "redox_syscall 0.3.5", "smallvec", - "windows-targets 0.48.1", -] - -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core", - "subtle", + "windows-targets 0.48.5", ] [[package]] @@ -1987,18 +1896,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", -] - [[package]] name = "percent-encoding" version = "2.3.0" @@ -2007,12 +1904,12 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 1.9.3", + "indexmap 2.0.0", ] [[package]] @@ -2032,14 +1929,14 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -2055,9 +1952,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f32154ba0af3a075eefa1eda8bb414ee928f62303a54ea85b8d6638ff1a6ee9e" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" [[package]] name = "ppv-lite86" @@ -2067,12 +1964,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.1.25" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 1.0.109", + "syn 2.0.37", ] [[package]] @@ -2101,9 +1998,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -2115,27 +2012,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.11.9", +] + +[[package]] +name = "prost" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +dependencies = [ + "bytes", + "prost-derive 0.12.1", ] [[package]] name = "prost-build" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck", - "itertools 0.10.5", - "lazy_static", + "itertools 0.11.0", "log", "multimap", + "once_cell", "petgraph", "prettyplease", - "prost", + "prost 0.12.1", "prost-types", "regex", - "syn 1.0.109", + "syn 2.0.37", "tempfile", "which", ] @@ -2153,13 +2060,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "prost-types" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" dependencies = [ - "prost", + "prost 0.12.1", ] [[package]] @@ -2180,9 +2100,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -2219,13 +2139,17 @@ dependencies = [ [[package]] name = "ratatui" -version = "0.20.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc0d032bccba900ee32151ec0265667535c230169f5a011154cdcd984e16829" +checksum = "2e2e4cd95294a85c3b4446e63ef054eea43e0205b1fd60120c16b74ff7ff96ad" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cassowary", "crossterm", + "indoc", + "itertools 0.11.0", + "paste", + "strum", "unicode-segmentation", "unicode-width", ] @@ -2241,9 +2165,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -2262,14 +2186,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -2303,14 +2225,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" dependencies = [ - "aho-corasick 1.0.3", + "aho-corasick 1.1.1", "memchr", - "regex-automata 0.3.6", - "regex-syntax 0.7.4", + "regex-automata 0.3.8", + "regex-syntax 0.7.5", ] [[package]] @@ -2324,13 +2246,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ - "aho-corasick 1.0.3", + "aho-corasick 1.1.1", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.7.5", ] [[package]] @@ -2341,17 +2263,17 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ - "base64 0.21.2", + "base64 0.21.4", "bytes", "encoding_rs", "futures-core", @@ -2418,7 +2340,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.28", + "syn 2.0.37", "walkdir", ] @@ -2449,9 +2371,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" dependencies = [ "bitflags 2.4.0", "errno", @@ -2462,9 +2384,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", "ring", @@ -2472,13 +2394,45 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.6", + "sct", +] + [[package]] name = "rustls-pemfile" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.2", + "base64 0.21.4", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +dependencies = [ + "ring", + "untrusted", ] [[package]] @@ -2552,35 +2506,35 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" [[package]] name = "serde" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] name = "serde_json" -version = "1.0.104" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -2609,22 +2563,11 @@ dependencies = [ "serde", ] -[[package]] -name = "sha1" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -2646,7 +2589,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" dependencies = [ - "dirs", + "dirs 4.0.0", ] [[package]] @@ -2687,9 +2630,9 @@ checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -2705,9 +2648,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "socket2" @@ -2721,9 +2664,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2756,12 +2699,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "strsim" version = "0.10.0" @@ -2769,10 +2706,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "subtle" -version = "2.5.0" +name = "strum" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.37", +] [[package]] name = "syn" @@ -2787,9 +2740,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.28" +version = "2.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" dependencies = [ "proc-macro2", "quote", @@ -2804,9 +2757,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.29.8" +version = "0.29.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d10ed79c22663a35a255d289a7fdcb43559fc77ff15df5ce6c341809e7867528" +checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2818,9 +2771,9 @@ dependencies = [ [[package]] name = "tabled" -version = "0.12.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce69a5028cd9576063ec1f48edb2c75339fd835e6094ef3e05b3a079bf594a6" +checksum = "dfe9c3632da101aba5131ed63f9eed38665f8b3c68703a6bb18124835c1a5d22" dependencies = [ "papergrid", "tabled_derive", @@ -2840,22 +2793,11 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "tar" -version = "0.4.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "tempfile" -version = "3.7.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", @@ -2866,12 +2808,13 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "1.0.1" +version = "1.1.0" dependencies = [ "average", "clap", "crossterm", "float-ord", + "hf-hub 0.3.1", "ratatui", "serde", "serde_json", @@ -2886,15 +2829,15 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.0.1" +version = "1.1.0" dependencies = [ "futures", "grpc-metadata", - "prost", + "prost 0.12.1", "prost-build", "thiserror", "tokio", - "tonic 0.9.2", + "tonic 0.10.1", "tonic-build", "tower", "tracing", @@ -2902,7 +2845,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.0.1" +version = "1.1.0" dependencies = [ "clap", "ctrlc", @@ -2918,7 +2861,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.0.1" +version = "1.1.0" dependencies = [ "async-stream", "axum", @@ -2926,11 +2869,13 @@ dependencies = [ "clap", "flume", "futures", + "hf-hub 0.3.1", + "init-tracing-opentelemetry", "metrics", "metrics-exporter-prometheus", "ngrok", "nohash-hasher", - "opentelemetry 0.19.0", + "opentelemetry", "opentelemetry-otlp", "rand", "reqwest", @@ -2940,9 +2885,9 @@ dependencies = [ "thiserror", "tokenizers", "tokio", - "tower-http 0.4.3", + "tower-http", "tracing", - "tracing-opentelemetry 0.19.0", + "tracing-opentelemetry", "tracing-subscriber", "utoipa", "utoipa-swagger-ui", @@ -2951,22 +2896,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -2981,9 +2926,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.25" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", "itoa", @@ -2996,15 +2941,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.11" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -3026,17 +2971,16 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.13.4" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aea68938177975ab09da68552b720eac941779ff386baceaf77e0f5f9cea645f" +checksum = "12b515a66453a4d68f03398054f7204fd0dde6b93d3f20ea90b08025ab49b499" dependencies = [ "aho-corasick 0.7.20", - "cached-path", "clap", "derive_builder", - "dirs", "esaxx-rs", "getrandom", + "hf-hub 0.2.0", "indicatif 0.15.0", "itertools 0.9.0", "lazy_static", @@ -3049,8 +2993,7 @@ dependencies = [ "rayon", "rayon-cond", "regex", - "regex-syntax 0.7.4", - "reqwest", + "regex-syntax 0.7.5", "serde", "serde_json", "spm_precompiled", @@ -3062,9 +3005,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.31.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40de3a2ba249dcb097e01be5e67a5ff53cf250397715a071a81543e8a832a920" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", "bytes", @@ -3074,7 +3017,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2 0.5.4", "tokio-macros", "windows-sys 0.48.0", ] @@ -3097,7 +3040,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -3134,9 +3077,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -3149,14 +3092,13 @@ dependencies = [ [[package]] name = "tonic" -version = "0.8.3" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ - "async-stream", "async-trait", "axum", - "base64 0.13.1", + "base64 0.21.4", "bytes", "futures-core", "futures-util", @@ -3167,30 +3109,26 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", - "prost-derive", + "prost 0.11.9", "tokio", "tokio-stream", - "tokio-util", "tower", "tower-layer", "tower-service", "tracing", - "tracing-futures", ] [[package]] name = "tonic" -version = "0.9.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +checksum = "14c00bc15e49625f3d2f20b17082601e5e17cf27ead69e805174026c194b6664" dependencies = [ + "async-stream", "async-trait", "axum", - "base64 0.21.2", + "base64 0.21.4", "bytes", - "futures-core", - "futures-util", "h2", "http", "http-body", @@ -3198,7 +3136,7 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", + "prost 0.12.1", "tokio", "tokio-stream", "tower", @@ -3209,15 +3147,15 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.9.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" +checksum = "c9d37bb15da06ae9bb945963066baca6561b505af93a52e949a85d28558459a2" dependencies = [ "prettyplease", "proc-macro2", "prost-build", "quote", - "syn 1.0.109", + "syn 2.0.37", ] [[package]] @@ -3242,28 +3180,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" -dependencies = [ - "bitflags 1.3.2", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "bitflags 2.4.0", "bytes", @@ -3310,7 +3229,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -3323,16 +3242,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-futures" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" -dependencies = [ - "pin-project", - "tracing", -] - [[package]] name = "tracing-log" version = "0.1.3" @@ -3346,12 +3255,14 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.18.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" +checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" dependencies = [ "once_cell", - "opentelemetry 0.18.0", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", "tracing", "tracing-core", "tracing-log", @@ -3359,17 +3270,16 @@ dependencies = [ ] [[package]] -name = "tracing-opentelemetry" -version = "0.19.0" +name = "tracing-opentelemetry-instrumentation-sdk" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00a39dcf9bfc1742fa4d6215253b33a6e474be78275884c216fc2a06267b3600" +checksum = "752ddd669b14a08036a89045e4ac4497ff7ce254e11386647751f36d7c7849ea" dependencies = [ - "once_cell", - "opentelemetry 0.19.0", + "http", + "opentelemetry-http", + "opentelemetry_api", "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", + "tracing-opentelemetry", ] [[package]] @@ -3411,15 +3321,15 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] @@ -3432,9 +3342,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -3462,9 +3372,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unicode_categories" @@ -3478,11 +3388,30 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b11c96ac7ee530603dcdf68ed1557050f374ce55a5a07193ebf8cbc9f8927e9" +dependencies = [ + "base64 0.21.4", + "flate2", + "log", + "native-tls", + "once_cell", + "rustls 0.21.7", + "rustls-webpki 0.100.3", + "serde", + "serde_json", + "url", + "webpki-roots", +] + [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -3503,9 +3432,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utoipa" -version = "3.4.4" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de634b7f8178c9c246c88ea251f3a0215c9a4d80778db2d7bd4423a78b5170ec" +checksum = "d82b1bc5417102a73e8464c686eef947bdfb99fcdfc0a4f228e81afa9526470a" dependencies = [ "indexmap 2.0.0", "serde", @@ -3515,15 +3444,15 @@ dependencies = [ [[package]] name = "utoipa-gen" -version = "3.4.5" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcba79cb3e5020d9bcc8313cd5aadaf51d6d54a6b3fd08c3d0360ae6b3c83d0" +checksum = "05d96dcd6fc96f3df9b3280ef480770af1b7c5d14bc55192baa9b067976d920c" dependencies = [ "proc-macro-error", "proc-macro2", "quote", "regex", - "syn 2.0.28", + "syn 2.0.37", ] [[package]] @@ -3556,9 +3485,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "8.2.4" +version = "8.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbc5ad0d9d26b2c49a5ab7da76c3e79d3ee37e7821799f8223fcb8f2f391a2e7" +checksum = "85e7dc29b3c54a2ea67ef4f953d5ec0c4085035c0ae2d325be1c0d2144bd9f16" dependencies = [ "anyhow", "rustc_version", @@ -3575,9 +3504,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -3619,7 +3548,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", "wasm-bindgen-shared", ] @@ -3653,7 +3582,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.37", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3676,23 +3605,33 @@ dependencies = [ [[package]] name = "webpki" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e" dependencies = [ "ring", "untrusted", ] +[[package]] +name = "webpki-roots" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" +dependencies = [ + "rustls-webpki 0.100.3", +] + [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix", ] [[package]] @@ -3713,9 +3652,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -3741,7 +3680,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.1", + "windows-targets 0.48.5", ] [[package]] @@ -3761,17 +3700,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -3782,9 +3721,9 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" @@ -3794,9 +3733,9 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" @@ -3806,9 +3745,9 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" @@ -3818,9 +3757,9 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" @@ -3830,9 +3769,9 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" @@ -3842,9 +3781,9 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" @@ -3854,26 +3793,18 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winreg" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" -dependencies = [ - "winapi", -] - -[[package]] -name = "xattr" -version = "1.0.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "libc", + "cfg-if", + "windows-sys 0.48.0", ] [[package]] @@ -3882,46 +3813,8 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ - "aes", "byteorder", - "bzip2", - "constant_time_eq", "crc32fast", "crossbeam-utils", "flate2", - "hmac", - "pbkdf2", - "sha1", - "time", - "zstd", -] - -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" -dependencies = [ - "cc", - "libc", - "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 9f526b27..9ca1e6d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.0.3" +version = "1.1.0" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index fd2afd0d..dc074d50 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ The easiest way of getting started is using the official Docker container: model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.3 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0 --model-id $model ``` **Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. diff --git a/benchmark/Cargo.toml b/benchmark/Cargo.toml index 67e04f0a..2dd2e64d 100644 --- a/benchmark/Cargo.toml +++ b/benchmark/Cargo.toml @@ -14,18 +14,19 @@ name = "text-generation-benchmark" path = "src/main.rs" [dependencies] -average = "0.13" -clap = { version = "4.1.4", features = ["derive", "env"] } -crossterm = "0.26" +average = "0.14" +clap = { version = "4.4.5", features = ["derive", "env"] } +crossterm = "0.27" float-ord = "0.3.2" -serde = {version = "1.0.142", features = ["derive"]} +serde = {version = "1.0.188", features = ["derive"]} serde_json = "1.0" -tabled = "0.12.0" +tabled = "0.14.0" text-generation-client = { path = "../router/client" } -thiserror = "1.0.38" -tokenizers = "0.13.3" -tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } -tui = {package = "ratatui", version = "0.20", default-features = false, features = ["crossterm"]} +thiserror = "1.0.48" +tokenizers = { version = "0.14.0", features = ["http"] } +tokio = { version = "1.32.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync", "macros"] } +tui = {package = "ratatui", version = "0.23", default-features = false, features = ["crossterm"]} tracing = "0.1.37" -tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } +tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] } +hf-hub = "0.3.1" diff --git a/benchmark/src/app.rs b/benchmark/src/app.rs index 6a9881fb..49654c1b 100644 --- a/benchmark/src/app.rs +++ b/benchmark/src/app.rs @@ -6,7 +6,7 @@ use tokio::sync::mpsc; use tui::backend::Backend; use tui::layout::{Alignment, Constraint, Direction, Layout}; use tui::style::{Color, Modifier, Style}; -use tui::text::{Span, Spans}; +use tui::text::{Line, Span}; use tui::widgets::{ Axis, BarChart, Block, Borders, Chart, Dataset, Gauge, GraphType, Paragraph, Tabs, }; @@ -244,7 +244,7 @@ impl App { .batch_size .iter() .map(|b| { - Spans::from(vec![Span::styled( + Line::from(vec![Span::styled( format!("Batch: {b}"), Style::default().fg(Color::White), )]) @@ -468,7 +468,7 @@ fn latency_paragraph<'a>(latency: &mut Vec, name: &'static str) -> Paragrap // Latency p50/p90/p99 texts let colors = vec![Color::LightGreen, Color::LightYellow, Color::LightRed]; for (i, (name, value)) in latency_percentiles.iter().enumerate() { - let span = Spans::from(vec![Span::styled( + let span = Line::from(vec![Span::styled( format!("{name}: {value:.2} ms"), Style::default().fg(colors[i]), )]); @@ -483,16 +483,16 @@ fn latency_paragraph<'a>(latency: &mut Vec, name: &'static str) -> Paragrap } /// Average/High/Low spans -fn statis_spans<'a>(data: &Vec, unit: &'static str) -> Vec> { +fn statis_spans<'a>(data: &Vec, unit: &'static str) -> Vec> { vec![ - Spans::from(vec![Span::styled( + Line::from(vec![Span::styled( format!( "Average: {:.2} {unit}", data.iter().sum::() / data.len() as f64 ), Style::default().fg(Color::LightBlue), )]), - Spans::from(vec![Span::styled( + Line::from(vec![Span::styled( format!( "Lowest: {:.2} {unit}", data.iter() @@ -501,7 +501,7 @@ fn statis_spans<'a>(data: &Vec, unit: &'static str) -> Vec> { ), Style::default().fg(Color::Reset), )]), - Spans::from(vec![Span::styled( + Line::from(vec![Span::styled( format!( "Highest: {:.2} {unit}", data.iter() diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs index 433c6f67..1875652c 100644 --- a/benchmark/src/lib.rs +++ b/benchmark/src/lib.rs @@ -33,7 +33,7 @@ pub async fn run( watermark: bool, do_sample: bool, client: ShardedClient, -) -> Result<(), crossterm::ErrorKind> { +) -> Result<(), std::io::Error> { let parameters = NextTokenChooserParameters { temperature: temperature.unwrap_or(1.0), top_k: top_k.unwrap_or(0), diff --git a/docs/openapi.json b/docs/openapi.json index 23c4f198..4a1ab6dd 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.0.3" + "version": "1.1.0" }, "paths": { "/": { diff --git a/docs/source/basic_tutorials/gated_model_access.md b/docs/source/basic_tutorials/gated_model_access.md index 08c76de2..827f6f4f 100644 --- a/docs/source/basic_tutorials/gated_model_access.md +++ b/docs/source/basic_tutorials/gated_model_access.md @@ -19,6 +19,6 @@ docker run --gpus all \ --shm-size 1g \ -e HUGGING_FACE_HUB_TOKEN=$token \ -p 8080:80 \ - -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.1 \ + -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0 \ --model-id $model ``` diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index b91e77cb..083dffa5 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -8,7 +8,7 @@ Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/ model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.3 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0 --model-id $model ``` diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index bb881d8e..aff6a377 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-integration-tests" -version = "1.0.3" +version = "1.1.0" description = "Text Generation Inference integration tests" authors = ["Nicolas Patry "] diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 3e7f86d4..77fb5ee0 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -7,17 +7,17 @@ authors.workspace = true homepage.workspace = true [dependencies] -clap = { version = "4.1.4", features = ["derive", "env"] } -ctrlc = { version = "3.2.5", features = ["termination"] } -nix = "0.26.2" -serde = { version = "1.0.152", features = ["derive"] } -serde_json = "1.0.93" +clap = { version = "4.4.5", features = ["derive", "env"] } +ctrlc = { version = "3.4.1", features = ["termination"] } +nix = "0.27.1" +serde = { version = "1.0.188", features = ["derive"] } +serde_json = "1.0.107" tracing = "0.1.37" -tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } +tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] } [dev-dependencies] float_eq = "1.0.1" -reqwest = { version = "0.11.14", features = ["blocking", "json"] } +reqwest = { version = "0.11.20", features = ["blocking", "json"] } [build-dependencies] -vergen = { version = "8.0.0", features = ["build", "cargo", "git", "gitcl", "rustc", "si"] } +vergen = { version = "8.2.5", features = ["build", "cargo", "git", "gitcl", "rustc", "si"] } diff --git a/router/Cargo.toml b/router/Cargo.toml index 10396826..87b5a8d3 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -15,36 +15,38 @@ name = "text-generation-router" path = "src/main.rs" [dependencies] -async-stream = "0.3.3" -axum = { version = "0.6.4", features = ["json"] } -axum-tracing-opentelemetry = "0.10.0" +async-stream = "0.3.5" +axum = { version = "0.6.20", features = ["json"] } +axum-tracing-opentelemetry = "0.14.1" text-generation-client = { path = "client" } -clap = { version = "4.1.4", features = ["derive", "env"] } -flume = "0.10.14" -futures = "0.3.26" -metrics = "0.21.0" +clap = { version = "4.4.5", features = ["derive", "env"] } +flume = "0.11.0" +futures = "0.3.28" +metrics = "0.21.1" metrics-exporter-prometheus = { version = "0.12.1", features = [] } nohash-hasher = "0.2.0" -opentelemetry = { version = "0.19.0", features = ["rt-tokio"] } -opentelemetry-otlp = "0.12.0" +opentelemetry = { version = "0.20.0", features = ["rt-tokio"] } +opentelemetry-otlp = "0.13.0" rand = "0.8.5" -reqwest = { version = "0.11.14", features = [] } -serde = "1.0.152" -serde_json = "1.0.93" -thiserror = "1.0.38" -tokenizers = "0.13.3" -tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } -tower-http = { version = "0.4.0", features = ["cors"] } +reqwest = { version = "0.11.20", features = [] } +serde = "1.0.188" +serde_json = "1.0.107" +thiserror = "1.0.48" +tokenizers = { version = "0.14.0", features = ["http"] } +tokio = { version = "1.32.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } +tower-http = { version = "0.4.4", features = ["cors"] } tracing = "0.1.37" -tracing-opentelemetry = "0.19.0" -tracing-subscriber = { version = "0.3.16", features = ["json", "env-filter"] } -utoipa = { version = "3.0.1", features = ["axum_extras"] } -utoipa-swagger-ui = { version = "3.0.2", features = ["axum"] } -ngrok = { version = "0.12.3", features = ["axum"], optional = true } +tracing-opentelemetry = "0.21.0" +tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] } +utoipa = { version = "3.5.0", features = ["axum_extras"] } +utoipa-swagger-ui = { version = "3.1.5", features = ["axum"] } +ngrok = { version = "0.13.1", features = ["axum"], optional = true } +hf-hub = "0.3.1" +init-tracing-opentelemetry = { version = "0.14.1", features = ["opentelemetry-otlp"] } [build-dependencies] -vergen = { version = "8.0.0", features = ["build", "git", "gitcl"] } +vergen = { version = "8.2.5", features = ["build", "git", "gitcl"] } [features] default = ["ngrok"] -ngrok = ["dep:ngrok"] \ No newline at end of file +ngrok = ["dep:ngrok"] diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index 43f444e6..d0131784 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -8,13 +8,13 @@ homepage.workspace = true [dependencies] futures = "^0.3" grpc-metadata = { path = "../grpc-metadata" } -prost = "^0.11" +prost = "^0.12" thiserror = "^1.0" -tokio = { version = "^1.25", features = ["sync"] } -tonic = "^0.9" +tokio = { version = "^1.32", features = ["sync"] } +tonic = "^0.10" tower = "^0.4" tracing = "^0.1" [build-dependencies] -tonic-build = "0.9.2" -prost-build = "0.11.6" +tonic-build = "0.10.1" +prost-build = "0.12.1" diff --git a/router/grpc-metadata/Cargo.toml b/router/grpc-metadata/Cargo.toml index 9e01f527..da163ec5 100644 --- a/router/grpc-metadata/Cargo.toml +++ b/router/grpc-metadata/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -opentelemetry = "^0.19" -tonic = "^0.9" +opentelemetry = "^0.20" +tonic = "^0.10" tracing = "^0.1" -tracing-opentelemetry = "^0.19" +tracing-opentelemetry = "^0.21" diff --git a/router/src/main.rs b/router/src/main.rs index 4903c066..f3028674 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -324,7 +324,7 @@ fn init_logging(otlp_endpoint: Option, json_output: bool) { if let Ok(tracer) = tracer { layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed()); - axum_tracing_opentelemetry::init_propagator().unwrap(); + init_tracing_opentelemetry::init_propagator().unwrap(); }; } diff --git a/router/src/server.rs b/router/src/server.rs index 91164098..fbc444fc 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -13,7 +13,7 @@ use axum::response::sse::{Event, KeepAlive, Sse}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{http, Json, Router}; -use axum_tracing_opentelemetry::opentelemetry_tracing_layer; +use axum_tracing_opentelemetry::middleware::OtelAxumLayer; use futures::stream::StreamExt; use futures::Stream; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; @@ -396,7 +396,7 @@ async fn generate_stream( // StreamResponse let stream_token = StreamResponse { token, - top_tokens: top_tokens, + top_tokens, generated_text: None, details: None, }; @@ -458,7 +458,7 @@ async fn generate_stream( let stream_token = StreamResponse { token, - top_tokens: top_tokens, + top_tokens, generated_text: Some(output_text), details }; @@ -695,7 +695,7 @@ pub async fn run( .layer(Extension(compat_return_full_text)) .layer(Extension(infer)) .layer(Extension(prom_handle.clone())) - .layer(opentelemetry_tracing_layer()) + .layer(OtelAxumLayer::default()) .layer(cors_layer); if ngrok { @@ -792,7 +792,7 @@ async fn shutdown_signal() { impl From for FinishReason { fn from(finish_reason: i32) -> Self { - let finish_reason = text_generation_client::FinishReason::from_i32(finish_reason).unwrap(); + let finish_reason = text_generation_client::FinishReason::try_from(finish_reason).unwrap(); match finish_reason { text_generation_client::FinishReason::Length => FinishReason::Length, text_generation_client::FinishReason::EosToken => FinishReason::EndOfSequenceToken, diff --git a/router/src/validation.rs b/router/src/validation.rs index 6c67f0ff..36cbfb9b 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -276,7 +276,7 @@ impl Validation { truncate: truncate.unwrap_or(self.max_input_length) as u32, parameters, stopping_parameters, - top_n_tokens: top_n_tokens, + top_n_tokens, }) } diff --git a/server/pyproject.toml b/server/pyproject.toml index f6eebf92..c06c298a 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.0.3" +version = "1.1.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] diff --git a/server/requirements.txt b/server/requirements.txt index 1b038cca..7c81c5f9 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -9,19 +9,19 @@ certifi==2023.7.22 ; python_version >= "3.9" and python_version < "3.13" charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "3.13" click==8.1.7 ; python_version >= "3.9" and python_version < "3.13" colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows") -datasets==2.14.4 ; python_version >= "3.9" and python_version < "3.13" +datasets==2.14.5 ; python_version >= "3.9" and python_version < "3.13" deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13" dill==0.3.7 ; python_version >= "3.9" and python_version < "3.13" einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13" -filelock==3.12.3 ; python_version >= "3.9" and python_version < "3.13" +filelock==3.12.4 ; python_version >= "3.9" and python_version < "3.13" frozenlist==1.4.0 ; python_version >= "3.9" and python_version < "3.13" fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "3.13" fsspec[http]==2023.6.0 ; python_version >= "3.9" and python_version < "3.13" googleapis-common-protos==1.60.0 ; python_version >= "3.9" and python_version < "3.13" grpc-interceptor==0.15.3 ; python_version >= "3.9" and python_version < "3.13" -grpcio-reflection==1.57.0 ; python_version >= "3.9" and python_version < "3.13" -grpcio-status==1.57.0 ; python_version >= "3.9" and python_version < "3.13" -grpcio==1.57.0 ; python_version >= "3.9" and python_version < "3.13" +grpcio-reflection==1.58.0 ; python_version >= "3.9" and python_version < "3.13" +grpcio-status==1.58.0 ; python_version >= "3.9" and python_version < "3.13" +grpcio==1.58.0 ; python_version >= "3.9" and python_version < "3.13" hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "3.13" huggingface-hub==0.16.4 ; python_version >= "3.9" and python_version < "3.13" idna==3.4 ; python_version >= "3.9" and python_version < "3.13" @@ -32,7 +32,7 @@ mpmath==1.3.0 ; python_version >= "3.9" and python_version < "3.13" multidict==6.0.4 ; python_version >= "3.9" and python_version < "3.13" multiprocess==0.70.15 ; python_version >= "3.9" and python_version < "3.13" networkx==3.1 ; python_version >= "3.9" and python_version < "3.13" -numpy==1.25.2 ; python_version >= "3.9" and python_version < "3.13" +numpy==1.26.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "3.13" @@ -43,32 +43,32 @@ opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13 opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13" packaging==23.1 ; python_version >= "3.9" and python_version < "3.13" -pandas==2.0.3 ; python_version >= "3.9" and python_version < "3.13" +pandas==2.1.1 ; python_version >= "3.9" and python_version < "3.13" peft==0.4.0 ; python_version >= "3.9" and python_version < "3.13" -pillow==10.0.0 ; python_version >= "3.9" and python_version < "3.13" -protobuf==4.24.2 ; python_version >= "3.9" and python_version < "3.13" +pillow==10.0.1 ; python_version >= "3.9" and python_version < "3.13" +protobuf==4.24.3 ; python_version >= "3.9" and python_version < "3.13" psutil==5.9.5 ; python_version >= "3.9" and python_version < "3.13" pyarrow==13.0.0 ; python_version >= "3.9" and python_version < "3.13" python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "3.13" -pytz==2023.3 ; python_version >= "3.9" and python_version < "3.13" +pytz==2023.3.post1 ; python_version >= "3.9" and python_version < "3.13" pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" regex==2023.8.8 ; python_version >= "3.9" and python_version < "3.13" requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" safetensors==0.3.3 ; python_version >= "3.9" and python_version < "3.13" scipy==1.11.2 ; python_version >= "3.9" and python_version < "3.13" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" -setuptools==68.1.2 ; python_version >= "3.9" and python_version < "3.13" +setuptools==68.2.2 ; python_version >= "3.9" and python_version < "3.13" six==1.16.0 ; python_version >= "3.9" and python_version < "3.13" sympy==1.12 ; python_version >= "3.9" and python_version < "3.13" texttable==1.6.7 ; python_version >= "3.9" and python_version < "3.13" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" torch==2.0.1 ; python_version >= "3.9" and python_version < "3.13" tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" -transformers==4.32.1 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.33.2 ; python_version >= "3.9" and python_version < "3.13" typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" -typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "3.13" +typing-extensions==4.8.0 ; python_version >= "3.9" and python_version < "3.13" tzdata==2023.3 ; python_version >= "3.9" and python_version < "3.13" -urllib3==2.0.4 ; python_version >= "3.9" and python_version < "3.13" +urllib3==2.0.5 ; python_version >= "3.9" and python_version < "3.13" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32" wrapt==1.15.0 ; python_version >= "3.9" and python_version < "3.13" xxhash==3.3.0 ; python_version >= "3.9" and python_version < "3.13" From 36c286885322c9a17ea0c674d79d22256010c928 Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Wed, 27 Sep 2023 11:06:07 +0200 Subject: [PATCH 431/793] Added note on weight-cache-override (#994) Added note on serving supported models from a different folder without re-downloading them. --------- Co-authored-by: Nicolas Patry --- docs/source/supported_models.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md index d997c8d8..c5768d9a 100644 --- a/docs/source/supported_models.md +++ b/docs/source/supported_models.md @@ -28,6 +28,12 @@ AutoModelForCausalLM.from_pretrained(, device_map="auto")` AutoModelForSeq2SeqLM.from_pretrained(, device_map="auto") ``` +If you wish to serve a supported model that already exists on a local folder, just point to the local folder. + +```bash +text-generation-launcher --model-id +`````` + ## Supported Hardware From 95a4bb696a2388f34d50c6873ff93911d6b235de Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 27 Sep 2023 11:42:57 +0200 Subject: [PATCH 432/793] Support eetq weight only quantization (#1068) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: zhaosida --- Cargo.lock | 8 ++-- launcher/src/main.rs | 32 +++++++++++-- server/.gitignore | 1 + server/Makefile | 1 + server/Makefile-eetq | 13 +++++ server/text_generation_server/cli.py | 1 + server/text_generation_server/utils/layers.py | 47 ++++++++++++++++++- 7 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 server/Makefile-eetq diff --git a/Cargo.lock b/Cargo.lock index 5fa01f0c..8fa7b726 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2896,18 +2896,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 09e32f89..b4fc86b7 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -21,11 +21,32 @@ mod env_runtime; #[derive(Clone, Copy, Debug, ValueEnum)] enum Quantization { + /// 4 bit quantization. Requires a specific GTPQ quantized model: + /// https://hf.co/models?search=awq. + /// Should replace GPTQ models whereever possible because of the better latency + Awq, + /// 8 bit quantization, doesn't require specific model. + /// Should be a drop-in replacement to bitsandbytes with much better performance. + /// Kernels are from https://github.com/NetEase-FuXi/EETQ.git + Eetq, + /// 4 bit quantization. Requires a specific GTPQ quantized model: https://hf.co/models?search=gptq. + /// text-generation-inference will use exllama (faster) kernels whereever possible, and use + /// triton kernel (wider support) when it's not. + /// AWQ has faster kernels. + Gptq, + /// Bitsandbytes 8bit. Can be applied on any model, will cut the memory requirement in half, + /// but it is known that the model will be much slower to run than the native f16. + #[deprecated( + since = "1.1.0", + note = "Use `eetq` instead, which provides better latencies overall and is drop-in in most cases" + )] Bitsandbytes, + /// Bitsandbytes 4bit. Can be applied on any model, will cut the memory requirement by 4x, + /// but it is known that the model will be much slower to run than the native f16. BitsandbytesNF4, + /// Bitsandbytes 4bit. nf4 should be preferred in most cases but maybe this one has better + /// perplexity performance for you model BitsandbytesFP4, - Gptq, - Awq, } impl std::fmt::Display for Quantization { @@ -47,6 +68,9 @@ impl std::fmt::Display for Quantization { Quantization::Awq => { write!(f, "awq") } + Quantization::Eetq => { + write!(f, "eetq") + } } } } @@ -127,9 +151,7 @@ struct Args { #[clap(long, env)] num_shard: Option, - /// Whether you want the model to be quantized. This will use `bitsandbytes` for - /// quantization on the fly, or `gptq`. 4bit quantization is available through - /// `bitsandbytes` by providing the `bitsandbytes-fp4` or `bitsandbytes-nf4` options. + /// Whether you want the model to be quantized. #[clap(long, env, value_enum)] quantize: Option, diff --git a/server/.gitignore b/server/.gitignore index 40c37f48..dcb8fe67 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -160,3 +160,4 @@ flash-attention/ flash-attention-v2/ vllm/ llm-awq/ +eetq/ diff --git a/server/Makefile b/server/Makefile index b21d79d4..52543e3d 100644 --- a/server/Makefile +++ b/server/Makefile @@ -2,6 +2,7 @@ include Makefile-flash-att include Makefile-flash-att-v2 include Makefile-vllm include Makefile-awq +include Makefile-eetq unit-tests: pytest -s -vv -m "not private" tests diff --git a/server/Makefile-eetq b/server/Makefile-eetq new file mode 100644 index 00000000..5e8e9830 --- /dev/null +++ b/server/Makefile-eetq @@ -0,0 +1,13 @@ +eetq_commit := 323827dd471458a84e9c840f614e4592b157a4b1 + +eetq: + # Clone eetq + pip install packaging + git clone https://github.com/NetEase-FuXi/EETQ.git eetq + +build-eetq: eetq + cd eetq && git fetch && git checkout $(eetq_commit) + cd eetq && python setup.py build + +install-eetq: build-eetq + cd eetq && python setup.py install diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 7464934f..cf9596c9 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -18,6 +18,7 @@ class Quantization(str, Enum): bitsandbytes_fp4 = "bitsandbytes-fp4" gptq = "gptq" awq = "awq" + eetq = "eetq" class Dtype(str, Enum): diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index fb27764c..14cb55cc 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -5,6 +5,8 @@ from torch import nn from torch.nn import functional as F from typing import List +from loguru import logger +from functools import lru_cache HAS_BITS_AND_BYTES = True try: @@ -42,6 +44,13 @@ from typing import Optional +HAS_EETQ = False +try: + from EETQ import quant_weights, w8_a16_gemm + HAS_EETQ = True +except ImportError: + pass + # Monkey patching @classmethod def load_layer_norm(cls, prefix, weights, eps): @@ -120,6 +129,30 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: return F.linear(input, self.weight, self.bias) +class EETQLinear(nn.Module): + def __init__( + self, + weight, + bias, + ) -> None: + super().__init__() + device = weight.device + weight = torch.t(weight).contiguous().cpu() + weight, scale = quant_weights(weight, torch.int8, False) + if bias: + bias = weights.get_tensor(f"{prefix}.bias") + else: + bias = None + self.weight = weight.cuda(device) + self.scale = scale.cuda(device) + self.bias = bias.cuda(device) if bias is not None else None + + def forward(self, input: torch.Tensor) -> torch.Tensor: + output = w8_a16_gemm(input, self.weight, self.scale) + output = output + self.bias if self.bias is not None else output + return output + + class Linear8bitLt(nn.Module): def __init__( self, @@ -211,10 +244,20 @@ def forward(self, x: torch.Tensor): return out +@lru_cache(1) +def warn_deprecate_bnb(): + logger.warning("Bitsandbytes 8bit is deprecated, using `eetq` is a drop-in replacement, and has much better performnce") + def get_linear(weight, bias, quantize): if quantize is None: linear = FastLinear(weight, bias) + elif quantize == "eetq": + if HAS_EETQ: + linear = EETQLinear(weight, bias) + else: + raise ImportError("Please install EETQ from https://github.com/NetEase-FuXi/EETQ") elif quantize == "bitsandbytes": + warn_deprecate_bnb() linear = Linear8bitLt( weight, bias, @@ -298,8 +341,8 @@ def load(config, prefix: str, weights): weight = weights.get_tensor(f"{prefix}.weight") should_gather = False - # GPTQ and AWQ don't quantize heads (nor embeddings) - if config.quantize in ["gptq", "awq"]: + # GPTQ,AWQ,EETQ don't quantize heads (nor embeddings) + if config.quantize in ["gptq", "awq", "eetq"]: quantize = None else: quantize = config.quantize From b32e9ce9d5e5b73082e31bf5d75c458dba29d30c Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 27 Sep 2023 12:13:45 +0200 Subject: [PATCH 433/793] Remove the stripping of the prefix space (and any other mangling that tokenizers might do). (#1065) Superseed #1024 # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: bangoz --- .../test_flash_llama/test_flash_llama.json | 27 +-- .../test_flash_llama_all_params.json | 13 +- .../test_flash_llama_load.json | 98 +++++----- .../test_idefics/test_idefics.json | 50 ++--- .../test_idefics/test_idefics_load.json | 176 +++++++++--------- .../models/causal_lm.py | 7 +- .../models/flash_causal_lm.py | 12 +- .../models/idefics_causal_lm.py | 12 +- server/text_generation_server/models/model.py | 5 +- .../models/seq2seq_lm.py | 7 +- 10 files changed, 208 insertions(+), 199 deletions(-) diff --git a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama.json b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama.json index 49bc996c..a7f7d2f0 100644 --- a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama.json +++ b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama.json @@ -16,7 +16,7 @@ }, { "id": 2009, - "logprob": -11.5546875, + "logprob": -11.546875, "text": "request" } ], @@ -24,65 +24,66 @@ "tokens": [ { "id": 363, - "logprob": -1.5380859, + "logprob": -1.5351562, "special": false, "text": " for" }, { "id": 847, - "logprob": -2.5917969, + "logprob": -2.5722656, "special": false, "text": " /" }, { "id": 2754, - "logprob": -2.2773438, + "logprob": -2.2714844, "special": false, "text": "api" }, { "id": 29914, - "logprob": -0.034362793, + "logprob": -0.03414917, "special": false, "text": "/" }, { "id": 29894, - "logprob": -0.96533203, + "logprob": -0.95996094, "special": false, "text": "v" }, { "id": 29896, - "logprob": -0.36669922, + "logprob": -0.3635254, "special": false, "text": "1" }, { "id": 29914, - "logprob": -0.013122559, + "logprob": -0.013031006, "special": false, "text": "/" }, { "id": 16418, - "logprob": -3.1503906, + "logprob": -3.1523438, "special": false, "text": "projects" }, { "id": 29914, - "logprob": -0.43652344, + "logprob": -0.43701172, "special": false, "text": "/" }, { "id": 29896, - "logprob": -1.9404297, + "logprob": -1.9394531, "special": false, "text": "1" } - ] + ], + "top_tokens": null }, - "generated_text": "for /api/v1/projects/1" + "generated_text": " for /api/v1/projects/1" } diff --git a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json index 5be2870d..9f145377 100644 --- a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json +++ b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_all_params.json @@ -16,7 +16,7 @@ }, { "id": 2009, - "logprob": -11.5546875, + "logprob": -11.546875, "text": "request" } ], @@ -24,19 +24,19 @@ "tokens": [ { "id": 5229, - "logprob": -2.5683594, + "logprob": -2.5839844, "special": false, "text": " failed" }, { "id": 29901, - "logprob": -0.45336914, + "logprob": -0.44970703, "special": false, "text": ":" }, { "id": 4829, - "logprob": -1.8408203, + "logprob": -1.8339844, "special": false, "text": " Error" }, @@ -52,7 +52,8 @@ "special": false, "text": " test" } - ] + ], + "top_tokens": null }, - "generated_text": "Test requestfailed: Error in test" + "generated_text": "Test request failed: Error in test" } diff --git a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json index 9bbb5322..3543dad2 100644 --- a/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json +++ b/integration-tests/models/__snapshots__/test_flash_llama/test_flash_llama_load.json @@ -17,7 +17,7 @@ }, { "id": 2009, - "logprob": -11.5546875, + "logprob": -11.546875, "text": "request" } ], @@ -25,25 +25,25 @@ "tokens": [ { "id": 363, - "logprob": -1.5380859, + "logprob": -1.5351562, "special": false, "text": " for" }, { "id": 847, - "logprob": -2.5859375, + "logprob": -2.5566406, "special": false, "text": " /" }, { "id": 2754, - "logprob": -2.2695312, + "logprob": -2.2519531, "special": false, "text": "api" }, { "id": 29914, - "logprob": -0.03439331, + "logprob": -0.03414917, "special": false, "text": "/" }, @@ -55,13 +55,13 @@ }, { "id": 29896, - "logprob": -0.36694336, + "logprob": -0.3647461, "special": false, "text": "1" }, { "id": 29914, - "logprob": -0.013114929, + "logprob": -0.012901306, "special": false, "text": "/" }, @@ -73,19 +73,20 @@ }, { "id": 29914, - "logprob": -0.43847656, + "logprob": -0.4362793, "special": false, "text": "/" }, { "id": 29896, - "logprob": -1.9433594, + "logprob": -1.9394531, "special": false, "text": "1" } - ] + ], + "top_tokens": null }, - "generated_text": "for /api/v1/projects/1" + "generated_text": " for /api/v1/projects/1" }, { "details": { @@ -105,7 +106,7 @@ }, { "id": 2009, - "logprob": -11.5546875, + "logprob": -11.546875, "text": "request" } ], @@ -113,43 +114,43 @@ "tokens": [ { "id": 363, - "logprob": -1.5322266, + "logprob": -1.5332031, "special": false, "text": " for" }, { "id": 847, - "logprob": -2.5585938, + "logprob": -2.5625, "special": false, "text": " /" }, { "id": 2754, - "logprob": -2.265625, + "logprob": -2.2617188, "special": false, "text": "api" }, { "id": 29914, - "logprob": -0.034088135, + "logprob": -0.033996582, "special": false, "text": "/" }, { "id": 29894, - "logprob": -0.96240234, + "logprob": -0.9609375, "special": false, "text": "v" }, { "id": 29896, - "logprob": -0.36816406, + "logprob": -0.36572266, "special": false, "text": "1" }, { "id": 29914, - "logprob": -0.013191223, + "logprob": -0.0129776, "special": false, "text": "/" }, @@ -161,19 +162,20 @@ }, { "id": 29914, - "logprob": -0.43774414, + "logprob": -0.4362793, "special": false, "text": "/" }, { "id": 29896, - "logprob": -1.9443359, + "logprob": -1.9394531, "special": false, "text": "1" } - ] + ], + "top_tokens": null }, - "generated_text": "for /api/v1/projects/1" + "generated_text": " for /api/v1/projects/1" }, { "details": { @@ -193,7 +195,7 @@ }, { "id": 2009, - "logprob": -11.5546875, + "logprob": -11.546875, "text": "request" } ], @@ -201,43 +203,43 @@ "tokens": [ { "id": 363, - "logprob": -1.5322266, + "logprob": -1.5332031, "special": false, "text": " for" }, { "id": 847, - "logprob": -2.5585938, + "logprob": -2.5625, "special": false, "text": " /" }, { "id": 2754, - "logprob": -2.265625, + "logprob": -2.2617188, "special": false, "text": "api" }, { "id": 29914, - "logprob": -0.034088135, + "logprob": -0.033996582, "special": false, "text": "/" }, { "id": 29894, - "logprob": -0.96240234, + "logprob": -0.9609375, "special": false, "text": "v" }, { "id": 29896, - "logprob": -0.36816406, + "logprob": -0.36572266, "special": false, "text": "1" }, { "id": 29914, - "logprob": -0.013191223, + "logprob": -0.0129776, "special": false, "text": "/" }, @@ -249,19 +251,20 @@ }, { "id": 29914, - "logprob": -0.43774414, + "logprob": -0.4362793, "special": false, "text": "/" }, { "id": 29896, - "logprob": -1.9443359, + "logprob": -1.9394531, "special": false, "text": "1" } - ] + ], + "top_tokens": null }, - "generated_text": "for /api/v1/projects/1" + "generated_text": " for /api/v1/projects/1" }, { "details": { @@ -281,7 +284,7 @@ }, { "id": 2009, - "logprob": -11.5546875, + "logprob": -11.546875, "text": "request" } ], @@ -289,43 +292,43 @@ "tokens": [ { "id": 363, - "logprob": -1.5322266, + "logprob": -1.5332031, "special": false, "text": " for" }, { "id": 847, - "logprob": -2.5585938, + "logprob": -2.5625, "special": false, "text": " /" }, { "id": 2754, - "logprob": -2.265625, + "logprob": -2.2617188, "special": false, "text": "api" }, { "id": 29914, - "logprob": -0.034088135, + "logprob": -0.033996582, "special": false, "text": "/" }, { "id": 29894, - "logprob": -0.96240234, + "logprob": -0.9609375, "special": false, "text": "v" }, { "id": 29896, - "logprob": -0.36816406, + "logprob": -0.36572266, "special": false, "text": "1" }, { "id": 29914, - "logprob": -0.013191223, + "logprob": -0.0129776, "special": false, "text": "/" }, @@ -337,18 +340,19 @@ }, { "id": 29914, - "logprob": -0.43774414, + "logprob": -0.4362793, "special": false, "text": "/" }, { "id": 29896, - "logprob": -1.9443359, + "logprob": -1.9394531, "special": false, "text": "1" } - ] + ], + "top_tokens": null }, - "generated_text": "for /api/v1/projects/1" + "generated_text": " for /api/v1/projects/1" } ] diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json index 0edd81b6..2c5d05f6 100644 --- a/integration-tests/models/__snapshots__/test_idefics/test_idefics.json +++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json @@ -11,22 +11,22 @@ }, { "id": 4911, - "logprob": -5.7773438, + "logprob": -5.7851562, "text": "User" }, { "id": 29901, - "logprob": -0.0069999695, + "logprob": -0.006996155, "text": ":" }, { "id": 32000, - "logprob": -0.8125, + "logprob": -0.81347656, "text": "" }, { "id": 32001, - "logprob": -6.651878e-05, + "logprob": -6.687641e-05, "text": "" }, { @@ -36,67 +36,67 @@ }, { "id": 1815, - "logprob": -4.2265625, + "logprob": -4.2148438, "text": "Can" }, { "id": 366, - "logprob": -0.013977051, + "logprob": -0.014137268, "text": "you" }, { "id": 2649, - "logprob": -4.4375, + "logprob": -4.4335938, "text": "tell" }, { "id": 592, - "logprob": -0.29077148, + "logprob": -0.2919922, "text": "me" }, { "id": 263, - "logprob": -4.2109375, + "logprob": -4.2070312, "text": "a" }, { "id": 1407, - "logprob": -9.4296875, + "logprob": -9.421875, "text": "very" }, { "id": 3273, - "logprob": -1.8671875, + "logprob": -1.8720703, "text": "short" }, { "id": 5828, - "logprob": -0.26586914, + "logprob": -0.26489258, "text": "story" }, { "id": 2729, - "logprob": -3.7460938, + "logprob": -3.7441406, "text": "based" }, { "id": 373, - "logprob": -0.0005350113, + "logprob": -0.0005393028, "text": "on" }, { "id": 278, - "logprob": -0.13867188, + "logprob": -0.140625, "text": "the" }, { "id": 1967, - "logprob": -0.06842041, + "logprob": -0.06756592, "text": "image" }, { "id": 29973, - "logprob": -0.15319824, + "logprob": -0.15454102, "text": "?" } ], @@ -104,7 +104,7 @@ "tokens": [ { "id": 32002, - "logprob": -0.0019445419, + "logprob": -0.0019140244, "special": true, "text": "" }, @@ -116,13 +116,13 @@ }, { "id": 13, - "logprob": -1.7881393e-05, + "logprob": -1.7642975e-05, "special": false, "text": "\n" }, { "id": 7900, - "logprob": -3.0994415e-06, + "logprob": -2.9802322e-06, "special": false, "text": "Ass" }, @@ -140,30 +140,30 @@ }, { "id": 319, - "logprob": -0.9057617, + "logprob": -0.91064453, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.2314453, + "logprob": -1.2412109, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.00024914742, + "logprob": -0.0002439022, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1621094, + "logprob": -1.1630859, "special": false, "text": " stands" } ], "top_tokens": null }, - "generated_text": "\nAssistant: A rooster stands" + "generated_text": " \nAssistant: A rooster stands" } diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json index 81cc1b19..f258e38d 100644 --- a/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json +++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json @@ -12,22 +12,22 @@ }, { "id": 4911, - "logprob": -5.7773438, + "logprob": -5.7851562, "text": "User" }, { "id": 29901, - "logprob": -0.0069999695, + "logprob": -0.006996155, "text": ":" }, { "id": 32000, - "logprob": -0.8125, + "logprob": -0.81347656, "text": "" }, { "id": 32001, - "logprob": -6.651878e-05, + "logprob": -6.687641e-05, "text": "" }, { @@ -37,67 +37,67 @@ }, { "id": 1815, - "logprob": -4.2265625, + "logprob": -4.2148438, "text": "Can" }, { "id": 366, - "logprob": -0.013977051, + "logprob": -0.014137268, "text": "you" }, { "id": 2649, - "logprob": -4.4375, + "logprob": -4.4335938, "text": "tell" }, { "id": 592, - "logprob": -0.29077148, + "logprob": -0.2919922, "text": "me" }, { "id": 263, - "logprob": -4.2109375, + "logprob": -4.2070312, "text": "a" }, { "id": 1407, - "logprob": -9.4296875, + "logprob": -9.421875, "text": "very" }, { "id": 3273, - "logprob": -1.8671875, + "logprob": -1.8720703, "text": "short" }, { "id": 5828, - "logprob": -0.26586914, + "logprob": -0.26489258, "text": "story" }, { "id": 2729, - "logprob": -3.7460938, + "logprob": -3.7441406, "text": "based" }, { "id": 373, - "logprob": -0.0005350113, + "logprob": -0.0005393028, "text": "on" }, { "id": 278, - "logprob": -0.13867188, + "logprob": -0.140625, "text": "the" }, { "id": 1967, - "logprob": -0.06842041, + "logprob": -0.06756592, "text": "image" }, { "id": 29973, - "logprob": -0.15319824, + "logprob": -0.15454102, "text": "?" } ], @@ -105,13 +105,13 @@ "tokens": [ { "id": 32002, - "logprob": -0.0019445419, + "logprob": -0.0019140244, "special": true, "text": "" }, { "id": 29871, - "logprob": -8.416176e-05, + "logprob": -8.392334e-05, "special": false, "text": " " }, @@ -123,7 +123,7 @@ }, { "id": 7900, - "logprob": -3.0994415e-06, + "logprob": -2.9802322e-06, "special": false, "text": "Ass" }, @@ -135,38 +135,38 @@ }, { "id": 29901, - "logprob": -3.2186508e-06, + "logprob": -3.0994415e-06, "special": false, "text": ":" }, { "id": 319, - "logprob": -0.89941406, + "logprob": -0.9057617, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.234375, + "logprob": -1.2294922, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.0002465248, + "logprob": -0.00024533272, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1660156, + "logprob": -1.1640625, "special": false, "text": " stands" } ], "top_tokens": null }, - "generated_text": "\nAssistant: A rooster stands" + "generated_text": " \nAssistant: A rooster stands" }, { "details": { @@ -181,22 +181,22 @@ }, { "id": 4911, - "logprob": -5.7890625, + "logprob": -5.7773438, "text": "User" }, { "id": 29901, - "logprob": -0.0070152283, + "logprob": -0.0070114136, "text": ":" }, { "id": 32000, - "logprob": -0.8125, + "logprob": -0.8208008, "text": "" }, { "id": 32001, - "logprob": -6.651878e-05, + "logprob": -6.699562e-05, "text": "" }, { @@ -211,17 +211,17 @@ }, { "id": 366, - "logprob": -0.014190674, + "logprob": -0.014175415, "text": "you" }, { "id": 2649, - "logprob": -4.4140625, + "logprob": -4.4296875, "text": "tell" }, { "id": 592, - "logprob": -0.2919922, + "logprob": -0.29516602, "text": "me" }, { @@ -231,7 +231,7 @@ }, { "id": 1407, - "logprob": -9.4375, + "logprob": -9.4296875, "text": "very" }, { @@ -241,7 +241,7 @@ }, { "id": 5828, - "logprob": -0.26904297, + "logprob": -0.26879883, "text": "story" }, { @@ -251,22 +251,22 @@ }, { "id": 373, - "logprob": -0.0005402565, + "logprob": -0.0005354881, "text": "on" }, { "id": 278, - "logprob": -0.13867188, + "logprob": -0.13671875, "text": "the" }, { "id": 1967, - "logprob": -0.068359375, + "logprob": -0.06719971, "text": "image" }, { "id": 29973, - "logprob": -0.15539551, + "logprob": -0.15551758, "text": "?" } ], @@ -274,7 +274,7 @@ "tokens": [ { "id": 32002, - "logprob": -0.0019168854, + "logprob": -0.0019130707, "special": true, "text": "" }, @@ -286,7 +286,7 @@ }, { "id": 13, - "logprob": -1.7642975e-05, + "logprob": -1.7881393e-05, "special": false, "text": "\n" }, @@ -310,32 +310,32 @@ }, { "id": 319, - "logprob": -0.90722656, + "logprob": -0.9013672, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.2373047, + "logprob": -1.2324219, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.00024938583, + "logprob": -0.0002477169, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1708984, + "logprob": -1.1660156, "special": false, "text": " stands" } ], "top_tokens": null }, - "generated_text": "\nAssistant: A rooster stands" + "generated_text": " \nAssistant: A rooster stands" }, { "details": { @@ -350,22 +350,22 @@ }, { "id": 4911, - "logprob": -5.7890625, + "logprob": -5.7773438, "text": "User" }, { "id": 29901, - "logprob": -0.0070152283, + "logprob": -0.0070114136, "text": ":" }, { "id": 32000, - "logprob": -0.8125, + "logprob": -0.8208008, "text": "" }, { "id": 32001, - "logprob": -6.663799e-05, + "logprob": -6.699562e-05, "text": "" }, { @@ -380,17 +380,17 @@ }, { "id": 366, - "logprob": -0.014190674, + "logprob": -0.014175415, "text": "you" }, { "id": 2649, - "logprob": -4.4140625, + "logprob": -4.4296875, "text": "tell" }, { "id": 592, - "logprob": -0.2919922, + "logprob": -0.29516602, "text": "me" }, { @@ -400,7 +400,7 @@ }, { "id": 1407, - "logprob": -9.4375, + "logprob": -9.4296875, "text": "very" }, { @@ -410,7 +410,7 @@ }, { "id": 5828, - "logprob": -0.26904297, + "logprob": -0.26879883, "text": "story" }, { @@ -420,22 +420,22 @@ }, { "id": 373, - "logprob": -0.0005402565, + "logprob": -0.0005354881, "text": "on" }, { "id": 278, - "logprob": -0.13867188, + "logprob": -0.13671875, "text": "the" }, { "id": 1967, - "logprob": -0.068359375, + "logprob": -0.06719971, "text": "image" }, { "id": 29973, - "logprob": -0.15539551, + "logprob": -0.15551758, "text": "?" } ], @@ -443,19 +443,19 @@ "tokens": [ { "id": 32002, - "logprob": -0.0019168854, + "logprob": -0.001912117, "special": true, "text": "" }, { "id": 29871, - "logprob": -8.404255e-05, + "logprob": -8.392334e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -1.7642975e-05, + "logprob": -1.7762184e-05, "special": false, "text": "\n" }, @@ -479,32 +479,32 @@ }, { "id": 319, - "logprob": -0.90722656, + "logprob": -0.9013672, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.2373047, + "logprob": -1.2324219, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.00024938583, + "logprob": -0.0002477169, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1708984, + "logprob": -1.1660156, "special": false, "text": " stands" } ], "top_tokens": null }, - "generated_text": "\nAssistant: A rooster stands" + "generated_text": " \nAssistant: A rooster stands" }, { "details": { @@ -519,22 +519,22 @@ }, { "id": 4911, - "logprob": -5.7890625, + "logprob": -5.7773438, "text": "User" }, { "id": 29901, - "logprob": -0.0070152283, + "logprob": -0.0070114136, "text": ":" }, { "id": 32000, - "logprob": -0.8125, + "logprob": -0.8208008, "text": "" }, { "id": 32001, - "logprob": -6.663799e-05, + "logprob": -6.699562e-05, "text": "" }, { @@ -549,17 +549,17 @@ }, { "id": 366, - "logprob": -0.014190674, + "logprob": -0.014175415, "text": "you" }, { "id": 2649, - "logprob": -4.4140625, + "logprob": -4.4296875, "text": "tell" }, { "id": 592, - "logprob": -0.2919922, + "logprob": -0.29516602, "text": "me" }, { @@ -569,7 +569,7 @@ }, { "id": 1407, - "logprob": -9.4375, + "logprob": -9.4296875, "text": "very" }, { @@ -579,7 +579,7 @@ }, { "id": 5828, - "logprob": -0.26904297, + "logprob": -0.26879883, "text": "story" }, { @@ -589,22 +589,22 @@ }, { "id": 373, - "logprob": -0.0005402565, + "logprob": -0.0005354881, "text": "on" }, { "id": 278, - "logprob": -0.13867188, + "logprob": -0.13671875, "text": "the" }, { "id": 1967, - "logprob": -0.068359375, + "logprob": -0.06719971, "text": "image" }, { "id": 29973, - "logprob": -0.15539551, + "logprob": -0.15551758, "text": "?" } ], @@ -612,19 +612,19 @@ "tokens": [ { "id": 32002, - "logprob": -0.0019159317, + "logprob": -0.001912117, "special": true, "text": "" }, { "id": 29871, - "logprob": -8.404255e-05, + "logprob": -8.392334e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -1.7642975e-05, + "logprob": -1.7762184e-05, "special": false, "text": "\n" }, @@ -648,31 +648,31 @@ }, { "id": 319, - "logprob": -0.90722656, + "logprob": -0.9013672, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.2373047, + "logprob": -1.2324219, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.00024938583, + "logprob": -0.0002477169, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1708984, + "logprob": -1.1660156, "special": false, "text": " stands" } ], "top_tokens": null }, - "generated_text": "\nAssistant: A rooster stands" + "generated_text": " \nAssistant: A rooster stands" } ] diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 696f0fb2..35d74b2e 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -641,8 +641,11 @@ def generate_token( if i % self.world_size == self.rank: if stop: # Decode generated tokens - output_text = self.decode( - all_input_ids[-stopping_criteria.current_tokens :, 0] + output_text, _, _ = self.decode_token( + all_input_ids[:, 0], + prefix_offset=len(all_input_ids) - stopping_criteria.current_tokens - 1, + read_offset=len(all_input_ids) - stopping_criteria.current_tokens, + skip_special_tokens=True ) # Get seed if isinstance(next_token_chooser.choice, Sampling): diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index d6af07f4..12d8efeb 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -793,11 +793,6 @@ def warmup(self, batch: FlashCausalLMBatch): return int(num_blocks * BLOCK_SIZE) - def decode(self, generated_ids: Union[torch.Tensor, List[int]]) -> str: - return self.tokenizer.decode( - generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False - ) - def forward( self, input_ids: torch.Tensor, @@ -1008,8 +1003,11 @@ def generate_token( if i % self.world_size == self.rank: if stop: # Decode generated tokens - output_text = self.decode( - all_input_ids[-stopping_criteria.current_tokens :] + output_text, _, _ = self.decode_token( + all_input_ids, + prefix_offset=len(all_input_ids) - stopping_criteria.current_tokens - 1, + read_offset=len(all_input_ids) - stopping_criteria.current_tokens, + skip_special_tokens=True ) generated_text = GeneratedText( output_text, diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py index f4177145..30cc2299 100644 --- a/server/text_generation_server/models/idefics_causal_lm.py +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -611,11 +611,6 @@ def __init__( def batch_type(self) -> Type[IdeficsCausalLMBatch]: return IdeficsCausalLMBatch - def decode(self, generated_ids: List[int]) -> str: - return self.tokenizer.decode( - generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False - ) - def forward( self, input_ids, @@ -728,8 +723,11 @@ def generate_token( if i % self.world_size == self.rank: if stop: # Decode generated tokens - output_text = self.decode( - all_input_ids[-stopping_criteria.current_tokens :, 0] + output_text, _, _ = self.decode_token( + all_input_ids[:, 0], + prefix_offset=len(all_input_ids) - stopping_criteria.current_tokens - 1, + read_offset=len(all_input_ids) - stopping_criteria.current_tokens, + skip_special_tokens=True ) # Get seed if isinstance(next_token_chooser.choice, Sampling): diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 806e9833..73329b24 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -64,16 +64,17 @@ def decode_token( all_input_ids: List[int], prefix_offset: int = 0, read_offset: int = 0, + skip_special_tokens: bool = False, ) -> Tuple[str, int, int]: """Hack to hopefully support generate_stream for the maximum number of tokenizers""" # The prefix text is necessary only to defeat cleanup algorithms in the decode # which decide to add a space or not depending on the surrounding ids. prefix_text = self.tokenizer.decode( - all_input_ids[prefix_offset:read_offset], skip_special_tokens=False + all_input_ids[prefix_offset:read_offset], skip_special_tokens=skip_special_tokens ) new_text = self.tokenizer.decode( - all_input_ids[prefix_offset:], skip_special_tokens=False + all_input_ids[prefix_offset:], skip_special_tokens=skip_special_tokens ) if len(new_text) > len(prefix_text) and not new_text.endswith("�"): diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 34932c0b..f67874be 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -710,8 +710,11 @@ def generate_token( if stop: # Slice with decoder_input_length to remove padding # Decode all tokens - output_text = self.decode( - all_decoder_input_ids[-decoder_input_length:] + output_text, _, _ = self.decode_token( + all_decoder_input_ids, + prefix_offset=len(all_decoder_input_ids) - decoder_input_length - 1, + read_offset=len(all_decoder_input_ids) - decoder_input_length, + skip_special_tokens=True ) # Get seed From 47954b81e9988ae069c8b790ce81f0cf2d372378 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 27 Sep 2023 12:22:09 +0200 Subject: [PATCH 434/793] feat: format code (#1070) --- clients/python/text_generation/client.py | 2 +- clients/python/text_generation/types.py | 4 +- integration-tests/models/test_flash_awq.py | 27 +- .../models/test_flash_awq_sharded.py | 23 +- integration-tests/models/test_idefics.py | 4 +- server/tests/utils/test_tokens.py | 7 +- server/text_generation_server/cli.py | 13 +- .../text_generation_server/models/__init__.py | 28 +- .../models/causal_lm.py | 9 +- .../models/custom_modeling/bloom_modeling.py | 5 +- .../custom_modeling/flash_llama_modeling.py | 6 +- .../idefics_image_processing.py | 30 +- .../custom_modeling/idefics_modeling.py | 360 +++++++++++++----- .../custom_modeling/idefics_perceiver.py | 89 +++-- .../custom_modeling/idefics_processing.py | 44 ++- .../models/custom_modeling/idefics_vision.py | 105 +++-- .../models/custom_modeling/neox_modeling.py | 5 +- .../models/flash_causal_lm.py | 9 +- .../models/idefics_causal_lm.py | 94 +++-- server/text_generation_server/models/model.py | 3 +- .../models/seq2seq_lm.py | 6 +- server/text_generation_server/server.py | 22 +- .../utils/awq/quantize/qmodule.py | 12 +- .../utils/gptq/quantize.py | 6 +- server/text_generation_server/utils/layers.py | 102 +++-- server/text_generation_server/utils/peft.py | 6 +- server/text_generation_server/utils/tokens.py | 2 +- .../text_generation_server/utils/weights.py | 37 +- 28 files changed, 767 insertions(+), 293 deletions(-) diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index 015613c2..ff7f66a3 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -137,7 +137,7 @@ def generate( typical_p=typical_p, watermark=watermark, decoder_input_details=decoder_input_details, - top_n_tokens=top_n_tokens + top_n_tokens=top_n_tokens, ) request = Request(inputs=prompt, stream=False, parameters=parameters) diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index 20083b19..6d6a0536 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -133,7 +133,9 @@ def valid_best_of_stream(cls, field_value, values): and parameters.best_of > 1 and field_value ): - raise ValidationError("`best_of` != 1 is not supported when `stream` == True") + raise ValidationError( + "`best_of` != 1 is not supported when `stream` == True" + ) return field_value diff --git a/integration-tests/models/test_flash_awq.py b/integration-tests/models/test_flash_awq.py index f0b99a3b..62a95f48 100644 --- a/integration-tests/models/test_flash_awq.py +++ b/integration-tests/models/test_flash_awq.py @@ -3,7 +3,11 @@ @pytest.fixture(scope="module") def flash_llama_awq_handle(launcher): - with launcher("abhinavkulkarni/codellama-CodeLlama-7b-Python-hf-w4-g128-awq", num_shard=1, quantize="awq") as handle: + with launcher( + "abhinavkulkarni/codellama-CodeLlama-7b-Python-hf-w4-g128-awq", + num_shard=1, + quantize="awq", + ) as handle: yield handle @@ -12,6 +16,7 @@ async def flash_llama_awq(flash_llama_awq_handle): await flash_llama_awq_handle.health(300) return flash_llama_awq_handle.client + @pytest.mark.asyncio @pytest.mark.private async def test_flash_llama_awq(flash_llama_awq, response_snapshot): @@ -20,11 +25,13 @@ async def test_flash_llama_awq(flash_llama_awq, response_snapshot): ) assert response.details.generated_tokens == 10 - assert response.generated_text == "\nWhat is the difference between Deep Learning and Machine" + assert ( + response.generated_text + == "\nWhat is the difference between Deep Learning and Machine" + ) assert response == response_snapshot - @pytest.mark.asyncio @pytest.mark.private async def test_flash_llama_awq_all_params(flash_llama_awq, response_snapshot): @@ -49,16 +56,18 @@ async def test_flash_llama_awq_all_params(flash_llama_awq, response_snapshot): @pytest.mark.asyncio @pytest.mark.private -async def test_flash_llama_awq_load( - flash_llama_awq, generate_load, response_snapshot -): +async def test_flash_llama_awq_load(flash_llama_awq, generate_load, response_snapshot): responses = await generate_load( flash_llama_awq, "What is Deep Learning?", max_new_tokens=10, n=4 ) assert len(responses) == 4 - assert all([r.generated_text == "\nWhat is the difference between Deep Learning and Machine" for r in responses]) + assert all( + [ + r.generated_text + == "\nWhat is the difference between Deep Learning and Machine" + for r in responses + ] + ) assert responses == response_snapshot - - diff --git a/integration-tests/models/test_flash_awq_sharded.py b/integration-tests/models/test_flash_awq_sharded.py index 39ea464a..1c687fc9 100644 --- a/integration-tests/models/test_flash_awq_sharded.py +++ b/integration-tests/models/test_flash_awq_sharded.py @@ -1,15 +1,22 @@ import pytest + @pytest.fixture(scope="module") def flash_llama_awq_handle_sharded(launcher): - with launcher("abhinavkulkarni/codellama-CodeLlama-7b-Python-hf-w4-g128-awq", num_shard=2, quantize="awq") as handle: + with launcher( + "abhinavkulkarni/codellama-CodeLlama-7b-Python-hf-w4-g128-awq", + num_shard=2, + quantize="awq", + ) as handle: yield handle + @pytest.fixture(scope="module") async def flash_llama_awq_sharded(flash_llama_awq_handle_sharded): await flash_llama_awq_handle_sharded.health(300) return flash_llama_awq_handle_sharded.client + @pytest.mark.asyncio @pytest.mark.private async def test_flash_llama_awq_sharded(flash_llama_awq_sharded, response_snapshot): @@ -18,9 +25,13 @@ async def test_flash_llama_awq_sharded(flash_llama_awq_sharded, response_snapsho ) assert response.details.generated_tokens == 10 - assert response.generated_text == "\nWhat is the difference between Deep Learning and Machine" + assert ( + response.generated_text + == "\nWhat is the difference between Deep Learning and Machine" + ) assert response == response_snapshot + @pytest.mark.asyncio @pytest.mark.private async def test_flash_llama_awq_load_sharded( @@ -31,6 +42,12 @@ async def test_flash_llama_awq_load_sharded( ) assert len(responses) == 4 - assert all([r.generated_text == "\nWhat is the difference between Deep Learning and Machine" for r in responses]) + assert all( + [ + r.generated_text + == "\nWhat is the difference between Deep Learning and Machine" + for r in responses + ] + ) assert responses == response_snapshot diff --git a/integration-tests/models/test_idefics.py b/integration-tests/models/test_idefics.py index 5659dd5c..5f4571b5 100644 --- a/integration-tests/models/test_idefics.py +++ b/integration-tests/models/test_idefics.py @@ -3,9 +3,7 @@ @pytest.fixture(scope="module") def idefics_handle(launcher): - with launcher( - "HuggingFaceM4/idefics-9b-instruct", num_shard=2 - ) as handle: + with launcher("HuggingFaceM4/idefics-9b-instruct", num_shard=2) as handle: yield handle diff --git a/server/tests/utils/test_tokens.py b/server/tests/utils/test_tokens.py index 4187ff25..0585f1fb 100644 --- a/server/tests/utils/test_tokens.py +++ b/server/tests/utils/test_tokens.py @@ -45,12 +45,15 @@ def test_stopping_criteria_max(): assert criteria(1, "") == (False, None) assert criteria(1, "") == (True, FinishReason.FINISH_REASON_LENGTH) + def test_batch_top_tokens(): top_n_tokens = [0, 2, 3, 4, 5] top_n_tokens_tensor = torch.tensor(top_n_tokens) - inp_logprobs = torch.tensor([[-1., -3., -4., -2., -3.]] * 5) + inp_logprobs = torch.tensor([[-1.0, -3.0, -4.0, -2.0, -3.0]] * 5) - topn_tok_ids, topn_tok_logprobs = batch_top_tokens(top_n_tokens, top_n_tokens_tensor, inp_logprobs) + topn_tok_ids, topn_tok_logprobs = batch_top_tokens( + top_n_tokens, top_n_tokens_tensor, inp_logprobs + ) assert topn_tok_ids[0] == [] assert topn_tok_ids[1] == [0, 3] diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index cf9596c9..301acb6b 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -125,8 +125,12 @@ def download_weights( if not is_local_model: try: - adapter_config_filename = hf_hub_download(model_id, revision=revision, filename="adapter_config.json") - utils.download_and_unload_peft(model_id, revision, trust_remote_code=trust_remote_code) + adapter_config_filename = hf_hub_download( + model_id, revision=revision, filename="adapter_config.json" + ) + utils.download_and_unload_peft( + model_id, revision, trust_remote_code=trust_remote_code + ) is_local_model = True utils.weight_files(model_id, revision, extension) return @@ -179,11 +183,12 @@ def download_weights( import transformers import json - if is_local_model: config_filename = os.path.join(model_id, "config.json") else: - config_filename = hf_hub_download(model_id, revision=revision, filename="config.json") + config_filename = hf_hub_download( + model_id, revision=revision, filename="config.json" + ) with open(config_filename, "r") as f: config = json.load(f) architecture = config["architectures"][0] diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 0d96d43b..96dd1ed1 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -153,7 +153,11 @@ def get_model( ) elif model_type == "mpt": return MPTSharded( - model_id, revision, quantize=quantize, dtype=dtype, trust_remote_code=trust_remote_code + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, ) elif model_type == "gpt_neox": @@ -252,13 +256,13 @@ def get_model( ) elif model_type == "idefics": if FLASH_ATTENTION: - return IDEFICSSharded( - model_id, - revision, - quantize=quantize, - dtype=dtype, - trust_remote_code=trust_remote_code, - ) + return IDEFICSSharded( + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, + ) else: raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format("Idefics")) @@ -269,13 +273,9 @@ def get_model( "gptq quantization is not supported for AutoModel, you can try to quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" ) if quantize == "awq": - raise ValueError( - "awq quantization is not supported for AutoModel" - ) + raise ValueError("awq quantization is not supported for AutoModel") elif (quantize == "bitsandbytes-fp4") or (quantize == "bitsandbytes-nf4"): - raise ValueError( - "4bit quantization is not supported for AutoModel" - ) + raise ValueError("4bit quantization is not supported for AutoModel") if model_type in modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: return CausalLM( model_id, diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 35d74b2e..fccfb0f8 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -643,9 +643,12 @@ def generate_token( # Decode generated tokens output_text, _, _ = self.decode_token( all_input_ids[:, 0], - prefix_offset=len(all_input_ids) - stopping_criteria.current_tokens - 1, - read_offset=len(all_input_ids) - stopping_criteria.current_tokens, - skip_special_tokens=True + prefix_offset=len(all_input_ids) + - stopping_criteria.current_tokens + - 1, + read_offset=len(all_input_ids) + - stopping_criteria.current_tokens, + skip_special_tokens=True, ) # Get seed if isinstance(next_token_chooser.choice, Sampling): diff --git a/server/text_generation_server/models/custom_modeling/bloom_modeling.py b/server/text_generation_server/models/custom_modeling/bloom_modeling.py index 297d5c68..5423d75a 100644 --- a/server/text_generation_server/models/custom_modeling/bloom_modeling.py +++ b/server/text_generation_server/models/custom_modeling/bloom_modeling.py @@ -40,7 +40,10 @@ ) CUSTOM_KERNELS_ENABLED = False -if torch.cuda.is_available() and not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True": +if ( + torch.cuda.is_available() + and not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True" +): try: from custom_kernels import fused_bloom_attention_cuda diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index cb0c1e85..7c743a88 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -169,6 +169,7 @@ def load_attention(config, prefix, weights): bias=False, ) + def _load_gqa(config, prefix: str, weights): assert config.hidden_size % config.num_attention_heads == 0 assert config.num_attention_heads % weights.process_group.size() == 0 @@ -211,7 +212,10 @@ def __init__( # config=config, prefix=f"{prefix}.rotary_emb", weights=weights # ) self.rotary_emb = PositionRotaryEmbedding.static( - config=config, dim=self.head_size, base=config.rope_theta, device=weights.device + config=config, + dim=self.head_size, + base=config.rope_theta, + device=weights.device, ) self.softmax_scale = self.head_size**-0.5 diff --git a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py index aec9a3dc..6fb00999 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py +++ b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py @@ -20,7 +20,12 @@ from PIL import Image from transformers.image_processing_utils import BaseImageProcessor, BatchFeature -from transformers.image_transforms import resize, to_channel_dimension_format, rescale, normalize +from transformers.image_transforms import ( + resize, + to_channel_dimension_format, + rescale, + normalize, +) from transformers.image_utils import ( ChannelDimension, ImageInput, @@ -121,7 +126,11 @@ def preprocess( a PyTorch tensor of the processed images """ image_size = image_size if image_size is not None else self.image_size - image_num_channels = image_num_channels if image_num_channels is not None else self.image_num_channels + image_num_channels = ( + image_num_channels + if image_num_channels is not None + else self.image_num_channels + ) image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std size = (image_size, image_size) @@ -160,9 +169,13 @@ def preprocess( images = [resize(x, size, resample=PILImageResampling.BICUBIC) for x in images] images = [self.rescale(image=image, scale=1 / 255) for image in images] images = [self.normalize(x, mean=image_mean, std=image_std) for x in images] - images = [to_channel_dimension_format(x, ChannelDimension.FIRST) for x in images] + images = [ + to_channel_dimension_format(x, ChannelDimension.FIRST) for x in images + ] # TODO: this converts to torch tensors - switch to convert_to_tensors once it becomes available - images = BatchFeature(data={"pixel_values": images}, tensor_type=TensorType.PYTORCH)["pixel_values"] + images = BatchFeature( + data={"pixel_values": images}, tensor_type=TensorType.PYTORCH + )["pixel_values"] return images @@ -185,7 +198,9 @@ def fetch_images(self, image_url_or_urls: Union[str, List[str]]): response.raise_for_status() return Image.open(BytesIO(response.content)) else: - raise ValueError(f"only a single or a list of entries is supported but got type={type(image_url_or_urls)}") + raise ValueError( + f"only a single or a list of entries is supported but got type={type(image_url_or_urls)}" + ) def rescale( self, @@ -255,10 +270,9 @@ def normalize( `np.ndarray`: The normalized image. """ # TODO 4.32 - return normalize( - image, mean=mean, std=std, data_format=data_format, **kwargs - ) + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) import transformers + transformers.IdeficsImageProcessor = IdeficsImageProcessor diff --git a/server/text_generation_server/models/custom_modeling/idefics_modeling.py b/server/text_generation_server/models/custom_modeling/idefics_modeling.py index 8b43ae4d..1ffe6276 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_modeling.py +++ b/server/text_generation_server/models/custom_modeling/idefics_modeling.py @@ -28,7 +28,11 @@ from transformers import PreTrainedModel from transformers.activations import ACT2FN -from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, dataclass +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, + dataclass, +) from transformers.modeling_utils import PretrainedConfig from transformers.utils import ( add_start_docstrings, @@ -37,8 +41,12 @@ replace_return_docstrings, ) from text_generation_server.models.custom_modeling.idefics_config import IdeficsConfig -from text_generation_server.models.custom_modeling.idefics_vision import IdeficsVisionTransformer -from text_generation_server.models.custom_modeling.idefics_perceiver import IdeficsPerceiverResampler +from text_generation_server.models.custom_modeling.idefics_vision import ( + IdeficsVisionTransformer, +) +from text_generation_server.models.custom_modeling.idefics_perceiver import ( + IdeficsPerceiverResampler, +) from text_generation_server.utils.layers import ( TensorParallelColumnLinear, TensorParallelEmbedding, @@ -49,10 +57,12 @@ ) import dropout_layer_norm + @dataclass class BaseModelOutputWithPastImage(BaseModelOutputWithPast): image_hidden_states: Optional[torch.FloatTensor] = None + @dataclass class CausalLMOutputWithPastImage(CausalLMOutputWithPast): image_hidden_states: Optional[torch.FloatTensor] = None @@ -78,25 +88,39 @@ def expand_inputs_for_generation( **model_kwargs, ): expanded_return_idx = ( - torch.arange(input_ids.shape[0]).view(-1, 1).repeat(1, expand_size).view(-1).to(input_ids.device) + torch.arange(input_ids.shape[0]) + .view(-1, 1) + .repeat(1, expand_size) + .view(-1) + .to(input_ids.device) ) input_ids = input_ids.index_select(0, expanded_return_idx) if "token_type_ids" in model_kwargs: token_type_ids = model_kwargs["token_type_ids"] - model_kwargs["token_type_ids"] = token_type_ids.index_select(0, expanded_return_idx) + model_kwargs["token_type_ids"] = token_type_ids.index_select( + 0, expanded_return_idx + ) if attention_mask is not None: - model_kwargs["attention_mask"] = attention_mask.index_select(0, expanded_return_idx) - model_kwargs["image_attention_mask"] = model_kwargs["image_attention_mask"].index_select( + model_kwargs["attention_mask"] = attention_mask.index_select( + 0, expanded_return_idx + ) + model_kwargs["image_attention_mask"] = model_kwargs[ + "image_attention_mask" + ].index_select(0, expanded_return_idx) + model_kwargs["pixel_values"] = model_kwargs["pixel_values"].index_select( 0, expanded_return_idx ) - model_kwargs["pixel_values"] = model_kwargs["pixel_values"].index_select(0, expanded_return_idx) if is_encoder_decoder: if encoder_outputs is None: - raise ValueError("If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined.") - encoder_outputs["last_hidden_state"] = encoder_outputs.last_hidden_state.index_select( + raise ValueError( + "If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined." + ) + encoder_outputs[ + "last_hidden_state" + ] = encoder_outputs.last_hidden_state.index_select( 0, expanded_return_idx.to(encoder_outputs.last_hidden_state.device) ) model_kwargs["encoder_outputs"] = encoder_outputs @@ -120,14 +144,17 @@ def update_model_kwargs_for_generation(outputs, model_kwargs, is_encoder_decoder # update token_type_ids with last value if "token_type_ids" in model_kwargs: token_type_ids = model_kwargs["token_type_ids"] - model_kwargs["token_type_ids"] = torch.cat([token_type_ids, token_type_ids[:, -1].unsqueeze(-1)], dim=-1) + model_kwargs["token_type_ids"] = torch.cat( + [token_type_ids, token_type_ids[:, -1].unsqueeze(-1)], dim=-1 + ) # update attention masks if not is_encoder_decoder: if "attention_mask" in model_kwargs: attention_mask = model_kwargs["attention_mask"] model_kwargs["attention_mask"] = torch.cat( - [attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))], dim=-1 + [attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))], + dim=-1, ) if "image_attention_mask" in model_kwargs: image_attention_mask = model_kwargs["image_attention_mask"] @@ -180,8 +207,12 @@ def freeze_model(model, module_exceptions=[]): } module_exceptions_mapped = [mapping[m] for m in module_exceptions] for module in model.modules(): - if module_exceptions and any([isinstance(module, t) for t in module_exceptions_mapped]): - module.requires_grad_(True) # Explicitely setting it to true to avoid any mistakes + if module_exceptions and any( + [isinstance(module, t) for t in module_exceptions_mapped] + ): + module.requires_grad_( + True + ) # Explicitely setting it to true to avoid any mistakes else: module.requires_grad_(False) return model @@ -195,15 +226,21 @@ def __init__( ): super().__init__() self.num_embeddings = config.vocab_size - self.weight = TensorParallelEmbedding(prefix="model.embed_tokens", weights=weights) - self.additional_weight = nn.Parameter(weights.get_tensor(f"model.embed_tokens.additional_embedding.weight")) + self.weight = TensorParallelEmbedding( + prefix="model.embed_tokens", weights=weights + ) + self.additional_weight = nn.Parameter( + weights.get_tensor(f"model.embed_tokens.additional_embedding.weight") + ) def forward(self, input_ids): # Clone so that we don't modify the original input_ids later on input_ids = input_ids.clone() additional_vocab_indices = torch.where(input_ids >= self.num_embeddings) input_ids_additional_vocab = input_ids[additional_vocab_indices] - additional_embeddings = torch.nn.functional.embedding(input_ids_additional_vocab - self.num_embeddings, self.additional_weight) + additional_embeddings = torch.nn.functional.embedding( + input_ids_additional_vocab - self.num_embeddings, self.additional_weight + ) # for successful lookup replace input_ids with 0, the results of these will be discarded anyway input_ids[additional_vocab_indices] = 0 @@ -234,7 +271,10 @@ def __init__( config=config, prefix="lm_head", weights=weights ) self.additional_fc = FastLinear.load( - config=config, prefix="lm_head.additional_fc", weights=weights, bias=False, + config=config, + prefix="lm_head.additional_fc", + weights=weights, + bias=False, ) def forward(self, input: torch.Tensor) -> torch.Tensor: @@ -257,7 +297,10 @@ def extra_repr(self) -> str: # Copied from transformers.models.bart.modeling_bart._make_causal_mask def _make_causal_mask( - input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0 + input_ids_shape: torch.Size, + dtype: torch.dtype, + device: torch.device, + past_key_values_length: int = 0, ): """ Make causal mask used for bi-directional self-attention. @@ -269,8 +312,18 @@ def _make_causal_mask( mask = mask.to(dtype) if past_key_values_length > 0: - mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1) - return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + mask = torch.cat( + [ + torch.zeros( + tgt_len, past_key_values_length, dtype=dtype, device=device + ), + mask, + ], + dim=-1, + ) + return mask[None, None, :, :].expand( + bsz, 1, tgt_len, tgt_len + past_key_values_length + ) def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): @@ -284,7 +337,9 @@ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] inverted_mask = 1.0 - expanded_mask - return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + return inverted_mask.masked_fill( + inverted_mask.to(torch.bool), torch.finfo(dtype).min + ) class IdeficsRMSNorm(nn.Module): @@ -346,7 +401,6 @@ def forward(self, hidden_states, residual=None): if unwrap: normed_hidden_states = normed_hidden_states.view(*shape) - return normed_hidden_states @@ -367,7 +421,10 @@ def __init__( bias=False, ) self.down_proj = TensorParallelRowLinear.load( - config, prefix=f"{prefix}.down_proj", weights=weights, bias=False, + config, + prefix=f"{prefix}.down_proj", + weights=weights, + bias=False, ) self.act_fn = ACT2FN[config.hidden_act] @@ -375,7 +432,9 @@ def forward(self, hidden_states): gate_up_states = self.gate_up_proj(hidden_states) shape = gate_up_states.shape gate_up_states = gate_up_states.view(*shape[:-1], 2, shape[-1] // 2) - return self.down_proj(self.act_fn(gate_up_states[:, :, 0]) * gate_up_states[:, :, 1]) + return self.down_proj( + self.act_fn(gate_up_states[:, :, 0]) * gate_up_states[:, :, 1] + ) # this was adapted from LlamaAttention @@ -445,14 +504,22 @@ def __init__( self.qk_layer_norms = qk_layer_norms if self.qk_layer_norms: self.q_layer_norm = IdeficsRMSNorm( - prefix=f"{prefix}.q_layer_norm", weights=weights, eps=config.rms_norm_eps - ) + prefix=f"{prefix}.q_layer_norm", + weights=weights, + eps=config.rms_norm_eps, + ) self.k_layer_norm = IdeficsRMSNorm( - prefix=f"{prefix}.q_layer_norm", weights=weights, eps=config.rms_norm_eps - ) + prefix=f"{prefix}.q_layer_norm", + weights=weights, + eps=config.rms_norm_eps, + ) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + return ( + tensor.view(bsz, seq_len, self.num_heads, self.head_dim) + .transpose(1, 2) + .contiguous() + ) def forward( self, @@ -470,20 +537,42 @@ def forward( bsz, q_len, _ = hidden_states.size() if is_cross_attention: - query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim)# .transpose(1, 2) + query_states = self.q_proj(hidden_states).view( + bsz, q_len, self.num_heads, self.head_dim + ) # .transpose(1, 2) query_states = query_states.transpose(1, 2) - _, kv_len, _ = key_value_states.size() # Note that, in this case, `kv_len` == `kv_seq_len` - key_states = self.k_proj(key_value_states).view(bsz, kv_len, self.num_heads, self.head_dim).transpose(1, 2) + ( + _, + kv_len, + _, + ) = ( + key_value_states.size() + ) # Note that, in this case, `kv_len` == `kv_seq_len` + key_states = ( + self.k_proj(key_value_states) + .view(bsz, kv_len, self.num_heads, self.head_dim) + .transpose(1, 2) + ) value_states = ( - self.v_proj(key_value_states).view(bsz, kv_len, self.num_heads, self.head_dim).transpose(1, 2) + self.v_proj(key_value_states) + .view(bsz, kv_len, self.num_heads, self.head_dim) + .transpose(1, 2) ) else: qkv = self.qkv(hidden_states) - query_states, key_states, value_states = qkv.split(self.num_heads * self.head_dim, dim=2) + query_states, key_states, value_states = qkv.split( + self.num_heads * self.head_dim, dim=2 + ) - query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim)# .transpose(1, 2) - key_states = key_states.view(bsz, q_len, self.num_heads, self.head_dim)# . transpose(1, 2) - value_states = value_states.view(bsz, q_len, self.num_heads, self.head_dim)# .transpose(1, 2) + query_states = query_states.view( + bsz, q_len, self.num_heads, self.head_dim + ) # .transpose(1, 2) + key_states = key_states.view( + bsz, q_len, self.num_heads, self.head_dim + ) # . transpose(1, 2) + value_states = value_states.view( + bsz, q_len, self.num_heads, self.head_dim + ) # .transpose(1, 2) kv_seq_len = q_len if past_key_value is not None: kv_seq_len += past_key_value[0].shape[-2] @@ -493,10 +582,14 @@ def forward( ) shape = query_states.shape - query_states = self.rotary_emb(query_states.view(-1, *shape[2:]), cos, sin).view(shape) + query_states = self.rotary_emb( + query_states.view(-1, *shape[2:]), cos, sin + ).view(shape) shape = key_states.shape - key_states = self.rotary_emb(key_states.reshape(-1, *shape[2:]), cos, sin).view(shape) + key_states = self.rotary_emb( + key_states.reshape(-1, *shape[2:]), cos, sin + ).view(shape) query_states = query_states.transpose(1, 2) key_states = key_states.transpose(1, 2) @@ -571,8 +664,14 @@ def __init__(self, layer_id: int, config: IdeficsConfig, weights): prefix=f"{prefix}.mlp", weights=weights, ) - self.input_layernorm = IdeficsRMSNorm(prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps) - self.post_attention_layernorm = IdeficsRMSNorm(prefix=f"{prefix}.post_attention_layernorm", weights=weights, eps=config.rms_norm_eps) + self.input_layernorm = IdeficsRMSNorm( + prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps + ) + self.post_attention_layernorm = IdeficsRMSNorm( + prefix=f"{prefix}.post_attention_layernorm", + weights=weights, + eps=config.rms_norm_eps, + ) self.dropout = config.dropout def forward( @@ -583,7 +682,9 @@ def forward( past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = False, - ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + ) -> Tuple[ + torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]] + ]: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` @@ -650,14 +751,22 @@ def __init__(self, layer_id, config: IdeficsConfig, weights): prefix=f"{prefix}.mlp", weights=weights, ) - self.input_layernorm = IdeficsRMSNorm(prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps) - self.post_attention_layernorm = IdeficsRMSNorm(prefix=f"{prefix}.post_attention_layernorm", weights=weights, eps=config.rms_norm_eps) + self.input_layernorm = IdeficsRMSNorm( + prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps + ) + self.post_attention_layernorm = IdeficsRMSNorm( + prefix=f"{prefix}.post_attention_layernorm", + weights=weights, + eps=config.rms_norm_eps, + ) self.config = config.dropout self.act_cross_attn = nn.Tanh() self.act_dense = nn.Tanh() - self.alpha_cross_attn = nn.Parameter(weights.get_tensor(f"{prefix}.alpha_cross_attn")) + self.alpha_cross_attn = nn.Parameter( + weights.get_tensor(f"{prefix}.alpha_cross_attn") + ) self.alpha_dense = nn.Parameter(weights.get_tensor(f"{prefix}.alpha_dense")) if not (hasattr(self, "alpha_cross_attn") and hasattr(self, "alpha_dense")): @@ -673,7 +782,9 @@ def forward( use_cache: Optional[bool] = False, past_key_value: Optional[Tuple[torch.Tensor]] = None, no_images: Optional[bool] = False, - ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + ) -> Tuple[ + torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]] + ]: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` @@ -695,7 +806,9 @@ def forward( ) if past_key_value is not None: - raise NotImplementedError("Past key value states are not implemented for Idefics cross attention module.") + raise NotImplementedError( + "Past key value states are not implemented for Idefics cross attention module." + ) residual = hidden_states @@ -711,7 +824,9 @@ def forward( # hidden_states = nn.functional.dropout(hidden_states, p=self.config, training=self.training) # when there are no images the model is used in pure language mode gate = 0 if no_images else 1 - hidden_states = residual + gate * self.act_cross_attn(self.alpha_cross_attn) * hidden_states + hidden_states = ( + residual + gate * self.act_cross_attn(self.alpha_cross_attn) * hidden_states + ) # Fully Connected residual = hidden_states @@ -896,11 +1011,14 @@ def __init__(self, config: IdeficsConfig, weights): self.gated_cross_attn_layers = nn.ModuleList( [ IdeficsGatedCrossAttentionLayer(layer_id, config, weights) - for layer_id in range(num_cross_layers)] + for layer_id in range(num_cross_layers) + ] ) # self.gradient_checkpointing = False - self.norm = IdeficsRMSNorm(prefix=f"model.norm", weights=weights, eps=config.rms_norm_eps) + self.norm = IdeficsRMSNorm( + prefix=f"model.norm", weights=weights, eps=config.rms_norm_eps + ) # self.gradient_checkpointing = False # Initialize weights and apply final processing @@ -932,7 +1050,9 @@ def __init__(self, config: IdeficsConfig, weights): # self.embed_tokens = value # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask - def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length): + def _prepare_decoder_attention_mask( + self, attention_mask, input_shape, inputs_embeds, past_key_values_length + ): # create causal mask # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] combined_attention_mask = None @@ -946,11 +1066,13 @@ def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_em if attention_mask is not None: # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to( - inputs_embeds.device - ) + expanded_attn_mask = _expand_mask( + attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1] + ).to(inputs_embeds.device) combined_attention_mask = ( - expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + expanded_attn_mask + if combined_attention_mask is None + else expanded_attn_mask + combined_attention_mask ) return combined_attention_mask @@ -974,23 +1096,35 @@ def forward( ) -> Union[Tuple, BaseModelOutputWithPastImage]: device = input_ids.device if input_ids is not None else inputs_embeds.device - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states ) use_cache = use_cache if use_cache is not None else self.config.use_cache - return_dict = return_dict if return_dict is not None else self.config.use_return_dict + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) # retrieve input_ids and inputs_embeds if input_ids is not None and inputs_embeds is not None: - raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + raise ValueError( + "You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time" + ) elif input_ids is not None: batch_size, seq_length = input_ids.shape elif inputs_embeds is not None: batch_size, seq_length, _ = inputs_embeds.shape else: - raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + raise ValueError( + "You have to specify either decoder_input_ids or decoder_inputs_embeds" + ) seq_length_with_past = seq_length past_key_values_length = 0 @@ -1006,7 +1140,10 @@ def forward( elif position_ids is None: device = input_ids.device if input_ids is not None else inputs_embeds.device position_ids = torch.arange( - past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device + past_key_values_length, + seq_length + past_key_values_length, + dtype=torch.long, + device=device, ) position_ids = position_ids.unsqueeze(0).view(-1, seq_length) else: @@ -1016,29 +1153,52 @@ def forward( if image_hidden_states is None: if pixel_values is None and image_embeddings is None: - raise ValueError("Either pixel_values and image_embeddings have to be not-None.") + raise ValueError( + "Either pixel_values and image_embeddings have to be not-None." + ) elif pixel_values is not None and image_embeddings is not None: - raise ValueError("You cannot specify both pixel_values and image_embeddings at the same time") + raise ValueError( + "You cannot specify both pixel_values and image_embeddings at the same time" + ) elif pixel_values is not None: no_images = len(torch.nonzero(pixel_values)) == 0 - pixel_values = pixel_values.to(dtype=self.dtype, device=device) # fp16 compatibility + pixel_values = pixel_values.to( + dtype=self.dtype, device=device + ) # fp16 compatibility batch_size, num_images = pixel_values.shape[:2] - pixel_values = pixel_values.contiguous().view(batch_size * num_images, *pixel_values.shape[2:]) + pixel_values = pixel_values.contiguous().view( + batch_size * num_images, *pixel_values.shape[2:] + ) # Get sequence from the vision encoder - image_hidden_states = self.vision_model(pixel_values=pixel_values).last_hidden_state + image_hidden_states = self.vision_model( + pixel_values=pixel_values + ).last_hidden_state elif image_embeddings is not None: - batch_size, num_images, image_seq_len, image_hidden_size = image_embeddings.size() - image_hidden_states = image_embeddings.to(dtype=self.dtype, device=input_ids.device) - image_hidden_states = image_hidden_states.view(batch_size * num_images, image_seq_len, image_hidden_size) + ( + batch_size, + num_images, + image_seq_len, + image_hidden_size, + ) = image_embeddings.size() + image_hidden_states = image_embeddings.to( + dtype=self.dtype, device=input_ids.device + ) + image_hidden_states = image_hidden_states.view( + batch_size * num_images, image_seq_len, image_hidden_size + ) if self.config.use_resampler: image_hidden_states = self.perceiver_resampler(image_hidden_states) - image_seq_len, image_hidden_size = image_hidden_states.size(1), image_hidden_states.size(2) - image_hidden_states = image_hidden_states.view(batch_size, num_images * image_seq_len, image_hidden_size) + image_seq_len, image_hidden_size = image_hidden_states.size( + 1 + ), image_hidden_states.size(2) + image_hidden_states = image_hidden_states.view( + batch_size, num_images * image_seq_len, image_hidden_size + ) else: no_images = False num_images = pixel_values.shape[1] @@ -1050,7 +1210,9 @@ def forward( text_seq_len = image_attention_mask.size(1) image_attention_mask = image_attention_mask.unsqueeze(-1) image_attention_mask = image_attention_mask.repeat(1, 1, 1, image_seq_len) - image_attention_mask = image_attention_mask.view(batch_size, text_seq_len, num_images * image_seq_len) + image_attention_mask = image_attention_mask.view( + batch_size, text_seq_len, num_images * image_seq_len + ) image_batch_size, image_sequence_length, _ = image_hidden_states.size() image_hidden_shape = (image_batch_size, image_sequence_length) if image_attention_mask is None: @@ -1060,7 +1222,6 @@ def forward( # if list(image_attention_mask.shape) != [4, 1, 1024, 64]: # raise ValueError(f"Image hidden_states {image_hidden_states.shape} - mask {image_attention_mask.shape} {num_images} {image_seq_len} {text_seq_len}") - # if image_hidden_states is not None: # else: # image_attention_mask = None @@ -1070,10 +1231,15 @@ def forward( # embed positions if attention_mask is None: attention_mask = torch.ones( - (batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device + (batch_size, seq_length_with_past), + dtype=torch.bool, + device=inputs_embeds.device, ) attention_mask = self._prepare_decoder_attention_mask( - attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length + attention_mask, + (batch_size, seq_length), + inputs_embeds, + past_key_values_length, ) hidden_states = inputs_embeds @@ -1094,7 +1260,9 @@ def forward( if output_hidden_states: all_hidden_states += (hidden_states,) - past_key_value = past_key_values[idx] if past_key_values is not None else None + past_key_value = ( + past_key_values[idx] if past_key_values is not None else None + ) def vblock( main_block, @@ -1194,7 +1362,11 @@ def vblock( next_cache = next_decoder_cache if use_cache else None if not return_dict: - return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return tuple( + v + for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] + if v is not None + ) return BaseModelOutputWithPastImage( last_hidden_state=hidden_states, past_key_values=next_cache, @@ -1230,7 +1402,7 @@ def forward( inputs_embeds: Optional[torch.FloatTensor] = None, pixel_values: Optional[torch.FloatTensor] = None, image_embeddings: Optional[torch.FloatTensor] = None, - image_hidden_states: Optional[torch.FloatTensor] = None, + image_hidden_states: Optional[torch.FloatTensor] = None, image_attention_mask: Optional[torch.Tensor] = None, labels: Optional[torch.LongTensor] = None, use_cache: Optional[bool] = None, @@ -1264,11 +1436,19 @@ def forward( "Hey, are you consciours? Can you talk to me?\nI'm not consciours, but I can talk to you." ```""" - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict ) - return_dict = return_dict if return_dict is not None else self.config.use_return_dict # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) outputs = self.model( @@ -1298,7 +1478,7 @@ def forward( past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, - image_hidden_states=outputs.image_hidden_states + image_hidden_states=outputs.image_hidden_states, ) def prepare_inputs_for_generation(self, input_ids, past=None, **kwargs): @@ -1316,12 +1496,20 @@ def _expand_inputs_for_generation( return expand_inputs_for_generation(*args, **model_kwargs) @staticmethod - def _update_model_kwargs_for_generation(outputs, model_kwargs, is_encoder_decoder=False): - return update_model_kwargs_for_generation(outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder) + def _update_model_kwargs_for_generation( + outputs, model_kwargs, is_encoder_decoder=False + ): + return update_model_kwargs_for_generation( + outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder + ) @staticmethod def _reorder_cache(past, beam_idx): reordered_past = () for layer_past in past: - reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + reordered_past += ( + tuple( + past_state.index_select(0, beam_idx) for past_state in layer_past + ), + ) return reordered_past diff --git a/server/text_generation_server/models/custom_modeling/idefics_perceiver.py b/server/text_generation_server/models/custom_modeling/idefics_perceiver.py index def78390..477d4d70 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_perceiver.py +++ b/server/text_generation_server/models/custom_modeling/idefics_perceiver.py @@ -46,7 +46,8 @@ TensorParallelRowLinear, ) -EPS=1e-5 +EPS = 1e-5 + class IdeficsPerceiverResampler(nn.Module): def __init__( @@ -78,7 +79,12 @@ def __init__( """ super().__init__() - self.embed_dim, self.n_heads, self.head_dim, self.n_latents = embed_dim, n_heads, head_dim, n_latents + self.embed_dim, self.n_heads, self.head_dim, self.n_latents = ( + embed_dim, + n_heads, + head_dim, + n_latents, + ) self.qk_layer_norms = config.perceiver_config.qk_layer_norms_perceiver # Create Latents for Perceiver @@ -107,14 +113,16 @@ def __init__( prefix=f"{prefix}.blocks.{layer_id}.1", intermediate_size=self.intermediate_dim, config=config, - weights=weights + weights=weights, ), ] ) for layer_id in range(depth) ] ) - self.layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.layer_norm", weights=weights, eps=EPS) + self.layer_norm = nn.LayerNorm.load( + prefix=f"{prefix}.layer_norm", weights=weights, eps=EPS + ) def forward(self, context: torch.Tensor) -> torch.Tensor: """Resample arbitrary length context & *compress* down to self.n_latents latent embeddings""" @@ -130,25 +138,34 @@ def forward(self, context: torch.Tensor) -> torch.Tensor: class IdeficsPerceiverAttention(nn.Module): - def __init__(self, - prefix, - config, - embed_dim: int, - n_heads: int, - head_dim: int, - qk_layer_norms: bool, - weights - ) -> None: + def __init__( + self, + prefix, + config, + embed_dim: int, + n_heads: int, + head_dim: int, + qk_layer_norms: bool, + weights, + ) -> None: """Perceiver Cross-Attention Module --> let long-form inputs be `context`, resampled embeddings be `latents`""" super().__init__() self.embed_dim, self.n_heads, self.head_dim = embed_dim, n_heads, head_dim self.qk_layer_norms = qk_layer_norms # Normalization & Scaling - self.context_layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.context_layer_norm", weights=weights, eps=EPS) - self.latents_layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.latents_layer_norm", weights=weights, eps=EPS) + self.context_layer_norm = nn.LayerNorm.load( + prefix=f"{prefix}.context_layer_norm", weights=weights, eps=EPS + ) + self.latents_layer_norm = nn.LayerNorm.load( + prefix=f"{prefix}.latents_layer_norm", weights=weights, eps=EPS + ) if self.qk_layer_norms: - self.q_layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.q_layer_norm", weights=weights, eps=EPS) - self.k_layer_norm = nn.LayerNorm.load(prefix=f"{prefix}.k_layer_norm", weights=weights, eps=EPS) + self.q_layer_norm = nn.LayerNorm.load( + prefix=f"{prefix}.q_layer_norm", weights=weights, eps=EPS + ) + self.k_layer_norm = nn.LayerNorm.load( + prefix=f"{prefix}.k_layer_norm", weights=weights, eps=EPS + ) self.qk_scale = self.head_dim**-0.5 @@ -164,10 +181,10 @@ def __init__(self, self.q_proj = TensorParallelColumnLinear.load( config=config, prefix=f"{prefix}.q_proj", weights=weights, bias=False ) - self.k_proj = TensorParallelColumnLinear.load( + self.k_proj = TensorParallelColumnLinear.load( config=config, prefix=f"{prefix}.k_proj", weights=weights, bias=False ) - self.v_proj = TensorParallelColumnLinear.load( + self.v_proj = TensorParallelColumnLinear.load( config=config, prefix=f"{prefix}.v_proj", weights=weights, bias=False ) @@ -202,7 +219,12 @@ def forward(self, context: torch.Tensor, latents: torch.Tensor) -> torch.Tensor: # Multiheaded Self-Attention w/ stable softmax (subtract per-row max -- `amax` -- before softmax call) # =>> `attn` should be a 2D matrix of shape [n_latents x (context + n_latents)] # einsum.rearrange(x, "bsz seq (heads embed) -> bsz heads seq embed", heads=self.n_heads) - q, k, v = [x.reshape(batch_size, x.shape[1], self.n_heads, self.head_dim).transpose(1, 2) for x in (q, k, v)] + q, k, v = [ + x.reshape(batch_size, x.shape[1], self.n_heads, self.head_dim).transpose( + 1, 2 + ) + for x in (q, k, v) + ] if self.qk_layer_norms: q = self.q_layer_norm(q) @@ -219,25 +241,34 @@ def forward(self, context: torch.Tensor, latents: torch.Tensor) -> torch.Tensor: class IdeficsMLP(nn.Module): - def __init__(self, - prefix, - intermediate_size, - config, - weights, - ): + def __init__( + self, + prefix, + intermediate_size, + config, + weights, + ): """Simple MLP block with intermediate_size and embedding size""" super().__init__() self.embed_dim = config.vision_config.embed_dim self.ln = nn.LayerNorm.load(prefix=f"{prefix}.ln", weights=weights, eps=EPS) self.fc = TensorParallelColumnLinear.load( - config=config, prefix=f"{prefix}.fc", weights=weights, bias=False, + config=config, + prefix=f"{prefix}.fc", + weights=weights, + bias=False, ) self.act = nn.ReLU() self.c_proj = TensorParallelRowLinear.load( - config=config, prefix=f"{prefix}.c_proj", weights=weights, bias=False, + config=config, + prefix=f"{prefix}.c_proj", + weights=weights, + bias=False, ) - def forward(self, hidden_states: Optional[Tuple[torch.FloatTensor]]) -> torch.FloatTensor: + def forward( + self, hidden_states: Optional[Tuple[torch.FloatTensor]] + ) -> torch.FloatTensor: hidden_states = self.ln(hidden_states) hidden_states = self.fc(hidden_states) hidden_states = self.act(hidden_states) diff --git a/server/text_generation_server/models/custom_modeling/idefics_processing.py b/server/text_generation_server/models/custom_modeling/idefics_processing.py index e24fc7bd..0fbcbeeb 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_processing.py +++ b/server/text_generation_server/models/custom_modeling/idefics_processing.py @@ -21,9 +21,16 @@ from transformers.feature_extraction_utils import BatchFeature from transformers.processing_utils import ProcessorMixin -from transformers.tokenization_utils_base import BatchEncoding, PaddingStrategy, TextInput, TruncationStrategy +from transformers.tokenization_utils_base import ( + BatchEncoding, + PaddingStrategy, + TextInput, + TruncationStrategy, +) from transformers.utils import TensorType, is_torch_available -from text_generation_server.models.custom_modeling.idefics_image_processing import IdeficsImageProcessor +from text_generation_server.models.custom_modeling.idefics_image_processing import ( + IdeficsImageProcessor, +) if is_torch_available(): @@ -124,7 +131,14 @@ class IdeficsProcessor(ProcessorMixin): image_processor_class = "IdeficsImageProcessor" tokenizer_class = "LlamaTokenizerFast" - def __init__(self, image_processor, tokenizer=None, image_size=224, add_end_of_utterance_token=None, **kwargs): + def __init__( + self, + image_processor, + tokenizer=None, + image_size=224, + add_end_of_utterance_token=None, + **kwargs, + ): if image_processor is None: raise ValueError("You need to specify an `image_processor`.") if tokenizer is None: @@ -142,7 +156,8 @@ def __init__(self, image_processor, tokenizer=None, image_size=224, add_end_of_u self.tokenizer_was_trained_with_end_of_utterance_token = ( True - if "" in self.tokenizer.special_tokens_map.get("additional_special_tokens", []) + if "" + in self.tokenizer.special_tokens_map.get("additional_special_tokens", []) else False ) @@ -265,7 +280,9 @@ def __call__( # if the value isn't overriden by the user, check if the tokenizer was trained with this token and then use it if add_end_of_utterance_token is None: - add_end_of_utterance_token = self.tokenizer_was_trained_with_end_of_utterance_token + add_end_of_utterance_token = ( + self.tokenizer_was_trained_with_end_of_utterance_token + ) # turn non-batched prompts into batched if not any(isinstance(i, list) for i in prompts): @@ -358,10 +375,14 @@ def image_tokens(last_was_image): current_images = images[:local_max_num_images] if len(current_images) > 0: - padded_image_tensor = torch.zeros(max_num_images, *current_images.size()[1:]) + padded_image_tensor = torch.zeros( + max_num_images, *current_images.size()[1:] + ) padded_image_tensor[: current_images.size(0)] = current_images else: - padded_image_tensor = torch.zeros(max_num_images, *self.default_image_dims) + padded_image_tensor = torch.zeros( + max_num_images, *self.default_image_dims + ) output_images.append(padded_image_tensor) output_input_ids.append(torch.tensor(padded_input_ids)) @@ -373,14 +394,19 @@ def image_tokens(last_was_image): output_attention_masks = torch.stack(output_attention_masks) if at_least_one_image: - image_attention_mask, _ = image_attention_mask_for_packed_input_ids(output_input_ids, self.tokenizer) + image_attention_mask, _ = image_attention_mask_for_packed_input_ids( + output_input_ids, self.tokenizer + ) image_attention_mask = incremental_to_binary_attention_mask( image_attention_mask, num_classes=max_num_images ) else: # in full language mode we set the image mask to all-0s image_attention_mask = torch.zeros( - output_input_ids.shape[0], output_input_ids.shape[1], 1, dtype=torch.bool + output_input_ids.shape[0], + output_input_ids.shape[1], + 1, + dtype=torch.bool, ) return BatchFeature( diff --git a/server/text_generation_server/models/custom_modeling/idefics_vision.py b/server/text_generation_server/models/custom_modeling/idefics_vision.py index 30f07095..c521dd0a 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_vision.py +++ b/server/text_generation_server/models/custom_modeling/idefics_vision.py @@ -75,7 +75,9 @@ def __init__(self, prefix, config, weights): self.image_size = config.image_size self.patch_size = config.patch_size - self.class_embedding = nn.Parameter(weights.get_tensor(f"{prefix}.class_embedding")) + self.class_embedding = nn.Parameter( + weights.get_tensor(f"{prefix}.class_embedding") + ) self.patch_embedding = nn.Conv2d.load_no_bias( prefix=f"{prefix}.patch_embedding", @@ -91,12 +93,16 @@ def __init__(self, prefix, config, weights): self.position_embedding = TensorParallelEmbedding( prefix="model.vision_model.embeddings.position_embedding", weights=weights ) - self.position_ids = torch.arange(self.num_positions).expand((1, -1)).to(device=weights.device) + self.position_ids = ( + torch.arange(self.num_positions).expand((1, -1)).to(device=weights.device) + ) def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: batch_size = pixel_values.shape[0] target_dtype = self.patch_embedding.weight.dtype - patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype)) # shape = [*, width, grid, grid] + patch_embeds = self.patch_embedding( + pixel_values.to(dtype=target_dtype) + ) # shape = [*, width, grid, grid] patch_embeds = patch_embeds.flatten(2).transpose(1, 2) class_embeds = self.class_embedding.expand(batch_size, 1, -1) @@ -132,7 +138,6 @@ def __init__(self, prefix, config, weights): self.num_heads = self.num_heads // weights.process_group.size() self.embed_dim = self.embed_dim // weights.process_group.size() - self.k_proj = TensorParallelColumnLinear.load( config, prefix=f"{prefix}.k_proj", weights=weights, bias=True ) @@ -147,7 +152,11 @@ def __init__(self, prefix, config, weights): ) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + return ( + tensor.view(bsz, seq_len, self.num_heads, self.head_dim) + .transpose(1, 2) + .contiguous() + ) def forward( self, @@ -186,7 +195,10 @@ def forward( f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" f" {causal_attention_mask.size()}" ) - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + attn_weights = ( + attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + + causal_attention_mask + ) attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) if attention_mask is not None: @@ -194,7 +206,10 @@ def forward( raise ValueError( f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" ) - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = ( + attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + + attention_mask + ) attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) attn_weights = nn.functional.softmax(attn_weights, dim=-1) @@ -204,12 +219,18 @@ def forward( # make sure that attn_weights keeps its gradient. # In order to do so, attn_weights have to reshaped # twice and have to be reused in the following - attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + attn_weights_reshaped = attn_weights.view( + bsz, self.num_heads, tgt_len, src_len + ) + attn_weights = attn_weights_reshaped.view( + bsz * self.num_heads, tgt_len, src_len + ) else: attn_weights_reshaped = None - attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + attn_probs = nn.functional.dropout( + attn_weights, p=self.dropout, training=self.training + ) attn_output = torch.bmm(attn_probs, value_states) @@ -253,11 +274,15 @@ class IdeficsVisionEncoderLayer(nn.Module): def __init__(self, prefix, config, weights): super().__init__() self.embed_dim = config.hidden_size - self.self_attn = IdeficsVisionAttention(prefix=f"{prefix}.self_attn", config=config, weights=weights) + self.self_attn = IdeficsVisionAttention( + prefix=f"{prefix}.self_attn", config=config, weights=weights + ) self.layer_norm1 = nn.LayerNorm.load( prefix=f"{prefix}.layer_norm1", weights=weights, eps=config.layer_norm_eps ) - self.mlp = IdeficsVisionMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) + self.mlp = IdeficsVisionMLP( + prefix=f"{prefix}.mlp", config=config, weights=weights + ) self.layer_norm2 = nn.LayerNorm.load( prefix=f"{prefix}.layer_norm2", weights=weights, eps=config.layer_norm_eps ) @@ -318,7 +343,11 @@ def __init__(self, prefix, config, weights): self.config = config self.layers = nn.ModuleList( [ - IdeficsVisionEncoderLayer(prefix=f"{prefix}.encoder.layers.{layer_id}", config=config, weights=weights) + IdeficsVisionEncoderLayer( + prefix=f"{prefix}.encoder.layers.{layer_id}", + config=config, + weights=weights, + ) for layer_id in range(config.num_hidden_layers) ] ) @@ -362,11 +391,19 @@ def forward( return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. """ - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict ) - return_dict = return_dict if return_dict is not None else self.config.use_return_dict encoder_states = () if output_hidden_states else None all_attentions = () if output_attentions else None @@ -406,9 +443,15 @@ def forward( encoder_states = encoder_states + (hidden_states,) if not return_dict: - return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) + return tuple( + v + for v in [hidden_states, encoder_states, all_attentions] + if v is not None + ) return BaseModelOutput( - last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions + last_hidden_state=hidden_states, + hidden_states=encoder_states, + attentions=all_attentions, ) @@ -419,13 +462,19 @@ def __init__(self, prefix, config, weights): self.config = config embed_dim = config.hidden_size - self.embeddings = IdeficsVisionEmbeddings(prefix=f"{prefix}.embeddings", config=config, weights=weights) + self.embeddings = IdeficsVisionEmbeddings( + prefix=f"{prefix}.embeddings", config=config, weights=weights + ) self.pre_layrnorm = nn.LayerNorm.load( prefix=f"{prefix}.pre_layrnorm", weights=weights, eps=config.layer_norm_eps ) - self.encoder = IdeficsVisionEncoder(prefix=prefix, config=config, weights=weights) + self.encoder = IdeficsVisionEncoder( + prefix=prefix, config=config, weights=weights + ) self.post_layernorm = nn.LayerNorm.load( - prefix=f"{prefix}.post_layernorm", weights=weights, eps=config.layer_norm_eps + prefix=f"{prefix}.post_layernorm", + weights=weights, + eps=config.layer_norm_eps, ) # copied from transformers.models.clip.modeling_clip.CLIPVisionTransformer.forward @@ -440,11 +489,19 @@ def forward( Returns: """ - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_attentions = ( + output_attentions + if output_attentions is not None + else self.config.output_attentions + ) output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + output_hidden_states + if output_hidden_states is not None + else self.config.output_hidden_states + ) + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict ) - return_dict = return_dict if return_dict is not None else self.config.use_return_dict if pixel_values is None: raise ValueError("You have to specify pixel_values") diff --git a/server/text_generation_server/models/custom_modeling/neox_modeling.py b/server/text_generation_server/models/custom_modeling/neox_modeling.py index c5b0c7fd..24ba6796 100644 --- a/server/text_generation_server/models/custom_modeling/neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/neox_modeling.py @@ -49,7 +49,10 @@ CUSTOM_KERNELS_ENABLED = False -if torch.cuda.is_available() and not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True": +if ( + torch.cuda.is_available() + and not os.environ.get("DISABLE_CUSTOM_KERNELS", "False") == "True" +): try: from custom_kernels import fused_attention_cuda diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 12d8efeb..34c7f633 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -1005,9 +1005,12 @@ def generate_token( # Decode generated tokens output_text, _, _ = self.decode_token( all_input_ids, - prefix_offset=len(all_input_ids) - stopping_criteria.current_tokens - 1, - read_offset=len(all_input_ids) - stopping_criteria.current_tokens, - skip_special_tokens=True + prefix_offset=len(all_input_ids) + - stopping_criteria.current_tokens + - 1, + read_offset=len(all_input_ids) + - stopping_criteria.current_tokens, + skip_special_tokens=True, ) generated_text = GeneratedText( output_text, diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py index 30cc2299..2472caf6 100644 --- a/server/text_generation_server/models/idefics_causal_lm.py +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -8,7 +8,13 @@ from dataclasses import dataclass from opentelemetry import trace -from transformers import AutoProcessor, AutoTokenizer, AutoModelForCausalLM, PreTrainedTokenizerBase, ProcessorMixin +from transformers import ( + AutoProcessor, + AutoTokenizer, + AutoModelForCausalLM, + PreTrainedTokenizerBase, + ProcessorMixin, +) from typing import Optional, Tuple, List, Type, Dict from text_generation_server.models import Model @@ -23,7 +29,8 @@ import re -IMAGES = re.compile(r'!\[[^\]]*\]\((.*?)\s*(\"(?:.*[^\"])\")?\s*\)') +IMAGES = re.compile(r"!\[[^\]]*\]\((.*?)\s*(\"(?:.*[^\"])\")?\s*\)") + def split(string): parts = [] @@ -41,6 +48,7 @@ def split(string): return parts + tracer = trace.get_tracer(__name__) @@ -94,7 +102,7 @@ def from_pb( cls, pb: generate_pb2.Batch, tokenizer: PreTrainedTokenizerBase, - processor: ProcessorMixin, # Hack + processor: ProcessorMixin, # Hack dtype: torch.dtype, device: torch.device, ) -> "IdeficsCausalLMBatch": @@ -137,12 +145,16 @@ def from_pb( padding=True, truncation=True, max_length=max_truncation, - add_end_of_utterance_token=False, # Already taken care of inside the prompts, so bypassing the processor's handling of this token + add_end_of_utterance_token=False, # Already taken care of inside the prompts, so bypassing the processor's handling of this token ).to(device) for _ in pb.requests: input_len = tokenized_inputs["input_ids"].shape[1] - prefix_offsets.append(input_len - 5) # To decode without potential fallbacks errors - read_offsets.append(input_len) # To decode without potential fallbacks errors + prefix_offsets.append( + input_len - 5 + ) # To decode without potential fallbacks errors + read_offsets.append( + input_len + ) # To decode without potential fallbacks errors input_lengths = tokenized_inputs["attention_mask"].sum(1) max_input_length = input_lengths.max() @@ -158,14 +170,21 @@ def from_pb( attention_mask[:, :max_input_length] = tokenized_inputs["attention_mask"] # Do the same for image_attention_mask image_attention_mask = input_ids.new_zeros( - (pb.size, max_input_length + padding_right_offset, tokenized_inputs["pixel_values"].size(1)) + ( + pb.size, + max_input_length + padding_right_offset, + tokenized_inputs["pixel_values"].size(1), + ) ) - image_attention_mask[:, :max_input_length, :] = tokenized_inputs["image_attention_mask"] - + image_attention_mask[:, :max_input_length, :] = tokenized_inputs[ + "image_attention_mask" + ] position_ids = tokenized_inputs["attention_mask"].long().cumsum(-1) - 1 position_ids.masked_fill_(tokenized_inputs["attention_mask"] == 0, 1) - all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) # It's input_ids but splitted into a tuple of tensors where each tensor is (seq_len, 1) size. It is then transformed into a list + all_input_ids = tokenized_inputs["input_ids"].T.split( + 1, dim=1 + ) # It's input_ids but splitted into a tuple of tensors where each tensor is (seq_len, 1) size. It is then transformed into a list max_tokens = len(inputs) * (max_input_length + max_decode_tokens) @@ -259,7 +278,7 @@ def filter(self, request_ids: List[int]) -> Optional["IdeficsCausalLMBatch"]: self.image_attention_mask.shape[1] - self.padding_right_offset ) + new_padding_right_offset, - : + :, ] if self.image_hidden_states is None: image_hidden_states = None @@ -308,7 +327,9 @@ def filter(self, request_ids: List[int]) -> Optional["IdeficsCausalLMBatch"]: @classmethod @tracer.start_as_current_span("concatenate") - def concatenate(cls, batches: List["IdeficsCausalLMBatch"]) -> "IdeficsCausalLMBatch": + def concatenate( + cls, batches: List["IdeficsCausalLMBatch"] + ) -> "IdeficsCausalLMBatch": # It adds new requests to the batch # Used for padding total_batch_size = 0 @@ -383,12 +404,20 @@ def concatenate(cls, batches: List["IdeficsCausalLMBatch"]) -> "IdeficsCausalLMB curr_batch_max_num_images = batch.pixel_values.size(1) if pixel_values is None: - pixel_values = batch.pixel_values.new_zeros((total_batch_size, max_num_images, 3, 224, 224)) - pixel_values[start_index:end_index, :curr_batch_max_num_images] = batch.pixel_values + pixel_values = batch.pixel_values.new_zeros( + (total_batch_size, max_num_images, 3, 224, 224) + ) + pixel_values[ + start_index:end_index, :curr_batch_max_num_images + ] = batch.pixel_values if image_attention_mask is None: image_attention_mask = batch.image_attention_mask.new_zeros( - (total_batch_size, max_input_length + padding_right_offset, max_num_images) + ( + total_batch_size, + max_input_length + padding_right_offset, + max_num_images, + ) ) # We need to slice the attention mask to remove padding from previous steps @@ -409,11 +438,9 @@ def concatenate(cls, batches: List["IdeficsCausalLMBatch"]) -> "IdeficsCausalLMB image_attention_mask[ start_index:end_index, left_offset:-padding_right_offset, - :curr_batch_max_num_images + :curr_batch_max_num_images, ] = batch.image_attention_mask[ - :, - batch_left_offset : - batch.padding_right_offset, - : + :, batch_left_offset : -batch.padding_right_offset, : ] # Create empty tensor @@ -550,7 +577,9 @@ def __init__( dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): - from text_generation_server.models.custom_modeling.idefics_modeling import IdeficsForVisionText2Text + from text_generation_server.models.custom_modeling.idefics_modeling import ( + IdeficsForVisionText2Text, + ) if torch.cuda.is_available(): device = torch.device("cuda") @@ -650,9 +679,13 @@ def generate_token( # this is due to the nature IDEFICS: it's an encoder decoder, and so when decoding, only the currently generated # token need to attend to the encoder hidden states (i.e. the vision encoder) # Also see seq2seq_lm.Seq2SeqLM.generate_token which has roughly the same logic - image_attention_mask = batch.image_attention_mask[:, -(batch.padding_right_offset+1)].unsqueeze(1) + image_attention_mask = batch.image_attention_mask[ + :, -(batch.padding_right_offset + 1) + ].unsqueeze(1) else: - image_attention_mask = batch.image_attention_mask[:, : -batch.padding_right_offset] + image_attention_mask = batch.image_attention_mask[ + :, : -batch.padding_right_offset + ] logits, past, image_hidden_states = self.forward( input_ids=batch.input_ids, @@ -725,9 +758,12 @@ def generate_token( # Decode generated tokens output_text, _, _ = self.decode_token( all_input_ids[:, 0], - prefix_offset=len(all_input_ids) - stopping_criteria.current_tokens - 1, - read_offset=len(all_input_ids) - stopping_criteria.current_tokens, - skip_special_tokens=True + prefix_offset=len(all_input_ids) + - stopping_criteria.current_tokens + - 1, + read_offset=len(all_input_ids) + - stopping_criteria.current_tokens, + skip_special_tokens=True, ) # Get seed if isinstance(next_token_chooser.choice, Sampling): @@ -761,7 +797,7 @@ def generate_token( else: prefill_tokens = None - top_tokens=None + top_tokens = None generation = Generation( request.id, @@ -771,7 +807,7 @@ def generate_token( next_token_text, next_token_id_squeezed.item() in self.all_special_ids, generated_text, - top_tokens + top_tokens, ) generations.append(generation) @@ -793,7 +829,9 @@ def generate_token( # Update attention_mask as we added a new token to input_ids batch.attention_mask[:, -batch.padding_right_offset] = 1 - batch.image_attention_mask[:, -batch.padding_right_offset, :] = batch.image_attention_mask[:, -(batch.padding_right_offset+1), :] + batch.image_attention_mask[ + :, -batch.padding_right_offset, : + ] = batch.image_attention_mask[:, -(batch.padding_right_offset + 1), :] # Decrease right offset batch.padding_right_offset -= 1 diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 73329b24..f6e66d30 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -71,7 +71,8 @@ def decode_token( # The prefix text is necessary only to defeat cleanup algorithms in the decode # which decide to add a space or not depending on the surrounding ids. prefix_text = self.tokenizer.decode( - all_input_ids[prefix_offset:read_offset], skip_special_tokens=skip_special_tokens + all_input_ids[prefix_offset:read_offset], + skip_special_tokens=skip_special_tokens, ) new_text = self.tokenizer.decode( all_input_ids[prefix_offset:], skip_special_tokens=skip_special_tokens diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index f67874be..d4d3cd19 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -712,9 +712,11 @@ def generate_token( # Decode all tokens output_text, _, _ = self.decode_token( all_decoder_input_ids, - prefix_offset=len(all_decoder_input_ids) - decoder_input_length - 1, + prefix_offset=len(all_decoder_input_ids) + - decoder_input_length + - 1, read_offset=len(all_decoder_input_ids) - decoder_input_length, - skip_special_tokens=True + skip_special_tokens=True, ) # Get seed diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 67137aaa..75d2b159 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -16,6 +16,7 @@ from text_generation_server.tracing import UDSOpenTelemetryAioServerInterceptor from text_generation_server.models.idefics_causal_lm import IdeficsCausalLMBatch + class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): def __init__(self, model: Model, cache: Cache, server_urls: List[str]): self.cache = cache @@ -26,7 +27,6 @@ def __init__(self, model: Model, cache: Cache, server_urls: List[str]): # Force inference mode for the lifetime of TextGenerationService self._inference_mode_raii_guard = torch._C._InferenceMode(True) - async def Info(self, request, context): return self.model.info @@ -55,9 +55,15 @@ async def FilterBatch(self, request, context): return generate_pb2.FilterBatchResponse(batch=filtered_batch.to_pb()) async def Warmup(self, request, context): - if self.model.batch_type == IdeficsCausalLMBatch: #Hack, i would rather use kwargs in the `from_pb` call + if ( + self.model.batch_type == IdeficsCausalLMBatch + ): # Hack, i would rather use kwargs in the `from_pb` call batch = self.model.batch_type.from_pb( - request.batch, self.model.tokenizer, self.model.processor, self.model.dtype, self.model.device + request.batch, + self.model.tokenizer, + self.model.processor, + self.model.dtype, + self.model.device, ) else: batch = self.model.batch_type.from_pb( @@ -70,9 +76,15 @@ async def Warmup(self, request, context): ) async def Prefill(self, request, context): - if self.model.batch_type == IdeficsCausalLMBatch: #Hack, i would rather use kwargs in the `from_pb` call + if ( + self.model.batch_type == IdeficsCausalLMBatch + ): # Hack, i would rather use kwargs in the `from_pb` call batch = self.model.batch_type.from_pb( - request.batch, self.model.tokenizer, self.model.processor, self.model.dtype, self.model.device + request.batch, + self.model.tokenizer, + self.model.processor, + self.model.dtype, + self.model.device, ) else: batch = self.model.batch_type.from_pb( diff --git a/server/text_generation_server/utils/awq/quantize/qmodule.py b/server/text_generation_server/utils/awq/quantize/qmodule.py index c658e17f..ca8caf50 100644 --- a/server/text_generation_server/utils/awq/quantize/qmodule.py +++ b/server/text_generation_server/utils/awq/quantize/qmodule.py @@ -11,7 +11,7 @@ # super().__init__() # self.act = module # self.scales = nn.Parameter(scales.data) -# +# # def forward(self, x): # return self.act(x) / self.scales.view(1, 1, -1).to(x.device) @@ -19,10 +19,10 @@ class WQLinear(nn.Module): def __init__(self, w_bit, group_size, qweight, qzeros, scales, bias): super().__init__() - + if w_bit not in [4]: raise NotImplementedError("Only 4-bit are supported for now.") - + self.in_features = qweight.shape[0] self.out_features = qweight.shape[1] * 32 // w_bit @@ -42,7 +42,9 @@ def __init__(self, w_bit, group_size, qweight, qzeros, scales, bias): @torch.no_grad() def forward(self, x): - out_shape = x.shape[:-1] + (self.out_features, ) - out = awq_inference_engine.gemm_forward_cuda(x.reshape(-1, x.shape[-1]), self.qweight, self.scales, self.qzeros, 8) + out_shape = x.shape[:-1] + (self.out_features,) + out = awq_inference_engine.gemm_forward_cuda( + x.reshape(-1, x.shape[-1]), self.qweight, self.scales, self.qzeros, 8 + ) out = out + self.bias if self.bias is not None else out return out.reshape(out_shape) diff --git a/server/text_generation_server/utils/gptq/quantize.py b/server/text_generation_server/utils/gptq/quantize.py index 9547d534..ca113d8f 100644 --- a/server/text_generation_server/utils/gptq/quantize.py +++ b/server/text_generation_server/utils/gptq/quantize.py @@ -578,7 +578,9 @@ def __init__(self, input_ids): return trainloader, valenc -def get_loaders(name, nsamples=128, seed=0, seqlen=2048, model_id="", trust_remote_code=False): +def get_loaders( + name, nsamples=128, seed=0, seqlen=2048, model_id="", trust_remote_code=False +): if "wikitext2" in name: return get_wikitext2(nsamples, seed, seqlen, model_id, trust_remote_code) if "ptb" in name: @@ -927,7 +929,7 @@ def _unload(): seed=seed, model_id=model_id, seqlen=model.seqlen, - trust_remote_code=trust_remote_code + trust_remote_code=trust_remote_code, ) tick = time.time() diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 14cb55cc..8be2463f 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -22,7 +22,7 @@ HAS_AWQ = True -try: +try: from text_generation_server.utils.awq.quantize.qmodule import WQLinear except ImportError: HAS_AWQ = False @@ -36,17 +36,19 @@ if os.getenv("DISABLE_EXLLAMA") == "True": HAS_EXLLAMA = False elif CAN_EXLLAMA: - try: - from text_generation_server.utils.gptq.exllama import Ex4bitLinear - HAS_EXLLAMA = True - except ImportError: - pass + try: + from text_generation_server.utils.gptq.exllama import Ex4bitLinear + + HAS_EXLLAMA = True + except ImportError: + pass from typing import Optional HAS_EETQ = False try: from EETQ import quant_weights, w8_a16_gemm + HAS_EETQ = True except ImportError: pass @@ -74,12 +76,18 @@ def load_layer_norm_no_bias(cls, prefix, weights, eps): ln.bias = None return ln + @classmethod def load_conv2d(cls, prefix, weights, in_channels, out_channels, kernel_size, stride): weight = weights.get_tensor(f"{prefix}.weight") bias = weights.get_tensor(f"{prefix}.bias") with init_empty_weights(): - conv2d = cls(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride) + conv2d = cls( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + ) conv2d.weight = nn.Parameter(weight) conv2d.bias = nn.Parameter(bias) @@ -87,10 +95,17 @@ def load_conv2d(cls, prefix, weights, in_channels, out_channels, kernel_size, st @classmethod -def load_conv2d_no_bias(cls, prefix, weights, in_channels, out_channels, kernel_size, stride): +def load_conv2d_no_bias( + cls, prefix, weights, in_channels, out_channels, kernel_size, stride +): weight = weights.get_tensor(f"{prefix}.weight") with init_empty_weights(): - conv2d = cls(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride) + conv2d = cls( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + ) conv2d.weight = nn.Parameter(weight) conv2d.bias = None @@ -215,7 +230,10 @@ class Linear4bit(nn.Module): def __init__(self, weight, bias, quant_type): super().__init__() self.weight = Params4bit( - weight.data, requires_grad=False, compress_statistics=True, quant_type=quant_type + weight.data, + requires_grad=False, + compress_statistics=True, + quant_type=quant_type, ) self.compute_dtype = None self.weight.cuda(weight.device) @@ -246,7 +264,10 @@ def forward(self, x: torch.Tensor): @lru_cache(1) def warn_deprecate_bnb(): - logger.warning("Bitsandbytes 8bit is deprecated, using `eetq` is a drop-in replacement, and has much better performnce") + logger.warning( + "Bitsandbytes 8bit is deprecated, using `eetq` is a drop-in replacement, and has much better performnce" + ) + def get_linear(weight, bias, quantize): if quantize is None: @@ -255,7 +276,9 @@ def get_linear(weight, bias, quantize): if HAS_EETQ: linear = EETQLinear(weight, bias) else: - raise ImportError("Please install EETQ from https://github.com/NetEase-FuXi/EETQ") + raise ImportError( + "Please install EETQ from https://github.com/NetEase-FuXi/EETQ" + ) elif quantize == "bitsandbytes": warn_deprecate_bnb() linear = Linear8bitLt( @@ -305,7 +328,14 @@ def get_linear(weight, bias, quantize): raise NotImplementedError( f"The passed weight is not `awq` compatible, loader needs to be updated." ) - linear = WQLinear(w_bit=bits, group_size=groupsize, qweight=qweight, qzeros=qzeros, scales=scales, bias=bias is not None) + linear = WQLinear( + w_bit=bits, + group_size=groupsize, + qweight=qweight, + qzeros=qzeros, + scales=scales, + bias=bias is not None, + ) else: raise NotImplementedError(f"Quantization `{quantize}` is not implemented yet.") return linear @@ -392,9 +422,7 @@ class TensorParallelColumnLinear(SuperLayer): @classmethod def load_qkv(cls, config, prefix: str, weights, bias: bool): """Specific method when the QKV was joined after the fact""" - weight = weights.get_weights_col_packed_qkv( - prefix, quantize=config.quantize - ) + weight = weights.get_weights_col_packed_qkv(prefix, quantize=config.quantize) if bias: raise NotImplementedError("packed_qkv only implemented for baichuan") else: @@ -530,14 +558,16 @@ def forward(self, hidden_states, residual=None): def _create_inv_freq(dim, base, device): inv_freq = 1.0 / ( - base - ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) + base ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) ) return inv_freq def _get_rope_config(config): if os.getenv("ROPE_SCALING", None) is not None: - rope_scaling = {"type": os.environ["ROPE_SCALING"], "factor": float(os.environ["ROPE_FACTOR"])} + rope_scaling = { + "type": os.environ["ROPE_SCALING"], + "factor": float(os.environ["ROPE_FACTOR"]), + } return rope_scaling return getattr(config, "rope_scaling", None) @@ -563,9 +593,17 @@ def static(cls, config, dim, base, device): if rope_scaling["type"] == "linear": pass elif rope_scaling["type"] == "dynamic": - return DynamicPositionRotaryEmbedding(dim=dim, max_position_embeddings=config.max_position_embeddings, base=base, device=inv_freq.device, scaling_factor=scaling_factor) + return DynamicPositionRotaryEmbedding( + dim=dim, + max_position_embeddings=config.max_position_embeddings, + base=base, + device=inv_freq.device, + scaling_factor=scaling_factor, + ) else: - raise NotImplementedError(f"rope scaling type {rope_scaling['type']} is not implemented or invalid") + raise NotImplementedError( + f"rope scaling type {rope_scaling['type']} is not implemented or invalid" + ) return cls(inv_freq, scaling_factor) @classmethod @@ -583,9 +621,17 @@ def load(cls, config, prefix, weights): if rope_scaling["type"] == "linear": pass elif rope_scaling["type"] == "dynamic": - return DynamicPositionRotaryEmbedding(dim=2*inv_freq.shape[0], max_position_embeddings=config.max_position_embeddings, base=10000.0, device=inv_freq.device, scaling_factor=scaling_factor) + return DynamicPositionRotaryEmbedding( + dim=2 * inv_freq.shape[0], + max_position_embeddings=config.max_position_embeddings, + base=10000.0, + device=inv_freq.device, + scaling_factor=scaling_factor, + ) else: - raise NotImplementedError(f"rope scaling type {rope_scaling['type']} is not implemented or invalid") + raise NotImplementedError( + f"rope scaling type {rope_scaling['type']} is not implemented or invalid" + ) return cls(inv_freq, scaling_factor) def _update_cos_sin_cache(self, dtype, device, seqlen): @@ -645,8 +691,13 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): or self._cos_cached.dtype != dtype ): if seqlen > self.max_position_embeddings: - newbase = self.base * ((self.scaling_factor * seqlen / self.max_position_embeddings) - (self.scaling_factor - 1)) ** (self.dim / (self.dim - 2)) - self.inv_freq = _create_inv_freq(self.dim, newbase, self.inv_freq.device) + newbase = self.base * ( + (self.scaling_factor * seqlen / self.max_position_embeddings) + - (self.scaling_factor - 1) + ) ** (self.dim / (self.dim - 2)) + self.inv_freq = _create_inv_freq( + self.dim, newbase, self.inv_freq.device + ) self._seq_len_cached = seqlen t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) # Don't do einsum, it converts fp32 to fp16 @@ -656,6 +707,5 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): self._cos_cached = torch.cos(freqs).to(dtype) self._sin_cached = torch.sin(freqs).to(dtype) - except ImportError: pass diff --git a/server/text_generation_server/utils/peft.py b/server/text_generation_server/utils/peft.py index be1f9444..e37447dc 100644 --- a/server/text_generation_server/utils/peft.py +++ b/server/text_generation_server/utils/peft.py @@ -6,6 +6,7 @@ from transformers import AutoTokenizer from peft import AutoPeftModelForCausalLM, AutoPeftModelForSeq2SeqLM + def download_and_unload_peft(model_id, revision, trust_remote_code): torch_dtype = torch.float16 @@ -33,7 +34,7 @@ def download_and_unload_peft(model_id, revision, trust_remote_code): base_model_id = model.peft_config["default"].base_model_name_or_path model = model.merge_and_unload() - + os.makedirs(model_id, exist_ok=True) cache_dir = model_id logger.info(f"Saving the newly created merged model to {cache_dir}") @@ -41,6 +42,3 @@ def download_and_unload_peft(model_id, revision, trust_remote_code): model.save_pretrained(cache_dir, safe_serialization=True) model.config.save_pretrained(cache_dir) tokenizer.save_pretrained(cache_dir) - - - diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 7b003f1d..f6339d7c 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -363,7 +363,7 @@ def batch_top_tokens( # Find the new "fuzzy" top n values top_n_indices = (logprobs >= nth_highest).nonzero() _, top_n_ishes = torch.unique_consecutive(top_n_indices[:, 0], return_counts=True) - + k = 1 if top_n_ishes.numel() == 0 else top_n_ishes.max() # Take a new topk for these new max n values top_k = torch.topk(logprobs, k=k, dim=1, sorted=True) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 266fcccb..8a19fd9f 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -62,7 +62,7 @@ def _get_slice(self, tensor_name: str): def get_shape(self, tensor_name: str): return self._get_slice(tensor_name).get_shape() - def get_tensor(self, tensor_name: str, to_device = True): + def get_tensor(self, tensor_name: str, to_device=True): filename, tensor_name = self.get_filename(tensor_name) f = self._get_handle(filename) tensor = f.get_tensor(tensor_name) @@ -110,7 +110,6 @@ def get_sharded(self, tensor_name: str, dim: int): ), f"The choosen size {size} is not compatible with sharding on {world_size} shards" return self.get_partial_sharded(tensor_name, dim) - def _get_qweight(self, name: str): slice_ = self._get_slice(name) total_size = slice_.get_shape()[1] @@ -119,14 +118,16 @@ def _get_qweight(self, name: str): world_size = self.process_group.size() rank = self.process_group.rank() - assert single_size % world_size == 0, f"Prepacked quantized qkv cannot be sharded across {world_size} shards" + assert ( + single_size % world_size == 0 + ), f"Prepacked quantized qkv cannot be sharded across {world_size} shards" block_size = single_size // world_size start = rank * block_size stop = (rank + 1) * block_size q = slice_[:, start:stop] - k = slice_[:, start+single_size:stop+single_size] - v = slice_[:, start+2*single_size:stop+2*single_size] - weight = torch.cat([q,k,v], dim=1) + k = slice_[:, start + single_size : stop + single_size] + v = slice_[:, start + 2 * single_size : stop + 2 * single_size] + weight = torch.cat([q, k, v], dim=1) weight = weight.to(device=self.device) return weight @@ -137,14 +138,14 @@ def get_weights_col_packed_qkv(self, prefix: str, quantize: str): """ if quantize in ["gptq", "awq"]: try: - qweight = self._get_qweight(f"{prefix}.qweight") + qweight = self._get_qweight(f"{prefix}.qweight") except RuntimeError: raise RuntimeError( f"Cannot load `{quantize}` weight, make sure the model is already quantized." ) - qzeros = self._get_qweight(f"{prefix}.qzeros") - scales = self._get_qweight(f"{prefix}.scales") + qzeros = self._get_qweight(f"{prefix}.qzeros") + scales = self._get_qweight(f"{prefix}.scales") scales = scales.to(dtype=self.dtype) if quantize == "gptq": g_idx = self.get_tensor(f"{prefix}.g_idx") @@ -154,21 +155,23 @@ def get_weights_col_packed_qkv(self, prefix: str, quantize: str): bits, groupsize = self._get_gptq_params() weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) else: - slice_ = self._get_slice(f"{prefix}.weight") + slice_ = self._get_slice(f"{prefix}.weight") total_size = slice_.get_shape()[0] assert total_size % 3 == 0, "Prepacked qkv is not divisible by 3" single_size = total_size // 3 world_size = self.process_group.size() rank = self.process_group.rank() - assert single_size % world_size == 0, f"Prepacked qkv cannot be sharded across {world_size} shards" + assert ( + single_size % world_size == 0 + ), f"Prepacked qkv cannot be sharded across {world_size} shards" block_size = single_size // world_size start = rank * block_size stop = (rank + 1) * block_size q = slice_[start:stop] - k = slice_[start+single_size:stop+single_size] - v = slice_[start+2*single_size:stop+2*single_size] - weight = torch.cat([q,k,v], dim=0) + k = slice_[start + single_size : stop + single_size] + v = slice_[start + 2 * single_size : stop + 2 * single_size] + weight = torch.cat([q, k, v], dim=0) weight = weight.to(device=self.device) weight = weight.to(dtype=self.dtype) return weight @@ -205,7 +208,7 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] weight = torch.cat(w, dim=dim) return weight - + def get_tensor_shard(self, var, dim): world_size = self.process_group.size() rank = self.process_group.rank() @@ -220,7 +223,7 @@ def get_tensor_shard(self, var, dim): raise NotImplementedError("Let's make that generic when needed") tensor = tensor.to(dtype=self.dtype) tensor = tensor.to(device=self.device) - return tensor + return tensor def get_multi_weights_row(self, prefix: str, quantize: str): if quantize == "gptq": @@ -303,7 +306,7 @@ def get_multi_weights_row(self, prefix: str, quantize: str): scales = self.get_sharded(f"{prefix}.scales", dim=0) g_idx = None use_exllama = False - + weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) else: weight = self.get_sharded(f"{prefix}.weight", dim=1) From 1e3ec3c91f140c5f0dddad95a9bcb843d67b382b Mon Sep 17 00:00:00 2001 From: zhangsibo1129 <134488188+zhangsibo1129@users.noreply.github.com> Date: Wed, 27 Sep 2023 18:25:59 +0800 Subject: [PATCH 435/793] Complete FastLinear.load parameters in OPTDecoder initialization (#1060) # What does this PR do? `FastLinear.load` requires 4 parameters, but in the following only 3 are given. This PR fix this. ```python # server/text_generation_server/models/custom_modeling/opt_modeling.py if config.word_embed_proj_dim != config.hidden_size: self.project_out = FastLinear.load( config, prefix="model.decoder.project_out", bias=False ) else: self.project_out = None if config.word_embed_proj_dim != config.hidden_size: self.project_in = FastLinear.load( config, prefix="model.decoder.project_in", bias=False ``` ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../models/custom_modeling/opt_modeling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/opt_modeling.py b/server/text_generation_server/models/custom_modeling/opt_modeling.py index fe6f1e52..ce3f5e21 100644 --- a/server/text_generation_server/models/custom_modeling/opt_modeling.py +++ b/server/text_generation_server/models/custom_modeling/opt_modeling.py @@ -444,14 +444,14 @@ def __init__(self, config: OPTConfig, weights): if config.word_embed_proj_dim != config.hidden_size: self.project_out = FastLinear.load( - config, prefix="model.decoder.project_out", bias=False + config, prefix="model.decoder.project_out", weights=weights, bias=False ) else: self.project_out = None if config.word_embed_proj_dim != config.hidden_size: self.project_in = FastLinear.load( - config, prefix="model.decoder.project_in", bias=False + config, prefix="model.decoder.project_in", weights=weights, bias=False ) else: self.project_in = None From 259a2300285a20006d3ada56a7455c4d97afd76f Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Wed, 27 Sep 2023 16:01:38 +0200 Subject: [PATCH 436/793] Automatic docs for TGI (#1045) I had to open this PR since I initially worked from my fork, and it requires a handful of work to trigger a new github action on my fork's specific branch (couldn't find a way, at least, despite trying all of them). --------- Co-authored-by: Nicolas Patry --- .github/workflows/autodocs.yml | 21 +++ docs/source/_toctree.yml | 2 + docs/source/basic_tutorials/launcher.md | 237 ++++++++++++++++++++++++ update_doc.py | 31 ++++ 4 files changed, 291 insertions(+) create mode 100644 .github/workflows/autodocs.yml create mode 100644 docs/source/basic_tutorials/launcher.md create mode 100644 update_doc.py diff --git a/.github/workflows/autodocs.yml b/.github/workflows/autodocs.yml new file mode 100644 index 00000000..a981c09c --- /dev/null +++ b/.github/workflows/autodocs.yml @@ -0,0 +1,21 @@ +name: Automatic Documentation for Launcher + +on: + pull_request: + +jobs: + update_docs: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Install Launcher + id: install-launcher + run: cargo install --git https://github.com/${{ github.repository }} --branch ${{ github.head_ref }} text-generation-launcher + + - name: Check launcher Docs are up-to-date + run: | + echo text-generation-launcher --help + python update_doc.py --check diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 313b6d32..6fa50a6a 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -17,6 +17,8 @@ title: Serving Private & Gated Models - local: basic_tutorials/using_cli title: Using TGI CLI + - local: basic_tutorials/launcher + title: All TGI CLI options - local: basic_tutorials/non_core_models title: Non-core Model Serving title: Tutorials diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md new file mode 100644 index 00000000..c689b550 --- /dev/null +++ b/docs/source/basic_tutorials/launcher.md @@ -0,0 +1,237 @@ +# Text-generation-launcher arguments +``` +Text Generation Launcher + +Usage: text-generation-launcher [OPTIONS] + +Options: + --model-id + The name of the model to load. Can be a MODEL_ID as listed on like `gpt2` or `OpenAssistant/oasst-sft-1-pythia-12b`. Or it can be a local directory containing the necessary files as saved by `save_pretrained(...)` methods of transformers + + [env: MODEL_ID=] + [default: bigscience/bloom-560m] + + --revision + The actual revision of the model if you're referring to a model on the hub. You can use a specific commit id or a branch like `refs/pr/2` + + [env: REVISION=] + + --validation-workers + The number of tokenizer workers used for payload validation and truncation inside the router + + [env: VALIDATION_WORKERS=] + [default: 2] + + --sharded + Whether to shard the model across multiple GPUs By default text-generation-inference will use all available GPUs to run the model. Setting it to `false` deactivates `num_shard` + + [env: SHARDED=] + [possible values: true, false] + + --num-shard + The number of shards to use if you don't want to use all GPUs on a given machine. You can use `CUDA_VISIBLE_DEVICES=0,1 text-generation-launcher... --num_shard 2` and `CUDA_VISIBLE_DEVICES=2,3 text-generation-launcher... --num_shard 2` to launch 2 copies with 2 shard each on a given machine with 4 GPUs for instance + + [env: NUM_SHARD=] + + --quantize + Whether you want the model to be quantized. This will use `bitsandbytes` for quantization on the fly, or `gptq`. 4bit quantization is available through `bitsandbytes` by providing the `bitsandbytes-fp4` or `bitsandbytes-nf4` options + + [env: QUANTIZE=] + [possible values: bitsandbytes, bitsandbytes-nf4, bitsandbytes-fp4, gptq, awq] + + --dtype + The dtype to be forced upon the model. This option cannot be used with `--quantize` + + [env: DTYPE=] + [possible values: float16, bfloat16] + + --trust-remote-code + Whether you want to execute hub modelling code. Explicitly passing a `revision` is encouraged when loading a model with custom code to ensure no malicious code has been contributed in a newer revision + + [env: TRUST_REMOTE_CODE=] + + --max-concurrent-requests + The maximum amount of concurrent requests for this particular deployment. Having a low limit will refuse clients requests instead of having them wait for too long and is usually good to handle backpressure correctly + + [env: MAX_CONCURRENT_REQUESTS=] + [default: 128] + + --max-best-of + This is the maximum allowed value for clients to set `best_of`. Best of makes `n` generations at the same time, and return the best in terms of overall log probability over the entire generated sequence + + [env: MAX_BEST_OF=] + [default: 2] + + --max-stop-sequences + This is the maximum allowed value for clients to set `stop_sequences`. Stop sequences are used to allow the model to stop on more than just the EOS token, and enable more complex "prompting" where users can preprompt the model in a specific way and define their "own" stop token aligned with their prompt + + [env: MAX_STOP_SEQUENCES=] + [default: 4] + + --max-top-n-tokens + This is the maximum allowed value for clients to set `top_n_tokens`. `top_n_tokens is used to return information about the the `n` most likely tokens at each generation step, instead of just the sampled token. This information can be used for downstream tasks like for classification or ranking + + [env: MAX_TOP_N_TOKENS=] + [default: 5] + + --max-input-length + This is the maximum allowed input length (expressed in number of tokens) for users. The larger this value, the longer prompt users can send which can impact the overall memory required to handle the load. Please note that some models have a finite range of sequence they can handle + + [env: MAX_INPUT_LENGTH=] + [default: 1024] + + --max-total-tokens + This is the most important value to set as it defines the "memory budget" of running clients requests. Clients will send input sequences and ask to generate `max_new_tokens` on top. with a value of `1512` users can send either a prompt of `1000` and ask for `512` new tokens, or send a prompt of `1` and ask for `1511` max_new_tokens. The larger this value, the larger amount each request will be in your RAM and the less effective batching can be + + [env: MAX_TOTAL_TOKENS=] + [default: 2048] + + --waiting-served-ratio + This represents the ratio of waiting queries vs running queries where you want to start considering pausing the running queries to include the waiting ones into the same batch. `waiting_served_ratio=1.2` Means when 12 queries are waiting and there's only 10 queries left in the current batch we check if we can fit those 12 waiting queries into the batching strategy, and if yes, then batching happens delaying the 10 running queries by a `prefill` run. + + This setting is only applied if there is room in the batch as defined by `max_batch_total_tokens`. + + [env: WAITING_SERVED_RATIO=] + [default: 1.2] + + --max-batch-prefill-tokens + Limits the number of tokens for the prefill operation. Since this operation take the most memory and is compute bound, it is interesting to limit the number of requests that can be sent + + [env: MAX_BATCH_PREFILL_TOKENS=] + [default: 4096] + + --max-batch-total-tokens + **IMPORTANT** This is one critical control to allow maximum usage of the available hardware. + + This represents the total amount of potential tokens within a batch. When using padding (not recommended) this would be equivalent of `batch_size` * `max_total_tokens`. + + However in the non-padded (flash attention) version this can be much finer. + + For `max_batch_total_tokens=1000`, you could fit `10` queries of `total_tokens=100` or a single query of `1000` tokens. + + Overall this number should be the largest possible amount that fits the remaining memory (after the model is loaded). Since the actual memory overhead depends on other parameters like if you're using quantization, flash attention or the model implementation, text-generation-inference cannot infer this number automatically. + + [env: MAX_BATCH_TOTAL_TOKENS=] + + --max-waiting-tokens + This setting defines how many tokens can be passed before forcing the waiting queries to be put on the batch (if the size of the batch allows for it). New queries require 1 `prefill` forward, which is different from `decode` and therefore you need to pause the running batch in order to run `prefill` to create the correct values for the waiting queries to be able to join the batch. + + With a value too small, queries will always "steal" the compute to run `prefill` and running queries will be delayed by a lot. + + With a value too big, waiting queries could wait for a very long time before being allowed a slot in the running batch. If your server is busy that means that requests that could run in ~2s on an empty server could end up running in ~20s because the query had to wait for 18s. + + This number is expressed in number of tokens to make it a bit more "model" agnostic, but what should really matter is the overall latency for end users. + + [env: MAX_WAITING_TOKENS=] + [default: 20] + + --hostname + The IP address to listen on + + [env: HOSTNAME=] + [default: 0.0.0.0] + + -p, --port + The port to listen on + + [env: PORT=] + [default: 3000] + + --shard-uds-path + The name of the socket for gRPC communication between the webserver and the shards + + [env: SHARD_UDS_PATH=] + [default: /tmp/text-generation-server] + + --master-addr + The address the master shard will listen on. (setting used by torch distributed) + + [env: MASTER_ADDR=] + [default: localhost] + + --master-port + The address the master port will listen on. (setting used by torch distributed) + + [env: MASTER_PORT=] + [default: 29500] + + --huggingface-hub-cache + The location of the huggingface hub cache. Used to override the location if you want to provide a mounted disk for instance + + [env: HUGGINGFACE_HUB_CACHE=] + + --weights-cache-override + The location of the huggingface hub cache. Used to override the location if you want to provide a mounted disk for instance + + [env: WEIGHTS_CACHE_OVERRIDE=] + + --disable-custom-kernels + For some models (like bloom), text-generation-inference implemented custom cuda kernels to speed up inference. Those kernels were only tested on A100. Use this flag to disable them if you're running on different hardware and encounter issues + + [env: DISABLE_CUSTOM_KERNELS=] + + --cuda-memory-fraction + Limit the CUDA available memory. The allowed value equals the total visible memory multiplied by cuda-memory-fraction + + [env: CUDA_MEMORY_FRACTION=] + [default: 1.0] + + --rope-scaling + Rope scaling will only be used for RoPE models and allow rescaling the position rotary to accomodate for larger prompts. + + Goes together with `rope_factor`. + + `--rope-factor 2.0` gives linear scaling with a factor of 2.0 `--rope-scaling dynamic` gives dynamic scaling with a factor of 1.0 `--rope-scaling linear` gives linear scaling with a factor of 1.0 (Nothing will be changed basically) + + `--rope-scaling linear --rope-factor` fully describes the scaling you want + + [env: ROPE_SCALING=] + [possible values: linear, dynamic] + + --rope-factor + Rope scaling will only be used for RoPE models See `rope_scaling` + + [env: ROPE_FACTOR=] + + --json-output + Outputs the logs in JSON format (useful for telemetry) + + [env: JSON_OUTPUT=] + + --otlp-endpoint + [env: OTLP_ENDPOINT=] + + --cors-allow-origin + [env: CORS_ALLOW_ORIGIN=] + + --watermark-gamma + [env: WATERMARK_GAMMA=] + + --watermark-delta + [env: WATERMARK_DELTA=] + + --ngrok + Enable ngrok tunneling + + [env: NGROK=] + + --ngrok-authtoken + ngrok authentication token + + [env: NGROK_AUTHTOKEN=] + + --ngrok-edge + ngrok edge + + [env: NGROK_EDGE=] + + -e, --env + Display a lot of information about your runtime environment + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +``` \ No newline at end of file diff --git a/update_doc.py b/update_doc.py new file mode 100644 index 00000000..a4c95743 --- /dev/null +++ b/update_doc.py @@ -0,0 +1,31 @@ +import subprocess +import argparse + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + + args = parser.parse_args() + + output = subprocess.check_output(["text-generation-launcher", "--help"]).decode("utf-8") + final_doc = f"# Text-generation-launcher arguments\n```\n{output}\n```" + + filename = "docs/source/basic_tutorials/launcher.md" + if args.check: + with open(filename, "r") as f: + doc = f.read() + if doc != final_doc: + + tmp = "launcher.md" + with open(tmp, "w") as g: + g.write(final_doc) + diff = subprocess.run(["diff",tmp, filename], capture_output=True).stdout.decode("utf-8") + print(diff) + raise Exception("Doc is not up-to-date, run `python update_doc.py` in order to update it") + else: + with open(filename, "w") as f: + f.write(final_doc) + +if __name__ == "__main__": + main() From 3b56d7669ba3b98ae36cbc67904d95c6278e3b05 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 28 Sep 2023 09:55:47 +0200 Subject: [PATCH 437/793] feat: add mistral model (#1071) --- README.md | 1 + clients/python/README.md | 8 + clients/python/poetry.lock | 201 +++++-- clients/python/pyproject.toml | 2 +- clients/python/text_generation/client.py | 1 - clients/python/text_generation/types.py | 8 +- docs/source/basic_tutorials/launcher.md | 11 +- docs/source/supported_models.md | 2 + .../test_flash_mistral.json | 89 +++ .../test_flash_mistral_all_params.json | 89 +++ .../test_flash_mistral_load.json | 358 ++++++++++++ .../models/test_flash_mistral.py | 60 ++ proto/generate.proto | 1 + router/src/infer.rs | 3 +- router/src/queue.rs | 49 +- router/src/server.rs | 1 + server/Makefile-flash-att-v2 | 2 +- server/Makefile-vllm | 2 +- .../text_generation_server/models/__init__.py | 27 +- .../models/cache_manager.py | 135 +++++ .../custom_modeling/flash_mistral_modeling.py | 532 ++++++++++++++++++ .../models/flash_causal_lm.py | 173 ++---- .../models/flash_mistral.py | 357 ++++++++++++ server/text_generation_server/models/model.py | 6 + .../utils/flash_attn.py | 8 + server/text_generation_server/utils/layers.py | 1 + update_doc.py | 14 +- 27 files changed, 1928 insertions(+), 213 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral.json create mode 100644 integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_load.json create mode 100644 integration-tests/models/test_flash_mistral.py create mode 100644 server/text_generation_server/models/cache_manager.py create mode 100644 server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py create mode 100644 server/text_generation_server/models/flash_mistral.py diff --git a/README.md b/README.md index dc074d50..c6db2822 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ to power Hugging Chat, the Inference API and Inference Endpoint. - [MPT](https://huggingface.co/mosaicml/mpt-30b) - [Llama V2](https://huggingface.co/meta-llama) - [Code Llama](https://huggingface.co/codellama) +- [Mistral](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) Other architectures are supported on a best effort basis using: diff --git a/clients/python/README.md b/clients/python/README.md index 4e0e564c..82f3ee0c 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -140,6 +140,8 @@ class Parameters: watermark: bool # Get decoder input token logprobs and ids decoder_input_details: bool + # Return the N most likely tokens at each step + top_n_tokens: Optional[int] # Decoder input tokens class InputToken: @@ -189,6 +191,8 @@ class BestOfSequence: prefill: List[InputToken] # Generated tokens tokens: List[Token] + # Most likely tokens + top_tokens: Optional[List[List[Token]]] # `generate` details @@ -203,6 +207,8 @@ class Details: prefill: List[InputToken] # Generated tokens tokens: List[Token] + # Most likely tokens + top_tokens: Optional[List[List[Token]]] # Additional sequences when using the `best_of` parameter best_of_sequences: Optional[List[BestOfSequence]] @@ -229,6 +235,8 @@ class StreamDetails: class StreamResponse: # Generated token token: Token + # Most likely tokens + top_tokens: Optional[List[Token]] # Complete generated text # Only available when the generation is finished generated_text: Optional[str] diff --git a/clients/python/poetry.lock b/clients/python/poetry.lock index e038ad9b..2d4e45d2 100644 --- a/clients/python/poetry.lock +++ b/clients/python/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -124,6 +124,20 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "annotated-types" +version = "0.5.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.7" +files = [ + {file = "annotated_types-0.5.0-py3-none-any.whl", hash = "sha256:58da39888f92c276ad970249761ebea80ba544b77acddaa1a4d6cf78287d45fd"}, + {file = "annotated_types-0.5.0.tar.gz", hash = "sha256:47cdc3490d9ac1506ce92c7aaa76c579dc3509ff11e098fc867e5130ab7be802"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.9\""} + [[package]] name = "async-timeout" version = "4.0.3" @@ -693,55 +707,140 @@ files = [ [[package]] name = "pydantic" -version = "1.10.12" -description = "Data validation and settings management using python type hints" +version = "2.4.2" +description = "Data validation using Python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, - {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, - {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, - {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, - {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, - {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, - {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, - {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, - {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, - {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, + {file = "pydantic-2.4.2-py3-none-any.whl", hash = "sha256:bc3ddf669d234f4220e6e1c4d96b061abe0998185a8d7855c0126782b7abc8c1"}, + {file = "pydantic-2.4.2.tar.gz", hash = "sha256:94f336138093a5d7f426aac732dcfe7ab4eb4da243c88f891d65deb4a2556ee7"}, ] [package.dependencies] -typing-extensions = ">=4.2.0" +annotated-types = ">=0.4.0" +pydantic-core = "2.10.1" +typing-extensions = ">=4.6.1" [package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.10.1" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic_core-2.10.1-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:d64728ee14e667ba27c66314b7d880b8eeb050e58ffc5fec3b7a109f8cddbd63"}, + {file = "pydantic_core-2.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:48525933fea744a3e7464c19bfede85df4aba79ce90c60b94d8b6e1eddd67096"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef337945bbd76cce390d1b2496ccf9f90b1c1242a3a7bc242ca4a9fc5993427a"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1392e0638af203cee360495fd2cfdd6054711f2db5175b6e9c3c461b76f5175"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0675ba5d22de54d07bccde38997e780044dcfa9a71aac9fd7d4d7a1d2e3e65f7"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:128552af70a64660f21cb0eb4876cbdadf1a1f9d5de820fed6421fa8de07c893"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f6e6aed5818c264412ac0598b581a002a9f050cb2637a84979859e70197aa9e"}, + {file = "pydantic_core-2.10.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ecaac27da855b8d73f92123e5f03612b04c5632fd0a476e469dfc47cd37d6b2e"}, + {file = "pydantic_core-2.10.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b3c01c2fb081fced3bbb3da78510693dc7121bb893a1f0f5f4b48013201f362e"}, + {file = "pydantic_core-2.10.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:92f675fefa977625105708492850bcbc1182bfc3e997f8eecb866d1927c98ae6"}, + {file = "pydantic_core-2.10.1-cp310-none-win32.whl", hash = "sha256:420a692b547736a8d8703c39ea935ab5d8f0d2573f8f123b0a294e49a73f214b"}, + {file = "pydantic_core-2.10.1-cp310-none-win_amd64.whl", hash = "sha256:0880e239827b4b5b3e2ce05e6b766a7414e5f5aedc4523be6b68cfbc7f61c5d0"}, + {file = "pydantic_core-2.10.1-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:073d4a470b195d2b2245d0343569aac7e979d3a0dcce6c7d2af6d8a920ad0bea"}, + {file = "pydantic_core-2.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:600d04a7b342363058b9190d4e929a8e2e715c5682a70cc37d5ded1e0dd370b4"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39215d809470f4c8d1881758575b2abfb80174a9e8daf8f33b1d4379357e417c"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eeb3d3d6b399ffe55f9a04e09e635554012f1980696d6b0aca3e6cf42a17a03b"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a7902bf75779bc12ccfc508bfb7a4c47063f748ea3de87135d433a4cca7a2f"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3625578b6010c65964d177626fde80cf60d7f2e297d56b925cb5cdeda6e9925a"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:caa48fc31fc7243e50188197b5f0c4228956f97b954f76da157aae7f67269ae8"}, + {file = "pydantic_core-2.10.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:07ec6d7d929ae9c68f716195ce15e745b3e8fa122fc67698ac6498d802ed0fa4"}, + {file = "pydantic_core-2.10.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e6f31a17acede6a8cd1ae2d123ce04d8cca74056c9d456075f4f6f85de055607"}, + {file = "pydantic_core-2.10.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d8f1ebca515a03e5654f88411420fea6380fc841d1bea08effb28184e3d4899f"}, + {file = "pydantic_core-2.10.1-cp311-none-win32.whl", hash = "sha256:6db2eb9654a85ada248afa5a6db5ff1cf0f7b16043a6b070adc4a5be68c716d6"}, + {file = "pydantic_core-2.10.1-cp311-none-win_amd64.whl", hash = "sha256:4a5be350f922430997f240d25f8219f93b0c81e15f7b30b868b2fddfc2d05f27"}, + {file = "pydantic_core-2.10.1-cp311-none-win_arm64.whl", hash = "sha256:5fdb39f67c779b183b0c853cd6b45f7db84b84e0571b3ef1c89cdb1dfc367325"}, + {file = "pydantic_core-2.10.1-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:b1f22a9ab44de5f082216270552aa54259db20189e68fc12484873d926426921"}, + {file = "pydantic_core-2.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8572cadbf4cfa95fb4187775b5ade2eaa93511f07947b38f4cd67cf10783b118"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db9a28c063c7c00844ae42a80203eb6d2d6bbb97070cfa00194dff40e6f545ab"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2a35baa428181cb2270a15864ec6286822d3576f2ed0f4cd7f0c1708472aff"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05560ab976012bf40f25d5225a58bfa649bb897b87192a36c6fef1ab132540d7"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d6495008733c7521a89422d7a68efa0a0122c99a5861f06020ef5b1f51f9ba7c"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14ac492c686defc8e6133e3a2d9eaf5261b3df26b8ae97450c1647286750b901"}, + {file = "pydantic_core-2.10.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8282bab177a9a3081fd3d0a0175a07a1e2bfb7fcbbd949519ea0980f8a07144d"}, + {file = "pydantic_core-2.10.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:aafdb89fdeb5fe165043896817eccd6434aee124d5ee9b354f92cd574ba5e78f"}, + {file = "pydantic_core-2.10.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f6defd966ca3b187ec6c366604e9296f585021d922e666b99c47e78738b5666c"}, + {file = "pydantic_core-2.10.1-cp312-none-win32.whl", hash = "sha256:7c4d1894fe112b0864c1fa75dffa045720a194b227bed12f4be7f6045b25209f"}, + {file = "pydantic_core-2.10.1-cp312-none-win_amd64.whl", hash = "sha256:5994985da903d0b8a08e4935c46ed8daf5be1cf217489e673910951dc533d430"}, + {file = "pydantic_core-2.10.1-cp312-none-win_arm64.whl", hash = "sha256:0d8a8adef23d86d8eceed3e32e9cca8879c7481c183f84ed1a8edc7df073af94"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:9badf8d45171d92387410b04639d73811b785b5161ecadabf056ea14d62d4ede"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:ebedb45b9feb7258fac0a268a3f6bec0a2ea4d9558f3d6f813f02ff3a6dc6698"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfe1090245c078720d250d19cb05d67e21a9cd7c257698ef139bc41cf6c27b4f"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e357571bb0efd65fd55f18db0a2fb0ed89d0bb1d41d906b138f088933ae618bb"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b3dcd587b69bbf54fc04ca157c2323b8911033e827fffaecf0cafa5a892a0904"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c120c9ce3b163b985a3b966bb701114beb1da4b0468b9b236fc754783d85aa3"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15d6bca84ffc966cc9976b09a18cf9543ed4d4ecbd97e7086f9ce9327ea48891"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5cabb9710f09d5d2e9e2748c3e3e20d991a4c5f96ed8f1132518f54ab2967221"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:82f55187a5bebae7d81d35b1e9aaea5e169d44819789837cdd4720d768c55d15"}, + {file = "pydantic_core-2.10.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1d40f55222b233e98e3921df7811c27567f0e1a4411b93d4c5c0f4ce131bc42f"}, + {file = "pydantic_core-2.10.1-cp37-none-win32.whl", hash = "sha256:14e09ff0b8fe6e46b93d36a878f6e4a3a98ba5303c76bb8e716f4878a3bee92c"}, + {file = "pydantic_core-2.10.1-cp37-none-win_amd64.whl", hash = "sha256:1396e81b83516b9d5c9e26a924fa69164156c148c717131f54f586485ac3c15e"}, + {file = "pydantic_core-2.10.1-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6835451b57c1b467b95ffb03a38bb75b52fb4dc2762bb1d9dbed8de31ea7d0fc"}, + {file = "pydantic_core-2.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b00bc4619f60c853556b35f83731bd817f989cba3e97dc792bb8c97941b8053a"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa467fd300a6f046bdb248d40cd015b21b7576c168a6bb20aa22e595c8ffcdd"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d99277877daf2efe074eae6338453a4ed54a2d93fb4678ddfe1209a0c93a2468"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa7db7558607afeccb33c0e4bf1c9a9a835e26599e76af6fe2fcea45904083a6"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aad7bd686363d1ce4ee930ad39f14e1673248373f4a9d74d2b9554f06199fb58"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:443fed67d33aa85357464f297e3d26e570267d1af6fef1c21ca50921d2976302"}, + {file = "pydantic_core-2.10.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:042462d8d6ba707fd3ce9649e7bf268633a41018d6a998fb5fbacb7e928a183e"}, + {file = "pydantic_core-2.10.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ecdbde46235f3d560b18be0cb706c8e8ad1b965e5c13bbba7450c86064e96561"}, + {file = "pydantic_core-2.10.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ed550ed05540c03f0e69e6d74ad58d026de61b9eaebebbaaf8873e585cbb18de"}, + {file = "pydantic_core-2.10.1-cp38-none-win32.whl", hash = "sha256:8cdbbd92154db2fec4ec973d45c565e767ddc20aa6dbaf50142676484cbff8ee"}, + {file = "pydantic_core-2.10.1-cp38-none-win_amd64.whl", hash = "sha256:9f6f3e2598604956480f6c8aa24a3384dbf6509fe995d97f6ca6103bb8c2534e"}, + {file = "pydantic_core-2.10.1-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:655f8f4c8d6a5963c9a0687793da37b9b681d9ad06f29438a3b2326d4e6b7970"}, + {file = "pydantic_core-2.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e570ffeb2170e116a5b17e83f19911020ac79d19c96f320cbfa1fa96b470185b"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64322bfa13e44c6c30c518729ef08fda6026b96d5c0be724b3c4ae4da939f875"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:485a91abe3a07c3a8d1e082ba29254eea3e2bb13cbbd4351ea4e5a21912cc9b0"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7c2b8eb9fc872e68b46eeaf835e86bccc3a58ba57d0eedc109cbb14177be531"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5cb87bdc2e5f620693148b5f8f842d293cae46c5f15a1b1bf7ceeed324a740c"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25bd966103890ccfa028841a8f30cebcf5875eeac8c4bde4fe221364c92f0c9a"}, + {file = "pydantic_core-2.10.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f323306d0556351735b54acbf82904fe30a27b6a7147153cbe6e19aaaa2aa429"}, + {file = "pydantic_core-2.10.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0c27f38dc4fbf07b358b2bc90edf35e82d1703e22ff2efa4af4ad5de1b3833e7"}, + {file = "pydantic_core-2.10.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f1365e032a477c1430cfe0cf2856679529a2331426f8081172c4a74186f1d595"}, + {file = "pydantic_core-2.10.1-cp39-none-win32.whl", hash = "sha256:a1c311fd06ab3b10805abb72109f01a134019739bd3286b8ae1bc2fc4e50c07a"}, + {file = "pydantic_core-2.10.1-cp39-none-win_amd64.whl", hash = "sha256:ae8a8843b11dc0b03b57b52793e391f0122e740de3df1474814c700d2622950a"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d43002441932f9a9ea5d6f9efaa2e21458221a3a4b417a14027a1d530201ef1b"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:fcb83175cc4936a5425dde3356f079ae03c0802bbdf8ff82c035f8a54b333521"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:962ed72424bf1f72334e2f1e61b68f16c0e596f024ca7ac5daf229f7c26e4208"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cf5bb4dd67f20f3bbc1209ef572a259027c49e5ff694fa56bed62959b41e1f9"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e544246b859f17373bed915182ab841b80849ed9cf23f1f07b73b7c58baee5fb"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c0877239307b7e69d025b73774e88e86ce82f6ba6adf98f41069d5b0b78bd1bf"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:53df009d1e1ba40f696f8995683e067e3967101d4bb4ea6f667931b7d4a01357"}, + {file = "pydantic_core-2.10.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a1254357f7e4c82e77c348dabf2d55f1d14d19d91ff025004775e70a6ef40ada"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:524ff0ca3baea164d6d93a32c58ac79eca9f6cf713586fdc0adb66a8cdeab96a"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f0ac9fb8608dbc6eaf17956bf623c9119b4db7dbb511650910a82e261e6600f"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:320f14bd4542a04ab23747ff2c8a778bde727158b606e2661349557f0770711e"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:63974d168b6233b4ed6a0046296803cb13c56637a7b8106564ab575926572a55"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:417243bf599ba1f1fef2bb8c543ceb918676954734e2dcb82bf162ae9d7bd514"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dda81e5ec82485155a19d9624cfcca9be88a405e2857354e5b089c2a982144b2"}, + {file = "pydantic_core-2.10.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:14cfbb00959259e15d684505263d5a21732b31248a5dd4941f73a3be233865b9"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:631cb7415225954fdcc2a024119101946793e5923f6c4d73a5914d27eb3d3a05"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:bec7dd208a4182e99c5b6c501ce0b1f49de2802448d4056091f8e630b28e9a52"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:149b8a07712f45b332faee1a2258d8ef1fb4a36f88c0c17cb687f205c5dc6e7d"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d966c47f9dd73c2d32a809d2be529112d509321c5310ebf54076812e6ecd884"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7eb037106f5c6b3b0b864ad226b0b7ab58157124161d48e4b30c4a43fef8bc4b"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:154ea7c52e32dce13065dbb20a4a6f0cc012b4f667ac90d648d36b12007fa9f7"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e562617a45b5a9da5be4abe72b971d4f00bf8555eb29bb91ec2ef2be348cd132"}, + {file = "pydantic_core-2.10.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:f23b55eb5464468f9e0e9a9935ce3ed2a870608d5f534025cd5536bca25b1402"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:e9121b4009339b0f751955baf4543a0bfd6bc3f8188f8056b1a25a2d45099934"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:0523aeb76e03f753b58be33b26540880bac5aa54422e4462404c432230543f33"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e0e2959ef5d5b8dc9ef21e1a305a21a36e254e6a34432d00c72a92fdc5ecda5"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da01bec0a26befab4898ed83b362993c844b9a607a86add78604186297eb047e"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f2e9072d71c1f6cfc79a36d4484c82823c560e6f5599c43c1ca6b5cdbd54f881"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f36a3489d9e28fe4b67be9992a23029c3cec0babc3bd9afb39f49844a8c721c5"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f64f82cc3443149292b32387086d02a6c7fb39b8781563e0ca7b8d7d9cf72bd7"}, + {file = "pydantic_core-2.10.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b4a6db486ac8e99ae696e09efc8b2b9fea67b63c8f88ba7a1a16c24a057a0776"}, + {file = "pydantic_core-2.10.1.tar.gz", hash = "sha256:0f8682dbdd2f67f8e1edddcbffcc29f60a6182b4901c367fc8c1c40d30bb0a82"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pytest" @@ -816,6 +915,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -823,8 +923,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -841,6 +948,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -848,6 +956,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -929,13 +1038,13 @@ files = [ [[package]] name = "urllib3" -version = "2.0.4" +version = "2.0.5" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.7" files = [ - {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, - {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, + {file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"}, + {file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"}, ] [package.extras] @@ -1050,4 +1159,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "0db2f97d52c557dd7f90c55b4ad5bbe308c957c5f7f99fec53c57e0a13822cb4" +content-hash = "b7fab8703967f2616ea59a98a437cd30f97f0c8d2a06e399d688814a2a2c64f8" diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 915ac7aa..4fe6e8b0 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation" -version = "0.6.0" +version = "0.6.1" description = "Hugging Face Text Generation Python Client" license = "Apache-2.0" authors = ["Olivier Dehaene "] diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index ff7f66a3..0bf80f8c 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -482,7 +482,6 @@ async def generate_stream( headers=self.headers, cookies=self.cookies, timeout=self.timeout ) as session: async with session.post(self.base_url, json=request.dict()) as resp: - if resp.status != 200: raise parse_error(resp.status, await resp.json()) diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index 6d6a0536..aa02d8d8 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -40,7 +40,7 @@ class Parameters(BaseModel): # Get decoder input token logprobs and ids decoder_input_details: bool = False # Return the N most likely tokens at each step - top_n_tokens: Optional[int] + top_n_tokens: Optional[int] = None @validator("best_of") def valid_best_of(cls, field_value, values): @@ -188,7 +188,7 @@ class BestOfSequence(BaseModel): # Generated tokens tokens: List[Token] # Most likely tokens - top_tokens: Optional[List[List[Token]]] + top_tokens: Optional[List[List[Token]]] = None # `generate` details @@ -204,7 +204,7 @@ class Details(BaseModel): # Generated tokens tokens: List[Token] # Most likely tokens - top_tokens: Optional[List[List[Token]]] + top_tokens: Optional[List[List[Token]]] = None # Additional sequences when using the `best_of` parameter best_of_sequences: Optional[List[BestOfSequence]] = None @@ -232,7 +232,7 @@ class StreamResponse(BaseModel): # Generated token token: Token # Most likely tokens - top_tokens: Optional[List[Token]] + top_tokens: Optional[List[Token]] = None # Complete generated text # Only available when the generation is finished generated_text: Optional[str] = None diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index c689b550..eb34c1f6 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -34,10 +34,17 @@ Options: [env: NUM_SHARD=] --quantize - Whether you want the model to be quantized. This will use `bitsandbytes` for quantization on the fly, or `gptq`. 4bit quantization is available through `bitsandbytes` by providing the `bitsandbytes-fp4` or `bitsandbytes-nf4` options + Whether you want the model to be quantized [env: QUANTIZE=] - [possible values: bitsandbytes, bitsandbytes-nf4, bitsandbytes-fp4, gptq, awq] + + Possible values: + - awq: 4 bit quantization. Requires a specific GTPQ quantized model: https://hf.co/models?search=awq. Should replace GPTQ models whereever possible because of the better latency + - eetq: 8 bit quantization, doesn't require specific model. Should be a drop-in replacement to bitsandbytes with much better performance. Kernels are from https://github.com/NetEase-FuXi/EETQ.git + - gptq: 4 bit quantization. Requires a specific GTPQ quantized model: https://hf.co/models?search=gptq. text-generation-inference will use exllama (faster) kernels whereever possible, and use triton kernel (wider support) when it's not. AWQ has faster kernels + - bitsandbytes: Bitsandbytes 8bit. Can be applied on any model, will cut the memory requirement in half, but it is known that the model will be much slower to run than the native f16 + - bitsandbytes-nf4: Bitsandbytes 4bit. Can be applied on any model, will cut the memory requirement by 4x, but it is known that the model will be much slower to run than the native f16 + - bitsandbytes-fp4: Bitsandbytes 4bit. nf4 should be preferred in most cases but maybe this one has better perplexity performance for you model --dtype The dtype to be forced upon the model. This option cannot be used with `--quantize` diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md index c5768d9a..5d645759 100644 --- a/docs/source/supported_models.md +++ b/docs/source/supported_models.md @@ -18,6 +18,8 @@ The following models are optimized and can be served with TGI, which uses custom - [Falcon 40B](https://huggingface.co/tiiuae/falcon-40b) - [MPT](https://huggingface.co/mosaicml/mpt-30b) - [Llama V2](https://huggingface.co/meta-llama) +- [Code Llama](https://huggingface.co/codellama) +- [Mistral](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) If the above list lacks the model you would like to serve, depending on the model's pipeline type, you can try to initialize and serve the model anyways to see how well it performs, but performance isn't guaranteed for non-optimized models: diff --git a/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral.json b/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral.json new file mode 100644 index 00000000..4e7de9a6 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral.json @@ -0,0 +1,89 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 3735, + "logprob": -12.9140625, + "text": "Test" + }, + { + "id": 2159, + "logprob": -10.7578125, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 28747, + "logprob": -0.54785156, + "special": false, + "text": ":" + }, + { + "id": 3169, + "logprob": -1.4091797, + "special": false, + "text": " Let" + }, + { + "id": 307, + "logprob": -3.0273438, + "special": false, + "text": " n" + }, + { + "id": 327, + "logprob": -0.94433594, + "special": false, + "text": " =" + }, + { + "id": 28705, + "logprob": -0.81347656, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.2958984, + "special": false, + "text": "1" + }, + { + "id": 28734, + "logprob": -2.0644531, + "special": false, + "text": "0" + }, + { + "id": 387, + "logprob": -1.9580078, + "special": false, + "text": " -" + }, + { + "id": 28705, + "logprob": -0.5073242, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.1816406, + "special": false, + "text": "1" + } + ], + "top_tokens": null + }, + "generated_text": ": Let n = 10 - 1" +} diff --git a/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_all_params.json b/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_all_params.json new file mode 100644 index 00000000..c0dc6471 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_all_params.json @@ -0,0 +1,89 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 3735, + "logprob": -12.9140625, + "text": "Test" + }, + { + "id": 2159, + "logprob": -10.7578125, + "text": "request" + } + ], + "seed": 0, + "tokens": [ + { + "id": 28747, + "logprob": 0.0, + "special": false, + "text": ":" + }, + { + "id": 3169, + "logprob": -0.1307373, + "special": false, + "text": " Let" + }, + { + "id": 332, + "logprob": -2.3359375, + "special": false, + "text": " u" + }, + { + "id": 347, + "logprob": 0.0, + "special": false, + "text": " be" + }, + { + "id": 325, + "logprob": -1.0234375, + "special": false, + "text": " (" + }, + { + "id": 28734, + "logprob": -2.0292969, + "special": false, + "text": "0" + }, + { + "id": 648, + "logprob": -1.0439453, + "special": false, + "text": " +" + }, + { + "id": 28705, + "logprob": -0.24499512, + "special": false, + "text": " " + }, + { + "id": 28770, + "logprob": -0.5073242, + "special": false, + "text": "3" + }, + { + "id": 387, + "logprob": -1.5507812, + "special": false, + "text": " -" + } + ], + "top_tokens": null + }, + "generated_text": "Test request: Let u be (0 + 3 -" +} diff --git a/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_load.json b/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_load.json new file mode 100644 index 00000000..9d133077 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_mistral/test_flash_mistral_load.json @@ -0,0 +1,358 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 3735, + "logprob": -12.9140625, + "text": "Test" + }, + { + "id": 2159, + "logprob": -10.7578125, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 28747, + "logprob": -0.55078125, + "special": false, + "text": ":" + }, + { + "id": 3169, + "logprob": -1.4140625, + "special": false, + "text": " Let" + }, + { + "id": 307, + "logprob": -3.0273438, + "special": false, + "text": " n" + }, + { + "id": 327, + "logprob": -0.94140625, + "special": false, + "text": " =" + }, + { + "id": 28705, + "logprob": -0.8173828, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.2978516, + "special": false, + "text": "1" + }, + { + "id": 28734, + "logprob": -2.0664062, + "special": false, + "text": "0" + }, + { + "id": 387, + "logprob": -1.9560547, + "special": false, + "text": " -" + }, + { + "id": 28705, + "logprob": -0.5078125, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.1787109, + "special": false, + "text": "1" + } + ], + "top_tokens": null + }, + "generated_text": ": Let n = 10 - 1" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 3735, + "logprob": -12.9140625, + "text": "Test" + }, + { + "id": 2159, + "logprob": -10.7578125, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 28747, + "logprob": -0.54785156, + "special": false, + "text": ":" + }, + { + "id": 3169, + "logprob": -1.4111328, + "special": false, + "text": " Let" + }, + { + "id": 307, + "logprob": -3.0292969, + "special": false, + "text": " n" + }, + { + "id": 327, + "logprob": -0.94433594, + "special": false, + "text": " =" + }, + { + "id": 28705, + "logprob": -0.8178711, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.2939453, + "special": false, + "text": "1" + }, + { + "id": 28734, + "logprob": -2.0644531, + "special": false, + "text": "0" + }, + { + "id": 387, + "logprob": -1.9550781, + "special": false, + "text": " -" + }, + { + "id": 28705, + "logprob": -0.5078125, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.1796875, + "special": false, + "text": "1" + } + ], + "top_tokens": null + }, + "generated_text": ": Let n = 10 - 1" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 3735, + "logprob": -12.9140625, + "text": "Test" + }, + { + "id": 2159, + "logprob": -10.7578125, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 28747, + "logprob": -0.55078125, + "special": false, + "text": ":" + }, + { + "id": 3169, + "logprob": -1.4140625, + "special": false, + "text": " Let" + }, + { + "id": 307, + "logprob": -3.0273438, + "special": false, + "text": " n" + }, + { + "id": 327, + "logprob": -0.94140625, + "special": false, + "text": " =" + }, + { + "id": 28705, + "logprob": -0.8173828, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.2978516, + "special": false, + "text": "1" + }, + { + "id": 28734, + "logprob": -2.0664062, + "special": false, + "text": "0" + }, + { + "id": 387, + "logprob": -1.9560547, + "special": false, + "text": " -" + }, + { + "id": 28705, + "logprob": -0.5078125, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.1787109, + "special": false, + "text": "1" + } + ], + "top_tokens": null + }, + "generated_text": ": Let n = 10 - 1" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 3735, + "logprob": -12.9140625, + "text": "Test" + }, + { + "id": 2159, + "logprob": -10.7578125, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 28747, + "logprob": -0.55078125, + "special": false, + "text": ":" + }, + { + "id": 3169, + "logprob": -1.4140625, + "special": false, + "text": " Let" + }, + { + "id": 307, + "logprob": -3.0273438, + "special": false, + "text": " n" + }, + { + "id": 327, + "logprob": -0.94140625, + "special": false, + "text": " =" + }, + { + "id": 28705, + "logprob": -0.8173828, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.2978516, + "special": false, + "text": "1" + }, + { + "id": 28734, + "logprob": -2.0664062, + "special": false, + "text": "0" + }, + { + "id": 387, + "logprob": -1.9560547, + "special": false, + "text": " -" + }, + { + "id": 28705, + "logprob": -0.5078125, + "special": false, + "text": " " + }, + { + "id": 28740, + "logprob": -1.1787109, + "special": false, + "text": "1" + } + ], + "top_tokens": null + }, + "generated_text": ": Let n = 10 - 1" + } +] diff --git a/integration-tests/models/test_flash_mistral.py b/integration-tests/models/test_flash_mistral.py new file mode 100644 index 00000000..63cb09b5 --- /dev/null +++ b/integration-tests/models/test_flash_mistral.py @@ -0,0 +1,60 @@ +import pytest + + +@pytest.fixture(scope="module") +def flash_mistral_handle(launcher): + with launcher("mistralai/Mistral-7B-Instruct-v0.1") as handle: + yield handle + + +@pytest.fixture(scope="module") +async def flash_mistral(flash_mistral_handle): + await flash_mistral_handle.health(300) + return flash_mistral_handle.client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_mistral(flash_mistral, response_snapshot): + response = await flash_mistral.generate( + "Test request", max_new_tokens=10, decoder_input_details=True + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_mistral_all_params(flash_mistral, response_snapshot): + response = await flash_mistral.generate( + "Test request", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + stop_sequences=["test"], + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + decoder_input_details=True, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_mistral_load(flash_mistral, generate_load, response_snapshot): + responses = await generate_load( + flash_mistral, "Test request", max_new_tokens=10, n=4 + ) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) + + assert responses == response_snapshot diff --git a/proto/generate.proto b/proto/generate.proto index 3f607dc5..c873e661 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -31,6 +31,7 @@ message InfoResponse { bool requires_padding = 1; string dtype = 2; string device_type = 3; + optional uint32 window_size = 4; } /// Empty request diff --git a/router/src/infer.rs b/router/src/infer.rs index 67b5bde2..787ccfcf 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -50,10 +50,11 @@ impl Infer { max_waiting_tokens: usize, max_concurrent_requests: usize, requires_padding: bool, + window_size: Option, generation_health: Arc, ) -> Self { // Infer shared state - let queue = Queue::new(requires_padding, 16); + let queue = Queue::new(requires_padding, 16, window_size); let shared = Arc::new(Shared { batching_task: Notify::new(), }); diff --git a/router/src/queue.rs b/router/src/queue.rs index e97a168e..1ab9eb11 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -2,6 +2,7 @@ use crate::infer::InferError; use crate::infer::InferStreamResponse; use crate::validation::ValidGenerateRequest; use nohash_hasher::{BuildNoHashHasher, IntMap}; +use std::cmp::min; use std::collections::VecDeque; use text_generation_client::{Batch, Request}; use tokio::sync::oneshot; @@ -33,12 +34,17 @@ pub(crate) struct Queue { } impl Queue { - pub(crate) fn new(requires_padding: bool, block_size: u32) -> Self { + pub(crate) fn new(requires_padding: bool, block_size: u32, window_size: Option) -> Self { // Create channel let (queue_sender, queue_receiver) = flume::unbounded(); // Launch background queue task - tokio::spawn(queue_task(requires_padding, block_size, queue_receiver)); + tokio::spawn(queue_task( + requires_padding, + block_size, + window_size, + queue_receiver, + )); Self { queue_sender } } @@ -84,9 +90,10 @@ impl Queue { async fn queue_task( requires_padding: bool, block_size: u32, + window_size: Option, receiver: flume::Receiver, ) { - let mut state = State::new(requires_padding, block_size); + let mut state = State::new(requires_padding, block_size, window_size); while let Ok(cmd) = receiver.recv_async().await { match cmd { @@ -126,16 +133,20 @@ struct State { /// Paged Attention block size block_size: u32, + + /// Sliding window + window_size: Option, } impl State { - fn new(requires_padding: bool, block_size: u32) -> Self { + fn new(requires_padding: bool, block_size: u32, window_size: Option) -> Self { Self { entries: VecDeque::with_capacity(128), next_id: 0, next_batch_id: 0, requires_padding, block_size, + window_size, } } @@ -204,11 +215,17 @@ impl State { if self.requires_padding { decode_tokens += entry.request.stopping_parameters.max_new_tokens; } else { + let max_new_tokens = match self.window_size { + None => entry.request.stopping_parameters.max_new_tokens, + Some(window_size) => min( + window_size.saturating_sub(entry.request.input_length), + entry.request.stopping_parameters.max_new_tokens, + ), + }; + // pad to block size decode_tokens += - ((entry.request.stopping_parameters.max_new_tokens + self.block_size - 1) - / self.block_size) - * self.block_size; + ((max_new_tokens + self.block_size - 1) / self.block_size) * self.block_size; } if prefill_tokens > prefill_token_budget @@ -342,7 +359,7 @@ mod tests { #[test] fn test_append() { - let mut state = State::new(false, 1); + let mut state = State::new(false, 1, None); let (entry, _guard) = default_entry(); assert_eq!(state.next_id, 0); @@ -358,7 +375,7 @@ mod tests { #[test] fn test_next_batch_empty() { - let mut state = State::new(false, 1); + let mut state = State::new(false, 1, None); assert!(state.next_batch(None, 1, 1).is_none()); assert!(state.next_batch(Some(1), 1, 1).is_none()); @@ -366,7 +383,7 @@ mod tests { #[test] fn test_next_batch_min_size() { - let mut state = State::new(false, 1); + let mut state = State::new(false, 1, None); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); state.append(entry1); @@ -398,7 +415,7 @@ mod tests { #[test] fn test_next_batch_token_budget() { - let mut state = State::new(false, 1); + let mut state = State::new(false, 1, None); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); state.append(entry1); @@ -431,14 +448,14 @@ mod tests { #[tokio::test] async fn test_queue_append() { - let queue = Queue::new(false, 1); + let queue = Queue::new(false, 1, None); let (entry, _guard) = default_entry(); queue.append(entry); } #[tokio::test] async fn test_queue_next_batch_empty() { - let queue = Queue::new(false, 1); + let queue = Queue::new(false, 1, None); assert!(queue.next_batch(None, 1, 1).await.is_none()); assert!(queue.next_batch(Some(1), 1, 1).await.is_none()); @@ -446,7 +463,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_min_size() { - let queue = Queue::new(false, 1); + let queue = Queue::new(false, 1, None); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); queue.append(entry1); @@ -479,7 +496,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_token_budget() { - let queue = Queue::new(false, 1); + let queue = Queue::new(false, 1, None); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); queue.append(entry1); @@ -504,7 +521,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_dropped_receiver() { - let queue = Queue::new(false, 1); + let queue = Queue::new(false, 1, None); let (entry, _) = default_entry(); queue.append(entry); diff --git a/router/src/server.rs b/router/src/server.rs index fbc444fc..f254afd8 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -595,6 +595,7 @@ pub async fn run( max_waiting_tokens, max_concurrent_requests, shard_info.requires_padding, + shard_info.window_size, generation_health, ); diff --git a/server/Makefile-flash-att-v2 b/server/Makefile-flash-att-v2 index a7d63356..cdea8431 100644 --- a/server/Makefile-flash-att-v2 +++ b/server/Makefile-flash-att-v2 @@ -1,4 +1,4 @@ -flash_att_v2_commit := 4f285b354796fb17df8636485b9a04df3ebbb7dc +flash_att_v2_commit := 601b4dc48dbe9d87c468daa2b4c0c8388b83753c flash-attention-v2: # Clone flash attention diff --git a/server/Makefile-vllm b/server/Makefile-vllm index 96bfc108..2e965da0 100644 --- a/server/Makefile-vllm +++ b/server/Makefile-vllm @@ -1,4 +1,4 @@ -vllm_commit := e86af624d059969b0fb07b075b1d338bf10c3365 +vllm_commit := 25dbff97d5a8f2ba331847237b458b2692e9ae78 vllm: # Clone vllm diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 96dd1ed1..dca3612f 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -67,6 +67,16 @@ __all__.append(FlashLlama) __all__.append(IDEFICSSharded) +MISTRAL = True +try: + from text_generation_server.models.flash_mistral import FlashMistral +except ImportError as e: + logger.warning(f"Could not import Mistral model: {e}") + MISTRAL = False + +if MISTRAL: + __all__.append(FlashMistral) + def get_model( model_id: str, @@ -237,7 +247,18 @@ def get_model( trust_remote_code=trust_remote_code, ) - elif model_type == "opt": + if model_type == "mistral": + if MISTRAL: + return FlashMistral( + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, + ) + raise NotImplementedError("Mistral model requires flash attention v2") + + if model_type == "opt": return OPTSharded( model_id, revision, @@ -246,7 +267,7 @@ def get_model( trust_remote_code=trust_remote_code, ) - elif model_type == "t5": + if model_type == "t5": return T5Sharded( model_id, revision, @@ -254,7 +275,7 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) - elif model_type == "idefics": + if model_type == "idefics": if FLASH_ATTENTION: return IDEFICSSharded( model_id, diff --git a/server/text_generation_server/models/cache_manager.py b/server/text_generation_server/models/cache_manager.py new file mode 100644 index 00000000..2e6ae086 --- /dev/null +++ b/server/text_generation_server/models/cache_manager.py @@ -0,0 +1,135 @@ +import math +import torch + +from typing import Optional, List, Tuple + +BLOCK_SIZE: int = 16 +# Will be set in warmup +CACHE_MANAGER: Optional["CacheManager"] = None + + +class CacheManager: + def __init__( + self, + num_blocks: int, + num_layers: int, + num_heads: int, + head_size: int, + repeat_slots: bool, + dtype: torch.dtype, + device: torch.device, + ): + self.block_size = BLOCK_SIZE + self.num_blocks = num_blocks + self.repeat_slots = repeat_slots + + element_size = torch.tensor([], dtype=dtype).element_size() + x = self.block_size // element_size + + self.kv_cache = [ + ( + torch.empty( + (num_blocks, num_heads, head_size // x, self.block_size, x), + dtype=dtype, + device=device, + ), + torch.empty( + (num_blocks, num_heads, head_size, self.block_size), + dtype=dtype, + device=device, + ), + ) + for _ in range(num_layers) + ] + self.free_block_mask = torch.ones(num_blocks, dtype=torch.int32, device="cpu") + self.slots = torch.arange( + 0, num_blocks * self.block_size, dtype=torch.int32 + ).view(num_blocks, self.block_size) + + def allocate( + self, + needed_blocks_slots: List[Tuple[int, int]], + blocks: int, + max_blocks: int, + device: torch.device, + ): + # Get free blocks indices by finding values in mask that are not set to 0 + free_block_indices = self.free_block_mask.nonzero() + assert ( + len(free_block_indices) >= blocks + ), f"Out of available cache blocks: asked {blocks}, only {len(free_block_indices)} free blocks" + + # Slice by the number of required blocks + block_indices = free_block_indices[:blocks] + block_indices = block_indices.flatten() + + # Padded block tables + block_tables_tensor = torch.zeros( + (len(needed_blocks_slots), max_blocks), dtype=torch.int32 + ) + + # Allocate paged attention blocks + cumulative_blocks = 0 + slots = [] + block_tables = [] + for i, (needed_blocks, needed_slots) in enumerate(needed_blocks_slots): + # Get allocated blocks for this sequence + allocated_blocks = block_indices[ + cumulative_blocks : cumulative_blocks + needed_blocks + ] + # Get slots for the allocated blocks + all_slots = self.slots[allocated_blocks].flatten() + + # Repeat slots in the case of context sliding window + if needed_slots > len(all_slots) and self.repeat_slots: + repeats = math.ceil(needed_slots / len(all_slots)) + all_slots = all_slots.repeat(repeats) + + allocated_slots = all_slots[:needed_slots] + + slots.append(allocated_slots) + block_tables.append(allocated_blocks.tolist()) + block_tables_tensor[i, :needed_blocks] = allocated_blocks + cumulative_blocks += needed_blocks + + block_tables = block_tables + block_tables_tensor = block_tables_tensor.to(device) + slots = torch.concat(slots).to(device) + + # Allocate the required number of blocks by setting the mask to 0 + self.free_block_mask[block_indices] = 0 + + return block_tables, block_tables_tensor, slots + + def free(self, block_indices: Optional[List[int]]): + if block_indices is not None and block_indices: + # Reset mask + self.free_block_mask[block_indices] = 1 + + +def set_cache_manager( + num_blocks: int, + num_layers: int, + num_heads: int, + head_size: int, + repeat_slots: bool, + dtype: torch.dtype, + device: torch.device, +) -> CacheManager: + global CACHE_MANAGER + if CACHE_MANAGER is not None: + del CACHE_MANAGER + torch.cuda.empty_cache() + + CACHE_MANAGER = CacheManager( + num_blocks, num_layers, num_heads, head_size, repeat_slots, dtype, device + ) + return CACHE_MANAGER + + +def get_cache_manager() -> CacheManager: + global CACHE_MANAGER + if CACHE_MANAGER is None: + raise RuntimeError("cache manager was not initialized") + + return CACHE_MANAGER diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py new file mode 100644 index 00000000..77b7f230 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -0,0 +1,532 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.distributed + +from torch import nn +from transformers.activations import ACT2FN +from transformers.configuration_utils import PretrainedConfig +from typing import Optional, List, Tuple + +# Flash attention imports +import dropout_layer_norm + +# vllm imports +import vllm_cache_ops +import vllm_attention_ops + +from text_generation_server.utils.flash_attn import attention, HAS_FLASH_ATTN_V2 +from text_generation_server.utils.layers import ( + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, + PositionRotaryEmbedding, + TensorParallelHead, + get_linear, +) + +if not HAS_FLASH_ATTN_V2: + raise ImportError("Mistral model requires flash attn v2") + + +class MistralConfig(PretrainedConfig): + model_type = "mistral" + + def __init__( + self, + vocab_size=32000, + hidden_size=4096, + intermediate_size=14336, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=8, + hidden_act="silu", + max_position_embeddings=4096 * 32, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000.0, + sliding_window=4096, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.sliding_window = sliding_window + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +class MistralRMSNorm(nn.Module): + def __init__(self, prefix, weights, eps=1e-6): + """ + LlamaRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + + weight = weights.get_tensor(f"{prefix}.weight") + self.weight = nn.Parameter(weight) + self.variance_epsilon = eps + + def forward(self, hidden_states, residual=None): + if hidden_states.shape[-1] > 8192: + if residual is not None: + hidden_states += residual + residual = hidden_states + + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt( + variance + self.variance_epsilon + ) + + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states, residual + else: + # faster post attention rms norm + normed_hidden_states, res, *rest = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.weight, + None, + None, + None, + None, + None, + 0.0, + self.variance_epsilon, + 1.0, + 0, + None, + False, + True, # Activate RMSNorm + ) + if res is None: + res = hidden_states + + return normed_hidden_states, res + + +def load_attention(config, prefix, weights): + if config.num_attention_heads != config.num_key_value_heads: + return _load_gqa(config, prefix, weights) + else: + return TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + dim=0, + weights=weights, + bias=False, + ) + + +def _load_gqa(config, prefix: str, weights): + assert config.hidden_size % config.num_attention_heads == 0 + assert config.num_attention_heads % weights.process_group.size() == 0 + + weight = weights.get_multi_weights_col( + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + quantize=config.quantize, + dim=0, + ) + + if config.quantize not in ["gptq", "awq"]: + weight = weight.to(dtype=weights.dtype).to(device=weights.device) + + head_size = config.hidden_size // config.num_attention_heads + num_heads = config.num_attention_heads // weights.process_group.size() + num_key_value_heads = config.num_key_value_heads // weights.process_group.size() + assert list(weight.shape) == [ + (num_heads + 2 * num_key_value_heads) * head_size, + config.hidden_size, + ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}" + + return TensorParallelColumnLinear( + get_linear(weight, bias=None, quantize=config.quantize) + ) + + +class MistralAttention(torch.nn.Module): + def __init__( + self, + prefix: str, + config, + weights, + ): + super().__init__() + self.max_past = ( + config.sliding_window if config.sliding_window is not None else 0 + ) + self.num_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.num_heads + + self.rotary_emb = PositionRotaryEmbedding.static( + config=config, + dim=self.head_size, + base=config.rope_theta, + device=weights.device, + ) + + self.softmax_scale = self.head_size**-0.5 + + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) + self.num_heads = self.num_heads // weights.process_group.size() + self.num_key_value_heads = ( + config.num_key_value_heads // weights.process_group.size() + ) + + self.query_key_value = load_attention(config, prefix, weights) + + self.o_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.o_proj", + weights=weights, + bias=False, + ) + self.num_groups = self.num_heads // self.num_key_value_heads + self.kv_head_mapping = torch.arange( + 0, self.num_key_value_heads, dtype=torch.int32, device=weights.device + ).repeat_interleave(self.num_groups) + + def forward( + self, + hidden_states, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ): + qkv = self.query_key_value(hidden_states) + query, kv = qkv.split( + [ + self.head_size * self.num_heads, + 2 * self.head_size * self.num_key_value_heads, + ], + dim=1, + ) + query = query.view(-1, self.num_heads, self.head_size) + kv = kv.view(-1, 2, self.num_key_value_heads, self.head_size) + + self.rotary_emb(query, cos, sin) + self.rotary_emb(torch.select(kv, dim=1, index=0), cos, sin) + + if prefill_cache_indices is not None: + kv_to_cache = kv[prefill_cache_indices] + else: + kv_to_cache = kv + + vllm_cache_ops.reshape_and_cache( + kv_to_cache[:, 0], kv_to_cache[:, 1], kv_cache[0], kv_cache[1], slots + ) + + # output tensor + attn_output = torch.empty_like(query) + + # Prefill + if cu_seqlen_prefill is not None: + # flash attention + attention( + query, + torch.select(kv, dim=1, index=0), + torch.select(kv, dim=1, index=1), + attn_output, + cu_seqlen_prefill, + max_s, + self.softmax_scale, + window_size_left=self.max_past, + ) + # Decode + else: + # kv_cache[1] => [num_blocks, num_heads, head_size, block_size] + block_size = kv_cache[1].shape[3] + vllm_attention_ops.single_query_cached_kv_attention( + attn_output, + query, + kv_cache[0], + kv_cache[1], + self.kv_head_mapping, + self.softmax_scale, + block_tables, + input_lengths, + block_size, + max_s, + ) + + return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size)) + + +class MistralMLP(nn.Module): + def __init__(self, prefix, config, weights): + super().__init__() + act = config.hidden_act + self.act = ( + ACT2FN[act] + if "gelu" not in act + else lambda x: torch.nn.functional.gelu( + x, + approximate="tanh" + if act in ["gelu_fast", "gelu_pytorch_tanh"] + else "none", + ) + ) + # Fuse gate and up proj + self.gate_up_proj = TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.gate_proj", f"{prefix}.up_proj"], + weights=weights, + dim=0, + bias=False, + ) + self.down_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.down_proj", + weights=weights, + bias=False, + ) + self.intermediate_size = ( + config.intermediate_size // weights.process_group.size() + ) + + def forward(self, hidden_states): + gate_up_states = self.gate_up_proj(hidden_states) + gate_up_states = gate_up_states.view(-1, 2, self.intermediate_size) + return self.down_proj(self.act(gate_up_states[:, 0]) * gate_up_states[:, 1]) + + +class MistralLayer(nn.Module): + def __init__(self, layer_id, config, weights): + super().__init__() + prefix = f"model.layers.{layer_id}" + self.self_attn = MistralAttention( + prefix=f"{prefix}.self_attn", config=config, weights=weights + ) + self.mlp = MistralMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) + + self.input_layernorm = MistralRMSNorm( + prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps + ) + self.post_attention_layernorm = MistralRMSNorm( + prefix=f"{prefix}.post_attention_layernorm", + weights=weights, + eps=config.rms_norm_eps, + ) + + def forward( + self, + hidden_states, + residual, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ): + normed_hidden_states, res = self.input_layernorm(hidden_states, residual) + + # Self Attention + attn_output = self.self_attn( + normed_hidden_states, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ) + + # faster post attention rms norm + normed_attn_res_output, attn_res = self.post_attention_layernorm( + attn_output, res + ) + + mlp_output = self.mlp(normed_attn_res_output) + + return mlp_output, attn_res + + +class MistralModel(torch.nn.Module): + def __init__(self, config, weights): + super().__init__() + + process_group = weights.process_group + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + self.embed_tokens = TensorParallelEmbedding( + prefix="model.embed_tokens", weights=weights + ) + self.layers = nn.ModuleList( + [ + MistralLayer( + layer_id, + config, + weights, + ) + for layer_id in range(config.num_hidden_layers) + ] + ) + self.norm = MistralRMSNorm( + prefix="model.norm", weights=weights, eps=config.rms_norm_eps + ) + + self.gradient_checkpointing = False + + self.head_size = self.layers[0].self_attn.head_size + self.num_heads = self.layers[0].self_attn.num_heads + self.num_key_value_heads = self.layers[0].self_attn.num_key_value_heads + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlen_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + prefill_cache_indices: Optional[torch.Tensor], + ) -> torch.Tensor: + hidden_states = self.embed_tokens(input_ids) + + # Get rotary cos and sin for this forward + # Avoid to index in each layer + cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin( + position_ids, max_s, hidden_states.dtype + ) + + residual = None + for i, layer in enumerate(self.layers): + hidden_states, residual = layer( + hidden_states, + residual, + cos, + sin, + cu_seqlen_prefill, + kv_cache[i], + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ) + + hidden_states, _ = self.norm(hidden_states, residual) + + return hidden_states + + +class FlashMistralForCausalLM(torch.nn.Module): + def __init__(self, config, weights): + super().__init__() + + self.model = MistralModel(config, weights) + self.lm_head = TensorParallelHead.load( + config, + prefix="lm_head", + weights=weights, + ) + self.max_past = config.sliding_window + if self.max_past is None: + raise ValueError("max_past cannot be None") + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlen_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + prefill_cache_indices: Optional[torch.Tensor], + lm_head_indices: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if prefill_cache_indices is not None: + # Slots also need to be sliced as it has the same size as the whole kv tensor + slots = slots[prefill_cache_indices] + else: + # Clamp in decode mode as paged attention requires clamped values whereas the flash attention + # kernel requires the true values + max_s = min(self.max_past, max_s) + input_lengths = torch.clamp(input_lengths, max=self.max_past) + + hidden_states = self.model( + input_ids, + position_ids, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ) + if lm_head_indices is not None: + hidden_states = hidden_states[lm_head_indices] + logits = self.lm_head(hidden_states) + return logits diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 34c7f633..1fe40c0c 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -19,99 +19,17 @@ GeneratedText, TopTokens, ) +from text_generation_server.models.cache_manager import ( + get_cache_manager, + set_cache_manager, + BLOCK_SIZE, +) from text_generation_server.pb import generate_pb2 from text_generation_server.utils import StoppingCriteria, HeterogeneousNextTokenChooser from text_generation_server.utils.dist import MEMORY_FRACTION tracer = trace.get_tracer(__name__) -BLOCK_SIZE = 16 -# Will be set in warmup -CACHE_MANAGER: Optional["CacheManager"] = None - - -class CacheManager: - def __init__( - self, - num_blocks: int, - num_layers: int, - num_heads: int, - head_size: int, - dtype: torch.dtype, - device: torch.device, - ): - self.block_size = BLOCK_SIZE - self.num_blocks = num_blocks - - element_size = torch.tensor([], dtype=dtype).element_size() - x = self.block_size // element_size - - self.kv_cache = [ - ( - torch.empty( - (num_blocks, num_heads, head_size // x, self.block_size, x), - dtype=dtype, - device=device, - ), - torch.empty( - (num_blocks, num_heads, head_size, self.block_size), - dtype=dtype, - device=device, - ), - ) - for _ in range(num_layers) - ] - self.free_block_mask = torch.ones(num_blocks, dtype=torch.int32, device="cpu") - self.slots = torch.arange( - 0, num_blocks * self.block_size, dtype=torch.int32 - ).view(num_blocks, self.block_size) - - def allocate(self, batch: "FlashCausalLMBatch"): - # Get free blocks indices by finding values in mask that are not set to 0 - free_block_indices = self.free_block_mask.nonzero() - assert ( - len(free_block_indices) >= batch.blocks - ), f"Out of available cache blocks: asked {batch.blocks}, only {len(free_block_indices)} free blocks" - - # Slice by the number of required blocks - block_indices = free_block_indices[: batch.blocks] - block_indices = block_indices.flatten() - - # Padded block tables - block_tables_tensor = torch.zeros( - (len(batch), batch.max_blocks), dtype=torch.int32 - ) - - # Allocate paged attention blocks - cumulative_blocks = 0 - slots = [] - block_tables = [] - for i, (needed_blocks, needed_slots) in enumerate(batch.needed_blocks_slots): - # Get allocated blocks for this sequence - allocated_blocks = block_indices[ - cumulative_blocks : cumulative_blocks + needed_blocks - ] - # Get slots for the allocated blocks - allocated_slots = self.slots[allocated_blocks].flatten()[:needed_slots] - - slots.append(allocated_slots) - block_tables.append(allocated_blocks.tolist()) - block_tables_tensor[i, :needed_blocks] = allocated_blocks - cumulative_blocks += needed_blocks - - batch.needed_blocks_slots = None - batch.block_tables = block_tables - batch.block_tables_tensor = block_tables_tensor.to(batch.input_ids.device) - batch.slots = torch.concat(slots).to(batch.input_ids.device) - - # Allocate the required number of blocks by setting the mask to 0 - self.free_block_mask[block_indices] = 0 - - def free(self, block_indices: Optional[List[int]]): - if block_indices is not None and block_indices: - # Reset mask - self.free_block_mask[block_indices] = 1 - @dataclass class FlashCausalLMBatch(Batch): @@ -481,7 +399,6 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": max_blocks = max(max_blocks, len(request_block_table)) - global CACHE_MANAGER block_indices_to_free = [] # Iterate on all requests for i, r in enumerate(self.requests): @@ -489,7 +406,7 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": if r.id not in requests_idx_mapping.keys(): block_indices_to_free.extend(self.block_tables[i]) # Free blocks - CACHE_MANAGER.free(block_indices_to_free) + get_cache_manager().free(block_indices_to_free) # Needed to avoid dropping blocks when the batches will go out of scope self.block_tables = None @@ -508,7 +425,7 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": # Move to GPU now that we have the whole tensor slot_indices = slot_indices.to(device) - return FlashCausalLMBatch( + return type(self)( batch_id=self.batch_id, requests=requests, requests_idx_mapping=requests_idx_mapping, @@ -665,7 +582,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch b.block_tables = None del b - return FlashCausalLMBatch( + return cls( batch_id=batches[0].batch_id, requests=requests, requests_idx_mapping=requests_idx_mapping, @@ -698,9 +615,10 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch def __del__(self): if self.block_tables is not None and self.block_tables: - global CACHE_MANAGER # Free blocks - CACHE_MANAGER.free(list(itertools.chain.from_iterable(self.block_tables))) + get_cache_manager().free( + list(itertools.chain.from_iterable(self.block_tables)) + ) def __len__(self): return len(self.requests) @@ -718,6 +636,7 @@ def __init__( device: torch.device, rank: int = 0, world_size: int = 1, + sliding_window: Optional[int] = None, ): self.num_layers = num_layers self.num_kv_heads = num_kv_heads @@ -731,6 +650,7 @@ def __init__( device=device, rank=rank, world_size=world_size, + sliding_window=sliding_window, ) @property @@ -738,15 +658,14 @@ def batch_type(self) -> Type[FlashCausalLMBatch]: return FlashCausalLMBatch def warmup(self, batch: FlashCausalLMBatch): - global CACHE_MANAGER - torch.cuda.empty_cache() try: - CACHE_MANAGER = CacheManager( + cache_manager = set_cache_manager( batch.blocks, self.num_layers, self.num_kv_heads, self.head_size, + self.sliding_window is not None, self.dtype, self.device, ) @@ -775,48 +694,36 @@ def warmup(self, batch: FlashCausalLMBatch): num_blocks = ( int(free_memory // total_cache_size) # Add batch.blocks as we allocated it above, so it is included in the peak memory. - + CACHE_MANAGER.num_blocks + + cache_manager.num_blocks ) - del CACHE_MANAGER del batch - torch.cuda.empty_cache() + del cache_manager - CACHE_MANAGER = CacheManager( + set_cache_manager( num_blocks, self.num_layers, self.num_kv_heads, self.head_size, + self.sliding_window is not None, self.dtype, self.device, ) return int(num_blocks * BLOCK_SIZE) - def forward( - self, - input_ids: torch.Tensor, - position_ids: torch.Tensor, - cu_seqlen_prefill: Optional[torch.Tensor], - block_tables: torch.Tensor, - slots: torch.Tensor, - input_lengths: torch.Tensor, - max_s: int, - lm_head_indices: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - global CACHE_MANAGER - + def forward(self, batch: FlashCausalLMBatch) -> Tuple[torch.Tensor, torch.Tensor]: # Model Forward return self.model.forward( - input_ids=input_ids, - position_ids=position_ids, - cu_seqlen_prefill=cu_seqlen_prefill, - kv_cache=CACHE_MANAGER.kv_cache, - block_tables=block_tables, - slots=slots, - input_lengths=input_lengths, - max_s=max_s, - lm_head_indices=lm_head_indices, + input_ids=batch.input_ids, + position_ids=batch.position_ids, + cu_seqlen_prefill=batch.cu_seqlen_prefill, + kv_cache=get_cache_manager().kv_cache, + block_tables=batch.block_tables_tensor, + slots=batch.slots[batch.slot_indices], + input_lengths=batch.input_lengths_tensor, + max_s=batch.max_seqlen, + lm_head_indices=batch.prefill_head_indices, ) @tracer.start_as_current_span("generate_token") @@ -828,19 +735,19 @@ def generate_token( if batch.needed_blocks_slots: # Allocate blocks to this batch - CACHE_MANAGER.allocate(batch) + block_tables, block_tables_tensor, slots = get_cache_manager().allocate( + batch.needed_blocks_slots, + batch.blocks, + batch.max_blocks, + batch.input_ids.device, + ) + batch.needed_blocks_slots = None + batch.block_tables = block_tables + batch.block_tables_tensor = block_tables_tensor + batch.slots = slots try: - out = self.forward( - batch.input_ids, - batch.position_ids, - batch.cu_seqlen_prefill, - batch.block_tables_tensor, - batch.slots[batch.slot_indices], - batch.input_lengths_tensor, - batch.max_seqlen, - batch.prefill_head_indices, - ) + out = self.forward(batch) except Exception as e: del batch raise e diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py new file mode 100644 index 00000000..919e4625 --- /dev/null +++ b/server/text_generation_server/models/flash_mistral.py @@ -0,0 +1,357 @@ +import math +import torch +import torch.distributed + +import numpy as np + +from dataclasses import dataclass +from opentelemetry import trace +from transformers import PreTrainedTokenizerBase +from transformers.models.llama import LlamaTokenizerFast +from typing import Optional, Tuple, Type + +from text_generation_server.pb import generate_pb2 +from text_generation_server.models import FlashCausalLM +from text_generation_server.models.flash_causal_lm import FlashCausalLMBatch, BLOCK_SIZE +from text_generation_server.models.cache_manager import ( + get_cache_manager, + set_cache_manager, +) +from text_generation_server.models.custom_modeling.flash_mistral_modeling import ( + FlashMistralForCausalLM, + MistralConfig, +) +from text_generation_server.utils import ( + initialize_torch_distributed, + weight_files, + Weights, + HeterogeneousNextTokenChooser, + StoppingCriteria, +) + +tracer = trace.get_tracer(__name__) + +# Will be set in init +SLIDING_WINDOW: Optional[int] = None +SLIDING_WINDOW_BLOCKS: Optional[int] = None + + +# Adds windowing logic to FlashCausalLMBatch +@dataclass +class FlashMistralBatch(FlashCausalLMBatch): + # Prefill cache indices is used to slice into the kv tensor before caching it into the paged attention buffers + # as we only keep SLIDING_WINDOW values instead of the whole tensor + prefill_cache_indices: Optional[torch.Tensor] = None + + @classmethod + def from_pb( + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, + device: torch.device, + ) -> "FlashCausalLMBatch": + global SLIDING_WINDOW + global SLIDING_WINDOW_BLOCKS + + batch_inputs = [] + max_truncation = 0 + for r in pb.requests: + batch_inputs.append(r.inputs) + max_truncation = max(max_truncation, r.truncate) + + batch_tokenized_inputs = tokenizer( + batch_inputs, truncation=True, max_length=max_truncation + )["input_ids"] + + position_ids = [] + cu_seqlen_prefill = [0] + needed_blocks_slots = [] + start_slots = [] + slot_indices = [] + prefill_cache_indices = [] + + input_lengths = [] + prefix_offsets = [] + read_offsets = [] + all_input_ids = [] + requests_idx_mapping = {} + + all_prefill_logprobs = True + no_prefill_logprobs = True + prefill_head_indices = [] + prefill_next_token_indices = [] + prefill_cu_outlens = [0] + + next_token_chooser_parameters = [] + stopping_criterias = [] + top_n_tokens = [] + + # Cumulative length + cumulative_length = 0 + cumulative_max_length = 0 + prefill_out_cumulative_length = 0 + + blocks = 0 + max_seqlen = 0 + max_length = 0 + max_blocks = 0 + + # Parse batch + for i, (r, tokenized_input) in enumerate( + zip(pb.requests, batch_tokenized_inputs) + ): + # request id -> idx in list mapping + requests_idx_mapping[r.id] = i + + tokenized_input = tokenized_input[-r.truncate :] + + input_length = len(tokenized_input) + input_lengths.append(input_length) + + prefix_offsets.append(input_length - 5) + read_offsets.append(input_length) + + all_input_ids.append(tokenized_input) + + # Position ids + request_position_ids = torch.arange(0, input_length, dtype=torch.int32) + position_ids.append(request_position_ids) + + # Add cumulative lengths of all previous inputs + cu_seqlen_prefill.append(cumulative_length + input_length) + + next_token_chooser_parameters.append(r.parameters) + + stopping_criteria = StoppingCriteria.from_pb( + r.stopping_parameters, tokenizer + ) + max_new_tokens = stopping_criteria.max_new_tokens + stopping_criterias.append(stopping_criteria) + top_n_tokens.append(r.top_n_tokens) + + # Paged attention + # Remove one as the first token des not have a past + total_tokens = input_length + max_new_tokens - 1 + + # Needed blocks can not go over SLIDING_WINDOW_BLOCKS + needed_blocks = min( + math.ceil(total_tokens / BLOCK_SIZE), SLIDING_WINDOW_BLOCKS + ) + blocks += needed_blocks + + needed_blocks_slots.append((needed_blocks, total_tokens)) + start_slots.append(cumulative_max_length) + + request_slot_indices = torch.arange( + cumulative_max_length, + cumulative_max_length + input_length, + dtype=torch.int64, + ) + slot_indices.append(request_slot_indices) + + # Create tensor to slice into the kv tensor in prefill + request_prefill_cache_indices = torch.arange( + cumulative_length + max(0, input_length - SLIDING_WINDOW), + cumulative_length + input_length, + dtype=torch.int64, + ) + prefill_cache_indices.append(request_prefill_cache_indices) + + all_prefill_logprobs = all_prefill_logprobs and r.prefill_logprobs + no_prefill_logprobs = no_prefill_logprobs and not r.prefill_logprobs + + if r.prefill_logprobs: + prefill_head_indices.append(request_position_ids + cumulative_length) + prefill_next_token_indices.append( + prefill_out_cumulative_length + input_length - 1 + ) + prefill_cu_outlens.append(prefill_out_cumulative_length + input_length) + prefill_out_cumulative_length += input_length + else: + prefill_head_indices.append( + torch.tensor( + [cumulative_length + input_length - 1], dtype=torch.int32 + ) + ) + prefill_next_token_indices.append(prefill_out_cumulative_length) + prefill_cu_outlens.append(prefill_out_cumulative_length + 1) + prefill_out_cumulative_length += 1 + + # Update + cumulative_length += input_length + cumulative_max_length += total_tokens + max_seqlen = max(max_seqlen, input_length) + max_blocks = max(max_blocks, needed_blocks) + max_length = max(max_length, input_length + max_new_tokens) + + next_token_chooser = HeterogeneousNextTokenChooser.from_pb( + next_token_chooser_parameters, dtype, device + ) + start_slots = torch.tensor(start_slots, dtype=torch.int64) + + # Padded all_input_ids_tensor + all_input_ids_tensor = np.zeros( + (len(all_input_ids), max_length), dtype=np.int64 + ) + for i, input_ids in enumerate(all_input_ids): + all_input_ids_tensor[i, : len(input_ids)] = input_ids + + # Create tensors on device + all_input_ids_tensor = torch.tensor( + all_input_ids_tensor, dtype=torch.int64, device=device + ) + + if len(pb.requests) > 1: + input_ids = np.concatenate(all_input_ids, dtype=np.int64) + position_ids = torch.cat(position_ids) + slot_indices = torch.cat(slot_indices) + prefill_cache_indices = torch.cat(prefill_cache_indices) + else: + input_ids = all_input_ids[0] + position_ids = position_ids[0] + slot_indices = slot_indices[0] + prefill_cache_indices = prefill_cache_indices[0] + + cu_seqlen_prefill = torch.tensor( + cu_seqlen_prefill, device=device, dtype=torch.int32 + ) + + position_ids = position_ids.to(device) + slot_indices = slot_indices.to(device) + prefill_cache_indices = prefill_cache_indices.to(device) + input_ids = torch.tensor(input_ids, dtype=torch.int64, device=device) + input_lengths_tensor = torch.tensor( + input_lengths, dtype=torch.int32, device=device + ) + + if all_prefill_logprobs: + prefill_head_indices = None + prefill_next_token_indices = cu_seqlen_prefill[1:] - 1 + elif no_prefill_logprobs: + prefill_head_indices = cu_seqlen_prefill[1:] - 1 + prefill_next_token_indices = None + else: + prefill_head_indices = torch.tensor( + torch.cat(prefill_head_indices), dtype=torch.int64, device=device + ) + prefill_next_token_indices = torch.tensor( + prefill_next_token_indices, dtype=torch.int64, device=device + ) + top_n_tokens_tensor = torch.tensor( + top_n_tokens, device=device, dtype=torch.int64 + ) + + return cls( + batch_id=pb.id, + requests=pb.requests, + requests_idx_mapping=requests_idx_mapping, + input_ids=input_ids, + position_ids=position_ids, + cu_seqlen_prefill=cu_seqlen_prefill, + start_slots=start_slots, + slot_indices=slot_indices, + needed_blocks_slots=needed_blocks_slots, + block_tables=None, + block_tables_tensor=None, + slots=None, + max_seqlen=max_seqlen, + prefill_head_indices=prefill_head_indices, + prefill_next_token_indices=prefill_next_token_indices, + prefill_cu_outlens=prefill_cu_outlens, + input_lengths=input_lengths, + input_lengths_tensor=input_lengths_tensor, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, + all_input_ids=all_input_ids, + all_input_ids_tensor=all_input_ids_tensor, + next_token_chooser=next_token_chooser, + stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, + blocks=blocks, + max_blocks=max_blocks, + prefill_cache_indices=prefill_cache_indices, + ) + + +class FlashMistral(FlashCausalLM): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, + ): + global SLIDING_WINDOW + global SLIDING_WINDOW_BLOCKS + + self.process_group, rank, world_size = initialize_torch_distributed() + if torch.cuda.is_available(): + device = torch.device(f"cuda:{rank}") + dtype = torch.float16 if dtype is None else dtype + else: + raise NotImplementedError("FlashLlama is only available on GPU") + + tokenizer = LlamaTokenizerFast.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + + config = MistralConfig.from_pretrained( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) + config.quantize = quantize + + # Set context windows + SLIDING_WINDOW = config.sliding_window + SLIDING_WINDOW_BLOCKS = math.ceil(config.sliding_window / BLOCK_SIZE) + + torch.distributed.barrier(group=self.process_group) + + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights(filenames, device, dtype, process_group=self.process_group) + if config.quantize in ["gptq", "awq"]: + weights._set_gptq_params(model_id) + + model = FlashMistralForCausalLM(config, weights) + + torch.distributed.barrier(group=self.process_group) + super(FlashMistral, self).__init__( + model=model, + tokenizer=tokenizer, + num_layers=len(model.model.layers), + num_kv_heads=model.model.num_key_value_heads, + head_size=model.model.head_size, + dtype=dtype, + device=device, + rank=rank, + world_size=world_size, + sliding_window=config.sliding_window, + ) + + @property + def batch_type(self) -> Type[FlashMistralBatch]: + return FlashMistralBatch + + def forward(self, batch: FlashMistralBatch) -> Tuple[torch.Tensor, torch.Tensor]: + # Model Forward + logits = self.model.forward( + input_ids=batch.input_ids, + position_ids=batch.position_ids, + cu_seqlen_prefill=batch.cu_seqlen_prefill, + kv_cache=get_cache_manager().kv_cache, + block_tables=batch.block_tables_tensor, + slots=batch.slots[batch.slot_indices], + input_lengths=batch.input_lengths_tensor, + max_s=batch.max_seqlen, + prefill_cache_indices=batch.prefill_cache_indices, + lm_head_indices=batch.prefill_head_indices, + ) + if batch.prefill_cache_indices is not None: + batch.prefill_cache_indices = None + return logits diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index f6e66d30..17d2ea9b 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -21,6 +21,7 @@ def __init__( device: torch.device, rank: int = 0, world_size: int = 1, + sliding_window: Optional[int] = None, ): self.model = model.eval() self.tokenizer = tokenizer @@ -30,6 +31,7 @@ def __init__( self.device = device self.rank = rank self.world_size = world_size + self.sliding_window = sliding_window self.has_position_ids = ( inspect.signature(model.forward).parameters.get("position_ids", None) @@ -40,10 +42,14 @@ def __init__( @property def info(self) -> InfoResponse: + if self.requires_padding and self.sliding_window is not None: + raise NotImplementedError("sliding_window is not implemented with padding") + return InfoResponse( requires_padding=self.requires_padding, dtype=str(self.dtype), device_type=self.device.type, + window_size=self.sliding_window, ) @property diff --git a/server/text_generation_server/utils/flash_attn.py b/server/text_generation_server/utils/flash_attn.py index c472d1fc..caf072b7 100644 --- a/server/text_generation_server/utils/flash_attn.py +++ b/server/text_generation_server/utils/flash_attn.py @@ -57,6 +57,7 @@ def attention( cu_seqlens, max_s, softmax_scale, + window_size_left=-1, ): if HAS_FLASH_ATTN_V2: return flash_attn_2_cuda.varlen_fwd( @@ -72,11 +73,18 @@ def attention( softmax_scale, False, True, + window_size_left, + 0, False, None, ) if HAS_FLASH_ATTN: + if window_size_left != 0: + raise NotImplementedError( + "window_size_left is only available with flash attn v2" + ) + # Flash attention v1 requires q, k and v to have the same number of heads if k.shape[1] != q.shape[1]: # MQA expand diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 8be2463f..cf61e47b 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -53,6 +53,7 @@ except ImportError: pass + # Monkey patching @classmethod def load_layer_norm(cls, prefix, weights, eps): diff --git a/update_doc.py b/update_doc.py index a4c95743..7e8fb769 100644 --- a/update_doc.py +++ b/update_doc.py @@ -8,7 +8,9 @@ def main(): args = parser.parse_args() - output = subprocess.check_output(["text-generation-launcher", "--help"]).decode("utf-8") + output = subprocess.check_output(["text-generation-launcher", "--help"]).decode( + "utf-8" + ) final_doc = f"# Text-generation-launcher arguments\n```\n{output}\n```" filename = "docs/source/basic_tutorials/launcher.md" @@ -16,16 +18,20 @@ def main(): with open(filename, "r") as f: doc = f.read() if doc != final_doc: - tmp = "launcher.md" with open(tmp, "w") as g: g.write(final_doc) - diff = subprocess.run(["diff",tmp, filename], capture_output=True).stdout.decode("utf-8") + diff = subprocess.run( + ["diff", tmp, filename], capture_output=True + ).stdout.decode("utf-8") print(diff) - raise Exception("Doc is not up-to-date, run `python update_doc.py` in order to update it") + raise Exception( + "Doc is not up-to-date, run `python update_doc.py` in order to update it" + ) else: with open(filename, "w") as f: f.write(final_doc) + if __name__ == "__main__": main() From 7a6fad6aac67d9bf21fe75c034a6bcab5dbd88d2 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 28 Sep 2023 10:18:18 +0200 Subject: [PATCH 438/793] update readme --- README.md | 2 +- docs/source/quicktour.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c6db2822..339b5db7 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= -docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.0.3 --model-id $model +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0 --model-id $model ``` ### A note on Shared Memory (shm) diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index 083dffa5..0a874b57 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -85,7 +85,7 @@ curl 127.0.0.1:8080/generate \ To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. ```bash -docker run ghcr.io/huggingface/text-generation-inference:1.0.3 --help +docker run ghcr.io/huggingface/text-generation-inference:1.1.0 --help ``` From a7808ff8538645e2d29c3dceb973008a1b47640c Mon Sep 17 00:00:00 2001 From: Mishig Date: Thu, 28 Sep 2023 15:37:50 +0200 Subject: [PATCH 439/793] Fix launcher.md (#1075) Adding a new line to escape between heading and codeblock. However, it is a hotfix and I will work on a permanent solution on https://github.com/huggingface/doc-builder --- docs/source/basic_tutorials/launcher.md | 1 + update_doc.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index eb34c1f6..b3498247 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -1,4 +1,5 @@ # Text-generation-launcher arguments + ``` Text Generation Launcher diff --git a/update_doc.py b/update_doc.py index 7e8fb769..3d68fc7f 100644 --- a/update_doc.py +++ b/update_doc.py @@ -11,7 +11,7 @@ def main(): output = subprocess.check_output(["text-generation-launcher", "--help"]).decode( "utf-8" ) - final_doc = f"# Text-generation-launcher arguments\n```\n{output}\n```" + final_doc = f"# Text-generation-launcher arguments\n\n```\n{output}\n```" filename = "docs/source/basic_tutorials/launcher.md" if args.check: From 724199aaf172590c3658018c0e6bc6152cda4c2f Mon Sep 17 00:00:00 2001 From: Mishig Date: Thu, 28 Sep 2023 17:30:36 +0200 Subject: [PATCH 440/793] Update launcher.md to wrap code blocks (#1076) Wrap code blocks in `launcher` doc page using https://github.com/huggingface/doc-builder/pull/420 https://moon-ci-docs.huggingface.co/docs/text-generation-inference/pr_1076/en/basic_tutorials/launcher image --- docs/source/basic_tutorials/launcher.md | 2 ++ update_doc.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index b3498247..bdb8cb73 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -1,5 +1,7 @@ # Text-generation-launcher arguments + + ``` Text Generation Launcher diff --git a/update_doc.py b/update_doc.py index 3d68fc7f..81e6a94e 100644 --- a/update_doc.py +++ b/update_doc.py @@ -11,7 +11,8 @@ def main(): output = subprocess.check_output(["text-generation-launcher", "--help"]).decode( "utf-8" ) - final_doc = f"# Text-generation-launcher arguments\n\n```\n{output}\n```" + wrap_code_blocks_flag = "" + final_doc = f"# Text-generation-launcher arguments\n\n{wrap_code_blocks_flag}\n\n```\n{output}\n```" filename = "docs/source/basic_tutorials/launcher.md" if args.check: From 5ba53d44a18983a4de32d122f4cb46f4a17d9ef6 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 29 Sep 2023 11:19:06 +0200 Subject: [PATCH 441/793] Fixing eetq dockerfile. (#1081) # What does this PR do? Fixes #1079 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- Dockerfile | 9 +++++++++ server/text_generation_server/models/__init__.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/Dockerfile b/Dockerfile index 56f4775b..9c15f023 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,6 +123,13 @@ COPY server/Makefile-awq Makefile # Build specific version of transformers RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" make build-awq +# Build eetq kernels +FROM kernel-builder as eetq-kernels-builder +WORKDIR /usr/src +COPY server/Makefile-eetq Makefile +# Build specific version of transformers +RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" make build-eetq + # Build Transformers CUDA kernels FROM kernel-builder as custom-kernels-builder WORKDIR /usr/src @@ -178,6 +185,8 @@ COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39 /o COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages # Copy build artifacts from awq kernels builder COPY --from=awq-kernels-builder /usr/src/llm-awq/awq/kernels/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +# Copy build artifacts from eetq kernels builder +COPY --from=eetq-kernels-builder /usr/src/eetq/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages # Copy builds artifacts from vllm builder COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index dca3612f..5b1b5715 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -297,6 +297,8 @@ def get_model( raise ValueError("awq quantization is not supported for AutoModel") elif (quantize == "bitsandbytes-fp4") or (quantize == "bitsandbytes-nf4"): raise ValueError("4bit quantization is not supported for AutoModel") + elif (quantize == "eetq"): + raise ValueError("Eetq quantization is not supported for AutoModel") if model_type in modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: return CausalLM( model_id, From bd998d87971b22da1e6257052ceab668e6474cce Mon Sep 17 00:00:00 2001 From: Peter Lowrance <46451172+peterlowrance@users.noreply.github.com> Date: Mon, 2 Oct 2023 14:53:14 -0400 Subject: [PATCH 442/793] Fix window_size_left for flash attention v1 (#1089) This fixes flash attention v1 which was always NotImplementedError("window_size_left is only available with flash attn v2"). Currently flash_llama_modeling.py doesn't override the default value of window_size_left when calling attention(..) (line 282). This means that window_size_left will always be the default of -1, but flash attention v1 throws an exception if `window_size_left != 0`. To fix this, we should be checking `window_size_left != -1` before throwing the NotImplementedError. Fixes #1084 ## Before submitting - [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @OlivierDehaene OR @Narsil --- server/text_generation_server/utils/flash_attn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/flash_attn.py b/server/text_generation_server/utils/flash_attn.py index caf072b7..8f0fcee6 100644 --- a/server/text_generation_server/utils/flash_attn.py +++ b/server/text_generation_server/utils/flash_attn.py @@ -80,7 +80,7 @@ def attention( ) if HAS_FLASH_ATTN: - if window_size_left != 0: + if window_size_left != -1: raise NotImplementedError( "window_size_left is only available with flash attn v2" ) From b8fefa6b55fc1f562f470360fface75f4fb8f2fe Mon Sep 17 00:00:00 2001 From: Leo Tronchon Date: Tue, 3 Oct 2023 10:26:10 +0200 Subject: [PATCH 443/793] raise exception on invalid images (#999) # What does this PR do? This PR is meant to handle cases in which the images provided are invalid. ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @Narsil --------- Co-authored-by: Nicolas Patry --- .../models/custom_modeling/idefics_image_processing.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py index 6fb00999..061243fb 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py +++ b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py @@ -194,9 +194,14 @@ def fetch_images(self, image_url_or_urls: Union[str, List[str]]): if isinstance(image_url_or_urls, list): return [self.fetch_images(x) for x in image_url_or_urls] elif isinstance(image_url_or_urls, str): - response = requests.get(image_url_or_urls, stream=True, headers=headers) + response = requests.get(image_url_or_urls, stream=True, headers=headers, timeout=(1, 5)) response.raise_for_status() - return Image.open(BytesIO(response.content)) + try: + image = Image.open(BytesIO(response.content)) + image.verify() + except Exception: + raise ValueError(f"Could not load image from url {image_url_or_urls}") + return image else: raise ValueError( f"only a single or a list of entries is supported but got type={type(image_url_or_urls)}" From 702d26972951ae73f2ec7bbc589caa3fd03568f6 Mon Sep 17 00:00:00 2001 From: Mishig Date: Tue, 3 Oct 2023 11:11:10 +0200 Subject: [PATCH 444/793] [Doc page] Fix launcher page highlighting (#1080) ### Broken highlighting (current) Screenshot 2023-09-28 at 22 38 15 ### Fixed highlighting (this PR) image --- docs/source/basic_tutorials/launcher.md | 2 +- update_doc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index bdb8cb73..08a6ed86 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -2,7 +2,7 @@ -``` +```shell Text Generation Launcher Usage: text-generation-launcher [OPTIONS] diff --git a/update_doc.py b/update_doc.py index 81e6a94e..1fa398b0 100644 --- a/update_doc.py +++ b/update_doc.py @@ -12,7 +12,7 @@ def main(): "utf-8" ) wrap_code_blocks_flag = "" - final_doc = f"# Text-generation-launcher arguments\n\n{wrap_code_blocks_flag}\n\n```\n{output}\n```" + final_doc = f"# Text-generation-launcher arguments\n\n{wrap_code_blocks_flag}\n\n```shell\n{output}\n```" filename = "docs/source/basic_tutorials/launcher.md" if args.check: From 85acb11ba0ed9de0bdc18047478adaaa041baacb Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 3 Oct 2023 11:55:10 +0200 Subject: [PATCH 445/793] Handling bloom prefix. (#1090) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/models/bloom.py | 2 +- .../text_generation_server/utils/weights.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 0151b017..8e8daad3 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -74,7 +74,7 @@ def __init__( torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights( - filenames, device=device, dtype=dtype, process_group=self.process_group + filenames, device=device, dtype=dtype, process_group=self.process_group, prefix="transformer", ) if config.quantize == "gptq": weights._set_gptq_params(model_id) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 8a19fd9f..4bae8cc0 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -16,6 +16,7 @@ def __init__( dtype, process_group, aliases: Optional[Dict[str, List[str]]] = None, + prefix: Optional[str] = None ): routing = {} for filename in filenames: @@ -33,6 +34,7 @@ def __init__( self.device = device self.dtype = dtype self.process_group = process_group + self.prefix = prefix self._handles = {} def _get_handle(self, filename): @@ -43,15 +45,22 @@ def _get_handle(self, filename): return self._handles[filename] def get_filename(self, tensor_name: str) -> (str, str): - filename = self.routing.get(tensor_name, None) - if filename is None: - aliases = self.aliases.get(tensor_name, []) + + names = [tensor_name] + if self.prefix is not None: + prefixed = f"{self.prefix}.{tensor_name}" + names.append(prefixed) + for name in names: + filename = self.routing.get(name, None) + if filename is not None: + return str(filename), name + + aliases = self.aliases.get(name, []) for alias in aliases: filename = self.routing.get(alias, None) if filename is not None: return str(filename), alias - raise RuntimeError(f"weight {tensor_name} does not exist") - return str(filename), tensor_name + raise RuntimeError(f"weight {tensor_name} does not exist") def _get_slice(self, tensor_name: str): filename, tensor_name = self.get_filename(tensor_name) From 1bebb9e76b795c74a85c1706461a211cebf20615 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 3 Oct 2023 12:25:06 +0200 Subject: [PATCH 446/793] Update idefics_image_processing.py (#1091) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../models/custom_modeling/idefics_image_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py index 061243fb..28525e86 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py +++ b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py @@ -198,7 +198,7 @@ def fetch_images(self, image_url_or_urls: Union[str, List[str]]): response.raise_for_status() try: image = Image.open(BytesIO(response.content)) - image.verify() + # image.verify() except Exception: raise ValueError(f"Could not load image from url {image_url_or_urls}") return image From b4f68c3cf4aa91eb0129e53a00438c98c51dee76 Mon Sep 17 00:00:00 2001 From: Fluder-Paradyne <121793617+Fluder-Paradyne@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:55:45 +0530 Subject: [PATCH 447/793] fixed command line arguments in docs (#1092) # What does this PR do? Just removed `--` from the arguments. With `--` bitsandbytes and bitsandbytes-nf4 are considered an option which they are not ## Before submitting - [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- docs/source/conceptual/quantization.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conceptual/quantization.md b/docs/source/conceptual/quantization.md index 1a44e3c2..9bd77b93 100644 --- a/docs/source/conceptual/quantization.md +++ b/docs/source/conceptual/quantization.md @@ -45,7 +45,7 @@ bitsandbytes is a library used to apply 8-bit and 4-bit quantization to models. In TGI, you can use 8-bit quantization by adding `--quantize bitsandbytes` like below 👇 ```bash -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize --bitsandbytes +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes ``` 4-bit quantization is also possible with bitsandbytes. You can choose one of the following 4-bit data types: 4-bit float (`fp4`), or 4-bit `NormalFloat` (`nf4`). These data types were introduced in the context of parameter-efficient fine-tuning, but you can apply them for inference by automatically converting the model weights on load. @@ -53,7 +53,7 @@ docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingf In TGI, you can use 4-bit quantization by adding `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` like below 👇 ```bash -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize --bitsandbytes-nf4 +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes-nf4 ``` You can get more information about 8-bit quantization by reading this [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), and 4-bit quantization by reading [this blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes). From 8ec1b87f16f85f110bb3e7e6d6871525b571dbf9 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 4 Oct 2023 12:57:21 +0200 Subject: [PATCH 448/793] Adding titles to CLI doc. (#1094) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- docs/source/basic_tutorials/launcher.md | 122 +++++++++++++++++++++++- update_doc.py | 28 +++++- 2 files changed, 148 insertions(+), 2 deletions(-) diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index 08a6ed86..62abe8c6 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -8,34 +8,52 @@ Text Generation Launcher Usage: text-generation-launcher [OPTIONS] Options: +``` +## MODEL_ID +```shell --model-id The name of the model to load. Can be a MODEL_ID as listed on like `gpt2` or `OpenAssistant/oasst-sft-1-pythia-12b`. Or it can be a local directory containing the necessary files as saved by `save_pretrained(...)` methods of transformers [env: MODEL_ID=] [default: bigscience/bloom-560m] +``` +## REVISION +```shell --revision The actual revision of the model if you're referring to a model on the hub. You can use a specific commit id or a branch like `refs/pr/2` [env: REVISION=] +``` +## VALIDATION_WORKERS +```shell --validation-workers The number of tokenizer workers used for payload validation and truncation inside the router [env: VALIDATION_WORKERS=] [default: 2] +``` +## SHARDED +```shell --sharded Whether to shard the model across multiple GPUs By default text-generation-inference will use all available GPUs to run the model. Setting it to `false` deactivates `num_shard` [env: SHARDED=] [possible values: true, false] +``` +## NUM_SHARD +```shell --num-shard The number of shards to use if you don't want to use all GPUs on a given machine. You can use `CUDA_VISIBLE_DEVICES=0,1 text-generation-launcher... --num_shard 2` and `CUDA_VISIBLE_DEVICES=2,3 text-generation-launcher... --num_shard 2` to launch 2 copies with 2 shard each on a given machine with 4 GPUs for instance [env: NUM_SHARD=] +``` +## QUANTIZE +```shell --quantize Whether you want the model to be quantized @@ -49,53 +67,80 @@ Options: - bitsandbytes-nf4: Bitsandbytes 4bit. Can be applied on any model, will cut the memory requirement by 4x, but it is known that the model will be much slower to run than the native f16 - bitsandbytes-fp4: Bitsandbytes 4bit. nf4 should be preferred in most cases but maybe this one has better perplexity performance for you model +``` +## DTYPE +```shell --dtype The dtype to be forced upon the model. This option cannot be used with `--quantize` [env: DTYPE=] [possible values: float16, bfloat16] +``` +## TRUST_REMOTE_CODE +```shell --trust-remote-code Whether you want to execute hub modelling code. Explicitly passing a `revision` is encouraged when loading a model with custom code to ensure no malicious code has been contributed in a newer revision [env: TRUST_REMOTE_CODE=] +``` +## MAX_CONCURRENT_REQUESTS +```shell --max-concurrent-requests The maximum amount of concurrent requests for this particular deployment. Having a low limit will refuse clients requests instead of having them wait for too long and is usually good to handle backpressure correctly [env: MAX_CONCURRENT_REQUESTS=] [default: 128] +``` +## MAX_BEST_OF +```shell --max-best-of This is the maximum allowed value for clients to set `best_of`. Best of makes `n` generations at the same time, and return the best in terms of overall log probability over the entire generated sequence [env: MAX_BEST_OF=] [default: 2] +``` +## MAX_STOP_SEQUENCES +```shell --max-stop-sequences This is the maximum allowed value for clients to set `stop_sequences`. Stop sequences are used to allow the model to stop on more than just the EOS token, and enable more complex "prompting" where users can preprompt the model in a specific way and define their "own" stop token aligned with their prompt [env: MAX_STOP_SEQUENCES=] [default: 4] +``` +## MAX_TOP_N_TOKENS +```shell --max-top-n-tokens This is the maximum allowed value for clients to set `top_n_tokens`. `top_n_tokens is used to return information about the the `n` most likely tokens at each generation step, instead of just the sampled token. This information can be used for downstream tasks like for classification or ranking [env: MAX_TOP_N_TOKENS=] [default: 5] +``` +## MAX_INPUT_LENGTH +```shell --max-input-length This is the maximum allowed input length (expressed in number of tokens) for users. The larger this value, the longer prompt users can send which can impact the overall memory required to handle the load. Please note that some models have a finite range of sequence they can handle [env: MAX_INPUT_LENGTH=] [default: 1024] +``` +## MAX_TOTAL_TOKENS +```shell --max-total-tokens This is the most important value to set as it defines the "memory budget" of running clients requests. Clients will send input sequences and ask to generate `max_new_tokens` on top. with a value of `1512` users can send either a prompt of `1000` and ask for `512` new tokens, or send a prompt of `1` and ask for `1511` max_new_tokens. The larger this value, the larger amount each request will be in your RAM and the less effective batching can be [env: MAX_TOTAL_TOKENS=] [default: 2048] +``` +## WAITING_SERVED_RATIO +```shell --waiting-served-ratio This represents the ratio of waiting queries vs running queries where you want to start considering pausing the running queries to include the waiting ones into the same batch. `waiting_served_ratio=1.2` Means when 12 queries are waiting and there's only 10 queries left in the current batch we check if we can fit those 12 waiting queries into the batching strategy, and if yes, then batching happens delaying the 10 running queries by a `prefill` run. @@ -104,12 +149,18 @@ Options: [env: WAITING_SERVED_RATIO=] [default: 1.2] +``` +## MAX_BATCH_PREFILL_TOKENS +```shell --max-batch-prefill-tokens Limits the number of tokens for the prefill operation. Since this operation take the most memory and is compute bound, it is interesting to limit the number of requests that can be sent [env: MAX_BATCH_PREFILL_TOKENS=] [default: 4096] +``` +## MAX_BATCH_TOTAL_TOKENS +```shell --max-batch-total-tokens **IMPORTANT** This is one critical control to allow maximum usage of the available hardware. @@ -123,6 +174,9 @@ Options: [env: MAX_BATCH_TOTAL_TOKENS=] +``` +## MAX_WAITING_TOKENS +```shell --max-waiting-tokens This setting defines how many tokens can be passed before forcing the waiting queries to be put on the batch (if the size of the batch allows for it). New queries require 1 `prefill` forward, which is different from `decode` and therefore you need to pause the running batch in order to run `prefill` to create the correct values for the waiting queries to be able to join the batch. @@ -135,57 +189,87 @@ Options: [env: MAX_WAITING_TOKENS=] [default: 20] +``` +## HOSTNAME +```shell --hostname The IP address to listen on [env: HOSTNAME=] [default: 0.0.0.0] +``` +## PORT +```shell -p, --port The port to listen on [env: PORT=] [default: 3000] +``` +## SHARD_UDS_PATH +```shell --shard-uds-path The name of the socket for gRPC communication between the webserver and the shards [env: SHARD_UDS_PATH=] [default: /tmp/text-generation-server] +``` +## MASTER_ADDR +```shell --master-addr The address the master shard will listen on. (setting used by torch distributed) [env: MASTER_ADDR=] [default: localhost] +``` +## MASTER_PORT +```shell --master-port The address the master port will listen on. (setting used by torch distributed) [env: MASTER_PORT=] [default: 29500] +``` +## HUGGINGFACE_HUB_CACHE +```shell --huggingface-hub-cache The location of the huggingface hub cache. Used to override the location if you want to provide a mounted disk for instance [env: HUGGINGFACE_HUB_CACHE=] +``` +## WEIGHTS_CACHE_OVERRIDE +```shell --weights-cache-override The location of the huggingface hub cache. Used to override the location if you want to provide a mounted disk for instance [env: WEIGHTS_CACHE_OVERRIDE=] +``` +## DISABLE_CUSTOM_KERNELS +```shell --disable-custom-kernels For some models (like bloom), text-generation-inference implemented custom cuda kernels to speed up inference. Those kernels were only tested on A100. Use this flag to disable them if you're running on different hardware and encounter issues [env: DISABLE_CUSTOM_KERNELS=] +``` +## CUDA_MEMORY_FRACTION +```shell --cuda-memory-fraction Limit the CUDA available memory. The allowed value equals the total visible memory multiplied by cuda-memory-fraction [env: CUDA_MEMORY_FRACTION=] [default: 1.0] +``` +## ROPE_SCALING +```shell --rope-scaling Rope scaling will only be used for RoPE models and allow rescaling the position rotary to accomodate for larger prompts. @@ -198,50 +282,86 @@ Options: [env: ROPE_SCALING=] [possible values: linear, dynamic] +``` +## ROPE_FACTOR +```shell --rope-factor Rope scaling will only be used for RoPE models See `rope_scaling` [env: ROPE_FACTOR=] +``` +## JSON_OUTPUT +```shell --json-output Outputs the logs in JSON format (useful for telemetry) [env: JSON_OUTPUT=] +``` +## OTLP_ENDPOINT +```shell --otlp-endpoint [env: OTLP_ENDPOINT=] +``` +## CORS_ALLOW_ORIGIN +```shell --cors-allow-origin [env: CORS_ALLOW_ORIGIN=] +``` +## WATERMARK_GAMMA +```shell --watermark-gamma [env: WATERMARK_GAMMA=] +``` +## WATERMARK_DELTA +```shell --watermark-delta [env: WATERMARK_DELTA=] +``` +## NGROK +```shell --ngrok Enable ngrok tunneling [env: NGROK=] +``` +## NGROK_AUTHTOKEN +```shell --ngrok-authtoken ngrok authentication token [env: NGROK_AUTHTOKEN=] +``` +## NGROK_EDGE +```shell --ngrok-edge ngrok edge [env: NGROK_EDGE=] +``` +## ENV +```shell -e, --env Display a lot of information about your runtime environment +``` +## HELP +```shell -h, --help Print help (see a summary with '-h') +``` +## VERSION +```shell -V, --version Print version -``` \ No newline at end of file +``` diff --git a/update_doc.py b/update_doc.py index 1fa398b0..6206e211 100644 --- a/update_doc.py +++ b/update_doc.py @@ -11,8 +11,34 @@ def main(): output = subprocess.check_output(["text-generation-launcher", "--help"]).decode( "utf-8" ) + wrap_code_blocks_flag = "" - final_doc = f"# Text-generation-launcher arguments\n\n{wrap_code_blocks_flag}\n\n```shell\n{output}\n```" + final_doc = f"# Text-generation-launcher arguments\n\n{wrap_code_blocks_flag}\n\n" + + lines = output.split("\n") + + header = "" + block = [] + for line in lines: + if line.startswith(" -") or line.startswith(" -"): + rendered_block = '\n'.join(block) + if header: + final_doc += f"## {header}\n```shell\n{rendered_block}\n```\n" + else: + final_doc += f"```shell\n{rendered_block}\n```\n" + block = [] + tokens = line.split("<") + if len(tokens)>1: + header = tokens[-1][:-1] + else: + header = line.split("--")[-1] + header = header.upper().replace("-", "_") + + block.append(line) + + rendered_block = '\n'.join(block) + final_doc += f"## {header}\n```shell\n{rendered_block}\n```\n" + block = [] filename = "docs/source/basic_tutorials/launcher.md" if args.check: From 66ce2fa7c1a2ef56237578f3058267851d2b0291 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 4 Oct 2023 17:35:29 +0200 Subject: [PATCH 449/793] Receive base64 encoded images for idefics. (#1096) # What does this PR do? Fix #1095 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../custom_modeling/idefics_image_processing.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py index 28525e86..21aa3ff3 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py +++ b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py @@ -35,6 +35,7 @@ valid_images, ) from io import BytesIO +import base64 import requests from transformers import TensorType, is_torch_available @@ -194,10 +195,17 @@ def fetch_images(self, image_url_or_urls: Union[str, List[str]]): if isinstance(image_url_or_urls, list): return [self.fetch_images(x) for x in image_url_or_urls] elif isinstance(image_url_or_urls, str): - response = requests.get(image_url_or_urls, stream=True, headers=headers, timeout=(1, 5)) - response.raise_for_status() + image = image_url_or_urls + + if image.startswith("http://") or image.startswith("https://"): + response = requests.get(image_url_or_urls, stream=True, headers=headers, timeout=(1, 5)) + response.raise_for_status() + content = response.content + else: + content = base64.b64decode(image) + try: - image = Image.open(BytesIO(response.content)) + image = Image.open(BytesIO(content)) # image.verify() except Exception: raise ValueError(f"Could not load image from url {image_url_or_urls}") From 6df43da0a4f2721c12f0a5636526bb6829455565 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 4 Oct 2023 17:38:42 +0200 Subject: [PATCH 450/793] Modify the default for `max_new_tokens`. (#1097) # What does this PR do? Now clients which do not specify a max_length will be implying `max_new_tokens = max_total_tokens - input_length`. This is a serious change, but which seems more in line with what users expect from standing server. Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: OlivierDehaene --- docs/openapi.json | 98 +++++++++++++++++++++++++++------------- router/src/lib.rs | 8 ++-- router/src/validation.rs | 31 +++++++------ 3 files changed, 88 insertions(+), 49 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 4a1ab6dd..72b073b1 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -367,7 +367,7 @@ "type": "integer", "format": "int32", "example": 1, - "minimum": 0.0 + "minimum": 0 }, "prefill": { "type": "array", @@ -380,13 +380,22 @@ "format": "int64", "example": 42, "nullable": true, - "minimum": 0.0 + "minimum": 0 }, "tokens": { "type": "array", "items": { "$ref": "#/components/schemas/Token" } + }, + "top_tokens": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + } } } }, @@ -432,7 +441,7 @@ "type": "integer", "format": "int32", "example": 1, - "minimum": 0.0 + "minimum": 0 }, "prefill": { "type": "array", @@ -445,13 +454,22 @@ "format": "int64", "example": 42, "nullable": true, - "minimum": 0.0 + "minimum": 0 }, "tokens": { "type": "array", "items": { "$ref": "#/components/schemas/Token" } + }, + "top_tokens": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + } } } }, @@ -486,8 +504,8 @@ "default": "null", "example": 1, "nullable": true, - "minimum": 0.0, - "exclusiveMinimum": 0.0 + "minimum": 0, + "exclusiveMinimum": 0 }, "decoder_input_details": { "type": "boolean", @@ -505,10 +523,10 @@ "max_new_tokens": { "type": "integer", "format": "int32", - "default": "20", - "minimum": 0.0, - "exclusiveMaximum": 512.0, - "exclusiveMinimum": 0.0 + "default": "null", + "example": "20", + "nullable": true, + "minimum": 0 }, "repetition_penalty": { "type": "number", @@ -516,7 +534,7 @@ "default": "null", "example": 1.03, "nullable": true, - "exclusiveMinimum": 0.0 + "exclusiveMinimum": 0 }, "return_full_text": { "type": "boolean", @@ -530,8 +548,8 @@ "default": "null", "example": "null", "nullable": true, - "minimum": 0.0, - "exclusiveMinimum": 0.0 + "minimum": 0, + "exclusiveMinimum": 0 }, "stop": { "type": "array", @@ -549,7 +567,7 @@ "default": "null", "example": 0.5, "nullable": true, - "exclusiveMinimum": 0.0 + "exclusiveMinimum": 0 }, "top_k": { "type": "integer", @@ -557,7 +575,16 @@ "default": "null", "example": 10, "nullable": true, - "exclusiveMinimum": 0.0 + "exclusiveMinimum": 0 + }, + "top_n_tokens": { + "type": "integer", + "format": "int32", + "default": "null", + "example": 5, + "nullable": true, + "minimum": 0, + "exclusiveMinimum": 0 }, "top_p": { "type": "number", @@ -565,15 +592,15 @@ "default": "null", "example": 0.95, "nullable": true, - "maximum": 1.0, - "exclusiveMinimum": 0.0 + "maximum": 1, + "exclusiveMinimum": 0 }, "truncate": { "type": "integer", "default": "null", "example": "null", "nullable": true, - "minimum": 0.0 + "minimum": 0 }, "typical_p": { "type": "number", @@ -581,8 +608,8 @@ "default": "null", "example": 0.95, "nullable": true, - "maximum": 1.0, - "exclusiveMinimum": 0.0 + "maximum": 1, + "exclusiveMinimum": 0 }, "watermark": { "type": "boolean", @@ -653,38 +680,38 @@ "type": "integer", "format": "int32", "example": "32000", - "minimum": 0.0 + "minimum": 0 }, "max_best_of": { "type": "integer", "example": "2", - "minimum": 0.0 + "minimum": 0 }, "max_concurrent_requests": { "type": "integer", "description": "Router Parameters", "example": "128", - "minimum": 0.0 + "minimum": 0 }, "max_input_length": { "type": "integer", "example": "1024", - "minimum": 0.0 + "minimum": 0 }, "max_stop_sequences": { "type": "integer", "example": "4", - "minimum": 0.0 + "minimum": 0 }, "max_total_tokens": { "type": "integer", "example": "2048", - "minimum": 0.0 + "minimum": 0 }, "max_waiting_tokens": { "type": "integer", "example": "20", - "minimum": 0.0 + "minimum": 0 }, "model_device_type": { "type": "string", @@ -717,7 +744,7 @@ "validation_workers": { "type": "integer", "example": "2", - "minimum": 0.0 + "minimum": 0 }, "version": { "type": "string", @@ -743,7 +770,7 @@ "type": "integer", "format": "int32", "example": 0, - "minimum": 0.0 + "minimum": 0 }, "logprob": { "type": "number", @@ -771,14 +798,14 @@ "type": "integer", "format": "int32", "example": 1, - "minimum": 0.0 + "minimum": 0 }, "seed": { "type": "integer", "format": "int64", "example": 42, "nullable": true, - "minimum": 0.0 + "minimum": 0 } } }, @@ -794,6 +821,7 @@ "$ref": "#/components/schemas/StreamDetails" } ], + "default": "null", "nullable": true }, "generated_text": { @@ -804,6 +832,12 @@ }, "token": { "$ref": "#/components/schemas/Token" + }, + "top_tokens": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } } } }, @@ -820,7 +854,7 @@ "type": "integer", "format": "int32", "example": 0, - "minimum": 0.0 + "minimum": 0 }, "logprob": { "type": "number", diff --git a/router/src/lib.rs b/router/src/lib.rs index 76e70bb7..560b8f74 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -107,8 +107,8 @@ pub(crate) struct GenerateParameters { #[schema(default = "false", example = true)] pub do_sample: bool, #[serde(default = "default_max_new_tokens")] - #[schema(exclusive_minimum = 0, exclusive_maximum = 512, default = "20")] - pub max_new_tokens: u32, + #[schema(nullable = true, default = "null", example = "20")] + pub max_new_tokens: Option, #[serde(default)] #[schema(nullable = true, default = "null", example = false)] pub return_full_text: Option, @@ -140,8 +140,8 @@ pub(crate) struct GenerateParameters { pub top_n_tokens: Option, } -fn default_max_new_tokens() -> u32 { - 20 +fn default_max_new_tokens() -> Option { + None } fn default_parameters() -> GenerateParameters { diff --git a/router/src/validation.rs b/router/src/validation.rs index 36cbfb9b..9adedc5b 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -67,8 +67,8 @@ impl Validation { &self, inputs: String, truncate: Option, - max_new_tokens: u32, - ) -> Result<(String, usize), ValidationError> { + max_new_tokens: Option, + ) -> Result<(String, usize, u32), ValidationError> { // If we have a fast tokenizer if let Some(sender) = &self.sender { // Create response channel @@ -84,6 +84,11 @@ impl Validation { let (inputs, input_length) = response_receiver.await.unwrap()?; // Get total tokens + let max_new_tokens: u32 = if let Some(max_new_tokens) = max_new_tokens { + max_new_tokens + } else { + self.max_total_tokens.saturating_sub(input_length) as u32 + }; let total_tokens = input_length + max_new_tokens as usize; // Validate MaxTotalTokens @@ -104,7 +109,7 @@ impl Validation { } metrics::histogram!("tgi_request_input_length", input_length as f64); - Ok((inputs, input_length)) + Ok((inputs, input_length, max_new_tokens)) } // Return inputs without validation else { @@ -112,6 +117,11 @@ impl Validation { // However, the inputs will be truncated by the python servers // We make sure that truncate + max_new_tokens <= self.max_total_tokens let input_length = truncate.unwrap_or(self.max_input_length); + let max_new_tokens: u32 = if let Some(max_new_tokens) = max_new_tokens { + max_new_tokens + } else { + self.max_total_tokens.saturating_sub(input_length) as u32 + }; // Validate MaxNewTokens if (input_length as u32 + max_new_tokens) > self.max_total_tokens as u32 { @@ -121,7 +131,7 @@ impl Validation { )); } - Ok((inputs, input_length)) + Ok((inputs, input_length, max_new_tokens)) } } @@ -200,7 +210,7 @@ impl Validation { }) .unwrap_or(Ok(0))?; - if max_new_tokens == 0 { + if max_new_tokens == Some(0) { return Err(ValidationError::NegativeMaxNewTokens); } @@ -247,7 +257,7 @@ impl Validation { .unwrap_or(Ok(None))?; // Validate inputs - let (inputs, input_length) = self + let (inputs, input_length, max_new_tokens) = self .validate_input(request.inputs, truncate, max_new_tokens) .await?; @@ -426,7 +436,7 @@ mod tests { let max_new_tokens = 10; match validation - .validate_input("Hello".to_string(), None, max_new_tokens) + .validate_input("Hello".to_string(), None, Some(max_new_tokens)) .await { Err(ValidationError::MaxNewTokens(1, 10)) => (), @@ -455,7 +465,7 @@ mod tests { let max_new_tokens = 10; match validation - .validate_input("Hello".to_string(), None, max_new_tokens) + .validate_input("Hello".to_string(), None, Some(max_new_tokens)) .await { Err(ValidationError::MaxTotalTokens(6, 1, 10)) => (), @@ -534,7 +544,6 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_p: Some(0.99), - max_new_tokens: 1, ..default_parameters() }, }) @@ -549,7 +558,6 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_p: None, - max_new_tokens: 1, ..default_parameters() }, }) @@ -596,7 +604,6 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_n_tokens: Some(4), - max_new_tokens: 1, ..default_parameters() }, }) @@ -608,7 +615,6 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_n_tokens: Some(0), - max_new_tokens: 1, ..default_parameters() }, }) @@ -620,7 +626,6 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_n_tokens: None, - max_new_tokens: 1, ..default_parameters() }, }) From 0e4ee4f107b67a8b41cd862ee089f178ed16657e Mon Sep 17 00:00:00 2001 From: Martin Vejvar Date: Thu, 5 Oct 2023 16:33:04 +0900 Subject: [PATCH 451/793] fix: type hint typo in tokens.py (#1102) # What does this PR do? Fixing a list type hint definition (I believe this was a typo). Allows backward compatibility with Python 3.8 (relevant for JetPack-enabled systems). Fixes # (issue) ## Before submitting - [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/tokens.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index f6339d7c..0ff07417 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -337,7 +337,7 @@ def filter(self, indices): def batch_top_tokens( - top_n_tokens: list[int], top_n_tokens_tensor: torch.Tensor, logprobs: torch.Tensor + top_n_tokens: List[int], top_n_tokens_tensor: torch.Tensor, logprobs: torch.Tensor ) -> Tuple[List[List[int]], List[List[float]]]: """Find the top n most likely tokens for a batch of generations. From 87f43814e3a026b6df603efdc309357543c52632 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 5 Oct 2023 10:11:27 +0200 Subject: [PATCH 452/793] Fixing GPTQ exllama kernel usage. (#1101) # What does this PR do? Fixes #1098 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/weights.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 4bae8cc0..2f330d9c 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -212,7 +212,9 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): g_idx = None bits, groupsize = self._get_gptq_params() - weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) + from text_generation_server.utils.layers import HAS_EXLLAMA + use_exllama = bits==4 and HAS_EXLLAMA and quantize == "gptq" + weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) else: w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] weight = torch.cat(w, dim=dim) From 3c373dcc53a499c2f902b88bcfcea9e0525cf0f5 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 5 Oct 2023 10:11:50 +0200 Subject: [PATCH 453/793] Adding yarn support. (#1099) # What does this PR do? Fixes #1017 Not sure if there's a mistake here but - NousResearch/Yarn-Llama-2-7b-128k seems to be working fine - TheBloke/Yarn-Llama-2-13B-128K-GPTQ outputs garbage Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/layers.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index cf61e47b..f38f130e 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -601,6 +601,19 @@ def static(cls, config, dim, base, device): device=inv_freq.device, scaling_factor=scaling_factor, ) + elif rope_scaling["type"] == "yarn": + return YarnPositionRotaryEmbedding( + dim=2 * inv_freq.shape[0], + max_position_embeddings=rope_scaling["original_max_position_embeddings"], + base=10000.0, + device=inv_freq.device, + scaling_factor=scaling_factor, + extrapolation_factor=1, + attn_factor=1, + beta_fast=32, + beta_slow=1 + + ) else: raise NotImplementedError( f"rope scaling type {rope_scaling['type']} is not implemented or invalid" @@ -629,6 +642,19 @@ def load(cls, config, prefix, weights): device=inv_freq.device, scaling_factor=scaling_factor, ) + elif rope_scaling["type"] == "yarn": + return YarnPositionRotaryEmbedding( + dim=2 * inv_freq.shape[0], + max_position_embeddings=rope_scaling["original_max_position_embeddings"], + base=10000.0, + device=inv_freq.device, + scaling_factor=scaling_factor, + extrapolation_factor=1, + attn_factor=1, + beta_fast=32, + beta_slow=1 + + ) else: raise NotImplementedError( f"rope scaling type {rope_scaling['type']} is not implemented or invalid" @@ -708,5 +734,76 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): self._cos_cached = torch.cos(freqs).to(dtype) self._sin_cached = torch.sin(freqs).to(dtype) + + # Inverse dim formula to find dim based on number of rotations + import math + def find_correction_dim(num_rotations, dim, base=10000, max_position_embeddings=2048): + return (dim * math.log(max_position_embeddings/(num_rotations * 2 * math.pi)))/(2 * math.log(base)) + + # Find dim range bounds based on rotations + def find_correction_range(low_rot, high_rot, dim, base=10000, max_position_embeddings=2048): + low = math.floor(find_correction_dim( + low_rot, dim, base, max_position_embeddings)) + high = math.ceil(find_correction_dim( + high_rot, dim, base, max_position_embeddings)) + return max(low, 0), min(high, dim-1) # Clamp values just in case + + def linear_ramp_mask(min, max, dim): + if min == max: + max += 0.001 # Prevent singularity + + linear_func = (torch.arange(dim, dtype=torch.float32) - min) / (max - min) + ramp_func = torch.clamp(linear_func, 0, 1) + return ramp_func + + def get_mscale(scale=1): + if scale <= 1: + return 1.0 + return 0.1 * math.log(scale) + 1.0 + + class YarnPositionRotaryEmbedding(PositionRotaryEmbedding): + def __init__(self, dim, max_position_embeddings, base, device, scaling_factor,*, extrapolation_factor, attn_factor, beta_fast, beta_slow): + inv_freq = _create_inv_freq(dim, base, device) + super().__init__(inv_freq, scaling_factor) + self.dim = dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.extrapolation_factor = extrapolation_factor + self.attn_factor = attn_factor + self.beta_fast = beta_fast + self.beta_slow = beta_slow + self.mscale = float(get_mscale(self.scaling_factor) * self.attn_factor) # Get n-d magnitude scaling corrected for interpolation + + def _update_cos_sin_cache(self, dtype, device, seqlen): + # Reset the tables if the sequence length has changed, + # or if we're on a new device (possibly due to tracing for instance) + if ( + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype + ): + if seqlen > self.max_position_embeddings: + inv_freq_extrapolation = _create_inv_freq( + self.dim, self.base, self.inv_freq.device + ) + freqs = 1.0 / inv_freq_extrapolation + inv_freq_interpolation = 1.0 / (self.scaling_factor * freqs) + low, high = find_correction_range(self.beta_fast, self.beta_slow, self.dim, self.base, self.max_position_embeddings) + inv_freq_mask = (1 - linear_ramp_mask(low, high, self.dim // 2).float().to(device)) * self.extrapolation_factor # Get n-d rotational scaling corrected for extrapolation + inv_freq = inv_freq_interpolation * (1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask + + self.inv_freq = inv_freq + self.mscale = float(get_mscale(self.scaling_factor) * self.attn_factor) # Get n-d magnitude scaling corrected for interpolation + + + self._seq_len_cached = seqlen + t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) + # Don't do einsum, it converts fp32 to fp16 + # freqs = torch.einsum("i,j->ij", t, self.inv_freq) + + freqs = torch.outer(t, self.inv_freq.to(device=t.device)) + self._cos_cached = (torch.cos(freqs) * self.mscale).to(dtype) + self._sin_cached = (torch.sin(freqs) * self.mscale).to(dtype) + except ImportError: pass From e9cdf6225fa73484c0c09b7aaec1e8acbda729cc Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 5 Oct 2023 13:35:26 +0200 Subject: [PATCH 454/793] Hotfixing idefics base64 parsing. (#1103) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../models/custom_modeling/idefics_image_processing.py | 7 ++++++- .../models/custom_modeling/idefics_processing.py | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py index 21aa3ff3..4760ae6f 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py +++ b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py @@ -201,8 +201,13 @@ def fetch_images(self, image_url_or_urls: Union[str, List[str]]): response = requests.get(image_url_or_urls, stream=True, headers=headers, timeout=(1, 5)) response.raise_for_status() content = response.content - else: + elif image.startswith("data:"): + # https://stackoverflow.com/questions/17090571/is-there-a-way-to-set-background-image-as-a-base64-encoded-image + # data:image/png;base64,xxx + image = image.split(",")[-1] content = base64.b64decode(image) + else: + raise ValueError(f"Unrecognized image {image}") try: image = Image.open(BytesIO(content)) diff --git a/server/text_generation_server/models/custom_modeling/idefics_processing.py b/server/text_generation_server/models/custom_modeling/idefics_processing.py index 0fbcbeeb..98e43a27 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_processing.py +++ b/server/text_generation_server/models/custom_modeling/idefics_processing.py @@ -112,6 +112,11 @@ def is_url(string): result = urlparse(string) return all([result.scheme, result.netloc]) +def is_image(string): + """Checks if the passed string contains a valid url and nothing else. e.g. if space is included it's immediately + invalidated the url""" + return is_url(string) or string.startswith("data:") + class IdeficsProcessor(ProcessorMixin): r""" @@ -314,7 +319,7 @@ def image_tokens(last_was_image): if isinstance(item, str): item = item.strip(" ") - if is_url(item): + if is_image(item): image = self.image_processor.fetch_images(item) full_text += image_tokens(last_was_image) image_objects.append(image) @@ -339,6 +344,7 @@ def image_tokens(last_was_image): image_objects = self.image_processor(image_objects, transform=transform) + text_encoding = self.tokenizer( text=full_text, add_special_tokens=False, From 00b8f36fba62e457ff143cce35564ac6704db860 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 5 Oct 2023 16:09:49 +0200 Subject: [PATCH 455/793] Prepare for v1.1.1 (#1100) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- Cargo.toml | 2 +- docs/openapi.json | 2 +- .../basic_tutorials/gated_model_access.md | 2 +- docs/source/quicktour.md | 4 +- integration-tests/pyproject.toml | 2 +- server/poetry.lock | 645 +++++++++--------- server/pyproject.toml | 2 +- 7 files changed, 346 insertions(+), 313 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9ca1e6d2..6b26bb0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.1.0" +version = "1.1.1" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index 72b073b1..f2619a95 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.1.0" + "version": "1.1.1" }, "paths": { "/": { diff --git a/docs/source/basic_tutorials/gated_model_access.md b/docs/source/basic_tutorials/gated_model_access.md index 827f6f4f..da585039 100644 --- a/docs/source/basic_tutorials/gated_model_access.md +++ b/docs/source/basic_tutorials/gated_model_access.md @@ -19,6 +19,6 @@ docker run --gpus all \ --shm-size 1g \ -e HUGGING_FACE_HUB_TOKEN=$token \ -p 8080:80 \ - -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0 \ + -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.1 \ --model-id $model ``` diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index 0a874b57..efcaae28 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -8,7 +8,7 @@ Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/ model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.1 --model-id $model ``` @@ -85,7 +85,7 @@ curl 127.0.0.1:8080/generate \ To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. ```bash -docker run ghcr.io/huggingface/text-generation-inference:1.1.0 --help +docker run ghcr.io/huggingface/text-generation-inference:1.1.1 --help ``` diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index aff6a377..9303ba52 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-integration-tests" -version = "1.1.0" +version = "1.1.1" description = "Text Generation Inference integration tests" authors = ["Nicolas Patry "] diff --git a/server/poetry.lock b/server/poetry.lock index 7c18ec76..0caa1d34 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -214,86 +214,101 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.2.0" +version = "3.3.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, - {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, + {file = "charset-normalizer-3.3.0.tar.gz", hash = "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-win32.whl", hash = "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-win32.whl", hash = "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-win32.whl", hash = "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-win32.whl", hash = "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-win32.whl", hash = "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884"}, + {file = "charset_normalizer-3.3.0-py3-none-any.whl", hash = "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2"}, ] [[package]] @@ -580,148 +595,166 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.58.0" +version = "1.59.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.58.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:3e6bebf1dfdbeb22afd95650e4f019219fef3ab86d3fca8ebade52e4bc39389a"}, - {file = "grpcio-1.58.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:cde11577d5b6fd73a00e6bfa3cf5f428f3f33c2d2878982369b5372bbc4acc60"}, - {file = "grpcio-1.58.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:a2d67ff99e70e86b2be46c1017ae40b4840d09467d5455b2708de6d4c127e143"}, - {file = "grpcio-1.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ed979b273a81de36fc9c6716d9fb09dd3443efa18dcc8652501df11da9583e9"}, - {file = "grpcio-1.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:458899d2ebd55d5ca2350fd3826dfd8fcb11fe0f79828ae75e2b1e6051d50a29"}, - {file = "grpcio-1.58.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc7ffef430b80345729ff0a6825e9d96ac87efe39216e87ac58c6c4ef400de93"}, - {file = "grpcio-1.58.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5b23d75e5173faa3d1296a7bedffb25afd2fddb607ef292dfc651490c7b53c3d"}, - {file = "grpcio-1.58.0-cp310-cp310-win32.whl", hash = "sha256:fad9295fe02455d4f158ad72c90ef8b4bcaadfdb5efb5795f7ab0786ad67dd58"}, - {file = "grpcio-1.58.0-cp310-cp310-win_amd64.whl", hash = "sha256:bc325fed4d074367bebd465a20763586e5e1ed5b943e9d8bc7c162b1f44fd602"}, - {file = "grpcio-1.58.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:652978551af02373a5a313e07bfef368f406b5929cf2d50fa7e4027f913dbdb4"}, - {file = "grpcio-1.58.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:9f13a171281ebb4d7b1ba9f06574bce2455dcd3f2f6d1fbe0fd0d84615c74045"}, - {file = "grpcio-1.58.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8774219e21b05f750eef8adc416e9431cf31b98f6ce9def288e4cea1548cbd22"}, - {file = "grpcio-1.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09206106848462763f7f273ca93d2d2d4d26cab475089e0de830bb76be04e9e8"}, - {file = "grpcio-1.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62831d5e251dd7561d9d9e83a0b8655084b2a1f8ea91e4bd6b3cedfefd32c9d2"}, - {file = "grpcio-1.58.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:212f38c6a156862098f6bdc9a79bf850760a751d259d8f8f249fc6d645105855"}, - {file = "grpcio-1.58.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4b12754af201bb993e6e2efd7812085ddaaef21d0a6f0ff128b97de1ef55aa4a"}, - {file = "grpcio-1.58.0-cp311-cp311-win32.whl", hash = "sha256:3886b4d56bd4afeac518dbc05933926198aa967a7d1d237a318e6fbc47141577"}, - {file = "grpcio-1.58.0-cp311-cp311-win_amd64.whl", hash = "sha256:002f228d197fea12797a14e152447044e14fb4fdb2eb5d6cfa496f29ddbf79ef"}, - {file = "grpcio-1.58.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:b5e8db0aff0a4819946215f156bd722b6f6c8320eb8419567ffc74850c9fd205"}, - {file = "grpcio-1.58.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:201e550b7e2ede113b63e718e7ece93cef5b0fbf3c45e8fe4541a5a4305acd15"}, - {file = "grpcio-1.58.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:d79b660681eb9bc66cc7cbf78d1b1b9e335ee56f6ea1755d34a31108b80bd3c8"}, - {file = "grpcio-1.58.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ef8d4a76d2c7d8065aba829f8d0bc0055495c998dce1964ca5b302d02514fb3"}, - {file = "grpcio-1.58.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6cba491c638c76d3dc6c191d9c75041ca5b8f5c6de4b8327ecdcab527f130bb4"}, - {file = "grpcio-1.58.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6801ff6652ecd2aae08ef994a3e49ff53de29e69e9cd0fd604a79ae4e545a95c"}, - {file = "grpcio-1.58.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:24edec346e69e672daf12b2c88e95c6f737f3792d08866101d8c5f34370c54fd"}, - {file = "grpcio-1.58.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7e473a7abad9af48e3ab5f3b5d237d18208024d28ead65a459bd720401bd2f8f"}, - {file = "grpcio-1.58.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:4891bbb4bba58acd1d620759b3be11245bfe715eb67a4864c8937b855b7ed7fa"}, - {file = "grpcio-1.58.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:e9f995a8a421405958ff30599b4d0eec244f28edc760de82f0412c71c61763d2"}, - {file = "grpcio-1.58.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2f85f87e2f087d9f632c085b37440a3169fda9cdde80cb84057c2fc292f8cbdf"}, - {file = "grpcio-1.58.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb6b92036ff312d5b4182fa72e8735d17aceca74d0d908a7f08e375456f03e07"}, - {file = "grpcio-1.58.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d81c2b2b24c32139dd2536972f1060678c6b9fbd106842a9fcdecf07b233eccd"}, - {file = "grpcio-1.58.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:fbcecb6aedd5c1891db1d70efbfbdc126c986645b5dd616a045c07d6bd2dfa86"}, - {file = "grpcio-1.58.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:92ae871a902cf19833328bd6498ec007b265aabf2fda845ab5bd10abcaf4c8c6"}, - {file = "grpcio-1.58.0-cp38-cp38-win32.whl", hash = "sha256:dc72e04620d49d3007771c0e0348deb23ca341c0245d610605dddb4ac65a37cb"}, - {file = "grpcio-1.58.0-cp38-cp38-win_amd64.whl", hash = "sha256:1c1c5238c6072470c7f1614bf7c774ffde6b346a100521de9ce791d1e4453afe"}, - {file = "grpcio-1.58.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:fe643af248442221db027da43ed43e53b73e11f40c9043738de9a2b4b6ca7697"}, - {file = "grpcio-1.58.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:128eb1f8e70676d05b1b0c8e6600320fc222b3f8c985a92224248b1367122188"}, - {file = "grpcio-1.58.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:039003a5e0ae7d41c86c768ef8b3ee2c558aa0a23cf04bf3c23567f37befa092"}, - {file = "grpcio-1.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f061722cad3f9aabb3fbb27f3484ec9d4667b7328d1a7800c3c691a98f16bb0"}, - {file = "grpcio-1.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0af11938acf8cd4cf815c46156bcde36fa5850518120920d52620cc3ec1830"}, - {file = "grpcio-1.58.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d4cef77ad2fed42b1ba9143465856d7e737279854e444925d5ba45fc1f3ba727"}, - {file = "grpcio-1.58.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:24765a627eb4d9288ace32d5104161c3654128fe27f2808ecd6e9b0cfa7fc8b9"}, - {file = "grpcio-1.58.0-cp39-cp39-win32.whl", hash = "sha256:f0241f7eb0d2303a545136c59bc565a35c4fc3b924ccbd69cb482f4828d6f31c"}, - {file = "grpcio-1.58.0-cp39-cp39-win_amd64.whl", hash = "sha256:dcfba7befe3a55dab6fe1eb7fc9359dc0c7f7272b30a70ae0af5d5b063842f28"}, - {file = "grpcio-1.58.0.tar.gz", hash = "sha256:532410c51ccd851b706d1fbc00a87be0f5312bd6f8e5dbf89d4e99c7f79d7499"}, + {file = "grpcio-1.59.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:225e5fa61c35eeaebb4e7491cd2d768cd8eb6ed00f2664fa83a58f29418b39fd"}, + {file = "grpcio-1.59.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b95ec8ecc4f703f5caaa8d96e93e40c7f589bad299a2617bdb8becbcce525539"}, + {file = "grpcio-1.59.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:1a839ba86764cc48226f50b924216000c79779c563a301586a107bda9cbe9dcf"}, + {file = "grpcio-1.59.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6cfe44a5d7c7d5f1017a7da1c8160304091ca5dc64a0f85bca0d63008c3137a"}, + {file = "grpcio-1.59.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0fcf53df684fcc0154b1e61f6b4a8c4cf5f49d98a63511e3f30966feff39cd0"}, + {file = "grpcio-1.59.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa66cac32861500f280bb60fe7d5b3e22d68c51e18e65367e38f8669b78cea3b"}, + {file = "grpcio-1.59.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8cd2d38c2d52f607d75a74143113174c36d8a416d9472415eab834f837580cf7"}, + {file = "grpcio-1.59.0-cp310-cp310-win32.whl", hash = "sha256:228b91ce454876d7eed74041aff24a8f04c0306b7250a2da99d35dd25e2a1211"}, + {file = "grpcio-1.59.0-cp310-cp310-win_amd64.whl", hash = "sha256:ca87ee6183421b7cea3544190061f6c1c3dfc959e0b57a5286b108511fd34ff4"}, + {file = "grpcio-1.59.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c173a87d622ea074ce79be33b952f0b424fa92182063c3bda8625c11d3585d09"}, + {file = "grpcio-1.59.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:ec78aebb9b6771d6a1de7b6ca2f779a2f6113b9108d486e904bde323d51f5589"}, + {file = "grpcio-1.59.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:0b84445fa94d59e6806c10266b977f92fa997db3585f125d6b751af02ff8b9fe"}, + {file = "grpcio-1.59.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c251d22de8f9f5cca9ee47e4bade7c5c853e6e40743f47f5cc02288ee7a87252"}, + {file = "grpcio-1.59.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:956f0b7cb465a65de1bd90d5a7475b4dc55089b25042fe0f6c870707e9aabb1d"}, + {file = "grpcio-1.59.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:38da5310ef84e16d638ad89550b5b9424df508fd5c7b968b90eb9629ca9be4b9"}, + {file = "grpcio-1.59.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:63982150a7d598281fa1d7ffead6096e543ff8be189d3235dd2b5604f2c553e5"}, + {file = "grpcio-1.59.0-cp311-cp311-win32.whl", hash = "sha256:50eff97397e29eeee5df106ea1afce3ee134d567aa2c8e04fabab05c79d791a7"}, + {file = "grpcio-1.59.0-cp311-cp311-win_amd64.whl", hash = "sha256:15f03bd714f987d48ae57fe092cf81960ae36da4e520e729392a59a75cda4f29"}, + {file = "grpcio-1.59.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:f1feb034321ae2f718172d86b8276c03599846dc7bb1792ae370af02718f91c5"}, + {file = "grpcio-1.59.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d09bd2a4e9f5a44d36bb8684f284835c14d30c22d8ec92ce796655af12163588"}, + {file = "grpcio-1.59.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:2f120d27051e4c59db2f267b71b833796770d3ea36ca712befa8c5fff5da6ebd"}, + {file = "grpcio-1.59.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0ca727a173ee093f49ead932c051af463258b4b493b956a2c099696f38aa66"}, + {file = "grpcio-1.59.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5711c51e204dc52065f4a3327dca46e69636a0b76d3e98c2c28c4ccef9b04c52"}, + {file = "grpcio-1.59.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:d74f7d2d7c242a6af9d4d069552ec3669965b74fed6b92946e0e13b4168374f9"}, + {file = "grpcio-1.59.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3859917de234a0a2a52132489c4425a73669de9c458b01c9a83687f1f31b5b10"}, + {file = "grpcio-1.59.0-cp312-cp312-win32.whl", hash = "sha256:de2599985b7c1b4ce7526e15c969d66b93687571aa008ca749d6235d056b7205"}, + {file = "grpcio-1.59.0-cp312-cp312-win_amd64.whl", hash = "sha256:598f3530231cf10ae03f4ab92d48c3be1fee0c52213a1d5958df1a90957e6a88"}, + {file = "grpcio-1.59.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:b34c7a4c31841a2ea27246a05eed8a80c319bfc0d3e644412ec9ce437105ff6c"}, + {file = "grpcio-1.59.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:c4dfdb49f4997dc664f30116af2d34751b91aa031f8c8ee251ce4dcfc11277b0"}, + {file = "grpcio-1.59.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:61bc72a00ecc2b79d9695220b4d02e8ba53b702b42411397e831c9b0589f08a3"}, + {file = "grpcio-1.59.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f367e4b524cb319e50acbdea57bb63c3b717c5d561974ace0b065a648bb3bad3"}, + {file = "grpcio-1.59.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:849c47ef42424c86af069a9c5e691a765e304079755d5c29eff511263fad9c2a"}, + {file = "grpcio-1.59.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c0488c2b0528e6072010182075615620071371701733c63ab5be49140ed8f7f0"}, + {file = "grpcio-1.59.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:611d9aa0017fa386809bddcb76653a5ab18c264faf4d9ff35cb904d44745f575"}, + {file = "grpcio-1.59.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e5378785dce2b91eb2e5b857ec7602305a3b5cf78311767146464bfa365fc897"}, + {file = "grpcio-1.59.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fe976910de34d21057bcb53b2c5e667843588b48bf11339da2a75f5c4c5b4055"}, + {file = "grpcio-1.59.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:c041a91712bf23b2a910f61e16565a05869e505dc5a5c025d429ca6de5de842c"}, + {file = "grpcio-1.59.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:0ae444221b2c16d8211b55326f8ba173ba8f8c76349bfc1768198ba592b58f74"}, + {file = "grpcio-1.59.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ceb1e68135788c3fce2211de86a7597591f0b9a0d2bb80e8401fd1d915991bac"}, + {file = "grpcio-1.59.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4b1cc3a9dc1924d2eb26eec8792fedd4b3fcd10111e26c1d551f2e4eda79ce"}, + {file = "grpcio-1.59.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:871371ce0c0055d3db2a86fdebd1e1d647cf21a8912acc30052660297a5a6901"}, + {file = "grpcio-1.59.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:93e9cb546e610829e462147ce724a9cb108e61647a3454500438a6deef610be1"}, + {file = "grpcio-1.59.0-cp38-cp38-win32.whl", hash = "sha256:f21917aa50b40842b51aff2de6ebf9e2f6af3fe0971c31960ad6a3a2b24988f4"}, + {file = "grpcio-1.59.0-cp38-cp38-win_amd64.whl", hash = "sha256:14890da86a0c0e9dc1ea8e90101d7a3e0e7b1e71f4487fab36e2bfd2ecadd13c"}, + {file = "grpcio-1.59.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:34341d9e81a4b669a5f5dca3b2a760b6798e95cdda2b173e65d29d0b16692857"}, + {file = "grpcio-1.59.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:986de4aa75646e963466b386a8c5055c8b23a26a36a6c99052385d6fe8aaf180"}, + {file = "grpcio-1.59.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:aca8a24fef80bef73f83eb8153f5f5a0134d9539b4c436a716256b311dda90a6"}, + {file = "grpcio-1.59.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:936b2e04663660c600d5173bc2cc84e15adbad9c8f71946eb833b0afc205b996"}, + {file = "grpcio-1.59.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc8bf2e7bc725e76c0c11e474634a08c8f24bcf7426c0c6d60c8f9c6e70e4d4a"}, + {file = "grpcio-1.59.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81d86a096ccd24a57fa5772a544c9e566218bc4de49e8c909882dae9d73392df"}, + {file = "grpcio-1.59.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2ea95cd6abbe20138b8df965b4a8674ec312aaef3147c0f46a0bac661f09e8d0"}, + {file = "grpcio-1.59.0-cp39-cp39-win32.whl", hash = "sha256:3b8ff795d35a93d1df6531f31c1502673d1cebeeba93d0f9bd74617381507e3f"}, + {file = "grpcio-1.59.0-cp39-cp39-win_amd64.whl", hash = "sha256:38823bd088c69f59966f594d087d3a929d1ef310506bee9e3648317660d65b81"}, + {file = "grpcio-1.59.0.tar.gz", hash = "sha256:acf70a63cf09dd494000007b798aff88a436e1c03b394995ce450be437b8e54f"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.58.0)"] +protobuf = ["grpcio-tools (>=1.59.0)"] [[package]] name = "grpcio-reflection" -version = "1.58.0" +version = "1.59.0" description = "Standard Protobuf Reflection Service for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-reflection-1.58.0.tar.gz", hash = "sha256:e6048a758d17b6ca1705258e7ee5d926d2960a95ae08ba0929dd233e505acd3d"}, - {file = "grpcio_reflection-1.58.0-py3-none-any.whl", hash = "sha256:fa18885d8a09cef02c9a6b1d17dfed0279f1f401b06bd1f75958b78ebf1b5c0c"}, + {file = "grpcio-reflection-1.59.0.tar.gz", hash = "sha256:1fe8f0dd6c180fdcf4e12ced2a8f784d9c741ccbc0b198585b1df024b7f8f3f2"}, + {file = "grpcio_reflection-1.59.0-py3-none-any.whl", hash = "sha256:bf4efc7e2e8162e5be9736f4d0a0b324c9bf0c04ad597a9d78fcaf1fbdf818ec"}, ] [package.dependencies] -grpcio = ">=1.58.0" +grpcio = ">=1.59.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.58.0" +version = "1.59.0" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.58.0.tar.gz", hash = "sha256:0b42e70c0405a66a82d9e9867fa255fe59e618964a6099b20568c31dd9099766"}, - {file = "grpcio_status-1.58.0-py3-none-any.whl", hash = "sha256:36d46072b71a00147709ebce49344ac59b4b8960942acf0f813a8a7d6c1c28e0"}, + {file = "grpcio-status-1.59.0.tar.gz", hash = "sha256:f93b9c33e0a26162ef8431bfcffcc3e1fb217ccd8d7b5b3061b6e9f813e698b5"}, + {file = "grpcio_status-1.59.0-py3-none-any.whl", hash = "sha256:cb5a222b14a80ee050bff9676623822e953bff0c50d2d29180de723652fdf10d"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.58.0" +grpcio = ">=1.59.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.58.0" +version = "1.59.0" description = "Protobuf code generator for gRPC" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-tools-1.58.0.tar.gz", hash = "sha256:6f4d80ceb591e31ca4dceec747dbe56132e1392a0a9bb1c8fe001d1b5cac898a"}, - {file = "grpcio_tools-1.58.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:60c874908f3b40f32f1bb0221f7b3ab65ecb53a4d0a9f0a394f031f1b292c177"}, - {file = "grpcio_tools-1.58.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:1852e798f31e5437ca7b37abc910e028b34732fb19364862cedb87b1dab66fad"}, - {file = "grpcio_tools-1.58.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:149fb48f53cb691a6328f68bed8e4036c730f7106b7f98e92c2c0403f0b9e93c"}, - {file = "grpcio_tools-1.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba3d383e5ca93826038b70f326fce8e8d12dd9b2f64d363a3d612f7475f12dd2"}, - {file = "grpcio_tools-1.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6997511e9d2979f7a2389479682dbb06823f21a904e8fb0a5c6baaf1b4b4a863"}, - {file = "grpcio_tools-1.58.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8de0b701da479643f71fad71fe66885cddd89441ae16e2c724939b47742dc72e"}, - {file = "grpcio_tools-1.58.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:43cc23908b63fcaefe690b10f68a2d8652c994b5b36ab77d2271d9608c895320"}, - {file = "grpcio_tools-1.58.0-cp310-cp310-win32.whl", hash = "sha256:2c2221123d010dc6231799e63a37f2f4786bf614ef65b23009c387cd20d8b193"}, - {file = "grpcio_tools-1.58.0-cp310-cp310-win_amd64.whl", hash = "sha256:df2788736bdf58abe7b0e4d6b1ff806f7686c98c5ad900da312252e3322d91c4"}, - {file = "grpcio_tools-1.58.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:b6ea5578712cdb29b0ff60bfc6405bf0e8d681b9c71d106dd1cda54fe7fe4e55"}, - {file = "grpcio_tools-1.58.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:c29880f491581c83181c0a84a4d11402af2b13166a5266f64e246adf1da7aa66"}, - {file = "grpcio_tools-1.58.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:32d51e933c3565414dd0835f930bb28a1cdeba435d9d2c87fa3cf8b1d284db3c"}, - {file = "grpcio_tools-1.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ad9d77f25514584b1ddc981d70c9e50dfcfc388aa5ba943eee67520c5267ed9"}, - {file = "grpcio_tools-1.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4882382631e6352819059278a5c878ce0b067008dd490911d16d5616e8a36d85"}, - {file = "grpcio_tools-1.58.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d84091a189d848d94645b7c48b61734c12ec03b0d46e5fc0049343a26989ac5c"}, - {file = "grpcio_tools-1.58.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:85ac28a9621e9b92a3fc416288c4ce45542db0b4c31b3e23031dd8e0a0ec5590"}, - {file = "grpcio_tools-1.58.0-cp311-cp311-win32.whl", hash = "sha256:7371d8ea80234b29affec145e25569523f549520ed7e53b2aa92bed412cdecfd"}, - {file = "grpcio_tools-1.58.0-cp311-cp311-win_amd64.whl", hash = "sha256:6997df6e7c5cf4d3ddc764240c1ff6a04b45d70ec28913b38fbc6396ef743e12"}, - {file = "grpcio_tools-1.58.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:ac65b8d6e3acaf88b815edf9af88ff844b6600ff3d2591c05ba4f655b45d5fb4"}, - {file = "grpcio_tools-1.58.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:88e8191d0dd789bebf42533808728f5ce75d2c51e2a72bdf20abe5b5e3fbec42"}, - {file = "grpcio_tools-1.58.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:a3dbece2a121761499a659b799979d4b738586d1065439053de553773eee11ca"}, - {file = "grpcio_tools-1.58.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1086fe240c4c879b9721952b47d46996deb283c2d9355a8dc24a804811aacf70"}, - {file = "grpcio_tools-1.58.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7ae3dca059d5b358dd03fb63277428fa7d771605d4074a019138dd38d70719a"}, - {file = "grpcio_tools-1.58.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3f8904ac7fc3da2e874f00b3a986e8b7e004f499344a8e7eb213c26dfb025041"}, - {file = "grpcio_tools-1.58.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:aadbd8393ae332e49731adb31e741f2e689989150569b7acc939f5ea43124e2d"}, - {file = "grpcio_tools-1.58.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1cb6e24194786687d4f23c64de1f0ce553af51de22746911bc37340f85f9783e"}, - {file = "grpcio_tools-1.58.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:6ec43909095c630df3e479e77469bdad367067431f4af602f6ccb978a3b78afd"}, - {file = "grpcio_tools-1.58.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:4be49ed320b0ebcbc21d19ef555fbf229c1c452105522b728e1171ee2052078e"}, - {file = "grpcio_tools-1.58.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:28eefebddec3d3adf19baca78f8b82a2287d358e1b1575ae018cdca8eacc6269"}, - {file = "grpcio_tools-1.58.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ef8c696e9d78676cc3f583a92bbbf2c84e94e350f7ad22f150a52559f4599d1"}, - {file = "grpcio_tools-1.58.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9aeb5949e46558d21c51fd3ec3eeecc59c94dbca76c67c0a80d3da6b7437930c"}, - {file = "grpcio_tools-1.58.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6f7144aad9396d35fb1b80429600a970b559c2ad4d07020eeb180fe83cea2bee"}, - {file = "grpcio_tools-1.58.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ee26e9253a721fff355737649678535f76cf5d642aa3ac0cd937832559b90af"}, - {file = "grpcio_tools-1.58.0-cp38-cp38-win32.whl", hash = "sha256:343f572312039059a8797d6e29a7fc62196e73131ab01755660a9d48202267c1"}, - {file = "grpcio_tools-1.58.0-cp38-cp38-win_amd64.whl", hash = "sha256:cd7acfbb43b7338a78cf4a67528d05530d574d92b7c829d185b78dfc451d158f"}, - {file = "grpcio_tools-1.58.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:46628247fbce86d18232eead24bd22ed0826c79f3fe2fc2fbdbde45971361049"}, - {file = "grpcio_tools-1.58.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:51587842a54e025a3d0d37afcf4ef2b7ac1def9a5d17448665cb424b53d6c287"}, - {file = "grpcio_tools-1.58.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a062ae3072a2a39a3c057f4d68b57b021f1dd2956cd09aab39709f6af494e1de"}, - {file = "grpcio_tools-1.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eec3c93a08df11c80ef1c29a616bcbb0d83dbc6ea41b48306fcacc720416dfa7"}, - {file = "grpcio_tools-1.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b63f823ac991ff77104da614d2a2485a59d37d57830eb2e387a6e2a3edc7fa2b"}, - {file = "grpcio_tools-1.58.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:579c11a9f198847ed48dbc4f211c67fe96a73320b87c81f01b044b72e24a7d77"}, - {file = "grpcio_tools-1.58.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6ca2fc1dd8049d417a5034d944c9df05cee76f855b3e431627ab4292e7c01c47"}, - {file = "grpcio_tools-1.58.0-cp39-cp39-win32.whl", hash = "sha256:453023120114c35d3d9d6717ea0820e5d5c140f51f9d0b621de4397ff854471b"}, - {file = "grpcio_tools-1.58.0-cp39-cp39-win_amd64.whl", hash = "sha256:b6c896f1df99c35cf062d4803c15663ff00a33ff09add28baa6e475cf6b5e258"}, + {file = "grpcio-tools-1.59.0.tar.gz", hash = "sha256:aa4018f2d8662ac4d9830445d3d253a11b3e096e8afe20865547137aa1160e93"}, + {file = "grpcio_tools-1.59.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:882b809b42b5464bee55288f4e60837297f9618e53e69ae3eea6d61b05ce48fa"}, + {file = "grpcio_tools-1.59.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:4499d4bc5aa9c7b645018d8b0db4bebd663d427aabcd7bee7777046cb1bcbca7"}, + {file = "grpcio_tools-1.59.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f381ae3ad6a5eb27aad8d810438937d8228977067c54e0bd456fce7e11fdbf3d"}, + {file = "grpcio_tools-1.59.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1c684c0d9226d04cadafced620a46ab38c346d0780eaac7448da96bf12066a3"}, + {file = "grpcio_tools-1.59.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40cbf712769242c2ba237745285ef789114d7fcfe8865fc4817d87f20015e99a"}, + {file = "grpcio_tools-1.59.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:1df755951f204e65bf9232a9cac5afe7d6b8e4c87ac084d3ecd738fdc7aa4174"}, + {file = "grpcio_tools-1.59.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:de156c18b0c638aaee3be6ad650c8ba7dec94ed4bac26403aec3dce95ffe9407"}, + {file = "grpcio_tools-1.59.0-cp310-cp310-win32.whl", hash = "sha256:9af7e138baa9b2895cf1f3eb718ac96fc5ae2f8e31fca405e21e0e5cd1643c52"}, + {file = "grpcio_tools-1.59.0-cp310-cp310-win_amd64.whl", hash = "sha256:f14a6e4f700dfd30ff8f0e6695f944affc16ae5a1e738666b3fae4e44b65637e"}, + {file = "grpcio_tools-1.59.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:db030140d0da2368319e2f23655df3baec278c7e0078ecbe051eaf609a69382c"}, + {file = "grpcio_tools-1.59.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:eeed386971bb8afc3ec45593df6a1154d680d87be1209ef8e782e44f85f47e64"}, + {file = "grpcio_tools-1.59.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:962d1a3067129152cee3e172213486cb218a6bad703836991f46f216caefcf00"}, + {file = "grpcio_tools-1.59.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:26eb2eebf150a33ebf088e67c1acf37eb2ac4133d9bfccbaa011ad2148c08b42"}, + {file = "grpcio_tools-1.59.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b2d6da553980c590487f2e7fd3ec9c1ad8805ff2ec77977b92faa7e3ca14e1f"}, + {file = "grpcio_tools-1.59.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:335e2f355a0c544a88854e2c053aff8a3f398b84a263a96fa19d063ca1fe513a"}, + {file = "grpcio_tools-1.59.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:204e08f807b1d83f5f0efea30c4e680afe26a43dec8ba614a45fa698a7ef0a19"}, + {file = "grpcio_tools-1.59.0-cp311-cp311-win32.whl", hash = "sha256:05bf7b3ed01c8a562bb7e840f864c58acedbd6924eb616367c0bd0a760bdf483"}, + {file = "grpcio_tools-1.59.0-cp311-cp311-win_amd64.whl", hash = "sha256:df85096fcac7cea8aa5bd84b7a39c4cdbf556b93669bb4772eb96aacd3222a4e"}, + {file = "grpcio_tools-1.59.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:240a7a3c2c54f77f1f66085a635bca72003d02f56a670e7db19aec531eda8f78"}, + {file = "grpcio_tools-1.59.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:6119f62c462d119c63227b9534210f0f13506a888151b9bf586f71e7edf5088b"}, + {file = "grpcio_tools-1.59.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:387662bee8e4c0b52cc0f61eaaca0ca583f5b227103f685b76083a3590a71a3e"}, + {file = "grpcio_tools-1.59.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f0da5861ee276ca68493b217daef358960e8527cc63c7cb292ca1c9c54939af"}, + {file = "grpcio_tools-1.59.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f0806de1161c7f248e4c183633ee7a58dfe45c2b77ddf0136e2e7ad0650b1b"}, + {file = "grpcio_tools-1.59.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:c683be38a9bf4024c223929b4cd2f0a0858c94e9dc8b36d7eaa5a48ce9323a6f"}, + {file = "grpcio_tools-1.59.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f965707da2b48a33128615bcfebedd215a3a30e346447e885bb3da37a143177a"}, + {file = "grpcio_tools-1.59.0-cp312-cp312-win32.whl", hash = "sha256:2ee960904dde12a7fa48e1591a5b3eeae054bdce57bacf9fd26685a98138f5bf"}, + {file = "grpcio_tools-1.59.0-cp312-cp312-win_amd64.whl", hash = "sha256:71cc6db1d66da3bc3730d9937bddc320f7b1f1dfdff6342bcb5741515fe4110b"}, + {file = "grpcio_tools-1.59.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:f6263b85261b62471cb97b7505df72d72b8b62e5e22d8184924871a6155b4dbf"}, + {file = "grpcio_tools-1.59.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:b8e95d921cc2a1521d4750eedefec9f16031457920a6677edebe9d1b2ad6ae60"}, + {file = "grpcio_tools-1.59.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:cb63055739808144b541986291679d643bae58755d0eb082157c4d4c04443905"}, + {file = "grpcio_tools-1.59.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c4634b3589efa156a8d5860c0a2547315bd5c9e52d14c960d716fe86e0927be"}, + {file = "grpcio_tools-1.59.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d970aa26854f535ffb94ea098aa8b43de020d9a14682e4a15dcdaeac7801b27"}, + {file = "grpcio_tools-1.59.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:821dba464d84ebbcffd9d420302404db2fa7a40c7ff4c4c4c93726f72bfa2769"}, + {file = "grpcio_tools-1.59.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0548e901894399886ff4a4cd808cb850b60c021feb4a8977a0751f14dd7e55d9"}, + {file = "grpcio_tools-1.59.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bb87158dbbb9e5a79effe78d54837599caa16df52d8d35366e06a91723b587ae"}, + {file = "grpcio_tools-1.59.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:1d551ff42962c7c333c3da5c70d5e617a87dee581fa2e2c5ae2d5137c8886779"}, + {file = "grpcio_tools-1.59.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:4ee443abcd241a5befb05629013fbf2eac637faa94aaa3056351aded8a31c1bc"}, + {file = "grpcio_tools-1.59.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:520c0c83ea79d14b0679ba43e19c64ca31d30926b26ad2ca7db37cbd89c167e2"}, + {file = "grpcio_tools-1.59.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9fc02a6e517c34dcf885ff3b57260b646551083903e3d2c780b4971ce7d4ab7c"}, + {file = "grpcio_tools-1.59.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aec8a4ed3808b7dfc1276fe51e3e24bec0eeaf610d395bcd42934647cf902a3"}, + {file = "grpcio_tools-1.59.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:99b3bde646720bbfb77f263f5ba3e1a0de50632d43c38d405a0ef9c7e94373cd"}, + {file = "grpcio_tools-1.59.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:51d9595629998d8b519126c5a610f15deb0327cd6325ed10796b47d1d292e70b"}, + {file = "grpcio_tools-1.59.0-cp38-cp38-win32.whl", hash = "sha256:bfa4b2b7d21c5634b62e5f03462243bd705adc1a21806b5356b8ce06d902e160"}, + {file = "grpcio_tools-1.59.0-cp38-cp38-win_amd64.whl", hash = "sha256:9ed05197c5ab071e91bcef28901e97ca168c4ae94510cb67a14cb4931b94255a"}, + {file = "grpcio_tools-1.59.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:498e7be0b14385980efa681444ba481349c131fc5ec88003819f5d929646947c"}, + {file = "grpcio_tools-1.59.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:b519f2ecde9a579cad2f4a7057d5bb4e040ad17caab8b5e691ed7a13b9db0be9"}, + {file = "grpcio_tools-1.59.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:ef3e8aca2261f7f07436d4e2111556c1fb9bf1f9cfcdf35262743ccdee1b6ce9"}, + {file = "grpcio_tools-1.59.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a7f226b741b2ebf7e2d0779d2c9b17f446d1b839d59886c1619e62cc2ae472"}, + {file = "grpcio_tools-1.59.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:784aa52965916fec5afa1a28eeee6f0073bb43a2a1d7fedf963393898843077a"}, + {file = "grpcio_tools-1.59.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e312ddc2d8bec1a23306a661ad52734f984c9aad5d8f126ebb222a778d95407d"}, + {file = "grpcio_tools-1.59.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:868892ad9e00651a38dace3e4924bae82fc4fd4df2c65d37b74381570ee8deb1"}, + {file = "grpcio_tools-1.59.0-cp39-cp39-win32.whl", hash = "sha256:a4f6cae381f21fee1ef0a5cbbbb146680164311157ae618edf3061742d844383"}, + {file = "grpcio_tools-1.59.0-cp39-cp39-win_amd64.whl", hash = "sha256:4a10e59cca462208b489478340b52a96d64e8b8b6f1ac097f3e8cb211d3f66c0"}, ] [package.dependencies] -grpcio = ">=1.58.0" +grpcio = ">=1.59.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -1254,13 +1287,13 @@ files = [ [[package]] name = "packaging" -version = "23.1" +version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -1637,99 +1670,99 @@ files = [ [[package]] name = "regex" -version = "2023.8.8" +version = "2023.10.3" description = "Alternative regular expression module, to replace re." optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "regex-2023.8.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:88900f521c645f784260a8d346e12a1590f79e96403971241e64c3a265c8ecdb"}, - {file = "regex-2023.8.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3611576aff55918af2697410ff0293d6071b7e00f4b09e005d614686ac4cd57c"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8a0ccc8f2698f120e9e5742f4b38dc944c38744d4bdfc427616f3a163dd9de5"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c662a4cbdd6280ee56f841f14620787215a171c4e2d1744c9528bed8f5816c96"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf0633e4a1b667bfe0bb10b5e53fe0d5f34a6243ea2530eb342491f1adf4f739"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:551ad543fa19e94943c5b2cebc54c73353ffff08228ee5f3376bd27b3d5b9800"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54de2619f5ea58474f2ac211ceea6b615af2d7e4306220d4f3fe690c91988a61"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ec4b3f0aebbbe2fc0134ee30a791af522a92ad9f164858805a77442d7d18570"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ae646c35cb9f820491760ac62c25b6d6b496757fda2d51be429e0e7b67ae0ab"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca339088839582d01654e6f83a637a4b8194d0960477b9769d2ff2cfa0fa36d2"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:d9b6627408021452dcd0d2cdf8da0534e19d93d070bfa8b6b4176f99711e7f90"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:bd3366aceedf274f765a3a4bc95d6cd97b130d1dda524d8f25225d14123c01db"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7aed90a72fc3654fba9bc4b7f851571dcc368120432ad68b226bd593f3f6c0b7"}, - {file = "regex-2023.8.8-cp310-cp310-win32.whl", hash = "sha256:80b80b889cb767cc47f31d2b2f3dec2db8126fbcd0cff31b3925b4dc6609dcdb"}, - {file = "regex-2023.8.8-cp310-cp310-win_amd64.whl", hash = "sha256:b82edc98d107cbc7357da7a5a695901b47d6eb0420e587256ba3ad24b80b7d0b"}, - {file = "regex-2023.8.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1e7d84d64c84ad97bf06f3c8cb5e48941f135ace28f450d86af6b6512f1c9a71"}, - {file = "regex-2023.8.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce0f9fbe7d295f9922c0424a3637b88c6c472b75eafeaff6f910494a1fa719ef"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06c57e14ac723b04458df5956cfb7e2d9caa6e9d353c0b4c7d5d54fcb1325c46"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7a9aaa5a1267125eef22cef3b63484c3241aaec6f48949b366d26c7250e0357"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b7408511fca48a82a119d78a77c2f5eb1b22fe88b0d2450ed0756d194fe7a9a"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14dc6f2d88192a67d708341f3085df6a4f5a0c7b03dec08d763ca2cd86e9f559"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48c640b99213643d141550326f34f0502fedb1798adb3c9eb79650b1ecb2f177"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0085da0f6c6393428bf0d9c08d8b1874d805bb55e17cb1dfa5ddb7cfb11140bf"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:964b16dcc10c79a4a2be9f1273fcc2684a9eedb3906439720598029a797b46e6"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7ce606c14bb195b0e5108544b540e2c5faed6843367e4ab3deb5c6aa5e681208"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:40f029d73b10fac448c73d6eb33d57b34607f40116e9f6e9f0d32e9229b147d7"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3b8e6ea6be6d64104d8e9afc34c151926f8182f84e7ac290a93925c0db004bfd"}, - {file = "regex-2023.8.8-cp311-cp311-win32.whl", hash = "sha256:942f8b1f3b223638b02df7df79140646c03938d488fbfb771824f3d05fc083a8"}, - {file = "regex-2023.8.8-cp311-cp311-win_amd64.whl", hash = "sha256:51d8ea2a3a1a8fe4f67de21b8b93757005213e8ac3917567872f2865185fa7fb"}, - {file = "regex-2023.8.8-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e951d1a8e9963ea51efd7f150450803e3b95db5939f994ad3d5edac2b6f6e2b4"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704f63b774218207b8ccc6c47fcef5340741e5d839d11d606f70af93ee78e4d4"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22283c769a7b01c8ac355d5be0715bf6929b6267619505e289f792b01304d898"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91129ff1bb0619bc1f4ad19485718cc623a2dc433dff95baadbf89405c7f6b57"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de35342190deb7b866ad6ba5cbcccb2d22c0487ee0cbb251efef0843d705f0d4"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b993b6f524d1e274a5062488a43e3f9f8764ee9745ccd8e8193df743dbe5ee61"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3026cbcf11d79095a32d9a13bbc572a458727bd5b1ca332df4a79faecd45281c"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:293352710172239bf579c90a9864d0df57340b6fd21272345222fb6371bf82b3"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d909b5a3fff619dc7e48b6b1bedc2f30ec43033ba7af32f936c10839e81b9217"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3d370ff652323c5307d9c8e4c62efd1956fb08051b0e9210212bc51168b4ff56"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:b076da1ed19dc37788f6a934c60adf97bd02c7eea461b73730513921a85d4235"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e9941a4ada58f6218694f382e43fdd256e97615db9da135e77359da257a7168b"}, - {file = "regex-2023.8.8-cp36-cp36m-win32.whl", hash = "sha256:a8c65c17aed7e15a0c824cdc63a6b104dfc530f6fa8cb6ac51c437af52b481c7"}, - {file = "regex-2023.8.8-cp36-cp36m-win_amd64.whl", hash = "sha256:aadf28046e77a72f30dcc1ab185639e8de7f4104b8cb5c6dfa5d8ed860e57236"}, - {file = "regex-2023.8.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:423adfa872b4908843ac3e7a30f957f5d5282944b81ca0a3b8a7ccbbfaa06103"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ae594c66f4a7e1ea67232a0846649a7c94c188d6c071ac0210c3e86a5f92109"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e51c80c168074faa793685656c38eb7a06cbad7774c8cbc3ea05552d615393d8"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:09b7f4c66aa9d1522b06e31a54f15581c37286237208df1345108fcf4e050c18"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e73e5243af12d9cd6a9d6a45a43570dbe2e5b1cdfc862f5ae2b031e44dd95a8"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:941460db8fe3bd613db52f05259c9336f5a47ccae7d7def44cc277184030a116"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f0ccf3e01afeb412a1a9993049cb160d0352dba635bbca7762b2dc722aa5742a"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:2e9216e0d2cdce7dbc9be48cb3eacb962740a09b011a116fd7af8c832ab116ca"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:5cd9cd7170459b9223c5e592ac036e0704bee765706445c353d96f2890e816c8"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4873ef92e03a4309b3ccd8281454801b291b689f6ad45ef8c3658b6fa761d7ac"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:239c3c2a339d3b3ddd51c2daef10874410917cd2b998f043c13e2084cb191684"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1005c60ed7037be0d9dea1f9c53cc42f836188227366370867222bda4c3c6bd7"}, - {file = "regex-2023.8.8-cp37-cp37m-win32.whl", hash = "sha256:e6bd1e9b95bc5614a7a9c9c44fde9539cba1c823b43a9f7bc11266446dd568e3"}, - {file = "regex-2023.8.8-cp37-cp37m-win_amd64.whl", hash = "sha256:9a96edd79661e93327cfeac4edec72a4046e14550a1d22aa0dd2e3ca52aec921"}, - {file = "regex-2023.8.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2181c20ef18747d5f4a7ea513e09ea03bdd50884a11ce46066bb90fe4213675"}, - {file = "regex-2023.8.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a2ad5add903eb7cdde2b7c64aaca405f3957ab34f16594d2b78d53b8b1a6a7d6"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9233ac249b354c54146e392e8a451e465dd2d967fc773690811d3a8c240ac601"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:920974009fb37b20d32afcdf0227a2e707eb83fe418713f7a8b7de038b870d0b"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2b6c5dfe0929b6c23dde9624483380b170b6e34ed79054ad131b20203a1a63"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96979d753b1dc3b2169003e1854dc67bfc86edf93c01e84757927f810b8c3c93"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ae54a338191e1356253e7883d9d19f8679b6143703086245fb14d1f20196be9"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2162ae2eb8b079622176a81b65d486ba50b888271302190870b8cc488587d280"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c884d1a59e69e03b93cf0dfee8794c63d7de0ee8f7ffb76e5f75be8131b6400a"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf9273e96f3ee2ac89ffcb17627a78f78e7516b08f94dc435844ae72576a276e"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:83215147121e15d5f3a45d99abeed9cf1fe16869d5c233b08c56cdf75f43a504"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3f7454aa427b8ab9101f3787eb178057c5250478e39b99540cfc2b889c7d0586"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0640913d2c1044d97e30d7c41728195fc37e54d190c5385eacb52115127b882"}, - {file = "regex-2023.8.8-cp38-cp38-win32.whl", hash = "sha256:0c59122ceccb905a941fb23b087b8eafc5290bf983ebcb14d2301febcbe199c7"}, - {file = "regex-2023.8.8-cp38-cp38-win_amd64.whl", hash = "sha256:c12f6f67495ea05c3d542d119d270007090bad5b843f642d418eb601ec0fa7be"}, - {file = "regex-2023.8.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:82cd0a69cd28f6cc3789cc6adeb1027f79526b1ab50b1f6062bbc3a0ccb2dbc3"}, - {file = "regex-2023.8.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bb34d1605f96a245fc39790a117ac1bac8de84ab7691637b26ab2c5efb8f228c"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:987b9ac04d0b38ef4f89fbc035e84a7efad9cdd5f1e29024f9289182c8d99e09"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9dd6082f4e2aec9b6a0927202c85bc1b09dcab113f97265127c1dc20e2e32495"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7eb95fe8222932c10d4436e7a6f7c99991e3fdd9f36c949eff16a69246dee2dc"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7098c524ba9f20717a56a8d551d2ed491ea89cbf37e540759ed3b776a4f8d6eb"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b694430b3f00eb02c594ff5a16db30e054c1b9589a043fe9174584c6efa8033"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b2aeab3895d778155054abea5238d0eb9a72e9242bd4b43f42fd911ef9a13470"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:988631b9d78b546e284478c2ec15c8a85960e262e247b35ca5eaf7ee22f6050a"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:67ecd894e56a0c6108ec5ab1d8fa8418ec0cff45844a855966b875d1039a2e34"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:14898830f0a0eb67cae2bbbc787c1a7d6e34ecc06fbd39d3af5fe29a4468e2c9"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f2200e00b62568cfd920127782c61bc1c546062a879cdc741cfcc6976668dfcf"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9691a549c19c22d26a4f3b948071e93517bdf86e41b81d8c6ac8a964bb71e5a6"}, - {file = "regex-2023.8.8-cp39-cp39-win32.whl", hash = "sha256:6ab2ed84bf0137927846b37e882745a827458689eb969028af8032b1b3dac78e"}, - {file = "regex-2023.8.8-cp39-cp39-win_amd64.whl", hash = "sha256:5543c055d8ec7801901e1193a51570643d6a6ab8751b1f7dd9af71af467538bb"}, - {file = "regex-2023.8.8.tar.gz", hash = "sha256:fcbdc5f2b0f1cd0f6a56cdb46fe41d2cce1e644e3b68832f3eeebc5fb0f7712e"}, + {file = "regex-2023.10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4c34d4f73ea738223a094d8e0ffd6d2c1a1b4c175da34d6b0de3d8d69bee6bcc"}, + {file = "regex-2023.10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8f4e49fc3ce020f65411432183e6775f24e02dff617281094ba6ab079ef0915"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cd1bccf99d3ef1ab6ba835308ad85be040e6a11b0977ef7ea8c8005f01a3c29"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81dce2ddc9f6e8f543d94b05d56e70d03a0774d32f6cca53e978dc01e4fc75b8"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c6b4d23c04831e3ab61717a707a5d763b300213db49ca680edf8bf13ab5d91b"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c15ad0aee158a15e17e0495e1e18741573d04eb6da06d8b84af726cfc1ed02ee"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6239d4e2e0b52c8bd38c51b760cd870069f0bdf99700a62cd509d7a031749a55"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4a8bf76e3182797c6b1afa5b822d1d5802ff30284abe4599e1247be4fd6b03be"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9c727bbcf0065cbb20f39d2b4f932f8fa1631c3e01fcedc979bd4f51fe051c5"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3ccf2716add72f80714b9a63899b67fa711b654be3fcdd34fa391d2d274ce767"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:107ac60d1bfdc3edb53be75e2a52aff7481b92817cfdddd9b4519ccf0e54a6ff"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:00ba3c9818e33f1fa974693fb55d24cdc8ebafcb2e4207680669d8f8d7cca79a"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f0a47efb1dbef13af9c9a54a94a0b814902e547b7f21acb29434504d18f36e3a"}, + {file = "regex-2023.10.3-cp310-cp310-win32.whl", hash = "sha256:36362386b813fa6c9146da6149a001b7bd063dabc4d49522a1f7aa65b725c7ec"}, + {file = "regex-2023.10.3-cp310-cp310-win_amd64.whl", hash = "sha256:c65a3b5330b54103e7d21cac3f6bf3900d46f6d50138d73343d9e5b2900b2353"}, + {file = "regex-2023.10.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90a79bce019c442604662d17bf69df99090e24cdc6ad95b18b6725c2988a490e"}, + {file = "regex-2023.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c7964c2183c3e6cce3f497e3a9f49d182e969f2dc3aeeadfa18945ff7bdd7051"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ef80829117a8061f974b2fda8ec799717242353bff55f8a29411794d635d964"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5addc9d0209a9afca5fc070f93b726bf7003bd63a427f65ef797a931782e7edc"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c148bec483cc4b421562b4bcedb8e28a3b84fcc8f0aa4418e10898f3c2c0eb9b"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d1f21af4c1539051049796a0f50aa342f9a27cde57318f2fc41ed50b0dbc4ac"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b9ac09853b2a3e0d0082104036579809679e7715671cfbf89d83c1cb2a30f58"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ebedc192abbc7fd13c5ee800e83a6df252bec691eb2c4bedc9f8b2e2903f5e2a"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d8a993c0a0ffd5f2d3bda23d0cd75e7086736f8f8268de8a82fbc4bd0ac6791e"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:be6b7b8d42d3090b6c80793524fa66c57ad7ee3fe9722b258aec6d0672543fd0"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4023e2efc35a30e66e938de5aef42b520c20e7eda7bb5fb12c35e5d09a4c43f6"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d47840dc05e0ba04fe2e26f15126de7c755496d5a8aae4a08bda4dd8d646c54"}, + {file = "regex-2023.10.3-cp311-cp311-win32.whl", hash = "sha256:9145f092b5d1977ec8c0ab46e7b3381b2fd069957b9862a43bd383e5c01d18c2"}, + {file = "regex-2023.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:b6104f9a46bd8743e4f738afef69b153c4b8b592d35ae46db07fc28ae3d5fb7c"}, + {file = "regex-2023.10.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff507ae210371d4b1fe316d03433ac099f184d570a1a611e541923f78f05037"}, + {file = "regex-2023.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be5e22bbb67924dea15039c3282fa4cc6cdfbe0cbbd1c0515f9223186fc2ec5f"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a992f702c9be9c72fa46f01ca6e18d131906a7180950958f766c2aa294d4b41"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7434a61b158be563c1362d9071358f8ab91b8d928728cd2882af060481244c9e"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2169b2dcabf4e608416f7f9468737583ce5f0a6e8677c4efbf795ce81109d7c"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9e908ef5889cda4de038892b9accc36d33d72fb3e12c747e2799a0e806ec841"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12bd4bc2c632742c7ce20db48e0d99afdc05e03f0b4c1af90542e05b809a03d9"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bc72c231f5449d86d6c7d9cc7cd819b6eb30134bb770b8cfdc0765e48ef9c420"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bce8814b076f0ce5766dc87d5a056b0e9437b8e0cd351b9a6c4e1134a7dfbda9"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:ba7cd6dc4d585ea544c1412019921570ebd8a597fabf475acc4528210d7c4a6f"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b0c7d2f698e83f15228ba41c135501cfe7d5740181d5903e250e47f617eb4292"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5a8f91c64f390ecee09ff793319f30a0f32492e99f5dc1c72bc361f23ccd0a9a"}, + {file = "regex-2023.10.3-cp312-cp312-win32.whl", hash = "sha256:ad08a69728ff3c79866d729b095872afe1e0557251da4abb2c5faff15a91d19a"}, + {file = "regex-2023.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:39cdf8d141d6d44e8d5a12a8569d5a227f645c87df4f92179bd06e2e2705e76b"}, + {file = "regex-2023.10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4a3ee019a9befe84fa3e917a2dd378807e423d013377a884c1970a3c2792d293"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76066d7ff61ba6bf3cb5efe2428fc82aac91802844c022d849a1f0f53820502d"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe50b61bab1b1ec260fa7cd91106fa9fece57e6beba05630afe27c71259c59b"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fd88f373cb71e6b59b7fa597e47e518282455c2734fd4306a05ca219a1991b0"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ab05a182c7937fb374f7e946f04fb23a0c0699c0450e9fb02ef567412d2fa3"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dac37cf08fcf2094159922edc7a2784cfcc5c70f8354469f79ed085f0328ebdf"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e54ddd0bb8fb626aa1f9ba7b36629564544954fff9669b15da3610c22b9a0991"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3367007ad1951fde612bf65b0dffc8fd681a4ab98ac86957d16491400d661302"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:16f8740eb6dbacc7113e3097b0a36065a02e37b47c936b551805d40340fb9971"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f4f2ca6df64cbdd27f27b34f35adb640b5d2d77264228554e68deda54456eb11"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:39807cbcbe406efca2a233884e169d056c35aa7e9f343d4e78665246a332f597"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7eece6fbd3eae4a92d7c748ae825cbc1ee41a89bb1c3db05b5578ed3cfcfd7cb"}, + {file = "regex-2023.10.3-cp37-cp37m-win32.whl", hash = "sha256:ce615c92d90df8373d9e13acddd154152645c0dc060871abf6bd43809673d20a"}, + {file = "regex-2023.10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0f649fa32fe734c4abdfd4edbb8381c74abf5f34bc0b3271ce687b23729299ed"}, + {file = "regex-2023.10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b98b7681a9437262947f41c7fac567c7e1f6eddd94b0483596d320092004533"}, + {file = "regex-2023.10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:91dc1d531f80c862441d7b66c4505cd6ea9d312f01fb2f4654f40c6fdf5cc37a"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82fcc1f1cc3ff1ab8a57ba619b149b907072e750815c5ba63e7aa2e1163384a4"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7979b834ec7a33aafae34a90aad9f914c41fd6eaa8474e66953f3f6f7cbd4368"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef71561f82a89af6cfcbee47f0fabfdb6e63788a9258e913955d89fdd96902ab"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd829712de97753367153ed84f2de752b86cd1f7a88b55a3a775eb52eafe8a94"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00e871d83a45eee2f8688d7e6849609c2ca2a04a6d48fba3dff4deef35d14f07"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:706e7b739fdd17cb89e1fbf712d9dc21311fc2333f6d435eac2d4ee81985098c"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cc3f1c053b73f20c7ad88b0d1d23be7e7b3901229ce89f5000a8399746a6e039"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6f85739e80d13644b981a88f529d79c5bdf646b460ba190bffcaf6d57b2a9863"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:741ba2f511cc9626b7561a440f87d658aabb3d6b744a86a3c025f866b4d19e7f"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e77c90ab5997e85901da85131fd36acd0ed2221368199b65f0d11bca44549711"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:979c24cbefaf2420c4e377ecd1f165ea08cc3d1fbb44bdc51bccbbf7c66a2cb4"}, + {file = "regex-2023.10.3-cp38-cp38-win32.whl", hash = "sha256:58837f9d221744d4c92d2cf7201c6acd19623b50c643b56992cbd2b745485d3d"}, + {file = "regex-2023.10.3-cp38-cp38-win_amd64.whl", hash = "sha256:c55853684fe08d4897c37dfc5faeff70607a5f1806c8be148f1695be4a63414b"}, + {file = "regex-2023.10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2c54e23836650bdf2c18222c87f6f840d4943944146ca479858404fedeb9f9af"}, + {file = "regex-2023.10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:69c0771ca5653c7d4b65203cbfc5e66db9375f1078689459fe196fe08b7b4930"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ac965a998e1388e6ff2e9781f499ad1eaa41e962a40d11c7823c9952c77123e"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c0e8fae5b27caa34177bdfa5a960c46ff2f78ee2d45c6db15ae3f64ecadde14"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6c56c3d47da04f921b73ff9415fbaa939f684d47293f071aa9cbb13c94afc17d"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ef1e014eed78ab650bef9a6a9cbe50b052c0aebe553fb2881e0453717573f52"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d29338556a59423d9ff7b6eb0cb89ead2b0875e08fe522f3e068b955c3e7b59b"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9c6d0ced3c06d0f183b73d3c5920727268d2201aa0fe6d55c60d68c792ff3588"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:994645a46c6a740ee8ce8df7911d4aee458d9b1bc5639bc968226763d07f00fa"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:66e2fe786ef28da2b28e222c89502b2af984858091675044d93cb50e6f46d7af"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:11175910f62b2b8c055f2b089e0fedd694fe2be3941b3e2633653bc51064c528"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:06e9abc0e4c9ab4779c74ad99c3fc10d3967d03114449acc2c2762ad4472b8ca"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fb02e4257376ae25c6dd95a5aec377f9b18c09be6ebdefa7ad209b9137b73d48"}, + {file = "regex-2023.10.3-cp39-cp39-win32.whl", hash = "sha256:3b2c3502603fab52d7619b882c25a6850b766ebd1b18de3df23b2f939360e1bd"}, + {file = "regex-2023.10.3-cp39-cp39-win_amd64.whl", hash = "sha256:adbccd17dcaff65704c856bd29951c58a1bd4b2b0f8ad6b826dbd543fe740988"}, + {file = "regex-2023.10.3.tar.gz", hash = "sha256:3fef4f844d2290ee0ba57addcec17eec9e3df73f10a2748485dfd6a3a188cc0f"}, ] [[package]] @@ -1834,36 +1867,36 @@ torch = ["numpy (>=1.21.6)", "torch (>=1.10)"] [[package]] name = "scipy" -version = "1.11.2" +version = "1.11.3" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = "<3.13,>=3.9" files = [ - {file = "scipy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2b997a5369e2d30c97995dcb29d638701f8000d04df01b8e947f206e5d0ac788"}, - {file = "scipy-1.11.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:95763fbda1206bec41157582bea482f50eb3702c85fffcf6d24394b071c0e87a"}, - {file = "scipy-1.11.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e367904a0fec76433bf3fbf3e85bf60dae8e9e585ffd21898ab1085a29a04d16"}, - {file = "scipy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d690e1ca993c8f7ede6d22e5637541217fc6a4d3f78b3672a6fe454dbb7eb9a7"}, - {file = "scipy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d2b813bfbe8dec6a75164523de650bad41f4405d35b0fa24c2c28ae07fcefb20"}, - {file = "scipy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:afdb0d983f6135d50770dd979df50bf1c7f58b5b33e0eb8cf5c73c70600eae1d"}, - {file = "scipy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d9886f44ef8c9e776cb7527fb01455bf4f4a46c455c4682edc2c2cc8cd78562"}, - {file = "scipy-1.11.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1342ca385c673208f32472830c10110a9dcd053cf0c4b7d4cd7026d0335a6c1d"}, - {file = "scipy-1.11.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b133f237bd8ba73bad51bc12eb4f2d84cbec999753bf25ba58235e9fc2096d80"}, - {file = "scipy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aeb87661de987f8ec56fa6950863994cd427209158255a389fc5aea51fa7055"}, - {file = "scipy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:90d3b1364e751d8214e325c371f0ee0dd38419268bf4888b2ae1040a6b266b2a"}, - {file = "scipy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:f73102f769ee06041a3aa26b5841359b1a93cc364ce45609657751795e8f4a4a"}, - {file = "scipy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa4909c6c20c3d91480533cddbc0e7c6d849e7d9ded692918c76ce5964997898"}, - {file = "scipy-1.11.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:ac74b1512d38718fb6a491c439aa7b3605b96b1ed3be6599c17d49d6c60fca18"}, - {file = "scipy-1.11.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8425fa963a32936c9773ee3ce44a765d8ff67eed5f4ac81dc1e4a819a238ee9"}, - {file = "scipy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:542a757e2a6ec409e71df3d8fd20127afbbacb1c07990cb23c5870c13953d899"}, - {file = "scipy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ea932570b1c2a30edafca922345854ff2cd20d43cd9123b6dacfdecebfc1a80b"}, - {file = "scipy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:4447ad057d7597476f9862ecbd9285bbf13ba9d73ce25acfa4e4b11c6801b4c9"}, - {file = "scipy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b0620240ef445b5ddde52460e6bc3483b7c9c750275369379e5f609a1050911c"}, - {file = "scipy-1.11.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:f28f1f6cfeb48339c192efc6275749b2a25a7e49c4d8369a28b6591da02fbc9a"}, - {file = "scipy-1.11.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:214cdf04bbae7a54784f8431f976704ed607c4bc69ba0d5d5d6a9df84374df76"}, - {file = "scipy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10eb6af2f751aa3424762948e5352f707b0dece77288206f227864ddf675aca0"}, - {file = "scipy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0f3261f14b767b316d7137c66cc4f33a80ea05841b9c87ad83a726205b901423"}, - {file = "scipy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:2c91cf049ffb5575917f2a01da1da082fd24ed48120d08a6e7297dfcac771dcd"}, - {file = "scipy-1.11.2.tar.gz", hash = "sha256:b29318a5e39bd200ca4381d80b065cdf3076c7d7281c5e36569e99273867f61d"}, + {file = "scipy-1.11.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:370f569c57e1d888304052c18e58f4a927338eafdaef78613c685ca2ea0d1fa0"}, + {file = "scipy-1.11.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:9885e3e4f13b2bd44aaf2a1a6390a11add9f48d5295f7a592393ceb8991577a3"}, + {file = "scipy-1.11.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e04aa19acc324a1a076abb4035dabe9b64badb19f76ad9c798bde39d41025cdc"}, + {file = "scipy-1.11.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e1a8a4657673bfae1e05e1e1d6e94b0cabe5ed0c7c144c8aa7b7dbb774ce5c1"}, + {file = "scipy-1.11.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7abda0e62ef00cde826d441485e2e32fe737bdddee3324e35c0e01dee65e2a88"}, + {file = "scipy-1.11.3-cp310-cp310-win_amd64.whl", hash = "sha256:033c3fd95d55012dd1148b201b72ae854d5086d25e7c316ec9850de4fe776929"}, + {file = "scipy-1.11.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:925c6f09d0053b1c0f90b2d92d03b261e889b20d1c9b08a3a51f61afc5f58165"}, + {file = "scipy-1.11.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5664e364f90be8219283eeb844323ff8cd79d7acbd64e15eb9c46b9bc7f6a42a"}, + {file = "scipy-1.11.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00f325434b6424952fbb636506f0567898dca7b0f7654d48f1c382ea338ce9a3"}, + {file = "scipy-1.11.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f290cf561a4b4edfe8d1001ee4be6da60c1c4ea712985b58bf6bc62badee221"}, + {file = "scipy-1.11.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:91770cb3b1e81ae19463b3c235bf1e0e330767dca9eb4cd73ba3ded6c4151e4d"}, + {file = "scipy-1.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1f97cd89c0fe1a0685f8f89d85fa305deb3067d0668151571ba50913e445820"}, + {file = "scipy-1.11.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dfcc1552add7cb7c13fb70efcb2389d0624d571aaf2c80b04117e2755a0c5d15"}, + {file = "scipy-1.11.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:0d3a136ae1ff0883fffbb1b05b0b2fea251cb1046a5077d0b435a1839b3e52b7"}, + {file = "scipy-1.11.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bae66a2d7d5768eaa33008fa5a974389f167183c87bf39160d3fefe6664f8ddc"}, + {file = "scipy-1.11.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2f6dee6cbb0e263b8142ed587bc93e3ed5e777f1f75448d24fb923d9fd4dce6"}, + {file = "scipy-1.11.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:74e89dc5e00201e71dd94f5f382ab1c6a9f3ff806c7d24e4e90928bb1aafb280"}, + {file = "scipy-1.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:90271dbde4be191522b3903fc97334e3956d7cfb9cce3f0718d0ab4fd7d8bfd6"}, + {file = "scipy-1.11.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a63d1ec9cadecce838467ce0631c17c15c7197ae61e49429434ba01d618caa83"}, + {file = "scipy-1.11.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:5305792c7110e32ff155aed0df46aa60a60fc6e52cd4ee02cdeb67eaccd5356e"}, + {file = "scipy-1.11.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea7f579182d83d00fed0e5c11a4aa5ffe01460444219dedc448a36adf0c3917"}, + {file = "scipy-1.11.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c77da50c9a91e23beb63c2a711ef9e9ca9a2060442757dffee34ea41847d8156"}, + {file = "scipy-1.11.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:15f237e890c24aef6891c7d008f9ff7e758c6ef39a2b5df264650eb7900403c0"}, + {file = "scipy-1.11.3-cp39-cp39-win_amd64.whl", hash = "sha256:4b4bb134c7aa457e26cc6ea482b016fef45db71417d55cc6d8f43d799cdf9ef2"}, + {file = "scipy-1.11.3.tar.gz", hash = "sha256:bba4d955f54edd61899776bad459bf7326e14b9fa1c552181f0479cc60a568cd"}, ] [package.dependencies] @@ -1971,13 +2004,13 @@ mpmath = ">=0.19" [[package]] name = "texttable" -version = "1.6.7" +version = "1.7.0" description = "module to create simple ASCII tables" optional = true python-versions = "*" files = [ - {file = "texttable-1.6.7-py2.py3-none-any.whl", hash = "sha256:b7b68139aa8a6339d2c320ca8b1dc42d13a7831a346b446cb9eb385f0c76310c"}, - {file = "texttable-1.6.7.tar.gz", hash = "sha256:290348fb67f7746931bcdfd55ac7584ecd4e5b0846ab164333f0794b121760f2"}, + {file = "texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917"}, + {file = "texttable-1.7.0.tar.gz", hash = "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638"}, ] [[package]] @@ -2106,13 +2139,13 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.33.2" +version = "4.33.3" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" optional = false python-versions = ">=3.8.0" files = [ - {file = "transformers-4.33.2-py3-none-any.whl", hash = "sha256:5a9a757bea5b5a1b94796805bcb5978b552208a3ac193f46edda66be6f4a5488"}, - {file = "transformers-4.33.2.tar.gz", hash = "sha256:47dd36f302afec86d9cdcacab61bbd0296e6bb02e64d2ed7855daaab14ee290e"}, + {file = "transformers-4.33.3-py3-none-any.whl", hash = "sha256:7150bbf6781ddb3338ce7d74f4d6f557e6c236a0a1dd3de57412214caae7fd71"}, + {file = "transformers-4.33.3.tar.gz", hash = "sha256:8ea7c92310dee7c63b14766ce928218f7a9177960b2487ac018c91ae621af03e"}, ] [package.dependencies] @@ -2217,13 +2250,13 @@ files = [ [[package]] name = "urllib3" -version = "2.0.5" +version = "2.0.6" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.7" files = [ - {file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"}, - {file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"}, + {file = "urllib3-2.0.6-py3-none-any.whl", hash = "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2"}, + {file = "urllib3-2.0.6.tar.gz", hash = "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564"}, ] [package.extras] diff --git a/server/pyproject.toml b/server/pyproject.toml index c06c298a..673968be 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.1.0" +version = "1.1.1" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From dd304cf14cc342e40eea83015489d0b136f85ed0 Mon Sep 17 00:00:00 2001 From: Omar Sanseviero Date: Mon, 9 Oct 2023 11:59:06 +0200 Subject: [PATCH 456/793] Remove some content from the README in favour of the documentation (#958) --- README.md | 104 +++++++++----------------------- docs/source/index.md | 3 +- docs/source/supported_models.md | 1 - 3 files changed, 31 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index 339b5db7..2dd8551d 100644 --- a/README.md +++ b/README.md @@ -18,71 +18,43 @@ to power Hugging Chat, the Inference API and Inference Endpoint. ## Table of contents -- [Features](#features) -- [Optimized Architectures](#optimized-architectures) - [Get Started](#get-started) - - [Docker](#docker) - [API Documentation](#api-documentation) - [Using a private or gated model](#using-a-private-or-gated-model) - [A note on Shared Memory](#a-note-on-shared-memory-shm) - [Distributed Tracing](#distributed-tracing) - [Local Install](#local-install) - [CUDA Kernels](#cuda-kernels) +- [Optimized architectures](#optimized-architectures) - [Run Falcon](#run-falcon) - [Run](#run) - [Quantization](#quantization) - [Develop](#develop) - [Testing](#testing) -- [Other supported hardware](#other-supported-hardware) -## Features +Text Generation Inference (TGI) is a toolkit for deploying and serving Large Language Models (LLMs). TGI enables high-performance text generation for the most popular open-source LLMs, including Llama, Falcon, StarCoder, BLOOM, GPT-NeoX, and [more](https://huggingface.co/docs/text-generation-inference/supported_models). TGI implements many features, such as: -- Serve the most popular Large Language Models with a simple launcher +- Simple launcher to serve most popular LLMs +- Production ready (distributed tracing with Open Telemetry, Prometheus metrics) - Tensor Parallelism for faster inference on multiple GPUs - Token streaming using Server-Sent Events (SSE) -- [Continuous batching of incoming requests](https://github.com/huggingface/text-generation-inference/tree/main/router) for increased total throughput -- Optimized transformers code for inference using [flash-attention](https://github.com/HazyResearch/flash-attention) and [Paged Attention](https://github.com/vllm-project/vllm) on the most popular architectures +- Continuous batching of incoming requests for increased total throughput +- Optimized transformers code for inference using [Flash Attention](https://github.com/HazyResearch/flash-attention) and [Paged Attention](https://github.com/vllm-project/vllm) on the most popular architectures - Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) and [GPT-Q](https://arxiv.org/abs/2210.17323) - [Safetensors](https://github.com/huggingface/safetensors) weight loading - Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) - Logits warper (temperature scaling, top-p, top-k, repetition penalty, more details see [transformers.LogitsProcessor](https://huggingface.co/docs/transformers/internal/generation_utils#transformers.LogitsProcessor)) - Stop sequences - Log probabilities -- Production ready (distributed tracing with Open Telemetry, Prometheus metrics) -- Custom Prompt Generation: Easily generate text by providing custom prompts to guide the model's output. -- Fine-tuning Support: Utilize fine-tuned models for specific tasks to achieve higher accuracy and performance. - - -## Optimized architectures - -- [BLOOM](https://huggingface.co/bigscience/bloom) -- [FLAN-T5](https://huggingface.co/google/flan-t5-xxl) -- [Galactica](https://huggingface.co/facebook/galactica-120b) -- [GPT-Neox](https://huggingface.co/EleutherAI/gpt-neox-20b) -- [Llama](https://github.com/facebookresearch/llama) -- [OPT](https://huggingface.co/facebook/opt-66b) -- [SantaCoder](https://huggingface.co/bigcode/santacoder) -- [Starcoder](https://huggingface.co/bigcode/starcoder) -- [Falcon 7B](https://huggingface.co/tiiuae/falcon-7b) -- [Falcon 40B](https://huggingface.co/tiiuae/falcon-40b) -- [MPT](https://huggingface.co/mosaicml/mpt-30b) -- [Llama V2](https://huggingface.co/meta-llama) -- [Code Llama](https://huggingface.co/codellama) -- [Mistral](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) - -Other architectures are supported on a best effort basis using: +- Custom Prompt Generation: Easily generate text by providing custom prompts to guide the model's output +- Fine-tuning Support: Utilize fine-tuned models for specific tasks to achieve higher accuracy and performance -`AutoModelForCausalLM.from_pretrained(, device_map="auto")` - -or -`AutoModelForSeq2SeqLM.from_pretrained(, device_map="auto")` - -## Get started +## Get Started ### Docker -The easiest way of getting started is using the official Docker container: +For a detailed starting guide, please see the [Quick Tour](https://huggingface.co/docs/text-generation-inference/quicktour). The easiest way of getting started is using the official Docker container: ```shell model=tiiuae/falcon-7b-instruct @@ -90,46 +62,21 @@ volume=$PWD/data # share a volume with the Docker container to avoid downloading docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0 --model-id $model ``` -**Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. - -To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli): -``` -text-generation-launcher --help -``` -You can then query the model using either the `/generate` or `/generate_stream` routes: +And then you can make requests like -```shell +```bash curl 127.0.0.1:8080/generate \ -X POST \ -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ -H 'Content-Type: application/json' ``` -```shell -curl 127.0.0.1:8080/generate_stream \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ - -H 'Content-Type: application/json' -``` - -or from Python: +**Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. -```shell -pip install text-generation +To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli): ``` - -```python -from text_generation import Client - -client = Client("http://127.0.0.1:8080") -print(client.generate("What is Deep Learning?", max_new_tokens=20).generated_text) - -text = "" -for response in client.generate_stream("What is Deep Learning?", max_new_tokens=20): - if not response.token.special: - text += response.token.text -print(text) +text-generation-launcher --help ``` ### API documentation @@ -241,6 +188,20 @@ the kernels by using the `DISABLE_CUSTOM_KERNELS=True` environment variable. Be aware that the official Docker image has them enabled by default. +## Optimized architectures + +TGI works out of the box to serve optimized models in [this list](https://huggingface.co/docs/text-generation-inference/supported_models). + +Other architectures are supported on a best-effort basis using: + +`AutoModelForCausalLM.from_pretrained(, device_map="auto")` + +or + +`AutoModelForSeq2SeqLM.from_pretrained(, device_map="auto")` + + + ## Run Falcon ### Run @@ -279,10 +240,3 @@ make rust-tests # integration tests make integration-tests ``` - - -## Other supported hardware - -TGI is also supported on the following AI hardware accelerators: -- *Habana first-gen Gaudi and Gaudi2:* checkout [here](https://github.com/huggingface/optimum-habana/tree/main/text-generation-inference) how to serve models with TGI on Gaudi and Gaudi2 with [Optimum Habana](https://huggingface.co/docs/optimum/habana/index) - diff --git a/docs/source/index.md b/docs/source/index.md index 097217ad..8bf45dce 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -18,7 +18,8 @@ Text Generation Inference implements many optimizations and features, such as: - Logits warper (temperature scaling, top-p, top-k, repetition penalty) - Stop sequences - Log probabilities - +- Custom Prompt Generation: Easily generate text by providing custom prompts to guide the model's output. +- Fine-tuning Support: Utilize fine-tuned models for specific tasks to achieve higher accuracy and performance. Text Generation Inference is used in production by multiple projects, such as: diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md index 5d645759..8b4c33b1 100644 --- a/docs/source/supported_models.md +++ b/docs/source/supported_models.md @@ -45,4 +45,3 @@ TGI is also supported on the following AI hardware accelerators: - *Habana first-gen Gaudi and Gaudi2:* check out this [example](https://github.com/huggingface/optimum-habana/tree/main/text-generation-inference) how to serve models with TGI on Gaudi and Gaudi2 with [Optimum Habana](https://huggingface.co/docs/optimum/habana/index) - From 20ee71dcf55098bbef617e63ba1869ed6f206b48 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 11 Oct 2023 10:46:40 +0200 Subject: [PATCH 457/793] fix: force one of max_new_tokens or truncate with slow tokenizer --- router/src/validation.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/router/src/validation.rs b/router/src/validation.rs index 9adedc5b..d0ea137d 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -116,12 +116,16 @@ impl Validation { // In this case, we don't know the real length in tokens of the inputs // However, the inputs will be truncated by the python servers // We make sure that truncate + max_new_tokens <= self.max_total_tokens - let input_length = truncate.unwrap_or(self.max_input_length); let max_new_tokens: u32 = if let Some(max_new_tokens) = max_new_tokens { max_new_tokens } else { - self.max_total_tokens.saturating_sub(input_length) as u32 + if let Some(truncate) = truncate { + self.max_total_tokens.saturating_sub(truncate) as u32 + } else { + return Err(ValidationError::UnsetMaxNewTokens) + } }; + let input_length = truncate.unwrap_or(self.max_input_length); // Validate MaxNewTokens if (input_length as u32 + max_new_tokens) > self.max_total_tokens as u32 { @@ -393,6 +397,8 @@ pub enum ValidationError { Truncate(usize, usize), #[error("`typical_p` must be > 0.0 and < 1.0")] TypicalP, + #[error("one of `max_new_tokens` or `truncate` must be set if a fast tokenizer is not in use")] + UnsetMaxNewTokens, #[error("`max_new_tokens` must be strictly positive")] NegativeMaxNewTokens, #[error("`max_new_tokens` must be <= {0}. Given: {1}")] From 3af1a1140141245c20df20a787831ec71b124eae Mon Sep 17 00:00:00 2001 From: Mishig Date: Fri, 13 Oct 2023 09:48:35 +0200 Subject: [PATCH 458/793] Fix link in preparing_model.md (#1140) Fixes a link in doc --- docs/source/basic_tutorials/preparing_model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/basic_tutorials/preparing_model.md b/docs/source/basic_tutorials/preparing_model.md index 0f5739ea..97c9bbe0 100644 --- a/docs/source/basic_tutorials/preparing_model.md +++ b/docs/source/basic_tutorials/preparing_model.md @@ -4,7 +4,7 @@ Text Generation Inference improves the model in several aspects. ## Quantization -TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323) and [AWQ](https://arxiv.org/abs/2306.00978) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq` or `awq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq) when using AWQ quantization, you need to point to one of the models [here](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to (./conceptual/quantization.md) +TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323) and [AWQ](https://arxiv.org/abs/2306.00978) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq` or `awq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq) when using AWQ quantization, you need to point to one of the models [here](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to [quantization guide](./conceptual/quantization.md) ## RoPE Scaling From 7402a355dcbf9ffe7a0b2a788f2062aa9e0a3ed5 Mon Sep 17 00:00:00 2001 From: momonga <146910567+mmngays@users.noreply.github.com> Date: Thu, 19 Oct 2023 17:42:03 +0900 Subject: [PATCH 459/793] Fix calling cuda() on load_in_8bit (#1153) This PR addresses an issue where calling `model = model.cuda()` would throw a ValueError when `quantize` is set to "bitsandbytes". ``` > File "/opt/conda/lib/python3.9/site-packages/text_generation_server/server.py", line 147, in serve_inner model = get_model( File "/opt/conda/lib/python3.9/site-packages/text_generation_server/models/__init__.py", line 295, in get_model return CausalLM( File "/opt/conda/lib/python3.9/site-packages/text_generation_server/models/causal_lm.py", line 515, in __init__ model = model.cuda() File "/opt/conda/lib/python3.9/site-packages/transformers/modeling_utils.py", line 1998, in cuda raise ValueError( ValueError: Calling `cuda()` is not supported for `4-bit` or `8-bit` quantized models. Please use the model as it is, since the model has already been set to the correct devices and casted to the correct `dtype`. ``` Co-authored-by: mmnga --- server/text_generation_server/models/causal_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index fccfb0f8..8056a8ec 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -511,7 +511,7 @@ def __init__( load_in_8bit=quantize == "bitsandbytes", trust_remote_code=trust_remote_code, ) - if torch.cuda.is_available() and torch.cuda.device_count() == 1: + if torch.cuda.is_available() and torch.cuda.device_count() == 1 and quantize != "bitsandbytes": model = model.cuda() if tokenizer.pad_token_id is None: From 9179605e1eaa5532e81552e7e7fc92ec32652592 Mon Sep 17 00:00:00 2001 From: Mario928 <88029051+Mario928@users.noreply.github.com> Date: Thu, 19 Oct 2023 15:24:26 +0530 Subject: [PATCH 460/793] Fix: Replace view() with reshape() in neox_modeling.py to resolve RuntimeError (#1155) --- .../models/custom_modeling/neox_modeling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/neox_modeling.py b/server/text_generation_server/models/custom_modeling/neox_modeling.py index 24ba6796..dbcefbae 100644 --- a/server/text_generation_server/models/custom_modeling/neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/neox_modeling.py @@ -283,10 +283,10 @@ def _attn(self, query, key, value, attention_mask=None, head_mask=None): batch_size, num_attention_heads, query_length, attn_head_size = query.size() key_length = key.size(-2) - query = query.view( + query = query.reshape( batch_size * num_attention_heads, query_length, attn_head_size ) - key = key.view(batch_size * num_attention_heads, key_length, attn_head_size) + key = key.reshape(batch_size * num_attention_heads, key_length, attn_head_size) attn_scores = torch.zeros( 1, dtype=query.dtype, From 648ea06430366a735c92b0c688b09b022ad84438 Mon Sep 17 00:00:00 2001 From: star Date: Thu, 19 Oct 2023 18:15:05 +0800 Subject: [PATCH 461/793] fix: EETQLinear with bias in layers.py (#1176) --- server/text_generation_server/utils/layers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index f38f130e..7bb95dd2 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -155,10 +155,7 @@ def __init__( device = weight.device weight = torch.t(weight).contiguous().cpu() weight, scale = quant_weights(weight, torch.int8, False) - if bias: - bias = weights.get_tensor(f"{prefix}.bias") - else: - bias = None + self.weight = weight.cuda(device) self.scale = scale.cuda(device) self.bias = bias.cuda(device) if bias is not None else None From 72b8f88be8ab6ea9e0ea1b499a9064825c7b5dcb Mon Sep 17 00:00:00 2001 From: Remy Date: Thu, 19 Oct 2023 14:04:44 +0200 Subject: [PATCH 462/793] fix: remove useless token (#1179) This token is not used by your action. Secret is removed from the repository. --- .github/workflows/build_documentation.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml index a0f1d6f1..4d0b19a3 100644 --- a/.github/workflows/build_documentation.yml +++ b/.github/workflows/build_documentation.yml @@ -17,5 +17,4 @@ jobs: package: text-generation-inference additional_args: --not_python_module secrets: - token: ${{ secrets.HUGGINGFACE_PUSH }} - hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} \ No newline at end of file + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} From 5e28f44a834c20602d4cc18d28703e024d3bbbe0 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 20 Oct 2023 10:28:45 +0200 Subject: [PATCH 463/793] #1049 CI (#1178) See #1049 --------- Signed-off-by: Wang, Yi A Co-authored-by: Wang, Yi --- router/client/src/client.rs | 10 ++++++---- router/client/src/sharded_client.rs | 5 ++++- router/src/main.rs | 2 +- router/src/validation.rs | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/router/client/src/client.rs b/router/client/src/client.rs index d427d3a4..f8f5df95 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -103,17 +103,19 @@ impl Client { &mut self, max_input_length: u32, max_prefill_tokens: u32, + max_total_tokens: u32, ) -> Result> { let mut n_tokens = 0; let mut requests = Vec::new(); - + let mut truncate = 0; // Create requests while n_tokens < max_prefill_tokens { + truncate = min(max_input_length, max_prefill_tokens - n_tokens); requests.push(Request { id: 0, // We truncate the input on the server side to be sure that it has the correct size inputs: "_test ".to_string().repeat(max_input_length as usize), - truncate: min(max_input_length, max_prefill_tokens - n_tokens), + truncate: truncate, // Set sampling parameters to also take these ops into account in the max memory parameters: Some(NextTokenChooserParameters { temperature: 0.9, @@ -126,9 +128,9 @@ impl Client { watermark: true, }), stopping_parameters: Some(StoppingCriteriaParameters { - max_new_tokens: 2, + max_new_tokens: max_total_tokens - truncate, stop_sequences: vec![], - ignore_eos_token: false, + ignore_eos_token: true, }), prefill_logprobs: true, top_n_tokens: 20, diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 112b0035..b4bdcd42 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -95,11 +95,14 @@ impl ShardedClient { &mut self, max_input_length: u32, max_prefill_tokens: u32, + max_total_tokens: u32, ) -> Result> { let futures: Vec<_> = self .clients .iter_mut() - .map(|client| Box::pin(client.warmup(max_input_length, max_prefill_tokens))) + .map(|client| { + Box::pin(client.warmup(max_input_length, max_prefill_tokens, max_total_tokens)) + }) .collect(); // Take the minimum value let results = join_all(futures) diff --git a/router/src/main.rs b/router/src/main.rs index f3028674..d90632ef 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -212,7 +212,7 @@ fn main() -> Result<(), RouterError> { // Warmup model tracing::info!("Warming up model"); let max_supported_batch_total_tokens = match sharded_client - .warmup(max_input_length as u32, max_batch_prefill_tokens) + .warmup(max_input_length as u32, max_batch_prefill_tokens, max_total_tokens as u32) .await .map_err(RouterError::Warmup)? { diff --git a/router/src/validation.rs b/router/src/validation.rs index d0ea137d..37465272 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -122,7 +122,7 @@ impl Validation { if let Some(truncate) = truncate { self.max_total_tokens.saturating_sub(truncate) as u32 } else { - return Err(ValidationError::UnsetMaxNewTokens) + return Err(ValidationError::UnsetMaxNewTokens); } }; let input_length = truncate.unwrap_or(self.max_input_length); From 63fa5346127e03b514634078dc8c41f408279b6a Mon Sep 17 00:00:00 2001 From: Aastha Varma Date: Mon, 23 Oct 2023 15:42:21 +0530 Subject: [PATCH 464/793] Fix link to quantization page in preparing_model.md (#1187) --- docs/source/basic_tutorials/preparing_model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/basic_tutorials/preparing_model.md b/docs/source/basic_tutorials/preparing_model.md index 97c9bbe0..56124a3b 100644 --- a/docs/source/basic_tutorials/preparing_model.md +++ b/docs/source/basic_tutorials/preparing_model.md @@ -4,7 +4,7 @@ Text Generation Inference improves the model in several aspects. ## Quantization -TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323) and [AWQ](https://arxiv.org/abs/2306.00978) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq` or `awq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq) when using AWQ quantization, you need to point to one of the models [here](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to [quantization guide](./conceptual/quantization.md) +TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323) and [AWQ](https://arxiv.org/abs/2306.00978) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq` or `awq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq) when using AWQ quantization, you need to point to one of the models [here](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to [quantization guide](./../conceptual/quantization.md) ## RoPE Scaling From 12590fdccebb34f39fb85b7dae29b80fade2b6b0 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 23 Oct 2023 12:29:25 +0200 Subject: [PATCH 465/793] feat: paged attention v2 (#1183) --- server/Makefile-flash-att-v2 | 2 +- server/Makefile-vllm | 4 +- .../custom_modeling/flash_llama_modeling.py | 15 +-- .../custom_modeling/flash_mistral_modeling.py | 14 +-- .../custom_modeling/flash_neox_modeling.py | 14 +-- .../custom_modeling/flash_rw_modeling.py | 23 ++-- .../flash_santacoder_modeling.py | 15 +-- .../utils/paged_attention.py | 100 ++++++++++++++++++ 8 files changed, 126 insertions(+), 61 deletions(-) create mode 100644 server/text_generation_server/utils/paged_attention.py diff --git a/server/Makefile-flash-att-v2 b/server/Makefile-flash-att-v2 index cdea8431..583437b2 100644 --- a/server/Makefile-flash-att-v2 +++ b/server/Makefile-flash-att-v2 @@ -1,4 +1,4 @@ -flash_att_v2_commit := 601b4dc48dbe9d87c468daa2b4c0c8388b83753c +flash_att_v2_commit := 02ac572f3ffc4f402e4183aaa6824b45859d3ed3 flash-attention-v2: # Clone flash attention diff --git a/server/Makefile-vllm b/server/Makefile-vllm index 2e965da0..c601e452 100644 --- a/server/Makefile-vllm +++ b/server/Makefile-vllm @@ -1,8 +1,8 @@ -vllm_commit := 25dbff97d5a8f2ba331847237b458b2692e9ae78 +vllm_commit := f8a1e39fae05ca610be8d5a78be9d40f5274e5fc vllm: # Clone vllm - git clone https://github.com/OlivierDehaene/vllm.git + git clone https://github.com/vllm-project/vllm.git build-vllm: vllm cd vllm && git fetch && git checkout $(vllm_commit) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 7c743a88..69608e1c 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -29,11 +29,7 @@ # Flash attention imports import dropout_layer_norm -# vllm imports -import vllm_cache_ops -import vllm_attention_ops - -from text_generation_server.utils.flash_attn import attention +from text_generation_server.utils import paged_attention, flash_attn from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -269,7 +265,7 @@ def forward( self.rotary_emb(query, cos, sin) self.rotary_emb(torch.select(kv, dim=1, index=0), cos, sin) - vllm_cache_ops.reshape_and_cache( + paged_attention.reshape_and_cache( kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots ) @@ -279,7 +275,7 @@ def forward( # Prefill if cu_seqlen_prefill is not None: # flash attention - attention( + flash_attn.attention( query, torch.select(kv, dim=1, index=0), torch.select(kv, dim=1, index=1), @@ -290,9 +286,7 @@ def forward( ) # Decode else: - # kv_cache[1] => [num_blocks, num_heads, head_size, block_size] - block_size = kv_cache[1].shape[3] - vllm_attention_ops.single_query_cached_kv_attention( + paged_attention.attention( attn_output, query, kv_cache[0], @@ -301,7 +295,6 @@ def forward( self.softmax_scale, block_tables, input_lengths, - block_size, max_s, ) diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index 77b7f230..2d731406 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -29,10 +29,7 @@ # Flash attention imports import dropout_layer_norm -# vllm imports -import vllm_cache_ops -import vllm_attention_ops - +from text_generation_server.utils import paged_attention, flash_attn from text_generation_server.utils.flash_attn import attention, HAS_FLASH_ATTN_V2 from text_generation_server.utils.layers import ( TensorParallelRowLinear, @@ -272,7 +269,7 @@ def forward( else: kv_to_cache = kv - vllm_cache_ops.reshape_and_cache( + paged_attention.reshape_and_cache( kv_to_cache[:, 0], kv_to_cache[:, 1], kv_cache[0], kv_cache[1], slots ) @@ -282,7 +279,7 @@ def forward( # Prefill if cu_seqlen_prefill is not None: # flash attention - attention( + flash_attn.attention( query, torch.select(kv, dim=1, index=0), torch.select(kv, dim=1, index=1), @@ -294,9 +291,7 @@ def forward( ) # Decode else: - # kv_cache[1] => [num_blocks, num_heads, head_size, block_size] - block_size = kv_cache[1].shape[3] - vllm_attention_ops.single_query_cached_kv_attention( + paged_attention.attention( attn_output, query, kv_cache[0], @@ -305,7 +300,6 @@ def forward( self.softmax_scale, block_tables, input_lengths, - block_size, max_s, ) diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index 9dc374df..af4ba96b 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -27,10 +27,7 @@ from transformers.models.gpt_neox import GPTNeoXConfig from typing import Optional, List, Tuple -# vllm imports -import vllm_cache_ops -import vllm_attention_ops - +from text_generation_server.utils import paged_attention, flash_attn from text_generation_server.utils.flash_attn import attention from text_generation_server.utils.layers import ( TensorParallelRowLinear, @@ -141,7 +138,7 @@ def forward( self.rotary_emb(qkv[:, 0], cos, sin) self.rotary_emb(qkv[:, 1], cos, sin) - vllm_cache_ops.reshape_and_cache( + paged_attention.reshape_and_cache( qkv[:, 1], qkv[:, 2], kv_cache[0], kv_cache[1], slots ) @@ -151,7 +148,7 @@ def forward( # Prefill if cu_seqlen_prefill is not None: # flash attention - attention( + flash_attn.attention( qkv[:, 0], qkv[:, 1], qkv[:, 2], @@ -162,9 +159,7 @@ def forward( ) # Decode else: - # kv_cache[1] => [num_blocks, num_heads, head_size, block_size] - block_size = kv_cache[1].shape[3] - vllm_attention_ops.single_query_cached_kv_attention( + paged_attention.attention( attn_output, qkv[:, 0], kv_cache[0], @@ -173,7 +168,6 @@ def forward( self.softmax_scale, block_tables, input_lengths, - block_size, max_s, ) diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 8419fa4f..00f953a6 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -6,10 +6,7 @@ from transformers.configuration_utils import PretrainedConfig from typing import Optional, List, Tuple -# vllm imports -import vllm_cache_ops -import vllm_attention_ops - +from text_generation_server.utils import paged_attention, flash_attn from text_generation_server.utils.flash_attn import attention from text_generation_server.utils.layers import ( TensorParallelRowLinear, @@ -191,7 +188,7 @@ def forward( self.rotary_emb(query, cos, sin) self.rotary_emb(torch.select(kv, dim=1, index=0), cos, sin) - vllm_cache_ops.reshape_and_cache( + paged_attention.reshape_and_cache( kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots ) @@ -201,7 +198,7 @@ def forward( # Prefill if cu_seqlen_prefill is not None: # flash attention - attention( + flash_attn.attention( query, torch.select(kv, dim=1, index=0), torch.select(kv, dim=1, index=1), @@ -212,9 +209,7 @@ def forward( ) # Decode else: - # kv_cache[1] => [num_blocks, num_heads_kv, head_size, block_size] - block_size = kv_cache[1].shape[3] - vllm_attention_ops.single_query_cached_kv_attention( + paged_attention.attention( attn_output, query, kv_cache[0], @@ -223,7 +218,6 @@ def forward( self.softmax_scale, block_tables, input_lengths, - block_size, max_s, ) @@ -310,7 +304,7 @@ def forward( self.rotary_emb(query, cos, sin) self.rotary_emb(torch.select(kv, dim=2, index=0), cos, sin) - vllm_cache_ops.reshape_and_cache( + paged_attention.reshape_and_cache( kv[:, :, 0].contiguous(), kv[:, :, 1].contiguous(), kv_cache[0], @@ -324,7 +318,7 @@ def forward( # Prefill if cu_seqlen_prefill is not None: # flash attention - attention( + flash_attn.attention( query, torch.select(kv, dim=2, index=0), torch.select(kv, dim=2, index=1), @@ -335,9 +329,7 @@ def forward( ) # Decode else: - # kv_cache[1] => [num_blocks, num_groups, head_size, block_size] - block_size = kv_cache[1].shape[3] - vllm_attention_ops.single_query_cached_kv_attention( + paged_attention.attention( attn_output, query, kv_cache[0], @@ -346,7 +338,6 @@ def forward( self.softmax_scale, block_tables, input_lengths, - block_size, max_s, ) diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 2dd0a5ee..c3c7617a 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -5,10 +5,7 @@ from transformers.activations import ACT2FN from typing import Optional, List, Tuple -# vllm imports -import vllm_cache_ops -import vllm_attention_ops - +from text_generation_server.utils import paged_attention, flash_attn from text_generation_server.utils.flash_attn import attention from text_generation_server.utils.layers import ( TensorParallelRowLinear, @@ -18,7 +15,6 @@ FastLayerNorm, get_linear, ) -from safetensors import SafetensorError def load_multi_mqa( @@ -258,7 +254,7 @@ def forward( query = query.view(-1, self.num_heads, self.head_size) key_value = key_value.view(-1, 2, 1, self.head_size) - vllm_cache_ops.reshape_and_cache( + paged_attention.reshape_and_cache( key_value[:, 0], key_value[:, 1], kv_cache[0], kv_cache[1], slots ) @@ -268,7 +264,7 @@ def forward( # Prefill if cu_seqlen_prefill is not None: # flash attention - attention( + flash_attn.attention( query, torch.select(key_value, dim=1, index=0), torch.select(key_value, dim=1, index=1), @@ -279,9 +275,7 @@ def forward( ) # Decode else: - # kv_cache[1] => [num_blocks, 1, head_size, block_size] - block_size = kv_cache[1].shape[3] - vllm_attention_ops.single_query_cached_kv_attention( + paged_attention.attention( attn_output, query, kv_cache[0], @@ -290,7 +284,6 @@ def forward( self.softmax_scale, block_tables, input_lengths, - block_size, max_s, ) diff --git a/server/text_generation_server/utils/paged_attention.py b/server/text_generation_server/utils/paged_attention.py new file mode 100644 index 00000000..57a59599 --- /dev/null +++ b/server/text_generation_server/utils/paged_attention.py @@ -0,0 +1,100 @@ +import torch + +# vllm imports +from vllm import cache_ops +from vllm import attention_ops + +_PARTITION_SIZE = 512 + + +def reshape_and_cache(key: torch.Tensor, value: torch.Tensor, key_cache: torch.Tensor, value_cache: torch.Tensor, + slots: torch.Tensor): + cache_ops.reshape_and_cache( + key, value, key_cache, value_cache, slots + ) + + +def attention( + out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + kv_head_mapping: torch.Tensor, + softmax_scale: float, + block_tables: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, +): + # Adapted from: https://github.com/vllm-project/vllm/blob/f8a1e39fae05ca610be8d5a78be9d40f5274e5fc/vllm/model_executor/layers/attention.py + # Copyright 2023 The vLLM team. All rights + # reserved. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # + + # value_cache => [num_blocks, num_heads, head_size, block_size] + block_size = value_cache.shape[3] + num_seqs, num_heads, head_size = query.shape + max_num_partitions = ( + (max_s + _PARTITION_SIZE - 1) // + _PARTITION_SIZE) + # NOTE(woosuk): We use a simple heuristic to decide whether to use + # PagedAttention V1 or V2. If the number of partitions is 1, we use + # V1 to avoid the overhead of reduction. Also, if the number of + # sequences or heads is large, we use V1 since there is enough work + # to parallelize. + use_v1 = max_num_partitions == 1 or num_seqs * num_heads > 512 + if use_v1: + attention_ops.paged_attention_v1( + out, + query, + key_cache, + value_cache, + kv_head_mapping, + softmax_scale, + block_tables, + input_lengths, + block_size, + max_s, + None, + ) + else: + # Run PagedAttention V2. + assert _PARTITION_SIZE % block_size == 0 + tmp_output = torch.empty( + size=(num_seqs, num_heads, max_num_partitions, head_size), + dtype=out.dtype, + device=out.device, + ) + exp_sums = torch.empty( + size=(num_seqs, num_heads, max_num_partitions), + dtype=torch.float32, + device=out.device, + ) + max_logits = torch.empty_like(exp_sums) + attention_ops.paged_attention_v2( + out, + exp_sums, + max_logits, + tmp_output, + query, + key_cache, + value_cache, + kv_head_mapping, + softmax_scale, + block_tables, + input_lengths, + block_size, + max_s, + None, + ) From f9910d13e296989f41e714c43eb60ce051359db3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 23 Oct 2023 15:51:12 +0200 Subject: [PATCH 466/793] feat: remove flume (#1184) --- Cargo.lock | 44 ++++------------------------ router/Cargo.toml | 2 +- router/client/src/client.rs | 5 ++-- router/src/infer.rs | 58 ++++++++++++++++--------------------- router/src/queue.rs | 18 ++++++------ router/src/validation.rs | 44 ++++++++++++++++++++-------- 6 files changed, 75 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fa7b726..b1f7279a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -743,18 +743,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853" -[[package]] -name = "flume" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" -dependencies = [ - "futures-core", - "futures-sink", - "nanorand", - "spin 0.9.8", -] - [[package]] name = "fnv" version = "1.0.7" @@ -900,10 +888,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi", - "wasm-bindgen", ] [[package]] @@ -1508,15 +1494,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "nanorand" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" -dependencies = [ - "getrandom", -] - [[package]] name = "native-tls" version = "0.2.11" @@ -2313,7 +2290,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin 0.5.2", + "spin", "untrusted", "web-sys", "winapi", @@ -2678,15 +2655,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - [[package]] name = "spm_precompiled" version = "0.1.4" @@ -2808,7 +2776,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "1.1.0" +version = "1.1.1" dependencies = [ "average", "clap", @@ -2829,7 +2797,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.1.0" +version = "1.1.1" dependencies = [ "futures", "grpc-metadata", @@ -2845,7 +2813,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.1.0" +version = "1.1.1" dependencies = [ "clap", "ctrlc", @@ -2861,13 +2829,12 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.1.0" +version = "1.1.1" dependencies = [ "async-stream", "axum", "axum-tracing-opentelemetry", "clap", - "flume", "futures", "hf-hub 0.3.1", "init-tracing-opentelemetry", @@ -2885,6 +2852,7 @@ dependencies = [ "thiserror", "tokenizers", "tokio", + "tokio-stream", "tower-http", "tracing", "tracing-opentelemetry", diff --git a/router/Cargo.toml b/router/Cargo.toml index 87b5a8d3..55af635a 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -20,7 +20,6 @@ axum = { version = "0.6.20", features = ["json"] } axum-tracing-opentelemetry = "0.14.1" text-generation-client = { path = "client" } clap = { version = "4.4.5", features = ["derive", "env"] } -flume = "0.11.0" futures = "0.3.28" metrics = "0.21.1" metrics-exporter-prometheus = { version = "0.12.1", features = [] } @@ -34,6 +33,7 @@ serde_json = "1.0.107" thiserror = "1.0.48" tokenizers = { version = "0.14.0", features = ["http"] } tokio = { version = "1.32.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } +tokio-stream = "0.1.14" tower-http = { version = "0.4.4", features = ["cors"] } tracing = "0.1.37" tracing-opentelemetry = "0.21.0" diff --git a/router/client/src/client.rs b/router/client/src/client.rs index f8f5df95..341e70fd 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -107,15 +107,14 @@ impl Client { ) -> Result> { let mut n_tokens = 0; let mut requests = Vec::new(); - let mut truncate = 0; // Create requests while n_tokens < max_prefill_tokens { - truncate = min(max_input_length, max_prefill_tokens - n_tokens); + let truncate = min(max_input_length, max_prefill_tokens - n_tokens); requests.push(Request { id: 0, // We truncate the input on the server side to be sure that it has the correct size inputs: "_test ".to_string().repeat(max_input_length as usize), - truncate: truncate, + truncate, // Set sampling parameters to also take these ops into account in the max memory parameters: Some(NextTokenChooserParameters { temperature: 0.9, diff --git a/router/src/infer.rs b/router/src/infer.rs index 787ccfcf..cc34c466 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -2,22 +2,21 @@ use crate::validation::{Validation, ValidationError}; use crate::{Entry, Queue, Token}; use crate::{GenerateRequest, PrefillToken}; -use flume::r#async::RecvStream; -use flume::SendTimeoutError; use futures::future::try_join_all; -use futures::stream::StreamExt; use nohash_hasher::IntMap; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, }; -use std::time::Duration; use text_generation_client::{ Batch, CachedBatch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, }; use thiserror::Error; -use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; +use tokio::sync::mpsc::error::SendError; +use tokio::sync::{mpsc, Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; use tokio::time::Instant; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tokio_stream::StreamExt; use tracing::{info_span, instrument, Instrument, Span}; /// Inference struct @@ -90,7 +89,7 @@ impl Infer { ) -> Result< ( OwnedSemaphorePermit, - RecvStream>, + UnboundedReceiverStream>, ), InferError, > { @@ -113,7 +112,7 @@ impl Infer { })?; // MPSC channel to communicate with the background batching task - let (response_tx, response_rx) = flume::unbounded(); + let (response_tx, response_rx) = mpsc::unbounded_channel(); // Append the request to the queue self.queue.append(Entry { @@ -130,7 +129,7 @@ impl Infer { self.shared.batching_task.notify_one(); // Return stream - Ok((permit, response_rx.into_stream())) + Ok((permit, UnboundedReceiverStream::new(response_rx))) } /// Add a new request to the queue and return a InferResponse @@ -493,10 +492,7 @@ fn filter_send_generations(generations: Vec, entries: &mut IntMap "dropped"); err }).unwrap_or(true); @@ -510,9 +506,10 @@ fn filter_send_generations(generations: Vec, entries: &mut IntMap Result>>> { +) -> Result>>> { // Return directly if the channel is disconnected - if entry.response_tx.is_disconnected() { + if entry.response_tx.is_closed() { + metrics::increment_counter!("tgi_request_failure", "err" => "dropped"); return Ok(true); } @@ -520,10 +517,9 @@ fn send_responses( if let Some(prefill_tokens) = generation.prefill_tokens { // Send message - entry.response_tx.send_timeout( - Ok(InferStreamResponse::Prefill(prefill_tokens)), - Duration::from_millis(10), - )?; + entry + .response_tx + .send(Ok(InferStreamResponse::Prefill(prefill_tokens)))?; } // Create last Token @@ -558,22 +554,18 @@ fn send_responses( // Generation has ended stopped = true; // Send message - entry.response_tx.send_timeout( - Ok(InferStreamResponse::End { - token, - top_tokens, - generated_text, - queued: entry.queue_time, - start: entry.batch_time.unwrap(), - }), - Duration::from_millis(10), - )?; + entry.response_tx.send(Ok(InferStreamResponse::End { + token, + top_tokens, + generated_text, + queued: entry.queue_time, + start: entry.batch_time.unwrap(), + }))?; } else { // Send message - entry.response_tx.send_timeout( - Ok(InferStreamResponse::Intermediate { token, top_tokens }), - Duration::from_millis(10), - )?; + entry + .response_tx + .send(Ok(InferStreamResponse::Intermediate { token, top_tokens }))?; } Ok(stopped) } @@ -591,7 +583,7 @@ fn send_errors(error: ClientError, entries: &mut IntMap) { // unwrap_or is valid here as we don't care if the receiver is gone. entry .response_tx - .send_timeout(Err(err), Duration::from_millis(10)) + .send(Err(err)) .unwrap_or(()); }); } diff --git a/router/src/queue.rs b/router/src/queue.rs index 1ab9eb11..bbb8db0e 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -5,7 +5,7 @@ use nohash_hasher::{BuildNoHashHasher, IntMap}; use std::cmp::min; use std::collections::VecDeque; use text_generation_client::{Batch, Request}; -use tokio::sync::oneshot; +use tokio::sync::{mpsc, oneshot}; use tokio::time::Instant; use tracing::{info_span, instrument, Span}; @@ -15,7 +15,7 @@ pub(crate) struct Entry { /// Request pub request: ValidGenerateRequest, /// Response sender to communicate between the Infer struct and the batching_task - pub response_tx: flume::Sender>, + pub response_tx: mpsc::UnboundedSender>, /// Span that will live as long as entry pub span: Span, /// Temporary span used as a guard when logging inference, wait times... @@ -30,13 +30,13 @@ pub(crate) struct Entry { #[derive(Debug, Clone)] pub(crate) struct Queue { /// Channel to communicate with the background queue task - queue_sender: flume::Sender, + queue_sender: mpsc::UnboundedSender, } impl Queue { pub(crate) fn new(requires_padding: bool, block_size: u32, window_size: Option) -> Self { // Create channel - let (queue_sender, queue_receiver) = flume::unbounded(); + let (queue_sender, queue_receiver) = mpsc::unbounded_channel(); // Launch background queue task tokio::spawn(queue_task( @@ -91,11 +91,11 @@ async fn queue_task( requires_padding: bool, block_size: u32, window_size: Option, - receiver: flume::Receiver, + mut receiver: mpsc::UnboundedReceiver, ) { let mut state = State::new(requires_padding, block_size, window_size); - while let Ok(cmd) = receiver.recv_async().await { + while let Some(cmd) = receiver.recv().await { match cmd { QueueCommand::Append(entry, span) => { span.in_scope(|| state.append(*entry)); @@ -195,7 +195,7 @@ impl State { while let Some((id, mut entry)) = self.entries.pop_front() { // Filter entries where the response receiver was dropped (== entries where the request // was dropped by the client) - if entry.response_tx.is_disconnected() { + if entry.response_tx.is_closed() { metrics::increment_counter!("tgi_request_failure", "err" => "dropped"); continue; } @@ -321,9 +321,9 @@ mod tests { fn default_entry() -> ( Entry, - flume::Receiver>, + mpsc::UnboundedReceiver>, ) { - let (response_tx, receiver_tx) = flume::unbounded(); + let (response_tx, receiver_tx) = mpsc::unbounded_channel(); let entry = Entry { request: ValidGenerateRequest { diff --git a/router/src/validation.rs b/router/src/validation.rs index 37465272..7a84640d 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -6,6 +6,7 @@ use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParamet use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokenizers::TruncationDirection; +use tokio::sync::mpsc; use tokio::sync::oneshot; use tracing::{instrument, Span}; @@ -19,7 +20,7 @@ pub struct Validation { max_input_length: usize, max_total_tokens: usize, /// Channel to communicate with the background tokenization task - sender: Option>, + sender: Option>, } impl Validation { @@ -34,19 +35,25 @@ impl Validation { ) -> Self { // If we have a fast tokenizer let sender = if let Some(tokenizer) = tokenizer { - // Create channel - let (validation_sender, validation_receiver) = flume::unbounded(); + // Create round robin channel + let (validation_sender, validation_round_robin_receiver) = mpsc::unbounded_channel(); + let mut senders = Vec::with_capacity(workers); // Create workers for _ in 0..workers { let tokenizer_clone = tokenizer.clone(); - let receiver_clone = validation_receiver.clone(); + let (tokenizer_sender, tokenizer_receiver) = mpsc::unbounded_channel(); + senders.push(tokenizer_sender); // Spawn worker tokio::task::spawn_blocking(move || { - tokenizer_worker(tokenizer_clone, receiver_clone) + tokenizer_worker(tokenizer_clone, tokenizer_receiver) }); } + + // Create tokenization round robin task + tokio::spawn(round_robin_task(validation_round_robin_receiver, senders)); + Some(validation_sender) } else { None @@ -118,12 +125,10 @@ impl Validation { // We make sure that truncate + max_new_tokens <= self.max_total_tokens let max_new_tokens: u32 = if let Some(max_new_tokens) = max_new_tokens { max_new_tokens + } else if let Some(truncate) = truncate { + self.max_total_tokens.saturating_sub(truncate) as u32 } else { - if let Some(truncate) = truncate { - self.max_total_tokens.saturating_sub(truncate) as u32 - } else { - return Err(ValidationError::UnsetMaxNewTokens); - } + return Err(ValidationError::UnsetMaxNewTokens); }; let input_length = truncate.unwrap_or(self.max_input_length); @@ -309,10 +314,25 @@ impl Validation { } } +/// Round robin tokenization task +async fn round_robin_task( + mut receiver: mpsc::UnboundedReceiver, + senders: Vec>, +) { + loop { + for sender in &senders { + match receiver.recv().await { + None => return, + Some(request) => sender.send(request).unwrap(), + }; + } + } +} + /// Start tokenization workers -fn tokenizer_worker(tokenizer: Tokenizer, receiver: flume::Receiver) { +fn tokenizer_worker(tokenizer: Tokenizer, mut receiver: mpsc::UnboundedReceiver) { // Loop over requests - while let Ok(((inputs, truncate), response_tx, parent_span)) = receiver.recv() { + while let Some(((inputs, truncate), response_tx, parent_span)) = receiver.blocking_recv() { parent_span.in_scope(|| { response_tx .send(prepare_input(inputs, truncate, &tokenizer)) From 96a982ad8fc232479384476b1596a880697cc1d0 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 25 Oct 2023 10:18:58 +0200 Subject: [PATCH 467/793] fix: better warmup error --- server/text_generation_server/models/flash_causal_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 1fe40c0c..f1a4854f 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -670,7 +670,7 @@ def warmup(self, batch: FlashCausalLMBatch): self.device, ) _, batch = self.generate_token(batch) - except Exception as e: + except torch.cuda.OutOfMemoryError as e: raise RuntimeError( f"Not enough memory to handle {len(batch.input_ids)} prefill tokens. " f"You need to decrease `--max-batch-prefill-tokens`" From 414a911b34960b49afc699573defa051f1734b9d Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 7 Nov 2023 01:01:40 +0100 Subject: [PATCH 468/793] Adding the video -> moving the architecture picture lower (#1239) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2dd8551d..84bca08a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@
-![image](https://github.com/huggingface/text-generation-inference/assets/3841370/38ba1531-ea0d-4851-b31a-a6d4ddc944b0) + + # Text Generation Inference @@ -136,6 +137,10 @@ this will impact performance. `text-generation-inference` is instrumented with distributed tracing using OpenTelemetry. You can use this feature by setting the address to an OTLP collector with the `--otlp-endpoint` argument. +### Architecture + +![image](https://github.com/huggingface/text-generation-inference/assets/3841370/38ba1531-ea0d-4851-b31a-a6d4ddc944b0) + ### Local install You can also opt to install `text-generation-inference` locally. From b9184093d92c61dd4d42486f26119eea23e7bd36 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 7 Nov 2023 10:13:09 +0100 Subject: [PATCH 469/793] Narsil patch 1 (#1241) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 84bca08a..ff222647 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@
- - - + + + Making TGI deployment optimal + # Text Generation Inference From 7b5c16748707e37a3c68ba5af670ecfe3d8b87be Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 7 Nov 2023 10:24:53 +0100 Subject: [PATCH 470/793] Update README.md (#1242) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ff222647..a91caaba 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@
- Making TGI deployment optimal + Making TGI deployment optimal # Text Generation Inference From a5def7c222174e03d815f890093584f3e815c5ce Mon Sep 17 00:00:00 2001 From: Omar Sanseviero Date: Wed, 8 Nov 2023 10:34:38 -0600 Subject: [PATCH 471/793] Fix link in quantization guide (#1246) --- docs/source/basic_tutorials/preparing_model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/basic_tutorials/preparing_model.md b/docs/source/basic_tutorials/preparing_model.md index 56124a3b..ea74d18c 100644 --- a/docs/source/basic_tutorials/preparing_model.md +++ b/docs/source/basic_tutorials/preparing_model.md @@ -4,7 +4,7 @@ Text Generation Inference improves the model in several aspects. ## Quantization -TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323) and [AWQ](https://arxiv.org/abs/2306.00978) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq` or `awq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq) when using AWQ quantization, you need to point to one of the models [here](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to [quantization guide](./../conceptual/quantization.md) +TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323) and [AWQ](https://arxiv.org/abs/2306.00978) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq` or `awq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq) when using AWQ quantization, you need to point to one of the models [here](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to [quantization guide](./../conceptual/quantization) ## RoPE Scaling From 457e72c386611c4cd6c0bacdd5545a221ace9dcb Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:54:58 +0100 Subject: [PATCH 472/793] v1.1.1 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a91caaba..4fcef3dc 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ For a detailed starting guide, please see the [Quick Tour](https://huggingface.c model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.1 --model-id $model ``` And then you can make requests like @@ -104,7 +104,7 @@ model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= -docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0 --model-id $model +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.1 --model-id $model ``` ### A note on Shared Memory (shm) From 8acdc1fae79053ae08f7cf809e1d6331f3a6a8c8 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 16 Nov 2023 18:35:09 +0100 Subject: [PATCH 473/793] hotfix 1.1.1 --- .github/workflows/build.yaml | 38 ------------------------------------ 1 file changed, 38 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 124e6a33..11a95f4b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -79,11 +79,6 @@ jobs: install: true - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v4.4.1 - - name: Install cosign - if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 - with: - cosign-release: 'v1.13.1' - name: Tailscale uses: tailscale/github-action@7bd8039bf25c23c4ab1b8d6e2cc2da2280601966 with: @@ -150,39 +145,6 @@ jobs: labels: ${{ steps.meta.outputs.labels || steps.meta-pr.outputs.labels }} cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min - # Sign the resulting Docker image digest except on PRs. - # This will only write to the public Rekor transparency log when the Docker - # repository is public to avoid leaking data. - - name: Sign the published Docker image - if: ${{ github.event_name != 'pull_request' }} - env: - COSIGN_EXPERIMENTAL: "true" - # This step uses the identity token to provision an ephemeral certificate - # against the sigstore community Fulcio instance. - run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} - - name: Run Trivy in GitHub SBOM mode and submit results to Dependency Graph - uses: aquasecurity/trivy-action@master - if: ${{ github.event_name != 'pull_request' }} - with: - image-ref: 'ghcr.io/huggingface/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}' - format: 'github' - output: 'dependency-results.sbom.json' - github-pat: ${{ secrets.GITHUB_TOKEN }} - scanners: 'vuln' - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - if: ${{ github.event_name != 'pull_request' }} - with: - image-ref: 'ghcr.io/huggingface/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }}' - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL' - scanners: 'vuln' - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - if: ${{ github.event_name != 'pull_request' }} - with: - sarif_file: 'trivy-results.sarif' integration-tests: concurrency: From 3dbc649b11d14955a3d3448ed6db373a0563dfe9 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 20 Nov 2023 10:33:44 +0100 Subject: [PATCH 474/793] fix: do not leak inputs on error (#1228) Close #1225 --- router/src/infer.rs | 6 +++--- router/src/validation.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index cc34c466..aa6dc664 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -82,7 +82,7 @@ impl Infer { } /// Add a new request to the queue and return a stream of InferStreamResponse - #[instrument(skip(self))] + #[instrument(skip_all)] pub(crate) async fn generate_stream( &self, request: GenerateRequest, @@ -133,7 +133,7 @@ impl Infer { } /// Add a new request to the queue and return a InferResponse - #[instrument(skip(self))] + #[instrument(skip_all)] pub(crate) async fn generate( &self, request: GenerateRequest, @@ -214,7 +214,7 @@ impl Infer { } /// Add best_of new requests to the queue and return a InferResponse of the sequence with /// the highest log probability per token - #[instrument(skip(self))] + #[instrument(skip(self, request))] pub(crate) async fn generate_best_of( &self, request: GenerateRequest, diff --git a/router/src/validation.rs b/router/src/validation.rs index 7a84640d..1b47fc97 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -69,7 +69,7 @@ impl Validation { } } - #[instrument(skip_all)] + #[instrument(skip(self, inputs))] async fn validate_input( &self, inputs: String, From b226e469c95d5abe4a67ca0aff758b49338654b0 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 21 Nov 2023 10:39:18 +0100 Subject: [PATCH 475/793] Update README.md (#1272) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4fcef3dc..e4f90d53 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Text Generation Inference (TGI) is a toolkit for deploying and serving Large Lan For a detailed starting guide, please see the [Quick Tour](https://huggingface.co/docs/text-generation-inference/quicktour). The easiest way of getting started is using the official Docker container: ```shell -model=tiiuae/falcon-7b-instruct +model=HuggingFaceH4/zephyr-7b-beta volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.1 --model-id $model From 91111a0dc2936741631bdd3c2be8fd354ac4c0e3 Mon Sep 17 00:00:00 2001 From: Diwank Singh Tomer Date: Thu, 23 Nov 2023 03:41:05 -0800 Subject: [PATCH 476/793] Fix missing `trust_remote_code` flag for AutoTokenizer in utils.peft (#1270) Peft loading function was missing the `trust_remote_code=trust_remote_code` argument causing the custom tokenizer code to be not found. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @Narsil --- server/text_generation_server/utils/peft.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/peft.py b/server/text_generation_server/utils/peft.py index e37447dc..d37e8940 100644 --- a/server/text_generation_server/utils/peft.py +++ b/server/text_generation_server/utils/peft.py @@ -38,7 +38,7 @@ def download_and_unload_peft(model_id, revision, trust_remote_code): os.makedirs(model_id, exist_ok=True) cache_dir = model_id logger.info(f"Saving the newly created merged model to {cache_dir}") - tokenizer = AutoTokenizer.from_pretrained(base_model_id) + tokenizer = AutoTokenizer.from_pretrained(base_model_id, trust_remote_code=trust_remote_code) model.save_pretrained(cache_dir, safe_serialization=True) model.config.save_pretrained(cache_dir) tokenizer.save_pretrained(cache_dir) From e12c34bd25c8297c1df33493cbcedef8ffa1306f Mon Sep 17 00:00:00 2001 From: Traun Leyden Date: Thu, 23 Nov 2023 12:56:17 +0100 Subject: [PATCH 477/793] Load PEFT weights from local directory (#1260) # What does this PR do? Enables PEFT weights to be loaded from a local directory, as opposed to a hf hub repository. It is a continuation of the work in PR https://github.com/huggingface/text-generation-inference/pull/762 Fixes #1259 ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? **Yes but I don't know how to run the tests for this repo, and it doesn't look like this code is covered anyway** - [x] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. **Yes, @Narsil asked for a PR in [this comment](https://github.com/huggingface/text-generation-inference/pull/762#issuecomment-1728089505)** - [x] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). **I didn't see any documentation added to the [original PR](https://github.com/huggingface/text-generation-inference/pull/762), and am not sure where this belongs. Let me know and I can add some** - [x] Did you write any new necessary tests? **I didn't see any existing test coverage for this python module** ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @Narsil --------- Co-authored-by: Nicolas Patry --- server/text_generation_server/cli.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 301acb6b..b741a84c 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -150,6 +150,17 @@ def download_weights( if not extension == ".safetensors" or not auto_convert: raise e + else: + # Try to load as a local PEFT model + try: + utils.download_and_unload_peft( + model_id, revision, trust_remote_code=trust_remote_code + ) + utils.weight_files(model_id, revision, extension) + return + except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError): + pass + # Try to see if there are local pytorch weights try: # Get weights for a local model, a hub cached model and inside the WEIGHTS_CACHE_OVERRIDE From 35509ff5dee2035de8ade3372a6e1bc2d032d4e7 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 23 Nov 2023 13:38:50 +0100 Subject: [PATCH 478/793] chore: update to torch 2.1.0 (#1182) Close #1142 --- Dockerfile | 39 +- integration-tests/conftest.py | 1 + server/Makefile | 10 +- server/poetry.lock | 1938 ++++++++++++++++++--------------- server/pyproject.toml | 8 +- server/requirements.txt | 70 +- 6 files changed, 1133 insertions(+), 933 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9c15f023..cf5e0ed6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,13 +37,13 @@ RUN cargo build --release # Python builder # Adapted from: https://github.com/pytorch/pytorch/blob/master/Dockerfile -FROM debian:bullseye-slim as pytorch-install +FROM nvidia/cuda:12.1.0-devel-ubuntu20.04 as pytorch-install -ARG PYTORCH_VERSION=2.0.1 -ARG PYTHON_VERSION=3.9 +ARG PYTORCH_VERSION=2.1.1 +ARG PYTHON_VERSION=3.10 # Keep in sync with `server/pyproject.toml -ARG CUDA_VERSION=11.8 -ARG MAMBA_VERSION=23.1.0-1 +ARG CUDA_VERSION=12.1 +ARG MAMBA_VERSION=23.3.1-1 ARG CUDA_CHANNEL=nvidia ARG INSTALL_CHANNEL=pytorch # Automatically set by buildx @@ -75,20 +75,19 @@ RUN chmod +x ~/mambaforge.sh && \ RUN case ${TARGETPLATFORM} in \ "linux/arm64") exit 1 ;; \ *) /opt/conda/bin/conda update -y conda && \ - /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y "python=${PYTHON_VERSION}" pytorch==$PYTORCH_VERSION "pytorch-cuda=$(echo $CUDA_VERSION | cut -d'.' -f 1-2)" ;; \ + /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y "python=${PYTHON_VERSION}" "pytorch=$PYTORCH_VERSION" "pytorch-cuda=$(echo $CUDA_VERSION | cut -d'.' -f 1-2)" ;; \ esac && \ /opt/conda/bin/conda clean -ya # CUDA kernels builder image FROM pytorch-install as kernel-builder +ARG MAX_JOBS=8 + RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ninja-build \ && rm -rf /var/lib/apt/lists/* -RUN /opt/conda/bin/conda install -c "nvidia/label/cuda-11.8.0" cuda==11.8 && \ - /opt/conda/bin/conda clean -ya - # Build Flash Attention CUDA kernels FROM kernel-builder as flash-att-builder @@ -148,7 +147,7 @@ COPY server/Makefile-vllm Makefile RUN make build-vllm # Text Generation Inference base image -FROM nvidia/cuda:11.8.0-base-ubuntu20.04 as base +FROM nvidia/cuda:12.1.0-base-ubuntu20.04 as base # Conda env ENV PATH=/opt/conda/bin:$PATH \ @@ -172,24 +171,24 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins COPY --from=pytorch-install /opt/conda /opt/conda # Copy build artifacts from flash attention builder -COPY --from=flash-att-builder /usr/src/flash-attention/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages -COPY --from=flash-att-builder /usr/src/flash-attention/csrc/layer_norm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages -COPY --from=flash-att-builder /usr/src/flash-attention/csrc/rotary/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=flash-att-builder /usr/src/flash-attention/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages +COPY --from=flash-att-builder /usr/src/flash-attention/csrc/layer_norm/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages +COPY --from=flash-att-builder /usr/src/flash-attention/csrc/rotary/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Copy build artifacts from flash attention v2 builder -COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Copy build artifacts from custom kernels builder -COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Copy build artifacts from exllama kernels builder -COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Copy build artifacts from awq kernels builder -COPY --from=awq-kernels-builder /usr/src/llm-awq/awq/kernels/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=awq-kernels-builder /usr/src/llm-awq/awq/kernels/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Copy build artifacts from eetq kernels builder -COPY --from=eetq-kernels-builder /usr/src/eetq/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=eetq-kernels-builder /usr/src/eetq/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Copy builds artifacts from vllm builder -COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-39 /opt/conda/lib/python3.9/site-packages +COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Install flash-attention dependencies RUN pip install einops --no-cache-dir @@ -201,7 +200,7 @@ COPY server/Makefile server/Makefile RUN cd server && \ make gen-server && \ pip install -r requirements.txt && \ - pip install ".[bnb, accelerate, quantize]" --no-cache-dir + pip install ".[bnb, accelerate, quantize, peft]" --no-cache-dir # Install benchmarker COPY --from=builder /usr/src/target/release/text-generation-benchmark /usr/local/bin/text-generation-benchmark diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 3f7a24dd..c1cbe7f3 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -318,6 +318,7 @@ def docker_launcher( ], volumes=volumes, ports={"80/tcp": port}, + shm_size="1G" ) yield ContainerLauncherHandle(client, container.name, port) diff --git a/server/Makefile b/server/Makefile index 52543e3d..92958d02 100644 --- a/server/Makefile +++ b/server/Makefile @@ -16,17 +16,13 @@ gen-server: find text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; touch text_generation_server/pb/__init__.py -install-torch: - # Install specific version of torch - pip install torch --extra-index-url https://download.pytorch.org/whl/cu118 --no-cache-dir - -install: gen-server install-torch +install: gen-server pip install pip --upgrade pip install -r requirements.txt - pip install -e ".[bnb, accelerate]" + pip install -e ".[bnb, accelerate, quantize, peft]" run-dev: SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded export-requirements: - poetry export -o requirements.txt -E bnb -E quantize --without-hashes + poetry export -o requirements.txt -E bnb --without-hashes diff --git a/server/poetry.lock b/server/poetry.lock index 0caa1d34..48ae40fe 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,10 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "accelerate" version = "0.20.3" description = "Accelerate" -optional = false +optional = true python-versions = ">=3.7.0" files = [ {file = "accelerate-0.20.3-py3-none-any.whl", hash = "sha256:147183e7a2215f7bd45a7af3b986a963daa8a61fa58b0912b9473049e011ad15"}, @@ -30,111 +30,99 @@ testing = ["datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pyte [[package]] name = "aiohttp" -version = "3.8.5" +version = "3.9.0" description = "Async http client/server framework (asyncio)" optional = true -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8"}, - {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84"}, - {file = "aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa"}, - {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691"}, - {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825"}, - {file = "aiohttp-3.8.5-cp310-cp310-win32.whl", hash = "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802"}, - {file = "aiohttp-3.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df"}, - {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9"}, - {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975"}, - {file = "aiohttp-3.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff"}, - {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5"}, - {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c"}, - {file = "aiohttp-3.8.5-cp311-cp311-win32.whl", hash = "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945"}, - {file = "aiohttp-3.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755"}, - {file = "aiohttp-3.8.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd"}, - {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af"}, - {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"}, - {file = "aiohttp-3.8.5-cp36-cp36m-win32.whl", hash = "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e"}, - {file = "aiohttp-3.8.5-cp36-cp36m-win_amd64.whl", hash = "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6"}, - {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a"}, - {file = "aiohttp-3.8.5-cp37-cp37m-win32.whl", hash = "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22"}, - {file = "aiohttp-3.8.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d"}, - {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced"}, - {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690"}, - {file = "aiohttp-3.8.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8"}, - {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b"}, - {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35"}, - {file = "aiohttp-3.8.5-cp38-cp38-win32.whl", hash = "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c"}, - {file = "aiohttp-3.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e"}, - {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3"}, - {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51"}, - {file = "aiohttp-3.8.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1"}, - {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd"}, - {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91"}, - {file = "aiohttp-3.8.5-cp39-cp39-win32.whl", hash = "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67"}, - {file = "aiohttp-3.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c"}, - {file = "aiohttp-3.8.5.tar.gz", hash = "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc"}, + {file = "aiohttp-3.9.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6896b8416be9ada4d22cd359d7cb98955576ce863eadad5596b7cdfbf3e17c6c"}, + {file = "aiohttp-3.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1736d87dad8ef46a8ec9cddd349fa9f7bd3a064c47dd6469c0d6763d3d49a4fc"}, + {file = "aiohttp-3.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c9e5f4d7208cda1a2bb600e29069eecf857e6980d0ccc922ccf9d1372c16f4b"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8488519aa05e636c5997719fe543c8daf19f538f4fa044f3ce94bee608817cff"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ab16c254e2312efeb799bc3c06897f65a133b38b69682bf75d1f1ee1a9c43a9"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a94bde005a8f926d0fa38b88092a03dea4b4875a61fbcd9ac6f4351df1b57cd"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b777c9286b6c6a94f50ddb3a6e730deec327e9e2256cb08b5530db0f7d40fd8"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:571760ad7736b34d05597a1fd38cbc7d47f7b65deb722cb8e86fd827404d1f6b"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:deac0a32aec29608eb25d730f4bc5a261a65b6c48ded1ed861d2a1852577c932"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:4ee1b4152bc3190cc40ddd6a14715e3004944263ea208229ab4c297712aa3075"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:3607375053df58ed6f23903aa10cf3112b1240e8c799d243bbad0f7be0666986"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:65b0a70a25456d329a5e1426702dde67be0fb7a4ead718005ba2ca582d023a94"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a2eb5311a37fe105aa35f62f75a078537e1a9e4e1d78c86ec9893a3c97d7a30"}, + {file = "aiohttp-3.9.0-cp310-cp310-win32.whl", hash = "sha256:2cbc14a13fb6b42d344e4f27746a4b03a2cb0c1c3c5b932b0d6ad8881aa390e3"}, + {file = "aiohttp-3.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ac9669990e2016d644ba8ae4758688534aabde8dbbc81f9af129c3f5f01ca9cd"}, + {file = "aiohttp-3.9.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f8e05f5163528962ce1d1806fce763ab893b1c5b7ace0a3538cd81a90622f844"}, + {file = "aiohttp-3.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4afa8f71dba3a5a2e1e1282a51cba7341ae76585345c43d8f0e624882b622218"}, + {file = "aiohttp-3.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f929f4c9b9a00f3e6cc0587abb95ab9c05681f8b14e0fe1daecfa83ea90f8318"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28185e36a78d247c55e9fbea2332d16aefa14c5276a582ce7a896231c6b1c208"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a486ddf57ab98b6d19ad36458b9f09e6022de0381674fe00228ca7b741aacb2f"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70e851f596c00f40a2f00a46126c95c2e04e146015af05a9da3e4867cfc55911"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5b7bf8fe4d39886adc34311a233a2e01bc10eb4e842220235ed1de57541a896"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c67a51ea415192c2e53e4e048c78bab82d21955b4281d297f517707dc836bf3d"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:694df243f394629bcae2d8ed94c589a181e8ba8604159e6e45e7b22e58291113"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3dd8119752dd30dd7bca7d4bc2a92a59be6a003e4e5c2cf7e248b89751b8f4b7"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:eb6dfd52063186ac97b4caa25764cdbcdb4b10d97f5c5f66b0fa95052e744eb7"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:d97c3e286d0ac9af6223bc132dc4bad6540b37c8d6c0a15fe1e70fb34f9ec411"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:816f4db40555026e4cdda604a1088577c1fb957d02f3f1292e0221353403f192"}, + {file = "aiohttp-3.9.0-cp311-cp311-win32.whl", hash = "sha256:3abf0551874fecf95f93b58f25ef4fc9a250669a2257753f38f8f592db85ddea"}, + {file = "aiohttp-3.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:e18d92c3e9e22553a73e33784fcb0ed484c9874e9a3e96c16a8d6a1e74a0217b"}, + {file = "aiohttp-3.9.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:99ae01fb13a618b9942376df77a1f50c20a281390dad3c56a6ec2942e266220d"}, + {file = "aiohttp-3.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:05857848da443c8c12110d99285d499b4e84d59918a21132e45c3f0804876994"}, + {file = "aiohttp-3.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:317719d7f824eba55857fe0729363af58e27c066c731bc62cd97bc9c3d9c7ea4"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1e3b3c107ccb0e537f309f719994a55621acd2c8fdf6d5ce5152aed788fb940"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45820ddbb276113ead8d4907a7802adb77548087ff5465d5c554f9aa3928ae7d"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:05a183f1978802588711aed0dea31e697d760ce9055292db9dc1604daa9a8ded"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a4cd44788ea0b5e6bb8fa704597af3a30be75503a7ed1098bc5b8ffdf6c982"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:673343fbc0c1ac44d0d2640addc56e97a052504beacd7ade0dc5e76d3a4c16e8"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7e8a3b79b6d186a9c99761fd4a5e8dd575a48d96021f220ac5b5fa856e5dd029"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6777a390e41e78e7c45dab43a4a0196c55c3b8c30eebe017b152939372a83253"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7ae5f99a32c53731c93ac3075abd3e1e5cfbe72fc3eaac4c27c9dd64ba3b19fe"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:f1e4f254e9c35d8965d377e065c4a8a55d396fe87c8e7e8429bcfdeeb229bfb3"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11ca808f9a6b63485059f5f6e164ef7ec826483c1212a44f268b3653c91237d8"}, + {file = "aiohttp-3.9.0-cp312-cp312-win32.whl", hash = "sha256:de3cc86f4ea8b4c34a6e43a7306c40c1275e52bfa9748d869c6b7d54aa6dad80"}, + {file = "aiohttp-3.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca4fddf84ac7d8a7d0866664936f93318ff01ee33e32381a115b19fb5a4d1202"}, + {file = "aiohttp-3.9.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f09960b5bb1017d16c0f9e9f7fc42160a5a49fa1e87a175fd4a2b1a1833ea0af"}, + {file = "aiohttp-3.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8303531e2c17b1a494ffaeba48f2da655fe932c4e9a2626c8718403c83e5dd2b"}, + {file = "aiohttp-3.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4790e44f46a4aa07b64504089def5744d3b6780468c4ec3a1a36eb7f2cae9814"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1d7edf74a36de0e5ca50787e83a77cf352f5504eb0ffa3f07000a911ba353fb"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94697c7293199c2a2551e3e3e18438b4cba293e79c6bc2319f5fd652fccb7456"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a1b66dbb8a7d5f50e9e2ea3804b01e766308331d0cac76eb30c563ac89c95985"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9623cfd9e85b76b83ef88519d98326d4731f8d71869867e47a0b979ffec61c73"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f32c86dc967ab8c719fd229ce71917caad13cc1e8356ee997bf02c5b368799bf"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f50b4663c3e0262c3a361faf440761fbef60ccdde5fe8545689a4b3a3c149fb4"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dcf71c55ec853826cd70eadb2b6ac62ec577416442ca1e0a97ad875a1b3a0305"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:42fe4fd9f0dfcc7be4248c162d8056f1d51a04c60e53366b0098d1267c4c9da8"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:76a86a9989ebf82ee61e06e2bab408aec4ea367dc6da35145c3352b60a112d11"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f9e09a1c83521d770d170b3801eea19b89f41ccaa61d53026ed111cb6f088887"}, + {file = "aiohttp-3.9.0-cp38-cp38-win32.whl", hash = "sha256:a00ce44c21612d185c5275c5cba4bab8d7c1590f248638b667ed8a782fa8cd6f"}, + {file = "aiohttp-3.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:d5b9345ab92ebe6003ae11d8092ce822a0242146e6fa270889b9ba965457ca40"}, + {file = "aiohttp-3.9.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98d21092bf2637c5fa724a428a69e8f5955f2182bff61f8036827cf6ce1157bf"}, + {file = "aiohttp-3.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:35a68cd63ca6aaef5707888f17a70c36efe62b099a4e853d33dc2e9872125be8"}, + {file = "aiohttp-3.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d7f6235c7475658acfc1769d968e07ab585c79f6ca438ddfecaa9a08006aee2"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db04d1de548f7a62d1dd7e7cdf7c22893ee168e22701895067a28a8ed51b3735"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:536b01513d67d10baf6f71c72decdf492fb7433c5f2f133e9a9087379d4b6f31"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c8b0a6487e8109427ccf638580865b54e2e3db4a6e0e11c02639231b41fc0f"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7276fe0017664414fdc3618fca411630405f1aaf0cc3be69def650eb50441787"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23170247ef89ffa842a02bbfdc425028574d9e010611659abeb24d890bc53bb8"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b1a2ea8252cacc7fd51df5a56d7a2bb1986ed39be9397b51a08015727dfb69bd"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d71abc15ff7047412ef26bf812dfc8d0d1020d664617f4913df2df469f26b76"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:2d820162c8c2bdbe97d328cd4f417c955ca370027dce593345e437b2e9ffdc4d"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:2779f5e7c70f7b421915fd47db332c81de365678180a9f3ab404088f87ba5ff9"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:366bc870d7ac61726f32a489fbe3d1d8876e87506870be66b01aeb84389e967e"}, + {file = "aiohttp-3.9.0-cp39-cp39-win32.whl", hash = "sha256:1df43596b826022b14998f0460926ce261544fedefe0d2f653e1b20f49e96454"}, + {file = "aiohttp-3.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:9c196b30f1b1aa3363a69dd69079ae9bec96c2965c4707eaa6914ba099fb7d4f"}, + {file = "aiohttp-3.9.0.tar.gz", hash = "sha256:09f23292d29135025e19e8ff4f0a68df078fe4ee013bca0105b2e803989de92d"}, ] [package.dependencies] aiosignal = ">=1.1.2" -async-timeout = ">=4.0.0a3,<5.0" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" -charset-normalizer = ">=2.0,<4.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["Brotli", "aiodns", "brotlicffi"] [[package]] name = "aiosignal" @@ -192,123 +180,123 @@ files = [ [[package]] name = "bitsandbytes" -version = "0.41.1" +version = "0.41.2.post2" description = "k-bit optimizers and matrix multiplication routines." optional = true python-versions = "*" files = [ - {file = "bitsandbytes-0.41.1-py3-none-any.whl", hash = "sha256:b25228c27636367f222232ed4d1e1502eedd2064be215633734fb8ea0c1c65f4"}, - {file = "bitsandbytes-0.41.1.tar.gz", hash = "sha256:b3f8e7e1e5f88d4813d10ebd4072034ba6a18eca7f0e255376f8320e5499032c"}, + {file = "bitsandbytes-0.41.2.post2-py3-none-any.whl", hash = "sha256:98e5e1979aea3d481ed06181c689f3a154d7f5dc1af770c5173485bc54cf7b72"}, + {file = "bitsandbytes-0.41.2.post2.tar.gz", hash = "sha256:d374da4700651f36a285ed53e012ee527736109614e3f5c0249985d41027136d"}, ] [[package]] name = "certifi" -version = "2023.7.22" +version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] [[package]] name = "charset-normalizer" -version = "3.3.0" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.3.0.tar.gz", hash = "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-win32.whl", hash = "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d"}, - {file = "charset_normalizer-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-win32.whl", hash = "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786"}, - {file = "charset_normalizer-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-win32.whl", hash = "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df"}, - {file = "charset_normalizer-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c"}, - {file = "charset_normalizer-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-win32.whl", hash = "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e"}, - {file = "charset_normalizer-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-win32.whl", hash = "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a"}, - {file = "charset_normalizer-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884"}, - {file = "charset_normalizer-3.3.0-py3-none-any.whl", hash = "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2"}, + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] [[package]] @@ -338,25 +326,26 @@ files = [ [[package]] name = "datasets" -version = "2.14.5" +version = "2.14.7" description = "HuggingFace community-driven open-source library of datasets" optional = true python-versions = ">=3.8.0" files = [ - {file = "datasets-2.14.5-py3-none-any.whl", hash = "sha256:dd4155091034cba04d5a28711f2ed3944275ed15c5d0c5a2d0b6b9ea34a2bdfe"}, - {file = "datasets-2.14.5.tar.gz", hash = "sha256:b738a86540ab8e1a7806c8a3790b67be0056318d0c5d5a58a1b0dbdd76c0f568"}, + {file = "datasets-2.14.7-py3-none-any.whl", hash = "sha256:1a64041a7da4f4130f736fc371c1f528b8ddd208cebe156400f65719bdbba79d"}, + {file = "datasets-2.14.7.tar.gz", hash = "sha256:394cf9b4ec0694b25945977b16ad5d18d5c15fb0e94141713eb8ead7452caf9e"}, ] [package.dependencies] aiohttp = "*" dill = ">=0.3.0,<0.3.8" -fsspec = {version = ">=2023.1.0,<2023.9.0", extras = ["http"]} +fsspec = {version = ">=2023.1.0,<=2023.10.0", extras = ["http"]} huggingface-hub = ">=0.14.0,<1.0.0" multiprocess = "*" numpy = ">=1.17" packaging = "*" pandas = "*" pyarrow = ">=8.0.0" +pyarrow-hotfix = "*" pyyaml = ">=5.1" requests = ">=2.19.0" tqdm = ">=4.62.1" @@ -422,13 +411,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.1.3" +version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, - {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] @@ -436,19 +425,19 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.12.4" +version = "3.13.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.12.4-py3-none-any.whl", hash = "sha256:08c21d87ded6e2b9da6728c3dff51baf1dcecf973b768ef35bcbc3447edb9ad4"}, - {file = "filelock-3.12.4.tar.gz", hash = "sha256:2e6f249f1f3654291606e046b09f1fd5eac39b360664c27f5aad072012f8bcbd"}, + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, ] [package.extras] -docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] -typing = ["typing-extensions (>=4.7.1)"] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] [[package]] name = "frozenlist" @@ -522,13 +511,13 @@ files = [ [[package]] name = "fsspec" -version = "2023.6.0" +version = "2023.10.0" description = "File-system specification" optional = false python-versions = ">=3.8" files = [ - {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"}, - {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"}, + {file = "fsspec-2023.10.0-py3-none-any.whl", hash = "sha256:346a8f024efeb749d2a5fca7ba8854474b1ff9af7c3faaf636a4548781136529"}, + {file = "fsspec-2023.10.0.tar.gz", hash = "sha256:330c66757591df346ad3091a53bd907e15348c2ba17d63fd54f5c39c4457d2a5"}, ] [package.dependencies] @@ -561,13 +550,13 @@ tqdm = ["tqdm"] [[package]] name = "googleapis-common-protos" -version = "1.60.0" +version = "1.61.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.60.0.tar.gz", hash = "sha256:e73ebb404098db405ba95d1e1ae0aa91c3e15a71da031a2eeb6b2e23e7bc3708"}, - {file = "googleapis_common_protos-1.60.0-py2.py3-none-any.whl", hash = "sha256:69f9bbcc6acde92cab2db95ce30a70bd2b81d20b12eff3f1aabaffcbe8a93918"}, + {file = "googleapis-common-protos-1.61.0.tar.gz", hash = "sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b"}, + {file = "googleapis_common_protos-1.61.0-py2.py3-none-any.whl", hash = "sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0"}, ] [package.dependencies] @@ -578,13 +567,13 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "grpc-interceptor" -version = "0.15.3" +version = "0.15.4" description = "Simplifies gRPC interceptors" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "grpc-interceptor-0.15.3.tar.gz", hash = "sha256:33592cb9d8c00fceed5755c71029f75aef55b273496dbced06f1d48f2571fcc3"}, - {file = "grpc_interceptor-0.15.3-py3-none-any.whl", hash = "sha256:96be2043b7e49f9deb444f18b61c373ea28d22d81c90cd3b82127a4744eb9247"}, + {file = "grpc-interceptor-0.15.4.tar.gz", hash = "sha256:1f45c0bcb58b6f332f37c637632247c9b02bc6af0fdceb7ba7ce8d2ebbfb0926"}, + {file = "grpc_interceptor-0.15.4-py3-none-any.whl", hash = "sha256:0035f33228693ed3767ee49d937bac424318db173fef4d2d0170b3215f254d9d"}, ] [package.dependencies] @@ -595,195 +584,199 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.59.0" +version = "1.59.3" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.59.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:225e5fa61c35eeaebb4e7491cd2d768cd8eb6ed00f2664fa83a58f29418b39fd"}, - {file = "grpcio-1.59.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b95ec8ecc4f703f5caaa8d96e93e40c7f589bad299a2617bdb8becbcce525539"}, - {file = "grpcio-1.59.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:1a839ba86764cc48226f50b924216000c79779c563a301586a107bda9cbe9dcf"}, - {file = "grpcio-1.59.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6cfe44a5d7c7d5f1017a7da1c8160304091ca5dc64a0f85bca0d63008c3137a"}, - {file = "grpcio-1.59.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0fcf53df684fcc0154b1e61f6b4a8c4cf5f49d98a63511e3f30966feff39cd0"}, - {file = "grpcio-1.59.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa66cac32861500f280bb60fe7d5b3e22d68c51e18e65367e38f8669b78cea3b"}, - {file = "grpcio-1.59.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8cd2d38c2d52f607d75a74143113174c36d8a416d9472415eab834f837580cf7"}, - {file = "grpcio-1.59.0-cp310-cp310-win32.whl", hash = "sha256:228b91ce454876d7eed74041aff24a8f04c0306b7250a2da99d35dd25e2a1211"}, - {file = "grpcio-1.59.0-cp310-cp310-win_amd64.whl", hash = "sha256:ca87ee6183421b7cea3544190061f6c1c3dfc959e0b57a5286b108511fd34ff4"}, - {file = "grpcio-1.59.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c173a87d622ea074ce79be33b952f0b424fa92182063c3bda8625c11d3585d09"}, - {file = "grpcio-1.59.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:ec78aebb9b6771d6a1de7b6ca2f779a2f6113b9108d486e904bde323d51f5589"}, - {file = "grpcio-1.59.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:0b84445fa94d59e6806c10266b977f92fa997db3585f125d6b751af02ff8b9fe"}, - {file = "grpcio-1.59.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c251d22de8f9f5cca9ee47e4bade7c5c853e6e40743f47f5cc02288ee7a87252"}, - {file = "grpcio-1.59.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:956f0b7cb465a65de1bd90d5a7475b4dc55089b25042fe0f6c870707e9aabb1d"}, - {file = "grpcio-1.59.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:38da5310ef84e16d638ad89550b5b9424df508fd5c7b968b90eb9629ca9be4b9"}, - {file = "grpcio-1.59.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:63982150a7d598281fa1d7ffead6096e543ff8be189d3235dd2b5604f2c553e5"}, - {file = "grpcio-1.59.0-cp311-cp311-win32.whl", hash = "sha256:50eff97397e29eeee5df106ea1afce3ee134d567aa2c8e04fabab05c79d791a7"}, - {file = "grpcio-1.59.0-cp311-cp311-win_amd64.whl", hash = "sha256:15f03bd714f987d48ae57fe092cf81960ae36da4e520e729392a59a75cda4f29"}, - {file = "grpcio-1.59.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:f1feb034321ae2f718172d86b8276c03599846dc7bb1792ae370af02718f91c5"}, - {file = "grpcio-1.59.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d09bd2a4e9f5a44d36bb8684f284835c14d30c22d8ec92ce796655af12163588"}, - {file = "grpcio-1.59.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:2f120d27051e4c59db2f267b71b833796770d3ea36ca712befa8c5fff5da6ebd"}, - {file = "grpcio-1.59.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0ca727a173ee093f49ead932c051af463258b4b493b956a2c099696f38aa66"}, - {file = "grpcio-1.59.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5711c51e204dc52065f4a3327dca46e69636a0b76d3e98c2c28c4ccef9b04c52"}, - {file = "grpcio-1.59.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:d74f7d2d7c242a6af9d4d069552ec3669965b74fed6b92946e0e13b4168374f9"}, - {file = "grpcio-1.59.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3859917de234a0a2a52132489c4425a73669de9c458b01c9a83687f1f31b5b10"}, - {file = "grpcio-1.59.0-cp312-cp312-win32.whl", hash = "sha256:de2599985b7c1b4ce7526e15c969d66b93687571aa008ca749d6235d056b7205"}, - {file = "grpcio-1.59.0-cp312-cp312-win_amd64.whl", hash = "sha256:598f3530231cf10ae03f4ab92d48c3be1fee0c52213a1d5958df1a90957e6a88"}, - {file = "grpcio-1.59.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:b34c7a4c31841a2ea27246a05eed8a80c319bfc0d3e644412ec9ce437105ff6c"}, - {file = "grpcio-1.59.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:c4dfdb49f4997dc664f30116af2d34751b91aa031f8c8ee251ce4dcfc11277b0"}, - {file = "grpcio-1.59.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:61bc72a00ecc2b79d9695220b4d02e8ba53b702b42411397e831c9b0589f08a3"}, - {file = "grpcio-1.59.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f367e4b524cb319e50acbdea57bb63c3b717c5d561974ace0b065a648bb3bad3"}, - {file = "grpcio-1.59.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:849c47ef42424c86af069a9c5e691a765e304079755d5c29eff511263fad9c2a"}, - {file = "grpcio-1.59.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c0488c2b0528e6072010182075615620071371701733c63ab5be49140ed8f7f0"}, - {file = "grpcio-1.59.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:611d9aa0017fa386809bddcb76653a5ab18c264faf4d9ff35cb904d44745f575"}, - {file = "grpcio-1.59.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e5378785dce2b91eb2e5b857ec7602305a3b5cf78311767146464bfa365fc897"}, - {file = "grpcio-1.59.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fe976910de34d21057bcb53b2c5e667843588b48bf11339da2a75f5c4c5b4055"}, - {file = "grpcio-1.59.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:c041a91712bf23b2a910f61e16565a05869e505dc5a5c025d429ca6de5de842c"}, - {file = "grpcio-1.59.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:0ae444221b2c16d8211b55326f8ba173ba8f8c76349bfc1768198ba592b58f74"}, - {file = "grpcio-1.59.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ceb1e68135788c3fce2211de86a7597591f0b9a0d2bb80e8401fd1d915991bac"}, - {file = "grpcio-1.59.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4b1cc3a9dc1924d2eb26eec8792fedd4b3fcd10111e26c1d551f2e4eda79ce"}, - {file = "grpcio-1.59.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:871371ce0c0055d3db2a86fdebd1e1d647cf21a8912acc30052660297a5a6901"}, - {file = "grpcio-1.59.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:93e9cb546e610829e462147ce724a9cb108e61647a3454500438a6deef610be1"}, - {file = "grpcio-1.59.0-cp38-cp38-win32.whl", hash = "sha256:f21917aa50b40842b51aff2de6ebf9e2f6af3fe0971c31960ad6a3a2b24988f4"}, - {file = "grpcio-1.59.0-cp38-cp38-win_amd64.whl", hash = "sha256:14890da86a0c0e9dc1ea8e90101d7a3e0e7b1e71f4487fab36e2bfd2ecadd13c"}, - {file = "grpcio-1.59.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:34341d9e81a4b669a5f5dca3b2a760b6798e95cdda2b173e65d29d0b16692857"}, - {file = "grpcio-1.59.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:986de4aa75646e963466b386a8c5055c8b23a26a36a6c99052385d6fe8aaf180"}, - {file = "grpcio-1.59.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:aca8a24fef80bef73f83eb8153f5f5a0134d9539b4c436a716256b311dda90a6"}, - {file = "grpcio-1.59.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:936b2e04663660c600d5173bc2cc84e15adbad9c8f71946eb833b0afc205b996"}, - {file = "grpcio-1.59.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc8bf2e7bc725e76c0c11e474634a08c8f24bcf7426c0c6d60c8f9c6e70e4d4a"}, - {file = "grpcio-1.59.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81d86a096ccd24a57fa5772a544c9e566218bc4de49e8c909882dae9d73392df"}, - {file = "grpcio-1.59.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2ea95cd6abbe20138b8df965b4a8674ec312aaef3147c0f46a0bac661f09e8d0"}, - {file = "grpcio-1.59.0-cp39-cp39-win32.whl", hash = "sha256:3b8ff795d35a93d1df6531f31c1502673d1cebeeba93d0f9bd74617381507e3f"}, - {file = "grpcio-1.59.0-cp39-cp39-win_amd64.whl", hash = "sha256:38823bd088c69f59966f594d087d3a929d1ef310506bee9e3648317660d65b81"}, - {file = "grpcio-1.59.0.tar.gz", hash = "sha256:acf70a63cf09dd494000007b798aff88a436e1c03b394995ce450be437b8e54f"}, + {file = "grpcio-1.59.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:aca028a6c7806e5b61e5f9f4232432c52856f7fcb98e330b20b6bc95d657bdcc"}, + {file = "grpcio-1.59.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:19ad26a7967f7999c8960d2b9fe382dae74c55b0c508c613a6c2ba21cddf2354"}, + {file = "grpcio-1.59.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:72b71dad2a3d1650e69ad42a5c4edbc59ee017f08c32c95694172bc501def23c"}, + {file = "grpcio-1.59.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c0f0a11d82d0253656cc42e04b6a149521e02e755fe2e4edd21123de610fd1d4"}, + {file = "grpcio-1.59.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60cddafb70f9a2c81ba251b53b4007e07cca7389e704f86266e22c4bffd8bf1d"}, + {file = "grpcio-1.59.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6c75a1fa0e677c1d2b6d4196ad395a5c381dfb8385f07ed034ef667cdcdbcc25"}, + {file = "grpcio-1.59.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e1d8e01438d5964a11167eec1edb5f85ed8e475648f36c834ed5db4ffba24ac8"}, + {file = "grpcio-1.59.3-cp310-cp310-win32.whl", hash = "sha256:c4b0076f0bf29ee62335b055a9599f52000b7941f577daa001c7ef961a1fbeab"}, + {file = "grpcio-1.59.3-cp310-cp310-win_amd64.whl", hash = "sha256:b1f00a3e6e0c3dccccffb5579fc76ebfe4eb40405ba308505b41ef92f747746a"}, + {file = "grpcio-1.59.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:3996aaa21231451161dc29df6a43fcaa8b332042b6150482c119a678d007dd86"}, + {file = "grpcio-1.59.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:cb4e9cbd9b7388fcb06412da9f188c7803742d06d6f626304eb838d1707ec7e3"}, + {file = "grpcio-1.59.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8022ca303d6c694a0d7acfb2b472add920217618d3a99eb4b14edc7c6a7e8fcf"}, + {file = "grpcio-1.59.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b36683fad5664283755a7f4e2e804e243633634e93cd798a46247b8e54e3cb0d"}, + {file = "grpcio-1.59.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8239b853226e4824e769517e1b5232e7c4dda3815b200534500338960fcc6118"}, + {file = "grpcio-1.59.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0511af8653fbda489ff11d542a08505d56023e63cafbda60e6e00d4e0bae86ea"}, + {file = "grpcio-1.59.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e78dc982bda74cef2ddfce1c91d29b96864c4c680c634e279ed204d51e227473"}, + {file = "grpcio-1.59.3-cp311-cp311-win32.whl", hash = "sha256:6a5c3a96405966c023e139c3bcccb2c7c776a6f256ac6d70f8558c9041bdccc3"}, + {file = "grpcio-1.59.3-cp311-cp311-win_amd64.whl", hash = "sha256:ed26826ee423b11477297b187371cdf4fa1eca874eb1156422ef3c9a60590dd9"}, + {file = "grpcio-1.59.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:45dddc5cb5227d30fa43652d8872dc87f086d81ab4b500be99413bad0ae198d7"}, + {file = "grpcio-1.59.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:1736496d74682e53dd0907fd515f2694d8e6a96c9a359b4080b2504bf2b2d91b"}, + {file = "grpcio-1.59.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:ddbd1a16138e52e66229047624de364f88a948a4d92ba20e4e25ad7d22eef025"}, + {file = "grpcio-1.59.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fcfa56f8d031ffda902c258c84c4b88707f3a4be4827b4e3ab8ec7c24676320d"}, + {file = "grpcio-1.59.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2eb8f0c7c0c62f7a547ad7a91ba627a5aa32a5ae8d930783f7ee61680d7eb8d"}, + {file = "grpcio-1.59.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8d993399cc65e3a34f8fd48dd9ad7a376734564b822e0160dd18b3d00c1a33f9"}, + {file = "grpcio-1.59.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0bd141f4f41907eb90bda74d969c3cb21c1c62779419782a5b3f5e4b5835718"}, + {file = "grpcio-1.59.3-cp312-cp312-win32.whl", hash = "sha256:33b8fd65d4e97efa62baec6171ce51f9cf68f3a8ba9f866f4abc9d62b5c97b79"}, + {file = "grpcio-1.59.3-cp312-cp312-win_amd64.whl", hash = "sha256:0e735ed002f50d4f3cb9ecfe8ac82403f5d842d274c92d99db64cfc998515e07"}, + {file = "grpcio-1.59.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:ea40ce4404e7cca0724c91a7404da410f0144148fdd58402a5942971e3469b94"}, + {file = "grpcio-1.59.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:83113bcc393477b6f7342b9f48e8a054330c895205517edc66789ceea0796b53"}, + {file = "grpcio-1.59.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:73afbac602b8f1212a50088193601f869b5073efa9855b3e51aaaec97848fc8a"}, + {file = "grpcio-1.59.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:575d61de1950b0b0699917b686b1ca108690702fcc2df127b8c9c9320f93e069"}, + {file = "grpcio-1.59.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd76057b5c9a4d68814610ef9226925f94c1231bbe533fdf96f6181f7d2ff9e"}, + {file = "grpcio-1.59.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:95d6fd804c81efe4879e38bfd84d2b26e339a0a9b797e7615e884ef4686eb47b"}, + {file = "grpcio-1.59.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0d42048b8a3286ea4134faddf1f9a59cf98192b94aaa10d910a25613c5eb5bfb"}, + {file = "grpcio-1.59.3-cp37-cp37m-win_amd64.whl", hash = "sha256:4619fea15c64bcdd9d447cdbdde40e3d5f1da3a2e8ae84103d94a9c1df210d7e"}, + {file = "grpcio-1.59.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:95b5506e70284ac03b2005dd9ffcb6708c9ae660669376f0192a710687a22556"}, + {file = "grpcio-1.59.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:9e17660947660ccfce56c7869032910c179a5328a77b73b37305cd1ee9301c2e"}, + {file = "grpcio-1.59.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:00912ce19914d038851be5cd380d94a03f9d195643c28e3ad03d355cc02ce7e8"}, + {file = "grpcio-1.59.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e58b3cadaa3c90f1efca26ba33e0d408b35b497307027d3d707e4bcd8de862a6"}, + {file = "grpcio-1.59.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d787ecadea865bdf78f6679f6f5bf4b984f18f659257ba612979df97a298b3c3"}, + {file = "grpcio-1.59.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0814942ba1bba269db4e760a34388640c601dece525c6a01f3b4ff030cc0db69"}, + {file = "grpcio-1.59.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fb111aa99d3180c361a35b5ae1e2c63750220c584a1344229abc139d5c891881"}, + {file = "grpcio-1.59.3-cp38-cp38-win32.whl", hash = "sha256:eb8ba504c726befe40a356ecbe63c6c3c64c9a439b3164f5a718ec53c9874da0"}, + {file = "grpcio-1.59.3-cp38-cp38-win_amd64.whl", hash = "sha256:cdbc6b32fadab9bebc6f49d3e7ec4c70983c71e965497adab7f87de218e84391"}, + {file = "grpcio-1.59.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:c82ca1e4be24a98a253d6dbaa216542e4163f33f38163fc77964b0f0d255b552"}, + {file = "grpcio-1.59.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:36636babfda14f9e9687f28d5b66d349cf88c1301154dc71c6513de2b6c88c59"}, + {file = "grpcio-1.59.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5f9b2e591da751ac7fdd316cc25afafb7a626dededa9b414f90faad7f3ccebdb"}, + {file = "grpcio-1.59.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a93a82876a4926bf451db82ceb725bd87f42292bacc94586045261f501a86994"}, + {file = "grpcio-1.59.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce31fa0bfdd1f2bb15b657c16105c8652186eab304eb512e6ae3b99b2fdd7d13"}, + {file = "grpcio-1.59.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:16da0e40573962dab6cba16bec31f25a4f468e6d05b658e589090fe103b03e3d"}, + {file = "grpcio-1.59.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1d1a17372fd425addd5812049fa7374008ffe689585f27f802d0935522cf4b7"}, + {file = "grpcio-1.59.3-cp39-cp39-win32.whl", hash = "sha256:52cc38a7241b5f7b4a91aaf9000fdd38e26bb00d5e8a71665ce40cfcee716281"}, + {file = "grpcio-1.59.3-cp39-cp39-win_amd64.whl", hash = "sha256:b491e5bbcad3020a96842040421e508780cade35baba30f402df9d321d1c423e"}, + {file = "grpcio-1.59.3.tar.gz", hash = "sha256:7800f99568a74a06ebdccd419dd1b6e639b477dcaf6da77ea702f8fb14ce5f80"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.59.0)"] +protobuf = ["grpcio-tools (>=1.59.3)"] [[package]] name = "grpcio-reflection" -version = "1.59.0" +version = "1.59.3" description = "Standard Protobuf Reflection Service for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-reflection-1.59.0.tar.gz", hash = "sha256:1fe8f0dd6c180fdcf4e12ced2a8f784d9c741ccbc0b198585b1df024b7f8f3f2"}, - {file = "grpcio_reflection-1.59.0-py3-none-any.whl", hash = "sha256:bf4efc7e2e8162e5be9736f4d0a0b324c9bf0c04ad597a9d78fcaf1fbdf818ec"}, + {file = "grpcio-reflection-1.59.3.tar.gz", hash = "sha256:5403c5a738c6eec4bb4080da77e450312dace41a86e292ac37cfd007b2657d4e"}, + {file = "grpcio_reflection-1.59.3-py3-none-any.whl", hash = "sha256:526064089d71cddce7244a83059f410fac6ccd30344a624c10f54420d417b3d2"}, ] [package.dependencies] -grpcio = ">=1.59.0" +grpcio = ">=1.59.3" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.59.0" +version = "1.59.3" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.59.0.tar.gz", hash = "sha256:f93b9c33e0a26162ef8431bfcffcc3e1fb217ccd8d7b5b3061b6e9f813e698b5"}, - {file = "grpcio_status-1.59.0-py3-none-any.whl", hash = "sha256:cb5a222b14a80ee050bff9676623822e953bff0c50d2d29180de723652fdf10d"}, + {file = "grpcio-status-1.59.3.tar.gz", hash = "sha256:65c394ba43380d6bdf8c04c61efc493104b5535552aed35817a1b4dc66598a1f"}, + {file = "grpcio_status-1.59.3-py3-none-any.whl", hash = "sha256:2fd2eb39ca4e9afb3c874c0878ff75b258db0b7dcc25570fc521f16ae0ab942a"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.59.0" +grpcio = ">=1.59.3" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.59.0" +version = "1.59.3" description = "Protobuf code generator for gRPC" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-tools-1.59.0.tar.gz", hash = "sha256:aa4018f2d8662ac4d9830445d3d253a11b3e096e8afe20865547137aa1160e93"}, - {file = "grpcio_tools-1.59.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:882b809b42b5464bee55288f4e60837297f9618e53e69ae3eea6d61b05ce48fa"}, - {file = "grpcio_tools-1.59.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:4499d4bc5aa9c7b645018d8b0db4bebd663d427aabcd7bee7777046cb1bcbca7"}, - {file = "grpcio_tools-1.59.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f381ae3ad6a5eb27aad8d810438937d8228977067c54e0bd456fce7e11fdbf3d"}, - {file = "grpcio_tools-1.59.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1c684c0d9226d04cadafced620a46ab38c346d0780eaac7448da96bf12066a3"}, - {file = "grpcio_tools-1.59.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40cbf712769242c2ba237745285ef789114d7fcfe8865fc4817d87f20015e99a"}, - {file = "grpcio_tools-1.59.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:1df755951f204e65bf9232a9cac5afe7d6b8e4c87ac084d3ecd738fdc7aa4174"}, - {file = "grpcio_tools-1.59.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:de156c18b0c638aaee3be6ad650c8ba7dec94ed4bac26403aec3dce95ffe9407"}, - {file = "grpcio_tools-1.59.0-cp310-cp310-win32.whl", hash = "sha256:9af7e138baa9b2895cf1f3eb718ac96fc5ae2f8e31fca405e21e0e5cd1643c52"}, - {file = "grpcio_tools-1.59.0-cp310-cp310-win_amd64.whl", hash = "sha256:f14a6e4f700dfd30ff8f0e6695f944affc16ae5a1e738666b3fae4e44b65637e"}, - {file = "grpcio_tools-1.59.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:db030140d0da2368319e2f23655df3baec278c7e0078ecbe051eaf609a69382c"}, - {file = "grpcio_tools-1.59.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:eeed386971bb8afc3ec45593df6a1154d680d87be1209ef8e782e44f85f47e64"}, - {file = "grpcio_tools-1.59.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:962d1a3067129152cee3e172213486cb218a6bad703836991f46f216caefcf00"}, - {file = "grpcio_tools-1.59.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:26eb2eebf150a33ebf088e67c1acf37eb2ac4133d9bfccbaa011ad2148c08b42"}, - {file = "grpcio_tools-1.59.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b2d6da553980c590487f2e7fd3ec9c1ad8805ff2ec77977b92faa7e3ca14e1f"}, - {file = "grpcio_tools-1.59.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:335e2f355a0c544a88854e2c053aff8a3f398b84a263a96fa19d063ca1fe513a"}, - {file = "grpcio_tools-1.59.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:204e08f807b1d83f5f0efea30c4e680afe26a43dec8ba614a45fa698a7ef0a19"}, - {file = "grpcio_tools-1.59.0-cp311-cp311-win32.whl", hash = "sha256:05bf7b3ed01c8a562bb7e840f864c58acedbd6924eb616367c0bd0a760bdf483"}, - {file = "grpcio_tools-1.59.0-cp311-cp311-win_amd64.whl", hash = "sha256:df85096fcac7cea8aa5bd84b7a39c4cdbf556b93669bb4772eb96aacd3222a4e"}, - {file = "grpcio_tools-1.59.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:240a7a3c2c54f77f1f66085a635bca72003d02f56a670e7db19aec531eda8f78"}, - {file = "grpcio_tools-1.59.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:6119f62c462d119c63227b9534210f0f13506a888151b9bf586f71e7edf5088b"}, - {file = "grpcio_tools-1.59.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:387662bee8e4c0b52cc0f61eaaca0ca583f5b227103f685b76083a3590a71a3e"}, - {file = "grpcio_tools-1.59.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f0da5861ee276ca68493b217daef358960e8527cc63c7cb292ca1c9c54939af"}, - {file = "grpcio_tools-1.59.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f0806de1161c7f248e4c183633ee7a58dfe45c2b77ddf0136e2e7ad0650b1b"}, - {file = "grpcio_tools-1.59.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:c683be38a9bf4024c223929b4cd2f0a0858c94e9dc8b36d7eaa5a48ce9323a6f"}, - {file = "grpcio_tools-1.59.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f965707da2b48a33128615bcfebedd215a3a30e346447e885bb3da37a143177a"}, - {file = "grpcio_tools-1.59.0-cp312-cp312-win32.whl", hash = "sha256:2ee960904dde12a7fa48e1591a5b3eeae054bdce57bacf9fd26685a98138f5bf"}, - {file = "grpcio_tools-1.59.0-cp312-cp312-win_amd64.whl", hash = "sha256:71cc6db1d66da3bc3730d9937bddc320f7b1f1dfdff6342bcb5741515fe4110b"}, - {file = "grpcio_tools-1.59.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:f6263b85261b62471cb97b7505df72d72b8b62e5e22d8184924871a6155b4dbf"}, - {file = "grpcio_tools-1.59.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:b8e95d921cc2a1521d4750eedefec9f16031457920a6677edebe9d1b2ad6ae60"}, - {file = "grpcio_tools-1.59.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:cb63055739808144b541986291679d643bae58755d0eb082157c4d4c04443905"}, - {file = "grpcio_tools-1.59.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c4634b3589efa156a8d5860c0a2547315bd5c9e52d14c960d716fe86e0927be"}, - {file = "grpcio_tools-1.59.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d970aa26854f535ffb94ea098aa8b43de020d9a14682e4a15dcdaeac7801b27"}, - {file = "grpcio_tools-1.59.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:821dba464d84ebbcffd9d420302404db2fa7a40c7ff4c4c4c93726f72bfa2769"}, - {file = "grpcio_tools-1.59.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0548e901894399886ff4a4cd808cb850b60c021feb4a8977a0751f14dd7e55d9"}, - {file = "grpcio_tools-1.59.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bb87158dbbb9e5a79effe78d54837599caa16df52d8d35366e06a91723b587ae"}, - {file = "grpcio_tools-1.59.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:1d551ff42962c7c333c3da5c70d5e617a87dee581fa2e2c5ae2d5137c8886779"}, - {file = "grpcio_tools-1.59.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:4ee443abcd241a5befb05629013fbf2eac637faa94aaa3056351aded8a31c1bc"}, - {file = "grpcio_tools-1.59.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:520c0c83ea79d14b0679ba43e19c64ca31d30926b26ad2ca7db37cbd89c167e2"}, - {file = "grpcio_tools-1.59.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9fc02a6e517c34dcf885ff3b57260b646551083903e3d2c780b4971ce7d4ab7c"}, - {file = "grpcio_tools-1.59.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aec8a4ed3808b7dfc1276fe51e3e24bec0eeaf610d395bcd42934647cf902a3"}, - {file = "grpcio_tools-1.59.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:99b3bde646720bbfb77f263f5ba3e1a0de50632d43c38d405a0ef9c7e94373cd"}, - {file = "grpcio_tools-1.59.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:51d9595629998d8b519126c5a610f15deb0327cd6325ed10796b47d1d292e70b"}, - {file = "grpcio_tools-1.59.0-cp38-cp38-win32.whl", hash = "sha256:bfa4b2b7d21c5634b62e5f03462243bd705adc1a21806b5356b8ce06d902e160"}, - {file = "grpcio_tools-1.59.0-cp38-cp38-win_amd64.whl", hash = "sha256:9ed05197c5ab071e91bcef28901e97ca168c4ae94510cb67a14cb4931b94255a"}, - {file = "grpcio_tools-1.59.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:498e7be0b14385980efa681444ba481349c131fc5ec88003819f5d929646947c"}, - {file = "grpcio_tools-1.59.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:b519f2ecde9a579cad2f4a7057d5bb4e040ad17caab8b5e691ed7a13b9db0be9"}, - {file = "grpcio_tools-1.59.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:ef3e8aca2261f7f07436d4e2111556c1fb9bf1f9cfcdf35262743ccdee1b6ce9"}, - {file = "grpcio_tools-1.59.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a7f226b741b2ebf7e2d0779d2c9b17f446d1b839d59886c1619e62cc2ae472"}, - {file = "grpcio_tools-1.59.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:784aa52965916fec5afa1a28eeee6f0073bb43a2a1d7fedf963393898843077a"}, - {file = "grpcio_tools-1.59.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e312ddc2d8bec1a23306a661ad52734f984c9aad5d8f126ebb222a778d95407d"}, - {file = "grpcio_tools-1.59.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:868892ad9e00651a38dace3e4924bae82fc4fd4df2c65d37b74381570ee8deb1"}, - {file = "grpcio_tools-1.59.0-cp39-cp39-win32.whl", hash = "sha256:a4f6cae381f21fee1ef0a5cbbbb146680164311157ae618edf3061742d844383"}, - {file = "grpcio_tools-1.59.0-cp39-cp39-win_amd64.whl", hash = "sha256:4a10e59cca462208b489478340b52a96d64e8b8b6f1ac097f3e8cb211d3f66c0"}, + {file = "grpcio-tools-1.59.3.tar.gz", hash = "sha256:cd160ac4281cd1ae77a2c880377a7728349340b4c91e24285037b57c18e9f651"}, + {file = "grpcio_tools-1.59.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:17017fe74734c158e0f93817f1ff17aeda37d0f105ed6e63b12c26b66743a7a8"}, + {file = "grpcio_tools-1.59.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:ac1013e4f84ffd15c45ead6d19c9d188b76c14466a799aa9c338ce3b9ebf6dcc"}, + {file = "grpcio_tools-1.59.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0a5d760619305eb51a8719ce9c081398f145a46bc7c86a6e2cebe0648a21f40c"}, + {file = "grpcio_tools-1.59.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de3d9649b7a3091ec785a67d5bf006584440f03896ee52259c6d9ff412d08afb"}, + {file = "grpcio_tools-1.59.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21868aa510317d3f39e5de40208ffb8ab1beb1cbcab333317939b59a9b5db055"}, + {file = "grpcio_tools-1.59.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0b116a888580317e421358f26bfaeec47d6f73079e8a47bad60d1f9f7b30f2a5"}, + {file = "grpcio_tools-1.59.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6bd4a72c27abda191e2360b2b720ada1880aba96a63604a6f9d7c37bb3bbf5c4"}, + {file = "grpcio_tools-1.59.3-cp310-cp310-win32.whl", hash = "sha256:d70cad744e92c7576c226a72998ae8dd59240c942f73798bbde40284eb9eb991"}, + {file = "grpcio_tools-1.59.3-cp310-cp310-win_amd64.whl", hash = "sha256:2b8a4aca0c11f2a8b3bfe103362984bdc427ab762428446ef2e12922fd48ee10"}, + {file = "grpcio_tools-1.59.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:b4418b78408ff56ee70a0b14484c07f5e48c2e6f4fa7be390d486a686d0cd6e4"}, + {file = "grpcio_tools-1.59.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:58de83ced4f86458f45288a5f76d9765dc245a9ce4e783a194decccc7e0674ea"}, + {file = "grpcio_tools-1.59.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:76b0cdcbcb38722840d3eaff6439ddb4b8f0210c6718553d7b7c911834b10e60"}, + {file = "grpcio_tools-1.59.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0cacf59513b100bfb3d8de51ba43db6140aa9bcb7bba872badb48acb430c002"}, + {file = "grpcio_tools-1.59.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:019fdd986c80b13187574c291df5054f241bdbe87dbc86e4cee73ffa28328647"}, + {file = "grpcio_tools-1.59.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ff304b9d6c23d8e2ecc860bebac1ec6768a2d920985bcea9ce4a7aaeeea44f76"}, + {file = "grpcio_tools-1.59.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ca286affe613beaf2d5a6b8bd83203dcf488917194b416da48aa849047b5f081"}, + {file = "grpcio_tools-1.59.3-cp311-cp311-win32.whl", hash = "sha256:8f69141ff370729ceaad0286b8c6e15352c9bb39aa8f18c0500ce3d0238c2981"}, + {file = "grpcio_tools-1.59.3-cp311-cp311-win_amd64.whl", hash = "sha256:05ec4ffe16b6eab12813476e6d7465a0027bee33999d4776ae1d9c0664d0fc54"}, + {file = "grpcio_tools-1.59.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:21d976419630f72a7cefebe7dcfc451b33d70c805a43ff5a60c43367813f0527"}, + {file = "grpcio_tools-1.59.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:396106f92ea6ab2157535e1a009bac99aa15680ca8addbc8e7c1a4d3f5b1fb2c"}, + {file = "grpcio_tools-1.59.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:4f064483e0046a4a193d6c67b26ea0f61737e8232ff61636a7fa0bc5244458be"}, + {file = "grpcio_tools-1.59.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6dc6da8e3780df25095c1952f45c334e1554f25b991ffe75dbf0408795d27a0"}, + {file = "grpcio_tools-1.59.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87111be05c1a159ce3fffbfad86ff69fd4bf1702cde127eb698d8e8c3a018ab6"}, + {file = "grpcio_tools-1.59.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:83453a13c2120238eb7fb993b03b370496e76071a7b45c816aa682d9226d29c1"}, + {file = "grpcio_tools-1.59.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4384b29d8e126bc6e24a5efd9d60a2a2015867c7109fa67ff2ed274b3f4a05c5"}, + {file = "grpcio_tools-1.59.3-cp312-cp312-win32.whl", hash = "sha256:ce1372c9acde9d74c7e54858598ac0c5203dd3ec24b9085f7a8b2f33cc156736"}, + {file = "grpcio_tools-1.59.3-cp312-cp312-win_amd64.whl", hash = "sha256:84179e3a7c9067e993700b3255f2adc10e9b16e8dd28525d1dd1a02b9ca603ee"}, + {file = "grpcio_tools-1.59.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:592208a9a02df75993cc4dba111d2b81f0e6a3f3837856be239e1aceb6651f31"}, + {file = "grpcio_tools-1.59.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:1abe30ce770ac4fed966d017771fa7f8ced6a279de7ce68023e2c07f07911e76"}, + {file = "grpcio_tools-1.59.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:4dce57668e2aa8c3bb0b2a0bb766a2654ee0f4d8d31e02a6001e98af18569285"}, + {file = "grpcio_tools-1.59.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acaefd3c362250a02cc93fc1b5440e3cb30ab8d7fd81594b2975ea19f123aae3"}, + {file = "grpcio_tools-1.59.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76542e1c11e3f2c22c19cff6b3233caab35924fad1f685ce63184d31b4050fa8"}, + {file = "grpcio_tools-1.59.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:64fd1efb23da054f61aca2346c5139f317b7f4c545f6dbda5ec246f281af8e86"}, + {file = "grpcio_tools-1.59.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ea6454acde508c9a62dab3f59e98b32e32b26aa60df20080982503bb7db51304"}, + {file = "grpcio_tools-1.59.3-cp37-cp37m-win_amd64.whl", hash = "sha256:a048d4bde526f3c6e364abea2c3a481f3bbebc4bfa7fdcfcc3e5ee4f8ab9c4c5"}, + {file = "grpcio_tools-1.59.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:b4db59a62d31c98105af08b1bfb8878c239e4cf31088f2d9864756cdfec67746"}, + {file = "grpcio_tools-1.59.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:05ac0f6683759e5508081c09af26cb6cc949c2c54d75ff8b76344709f78dda53"}, + {file = "grpcio_tools-1.59.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:a031e1ced828a00f1eb59db5f5d4dd39d3bd6a7df8187f84830d4a32a1bbc686"}, + {file = "grpcio_tools-1.59.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f48b4409b306675b7673dad725c9cf3234bf05623bf8a193ad14af39d0368ba6"}, + {file = "grpcio_tools-1.59.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36524d97cc936767a69815b90be76a1420b3218a7724ce69cde6ece794e72a17"}, + {file = "grpcio_tools-1.59.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7cc7e8b893a6c37a8a28735fede1aa40275988a668d1e22c5f234938a77d811d"}, + {file = "grpcio_tools-1.59.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:917be645a71cf9592d2f5a3589c20c074a6539954017e8e2dca5e8ba13025625"}, + {file = "grpcio_tools-1.59.3-cp38-cp38-win32.whl", hash = "sha256:a1394b7a65d738ee0ce4eac1fc95158dd9c97b5c3f690d259e6ee0bf131697de"}, + {file = "grpcio_tools-1.59.3-cp38-cp38-win_amd64.whl", hash = "sha256:007745bd3c5a788dcb73eeb6cd773613a834bd2442e7d062dcafe46dbd4bb5f6"}, + {file = "grpcio_tools-1.59.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:102b5f14a500dbb766f24a96884d9572a3ea7a56d69695461100fb71ec922ef6"}, + {file = "grpcio_tools-1.59.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:46c384a0e30a8422a3e2c5530b3cd69b652dd659549907e2eaac7ca4e0ab614d"}, + {file = "grpcio_tools-1.59.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:ee013da4f5a4ef50fdeca372470733bc402677a4dc0023ee94bf42478b5a620d"}, + {file = "grpcio_tools-1.59.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7883ce3d532c09f29c016fdac107f9a3dc43f9e6b60faf8b91fcba21824269"}, + {file = "grpcio_tools-1.59.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2861e4814ebc147854c2246092c433931f4c15f3c8105ae8716b1e282313a5ae"}, + {file = "grpcio_tools-1.59.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d93590a6a82469f3e58e39692230d99c43a39b215cb581e072dcd52286471152"}, + {file = "grpcio_tools-1.59.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f8396183e6e0a16178773963dc21b58c0c532783476fda314198a9e42f57af7d"}, + {file = "grpcio_tools-1.59.3-cp39-cp39-win32.whl", hash = "sha256:6747b1d82d08e0f5e1a6438532343a1c5504147d1a199c5756e702e5f916de4c"}, + {file = "grpcio_tools-1.59.3-cp39-cp39-win_amd64.whl", hash = "sha256:3a560dcb176dd42c37af5d37299e318341a572547e32b942247daa834d2164c0"}, ] [package.dependencies] -grpcio = ">=1.59.0" +grpcio = ">=1.59.3" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" [[package]] name = "hf-transfer" -version = "0.1.3" +version = "0.1.4" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "hf_transfer-0.1.3-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:862b6ddba8e236bdc73408c20d020cfe5069cac3fd0b6de901c46f031df2b7d9"}, - {file = "hf_transfer-0.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:569ef1ec6fec182e706ade4ea0c63f8510fd618ed7ced7c772efaafac7245b07"}, - {file = "hf_transfer-0.1.3-cp310-none-win_amd64.whl", hash = "sha256:c9faa88b3491c50d4aa75faf18ae24040cd91aa0565c7f7ba2357dbcbf8372f6"}, - {file = "hf_transfer-0.1.3-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d53954ecfaadc84c15481bf5d4c7282323196b4b6df1d1be54208d4fdedfb407"}, - {file = "hf_transfer-0.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:334862f4a82f8a09d6d3f550e67d7e498bb8882e678b7725638254fed3276801"}, - {file = "hf_transfer-0.1.3-cp311-none-win_amd64.whl", hash = "sha256:da92a1483a66cf2baa96de133c75e7d5d9a60e4a0e60d228f26c573c73a1feb6"}, - {file = "hf_transfer-0.1.3-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:617692a70cf299576d82cfc860923f29ec5c834a3f5242bc0853d4f106670398"}, - {file = "hf_transfer-0.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca218fb6efc5c22379d9e64989e84bff426fcf5664fdbbf7cd70aa8b79497652"}, - {file = "hf_transfer-0.1.3-cp37-none-win_amd64.whl", hash = "sha256:6e5201b648df6106c232fcdb507db734081fd6220dfb1c432bd27c6fa9453331"}, - {file = "hf_transfer-0.1.3-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:445edfcd9d59c9d2541957177a9c9225b1f0e8855f6311fb16e20f67c3426421"}, - {file = "hf_transfer-0.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c1bdfa554f0b0936c1623b99058c6998a00fdcd86f75d9203f3f66572d2e30c"}, - {file = "hf_transfer-0.1.3-cp38-none-win_amd64.whl", hash = "sha256:606f2fe436e5be73f07987a56cd97c695805413d29203ae39ebd9fc596405435"}, - {file = "hf_transfer-0.1.3-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9913f5ad2644a1f57c1b7755a7d959ca5e0189863bb0473817d0707af230bf6a"}, - {file = "hf_transfer-0.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d935946791e338f748e05a23df877d74fbcd39dc7b537f0aa2e5a5841cf7dde8"}, - {file = "hf_transfer-0.1.3-cp39-none-win_amd64.whl", hash = "sha256:79099ac043423b263a2843a24213418f309d5c8bc458776622bffe012ebced73"}, - {file = "hf_transfer-0.1.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ce6c5965a57d94db5e043aa488a4df929a32000db125d9c9a1d325e8c7006dc"}, - {file = "hf_transfer-0.1.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a7934c8e491bb395731f677f66dd5f6641432f338a3a9efc9f0b6c186d37cf8"}, - {file = "hf_transfer-0.1.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb8b41360c7e3d7700c147b70688aed0a03e86fbe5bcfdee079b0e634f026f9"}, - {file = "hf_transfer-0.1.3.tar.gz", hash = "sha256:7afd7eb03efad7812a48591b639b2e3f3d1f93c1e9060c18cc63ebf08d7e193c"}, + {file = "hf_transfer-0.1.4-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:6ff5fbde30a5bed35ef8f0d4ba78bde9f6d60a233dbff78a0e4035d6e6f71e4c"}, + {file = "hf_transfer-0.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1c5c20f76e7f3451cff476b85c55dcb8566ebc94a596cb9eb39c0bb75db8675"}, + {file = "hf_transfer-0.1.4-cp310-none-win_amd64.whl", hash = "sha256:84c3ce20c68863a7d998711b98726ba9ae8f2e3fc0d685bc2c9ac9833c0f4048"}, + {file = "hf_transfer-0.1.4-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:dab1cf4e2e6fcb963fe0e48e6b5e3a95cf65ee376c7b6618a05dbb2ef0dde183"}, + {file = "hf_transfer-0.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63c9c7aef90facf45391c86131ed00e74333637735cfec52da4f5170004d0b3f"}, + {file = "hf_transfer-0.1.4-cp311-none-win_amd64.whl", hash = "sha256:eca1fe6ae145e88455d0a174248080498cea52ad45cee50702070b47dffa421f"}, + {file = "hf_transfer-0.1.4-cp312-cp312-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d07c0d26b5c01ad50d22ddcff7d30c4e8cbb823565b7f61e0ddb35f7faeae415"}, + {file = "hf_transfer-0.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b9cf169c3c64883b07f7ded5e3f14ae1d437eb77448738b88c923fc5597c47"}, + {file = "hf_transfer-0.1.4-cp312-none-win_amd64.whl", hash = "sha256:6b8518b9ebb85b0238745be81f7b88383c7ea216dd8407d46444bcc7806dc0ef"}, + {file = "hf_transfer-0.1.4-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:ea32e9f91de3f2dad3567577c293f2e81a9309e680def4712ec0c4ea49be6833"}, + {file = "hf_transfer-0.1.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e81a10dbf2ac534083da06c200456b5d10ba7a1e8c4c5c48f7ea1ca4cf6af474"}, + {file = "hf_transfer-0.1.4-cp37-none-win_amd64.whl", hash = "sha256:97555bbff69a0459712e5d25d659c0dc74cb8f9726562ca66241f1e1b081f6a9"}, + {file = "hf_transfer-0.1.4-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:38bce7a511952e1b804168e956cd3a3b1ff7e38828259c3cdae27614060b90c5"}, + {file = "hf_transfer-0.1.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1977e94e8c8fc8a0e9ce74a651d4694629e526da246a492855fcfb710aa489"}, + {file = "hf_transfer-0.1.4-cp38-none-win_amd64.whl", hash = "sha256:6ca2d2c40e5e94c5de7e502037ad23ac1d803a2a12760b15b3e3f88c616202bd"}, + {file = "hf_transfer-0.1.4-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:c04a93acb58e50b8da1e2258185e54f6bf48ba24bf95e470310178b7047c1017"}, + {file = "hf_transfer-0.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3028a807363e0b2c64985c44732ba4ab187a569f013367d2115a6e09ae95031"}, + {file = "hf_transfer-0.1.4-cp39-none-win_amd64.whl", hash = "sha256:dc9c7c1d0d79fc06baf86d41620623bb6bb2736755329ea6b1ec5faf71e3e36b"}, + {file = "hf_transfer-0.1.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a466ae2b11d72df9e0005eb8ff7f537d5460c98b64fb6e49f3076ee14040dcf"}, + {file = "hf_transfer-0.1.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb34a023276936d4716112e17daea4ff98afc35b6113dd0f0383710dc208c058"}, + {file = "hf_transfer-0.1.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0647b84d7ff0eee1de6479179a5d43d0695001733f17eecc00153f0f8ab1ac"}, + {file = "hf_transfer-0.1.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27d0bc1f8b79a6d65751efbce7eb02d2c1bd7e4de1a46aac18995461590ce4dd"}, + {file = "hf_transfer-0.1.4.tar.gz", hash = "sha256:687e090639cd52a48dedbfaa9e455a2c99c5169ece3d911f95983b1d4d4c84ed"}, ] [[package]] @@ -844,7 +837,7 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, @@ -879,7 +872,7 @@ dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." -optional = false +optional = true python-versions = ">=3.7" files = [ {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, @@ -948,7 +941,7 @@ files = [ name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" -optional = false +optional = true python-versions = "*" files = [ {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, @@ -1074,61 +1067,206 @@ dill = ">=0.3.7" [[package]] name = "networkx" -version = "3.1" +version = "3.2.1" description = "Python package for creating and manipulating graphs and networks" -optional = false -python-versions = ">=3.8" +optional = true +python-versions = ">=3.9" files = [ - {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, - {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, + {file = "networkx-3.2.1-py3-none-any.whl", hash = "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2"}, + {file = "networkx-3.2.1.tar.gz", hash = "sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6"}, ] [package.extras] -default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] -developer = ["mypy (>=1.1)", "pre-commit (>=3.2)"] -doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.13)", "sphinx (>=6.1)", "sphinx-gallery (>=0.12)", "texext (>=0.6.7)"] -extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] -test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] +default = ["matplotlib (>=3.5)", "numpy (>=1.22)", "pandas (>=1.4)", "scipy (>=1.9,!=1.11.0,!=1.11.1)"] +developer = ["changelist (==0.4)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] +doc = ["nb2plots (>=0.7)", "nbconvert (<7.9)", "numpydoc (>=1.6)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.14)", "sphinx (>=7)", "sphinx-gallery (>=0.14)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.11)", "sympy (>=1.10)"] +test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "numpy" -version = "1.26.0" +version = "1.26.2" description = "Fundamental package for array computing in Python" optional = false -python-versions = "<3.13,>=3.9" -files = [ - {file = "numpy-1.26.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8db2f125746e44dce707dd44d4f4efeea8d7e2b43aace3f8d1f235cfa2733dd"}, - {file = "numpy-1.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0621f7daf973d34d18b4e4bafb210bbaf1ef5e0100b5fa750bd9cde84c7ac292"}, - {file = "numpy-1.26.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51be5f8c349fdd1a5568e72713a21f518e7d6707bcf8503b528b88d33b57dc68"}, - {file = "numpy-1.26.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:767254ad364991ccfc4d81b8152912e53e103ec192d1bb4ea6b1f5a7117040be"}, - {file = "numpy-1.26.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:436c8e9a4bdeeee84e3e59614d38c3dbd3235838a877af8c211cfcac8a80b8d3"}, - {file = "numpy-1.26.0-cp310-cp310-win32.whl", hash = "sha256:c2e698cb0c6dda9372ea98a0344245ee65bdc1c9dd939cceed6bb91256837896"}, - {file = "numpy-1.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:09aaee96c2cbdea95de76ecb8a586cb687d281c881f5f17bfc0fb7f5890f6b91"}, - {file = "numpy-1.26.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:637c58b468a69869258b8ae26f4a4c6ff8abffd4a8334c830ffb63e0feefe99a"}, - {file = "numpy-1.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:306545e234503a24fe9ae95ebf84d25cba1fdc27db971aa2d9f1ab6bba19a9dd"}, - {file = "numpy-1.26.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c6adc33561bd1d46f81131d5352348350fc23df4d742bb246cdfca606ea1208"}, - {file = "numpy-1.26.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e062aa24638bb5018b7841977c360d2f5917268d125c833a686b7cbabbec496c"}, - {file = "numpy-1.26.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:546b7dd7e22f3c6861463bebb000646fa730e55df5ee4a0224408b5694cc6148"}, - {file = "numpy-1.26.0-cp311-cp311-win32.whl", hash = "sha256:c0b45c8b65b79337dee5134d038346d30e109e9e2e9d43464a2970e5c0e93229"}, - {file = "numpy-1.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:eae430ecf5794cb7ae7fa3808740b015aa80747e5266153128ef055975a72b99"}, - {file = "numpy-1.26.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:166b36197e9debc4e384e9c652ba60c0bacc216d0fc89e78f973a9760b503388"}, - {file = "numpy-1.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f042f66d0b4ae6d48e70e28d487376204d3cbf43b84c03bac57e28dac6151581"}, - {file = "numpy-1.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5e18e5b14a7560d8acf1c596688f4dfd19b4f2945b245a71e5af4ddb7422feb"}, - {file = "numpy-1.26.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6bad22a791226d0a5c7c27a80a20e11cfe09ad5ef9084d4d3fc4a299cca505"}, - {file = "numpy-1.26.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4acc65dd65da28060e206c8f27a573455ed724e6179941edb19f97e58161bb69"}, - {file = "numpy-1.26.0-cp312-cp312-win32.whl", hash = "sha256:bb0d9a1aaf5f1cb7967320e80690a1d7ff69f1d47ebc5a9bea013e3a21faec95"}, - {file = "numpy-1.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:ee84ca3c58fe48b8ddafdeb1db87388dce2c3c3f701bf447b05e4cfcc3679112"}, - {file = "numpy-1.26.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4a873a8180479bc829313e8d9798d5234dfacfc2e8a7ac188418189bb8eafbd2"}, - {file = "numpy-1.26.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:914b28d3215e0c721dc75db3ad6d62f51f630cb0c277e6b3bcb39519bed10bd8"}, - {file = "numpy-1.26.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c78a22e95182fb2e7874712433eaa610478a3caf86f28c621708d35fa4fd6e7f"}, - {file = "numpy-1.26.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f737708b366c36b76e953c46ba5827d8c27b7a8c9d0f471810728e5a2fe57c"}, - {file = "numpy-1.26.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b44e6a09afc12952a7d2a58ca0a2429ee0d49a4f89d83a0a11052da696440e49"}, - {file = "numpy-1.26.0-cp39-cp39-win32.whl", hash = "sha256:5671338034b820c8d58c81ad1dafc0ed5a00771a82fccc71d6438df00302094b"}, - {file = "numpy-1.26.0-cp39-cp39-win_amd64.whl", hash = "sha256:020cdbee66ed46b671429c7265cf00d8ac91c046901c55684954c3958525dab2"}, - {file = "numpy-1.26.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0792824ce2f7ea0c82ed2e4fecc29bb86bee0567a080dacaf2e0a01fe7654369"}, - {file = "numpy-1.26.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d484292eaeb3e84a51432a94f53578689ffdea3f90e10c8b203a99be5af57d8"}, - {file = "numpy-1.26.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:186ba67fad3c60dbe8a3abff3b67a91351100f2661c8e2a80364ae6279720299"}, - {file = "numpy-1.26.0.tar.gz", hash = "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf"}, +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3703fc9258a4a122d17043e57b35e5ef1c5a5837c3db8be396c82e04c1cf9b0f"}, + {file = "numpy-1.26.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc392fdcbd21d4be6ae1bb4475a03ce3b025cd49a9be5345d76d7585aea69440"}, + {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36340109af8da8805d8851ef1d74761b3b88e81a9bd80b290bbfed61bd2b4f75"}, + {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcc008217145b3d77abd3e4d5ef586e3bdfba8fe17940769f8aa09b99e856c00"}, + {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ced40d4e9e18242f70dd02d739e44698df3dcb010d31f495ff00a31ef6014fe"}, + {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b272d4cecc32c9e19911891446b72e986157e6a1809b7b56518b4f3755267523"}, + {file = "numpy-1.26.2-cp310-cp310-win32.whl", hash = "sha256:22f8fc02fdbc829e7a8c578dd8d2e15a9074b630d4da29cda483337e300e3ee9"}, + {file = "numpy-1.26.2-cp310-cp310-win_amd64.whl", hash = "sha256:26c9d33f8e8b846d5a65dd068c14e04018d05533b348d9eaeef6c1bd787f9919"}, + {file = "numpy-1.26.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b96e7b9c624ef3ae2ae0e04fa9b460f6b9f17ad8b4bec6d7756510f1f6c0c841"}, + {file = "numpy-1.26.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aa18428111fb9a591d7a9cc1b48150097ba6a7e8299fb56bdf574df650e7d1f1"}, + {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06fa1ed84aa60ea6ef9f91ba57b5ed963c3729534e6e54055fc151fad0423f0a"}, + {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96ca5482c3dbdd051bcd1fce8034603d6ebfc125a7bd59f55b40d8f5d246832b"}, + {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:854ab91a2906ef29dc3925a064fcd365c7b4da743f84b123002f6139bcb3f8a7"}, + {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f43740ab089277d403aa07567be138fc2a89d4d9892d113b76153e0e412409f8"}, + {file = "numpy-1.26.2-cp311-cp311-win32.whl", hash = "sha256:a2bbc29fcb1771cd7b7425f98b05307776a6baf43035d3b80c4b0f29e9545186"}, + {file = "numpy-1.26.2-cp311-cp311-win_amd64.whl", hash = "sha256:2b3fca8a5b00184828d12b073af4d0fc5fdd94b1632c2477526f6bd7842d700d"}, + {file = "numpy-1.26.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a4cd6ed4a339c21f1d1b0fdf13426cb3b284555c27ac2f156dfdaaa7e16bfab0"}, + {file = "numpy-1.26.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d5244aabd6ed7f312268b9247be47343a654ebea52a60f002dc70c769048e75"}, + {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a3cdb4d9c70e6b8c0814239ead47da00934666f668426fc6e94cce869e13fd7"}, + {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa317b2325f7aa0a9471663e6093c210cb2ae9c0ad824732b307d2c51983d5b6"}, + {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:174a8880739c16c925799c018f3f55b8130c1f7c8e75ab0a6fa9d41cab092fd6"}, + {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f79b231bf5c16b1f39c7f4875e1ded36abee1591e98742b05d8a0fb55d8a3eec"}, + {file = "numpy-1.26.2-cp312-cp312-win32.whl", hash = "sha256:4a06263321dfd3598cacb252f51e521a8cb4b6df471bb12a7ee5cbab20ea9167"}, + {file = "numpy-1.26.2-cp312-cp312-win_amd64.whl", hash = "sha256:b04f5dc6b3efdaab541f7857351aac359e6ae3c126e2edb376929bd3b7f92d7e"}, + {file = "numpy-1.26.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4eb8df4bf8d3d90d091e0146f6c28492b0be84da3e409ebef54349f71ed271ef"}, + {file = "numpy-1.26.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1a13860fdcd95de7cf58bd6f8bc5a5ef81c0b0625eb2c9a783948847abbef2c2"}, + {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64308ebc366a8ed63fd0bf426b6a9468060962f1a4339ab1074c228fa6ade8e3"}, + {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baf8aab04a2c0e859da118f0b38617e5ee65d75b83795055fb66c0d5e9e9b818"}, + {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d73a3abcac238250091b11caef9ad12413dab01669511779bc9b29261dd50210"}, + {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b361d369fc7e5e1714cf827b731ca32bff8d411212fccd29ad98ad622449cc36"}, + {file = "numpy-1.26.2-cp39-cp39-win32.whl", hash = "sha256:bd3f0091e845164a20bd5a326860c840fe2af79fa12e0469a12768a3ec578d80"}, + {file = "numpy-1.26.2-cp39-cp39-win_amd64.whl", hash = "sha256:2beef57fb031dcc0dc8fa4fe297a742027b954949cabb52a2a376c144e5e6060"}, + {file = "numpy-1.26.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1cc3d5029a30fb5f06704ad6b23b35e11309491c999838c31f124fee32107c79"}, + {file = "numpy-1.26.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94cc3c222bb9fb5a12e334d0479b97bb2df446fbe622b470928f5284ffca3f8d"}, + {file = "numpy-1.26.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe6b44fb8fcdf7eda4ef4461b97b3f63c466b27ab151bec2366db8b197387841"}, + {file = "numpy-1.26.2.tar.gz", hash = "sha256:f65738447676ab5777f11e6bbbdb8ce11b785e105f690bc45966574816b6d3ea"}, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.1.3.1" +description = "CUBLAS native runtime libraries" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:ee53ccca76a6fc08fb9701aa95b6ceb242cdaab118c3bb152af4e579af792728"}, + {file = "nvidia_cublas_cu12-12.1.3.1-py3-none-win_amd64.whl", hash = "sha256:2b964d60e8cf11b5e1073d179d85fa340c120e99b3067558f3cf98dd69d02906"}, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.1.105" +description = "CUDA profiling tools runtime libs." +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:e54fde3983165c624cb79254ae9818a456eb6e87a7fd4d56a2352c24ee542d7e"}, + {file = "nvidia_cuda_cupti_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:bea8236d13a0ac7190bd2919c3e8e6ce1e402104276e6f9694479e48bb0eb2a4"}, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.1.105" +description = "NVRTC native runtime libraries" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:339b385f50c309763ca65456ec75e17bbefcbbf2893f462cb8b90584cd27a1c2"}, + {file = "nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:0a98a522d9ff138b96c010a65e145dc1b4850e9ecb75a0172371793752fd46ed"}, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.1.105" +description = "CUDA Runtime native Libraries" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:6e258468ddf5796e25f1dc591a31029fa317d97a0a94ed93468fc86301d61e40"}, + {file = "nvidia_cuda_runtime_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:dfb46ef84d73fababab44cf03e3b83f80700d27ca300e537f85f636fac474344"}, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "8.9.2.26" +description = "cuDNN runtime libraries" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl", hash = "sha256:5ccb288774fdfb07a7e7025ffec286971c06d8d7b4fb162525334616d7629ff9"}, +] + +[package.dependencies] +nvidia-cublas-cu12 = "*" + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.0.2.54" +description = "CUFFT native runtime libraries" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl", hash = "sha256:794e3948a1aa71fd817c3775866943936774d1c14e7628c74f6f7417224cdf56"}, + {file = "nvidia_cufft_cu12-11.0.2.54-py3-none-win_amd64.whl", hash = "sha256:d9ac353f78ff89951da4af698f80870b1534ed69993f10a4cf1d96f21357e253"}, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.2.106" +description = "CURAND native runtime libraries" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:9d264c5036dde4e64f1de8c50ae753237c12e0b1348738169cd0f8a536c0e1e0"}, + {file = "nvidia_curand_cu12-10.3.2.106-py3-none-win_amd64.whl", hash = "sha256:75b6b0c574c0037839121317e17fd01f8a69fd2ef8e25853d826fec30bdba74a"}, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.4.5.107" +description = "CUDA solver native runtime libraries" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl", hash = "sha256:8a7ec542f0412294b15072fa7dab71d31334014a69f953004ea7a118206fe0dd"}, + {file = "nvidia_cusolver_cu12-11.4.5.107-py3-none-win_amd64.whl", hash = "sha256:74e0c3a24c78612192a74fcd90dd117f1cf21dea4822e66d89e8ea80e3cd2da5"}, +] + +[package.dependencies] +nvidia-cublas-cu12 = "*" +nvidia-cusparse-cu12 = "*" +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.1.0.106" +description = "CUSPARSE native runtime libraries" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:f3b50f42cf363f86ab21f720998517a659a48131e8d538dc02f8768237bd884c"}, + {file = "nvidia_cusparse_cu12-12.1.0.106-py3-none-win_amd64.whl", hash = "sha256:b798237e81b9719373e8fae8d4f091b70a0cf09d9d85c95a557e11df2d8e9a5a"}, +] + +[package.dependencies] +nvidia-nvjitlink-cu12 = "*" + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.18.1" +description = "NVIDIA Collective Communication Library (NCCL) Runtime" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:1a6c4acefcbebfa6de320f412bf7866de856e786e0462326ba1bac40de0b5e71"}, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.3.101" +description = "Nvidia JIT LTO Library" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_nvjitlink_cu12-12.3.101-py3-none-manylinux1_x86_64.whl", hash = "sha256:64335a8088e2b9d196ae8665430bc6a2b7e6ef2eb877a9c735c804bd4ff6467c"}, + {file = "nvidia_nvjitlink_cu12-12.3.101-py3-none-win_amd64.whl", hash = "sha256:1b2e317e437433753530792f13eece58f0aec21a2b05903be7bffe58a606cbd1"}, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.1.105" +description = "NVIDIA Tools Extension" +optional = true +python-versions = ">=3" +files = [ + {file = "nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:dc21cf308ca5691e7c04d962e213f8a4aa9bbfa23d95412f452254c2caeb09e5"}, + {file = "nvidia_nvtx_cu12-12.1.105-py3-none-win_amd64.whl", hash = "sha256:65f4d98982b31b60026e0e6de73fbdfc09d08a96f4656dd3665ca616a11e1e82"}, ] [[package]] @@ -1298,50 +1436,50 @@ files = [ [[package]] name = "pandas" -version = "2.1.1" +version = "2.1.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = true python-versions = ">=3.9" files = [ - {file = "pandas-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58d997dbee0d4b64f3cb881a24f918b5f25dd64ddf31f467bb9b67ae4c63a1e4"}, - {file = "pandas-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02304e11582c5d090e5a52aec726f31fe3f42895d6bfc1f28738f9b64b6f0614"}, - {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffa8f0966de2c22de408d0e322db2faed6f6e74265aa0856f3824813cf124363"}, - {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1f84c144dee086fe4f04a472b5cd51e680f061adf75c1ae4fc3a9275560f8f4"}, - {file = "pandas-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ce97667d06d69396d72be074f0556698c7f662029322027c226fd7a26965cb"}, - {file = "pandas-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:4c3f32fd7c4dccd035f71734df39231ac1a6ff95e8bdab8d891167197b7018d2"}, - {file = "pandas-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e2959720b70e106bb1d8b6eadd8ecd7c8e99ccdbe03ee03260877184bb2877d"}, - {file = "pandas-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25e8474a8eb258e391e30c288eecec565bfed3e026f312b0cbd709a63906b6f8"}, - {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8bd1685556f3374520466998929bade3076aeae77c3e67ada5ed2b90b4de7f0"}, - {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc3657869c7902810f32bd072f0740487f9e030c1a3ab03e0af093db35a9d14e"}, - {file = "pandas-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:05674536bd477af36aa2effd4ec8f71b92234ce0cc174de34fd21e2ee99adbc2"}, - {file = "pandas-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:b407381258a667df49d58a1b637be33e514b07f9285feb27769cedb3ab3d0b3a"}, - {file = "pandas-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c747793c4e9dcece7bb20156179529898abf505fe32cb40c4052107a3c620b49"}, - {file = "pandas-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3bcad1e6fb34b727b016775bea407311f7721db87e5b409e6542f4546a4951ea"}, - {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5ec7740f9ccb90aec64edd71434711f58ee0ea7f5ed4ac48be11cfa9abf7317"}, - {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29deb61de5a8a93bdd033df328441a79fcf8dd3c12d5ed0b41a395eef9cd76f0"}, - {file = "pandas-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f99bebf19b7e03cf80a4e770a3e65eee9dd4e2679039f542d7c1ace7b7b1daa"}, - {file = "pandas-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:84e7e910096416adec68075dc87b986ff202920fb8704e6d9c8c9897fe7332d6"}, - {file = "pandas-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:366da7b0e540d1b908886d4feb3d951f2f1e572e655c1160f5fde28ad4abb750"}, - {file = "pandas-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9e50e72b667415a816ac27dfcfe686dc5a0b02202e06196b943d54c4f9c7693e"}, - {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1ab6a25da197f03ebe6d8fa17273126120874386b4ac11c1d687df288542dd"}, - {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0dbfea0dd3901ad4ce2306575c54348d98499c95be01b8d885a2737fe4d7a98"}, - {file = "pandas-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0489b0e6aa3d907e909aef92975edae89b1ee1654db5eafb9be633b0124abe97"}, - {file = "pandas-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:4cdb0fab0400c2cb46dafcf1a0fe084c8bb2480a1fa8d81e19d15e12e6d4ded2"}, - {file = "pandas-2.1.1.tar.gz", hash = "sha256:fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b"}, + {file = "pandas-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:acf08a73b5022b479c1be155d4988b72f3020f308f7a87c527702c5f8966d34f"}, + {file = "pandas-2.1.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3cc4469ff0cf9aa3a005870cb49ab8969942b7156e0a46cc3f5abd6b11051dfb"}, + {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35172bff95f598cc5866c047f43c7f4df2c893acd8e10e6653a4b792ed7f19bb"}, + {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59dfe0e65a2f3988e940224e2a70932edc964df79f3356e5f2997c7d63e758b4"}, + {file = "pandas-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0296a66200dee556850d99b24c54c7dfa53a3264b1ca6f440e42bad424caea03"}, + {file = "pandas-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:465571472267a2d6e00657900afadbe6097c8e1dc43746917db4dfc862e8863e"}, + {file = "pandas-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04d4c58e1f112a74689da707be31cf689db086949c71828ef5da86727cfe3f82"}, + {file = "pandas-2.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7fa2ad4ff196768ae63a33f8062e6838efed3a319cf938fdf8b95e956c813042"}, + {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4441ac94a2a2613e3982e502ccec3bdedefe871e8cea54b8775992485c5660ef"}, + {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5ded6ff28abbf0ea7689f251754d3789e1edb0c4d0d91028f0b980598418a58"}, + {file = "pandas-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca5680368a5139d4920ae3dc993eb5106d49f814ff24018b64d8850a52c6ed2"}, + {file = "pandas-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:de21e12bf1511190fc1e9ebc067f14ca09fccfb189a813b38d63211d54832f5f"}, + {file = "pandas-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a5d53c725832e5f1645e7674989f4c106e4b7249c1d57549023ed5462d73b140"}, + {file = "pandas-2.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7cf4cf26042476e39394f1f86868d25b265ff787c9b2f0d367280f11afbdee6d"}, + {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72c84ec1b1d8e5efcbff5312abe92bfb9d5b558f11e0cf077f5496c4f4a3c99e"}, + {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f539e113739a3e0cc15176bf1231a553db0239bfa47a2c870283fd93ba4f683"}, + {file = "pandas-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fc77309da3b55732059e484a1efc0897f6149183c522390772d3561f9bf96c00"}, + {file = "pandas-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:08637041279b8981a062899da0ef47828df52a1838204d2b3761fbd3e9fcb549"}, + {file = "pandas-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b99c4e51ef2ed98f69099c72c75ec904dd610eb41a32847c4fcbc1a975f2d2b8"}, + {file = "pandas-2.1.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7ea8ae8004de0381a2376662c0505bb0a4f679f4c61fbfd122aa3d1b0e5f09d"}, + {file = "pandas-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcd76d67ca2d48f56e2db45833cf9d58f548f97f61eecd3fdc74268417632b8a"}, + {file = "pandas-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1329dbe93a880a3d7893149979caa82d6ba64a25e471682637f846d9dbc10dd2"}, + {file = "pandas-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:321ecdb117bf0f16c339cc6d5c9a06063854f12d4d9bc422a84bb2ed3207380a"}, + {file = "pandas-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:11a771450f36cebf2a4c9dbd3a19dfa8c46c4b905a3ea09dc8e556626060fe71"}, + {file = "pandas-2.1.3.tar.gz", hash = "sha256:22929f84bca106921917eb73c1521317ddd0a4c71b395bcf767a106e3494209f"}, ] [package.dependencies] numpy = [ - {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" tzdata = ">=2022.1" [package.extras] -all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] aws = ["s3fs (>=2022.05.0)"] clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] compression = ["zstandard (>=0.17.0)"] @@ -1361,14 +1499,14 @@ plot = ["matplotlib (>=3.6.1)"] postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] spss = ["pyreadstat (>=1.1.5)"] sql-other = ["SQLAlchemy (>=1.4.36)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.8.0)"] [[package]] name = "peft" version = "0.4.0" description = "Parameter-Efficient Fine-Tuning (PEFT)" -optional = false +optional = true python-versions = ">=3.8.0" files = [ {file = "peft-0.4.0-py3-none-any.whl", hash = "sha256:2cf992772a6d703814477e0bdcdadd68cb8ea388111ce2d793dd2ff0e438f357"}, @@ -1393,65 +1531,65 @@ test = ["black (>=22.0,<23.0)", "datasets", "diffusers", "hf-doc-builder", "para [[package]] name = "pillow" -version = "10.0.1" +version = "10.1.0" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.8" files = [ - {file = "Pillow-10.0.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a"}, - {file = "Pillow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d"}, - {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d"}, - {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19"}, - {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f"}, - {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff"}, - {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf"}, - {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd"}, - {file = "Pillow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0"}, - {file = "Pillow-10.0.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1"}, - {file = "Pillow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1"}, - {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21"}, - {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54"}, - {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205"}, - {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2"}, - {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b"}, - {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1"}, - {file = "Pillow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088"}, - {file = "Pillow-10.0.1-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b"}, - {file = "Pillow-10.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed"}, - {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635"}, - {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad"}, - {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a"}, - {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91"}, - {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4"}, - {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08"}, - {file = "Pillow-10.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08"}, - {file = "Pillow-10.0.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a"}, - {file = "Pillow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68"}, - {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500"}, - {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21"}, - {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d"}, - {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7"}, - {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a"}, - {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7"}, - {file = "Pillow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3"}, - {file = "Pillow-10.0.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849"}, - {file = "Pillow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1"}, - {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37"}, - {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876"}, - {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f"}, - {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145"}, - {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2"}, - {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf"}, - {file = "Pillow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971"}, - {file = "Pillow-10.0.1-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db"}, - {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e"}, - {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4"}, - {file = "Pillow-10.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f"}, - {file = "Pillow-10.0.1-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf"}, - {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317"}, - {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d"}, - {file = "Pillow-10.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d"}, - {file = "Pillow-10.0.1.tar.gz", hash = "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d"}, + {file = "Pillow-10.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106"}, + {file = "Pillow-10.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273"}, + {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666"}, + {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2"}, + {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593"}, + {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db"}, + {file = "Pillow-10.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f"}, + {file = "Pillow-10.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818"}, + {file = "Pillow-10.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57"}, + {file = "Pillow-10.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7"}, + {file = "Pillow-10.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7"}, + {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610"}, + {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839"}, + {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172"}, + {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061"}, + {file = "Pillow-10.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262"}, + {file = "Pillow-10.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992"}, + {file = "Pillow-10.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a"}, + {file = "Pillow-10.1.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b"}, + {file = "Pillow-10.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d"}, + {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27"}, + {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312"}, + {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de"}, + {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651"}, + {file = "Pillow-10.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b"}, + {file = "Pillow-10.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f"}, + {file = "Pillow-10.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996"}, + {file = "Pillow-10.1.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793"}, + {file = "Pillow-10.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e"}, + {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2"}, + {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a"}, + {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01"}, + {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d"}, + {file = "Pillow-10.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80"}, + {file = "Pillow-10.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212"}, + {file = "Pillow-10.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14"}, + {file = "Pillow-10.1.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099"}, + {file = "Pillow-10.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616"}, + {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb"}, + {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219"}, + {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34"}, + {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd"}, + {file = "Pillow-10.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28"}, + {file = "Pillow-10.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2"}, + {file = "Pillow-10.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256"}, + {file = "Pillow-10.1.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7"}, + {file = "Pillow-10.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba"}, + {file = "Pillow-10.1.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4"}, + {file = "Pillow-10.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9"}, + {file = "Pillow-10.1.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e"}, + {file = "Pillow-10.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412"}, + {file = "Pillow-10.1.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b"}, + {file = "Pillow-10.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f"}, + {file = "Pillow-10.1.0.tar.gz", hash = "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38"}, ] [package.extras] @@ -1475,47 +1613,47 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.24.3" +version = "4.25.1" description = "" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "protobuf-4.24.3-cp310-abi3-win32.whl", hash = "sha256:20651f11b6adc70c0f29efbe8f4a94a74caf61b6200472a9aea6e19898f9fcf4"}, - {file = "protobuf-4.24.3-cp310-abi3-win_amd64.whl", hash = "sha256:3d42e9e4796a811478c783ef63dc85b5a104b44aaaca85d4864d5b886e4b05e3"}, - {file = "protobuf-4.24.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:6e514e8af0045be2b56e56ae1bb14f43ce7ffa0f68b1c793670ccbe2c4fc7d2b"}, - {file = "protobuf-4.24.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:ba53c2f04798a326774f0e53b9c759eaef4f6a568ea7072ec6629851c8435959"}, - {file = "protobuf-4.24.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:f6ccbcf027761a2978c1406070c3788f6de4a4b2cc20800cc03d52df716ad675"}, - {file = "protobuf-4.24.3-cp37-cp37m-win32.whl", hash = "sha256:1b182c7181a2891e8f7f3a1b5242e4ec54d1f42582485a896e4de81aa17540c2"}, - {file = "protobuf-4.24.3-cp37-cp37m-win_amd64.whl", hash = "sha256:b0271a701e6782880d65a308ba42bc43874dabd1a0a0f41f72d2dac3b57f8e76"}, - {file = "protobuf-4.24.3-cp38-cp38-win32.whl", hash = "sha256:e29d79c913f17a60cf17c626f1041e5288e9885c8579832580209de8b75f2a52"}, - {file = "protobuf-4.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:067f750169bc644da2e1ef18c785e85071b7c296f14ac53e0900e605da588719"}, - {file = "protobuf-4.24.3-cp39-cp39-win32.whl", hash = "sha256:2da777d34b4f4f7613cdf85c70eb9a90b1fbef9d36ae4a0ccfe014b0b07906f1"}, - {file = "protobuf-4.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:f631bb982c5478e0c1c70eab383af74a84be66945ebf5dd6b06fc90079668d0b"}, - {file = "protobuf-4.24.3-py3-none-any.whl", hash = "sha256:f6f8dc65625dadaad0c8545319c2e2f0424fede988368893ca3844261342c11a"}, - {file = "protobuf-4.24.3.tar.gz", hash = "sha256:12e9ad2ec079b833176d2921be2cb24281fa591f0b119b208b788adc48c2561d"}, + {file = "protobuf-4.25.1-cp310-abi3-win32.whl", hash = "sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7"}, + {file = "protobuf-4.25.1-cp310-abi3-win_amd64.whl", hash = "sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b"}, + {file = "protobuf-4.25.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd"}, + {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb"}, + {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7"}, + {file = "protobuf-4.25.1-cp38-cp38-win32.whl", hash = "sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd"}, + {file = "protobuf-4.25.1-cp38-cp38-win_amd64.whl", hash = "sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0"}, + {file = "protobuf-4.25.1-cp39-cp39-win32.whl", hash = "sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510"}, + {file = "protobuf-4.25.1-cp39-cp39-win_amd64.whl", hash = "sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10"}, + {file = "protobuf-4.25.1-py3-none-any.whl", hash = "sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6"}, + {file = "protobuf-4.25.1.tar.gz", hash = "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2"}, ] [[package]] name = "psutil" -version = "5.9.5" +version = "5.9.6" description = "Cross-platform lib for process and system monitoring in Python." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, - {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, - {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, - {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, - {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, - {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, - {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, - {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, - {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, - {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, - {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, - {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, - {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, - {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "psutil-5.9.6-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d"}, + {file = "psutil-5.9.6-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c"}, + {file = "psutil-5.9.6-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28"}, + {file = "psutil-5.9.6-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017"}, + {file = "psutil-5.9.6-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c"}, + {file = "psutil-5.9.6-cp27-none-win32.whl", hash = "sha256:70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9"}, + {file = "psutil-5.9.6-cp27-none-win_amd64.whl", hash = "sha256:51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac"}, + {file = "psutil-5.9.6-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a"}, + {file = "psutil-5.9.6-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c"}, + {file = "psutil-5.9.6-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4"}, + {file = "psutil-5.9.6-cp36-cp36m-win32.whl", hash = "sha256:3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602"}, + {file = "psutil-5.9.6-cp36-cp36m-win_amd64.whl", hash = "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"}, + {file = "psutil-5.9.6-cp37-abi3-win32.whl", hash = "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c"}, + {file = "psutil-5.9.6-cp37-abi3-win_amd64.whl", hash = "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a"}, + {file = "psutil-5.9.6-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57"}, + {file = "psutil-5.9.6.tar.gz", hash = "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a"}, ] [package.extras] @@ -1523,54 +1661,72 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "pyarrow" -version = "13.0.0" +version = "14.0.1" description = "Python library for Apache Arrow" optional = true python-versions = ">=3.8" files = [ - {file = "pyarrow-13.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:1afcc2c33f31f6fb25c92d50a86b7a9f076d38acbcb6f9e74349636109550148"}, - {file = "pyarrow-13.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:70fa38cdc66b2fc1349a082987f2b499d51d072faaa6b600f71931150de2e0e3"}, - {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd57b13a6466822498238877892a9b287b0a58c2e81e4bdb0b596dbb151cbb73"}, - {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8ce69f7bf01de2e2764e14df45b8404fc6f1a5ed9871e8e08a12169f87b7a26"}, - {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:588f0d2da6cf1b1680974d63be09a6530fd1bd825dc87f76e162404779a157dc"}, - {file = "pyarrow-13.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:6241afd72b628787b4abea39e238e3ff9f34165273fad306c7acf780dd850956"}, - {file = "pyarrow-13.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:fda7857e35993673fcda603c07d43889fca60a5b254052a462653f8656c64f44"}, - {file = "pyarrow-13.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:aac0ae0146a9bfa5e12d87dda89d9ef7c57a96210b899459fc2f785303dcbb67"}, - {file = "pyarrow-13.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d7759994217c86c161c6a8060509cfdf782b952163569606bb373828afdd82e8"}, - {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:868a073fd0ff6468ae7d869b5fc1f54de5c4255b37f44fb890385eb68b68f95d"}, - {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51be67e29f3cfcde263a113c28e96aa04362ed8229cb7c6e5f5c719003659d33"}, - {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d1b4e7176443d12610874bb84d0060bf080f000ea9ed7c84b2801df851320295"}, - {file = "pyarrow-13.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:69b6f9a089d116a82c3ed819eea8fe67dae6105f0d81eaf0fdd5e60d0c6e0944"}, - {file = "pyarrow-13.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:ab1268db81aeb241200e321e220e7cd769762f386f92f61b898352dd27e402ce"}, - {file = "pyarrow-13.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:ee7490f0f3f16a6c38f8c680949551053c8194e68de5046e6c288e396dccee80"}, - {file = "pyarrow-13.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3ad79455c197a36eefbd90ad4aa832bece7f830a64396c15c61a0985e337287"}, - {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68fcd2dc1b7d9310b29a15949cdd0cb9bc34b6de767aff979ebf546020bf0ba0"}, - {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc6fd330fd574c51d10638e63c0d00ab456498fc804c9d01f2a61b9264f2c5b2"}, - {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:e66442e084979a97bb66939e18f7b8709e4ac5f887e636aba29486ffbf373763"}, - {file = "pyarrow-13.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:0f6eff839a9e40e9c5610d3ff8c5bdd2f10303408312caf4c8003285d0b49565"}, - {file = "pyarrow-13.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b30a27f1cddf5c6efcb67e598d7823a1e253d743d92ac32ec1eb4b6a1417867"}, - {file = "pyarrow-13.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:09552dad5cf3de2dc0aba1c7c4b470754c69bd821f5faafc3d774bedc3b04bb7"}, - {file = "pyarrow-13.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3896ae6c205d73ad192d2fc1489cd0edfab9f12867c85b4c277af4d37383c18c"}, - {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6647444b21cb5e68b593b970b2a9a07748dd74ea457c7dadaa15fd469c48ada1"}, - {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47663efc9c395e31d09c6aacfa860f4473815ad6804311c5433f7085415d62a7"}, - {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b9ba6b6d34bd2563345488cf444510588ea42ad5613df3b3509f48eb80250afd"}, - {file = "pyarrow-13.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:d00d374a5625beeb448a7fa23060df79adb596074beb3ddc1838adb647b6ef09"}, - {file = "pyarrow-13.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:c51afd87c35c8331b56f796eff954b9c7f8d4b7fef5903daf4e05fcf017d23a8"}, - {file = "pyarrow-13.0.0.tar.gz", hash = "sha256:83333726e83ed44b0ac94d8d7a21bbdee4a05029c3b1e8db58a863eec8fd8a33"}, + {file = "pyarrow-14.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:96d64e5ba7dceb519a955e5eeb5c9adcfd63f73a56aea4722e2cc81364fc567a"}, + {file = "pyarrow-14.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a8ae88c0038d1bc362a682320112ee6774f006134cd5afc291591ee4bc06505"}, + {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f6f053cb66dc24091f5511e5920e45c83107f954a21032feadc7b9e3a8e7851"}, + {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:906b0dc25f2be12e95975722f1e60e162437023f490dbd80d0deb7375baf3171"}, + {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:78d4a77a46a7de9388b653af1c4ce539350726cd9af62e0831e4f2bd0c95a2f4"}, + {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:06ca79080ef89d6529bb8e5074d4b4f6086143b2520494fcb7cf8a99079cde93"}, + {file = "pyarrow-14.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:32542164d905002c42dff896efdac79b3bdd7291b1b74aa292fac8450d0e4dcd"}, + {file = "pyarrow-14.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c7331b4ed3401b7ee56f22c980608cf273f0380f77d0f73dd3c185f78f5a6220"}, + {file = "pyarrow-14.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:922e8b49b88da8633d6cac0e1b5a690311b6758d6f5d7c2be71acb0f1e14cd61"}, + {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58c889851ca33f992ea916b48b8540735055201b177cb0dcf0596a495a667b00"}, + {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30d8494870d9916bb53b2a4384948491444741cb9a38253c590e21f836b01222"}, + {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:be28e1a07f20391bb0b15ea03dcac3aade29fc773c5eb4bee2838e9b2cdde0cb"}, + {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:981670b4ce0110d8dcb3246410a4aabf5714db5d8ea63b15686bce1c914b1f83"}, + {file = "pyarrow-14.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:4756a2b373a28f6166c42711240643fb8bd6322467e9aacabd26b488fa41ec23"}, + {file = "pyarrow-14.0.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:cf87e2cec65dd5cf1aa4aba918d523ef56ef95597b545bbaad01e6433851aa10"}, + {file = "pyarrow-14.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:470ae0194fbfdfbf4a6b65b4f9e0f6e1fa0ea5b90c1ee6b65b38aecee53508c8"}, + {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6263cffd0c3721c1e348062997babdf0151301f7353010c9c9a8ed47448f82ab"}, + {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a8089d7e77d1455d529dbd7cff08898bbb2666ee48bc4085203af1d826a33cc"}, + {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fada8396bc739d958d0b81d291cfd201126ed5e7913cb73de6bc606befc30226"}, + {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:2a145dab9ed7849fc1101bf03bcdc69913547f10513fdf70fc3ab6c0a50c7eee"}, + {file = "pyarrow-14.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:05fe7994745b634c5fb16ce5717e39a1ac1fac3e2b0795232841660aa76647cd"}, + {file = "pyarrow-14.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:a8eeef015ae69d104c4c3117a6011e7e3ecd1abec79dc87fd2fac6e442f666ee"}, + {file = "pyarrow-14.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3c76807540989fe8fcd02285dd15e4f2a3da0b09d27781abec3adc265ddbeba1"}, + {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:450e4605e3c20e558485f9161a79280a61c55efe585d51513c014de9ae8d393f"}, + {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:323cbe60210173ffd7db78bfd50b80bdd792c4c9daca8843ef3cd70b186649db"}, + {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0140c7e2b740e08c5a459439d87acd26b747fc408bde0a8806096ee0baaa0c15"}, + {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:e592e482edd9f1ab32f18cd6a716c45b2c0f2403dc2af782f4e9674952e6dd27"}, + {file = "pyarrow-14.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d264ad13605b61959f2ae7c1d25b1a5b8505b112715c961418c8396433f213ad"}, + {file = "pyarrow-14.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:01e44de9749cddc486169cb632f3c99962318e9dacac7778315a110f4bf8a450"}, + {file = "pyarrow-14.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d0351fecf0e26e152542bc164c22ea2a8e8c682726fce160ce4d459ea802d69c"}, + {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33c1f6110c386464fd2e5e4ea3624466055bbe681ff185fd6c9daa98f30a3f9a"}, + {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11e045dfa09855b6d3e7705a37c42e2dc2c71d608fab34d3c23df2e02df9aec3"}, + {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:097828b55321897db0e1dbfc606e3ff8101ae5725673498cbfa7754ee0da80e4"}, + {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:1daab52050a1c48506c029e6fa0944a7b2436334d7e44221c16f6f1b2cc9c510"}, + {file = "pyarrow-14.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3f6d5faf4f1b0d5a7f97be987cf9e9f8cd39902611e818fe134588ee99bf0283"}, + {file = "pyarrow-14.0.1.tar.gz", hash = "sha256:b8b3f4fe8d4ec15e1ef9b599b94683c5216adaed78d5cb4c606180546d1e2ee1"}, ] [package.dependencies] numpy = ">=1.16.6" +[[package]] +name = "pyarrow-hotfix" +version = "0.6" +description = "" +optional = true +python-versions = ">=3.5" +files = [ + {file = "pyarrow_hotfix-0.6-py3-none-any.whl", hash = "sha256:dcc9ae2d220dff0083be6a9aa8e0cdee5182ad358d4931fce825c545e5c89178"}, + {file = "pyarrow_hotfix-0.6.tar.gz", hash = "sha256:79d3e030f7ff890d408a100ac16d6f00b14d44a502d7897cd9fc3e3a534e9945"}, +] + [[package]] name = "pytest" -version = "7.4.2" +version = "7.4.3" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, - {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, ] [package.dependencies] @@ -1867,36 +2023,36 @@ torch = ["numpy (>=1.21.6)", "torch (>=1.10)"] [[package]] name = "scipy" -version = "1.11.3" +version = "1.11.4" description = "Fundamental algorithms for scientific computing in Python" optional = false -python-versions = "<3.13,>=3.9" -files = [ - {file = "scipy-1.11.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:370f569c57e1d888304052c18e58f4a927338eafdaef78613c685ca2ea0d1fa0"}, - {file = "scipy-1.11.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:9885e3e4f13b2bd44aaf2a1a6390a11add9f48d5295f7a592393ceb8991577a3"}, - {file = "scipy-1.11.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e04aa19acc324a1a076abb4035dabe9b64badb19f76ad9c798bde39d41025cdc"}, - {file = "scipy-1.11.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e1a8a4657673bfae1e05e1e1d6e94b0cabe5ed0c7c144c8aa7b7dbb774ce5c1"}, - {file = "scipy-1.11.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7abda0e62ef00cde826d441485e2e32fe737bdddee3324e35c0e01dee65e2a88"}, - {file = "scipy-1.11.3-cp310-cp310-win_amd64.whl", hash = "sha256:033c3fd95d55012dd1148b201b72ae854d5086d25e7c316ec9850de4fe776929"}, - {file = "scipy-1.11.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:925c6f09d0053b1c0f90b2d92d03b261e889b20d1c9b08a3a51f61afc5f58165"}, - {file = "scipy-1.11.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5664e364f90be8219283eeb844323ff8cd79d7acbd64e15eb9c46b9bc7f6a42a"}, - {file = "scipy-1.11.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00f325434b6424952fbb636506f0567898dca7b0f7654d48f1c382ea338ce9a3"}, - {file = "scipy-1.11.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f290cf561a4b4edfe8d1001ee4be6da60c1c4ea712985b58bf6bc62badee221"}, - {file = "scipy-1.11.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:91770cb3b1e81ae19463b3c235bf1e0e330767dca9eb4cd73ba3ded6c4151e4d"}, - {file = "scipy-1.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1f97cd89c0fe1a0685f8f89d85fa305deb3067d0668151571ba50913e445820"}, - {file = "scipy-1.11.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dfcc1552add7cb7c13fb70efcb2389d0624d571aaf2c80b04117e2755a0c5d15"}, - {file = "scipy-1.11.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:0d3a136ae1ff0883fffbb1b05b0b2fea251cb1046a5077d0b435a1839b3e52b7"}, - {file = "scipy-1.11.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bae66a2d7d5768eaa33008fa5a974389f167183c87bf39160d3fefe6664f8ddc"}, - {file = "scipy-1.11.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2f6dee6cbb0e263b8142ed587bc93e3ed5e777f1f75448d24fb923d9fd4dce6"}, - {file = "scipy-1.11.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:74e89dc5e00201e71dd94f5f382ab1c6a9f3ff806c7d24e4e90928bb1aafb280"}, - {file = "scipy-1.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:90271dbde4be191522b3903fc97334e3956d7cfb9cce3f0718d0ab4fd7d8bfd6"}, - {file = "scipy-1.11.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a63d1ec9cadecce838467ce0631c17c15c7197ae61e49429434ba01d618caa83"}, - {file = "scipy-1.11.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:5305792c7110e32ff155aed0df46aa60a60fc6e52cd4ee02cdeb67eaccd5356e"}, - {file = "scipy-1.11.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea7f579182d83d00fed0e5c11a4aa5ffe01460444219dedc448a36adf0c3917"}, - {file = "scipy-1.11.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c77da50c9a91e23beb63c2a711ef9e9ca9a2060442757dffee34ea41847d8156"}, - {file = "scipy-1.11.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:15f237e890c24aef6891c7d008f9ff7e758c6ef39a2b5df264650eb7900403c0"}, - {file = "scipy-1.11.3-cp39-cp39-win_amd64.whl", hash = "sha256:4b4bb134c7aa457e26cc6ea482b016fef45db71417d55cc6d8f43d799cdf9ef2"}, - {file = "scipy-1.11.3.tar.gz", hash = "sha256:bba4d955f54edd61899776bad459bf7326e14b9fa1c552181f0479cc60a568cd"}, +python-versions = ">=3.9" +files = [ + {file = "scipy-1.11.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc9a714581f561af0848e6b69947fda0614915f072dfd14142ed1bfe1b806710"}, + {file = "scipy-1.11.4-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:cf00bd2b1b0211888d4dc75656c0412213a8b25e80d73898083f402b50f47e41"}, + {file = "scipy-1.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9999c008ccf00e8fbcce1236f85ade5c569d13144f77a1946bef8863e8f6eb4"}, + {file = "scipy-1.11.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:933baf588daa8dc9a92c20a0be32f56d43faf3d1a60ab11b3f08c356430f6e56"}, + {file = "scipy-1.11.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8fce70f39076a5aa62e92e69a7f62349f9574d8405c0a5de6ed3ef72de07f446"}, + {file = "scipy-1.11.4-cp310-cp310-win_amd64.whl", hash = "sha256:6550466fbeec7453d7465e74d4f4b19f905642c89a7525571ee91dd7adabb5a3"}, + {file = "scipy-1.11.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be"}, + {file = "scipy-1.11.4-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1b7c3dca977f30a739e0409fb001056484661cb2541a01aba0bb0029f7b68db8"}, + {file = "scipy-1.11.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c"}, + {file = "scipy-1.11.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:530f9ad26440e85766509dbf78edcfe13ffd0ab7fec2560ee5c36ff74d6269ff"}, + {file = "scipy-1.11.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5e347b14fe01003d3b78e196e84bd3f48ffe4c8a7b8a1afbcb8f5505cb710993"}, + {file = "scipy-1.11.4-cp311-cp311-win_amd64.whl", hash = "sha256:acf8ed278cc03f5aff035e69cb511741e0418681d25fbbb86ca65429c4f4d9cd"}, + {file = "scipy-1.11.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:028eccd22e654b3ea01ee63705681ee79933652b2d8f873e7949898dda6d11b6"}, + {file = "scipy-1.11.4-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2c6ff6ef9cc27f9b3db93a6f8b38f97387e6e0591600369a297a50a8e96e835d"}, + {file = "scipy-1.11.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b030c6674b9230d37c5c60ab456e2cf12f6784596d15ce8da9365e70896effc4"}, + {file = "scipy-1.11.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad669df80528aeca5f557712102538f4f37e503f0c5b9541655016dd0932ca79"}, + {file = "scipy-1.11.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ce7fff2e23ab2cc81ff452a9444c215c28e6305f396b2ba88343a567feec9660"}, + {file = "scipy-1.11.4-cp312-cp312-win_amd64.whl", hash = "sha256:36750b7733d960d7994888f0d148d31ea3017ac15eef664194b4ef68d36a4a97"}, + {file = "scipy-1.11.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e619aba2df228a9b34718efb023966da781e89dd3d21637b27f2e54db0410d7"}, + {file = "scipy-1.11.4-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec"}, + {file = "scipy-1.11.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d10e45a6c50211fe256da61a11c34927c68f277e03138777bdebedd933712fea"}, + {file = "scipy-1.11.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91af76a68eeae0064887a48e25c4e616fa519fa0d38602eda7e0f97d65d57937"}, + {file = "scipy-1.11.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6df1468153a31cf55ed5ed39647279beb9cfb5d3f84369453b49e4b8502394fd"}, + {file = "scipy-1.11.4-cp39-cp39-win_amd64.whl", hash = "sha256:ee410e6de8f88fd5cf6eadd73c135020bfbbbdfcd0f6162c36a7638a1ea8cc65"}, + {file = "scipy-1.11.4.tar.gz", hash = "sha256:90a2b78e7f5733b9de748f589f09225013685f9b218275257f8a8168ededaeaa"}, ] [package.dependencies] @@ -1963,17 +2119,17 @@ files = [ [[package]] name = "setuptools" -version = "68.2.2" +version = "69.0.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, - {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, + {file = "setuptools-69.0.2-py3-none-any.whl", hash = "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2"}, + {file = "setuptools-69.0.2.tar.gz", hash = "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] @@ -1992,7 +2148,7 @@ files = [ name = "sympy" version = "1.12" description = "Computer algebra system (CAS) in Python" -optional = false +optional = true python-versions = ">=3.8" files = [ {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, @@ -2080,41 +2236,55 @@ files = [ [[package]] name = "torch" -version = "2.0.1" +version = "2.1.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -optional = false +optional = true python-versions = ">=3.8.0" files = [ - {file = "torch-2.0.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:8ced00b3ba471856b993822508f77c98f48a458623596a4c43136158781e306a"}, - {file = "torch-2.0.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:359bfaad94d1cda02ab775dc1cc386d585712329bb47b8741607ef6ef4950747"}, - {file = "torch-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7c84e44d9002182edd859f3400deaa7410f5ec948a519cc7ef512c2f9b34d2c4"}, - {file = "torch-2.0.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:567f84d657edc5582d716900543e6e62353dbe275e61cdc36eda4929e46df9e7"}, - {file = "torch-2.0.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:787b5a78aa7917465e9b96399b883920c88a08f4eb63b5a5d2d1a16e27d2f89b"}, - {file = "torch-2.0.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:e617b1d0abaf6ced02dbb9486803abfef0d581609b09641b34fa315c9c40766d"}, - {file = "torch-2.0.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:b6019b1de4978e96daa21d6a3ebb41e88a0b474898fe251fd96189587408873e"}, - {file = "torch-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:dbd68cbd1cd9da32fe5d294dd3411509b3d841baecb780b38b3b7b06c7754434"}, - {file = "torch-2.0.1-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:ef654427d91600129864644e35deea761fb1fe131710180b952a6f2e2207075e"}, - {file = "torch-2.0.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:25aa43ca80dcdf32f13da04c503ec7afdf8e77e3a0183dd85cd3e53b2842e527"}, - {file = "torch-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5ef3ea3d25441d3957348f7e99c7824d33798258a2bf5f0f0277cbcadad2e20d"}, - {file = "torch-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0882243755ff28895e8e6dc6bc26ebcf5aa0911ed81b2a12f241fc4b09075b13"}, - {file = "torch-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:f66aa6b9580a22b04d0af54fcd042f52406a8479e2b6a550e3d9f95963e168c8"}, - {file = "torch-2.0.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:1adb60d369f2650cac8e9a95b1d5758e25d526a34808f7448d0bd599e4ae9072"}, - {file = "torch-2.0.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:1bcffc16b89e296826b33b98db5166f990e3b72654a2b90673e817b16c50e32b"}, - {file = "torch-2.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:e10e1597f2175365285db1b24019eb6f04d53dcd626c735fc502f1e8b6be9875"}, - {file = "torch-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:423e0ae257b756bb45a4b49072046772d1ad0c592265c5080070e0767da4e490"}, - {file = "torch-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8742bdc62946c93f75ff92da00e3803216c6cce9b132fbca69664ca38cfb3e18"}, - {file = "torch-2.0.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:c62df99352bd6ee5a5a8d1832452110435d178b5164de450831a3a8cc14dc680"}, - {file = "torch-2.0.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:671a2565e3f63b8fe8e42ae3e36ad249fe5e567435ea27b94edaa672a7d0c416"}, + {file = "torch-2.1.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:5ebc43f5355a9b7be813392b3fb0133991f0380f6f0fcc8218d5468dc45d1071"}, + {file = "torch-2.1.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:84fefd63356416c0cd20578637ccdbb82164993400ed17b57c951dd6376dcee8"}, + {file = "torch-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:0a7a9da0c324409bcb5a7bdad1b4e94e936d21c2590aaa7ac2f63968da8c62f7"}, + {file = "torch-2.1.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:1e1e5faddd43a8f2c0e0e22beacd1e235a2e447794d807483c94a9e31b54a758"}, + {file = "torch-2.1.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:e76bf3c5c354874f1da465c852a2fb60ee6cbce306e935337885760f080f9baa"}, + {file = "torch-2.1.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:98fea993639b0bb432dfceb7b538f07c0f1c33386d63f635219f49254968c80f"}, + {file = "torch-2.1.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:61b51b33c61737c287058b0c3061e6a9d3c363863e4a094f804bc486888a188a"}, + {file = "torch-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:1d70920da827e2276bf07f7ec46958621cad18d228c97da8f9c19638474dbd52"}, + {file = "torch-2.1.1-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:a70593806f1d7e6b53657d96810518da0f88ef2608c98a402955765b8c79d52c"}, + {file = "torch-2.1.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:e312f7e82e49565f7667b0bbf9559ab0c597063d93044740781c02acd5a87978"}, + {file = "torch-2.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:1e3cbecfa5a7314d828f4a37b0c286714dc9aa2e69beb7a22f7aca76567ed9f4"}, + {file = "torch-2.1.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:9ca0fcbf3d5ba644d6a8572c83a9abbdf5f7ff575bc38529ef6c185a3a71bde9"}, + {file = "torch-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:2dc9f312fc1fa0d61a565a0292ad73119d4b74c9f8b5031b55f8b4722abca079"}, + {file = "torch-2.1.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:d56b032176458e2af4709627bbd2c20fe2917eff8cd087a7fe313acccf5ce2f1"}, + {file = "torch-2.1.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:29e3b90a8c281f6660804a939d1f4218604c80162e521e1e6d8c8557325902a0"}, + {file = "torch-2.1.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:bd95cee8511584b67ddc0ba465c3f1edeb5708d833ee02af1206b4486f1d9096"}, + {file = "torch-2.1.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b31230bd058424e56dba7f899280dbc6ac8b9948e43902e0c84a44666b1ec151"}, + {file = "torch-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:403f1095e665e4f35971b43797a920725b8b205723aa68254a4050c6beca29b6"}, + {file = "torch-2.1.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:715b50d8c1de5da5524a68287eb000f73e026e74d5f6b12bc450ef6995fcf5f9"}, + {file = "torch-2.1.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:db67e8725c76f4c7f4f02e7551bb16e81ba1a1912867bc35d7bb96d2be8c78b4"}, ] [package.dependencies] filelock = "*" +fsspec = "*" jinja2 = "*" networkx = "*" +nvidia-cublas-cu12 = {version = "12.1.3.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-cupti-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-nvrtc-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-runtime-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cudnn-cu12 = {version = "8.9.2.26", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufft-cu12 = {version = "11.0.2.54", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-curand-cu12 = {version = "10.3.2.106", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusolver-cu12 = {version = "11.4.5.107", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparse-cu12 = {version = "12.1.0.106", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nccl-cu12 = {version = "2.18.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvtx-cu12 = {version = "12.1.105", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} sympy = "*" +triton = {version = "2.1.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} typing-extensions = "*" [package.extras] +dynamo = ["jinja2"] opt-einsum = ["opt-einsum (>=3.3)"] [[package]] @@ -2206,6 +2376,31 @@ torchhub = ["filelock", "huggingface-hub (>=0.15.1,<1.0)", "importlib-metadata", video = ["av (==9.2.0)", "decord (==0.6.0)"] vision = ["Pillow (<10.0.0)"] +[[package]] +name = "triton" +version = "2.1.0" +description = "A language and compiler for custom Deep Learning operations" +optional = true +python-versions = "*" +files = [ + {file = "triton-2.1.0-0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:66439923a30d5d48399b08a9eae10370f6c261a5ec864a64983bae63152d39d7"}, + {file = "triton-2.1.0-0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:919b06453f0033ea52c13eaf7833de0e57db3178d23d4e04f9fc71c4f2c32bf8"}, + {file = "triton-2.1.0-0-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ae4bb8a91de790e1866405211c4d618379781188f40d5c4c399766914e84cd94"}, + {file = "triton-2.1.0-0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39f6fb6bdccb3e98f3152e3fbea724f1aeae7d749412bbb1fa9c441d474eba26"}, + {file = "triton-2.1.0-0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:21544e522c02005a626c8ad63d39bdff2f31d41069592919ef281e964ed26446"}, + {file = "triton-2.1.0-0-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:143582ca31dd89cd982bd3bf53666bab1c7527d41e185f9e3d8a3051ce1b663b"}, + {file = "triton-2.1.0-0-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:82fc5aeeedf6e36be4e4530cbdcba81a09d65c18e02f52dc298696d45721f3bd"}, + {file = "triton-2.1.0-0-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:81a96d110a738ff63339fc892ded095b31bd0d205e3aace262af8400d40b6fa8"}, +] + +[package.dependencies] +filelock = "*" + +[package.extras] +build = ["cmake (>=3.18)", "lit"] +tests = ["autopep8", "flake8", "isort", "numpy", "pytest", "scipy (>=1.7.1)"] +tutorials = ["matplotlib", "pandas", "tabulate"] + [[package]] name = "typer" version = "0.6.1" @@ -2250,18 +2445,17 @@ files = [ [[package]] name = "urllib3" -version = "2.0.6" +version = "2.1.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "urllib3-2.0.6-py3-none-any.whl", hash = "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2"}, - {file = "urllib3-2.0.6.tar.gz", hash = "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564"}, + {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, + {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, ] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -2281,263 +2475,297 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [[package]] name = "wrapt" -version = "1.15.0" +version = "1.16.0" description = "Module for decorators, wrappers and monkey patching." optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, ] [[package]] name = "xxhash" -version = "3.3.0" +version = "3.4.1" description = "Python binding for xxHash" optional = true python-versions = ">=3.7" files = [ - {file = "xxhash-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:70ef7288d1cb1ad16e02d101ea43bb0e392d985d60b9b0035aee80663530960d"}, - {file = "xxhash-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:44ff8c673cab50be46784e0aec62aa6f0ca9ea765e2b0690e8945d0cd950dcaf"}, - {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfebc90273ae2beb813d8118a2bfffb5a5a81ac054fbfd061ea18fd0a81db0ac"}, - {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9084e68bedbd665c7e9241a7b597c28f4775edeb3941bf608ecb38732a5f8fb5"}, - {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d72493a14a3e89564b1a6c7400b9b40621e8f4692410706ef27c66aeadc7b431"}, - {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98779cbe9068dd7734cc3210693894d5cc9b156920e9c336f10fb99f46bebbd8"}, - {file = "xxhash-3.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:499f8a12767dd28b98ab6b7c7da7d294564e4c9024a2aaa5d0b0b98a8bef2f92"}, - {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4dabda7f42c548f98d8e07e390bda2953fc58302c0e07ded7b3fe0637e7ecd2f"}, - {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c416409646c793c46370f0f1859253302ee70aeda5278c2a0ca41462f8ec1244"}, - {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b8bd31aaad8a80a7302730676cec26bea3ef1fd9835875aa47fea073aca9fe05"}, - {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3af8e3bcd630f905efbdfe7a51b51fc1ca3c9dca8b155f841925f3ad41685d41"}, - {file = "xxhash-3.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d86b79c707fc7025d967af71db652429a06a8179175e45bd2e9f17b8af6f5949"}, - {file = "xxhash-3.3.0-cp310-cp310-win32.whl", hash = "sha256:98fe771f36ee9d3a1f5741424a956a2ba9651d9508a9f64a024b57f2cf796414"}, - {file = "xxhash-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:0a65131f7f731ecf7e3dd27f09d877aff3000a79a446caaa2c0d8d0ec0bc7186"}, - {file = "xxhash-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a9761e425e79d23797fa0bec2d781dbadb9fe5dcc2bf69030855f5e393c3bec8"}, - {file = "xxhash-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d28c7ef1deb3c3ac5f5290176ca3d501daa97c2e1f7443bf5d8b61ac651794b2"}, - {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:701b7cefffc25de1b7ddfae6505da70a3b3a11e312c2e2b33b09e180bbceb43d"}, - {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b1644f8b8e19a242c3047a089541067248a651038cabb9fcab3c13eb1dfcd757"}, - {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:20e7d0e3488cc0f0dbe360731b7fe32e1f2df46bf2de2db3317d301efb93084c"}, - {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:156c52eca2b20f9839723bef0b929a290f6c2f1c98ccb24e82f58f96f3c16007"}, - {file = "xxhash-3.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d6ce4d3828d79044ed08994e196c20f69c18133ed8a4286afe3e98989adeeac"}, - {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b85b63757ade2439c8d7d71842c40d42c0ab3b69279ed02afbd3b1635f7d2b4b"}, - {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b2b9051e40b7b649a9a2a38fb223ca6a593d332012df885746b81968948f9435"}, - {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:81b7ce050f26fc1daaaa0d24e320815306736d14608e1ba31920e693a7ca9afb"}, - {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:7442500fcce71669953ca959682dcd47452bc3f9c95c8d88315874aeabec9f82"}, - {file = "xxhash-3.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:36a05bf59a515cfb07f3f83373c527fff2ecaa77eaf30c968c788aea582070a1"}, - {file = "xxhash-3.3.0-cp311-cp311-win32.whl", hash = "sha256:da16f9cd62c6fde74683be1b28c28ef865e706da13e3bee4ba836fcc520de0cc"}, - {file = "xxhash-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:40fd49ef6964b1c90c0bea63cd184f6d0b36e59144a080e8b3ac2c4c06bf6bf2"}, - {file = "xxhash-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:672c60cce1f8026ae32c651f877aa64f342876083a36a4b1ff91bc876aaf0e34"}, - {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bb6c83d7a65dd3065566c77425ba72df96982174e8ef613d809052d68ae77ab"}, - {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a4170f3016b621e3200ebfcc18de6f50eb8e8fc1303e16324b1f5625afd51b57"}, - {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bfb9c45d502ab38c0f4edf98a678694ae0f345613ef4900ade98c71f64db4d78"}, - {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48af026a2b1569666da42a478248a1f03f4e2350a34eb661afe3cb45429ca1d7"}, - {file = "xxhash-3.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe627de8fe8ddfa8b6477bda4ae5d5843ad1a0c83601dcff72247039465cc901"}, - {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:427fc60a188e345534f35b0aa76f7640c5ddf0354f1c9ad826a2bc086282982d"}, - {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d80acb20c7f268fe3150ac0be6a6b798062af56a1795eef855b26c9eae11a99c"}, - {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e71100818943422d1fbbe460e7be7fc4f2d2ba9371b2a745eb09e29ef0493f4a"}, - {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:e3b9bb5fdbe284c7b61c5d82c76688e52bbaf48ab1e53de98c072cc696fa331f"}, - {file = "xxhash-3.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1e25f6c8c46cf1ed8237f610abb231093a748c97d6c2c092789a7cad7e7ef290"}, - {file = "xxhash-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:928208dfecc563be59ae91868d1658e78809cb1e6a0bd74960a96c915db6390c"}, - {file = "xxhash-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bd1b4531a66da6dde1974662c1fd6fb1a2f27e40542e3df5e5e5dbab8ea4aee7"}, - {file = "xxhash-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:deebb296df92e082b6d0171a7d6227b503e2897cea4f8bdd3d708094974d4cf6"}, - {file = "xxhash-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cd96e9cb0e2baa294e6d572207d9731c3bb8e2511f1ff70f2bf17266b4488bd9"}, - {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3756b44bf247e422a2e47a38f25d03cf4a5ed539fdc2be3c60043e872e6ff13d"}, - {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69550c3c053b8f135ceac97b85dc1b2bc54b7613a966f550f32b43bed81c788a"}, - {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fc8736fc3e0c5aad435520873b9d2e27ddcc5a830b07e00e9c4d3a61ded9675"}, - {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80ead7774392efbd95f9f701155048f9ca26cf55133db6f5bb5a0ec69376bda5"}, - {file = "xxhash-3.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8737c9b3fd944d856faafa92c95f6198649ad57987935b6d965d086938be917"}, - {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2c8e078d0b9f85212801c41bd9eec8122003929686b0ee33360ffbfdf1a189ab"}, - {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f399269d20ef1dd910331f9ad49e8510c3ba2aa657b623293b536038f266a5c5"}, - {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f3661decef5f9ff7ab50edbef463bf7dc717621b56755dbae5458a946a033b10"}, - {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5ec374d0f1e7d43ef48a4ff643600833d7a325ecc6933b4d6ad9282f55751cf7"}, - {file = "xxhash-3.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39a947ff02d9a85673f5ce1f6f34059e24c714a797440485bd81b2c3cb69a7ff"}, - {file = "xxhash-3.3.0-cp38-cp38-win32.whl", hash = "sha256:4a4f0645a0ec03b229fb04f2e66bdbcb1ffd341a70d6c86c3ee015ffdcd70fad"}, - {file = "xxhash-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:8af5a687c0fb4357c230eec8a57ca07d3172faa3cb69beb0cbad40672ae6fa4b"}, - {file = "xxhash-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e5bfafda019ecc6202af6f3cf08220fa66af9612ba16ef831033ae3ac7bd1f89"}, - {file = "xxhash-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d113b433bc817adf845689a051363777835577858263ec4325d1934fcb7e394"}, - {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56aacf4bf65f575c0392be958aceff719d850950bb6af7d804b32d4bc293159c"}, - {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0f5d3e4e0937dad05585e9bd772bbdf0ca40cd8b2f54789d7a1f3091b608118c"}, - {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23605d7fc67bc7daa0d263b3a26de3375cfcc0b51ab7de5026625415c05b6fed"}, - {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe525be0392d493558a2b10d764bcaae9850cc262b417176a8b001f16e085fc6"}, - {file = "xxhash-3.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b234d08786884f5c8d55dfebb839cfbd846d812e3a052c39ca7e8ce7055fed68"}, - {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b031395b4b9c3085d9ea1ce89896ab01a65fc63172b2bfda5dd318fefe5e2f93"}, - {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5afe44da46b48c75169e622a532dca3fe585343c0577cfd7c18ecd3f1200305d"}, - {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c59f233f38b6a49d5e4ddf16be910a5bbf36a2989b6b2c8591853fb9f5a5e691"}, - {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:ed016e278c5c4633270903c7cf3b9dfb0bd293b7335e43fe695cb95541da53c9"}, - {file = "xxhash-3.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7a8bd6612fb35487e9ab329bb37b3df44f58baf752010dde9282593edbfed7e7"}, - {file = "xxhash-3.3.0-cp39-cp39-win32.whl", hash = "sha256:015a0498bde85364abc53fcc713af962dd4555391929736d9c0ff2c555436a03"}, - {file = "xxhash-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:06a484097af32caf1cfffadd60c3ca140c9e52b40a551fb1f6f0fdfd6f7f8977"}, - {file = "xxhash-3.3.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6c3809740124bbc777d29e3ae53de24f4c13fd5e62878086a8feadf0dcb654a5"}, - {file = "xxhash-3.3.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae092f0daaeece2acdd6ec46e2ab307d8d6f22b01ecca14dc6078844dbd88339"}, - {file = "xxhash-3.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3498e72ff2610b049b97bb81d1ea6e7bfa5b7a45efb3f255d77ec2fa2bc91653"}, - {file = "xxhash-3.3.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0004dded9d86f129961326e980420187640fb7ba65a184009429861c1d09df7"}, - {file = "xxhash-3.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:41c8bfd27191928bae6fd2b66872965532267785094a03c0ee5f358d9dba51c2"}, - {file = "xxhash-3.3.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:71db8498e329cef3588b0617f762a3fe31d899872e76a68ce2840e35a1318a5b"}, - {file = "xxhash-3.3.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d1d24d71b6209bc0124286932c4f0660c1103cb996fe34cb374bc12ac251940"}, - {file = "xxhash-3.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61004587a09b5b385e43d95ffe3a76c9d934dfd79ea38272d5c20ddfba8eab8f"}, - {file = "xxhash-3.3.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f0c92e3fa826425c73acafb31e022a719c85423847a9433d3a9e61e4ac97543"}, - {file = "xxhash-3.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:367e03f1484ce471c94e731b98f5e4a05b43e7188b16692998e1cc89fd1159a5"}, - {file = "xxhash-3.3.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed04c47dfaab98fcda0b748af9ee6fe8c888a0a0fbd13720e0f0221671e387e1"}, - {file = "xxhash-3.3.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cbfde62516435ca198220aff048a8793383cb7047c7b88714a061968bca786d"}, - {file = "xxhash-3.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73682225faa973ee56743f0fcd36bfcbfec503be258e0e420fb34313f52f1e7b"}, - {file = "xxhash-3.3.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d49efdce2086c2c506af20ed18a1115b40af7aad6d4ee27cb31d7c810585a3f2"}, - {file = "xxhash-3.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:546a0bb8e5a657cadf0da290b30ccd561cb89c256a5421ab8d5eb12eaf087349"}, - {file = "xxhash-3.3.0.tar.gz", hash = "sha256:c3f9e322b1ebeebd44e3d9d2d9b124e0c550c1ef41bd552afdcdd719516ee41a"}, + {file = "xxhash-3.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:91dbfa55346ad3e18e738742236554531a621042e419b70ad8f3c1d9c7a16e7f"}, + {file = "xxhash-3.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:665a65c2a48a72068fcc4d21721510df5f51f1142541c890491afc80451636d2"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb11628470a6004dc71a09fe90c2f459ff03d611376c1debeec2d648f44cb693"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5bef2a7dc7b4f4beb45a1edbba9b9194c60a43a89598a87f1a0226d183764189"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c0f7b2d547d72c7eda7aa817acf8791f0146b12b9eba1d4432c531fb0352228"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00f2fdef6b41c9db3d2fc0e7f94cb3db86693e5c45d6de09625caad9a469635b"}, + {file = "xxhash-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23cfd9ca09acaf07a43e5a695143d9a21bf00f5b49b15c07d5388cadf1f9ce11"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6a9ff50a3cf88355ca4731682c168049af1ca222d1d2925ef7119c1a78e95b3b"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f1d7c69a1e9ca5faa75546fdd267f214f63f52f12692f9b3a2f6467c9e67d5e7"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:672b273040d5d5a6864a36287f3514efcd1d4b1b6a7480f294c4b1d1ee1b8de0"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4178f78d70e88f1c4a89ff1ffe9f43147185930bb962ee3979dba15f2b1cc799"}, + {file = "xxhash-3.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9804b9eb254d4b8cc83ab5a2002128f7d631dd427aa873c8727dba7f1f0d1c2b"}, + {file = "xxhash-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c09c49473212d9c87261d22c74370457cfff5db2ddfc7fd1e35c80c31a8c14ce"}, + {file = "xxhash-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ebbb1616435b4a194ce3466d7247df23499475c7ed4eb2681a1fa42ff766aff6"}, + {file = "xxhash-3.4.1-cp310-cp310-win_arm64.whl", hash = "sha256:25dc66be3db54f8a2d136f695b00cfe88018e59ccff0f3b8f545869f376a8a46"}, + {file = "xxhash-3.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58c49083801885273e262c0f5bbeac23e520564b8357fbb18fb94ff09d3d3ea5"}, + {file = "xxhash-3.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b526015a973bfbe81e804a586b703f163861da36d186627e27524f5427b0d520"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36ad4457644c91a966f6fe137d7467636bdc51a6ce10a1d04f365c70d6a16d7e"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:248d3e83d119770f96003271fe41e049dd4ae52da2feb8f832b7a20e791d2920"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2070b6d5bbef5ee031666cf21d4953c16e92c2f8a24a94b5c240f8995ba3b1d0"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2746035f518f0410915e247877f7df43ef3372bf36cfa52cc4bc33e85242641"}, + {file = "xxhash-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a8ba6181514681c2591840d5632fcf7356ab287d4aff1c8dea20f3c78097088"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0aac5010869240e95f740de43cd6a05eae180c59edd182ad93bf12ee289484fa"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4cb11d8debab1626181633d184b2372aaa09825bde709bf927704ed72765bed1"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b29728cff2c12f3d9f1d940528ee83918d803c0567866e062683f300d1d2eff3"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a15cbf3a9c40672523bdb6ea97ff74b443406ba0ab9bca10ceccd9546414bd84"}, + {file = "xxhash-3.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e66df260fed01ed8ea790c2913271641c58481e807790d9fca8bfd5a3c13844"}, + {file = "xxhash-3.4.1-cp311-cp311-win32.whl", hash = "sha256:e867f68a8f381ea12858e6d67378c05359d3a53a888913b5f7d35fbf68939d5f"}, + {file = "xxhash-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:200a5a3ad9c7c0c02ed1484a1d838b63edcf92ff538770ea07456a3732c577f4"}, + {file = "xxhash-3.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:1d03f1c0d16d24ea032e99f61c552cb2b77d502e545187338bea461fde253583"}, + {file = "xxhash-3.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c4bbba9b182697a52bc0c9f8ec0ba1acb914b4937cd4a877ad78a3b3eeabefb3"}, + {file = "xxhash-3.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9fd28a9da300e64e434cfc96567a8387d9a96e824a9be1452a1e7248b7763b78"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6066d88c9329ab230e18998daec53d819daeee99d003955c8db6fc4971b45ca3"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93805bc3233ad89abf51772f2ed3355097a5dc74e6080de19706fc447da99cd3"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64da57d5ed586ebb2ecdde1e997fa37c27fe32fe61a656b77fabbc58e6fbff6e"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a97322e9a7440bf3c9805cbaac090358b43f650516486746f7fa482672593df"}, + {file = "xxhash-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbe750d512982ee7d831838a5dee9e9848f3fb440e4734cca3f298228cc957a6"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:fd79d4087727daf4d5b8afe594b37d611ab95dc8e29fe1a7517320794837eb7d"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:743612da4071ff9aa4d055f3f111ae5247342931dedb955268954ef7201a71ff"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b41edaf05734092f24f48c0958b3c6cbaaa5b7e024880692078c6b1f8247e2fc"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:a90356ead70d715fe64c30cd0969072de1860e56b78adf7c69d954b43e29d9fa"}, + {file = "xxhash-3.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac56eebb364e44c85e1d9e9cc5f6031d78a34f0092fea7fc80478139369a8b4a"}, + {file = "xxhash-3.4.1-cp312-cp312-win32.whl", hash = "sha256:911035345932a153c427107397c1518f8ce456f93c618dd1c5b54ebb22e73747"}, + {file = "xxhash-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:f31ce76489f8601cc7b8713201ce94b4bd7b7ce90ba3353dccce7e9e1fee71fa"}, + {file = "xxhash-3.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:b5beb1c6a72fdc7584102f42c4d9df232ee018ddf806e8c90906547dfb43b2da"}, + {file = "xxhash-3.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6d42b24d1496deb05dee5a24ed510b16de1d6c866c626c2beb11aebf3be278b9"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b685fab18876b14a8f94813fa2ca80cfb5ab6a85d31d5539b7cd749ce9e3624"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:419ffe34c17ae2df019a4685e8d3934d46b2e0bbe46221ab40b7e04ed9f11137"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e041ce5714f95251a88670c114b748bca3bf80cc72400e9f23e6d0d59cf2681"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc860d887c5cb2f524899fb8338e1bb3d5789f75fac179101920d9afddef284b"}, + {file = "xxhash-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:312eba88ffe0a05e332e3a6f9788b73883752be63f8588a6dc1261a3eaaaf2b2"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:e01226b6b6a1ffe4e6bd6d08cfcb3ca708b16f02eb06dd44f3c6e53285f03e4f"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9f3025a0d5d8cf406a9313cd0d5789c77433ba2004b1c75439b67678e5136537"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:6d3472fd4afef2a567d5f14411d94060099901cd8ce9788b22b8c6f13c606a93"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:43984c0a92f06cac434ad181f329a1445017c33807b7ae4f033878d860a4b0f2"}, + {file = "xxhash-3.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a55e0506fdb09640a82ec4f44171273eeabf6f371a4ec605633adb2837b5d9d5"}, + {file = "xxhash-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:faec30437919555b039a8bdbaba49c013043e8f76c999670aef146d33e05b3a0"}, + {file = "xxhash-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:c9e1b646af61f1fc7083bb7b40536be944f1ac67ef5e360bca2d73430186971a"}, + {file = "xxhash-3.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:961d948b7b1c1b6c08484bbce3d489cdf153e4122c3dfb07c2039621243d8795"}, + {file = "xxhash-3.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:719a378930504ab159f7b8e20fa2aa1896cde050011af838af7e7e3518dd82de"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74fb5cb9406ccd7c4dd917f16630d2e5e8cbbb02fc2fca4e559b2a47a64f4940"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5dab508ac39e0ab988039bc7f962c6ad021acd81fd29145962b068df4148c476"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c59f3e46e7daf4c589e8e853d700ef6607afa037bfad32c390175da28127e8c"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cc07256eff0795e0f642df74ad096f8c5d23fe66bc138b83970b50fc7f7f6c5"}, + {file = "xxhash-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9f749999ed80f3955a4af0eb18bb43993f04939350b07b8dd2f44edc98ffee9"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7688d7c02149a90a3d46d55b341ab7ad1b4a3f767be2357e211b4e893efbaaf6"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a8b4977963926f60b0d4f830941c864bed16aa151206c01ad5c531636da5708e"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:8106d88da330f6535a58a8195aa463ef5281a9aa23b04af1848ff715c4398fb4"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4c76a77dbd169450b61c06fd2d5d436189fc8ab7c1571d39265d4822da16df22"}, + {file = "xxhash-3.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:11f11357c86d83e53719c592021fd524efa9cf024dc7cb1dfb57bbbd0d8713f2"}, + {file = "xxhash-3.4.1-cp38-cp38-win32.whl", hash = "sha256:0c786a6cd74e8765c6809892a0d45886e7c3dc54de4985b4a5eb8b630f3b8e3b"}, + {file = "xxhash-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:aabf37fb8fa27430d50507deeab2ee7b1bcce89910dd10657c38e71fee835594"}, + {file = "xxhash-3.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6127813abc1477f3a83529b6bbcfeddc23162cece76fa69aee8f6a8a97720562"}, + {file = "xxhash-3.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef2e194262f5db16075caea7b3f7f49392242c688412f386d3c7b07c7733a70a"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71be94265b6c6590f0018bbf73759d21a41c6bda20409782d8117e76cd0dfa8b"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10e0a619cdd1c0980e25eb04e30fe96cf8f4324758fa497080af9c21a6de573f"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa122124d2e3bd36581dd78c0efa5f429f5220313479fb1072858188bc2d5ff1"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17032f5a4fea0a074717fe33477cb5ee723a5f428de7563e75af64bfc1b1e10"}, + {file = "xxhash-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca7783b20e3e4f3f52f093538895863f21d18598f9a48211ad757680c3bd006f"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d77d09a1113899fad5f354a1eb4f0a9afcf58cefff51082c8ad643ff890e30cf"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:21287bcdd299fdc3328cc0fbbdeaa46838a1c05391264e51ddb38a3f5b09611f"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:dfd7a6cc483e20b4ad90224aeb589e64ec0f31e5610ab9957ff4314270b2bf31"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:543c7fcbc02bbb4840ea9915134e14dc3dc15cbd5a30873a7a5bf66039db97ec"}, + {file = "xxhash-3.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fe0a98d990e433013f41827b62be9ab43e3cf18e08b1483fcc343bda0d691182"}, + {file = "xxhash-3.4.1-cp39-cp39-win32.whl", hash = "sha256:b9097af00ebf429cc7c0e7d2fdf28384e4e2e91008130ccda8d5ae653db71e54"}, + {file = "xxhash-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:d699b921af0dcde50ab18be76c0d832f803034d80470703700cb7df0fbec2832"}, + {file = "xxhash-3.4.1-cp39-cp39-win_arm64.whl", hash = "sha256:2be491723405e15cc099ade1280133ccfbf6322d2ef568494fb7d07d280e7eee"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:431625fad7ab5649368c4849d2b49a83dc711b1f20e1f7f04955aab86cd307bc"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc6dbd5fc3c9886a9e041848508b7fb65fd82f94cc793253990f81617b61fe49"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ff8dbd0ec97aec842476cb8ccc3e17dd288cd6ce3c8ef38bff83d6eb927817"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef73a53fe90558a4096e3256752268a8bdc0322f4692ed928b6cd7ce06ad4fe3"}, + {file = "xxhash-3.4.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:450401f42bbd274b519d3d8dcf3c57166913381a3d2664d6609004685039f9d3"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a162840cf4de8a7cd8720ff3b4417fbc10001eefdd2d21541a8226bb5556e3bb"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b736a2a2728ba45017cb67785e03125a79d246462dfa892d023b827007412c52"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d0ae4c2e7698adef58710d6e7a32ff518b66b98854b1c68e70eee504ad061d8"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6322c4291c3ff174dcd104fae41500e75dad12be6f3085d119c2c8a80956c51"}, + {file = "xxhash-3.4.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:dd59ed668801c3fae282f8f4edadf6dc7784db6d18139b584b6d9677ddde1b6b"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92693c487e39523a80474b0394645b393f0ae781d8db3474ccdcead0559ccf45"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4603a0f642a1e8d7f3ba5c4c25509aca6a9c1cc16f85091004a7028607ead663"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa45e8cbfbadb40a920fe9ca40c34b393e0b067082d94006f7f64e70c7490a6"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:595b252943b3552de491ff51e5bb79660f84f033977f88f6ca1605846637b7c6"}, + {file = "xxhash-3.4.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:562d8b8f783c6af969806aaacf95b6c7b776929ae26c0cd941d54644ea7ef51e"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:41ddeae47cf2828335d8d991f2d2b03b0bdc89289dc64349d712ff8ce59d0647"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c44d584afdf3c4dbb3277e32321d1a7b01d6071c1992524b6543025fb8f4206f"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd7bddb3a5b86213cc3f2c61500c16945a1b80ecd572f3078ddbbe68f9dabdfb"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ecb6c987b62437c2f99c01e97caf8d25660bf541fe79a481d05732e5236719c"}, + {file = "xxhash-3.4.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:696b4e18b7023527d5c50ed0626ac0520edac45a50ec7cf3fc265cd08b1f4c03"}, + {file = "xxhash-3.4.1.tar.gz", hash = "sha256:0379d6cf1ff987cd421609a264ce025e74f346e3e145dd106c0cc2e3ec3f99a9"}, ] [[package]] name = "yarl" -version = "1.9.2" +version = "1.9.3" description = "Yet another URL library" optional = true python-versions = ">=3.7" files = [ - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, - {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, - {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, - {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, - {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, - {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, - {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, - {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, - {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, - {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, - {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, - {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, - {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, + {file = "yarl-1.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:32435d134414e01d937cd9d6cc56e8413a8d4741dea36af5840c7750f04d16ab"}, + {file = "yarl-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9a5211de242754b5e612557bca701f39f8b1a9408dff73c6db623f22d20f470e"}, + {file = "yarl-1.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:525cd69eff44833b01f8ef39aa33a9cc53a99ff7f9d76a6ef6a9fb758f54d0ff"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc94441bcf9cb8c59f51f23193316afefbf3ff858460cb47b5758bf66a14d130"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e36021db54b8a0475805acc1d6c4bca5d9f52c3825ad29ae2d398a9d530ddb88"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0f17d1df951336a02afc8270c03c0c6e60d1f9996fcbd43a4ce6be81de0bd9d"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5f3faeb8100a43adf3e7925d556801d14b5816a0ac9e75e22948e787feec642"}, + {file = "yarl-1.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aed37db837ecb5962469fad448aaae0f0ee94ffce2062cf2eb9aed13328b5196"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:721ee3fc292f0d069a04016ef2c3a25595d48c5b8ddc6029be46f6158d129c92"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b8bc5b87a65a4e64bc83385c05145ea901b613d0d3a434d434b55511b6ab0067"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:dd952b9c64f3b21aedd09b8fe958e4931864dba69926d8a90c90d36ac4e28c9a"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:c405d482c320a88ab53dcbd98d6d6f32ada074f2d965d6e9bf2d823158fa97de"}, + {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9df9a0d4c5624790a0dea2e02e3b1b3c69aed14bcb8650e19606d9df3719e87d"}, + {file = "yarl-1.9.3-cp310-cp310-win32.whl", hash = "sha256:d34c4f80956227f2686ddea5b3585e109c2733e2d4ef12eb1b8b4e84f09a2ab6"}, + {file = "yarl-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:cf7a4e8de7f1092829caef66fd90eaf3710bc5efd322a816d5677b7664893c93"}, + {file = "yarl-1.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d61a0ca95503867d4d627517bcfdc28a8468c3f1b0b06c626f30dd759d3999fd"}, + {file = "yarl-1.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:73cc83f918b69110813a7d95024266072d987b903a623ecae673d1e71579d566"}, + {file = "yarl-1.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d81657b23e0edb84b37167e98aefb04ae16cbc5352770057893bd222cdc6e45f"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a1a8443091c7fbc17b84a0d9f38de34b8423b459fb853e6c8cdfab0eacf613"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe34befb8c765b8ce562f0200afda3578f8abb159c76de3ab354c80b72244c41"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c757f64afe53a422e45e3e399e1e3cf82b7a2f244796ce80d8ca53e16a49b9f"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72a57b41a0920b9a220125081c1e191b88a4cdec13bf9d0649e382a822705c65"}, + {file = "yarl-1.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:632c7aeb99df718765adf58eacb9acb9cbc555e075da849c1378ef4d18bf536a"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b0b8c06afcf2bac5a50b37f64efbde978b7f9dc88842ce9729c020dc71fae4ce"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1d93461e2cf76c4796355494f15ffcb50a3c198cc2d601ad8d6a96219a10c363"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:4003f380dac50328c85e85416aca6985536812c082387255c35292cb4b41707e"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4d6d74a97e898c1c2df80339aa423234ad9ea2052f66366cef1e80448798c13d"}, + {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b61e64b06c3640feab73fa4ff9cb64bd8182de52e5dc13038e01cfe674ebc321"}, + {file = "yarl-1.9.3-cp311-cp311-win32.whl", hash = "sha256:29beac86f33d6c7ab1d79bd0213aa7aed2d2f555386856bb3056d5fdd9dab279"}, + {file = "yarl-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:f7271d6bd8838c49ba8ae647fc06469137e1c161a7ef97d778b72904d9b68696"}, + {file = "yarl-1.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:dd318e6b75ca80bff0b22b302f83a8ee41c62b8ac662ddb49f67ec97e799885d"}, + {file = "yarl-1.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c4b1efb11a8acd13246ffb0bee888dd0e8eb057f8bf30112e3e21e421eb82d4a"}, + {file = "yarl-1.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c6f034386e5550b5dc8ded90b5e2ff7db21f0f5c7de37b6efc5dac046eb19c10"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd49a908cb6d387fc26acee8b7d9fcc9bbf8e1aca890c0b2fdfd706057546080"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa4643635f26052401750bd54db911b6342eb1a9ac3e74f0f8b58a25d61dfe41"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e741bd48e6a417bdfbae02e088f60018286d6c141639359fb8df017a3b69415a"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c86d0d0919952d05df880a1889a4f0aeb6868e98961c090e335671dea5c0361"}, + {file = "yarl-1.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d5434b34100b504aabae75f0622ebb85defffe7b64ad8f52b8b30ec6ef6e4b9"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79e1df60f7c2b148722fb6cafebffe1acd95fd8b5fd77795f56247edaf326752"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:44e91a669c43f03964f672c5a234ae0d7a4d49c9b85d1baa93dec28afa28ffbd"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:3cfa4dbe17b2e6fca1414e9c3bcc216f6930cb18ea7646e7d0d52792ac196808"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:88d2c3cc4b2f46d1ba73d81c51ec0e486f59cc51165ea4f789677f91a303a9a7"}, + {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cccdc02e46d2bd7cb5f38f8cc3d9db0d24951abd082b2f242c9e9f59c0ab2af3"}, + {file = "yarl-1.9.3-cp312-cp312-win32.whl", hash = "sha256:96758e56dceb8a70f8a5cff1e452daaeff07d1cc9f11e9b0c951330f0a2396a7"}, + {file = "yarl-1.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:c4472fe53ebf541113e533971bd8c32728debc4c6d8cc177f2bff31d011ec17e"}, + {file = "yarl-1.9.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:126638ab961633f0940a06e1c9d59919003ef212a15869708dcb7305f91a6732"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c99ddaddb2fbe04953b84d1651149a0d85214780e4d0ee824e610ab549d98d92"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dab30b21bd6fb17c3f4684868c7e6a9e8468078db00f599fb1c14e324b10fca"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:828235a2a169160ee73a2fcfb8a000709edf09d7511fccf203465c3d5acc59e4"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc391e3941045fd0987c77484b2799adffd08e4b6735c4ee5f054366a2e1551d"}, + {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51382c72dd5377861b573bd55dcf680df54cea84147c8648b15ac507fbef984d"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:28a108cb92ce6cf867690a962372996ca332d8cda0210c5ad487fe996e76b8bb"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8f18a7832ff85dfcd77871fe677b169b1bc60c021978c90c3bb14f727596e0ae"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:7eaf13af79950142ab2bbb8362f8d8d935be9aaf8df1df89c86c3231e4ff238a"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:66a6dbf6ca7d2db03cc61cafe1ee6be838ce0fbc97781881a22a58a7c5efef42"}, + {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1a0a4f3aaa18580038cfa52a7183c8ffbbe7d727fe581300817efc1e96d1b0e9"}, + {file = "yarl-1.9.3-cp37-cp37m-win32.whl", hash = "sha256:946db4511b2d815979d733ac6a961f47e20a29c297be0d55b6d4b77ee4b298f6"}, + {file = "yarl-1.9.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2dad8166d41ebd1f76ce107cf6a31e39801aee3844a54a90af23278b072f1ccf"}, + {file = "yarl-1.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:bb72d2a94481e7dc7a0c522673db288f31849800d6ce2435317376a345728225"}, + {file = "yarl-1.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9a172c3d5447b7da1680a1a2d6ecdf6f87a319d21d52729f45ec938a7006d5d8"}, + {file = "yarl-1.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2dc72e891672343b99db6d497024bf8b985537ad6c393359dc5227ef653b2f17"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8d51817cf4b8d545963ec65ff06c1b92e5765aa98831678d0e2240b6e9fd281"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53ec65f7eee8655bebb1f6f1607760d123c3c115a324b443df4f916383482a67"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cfd77e8e5cafba3fb584e0f4b935a59216f352b73d4987be3af51f43a862c403"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e73db54c967eb75037c178a54445c5a4e7461b5203b27c45ef656a81787c0c1b"}, + {file = "yarl-1.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09c19e5f4404574fcfb736efecf75844ffe8610606f3fccc35a1515b8b6712c4"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6280353940f7e5e2efaaabd686193e61351e966cc02f401761c4d87f48c89ea4"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c25ec06e4241e162f5d1f57c370f4078797ade95c9208bd0c60f484834f09c96"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:7217234b10c64b52cc39a8d82550342ae2e45be34f5bff02b890b8c452eb48d7"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4ce77d289f8d40905c054b63f29851ecbfd026ef4ba5c371a158cfe6f623663e"}, + {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5f74b015c99a5eac5ae589de27a1201418a5d9d460e89ccb3366015c6153e60a"}, + {file = "yarl-1.9.3-cp38-cp38-win32.whl", hash = "sha256:8a2538806be846ea25e90c28786136932ec385c7ff3bc1148e45125984783dc6"}, + {file = "yarl-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:6465d36381af057d0fab4e0f24ef0e80ba61f03fe43e6eeccbe0056e74aadc70"}, + {file = "yarl-1.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2f3c8822bc8fb4a347a192dd6a28a25d7f0ea3262e826d7d4ef9cc99cd06d07e"}, + {file = "yarl-1.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7831566595fe88ba17ea80e4b61c0eb599f84c85acaa14bf04dd90319a45b90"}, + {file = "yarl-1.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ff34cb09a332832d1cf38acd0f604c068665192c6107a439a92abfd8acf90fe2"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe8080b4f25dfc44a86bedd14bc4f9d469dfc6456e6f3c5d9077e81a5fedfba7"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8535e111a064f3bdd94c0ed443105934d6f005adad68dd13ce50a488a0ad1bf3"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0d155a092bf0ebf4a9f6f3b7a650dc5d9a5bbb585ef83a52ed36ba46f55cc39d"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:778df71c8d0c8c9f1b378624b26431ca80041660d7be7c3f724b2c7a6e65d0d6"}, + {file = "yarl-1.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9f9cafaf031c34d95c1528c16b2fa07b710e6056b3c4e2e34e9317072da5d1a"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ca6b66f69e30f6e180d52f14d91ac854b8119553b524e0e28d5291a724f0f423"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0e7e83f31e23c5d00ff618045ddc5e916f9e613d33c5a5823bc0b0a0feb522f"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:af52725c7c39b0ee655befbbab5b9a1b209e01bb39128dce0db226a10014aacc"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0ab5baaea8450f4a3e241ef17e3d129b2143e38a685036b075976b9c415ea3eb"}, + {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6d350388ba1129bc867c6af1cd17da2b197dff0d2801036d2d7d83c2d771a682"}, + {file = "yarl-1.9.3-cp39-cp39-win32.whl", hash = "sha256:e2a16ef5fa2382af83bef4a18c1b3bcb4284c4732906aa69422cf09df9c59f1f"}, + {file = "yarl-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:d92d897cb4b4bf915fbeb5e604c7911021a8456f0964f3b8ebbe7f9188b9eabb"}, + {file = "yarl-1.9.3-py3-none-any.whl", hash = "sha256:271d63396460b6607b588555ea27a1a02b717ca2e3f2cf53bdde4013d7790929"}, + {file = "yarl-1.9.3.tar.gz", hash = "sha256:4a14907b597ec55740f63e52d7fee0e9ee09d5b9d57a4f399a7423268e457b57"}, ] [package.dependencies] @@ -2547,9 +2775,11 @@ multidict = ">=4.0" [extras] accelerate = ["accelerate"] bnb = ["bitsandbytes"] +peft = ["peft"] quantize = ["accelerate", "datasets", "texttable"] +torch = ["torch"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "f2ef5f41a172d14985367a385ad6ce844c8c05b2d68d9ddcc11b41f581921c96" +content-hash = "cd3fb4b4e4aaf100f6015afa8c9adc28c22e6c0b48752452892dc3d004c1562a" diff --git a/server/pyproject.toml b/server/pyproject.toml index 673968be..52431eea 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -30,14 +30,16 @@ transformers = "^4.32.1" einops = "^0.6.1" texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } -peft = "^0.4.0" -torch = { version = "^2.0.1" } +peft = { version = "^0.4.0", optional = true } +torch = { version = "^2.1.1", optional = true } scipy = "^1.11.1" pillow = "^10.0.0" [tool.poetry.extras] +torch = ["torch"] accelerate = ["accelerate"] bnb = ["bitsandbytes"] +peft = ["peft"] quantize = ["texttable", "datasets", "accelerate"] [tool.poetry.group.dev.dependencies] @@ -47,7 +49,7 @@ pytest = "^7.3.0" [[tool.poetry.source]] name = "pytorch-gpu-src" -url = "https://download.pytorch.org/whl/cu118" +url = "https://download.pytorch.org/whl/cu121" priority = "explicit" [tool.pytest.ini_options] diff --git a/server/requirements.txt b/server/requirements.txt index 7c81c5f9..bc1b8891 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,38 +1,23 @@ -accelerate==0.20.3 ; python_version >= "3.9" and python_version < "3.13" -aiohttp==3.8.5 ; python_version >= "3.9" and python_version < "3.13" -aiosignal==1.3.1 ; python_version >= "3.9" and python_version < "3.13" -async-timeout==4.0.3 ; python_version >= "3.9" and python_version < "3.13" -attrs==23.1.0 ; python_version >= "3.9" and python_version < "3.13" backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13" -bitsandbytes==0.41.1 ; python_version >= "3.9" and python_version < "3.13" -certifi==2023.7.22 ; python_version >= "3.9" and python_version < "3.13" -charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "3.13" +bitsandbytes==0.41.2.post2 ; python_version >= "3.9" and python_version < "3.13" +certifi==2023.11.17 ; python_version >= "3.9" and python_version < "3.13" +charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "3.13" click==8.1.7 ; python_version >= "3.9" and python_version < "3.13" colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows") -datasets==2.14.5 ; python_version >= "3.9" and python_version < "3.13" deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13" -dill==0.3.7 ; python_version >= "3.9" and python_version < "3.13" einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13" -filelock==3.12.4 ; python_version >= "3.9" and python_version < "3.13" -frozenlist==1.4.0 ; python_version >= "3.9" and python_version < "3.13" -fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "3.13" -fsspec[http]==2023.6.0 ; python_version >= "3.9" and python_version < "3.13" -googleapis-common-protos==1.60.0 ; python_version >= "3.9" and python_version < "3.13" -grpc-interceptor==0.15.3 ; python_version >= "3.9" and python_version < "3.13" -grpcio-reflection==1.58.0 ; python_version >= "3.9" and python_version < "3.13" -grpcio-status==1.58.0 ; python_version >= "3.9" and python_version < "3.13" -grpcio==1.58.0 ; python_version >= "3.9" and python_version < "3.13" -hf-transfer==0.1.3 ; python_version >= "3.9" and python_version < "3.13" +filelock==3.13.1 ; python_version >= "3.9" and python_version < "3.13" +fsspec==2023.10.0 ; python_version >= "3.9" and python_version < "3.13" +googleapis-common-protos==1.61.0 ; python_version >= "3.9" and python_version < "3.13" +grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13" +grpcio-reflection==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +grpcio-status==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +grpcio==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +hf-transfer==0.1.4 ; python_version >= "3.9" and python_version < "3.13" huggingface-hub==0.16.4 ; python_version >= "3.9" and python_version < "3.13" idna==3.4 ; python_version >= "3.9" and python_version < "3.13" -jinja2==3.1.2 ; python_version >= "3.9" and python_version < "3.13" loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" -markupsafe==2.1.3 ; python_version >= "3.9" and python_version < "3.13" -mpmath==1.3.0 ; python_version >= "3.9" and python_version < "3.13" -multidict==6.0.4 ; python_version >= "3.9" and python_version < "3.13" -multiprocess==0.70.15 ; python_version >= "3.9" and python_version < "3.13" -networkx==3.1 ; python_version >= "3.9" and python_version < "3.13" -numpy==1.26.0 ; python_version >= "3.9" and python_version < "3.13" +numpy==1.26.2 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "3.13" @@ -42,34 +27,21 @@ opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_versi opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13" -packaging==23.1 ; python_version >= "3.9" and python_version < "3.13" -pandas==2.1.1 ; python_version >= "3.9" and python_version < "3.13" -peft==0.4.0 ; python_version >= "3.9" and python_version < "3.13" -pillow==10.0.1 ; python_version >= "3.9" and python_version < "3.13" -protobuf==4.24.3 ; python_version >= "3.9" and python_version < "3.13" -psutil==5.9.5 ; python_version >= "3.9" and python_version < "3.13" -pyarrow==13.0.0 ; python_version >= "3.9" and python_version < "3.13" -python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "3.13" -pytz==2023.3.post1 ; python_version >= "3.9" and python_version < "3.13" +packaging==23.2 ; python_version >= "3.9" and python_version < "3.13" +pillow==10.1.0 ; python_version >= "3.9" and python_version < "3.13" +protobuf==4.25.1 ; python_version >= "3.9" and python_version < "3.13" pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" -regex==2023.8.8 ; python_version >= "3.9" and python_version < "3.13" +regex==2023.10.3 ; python_version >= "3.9" and python_version < "3.13" requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" safetensors==0.3.3 ; python_version >= "3.9" and python_version < "3.13" -scipy==1.11.2 ; python_version >= "3.9" and python_version < "3.13" +scipy==1.11.4 ; python_version >= "3.9" and python_version < "3.13" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" -setuptools==68.2.2 ; python_version >= "3.9" and python_version < "3.13" -six==1.16.0 ; python_version >= "3.9" and python_version < "3.13" -sympy==1.12 ; python_version >= "3.9" and python_version < "3.13" -texttable==1.6.7 ; python_version >= "3.9" and python_version < "3.13" +setuptools==69.0.2 ; python_version >= "3.9" and python_version < "3.13" tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" -torch==2.0.1 ; python_version >= "3.9" and python_version < "3.13" tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" -transformers==4.33.2 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.33.3 ; python_version >= "3.9" and python_version < "3.13" typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" typing-extensions==4.8.0 ; python_version >= "3.9" and python_version < "3.13" -tzdata==2023.3 ; python_version >= "3.9" and python_version < "3.13" -urllib3==2.0.5 ; python_version >= "3.9" and python_version < "3.13" +urllib3==2.1.0 ; python_version >= "3.9" and python_version < "3.13" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32" -wrapt==1.15.0 ; python_version >= "3.9" and python_version < "3.13" -xxhash==3.3.0 ; python_version >= "3.9" and python_version < "3.13" -yarl==1.9.2 ; python_version >= "3.9" and python_version < "3.13" +wrapt==1.16.0 ; python_version >= "3.9" and python_version < "3.13" From c6bb76703f2e7d1904417130994e8a8fdb0b6531 Mon Sep 17 00:00:00 2001 From: Vince Jankovics Date: Thu, 23 Nov 2023 14:00:09 +0000 Subject: [PATCH 479/793] Fix IDEFICS dtype (#1214) # What does this PR do? This forces the use of `bfloat16` for IDEFICS. The issue is that with `float16` the 80b model gives garbage output. Let me know if this solution is not appropriate and I'll adjust accordingly. For the details see below. The current behaviour: ```sh $ curl 127.0.0.1:8080/generate -X POST -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' -H 'Content-Type: application/json' {"generated_text":""} ``` On closer inspection with: ```python import requests headers = { "Content-Type": "application/json"} query = "What is Deep Learning?" data = { "inputs": query, "parameters": { "max_new_tokens": 10, "return_full_text": True, "decoder_input_details": True, "do_sample": False, }, } api_url = "http://127.0.0.1:8080" response = requests.post(api_url + "/generate", headers=headers, json=data).json() for i in ['prefill', 'tokens']: print(f'### {i}') print(repr(''.join([t['text'] for t in response['details'][i]]))) ``` Prints: ``` ### prefill 'WhatisDeepLearning?' ### tokens '' ######## ``` With the change in this PR it prints: ``` ### prefill 'WhatisDeepLearning?' ### tokens '\n\nDeep Learning is a subset of machine' ``` Note, using the Transformers implementation (with `IdeficsForVisionText2Text.from_pretrained`) produces the latter (correct) output as well. This only happens with the 80b model, the 9b model is not as sensitive to the dtype (as also mentioned in the code). The reason for "forcing" this in the IDEFICS init method, is because if quantization is used, then the dtype cannot be set explicitly. And since it's left as `None`, it's set to `float16` by default [here](https://github.com/huggingface/text-generation-inference/blob/96a982ad8fc232479384476b1596a880697cc1d0/server/text_generation_server/models/__init__.py#L90). I.e. there's no other way to manually change the dtype if someone is using quantization: ```sh $ docker run .... ghcr.io/huggingface/text-generation-inference:latest --model-id HuggingFaceM4/idefics-80b-instruct --dtype bfloat16 --quantize bitsandbytes-nf4 ..... 2023-10-31T12:42:26.710401Z INFO shard-manager: text_generation_launcher: Starting shard rank=0 2023-10-31T12:42:30.315734Z ERROR shard-manager: text_generation_launcher: Shard complete standard error output: Traceback (most recent call last): File "/opt/conda/bin/text-generation-server", line 8, in sys.exit(app()) File "/opt/conda/lib/python3.9/site-packages/text_generation_server/cli.py", line 80, in serve raise RuntimeError( RuntimeError: Only 1 can be set between `dtype` and `quantize`, as they both decide how goes the final model. rank=0 Error: ShardCannotStart 2023-10-31T12:42:30.414010Z ERROR text_generation_launcher: Shard 0 failed to start 2023-10-31T12:42:30.414044Z INFO text_generation_launcher: Shutting down shards ``` ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @Narsil what do you think? --------- Co-authored-by: Nicolas Patry --- server/text_generation_server/models/idefics_causal_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py index 2472caf6..dcad1fa9 100644 --- a/server/text_generation_server/models/idefics_causal_lm.py +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -583,7 +583,7 @@ def __init__( if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.float16 if dtype is None else dtype + dtype = torch.bfloat16 if dtype is None else dtype else: if quantize: raise ValueError("quantization is not available on CPU") From 3c02262f29fba3bf1096f22017f70d59ff4daa4d Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 23 Nov 2023 15:42:48 +0000 Subject: [PATCH 480/793] Reduce race condition on file system for test --- router/src/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index 560b8f74..d6df2f56 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -300,15 +300,22 @@ mod tests { use tokenizers::Tokenizer; pub(crate) async fn get_tokenizer() -> Tokenizer { - if !std::path::Path::new("tokenizer.json").exists() { + let filename = std::path::Path::new("tokenizer.json"); + if !filename.exists() { let content = reqwest::get("https://huggingface.co/gpt2/raw/main/tokenizer.json") .await .unwrap() .bytes() .await .unwrap(); - let mut file = std::fs::File::create("tokenizer.json").unwrap(); + let tmp_filename = "tokenizer.json.temp"; + let mut file = std::fs::File::create(tmp_filename).unwrap(); file.write_all(&content).unwrap(); + // Re-check if another process has written this file maybe. + if !filename.exists() { + std::fs::rename(tmp_filename, filename).unwrap() + } + } Tokenizer::from_file("tokenizer.json").unwrap() } From ed2a3f617e6116c1fe8ea79aead4fd89b8140a6f Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Sat, 25 Nov 2023 22:38:38 +0100 Subject: [PATCH 481/793] Exllama v2 (#1211) # What does this PR do? See #1165 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Florian Zimmermeister Co-authored-by: Ubuntu --- Dockerfile | 10 + router/src/lib.rs | 1 - .../exllamav2_kernels/config.h | 13 + .../exllamav2_kernels/cpp/util.h | 12 + .../exllamav2_kernels/cuda/compat.cuh | 56 ++ .../exllamav2_kernels/cuda/compat_gemm.cuh | 38 ++ .../exllamav2_kernels/cuda/matrix_view.cuh | 121 ++++ .../exllamav2_kernels/cuda/q_gemm.cu | 211 ++++++ .../exllamav2_kernels/cuda/q_gemm.cuh | 33 + .../exllamav2_kernels/cuda/q_gemm_kernel.cuh | 487 ++++++++++++++ .../cuda/q_gemm_kernel_gptq.cuh | 219 ++++++ .../exllamav2_kernels/cuda/q_matrix.cu | 623 ++++++++++++++++++ .../exllamav2_kernels/cuda/q_matrix.cuh | 73 ++ .../exllamav2_kernels/cuda/quant/qdq_2.cuh | 103 +++ .../exllamav2_kernels/cuda/quant/qdq_3.cuh | 169 +++++ .../exllamav2_kernels/cuda/quant/qdq_4.cuh | 227 +++++++ .../exllamav2_kernels/cuda/quant/qdq_5.cuh | 207 ++++++ .../exllamav2_kernels/cuda/quant/qdq_6.cuh | 44 ++ .../exllamav2_kernels/cuda/quant/qdq_8.cuh | 38 ++ .../exllamav2_kernels/cuda/quant/qdq_util.cuh | 51 ++ .../exllamav2_kernels/cuda/util.cuh | 42 ++ .../exllamav2_kernels/ext.cpp | 134 ++++ server/exllamav2_kernels/setup.py | 17 + server/text_generation_server/server.py | 2 +- .../utils/gptq/exllamav2.py | 191 ++++++ server/text_generation_server/utils/layers.py | 22 +- .../text_generation_server/utils/weights.py | 20 +- 27 files changed, 3144 insertions(+), 20 deletions(-) create mode 100644 server/exllamav2_kernels/exllamav2_kernels/config.h create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cpp/util.h create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/compat.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/compat_gemm.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/matrix_view.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_2.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_3.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_4.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_5.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_6.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_8.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_util.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/cuda/util.cuh create mode 100644 server/exllamav2_kernels/exllamav2_kernels/ext.cpp create mode 100644 server/exllamav2_kernels/setup.py create mode 100644 server/text_generation_server/utils/gptq/exllamav2.py diff --git a/Dockerfile b/Dockerfile index cf5e0ed6..d45aaec5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -112,6 +112,14 @@ RUN make build-flash-attention-v2 FROM kernel-builder as exllama-kernels-builder WORKDIR /usr/src COPY server/exllama_kernels/ . + +RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" python setup.py build + +# Build Transformers exllama kernels +FROM kernel-builder as exllamav2-kernels-builder +WORKDIR /usr/src +COPY server/exllamav2_kernels/ . + # Build specific version of transformers RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" python setup.py build @@ -182,6 +190,8 @@ COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86 COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Copy build artifacts from exllama kernels builder COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages +# Copy build artifacts from exllamav2 kernels builder +COPY --from=exllamav2-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Copy build artifacts from awq kernels builder COPY --from=awq-kernels-builder /usr/src/llm-awq/awq/kernels/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages # Copy build artifacts from eetq kernels builder diff --git a/router/src/lib.rs b/router/src/lib.rs index d6df2f56..b547dc15 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -315,7 +315,6 @@ mod tests { if !filename.exists() { std::fs::rename(tmp_filename, filename).unwrap() } - } Tokenizer::from_file("tokenizer.json").unwrap() } diff --git a/server/exllamav2_kernels/exllamav2_kernels/config.h b/server/exllamav2_kernels/exllamav2_kernels/config.h new file mode 100644 index 00000000..86baaf41 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/config.h @@ -0,0 +1,13 @@ +#ifndef _config_h +#define _config_h + +#define MAX_Q_GEMM_ROWS 50 + +#define QMODE_2BIT 1 +#define QMODE_3BIT 1 +#define QMODE_4BIT 1 +#define QMODE_5BIT 1 +#define QMODE_6BIT 0 +#define QMODE_8BIT 0 + +#endif diff --git a/server/exllamav2_kernels/exllamav2_kernels/cpp/util.h b/server/exllamav2_kernels/exllamav2_kernels/cpp/util.h new file mode 100644 index 00000000..919703a8 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cpp/util.h @@ -0,0 +1,12 @@ +#ifndef _util_h +#define _util_h + +#define DBGS(__x) printf("%s\n", __x) +#define DBGI(__x) printf("%s: %i\n", #__x, __x) +#define DBGI2(__x, __y) printf("%s, %s: %i, %i\n", #__x, #__y, __x, __y) +#define DBGI3(__x, __y, __z) printf("%s, %s, %s: %i, %i, %i\n", #__x, #__y, #__z, __x, __y, __z) +#define DBGF(__x) printf("%s: %f\n", #__x, __x) +#define DBGF2(__x, __y) printf("%s, %s: %f, %f\n", #__x, #__y, __x, __y) +#define DBGF3(__x, __y, __z) printf("%s, %s, %s: %f, %f, %f\n", #__x, #__y, #__z, __x, __y, __z) + +#endif diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/compat.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/compat.cuh new file mode 100644 index 00000000..12684ff8 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/compat.cuh @@ -0,0 +1,56 @@ +#ifndef _compat_cuh +#define _compat_cuh + +// atomicAdd for half types, to support CC < 7.x + +__device__ __forceinline__ void atomicAdd_half(half* address, half val) +{ + unsigned int * address_as_ui = (unsigned int *) ((char *)address - ((size_t)address & 2)); + unsigned int old = *address_as_ui; + unsigned int assumed; + + do + { + assumed = old; + __half_raw hsum; + hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff); + half tmpres = __hadd(hsum, val); + hsum = __half_raw(tmpres); + old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x; + old = atomicCAS(address_as_ui, assumed, old); + } + while (assumed != old); +} + +// atomicAdd for half2 types + +__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val) +{ + unsigned int* address_as_ui = (unsigned int*)address; + unsigned int old = *address_as_ui; + unsigned int assumed; + do + { + assumed = old; + half2 old_val = *((half2*)&old); + half2 new_val = __hadd2(old_val, val); + old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val)); + } + while (assumed != old); +} + +// + +#if defined(__CUDA_ARCH__) || defined(USE_ROCM) +#if __CUDA_ARCH__ < 700 || defined(USE_ROCM) + +__device__ __forceinline__ void atomicAdd(half* address, half val) { atomicAdd_half(address, val); } + +#if __CUDA_ARCH__ < 600 || defined(USE_ROCM) +__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { atomicAdd_half2(address, val); } +#endif + +#endif +#endif + +#endif diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/compat_gemm.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/compat_gemm.cuh new file mode 100644 index 00000000..19b1e4a6 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/compat_gemm.cuh @@ -0,0 +1,38 @@ +#ifndef _compat_gemm_cuh +#define _compat_gemm_cuh + +#if defined(USE_ROCM) + +// For some reason this include is not present anywhere in exllama_v2 codebase, but it is required +// for symbols as hipblasHalf. +#include + +__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle, + hipblasOperation_t transA, + hipblasOperation_t transB, + int m, + int n, + int k, + const half* alpha, + const half* AP, + int lda, + const half* BP, + int ldb, + const half* beta, + half* CP, + int ldc) { + return hipblasHgemm(handle, transA, transB, m, n, k, + reinterpret_cast(alpha), + reinterpret_cast(AP), lda, + reinterpret_cast(BP), ldb, + reinterpret_cast(beta), + reinterpret_cast(CP), ldc); +} +#define hipblasHgemm __compat_hipblasHgemm + +// Previous version of PyTorch were converting to rocBLAS instead of hipBLAS. +#define rocblas_operation_none HIPBLAS_OP_N +#define rocblas_hgemm __compat_hipblasHgemm +#endif + +#endif diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/matrix_view.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/matrix_view.cuh new file mode 100644 index 00000000..55af84f2 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/matrix_view.cuh @@ -0,0 +1,121 @@ +#ifndef _matrix_view_cuh +#define _matrix_view_cuh + +#include +#include + +#include "quant/qdq_util.cuh" + +class MatrixView_half +{ +public: + const half* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width) + : data(data), height(height), width(width) + { } + + __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; } + __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; } + __device__ __forceinline__ half2 item_half2half2(int row, int column) const { return __half2half2(data[row * width + column]); } + __device__ __forceinline__ const half* item_ptr(int row, int column) const { return &data[row * width + column]; } + + __device__ __forceinline__ void item4(half (&items)[4], int row, int column) const + { + half2* ptr = (half2*) item_ptr(row, column); + half2 i01 = ptr[0]; + half2 i23 = ptr[1]; + items[0] = __low2half(i01); + items[1] = __high2half(i01); + items[2] = __low2half(i23); + items[3] = __high2half(i23); + } + __device__ __forceinline__ void item4_f(float (&items)[4], int row, int column) const + { + half2* ptr = (half2*)item_ptr(row, column); + half2 i01 = ptr[0]; + half2 i23 = ptr[1]; + items[0] = __half2float(__low2half(i01)); + items[1] = __half2float(__high2half(i01)); + items[2] = __half2float(__low2half(i23)); + items[3] = __half2float(__high2half(i23)); + } + + __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row, int column) const + { + half2* ptr = (half2*)item_ptr(row, column); + half2 i01 = ptr[0]; + half2 i23 = ptr[1]; + items[0] = __half2half2(__low2half(i01)); + items[1] = __half2half2(__high2half(i01)); + items[2] = __half2half2(__low2half(i23)); + items[3] = __half2half2(__high2half(i23)); + } +}; + +class MatrixView_half_rw +{ +public: + half* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width) + : data(data), height(height), width(width) + { } + + __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; } + __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; } + __device__ __forceinline__ half* item_ptr(int row, int column) { return &data[row * width + column]; } + __device__ __forceinline__ void set(int row, int column, half value) { data[row * width + column] = value; } + __device__ __forceinline__ void set_half2(int row, int column, half2 value) { ((half2*)data)[(row * width + column) / 2] = value; } + + __device__ __forceinline__ void set4(int row, int column, half v0, half v1, half v2, half v3) + { + half2 v01 = __halves2half2(v0, v1); + half2 v23 = __halves2half2(v2, v3); + half2* ptr = (half2*) item_ptr(row, column); + ptr[0] = v01; + ptr[1] = v23; + } +}; + +class MatrixView_q4_row +{ +public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) + { } + + __device__ __forceinline__ int item(int row, int column) const + { + int shift = (column & 0x07) * 4; + return (data[row * width / 8 + column / 8] >> shift) & 0x0f; + } + + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const + { + int shift = (column & 0x07) * 4; + uint32_t d = data[row * width / 8 + column / 8] >> shift; + items[0] = d & 0x0f; + items[1] = (d >> 4) & 0x0f; + } + + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const + { + int shift = (column & 0x07) * 4; + uint32_t d = data[row * width / 8 + column / 8] >> shift; + items[0] = d & 0x0f; + items[1] = (d >> 4) & 0x0f; + items[2] = (d >> 8) & 0x0f; + items[3] = (d >> 12) & 0x0f; + } +}; + +#endif \ No newline at end of file diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu new file mode 100644 index 00000000..351b9cd5 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu @@ -0,0 +1,211 @@ +#include "q_gemm.cuh" +#include "util.cuh" +#include "matrix_view.cuh" +#include "../config.h" + +#include "quant/qdq_2.cuh" +#include "quant/qdq_3.cuh" +#include "quant/qdq_4.cuh" +#include "quant/qdq_5.cuh" +#include "quant/qdq_6.cuh" +#include "quant/qdq_8.cuh" + +#define BLOCK_KN_SIZE 128 +#define BLOCK_M_SIZE_MAX 8 +#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32) +#define CLEAR_N_SIZE 256 + +#include "q_gemm_kernel.cuh" +#include "q_gemm_kernel_gptq.cuh" + +#include "compat_gemm.cuh" + +void gemm_half_q_half_cuda_part +( + const half* a, + QMatrix* b, + half* c, + int size_m, + int size_n, + int size_k, + int m_count, + bool clear +) +{ + if (!b->is_gptq) + { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + blockDim.z = 1; + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4); + gridDim.y = DIVIDE(size_m, m_count); + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE); + + fp_gemm_half_q_half_kernel kernel = pick_gemm_half_q_half_kernel(true, m_count); + + kernel<<>> + ( + a, + b->cuda_q_weight, + b->cuda_q_scale, + b->cuda_q_scale_max, + c, + size_m, + size_n, + size_k, + b->groups, + b->groupsize, + b->cuda_q_perm, + b->rows_8, + b->rows_6, + b->rows_5, + b->rows_4, + b->rows_3, + b->rows_2, + clear + ); + } + else + { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + blockDim.z = 1; + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4); + gridDim.y = DIVIDE(size_m, m_count); + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE); + + fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count); + +// DBGX((uint64_t) b->cuda_q_perm); +// DBGI(b->rows_4); +// DBGI(b->height); + + kernel<<>> + ( + a, + b->cuda_q_weight, + b->cuda_gptq_qzeros, + b->cuda_gptq_scales, + c, + size_m, + size_n, + size_k, + b->groups, + b->groupsize, + b->cuda_q_perm, + b->rows_4, + clear + ); + } +} + +void gemm_half_q_half_cuda +( + cublasHandle_t cublas_handle, + const half* a, + QMatrix* b, + half* c, + int size_m, + int size_n, + int size_k, + bool clear, + half* temp_dq, + bool force_cuda +) +{ + if (size_m > MAX_Q_GEMM_ROWS && !force_cuda) + { + //printf("cublas\n"); + + // Reconstruct FP16 matrix, then cuBLAS + + if (!temp_dq) temp_dq = b->temp_dq; + b->reconstruct(temp_dq); + + //cublasSetMathMode(cublas_handle, CUBLAS_TENSOR_OP_MATH); + + const half alpha = __float2half(1.0f); + const half beta = clear ? __float2half(0.0f) : __float2half(1.0f); + cublasHgemm(cublas_handle, + CUBLAS_OP_N, + CUBLAS_OP_N, + size_n, size_m, size_k, + &alpha, temp_dq, size_n, + a, size_k, + &beta, c, size_n); + + //const float alpha = 1.0f; + //const float beta = clear ? 0.0f : 1.0f; + //cublasSgemmEx(cublas_handle, + // CUBLAS_OP_N, + // CUBLAS_OP_N, + // size_n, size_m, size_k, + // &alpha, temp_dq, CUDA_R_16F, size_n, + // a, CUDA_R_16F, size_k, + // &beta, c, CUDA_R_16F, size_n); + + //const float alpha = 1.0f; + //const float beta = clear ? 0.0f : 1.0f; + //cublasGemmEx(cublas_handle, + // CUBLAS_OP_N, CUBLAS_OP_N, + // size_n, size_m, size_k, + // &alpha, temp_dq, CUDA_R_16F, size_n, + // a, CUDA_R_16F, size_k, + // &beta, c, CUDA_R_16F, size_n, + // CUDA_R_16F, CUBLAS_GEMM_DFALT_TENSOR_OP); + } + else + { + //printf("cuda\n"); + + // Quantized matmul + + //if (clear) clear_tensor_cuda(c, size_m, size_n); + + int max_chunks = size_m / BLOCK_M_SIZE_MAX; + int last_chunk = max_chunks * BLOCK_M_SIZE_MAX; + int last_chunk_size = size_m - last_chunk; + + if (max_chunks) + { + gemm_half_q_half_cuda_part(a, b, c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX, clear); + } + + if (last_chunk_size) + { + gemm_half_q_half_cuda_part(a + last_chunk * size_k, b, c + last_chunk * size_n, last_chunk_size, size_n, size_k, last_chunk_size, clear); + } + } +} + +__global__ void clear_kernel +( + half* __restrict__ c, + const int size_m, + const int size_n +) +{ + int m = blockIdx.y; + int n = (blockIdx.x * CLEAR_N_SIZE + threadIdx.x) * 8; + if (n >= size_n) return; + int4* c_ptr = (int4*)(c + m * size_n + n); + *c_ptr = {}; +} + +void clear_tensor_cuda +( + half* c, + int size_m, + int size_n +) +{ + return; + dim3 blockDim, gridDim; + blockDim.x = CLEAR_N_SIZE; + blockDim.y = 1; + gridDim.x = DIVIDE(size_n / 8, CLEAR_N_SIZE); + gridDim.y = size_m; + clear_kernel<<>>(c, size_m, size_n); +} diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cuh new file mode 100644 index 00000000..c69f1a70 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cuh @@ -0,0 +1,33 @@ +#ifndef _q_gemm_cuh +#define _q_gemm_cuh + +#include +#include +#include +#include +#include + +#include "q_matrix.cuh" + +void gemm_half_q_half_cuda +( + cublasHandle_t cublas_handle, + const half* a, + QMatrix* b, + half* c, + int size_m, + int size_n, + int size_k, + bool clear = false, + half* reconstruct = NULL, + bool force_cuda = false +); + +void clear_tensor_cuda +( + half* c, + int size_m, + int size_n +); + +#endif \ No newline at end of file diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel.cuh new file mode 100644 index 00000000..0b899a84 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel.cuh @@ -0,0 +1,487 @@ +#include "compat.cuh" + +#include +#include + +__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result); + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); +} + +__forceinline__ __device__ half2 dot22_16(half2(&dq)[8], const half* a_ptr, const half2 g_result, const half qs_h) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result); + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); +} + +__forceinline__ __device__ half2 dot22_32(half2(&dq)[16], const half* a_ptr, const half2 g_result, const half qs_h) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result); + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); +} + +__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr, const float g_result, const float qs_f) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result); + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result)); + return fma(result_f, qs_f, g_result); +} + +__forceinline__ __device__ float dot22_16_f(half2(&dq)[8], const half* a_ptr, const float g_result, const float qs_f) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result); + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result)); + return fma(result_f, qs_f, g_result); +} + +__forceinline__ __device__ float dot22_32_f(half2(&dq)[16], const half* a_ptr, const float g_result, const float qs_f) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result); + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result)); + return fma(result_f, qs_f, g_result); +} + + + +typedef void (*fp_gemm_half_q_half_kernel) +( + const half*, + const uint32_t*, + const uint32_t*, + const half*, + half*, + const int, + const int, + const int, + const int, + const int, + const uint16_t*, + const int, + const int, + const int, + const int, + const int, + const int, + const bool +); + +template +__global__ void gemm_half_q_half_kernel +( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_q_scale, + const half* __restrict__ b_q_scale_max, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int groupsize, + const uint16_t* __restrict__ b_q_perm, + const int rows_8, + const int rows_6, + const int rows_5, + const int rows_4, + const int rows_3, + const int rows_2, + const bool clear +) +{ + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q4_row b_q_scale_(b_q_scale, groups, size_n); + + int t = threadIdx.x; + + // Block + + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + int offset_m = blockIdx.y * m_count; + int offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n); + int end_m = min(offset_m + m_count, size_m); + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + int n = offset_n + t * 4; + + // Preload block_a + + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) + { + for (int m = 0; m < m_count; ++m) + { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + half a0 = a_ptr[b_q_perm[offset_k + t]]; + block_a_ptr[t] = a0; + } + } + + // Clear + + if (n >= size_n) return; + + if (clear && blockIdx.z == 0) // && (threadIdx.x & 1) == 0) + { + for (int m = 0; m < m_count; m++) + *((uint64_t*) c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + + int group = offset_k / groupsize; + + // Preload scales + + float scales[MAX_GROUPS_IN_BLOCK][4]; + + int groups_in_block = DIVIDE((end_k - offset_k), groupsize); + for (int g = 0; g < groups_in_block; g++) + { + int qscales[4]; + b_q_scale_.item4(qscales, group + g, n); + qscales[0]++; + qscales[1]++; + qscales[2]++; + qscales[3]++; + float maxscale = __half2float(b_q_scale_max[group + g]); + scales[g][0] = __int2float_rn(qscales[0] * qscales[0]) * maxscale; + scales[g][1] = __int2float_rn(qscales[1] * qscales[1]) * maxscale; + scales[g][2] = __int2float_rn(qscales[2] * qscales[2]) * maxscale; + scales[g][3] = __int2float_rn(qscales[3] * qscales[3]) * maxscale; + } + + // a, b offset + + int pre_rows_8 = min(rows_8, offset_k); + int pre_rows_6 = offset_k > rows_8 ? min(rows_6, offset_k) - rows_8 : 0; + int pre_rows_5 = offset_k > rows_6 ? min(rows_5, offset_k) - rows_6 : 0; + int pre_rows_4 = offset_k > rows_5 ? min(rows_4, offset_k) - rows_5 : 0; + int pre_rows_3 = offset_k > rows_4 ? min(rows_3, offset_k) - rows_4 : 0; + int pre_rows_2 = offset_k > rows_3 ? min(rows_2, offset_k) - rows_3 : 0; + int qk = 0; + qk += pre_rows_8 / 32 * 8; + qk += pre_rows_6 / 32 * 6; + qk += pre_rows_5 / 32 * 5; + qk += pre_rows_4 / 32 * 4; + qk += pre_rows_3 / 32 * 3; + qk += pre_rows_2 / 32 * 2; + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + + int scales_idx = 0; + float qs_f0 = scales[scales_idx][0]; + float qs_f1 = scales[scales_idx][1]; + float qs_f2 = scales[scales_idx][2]; + float qs_f3 = scales[scales_idx][3]; + int nextgroup = offset_k + groupsize; + + // Column result + + float block_c[m_count][4] = {}; + + // Dequantize groups + + int k = offset_k; + + while (k < rows_8 && k < end_k) + { + if (k == nextgroup) + { + group++; + scales_idx++; + qs_f0 = scales[scales_idx][0]; + qs_f1 = scales[scales_idx][1]; + qs_f2 = scales[scales_idx][2]; + qs_f3 = scales[scales_idx][3]; + nextgroup += groupsize; + } + + #pragma unroll + for (int j = 0; j < 4; j++) + { + int4 load_int4[2]; + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n; + load_int4[1] = *((int4*) b_ptr); b_ptr += size_n; + + half2 dq[4][4]; + dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n); + dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n); + dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n); + dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n); + + for (int m = 0; m < m_count; m++) + { + block_c[m][0] = dot22_8_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); + block_c[m][1] = dot22_8_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); + block_c[m][2] = dot22_8_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); + block_c[m][3] = dot22_8_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + } + a_ptr += 8; + } + k += 32; + } + + while (k < rows_6 && k < end_k) + { + if (k == nextgroup) + { + group++; + scales_idx++; + qs_f0 = scales[scales_idx][0]; + qs_f1 = scales[scales_idx][1]; + qs_f2 = scales[scales_idx][2]; + qs_f3 = scales[scales_idx][3]; + nextgroup += groupsize; + } + + #pragma unroll + for (int j = 0; j < 2; j++) + { + int4 load_int4[3]; + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n; + load_int4[1] = *((int4*) b_ptr); b_ptr += size_n; + load_int4[2] = *((int4*) b_ptr); b_ptr += size_n; + + half2 dq[4][8]; + dequant_6bit_16(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n); + dequant_6bit_16(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n); + dequant_6bit_16(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n); + dequant_6bit_16(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n); + + for (int m = 0; m < m_count; m++) + { + block_c[m][0] = dot22_16_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); + block_c[m][1] = dot22_16_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); + block_c[m][2] = dot22_16_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); + block_c[m][3] = dot22_16_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + } + a_ptr += 16; + } + k += 32; + } + + while (k < rows_5 && k < end_k) + { + if (k == nextgroup) + { + group++; + scales_idx++; + qs_f0 = scales[scales_idx][0]; + qs_f1 = scales[scales_idx][1]; + qs_f2 = scales[scales_idx][2]; + qs_f3 = scales[scales_idx][3]; + nextgroup += groupsize; + } + + #pragma unroll + for (int j = 0; j < 1; j++) + { + int4 load_int4[5]; + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n; + load_int4[1] = *((int4*) b_ptr); b_ptr += size_n; + load_int4[2] = *((int4*) b_ptr); b_ptr += size_n; + load_int4[3] = *((int4*) b_ptr); b_ptr += size_n; + load_int4[4] = *((int4*) b_ptr); b_ptr += size_n; + + half2 dq[4][16]; + dequant_5bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, load_int4[3].x, load_int4[4].x, dq[0], size_n); + dequant_5bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, load_int4[3].y, load_int4[4].y, dq[1], size_n); + dequant_5bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, load_int4[3].z, load_int4[4].z, dq[2], size_n); + dequant_5bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, load_int4[3].w, load_int4[4].w, dq[3], size_n); + + for (int m = 0; m < m_count; m++) + { + block_c[m][0] = dot22_32_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); + block_c[m][1] = dot22_32_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); + block_c[m][2] = dot22_32_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); + block_c[m][3] = dot22_32_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + } + a_ptr += 32; + } + + k += 32; + } + + while (k < rows_4 && k < end_k) + { + if (k == nextgroup) + { + group++; + scales_idx++; + qs_f0 = scales[scales_idx][0]; + qs_f1 = scales[scales_idx][1]; + qs_f2 = scales[scales_idx][2]; + qs_f3 = scales[scales_idx][3]; + nextgroup += groupsize; + } + + #pragma unroll + for (int j = 0; j < 4; j++) + { + int4 load_int4[1]; + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n; + + half2 dq[4][4]; + dequant_4bit_8(load_int4[0].x, dq[0], size_n); + dequant_4bit_8(load_int4[0].y, dq[1], size_n); + dequant_4bit_8(load_int4[0].z, dq[2], size_n); + dequant_4bit_8(load_int4[0].w, dq[3], size_n); + + for (int m = 0; m < m_count; m++) + { + block_c[m][0] = dot22_8_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); + block_c[m][1] = dot22_8_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); + block_c[m][2] = dot22_8_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); + block_c[m][3] = dot22_8_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + } + a_ptr += 8; + } + k += 32; + } + + while (k < rows_3 && k < end_k) + { + if (k == nextgroup) + { + group++; + scales_idx++; + qs_f0 = scales[scales_idx][0]; + qs_f1 = scales[scales_idx][1]; + qs_f2 = scales[scales_idx][2]; + qs_f3 = scales[scales_idx][3]; + nextgroup += groupsize; + } + + #pragma unroll + for (int j = 0; j < 1; j++) + { + int4 load_int4[3]; + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n; + load_int4[1] = *((int4*) b_ptr); b_ptr += size_n; + load_int4[2] = *((int4*) b_ptr); b_ptr += size_n; + + half2 dq[4][16]; + dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n); + dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n); + dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n); + dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n); + + for (int m = 0; m < m_count; m++) + { + block_c[m][0] = dot22_32_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); + block_c[m][1] = dot22_32_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); + block_c[m][2] = dot22_32_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); + block_c[m][3] = dot22_32_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + } + a_ptr += 32; + } + k += 32; + } + + while (k < rows_2 && k < end_k) + { + if (k == nextgroup) + { + group++; + scales_idx++; + qs_f0 = scales[scales_idx][0]; + qs_f1 = scales[scales_idx][1]; + qs_f2 = scales[scales_idx][2]; + qs_f3 = scales[scales_idx][3]; + nextgroup += groupsize; + } + + #pragma unroll + for (int j = 0; j < 2; j++) + { + int4 load_int4[1]; + load_int4[0] = *((int4*) b_ptr); b_ptr += size_n; + + half2 dq[4][8]; + dequant_2bit_16(load_int4[0].x, dq[0], size_n); + dequant_2bit_16(load_int4[0].y, dq[1], size_n); + dequant_2bit_16(load_int4[0].z, dq[2], size_n); + dequant_2bit_16(load_int4[0].w, dq[3], size_n); + + for (int m = 0; m < m_count; m++) + { + block_c[m][0] = dot22_16_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); + block_c[m][1] = dot22_16_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); + block_c[m][2] = dot22_16_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); + block_c[m][3] = dot22_16_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + } + + a_ptr += 16; + } + k += 32; + } + + // Accumulate column sums in c + + for (int m = 0; m < m_count; m++) + { + half2* out = (half2*)c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1])); + half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3])); + atomicAdd(out , result01); + atomicAdd(out + 1, result23); + } +} + +fp_gemm_half_q_half_kernel pick_gemm_half_q_half_kernel(bool first_block, const int m_count) +{ + #if BLOCK_M_SIZE_MAX >= 1 + if (m_count == 1) return gemm_half_q_half_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 2 + if (m_count == 2) return gemm_half_q_half_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 3 + if (m_count == 3) return gemm_half_q_half_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 4 + if (m_count == 4) return gemm_half_q_half_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 5 + if (m_count == 5) return gemm_half_q_half_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 6 + if (m_count == 6) return gemm_half_q_half_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 7 + if (m_count == 7) return gemm_half_q_half_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 8 + if (m_count == 8) return gemm_half_q_half_kernel; + #endif + return NULL; +} diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh new file mode 100644 index 00000000..ebaa42d0 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh @@ -0,0 +1,219 @@ +#include "compat.cuh" + +__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result); + return __hadd2(result, g_result); +} + +__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result); + return __half2float(__low2half(result)) + __half2float(__high2half(result)); +} + +typedef void (*fp_gemm_half_q_half_gptq_kernel) +( + const half*, + const uint32_t*, + const uint32_t*, + const half*, + half*, + const int, + const int, + const int, + const int, + const int, + const uint16_t*, + const int, + const bool +); + +template +__global__ void gemm_half_q_half_gptq_kernel +( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int groupsize, + const uint16_t* __restrict__ b_q_perm, + const int rows_4, + const bool clear +) +{ + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + int t = threadIdx.x; + + // Block + + int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + int offset_m = blockIdx.y * m_count; + int offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n); + int end_m = min(offset_m + m_count, size_m); + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + int n = offset_n + t * 4; + + // Preload block_a + + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) + { + for (int m = 0; m < m_count; ++m) + { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + + half a0; + if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]]; + else a0 = a_ptr[offset_k + t]; + block_a_ptr[t] = a0; + } + } + + // Zero output + + if (n >= size_n) return; + + if (clear && blockIdx.z == 0) // && (threadIdx.x & 1) == 0) + { + for (int m = 0; m < m_count; m++) + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // a, b offset + + int qk = offset_k / (32 / 4); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + + int zeros[4]; + float scales[4]; + half2 z1z16[4][2]; + half2 y1y16[4][2]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_f(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + +// __syncthreads(); + + // Column result + + float block_c[m_count][4] = {}; + + // Dequantize and multiply + + int k = offset_k; + while (k < end_k) + { + if (k == nextgroup) + { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_f(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + } + + #pragma unroll + for (int j = 0; j < 4; j++) + { + const int4* b_ptr4 = (int4*) b_ptr; + int4 load_int4 = *b_ptr4; + + half2 dq[4][4]; + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false); + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false); + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false); + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false); + + #pragma unroll + for (int m = 0; m < m_count; m++) + { + block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]); + block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]); + block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]); + block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]); + } + + b_ptr += size_n; + a_ptr += 8; + } + + k += 32; + } + + for (int m = 0; m < m_count; m++) + { + half2 *out = (half2*) c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1])); + half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3])); + atomicAdd(out , result01); + atomicAdd(out + 1, result23); + } +} + +fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count) +{ + #if BLOCK_M_SIZE_MAX >= 1 + if (m_count == 1) return gemm_half_q_half_gptq_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 2 + if (m_count == 2) return gemm_half_q_half_gptq_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 3 + if (m_count == 3) return gemm_half_q_half_gptq_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 4 + if (m_count == 4) return gemm_half_q_half_gptq_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 5 + if (m_count == 5) return gemm_half_q_half_gptq_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 6 + if (m_count == 6) return gemm_half_q_half_gptq_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 7 + if (m_count == 7) return gemm_half_q_half_gptq_kernel; + #endif + #if BLOCK_M_SIZE_MAX >= 8 + if (m_count == 8) return gemm_half_q_half_gptq_kernel; + #endif + return NULL; +} diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu new file mode 100644 index 00000000..6aed7470 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu @@ -0,0 +1,623 @@ +#include "q_matrix.cuh" +#include "matrix_view.cuh" +#include "util.cuh" + +#include "quant/qdq_2.cuh" +#include "quant/qdq_3.cuh" +#include "quant/qdq_4.cuh" +#include "quant/qdq_5.cuh" +#include "quant/qdq_6.cuh" +#include "quant/qdq_8.cuh" + +#define BLOCK_KN_SIZE 128 + +#define THREADS_X 32 +#define THREADS_Y 32 + +// Shuffle quantized data on load + +__global__ void shuffle_kernel +( + uint32_t* __restrict__ b_q_weight, + const int size_k, + const int size_n, + const int rows_8, + const int rows_6, + const int rows_5, + const int rows_4, + const int rows_3, + const int rows_2 +) +{ + int n = blockIdx.x * THREADS_X + threadIdx.x; + if (n >= size_n) return; + int k = 0; + uint32_t* b_ptr = b_q_weight + n; + while (k < rows_8) { shuffle_8bit_4 (b_ptr, size_n); b_ptr += 1 * size_n; k += 4; } + while (k < rows_6) { shuffle_6bit_16(b_ptr, size_n); b_ptr += 3 * size_n; k += 16; } + while (k < rows_5) { shuffle_5bit_32(b_ptr, size_n); b_ptr += 5 * size_n; k += 32; } + while (k < rows_4) { shuffle_4bit_8 (b_ptr, size_n); b_ptr += 1 * size_n; k += 8; } + while (k < rows_3) { shuffle_3bit_32(b_ptr, size_n); b_ptr += 3 * size_n; k += 32; } + while (k < rows_2) { shuffle_2bit_16(b_ptr, size_n); b_ptr += 1 * size_n; k += 16; } +} + + +// QMatrix constructor + +QMatrix::QMatrix +( + const int _device, + const int _height, + const int _width, + const int _groups, + + uint32_t* _q_weight, + uint16_t* _q_perm, + uint16_t* _q_invperm, + uint32_t* _q_scale, + half* _q_scale_max, + uint16_t* _q_groups, + + uint32_t* _gptq_qzeros, + half* _gptq_scales, + uint32_t* _gptq_g_idx, + + half* _temp_dq +) : + device(_device), + height(_height), + width(_width), + groups(_groups), + temp_dq(_temp_dq) +{ + cudaSetDevice(device); + + failed = false; + + cuda_q_weight = _q_weight; + cuda_q_perm = _q_perm; + cuda_q_invperm = _q_invperm; + cuda_q_scale = _q_scale; + cuda_q_scale_max = _q_scale_max; + cuda_q_groups = _q_groups; + cuda_gptq_qzeros = _gptq_qzeros; + cuda_gptq_scales = _gptq_scales; + + is_gptq = (_gptq_qzeros != NULL); + + groupsize = 1; + while (groupsize * groups < height) groupsize *= 2; + + // Create group map + + rows_8 = 0; + rows_6 = 0; + rows_5 = 0; + rows_4 = 0; + rows_3 = 0; + rows_2 = 0; + + if (!is_gptq) + { + uint16_t* cpu_q_groups = (uint16_t*)calloc(groups * 2, sizeof(uint16_t)); + cudaMemcpy(cpu_q_groups, cuda_q_groups, groups * 2 * sizeof(uint16_t), cudaMemcpyDeviceToHost); + + for (int i = 0; i < groups; i++) + { + int bits = cpu_q_groups[i * 2]; + if (bits == 8) rows_8 += groupsize; + if (bits == 6) rows_6 += groupsize; + if (bits == 5) rows_5 += groupsize; + if (bits == 4) rows_4 += groupsize; + if (bits == 3) rows_3 += groupsize; + if (bits == 2) rows_2 += groupsize; + } + + free(cpu_q_groups); + + rows_6 += rows_8; + rows_5 += rows_6; + rows_4 += rows_5; + rows_3 += rows_4; + rows_2 += rows_3; + } + else + { + rows_4 = height; + rows_3 = height; + rows_2 = height; + + if (_gptq_g_idx) + { + if (!make_sequential(_gptq_g_idx)) + { + failed = true; + //printf("FAIL\n"); + return; + } + } + } + + // Shuffle quantized data + + dim3 blockDim, gridDim; + blockDim.x = THREADS_X; + blockDim.y = 1; + gridDim.x = DIVIDE(width, THREADS_X); + gridDim.y = 1; + + shuffle_kernel<<>>(cuda_q_weight, height, width, rows_8, rows_6, rows_5, rows_4, rows_3, rows_2); +} + +QMatrix::~QMatrix() +{ +} + +// Reconstruct b[k,n] (GPTQ) + +__global__ void reconstruct_gptq_kernel +( + const uint32_t* __restrict__ b_q_weight, + const uint16_t* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + //const uint16_t* __restrict__ b_q_groups, + const int size_k, + const int size_n, + const int groupsize, + const int groups, + half* __restrict__ b, + const int rows_4 +) +{ + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + int offset_k = BLOCK_KN_SIZE * blockIdx.y; + int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + // Preload remapping table + + __shared__ uint16_t perm[BLOCK_KN_SIZE]; + int t = threadIdx.x; + + if (b_q_perm) + { + if (offset_k + t < size_k) + perm[t] = b_q_perm[offset_k + t]; + } + + // Column + + int n = offset_n + t * 4; + if (n >= size_n) return; + + // Find initial group + + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // b offset + + int qk = offset_k / (32 / 4); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + // Initial zeros/scale + + int zeros[4]; + half2 scales[4]; + half2 z1z16[4][2]; + half2 y1y16[4][2]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + + __syncthreads(); + + int k = offset_k; + int lk = 0; + + while (k < end_k) + { + if (k == nextgroup) + { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + } + + for (int p = 0; p < 4; p++) + { + half2 dq[4][4]; + const int4* b_ptr4 = (int4*) b_ptr; + int4 load_int4 = *b_ptr4; + + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false); + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false); + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false); + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false); + + b_ptr += size_n; + //half* dqh = (half*)dq; + if (b_q_perm) + { + for (int j = 0; j < 4; j++) + { + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j])); + } + } + else + { + for (int j = 0; j < 4; j++) + { + for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j])); + } + } + } + k += 32; + } +} + + +// Reconstruct b[k,n] + +__global__ void reconstruct_kernel +( + const uint32_t* __restrict__ b_q_weight, + const uint16_t* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_q_scale, + const half* __restrict__ b_q_scale_max, + //const uint16_t* __restrict__ b_q_groups, + const int size_k, + const int size_n, + const int groupsize, + const int groups, + half* __restrict__ b, + const int rows_8, + const int rows_6, + const int rows_5, + const int rows_4, + const int rows_3, + const int rows_2 +) +{ + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q4_row b_q_scale_(b_q_scale, groups, size_n); + + int offset_k = BLOCK_KN_SIZE * blockIdx.y; + int offset_n = BLOCK_KN_SIZE * blockIdx.x; + + // Preload remapping table + + int t = threadIdx.x; + __shared__ uint16_t perm[BLOCK_KN_SIZE]; + if (offset_k + t < size_k) + perm[t] = b_q_perm[offset_k + t]; + + // Column + + int n = offset_n + t; + if (n >= size_n) return; + + // Find initial group + + int group = offset_k / groupsize; + + int pre_rows_8 = min(rows_8, offset_k); + int pre_rows_6 = offset_k > rows_8 ? min(rows_6, offset_k) - rows_8 : 0; + int pre_rows_5 = offset_k > rows_6 ? min(rows_5, offset_k) - rows_6 : 0; + int pre_rows_4 = offset_k > rows_5 ? min(rows_4, offset_k) - rows_5 : 0; + int pre_rows_3 = offset_k > rows_4 ? min(rows_3, offset_k) - rows_4 : 0; + int pre_rows_2 = offset_k > rows_3 ? min(rows_2, offset_k) - rows_3 : 0; + int qk = 0; + qk += pre_rows_8 / 32 * 8; + qk += pre_rows_6 / 32 * 6; + qk += pre_rows_5 / 32 * 5; + qk += pre_rows_4 / 32 * 4; + qk += pre_rows_3 / 32 * 3; + qk += pre_rows_2 / 32 * 2; + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + half qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); + half2 qs_h2 = __halves2half2(qs_h, qs_h); + int nextgroup = offset_k + groupsize; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + int k = offset_k; + int lk = 0; + + __syncthreads(); + + while (k < rows_8 && k < end_k) + { + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + for (int p = 0; p < 4; p++) + { + half2 dq[4]; + uint32_t q_0 = *b_ptr; b_ptr += size_n; + uint32_t q_1 = *b_ptr; b_ptr += size_n; + dequant_8bit_8(q_0, q_1, dq, size_n); + for (int j = 0; j < 4; j++) dq[j] = __hmul2(dq[j], qs_h2); + half* dqh = (half*) dq; + for (int j = 0; j < 8; j++) b_.set(perm[lk++], n, dqh[j]); + } + k += 32; + } + + while (k < rows_6 && k < end_k) + { + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + for (int p = 0; p < 2; p++) + { + half2 dq[8]; + uint32_t q_0 = *b_ptr; b_ptr += size_n; + uint32_t q_1 = *b_ptr; b_ptr += size_n; + uint32_t q_2 = *b_ptr; b_ptr += size_n; + dequant_6bit_16(q_0, q_1, q_2, dq, size_n); + for (int j = 0; j < 8; j++) dq[j] = __hmul2(dq[j], qs_h2); + half* dqh = (half*) dq; + for (int j = 0; j < 16; j++) b_.set(perm[lk++], n, dqh[j]); + } + k += 32; + } + + while (k < rows_5 && k < end_k) + { + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + for (int p = 0; p < 1; p++) + { + half2 dq[16]; + uint32_t q_0 = *b_ptr; b_ptr += size_n; + uint32_t q_1 = *b_ptr; b_ptr += size_n; + uint32_t q_2 = *b_ptr; b_ptr += size_n; + uint32_t q_3 = *b_ptr; b_ptr += size_n; + uint32_t q_4 = *b_ptr; b_ptr += size_n; + dequant_5bit_32(q_0, q_1, q_2, q_3, q_4, dq, size_n); + for (int j = 0; j < 16; j++) dq[j] = __hmul2(dq[j], qs_h2); + half* dqh = (half*) dq; + for (int j = 0; j < 32; j++) b_.set(perm[lk++], n, dqh[j]); + } + k += 32; + } + + while (k < rows_4 && k < end_k) + { + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + for (int p = 0; p < 4; p++) + { + half2 dq[4]; + uint32_t q_0 = *b_ptr; b_ptr += size_n; + dequant_4bit_8(q_0, dq, size_n); + for (int j = 0; j < 4; j++) dq[j] = __hmul2(dq[j], qs_h2); + half* dqh = (half*) dq; + for (int j = 0; j < 8; j++) b_.set(perm[lk++], n, dqh[j]); + } + k += 32; + } + + while (k < rows_3 && k < end_k) + { + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + for (int p = 0; p < 1; p++) + { + half2 dq[16]; + uint32_t q_0 = *b_ptr; b_ptr += size_n; + uint32_t q_1 = *b_ptr; b_ptr += size_n; + uint32_t q_2 = *b_ptr; b_ptr += size_n; + dequant_3bit_32(q_0, q_1, q_2, dq, size_n); + for (int j = 0; j < 16; j++) dq[j] = __hmul2(dq[j], qs_h2); + half* dqh = (half*) dq; + for (int j = 0; j < 32; j++) b_.set(perm[lk++], n, dqh[j]); + } + k += 32; + } + + while (k < rows_2 && k < end_k) + { + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + for (int p = 0; p < 2; p++) + { + half2 dq[8]; + uint32_t q_0 = *b_ptr; b_ptr += size_n; + dequant_2bit_16(q_0, dq, size_n); + for (int j = 0; j < 8; j++) dq[j] = __hmul2(dq[j], qs_h2); + half* dqh = (half*) dq; + for (int j = 0; j < 16; j++) b_.set(perm[lk++], n, dqh[j]); + } + k += 32; + } +} + +void QMatrix::reconstruct(half* out) +{ + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + gridDim.y = DIVIDE(height, BLOCK_KN_SIZE); + + if (!is_gptq) + { + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE); + reconstruct_kernel<<>> + ( + cuda_q_weight, + cuda_q_perm, + cuda_q_scale, + cuda_q_scale_max, + //cuda_q_groups, + height, + width, + groupsize, + groups, + out, + rows_8, + rows_6, + rows_5, + rows_4, + rows_3, + rows_2 + ); + } + else + { + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE * 4); + reconstruct_gptq_kernel<<>> + ( + cuda_q_weight, + cuda_q_perm, + cuda_gptq_qzeros, + cuda_gptq_scales, + //const uint16_t* __restrict__ b_q_groups, + height, + width, + groupsize, + groups, + out, + rows_4 + ); + } +} + +__global__ void make_sequential_kernel +( + const uint32_t* __restrict__ w, + uint32_t* __restrict__ w_new, + const uint16_t* __restrict__ q_perm, + const int w_height, + const int w_width +) +{ + const uint64_t* w2 = (uint64_t*) w; + uint64_t* w_new2 = (uint64_t*) w_new; + int w2_stride = w_width >> 1; + + int w2_column = THREADS_X * blockIdx.x + threadIdx.x; + if (w2_column >= w2_stride) return; + + int w_new2_row = blockIdx.y; + + int q_perm_idx = w_new2_row << 3; + + uint64_t dst = 0; + + #pragma unroll + for (int i = 0; i < 8; i++) + { + int source_row = q_perm[q_perm_idx++]; + + int w2_row = source_row >> 3; + int w2_subrow = source_row & 0x07; + int w2_row_shift = w2_subrow << 2; + int wnew2_row_shift = i << 2; + + uint64_t src = w2[w2_row * w2_stride + w2_column]; + src >>= w2_row_shift; + src &= 0x0000000f0000000f; + src <<= wnew2_row_shift; + dst |= src; + } + + w_new2[w_new2_row * w2_stride + w2_column] = dst; +} + +bool QMatrix::make_sequential(const uint32_t* cpu_g_idx) +{ + uint32_t* cuda_new_qweight = NULL; + cudaError_t err = cudaMalloc(&cuda_new_qweight, height / 8 * width * sizeof(uint32_t)); + if (err != cudaSuccess) { + cudaError_t cuda_status = cudaGetLastError(); // Clear error + return false; + } + + uint32_t* cpu_g_idx_map = (uint32_t*) calloc(groups, sizeof(uint32_t)); + uint32_t* cpu_x_map = (uint32_t*) malloc(height * sizeof(uint32_t)); + uint32_t* cpu_x_map_inv = (uint32_t*) malloc(height * sizeof(uint32_t)); + + // Group histogram + + for (int i = 0; i < height; i++) cpu_g_idx_map[cpu_g_idx[i]]++; + + // Group map + + for (int i = 0, acc = 0; i < groups; i++) + { + short tmp = cpu_g_idx_map[i]; + cpu_g_idx_map[i] = acc; + acc += tmp; + } + + // X map (inverse) + + for (int row = 0; row < height; row++) + { + uint32_t target_group = cpu_g_idx[row]; + uint32_t target_row = cpu_g_idx_map[target_group]; + cpu_g_idx_map[target_group]++; + cpu_x_map_inv[row] = target_row; + } + + // X map + + for (int row = 0; row < height; row++) cpu_x_map[cpu_x_map_inv[row]] = row; + + // Reduce to uint16_t + + uint16_t* cpu_x_map16 = (uint16_t*)cpu_x_map; + uint16_t* cpu_x_map_inv16 = (uint16_t*)cpu_x_map_inv; + for (int row = 0; row < height; row++) cpu_x_map16[row] = (uint16_t) cpu_x_map[row]; + for (int row = 0; row < height; row++) cpu_x_map_inv16[row] = (uint16_t) cpu_x_map_inv[row]; + + // Move to CUDA + + cudaMemcpyAsync(cuda_q_perm, cpu_x_map16, height * sizeof(uint16_t), cudaMemcpyHostToDevice); + cudaMemcpyAsync(cuda_q_invperm, cpu_x_map_inv16, height * sizeof(uint16_t), cudaMemcpyHostToDevice); + + // Rearrange rows in w + + dim3 blockDim, gridDim; + blockDim.x = THREADS_X; + blockDim.y = 1; + gridDim.x = DIVIDE(width, THREADS_X); + gridDim.y = height / 8; + + make_sequential_kernel<<>> + ( + cuda_q_weight, + cuda_new_qweight, + cuda_q_perm, + height / 8, + width + ); + + // Replace qweights + + cudaMemcpyAsync(cuda_q_weight, cuda_new_qweight, height / 8 * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice); + + // Cleanup + + cudaDeviceSynchronize(); + + cudaFree(cuda_new_qweight); + free(cpu_g_idx_map); + free(cpu_x_map); + free(cpu_x_map_inv); + + return true; +} diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cuh new file mode 100644 index 00000000..dda83a4f --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cuh @@ -0,0 +1,73 @@ +#ifndef _q_matrix_cuh +#define _q_matrix_cuh + +#include +#include +#include +#include + +#define MAX_SUPERGROUPS 16 + +class QMatrix +{ +public: + + int device; + bool is_gptq; + + int height; + int width; + int groups; + int groupsize; + + int rows_8; + int rows_6; + int rows_5; + int rows_4; + int rows_3; + int rows_2; + + uint32_t* cuda_q_weight = NULL; + uint16_t* cuda_q_perm = NULL; + uint16_t* cuda_q_invperm = NULL; + uint32_t* cuda_q_scale = NULL; + half* cuda_q_scale_max = NULL; + uint16_t* cuda_q_groups = NULL; + uint32_t* cuda_gptq_qzeros = NULL; + half* cuda_gptq_scales = NULL; + + half* temp_dq; + + bool failed; + + QMatrix + ( + const int _device, + const int _height, + const int _width, + const int _groups, + + uint32_t* _q_weight, + uint16_t* _q_perm, + uint16_t* _q_invperm, + uint32_t* _q_scale, + half* _q_scale_max, + uint16_t* _q_groups, + + uint32_t* _gptq_qzeros, + half* _gptq_scales, + uint32_t* _gptq_g_idx, + + half* _temp_dq + ); + + ~QMatrix(); + + void reconstruct(half* out); + bool make_sequential(const uint32_t* cpu_g_idx); + +private: + +}; + +#endif diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_2.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_2.cuh new file mode 100644 index 00000000..3beaeefa --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_2.cuh @@ -0,0 +1,103 @@ +#ifndef _qdq_2_cuh +#define _qdq_2_cuh + +#include "qdq_util.cuh" +#include "../../config.h" + +#if QMODE_2BIT == 1 + +// Permutation: +// +// ffddbb99 77553311 eeccaa88 66442200 + +__forceinline__ __device__ void shuffle_2bit_16 +( + uint32_t* q, + int stride +) +{ + uint32_t qa = q[0]; + uint32_t qb = 0; + + #pragma unroll + for (int i = 0; i < 8; i++) + { + uint32_t qa0 = qa & 0x03; + uint32_t qa1 = (qa & 0x0c) >> 2; + qa >>= 4; + qb |= (qa1 << (i * 2 + 16)); + qb |= (qa0 << (i * 2)); + } + q[0] = qb; +} + +__forceinline__ __device__ void dequant_2bit_16 +( + const uint32_t q_0, + half2 (&dq)[8], + int stride +) +{ + const uint32_t c0 = 0x64006400; + const half y4_ = __float2half_rn(1.0f / 4.0f); + const half y16_ = __float2half_rn(1.0f / 16.0f); + const half y64_ = __float2half_rn(1.0f / 64.0f); + const half2 y4 = __halves2half2(y4_, y4_); + const half2 y16 = __halves2half2(y16_, y16_); + const half2 y64 = __halves2half2(y64_, y64_); + const half z1_ = __float2half_rn(-1024.0f - 2.0f); + const half z4_ = __float2half_rn(-1024.0f / 4.0f - 2.0f); + const half z16_ = __float2half_rn(-1024.0f / 16.0f - 2.0f); + const half z64_ = __float2half_rn(-1024.0f / 64.0f - 2.0f); + const half2 z1 = __halves2half2(z1_, z1_); + const half2 z4 = __halves2half2(z4_, z4_); + const half2 z16 = __halves2half2(z16_, z16_); + const half2 z64 = __halves2half2(z64_, z64_); + + uint32_t qa = q_0; + half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024 + half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024 + half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024 + qa >>= 8; + half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024 + half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024 + half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024 + half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024 + + dq[0] = __hadd2(q0.as_half2, z1); + dq[1] = __hfma2(q1.as_half2, y4, z4); + dq[2] = __hfma2(q2.as_half2, y16, z16); + dq[3] = __hfma2(q3.as_half2, y64, z64); + dq[4] = __hadd2(q4.as_half2, z1); + dq[5] = __hfma2(q5.as_half2, y4, z4); + dq[6] = __hfma2(q6.as_half2, y16, z16); + dq[7] = __hfma2(q7.as_half2, y64, z64); +} + +#else + +__forceinline__ __device__ void shuffle_2bit_16 +( + uint32_t* q, + int stride +) +{ +} + +__forceinline__ __device__ void dequant_2bit_16 +( + const uint32_t q_0, + half2 (&dq)[8], + int stride +) +{ + half dqh[16]; + for (int i = 0; i < 16; i++) dqh[i] = dq_ns(exb(q_0, i * 2, 0x03), 2); + + for (int i = 0; i < 8; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]); +} + +#endif + +#endif \ No newline at end of file diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_3.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_3.cuh new file mode 100644 index 00000000..10117376 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_3.cuh @@ -0,0 +1,169 @@ +#ifndef _qdq_3_cuh +#define _qdq_3_cuh + +#include "qdq_util.cuh" +#include "../../config.h" + +#if QMODE_3BIT == 1 + +// Permutation: +// +// v9997775 55333111 u8886664 44222000 (u, v lsb) +// vjjjhhhf ffdddbbb uiiiggge eecccaaa +// vtttrrrp ppnnnlll usssqqqo oommmkkk + +__forceinline__ __device__ void shuffle_3bit_32 +( + uint32_t* q, + int stride +) +{ + uint32_t qa = q[0 * stride]; + uint32_t qb = q[1 * stride]; + uint32_t qc = q[2 * stride]; + + // qa: aa999888 77766655 54443332 22111000 + // qb: lkkkjjji iihhhggg fffeeedd dcccbbba + // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll + + uint32_t qd = qc >> 26; + qc <<= 4; + qc |= qb >> 28; + qb <<= 2; + qb |= qa >> 30; + + // qa: ..999888 77766655 54443332 22111000 + // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa + // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk + // qd: vvvuuu + + uint32_t za = 0; + uint32_t zb = 0; + uint32_t zc = 0; + + for (int i = 0; i < 5; i++) { uint32_t t0 = qa & 0x07; uint32_t t1 = (qa & 0x38) >> 3; qa >>= 6; za |= (t0 << (i * 3)); za |= (t1 << (i * 3 + 16)); } + for (int i = 0; i < 5; i++) { uint32_t t0 = qb & 0x07; uint32_t t1 = (qb & 0x38) >> 3; qb >>= 6; zb |= (t0 << (i * 3)); zb |= (t1 << (i * 3 + 16)); } + for (int i = 0; i < 5; i++) { uint32_t t0 = qc & 0x07; uint32_t t1 = (qc & 0x38) >> 3; qc >>= 6; zc |= (t0 << (i * 3)); zc |= (t1 << (i * 3 + 16)); } + + // za: 9997775 55333111 8886664 44222000 + // zb: jjjhhhf ffdddbbb iiiggge eecccaaa + // zc: tttrrrp ppnnnlll sssqqqo oommmkkk + // qd: vvvuuu + + za |= ((qd & 0x01) >> 0) << 15; + zb |= ((qd & 0x02) >> 1) << 15; + zc |= ((qd & 0x04) >> 2) << 15; + za |= ((qd & 0x08) >> 3) << 31; + zb |= ((qd & 0x10) >> 4) << 31; + zc |= ((qd & 0x20) >> 5) << 31; + + // za: v9997775 55333111 u8886664 44222000 (u, v lsb) + // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa + // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk + + q[0 * stride] = za; + q[1 * stride] = zb; + q[2 * stride] = zc; +} + +__forceinline__ __device__ void dequant_3bit_32 +( + const uint32_t q_0, + const uint32_t q_1, + const uint32_t q_2, + half2 (&dq)[16], + int stride +) +{ + const uint32_t c0 = 0x64006400; + const half y8_ = __float2half_rn(1.0f / 8.0f); + const half y64_ = __float2half_rn(1.0f / 64.0f); + const half2 y8 = __halves2half2(y8_, y8_); + const half2 y64 = __halves2half2(y64_, y64_); + const half z1_ = __float2half_rn(-1024.0f - 4.0f); + const half z8_ = __float2half_rn(-1024.0f / 8.0f - 4.0f); + const half z64_ = __float2half_rn(-1024.0f / 64.0f - 4.0f); + const half2 z1 = __halves2half2(z1_, z1_); + const half2 z8 = __halves2half2(z8_, z8_); + const half2 z64 = __halves2half2(z64_, z64_); + + uint32_t qa = q_0; + uint32_t qb = q_1; + uint32_t qc = q_2; + + half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024 + qa >>= 6; + half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024 + half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024 + half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024 + qa >>= 9; + qa &= 0x00010001; + half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024 + half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024 + qb >>= 6; + half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024 + half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024 + half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024 + qb >>= 8; + qb &= 0x00020002; + half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024 + half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024 + qc >>= 6; + half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024 + half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024 + half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024 + qc >>= 7; + qc &= 0x00040004; + half2_uint32 q15((qa | qb | qc) | c0); + + dq[ 0] = __hadd2( q0.as_half2, z1); + dq[ 1] = __hfma2( q1.as_half2, y8, z8); + dq[ 2] = __hadd2( q2.as_half2, z1); + dq[ 3] = __hfma2( q3.as_half2, y8, z8); + dq[ 4] = __hfma2( q4.as_half2, y64, z64); + dq[ 5] = __hadd2( q5.as_half2, z1); + dq[ 6] = __hfma2( q6.as_half2, y8, z8); + dq[ 7] = __hadd2( q7.as_half2, z1); + dq[ 8] = __hfma2( q8.as_half2, y8, z8); + dq[ 9] = __hfma2( q9.as_half2, y64, z64); + dq[10] = __hadd2(q10.as_half2, z1); + dq[11] = __hfma2(q11.as_half2, y8, z8); + dq[12] = __hadd2(q12.as_half2, z1); + dq[13] = __hfma2(q13.as_half2, y8, z8); + dq[14] = __hfma2(q14.as_half2, y64, z64); + dq[15] = __hadd2(q15.as_half2, z1); +} + +#else + +__forceinline__ __device__ void shuffle_3bit_32 +( + uint32_t* q, + int stride +) +{ +} + +__forceinline__ __device__ void dequant_3bit_32 +( + const uint32_t q_0, + const uint32_t q_1, + const uint32_t q_2, + half2 (&dq)[16], + int stride +) +{ + half dqh[32]; + for (int i = 0; i < 10; i++) dqh[ i] = dq_ns(exb( q_0, i * 3 , 0x07), 4); + dqh[10 ] = dq_ns(exb(q_1, q_0, 30, 0x07), 4); + for (int i = 0; i < 10; i++) dqh[11 + i] = dq_ns(exb( q_1, i * 3 + 1, 0x07), 4); + dqh[21 ] = dq_ns(exb(q_2, q_1, 31, 0x07), 4); + for (int i = 0; i < 10; i++) dqh[22 + i] = dq_ns(exb( q_2, i * 3 + 2, 0x07), 4); + + for (int i = 0; i < 16; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]); +} + +#endif + +#endif diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_4.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_4.cuh new file mode 100644 index 00000000..5fb070d0 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_4.cuh @@ -0,0 +1,227 @@ +#ifndef _qdq_4_cuh +#define _qdq_4_cuh + +#include "qdq_util.cuh" +#include "../../config.h" + +#if QMODE_4BIT == 1 + +// Permutation: +// +// 77775555 33331111 66664444 22220000 + +__forceinline__ __device__ void shuffle_4bit_8 +( + uint32_t* q, + int stride +) +{ + uint32_t qa = q[0]; + uint32_t qb = 0; + + #pragma unroll + for (int i = 0; i < 4; i++) + { + uint32_t qa0 = qa & 0x0f; + uint32_t qa1 = (qa & 0xf0) >> 4; + qa >>= 8; + qb |= (qa1 << (i * 4 + 16)); + qb |= (qa0 << (i * 4)); + } + q[0] = qb; +} + +__forceinline__ __device__ void dequant_4bit_8 +( + const uint32_t q_0, + half2 (&dq)[4], + int stride +) +{ + const uint32_t c0 = 0x64006400; + const half y16_ = __float2half_rn(1.0f / 16.0f); + const half2 y16 = __halves2half2(y16_, y16_); + const half z1_ = __float2half_rn(-1024.0f - 8.0f); + const half z16_ = __float2half_rn(-1024.0f / 16.0f - 8.0f); + const half2 z1 = __halves2half2(z1_, z1_); + const half2 z16 = __halves2half2(z16_, z16_); + + uint32_t qa = q_0; + half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024 + qa >>= 8; + half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024 + half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024 + + dq[0] = __hadd2(q0.as_half2, z1); + dq[1] = __hfma2(q1.as_half2, y16, z16); + dq[2] = __hadd2(q2.as_half2, z1); + dq[3] = __hfma2(q3.as_half2, y16, z16); +} + +__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale +( + const uint32_t zero, + const half scale, + half2 (&z1z16)[2], + half2 (&y1y16)[2] +) +{ + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero); + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + + half2 scale2 = __half2half2(scale); + + z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half)); + z1z16[1] = __hmul2(scale2, __half2half2(z16)); + + const half y1 = __float2half_rn(1.0f); + const half y16 = __float2half_rn(1.0f / 16.0f); + + y1y16[0] = __hmul2(scale2, __half2half2(y1)); + y1y16[1] = __hmul2(scale2, __half2half2(y16)); +} + +__forceinline__ __device__ void dequant_4bit_8_prep_zero +( + const uint32_t zero, + half2(&z1z16)[2], + half2(&y1y16)[2] +) +{ + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero); + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + + z1z16[0] = __half2half2(z1.as_half); + z1z16[1] = __half2half2(z16); + + const half y1 = __float2half_rn(1.0f); + const half y16 = __float2half_rn(1.0f / 16.0f); + + y1y16[0] = __half2half2(y1); + y1y16[1] = __half2half2(y16); +} + + +__forceinline__ __device__ void dequant_4bit_8_gptq +( + const uint32_t q_0, + half2 (&dq)[4], + half2 (&z1z16)[2], + half2 (&y1y16)[2], + int stride, + bool scaled +) +{ + const uint32_t c0 = 0x64006400; + + uint32_t qa = q_0; + half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0] + 1024, q[1] + 1024 ) + half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 ) + qa >>= 8; + half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4] + 1024, q[5] + 1024 ) + half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 ) + + if (scaled) + { + dq[0] = __hfma2(q0.as_half2, y1y16[0], z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s) + dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s) + dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]); + dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); + } + else + { + dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z ) + dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] - z, q[3] - z ) + dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z ) + dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); // half2( q[6] - z, q[7] - z ) + } +} + +#else + +__forceinline__ __device__ void shuffle_4bit_8 +( + uint32_t* q, + int stride +) +{ +} + +__forceinline__ __device__ void dequant_4bit_8 +( + const uint32_t q_0, + half2 (&dq)[4], + int stride +) +{ + half dqh[8]; + for (int i = 0; i < 8; i++) dqh[i] = dq_ns(exb(q_0, i * 4, 0x0f), 8); + + for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]); +} + +__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale +( + const uint32_t zero, + const half scale, + half2 (&z1)[2], + half2 (&y1)[2] +) +{ + half z = __int2half_rn(-((int)zero)); + z = __hmul(z, scale); + z1[0] = __half2half2(z); + y1[0] = __half2half2(scale); +} + +__forceinline__ __device__ void dequant_4bit_8_prep_zero +( + const uint32_t zero, + half2(&z1)[2], + half2(&y1)[2] +) +{ + half z = __int2half_rn(-((int)zero)); + z1[0] = __half2half2(z); +} + +__forceinline__ __device__ void dequant_4bit_8_gptq +( + const uint32_t q_0, + half2 (&dq)[4], + half2 (&z1)[2], + half2 (&y1)[2], + int stride, + bool scaled +) +{ + half2 dqh2[8]; + + uint32_t qa = q_0; + for (int i = 0; i < 4; i++) + { + half d0 = __int2half_rn(qa & 0x0f); qa >>= 4; + half d1 = __int2half_rn(qa & 0x0f); qa >>= 4; + dqh2[i] = __halves2half2(d0, d1); + } + + if (scaled) + { + dq[0] = __hfma2(dqh2[0], y1[0], z1[0]); + dq[1] = __hfma2(dqh2[1], y1[0], z1[0]); + dq[2] = __hfma2(dqh2[2], y1[0], z1[0]); + dq[3] = __hfma2(dqh2[3], y1[0], z1[0]); + } + else + { + dq[0] = __hadd2(dqh2[0], z1[0]); + dq[1] = __hadd2(dqh2[1], z1[0]); + dq[2] = __hadd2(dqh2[2], z1[0]); + dq[3] = __hadd2(dqh2[3], z1[0]); + } +} + +#endif + +#endif \ No newline at end of file diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_5.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_5.cuh new file mode 100644 index 00000000..454e4b93 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_5.cuh @@ -0,0 +1,207 @@ +#ifndef _qdq_5_cuh +#define _qdq_5_cuh + +#include "qdq_util.cuh" +#include "../../config.h" + +#if QMODE_5BIT == 1 + +// Permutation: +// +// v5555533 33311111 u4444422 22200000 (u, v lsb) +// vbbbbb99 99977777 uaaaaa88 88866666 +// vhhhhhff fffddddd ugggggee eeeccccc +// vnnnnnll llljjjjj ummmmmkk kkkiiiii +// vtttttrr rrrppppp usssssqq qqqooooo + +__forceinline__ __device__ void shuffle_5bit_32 +( + uint32_t* q, + int stride +) +{ + uint32_t qa = q[0 * stride]; + uint32_t qb = q[1 * stride]; + uint32_t qc = q[2 * stride]; + uint32_t qd = q[3 * stride]; + uint32_t qe = q[4 * stride]; + + // qa: 66555554 44443333 32222211 11100000 + // qb: ccccbbbb baaaaa99 99988888 77777666 + // qc: jiiiiihh hhhggggg fffffeee eedddddc + // qd: pppooooo nnnnnmmm mmlllllk kkkkjjjj + // qe: vvvvvuuu uuttttts ssssrrrr rqqqqqpp + + uint32_t qf = qe >> 22; + qe <<= 8; + qe |= qd >> 24; + qd <<= 6; + qd |= qc >> 26; + qc <<= 4; + qc |= qb >> 28; + qb <<= 2; + qb |= qa >> 30; + + // qa: 555554 44443333 32222211 11100000 + // qb: bbbbba aaaa9999 98888877 77766666 + // qc: hhhhhg ggggffff feeeeedd dddccccc + // qd: nnnnnm mmmmllll lkkkkkjj jjjiiiii + // qe: ttttts ssssrrrr rqqqqqpp pppooooo + // qf: vv vvvuuuuu + + uint32_t za = 0; + uint32_t zb = 0; + uint32_t zc = 0; + uint32_t zd = 0; + uint32_t ze = 0; + + for (int i = 0; i < 3; i++) { uint32_t t0 = qa & 0x1f; uint32_t t1 = (qa & 0x3e0) >> 5; qa >>= 10; za |= (t0 << (i * 5)); za |= (t1 << (i * 5 + 16)); } + for (int i = 0; i < 3; i++) { uint32_t t0 = qb & 0x1f; uint32_t t1 = (qb & 0x3e0) >> 5; qb >>= 10; zb |= (t0 << (i * 5)); zb |= (t1 << (i * 5 + 16)); } + for (int i = 0; i < 3; i++) { uint32_t t0 = qc & 0x1f; uint32_t t1 = (qc & 0x3e0) >> 5; qc >>= 10; zc |= (t0 << (i * 5)); zc |= (t1 << (i * 5 + 16)); } + for (int i = 0; i < 3; i++) { uint32_t t0 = qd & 0x1f; uint32_t t1 = (qd & 0x3e0) >> 5; qd >>= 10; zd |= (t0 << (i * 5)); zd |= (t1 << (i * 5 + 16)); } + for (int i = 0; i < 3; i++) { uint32_t t0 = qe & 0x1f; uint32_t t1 = (qe & 0x3e0) >> 5; qe >>= 10; ze |= (t0 << (i * 5)); ze |= (t1 << (i * 5 + 16)); } + + // za: 5555533 33311111 4444422 22200000 + // zb: bbbbb99 99977777 aaaaa88 88866666 + // zc: hhhhhff fffddddd gggggee eeeccccc + // zd: nnnnnll llljjjjj mmmmmkk kkkiiiii + // ze: tttttrr rrrppppp sssssqq qqqooooo + // qf: vv vvvuuuuu + + za |= ((qf & 0x001) >> 0) << 15; + zb |= ((qf & 0x002) >> 1) << 15; + zc |= ((qf & 0x004) >> 2) << 15; + zd |= ((qf & 0x008) >> 3) << 15; + ze |= ((qf & 0x010) >> 4) << 15; + za |= ((qf & 0x020) >> 5) << 31; + zb |= ((qf & 0x040) >> 6) << 31; + zc |= ((qf & 0x080) >> 7) << 31; + zd |= ((qf & 0x100) >> 8) << 31; + ze |= ((qf & 0x200) >> 9) << 31; + + // za: v5555533 33311111 u4444422 22200000 (u, v lsb) + // zb: vbbbbb99 99977777 uaaaaa88 88866666 + // zc: vhhhhhff fffddddd ugggggee eeeccccc + // zd: vnnnnnll llljjjjj ummmmmkk kkkiiiii + // ze: vtttttrr rrrppppp usssssqq qqqooooo + + q[0 * stride] = za; + q[1 * stride] = zb; + q[2 * stride] = zc; + q[3 * stride] = zd; + q[4 * stride] = ze; +} + +__forceinline__ __device__ void dequant_5bit_32 +( + const uint32_t q_0, + const uint32_t q_1, + const uint32_t q_2, + const uint32_t q_3, + const uint32_t q_4, + half2 (&dq)[16], + int stride +) +{ + const uint32_t c0 = 0x64006400; + const half y32_ = __float2half_rn(1.0f / 32.0f); + const half2 y32 = __halves2half2(y32_, y32_); + const half z1_ = __float2half_rn(-1024.0f - 16.0f); + const half z32_ = __float2half_rn(-1024.0f / 32.0f - 16.0f); + const half2 z1 = __halves2half2(z1_, z1_); + const half2 z32 = __halves2half2(z32_, z32_); + + uint32_t qa = q_0; + uint32_t qb = q_1; + uint32_t qc = q_2; + uint32_t qd = q_3; + uint32_t qe = q_4; + + half2_uint32 q0 ((qa & 0x001f001f) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1 ((qa & 0x03e003e0) | c0); // half2(q[ 2], q[ 3]) * 32 + 1024 + qa >>= 10; + half2_uint32 q2 ((qa & 0x001f001f) | c0); // half2(q[ 4], q[ 5]) + 1024 + qa >>= 5; + qa &= 0x00010001; + half2_uint32 q3 ((qb & 0x001f001f) | c0); // half2(q[ 6], q[ 7]) + 1024 + half2_uint32 q4 ((qb & 0x03e003e0) | c0); // half2(q[ 8], q[ 9]) * 32 + 1024 + qb >>= 10; + half2_uint32 q5 ((qb & 0x001f001f) | c0); // half2(q[10], q[11]) + 1024 + qb >>= 4; + qb &= 0x00020002; + half2_uint32 q6 ((qc & 0x001f001f) | c0); // half2(q[12], q[13]) + 1024 + half2_uint32 q7 ((qc & 0x03e003e0) | c0); // half2(q[14], q[15]) * 32 + 1024 + qc >>= 10; + half2_uint32 q8 ((qc & 0x001f001f) | c0); // half2(q[16], q[17]) + 1024 + qc >>= 3; + qc &= 0x00040004; + half2_uint32 q9 ((qd & 0x001f001f) | c0); // half2(q[18], q[19]) + 1024 + half2_uint32 q10((qd & 0x03e003e0) | c0); // half2(q[20], q[21]) * 32 + 1024 + qd >>= 10; + half2_uint32 q11((qd & 0x001f001f) | c0); // half2(q[22], q[23]) + 1024 + qd >>= 2; + qd &= 0x00080008; + half2_uint32 q12((qe & 0x001f001f) | c0); // half2(q[24], q[25]) + 1024 + half2_uint32 q13((qe & 0x03e003e0) | c0); // half2(q[26], q[27]) * 32 + 1024 + qe >>= 10; + half2_uint32 q14((qe & 0x001f001f) | c0); // half2(q[28], q[29]) + 1024 + qe >>= 1; + qe &= 0x00100010; + half2_uint32 q15((qa | qb | qc | qd | qe) | c0); + + dq[ 0] = __hadd2( q0.as_half2, z1); + dq[ 1] = __hfma2( q1.as_half2, y32, z32); + dq[ 2] = __hadd2( q2.as_half2, z1); + dq[ 3] = __hadd2( q3.as_half2, z1); + dq[ 4] = __hfma2( q4.as_half2, y32, z32); + dq[ 5] = __hadd2( q5.as_half2, z1); + dq[ 6] = __hadd2( q6.as_half2, z1); + dq[ 7] = __hfma2( q7.as_half2, y32, z32); + dq[ 8] = __hadd2( q8.as_half2, z1); + dq[ 9] = __hadd2( q9.as_half2, z1); + dq[10] = __hfma2(q10.as_half2, y32, z32); + dq[11] = __hadd2(q11.as_half2, z1); + dq[12] = __hadd2(q12.as_half2, z1); + dq[13] = __hfma2(q13.as_half2, y32, z32); + dq[14] = __hadd2(q14.as_half2, z1); + dq[15] = __hadd2(q15.as_half2, z1); +} + +#else + +__forceinline__ __device__ void shuffle_5bit_32 +( + uint32_t* q, + int stride +) +{ +} + +__forceinline__ __device__ void dequant_5bit_32 +( + const uint32_t q_0, + const uint32_t q_1, + const uint32_t q_2, + const uint32_t q_3, + const uint32_t q_4, + half2 (&dq)[16], + int stride +) +{ + half dqh[32]; + for (int i = 0; i < 6; i++) dqh[ i] = dq_ns(exb( q_0, i * 5 , 0x1f), 16); + dqh[ 6 ] = dq_ns(exb(q_1, q_0, 30, 0x1f), 16); + for (int i = 0; i < 5; i++) dqh[ 7 + i] = dq_ns(exb( q_1, i * 5 + 3, 0x1f), 16); + dqh[12 ] = dq_ns(exb(q_2, q_1, 28, 0x1f), 16); + for (int i = 0; i < 6; i++) dqh[13 + i] = dq_ns(exb( q_2, i * 5 + 1, 0x1f), 16); + dqh[19 ] = dq_ns(exb(q_3, q_2, 31, 0x1f), 16); + for (int i = 0; i < 5; i++) dqh[20 + i] = dq_ns(exb( q_3, i * 5 + 4, 0x1f), 16); + dqh[25 ] = dq_ns(exb(q_4, q_3, 29, 0x1f), 16); + for (int i = 0; i < 6; i++) dqh[26 + i] = dq_ns(exb( q_4, i * 5 + 2, 0x1f), 16); + + for (int i = 0; i < 16; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]); +} + +#endif + +#endif \ No newline at end of file diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_6.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_6.cuh new file mode 100644 index 00000000..c2eb8cfb --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_6.cuh @@ -0,0 +1,44 @@ +#ifndef _qdq_6_cuh +#define _qdq_6_cuh + +#include "qdq_util.cuh" +#include "../../config.h" + +#if QMODE_6BIT == 1 + + // Not implemented + +#else + +__forceinline__ __device__ void shuffle_6bit_16 +( + uint32_t* q, + int stride +) +{ +} + +__forceinline__ __device__ void dequant_6bit_16 +( + const uint32_t q_0, + const uint32_t q_1, + const uint32_t q_2, + half2 (&dq)[8], + int stride +) +{ + half dqh[16]; + for (int i = 0; i < 5; i++) dqh[ i] = dq_ns(exb( q_0, i * 6 , 0x3f), 32); + dqh[ 5 ] = dq_ns(exb(q_1, q_0, 30, 0x3f), 32); + for (int i = 0; i < 4; i++) dqh[ 6 + i] = dq_ns(exb( q_1, i * 6 + 4, 0x3f), 32); + dqh[10 ] = dq_ns(exb(q_2, q_1, 28, 0x3f), 32); + for (int i = 0; i < 5; i++) dqh[11 + i] = dq_ns(exb( q_2, i * 6 + 2, 0x3f), 32); + + for (int i = 0; i < 8; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]); +} + +#endif + +#endif + + diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_8.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_8.cuh new file mode 100644 index 00000000..e2409efa --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_8.cuh @@ -0,0 +1,38 @@ +#ifndef _qdq_8_cuh +#define _qdq_8_cuh + +#include "qdq_util.cuh" +#include "../../config.h" + +#if QMODE_8BIT == 1 + + // Not implemented + +#else + +__forceinline__ __device__ void shuffle_8bit_4 +( + uint32_t* q, + int stride +) +{ +} + +__forceinline__ __device__ void dequant_8bit_8 +( + const uint32_t q_0, + const uint32_t q_1, + half2 (&dq)[4], + int stride +) +{ + half dqh[8]; + for (int i = 0; i < 4; i++) dqh[i ] = dq_ns(exb(q_0, i * 8, 0xff), 128); + for (int i = 0; i < 4; i++) dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), 128); + + for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]); +} + +#endif + +#endif \ No newline at end of file diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_util.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_util.cuh new file mode 100644 index 00000000..71657191 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_util.cuh @@ -0,0 +1,51 @@ +#ifndef _qdq_util_cuh +#define _qdq_util_cuh + +union half2_uint32 +{ + uint32_t as_uint32; + half2 as_half2; + __device__ half2_uint32(uint32_t val) : as_uint32(val) {} + __device__ half2_uint32(half2 val) : as_half2(val) {} +}; + +union half_uint16 +{ + uint16_t as_uint16; + half as_half; + __device__ half_uint16(uint16_t val) : as_uint16(val) {} + __device__ half_uint16(half val) : as_half(val) {} +}; + +// Max_scale premultiplied by 1/256 + +__forceinline__ __device__ half dq_scale(const int qs, const half max_scale) +{ + int qs_i = qs + 1; + half qs_h = __int2half_rn(qs_i * qs_i); + qs_h = __hmul(qs_h, max_scale); + return qs_h; +} + +__forceinline__ __device__ half dq(const int q, const int qzero, const half scale) +{ + return __hmul(__int2half_rn(q - qzero), scale); +} + +__forceinline__ __device__ half dq_ns(const int q, const int qzero) +{ + //return __hsub(__int2half_rn(q), __int2half_rn(qzero)); + return __int2half_rn(q - qzero); +} + +__forceinline__ __device__ int exb(const uint32_t q, const int shift, const int mask) +{ + return (int)((q >> shift) & mask); +} + +__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0, const int shift, const int mask) +{ + return (int)(__funnelshift_rc(q0, q1, shift) & mask); +} + +#endif diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/util.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/util.cuh new file mode 100644 index 00000000..06a58d18 --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/util.cuh @@ -0,0 +1,42 @@ + +#define DIVIDE(x, size) (((x) + (size) - 1) / (size)) + +#define DBGS(__x) printf("%s\n", __x) +#define DBGI(__x) printf("%s: %i\n", #__x, __x) +#define DBGI2(__x, __y) printf("%s, %s: %i, %i\n", #__x, #__y, __x, __y) +#define DBGI3(__x, __y, __z) printf("%s, %s, %s: %i, %i, %i\n", #__x, #__y, #__z, __x, __y, __z) +#define DBGX(__x) printf("%s: %x\n", #__x, __x) +#define DBGX2(__x, __y) printf("%s, %s: %x, %x\n", #__x, #__y, __x, __y) +#define DBGX3(__x, __y, __z) printf("%s, %s, %s: %x, %x, %x\n", #__x, #__y, #__z, __x, __y, __z) +#define DBGF(__x) printf("%s: %f\n", #__x, __x) +#define DBGF2(__x, __y) printf("%s, %s: %f, %f\n", #__x, #__y, __x, __y) +#define DBGF3(__x, __y, __z) printf("%s, %s, %s: %f, %f, %f\n", #__x, #__y, #__z, __x, __y, __z) +#define DBGH(__x) printf("%s: %f\n", #__x, __half2float(__x)) +#define DBGH2(__x, __y) printf("%s, %s: %f, %f\n", #__x, #__y, __half2float(__x), __half2float(__y)) +#define DBGH3(__x, __y, __z) printf("%s, %s, %s: %f, %f, %f\n", #__x, #__y, #__z, __half2float(__x), __half2float(__y), __half2float(__z)) + +#define DBGIH(__x, __y) printf("%s, %s: %i, %f\n", #__x, #__y, __x, __half2float(__y)) +#define DBGIH2(__x, __y, __z) printf("%s, %s, %s: %i, %f, %f\n", #__x, #__y, #__z, __x, __half2float(__y), __half2float(__z)) + +__forceinline__ __device__ half dq_scale_(const int qs, const half max_scale) +{ + half qs_h = __hmul(__int2half_rn(qs + 1), __float2half_rn(1.0f / 16.0f)); + qs_h = __hmul(qs_h, qs_h); + qs_h = __hmul(qs_h, max_scale); + return qs_h; +} + +__forceinline__ __device__ float clamp(float x, float a, float b) +{ + return fmaxf(a, fminf(b, x)); +} + +#define cuda_check(ans) { gpu_assert((ans), __FILE__, __LINE__); } +inline void gpu_assert(cudaError_t code, const char *file, int line, bool abort=true) +{ + if (code != cudaSuccess) + { + fprintf(stderr,"CUDA error: %s %s %d\n", cudaGetErrorString(code), file, line); + if (abort) exit(code); + } +} diff --git a/server/exllamav2_kernels/exllamav2_kernels/ext.cpp b/server/exllamav2_kernels/exllamav2_kernels/ext.cpp new file mode 100644 index 00000000..5e52e6ab --- /dev/null +++ b/server/exllamav2_kernels/exllamav2_kernels/ext.cpp @@ -0,0 +1,134 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" + +#include "cuda/q_matrix.cuh" +#include "cuda/q_gemm.cuh" + +#include "cpp/util.h" + +// Some decluttering macros + +#define TORCH_CHECK_DTYPE(__x, __dtype) TORCH_CHECK((__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype) +#define TORCH_CHECK_DTYPE_OPT(__x, __dtype) TORCH_CHECK((__x).device().is_meta() || (__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype) +#define TORCH_CHECK_SHAPES(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes") +#define TORCH_CHECK_SHAPES_OPT(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).device().is_meta() || (__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes") + + +// Quant matrix + +uintptr_t make_q_matrix +( + torch::Tensor q_weight, + torch::Tensor q_perm, + torch::Tensor q_invperm, + torch::Tensor q_scale, + torch::Tensor q_scale_max, + torch::Tensor q_groups, + torch::Tensor gptq_qzeros, + torch::Tensor gptq_scales, + torch::Tensor gptq_g_idx, + torch::Tensor temp_dq +) +{ + TORCH_CHECK_DTYPE(q_weight, kInt); + TORCH_CHECK_DTYPE_OPT(q_perm, kShort); + TORCH_CHECK_DTYPE_OPT(q_invperm, kShort); + TORCH_CHECK_DTYPE_OPT(q_scale, kInt); + TORCH_CHECK_DTYPE_OPT(q_scale_max, kHalf); + TORCH_CHECK_DTYPE_OPT(q_groups, kShort); + TORCH_CHECK_DTYPE_OPT(gptq_qzeros, kInt); + TORCH_CHECK_DTYPE_OPT(gptq_scales, kHalf); + TORCH_CHECK_DTYPE_OPT(gptq_g_idx, kInt); + + TORCH_CHECK_SHAPES(q_perm, 0, q_invperm, 0, 1); + + int device = q_weight.device().index(); + int width = q_weight.size(1); + int groups; + int height; + + if (!q_scale.device().is_meta()) + { + TORCH_CHECK_SHAPES(q_weight, 1, q_scale, 1, 8); + TORCH_CHECK_SHAPES(q_scale_max, 0, q_scale, 0, 1); + groups = q_scale.size(0); + height = q_invperm.size(0); + } + else + { + TORCH_CHECK_SHAPES(q_weight, 1, gptq_qzeros, 1, 8); + TORCH_CHECK_SHAPES(q_weight, 1, gptq_scales, 1, 1); + groups = gptq_qzeros.size(0); + height = q_weight.size(0) * 8; + } + + TORCH_CHECK(temp_dq.size(0) >= width * height, "Insufficient size of temp_dq buffer") + + QMatrix* m = new QMatrix + ( + device, + height, + width, + groups, + (uint32_t*) q_weight.data_ptr(), + q_perm.device().is_meta() ? NULL : (uint16_t*) q_perm.data_ptr(), + q_invperm.device().is_meta() ? NULL : (uint16_t*) q_invperm.data_ptr(), + q_scale.device().is_meta() ? NULL : (uint32_t*) q_scale.data_ptr(), + q_scale_max.device().is_meta() ? NULL : (half*) q_scale_max.data_ptr(), + q_groups.device().is_meta() ? NULL : (uint16_t*) q_groups.data_ptr(), + gptq_qzeros.device().is_meta() ? NULL : (uint32_t*) gptq_qzeros.data_ptr(), + gptq_scales.device().is_meta() ? NULL : (half*) gptq_scales.data_ptr(), + gptq_g_idx.device().is_meta() ? NULL : (uint32_t*) gptq_g_idx.data_ptr(), + (half*) temp_dq.data_ptr() + ); + + return reinterpret_cast (m); +} + +void gemm_half_q_half +( + torch::Tensor a, + uintptr_t b, + torch::Tensor c, + bool force_cuda +) +{ + QMatrix* qm = reinterpret_cast (b); + + TORCH_CHECK_DTYPE(a, kHalf); + TORCH_CHECK_DTYPE(c, kHalf); + TORCH_CHECK_SHAPES(a, 0, c, 0, 1); + TORCH_CHECK(qm->height == a.size(1), "a and b have incompatible shapes") + TORCH_CHECK(qm->width == c.size(1), "b and c have incompatible shapes") + + const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); + + gemm_half_q_half_cuda + ( + at::cuda::getCurrentCUDABlasHandle(), + (const half*) a.data_ptr(), + qm, + (half*) c.data_ptr(), + c.size(0), // m + c.size(1), // n + a.size(1), // k + true, + NULL, + force_cuda + ); +} + +// Bindings + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) +{ + m.def("make_q_matrix", &make_q_matrix, "make_q_matrix"); + m.def("gemm_half_q_half", &gemm_half_q_half, "gemm_half_q_half"); +} diff --git a/server/exllamav2_kernels/setup.py b/server/exllamav2_kernels/setup.py new file mode 100644 index 00000000..518db1df --- /dev/null +++ b/server/exllamav2_kernels/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +setup( + name="exllamav2_kernels", + ext_modules=[ + CUDAExtension( + name="exllamav2_kernels", + sources=[ + "exllamav2_kernels/ext.cpp", + "exllamav2_kernels/cuda/q_matrix.cu", + "exllamav2_kernels/cuda/q_gemm.cu", + ], + ) + ], + cmdclass={"build_ext": BuildExtension}, +) diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 75d2b159..fa831682 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -168,7 +168,7 @@ async def serve_inner( # When using GPTQ, Exllama kernels need some global kernels # For which we have the finale shapes only after the model has loaded # This will allocate those buffers. - from text_generation_server.utils.gptq.exllama import ( + from text_generation_server.utils.layers import ( create_exllama_buffers, set_device, ) diff --git a/server/text_generation_server/utils/gptq/exllamav2.py b/server/text_generation_server/utils/gptq/exllamav2.py new file mode 100644 index 00000000..1945338b --- /dev/null +++ b/server/text_generation_server/utils/gptq/exllamav2.py @@ -0,0 +1,191 @@ +# Adapted from turboderp exllama: https://github.com/turboderp/exllamav2 + +from logging import getLogger + +import torch +import torch.nn as nn +import math + +logger = getLogger(__name__) + +try: + from exllamav2_kernels import make_q_matrix, gemm_half_q_half +except ImportError: + logger.error('exllamav2_kernels not installed.') + raise + +# Dummy tensor to pass instead of g_idx since there is no way to pass "None" to a C++ extension +none_tensor = torch.empty((1, 1), device="meta") + +def ext_gemm_half_q_half(x, q_handle, q4_width, force_cuda): + """Matrix multiplication, returns x @ q4""" + output_shape = x.shape[:-1] + (q4_width,) + x = x.view(-1, x.shape[-1]) + output = torch.empty((x.shape[0], q4_width), dtype = torch.half, device = x.device) + gemm_half_q_half(x, q_handle, output, force_cuda) + return output.view(output_shape) + +def ext_make_q_matrix(w: dict, temp_dq, key: str = None): + """ + Create Q matrix + """ + # EXL2 + # won't work as the moment because the tensors are not the same. + if "q_weight" in w: + w["q_scale_max"] /= 256 + w["q_perm"] = w["q_perm"].short() + w["q_invperm"] = w["q_invperm"].short() + return make_q_matrix(w["q_weight"], + w["q_perm"], + w["q_invperm"], + w["q_scale"], + w["q_scale_max"], + w["q_groups"], + none_tensor, + none_tensor, + none_tensor, + temp_dq) + # GPTQ + elif "qweight" in w: + if w["scales"].dtype == torch.float: + w["scales"] = w["scales"].half() + + # GPTQ with g_idx (act_order) + if w.get("g_idx", None) is not None and not (w["g_idx"] == 0).all().item(): + w["q_perm"] = torch.empty((w["qweight"].shape[0] * 8,), dtype = torch.short, device = w["qweight"].device) + w["q_invperm"] = torch.empty_like(w["q_perm"]) + # make_q4 segfaults if g_idx is not on cpu in the act-order case. In the non act-order case, None needs to be passed for g_idx. + return make_q_matrix(w["qweight"], + w["q_perm"], + w["q_invperm"], + none_tensor, + none_tensor, + none_tensor, + w["qzeros"], + w["scales"], + w["g_idx"].cpu(), + temp_dq) + # GPTQ without g_idx + else: + return make_q_matrix(w["qweight"], + none_tensor, + none_tensor, + none_tensor, + none_tensor, + none_tensor, + w["qzeros"], + w["scales"], + none_tensor, + temp_dq) + +DEVICE = None +FIXED_BYTES = 0 +LAYERS = [] + + +def set_device(device): + global DEVICE + DEVICE = device + + +def create_exllama_buffers(): + global FIXED_BYTES, LAYERS, DEVICE + temp_dq = ExLlamaV2DeviceTensors(DEVICE, FIXED_BYTES) + + for layer in LAYERS: + layer.post_init(temp_dq) + + +class QuantLinear(nn.Module): + QUANT_TYPE = "exllamav2" + + """Linear layer implementation with per-group 4-bit quantization of the weights""" + + # def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs): + def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): + super().__init__() + if bits != 4: + raise ValueError( + f"Exllamav2 kernel supports only bits=4, requested bits={bits}. Something is wrong in the model initialization.") + self.q_handle = None + self.q_tensors = None + self.bits = bits + self.maxq = 2 ** self.bits - 1 + self.infeatures = qweight.shape[0] // self.bits * 32 + self.outfeatures = qweight.shape[1] + self.padding = - self.outfeatures % 32 + self.outfeatures = self.outfeatures + self.padding + + self.device = qweight.device + self.qweight = qweight + self.qzeros = qzeros + self.scales = scales + self.g_idx = g_idx + self.bias = bias if bias is not None else None + self.group_size = groupsize + + infeatures = self.infeatures + outfeatures = self.outfeatures + assert qweight.shape == (infeatures // 32 * self.bits, outfeatures) + assert infeatures % self.group_size == 0 + assert qzeros.shape == (infeatures // self.group_size, outfeatures // 32 * self.bits) + assert scales.shape == (infeatures // self.group_size, outfeatures) + assert g_idx.shape == (infeatures, ), f"{g_idx.shape}, {infeatures}" + + global FIXED_BYTES, LAYERS + FIXED_BYTES = max(FIXED_BYTES, self.scratch_space_fixed()) + LAYERS.append(self) + + def post_init(self, temp_dq): + assert self.qweight.device.type == "cuda" + assert self.qweight.device.index is not None + self.q_tensors = { + "qweight":self.qweight, + "qzeros":self.qzeros, + "scales":self.scales, + "g_idx":self.g_idx + } + temp_dq = temp_dq.get_scratch_slice(self.temp_dq_size()) + self.q_handle = ext_make_q_matrix( + self.q_tensors, temp_dq + ) + + def forward(self, x, force_cuda = False): + output = ext_gemm_half_q_half(x, self.q_handle, self.outfeatures, force_cuda) + + if self.bias is not None: + output.add_(self.bias) + return output + + def temp_dq_size(self): + return self.infeatures * self.outfeatures * 2 + 128 + + def temp_fwd_size(self, max_input_len, max_batch_size): + return self.outfeatures * max_input_len * max_batch_size * 4 + 128 + + def scratch_space_fixed(self, max_input_len=4096, max_batch_size=16): + return self.temp_dq_size() + self.temp_fwd_size(max_input_len, max_batch_size) + + +class ExLlamaV2DeviceTensors: + + device_idx: int + scratch_bytes: int + scratch_idx: int + scratch: torch.tensor = None + + def __init__(self, device, scratch_bytes): + self.device = device + self.scratch_bytes = scratch_bytes + + def prepare(self): + self.scratch = torch.empty((self.scratch_bytes // 2,), dtype = torch.half, device = self.device) + + def get_scratch_slice(self, size_bytes): + + if self.scratch is None: self.prepare() + + size_bytes = ((size_bytes + 127) // 128) * 128 + size_half = size_bytes // 2 + scratch_slice = self.scratch.narrow(0, 0, size_half) + return scratch_slice diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 7bb95dd2..e6a90116 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -31,15 +31,31 @@ major, _minor = torch.cuda.get_device_capability() except Exception: major = 1 + HAS_EXLLAMA = False CAN_EXLLAMA = major >= 8 +V2 = os.getenv("EXLLAMA_VERSION", "2") == "2" +if V2 and int(os.getenv("WORLD_SIZE", "1")) > 1: + logger.warning("Disabling exllama v2 and using v1 instead because there are issues when sharding") + V2 = False + if os.getenv("DISABLE_EXLLAMA") == "True": HAS_EXLLAMA = False elif CAN_EXLLAMA: try: - from text_generation_server.utils.gptq.exllama import Ex4bitLinear + if V2: + from text_generation_server.utils.gptq.exllamav2 import (QuantLinear as ExllamaQuantLinear, + create_exllama_buffers, + set_device, + ) + HAS_EXLLAMA = "2" + else: + from text_generation_server.utils.gptq.exllama import (Ex4bitLinear as ExllamaQuantLinear, + create_exllama_buffers, + set_device, + ) + HAS_EXLLAMA = "1" - HAS_EXLLAMA = True except ImportError: pass @@ -308,7 +324,7 @@ def get_linear(weight, bias, quantize): ) if use_exllama: - linear = Ex4bitLinear(qweight, qzeros, scales, g_idx, bias, bits, groupsize) + linear = ExllamaQuantLinear(qweight, qzeros, scales, g_idx, bias, bits, groupsize) else: linear = QuantLinear( qweight, diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 2f330d9c..f3344988 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -278,23 +278,13 @@ def get_multi_weights_row(self, prefix: str, quantize: str): ) use_exllama = False else: - logger.info("Using exllama kernels") + logger.info(f"Using exllama kernels v{HAS_EXLLAMA}") if use_exllama: - if groupsize >= 0: - # Exllama reorders the weights in advance and the activations on the fly, thus - # the scales and zero-points do not need to be reordered. - qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) - scales = self.get_sharded(f"{prefix}.scales", dim=0) - else: - qzeros = self.get_tensor(f"{prefix}.qzeros") - scales = self.get_tensor(f"{prefix}.scales") - - # For tp > 1, at this point we know we do not use act-order - if self.process_group.size() == 1: - g_idx = self.get_tensor(f"{prefix}.g_idx") - else: - g_idx = None + qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) + scales = self.get_sharded(f"{prefix}.scales", dim=0) + g_idx = self.get_sharded(f"{prefix}.g_idx", dim= 0) + g_idx = g_idx - g_idx[0] else: # The triton kernel reorders the scales/zero points instead of the weight/activation. # Thus, each rank needs the full qzeros/scales. From b2b5df0e946cfcbb7c679d94c0642bc7a6ad8f5e Mon Sep 17 00:00:00 2001 From: fxmarty <9808326+fxmarty@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:08:12 +0100 Subject: [PATCH 482/793] Add RoCm support (#1243) This PR adds support for AMD Instinct MI210 & MI250 GPUs, with paged attention and FAv2 support. Remaining items to discuss, on top of possible others: * Should we have a `ghcr.io/huggingface/text-generation-inference:1.1.0+rocm` hosted image, or is it too early? * Should we set up a CI on MI210/MI250? I don't have access to the runners of TGI though. * Are we comfortable with those changes being directly in TGI, or do we need a fork? --------- Co-authored-by: Felix Marty Co-authored-by: OlivierDehaene Co-authored-by: Your Name --- .github/workflows/build.yaml | 93 ++++++++++- Dockerfile | 8 +- Dockerfile_amd | 153 ++++++++++++++++++ README.md | 6 +- docs/source/quicktour.md | 2 + docs/source/supported_models.md | 6 +- server/Makefile | 5 +- server/Makefile-flash-att | 2 +- server/Makefile-flash-att-v2 | 25 ++- server/Makefile-vllm | 15 +- server/custom_kernels/setup.py | 9 +- server/requirements_common.txt | 46 ++++++ ...requirements.txt => requirements_cuda.txt} | 0 server/requirements_rocm.txt | 46 ++++++ .../custom_modeling/flash_llama_modeling.py | 37 +++-- .../custom_modeling/flash_mistral_modeling.py | 34 +++- .../custom_modeling/flash_neox_modeling.py | 3 +- .../custom_modeling/flash_rw_modeling.py | 6 +- .../custom_modeling/idefics_modeling.py | 49 ++++-- .../utils/flash_attn.py | 43 ++++- .../utils/import_utils.py | 4 + server/text_generation_server/utils/layers.py | 65 ++++++-- 22 files changed, 575 insertions(+), 82 deletions(-) create mode 100644 Dockerfile_amd create mode 100644 server/requirements_common.txt rename server/{requirements.txt => requirements_cuda.txt} (100%) create mode 100644 server/requirements_rocm.txt create mode 100644 server/text_generation_server/utils/import_utils.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 11a95f4b..395a0b6a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -59,7 +59,7 @@ jobs: build-and-push-image: concurrency: - group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-build-and-push-image-${{ github.head_ref || github.run_id }} cancel-in-progress: true needs: start-runner # required to start the main job when the runner is ready runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner @@ -146,6 +146,95 @@ jobs: cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min + build-and-push-image-rocm: + concurrency: + group: ${{ github.workflow }}-build-and-push-image-rocm-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + needs: start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + permissions: + contents: write + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + security-events: write + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Initialize Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + with: + install: true + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + - name: Tailscale + uses: tailscale/github-action@7bd8039bf25c23c4ab1b8d6e2cc2da2280601966 + with: + authkey: ${{ secrets.TAILSCALE_AUTHKEY }} + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to internal Container Registry + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.TAILSCALE_DOCKER_USERNAME }} + password: ${{ secrets.TAILSCALE_DOCKER_PASSWORD }} + registry: registry.internal.huggingface.tech + - name: Login to Azure Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.AZURE_DOCKER_USERNAME }} + password: ${{ secrets.AZURE_DOCKER_PASSWORD }} + registry: db4c2190dd824d1f950f5d1555fbadf0.azurecr.io + # If pull request + - name: Extract metadata (tags, labels) for Docker + if: ${{ github.event_name == 'pull_request' }} + id: meta-pr + uses: docker/metadata-action@v4.3.0 + with: + images: | + registry.internal.huggingface.tech/api-inference/community/text-generation-inference + tags: | + type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}-rocm + # If main, release or tag + - name: Extract metadata (tags, labels) for Docker + if: ${{ github.event_name != 'pull_request' }} + id: meta + uses: docker/metadata-action@v4.3.0 + with: + flavor: | + latest=false + images: | + registry.internal.huggingface.tech/api-inference/community/text-generation-inference + ghcr.io/huggingface/text-generation-inference + db4c2190dd824d1f950f5d1555fbadf0.azurecr.io/text-generation-inference + tags: | + type=semver,pattern={{version}}-rocm + type=semver,pattern={{major}}.{{minor}}-rocm + type=raw,value=latest-rocm,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + type=raw,value=sha-${{ env.GITHUB_SHA_SHORT }}-rocm + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile_amd + push: true + platforms: 'linux/amd64' + build-args: | + GIT_SHA=${{ env.GITHUB_SHA }} + DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}-rocm + tags: ${{ steps.meta.outputs.tags || steps.meta-pr.outputs.tags }} + labels: ${{ steps.meta.outputs.labels || steps.meta-pr.outputs.labels }} + cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache-rocm,mode=min + cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache-rocm,mode=min + integration-tests: concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} @@ -153,6 +242,7 @@ jobs: needs: - start-runner - build-and-push-image # Wait for the docker image to be built + - build-and-push-image-rocm runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner env: DOCKER_VOLUME: /cache @@ -187,6 +277,7 @@ jobs: needs: - start-runner - build-and-push-image + - build-and-push-image-rocm - integration-tests runs-on: ubuntu-latest env: diff --git a/Dockerfile b/Dockerfile index d45aaec5..02540f81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,7 +106,7 @@ WORKDIR /usr/src COPY server/Makefile-flash-att-v2 Makefile # Build specific version of flash attention v2 -RUN make build-flash-attention-v2 +RUN make build-flash-attention-v2-cuda # Build Transformers exllama kernels FROM kernel-builder as exllama-kernels-builder @@ -152,7 +152,7 @@ WORKDIR /usr/src COPY server/Makefile-vllm Makefile # Build specific version of vllm -RUN make build-vllm +RUN make build-vllm-cuda # Text Generation Inference base image FROM nvidia/cuda:12.1.0-base-ubuntu20.04 as base @@ -209,7 +209,7 @@ COPY server server COPY server/Makefile server/Makefile RUN cd server && \ make gen-server && \ - pip install -r requirements.txt && \ + pip install -r requirements_cuda.txt && \ pip install ".[bnb, accelerate, quantize, peft]" --no-cache-dir # Install benchmarker @@ -224,7 +224,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins g++ \ && rm -rf /var/lib/apt/lists/* -# AWS Sagemaker compatbile image +# AWS Sagemaker compatible image FROM base as sagemaker COPY sagemaker-entrypoint.sh entrypoint.sh diff --git a/Dockerfile_amd b/Dockerfile_amd new file mode 100644 index 00000000..dd331a5d --- /dev/null +++ b/Dockerfile_amd @@ -0,0 +1,153 @@ +# Rust builder +FROM lukemathwalker/cargo-chef:latest-rust-1.71 AS chef +WORKDIR /usr/src + +ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse + +FROM chef as planner +COPY Cargo.toml Cargo.toml +COPY rust-toolchain.toml rust-toolchain.toml +COPY proto proto +COPY benchmark benchmark +COPY router router +COPY launcher launcher +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder + +ARG GIT_SHA +ARG DOCKER_LABEL + +RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \ + curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \ + unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \ + unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \ + rm -f $PROTOC_ZIP + +COPY --from=planner /usr/src/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json + +COPY Cargo.toml Cargo.toml +COPY rust-toolchain.toml rust-toolchain.toml +COPY proto proto +COPY benchmark benchmark +COPY router router +COPY launcher launcher +RUN cargo build --release + +# Text Generation Inference base image for RoCm +FROM rocm/dev-ubuntu-20.04:5.7 as base + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + ccache \ + curl \ + git \ + make \ + libssl-dev \ + g++ \ + # Needed to build VLLM & flash. + rocthrust-dev \ + hipsparse-dev \ + hipblas-dev && \ + rm -rf /var/lib/apt/lists/* + +# Keep in sync with `server/pyproject.toml +ARG MAMBA_VERSION=23.1.0-1 +ARG PYTORCH_VERSION='2.2.0.dev0' +ARG ROCM_VERSION='5.7' +ARG PYTHON_VERSION='3.10.10' +# Automatically set by buildx +ARG TARGETPLATFORM +ENV PATH /opt/conda/bin:$PATH + +# TGI seem to require libssl.so.1.1 instead of libssl.so.3 so we can't use ubuntu 22.04. Ubuntu 20.04 has python==3.8, and TGI requires python>=3.9, hence the need for miniconda. +# Install mamba +# translating Docker's TARGETPLATFORM into mamba arches +RUN case ${TARGETPLATFORM} in \ + "linux/arm64") MAMBA_ARCH=aarch64 ;; \ + *) MAMBA_ARCH=x86_64 ;; \ + esac && \ + curl -fsSL -v -o ~/mambaforge.sh -O "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh" +RUN chmod +x ~/mambaforge.sh && \ + bash ~/mambaforge.sh -b -p /opt/conda && \ + mamba init && \ + rm ~/mambaforge.sh + +# Install PyTorch nightly (2.2.0.dev2023) compiled against RoCm 5.7, as VLLM can not be compiled with RoCm 5.6. +RUN pip install --pre torch==2.2.0.dev20231106 --index-url https://download.pytorch.org/whl/nightly/rocm5.7 + +FROM base AS kernel-builder + +# Build vllm kernels +FROM kernel-builder AS vllm-builder +WORKDIR /usr/src + +COPY server/Makefile-vllm Makefile + +# Build specific version of vllm +RUN make build-vllm-rocm + +# Build Flash Attention v2 kernels +FROM kernel-builder AS flash-att-v2-builder +WORKDIR /usr/src + +COPY server/Makefile-flash-att-v2 Makefile + +# Build specific version of flash attention v2 +RUN make build-flash-attention-v2-rocm + +# Build Transformers CUDA kernels (gpt-neox and bloom) +FROM kernel-builder as custom-kernels-builder +WORKDIR /usr/src +COPY server/custom_kernels/ . +RUN PYTORCH_ROCM_ARCH=gfx90a python setup.py build + +FROM base as base-copy + +# Text Generation Inference base env +ENV HUGGINGFACE_HUB_CACHE=/data \ + HF_HUB_ENABLE_HF_TRANSFER=1 \ + PORT=80 + +# Copy builds artifacts from vllm builder +COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages + +# Copy build artifacts from flash attention v2 builder +COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages + +# Copy build artifacts from custom kernels builder +COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages + +# Install flash-attention dependencies +RUN pip install einops --no-cache-dir + +# Install server +COPY proto proto +COPY server server +COPY server/Makefile server/Makefile +RUN cd server && \ + make gen-server && \ + pip install -r requirements_rocm.txt && \ + pip install ".[accelerate, peft]" --no-cache-dir + +# Install benchmarker +COPY --from=builder /usr/src/target/release/text-generation-benchmark /usr/local/bin/text-generation-benchmark +# Install router +COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bin/text-generation-router +# Install launcher +COPY --from=builder /usr/src/target/release/text-generation-launcher /usr/local/bin/text-generation-launcher + +# AWS Sagemaker compatible image +FROM base-copy as sagemaker +COPY sagemaker-entrypoint.sh entrypoint.sh +RUN chmod +x entrypoint.sh + +ENTRYPOINT ["./entrypoint.sh"] + +# Final image +FROM base-copy + +ENTRYPOINT ["text-generation-launcher"] +CMD ["--json-output"] diff --git a/README.md b/README.md index e4f90d53..0fa7a538 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,9 @@ curl 127.0.0.1:8080/generate \ -H 'Content-Type: application/json' ``` -**Note:** To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. +**Note:** To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. + +**Note:** TGI supports AMD Instinct MI210 and MI250 [to some extent](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0+rocm --model-id $model` instead of the command above. To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli): ``` @@ -189,7 +191,7 @@ sudo apt-get install libssl-dev gcc -y ### CUDA Kernels -The custom CUDA kernels are only tested on NVIDIA A100s. If you have any installation or runtime issues, you can remove +The custom CUDA kernels are only tested on NVIDIA A100, AMD MI210 and AMD MI250. If you have any installation or runtime issues, you can remove the kernels by using the `DISABLE_CUSTOM_KERNELS=True` environment variable. Be aware that the official Docker image has them enabled by default. diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index efcaae28..b0a77599 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -15,6 +15,8 @@ docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingf To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) . We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. +To use TGI on RoCm-enabled AMD GPUs (only MI210 and MI250 are tested), please use the image `ghcr.io/huggingface/text-generation-inference:1.1.1+rocm` instead. For details about the usage on RoCm, please refer to the [Supported Hardware section](./supported_models#supported-hardware) and [AMD documentation](https://rocm.docs.amd.com/en/latest/deploy/docker.html). + Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section, where we show examples with utility libraries and UIs. Below you can see a simple snippet to query the endpoint. diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md index 8b4c33b1..d7d45b70 100644 --- a/docs/source/supported_models.md +++ b/docs/source/supported_models.md @@ -39,9 +39,9 @@ text-generation-launcher --model-id ## Supported Hardware -TGI optimized models are supported on NVIDIA [A100](https://www.nvidia.com/en-us/data-center/a100/), [A10G](https://www.nvidia.com/en-us/data-center/products/a10-gpu/) and [T4](https://www.nvidia.com/en-us/data-center/tesla-t4/) GPUs with CUDA 11.8+. Note that you have to install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) to use it. For other hardware, continuous batching will still apply, but some operations like flash attention and paged attention will not be executed. +TGI optimized models are supported on NVIDIA [A100](https://www.nvidia.com/en-us/data-center/a100/), [A10G](https://www.nvidia.com/en-us/data-center/products/a10-gpu/) and [T4](https://www.nvidia.com/en-us/data-center/tesla-t4/) GPUs with CUDA 11.8+. Note that you have to install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) to use it. For other NVIDIA GPUs, continuous batching will still apply, but some operations like flash attention and paged attention will not be executed. + +TGI also has support of RoCm-enabled AMD Instinct MI210 and MI250 GPUs, with paged attention and flash attention v2 support. The following features are missing from the RoCm version of TGI: quantization and flash [layer norm kernel](https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm). TGI is also supported on the following AI hardware accelerators: - *Habana first-gen Gaudi and Gaudi2:* check out this [example](https://github.com/huggingface/optimum-habana/tree/main/text-generation-inference) how to serve models with TGI on Gaudi and Gaudi2 with [Optimum Habana](https://huggingface.co/docs/optimum/habana/index) - - diff --git a/server/Makefile b/server/Makefile index 92958d02..2810a528 100644 --- a/server/Makefile +++ b/server/Makefile @@ -18,11 +18,12 @@ gen-server: install: gen-server pip install pip --upgrade - pip install -r requirements.txt + pip install -r requirements_cuda.txt pip install -e ".[bnb, accelerate, quantize, peft]" run-dev: SAFETENSORS_FAST_GPU=1 python -m torch.distributed.run --nproc_per_node=2 text_generation_server/cli.py serve bigscience/bloom-560m --sharded export-requirements: - poetry export -o requirements.txt -E bnb --without-hashes + poetry export -o requirements_cuda.txt --extras bnb --without-hashes + poetry export -o requirements_rocm.txt --without-hashes diff --git a/server/Makefile-flash-att b/server/Makefile-flash-att index bc1d37ef..b4b2e40c 100644 --- a/server/Makefile-flash-att +++ b/server/Makefile-flash-att @@ -2,7 +2,7 @@ flash_att_commit := 3a9bfd076f98746c73362328958dbc68d145fbec flash-attention: # Clone flash attention - pip install packaging + pip install -U packaging ninja --no-cache-dir git clone https://github.com/HazyResearch/flash-attention.git build-flash-attention: flash-attention diff --git a/server/Makefile-flash-att-v2 b/server/Makefile-flash-att-v2 index 583437b2..8b9f289d 100644 --- a/server/Makefile-flash-att-v2 +++ b/server/Makefile-flash-att-v2 @@ -1,13 +1,26 @@ flash_att_v2_commit := 02ac572f3ffc4f402e4183aaa6824b45859d3ed3 +build-flash-attention-v2-cuda: FLASH_ATTN_V2_COMMIT=02ac572f3ffc4f402e4183aaa6824b45859d3ed3 +build-flash-attention-v2-cuda: FLASH_REPOSITORY=https://github.com/HazyResearch/flash-attention.git +build-flash-attention-v2-cuda: BRANCH=main +build-flash-attention-v2-cuda: PYTORCH_ROCM_ARCH="" +build-flash-attention-v2-cuda: build-flash-attention-v2 + +build-flash-attention-v2-rocm: FLASH_ATTN_V2_COMMIT=8736558c287ff2ef28b24878e42828c595ac3e69 +build-flash-attention-v2-rocm: FLASH_REPOSITORY=https://github.com/fxmarty/flash-attention-rocm +build-flash-attention-v2-rocm: BRANCH=remove-offload-arch-native +build-flash-attention-v2-rocm: PYTORCH_ROCM_ARCH=gfx90a +build-flash-attention-v2-rocm: build-flash-attention-v2 + flash-attention-v2: - # Clone flash attention - pip install packaging - git clone https://github.com/HazyResearch/flash-attention.git flash-attention-v2 + # Clone flash attention + pip install -U packaging ninja --no-cache-dir + git clone --single-branch --branch $(BRANCH) $(FLASH_REPOSITORY) flash-attention-v2 build-flash-attention-v2: flash-attention-v2 - cd flash-attention-v2 && git fetch && git checkout $(flash_att_v2_commit) - cd flash-attention-v2 && python setup.py build + cd flash-attention-v2 && git fetch && git checkout $(FLASH_ATTN_V2_COMMIT) + cd flash-attention-v2 && git submodule update --init --recursive + cd flash-attention-v2 && PYTORCH_ROCM_ARCH=$(PYTORCH_ROCM_ARCH) python setup.py build install-flash-attention-v2: build-flash-attention-v2 - cd flash-attention-v2 && python setup.py install \ No newline at end of file + cd flash-attention-v2 && git submodule update --init --recursive && python setup.py install \ No newline at end of file diff --git a/server/Makefile-vllm b/server/Makefile-vllm index c601e452..ddb648ea 100644 --- a/server/Makefile-vllm +++ b/server/Makefile-vllm @@ -1,11 +1,20 @@ -vllm_commit := f8a1e39fae05ca610be8d5a78be9d40f5274e5fc +build-vllm-cuda: REPOSITORY=https://github.com/vllm-project/vllm.git +build-vllm-cuda: VLLM_COMMIT=f8a1e39fae05ca610be8d5a78be9d40f5274e5fc +build-vllm-cuda: BRANCH=main +build-vllm-cuda: build-vllm + +build-vllm-rocm: REPOSITORY=https://github.com/fxmarty/vllm-public.git +build-vllm-rocm: VLLM_COMMIT=ad9b7c4095ef54419a0533d254f2ad84bd2dfcae +build-vllm-rocm: BRANCH=rotary-no-positions-split-cos-sin +build-vllm-rocm: build-vllm vllm: # Clone vllm - git clone https://github.com/vllm-project/vllm.git + pip install -U ninja packaging --no-cache-dir + git clone --single-branch --branch $(BRANCH) $(REPOSITORY) vllm build-vllm: vllm - cd vllm && git fetch && git checkout $(vllm_commit) + cd vllm && git fetch && git checkout $(VLLM_COMMIT) cd vllm && python setup.py build install-vllm: build-vllm diff --git a/server/custom_kernels/setup.py b/server/custom_kernels/setup.py index 43b8ee4e..69f6b72a 100644 --- a/server/custom_kernels/setup.py +++ b/server/custom_kernels/setup.py @@ -1,5 +1,10 @@ from setuptools import setup from torch.utils.cpp_extension import BuildExtension, CUDAExtension +import torch + +extra_compile_args = ["-std=c++17"] +if not torch.version.hip: + extra_compile_args.append("-arch=compute_80") setup( name="custom_kernels", @@ -7,12 +12,12 @@ CUDAExtension( name="custom_kernels.fused_bloom_attention_cuda", sources=["custom_kernels/fused_bloom_attention_cuda.cu"], - extra_compile_args=["-arch=compute_80", "-std=c++17"], + extra_compile_args=extra_compile_args, ), CUDAExtension( name="custom_kernels.fused_attention_cuda", sources=["custom_kernels/fused_attention_cuda.cu"], - extra_compile_args=["-arch=compute_80", "-std=c++17"], + extra_compile_args=extra_compile_args, ), ], cmdclass={"build_ext": BuildExtension}, diff --git a/server/requirements_common.txt b/server/requirements_common.txt new file mode 100644 index 00000000..5a321834 --- /dev/null +++ b/server/requirements_common.txt @@ -0,0 +1,46 @@ +backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13" +certifi==2023.11.17 ; python_version >= "3.9" and python_version < "3.13" +charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "3.13" +click==8.1.7 ; python_version >= "3.9" and python_version < "3.13" +colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows") +deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13" +einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13" +filelock==3.13.1 ; python_version >= "3.9" and python_version < "3.13" +fsspec==2023.10.0 ; python_version >= "3.9" and python_version < "3.13" +googleapis-common-protos==1.61.0 ; python_version >= "3.9" and python_version < "3.13" +grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13" +grpcio-reflection==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +grpcio-status==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +grpcio==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +hf-transfer==0.1.4 ; python_version >= "3.9" and python_version < "3.13" +huggingface-hub==0.16.4 ; python_version >= "3.9" and python_version < "3.13" +idna==3.4 ; python_version >= "3.9" and python_version < "3.13" +loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" +numpy==1.26.2 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-exporter-otlp==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-instrumentation-grpc==0.36b0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13" +packaging==23.2 ; python_version >= "3.9" and python_version < "3.13" +pillow==10.1.0 ; python_version >= "3.9" and python_version < "3.13" +protobuf==4.25.1 ; python_version >= "3.9" and python_version < "3.13" +pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" +regex==2023.10.3 ; python_version >= "3.9" and python_version < "3.13" +requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" +safetensors==0.3.3 ; python_version >= "3.9" and python_version < "3.13" +scipy==1.11.4 ; python_version >= "3.9" and python_version < "3.13" +sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" +setuptools==69.0.2 ; python_version >= "3.9" and python_version < "3.13" +tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" +tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.33.3 ; python_version >= "3.9" and python_version < "3.13" +typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" +typing-extensions==4.8.0 ; python_version >= "3.9" and python_version < "3.13" +urllib3==2.1.0 ; python_version >= "3.9" and python_version < "3.13" +win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32" +wrapt==1.16.0 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/requirements.txt b/server/requirements_cuda.txt similarity index 100% rename from server/requirements.txt rename to server/requirements_cuda.txt diff --git a/server/requirements_rocm.txt b/server/requirements_rocm.txt new file mode 100644 index 00000000..5a321834 --- /dev/null +++ b/server/requirements_rocm.txt @@ -0,0 +1,46 @@ +backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13" +certifi==2023.11.17 ; python_version >= "3.9" and python_version < "3.13" +charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "3.13" +click==8.1.7 ; python_version >= "3.9" and python_version < "3.13" +colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows") +deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13" +einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13" +filelock==3.13.1 ; python_version >= "3.9" and python_version < "3.13" +fsspec==2023.10.0 ; python_version >= "3.9" and python_version < "3.13" +googleapis-common-protos==1.61.0 ; python_version >= "3.9" and python_version < "3.13" +grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13" +grpcio-reflection==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +grpcio-status==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +grpcio==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +hf-transfer==0.1.4 ; python_version >= "3.9" and python_version < "3.13" +huggingface-hub==0.16.4 ; python_version >= "3.9" and python_version < "3.13" +idna==3.4 ; python_version >= "3.9" and python_version < "3.13" +loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" +numpy==1.26.2 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-exporter-otlp==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-instrumentation-grpc==0.36b0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13" +opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13" +packaging==23.2 ; python_version >= "3.9" and python_version < "3.13" +pillow==10.1.0 ; python_version >= "3.9" and python_version < "3.13" +protobuf==4.25.1 ; python_version >= "3.9" and python_version < "3.13" +pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" +regex==2023.10.3 ; python_version >= "3.9" and python_version < "3.13" +requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" +safetensors==0.3.3 ; python_version >= "3.9" and python_version < "3.13" +scipy==1.11.4 ; python_version >= "3.9" and python_version < "3.13" +sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" +setuptools==69.0.2 ; python_version >= "3.9" and python_version < "3.13" +tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" +tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.33.3 ; python_version >= "3.9" and python_version < "3.13" +typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" +typing-extensions==4.8.0 ; python_version >= "3.9" and python_version < "3.13" +urllib3==2.1.0 ; python_version >= "3.9" and python_version < "3.13" +win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32" +wrapt==1.16.0 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 69608e1c..4aeb447d 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -26,9 +26,6 @@ from transformers.configuration_utils import PretrainedConfig from typing import Optional, List, Tuple -# Flash attention imports -import dropout_layer_norm - from text_generation_server.utils import paged_attention, flash_attn from text_generation_server.utils.layers import ( TensorParallelRowLinear, @@ -38,6 +35,12 @@ TensorParallelHead, get_linear, ) +from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM + +if IS_CUDA_SYSTEM: + import dropout_layer_norm +elif IS_ROCM_SYSTEM: + from vllm import layernorm_ops class LlamaConfig(PretrainedConfig): @@ -120,7 +123,7 @@ def forward(self, hidden_states, residual=None): hidden_states = hidden_states.to(self.weight.dtype) return self.weight * hidden_states, residual - else: + elif IS_CUDA_SYSTEM: # faster post attention rms norm normed_hidden_states, res, *rest = dropout_layer_norm.dropout_add_ln_fwd( hidden_states, @@ -143,6 +146,22 @@ def forward(self, hidden_states, residual=None): res = hidden_states return normed_hidden_states, res + elif IS_ROCM_SYSTEM: + # We use VLLM RMSNorm kernel that can be compiled for RoCm, instead of Flash Attention ones that can not. + if residual is not None: + hidden_states += residual + residual = hidden_states + + out = torch.empty_like(hidden_states) + layernorm_ops.rms_norm( + out, + hidden_states, + self.weight.data, + self.variance_epsilon, + ) + return out, residual + else: + raise ValueError("Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") def load_attention(config, prefix, weights): @@ -204,9 +223,6 @@ def __init__( self.hidden_size = config.hidden_size self.head_size = self.hidden_size // self.num_heads - # self.rotary_emb = PositionRotaryEmbedding.load( - # config=config, prefix=f"{prefix}.rotary_emb", weights=weights - # ) self.rotary_emb = PositionRotaryEmbedding.static( config=config, dim=self.head_size, @@ -261,9 +277,8 @@ def forward( ) query = query.view(-1, self.num_heads, self.head_size) kv = kv.view(-1, 2, self.num_key_value_heads, self.head_size) - - self.rotary_emb(query, cos, sin) - self.rotary_emb(torch.select(kv, dim=1, index=0), cos, sin) + + self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin) paged_attention.reshape_and_cache( kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots @@ -297,7 +312,7 @@ def forward( input_lengths, max_s, ) - + return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size)) diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index 2d731406..959949f0 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -26,11 +26,8 @@ from transformers.configuration_utils import PretrainedConfig from typing import Optional, List, Tuple -# Flash attention imports -import dropout_layer_norm - from text_generation_server.utils import paged_attention, flash_attn -from text_generation_server.utils.flash_attn import attention, HAS_FLASH_ATTN_V2 +from text_generation_server.utils.flash_attn import attention, HAS_FLASH_ATTN_V2_ROCM, HAS_FLASH_ATTN_V2_CUDA from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -39,8 +36,14 @@ TensorParallelHead, get_linear, ) +from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM + +if IS_CUDA_SYSTEM: + import dropout_layer_norm +elif IS_ROCM_SYSTEM: + from vllm import layernorm_ops -if not HAS_FLASH_ATTN_V2: +if not HAS_FLASH_ATTN_V2_CUDA and not HAS_FLASH_ATTN_V2_ROCM: raise ImportError("Mistral model requires flash attn v2") @@ -126,7 +129,7 @@ def forward(self, hidden_states, residual=None): hidden_states = hidden_states.to(self.weight.dtype) return self.weight * hidden_states, residual - else: + elif IS_CUDA_SYSTEM: # faster post attention rms norm normed_hidden_states, res, *rest = dropout_layer_norm.dropout_add_ln_fwd( hidden_states, @@ -149,6 +152,22 @@ def forward(self, hidden_states, residual=None): res = hidden_states return normed_hidden_states, res + elif IS_ROCM_SYSTEM: + # We use VLLM RMSNorm kernel that can be compiled for RoCm, instead of Flash Attention ones that can not. + if residual is not None: + hidden_states += residual + residual = hidden_states + + out = torch.empty_like(hidden_states) + layernorm_ops.rms_norm( + out, + hidden_states, + self.weight.data, + self.variance_epsilon, + ) + return out, residual + else: + raise ValueError("Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") def load_attention(config, prefix, weights): @@ -261,8 +280,7 @@ def forward( query = query.view(-1, self.num_heads, self.head_size) kv = kv.view(-1, 2, self.num_key_value_heads, self.head_size) - self.rotary_emb(query, cos, sin) - self.rotary_emb(torch.select(kv, dim=1, index=0), cos, sin) + self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin) if prefill_cache_indices is not None: kv_to_cache = kv[prefill_cache_indices] diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index af4ba96b..eea5f787 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -135,8 +135,7 @@ def forward( qkv = qkv.view(-1, 3, self.num_heads, self.head_size) # Inplace rotary - self.rotary_emb(qkv[:, 0], cos, sin) - self.rotary_emb(qkv[:, 1], cos, sin) + self.rotary_emb(qkv[:, 0], qkv[:, 1], cos, sin) paged_attention.reshape_and_cache( qkv[:, 1], qkv[:, 2], kv_cache[0], kv_cache[1], slots diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py index 00f953a6..6a530f3c 100644 --- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py @@ -185,8 +185,7 @@ def forward( kv = kv.view(-1, 2, self.num_heads_kv, self.head_size) # Inplace rotary - self.rotary_emb(query, cos, sin) - self.rotary_emb(torch.select(kv, dim=1, index=0), cos, sin) + self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin) paged_attention.reshape_and_cache( kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots @@ -301,8 +300,7 @@ def forward( query = query.reshape(-1, self.num_groups * self.num_heads, self.head_size) # Inplace rotary - self.rotary_emb(query, cos, sin) - self.rotary_emb(torch.select(kv, dim=2, index=0), cos, sin) + self.rotary_emb(query, torch.select(kv, dim=2, index=0), cos, sin) paged_attention.reshape_and_cache( kv[:, :, 0].contiguous(), diff --git a/server/text_generation_server/models/custom_modeling/idefics_modeling.py b/server/text_generation_server/models/custom_modeling/idefics_modeling.py index 1ffe6276..946f7683 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_modeling.py +++ b/server/text_generation_server/models/custom_modeling/idefics_modeling.py @@ -55,8 +55,12 @@ PositionRotaryEmbedding, FastLinear, ) -import dropout_layer_norm +from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM +if IS_CUDA_SYSTEM: + import dropout_layer_norm +elif IS_ROCM_SYSTEM: + from vllm import layernorm_ops @dataclass class BaseModelOutputWithPastImage(BaseModelOutputWithPast): @@ -370,7 +374,7 @@ def forward(self, hidden_states, residual=None): hidden_states = hidden_states.to(self.weight.dtype) return self.weight * hidden_states - else: + elif IS_CUDA_SYSTEM: # faster post attention rms norm unwrap = False if len(hidden_states.shape) > 2: @@ -402,6 +406,32 @@ def forward(self, hidden_states, residual=None): normed_hidden_states = normed_hidden_states.view(*shape) return normed_hidden_states + elif IS_ROCM_SYSTEM: + # We use VLLM RMSNorm kernel that can be compiled for RoCm, instead of Flash Attention ones that can not. + if residual is not None: + hidden_states += residual + residual = hidden_states + + unwrap = False + if len(hidden_states.shape) > 2: + unwrap = True + shape = hidden_states.shape + hidden_states = hidden_states.reshape(-1, shape[-1]) + + out = torch.empty_like(hidden_states) + layernorm_ops.rms_norm( + out, + hidden_states, + self.weight.data, + self.variance_epsilon, + ) + + if unwrap: + out = out.view(*shape) + + return out + else: + raise ValueError("Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") # this was adapted from LlamaMLP @@ -581,15 +611,12 @@ def forward( position_ids.view(-1), max_s, hidden_states.dtype ) - shape = query_states.shape - query_states = self.rotary_emb( - query_states.view(-1, *shape[2:]), cos, sin - ).view(shape) - - shape = key_states.shape - key_states = self.rotary_emb( - key_states.reshape(-1, *shape[2:]), cos, sin - ).view(shape) + query_shape = query_states.shape + key_shape = key_states.shape + self.rotary_emb(query_states.view(-1, *query_shape[2:]), key_states.reshape(-1, *key_shape[2:]), cos, sin) + + query_states = query_states.view(query_shape) + key_states = key_states.view(key_shape) query_states = query_states.transpose(1, 2) key_states = key_states.transpose(1, 2) diff --git a/server/text_generation_server/utils/flash_attn.py b/server/text_generation_server/utils/flash_attn.py index 8f0fcee6..aca95e11 100644 --- a/server/text_generation_server/utils/flash_attn.py +++ b/server/text_generation_server/utils/flash_attn.py @@ -3,6 +3,8 @@ from loguru import logger +from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM + if os.getenv("USE_FLASH_ATTENTION", "").lower() == "false": raise ImportError("`USE_FLASH_ATTENTION` is false.") @@ -15,7 +17,8 @@ is_sm90 = major == 9 and minor == 0 HAS_FLASH_ATTN = False -HAS_FLASH_ATTN_V2 = False +HAS_FLASH_ATTN_V2_CUDA = False +HAS_FLASH_ATTN_V2_ROCM = False try: try: import flash_attn_2_cuda @@ -30,7 +33,8 @@ f"GPU with CUDA capability {major} {minor} is not supported for " "Flash Attention V2" ) - HAS_FLASH_ATTN_V2 = True + HAS_FLASH_ATTN_V2_CUDA = IS_CUDA_SYSTEM + HAS_FLASH_ATTN_V2_ROCM = IS_ROCM_SYSTEM except ImportError as e: try: import flash_attn_cuda @@ -41,10 +45,17 @@ "or install flash attention with `cd server && make install install-flash-attention`" ) from e - if not (is_sm75 or is_sm8x or is_sm90): + if IS_CUDA_SYSTEM and not (is_sm75 or is_sm8x or is_sm90): raise ImportError( f"GPU with CUDA capability {major} {minor} is not supported" ) from e + elif IS_ROCM_SYSTEM: + for idx in range(torch.cuda.device_count()): + if "MI210" not in torch.cuda.get_device_name(idx) and "MI250" not in torch.cuda.get_device_name(idx): + raise ImportError( + f"AMD GPU {torch.cuda.get_device_name(idx)} does not support flash-attention" + ) + logger.warning(f"Unable to use Flash Attention V2: {e}") HAS_FLASH_ATTN = True @@ -59,7 +70,7 @@ def attention( softmax_scale, window_size_left=-1, ): - if HAS_FLASH_ATTN_V2: + if HAS_FLASH_ATTN_V2_CUDA: return flash_attn_2_cuda.varlen_fwd( q, k, @@ -78,8 +89,28 @@ def attention( False, None, ) - - if HAS_FLASH_ATTN: + elif HAS_FLASH_ATTN_V2_ROCM: + if window_size_left != -1: + raise ValueError(f"RoCm version of Flash Attention v2 does not support window attention (window_size_left != -1, got window_size_left={window_size_left}).") + + # RoCm flash API does not take the window_size_left and window_size_right arguments. + return flash_attn_2_cuda.varlen_fwd( + q, + k, + v, + out, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + 0.0, + softmax_scale, + False, + True, + False, + None, + ) + elif HAS_FLASH_ATTN: if window_size_left != -1: raise NotImplementedError( "window_size_left is only available with flash attn v2" diff --git a/server/text_generation_server/utils/import_utils.py b/server/text_generation_server/utils/import_utils.py new file mode 100644 index 00000000..428c9f3e --- /dev/null +++ b/server/text_generation_server/utils/import_utils.py @@ -0,0 +1,4 @@ +import torch + +IS_ROCM_SYSTEM = torch.version.hip is not None +IS_CUDA_SYSTEM = torch.version.cuda is not None diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index e6a90116..a93ccd0e 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -12,14 +12,13 @@ try: import bitsandbytes as bnb from bitsandbytes.nn import Int8Params, Params4bit - except ImportError: HAS_BITS_AND_BYTES = False from accelerate import init_empty_weights from text_generation_server.utils.gptq.quant_linear import QuantLinear - +from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM HAS_AWQ = True try: @@ -525,11 +524,14 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: try: - import dropout_layer_norm + if IS_CUDA_SYSTEM: + import dropout_layer_norm + else: + dropout_layer_norm = None class FastLayerNorm(nn.LayerNorm): def forward(self, hidden_states, residual=None): - if hidden_states.shape[-1] > 8192: + if hidden_states.shape[-1] > 8192 or IS_ROCM_SYSTEM: if residual is not None: hidden_states += residual residual = hidden_states @@ -561,14 +563,16 @@ def forward(self, hidden_states, residual=None): residual = hidden_states return normed_hidden_states, residual - except ImportError: pass try: - from flash_attn.layers.rotary import RotaryEmbedding - import rotary_emb + if IS_CUDA_SYSTEM: + from flash_attn.layers.rotary import RotaryEmbedding + import rotary_emb + elif IS_ROCM_SYSTEM: + from vllm import pos_encoding_ops def _create_inv_freq(dim, base, device): inv_freq = 1.0 / ( @@ -597,6 +601,37 @@ def __init__(self, inv_freq, scaling_factor): self.scaling_factor = scaling_factor self.dynamic_args = None + def forward(self, query: torch.Tensor, key: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): + # Such controlflows may add some overhead. + if IS_CUDA_SYSTEM: + rotary_dim = cos.shape[-1] + q1 = query[..., :rotary_dim] + q2 = query[..., rotary_dim : 2 * rotary_dim] + + rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False) + + k1 = key[..., :rotary_dim] + k2 = key[..., rotary_dim : 2 * rotary_dim] + + rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False) + elif IS_ROCM_SYSTEM: + # NOTE: On RoCm systems, we use a ROPE implementatation adapted from VLLM which launches a single kernel for both query/key, contrary to flash-attn implementation used on NVIDIA systems. + # Compiling flash-attn rotary on RoCm, it appears hipcc is unable to unroll loops, resulting in an even slower inference compared to eager: https://github.com/pytorch/pytorch/issues/113773 + + head_size = query.shape[-1] + + # Inplace operation, updating query and key. + pos_encoding_ops.rotary_embedding( + query, + key, + head_size, + cos, + sin, + True + ) + else: + raise ValueError("Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") + @classmethod def static(cls, config, dim, base, device): inv_freq = _create_inv_freq(dim, base, device) @@ -699,21 +734,19 @@ def get_cos_sin( """ Return cos and sin for the asked position ids """ + if IS_ROCM_SYSTEM: + # For RoCm, we always use float cos/sin to avoid a cast. + # For NVIDIA, for some reason, the flash-attn rotary kernel requires cos/sin and query/key to be of same dtype: https://github.com/Dao-AILab/flash-attention/blob/017716451d446e464dde9aca3a3c1ed2209caaa9/csrc/rotary/rotary.cpp#L26 + # But later on goes and cast cos/sin to float anyway: https://github.com/Dao-AILab/flash-attention/blob/017716451d446e464dde9aca3a3c1ed2209caaa9/csrc/rotary/rotary_cuda.cu#L29, which looks suboptimal. + dtype = torch.float32 self._update_cos_sin_cache(dtype, position_ids.device, max_s) cos = torch.index_select(self._cos_cached, 0, position_ids) sin = torch.index_select(self._sin_cached, 0, position_ids) + # Note: this unsqueeze is not necessary on RoCm + VLLM ROPE implementation, but we leave it as is to avoid yet an other controlflow. return cos.unsqueeze(1), sin.unsqueeze(1) - def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): - rotary_dim = cos.shape[-1] - x1 = x[..., :rotary_dim] - x2 = x[..., rotary_dim : 2 * rotary_dim] - - rotary_emb.apply_rotary(x1, x2, cos, sin, x1, x2, False) - return x - class DynamicPositionRotaryEmbedding(PositionRotaryEmbedding): def __init__(self, dim, max_position_embeddings, base, device, scaling_factor): inv_freq = _create_inv_freq(dim, base, device) @@ -722,7 +755,7 @@ def __init__(self, dim, max_position_embeddings, base, device, scaling_factor): self.max_position_embeddings = max_position_embeddings self.base = base - def _update_cos_sin_cache(self, dtype, device, seqlen): + def _update_cos_sin_cache(self, dtype, device, seqlen): # Reset the tables if the sequence length has changed, # or if we're on a new device (possibly due to tracing for instance) if ( From 3c71c656c7075fa39d1354f7cb97a2107d28d7e4 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 28 Nov 2023 16:28:40 +0100 Subject: [PATCH 483/793] `make install-flash-attn-v2-cuda` should work like `make install-flash-attn-v2` used to work. (#1294) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/Makefile-flash-att-v2 | 39 +++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/server/Makefile-flash-att-v2 b/server/Makefile-flash-att-v2 index 8b9f289d..71c6cabe 100644 --- a/server/Makefile-flash-att-v2 +++ b/server/Makefile-flash-att-v2 @@ -1,26 +1,29 @@ -flash_att_v2_commit := 02ac572f3ffc4f402e4183aaa6824b45859d3ed3 +flash_att_v2_commit_cuda := 02ac572f3ffc4f402e4183aaa6824b45859d3ed3 +flash_att_v2_commit_rocm := 8736558c287ff2ef28b24878e42828c595ac3e69 -build-flash-attention-v2-cuda: FLASH_ATTN_V2_COMMIT=02ac572f3ffc4f402e4183aaa6824b45859d3ed3 -build-flash-attention-v2-cuda: FLASH_REPOSITORY=https://github.com/HazyResearch/flash-attention.git -build-flash-attention-v2-cuda: BRANCH=main -build-flash-attention-v2-cuda: PYTORCH_ROCM_ARCH="" -build-flash-attention-v2-cuda: build-flash-attention-v2 -build-flash-attention-v2-rocm: FLASH_ATTN_V2_COMMIT=8736558c287ff2ef28b24878e42828c595ac3e69 -build-flash-attention-v2-rocm: FLASH_REPOSITORY=https://github.com/fxmarty/flash-attention-rocm -build-flash-attention-v2-rocm: BRANCH=remove-offload-arch-native -build-flash-attention-v2-rocm: PYTORCH_ROCM_ARCH=gfx90a -build-flash-attention-v2-rocm: build-flash-attention-v2 +flash-attention-v2-cuda: + # Clone flash attention + pip install -U packaging ninja --no-cache-dir + git clone https://github.com/HazyResearch/flash-attention.git flash-attention-v2 + +build-flash-attention-v2-cuda: flash-attention-v2-cuda + cd flash-attention-v2 && git fetch && git checkout $(flash_att_v2_commit_cuda) + cd flash-attention-v2 && git submodule update --init --recursive + cd flash-attention-v2 && python setup.py build + +install-flash-attention-v2-cuda: build-flash-attention-v2-cuda + cd flash-attention-v2 && git submodule update --init --recursive && python setup.py install -flash-attention-v2: +flash-attention-v2-rocm: # Clone flash attention pip install -U packaging ninja --no-cache-dir - git clone --single-branch --branch $(BRANCH) $(FLASH_REPOSITORY) flash-attention-v2 + git clone https://github.com/fxmarty/flash-attention-rocm flash-attention-v2 -build-flash-attention-v2: flash-attention-v2 - cd flash-attention-v2 && git fetch && git checkout $(FLASH_ATTN_V2_COMMIT) +build-flash-attention-v2-rocm: flash-attention-v2-rocm + cd flash-attention-v2 && git fetch && git checkout $(flash_att_v2_commit_rocm) cd flash-attention-v2 && git submodule update --init --recursive - cd flash-attention-v2 && PYTORCH_ROCM_ARCH=$(PYTORCH_ROCM_ARCH) python setup.py build + cd flash-attention-v2 && PYTORCH_ROCM_ARCH=gfx90a python setup.py build -install-flash-attention-v2: build-flash-attention-v2 - cd flash-attention-v2 && git submodule update --init --recursive && python setup.py install \ No newline at end of file +install-flash-attention-v2-rocm: build-flash-attention-v2-rocm + cd flash-attention-v2 && git submodule update --init --recursive && python setup.py install From ba552e1a821d7a8f60ac0c576aab1857a523038a Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 28 Nov 2023 17:54:26 +0100 Subject: [PATCH 484/793] Let each model resolve their own default dtype. (#1287) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- integration-tests/conftest.py | 8 ++++++++ integration-tests/models/test_idefics.py | 2 +- server/text_generation_server/cli.py | 2 +- server/text_generation_server/models/__init__.py | 4 +++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index c1cbe7f3..d2216241 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -210,6 +210,7 @@ def local_launcher( quantize: Optional[str] = None, trust_remote_code: bool = False, use_flash_attention: bool = True, + dtype: Optional[str] = None ): port = random.randint(8000, 10_000) master_port = random.randint(10_000, 20_000) @@ -237,6 +238,9 @@ def local_launcher( if quantize is not None: args.append("--quantize") args.append(quantize) + if dtype is not None: + args.append("--dtype") + args.append(dtype) if trust_remote_code: args.append("--trust-remote-code") @@ -269,6 +273,7 @@ def docker_launcher( quantize: Optional[str] = None, trust_remote_code: bool = False, use_flash_attention: bool = True, + dtype: Optional[str] = None ): port = random.randint(8000, 10_000) @@ -279,6 +284,9 @@ def docker_launcher( if quantize is not None: args.append("--quantize") args.append(quantize) + if dtype is not None: + args.append("--dtype") + args.append(dtype) if trust_remote_code: args.append("--trust-remote-code") diff --git a/integration-tests/models/test_idefics.py b/integration-tests/models/test_idefics.py index 5f4571b5..5a81a4f0 100644 --- a/integration-tests/models/test_idefics.py +++ b/integration-tests/models/test_idefics.py @@ -3,7 +3,7 @@ @pytest.fixture(scope="module") def idefics_handle(launcher): - with launcher("HuggingFaceM4/idefics-9b-instruct", num_shard=2) as handle: + with launcher("HuggingFaceM4/idefics-9b-instruct", num_shard=2, dtype="float16") as handle: yield handle diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index b741a84c..3abe86af 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -76,7 +76,7 @@ def serve( # Downgrade enum into str for easier management later on quantize = None if quantize is None else quantize.value dtype = None if dtype is None else dtype.value - if dtype is not None and quantize is not None: + if dtype is not None and quantize not in {None, "bitsandbytes", "bitsandbytes-nf4", "bitsandbytes-fp4"}: raise RuntimeError( "Only 1 can be set between `dtype` and `quantize`, as they both decide how goes the final model." ) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 5b1b5715..ab3b25b7 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -87,7 +87,9 @@ def get_model( trust_remote_code: bool, ) -> Model: if dtype is None: - dtype = torch.float16 + # Keep it as default for now and let + # every model resolve their own default dtype. + dtype = None elif dtype == "float16": dtype = torch.float16 elif dtype == "bfloat16": From 624800c4de31399c305c9f62e54db10c096757a9 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 28 Nov 2023 21:22:35 +0100 Subject: [PATCH 485/793] Make GPTQ test less flaky (#1295) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- integration-tests/conftest.py | 13 +++++++++++-- .../models/test_flash_starcoder_gptq.py | 12 ++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index d2216241..35c8faae 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -24,6 +24,7 @@ class ResponseComparator(JSONSnapshotExtension): + rtol = 0.2 def serialize( self, data, @@ -58,7 +59,7 @@ def eq_token(token: Token, other: Token) -> bool: return ( token.id == other.id and token.text == other.text - and math.isclose(token.logprob, other.logprob, rel_tol=0.2) + and math.isclose(token.logprob, other.logprob, rel_tol=self.rtol) and token.special == other.special ) @@ -68,7 +69,7 @@ def eq_prefill_token(prefill_token: InputToken, other: InputToken) -> bool: prefill_token.id == other.id and prefill_token.text == other.text and ( - math.isclose(prefill_token.logprob, other.logprob, rel_tol=0.2) + math.isclose(prefill_token.logprob, other.logprob, rel_tol=self.rtol) if prefill_token.logprob is not None else prefill_token.logprob == other.logprob ) @@ -148,6 +149,10 @@ def eq_response(response: Response, other: Response) -> bool: ) +class GenerousResponseComparator(ResponseComparator): + # Needed for GPTQ with exllama which has serious numerical fluctuations. + rtol = 0.75 + class LauncherHandle: def __init__(self, port: int): self.client = AsyncClient(f"http://localhost:{port}") @@ -193,6 +198,10 @@ def _inner_health(self) -> bool: def response_snapshot(snapshot): return snapshot.use_extension(ResponseComparator) +@pytest.fixture +def generous_response_snapshot(snapshot): + return snapshot.use_extension(GenerousResponseComparator) + @pytest.fixture(scope="module") def event_loop(): diff --git a/integration-tests/models/test_flash_starcoder_gptq.py b/integration-tests/models/test_flash_starcoder_gptq.py index 608101fb..5e448d55 100644 --- a/integration-tests/models/test_flash_starcoder_gptq.py +++ b/integration-tests/models/test_flash_starcoder_gptq.py @@ -15,20 +15,20 @@ async def flash_starcoder_gptq(flash_starcoder_gptq_handle): @pytest.mark.asyncio @pytest.mark.private -async def test_flash_starcoder_gptq(flash_starcoder_gptq, response_snapshot): +async def test_flash_starcoder_gptq(flash_starcoder_gptq, generous_response_snapshot): response = await flash_starcoder_gptq.generate( "def geometric_mean(L: List[float]):", max_new_tokens=20, decoder_input_details=True, ) assert response.details.generated_tokens == 20 - assert response == response_snapshot + assert response == generous_response_snapshot @pytest.mark.asyncio @pytest.mark.private async def test_flash_starcoder_gptq_default_params( - flash_starcoder_gptq, response_snapshot + flash_starcoder_gptq, generous_response_snapshot ): response = await flash_starcoder_gptq.generate( "def geometric_mean(L: List[float]):", @@ -39,13 +39,13 @@ async def test_flash_starcoder_gptq_default_params( seed=0, ) assert response.details.generated_tokens == 20 - assert response == response_snapshot + assert response == generous_response_snapshot @pytest.mark.asyncio @pytest.mark.private async def test_flash_starcoder_gptq_load( - flash_starcoder_gptq, generate_load, response_snapshot + flash_starcoder_gptq, generate_load, generous_response_snapshot ): responses = await generate_load( flash_starcoder_gptq, @@ -57,4 +57,4 @@ async def test_flash_starcoder_gptq_load( assert len(responses) == 4 assert all([r.generated_text == responses[0].generated_text for r in responses]) - assert responses == response_snapshot + assert responses == generous_response_snapshot From ccd5725a0c0b2ef151d317c86d1f52ad038bbae9 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 30 Nov 2023 15:18:15 +0100 Subject: [PATCH 486/793] v1.2.0 --- Cargo.lock | 884 ++++++++++-------- Cargo.toml | 2 +- README.md | 6 +- docs/openapi.json | 2 +- .../basic_tutorials/gated_model_access.md | 2 +- docs/source/quicktour.md | 6 +- integration-tests/pyproject.toml | 2 +- server/pyproject.toml | 2 +- 8 files changed, 492 insertions(+), 414 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1f7279a..fa1a45a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,38 +19,30 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - -[[package]] -name = "aho-corasick" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -62,15 +54,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -86,9 +78,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -136,18 +128,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -269,9 +261,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "bitflags" @@ -281,9 +273,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "block-buffer" @@ -302,15 +294,15 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytecount" -version = "0.6.3" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" +checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -341,9 +333,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.5" +version = "4.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" +checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" dependencies = [ "clap_builder", "clap_derive", @@ -351,9 +343,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.5" +version = "4.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" +checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" dependencies = [ "anstream", "anstyle", @@ -363,21 +355,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "colorchoice" @@ -416,9 +408,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -481,7 +473,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "crossterm_winapi", "libc", "mio", @@ -557,9 +549,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", +] [[package]] name = "derive_builder" @@ -681,30 +676,19 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] name = "esaxx-rs" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f748b253ceca9fed5f42f8b5ceb3851e93102199bc25b64b65369f76e5c0a35" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" dependencies = [ "cc", ] @@ -723,9 +707,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -766,18 +750,18 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -790,9 +774,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -800,15 +784,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -817,38 +801,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -883,9 +867,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", @@ -894,25 +878,25 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "grpc-metadata" version = "0.1.0" dependencies = [ "opentelemetry", - "tonic 0.10.1", + "tonic 0.10.2", "tracing", "tracing-opentelemetry", ] [[package]] name = "h2" -version = "0.3.21" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" dependencies = [ "bytes", "fnv", @@ -920,7 +904,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -944,9 +928,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" @@ -962,28 +946,12 @@ checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hf-hub" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca0a2d78bbd80fdb804b213f675d261eb87060571e1c389bcad8c24add066cf9" -dependencies = [ - "dirs 5.0.1", - "indicatif 0.17.7", - "native-tls", - "rand", - "serde", - "serde_json", - "thiserror", - "ureq", -] - -[[package]] -name = "hf-hub" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9caf63dfcf84b1b62532c836035516f5b475b06c906da6f16990d22847aba216" +checksum = "2b780635574b3d92f036890d8373433d6f9fc7abb320ee42a5c25897fc8ed732" dependencies = [ "dirs 5.0.1", - "indicatif 0.17.7", + "indicatif", "log", "native-tls", "rand", @@ -1015,9 +983,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -1070,7 +1038,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -1110,9 +1078,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1130,27 +1098,15 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.3", "serde", ] -[[package]] -name = "indicatif" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" -dependencies = [ - "console", - "lazy_static", - "number_prefix 0.3.0", - "regex", -] - [[package]] name = "indicatif" version = "0.17.7" @@ -1159,7 +1115,7 @@ checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" dependencies = [ "console", "instant", - "number_prefix 0.4.0", + "number_prefix", "portable-atomic", "unicode-width", ] @@ -1194,27 +1150,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" - -[[package]] -name = "itertools" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.9.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" -dependencies = [ - "either", -] +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "itertools" @@ -1242,9 +1180,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" dependencies = [ "wasm-bindgen", ] @@ -1257,27 +1195,38 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.148" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libm" -version = "0.2.7" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libredox" +version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall", +] [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1300,9 +1249,9 @@ dependencies = [ [[package]] name = "macro_rules_attribute" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862" +checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" dependencies = [ "macro_rules_attribute-proc_macro", "paste", @@ -1310,9 +1259,9 @@ dependencies = [ [[package]] name = "macro_rules_attribute-proc_macro" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" +checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" [[package]] name = "match_cfg" @@ -1337,9 +1286,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -1367,7 +1316,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "hyper", "indexmap 1.9.3", "ipnet", @@ -1387,7 +1336,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -1438,9 +1387,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "log", @@ -1450,9 +1399,9 @@ dependencies = [ [[package]] name = "monostate" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f370ae88093ec6b11a710dec51321a61d420fafd1bad6e30d01bd9c920e8ee" +checksum = "e404e13820ea0df0eda93aa294e0c80de76a0daa6bec590d376fbec6d7810394" dependencies = [ "monostate-impl", "serde", @@ -1460,13 +1409,13 @@ dependencies = [ [[package]] name = "monostate-impl" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "371717c0a5543d6a800cac822eac735aa7d2d2fbb41002e9856a4089532dbdce" +checksum = "531c82a934da419bed3da09bd87d6e98c72f8d4aa755427b3b009c2b8b8c433c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -1549,7 +1498,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "libc", ] @@ -1591,9 +1540,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", "libm", @@ -1618,12 +1567,6 @@ dependencies = [ "libc", ] -[[package]] -name = "number_prefix" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" - [[package]] name = "number_prefix" version = "0.4.0" @@ -1669,11 +1612,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -1690,7 +1633,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -1701,9 +1644,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ "cc", "libc", @@ -1820,9 +1763,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "ordered-float" -version = "3.9.1" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" dependencies = [ "num-traits", ] @@ -1856,13 +1799,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", "windows-targets 0.48.5", ] @@ -1875,9 +1818,9 @@ checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" @@ -1886,7 +1829,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.0", + "indexmap 2.1.0", ] [[package]] @@ -1906,7 +1849,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -1929,9 +1872,15 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "1.4.3" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" + +[[package]] +name = "powerfmt" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" @@ -1946,7 +1895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -1975,9 +1924,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] @@ -1994,19 +1943,19 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", - "prost-derive 0.12.1", + "prost-derive 0.12.3", ] [[package]] name = "prost-build" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" +checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck", @@ -2016,10 +1965,10 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost 0.12.1", + "prost 0.12.3", "prost-types", "regex", - "syn 2.0.37", + "syn 2.0.39", "tempfile", "which", ] @@ -2039,24 +1988,24 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] name = "prost-types" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" dependencies = [ - "prost 0.12.1", + "prost 0.12.3", ] [[package]] @@ -2120,7 +2069,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e2e4cd95294a85c3b4446e63ef054eea43e0205b1fd60120c16b74ff7ff96ad" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cassowary", "crossterm", "indoc", @@ -2152,12 +2101,12 @@ dependencies = [ [[package]] name = "rayon-cond" -version = "0.1.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd1259362c9065e5ea39a789ef40b1e3fd934c94beb7b5ab3ac6629d3b5e7cb7" +checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" dependencies = [ "either", - "itertools 0.8.2", + "itertools 0.11.0", "rayon", ] @@ -2173,43 +2122,34 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ "getrandom", - "redox_syscall 0.2.16", + "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.9.5" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ - "aho-corasick 1.1.1", + "aho-corasick", "memchr", - "regex-automata 0.3.8", - "regex-syntax 0.7.5", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -2223,13 +2163,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ - "aho-corasick 1.1.1", + "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", ] [[package]] @@ -2244,13 +2184,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "bytes", "encoding_rs", "futures-core", @@ -2271,6 +2217,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tower-service", @@ -2290,12 +2237,26 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "rust-embed" version = "6.8.1" @@ -2317,7 +2278,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.37", + "syn 2.0.39", "walkdir", ] @@ -2348,11 +2309,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.14" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", @@ -2366,50 +2327,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", - "ring", + "ring 0.16.20", "sct", "webpki", ] [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", - "ring", - "rustls-webpki 0.101.6", + "ring 0.17.6", + "rustls-webpki", "sct", ] [[package]] name = "rustls-pemfile" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" -dependencies = [ - "base64 0.21.4", -] - -[[package]] -name = "rustls-webpki" -version = "0.100.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "ring", - "untrusted", + "base64 0.21.5", ] [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.6", + "untrusted 0.9.0", ] [[package]] @@ -2450,12 +2401,12 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring", - "untrusted", + "ring 0.17.6", + "untrusted 0.9.0", ] [[package]] @@ -2483,35 +2434,35 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -2553,9 +2504,9 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] @@ -2625,15 +2576,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -2641,9 +2592,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2655,6 +2606,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "spm_precompiled" version = "0.1.4" @@ -2684,15 +2641,15 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.25.2" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -2708,9 +2665,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -2725,9 +2682,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.29.10" +version = "0.29.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5" +checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2737,6 +2694,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tabled" version = "0.14.0" @@ -2763,26 +2741,26 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", + "redox_syscall", "rustix", "windows-sys 0.48.0", ] [[package]] name = "text-generation-benchmark" -version = "1.1.1" +version = "1.2.0" dependencies = [ "average", "clap", "crossterm", "float-ord", - "hf-hub 0.3.1", + "hf-hub", "ratatui", "serde", "serde_json", @@ -2797,15 +2775,15 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.1.1" +version = "1.2.0" dependencies = [ "futures", "grpc-metadata", - "prost 0.12.1", + "prost 0.12.3", "prost-build", "thiserror", "tokio", - "tonic 0.10.1", + "tonic 0.10.2", "tonic-build", "tower", "tracing", @@ -2813,7 +2791,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.1.1" +version = "1.2.0" dependencies = [ "clap", "ctrlc", @@ -2829,14 +2807,14 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.1.1" +version = "1.2.0" dependencies = [ "async-stream", "axum", "axum-tracing-opentelemetry", "clap", "futures", - "hf-hub 0.3.1", + "hf-hub", "init-tracing-opentelemetry", "metrics", "metrics-exporter-prometheus", @@ -2864,22 +2842,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -2894,14 +2872,15 @@ dependencies = [ [[package]] name = "time" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", "itoa", "libc", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -2939,18 +2918,18 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b515a66453a4d68f03398054f7204fd0dde6b93d3f20ea90b08025ab49b499" +checksum = "d9be88c795d8b9f9c4002b3a8f26a6d0876103a6f523b32ea3bac52d8560c17c" dependencies = [ - "aho-corasick 0.7.20", + "aho-corasick", "clap", "derive_builder", "esaxx-rs", "getrandom", - "hf-hub 0.2.0", - "indicatif 0.15.0", - "itertools 0.9.0", + "hf-hub", + "indicatif", + "itertools 0.11.0", "lazy_static", "log", "macro_rules_attribute", @@ -2973,9 +2952,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.32.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -2985,7 +2964,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.4", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] @@ -3002,13 +2981,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -3045,9 +3024,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -3066,7 +3045,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64 0.21.4", + "base64 0.21.5", "bytes", "futures-core", "futures-util", @@ -3088,14 +3067,14 @@ dependencies = [ [[package]] name = "tonic" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c00bc15e49625f3d2f20b17082601e5e17cf27ead69e805174026c194b6664" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.4", + "base64 0.21.5", "bytes", "h2", "http", @@ -3104,7 +3083,7 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost 0.12.1", + "prost 0.12.3", "tokio", "tokio-stream", "tower", @@ -3115,15 +3094,15 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d37bb15da06ae9bb945963066baca6561b505af93a52e949a85d28558459a2" +checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" dependencies = [ "prettyplease", "proc-macro2", "prost-build", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -3152,7 +3131,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "bytes", "futures-core", "futures-util", @@ -3178,11 +3157,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -3191,20 +3169,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -3212,12 +3190,23 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] @@ -3233,15 +3222,15 @@ dependencies = [ "smallvec", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.1.4", "tracing-subscriber", ] [[package]] name = "tracing-opentelemetry-instrumentation-sdk" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "752ddd669b14a08036a89045e4ac4497ff7ce254e11386647751f36d7c7849ea" +checksum = "f523eba1b52bb854b804d43a039aafeaee5a623015065adbfef8016825319c15" dependencies = [ "http", "opentelemetry-http", @@ -3262,9 +3251,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "matchers", "nu-ansi-term", @@ -3277,7 +3266,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0", "tracing-serde", ] @@ -3356,19 +3345,25 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "ureq" -version = "2.7.1" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b11c96ac7ee530603dcdf68ed1557050f374ce55a5a07193ebf8cbc9f8927e9" +checksum = "f8cdd25c339e200129fe4de81451814e5228c9b771d57378817d6117cc2b3f97" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "flate2", "log", "native-tls", "once_cell", - "rustls 0.21.7", - "rustls-webpki 0.100.3", + "rustls 0.21.9", + "rustls-webpki", "serde", "serde_json", "url", @@ -3377,9 +3372,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -3404,7 +3399,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d82b1bc5417102a73e8464c686eef947bdfb99fcdfc0a4f228e81afa9526470a" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.1.0", "serde", "serde_json", "utoipa-gen", @@ -3420,7 +3415,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.37", + "syn 2.0.39", ] [[package]] @@ -3453,9 +3448,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "8.2.5" +version = "8.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e7dc29b3c54a2ea67ef4f953d5ec0c4085035c0ae2d325be1c0d2144bd9f16" +checksum = "1290fd64cc4e7d3c9b07d7f333ce0ce0007253e32870e632624835cc80b83939" dependencies = [ "anyhow", "rustc_version", @@ -3497,9 +3492,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3507,24 +3502,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" dependencies = [ "cfg-if", "js-sys", @@ -3534,9 +3529,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3544,28 +3539,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" dependencies = [ "js-sys", "wasm-bindgen", @@ -3573,22 +3568,19 @@ dependencies = [ [[package]] name = "webpki" -version = "0.22.1" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring", - "untrusted", + "ring 0.17.6", + "untrusted 0.9.0", ] [[package]] name = "webpki-roots" -version = "0.23.1" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" -dependencies = [ - "rustls-webpki 0.100.3", -] +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" [[package]] name = "which" @@ -3651,6 +3643,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -3681,6 +3682,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -3693,6 +3709,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -3705,6 +3727,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -3717,6 +3745,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -3729,6 +3763,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -3741,6 +3781,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -3753,6 +3799,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -3765,6 +3817,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winreg" version = "0.50.0" @@ -3775,6 +3833,26 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "zerocopy" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "zip" version = "0.6.6" diff --git a/Cargo.toml b/Cargo.toml index 6b26bb0f..f594e0c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.1.1" +version = "1.2.0" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index 0fa7a538..c8de594f 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ For a detailed starting guide, please see the [Quick Tour](https://huggingface.c model=HuggingFaceH4/zephyr-7b-beta volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.1 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2 --model-id $model ``` And then you can make requests like @@ -76,7 +76,7 @@ curl 127.0.0.1:8080/generate \ **Note:** To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. -**Note:** TGI supports AMD Instinct MI210 and MI250 [to some extent](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.0+rocm --model-id $model` instead of the command above. +**Note:** TGI supports AMD Instinct MI210 and MI250 [to some extent](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2+rocm --model-id $model` instead of the command above. To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli): ``` @@ -106,7 +106,7 @@ model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= -docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.1 --model-id $model +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2 --model-id $model ``` ### A note on Shared Memory (shm) diff --git a/docs/openapi.json b/docs/openapi.json index f2619a95..6a15ae66 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.1.1" + "version": "1.2.0" }, "paths": { "/": { diff --git a/docs/source/basic_tutorials/gated_model_access.md b/docs/source/basic_tutorials/gated_model_access.md index da585039..e1abccac 100644 --- a/docs/source/basic_tutorials/gated_model_access.md +++ b/docs/source/basic_tutorials/gated_model_access.md @@ -19,6 +19,6 @@ docker run --gpus all \ --shm-size 1g \ -e HUGGING_FACE_HUB_TOKEN=$token \ -p 8080:80 \ - -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.1 \ + -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2 \ --model-id $model ``` diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index b0a77599..80561c27 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -8,14 +8,14 @@ Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/ model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.1.1 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2 --model-id $model ``` To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) . We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. -To use TGI on RoCm-enabled AMD GPUs (only MI210 and MI250 are tested), please use the image `ghcr.io/huggingface/text-generation-inference:1.1.1+rocm` instead. For details about the usage on RoCm, please refer to the [Supported Hardware section](./supported_models#supported-hardware) and [AMD documentation](https://rocm.docs.amd.com/en/latest/deploy/docker.html). +To use TGI on RoCm-enabled AMD GPUs (only MI210 and MI250 are tested), please use the image `ghcr.io/huggingface/text-generation-inference:1.2+rocm` instead. For details about the usage on RoCm, please refer to the [Supported Hardware section](./supported_models#supported-hardware) and [AMD documentation](https://rocm.docs.amd.com/en/latest/deploy/docker.html). @@ -87,7 +87,7 @@ curl 127.0.0.1:8080/generate \ To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. ```bash -docker run ghcr.io/huggingface/text-generation-inference:1.1.1 --help +docker run ghcr.io/huggingface/text-generation-inference:1.2 --help ``` diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 9303ba52..75411131 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-integration-tests" -version = "1.1.1" +version = "1.2.0" description = "Text Generation Inference integration tests" authors = ["Nicolas Patry "] diff --git a/server/pyproject.toml b/server/pyproject.toml index 52431eea..b96c1204 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.1.1" +version = "1.2.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 25b5f8194186f651111dac8a0bfd3d2a1d244c40 Mon Sep 17 00:00:00 2001 From: fxmarty <9808326+fxmarty@users.noreply.github.com> Date: Mon, 4 Dec 2023 14:09:51 +0100 Subject: [PATCH 487/793] Fix AMD documentation (#1307) As per title --- README.md | 4 ++-- assets/architecture.jpg | Bin 342347 -> 0 bytes assets/architecture.png | Bin 0 -> 952555 bytes docs/source/quicktour.md | 10 +++++++--- docs/source/supported_models.md | 6 +++++- 5 files changed, 14 insertions(+), 6 deletions(-) delete mode 100644 assets/architecture.jpg create mode 100644 assets/architecture.png diff --git a/README.md b/README.md index c8de594f..616a8d60 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ curl 127.0.0.1:8080/generate \ **Note:** To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. -**Note:** TGI supports AMD Instinct MI210 and MI250 [to some extent](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2+rocm --model-id $model` instead of the command above. +**Note:** TGI supports AMD Instinct MI210 and MI250 GPUs. Details can be found in the [Supported Hardware documentation](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2-rocm --model-id $model` instead of the command above. To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli): ``` @@ -142,7 +142,7 @@ by setting the address to an OTLP collector with the `--otlp-endpoint` argument. ### Architecture -![image](https://github.com/huggingface/text-generation-inference/assets/3841370/38ba1531-ea0d-4851-b31a-a6d4ddc944b0) +![TGI architecture](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/TGI.png) ### Local install diff --git a/assets/architecture.jpg b/assets/architecture.jpg deleted file mode 100644 index c4a511c9db7a690b991e95e36aab4e1a5b5fef57..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 342347 zcmeFa2UJsUwmy6?6lq3OinOSpsE8hHx%KXQ%pOuw`m3==42m5|@b`CBm7YFA7 zPIh)~9_|Csg9mvIa&R3weCXg|@b`m%T!iV5S3;QggD*VD$<7J>?_YT^>Hwbo%r#ti z2on#mkB14u!^CI;VBqIuVfwcN_3p(`U}g z$tx%-UAnAsO;bx-M_2FWEtA`(ckY@!d}L#5XYb(X`NYfH$JfvQ`HSF?(6I2Aaq+Jc z5|iG%P0q~9&Uv5v;bUG=aY<=e`PYidhQ_Ammez0I+j{%@2L^|J4v$PuP5+*mots}+ zB(864ZjrXhJG+03iwS`I`>??8|30w)HZE`ynD#M)Q_A+oxS00&fd3#o%q+(*vL3o( z#AfY&_=L=}{YS38$|$U77n3z6@ILV9;ouXOn>V4~G06~=M*YbZW5q{lnd(~z#M81fnXvR}?B9}3>T zWnA-VQ>Y%3kt}zFsl|!`)LSqBTPIyE?^-@`xIdeA#P#A^@a?f)OmCpfECUE?pfX#L zx5;kP+YA83d6I6*8hD%mgf>#43?S2b8bg`wVgNCQObh@TN`^6jYmKe&?G6kB5KV{B zVNe2`9v&KvUQ2<~Rn^(3YOo13?J!iB0pJzTRR2II840gr0K2C|FEM~xX9hst0$>C( z`WXY*ysqNO0Fc8BfVc?2CkYGy!T{#q_|DOHiW$IkKY(tWq_Z)Ao(se-+J+qiX#D`} zeXzY5+goFMyK>K5>>1cSNZP~7zqxk-nFd{?MkI&CHlAC>H;eXWd6WS-y##PB41fhq zOZk1jT#bz!bHM*g-|`(OcGnWE2P|%)$!C3*UwvGmxZD$m|iCyy{~3}%6gRF+nVI%lby zjT9yO<=#~m)=EAVs+R9tG0*NXkDwJDVY-JD2iJHu(n}5fQbm7i>_p4l+UG3nsd%QY z?Sf-AZq#6Pp;`g(6iw>2fjNp>Q%sbg{Oh9B4Ychr;)8P9j^A|^$^>1GX)zf}JtRAk zkbave(c~uRcwO17v>@;BDwnZv!2aECwx>B@G;$IBsEWnNXEMOkSh9c2$@8Nd_Vvxp z^aE+FboOluY+<1Y_M9(fP3Fl`4aGZ_w z?(te727p5_faX82fXje#2HnF=z|(fo^s`{(@xX0ZpC0BG!T{F#z#!um7*HHWwOMUy zAQiz-2aI4|f)R{B3dtA@bc7~xluZUeH36fOCVWv6?I4~TgxFjLF!e{&5ps(58aE$8 zcB9ka41gC5bS^3`gTuL|*ofX<0Hc~bFoN0pOndXRx1RR)-JUJjvzdF?vWLO{Uw#2Z zoEC<*Vb>CxK!=7>^3O5=iF6J{_!sTv@FzJoWXboZRgqPr-@0atxeVEqKceg9;g2M# zw3NveItH^t8gjb2?UoYqPKDOjObc$ideepUk8(m7)ojbB~(E-e~8k%?i^0{t6Hwc6UU@)>KwTEwFe z1*0XF`fzjFSnjAC@SOA$^!>t-`<2zs_!_U$JE2KGh>F1@N-tydBis+7x68|?)kSLv zeRR&llgfR(m}Y#iV00n_@LDSoN~ny)++?mg%GUFX0h|SPM<=aT&=e)4ZJZ=xAJwF1 zw%@kSLSbj5h4_X6a5O7v#m0~nq5c5r@?Ik1AC!pr+rI6-vaF8|BdU?R@3Cl?hc=EO zx~r}>cQV7-ZreTB_iPaiY(j}xI!8D~1u3H_Xccs-k{tZ0Ie|Ur!pr)5-cIGr7Z`x! z$=IjeY~Lz#SL`TA)KK#45mQ}_?GXU?v%Q2y7 zf*v%iRubGXFCwOK&8OvdTvIx~mi0wA@)yPT>$;!kvHHcX2?SUW=IHDj6iMWz_1u`r zUf(#}adQt}K{oC1l*u%$ZNNd5^IHMtMs=NggqZ(bNQ>b%rdJ z|0)ajDrDa$Cbl4Ovl1CV7`_O)JW9OTk<^T5my`85uWz|P6yl7-k2^@U$zryPR6jsw zU@y@G9US-gr*b?i{p7eqvByos&=mi{_<4yi$xPbASV}kBD}K<}#GsnZZOv1COb7)@ zhC&oi9Ukci2OgCo(7!M!d&oRA2uCn=3mstq!<(3BHSS76>8Q1pNw0?Go%pe2>8&29 z%?6?d2xy&vhtcd^S+5p+WUv_CR4xD}EYqC2L#)+dN)C8dv?~J`eNVzS!FaZ6w67CNIwgW! zT#{Q6*Hl8v3L_!MM|%z4ICFg;4^*olOM<}MO~arBT3IuuCC{LaIT=kRyWHJ3EQv0( z&(5%NxQ+=1ZO!|IWY?&NtV!7pq(Oz--^4;jMozvw{rZWCCUBaxFYsLVKh=Ht%eYIX zN%T?p1I+NywBs)e-}pN35!*sF1gRxFLb7I#JZir_QLG zJFmF8=MCSDFwh!qktL-la76ljKQj}{_UoZKEQ0}X)?P%)kozWQh}w^!K;^UlDhB_bUK|6NfsdrnIrs@lBtMF2@FrhOi=|@B#nkg{ z6P!=^7w8E}->yQbMfY$5$C>+PQ5cr0H9ze(xdIDYi7kli>f^uY;WDav&sHT)Cz1o8F&E+r2hOGUm$ZL7zj8k7 zZu-}tE8-q4jc421W81DU^VFxa&+p0Q{>K&p@*5kLcA6-UjV>K^vf^=k+$=(Ed)4Z5 zA}_EN03NfTc+za|wE3l-I9MS5dgf8~3hv{^o#X*AGhN-aUvucm3q&^>_hlTNO&{d` zO%rJOF$@5+!2s%V=(cL(*bR6!{J)-K^yjBouPibESxO`QFqDr0bob>mfQctKstIi5 zD+AbJ0B|aY)vV~h^kq8G?7rCM!(@zJ2>Os?k=C}mq}aHQ26tl+`)rjAwM$6F=5$-h zh)2vwz|h;AvjZYJxe}F&0ne!qtNP1=?cu`4|CxG=A0;aP;u17;0ll?)&oZXUvR)F^EC z`7A^o_Vn)X@WNp1PY2SMk&#iYLJh8qFQVm)f=!I-04_i&o&Cd}#n{1+QfSXnhfv)M zo>M|P{0(Vxv3!C9kzzJ|B+4zZeOBA`cP}HGxJ#Ib62bJ7s6jNr0CTi)5-zaNRp2YS z`ru(IkB6aR{+SM{obwkpBQhBp^?PYoaVVj(?L?<=9t_wCh8IgIa%q> z#!&Lh4F}YtHus?F)aaYWau z%P$mP{q~=$U*jWB->D&bhmqj-2j-qup)Wtc&Qyp$0v@}sS+RvrI-|TS{X&G2h6h}Y zs)~yTEX*~2q;b4IapRMeUEakt6cbgu#JIv>+Im^;t&um?-vN+GYLI{bLHAM(-oT zr?f+`DzIrR^sbY?{H}9a@2cvz@kpqNb-zq(n;)|vKy9LBpdVl&U`1}>Nz=IHo+<7J zbJuzwIlWg{sWR+|C%*{fx@Z3Ic=*lr-#R&0JeaoJmO=e93nw9Tg}Dq3rJvwL96(+j zIV8?q)^+Dtz>-Jn@1#LK@w5--vFWgfdjg@qc6WJLULa>pLRms0jQg2~?msdco%~Jp zYoLppK1(K^w|f1wG}n&-oby7fMa+7dHuq*^r`5z9BATwSPj%RN)rJcDYgTl1z`7)% z-l}kb8|D7iz0*2!`@&!jOv+{(*Msaj_1Wh3``8{~o3eyyw6Mk7^pgH9hmb4`-XV=4-_{TAl#L3UPHYZg_Y zDFcH>kpVVz)5{MNhZ0HJUJk@uO#*j`C1-z0O2ul43E$0j&hsr#CK5UiD6PHRRe|#x ze*36>uHc4o!pXLg(iAggD{~R|Rt;`mE&V(5&I|Ywc%Zai&bNN&H-zim`?83);cYdS zh5Zr+dL50nOsOdf(RNzqViz+?D?L zxI4e&bj9Ng0H^;(Gcvn(F?XGFHxR0Z&a~y#y>`hc`?Bm^^iH%%bdxFgDG^$C*)2*A z%o*;6Mm>>q<8vJB^0Ut9_;4+0+wlB)uQrp*YP$BBZvssounA|t?Yp@9@i^04Mgwmq zH~_D|b8PS@+x*v>g#W8LOXhtSw2v_DP$;$_X{l^I^r(_tL%HP8{(gyYu?k1n%2>~; zy6@vR=v(*lz811D*fqcO>tjv^@MW8N|fpWUh>@-;p>gqat~X{Bi1f|7?xp8dN_{TS~h zq_dZ%Y_oXro2%4Uyl1su`@LU|!Kk2#+jjn&^>Z*e4(xiaS-rMH zx`ek{Mzd`{q0n+S7S@M;5HB8$Tzp!xX%${ae%PQUMr_F?8Hf(*hqJTZeV6*``PgVV z+J6UGb*_1Kt0qxUr7M5FdX)aQRWEX>ty>#MmY#qGZ!$+OSwT!|Z+;CGc!o$vOi}%J zO&9&NAroI5ZndD+(K}~%ymn}<#Tw^x5qCje1JZXQWTcZvm`9sgN9mSfUe>%4FDdrL z>DI(E;z&8~2#BTN*LQJcA2Tgy%w}5_`Bo|wHK_GimdjrqlPzT|KVk=j$2VcN{zHKbz}^6X zc+y9;eNP$+HqW-gOH`qesM^`uwUg8Dw2tQ8jln@@Df&aWxkdjw$Tp*G$Hl5GRf_(= zlNHw%DSf@_=*1E1*KxA3=sJq&qVGKuVV9HU(37WYl2t$4kmr}P=&7e_kuPTYwq24J zkrM0MLO#;T51UnsWzU7O$+Ts(kw(WL%)X_`6skM_3;F zIUQi10pI}_Q3>!OU?*_Vbn%51YtkAQ*~f6;95|P7ySPOmA78OpR%4BCJl^^HeC{9A zNr%2*0CrgNImPE&RtFr*&?PaU44_cHM&icPD#;a&@w*d)GsgJ&e5UVZsi8@wX&Ric z3&nN&@&V*=0w<}HTu*>a+vXFKyKd4);+vD!Ey*~&cc1u1=29fv#bjEltly}_$r)#& zmW#Oygmi@ht&!HWTs3j3IW66Wmho9$KMHPMG+>P*Y)YN|^*(~TWZWjT`bw7*l$HfgVmTI3BE^K-@XV$pMUsLkLdMbTeQ$w>K42_weC;u{ zS+T-F{G@kafXyVDEGhk51wuLPt8JjKj6Y1ZD(yLtAoXrR`R?d1^9w;jRZDLh#5X*97?Bmn#Io{-|psx2rOpGMDF*o1@duozE_;QYdGa9$tN@6n9i1zDQHtRU=6D zTRLa|KNN)jXE5=9>_H&iD39q4f~KwJl9+HSE*Gg1P%fcR=$gh9HaWzj&3=5oIPgM| zD3pWh@(9?#{xl_R3_g*v#gp7$fO3fZQ_mvDSV3hrewNY z2ieTr>s;{{cCPr1I!4v$QDMq`>n4B*IEyJB5kqzcm{aoCc1)k85f2E7<@iwsM0?C} zio<@i*7Fj)Trxwb*7orW#Il-c7u`~B$N8cl%JjPmaMFcAfoiM?6X-Xt8mq*(p(eJdIznDw~j@s@>HzY}8Ja zByPCntG!G;Z4`5kFO-`InMOUEyuM7hTp2m+PSPruDY0I5^g_TNf zikZ!!f}yZ^7dFYHcECOPa#h3;4yG)gI;T4sHoCKUqZ3!pT(Jg)UjH zaRbv72Uom<-A=Xw>n)yTU*Mg^I{BxKO)vnMueL^EG2wPWL8C*SL)f`F-J*}P5^T_I_p}{s zj|Xx?x~@An@qy#%f%mU1Hcib`y;U6rl5b6Y=ol`5UeVS)a`}Rdb?lXICu(ctPkI%O zRXm^S+DEo)@h)&HSopTiJCd}2Uk5CMn|46E)bH+l-Jp(%>F|&m@shYrbXIpmnP{q) zL)I)N9L6ah16cUAD*E5+2y;#FM;A?#5gXW6S-No3BQI%b|CAA{tYrqE7W({UgFYAK z^9&*s&$ERA9M!zCjxO7{ZANH0pL^c#ulSYbfBTYXK=`*M&)z}kRE#zq(X}Q44Nns_ zHTPEgOqdHR!SXsBbTU(|OB}am0Qp3t7h0OZF|@9(^mbH7-O zuUPj9S5eF(-!#Q&4odyF+29ZA6|lB9e+g1$VS!kX6A(IBaFV=(J%yYxGtQD=ceNQh z`UdnK{+E%Mu#0rCpswsj6y-emqTJa2QAbhuP^Q~*1e z)CtX(;TuYZXH$%UckfPW-z#lH1P`Yy^)Zz*~N9IW5wBy;;$Z@ zF1&7Ib^M24oLEUW8_3FVpdVmIo=>|4g*G8V(ClkoZU?^0b!$95Ny|z9YX3F1A?Z+7 zTSiez&t0GVbmdopz&c)le7F?`mAC0e9rIPZN9cS{C}O!JIZmVb>O3xsY~IAbZ|-LAGB!=X74zNqo|5r_*6=o+z?E3> zr;UE@D*#Od)Lwr+s;}T@m>C@F$avjP;8WyiOxH}RJ3awa`!+HoNEkdq+ z8-VU@0Mh+0nZP5jkydD7=TNE+vPeC$yA2DrN$N6m{Mv=^f`;c|?=}yg7Kqm9w-%Uo zI>rXJ3}N1)ef82G(;6k&kvIJ0YpWj3A-d&NHO|NtX?cU*_OI37*Rx%;T+M}`UFE)P z^ELmPjL={0j3Ud4YmGyf;n64iMRgyQ4;(}s)@d$FiwsPw+Z1`W4|iz#9v9(lc3#;p zM@Q_o{5tipcIWJo)XSX~M`ugdCD=*OhPO}dQdzRUXQ;Xvd5>V-EJ zuVi`0Vdj=4?_F~%VnF`RuU%)M=NY0J@5q!XX&y4VQDOR8{I?*|?VwpN%HqY?bW*3# zI#w+DBKZg30ZFcvqh;BH6#Mvv0Iv%_9Q$((Je;@H9?p!O`&IZmnQBjf=Bx1`WeID+ zaw!&Ouu}}cuqxO_@#{B^x`Sd}S0X+8=?N4x_v6*W=Q4z^mMrQ@bo5ySjJJ3BkYU;{ z8X6*RmAhegM4wsR@x};ILCHtlvneQLzX1ct5Tri^yVLx30^J^Aoa{`YRSPCx850CM z;wl%QC(Yqh9=Yo^>RcYDfjObkZC4cET6k$XdG`i%>e591xdW%HsuDp?)n;=CUcdBf zx$eQ(rKgu_U=snbUfg}vzD6|}s&(I~;!1)~Xjaf>n@UxFk|8lSFJ09f-aIPo`Hk8d zq4YV*%%hzUuD7d)FqyBW?mihZnVv2zbm+~+di%N5A}EE@ziK8I<;IZjaHKHm)Xk}U5s?n=tEkXysA&Ij;8 zpl1(Om)ixP(Kb259UmO|&$zklON!H0;*eZi8p;jMJv)%o&YSPN{Q5P&T$!2?8B5&n z%K$9zuDB#h3SD?m))sTIUZog@e5{P}B3m7r!;HScI_w|e=~cz2Y_0UIsd*j^-)&k= z44YhQGfltTKUq_v00Eg5*P;TS0Vm|$Civ|8O zNe?|v`+x>>m+bT(XwEeipDFGn|H!71`AWlh=j7nYqqny146P|o&lrup{0LA-XbCtw zoCsZISnLKK;bH{xP;Ck%EGPs80&F zZ}*swj<$fp4ILgaO#u@VVUUbq>J2>k=dI!L^Bj|HL*H90`HyEgE#t7=VS@|2m>fxD~)1ZqMU7~Kuyax zyp)nU3u%+(n)mLtLXbZWPD2yUorVz80@ZM|m+*veWsAgmD10uCG0H5bo?iV^S)F_48389zzhf%PWQ zU9f3^ENexD*pD|MufB2VrMrCKRX1dsFJU5VN1-R;tcqJ-(q+t%T76q0H#G=9wqpW5 z`Ck6cdv4Wg=7glft^DD-PJwEts!&Nz^?fb?#Y(TYq=r+!xO-DoVdzF*$&&BNx#gAP zckY)5$#V)DWd?qDnSR<3oW9c#$P}UhABq-Gk%m84lOQYV6Hi5{%~Is4`w*wJqNX=8 z&^WGo@yA6iw>5w1eNpLVn~7OT0&(QSP&I2YRRip}um$ItHxTc|lPyxZaD9d|{UeI_ z$`@R@i9R>JYP>C1KE^f&ihDBA^-6Fj{4j##O-r*Ogf^jH4w)gsBan-@HVO%H)U!iH~XC?S>ICp8+O3VQ#QJ!qyj? zudlv%<;&M?w9*uAFh8%=i2ig0vOKaUqy7twsS++ija^iM@NTedUTw@SCE{kZK2saI zE);gpw9{P+K`VUrIJ8yd;nDzdX!>FNQH{~QqA_P=^gOXj`$U3`(lL@eS>tvOM(!zS zlctSw7jKd;({1iZhbQ(Iww!__fT26IA8JN}1a6@7hZ3z!D8+;Dsg(_@mm8}0ig4#r zC59f3A|Fmxq4|AXDe9>Ss6J6K580wCZRHc{*j`y=<)zOY?+=-IO2O=UXmbUYaae=; zEj9W6&7~SK!>@hPnHPVV>-6a47DHarpJK1R7yR^L;Pz_^OrDyV(c}2Ya_t^C*!(T} z;ouXoN&R8%(I<7+FR6TIf`9IH-#mf4JK;R@;4$9t18w)IUn`w;JW;i}45y+RKZepS z7#dG|OT;yG?VJ`p2Bywl;CFejb0Eh;*ZLYmY~N5b=cPLA7AN06TmKql6U0jWSlKds zpHG{<^ulwF_`2L1b&hDO{;GXSKUV3e>NAmmfTyPY(_hO)qUtqRp58S>e5!fsfb*Hj z7`-WCoFk7|+fbn^&7^_hrGvO2N;LnI(`DOdxgB|n z+pk8=)n!)aV1Zi_UwF5bsx7#<`K@?P3Vg=y;=gU$8Ou#~NIXK~i274tSzGjKN7Sph zQLW&ZgaeQJL_6v9=jk*jqN~`H+3*;`4(c5@?6ZN+oJZ27kDk*g>XiV()+<{D`Mi}( zAmSLkohK>Y+I(}?dDN4{!I+F@?AZLO%Ws4A6C%A0wV!LWH`iWnyQ|2*oxIS_Z=5pY zU*%e3is&nd{y4F8Rd~hTy@^-+;1!W8Smv2S|C?L~$l;b+83Mksi+!H*x5Nu8VCwVO+;PIcF!DPxn=_9SO8#OpcJ~X6Xz2aez@Hz`iCbOQb2VAIca5NE`&6zi%aA*OdR?bC?Py-NtU?Xd;5 zQ>8DI0%UIyXs^BdT#cE$OQMg4^lESyNuQN{6{`+6#684}#!L>acxH}`{QQaG_U+R% zO)(<;+O~Ykd4JM)k-h9#vRDOgt%?1w6`PTVet;Utodgxon~C#D{FqWiqk&Arkbdg4 z)zO~~*Ulw*v=z2~$UIz19Tva#`n=E&$x}mk*>tv98vuO;Ukqe%ci7E zHlqukLcq}?XyUY5W}vEfpiK4MDHpMNOTIyshuBW*J~>IwD7|&fqms-QOQO|Ry+2L` zs-ejyAiV@3vk~K4ceNX>SS7;ocL&q1mGKMSkA{ymtryxJc)U-mRcS>XruXlwlPP`* zf7l_7gYda{TFSXiZiNj|(ZBjbx2EZWRl(aw=A}+NWB2@^-iEO0O z=IWs{q%jo@zHdXG$oeiNQaUu_PxJ_vv>D+I*7M*hWbq)w zJY_Tskjlh@AY~SrtBJ)gm}yfg14^QGm;xX`k>EypO_n6Cj+>GnPNx{?Espv8a#+tF z9lxr2U53S@dfGpzVp@)Aq5q#C1BJ4n|H2o?gyU@2!f(S$ed=zf==DWsC;5D|t#%NM zP~5GnQR}#PQtwrXtPCts(~xN!znn|a@%164l8tPp3ocrh46J-B9(vM*Op$B`z5weF z-8Zs46O<%8z5t@%{}o-^zdD=F)(Q5%r3yNbBRZI_--yFWz?B^t@Eq7i#s(&IM66-W!ed%W0 zJCWo8|Fj3L=jV6vljtT+E75uO$DJQPj53M*)`(P4Bg4kvT_xO$5wgGKy=_dm((%sH5S)G%X!|vDF%LI8w>+?h91S z3#)2<6KZ>V2TJlrs{)*T#k2&>*}UDDjmck3J325qAUB*gvfi_vE3$uazpC~F$@Nd? z&z7-PJM|!oBl>o!JyQ)HxhGGr#(bz1McxDZ_C^&vC`+4CmbwEJnd|&KFkTqn54(}I zg&pw_J<~8txHa2eJS1^ zYf6)KRPs+4OOMG!2~&l8R1SXpV9DDr@9e28{hhz)%shHzX%jnGbl`B~BT|7WJ|CaC zY4w>NS`l2=5LzSk$ca2XWg1e=>1=LXaoCcFQl%6oqxc?OsoJ$YdqjY z`3}j_d5Gs_Zm1=R7V{SAMTA-rXSOjPJ7z{&5U(6Vx7}kF13FOh@9f@GFJje>Q}XQ# zl6G*ibxjZK!V+f6A0F23@iIWsRb{n{5*KH02J5t-56`Q1_>rBZoNY|Bq5Nmm!dea{ zXt(ov2S^ZFCt6Y_wqi^Nn|vy9CQ3iNsZR|3%PCe#1t?tOJ28F9GBI8U6ROz47^wfL zc~RFAQ=@9LBbux(##-CG2~{I@)w^+!tM&by4))3O9X(u^E`}j>KSpJ-Em@s>M?kU3 z)#&LD_XphW8bIa!m~6W5FgRe}^^WQ=5#?4_nM<-Q78IGNmTB>bd+#4TJ@2%BZ}fE2 z?*UrKNq_lbi=N4$3>B3M+bjP?wi5CuX{FrXSrX5*qYVPZAzBug+uWE+O&c`GQsqj1lTxy;M-yLsYYu#75~y~X3<0SPIxPbZvRNU!NzE{(JM)8; znpqv(L5|zTX6>Ka55+CmQOdNKT9amVh^@0WKGZm8U;_*(irre!F=nK~Nw;7WPsHuP z5`m_XkW{q-8*8saZl`~os7zLTj1Bfke_^LSs{dL2G}4ZIm}p8u5;d{U)uhSh6v*A0 zF9yb(Zsq0{Ze@PrQC0!f(a)v1o-@zQbY8e}Oqdt&N7TcITc;IQc+q(dXjUpn0kX4n zMsfM@f2UBK z-pJ!$!uOY^O6u(wPcnX%SzBDrsu^`{hz~3NIMK(%NbJmmMxdK5#*pnMp?nGvPKQ!8 z32R_Vj&;mWur*|BKX#>9JY0Ab(ZyA3E-7bx-2S(E;}h4#8|>-OG4v>YI^;PB`^H3X zoQn7MwwhESn#=x+PRdk8MQw=YFHhlD3DHa{lt)bB2Y38fngNs%x)H4Jv&WF_+{g~9 z<9zf6dL5|b!cZhB1~^cZQLM02i^5BP1G}B0@yi?K%RAmXxXN^>I_P&IkW2)&=CvsQ z&}gK?=QCAg7m}2nPIsJ)++(Fgz9xW3j^I~|h!S@5MPMZwfh|1QAO7L|dx(P{VCPe&z zuN_twjR#lRS29=?x!-3+T;-(aW8qIcU|Fm& zR1_p3gXjd*rD^*V{VfH z$DJo0)SUl}XL`29qj3Sg)osYOIQH-P98o`tCb%AIkqA&wrDz6ft&f6f&|*3}$WKaG z=b(SWz^;oux0onWfATj_qVz$$2-SvSNcFDl+YDcmE-|+-C~@O(IQTn+^~#~j^rdq@ zAcs6qHuomMV8@h@LT49A*H0hCQ=Q16`JeOM1$9rQPBMT~r&rjIUM%U&PIF)j0ySms zX(dcUzWH=^7(t5xAl~`HTj>{j$%yc#22{C zirKylT8DdQn|4p=mOPrg$NNg_2A?MLZ$V(|KcflIc>IN$mH%aDa&;XhJ6$Cvg?dSI z_+kt~4ST(wI-FaRvOTK%!0Y(yZDG-@SU=%zw%>G&&oqX7%5=rDN6*bm8d&-YUHG`b zMOCz+m{dsg#rlg^2cmHEmsew^q7Ea75}~C6Zksj6Bg*Z{AIvXyalJe$g2*uD2CV*9 zP1FJNDgI{ZP&kL28z|@r7nSc-x%9x}kUk|RtgSJwFhe=pjbwNjjIG3dx9J?C6cu0B zh2`p_=^zdCG_ zMlt}Jxf|UQPHX(*feL7PK{*4EhS4Ygc-{ioee^acm`DP*_Tz3Q@R7{EXoC{3Pb0KeqG;C+q(ERLeri40&>5NuKkR!MKU;C8^|&^9;N>l7?^ z-YvsZtI?E@Kc3aF_ttw8zPHTwHrM|xlOk7wA)jjvwN22fB5)cN$G~OHd_$BIAwT7- z{iyShF%%zI5CdlPZP4Me8rJBk6g;a{d*E^6$uH$$<><|@;Yp*_)oO2%(~w1FP73$! zHg0|WnxOUG8Rjp)uAKNT{6R}Y^33sPTStJUDt&XxU1#il-SkurU7^U3oBe{FrGj1y zl_n-rn+{u7U*AedYi);v@IFrFqhXhM=6XKEwvDGx9s1*8G#P7FVMJ-}f$xqKJ^!A> zuJU*q%gi*n>%4sk5@Ya zI=oTZs)D^4YQ7j#U;Zi9;OP!I3y0$Us}GGx5vF2DgU!|25S;sW&tD2Y+G+76gT$xU z4U4y%zR2qrd8&Pu?<230rP9egzx$WpIgDgID)~^Mn?fty+7_lcwRe5h;7(DrjQE|8 z&)Txe+xgXc_*pR8|FUcYQiFhLP2=}G*%>8A_&7DyCnM26oj_H`{&1(tzn$saa!{yh zH6c6JI*7{O6LB=he%(fMBH;w9FzT6Xyi+AhX~%NetCICXn^!5qiN}HvgE;**58?6) z?H^2q+d{qo1gTl*LN z8)+PBovPbOJJPNF8rQXjf?K9-ZB|HgJoL29)AhJXw;Ss5 z+WNWNIN@kIFdvhp0-JDy`krXIJ6~pP5pwb?%H}l%Gp~dl@_3iT9&b$YwO<-U_$)F2 z&5TvU#zv^n6%iKymft@7e5BaY=6+)1A+F+Y^(NXMC7u}>XP?pR%+fuNy>_bKWcat5 z3N5{)Kk7mvPvxY|3g ztpd3-_y?I_fVflh2bqxQcL2>aubRL9>&DqKnf!vV_}6JGY3Jt7#w6cZuJ|uOdfA_X z^jm-5ARSo+x;^`7A7Ykg2+x+0@&Plxqs>$1+($BYmZt%~ZOom4K*X;7{Nt-%(xDji zM3PnUxn#8@#g&uOPJHySmGi_%kmJj$H0vqNu^O2? z&mrU`<^Wx+yqkb^*TP$i~ZZr&u`e&v-ZznbI1o0D42Z&+>I zc2P`k{Vc96hgbKKRLg$087aVDVwpzJXvURjH>W}-O{KRYyYV{Q5Lla`LouU_@fwj`~esnpa4iV%||BmLb6T)>b5xg zgXgEXmGYSG=~=l*?%|^k)NgSu=c&Up+=A4Yh|=IR2dBXZ5|OCmj`pErz76E00uD7I%24JiRux4{}l(dVhoX{|vdXHi% zp!QqeoqQ5_I!8UU-3lip(A&$BnTz=`4Q4g7i5d##?cV&SQJ4i5ZOWS!b6EVH{a=l4u|aeYz^1LZBVLsVW$>$q(W$n8!Bl-ALj_|3L{ z-~GL2&0Y2MVS5G02Ry}4!OnVH+mN2u3HrYx;Bmb4@USV(eS^DV|JIo z+)*LrI=$ZtinE37*T#}dzN7ONS*CmC%+;)ue)ImGk|4-LEL7DBS3vfdqr2AM;Es?caiVz zt|obX)P9(4S?^}!T5mQrb!}9TU**JHqO&mYSYYBH^zI+pT2Z>K$W19h;8q-NDDz5|>sRfmm6q3trtf!^qPhA`rPjp|{ zeQ`|qU=+tEKTDKpWPGHnWeL-66v&f>K1a#;3SB3MPN*7MIS6LkWId>Q6kgxhd;%;0 z$ltf0^iBMr4x1^+s>t|O&p&#h^L_S=McZRju@Bqn(7EmU%e7e)IMu&rE9Q{>`=?z@ zzR42(=k5SM2FnE!j-)}5QKglT=jyNuI7jVe>M8P2wq`z6f!OjkKQvPJWaf3gRlL}0 zJLG}Rw{3wz3H{|KT_1emH;|KkF?N*oT5HFl zgMO_lp6WvQqSob}_o()?{r%NE*}jz&tMdL*g#khR{3oI937OFf9C--lYpDoFU=t@y zo^HT#S4(^(Pq&OTUA7F*R(CzWn)7{s1=vt2@qvzsA^QV$AYL8z1}%u5$cOW)Xy*M| zz%a|1OLIDKv?xDa8hupVZ#x${vJ;}h06d+hDJ_92V8rW!ADwO>Xii0WHVck8cvNB( zjR-lPyyN{(ql-_2F?3N~H`^e($clO-Uyodh{Yh0Iw`c$KecI$75hk}{-0{{~I``p= z;8)-6A2|o+dP*w7j-UO+w5vJA)q;Kv9h@{?xRdZneWtXotM|HJXy@k zwtF`h2+ai>n_=)}iji;!Aq=0d`P4%PC#?MfYsN#85c-&beN%RYMeA#{G-lUP^)zHT z8Q2*j&C;^H+=j7}%xcF8Z;RXxFI|3p8TBT~$geOj^1EE*TcN;fqeCIDPD6shfuiU= z7*V*x@u{vuNd3!a$!M+hbIv1eb#=YtY8EXkb1x!KX~AdmHtN%%cWEEsj(CtoEZPpv z7*Nr{<~dcyf{kaq*O`--KPxIOcFT_^nioRgq8U;(7Ywx!W6^v0N&B-ME=m}5>HXT zfh8DJjmvks6(zb3FWKbYM=>hi=tH8O7#6>u`S$Ibux%h*xt1SYe+wl( z;!u08pMKWN-anzaEHSHbkngsEtE)U)!Y#jOfvJ-Qt7GgVz_C~AqRNVY46G)}BEUvX z+^muFl6CI;@JB8to~eOyTGyW@Nzn%Y)Mxq+25=DRPBfqJ306Bn)#&*oS7T>cq8P&x z>lM~u@dC_gH~X~k5Wi0Q&IYQg{Rh|l7bwX7A*VpOXoYAyPUy(ZNtU$`m#JVCgB;g> z71bwXi=-=l!6_vk&kCy`g3k07U|+>sR3BY*UfcnD5Q;ovW}| zelV?WegE+pqv(qlBvxMnv@`$B@%u+otB^_js5zZOKAXx@TLVf>{3WG4Te?F+4RJ}^ ziT-k_3cRqTT#-Iy!)foSqo*M=U@=~y^a?lS?jn(TpS(6LNt3ufc=55cmyBIM_PB)P zZ}rDtnzYBpO4ZLA9d*A~>TZ;q?PcrnA+dD?eE@zRI_xt!P#Hru?I9;`xDHhlk6>Kv zz+$Rv``HE_vVNTj|9HMk)-zu9a^D2BE@FRm3M{A=QjS<{oottfWqV;tG2RH-RRmbm z1E0Xs4ndhgK!<}}4O-?&@v9BX@JVedFDM1NGJ&B=tiZ`*qP+n>EkdcF{yPY_siOj7 z>TVuZ3r5LKLCK>rEUiO|E@g3tsEm`<5~@>F-*+8aj5qH};$|1q%t?qL#!>Kfyq zwtT*kegs`_wJRMlkd&>-fG#~p?kJv&no@{k;oR6Ls3L^`F|BNA`4BN ziSR;`wOv5$T+0xE7jS$Ux~*pSivW#)6Rv~IhJ4V}#^mNm%(GHcmQ23h(t&U^Y2tMv zWLR8XRrxldxf9Npb2iEqndOj2AHop~V))HlRhH0`-2V@A-x=0qxMdrpi-3Ui5)=?o zx*$@6q7)HPkzS(GrAY?~2_31S2nZ^mfJlvW5F~U|dY4W>iqwP>0x90l)8?KtbMD-E zo;l^0Kh%8fec!$J+H0+??L{MGJF}LR;m4H3Hw8QOgT}k=%^HLH=z3T2p@FTK6mn!> zG6SDW2qfMmh)~#(^tW6ORHZ65_{2Iw%YBYs3Y^eZ>iRAw3ZesHOLlM>x)pzQz{po+;~SRLfqTvQdVDb>`L_3pwwtoQ~s?Di`}HYJb-z zw)L;mSW?M8tOPk6TbQW@T|us+d$!e|@aEhqS$Wo+yo6xb_)u055Y^>H=I8@bjl-UY zl(a*$+Z!_7GBYM|yPtDr4%9C^mUuw0LuP24vw0#*ID`@v``Uy@&rer;Kjp422YtR; zQ+R5G?d$o^N3NuSHyvM#hpX26pICmr^YF%mk~AyL!V&^ITvR+=ZS$rPh zFw@dxU@FrFF=Uqap&L3|&iW+`#UD$d<<%d84A|-p6+xmTchuC0ssY_bOwv{AwP|c> zC3TtS)%{T5OwN6R5KV7|2L5cZ%Yf;j6ojKP+tTD&kJbcC1nbxeKjXB0GjmktGy?Uh z9TaS=!a9nyAkLs7i7D5)6rg&aGBoywFRf4@d>!24~QD{kY((E{!vC5p|HJK zApUFF#+BNK%+q#=mq>eooEB3$C56`IbwY8h1`68feKDNa<6(P z6wU481@amEZlG@pIL?>B5ieJ-&Wu}qFHR_*7TcXZQ2SILn>OFv|E#I%bS`ra9Y89d z%tuUYLXS_E&(x9eNCE(W6o}k{bRdg0#!zcZ8~^wm{^Y0kMTJgT_H#8QbDfU1k7zPX zO*v%EFp+1f8aW1OX;Rk!8D6oo8o*(p6(m#^Bu1y z0S=Q(f_snU$ckTd`vktqT8D^*v-E9=+=$1mx=# zCl7yt?x+NV;Jtv-0rFn~%YP!JP~){IjJ%5jlN)yl`hghei8a4t!;a5tnN9aSM9n=8 zE@Xxzu1olpyHQ;X<6PtdUzNI${*V>zM7_sNVe+62cbaE#x2r#gS1ofp8&`GXtAm|K zOQ{xxRuh~FKY=@phjSC!^>K&oKzTd6mMuKw$jxOs-b>x}T{w*4;@Y=cm9iJsdrMDN z&`=-Kkdt7}xV>%!jZc{)J_*YoVB3jns5^I4w_vm^!t>>hSgnnlm{&iFiS}%s10&z3 zcOgN>e|*zmd7!PR4HrVRA&3_VSpiZVbo_K zSBKQ#H&mW3c1}O;`y>=vmM^D=^pdtEaeN=7FGIANQ$&C&1%h>KN3%$|U$G+8nG$cT zL1Kr80MZkFO5Y1I3kji_$LgV@A=pIH=?VBbtLcaa9Q>ngB<^ra>P%NO715OrN8lY5f823V|p>JY=|_FcUJ0bnm^>Q4^)Vu@>9)=A_r{q~b0d z(%WkU?n6a9i=A&xhq5$-16*1x{Fu?v)g{()JP2?q`&--XTPeYPAh-m4EQhU0i+BS* z%D11Ms@Q8#ifB{95D~I3hv$@=Ue+E5A~Ow%_iI4#CU`%jDmn!12tHvRVB2P1PvcSi9IaQJVGeJL+Sn?K8Z5~vB{@ZOQwE3*lmE-I;e(`K?=wG>= zxqv>8ggf9cgq5R#e@YoI}XM{9!rYA4rw1(@Kat0ye5 zkfweU1n;8^4tGN~cX0AfcBYAjv>TG~g=a%6UW74&;Ai0j5C$2FD#;c*&_?97;5l$< znwycnO#AwYUG}wXC5Exu@Yct9pFwb30Bef|E5RP)0;VN!XjTvH2dl^laE*Vh_F4XZ z;O}0#sW10c6u`rjC?^exmkGArXvP}D_+u_4ZCxBmjD2=yrk%$G{gFnpNCF(tRoH8y zEG%*XQxC!-38KhO%dN}Z6bXWIpjWrS_$H)wGtXgB^$Rb%LEgCTYxZDa3lLfF53mJc z72ge1)4@(T)pP|)9Ujro-V{bRx#Nmw#oE~Hi`{rkUgSy%%#bV1*{w3U(T}folG;>+?4WE8*BS)DqY3^U7T`FaOJEMbTcdWgy6g^3Lv>^AQ(Pa%si^xGsP}9sCuebp z`#BxWT!gcQ)=kfX3tmnfDCjIY|L{+w0eY<6BDR!Q)Zg#Y=ObDbAoe*7P6+1x9ps$! zP?@g`M?jd~`Jt>8;C3cWDjK^`JZg+~7yFvLPUrIU*imo4+TfSE(;uvQzbb5dc;!lL zfOx!~z=w8PbIY@#0;dCVS-T)WH(>MN9eiM(uWY-C*;P_ad9l&9*vz6s*HU5(KXCGy zuv>X<@R0Dmkar;vz)IFOC-!h2*pZGh^a0c6^_PL}t-YJ7QSvJpvA(KmDG@}C@Xe%O zpov=`QYyIt@?*1E4DsCzvG#I}dVc_~Moxxvk`xJYm|2|;4Vs|?1ZP0FLHNBfl&?O| zwtYe&z#DZilm$zjmc7o+ADiNx1==S=@$LkD@pc41NfBEYu7G0sNl~fG^raoyiqOMH z$34FNLBsgM^~y-liOcb+NvvOxlZM?EsLffTnNtonCG~Obv!*$29L>#-72TpybJnZF z$|t`50=c4r4(Qih)-PbfMV(F*;Z8F|Yk{9B=duA-Fm^JH{-Q@AljJ6AhDK|~< zJ7`5=8!O>Hq6!s367zTY@nphee(JdXQ8dxMU)0VcU^E}98wrBz!VtLm=Li51nnD;g zer(`oF5jpC%ph>p5#55Q*TS{N8zGY{-@(sx)Yqu-N(=OxM_8a)>5rAliBRLtpDvq7 z)?ZK~@;+}H zpgzzsN_@?hG2O6&RO^KI~Ruda`OOV=%T?dW29ow$CN+v>-O z53|7iw!z2p!IK$_$>9A9>p91__o)c&L>ZWH^io~72P5fnbBa^-)~%n*qjqsomb$ji z1pw5~Xdj>J-n)dl1tNd&0Fvh*1Y=?7{K7!Qodf~+{T81Obx3^EOyDcx&TOL^FpYf8 z;t%289tihSIxc?*_k-H9$v0v-*>oPB0cpITdh+iM#-0d)$UzqJA#{rK;b_ik)+saw zky-d^_B#FO`8p_JZwzzvu3qH*q_ONIk90!g`P+Bt*Bm#0;;OXBZ#7N_AQls2MPd$k z)IMJyvV_<+BbT!|vY!gD=kMN2M#qMzU;KA@{v#R6NnWBkdUFFi4q+l(A=~2S|b8M3hlR^lj)Ij<*&=HCPK#@d_=?z&+_#lMe$D=N|B61?yr{rKxlxB5n zFE5zf{c@~c`ABlf$JvdTvq>tln1#=)4UW2P0GI7)` zsk6|4JI4-J*uVVHLD-q`R&TtcQ|shuw|y-dOGRrsmS=8^%xtpnMNS}qJ`lT_49>rW z58Q#In+wjWwdX z$O#-@PCg#pInR!Irz0y#Q+uxi9Z5q&q+>{Sv!fVH@Zxce32 z(O#EL{K|=Hvc;&&av&BBr+$oc?!oAIZOWWb#6XUV`)_f2!K*SKNInG5 z!tgrN^qbtXtu>Og^!Fu#b=hXaMDWvn79L zL#<-t*s^tA;Sze$^Fg<1(fC|6ho`rv?w$!23pMW^parSVzg?~sVm^Q6D0ExxC+OPu;@iw19|-o2;Uo#hPvG}h zEOYPvhEq!Hcvd!d(2-=m1+fn~1dH%8EJVEsKLq3i(EqfJujsK~m@;SKq;j~rMhk*7 z!RMhh+mS>|*o|Rd$Vq~bApZ^Q=W4u((xDs*pPiqgQlWGzzqqTwTR^k6U9{IM^U4RH z*d?(eySd?PILX5O4kMZdti1{3@+Q8HaI8ou|bQ!sfe%Yc2Yy)(zlcPil1zmt3z04k+EQ*W*(S0 z!O$0%+F^5Pco~cR=rSvBCi7Or%$C~V6qmb|=8?MRsD0CiZ8a(>wn0i_?P+tQ(^0%wJO}IHb%Q1#HHAF+J+O?zcn{~-)7H#RKw%R6euyPTJ2lH(d;R$8 zqTMHtR@*v&OZN-(7{Rm-1_WKMNY;&KKOQf%7F!V-AHbf9#$?&MKI4yy@FuT=$Q53# zJh`+VpR#&ug8OqK6>`oHUUg+@142@TY>_d=3sU z^jsN&J(3`yL?RI80U}Bqf^r++k@EsIfdSyC`CGf+wg~)1>ILeIQS?K@X{dSgs|RBg zPUyVNOzly*H0)Yok7U_;|Ac78;rS4VF?l{2;1)^IA=%GX3o*kQ>fay?)2LVp`WDp0LLX(6tx0Fk(w zJ9dY}kL|H;{ygeKM z3nbryJ3eoCaPjD=`JEM-EzmxKAPVi2db~T|g`oH4ugz*V)UBD76Z|Ucfsn?W5qfUc z@tub%F1yr;dKFEOc%30bv79f)-{NXr>_uU49byLk8 zIBMte>^bMs-PE8SH0P&ovP9Yv@4ukA;+IC_4;rM~A<1`>&b*?)U;y(evJ(vmu6LJXn zGZbRFh7^q_wzqR@T|PLTpIwZ0agUr0^c3eE0QgdS9QYr=BB~^${Q^A#+Q08?b6IEl zR2a5=h?Y*;>^%1sg+eX>gL46p>yX6Bfng;HL%?~sl^nO_pK$dm?uL7j+=E+*7@g5T zd5)QHqg027ErwI7w{YB>!G$nyoZ@_@c(e}?TWWXbRcFo5>OmInzc2-w&jzOBArM)J zJ(Bv#eD&d$OTkRQTa8J<6%-`4={nIX;fz<{?j`D+bUkx`hE*FD0&!DZ&6Z-lmon;L zSDl#$-lDlm28gFiSa5Wc=wDX+Dyg?_2uw&TAnJAp za&KnfPD)+7Vl5uJGw%E4YLa;^CVApDK>7oyik}!($VI0T*&DfN%^dCXcQiS{Y;#{5FeGrCUCnbngq>#4G?H`2onFGW`6; zd=+|&V-YcFyLdatF50R&W%!IAc}As!m*MVf=F|>ZenM<(BEY$;`A=!VzbMIv%OaKV zIC8Tz_-=j5Qd`Ui-3KT9-bX&Z(*d5{dv3LqH&apS@&NzT>-?Q>s~Zur?H(sBr#q5w zvPaywd9pb7A&4GCHPsaUGWc)6=zr1U{LN*lmLULVq}39P%%O!{!yX5W5U=aPr}KK- z$=YvEOEEcqw0-#&2jQ!cy%phS7=dkSj!|c_Y%De zgEAwwFaH9)kDCIK9{`zYV!#Q+b3{dV$0c=$PF3@$g1(vBwI>r))D~s0e=7D4RRvFQ z#d$TsdIggl{TS#MZvF}F_dv8>YElJo+NH&nyai-}5LF$HH4gO%7~;(G0qs1(bMw@^ zOX!y2_M8!aV6nX(_vw^oqj#Agh;wz{WrHD~4Fj;w$X2POq;TTY_&6|wo#*V$lgVAN zCyI+hPJ{p+keXahf9{XqGVll3<(*dKvXEc;`Xt0kf8d&h@7D3Vvq!o z_dJbjImCHqI$VEjr>wxm<20cb!8Y3|Jdojh2I=$IJ(oH8Ir=pH6A-yE_Y_!&pks&w zyw^zA7Tg{n-V(xZe;N_Lx|VulC`G+%6X~f54$Ef8Wjt>_1=wR_mn_v(606?M6I70r zY`?L3a%Zz*m{^O>P#r=@?5vo?2VTa0@w89GI0i zOF0Zru52kJbt$SVJ1}d!^3fSMRWkZ8yyiJo#;L0cRNn)-yEJ$R2uy-bmyP_m+kP4N zR%p5OcA@J<;r;DiLn>{(KN@W<_h$lmvIzNuS|Bct!iJ9WJEp;FoSUFc3@upaGxI)k zzo(K-G`X_jtgiHA%MlU;R6pHY4C2tqk;VJ7x+%BSI7?#$&lFb$M_DB?Ugh0NJy7@! z^qnI)R|3{Yd+pHEH8~1vLrt|RA7hwgtKXcw94S!SW^$XwFe4O*;(iKlF1k3ORA-bE z-<$G0z|pCWtIn^oG}(avR7sw+)(O>E(2+Rhq(1Qmpj@hJO)$jVc~h(<69;p1myFDJ z<=yhZd(8Ie7bZKu2{1Hs<|!Bha;jfh%ZJZv4{CMhQRL9*z*w8WqjTooBb4TgX2>Yi(^RUg;)Q=I_MwHn2T=>QO>SI-$XwGm+6GLPi;=7|r2 zdg8gctOGIx3CNZ&cFBz3?T=-`tbBG{6x28U6k1bKB5`AU{O)vb<@2U7RSR25=QOLW zlhYq{&rrPrGzrlJ=i=)CRn-qa(etrE*N2Ib$NN$VW4RpV3%O;$OP9i-DG*KKL}C2i z=96w?Cpvp9ANzbp+~zarjlHTr?dQjjbi=7^36e7Qb6I_XQ^1ijJjaJCY-&KUlnF}+XvU1Rf%|2sJfLyPfDzJ1)TCS8ce067EbFJ4|$YWW3X4j?Y} zm1nw@2ZT|>lYRn&W1O8Mq~I8v2~f#SKq$LCc*}{OxhF0V{3W@4f~8D6EBRC-?A$AG+i-k>0-Zq0BmU#i5-fe$8)#u zu#p@`%PM>qgWeYWENw4HtI%*M+7FB|+Ht=?y^}(HqLhl}Vfb7Kgr@m?fNcj9Ko7Pq zAqp*~N==&?aiSL4sVhJ7KX%IxIIX_DICNyUHG86({Z1%Dr!&UL&=6fcXo#EaBbFYS^518OA3z%DAo?x{N$$+01fxF?D+T zdt+}BIUm?n2!r%;S$k#Z5Un?Vfu5jZpuAErk%bE|Rk6C5gI4E!=ZRy}my!!9R>H}P zK7I^2mn%FlwtyG=pvP3HEICu-LbPxl>72@YT^<^8CK0Ic0GUZ4PoiS`sZGZn06BPqj>@?f6Y!m5lLAOGtWfb1<;Zr-9QU3i~;KYN8{D z&%m8|>`nQbf@%H48{TTex6v_S)C5x&#kJ%#o9LegPkAE$E%Q#k8*YnOhNQ)U;8({Y^H}E`Hhd8xKYjM5{`4}b7{{7lSoamJD%gU?N}Bts|9D8 zvxZ4_PhP{_9uYOd8|Gp?B{|9TwiU;1#jo`PX(8 zOY2_X)%nPf_9-|pNY3^>389q&t!DSzfxtszWu$Asu8mO7N(N6$NDAT zy{A%Iw(|w`h72!I%t`Hoqq(s^p+HOD6&;z){kcA_KWf_IjbuTJ+S0T95s6Dl+M1mp zpD7&@TV7|j@NntDEIs}?Hof5e`f0Jgg5~TS$w8~@>%5|gH#k0_v{YJ12g*h9N-d~EaaCx?D9~l z3Ip1UTcua^O!NXlG;7o^8;tKyf95wI*+T6T38FQP)t@5mqZYaEqXeF}l;lYW7tx&7 z1*Dg0I535w$}M=&WmX^K*yU8+$G+~LGOs^YU6-YQ;R{c^l_g3i8W+ z0E_74vN_Q}K<5WEcIoc##ZL#!T|0?X(!_(v1UON`h6B2N>D=?RJ zI%o2b!dOHU4bV5i)^+^?(YpxA;$W7NYBw>H2f_EaXk5F!U3`i>XjlvxTJ63iJ^kWD zy>;u)#-C6d3u)*{J>RvpY#hS%;*vMe`%sN+wjdRxBy{kkuWc7zA#iNR3 ziLSsF`ZZ?I8LA@#qB*4{ryAhFIze~>+A%Wiz$|v9MW)WNv-Q>%UDJnq=RUVcD12!- z>N0x$Fibg@^%LUpFHpDzx+Q0FwLO~-rygcb2+dl$JGN-fZjf>LR)>UB&n3?I8=3d7 zO#<rvT6maY26HYn`u=iQX#6-Xn5!PTg?LviX?2!_W9N%x9tL#;&aYSy2>iS_nkx z0Q?<=pCy178BuDSm@Pyvs4J^3{J=XiNdUL3a)%z!x58h5w)F9?Kw26K^U3O;@1h9e z=1?^E!`X0GP|NMEhTp8jdh8+Fkk15RG`CyRilBoTqQ{#Tc?q#l11Iz>XIcPHNgc*mFN$ zlX7*$`*cbALwQ|lwPa0jdJZFG;bo3clVOPqpJS&XOI^4dqGCDY)@Xl$eIPeJwud`I zpyJx)25n?NO^96U*A^2{w_ykvdv_;f*ebxX)da3lJEC7ZPq=mrn?*jH2iJa`-F!-F+#>m21o75~nQ{=*O8K2ZD$}+F>OEp-4%RSdYP#h( z#u5>=yTR7xe>3E%mveR3zfwXiA`iXu5;4WCt)Yd63SrGbd7mi7>XHu8)~V z`i%G&JaMAB;MYS_?3H@syQ-bh7j=~|2GP^3-{#}MSOf6XWlVbWR9#yKl;@}4NFd%c z%Wc?Vor`$qZquD_{$ccTmydCSpocRspr;y^6wq@!wbh~k**QI{IM^xmKCi?|toj2Y zbjta<3Xxqim(`8@60!ggnGO8!;Oe$`#S%V8z7F%m1O?vg$dFVvZb)NIFM7@X9Qt%2 zZ6ySvzYxY`CQ@4)c`iznQNZb~@tIQ~&;m#m$ZN#6%!#bJ2W+bXwp za){|Nx$!9KzG@ib87ji;Z}%_smR2EQ$!esD1H0@CVYyjkD3tc$N<(4ys`*j$H$Odb z^D~>%^_>0QKuvBz;UKAxXmF4euoLaUVCPya{uMLwC7s$&PDG%DQ2Fv}Vnr@XWr&6+ z8HKiGWuS~cyV+d8yz8W)F#K4~`S@epGkq@q;B1VJl#t%VYcr8CyuUZNiZt35IgMO! zZ&V)$Yf@g+Er1A*sewyUq)!V$Mn8R{&-F3d(qxAMQ$^|yyIpz|aoy2PZpGRu$cxZ3 zPbgLzu@=b4=+fvo_nNNRu zpjRB-wQgO1gT7!)TNe_Pu|zs|7FYFnuT2nTCzX1~ zdrlO9;7R`lKKI`gCI5<2;S5zPEP!D29uP&}Ttr5xBqbnfONCU;t?aJ()dz_E0)>40 z&^>xWb&$Rd;LsvsKV?Lae3uL-8#@!k^6SnUn6!=ChBK2J*sR;=4<}DtQDySWWgX@F zOAPy~ln7|t{o~o=f4PF1Hy((&sb+*CpubuJG-6{FsBk+ZcBuz2bwUV6d#Q9PEGS9% zD#f9Rxth$vwz|24GUun7xIL!FxRr)E*y%|c zjc_hRW~Uy}9(Mb_-R5q0b9 zE70avS4P5;RfL6W=}x^qap#UKY>LFD$h9`!xRzBnH-bo>){l8NWh^o2>pYLGBn`ZU zu`LapB;6QX!m&p9I-$x0UObhryuMVd8@-~97fHItOT&TtKEn)aF3EeIkh4G<*&s^| zyla14^cb^sp%Jy=4R2s>`zz@FFQ4$AsfVWCZC}&{vf)by@_vyzqv`8Te<@YmffCNNE50Wib`ufapVrGbIi06!K9@y;;3Wjx_AH%nl2So{zGf}gu|u3aU5+CIFg@UDM<^SD_MRr6 z&wqti+?()uTfUYR_1u?fgX)7t&lC4eN7~XB4b%Jjp8d2ySqvb?kak1D zd1z>p3y}wyRf8rp4G}8$v_9bk!!4{VYoi@$E?P~^R!Q$&V7ry_MfBRK_y>7MQY#_B zia-kFg%iZ1h|(J>Qk0K}TGw%rTQzS=`sydAzPh={8ru{nC}gmxayheF4bsoP`2BD~ zOvC`f_W)|V2BD<<6}X06qM3SQ_i$^P{O+giiX$0Wz0)0BzoQAJqI?BoZ_7j@sv(;L zg)l1|cL$Q~XZAUq5`dLdE>%XcR+t5H%TJ5v@E4^l$xtg#i$X6&@`!kXNNWH{hWZ{& zVVuRP;tL3L0P9Y1KE_767RTqp{mg!>_Lactsh>Ni6H?BzcWgcFQWkmfr96(f@XDM2 zGB6woMoWMckf;eaG3W@3E1!!tuD}KBQh0WJ#Z9t}L{XQ52T$ZWpM1Ah!^tfbpTt7> z*Kd}8>Oub3rvT&v!qrXYF8Kw-9l{C=z;;%37jF1JC8XLxFQnX^Gs*MFF+Wo!_X|{* z1tNP4*Qhdk+^!n;DS6RQi4IdOaZXyciy4)d#S4cM25F1`*8>&9LPOmrEvZsmZ|n}Q z(|1{xc;gI_dVAr-4z{UR22Nihbsty_I+xTG`fN@VI~Pu_>FIv_A4SDKg*DT^3TuZX zb{UG^JP->u6hA@M-!z$-Vz{3R}njJTAbhB}{M@dO{O`qOXX3jRpEJjTOzuh*wAvGy)1QHze}Eo5*H}5GV9DzdxtYN%(XealV3*9uKn)7K+jxgnm=hwN z?rEY@t8%(@Bx1^>rp(`^usPdpe!_TX$xA>=O|szD`aRRnJ3;D6!7lriWZD)5018O3cocf)f1U$w)`}9Rd}JRSAZ?N)heVf zhw1rK1zbI21L_&$SBL_NutnHTUbpAT#8li{ex{dCK`b{MD?*dNx&LUe>Hj040|u&f z>>TN90AxU0|0wi+V*h6=2sYzjbS=6KIgzy{`OnRn{_m~EVzgL?1p5Aw2xhf=IIxj% z5Q4?oR3#?4aGmIPMeDh?<1LC1gyT^;Zlruuw^!#coWI z?kQA#;DxkzV&nu4L^k_-OI_^{04rMEsPRz*twnJ7ocNd`v(gHeFBo&w|=a@4&jW@y#bRZb40;(S1&ZlZ*C??=hMcrpZ zMVq7MbR(X2o^g$@N*8{xz`gigPP;r(Bucx!yzzR0#%t6u(9Y#!9@Y?D>!56rD(g_iEGAff#bU+S`ybp zU+KTSV3yVj;3Bn6)IbllkWg5l<;~qO5N^YZG%-w9{4S!J83NHmI~Y!;w&YaigetN|qgj0u z5cfuE1Z}Jx%zZO4>GqkJsB^`SV-=ZdX($3e>aMncgc|>a`PP4xAN{lII2h1*rPU^w z0)*g!O>Fj+N}zqodSaasQ7>(|>CJX(UA>OSPGg770$Um4l~ z@u+Qmq1$&OgC5i8hUuoN_qy$azz=}L=zleN6%xa>FKIa?TM=~Hl!83I$j!}*ENxCM zbn~v;ZKQ*}bB`JfO{qfh3C6|MYmL7^EhMrM=`gOo03#yFLT5u6P9bXO6KAIg&w}Pz zzo1zWZhagJhweoO(c1*^_U&`|G=fFGY62mr#WXg<70<=G_1c zP-poFj*x3JgB^->6i*<>)?q$BmU};W>&!ae8XrOj231z+b?e2P9uN7Xr zmCAB16Ra~Rn+U%0?2VvEoD)fw7AY83Dz1r|GD+Omq#%6e_Ii_^ytLkCm4celuCypdS0b58%IB2H^&# z`6&j_#X@rI1Yi+1p7Y1 znRas5n#xblSO!Gl%^YIBskzrl8>dAJhf&dZF~cX*uPe&0%VvFgnlQ4lowujv^0aI8 zg>_acY4Y|FV?r};&D*w=C) zt&cUTXr?XlXNUKPk9``04@WmU90QsRz_yn}4_*8>P&T#t+k?2D_TA5pIlf5rh+Amg zRJJSdN>OYU(*FhG=5A}Bu&4msiGEs{0mcEh(3p%L{jX(kn055!gZpSUi2h%m?Ee`g zshLc1tD#MhXFgj&UC+2)`4tPa27YKi6>6we^Id@NiPUDat-CV&1M|neK2Leg?_8sQ zOP3V==M6$fPm*M%LOcODt%x>{T!eEVAVegB@h>zGZyG%^mbYio{`{V=afWGAc)?7x z?q2|e-wPx35g^xjR{jpKNnLU~u}jgm)3*%1#e3cXlyc${mncYYsp9?Oc&%W<@%6v= zEYz&#xQQ^Bs-FU&$`0t(8j+4=q~Xqrkp22`l^M8y0iocc4|SD-2`?R@R(Q6hmh~Q| zX?35Iwf7Yf4!bIxxF7M~8jw-%b}#yq3UYv#UeVYNJ=SrwDF+kukJo*)FghE3K3xCo z^X@;cN8gFSny|(|vCk=eg8}Epm+a>g4xUR1Qng-52h=U>9|1ydjCCzvZ65i8zgg}tNOt5*ioi9Z2uzc}k8Q*x z<~Z8>Vx}{FW<~T3&u$-B25^=Zq-bglC}&Mg$lvN6c0F;u<#h=B35*K3V?&q=p>cuP zuq#`69D=d5PWt)QnXf-X#Cq;aF6mIB!KVM*!UZv#>q_X1>a$7I%czK)6U`DZcLI{T zEhoB}v!MycDF5oid7lZ>M(t?qt(~~+tK>|Q6fUooZ>(k~4yMI0 z#u~BR^y=x9>*N&_^L&=Ad^OyvQ*^g$GByRtW)9K-3hJ;Y@PSWt5oo&vt=4&d5;Zt& z5vZDN*7tr(ne|R3U9_BjXEilbcpvXze5(H9&`03Q-6DYrOxU;up^Rd{%AkqU1v9Pg za8z^YJ0{Egc2yeFVS%5$CiilJ?#&*zd-Xr4(SI-7{Ck`2Us%@uV{WMpw;JICAbs$X}`634(hAN%VQOM<0M8%Q_u0q42Zk=N69#XBe#O z^*0#f?|0=9JI{Lszr~-SI)3}VkTg&&&vT)jQDI_R#w@km2hnn=G$V)P1r2)9cd# z!guMG{|Lkh8R7~TeRrbSnxx)Jjvr2(j?Th{MO~c^Ij8=8VYz#@6|}vqF2KcU3?B{x zxaOCa0`k<0!k`b@XAR2iYf>pth+>_AA8CgC4lWD*`oRWZ(2~3-uQBe{T#8P}OM!*J z@|pJ-zKBSX0+)U)lVkKYVcCXhTDPq&mGGt>N~_Bw6CRfdFoier+7mSf)E1-XV=7$m zE^?v_K@E@FrIowE61*D;pLuM|Gy-onPdPpH^ccrg&(SkIFSsQYL>j4nL+LBs;;IYk zapOE{P9|^?t)8G66%P}dBo;sGw`c6qjVj=rct^2n1g{oGO%Pkl(-XC_rKvCP7A8g| zJ#lgLXgz{MpTVK@V_i?v#SdS8V{h*x4&gr|vXl}UQhvTmG4ZRm2bTewvvTzM_uuIP zh7&NvEm?t=kqPA0Z3IwJ6?zlV3(+LS39==F9>h+}?cr zd~L>aDTS*F9?UQ{K-tt-cK4I|N)=|rWM z-%UUvGQCGEvi+Vfo>sI!EWRF5Y_0qtLuAMGs^ZaY>zWSz?)9e)nPb)ZMy|k|hhA;8I@cbeIT7Xe5 zvwSd|eRg@@U9fN6q27NhDdaZ86s-RJ7$AnMfYxQ23`GBnWt@(xJJSQ^IlRYLvz`{8*a5k;JzJb}&N#Zuj($hI_~N15R3hANR2Kfg+Exw*ddilZFdO-t}>Sid=$05pK9| zH^upkz5T@D+^2kBaAk&b%U1fd#QD3_Dci-71BzCsa#8f&b*!(h=iMtVTOFa$F9xS4 zrkwlAPiF!G1~ck>D5stf)Z{#1zTU3;v0=>DawF){r3*HTD=Uk}iNO3<$?fe9TI0KV zq6;9s_o?n*3_g^6M#fP9Zg{I6Ne-jmgFZ##9DIwDe(NR?bMOjpKSC`LpL$Y+QTjpn zMfX|uC}Vs1qSfX^sve8P`OF`Wy`5TjhfITy5v?Q*$G$prwdqzJZg>t$ur$KG0d!2k z_-JJ7@r|UiSF*uLnOh=zwX}~<=!rNwiF;povhhH6AASi2eDLg`PFp#pW%oIK((`k` zA_@7aWK#DPSXe`IO6GYcl+lLk55lOs{3V2nmuT!xYXdwohi5_P5cKrBogKLNJ?s+Y zq#kahZDJk=y;eY&HuRS|XF&5TcsL>AVyDz6hEeyaZrB}NniCCQVqfxk*=$EuS3Mbo z{{l5N@AQz6gEByi2n-j54{Gq`bik?5IleENZ{puPixXT<71-RnXC5^HO7nFvisGF~ zELkzmQ;{B4@6|Y;eNk-?SQ^jN?R;oK0jpaHfRqe{Goo|?xCr@-6ZO&#AaEn4qQSpF z-=6?=$ql|e9GYFOq~Wm7llONvY-%*7Fw50tZD=+swOP6V>QV1_cXdRRDqZbV3g}SS zxi_GsxCcoMz$e8i|dN8_{!0X^#5^qE;?=^XUCdL1diA$S+W`1bXW% zw&EG3ca95!y@a6LEdB*D1wiS2LCEimq;liCI4GgE1jMeGKe+(TDFdAhZ;PhTy8#AV zFVJJX2r|7P@P2=Resq&7B7T8#%_v|}%-)MiUsSm(mR-U?n%*KM0TC_lxTq7UL)P9z z>_0O-4aMQ+m1#88-tP+)#=D zJ>VPh5MtYA54Lz$yf;O$@!Qr`jQlc7McvE~8EztY?ki`gGfCH`&Ch;A9ioihnKRWj z8bJ*SV&BCR7v&@kk`S`N%)#Z*myh;tF|kGY#}P+1C$!NECGRsA7iAA*YXnu|paXsE zrk){x$hreiTM0PJ&Uso0@C8o8ZekT`D_m4y_czD+>?5DB^x(_I`o&@lAgMh{PdUbb_}NRzp{7s+gFYc_K!BN(c@5ZC90Qy+kR023=J~zQ2-hr z(1@*z(BPT**|W%eU)fjcTKQJTBEKxGKIA?kq2o0m^%mK%HhXfiY^GKrBCt-a1g}bN zps16a$+=C+*wt>ns5t3_qE6UIS#8$GuP@8Q(qvn3d*>*;WiA-Z9Twf9Rxuwq(v<4? zUzHwIgjL|tfU`QabPT=8R)|>3I*`q2SCJrzZ$yV8e&}Kk$zf%X{aX{6Qj_6I5)6MT zb4mBXyUA=o8CnSVrl)NndXO1wKEcRjqNR;qKTtaXb0kK73${Xjd)xrV{sEcLdt(Re zLqAm%n_{cxRLCo$t35GZO3L)SpcA_E77y%=Cpmc^@1dE#QJN_mB(y|lEaNdiK(G>Q89?Ukf*lNQ_&GEsAeUMIFjEJ+4 z^EtP5!UNk#Ipm#yt0wK*Y+koxt^uzt+>|XDpgeISzHSDnQrcCbDXCVbWl)0TS`OeT zW18X}`FEPIR7Z@!RrP?^n++v$Z9?{yOGerdr{=P|*F9Q)fsRD8flg?JHxRvsoi_S& zI(ltZt?vu}Hzfl^5-yJTX1cRvn)4v z<}*(5sBdh_d>kvc-8YjBTE7DVEP>R3xt~TPI>G>ICcqLi-@z2DZ&Ih7x|w2W=q6f$ znhS{d()>{H&PLQHbuMtF+&`3Bp zIq%@ji3d1D6pt=*LxNk4gsnmC@NC!JOtHGFPWdLCRB~hBa8{V{OI9 zicqOCH@A0YZ^O>qE#f;DQe2!1$_4mycWJZ!*IS5x7x(*r!d(CC#qrnI1ICAT-$Xc~ z2^SMH7B95ajKHp;rfj&?B%&Y0eQ8Mk98uXnj}B}jX>}>p9i`WARKo)l4HiXvH@B> z&Cxu*rGxEbJF@b63deq(x(3kkZUfeNgbR-Ckhpn!YKt}W4 zNa?s%Jm2eC2yY)K*l{9RlL=Vd-pyCB8mUX41*`jXc~+1zZeHOs)W^i4mjA=vdq*`D zw)=u8DuM{oI{^Vvs`O3}q=^ve0s<-`O?rorC`geiU5Z5MJ<>aoF1>f@y%TC6#k)P{ z-gV~AI&021=eu*)oHg^0ELlKycHaHI&+}`~@48jLJ2&TpeP6Bh)AaS17s^yoS{bWV z2LJ+ep&ev=h2>S)@E@Wp*<;qpPbVC+eoXG(N#oyz?fZ0dU!8jsb{rJ-OLIkT2+#IF zh3@)&u_LnDbmLpE=U5E_=8NhHw#}#!F9%3MqsVfA^HOJxVtnCDpWuXfO;RXJENUZ@ zJ*AV9^3!yo{udn!+M{?jATsR+ z#{b!{2P(>RZZ4^GviIyO#mzS+->a{L9cOcK3@~Zj^bHHDCB4D5)L)@4x3sxg-%PMT zAaXAOXW?@18tbHFxTpR(r^lG%S(>qqWlxpv!g3Aoc-3*&?gu4C3yDMHP#pZ5_R=7= zDN{)f=SnQawBs64aFQf}fl2;^Bax$k;~=!%pmogfK@%K&&(~x?J81AnT+51;Pm}%* z!-#GGGtO{STN2T2aUCBHOz%Ly{X_I@Y#&)A#&Jy11pW1gsP45Ihv747y(~LDik}ZD zTBRO^kPI-7o=Yf()@y<;IZ2U7U5 zm=#jFg%dth%mt2jPn#WZ}?_ z41}=DrUvU+kDVOV50tQ*C|Kq7em_-k&4|4q!CnW(2T%F15SHPWj`c`)CL9wKuxGn1 zoaf_!suk9391@@#v5dHe5$lrtJmoXlDk~f&=(u|PJ;>lk_`dXN3rpe~qbol&er;SE zS91#~^SbM?q-$yi);|t9TLYp+r4C<@^e)5O?wmC&ll+BmQTvCt*SSh_PJ zdO1HvC4Md)=~=Qmb@`HP{5xY~a9x^lX`}$+)YSm=cM{D44$V>+>rx+*cB#qaBMjgA zw1ATk({uG#F43IT`sZO$CF`BbE~R0$24pg2Cu8@r>%UHG`x@jf30GWzUKx{aBa%Ti z410F@I3qOMK`ZL2J(}{W9m5sg+KM!niT!7RZ){k_Z>~@k#uXbp(SjZHo<00xt?k|m z|8>#W#k`|A5_E11pasMP)-%X7h_o{|2XF+jz1~~5*n~P4o{kE9tEi~>#8Fcd zD8l-z{XJhM-y+|8A|i4k5et9EhKeaoo;MH;n;2*SK8e6U7-H5Q#Bnnln0kM@UKifh zE?Ks`K8Ja*PNY;D>DZNor$fAoibglFKHhAS+;w~H;(_L$HwySeR5F+W2oiusE^Rrv z6%kg0x&!rJF2KppJRP*Mw#%GT72SF0bba2fr){^C)-fva?OZ6nS1q`{OuE(}RKO6J z7UZb3oKO_2SZD9@*O8-eFYK)8L8gUGJDOe8kY7M_MiwPG zxrtbZVg(Y<)CcaSmt#kmbWCKAm3rK|5B~n6r>82D)9G9AfYv%$Jj5CIQ1c!pdFlR& z^8V#sVBv>2&Dy6|Uib!Sks6}b{<2YgO+GwxFq>_~?4K5z7&xb82h@!J?I~(+0e_G8 z!1qZrH3;0FU;**FFzGv`)sj_MNc6e;`fC5tk?WxIAOqHng#Pfij=FNpi=z!CXft%6PET_H2E;>RIGDgE z1fsf70JUjF41O4an*E4Y*OqGEP#JTLqvEO7bX>s4*752Em|ypMLHI-Tpy{;i5&?jC zGZ^4Oxy5_tRdG0uA51C4(7R{$Fs`Levg}#1C2)#RUM@Nebg0C6@4wPDK^RSnTovPc z;Qg{Ni#z^3AmkIqtmxI_0VAuU=5j@n=83t!F_xzxi&Zsqxp%rfLfI@=MmEO?{~mSgCQUB-;wt>s8V#b8m=2Mai;@uW^Fhd ztRtaU@hU1n*aFj$-RsR{?Q`-gZ_Mk5G++O_IKIz5EqAM<*|V!Oa!5r{A5l(-Y4uL; zg4``QV`Z;dimXt9vuj>%fP8P*h&-^{q(uIqDR|z)s*5`JiT?mjo0nS#4^fM z@7I*|_&A?o@Nqa@XsTm!2bN!RRa(E``J9RBsc1iSCn|z0FQl-ZGEU>XL9SO@ZG;7{ zxXqI0@R7PrT!>xfPGpD?>m7AvezvfC-zYmcfh9u)Ix~fz%UIOlYA;}+>Qfh>s;brW zC16Fu#DL9KqtUp?C*}-^zL%U|hm{Hy6cF!a{KfU=-n0I5akb6U4Lfp;ahHEO)qbj$yMDQpAXd?H!^E4W%02kS6$?+(xB~tP zCe7}QJONz@O`E(Ox7!0_bpZVvyW_rg2IS2&sOZIp0N> z30pvvE24a==@E>Mf@|*YohxT&HGm*Rp5SLN0lp08!c9d@_JeE(X7w=f*#MhIv15s^_HVx))#-~{yZT^dF)(gjuP+%2#2`S}+;E}tJUGk9BV(o(|`xM$0mdT>&hSs)Y&yG?}?M-QYd(~o@A z_7V6Ue>2{zb(*-e3s<7oemy=TL3Lm=)CKkI}4Bmo! zJ+4s~LD;ku1~bPShcid`Gs6MhEP!v*+>Z&r>fYNB135uRL%}InH`E zJ+dDrcd6Zhwu{25o( zgmmRQ6(i!G81ShWedx$KMiR*4ZsPM2Vkg_Y82=D8ub1xf9`@Vq!g8Si+!?327=vXH z2lo8v#x2~_j(yMZyZ-^7_m6PyBz>St_};-GauGh$D7Dnhg$|8ao?|Mj_!5vhpo7?= ziG2TP!0KtU8bu$_yokWWVMA~_n4$xG7DNTrS6hguDz#gaGt0kG->Yiw3|escU)4eWSM0I>=Z^ay{vWO%IBXg) zswWF{P-zU}rKwz6gjvZwllYLx*94uM<09ghe$1bHJ_L< zs^;Deq8GsW*CCL>0Dgk38ElJ~0h%jy)^jQjhbGY#Doc|Mf}d)fYkbtU3F&6%zosCs z!S3SZh6voY!rAsktH0HliVt+%d5S}8PxU|4xrvgzzQQK`;^|?dByH!vSxjVbgfg zc}e#Pn-2g2NBT^jb>h?HiSwIdOK|$5rNVk#4??xV9hHdd60$U%Vwg3^ShF6!YT>v_ zx*0-WJU*uLj__rwpYA z>K@=uyx&y(+{wgyxr4pGd_xkFE`A}vD}PJ_Igzty{^fmN%FdwMT^K~M2$k~ zZNtN7ZiT6vzSB9X7rhzDyi@b6cLjz zPJWLrT%s@>Lmw5Al^E;pfd(&e?_ime(E%L)#%-|6V6EWYMK}WhUpM_j1i0dui7;3K zDZ?Kk?t?BM^^+MuPlj*&58K`GzuN9#ZkTg(AR8?5bkA=;_L=E&X! z;1s0J>OQa#$o0yP<>?MFNKv{cNq*%_ChK-o5+xOc>h=rnY-DxxN`b0T9-s5uRP4Y! zDleu_`LCCQc>QlK04`NH0=~1O1<)6p4gle2@IsMm9W*7j6oBdkCKIec+5kzJlyO6t ziQGvzn&VjTH<_;X{hSyes-g=BK_k0Y!}l6?6vSS7c>t(aWcLL6HF3% z(kIENP)#M(1wY8 zZI;Y*!mb>`UQTJc-7#o*&0Kv54Z`q(a96Ev@IZLHH|<|dm? z2N0qFKGd87b{WbS+Y8qkee>;^6N?@1+Q}|=q&2#i%Y}{8v7TIsx8t@>aDOb4?3H?5 zzixfuhw+{n5P6KE79fz|RGS-|Nw?1zrd?;}S2Z*3$*~Mh`+C429PTh5J@3dnvq&mc z9eMO`_o(sV%V7(<4;P65cO{B1sG3F64q|07yaq{wnxmOfI4uFnDx^)UN=X?P!6Z{O ziRmKchkEzm_m7z+drVc*r`E#Lc?8lZ-gf8fTLdtscewAMzDJ!jv9Sd;8|$eX&s|A2~EP9 z5X({fB#E%F2f5Ml7U=%^VkNXEPKI<}bPiJ#MR!G3eE}`gozBgdWzMK)$tM@8{?cGd z)O!tk>oL%3w%!`K8t70hqHD30sre3eiD4i-j-iB4@r?p9ffAqcx%9pZegc^o)_4be z1FB$kh+LY_K2zjFOEkQgTX^g$gveWq7s)8%jw~*0OT4-ID*VTG>##m_WSZgTf)TU@ z%Li{!z-eKXe0&vop^woDgP~BV0G;}E@*yU zqLpyrpN(Hm=Ys~2q2OfEP& zxvKUN{TSIfI_WucP|5f@BPd*jpm!I%Tlt4*->u2{!u1GG5mR;qc5vvHX$2l}+z;TI z@}M-6;^0pcaF$|0Zp34>V5Ow-DEFHPv#C3GKT#^NbEHAsS)$t%qp34pS9hJYx2(qg z5WR`P+&}+N+Q^>`mBAeNU9y+uK4Ob%Zu~W9W1zouk#245v3A_Sbk zSukl!lkI5;28=AexGIf5ZFA2_*X^*fo115Jxb;^DX|DX7q#ChO?ZVv~(PCSNJCq<*ny(KN#%$ z!f`alD_vnY5_In15_?wu0D%epDT%WD$!bo{WtP0#H?T)@8_Sib*|GFJ#!))$C#-YN zw$-*bc7lCFNIETREO-6Ua%VBemdatwcY5ULH-bXw+rIT~4Tht-FKjKth z7a?NBy10rcEyUyX#6Ko@%LtRgTezCpXQx#Gr^aLloXT>Zt`9XWIb^EFG7KG0_^hBK zwkd1a%OYGJvfUgy2=rM}yHtSd2jfH;)=Y+$m`5$SyRzkMTPPQKzCQRI-S?iWv+wE# znTlv6>#Cy)&7LLy?|ev_T7V_+pmtyumO-BuNUIRn0Pa0acTAqr^P1$jSe+entEW4e z82j@qRnb7m+?1Sqe?;zg{?cp%BX7k(d=9`|{BC{#fV;^+5wt~K6)e20K2F0nj2S$H zoCTq~TG)s?Q+eA)1)Gs8W@hHHa+v2A6ZUZ;Z^MY(xk&Ek74CDpUH2P*P#7iSA>|5j z-$n!>4xcR0U>Z~+I>5|0U-u>KoGNS>A8+kcG1rAk|NV5Y>dorf)fx*k8M=P?*AbOe z6gkz3jras$!G0wzzl8q^y;Y=T<#4+kvixZ=pj>Ap-={>1!tk24c^BF@Jnwx=y-R{& zQ`QB8j)F-Ktass1nbjF{jet-{&P5!Jk%Wd^OxEBL7l?wDSy zsf1S65~Q-RlS7GMTG5 z2YmcL+cqtp$Kb9#BPj-My@LF6f)M^Q3^ex?n>XrMDxPw~z0&MOFD(Ft7u%q}QD*(~ zT8EoXdxFTlw`EN=(fLjRRqG?$g|nM1wI0Da4fOvQ0_VmCpxViMajmEWlJ1R z(~i9mo$5XQIlZs^&H>-xBtE9>5y^>mLn%!%7laID;&o5?GR%eL7iK+#o1_!xI=PH!HoRK(L))y>;qaq z+;_i;TOdO(*_?!>`AJK$4F&(DysiTb?_dPYG7t;Wjs^uf=hRe>Sx1BKv9wHf*Vpj$ z2{-gn_2v9vIUwdz^Gkg4eCnv%B%M0TQUFmI)Cez`+7EFr%6w%lvF` z3p9IyBU@`>Pb|d2|1Jrc202}b61R;sZ;8=eAx0 zbPoxT(se5fm&bs(dL;jEw`7~{8;S+WuhP5?#BMwmVz*GC^L^QD4sb8ZZccu$`eejX z71VQ-s;t>l?Y{2sfz+o7^&HWh=?V($6yS=tKUe$=D0lNQ4ur6EK!z6-222DPcZ1vX zUc{(i--5d_Ulal2Brtz_)b9M5kN)C#P7eBKRsEP}Qr#!mtKJq?k))p0mrovQ$Vj*3 zuk{d4!jjzf_S-D=H~0gDS2p&t(5Bkjj-^#idg~gFd5t3Ceoj&EyuXZ3T?Gc1nmiX4 z+GYK~9qH{kpF~eTdu)@D2n{d;n#$`0%SruHn?>)I14`&EjAIT??hsq@vqE2R5B=eF z9AWPw&ihsdgzQ@qU)ZPG;2VwhxI%IZ+7BGK+|gWTP)vd2A!PaN16uZ2 zE!i+s4QsTckxM0>;aLn!ki16D^k{3A?u2d&YwC2X06k^eOKE6e575+HyMq|Nm=M%v zv^^Xn4ICs<#w2G@)R$AB(6PcHK1Ka~^bHQUD;h%E7OuYv+{*H-tZkZnT9&o4X2+>P z`-9|Keysbb%Z@?${t@oxzuXZ(tMDE=y6g;a+2Crac7A^oM`WXRVS9NRn;7K1s$K4E zW0os%aQMT1W~Lm+_M2_Lrc8Gt?Os_;vqLv5L3*>zbEwjdZHC>BLrKc%^CiL158Mrb z$A(FE5^S%xe|N7z{5hV@5~l}ijNEUz;(hZx(dadQSG&>K==x>=J0<0*=@T};OT=HA z%iy#`=QhLAVCd@x&rzgPmE3ID4ashCQm0s%gG94WfZP%~`SwNPDxOkKCjJT47IM=M za7k;aHI4vd4E>9CJ~vw+m$iClq3}V8ked|nMHdC@81EuyuB-iw>gJ5H^8e(?|LyqBL-W8Vg{0eFg^?LVtj4RSjV}?E z$pSeJP2X=pOAnkTC+p5CF?W;s8Vzx4{J6cmxo_)0Xqt^0c7f6wd`=7C3M;3{I5jdv zKNxovagj{O9ws~oWb=Al&eP6k&II95J^13pIgsu=0(&@S)&s{}2Om?O-itW`prxyy z0qa7A)b5J4JKb;a?Q>1y;;`~<1~*P?nJcmDd~MMxS8px`@{=`-^ta?Z*uzy$E`OaY zoEl-J($rLZjz5Bb*t~Na{B-oyu!jQQpw_745ocvZ@as=kWHN8CCdBwDeAkr&du|lvr-}=mD{L(~Nb@Xyy>BzYAiA`muv9Cj_zJxA^ zFMo#NZ3Bpkb_eZ`4Pqcztu%Xet7Fig{}Pd))1xzc%ar=Be76sAs0X?bx1pV^_&(d_6-k0xk0Z)_38Q<2#K1uJ?p8Jgh4y-^|KwmVE5hK*vBQBk5k?H{ar1Y^Wf>Fga0;HYFoX%Sodd6N4!E~tC_-jx@JhG9Qz;kuCx9G*rWF#&$ z^=3}Jo;tMWQ|v*%-}nL#ua)l>&Dl(2d^#{iI*mXkv)Bs;LRwAt=92;eqnVvusiV%- z-n$90p$(e1>dvh!O6gyWjSqx7p}}U&G(H({W}LvGb7X1s+sQ_7p1m6!rBJQ`HcFrVNVMwU)~iU`F6GGE&i*ZELWB- zqRzF8Rkv98ZYBp=Vb&F;O`=`?E}770B+Jfw_$4g}DQrSfrh0TyqvZ5KNtK}?BYv%; z(d|js@18f{D8DH?cRlsTY|F4-pZ^I^@ztCAB8HAQ+e6C}`0>T^zOKGv7qS5-rZ1i- zI`hf-SVab3i!Lsm8Z?sFUK%gp$^?;~Awq%Aw*Xwh8zRmo)#e9&-^@8XWoPtx+O$2@ z@x4K~_|0&lhMzyL*0ED(c{?&;WyDkn?7dhy_)sF61vq%lz_?`fv2AbVvv;^9%rN@a zBXmrcFg5NO#@8_*XUaZx)IpNelh*Zx3jf2{lq5-OlD^+QRy*cn%PZHJSa_`8!p==z zMD$7v{rs81^O;inCuqk)qz5KId7%NEt>pkvRM&HgTI=dPkyY&ag)c#JZ~TQq<=QRS zhM;Dd78r;2OI2XluDuAOQ1N~ywW=ul_k=_E_m`!BBc`H+GX1=e3pQ{iggb8q2XqZ}N?=s{@3J+QG znb^gQ;}+8lQ$i*6q1f6lQMvrm96rNIcdbsFdCM~Ay)y)xrw@z)`fPMhe1Wl>rYU&u z+q6UBHS@kS(t9%1fm2LISITOxKfJ6`s+>t#B8?co`58^gL z(WcIbY%z|d04)p@ZBbd3btDV=37Y~G^6540pl~<^jCPB^1gDP9jbGZ4AqANAHQn5o zqDJD_5lNYuZwUMRR(-Ep0qTv%f+h?iKz1vi{pCuF2 zS;Eu54v{nd1TxSec*XnY0WDPM5U^g$JKQGJ+^(P1&NnqxCwEhIrTtl^75WZ#@I#`C zReY;ov(n(@Xf3>jBPFB^t?K6DSQ)6^yn{q%u~aG}RrY`}2>=z;yNtkp@>?K$?tS5b z`UEntVH>_M2JntejCT%H1)o=GRWenwmKRdIFL6v0`8|pH(=94OUwP-soT}gR0_jn& zZAaZ{P({@m>yi>Eq*`2vj3Hv5LYl+& z6DJ>gl-z|a&&o5qATG?6t7K*Z~*cW80n)o+6p*@ zyu-Q6KD`b#t0^D%y8n|}^J;PE_K4bUz|>?>aN<^0X4*MqQaA`m-P4q(0x}5K(*VQ3 z@MH@3(-(z-`s_#lcF_O3G&{@~s-Maa$MDmCb#>)arL_<^oNWLUQo7b}HMviSy#EhP z{$|XK3W3Zz)Aq(eE4bpcQMUP8PG)kuvF`c+%}|=ppcvHv+mH0C*WOP)ve|HsggPt_ zhADE?p&?O?Dgh<-lQyDr7Ej)|$=x3$%PNl~k|rV}>Hzp!IZzPlwkIS47$Jj`Ib6_~ zuW(52cIGY=TUAoD%jwEgD+LHNzyPo~c%p`|(~*#zs3JS zp$WS%R*5hHCk5MRa8;vWSqg*(PpYI@)zQQ^^&XpSN6PJhS&rtj4#m*a-S>)&XmuPd z|H_LaULTA^-I9O@ zX2>>SfEhHFkH=2Ud9BO`v-<^pWN9)>hgHUY|6TYj#%vC$Pfg=Mn+|mB&6yYR1(L_XXO2G7x2&}x%trwVAhw3fOSJ4tZ{e_z| zFI>zM1GeL+pLdr0zPI;2`lhZDdxR3ZKGwkJ2)l&bJzt4outV6HO?fPonf5xmyiA&R zD4?E^8UFdpcIFjFC+YNiH8;t&7K0+jiZmwO;qMPh7cxSDFk5ZZ$9l_X7{C3z;^e#C z13`Hzhilb829vDEoY)#a1=8`giifzzJu(&%jf_O*MCUJEryE1NrIO_s%LE|$@L(kC z<-@}&NNtM|XWyA&B|U5{EFoTx`r=F6*`O#x|cVx8Cj?HF%$XO<mlW2Le}ZC$Y=L}_E4h|-XT~-fXo|EFJM>&m zqbS{tbqU+m-}1l#K6ziZj=f*hfmBNO-yW z`$k@X%K3Ph_W|dc1GlqHO`dHYci8BqOj(;6P3nYdl0^L&(&?1|g_2KBN>*%uVM5UFXK2sVnujaNuQKMq)f%@-#0RKpDYGd;Eiu*fHO_W|75*hOW0fDo_0caWg=kVjqg4?D-cxe<@_l27FZuEp0V^K`(N$C=rbeP5<*is`nro1qr9VFR)URf;+#E9R zmd@=`Yg=1lit5e8h83+-K68+!OV4|iE|~R$FC@SlFkeIE&$`ppr;sc)73tY*E%H|rz6uVoQT!N_iSWO zo)%pEtC$Ab4K!9j9TTsLcE$(T!7t5kd%S{&M})TfQsSt^2%3`s#~ z=q(q6c`(^3jDa*?&5J(Su@o~nWSeU0DSa{3Uo98R{_{eVX6mv1!}v9Xu&ch8mpl+z zTY^=$W2eNZqiI^2MN#|?i~c4!leam|Ugfh9VMG2;%09~`)s~1Dq-Uq@eA`wx%jy71 z*+G(Rz(J4L3xsj{R0~(O)NkX0Z~~1~!66{3zB_uO zW(f~bb4osQH$`BW=?GP8c{c9E=$Nx>6L$}j_W~BboF-sJQvLUYRKGadZX`|E4!ih? zd@fu;N+XS_>k8yp2A?CU68~z|K{4P%eI%914x&nMFSxjH@4Iahv&WQ7`$u^U?%Iw; zE|qD@Fk}g=-+bLR*@79vbBJxL8BVAh!3pM=4CLj<$!Uy-uHW_@XIo!L3Hpum#R$5B z+6HH>+J@|RR-?qa^0X1YM2}7PzVUq8q^*XGG(E{UWpL_Jnc&?!^(A=-jJ*Y9U_nMT z088&+N!#Ysf~wdP-%;nvMcc8&Vw}6im`_~_$726p!%KOfnz*YTirS^K13p}%#yY1y zCQJd=YRjq(UX|A_Eg>f?DTYkD;CRcw-XWX4_=fd;FUP8Tzuy?!*57vi@>L$Z1q^?U zH-l7-djJj4RR^;vy=7AUcG(o;s4>wWcQ-yiy3u(4sjw2YYBhMb3ny1|29z6_%J9Q) zzFYhy6 z{lS4V{J*xP|E+LsFrjTT3=!gHg8-wVQ8_N&q|9}F{Og#qMy|@Z3FJnqlhCF=C=@{j z1v_FDVdFE=F-t6Urn4;sf$#vCB;_|6ac`2Odu@0UHI)572e?`^T2|(HkF(vH|Afu6 zkm-Z*5jqx*{MmsL>&L)%pxv6J+c75GlI1-_R2lxuDhSV@{X>-S(G)&y3dhS<`Hc`r z{z3{ZSeBIo1`{9RpYwOo6Im>p4oh=<2ZkxMM_bplId%7y7Xuu6B7L(nUnb7Qbi|O_ z{viVXj+67uJr*|a_D5Cbz|4gDhBWs_xzE*;ti$CG!}9DoxCGjG1^O*2(_KzZWZAyO zj6kjd6`d5^8cu4WLSM9qJ)pHv%L`R1!}%*Ne1N`sw{ac_Ud}@Zz?KrH-%M>JvU8k+VQri-1yS=*w@EttlYn2 z#DV7K5&W0Kawy(!3v|ggZWlGfW4Ia6g4F_WwK^!EB`gy*1E>%thv;N9*LLJeJy-_3 z7!N;k%>cHWGQQ(KL|VlTC?=f5wu7V3ab)u{{ zSIjkvBq1;BWRT|AF^^r3rq{5A$!mKOQfE90-)WCE_-z#12d^s(WNv*~z)0d#(Eq~~`2VZhga1!moR~9^wu-2d z{Js+Q{+0gZf9RSCII zoVO5ZLwjW$$_qTH=}@!7xA`-?T_B2%IqiqDxrhSO^!IJA3)`_uV9v)f^0ll$ zZazaxOf>2?Oab!}leIKZE3^`vC@ipUDYW0CcI{)T{MWjcwyTEKUyEIAiHH74{`?|_ zK&D*?e+%numN#@Px2R6BM;r<&{|1C<&4&5Jfc`D+@_8jby=qSNGYJaqIeMeZK+DXK z`0(W)qOU{&W&gF&jDM?p|3O&dpZo<@aF@1#1N>Su$7Wwz$9Zs-**m|3%2;>_5fm`t zUb7h|Z)V5jWhMcfm)YFCP0GvdWPSuL@ILX$KPXVS7(E-r7Ye{_gs(;jlKJqWJDXVU z#lHI~U9A)4&6gpGR3!<8JydQcQdGnz!(T5j!ba*pV;0b(FUHhEJ+(sW+28d!pLX63 z>=+bL*62KFRBFA>=yK&!cM%fGPK>e25)LJRFp#H1?--cEdx%1m!QpDy5h?6b>* zasU1vF5?_-dB+1DY)OH8gr+g)+V#8rG5K9ubxJz9`W^7RH($_PFcFNxr>K1f6^YXZ z)T!+Z_g3^LkoQmx$=H#ewW$wNZ_+8+!ijqSdP~0J)BX^>1pX#VbMLYxO?ETo;m6v~ z7IQ#Ij4#OY#D{|PniaJo0|`Hh@f+nRDhNyjqsLtzg8v|}`8e^b$0{|9-D|I@`RJXb zC+fR;zm@!J;eFy^;4z{k!`GfPsT<=HalUpr)7qZmE+%v0uAFc0wE15vuOr6jiF;d3 z8)S@+ezY@+0=&}{0v$51vXhh7V^V+|fO(FQl15cDWoyL?ssejSQ?KvN^qriGHNGQ8 zbCKi{V_er(_gAE)cluuMF|m7TbLU8(df~uJgF}}`gFb&>zGrYcw7ni41z!9nNe~&K zH!Pd<{6j>w?UiXbv;1Js`HglPFNp(r;)k|BJo*rZhrE`h0f}A zfWUPJ@Ed}9MB1y;rbZZid?grBF5cyz;`dS*sY( zqseb0bM`2t3NV6?*SIr4ZmFxI2z7O*^z#-$LCgQX&C!v*?x(cf9y6C#@Kq4V7H$GDCA(6s#jd&Cj;384bvwZ3H=5d>=3_y z{rY4*QN@s&B!0dw2k4SX;_5cU45{hiPu78-(q15$lAkQemsB)j?gAZ zS2EmQuQ$Z(<}LNBh$5zJlKNj&8_OI0iWZ#`Y;!TijX+U;w-bO?EEqtSf<-A#%3xO1 z|H&@TKL+^a)5b868RnyC_oG{9M;Ov&)O$-dZVYPiq&`#$5J%zb^6Vwi>2~#ql=Vh| zIl9K5Z(^jc>)h~?@bmEt&z7-RKTH;_P^`hw0N8>6AYFGRpPq%9j-!@tqaTYDdk31? zS#+)|$;2_;k?c_L>JH3ar0;}CJQY~R&*@U6nK8o|{`+3A@d2vf&imeT0*wq~=Q@*< zlk>HzF}W#cYL{*J?ENWeT~mk~?Zb-r^GcyZug-(VA5sSyDc2vXh~aBFS`w& z&EOy7R#~l2k-{fJltw_Y&p?S%cOHHz?Wr)}v_2A#*HzzR1h5$ZoS|D|-B^{v(Rwc` zRvp%?ICgbt(*wclaw}#rH65Nr9H!eZiRIt5?Kk`od$1yx26CCq3(n7@Fs_%%<1Yf~ z!v+9V1^i;-3<4D?$H%a1IP;>v@EStucSN59N z84iQl-?;CX&=O2r$2?yXukyi$9;z+ZD5o;U8CB)miTh8)pk1OOnrn-%R^mCN~qYOd1$L;RMKVWnbYTH zo%mAIPA7873!ax5BFBl4`p6H_b48Wcf7>@R^&cy_2cN1>Fu)I=A*OGP%sEkZjHS8s zWXno^8-Qi~T0wWX2bdyX%W%79fPjZ89x9C9=s=M}1u>qBd(H-sJNB`5gN5_;CLMR& zV??D1-tjf$T!lU+YC?KeXO8?8JKO7GVA@bz_G9=ofNT6-^IJGtS-WHdS*^k*CjTYL z?FyO~z@GCI=`aCk?+(OmMNX*1SXTMreuaCkYN4H!om@}ZlV@@*9NJByUcdYfr8i1m zv1aDb+wJa`#ykRqR*P8+Rvyd=9)d->1JP~HzAFMXFAYqgRzzwp|)qmgINFg>X9 zjV?^1`ieRAq3O?LQ~El4Ans+{qyQQci#`F?Paqo-+J`bNlQ!Cw-dDXYr3 z`SsUgx2wP_I)x_(&Z-g*TgT!jl?-OV_nYry5S^BH9`z^>M4j%s1iB z`;ef0_X%;^8O}i0wH28gP4{`|dg@vD1*xzNhj4->ZyZ?SYo$VMf=PbPQ(weUC3IFYHi{QJbK8NlS>`&T>D@)v9`qe_uE zbDEC6RSrCqv(wv#oI#r)REojlDhMGH4Y*;X-OS@OvrnK$(f+ zC*`Q{drMY%eykEsYLvj?^D@lT^0&R@Z4O@nVOmO2dM+knj-r=NMn*peh)A;jt;YLL zG=or}qQ7ohpR{lW$+$m2ztJx10ol*$gRcj^|69JiGRZ;ire*77-8e8F%XN0UH;w!J zjAY`UkVyO&mmB;iZZYi@#4?Ncn8$4hL?@p^D*{u{1KU{xI8sF2TWaSmv)Cv!)? z?fg7j*mGs9E%AkLSt7hP&|3b7#*Uj#`+LQL_nMo&a@JHwlk9<^4fKqXdd? zrW$nuJA-DQ9205O3O*E8Z)i`>c1ZoD zwnZIRJq#-P#2WPLXa#t-j-%hp0ak)*uzOHf^v%PAwixmk_IlqfD}Tq9PYgrQ+7B+C zxwt5zTV_a?--ET`Zvk05FRBe3v|x>viU|h#a1-`vx{V0@)$T5EU2rp&ctlgw=epr- zq5!LZg?z3HI;)M~)Uzxx{$>?gO>zcRNj>{>$JMsn%F&^$i$zpjCE~x$hjQq|R{#>6 zW(9s7XMhhS$Uona{cr5OcU+WNwl!P`iUcVn=R!frAd*Bvfq^I>29T_Rk^}_F6jcPt zNku?Gf&v0ckSsxpj0lo*DuO7PDq=we-{Z{f>7M(}-0ALl=eu+7*MI7-63;njpS9Os zdu{tz&39>L1=Lp)p{IwhmEJV$9gbE=Iu?74>NMLYd3raPGjph9YI^6M>C>h8!G0^D zNt9W77VH4rp(q5#C!0c%=~$6$6;Wv2(iSEqdEG_e`Ss^R;&ZFQtej1)(DtAPZc*ho zBR;*p1xo4ap{;$**|xUh`xDzAG!6T=VpP8w4Hgxw+;Dfow{P#8d}+l?nPOqSyB?%?9Nt(pzeK#(BrKv z%LtM=nNyDP?C*Rc$=7fC6a*Uzd7>XR)v*5Fw_IC; zS}R?J83V*%61d2|tSBG(4snH(T#5j#;&46HhV zjUUxQ$5_ARG7*puO`a;6JKChWt*Wvbco+;;UQvmzv|BC1Fq_}Wtcrg0CaGvp@QUX8 z`23yrM{h-Zq?@NF_ZFp+Knt+=XS2~^-!0TPGRkLHH6_n}tSjB=tv&nB@Q_0BiI(!u zWpBl=UZ`hv_Yge&NS%s@(Xq=u=dPc2o>)PYY^V>q6*k5`qfr0h%p_3_cZqGch&{zA zYy7q&A>(|)=wU_LRs}VTvBcl#R5A642Ua%Wg3g1qqnS=6#YtWXY9Tq5l^Kf2Y=f0=o z-Aa~B&Kyp$*WQ*z1Xdz!V4KE8LbCQ^cGJnpio~SiGQ$})rAxgv8|p&hK0+KEluIJ6 zvDW&urw!xR`q9&UL?{D7CHQGoykk3jQ;&Qyy$Lg_JM*o{g=3K{-?LqGwA=snUfne> zFGg9tuNQC?)9>=&Voy$Kp4m56g+~u(1X9RJ+KB}kN*y7HZq`Wuy9RK z1t0t=>iTBgQ|sL#vX;Xr-pGVaa9~YW>7ZfRh5_+Fg818@VmSegw9}bSlXW#~dQlxO zJJ?hEqh92;b{GR9!P!dnNI%pps)(?N!YA*C&3Nj6hs>{9)xn5BogmUGRr+HJ^oCm6 zk-mh%=t@Znsyx@=mHRUK=^>-yC$mKco*38)kM1d5RDz#OzZ+zJHRh?Qj#yXswx@i- z>P&fv#OfRk7rkig;O>YrFMZv^`|oe$Hj1`OEgS7RElm|&{kCkB7oNK`=oLTt#Mto1 z4mb$8kLFf|dfU!*H>GOM#6_)_E9psHeL{00MKAHp8cU1lryA1>9-KCic>}7gWn+Ek z7gdk?ORKs-@?HuHK;PA4Hts(_Z$%3? zc(UfJmdl?DeMa%ZZyE%p!^r&Rz@o;oV75TYsHP%xYTn)Nt=+2^%{4|EOyeJ0b`F}a zwI20e(%-_#TPvLPGrO-*U_RHRO{#whw*W*i3ZrX2!y@Qn?>!j z>TKD5jwh;^&*7x+#ExOrK8UZ^VepXE{g6m`ZF9>QHMky)l*=1mt8>w&!O-qjOz^;mt-7NnQ_pFTJe$% z@tIV~f=u{I#I^Rg{Lt#m=w!^5L5-`}A|jLDbZi2O7ZZtan*HD@+=cv9bR_GFpW=0? zxLp&24^U2p5^BSAOFZ+;ZV(uS<&0@mg6ujK%m^68M%dWhtJ|p_6=~&>>;Wc5( zd4tGwtR8B#ZB{;0wLH_rA&tK7z0-zd*lCHk3J-RCSRYfD`B5W0x-Usz7swpwy=J!w z6fZzJ#qe+@8~K;C;1po9qlq%88iB|TD4SFUsueuWwYkSerVOlA1xJM zQPD80fy?OJ95mbm#*Q&@^kt}etG{bpnS7Dl>#D?4TJ}IZoJPenq+4aLU?$=PYc<>J z-Y>@KSS?Jb{@y!$`K-5bpQEmi!Pf&rXp5xVAKog|3Vxty~r!OYId~_zH9yta(g#e1TAYTL<-Vmp&hp(K$FyZ^=jQiGdYC`G8$Biqb zM9-eOqkG2<@@S4hky3HJSNYO6KI7XH<3_6Kg>Im9bh95UNLu>AGEDSiqDyddot%>! z{HQ{N(dfF9Ud~udL;Bs`d$5iv*OJlklZK}6BnHz!tE;58nwHm2Vp zfpX8+ZoR|tC3$2`YTx@*Dk#(3t90zn9eUPwsyxnC^+l6b^8#&mCQmPA;~igs;#rWN zEoDF8Fo6j5*^Xtxs-)gYt!>8|`&*2)#X2_*XT(%Ghd@q*Hz>OcIJ}RzZeAjK)y>GZ zr#z5Cjnh&Dn-fZO$*}Y`YuV>bE6Q>>x@Ro!{M~2x0h|YrNU3K@AJMKiGf8 zc$yiDE4;=n)tGRE>CHQLN~;qj3V4FkQ*T?Ws8j|H^Y6TMmd{~)pa6@w;V(#Ss8JKIKd z?78J;BrFzw7++`6$9t~p=#F1IP~T)Vp5lmq@$o(4MJm1pg1iOBm87eRA}EL9#dXKM z-E{5lxK$)Ssjh>4kLZ!lky zbgo%=Tb|t@4@1DYZyzAY_}@r+xS?58xx9^mXf{yJC9F=SM1jk)fF|Y>qhP4|Cp}L7 zRri~m`iDP$VbtP>C8LX>v2sPDl{?u$xamD)b#l0wyfUh6so0CA6eY)!i81 z1Cp$9bZW3SPyd-xJ?zJJGU0}m)Se%1;qnIB{0BXUiipg&VPZRQEP#65j;V$KEFl3; zGipbLKp1QgHh!H6sP(>%{N`Pw(n$shWV^B8llDN^el&4|A#&ns#9rSxDB)APp8|sj~~cm0ntN z?T{8X5KM+l{v0B}Br7XAA`Q;3aRaY33k#bb1YS$pgS^iLL$4D`G> zEx&&9?2~LsyHe4CTb9ZRti9N8&2apQRy6BwIyQY)BmIG`U>J$L*|7F>1K%My>cE-j z5#7X+Xo}eKg{D(RyTcRSgKj=nyH^O{PUK0gBr<`1>`e0ul}H(^;AM70u{Iy?wY)T) zyo$@s^wVsd+q@tREECy54R^NNY*&}#LHT&wY6V9Cl_&gBWSU4_ZAwd>Nc3%9jh)Hc zLH%D{lBXuvM&@;bR@O!@<*9Y%MAk`qB#HL&rLjxqgd)04K%M_2ktSIz9WNrc-OMEZ zSa%k~T7C9K=+z^8R7loIGgAK-^!d&v$7gU8WX{q-?{B?n9)$Bm#UZi;f)l6Egl4d~ zG45bGSZiq3;k))o#3b`U?WK}(+_mN(_dJWc0Dj7$zkH$*Hh!?55_dzyUoR) zY3C`|AJXhc<5+seb;O{cL3l0JWk-gTrg{e1Rl1x^9fQ!EQ90k3{D$sh!Paurb7%em ziVIV9HqyD8;fy4G>nIM2Qtvlg%cCty&!)5x?m-YLJo zouDmu2TA$>t$F)dCF9S#k0Sp3kh^{ zuC@jz)Tm-9b=cwRUa4x5_(?r^`)6OM#=OP~XWdxNQUpBCi7lU`!Hnuoo+Y*d4lNGoTpKAyizX2wCHr&<-cZ)-Ens?oqjq7=IhDHfvh zXO_lOaGI&x;&*u~>4@oDog}*R0Qm*sqpn2h^nb-;ST_BK>jvb+gf$8M=4Um4&$-jg z?ZQ12g~pl18cCNHq^%WpolH}SYiebW$wlZ6$zrI-Yd2_!qr+F~9<9;~($GD7Zt^V1 z^k#jik^yJLTwa@&gQKoUf#FnUCAhSTp6`BZKm~vzO5Fi(Un?k4rFV+byZr=dyEDyc zax1*%dUiZzVB#?r;FB+!QEJ=y*3={Qi{CSZoU*KseP+*Ff%_Ry9$-BEWsoe28Yuv| z|Em}X;s-W~0xr^CK!-s%0&v$tuq9GqRzuE9b+1t1FF=7opM1#@pm_`Ky=1aSx9z=d z=I;=gZIeB%-v}4}2G9L_i2uI=2FM#kK!ZFjT}16DcsqWFXss*{ zHh~lgWuYIA5GMH)1~mf0{}WC#1R*zP!LZz9DcO_n5hJ=SZ(#33+rb<=p>W@a%8lW0 zLk6_l0JPoGB%pFgZ(Go%ElpG7NBTNu-j1UEIhHP#b)vjgn9gy>@X8pY@7D z$YtS+v^@QeIJ6!)H~8%4v%4`i=TOt)!j$^|xa#qD!3n;JO#2x)?i5jzV&q}_qO$?= z;4VqQY}i@?r`*EoBJztufT8~0eNPFN!vQwdKZ998&%7^EK3{Vk$Ck-0pO7)G!{+#c zhg0z0Q5H3 zVOm3c3m%mws{ONfKe>gwn{>vchkDun*LV>k0>Qz)SVs_A!B`)1-NCSos&6;5pGK{o zg{%S*fhXN9f*_t}g?1M{rT}fbGQ7(H-~DI`*+2QKV<+@1{sBP^A%ZWOEn@Oks3(hn z!qzoxGx>Vw`9N0=S}s43dP=-B;b^t+Jw?4%duhKbda*@E_d#0X$prf|>PvAK^J4=t z7{um;TCAXe0eg^7Jcr}y+4J?N@qaQnFI~PpO5I*@jy2pCail@ZuQX}u zxn)hL-ayj%%~CQ=)ihlZE73Q-&5O=s$AH?tf8u#>lC{O8m{}Xo^SyApwE0%0=F@(# ztGueFh!5S=qFLXp(kqv9_R{lM=hC%)zJUI|s^GWxkN)YiC_3;#pL87)t1ej6qowcN z&-V=mR=!+Bg;+P8Dy004PlnD32EHVgv_-VSn-7&Gqv;PG&)o{o$ZaR!`X+}BHufym zZq+925#PYnUI9BM**9Fdk7eTfsTjE5Cp;64*I$y-7Znc%o9*-COKv9H;J@EXQ#CUiW6rd9) zYd?+26--s-ryAWkEr`y=STEXc zt>1y<^ZIzGVZeR@JmT$%FS{<4R_VGz&V6Vk9R3e=2Y>el{wJG1ettVh*ds6-iT+aL z*7YN(Mc&lEv{8k?isX83{BA%;{Vj*cS~r>8KjSn46L)h2<-5ktI1*J4^)RfYH8&Z( zGKuY4g*19w=>t>i(1YRRTsz8v1<21S@o#)O&}jH{KB!q@Q6v4g0PZdnrgS0Gd;l=T zW7=hZ`3>lx|MWFL_4(ia0@~`ryj!pAy^@FW+$wgA*fuQ8J29ST9_uQ>Zc@ln;6awH zHUvs;n6V@?TwJW}dp$<2u~ee&u}-$bGZOjM0&8EKu4ph&#q3rGAd?q-*{Q^Cd9;m*A@JIL;RB>!6P*6up*!fWso%ftM;9Y?y(DO(YfV1X&u5NF zQ~chkPoKPRYEp4Qi0!18NWP`?V$>=P5MrU}PCOacsd|65FrHb-DPr!XTgR_!*?LPq}fGyB)?&VSiA zP;P*!2)tlGl-!&FfyD*BHDjv=E7CwWhx7R?7J)&_pXC;fN{OZf+w0gf#OHA6CGPi(tdp4hb{n=oq|c|mykEaQSs z{H6$%2Nl3ZB@+PCKYIT!iAH_^?nGUp4H^Gnqz6_rX`m`brZtWcGy@V z%zg9N$77t&E^tWrZWX&POPVIbRpMS0B@H#{rp1hHOrk8U0t>KU4v!PgwkfgQ8pqz4 ztQl<`D|BER&QqG^=k>Vk6zYa(dJ*fT7S;8VH_u#=wifm%@_zKkkn?t z8Th)KqWki*GOHk?ipo`HI{UR+wPD7#ld4AELsRk|)NjJB_+PwRGVgx}&s94=uI0E4 z%6zz}XDA1kadb3hv{Zds`}kYz8v)&7qHI44+Y+XgYa{^BP4j;RTNq z@J0c-`TH*@lg?~pdJws8b1Qxag+DrDFuGpP=q=QArNc+L#J}k3z+G;ex*Ac|tnnuT zYAm7}G_)BurEH!Xjz$SS4`dcjjo8iDZ)kq?z5Zju{ogxazgX?x;m`Qzg%=R5ehR3R zQ|g>l*RJKzzH?s^oE=Q|eVw*g*rK6-L`U*7Cwm%uzq78g+4O_>4Oo&;9dv`ED72HP z{7HK4YNqcMFT7Qa*I*0g;+Ul9m>Az0dw8+8d!~|}ecjHsRZ3M7Jg89q>k)O{54^N- zoy=CpqOdtkFA?&hr<3e2w)ii0Y^eB}o9K1ymL*$)bodpqIjn)Xbv9h6F@vo zgZ}Q*_xG?$-k|N7=mT?$g}I*H1Jg zDX*-m!8{O$Nx5|_^NZe;ZAH5lfcrhsRks~TU;kDoyC z6;sx{F-GpPV#x`q_?SjN6q$a0N5m%s8i1Nk-03NNjAL5i-?aRu7hRz^#x@MD{&5)`DztI8zjGsT#r*vwqpdl!P^D7ji!+ zjD8nP?sPi~8m8*5U8a8jyRp#kU({a}rGB{!e$yK${$#rs3uumHTn;HxA$|XTDX7!z z|C%HbUqhxX25$DyCaWEVaR9DZ=OMnB6*+*nGO@elT4(rHp#x;s;n#fDf1InYwO6s- zGi|ZXw?5!1!45ux51JjNzufU=@v=p&mcGtgt>|mEFKFKnyijH9eX-D=yM88TDcTze zAs((w5DK2o3CK*8XNXPXE~U?BP^Z#tIFw$IN9zw$tC-`^Gu>I4A zGd?<-47rsiO}w^3sW%{i&z*;`-;a+Oa0{4Gwte}kUioxd=%XJ~#eesz{Yq%@55IwW zKCXFvPggYw3_5mpiT)|n#44Tk#KNrublWn zg1^duzCz!2*^@(sp?2s|B$x!m?^mVhfBPv>;KGQkDP%oMD1qx9eCv7(1Ym>)1@xD$ z+9QBJ_pl-2x|p!jGC>Q0-@2IMS@!oZ;n8 zfmJjXlBL+EoN;0Ewr$;KEsCf;RygXge|>f>Xsyps&}r%PSw?UE4JZ|*8rJ~DSF3Vr zVv22x6@h8Hh;%$LK$RAJhdJsa8b?h=%dV3SI6w%AKa&!B%sWG**T`cCF`QM1Di85E zI+EA#YV@Qfr9E%c>5sEw-U&8T2Rch&mh2#?zCkFs6qw@ubjxwP`<6>Rqm4$HDqUFO31s?ifNbtRioD9LQe)H&!!X`#QhLEkIj6_6AFNW(?JaE7nluvMA%r9m zI1MJ-Iu$dZ`u{1PfPxS-ode&{tOY_JYf;pq_!h;s@ro3PB}v$Eas#M|WHa?p*ZdQ= z<3BtN|1B~4-z)(CrUMTZAQi6K)RK~rqD_n^k{V54Rja^)AtwjP3&dQ=Ncto!0m98r$lS=F zbO1A^k=G#a`JVpb9RKBAe>*b#3tvgGXE7&os0L7mclyVAY(L>bBXfAZL)t!MzeADO z2A6LQul*D}KTW**Ph0%g14d{ddb$9%Axu;PmEHv410D&Cw&K4t(OPfp=E3ASOe$*2jX$dZn{Pz&UuB{8(2Fb`aGd3IAe+4+b^TWG`}{9K^rl zr)%}d6Rp{p?1aIi6nF2QfiPqNAo~r^$xQA?SF9D|bT;JQ_aKz+?-$fzb;75c<>|U7 z?>$W7%iW^baalcx8*fT4-3`dXC004??#f&Oez#{852#EwP_0e|7oQ92963YzQt$v6 z`WRX#hiXu(lUfvLO=lp(?2OLRcXiv=2_I)a45iQ;Xn5f{bs&#_I`%0=Tim$W4m!B) zF;Tn*YZc0myE=Iya&gz!uR%b+p2fVpF{kse=z~yt=@*d@g6(u@4le9E_OvC!tw(m1 zySa7gveP|YE@@}!`6pMj56x}be%+b|Hu6?1Xy-T>K5mvCLl3`eaOh&;gt}P02d6~Z z;+hEqZ$^aH^%yPC$oC~ku`!y7qT9i$+-vnYR>zOmV9vIQPArCD>HZV#6VD#LTb|Ft zD#j|bsVy~6;~MxWSD_(R%tUEiSbeoi98o~L;8yP2yCMN51@l*~&ptY!c1NI?KN^U* zdZ<66xqo!GFd^7TnV^hDHFQsQua?!`{z%JcWc?(4zs=Vfm1+pbOpocV~h zcNzDZ-EW@-Jx4UR`)kjvU~C?Ziod=_P4+WCot0BzeOc!5z~zJMA+SoT<{0i-3v?s! zjfOW2RVybMOOK`uRy^r46XBV$w3|rq9!c06wu7wg0l80y8jJ;Ti&RB)Lg5%t4}qD5 zZSbq%a&E`4()i1XlWUhYj~zECZGLzpQoXZK=Mhc4xisXfoXc4H&M;n7LJl2`q8E3T zRs44SiFW=&bJ*V9((x*-7twRRl}OP2lzzZ$(B9Kj#OW(U&+#me z@)d{{{TUUXgId#(5Uf=U9E>ikJBsYvhc~5O-*R-J>acb3$g|LM5-@Kq6r8voJol#X zP@kVy22=)3dIxt%k3L(zidosjR(KN??8Pp!HuQZSDAa_1NQiq%X~=bQo`*#Pd;NFXT!0s#wvkwN*{wy4@c)CR0tXfr3yB7LD|B@(6Miy z#3>Rvk0x`T`!cWgW!f23Jl1E^V~8F(L}@+=ywHrJ3x~sSJ5{p#5y=rtPK_DGy{9E1 z{HqgL-*PjcpRK~ka->)NJ1Il81~X{-@g+3Vl;c9t)^fOdth`}~4Xxni0|#kOkyt!3 zpjBwwI9}wCr4TNAHLML4W_mX;Lo>Tw?!)J@W4uLF#Z;FfFhMaQ1aza~irO*KTe#z@ z!j8Xu9UGSXn5}Y_q(R}zTibc@LzZBI$%*kz>AP2`9Mt)lOuQ#3X3x(R5m{9GbzOKS z5E3C#x5n{=a@D^b;B{=tX=n90Ol5tOblhNGb&Rt4Is#77X@!+#WxY z3(ymF7$D0i5~c)%`mLfpIanBad(IVLya$)5YyM?P_HF!dlhKqc*7?Kq^3b?T;bM}h zO+oaGP(-G6TqYpRUjs;OhA&*Mhw;`rV!fIfNl(^IOKj{gVeDgwLi!uqqY@49?XLH@ z$F(dqoCBujB(0(|_5Yp=d-z)#_YJ|Ss)XaAET^^B?!U>fdEy6@{~Nr{&4)++m2(Jc{FqUk=%z8X41?~_{TLwi67U_OFpgKRBe+RB%ik&zk5}v>oHiP zxIGr+59p%v3-0Sr-DN-WLmaQYlI^{ALT6e0JWL`I4c1 z)RBO=P9|-hSdR5O+^$F0(=I{~pyv>3fd^8J3*TB%2}R!_yg<^ySA%^(j_8j54iU2= zD**>>6|d=UB<6HZvJsl-31lF?>kZxeLaH*v0u+7~7<+GHg44J35w?3ksfGI?^;kwz z!xvuQyEn>#RI&_sdJiKRP`!${^m(8!YWpD(2`eN5DER8Z9Q@PpcSvjx#s8)s;w`kA zN4V!rxAPpY2x~&o;0>($*5hwf3|!g|5BoIArIn5HTyaA*#7Z}09D$8q-C4r>VVS3m z!;q}_xJJI?10F|s*SXB1j!{1$drXu{PHr5Uv4MQ?I254bNalMEY;+>NMZe_5Pyk=qVSVRLBuDlY4rIsx)3@&O+aS z(c8-mTUhYgq{FjiZEKJ3)AcBuR7;idYnk8dTKgakkS!qGZ6ii~hcKd+H}!$j`$=#m z+WkkSP1%TYGKj!n^N|q8{xNeO)AVpiPyYRpoBcs2ZpEtKFuDS@t_V2uqk%iD35q{g zN2?HB9-N1)Mc*%ly}Lrakt*Kze1=E9yeh~IyjW+a$j@8 zEQ0A!la*flX?x&Iy)svhBaBQ(kecQh;rpN2h>Jn2`U_N!0gt zfGQ8^t%s9B8@!@B$Q=`wbZW{Wk;$IRe_#lG7ECr(4s*4niQ!Xe!+Sn*Et{rXL5&r) zRHhv!r6F8rc~x<%&hdH#K^cwv3%g8$ki3-#QdI8CYA^<5YkFB+1AKI|$@fc|bZr~e>^fiHetVuEks`4$ZtQ^cKd=f z$#qqY&x2s%Z8_eGsg#eOCJcC%53QAn(FMeWZrs`E{V0(0N&7PdnP6&aI%0a>+S-MO zZKpT?atl|8R<8!@HHH8PptJXYociE}-W4Z=E*3O-o>GhO^Rq=c%D#DjH;dWjLuy`7HSqsmNW5MpHmw*|S>XMSkXqOZ{ax;|NOua1F$X89nOy zDX#unfteUpB?&2=N?k|0%ht4+oI*8x*>QqT3{D2f78VxVbISfuR~0!mHOXaiH=$9` zF7w3mI0)plb_Vny&UUqzAcxo4uuvc~dkuaI4&}P|B1xKgp(H@SnOuOZbgQ@5c?ze9*wx~-TU&_hX%@r96}z*BuL-tuI*0tETaB+9t1xF2jHTOQz(2rIHv3*DXWmV0zk92A^y}J_*?X|m zr#G#K0;JJ1|8K$`xE=qw_&Wl6vU5Ls^9%vgGWZ6`lm~vy3RRL!RirJ%ms!xUp1?n3J{`naIMnlb2tL zqA1nabN>fPGlnN_|@RG zu|}wH%M-cJ>c4W{rf zhCgP;+!6!!hpem+V1dPqYJdAD`p_B#sy#c(@rA#6FesC8yG^JA#4Ei5poa)XKe}9P zV@&6nIIN>1{we-?@GZ?}S2;J9lCMG=9P+y%H> zvH-b*d=x0tkUaKWUFH5KUr9xgbo5fzQN!hun497qEo=az#)C z8B{M9KrMe+63_#gLJl7e`wscbqQJofRLe7fpoM8 z`pc~XCp||m#CW5A-YOvP`waGqW7BuYe{7absWHvz?iDoFU>Xx%>kCXI?prmu>^Mnq z$*bPkLJKOh?FAK2H4$%!xhIK_z#4e@)s zloV6JZKieUi9%le0`--)07XGs2th`h6qtjz#!Ahyx4@&$Ml0|gdxyPSt`Zt&*Dzi3 zbflB=vyir+IhK9ih;xE?lai8516YaU0L{VniT&s;nq0~?)1U5c;wI_+_GjHW%~f`w zxELigT}|YA(Bvd-o}k-NlD}5(akO6G-cH9BkQ@@RvR3J3Yli%;<9hG@(XVB zG70w_mzoClzucs9#KjqxBgk$`FtN=z)X8IEEtwm*ZHC&YyRQu}SbS7l zd9K0bRSf$B#foI5pf^fSj`LFt%TjC#5t-i;`EcnWa@}EMX{<}A9Gv+ErsCzj{n#WI z+5S;&E|J4s*(n|F_hK6_?q)!X;WOF@5GP-0PNRv8rYKxBWE6`DQa(C!s0z%)(LCSF z_03@-RlbqiT5=)*I-ffk)5f{@9+-ADpt;fW0!YxC_%1Po_#^;+K9_RZ3NLC)L=b_- zpwJh+S(X8PjXdM6LuiAStcN4yUaR#_nR`1Z7R;(ACTba;yk@x6dWyOV;K*(w^BaSQ zmSTN~i`Z|iz&Cm-XO^P~UVhJJPIxky0v$T?5yaOm1g`;ex04_Za(`o^Rjn}U81n{!CR>7 zckhp^&-YLp6U4}!sDrX`4=q`6PP@UfaYr3ZcM_c5SXeE2E_zvHM3oQkr_(21?&Fej9qgeSv)%_vXSo6tk>6#RuSAiR`FagB*syAriN1q7+6 zXE2ofM4PID8zATfa2(>pwvnPi_ghwJ4|Ol9#3ca13x-%r;az5mc)gLsCVgW~-SU`PSfj+E%b;SDJ(Uc;^kHCoR;0i^635G^=>3cK&UQlwNFsKLMDv8Vj z)1Du`8cdVCnpN3TOH1TZT_obk6vCTO5-9`vqV_!D40+h9gzqIFn?ik8zh=G~zv~w@ zNqY`grrqVdlCB6jV9;?~m~%pyatSqRyYm$r*Scvxjpk}|nOWz4Ps5}3*go$R&oj;l zVIu0yZG&Ci2LaH$pGb$l2^9X_|MQ>d0I>jB`wh^VNj+0U;0CZ|D{>=1#bpPu)bWOR zKVa6Z0NXc~gZP#IME$e;HiY&o&^HgXO)8j=5l_N)8{_ZH=^{p#4A+9@Svc<%8lMyk zFYWCi)6M_21br84ZRGH-RV2u`03qr; z>zW{bSg0da@v7(Ju70!B7PX+C4&uK{o<;dJZmel%3JYCX%nRAVJbc`0R%EeNR4joz+v({3mp8dDkDhxI4LJniOE}N=Pdp4f zH<`8+0ijjn zh+z^Rb8ygEExb9SF-iwQ62rr>LNjsopAXxmLlKIsnOYeZRvi&iQis>YvurlIdP6vs zu3bI|`z=L%m=J=R6t8*=uQ^Aq#Es8RRnC0Y+1Q#gAHDe6+(72M$9??WlQ%LO^DixA z*oN{-tHEycP`AP>zO_DA*38!=2zyI4B$AH}Bh;LXmdwKLi+_C_Twi?f4r!X(MzZa8 z9w$8=grWieqiF=^+1Q|qu<0-JnWvbqEM}Js4FBSMP{I1O>sWvyCK7%CZofWi4eHIZ zQqjA{V|PNDwcUrjc4>P#xo^up(tY;jT+6lKnnal7X*KEzSZQ2~SWyh@kh0Po>B<;~ z2i?M*p#3vp?K|Oj>{0_(I9_8F>u;WRRfC;H*nk@kfwAzeHVB86u1CGZekM$C7v~s= zSK!7k>FbUgFiJ6YY3aSE!ZzH8P@h5B!tlZ;3!F&tC|rGmV@wF9c~`AuOVMsla>5q~ z;NT_SA@v`ureT3{&?Amh;=sV0tD@j!k*nLI5-P7q;ZM$<5<>g=>q-x5aya? zM)=^_^|7||`1RMzs$dXjzvR|4k;To^Wr-EN z=jMHx-cSgrya$5@RnGu?1iv$ku#jpt2(%OpZsN)sK7=dV8`HWKIj0nzxOsuDsr#7c zBbw>ldI(Pj37~@MjNzRot!n&q&$zYyW1^uuN9>Dg4KOtw;(vl5i8wf{dca%Ls1Kn# zM}=d^D|CR587t5*=W1>Y(yoKQda`Sly+{YLxelgC7Bx$STa|%EUsXA~>h7?YJzGkt z#~oAr-bM_qCJxPYK02N+{xE@1&PnyCGQ@Ay38Mv^?nNhdDs@ zN_U=;B8V1q=@C}g_jPpm!fuV>LcvJC_mZ}WBL&S4koQ4$O{{LLr?V&?nR%VbU_kG< z;ynna-VfTC9Z|f<*UZDo;HKaOSjh_0>)#6V|e48 zkXT;NVMB5Q&f=-%CDWj$ubQrm?ML+kg46_O8a{o&HUT>>WFPtt&?j^>485J{rX$)! z0Ci#H!Kh}h09((x#JGw0}>QJIJz%exi~72{j?ooji#y@y7D@fZN#1!0Buk=a9ccSCcd;$bUL} zW_Hy5sN$m4qe83H*qYVqnFxfz%pz0MjcfJHPpV?S2oqkZaSOcO zLO`U|_;&w^$o~HxZ~W)G_mYc31jmqt+Qm$4XUdlbw3X2B)J6 z+iwdBF845oRBk@2pM7n3#pnsM@kB=-YC7AhT&i(=2#aX}wcvfvm;R9E$4k}bW7qMD ztv4hGD2s?CFw?Jz3{a=T?!)JcL97WvPi5VxBc)F#8?2w7eZWQ57(}OP4xnqL?tC=> zRR+)}{1&i!9e{6ppXaYjzy$b7odO@;fEug;E}!FIy27 z|G&N)WR$Le8}$P|^fIcLd~k6nr*h2WTRLpFlSc1mk`Z4==Ug@^8}0&7d$S&>R)o}X z4~WawG~Rw(E1e^kC0f-AyZk&NY9ivQGA>rk?p9s{#_j3xI9F0KYWit1G6qEr%>A8A zG3K?#%xyE1s&^foNDn*|DY6xpB0@gSPinLFnC!!H?5jQFa>nh?UJUsV6Z1m%U^y7$ z#B#YclQujO@?*RM2jud>5>P<)nL)Z+w!72jI zEr**~4?}XNUw%wWdOcsKYjzRdBAYX>K1{jJ!<&%?Mq;M#c#;C^xDm$)QW1#z{J5~l zx$)!e%2o^VF2lR0Dls>GwNDyA-n`6!<^c%-^fW0K{$*TM6bKNL3Q&tIWfWLrBGVih zc-e=;xA$^T3(S|PC(3@o>%l0HLunNF=h*n2xMET+f_>(>$7k^m<&(n+%%3Xy8V)oa z*JrtVW=f7DK0|Pe{{Y1gYk2HcbjWG@#glnhLQCRqi@LWH`LnEgO`w3*!#MxoMKwNS zoa5Xsz=z`+y-0Qh#-Lkr3i1QoM^?-01L5&wwg)iTzH#oYLDOsCg+5jb~_L-+g#(%Vns#%w_5& z*1zJRep5p7i)H$6c^~z>6_H7atnb*Gz9T$^T5RkJfSdj8`ubm=aLU{ApW- z-|^NOYEs$LOXs_`zBFXHDa-?bEC^F<{o*Qx2_w$oa-#3mx)z$6;7d({4!gY8nguhe zE?jPdKt>Lr>AN-&jfrz2dM`m8=%eITsuUK@bv*g9qwlMd0XKHQBK6#)hJ0KuSke>QQwJeuf7|1woJHeB$ffqwO_)R8K3`xCwL^D5#P~g%3CM(Xxf2&CSBgeDw5oc* zRJ6EIBL%H?fdxj3TcHf2?axuT(Sr~_mA{3*Vb4$%TN2peodN7_ob1=5-ClfAvj_Zq z9$mWi4O?zbvZLJE_-()QUkqh`lPuwXupIlpJifmO;{PrA&=21Xqq>2zRY@uMmAcCE zxYvtWhoRqY7&rGewx!h$7^moNp1c~(`enr}IjI#M#&t_XD&p4jhSm`QANXbI{o#=C z*@<*$W`4?s9$BYNws_!JZJfX`(0^98akZ>Rm1!0e4iza)j_>ueq9ob{_$h`1HakyE)OEdeii&E#>*QN=LM4-1`Dd6gKYS6SUg#8nxPo<&X< zG?=uU@X*uk)7>t9QL=n zSM7IK9_WYYpjb?$5A#n)b8U7${AGqVigh8zBx{n8AhUl-yV`-mfH48F@ck>YnMAGq z%vIC>!`^!bG_`K&!$DM3z=$-dQ9*i9iXbHs3y6q-2q*|q5fG3jEtDi6DoTrph#(LF z>0szc4G;urBE6S@^p;QqDSq2~=iEDY=H4@BerLX0X8cdn_U3)xy~?wm^(;mNH1Ai_ zxPL~qoPT$e!jb|_L61&qt(=8KzFIj$`kfwWJZ&$av#(nB<5@TzO5-$mUWtmxy zyfX>#rzB#QPC4$nF>F7|(^bOyDEC#T$CD@-J?p&_*DeV?{D$6j_#GsF$2q||Wv;jU zYV3p8n_h;g#g&p<*3Y@5GB!a>XL;`bHRDF>1YpRt!F|?S32i?R>>Z6APg#Mk2K4*nXsqE;c44NjO^eBL#OUDIh4*u(Uo4A?G0#BqwocW+#?)W67#5PSOnw)e1!|`^@<+mC+YZU7-zGMWo?k%uNwhT$<9eRveQEQ;L#|HlU;}A^Y~OLz35`c(?wp@C2ErqPx+<7 zUgv)u=)&JegRG+UA~Oa~jmN&*qcf3%j`sCXU6am!-<>j#+dl0i_$92gTs;B&2`Wfd z683tYV?HVgQAi12V5}zUOi0h=-An{4U{3QS$E5U4m#RzlrkhwDoECr1;6y+^ox%@# zJX>-Wh;(z?A2WGsIg`BiFrKGx*_tp-eyD6QOIceN=}YR z;BHTr3|xfj?q8e+J9aNHeQMx6x^EG_bLhixDSZ<|0gO_r$YoePCJWkwo1I+vbqw#% zijpggI)K}^BP9jek=_*h!TozZD;2;MGFMM_d~6~}vh0p{{D^6Li!9C9&+$PlMR}%nEaWxVc9rL8OTA$e z*9-aE&PUA-L(r1 zWuO&FU0|~kaK|YG2n7ZJtCK&vihh3T=Suv~oC%=&?U6OA@9{V#g}|)&1rN0FHr!_} z{XrMs{fz9{L-U67Fr??aUI!BNT;82O(b^^FW2CfjBy5sWS-QgyqBB#@ldOsDHTZpm zaop2tp-{a%?KLs~mBjg5?lPT_I2~wuz)3EkVklJg*MkF;6T0>~`dbo3gi%unU>iFi zIu~h-7FrIc`v8$y@+Z*!-_ce0kBvwm(P+}8mF+V)esq6|&v#Hds~U*z@{9HSHNyxF z#gAu0R?q%u5VA(%8Ie|oEV&Ehxll#UXJAWW?%_+RP1Tfs;edwD$dQYS`fByw#A!+8zAb>fPGcrj z2SkoIjyr5MdmoxSB)Pty2FJiJ(+woX{dw4>-n5Sbw73&+IXvv+=6##)xdl!5xbHo~ zp__Fgk0i#Qvm6NFzK&$sRo!{$$&uAhqA^Xvk1k7}Pv}9Npnxx)aG@F9%c(Lwtx)!MXXOxiyX0@%ag{d4gTmnKrX=e7%$8m6r&V?j-Yy z#pi+MLH1@ac%9q`vn>;liGCy9qXS)T}VR&Md%Xv7`tA<*OyaU$e(j zHY{ljYW$5HHfg_(PMtRX<)AEzqjAio-Up+j)11wv=hP4Rax6=e7p)Ff?B;I=b^u+EVd%F`Z1?#lQn&1avJ0k7KC2+P{NRW|`NG3FuY1IrPR93nP?^DdlHg z`7^Kl^CQF@NeLp0Tp;aDw5hzKxt@h;zaPGKX2WF}ho7?A>sPP(AR>BZ<+|TRXWE#} z%$DHC`0FQIEU6HA1fgMYE4)~&TRA(qdH?Ed<7Y(U0iCIKV|b`-Y>Bi=?oz%2-loG> zehv#WuUu$}d(cC%C$$?>0y7V>TG+o{gm$?D6lNi~DJP3ZNq(@UHAsBv8_61 zN6I5fTO-e0P^BImzQgj^TwmofYv#OzSAhs>UWCzgFuflYy>@1W@6N-9St8!Y2t6*{ zv$YC89ied*%7yHAXx~eQ)XVcD2FD5L%&X+)G1=5=Q`l`TPufGrRT@E~RnbS;$|-%* zG_b-%!dUPl#+|+lszpo?Py_wSabTDctO}mgWoj%V$)hHp7~tj z&F2{@rg28z9hUbu`ZqG}+8c+YsMr>1?hz6-oUM(1s>O0NWM{oQy0_758XT&@O2qOF z7FU<1i+nb7xOJ3O3H4ZI{fuzWlx?gSt4dIW82|dqPQ4BW=KO(vU(cR2OdLwoU#3@s z$#X_Jg|r>vgVOtD6qhTap%8zaUVo;H z?cOlE{AhfqEMHy$*l2cia%1ksH=j*$k=%B@8?X>fAXA1_#zVyrK9)}XI~9b)BGh?H$?NNPpxs{_(_<*@h@WihvG?*PW|0Y!zO-6q1o z-vwm0o_I~+SBlV$5V|D45knMdw3&<6f!z3242miMHla<0O>}NmkUj%FQACe}^Dc7{`kGq`h)R30-9pny|uT}D2zN6cfGu4sk z2tV$My-kyhxrnVXzY@o_pACBH@X>iOLFkHch4`$F1AXnu)YC+hi$s%XAQ<8u#3!_; z)qYgt_;#ipQ+=Ea38pDJE9q)Hlef7qX9urN;X43+K*UR}s!l?}YHv^Eed&1Xpzl)h z92(PS%+`>5*`rUa-F+HrSFfOZOrb_Cc#im`5SPtvN(2J1&tW52L~9Hw0w z>qi8tNw`j060jVxAfp##?uS7anU>3Qt{bFyc-H-hcS_n_jB<>uJ*m^3OiHUjw5MB^wD^dgSrygX z>nEw<$C-Er_kNrql6at?X2OPAb|1G1H;AGMSAA*SE87)Q__l+=(m|KrWFFB_oRn`6 zQgBqqu^@a^<*%f)l$Uve`F*pPdkSfm`HdQwRzzXQY)`rje%~wH`}Z4Gq3!MI`kq!$ zbpf6PDc>!-pyL`5`3YWS7h(P|*Zfw#dC8Y1T`hM#>nTVGAc7c-*_cY~9k`ocxL5i?#+o2!%c-qK z=F%DX6SABSy)0z&MR<~s$jCO{Y6QAAbLtH?=jz=32;pjvZ$PBOY*&k4J#7!!pz<;v zOP9)i2O(1!s=#0pAfY9T%wvijcT{(3el_|3C*@cFw&(444={kdg_BGm+Y+dKz)6+) zp~@NnyQrZ8^sO()073|c=-K?U{7Qd%zJK@i{=SCmU!Ax6=UDzPN=3iDsPsSoNmYC( zbieRvQhaa8EOf-~ZK!dC9#b>ACKz(d?2DVW0X;v^HMmj8dZEX^Z_+x7@BC0#!N>#u zh*@%l&hcm))uWrrl=G$?SS3IC#idE3^m${78pGy>{PWk{`3+*s(Ez6l+zmFTGY71p zv-@7dO{j&vu!*HLc*vUOtpdDEf_R_xkiRvt0?mtXrk+iRMs*2`JX9>uYAzi=i|bVM zFF*6)K-yt*cUg#C2Yn{j>LpIUb_1&1y}nQ-V)(I3KKazJ#ifo*>4nT83_sktH2$27 z9@AHr*es~Ik!w^~sZIwBKvyHIdp$&mdILH`9yM3aXf!dwK6JXRAIIMue=v2|b<8k> zzT@Z9$k;pjTC=hQub~+=g3`(D8%>M_41fEt8~!*`soGaBv_6Xz`|M>&LIh2kg?Ns! z4kDax4Lt~LuKcJYruRaXPcu*miS#q;p8QxU6kdCNkKavmgPbZqd%X8l(vY#xwNzz; zl{F29>Qv&z5K((k_a`QNGW!1ISFg6Xhq}*VgU0qY1Wh4QJ{@Atcp*GEfS(%ccuU7S zk6B@JT+{c{^>e&50qEKm)-p|_mHRF&-_C=!Z$+!jRtRV3II7tOrnhCyZ$%DSoK;w! zfdp)b=W%bTR$8*L^TIh!9QlCT8T-6$cTH)sP5hoeo~$lAT2+%m+pc?7&_x2fWwN$+P|pc|Mu*1)bucXRGCn{5)Q&Cd{JP%iA7r zy>xhH?qtzUkb5ynpKv3y%TC|)SC0#I)&br_XMH7P%S}T$mUyy-_uAVj-wNj{GlQ<& zh|KYYbAn6v-RikT4`~T$;dW0Q4)_T+{kPgf{*zJu2{!$WBA|9YP2XXf70F8O34Pw@# zp_om`vMacM$TylLzdGzT|B?-t>*_luwVRu3k18ff)H~P|ARQ zLjm(oPMY;z=6Ou_N+R=ws;tE`pb0t)@S<-T#!uSDGutvu7I^!P#}?n6Y7{bcRF!v={uP{5@lUDO%h& z2hY7HDE_o*$sC6_!(W`mThhVqtbtF!zUH6%$c^rMowZMjdGI_uUC7Ir8&qbhzFk4 zfi1lor)z_ItM$%|(+=AEzKSW#*vZ9a(M@6r&uY4IvpTS6C|$MuV}YIqi*2?_>a_@) zu&EPQHiR0#_I)cWF?!yVO=Y{h?r2+)LcpCv_&jV4`uCX$+n)#@#{s6`&vM6qnMeHe zK>zF>=zy8|XsES#v|sjs1%Aus;{{lx5)%vUgfJgKxCjNzL=8YaX#hF~FKYm1>^x5_ z5ZMDf_;5hR_Z7m9@l+Ogbr(>c=LaN%B}nabHgh}1o<;`1?Wr>&Vt0-dwbNKJv>$Ue z9;Xms>!<9o6kZ7B$B+$xgNmNL+XW-Z!N`D4@L>!afbIffmgBe3lK>Cs=NFeGew&fsBd6MEwA6J@XBUEk%n|F#0iTZ2UEniOP2u{NB%C_e7nI2b- zS1*LSTVf;IwTm(g62*H)_fP_p5NrCp<`3%B-*h9Bdp?hGBGqw8xZ#Jlw%8LoX5KF; zS6VI5`fQQH5KguWE<)5JkVDn1_n}HY%If0$UKs*#;o^9iaXGfzpaGk61 z!LqFi^W1LNU!_fa*wC-=EWFA{&(~qs&Xp)@wBX;wvsDY&vx$ci%>8YAMos~)W7V-U zo#1ntZ%yjd_QH0FL*5&NWAjr-7#E?Jdyyh?D=@z?UtAI2AG zL?nArPxY;K_%CvP`kIo;nEXt$pVLwTEG`S9~r-OhFx90)TX`U(xT&dp|kp@ zyHV+nFohW86hn`^q0zkkY9>|LhLWi3>FtevxMdb|*gzw<>|120BIj@vob}rK`KB)^ z6@H}^8ok2_1EUDear|@IZL;Oag+b>xjbalv+wthUDucfCDx{oV-J9Wote3Z5eFw#E ztXm?z)K60Qqc?m-X4_dP0))|h`8>X|Z_MsIcZS5;KxcLiN^cCOpQyXnU6shMwh=Y5 zgYUTS_v$2x(C}49%PPN2=+mdF_p)xD8m{$0GOH&|jOKO6uV2qRVC0_Bsh8ccznI(0 z2aJK;mzCDLsq`Jha(ry$VcJ~dvNbe&qD!d00r}d z1oICiQ9x?jbKGTT0jO}zH3KSzvA`@iXr{7skpop<}+{QAH0NJ5~n!=etvz0XklC>BINJRj{G zSvKNH%1yQwPj9iF_IrjR(&! z9vvGMe38a__H>fwS+`>%c^cQ82lG^=T?X742Y9JEJUt9CO1p8VlkJ{P80W&vT4Au^ zYcC=VmlQ6Ycq7hj?bsKIAF}nkF_bhiV6`W3PmV=V^M-~$<6})2CJ;-^U1XLxz|hh5 z*9J~@6%kKkN{s2V?$-onqIlTVyX;7h0*t;IzS*5cKbB&EfP65F68;@@(T{+pFlz$e zzrC3OBX(dp@Uu9C%XbjSE@dHqQMiYNrlvTdLg!uq;FWQ(Og|Kd8B4~=<# z*CC|Y_Ge@3W|crWDnPMgqkj|Z5X2DTBMUG>wg4gmX|FG|Lb!Ix5KTydil95JNMj68 zyUx#h{L_B@T*sg1@#i!1)As!H+ef2oXo@H>S5;Qd&69`?Nj#*OE-Cub$~+Ht{rTtg zjnAIvH#6F?CO{KT6soa#@Ch)?Mb?aPn%U!}{>eU%{C%)b9}9=X{1~%qD;DRf)ZQw* zit679+WaA{o8|BDZVOiUlg-z�kThn$ zT%T&=wFTIv`1`Y0j0PxQK0OWsjSR`}_AT(D!KCOAsmb-^%xS~E?UWtdDive?)UGpY zI}!IDF;=HGFKusiEI%H9VeK5AzS<8?pnM`z?ayN3 z!l0Ucj5c@IXUh9a7QL7SFjveb&Meszl`v>4ph>p!T(OgSGxn zB?kjysCqjX^=0%_cU?)@=hW@&>PE`f)xX{xF6 zD|r@=#g2eLHAk3>ky`yW3n)p_O0bB6qi%!J}Hmx|wD~y)@_5asvNEsZ1_)wv-npYCSCRjUj4An6^54?U8GZ@ie+kmUpCL zyMHa)Hq!i>*@lM8Cnyvs(b%pjx0&7)VUgS=6e-`K!?(2n$4;O5qM{sgS}Mh{q{+X8 zZXoyJc&hhGVIoeZ{{IM@Nq9figwZbyqQ`g~d7919>u-{I;a=yuglPY&gs4v!6dZl% zXxRAcqfc1at~0OA`kMmJa1f)H6N)1WS=XysJr;G5nC%m}n6DM}igT|HR%oC6G2w0| z7)=qpMEMR<_t)!H7>=j(PM!gh##sPAehQ~o<7d+x01W{1VKmtT=nIu+jKSB9mC-a= zXBed%K%fD#lM8M~G7it+DTUB10g$@oJE*je&eaA7CjMDq7;*rTZ3E~yE-*rDae$g= zF3@hv3!u#k-$6?oK-jBj_8nB$3&i!I03^+a-UOI{$+6kMI4DN|^#1e%wr0aqk?4*q zJwV`E3_#-(KX2{-%&J*Ok`>K9e+Om$@qirY6FvgTAIn`EA{2?*!BniR)Ob&7BK z(lY}cnNH>M#Mr%xiL|dQqp$*%rQC7Mpr1aH$Is@ghRYqMxGf!f*7xK`Oo=0(1-e9S z3|b$Lf4lRHrSq@T^nO;-{sw^YkLPj#^T*Nkxc&Y|Ny3F>NW)G*VaK|jwVOH{?~Ct+ws~pXm8woJtLPAe0DMTuUZckGmW^JDB-c$LE9P@g}qvI@IIMHWzl=_Ki_s zgAdg5a%a}Y$yf2Ens*CD|K}Nl|4nS?FLw(RptbuRKZr83SpYmSz7)vBXsqYN%42ie z1R|Qhp5OKn*P;IT^h{k2)Qwc}_aPwBgz=e&8%FgX?PZPL+xE^xplY!t1|ijKh+fzL zdXg4%y8N=0ki7oKZq=Ccm19K~mm3+D10Q$46cKgLn?j0qnQ70=r=C0jn$)l+y{%;^ zf;C4l1M)X!mP~7;qZnUj$1u>1wb)Wto9*Hm)z$Xxl$7mtw5I=;%M>+}4Ttt}4ZnJL zVS1?2w$QpjHu4&nxTi7`eF#CrOu?~}6!1wxQ>CJb$QG<@j*@6|s~`u(ewD?TWEQ5R zW7F-gnPIUs-@r473R8sz;$rHjR2M~xeHGy1E_mw@4=KV#C^VTQ7Mnawk-wlahF}0G`#zc&`W7JqOs2Rwiqz+GVY!zn$=Qla-FX!M|zsVrchuI62U&U6rkYIC_b*Wib${UKo!wlaW6+4MpU8~B8Ru3V6q}fym^V$F_Bj#x<>pA}VWh6| zTH&iK-Dc;d4LfP4iUO&~Kk(W1wg;iVJ?MljY|7L3nK@f9s5&&j6?aCa{;vU$55!ex z-+4EuQa+vSns?K_chz#Zc|;^cS!NLxzzHZfVfR+bXP|~5#o%Yv$7$LM)uFrbgQdN- zUo>72Wo5WFOa?!SJ~UPNrf3@gyYfIr#v@w*sd%OEQRkOl<)^F&5spSKhm%=Zs)nE+ z;zz&BGVm##944Hos@IjdmZ_s-*xanW?DEsMhY@ zJP3u}J9FefV_N^qvk4`IZHYnqi=z8Wx~b`#UKCEE%fJjV;_W#2;CX@J=)I;x_S4F|P?X*&(9V%c1A?6F_182u+6?BA#F0Rr@1)as*c)#bS< zfD+by@IHGjPe5=zS*-Gym5UPT^z%0dap0CUbo=oE)$p^6btZ-1{P??~i!|Q?#U`q@ zCC!r}tOVwS3t+h@+!LKE@7-hNoGxPvhEFYjGMc>WBgqY;XZeT3-!=wU6eAWNvpN(&ZMe$BC-iT{>>hr?qhmkOgfS zLQ^CmO9H&10+w+L=ilnum!j{Uj(@ZV19QC$L&m#(Ec<2$n7C#5z}jX5#fZFf#G^vG z;~nCCn$6dhDW>7cC>L>7)rA^DzOwl5&KixfBxI zp9+)_nStlTqW19Y7>C?B#;RU|!f9w$rt4V%HN(uDg?Rj4@fVhcF_Liw;w=&OL~a1) z2YFUm>~hi3UKbK4KXn#jjv{O+S~8lxgV;|bv?$Cn48^ILG9f9X1Aw$>^iPBPFEbKA z(O(aDgJMRA!-Xg-Tuj8+XwqwiE*NQB;x8&OZOa1{!Cum664`Mxmh9ay#)T;!p55G1 zdsPS%r|KC-*Pt6<#i;0}&=?smzu0asQ<(8W8xSr3uHfMRr?2m-PEEhD!0*``l{whx zZs*53G<^POvmR%rsi{!y?x2k%b%y|2IXysbLq1@v)VYjgwKLUL{oWJK-t>`HpqXz! z{)QS|dJMgQtMYS=(YPM*e)$8c(uC~~f2&4HJ*;pUGa>Yldh2aHubbxjiu&>T_mbA} z%yE<}-n+gw-gUTZb2I()NKSd8)0GlsvFWL@1ONqUNdnl4Z9wl0%8ce1!#FkyVW3-< zFQ9GCZ@)nk6lT%uidNr2W6=OJ5;h@3{0^F-ReysF7&2@jq}Fl>ovjbQHHB`#P=XQY z?e8G@GM3Hjuvnn=RW2BySt_s4x8dZJUw&4L9u1zp^B=AXFp!^l10Tk&QeXV@Ue=-$ zZD%Q<+r8weO1p?yaa3sLPBPqY9PgQ%M`I2q^ggjMH;7Izm|ecE@VV~tts&tGILi0V zulo0YQbq_a1E9_H z1oU_^;}i)t;4e!~R>yMPcYHS4V;Gcajx&gVxu*`Z`3$8yJS_WzAi^@C@(3zH@r7_B zSn6N{PKPV0}XL{L}CTZIb{IO$p}r@4%<%8R&nTuM?XC&|Cz80Xx8ql)pV- zhov4?s?#v`%nQ!xF9Zmo@rEeNN}HB(GgFR}8wa z7!!Q$dS0H23O{b}i|L&M&38E-)PmF=d3%0t#F`%DF z41i|Qp8)y?oeTkVza&7ZNj3*<`TjN2_mh0{R|)>4cZN&O{91gs4hAoNbrdj9Dpw_b*gX#UbB8P=Hhu z&_ataz!70Hi)hjbhBcb#K$D*X>I@=vV-(dNpPgki0riFU=Kw_pC@>t|VQ?}~zze(J z(;i>j*jpvtT%`24vV??=C=?`W?RLHB+AJsk`f}^MzOE(s{mFj--RwWPaOTy*dUEMg z{CJL+w1cZHIpSrbi=d@>4Ny+eF81yB_PsP4T!-nKAI|%KzT1D+&V43Q42^4RoZMXr zOY(ov>EwR;1wp50z%0s1gmZWBSD7bK!>!e?n6}#iXXS}!hf34VQnV&S$vTAg!z1>* z*?|(_r59a;C?~?E?!j$YSR#Xh&O%b~g{3qu#Tv-r>NX-KB-D`zJ`5GnOM9p^1RM-o z?K6Rxt$XT7!Ke_>X8Rx#2p?>5AXR*_+i>{fe` zu#2xPuX4Yzvu=7LlkDR0{6^5izIWhkv;%&S2R1NCoN(+teQ0c9Ft@RPJ==cm+X}OL zi&*FW3syCc^dvs-e!;X>`l#|46-{#_jg;1(B0p^>oc2wo=Bp~md)S1h;m}WAS7M4&v|*gpfqlo)!6aan?SZlCV`MJ$L;chXUCsNR90|+%aAdtj7_>*8cL878x7cCh zGKuD)e1ILM`0{lvRV1oYINd;^pCWC;Z@geW#;nTsZ5PuftOgdY25H0w^FTE^#(ARD zi%XJj+Zy@dKx2$jp#!k$lJ63Y>*7%3BO^SfYc z$tJ<|;{ggZKB_UJ0|tW1nfsrnr;4jRjq#Mx(%+Z$uZgqLv zH#p8hk{&dNSoTY0G>MKgorUZ|ENo^{=;W;lPyhr^Gcy+15)#=F__G z_uPIQ&pi};M)2hjXb+edFzB^s(RQ-38r~DIBRUhZ2L`YEO)2PPoIfH8D%^W9`cPDO z$BVr>tqKyr4D*4>$=9J|pg`JF8tySAtZ_4@rC&Bo^+CP~UwPjW|7Qfr-9V}tg4-w9c1I_9>;oXG z-h?oYEYg7AmkZ!L11m;dM|+ir{47QQp&O%oS+TR!8mSh%qa;(H5aJ!U4fAvdyu^W| zUUBpl52D?4aUn~M&KLpODc3&6@}ChtEq!LVBk$-WCMr#pi)qVjkxYaQE@+Wcmu4E< zh3wBr4%5aya7ScCb(li#XMi(aShNv6)bp4+8kyiy~Fz z*Y6_vmNhU%$|+ae*iN1Q>@y}+bMok3cyy@$J7HHS1YN7UxL!hzLu{H=aQMtzrXnk& z)`zd3wLa;*R0O$mRydt1%;+5~ZBQR-e~RXmNy;ZDMzLNhltCE`^`0J*xc_X|nHS>) z5&MSEWn7bBp8rz_MOk4Sa0l+sj{wXP} z{}br$9(L<`esFLuqYt;>#eWa7*1I}&%%JTv7t?g&ae6M=7S8Izk8gzST`{NuJtqK)w4>1B{*{u^G7xyJMV}LM)|mKQtHxT0|(r)}=HtFX^!>x zz)*!h-U{u%z6croaij2U$4T2px?+CR4*!|SVa@QXz`>~lN>qUOR=Yo(siNOo7;!Tqy?r-rHU~MJnAJOAU=3v^>YT*&+ zDBhWLos9B3XI#!J8lwf6ty*I=pfLPnzEuhl`FwOqYz8sz%nzo)5PM`5(F5m zthi6b{^v>32IY*kn~v=+lmL?Mwr!x$G7#A9@ue7@fm1FqxDX|cFuz6PA$ zvqB6Y%h;n$k5=|F%5Xr5bufFX>?o?jaRYe?X-XH45MXhIX( zNl0g_1j4QXH~y6qT8$kicRDKW7X>*7b{Uv=cZOWr}$!=JtPjk*P(l768pT z=^tUUZ|&g}Ju5W1nfbM45JOm;>Wc0--H9i?22{8vU$Mwku+TINHpc;qj#Da{DC+}2 zs1gTndVwhj>=N)J7ltkaXUBXE&@?d=ELen)$grUx#`6*&sCryH1GJ&MJ7d!iBTk@8 zS?HWj(m+S5V>{!_*@rE8XaaEJoWVO{gJz7weOTJVfSgo#X zXds2L2`tQAB#d#OmU`~KN+=|}ng^kM9|OJRRr*>{1=qUz8Acx5)Wetm3MsFg-s%)z>zk4X#^tv;ORpn;fl>dbRJ`zR#k9!jD6~*RE6o$IE^B|-N!*%Jn4#PJzk{t zVbrmi`XTWx5~csPh2unUy}u;{Ej~kc!FO?!?K=NKCBlDyS(YhG{mQl;GzJj7_kN0B zkN8q`3r33B=>9`O9XBiU|BS`|uAx*ukR=tW)=fqm7VK1sMiQSF&n|o3@W^-K{#q(^ zFT!l>W45bCjottaSeU zo7$1M6t}6VfBUBYLagnP^8y@!Vrpw)Us-?^B=mZKca9GRGY)qBvMV>g!e&DqAQbg~ z+eo{!z-r58pvRZ~xY!#B%#3FxNx&8r{o5A(?%!Z)_dgHBb&&{>34_3mcBX}~J<811 zANpJDq}d%5tcUS3p;T;T?o(kPwkAry;qRTcusd2L*YQ&Ao}l4wk3SSy+s46zQ470m6`o6Y%^2YoJpOC_6~MgqV}Obd9D9%h@Dar8T= zZyKoMywg7mdH74omE{Y%6kESSF~fs4WTn1LwscooN_5-?(du~jd=L;$yjFm`XAZJk zL$i)-GSH#^MVRGXHsdw*jdh3-Rjz*+pQNiFB9f*Rv^-n!YP4b}QtBaz%i zy)PU2v$I9=)E^!^;{V=5i3tc;8!?=7@FKINgz*;e<1KNM;45lg@jKz~Mg)9Rt{EuJ z5>|BhFQJ@+TU0+Eeeol(eE>VuM#K(eH~NMhu6(GMo*E$G>gKNG3h6Y)KKRN~hb<^w zf~TR{g*keP?pJf&TG-ktN!pCvIO0;jyFNdwalf#hpyb*S_wOKCFc;HYH4NCZCpRfu zHG{xEa46<^R-Ur7;Tyb23q)8GpCN?`JE6;_S@VwFfO2AKG?fi)$GX(4%Q<=U ziR+>q_s-4}gXRELmq9e4H&K>eRRm$1=V@}Y%N(2K=`(_jAdWPM8ZR1qf>2RXSs5lE z&`Hqj9*{p^Joe}o1uDaXRThqL>bqo}TN=(%&Nrg)x_k+MzNIJm8+gBCvK5#yfBj-Uc=pb%mZ#3CkTwudBcwn&)qu(37R9)UoF^1c<7x z#>XpO8bM=HLB2DgfSpM$LCKTS3W*jg0p3e4IX7J`&&F+I=eRqk)KE3gJ!#Md>ddO=nMgU19BvTgM>-&kRrkA@JG>$xsAzlU$kn6 zlnL5CO9VrfSw?J{{f^vQ`;J`|g`n zi;s?v!4z|zFS6Kbp;8&Jj*8*0KGwm(YW3c4`MTQ(LA#mO`j#B21`c>34VL%!y}yI@Dq{{1Y_eq+)uNXO<$Bau!@75M zLE^3tmSOHm^B^dh{^pwx%nldqB*G&?DA9|WJq`kYJ^-R@ZKA(}UOh()G3rpqfG7nA zpib*d+u|uSr(%-IlHG($7(f2;G}+=}t^>}_aT6<X8)sXNKXaMivn+rga0F z#t3>g$N^0`^BuJ83j%cHACE#~8sH_HPsl1dq_7<>enzEbPAT7R?TNi1wsIU=!IqY5mLtwNNbk|Lkm{!sCXo|Fat-(QEWh?==Z-H?#QDgCAjB|pW zcpCrNE|l8=5Q*KGalnjfK{M(W?L-|$aO&rIeFvHGV3OOitmJ&}Rd2XZ*Cnc+M)`4J ziCZ+Tl}VR2+=8%D*Ww3QFYCx%O@21WiiLJZzVPVHPvMt{uuMoU>lV%-CN`mAYSx8XBh?o7*s?>9*tdb|P7|!BZhIrdNuC z)5?Vy;nZV{R$rd6JQ@e-eT6Twpxz3FLeV;BVjlJ!kqdT8rO8mNllSn;6ELiEMAXWF zhaVz;eeh9?qNKL;J&(Lv_M*E2h-(sk$;nTcNn9b2UqbtTA0jfHWdc3gw_A9h64R4R z3Tw^aD7wU-HwnJHu$imI2UujJWb)!w$>j*eoAel%ovfc*H)LL9?#rD>Tlb&8xjVA^yry`qB8Q+=yHsKFCFTJ|IOybzacWRQbB*atK{3 z$zXvyqYqMs8q}_b$IBF24Au+KB}ZZ*FQOu^@vw9Q%}`ZO^15LxJA(?xvDKBHrZwtI%CCqVXWUl4r2prle|&j zKornx#M?D4m*6emzoLis&^l@6FRbtm;|tn|`^A#{t$-U0I0HurMPe%m#eQoP0VZDH zLCKIG3nZ77*|XXY-V1v4p;;?YYvfQs`qSH5mqFWR1V|xl$%Sg?0N{Uzs!vhM*DiNB zgJYe%bDw^G4ghNB^YoJrk#sn+4{AAhsYw!NjLnjv1wR^-c9ZDwD99(loyOp0f9s5l zJs*_IO6$SMlDQHMKUMdYtBto#c28EPf3*@O*#*%=b`4UD>*AoHyV$dZvDVxLfwQ|E z_Da6ZNG&*h|HyBvzaFx>G2n1|$v)#|ttT^mHZ4pIlA&QJ+kT7^2mrueAcv2?rMeTX zFm|4&4ThB7RM)Cn(`=Fk1py}b5#JC~#b|!_5r~NYHDFh45ev26EtBAZ_Qq0`7#{hV zp#iJmP)%(fAvRv4C2g0CP~o)bh?_+IUN$^QfN?;DtU}~{W=1qyf+sn6yWPAjo7!Vr za%%py@KXc{`ZeJ8GU$@Fg~Gi9IwDu)TIu4X(vV7(Ni=u>kzJ=Y0<=XsoZ6r3B75TS zMJX!B@F6Xh7h1b_?R%%4`xmzCUw0}QxqV20p%0%(r-$?bu(`xnmc}vCCH%HH3W&n; zdT@+q$}^KFQEM~zmADCoKNW8&BK2xhxL~NJgBpGH^5zaabgDA;E~rA(JGEazbeC&7 z-%-C1dJI_mMtmICoyts6+zcxu-ZUd)`dYT^t0C%pq&yiuX-5E4Ov+pODWwM8X}umz zIZLv#VmHO*HDt16~s!PfYoaCN>&}+^m3> zLSl)~Oz6$DZ*jUVS@Y#SV{2b2mv3q=Y($NRJZy^FYyA3hWQsd6deYoqpA_BpegXcf^(BDpT_&t+KPE{I@19Cw#`udQDQLYY})*ZnX*OKzw zFf-SjcZUQ;Lz<85y?s#eDYOVs`X;C{+TrV33?ZO&Cl8bb8Zv-P{HYr~5kDJ5QwMNU zD*%EDHw1!iwn?fcx`XA%aMH(B7=X|AqcgH`(-%k(!K4)cHT8r1VQ{A+W^n_Ej3C4@ z#^FsE%>hWs?H~gSjAt+co_1mpP3ZxW_Aju)KKdbaFq)#*{~bim0y6t}v>k2-$j1%> z9hEWpKz8|K;^+UVyMvrS*Iq?&(qu^B#estrEy_eXz{szD&T~-SPtd)I289N%g0ywMQXv z8Yjh=#7une-bi}Xe%`Mv&?5A+{Oir7zM!J!n{VVnOS|Q?-Qqy`)GL(3)CqVEnvbG> zIWWDtqKihoWW1uMaae}jpgE9y05BA*?KIBNN$;l~ND)(DkC0b|)Qzy$)FNIu-`iIz zcHzRy5%&N=r8|#iWUKPG?lZ4Qibfn6ylH84h{x7C05q@rj%|eb6O@m9OGQPPmwfPT z_9q+H-k8HTMno6b=99u67``~N_uBPUfJ!h94z!lh0{LV&6G@)Nt;m4> z;F%ac-NE&hZIJ`kPB_WHHuS`%$u zaS+%X)bh2`goxon=1?5#&24jJ#Hq?1`4tgvyFap6FN!EJP+Cwxq>D7Ek=~{E8mjb!5&|in_nX->`t1U;;JtToD;o)i^C}>J$PRpIFv5}p(7v$Swy*?dx_Y|s z(Zp(#7!9+XJ0Ck}aN!rZGLoI7Exw-%QgjC@s}zDMy0hF31?<%N+lA+`xt|vm3yTD< z@|PA<%}9TGC9AI7{qUIF3F`BhavWJnNyzAG8^kjKoX#WT9B^#ll`Stz0D=vP@kzhr zwJc|~tKe)>8=UuoH!0xYL7l?XQ?Tw2Z=fqo2NAl_6U~TkX}fKu8Y@(JwL`}=`0vlY z-$1>_Paaw$3i`jmnx;rA!QCC8z|>8@vJYiW6KBn_U(Bqg<~zA->uOS-Md>Rrn`qF2 zzvNtWSze?Oe#?o@@JZHWnH#Pe-b6ab?`pBcR8mzUL015p)5O0`Gr@5H=28GXP-_$c zAaOqUroa)|QSFX5N{7a05G##y`-E$2Gc{}|$dc}Lk>Ho!`zTaz@jhv;aZI0s9I$h(PPL(jZP(@;kt!pz|Q#v{>X)7NpWU>P^zG8N+gvb3_j~ ze~Wf_?^xs~7sy{|@x^1xKC8<}Tb&sxtbJo0E*kCMG3nQgc>38y=*Y<58TXNr=VuK? z_>eNSQ^c+C_Cx2CYEi0tI9+P3a; zKMwNw4fiW;Fv|Q;&~rQI2%EIQcrty*_z=_akGEnTW;%ck-oZ?l^js&O{E*; zhYrhD1O`g%JL0EC9iAYl{r;sbOUT!H?wj$CA_m9TCV zJv#ZD5J4JPrdF_rnEX*6#L8s5Qr;Xe^>KFw=$tCsdk6UXo zVq5SCtFLN1UFhxJEP0A%uBK?nrsd7F-TdaS2|!)Z?U_+{+l}RNyz;_PDMWp!Bw1_d zs6jz?M7Yk)>0BENk@4QST#si-Q?-BHga5~l{7?Uz=8SWk;mtKtJ8xqA?5=_mdk4su zvR!x@jCIGQe}hIUGCB^;Yb3decgKcx?(Z`Nl^v2us9pU6?y;Zx9|eaGx`1&9j{Cv~ zE-@<@J1)_kVnvW6YniI8vunuwe$JPpeyu(KiKto+YHOouhqO&7JJRJhXbNsn1wdu( z{F+3)wuj5oE)q3THDWhNtj-4^5i#3LP zg)z3dcIDi)-;W`(q?F8QVol z8Mw!>`_H`7|M;!~FCbe%)JPH1ksUgC#&RH?JjnsaBbO542igPx1GX-?Ejn zmTfFb{GC=!b`T$nw1d1|;KGDAEYq+%wss~2}hm)UK)?hOZu;QY29kCoY0RR!OEIx@g{{sFW3vkFjvv)cVnMvkzO~-t4}79seC!-bU^7}TJ=WmI7wCf#PML$ zan!)!s9pJrU8_}){XZQda2{CT z!b5@phXy}fmoIu$3C$5!G-En#I!}|)^Vx(sU#s1k>N?}38f{VAhf4Lf59WBCwwdH@ z*$W>^G_Q7^OuU@?#@ns?>w56fz$*y{n@bld{?-J;7@&Y|MFCp1Pq>4?8P!u_-3UAg zbYwuoV8-A9yh^ceeMQcD_p~C$zCp35388RiJ~>@%j-UA6|3|A&4SJRd$SHYO5VLC7 zr#R1m+iA->(Pzq*57uRFj3&7#C3YFORdNNZ?_H1K;5QyB6x8Lt|7fGC9CycPC$i{! zq@qA+aq^G)5#djeli>)rZGH5(ki*v(Bc5gwzjUJzT!Wo(2JNT(}M`9M) zT*P#c#Ji6`{7g-s+v0lwV?skn7x`BPqv4R(I&Vx1+IG)%wiz_nSNRwFu8ZC3U?`)a z^-6m5ltV6kr!eesXv%+>#1dvL^+a^+7dJj?z4 z@GOu&_y79C0Ci6UfiZ)mWeLW!ynr4Vbbx>q*&>Bp{n-e#aTcCdL>coT%2Dw@WeUKD z1pr!G9yQ+uP!25&{Ue-O-n{Eqbmsl-#}^m*7yEO<6ynpam~(U*0jl5SEs%ks7gdzW z`z4U5Mh>_P>EMRrLBuz62V?;0o6&T;m4g*nCM7}7jUq|?FGS8E_SI=#w_f)6gY17v zkpTw9DWRqa!sx>`!+MpZrCwW$kvVCS%rhA$*7?XEOHZFZrc}TK&Sk!F%u z9x*{#q5b39JBaL~B&j=_K?9TGWJfL_m@?jqV2!D57mfmkF5kdy8CnuXMv^r%P2)S% zyiVK(9}K)hIMr8pZ+ddrqYifY4?$1>-3qbt5t5UhG{xnY4tiJl#yZuIf0DzB<-~oU~!Ne?Lu>0phVj3}J z!}mAn8DgL-YeQ&$UJ=!0>Gzm&=!mSH@D!)K1R#P7)OBd7Siv=CjVy2TXNJ8A$W^BW~kDOHIZj#gU#VqkQ`71 zg}4*sLQyF|L;Df)=)foU$@dU%jNo`k?-13jAe-c}%2W&uIli?$LF2~x9B{tkbssL6 z6wVj5NOnmFuZAbr4sY8H)B(Ql^9#g`4A%fDHoqatFwzL*>q$6C#wV9VKTp{q>SNn3 z`iXm+m|(8tUE!6RLK~9=VsN7fVYpPc9>hb;gxk4sz}PVD3u2h~PPuXh-ch^JPhWl3 z?_{jT*6t@6cFH$C6~=h3B+%hkCU>UM+t|>NoEnT>Bi1iJDy3q)q_?ykboh1kGA6ml zkmEa_wj=o(F!c{{!cA(XJG+pt)Xz@!Pd^pkw6=%r@;y?LKFI-F5Ho*+Yy|0-)S^eb zPMQ^vlgHne(kZB)r%tpyyKd8KGM~YI0*w9xr2eg@g8(RPi#B{U-p7{IsSm^l6CjqB zPg#+)gi*D`5X5iLL+%`~_rG=0fAU@;{00%5q0jIxx2U0x;5TPaM^`Hobe37=Zm#Xh zeG~pfb7CxYHqbm69Kl7tRvr)KkZ_AT0|$EwGl(MJ%0sQ)^S0>e6vwN_bHCqsEl?W1 zV{$!+3kI~XM?JZ1k0wz@EdVbJalTO){HB@R=hducWvp5j-xufT0?5ILlgU=?W-P&a-J?=Zh6x?Qk&cC62que6yatGw)K=X~~F_cP}&Pr^dRv z1ta=;=9ky7m}{E_iZ0A8mT8sVW{`r#bBeq(uoplkaEmfbEq*S6zal=dg->M-vyXFZ z*}JCiNpTqb^OB=S*4jVFg=82t0s{hDn1PZC(OcRNksxP^B3L}{#4hu+r+d9`&%F!} z*HC|^H^$fPBXK36(Tz=Ai^3myL2s=I5|E_ZQ%OUCEVjO}3#*R5b5%+`TtzP8zUofLtyJ!Ek9`q(1 z`uU}iv>msOsj4S$QDUXyMPj0&5Uvd?K+LHY1{%o~3dA$?L>(-3@~Rx7a%Nug+T!EL zc#cQW)#jLk6#JI9r5bD6bl?<3$-Aq6D=C40{{L|%EY+T?%g*jvLqoRojbIiHyTp)B zw_;KO&v%feMwZvd!hU?q%E^}0P2%CP zO_ASyDaE@dO2=%XdW;@_IlvbU2kc@SbLD?*BxIuXkerof64SH(8qX1Zs_7Q&9@*dj zQCazyYyW>oHTHGu(uwIoS8CQ&kJwh#PrCj5W&I)2*qZW#g?@+<=ku7DtS%mF%3nDD zTcpV^Yld5y-p<84h~<;bg^l=(b>#HWld#Oir3U0Ei422`f;lZCnC1h@+s}x5d0YEj zHonnZD&2x>1@kd%bPQxaR)VbxtzR0yp!L0req#Efw${7h6hUo8Z^kI}e8g??X!k&- z&hDfvlmVC=PJ~1yYtD+(<_o+F*H6{17JQee>QuRqs5;nYKab&TMf~SJF6M829OskY zp!gmrc$FVl9t~Y}HzUmh0o5k(Bn4jAwsHFHm7_r==22^Z3t;eH+ZMmji|c z=;ka$S~&{G!gxaY_FtARL=@hl4qqLHnf(S?nOc$3lr64XEq_8`AhGC{qRk-_T=<-h z8*|<3>U!0g2k5u+HHqDJhn$MDVSecfbfG5*pgSvY!xSuay&-Y6|8f|wI>Nu=QQzn6 z_okwJ%B|K_1NOrdwM`GSuP0Sj{7Rt+0|f$QT+1Ux9xIp*@uT>D7zdlMuN#WWcJxz- z(4(2GVZ(RAHtwrxas9E|mZ{g+64nE>HjDW)r3m!#B<+(H#Bm#ta7F_pe!Ub@d&|Dt2`fOUt=7fIgf2(3^D89sd}I(8gRO4RTP8 zFSPauEZ1YWup^A=-tVr(4JGz=t-bStbgkc!KgWusR*~3cu9iM_Aj8W7mnD%F=)uk=?TM$zJILXcER!Z`jmyC}Hvb7tma|U*g$TSjyhV=02fIbgJ%TXfmhmcM z@)9VFJN*1!hSzlZ(oJ<+$9aRXwUf+q$OcYtPNA~fVxLFmUZp!LAK8C@q|;$F)j-m* zZCS)Jk3o;pmzw}r+ZKlhBzkO7WAguTd;fm3!QDDIJ{>~l0e}M5S_9{^4OnK=&6|9ydlaK zWK{%#VNz?Pc^oA?y|tilf5)CzUc;BpWm$o`CM;rKC%60D>dP{K$g}~kIN2@7%v$2M za{wDiaMU_RF?Km4^;;@`v}3BKlZl3gjpO*kpdYS!DKln4FYi$T+R?v<{eOHZf9mic zsc^Ibm^IdIMJ=N7ikq|#Ozac2-Mz)FJ=Z*eHg=>ZdmTXq6fldw3Gev^lLS)bdE^1W z4Lg3*0D@%1KvoMZrG?+1WLdNn(Q_Prv<7s9kB0tq z0X9A>9oI-yXrs6CPTsDEEjdo!et8u)+td^8k1%qq3f&<`7j#&1j}si0U5q;Fk6RGB z6K=Pg>yz;#-D?@-Uwf9kt}~F$5$K6oio68aIH_ex>wIjC(A9mz`hG*hCPl(~xeN;S zx6BPIY(tcr@j!z>597yOtvTBrqgjSu>m+A4J-hM^2!ES191pDauMSSGgO-C7BY-+m zz6<*i`0g)o#XzZQU-=u9_X_x#;r|7B`i~2PY2b|m2N6t1gE3fWRl$#4g=&lYkX!zJ z(ws)w%u6wHYGZ?X=H%Q`>Uqi6=)mR=NFppwLIs7{!~KWAzwk_ zcn1lI`?T<>Uc{NE!XIrV4Y1$x34g2|J>dR5sQR1j*MD3gj1mF-+LNz~=fOtRn#Js@ z3z}wM*zD}0RQP@cL?r<-KY}=}Wr@1^8tgF!+BTiDGN^PYeB9H?1 zQp>DU$a;vNRL~sh?Ttw7ZmdvG{<`<^)5V3TwYpAP{dm!#nKxrRg?JuYxNVV%QlV2? zg2jx5YNe7pMTq)ZY`DRA+Rb+%RGcCF?~ZDD{^BeAJxu-i-u(#z{~LA+m4+`BAgiQ8 z8?~(S3DoHPBbKc7lp>+hl6kXk4t^8cIOC*A&QP9xlT0|>IJ+iU)1B&KX^aH@$(3U! z0=*R!Am7uiB95~30GT;L;e|Uw=g*t!@1f#?RII$kCmdy~>|`uK&3)C$@6wyIu-Q2E zC@!HEHLn{_=~jamf}nTcb_i-`XpPjaSvkxV7fWvLWU4T;ZS+`1m?_ijnxwLVtxCMK zf*~!WNsImZbJ$_SG^Ef#?epJ1=ucbvpLh`F4m3o!an#^=(siDWCb~f{Is+Lezjb9sP@4%9>~8TzA*o^6^9>YT5{%1P6Umm&swZcd|8F% zZ+f`MC}=o#wrceFWM)_MWCkexThj>OTmYWb!iij6Z*&57aqI>uQ{Wf*aq~aJpZb5& zRhcu}NAPLV)|MkGoIzBBD~9pOy&fAK5ZOJN(j>2)NGoQ2k zX=Q`IqRX<$2gO1@?*?DDAH3nfdik8jjl_oM<;aj$0aUiy%ebmzpf3a5@bo=-Kq)14#LNUgwg zFkruj6)v%A8R?H5^$^rvd{THm(>lKtH{r!ql;_43V>@&Ao7+unMJxMo!JzF+kCw5u zq~RSrNo|u&qJ2t;r?pY_n&pv2K9k*@<}{5amg&!t8jVIUPxPZv8E!QI_< zLoD{!NS{F26;2gqdwY2hh%0&I4_lg7eEMP6Wm8z5Jlt)%s_C`(Af9IkaGNZ@Kqg-%Iz{W{Cm}lXZqp;6toV%7;7ib^iF@hO=>O4K z*H1NI(W_qNXsb4*>@>iuadhwgKU^W?kPZ$JbVP_a?%0jRr&w)B!0x?Smq(G6iZgw$ z!TTG&*Uxnr#EE6{D#YsB@;0cLM~;O__%;ZOWmV!ZXt~D?A+2pIQo*jSWzUSQ!y#8p z8k4%aE*5=yRC)Ct=#@s%>*s>zR&)2;hh2M}_tiX2GX5o=QpN>N0mrieXe)Hb5D8>g z)gVP+AQYkk#9%oLfN+GD!jF{7p{y8AmIefD^2#5eJLCcZh7n)j!j&#luZjs6N(mMy zvn4TIDS2p$XM|;&) zQL??H2&lrp#A5IV=u#f^z_D5IH>kW+_%}%NV;RsTJ$Z?XAKVW^UWB6T1KSZ-kBeF1 zVg_iQ$QzSOH(;H!J1s(K3Tb^r)Jd3gjmK(9nup;oSHO9o+If#_cuz1vE9=$E=*lbS z^Gn&*@V{dQ=BonN?;IzbO?7Re$rKte$mX+PE87AV8??S-^>rzCTBtxqUB)BGF$&P~(*?pZxT3 z*j2-u;%Kl>(FFSKyUi8LT3XCiO?1=`+(bh|1npyJ~k&4Eo^VnKeK$jP}c=_w}1-*C^?KJ!}u{fIf%LAX=;{ z!Eda`tG}}vOjc`c<~a^)t~K=uNy#;X)2^=CebrV6(T|#nTNv3@Jc&*`UG&8J2hLhC zNt~88v&c`hpKH@ZcZH7}CyKS)R$ot>l>wa@d5`~U@n4KA=Q3R13&RHnP9raPM%&?L zQJST?tq!ca@=B<~JabvPPt3MacUk9?bn1HckUY!lK zdWv!)f9GPpyy;+k?z?Fph7vd zSo&q~Il0?_{TDJ*S_JuvmjXic2y5aGPxYHX8$247r>f*a*RBM#}* zo8lXx&)jCAtyWdoEcZO~&CML2!AH?Q8%JRKqB1B7;TgfXAQRDhc2Lo4p0Op>Z1&z zEdAB?AE&fm7*S6Zw^e`S3ftrveb5yWb$T!1VDBYy3)uhU=t-;_W+S}$df=EJG@fAX z=khuL;Na>t)F&EwTox=l_o>{=p07wQm2rDKI!jUib&sW_TEic5D=oYROlxxQ^I4i-7NYT5D&8oo%TAXa6(vRm zN#lYgbfzHqJtBCIFL@-ua`E=0S-*mvwj!qm z`XLgVWOG{Rtjjw-!*svZ zOuP{)^(@%-6WyG&=bSd>iuu^3`pkOA(Z^KJ=&iqWQ~XGIkQEz&6R!-D`i6@}kqL-s za+)qoKT=BV-2@5KB(|VZg4qq;VKTvr`BD$fOL{d^rqdD31id001Q@%-!JIydYZf|k zu#Q#BiSfB9$zK~$4@>(#m&@67hnttHJ!L*V2B2Hu?m*tktK|?>Fjl+ZGN+2oY=(+& z4VBlf@iTmDOi~Z$dJOm3TX_8B2Ot~lVfXynBmQx03!YKA}P565Rv(PR* znCoP@@TcKtUu#Em8%CBY6S>^Qv=BmEf~_4Vm6O#T%hpt836Xg<3TfDMVgU?{7r}`_ zFXK`mPsXdSVjR!JMJZCTDxJ&5p=+HPX&wpg-$B$j*a~$$EfPhdlcBN_kXgVLVHj7v z%q-B%>{h|!aXST74(2`zkw+W%u#H@$G-OpeV&|{3E>!xL@SDM+w!4XUCNPf+KEn;5 zq6ai}((jczFGK0r`Air*uV}J8vHl7cD5LzChdz1^qx~arprYkfF{HwZ5^$7E^5GuqtiiQlBlZE_vDmd zKhM4dB`P(*`PUb-k1fE};+3C2B8g56ltr^;?25}>9iCQ-dsF7Y4{WT}Y;DQ7JLKJW z$oBw=ZdXh4vM|IM!hGzkHKBOz?dF-SD0=d+VE;PA7{vv|D6KJU4tTWpf6_36a%iEf zxY0HQ9f7+Dw+JL^FInx_7biw~GHiE#6+t84qYTuz-;?#^fLqHWvxPA{9r4UQ8OuLw z>C?umtwg=%>z;~6-2FklQ_*e^kjp@Z=l=f~sxcZ9#|xUaZfK5`+SXMyhVAa%O!Vfe zs86x8p1Z8h2@Hg_N03V&0BVu|%$Db^u&2)D#b3*Avg~x4VaECnie|mJ1FzY?_D*#g zAE;A~n_dh45$g8Zpp%Mz{VCOMdq|K%(qHuy;9e^nuTgmEYvCCv-f-&D z!p)%<)y!xSe20?@@`x1}kdGk%8pnSlQ@!921zj~6{|(x1T>%i1YS16^ZA7JIGXBxv ziW|+Gv#t|MSz3OJf4*hd;#IBnU0%LtuBwz2p()aw(tI|r*2ZRHC3Ju79wAr)0f(v! zRm~ie;+t-bH#bD$7S>F+)#_;HsvB1>P|9cJ6!x4STD5m!3FFnz*}*E7<1~P7UKmHe z(h=3w^<)7@uOCR&bYZBry~;0{J~uCZtXfU}FU9$P{u%x;z;d&x`P--qH*(DFMs;**e`}(8E|bDvtrnCKXyD$_OV!hwcpP2 z$`;HGLynyyenviLwqU!0k}eivMBS(Y2eFFh zzkWa1Yji2Pk!b(GQ_csd>3kcn!e{a8*2JxkV3}&a zhbyHdN$Hm4c~e}4vQUg4L8ai_oM#z1wa7M*AO*$&w$S;se^TpC~Y!CxgGG&z~L|pU|j-LZW1M` z9M8H7@HV3kf)w1afRrNn3C{mF$WjV&W+Vaful|ScM%jg|h7cH7+Lol+d2kLWH}ZnF z$G}O9dE*Y>nMmgP%m#bm|%>FulBPn+XC={n!T*6({kdP4vUjeK-n&d1Bu4 z@ww=i#Gf*f2QMq*n(i*0dq_3_@?ZPckLBP75|yPuxLkl-&jAc7&769B5Fx$sq7nAJ z)4-3-wTF{>Yi;=RY}yQ|Mv*3Fdxewn;l!u2vhu11v)A?e#fI_gUAs~xKFcc|k@?#- z>yygyqE5~+|ZVBhQpFq|1O6i zlgHJ2j~h;SAGcN{WIeoU9B06Bwz;`+y5rL*7S)56$>&`iyuvpyt-|M(G+!vlB$Vd5 zojNzEWf4#f_z(afhduxe@*5mj?JO=3z`x3|`+I*lFZ}yo^x>6oe>C`yhk4$ShIa8T z4-iTlA`=YB?CxgH-@aU3rkHRBN|S!>lf z>#0c7PAL%_fsLrrmucF!?Q`Dj+m)$aYo0jqErt6h;P5|Q;Wpaf*Ab7bcE-0S^$rCR+hebt zU{1n`hgG(b?G}12vx6k+UvPB475Xvc^bli2x>kcEq?rWRURpbW|BzKIj61|np1y-= zErD@&pl!3P9)%V#BQxESE3e}(&poHU`lUxe@OE+^$8NB;SnI&1TLYmdOCW%|T&idB zz(Eh~K)R@bfwp=lIxRB2_=JgzI{=-$>|ig;eetG~S~u9{JzvQi&Z0`O!#m`Rj4$t| zejQGKJn&p;rmPr4LKY+P0vRzn5d=5XfLzA{wfiOH zycCzIZZ#IFv>fz#{IZrFi^{YZm3aOTN)J0<3Jb(#E%!mlagfezzOm{i%>le%If3h{ z^183Pc==%;UrsA0hx*UEGC#_5((|?DM;>DTLhkV;eTIkY8EAY`TyKJ2(n;9oqRkR9*Z6@?W=d0oA zLCzaL@cp2l7hb7I=+Q`x0xuP{Q-f#(lqVxs0gHh}^kcxKNb$n>PSt+oBpm(0#|0Mc zA1r?5bFRxpUR}(i2H3QWp&QiqRgl~L{$q)k`s?MFn2lj0rIIN+zJbum!ZW+2#p^h? zWy-LlMP{6`VW{r#vE%uU(te(s@6h^jMO#0LdE5B;c|jrkJ@3pR0=0@!S?c3ane3yI z&ou!#N)&)R5*X1|Ek@09nDqjzmg3}w{usex~HU>L7@H)1C#J~o4qA< zHg={}chMdhLxg2q7v7uThU+PldK1V1%#3I@nQHIP>QoPpXBang=N`26fACDGXX9@E z$?Z7RE2CH{z%Hwq?uYzwdUl%f{9rZz)4}4|5#gaFBajK;fuo2ksP7oHx49!#tQlCd zPPePiby*u~-n%)mJxG>q*7?ajgOBX~`BIZr6;WS(#30(QW#wn;hs43#e*suSh!x@0 zV}w1XJop(GY+19ReI}sdg~u5%U~tsi8Q{<{%e9I><R^~G`5#cuKi>QzkuC`yMApg~x81vqjp*-b#xP9c7QJMNYVqt(r#PI4S9c ze0$jA%Xtv_(12@$$Lm;(JL!vxj9r)C1<}qny$QEZqv8RI>m=+_O4P#4?Nk#w>eiwV z9Z)vq^Pi>0PolF_lPt`>iHFJPrwK65?)U+Iwhfr=$5r82aKxd2_mz?_l~yxtHw>A{ z64z*B2l;+n_PN1=;c2fk=UU#{oz@`nnh@f(WrSokH1r5Yr-^-dp1`5llR(IC(3&B@ z$Bfs9mz)|AhyZSBM*`q&n-%27;xe+om}e&*|3FG70(oj!0Z^vKkQ1nPS%-(qTL2Q<%nJ=dM+h z224uqNXjNc5)IO3>&X5yk0qv^`CQl)BaCpQ>x*N=^@l8Xp1rwNJG#tCq4NEL&=11M zWxCOdd#rsVt*&Oykz5TFdiE&xxcMY2JJD2vb;0bkYuCpkY&C!PIguzhBQ@V#Ua+zC z?V)k1Jy&v)#nx)k z=TL}>bBtG#ox$8G&x5)#KwJo(W!Bx(2rCaI+?zPL6k@uf@&0KAd7-n3axihn&q%Y{ z&+;>ZO=nilcd0%^uGlHb*j|b2)y>}e!~`XFCC1gvu2z6yFc^)Tg3=vHWTlYtF3PMfgICk@&NHy-0+FrWeEfDhy~WD1de*xe4t+d^rw1T6DMYDMyJt1OV)N z>W@v{f@qWcjox`6-f{?!5!CR{0w>DdE)m{iyqfF1vRBj01Vz}271V4(oASHP? zUOXA4&+Es|KOcz~L=U3kLliIL?sxqlkqwm|H3@az{b>87*`bs68`6a9#j}c4qI5Z* zhlh9K&so23i*YbY6gV>QdK20$?lp~;t=br;$bieMj0kOJ9eww43w1mBkV|)ZTISDr zO%cp7hKOaWsbXw-Hv7ecWhZ&o5;YIUo!Lv+tXm}^Qk}YP8JR| z;SjYfmiZ1P07JSNaVK@+jB4+x`*5x&Z2^YDC}BJ+&K9GGt-&DPHbY4hrlgGg(=^xm z8Jwn4&0@+d!=l5z><|sjp5u3urK)^c+;F{5{HZ+KNa)@S7^!ExZ7g z+uk@Rm9+18$dg!KVgHj1Wpwc_h;d#2%a-Pc_mZo7Khrw1FQNm%@ZG&{AAX( znx|aBvuv)8*`n^98s+wxr%QRSJLb)ai-4jzl7j;v4Hy)VB26**4Kg22|ESdOU$EqI=vjQ(eDe^3#t#EBv%*(AM9~@uq>AX)BD7C`Tc{&7(@tx z9|LYPS^#4{M>rL4vc%Z)N1MHGE2irgJ2B=^roDXcMhO2M-~y@TUIg+uov;+_B;rk6 zD#!8?d|2CeUAS;<7EUDlNm@$6QLRUU?6sAga+8^AVZGetNc?*$&Ks*N2XwX=b$IUs zHDW4)_|vHLH%Lg)>fAdt zdZ)9u=I2n_Ht%f7`(svL)ABcUX5X~(a8`AgLQ+DPib+?-`&o=S_s}4zJb11CZp0Z{h``rp#DA-fxP4coVvQK)=2{fIVDl;H7YNkAGJ-0dD}RRW#SxG z5(o$`cj`-G>)&ch8JrgmgHkn;p^aK47ia)BJ3c_P^}HVcr+C@D#~`ss#AhCH8fUa1 z5N1P0-#f8?);}G3S)LrF6G^%}MgXr^+`6QuG5*x&c9cHt3$LQ;fIlyg8L7ub^wos9vg zGfJ_CljfZV?oB-Dl(9?#CgAA@aSQ}z{0FX941lIX6DUBA!cAB63F>E`IP0PG&F5ja zClzI2dP9m#jvm*|OzI&{h@#n9)%^AMac@ejyD1BAc$Zt|jZ4>webVmT*mGzFhWQ+1 zI4P8z)diS5dbcK}z>F_KBxg?2WComL5W+}hH+t9u4(quBJ|CV4xtR)1L z!=4k1<&{2dF1W1vDSm2Z7%myEJZvTD6>Y>cwIpm1bL5OlBccklpdW_MJ0gk^m4iP9 z0U{$Hi>gA9`}k{DYqrh+i-FV1S{l(Jkw(L zrN@2f|G&)})LCX~FFyz4MXxRh;KG)j1?e9}#7m+`POQw+EVNZ|FJBf>OK)17M`R$X z;sM*j&TKinQJ5lihbKzw{2^Of&yF^rpXACGxd*93R7$l{1yaHo z(33&5juKRYVG~8$q3c#!hXdXC7ULZ~;+QGunB}kGroV>V$UophF~UE9nVSMMO7Vuz zr!JMz978wJ^~Wr_7razM*At11N<1S$(7RAa;TfJ~o!yR=CSjCts}9Z96`Y@3E_YUt zOgg5k58Lof(Ri34>B7Std+tb15WCfG0h+!IUWwf4L+}4MyWJPPF_~XCe$4*edA76q z=Hm2)bDyw}#gdE@z%_tONBC4yCwvhLswJrgDFpr%!1_<3h`2V=#ev_T+vSwRSV_nk z-*S*7AXm-+`cumb05(-$PRcBE1ddes2X~45fD#*mb6#>9W!7THA)J|H?KnaL)8bO70LcD=6S7rDkFX765PU^nq2uy^Fw)m2- zq)c_41i^i!gqwYG!1@$-Y4a}oQanEQ*CelhihIT3j?uw};ndLn4>KNP;$vRuu8ZFm zl$LOemc1S@Ez|RW*K0*_us(E{F%tS%Xyln5edp^#@ZrxiDcvIU&N@Bf~vr zcF7R&gCrbhR{w5LVGtRf-oe4SU`EN>)-}P*O!aFUgDnlqoz^exL7 zpZ-&r6S{GJwiQ#p=Iu@b%wnKR4dD%Fudvr`7hFUq?kn^^bR@U`&#xyJ>wp{xZ2blu zquc@erI{e85OgUSdY}vQ{0(|44L!;+l&DcNWLkFE3_A{sD|C>|Dx6mGc77QylanpN z^@?|^q%=XTqJ(Z`9UKKmYYR`$F0&L)>ub*C-+a0rs=&+wIrgg6j8fsbf0n}=_);my?OKsG zpwX0oIYkH2>ZS-QqO(~9>m!LgmH>r?J-N?j+V!sm838H`^HMaSx zeN4ip)0g~}LgsG%3y0aVHih?fKGKyi>GjEL0atmKnD+W=+oN%q%CaExh<1cs>(nIIY+?NASjU(CzVK`eLPuJo^Q1N^x%TU0NlzIL$&AkG= zVmF>4Hl`q(i#sB6gq( z*8ftpi|a%p*=6tboiD|l>u?)?sgXefbc?*+ZzB@ zF><<{cQ{z^ac4fU@%Ir5)Iqi+Sx z5wD|3(GI94N-gwWW`>}6gZhgcbK5JO!3=>9`39Z%`I&zGlkSQCk?{MkbhUMzUHcw1 z@I9_9Ekatg>MiGfvCOxT2mMMWW||lw+p+D^A8|PhU}S zCMfOSgE$8QDFm(S#*AayX=cF9GfqZTakaZ|4JUZY863Y-XX1m-+&s$~Xa%_F;I{op z#q+pgR5#27jc=bYT}Xp+cvk+OCsck0c*cCs(=8FDPKTSOrJu}a=E{0*1-|j*SPCJd zctu6=H0q1?@=b0m)$p_s@FEbT(Xt1#lfjuK@FO&t>I;Ldc~nl7D?O0Kr8nw z`F*aHi%{92}pR}&FMG0j1SF@maZ4yZC3NpXAlB0A`$Qj@c*r0(6c|eTm^PlDM{f8*v6cskaWn zak~*Bnr{K1Vs-|ga2d4t5H~D8m$8iKJBBJd_bDvpfP;`rpCRW7H*r{0Uw*#pnbHN)&bYx*qgf-YMO`QbGGrr<0PR+ctj5ChM20#v0OB zk7WvbGk*OBL2#6dnL`=4jT_a$O83dK&1^yN29Y`P;Q-T;3?}yl$42lTs(Qa$O}xE6Gi&-iPKjlHsvGux;X&6vw}vT&Aq7P6z2s74 z<DJ?tmUir?1P{zDwPc zoKX&*rJTsNW%TLJOn2#g`KuHMnEgZCpUcJ}@|;d&Yf1y$>}>65=FI2j9#(WvsOM+R zPX!efQFCR2s3kT4Hnu|Qn88BVDrM(>6#O#ug>dZ9G|otS+|8v-hz^0Wphy{Lq-4zs zlDo3#kRcG$e&@RIT6>=VyTUFSorTlNHFw4$+hSAxVYpS~nFL;x)E7t#*saTdGgH3y zae4RtH0OAADn;ch=A6(QzAwW;OCa;2v>pSRH5-wuWrTP*b!hlCS!baV?97JTxgZgO z6$*UG2j6BI5!z^*YqnJ{Sxg4jhduTA$F{TatH~>Pt_{q1r1Rs`2PI zFCOTc|W7ca#jzxaG!8;tf;Z9EkBJvLJx#OhO_mZutv%`aTk`f)MvNrg>c_^;CF zEQ7v^j_|%h>($lc0O3&*7_l0_ZeEb{zwo}HT$gMruymB|AVqvB%3sLx5X{>u{Z7qk zQ$qHCvG*QOO>XPDa1am?B!bdQRGNT*G^IzRi-`12L_nH|2uM#91OzDp0t!Mzqzg!w z8hQ~0rT5-zXdyrn?{w|G_Bm(obd@;vYRyqs!E3mETo zGwE>Myi*_ItfKX|t6t5kJ$PGjYtk*|((6sLmsfA~Zti_E>Q?QUl9*UwrXLwqVG(_E zDT0xi=ZUxmD?ek3*{AedhJtdce3S3TdQxNzVxM{Wa_^p*Hf}6DMSx+)Q-VX19r>U0 zRKP}jQ^`AdKAx=-il@Cmo9=XnRxp-^L1)LF%WdKO3>S=UZ>dVjW8d?Zi`?pbl_M-4 zyGhrRp?S->f6ui5(@A{r`1s0v7@v>K?p>j6%|+9AO_{2l>UF-vv6RI3ieIEJNgBpW z?Vmr@57_I(F!*MPF0tsf=)rR|iAr<_)`$HG+1Y?UVi~O9&S*yO4}Z)^SrEFf0z8Ba z>WvmPH!e`cxJZFIR1qAA{HDum^kHf-Hu{(=L*^MZI)gRm91eO@c6DxLT#Wu4=~< zbwUigx1sY@kMa2U!CVlzn#6d_D*6oFjp(moc@jIDvs`)E$YFT-sJEYTb*%PPy#X66 zF8Pw}<bw{}R>fH&axo9aHC&P6TP&D7pB#=fc5c}5J~DzV54l3FaW z9Hhy#WC_hX(5~2P@o^hx;oe#VzeQOyS+*Et=Bf8XDW}F1-iD)YS97rF zQpS0X>1(hU7}j7(_&qUmDi)HqfNgp4?P)(@xX+$+EbsT`!$%fd>T6;`a64%^I^8nwGU8q;^Dy5pe za~kft+Ab6L1P&?`rZOIXS2{Aqhnrjv{oK1d%v(SAsZ4co(y4ISs(=}h#lE)sN|__d?xb>QZc~}=uV=}-2Wm3i(JyRBdKchp;7e>gs&zeZ z+a}UQ7j46xojYV2e`SEbX1JZfVj?}>c)(b^-L(gHhkrr_%$%5V+|Kx|KU51?&XKN8 zn`mo=0TO8{5p#-QZ0%F%PgjXL>71L-<7=@aKjPB4H$rnygIb}%mgCfuN3uCHbET=- zJyL9AcmO2hyJw?^6LEoM*wzwUr;GSch!GcT>q_LM*D5bN=iZ!LbVy)c^m@qZ9#T-% z*KcCU%=007Bx(DW`<6QHN}wEiRKKC<3mLgZ?aKfo>LKJi<9r1E(h|B6IcOZ~4jPaJ zz4b|Y^x$CIfxo*e{i|AdnCSPVBdT@X^Gf&5k#+Eao>#&$@dH?YHd|`^jNBvzsoTs7 z?aufr-$nHK=F6^f}>5!+HkR>=L;k$Qa8LpCy2!8TqlaJ*itv?s}Zw4 z%dcDXc`N?gW>U1ITeg_Q*@v8cQ>nuoPYrM7vua+<4R>bbhvO8d>85ZhO6C`~7 zV7X|jzpF^Sf;uDirz%pziCHz~@F(!8^rW_^d>4il6y1uMTK-uE1}5Vmt@4*b^z9Vy zs5tZbn}GN;%JSB8|BKt3;V>rBd4la0PpTmP#f)>l)m|9=rO4+lFW(+KY*jSr9lqAB z9~Wmy`8N2s;zk>6Q0oBfRNaLo^yxC#Db>QP&ofAh7=%BRkQ%z z)-d<`gsA73r7*`yynNj6bB~<2n2Gela$o>R?C*1?Ks1n`#nu3}YW1f-V^hi4G-@e~58&3qs{uy7+Z2a?(aY4%BUL zpS7K`jlHG_K#agofMmZ6I_5*cdU_tegT6=o-vg*ZTkEK>iOsIGlipX`WGOSt(wzk6 z$A-uG=G^*D`{t1cvTE3!JFUd}4*EdW-0R=brT!zn{tHori({s8P}?R10Z>-+EXQy@Pkw!m6G_psRl)t(Zi2AaLIJ?60=Ed@A?saG?QZxE8*?ynl_I-2Tr&S;X<`N ziq>Ca>)vZ~9^skM@m#c08MpLdY`{wVy?fZArS2<(n5o!Qu?l_?eRM>X6aZT>JsG9D zq}k5R1E`hIs7SLs_fnTPuq=vTgc+2gmFk&1DpP}-Mdb%U+w=al5jCpUF=X}Rg>IwL z$ev3G<2=sSn|MPE)}Zht$LOln)f!bEArNAly988o^UudlC-1;+h-!xldys>{8H_@W zuXjpO99C-8rg0>CE$8*7cdQz$b)RK69Llb6^nZtqGLue1r+-9x)yu|gbNHY~xTur3 zY8lJl+w98!=z!q*6IQY004WzDev}N3J*7Y0_Uvnn-3RCVlcXz35!;lFPsF+}MB6dJ zOtY)bfJ%b~6Xa!{=dMV0Z&?GFzsiGj;kLVW_ubbVa2FNJL|P-jeC}u%_33Ze(57`3 z>q_I>Z!|gUrKJ_|lC|L*@cgnDH(P30dh#n4WY z-)Fnr^r)b2MPq|(n+{rcimSy7`qy79B$_XhEYZX^t)YYHRivt1AzmozM8R{PE)3obR%JRH~go-#Pm$X5|0cN7U&&LRZ<22ynO@CnN}5uK%MkG61iEYchZ zpm#o5RnjLf2Cf<*CipArmO)RgTAlZ3IF+-q9i{$An2-U#F1D_>(vfsYLDF~s84$eI zc|tnwFpsTh6g_Fu%{*2~`Wmhk3>vBmXn=+4r_i&4zi)mhi0f8N1;6t*THs$eXlO_G=vVMr zqpOVrdI6L8tak*qS@@9@rSgpWa+&GcZ;|z%B1Hdzv>-bO1J6j`s0{yY>f4PoVd|mR z^q2HD^x#H0_UIS*n8P+_>8kROd4udEsh8xV+>%k(&3-3?|G#ZgctIMRJRaHxeL^~q z%}B{kHZ6`mGFPkb3@C=&cz@I$qJde@fFJ=waV~-$7;V6l(khDhtKhHBNE7t2i_QDK z3QLV&xLv-Fy@Nk;NEWzobXi(Hf`bi06ecEv)13~7BKe7ljgw4FgAH|j{%VQZ=h^IL z!Qgssj`}uqf^tPB2bf}Tbjs9HIsLd1D+Nd3MKXtD2_@-{$n!lL9+pmDb+MolbwyEXWzOp+_iq}c@!mu4?MvnQ=HHT-c5OgV`wJX- zwni^nAtp7Z3m<&4jT$T0qo{ApZcJ#Kw2rGcwEFtM+L+x=fnlk7pHzOdxnm*LP>G`P z)12NEjB}kxSh-pizLtskCxb2%GV;&G+ajoWoL&+NBn z5w(|sb)QF)%iWn`n|a2oyWL`Om=tfVCg1y5y-g7~h`Xou_69yXr?^CD95cH7PqG4lpp7gBOsNmZyfUzX5VLl}b+M^H10 zN^*3W2D$#REKK-%^mQ(QM*-W4@vb74Ja!xlS~H7en85F|H$UZ{_8l7C#9yFyPahc=kKzkXgPjMX2AQ_4aET>=1gj; z^|~0$cz&gSbA+VcyVvTg{u~Kh$F_5-0%%Q=LlFf-sxD(%!`fb>s&x)m{F{L7$o~m> zx!;U9-04N_%B+@%jqoSO`RgAbvzA}$@FVF9bQI@292Qp!m+)tchRI8p7At6tzF zaQ4j+g|jP{UsA6)HJ3fDePsAxl>O#DPZ((rY1+q7z7~Z z0jLm=W#zMqeu?^oAf$vo#cY6lIde33Uku%l*tw%9GexTq44hSVb z9+QE0pSFOV__!gQIUsN8M!Wxzfpz_(@Vu$Yv*VPk3sJm!Uku~<$m82L>-8R?;VM2d zWu7OkIv)Ilh_3F*xZ-&qJk2u)1iQShRKd8!?V8^E!dW|KH&e&?ec~Q?oyv13>ws_) z7?khsunM)X&i!zbNYsWy@$P-#$fZr5cJ)7VBsab+Yun8T@Ou_XqgbJI+M>~yJ2$Uc zS(;jJTY6L4pPS?0xO`hb&+Gg0H^C}zKanaFB4!$my$K6u8a%8Bs}o-tad_$I=b_DJ zzn#J`Js8lN`_fjEN#a;2rIJ>d@j}OF?`wl%gTb7ZSGv{l_Pwfype@RN$@ABf`s~#GG+ZbIB|5sxFaH(K=0dtwXmlZ+L=iQtT$4FJRe2%i@b>cbEDq zqQ$3gUSGUmVdG$~PP4RbKJ94Wq@c8u<6}e&^&O!vk2F>;_$aJW8%XM&_Yq3cm()Gy zBb4ec1%3pqCh1G*3f07S3x*Vh(R6J72E{vHFLz!yo4Dj3K4C+sWU=zW$wOOfH%@m> zxTM@pcYQ3HWM*k5JA9xyauxNpVrZ&MjD|Y!?O~Z+ewnZ1GNyf(jg$%({mt9`2PNL; zfdNuOb;tq}MhscU>?-BEVHgAp7W+_T{EdeAVJvq`9VCP@PPekyIbT$sK7eZT90WJ^ z+HLzaH57`A4)fRDD$y_h%BG@xy=OQ?sqz)oXSbyeSOP|K z({NDerCN8x*7h{k(+ zKui&7lAg*RVn`b+U_|vQ4Aa#lzBJc>td-Ne-}3uwp3Wxmlg0OkQk+&g zUK8?ht?RB|ameVqz{FaTNeX4es$)_r&x)w*G>g*Ssl z08>fMs5k1GZhgv``}Ci8O<^8yccJYpg;uWu;^E%Q;qy^h>*~Ss3XapFG!7Gt08ELt zWY?+aod)${4_k{0m)_UB0p*J+D@Xx?eg!zz_dwvnZ`7Cit&MPDLc;R#z-hNd)8wb_ zCl`s?mP&^)R;9v5iMg`OwP$b@jT8LeYaFM)U2REa?YoQVDf4sKSQoh~V|u~uWk9ri z)Vnz@RUcvLY6-N>4b!`U{7H^X(zAM9YY%hNV>J~h&%=UIW&6$R5;Obj?6U~=CF2XT zH>#=*u zHW*QmyXF#;Bou05VbHv3>PL8X(IJ>ZGOZI$LiIVzVH|qPztK`_lnao1wHq^+X zNgdul)4RrSX0E))Htfd1mwP7Z$rN_VE)0*3OLWN;TG0&m>{oGT_!adLzZRZbfHR`* zZXPH^Q=x|JZv&{zIeaeak$;;7nyFL zjGm2Rl7!LtpMcXOVXsuo%2m5_7IPGvncs|)FI=jgja1L!hj|wDKMQhSSaa029NUKB zd6)?ybr(^C1u>w|VBF~Q_3Vpi_ zn(%bGrfNNh~;W2}ifcJ}NqI8?HI9eVY9*mUi zZ*;Y)DGgZxW3S+Nf*?mMM_zP9G5v%%>M(nwuhnSfUl=hdt)_lC&l@!&6`7*r>Byv} zKX$p$>Y0qdhpyK`+&IJ(5SNM7p+F5)0`_ZJ(sN#C#$Il0HCN?dyxt#X5mfWWEuHKN zI;4|j_L(bJiY!Jf@mWoubSSb}c_TmG?~i#p6-Z6N)mm!-$0qLeErm8lI}$&jrrzk+ z4_b*3Q#NRSyzLYXakW1f92xL}Uk3P!a_VRI#z7_fdf$mqFmBDT>U)jyhwupoJN{L;U{z z968qTxrYrpPC)g%w}u{DZPdG7W_h(Cm9?^@HdsV>A^PT%k1k$g8MS#009NzZrB~tGtpmoQvsC32j*D*_ci~w+#iu)yn18OV|WQc?8C@x zR-7H1&9)h&-_YQ8cA9|)INBmj8F3^|cT$h#(}uPojp3F^7f<5abIyIY|Y63TOgf0PPAY84y>{vh(+t z!96fyK{W+Isu6+jngcm@5GL7^Zsx34r3ZW6D4q?`Jq`*4iCnq3Ir ze0AWZMJxM5O&dD0y@t&W!V}woy`KB}j)cNt0lB z7kY@HVb41V*>|$y_@M*iyDeEy<>{@@C$%+Hah3)@PJD4In40SiP4CkK-Nzi+J*7P! z)l}j`j^*WDO^y!M20XsDC#mlC|FOh^ek$i>UBl{p13}*%Mq(WUc;yxN zTsMIeh=j{#krKNt`BMWF{^1GyX0Bmz=n8(q2N-B%*_&)aA1XMgHvI(-`uhj=pIG;Q z{e;O5BEi}~fD@ZDg*jrL1oI`VGE$KeGwLf7z*(M47CqdL{s~b!8+CpB_b>Nw6rqNb-X#V{gcH{qQVnh<%7iXhL7QwF=D z62xMsKrkPZ4O~KKAgLX-YYg9hnv2?&0&|S9h+!xxe35ipN{6Hd#^!R-9Dy9=2e@Tl zj)NG#y$>)t4Jcdqp%D7uT0i`N4R{f6(KUdcsDk5skqsPU?q9=|p_D`r^eq*j=E)5J zU+)Bl^8PPa5400CVxvW25db@qCXHbjNynthpZYwU5WqgO73dMolv|I(3}?!Sp*}0~BVL_Jf)F3# z{buuNX6yqcbzt~=ny{OL_h)M=b_}SlHNARBP4;SDgCZiR;~ibd6QRk;p=G11vY9!4 zlqvCVMcu%kb5R%XUNapZe}3}aF)(uV*e@N}GfskQgiw6TmS`KQ+_f!FSg7f;6V%Sn zTXK@tQDVFN$|Czhj1BGCPs)Q^! zBZdOWxGdO~>wI$np7)w;Tcc|*@_gXPAOgsqXy0NNE1!`2#x`R2><;ZYkMrK-Ss8b7Z^>Qp zyb&z0d@*ZVk%tH#r;RFJ0C*Rg9h`e@-){PJj&*Hi&wC#ctB%unVm?VzIsh`0Q-!F=H*H!+ia>zE*+S@yV&~0kbWgEP3VssK9o6 za%#VAG7NR>fu|NGl&!2RBB@c8b2zC>X8)sIxQ(~;(Rxp9c&WA7*y6YE&3#Rly?&`F zV4I#KX2X{r^un>);5#-C!1dCHmPmcg`viIlSefg3+LrtJ1S1ynLS8+$Ly~XC8F6YU zM!4^CUSU_UStEa>@>#ux&km|HWeQVl@l- z^xXY>#tx;gOU|S*+D1CXMmI)>!cP-)v3ad#sAzvyAI5@`j04QQ1QR1eggAfLcEkxd zPlU|gw*XEJ!I=4kww9heK?(hkCEO<7+jw2+Cq()28vs%`yOQ`S>oU}fhKV_0n|N+T z#Hbdoa;v6rI#ObWDNXXQ;mXSqM_GkjOF7MY4K5b{DZ=!PD_FOnq{7_qA^G=p2Dc0q z`h|~pNyd_O%%o;mDZDiRsqNhA3+{XHN7o0u=-UJG8ilxy3=!IA=9OnUxtZ<9j_AwF zZM(>lURN2!rx&IfJUYaOSvhqA_C>h67f>(=I(Z3fTz!&y6Naiu*3nFo@1k~doo%s% zoP4a@LmIj2%KDQu)`kX@T&g#sKE9o=+besnlyDvx4z+PfRJ1zgoA+phR# z*)~xJCUXRXf8PU*8KltTaYt8GQMB`uW`m-I8NdSO*=7{!F7UWhe$yy2`x zE%ILGcGLY&A!<_*(@}Iv4HU-99wi^EO3%)kT?rV`*?5e4y5-*K<}v?e854_{DTvy^ z+2cL8&RoaOPUo9jVws=suCgc&FN|9*yEmQn`?z2@w)T#;2x7Z;&N#Z+CxoHSTrWki z=v7Loo;E@2y?w?jYoVUj>-s@jmSzEOZ~>z;3%aCHlq^ZIE9-t6JxK;clQ)T-#|uNV zNQ?)JRyw|0b`nJ@Pv;7JZuYAfmtqty6~H$~jqev*43dC*nqmO1g)tJu1KmR)gV7?i zbG7LzToOTT_M(E_JGsAMmYA|gWTM+rCTsizVl;^Y^EG%)N}^Y%5;SHBlNg^BUmE`j z`6{eKIEI*WC>nwv9Ki_Gs5fUEtFnmX4=|gZzuxCu7kr7KkOXELvt2?$Nys*!eTP^O z1L`yCCnQ(j7S(@Mfi|a7+S9RhG!IAu*rat&TeMe%vv?^?Tb5q!nSGy{vVA-V6V`ht~Q89km!gx{`@mAW`Azb>7QZNkR7&JCVYHl zU_yLALNFv&<)rvmNXOhup`PZxXE~)wSCX}1`SA-ZFo9pJN0C(G`guoaq(7(6@x1}G zZ_=!Ij4JB>5UHL^7+RZ6HE_gXv0z2}XQ!b%t0^jFbCp*WhvsgQJ7i_QzEqt`6` z8NQUP`bHDU47ZR+_@*v{_%DQ9nma`@977T6rTaETZx;jTaruu}(R|1;bde)5UGfOQ z?oH=xF3Pcl4M;I^pKr=FCy$|xj*eC34VRf+5FDZq_n2Ni66R`si56LGtfStFUI#)g z9an=TU1(Yk6;pZdxybITUM;7c+x}jpt`yhd@cOudWE{~Zp@Df??lo`)+5tK ze;O~x86QcxMb%23wO2ioQ=!q5x45;J3978g($kv9GW#yBh(3~i`xCNGb42ZLfLG1K zJF8-Kp4I5j7M45cjyvriUWs(`a;1BJ@0I3BmUQ0R=R(;@_d4#=t}|-Uvu2Kdnw8v# z->G4DRmgfKDQBK-#hM1>?tcl;MO?UT5cEB$Rj4yzt$xRK1ch_NMz2dv!SJeO%Tee& z6FG~eky5wN*e{PRD{;7nVaYSq9)37&sL>N?e8X5dqSEvr)DH{8^%G;|jotFR`7SF(u3Ko80{uposle}>{U zA#069SI&TOJ;3pBL=9!ET@=MXoC_O6f1M z0>PL2k=BtuiT(&v91sWiz9aRN`L*qG|D&A>SD6-HWDi&CpmD{LtLGPF_2BC}AS0rh zOI)YN3gN2pUR(QZurnkMZ$#^39NR*rSN8A$E*K{a{^HdTs^Nvf8hb7>c26jcvXIAK!cfJk&d_%hc6R=0a)A>t zNp0&A97wi-OLga+aT1DbpH}M2&^ha zFe-{@+WE~a&2r!D&7b@hbmP(qN7h5lD?J7-Ukc1A!zg{+SkMlYf8FKZz%GwL73=)E zGm31)!Wy9&9eV%sTbYR;%$)|ewX?03m~We_MY>8GIa*zQ!f=fuRg-1aD#hugN0K&s zM}hucPfZh$+G#JOB|!1LRO>%nW`;?p@dFB{ko=18M$(s zU1D8sR>?U~8|VDUjY^$?Z#vq=1Cpb)+V2-{rK6cN2N$n(A{=G4*(Mj)Wh+;`i+_tk3S zM;GoiZA~ZVrXyRB9$)vAb08>WjyT663WjP%QIsDNZ}~o|Q?iI#3QDqS3>@5JzC;(= zfcm7~hIru%jX+*TewPjTR7Dt(T07)><0+G@SJy`&a%ZPa(Lkv@__=j7`N{NC;?0ex z)Y!?B%(#0|Qwa!KBo9HMzm8jLb~AK>7u$m{Uv_pHX^m5O`8LmDgL>ipM(cc+L}ZhK zs{M<%wuwTOQf!ZlaQy4knNDVd^*Uj)iji4klarI)>c-_|W#wTRNj|Avh8jQ4GTpm( zjuH~9ajep@TKno4s1~1!#`CE+iiVTO@OuTTT(WrUnQl50?P>NrhmA59riCu%*Au3n zBIL7kO>%)I{lgD9snHj2?I_+KCiOYyLwkL9zFvQ#XiBU=*_gl`wV8TbxK4jA>m_)m zHQp(YMI~@vALXs<7J70cp7#y=OPb5vrz(9f?-AolhCb`O!m~G|o+nnKY+qDI(>Wwj zz1TJ9dsUMZ(E8GKcNvl9FzVbE>y{7}@=#p+affbxo|J#FtobB6cnr!Yr#v6|<2dBS zMAg3XHct*SP)_tT0;@8}W%fY1FI|QT=Fyk^J$>`!_oz(%b@HR*z{Y{(`a~3Z8cKtY zYrC2h65G4##l{rl=Huex&v(ByM7qO(zVr@r!`+WVtRsS{7bI%OLs>JC=gO1;S8PmL)TFcYwLQMg@KWCzVtRanIA$9X0Ki( z-^(LWS8Tl|sNw-&a6%-x!&a)Z*w+}6_?$%jlHpnCWRIOv~UHH;S=7_jN)hS^E2 z>`>xyv!29e_Zd`(7f7n355Q+qp$5A^CX}%=sJV#N`QwkGnp<24L8Sl{7DM#HE|Q|n z=pTaeEEglcf9{d~<#8ZwPz}cLxm<5iQp-M|_!2?YR|I(5iVvuEzD`mtu-V^LxnEZN zeyU*cD6GcjflMJAYAD_3CnSqT7D9CWm&EQ5ir4-MIzwN8gc7!WjQ~oDYyvQ(7^~Y< z`N$?)(X}+}-s>lnkqtaJaMP2X$m`?YfP6Z}VS=Kc9}BPV6oxlfmGZu?8n74UjjTzz zay?t=vvv2k&*#qBpw2iaWx4dVusl{zZGj-rpWjhm2x4^*t zrHWHl!R$4z?rBjx&t%Y4#i`lC7iKNHYvTHvMdAIY4=65g^5-#JB5% zXI4qFRf@$cMom8<*DB7E*^oH>j;^4(K?4bhI~<{Z=|1`Y?`zO5#E201?ofHoEw5(9 z6L?d_GlEa zM<)qXdEN{suFG44ksJO4eUj!h!m|BH zeHN{X__;?Xy0&mCkU(iK3XX`!3k*(T(e7V6rrypg=1xWv#16%*bT@+`N`7{BIl~r? zBat7j8YSw=pJ|Z43udD|rH#r8(F4rq2}ypJ+ezA{=y+6dXs~fTrXnPH>RHWI=jnC( zX13d(g-XAiE2(*bl(WC*wrMrMXMwjD!4aWJ4a&o_269(0$C3Gi;Q?YZFi^pkv>1(A z?fQkh|2CB8z1sq8qnA{L2CQj=5Nb|giQS}V`-nc9i6NDUKkz}MlW(StYOt`UH4j9O6~yI@EvrqsinD)K zN!uB99eUmtKJPL_A)Q2n8w-J-jc;h*JAED+~UGogpze=k>d$Xx4 z{Yq2DBuN>8Elf}3TkZ-qvmVpd32xp3_n-GWU z_ErTjyw>50hw`-g>NgLjvyfr#D3cKZT}v>P!}_Jt$_XGw!iMaDVQ)dn2x>>h;dOMi zh0o+w4r11>&}WQej>4%cY*wCR!Q6SktN>@i4ZQIdIncM{1IS}TWV23$hX*yu+Vkk=}z|ZJmpq`qG^bFOmpn1ADDh#jZyeGtsJ06 z;{J>q{MR?;|9!T)eUr=IfcSnLjU2AG3O$g^NK*r9fFeIZ0B)cnNz$_^(1A>ywhDkOR`Pw zgWg9Fu!|SUjY430q!!Gl%}~HX-p)@hn|6=E z1!pN=ldt`)A>r>V6?q5BDLO)za0-kYhI1&(w&k)Q8`5)H%?|GvT_9U6S$Gw-%6^+_ zdMs8`GE%ZfqJ7QuO_bJ|zPPdP#0is|{9S z9PPo$``wc7ZBlp$tylhsALFtYZ)-j_VKuPXyxc`4RmXIL@%?Z|R8UUnTjMq+|EZaRYY7IRai5+mxw3CqTt>d*H@a9T=H9Ccs#KfCDmliO!Xd8Gug@%>-$h)G(9BD{g z6j|2UxgM1-H|QTFv=crOU;69%%tnvCP}#~%qquG9e3)9Dt8>m@}#s~(dLzh>bO9`S58-ZMdK3ru}+%xoShcG&TSH4 zp+e=)kf=M4RH9Ge16`k{iVa+GG#fpb;PQJ@A2otUrw%`=q27sSf3kDtWX6J2Z&cdX z+7DjlZ)J>xVy#+T)hh~J(R-DLa4X$i(ANKF>+XS(p+XY^@D>>S)Fkd+$D}5!mb83%k#SWxuEyv zkhRN)!6Aj#F1gFb1N-I1pE6p?zSZRM=r^yha&l_Y0+AiSgwb1ZYs*Qj#OWEEGYQaw zV@6xLr5-W|dj9FOed||?Oe2+ce>_AxGG*YXYic%ztLMifY@d$l&p%!$f40+>9D9h9 z5;*%$sOtQd+mf|A*`?Q|Sll^?KplGoS4MQb)uf)@u}~2dQuKkxc_Dlq(eLLAq%OZ2 z^W)uj0JT91bA_p|x>bL1WsYC_Rf53}K$&eynh3*-rcBZ!J`|Ck;|RH{c>i!G=FoVz z@Z=q=(1o;(lHGEVJe%2F^8gMKHP=f)T%>M#%%JQ?C?*BVC^B=bv^fE zMql%?h&t#YN$y=(^XeyxAn*No7xTaEN>~E?3}*h)A%Yk=!3H1zIoZ93AV@_F>c5Ab zgj+ek`}CVzokwyLM6l47?o&Xq<#{Sg$w2Awqc3p*YN_LJu?j`h{!+$ln#3M*2 zK~~Ds1f*-ZK%lER@_>Ug#>uKXu%+aFzWAWAyv1xh2^Cvlg-trQ=4RmY7!Ly)KhH}pQL1C5s>)p_a=MN_ME z%s*#;g}4KIWo5 z;#Xl(zQ!`o>y?vWzZ1prs#y4RV*MmEnvNiiZK+FP4?fcVL7eX&h0Y}D@5sze1^kFB zC2N6rz4|MtOLky)WQFV?@et93+mnnWmViosj{O`j%9alifUT|;Cud`}5O3vaix)MN z(~TnL^OuA)=l+-jt?h4iOxDhR>cN$gV4a6ceu(;Lya4>!=58_+>hE1z-7Dp7vz@s` znNpEr?_OpSW*2gPL1jHyk6T@D&78jQOU>RV@mssNwa3;a4yEY`^|8HLbsUix|FQHa zxzqJ|1@n#)O($nYU6_CIn0U?wN%8B?S)bR7e5vHVbX@z+C(q>3ZP`@&kQ*zAAVp1l z(~&AcpIBaFm_=a1Yfk5V<8P{9mYXkge4*)`C)(lZza(~+zA$0oe5K)mu+WKeTu&M; zPr!!>%s^Wrg3saun-cqQiGX}{ysVEI$MSY^&ZbgP%n7>yvtU_?@NtpXx(KmC0#xl zonSC3IQ!6Y{;HT-p(wrjPSe26gCCRp`!2 zVD-mH%s-mk&jx@CAF|FkB;fX6_Bn1qG$FS^H8PU?^MsqmcO*rKT_i% zV{mx${Fv>vT-_*6`p46Q_iOPt^)f5@i{|dJP?;W?neSPG*3H%Kh0WnST=UlCTCW$p zsgT_vaik)6eKSceQgEvD>WJj~>-?vwTIsx)DgxfMOFMLGd>cg2u#pH>m^X{lV!e%U zdnwP+Tv9d6*ZT~vlbBqCa^fq-r)4bd)3|c%jI4GOldy@fmHlt!0-U#$=;fY9%tf48 z=%z8^RGb~nif-yRyD?G4i6BVw*TWA*xIjH=V?rkw>T2M}hWSF8MdaTmdG4n#c1Oan zdZHnU45f+McvXA;h_XaA!|Eg%;nLN6%|VweE|oCaMXaeB*dci-?p8V8u@|;~FD=HJ zFD<0kPl65Zxi7d5$4$dqF(ZDnoZT&XKOyYSBPvW@nJ&&P1HG?BMvSyGdwwKBH16I> zHt>1atY&;%`omhYK~`v1dt&0tE=p;N{ z*c3E;p3n0eHln8$pMb(Gj4z&aaaz{mTD*f+Y^h<~(qRVgYJgGhVU}&QjnKEO#5zr( zhU&W2UIKdR(cs9fta~P%PHQjXYLCy?Z^ra)wIIcku&u7~m3~gbEtWf~?rNQR&US+n zCFM&;H$@zST6jAolgp#ky?QN4#^S?7p?Q0W0bQ0`&P*GYA015`57!%6?$1_g*jbco zY7FbJE(SFa@>aI}6Q6Irnt+kB#boGOu1r(&+dUpn7s@KGcDDU7H!D`@!s>TpLqvD? zk#@j3k}X9sbh^DX9TzriSIJHGWSZ(ByzRgpJ&0F-y7X}MaNvAy{G_=fql6b8??Pk1x<6th|YjzE@B66Ej?q}2c9*HFAQ2-M&)G#;=Q6)@UG zppeHKoRB%9I#6DaYT)W+7z8~k3qOPBd#{Uhq}ojVX;(Y5&_N`C=7n@f#q zp&A&V>|qk)NizhgC`DqHupGhXW*Nln+uH@+-lppPJvjRnqn(PCo{xMV&-kOk2q;x> zy^Go<+evJO4^iRVwut+<`0}UO-g@(M;y3-}9+huty@Vk6RAO2BY@{R4#JVS~hFtl` zScuZ^&P%XfsUDdK{Z;Qy(Mm*5eGYf@*Pg9@7TKS~Dn)ZG7V^k2eDrw3(nF=)3x%tD zexj=jw#DHzc?5X-A^&t?BQyj>RhlE|8Ztrm>HGTPxh`yo&&K7h&V}ytj&>uq0o6n6 zwAZKHEXoAC@@mGrug9PCV9d#by@i)pVY0k|usTDkz>oQz`%=;L-if!%;wW50%$*07 zgSrJ9Yt6RY8n^2@cNJ08yz(gV;YwuIN`QA;&)dQlSg0(2OCcGk#iApqdj_~KF_&4zYLvl-^_VY_)_DReu4tMZZrwDi#j2Ie2rUlU z1PVRP9aFlfME|H-KHH|@pvygVT*z!F%ve(^EsC8AXio+C?kglzc71wBM>T8TphHRU z$7*>qx4|AneS0ud7dX>u7d12Cy-l8GXFJcOay~v}P`aUk4b+s2FNB3~Luqfy@VQNr zp={9rDy<@A6nev2ke91S{UuC^MrQH53}$s4;x}xa=>-&eBYi z)%w%z#*QziRwqd;xWh#`=vzewWFzcE9WCK{PaO{)(G1?GZx;n`4<~K+Pq|*9KJW4+ zVX2Kqh(3hlY2QQNm05lxkyG5=X$d_|eTEmt2eS@f>VxN({^n79>|5c(iBpqpk`nmU z#%xc+;IvTVB6nB6)7+3vE1pq;kYH8BTTy;Fy=H@SY%QG={)RL<(s?px8w1$?gwv&o zgI47Ds%4ltJ10VmR-Jr4VS%4>+v^fn5a(*KOz4iv4u9N(X?QIXgQdHA9awlvf2+@Z z?_V@ep7w`XXQB5>hRXWKI!S9fXKzft2>-Egx;*TSI)CnDEw3w@Kt56j0E`BSI(qNB z)_z;e&Ax?^{F)s3RfRa0WHkdyx|r%4mgepc&IrDJVjNVRnyL5*&xkowoS=V$tSxYho7KHX z|H5h`f6Y(X(kHbz1C4^S4&<|&cpd~H4g&*ibWVeyjgD<+3&S9mSGoVqjrDK$68L{| zzCfC1+Z*RKjgJb_LRT$@-txiNq4r@}tZk*S&Kvh$md>}y9iD=?bN>^5^}m!z|HrNr+6fDr z+)fA(CADszLM?Nq_~JUfCj?~rM{c|`?>9Z5*ty05_hZ-?0}E!1mJ#lG5>?T z_YP~i&DMnzL85>J5$O=5D@~dxEnuOD2sT6zqtZb@n)C!gdKVEzAt)d!BGLtc(5o7H zhfoEn2_*y);(2^$&+K>ho;mv+_nG~j^Ih|o7x)WFp0d`x?p5yX^F7dY<3#{DBwC5X z^cpVEhSdSf@A&n?7z7WP35}cX9S6y#SAcCgbKQiAKs(@1Ketdr*yz9V1Ja%R1rn(A zi#{LF!-v3O0WvI5@DHyXTEb{5@T8{5<`w4~3p!k9y<56`<9R+3R-3*W^Y3N8%U=Zv z^#8^8`WL?Ve+FLwwK`Vh#3>&erB+e^Bm9<)H(ex1Q<}baW0ww9)QA0T5GZGGXld}4 zyii_C3!(u?o%(md#a{zvX7jNkFxf{JM`A1Ud7OGL#YuK)raq*UNA%?Ktibq@q~>hp z=V=Mq;*2x0iL1C;0!7NZ-Zer?t6xKUN-BdG_N0Gz<%zeb^BnxqUM9?zC$?@rO6WTL z^-JDNQu)DGi@CwiMY09`(`{J0(yo2PpXvMo!ES)UwgvM^P3;WO>zIdgNMji1zAt~L zc;})dsj0x0H1}AcLb-HiQ)H~=-8)9N54O-pk-W#)R14Q9J}ei%Cm%2P_;Dhm3%xc> z!>1rYVvAW$#h7P-z_YHIuX40&iOxuQn%%ss1l^CPMA zzU625zx4XcYGL}nq2u}=`A__j8zgjsYPAbv6Cd@27s&sDYBF&7`5=@{`d%(QnlwIw zB{4D2{5fWr;RafDQdh-WYx}+EU+gWT`ltQc`wP$c>F-w0;Jf`1xjKNpyd1=M75tU= zh-7Z<<6`yif|ENq_U*JN2burDz5DB({6G8?QbtRHoj^1Vly|_>u8i@2U$V1H{i_5( z?!P4f{u6xt1E1vo{V?#?{UNOb;FniJZPy&a}yh zja^)?kSLyDdQolq;$>oDzcG%!_k8s&@pijSi}ho4eoU=~mNHF`BtVy->`WRc^JJG3 ze5|XDj7@REO$KWR$CO#a&TxP!h{B)GW$~px8MBhnE>eH6!#obP|frY1L>uhjO6A0q2Y&F%jJo&GxN{C#Zj3x5TXjD{X2 zSp4%ZuWnCbes|tNpQPPi7_0C;+8Q;I%lktZS?0>XM ztXOt1jwM|#SQ^FUV`b3y;{NAEz1M?!?gga^<)@>xk57pN$$hV|eU@pall`49up~D5 z?Vj<{(#YFUVuA-RXftaDWw0p!dP@MT2|+Y<-BR&EE;@Vw;}@=SLdyUpF$S87)xb1q zuU^dh45%A%Yf1zA@S{mB1BCGtQS`} z8~KiAK~)dX=#z=&M8Sv5S}f@P7Q$s9{Vj5O9XkK-3kjPoTf$MTXg7$6#^L->3~Nc^ zM7t)_*kHa~W@(heie0MIEJFEht>_g`6V6@6Pd|P*ba@wYaE;DzFI%>r%!=3VN4;sNx9Tb-$ohB1mhSNQk7VPYH7LH>7j+EtWXTZFmol4t_3d z6#mM_dtPyma9l<(+n$VlYM|rgkBe4T-Sqw&Yx74h0dy5b<6NN|CE^xufz!}olXo~k zPfL z4_$DYf^#%K8S_k1bAARgcOuU}*Ha6gd+n|Tp8F{pbgtHFW9&I8FEVkl0R~Lv8S~~3 zj+IrolASAcL=x*Bhi0C+JI^mFqa)0OH;3$8{Ht5WuUYcY|8{go$(DK$em4M?tMFS6#o^N;ML$}3*ic*D1JJ%;wRYK~zROY!wy z>|dD72hA@(AOb(bNR>1hh32QeS?4YX z24u{xXTUQw_@}Ti+R084k|dZkn6uz#o%MEP@O+{0tzHX(T(4iYrw^C)1d; zXa1R2f12#>*w;_Kv4?KjgdzLnkbIC4iaIG85cmR>eAQ~i6IE(f*W2y224TsPPlZ$i z$~D|ie1UlXZFBoSJgDYpu|;mWk{Y%les_Rokm&p9ETcCkPCZ_8nHekyzs=RFFX$%u z5#T!PgKGAa36SR_?D657x`oy5SoHkf7TEV#YQJE3ZwR6(NUiOkhx?Jd>xQp(9)Q;B zs+0BCB9|(q-2zoI^?gu_hTq_<<;x;k7hHV=yxuuoA}82CmCi-vMSK^LN&IMYllj95 zMxn{dkSG6FL?!>puQPl>+ONb^*wvsy#tu>9(VZn<(#k{5m05uVn`&3U2fer{@6Q8s zQL`luXR*bpkr%FiZEgasN|*;_+htLeHYDoxsB5C75p(9At825@Kewy&JViFVEnwYT zd)8GIlC^X4)gLuBzwWgD&O7|m$);botN|NTZ<=B?B9F5%4KvuAWL7%-n_!K z+v`c-X^wL>Yz8JUbI2-+BB9#EG0Fh|sMy@VE{|>y2c1wd(g{8NN?CN`iw}Kd_R!5@3?MF55~LOKT%QiR23! zAB9RfHLms1D;`Wu^!IsXZpy8c zxLhvcW8B!a6zfTCyh{^4>5?k{Pl4HN$r`$gh=%LrMKK&Xh^xc+fbm zTAOlX7bz}pVk_TE{mHUe_xp6FWK-dj1sC^19=d4*r3%#sOj%BY?0yZyaV!VTq|dWl z%;({9w@KQR-K8+d^87xC;|S0jYg&J#>Vb%nDTql6o1co`ym}9F1Tsd^0gJ>;86^@L z4KipBN|YqMI5$_!zL}T9^QNN0N{6pp{9O6);VxsN_P!vbv8Sx-S<W-4-v0Xmh{*6PnvPMQ7ek)uoh;hX=3Avr%@fE%L2Ii+_ZfTwx_Y z{=i6>nGIU(%GehYgy1kg>#XtifJD&pVvcIRR63N+`Dc;$9Y;M-)v^U{x;}?wPJ{ua9hW80jC=VI9;8` z#n~hQP;r|EH2tjTK&Pmdu(1Y|^a3`zd}q6Rpx>~3IQ#feEL0h@gb-SoPf0rDZ*-)e zV+;&QDHqb^7Uds!%h2R;UmNk1P^`C9PBNZ)@v-BMtt$)V?8&RtBW+gSuIKD%d_D^L z*6;2<|Ma4+J;N-RNO+38m!b(q78p-8Vc*UltG10EpRsf#erT&~8z_yTKD|rPG?fI1 za^cUu^tb!_zwiI_kz4|GiV`oTR<IlB?6#;F|=QaygIhKmv+W0^OQrT z(OZ^ids9AGx`I6T!JdSF<)Q$dHbd-)bA?BR5&U?=84MPHSO|J$Mqoul&Qmr66T{qvwZ#ebJ z=b3&|)`zTTwz}u0zMSOAymxDai;;o+a_AdNyZ!s6ERIuZ!12tr)e8d+d+|R*BzQA2 zYD|S>7lXMI{hD@Rl7H!}CSsD|dqsdpo4LB7eTi%&r_<(zlW*4r5|bW+@x=pw@un2m zZ-t@4gfP?w;VM9Z&h`vFpb8-sx9Y5b8MsFU+$liQo`2JR|Bu26Rsx4l{3&7>$hl@V zR6J5cCudt!jn4EmEJlg(+8&=KszD4iAVv`Md74nSiYu5S)tl0!Hd+)b;3(O^O)@BX zz-N;yGHhOYl>eUW#O)L6X;ePQj_04(@DK2E|L`O5IIxnLxWYRGA-bR*iJ?}_4cjcV zFIX>(CrQ!K4{g|bN1UbnE%mZV}<~OZvVo41{*)2_14jK%>4;^O+2sO?<8DAJXwm7zbMi2PSTI_={9twhp z*8$q?Jog6dAJ>U0Nx{7owzAW}2rh4#<_}+C5av0VrN6-v{v%69`8nLBs9Gg}h0+ng zGAe>kQ}BWIsjw81-b6^d>jX8o zuc}k-gwIme?3eCuv*Pz2Lk_GpG*~CQL$ua+(vXgraAdJ*d>Y%h16s7?fx&D0s}~tG zG@~0vd^_!8=b^loZP`J>LSO#8)B>2sm2f`0Mtm4rgyNBV(9ElyJNSedhw$@T6Dg3G z#pDYlf{PRbk#N1!3%PH>9L&DT)hp@o{PgC|`*BPaXJ@(IsbyY$n$)gj2q$#IdvO7v z3MeI(n7LTZONk_7c_`+Z1#cc-pFehCmVWIlVd3&FH-YfNP)`Qnj1Em8V=!!JW!mxH z07K$RL+Mc3$L?>`(~!mTVh{d+q|<17{Jxl&V|y>wr$T6pU=M&pZ8aeE?*swo(skdg zZ&1(-6^VVdaew6Z5gmdBQ)`NuK11Z*34ISlLeF-s}ylVisT{TkrA^QrP6@xY{PbA6s`&GuN1m$UQk_Uq>D z%uT0rKSFqBg+T$0W9_4(%SzfpzNCdI-9#dMVzjxz(nGJZXUTc_X6Z3V5Q{F`8N6n-}!U3NxRw=JD>nC!{d5xsd^PqJUWmyZMOlx|L8NZc3mR zkzo0$yu)+u&Z|`ZC#EN^q1WE~&Wer4lQQ6dr*J~VRUnt(q_aqby`T7UvXfA)DN1C%Yz&7&T;G_=nw-ltgStqJd-E$}W<^zYDTb*X4>~h0 z>r_A|#dSP;X7mYCrhjO1_-y@{a+r1w&`x6L)&l!{rkXwALZ57<(q!7Pm0GpDg?IOs z{MGO_G7rWjhai>@$Z}CtMbDWb+Z91twtYw!(2N-!dQ`#HFckv(d|#?apeO@#l%#1WVw()q;4HJBCk*H#WTCyl~jt`{vsy zCIc}dL~xFQge1rHd*Y*T#VOGxNQmCf3+MMvxBJgik2BsdcV$n0g-FtF><@wV3~!m-=Cqc=q|Xv*)^Q zCFojTsd2CgK*$}{=dBeZLhBDgDC%aw)CAG{FzjA>Wki|B6<19yG~uTV%XituHrXUc zPP4{%40vX+Mk1Xt!|FBL51c1m4-y&-kaHEy;dw8wY3d!#q@Ol{flA#%)g=O=8P0x& zI6Ns4ZI|Wq(kI=JC0F8>6Z(RohQGk$IPHuN{7;UWQnMi1>d0ZcDenft{V`aF3L%W^ zarf9f-<#s{1A{6p`+em`c8kJ}oXCK;30`Wd7VXgzwWOV?G)uZs-Z$NseSwYrhVnOC zez;C0Hih^z891;;&ZlgWo=q1mvrh&*N9#nu zJ)&d{oCz1B;p#0R?y);Gwqp5X6_#pen~hQXE-^Ul%7Y%K7&R}`TF4nw<$q`_g7jHWmQ(-jV|IQy4r^Y z&g+RLjLtDE69(Fm)V@t}^%OW7i={#ljun-4|r&(;m>?)GN1lG2BTU93T9 zVQS4V)BMDC`X>Nqw)f>hd||cm!|uj|U<@<5loB|Og`axk^LFY|<)_ErXsUxUlkWGm zt{lqtD1L6EJtbal4w>CR3}eWx)~NjkQ@O*^yicB3Y9MV(KWgqO(mv&HowTAYv_L%u zZk>L~s71pzCK#s3N^3!}0hbOLz?r~_zLidFNNM%SI1EbWcui*sDJv`rPK}g#fys-M}K`RI~ldQh;Wc>$Z@Y6*9pljQ8$|uzRk%fN8!P8!_ z_R@Iz)8_K9<#(T#I^Mp9RsY@@{2spi4UGH8(ePiML}04}{us7r_FG%2VUSN*Es7#l)@U5;mqR;381k+&uYR4X z!}V~q6`quLZAoD7L!;cg3UOw+3Eo_OQ@IZ=_VMwyrIED;C|=at^e@;0OMS*{G_y|U z_s3@5Z8+eHmN_?}lEXNm*Jl3x0jqjY4SF3OIQ~8V`gwZ8YpH=MI zsLUXCBR-^f&_lLe>AUsl%GZK0b%qo8ZTNQ_ga-(#BRPKY(Td`a>5&!xd|5}fKosWOrmw(xhn|6*U4z%E0MjG(G0tGBRBl925 z#8bbAGIRI~7ik=r#L_rXWCx1d!A1ZeA_HvRp3V|~AA)$sqIzef^w4m6HgAi-IFXH5 z*OR`@nK4MZhak4iI;^TWILuvLf`!Mp13@>tn8kXZ5APtPwItGPxe0(dyS?rvPm&~ zkNV}bZwtxwL4eW}j%|!$+{7pTJ}=8rVe9I#l*8o$UX0$Ru-#_tZQ8=Y&cbvJN1-KE|a4u5*~{ItMo7osgl)9@@fOm6~CKFB8%>TqnM_hSh* zmFWbe(U>jL95RQQSVU}S1JTs)KOoC$K+FXEsDu0iKW)LLRvKmbr;rUxhLgJS<2Z^g zznNM3Y72@b%PG_(@5qNJ4Oix@m;;|Yte0ybYZX72aUX`o2j7U^)eDQz;LjiX&iBBr z7#;X7o23c8dVFr~u3`*x`gjMd3;SOPvHuyIP5>~Ar$ton0{sLf1bU=7WJ`TQa_K24 ze>^AT7Ev--UfHuNBi-VGMV`jliRlxr&qQN7VAp&Buol|MS-xfeqQ6*O_tt$eHH|YV zBd#Mz1#KaV3%V=M?vrM63?nV*Sg{Gx> zR(-Wh4a5K`y1tXU=(1rvl-ZnW^FYGl8IE0=iTZQ&LQ`)&7H*Yf;EE_08^M6MH{oN|5Z_Cf72t4yRxF>2C$|q{o7;c}hKJdNT zcGxG~z;cw#%H;l)pky6%m+d$ZDL?#Mdexrd9;n!V<3nX%aZ^t*)#*X{a;x3yzRyzL z8YhkNJ`@IjDt&HuToWV_D4=r2O^4$Yyk>XY@tz|hluYR5yAXucxCsO94_k>pZcqOD z$$9!gOsxTK!j`5_oULD>TtW=uhzX8I2Lkiy9P;WdV*`Yu#oSLq&%*nHv^;mpku$<{ zZeTX)AXMpGI4OtVKpMTp-vlNH&badlGu@*R_yclX2D4r<21YB;`6)m-P_jPO&(eG1 zartWp^LKsuPGWufe7g6aUV9V_=TC&6uM5(YqW5A;26=#vOUfjU21C(RwFT^A=^8XQ zQt$Ir`8X{J$}^L~gxeN}SFH6BJCys(ga~EmBI1L0G{!ETiDppNi(9pMAU6-c5 z;O)9I7zCZBi#>BYWO@G#Ozh285a_fh{J@&UuB#*12toVM`%wieONU8GQ*uQl*vSu_ z8%9TLg1ZgNEU#ESW-;r4WZN)vrYN3(Mzp6wsD|SywFKq>c1mD2MYNtTu+qrc1?%$Z z^e)4N=jc@j{#=g18V#ws7|3pjix0K7nw7K?ssi8VLp3PqQFeN5c0R>1+loi@v#IOH z_UGX|M|>GgU{k8K1B6!DY|mLw*GXZNyU&dIIb{P^ycatYMDHd9lcNxfRGNHnGE) zsKdnb!DEMO$>gE)v2UuXW_pIQEAo1ym!oX;dk4Q--Ja*wJLDe+QFr|*Js5)K#qkx<#*y00}M}@x5A0^7Ry=)a*3=cmd79*zvd8~Od z0lKa3oil(MXwN2?4UY`s_lH^dzPuWx-%_i4@#y8aqqIlF4g0pEE$1h>Ud2RvekH=I?g3^XScb zIHY<%-sGCJk~~LE_{rGkPo_%O0-o6j-sL~^6*hKHbBP&PGxV+R#sy9@mBRq$4%}^F&Izc>yV5A_C&AQ z5DkyadSBGe90dBSHG}q=)t;!71?xzgV3h~WJy|8)T8KbF1)wj{z`@X_LmF0 z4a}Bh8JqZH;^r@NU!UH!{}VH0$8sm<#7-Jo91IdTSqeN=78vrGh$`}~dyNKT&xp2F{5f|cWoXqsV>JrpH}kVd!h?s z$@j^c>L%gSJ>ReP43*uU3xbIAC0(BFW=roDUl=<1Sn-&>Jj47JmH~m+1<3%Lhj>CD zUcql2NoN6kIIsfi+4I+Cr?kWZi>T-5eF)-bzN^7{G+`w1A*lW{boKy89#yb|U0eWz zTh3V%KH|py$6H#Z4ad28o!R=)8G8 zRhpEs`XcwmVP7oq(GsDcZMk{it5VFAM35Bt03*100%jFt0Dr@3%cu=k+!jbaYzhK} zyG{A+G6JpQ2ZS02wCns+b}UxtmYAJpppd88PS4x?0g=Per+${4&isIwg0hMfaNONG z0ObRB2O3adY1;FDKt9faZ3r*N^a65lrvKl*>HimR3k?Avxb|9r_uU2{>c$juvnrIF zsWOO&lDU=v*C2t)VLbU6Y1iAi{u_M6uY}2NN=n-zg0$AcAphq98IzB%Ux!T-?E^#4}X0t22dLxWOIklKwc)hDs_JJL7y zzaMb1iD~wNtjA3%7G}&IH-&661!<{~R%rXDCc^UEH<4`UBMWo^J@M)I$0)(%K1IW2TA5g6Xagq9YR{r(!e=XgA zb!6Aw?DWZ!3Ce5fr!}#e4jC~4HjeEsbGS_J~0Y$03!g1t5zsh9X@6IAZ3gwfR?Z%~s zKB9=lmk`*DHs7P)rO^J76cw1?DNpB!X9d-hjju4{*D+Jzgo_LD{(!uQ2DZKCBamUBLa1ZGwPmlW|NnjKV!$PIH~g zosAmgIpyri9G1T7zSr#K^_T+4^M??~Lv2pzR`YwB;m+$H5ElXu%EzLn!_r}l=NzwR zyHgSq)&Y9_lCKu@X}o353E@XNAwh6IiX~`+s-VW=*feTMX>x^$(mbUXk&hPg z@jla8sbqyD-+RiGWV5fBqmk`0J|g)PFTy`e^j_Z;e47+Q`)f!BWLLwHV1se!Sdi9x zia7XYG)nkX6_+btfQhOQoJcb*yx!;_=F^kE>y7Z9;hmGu5ve+#?mrL?%V5IPN?@k5 zC=2wOyO{6V@0{+$y_0Xicl*^W|DI2VhXTd|xhg_oq1W|ew#Drkx^Uyqk*jz3P&=Vs z@Ev|_#Can|ZiOfB55-+}VXg>+5dqsppmAD|Jt+Q8#4mJ3$|da&oho?}s~#MlC+jQoMTi#1sSs5sdkyzPwzDXcG#)@?OXS$u?#_HV`)3X7NJ)Nf6nx z!=*X5U>TH*V<=K9v^1t%AYh*dR{4wxY~ea9AJ2dgb1?DU-il84hPdqt;n1^zzL)A zTsIVpC}GnJ^-vTO=u^qBtHgOga}8=&{X>mR-U=L`Q1BntT5~9jVmJsZ^CXcx|VP{ zV;-e@%YshAHP8zQl=gNzrQF~nxrF;?IX-qS^f&x?&i&BhUB8f?{j~zYXCO8pc;wUK z@0TR@M|cw0hb=6wx4s%}^!#wWK`hKUhTl5E$NJ9AH{;+@bQc9yi`%6oLXkdA>bp-Y ztHs}XZ`SA!r|B;lXwUD}-O^fp{zfoDBuEuEVH5yB_sdJ;0K1!mf_;Eh>QaBUik!{? z<2yu305kOyR)?w(L+FvlAvoe0Cm`Vb6T&b%O~7_IurSPc6bQ^B0OsQCuY+%kYq5bq z=^%*R_5-VbIR61+{cM0BI1SP$Foy(AxrEw-oO?u32PfAQM>_*C>!u$JkneKM=vmJX zqjMEjmb!_la&2o?x;IQ^N*~#kI79n}>YquT+^J9CTQ9(30%%+_}+ab6C7&qtN)kVIJY!U^cL7M)Wac z(MmXacff^0Y^~`Jhz)Tw%J~L)2=8{wK}~D$<+-&PT1)#wx5~I66W9oWXbiq~Dz*VR zh%?~((pmnA*gG-=eY7(ra3p?@t5=j+|6FDj#2r%-zchP`c*+txfv+Fs>b4{qRo*0A zw`@VT<`hA$Fu&L*^F%nA^@Y2lK??*SJdEzdmPt1GegjP|#tlMBJ(&*7kBDQ*GkR>8 z2UW#4Ukkk(+)#$*I#7kt!x-*Y@$>^`8=+;l;X?seCJ%_b*^s(nfFZR(Ncd25O?)`U z9t)$OSIulbc568047L#q6pESyA{q@K$zlT5Z0dj-0j{$#9+>Xd87D0Hk_%>i9`I?# z+K5QZ0W(W_@7NJ~A0(HiQ)Lk}5`-I)+cWz?zy!9G{ZoSDXWKCtp~OT}1Z?LvjvnHq zEtCm%wkuL}C$+cpu66&))2gon;j3WOVqONkl|VUcM|%!z(7>+dtyXQs3&JiTpC1rg zIJEVSk|IU)yh1!V^r01~T5WU+H zpdlY`PJ!u+r|7(ybmjv;AX+!T$Ea(A)6as#_@95rW=kalRm%48+{suRR&zFwp!00YbVR2HxC@pQc(?x}Dp6j&0appXFdN9Z^(U1MK@2ILP)o z0=VL*iB0FU5Y?V!^_QsWrXb|ixj$urXieaW_n6kpMAT073h1GczlcZjl0-My7HZSs zqlzuy{e5kzHGP5hO<-$Mnx}jY#wW7G`Z#RG8nbBDUg0l@SeAg%%6=8vGO0DGa7vj~>>wB*Dror8>|IRb-WXABz4 zBaG|uH1S0Yr59j@uh^p59d>LumSQ*nOx9Zf6#uXNXx7*w%;MS+PRxBI{BEFzl6B77 z{%I2!%ari|l6;-?R>=^sMYgdSHyx)r5mp z-PtfE^@bXU1IX2G{{CvBdj=x&sPXQe{UgKYl1?e^6@7S&^?jE0QPJQ%v&vBMpBMTs z#bo~o-v6I|T|g1_CcROS-fd0~pB&|G>X=hYPqy{^{_tA|o|EVsc1AdN*SqZL#ZxW_ zZ-F2`APBQbbit1!>YTYJVqbmO&UL8vOd2b`aTTjvSR8lKddbAXo>xN(3?|SN`o%(l z{}#peS8Ykzc@{?gnS{aN*A8~2kHr|LWLmU$0MD+DsjmXn`M7Pdg&&ZRB@iS%{EhGO zr}tyl4S)xR4FOao_8YmdxdX_`xukqLyckc{2UksSg$mGnrzDDoaKO?M!9`OikUxRT zcOFO4>PBw;bj;qkC2w%`(lErpSD01c59@u`&C@ z`_&x}xCKHuTacx>8bWV+2H!d8rp=Eb8m(+8GPvL}hSaF?tyV)Zkzf`NjIw?A}s);uKj z$FxhfxQ(VNf8`ctSg)VJ)uWgKB#6n_;7XRvQ#lT$j>kjA;-(f8v3qT-{ zd5}8%Qvq!4PtEHXz46%^XGf}W#nA<{Fk-;(g6W|ygXGFq9ma!)*%L$9;%7nttQG)q z4%~<$PCY@pOrhrT!=ZCx19eon4gbnUE2r-8B$&s|kOBrRHn$j%HnO2=pg~0@Go4Yk zy4+(IC9|=B)@)Y&z`5*BJcS%Ce!B_w)zDzG`X*ZkbQulk|3o{mU->NYJ6t``EfB#o zf=K>h(o{Yb3O;1^u7bQmwGX%hNTUUYkAMF7Kfi?*7x0kpK%gEyE>C3v$8s(?gUyMo zx>G>k6Tm|`WHXKubgam(&UM9jt7q|fCP}m++!NZH&qBz9nDHz`5v%<*l{~+b$WYHn zMT`bBWD8EN+bP3LWOzPvh58^)S9NJf@2mM#;XfHXzkM11nAfB*%PDPOq@i?g(+E5K zEoS^VW_JlT47Jw_&P_?JGn6%M>OVTF=XxYLQRbE@*G$j4jCFDieIe&P1oj01+I^Yk z{ZyM}Rhu~K&ZkXk@`(lC81wqXnXL7~Tmq06QIPHC@tmb@qI4rjUKD@A^7>3R*WVB> zDvA<3-+o~fv66hUs9``_L}7Q>-kW*s6I|8VTXb{%pRaWvEQbD}Vd4oN_-MD{hq7f|$=OITH) zodDY)Yxo1AtD}v$MQ_8bIurH*qi0zJuKevFq!w=c6>ilHltPhZ2QhPlAQQ)5hg+2a z%=^ZKD=-4!sEDF9D5_1ELe!F`WTXa?!J?My?GRwo<;*qeHH*rv_BLC7O69)6c4Ax+ zXw>bd*ib#tcm#W;I#cdz%D3jaia7jwo|5?^9WgnzP>)sp>d^E0^H6|iRp>*6GOcHm z(k!mwNJypS>n721i?mA(YIjzLmIo6`iN}X#3~xPcf(~VHfkyf{{D74{r36;{1ZYo% zG-?U=mdLY@1io()y#37}yQF}bOU_#Se!?=^_y~ZHEVPqg21hq1_ed)(7ZtBazV8)<(~Q*(UYaA501yJeoiNyWzxp0pEpTRd=-RL2K~_3 zflHH^T2zTobp0F2=m2ppYt_C4Z1IpJYCS?%A7ZB{ca=fX7IbPU1Fwq6BoNObVpf^KU5mE}Vuflz>!g{T8NMX1MaYucZw zpVa$aa%~Z~=l>k{G=TT87*?-Z;vh;_hbRF|5mr{=H- zx|DhAIQgORJ|CCfcX#B(VlpjT@D#=$aoWR-bjlc-wK>@VSBF}V_BdxU4bPkeRvNCjPU$7zm4 zObr%_qVzX<6SEQ<7IFrjLQ?6o2r-5HOGz<_b?*(vK{0`Ao%6-XwW*X4mE0|FJTh8& z;SH8&WXYFbf!3Mfa>oiow>=@LC&yrwMv@)4UU2lQv5qLxMj1>_nKAM~D5 zRz2G6=6b}C^R1vDk77k)f{p;Yo*#q;x&hIGr~vhyESiBA+2de*WR@km?n(=M)Z83PAG$nW^ZCwrfO01hws(s@R|J0J@h%WkCc#RgVhF`!X*m@C zN!RH01>`}W@@ic>=4LJV)*kR*`r5(6RH+m&cWg}pg z)F9jMgOHM#@oem>dp=-{SP-+~5Ogg1OO5K{8nM%C7yN-|g3by(QV@bZ6L5z3t%`aa z-O=C2)mJBtgZTt&>YZCtyr_TiN_sE;xbG5pj+cOO_Ul)??$rIWq-=+vCqL>CpE3jzngMb*gcUIet1sUJ2v@=i2I^3 zu>kD>8)z6&G|5+N&Fs%dqfJL(V{<>By);7-hKV8Pj?waI;LMb!$v}HYr|47waPgA zDl1fR)ecs}_1=nah6R5blVR{_oUQJ#Y^Gh(3`aK1B)gdz)U{Ls6!3ozp8npG|DNvr z|Is_JF#r=OL}&NGHv5&;%OaXnt<$ACRNJiz}YNmS1A_Wz9;8G6>Ew=>BqO@lRK&`1Ot4nkBDM z!o7%_WS9Q9G*9Q1NdA1SwDC_onX!8r z4cxp0JX)ouG%~fH0N`V43kz*s`o49%nm;o> z3y?t-#tuV~D5LrN@=|{$vK1KpCD{fg-igLzM$rj5qHwENBaff-G^#4i-0eml6shJ{EcU=?s`2fZ1g`*bDWl(OSaC z5p_eM>F^oHeGot6ddz@7`7XsSkSj08N5Dz#!ESTuG#4u$tDEk;R(DSG2(Z^G?ir7Q zY`%p|X-NklD1oH>x|In;L@J|S^5Y+nV=z{$)S>G)4LGCML_O94Cm?15Yx5=$&*H+| z=b1q52PQIaF`7g8@z`E#hVIW$J-WKKQ0_}~Ef~uK5A$28cjDK)jNmxKsuova7h7^n z|5cpL>BLNhrqIcgS1!Q>rh~L${zMjXAkl7db^@@-dwoPN_T@XP22>?Bc0f)S|Qv>z_?*$tE2hZGc79Y+%N6M&?yK=?QZ$g`#0T-lvrydA6 z0&3qJ(=|wsag(29Ppb30WAw8eEx!9?%I>C{7~62Q2lkl&sJ9<`Co|do&Uu}}!jpGf z-=v`RNCp%ezzq4OB^e+AmN33Pb^;Yq&0pHPsOp>O+ScwWq44dDNJ3{Ro8q39@o5ez zh8bBW`d$lS={)IG2U?7hUN`zogg+vq{D^u`?ImK1)>a1F+60Prpne=t?|eg^-SO@` zTCTdx#)tb_BHs`0lG%xIc)r3Sj=}CZl1qmLVWXadV zw>g;ksgw(AA9dQx^3_Onor$&U*S%n-HAQT7S0<9aK%$9z#y!X@Wl>D774V z3Ldna2Wo;#8IVgoO|L<*Y02*7HdODTB133bmJdwYg8 zr^&L~_(;TVueAw6L^|KcZkyW#yN^v&rxHXnYvznwOUt|>1H>m|pZp+xQa78JIdwHu z!zRboE5syjX!+m|$YXL5Le9F`{X(ly3@6yI;i136^#5BiKZcf{6AX6w=DJI1kod`M--;x9znM<0W%4sT3|Xs9M?t$@oVePS`8?@ zkFailTdn$y2eIr%j;EtaENSfr`-2ued#ysm`^s$Fs%P60@zYWabGZ8`;<;!+2Gar( zvT225a6zKCp2%m%tQHo3`c|;&`i39V7FmLBBwg@{#r+ecWWA3@2Sws)=mmQn50I2^4FHxSx ziI_-!NinSR(`#@(*61!;QvB$P6PMrOp;Qo>1SM(<=HCaf4L5XZ>T0A2Np)%WWb~-W z=LFkl7ke202ttpVq$X_rUPB+^wf17&Ff8_gGruOTqtF%!?EpZTY8 zZ5!D(ieKC>uzcWbUb)SP;#C;00Ei?si8FoqffGw|n+<$+WS2K?c8q0x=-aJir!Z~n zXm^ghlE~|Cgr|f-MS41}WOGrJWY~Z>P?8@>*y}l0%W&KQt4xc3133 zq3AKQo6B1f&5(pW$lLhAO>!#5j)d1CA)+;4Z$V}{jHTUTv%E)mFQ>aUbx-t{5saS* z^{xN52@D{k;_$_(@gI;OWUbQ#fnC3kFFQnINH>pz>ETyX9^p9ErUCvW(c_Hl+QJNL znb&t>z#+kh2#GwV)@+;RdKD?uzw^0zNorNXBldNx+B#$KNBl{*R7k!TRTDje;g`>Q zS84UaXZS+gVI=_&8Qf6!zeBvv5yYu;vL2BtqXUgY*L9@@P4ue7qfe|#UTBvvIHD(ImY8tOjA7gm3km=}Q8?~x-51+gG7JRXQG*QnQFqAf-6c{YA3y2?hLTYpfq2MwHCUbF4lYRj8i(x zgY7`nM?JCn*T1nnc2?#|>~h3v|~2&B^5n-6s4y-1+;2D&i3c z>z+z3=2P7qup9x$zBuK~d*7o=?d2nir0T_O3#p&DMiJyP0a4416C9VX-S+al%X{s- z4(tU}kl-JAgTFwozyBHj*^j_x!)XWko3LD9sEaGpAO+LFH|CiT$ zE$${J#C|T=I-{fEa$CIv2KbAD8$~1r)J`f5ksDxGUR}{A8ReeBJ^k$I(&=SkUl#X< zCT><=o1un>?%JAlzl^8svwB3w*SQ_mm*4MF3KA@lSMm0oc2IZ&d*+~N=1?<%{Y{{j3j}Xyhi_8 zEmTU(ufaghxz}^*}EY!zv-$TYi{KWr)ub|_|V(vv0v5fqne@Vk4?}nqF zL>Z%#tDEnuVymSD`O#}vdV<>&4>o}SX`E9EW<$J6_M}`P!VR4e5PN=kNuw0Ml8FN- zOV*I^6b6<$TEsy`rLq%Z4BfvSrvFCV@ekf@1gMCrJrAS`2Knht{O}zn^RAQzk4a3J zH&26p zIBzgL%k!^tlIZiG@@F*x!}0^tk72{iZM@)yMD#om_(LcQ+)k0?c&ED>TVBH2ooh#4v&MUp*~vWLjN%!sU6icknyD_cS$j9rps z&pO7Eof+#eOTVXk?>YCLbDVQK&gp*dJ-_}mX8C-c&-1+3*ZcJ%3%EzS5t@TwnE+?$ za~(Z3fFRRckf#6PN~l&r+k++yG_hjw8RTH8#I+mw;GC`fBuM=gv++0lK!K}6cL6#8 zDXfgQ=c=VV;-Ok;i5A!id#qGXtn-FflH->e!``JiwkL90gK&^4}{$NowuedU?{wo(3XFV-Hm$0lbE zNk@{;Z);%FBp7G~Z;}b(Xq87l1)xZfZqhu9?tmi*eg{mMRj{1*Sk*5X>9bN}ypi4? zXeR;d-Is-jMJM949@-cT$M|8xVHWCir>;p;m|bobAVYz;$GG|Q+JXv>4O@mCJ!Po4EY|AIz<$Sngke?1fk#4-d>e1$Dl zhotd{>f?z7zIqbK7=6)GV;2E8;p$ZCLUYEGb-mEPVe9{$*Ysnl6>$VWOcUp|WZ3|g zhX_UwpS?K`UZ}8DK7rKsRQ5|GhT`KNNtWZ_xx^7gBszC}LWp6zI?SJ%S*M zzQ&%=g4QrjD^nRW8182mjtQvMs6i_g6!U+E04VVcB2S_(2K@qEU7R{uaM*l2n^#9+f{fm-FlM7%UEY9EB_lcyEDuUG(l!`y6CvVnOPf|1HZyiOg2cYw&>UgOLo+udp1bc;9;Z8uA7` z;us4RQS-6IasqdYIC)bAqfNU0N%O68r|gpy$&e)a_6@G=H^-vbyC67_OnQVCIF}*m zG>&A%=f=vIF1KiclE`~0cCg|7HB z0(<=aMiTPn1Y}N%&h!&em4aHZUBA6_)9g!CUe70qojuL)H#!#q|d`H0@=NHVq zCn9;5M;mWnGdZkv=dJ?GbA;C}sG8UE zt=Yl7SsC6uK8N*{M!cKLaLVr)AYK?E9kQk;yoxi8P>eU}%iBzm#?>L8-3?ZxHQ2Ww=!gwNR9!54eoIC^`zSd#8Xkp&6=F| zN{-H^!rURaP;<(*YT_ZQxbk`|gN_P+Swf-1$2GU6y-8ceL88<=ou8o`?t*QyyGZA7 zVNUPiNiWZcfO#NGT#dz4qWlosl3bB{2(+g^=xc2gT7FsG9Hk_`APIzK-dRu=Y;Ze% z*{c7=$O#gFSNc2w9mkV6CGstE5Qzc)TmdY4mCX^oHV^zOn4)W0=hio-GxU4C=mvVk z>+G=K#4rBdOZwsa|Dz%+U<-+W=ahwNpvHBPLji_%P0>xByNJkuYIs7D1HfC(aI#>m zw_MNq;p;zR&fl_}|H1!3-vbQTOokEWt}bio3Z$xe((EUmlQoOh+Ugi*u#o1cu_q5B!!o9!W(fAZLS`U;VS1r zcICmg5E?ZL+B~g?RP2_xL|hhTnANn!loJd_wW0^)f~Wl7J^>L#-rox&peRVZNa;!} z(5XQ8SZm$Rx#u$VqW1gQ9!e|y zlytPU$AeCbRL#5N)rYD-`Y=GwZO~18`#9^1+8d)H zqpJ5wFDdLMBj2Strn{*jo{s}Lm3QxjrHXIx6eDd=l^0PXk3sjPA|Di~)*xgSF#LR{ z#F*@+03hkL5R1Pr{!hN&%r%9g_{f-2DuP%vI8QrTPhv8soKX)$Pt(#;?$g!1d^1@^ zI!l(y+A;VCwjmrq>3tpn%ltx$93lx@+g_L^8aM{(U)yIhNzAK zCOde-`o9HjXzq=qT!3OxYjfCUGIzm#ES_hvCu{k4hE+c~AphasGaHaqL5$oYSX1hV zDDwW&B&Y$|A3f5JOKglll0M&xvG-0Gl|*- zLFjCV%SI3EH?Nkt)s9V%;*Ouk&y&uV60B1ihfL!Qyu?p*pxf$_M+WprCh6??8oh<)^RB`hYeOkg12-Wc=!S0V(DBF;ef22qZIsHJGhrj`*-F(5PL)(v0JB&JHK<=KFmxfTcSp38{{2$!xb~KAc z9wD+?1|48}h*>M|)eDpUBSO~$5Gx&k84_8WxD$;{L|X|OKsix0IrLp6xbzoh6-3_9 z{eA=P2Mq#Ews%(;))zvM_5TR92%|(#(zyacG4aggc~=MZB0Yk`R!=Dl+*3Qa?xP;V zB(FJ4f1zglZAZvkpBNSz&fYSSH3@AFEM~&ynbFAYFzPDEYzH9!Yt<_pU2^zk-7YUjI zdF7bT5ary1)SaN-nY$_h3qUUo+XI}k| zzx`ktRvDN?JB2BVna)U(Z*9aI^{m$S`&ZRXV&m5~<;}n@$b%8Xdnij7LBKYD_!0L) zr7?};ow*$oT~9t-nsN!rMDboftj%&ldwU3p_z>s%5vU0RgW%&yX)Hn4%muQ-*km=* zOO1Bm3O^LTv{azhXy=g*-HBp4J$O9f_7gRSUU`J>hTRVJFz9edkzz|>%i2&$)uF_pzZi9vw#$zm|O;ieI#-0+7^k`0S zCWp^{pnWe>L}H2=zQiQ846A`|HmZYe`DBpk!ZZH-71?5w@KMS(4SbK=Fh^0CEWITa z;`qs%F5)m}b?R3Q%;8I15{| zc&4N!PPMH4Fx|35B+(|-V$G{CG&Ek7p{%e#mF0Y03SD3@M2p%RQjtV2zR^O$w?_AE z^t!&dyT87n1EH2rg)83`C*0hSIHMU~*utq2Pp{6+X19@h`%oD0^wld-92qrsx67gU zx9@l(^;@zuMq>8GkP|=j8~)I2$zQ3yM_ZAIyI_0^ zV6F}VRbr$41Qgary~bT4`r(22=uSkbwL!`ev%B^2*bjA55T)5-0Oa%9N91&6u17GS z&5h{@Cx?W@dZ)DSILuiKB`XyM&GNb?3vFZH+E|CYG&3X5-L=7BVPjGBdS_OmpXYRh zl9uA;hYtoJG7%Qk-hgf`fItO;iDv8yg{9M&-|o_v4&|2jyx$KFf0Dfn$XYTSH+V1qKXf!8(IO=plJi)djO!E)`=5MctFq}otx$X z6pYL-F9G3g?1F1yXn>r~Tzv8%F*4 zjxEeW8@Hb%ud=h@o2^c%Sur}8gY4jpy1z_#u;~ zVzW~)6E4w=FEaaj8|b3P=>;H(@4XSifTXemQ2-+fek-6Xb{>Klj6xJ}yWtb7-=Vt5e|wEKlH^E^%0ZdO zDL}(0!mct?Mp4w2k1e70H(lSlANLPpW=F5ODV8kuei9*KA%qQ9d~;my{H9}om3U9u1JS9W}BNPrz=B8EH-uB&%63>|#re=r_*N04PG+Edv z9Oh|ew`5wH(Zm}p$K-(WNyfWr_d~#(%_^V*!a@~Y zsMx20%gtMtY!a?|@mRfmRw8pFTRulJhFubCK900ECxXGT_{3P&?)a%n&2;Syfo6iK;4bM>00-rMS# zShwlzrsZm{wrrU^ylJATEk+EXS_&ODW&lw1=%``T)=2@cRBP6$o7Q40njN|*dV$S) z>icfd-qUt(iHInXvN*W0M~M(e*ug^G>+5ra-KC}*aWTlJoBh+HyHu}Zgef`ojf`U% zFC5Q_U>aPE2Vwo|MbwbqsAnr;}kXPy2Ry!1E`5AU;_-szrxB;56uw{)syZb7*HQE zfC4X7=RVaG>UkF_HoO|Mh6lL#*5W7B{ZGE)tNt6_LReWIBInE61C_9tYRY!pmRl)( z1C!dcnkBD0_D5XQ4aw#>9T*n>Ns2eZm4~_v;(?lO#79gGcU6Yc4jHASR;Oc(S8+xL z^13zoCk0P8vD-So{7}K9+#eEu&D;IaFDqGpUP$|E$LW{9VD9Q$Bg5t=Zk@Z65T~2p zTF|)rU6E8wq|o`A@~n6A+=-fiYr?nyO=789I8W5^MVmW%s9t@Lj+bjc7OQ}n1|@-y z^`wHkwj)>;?{+FI41&5k2Vlho%UJ^=?KvU^XsD4|n-L*S2O|_?tN#9q_VL+E@uM%8 z_l_=9Ldd5G>_jO*8I z&^K9_dBA1&u?^Jil5-AH+flZafr_2epxxTkGtdit3wRzLZu|$fb|7qkjHMDG9uR%n zm7a4fiPyn&M;hLdlsw;W`_aVG;|k@%$DSPy6`L%m`@iNSt+)L*YRdrt!gqO`9OP-3 z;#xAb979e>E+ty^i#*M=)VI3lV(plEq`Yw3f~2mUBkZurrE5$(`z(7eB9=ja#3y!K z-!D7kHp0TgrA6mi3DS4M*HL})4IOjj?M=_4MDmUc3~9ES_a(67js0aU zQ)_cwPu9rT7N$2-a%QV0_=C!d_VK{`6;0X<9;u|~gz9hk`VIQecha~>Np@<_8xCAj zqS^V>?Lbsj^1+b^bqEw}(Lcc14lY^kMxh{P0_^VQFuo1AW*MCJk++?Vmo)Hg{%oDz%d2QuQokxNb|zILFy>rKJ@JzC3m%^hEN( z=}q2{<#ZK?Uvn|@%{}Uaw0@vT0SL)^0;)elcG48R{tO9!#G}z&JS}>WD*~z|eQ1hw z=H0PlfLP_rEc@zk^T#_w$V{BnumJx zyGB;77J~{o%@{dMgu*s)2j4j4f8PY}oNs$N{OXO2rq&r=n0OepKjHihrHdyE246n` z_^+{lJ&)fEd0nec1?T!{hOt^6dbl5x_!+X z`VJGAxEo|3y@K`5CCn$iG8=??3z~OWGCnUT*t3(inrSy4u(21g?cBA8w~wq1k!Tdb@Myht%^mk9k5_ z4kh36hU|gRT0tWIzycAfDRfQ1;pEbiLjm;h(2%zz7=Vt$%$pl+0TK@-xs-Uo@ih^< ze$VJDo&bTq7EnkevP}UEfF&qf^>9Bcf^O1%BsS|=m0eLiOC-Dg0r@!k?NQq=IUfT8 zq{zPmL%u2t`f=xhynTaHJYRkXCgbfArpoTI>iNK5kq>F!r2ySSE6;6V#T)K zycv=&LfXECg$j=dLJTG%3NaO2W+*1oCVb}9UD5r9Z0BWc7V4yHUd|s=f=vhUHfLC! z<9{{G&!u_DGO=iE#s#M~D@QwQUcJv#TMN?6^W5LBYM?! z$gHS+1-c0UWs&q#<%a06Ql24E`uBu>AqCNy$S#q%Youg}9e2j<_FOcYtS@s_Uw#SL zaIBwO7_*baxX)e>UYna1JF>X#24m22oz))R5{_rjqu(tZXHZq^u!&w#MHiJ8p?<#{$|u|v8GI;#N~TC_m6!{3D4DU(J(w~ z8XQT`xCfnX0oC&@LxjCcxF2`71pR;ve6U18V;-*w;|!LA3F`PcB|}y0XGoAq?95KN&M1ddI-LpS z*Yq0LhuvpSc84Bq@zZTLnwYrX^!ti9jd-bGPpQEsj^P?3;jZiAUSb_C=UhBws+qj> z0@>vPG<$AICg(IaC-$gP==if--_3Mt$$o z&G~0{Inwg|a}KS^FQPtXP__*XFqP#KYKaOTm0X5)IlXRO-1gv1^ab-9uS#SC;$)vW zJaM~TzKJIUOnd+9l~d2>Qnqn@hVV-as}ou55Q|uId06)1SC|SaybEd9Rf;mqA3beX z1d=_$w~!_DM{CMCCF~s8wGTCMshw)J{VTrdzu_$o32;It4uiGH&K|2#=d|R81+Tty z#@H6?_L%MN&r=uLa$0#eJA0iX#Ll9p^WKo~p)SJ@5~_x(YUM%Zg|7=An)+Mx-hDHv zeA!D8V)&CK{^^4MXa3@O^~%3>z#ucAo8@5Z@vFzqi>Z&w!ycjw>Pak_HE;oJBnhzD5W&%Z(0Q67QgzHhbHxn9%gl`rcZG=WiO>xUw^rB zI4kefqUmJ?k_B~@tQ&DN@@?WyZq+!Djj(bk_B}*cd>D;X?@`wrtvRKHKPk^mHLOnH z6|H_fCuSV*8N%wRkb0z1NWU;%Y9ev=6N2~FtOK%uYD7Pr#DrOn$DM1mGTu?nLvWbs znk`IEbIRzKltFC)J_L2JurAJkrP&)X_zbpvi?FTiP1JUTeM-knVH!_2hl&Artx{`R zmFQ!uf-a4KWk8MLbK6t)7#@8~f}Mme?HThKeW|_jPGD#KtEbBv!X{Sxm!p7ZGA87i z@<$EX`5Cd6I39BV2I)Gtqwrl0>k;uJObQk58?X(mF9~9rV zY8I2XAZ*fg(W%6aw98}geg(>!1jm;>l|we~N*zSr%$QG`*6lnb7Zz4eq4Odh1v}oD zP0yU-F>%oTGXyzmW%O3j%WMld_X}?A+$%`qaMOuixwH>+I}me$J>jPN$R>$5wR@K- zkPy!nx`8bj>75ytuJ(K;)xJyjrHJ2%oU=p3m@n{W&@3)@nYAbKn}766}l&4 zn!A&$n!-`E4BA7JLl-7a zx6sX@!r)LIa;Tal;x1p9iZj%D$0wWSnKYtM&sY|wMzc{2m|dqcmOrz>K! zC_B4uzBWGN5XtQD)zLHz;CD)@up~-5vw{0+1RM}P!N`Wg=5Go2ob16H9vg#-5av9< zi1<#i5FY^gV@>03WE)&@B`6oHrmVORxCa!wB~gHRDc1ZrnBT0_hEo~%l(_iHx;PI% zwmz`XHIv6IQJ z`jazdd7s3S604qG2!b+8DrbZJqfLmLPfH9JI#mj>nQ`PwnLNeorP3s^HRrZW2ke3} zYVZ-_Wl>F+9_4`RWEDe=cw#T*z$^59FBokkbVMJkLh)unwYcRP5{x}H$`Jy+`x+nk!@)|7ip7o~hBX?#Q* z5kXUGpQd(~q`Uy3gikW%y{5>7xcVN0#My>)1F)=NeT<54*ZL7+#$V*mru%_V9XHrT zm6s4C_dDTJ7pp;MiYSF@4ggyZrvnAt^U2tyXP^KGMUdmrW7Tyif*`PG;TkvAIu3&( zNg|~Qvw$5&kamHRO#PErURDoK)qwoPY;F~YPo&U+5OBEt61?y%0y_rKr!FLdEBNo` z8$c>;QUs=-4*Mdw*8Q5|DQga{_Z5n;#0JgQgsyXO0i90+zO=}eZ9hX2q3j?N%z;tl z6sx~q^H0`U2`pn0rD-zFKW_Lm&+}Gn18MBbBQ)Dl2-aPPaMT7Y-E9@khi>PC@_u*EiHHntU_SiCX5M0JJdZl{Jeznz(@vM>S{e)8-p4Z%z$kFqPJh7Vh zDSTf2L}?1>Ol?7%5tx8-YL9_6oZnPXDo{dPrl97_8i{(r?Wo{9&6ZRD;yGLLH+T` zuLefHUWD3ssJ6~`(#4jg#;j@0ga{?8#<}gk&`)K$*Sr@=Fk_Jp?55(S^II?^W)I5QsS3^Y{aqr2*#VEwa{+LaKO0~Bt2K}t%fb$2<`C= zKJ*PYSbqL{+sOvi1F){)d8dA;dV5NpWPdMjHTQ$d7Sw?qD6_flDT6*I^;_keUO7+k zxH^AO`3&(2TmeD!8!OcF`stWVCfo9@?^sG5vXl~RbSz3(0%clUKi z4lsC}r>GyLgN)^(2cr{Tlno%8t%ou}IMmE=_1I?!IqC*}jv?He<+?17wQx^4CBCFS8m~)Ap&NCDiDs6QF z&&sXvwcJpd$USGRGPdy?_V$EVfD#?I5=f#-#JvLd=XuXpy6 z`}ZFJwDJDl!oY^p+#=Y5moMW5QLnFV`l3j$A5Pguj|)NN5S|V#e}*^ z8mdUD(+X(tW#MD>7CBkTv84PH973w(EwT+1o?f)n;s`UVt&W1XSAM#HqS0Q7_Qb zT`9xU2&z)rVt4T#hJts9!3#VbkeLGYO|GzNMfCt#&c_pq@VSo#C`R6T*n)x_$BXx=>rwvU6fit>=@EN(pkRM)mG~6Xxur! z#ZUajd>tv$D@i+yDO^oN4v^gK`3z|c0P8an3*=b)FnPIT#%9thCu54r7yZfa@A*Gk zpOzNHW%X`)`4YBh#Ae5dYrBV5&X<*{6NlU$p@5>I`ZfZzg0ks_jNL5~SF$UzefaQ% zGoyZr*C{J1kaj!&vP$v2>h|AI^JDolFXEfSa5Ze`PkHG{th7uUXoekSGKzCN`i%P- zZ{46=elVnLgD!VtqY3}oBl5FOtB%&rqgNT;c!g|RkgW<{y@1{N=FJ;`P`dwnV~~Hur5k1F_<30SZ)2YS zn-6@)hWu!K_o&z8`XthoA67Dj)u5>-N>-NbuUf4fJau=rZZIeHux5ejP2XGhY9WqD zPF&}$K{8!)HJr6&Mm$?kJ^XE9eqo>zi%R_9!6n`Z_Xri)SV-Fjx7?q;6X3dJbh)gj zDuI5`Zo4&V`%<9zRGGSVMCa&ee(<2U2~!2t{=3p#iVzRj^xQC)#}Kpt6 zw3az5o}lqZxxscAVbPcbM+lQFc#iia}ye{#9&(J zwwOCT{eIWT!nIpw3rBovylWj;!nW=4_R9P2>VF(5`QdqeclAG7Ygx?DXUGgA4mEhf za%>27XSg-3gWfBmUpW0$tl}kGuX}=rH4;nZ*$1!T<+)J4rB|`xf!yqEB-6ouGX|a0 z9s3uuMF!tDupLmR*|hcY#itOTt$TMWLazU}IC&WTmo5Tor5aF9O@Iib1@VH7 z6vtm}M}!!BhVaO9qc?a)zL?f;9?1W&X@7V8^9SHqzc?|!`^Ufe2fmMo{os9}At+nS z$AuyFnQcSOI|yyq4oXC;WsiO9?DX8M{w9_)t-OIJUL-Ds?B2szXdfRB@yM9Y!+Gcu zm~kiN33fwVXL^X5c~Z>%4OY4pFU}opcp+B7#B!K@?=_zrN9DN-|G?qc@ZMc-`*@0i z&JJkNMO*>O3s689nF_FqrzmfX=dUL+pr@(qNI)2jyun)a7x*F`DB{IY&KW_89M=(x zTD1@xU^}GHBkxNYK0^$hjLFiMfC`@Yde`bs*| zuTJvU!g*gNysm(6!IR@Jjdzr9<)JAnx^{V*+ zYHdP;1qCk^SPm=f4m}|1l!?->-cgZr=L9;Njgf zjj#ByuX1x_hP+`LtVVo~)&Kv9)z!)Q2$Qm_%h7mNUr^bmCsfrdwL89MPVYO3qA$2m z{YgCVZ9_yBeYpb7p0Lx>kY(fU#=#y>8p6qBPch0zN=Ka9!f0_R)Tezj)tnx zUL{$-hLI1nY2VNFQ2;*eG6XmYH65U?*tFqV8&b`63ZOz~5Kcs0EB&T`+ayE2Owr(k zG4(E{)|=O(0sjH~Z03zEwSHF*9eK4`qhL7gM`@tnohp5=ga1l|@SP^@Pb*CRJj(pv zkF+@ zAXIB;r2IxWPd1q)o?*348Tbs$v=(s7dk9m@*KsjH@O=Y@Mw2Vs91^d;Zhqvn>5}q2 z<&pWCi8aL{~{UtjvM(gk~lco!y1$;AKfz>e)UN;r=9ZbOslDOof!2ds!Cbnht039 z`1kCdnHf?}4mz=azu(34harHleMTPo2dYaOcS-A%T;Ob=s{b4)4ju+tumNd+kit0$ znvA+8x+1RRRYp9mQqmf2n}#x!KI)~p7e zJdYk}_;gh4I>u-9z)8Q6kgH%a_|OJ#<$INTJ#YTks&c#X&j=GuJmts_=Nuog)#$@B zvnJO{g`K$4ycOrOt9LDM^?8d``greferaDPFDHnF)*923tO*{efMU$Pn8Pe9O#AqQ zs&m@|rz1{JZlj8d3ZU_=qyf6C&KpKc{6?C0QwPH&OP@Y)?h0zidrBHl$RsuT5QDRVnK4u4c z+^+32WG7) zpU2x>IXB!)Z9upu%ZUhCJh-$fFOUXL*;;x%B;Lb{XGBs(NF%X&-!jk z|N5rXvxBIItc^v?$UPe$;0>?KceT@NwFwQ1nchDuF!Ro@>i)oOv1o^_wl0^$2B|yY zTRiu+z-dTFS}5!mY;72W2@`FP;&!1JrUoA6S@}WFck*1`)cj8&a~5B<#n#<-zxpBp zGh|M>58a9Sc*2gnwQFGj1>mrklnA4%iNe7YakAdKuDh3i25qCtAH=M3iifX}?|g=w z*wqVab>%G}qeCsGS}GClEh+UBlJ|IBUhhf&Xwv@*FoS5M1UVdSc}{aVrhSPEQ7si= zV8zWJqx}-Uvp7V7>$M>}t8=8YR)_NOBWOWC+7NGK`fskuubb8H^~+z`z~Aea|LR@V zS)@(UvwI7JGKwGW589&F@AHIvr2V~p2Aja^106vHQQb8FDZ_z;jSIC%<}P=~WI5mY zbd|Ds1*N|l@3PzU7=Po8G{$0zFhhusIw=j}SRNld7R4MH7W52CZn>Bfx^w`8qRnZ^c}tWVCoIJxy)VDtMe3dcSb zJg(Bcz8w2*vi-iP4#rZyG0s3RIpvoB0S`;b_(I|vQIwgSMEhrmxoi3m-^`c+ZSzP~ zdQrf|SaOGRnMvZvhqNfA={KWJ;oZ%3&Iem3T(tGQjZO$97~BJdN)=@5>e?IJrixOE zb;z)0d>dXmBU(GW+RSYrADfM;Zgb=4jC3;Vu2{Ib7xsAEdIayfM{Ucc2eG!IO(>mX z=4TxH)3eNU_WjF&yiSm!oNnTwpiMqQKKMD0qUQ=w^)RvkhX#e$OJg}3r3BOcnVJ02 z*8S~ksBmb~t^kTk0MIU!6Qx8JLcExX`%n!pJS73nYBPFXoNDS1V8Fh)6ve-+#c$r{ zPw)E6UmF$0T{pP=uY2*k7{cG95)xpy@=0pIhq%U7;$5#10w|0;m#3@GF|G1zEU0;6 zL_-IRqw-yd9;je%cmg?umk=sCjlr@JQk{5m+*}}B%7;rNt6X-vy{24W^{`KHo@s)3 zY*dGNon1mB6%!rnRS*-?b)>g)XWzVXrIu2$#3QSA9M0)b!)Zb1#S{1MYmF?*dekKr zXg|E%Y&u9(@H$N6_3VcBUR1XcJ@Qy?bH1%ynqrBYqg&#FT)CzhCk5Z@?Tse3HmMJ<_q9%db;?R1*UXBp2xnKJ z=)<-y(LUxLLLaFeoEkh-a_qsY21it9<~#hPW)y#;gqgXHz(4HUmS$hn$n{pl)Uqgf ztB$P!g{VZb2bIBlX|N%s4wzNfqW#jX->i%O??Y~op02qRIt4tn0$~h=H9&115623N zUW!zSU{eVtIz@fZ_UOnJU>;K_0@ z*+X=KCx-EkKbPJl5hmf|>K=lC39JK)==jKl?=S{+y<^Y2~FLYF>3wH-!19G+?|yX5P4M7k;S z;);L^V85!Sc(})5pC8SgliTmbhDB|5wX0nA={nJN_4a27%R8wvWBuWXfHPz}p@!#f z=iq0Bg!^YPnUs688r5w{*)*boo|ae?UCp&Fr#KIjEvHP2l#zjegIO#5{b6a7@kj~4z|W&KMKu) z896ndvCQih9!%kKEvk%w&14JLuvhgdF4SOf``X5DSPvhS4d=T5_Tv*h7yVo`X}THy zk|T|FF1g+Z$wx@6Sb^ST!fo+J}cet1OX1eO=r5f5q>wV-A^R9lc`U&F~BgM;X$J3e=dsJP^x>j?K zVycI(1nPv5?<5G}oPCGoBnoD1o38DKU&^ex2h*~}s71BJn(n?d6rNX8M^4ll2Q!6l zvrp_AZWQ-?jfY}$@&_4Cn!b@@Wri(!6kZZUy{IM_^saUdJ^Z8)^k#Cty%TEbh*Zy? zCx$1~V9oub68hi6OL-pr&GJclf~h+mGmw%P=b^?=ymr8>^dN@ADfdG`X_g88gs6>9 z3D5d%yG1aj&;jB+-s(!&GIz{ph)Rz)JcxW3J<<*5M19=E%t!>cyf?xjD+VJ&D z&~Bb(j}GL%qFT>&&BKH<&0pXO} zgLkMGQV00SW;}7N@v_tjUxU`JMHEj7aUy7 zvq|g(`SQI-2pKiYV?=2K$S`8=8O-i{SutbX2`h^vw~%-+xov4ky###i^^=Zb7-vM&a$<%lVU1go|H%Ea z9(|FqSPM8OUOn7NC*WMstMXS@%Pmt)PK?YOHmlc_AKx4^d^C2JW&ap|CMxs#V8)sB z{n}w!OWFL(rYAR*7Y$wDr~HAl0#W$_R#M4IK)VmN4A0Dq@2P$<=~~$oLwgyx$#OjoM|*AYy^8ZpaB)oapeg zSWp-@EPna2tSF*&cIzj9EdgGKgQli(3baphV((-1cRV^3Z!|Ia&_+P;=-Awy#La9< z!6+8jGklWP4@Ium^D*|fb-0f>uQ`%3f{&e?)QFaXlr@dQMi-0>iS9N-!#Chf9%l@k zW#dkDNvQ9bJU`hZ*W24y;gExV6L#B!gK(Wsi)d#P-XXdX8S3uj#(?}uy-%=5mKP9AWSz**5N|JTHm1D8 zXszNFYZ=M7AqGh=9-TtfI|hC|aUL7`yA9kxrq@{Wzl3AKwi@(aaF7DR#ftoE zvmDu8YohR?3&xSMLT|%)$;sUm^YE@p?kh{pRZ!;IAr!Ul8Of|=WuY;R?z@*^wKq^Q zWQ^a@T@Oa@V|oHgCX;c8K5qAO3 zIgZ~yJDO89kX19D&79}cKy@bdJ%W8LN*=eghfApB{&S6jYmp0RgYuj;n1)}dgo=#E z(ezx4akv!W;b+JJ>KfSc&N$E)l_0GE#K@a2Mv64jywL<5iqJMD=nPLKZXyIrr#jn6 zP6y@;HFK&fQFpjgu9i9CKDo%o>`uqbj;6-MptM#E*>2a!%Be1TBx+4^e7cu5VUvxS z+*xaq7bH91eIfN4V_5M_SRpag(-y{{Fl5zq@PNl+QH;mp%dwWXsaLq&gKRnCA(#7{Cl3EXWccA=-@Nl{idG)hwz;29wSLD^2Z70+aDI? z)NAxpIZj7jZ|mzjd{N_rN90E+=m_z#zLY7rvB=}5YD#vTwXb4&dSG|9-coLLF4ae% z&3X>b@S?vl;PtC|F+|d5$TjC(#Fq*Y7vA8W-FJ%XRDTT?=fm1r5w0uPw?&he@G&8a zb~KjIXOnH)=l9)GZ$A1N;-s`7mzg5fgj<#5>dGOrX%j^|7DdWjuM9WwjG%YBqIK{& z`6Bt5s?m1Q2V-VWJ&ICVZ9?vfHFFK=-HAUIklF31`hrpjw<@&MNt6wdSqSj~khg9e zpoNK5A|q=7#D{LC5rqLEVHjfVf_Mr^c`ao|pArUt{+o%%|332f9hx0k@fiYJ$_-!3 z$-ioDGQibNQj$zB!M0!FzR1+a?jo*OiaDZHLbLbz;BYL2EWSaI=65d1U-;&~yKhfJ z0KL`))J3$|Tq5TT5b_D0_m;C;1|;5*8U&Be5F;trnvMj-qj19`Ro|rA_N&3iE+;O{ zv#r9lo!L)%y5)8EMhuyT(sTO?$u}$}(I0v{$+cVT`J25XjPmrqSuX#TtA7Q6DJXefdX5FQ24=AmOt#X5k)3byYJi4ZN^&mGu zDL%iAS!2u{EBy?ioPHIKZ(M*I>>^pNrWrZCt}3fK zBy~A;tLfbEnjwp7sJ2ZNjBXsA+jvq%D}E@2Kbg5Yfuqw?>#0(jMYOv8@p=0d4Uc^8 zs*oMjj1RRh9&Y&*Ns`4}W0^QpP<*)%o1Rs@{Nq4y0majnpmTt}wm;!y*0?@8AlF!}vtTuwU+wAsdx`dwaKYYqiuU)TyRW(9G{1No#uMYn#%I?fve$Ht2@Fg#O=qOY7#6w0|{?pw%VdS*3sw zYh*oA0O$XDu$gYEd_Qg};n9K2UQ6dzLr&|u2hq;Khjr~vGWplR&zh$}yp9le6HeBb z(~#pm4`9QCb{ZO3OFq4UY;^2RFD$Ygu`sb!JHmcR6zB8**!%9VruuB%ARr=Q6e&`p z(z_yvv;_4NM5HRcL`9?*={3PZZvp~>0wP_Sbg7XpO?oeZ0Ma3$gg^>+JLR64d*++_ z&CHo|=H6%i(B}b?U4HwwS9#aF-o+OA1;jJ?U*y67C;-w93(~-dUd0qLO>wLAUwOt6 zWfkH{T?-!ar}YD~W7pd6A5z#6+oCh-8$f;(wyT=-_L0&@{Of6iV9!+E<_TCl;ukN~ z0VFj=Fek)~KoT!BrOF%69^;;T7ZR+M4<2$I8(o{?Y_a21dw4~1KgN6wX(Ks|5H(+_ z4QM6aAqsDz&gBA#4luRxdFFxY6OPEw0wW7i!8Nj<(gurjq3E;DWU=kQ$~;IQ;OZ!x zDc9>nTH1T;{R#`0je7&#jijgbM@&reJ?52g{cm=8rwlOH>v(u23YIaK=K0BHu#Su+ zK~1-;F8-&!g3YLjJt-+;I?v^HsX;+8OScGG zdFiasJcp_eZ*7TJUnD-OvI%U#otdy~46 zDz+rtV53l$iG?&vp7q*ZW)RY-4};9i+(lg?g0~=#4h;(9oGFqDdA1^bCLQajo&(90 zNG!n!1F5ZYUD-PB#ioqC_Fl}*7nyR9JlFOlzP|U-#sJE?Q_56AU3GkJq}W>tO#+KY ze@G7w;<*;uH(KOq=@s0}!4_t3+-x@Zu_iQ#>QS(k`=f8LyL0V}UrfKpWre+nB^hK) zvkzyS>`Jg*>I9~AUjgRho%9ObMT70^kjLqttoB_)%spy`Zl`{h|KSB*9T+V~qH|N7ZWQo7JB)@`{ z{VWIW*ilYq0GT=%C^ymq_=Rd8-EvM4Fki?FP)=U!1d~~Xs8#;XKC)jw?ccjDf9v;t zHwr+7gf2%#a=Dm@BwntR0rd@Ey~Gx?(B?bbLjEWrfY{0);(uNE52pY3ommp=!uMN za@+*p%etVbNdxfq-ckm#Dy&B8T&=RZ8H9cN=ZJ#02JP=YVv4@W(ooT>sK?PFXXW0ILS3IzB*J>#R3Veg(#0!78fAjHy5p)JX0H_{|x_ zo6%2sW~hK`pCcEdmhG|o6!0dJFbtVs_Pvb9f(f4B4WNblmQ^qciv(|G)Ml%cw`i zlNn9A^rS18(o)sV%0$e#!x-doo8o=LLonq$Vul)E>)ARHgjClHeYvYzGDl^^rDRXs zZ%we$u6Voj5uHBMP9FcZ@PuO}M?VGrd}=@nKt?AZWo#tocXNX63ij7gGW^IR{7zP&t% z?nFsq8klUlAOTVDIz@NjpNP>A?&mXwuw=!X1DjhKwNcyOZNxVnrW74t<@27N?;HKG zyQeAltZHhG%orlZ92;jOvgwhe`rnA;g%sn+Xz$}CL!3Kn0KfDUm6h~}$QfGIP> zHP2zV$aPIFcW4|Mdjt|(EL$1dcYaG4HjS4?Ce0Ai8Yw)uz{Ih=4J8R?>cwx@V-@%H zxuO^0_meEIE6${qwch+mHvHZ>Bs{90tO##GmYHWUNYAj1Jg$z;4O+H$R(k7<(R?U;lir>IRo1igr7KI982W>SXGSZQpwCk zo#5J+nTt!}Y*R%5cPTd8h9ye|dH-9T2~Mze=IT>3i~Gnj)~U)8zZl6J>LMjg4L??^ zG>aXN5dp;3b|LS+`@z(X-DBHP!c0np1LW5594dL5+t26P$_?~%6_jXOxF@86m# z1yaC~sr2)`QIrcfsg9iGyAygZEe$gehuX-x>EO?#9t1q)S?X0O%>>B48>vBX_0UJM zKJ^;!S|6g4 zB%QnKB#Xa={bWVf+ssWnP!L4|zux2iyQ^8wiST2H)8>RZDBmai1VqHCVe9OL74zVNV*&hzgC=l(y|1C_GTKk+1~MkOn{?^GY%Vj}764`2*ESDlTA zxA2_GP2@~n0PxhxsSZbf%i8@&Isf5L8VpGj8I9Zguj-Xl)s@xU zV#e>E6^pqW{Gy=X#6$lR+3z%$eIchQLvXCd7K2y#y;@x2I{6V#Wby498#k~lt3gJd zp)6!dZImHR40}1nZlHO7k*&)>Z)zsRqdevog`P}wCd4&22vm@hP}U5S@srQUiRre+ zA)>C9ZiSx>O%oRK9+fcoeH*(KFIqOLA>&>$gm^#5zL>@YrL#eZ*vKHk(msgncGtY~ zK^B{I)6n_tB^fN~M&lb-R@`}on;Fyoi4?6K01*D(`jFQEos&TVr&b(+PT5gd>g~i` zHIfJPV4{}~Or}lJA$Lg`oQW8h)zmH)I<}?J{&k9KsP3B;s8K$*!OI32ujZy_S#jTM zB4c?jR(FUfu^dL{XxZYoi^RKK%JXu1X$RmL(+JUT%jr@1sv4t7YaFl*_SvpQolg_y zD+6bDEPu&`o$VmyZ+@;rCJw7T<`sg~BRD8@S*O0gY2fQ$Y>>o)!%6BG4b_%po9o5P@VIAo_4Uj0-?OR?J_uLkI5+gKyi@ldHq=rJuc6E74M2`9Y0R-WnfXd zrCuLQ-abLL8*aw(*ccFsF7J7gP~9_fmNtI+s@`*3bzY`Ob2Fl8Lr=Xisoz zbYjlg(C?H|D#uCD4y1w7iTjZ<)}l=C)#Rt~Y-xF`toV{6kdMtV@JLaEND!;y-j9!U z6Sv;X(|RlL_+liZDRlketvQMl=m-o>h(^c6b7NVF{vhWfIceB`Zzu@X!2tQ50I#v_((h zTmAfx{oXw8iaS%CgH!jmzb%P_Z_Yh zZ{e@w0>7E2$Zx6FKq7aq9D#&|sG|XZ&e#?Z8pIm}jd~!<>Tcwk_+MQFf}RR%>a~eq z>HVOhKgnuk|XOPSz4=t3lT#XA2oItbkNJu zJTiUl^IVz7897^sE(_oT(~{JnN1*Xa2q{UKj--UbI|Fc=C`OCIU=KTx3IG#)k?Vj7 zzG={sMq(X9Zd-yNSUm8BX3DGLP#8fRP*z*$1g6phT$i$7GCiCLi8XlywDPdBRV_W-}qB=oXFJ(n*Rsxdh`?EBcd!pl*})i?H2^ zYq7vSdtL)fZtLq|Y!QCuB%;pzi5nm&mBzfj!F5K}2ju^k!&HO=vhe|OA)UZW`C4`a zT0R7knXzb6cnzHFed7r9u@z+fcM`ULl~(l+d;h<{ocasLJPwK4gRR4<3BiWqRqGsj z)8GBC-RvO)1)wy_iSew18F@l{A%KZTn+Y_HLR0MrWGVf?U-zK@Z&XUaom&LqBT&26 z89yyT-bYF&`gCUOHd?c~lG0yQDQCA~?svlB`}IpQZu8gCyiY8T^LN~qQ+oZbvfywY zZ%s}mNfPBq*DJ75kzAe}6$4pqRe2i+TaNf`*CCZ^k?=9suB+#EzJzTTL#?WAy~v!J z?lmen@hiP)f0bJLw^p$J=7RxS_oN2#ip~DvskCyf7xt$(8j|}3Lq0xtmT(~aQuB4G zJ7Qg5sqe+Jvb6hDAvQ!|hrX5c`{w%oSAotX0y$FxX07%7vt+0wq921kA#Wq!T=rBw ze)64f_I?c2Dc?F59x9eKZq>n|uX&$rgv@#;wV&*{uZak zK+BE0k{Jd9&XKpf_}#ptLr_*)Cv9Dn+CsgbWhuBWM$fcB1GZuEggPW>wuH&Gsb|o? z3Lb?DT^jBZ?lM&1I-N(9^BU=YyE3ZJ_6@o@e##z^9zeY#vrR}n*1ZCozTV?APVPbE zFmHGE6O65LyqqwqFBJvT$Gxn~M=~mg+`j0KtX_(kQwD{Wrv4{8)Tw(Z!IsjkDQQTU$DK2HI+;V__Z zr-}CPN*Cg3gC|=Xdjn7cXN&O&tD4UzX{yby^%mBj7B~XkA97clSkrAtU9}<(Xk7Gw z>DmG8w)~K^@Zj}uFc5MKwR_R5+KIRU!LsU9RitZG)}%vdg^h%#->u?Vw4M#>Z~-? zjunYX(8MJA1w{(Gxn3U}N*Pn}5yOAZB|A*+eE`?y{>rNb?kb*qhtI)nwWWoYQM%yN z6N|fFoRFmztfh~B*s*;=db8s89WWPPx=;zNw_7t)4f$OJVvlE}D_BY#?NV#)yEJAy zsS7q6536G*+K#tR-(oj<(nz`Vl?(jF6SQ}Q$V%xJ0YT^syyo9c0#2jCnhH4=j(Q3? zjZ#XOtVZ-z&MC_BRDG1uiOJ~K0N-zBwr4~ zB^EAhrLE=eD8J$OhH4RAD!UJVUL__G|Cs|2!+?uliK#Zlo=vgP#IX&QD`F+`*b8oY&628>?;=3B&y`1|3LJ zP7Hoj=MrH#@Emf1L`fs+W$_3|{TGzn+a9WfEjHGHCW zDnw_9{2)d{md#CieG_4aY?PCUgwG~!)F2Ac1LaN zKZ9%0ySA#iD=}{tA?oxZez%)`Lo=AG%)mS+dvB#?Uc0YjieottlenTk3bdcz< zWo=N|3gIOJQh7+`PF2JUtVvziiLFjx5aSM*LLW1`rU7qlkS0XuMMH_|RaGSw-q5Fv zE(40aaU{b~6`#4u(bwRQBL~LMqXb@Wh6w?2qktsyB6*;hp>5U zDb+#`%!DWy1Nyat0-%TFw~ChD^8h%qG`AJ*F5JCeH9`BK=FR&g!AKexxqXQUEr1`W z^_NI{)H|G%le{8cwfGz?>Xp5#Vu=lh<2$~)QhqB%KOUlVW7`k<=vMyzIZS`!tMh;E z*njlvR67PFW{VbTWpa$7ZMcwHh3e-)OfSxEcWAHGRXvN9=NOU zt{^s=#NyZ}ZknFfX6zc>JW?I6l++!1i;wjqlAl|tA{+pD^&F|iitb2B@uL79M$qYbK z{iFvx4jE}h$zO!*U@je|fEWM#FOby+og)zR66Ah5DHk^P;CJfonqAvYxD2vECv2(Z z)~XvUo{l6E4alquk0UfF(#TH-sw6^)zh?0E2cWK`1E{(ODoA#q+M@U1LPwyqv*5fw z$^!`Y6BH0w&#^p^CG`LP^cVN!A)PNNEUJLMJo7pVNX9r|!+seQzh9s^w+^(CO}Rhi z<8P)UE}-XC4~yR*dqaYUmXRW0f*u6Walco;ms?4goC^-H-{h^S zz_z}3z1i45>Y*2WYvfZJlgVHJj zJ&kO?J76jE;LuUZH~eEw4i%@xVCYi7}scG^#G4h=RX zztbC*R$mikuX5PlCa?JU>d5yZ@nI7+VKN-x(yH?7%@Uw*#-RPABo5B5a*VMG-RSrx+%sIv&vIyZeu6-`8 zotwrE4OUcTT#kG&ZKOu5t-S_o7E_I1x^TJkJ$HdmADu6-W4)s{ZUM0LIL?65(gTzW z?<{@*zW@;IM98)z3D|+fJqkx4<>Ub9;(u83>*!F(c|fP*QblP!Arh1C&&sW$8aC9$ z70Yg ze$D9d3&B)x*6Z`So9oV>GGf(~fQM@4CtM3D&ml3NAz(V$C8T}L9&6ZGil!uZ2r>1K zd33egUm2?hAu|5L0sm2N@!x}*LXIOF&FYNEY1p_%a|Sh!hKk}2L?>91yWmu&zpUBC z!{^EnO++*041f&r;x!3Y8RsjN+Cq}^Uyi!ZSBwHh6BJ2ao38q=zJGs+UjOI2vAmWx zB7wb7jJr>1wL1a;Zfoj!NdjLlDQ*j3P>($YN;SW5;C5h9&|nE(H~%^FM?YmhO;(rUM!(@Q0S~CHIyALD0H3X3ZLX8c9cO?D!b%uJF%)% z%gAUA(WO!&TR)O`K$^VnrRIB*VP+MrUX<$Z>$8hRGI^orHHJUnSgQ*O*%QsWCqu7a zH6HIi>3rsbVU(U)QM?=^(^VqG+RgQ+yt6n;EU5s1gdVWNXKA65@4`WF_DKgiV4^4R&xW@CCjQ=A>d zBb7r(^Dw}!v*k&Whk%5PkgC~)8m!)kB4{aw5Q)>2;Ysy`yY?NX&4M@B>~Kn`IaZvs z2Oi2FcFl`aRw=q`t`@2GEyf_>-unPU=M(UrFX2I1@yPN=TOwLp(ZzAg6fG0kJ)@_# zC22KznMWWP0Sbp{ITpt%z5d(Z>p#sPe{;b4yX53oEC=Z;Asi?;!UtgMj@VI3tkG!S zL)7#<*gL4u@dPy$SLjgvaAj}$%<3eWG97bhj5MrX09n$RC z`~Z~p(U&FiClLA3pWOxZHqh{L4vVC)RJ99}?q^RN_`sshRv6xm5SckfJK(P#|KI4)~5dktJ!Fe#V*EzFF9PDMq+B8J3+@WvU_^yTf9v%0CT@jZPvoD0GciHfOzkd<4 ze25)0ors^@*WvEtN6>d$$ESB8!1C} zVr;D21u=Icq}YwrPMui3(F*eJ{o7p*Ode*F|1Nq$72ko~Z;X^Mh`ZaD9j^X9!x^1>1ZKtET678%Drc)9+_JEuXv+6h_PkM?CI3*=(+xPN-_1lNjgG~)r zusKm~=|x_ELqk;LBym7045!5UolG6gI`4e@Ha`;}>@|ro01CfB_!zX;PMf@DAoKVc za^(uI&7IY=YEy)*at_cEiJ$lji4pfboi%RGDa@RwuHj2u=Zm4~g8TQ5K+|^?CcY!5 zj|V8R2FP_#c)gBKw&J{w&9})bBKT8o-Oj9-V94QQSWr!AU!<*ZWB+j4vkg+N^>vHN zR@Zt)l#upe4Ba&Ow1N$4Um}~4=^F-l3hFz)^#Tx0kt4(~4-mj@6VO_`GlyGgtr}kl zPhcN+SDLdBb?(mtrJePo!~STMk>n3PntJS`T6wk7ibX1k{0b}O zQoWv?zl3_ZN#3nvHw-(bTv%#7q^UyZqod)+DV(d2ah_|0$d40}f zS<+Q1_u7J16-6PVN_^@oGlkO-iuFhxLB*>+$sM# z^z*{$C1$D-JC2d)xs7Z77UskDyORu-Xr?zGW%8r-p*|@2;zK_p~C+InPl?_PUo!5)Z!$Qk0Z=QgUqN5yckA^D@zsGvm`L{4X#w zda=UV+dtnnaIQ4f(?4dCP#}(b@gCqR@L=1?5qSV4SJp>24F8NGCBvJYh_o=u8+B!# zt^bURQsI_1$VBaNKLu>$Xd{{q-7XoXsA}7{s>ONCZIYht6el--yqX^O&DbeRdJsT5 zqgf{~`eS?ND$p&G%r8P?}>;0 z;hZ0#hiuf6@jU{CBFpz*AWkp4c$XQD85_1j%&H~=+wb{)sQdsP;ikJ?f}IJ>@$$W= zk>F#Nc};(fJeHvx2IFU@`b|^#{%afbvI^`ttb%vDQml;XqXs(r6+5yJVbGVcY8OCK zz(2K5ulRNEUNNv=YJHqLo7;6ZBvFhv-|3JTJ6zA2!;hWB09hnOKxQNR?UV8$d$aJd zsn`70vX2C`<@0E7S!|XxeiP{#4f{IW#4l`{28W_YjzHeC*dq|jtQVj#sIzZ-b3YuN zaH!^KVvc6$U&bs>8=T#3{G0Z`KN8wuM^N5b#(plSn$Z@A>>HSB^IdheUGKScNh*oJp%2l z7I$b%*(!%Lz1P()N>qS)wHqPT*XRj5wt!c@cU>n@)6|B2Z zZ*snecq)994CFS_4S+#@7t<=eoR!_s!ol%~h6mWnV*CdE0@^hleN`zliP-&8F0^+) zj{W-7`BU0-)ACyz&#+baSLC-8p)u|RWj9G;-{cmZzn&^B z1o`a%CK^B@V~`jy8DWDyq#2^x7ysi$y26-9&vHmI1mLecwqBR5J{| z_P zxxHR`KPX@X0?HLu9N1b%*#yWKeq(i$XFQLm$BnY?Js*+))4ZT6!&>3b5Lj%m^QvEP zOF{4!;~zN|jKdp4Rr_c8oGu-|(o!N1%71_SLdjaHGYER0Fo)&+nnr`Pg)kCCd~Nn% z$L&hT%-_jvaz6gNbKB?-;$qYeu-Yk%xEFe*IZUp53<0iz-gWJSCNXsLmFQ!@@1V&OH z%Z3}iUFrH&GP@JsYWdceqZ@_Anx$bh-{_cH_*n z07GoYh1*YcPt$Wg`*wl4AHdPmB8`_?qXXHDWy7`v>PP(eT@rN7doISByt830m*nM9 zk1TqXnoh+RYfZfvJ!!tv$_lU89q`G-^cgLh4(@88L-D=E%U1ZR6Fo?bdW5v1l|s2 zAhge=mU$gIj5w|9{?HhZOqjE(cpX69rZZ%fclySH;{`|-LI@WWJtukGHdSFL(<0l( z*sbP$;rT9UVRqMYPrlQ1dCXJ?lh~_s0w|)hpBB(50}tnz-ZiF;Y)TKB(Wwq+>L0&( zr{$T-+bQLaDOhMy09`%_xYQR&lGGrDYqddupv9AswdTW}nb_qZK5u%*mY&>!Hc%m^7+^nGkj*p?M)xQ<);a`^ayBp*I5`0|!Tm(<$@(<6|Z zpkFfRU>?7KGYC;UM>JlU5y2KTm>1{Ruu2)B66*?bdbDruD`J&=0hRIIKp0jFa>}-b z1|dw?Dq5>44s(Bw@9Oe$Qmm-*K8`7sJGI?Oukg;9{aa%jhH6hTh{8s&Xi7VUA!ubX zWcJMsE)KG}p&Pa{PjPL5{MvoF0^~k`Q|f?13C#pS=ELN1Tw6tqzJs#ID=*~ z({`^;O{qUrO#qeP{K!g`*DjW7n#pxE+6Q!S4j+PJ3rE@>fhK%L@xIuEV)3+KH_rCb zFMRqXSETrY}oKKtJokcbyuP9pLZYh6G72B#XNmu&TPQD!OmPXhB`8nW5DJL!5 zb0?<>cC*kxnYT&zc;Q2@y_*#sU!Y&-^k?$z>{ER>0kedR*|kt4Mhg*F5q&Y!Vqj+( zPJd4A?nIx6%*}J=PK(-A3Y&O9;6f}|F>uhM3a?LBojZ(@inB?gI^8Zb`dn__$2AciSD|-(58{&oHaek$! z+mP;9XAoDcGP}U1GjA3$Sn{HH&#E)r)yuyNXsxYb%{)}w060afJw#%CX?QO`)Lxo_ zwKYC!mM;~s*~$oSLo4d*H&UO~_x3-!+}G5ew1wVPoge>U6^^mL+uWz0T?r?;p$e}V zIH2d180HPy7}eYR?97i1AAw4o9F}BRQ}5|WO{g5F_jKbQr3qquJsGyqOj}fB`!;jA z@cs%JX%?|+=`G7}lfClNYL?Q2H<_7>MPVXp3aiy-gOLYCMZJTl@r-%K;uhp_ljRz8 zsMS#2qs-S4##wh%VyE?#Rfw)RL`X)ln=YZ)!-$JUawDE-XGl|u}wOA zvd>-r^fGMUt{L;vbKvl=s>0EgK7MtXgVrioGhm-On6tnUu~ey-DOq;rj1Vgy&vY@s)jw zM)T@f<+X#Ge%D(sYL!m$jk0o^hFjq)p{7}RT0FzP}eLbkz zO+I}t@=jjkw9?x|S^W`hrQyXE?OPB@z9JTbDn6_*|Fg=30)W%;s;UXKd({R;;Diu{ zUAjJ3&riYT#IeQXQxjU0dBh+KX;fW%&d;W84;UZyH_;S(#3 zUtRjqw8{B4^lL0TbrB*XOhe|&pw?rbTYms66;VI;96$vmV{w)|!i2Ip!qSV40;`I$ zFGT;Lb443ne5_4s785$p#XcPW%Ev@Cao(ql@YwZ1n|2mX4S|g5GNMewOSAT-ShY!p zflt1~JXU#k2Nia?+nGV(D+^nyQ3%N84vQ1Lk$>8^o~PHJZuAgf+e)5IGjdq1D2Jt5 ztX6~@?92^&jNIZt#RU;Z-$3~-KCxW9ZzxS>?2K&!OfIf>l9QqV z$K;U10CwM@P#;|z9IKXgAUFdkiG1#$Au8P~hcQwc+C7Cc@;Jxtj1y%8=)Qn)~XM%{dtzLP?`JBxz zw^)C0SgYmv&bOi8FZWlDI*8Xu_Kv9xN1)F59Drq5)Vo(E2oy8X0Dtx19C+Q- za~mN_>Ks6H10lM}CV>~=xauQG9?}2%V@IHavp_y#fr`Vye|R&Oy8mbhO|{W9x-yeh z8Y=^ZbftwL&O`RtOOHVPE2eVM#1uHTYuieNjnKCTWJ_3%oxe-W5A}pWyk)j*WHXvl(Zuq($h7cu%T|H>sjEK_~U*z9*FnRp(C~B2|Xs zj0VKqy*^qG98QSFtH_CP-vS|$iUbPQpX^HL{^1N%+n0fOF2qZl_AO>QlDsq zb0S?rPCy)^yLJ%ccqfIYQybGe7ILkgWEJzS33OHuIehU)^>Ru7>ET)F6ItLn;Xfs-KPa*MBS?99e(3USbk@^hK=FFu z>^7_kP7v}8M2PvB4>T|tl4OU=iIMu7zQ5Eb9LoN3MU{6U*K}ubzSw6g|9QNKW5&~g zkMlfsX{H8LA%C{Cf080Y#3}vgs^B1i3?A}w?ngeBcz?u7-cR-jWK&fa#n%$})vO?M z!G3Mato*V|@oBephm)GN`_R|e%38KCzblzu$qb$^l+2rE$d_CgR7%^5m}7aQXf|^Zc4tSxjb50np5o;k^`DppEOY;70xt=(Zr>R+>Hn z$!%mG70_2nPXq$hezidqHn$FlTik#Stz(D@)UK}E7D+@BMwkX#yZ(Kwe{8+D+FMX# zTKM9AMTSDJcWpd^gB3l3%BAo0tIIeRTZf`vt9);}KBIUwkn#Czts6kBpM!r&fa84U zNygv9u`k<7R_x~bPgSuR$d}_S>|3$8%ty~TUvDZtjPErD?Mo6KfhQ#?VG@95Tc8a3 z2*ml5QGN^G1u)9P0no7T0ABekm_6!Hs0K-4 z9z~KI0p|HXaI~BPN((@H|NcA!ZCBIVLYA1+8Z6AJtx457XYLSoeyFU3j8^^tZ~>vP zTCc2{WOXn6FWXly0m{B=vxo;&EI-i|I{2(saHhl{qGVxfWhdZLv!w6 zuf4?PXuc=sv>3oA*uLeYm3~!enh2KiqH5HJj?ZU8srZg{Ewk1@>+91A*KrtCkPYokbWV!yB7ID2xF;i z{TahgE0e$4jwURa7Cd8hd7W0HHvS8nwi^#zj2cvni?%Bh%fikIP0F)M3oor-#Z!`RG39w2EcR`0> zd~)hG8h-<)m;hS?2XE-0*4{~r3R12c2SCNi2~b8vKm2SReI8)1=nz*0U+vcV8>`#B zgKVhRKHfie{%xl|v!$H=?GCwv6YE0MV?{_y@Z^i_2^00=2^;<(H|qrz-0{p?qNmOv z-TPiD^Mn(*5uFqTa*BHz(<+Y`?sRDGjlzk^W@aDzxGM&cQ;T)q2Fq{}d(u>Ms0y=8 z;Bi)^1r#{O`N_&6xB<0q{kld-%m;A)UC_4Egu(oeB%+d^f)lGLumlqvDeIW_yJdp( z;*I(a182WQ>Zx5{z#hN)nra8~f}h*~_&7!xKZk6uEln&iP4862+X-=|JB@A2dc3hN z)!*62tV#ef(+_k-abt88gwc;K(+VO}MZbY#_ z`4Iy%;sXdy!it|T##_?eu39pA+xPLRgx0$=)_QMaO|J7ogq4&(fIp(<5@3$VLpdB^ zWKIL^25q@iX^g1H;HU;3!uSV2Q(SLL3Y|%s*3id#R7$bLkJ>N4aNe2byY+11Sj6Yq z#{tkEe=NTLC05c34-LT*M~4|Do$Hz#eT86O?YEg2RAu~{YSJNY(5W7}AGu7w#2ewN z%%cW4t&2oU;8t!A0&jPA?H|0>3B)*ZSI6xLL=r9n?AuSj)dZ=8lbOw6fDg1o{oV%# z|AeG(vjNPfrsS@1=&OnM))d#$&MFFYqPBqL?2gNXIGDe0D+Hqwdie&z&MT%0xmj{USf_)9q!Gg-JeRe8m< z%UdyqFY~dUrD0P|G3d(0sP(e}r+a_4)4zx(+-eJ&Y(jC(R$v=S2~kQ; zLhGDkuY;W)-cG0SKF1^b4HhofR0Xnfq>@7#NW?3AA!p2f`5q*t6KOINwp;_f?t zRsxf`-$BYp;^sr@43itGoh97l)WX%R&t&S8Cj?AWO5Z;zfn5hh(2k{M4l3G*6|oPv zt>kMJdoHak4la3q_mip|E*2}=s8hnM`CnW3;=M7`CL8m{fAsWJJojb!bA669dvnYS z$vL9X5hy&;ZWcRiGt}+uSRiiQ@!*ZFZ-r&qz&=sU@xZ>`cMkjl3iS%c@Fanoi$f%j z*V3d}N8zLimBt_D7UM%M+zx6!c6xU5TFJxglJN1qxE8JH4a?%#9a5E|IeY7DNp+Uc zLbb2nU`=9WvW}UhU~z3>vds=Wp{G1;5{}JA*?|ugw@B^APCyqWoPPu0-S(nd)B4Su zSpYT`Y8&A9{`qHAD>48c@ezoX=Z~N8b^=CuDd01ZrzTb>+BY~Q!`*r$o_?u$AxUp+ zEK|&-X%QL{ui8zo_<`?vc{N;1v{-%O`mL$Bes@&-47#{5I&gl9#pq#rNQ;hF@&mMV z?3?ti3BMAQl<{=^|@>iDXMhi%1p+86z=(`JN2Ihlec z+fF7r8=kG|;r>qA{Js>s+Mko4a=ykik?^Q{k$+OZsBovWl(KS|g}twgg9gs|C-1H& z?0k+L=;#Z6uBj#n^L!>ZK3Por)*vt{Pcuu)6;VJ$ur^|_ZxtS!#f@B9MRgJqAY5~waUFiO%lt4^gptUl47eiLChcC^P z|EKw=Eb-l|sd^zn9WYgj$5jd&VjNwr`ZdKv3rzTQ1k%AVMrO6gBhv}*I~^ZhXyfKh z)ONmtLdid@~#K!q4Zm+rqqf|L&VlZRIR?*1sow=p$Vv zs9{lXLJyANF2P~Ww+A|Ue*|#ImrM@0FTh3U1su9V)}=}=pTFP1O~(*h0f)8@iB8*<^#VX-pz3hIR1GT^_CSty@4`w&_u*uMy1j=?Qr-_pIok&9Z1MPfqXs6LSq)!!bIs3KC5HLA z)sPkF3H8fTna(8s3$JLLyT3(Oa7Y!4^1fpU3u5IOK!4ms*b}zqksE@<$AnU+<6a1~ zUiF~k0-7#xS6hGJgyOfG6@wApVWndCEWaq69gE*?cHk)x)&-zwGLRrF`;qUd@N!Gs z*%(Kluv!nWvLvRh_KAiBNk1eG=+2%V3gwC!9~-O8T&Z^H5M{rl=MIzn_T+vCV}up^ zkF#3S2J$uwJF#_(PVJi$aW{w}B9v1-7n_ z0)i-&0H_qOp#WB02NbtO4st;4-YAA#CCo3oU}RNiSblK*B%VBjurvs*pcu;UIBhyG zZfq%XSjemCjbYhvlOK9`XzLZ#Ok*C>U_+o_KB)Z?E_P{zVV`qn}iQQjD zplIgXfmREO3nAzgw$z)}T2(0&wZSFt7Bqb33Z2s?_WN-VXV=-aj^5R`nw*{3XN!?6 zf>~4bKQ>0q*ivFu+YNf2#ZOO7><(^|9bLCc+Jq>1KV=+{^?fM-LOwlrq{Sk77YrsH zgDpHR87n>@N|NZ0KuNk!r@qD7< zX&!}?$Bwv1XYuo!8<;JiXLpfT`@Bv`kBVkDEND473(#uKz(-KG=YVA)n!n33A$&#? zyK3NR8hTZw2lkWVhY>?7Brw(=)m1$0spKy=|2vG9t_OzIaXOIhWkrsTBmsQ^pGTk4 z5!RLtIJF7A-xzsyhU;LtV_tD;$-w*vhRiQz)jIovq47g`EAa|ojFR1qjhazPu^D*A z=@tCUE|4Xks*}NS-kbQmHsNNV#qmDmDKArN+Jo2f5clGwe2sy3FPFUsqD4zaN1gFhjV$rU;vuEN(BOV@gUJ;)@nV{@1W_=qcy}0WaC}r!b%04NV9@I z6sccF5rj6kuPDVWay=*owUmUoic;xIGN1T zhJ0sI^LS~+;4eNIIElyae!gxvL|(wl3k~Y-CvCSk!p7k9#T)dQ#nFp3rTx+l_saWL zeWtdhN6}Y~{}Opi;h5_Vv~T)`%%lnX-Ioi~2b^~c9;L5vERmSPG2{Yq!C-((s;=X1 zdf<5pGgh)o!yDZT34Br};^g+tFl0}qcs-bS1u+CUc?7cd=V@+Dn7M#m3bcQ97?i;u zm4(tAI(E?z<0RbeXf%GhowG8Z>(sOB1-Cigyo)LjdGIPD{Sa<0_YEOMX+v-O@SFh( z8ExqRHFA_?_k8ib=d!0Bg%kEPf9d;~UyI3Md_X}Iwe8Aq1bVy?O$nCqt@gwstQ^1r zWM#r_V90q|BOEWWf;!aT_8LaWuDYmWtW1nz|NII5r=NZshCAvO$)U!rbB6V)MF413 z?E^RxWdk7yZ3Q%N`sX*HPB4wO@$lVi+pp~ySVvC}y>xCS<^q~qRWX$Tii!Ap?1UXql+ul#sFdg+eu zH-B6inQ(I;F9z{PKsv0cAOB_8_&?6_*V4CR3gKQ7v#}$q%n9tT+xZ;Rf#Ttr8>=Ta zlp)cuvhFy_HxyPINV5YcB8r_dRyk7IgXmwAiRvFqSGKwK#qqjY4|{q*h^jxLZa={{ zzfb`B;Uxg?)CtulQDiWO*+`E;+u-@^6Jt*X=R^il<((`zeDmV^0FBB*;PtW49IRk8 zKV+0tEBR7iUla5A#mtOAtdZ$^T0L!bCLf?Wwh0-Z1aj*FJESEJ-rlh|P;h$O0!uFV zV0KUb`7&iQh=&{9jB)R|Z@jY`FuWWt-0o$t38b;yh)yXd9;$ZO0u*PcGgKwkf(ZE z==RGz+f>$@!DbP)0P|=Thg|Gzlbf(ul&!*%7GYuTvIjyp?0gr)%Fu3R?w2ONCY-W4 zE#?u%WcD@+bgU0dkuboWI0LSncFL5kOE1+b-@r@LuHWyg3_LkVUl`bz1^!6+)Cewu zcZ|1kI6ikeZcBT3r^{2q*X(hWrU=^?t}JB%?H93mL6F--;Gn)JfJ>o-R?02Eqd6A3 z4n6^#`KHjNH4KX40rdYffXPfBd@zRcoiN4^&1pZ}ziy?meN9Kr-AiW9O_^~JnE`|z zTg1nB&PBOxZoKJzoO@iOqE`VNdCzMY6RVa*nI%>ur>gMrBkLU=VLM*~_4Pv~hGUQeDf&nKL|Yn3hE z+jJK~Xeqe4Z|f)nSjxPEIsxra=xzk^5CGKC6d5j>WAJtGIUu5F(4&Yc7`L#+KK_3H zA|#VDM^}O9!(Vr9zd|g2YblqM_=-5_0s-xx6KMCW5F37?9%g_MJ~n-S#LJy!Bl*~> z)74?E^W|4<5li-W(QalZ!_S0^zEnLY%uF&zc;J1o^1xd&ScD_M1slK^i$I)r3Crsx~o^O$^&w0b&N9hF)m`sB5Jz1iT2y*j^fF( z1PXQN$>67s*eFeVsQ*Z)oxXY~wJ>0iCR46*yj>VP{UbLB6DQ4@taBR%CfH`F4d`LGFN>XAoF@q({9Rt9c# zM*x&bEu24L23G_0``;+dvuJ=9O76~eyBNI58?9nkioO2i&ULmte8}cgq1unEx&Hu3 z(JrBh7YE4kyG+Q5?yO^jYoGu#5)fy)a>+_SpfUwAgIkp}I|u))_xSg5{r}<%{h9-r zdt?zcsQ^7i=J{IJmKHj&D5OWz%+^^i@F>76BOiJ{1qUK26SRni0RCx8@WNRH?Z`C7 zXv?U&S7}Ly^VFQ|yINDI?ZFF)1aL*ie=kD(m_x*8d49h z01UiCzkf4q(e7JO88dJ$B>A>9)=r@d<7_UH3a*cn;5;UUN_$|R$gBRRdB%bur$4g# zP%@#<;@%)D`T)gI5|UDf3HbFvb5zv!^vRWI)}HILfjjpSlg&S}jv5Dnt9)9Fr@+Qq zxXdzwke|xYtfTUF*_>8f;}7}QZ;=5E-$kC^eQ{m52MA$08f?~@VpY8t739yy>Mq~u z-qdD%3qKco#ZW0@NIukAtHpU^ePk1n3MiG%gy@dA%VL4-5zBf8#+<^27Vk~wwGdgA zZV(VebYDNYGu3yX!u0S3xSWWE^eFemqP{&T%D9vOnP zH-4RYkr=`fgKIgST-m=}eWc8xfpiFBy=QDgWK{k-Oy&UrQA4ET?^=q(r|#0d@CA z;gHdbj=^suoj)_=?5WghYJHY;1cnLC90#9}QNBZxvK~)y+|Hb`l+!BZ+>B$RQ~~y( z;~D@sWCO4yj?I`oA0uNp<1PcnG+BqEzapp%R=+{#cO%h=S8-PpFVt@{Sltd8un%pad3|_^m(hpl&TF;(z;+5j(uLDYxEVI2Pm~O;(RwWyj z1)ZD}z+u31244~NVmZdB<}1)rgmOQ$fs1t7oOZc)Um?SM{GMK`Q2+Sz%4eaTG+GJq zTaCvi@3n`c(WtFEU2AZCp$sbUj6D)NP za9SG;L?%LE(yfpJN!n#&;zib$fHQ4brv}<}i!Wiu8$~>x#rc_qj!YQJCAgNWJQ7b+ zJ++N(`|#phjn5-zmC@z(0cpi-z9GUli{%$!oPM9JX_gK#?xTUPh!ep_&3gIl+D@lJ z&&!0`BfDTW$1AfK*ZH%e6t~dj8)?_l$B!z0J)_=Z zKko~6KG7F;_xZ=fSQ^DOZCOK6|Eme14|B_*!Ww!dWNk_X+EKSXnq8*|p1ibOZNvK9d0@qpB!i)mmr35{Ek$IAnjm|IzK>u$hgOz6voPVq{ zf=~q;Q-P>_V3==J%r|Vzww$|%>kE!9F1%Wjr{|-a(ihJKU)DS~VwmqK{w^uA^F87$ z<$V)7`Q)f#iPd+tOIraYx7$lm_M}_Y)$KJhpJlbpxz7pfd7n3Ez*^V;Y%?m?w`O?; zVM*qzp(~;m`+a-j1u&2QOX6sHp#NP-A0^P1#d-L$LVbE8T492F@S^6~TOqMIeOKQa z8J)%kL0vG<_?zuKM;;8!1^P?_GuO%?^eQMUJ^?EP0q|CVOuI`}2NLh&^(`+l6<&J3 zd%gQ^mtG9FSmIowo#@>(w~qjL9>3pyA$}MkT|_1dPg%o6IAsb+w?x|(4 zx-5XF4U}g8E`%_^dWtOVE*#jIc@>|T<-Y1{a{_-`dChqxCaJyg#Y36{X{JyDyP{u3 zVOOL{fXVQRQ|`=hE0w_v??5tb2^1qL7FzGr4oUF$zmJ}wBtJR()PyqnPCCFru{HH> zWs=rtC%ceu+&7nA)#x}LVvZh8CrmMpcWcD=f%j`$NVX@X&E&jyXW{qnLj6)XJgSnV zIr=+4$Z>!<&$S}zFto~Sx6PRAmVJa#=6;7?-pd~-0r%i@#sY}dNV;n#ytXI4&D=^B zjt~uCnQYRvhPN{w(TSV`CvQ{T{ic=KH`T|F8Yf{kT{txJfW@Ad?$l zRpu|U6b$`p>$J_E1|9hf({#6mMlXq6YLi-i8Xb+*4&CZWrfXqw`$3ckh=OK-Qm!VY zUxOAm-ul2gQdELaG2pMxgqw_6G=oe$^UJK-Xq5@JAvxS=EiFc2jEPda?G4hS5Cx(a z#_++9=RNr!-tY}H9r;?|CU2EbK*PSm;Rg;t_6-7R-teqShX}hdGDCr$&`R>pQO~09 zg+BJ3*>)$xZ-!KjCaH)7pB8pSXsoO2uB8hK=NFEh6!yipF4eT|0_b@2z!Dg|DI4Fw zTU=c7-M?s~EK62Yl=fBElN$w_+S*<@+FmO?92k1#rwF>mKj-gS`tT4wLdH(&fJ--4uFs-Vhgj8Bt)95JS?ZM_PqR| zBeP5+F5jEIckfdEGy?g+?CQky3=(d@($u4&PT0nehe5>2p2F8~MrCJnb6nKUW?if7 zYUaHW9h?8o%kELP?@0@77LI&_WUD-D(Io+S0{!>jpgl1-lG^N@b$rpKIjgN-q82{V z(R7yRgNeD>VWTisa!>-k9ogsMrT+MyCy1rE?Jf6pvDcszhHRl<43eeq=ygqkv$8nR z3q|$wg!C~aeUsouk1Z!Q_BW_aPBc#KH^~1-*hmkdH)e-k|IlgW%X}m9#fYF9>Gm>N z6)1`=i9hWSaf?|raCNhCdO7Y-4<~yALc=Y&0wtYSl1E%-pXEoyX$MS`)B{JdSbzbp zEPY>y5U1iQb>^JAL%WP`KHT+FJ{b|6j(u@(-sqlh6>h(^!N4}nb4ED7bX=8w3D6UC zYerC#jVXQOgUDVvO2e_x{NC1bVE_0)`gtI+2~+@VSv*P8;Hi<0ggK5Lc$!xmlYb|H zmn%?V489;SS!p?comqzfGbU=Bhm5LH!>LQu$S_P_kfod z8*LG0qNtQx*D0O1p+4hl{H^2~ zvG-KkkIR|#iU%t1>+U$YXV071oYfftb3+%3d@eiJ@cT5QB9eX1-ci*(ycVzwb)=1H znG(P8X2va1RM^azyUSQ+_z*l}CkWk_j~GHO6u|_rC~mUm0+bFmMkKW)z{nO<^fFPl+#%LC+pmw zGda{#!Wl~$?u*`uIq;hXIM<{Sa>|Q`KQ@Pc3`@H)_rEL) zJCo3kqY-7?ZhJ&Yj0!JF8_n%;5thb7sf<;N;bZodL?{5e0}bY|ZXI9LYx}eX+SbTV zp6m{T6-otY^NoJLc>e5_qpS%FLN+mrWT3hDi;T>H2lmgm=5?!ERyH_&lGjsC ze+W&L4ynBL?aEQZxRN|8)3rLk)Sr3TF2Wdm))FiTLJPm@E`YfHllj}1nj}}()2tr; zSNPa(gvRvEEC+!^-=1`wNgSavHsEg@nY=|nfupoU+W2H7uvTs# z)iBAoatgFX>25Yy3Fnviz3q9F(l|N%b*o60NK0izFMV*;mu!+mC=JOStlC+M4*~#r zWJ<%y)MZOnlKNY)iT>YxQ^G=?4zbj^-yn94^LW^jc;XU(2yAW?IPuqnU;6y|`xO3a zk6Po)iz`Bwx>A9S!3^uorT~-n>fFlEo9Jf)W5%JdJv}lIPoC_20Fc+A@d%5hzKbsl|z<+SJfAtdh2mv5m?@$|Hf->9%_y#U*tzU-i;HmW)qk$&_(c_i{cj(`d zJt=vCoPh&=#?u-;C~0R??WYw(#dw*+@$CD1O0t^2X1?IG^%jGma!AMdX`jV`&ZF8P zj+MqM7(NEGp=aW%8PJW@XQ$o+p8sJ+^gZzaOfVM&0kqA*j}ZKjOT_E~^2495K2CEi zLCsGr11%L9MoV>USK1r1Zi?CmLGKp)W0mD2}Q*^yXcYF?{2jn(>sIkQJ^E%NfJSmh{97BWp3(! zrMBZb$cCl!9k)bj!`UR5)t~G1gq?hx3TVW}7l85?mqS2))y7rF6R=)OlOCloN&h;J zqhP@&qR`pfPibn)_&%~e`p9~PQh;2ZhDZUa4jSmbOmox(Ofbjnym1iD`V1GiUQr!(L-dZ**H1xe z&jC`TC9vK?*MT^z01o}6>M3)or^CnMInq|Zz;Urbs+{(UYtWT9J+dsjDIOAtKa9p! z^ZIZ3+szB>hoSL&?aS3$=gx;U%ytiD`m_1Sk;GoKgZ$~EGz2hU0RVoRgj~gSB$Zl* z)M^$?c0P}qpQCARJ`xEToBBx!Wm5_yD`0kJS7?MW2YeRMEi*d&Reb7vl zBqrv3xMHlzsMLjMI@%zR5FN-Kp3-7-7N42!8OH;6zdcs0H%kb(F{~cBzscPUA{~BY z5uPX5QZh^O_Lqo?7;$$!g&)f>(#k7vF33+dBX48sTKWU;#QbT?2iF?_Y+fu@K{)CQ zh<4*T?^J$kRz&?;q)ek_37oTaD!k5qNU=KKE4Nu@Mn=Fb`g3pYXwl1B4Ph;i=RMH4 zR=mDixH<2+=wJW<>f1%yqk)(TP*-E-o97u6w%gyArf^7eywo$J(L0U;ZAsu*U%4T! zHl3-%*@P%Zq&QlO8>*eH5U_elT`ukCJ)yD?J$NcFHFqOTJj4C7> zX|Bs1IW+8CbY`?4X}M@p7I$gj5}gE-jV2Xbay-iF#;R`4=(DBtTg#8i-RYe=`FIs* zQWj7lFvdW^xvoN@0EQ6uX|9JoY^UVXkoc7Z24i`F>Vz-4{VKx2L2A<ri6yx$kQVCHtP#j3{Y+5(Is`{x{zvahd=)oWQN=Uv< z*}YnoJMdmO{sYCTpyjpkNgA{h-Fw`bqg6rzcWQom#@tx#APj4z&8<$EN8NvF z`u4}BCSZ)n;z+wQKZ&L?Iux(lkGH@oxf|=Nf~L7_E>uPal(LT~v&(TxF3G+}(8ixv zpg9)7fcfASol|tEC=H&)GeQn}jn;#+lf9zahxA~th#UDQg-@RfQgS>Y{lUe*$JZ5j za=3(c?ETMox-ML9Hm46hEz0JUx42VtAy>sLB;Q2hZ1KoXg(E|6)%g=-B^#@l1GaZT$Pj1P;A>vc3~RNoLzueeLb9@upipvob3BhDLQ3Eehvmmmx^6nwwPW3u@FA{~gqoMq9hjWignh|yjYD^@#=u-vpt zS$GPMr?JjDuhY!DfB77%HXnlHD0S&xzmzhm_H*MQz)}-~y6;X6G%Lj$F77g{kLV!x zrv{4GdCoSE(^4#Bb(12_9c_(ejTRVVfT0Rg@Gma&2CHaHToKvxI+^8)D)6m$avya80(aD&IZT{?&8`V;RAX5M3`SG1%(yi| zCouyku%~u*7tZEp!)@ePHBl~p4`!NOp4Ws+E7Uo@_D;`Rd}P>Z&te675#taR>ga!O zb{oyffB!hK-Tzcl>3C+!KBS|Rc1+_Y0pR;5J_MLQz5oujaZhn5jH+Oy(Mp*@j+?H6 ziDK^Lh^7kwbt)6cIt&8&&FKdLqAxAg3?UIX&Aj_u`OCKQ`#FQ-}JmRn8vvolXIA#?EvLowZ-` zhkK3ko54X4phgakrVc>_KD4kSSa7h&?J5D%)~qCW&QWw>G)UQGP z)EeiXa{87>7nsL+eZSt_xfW%nGTq%Pp8n24bNOI}SKr;?7}=Tu1Nvf}NL(!9R5dQ3 z>gUHgyf=!j@vA;V5>{sa`JT>zfW5&Ty#Wqe!C(;@7U%TY7!T+4cdF`WB6QgoAb-qb z+87=$f#1}aeBU>?3V+$2Y^gDkWF{*cA~7Sif>o-y#N-CK6X@!}u$jGoxog8o3N>t$<`C6&&8frTpLjQZkian)vTCfq`yH~yZ0RaT1?Ey5Gt;v+=7hy4Qj=8 zw&zn9YM^8-2TEBAa)5|jF!>F-O3i%@$X;KPQuU2Pk!Sl~;2PTJiK2A?drOi!-t!x< znuFpbe}mrt!|UE#A$W*r9}n(g*ctLYlzg$X-LOOqjRdVnd6>zj9>`0S@wJ1h=Y{31 zR^wd>qQuK&dXCv7)qvjP4+IxVnjAzBCea3se5A8@Iy8P`%2PhVO3O}iPxps@LwSYd z9sMuq8NE6wzZ`_|1RmxvPsfIKa{AOyv4DG~6)&AB8h}h$y)~)Gd@VC3V90vj`&e!x zfL1j3^%GwSadZ-%H+4};zeLAZubN_0ObJht9El3`e!xAazx1weiGx06a9rby!cNRR z_w13wRdhb5p*1+Z>apV=`6(C>9X4yBmq#>!92xe&ftuWE{T{r8%35~4xujuAW$^## z6evwp1t?6%E9EmzQ{VRP&$i5phFh8Iz}oMRYuifj$0x`(R3_>G9~-?F#e$$qDJ>~( z)Oz{2deEz|Qs#;FFAggPs~)FF>dtUt+>v>MGvJVA?PZ~qKzKVy`FN7;wCbrQ_yR;8 z*nU}v2S71)b73_LH(*phSpc8fr<}Xkdgja%l!VcT&zDj4yTVoLcB6G)JetA}M0{ao z##Dw$Tnd$KV6wFh#$I*CH7zdo?Be}5vV6~EhIJPO4`x;!2E3j^-r;}xZWhRflt_2a z#>E3W^k0It*2LWfXv(VVKd{6CWy#)n)ElUa`&6v81TY8_GpZPdJBm2(EiUb^DVJRS zT!nf6_FB4|+jF`AW?mg;?cJXlIQ?|i(ScxO1)z+51re+q){wxs@Ky0Hnl+5Ml?)q9 z*In6{*VUBzyc~L$%Uehf@A5>-+$T|G<*un;N|P*g0LBogw9cajjzyfeRjAN5XVyP` zSAAM%8-M%Mg65i2clzU3f_HOmJgpa7+({WUo@zuZ=UJQLMAB5nsmZ#00Iuu-8MBVbwCxvT* z@3^d0tFU&0r{63FHsJFK$zux4s-?2q2vt9Xe2(QgiK`vdhwT>GP7wC2elyuiaPlYO0r3nDwPf?k<0J8?#a!7{!zulTq#q((mNo|Wutp9KRb3^xSqjAjfJfxp4@SDZ`Ux{gln?dEYz<)ZG~QK!FA zN}t1ctma|%D6c*PpK`U1^lbs)O*aCnpGOVG6t1d$d1C6?wShh)QHrM{<7OL}<5?i% zMony*PS`@!(VZ}(lT)Ex-_H1jhK;y4qOdj{)_U5)Lp%Ln*vnTT8I%}613z$rEJh6Y zOhmO)`O1E3yeOU$n_cN!Hn{xN^3*|1n3Yu6y`X+o``P{(n)ax6{oH?oruP`q^|6={1me=yaJLYbkyyrS6*>p?&%<}d2wc`0| z8*=ub8289VJ(3vpC!5tg49NRKJ*)#bEuTIqv%b%VsMF&k{SNW4Wy)nHAcm% z*JwiGGr24R&u|qq>23F|k1h{wBW~gh2w#c2q>G!Ja2c%K2feC{rg2P$AA<-8 z#4Fo1x|eh2%%Uw)EojU$%YF(DXG>xW53O@&b1B6ZV>~7gryrh6xS0KX;{yBJ?$+Sb z>sl`qUHM)so`Zsl8?StUk1yV~VHpzPaOPjFR@gGT##CWk=COXYRbsd)yX%JFz^7}{ zlMgAWL=loCLVzfQ=?Rd&iOUVo{em@b9P>K!G;?$Dk?Nyf*5tZ=_RI8jT~MeM=iQ11 zKSBAzixuumJliB3ddVSlw^4p^_l(o3OQf}{mn1gHtvFBWhJj!RbCQa>-qb|v36(-% z%i8t*-O^2hmQhi~WGM=&Q7-3Ovz}3va(LCr8#lVeWN1t`Pk!KKz4w5C*E9P#pVVLf zCXI=bh00VhzD#TetQ$m#yI&mDP;K+9dWGnS+w=A@vY%gxT6&+=yZuyNQt~rXwBYeu z@7%~jEk*je7wOq$KZ&LoRO|eSNv(Xiax?$gH|~tsdu3mvgw>Y|x+LmUCBEJIV&t~x z!u9ykK2?5L#u_39K62{?l3uH(U_^vRdn1uc}rmKLD$t5d`}Hcb*fn_P|dt* zR-6q&iJ4B;#pciVKj(`7!F`}8#{Xh1K ze(+5K8!K8%VorbV+kc7{@?DutJ*~a_?(ExX|E&0V+#PV#R82ch|3%C*zy;w?=})QoUXJKWU4uVH*tzD+Pa zRfrq_wkW1-<4@kqnOy~qVxn)aBrx3&|vho^$7AHP3>1V zTHsJc3@=|&-1Q%z0<`#q%I&YZ2)w;dJZs6fau^8cldX$qM9~{$e@N;jy3#E zJO9t*#N0`V)EB!K;%A35z+<55jc8` z&HoMyQiBwFW!uUsFRi-V$ zdFmeLij8z_kU;}N_$JrAfJWQycYq-J*!mwj4T5WJ8DJ|sYb%;-U6{ZA>{L_!r+Leb z{j<|654&hT8)iRCj^j93vS^Q`w3g~_LN}khGR1L0! zPXc+cEHCmA`NqTOH+?EVh6u_c0|aB|e}<;9vQa)EpAJOKSE|&E9$FP;aJDJkd4KNb zvmxF_M7-dS`7IMYQIJ`=}l7IEP+k@}>icQ&L#h>iyNf`BKrVrAOE18@7xu)}b zXSpz!loT5kt)WOEM+XD{Mrz zO%33bdVJjQw_*u4gCJUKCt1js*RkLd&~q?OZ{SrN;0BnM)nVv7n3um)7-~*4z8Ys( zV{i62QzupK{lvPtN+e*K?)}dx?*GA4()S`O(c40}@g?l?+@+zNxX?!HbK+JctLKEH zzKNNe@pTQpQ8yovr*R1*5R`Tm(ZC$n$_G%Iq5eH4lF%OczV8RHF$K(9w`LW$Rmsq%j&eFGW9!7VQ^ zSvVgcz5_NQ!sD3T;|pqWR91jS8~w*HhSddkL}3@w0V=>kzy;vXJj;kGqCYh%6AuSi-ins}#lK7gi;xO%1h1j5_vk{8jk3UK+pzZ(b*uO@*u+|wD1wNsO zN0C+Wm&R;tFE*4VUbUQ)r1E7;Fn3&k9L}eH{|-IYxEcDI0q>8|35ST5L`+!Od^JdQ zeI=@OS?sBqL_+q@ul~{eAnZCVw2gl!3FAtC7q9fqTgJ2R!mtZzvRnSwom*kc0bkAt zN+kv{s4VcmFftUq{W+WG;b4H4B1%7*&pkNe2_o|POeJP?ZI(KaZt$C_K; zSXXN49(lwa{eJM5Q`ObF>DmNDRMnd0-0kDrkh95r56;+7j17y8LKfYpSqbvEV&Su+ z5P1aWP(*~rP0XmF_LlK*l$DjCJBV&I{7Tj1G({7MogcGhUrR(z1Te`;SrEJwZ{zV3 zrwi&w&C;+Gq)mPwvdsQl5xj^nfE)NjC#-VzYhSHE`a_coR|jOmrhky`&^@f+3bi;Fj{B5?Xt?%pEJ@UMm(J7xL2od$<%V#VAm_gh^ zN>SVHd)EX^@F`gQXQv~z>JsG+18D(Co5j`D#TKf{$JbmH4GrfQztDGNL@-)U90 zv-L3*;`ZrtPwgVY9#ZnqjwN@uFk^3kjwBZ`(|^mS?t@+vUNh=UrI zfpE)K&jIqoZCb4;Jvr6aZJ7@`&ScH{rhX?iRCKtkZ%WpZNMIhlQB#}lqrDlzE>Rb~ zkPY77#a95GQ@;p+f>`SIP%dS`+rt(>PLJdv*0cm1z88fl(eGT+8s+a&s<7ImHwo_6 zN(?SfjcO@NU)Vi=eXHF>DZl<#)nUyq;Ut_tm7x$X-DZJ@k!?q@wNcCO%*e0sF=5{BE^ui{g<>0P5k>y#xbY^-w~4XBpqn5e^x0;2$8ZJ)F(<;K6m4iE+^cy)JvvWcw$Usj2T1^dJvEul~xd+ihXbBy2`a-;Z)tg?LyEAWrOFE*xJhhoQZM6F^)0&16cQ=JCqVSmymNtB!UWDs$ zz3mQYK522Fr50a8cb8?OD1UjN*N%rlrOPr$$?}@v{Ya6h)#FuZb#-5d+FGwD$xk`Pb)0d27O}!1g?>$)_HZTS9r{H-ISspj7KXTY-d)r{(Y!_?lwYFuI z3Lnt>Pj~d0-%qbHRoLO0tsJQVOK7$)Bk5m9Aw0O;Pfpes_10(S8tY}$6n&0iEVy0= zZu7h?pJUlHpM7lh)+t};w_}T?k;U}kyD!aTmUDz42lI0Sb+LvadJ+QU`Ck!P5=UtL ztiv2d0CEK;Wg~weCx~*(WZ180mqL1NinQtr8)x*&F4X``y_K8zC~E@vRlMTz{Qk%R zcPOz!$*FTXkR52Q=q}fa?sgH;`%+yOpBP+`7#;6hyrL-GH(Dfd5mGWs**Ujw zZ0W7#lQ>tKe`ogAtCdpU-k-Nh_x#5dHu(!O5@Npa3_d{@7Pd|{YkHbPh0^Ojugp9s zzX50te&tgcWq=bXb#dVMcm#S-&A$y01BC!z0ug60-!l3yzs5{Jw*U3#AHT*X2YlyX zqxSl~98n6{;=T=z^kZ$uDvi6MUSp9JhdgErHq)=l-XAxL-9wJkTG{>d{kYK-_ISms zJ0On4Nd|Dvux64v(dB+a{eG3}S?IRRo#dIhfLg}V&Yz1{bOUC_v)<%xf2kVUd~B{! z4!wqSj^l6!*hf6rs5u7Qx8}4tA$``<8<%LFYQ^5D=)Jz#5MZ%+8RLfHmTS z?3#wFQ)``($C}stYyZMTz0HYy$NNhHh1&>`lDVmHd8>_@tDQBaE3YFvo3!AUEfRD% zX2>DH|Ms1*nhzYyWL9RVwbCi*4_Xpvk6r+`svf_xA*fD;`~sw%?ys`SllIq6-=SYl zA#)uWS5;8*&Mwm3IhqqHt4)vH5q4~b;&{jRP(HFgIs5942>K15?O*;c3IP*L*8=M@ zryYbyPLGIiai0Ub1YTo2&iX-d$_G^my!@W>f?^@w0-K1@K}#U!RIGj}XBu?Av~MQg z{Wdj_!h)T^X1;a6#vnGn17t*CL(mC~ER<7<8Vxk0xKYn4r(a+8tC_UDP1Tj2|E@j= zldq5a9yhj2`^Sv}$el=ZQ^@sP6eE&TC39_Y4eRa|P(&S_XPdl`;Gek2=lUtmo`Ad* zIdo&c;ruwDxc#yU#unR*hC_oVLK zrM^ewnve?x&;vRm(3S|_l%w9rnOJB*p{b`|nov_01C@xzq{{>d2MxACwGKmdc;aAW0hn*o8}DBs%;R zD~U-9*2vy_3yUD9;l8H@c-deO%<8T#Gx5NI%i$$#QL*4@jR@F9G_M>HEp zskyg;x#nTYp^82DJZq=%bBTF#x+IRL=Brml&*|oXg4=?-rJvagD}G!P8uEJdxwHmq zUDToy*FHbV(JcTlg(R8>jb0^rZ50NI5Obm&Oo`m5nj*Nc5rGk2#+V91R1+Kn=ml+Y zC~yt-Bly4!=w2FyH-*D?{!(s3G32<(?q5rx#Bl-uyad06%VD&?`7gMKPo01_SdT0_ z%DFAq?p2a3!!@tpJDWNhecAepn!705HAY?rvxg(z+PII)r317h=x1_7NuG93>LAf; zA$6=DRtI^Fk;t43Oz0Pb^%WIjo>x|WAWi01tKpt{>>$j>Wqc1`b=KBt|zS zVf?3S;p?j|aAT^vyhX|T29sYIX7jay<>TQm08+37Tzs<8nB=lyGMDKdDE4_|ce9mb z;2*N{TxWEA$Z!uc+YlJxKb-Xgh@%-QCxW2ofhC=Hmd%nns$9cj8`aXeZJ zti+2z5gof$;ZjRhEn1(duqtwuKNFxdb~lJ1VK2Y?FVLtMB8H@cFsFR*_+ec`oJ5Ue_nU@EzObp|6SG6|ESXK|K!auTA*>}>)IBEL>BD& z2=T2PRZYft|0Byypu!DK1ojg~OBo$l;5jt`!@_9)d{NO^5+;-24jJ^DDDop#d@6wpJ`p!Oh{0Oxi-Y@^7N zqx!xnSM=Df*@K-02Xz`+_RjRjJh3!S+U$d_h5wr}$iE%xq1{s^9N>5&p?1ir=^KEi znZ|W_0v=0r`HDSxOHuL%PxWpb=DpHBWfyWCf&~SpA=L1zukg_C^*n9hGc&$ZHfURJ zY@Mjv_11UKJiVS3vGjiS{>T#h|GKpfe15pAhRXuhr+Iwm?zGK_Y-QJb7IW1_$BDS+ z7*L>oU~8t*hXtKZ6bBH#BBAB^W2^kPf^Eazm{!M+DTopR_P%5;=RaHq1t$I{;FNzA z>HMRD<+{1}5XMG54|iCI3$EErNz+)vzOZ6mwbowvq~%t-bs_e&(8}~rT@W}QeM*BK z(88iIa-k4m`Qij3QoHhN#^|y}7yoNsf1Vj6KWO_ug|7YWINbk{SJLh|5K_kv z(V;*_qBwTG^_Y3u(fo*=+{G{AKI$I8qEStoemPj<z9 zoXOV5c5YeTk{7N=hOB>~13CTM9P@t_4YPHbJA?Cyipt_78f%X+%Kjt;gj( z1Ge^ibPIrZ*+C~OsF6M)P@E|b@eXhfG3Aq#kV7mu2P)$eU~K?M)w|yy`+?*0b4&nu zbsL5OV6p%Fy@06QSrzyBptNLO;LZ#F{24IN_o#G#jbL4ji*6GAMs&f<%4y-MN`}gY zu?N#$;e6*KmiKWVpZ;k6+_lC85V`jsfFx41a)BJlyn zg)nzPodR4lFHCE|M|U|X?<{4@G1RlvgrtvYnerOy9 zK06d~qgZoc+=yM`fiksHrQ+;Tc%ls~4dMuVuIB4%h8^GVoNfCL+(r*b*OXeO0=AVj zpYJzz)ujo$;7o9I2^Kl4p({M@M#*Szi3Z|%Qy zk3!0gQs_oKu~3VV9nXvh9dm>Z0#6MvTl)3WD0qf`z1rpTN$=ij@}7I^(hCSn{Yx%N zUyJ)8_l9~3ue0q+(wRnw9x_aNrY?+YT=UX+?@}aTFMO{>g2_T!u`y(MpZg)!?78$z zu3}W~uAlsh%SbXU`_92nKjZ`O7xZbWxFeD`P|y)Dw7s@BwWajQA~aci%1cZX4SJ54i8r zXV!NNt_#=oXs<36a?hvqkG$CN)V z29gZ;yuFvXR|Ws@ zhw)TYf;TK!XV(`YVtmF>ryLoG!t?8LuhiIneZ5>PcBhKtY~ozqio6yF{a;rz4*G<; z+&e#Sy;?phb-MlM`qw2ei)U&#I|NjubYFN=Rb-EQ1Ghs@vQWwnb5dCG@t^E~93*E9 zJNX`3rKjeJY{I3Ik~f^(cD0BV<*1FEz3YE{*#AXjR# zhmaif@6>sCxt$4x5i&G>ps~3xG0r5Z!h@^1vXLHFaZgtp80X z&+A;ZZ|)nKd$kGJ!__B!u1n_BClq-9MH8UT&+o{aPS;S!Yk#(7Vq zUe$bUnU&1hKsB0fofjBb}`d{O( z%31=Y$Y+1f1HL7tm2pIayuBy5{K!&;03&Y!vq#9QV|sWBFolG^fL5qz-5FJU2l35h zUTlDwb!uWc_Ws!Y3~Rwt=B97IADlbl-p>bZg2*sR1=%x80O@S#L1n{ErL)ag!9DcZ z$n#F{GQP=)@Qlw>0J+D1d=PVf+p*N$v?*aZJbxSfN<$GGOlHUI(%)##8LTWayu$0l z)=~x_E3xAtsvccEbuF{A106ALzCef^m_EC8XEZxa5+Sw}0^h1xOK7w94SGeLVLJUA z1h;5HBO}T;V^?;+Q%b&~%YR;SJ-nv2DgD8qHUTdFIz+UrmZ|u&7vLCsgvxa8@5M-b z7Qw3%rto7~_%nobd{a%%Oi6oK$>@WJ9Ot`hCLilQEZkvweg5QgP40vHx+*474gq34 zCj)!n0A`0`1=K&XBY=gn41!D*fBtRYhw=1QbA8+^q%)Gs`dxyg;#kf2CGmCZr!hCb zLPkW3euHRG{&b3J4Bms-PrD}rzan^v+1X?}qTNS?_MkZPDU>5h!B?PUhi4`2?12;} zqKQV3-A3roxyRQjS=%6A{TDHT7XSL6aEx{6Di3v*!m|6zYY*z+@{hJ-Gz_4pgY=8a zodVOW{DEUs@mk``vfi|;XpIt8?B$*pFCFgfi%7S+r(8O{)o^WVeLbJ?Ojg-E^SRQ% zS&M!_`X#w-SKwgH#;}r@oWaM2FJPJgd&>vF6kq_R;M|1xdrX0xh0UiV*#i{gY*cm% z3H3)|3(!qP4&#<;A{C7p$ej42*MNuTp^$XzgQS(>H)E={)$su6wjc_q;9GOJrQ+Z0 z#HvVliR}3kn20b8>ohNUUWx65nCg!?_-H6&cfX79wyuCd(1HJ?^J8G>DM z)=v^&^fpTb=F!+KMoC`KJr^Sqt9Uvxh|Ww91X`E=u40^oT*!q$h7ockrA=YZOsdKu zl*eXja>{D$C_#q_KOp~#GALpuk@wVO#=`9ci&tG;AaVPID7-JJloX3^+euLVTm%_%ML_Z(463A_c zM1KEdM^Ht5$|r5B8GNZ(_vxH}MbgXJJa&jw_#oq>^(|`Oc+_>Np{n+EdE(6r;X0-> z+~?v-5c@{5y%^^yo{Oi+j{u8jpM2sGAq#IemiR)D@IW<0JL;Y)69FHV62EaJ3CVaY z$R#LQbf~^wqzNvCoklMw5~Zo_v&w)PG6%(o>Yx~1MBLtTx%w9(_V1Kk zP!Rm788hqFCwqW3ZOa6gH`X8<6gciriDY*^Db`u~7ih!~)QOcSJ`jUA?bc|4Gcw5H zUYD-7Q~*v8?>FP8K_^ypNavuMRdvo=iV*2c2CPHE)GAMuWBTX#`lP3io@>&-)x=(B z0uFd}xsXgr+&~C`*7&puE8-~0HJEid6EBxDJufs~PCm3aXM^X5TNI~k&HME7xtFKUtp)*)@qbfV{XZ$i|KD~mw8n;%j3&?q zubNLY^h8U~P-g_z;5TSqi-F=uTRC(f#M(NBemtn)B6M{kW6m(6{zmHIR1ni3-SI!; zv;24JPrUK;nFS#GrD)8>SEKQUR9CxVP<8{3n)@h>T+YSnh|5vJigayfq+G2`$ znNGGK-cGJ=QmqaZJ52lLGIxqu-{165j5|}>csJd zGcoo{1}2Po#ZSMEnqB|Sd{LVNbzAj@?wZF_e$W;avw=Rmf!}CLq#_L^@KW_Zo_HNGKtY;(eWSX70>AbMJgJ=bIVN@A?-%LP*|s zzt4X5v)5kFT5G`_(2avc!yyYr78=vOQ@p2E#yCZu1t zBBUp*;)Q}-b2|=udYXf-PciHC(a^zx^4SNoloPqN! z9d0*Cu)|*6A8=zgX^HRX!cWFeQO8%25Bl6-n#o`~1uST>avNy4|K?>fKMnvqtlEx#?-oYrl ziAgzH_jTreec{P_nw7aZ>jl7GY|{izv4)k8`}cr3;Py%u8~)^dTn>F5tlOsb-6pc1 zBzbP}_XH{5pq~l`EJVP$fJ6ae<1+g79M9x>mF)UFf>Vv0{~n2@5}>85P6(Jt97Ncy z?}DDFfYttxq`d4lRna7Xp&Ry@i3ca8xjS*Rtb;es)>Hkap@rK$QXNKD{$vmpA zZOl=+C%b`;KZwuN9~#Yx$zEo(?ZZeGtfk$iDn}}Zt%<>w2Pi?VBxOunAbPaXyEj%* z-*wR;mH$nOYurl7nf{$t0j#m4g3}EHwPjnYcLmJuLRkXY#`!#YuE`yQa$=O?)dE-* zb^tvaSy7Yhq;B?Uq=N1X`?0WeN4hExT6yjVXODe)1CVsKM01eRz9YwUU>%(nb);KF z#}ekF;1-Az{7|MBpm2?*AxTNsExDizHx{}f)44=@;@n^C00_T!Gv^7}JSwZAoeKO>4Bchb6XGQ=T<3gMLY{$|3ZoxI6c z2K?P@JgZG3SgFwmE(_>cvz#r0mh%p&b__R5qFKeOFon*tSOA?7#n}R2Oht>{#1p~i zmZouWaTzF+yEk+^63p%j6=d;jxAkM=mMuy;cAnPje$3yTc7>VN;nQsAG8cw2hq=^&Bz4V)j+sf` znxEvf_aSHf8GwVQHpEBHc9iRU5#vs$ldRf+qDD^n%|R_&K|%Sb1oqf2zW$HZwaE>~du;BgYeunD9)+Z(`KkJNh z$^&kY@2SssewO^0beH-h?CJC1rR3$IrYD8$@(S-B86^XlEENZmi^KzIrBMdp%qENb ziq`z8GXTJO>C`yw))+7Et-woXQ_@Igv(XvxxgQZ{%PT{bXAtFBxq~|XoClBC$QBTl za)eZ5t1h0s1n4@~&tQr|&Lu*2bnAV-gREaW6PX>`yGim#eAUtY2O(Zz7XgS^KnAZ$ zU`5RPCmpXseGYChzw?6*v#!w)2ki$q~2d+t(XXv_{G&itH5pW>EkbA z7S8umOu8OR&o7<5UuCD2;gDc1Xn99`T!_nft8jY@c61NvGAF=Q_`)z5!c}DO_M`dC zrCF<##IlMKUNJ)_iC$QuT&S!g!_^rUPv|6Knz z2n`$stOizGXySB9eBr~m?;xj#8O2<8ra+Ut#Ox?l0BO-?l*pkf5(7|Ja=8tAY3Q zZ>cS5?Ha)$(di>p7)9B6B@0b}s1qJcbK%iVfv0(N6%9uP0ax{S`y1znC8m3A-X>W= zJ=D9Q^hl@e30%zMOjBNOq?+!$Mn(IPl?AmKpb}7XVL5aoD1y zCpgjwEuF9I;JC(;#kJP^kQf5DhO^+HEgn!Ff9X~5_<_0CZn+qv3w?hNLC!FCPV{R> zZ#OT!M-pgv;Zmg5_~2pLl?uZwKX#;?hlzKUcrAS+C2sQ<6iNsoV^LRerWls^Fhd}mwec*lJ;~{uL>r-j-1v51BD(j zw#kzp#8scQr--{`Ri1gBs|5MYU4}MqpALyTk8B@zYVl~GNAuTLH)~)OstAEKp9!v} z?<-S0iN4`S_1ew>5PtY0=){SIl;DcAYxtH#nCzfgq@}9cbB?lstSZCZ?2l@ocyGc- zg~ZiXZEtyRwh0sD{JAFP=Ue{iqft+Pi;2LY9A;sGYSRWyx3C-|W69g1e5v1!cb3_5 z%AqSy<$&i7c2kP&;`_C}&GRaqcd>IZN}bYqCOK>ak7uPSS)R>@t-{eph>0t$vR3vp z)I+eg^=Mg?1t6CCHES<(X~?Q4qA#KwwvAmR<~LlK1d}^>UK%@-T||)z4{=-N@Tss4 z7MLHodwGC;>(PSPvm>sjjvIEGlqP{htucelJbmz#LlSd%poB#|# z#mNcs>#6B%^LLu6oX(IFm*VDD{}I*a{}9#JenfQ}bZp$ze?pnXAXm_INb(Xu$N3NQ z0FOWO4q$52r7Z*u=gsN)Vk3J29}b?sVC;+`neCGW&h)uUFsz05+7^uW*gP$8xn+>4 zPOjYTxI8~)Za=T{b$LN9mc&JPN{na_Z=Bbn-hPy#{V5U8)WXu(FhF9Zrfq*Q)t?~1 zv3P%<%Vv)`Uzby-_gl=8$Z&M*-4qgS4JL0jf&#)s;HW~W_@uPXM70AJ0N++FR4UztYYlrB2N4>Hu_4T1#rg^=8VXwEEIs z@q!$9{dvKlO!e8<_%BA*3}*94FaU4^d9v3TW#P~6vD5tB`2lOGbI&?x zSTE{|y^N8&4EjJ8t4fP0@wbcz5{M7D3_;xprd8yiZWIt0ME+=HJNnVeX8U`zvXN?g zbSa3LK`W-3)NJu=M57_^Z?j#Fjkw$;}JL?aAALjS$l)0(bUyn={UAmTZ$C zmOp)1I~r&y9T6;R_P*mxKPiy>24p=D*wy5({Cl>P1O%`!#}U|ob2DF;e#k>;$h;Pv zrKds6*1Y|RrH`Y63I}pdWd$iGmt8N6qXIIRkZ`+_=yU;MMgyBYl9ixrAX^{%Wk`(N z_I*hlnKTuWVo3wv(J|YqA`t(hV);v@IyYNHP2lAiGZI+ z_2~flcf?e<-I;qmXmV7I&|)G=i$3u?$Ulz^f_eo+4%3m`cIYfl(-e*=06?8zFZ~*c ze_l*bZS>>i$>Z^mAHD)j2F-wds2-Iqs+S6<)rjq}MSX(#jf`P`GyRqAle{%KeGBu6mfc>(VwTg;@Fl-u8`9HuS?AK0da)5XUPwB(+WQUQNRJ@Y&V=BoAqjSaD8 zh*`XM~8xqH5<-xiZc472V?_SOM$QM~h z?fJY8AMna)ueMJljoW@Q#QUX)g_<+uBb7q=>B@c8$gx$N&1rFVMSXA~k<{B$%xAfNQ1*eQMuh@jvN3@CG z-fb>kkaZFf=IJE)!ROMefL^^uJ%1Y2+^H|wIB?s6+!wlvaybrcT&qx~POBFOs`K2V zV~tz^8nk1Lt+x9Th3bZ1MJ8|75s=-7%eOJH?Xx%$XbYVL&6vZNES#=-&516Cw{>O? z@f}+n-&V(Bm6yAw!j&y2!Cf~P&J`!|n2PKweWfufNj1{f?1w(x-vV}#OU~myO-Jw` zi1Q?ra#iY%mQ4mAe>t$3U2W@NB_if*b8Sh5GBWQAh<+L5mK&a!&iiU#Ymej4C_;{H z$}j&L6d{I^3kN|zv_$7JX3Gc&dXlhQ%jWU%-Z0-Mu3{~dCbv0KGY^;5JvxrvGdJS> zc=MIh4HmyI8l!$5@4_U+$eZ$c>DZ zgiEz&g_i`p4z>c0hoKSJSU}Y{8(g*=QKU`~?k$2T?expNt-~eeqv7Qr zkUcoajw5ha!<;qP#-Esf{tb`!#{d%z~_5oX3jv^8pMlB98tRE#0HJx#Uu&e>n_XP!a_F4 z&U=&D)l^~$rD?6{Vo&5>_TPz{-MQF!*DfNZq7Y4zINByk3_Zm3X48T&y_38oZRmV1 zF>(~Jun_5^0<_&V5Qk@Hlz^t9o*zv`3*X>;evEh2R`5;r%`g85kdvQBVMJrGvAZRW zyFfM56avQpVoREKD>zD0mo7ncN7XK&-r?4^{z)&TbpU)a(qbsFWQ=b<1Y=4*ri72D zeg~zvkyhb5>Os{O)TE(?ebNo>dC|rJ5x5{p=0&|m4=156_W(FQOcZl@$3V>YTL--F zsC`wvgfK1l!WdCHWLy3f#dIx?dQUvhx#qSyVG!wIur@$?dWlc$S&o`JTaMf7X*a7d zRcIN4;{Pb+{w!0o$yicdAHBCD^NFWjFP3ZE{c%Fc;^#r_qA(wpRRM1FOknkjGI-E! zdVv9}braQbFmc?W=JMml$^+@O8^FgNe^7#oWq<`_j@B!^d#HBlnOt&wPr3U7OuoS2 z$#}GErSL$x66dvNllG~q$q1UpOB0jXns7>xfzzj^0||p4dRNRdW@c*~+YWFChHh3D zj5e2>bDkoP!=&{~eE{b0Vr%*)K?ZXdraG-UFh|d1Rp)BFd3MUTx2CV1&!w$YmFMlr zrh+Jt5-3l|{f{-P?9GKFo^$$^eAa-DlE5>)v0LHgb@~Ohs6fITeE`FVCzi3P3}!Go`3Pi!v+}i)howCGOju8iE%XRM3lZLue@or zkV^~7UR9=`tNtgI0C z&7@_)(*clk@CQs0{vyO@aEDK>bBNZu&`R`3J}lKgN0jws#L>h<{pIXO@8tv_9*2)! zeg=VyI@V8zsnNfe1PDUjeaHI#Ga!Eb_~X?0^N}D!0|Hd*Si}{MNJN=uK*@x*v(*ha zi&ClZT-hV8O=T<3ZWlA}Ll^~%7zjAs*kHkntK#K+PenTUf4SW@BWhZ*KYEg*lIslb zMZeSZp57h3U&vg@_NM=s9{%@){PS=9(=7d)FOuy#117<=zGB2oMQf&588HjvKACqn zLP>s*F58A^w)32~DX5IeuE(N4%z(PuDufbghX{dy%58Fdub`(qcDn<`_Z5PK8|vF3 zpHe^ziW_=)OsZ^TtI~gQ4PId_knzMhiB_%cO5!sBM~6y%lASzVc1pTU8gVKMrmh%ush- zdN9cY%DSK-&zGR>+Nq-T1WC44eY_I2y63=!Ew(gMDo}8#xIbRaDJyN}!i) zSIXIlWU8hY9cxifWavB+J_5@Hr{>2PT3WyXLijpR?KNYng6gF(oK2%7I!y8_VrJIE z@Y3l?uL0F>=c`@h+Bz4$87s036r6BLk@f^cYKNac8;Sa+%vu>>{hz>k3O&eDf=xE;AkXjyH)8do^zfv zc(2D}*x<(VO_$>d40FfrZMk5k0m}DnA5$qBH)ZG7wT^u);&D$Z5Jkws>VVvimNHK` z+HN}f4vJK~EYmJqx$4S4oR<}y*x6KjnY%7yl z6QP~R(^*|sZ123!3w!f7OXG(5GS~fPoEqy znc$=^yppHY!Q46a#!LR@)*He`P8^0Pj9%4sM3O#<;@n>j*m%__hfh% zCvMWa9M_5 zFVBC=CUh=-ohNb zWc1D3;qT?MzB<(v#{~3=&%HsES$zV4jZtL9Y+Jl|4(mh7X?x_OJ89~MGq+NnvBZjV zbljr%3XbwJJqgYm`g;T6H^j;x-ojS#N#9M0=R3&Uhj(&#aMQL{c0`YR>kM^Br7F+P zj_k#P%R+%Oskc4YaynltbK(?$_!6xc^7Sh=BitXf0m$L`ceu+f>f{CN2I7rGg4|WC zZ12|9w;D95Syy%0G9Ssfa71E#(#7bn4*0p%`PDrKarXQ*|q=94D zpBHz+Jm#R?Jg02pCf2ozpkWio=4EwrjIEkn+59+tQN@_k1h|SW)Sc%|C81Xe923n| z>*Bvl07`s-W<=43Q^=pxF7b64-uxCcur^*U#-|~`vEhTh@BZ<+YePv?F#e?6(g8lC-2)#q)-)e|^6f}UhF#!sY)Fs%_Z_2#< zT&V~0lJi$`oGMnuQ&;wGYuq_fe0{}Kctf=0DIR<1v7VcLa8caA$8%izQP`CGk@x7r zt57=j6eFx@?PDg(5$p!mvBwhZ9@?Vcs-7P8A|+*gvOmd6b0=4CLtjuRy&^dAL4;|I zt!Dv+Y_fsoRjMWZH{??S(~hf8@61AsW-ODwIw{~wZ-4ZbO?w?wH!bsE+h@&7p1@)L zt<NsS?M`=-A=vVd_=cmhS`?HPNm>FgfJxeBXZDiilNSRxcxVs{f2Hs zq}WJN!^h#`d}oh|r9(x4R3ZsBZYpstti+Csca`|3ieW|f5|@6{aQ}1X{NEC-1Wx+? z7UTc7E@1zS8zJ8rCDES4i>yPa>#aEf59X_OgHG^r!A~PurMwoJL!Ln?r3=|lmE~el zam(>%*vQwxW6RjwK7i(r3}c!7ipBb0?2kV=x;G@>^0H~_#wXwWi#JK@j}3Il?lSrR zp)u@Va2NmMRvJdXyk9*^7_QG;_cYX1ROcwd?ZO-YeMM%N*ZMUWlR($A z!M?ga3U{omCax||TGcsC?M!m$*KgW1$X+v7x7(#Du9g2XaHqt`YwS&B@~K_*Y53^% zC2~9yX&g$kZ(E0R^evmRoP4?5fUb&YXfl=~U%ELcQ&*{3c=xy!)n)d{{aSWTB(4PK z+D-b}+7C41D8S>z4mL2rvcrL)d@~_X2VYPqxA6T>0c|`9PUdUK9k#VIfG=zE$UKc6*WVZudCVC@09gaK@*$&tVokSaFTN_(i zi%AQMmeN*=o0tW(ND?_lnxRdHcnkeD+Cm(pTrsu5@9dx7qA#CN?S z?$i;~Z(nX7Ns%tpTHM5mo*pQ8`%IO#;39|%bh}3Lg-D}e4%^#u4N;*wA+NzV1sa`? zH}~%iC;ZkzWEa&XL&A&(gqAX zgiKs3Mq%EWz~;Fy@*4ddrO3(AKs#vXAV$&I#paSfDV`5ga2Zoo4CsuMt9uM0-o&w^ zuQvk6Sk2LLsH(t6SaY*j$+a~VMLhl;d3W`TnBp;k&P=1bB5Y(U(m&B7_*-7qz!}vI z9?TDJaG1EDn;*)`y8BO$a=_vP8@&y0EXtU$y23*baqK07U-1UouSTaNmem3?(2o@g zNN(i}R6)ccf42blW4|ej{&UL_rf#Ni#7+`Xkkq!#0ACeO^E*yQoPmrQ{t$W_JzAIf z39j*2z~$L+fM<9Lqt$qX2q1XAy;`u6T7QQ3wL_Txu-VNe{+smgFB%7&3=$#V5o{-s z;Q;m^MGtPn+({5o9>;`P2PnCl zOmh}+J{+C?OztC3ENh&&8$RlO^ZA&t{;U?9bLj-AaylR5mWrZ?WkDBw@^pVM&VJJ+^7qqyMcZ1 zr$S$71YF33mR`LK@KOD#o`_fP=BbQ3uw0-^ED5!K<3SA){jQLn_PCa&)BQ0BM=L#3 zFVJI+PLY&%X9_Gxz|j8hIaB1pYb1%j}Im=bGz1j&34ziO|wdG4uY) zMKE>E(yIdk7(kIrnNOSTU}whSYKvJf-FF!2e2Q z{CDJgzV;SGXag;-X0Av5GlW@MM8=8JDE5Az=4BHSx1e?B&D^9XK}?z;LM$!`a4;z$ zoX}m6sftyk$?GkEDHyzLb?`fV!YR*U`AKPm>Sr?*icqeva_vFfM&x_J*eAqyBtFsr ziCN$@+G%#YLP6DdxZEP__*&ET^;jgpXpN(nRk&yi5~%q(2IHSvjs6|^iY#IQ$_q0@ zjfP4P>=yhyM%DY>I-*8zdrE{{9U~il`G(S%@?{)IC{P8Q^c}?GuRyQ`wmhM}y5+dD zXf4(HGPJkux191Qn-ixm_FyyC;3g zfpSH050}}tEx0KQACsQN^XFOlcurlfvzdY=PCW;(q4zyg3mvP% z?hX@J=FJ3pn@8nFc~0n_?L>|r-1<1K(}F2yd29;Eeb26_J_Wid%Tq0iZWXkro?Ywl zJ#6|r9>~9NW>T?Z5c4t5(RQfVaAhIK;s%{q`+@5;@^YLEJI&MwUbH2Z8Y>JAny(9G zD2MXeUyb3yNHpirqsAD|Rwy^Y2XpL;qPKC~D*8?ba=n7ClJ(ZycaWEj0T+C*X>?F1 zIebrd_{!a%<=&5oRh7`FM{AdTw`ejUSae8=a)CgH+O`9@P-@Seg6a?6X zLWGg_h;hS(vm56YaQ^bX)FTxN2%4*x>?^2Kx?kTiv3~8iNd73Uik2@r`t#kQDi^A8 z=tuR^ml6p-Rvy{d(v=t)h_XCF*J)JH)d3**3NEV@bbR0#MF8g za4>;9_kmZ?mx707d2-$_=KK!1-1HMGs5&#=&?gI#53T%sa3Ic*L<`5Cn<5yZ??Ctz znR2B`XV4*qs`-;QYJ8^8XT9I3``Tw`?n5lDdP+3=5LHHCyfSnMl^nyLuHXIn?qB2>Z(W@Q zomojq@x98*%IJ5g4sTAYMM`#e!$2U7vjLRSC2s*J8t`{Jn1@b+=f6gv$>);Zr-lYI z=AA4(7f&qo2`X}H$DNG66{4~7db&2C;TsdfhE1>7_@pM{blntDjhN<74TEF7aO#-^ zi9YTJvMgS;A#-$z62b1L3`Qo3sNFU-n+3nsem>EMuOu5Ar>Wbdiy zcB1d=4qQl(<+8O^WCrk?Mj(C3z#S2|Gmmp~)}(yRiAnd8X^Ix{PVJ#5U$#rL=1+fH z-Hy9T@t8S*TJYdLp)`Pgd)$>oYgVwrS?ViJ%zs-GX`#W83X9(y7Z;n>nH487%dt{7 z#;eV#%w%u5kR8coJM!p;OLtj1UmMzV=U2uGr%r5**d}vlDRJ{_&@<&plRf^OL>2x@ z?fb7uOTSNQJ*2||eds}9z7QN&l|MZW=A>x3@QmR^Tf*ZHgW%kw8+Dsx7|I{^z1$=t)H>zr5O!Y0lz|2 zrVvgfws;7fuECAsFOF$dce~Sl2Qiw~M9z=Q0nIBA-N~pZ!i?UjB;@cSb7bmDiz~eB zfWU^-Lm!M2sPH%Kt763x{o)B%ooMj4Y;^|2H$Tu_T=|NSJBjx5XN>vQ>EcCpc{T zQ(mTCPN%a^^?8*b5@(w>c0TXmY|ceR!}+kyO#EfS`HGq=$QBC*3;jTyX{j4e{GM(> zf)DJvqLJ%adBu@1%?eqlSGI$uB~vg-#J?X=F1I;8Z`#1(dm3%IpB^IpdRF7Zi#naa z>QvI==8bSZ{@h1*;}bU6MvKB4o5Je%?TpTeI1b_QSW#kxq6}W#UZQy2)xr_VEMrgM z9h$?F$dG*_$f>9bbnQOFR~5UHE+AS32HiaSKVAJ#GHI1vi1|os#4}|1^!wVLA;D&U z>6_R|Tu>Xc`5pT6)2g4VRY0HwK%MTcBsJCYGoIW{fbBvtj)`1N&8dUJuI?h<;(wpUl>_O+}eN1e9(c>jojquUV{MucQzJ{V&9Uju`XLzfXWXah9NQ zxiUwud|Xo`gDLYTtTle0Z1-C~MG?OMZbybd$lRDVzk?zg#3}|kRBap|>yxo_lW5P_!+ni9wHOXtAXr(Fw_2@<$3HcEA@29wZISo)f%EfRc3)>BK+@ z^tt(}p_^0`%wP#eLWC$fLcYA}R>*w_iT@Mc4c@Og07n;o0!-4}Y6QnF1pk*;pQ?W_ zqarByU{?Sg>!fTzNcT2CL~dJIjJ^Aj3FmRJ2{gH=ifRCjA;kwk`(ghVfyVn_^ycVYzGj28bS;-9tDjk zAyA$KmID+vR+=ZQ6&Yck8$Ru_nY$`G-!z5TDt~XK4t3w@-M3Mm#yOx@tN&$w0NP8UYEhbh{=wZu8L*0P6|X3w5Y zakce(Tz38Zk>tgbyB&LgK;^RMrZ7-1r&m-bEVO~k#7nTeNoQl1f!tL0%Hg2td*?gP1HcM&j}BYhW4LpA=O*RXNgpwct9lJOe{Ng-+T{iF#Sk zZG9E6KY@u@Orr3GhjRI{EQp*Oxqqi2EPsB2$SA22^+qe^+Zh*yR1o&zzo%*LCx9s! zug~kY&W6INrEN+GTA1P#6kd`hYPu)lfnZ*waCbr#UgW)B9{HSZG!ktz%Zf`S;Lc+2 zp`AK=JJk3d~VEO8qj@)plIOqr9)5PEKr`$E|hCYmG=rc z*3PfCV|t`)mmce0(2)uZYz28r|3r}5&vHcZx1hhD^*Z@-Ti_v3Mt=#IA3+Qw(Kulj z?l$_xwB*!O|0OO7wqT|c3%K~CPb=iRz;42mzweLlprsj3aiP zn5xVyggwe_q4V=G3KPUjHRBrId|1mF3b#@2J)H`|&9;Nl2F?@U1*>8b6)}~>)0^Gp zo)od^WMMpg= zk4V8630{h132yH>bA^qK>|p!z_PE_#*aqg_DV*b6oul8?#QR6Bvqx%~^4ql!xb=MA zQtD#u$wz)VSm`H7|Nn}3`6u4yIY%Dx*m-{01+~$=*Gjs&wYuUyS{)qrw$QCw`SZI-1&QtFui$Hed%OC7=Q4kz|g8P#rVq2Luo8!729`ANW;nrH*VyT zh_s-i6OHf+=Z#qmr6GyMIa+`t-OPXKQ_s%$l-vL}763tF`l#}+KA{{s=jy2Fs(NxN zL9H<jU*;7M z8->FB$iCBhiwl_-iRJ;yA-L;6(fS;xjY_E+B1yp7jW9_A%)Pne!MS6J)28`S@&`-; zej?+61}KuZ!O`kW5M0*=$UDaX*!q7KB=_gVG?uS93;v(%!D!`-&dP5qWkl2!m z4OM2d7C z`2v#ExP8FV@W445v8S!b@*U(0XwhTV08zuOSzgFWDol`cSO=7PLx8K{bOT^nD2-Mp z?s|R)wdVm}A6ECHHRP?sTrB<@fB?t-a5iM}z>NdE?!iXzLC@@WkN{wiD6oksBSqu} zLAGBbNlJjTq0;1n9r1)N1$UDk(xKEG}-aPRRAsny&jPYf&4YEAY_3m{G-9hx#+lHzV9GrWqAd7 zua+gRI^Ci6g)pGZ(xYm?fk`yMcwrc%E5!g^Tr!`(&}ioV$>f^UnoZSCmUGrQHnP3j zfh10XJkIXE+jdwwMW63wa*^AXQo4~-Ur2{4bO;ZjJ}TW`+mBy6u>TJ*ge}P=TFUa+4Jb#VnPu14_>v8;g9KVXaU&Y?9V((Y6_p3PhIafof@8kaq*$25e diff --git a/assets/architecture.png b/assets/architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..1bcd1283794cd9d3118e4cda0bbbee0dff71ea14 GIT binary patch literal 952555 zcmeFabySq=`z}5LiXtu24N6E$OG+b+fFKP+cXx<%2`VTO3L+pSAkrlz($XM}j^xlC zzh{WOxBIg{`~1FZo%6>zi+3%@nR(y$iR-!V>%Q*$nHfTr6r?cFh|wSr2!@QbxC#V9 z3ZCQKynG3~^Ma@~2m&GM^|=4QNyWgG($2xw#M}x>>Evz)rG&bfn?N9L!?_8TE;HP* zfk%$li%=++bus-Q9WTk+R$&!#S=wr!I!isvjc=vnJw?IhSwfz1JcQN2*Yx(Y)mNY1 zn|JOd-j0_d=iYe}+*z~Z6HQe!UNaWHH$SR3=9|;h8Qt_SbYOi)EI(yn``~aZv|4w1 zCS$cHboX$#es=?l##cA@$c1{Zk~UIWcyA!IX&^M#`w+WEflasv&3S!u4X=NvG8sO* z)p-$%Xnw~uK(azh>l&^aSJ+?m%bX&)>AO3`e1oQah${wuj z#>1OeNA_kt#ozZSYUGUz59Pyy$wydw7JT=8^S5BtPfhS|Ygnk(+}sds+@izOW|H#| zR^PGM*V`_u(hJBx?7Bp<;5KNmQal|Z>22HAJsrW5yye{7j$K4%S(#vc5AG?*Me<42 zjoz(aqq6^-bTWVAIIDs~A%#&W+_~AajO-4<=9d_JzLyh5(bWfZ)rXtnTkppvGHcSK zZM{>oDhPuZGvK5WqEiP=%I*Z)d-Geq?=4dw{m}W@fnxd~MKuSv=EDur^pdVn%XpR7ZmQ)K zU2K?ALM4o*tddT1=VC&yEcz86ttEZ4xcd$a>2c~FM9}{Gj?BgV>W!&I>2VLui;Me5 z%!)FSJ!gwc#!`fKNx5;E9!io~ud_ELYCAdnY{{TQZ7Z`%f3BhFT--PCz!5nm$#uxN zWHd!^(`ju6J4(ZL%zIm41(N<$k#FN;PbcphQt>OB>L2|hHZ{{76l&p}89BbYyD+uz zakZin75)zgnZy*}5 zaV=EcW?#8Nm~rdgXma9_o^uiWt03d3Z>mGI>=w`SJ;%|*R121&VJbn}z}_IO-X~_l zm*zWJ!4ZJG2=$4%m@pJrL{y^BM)`=;+CayI*8 z84P&AWUg~W2Y=P+P({AyI?QrkKV87Wa2Ab}LQ=*u-Tw16&0J-dA{TlV%dx_YPm1H8 ztU9JmvGfAbXmhP?{W5Y$lpiP9qMDCE`2BK%16~Hh<@@q6^6^EQ2D6IEmbAt$`K?qi z#9yR9&*^N`&njrYNNfPlfVJs~ZcI%K(a=pKimcCkAS2AXbFX5{tSkTFZ^@?xpvFD_qvA$FQ(Iz}I!Tre{zdP!0@!G0xsx-8sY>- z5RA|HmYJ?69>h4((?l-Sv4*+3U5FXuf?BOK&v@Gv(k+G2n?q!WSxjbDa62$6MM!2% z<|P+#eAX6s6?s5Gp{cmFaafJQ&<_cG?#Rz04J z*UFolk(rcHYK? zXS%k7*X@*&Dv??fy12$$eg+9p-p}Om_3J1@ZmX1?94MnMj&wW;lL(`ZxFavTg4O;cp}ZqxAbptOak>Uh#y}E>c$e zH}P*}rcV>#DFs{}(|FxH67bCPE4I|5WOLV2sQz_7r?y_=P z=gt}O6=KVXWr39QB%#w9Tlz2J4LYnYX7cHjs<516K?_*o3=tw{Xl=B;eCd3H9sNdQ4Sd zbII22^|oTA)EMOiXSdtUKGX@L>b<*X*Oe~Vzp1b{6Y!PTL_G?Z_ql&^%&%!l#+SpX znQLWeTlOtQ`n^iC{DyZ+X?>X$`vb6y2sx%?;v4EvS%2CWQzrFTM9q2%ZHp)HsY2dx zs^Adt?~I0|z5da5W0&X>#gy{xD&~SQJ1lFJ87gtyuOBBjOpm_KFPKjKyvbRsZN4k| z=zDBC(t0h%I>f@jz3=jKdSn*!SnZ$`oG#V+ScP(QiWgTer8x1urlaQiU>UH*=l(b@ zh!NkvG5u;$y*O`Ns5>(Ar_f050L zGHLESHd}18MTIM_DYP-r07+ItNNJ&8mj z5TdX8r6jqsS;U&{-4+!QIty~KwG7Fw*gjHcSAg1M(&<3w*`a_O{D=a~a8Gum7ZEpJ z+2L93zp1Ic_2~f>!6y-iI+aE(I@GTLPgKmQe|c##HL&MB6s4Je?EB!F#n)Ty3|OH$ zt%6_3S7syXE1VdOD)7kPJa6Osf<*2%H^#~ZP002-qH*9 z{!VREvsB4JPKJJDV?5$m0b@89&B)%Q+}V7rV;0B-;8`T^@i&HPaXT` zrCLoBUMw`<^y8LuTShg+i=$=P0m!;-n?A%#NSxP zcs%VMOeG^Ma3f2ZRe~f-FTWFRyI+xiRCevUq0+a{s=9L`-K{qcNSb=82s<&3 zJ14xQD-AYi^o%LYjaxlR{r)3C!C+*o5!c zRfJn(xFXnf?cXaFc(RKvpjcY2Qj73h`!%ze>IY-m9DZn{_%Qjcbg9RaY?FF?{9)V@ zHJP?5{KD%P7|fGitiGcfnyJ_t1J2Zs-(8#)EzJB<36Bf16ie6L^=X-BSF4SBm5=Ls zDLGVYHI2meDFx%ru2ZMIcXEs3y|~rcDWSxLuco{tIQPL6hE5m_hqXr^MphTa-=; z6+z$Py;I7)T!fItE+t+T!bC=bazyZcT#UK>j-ifjaKwAf_<@{kP~85-Z1_D&fkU(4 zOL_+H@h=m5ShF!Y3>N&fMq&tp{t_A>MY;pIkvc`AG_8r&F2G$CRB_>3o;;=8$h*X4 zdLuuX0NhtJCQ7-u*UBPxChlVD8b%LctJI_z2pl!@UJ7n^VpR@#TSjN?M`VEM&o(-{ zs%GGQ{~DiRAR8N@WyB&CdDpmnLX^N$9u~+Ym=JtopkV)_T$6N^^}A}EkCUa-Ze5)@ z7*J9ZxnD>woex^mt57%uspdV>g*B;DHTo~n3f;8`$u5^%bQRy9M{dJ;Teit87y5}A z-gHs7ypJ=@g=;1I#o_)Z+%h<0x?A`QR;1$AW$bS$kJbDe-?hzSk@xE=jQk>8mSec} zwwUoXR^d}EreRYxF2Nkm_-2a0wo0s}M4z-~2xoZEJY~>yq4vYPoFj!47ZZ$|S&yZ8 zBH!qIyDD~5^xgbi5(<-ZhR7v8$n%hC-!0ycxka=^4L2b5t7z@$=&|!I=B%qW4E}c7 z>`q2roBSUKJ{vNZH+&kJ_tDUMcPDT#1|{XGK*;Jm-g7_@LswoZy?4wLjO%HBu$yqbDlfe=fXXL5th)GtsRGBjm*d35;GtDDna zADcgMaXN`GK>S~uIkC83?___!P5+XCL2Z%o3NMqPwqN-D&@5u~0nDqx+&eGB7Osr- zk;w&-7T>v=#NZ~3=QVwII2CK}F`L<}bPJQ^k<_dAgmlEJ1fNmnBnO%}W3PPSQ6g-& zN-M>$klT775&i2r#oe6{g09;(A@pP0q)PRM9I}#%0_5JtnPH8stjX`T_DHoTOI=vF z=Y7aN_+VVTjm*IxL{>>$tQ-WfCFIK_V$6HIsyR*r^O=)ycisCq$Irw(pZE>7%*YRR254T^b@TDM!J@;BA2ZGoqlz91 z1c^VCS3NO$iuG3k>ima<{i{w+l+o_b3)iH^C&;gSa#(zq0b=KU<~jb?W8a(#F5DT-3N9uW41Rr=M}Yt^T4 zp)!7cghYO*w1jW)5HXY9UfDZT{<}Q zGP-CJav(=*!AFR`@av6$a6q;Ele+im_O zgxhoUN;Bzzp~y=J5_`h^oAXEmAH?52idBDo6l$HRc57AanVRVA6CZSgv|`JSXv@|Z zv5D~+g=$$3{2vym+6PgFxDnYl_4>rpUYx{K23S`1^=Yt=m&xfiPZs&h(A1`fw z;yq^MFEJ{~d*!I~uCF&3htVNNqORN_j%-3~X=F=bn%}aLtJElqJ=AE zkObYA4n}H(mYHfE5w1S(ze$9X({ESYV{-$EqU`zfeIbGNJL^!cyTPy5-4})fA^?HE{KN5h+0v+xD>By?`csVk$P4!@f9Oq4$p! zyjazmaG4r{Oi*@{CIVJ*;ka~@j5O$^SS`+6?r(?1v$|iIUf*unn|OVb3frVz`62Hw z0T+*3TvJ5+CTMBo=^5ojff_93WwrMAN@kwO2oi5T@#J3%E64FOm=qqZp$SGS9^QMM zWq&KYG@w}Nd*{oS*ywLDNwJvi_1z0&j3c-|e^3=2?GJ{@Z(s}(sf>382yS1^S*eOM z(-V9(aoaS4q#bqFlPY|j$Mw1u%GAB7hZeELxIRRsIEFN1SF?hbIy*fHYwWkyciDEg%jmwO49Gs)}Pd83#;nmNCoMg!T0=`@$s=mJxF=xH=%?=a=v>D90t zq{F|Hn1f2*ASaC?v&9hBdBep_7mKtn?Nz78vw(Cz)w%cY;P#aH(^YsP9bbYrJK8^X zS`M;9uDn%#NZn=gm5c06j%;UkB{wFOMfQ8}q?_MavLrmRpbd4~Py3-nW{hUAB|m1b1EzNrqx|rn+LO|iY-YZTeG*eEs=)#^ z<76o4!fKuradpkISy>nfev^>zJ;ssjd$6iIDlJ)Gn7ri&7Tu+}B$O@MaKr;Rb6s$0 zU1*->;ZQxe;Ad2^>FMZd*uE}}t||I1oHr&5ifv{hNe~>7K-I(YNDz zZFP7gc^E9o36U8-d*1GNym?L8`PJ^IZXi-Wk`QNbq2%aHTR2XoL$`l__7{r9JkLJD zn9)9NLf&uP-==TE_gAK6Twc`$UWPJuiPUX#e&qc8gzaN=I-L&YbhnKxF;`5YV8rsT zri;bWMRf{}`#W#LnhYU}*SwML3=Uiiy)en#6}Qel6tGJoWhmZ3m9N3O_4emv z5!N3bN|-$84Dgamh$0dU3@# z+Ao!c4lFY-C>B20=%i#Wf zcCI*yip;N5&X)Dr*l3zDnc86^>>GXdqj*AX7_yDu2u8EnM>U75oGZt!kws_(ULC!V zN+FV|XN|PjWxkjukd4Z(;sH0FIQUXjfHfwi%4nU1`f@3aj$(qBACsrE&}n#s_hv0)Jfl;INy#*{R09QfSFxP2vr65^zU=;?ImR=>s|{z?Ic{vU?&cuq}4)${--^ctmLfkDVpWKaw^qi6|^bx z&mwoGpD#l;M6Sas|8|QCYUjSbRTc-S9lB zDr|6`m~OP;U^mP(UbZ&=K`*#>&n{^K5=rQ>pI8}{Y5|Ywpru+sy=Q;Ht{a&>h@ts_ zrIaI7IKM52ctFt`k$vN%FrK)AraE>wNyDhsXNM z?5qs_6qk6KktP2U(n!b$%Ib#dyo~3%J2mq~;m<5x1SNRWZ-}s@)OSmb5RnUYythR5 zPwqz!n3KFtP$GoKb2xWJhVx>O*C7E+3OnE>yhZ^6x$xLrOiW2eOzdob2kh!3ddCP# z*DK)#D)xj?U&px!ALol5I{F$-H-&@P{9m8>w@0P037UzMp-@YAm46XKUa8Rjq&FX$@Oq zk49XX0@q^{`~s9K)Oh!m?-Fqvew=^%K)T!gd$aJb-5U-JGmQQ*Mi;J|J<$=~rr1x; zeV#(N75mx#^>m}XP>_Fz&TD1IM7UA2c_O6FbVAm4=a#en&!_8J_rl39RE^X&I^R>> z^lK&4pJ;g^KlJ?3{kuX~l_}jkQ2)b+uRYseJzOKrz!FI?d0u$|6VpEi37WrGS&{8+ zb2Mh&0Y|k|ad@lcYN-MC;fT$_e%k|iIesHsYZe1zTSF*|o3$O-kApx2gx%~6j4Yu} zl!j1Ka~nZwcts60rMa;nwFZ|wo4lPE)XZGk!vU)5p>W^G!_tV)m|9o}O~8#GG++&N zGN5#`wz6^LcN3&OZkHcCN4(8SO?iBYlcgZ_19>G%FHB_|6f3mdb9o4E@IwGbMm zfP=9KzlylzX%pa;Ahns3lN~=RtE;Oki|cI`TL)8Cc0N8nRyGb+4i08;1+$~Ojgx^J zvyCGSqKT6>#G#Hx4(4`F=C(GJh&BxjZJnJ2sj0zv%CqrV+sVuS(Y%f0DGC4&RyPAX zR(2LPR%>h4zuw{KB;f)YIUUeH-r;y3Y+JIbKpkzJ9gLt7E>IgMn!h$-Z1l(dcFqn~ z$I~%3Vue~kt-+;^pjYb8?SINusi`yDGBZem5lw-{STYi!KT%*khL$YsRN&dy2V(j!1+a$ zWCW=>SlIr2N6E^-$pqXWNG)e>_zo}^elZ8Afs?JneOp^AL2AU1C=qua zn>VGv$)ZS`JAy0R5f*=H^Qut$lUFA$U}b)Mijwl!Z21k0PC9WkaDf^hcLeS`Ib~#K zU}Fje{(WjtXYZT;!(tg3Kn+X`*bSLYjEp&%Id6k!h8(g3ep*OeWi7f3SBb1 zCnXNyfH*|-5C|ni zMqKp1+wj7u=mqfNFr=Z=pxd8KP~cw9{azah1@-#^6%)Juk!DB zK+X;OZ-AWp^4|eDr?0;wa?aR)L*zV2{SA=wxcoOj&ZWrT0XdfNB7Z~VT#Eb+kpG`jq}CK3brEsvrJaY0hNhwaTnvd2|F;?n8}7f=Dr%+w zwG2Y@uN9W_j(=a)bEtpEmiuIf48R$p3;so@IK5pM4hKJahaVx^r>=J0Aa+xR%JmXaQp2 zwUoZ|G7C&GzfVFkRpj?KetSh2`fndG(Wm{_Pp`!M>!(D{JN^x6$T`%%<8f|WzvJO| z?oPi0a!w$><8e+PzvFREAiv{rP9VSI0XZj--vK!%kl*n*Cy?LqI46+b@i-@tzu|FC zAio20P9VSI@&6Wqgj7}3Skp)$^X{7zT$Dq7!}ui=N#^=sR;|j#Pq&sck+U9sxh_uj zVpd8Q7nk5PI}^+x1(xPE(OCjGC~ z6k^t3CXjPV`3F$WDditP`QJn-5?8TqoP8GHA9{hD^Z7ra^1lM#hv)Q{1$@*QZa8$T zY|Ia27B}%mDO*OWd`&q#c&6f{L%@(O&CqDY9sW#Z-P`uXKXp^v;8V6a4appQEa*yZa^2_96{ghVs1dym@r2KU&lkJD_sDc8Nl-OGTss;h6E!UD;WvEtr2Tdv>uT!wmpHMn(pGdZkwt zH~Kb>&Op^;a)0>{w%F@?3oIBJ+_91 z2L2y?i8I_#&sLjB9N*VckzH9{#%8#26{N_dfr*Y@2fJLL%fz);aEz2r7+5&~chOMW z+VwwrF_k%;%R~g(w+o)aY4E-6{hcB)n$s&LzWhj>o^Fm-wiJ9}Da+89p_l>;h<=AL)Nw3||=s_#Zb-76n+&@H2EFwJq09(`{a{;{HLKl7b;C z%tMio0m2}vy!`yMdH(RLr~OI&1Wm@Z{;1O(wR|hV&Pm zFnfD@oVA_rx_{cBLhLD4wJ%%vnk|aMRKCiLf|Uz0uw$b*ZJ9J?!-K4zjdb4`vmP7# z&>%i=1-EN_um|}AQWU8BF*{!5z_Lt=G~YgB;y=9y0--xw>j}AMDpQ-A+?#s`GUG6Z zvWSX`JHcY7*VX#pN^dac;7gbvwc@t-_O_k`CZzY16v^M$1h^FdZQ%;)S(_8T4optJ zxRYU9rQ7VpFSsMEFWvy=+hUA6zcKt7EFa>i^=C|PXpW|co=XTe;H}^VEi7|8bQC;b zGAwwI_`^}UgU=#&><^S7q*bn+;AlFz+b#iIs+F7i%pq>;*e#~!=Dxlla0dCzdO`{ovCelVoFd3mjaKU{Zu{B_hB{lwXQjQHV=Ly|}| zhu%s}4rK!K52#PCBu&`x@O~Ck8xmqq8ofDu0peHxY1GDuKPlOpmThW0=FeY;;k|g~ z;&dhzqfN#f_Vl9^3oaEetWrI%*&nfvq+>l2Wxn&uoys?V=OKWOM z0Ato3S=Wq%rHD){GT0X#-}-BlzcFWXc0cj_20!c01kQzv08=GQ5EwwMVpeezun7sIqr1P2vgMD7-vu_!gm3TYc&%6+c}J$WA9&f# zr!o=KW|XJBFJqP0)Cgzo4K2DH5AUHY1H!0k#z~s=N=r-G{`^{*LeQCG;edV@7p(#A zU-(F*Y0{qoaF5N)@XuCxi>ok;`@teT0Q8_%Errx<&ly4SYj5l3BFeIs*~|&X*8&zMpPSiZG<_-rk4% zd@QW2%xnO2v2hyzbiDd)|7R)!LPErUj-GDXmF8s1yjIh^h~dv9VoVdzr#ArqH6Wat zoSeMjhPWVOx$RDr5#Y1gSq51K5L)d?^$z!sHUtS@sCevfrQ^ZJYod>XD;2nULQYUn zFlolydvCLS3Xr?-eaq`tyl$j0oEa>{&f=_qhY32o&|S@71; z(b3cgV_*$MsUB-=3`7j*{hp2Z+{YGsF_+S9yIv;Qq>05i!>DG8w z_9&y7M6AQ3Yt_wHxAqQN;Tzr*-s8<)LeWE3_#l2D*qH8JzXia&cBrhX3gCe;^?bLQ z=TB@EzE;3sFjJ$GzIoBppFPM`k8$LVKz^nY795OPfTYlGQ*s*xb7_2Z$IAxH_9(5N zcx!8mwV+B>!1&0Mb9rj&bLV4lza6!DcAt#T%`$*YPlh5sA;ibu=tz5Jb(Nz$ujq-L zBmfY-ULQLjwA|b*6>!oec$|2ceU7_JT217 zMfVRV)Mk+|fItp`;?b-23QcsJNmM-aX||65230gviSjl-FK^%LgiB4WDn`jSV6@}H zYmg4H9Rkm-Ypc>8uq%fHy5p;C4l#I*{Jixm{dLo_Y}d1@ z3=PI*rRjIdA?X7@s^30daT`UgS-Csyd>c7`b#QQ5QrD&6P06!X@=q(nMDDkzHA$~j zG&g#FQQpnXY5Tert;;~YzT-D*qK* zv}tnEfL3PeCa^5T!I;6+`fF>Ccq(>jv?2pUg44r{RMh|E=^Yh3i3rKGFKd;21kUr3 zA^ArV4I1S*w$)@S&xOOceOp70U|TpfI~gH*zPKyC`vOxXHRj?+7{T?3knQ2Ym=(uZ@p2FOnDn_MyJ`zbB*;e7Kv{OiW9i>oLAHrC#T=5F5 zfJ^Le58zN7a!%!zkaU-rCrFlzL_U3bjRzqz;`wg@)VtwZMnWl9uT&b6UNe)d^SB}GhZu&-v=$Bm{ zqu0^`0^+%yyJp-D=F0Wa3RlIyEQIt~$2LHdhuf5lB&LF?ldV+gNqTLkg*+s28(sgo zXzHuJSz+qyg3J`(3g!eBRRFXO5pWfvWRq9DNNp>f4C`KWG2$dPHv;cxK6!WP;#irZ z6b#G|2dDau7sl_)FRj;mia8HW=bFj&w>2A*4HF|ca9^j}zK5?$XngkPLsE{!8WgLq zt&EKKR=0fPjAUx1KK6%oB?#n$s)TYwON$Itcs#yl{|6yvFn?H+788EVJM(AIEcQ>d z$LrrTZBI%@#>I_@&idnOf{zXOljhzYW3gy&ZkTFa9on~Ib9ebC*@D~AfgJyREeYHw$!G}JjTzcID2AScz$>z>?f#;Jns&>Tw} z$|*m+ysU(PjZ<1PZY7=7JL_9Yw<51_?x+pVTq3PZtlR!}LDWS*=2^rNyITt%4Gn3o zdkB(U*}PLZF)=}@ps2`o3MXhKa{Z$G*~r5isez|B-bEVsxHIKh6S2K*I&$?&gxjX8 zH&|SX{kb7iGe4%*)KQyeEAF+|0M~0_-E}TM+{q7ogm(|BVfoZX6L!4!6G0yJ+J8@Q zF5SG2_Ynv`ej%pg*FH&Am3wY2ng7{0@x3#RacRb#9E=~Vz%!`k(baOWzC6_=D+hWK zUK^Gbd!&oPk=rZzG}SxxrYTt$zHwTszo3Ocir-L@SxITevA>i{9fwOL!-(HLGBT(b zjT*9g#||b%Jkj!$j`)oUCKeWnv*7U>hAbbhB=9FiNuJCZJiabCb8qeXovEYjzV}1~ z8y%xyx{eKd6cXRxf0=*lIqtFfa>=JFXp>3F-~_fdm}8`} zJ#g}h(NC;EuVwP>mG90v@dL^swr7@!ZK$jBSGWlA6O4zI9n5lVsh_zL7=szd8P{s` zmQMZQQTs(9JhmF}wJP^JQ}q-Q`y0YSdpQ_SpqhYoUbLH#(?cLO+M0317i^@J@eouZ zKed)P+nv*aLR(F~P{=4iOI`X;)5l zd+F;!CL@lQ;x*&83M)shOOnPl;ciEORJc5y*AJySh4x0Vh`X;#D2oFc?KY!*0eT`C z>V&)+Z{FBDAdGHKF1WPVN)QhIMgl+|$8*)gqF9TaJlfJ5qi2>gmbUvPFjyGzTGA@n zoD)CsY4`(;NM1Xw5!hWgTgo*P?XHKmQUGyDuj5r1*y10nBwLw`rWV?tEwZ#gxWPF5 z>P^5DY3kX}CH*k?`S?O@O0s_$)dQTSS9ote4SkIOBY{5bE=_N5cek>2z~-?7I|F{j zrlX@Ho=n)gb++(e%zrLKCqiEXUIq)N;J+PUE9G^hfunb{_S30%bc-6ncD7z6IPt0= zE;mmDhDCdb1o?oYm?G47`NIXkKgJxPlXmpdd38V6K#0=ubO7TZUg%luoX-a4u(D#$ zEW9^6n}rEnF%@WoHEl8NfF~kg2q(v;|CvR^KOi`SyHvC@Oe`O}1mjddO=M|b*MK(6h3O@4jkSA`XVefL@JtGA{nNP*9faNC1}7r8*#1B z_9I{c--9-w`jXgNM(bS}-MtHVQSj-b=7O6*Wgi^oBWRoTS&^t2!wa8pi(MUKRO(#qu(6pRD|{a@T-!P;+A0HNVoY#No|!lD8O%eW=8x739_v92x)RvGA#ya*-JLz9T+qxZ~zgB zV?b-Gwt!#I%BX~P7Gbv7NrE$#zb5O0Bg>N&GOnd~t;&#+l3M+M^Me8IcrE?opj3`o zIBguxL&r7^^vMvMddd49YRy-$#6pJ>4RtlFIxD0X7=RIQQizR1?2wNKOp&urLS&;$@8Iv@)a=6wNCuEByhC9#y(vcvldE z+B8&5pp^lyB(nRyWu1)(VkdS#?4+){DS5nFK<=A7&KmWgnR+E0v3)3vi0%B_g$Q1q z)|2wu?rB1~^v;|Z=m3x>tOK9xyn^;PTr(I2jcCFdFlT+e7zRM+gIGu9W<78egq_UJ zny-6QksMTXprVYekAwbtM=Xjh8eF>{=EnVn(gu-EiB@F#x&+R#T=8)XyEt~HnP(>~ zOMZCbP#jHZzT%$op#b`f5FM`>1wu(poh;y&2Zl%&kG>X)&_F@@FeV+?e$dr1D(M~tQIGR z{kjE6a==0Kt%XDc>-{4*Pg5ij$hbZ|jX2dSJUA2wts&!w?>6H6Vix-xzlWFOs_luf%^2Q#XmDBVusE&!br(>223}Q{MD7vj2%hC#@_OP z@CeeIKl3cWnQ>5{s7cAm=Zt{P>3uQbR7!OhsVsBW>HYEGBm~@7#c8mU6zBkX0_+e# z>RTo(`?v$4Eq&HAWH6T1>=YUqFycCW>;s<~J}Q5`qi~?gik{S3m+$E|G}nZb?ui;D zhfmrK<}AxWAS`ka;>uw&&eaGzDpXX|n0Hx0pvlb3a6yF8&{gsK&;QZI2f|{#mY}>vPx@&yDUrL2ygclRsSh{dukvM&$mqXA|b56)2LfHFzPIC z$lx|NHxq+#zrK6Sqxy;-TwoYL{y71udU{xez7{!t`g#+j?=~eQ(<`M!7YGsYtI3rH z5eD*oTdy2<7jqTwFc7Oj9(FGF6gR77(yX-z$n%aA9CPmUc_r;dAz*-Lty}S#w z4dnW9ZNN{*?DH0aI5TL#yiX4O@H1*JR*Qqm-Fymsy!}dz8%R&E4*8}IqN#1%pQ|!# zRbO9OVWmb0-ByM$YQR81=t^*2fDl*PA_-ZSX`@}4`3&L*jSMcr5tZ1-?7@6LT$SG@ zm(1wtjydZN@&Xfh1A9#a;=oJDw&`NkhD*bi57Ej>C-qomZb0^2`v^#n>8J9K zRy0$%ME=UvoLy;B(VR$$2#F&}gHabKm!T@b0Xk(gE7m$rAzV_I>NaxKTknAU?(`Tu zlhGo%K!br=5RmHskWcWJvVa^NNBD4~sM^vml^&rw%s4HY94|s@Gx|MhDN=%cYHY{r z13@+@k?`(#O&={Rg3bq@(Z<|;kS6E9($Y0Umu;!6OXqIH#`)sD>&2-if_}4XNX~eNJ=-b3@qcUs4kNy)%<}9uIS0-#efW zOFblT1hNH@8qE9V8NK?sg@guc{&ZE5kcn(rb8tN&K2-hG%dX+Tel;hHa^z*MvbVoP_GRwx&YUQds z@h1UO0q!@|3b-!g@Q9Vf6fik1R8vwJPdN^nqnuR@)Dflth6S7SdsT>uuAQsCvOHN2il@&Pqi3CVy|4fsiWKJ}6x!`?W9v5WK0WDGxbyolR0w(qf~tkk|HyO)AR0ZyHUce)I5L z{S^2SNa?8#i$G=&92^YlBiOm>mKiDOJ+tCK3$3`3g8B}q>^B+~6tUlui3KXsLb%Srk+$zD;lzbr}(R9UkB+f$TeNbF}E23CA$ek zfkA1;Syzn%;eya=pAZ8{*_dg$_Gj*k7?Fszmng75#G%IVsvVR^`{4l;UU#VXlAeg@ z!oiwS98phAjQ1|3<*R3dU4&h1$ZN@Yo-J9bLB6OohqYa zHCy<+W^H}l$Pw1Z)s{!2ghhW_Jd=XK52!IP- zjo<&8(l?jZ30ni{A_ed|5m0#WuLArGLQz%aX89$QYBMrI-XsOvb>gM*8j2)^W-*{G zlJQzcv7j^2Od>@qG<<>GTtnZ+iX+2goU zDERK+2G5Uu-u*)Mp{iz~$eO0H(I`;@O_I6r-5e++N8$}}XLJ#l->adm2NjS3oea(F zg6OsMvO<3EwW*^+!=f7=fuG8-h(J_aFNKk$` z@vr<~_2mP8q>wQkT>1euS;+f6&fqQ0xJ7vplkC1QgwzIUQnrw;-d2_8mLVsxjz?tg zN!VdezXS^TKx=`(?PbDzU?|!aE*(3w4$O>JTz8Bz^)eeH9L62hO zoqRMCPW--_&iuj-)CA$JJoW7OWDIR*=G{W?uAMSS!Y+D}UXAV+X`L4S#FPkQg^Up_gxJZ-k-!uI0>#sV{) zoae8=#tfjAVQ(9_vwqQD%7*7Bon#DJnboWf()ippRC^$FJ14Tzi_{Siqh8jT-Yhzb82=2YR=H ziV&ddv0rQYd0tf;JNr9Zk2(w|hKlTcmUMH~AMSV#_*(&=Tv@7sC*-^PunM`a&g@i` zW*>a@&@E2RS@Np(DJcx!@e=e&s*&>Q1T|xD4Dy^2RnmK_2R)z&Dd$7H(w9F=>jGan zy1pX3&x2EQApb*1>m&ul`I*eT_P)jk#L=fupC%n-Zc_IL5?vSYHvu_YJX+U#koYdF z?JlJ{Zh$=DGa(FQnjnhgAHYD+Sp2NIljeM()*8RBJOTtUuoZWK#bwj#yOVueYaT49 z{*CFIH=nLQb{XA>EyDODh@X;}xUle2I+hmX1|ZMlw)w6TJg~qRv0i@Q;KNiHJd%To zMc&JSGUiD(*v9|ytK;XLB7cR9wCT+6j=+~)%3ZvbfyaVEPpK+0&(Rt9`F=1?vcW^oPgxC*KXIJ$N;R<|e3d;0JvErC=&l|DIIaRlu} z4U;MRg2kb3O8$MQUb&BykCW4kC$Xm!1-#vECr-lMZlQ-=q8o}+7@?zL!AD#d6*I zo_kc}b2#^NVIk++)-RU5PaURf9q6_$f({N2xbDdVt7LWj4{DRmga>je$RFxy8}&^_ z$P=Q*-=OarZXc|8S-enJCkm1u%^qFR(Eudan)~-MN7H$*4{;d%h6Plviv^G$rZ`?9 z;R?1HK9*g|SzWn6A{oP%`UEV&fl47bK46{2?K^7|b;ymHZ!UlmHW-TeWeYJSN*?CU zHBgw0yv!3m82vc{NO@4cmAkNd6EE2I+Y2(BmF<=3d&0u7tFH+pa6p6spD9Q=8plat zc|5ZIs;hyWQ|@z14AiBWKYVqS%izCc4nR`;xXiep6r}wNJpFwax{wQb28+Yu>9Zba$BPAHcj#VrfBn0aiN{RD@x<5IG37A7czI?q3mXFQ?xH zALv6=6Br4X*AVp~@KKYo8XCP6=iJV+ME6CPD6jqPm5#&1@{}V$sItK#{qjj-?{g(` zH9IYfwO01VkH(6eB9 z^t@MRS~99jR;H?-x5AG$cLYUs?uxTk*j$^iU)ihq3Ztml7jlS$0%5?&lgB-~7cW09 zR^7IHuwDVLwzPG>poS?z9^Gtzba;>?M4{%*;}!$c7u4{?%%O_x`{{VPEF#Wc%BLc= zA{P2m((lp(miR|V{#6<$T5Htz(mvy!*^eutA+Mes%N9`D>V!ps4=8}7QCc5^lR09p z(Xw?-;2Wn}>ARmmBz%1Oy7s2P|3}t!2U7iiZzR!DQlyN8N@n(!QXw<4T@8Ehd0nH0 zlnB{cWUuU9Q3#cBadFLVTwMF&8ozU?&-ag?zxw2p>-`$%Jm-0ybKdN5&739uw@%X@ zbT2K$7e_#802thl{g)oDcHm*-U4XN@aZq^j{<@8|H_`*1M1nRr8Y@{0fww_3konl+b73nnvY|~WRw2!8S#&_J#ji3-) z_$Sa#D;dT@;u514RZPK!y{GV#qj`4i?dq?%THVRJBZ**ifs=B5$72wE%@hOTk#0~N z5A}5J=cOq}>EwXlbuyq)foBA?12Fu6K;E$km|BVI4*l0qeJN>a1_o~x_}(Uo2*vf% zlJ1?z{l-jUwq+RL&`+^577Lhk5{PyPA#a33)#`2Fbm6a;{c}=7c$qc_^mpQ3sAp9T zlhr9utf6S~tQssYjzAh-v0r3sTa*QD8ScNT!l^3S;+mVOz%K(2Tf6jL5CqC0Ok}-& zE`6e?L|#X0N2*Wmf%v>D%oOdZ2SDqrkz|ro(D-YRxaKE>cIsSD%2n~J+|i;XMw8DK zuxh5r_N`YB&3K+F%dqR0o&Y?tk~w|9P9(7{CH3NLtDlzD>~gJDnMH|_9BQA@M8)s+ zWSI#0L<1s(xZi@sW$uS>X=1igt{KL5U=z_&Tyu>+zOroBs|HsB@_5D;*Cj5fAEpz% z?KB%O=~?v;^d|f(e5b#_wHKojeKLBuzE%ps_x3^~g}^=dEW!JO@7OW5d_QvAiPnSR z(xP{eWaVGC;gGFs&@e=A(8#3M_Ji^b6p^o<$F-Yr=K1Dfr^7R|4a(fBd|*dD>rBT3 zz+<{U^kj4@&`c%1@>w5OIHe9MiY8l31QEF#twAB8C zfOBsd=Wg!lYF^_+hy6j_-eE{jX@BvRP8Lu9p{C7P!vCu80lTBbq#4!NmldG(p-OF{ zXYsNuIBd>9J=eF~4|a^+4?IYZTSUV8$&=5&`VpU=S1L2bg||szR&#ig9YhBqOFB)T zJb91?h&Nj?HX>xD93o9f!=h3ahPKyXecf4z`4xLmQ|{%xmem1r{jeaH;$Q7yxs-u@ z35-h_sS>!3=;kknXsWx^XO;0MoIgSs>6y);F14uemk(`f+Ud*35AmPD6)OAhTW7{IfgN+%Fb?k9TQ*LPZqbXSjC^X)|n)Hc~EeN)-VNByq? zGVw@bBD+15^N?WcDI_GXi7uOr08SEC{yOYGB4q_yiliQLc2Zp$aoR04^~JKmNOD7) zkw9EWc)Sc|b%UfGh~J9%dbA_#HoAo4H4 zhGtn>01uf+Fim+Va?5vHmhCxlJwBlWN0l9@|13I@?bM2t?6`%WF3w`^x=)I6qG_a1 z-c(hnpEvTid}@asnhn(Fl})J~HE(}gMg6*VnvLALO=nNtfnHo%^|IMlzYjC>elN2z zUkTmGsU5|9ZpW%EQ-B@>ON=P+An>Ji;J?N3Tl(|j8byb=;skI;dy#o*bP7$7{D0z? zS$?ux8202vUWv6R5j^bS?{6;@c&e)_m>tX;H#i~;QRij7d~lyLrxmhnqOA~sAxkBH zs}F8}qH-#A*cTGCQZrtwE2&mOz^j9;s)Q}1wC1rv{xUh@UptGhIpv6-_vWi+{E)m% z5*tG=chEwpsVHqimNzdZH{Kn|nMs;(1sN23lEwM070stB@|eyT!F`u;=`{UjJJV%+m8v$`0{*9y;-g+U4hkcoSK3^3_rR(3u->-i-L> z*yoV~;mgrl-BRSZV~PfHO;p`Os3b$HZ5a|6;KH8?VeOX_d!00Rj_r*>{H*P_SoFJS zbqZZ|?(ye4H+halXbUHr+yj!#h{?hp2-Zu29#E zdpC<;a!9Uls-%Z)z~K|{NuVl)H!9uJb>6PaoNum}$i`U_x7ub>pSC4Bq0?!!NE-1Z zAm-q!m;S$_ag2wiZTzvd;;o(wc=KafbiISZ{HZr*C?<#gFUKZ=#XmXM{Vrb+%tpBjtxGk3!{#SP2D%F0nu02)qUsnyU&>MH_wZ_7D^mK7H6upO1DbU7#T& zF4^fRYgb>TWV(`Aq6FLXQ&O|t-5&eB`<%=iZ)#gh=OOc@vTNciG^%cY=4pBiNX#&{ zyO!Va()l@mMe|sTriritQbwtJqWGUCU51d+LQ7FHyiI8S|X`U zNZ-?d8MI@$N5IqBTBPRek$F&UACNl$uF66Ka0C+&?Wu<(-LPAgR+r+v^L_U~9P%O^ zuR`bdGvR$qB7ZkN{d$kM`KafaO@i%H{#*oH)m-#TT*p~iRAP)p{NF77f$LwE?pX`k zD;munzsG~EZhbQV{&j>)2)<7GgfV~(Ks2X1>GcG=ZZj(99-!eSXX?FK@C~uqn|F3R zJzW)Dv5HxVap^V?5O4y=`(*l^9G%+r$F~=V2^^t`*kS;S{rd(aCIaf(kwY&dq3=Nk z__Z4(4W4KmWst5L(mI7Y>!UnGPnLV(jM@2U5m&?gy$B@oU0dL?q;Jf8C6-{^Np^ft z)mqC^zL9W}Ws|{W&2L#ah_Z-TvgN924cnHu@)SQ5Khdw7yH%b8k^;S;&F1C_yHtEG z>!4ee;jNzeDv$QFrRb&y^?#k2Z z1+w2PAN@ogZpbg9?_l^h^^DFNQC~hEZVFX<&Z?y!-F%g!JNCX|ImLiKCudV0BU^yN3nfRWDf~-drEyVUb8P(Zac-8#tiX<-U5wM;nlUUH8 zBBb3=#6ho%+b3sO%M{oRF80_xHF`Yw-X?1#&l{`BbL@~yv-q*0O1~YjlH0cNNA@nM z<3s6g4#`sbYt*DD5DsuZ+|>;CH29~L~1st>7P^5t&f55%c{1>4VP!rK8mOop?ISkKwG?vKHZM-vb1HvzAc zW7%2tTMb{DI0Fx~Wqz8@F1EVFq>ENgeF2wQlm+U99&5Bc;ivVxGl2(EPlAxme2@kn zTa+c@DB9za(f11r@6TR)j9pDt3_qLxMT-bpLO;sVH0=fc*7)|~1-4RSs!C7%WM@WPfLTK6*3i`(=1^EUmK{Hlw`TCsfKQaV zKnwyv{O!FsDpK&&tPR2|-IP%os-mX7Mg-LqT+g+7J1in6{Bs8wP)jJ4wI$EK6$S$h z5WW&qesmsCuX`ySPfF9>xbCiWu^)9k^=v6j{mnsrf6A!YxbKg2-0h;ogHcv)B!ZH!=FTLYI@n zJ=uWsaF$%s`g|z6Gs)$r&)zA^5+eY>db)OBAMBmol3J~yv5Yx1)A{kj zbQzh_pA9I-%=BMz=mjn(y=&-h;gKaB${lCm&m%JZM1*B(=NA#8S-yUAM~Qp~LLknu zi&f$5Cn_(}Y5x?+1&|hGYe)Rd0tvY1``TNwp?D(GX-mjXFtxT`N-ntv@%S&TKK=5q z6bur!U=HfzEGz;!{NJj72VkG43xrwKy^9k*W_WfJY#j>w`Jge&#RN_j2FkZx?Qmn9 zEB71b-Ug5@`sG>wmY$vcLrSM-Zvs-q!97QP`$j_gCqA#?H}{mZ86)`b`o#Q=0>{FB zD26mBavXNP*F$1;yDqoh?61Y2zdsvxt-asTIiJIXW(AYoZoT*85Z^sPkJ7_TZ&(1o ze76)iR2U*Rp}bwuVE*G<&CMvTm_fYXKGWpQ^y1aQlFinpWRFyDuKhtmzcq&HxVq0BY5_OdGP`e`qIOhEUtO#;g;&w^C{_br;$d$-ID7-Q8^g+8;|0+0!IB57%dbi$SbUi7T>|7<&UhJv z>#5kP^FlRpHVL!HAf7Wp-RVzyS2~WFS&1i#Rchwxd@M)f4KZS1t_vfp+kp}7Wj&JX z5%`V{oI203%c912O{v=twm;vu@(BZUW3j7knOT8&b{;_K4WICqBL5lYTDa4jtOJlK zdogY!TL2YV?zRF)wiIvmg+Nv6R!1`XRV-5LSY~IEmj>NGutO_ru(v3AuV=g3kbUCc zLv4J_uI#uAT`Dx4Pk~MHG zg!>>96ObAqZ8$f(E9Upya;Zw!pSR^2w-!D9Y2Ecl%(i>n$I+D94>&o^V2%05F4tyfNx;x#hvX?DC{NrFUs#s25I7X@&0B1IM?Hrm=+oIib9 zLVJ5V?^?xo+_FzCajnXFXQS&K|HbsuJYI!2du1_-i?Hlxg2c2lKAi+g^61{~)}jHQ zM0CjwlPIqJj~ZfePtuQ^S58imA=)9&M^%uyWSWqm<>ABk4NK#cx6Va}M%E{2UA3<= z6C;8bWikyldtG+a7wC3?)qkWkuGq?YyX&KbXN4#p+J^N*7%Hmq~sEa z*QzfybPHDMsVXxOK%6XZXoH>uZ2~hz2a0vt?FsF2fdL;JKgT?d5{UwS%ZNdVLu@`rvxGml9|C!OmGJiVNa^TuuQubj?|RFCm*4wRklt z*VbDWP3BEkKFJ^71{k&f1iH&x_V!i#1LRBgOPCXiLIep|fV^cwnNe=b361+AaQn3m z4yywFrC9vPFm<`UmLct6L84qH@uu!iAT)SxkGV7&qCHoh3xZ9_Z(eSIcY{W0*hCJT zz;A!vxKhAG^fj$pM0QKL2r>7fb#>uaOV+r!)>nNmlfWzi>0{S?R&e~k>dgY5-xsgFF(+WNLpv>=Q16@LHbysdHl(=)g3IjwA}!?q+lbop!OF}lml_=9lecZ{ki^iAVV3x&&Bu~%BBhk3`E zPYYpu6u?a9)7o_mZo)hektAA;_O#_1&8sX88 z^>|||=n45}T8nDp>5-_sesB|BTuZ^o#QGZN-X6-|Ey9fdxpaCfxm18iv!S(qrOueX zw>%Gype&o+lLRt-cfPk5hzqwmhbrdVrl=corw!a$N;!!q&NLdOciTEl+mjVK$9IpOUxBk3&Y)YvPCBoQ@6(?} z1`g&3zUJ+@BdGLy3j4grLjvDubsy0_aMqJ_>IqO5H`AdFX&HHn#R;mMt(segu}V}I zfn~qFP!?_&8{g6Pm%eA>{8vy`-p=OMI>?gVIY~>g*VECo>oHlPEjj-~6sjlTeU8Pi zz(4-N!jKNx_Y60Kk_g z4k-xQdC1$9=cw1-larJ@Vq3cp?;nUhs#}A=-#Sw?{Z_VPrj5c=XvN4&>o|nS-YZw2 z84_%yR|X;%9gM-igIlaApS_)3QcOUBX#q@NjHj=)NJ4AV4&WV5-fu6D|iC z30m4T^J;6g=l@h87+aJUC=E^eKqd*O`6L1%>c$w}$A)&ToeJAfJ%NR|PpwR$< zNc>$XagvdFe!BX~$kuP8eImWlRHqJ3xEK7Un!A_GHm}WkO!Cwde)*dXZyJ zDF|0fOCO`~=LiM~679)JNo4K0X?L|8ACPnH7HVbo+|{BP*&`bgQ=in($i%*-H;%cM zo+riLKAP227IzOhbVYgjO~en*vnSK#R`@tu(o(F>9z9wW%l7t($-$1OLr3Wto|;MA zw~S+|DPVkYs&7sg1pG-hNY$T31ls5ZumC zMd@#&>$0XbvCs^&0t1zWFKLz}Z#f!OD2n70lr28p>DvfXa6oa4{0#c1gjE4yrU_aoDwM!sy za)Ni{ErY$;vP_R>6)W&HeYQ;yQjizsnuL|!Oj!bq7L4Aj6bSLMZhk_GWX)!EPEov zAgbD~wLM&u)!c15iL=mweiryj^G&_A3xXAo14UxN zJ|(Z>I0hCU)!%tax0Zct{&-fETtvg#zm2*m;UG^uy!(vh0{>nS9Fw^e)m|q5WOY?g zjLOG*oo{r66tn50LKVcDi^lg34(c5}3HeZk+5vXekjtZdR+0OHS%N@oQ7k1*8Xc&a zC+a?@gjSNz%16sueTHvg0~3sa?S0pVxF##*A#*@$|Jwk!?C#1~RA=AyEjv^o8mD#d zN-?yJ+qWY|9N_AFpbq4ZKL0oiAdpJ12)2_2k(ck7==s|$cA^LGEqQv%{v`B9NBBLn zj%~dALXRCgI9TGCvnFf$)u+=xr4DFS`CU^@-ovXwJKL#f0V)4IYP(6lmU%up)sFVp zO7WA`b=*b%Pv$~D4l7zvFWYFyWS9di(`IZ&6Ya&TN>MV6SJ|LWyh3~Y8}|!Uz?OF% zITV7sMX&Ixl#N`%KRg|LNeW|46Y5>QA07X|A4tclS0X`=a@BJ#Cpy5Gm#wYZurzHQ ze@!~`>4#ntV9{H~oSSN>a?FEL4-Cqs>k=@PM$i8@ED1OUKF}7$s(muUjW0`p2C#P6 z4~4>3>Hj6K>mUAm*_JM}li3}zUNbb?_~Y`0eIxb}hXebjdducbAR9wM_K*f>+HmdByOK?|=as)h ziS$?w(sY|WTU+%ziro?#4X?`(zR`j3XPn=3x^48JrbcdtLe~<7suQj&IUfeo1OPn& z5QIRSYh1?3ISReLN9S9Ijdsr#7|o^y-UAuI_2NN3C`mB%V1cRr`rDr|=gL%tB|FW9 zgnE^fUFqJr5NAyzRQS<)>igx(46n!S7d7=N_w%O-@OXK6cM8;pGFoL;HXPslMAbW+I+ssj!Esr0r^uy4nprMzVGpcTevs z`N0AvWFp%98rqm+qG5^=n4g=@7sY%oOIBaMeO4J-rGSf_Y3cs5w$be*D6!~sRb*BO zE~uoWNnGV;KAPzC%jVDe~dotB+$&yoOm2 zo>l!+AFj?`M(%VA9VgBXhHGTFy&g_V96kwY66VE3<>$X-NP@zU?5_B=T(dKl$i%Rq z6rRPz{0V%7kbv&_I0`MyL0%dsnR7U{oOJlX?$R!Vp_;fLVQeFhbz8RGVk>m*OQmcb1wW@8!QZGXbq9?vgPx z^6QB^=~ankYZY-_7HP6pd_e1AQgos{)gP~_LA8Amf~^Y{Q7@k$XojR^Wp9de=%~_Z zzzGKmAlnejyMysezB zE1}ElBM=chvKmr1x`6pK%ReW4yf5!0L}k}zA2)fjp|!BA3}-)f>&8%vMQT322bv^@ zhtpQ@DMfs5p@GdX)&<{_8%N5Qm}wBNv#Bs?iaw`lMswl2)T=Q(kc-qBk7)p~_nm9= zPF#bVf-ug26)p9zQpvZ$vkIQ(9z18fkTWC+cqlN|(o-2q!xK3UwF#RS z7Dtqw^|=_=hp^Aa-puHPq?_BF0R1$LU~)F2IZ=JX)V|%Znz z@IfBU+eq|E1?LGYI{bF|%W`mOk?{udjb2O)E-rn=Ar8}*tC^}_wRy2XW?3N;qfg_w zMYyftKjNL%5#1l%79?_E#+K*w^mMf>Q$U=^Qrp^^9t!6-=+dG0AYeiVS6Br0c=#i z#y{!j!Y}&#tFd}*CEZ+cSy`OP+*62=u+AW|M%iFCH#gsr{^*(5rvOk{E{2nhC7g18 ztyDohcZ@l>vYCIx6UPob+YMEy% zE{20ZjN~wsIR+nha$_%%!oVUGR#hbq^gSBSHTj5jaue)?(0Aw;*qouu2Mn%5C4^k@ z=6$5it=6=*3hv$jkAn=7tO*xN+}2xj)PY-r%Y7i3iBWmpm2H|x62z*pvawmFiaM7V zk-c7z03jYf@o| zcj|ZRWReK&OyFt64!nd-jL3kvU!}`ArQh*G!fqY{9i)_U_d&Ra;Nn|vk8S!m>0a5r zRt=rv9XP>bFEeygqoV)ccJ6fdX!+DHoT3l7Xy%XGcXWHH4t>6>qtogrY9pUlu*qEL zh|qx#->pSkewUH;=EIL;_6Cw$%3*{g3Z;-aoDLRh6|VhHS1mZ>yF$L2H8%eK{Q2{{ zfJO1(;n|s)Mwn2Ozk!9hTKBs!p`-HZ~RS6=gRn!3q2iRkG8Nn1wtIC@OR-ifrAm9 zDbZ|P@h7Q3J*2@LKZfU_UC(ad&Njjyq6ryqSI72DPEM9|J3%m+boZUuo`pgnV5|C> zZ$%dFndP_1&3^HY{g6~apoilI9}70Y9HIgADf-r=6rRye=@h!M(ZzN zH9Q;q9u8eE2GeO1c=4nk+*jz@A}HN+if(t#R^@bsWLfTk?}^#20@~_^bf4pKECu4L z#HIXMFISa-YANH)sZ%j+@g!%-o3`xWBLM=hw{@<4bF&is_5D1ul~2IB1KqVnbyx!N zEf}R|XB~C?&|pi;^^b1pent-q;pyoKP2zRS-i3x^sVOO7T6Ph~M|bq2?Jv$x>AjR7 zx-Dp8%FL`7rgm=F^*U^wE>(FE-`Jl&A7VYqjdF{sKI>Y%S%$}kb#Go9@QT~H?Dcx9 zgn;>Q#D2{2%ggLmpd|eFT`Y zl(+*`Bo!3UC*EzIZ?i0{t4m#SvYyJ`I^8cY8po9Pb<71WL;p*g=pO>d;n+8G%|Ize zwvD_aP4i@qL(_kDIg;dgHXz$YXY;LKog<84I(XRu>Mxma4y@j}ckAy;hF~==HPr0L z-D5_v{3RY^9{HhgFPH`$iK^@CK-@#s3Yg4r-y6S`X+s7|b1);N$dLtvE{xCmhf$naOqI1-IQFE=Egr=yqJlQxiNkEx8wJ*3jYQo}BHOlW=nc zqxlitwaasJLICnhmwzL5*KOw`Y|^c$QpM!Qo%1~O#=e;gdE>tXIu+wiMqD#|UIV{u zu~kRWnp|d`SL>vlrv@P|OQUyQ#V_P#d!u-0$4u*DKYjqxPK4ELqRVi%8xdOCpaOtd zjo5O_forLKSZv4L04$eNmfEEPK!LihuPgIIT-Q?m@b+h-@gV=z6cIYz6s`|!G}YA9 zaJBS*T>JvzRft*F6*0kyM3IMrII2HG!Z0!zFIdud0N?~9Jmi=JgGcl)R-WDTP%EDS zBW&O+P&!jNjqtpVX?})YjAnU(iCr&+ONf)Yg^D-kisElLe9+`kQuQtAm-6P58@!o! zATa2SJ_v_qZ>GZ^U-i3z7yVGF)*f(;HjjFBaVwqxgAQM?23%wq1(K!5S+LvEr?UJo z=~&}?$7AmTv`})sDfJ->EY-P{WCr2_Ygur<%1v1hxY(G-Z_Q>l_($h zNKmx&D`Nrb18GcNg+3ZNjSH)yT7vKo+B-TBJcd;*A_G9H!LQiI<>WW$TN4{dLrRv& znSqd(8%2HXUG!D|vDMYyJujN&K^pi9M`QHQpFbiU0}_jWet)m&oUE30gQ>FQyUR)D z!7r`-$o2y&E!l2U4yvzkt{^z#@D1H;r!;W_@2$S#fZM-Rv!D~$?3v8xkvUaTx8-)^ zFaPyC9jSZG7B>Up)|NIls*9W+h?7r3ZMR5?%CAg<$+m%hhREZc9=vnQ$sVd+F!G1I z`Rh^tx;SIt49T+TBO>!R1HNFl6hpM=HB*)SaYV>0-m>lDw9Hy4DK3dVlY(5lfpq-M zsFOJS3FgUSs3MHGKUO>mr|wbZuW!aT1O`rxA9+%FK5YDJz`~0wf!|?)!1@(yVONG( zmhAZMjO<}dtQqy#cfX%Hv0aSTbbsh0!RQAy0^&guc$N=#fa8u`&AAXn>voH=v2|Q= zac!ez&m+ekvc>t?vuC3l8?xl>hIap`O+eTBDgHLm%tZKGI5`ZYbjPwpiz3Celdj%% zD=I51E4)Y?m_4EI!J?whC?5?CufrpmYyJ06*=4FMyKQS^#|D!s zG&4HwP^m9Jd$9l@hoi*bF)uM@#f!C6;nP)shyb=07;0ncH<|+;BseUF0jWtXEGRJ3 zjPhAW50nUCdmVuhq3V=Wx&=paE~n(88HbQ5mL+wsXuORHMSE6B^2{^h|1W*+hG z>>kBbI+WuuHDiL#+;IsdW9GjE9ZhSeSRUFUHy4j8kVq}5t-8T-6h5kQH$L&qUr81$ zJfRF@e`P@szC}8alSq9I&6+gAi}#de18yat2NTviAK`_VhY0uV}^U zGIaU(%U*m1cp*wUZ`t(*tx&Rt-Ko=VfC9g|R@}Ucb%i!!a+j*{Td_Z-9RuPjxjI9s z{!xrnKi;R7K+yiilP1$PQ|D{5vD z{s8QaEVMC&%Oib#ZPjTHy||*au9=WH#USZ@>)F7R4~(|TN2~EjI^BXC&^hUzjn9O5 zz=Lh(&unn5_TJ{0Lz<+6ly&ETYcCTDvq3Bjq4i= ze07ev+fqCWhH&M)y_svwIzn`dYZ2WE?m+%P1soO7u_oP|71#mYqaaq?7GHs;1n;d4 zpA&vse%g6;scHs=ff(budF0x|lt#_U)9iFMXT zvgSkT!n{`lCK^9nvX)Z3+TW}$9+!V>Ki3d}7)^Wa#ogeC^WI5I#j64;GPm|Cn#8_; zV4x);MJ-hJ_>Dk^P7CEFKv7FY9tqxxG)4EusT@1GFdAXsRyg<@C#thUROyS-{UjI&8N^ z=@J8Doa~}yS-xSQ8oFJV0gf&48`EIH4X8`iLmFrI*eL+j#*2vlT`CX@(XcZGLIQf! zIJLoCi*%p9$t=qxKYiV_;rFwvs#Z6_^Bn4wt)toA^K!GzU%o@$(5Qv57l42(R_8EH z2-iyX1=NAm5a0oxv&m)oZ$P_sA9qPEDZySm(L$jty~v5L^%6OVLr8~F7c;oTm)JsR zY8+n=;3bUQ+F~R3*u%h?eScpCAJ`A`=Sps*yWX37_=J7SAf&()a?|N8jfqC69QzPW z`=%U9jJW+SSDglTXhh-DJ*$$xNQDK8b$g2q>m{!QRD3tYFISe@Jz4fmc8`kC?I2(C84{TJ#d*-~ z{0d;zJIeP_!(*WocMgIT`8fbKwj#B8n_pusH&n!4$Qw#sjLphWhO3xGHne<0c%@WM56sqyZ7Jf)hB>Rh}k;&FQO8Ex|{ zJBo!U+sHxy5trh*j0qR0bB+c&lEfP}@h4VG_*%!KoKkT!k^$d7#7u%sh$~Ur#lESlN&ls zQB$5(ebhxv-%ol2EtYK&^$=niR9WHD;2|}v-k0u6UWN(?mOOqd20MIbe_W^E58pm6 z;e?w!hBI+kt{4K3f(k3~wbYD08t?H7`z1q~rhB;tvEOY9leJh0S%L$vh4fN0s>RQ{xp|lPYK{DdpK2!EjW+kD{R7gN`9X!y)h&sO{Gd!g1zxgOtKz5i z%lj3#4o?-IDd<>kD+JfXWG#7guEf^%t;!95clY~g4-=ZXCC7m?7vqUZXtQjV*UU`4 z{x0%~0Y&9ZKlQ6TyU1*_{5W;<_DY&GH8Dk9P-#yejz(=w4b??_T?QtS32t1}66uiq zg)^>DJ+Jwv_zSgo#ayA#Z+79>b_+^s8mQzpMRl@BXK-LIqzb;zIv+;H>C0yJPEH*s zsUnl-y-o1c*8s>QR+g3y^S<-{ow6PP9V9(l2c4lS!_F3L1kmrjnHAAdzb(al zFiK%e^UhOX4#XtweGX>fEAH|(7Yun$XX6vhD@-iWWo|bZUVW++n^=>LAzrx9f%`{r z{O>sWSr`V~9d6Dg^%Jgd%+QO{qVbc(PaYHW&7PhX@Y^#9LUl9EpVl+9)`qb_O4(ws z>l}nRjGobl=L(E+hnU?j#!!5L>$uiB7x~tNs&pdCIn@0<3r8^;Z{foBLaBbKxLAa~ z<%B)eO*FDSi!?-Vkn>r17fZ^}J=X9(rQD;l4xdu3_hZx-LJAPwIb#Z+voEVjTt1(8 z;?G+26sMjP&Go$jHiZWFqbag?20g39K);ifh{Uc~JFn$XSf}8-^kKzsNh2e=tHvBz66c$uJpgK zZv`5fREQCpYBb)iu2DC&+DwA&6&<-$f?CzYF6ap~*mC}&HE{FypT1LZEtC4y(2X+{ z7FqZwy3w`}8|>S~$+b1fNysKsuWK*KW}*MBh@<{H89@5cyQt;Cf;zo~_{8 z-Bj;C`;??+zlAwePdF4_f%IG~!KbPa8tuP}YIFAxp8&ZGbPG)O-CW*|;e z{V=_S(BQ9Mx^|fI&s1#nT}a$(Ngh>$)9vCA#rE>&J>m*7xxFLjMr|Ep5LFCt-5;3b z%Ndcx5ClW9yvy@cVDbQOQA-G?H)*{T+djY z8AyC-5kKhdP4?FOTX1T3#uI8!W9V;xY`2hLzBBi5_k0Uf1efdUl9}EcRP6p<6 zPTohKp!y-EnMIi{!L`3SMebIZ>p`KoPkoO#CNT)V8w^j@M99tDXO%2nRsKL(yFeIl zRi)u9Twb23f!9++S#+Ga1z!fK01Can`x#QzYQih7eI7Pp$n#%Ch?VEaEf!IG zZx=iVbtfV(gfUtNx>P0xUsP`MjCH~URv6aqQ)D+$S@$)-*hB5aD~wM%VW{j}qZgl4 zwdNX^2Rj<5IsRkscjrDVpZg0aB9qANuM=%d3Z~Z&)}{%Uv!GT2$Qn9Gp=y_1f{aw= zgFz)|TVR0xlsu$)>Px{s_|v^|WVz8BMUF-nGO3WrCG=SCLt$L2s+i67XDa17H%=oxJ%7LMyNkLZb(Emj z=5!&YdHmHkQBJa*r$hkZNcTNUJXs@J{w>Ut{5gj6kh>>Uv%N4_C0v=g=(9gmx5jSQ z6Ct4VMsX(qbErTmwchH?juITju-hIKz*WH*{~gV}Z7(vg@`jjo6R6mw4mw1R{h2&p z`ff-`Zr&|?0xUdPnKbciU%n(uVgHV~9GIc|i6+IfU>w+AZIrpssr5`fij}CAsdDB= zk8-yhv)h!_sS7G{g%_J*v$?#_j6*ZFMR7;#1Yg~tqt($rGTmNf?ML$T4>xD!Jgo0c zWKl7)kyep;szUhW_hSZPeT0IW<~*yWHe(-8N*2lIyKhY?dr5m)d)>7QJaGZhlkQWO zv7-TJ8|Cbtg*Ko4b>khc)NV_y^U;4IxC&Svp}Nfy{Duxt7#*wY;xPFRVgz>03)CfO z>#!{1y2HF_E5g#edu*KRr!sgz|{0K zU&SR{EcK>G?Kv2fi5=svwyK4I(gTNxm;$x;{P5hxF`KEWqIONOilq$Uj)aaxa3Lf8 z-J5nF&*Z_s!2Szr!ihNf9h=L(?Z-{SgR&d8J6E(}FETApr#dZ}#Eq#u7yfNT2L-T| z?fD@J^g9@_Yp`E&Mdwgtd{t%sdp8xXhVBAO#}#?8vIl(m$3DJV^Uq@tLe*ZIQ!1eG zxLuI!!}%D?RVbFLroyH*jmz=*IhHbC0A)p@+rlzF?&JpEDWypRzD1NX1cL-q? zI{6URS#PCoi;{2Mk)ig5j+>F#ZpVnY8AJjgbrTnr)2wyU#tIB&G*l6-qqzc}v0;9?% zt3so2fH4)^P8xsitgkxFPqNnS_RtwoYxk|p0pOj7Krjco^#o5{mIiy!h8*6U# zKg2RW53>4Agxh`cYcMWz1XMXhx4_$6pRXYhO4VBQGSaIRng_n)R!&8*dF||C;7rBq zzaqqGpj$vU%v^G93{TBFOM`f_+DWiIB0q@Q!2W|`FweZ}gd#s(f;Iv=tfeKv2K}9r z158Tf37*hy73i$IdHWItMFM9dMZ7w#^+gKS3qJ5vx}T08QGD9El*~i@e5ka+oTHv6 z)%w`|0FZ|J`ubYO1EJd2ZeHxcmsn9>id3U2gNaxceeyuOhrPKG5YWthhFc%T(bo5p z@~WfLF5rKpCoT4Dni60L3OK3ge;&tsRxzM!IUC1?$U?PcR_|}Q2S_ogocY{S`6&d6 z@3`NK?oD5z$I3f!_qo0PkxG#ffV0TfpOPRh`7N_A`(R|E8lsb9qw+B#!h1>ky)vlL z;DRruu)WxLpkKZGequ0y9jirSxXEDIJP_%r>_JDN9(lKbB{XuXAt747TdwduZfae0 z!XDo1VpR5Ac+IyTO5m&~k1$-+vNAGY#a+83WgZP2t*ZwHKZgK;^PXk+IXB z0#ga6AzeDrcA%A`@Te@@24U*BGKAAF#1>_3(NW_FMur0_jNh^WSIO+X5FkUe;gTMz zWlUB0YpYXC_)l3f0Z$n@AqJBj2+y~5Lgun}+)+T4L5)S_L0R%U$^qGp-1>Gwkz6H1 zxS`dbvsa~|m?ZC8(JCud`yKwJKI(5>ULSB~4_eiPeZU~lG;Ts?Fz_As2ks-7NRGhZ z(dU>lr>}m58m6kMDj1~#UJy7OAFV`I0<9|2?S3T@xulXp$i-22>x3!qcr6w--sUQ1 zHKo7@1Yc!N6hYw0ABvD5&8bY|Ks2h>tKcygmNo}Z{LTSE&_+U>Z+& z<0m(2QD51crn5m)Zc_VP9@S?%4pzQrpa6TYSLU36&HzD_`>6hY5>sBRh}YOLeH=tn z-k`YZdy}19uxMWCZ;f-&8g-Z(tEJ@y-YkV?~ z>f*TSkE8$ZB0}@%3B}y^-p!scTWbabujG;Upf5_kcVuqTz5msgM>kCJP%H`bp|hJz zz+kaxXDyLnCetA>%{sCc>~uwZ*mm}Sh9A{$b0=ZM1H9SDXsv2Ei9B4DTam`@$LVpl z5AtzXFJIp&nAA*R95Z_I2iS61eu~$gvp<0y+pGlDbi742u?_4|24mfU#Ng_dGL|+I zk$@;b+qU2N)#goYqGyG=k+n%}mbL1;F?J~QUncb2Epc|P9#eH45#BYldFKlgO7pE~ zAK524xUJUBUt>VmkmPmsFni*%)9cqjjGWVXkvrKtsX5^5EQxs5n!hKncA*YS5 zqqF&`J-9M_dV4Kzil@Q^Gk9Ob@fQsuphT0$=HFEel=#Lyf&=KVLdq@Ti$?ev7gH&g zBu#XRLN4^vnnu`uB6|DC;xnIRH=Q3Y9$#Q99CWn1iTvuHas=DI`v`<7x2&BYLh()8 zFt()psXy6(xH|G&?8YrQdwQlb6gGt^O0Wst0d1;^w+vXi)8+VdrpQZq`-uF%&LgZ@$&$y3#J#x9z8> z7nehAnQ$wTIhIUnzxzwF%yiP4`*thB4_rJ5d<0#K*mn2jjRZo+b&}c)>{IG7QTa|3R8PF%+S#+#Kb@XZ|?# zhF=y@aenAi;|okwIG1dNF}z`mC1UjNl0r*2^Zp;ha;CSaIIxu-ttpRl47~PG^rZHR zNZcDFV3aq*vvJ8BCdnpIYKcz&`d-F0-ze^EIfor+iO^PB%Yj|z*a6(?^Gk+bJ`;Dh z{N;fgVQKs3CT$DTb1T3=0zseln>l;xo`x$wuoMZ+eKhM@%hgPoUfc38iFj~uFs_+0 z*Mc&KopV9z*an{z*VT_^Ft>aCSVChwDdV|j)A{c%TJ#mwP6K^?KVPl?oyQs<{g?6l zW(NEslzcu?|H0+v!4A$*d}@9*=slWmPIE{J^u3`+Rstz3dD?bdBJ$?Z1(wC{DswB) zyZwfNB8QT~)(#>Fctgm*gB-XE4L&eZ*!@7fxI$G^_Tq}t)zBj_Tq7OYM;OYN*l6-X zyn>li>tFeIyGknb?4k&Y;Wn{Ln60NKz1-kGWD_H#j{(xN~=Jb*JiuzMxkRb&K-bu=E zMlwO3s!**TVy=Ic^dO5+GbOWo5uG>x`+TxdXMOlCmZS;3Ypr|ZQ-bngtFx#N)y#`) z)xtqbf&R4b=u>yicD$pCnzOJ35jxi@7oNH)PJ2Z?lF+54hMwZX(_dFdr_8xq9;YMQ_gSYCO3t%MS}Y^vvakJyVuAZ z^WSQ`-%;2yKOPhesC?95A@YQo@X&^K|q5uphtb5 zCoT=plI+h{6hAoH#RJoyAsMKt3_{FeuLjxpz$QF~v*^c`_S63uwHCV+m*2%!oh$#x z-kZlm*}i?iLque$6iJp+D50{}*dvNck|kRyOV+aQdy2BON5)zr5wbI52?-5Z#y*yq zWE%!!_a0;E`91gVdH;MrpZAY@{^+iIn7OX=I?rSI9^d18Y@iVe+qNK3@gHzfmZ@9i zm&aiu260kCRfJMmrSmnW)Ht4t(<3+&g)i3nuQdU0j*IHlK?0P95S%}G6^{B%am0t7 zyd0o_$hh!qH?2ns+?joiJFL2s}$2>XZ{{xIx$ls^=}QZGw5{kRAONbIYOM5E zwiT9`?4i-eY(x0rWL&Aoo={nSFKXJqk$@w_1Gk>>VW@>d^zq`=;8qt#f&$Lzp()&NJlg(KpiIL zwgLOX`0gXh==^x?5@A698sDVl-x_$l$mWs?{1cn;VQD!$cV+ z-pF9sKObmBK3TEbLZ>yvPO5pc5ViF4nE|gM2MhGi)_711&)#GjK>M_)^jQU_b(e9a zb`fqm{j9pb{TS8Yo?n`YQX7lFd*tue1NH`Z=l8mMf4%RqP2xWV`Vbf4+l?UNA+y3n zVT!xP4YnyN&yC7h-c{0t;Toz}uKc+T$p@G{=m2~{bwzu(MLZt%m|yA1k*P~y@YY|^ z=Mxpy{}Mk1#3%4#ItC8_TdR4pHfwT@0_yYA0`GHU-C?!Y50js7%a{25P!EKv;$UZ? z$Ro7>C?!*!KSo*SLDv?_&{#8`eA80ld=5`*%{Uxs0A6o5@#|b*mxJGPb`w<l_Q8dMho9VH|UF_ z1sRn2JzUJrE|w1=A-zMo0d)Jj&)g<$2`xqSALr1f^|Y3|slrkNn=F4VuOH}d0gkR2 zVUzk|a#V=6p!JI=D_(qiY~ym~-$8*9ct_LsyWNO4cF8YiZBLDX+#LE4bc`vZwm`$m z6Vb7Yi17YB4)tdW;A~^Yo|@q9Ib%qf6~Tb;^A^n&Q0G;dz%BIz+;z|ZtO@<9P|*OVvCOMdojMthQ`_o(U;A?K6E4gQpvWV?2>J?^^T@! zIH|Y(k^V41l3yMjLrX;gvAD8{xmDXtC~~90=^7dIR?Aj zH~PbvS(v~))&Wd67P zAs>>I_yrv>x)0-4yoXk@@RQbzyw;GwqAv7Zy^p!=>EOV7WkQ}Z?g~0B=+BCmK?)#= zxdUCF&?)ypVH|`&ULdaQ*-e{m3#;?AOsbuq26_Pj@7@EtydPT^bNfnx=6{Uwvrk{_ zu*nLLh~#2uF0>b$K0aiDZuR>ZOVXBibv|F_75-m~7zaFM3jO)rt555&pA?#F>RqN) zTLduH3Jsaw)CuwZ243T@=hn}ZKOas{;(Im4 zAu`zZ>BcetPtZI{$rMIs4el%tMt|tBgGma|1l9x8u4-q$6Q_80T53 zu1O!xd~jBy-EreMbZ2XN_`=1pj^pc7Ax4f0U%tPxS9?T)QsH||9W3&K{)QfTZ z1E4oDF98cxs-WCPCqt_O;{uyL7&4YuerDe$mQq;^jgn6;Np2|n0QHkXDFBBP21pD1 z3rff7Q3%hX6MzalS1z;IkwJxM-)Hf@y*S=CD(Nl8b-Q(SnANoB4K??%EQ<{<-}_?I z;UerCp|u)fOylqafuiS@i8Y+JZ~RBmC$V|Nk_9opQ^q7acD-9ld|ChuDD+-kqY*bX zK{dZdDf2q~Dclzj_LPPWd0B7iCsI?^9_pKzryLHT{)53w`E2_^bC=RY_{8coRU8!} z-rbWst-qT^4C8NcY%ltFt}(wXdI}a6UVjS@e%MAJorC#VcMK3zuBv|}WAv?nPWBK$(<+G9sy`@Z4&+%@h|jUZ4< z;GQGC!S7H^nf2wo_}05Jd%n&Gt!@KH^mg)xA6nVNbAy5RKX^yk4S?7vmhP{a`R44t z(WFap+Geb8P zbDr(Gb%i4ANBlUtIwOGIm`;v>S{#(sW|q81x#G1vi(4dy9?ORWJsgIBAYH!6Re@*l zJ}}v@QFA==ZGZ}Ob&KnkLH?UcBi5sJO2$?3^1uo{oSkPy^Q-;p1J%O2CTm__v8(8x zhaj(kfUuL=G8vMtLc^o_c4;pLDNp>lN?E%tpj<%j=K+P0I|}cCGIqUo7EI?$b-UEL z{pwNt&Wp^y1Y1v2TO4RD1{C{3U+4Jbq{Tupu;Cz6SrU$K@+@YEq zU(W&$oifA*63#mc4`3G0^+(iDo)q54s4i@|e5MGPk`VX#DO|rQ%L6=@-DM1?pl$f+ zPKSAYvk9j&x%t#@8E><8%jU)}vcIS_9E%l1Ai5~QIe#6tGx2l<9m&=z*}4$P|s zK7#~zWRKh1z?R|XZx=7$wIFZF7dv@h+U&h8DNPgvlJ|!V*mH7~o$0=g4^ej`8vqf| zD-!WdSDDl%uS)8UgS^7Aerg~tn+5DpZl;V8D#n8@i@31v(bk-xx4ZqQZMut`kJsLh zi;cBHe(MGTBaruSqg4wFh2Tx2&a?gklA4iH*X5^`XI1cDK=xe|EkH9Cl?fUU(@&0= zBJ&>bPpvjTcijwO{#F_K4g%r>_K=c0^f9i-GKJ|7%P{+0ZdjyeeMJt-!}n0fWy(rgm)< z0vb79{TDZ_x~ro@J-a(##8C$VUFd1wdv6DPegMD1d)NUk81?(w8COH8KzcYgS3EK{ zQSfQcMXCj82CX-5I2(sO?t7l`ic~7v#Ipxb?l$nhpKg^Jj{-xE$PAGGbhkGQMngiQ zH~HIcMk<-_NA%-QdMw%Xmb|%L*z;<`WvM;e@(Ju9I`WZ4j<}hM6nyxnRYM%=VeAAd zT38wCnmwp>vN_X2t|j{{G6A4Ub8-y-!zu#o_YK93 zPU=31RO9Tu>Cts_ac=|h*C{IM;BFnE%}m%A2QC8wQKtHLxF+~YeV6hle%;BOeiHy@ zPhbBA#rR8Sb^y-}`c5SH%Wj|;fAQkt^HQ<3g|nv-h@bbN&wdF}X1owexl~Q?OvCw5 z46?pLDnxH2aFIfPFBGKoDEvM-j-@qh@5-|D(_zrY5aW1xtRA#1+a7w;{heFS9^^}S9VIxpWf z>T#cFbKXvC?Z-wCKAARp#wP!U0PF7i-rfgirvO!%{E3(lZ0mbWez%x$V(z3&?&wa0 zU%}TBx#!<=5X5^~6K0^ZoseNIz^ZolhZ}M7Ox!o~oGe!}?5!RG3>|w+*f=UtTLSb% z8ZaVY8tLT2c~+KPmFGt>&PsB_F|K^eQ)^DdN=g*zb8h?Rh?5f6?R!h^Q5WWa{CK?l z3RaD{FBy#Uk%i(vMa;lQ4|O35X6=jD{=8icbW9)RA!2zw)h^cGya?f#8gE%ty`uL1 zVmV-L9C3`LLX@FF1$o?1{DlQo-ylVG5;;+nQQ6x1Vh=t4k$1Bf@$bfPA9XVmW#{VZ zE(&a*1iXi~lq6B#EXB{bDbAU!dLOKqH<8}YL-Jk4@_CV36)CL@Dd+*^gGx4f#jW+uoNLT~yD1qJ?c7x~JL@C%fgSzQi+m9E{MxD6lbF01-l zDE_itW{1si&reiltRDHufla=}z?P6x-;NWn>MGf*7sKQk`z_Xf*1F-@F+3FwJUlb3 z*}{zABMK4okM$UrTAWYx1$pk;p34){P+huTOP;~iJ>w5~sdg3JVPo#qR}M_N7Mz^I z1-M#zMYrS0{B4JRsc6w38lbbL%F3AtmigUN>P|YEN}3*Sh~Z!KyOtq}Kn#FX>A-$E z%XcvRfa!gwjbw#}e{%Ad2P4JLSHhf;?3XrTMOOAL&Bn_>1QLtGviS1)&gZbM4#-RM zJ+bc%CRSZ$@qwYV*KTV2oqf1)cpOBnXzmlNkGvecBNrC}MPU`$2%)-Z!O(jcx2Z#a z26e?^i#=8roqu9rJhX$e^Uyg*dr*r#Ym9RqFQR)o{ipS#WrVk6%$pcx>bx%7FEzk% zidFC-rtdjxc6>PA^;Ksv2t*7m!ffmFe`Y5ot$*vr$n2~pDUr^DfO7( z3d%Td71npRpW2p!o7*iFtsqc9AP&wqVOBq}u1@g8MzSi+2jZpY`Jf69i9{^p#%-jf z(Y$pK$nuJeCAdRW2~uOw_mgBh3EQ%&N@W_?8sWP<_#whXk#U05eb7-WyjV`Tnrp1Q z|EnRt_HW6^r_cak`$5YH7aE{Mjb2?GwOqpLRc7{Ur=|U*e;wFsCRfy9^R$qxtv{*e zExQll=Tes~3`9N4h&7N`Qx6I%^;tR7X=d{2dYcH9WA)2yS2N75)d!&A#ZQ7t1?9!- z{PzHqyCLP~Ywlo{V=|RE)?PlFn)DF}vPYl!&G*Zl+xasLlYsN(oK>4G9D7dA3R;M? z4eubi@vDx&CS2?V$=q6BnmDChmcb!zYHQEa0OAym^bB)P@3oPj@J=2;J1Z!voaLhtb-eAM@|I~36f#23Z-QPp05rB1LUEu*he z(<>2QMeMbM>_&gUXgPTP}M1)O>`YC|IiKr00F24dDoxS5;`{elNoHO+%)$0{mK>3OP9zOZ% zYu!jV8n(p6+0hRXcu!w7=)ngz@j1_B2w`j_4+!iQv#?1h%AhS9*Q%e?lZ<&XwK0Z* zm-_GRsG|s<=&vp`VSS~7FsogIXQ}WdjQwGFnfaR_b@Hq7I0B_SMw?KV&E`-+S2dI6 zlZjg@M^c8m2O{@sSmtM9Rt*k2Sb&_x0$va240Z24EtRoV#(tM5yQ+QtAtb1>X6a4h z+bjDVY`yATik)|Ngh8+8ZJ&!{qubz8-z+%Yt;(fBJk*96rI{@4nKV{mGJ0w}w8_cM zBm87E$SUse>w4CXxd~c1m)6loghJhp%}SD+YEskhKAx4;>B&(d)@Mm7B6BKp)tw&i z<=m(YN)CXUNKuz<1A>Ed<6R@1+?OFbh(sv^khvynlrkX(ZXw(R(&3HQXM_Fjsx%iy zz_lgqhyq|T^!`1IEcW<48nnsc@)0tYJ)PUXe##gXjJR=yH`Vw+TS+^ibT^61i4r8Z zAQ|~$sh6_kaR@}@%hw?Ck>BNT?_NK@QX=FcNphz^pTA8;L6g`w!+`HvebrA?7Qg^ z3Nc?-xuGpAxWRnzoKgUcR$bj+Bz)2OwV#aHMID_ZP9B#B-NzK4-O+zA?lO9NBf|Nz zr6BBQ@$!y7;1VJ7k?T>$6wgn4;$Nc-Jr;`rFAyPfU{ZhkR+@@kkq%^4qJQ=3m6j1> z-0`O%`JcTWdtkmil=a3S?LUeJ$teFbl_m zrtIH!eKT3Uv~j#(4?e#$5mFJ5xo-NbM z1t*TgG^o~;m%?UE64sNAX{dmFl~+PeFgQSh&+A6FWpU4$1ce#z`(x4ZxBLimls4;V zv(loX3yEUh!dLZ4yG!cOF!A)ZPn~(r4$7|J8oiZ8Pca{Pn9(J$3hgPlOcQ~rw}%V1 z{e&I3d+C*V&YRB)ydFZIqA>Q*AC()Ik(TD*;lcSm+or16?^;Ez?#CCKCNVyEO96Qd z{#Y+Rf` zmU&5!-6vaw=ZN(!u{Rn)G!q>#Pjolt@XODq2S9)F>Jl(3Y*sk5bad{kxN6f&fj?~^ z#Ig;i!DTqm@e96Kxp+taPmpnxwZz5y&h%Kn z5*}j2HYYwMq4DPV_T^OIcNV`&ho=uE0q_?ri|vwQ>gq?{cDrq@N9XxXwf{2I6V;?Z zcER}}Jgz@(WzPP6^9S8E9)%vqE%jpoLt}ccA;&!O5toax;Gy_}t!N-pue-m~F z^m7Zm3y{#!35Lg&bnt*=j3zum7}8TePak#RNJ;J+doGf+>>G`Wx>ur=06Q|qTo03I zw}2TTk`Hq-M#VUGmarfrh3lTur$gRlXl#5YVNB)1BrZDFKkva*uFO|r0{Eexa}e=( zprgO{`axTUA#vUtaAPoGqlGxayd(LshW6E zx&jZR0ML zx3U}pm_I=HF?H5Fr=w}#ngu3X_~KpWqAs5NA?~|v$F6!3TET`EyC4v1lkNiVcrzNz zpZ&_IGV+0GM8>#Way`iXVcgj_@^lE;0_o(blSOhHuu7%74!wWkTJCq$tJy1x^5;Jg>6??5QTU@e;k;M|QI z%h1MmUZ2MHr9nkW{Aig?qMx&~du(xy0OdP}yHx(~|d#R=o;i*L8>d5xQIS zrk9P5VI`f7LZGitfX%V-e7$lP>%Mlb^PwttLMjttz#@^_w#k|$bHB_l9$sA~J?9m< z8fEa>ioV}vvMFn<*=_A$yqNYw-B9aaJTNcZpzAV3&2Whmxv1VTAqkZHCm3eTMm5w> zGNkP3u?Z8DC2YBB>BvMugQ@byh}Bv9V4~|9hsm~8W%_;=pTWKms5&cy0J-9}v76l* zC;F1p(jcvRwo_)|nqpiJKJHIQ9-K+g`ffc*URpWdV;^S7sel$ zj~7LfPfqNoguS)2oNz~g@E;E_K$)SZ3v-_oWyoeW8l22*ZgD;cyZd;#UaB|Bfqn5?zLKla>5Qz~!y3`G;g75kdJkHci2hG1w_2Mxj)42!c1dY| zBCw@hnG`)R?<(}W8LI8-+uuTf!!@)b0ePyt>mB{0TJ2Kr;>%c-V6)Y351aPF-c9{+ z0Cm3`lDhnIIZ@*hdayiT5QSj?O4q>kh!Zqdan5j#Z)WKQg@uJp2D81qliF8}LQ1k0 ze2VTH2Rr&ZtPR@O!8BHcv5QWyQNO-gc2ciPt17yL+$O1}G)Bu9<1nTC zLju}e;M+ys?^8c9W&l%4(S7ZQ~Un3Kdx>GQSoMP8M_|{O~>!_r^cP5YQ4Y3LY0Dif*(74JZZ3Xc?#US?7 zm43E5zJ^@tF?+05?o=?fO8KcAN-A{mmPwodvR!1t?Th-}1L&7ulXRhU~FwHd-v!dr?Vd>qFm(HX!#D4+{ z@DpQCHtT}D*tT!J zbdBeoDR9qMYw)i8ASZ52HKi_7lB?gD8F*~oQ_6$0+wUlqol?>UF^5SE@P?lR-Tn$15L$r@ZZ zS+Yq_>{Q>3$P^eM02E-dbbm;JOjkSX*UTx|*0^2z+6soHKPHpegC6=1Arq;fmG%QK zB}}u7PD_J5Se2VVH+L%~$CF;{ay_eF%0`-{H|-XRGUGqVuweiZ%*JE$dY6 zB1}w%$0VeESJ3<&%r@i0_1mFPe z+3iCh5*>GzAAADu!cnOl8L>J}jg0o2g{u=_3CSA1bPTGL(>LGqdHG((YomQV$4-t< z&&(UYK!FrKJM=BOFu^7@c`l+ z1P!q1*1v7%+ipv9aaYZfR^8ABpA4i>@~u)DTmImRPx3}(5Yb7AZ$?K)1AV@%UG`mw zQ<|4rI~KGy>uYUs*qajn$*)bthrri2v9-}zew=dix~5{I`>}QWd>&w(z{@fQn>1F1 zj!}}s;c&19a@H|Gk8%KFh;tMnEiG~9Zf@p~%c~YY2qfb4P~C^gVu-y>#poe3>%@Z? z|I>-%F}G+M?8j$4`hWfUb;;WdfOA+%QOxow@CBbcY~&1*Zx#1&BU@r?@QiT(6e0HV zChLII@H-+sLH1bloJhb>;n!zJae$^FJ^9cuUpFH&Q~vA^WYt^X0@14k(>D-El(3o2 z|1e)r*D8_K`asvXWQjXzxC18{jd$D07x|U1N+}pjJ{;lmPO+rp*mC^-JM>GR0TI2&tiWKnV7vA4n41xG+Bh3Z-;2yEB#9MAu z+|xYTwJZ}BX>N9;XGQX3>z}}zW0&9Um~aa(wf%eh zS?JFNy1vAE+kzr=>0_wEj9Dr{2?Sn#bF7;k){VR2ix<4OA;!MR@{7fHpVE~410sIa zb^LDpf8sl}18InpLZJhe+k9Q*zx`N8kl`R!0Pv~w@($EKJR`J=s{%2AKX={Qza;Wj z{EbGU$a{JC^XxDZYaga6I~x~MAq!YE_%a4K^WO(Ams&1n^eD6rb1q*o(bLiSFuG{0 z@|{idawUkk&Y5yNtjeO*9uKF|v^1sX1IyW!=mY^&zT?Q!K5utl?j9P}u^ex0`kn`7 zEl8;<4(T;RYl@^@zT-koy6y4+JEkCZNx!S_wFD`KEiB3R15d{aMt*fF0z<~EEskbl zaF@9*md{@Uh2L!9v!NEzWX3O&xP_4F@?lO4Hb=t<$6&(U-|nChJW^QP?_kNZ6M<-w zpX9P8)-8?Fsc3bh8GDReF%jmU65LrjOSbj6vKB+|3!MlR&}tNJhli*wpHG?3Ystrn zT-rBLwCcUq-P8Be97~eJJsvG$1|$Ti^u=X-hhRBChGKGD>L_P?2hFdP2~2 zFmc3-50WR)bt#-!@`?z+T!1lDIGu40+~ECq9^Pyeh*t?sq=&&)eTt{$i{=}Mb-rX> zzGbO3=d+yaMYVE!u4OU6-Ex9P5Dvb8*umybVwCRC!Ohwx#F4jkL`Oc0@OJ3(Qs{c0 z*xq3OC@X&TZf?;D^W9zEzqnVvl$$!-@IqEwgqaCNuO`S%q%KOSG&h6higJF}Ui$K; zv%{uoMw)zeWJY-j$KMfq>uf@A^-8(T74CQ+APpM%;4Z^Ge6k^YNH{sFZGFIY?3q-I zWBC&2!5B7k4Daz5ojR@G4W@N;fRuWD8f$&dw^zb1edh!1s7fg?_CjE-y)I zH8bu3u|bzkHVBFJWv!qo>jz*3A_suYn>lz`t-qB)wWgr3@X>UE+00v&_#EMjhk`zA zJ+3Fy^GhpenXx^F+aSETl#+Dv&?|yF=sN(6!0p^97R3AdkK>9@Gr5L;xY1VHyrC1C zT3g|BN2PUM-f>%5%uS5{LOE>Vr!<&wlCGqoS0)}0q_2WrBRGM(^h$tGN=mxYB6#-# zl>z=&=N=+e7Mcz~%l_qMLkGGh=->qkhv$Oap=xqK*5rv`GsuCJFnBUb=<0@52?QRu z$}Zr70{QBa>Y+h*dOkpZK7&-g!ul$iiuxD9Ug7uP<_u|XIPzrRv2_g26*MYW4 zUr%Bsv7AL=37T(JZ1&{z*uh5DxmM!^vEk792EevE7zJT7@P~?nYFgj}H~H#OT&A<| z<-Wy5gTws5>=XP!(_d_Q?B^*M8$^BXgAl`oGf=aJlfoI$HghsE5j3h>{_OZ#G+aCA zd?ps~rU$$myytOgDhq>-8Y^<=+I%feMq=)_&$bN;$fCZK!V?YI5GsnHsOw4gCzs zdju+^2~bQ3&rM5%;fskj1aG#g3qnDbFowa=XyRC?63+q+8%g~l2CvJ(8ep2n28X~+ zh#{IX!Efj^|41##a82nc6knmkN*Vby;MH*a#9t$6DqG=c@P+pJL#u=7c4 zd_kC{no6!Ro@G_~HDX-EDiUh-3>L{L8KaieFsW4i1K`GzBB8hJRR+Qm)e`5g*O#AN z@iD2HRttUdE=Bkv)}1woI=K@AH!u(R*JTas1IgGTgpfRzI6rl)de#>8<9^{z{g+k` z4@<;hX&JU5B4@mCq)P@gu92aMf@As@fSH$tVLdb+5qvS`fLx2Ec=qH^w~d-Fk@xS? z+aS4lO2lyM+30EmrgP0?nNlu+1GfSNjpL_wuu?kc98_-s0cLdDCD;?-pTEz^%FE-I z-b&H}gI8aA3Ecjf7Xk(NV3BtgXJRKx<%9BSvo!=2ZqaBfmax%kVqw@G`rPpnkJ$@{ zBa<(s4C)W>Ff=i-4mxZu|8)vlE<#P=`j<)2)Y8hjV^_E0h4;;=4hdiBPX|kb=3Ss~ zPHiROK}hdm3;Z1qAi}GzZy*FJb}QT1XU&ku)6KIgysW!6g$4f63L#TzejtTztplWj z&if*wfY!iZ!JnKl+Q$9^P(rOdTNEJy=DE*KY6uYzRkM_ZB{s^-QB_s{6pv}>=HZe} zKv>P$Qrq!eg37nvc_rXHu(Pvs23${Pbi=8HW(#*Mov{do(txV;eM&?a7iazOFsS`0 z>A&QJ!u2oM4!7ryPn}RSOg>j|#Q?BLFu6ZjSrdQ>y6ce6%OOSbw;}h5MRg4w9n?mh z#Lq%t%cl%MWfkb8z|mNnTwA&8>)T4szmtwCCP)<;JTb^9MX!85+1qjNmM zHcRizT>HeS8krV+OvOqeID3FnoN7KK&`(Gq(4CXz!b-#;P- z28@^PhH0ol;buk%RFp$b%^o&2dOSMfB%Lg?J1T@S*9d`-=7gapK7K*;1BY@gocfil zWSmU*62&0+%odh77u5OrJV-+}Dp=J|4NN8&GKq7CIt%j3@BNsCS!xe>)qClXmyB?2rSyDW2lQm) zuoCA6R^H8CMj*mx-9gIX?!$BR6TA|4^chkve1j8Y&ajJIGuKd+%H;VAMsd!PzgCm< zFzk?S!mIQ0)_U`^nRi7%2f?76-?tZcgjkRBL9t4r27)3=0r$s|t}Co^FBWVbE3yDl zhd6(@#d$nw?W@F1e;}EfIeY=28JH<##;qc{5Y`ELUCUl62#&Bx=G+|7Exkykf!x{ z+XE3mF75z!4=1*3hH1m6Mef_ezjuwydIUtEV74I+jd%pxKh#9v7QoG|cUwcv9u=Z`?JqI(WcxC<}^oK@4zF;SP+foj)I z9UUwBhgY=NfVuEMxK3O($a^w@JmY%i;GjEr2o=Z-;G9!lZuhs0>7moy5#Hi6JExJl z2x{JwP>X?~IZRQ0kYxt?mKT*&a%7b}_v_}(}f=9|6nNxQMEzA?PyZQFsVvc?Q zL=^B@4i(VNJWjUTl;$NhK0#Lr{nO-cYn3`WLvQ?fSoIEK;I4pJNx3UBJ)=eqP^9Vk zRrE~HK$}GA1J+lX6m(RDVAhrQo-Uq@N+tVO2C}g?k2IYagz8&8x2NXP&<_MTIz#t_h#o#d2 zH07M7$KR8}z^GA~pq%etlam(}uAE_8Hi2vdpL{H|VyX8$;HsQ(<@-1?kMcLZWX~wa2j(2aHZ(kmfBCJ|`O`xVI*5Pak#Pp_!#Gvi8$7=$5g<{qf_m^UKhW+b zR4Ko7FY3@ue=AmVhAF6V^nO$|hwT5C=Q9?e&Sr^WgQ$H_rSy#t(6G?BH>6&e0X^9n9~6_kh)ajE28<4fVj_Wi{uW0IGzuc^>c1K@K@7IaRn2Mz+4k3WN&;`LM8<4*-Gb*N-zP-VdUEPhkPlJnVKu?wi}!&xJYR zxwy3A11ZBv)OkAy`ie3rF)X}@{97RXIbp0L&s$kIFLPOY%XR^)3Mltg-iNN^W{m?W z6bFha&C|u;?Nvxb99G)(7_~a5DMWOKmqMYweMMpi8W@^LJ<*w z_UHJH=;=EO=xFMhrZhKkM5U!2gPP@C`iF)XJPU-V%I6X@Mz7D7f>bs7;?|xzvGsVO zMrS>4igQ{$^aN&MCsUu=i#w^Wt-fCW*xE~dX>R|8c`ks!MkBJIes@}+%SKYmn+8QF zL=*i9`>{_UcKE%q46o`S@CM|(C{bc6Oorybyj*bDpl=3i?}Y=yE^V}^ODIDB@bIuF zHZ)XEmc;``6gv`08*MSP!T?{9XbYi9ZAd@C2r~#lbHD3FjT3+SQ!7vEo;5 z9Vt|lG_!*GFtj%H7P;=%oX;dMZ9@BgTj!bv(hVNQp1>PcQc2KJiN<`HRocxFWO>Xn=4sYoH-c2gqrs{nFI zP{*~FnB*L7qp0{(rfI{;h?vQe;O_j?fw>v`%*<^ye|UdfzXJ!wp81^v#DNHn!IpCb zo+PkK6cdKODZV7~gmABS2`*#Huk7CPE4w4_yt!pV=H+1UV@0o+YTt`<8$JWV;P!-% zX=%U1(yC{6!4%$FuNh!i0!plPkOV`{S$dnzbDHRbSka6GEe8?V^>EH=rEiZJ*-K9M z%hBDM?Q;;Za74zxcgRw5y7xE^uD-jwTiOI}@|%P7vgLe=i|q30Vj1EJ*uy7fvEsB^ zvyLHgCLr$V4Yg}z!qJjS_tF?B-oliS?H)BVw&_ln{6kn;#%?* zUap!IA4f+s+oY5-M@GnaZr(8mR&m~+sN zbnzyz_K7pm&T*i_+K^s_Jx5(2JvWL+y_^f@gGjiDi9cgu6~xLMqxQkIH_aA9Buu&H z;|BF|x7P2juW-wfs{wI&I;v4%%aSiE=)Z)Bf}zneYTduh_G0T?ikTp%`jdBnIYf(_ z>~uzRc<~@S#b8ZImDs)M1)lO$lN84Rg66)D^t4TlPL_u>bkJ=3Qe)RI6MVO?fZoA& zGUsI@WG2ZS#&MpjbtJ=>JX5=L7IZP7zXr|82Uq&N%5b`;v5d-v?(YD5s>QiPMmwhuv*y?$grP z8wtf+n2T!$a1%G7AHHaBwXpO_rFzaQuorEQusmE-QK+Q|!c>o$l5jSaFu z#vT38#tG#PF4aBN;Bm#PQ4;-U@91}8X?PmbYv->Ii`m-ya5e)tPpBm!Q{BfP#Br*D`5`o+-TsLq-LoBDrN+nUTPw26fi({2#o)Nx z@SMcU-xhwA*zvvFdm{4EZOgPR8SIP0G0o|F<&#-UWDJT?I?&Am$Bc7u4<*^|(~2hU zBzHD9jNnFRO9SsxA^cJ{06l8cxn%q#<%93q7bF3x42-R}m?tO8*jZ{hN#7=R`8e}9 z!!*i6#iPycG09?jyyej4a2((@_qT2a*-X}eb$=JjI|#`csr~4tvc^ePH+HSSaDlY6 zP@mOT0T3i=_@_XD;|}6o#fa z)pb&ucYs|#FCSqYih(grba*_#q9H`=!hn-*L(k*2X{d(@Ve z$KeT%Z`K2uRkyOj#B5>DYyID^K^r^IID#lBm*>4WXVX=a3+@-HV&28??|YlC1FNTS z%P<@6R8Y3i@Y@TBA#H8(*bvrht_3daEpF192^zs%apvJIlVn;NY8DheCIo34C(IjZ z_FF34{Rc2ZVB!d8&fm?Ja|cIA{!4RJ5Y?D0pL6_bW+-Yux6m#y?=-k@zp#Xc zgf`jY0+gakDmTD5=D$X%BATTx{KYLQ1ah_@?ZaxNMAfOz`FrCX9r`olF$S-10C~i# zWy4uEar=TrE;WwP#Q-tvSoDu)^+Id`?%a?BLGpX$lM2mtxd{)uP;Q8pai@v5Nl><8 z@w^UFzx^;!jG9^|T8<(Bkw4+|YROA7Mz%w?cf`q1=%!Y_y4%BF(cwywoY1^D? zVKJf|c6Im-WSlJ46A{{MDqz7-!F_MqQ>Gl|4pJ=+RL+&R)nVi?j>Sh``AR6j($eNFm59E&h!vPDr1)WE-AWtH z_9B9hH0~M^eCUAhuvJ@kaI){JjHQ&9wK%8yz5t87VkvcnA1UM8Uw69FAazaB?HA^e zO&$v=H6P5GZ8N7*04nLShCz}=8wSYl`c~tT0{}@)TEQMb*@94G4@+52=LurchY?o@ z)WCFyUU{}oDSU)(z+$n_mO13Gq~w6Oy6m0KM(5WK1SR_r>O?Uu^hk&kE4<`bpjC|# zT7_?hdcrzWqHs8h!8C{Y_(_A_$^b_6hQ{`gwPqxZ6n+%hh%MohH>vPv2LL&;`f^9)Q2MUwdBZ47=kLS$nd2Yf4col@CEyS4cp8ZQt3ePR!E3 zkoeAOGER~vuOb8_l+U%3>=c?U#XaU0xbGS6X_)m+_xIOySn!f_Rs76eY;9U|rG8HN zW(#T!n!Qe6{vMtC09u0%ju+EeCiSfN))a|aNooA1oEZ%&?N%eYlWf^nt= zQfd=5+%+F{bp5Uo(%hPUy)Z6wn!IBNO9zL6u$2n9TV zULhge(^Jy{Z2FV-YNe6aw7;wX8S@c7$KVtEu=>E^nfGO7fSp2^WoTq{WLL>T7J8-i zqi()8a*QMg!4)9EA+URZ

`&*#qNkr(ro!W1Q*wJeVzF$O8{7?LU8x0qrxn<|Wj6 zi(mS;W!l0&iO_BZ)SsuuU6#Y_4A}A&91TGG*RX=1SOb?enJegyB0ysi|5DSn%t=V| zz|;Uos;;i?bC2D#l}`=QdThb>A_F6i^9At};u<*p6rk2gtcSk63aH;?PcFkPLOOSv z^;~jCiWcxe7C&5D8l=?xF+R2|yz!!PeTGvYa=y9~7rDvGmNvl5eKcA5qZ_@Ep!XF3 zNPN5EIwyqpP>2#>_q%=};TzVD5A?AID-C!@(02J4?Z*L<(A-9;`U9UcXxt^n=CuKM zExsTz;QDE?QMZNz&YaIjb&@^PilGIXV+EaP&dK8K7YFsWJ&RsI^E$X0K;#8bU8h3s za4!#YNCIwD<|guY;V-Z~i(-!0BVWy3WqIFwUKRtF>%iU+y#!`zQPwgiqLKT6+W9@F zoS@@zMXn4^C#ANXiOV6-Kg<@jYx_1@B&qw$_)pPluf+3;zYGdq+P~)lU}Pxgsq!2J znkR&042#sPtsLEKO_oYNooA{gRZc}W3iDWObKQGUF$YTGfl4#ZU*#}>*WhDJzFy}# zT9@6G$PFhW-*K0lua{M5Q|`hU`%o_69ncrjJVkkmJFezs+KR7aEy#_f_)0r9I=;M< zk6p?%!7!_66_DLuT9-6n`5Q>Hb_y9wey=4)>l-?CW9ACSdhJ6GLcq+@6yzq3McVdQ zn3Yn6YscX3xD2_k29g61sg`r!C6I4)u=Dq*Sk3%m zo}TUv1{dhqaJYYex6-`GwNHg6{Tb1ti(nH=RtVnVCm}3)&^Q);BEzh##B8m+O4Ao& zw~EU?(|Z&@H9ke(G@<T>-dDnOlvnMB`^~*dyfL>xXnOF{h{9&Gba!@hKFg5-j5UJN?7*zfgre zg}0j7_u$=nACENn%D1@n($H{j^pW|sY?X+4p>Ph1V6bN&Q3kv-WX=!~J)={|{!@fL z5f5Yyn2_gl?}kRw0)i}J=DH`mmF-wVw(#;V0r^5E39T|Z>F4j`6@btQ3x zB-s(#DEtO$+?3GCxgq$6@>sdC6PvgIRn5cR+ddc|&};{Z5YhSrOp2R zji1^2la+RBzk2~F1Tk^Ydd%HfhkrwGhY_AEkpJJ-ALifs`TzA_t*tPjESAE@1OvL^ zDcpa`!`LJuG)P@xQ-Ly+*+v5UzkV0!CA&9iIc0&iz<7f4InKoQAT*}}R&-5gw$@hw zO9JBpc?dA1;Dp5j1qbLioEyOHKR!$?U=yW6psJSq{qpm74vZZ~F=GZ&nlk&$kh%$gQjVKi_v#+A^>I=i7ZdwvOfh`S$R2vdJbc~@(OD~6BlsrpQ-)f z4Hmyjty<5o#$|Bm6YKFRWWE4sXDnkFSFIN7| zjq-p0gyO%y{EvnCcftO#APB_2o8+HM^6w`3=ZgHhN&ag^{@o=1wIu&ql7BAAznkQr zEAp=;`L7lEca!|rlKg8){<$RoZjyhl$iJ54zgFbmP4Zt$^8Z^);#B_+#L8H>C$`r= zGhe{q0f0h&2RG_1lpCTB+877$_vTbJF!CRkx>!cb=aT+R%>5zv7|YJj=64?0nTC;? zufa;zaL)dj8f>+Zyqk7fe|+k?rt_s1CyVUph6nvX9C%D*UZ|U^TyzxsygA7MK3Spc zBFJ7ZRsa2x^}jfLvF+_iYw#qGrWD`Eb)O^xmcnS>!x+Ef2 z`J;u~zV#i2gc5uS?nE_eU%A?dj<;{**zc zT>Bv)St69Pm5oi%IwqgG_ZQ_jyL!$p{AACSK0BVc^#Smi|3qcZmksxPD)O*REh;We zkR60oVstz$0+?wT^TM_9N7rM=G>U}({=VwV%_92N|0#>SVSv_=dWqp)bo_xu-PeaV zT)tUk&%gYiC*s$1^go^mk;lo^BiG&jBlqO8R)njkFJ^Vx?c@7iX|2r1`G0O({EYV_ zq~Up5R>nhSO_#LbHga)1*K}Ip%Kq18x=Dq=&fo`@czlo!SS!f`60I_nOW#hrmh=&q zfs?}9aXrdG$*<%w!j6{aUs;WpE1nvP>fpno&w`NT){DdHKUZY!>fg=rVdBzc>ZE-{ z{_3qsJF|~P*mF(tr#Y>h@?xV@XD^=8;(FS$txA`@^10f#+e*Rr6Z+%Tm?GyYWy_qd zjRgHtje7dv^xc?9?%1MW&!k?z2=S{KnwaFTV!IFVnX{%0)399at{DE^ZpLF_;Lt+l zIr-u0KxvX!^pd~L?Z;DHK2tsmD{{LY;(C4Bd{^*o2G~8Gu5x?FKi)_xOjShq2dFbY z`P$U0eNyuOVe31u7EEGiqMUW1HmtHL>QUsCSf)qh|k!A=16=?#30s_*d zN|D||Q+kW^9+2KU1QJ5t8By>3-+TE!u9uL^oH=Kgwbx!}X5ZcKr+#^)BaS zeQh=D~j6jjp+sow74O5zQ%8~<~9SenKE zj7H}A`@lGcxgak%++dyGf&6eoc2I*OBKEAB;e(?``RbJSEis7{;qBevRJ>Hd^(eR9 z0asVqfOs;QQ{uyp+z&rVMfdRsZO6Xlx^rzUuBuAe!G%3O7WYKn+u;)&!#BlX9DsvPT7mB(4IU7t(()&N*dD^~YM(eFUm(|s*%8kR|mycs) zltTQGLBgmt=Uc&|VNLav$4=MPn&@~Qim%_?jOBDC+s(CPYM}CPzH66o#gXK~xs2%R zV@<5E1~(&x!BoZtCqt5KHHUFFHWt1M*$*qd{eKS^C`N;rG# zDD9;8VISt3VLUtbsR9GDi}SRGu(Ye^-_MB*BJ$W99-a=a_v@NDa&i5~K#%^1QdDl1 zc17pG)zY12$Y@-t*tYBi+gN_Tx{vGT+b5?1L%x-&+pOg3lZQ#{9wcly)oOeqHz4EdB3$tb+4+ zM#cXpz*Xz(?X1dooacJ@xif@;{;ZYZLpChIR+jX(a-R}9g`trEu;OF>_=}Gttr$?a zPln&mJ;zn*=8jfWWc5JL-e3>$YkDr5bG5H1%i-ML(4KYz#r(Su(*08L`lZK_nKIn3 zrc;jI0%PFptsYAbb#Gtmnn~1WzB|Ty{dyPVl(TW~85jd6fB5bnhh+>Od!;GAF1ixO zC6b81PmiT7Z!0?T(VqLdbF*GzaKVywMTq;X85Pc#C7sAc=U*mep z4_xIgr+yhmRpxL>z~m3PCMnCs`3ADnFm>MdyXfbR0hSZJqF-gq@O;!rrbOgcbAN>+ zA@TiAFYCDuY!T<+kVajQb@Vmn@ozp6Muw}+S+#|i7y3l}z}Z+s85q-j1t*vL86&NP zcuAO$*@dbLhpQ^xH~Ou!zzr&8$muVj6IBeoluYwv`TjH)xuySigrbqWb2IwN9Ms2K zdEd8%FXr2PR)xlAvpDE_qE8)l#!f4K8S4_@yjX_6yVusGTvuGly{>uu5|ch1^n-A= z4hF_lTMdS!Uhu|!pS1VTYPbH)HwM0tlEOEiR4HpIAW7qr((h!lbpoKZ~uu zcj%ZWXP@4;#96AG5_t6!#-@Z2;JTC^!$Dh?6S_k}ltAYu6P9)L=3{TZ=De-7p#HpP zuV9D4>CzR2+V|!YWvFG~RU#mYftO6y|6atFlV2#!4|5v}cDE_Nz9*-%Uy0tR1XHF)JCOzl(B$|wIz>CX)s`gdld zX0@sw-bihj%;~N7!(nKNj=dh%$%cnrobqj#IhxEsDx(mAsg;r{tkvCD7g}UVpM01s z{HE`DlRc4Y`Wf<)$q_(@Y-tddIX8i*O|F*y9L%#${&euY>(PN_cW>VapM6!s_175h z{x!yU3t%vBO|VVx_T|8voSoHUQy0pGV*B`)t*unp?cM|iXLCGAs1CZ{cU;I$6HJ*o zydn>_5K}6-r=lQ%PL!=a6yIz;QgAhh0a-p9m%k9Ld|ZfIhJSi@8M=v;S{zu26*ID@ z`Hn+RYsg`S9YIwHUgwoT-ZKh;qjULxhcx+LY`{618fQkHa2%MltmET5kE8*XyDG%_ z<9MsFbYH{PDTl<4!w*bWnSS{1tIIY>a*W4 z$&)_mFPu=JNfDr6Iiymatt?NkY6Wa=2W5aLHI)6wSz`AtI?px-z{DccuKH+iV$?d{ zVjoy_+r8OjbNz&O$6?(Wn%hf(lH#$Pjk$vW!V8~9Q#<*+uKq@5DiBAv=}INtJx&ir zC{^PLCh+^x7flLI7|boQ*#v}f<}(2RpC$g8o>TbqR}a<-2Fs}VO!fdK6VlViIv$2Z zBvoVymIYT`IIFZuhz zV7Z9C?S6dVsSdZ#;-TM@0`_NV{1psJJ?#SKh!$W3RE|US4VZ#uI^Jq;svK%G>^GiJx%ejsc(M*wY^0autYOHF z7`L_zHjltOjR@Qu-dDI^>D7P&EceR9Vt7T~U`G_Os2%MxYJc$#WP$o1!t!6;HnbFW5SVtwmlwEK_l4)Ie;iZq=T zm>k;Ar5#Y5{svG@yvmpaIN{D%U&NmR(xRJ*@(&JHIVIKNeiaz^sz}-|Kat7$qResx z{5hUN>2oK!^mJU0HZ;u_0CsOQk!%o)_%HZS{5Rfnno!G;#fLZCI-vm>4W~T6KX`O0 zton8jhY%G^=1|OTkL<0Fci>e!(4Rrl=RCv&DWqUt|r7I{zEpq}l&l=c-?QzpHl!mZtrfV*2-bfp2|P#7awFIertI z(0FmMMCM>1GJ*fSwm6S$nRBp0uA8O|zmE$(Ww_?Cx1PV&-`6+o^cL7o=$&gXMU_?% zGwb+WC=O5kY)|{5eEj;s0o88&XZ$z#BWaJoEF4cNUOfC~9bqu7D8=c3o!+80HzqT~ zRe!^Wjz9Y%Ht@TJaNUa_wS+`AQ~bXN1IzsK!T!Gok&%KjBXHy#V^B$+Y2;%C(2+T; zec^(xkaZ*B!og^&*Z*^Qt_JQ$VH=g*n4s*zuQL4+pJ)xNvBUBQG=@^qdyCv z`t;TY7z~8Z1s=zqKlxhUhi)&Pf@b=w`?$|Gv1J)V8O^FM{y*9+wMvf^$E7 zOC%(s`Y<`f!De6IoqraP7Y{z1_Wa)!qrn7VVBL9&+c*DXAchZ81mf=By^9#i0P{QP zl=%xj(xVz6sD1tyY$~DJs#+$i1=T&+1sqB#?+KKHM^Z|gPT#(;9YRn4P{0BR&#qUif3i}-`D{O`fdpw)fz z00u0+HMR2L-`RL7oc~q70bl6Gw6Rk@V<8ZJn8~1NVFm(4AZpqWae-<&he3u0FHeg9 zF0X(#Ie-HCaTkyMYjs+Qkt-%v#QHbr;7CxIm5Mx6OqqBEhXb=KuiUL&nz=q9`s84A z|Fga;_c(b#=*m=_c7#@Vx@Wli${(v4eqwQtzYL%l04oJ@jIV?17FE-&Bcpu8kXw

_U+#9kc0>be4ijcC3uL?+g=K1q}zP=^J zbC{$`-m^}AcMG!3{mRN-8*0S+HAT;OFo#tx4J(`GgMu2nH7qPFR2$dB3UFmjXTjj1N^6|9govtC}Y}@uW*23uI2B%8IR<@FV$IK92 zT7@Xi9kP{`cw?^Y7_C5RqogkS9pO|xWw=GQ*%b%JKD|pIZ`ACNS7-4< zWB}TK6x8m3GV%lHP@O4<>S)d+^6hp}uek9nGBR(NNozJX*zC>0Z2SJL-{RvP9#Cg1 z?_b@Yns?o6=~>(9Q+=K5#_`ojUyK3)iqL}&~ z&+W_(HjpczC5gku#f9~nOjaS2rct-`ja8OI?+6n<-VWlgr==Y))@+#Q{GkOMy<`uQ zkZTlL0$#&0|jO&rEfMdlG_-IWeAn@#J zBCAkPoaMjWlU08xK4toQsmIrB-m?I5)@d@8Rb|vuE*Z^PC#I+Wd>XA~k5XcJaD>yA zQFFB1@NU9-KgVr}JnU!$P5tVn~fC3pn{!NadKp}#B07sXf#1IFap zP1#DJ5rr>H&VtTPd%cA^zRqE~hoy=!*gwQPkQr{aoDrh;FP7llzt4f-HYfa2N1+(%Vah zRZdhYH{hMzXei$LUbui(J#XA3y-hW_wE-^cp5sIeZUoJjzqf${&P7o+^H?*&uIk`1 zfbsQzeeazQAp6{JxF*Lx@R|;dY`*8*!q6%CNw(tKF%T8ALuwqo9^-tKx|gM3kPqwXw=7vwzx?E1RwUK!j0s0j zVz;O1`=EL|x0_inPL2J{`rHC9;P)%{1IRC%nJr8}^+8s&kcP=EG~gC1QZdkaqR&ey z2|F)MK*X)cbexIv7uYs@b4ufUOHODOc)GgUHeL?~jccVol_7xWxW6aVL3RtEwp8?|aLGWw6Uv{ytzg0Bix+_$+89 z*nI9R_79+57t?uYu$*OG$65bASkL}i)6oq_h=1{^Z!SYMwmD)+D~DUe>La^`v22t6 zTwJJ{InuJkKr2vFDd=b)Mv!vn!0BcXBg2PFaLP;uWnpRqh1w&I z?J+|_5uY2|9F_wf!VfQx?0(obk^1WCo@Q(kiwT9b|87d}k3`SdnJ= zSykd#N97As^x9wxeVxG0wj+TOzkD0iqkHResnr@^Z5Q!FFM}@*z81#0ups6$~j>^|k1PppEV#c{Cp5n9Kxy5!*gzBi4NgK#1!p^9ll z{e5@1qXR*F=vOU(22>``mie?Zw&Y#^qYW|ThT%Dr;r z3{FUhIyo3r%2)AtzHlYNmu=CuU)9kym$$EkRp%N_^40+6;~NqYakiGI&kp3XbO(`2 zrO2C%Qd^DHMM?AVxnM;h-FFvgxE@V{Z@ze;OtUA!n|yf-w2p>edQIJ6Yig4$LnzDp zN(Kab;0P~^C**mkMMC6x2!i^O`)N;e3iT3ksKqK3+B(){AvO)W#G;|P&21G9S)cTH zqCa2_+ljzs)HT_^p4k9|NikiPilI%Gqnm31_cDBPdIN--wc%L(++6?nng%5=` zpFzZ!DRcw>4cF4yIQxxZ4OEDa11B@GSTO%vXMRla9`2{EW&yRk^b64MiP0Zn9kG3e zOiQ-NX>&lw2@wa?CeuY6+d-OpP@yyzCh zw3{s)9iYa(x`UWWoT+UJEv(q?sb}-lDRAm~gT^U=)ll1#m~@C;8^F6ZWa7?Cw>=fC zrXRRevPDeEE8a#@?;lhlwn!ki<_X+l*c{zPaLRPS*YILgKUR1P+@SqHR$4cK;T<=QS~UewjFDLJjGcZEuze+;0~ zqa7Ig;ET+2+wRmVJ-kobuZHW~SjFBm=HQ+&0K{v4@kudeS^u4dX4 zqAHW6nvt|uqnGF(%aynyov&_zrsnS6I%p#MYsB{!ijoQ=xz;x~-z(QXY>K9wz86wI z3_{!UE%bTgzWSCt$tjAX`+GeR1FA%HT0W&1cW0UcRnByP$yEmamQf#{0JB<&tI~SJx8KGpDnUQ$VS`%4 z?y4;gl#ZM*cnutO?I}i_<`LR5dm~Qb20bw0P_J7T(`zGIS9+`D6g72(RnxzHs!9j} zP1f`@eqL~lpfB}Rf3yIj#QlX(2ANYIqd4^h;%^e^#@pcN$3+uMpb77Tzy<|-2s45Z z{B$7nN|>X4h;Q@>d%iuqgG4bwy!Rg&rJM!_ax!3IwfkncsPldOt|CJ_{l6Rc&Nf4O zVNKKtQHr1URX}Fx#6HZHD??C!CoS}8HfCyU=rt^dRPEm#FcA|gFXKwI21HOSfy2w9#K(3e z4c~6PTzV4MSR+xWWqc0=RjZNiCqa*z?bBuILQdlt0?wymcw1I};8g<7pHO)P&+Xx z$dq01a==B`hvSS&zjH^=Mmu<79>?q9%9Y!_a!yYEC%!o}ndFg+w}Zx|8`~3}!eL2@ z4Y%ek{Nl1?p;glsWvPv`bjb~=mZ$4dQ2%>vnh$GG^!j_xJBAX~26TC#7Z|2~w5gr- zG-&&0G`@TSe6l_);Vk)>f4u5Y?a1VI(D3)FUHAIvD%aF;jCJ-IPofO_t6lvtNnNoJ zc|MEaNV3e))O9ZEl3{B|qd}oMd&YC!$v9}{L$TH1JDkVJu~!ipc`9R55)IRVa5w_z zD6o6%R2Rd(P$^CrK+*!ugX&o2)?dT7_!T|+P`MVAM;;1;B6GBI{l((Iu9^gK@+ljN z7O=8AHKY-B!KSLGmWN~XZDrvka=zEM{Q~tb2WQ6`tBYfS@t1kHb7Zg%Y_+a&g8~BB z@jWumG=a!gIyd6pZ>0*2aB7@f@&u40j8~!etU!Y&c{WZn-Ck@EAs|%-lK8Vx1ETcT zRjDG1_a0ODyWWQDk&bE@v1fStXW3*hTsJZOJXOo&i$nh18uJVu)BS>=j|d3)9(H!h z9#xU2(`cDTa}tEbO*S{8J7~lMfrEoK%BPnGr@1Mb_J$u*sO{1Bw6<;L7Ko(b?%PjKvS;*m|xM+7k7nf5fw;EszY0=Khps}r& zYY#fw41ASQL7;DusdYuAYvbo||5|Ylni1hFMWZZTI&cCWZ^3g!rG*?8KdfXbl`|*0 z*ILQROigLxHlw1hBl1*C)+F2}-O>jf)|OF{)(n3ve2W0ID*i<|lY`u?g}WV~UREY- z5B70(6h=A%C5$??KW2%Xo%W*{A|p?9+kGh+I2_a2rF$bD(!$ z?xK#-Gd5gRHIN3(+l`qEh@_*_33Vz;1g{N)$1}%3yTsEC;-dk<97!;~dNB1ciwB^r z2128?)(rx{0!>j|=R`(2uuB3HK)DiYbqle1R5iXCNdaj%ubp2-tz{E1n;~ ziRuE5omRJ2ozXkw8OJymOcWs3UaE%#@>Dgmmy@x9u`s<0Z5_d zf~zv<@or4XiGhpQ%?61c+Q?)L|0)~?WF~cer+E;N{wzmoM@F`u2&&ttQlVo<`2gtk z7%41$UDzJ#vD`u^>&LMP9gS&5w)U+-g)6v_Y6RGhGo&|#6!y}O5Ts?%8z#OVfAg8 zRvSBtEyr<7;?j2a*M8yw$#upsQcoF~HyhW)nwZmz&;i{j>caR*2gLEh=Q*y-VL?^M zsjZ>uBdyjJArB&KX<5b8gmAAwFwQ9X$h0h~AtcKu?`F+8S^Bz!N5G(zVA}Jd${Bo( zX>ZZ@O|CD}>$_yb__1T<7Tcw{!yC7HN>FMFb@8bii%irmSg3BX%qwYZ-TG{1Pa(~w z{TA4=eF4ry+Z~55y9_IpnZ(*iT_x@lJ=1&2N!**qe>3&dUmdqG8nB~m5yNuGJ zszX3&*TPB_@%GHRX21!hm8Xk?4KCEzuSg3|&!*mtv_kI;)F1~}uBAupP~mdsC%-BR1jvnv{`N{GmPBbFZnHR7qdMI;7C}hqDf77j z!~(ri#B@t0D~8`L25FQ>YJqnIyU46 zGVKE{!GS>DQ$RZWYtB_p{ms-Fzha2{k|EGAM$i$b?RRE}1S8$E^S_86^+0ML{`eGF zjjePs+=xyqQ7`YwiK8ovy1(pwjpN;=bme6++^@^oy=w~6_=6i%VN$WYS3)d#(`^ee zGf)%;a~~PaZK%YRaO#Zx%_kX&OJ^Z#ru@Wj3yCH3II*f05N4q|&FHstZRt>Wi`w<_ z_-{hMqM{b^L0Y9%mpdCSK=;))L(4E1y`!_cH2}KRA5%5Q3ZL0pKG}CVgxQ~@sO#$_ z4uZH=E-Zv$Ae0l}xW)cfOV+S#|9v)1>VjSl+T-4US21#5A=C#B;3JtBB#fw1VawaY zeTtxz)6|e9y9OG~ISyopEKDjy;((B(iQtHDr&9CCyx!$Dif~Hq=>@|R#Jez z@t7T{(axCxf<}L+cX?2>s~r7zX2Bis1EH8oN5zn_`#P8sgrfS<%ZnN5ER2#BspE52>w|(Rw2P)XP|c#(eyQ%T4~xi7Cl!^`gAB?riIn!P2mp_& zBGEBtIQ{4_^CLo_A@brx2872WUzWKvS5e zz|t&LX{^~mv*cjS`{4RU9oqVf2BivTr38*a{3ffAOLPEfi7RY~$;H{JPwml7pk~X(_KlAg*@Kk(W{HCFq($gq1h${m67}z6cQhD&Ybt4z~$3s9rb}-lHd*3aS&X zmdVj%hI~%Ri`-Len%vELW6l(zJa)X1`0t-cSdb66DbEv4TMpCMTUYLsCF;8rl@<`_ zf$}A&asgES(kXpA(J78ZbBH+O{iLq3l z<^Z4oM9rfshJF)o2q!5&V&H`SD1lK?YfeEN+6Lk=Bu#fmbeSydfhpJCuE*Q%JKaCj zdn1e};RcUg$N^X6>gv5#c*fCf@qU~2V%s|M0x z^`8A807zz+kuKU&5|wq2kDT5A@`$NauDauamXE*}s_{D7Pe zMPY!%#b1Mu;d>&Urs)kGTZEP+rhVyU?gt^9sczTybZ{34+bmeuCDR&Dihl&dDw6EB z06^;)*wSd*e%3H5NmuUDChgqokhc|!eV^;pbyODfBxIY^AW0K((!H!bd)ZW(!ay_5 zw`!+ZGtp)+7GoB=7thdJ54y?r@b8+5ydmIL!Zz9Jw(g?RpiW!MiF4269<>KCu|3-ck*e^| z4cBxlO5YCzbd}zAR`c%QUQ@{O4u)^VF=3<*@F@BF72l+Ufr7{ibmFc?MbleI(GJ#r zBYbBeD0zI8Q6Qk!4)Zd&HjU8o}PqJhJ{Bl+!|5HFZ09ge(C>My^xXqXAhG(9L zPOq2Fyk^wyI0xiOX{5YUm@SX~{+)&vaQm&jWqlf=-a!*yM)0r1D~%pVkIr2NPdtNw zzR)NKZXN#E_`7u-xJ}qOd*zY1dv|LCo1*=mq~;BjNJR!UslCwl|33BHd8=0U7)q`1 z#AUz%hAo;7K!RgqB4}avmQ8*`0Me-*uo95;3`B-zg6a4!RJYu!yYCs@oW3fPePIr% zbm~!@T&q|wx69SeIcmyQ43rxtrGvvIp2&#O$wVq0BvU$2UT7zIQQUO4-Yfe2vQUrZ z#>@`E%($UyRANRaD?H%Hy}O|E*gs{dL8Z7R%SpUR4?U$^adP-yQSyAm;DPp(leSI)U3In(gH=gYU!(SAQpQ|HH%DFW4sQ_SR4lfh{QI^L| z+RkuQ9%>DSE*^V`)Hcb8iX*+UU=Vh|AeB(P*bKaj1jOWgy+*gr)Q6?S7P zP1{GIUd&pff4Hj^F^W|h^?t~uQs5t2p#My)Vk$CgcntN--W;@}Rj&V=H=8#QaU9w& ziz3$_Q2Y}O3J~|_Te!7gBAt9xw=wX%iIvzd%|`Y>_HlP~cl>&=E!e>TLRGChyHc^zmDpkr8s@s z&i65nSsL>qfMW7UvgMS0)TM{oggVYZPY-yIXCT$w)z=>nDs>mlbYo@)RQ(_qOO>0lALY@$ z9yTMO8U@YQg-{%m8J6;GiCf$bF#W{QJ_(Q!t@Ntuv2)xhdHE~PKI9>%wuNX1*+2z9 z{*!f>kF%$6_xme-)*-HD{b(Rk&{}8guQu@gT`b(AKi~ynhBj2vZ(x7pO(S&LV?+LeiAVB8O)D^S1eD z+**VKFI4^7<^(kyV>0mbg44}ZuEt(g=9v?HHT4`onvQY)A+65+!Z4t4Y;%)5EHdO_ zI10!^0tp#EOYj<5`J_rs-|Ra4QObaO+U-;11Zc44g1-5OOcZ))BL~R~6M3RhV#T$P(Jw&JrCv{W2>iL;*Y1Q?pe6vFGEC<^;4MT^e>>r&gErF<8&*2%Kc zSS;7|sUbG}xeYL3SGlzwN-*MgK(<=&xiI&_nt-2K*CbTx%tkUGC3!zxAvuHUqjPK(b=D@m_L^?09kRD&*d=x{Ze+PYLVz-?hHZnoJacXPL!AYsqa2G~q6 zZoZQKl5M}NhYf7z0{e%?=h%cboyw(#9Wy5_w{B-AFKZ~~U3l=*vlHmJJT?Y-jjrZH zxdpkEK~BSC89OoRK*acgLl-o*fyxiW2Tt0?P_X$el*Zl?$aS2ph8TeGfM#X~zMQo@ zP*^eRZL%2CK4PYx%QI5gIau*vuokIbjO8E67Szbg(+&$Gdtq)w@OY9BXg(pFqfr~yxTp7_Fq+(=ti2+c^(ku7JmoP#H8spU2y3kZK0W>Vnm=zu`zV-|wsA1Jup@=vrH>v8*;?abQiY3CB-i3R=?) zqIl;HX|o1E)5hu#DfC7_gAL-6tY9;q=<8Ek8iWc6`Oq64DOtfT)F#8|Asyrk)e@`= zrp_C3yac0oyZyc$*SCOKXJTCBoF)3-W{Zp)AW@fpTv9|5={QqV#xST&Kyi|4#=>4= zEWuW) zK+pLqiM(cFs6fcdO|ss1K!La3&+weK;skCOUL^_sXGxTg+2#>itA1b>X_#o=r%!} ze=P1D=h8?rwrggcDTl5kPCRV)_MK>1&=yEs5|6;UC49QQEKY)I&+XC!t%#>Z%AanN ziWAhc5>}R`v85H^kmm>$Z)3xa!+|XPCCTHay6zKiXit~JFhHnh3E`rmqnj)V$f~jn zk`2ndco(3+?PPn#A*YI$=G?=Tu{}!Fy67m@g-V!xFq;Br!>nLwK!gd~bQ#WjW(>*O z*|6fpvr?uUzdF|QLBa+I?ZaHTQ*y5TAl-`>SqC*R>gRv9w@e2*7D!VBN@ck3O2;S8 zy%p6sxeLcQC;@#y5moZ_<3g$4w>p&TiryZI?ICT&87#;*SoybVv&>%5*5sLOsmH7Q z3Dn-FQlhisnbpN29L^cF zEVAe`0C6ak?g+b+?lT*7WFx7x0`s9jgpkcC|)*B_nn z?^!vG8#z=`4oONGTSC8IqY=dFwQu08;S~-wmOp@AInXM+;w6>G<4=g&2xp1HMU2{% z?hQLkP^t}$*MW*49=(@)ppl2cdS=0fH98ir2cSxVHnISfix~tZP2bm!gTFM4k{0jF zDs_f(WvZM4+AjVOPWQa`AO>**@_){uPT0fk^UWeDZ0^^%XBO;VMh`WQ09gojuTGqG zHxn{)TjJtJDw9EY-lUrVUqUq_SV)-yvKu8Yx z`OtSkduPyi)d#(fqk~1FSo~bDKQ{HZBol(x(|i<<&!hvD(GDSw$PTng#c9UhvRH(` z{u+EKDZY`MIlf;Y(h3URQ~eMS@(*kQ$^#nHV78gvkx`BS$!dt=F2+Uv+NYGa4fVueOe zrMt9kP`l__o;G5sqE830Ldq#q9db*vBt97u96ZT6kuU>^8zs)s#&8f9bc`J4bJYi5NE92cY{mbk7;!CopJV}ca%j1i|Vs=ZU z63Ig`1X+!t^B++o1Etmri5|;Yq^+$`@1Hz#9+{%8miuemPF1t+9uD|#tPg^e?0z0w zCX~!iXY_Sb&f`+uL&fW=2TN$^`cy&ad*zt{q`l6cZW}d zuCTl6x84dsZ|VDg!Zn1$8|H`4!`CtGeBFJFqt;E#4}Pu-Sl3Nc;CT%kYPEotGzHzJ zvWmSQp`o^OWwRovBKR^>Li}j&PnZnHEcb0;665Cneg!KJ_12hRvX*QTyNV`*_YHQg zz+4n*Y%Gydo7X$W-w$?!4?D~fhaA@LzU`Xe2`7jtO&WONi%C6uXcLdK!**`k9r*OZ zF>SIX)*$Zg$FV&?6^x-n3-BU)xc~KJZcQMjwrU`=7V_Fn|zMd zEj$*tB^hJrf;fAJx)4R?Ua*i1ik8hsl;y(v)3}%eFTTRoZE_H*)bUk8inIo&BmwXnxc122s&7 zXf$L8i>LNKE1|-Ek8Y%-#(}io8-CC2%a+qpRVu+XIEjoa^RwC$73J?UTaD4y5(e_s&I&7>=xvH)ETG9iQzmJ zTP#hvCq#P?_Y{U8e=PX&J&#=N`;@7}q@hpe*J`!vT*w(-@p1PD$@zoz^{=9eZNsju zEEi9D=-feOF)YphU>qioFJK1p0}cUHe1+1tFB%q>qbD$_Dhy({rBMr{{@_G84*|U+ zgNR;7phoKy7BLv($&Hly{yj;zV#)U-F0)_fD!$yG8{R-X3)ao=&&4c+;6|Tmdf3Er zcSbdwqChlg96!o+Pb#a+A3yY%uOgfG^9N z`;(jAEhkIZJ}Q_2n~f^#qq+SyYbW+9ta{%eV3;k4<9NzSxoZ@gdt*VDzP@*eM*n+4faF z1h460yarrm?K+lYG)`TMlqQM%z+R6*ObiA{9%dh5z{+EC_FKvcTSq@r1v`wb zX_b9524VjB@@~ZxPE~f*TP04 z8x!M)H>1{h?o0&@E?lR&K@t|Y;Trh9ljY0eWlY~-XWFLNe*9#KW1P?2@EyeJ3!5;z zCN{Tx>*D9jfzq3L%+#ALh?T{uo%V8CEo}Ikdq`^4-zlSRCC8X=_QXrc-YPv z_JSBe+ON5l?AYVocf{;?0_tV$s&D$tC5_YkM`0p&+IQ?9k&w?Pj&`jyO}#}=jV@ij zfVOz?@FLPw@^I_xg^BqpDoab7yf*awqZRM|rAxy9E;I9g_m?Gshv>#$5Id8e@eafz z1%!LCb-?wz8C+p>$2jGJln2YqeddF`@7${5p%kO_N7v3Lo=ea8`urAswg2;DRQ*?3 zzQ%8dooYVJ$@m;3vX{fuYj}z96R%$AWbp)^`x0P5yi9%T$R|WaerFl4p22n7MXNkD zX)vpzojp&|1(ndrk914qINyhx`gzMKb4{OZBMfX<+=p+~O|r9;Ejx?I_5IpOn8b^UR1S%Huv{Up zE*BkV=5n<~m3KN7J|;_6m*ZX&qXw6Ho{90f=_9O~7+2~G%xozr62Gcd54~RwS*b|c z9+pT?fREH#AR44~9gB)1c|Vibol&Ey-3T&ovmNKsZR+hK58drn&HCbSV%vkuv7Z-q z>Z7ASOI8jE5m!MXI%jL6`a)?;@??oRh~R@Om!LHbDEtg!1!;ZRBa6@=dW{by;vDLQ zyYTpCz5@MwgSdt~gf|T4vf(CqZpdoCVzhQOEtwSK$Ja(gCd9)ngl!C~Khj_83oD`n z!+(QUDq8UQhPVo%{WEN3*zO+{4$M>O%-^lXU?=9gcg`VvZJlu!iorX zL-dt$O~07KkJ!2d5F-n{@A;`5IXm`3bJ$7ebH=xGC>Qc=0byjUfOY|pHp6^yJ|SwB<5f0|QU z(Y85l7XvN^Nw>MoTGfLu%i0d1eBvcG+VbFJ@Tx(OkvEc(YlVgkHk`E`wv{q8%Ovj` zg3)z0aGUya_OW$0x?pE(MDLB6>4s7pR-=3w+haadd7OODVRD4n$vE0p{nuB&p7`(G z`bA=a1?lI;(ftSlX~B<*N-W#(n}HJ3j()CvO=)_hcJh*J#`hv7HX9zr6Wp?#F=3A@ z%nAx^XUJZi+A7pJd@SXuP0Ji*#KqSN)L#Oc!2t%dS10y%qnpQjDHDz}kDc@$JIMuK z$y8Ci4o#*{rudAnnx+M%+NY?>Hb1hVOv#7av0Zu7vjtDC7}QkW@#%p(W^h_vJg0TW z_L-UnTG(vTS~smY6_#ed%{U!d7BA}Flnx+N|BI}I@!?NVkIJbgw%jHG#4K_Ty3Tx#m+!F{lQc4R z=4rq@>eAw6x?jc|AA6}QKS(V5O-Ns!&vrf`BQWkKChn5^`lE=Ymut^2k3tCyjPJ() zHf6EY&A~l%jJG5xD1LVrtZ(>fR*w(g5Tat=$%1k2?J%_xAu)%q--`&~&JZr(!lMkz zC*Pv=hz{MMq{_*X-9oiH#Op`>PYBz#aUzvjs5j4l`@P8!eM%^O$UcT*U(`;X++mrZ{7S&5W_+WqxhQsLHTE9BszoJyvlMD* ze2>#Z+Wx0Y2}vr}x{mLy`xpRp3d22MpN#+rrFctg?I28MN{SyfPwmv^`6;t8P*}2Y z8}dYGm|*J)tQ-Dnv7q=o$i+)%#d!LUTFf2~_7fI<05Z+c=!W`|w7aCFf@kZwUV74c z3gQl#6S;Ur?*F0cI>4!J+xS7UvO*#wA|tzyEhX7|l&pl1J+h8fB4lJ{%gV?|$fiih zNLKdV9D9%NKK1r~z5na#>h)IV{Ga=|=kNaA&r{sIed9cPhvKKiB>l~4>cMARl-F#2 zY$zAueD7@FLm0pWma zgZfGr9Z>0fn*V5yPYOKw<|B z!S8w;>?1={0q$xL080yf62fC@9LR&+?&&AXgg81~zx+*Vir%QBf~N*?5D4k`*UAGc zvINzfLLV29H2&BEzQ#8yS4cY(t`&ZO3bG_m3^vmI?dZc+@$aO3b&2>QOw zwergiyf^lC4xO?63_p}|tyG2Cy1qWtZPMrZ=uHU5Qo&hpqhf}p??kvcrLJQRH^O?P zriXXuxZT<5h7Ql(v$-?~9Bq_RHTf#0#~7)V2ln9aTLUd_)peUwm{EujxLeH_N=Z zr`n?MSX=3EDJt!bQwI87O1ItJI&9li3kBGF4gfRR;S0_ft`)3(KeCA`oe(*b?Wpx? zawh3+$>YJ6I==Zu^ODgI(iFkkf3^c*$ouz&CS%k}#JC)2x57qe+T`ot5Ew+7uXn`A za(0YS_$dw~=OJvO_i-nCGEfuD__xLUR64m?#ILi4TbTI`>{rpv50Ult9!tM8NC=w%UOYdAt7igi#`4UCicdeYOihLC4DZ6Lxhdsx+tT6hGh0vc`-g*&bc>Ow&Jm0c0VIYGKu zoVP>~a4p_ms+V#dw+IEr1-cTW6vyn3Vj6PVg%J^-Dm6z~4qWrsbkN}y3QPs^iR&;oD(B zm-@y7IFs<7hn4W}m{I5BNKRaN+$TG@^)z+&rb@E^g8^^dxifSS3A@aTZPQrrVq#>W z29+NKeSd#P(=TCnHN}{KbdmiUH{+|8hs;6(^%pT8GmQ*AtPGP3cb85QJju*_d-Sl$ z*GGik?~Mm?@q!Yaj@HgvL<@moP{0TNK|ykEv#AU(m1Q>$02{1UyDs%bW7IY*bT45H zuu1AZVfFrj{}rncr=a7K43YC4@GZJ0B95oLW`nEJaS0lbFUD1;jk=WZop;TZQ>690 zzx8m-P`M~Cpk^-j`OD4YItvfP)Pf-J`*CJoDjbN5)CKmeOu! zBz!Q`27Ke$?0iQJiEIyRPm^6?EaH$Sa+CV@Lti=!=`xC|6|2O>7AGP7zM zdFCYqqBL#7HKwPRBdg=&iNj)f4*tkZ70ImxKJuy9KQcz7d^~;umo)x)qAy34FAjW) z^f+UNEGfu~`9aCeX0-v;7Vo-i6GiJOZ$FUuqo1Wc;q(V(rPt&h>{ckW4QY)ygB^Aq zcIbY@;Y@Ko&@vAdsb$qz%)c#5*F|>VTQoL(d!$+&8OwF8Tp~rSoG|dmCB6FdW>p zdUZs{Zh`d42tfKZuwa==VVWX?^9RG1=jTPzs)na_Bd@JK?%J}yt$&>S!=gzWH1BJ* z8^8)h6>nd;s-KK$>K1SY+7a3~&D}HRb=V|rT)fBfJU`&@(vWPUIHH4}uD5)4r)#xR zG2}c8_)gc|nZ)3uQJWuohvMX$BF?C<@0$9Wn>2bHHXxRvFg|^^9*OH&KWuDX^hTp1 zOOIbkB<*Xo#?_#PO!^9v&n-UE#_2Ohcyl%`2sZOZI0R1Zrl)t^mG`KOmP128y$_SS z`FO}<)vKER46ma1r18)FsOTRGZw)soe_S7>g^aJ*c=!w==!d2#M^yX08pB3j}C z*0DIr@W&gs+&&x=Rr<392IT2!W*)jp@QK&Uo|4ZOQQFEO?y{32U5coljVvv@GV`s_ znKYb#+N5QsS-Xa&mI?(B^i@a)gDqh1qblaQA0PqU^1emFu9!{078 z0lj&szQeD4?klN^|M6t)EDFgf1$OjJ8SG+u!!5F^sS|?l*$w}ZjhWeK&~-CIw2Xg0 zVFsQ5ke$&#k11SWzY@m9b7y0IHauOJHPv!HtEu1fc`vE|yJ#riT03cX+OmJr8&zuKTj zANeZ**jA!EYj(`axbSpjFfN{E*dTmC4vJT~_O_}1XNyhAeY1A=@1z8L9kAbW|03lc&)UA5fx9lmEBxU{#SX$@wHD?0C>U>M4 zuataPj0}?`4R@l(H<}F=^+Ou60b&~Ue4OGMT%&xz*njl2FS2xYGYpS3eC`XO+)Q)x znyfrdZVpj+LI)xC6oyWa`b|vKJJ;@x*;ziS(=S`Mi1jatn&L5PS|rK99#TCc2u{_K zy1*nPkSaPiOgL-$*g->9B#p}}AD!!bttV>oI~5r4_$~ewo2yYpJ0ouwRJ!(X$JyS2 z9?B}cCCqjB>^as>s~h>jVJmm>jMR09q#RzuUcwzh&H4=jfcT;@_9J)rK0QMqlw#Fm zk)aGx&C!R|b`N?(TS@dY!@K=VFl zb6E(Fa!EH1NMydVNHJZdtOGIzJe4J9Rg!G_;a0YUk6s(e#t#2J#!8m5v! z{HKd&>WZG){*=SB9I)gp;m>VVpzJcZ!*12FgLD+H4eb$nqHmq-O6Sul50(-6%mt92 zp((xV*E{DoP7)8E%f74J6t$5HQC|PHExR$Kc}35T8w+tg8LGy9(+$)smgh3^`zW4D z9BTai4Cl$yr=KzE7Yv9MPxX^0{$?nEH2+)3AZX-d)(S8DD6{IpX?@yp#j3!IwZ0CM zM#)L_hc6@xd3KMATKwkNbsUXn1hCWYTNl5QN=SMpwp<<$3u=>7x;Uu2$Y`hj|yEzYRNf=5U@GtEUliIr9s;>Sd<$ zhKPZA+%{WAS|hXPYfpsFG57s&*J>S+gQN^V-%?ca$$1Uku{KT&9>xXC5*r0w2c-m> zzB2mR(k)@c+ze>Ksfad!EuTe}>(oxzrVPYbGLhSM3{-6kLZxAZ+N2lWX_!AnD$B~1 zQng&Bs=YP(TyCR72%Ms|l7qEy&%k`dRiyTEU zHCD-xzbOy3a;_xG}6Mu ztY|-`FkmI^s`;+qr${vmtS=zU`AYkD<64Kls>8m(FRJ#B1MBBhTKSjE&wm-U5~Ad} z)`m4n)URB8RQ*tAxjKC&10N(?vChZ598A)W?T^OSK$Kf0E4nAFxZ8JEkHy8!bDx9- zy+8}=g>z$RoOPPUDC$h?nN%X|qxZMJ4-oFm^A7*fIHgAd3oXapnmh|%zFG0Elb?Q4ywzjGxvZas%jnJM zYVH_}a^D71~-(JYlq{ts6xTl?exxv9= zUjVBsdn0>n!_haE`|AlS1j(fvF|kHd(_S9g+rwANu%!;F=&nsvv|pXJa;8TWJ^b#7m^n1Lue|CV$cW=i zF_Edqu+~IUq@6!!$q|=l*Go` zUg3LZLt(|IF$fX>&e{%N*Ue?!yeuSclI^8!|5z$GA?b}*3D=p(Q{EGK5Jr_(9=cqf zyqy#3-dg%DfAO&$DVMo~GM(EEI+sA$5Lgd z>J4Xfv9paqh|b!GDDZcg)l96#qpq&=t!7zDA5}tMD$Y}xL5d2cP&hHg0VzNdsY^1hY2jeOKo2pG&63>?DboAS$F2tkbM)uY8f zGY)(#Df})eDSCZQd3JRs+hD;p=4%^iO6kQq0u0C>w@v=e+Gr`MJQrYAYy4$EI( zT_-PaUO=jR-q5UU%gQv}6fMqm&GyAGec}FMVDiH7$C0R`w+pU62uF}lZys9Bl&%ZF zelB7SF={TH90^_OI1t)^h&K<1gMLOi&FpAJOh!l(P%Hfo@ewDcE2x2)thgHD__%V; z10q_}e@>r5tKYpp_x=9TMWRy1I|%9Y3N@*?eaHY?P@{X=^Ydb@hFFimWLR7z`GSz?p34-i=}9ncs{`0#;^brwXEOB3r+y&4ka9Jun|2>2$sYf;)}z@iY|aJi-Ev z!U~N}>fd8`2lGk-H!e4^M3sT&+&37aLO8CwC25Xi8XYEvI8j|nro3sylNf)~M+5uo z9RB0?Pr)fFwH$!n&g=A#O*mYAJ2y1t`dWQfo!S@kB|YRxFSun&KFF_H&S&u5ou`*$ zwd`m+5FPK~u2|nka7+Oxz}aDp8%yysk1beptC*pZF_sQ7b{ZTBqvp)4CUmeVr%{R_ z@h6$kS=*vkX&3FMR< z0scR={6yjvbk`6Vj~6Tnv8`dJ^4Pc?kfS;5N;@f}isK=L{wbLxV5(}+!7I*%kJ z#C=1<)toNhzKuRAXyb(}T%nNe#qR|(*9e$U`{IAsoM72zxGIAs*c3_8n1~_ClDre+ zh^bKd<~q&SHir6~ZJ1%H%wN7)_B@<>>dS`#AvecKH_GJrKq=9zS|dYYd+}*IB>kL= zU~`*~y_UVq+Q^Rz$W{kqa<*@r2#B=SJzGY`e=pX5adwr_QY*$B?}6&xXiUR*?T0IZ_(8tG$FH5%_ZtGWsCWjG+Bu*Xtb9fa6WhJ0n3vASb^&>lKgsnIvc zqU$pi24IOak2tgpZtaVr2?%ya{QT~^cXexloFlwcZuSl31v#G6moLFdjMMS_@6<66 z-ok)!7Cdh?aI`YW!4I=r95H7dd*XC{@2Q4FJYE5pgO!1JF00OykOw`sDGAWXU@5H0 z@og{hj+_vZPH!0=4`slvFTTyy0b*fp+I;rqS!?UueNxUF9D=6SKZ+R7;)^P?AQ1PB z^vd^E#{nQIbsbf<>~-wj6vQUnioZSAr%wxQtK36x38#MH zv^OWLUBV(;-}+GD;Lx_fx8^ofd6H&!FYJW#7kU>q4 zEj%aiOKQa!~kmD(3r2%Y}2*Om7Mm|e!`p-2kzY~10%94;)aQh3Ok z`a$ujZw1_8qmR1ki1`=>=yNu&B&7ue+%m`y2XMwnZ2MGd-3C{!a}M6m!j>u>8M|4P zPGvhdOq_!YT`_45AEUsW<~6Wq_^5<2_+s{y?&@ASY`s4m)4_%>%hW)oHqfpdBN zaD+OO9OuS!?U?ck3GLk&Hdim6sP+os6mp`ErQdSA{O7by@|Xz(H0I0wmD;}9FJmC; zq%92Ya%XYt^Yz?DgO|X0ShhI~z2Sg5=&sVJ_u+cgzAXT%Qp{Q#g!=rilS;w=a|b;7 ze;G0mpT_hQ6T33%xud80X*r=CXhA?JnF`Gk=Nwr}*tzjYQPZ#C3BOwZslpVXf0cZw z%&ZD-uU3{x(PfXI0H<8=Mk=Q;*{r%&W5l%mE7b3>dSpo4KgJ1kp>?f6>t18Xa`c)d zHO2-Nde2(kqisyY!^cF_?RU-Rb(y6t-mlRvupmAP`Q&}QUmY@=$WJ_|RrZ+Vcya#O z%M3}jZ-lT@%?QT#s`-avRm(;vAwZne)D>58W87)~YUx@l@C`XhzoVjE){hvTM0CGZu0MJ(T52HdS2ZeefrE<%y zi>{+I|B5O|>MtOu@D?Di9K;_{PxH;H$zxPANbqB6;$Mlp&SN?emc5u_iYD2p|0wP% zxSqH0qDeR+>NquTlb9Ke5d#SjI#3N57#vbEDk=Gxvd$&>htgI%YWTtxG5L(2ZYS-* z1f))!#I^@lcJRK`j8&WBS^GKh^p>sji#)2O23F_2xc6$Mwe4mJHJ{01A`G2IO}kHH ze-vZwY}!Dw+TUE;_)wB-kgiU#d)!1nf9iUB-_?cWQJEa}Ih`Pta<2;O9=X1qtiD`! za$)(87_nL6hKfQPJ$s*WTOv>U2L&i7j%FHTAZ%Y-)xL})a+W_6YJrDe#RI*)Lpvub z)CbDDz@TG#UeMaDU^g3;t)L~ovZsOm^}Dy0Zldsek`6>y=lXY<6dgcZUZ?}fW;yTC zOMppnu{lz=cde=R76Cl|*%kbBO;$EU8T|Ow_m)js*+^G0zi98X2RClA_T!)M2|Ls2 z`Nd{s)~3Go4RS@g;otYN;gNx31cLgW9KUC(p;jHSPE8?SEMjqAd2Mz+faq24hbJ;< z`^)S2dXn;?T4l^tShLyAXtaCTJ`k(X0QchoRe4|93>Dm{R`(*)w1@JELDN+ed6lF5 zb4{A#$fR;QkPTB+oaHwU%T{f6V*JCwB@S8wgm-I$XIZnOis6Zh6tFI$POP*QYc_to zx4MG!@zjWWmfkk4@cs9w&-?yofoBm0%4Jm<$D~Fv^OwUCJ^Q!irN}~Y@@i6@PgsCK zy||36)07kvj}T|4$W3Dv(r{qV(eHS$!>@hy?_Qr|e7ZVSAMsQLK2Rmrcdr%?U3X8t zkrJQlca4Hn$z-Lg98B46!ynycO9r7pis>SMX|X8lmXWQ~jNC^vt<`s#u6qvp#5ka6 zLGmGrY*seX-oAP4Q~jY&stdAup1b7P?NPTk4V-deqD5)*(ic7#p(=4#5-09cGnq}y zOyRE!+35aszpa&j#w^gXV~Y!s?xn!832PCaORgyXXi-0DBa@yMz5CtenDTpXLty*j-VC3 zOI32E2cW6UcgJ|64U znvl6g=hqLtih3H=wd4CIkl|6NjNvrSW)Ak*wB)~bO){^$t2VHW<>G0Bo%7T`%e0@k zn>6h7M`RK^<9o0(;cX1S=LTnLh(QfGE9{5oUOOtRd% z#ROClbaz6+@O0s-Yu3|_$y5jW!cvb6g|@exYB5(m(~VPyh)=7Q2e=ga3f6}(G1^cc4F~PVyZd@c2{7ZZ9=@oluIUte|g&cQQ~Zs zXl#Z+v%gh$`t*$~MWWNess2+qgoU3={D{lw*YmWIDr1S? zo;c8=@0Sv>I?;}@ zN-_EQY&QZyFf0;G-!m#AIwV3j)zi%~dI7x)W4GT`QY)76^6Z|l@9CaCUGRE;50K3d zZBd0TKuQ-H=rQ_fBc%2^&yikY?Z=LhN6eoC%MoK-C%67yUmA1M<#g1ciL8=BV0hX^ zKXtHZAt2%jZ8hG7P;C6Ow)o$j?JCWcrCt!UTSY|Je;J*B#&{Q95qaC|ifw~sa_JRa6v($}Cu1}cQ$480{i=@6A_St}DCv$W+zWEwk_)Nr8c)7h*12XP zLE?RS^yIB4w`;ldO=&qCOey*&l_tr#!jk9BW+_$!+iD^3?$);}>6-`riR6J) zErqG~2AH})WV!Qo!cYIV%oqwvo{SJ&PBCOJe0cj9XWHsHCV3G1%ochUhIL=`+As20 zcLB4n=ZmsyUG?6V#ebjoqHbw~C`_a9F>!<|qm=c?3!wB{!jwA23a=v#WmtoNjAF@e zr_9rH+y3#V?#ek}FakDAqY@NSgtctdRW4|~=la;*V?7J?88#MV{pX8HjlZ0)!l^~Aq+ zjX@F;LWEKglsufvbju4Q1wp>w;9xo`1($S@pQy;Py}X#;U)4TgoQUPWSO4_~9>Tk; zqe%2yQDEcP2ru^_WQ~WtO!~M- zBJv^+i4G=h<8!T3(ceLr5Z*Uoa5GBd_ECk%$k45i0vw;Secbdhuc3u>-74{B03pqn zuen~H9iE8QYqQPVfVn7d4qaV9Wj{NGg&5O^cI1!I1qfXVM~6KHw-t^&sxrc{%(47X zpJBGbY7MCapaP%-fBC>VpZ`!B#!MlP`UO4JRVqPCUtgzGp*K3Fd0?IIE}e@$#FL2) zS5GZmE`Vp|gJQ0C%9Eq}?4gx#SZ_|kl89LlF$XwNbbbPT-HA$SD>6r1guKbr2Kf!> zrAVqmY?%TPO)tavgL#I_d5&k8Xflh-qhYDS^4y=xNPO*gqaJyq-Yl-YlNn^*EURnn z*Z^*Qy>B<0G-5UfN06p;myX`f$g-J7jRRp19jw&!;Pl&SSIge+n%TG>lco4U1sDOi z*4?*`gjD38NACIG+Lj+FXO?lWi_Q%19orBV+}l&Rc#(dGf|C9jrnl3y{jbmRdWuO+9Rw2|0PUYQj zKXP$g#Sq1>*(1fJxN#gFBm)(>vLIcdfM1K_sBa!Ae%11p`wWGR^(z(c3B|&5^atVa zXD$e%hwxi_iLaQ&*z6)bxBC?Q*a$2+s2PkFDie-Vp8>m_dw=%$!MbkB9j92VM-I+} zXK1LBBcWPpX#5V1f4hEp zq{GLt-L{9*lS#@MJBQ2=J{VFj8k7F{&^oTalfW^2V2(j<%>e_O>0UArz1z6^NY2cW zK=KF71X*CMyLISA=X%kFYY1BoZ(NLsDkG+i?$uK~J1R|rf$+Z{wD*w~K3ljCD$Re# z7aVyw=7qT3qtRb#h|`LNA&h}AjO%FfytwpY#|BV^C2nBuK`u>tscf@wreSsNjV~q| zmN<;J6tX2E&(4no-y5(QvEKtxX|RQ)VPN7|P8jTjS)nNpLJ5j244<>9I~ zGB~PKcP^!VN8Sg-^-CbAJagaG^Q8D?Iyk>V01tYz%RqD%Yo z3P0wecI!Plo!cx3gaR;7pS`Q^ouq6uK3_!!ncC*J`A;TSn1#g4rv%VN4BhBa_X!dJ zsO;LZR8KOT_IfFdTAD{T5iXgkuZ+p;-=h*>lUt9g(N)TmuDj-z<`0pO@&0}c?fzp! zAAIU9yO4*szI)lg4nmD7m*+|5SE84hTB8L%+yuIt$Af#V_An*GOpxu70JKvyL<^uN#5z4k>X%OHIEXKeeHwdS?*^MKvbrz2RIJ;H z?>m8zzgM|0EsW6~hx4dEBDVG6oX!}pZ%M2s=CK3z!uv9>f)-n63>O>zg}_HB)B`lfwcZ?E`BC7Mu-&a*Z>Ocr{V z$WAI3$s%Ck(8{z;S&cxadSLw>C@=bR`Xyawy|fuP!o+z0EE}=N@SjCof7}3TpNjxc0 zjDF4buNz#!H>s*(FeE>VYTkOtb(3WNX;<7QvgD2RV?ZoEvOwg6k;Tk`>pkwBj{Q`!M%GFG$YGC3G#%eiX>W!K342-U>dm&wdpJKA|@L!c@5xw|s{J2V#B*j>7 zDI}Be;dqY;x^d3ci#$w^uFinQ7L>M*tD^eJ`_&~HaU|zOO$oy@J_1&59SLPT83G%3 z?Uc%yitP^jQ!RIWf7bcqWPFT4bn+-uknSZ{LLvyH|;Sm-4i)`1xun{Pws zCV4la!GpJl_P%IFL!R=aFpj*X;$2jD@{Di$kmXF zcJV<4zQoznga~isX+bhE(Oo z)`$>z)pPHq)YlxhgE{QE*Rs8c;^iD90LRR!Egfq1lxY%osh*|<*ROI)j95lfBaKKdlSt;XxR*DOTi=MIR z^#u0Mh*wtP8*6jeO>?sWn)JixrG_wlxz+g;uNa?S5F{pRwM1%lgR zi_59>X?dYzVwF+fL?aVuhOrSU(o!D?aC>DWORazZU8sStv8F@A=3yY=!9A?}&5cT_n2=N#PEm+zereXiJyVpM}?r43sxQh@dVG z0k$Me)Iz>8Rs%N*@soL}6QYjmKKy@&R5$DD08noS5&iQNBK~Ft^y@>z2vY3mQjwiQ zeQM9L7fQEf(GDIHD`@M+Bgp1z9Af4@KKk}8!XBdcVoCJLHHr>5rO!oAUw+Y?5t+8i zzrvk{c0s90kmGl$aDcmudhWIzMJMBgF?xX-zvs<0eO%FkjNxpmF93>YQFfCMHPFD& zJ&*;atsl3$!K2*6qillS=3jd(eKvQLNk6%~ z+Q&yij0XQYU(IH-tUlc-TQ?lCNo!(a2EZzqy#ogTHgJ!fJTc zdjl}OPkI(t9=y!2QMYnI{YYY|11|lumy)w4$6YM>$!X&_RAU44|64EhB0U zYBs;!D!QL4ca3sh0uS12yOyQta`a(4der;5Tg{cHZY2ee!_e^)2#l(pqil+!`Vhu{ zKV$QM^B3xR(R19y%#me-RmfZ*ymZwUrlWNQs*iEaD{%Ax3>t;=;#<`3MO(59UvsjA zn2h44&S$pNB0D>0S`mm-z~&W>hKWp|9LdWVMh{^(n%@l-y9e#6HuK&GN+d9=^$Z)R zd6xmdpncZbEsiCB_9E8X*I76{`Qx$4SUPA1bBE8*tM5BSZAu1ai=dGb?2#0LegE6+ z7He`=V8DGd+P#PiaVx1+Y0%}Mo9?021)#v*Lrrbo>l7B8&8TayTf*IEE19AR4H@)F z60H?8^&5=MJ4S!jjbQ#y-lh9PwVM~%HtrsRi3IWl*oQEErH4j6-qp@-?zK7!&~U>* zL)S{%yv^Lu!LdMEP3K}ki&j^WbrO~Zy9NokcLz}LaI}KAJd8q%@5)Bt+59F)KR8WZ zUnjE$cVN^M58_1~QE$H51rkT&=RACpesMAEk)?uCyZGMN9K>^Mf$d4*{Li$riG68O zzls4^yoL_WjvlW;^hzXvvdB_0`KHSt7T<8Hwv zu@1oUq9^g%=;d(N19VZ+Wh>}6zZ7Q2L)`WQ1O&&bs^3KS13X zaz9XZm>P@$$Yu~VSL>z!??$Iw`#WG6>VicAkS{kF+#_v5SbwwdHE=KA8Tn|}(qu%8 zyVSC)(~H{znQ9!1kuQUxQ}LF8&mA2DI7A{DYoP*MD#DU4xat=-co3#h=Elx;eAWpP zYFhU;kgr6OWA8&>S4n1E4ZAYb`S6ZJZ-`dyXcFF&TNcF^LcE|J{w5HP7Y;5JlE2?*5Q|Li0Ao%WHS(1#O@v-XCZ!^C8gI|+O2fA5-g3$<)3aG-MjgHJRzLL zL-N*H>QA?k`o#4$)^zGHKk?xL|6L7;^ZiY~pcv5N3joE0Wg1@XHO^}sdh{;SiJsvG znnJ)jMp@92_Cb-hBO0UZuC{*W;ZW5Z^A39N>Ygf%)G9^%RNbb6+-|y2XwPLkMuX*w zEBE-6=iEUP4e#r_R+18;7U12@(B0>Tsr{mZT|8Njl1Fy(yiH#?WRmDhuZ2QBx@8x3+L*ebvT zJM|TD8O^DQd8grNO}!_YNHAvBYvE?y1hzVd>%Tx>b-M|ZD>-Wc_xnd)hrum}9t_34 zKER=T09Q*AHG7-GF^-x!;EU{aKNtk7WwqH6Mt%Etw<6E{T?{{7w7FUaL-V->8R|yv z?$ARmd)=aPjiQ}0JHLCK!RbWhQ_qpxR;wy69P(mnHS*Et#^M9*W6?KNZ9uwftVutcG&J{*^ zd_QUb9v-r=td+AMF@Nj)Y$#4&4fPq$@UTR@YAr6M^IU2?VO80R>Mt)dl&d;26y;On zK1`R#oJ|=*RODR#lpwfgY|nfPh9j@>)WhM@ryLyD>YN6A(~^37lko;c~ZA#Vuq zN1o)BRQHV}cjU4w_HMiN7WcX(nZ4UO(6_SF6&}CQb9`9yc!inm*-hkj&{gVlIYT!$ zB8XonHd8)W>YLOY&NsWw%~aF&<+(6A0+yL<1+EAkSdGT;_E|XauT6yZUalGD=Jw36 z`)wjK0e|e{Vub0sU&H6tE>@{i8NJ^}sA;VT&W^4$q!&uEk52SXbtJy_V7T3$STGP5ii#G@063*0pk-5=zuN za!$6FJ<3OidGPn_=`V@D=UKC&J4bYC=dls>xDuO~GUofmdYm{$ZQY1n=iH+`Wck~+ z9;RdKAKcs_BBd(R6>70JZ{J#?7uG1T61HFQVDIQThqUn_B0zlmdAR26s-0+G=y6wj zVpfvc*yFbjS%?0AZ8cS~k5f@N^pDg!69d1OnVL)t(p08@6puo259q4@Tv?r8I71&B zn-Fq|ja<+A@!KGr_;|;DcBiR^+3d$p_TFbuQzF=hzLRCCT}a-3rNv9?qmilLdCAxdU)sUZkSlZE^5U5Gp#!cnbn z=Nt?lsQm*I)ZPELr~9x=;kcHiq8inUlu1+L?P7I{P~J^=YTq59rqUlC@mcdOkHy1l ziN5(kHD!`!@6^a5e(>_SF{~~3&RD#;l#v3QaE~vf2hYWoJSxiv;#}NYZxbeXUkkj4 z7x(rIfjVw@@Y|J%c0?Gn@}Y^@#2`8BSN`nxsgAY}Qg^(U7VelU8a~|g<@e!RoqIKP z^G(jy@EOD25@x@!pP(#69v)Q-1NIN>g~bn=uD-A-6VN8pb0FEqLOj&&c%PAdN@~=j z-J(j$H4#g;*zLU;&#THbk)t%?0Q!DmxTvj5vS(%`l_0r zD!O$m@9y_^#ah|*k@zl8j_Sn75IIIPyuAF0yUt!!ZkpG6`^?Aql0xy~zSj81$E&gJ z3r27^e6Q`3fctzhNU23Z7M@Y7#inls^g@~FR~#28&B$3!=XaGqw6nN@=-EB284}%< zRG&aCYA#k%)zNb*Ge}-yoLFrh&#`O`|Gp8PEbHm90+r*KQqPQq+`g59@l8F8Nx6@M z4j1S7BfXk+gMQN!VhR5EXm5r6zGo70l?69cpRl*yY-{$!-LGt(&oQs`;+%3yVef_S zW*-!@-PO@7nM5lb-UIVr`sjC{mG+`}es_t!BmKP#A*I-B zktB<^w$gdHMC|RaU(R`?n06Lk>P?2ZKKtPD(k3PKc|^;78F$NxM&(P4&&5eEV(qwD zU#1;l9>Qatj;r(e-1ICW+NYg4h(`CKDzTto##ufQUVhq3v+dfNx+@p1PZ zcX%7#pg23nA`|`NBJ?iB$L^cG{5^HVgY*C0OHUyl8WMCkDPrF)E)w4h>+l%a%81+I z(2bL;e+$|XlC)c8F~Ixof$E~W_I$eL!fJEJC5ec`t@5H&F5v;)tAkvliVRC`mrrnL ziM#`Od4&>N`ai&*ofgRJtHWEo|NeT^UjJAp!%JNZ32k;*f&-pXN=9sS1B+fkfGolORP|oOn3+r?w1IvNd#+$-M&K2MOZ`2=QhP`Yz&&~ z9vtg_zPFQZmXycMJ<<(pDWVW1=*hj3u{9O`oj`LV&uFx>Df{uxi`u2`8yMb|(OMTb zAN#)iy#()TXn^w{CpjNq9{2uekov+aLNgWIkSsNJVNuxi1BtxDiA-~fpas`F_4D#k z)HK56d#@ey+m0-?O)IRruCsWGaPm-kx7dzFOxjIN*|JY|O}9o{j76qv)@kD^l=AoX zWvE>fpRl(x^P9v-smh?ZsIJIB9QGVMtD2CYUo}==2o`#OT1w5<{ram>ST9AJ_IvXb zru%D(M~nO{dO-dIZtL6CPbo${Wt0;FUt~M>eslMn+oMLR%a8G@3d%OA|=M5oo^6q^Z%!|ccxMAg@MCUgZa1-JsU%OvxyoeZ2cPL1! z<_8gbnTmMq2OAS;<~cM^Xq19^>3#2@zI-{$_}6w-&K9)?x`2(v4(=J>L+uz)72^v0 zljo^1{eN+uBATy#SOX=r3r3lTvMwyv>Xjbs&nj-59i4d=mY&mB%#lcZgqG^gcxvF| z;t!v6(3*Iz;c1(quqlO``C@x_>F~irlV{4ky-XL#Sz#aL3X!OquN_QY61t<6a-=`b z-nsJVf$dz%p6By-O1z?;4>zS6DMRQ%3l98=7VaP(YTd*3X35OF;@n0j=VV>_%?9A{ zY`4O4ww~V$-g(hP3w}hbHTrgb>E$RZ?e*g4^{F>`!@DZGL5CJ%5p`mVk1Q4K1_`Rj zZvB+oKPW^kEdEwXhku!qVY21VuvFQ3Syk3%7q))|1e9)h+oo~$BaXnO(h{Wg97s^ga=EJ?MGw5Ic+`Z)cA#7Z|!g1$s@y3Ve7ibg8qEF z&4yX(^DA8o6eq?Ov!)q1ZdeWyL;)m9_lUaiH{)XCLXvAt#7J^fAyoBlrjtnt!$*RK zu;;ghBweSFQ6kE6Nc756pCdwq1@v60P)~?=HaCQd`G$`{QdHCM?8*e7Y-koDji@}ka zCp_zZQjH&ae3V17v^a9otZp=(s<1ow!WEJw9DVnoVUps9mieifV>v@E1Xa>O7Vq;% z5ND1{KIO`@y_-N8$J7K4Kcxpz+&g;cCVg`IHg|(eGF`z(0kX5WlFYJKRD>BH>>Tgp zEC?|&neV;N?+(PTEp;exAiIO4A+ZxmaNLEE;}()mHOCKO_O$=08&drDs^5r2k#tRz zWW4H)VEISwtwyg3ATBn8#xG*gE^|^}zX|WC-7~O;-k;K~F)T%4G?xYBLh~;s^RNp^ zII866PNp73$|wl#mAkg)YDm)gUfKM$uHHoSFZ51P)E1yq9%r`lluG*(l56veH*Pgh zo;cRa-puh+1ofTFvRTfKwQNs1LS?`i=7ft*U*b2@pF4z3*i)I|FKV3yyC1Drqt4{bt!6=Oy+`=K{rWc6D-* z*IdfQJ8z}TpNHMPudjUE)3_G+DEG-}i2(3`D?VbOPHtaLBbY>L?u;LeLbwjT%a^{J zvpTpKjQ+nJWIiPP>q7z7u*E8w-YXot=6B486g{`RflVEwO$TaO^AE8&U8 zZG23%a`5AlbMOcTQ15pWwW9CgCrm8X9tI+F734+W!sM) zUhJ`KTXOP|X*Hplp5R2pe{VU z>HlDt(oAj8^JE}Om=O{77Nm6vDvUV+_bSRrw!xj1%5`M_5AHJ>hdV~HwWA+CzSCs+ zoruO>`V;286)arGjvLDMe$I3A4btI=3(;}g3{&k<`CTbM*FU@96||4wH{Ob1z(B?Uo(9gwW7 zEoE9!*clAgJTb^gZGzH^ookNerQMoB4GH95BISs$!I1lDU=~qdSO}J#brJ6cw`DOn zT74qcSby~qRCyY`KKb{RGc%2MYKZv;#rHfIN>RfsAk#pBLL;Qbq)(B&L1U|~GcLii z$}rRZ128TZhYstJ+|wCbdeTxFvj=Amgg1g3sy5#Ut9yygYR1ZsHBcAs z5+z8iHm_;)y*hB$mNrHvhTtNY#J{+29t52o)Q7y+k@Iez(Nzc~K50v8OC|WGt=#;kvLlS7(r{c!J`Bok7w|CEKB6#|s+Eb$? zMvD}kE#CX6ZkNEtK{EHq25Q~(N-h;PBOllZXJSv=9uX6?M?IpC-id44I9gO2bhZ;) z)uwDpR4`14o1KME3G&QuRF)D(tKWj>^I7uQdQnbc!Dn8alr;Sfd8~Pj9E}YgQ zP8L@WMSj*m2Om6DBoqxpdg>-?R{Ion$hxKO4GpEsA?cQU9& zQ*^qAUjxptGvH=kKe6Gx8t#~Z%;=JkmU=H!{(^zM!Zu=4H zwik-sFT6z(;-M#$WI4uj%kHCXj$_;JXm+Pu4jr440td3tfsi1KV*uVt-N*+tz6qS= zmwsly6^yq;x!R60#y8z8k!S&$30ma_%=-6T1^+C8v<#g!WU@B=ysuH_1^RWkmHCbW zX@HAH??ovS8M`Z=riZNI}q z(Avg#I8tXPyRV^WKWB9`#Jg{s%+LmfJIqdIotWF?)E0?1S_&Z~p@4cYS;=%0N45j4 z3Z=#<&e_AkotpUOojIEi9WqLn52~LVD*RG$@AAKP#D7MRxuy-Y+{6HvHJQ6%rJi_V zdJwlh*YwF*avn&zm1}vzvL{A&Dmuz?&F*Sn-maeZIZ*#;9ql`KM|*T2>N}s8KD-9F zX%jEnmRW*qf4C#6s@fz?=?O6?X9(!=w+8;a{;YiMbUxXagihZL>c{`b-g}2Nm3?i) zaU92)u`rGW6_Ig_qJSV4kQ&P<#sVmYA|)y)N{jTC5X&ekZ44qJB?>AMN<^eXVgp1< zL=B-PNGOsJ0)!BfkmTD(&=H^CHSfHCeb4i}a9zqdCn4wTwb#1Wz3#noPAvZV&MOT7 zMrNCu4*dDuovbLw6&7neY|r`!2fOVK^m}sO{-GO~(FC+|N#m;2dg ziZ=ah`jwg;DREe@Xh4`{i}etfj#SRTzavWci>kh(mTXJO7&QQR0{v;EV~Lz)A`l9i zeQpM<8v$1faHH^d#!Mn}<1c`3Ru1-UzTo=dwl^onCf&%~1k|@q%*Sc{RT*X$*aS2xP7pS9J5ukXz=BswUo?oD1KLDyc@M>(70eF?WeM#Dc** ztF+NUQIMdt!zfB&H4u^``p$|% z$^b}9nSty&6Y1*!2P=~4;sv>s=TiEMy1wswR&K*)rC!@G)6#BOyc2jAMx%9!gJT#d zdUdL?{UE_4s41XN1?~yZJg^eLxTaz}`I(X?$s)_CAvuJ&ZZv{8(6msz^KO63vHs4W zap79`3fK`cAi%b=KT1A41G#=^WX{aR3x4+n=__b7&J!?>j=1fBUi5)v#x`7dm1AGJ z2~6M#maICnRU=ig5`=12sCNg+*B9tLuJ6yzag5Q52AV^LqW}4y&5ijUYDw?;xsD61Xy}b6_2c;OrGN8f`*s~wtmBr7J!Su2}FnT zQ2*0{rKMX^yj1kx?n(<&T0T2NLDRYzEfdG&{#<&Mtff=81>+k*$ zK0jpso=zt7v99nKA80NCN|gAFUP&JCX$^q>XJ2e51Lo-uFS?|!=egJ9)|NTW=Q|Cb z<;`q8;WF~vQxSM^*X$aJq#1yv&ZtSOgg=*Px=_cneHjS5lxlA$-QNa7s+e8CV0eK@vEKl?0NvHXT94t8n0Z%?o<9@Q+~8D zsM7!uvh8eL963P=Hj4hHtB^QzbI@WJ?{hD}1HYdU<_8YOAGGZNB~4X{^6bOnH{@T- zZ&=nqz=Ou=qHg6tix=%hKGF^pq7=3?j;A3#sW+7{ReCi>P zl$mXLI$W7yhBFiJNXbvcm~ zmuMdA+x+QleaRrsVvALAaD%Ai|mj4#m9T&rREecmrkmh;*9D5T~dO>B< zs{)XLD8nmp1}c!ICH)PgpImJ7;mO+(9gCLDKKLmm!eB1V9Q9Hw&0~k=p5dVLG^`QL zj>9+=01NiG8G5)m|_RS zAKorC{>wac(HtK2QJ5p3Z%G~~uU(3gZ2}AB!#Z6@w3crjsm`^fo`mD2B2%zT+qQ8W>5o8jx!nrx&eKegh(P0chY{uxY6!y zu7FpPnE&F$j|bMwIx*kF7SOPgY!^W!R4Tl>`higYP9WZyWGLI+55(1>d<)pa@1unN zx`X<%p_cmcHRAGS_W7JOb5_S|5_|Uqmd+3D7|i`B%k-i>-TQv{q@%8nSO3lPJ;+n5Kg-KPZ=<5ag-|%}_Lg>ICBMBiVMS;XsT1GP!rn_=9U9 zqcHmK=yfkac6;-S{-C9=RoNS=Ik6*#eWk!!RJ!S1p2wN~;u>#9+GFi`wmr;c7;nf!8K zQ0#=(R33>|0M53;R2-aS5@-s+@aQ z(+I?{<0ZeorY*`@wAwuZp(9T}={WR}a4Ex~6=De=-np^|5p(08SfHG#`$8QYR?NBY zuLh+#K)02n!a@a=k2TAF+AfBq-hGzVfCU%un{9+V`+-Y5Xhp74Q>F8Mw$c8>E6R#i zo&|RHd$Qq5%1K9G!^#6E>cRKwHv*R!V$(DC9eytk5sajqOwL#wq@A-kE(A0(*}WIO z>LgtLesR@I@V1kJ{z0H>I(Lm`TcT%H4|%!dE+=bWS(s$$W`|X8hv?%%Tzr#_)?xRiGjz_Cv0?}|JV@QT|hRaQEZZGbp~ay;Saj=>WfucguggRv8#`A(6X(YMXV zM?&fxblksWkG9S2o%`-pkFh6bssf1QopVAfoBL1j>IYdyiSFhcDg)>dx_m~#VYdX;vyOMvFECVu4%IjmNbzyx*APcN8Fi3h`u(+&LYtAJeC^|c%qG<7c zrb`w;t9iiF9M;2#FH4^Ca#XPa5|0zU1^edS4g&NZt2f)X1<<-CP*_3p0~lKaSYI-3 z!zxUK5k9vpW^A_>W;-rZ+}63F_3`eul|`e?$JhV-x<_!S=KNmp_|6abrRFXUSO|FR z3Z8iEy71~j@sFWIF2n<|kx451N7}OPJqp}g^YpTCDAtCm(U;#7!bk)0{ZSj7vGLYtdIg6Y zX3?JemjO_RaCOa`{CW_8-(O#Au;-w2Sdh9;e*8?wJPBnz>@Qq7b8Ggtg|YFhioLQx zid=7X=dfrl1liyk_7@QEOHBKD8&v!?CJM4mU;2z+m4Twu`QzR*-;G|I^(z-3m8d!5Gp->;*+pW8wm#-S?e1i0(f^rSF>i1tYzYjA3 zmCQ=TS&2vWwTppR$0m(TfWT4DV&*>xGg-HUm!JyPL?tEmPICa|WY_Fx1JKr@lx+)p z;Vh{@tpA=kYeU^ZXj#4ztaGf9wKrc%yVlDCi#!bgzm)V@YOL4vKXM-QkaivhIUcOg zzf%5rs1@*ZUKgSYER9zs;7crrPh7Rh|1ka_=(x-x*@y<29qFqpH|93QU(x!y}^ zYP(`49yA%~m&viZAx_{+D2(zLkg9BVojqw<0D@wSYg14sE-D7nvfq>Z$`?v)czfs6 zRy+N5_070pQ}_FxsDmn<>SV9PJcEJ${O2dAue#k*Plj2wF$Q@LWg$6E^930< z#e#j-c>B-t_X z()ZuN{_0qP^)UoxYhD-CNZJZHb!uAn)+L37ZSRb>Gm_^3ac6a!vvaTxr_a)vKEJfS$)A}@Vo1FGcIpg90BY0**a8r`*sngG2Cx6S{Wy2>|C{0J$?W3 zJMT78*E55<#X#52Y`XIl)C}@&C0~`jex9oafgMxt-W}MtYOqpLqOxTC+PaujbVZo) zUqhY5`;?UqD)WKte}`kw8mo-kc?`HK_*49qgqVtUD8CM&^1g3jm6wWVZf{(i#z zT0{HCAM^+Yj5Q{^N1vGseKCtLLE?7Ro<>#C3Tpq&BiG zIW1iB;PngfFTj{=qG(1fOa9Qd@kJGhcvv+(#wr%14`Ea% zQtO?pFT_jZIaq}79#gyAn`LX3`=g~nA8vRE6ywADU{=n%+LM%@@7o1NE+43?%XJ;R znEA7$?*57M&$1R{f``m#NB(#OR>sfyPiYcDZ#v@FDMJ>h!559ZE-U+$ac#z( zj|eJdy;Ey$dR7YSCR@+Mh&&thn^}A-CTH$7Vvl!cMAi)Is*N?rcoJ_EAQLafCeW>Nn8HAc6irZ zW08l)M|2?F42NfKA*aSDGF2L@A^=CqX$+4HOatnX7P{cc3CRmUdp1+mTxn5#x0byv z3^+O0m~ZN51iW2-j@t5~ZEx6k9)RZ43$&D>gw{~-n?S1DY5L<+#-!%4{DR6+5P|`o z&8H83)P@?WuFF1DP+V8Ex%ck3*K;p83APVNO6=F^X*F33O5!i6z5{whyuG1mfF zZP@nTR}?#RE4PLMR#PVJIW^0Ez>?;i+yE@FoA(a*T!PLuZrM=q*J95cJKTBV`XinVHbqB>k zkBX}|%SuP;J7zDR;T34AHP~0s?dA@aO(0Dl@MRy|=~8Rn3Lk(1RD0}CZ$jl&9@ z1@m6S4`dQ79{{ik!ixfSBQRFLC_`?u{`|=?6RcbyoRWE_khTo4SH135j@c4TuaH>j zdqBf>=j^#lx!1-xD^9Vg+ffbRVCQ$3jVoV=P@!hD`Eb}-v!k~Y7oDTdS)Hap?;d~@ zXcX17siB}_*uXp4|AU*I<_|FTh0@J>^0=n~hf#Bn2an8-*ixwqfvgGQvQb$h5T~6% zJ@aO%zFXSmk|0rCUBW!fXBju6a_{2X4=T!~%PEd1++7Usbr7Bw zP%#UWdU)4&5h+I-lEoJ=>0x2kW_6wZV(x{D8w9;)BFW`VqS_67tG06GMsmdYUTJ0y z?1b|RM(^#vo^oG@oEx27eftacRFzh>=*Lpo&`8b(l{9>v!Xa1M1YEDINpt|UlEqY4 zi*Y1lO1$LlpF0+5!yAP`T${+9et5-ECkx$7t@Y%X` z`OJas&N#>Z5L~Q#-N779J8JKoKg)0%1c{%9Btg2TpVidKLiaLt_nWm|xK`u}gZ*_7 zbo>CK2j-g=KnwQTxpyLHwu={jeqPj(B*-0{*@Ekt+2h}x0VeEhFVr&wVw`9G{;i{z zJjospUk~21hmif+{GYUzH$5_a$G8C4BsXgrMLzCPb(5sC9BA9hh){Q5*RM!Rpxq(TDzG-PymwiZZ?50Clngpx2QK zNk{T)dShF|?GueA`Kd+#c5JueRsp>YU)m>bdP5!PQ8?AanImT_Q&sXV|Kh&loFD%z6YAoDR58{Z&9PPb z6V6w?uveR!Rt8|JG8G29Q8BKNwYRbXyk!kJxVCA-CqnAZ_?HF?jl=tWt@>LWif7#c zDe|Q>w7?8ZUZLYTb}4FnRTq$1hU5dhpLgPT;6uFZw8{-ArJmWUF4rx|nEkLw5`qp ze;CPeDB(jI!WQ%lTh5IR#*0R6DIOW-&ypbh0Jta$_tZ+aOaNTuQsB;g+?ixATeX=q zkqtfn!Pr~{2Hk8q5hH43xtazjnPt3byXVmS_A?L9r;%Dm1i?&Zx$8CL`GK(|FDB-XQSm+(nBs` z=NFDvHr*e()>?IM00qZWLqUgEElUB`>HF+OcK{k*xl-Fc8#IJ1l?sb~Ujp0yJNs^m znszoR|IU?~S)Oh|dZRPze2nIaf>+ualEr4Wvrjvi~DXnzvOt^Fpu z!C=e2JkRoB5TwU|=+0HV*p;C525=MC5wc^#-Wfy;E^X~vzpI+E)a9l>qRP5sps)SZmS`BbgyhnjeQ6n32*GzPoI z?u3O2hWZgZw89m~i&AL^X@cX?J&lVg*6%^*O9Rk|QL|?%#|#ah-0!S-b9d>oa?9(| z^*=7(mku)7^&R(+S^6_!+YPd29C;x+&di+X8-AtzWr>o$Zpp_WUUs=|z?rgs1#`>K z&rRMN>nxNy?aM>_s45u?N|IDqvQiS=^`l#+pr0<9|7`nv+T zXQX1^Y$e#<1NWEAw)$m!K>92Nr}om7y3NYJsu_5QL-ygBGqc&TC-W}ano7mZkkGz6 zDG{`le;i|8_=tPlTKv<=f<1o;^HUu>4;l}}CpVO)+CLj>X5Srcgo<;p)UK(q@e!mZ zXxpu@7(bgK+* zYIk15#bj~%oP@bamUH{Tm!b=7^j?C6bZ^iXtLyOK{P{EA?>OrI-8pWe>e5^E-b*|0 zWxVtb1vLevQ}_Aios^e-ba>nZY=h0N<{Z5qs^G2v;RmqLRn-w5-kFCtwrZs9!}w(p zp%`*_{3hhLqQEwp@Qp7B7|?!Z3L2u;QdBktO!&(~>=%)gw59Z^vjKL{Myu();Klun z%i8(IXQ>zQvt@wqf|h)7^B5I$)q=L5I36nz;AS*`!xg4_DGnAAUR9h{p5OocF;L(e z8cKd!iE}ybXY$x~L7@iFemg$`EI$A)5L$+V=KtmlFAx<~3lR<^^eG3xXdo&JH0GOi zwtH7ye5-kR&rSQ>5@^yTvfT|0Oi+%%Zg^nFIhTbC7C#U?7YKUdrvKSZu`CAj!~C0*$;ukjA+?& z5?UNrvXy`4cD|=kU;P2g7H)sl2$m(unw9rqup?aC@}dXE0p7-PvPHDL)kIV6vHI~+ zytEK6)vufIDAhA^|MeFWhiYkQ!LgeO7-r7W4|g_lpFwir;%>K;O)50W}@q!b;h*-!CQIy|9S2>^3()wb}iTlw9i`-GHpS!#SG1%a*S@22xfI z26PQ1ndmP8tugBodrc}te;D&8GXFz|&+Y}wxqfOvF_45#91qPr5w-ZO1*L2(aVc2% zSeyR9N@v?G$e|i%f!WshL;YmM?%BN%%MM}XPUtO9#&&=#qa^`QavHP1C-FSD&J}@B zK}X}`#&D0Cl-g#`IH$FK$&Z?JLrd^R2&g)v3kqpw0Eb_&OkBTZDyajHT7qc%H0(}0}MSLz4Zul+#7rw%Z6EjbdNNY;v&uB zwYF%_f}Jy3N+trs0`+o#!Tb(Kw@+Soj`JnIb*#OnHV(Rd^&yQDTvQBL^;ZmsBVHc3 zJE&e^2*UA@z12dNq*5PJG$8@uMV+V$v~>@;&h!6y?8pu@Xwd46SH>@-f`&QPK}Y$v zk-~S>m@7pes!}U1XhDCOdY7V2Nyxk^<#0PGmoa9bldsd_WB@4Jy8OE4T!N}Ma;WMp z+yOPO+sdq3`paE)2ffj z2tb5p{jy*PNNO)}LjOaFnZ=JO9PzK)^v@3m@C4G%qtEJ?x)EdV{8O80Hjd63?4@Y_ zHhZP-uQ|UrvU5(snIxF#;Xu6cZ+!>bzK;uI&Y5Wc{0;fV(P6NCzxCaD3YJbF;;|gu z;O3j3J9A~BpBJ~kd1+#|`b*pLe|-6;Uq4rF|HqgA+5vgP=}BJs)!WZ)`qF6zY;u{JVVquP=f-rd;?^T~ELKw*{un^0zI%+L|`YSH}2z^B>LfR6%2^w*EHv z`|ZDM`4g*j<)tj}Pp$Z4)#k*tORAq8n4#=2a}Mm6U%z}g>+g^KvUTpS&noGcLtk(F z`R8{NnfmL4U$%dP|0B|@y1V_$TmS3Ek3W5NOWptS^7g%-dyoIiOVELxJJ0{e;}Zw} z+YZ^JC+Xe1FBh&5wvTpiBYR*X8IQx5&@J=6`6>E8SkT1rn?IPct2%A|DfWQwlg23fk1Secs!Xy8ZL!fB3!h>OYAt$Kd~{=Uo3s-DLX2 zQ|&!%>Z$gacCM-RI5!p@#wa0W!PqoK?&zPQ$<8NjOHXX-PEixU)Q|6NpozA_CaQ|&QL3cj(&G?+{^$TWaVwZ}Ao zOtr@}fP7<*X#kmOkZAy!YLEYU0CIxw6grZ{k)6l$U(`O?vLeg!&Ia8r*Do%1iXt1iBt2 z%V-5MY)V#^;P}Uj#f=zS$UGyjcKuU#%bV3o|J2^HJ;wMS`&&$g|M6Ba(0-kT1!zilyPYZ_#xnB(uAX^{Eb7E`vSL1v0MzS{Z!7i5+jL3hUN za4WRcG88{McD5k^#p=-sWF|{=>{(b-epg^=bU>Lknmu{NXZ?%XLIo!SLe(h!Oxa;J zETAWIZukqldy=H= zFnit)7bBvL66*|wbn4q=37S)~FY43xy1|ek(R?LruUwn4wT@EL+EE|I;eIkr6O-XJ< zi8+O>!u)`vWZJ&Tv-DSXGvf<66m0WYO;1Kx%|L^+gr*j?C;mNNFVWE&G4*0#^38Kq zzcJO6cZ|3`vejA=4ZWpkId8;i9L^UJzMF)bjsJ)2DXbf&HLAeyj|BTFgN@% zr*(X5T$o)W-r0FIa+r*lX(ESdczIqdxCTAsu!KuiM>2zZli}6wNWL5}iT z1##zfSoL zc(9Oe6AQ;s#HZBJLb^Q8#tBLA@bG{m>xl>=JfZFslF)GaJ+f7tSvH){+~{vg6eEUoxnoxsIT2A92PU(M#{mz zn5Pd$TP|#bSKr+gZrJzXMYxezCKah8qgNOZ;09+qwi8@DJ+t$e`=j=MJtVN*@p_}* zf4w{I-*0+De0Cz&b{lm2?{i%;5zB&LEbY%j64HrbhDaaDNFTh~mu38F6 zSI?I0t73brv+Av6E&fNOCq)A_m6<|o!-N9_cxGlMk^uZg59!nDS-1viDj$lO9G>`n zx@jETz`OI0bd$fY`_q=q&3`?kr43c~4t zTmJ2=78!1kYHeIPS35)D)e3TfhU)zAWJ#ajZFsPtPCpi&Zb39B7!SQ%h$N^Jv73;? zZN9WMNG4t(Ggo{Hf~Y;oL+)++S|uDCYinx)Jg+$`_9lmsV!_L-%nb097PLQ>1LXNU zeKH|K9hp-)Py(-};G~8;H?${ZG_Rzj1=%|6#uCvcpRjgL;o(ngXQnYs#$7h`-C&f9 zK0%&ch8P{P8nHA#*tv^S$<9K#sJVwT2KvCX9c6hrd0 znPXk*h-PGrs?p6)ZbImCR+HMgCY_YFBD2%q8@3A0@D`~zRPKHsJk^RYhclO>zd1Bz zSF#x-lQ~t;F`Gh(v1K2C?t*qGw`aD00-HT)o>i*At2ST+e5xXh(ADfpJV;0~4hD8i zN66MAeR^sWJ^^7sgTKYv*!m)##H~G_92XZ?zs=s>K6w;{7UZ+ZIXMA6&24S{PpCtc zDzWfFW{5kXtK-?l)|-456@h$=VNu}K{tQRPnLq#rel7G6D3UyBg`<7GlMls$;oSJ_ z{h^5VO7$lqnLn-lfu~|rUOFIPFc`6Le!0kr(3Kp>Y}U(`xZweonM*n&bk-uBa6&q~ zdbS$&FL;7^dh0-5RN_O~6RAUh=~k}_)%O&4VjMXD<-}_p!T`91SGNbUlHtXCij)8+ zifcibn8mC-NWhMyskM*xTC#N_UOW_d(IuP}ZtXRlpOgG&qvrVSGW)#!pMJ}heaV@G zd!`HL{%~?CJ=IV-L&^YPUaQh zbLFv(#`%gCo9w{m;(Q{8gv)kMmsu+Ul5XP*TY_7BC&9FXb41k7EBEORR0+4_al%Rm z?z@j49-0|v-L^9n!DLnyvUN&h$kGu`VQ%hqcrZIkA3#dk;7&ppIKKs61B6Ia}xAvtzB2hSN_x#c26(JiHnc zCP;)=vttkdEIAqJ1UP$W8*)>!xe@z_*h7ySS?D3xz#x2|@HANk6D}Oc40a`SrGyS3 z2`fxOwjg}~4NynUUuk>`ZU@GeMOH1W&ug^d?V~fo;(2*38&t@$;tj3%!~$Mxm{E5t zZqh%_`nUGUL(;KA3_mxstXz~M6}T}fH#}h%4k^|2kOO#QH3LJM?JX^wthBcpcqy+i z7VcV~g!EaJ(#gjwLJ3_XeeaNjc&Cu{$W2hDk&g2A6FdN`s&x=xGs}|$JRSZJ zw&uB75axI*0XIs<=99NY0}KPy=DrSYJ#x6?qS9Jqv>JlhXXD1?P_5c_@n%l-;EsLW zik^Q-2<7NWmaKCWKhRbT?2DJS!mDXvqj~V^>R5F^F3N@01Q#fURVgcm&HHzxPj7M{xZaf}fw;_5 zpL+pr!;AYKBpR|IO4-;xP+Cb2pp)O5y5Ui1X@)mv1&Pv=8OGa2r<#zhlg;c##jbY2 zLUOPME4tJpY;v$U+D7dNotpR(Ggbz!E>FvoU)mbRlLe|^9$6{&W#a4_DcNK{JBO#? zYGJqFVlG;78*bpwIYQ`CQ;eAry864>t%Nc1d?^C94Jk<}!Q|5GyML1VU*Ir(6BA zb8^-q2SCb(S2v`Y#ll4_JU)HA(H_GaNj$JNIT#R~^s*lJ8cnniMRBEpw>3647H~6V zfFOF5vPK;=vc+IZUE7*H8j-C6rvN#RcK7`liuu5ks+TSJJPuy}C~Yyxu|1w43k)SY zwMbvEP1XNi-0dbh9LMo#dko`6;g(K&ynI^#v!p4ILT}`>pNWwyObH8Ca z#&~7ma7`o1P4=$Ouhl;;z>yck6QfE4D1c}^ka<|~;)Sg|*OZgoEIH4Zhc#871jIMt zqfRAJM(h0fPCO2U^7fhvsK!Tc7xvf%&{)h&o6>8D(^6i539l#)-AMIB_dY5qxeZS^ zuog*BMq_o6!@USaajdEEy%mxGp?3A9ZPz%AqXlFQ45K;4y#m$dU8Ra7n5_(kBZq;B z)sbGMJUsViVnk>*ueDGI>7$T=l3`^~$9Z`5 zMJ3Z)aD$-1W3FX=el5-D($Ce>xlB~Wp(jJ@??-LUD2o2%S)X8EEaT&eaPbQ@`Css0 zZg!y|J71ucNR3j?7>m&xNP8Wl2IYpX>t+PFlF5iA!9^jLAqhi4Nl-nMJk%QOUUto5 zeQ8u{#@LP9myZ zM_vZy2AVVu@+Oji9`95}64b^!W+MqK*+5|OC?H!QDSLJ6|{!#pSLI+tIl+?G^Y99L0QcOVH zIfJwU7*$B$eRwsfi@ES>9lW>(UY)Snl>qOpPu}mdig~XR!`|MqEyfK^qIB*ewYwsu zpsXyU2tj%LLJ5%sZ!W~3K^oHVY&&>{?S9l#agA|?eO5#JVWVAdK|w(eLi_XSCw_ON z{%a1aNP#kZ20AbN#fJiMd-Ud|a}B!zowqVw9%U$ut%gq)e1TLc@G4N#3nrOianEgc z!UVq>3M%7;xKm!+l)lDzdagmvS2er=7q|7Af)a7?+3Z+2Q9hC$3!kmDvOmig2pE-_ zGV7ymCi!Th&gbhcu<>5J*tMb3UMrLQq;qdh`Dlx|7|Iu2JCmvajK#vqRCx%Y>wN}@ zHMERl11gi0px=?h{du&tNKk_q%JlT~2o6`&G9=e78sOzvV6W9%F%=m{q+}cqR3dzA zuMv_U9Vo@L`krC?w`_}BPVIDQ2}(+S;=S?nWla9g{rguiU-PR8y8=Ach1xErQ5v^vlh2rd4PKRtNYuIleaJMEGw4Tx6I%0 z{x+;-&ee?>d(95axYY8jazR$y_QOYyV4by4?z=n|?w{9qA^C?5&T+*_xL|F{(s(09 zxbL|KRhz;=WssF6L>Id96v4UXVHPPBCk_*7y{K&KOj3P){aWdA*&axA$S3uh@Pkn} z83T_)LEJ3!D4BtBbZ|K0@8h#+8Di9^{&*H(WNT&fX=j$oDk)~NL9AKu*osMebX*+> zDZa+V;ar}?lvTbWuIMqEFIW-ITWzr^MNL)ppal1Eml}!F&JPPj@k3csk_0%$CcD6{ z{d>&2IrPj}@qFL)&hmHf-jPyf2+JZ8ga;@Y z8$TYCDX~!WAr4AnxZ@v;Uz}!n=*0@d?J?!~B(nH6IfFAxCuj!>EsjIm(S3@uIOF5k zO{^2Sw`z?VRe8>B8zoV)Xu(oh_#rh}Wtds;Hm@Z9hu+UwASl&vJml3Y`^cnL)g+IoJLYfP5{)cg~3EoaICj;`3r2#ElfYY-x>Q2VL!Kv z5;ga0cm8Hu|BVNg#aD$)JwDnWP)h0%i0|R@nvdi=Xp;>F&d*-?ki(aiQ*G!7sjDK3 zq>xb+BA2#`irpyoa_Z4+k|6>?98r-1ZwX70w)I;3SE&Xb5sm|Lk*ipGtB!P3Yf0>o zGb>FtC=Mu1?stB#L_1!Kc+IWDkFgepb62MHZi~q^oxG~23)_#&J2w`3m-mJT$8x)O zR|*bAm%aL^@1wd}|FIKUof{QAza}^41rFCSQ|%$A3O`@v}cq&g2>Yy;F_xnt>ky;n<>?KT~aN zI|?C5LmZ_e6upWvQr#&l`ib12{^zENDF3i9(=%x8cqp(AG^)^upIkjvx%?pss>Vq= z{&(btgpOuUn|44b-3K_YO_`;$3Rn&@THR1J{%Xpb|x3}`E7T8ph zsI1on)egijlq_B`j5bZfIq@AXr=_KlK24WUl{KUqD>ljn4Z4x0cnEOP!hj_f2cx*ckhhd}FLicRQD8aO5mheZVaS|byu4SPrlYM* zI+|@Ai;ENksd_788Jh?$7$uBijYXkwbQ>HBRnt9uTM$X}9}E6`c^#)lBB)~#lcS2j z59yFX08b-X)b~ZU)xKiPwIMrYbGo`*-pV6gjnhaNStBM}+t)BWCthSw+vh-L9d|mD z*L*;ND{)vjY1qkMgJW#_XHySN**iP4Zd58`LX!H~ycQ2V)Q0#Pq%@#3+sz?ViHI)= zt}JTy&`ZGN+8(InfoNJ8^6@T-ITQ^-LZpzh$Q%$o3i87WMXjxQ;x z6^wW4$UE;TUeO}k>Nga&9j5p7o$*hYvphHjL{Ggw?wvl$AJos*LXZReHz{8AMLN)g zD|!g?efPg;>J2HE-;hliCG1>PO!4Zeho z7fqv5A=J=X0-j&0_D@F&8>VDiMRT+ESv$*-?pfF#P$y3d+oos|tz&TW5#R8hHkW^$ zV!f={N5P8hS|K!kkPwX(e(Z3C7@=@QL4`u5V*5>tm$@Xv&EPaZSq@rYXvcK zi?HF?Oo=AGn|#C{$Xho4V1eXdkCBN)5rLFWMrEH%xvL4+4%H4;6H&N!2Cm&Dh3L)fkhz`*xb6enI+ zk|Yqt2_&YZvCvwWhmhu>RMpfhBvDEAz*xfS;yb#ErYe2BK;K6AIKDiKly1U16HC(_ z^&ShJ@4GuZ7(D0v>z|Gx=Yb!;KJd+UM`Z#}BrSIahdWLcwtHnlseTm7gg(VdPw<@! zi4P}2KrZEzISpoNzEVjbv3)-iHG!4yLziqM`47}xk~au%nv^Ta0F zx~efif13FpC{^QWvX37hlJfHMNY9!8wQtu|n~2{7`j4)ObU6H4>Gco*vt~+RX2kNIZ&B zjFOy&3GqjSPk>RZ!k9zFpFe!!_qi(a{{b_tLm@_;5!Q4QQQPJ-5F__2 zY^^TRWv#Qxq~Bxt5~XbW(V!JBT51 zh~u&N!;U?^mv#9esooFPk$^%Fp2dk(IbH z2P6QG%k+nlAaY7-b#Tv=u{%)Y5PBv_;)@@k0bfYZl^>t)UfS;Do2<(}Tkg=f6ybnL zp>1J&BLiCeLHW{9^7+(X|Ne?46Fp;$hxejvavXyDpBpAR=CJ}w%TGzMresc$d z3aYeo@|y1%k0esXiKue9kW7a1ZKhkU?UAiH<<1l+2gGUGK!dw=gBH@5M=AjcDN}MA z)KJyfv=xHSLxWj32*lXKP~M6+NV~q@(9n>?i4W_GCxz6xJ6X|ZBgURV73(KX=pD|D zY}DIa)2NQI`;A$lOTD)2yx+JI=IXiJY8 zFFaUt%8bD}3hT+srXHHI_jKNtFYJX~=T(^LW{U6EA5h2uAw|jZsiO>%gpQWcN#+(7 znTDpOIc^TV`iXU0d+h*ImEEU~#FHci7-<11jKP;UqBbmvs0K{enhvE?Ie_ZE&-@H# zJQRs-0E`%T?rPU9oSN>(CAF*Og$LjJ#8E=|<%J}u2A6aty**xkrYjYiZV5q0Ody2~ z&c;6?0e>dn>UZuJouzz$asj`ow@J~&9~+-C9@ta$#82&NQ;+_=SLA(4DhuO7a`l+U zGRE5pKyJP#zs)FnKvI+Ds%3tv=H+8!MI=BPo{(Z=W2=ob=_C?Dmagza;e}*5okSUn zpawmz2N~!6r&>j82`ZbxMH+oR(bHF|QAuF9$;JTQsrSWuOwV-PV-8-{KjII>X|%6A zSN;n5$yZlfOjb2~Q5XgV;5K%xAkw?0+vlxmqYCkJr^)2!j@8!vV>46Rw!b6P3HzxQ zHlhlQG9@3)KutO{5XP$u3m+a2!apN{SB%2jFQrZiTHT z0lF!5YX#Agb;=PHF3AFcfCLrO%f6b^+Fk)twlUt%R!wF$|4i4NN=E1&r!z}=YVyJG z%o5P=L6R^O5;Q1iz{dc*(-s_Dtr!_eN0gI3O*vUYBA4_~_SCyAWh)7+!@n&!_WPQB zHtB!*tzZ8|rXX%maPcxSNiIPvxTtcVXl1xL*z^jj!q{Pe0Qd_zbW(IojJhmE@;=>K z2t>)UiMkuAc_BsyYEk`!z}n3mwiwF8F5?hR*L1&C`@;THI)GF}swsQ>YPD+5jXk(Y z?B@0+E{WLfpMYL2&LB#QiJ6?OW6|C(!Fo?zGy!91b167i>~TM;=4pq2PAheBDA>`|;qI|%m z`g?mHqN@hB9ioe{G|>@%u(n~Iq6U~CN5S8Tee^{jdRlj?YmzJmDh62D9H2Exx>htn z9`ia*6P28j@~L9XCo#$g{88l{vvi7jLKro%QNd=~?pdL?%%*0QbLxbD3%DIux#Qzy zBw>gn1Dd*A3bRLCA6Rc-`yjIs?EtD=nWg_1@rv= zm9JxYNIn!JoLFCbNQxwdaP3(~9eWD)taR@x*<%pt&@A*hy}D}a=+r^Oh&BIuCM$(! z^%4)ZOllq|NFM-do9h;uh3vfrN|?+NCqtqDI#02&2Xx~=+5%~yOi)jvNK2`qZ!!Jc zM+yL}9H6A4l9Q4g1il-V%)x)@ZtZyBcZ<4QoC8_a2xtx@Cs7Fl6(3Bx&R=ejc|9FU zw%Wd^)q(yr6VVZ&tGt$&5J^qZ<-ZCBLdf!|VTaYe{cqtlr3h3*TU((Bv=1w7w+8eh z$G~euMF>&$0YDT;Oc6z75-=mVE~l9JA>S`cWTyD2md)`ehdQFDe36VoYUKJudO)n) zIb{g@-mDnVU3Mj<>hB&;FTK?YKgh8Gb@xw5grpafk&CVoAZCAqA^+HM1k8PT&sLz= zI`aQN#@+<3$unyo4%*h~RAoAjbwil3D6Ld*L-u4kj?$<=6=PJEST__DWC_TUgpTi& zae<7Xh_a=wNCJtd$ez%#fQS$ULX<7Y7D)^dLK3on=MG?Nzjxk${q@&CAR*8Fock=- zIoEY363JKCODov^$^8om%K?`jUQ)s3>MOZy={!gxZ-}ABK5Ped`dz_A$7QldC*P(g zmNj_u_qFl=*MQ-vfhRK<(48(hD_ek5MLuRiN`s;``e4Qe^Q0YHcPUbPiG{}gaB*X~ z%~Q#dgM$MhFE5Xv(lX6j0!0*0Ari!I=83-+0yuC(41xdkp(h*EID`vP($!J(Y~@|@ z4!gJ2U-{>0UhNk58uTg7Mf&NYP0@SIYO$${6{2Dh391r0<-~dg)Z-WIwvI|a_fObH zh)nA0V(G@6&Et4}Nz3zw&k#;k{j1yE{UkZ(lXorPo$n%km;u*Q95@fdUmm@8r5$`{ zfep=KOZBe2v6Az{00BX{#@qV^s}rD7id;lJN#IgdY_{&2v+mkdYT@CL4HxLDcF{o` zwV=)7ml97%hERo=_xf$f={ve%d>5F6@JM=CJ2$9e7XnwvSGtyJ+0c@{!Qw09O@0L2 z_@}t_SIle8F2fZ=mjYMGss~p(7fK^d&HRLBifzlio(MOMU}8z6UjV*;5(d zY-_KzBd#6pfVW>*rZk(xW<7ffUE2V2Ytr?po3>T_>Pqu}_nUpw&uQPH_^RM(Y&)E! z?_>cYGw~01@LO4c;~4JctBP2?PBcJO#^FNB*RUt7=rD?iv-gUhx#LXz>m=#FuGB zo}E5gOF{;lHH`o;lz;^l+>Q=AW5Z1U1#;y)RHaUZio<2y(c1}htnL_K_?4_Vwx@q! z5v%(b5{#+lH+BHGr=ku=J%lWxY71#%3!!FUx#G4|W9k)qj>J#86kI&BY{2L3ZJ)Z1 z@+D7xYi7&zL@+V3ElXxVm5a~WiT0B&D(86@Gx=Z*e-D{UnLg|$d$Z>Id zgiPZ-+5?utY*H{M01mV?W7ID_em}FCZzk^qIkCZA3y<)^QCRk&4F0EgvfgzM8AeO@ zWVx4Iv|FYvT^9j(3m@i(I8v>EMkQZK@_h2+yi{e{ujZF+>A!WWrR0dF%dfJ&nf3=& z6mITS7<*a0CUqIng>x1hABtSwao^@PuaW_bN3?t@m4kVZS%g{}cY7_lI+JsK=7KQH z;7_eLT}TFJw2yp6zciub8FrimEV1R+LrJEwVR4l z)V?{?19zFtM>mY%)bW>d-nGG$hEK)c{R%UA`*-1slx4?e`T>2zfbv36Ea1Nn@9rRn zX$%$&LP0WvibCO3%C>wtLA8=JzLHR5S!WpgNhDZI&l+bqMQCVf z__>iCwfya;n0&q8DNA^SdSbCV7p^o!B*W#?bDvvsUX` zkY)N2H#vu392Fa@90b-e(Z9S34Cc-0;jWT*pL*xJU$lR^4UqNRFwhq9;R4MDra6Y- zOdUJV*4dc{XuDw(ts2;BeyIB>$kP@( zuoAURq!0_X3AO9`h#+eTbc~aQ>=1&bF^xB`1IRD7)r?9A@9{^UT2gXAe3+z|;J1rO z4Mo{6tOhF)E{OXAEB#dAGEVxGzv+Vd8O_U#3KD|qq*#3t;=lZq;n9^e@>VUR86_;# z9@qS;^ys^_?zg6F-3Hj~Q10DjiTEZJq2}eh$jkQu#Anr$PhAKEM`5WYYUC#)+0rK7 z>n1Q|JAJWM9_P6)zI4B(xjO;`IhKc~XU;?yqy^CvKj$p*9gm{Yf?j-M3Lk9R+WUH~ z&!*_(pE_)cK9*W>f1SyKVxlo(0)O0*zkZ+jO}O86tU3nT7(^9}gSOheEmbTFmD1qN zuYbE_|H~IY!;-b3;_`e1mFlNjP}1yynai~Fzo74`$esuU^a8xKzZ7*ZrZ88_j(gA* z!3KcRzhg;#xwbpe=bWc{ISuZ9D zEu9zW2%@Y%3W5FvXope)2xzW~jnY&!Bx7mQLa&ttE+vD?$uz`W(1cb;A`Dd%+up!G zKW3Y5LiLGcKwNj)ACg@0VqCBI8Es;jJhh%*^-x2(|JGwNa|@t5^DugGLcS|t@C0Kq zUsc9xN`7jIPzWc5=>JG-IEx!|U{#t;dUli7)8`G3?9#~JJv7XaAEqzaVD9TGdiV0* z`R*4UDOr62dQND5!)7XO9&9x&7H_4*hTNN1J}6;izeJ8jeLneYU!z zJlXc=^4Lk~B8sW0ZvMvhRVYmz%AWEY08@;iKF$sCqtdi7lKKc~&s299vB9M4HJhS0 zA7Z8^{G4Z5u6EM=BiH?%>wfo}U!a?BUTokTK|#mTcSHCVv}!HTs>(8gK{BgIOfNgj zGAa3%ZB93qK#vBN1On>~ORBDO1{VxzEE{GMOvk!U4H$tJvCYyl(#R7Pn$!NUB{YN=Np)j< z=ExKI`{({%``I5p-fjQ8XN%LW{dvzHGB5or?UTR!GmYWm3-w~KF*1$Fcu4AI$QIpU2VJ&^Xu0&qL6{m zV#({6j+%E}(l3|&RPqN90ScGFa}hYz{!NSLMogG+rUBe4yi?6Yt9_%%L}eV_l(Q1Q zDgzJXpT0i>jW%B&F?rsp@-zJSE!P^LA^b4M`*vge#-%P=Aj;S`N~K|p!;z8B_`5t_ z?+xI%`nsPA-))GDjC8;|mG}K)Rg2WkmYy@QkPCC-zR4XWt%}#L?eQ>?o-Ltg)5tYA z^+|8C|8aM{F&H1`-H~<`7o)dp_w(Cm2Ar2RY7tbixOj<@Mtq5 zR~-7*lTW?e>^!;YS5M8nv*pok>zSeGiuQT%O&K3eqFR`eLLu;v>+f08Eu2?Lxxb}sa7nIt7AefX`*?2ay_6{u%t zc5(ys9bveqXr{ux5v9To?tFf+n6z8lF_uNsW@XsZ6P_H(PNsL_-qxX07ufL4?OzcQ z-nsC;$+?ig>~e)CyG_H&gehVNRT58iz>h;OgW_9>rZ^kb>Nv|3sJr=Aqz<@N1Y$4{ zB04+6#+`u@>JI59o`J3#_)U@!n>lBOO|c#)r7uevf!F&_y4lz+rIABc+sgK89P=Hq zG7~Hru*N>M9sAcRBarJdc=Q2YE;YZSgy}Q$@oJli`tNj<n$ff4qR?If7IN5+g;MtApq!LV0(>}^<0 zbMw;VjVC1~;Td2wtAf2rBh`DN!+|fosuv9n+2ft6Uf*5Ca~y@KQZV{5N^(f-vn_bW znKNf@RU3^)`^H!g?+Gok5al+0O--Kti=G}|i*0TC*UXZH68myxHA}CJRYQVC)un$MS?qAW*r-+NzbeMsd=e^lc1-iex5d znEcHX{u%bxw8@9oa>dfD$DtP219R~cn@b}P9lE~HRZ6|dP2R4OnM|fMENO&AC4~Gu zuYT%}MrcQaZ`&tG+`~}n)5B;xRKvyCU%2?qo{NL-Pm! z-M3;e4aa_7O$*om*|CXzH}z=3118zwx>f=wAVxfbMMNIp12$`c3dk5l^Qw%&*6q$f z+Vmi=o}Pb)C49VO?gzf!Qu5RAmmktf%umYG8wREa`GKn3tOAqT%w)S2PO?tlb}Kr> z<-q#eHy$iGHqJJHf3ghA`x=NkmIy|O1-A^9B<<_b(M_tNIAI9~u@EB1ixc-yhxjHL zA6R|R=Le^y-R+4$=#_XtjY3_92vuwV6y<9?otot3L$G|90pLllylgHXEj_DL?{(sTF*Sq5%A#Uo_S1D{o>`@TGQ&C4$pi zGiZEnf!h48an?4kYkylczWXD~cfX8d6^X@5=YC*W4f{q!N`b!Ds-A_!ARs(~*ANA9 z4bTFba(8O9D#Tzific*{lXp!~2=CU^^n6bgX^`^@MfPR4PMyj6H0I%H2mX$~}( zqN0ssyH{gj;y*iC2Mb=99Ey0L@ia|*hJ990G@fxdKECWMU!_vr%3aML@9FM7(ZbHB zc(V-Wl$E40>R8<(2fPXO!k?bzCT6i3fPV?TO&1c=>;fbKvDuW`lJS@?4c6|~E28L0 z8&yREKu5cWH2}CqVGLfiro6g3w41iuJyQSy_nuSxWad)l>VnR|29w9k0_7eqB^9bi zjBPk$0AuiOlhbyz*gah1hEnSSwY9hYg#dVOnhL?mR*n8ddgZThEHmvlJB!I31#R|? zV7)PvlD+nLD3}+)|6!tG6<#O0>tQ#)OA$wd@qi4VTwS-zvp7;7Ifw+D)7J`x1MmsQ zDi+oL_(Q_z?P#}sTd(h25bx4U=9QE@%eAwyu}9g*f;kKAGGVqmd|Wp0IwHO*aDl%H zN_H&}Yap2TY}9I0ju2U!RjbQ(d$W6wL`HHyr^UqFE2N2?7fOs?r)y{1{@Y%QV&<(} z82NV@0@mCnB`dyeA<-V6Z#GCA@Nsc*+2kKOiK6K6es`8?Aj+?rFVh`Z zd#wM6h5mN$UYl20RaMflpgq+DUZ$YFertw#CqLx2L^tV4(ZwnQ;fkikqO%wH$oonAwpcW&_I@^ zW$O_iCBq&FlqBIGNdwzGzywP+KGLQb9bJV-o*=p+#?8&`E2Q?ZiU*!0Th=x*@$_`D z8<6Nvv_S6K6Xd+mXJI@gNTE=;4&{0$sh>)4f#CRkO%yHpH8+dpzdYQw=X;2@oAq3# z8Fi%@_KQMJf5RC3(mD~;MqoVY)m3FFmsAQgVht_wey@*^Utcktl;)EeFhs2Mm|4S= zjN58{JFnupx=l!}=oBlx;XgbBg-LL{4@WcvAvUMUsp<=SG-zl@W8XEthzGxU`DB1t z*f%1rAJn$NGYqZH#$8j~VDPdrHuD9PRQ<7y+ZffV;z(bHm*J%~?D^NPH)^(fyFwWd zEpy?f1LM_w54NidXg@+B;Y>V(^cuH;v(TTC`#lf=FT~evxAR}vkSPWas7ww0B#rJITAnH1IigG0Hv*Kn zt_h!!E`lVJBMp=n4M5d#RmtPNjpDiXrSt+Z6U;xyzT(Y)KFY`yrxHhSx@YuNc+|r6 zoQ}r5y&=kEc~v20lYdfY8<6OMsqpJe)b;ppPQatj*R8NrR!-_gyf>)8Fw~KDOsVcz zN-#3n1yS2U83PG|R71_XozV@*3{LZ(O~6Z?(ZHw(8R`fR4(1*zOoSH5aQTA=9fM7c z9sFLRO}h%I5J2wr6$6NI8ayM}Fkt78jMcuerNxhNm_n&Z;Pnr$bOygNxWzf~+0~)t z1SZzjk*b%ef|iGycGs}c6MSBe&xmqFSj|pI43K-Vf{ig0V>(L$OA^3`Ts{aU%=qs3 zXduG4;>V%?qT4F^C+}@Q@3UtaolE(*TlB_Vr8t(DrsSLTd|=bT)D@mVPagaf4799} zLNh9H6Pt0sH9-tA! ziKSeQM*F>`tb*0bAc5mddV0nd>JLFuFR-Bd=HP&zjzI3svS2w^y@#Ts;eNT!_iKatOk>+L2Ncb2?0Vtd$H zzO)&1s}>(;Q8CJ&xZ$wUoiDzvR<)b1dUWveQR5TOK;szrh;jv6Ob-xr#m~HKYJpFQ z*8V>-vHD8*&2sj#w5@msQgpG?wayJDROsOaZZN7f5 zZDtnW)Q@ljAhBG<1O}lT&wxxx;EyB{R^ef~?`i|fp9Yo62V?KCqlzY%%}-6OmmnvE zGP9`N8pqMN`1qV;M@k(jm|MO&(~PD>HfhZ=z6mr;q-<_PE79ALneCeCkE5;jlkHV; zw7XU8WW9EeKrWXrUuq_pN<|?#Xulle;NE;|eL@#{=+MAZHBp-xDZF09<+^Hjn>!{m zrCBW6P=HnqNfzzdmPH^Eq%NVm$o|exw^`_bTE98t`Y4u!ZA%W(u=9m<;pl39fAQm) zMMZvEXvy-&5zdEafR@un{5a06#KAVmDm1#3iY2facPb4-S)}|y124Ce6rfL%zKgfk z20#3I(K|cf-EV5gmVA-bbTb0o87-0-8Wcf)5mH|ilr4%5^XJcJSgN<9mWmz{SD*_6 zOz(i_O>E~w=7I2vLeaf#q}AfOgU@UnX?~OEwFjPFke&AOv!vLztLIB~P^ILvCkrJ_ z(_X9e>_doVt3v2H)n79MMs$fpn>{t#g1I`%0}*|jC@WCKiS_jKOzo&j7ml(N;EvNC z+~e4};OI*Qy95NjNVju{sJpJ?+i7+U>>Gdym0?$KF5_~sN#3k_T+or{!PN_U!$MWz z!Q63aqRL_B!gK##YAvx;kQTl;0xj(uEw%iVb`z?yqp3Jwe&m}>Zt@Yv;D`RF|CYho zUHNnVJoQ%bemufV(l_(wzkE!8b^vBw$Z$t%6I9t7Q5`>hFQ>K>RAo~E64~YmBIXXF znKQ;+tEhRCp7eR+}c<*DaC4FOuhIGmIO z5eV$eWo$p>#LDh9rAWvHijw0TVl~lI1|A<^rLL~<&OjAN=cKAPi%r?_DBpU_mFS|S z$z*crhz90$sh6Z;mKaLntHx99O4pn9;qmdBr{o)UWe_w&GxgSx3{4dxzMgEHBQuew zE96FI25R>tPrTIyti|uve*Gl7V?e8veU($FTxCA-Qm^Yxq7z|fuoboZDb$} z;dWq#5qoMNsAf-clUJLuqq_1U-R=ZHdF0OamB|q-?@Z2cU~^k9Qa@@h==9F>%ZX+M zBT8zHI;~+~n2MY4508OT*2#AbIE>=vL-*H?^WAe{tf@YG8>eubyU z`~B8}t=tj%NT5A<$LOR&8h_f_IarWTu+(OSqT8@#k;z83f(8_52aKJA3dvSe-rzC< zCR8omsS}kxI~IG)s57`-34`U6RrmKJY&iPmKNqdSgMZ23(9dimMeUbBMdqvX*yBae zJvN(7v%Xn-?6FKhKQ~}XY^GbhyT_i&>(XbSQsS%c1q1@zO}`_r|LM3zVTM96Zf(M` zdQ{z(aG6Ph4IYl8hkLuRmKzhex>6vf6OtgarW~`Gh$4vC-V`I;yZLEs9%*Ka_Ss7{ zAF1@zFgyR7NzhJW+E*CHn=hTvS&lH~LfE_yv>IunBW%hWiw$Xpnhy`!(vUNa9!x%(ee8wPSyVLR3;-VsZ zJbosNRMvC91rJCuK?O1&&hnVZTDq~r*;`NJr%`poj2`%2E zQS@86*x-GEf!i6c<#HErsA3X#at*<}x{Ob=NUT7~)D{GJ1TQBiC-qYNZY?)?$Pu(o zl6sXE&9p0ApTfkB5`hL%4(AMe5EY41b|X9?{rjZ?YOf~hDR&Nq>Z7755Yqv$X3FbB zIZ;KXDk+Q2ut)TB>oxI!bR=*(xYwl!M7R6{V?JLT6+~>PWFvk76pF~W!;~rUpxIUnwy;SD?t05U$D9UV{}E36nf%)oD&Fh zKuFq69!hxt54VM63S|saj`sx8gg^yZGGbgqYXcBjBmfYR&oASSV3ZXwGA(P`zYPdcOxjB_XElOsQbP~3SY!;Ya3$j%M zx2d;f1ny(WZ4)*uIen>^m+zz6PV!58I0ajc#xT3ux#M#E!kIPjyVkiVKyTb8XBQgR z`^Y18h)p|%L9F6;3Q7S-hBYmKkv->aj3AIx!!uMW@|l^JPd%P;*!2I%<=$q|p7#>v zfIeET7@=zJ6DDx1(1t;Ll3^2IjQL$+&}lTF0V>8|g@6))@(PzBP=whenL>J=NMg>! z{zEuP2>KrR9B3FcYXP0;FYafL600Yoh#Ey)S(S^1zP;{bX@PX#&;-b z0?4AN00_@v8C}Vyc`)XQXSi0soW~?2lyYy6zv~wI(woiozrxUF-qrH>v!^KEa9fn~ zBWQM5>M>{*Y;J?tgkXqK&vqnXw+heU>7=Xp+lbneNW{~LH`xynEkq55J3M@paX2*8 z0ii(*B#>tT+ihftnSu`~C8izK!t#9gm{s_@0D=c`Cr6l&iZ@5WE_LTh9Q{^vq}Bb( zODyd(5%)0=W_&67ggvUu2(pLFcbF>VjuNY+6$84`zc=%FG`&tGcS};A*bt)E>r-#Z zl!bBNYgSsoP$kxhWyj`=AX?t{rciWoJstd(LRDdX4AhdJmnj5Pukda*GyUABGP{|D zaM;JE%MfuK1XWJgIcDoAl^)oEC4#RQhM)Wme+M+Dp^)NQYF%hUUOw|0%VUb?C+1~u zIUDc%XzH6^c!ZV%mJMvVeo?;1{146yPa-lUNUre>L%x;Bt)gBI&l-3aplb%>wG4Z3 zk$6T=4~Wb_5<8AVX~f)A9^2S*VjBEKDcTDB-Fx>kEKRPpDVvc*PKG_-*v0P&;u^myjl89nPhm4&L!bK}AD3&q)slDDue(Hh>FreA zyFapg_j8l9Ht*SwKYoeuosA+37@%$E6?Jk|fS|ZFYAGJBgkQCVY__C@VP?rV?)M0Af zPNdx?-*alU2-cnHk_yj;3 zC;v%0Hm=Z@rmKR4@u8swjb(NcULN^vFoT4STWH)^3gz4!q0s`;F;JR|XZX`}ZGcn+ ztS;q_P&HK~#vtAFiVmoC(5Y`*ex6t?U$EEd){?nf8*9y*KZLc8U-*CAj^d%{t4P$N z&*|_?1J$k3Vucn(=DPlFJMYwo>Z)hRFRsEv3T66Y3ZSrrQ_w56^r9Gpj~%%?7|Cc> zjBz+T+!4PX)r=|+!VtQB5L<`7PvkgfEzlkGUc52tRQqYag0@wD5;O||jia+7EOQ%C zng}3s+;%AHHY-tG%^@ZejZ!M)CjrV5f6~zIZeo(GA~4W=FBEjZP#eI&7PFTdy*P(% za!s6MS+$h%S9?(*pCT7w25o+1- zJdHpV)!VSso8-e!pMf6wT5hP@0H^D;YgcG=UEtsC8vzpPy}_pZ&1m|m0m=b9gUzX2 z4Q-}RXI=zD4tKx!L}eDIkA-ZGc`e|Q^=4fAbM-GcnA1P8DdOx&p0B+BTPV{4C% zTJm9}GVEm>@dsIM=p%=Z3M5Kej^Di7myStPgVj1=i$5d7_%}&Ao~`U2AFAI|ya)8Vq1*AM+nq zm5vdWQb>tU4sSutqXZX$!Q;!-qep=9nslGWk*Aks z@*?B;elY4&(BN&>%gDXl0fHdRzm&@l39|_|x`pNU5ZX_x2YP8vYFvYApiL6ckjQBj zk#U-yb)p(-h{&y{U9gVKPj__@>jlqR8;CT%IpNK%{@cT5h#Jb6FZub?_U#}(4{C|_ zjUe`_vv)OA#hK$@qaqY+cKjm!MC$QxqkoiIk`{4N^R_u#jV8l-f!9V|gfqJ-B} z9TFhjD{7h!(UzKi3Rj)R#E=-0udbE<0ZbrhAU?w^?=~dQzi|YBj)0b(s$ouHJ-`2Y z|IL0*|6Mx&oETM_s3|*M>?RwC+LW_=yAirqIVknSqo%p5>h?H$IW_Gmiz*edJ-~I{ zGP9VF7)@(VTQb?s&d#)Y{~=;jKooI6?aoxXr*f|Q)SXOI6rVH$colz4uQ%vTteA*k zGqJi|_mmdWA=zK&Uu;)8zo(aJCeX9z5X7udev3LPSU{v^a?BOiNX$sN1a72yNF^_t zaN){8uAxP*s^Jzeaq~(3IziL^NWJN`=!HpH(JD$vBif}>O;v|3(9(R;;_c?Rz`M{Q zUr7i!ntsgLXQ<6=Ltu&)5{=bda)ngP)W{OVAyfOMZ;;J*2%&0+@C?*Q#z#!X*C8}l zqFH7fvolA*ypEloAKoE$mMP8~#ryR9_;+{RsjfHtB{P}YFZSTfm$mDb&fww^{?n~q zh!EK@jRm62dcCF9GF)L9c7jgk935C0WAGw9`7mP;)D%xK4tv?UYFts;gp%L!K)xyv zfF$o%UxMqRS;osOE(`1;z<)G*wcqObx{NYT&32Aqxm10?bV9=bGK) zPcBSdh+9}d4vDIPX^UK9atL&@e5ng*6dTQuO;)y@i#4RZ9Bo5$0Q5A7#^h!f5tR6q zudW@<4X-Jbtj-6pYW{NWHqt=!iz9IDy(&(;i`KSmzrQ1g&9k^CGRw3kRNrTF5=Ye`u8w|E<$d~j>T<8Jkandp`L;|r>Ve~s?@I})5!pUg zFf?64adgKXsA!nh@noU(LWN2hgqttq8+8Fut=uXyUlA%3E#GVoc`D#8ZNS><1Vh*B zD@GwfhbGKyX-tW9CzECrq#6ylFbX$Xr=E7e@mQ6n5fl+=P>?}O404u#d*{ziyN5Z<6$oMv2M>!tql`5y^x{f#} z!Eqmee&C6MOv}W>I2t0aungTHX=;leFq*{WFfFJ_X({=bsoQ0s3?cYXA}aM$aX(*E9jc5W^?-b<+k~+|N>UZ+ zG;-GIX|#6KORD%@rTIyh^fC}gruL0=<8%5-ceAFsb5XnEQytDMF~d|fs71LnYWXo5 z{al@q^v+JjbU!e#{oZ}*o$r1DM=XCfeZJ1H4w?QSnk4|3IVoA0zL<5@C zfIwI}6EQUCfK~9y%ChXySWYX9NyIqsKJ)FDE3QAgeKz%673%FbR007SwL?KJHwLGP z!84e;LT1=FH&&$!8ERJrvWJO7B$Z4-^wuXlOp}SuN+mf`4Vr_@)vNB*zr3A{=H)>= zcjzg3q8y#y(#U_Tp!MuF-YF@GI;Jl_!l@pI<~I^F8;)V?s&v|aeB^Yje&~uI`6y!$ z3MujOFzWYHvmj7eY#Jjq?5D5pUER`z#(iOs9$e&b1e&hgJEu##cT75g?irJ{mqyRiPMV?h(Z#a?RsuC*jAVihYgCtJNE@Fox*MFwkE2 zBvc2JDT?yC?qlm2E-&TDJd=6iQgwC0rOVQUScYshuT`aJZ*71SeHyKnKwh%MN<`f1) z_f{{)G-Wx@`Jt?AQ3{9SggM%r{eCUXtbABoouJ$w;XFc5OS4+*=JvbVM~}X0GIYp} zE7Qe4ZN`F5mDMYS!uirNTiXx%a${Po`aqo8>5{)!u}qTKpLFnGKXDf)ZbUON@h7Fs zR8=7z9d)|ZMwNFuG|}nwMNt6_!e&?T+*&vwqoPB#!1e6)7xzMEziAKfUnt)M7|!vN zWr@{Bn1`NK3&)e}yHHR_|9-UIE@t)OuAbMgDv;LBtktmQnefkHCSz|VlFi6WT zqTL`GIHH!k;5$(N$X8KajgD<2)}SimipdI&+qcs+kyAv}RH>Gm1H|~UmiPxb16c@z zf3FWz7*D*gYhyAL?E)Y@&%?xtDVl>7&4kCy#N~?@f7VF`22&l?ZL9=pI;%S;Uff-g zcR%oOvWN8cbG18m*`hJ@qCK&{M-|K_(H6pZ0pAB_(>-leQ>HW_OxgQOC9Olr569*7|b6yHk z$C<=Ni~oblwOMx#{P>U1(w=!Ww7>nYrV%b_%l%(}@a~hwRmV0K&9+z_b#rXs`{<{e z9st3ue}ji-4y~(t9oZ^6I{J?(2*bvF5mLQ4H8u5o?0Q^R)A&5+e@AruvEbrlwrga_$0T__cC1Y#nuBagIVUMT zzJy97554Pf6^y8u(1z1_!X{cuA)E=UH@3{~|&OK}d zs0(vxP2ip{nDWC~P{TO>d3;_AFcSlJh@!xzB(qL|!uJxNwgaH6G^Y==HP_TL?{D|U zs?_RPr$9l{>P<%(k9>U?XIvSjJ!5D1VaV_k{&9ve=GHx0`i*@p(+(;(gM^hogVasoI=I9PrK~gE1hHnY3p>HM~OgeU$SUQ`b02w38g6bw%K^ z%C{%AH>6$3=eV_gr{Z6*ejz@bC}a^ZXsp^B0|Kid&p7+2!3|`U;$k9>8OvZ!D0evI zk6)RP8V5kW0aLrK!De>tmvffCR|{Ga-(ce4idM|;fh=dnLUPh|x=>EINMsjdRDwU= zfN|VYgy{%sc%pjL?Gp{hc+GkP6~ud|hvE1b+`ngDHuZX4bNhkcz53ydmE&gg#0~Ku zc+Gp?zV(0o;)bNAUP|Bku+#E(BU(<^YV_6p5|>+?FXCcj?;EHG56((K67(1Y3_&?! z^75MblwP6meYoMj!$1atv59%I4)Yq00K;4W|MIl0EntU3-d6HIHz;Mv?b=JGa09Bu zFc;|bG6R;0+?VF8_^03+YL%QEAJ28+b*b1MxU-5lEjvq35R$5>nbEgGI!X=qGIf3l zqRa?yr^7XIO`%k*EU{SB6#F2~% zP8P~SlUvmOSnXj9nhxN)U{$ZHr3CJ*zI#mLiI3+j@!w@Wn+npHm`|L7p5HYsu-?Z_ zr^k&w;7KRoSh)Iwdf3vOBOZ5y>d$`VZteN{PQSj=F5*==fp#=^n|pN7UD&J#D{akf zv|YR%NdI$7Vq0*!c)}PVy^76e%L;528-nRMjaSBX8e=<}FiPrh3vF`)?tolILMs%;! zTGX(B%=`Jid(+xA$J<3*s5h;i!vP(bNeIL9YtdBsLG2^ef0P28T?*LVA5tjl7X^Dc zJ(RzyV)J0u*76pK52TXhXzYCxHCV8wLvXb?JYo7L`$?*#A*traEfnhA9(_qP?ihu- zQK}($cWHi1i{Gelmggqs?GQ=3yq2X%cC19NO2m$T?mPYR|7vXTXQ^!)Q|brXX_imW zs`M=*{25+#{E+PR3DZ0!&`mMGHO9P>N@w*I7Ua+B`=14Vm>;-Y>lAdWzDrHDy1eet z2N-m`3kEU+{kb8aqn;O>@X1b}^tG{#`srhg4JZv`E?l^Pd3tQ)IYCj@=?igD`<$7v z_4W1VhSmeP(X)P37TYQqcVD;eS+oa}jLo2ruo@CQ<vl@?x)?P9$k29tm&WUt-yi>zZ z3~JIfhAuDo=#wuneMqvjX+o;XXD3z{HE1f$h{wA{yDR-Zv30x0J`4Wa6xBMg_+-oa zh{OEy!;OE$q~lDdF`um2zk8xA(eNntaP<>|Zn7ZQLp8QbcN``YqHHctWQZ?pzIU{F z)`7Zn3+qhxuvqtj3mY)c!`Q_bx7T+b?oW4>*WVY!$Dn!@lME_ZvpRwto{)b^Mh6aIx=%fvhdEJF_eq_)D>(j%dYn?F|DS5wms19@ z%Tt!y~#*}(mKOap#yhb);Twh@gW>-5% zeOf)ov5PB6?NDTeWicT!Idb@DG}fgDv+7%?W&6-eZ%D5AvH$viS>DtI-4O3i6Vm?m zwdFUxX>Y@0V`Jm6_4lL0{@(}feInWEv;6&9&@ixi_39PohkN(5Wr8^eYjWB06*ZXawJ=-a@5T4Vr^gOrE;*BWqj7oE zU^~%}o0IZha(xDYy~pYeKql5HC~v$2s-IR*MWV8fsPzM1OY4@oTU^Z4ZKek?pO727 zT`$W1M~f9Li0M&W*w6x`!@$V*4>U4X)^1P)J~AMg7>oh2m&Pc?JV7zf|FNDjZPDI= zD&@dVX8e-`fzEKqn0v_Rb*M`6XPJa2m3IgT?SeL~ppqBW{Z>PZ}XU7+Q z_5U`?iJamuXCK@9@sv^ezctF1mSOQ)=H<(nAK2`VQs6WL>lDxnpPjS~N_^{>1Vv2kf#^hl_{?dsE8L2U$jqfaZB?~miij6@fwl{j$4`S}ME zDJzZSI(%@8Rwa2JsdBU!Nc!U)vibd^4$mOFDyYZ+Wqv7 zGnN|+r0)+?ka*uQ0gR7cCa z`YGmdW>n#5>WI%6oRK)|4V>y_^`?Ijzue8~t6UEE0+UR;O?S>e-Txs5od$J{4Un)>HwU6S2wwb&QwL+oVGUPnV8+y7iHh6jIJu<7IbDR2A#NtL5J0R8`>+# zkX?ZSWDN$g3)YJbvIgQ139r{=SIlv$;5G6>kr zhtp}`CJHjr>Tf8$>zN7M?MBm+YQ-v0pVR~mf%oL!qH_1}xJuOp%~8LW%a8qI?ye&* zeX!1@=h?F$-jBaqK4F!w_#S=aLDUNTlnpb~(?KfL2);;0%^rXek6M2_&bKmQ0 z@l?Fm_hf_MJ&WB!qCM-WSWGI4fBNMdt2Zt2yag%9vi}aJ#=QT-vtyX2U0s2uKT9J_ z_Dk*T%HWJ)>Ewt^Hv7Pd6LVAG;NGPv`|cfGIClB80zBvv`dN`j`lnI(vFr0pjM!DH zR-Kc#H%aujvzlUCPZCE}LP$T`iI1-sj(9NT(s0Vko*N=gGMUE^&m)w@^vu^5brt5@ zg5PRw4DF%r<2Vs{+<~DAr>w+A0}{Eq98wv3d|kRj%YACAEjn}Q*=V8ozE6Kux)|== zkh@AzNpaoX|LVJC5uddR41=C+XS2Tj_S+RqdHusdC7A8(KG+$~9XX@X?E{2vIMEC< z*V=lKDwtlj#Ckmijz#!HzK_13b@&UT0puMWhq?PpS&JQ71zWdn-EWICK;#|W*3^h4d|pNkEFor?0uiipcHhRXN{Xw$vd2NO;! zF=(_3Vo}_z@{;n2|J+qpu5$nA%YB-N}MNK#TivDjh zjqgzMVQ)Uw|0u;AYBZH`>f5#LxqIZ7Jp@;@%bS(=@?x#h3TgSuX|tBz|NYx7`QN|j z+`R*nvFD8C7nbdVgXfOzork>grdudAt*orXJcnbQF{ce<4OW+PPlW6R8uSMiLxq%m z)+so7dDhy<-s%-KJRTNfSC8_|CJfRFs55*~ zCV$X1>karejtdRY`7OM}#Kd!ktfRhf#U3~h?s+KE_{ackmIP?{E({Aa#2)ZPj zKeDUg%g1@Tme9w1@M$Z|XR9t={1|~gg!{4!OEz>=)}SC<84_{o)SaP3a?Ro77Poxu zlB^7p|8`&h>Y|{EKc)c5N{IO)@Ilcl=Y5>?>-QTF4mx{nlKS}Ehti-?>&e2Wcg>u~?b)e63@}DpT`H5lc(%bc=01_ND!p1vT&1&Q7V_Mm5&527I z5L)|*wj~;0)*I5lcAyu|_YJ-=2FW8h!-9D8GbjFq@k#-n+uj!0`cc)*` z>>cuU;wChQW84P+y&2=ymogjk9FBcmmafzOvZ;L@5i|XvX z=xiH+ODlL#ot>XNxhV7sZr!?_Pxx>+D`qb**&UO%H6hA~>YoFrPW=IJNzRHq7<?QyjU~ zcmLIOe>1j?3br$=&j+X5SwijDo-5g)EN~o+1i8@_whNeK?H_S45&jVdG|k;GA`*>K zlAOogA*Pv)b>-TBEUG@Rzw0lmU84AY1JvZyCfyFx`n>U$JkLcbAZkf29=LPgy~&Me z=h{9%R4YbEskVuqVG44iw>23f)!Xr9Cdy7l!M(f*&4kE?xsufAQ?$2aRtg+z&wnJi z5}N-an#P=9(4Y951UR#yU$dKHlQnTSd3f5+zzjMkqm53Qj|X*!IK&RkqQS%iqs+(T z#v8z#)R;d&MdpG#!-?PIwO9Zu;5hxh7EOB14(IrEG^J|v)46NngXrG7SjA#hOc`)9 zHJC|g!M%qz{K*5&F}A9NrX7-d)cI^G^;(Z|AK62;KV-@TTO)bZsVg-nI}}z=5}K2T zYn(T4-dtzMl^90j`5)E7Q7Ow0G#m4)OZ#T5{t3b>!@O*B|O}mU5u;xgmVL@A?*-rM5tj0>wLT6M*JhsrzR9e$ zYuA3A4C4~-*A5QaV`$Ksv#LeZ!h;pak9v4E@y}w{t+VjU9_Px_&7jKg-T^?Bw^W`T`03;WB|2^elS}T{x5eYE(o)$Z zYsW8*Ku&D678Pk|$0Xl0AE|kycHNy9@gQp=^vUI&6N*7sRoAhNn~v%C@!{WR(9}9Zl-ibX7Du02&kD9p&>ZVjrzi78R4J69WKkRSw5to;&SFDKIU-oh{gM{u8S^ zkdQ7k{w3=BIM%@W5e|1*METXPT)WrgU(~+VZzlIq*T2k zZ=U!3+XHUQ7u3fwAfx$-@?K9O?%yNOuDyvtfZZ0#4cphQe?$f67TRsi<|VjR?$MUq z4{MJXp9`icS?kdm!dXI5V1?P&sy~S-{E(QzZJD6~e z!_g!JtM;K8SSyRmef~}A@^i7DYFrOjsqRm|i%=@rqBLyDXV$#F+k`0xxge|4udb$^ zQ%RNAu5kEVp&;o<1$oUhmUdjZQ(U%>Z+FYYqtc0k6sHRAdPQ+#LNio5^HP9%?}JHx zeaLQhAJUw$nrz}5wqxR;L4QyWo2Q$5MZZN!K6JODMi`1yGFt{|VHJhWt43~oq%clr zFK-r2m!vZn`Sxmm`9;nT_$dtJM%t|@9Cw1f-u*@OpPs+iXG$9fr&=>!OxNT2>Cp z%v=nq327B*N`E3(xfKs2jkMkzvNje{Ga+B?2q*Slym+zHDQl#XBs4}=cK$_syWzYg zVt|u2aJfZ%WHX6LNr9J~yL>$b*)HMxA2VYc>gzi?sTYS1LK(G%7m)rH_PU?*(1}G^ zWSgVLOx&+t6Fqs$O_Xg(^tAHBn`cp;>Fd#d`olqIn%}RuWCwV z&L1sudatj1AU*s=Z=v}olDFT3Wuelj`ynP4n-k{FTWKD#J9`F_^hGI5SNiXg##xjhRnpqXjSZTYqH_6C)wtRhw5tyu1RR5RHjR$W9(p^ytAz5 z^!|o~Dr(kv^W}B3Q&3N0WJ04UJHfBy-(22Vnp72api(Vf=0ROa`rLmxN#(Y2b6v!=5`h2Rd(Yws1T`d`02}WQG~%vH&4hXLompkOv9#QSb|H>#-?NZE<4iug!v$d1SJB-{Ow|(?7d3vgB>{g3^mFkuwRaSny z%v?lQN++8QmCL1LYlj|<1*ikaly?39$Jm=dHGO9N<6orebZli>+gg=nS{F)PP!ZXa z_H`^$1X`#f3dB|gSp-?a60$Jk+gdkJQBbzDiXgH?L3WbRxV_mpYuUf;6?cfRbD-0_3QS&lc#T$D$y73x@I z+Se_5FGv0O`XD-e>Da|x;Z|*Bxb!6$xx?G`rPmJ8 zeUE0$b0F})4)sp%?tlFvARzlHi19kFyKzSW(H`8 zqxhPIG3MeSW?3lOUpI73sb?YD7jnQYyDm7<=TNh;_&@zD z?CA0+db6R1S|6lo=ncTzWCjIzeVA#O8@9@X>8$>kJG2?Qy>3|(qZBGoh`iDQut!Gv z6nDFK4r6ymQ-&=7iX8v6SZ+uzT?vT!LqD1G^mh$MyyC>QS!F3xKxTa^aN-Um6wX4K zvX}>w&+voZOtYh0;`i(w2;)Z)Q&SMvT>ivcwq9GMPY1rTuuDf|KaoHgd0Hd9MObg> z3eZ}4zHGZu*dEX_?IpFRm!BrnbycFj)3rGDciFx^5@{KI3$B(t6oXU)&?4Uz&lih> zrtZ^9YssUGVVi*Jjog9t(gDD0a~ORY9hby%!0dI-+>_Qldb{296X za#_N!S^COgJIBc8(S)z61ni^k;tbf>epZc^jxXAJ&2j;xu5zVzr(K$>($l=MZY?ONdFxCej*8;xs9iJ~ zUzFYvCUKhHr_3Hw&}yj$k7}b$?#x8${vL+r^mXBs#z%I~wxK4xz940<$1&|g4N^T! zS;-UohbuT`y0p#G$QGihil*t~Y$O))JMEVT+O-E~S=LZ%uq+*x62=cM zC(jmhJULCW1e>b~gdAyo9QOm=v$;8i}!XtwAHe(msRAoj8myRK8 zW@cnDjltl&##@Gx`!!{J4Uw-=4`_HaWrmC8$sQkji@QhLFPA6h%exB)TLY;5cP615 zh;Q96I)jrwy;rWkytF{$%^v?S1AF-xPhNqwC|PUnPU0q-9Ix%0D5Lt9Z3(6b(+q{> zuby6spSQhO|JQ*+$5?*}IcMn@&cvHH{>i{#15+t3VZ2Lf3 zozC_ZtnF>JQ~k7muQX3#8vJp-xnIVIvZuU)%#_r3eSpp*SsWTgirf z42FmC1D(m^PSc^BbSAm9VK~kMgJf<3wspsj?@DSB(z12?_MU}*3P&M>7o-g(c{3$; zkk{f>0ckaX8sQa3tuJN5>9X=lDrJ39Up%y>ml$1ikJPxQ#&%({PqRa0m~p~hv9djW z2(J(T+IZgtxMZj9Yb{Vj*ny_RWA6or&<1OcVXZHJblfyl&Lxaiw+DCRXxqJ6gvOJ9 z9@IYqbmnaG{>lU2ULp8_NNdSqphzso(8h&>8qRfpeS~S@a$akRm=wimX387=bfA&Y zJzf*PsdNQBidT}uLvBQ@tH6cvF$1lXw>$~S4j*zBhd3IJp&=WVWW1FLx{0NkX!qxd zRb^*dNTF4tnrixN%U;I=cYis+s?O`;Nj^FhYtaU#=D=8nfXZaf^7{xq0j6q>p_vw8 zfO%nArV?k~El9yqjexU;cP z`p|pQduXF#O-S4kJ3L?Cl%fg>AhLa$dbViyQu-A0fPH@0x!$msujiB41-eST_NT8i z))@l@h?2rWiT(RxY~t5yb&#d=P8(X5{R7oYP~r*0TJkp5r0d_Mk){IIY7Luch&0f_ z9`Sm1Lj1rO(q(jt`ukExaax@IR$u2Qr>z&Y!Jfo$Yfhg&heWj&Kbgbcsd`3 zEgSYk9LDNL5<72rs+DKoy6D#(Z?30mFNSuY z!KihJuW5K)9foH6dD|vfVqJRa6KzMYzsI|oj+E&@82=xNZg%U>d91mWJk-4FCSQ$9!$>0%q>35ZJksl9}#uyi|se$YuBxwDjf<$Kz; zDEmbxH8z?>C;LC_)&!6xvPpdz)tCcvIQd4UDt1SQ=3`=V@`s&aCx45ycm;cav@#tF z*Y)Hr0(gw9+$Au5lUGI+?wJ?tH2>J0wsfv}YJ2_4|3iQF>c2*SxD^w zBnlY_zC!M`Z8wjRK*E4We9RD3Kl%3T3yR4zLDB zp2?vX<~I5^)s9ZwtOko{V1RGlmd)dcBSzutHP8HI3|nt`N_XUA;h%s0nLC{$Ey26H z57mUKsh{|JPX7brLd~z6YiK6)8Df(1NX-C7VDo58*Nr0C^g2Zm2{qOh^T6fy<{Gxc zr`>y~E>4mnXG1F7Vly!JRIrM@y=WFceI@`q39GLhTt#^)fBE;nj=7x0FZMW4}npJP#3-L-hvtWM(`j840?91x-engaDZ23%mOb_ zAv*#n$d)08%W^MlsA|uNR+RKwa!0s+%iy+ zO{igZ1yBZtt27&VdD9!=%{1(l%x_2t%B!&D%jR34(>(`tm3wty_L&FwW5!9lh(fL97-TzIVgOhAc8bor<7A|Y;ZMZ8gfG1C z+GfCN&l*p{uDE*7Em5h3hZ>Svgn5JEJw>Rq1Paa_J8hG4pmR2e5yg4;>P#qL7mw!u zn3~5W%$ZqxNZenZ2_WdlnJCEy8v$5pGe%cD4AE$9|wV_+|2j&D`TuIBT>olzq&VPlW6fTRcG~>b#P4~Tf zimHW-HwW0uu3tkCH-9LZ^Dfn2gLsa~JI_XK(CxgvQ1mYE^y`MQ;7)s%&MPQ}#yEDT{1gIuo~l``PurV6pGC=MaJTUwc;ldT}R_(MZeaE!h)Se*ch07 zH_dDV2kvGuqt{30KpPE5dyn38Ig?kCBSph6P*e^Wkdn0nu3uASX?}$Ok0*vWs#mqs z0JM$hx@OS^p>NCDP|U^;+U5lxytE{&W@Pen#YNXW9*TP$+s&M-&Gg*^0Cf*z^N@BW z$~A13-X9i5Hjz9+;sHLx5LLXp_2@z@zuHVY8+~X1g0-nWZ=S)?R4`bpGttd_ zn=^qQF7F(UYXb5D+7s3d@?cz_VMn-0$8UI%y^4-zAkrO96^Y!HXjc88R2hr3m%lAN zR8OeS(fPLY4Ob02_99|z#tUHZSrgiH^>gh5E4+24`F2Jin|%FfpDNi#bBhHR#gaDQ zpAThm+P!6Rh>*3yV6Oe*J-c#7^8%5!r*PeUcwC6Tr9@SJUG(sjWIy%e^&8T zX^uaWUcpgZxHZKiE|iA%*q&)w*s{k&fPNVj-T17vTrW*1=I^EM`@wXe!F$wL&LMdo zAP{L;R~$Lp^bFdi-=wBhK^kI!|7gP~T{2Qy?AWDREE20e4BNrx8OFVe!h+;i zHGlFohZ+noUTpo#tLIqa_Z{yjrT8w{>Tefj7Jdj0#+6OA6uO(9dAO>LKVF(lfBKX= zX}D^0vs?3dYjT5)g{ypIGQ-)fm7R{8t#ry)`ZS47r!Lxx+g3N)KB=>`6zjkGs;tMu zw4kzRl--%ka05FA12az*+H-{s7s$d(iM?N1zKnf6yONB~5Xkf94f~J(uyf_RuK{(! z>DTA@@yq8C1Rm3J{!7lkYp zTQE=s*XOG9#5ru~YJ89t-01-x1L7j5NdHUzk9iZ0o$HwoM`y&H=VlRrJncNaPjhZB zU$LJftyPQcOdn@9B}qO~$*ETxq3`)@GP21%4yaO2x@ywmF0>(*FQGqQsVH#uA(z<3 z!xooZJ)cZZ_-y*>eC6;by=Trnd|H$$2$@(bl+@y=&tPz|Ky1swIW|&+oqY1iY3yX@ z=7tZXeZ$$aN|@(A5~Ym04VUvfH9qW7qX=L(;W=t>_DXeQUNVw@`~V6{^*(-pSNJ+1 zu^A{=&;dgRD(F^2Shn#p^3mqe#*6BpDUvByDOZpcF@%kee>~^N`ZyCrYb!g;3KcSS zQ%57(7x|GoR4GxiO;ijVC@+7Y(WjG;D0W7Ax}8v3%UtCY@j)*Uet%^dCdC87Z1r>+lW<#eDoS}@4l`Zwi*s=|62%e}gdRV@xC zabV&T4(QK0(zKnR2V8pHpSi(y9Hm@d57DPUY$jG2X{C`Ko&Q~0)eK7V#tyjGp7yMvdDX6^^e)+*AGT;?Sp ziuaxY0@rF!D`)E(Yq`O5EhvNN-8T5_?vHh$E9M^X@(jPcIjp@7{I!XuUDX+ro*5L1 z2jAB$@<*4)*maG!Sl8xbU-VWRU?~Sduw1Tlf8D;6_ytTUtCSfy=+n)VXJ#ahR-pR% z!P;&;X=$VjFK9XUxWIm45})pbaf*_=`WDLdnH~l5Y~o(8rTps>yL^phYVuw$+Ez9B z=|ta&u_L0U0G4)e81OUt^sS)9-$xtD^|#w4eGA+f?UR1q?x)}X?KOd3aOZJf#*zwX zABhJqe5HEps331EPW|ZFd|$>jQi0?X@8BK0$T_QA<-K3n*TduERjrdxjyo}nQik8l zJjZ@%1&dZp@T(h)bWuuLCES*R&TK!X{r56Yaa-H@Pt7<+TfXKFSydo>YoIlClk&uY zuo0QpM>a)4%HOW(sF!i>Qk`Rx`7P{8R-O3yKUTaq?=l$8_OIS??Y`M_=A-*iR&SP1 z`Q;%J;nRO>QjrtzmDqlGcv;(>eH9o|MSU=0)|1OCYq5{*?Bb0lrC=A!08x`-=jwMi z6695=t3wJYM1ZGt!XUE(uES|2S(}JKGX?CUtG`Y$?$XH@W@lx6qynba^id{SF0q9b zVB8+2p*MT3X)ztuuflIH;sNCUQ6}_8q$)+jufvREg{#-B*(j?JOu^Z>XyCTDAQac|PEmEa*Tou1BEYMS3 z)CIadp4L!YF@%ylnyI_bhaTx2GhYOVd-yjG54B8}hq0d|&m{Hl3$7cQ4;N{j=QiaZ>~agPhh%b zkj)$&fghED&ELf9)?%y$v7_Znlq|5DdCSY?JFntCZTB2Ztj#yVgIJ)X`fDcW(Lhx` zbBmQj^)i-@)`U|+zimhj%a#Uo=5Dqe1OM{px_|-3utfW*`hb%CsVYZ#KcIuw_pP_Z zNyrynscx>G^J{_VZ99i7C$s+mx$JJ(^ltGf!(|;LoDp4dQWO}%bjLahcqGqq(akA) zFkG12aW&zxs;pi+(>w>WPM^568<)e-HExjhZ_nxzfzqa#*uodZ4&9uJ^hpIe7_&zH zf>2aetify3+haFmv$PcGNIc!l@Rk{VK_GkJqt6u4Yx9OjL%f3cEcW5pc;! z5$%;}W;z|0j#=8xe{snao4PBTRy%sdLy^mScqq#k-c8f>nUZibC=tOc4vS^rvd)ue z*A#_N9v@8y>5kMN(g@KNB`vcVzMvt7_HkCK69JFe*b%}%LbYFSXP$4+$lYo9hB9(M zNtX+Eg)pWV>I6TT7&3j`%;KJJv-);<759sh4EnM#lLrhc2aQ~R_H7!W#2Sd+H|m{P z@^=K63P#MNCv{TdNvX~WPk8hpAa6UGSb=l`Ue)Df;R?{mKb`q+*ohO(iKZz8QMkgA zK8r@}WC3V4mr7XjNYD7K#5I}}$2gpedXTp84(3>#AKrglkm~id3Mb2&D$I)&Fh&k} z(J!j5rc)!-Dprx$u#h7P;_;L7Ppj*yn@3iN^#h5l1nt>!&F;NLj2F@q#Y=70#0WG^dMcpD zO*|Lj%4sc+@^&V#FoWt;)IK>=AruXU3G8(c!2A9ImF-X9x_>7V8i=MQri^(gW&-y| zxn^$1S5bdhyi@Z}#$1m1nVd*-`CpVXpMs(#7nW3*(r%a^lpp?jMwRv)-@%X(i9`!< zZZ5zw)A-i+GC?UZu1Au09c-O(#85^*A|O8NQGiXUN`<^gI-TyU@|Jp=o7>=a0eB}5 zJ`P5LO_->njd#HwJBhrucVTfHh8n9zzt+f%IudoEkKKOL7`-F7 zBhR=Kbnavmc@18cP*R;uGA64{ecq4D?+3Ri%Q8lMJW83enQ*|iH?xrl`a0bm7&NDZ z>$9kU&hH_ekx$&r>aCZX=OIWH&9+m3v7DI4UTHP`40dlOnpg*^lC1Vbul z>>0@7&&{WU00(OvKf4iY1Zwgu7%Ky3=xJRVV0&NHhXn|?T7!YT>GS&FY+reci5CeN zxDW>Kq4ionW5Zy?8VoJjU8&v?FYv@{DYIt#-(T8L5Yndxl4th<$&s>JLVn)L%mg}Uk4*?Pp2q_l6uGTP=baiUI`H9H31;TKv|m$e2q-draa$Rr6^!diBMi=I64{5jE^f#G z&S?t5Bj(PL$kBP(?o4+dO+(b{e`uhK<6`*wA+kGvF>-! zlEOf{{5?0G5#nN(A4u||0cq>l^Qs(NzAr@ zWK~9UX>nQMCS?xwv9>FEd#V$I%Dwex#cO3&iQhi&;Pii;?EbDmW9gr(ImWd;Ud*r# zA0#t+Wg1KMPJA$X<&Cm*R&8yPjI!@$kaC+Ws@P-=9>HD0u1Wou#+hfS7I#(LZdHXH zN$H(0-FqzNhgg~69h+|QgyDm9Q|}?6ZvWx#3_0S3CgRoUJ@?9a#pN1*;`vn{m`kAH zEb*Y(@BOMo|HnU1JYH?q{;u=stpbhve9j?w`1_bP9ax69zX1F8HZLT63DdsI0|I{-I36M~Q>}xohm~948|)jFnG+dj8_Yrtx~f?B@55AZPa0ojdoF z>1tFelf7Zmy#_4mdz=c|#egb;hTX*|3$8ryG|`x>F_n0LcQ^8?qh2`s_glK1 z)DO%m!ObFjZJf&%v`QTt17F(#`E+E9A-cy$OMp#k7BAt&)md|7iT$E5n=+!?|@#ANG`eB`_@0dUGg zbG=s>VR$3#V>p4KKdTO~!n_7JT4}bFhN>@yjBPH@8yf}mNNWK|wXjzjMaYc|!oX9Q zf_>x|9y;NnAkMVgTLQiRRvWm^{@TW8&DG^{GRl<`Yt<+O$>J_}{OhgRJ7?!>Z8@q8 zdr|UZPP3`Bcso;NI-lID*OT6O1xQ=gvw^e;cR(LX=?yWNGw_jVuI`$f~{ z1ABg0woUN&==1ppZan_inQcEeIe+Q*kGH=3_Ft#|{`0B-!oIxwx3|8!Mw;x3(wy=l zaO50;Q=9dF>09Y9_y;qtRxDK441`A(dE=YUoV()9RQ?gu>XRN}blSOdfJ_k8@LSU% zx#RfwED8r(;X<^&t#R%#vS2s!qLb*AAzCJ-$m*!6PXz#Mfe1C*7Gt+>W%8}sEre;;D-_4uy;?ya*C*E)wFI({XgKl6mYG1xKah9Lc z73T<9njELb6pjee?Kn^RS|ix)?Oa1hgu7lbd)wPWPG2?} zG0e@d(dh@6v|l+Hu|JSV`79PIgeLseT;;5eu|E9v|J??boNl~6!+Xp8RC-ER-8bWr zV(&&YjG}@8O&$g~-Cr_|#yQ%B$QG$oI;ls_yrpAEFq(z;^n9lc)Za-X?1fJDTZfMx zeJc~J!`0GCX-Ca{htks0d{?7lv1;t_iab!%`B~cwR0fc@Vrc-`EHn2R)rzlV1B~^{ zk3f1vw{G^FcJD~PVXFFJB<15HilMsh#*4~c{~8vRK3hTypcT#49)J2Nh88537KvRi z8G5UGrlOneWYHj|4#YV$fGXS)bs{_T>y6vDl;OaKe71BU$j*)G6(IBCbViOp2cLX+ z(ttcgD9l&W}kp^n!Vdv`F!jvoGlC^3lnvsl|C#N{z5-0Unj?7hv z=_2;?-HwaK%*-4Vq4Zix&#?T@sw?p3=$u6)tK+yG|lnh6KIrTg7_~s9<%XHMZ1_e6ly&3z+4%4s&c$W4S^4viqmb=Y>=oh|H-w z{G>WswTR56lQsF?g^;m;6+)X8Hk{7fQehY|vyA{o_02A=0|p}aM1w<_x99kKSSv_J zmX1MS(*}%Vo%D{`n*vX}>59ZmA|cGv1*9TDdi83%T4&Q9xBdh%SY&N5cna1S$c~-O zGX9+E7j*fB*9_7dI7e(cLeht4yMN;D%DP|JfCMtils7lg2R`7fE` z&ODcdZ#axT-wv^ram=cjyOLCwJ!A*wt@!xyzspwyLjg*-@wBt zNC==<$4;8ng?Ya^+C48Sc;8FXhjiq$Vtu!&9YMd?{3M@usDWn0wW>m=zN1N+{`_&7 z2Q78mPC*lWeo6zLvPe6FzDN;rZ2fCEE^wuqlFz2tJ#OYNx=@`lwOz^3cF-2qU<~)E zG_1O1<}r~Yec|Fh)Wj~3?1Y?Rxdpf_gDp-DNsBbN6ZLmhxSv_9DW%milg}$$vXLGOo!UdodI1`H>P%rNgNrlK%^jv0KH%A>j-f* z6I%6wkzwO}T!))De>zprn(k*Hzycp^Q)uH|2Mw8?)}9`ScjE#^{o2L){KmPRu8w;s z#V3D%8?&Y@Os6_bT`mTSz*%*1ww9@zPUCn8UYT>MRhRbY0^>sbn_5wIL58(%qfb1p zmI?_3W)0oJAgVLnEuf>kKHF@+0EzS6_V~F@&+6Qyo`l(_-j>LNT-#2tSt@UJ z+R5$|2gxkH&a1>)-r8DEzrq7b<=r+&mTfMj2e_>xoXNWqk7Gjy{EkkwyGy6K*;VR= z@&#O_B{@_3TfSzl3yYEu-4qxnWnoMBuV<>i{=f7g0Xxr@PE^90nfm-k{ znTU1v^WqH<_jI^PXs|YKaT~ys!*#aNq4q`K2aU>sWrz=6(0Q{tjv0fU`Eq)R{rh?4 zd4(>jJCR=U5I?UH;hyyM%qW*S+uAT2??jflmpzi@wj^_7b>oc~(^WE`98s+DrprRtK^GRKv4yp^Q1DvTz3*>>Ba?Ou2VwzVLzv|5)JrO$9 zLI(>f25DG!*wlgt0T^{~j9{T@fBx83q!fBn!hgl#KY3NPq=H}*^{$Z_IH5d-)*E44 z-3E=6z$ufTf3?2l^gp>8x|;$*0}|Nw$yxZ?DP{AL%^9x?o-E>*_)&fC1`nn8b*D0b zw^|#1cv{$v9bh)A1n0`z=O}I*JN*)DOVji1m@Xo5SIQ_wc%7(>yf)S9EqdfAZRJk@ zMm0=4RHuxW5ry-q!nMxBN}O43CjGTk`04+cTmR$e=fjSPi1_LH^;OHv*6osvq;GQ^!?FCm(( z)h%vFY;IU8PB&kP?^v(COmctOW@#}G>!|*CkvHSlR2|N29yFO_AJBQtqLV78FV2V6 z$A%;oqwRX#RJZu6B6}v0df5=Ou-wYZr4xYhjTl;N#~0bNg5u1ys_W#nBl&8F)A!jW zH@SF_>{_%ThjO);zA!LHf!g3A9|?n@PaOeKs@VA7bq#+ z)K!|m<*TgZt|6SK6E|}Y^{^2vXdHJmF3POsx-T7Qp=sEr`M@W}(cpmR4^9MadD&EWJeWkk!)# zGu(vBQ&*IrJQ=QIr~QIInb-JaVUv2!?6EeJ6A1HKbNt9_|GMBsKU zPeN@@^`aFCyXJ@*0C@%r+1v_Qq5>Zn=fFA0BaTquDZPrfD)kJc6Rq3lQ&2*_RN~N z7#q1RGS9Mbe4l$_nbCU|v;~(7Fw=rLQPs%k6T)IS5CS2=f zaa1cz^V_gubr)A3Y?S=Or3ijA5|s@HG^8I#lwK$c3Sywju$Lo|se;r%;j*CaxeVCh z2E~%}nToOL-$6WYUf-K4`ntsZ`^)^@r#>>f%c0Vs$!cd~7~ps4BlYq(E9&3Otq$0P zED%C-m`t|LL<<8J9|jz6>0uJ7+06**SINJQ-A!um5ekLYj>8(YIwXDQC~~_(xYv$^ zYw9x(9u9-x@#+17Xf!t^W_dU}_dmQhT_Gfz?`+WJy_*TPg2pp>508!Cn-dm%Ziuz9 zLOZ`>UwTTjT~bDdXpxj$l@>;wCFVmYmvD=Qt?d<2M??d={A}!k{2llx?B*_XLe&XZ zrad~rQ|5;0U(^Pypw-#JBve3kQmaRm)P`039G11cp0oeZiN%Py9_Wkj$szZrfY4N! z1}kC~p&-sdqG>e`aw}J1sMWxp#nyH==SDK-Qh`geL=lu5OwRtQrikvqX`#e}!qeyeLgaIbW5jOcn{573Q#^mx6WGg+jc7-3O}WIW+_+ED0-2pS+I6LaWwI>_z(!szt;sLT^}ajzSjc zCqc2blb)c(I{r9&q9s=d>s~*{<{-qOTJ=1?IDEQ50Zxzp`@Gphb(D36c%Uru6YLW% zk;U#i^b-Aq=K0O+DRb*hpf+SRdMyiebn?Fr^=4@aNg>^{Ga<=hvj&U6f0q3^j z2m}dXtw4JyNQS&b{{{Q-Xs`Z+iY$T~7X|#8y*40$b-~cOJZv8AlZ#l`OZ!f4f~>bC z=8OgF#3!F_V1r9V|UGw|RX&Hvx>@%k0KdG-JPvS-uk zSC|c~g0k4^o6!LRPi6*O=>n?((G0WC>kw{4lGiysm%XjV%| z$2Dvzp9**o`%FQ9MX=>oPtTD}R`0&sEz{ndW4IC4QCBo*E48;pyIBFvIqXekMg0k% zMdRrrd|!W}bYC1Is}BQQw`dGLO@hOC7aVSRn^|2{0Tlma;~>ygtIC3uC{MJOqI)!C z_ZY%X-$nVo`J(UY?7CB;>`z-K{>2=Am`^^|vYLYuf5%X2iv@}fY34@O5WKh$)NXAn->*Hv3X zfiqJ;8W|Gg2;J!w>cUwz7@6K0TMJTQr{x-1a1Gh0l~}%f?83U|W_tGQ06$qwCWapZ z=YN!ygR)SAmele^{FGpQSs!f%nlX<~jdNTe9W_oCW>T*av}yUSK8F5zy)=OK*EXb* zZi6T=!!I(EO#P*7v-(=19}8vrFG4bdR~-ljEG*%;Y%Ryv+7SY4t~(ASRE0D88t)_o z>L1*%MfTh?ImilFf*_A0>TNfSeSnNLQAcq+57a$Sc3jK8cxFW!xM}- zOStn{71}AHO)#gZ5HAm+)v+0>8tqzQOlZ{dIayofnE~HaEHH7j| z(t2`=`>q6WgE0jqnPsV=aI8Dqa%n z{1MG}ReUt{g>^b>{Y=H=cnC-E)Ar9@6&*EA;N>)6qLUQ?(G>ge8Y9~InPa*V+~Fqe zqB`&^8LH|b#yT5#r%gjyW*f>&INnv|rlD|9*ufrFi<3aBr51S$`t_gJg@J8c;?8K- z-XYVee;2>ctX>mcINMyk$E#0WFhqTGos+h9DF4ih7kR=Hxbk zhP5&if%(tolghNO-7U$maxU+Rm`9#;YSXdTcB%%o)~ggF!bA7&s5A~dSJ^7eT7QSD z>UY{DH~PmmtFG%a<)-SWC%@;tI@cvzW^DgkXT0&-Z`_QJBO-?MRqRm)I|-k1oGLLa zKR1~y0Ma^uC=eyTuwyOClv>m0yp|m~QY6_8^oHsEOde$_sjWyD93)JVWC+dQb10eh z?*9S?e*N=9b{u~XK|=Y(#Y+xHL?AT90@LYRO!3q3oNhhTuuT+XP_f1as>7C<^KPLu zbWaB$;|kP_;aIxpU6>psMUZ+1DWwcIT5i&aj+vYa)*I&p=tmZ}$@(h_8bfO%{Avl~ z8K3|?!{3w95qVYhG280Ine2z(j(=_#M;{#H7Q>NoBAHCy&!vk7R11*gI@L@mp_Jem z(?#SG%Fz~UycU$XEX?d`D-9b+7Q3!0i^W$R46h?+f!;l0p?W$mkq+_x7+S{~6kpE2 z0s^7Ep(vF*Oyr#Ktqllmx>KAD?ZI67?8H2ZwmA%GDDg9wL=$?D0`1`Vp(q(L60c@f zt9~hD=m06{o!BV8s46Y2H9@5IJE#;j4+M`o(uovV0t5s{+4*?Os_IoJCSee#XylVC zxScg4G2{xvcO72RuMO)*LA$dQ69VyGpnEbM!%hZ{Xc2Tf+sziZ$;LWQdEy1(L6#d$ z7K3iqg+J|#-}6-fHQ=vHHY^W}D_2$qSVy>VeB8Gy_qgFCZaj;zc;jSza~wgwS=$In z^%f|+G=ElehrRUd!vKc%yAC*6>>ALIU}h5KrTH3yC0BpVbmDogFc*g~PQ&Wp^3&ml zgEBThjc(}mzs%^`l{DhJrRgmsvtMoY4EeE}K(oIy;o9tpSmSju0>HW>$DPD^vjg(q zm~%tRGJjmQGNs#lyrCYh?K+zQs=c;7A;wr^G8gW}D?|UNW69Dln`SqnwvliAp&q295-r=;n?X6H5F?B$aMig8jBMVU6_~#&Pw>VSgs%%x|ySA1EX7NLl-o<=# zH%arAvB4^s!MvS2+k;Q!F(30`@N!Yy%VIe2H+N1YcT>W9EF7=6RMoX=>p#(ui!Pz+N_yq8qsNX zxr`>SFke{BQ<`B3Ez(`DocV8##mj$MrPkZgLo}`0z53Mc75j^^xuWr|UKU@%=6N>R2$6Oztt4gOSyOyLqZe zQ}@msajqc$d29A3%J0*uBvLpx&aSrTTvdo|qev zsJV9zzJ>k?h;0mV^A{YafhUTf00xPQh?(>(FLyu|yL>S!DPWgciNXE@3DH1B%5fAg zgc{kV!>e9|)^yP!r5TzD7ipKqtvESg-4rOs^XH2pT^-!MOV6mFUD+`W9Bjj)yI{>Y zv2{yt*83bcy6w>Wk^LfDjVnflQ4&6(^PMwZ`$YHV^uYuQu{KmhM5wICYCAh+KEs_g z;iP>s1M08>qnGF(HZC*Q4G))aru!ZMO)o(b3yl9oo%8K(hc6u zMAXfxgoAP-I7{QRTp-Rrq9t6hPRLhSA$dD8@I~OjJ~Qwzq$d?}m3Hc_5Sr4SIuNOa z6Bv_wlbaNXLk13~V(=^2V34ZvO9*}LYVyEb1Kg_{D4=t%C2$LpO9P&KZwTZfRvr5oVA1i0+#p6mJr|!NxZ-Plvdv;mY_J;* z*J~Bwraqv7PcZ*;agJdQ&S9x%;>bhd#;QccGlApScds*mS-Jh`0u5Ehk!*^LGkYO)I zb&0fplDUc=!_G@PJDcQi8rj~%M4wFuaxSfQjN-TjFxlTIJ=RsDc{vG1cW_cG&T|=#G+ptPIT;2@vn(CT!ly6K4kVO4{$4?tlC{?7?#f+Z z=Wx1&!T0>i{IQ1~y(&*{&>Eb0juS>&@o5hvrA93MKRxiU>_U&c;uhuz@=7+IK(`xw zrhFqt*%@Z^FU1Axvr=0v513N@2DfMwHHI4scg-@qYB0cZs}n$_C-nnf+03bWLb0Sf zLC3Gouuc1eZF;ThDT$joc=evMLVs7L_q2cPK~HBqi5ITyX_!hnIjaA7gS&=ER~N3e zX#9ROBgrTYqRXx00Y3fp*Goi+a@{Ku66n0G*tw53FB%ZqS4+#4y|3`n)FPT6?6Q2- z6#T#b{PzU%Xm)l7hBz4qWMMOBs90KBni)WITx_>-u-aBqq$Jy8K|w*?33mz+OA`W_ z0|Fx3I$-DnATRk{UGJDd)N$8oP~1F?rSr`BTO@Tf@{?L&)o{a1x<=V)D{I0oC@Xo| zm}cACmyK64X7=fCg%8KyIq1mj36>u_q8|-w58&qUkWe_RpA5$4btVi=@1W?F+Fh9a{;^8Qwv1L^h0r%S>9f$sH?J3gvdMzwS@bQm_58@p1x;FAFLwf6Q zP67!zggcE0nF#%AzJr7`QmJ$fKT5F?W^|`Q>>hsuCCUm~phKJ-6l^A+-*!mEj}4>s zk?x4TJmKB$ZR0I-?c2z9^FI+^H+oChC9yM2+qp=(J(W<-EFp1q3x>wGxW~*1$An7@ zw*a0Vhz#vc9K}h=oreUA%+ngxahNY}4p()d$g-Gpep;pI;RP4LvrxMMY6ikbnj>Qb z2RUaZLPvJPfaLn?2af8EdbS4*uSTe7rtzO#N;8Y)8Offa3crx3HkY;9W?t&%dvS30 zrm3}@o^YwA<>+aes=>4Obq4awWXaqxNEd0`46KTJ}gLlmT zoyrX1!#$BLCdp=w#mPyxBF)}W(k_XYD-XxwL}Bi`2jM#GNa?S$SgF zleO7M>{4<@QsNjmMQqt*Z)Y{z2-cjSPUajzySK&6zL*ifEKStu5E|r>Se)$I#!0?# z)7nL?M%u8j-Vo6`wQ0g0-xR{t_;}A%aLQHLB`ylkcFjf2Rj7)F*pf7qXq2DapVq0Z zZkFAtmR2a%(h;Lpkm6GQuR>0+;wxd{}e; z>V}jWZyJpkg;p|U1G-7a%zJ83OmA%%)#ZtDtDW|VH#QHycT%8Md#qgU|<@9@_o+TD-nCqNOf|zmp?` ziw_x`8s1N)U%!4m3K!$?TusScp$y?tflUo-=+u#l%gf6n48W+?nxJeiGt0pz)2!-A zKzzjM&Q%7OE*p|doZ%Ff6+5aE;mkTqI<+hK_YeUA3ah)odGPXi&0*^(Q4LZ@ z^s8NjVxN;Q^uEYb!Ac;Z1u_vU^0Q*;ik{^;tk}FXzgD~CKfVm(G5aH5Uy{(AOCqa|$=2o}{DdaY#?b<5iIus)9S@a!*HfXdcp0yL6Vb!xfth5B|B-_E{Y@nmH?5 ziS!25hG-i?#s#N3<|y9~c_P3;QW>DVWmhh~Xz0PkG|fd@F*_=_j_*sX>gD_2e41^~ z%aYbIiz}Y!B;d)%$bkPX9x%cAw4Z5t1WH(E^K7^XROKG2UW`{qxFmx&aa7VI1mE21 zOtd2CfXP}jpC(}m`r_#-r6--TFwIyrfcIub$L>oSsbuh+BrIW=!Ncm!f%NK?Q}(eb z@$JpLNKg4Hj7-Yr!3U$x#bRsS6vd{AP^jsnOghmm%anw{7X5D}41B0)Gpmiu2b zA;sxWv$>$lVrqfMRG{rB~3?ZL){}fPOdDshJb^P@08aDIq%6m98bBkZC zasc7=v%@62<2~8uA&^J=e>i*du%^$v?K_AqZEazyZLJkyrY@ACpjBi~rehgJ6ckiM zmRL86Yz7ERfP{|IS~pNpkS(c-ARLB0Hpmy5vPMF1|xJgOA2mR=%DMx0?b&T*A$ zpGS0dxA6&*PRT*J)elsW&(-(|OVp=s(94Sl4qArkU%%dn!}?5gk|j=8prQ}U$kKsS zt1iODqi0rXUW{WaNlS-2G$v@j@t>WxLa&@39Oq-i=oT0}kRU?q+`;`qVbfg_WN9eL9YPHNYcu zRe-+$+dooY3Fx%%y87*CEX_}rPNLuRQS3eKn?$FclvU?nSXtUS8K7c{{y;O(RACT* zC2ORF>kC-<7*I+qdI_rgjW{MvFU?mAfWMh1i=)rhofSU{XTyG+V9{b1I{g7|t1wlu zj*6A`?(kv?w@e?<%0s`gDxsbvWKr&ko<{NrvARdTBev>}xtit&G)>l_iW>KsG#Nu) z0R>acM*5UYy?d3B4*yY86f(n<)(43%19st4)Z?#0d#O0Sc5#ihPm@j(z!@ML7)hB0 zsujE>ENDvqu2{$#Hwz3!(h)(=wv3!vvGYVs*3g=>lKO+%OjnCe14zhTylnn=|2#|^RbA0h_s~k19AOLuCE=QZQ?U5uHfznF?i*D9 zY}7A^=q05kixLgPFMl-X@)wRoJZ-Di{n67Iy27;yC?=#^$Bft<|8FKGmQC1 zHhJ?rl_M9~NRX#EBS+ym*RF}13EG^kB+YSH~kTW7% zGJ5bZTP?bQDGAH$z5k&cOpc|rJo8`~#?Jnev z+D%-CS=?s(W5I2)AgT6N7g4Y9zHqQ`v=#P5JYC&x%el~0%EYc6oj|U)!Z|?@*1LoL z8LC^>`aAv1xziIR51Kc@iFUw%OmT)D+Q{vIf?letI0O;$!TVzJ($+Wm0~`wN=ph@| zw_=PrP<(ci}3 zuK}9(g}74rg=KAt0}mrF4%US9N(=M2kCOXJ2o=g|ctK2&QvB0&477*#_?^_6!k%dZ zRM2J%O|j7+G!9;cRr|zUQT)Zs0f#XyNO&Bsa4#xFRlOE#jbwa|Zk@;3TaWg41$8rc zSGUW2;|iKwxk|t8z3pqq0@~ARwZbA1xVr3du$`vyA9WI2Nu}ypRfYZ+FXrh~n6BNp6+C-nMLh*Stgh|NOy78=eh283=JMOds zElnwZ`Xz6JHW3oau`d@dKV&qVouf`_R3q`VK~;@$jQJJ3iRLroN3FGHxM)M5Sbr$l z`1dK}c=EF)4$Xf`2iwql^@@~ZQ&Lq7HXxtNb{*a?EzP@D5lO(Y#rh zx4Qr&n5vV82zJWjF-LxI_akC0)M%!MVN}uyd(ST%fXe;yUNK?1p)w&Wb~o#GbIyjK zz9PcJ?uMd}4M?Akd>CVjJu=Qow>x8g9N#n<%2+{Hx@3TGdl-Jf`oKjkNSR(iN`S_V z>=85P>%PXE4&$n*jfH^~B!4Z>(rLmPD)U;<3=ZczOFk`e2_3fqDte~L3wRv!WmM2L zC&>6!#L-l2s|B6&@nu3U;S&(VcZNwE+9xa!I+U9wQzXMSIVGrzZzpW?g@CtS=a7cj zwdnZ|^tW=MZV(=#+hz{X{X)rVL0*jN-03_)o-r~YGiwYMO3|;~Ro4y~!24-BVN|3& zX+RZAC;4|Mfb&VCSk@H0T~V6c(I&3VR5(w9CKs{48X; zjJmbg7&~I0-r_oqcji0UO~`w#Qk15wC%^43ta}WX(qM-L2PA$8g^V!Hqd`Ea9mg3U z03r4Cgs&bYg+^3SkB(iRcAAjv4`n^QogF7}AzhxIaJ1&djx(TZX(V2_<&`)<*0lC` z7h0^Ts1kzd^|>QEg0n{q6Rh1NFPsQToi_-vYXNc=;sbjZSR>nL$ii<@xP33kp|St4NU zOtffJ)E?D>G`OgVS9bF4xBK6|mnWY8>WM)gLF5|ua-(AnkTZCel-+if(GR?5L z5Zb*1t#(lh8WUr~gzKBb%yF2)7G}vVQv)>-D3qn@HJJmaZFs#6dl~br2zGyqPJ#2y zQT9C~5%vQol5X;NM(YrGmDPGu6M$B*%|Z!?zZNq9&4J+p{fxxPm%UC*?+BlmSKSYfX!E5 zVIToJfHQp9R~T2@w}nV5`C(7pl$VqfnRKWyOflpO$gD)jQxuY!D7xsh=(U1jY>v=} zHLWiR1wtIyML<;(wUOXO94QYVT$%rU20rxfw`9?0*ML5H^h(88Zt6M%H~>Fv`QwbW z;GPi@MFxQ^A&;QKR_fWh;kK~O?abMpgnfgaQ5G(=CufV|u%Z*bPAZ=yrUh{sv9BBr z3%7As>U`eg%Jk&j89{PtDmner&Da#3tc#r`Q&Xu`PX>{X0SEQ2 zoLm^tmwakuJ%1b8K>G{&<%sOLxuP#2xQ>^c0dDDHYL*~USQdOzFV?=Zy}V zYX$;y;F@JoF-^pw);pr@t8A_a8}aRJv_hMnxd)YMDw9;r*L)#-TMSMZY&gPE)ju^U zhp5}n0tV+n4(r2%#?<|E>rzhp0{$0i(LlM{Q1bQ>9d3tw5TgYQU;B44wJCT0;3^(& z$4gaiQtdkE+zii87&WS;O8J%<%Y=?Tp~G97&7rh3S<2Q}x2-YcJMvAzjVEn!zO)l! z)={~~^y$t7rAHzXng1x@E|7FSOx@9O;U}gxc%<@4xE@a;0370x8kDS6X(LqRDnw%=X++cSGd(@~;tD2- znH-ecN`L)Nn=OMLdwOGiX;)*dkRle&nr+Ed@27J5QxZI=dDL*Ekp+oH{O%YSbm$L` ze68NeuELW(Q>J=yRcm#gKD8GQ!VF&CKQUJV#=9NU*kY#n~I|EW{EmprnLugR5Xaf_tr zsW}XDp5^Ngp5hcF+N4b^_wy}1v)*PHqD()+p~5(i^6FCpOqh)|za{ra-sbfAhxlkt z(yq!RNyP3ZIG2gCn5FWw#-6wl1KxrUPfVRt>H z=yZyiW2*1zd(fL0EwlE^a3s{QMxAXrS7XX^psf_zsqZN6gzA5b^s?+x%h4?7LJ3Im zBv8vGuF9cwuC6kG|Cv-pj54d?l#v2NTh({P-mc5cT!`6- zopu5b*`%COh5Qg#saZFu93YS`#kP|q_5R{S)_8c$kkeL&Yj0l`m9;u#jY=hz|3y9g zE8cinD4Nm!hV+w3#0kg(e)A-iNd-$TfnBIz`$&y{k#MDdzMH+`emTGN@Klm`T=EWaw$RxN`S zN>y^^qn>?Q`lXs>h2rlP%o~uEhD)Sc?-tl)sjSA+jZeQms0IwqM(d5y>Y(l7T}m zOZfm}ojDp{jjhgfe-NR0Kr3YS%<_|WuzBu90gWhFqC)1bR{|0yj(&A}Mpp83G)}1y zGu1$l!sEcuB7Ije7!mx+(SA&O#$)n(91>uJa8QZc$3*@?WK6(e{xDXNbw{$lMGcXw zQs2||Qg85eo5kg_=Bb?dQbqeDQPjvMQIy*qt<``=7XZQhWm69^6QuakWyG^}UpUkV zrM*UBlIUXRv}7*OhZWX52+Hcb)IPo3uBO1>V9RmM-023VmD=lQ6=8YXBP_Mez|v$= z3a=^j*Gy~5VUQtJRyqig_%5XS) z8rEbdPV-Gz<11jgXpyhuQ|_pUONnI53K^%kiJ!EO=}ieS&JHm7&LN7u?M_;oLM1e& z(oY!z)thQ`&4eN5C{NTQ{754IR;?Fd-4;@5;Oc?DoF{uA#3m$Q6L`O68N3Mj|~w&6&5lKpBSU>BO;|;Mn(Z5eE_t_9cuPWJaO*Tx*Fd&%4df9Qz<(Yx2AKI2VWS!nl>q(}) zc(q6Du^x7lDZbr}{EFcEc@FM{)(afaCr%Ehta(>cz8L%OOid}9u@d$_s(HI67tHoW zW^CAqa2L}~-Ir?MG*@Ur$?e#V6xRrB5T3mqJQcFIAguL|kt=r|Zi zdDahXZm1W`2uX>DtBaIb&g zj(-qn+{0|0)vj>(a_q{qjp#QROEzU}3ttYu*QmL6QdNOWk*n$U$chyrPXm$_=2O=z zvtxZ&p&%Q;O~UhqE1((FomAww$X;esS_dup4+px15%{a|VywPAjG?w+BqAT6OLY?f zYX4!z10)}#Hd?UX<6*fOogLxQH!BRuOzt>}tDhT)$QSYFvJ_FEPsV}rfnahNshtL( z8r0o`XxNG65!&`?|Iku-(XD8H*>D*3x~M0qjU=6ZK8iAj04Jp^4!ecxyuUJGxf?&Hmn^kRUOK#@#snIs}BMxE1!aVbbQ5H_DRCk zc`vO`@@*nmCX&>j1!&_C@R?<#cr2?EQ0$Q8(PUvjhoL@EBi=OUfA%!2 zMmR6uad_n#+GD7yJ^h6>_(80;LP1b$P^7I&JgG&X<-FwYSkXJ0N9V&6YxE_BCyWEu zW6or1u-HkN)J6o!83;F?vNF7DICrN*#hc$&v0i&=Vh@ynn*?HxuE+pdjPXS){#^vA zZUf~TXIY;(K&p7K@UM5B4rCDy7&B~R2^(k@4F>tepYvt0G+Emj%NLd$>f2(~s=%Bw z^A}v|D^u%tsVWaQsUUxwm^usUP#kQiQrtmEgy&!yq%M!%|G`XW;>Q?BCX;Fqw&FB) zu_K4zphIZ$=Ln7X1;V&))q3_>$8~H%Y;7l@@9y0u8$CQ&zr|zDD$27!7R3<()R$t( z)Fx=wh{I+cb1-qanZkd0SlstJ3QELe3!r3HQ=ut>gEMDAhU2^Pf*9Blj-q`~zO8To zdwbrrZEJaZKRY94U<(#(iRuHVgWS>*|0i5n z0qO5q6-6AnlWak&)6b(OR1r0-B7&mn5En}7AV~^b3NEw(8T5Qjb5&l#z_7yJp6%Qw z6bgl|iKXcP_qs8WxtYFhm3kLGmR>|?-R*i(eJrgO(^dnUh^j~%|E(>?!-qLvra#`5 ztM4WjyJrbH9s*C^!3kuFhZNTo05=U?8R?{2-sT(C+9t=Ak?7Sa)&WSod$evUXKq@< z=djP-n#`f>(v9lCl`VLXCMoGVTFc~bbBX3=$*T%Py&5od+U3{7NKO7lor`Zq9ZX`K z2Kl;Pg$-ar=FYSMw-Q{C@f(bJnTYPtIzje0sO=8br(|To5!PdXV!+tNWN~JfXC^&b zjYB-2{IOXlXe>WP=CR2ELSHQH*Sn&yl>3rjV&9~XJ|QI`C`Bg!I;Aj#p8PoujTz2U4yv-?Hup@~ zNmKu_-F;CyEW(Q;-0rqJwSyqR4539HX@~45loEm@d{~a3B4wJs#4MePolR}zU)fYT zyaS&4vOy)V5<)MyNwl_S1D<%g_;^I4z0F>qwlJ<3oKxGMo5!U=*<^a2!iH_vr%HE{ zk2~d}_13LZ|33cb++%)M{4>m5v;x-*RJ6rYX)*S^{WCgvp49K*Xmeq-cZamix2Vjo zc$*~R%Wv+1xmN~iK#u&^sbp&oQz!Vx?Q#8RiThY-q0CiiGwKn!?XAaYu==!lv;psF zux|&K8g2U|ox|vEh|xiVxqRN27S_gh#+EjemQ;tgEXcP-|G&S-|MM@3I)7-vV6%f6 zKmGL6iC+oNK2!#kAE=}vFa-7d`SUm|vw<2H7Y70*mFfNJ>cyy0c&`P_0fFmzTdw#J zi3-})x7hp)=3R1Z%97 z31>x6;lJAgvs4QN=Xh2_knX3S^?uR~{D`qsm`NY#>Aw09;(gy~fibrKqt((ZBSc;x zEGHIbMY*J}jW^~dT+;O@Yb!FteU<8x&=Z{XzQNc;9p-6Ij^xcc^@U32!9P!4W!$p^Lwz0XJ`wb)Bo}-%?(Oj3s(p{)?-uUi{sxCM3yme~Bx zT^h*yy33b(B{NiZpDsM8?@;$lr@_1CSDoW(O-|-@bl_eWh~3T7(dt`1!1gCfHMKPo zGOMgzgaViCND8uc#=r=@yBnNZ3UmP|PsVbT02;0@2X{WGDdf#F2p54DzYX#Z%=oJ@K6<|I{b zXxERNwum+uUKG})y_5b^!wIz?F;3_#rwQN<0(`g%2>UNoq1_l0~=kLEfpizG} z{TZo>1`^=dC5Eu!+<}ZiLHnt0FbenC^5rJ=295Mhve3W&^|wkP^m3J`5eO*~o~$`H zdkY_W46pTq^$?ZE<1Lr{fI~ zqa~E-YedGtyo`wK*V5AR*XKU)HlLfjGV$|AE6+*2o??Tgh6b0@oD~`-CTs)B<{}Iv zjq^?zD}=RV(V9db3uBYJ*+ujrQ!zCMDxy43>(Zlr=6bGf45 zLFqa@56F~#vu0#XE*5oem@NW0u*ZJ#-x4ac6~O5 z!8!hT_~hn@o4btQz%qG_@l}w-X$L`jyP(Ql`SmnY%jyv8{P2PFR3ZR%L!>0h-eKbj zJ>_!4}wr7{%5r`F2kKl9@3`2M&yP`GM zH*Kfej)pZ1GA+Ppes^&HOd$Vs5d@)|v_Lozuo_6KI7K;nKLfHs@Trv*DrJ)Lecz-- z3y~B_$MPuI`nb@BDQl1|V*PkX`=*H=aU4ync4d1Rfp5I%Wfntsj30YfD;-@;@@DOH z>5)|$LWatQ>iS4QBbTY`Np8r7ozxi-+Fir&B3zYqThP!>}TNy{spuxmMKqKRg>u(Tw!ah&cwu^i2&_e2(YVI&g$Xkbcma!)87tP zNaqZo;lFj=H?5Ow<43E0l#u8{bSf)SDDlU*C(AwHDrq|tToTh zV8JIxUkb!)lIebglGiN%h1N03SEBu1sd}%bxlWJp7xTVyp~DuHJXRQ0%Iv3|`Lx|t zvAa=ryXdNltw>#>GTw#=_<=;L&oF3g;7bHXo-_Xv%~yrO)K27 zFJSd&H#Cr|eWK+LM%ka01EtNZWhI2U z-E{Z!IjidE*mardkT&6swdy7)B+_;!NKELaq~<0t=Et4Srp!Prr{;(T47lY788bUm z;dVMFIz~Ysy0J>I9E{YH8(>{{%RW;xPflwi8Q=iDpKjwZ;e~S_v-P#@YM;&u9ih+n zOy9|5D8siKL9-Y7s1(lt^6ajLH3VcfJuebFbL>S^jjP6VlGvT>%?u5lk}plH5G&=E z9@&w0x-S#wG>@*u-3xAb5Yez^IJiy{?{7|~`dgI}`U&bCu1Yx*tblia4;Z;D>K(f- zTEpN``l@g*V6t$Y63EYTHG0V*1=r*y=r?}gQ}Sg?IqP||07GB&64n<{BtsxkEHcc= z$*giQyw^^jO+O{YQqR(63TM&he9OO7Mre{I_i?gmH2+*;;n#`52ag{Gm#9K6r-G0N%Id29+UA1=<;!OuD73R3=Hlb;R zA=-P&&4DaXG*3TSY2rNo>gb1b>{MFJs_L;L_aB|naK(y^$bHw<#~hBv3RyHB+7wlD zXv_j&XeBM6e0#42@P9Xs81E5-gBQax<&$(ocG!{l{r7)R$Lv~}C0(^^WM9gDKz2QZ z7k^s(~Igs$cA8xewP2+7D%Z}Zl{XOX~vJ=tK=Af{DANwOd z!r2|QT{$0iRMu4C0B8<(985RBySt;jBGW@$b6UakXGK-9VT|12B8>15E zmz06MBOcDqFO>Dnq{ns`C=ZtemwbPX)o5ewLvy!Jp#RfDHgI4iZDbl`bk^92H$-$< zJxXZjmj{`(Kps$F5$F4dKUe6J3OP4!+^7&e2jOm?(suB;S`H`9^!24P?nT%CK5TV` zI^bE18Z#;?Gf;VqL!Q)gvaXnQ!;cy&_^^Ke1_!8%ME;DAff}OWK~d?}kt%uq&}wcY zF2iH)AP$fo4xcF3k~)-M@-rtnDB}UM1VKFyV4ay9@HGfEwu#dlXWJ zq!do2#$V+$krU_92m`(Inyi`K9y@7QQcGw_pws^lWzH>XLE!YroLRq2@FM<&RMGmJ zOqd2A;wMOh%aP9Cw6f;&$6yLk{yAd}flS+@B9gU(J*4w*PsKn&FONK9N#LEI2_8p% z%_+I!f~G;!Bv9JJqxz*%(XVM(I8j%sJ?_ZQD>de;sRjkZl%p%`@DR3E=c_I!4lBRy zomqSO-`(it+Xv!|$^7Abj`w9S0|_&w(wHeHAsq8}-#B4Li8cSL0)P9>{1?w?_~tL( z9!@4xO=wxZeEIpYTFW%+@!9d&eOU3+|Z zcMq}6 z-%JzCNljiuX!U#PSkQ#Q-~4Q1+E9_=LYTX0`ts(N?<@pXT*$~kZubWjvW*k)M)Hv= z5_Q5aEg&w|WF9`NL07$5w2^C~HKAD9g!;s$ap>x}@NshqEiP?!@hN0HvQgSe5tVn* zDKybOHCfsE8L6QYQ1;1xUu%g;}3A4@$`Sxz?bWahDle78YBh{HV z(%tZ&>YkH=)(+u{kPiJQn=5SlT^M)LNp%aB6bYy27}eq3H;?}rJC~fL#=8r#jsL~* zeDlrvS;WcKeipv+J@o&(n8y}IM?sw1x7VU2s>#9N;K+A%rel`~?Z^j%p1hz*C&Uf~ z|7fM|UKLl8M@3z1CYl!=lGflSphJaHxkrpvvLSrbsw&7SJ5qM0RMeFX5}C?e`8Dpd zXMe)M07}(wC5@-_A7s;dWn1@iZw0X9nk@-~z3TRaz|fK=^k+H+4gO6|$iahOWdu0L zYeLd=jWpRcp~m%@ot2V&18*?xEz?OvCz`;sHsZRE5il;Lgx1+a38uG6JjZ>V|5%JO zs2RovDVY;(liN4g$>lCOrJHU|+#u}SZ}byNC?R#{HSX5>SKAZITgi1UoYiDpS@gtr@PHyYaF!`^^Zot znC>7^Wpmd5DYHUI_7L9x-V_JO274S%N!@x*drF>`P3n}@*XGOE?jHv!Wd;vwGx$g13n!r0VmAyJ{x_b@ZRA5VZw1^ zb$6o`7-7g-nPcccF#kDaXnzSqGlLDVP(0Ms0E5pL6*1=5vU3U0`kYVbktXgQHm6)9 z$YX*f3A5RC&F`ecSk~6l7+QC0)J}YPt_hi)46iMW_wT5XK@EdGKaVMbx&IC}HvOt%E1Q1n!*40>GXHYLqY!;)sXfS+u1V;oO%i$0%Mq6pb0&fM)_Kh|_+T-*0kA1_aN zK4t3|{Om)I-ET0uHa#1v3&C$0nB?0|L6&-E8}}bS`M6jle9%lQRahmL>D=B?>iaLI zSvQyVhLsMh*-WOw<0!F&&U0)H>64^7Y#l6!5gv>qQXVWTIliP*({`gNUojao6=1P` z>-0+cDrt4rxZ6MG4rUn|eTIJiylUyf$}2p-m)~HlRoHh5f0D*H%4xN3D*k7Kg&f7V zal$C&&%`Zm6yfLTe`M3P??*qcSsXQUCFY@h`_p@SvHo8xj;#!nZAd;DrPpZkjHb=d z2|N=DiKyORpG|~E1J?mpP&f1gHh(MYU33^i%W(xD*u<@0zaEEW1x!vx@QSupy- zfy4rr)N{+KwP&xLM;n&gIh!rc9>XYpm)_x4XP>cx<$T;Ir4ZcKB0*|0ScvBPR|2VU} zb;XJm69WcA*;;Gxd#=Y)hfNOFMX$oan7~tCNT=I#>!uts+@|VBnx%#Z8Aq80jbW$D zy^yBQuQ2L;Kc8M+)Pj0SqCrNr8#7SN;XqKCN!Z=GTUG8Pmsy+h@ zy(4)tbJthDjx;YT*(H^xj_h&ZrGk0G1;&LqXV4?c6|N7;FY@CIoavPulNKS+4NXn~ zQzTxNvozDSf=s)r*8~f@e8Q~d>)M7&kel2n3*2u3U-b`uYa11lJVQIP@ zv-!aer&wL6h)vO5rd-oMKzgf97m?2Cg8!%L@~-y|ABdGboda?fDwl1Ff61N{z=JqEp>f zpsSR&c7~fldhD0D7)p2kIjK*Dg;H5sVI*r|gVNndr>!c{xK@^m`ZX#ZAcoCbAh}~J zj_WMCa8po3+`*O@>gfo;@Igy&0CBj+FsjLoM=0h`)ZT78Sf$c{k!#LLL*frTPW=1XWA8;mq*s=_l~DFttoCPl<-?GJi=XIk}8SbO4!$<|5VFX8Yjrkq{<={t6J@smF1~R$5TpwJRnZ7%bT$d ziY#8O@6-<~vpN&MU*@(&c`W+GQ6Wu`$m)NgTDnq;|25`U#qf(%vElV!6?9s}ul+Fz z(^Kmhw>q^_y^O2+4x$P^O8k%;&#iLdT+SVL`vqE-Q2*84g{#Tzro5<(y49VDi~J4h z#S6dhsE4tfB$%_}3<_cHD_!ALUD`Qr6kDe+)W|EbF-Vb?$NbA%XViat-wa%M_d|p- zt(HMuwD?yJ#~Muq@5R&&;+oM)Ml}e>hhU($3&Q}jM>dj%|r*79w zGSsltn|b$P%U>@GDSCu}K=xvu_D2Y#R`^h*^JBUP?)u|1`Fe@$8^%n#8b!t7P#{4_ zCNW0=?&H10n;8M{K+}82g5*Tva!HcMo|R?Hvtctdpv9Rk5c3cLN)>3VIbLMxPsM6~ zo^_FGTIa08f-@YMDfL%aPO+;;Cd*Q1>xDKT242C?3O12N;7%gO4GH5)G=Ef$=Y4bei>x9awe~zGvyH1tWgD~4ev_hCa)QuWN2JbwS}r9 ztT?VVHWJH9yqM8WrDrZv*9?{j^qoT8M>s@E$IXM_dw&rFuuH50-+`mU)+d6qlJvlL zvmdz*lf_0xDR1?}SUs7e#DXux?qU~D*Rv5G6+SL&gfIu8?J~sbW>>l}U5%=`(3r?z zPQ!rjutSgN#}09-xy$IEa8!j8t*Sn{Z){JdVyV1C4vRZn0ig6l6#HB2tXiW_T7skA zg$HuSn|I{PgHL_S*ZUKshGr`F!~77!Uh+y^5g`YG$Mfaae$I9hrgYTgYb$Rp=}gi1XBDc3%o4);n9*}Y_D!`QvU69QQAw9K_BjeK zcXVv+HjH(q4F#3tb5?KMc(K*7WKp^lab~+v1MA;o+8(uKk&CV_e@o+ z(1Au=r!duzrE!|g@;DB0G-Jj&tYFa`H{5-btNnj1VE$i!^WRgG+FR&$NoX1-t1+w^ z>VuA`Q4bA!v&Xa0cr7|iB7L0x(aNn08rPvAOI-PZ_LmV&*AgxzWX)ZfQ(nnwUYx$f zzW&!A90ILj;PH5-$e#%w%Wi!FqS<)3|aPKLl`u)UDDc2aYhicJK9uqK4B-^s$0 zet~)55j7fb>a#LCd*!{|k5((l$!n=l1w7U~YRC|yY%7HeBu|>4mlFkJh#H^RDaff( z=QYHBe0x|}80c13Yqv}Gh6u>4{_Bm+n;Fh)h7oiX8-pmnR!{AxDT#K<)iOurWV)$2 z?l5M}fc7exPsyg_%`m%N)T3GF=k}?xf+)I63Dfnh9u>|El&cNM)^qUw7YoLlyB$yt zLWp*{qr{pBgVX2cTlwU#L{*lmKP+g@I3zPNNdjl!do3!q$^%t-jdMJ*+wSBO$src8 zy2t<`?}58Q2^Y%96>obiQ|W7NNd^)2>QXA?d)&6nVl*0V2gfycVDS}*s$zzv-0_{) z5R~Mv)OVO!;mgU!0_U)_r;do&Ypt4^`>1NBjY8SLIpE3 zUxr4Q%SqlYYpip9wRaRb(r-h2V+$IarJ7tFUnJXIt|GhOpGk>KQhXYn+~hKL)gOl_ zt&O3+5>iqKuRHTLyLaqh9l#)>CCbg>tkjaBtjsJQ*U~#e;6D3o;+89NQu)d%D>J@+ z|C}$4pIQ6jN^GJa=Bk<%-O6*}@99|JafT10`>z<8}JgU8|Y5FSJF7_CYUxy`WZCrSzpVWofl=A9Yzr>ZnX)=mh7d?Z9(XdBCdO^e`G%aPHsVvkTL#z_?&YcNm7e0|8yWDFsl= z4E&+frSc7bR$#1iBnV=IlBcOSU){*q0nxrp5!S8wa$qMIiz$FcjX& z0i++y6s^52CAwqUM{CD&MQeS~;+$l#qPp{z+UyQ?>yQYYMYR?j?7Ej%S<|$%VbllMHYx8{BeKsD+ zf<0YAsp>IYFofKAQW~Pi+@(@4k8-BBt*X17AD^Sdo_y+Hn{TJ|YZqq#D> zR0U&GwHIjjm%AiW;cK%Y*-_KNsE z%DFoP0G1A=M{65+W7qT4z3SmZW6iO6Tc$3fZC-3gRDXcO7DQ-g)7Y1(X)G8!8#Z1= z%*dI!L(5cZ(z3_eXyC-1#}toz))X0=7kMU!aWs^mOzW&EP)B2#w=;}*$@Jc?@>X#W;P!OQk{()Wa7{VbYO&Vm9B-9G{;CeT*rg%O2{UN+qooh6i zvVvIv9Ua#Ad+Ops$y#n|NTx*U54S(->6YoT!J}Q}?My3+$SL6hiyRJEcze6zM|Y=bOf1m+pfTs5m=R@f8oKiL z6Z!I^(L|~l3_Clue`x1q@3p)fsZaoy<|TWe^E^A18P@bXRlkWvbQcBlwBpH1iL)5cI}Lp5a3j++5kb zY2zNj7r#^6!ZH}5b^1{V5>{{yawmvog!c0 zT3)|CRf!jSxVZ&AD)UhR0Ol($&@u57Mb_}b5&G@bUif9=0v?Q4E7NLG;_1jG(HgzB zETExRT&VCiNGSQ5P-0^BaV_%=f#O^&+5+QHAkCi0oY;HX((0@pCc-*}Xmc9{k+$R% zS6F?SK*|6NNpx8WAZ=9YARXG4OCaskR}3Z&Q=MvzWm@6rYH^NYhYJe})9cvGTm15z zT?G#15iUYjVTZ0Vrd_F%k|Z%Whzpa}W`JCY6Eb=fjiE=bX|mL#N5NwpL12V;!Q`b0 z7ExagM>$c8#G;K6 z?-RpEog@8igrOTtDkYxvefk&m?+Fc|kurTu+OUs%tYD&KY0m6+gz5IX%7;{kcW|J5 z_Fagfyc1(vR}`QMvdx5~4bGHBx0Qq>f;G?^oB+`K*WjS7xpYKp$%*EH#d6zTdRt-! zk+Hq2y*eXom{YK^}e|^(tJvBrwl~VL> z->AJkxXR7?zYa(>O6hruuuRZ5J=7EXhJw1ri^G`=d&1aie7f4_H{KSZ z)21#p_pZlKo0nMrezoi1u)^iL{Txl%WA7!UzmOj{(-h@C)a-ZL!^<{@IQ5otU{(4% zpY}!ikK|4&ZMu5AiDTnYk!+;)cWJnDohhoU^M&%Cw0TN>`8rNh#@30>4Z8y0U$|KYiQ4r)Zn3=b_m&Mu*L`#vera_S^gdsYYhiM{QirK} z{o2P@Ji?OLt&d_wkI zR)RBJ8`YfZvc^?m#$8#_#o^3N&d%?(KnZji1u=YpDK4soqeqV%x#^r)l$GjWZ~s}l zbdC{^=0wQ%4-%DGJWu}PQ#A-EO~JhwI=Y$EmO!RTMK&DN!)f3aTb5Z6Hgb(}slKE@ zsxHc!xu?jDmr|YEtB75L_rKrv+}B0`lB(n$1?TRl5quH2mk7^68Y&9n=UdigoTExWI@7pny!jhO{TlO9=7jYFcQHe8PvuQH zyIoUOonv-4BP=7xfd`jNk-$N1#8Mj)*@{&-1jJZ$3Rai6Q-G2;g^gF^2<7#&Gx$%^ zLCf$_I+jnWR|&1tVdh$Skl}pH_{dJZJ6KAsv*Mi47Ud`#=;cHX`xOi%zG~fV%#D{w z|2|Cflsv>F*02{cYuQjY5iE9~ufn7SJWcy;RD^+xc89N6V=Pu2!C#2(+vEmdJD z8c?ik!NF4N@Lqc*bTTo$9Myv~Zbbis|3KIVst`X~hlGTZ&5l}Y!e>ZA&$=*ldDEFMWKZu157 zRu2jC*c+mzq{l<1AUbQWzUfv4gC+5uW7c_V$T}${d%j1R8rOQm%E%Yz&Uoo3$egvJ ze@vl#;8VQ`hj%R?8b{yy?vu-kdCUVPjs%+b6cQ2+H3tA=KAYX0Z{KW4~>WC=}C zpTm!=sC-bTbH@fcGic#X6**B&)4`IrGF|uK@0tT;@3a`_t&OJX;PU~=DdNLEe*E}I zo=9EwH_g~w53ISWd59nn{n>f_97x4)rp+mppHB2r%6X)2E2(-1JDRONfkt#c|)0Dj`={!nR!5yqQxxX?bBZqn`fa>Hp)kf>eM-(o=g*a2wnq5QN5^I^_Zh5c% z`(fWc1zCf{JL{p5+To;Jd&Q>~2BvtE76_LMDxi(HySg@dN@(0TH0(}>B)#QQsg9zb zSHk@nP@My%U4`uK)OJ6jKX*bX(fb+W3c!HDc}?6P`&#X#zo!UM$zi17E?$|HmF4LU z`cD*%UBfcor6}!knXM^gPJuG_gb^lFU()(%C@2*yq!s#oA#^Buq<=5uxw6T$eA$)g zTHoL-IVBnL4r~U;oL5RSVaUUmx?c$mvCr)j=cso{b;_?n%9%Q_KO|Zy{|6G zg|cQs#8p0Zu;bwzB=V)A9`XLVCXg!h%KOykMw_*g zX0lmI3S~|wFiD2u+Wb(FFqFj9mbkK6BcUbTQr8}->RV<~q(og}hkhslTSfU&ShPYl#v|R# zbv8}MyUNKba&3&GO3pPHxr<+_LR0N=-rE#~@vd@fhXjGyRM0oCf}X__{jIdz^6;l& zwQ4J6(9zuuR?~exE*rF-VQ081^QuDCqwTK3%?|s7)18UzfEBOS%YBm**$%nV^~BF# z?f+PwLdpDTN*lGlFr_b1BHv>}^DaysbCa=e%xvtZ-OrL6xCwEE^7-*RrJSJMgR|lU zEIV_&FyR4RxjXiuU4D`S_t3u&^V8w_zQ{PCtkTS*GPguQ4>LsXC~FYTzuqo0Yq6=& z-1`kHt0qil%69yxEzXs#w+=K7N3TTTz2SqG{zxlc2&+epsw_J@yD{1bNZe(J>HNuH z&7txNtxBn;WqOC3g>Ofv2&vy>CDBfScvsg$hCTB2yacdk4bz?_)(-YClx<;5(Hd*A zA*C}0{_KAb=o34WQy&QX?%G38$?-l5bMr+&1d8Q6y}_R`-krg;F)-}Kr6U2PCySn! z_w^i8`Xm42|NZw9uDzzZ1Q9XsAz_Wr%N${Q`z)x1Kf?ir2`a}4C52%pCyaTlL%%2} zGdxeeIr}Tlw$J6d^#_Ar-Lsxd6?a9CpOLErU#gbC1x93QL#N>h{VW}=Yh}UuJndl_ zcU*Qr5h84GNHWJc%`ZGpYLWH@PIh}7|GgcJcz;+o+}vuJyb))uZT0=!4g1Mi{aJY8KQ1o+ zJ-_dZLAyXUd)&AvrsfdlOE)?E{r5LsARpcfIf~^&XqOlE6QF<82Gc=+a0|5l<7RVz zRV-@)PSbUsraB+7ED$Iuhw#j8u!LoAZ=VoCpRJ<$uPq#P=wf&Kf)ilTMi|;VT7#`o zie;RFQ}35;bM5M$*5QF3{ZVcN_!55c9fyW@s>*Vyc(&~9;FF9d1Q1$h^dgM;!<1PK z1eXt6@_~|5p*$93m-xSbV!ZP0<;y(4_4Ce4!R%*=yTVYmeR+{FK?OkSM>APB*nNoSmJdk-n1fi`pGgBrmtvpa^ zx$>0fBT$->8IqWyqL7jhDk35vAn?6#@5A=}eZTMT=Xd|n!u2-@fj{R_%ImR*<&e{?4Z8K?2I7I52uI*RxI?s6*T97rcWAgniMb!pJTd zf!d_x#yx+@;yy-u(2~5!zjG&>hahTCSn1I+4jU zDO@p7L&yAGhN?Er=f36L^g;C&OBL(+u3OrF+-?wz9!3EOG{XF&XTdr2g~F5Rr*B{A>jLpo zuh;Dz>ZR#=Q{$6}675OK90441VsTp}#U z)ZF|Ny+3BusME=Le}w`p2{d}rW2^0o7v})6a$nVJ!0R>={^ce|wC~A}mrZH(5UEcA zoprm|ep2&DU*j9-{f59-x(gxV{xCErO8-Raj80#qRBf7y!mT>ekgtrpE8g^JB+lf` zCb>ekm+Ev{dFRcZpF#AS-qTk1#TCe!(T|?_CY|Aw-xg!@oKb-h9pSFVUatM!y;nYb z4=C?V;a3)}r`}a+!|ac?^wUufd{fl3bh)Nbe6lp@!wgqRaZm(dQW>LBnyv=!TV-Yp z>+aN2@g|c!HG%d))U{Uv(`F6c&E?@L(rE~{2cIFLybKPy@{a!bCL`nLrVsx4i~oA= zSo|OK&@0`_CB6Kro1J!j@;M7z4J?*zi}f;QdhfMm`25FLRskn_1O1yiw;U-;-9?c)-`)`Y_MLus zJv21H9;i;O!fj^L+S}Wc`anNBl)t_d05dA5ap}HO-mjE;k>O>)!5`k4U`sdOk_zzn zowCi#`(SiEeN%+Qn_dA<7AuzhwmnlTtLz`c(!H4UxZ2=9PjIH(O6*y2y8iP&WRme3 z7XA~Q(!G4yQ(LbtXQ0>n@=QT9c1~%wh~o#uTzwZoB^$7q!jk&$Abh2m^Vr9m#Eib1 zI_ch?FX;1q*LqPlS8cC?m#tw(Mh0x7Z?Mnt%NS^*~Gf5v+xdg<=}+yd@D zyy?zPPXi8c`&kK`7(RbCGh?%?F9IrkgE_#YF})&@#2N4b&sopklA8il`n`dRMzroE zrrbspuJ>BehixDeZE&HH-3cIMpIrfZ2rHHil%ImFk`gwHc__?;88*v(eEreX#dSOa>q@8cA2KWDizccfsB9Ra)-6qsxIQbIyB6 z4m;}rPGQwD@nW5#1hbsJ0e0ZWdNm`qx;eI zxIH7yhA!@`0M1Q<$KoowhS_;7T-Pn5b3Yr}HFC2!DcVE*zOq-*w>q^I=!o~M{rKH8 zr-~TI57Q<2>IyO?iZi52X+jf+lMcG6=No{sXtd2w@DsmI1d7(SNV~d>5 zuJV-!6kO2uv6Ml5-16ZtN^R%a+VfBCTklsXX&-CX0?K4yUh7Dqcvq*~2_YZ#m=Ru9} zNL?md|8GP3fBslPkNB5lf??{1{hdqtG1h1_%Mj5y0MD_|_pXXhfwo^uFUXO>)~O)5 zbN&~6Xai+#vLhH$ z>MsRf{QTWVA7{fK9(%E^ea-j3eDUrIQ{w5QI{q=@%2;yq<&$;1XVW|0)O{#hS9!Vb zn>~lxzx;;z#gEs2eE-#FC%*dWz|Gy=-IsoGcMCZwg8=bxq+b}!Iqm)JLb|d|7f|{L zXp>buE($91c}r98n34)hNks)#gI*hjkS{(Zme7E5PX411Hw)xlNMG;)x40m7P+bmP zUSrd9RBAWBWfpVdeZ+hJ->wc(?~uHvwKPo%Pz&DzKOGQ|4YjocLKG?F;LG@o2fso= z<{Fw>Vt2r0i{=<^3k+!I{pT=iozN8XG4TKqkOdWjuIiBnn%bwVtc+NqXVZ0S?<+n0 zrDPUANq>zY*4}h*+BlzcFKsa)_lO5YF=@obKlfMcAmOuf(hLmFVuYa?a)KNd(DY$t z(Mm_(6DuL_USqG+h4YHyBBf4>EC8EMqDSKd&o?@*ksl)dqD(X%OHT82s!R4AIwuWL z*oqrhp7)q;LsxjUXburvnUftN0oA(f)SQUoA;p%@v<%E^F1Bh{deYJeme;=g&#f33 zc&(|jU-{=R{@eMb4fWk0fghYzmrb80E`VCvDWErAO?_T#*7uPA@`yC#yU~o zy(c^1%mkOCgK0y-8$4^iFza~`Tz}dY&b;7oI5(eN9Gpet9i5IS?T0sbx!>gJ5xZss z3tsLX01;ye2hXxwb;!SweYAl(6K3Z-h4&Cid)I88!pZr@RMs2Fxz=HJ9 zlj8T>i9U_nTtE-c?=qP@8?446`XLk8?Ad+3GbTONh@Uk$ps-qt{~${tmW*Gz{hO6) z-JkFB-H?&EvlDY-orf6B$bphIB4l2ILLr$TKh(c7#F7a2=>Q_1su;nNjKPJ^d^q`k{!=E19;0V}5b8OJKe1I%Z89c4MZhkxv&#Fyf$g z0o^o9M!&>~QzUBRdI%7GHvEy<}p92*C*%ISV2LKN7w^W);&tKABHoX9$Q0;naoD(re8 zk^?TEUVZM~!29;q(E@6-Ta}xG)SlTVOdyKlbo?Mht;Gwa5VXJkbiBYEkXRgVLo5Iu z)j{F{*mFBYOz5#~JT#JQxLvOA2!0#MOD*H8F?7XkJ{j@}xb_{4bHGu2Z;PoSif=lY zCfN^ZV&8DR%oaDj8Wgb3M6!DzHG-d7J!Uf^&~3Y`p7Vn?s6`*6W*Q`TM&r(2tVbHj zNh+&fQig^z%u#i=MdT1}aej<+)h@=Q8$W&#Tgt2~W6fhvF>XKRK6)+9^UX0ZxP%Y1 z^;JliRMjrQuh)t{SuIUh2s~o%jG#V^eND1~r!b%ATfz~Dr?{oF>MKy;Is1HnmtGsU zT~8%_(Vf0$X5^Cn<|}`uuiu9x>u_bay+%f>?wTO%IjW*YCmxxNUP8UgLJS>dN&EBE z9tL+)JO&-{^2&3B_EiS#)6&*eG2+~E#0~N0Xe{;lh>ow=>DZD-S6A>AUro4vA#qft|)(SJYvcZ`}23&^$cKY6R1Azjcd#@3(io?qoL zw_-UgH8ipv-$MvAVhKc<4atUzhHI??5F0*mS}`=Cf}DHH`(4pz@mpAPB<2`P_DJS_ zz^!lRfhD>qu1L?0B9;VKS;4=~JgB&>4adBFD^^&-0r%|@RLJzUJ35ic%NJ(w!fOLA zk5wbyd!0eqII$T+)Zwgylule6R+_Szvd_1XeG9rKts_)@cGsbK-JAO$%z)~3(xM;o zz&vZ3AaxM?+H(aDT`TP-qH?Ryf=sE1-RMh90MeFVHJBaU4nG|j$ac{E1iJ2Ee8j!m^ZLE>t2&kIU-q{6wvhK`?xOgfF`&})U1|LLncVm8q3jMVo#Fp?p?a2P< zwtkK^{PbN4{sTMZv_|*?w>3m->zK*cUz|2%rs;m4G z?_a)SbzsF}@PYB^esQt`mHTF?=LZxp4wZpfN?<2PDbYrA}5XR{4M%~#Q9-TYu>_iAHI$~b`T#T^Pof- ziS{qpRgaEZ&^j+;8y_o!w*7NxZl_VV>KD!b(eba@`|iim%IQ#NhLa9P>8be*8Uz4S z#1fkxD10o|pIFz>kZYy`8Ttw0+l%!?&66ja`+v0EQ|My}CkNX-*5!1E_NyT5Vv5(k zd^&Lw?6LR~7m5^ZUc?0;lm;Riz;AAqdB_RU7plGsp)j*LHXPF*O*)@?UH-6H3os}- z&_tKQQ0gHDNZa%4yyLV@mCw)qT4KjEVsFu`!=Ldl*)nfGNKolW0pj=oiY?P z6B=v%wF;^c$)fhuP@7>V_HKKZa#gq^?EL-Ie-M8QlI~?!)pxX4GitA&OC_Hje^a!& zg`}ygz)hVHq3_oy4a=J8uz*I`NYo1I zC%NZEtl{eEQM-KK8cbXCv^7IKDz`gr{<)ojL0}2{Yjtwxs^cMs7>gt$L)&b(RT>|YA9*LrNiH@b}~ zzMU(4JRvaExu17wF!F58pD~$c$FQpwxX2n~bh1sx{6F#nju+L~jC@j6dU94B_~Rx* z%$c#-_o17g22JB$EdV1g*pa+bwgQoy-rR*8l*O^eb z=P{_LTi>>VGXVxP>uCS6lP520FF3ubIWmzxdC0%Oi3wrvH#3nqF%-aVewQ=y45HTE zbZr{3#HTc;#IDl57jn)%bzN87w^;-cPjZS__w?yEaF>J+AS5~ArB0S`Brs~qS>&Z` z;e@NlUK56@?qq$b{PNuU6D9F3g0MT*PK|}16RL-KKiBvo z&gYYovY?TVi)`mEqZua2HYhd5hKdlOq6V?6sQ4@>6~d_!rdiI3SQ&%Ak4i5YEF z9&r!Al_#JUQ){Meqeyw{`w@Fy?E)`a;}f&ReJ@=n8VH!Ri_gN|NdsCfe6X6e)2xii z{Fnk+ZZ;v`t*D*0cMAlSXpD6;TDe1KUjOx@&o1_6WKKFu!^91#?ru!8+Yd9QdD*I> z{`jVZ(HLFFnN+<*C{^Yd?)#fs78^ND{4KBR#|*MNp(tcy_N^TEq>IHs#_;(m1xSr zG_G6OhkRRan`a?c%TYh^QH7g1$>MEhv+67V9F$##)SdsoqxM;lb8Lk_d2*Qen?R@& znoqM<&BvI&0w`UvPZlCDFb8UD3px}Bi?g9ZfFd|Fm?liv0l!#(di3M78NY>Xhd-Hi zd9158V*^EroD>Z+kK!X@feBJj(LiCcI-o#3(R&|sYx#Py>=B$0b(iTBI zD;q+`W=vj32IF4H_u=i`rOf8GHXHazb%@txiX_Dji0w z`18DdmVHA}d{qa305KY*7PJlGZqVyLL9cktbeG^yk?2ZuEoxDEf_J0W;nrun*a5bzvHpxixiLs zeq#VX@cj=fy@FWzm5g|`KK(c{2;@s4xzJ2n2%Op8{u4BE&mE|w5D3b2ODg<8(ZAsu z^zoA&dmuxbk0CJRBIx#>niB0n&8$KV*JSRR5`!FfA|q+bfwtP==>7x;mJ@*>rAL*v zjKvZ;hx|bkfH`o)_l1dshA!=otf35RJ#m=_i=jo!>p(2Psbh{H3t(VtA~dxC)_#f` zO~_}Nuz9L2(T_rcTV2H|`%6k6H{+!$kmBY{N6$i7>y*PYkcpPHc^-ec-+jj=!N=S1 zMq;Z@wQsc+5RX8;ucRY$>m(LG{45C`P;+%SI8!Mgk)jRhiu?MDM@fbTjvmNcj)-${ zm)~^hSlG^n2uI|>oUqj>b=&nj!0DSHj>`9az|ua13g2ElMohrOJxiLNxdk0_CifJr z<|I6}dL{^2h^DHlU>_vIDma_pnSfP!4QHV@H9Q#N1Ln0kbr;6sv~1=~+x(u^R}k!7 zXygYe+}OJ(Zy?v!?jJv`Flgc?j(FHP1ak#a%bfjDPXjM+VWWVsvZ& zF5)tV7AaY}o!LsoH{n|I>otc}*L9h7doVy zJk!`sMZRVKPN*F|LVPPqvV`*_`-UxK&bh_Gw75bat4RN?IdIk0?^;s{f^;58BHH)u z1PQ{6^+GW3Kg$=LiBFZfrL9*QvBsk9wgermM%Wi-Yr|c(xUWOlI7#d8kjTNv{)Dd7 zbZnN~b%b}wSGq*UBKd*RCOcs3Ml8{b!x2kBie1onhr^e7X%8%AYnwRXxWDw~o}oag z_Ja*g{SQeI6Er_}!c&0p^%l5Zj|s$FJ0r2HrUSNg6{^phK(>IsujFhLGxY12qWBfx z_x03DV#&nFwC4L~f=*kx?d#}*s_`e+9>Sb#xaY^GVH4`WxeB3&+6!3w)j2?y6ZNkmm41*atH% z`Dw@TVvyoCs<3%EI~ckGG!(o?;;kG*HTL9DKP(97egh4-Gqw=lTO0&IlMM0OUMuO1 z>-YrDEG$C}?V~+vz33uMPT3E!5-E5J$FB7<{<(NH%FXA_dSv|KF~~TY(x33U)xzLe zgF}7*h9eqL7;Rmz&gz)2k;RiMWoctsGE&%`23?xp3@T&MHDt+s52b&XSZz$F?w{av zZ!IF!U}x&03y$w?5sucP9T(IOme_dhv)wxw(N|n$aS2lr#~4vQH;K_oxBt|H5-E?jOAeIaCw4&b?rWyTzDHJ{1rMHWc~E-(+Qo z519)U%WX9<23Lk}f`9*K$G@iZP)hCa>rX$#m_G17X{k>Y@@I8GLHG>utuO%!2kFN6 z*`pA!o5TH9cv|pjTTp-ad$)v~8r!riYjuqEQ*H&6JmKuXJ+gHzIqElM^^IC6znsoz4YX;SVA-56m1dI8)Tm%Yfz>Rkw&1n5|Id1BB1EZ!cL1Wl1x*Dhy)S6()ARzS%Y~k` zGonrW7F+d4>iZ1@ywFj|hRFsxsy}p~eLToG*8>qA&RF>b#g0QJgBk2X9vd{#KE64^ z54sRdGf~DhAdK0D0gF*ETlf@yieU-2hH1ZoGj|S45@Q)wt)=p|r_`sH-ZTg@D;tzc zpOLoqE8Eo~h=yDwUWq^Ja^ZR2&iWbbMD5Z(YOF=Efj%Zba-D(fCN**4uJUvl_u`~* z@$sDG7-d4!mf1RcGp_GhgKvmMjPt^x@lrAgdB%}1>dmK_PUo`)beQ{V>^lq@9|Q?# znT>HVFkt_^ER4G9<}Y$hnsw!30aQ5O1DhUulG=DbaG_kiAU^QEITbG`MJ$@ZnSdP? zKK2MtwbD|CmRxz+IPphbTXV+J0Lf{MT>WB*l5;FCQ?q2hZLCtHd*ZK2aq(&2Ed=u0 zB&go6;m{Py52J~pp(4r)mZqvk#8<5fE>Nv}w!xk?)N|Ib9M>*R`B&b80)bSqWDRKU zJl2?hLt*UwIshiU3e%o~Gy6$6zZ!Q6>n+x9d)G?3_|iYq!e`@iE<68ca~%B2>sY>x z{>K&uE2Rx<=n=I6HKY2HP`IA7_%rlzWfc5)*#HrG3*^Rkk8CBD0EI)R0LZy2mUpj- z%V}!CNEi@HG=E{&lZ>mN|sl${5u3t`ItMPT9+hE--DegN&OqfWw(zl%BYVf!7 zioO&eAC)!ZU$AfDi^5e~%+p$X!(I6GJ=I`_cGL@`mBnbtUD4e0tE((Qx5X)43>e8o zC@|0_u^oz`4}BtwCjXn z#)yD?(-uEakG`0;lw_lvb3Hg7FewGXVkcp}jW0AkA;CWjFv#xqm1V|SE^=O4HGH{L zV7OtWp*y9)97az?(|H)W6bpmJE?$5$Cup2=t?q(Ph$Xna^3`=uOFwCCYa*sZ3633T zIH`Qw&zhGt+tD)T=SQm-vX*++RH}$Eh<)_-bZ#FdEvkrQ+oPL>rM{j(kNOU&nvU91 zbq~i_=X$0BS^d>GS6g(;-dZCow8!8>QQqF^I}u3pNz0xbY?ZWvt}9obLN`qwyqty( z4Vj#L<+p)yMkz(+HukHk(FqECYip-~F{RFZ7HJ{!|6v=+YcLX{oWI!jQrvq^Rfwj> z|6DvkIOsQH@b*v{eWZv0YxL|)LOIT-wD`SqLWtY1IHzy;8MOGRt~RmFFp}3?Kg3+JZljLf z_J>Jv9*Vd_s=}OH!+nqJqaw;4yWdYxQ>c0D5t>5OCY~m11;JYKXr86UZkBv5R#whx zxV$*r<6#GH`Cty28$WNgoe*Fhqkw3Jybcj9q}RnVT++m(y{J?pNhE8%+?5SkMhd=G z*)-{nm)w7k`u%ZZMz-#nBF`S*o7{1ItpDifu z&(O|MG#|}N6o{>G)x-cR8GBeIk&T}Z7rF`xq|r+_=FT0Er}P2ysI)E)Y;=Jh2!)IB zG)ryUK#ya+wERS;e%6r)Gk`Ny zh2tE=CuU&2?-8So;VvHXYJ#PL#{!@%OE?g3!xIY>YmmtukAy*if!_05CiHE=e;-%W zfo+WgSgH*N6-7bHfMdqri3#nWSsfzkQ5cHI=8lRb8mz^UhIdX~d3kO{eK)S1G^2L3bJ6#& zHJUF-Rc)=EY`5SHduVS8$ypgsZl9o-LWA~tS2Qmbv{)dUV5Dm145@7LJ&DfImt;i;zD!l^s zc)c{%EvVZ4Nj^+IaEn2bq&QZRmllOC1A~0s{ZWDX_gf`Y`RbRy(IXpYT2x^n+VB_! z@?l{wVG83`dlJ(dW7raji^-18km4=vaz5%mK4Q`6hJRFpjY2CU!*Z*x<#{ExjVdOR zNu_*==U#>I)D4p5l#RyxzO%2*EEoR+EV~o?%+Cc<2|yJo8zy5?wWQLrSLNd`$%Vg@ z&FLwclH|-N64J(+6vIf9FsF~gUzgXsVB6_3I)gh>+IbOy=4}YAd zf1_RUQtB^f*(2JZRc%4_=v%9U^78_YwsHLZzB;cB3l2ovrefpcxn0{I68xWxjNCET zd_q8x)_9M&R(QNO)`@Oqrj1N4at1JlBuY{y$cXY_gRj)@_C=YHQ*4d>D%m^=3*~y5 zJ{@t4+v=OKm&Q;G?$^yd(%m+7MHbDXTC}@d5*O6Q^79*W9x@E;8=barQ^~B0=+-l{ z)$+Req~?15{ab_)Hcf`>QWsu3#}Hp_%TTj2)cV{ zZNofLeDF7n2ddQ%-x5q%YhDvKfA*a+P@QU?iCv6VhXq$X;%$U1N(ltmlVJy;{CvTe z5(mK;D>1jgZlw5P1`Uyu9^_RO!6Ne+>sE6Z{$!uaK_|ml+}TswL^s4?f)#@)4^7ilJIJADjCpl@g^wepbwU)y9udDH@%BJib+uVA z)BuMPs9Vp?Zy0@vn&2ItTpw?f-wB zJnQS|kGW!R;#5m6RNszG0lh0wC<9^1f@qRY zX*X`&M?;jmGFbaRH*veQwZk@%h`Z;C1jLf=P~r;{+88kSBbKak|L}c&ga5Y^>AdsK z2imGVvXBklaoD=6)n+X7o}+YmEJya@vOOfkxt%ZD%U~Zt&46f1Ouq48Xu;(!%S2&2 z%NrRv&77^OtUOEv4v9Fh3)jLT`w;8e+S(#hCR9MCW>x|c|34k?v`lPO`C}!8H9OX7 z3D+wHSiqj8MsvFUPMdH3mBF2h!zo_*j%2m^)IW_r?%ghTx#ABI(b$w^&K%`&Z%hjvolhu zF^;p+*k@^JOFrp-1EaM0Ez>ncZs<+)Q^}Y5h_^_=pE4rr$yHr$VYaIFemW*6wua2}VvL|N*xb|cx~6JPq)p^g#jco`(ncErDXj5{+f zi;h<2bheqg>YAb?$@kh$MyCWNilmq&4`0n91FHo4M(nl2sWZQiFqdGDYo^^~KUg!s zKFVQcrGp?MaRf(h&da||;Sq|%Vs)p=`vnt|%&G5@_oJ*QY8sQj{(#X+q=6|~UPSJ? z-F8zi)>oW?z1*)0_|9Br`pN5`BCQmjB$!`{t61@^GzqrHMWLC>Ww6R>kK!5EwIM}A zxF#-aZRv#o=Nrn)yI~dOw%R+5G*CV?$DRcLb0dS}lki@hNe;N#`I_0>(Xqc~9^3K6 z7N%4(8l`L8NNG8BXAM(|#`mvQ2OB=ZX@a=}z5TqcxxWfuG@@pjOvwC3o7Y~F{n-PA z$3ZM{5uO4SvU>4Fg|V6z>3iH@q0DtcR(-@eSKvF#Dy%Reof*;&wk^qA9DjPeBv(w= z1>HVc;g%O1+!ajy^jLvOIn0cZ>V|&lSn$KCPwrM))bQBG-w%kAskK@q9LE0iCM5q0{I|=`!z676zYjfT?lxz{RKnQfxiWZxq zjt5{^!PnOgR4TIYjaUp90}Gu*6?mWoC*26@_~6o8@7Zrn-vh}XuxHC*6!8Yi^55G7 z*%XaG53{?^_Wxp-$hYSU-G8--1zfn%+Hl0wM8RKS)uV5Ddu_`V>w{3*CQYPv*`X)0QtY1UfLkx zQuNJ4Lj!#~sK&;7)r=_NB~H?5;!QL*Hgh#-IUlVBE+P>Zc8`=5OTPS0#DCF13TFnjl_L<>YFJi)nH{ZgU zp?k7H?S99Md(-Lw##op>`6Z1W<-5PcwU>J&Lb~WzRxebZPkoPizu!i0ahwY!%j>J$ zA0%{1q|9Nl!bt>~SUiL%?H{hw5Ei%Gl=d6`<@SblNGt-|Rt;cbHXQs7Md32SMQ>mL zg#-K0^pZ@)y@|zBl!+Svt3pV5TTGOJ%ZG|4G^*PsMq(PRX}5!5y+b3-4Wx*U3PRk) zc-?KUOXX9^P5kk{k4tU3iWuhkieC{j!-n(X?Z%vQ(nD`+)TMXQt~S{fajWoY0=&>^ zR1h$;2_&Uv$y!52po_}Y?R&Y|rMUsY$oz%pjMnJ!2V8FN)l@tJc1k8yV>7EC)+(Nt z%{af`y27otcJ_}N5!L%ovZMaj!l6HBK{!&>m8hWTqt z@pyGB6mED!7)nvUDZZ23v|%#-;$-!_oH~qntuDOp%n(y_S021a5mLw=!=EIyTvPb> zL&wmpS10z?@?3(e3>#8mjG~9>=3L3-j5DW5<;}vE{77Rgo1cxSN?VKkU=P#lE8# z2r#J5nhG<^&N53=FQWj^An7U&nACWSc>MOYx8>U5R;NHjN>4n^>|x z19iH&a0s0e;?1ht*D#G=pdGUtX;!SRzlILqR3~=CP?|tbe&0Flr3spEn@$v?Y71Zu zB7O)cOc-J;`N|a;WWwkjH{V#SQ^^9)RP&FR$@>MWeW4i;nAO7|-G$KcEYB#$VR08M zrJL(bU_vIi~x9SuHc<5RJ#0fxVcZ@$$lhFJ--8emD7NE&% z+{8-;4Mrsa+ltz*+^yZ?M zjfVi}Spt&21@nEEz6rnq1=gJRQTC*XmMsv{+|O1}8coS-;%n*CHxCJb{n3AE zGhC_o3gA%G=Ow|wA$mC3ZU+ZY9d6n~;b6Z2q@OX8qQDu2-u8^R09+xqrJI1K!9F1+ zFQTPiOr$C|MmPfVf{q)S5a%w^ZlrZrA+6<_%buYw_6{4wqaZ4bq^r|}C?v+Suy<#_klai;!-1r-;+4{&1#gaUwX1SwG{f;sb1}xBC~hXdalM;qum`4a>{eV3~mfWz|Z`S+}Rk9gAJNtMtM6p_2V^jtjk=$|DQ@F@* z#?h8{*NnB2YamxurCMYLO9jT-=hki(q(K=Q#7 z+=-0Pic-bZs=ZI=JtP;r11O*rsu&?xkhPaHdhoYOMF)TG+v%z9J2sM)+*%z$y49FN zQcH0N8-&`vN5FlYt2RDZAj)reI+rigxew)5pQzN|(~>0sX(p1M1xt7h<`1JUNM z%SQ>VjuhqbFkWUURTzbPupSUXCVBUlzB`?rZ=}A|6V;f4H`Dwhd+_Q=PaJo8EO|g7 zO*w7eB={waGvD=0^)idpN*Oo>i<;r?o4q9Zklj#A*VH%*oR4@EV1bz5K~bYTUX{YV zT){<9bsYp>%qpQ=6iFv_=IPIfrvp!V(V3CQ8*Qm zqVBU}3TTG-XGL3Z$^pCiIHU^!@0h=}C$Ug{xomgQ?~M++39%F4ynC#Dj7<(3w}iC4 z&Gov4Y{b4K<9k!Fe(HTSmF|!z9p+aU*;CB(k%~zn&c)7vR<~%rr%3H`4aPcB zP7)exTaQF)-c05pzJ;fVKEZE;kDzjN7JSVQR)Gsi!|V{qq1Rha;G=7jFdA1|E#^S$ zfDf)JUC`&3D_@`8%gl?{Y0f*=F3QEVTG@WFx(OHiin|I?cE z4PyNPVFh~16;R>3=U2ON)$V|K3McAGsDDrlA^~=;2OX=6NIrQo)bT0ZGj$mkxLm8R zO_RUlHhAg!F4RS~_UTdAQBhHFOtl`F2RJM_Lcep(HMr>!+V%v>i5deMJoZ8esEC*m+s?_a(Z08`g( z#&3b;i;|)Vyz`p*`5LJ36T9Rep+dbM#bxY_-raR~I;@hq?FN(#K;+zYj?#)d+9tzw zkiR^S=v&D;+jc$>6VRS$GaLZuf|xd4f=XEGO^gM$lK>SZo&s8Dhk98z^%t_eUt3+@ z)s=lz4?0ZfKk5W0>eaSEb5rd^$dH7!=^(tW+sL>kskyaZtt%A1x-%HgT#-L`Z;TZb zru#6I9CU|aziDyl)?QK?Q?6WNPh!mF<1e0LYym8X+n_Gp!LAomqBrJT9?bWu!p!aI#P9_o=eEO)w*(h52KT$ai-#0d+`_+=#v=E4#y?7YDCpB<13dyixbBG z9llJfaVM{blatyhW!x#HfC6EI|4sODHa zhtv1fKmu{=e5)QABWKs_-z!OZE4J)B z=Wafq+{KwQdn%RpD&cUZLfoKI2v>8%`^x>z+QY9GkL-C^Q1xe zl($DcT3svgm|jb$Rb8SjCc&B1gbKW-o?54T>VLHDeP~(mtgkdhVrM)%KTn?GuH4qo zk19Uzb*l9K*VCvCk?w&}FZ!jph@^a$HL_}T6`}(xO*YIcN|bsW^@9YcY6C`g(qd1h z-u}gI&rYgfak`^~cS#*3oz4>cG~j7UsB41_IZM^S=3<4S+akqvgj}CZCCq84v^*2H z1<8jQ8^cuWfILpt-kC(Ht`7{8fmS_*ryYoGZ9MDhhf>X{c3KO@?&M*t1+D}M<50qU zauOHn7dgDU(ovrCWrw@Tx@e`Ag_o@bxGcNII8OUdV^uXp zj=y9}Qm>4pb#U+zqN8O(ja`KuFa4o-Bx**yo0U45(j#b?^cV4zNnqq5cAZ@G=lJp( zQE~S^^sGhZ;1cc>uHm$~%WeZ&$%oR-1u77%J-LvGid6CkGNg_agHsr`A*5pXfKUW>I24T&*$qqCukH`L zN8Ky1;{oK&S<^_I&aR8Sdp3rW_Ee%MTv*gYB)b<~Imo}!7nef?JJ3f@Ec>4wmlo>& zAmL39B-+AX)kOD;ARv5#B0W~eQs^mLsjY=(y8W<){lpTWy`-vQ*`nX>RW#`K%|D@?<0U-}!NqzlZ`0*+hd!b1$l}aqp8*UOyygcT!0cyXpWkJ6JUeOZ9klDdxa-qH@xD0lAUgkZA`(+RHA{=^5oS+=X!nh29>!Zj%kDwZ&4?5sLQNVYhv!_O*4i z7rNS>k*i&6<8;9RZcZqT+X+Hx*!cXb3f#A)XqDEJ=&(IC*o(~^=7dP-uc~Ra?A7ls zWLQ4~T78Yv2N_6jR(cNxoS!VB%wIu}=vK~sQuLIv?tQEI>5<7shM09Umuol{CeZ@9 zFsvrfcFB(VP6C-%pOc=7Rj8LbyT6AsPq=^up)h@nbStY6UtOa1I~T}hZ^lrZK*Cz(c_g59g6lh2D;XDjNDK2;58FV@LHY6 z*~T70Os8ALqC4-Brej={x8>H}o}dCpetvyx_tcXR*1G1)bQpItJcW~g^V$>Gh+BFqp%_;z7 z+gD|>zskx3@$`n=X&RIuB{cb<3meua)xJEO(%cu9D5QCzdG(m|3S z@J#E~wSIRiz*&re;V;Ats5Vi_F?Nl`?IS|Fis4`Io6POtsymGbon5pZ#f4*&E5+;>?;qHDXPnQ*xHu!*C5X-qz zURoOKV3V0-%lKLy*{hW{IQ5eDJzCXjU!PE4 ztnuiA+4?7|A$Z^4-Kg`Mqc(^Ww8~5Sj&zxrO2;C)r60Q7=Huo^oy|^D(aKVh_$%N} zWwk`f&Cl9$+0Lt8QJU!I@y&b{k2m*{`k*d}zY?$b7V=1~SoI{A(EiB7?T)I3AeJtO zUFcNli(REQ8fEN1uYMHWsJ~(r|4@5rXAcQMmM$<`D_23Yz0CRX37uT0c55 zotDL}W#NKd6W-uWc95wxNlE(}CIiC8LmecNn}p3STGwc6bNNBoM$M^79_537#6y^G zeQgP8MyBH6Q*;xIIGD^HCWLXrycP=9fH3Ly>u>Qmak}{9F7@Zt9=M1W;K2-q1CuV$ z_TlFveY)6j*f!Q&26Zum=t#kDr0$oPFYhf46I@S>a(KDA8-o)G%P?n+-h)aCxt4Y|=}&eGX@WC!V_kYvT5q%Rt@G~Z&SD}00Ss8NZ$Q?bjK1JDs*a=tNLMV;?KsgO07XC~Q1|+}b-YGCjzsv!u zSKkpAT&YsmZslh4>|P*r2Oxf5XF8I#%h8(3^<}OHXy>nlH)H_vlo5C4h113_9NiIM z``w8xID~%z5ytXQ6j;qS4p-7rWBA@u?E#NYO}Z?O@>1N?dtG0?|L15p+~#E@M(yPi zGABVyw_ljj081LUCq7RApzGnems|0_hmY|s1A78|(cEvR+jqZe%?o1g4yIWRgl#zjb z>~rTLd2zB-kDi21WC4SbJYaWv(F-ZR9>Vb!-FJulJ~_SGRM)f8d`|f#V_$#KdW)1c zs4yIMgr}iUxwQJ~Qt`gkX}Vd72d7`4aFPmg`rQi4Jj9g|(`6Cbx4C@0-)WaLT!3ce zl^R-$b{+F=V>{OQ)?Io&$bFP95WJDnw*2mf=gv6=Yi+f7b1vvGiDYMWV)BsO_5G$9 z=}!*TQXPY)yA2iY#s$7gndy8s5%`1AURGsDakw(5aaS2};P|8*2WusOOIAfWbBhH8l z*e*3QH8ZtRp)4~;H0KH2mGeO5Ku%CjsEDWt2nf8Zp8Y%LJ?DC!-}S!dI{$bf7cACd z_mkuUb9j(-1K@VqbQwJgD+tcKC&d9j{J{=!K{uga$v zF>#Dq9I+XPw2u#YJI8Wz1nKTIciF9i3~ut%w_1+3NADki7wi%ma<`Q!%X_;we&)D% zjE9*IGHK1wp$;ZK#4K%p{Y)oFgEK}DMph4qsjIcn`ShPwPPgsk>m?5Zn4_o!-qS9b?Gi!X1MGF&ZMacSY>aL<2i}+1dd|#$dG}=q{=AfG(V!4jh+O|*i zCp8vnnkvN%GRvE}30|qh4x(b#t=46-XPQ>H(bn-6&K39qh48LFZFqt`jo3LF)f~38 z4apHEAyG^QYJO%4_Q_gNcJ}Ea$H?phn@(5ND+~FcQ@*G&Q=kg+rMUjI3;xC9Lt^3p z*9U7H(macX@uhXH<7n+m;7c+f(Vu$93<3;Q)y3GQeW1iR6Vr!Z@IRZm7+2b_AFCy5 zotO*A{~WXDdBf&my56}{n^Qciwb0~?KqDteSVGy_0z@S+t!!>Nf)aF~d_g?}tOdiPyx;I3pe{~JLiSyxgIWPk%1wJroZHYzZHPuMv)uTHZ^Ig;na4X z*pFBJFFNVvfp}1jUDV-~_G=l(e^z?Q1bG4JYD!MRHjsUT1M`@}ySYvvMf0Eiv-hdw z--aLt-|Bz28XO(3@#BJ;gfX9PJtgZKkFW${u#++LnAu}ter;gT2EvRV|E?z3)7AZ$ z^g>o5ORDHDL2`}tI9vah>+>$32>0m}yno3_Zw$8qsZ3i5bE__W>&i3&I|O<=IPDke zh>{ui1H%%D&ad~H<3C1}^b;>VO-)NX;M0-jNk%XtsPKu+%V54dxF(OpX{GLW^!pp6 zcK&igxdiPL2-7)2YC3ZBXy0)Z`NC%Zwm4f8Zu@pyydg=0jV3C--KXWcu z5PC2CK@^UttK7c6APN;$Nvd4VD2Gwjw^Jk(;R<`>NnsCYhwyb!@$ysYq<>=aept=u z_-*J1Zx;XP;_{=BOd_m+&G(;Tv43M0Gg+&dM_q=LGt%0) zwQ~XbeBo`X7&FMO$0zUivazYZD_(t}B()N4HBRkki}gu|z70N#;4OXFPU&i_`dT5p zkDrPeH$5P>qCQ<=;T|GQB1y*5NFas!NAD@P&jGHO5H38N`czy=V8E!%lPE;UVg#J> zqitb+4@E>ga7o^H=~(_o*Ow4~!_Kz1uyi42K!|OZRiw#|8gJIMXE={Hv|rPx45(&i z5X+Zsq5bK|E6qm*i<;-4qh*c>%kvDRrMPZ9K~+Ey)Q=9<%54u6l|otj8{OXE<7%rg zxD9scGA9e_b(^%ik@ZrWo)GXf*)ZcTnH7Ie%CJ`ut} znn9{!U3Z0`Cxy647l+)D()ET^N!)DuW!|C2kBoeB>_dlDNHA@pR%*N0vfYUBYWZ zsd4?Xj94$&SX_{|Afc!;!Zm_kwXXf^)vC?jPn}OUK4^HqUw+fdr5?R~Z?fKKe|l{A zIKltVVB8V%*Biw9k!uerJp60z+O;dz{dI5mQ}0((HdcBWOJbM8;5clJFFF_j%il#I z;?0O)t>Q&0=jYX$D;A)%Z>r5X7{f3MM;J}&LF|J)aK5DMMa;JmSzF2d^KVrTE9GS( zWH*9z&9;OO%uEBT`V!0vN7%gj!lC(jOVlg>;v9ePBIeij@u~}pXdLDVRF$E{%Ol6f zUo?Fh`#2eLT27g%y^s;w^AUKDpOwM(_DZH}eBTE-y0q(e$N3_bAG6adDW(s0nw#!a zcM7kLfV{;BQhO;d;TCoP=K+8R&dqqMjv?wG_kmT9OO4hCYY_{tbb;rP7 z-Zx6t&(DhIPdJsLHRI*q68Y#Cht`6W;CPd49wH^D`*={!R}~)oi~g|mSZ`sNnYa*Y zKIk?6{llIDZKkjldLHU(B=&!X}D=($7uDu|BzdnO)C-!i6-d{M-SF~yt zz41sqlf}q?e?(xUe+uud>cTD-Sda=Tdr}gMEb6|b-z>7~M1~SLJ(W{qE{_I4PvEd^ zxdf3J{HpZ4emVVfiWr*}MsJ(e!Ri^EoDARYL6daFj1RZnn_u~2BgR$Q&>a-pA|S$- zPsFctb$qtNEm2#+z85lXQe*r~U{w8mo(VaM$*8QCXabS<6%9@kP!$g0+pkJs&uS#z zx}PBhU_=fum(Cbc0G6f{$2kjWn!=5FXFKd*T(#e5{s#YdcKg!6dEch^o-)t45|Djp&E4M29-xD&#k zKnvqW0+V`wQM}IO`IYmL@r)C$2`e znDflVriw`}+N#q@q~VoZIO&wk#ZQ>du4aoS_27C{-GmAE0_n$v`XgG&$=NxDs#wVa zi}CkmM_CrTY`#RmwPf$XuQ|7N&z&9IwY;TlY54s!tV!I^D;(!n4jm8y=G$c~)ZUBy-c*iJhh9zk>sT z-?sGtxUbj=ID=Bpz!~RFs}fWU8HYCw0V0<#``iNTU)UH}{BmE%|$IN$SWe!pwM$u4j~ffi z8r+-bPB}LqG3{6SOC8Zxa?*TtcdEq?5$fr&gv90Ywo5$H zCWHFoeGA4uj7A_5wpCeC@d&l*)=sUFtFFeUM#iRYg?g)rvlPi8;gy;@?D5=Naz12{U(fIPrGDLd;8a$FBzF<^}cLhjYHk6$UxL z>Sv?=?moL5)uuTYge-J2x9l>7q7_dVrQyswu3alIqWs(sG&~xbiEQib=B|Eo1YNFl zk51KLMvxHoa==^+&XH7e&p}$dTx04D(D&~uL@3fJY&KrF9RH_U-!`JtqJAfSXS5B z(O3GF*w-Y*Ab{T);RSW)(Z-2;iG9nn5Iuu%<@a^ha7o=xVzmo) zI}9Nv?kayZsFHB>s}hS>VJZmFfxGS6PEGd=aae)C2l~`)tOv!4JU+odx_^>=|H!AI z+XPn$p(B^7GX~Mtw^h@IUckvY^Z8m_=1eJkB@im~8skhEjFCyBX|6C$1Px2}Vv6i? z`3*DiLK}Z-ZHHN6r>s&* z>4VVd{sZYs1ZLOP1`{npuZahz1?zyhajkX+0t|sR z29ha=lN5x2v;%*wh_#|Qagg^bv7uoM?f%C1v(hNju z6jiMNzTDN#5Zk62@bi4-J8^yNNif?*5%>=Jw`77f`TRNHbnS3RoB7jKz!&dRasc=Z z=S01Jd%dr~cJzC<6Ny=t-|KnJ)|bFcaC|2@@NX~c$%|W*QNtgR;2`~XSjAOoQ9F+0 zN(J*|e1J$3McA{C`+hyK|1s!SYzkbJ&NBFN-#JS@_`kn=*#x}mp3?V!Es-)3$Q+~b z-(m0v1;O>8)5sd2PRb*pM><5|))Y2JyJq-UWVztVpB0pb0iPbhrH+%YnX^~o)(tam zJXC3NN%^FM+%2d@c$uC%_Ry8Gy2%xKJc}b9py)7;HdC>tYH^loY=ZTn0LPvbCh_W6 zmBq0I>|Dl`vFb{XI?C0iKMK#=QThhEStWgpUVraPvOFcVjew0+D{r&;;?dL4%9US~ zR{~dcW8{GKM?6rQTX1P>KcH4COt}ZQ*q_Jy)?YqGJiyBKwV>M<-{kie3a{cIUxLiHe;e+sy2vEeuztZ;(ul2+%6|$%^5QgHBR!@YZCYxc@dSw z-0AQ`BGCsdS_$h2il2XQjHR|f0}Vw!kF+q*GcDWO+ijzp?-ACsjR&mC8%jf0HQ!H~3Bi&E~pf%uqXQi!q|mSYkbc47J^}`bAg%<)+cEsgp9UlXwl#Ksr`$Gm$1rt9ur<5-AYwuya1HBNQ7# z265S3$3-6p$(K}8s?^zr8JHcSV|{76k53Ht_MyIzHp$z40$HdK-DrGQ@X2Ai^{l2T z_5PP$@$MkF;~iY}%k25Yz`%3we%wT5@sRu%rFXuU%{p z!c#9=W;f#=JWF$o+LwMn+R_C?o`l6hDi6C73Ld%=B9;t!h}Q#*>ui6{ADYetduM~w zrC$CHvWPD{1sKGV?%j*H7T8o&DYbnXqgt=r*0`>?s+X{y7lx;N*d^;3o%1Xdu_Ta9 z97|yjl}AL}w}+kJiT{rG#Iu)g@XSelez)M5VcGrnUNtu*O*(HE=Qi)Wr|dlHiz__z znM(N;@&-7dhl3N~y4cv08@tw*=%hI;tOOd2q}I!EkaK223B~2EgZ7~K3rjpF=6@B? zs*TI_S$~H^QmdG0l>7}LfG}G^6{Z9$ zAARs!*g!^MqFx$&bG@fj#lKVNUoKi~l%j8yoi@^|p9CYYyUL+4*yaXOePzjaF&6tJ zfF&swv2jRzF!nZ{#D&IFPiox`C=Cg}hl@Ewb0jpm-kW~W>oRUn$R?8JyjVQH;ZY<6 zWeI+m_>jxUw0(Qi4*gM9KdayK(QoPA5QkqA2Jc}NEg!{O^)y*^Z+IV+iCTEitezf! z{&5g(a(OIhuTCpvs>;>zg?gGp*}1+_)uh0EYG3E)=XzFO$r8RSdjkC!&d?Bcq}7cy zYA5Q>bRH?)`a*}1?514~Q*k!`_LA@&ZM6F4fYKJ*hRyzgm6;2y&t0!=@P|zz|s{BXz8;dyf{E15uDvC7e^m80WO&&ere<3ob`to}p+kKZ(Q-}CK-#yQmf|8U zrZ{mW!4vyf(j_nw045**s~7^;SHhY(=G}k0K>zcAe_KlY8wC1aaEGiVi$Pmi0l=>c zNRJ-+X4UNL2eTlcQ2YI55&@d7@#uEj0{gCj#oKbWg}AHK%Z=G_td}(Sbp}CTS`1YF zDmT7+<`)7$y%QCK;!XDgk@^X8Abc5w_(ephfW_1w!0xpxfQDaX3TV=9Fn)5ULXH%S zQ#cG3_cmB|$RH-+-(NyY^o`U5z>in=!F&y2txl~kg{J*GOQ|8wX>l)z)vUe0-;i{9 zzIZumz0wNcW#_@jR_rRi7#FwR2Y#wB;$1o5Rg?n{7yrn|a)6{TWBsAaV>*CI zzwaamd}oh-V667R;9Pf$U4FFJMb5^YIfZ1gRshG4br9f3lpzCZVLnW|e{Oq2>8*0! z^HZ-rU=u(^CEo7VtAIXSY6bEl^L~!_^%jHFloqYIJ7+gcMF?fmpDTe#owAJK1ZZE1 zyF>c*Bj`JJrCk(Bq5)XA)N*j9tnTLP(|#ARU%Nj`V^aD*y)=B`=pAin{7p7AVQ?oS z+I#0zoB-*+8E{UdY%wqRIa-Ro%Z>;n>J|3taQI%dPdU(F`4!KYz zNl!KhX97S-U%0$O6b^jjtF5)79mEq*OZnvS@vR5Ndqndkfb zI8V5{Pe5-<)#+ZK)i_-v$$B@iUKBy)VS2^nk31y;hks>s$9s5;Ql8L(IBX{GK#bg~ zZ|z!$wKlicDhOhS$ZB9H)iAA;t_`Wu6=6;DGl=J(JZYuN)#oeG%WHsQCcXAb-Ij?2a6sIYWJkX$!`+an#TKfw zn!{?vi(~ayv}#rW$5rCs>@tMs(Bb*h`6U<2C>pHfeTiWyEzFw1iJ)b$>d9*1VXGOl zRorf}hO_9_c)vjLuZzv4LF}6_Z%!{Q2I!zL%&|*ZjVW^Sz#ZA> zqF!=a(qYm4=3{GZ&*(!B@r`yxFFHd|e|Xd`pHXCf^|2SdJ^dy%Zf|eEa8-+9o`)t< zzR_TLNNvx9C%MaPyZLyj?<=W5`YSaSh}6PPT}hx+zNtqbwFV_TtYRO51h4$G(a7)1 zKL{~4Rjii1_Wvtf{CAIg(!cNRKL_d$dY`ur_`(8(OW<}%3D|wj0u;)&%Xp64q5{dG z^CQ7@0hF5CJWC^O1%V`e_gaM&`aJT)JRse_Ew*XIEm4V$G zOqk?eFX}jEBk-rPvj5EduMF&8F(6%mAr#-rLdQmh@RZikgh4Rz0{}q#P95OKFX)YIWr$60 zta4sJODiaCMQB4+=Fo`gAj;WvFwvRAzzT+e$k7C&jey^;#1B9^fs(Fi7{fgXtMe4E zdABQOAiM&3%VH00!nC5^q{7_!`~jsMVF7=4i#4{wW~JId!(T{84W;Ob){>qTd;9VC z_|H{#X(#A&3j>xbId`LK;4bBlArsWJ(G?g<sk^>T_I8%DI99za3!T)JeBm$(s<91&QcYG{{NdUC9Elq@|38cE`a6s82e zN&JEN6SWF}THb{zsLC7B7Lqo=gP+fjbG23jb+ULccaQXqa{%r=m7{Wza_ipPzt?@Bib0+5Q z#&(Y!c24>bT5sHCSI16dlmDuFU6g1SXKiyXJdB?A_M=sr!=X=eO>{uAWquQy~x~MuOZ3;fy<2Lg)4D-eb2>^d~3RVKpDaP&zpR0XW zvYSfoj2_3+*dZrXlIM3i?#9RH&p9WFwQ#uFyp$rXMb73OO1l`MCJYd1PLz4#gO}LrNN%qVGkA(G9s^4Gq=Syx^QQ=>-Q(6PQGYY#&XNqYH6KBK!4lh?;=h- zUDO=gvWHX65*70WIpiU46$?lC?*inaO-j^8N&@8+X%VJQ&=n2@@+B*b?CMkHOGIp8 ze;dUH6Kv2QY!r4hoLCctc*F;$pUcDnRuStUfVXUuwgikHlr zKDk`BL^=|z#u5#;l{1s|{bw9h)D!A;gfxoMgjid4wo`rF)$olpomhe~7G^?d^o=i> z{^Jwn`7e>{3Wr~!-isBloEE2us1+@m-4TWT=}?5k%qYPagVX)Aj4U^koPs^?VB{mO z?1!Y&*m+Y?ZS4y&=MRiqzT|vZ@S$H7RbQq$8VIweUJtD7G+C1;|9V18U1t2VNx{k$ zbFKH&A;0MtMU;X>WNvK!@jq1Hcf#lb?gXWrp&$u=oR)|4GKwDESb+OV`kuLYsgWyvNE6*8ix}$os|x4JO5K9 zCOSGJur&NPaGW|d@x8Fdu{5wWnNRLF(x@C{uK-5$L)uDN*7TN^mS_BvYggoufBwA1 zB=>VJyg89AsR;SErw8`Fz=@3mSYH5q1|ARm&Xxnf6c6AaUOnVj7_*HS?fx5_?r=Zi zD)DWHa{$12@wK9XKVkS+G~ur}D|K7}N4^-lf$x;+YL?+4UN=*ts#Q047Hr*usmp*q zyQ~`kNMXJC8KI?4jG)mk*G@V)T3yP3DSn{3#9bcou?$;#E@5A_iix?gN6BqjnjEl$ zE;Z5gAgfqk!Ti}L*~Ly!vB1vUM?D=?r+#IYk#7p(7tfJxkB$Zq)lDA+Kp0ha3F?%y z36A|~3pupq@;cFi9px23qFVi0k?Hi7HI$*f=gSngj-R~VwnA+9S=iV-V zpE?3r(mq`;0p0BPS32Ycyi{*afP#{H&@zlbIp3`ut^X)9?6T_I(PZ_k>jOXY2dtxJ zQ5zONx{>1#KJ%-b#?J__1N^IOcW+5F6fto8nWy}~Rv>aE8W;V24|3h}hgJ}7 z+t#-O#wr^M})%@vk}gLv4EBhJG)e3DEO3^Xw6=MQ<9yNbug4B!vl zO_#HK1o12sI1ssUH#WAcH0hGKCe!LR%Cz!2_G?J=3u8T319f<@L0hz9;=qFDLAstVTBXS70`* zned7>E}6w3x1PvTd31$h=+R{OFyCj67e6Bt4U}8M^A%WBJ6?Fg@T}N0{FttMUf7ei zI>m(q<5LbAW}|&azP?FOvH`|K!=70a4|#Q-tnrXeI&F3E`A#A{PgGsF(>QkD(|P;- zUA`^6JYjKP>B9@o2l*z=nj2h!0Q2R~p@^pWrLBy$K-EGUBu@Az{p=?_>Q^M`Y3OJk z1i8=d_`$Ge9i_op&(gOCz&6}QP~0wJWmrk=-U_Ee5*)QOrS%zFWDO^YmyOsR7-Ox^ zaLW9H+=b!Hx z0d&REFR}X|_>oKMjNAU>A2k~j`UN|~h6!Ef%4y*Kh}%HcdL3e?^o|FnE4~hrT#fQf z{$`(Sg0Hg(bl)k=8lK)_EyylR}4RpaA5Q;xAj_1sog zIP5hj2yL6MEKBL{|4e2j*N*o}F6QC-$dEfyXG?bClt<4dY;16SV$ZVETMzs0Ztwf4 z`&M$z2T126#lr2oa#}fM9>&+3^I^xN;TO3r1cU+S+T+&cU77F#{s;C@WmLmk$~U1e zh9zwfnPuGOR}HG67g3QVoZ)o~Qod_Wf7J-4}ghpdlcq(5m?BB`<% zB|C=nuLg>n)xxh%3H}R{yq%W*3u^vDHSu4M{0~s@UwEe-{y#7)(EP?WAaeIwu){Xw zcYoE)>jyIg@a*#P0tohXp(Wgm6c=im>mR@z7T+ddXZ+?IxVWW-)vE!!AMO2D=gj%; z-Ii(;zhj12&$z1#0HC>cNnA()#hv%OzT<;;TL8`^4@R0Er+mAHg9Q*4W-)mJ=Lb{gB z=E<~>OiGe6c>Ai49}%D1boU8F|P4^rW}pQPwAWt8}% zQdxLXg^A~^^-V_Kn-N6_wW+**Z{=3O!1<2JdDDk<9?zmyxw7Y2m?t7zW^!Yu>(e{DkcofCAHx@+4LGfqvWVhx$hJXJ>S z&qHGws)zK7WxEE8;!^~6>wZ-GJKXS!5#8b{g$Y`us z2Sw=hzBk~WRRM7JjspLiCuTQwW;Zq39uWoddT9WVpg)n4m6|?xvA>`O~*polaitgG~mBdRhxb04{Yg_nuL<$y}Y5qw(hvOKtz z!wK7s-a_a-?2ePTQk+ela0n|{1E>gV_Q?VAs&&N2v@GF~v^^)&d~zf_XqO#-_|w6Z zoLt@As_RVU01fLn0?bc#ABfyll+4-ei0N090~C#yir*+Yd9<6NE@M?xQ#g(@^_4B$ z?WE1&K`nb?F4CQEP?N&VC9&g!(+sTv2d;g3n6!H#EviY2JfRBBgkWBe7IF`*{cOxB z=60DGgSPi~4YJ=TxpBM+!W#0ieT-ZRP%t`JVOi1*GUp`iqkP;sS^_?{eQo zk{ZIEL|jxc&65w1r0G4xQe?QKW}iyQ7Qm0@h?WCVk6|4oZqyR>l2pW9T(_vUy{L8t zu!AFuFkpfFWmybD3hD-Bo3o z(o#B??T88=Y~4%QgTO^ilmEg{N4Z1CYkIz#MLo?NqU>9ggoj$ z?xV*zLiK{P>#wtK+H)OtFh6@XhlIT=Kskx+gtZ^9Acb36@u<=i7AYum{8prl)CZgW z1#3VAC!Dboh?tK<_V9!cMzv;p3#vS;^Wb(9qLyls-Hv*tE0z^KqFyK5HZ?ntz^vvH zh9Q<_^{V^`mFBKsFEFJsn4Uj~`S+Z=J>8#h#1wE=Y=P|_g5qHJqVE*7%^Mv?#1L|P# zr|C-m_xYw>sGx!1g;w>Izqxoh5T3+2-SCFFST#bm$dmR9r%jq*U#9y`HwEwiCOMdbVD%3@@s#+#l~@X!!&HuyW6!)BPGcV2 zJ*q?cvUy<)+IKszk(9cyvY)dBJ$_+elfgSX?}CNLpDwa!{<-UVUFv;DzKuR0Y(!1) z)#sLT@_NT^R4Q#J2SX84lgrkPc|prFFfiTL$`HNg{%o8E{W54BhH?xWR?xq7Btt$KbppVNrkPeP2Fj5k|MHes*ulBYWhj zp5%HPh8YX?0V76?(k259<)E3zx2m2zs;QcIva9Oj^0^~aMC;7=_`c2Rt+lFOgsnk|Y<2>aiQqVb7a^q3NULV{xJM_AgWT9hc<@;d-_t8z+ z<~(vb{v^eyThboY2bnyrI2bY43BOqHQg10r)>Ww|W}x>b?kVTKzu=`<_5BLDyc3g>sZ^s$p@Jp~%b`(W9cHo| zdxm!k621GL5p{FV%u|s|-1^F4H;wue#;+thE@oS_=iZa9<9{Zwv25J$d1bN45kiyCZ4J6=W4> zdaWD~GVG%cG@x*@=4;xC#jho@N^@{WTwFnkVrE7pW(0S+k*XA&_EIPGcwYeOZ4x=55FWiJ-_zf3oGf1*13mMBii8p>Y=c1aIm|aP8HKQq` zyXD2kG4A(tmn2!r2GVXJvAp2tyZd1YkoiN_rny5J)VM6uQCfJzf%k&2iwGxDmm+W=0~=a#{f_(a(eL zi1kqIhY1vmG5$wTN_K!HMk%UQVW)eiJg_`ZFdYx&d|Gh2vglH$i5a`U(RdRuCxC=O z0A~Rfow;3xkac~1%doBuQ}(&6kUiUlDS1c*Ha=IFT-fAusw^%xs~wSNJ>rf3=5qwN47=1}0j z5HP1M>(IM`W-gRDG`;V^lj^-x?qRmYa6;6r$?xrk;4n;)#i!2Fh@}B6YCmS!Owlu@ z-@{AxYT-ZBsvXC`uQ~*rZ+Y}{x@QEgAvmx$H{>XYum}m-by=eHtB7rRcLvsFH1cpC z03apPh)i*Xtc@?%gXf^WOGLawk1|3_%&?R*FkkP=K_C)zDv~6sM zXE3~%nQn$XG+=kel%qGcghqYkUfpiOasRaQOCtC^uIC^3VNCW&7KZtzYn@@+&rH|I z?PVTK6J>p&4I1LITN#zBqYi^?lhNwYZIj`IK9`$2bv@&+ik0lHWY<+$N)}Hxx~FFd zCOP+J;Tm`9YL1Xrhsj3O>o)^)v4K>YoLY#2a{>4D7vZ0p0MOBmv0hW+!k_WM?z~pc zTnlw?5cytUzeu}6wT^a+c{|kG2TKQYhfL|~dr;e_0uBp4wyQKM)W14-8U+*08bj7N zbu#XrxL0{*wBq$D-tCdxcGXKeS%*FDPX7UO7HO|vTy#zmF%*{%1)Em9AHrm^R_(GD zSKIO61&#N5Be`<(BMr=)D<@CyWAQc?LXF64(gmrvQ;C z9DA4Nsn00+Tkpmv!_G*=BouiUP`pTPK2Hi6zTEMo&s`d5(%HBh?#&vxV6uECN2f@z zQUU%JtBKsU^l^RK-DShqKhIix!ePkCi5{VQ$JMYiMpj^(qCB`?#Uih+UTWCUWJ0@9 zBUMHo9>vi0j;#SA#^aE*VB(@q!dUfXF{-`p!oEH~$FXPoOWKSIr;;|@c`Jz9+wGD) z9v2Zua6ndAE{=Rc$@IZWM0|z*<8fjH0L&FIXAVie`Nx+pJFufj{w~>)VVkU*v=Ud- zaT^0n?D1_B+p7dRmJJ&K@@;LD8aI2W^S*^Zxd%1yv(YZt>ATJy}7B5!Nz+ zxctMtaihaA21LwEl&u7u<9yP)LYrdCj1aZT&TeuKgoQY1^|SfvdH2dVtUhw&0gFU- z`h;bh_x8>i-GBvi=XNNrzQ>su^b)d8bC19}W6mG6>80Iv7&D@jYyykNP5}Uz8BXBA z-MzlG)NGEdNVfj~#GNC%b4NY_s6g@l>ayvo7A7|4Cni|Kkj z>4Hh<2OGkMXk;Rp*Zt{MijSh{cZQAGs%nxaM$Vz@+3(o z2zHalX$NO|7ES`M#=hq2%ND?Cb>E9ddp(e}iR}(2m;=O~3G@BimvqnQbVGI>Qmu4g zy1xIE-F~BC#@>_{uK!iAD5*$zoUreLX-2^7Fy_<)u6}^u>N_D8%tR+@FZqFH3jTe7 zM$j6npAgcg@LI{R@y%1#J+1A$%N{hGscMd5*9E-xz-+4c2WuB`n^SgciI>aNf^BMG1{*uwhP&Za& zhf;Opt?m=Y-$@%EERSx5h;@M$R#>u@0~A{r%`k2dy>Lq`{&(HB?cGpiaWY-?bT*RN5j3h13xs?+w`^YXW238+`Pd{s**?E!$qU0(b1`oSG)$t~YM(j(6{Nq=gW5{+1YSYp*3g_``ZLVB4<$bn zCx3a;g5;n6`+8HMjd>7H_r>bdTsbD*ej$q#KFs=jZ1FRFyC>zZ{Xxrre}C{eJpaVZ z(U?x$jvBwhwzgLLfx40qw--AH^uOy4RdppF?d&C-pQaby5AZ&qt<@!J#%~Vi59gng zg+~p%xz#yZMcKDD2$$lSW0?7eX^|UaMtGU&zRkm_Ee zPM+xRNW8GayjiEW?k4ZdMDJX&$ixwK40~>%TqY-80T4o$F98iP@Fj3XBKg)?X8=C!(Bn0{>j8q-R2;_1vQ za*R!8^POylyc+GoH7I@)*+Np)W&>&Sw=D9`EHd7#c7PhqjVSH+Tl@)jDz0ovl+EAz zy$J56Mtv*l3N>f!`X-Ib2=h3TV*(0)uP+ZG%{`zuWVN9=-Tr;};Jz;WLs|9okyx&= zxSzTLum%;>mlg*HydMi(*`WQ^U}DUS*?(_+KIi`a z@w-3w_5EJg{bzESXuMy`^Z9(d0)9DSl-QBOH4g=4D3=5DUHSgSdBKrosJIuQ6pdr{ z_(`m+|NYY>UwoW+psLFSqldxqc2TB*mfB~f6fIi+XD;}gFQPGMT8joRkT=B(FR*ngF@13uIe075 zl<#`LD0|soTCZ|VxL|1x%=*qo94bEmS~cuyY^Pbl%+|qN`K27@SoTtl9d7hG9;*9c z{Ex#Vms5VaA7mlBr2QlGsfl1sDN#5et(*>RDCUyWeFm&AEnuP{tS3NYD~W#$I2Jx@`#nAm#~#5zYQ>4 zZ4J{Nll7bW_QaKUcqxdiA{py-KU|7_;7aZFH_kyA8ZTOZle+rbPZn;PIZIdeG1%Rug=zoX-| zzJtAR^*BHNd>UBx5k^0jdhyz|%Xc&%aCg6~weNAd0Ye-!>!Dc|V0RK~tYY#M9TJiS*KPM9uPEVh6Gz^LxD)SrhUc9O^$jL>X z?znXIkV2w@h5@C|@M`AvM`+LDvX2mEhEhU)dj+a@0<<@C&i1fP)b<&$#u@7I!l|>- z_mjd3eF$r-SO?|W1@OlwXneLfwdaN%G3ZeB{&T8Y@Ph$qBNw`;FI7_!Zfg+OQqnv2 zR0epbSRoiUR3S2)rO06vwg?M5iMFkJcRB7=<9?vMiUlu3GvvD>LPrRnzq)0*4EEqG zrVqFZGj3iQ=<^IuUL9K~ZlR*Hm<589TDVCaOOc8yj2grgN1-e+=;moC7+ggV8OO{G zI02EE&CcjVT;5uc$M6ohTu#+Bh*?hGfMLNvf!`1)2%C^@N2RPZTgOnY}ec- z_m$i+h6imF0}cA`e-JD`;q5gxN*swU{hMW;8RrnDuaAEuM+vX|BBrRCmy?6; znyJDw+EyK$tk!6&5eaPpj`4@@Adjb&$(u_sL>j4*NB(+x5&1`i3Jtll&O=xy#B#*T zXweWQWMf&kz69_L1Vt_Ad$pY$oJdx=UmI1*8se_yW<>p6KUFzup2|w0`u#J`Bp$RS z|F0LGU%6lbYc$oHk{}wb>zfDBf?!-&Hu&c!AVkY7mG&Pd8F0kmAHNxY9NhJdkI-Eu z4S${oW2Qu*?`%zeJ^*YOA)jp+6al|6Ph;Q(CfbcYj1qbzY}7ADkB`;rByZsSAGNp} zLrw{Hpd=^f18T1zxDYJ2?Ah1gj?#2lTJvtv{jGp5Xd2lBP((uZCcqhsR01C08(5nG z&~^Ex>efE9lVO3fHh9D?#Q>o3kUJ^YN8IJaphKH(+7Uq>OV@OgK8lZ_5x%Uvx@;eC zdn)|$_CpwQ$0k5(WZYfzAiWpiCMY^sF@axGxRSU1!yOWG(pDt?P3+u!jdV42 znmqMl2t~ANBYeJ_D;V_q>#oO1W+=AaDiLc%P)(WAgGXK0d`2rnTY3-!ow9On!!Ao{ zsjTH>EJ#==^c$|VO4v`!zZlwQ799w=?kGRMbh%Y~^I%bw;N@>dQB(J(l3;U^uiD%` z+rN_1zNv4n$=8Bu#vJS~Vccl%f-qr_lHF*nEx-^b*P-0HX(XyY5qHh}vv1Jh< z2(%p*zV$NR90fV@@Y^?J5o3t&)4KTa;FAhsVxS>uM;YR|CFENYH@KEmL~p1Ns~gV4 z%lcr>3!EyJS%sGHn&sJi*CsAlmujb6jiu)TqG%z?uqU=T@tEgxUso~1puBcQ9=w!p z+0MI=>8D3D9xY34&!KJesX00E!n|5Lz%~a-{Nve#FqC#->$`yGU zlM(Sp@i&=0_Hi})O4F$UdsVs} zMz9kIDZul?-%yKsv5gYGW0%dH4&{y=lYGYcTSZIlRf^$PM?zcQM}yS;0+X`a(?^(= z*eH|1ZC8tlYy|YM-$UPG?t+2mtKmq3P2UQ8^GT5&X?#EM^Y5mK&A^b$tTXsaI=W;H zgcMax90Hz+1d;%-TyHkTY!s6e0|@@h?DF~>v!2C_!`@Sgu64$tvtob|s|S>pb{Vd3 zy-oM7l!kn*s|}pYF>~HWbQLQ1@%so7pot`yM-<^Rst5}D!aPJ(T{?ok4#%iagL{yF z!^hZqw~I$aG03x|mO82{#sY4MDH_v{hcmU{F@yZfXfs90VH_o9gbk{|mYJ&t?Bt`k zP&^W-P<}ZBA8F9|+EL+XwCvyZ2xwHRdo@l$4yJAbM%|THe)t(YrM6K^!W6`S({@~N zg_@)w+6O%AFsssA(g=dj>(1!!ioCb>@X^JV`59b?p^y`~iYk4vYaG^EkrAj-wgWqz z+~uJ&d2NZ=1B6hV5QTFcyy)fcZB|MjQ`(l z-c;&Yf~TRu@9zqq36S{Me53FW*Z`lJ?pgXPg}Uo4o)Je@FnKMiMMw604+t`4UYdY^ zj5B2C4_=!H4omg(nIg_yKPi&0;6)qN?gKuuCn03$b=yA1DP#Dd$Rich-ihig>cMrt8j0XRwtS#T6K;qIWmEkvU3nUaSR2Y`k#c8qlZ$KPh-myt&tg zaiCiaD73A)O+BF}T#zPi2FCo#+GxgBwyGR_<(IyUrqzj$Ext>YUv?%4zu#yzx@$Q< z<*X{7dt)P!JiPCtZr|ivra-P@nFM6s$9dmEDc-DM zC$RspcT4qF5V8=;a(DqM@m5^N%w%i!TXbiyM>#V=O(bh{J#$Mg*yK@kp=;NfTU1QI7AF?BtCD}wMZ+*1n?@IfP(6=kt zg?Pd}xYE85ZWIZ_>{*=hvRDZ%Ir*3CHDMuePvW(F+GxCk+r@sf2(r(}A64cam`66w z==l*7*c8F*XwQ~EZ)kZu*@{|Lcv&91<*Z0M*<2HFRWLQV`eFN?ytFsvl*IT~mawaX zb2a@^b4&U;xVQ2!he{8BH(!Xq@Aw=#G53=yGtfOs|5Po(hX^N@#K+i>a`$tzcy&X}3pHUo6FN>UzmR*eb5N#&s4DtHs2h1+Vas#Vn+3P z&{cDB0-lo{OeH zuHT7FkeC-m7xO`FCSRra7A7xS_$~@X#?Pdv&+IXcUYmpN4`5AXEaiEEgBGJ5yD~CG;jYU_zeRQivvy)hQXw! z-iY#!pF8sS0p4uN^O3O5is7P9A}Z40e*3 za4+z26#fe5$$lMFkwX7c@i>fsesU+g$L5Yaq}li6?;n zr^7HoS*Wkvco>sC4g+X<=wS{rI3;<1xv65q5^jU?;%0w{zW?kr^A1( zFqa8D4?sq=t16JP@Pl3VYof&1=A++pURa8r;ZQ8laQ1zxIsl9X(HmmpMGmpqc~oMH z=MEs3>>7o0#TXB41iC@^ydL;mDkuhBIg?U20H~kbj|YQ@?U->}{y_Cx^cyEo(@d-W z9e9$-jg?N5HFmu(zv}PZRMS=z+>-{{h&=Wp39FW;?&%fghod`pW~L_X3Nnld6n8_l zDCA>)9(BLFmRAVAnWXNk^-=Fti2>C#(18Y6QTX7n>h^R8@R6KS1B6&DGr*9w35zcQ zG^T8t54iarn=Cw*uD}ePyQMVyyj}J67}2=%O7Mv_&WS9&A*7B`IQpzNvKwDh5IcCmuV4<6- z3JGMIZ9WBq78XjUCg?O!?eryG&+N53VgC&-elamTl+`qd?C$kU^|OYHP!xxe3+~Xa zv0X_9zZpX6VbKbnp$ziGYJI$V>4zsY&m{0h$M4w_nKzGe6_M>R!EU-&H%*r}b4E z*WC;Zd(u9JD4V5hdmTH$~hW#Z^|xrEaIr=)C`hkpx=v5XL3pi(u=PQ?fG6M zeej0M3s8m#47^lR2PND1bDz5yy@}ur$t;D83|b23;736kWxZ)La+KtQ8uzbSqb0+M zrc;A)UXg)?R7+%bFV5pA&5hJB&3wL!XPV$=m_7|DB5vmu;iTyj6O0}{ByVa&!c;C-=SNR|L z-@g@ZGV5sLAZH^W`m9LhSE*q`&ofmRV^8z#`eu`vo1EoE^Rc31$QBa=}r(rbV7sSMo}JVy4U8??TYM z({QdvK5+dxc$qx@bi817C^i#&*00^TqIy#Dt5Gl&h5&;W>}*9JRJC6zdBD(ZjKUM^ z!yA%h8{D19_kS?zXmaUvle^b*5 z&*zJS_&^Qdff?=kK|FzULTfpSe{V3$M{!rz+qUZ0*C;AJ$MFQ_Uh8Na?=y5WU?gw} zr^ds<6#ki0XLUa0laIElWxH6lDtI$MWJp&ClOsMpNcQ|__=c10HqeIVB$9%!uBs2( zSq(=^IOEC!Rf!I>WG~Hl;!RA?WQ~-$?xz}foH5hD`X4{(L3aYVgESz-ql)$}SqC8xJm^UW1Kt}($t$=1?Zf}?_MMIqpTICUNr)X_qku+m z&fETPuLdZU0i~zKfHCB3>PApJm3J3MhW)C+=Mb2W3#8$dA%|#i?48lX4WH)9v}%Mf zG3t*6xcBb?;PS1h^$ZrVa$ieTAh|b@cpj6M zQ6#WsaWlY^*DZk=#Y;}%r}9r0+NC=iZ_YG{%Gp}=YO?tICr^I%B z5QKdl_;%3T67vXxr|l?|@7a2k%Nmx%W|7^*vOnlEUV?976G?=(=Ha8!7SpkxD-4Lp zNf(I1Tk82Hr9#q9lbKeS%8bfe-?7_uM3Ek*Kp^Y0U%8w^j=HK(l6s^p?V=OZ(Eodt z-_hdprCS;E@Iuc)?_!no0-Edj==qSk37HE{4~I)n=X_*1VP^a6MBXR@#nU1h^jhBG ziRu%dE!EHACd zz$0J?{P^g#lxHHbbl4%u(6mKrOt)Lo4TC|!I>G>ZdmsdN?2`yPus8qFR<#)zD88S;5$jWcC)DLUHe0A!lc8(_QUcrO&r-YvgemEH zU8R=eM)j-h^n&^+?)trso`S%#Y}(Zwck+ZZX(E4X_@yel6J8Qm2y$H!{jNLXh;1rM z`L^27zw=Yeylx!}Oq{D9)_((uX__SlZUVlyp`sIz9{v{H&)Z?#I=`j+C89EL(OLF5 zvdo@403}EFc4&JftOPSDBB)CaZC<}%KER34(sEqhWgskzxU&W=%rV*yLP)K?^7gr& zU(}viubyLkYy4QN=^kfwX?mXvGREe>HKKkJBB`u`AJr2EWwC(}MGsx`MSCk&&!_Cg z3?~mdY}~M~87|!bS^p-MWm`0*)GJ<*GDVx8ylUN)>6Ah{^aW0EbKb;$4!H{ zeEyFi+NTR_|E5}N6FkYQon%uFdRg4*hM-aIQw|}U+3xMz<(4EsM4DilYS-MF@m6j1 zs>A;Gr<^I)7K}dCE@|xgAmT3XC|5(j*1RPHIb=U!&@Z^nY+jv6TrR8O%9vHqXK+*D zGkS`!X*JgAJ3&?B-4S5akMIM$d@36I>8(1ehmPP_rOlAhJlbdqr<9W`xoghoD77nS z7mE=Gzqit~*H~>p-DcL0GSF4mttSQ!`-zAIq4rTbQZGf#%PxVSI9QrM?;Y`hUq@B& zT(Y?b7r3vKer{E0nlZr#s@gobvK4p8w*ycughyQ*T0{Jq2IaqYpS>hZ9zk_9y%$t@2jS4o`>n2xu{k8SddQC<&!FN3ajT_zh%?&9 zy89`=x5x0)Wb@>#tk1U#3(D2#cO^CTZ18LRs!F{Ioj3OWQ=k63>hOR2vA+M~_h9gs z{fS~gKg}07Zyn!_{QKCy&4eJTb|ad-J^DZTJf3;GXy%svbzO(;3-W7P3Qpsu0d!T|JY@a^UEKWLDYlZ@&dSEJMe5WTp8SS z=u&qWt0haMDGOXK%&7Kqc0AG}m5)6;HT&Bdedg>zqC#Z&*qJTBAHQEgGxii*ytdO# z=4h)PXj!(iiH;<2MU^HSu4s+{#q#Al#Iv^&=Og4hut&(A*HA%u za;4L@CgdGgRvQ}+KpB4lx#|PHYp)6;E-%t95B1aJRR5@>72UA*Sm)Qo2NBpfHz?N~ zlkZiDPYXsNmSiui)_ftPCz z1@6b?*S{eZz?#EbX6NgjC{IqSR9PieG-MuUW1f+Nu}mG{W41~p4zJ_yp_EgWtw`PzYo8Be9qgl9C4crlKW zX{HN1eaE|Ai*EuH1ap)Kem4r=UNiHDX-7f%s9jfwR;r;X@eA|j_>Ot@FBY86*iWe% z>|hErF&F)1XP?P@>wFGzKe(uqUp2UNC3D3Tn=gN@=6TE8GI=rhnnq7`8XK+ojM@@P z05g^v7#bK&qMGY&0-iNjsqfdTXpu!NH|yoiQ5yWg$5u6}c?rBp0k1XCi7~o|qpw-u zF00oj&e6n;ei|iK?MbAFYK6+Kl26CqLrUjavTRiggT0)yxGIVt=$)=-yn@ipw5S0t`tDQk9p#SKgZ?Zl2O3dn2#ixy7cVJxgf zC_cl87xs}+mAei0y9UFbpi<*!Huc*lDLka9?wAhDW-V98PHX|Hr%Wi~z?-PTy9U}b ziw0fUod|>dK>@!ceNgrzpIsiJTm9(@8yO+>#@xvcd_VOv8+rvxO|2+}{>|{hl!8 zO7FPHge`d`(t=$$_O+Kse|N=pC$yDm7_)l%L^Jz(L~cSGL4Ig=&~mvOvdh=aQGvD~ zw87c<&orqwxm^DcS36PbIvB*eSIomLyl_Sy?wVbyq7Z*S zYStxah2TORUo;;Pvi{kM~AE3+NT}S{m3IfXAmifku$yH@kjtX3*M} zQpUdUeNhLb^MawU{$+$3D;(ZtTOTbG!cy1=aQWQmd?*O8E zH^e{hVrT1ZVDWK3xS^Z5RwUi=$R|pST67W%2b`&S0|;sX7f&Wyl?g#G0k}fp&l!V* zXYgt)ZVRwT;4OFS7Oa+e7m!9Y@-gbp7Q9;x_Na*k_w-Z7deKW-`D5&*;AH}U#SETfy5B&r9`7@;^@Cq~kEm5VYfZ)6N&XZg3Q+)_t z?jGk`lM~WhYh;^l@Bc#cm%oDHUtGHS_DO@-a-d2UMsWo9&~%RZT)0TE3g@Z#Wu7s} z@@wf%W3_<>ie0ZDziG}>^eqb%#yBD5OqwO)avZ1J(>dLmYK1Z$ET}8=wgcgJ81Y4I ztPA@4Jfl)*wKcuxN#K(^Y{HM;V1y?B{nh4!z}WPw9ID(lNWQX(c@0xAy*cah*x!jYFb zlq4w?`1gxI(&k z<6h(K+i)-q(SA(2e`iiFd=UczOP@jI+no> zH<}RIteE%(u1H+onOwT7&(EYPD3Vs%7|v+Nrsu=+g;$d1>Iu#q zcO#A*s=9hVxj(3XC9FfR>DEWTQ&g4-tm|#F&b+8S*n-;a8k{s6F6D2JX@Y&h-&bDs zXz?w&Hl6sc)-aPFpfueL5j{N^J(3vJPMXjxiVjlkt~ztvbIb!@YFI?P{%FN>@e4Jk zMYD#B#2c)Wfslmy)#E_OC|W4KMC?y-gWTF@)VH&BP(%g6Kzo!GvX}CW0zAg(Oz0>h@dtxGA&c zGX{EkjOpCtbuald;Xu?F8)`a0xhtxT`@)tqDhkw}dZW`<-LZPWv*Da7Hz`{#ayGMp zZtE33LJp1Un8NCMNoZkuY)>d6X(D87F~xM&u64Kb(|(5n%l&MPfuB$c7JM;eLmJe&V>8n+<=i*w&lshNG75y~A zcseOJ^|eDqD3J49PH0QGwPr7b`s8Xpu_KKOR_4%wN+UpV{i+1-Vrm%cjcuGR*W z32K#P*CgRf0%xgxeY2fTlJbI&1;)}mYyB(2Nbs9A#+zntxMa1OaG;gf5P5$3;8eH% z_Vj#vUhIs6uT7Z34Gu~Y1up40&CqYYcLsg)yJ%>u1vGeq3HAhgmC-V%zUgkDsxt8* z+Bgr5ddj*Y*Hf=ZEt=ig3aMC$uX&?cxK6a6Z;`rioUYKnut2|Oujw;`$Vsj0L)G_d zC5@EbW6+a~SIR{(#V@pJj_?uJZbLI_Xsv*=U=E9xd!63+)p=VIh4H}8VqYHZYrupc ztsQDN=SN|9GERbiWgjf0$e5z&_*l}lmiU#KSrzSH&YxO=75kmOODxO{Rxa;~8U+E+Fwh~~$KL?H<%t3`cSm}*1mY?Td zx)WS=Urzns#*?>DvvsTR|J%{-fBL*Ot^Yd5Vydt)xV4x16ib7C|DPy|Uezk-MImjY z+n@{;E_NpUqyDocq2j-5mR%rLDl6UmaL_6WlgNz<{5hz}9vH&a*W~e= zpyy^te)@7b;Dt>9Bvyj9@0BkgDG}V$g01YYeRsDJ??tf9jGQ>jJ%Ol-8cr|0Cd&sSL1mK9lRnh;92q4w_!1pPs&F zr58U(r>mR5mbM$_2n(gpw5O%!O@x(f{kwO`^oCN}T8>};iWjlwW ze9t21n@sa%5kJgo?wN_R-L{8JWykE3QF4F$?NMw8rU@85=3|^ZWE|%vWR)9Bgqs(K zyj;jpSnu{aU17Ih@ph+I{B-O{>hV_D7tRM#?2^H4g^orVt@^!%=<#uf*w;0uHCxe2 za@H^>$TNx1=Vc$&I4fpAh-aNb-WDmFkY_lPcIW&-%`J~)LJ#E7&i{5ZK2|~JuE!l( z^0Lr^-qcjomIdiz%C~&RJw0}IwY@6bH<=MR*U|F%tyj}D^-<(&UzPE0 z#|dxP+9yVt^9=>;vMl;~p9ToIh+IcvINOo*pR{CzG$)WNuPEKn?F)JL-1NIYu}&8F z*<%%avp?vOMf`=ix)qz|J!eF%{FtM#6y2H=ndl&RX)uRf4#mnKDxEfgcOtdIj-ip{ z0{i0vmk#dluFJI*fvUw=wfvQAP6JE6ZJ9@y7Q2iEuTx2Pua}wU{YYAO?rQl^TQ-S6 z5UHRwy@8E>*j$oy&@hq$PRYLkjp`onR<_vp#YJ=n#dp->?F_)2-Qo{pkn@+IHt>kE zjd8LefGABe;fQLc??xINZSA~Q56>U}%@V^Ny0b`Et6N-{5;H{pQkdhe6)7mn+9#F`L%9=98^!dlvpzQM5(8|j_}j?vAmT$| z`DpY6coWi5qIt2Qxr0U#uj5q>(;}O_leyzrOC7(?Qvv@%Khg$@lC{i}wcX;&h(kfE z^HEf+H{Y6L@0(AzaHo`^dMBr1VF#ci1e^CaiphdfZe8U+)j{}W((`r5fAeJj=M(;~ z%^QhE7ytc*-p;o&>aUXujHx$5Qk{Uz)7njd?0wU68s19Jrft`rKP96`9{1t z6&|l}tHc42QNMr}f~!F$8APm&ES&=mM>({@3I`BOOFXH69rSZ+67Cmm1Ll16H!9BZ zI)2bT2sE53Lpi^LtlK`AIFRDie)5+x*YAM#`ukW^a3LhtJI)ot8hflrye;6=u91ZL z>OItS2g+vPj{=?PKVKe91>4x;FAdq(>~L><4euh(9cfL%DyPFFaI#WJ1wZhH;T3gM zVkh-G_-rd7yMZYJ&JXq`K&NOuRm;uuRk#0-4$s2AUY5+MbyMDv**XP=63__6dV&1i zLWk%rz~%c2BLgrw_$k_hWBUc4zJ2H{`|9rf`wOqQpeIre+;@4{>A+44ikHnNeGzH_ zcaAZcDJ}_xb zjs$>-wf2OzHMcTf%Gx*YFONJ=uc0Xg_`bUw<6^#VgJSGTXg;n)_2bc-kW*YKVt0I&Juw&9Z>4IPjxjj~kk%El;s9C!;U*o8T*J5_zP7C=coy5=I+}AcAv5{x7*U8ju zBONlQr5I6&HEUR=8SRNcUGHf1MUuz@$2dLxzg2%rvuwV={jUY$vA-o* z2BpyQ*AMqlPM6Gj4~CCDc5Be^iw_<6%#7aFDa!T9egC=Btll|2x{wVGjl@*lL@7hs-3%enm>gZ3*}zwTzN-K~fO)}ObZ3(h5^_>BB+KJj@~`y8B=QsK->EiV&P%`Y?*im+do5j`wpPH-8f@KMqDa4<%@6+n-4BI#CEcife$cTtphl z^#v%Cwa$NEgQ`3=d|vGAqW+4j`Q6Tr5+RKM*OM9zx2V{$@SugHgt>*lTArl80e7IU zpfts2yxRx+#JhRGtdMs6^@2W8t~Kh&0fC(STV%jEjV%ts!Ln%^*f198)`IgtC-g(x z%Z>7?#%~)u@hfrjyReqq^Ug1#e&)-#He5FAscpB2k^DEQcFIlkJD|&a6MIb3FMleF zZ{163Ie&*XuS;D)^+c#CMe}e5WHx`o$I!jz1`(LC3+FNzMDSXm9!q0Y z8~=eeUN9u^b-S8fA@DHpU+r*FVjcdvH^;8;60r)uAe=H=K7%LG5JNvI1csK-oT_M_ zaa3kDkGnaQ7*ipcm(q#dW*!jdW<**Yg#>AJJmv%uxB+0Y1bqUr))RhBhfkk95r?f`~g?~ zUskMop;vB!>kiDSNZOxSo0{XY{nyZ;a}Tp|F>RvfU=RI$=$F9l72cE_E-?k%5`D%?mx*w*WZH;N)h@ob#JW3+9&6A*G{ zM=FiQw#rHI+73oK+=@N1(IBhwn@3eDcZoZ*%C?yijp%mami+3>_3EH!&U}BbxZlQ-kL6m zM*jZM_WfJ0z|fd%K5b7N^oE>Ox|c#PAt{C1tDoOgTK7cN|e& zn)&KXQ~=|3Vv?JI<#OhoYy8*QSY=f*E=tzfCjEY2ha;B?)u=}`Goubq;dJ{;)ng|>L z;SS^<;5vvZR+hOrPWh_kB@1-!pe=-!FEl<2GssW1xmU2JtQF)eVbjg{9AC5RLlq`Z zR_tU#WYiO$m}{FQfgo=D<0)jp+ldslubYY8l(#)<_u0f3dK{#+$9+yDxh#)lW@&`E zA63C|UH28K&*qPxAv%KB>qeEsp(~TpuYL@TNb@SZQrBO=y7n4SXW0?Mf5iFCkmUD4 zvw3fmR5W~glxCn1P*qD_ForGbRC?D;l%rG+?uU{!4NiGkCltn_`OV-6**0%r; zCiF{068aGpldm&AOg6u_qg)a9WA76Sbwd>N*%-kz_Q~W`jToKlMX$Vz2ibJ8`=Ns~ zQ}2^SnQSp;S!sw;y25F-z%I0U!uq4)bs0CDeF5W$S2XPze{K2wi8&aWF9kIA7!&sZ zXRwh4LVQchh{nrGiH z(X4ow=X{ZxjfG~jK2LUwDn2%<_>5sSg=Z5O6$Q<^4EaAZ&dRF~%x-S2zfbtk&nzLS z*ejtMxU=zA??3%^cM-=O?65zv!ga6Wo43*ytao3HLavXiV^p}_Ll4uf9_!?sJxyr0 zWIk%9FYo5*X^n;kj9Nz<&`pjyyUJ$U z@PqVw#bKWFg|V|H`ywXX>l|Z>z2~h)wDt1FfA>lC$rAMVN<~3j&LtKY4h-V{iVyJ(dE#rJ_X96Yc@?O!3$pZ6LfshgWW*y@Edc&>Pt*k!F=I^fPtCb@>qmLgmvOer-XOQ~MOS1qI z4OLp1O{$=!_&Tz3?Gdi_=G2$JC*nbZV&gClgZU#z8VKg@C&n1kf*_MWY zjdSseY@>5|IKqVJ`^2T+hD$XJM1}01Sv1_|2&Ia_$5uJ7Gu$!N>yr`93WDAQrE$qZ zd0JRQPBDvAY%`|Uw^GMcpLnIQbH@Mw)u%|mQ`lgg!47<+yKdkC7k|lHAaSw(q%(yv0jhG{H4p^Tir6>p|1U4P zW`w{lPOb}4uy@OIZvqIa7Xw&h-Y}pM!X5)}-dEf==j7vDgt!WwJ42LYClKHCpE74;9`+mhcVVB^|kF_--Wv)%++i9tj&xW4>XsCluBv~A-)nBMBgK(~sdJ}gst zN7MCL7w;8Z#Q7G;%{D`Iy%|*?q+}-Kz&8%_xY{2lr(Z>}FlVZgroQdvRI1+x7j(RH z9*_m(yH~+CU-0VT)r5zDZ)w!(Xe-cAm z?Z`Uh-olWuqm;-f7ZA$tKa$>-#B{lTe&iNeVu#ik<6?B)8(JYoXw@@#apqy+90~1r z)P+7kp=<^ISjv^P$tK09kSgCOcqhzlSwLEC*YeVPHi>?~U;so05h{3%Y51ga34rtm+*ICeC31DS6*_T#ms}pqD zHpWgSZLhZfgkqiPBQ{S(2|%p&gYq+o>i_xmnvLYcfm_IR~|{rP+_?>^*1UD zdL(t16KV|pK-PE=^A63x>sY1Ll!~0(q2WibX{6H8JMS2$n1)swad-NyEOpw}`A0{; zqd%gtH%8vLWZrel4|bBb_ix*nQ|s1v>3%MA>w>gmC}^^uT1B9x!OdKZ@eN1~q3SmX zQ@nXoTZidr&uka8?`Po?6P_CuG*^wW82h6z&K?nR8P^s+tErEf4-EER8L>~NY52BV z*QqubSq2Mj^I8~6IZO^<_qc|W(JRb|HY%Q8B1D#bb2`j?T=NdGf6@v(-q3w$c3U?? zGql^gK+y{W0=;(t>*GZ#KHS&Wsm3sBY@O&?f;YiPO?+c~$FNauDx3uz7OVGuGyl*|pK1pkB?5u|2GUw|7=d4)KD`ZtW=W|j06)%Kq zUx@MQ)+LW#4&V&9PnNchMC*CaTuUK#o$IA@@7`AP9Q4mo_7~Wt-=3*@%IhZFPKRz) zvw+WZ+kdyk6O2`FM;eG9!(J~ZW@IRqcAYyEhfL%*EYHM4lUCNk+CTO2;{@_?U;9?Q z?Yh*#xCVBAjBMG(c2(PBTv0I_y9>~r?gydUgK*v*pfO9@ZZlBLFkjJ8oC|(@oAfhe zM!Q_Wp0sm-S?KASA@5pu8|_F2w+nD1hNxW#9j5jP-|%rZ$N405gdDuMev0XBX`V>@ z6wtwHTF_7S7!cXp&U7X)iEEB>OAT!7Li7_`>3QLqCSfa85|YMH6}T)IEg@?%q#M|~ z^VSnCF}9IKMS6^^=RNBIZQba}R5ET(oAkw506%^^xYZ6^5hzN)@k)|Mo8k%%MPy_^oNC}~Z zC;Ie^B3No=Gsr ziQ}GGwhC#&*_Kp+DY8Te|rJU(}}ddJ~wZ;tS(fAQmg=dfFVtR+yY00O)e;~yEf1kGAr?uD~9CvFsAz>Q=$sLeH& zrM8!I_saCf9md>eojXm+*m$5MENl%}m?mb2zk5YIa@{D$1%(aIFz{VAE(%D;NWJ>f z2AgugJ!-G<{FLc$wtCJA9YyKs1?*GY`U(Ik%_%7@`1upq1cl`tNE`f8eJnARF?Tya zrE_LfF5NXBg^%kCn&s8)4ssfS>=_LXo$`8;=V(5=V@7b%kLyI))M1z!NXtcuAJ6c} zp%^>%cJssc1*l{?Bv!1*)8_*L5y2Jjq)=bm)O6bPjVJ7hx=)Uhd5wLp%W`@uGcRH^ zGyPKH^OWZKIY$8S^jUs>2;{&^n#lb!=SWtt+|cJ)q^G#57tm9nx`0So2>-AVrs+V{tE+z*%)!S^TW4sh8Wm3q%! zhGWp|kc#{$0I1$uDEaKQ`}lG-hL;&J<%fu1LD!@HR7j2ts(DWq@*rE4+v9SU?y6J(S5!m!LPwV)D3Y@$k4gY^u2a@iQ5!bXnS zP}mc^SC*t_Ut^`0xCCd=WAP<82viw_EVt=fF(J9=Zh?OsvU}(=o$CZ2J272fB!59q zuWb3}+opk)$*`OK5UphuT=L{{@^+?J>2fH^b_g1Zab2 z>Q|p9+{@2(t{t>ZeKdPNVANANHd&|R1+@!1T9@i!aAn6BYa)&S3G99PElljnB&h^bzj|=OK-VTRaRQd_KeIJf21B( z#ae1f3dzCQiFK~>s?hqn+RiG)cE$|nX`&Vg z^yq6W zn?X&uQwtw4s?lVwH<3c6L>+9xps41DX{Vcq$B=pyREF4(-;LlTKw^Gh02?Yh3sm*-T&M9~?>!h**T| zo`+|q=_LFZ!>fuuxy78w^VM&b(z@GbdQL|veEP}tIbDbZu5@xh7ut1r#7$q-Nehb4 z;1Loy8?0dNuCpb)Nrq-_o@C#ok;Us3PPnx`UO4#6(0H@9vBY4aWNM<8=+xt~ula7(ZSU@Bl+YhhOyNy1LSA zdA0=h;`(F?XjEgpE^}+gaQ@b8Z1MDIc-M`89qov`Wv&0$Iiu^Ml=ZGHJ-_50@yh^@ zii&E^BCqI$Qff%m_krcLk%bo1+cWZ?2PMAtIUKt=gz&GB5hbN2ABOcJk_~>hiTyPI zc~|N`B`*JUoACY5Vfy$DC17_U6bGr{g-jwu0sw&GhgmNT>PhVvl;qko#oV_`SH!FL zo=4`^I`l2j$7_MdCE$B#^hIC}u6y`)%ql9b?`!S^1p$Z@03Hey0{wmpOfdamQ52|+ z&tYV*07c3Rwm2vpg$QM1jC9}lx3Xa5p3R6$Gp|iVgN-$_5#vduXL6YVA^;I71`O*9 zPM}N1v*4l?!{a3Rg-;+*4Fzx)YYkz!kd-oU~VtYKUeSq@{`h>YFYi_*J2YD;pvALKC z<<+LC6HW!tAd4u+A-XBEwoq_rXeBnK_kpDDir<3(@5_th3*ta(qywXWvrsX8`F_*c zKnOeEvY$He*c|e!EVuew7UvuVZY1BAFbV-}j!h{2v~6TtDkFV~c?CbkT08^Wd?n{O z#~()ugrOo}B@8ndJh`G&p!~HO%vsLvNci+MhXCEjQ%^HE`alwJZV+sp=SQUU%6UNe zCI@BJT+B-WL#3mDp1pNwX4M6p|3T?7aE$V7vyEhV)2{|(_dQzTIJ0ny?p?})OJ2;C zEFoa55_Nmv6rv*Xv#OpdH4iqBC>Fi!O)s5Lq89591WaFv zqen+S*Z^X|2(6ZW<-=ho0;i4RjLY>)0T<6HL6=B6Ns&c?p!=K)|^iL_(Gug)Dw#h8g(;n)l?gv3E{GtIho@hEEV#^izW@0ocI zy(CYL)+t>1ne4R_w_q}kx|Ht~WRu3@RV?Wy=Wl^P*IyF6=%*zPQ{v(<)rQKHvth^y z=I*}yGA^aAzBvt9$&QyXW4d@KkW;~!`XB=uJ za2$fuif^Vw-L6*+DU5S{Ry+H*TO*Dr(XGjtg?ZaI$KTaUKN@i?-5n_}sWF_rn?tXh z*ag_VTcG@+&|IEIpM(pr)A{|U2m~sNxK}$2{=mKUwL~)hM(f~6#^}PS6kg(_a`>p4 zkeUtxk>RW86{n1cjlvbpgK()Pms`;Z6T>xz6uXO=eK#FD*%_Dl zE9a1vDI&T`gxwBE!h{W#}7QcUEx)D zlcDNi%e9G{-Z_^W*CW(8FqB%~&loO;urvR7E#y7VQUijI&J|6$ox<-F!sw`-kz8XS z0Ge(xK?wjiUbhZ~0>2;fou$o?@J19-0wGwwa8o55lSEn^t66QDGk)4lOkUft&l9L^ zG;My!`ZV^s;SV}GhCPh&)|#!rxUdkBiQXZEl})$%6Xp;Z7zY^Ek7o?LwiiRxrp|%w zd+pJg@uJ(DGqn`5JZyC!xieq-DY5v!1{5iS|MxdXSdQib6A-_6I0jG~={-orMWmkr zlo^0Ygn*I5&+LMoIs>!VpS9_~3ts=aPpUia5Qo;83D<}Ax7A=ufxVc|2GHf*T5MW4 z&xH+yi8q1(DaxQfxV@Eh;;%oX{*)6q*EJT717865^XrTG4Xz2=K(|gK7PhO6i=m$v z4dzTZ34Kd-e5H}jnv2}FHejmhJx^|pR|m}BRFOd+5U1^D{v8fNg`xqR;n@B34WJ|O zogbOE1@pxyQI&^Wa5ewfkrda#H(Gfp?f}jeMY61l;*sQmESOrha-tW^0^*w|4}6RC z;64cl_v3|fw`OGL17&?MF8jiHRXy8|BCX{ifgIbK6nKYHXI@H3bJ8#-s0-Na_~mdz zKyA&KwauWmcTK2d%P#zW^re#mqf+HAyRPgeZcb#TiHsX{bV%sIk8#32wd>t{@((m| zgaZsgt>wouLo>HR@z4*nuO$jTH0Wf?(k#$3dCbLE!Z6|DY4-4n-v&^f(yBI;H1Z`h znnn#C+Bbi$GzecJ`Ne~4l5tD-2kqU5AHg4Y+GC_6U{`y0jz`?!Ew;U`Lo&m3u&w=( zwzCdi?-YW1u&nzoBeqJS+_(TswX@UXR0 z@i=vx$Pg5jhnJr;OoQEI(QTO4eaVZ|<{7Y*x*lt1`TGLt&)!;T3bil`nsMRmQKqC` z4VA==nnGsvT{@9U^-&LtkSU&o7Qo}if2~?{$>n>%)A;6Mon@llPm{E1Nh#f8eT*$z z4TjPudMNfokpT_n5S8w9ti`|vZIz-X&<*3C+F5UABk3A%7lz6RKuf>nCl((m^_E0BU6TC zG#v#vt?DDWQuhl5uLDS=JZ$Z|Z;{@ni#Lns9z49x?*Y1#zhcl@x-U*bMTk|O8zk11 zRrj#4utB|;`1X}Ub+G-;sd3!Ql9N{b@=7Zj+N`n|N_uVb$Bby`(fSiN)3gNJC`=Q^ zVO&eJY-EJBc_Jb9Xtpr{ZYp0UtXygRkuY_k1Cs_R?y17J&ThVt2*Q-jpvRBoG&A-a zEzu?HTaqHqgP$ywhUbD+k5Gip6!>e~l}&|;`&5^Yz3a2r$@zE`;`fxHI~1PRH)??k zPS>Uq7Hz%qX6M8A4`(q`gS=(GF-=MLz^{Vj&Udwb%>LfZn4M~{!hYy#xttIq2hhps z(n5gS_#As0~+zx;?mWIU`Vj; zt^FC9nz+CkD8aWxX#R5)-?y1sAAf8W2S1Qpk-JcxNu{OFo0Np7U1cI~={K&e-xsYx zpKaY*n+6jdtnvI*W<8~jyCmF&u;jf>C%tp=n$9>7egrZ@sduA5ankQZ=KI{y-lhgr z8g_NAJZ_5qd7){z#q190HqVb(Vb@&Muilfv#3-IT=>#@mblgwUUbzr??t)+27(U9` z-A43vDHCh~leb`s=cxJ1Xz@zN%!cBMTcx)5iGcDVRSh#r=jjJ(s`9Uk1PRlZ75h00 zfS(k=sPU%AqXJ6gLh*E~XVgR1_+kSGcx6+{S$>72?b29lqUxciPmf~q6LfmU5=)53 zyeaJBSIB=&3NsLCc;7!aI{bI3Xnk!h8jBp#>MdEB64fKsZES3@JI-&~bou0gtA=eq zw`u`f#H~(&t^v#=zP@=$%3H*LUn4z(n+kvFY*E<0Wz(@2hpfNG?(TS^^?t`j!Ye|M zZ7BXxw7w9khYcLB<;s*SD3w!|i0nQTnl(|Y=+Le}IKVe?^ zgjPZ}^B#E&j&=#{hn_OUjIupip^Kzk_lPd`Qy;xDIbE!1iU!Qie`k%vKv_Y9CSqOq zx{z!UjF5oMT~*xS)*3NRYii=yJY*vUiH4g#*T5mFuarB^>Z-$3VmTKVz1_(jFbA4T@Nb!{(v5PMHzl%(fu1wbab zLX+Ow+>Tefh7YRZ{eN(bQx^Q2YZenzx_@pFqGCS@itCVjo8wR1bkoHRz&*f0grh^k z&BbzfZV(|KU%UB%@(n7U)HvzxQGFOxIrhCCr&}^nH%b&JC*@vL^?^bb{rC37t7(xg zo(H?5Z_Hum6sS$Kl8^cp7cWZ25ob4XJ9FU5?%6;lSfAN&tK2O1R1v)OIBN-Ou3)@!Mq<($iS!^Y$RA8T+&O^X?>EGuB5iJ`0T&~cOCxo zDde8p7lhB=9x)C6@T>$o-DQoS>d4^sA*Gv8>VcQGDBH!6mBM&aO{Lb*dzMrjub>Hq z^L;BzpP)sL*0!Gm3ZWs)UN`3+W8IZzv75c>MxvR}M%HPeemSCM3Nb00SWR1?Pg3Vb z`SX6>E1li@CsyxZkq&;*(5}-A{)Qp^l!gWPLrXi&{ZV2hjF?UQNg11$e#+A|jX!8O zqmtW~v$G0s_!THd;%v~Thn@X8>7~2UaPCo-nQ8N{kAW9ZL9m%m@{DN%8|OaF_@arU zEBrU(-M4xH5WM9NucMGF~t6fA4Fnl{mYiS1Q0dtOTzGk!IXz;Q7F3DF` zKLb=8B>cs&qhz3>OW7Y0B~r1E(Tm^rF{ALJQ@w3TxtB<9PHEN~Cm46|zd^wLs)l6v z6OQ3Hlp31W4VxH}Rr7Pl?@XvbB9Yn8?QA;u+yPzh){lhpTO&SJ0t?v(`!hb(TkPXd z?mH{)qHo@tTc^GNb-mhodI^_Rd3w7Ow(%G!Ta(?majnU-r{V(mvwJqdM8&!j*Oq&Ml?;yGJ~&s2G zt=wzk+7Zc)DhiY-y`86Ddx`mxLcKz>s(+}K5cAGuHL$&9J$%YwM3cN>PV8Trtr;#cImJo#ey z6J4UlQmg*i$cM8Gn)qAHkz&30gt}pD{{ z#)R3h&|6hFkC%*^f12n86|f)j8}2yjF9};qpi2`;9V@~(qMV4+g!a3`iE-Db+M9zB zj{fHWe2=pVlKLpUp`}R7s zKmmOhzRVt2y5~Lo$szaxmj}I-x2pw`Ltc-sYOVd_U+q?pEKzdD3_JvZ(p2m>g5G9j zZ2|z|8}3je8v#GJw4E<~fy-PGI0K)9_A+MTH!Vgsk<=yHq}zKQbP4SuUde3UEi#~m zNP`{$Exw??YB97YbeDi;(#pH-TR@JOW9M{TX}@X7IKLgu7^c@-{bO#SUq(#c74}Cs z)OjBU-C@l2^>fN^_u6;WEIiLH5EVkcLp0E?0*1gzm-u({lUmqk2^p9xv5&*9;zpr) zR~Miap59(QmOqBV%kh+wq5^&jUQn#_JL0b6s^*(_-mc}v-}?rDo=x&a5yh#_?(AuA zcb%mX;U${kNz!j9RIm|vG)q8_b){bjx>6f`i)@d9Tpbt!l@u^DF54fM#frHLEr#e9 zG<`OLRN{}(621khCHdQ>#sW2NMcLjOHU^(C4-Af#+hOVHiSaMsZ%0;cObiVALv(dq z&ffyJ2!Uk6i=3&1$zL+H6ZcP!rXEX_O>I9Fq}WpTgH$0fEO2ym&pz!VdU4E_=l2c} zAnbJtXPMecl^$aW5B1UU4k+5;q;Q&foSaBNTE`?UIO_dRQkx0k?24 z_APZy{1Fqnk!D=XzyuY)V9Z(1-c&S5{HbJcm>7fe+r$c;Q~bm%sqeMvccnK!xDul1 z5?)l6?*%W8z?4Rm{s_L97olXMFXkN6@e$dmuYx*q5NaP@X@qYc$sD_4!%NcZuD$7R zyYq#&h$y}b;GLV`$H6K0Q3l%}Ke!bGG>bUbqb2q$MYsq*aL~>oD$A!C zT*e*aHz-ob9C%VOJ4%;Ms;?jKytAY5kR^1!wsib6Jc+p75g~bX#KVkKK5$}MT#s&q z6$jwhUiB{!Z-y@#czI=|dO0b{^-5vZjFtm)5Gkbk86(QPY_Dyw;w-m9#KT`jZ_1-T zi13Y?@HEM)jH&>9??9{5J;ejZ?6>wMMXLh=D+s|THn6bpfg69Xb}OJEX!(1AIt%IOuK+=?g<-z6ysqQE+7tzX2c{uCznaUPq^}E4>%??w;R~_;)qVmQn(4T_D!BCVNmaCy_~C z@Uec%tnb5c`tqQRRzxk-~L--a-0z*$cdrjEMkTxY|%GHv)1ZzquE3wrY?X8e2&rQd?<^O}*849LZPz_dA;*Q9&7L8h(|^>E zGFM?oL;?$9f_-b})Kgj6RC3aKkt*ot7#6Mp0@zYf3ZS3wpY8$**Cy1Jb7^Q>jDGvL zhn@Z_67cR{+{?-`{I~n}9}mI*%`beHKNs_NK;?C#mCXUrA6x&V>0t2G4WN+b|U2QQ}sUUPAS0t0PF>{wHIXS*}?Q?x&cy7q)#}z zL;D*rBuLx|`BF*O1s30R>t{O9(4NJJ1K2hI90$pFBytaeel`;YXFxv_beH}{gqp*E zt03gtV*J=)-KF}W9-1&jSV_OJ*Z`HJ9UjYG#`~?`2!ECHY4}dgr%dS7GHBdvw5U+i zRFr}77SPsXD8T44Z{!LL9SQO@ygTWBSNT-qn*-<7Oit0Zd+Qu!61_^G$^(ISt$OHF z=kv+k!9|)j3Z9t3I`3X!RWimQ#|j}fh?8+79=fN~0v9Gjl^ex_^5C9Flf9<%N{x6kQ%(xe$nzNbXQ*OByF#C#xoNzPw0*^4N5M2pISc zJ{W~Ad*HaQ-Kk!vR=z*LsDOdCz=FBY)qqH5WfUN*QcGii!7(#N2?_8{OMiJ?oE3oa6}JCy8VI@ zrSMFVIYNOa5&2uF;aFS$S;@JG>hrwk!HtWvZ~999a3jEJ<%%0XyS@M(UD%0;>9(uP zvBI052*wYCJvqtHp1R22cOo^3oAmu;0{oy|y7LXJj|iu({FUk9_cSH0&wi@+%TV23 zfuK!U{=x1m*ox5sZmGiTp@19=vz>SEy0-mp73bu)7^MFLXVROR{)qTdW<`|MSJhBb z?K>SkdnL}tOblEdbmv}sCJ3a=dZVRbi&3!@S04^r?1ge)lGYrYG<2G-5;W{Fj)hl6 zEz5m~rJ3Bp5{Dnl>Z0Q6wu&PGPLg)5z00S_VDFgbhJ~jtk+pE;r%O6*2}D;Yd-t;0qZ(gKA`N=%&gbb|~I87sf1z z9VVU)VEj=TWa!4K3Ru}k#LV)5KORXYih(*E&SUH$#L}$@FocLshFl?QU1=%D53I$hug(E8o z!pOU{Iz1$>8v+NMU;r%zpbrCVg21VsRONLFc<}yp9O?{Axp#<1^mt2H6>P?U;ZE+f z8Y*xezjofxI&%3x3Nrun;{Au;{QcPfcOLKC^to;2tPCI12nb|<`<6;d_KNEtltKfE zinhN#GGNJjx_&(){y*tAVk{vF9plA=H=_NT*;A2XAi(X1dO{`w8)Q&jtB8sR!sBlE zw#7deI6ziUrhesG{;x$IF=jGo2#6@(R6_&xQ*luKM$qfy+tzCo$Pt)HBDQ)*+X!$7 z!&wC2)hjmPMNA$Ihl2$nj*(l(vhj%eiQtj0nVF9ycCCXd=u!oKqJV#yidon<6sRHB z`R&__{B&?+U=o88v+O>3SUVdNoz9Mlj#ktPoZMctH5d_TC>xoECHYvt#$ zZy@@^va{Y&+vZy~j#pf&FOZWwiWENjuvu$3oO0jyY2l1+U(vCE$qO8bU`OjZkQ@Ka zt*)Vh?FC=UigTi?md|nWRq;|!66kMEoPX7{>()LXmslqc^r5&|X_s3XpHbGw9K@kH z$)t;3w|>FYj;Q|LF&rc~pssUA=g(9Y_-3%PiR4}%p`BfueS@x&D2a0}W2$xOyVOf3 z=Cgia&3Vtm_M^}y?Pb>2i#Xdf%^#jCKz!Du&nM(U{V(M}bmA`8W2o?eLrn~)h1g=& z*^v!6Dhk5{*NXfGZOT?qG&A+onj=171@UOdVfg$NRH;jeL(?T@*n9Yu_f*wd=KMm% zU7oP60u!td9a^k!Umksp`M_*qNE*kO6+(7)w02~EY)!IQ#{STDWG=~8`{+CKmSH~Z zJq~m^hgSa_**8ENsq*h0O(&dJ8{U}~ zt2V5*HqHc$GA(luh;3x97Lz?p6n!CMitZ-PCoa8Ovrt!8Pe9oCnF!PjXhwN1S0tVt z^pcU;>2&sq&Xv;J<*$XFdf<}f^4Y#J+YD|+V)?|+uQlPg0q()N^n`mXIQnQ;~=;+|WeuRV(4g=BlYy z4|c~Kr@VEHWaPcY^Bm>w2L%nEuII;bXd#H+iZaJ~^UlVpM+uGwU+3zF^=`gjxr;ge zu7In1_460ur-!9UVJj3C9t)9FR+LDASCMBP?g8b14jq)aQm!(>fUVQa4eO(aUm)RkyEh?uES6*Gb zKCYS-^?@VX*yp7r6uLxyi##ONA_B}^t_ewe%GyeSrD*sX&T%kD&r(npkXG_^q6~kb z;@8})G9$+N$z1jg8t%oAd@KH(=64jEA-rogTlBkAJBXiL&JlGi^Fl|2>qvaX63n8u zx5Zpm(3P!+e;V7#Tpah*;h}x81H6Ocom_suNMJpL327Ax<3w5E?gguSl^XCtTZoyN zwxFJ&VlX+z1K_xBcS&+JYH8vu4C1WrMHP9npQ_|A_GSsJo=el$sd1OoU)}w<98&>j z^l1Vnb_4E1V8u6eBK%KWsjY5^-zXi)6aFd6Swu~P*CmCvqha!Y)r$RZ9=QLL1~HcU z;@|QGpLM!-FX%NOl;5F^*Mc6Glx)fIn+1%TiGj}R>1rU0_CLY0vMbr1G?>FgeK|VO zahBe$)torUGxKm1o@I>9NF5DXl#XBszw z0KiM44tXq8Zrv72Vd|{2^mTaAV^SjBJ?*W>szU14&ryKRh^jBIZLh%A#|UA-N&6e` zRe=`+d5>0pwineUcoWuyKl@)++Zz=~~X~U>qPJ z<0y45(hPd&6H-xv_JF;-bEODrt(<%C?QCL${krL zMAg(SDdu~I-Y|yj*s4OH-P&BpRsX>=rX&vAcGu;HsM=MK9+TpA4MhUO7zfOeL-gb|M24(5`~LQj5i%Q}fJd{Ur(n#MPt5aYghnZdYVr=aQ^7BB7uL)4 zji58Unb`onWU(r)WC2PuuD&&_U4~1nWLf*w@%duiEG)O-o4r z877$e)fJHbZkuGKln`5Je>#r(%S7m%h3)eM-_{EMQ~1)6iKgA?QuhKlT_+cm9OQ<+II20tTrq0z;qXl2dg0GKZFkS< zfHU&lRh&INbp{*8zL3*foE9fAaF#?ui>(g!B+^7BBDAl9PX3AQqV(S1c)o`8WNuGz ziy56QNWL{UJ68d-$t&nOnH_cGRtKiB?A%bs2tRi8DVjYY0s39sYp?+{N`0^Y39fB` zOKm6|F_qz*N1WPvU+CIL4H`U*%d~Vc@)b6H$L;dq!Nai?Hi@1tiuX3UnXe4J$bQ_u z2=NS`ZT*twKUL|Ue>K>Ot9Mt${s${2brx&nJ5F1=gcw|eZ#a2Ul<=i%F%gX&_tp>}_reG>4#!FWq=rCrwmUsPL;g2OUUUfbXxsw{jAgou zXF+WoaxTy-Kl9pfOC5Fzk0qziwuP+sTQzU|e?-V-T>q}{cO1-Z$=YT!XzQPMelDL) z5e+z(0)7FnkurbO%{QAsxAs4`!Y1deetg0-w%D{zLYf}!3ZngU{&<<#T+0q*Pxyh! zdXX5-_3Fv>GqMbdkpoqF;!2({B_pA4fr6I)Cky^HQub*ZM!C+>W6w)bO@RIcKuP}-kgsLruz@`)Asyq}Q$2Zt>&NX~%~?=W&VBbT zU^NENiNXL}q6A0_NP~-WhlHVig;M1B>zG_+!{GT~sXRZbiJ92OvB{O+PgjifPk!Wp+kkJ+j{llmedu&Ei-Wssc*v+fyb+arYxeVG?xhaJ zy5*4wC4HB-qJs@$a(J=_->^Qqq?B8$z-1j4m-!=N{zLru9$CDDY{D1iPrsOZ+(+x@ z3~!L57{(;6ynUBKpw}*9(g`29H}HlZsU6>4lC&dsR8ctRnTwR&(Xx^bO&3K*BWiVe zWi-1&+8Y|sbx|_9rwsV2=!yrZJEb>m)bwoVY%6WGHxZt{x*doF(Bq6N)1672BV9^t z{e6V~F!ER(&6h_SVD-2Er2abGc9)E$Tx#_EgrQA{6nL{L?96gmTu2VsF?^=BYd<3N zE!qN+$J~Mv7+p%~y_y!=X$Ej?o1nPawO(`G&7j{4iHK{s*R$DfU+PWGbeAcHY=e!U zHD<*O`^GN4Ajv(6X=~lKFxE{a<0=<-iOVDYURPHwHB*_9+)~Is*xo`4#6*|$htnwE z*;?P%#m|wG9-;<4y$nt3W#p87%hpmFY^b(G=nwHH4JEr9O==|qjGinbVYmiM}^OdXP^1vJ1R2>5Mss}#%%viKxuL8%LiKidDo=mqo6atw-W_Vss;OlL8-P@z} z+8pPmttfNO%rtS7{-KdW9I%M@+q*1%hTA0xI7`8gpaY?AM|dN$=m+$>cC9B92!P*w zVMa|U_?bkX)jRafVFTZ>qi|E5Vh$}Vwvf=Y|4;2oU|=qRCHX_tL-mr23gI8WSzJ68 zP~~wC@F&fV_88%Ejpf3>E{m&F9NJWnkI8!xu72g4Zw}^`{zLz)j(nCAfiDpUR$0;l zxNks#^qPEylqhd3h*C=)2x%P)WJr9Me9Iv=JEcR5;ebkE zZ?~>!eg#uUOyKtc?XrK_EK|c|aw?4+q^`c#7QFVp#r8hOs(rB*4S9f7{H}IZcdpAS zB>S$g9jL7YWP&mG+?{u*^s?O0DF9LoUV93DL}4>zzAFRv#<2q*F8;Hk`+xDgo=bZB zFFpF3U*dYZ#@%^g;u6qpA^8_M_I-)cU)71>K)x7b#J9s$U7$M=5l7bX71>N5DzKGD)Ln$RM)dPthg@$9=QJ^ZYhuLSU7I-; zUb9esm)NK8A0!tMK)gkFyh7=#$*FT(J@2s7=4*9qWMdZoRr%cjhd+2bo6X*n_6z2m zrE(r+(s%nvD593h1xf`(I;RxaiA=JjMAEU6-t zu-C`knQBt-Co(G>SSkFfa8aO9;xg*m<_6B-!!HC7&a9~vfuCA9pWde=f73gAp=CJ8 z`QTvwsUHXSfA=|IqXBU+D`?LpH}*8Yiu9JqTTyxxgNsIEE;gQCy@s~r+8Pb!jyuIj zKJUp4dY?3$m5?n0V}W8XO12c`8TFNvPXCACqJWWwS7{0FTz)K{4r-Zf+@q86WN=Y(EA~NMZ^au>xh#PnbO45eVE-uVN-2 z(eN`m&WTStHEnKbEid&bssg3j>@4?U=m~E1n1j;bkh2|U8DV?BE^Ki6W7WBViu`<4 z%|zSA@FsZ9dj;0d{#@c|Ax~k2cgDu&?y<{p^pl$1Q35_0Y>TO#TMZp$0psCqkv(a) zCtjC}3A8FSck*~S|I={SBUa3;LV3=oV=L#+RoTx(a^~}1=LRRD)@MR31u8^8G7i~W z%`LAZc|an^+gjpi#uhh8uP0-@=uMrX?MFaZ{;?W_G*I7{96K=-Ba6{6x%V0Cu@oU@2A zw(m%oTJ*?iTiK2cA zhKA2LKuKRH1hxfg5d(2qRN(pfTK4XOWj8r7F%i%$w5DI!PoKcM*gLtg;Pd@xymcO3nP^ZLrSC7{s)^!T_*7A&%2q$mD>@@?RiJ5_wy z4~9JB---Cn|K(dR0ea!eS%?L!p1{Mtm5r7JRR+xVJmG%j1Ma%<@Iv|Z zC7_`|)WqdHpVWN%9il$6VybpMvZMiUyQZZM61%v9*W9#);JY5tS9o_BJ_N@*!JdQ? zdmdwWi&I_FtH3hG8|vCs2jNVr>v?ZSjb2W0t@}l3h``{Ssp*}AMi^dDkb&{F`_jEp zdCJs=fgi$oJ;e`o#5A8GviNy9v?a}N|NMpCuo3DJH3vR&rEX(@;Hf+Y7%8y1XbU=0 z%xzAtq}xBcvjWR>5Q(B*%y zR1QR10rymc;*kxYS0TX!+miZeUDdOxK9SiYY*YO+{_d|U1ghF8CCEP$Y?nHUF9W_C zAMw&|A8c-|e$Aq!I(Rl@sv_b~Hu0{ZX0O)%sgdsv=Nn{Y52RnaEL(YK9^Cct+q2CS zWZgtw?zMffwk`sStr+OcTDN$XR>>_)+w|Zi!P`vVl!g$l^ZK2O_CrAUHP%bE%Cjwh z6kmuM3=qKH0Nr8E^nE~-c%S?=JE!G}Tyiz7vW3vCmK-&SaSo2(mm7Vz?=aCf6o^lx zRsb8?-^LXNHtF_t?*(R7jiTtC(m)o97+4>Dc@7CI@z%oQ4R>je#!l26l`VR}C%dGv zwH7Pevq-6wd&f--XRW5+)1!=Met)#hs(w$B@&MI7=o3rGT1DWE1(XZHj3!2^ zak)}4`!q!Rdi&z__F`0_?9tTrTcprP)M*9}M_upf6;{P~$+b<}_bImbDtkG7tatt$ z^+@iT7`{Ty56+32P5hR1VzgWOo!%IKHMa}-Ts_xcb&z`A8Q}%I6418~vLfw|c}jNp zO^Vp8nIG6Lu2rN|u@J(@p(yytT_jfBPyVW(I%|A^c&iv8e|1Smqh6GOO(a)x!hqS(BbJE3`mNAq|q>=T8LzFqq$<) zPy84AUyjL2OG&FqceN7p0V=_AhZL{_)t=qIT^=~jwZDQ>T|=W~0mFrruJ6QbZ@L(0 z>UXpza87BHWyn!A)CY8*X5VbR3Yo)6QP|<(6j$-`$(%qto(cj66l>p*VVAB~6S5nz z+Cnkn(;O4~oPT=(fCx^t@|TLe|Eya4@6Y+y;m7B8|9#G%4=pL#2-`s4Xn_-&qRrex-K{TXKGWKFv@gL*_YuSFvTzrHvs)C zr*j=GZaOdPwinI5gedIGSqKmTU)F#%tZx>7GwP$i-R5OXBch}MrjJ*dlI>}bmOBZsgqn;Pzo5nMWQQvRoEAZ^chC-yw`7XU~A})d# z+6euF+UOiK%PnkNsD_rv7zM|Y5~5h2_a@Bf$77`vrcc`>T&YzTug`` zxef4z>Lg_zPm^X3o<^gnqrG%~^DB}B6#kNwE9LQPM$KDpsD(PCeWlOU8#VkLIo&C6rudDXE+F5 zD1OCItCDOL+|^*j!%W&jEaKF_PU`=Zl zY~p)(@|b^WTuZ*krK(I#*z9dk+XOM&b4uA#f1l2;AW@@-TIfvwszpje#J3(=Q^_M5 zRAz!9Q-d6QlGzH2Yr3ea2tEAl9orV6wABcnSJ~gK_-t|H*^!tBMwGAJ2PPZzY`flD z^XTCEs<4r%OyD+&l_0WP0Xusur_WD^$-$Pwz_=b*^Bw>0O7^4Z4t-2HnJWN2YWi?65W_O=mr z1P#6JA)E*PJ+?9tY4;Oy*t9Qo0)U8%T)iKeKZPsD-RC?-^N@dw1aegfdLRh+AC^Snt3zqE7TK=rlO#YhhkZ&X!g;IhF19Ne+FVbnLYM__GQb z)0=L|h)0Vl73-Mn*!htDJqo(epLgqzGjeWR>8Y_o$i3lhsm>GpF&v#EzbLpu1`064 zwhN&TZ3w0x$zFE|wt58dexsA8VTZ>j8u!KKHIg`io1V1RV_0W1#o0{JF{F7lX4X zfhlu8+;sa(Z3ErgIg?iXA&4`YftvfLd%ooG`=GfornA?J}UEc z7QRSqhBW-HM#Xt&VN~K73GuDgN^g1p2&CB@JUCd^mjAo)^`zJ?Uoy#BId@?>Fta$Gp-cPQ z2+ukHRtlU&z1{34v7IhKhrVPF#4ncT-b@R#`N$`&y?;*(Ad6oyRIn5{$~go&kEEE! z_NQ@V{MPJKxvW;01>b^xBPo~nQ^&`ZUG|2WVUU<>)4V8hEQa_lv${QfD6eScLi>rx z>UPVF)hnNv2rH!Ctd|vSXs|oPWX-~S+2CD&TK2K*MZ>TFPAp0cQbj#oY%pzwn{B=& z{`OFM0kX{x!-)h<9b0|r8gAlF_F3owGoQAwaJGgpt68!aN9 z&tY=y7CPRxyGs*nI0(k>oN4`(^Vf1E+%sA``~?ANsagDM(gXbhg8^(QM}84kuEhe0 zqKRC6tsy}fX?deEgJ5m$FLAY4AHd88M`*t(z1l(wDTJ;eDVjC5gloo)ei?8M#y)KL zO|+(>WCxjj`6zRnBdd$xm&t2H7k6mR!@^2m0H5AJayB${%%=r_j?figPbmVan*SVi z42|OjMB|*LIpeV+^`yg$(B&-jX}kfHw_qd$5z}4kdA$4^D{HobGx`gZVZ~!%dH?^| zrv0a6bc5>xTnWNQ%>WuZROORiidg6TEfR$vs1cAy+y8xg0ZbHP?V;!OEmAGiQ{#WD zh&BM+?o#GM4UeJURrH~7PvE!KN=eWGB^eO9t^4r6At!{m#`))fU)ewh{Fc0d&5*Sa zUC71c%)gTZ`pq|>U|Ig}NFWXi87xf=3114Uov8TUE&tR5HcCcaH9qfcYZox=Bt)pC zsIZC8lI$$!{g;?1XXhWTm4_;k<)fZi0ugC%aN$d%gaL-h@9*M)iybAMy?KKODRBjO z5r0&cp#}U14&-3GXyXrBWnDiBf2Mqze(G8*H~+9~swYRB5&m)foa^&>ck6Qe^zFSz?O(>=p{He{Z_H$mN`ojx!?xf)r2#!~+>8X~ElRiE zwjc_@{pmNCci-0R(_IUdb17$5L+?_xWVKoB9rt83xCCkJhH!1MjmX3iRp`DL`fT`Xud`S5hF*ZUQI#r!7sJ%|0bxVIcEJa#_+3je??%Ll4tLQ4>9sPQ$h>=X zG=)Cd3(BUUt>N|5_EC1sSkZJm<+rGmK3+C8+P9UCtbMgyCsW!Vd1#4LL{rRPNJ=N? z`1`wLG5W%;a}Pl79WxPVbmnZ?e~D{m%FsSi0;ScZw>TY*`iplnjiB1Ca|rT#AP<-8 zWAS}gYRS$r=&l!CDH=i$tFGgQIv%U;$**K?`c>oI?PzY0i>0SRiWA+&4z&9_mDlg^ z^-(Rk%gr#cV>mz>%gw{D_J@92E%>2d%_bH+UniT}`W%N$_7hTfLW0^qiMZ;#xiAg8 zFw$Rir(Fqmz?D1R$94L+x;Dtw%kh2VR;L_e|H&um`LFe!P=WDS!}B>6C4Q_`Nch5> zkee^h9UjPT)cI=WXX0s&&KMsDc741~7znOiXeZA!#J6GkF##r~=E>k9#)=f@34wg= zC;_Y*%fQcgdffaGdX2WfLgvZgfDxFgt`evG_H6CLy=AjXi;^(Y_*CYwk*zXe_H7E~ zguE zYAbpDLm{G9zWy9RQ*Q>lHuHV$$pC^yx5owcD`c7SzwmCwk-ZA(!(mr1cs*;^|J77_ zUt~5*ZdzwT(nKqfm1?HDCwEN3$(*+BoDuzP=HelOY*`)JGw%p4vV7psUabe&4cBRf zeq-pop_;Q=9p`eGT?iec?!4G+8!NPL6uY`Of$aY$kI{k++a`~anTNsh+|k;VbqOQK z<1x;+W}KRHoi1QL$6CLgsZ@um9|>-n?HCYOt34G$YJhiy?#HtuTv~M&!@DMu>YRe{ zuJCkIKQMX+!7y0X*^YDzT~(MTZZ>-jFPw*7%fv*YMnccDfrzXaJTo z4NFS>ARABipG)Mfi3EbdJiIUPOIq9R0oZ5kwes;CJC_iThY#P!fjhTfBMRx|^NjVf zLeS{~1z)P!_}ziDOMxW0uWOCVD-TpZ3A23C&X0Hw`XN%iaA z1le%(ib+!sMR4|GEqN%aJPNP@n&bh|C}^Y)JvU@d6fYuoDsz@MFx&lCb-Mm1cjn&* z>i^PUKNtJI!COkqMs1FZi!%gvncc{}V5xZfcGusL)q~l&+ksvGCXs)qUM{Kc3`Gr2 zU=RJE&Vngx1M(JUwMV_Z#*jubTo3|j zv~)?EO(l_!D*%)u#?Izph{1PA-?YycTBgIUl%S@c$j&k+Dd&=c=-eqpdqtc!{c?-e z>oFhV@N|UNsY9tBh7^Sj%3;M`rYpTRD?3Fx?M~D(jpD8pw_3i<6BxHu>3WZ=K^~c^ zXn_#W=v}p!)#XmR#OaFuj*f*rllGcAhtNC3oFq+gS!;q$V+>HOMDs{ z)lewY<6myP{E01`GS#p(VMi9dZf^2bwEU_zVv~9`)I=c0(GsheInfgc}cw zidbKk<;=@}k+L+Jz4D{f5|P7yL$;3g0X*q$+{Hr9`!({ZNZFIiGV%V}cM?4{5|Phe zwwh-0Rx)(6!ZY<9Y?(;@$l71IgxN*>(+exnnFUfwiFT)!;pdcT1I4Q+w4Gt6Eydxr zzf!uxq*2_}nu$z*-8R~*oV#mA6AIFit%LL;&y0YN)bq%syzp1Hq0WnPDTe;zuZX+$ z9TgWaH^Rrb>lN@9(hoCie^~%)Wqr|jgSQwL=}=9eNQQRw*XNwv;@3D=xkPI79CNzd zhwg;ezN71^pAMO8b&(bZtS2MFU#y`Ii<`7yuZ7a|b4Wm<0z#tLEh_Q8U2=hHSm}Y8 zhK#nPik3CUN`gP{V7Yv7g6q&xW|YC((hjH+W1U~ytddJemaNZ6#>SC1Ly;-zyzTvn z_x|j~((XS|^Csrm^`=_wET0VOu=x>X$1u) zMq`wZ<96UcE);vP@&Y}%UOIW}EC=vDFYkY24^)pI1$N2xfuUV$RAd*2C;$$>1Lg+& zD!_7K;8C$%+{YRK0sWu;z`)ccm8OKB91W#Oahc0tPk8g}DT1~^+Wc>Ji9e3~Cs}jD z`0|hMJ;NJo!XHl{8>uXtAeISQvjO4nfQLUgon!&hy9=?fl>L9(7>by#qJnx`K~;UE zHWUh!M`RD}vuDr#UN72U+jG6n)_V})AM7e9=kvL9DTu8Q8(@VjL=W5`a{}7V-*MEB z%WO|KBLy&Oeh%C7-+2+Hy!6!AZ#hrZv zafyo+Alz}{&)Q;RBkNuQX;yhwfW_ZO>nHrBbAHcF@{_aGgTY2SvNT_~w%ot?@cs5^ zS#pVOP9!#z0zmDodZy56`03jj?bhyhE1tz`Y=X$Aa-1QJo)WzDe9H6Jfjft{a>E`< zoV7WJ369qeQ8fLUo7jE2sli|C*JB%ln$Ost{IAAg<GON-pS8pxkL4*DvLm8qOsQHXb%mYnqL^{pa%4F&S}tNdW2C9cIX0*YAq zU}hn_!h6DaL(^CYPC|nNqA|1~?aG-iVIs@Q-z-MDrTH|37kl(+=xpDb>O#Q~u^WU9KmM(u@#jjmE7>ipk#ewO3CY>Szk@lUTY=T^i zOBicGYb(7aYp3KpJXLG>rqfSbaR}|4_)K%J@dSyU6{yZSnN1a?Od%{ycb_F z_x@O(-Dg_1&y^Dr+%>%%sSF#E!Zc~4?D`QP(=i5u^@4MjMovG z8a7U|TeATR2L=qFlk2dsn0e;v@e2Ys#Cb6_kXK0=%{sR>h$|!>Gt=@QvwP`3_t)Qk z_71=q8ybzE_;FDOv{W8 z3GZ9QP<1l?>}*EnjNL81$?f3J?*-B;}KPSjMI0j+iX$IGiQu(e}wQw;Wc*LIvRgJr0Bd|Cflem1--kLD0$Ft z$v1hfM&>>d9R+6Zq;%Ol_ThWk;><}f=FkcF zuUQn4Q2}bNq0TA@p$aBlp9pqyYtJDrwKG&a@PHHW zX#t66vdlWba1~W#5_g;chyJRiN8y3R0n1{(dDs>}ZQd7XC{gZ^8 zMk}xraH7xb1TF;YEs^niE~BepF635tVuRg-je9;|k#$>A!0or2jb27Z-J)2+Ue_~5 zfQRKuHEA;hEdX4*BfOc|?*jZp$`4Dk1kwaSl6l)CP*3%R=Epy_?7Av^br0{LkmSY2 z7ca6u`Wy@SYwAxw>7&Cq&N z7%m>Kn9iQ5vi ztA=yBSbTG6Vzb8~XYv)YxA@07cGtB+qnl#BGkZh>!I)dKei;Ov@qgc~9b35br@{8^ zvrk43yj1fZx(>V#kUSQ+Q)W&loPaMe9%c^qso0cx_Ni%uU|o*IV--PTj7e$#bmg;< zz~j77a~y%FDC_B>t1t-rW8nI&P!nbXBG8xErE>>%HJUo`DrR^)-LurOlFPcWT=0-r zSOG`O>WuuL_L{LJ#+w;tlJJ*3mp*DWZq?=na==P$()S^-4(eY(TPj5bfLAdpX-s$6XqVHU3u5T!l`%xE&ug(%_g_Arfvz>v5|8g^Hr9u4JO2Z%t)@G~=X7r+EWK2zZmOtuT}9@~gzh zjud`SNlBNPqF));tuuQuFB5CWF=t3K0%e&8;<8@x<;N3tVLEE~Qn#UUk_i|)PoM5k{wf9>71`g+ zZ^~RafrvoI$C|{AE?AODkJwoQ4mbl;;SB%Ir)G)EFAhzjZc>k&Kh{OUcad0KemF|g z9-}P~ow>8lD&4wF8=6VOdhpirOb(NIW2@v{w zy#tmSd}7)3;CsXmI_N@8WTER*#}!P* z0Hzdh~OJswjPjBN1!e*#2n?0tCoXA$;>oTk z-30mFXRZU7zaJ~@nt2P0y4#p3rFW1$SdCaOIJqb7Qh7AJu<1+M>3jm7+*Q}JMpQ2j zH^0~~Zj3b++*nrsVLaVU867FQ%=G-3)}~eATW|LXZtac(jfhTNnLji#K8R15&Gk&h zl=vWJwRdJMF;709_1wE7ZnVk5m8VGR(2XpA7C1 z;-N#};{=z8zOjnt=8Iy;)hIwy1axSMb^U9Dym!G(C7(kH4${!iqeI&UHQE&M!IPgd z|FGkhkcN3m))b$wBs-tIGvdj?w{Q)2_~$-dlD<0VHKE3%<`TF7mL4VY3!`ZXm+ldC z5Vzyi`9H_a=3KDs71SmOI&dUv6U2|~(ID&FWIrcDDNSrP)tKn?KcmYiK!mp}N*P1* zPIkg?QiX+S7mky9W84YY^6Y$#KfZBwDv`4Xc?%PBho%MM`9#Yo{uLWltRp$3@hk90 zGz``usUhq+y4~j8e6eb5)|0whwpR7iSEX9DC)UI(WaQ$Qbh1YszxFDq{+FN+zHg;P zaDyMHiYGLC7N(a=3{(`3hb2t^K68w#`{lilezB>L{K{)Hg-#={VcaP?&K5s&1ZWYE z^XDLqcx;1mw#qFKF&-N3df(S)xCQDMK<&c_Lv*Mc?QM*LWLEe)@rP#@Twt7okjWNU z!`8+GQt)`fLT%8Sa|ma;T+T`PersvU0zL?At)Go-B!B2RXw&g~3ByxA7EYxIRN+C% z1vICM1{lxC8h32_bl1BO=MZOQ3xc&*5!;Uc4tGY^>;qbhF$AOwV`RHI=0r{Kks=dZ z-3mzu*F0Dc`$8PaL}}>Ny0hgnQKtIojeO!mll8EZ*T*SJ!S(%9JB~EvPEmgI4496* z8ctdl(<7PUs6IVNCoAFTnHu%m{#FE7NYgb?AgzGUCMR~`xD`aoL(b17R4L4^oAezY z4u!71G=f0T^n4fLRsXm-uIF;NC8)kr8kcCgCADnbs7zhIT0wDa_)ds70MvzjGDv4n zungu33ah5JLmG|s4+%pwSDW@k1k~B|*=-bV&^68j4Cddrk$>+K|K|;D=<%22&Hp^d z%xD8BsVR)`gI%Z`53u%LnVlZuM>4%Nw?I0(yEg;ks|e)1Q?*C<|M`gje5s|F{I!Xd z8x;BpzOgOL7aM)2YeIrR`3lr>YRiQmxyQ^7!z=aEdN03L+QW~2fC5l3j zom(R=5-Wi{1ON$veq-Z}@9QUFv`tNw+{L+}hi}!E-eG0h?!N3H;KSSSeq-2Lz&Loq zqj36oI@p#DPJ`D9t#(JGOvp*dBpr(aG4GFr#4IW>P#KH_AuJ@D|QcHzS|LyGuKsuqVuXPmyh z@cpS)FJMG1H&p79laj-VM@??a;^Qk&>XnG{<+A(8M&9CCFhJSWi9;q$DaC#}y(%0- zrw%9)FNSLrzxOYGQa{vxQbVlcs_t0MFSu*asFjyH)6`mvI6}Yh(6M3!=geBRZI}xW zEN^g=>ZrC>_l;~4t1RSzsmv|{kp49)S41HeHxQd4`2{s~w_r+@Ch3Wq+ouZKe0H}y zM>2!%??}DR2u}KNF>SqU&nAd~WB@PqS1Pbk=vc@~epIGiF~Q?MS*U&hG&Aj|vbHSO zLPAmxn8@}!eIQ(}i-JjfvI00(r_cvj10D*}ZlM0d;H;EbKX0@NLg24SsAC*ch>9u^ zoQkH7*An^+)( zqf^pQ^SgjK7pOp{!yCZa@l~N>z?Mm{Sxt(WjPsY`m-Cs^yzdK3{ zc3b`B4rhhLjQ;3d33#Hj|8y;Gd~qwy?l;>eG_9?x&Q?~@0|K#$744XN{H}?eq#uxf z1D`Tnm(_2wZTdrYcEq|S!{1rv`L7ZLtT=6_d;2Y`=OGzMveQTd$GPSzsA-Z2tL}xA zyl1{@-+=?M-^1Lk@WZU05rn)M@j)BiK9Pf!oqy?-$hyWL&Z>9aq>NMe93br%@R>io zd}Ph#+NEfx{Er_Q+EmrB>fc&rh}I9Nx2(Nyj0Gm;)f?_S;k*0poX}5457#afE)aSj zTnSGts(is=uM^`0BcChuzT#P+uMTDZPtq!@9xXijf zs&{CLjqQ4yLB^wir~FU0Ny*7m$Q2*HXFZYmI535WDr6bKc&p|8Mr$tA_xz5d^sAA+ z`$dpu@~SX83blapOLHGlwPps4uKn=t7oW;942|)<(<8<>WsX)!PxG+Wm96=`8Pa&r zMF)hCeg_WwJ+FAY!whY+3DZB0m_=|R_TVB|&*wO*T_m7;dd2nBGO zn!3|j^-=K?qwXW&l`cr!s8(eZS_CkEU^^k2{PpRB@QNp4O~>FjCa4KOIwk{1S-{FH zl*rk--0B5Zf-3FL`CtCCsQ$m!`C_5#YQq22>U?xf;fA7O+_l=;tJ-PUcVi3KD7l?o zU7H~7B|F=jeWc4s`v2Zh0_2k^0~h#w&v(ZzRe!w)AO;*V{5LB|$F73=H*^hT(g^Q* zOWdEAU)27%LBv@saCRZb=|v{MNNCa>&XI&n0#{74j})4Y_zG?MSEQ^MBxu0^9+I^j zDo9>dhau3|`?3{`@QsaNn&Wyre2P7}n1|g8DH~n>Sci)3xZd~BGQ3Zbg#1XB5RB2^ zqEtG`Ut@ zIF317vc`u85igHT!$Yfui}ett!_mqpU%65WO_wnYQ0Xv^9%beVUCHaBd@5;VP(!(; zu94xMDBJGRUL_QOKH?xaWiD<0?p#CC&I5I^vq4L*ZM>I@&f0B;bbgyTu99EBi#*DC zPx2kgKV$Vr<|bQ4t<6->DOhoNYU06`%20Be;`IpnfjRZZO5U>ZTBI&9DJW_ia1NR_ z1cT8BibN+0Rve@{RYvdw0!G7-apEXR0@br7vGx6q#1GPyi0sUJcl+gCy=G6}xbyW{ zhtq|C(|eZRYM33}UU}$^aHKAey0#N?PYH*qJC&Y!p!8l=2Tf;)*xQCbqs3Z#cA~OB zb|UJ*eRfuN-fbh(ev|Te+2fZgPyC#xlIA`?8Q9a?5EQOu=4zxlmMiJkkZr!BPhgq+ zjN{-d1=8f<$yNfJX#Qlb>?Vjz@KA(S%r}kq14i}e>rVq+caIEDyibdLegft{%6*h# zAHeTF)gduKNvd!Zu%EaCX0q0Z>fVR!=ohC{njiVz(|LRJX zg$TlR+4$$1S>2oSdDb1>eIfO62yN;eJBLPY>AE z$q0{zM(LH^sqG4Rbxqd>{qpEZ$vOWFj<>|<_~2@mLO*%BKE6d{=vU@~N>b`WreiTU zdoe5?oE7_!5wvfF3jwZtUxygiN1Vr6o}E-psv3XHw@-K$10AnzN&u`%it*IoJMOUT=uJ_)_*vv70v zc*@If$^GBMu6p_Yo|Lm>9J#IwHs*P_10$=O^oQ8>JY6seTc|oQ&NygqV_vrm)j^-7T!g)$ zT4asE`**CUoY*FM$3~bI@v5b%eL#3N8JU5n+(*|2ge}F%+S7t3t53w@SqiE`mB4!= z*+gZp%^_&kOj&aSDu*%_i}_>e%IWk*#0yDACthxardbit0u{WaYj_UR?X~Wqu1O<` z>%wcJ$`-H25Jt%2=N4n_?x7>&+bkza0%+}1%cqk@GnUO)eA{_vB!{cCm-jO_ErTA9 z{7PS4o?(v@Vtom(Nhhd{j;5j!VXgzM8X-`>DBTD1;_bKE!bVKcZUOT~a96w|wLF0M z2>I~xV;}#r5lpY!oUP-vho?BO19uuuYV4w$s1ob?rX(1Zr%9d-BkrEEV!ANf9+(f# z!Ot81)jw8hu9YC~V0X`8q;~sS8($=u9|l$m7&}v=4zO(cD#*NsF7v1mrA*k{JDAw< z$y(?q=nWl&b@~zg6?t!)n{W2!XMViMyp!?>r%hO6%b_UtYXn8va|WVi@s^&E;M}ALxsMoaX z)wxO)(~55YA57EsG)`a7HhLX7`c;6^r^g>gzYngCOCj?VR0xgMD*wMY^kC&xi<-l-`Kz@T{HWQ3U!lCPSlIcm6R` ztu}7sYQe z9?B$$w~yb9ne>uHC`S4?Wil;OJPRP=KO*MW1~!8KGIdS!D~)7 znbuQ>)q;01cPZEPph?nxoo73`BN7Rrd7v|{h~=*#t!V5U0prVAqt=yv{Q79X-z|VV zGOu4W?U)N6L$g^OaIau~hjOiAHp*ZkZ{!4QWAR$|JnVk#zrTL{d3ovLom-Fp>Grtb z#hR@UkBPd2klPz3)-m4&Hy2PMRTzUSd?@bh9go z6cG&eKB5k~G07-)+!JUpXCIsgFoAE|@vnSYu}JNvSQ`5kkfnh8+(ne4vhr3EfTK3- zZ6wwR6h!6dB^)oP^K1(skN^{p%@7a-o!%_+=sef;wY?{`Cv!r7-+tx*a>1ts$Wm9a z#PS_dp1O_nOHSm+S5pMCqMgcPjoVj*t?|ZA7^ACN!tYv!V!~fJs+7cZ{xG|wUO_wQ zEE0GAJ?XXRfHLXoVU>fk>Mk0j;U(#oN_kFXZqu^D4GW^@8J4~mHEPpnMrs1@lBT^< z3cj~z?g*1L-j(y}ox_7YYS+H2{DC{M9aC-&6|exs07W@&BVu?yF~hDS-e&5;ZKN2A z4{9N-dQ#|pq)#^G$8m!ECsb320$VALQV=qx2y-@fradF2*lEo!KpuJF_g=y}z8lQc z+4*X;Dz>M+kK{*w|+GMoebE=?Ys6}v`ZO#eG;nGET185>XiLpPhxFr<>T=o z*<_tc>*Yh|_&zf}SES40ju%{vJ)Ra&bi`J{IcK5Np2+4ziK-jv*E0(2io^NC;(jT~ zXv5sAW*SOI#-6oh%Nw)@-$~&iOiKfx=Ymhl#l^{qmC^Z;<}fkHZ;s&i^6fwM$3*tc z5|^v;##`xyqdcGatoLg!*WMG3ZY!kNuDJSY-b0qCP26}CzJtlx8!H+PXG+e7ehPk& zX`Vvr-A*WuCo9B9NzoIc$v!5VJe0l`?ej}iE6j=xNSRHZ&M`ZnUlvWfBoGaD4C`Xa zsg zB?8ir6F2H}o9;^K{N&#C+{Xd4dPUnd?aCx4fr0H*>bF7M*hu`hC5_F-f!Q;NV+orrQBEM^D#dp!YiPLZU z)vt91zj9;5Ny#O9d|dwCVti&I%2C@W%cGwBsUyNxKw{RnP+wC3m+ANF4ybF$2mH%~ zg#AO<*T~C~Gos-Uh_{tTQ_T}-nnjsJjQZ5r6W`$ZU6FwHyo9#{C=u0I-=e;_OYi}S z>efWK+WXyx%aFA1%LY$;o~dD9AlVR!tTZBkv>1YLNr*Q;m;$IAd=qo;@acPk=vPNv z6t>ODdv)r`tA&P>JyU?w>|lXvHN42bb+iG|6(f<;)#7d^aDz1Vb!)cuho*TqEYm~| zxzX6A7#nv?W&Sw-9V>j?OXE!-B@j(Izr!9{Ygu@Yi}8W0)7l%L5nm6tO0aZfgC-+@ z%DkQm*yxe44WoX{1Z$gk|Tj#q?|a~^R(Z{AI5DwH(J`9<(tZO zUw0A{v%Uq=c*+RYyV6k=LtkVdER*4?ihF+SQ++%U;A9?_Y@oW*8S%mGq`U))p@RFmyx<%A7irr37}I_EmE^nqX%SGZ-+L})TqmR&0ok2c2iVp%kY{pM@eBXKP zeadOa3Hc0`B&DB@wdWaH+p>imGs!23K0o-e2Z@2zFfBydKsiyt6NCBNtt3Kr?N3M} z9LEQ0Y`AS;_&sbW?{@5fX*ToSF|irPcSfz3Ig@nL-bd3Ktf1fanUzFtAQ*m^{9zq& zE$<0WSR>zM?R}YP*zickkHLCpZh|4{CQhk$rHURiA49lwO)Cp^?tmpOQjSbmOTFbi z?&1*3nlOuG*H`|f_JCes>&~-XOiCrJY41y}y1J z6Wg|+uUw+j{!Eq+j$326P zBjLreCozNcR5fI-cj|qm0msNy_*eH!EvdF?dUpr7^3%gQP9=}>wqW&ys&Nu+}(iL2x9*W+xqV<#xD-(=brlK z5mS{dfA6guoA6W+OE6yVHD2R{CHNnb<{@Dc|JrFC0*}4=JUK?h(wPBqfCZj!Te}}} zJK=Ah2&6GcN37j*sAvoVY;>bVe z5iVyAKp=B-a}Wq6ga9zsjT^sqnxTs&r?*Vu@{6=s3a$6S7GPrVJ8O3#jt@SR4W48p zZq%4FJW5C2x7;5#u^b=-^FtzKAZJun_t-Fjq6PGP05-7+z$ntY-&kLmU8vPPzF)wW zPE>vQ?zgX+vz9{7E6)l3)HA#Yq;A>(UV^739BSFu;fHk0b3Ii~=A?1YLmGb=p>{)x zA_eC~2*Ngr#mq?FhfT}YZN5peKb;8W`33b3_j!nNt6xD19LLr<(rI_^JvUk#ul3h3wF^TY1VG>Ux>pgMV@I#U57P4i#4#e!A312TXL` z;x@ofMNFu>c0qWpS}dO6LFc|F7frG=iV}9WUrMmzegAdlz0_juJKqEY)R^3dPv|Os z+zEY-F3*zjwXQ($k<5O*d-zk>xcV21k>G&Lh3|0GS~%oU{k(Xh|8cF>D<7oqRgV9Au^~YtHfL zcdFzTbAQR3Ds)~%1fz0;ugtr6uIjB!Jx#nNRC8H()Cy7e1omPdd;I>`dY$ZtQKg+y zJZix1F*Q{7Fmd4;Ecb^FORL$jm0CcRKY{AQ-P2Y4keqr;Jk!JCxb}x7`|md-7WOCO z-u4fB)eCgRUOR4wERR3pn`nEg7M2FTHh~(QT&6u7SsE!wS!P~xb}k4D%)Eut+P3zp z1yk^_W$0_p!9Eopnobfo3fm0=&M2zdGR?vFBuqxCpIz;9$wD*EvtJJ4?Yl^_sfkJ~ z?D-GakY_r!?`9M=#AqR=RhyV2?g+2E}MG0N(U4FHxtS6)ovZx=x1{BQxD9}Jd7J$hsk+XubKrLV^4+fsN z9yxtZi}u^zW^G6|F6)b@>>6inT04NZ4JQbKH2Sn05&>2k)3FYM=Xb633gMmp*-wJ* zdATXS&@kt4RGZ&SH=eYp{Po6nV&(T36ka)jlqIEB)`>y9scjpaKp><6Fb`ni-hyh3 zUdU2P`R#V*zOGOqu5Ebg;|zCZP2<*Md6U6dIwyG|l2I^>&W6@ z8|={LxksfamMYpn$iprLNAj@6IDn;v@yLLe!kx5+-IF}_xy}#Mf>!wjjRn5KG*%9r z8D@^^PvKRA4HG~Ce7b^QE2L^J860wmtQB0!_;0S^u#L;&+s{^m zhF)4f?RwDFsUduOpRy)y$~dY~XhmYa0I&}BL%eZ9Onadd#PnQ=c_wsuDaeDDVsE1! z%r`0U;?=l&V3hVbi!&`s6R6c!bjy<1O=}%%H95-1D#CbHytM}n>60d4;r1GJQ;rL_bxFzJ`fX-pmZ(3wAZ4yX$XEToHZWwm8AzKu^$N)~3Bb=ex*++d5(%4s>(k`kgJ z!iKtNYN)yUYMk6!7R93av?C4@nforIJ`+@RbqHwD9}?vr7xCf04niJ*RTe;qZO0Qo z)P;4=%swY0rl2Is+`jaS;_H4RFVUanhE^)FHMZ(h28WEu>NbyPuO5i&lZ!3ljAL*8 zs5D|aj)%}VxnWJdYx=G(*Jd5w!2MC}1iy>5nlBYjSVaUQ*=+;eb{E0!cgrV~vKr0F z(3cHA51HiBK;gGRvSu%hP4Qhud5}ecbLt%biEMo;PNOFbKB?S7%*T^gY0L%!+N2J6 zlkKFPL3??uFN4O|RQQ^7Relw)<{3l8EDmThK-&z6CK0~3YX4CVK-s;qTLVy3=%sfL zem(@5L+yk9J%Oy**OAz7$y?Rj2bMe7>myYV0c7A{uJ!$2v+$417bjGCLs;QrU=u zs}DS=yJtNI9ZACShoz{Y&zA7JQ_{5fnjAS>i5NN3`l321?vSJQZ?6{hIJy(bZ=@is z&g$$;ShOar8qtzn+*lq>{y-RID&QUtl>Z{_GOuwEwgeL4G+YS}1*V-ZI#20GM$S^r z?+TN2MWHWNCiR;tCG zF9dE?pXP}X!aMhQUrn+Iue77?it9K_C@-T+S7x1}R3Ot{9<#3g+B#D<;1xOgw0U>? zuPZmr$=6~necYxG7^>}}ZwmgRFE&6g5IqBrBPVEd!B7>5Bz-Ph(4*fNu z-hk4g$gY_EEV>gm6E>>=-vrSiTjD>fU*8-$!eQ<`goR9?p>bv z-291lXw=0RGc|12;kkkN+R?>xaLhAWMz_#R+^oe)->HeeP#HBbn~}03d%Wsq*gdG4 zh{N{5oT`EjTa6e|SQm?K2OCz`uz&V^)Vlcavkw3Iuxg}T53~knW(-Sm$Sz6twI`2A zTCx^GA_~kaki*+m7^NJ7A5gZe*6|vO&GnI) zBf2eq*Hk)4FuPQ@IPDn&`;3}jT{=&yMz7}#Qg%&~x*u{FD~ivH$h_j^L>;tMnxLKk zo~}8MxPkgci_jwV5tK6F#tB#D@DHZzoTh@ki%cd_*8Af!MFjdA(ms6m4LmysLk&1-uGgN?VclG%oN1eq@7Pd7vhZwgesG`T zEFI}fVa4Q}io>s-=|^pwJ=x(gmZC9Q$6L!HKp=&z1-u6oQQbVU5c?#gJ`IfM1h%SE z@O$8>2h5-*FR7!hTM7I2D|p%t>sVb^7$O}P%Gy8M%XF1T2nVNRw9f>24X(MP%J#N~ z8t<&`4n9Lcx45ndoT0f%E{uSBzB(`A1Aj}3UY-*tv-12LdnlYJU@(=u#?FgFK&r+} z2|+d3(#pz89lz{L@~sXiU>z5ce&Ab8G@Sb=jA>$v;?($`Laa2>6Kuq8A+hx%+;6v}5Zl(;zI_9X zU|>NCE;QgU7}7I~pA85JQMbVBGS0DrdLm?i`=~%@MQW%;iSZPOw4O2aJf-ETGqSPt%6+L0cjM))qpljA21j3oySk> zs6bU6+5R{lLkOa?WG@c9)2$ISTodPXVHW4OUW^}=>wQyjVX=qbqhf#bTi-S(T?bv$ z!aY9E|_cI*@8l2-nFBn$al$8Q@JMOwr@ zW%!&~6h_`oXnw4_a3bv@%qb}vbp`ctd}S-L2`tD`^QS5WQM9PXf_Zv@W$ANjQC4#s zvm}VMOCQ=lf@yz1%7+b?mBZ$Jczc-EHY#Uu6S3dwJeLC!kskY2N?+>6>IsHt5mOSP z9rtjcFDr}iN>-l?4->VyLG018$Q6+@NMuvG7#J5BKUKz*BKyKHjl6MLeh;;f4}Hv0 zkc=r$$iN&aZe@bR(FnGnh?{v>S z1e~iRk{)~>czA9!{7K*g2&6e8u=PtHF&!lvVY|DA9!u0s3V$VmoqU*0)4}!e**pb0 z{EyX?7Wz{P=a=_^B~FJ5PUnAc_AcN||L_0!ODC1^t`y0kgHjPHF@|+`mn0-1IV^+} z=6tp(m2xa{$Z3^RA#y$&i50P&4`ah(G0e<1+id&4^!|K5r|bLq{y*36x?HZ;cx~6V z=ktEv_x-r<`|%hSSS4?swxHmGpnZLP4>7;^xuk_!g14^E&zJfe06SLQ5XfifR}ylP zQq2i-*iQmhS(JB6*Hdn;RC&k;veDo`@9Mzrh2VC^+l)6d%i0U)+bAzbavM!Tj)L8B zDc~^ACzop>*JBq_a@8wgpR$ej-VXjrkDq*bss8SK9ir- zj+}3S4&P~E8w4qfh`+e^3({S;^fa{9>dfg7(MvapNPp8B%nJ=v-4nWI)h-`d>*}y$ z@tU>7Q*Xv9YZt?^^z&S1&RB;V7O#P|`&C7~!4>0On-{)Z@xZuUIp$)J2Vu&+C115B z+BpT!jFG|j522Q$t2X$A;JsQ_Okw#4`iuW~O_X}dbQ&fAE(;l>W$Iz%GZv^ERJ$SJ7@)iumX2k z$AFMuDY3n$Z=M+lAJ3DEb?L%L7yV30>EzIi+B4oKcn{-*TOL=AVARX}^F^S8^cmdv zX^^hO@h~)g7QXk}Ne$TS3!X;pJ;_0$3d2Vg-?flWt@r*C=Cb?755e-Sp>s}kQWC1M|Oc)v)DY|8og6zud=8SPpm@AirJf*Ks%qU_=n^Ev-b zQ&wiC(=Eu>VlKyIIqX#mZR*%F5yN=yG5J1iVIeFy!J$g2`HT8^st|dpzDsrhSF|{Qka~E;k;C_oe zlgN{Y6LI`hjPNUbO-iugn85tHNnxAc-KJ*xxvsEDZZ}R>{6~y{_lxWOm5olD^CVjF zL=IEnE`!DSVTCLfdQFPy2+Cnr7TJy7mJ53qAiX6_!YuxDiCVFKQDm^=J4ELX)!{`Q zadIE3`olCk*8W#_#p`O8muFV^2isBP5i5!LYMp8?6|ueVg|CTa8~+L)7rfwZ0K}pe z%ZmthJ@ET$apvY4$e}1sj%R&4PpxQggpOsQ({iF3()PBr`pnG#;Db=fOI8?G-y6;AC?q zYB|JpR^i~z)x+bjU9+r(^JlKfiHAL#Wwgrqs{s?K9IuCs3oPz=UY6xjLoL3V;5iZZ zN4n}hmrKe++g7Xb?JRboifL$JNO1YMt@{E2s>;E_*`zu->GydzyJmit--qcvSB6}vI4 z>NQuTfc}QpU{w2)x>*Aa=Y}3JeOmuy!%naeoRLHz(;#c^+Xgi!xf?7{k)lqw_f-CP-V*QPe1o(wyxopTpPnBEfQm zZGi0^6I~=d^f3la_J^ffM!rLYUtbQzm|?6YJe8Z+F@uvOuRraD@h=EZ4Koye0-O@{o&0W-D=2Qi5CjfeD_fJE4ubGx1%MP6$d3` zM|qGuIizJ}50j8i(elQ&t$YmviB?x59OWM%#WzC>hUKPTM4lXvnsZ{s54T-$Ys}w8 zw@x{c7{6SeGJ-YHisi)cJ?~HF3s+$zt@{zP2gX>seUcB1At38hDJB@CNy@oMMYKU@ zBqg8EqP`pwW6yNx?g>E?hq@Oyha&w>^4QrXu#;*B z=(-dlj{?$R`T4NOYd88tQSN;=@T&zl)XlO0@i)&_!}z5iQ8!ajdwtM1i#0#QN8i0d zOVdQXEekuHILx2B(X?!E%ayTmEPwMJS}J3p+@DW5LYH?b*Zq}$o6Y>Oh>mq5DtrpF zi%l-F0jWopYN*=}H^$C1-r|#jpU{Jqkd~E6-UHi*YK^S3p9+wX{vFK%_Z!ZI^mAAF z6}K%$!>=M#Pd$05zmk46oqGO0y;>{e@ofEye&KzT|A&jHxxF3oyeIbQ{vo?%Pxj?! z4`vuW@Zec%qTotI&J$V3MD&T*lPlF$R5>@VGce-3?(jzzs*Ay_Y39)_3OmDYZSo@Z zXN=VIe=tE-YGJn0o1Z}vGP_4z5{lm6*~G^Eb7qfQla2As?noz7DO6)G1vGlX5! zdSaYO9v#UIvbB7OL4OKu37e&pxfQQ*^qO{<{GN?ZtuipB(X|78$JdV}B12#|l9jFoVD$NA=of=6 zO$PTjk$190+obL^+%#Hlx3__(Se)O>);9LvKK+6l+#D1uLU37#B|RNG{Js9qSM|&> zU(c}Uu&4_`)10&P4!b6H$c03Lw~h+Th`od}YZpYwE$r>AzU`_OpZ#^v0L za7>V>vrXLZ#ujI!j8E7f%<_Vcb8dwvLXFd$yxv$k7O9~2=%0Bw<}xOJNut~3_9;Y5 zs7c{P?k&rDe@ zj7>GKjT>uRss)RlW(OduHO8cx9=pamwvv7onl84$Nm#%yj&Tl@q#vA2#Q8o{#fQ`)Ho4N{ak+?H;Ywu%1JJNXoRb||c|?tycvvy>@Fjl$Es zo1wmKQ%w^xOeS_?qq>aM!%FvZX!>4FQ$e~;t|zQzlp-3^j|%$Z&9!-5dbS6;k#~{} zT}k>zpY51-ktb|0x&xZyeU&p9_tgV@mxs>=yyj ze#E16mPl#AZ6%iGl?BXn{yX_b*%CnwDGo$xe@~Q*jEp!jGAM`hM_@zNg$pcQtmk9O z^%M=&Q)I7Q;tG=uv5~LRjrRjT_ThP!s3ovONDS(+vM7FI`>aMq#W-nX=l3FT93>gd zFa~u7=D!Odi{F57%{5v%0GKr)kLKMvR>YfCM{Jdde??Mu-<(28j&+pS+qyApeM~y< z5!1D$DH37>b=j=JnE!=QNU##sv}J?dm5)W=CoR#8(?j36Fb-0mj?0wWB+nC;B{c_Q zu{m)bs#(VZbqnU>6{>;^&jmTPl*a{w@s2H#r$Qk;a+j|8Tuw@J1)86}9SHdO?P!1i z`EB#jDPV$rs*KUbfbzSqHcMQOt!zZ;{y(%dOW67$<$d8@tAp@nqb zW5*WdBgpwY#(uAmOI1rP@u|`|`!oD*i&kFaS;sx%s}GFbFFz)Gt3HVe9VMQZBQ|j% zoY#5sD4nU8T$wAQ&XLzpaLHPNBWCxURvcpTjC>`M<*L(uo%TPy04QkbV^2xvhm@>p z_b{JSUAf+-q@-`>lTvx#U8!Rx`>*fdOuu`65zib*<)8nPeY|G;>KreDDb(gLA?0TG zDbx`v!tm7^zQZUP*$Aj%YQA|WL1SavLY*r6uhUm7ud6&LS^RVg>_}o*bXem3!-)IE z8|dCj$DesW@vj;z*|Ua)j-{y2v1YUP-!u}O1tu&*GAd56@z(^1j1ldm&V zZHAsz=Mopw`B&d#qrd-rbBtB(cz^lkC5vL)AE+YoMi%0QZv`768NjCc2g=L_$;DUa z<$bsNuKHb=mQGHr(g|3qIV`qm0PouVyfF7t*Z_Rqx-s*yhXd?~*&f+ztc`ugtPc|R z&Wv0y#Sl<-tiVF8VrQMPm}(yVTLYUhtTFn+%QQLXybjfipH=?AO?JjzHsQlSn>;}r zM~>BZ>^L9o|3?1nC7DRCyUCZ%%UA^4+&W8Iz3%1y4fd+DyDxd|LXGhv@!GrzY#yHQ zXDl-rXA;vMajvOalBn5z=MKZae*6xP!!x~3z{~bD8E5qkyw$mEdN|IdHcAYB6*h=x z_^seBvl>!(At*}nw7XEtK)V(ijz)fE9H9Gp?cu;G;0smww?~-e zva2SBBD{03_{*}f76X_*08!Bc7RIv`NM73W^#hgr zC}<91s1hpBo_Mh(u=02?*coBb+skGASQ;jNCwcd;9(QU5T!N`tx z`4ja~rq8$+o|*YN*RG5yDS1+iPVPAqa|GQm_JLt*kN*oDd$Gip3lDsJ> zn5g^DFAYL}d~g79fI~LwRUgmeaCpe2S^{YgNCMD0xZ;|i&^bpg0ZTPNxPQGUASD*r zLPqLLJv$5l!t$X@nWGU&O+;|e{0hi~&aC@ms@f!r3{-{U<~rg0(;z!-o>7aQpDu}$ zqueQ5hH7l;xhJuj1j_r#0~5c=FJ=6P?eRI}26S8sV(u&(ec8&#)}FqztwQo9o%DH7 zt1D19PmY>8e(P?_Y=4KYr-vvoA!wq5BApT4!8@txc`XjDufOdN7V&qZ(Q}1>iH6Sa zdYP~Y4G^V5ZU6~xw=hG8ebjyh$4y`!6tHm}%P+_#BG&*jN z>-OO|eoUVI#vf*0>@TMnsBZ>-`%JsQyW3Y(MDNKLbo zZ|0hLL46#n4$p*u%k=2z5w&n!JY{GnszhBVc6`BmCRPr7tDm$cdh zPO=ME_XzO!`u2z)N95t^`XkL9SfaZV%MXDOir1DCM72_5&XkeftLw@u9%m_$%eT`bTr(osTca+fJ)zGmjGYqy z77Wr3$IE};cpN)W7Ns`Bi8&^Dme*2|N%&wjY|(y}lDoEAS1)6nRc`L}CeT)8>iwRt z<~I5dr&xaPkFp0LQckDJC?DO@=*>cG$OzIrR!hE#8rHOM0pWEn=*lC$^2w8%>GPES zZY?S!oWyY46<1$sl8zDoN%GNhwF_vR9BL#yml^mDo%T|r+UZU5_x&iD^;gDB;?8Ej zzxOloGdp%ayR@z-g7FmsPvl=0dzThGu;&ntZhVz@<_`DU+!XcRA79s5SwvfF4nJUr z-mug9d{0u~Y7Vb%X&$WZJB{&Ge}620Op>AY)$HhVO8GPymEQNRsU+y)&H5%oEsQBB zeU*7gy4_e&Y`%4vr8-9Vp{i>8DG+5reZh|Y1+PbbO@-5W5UEi<)wqwa8kBs-jZlk2 z+3>R;^SANh68t~p$+-rYb*u8?HbMUs1ZZcTqt@u`n3ew-lzse})}PAxDD&C|{UZ$YM@PFzCYIFvh-R5wuPd@K^*JH#V#xIU zeujNK)6v=aDPy#u=C%B3Sm{%=Y8VAlH*zPc?A9{@fp-#kEI}JBVht4N&snOa>61 z9j^y6+3;BydBC$}v*ae0CVdB976qmPVIBdEL;~(FhTVKU#)pUFuVg{#)l#J?|L^o| zT@(*unt$mj?h&R$CZ|2he|d=D)edu|yi{KCjUR#$tIT zDs1ZrsWr~uI1xvsX%O+_6|Q|{>$>2#WQ$Rj9J?oQ#2g)gq^Ebz@s8p`2=wFuq8%D! z+#FA9N|F6zYVl&)sEGndKrFwlxW36#x2GR!q}$goZdZbizkC=t2U4hn*)3jD4c-Y% z)Z)TCJR8B(iZfF@_jWu};r@3^$FwqtYpBfL)T|7jOgAf9b0b)@F4z0aeazrKJ!!dpekap0H@LmV zOt2s}UHGxWERympH}A9Ct3qi;Ah&hTTz1+y?Dl>$M!`?j;@&so+M4yXkmIN1eM32d z(g3Pyx{cpMD#kBJh5E7HXZG=?3tPWR?Y;N&&utx^R)H_l!#70iwpiLB{PIYK%``Hvgr@8sBP7M7+g`zTtuqkatpfUL#Wl` zNh|}+pSh|lEIMwnV6?LE{t%MMAK=ah2m5>Y+Z*ll@Z4jPeSewe>lor?;;-u$P%Y07 z327?mg-A#T?n9WRWDyl1Q{s@APf*zG(3jQ1*_j}n!RZBWDN^I&&}X8MXAY}U<;DYp#|#wv z%~=ElsvJlB&``i){{oqzP)L12_scBRblG}PyhT!E%VT7SZg9pL6{0WPWY>tL)ksS~ z`I$4nd~DNY5d`D%-9-iVC4_oeG>e1h^14fz52!tN7I{?t4{Imv+I^7<)^<9dRJQJp zvTI~brDsof2e?(2TXCRx&uadLSGhFOHLL1IoR-zOmCR*wMO7>=sMTIZ*f~gWK@jkM z3sh8`btL>7-k=0N7mOw?4E9!zpzAqkv@hz0kNg+9@%Sa9%0fm;98nNBz_{U7eHSOx zXZDkykIB!$@p|m5yXU;PKBcme^Mk9kJ|S1eulG$K_Pd`?4+J@2U0IEGE-FD;cM7AQtl4y=8R zU)u_-vl;JbPH9@nEgIuDJX%;i5~oS>dS+QwRRtcGJ#>=)+D;a?!?J0EOr|E|y}s^! zGA6_z!B@JbgzjV3wB9uUtP!X@IUo^&I^b?X@{tY@mw|vF5RV%Wx(MS0JP#2-OGqUO zu9~kb9x50l{C7X~`BmG&J9ErT%*{u%nns37_XY()JV9&BYL}OJ+-n!2g&SuGtFOG%R&4pz- z)*hFUQi=3JNf&27XC2NYKT#I)q>Pu!3BT22mOSPD@aU$K%qKTEES!Rjc$w_vM)=F6 z8`Oxi@RG%(dF`>t*hg#pYg2wFu}Z0kFPt6U(g|E7fLt8GJ?kon8G_=H<1JRI?^W&T z9q>dCKV!Xpzl=M-f6%1rGYxCV)UAK}tF7yJw-M=P6uPfQ^BJ0?2yz2WBG+&x8zvGI zBmKQ8YJ^l>ayxe)?D(ut?H(%iMgt}y)i`Y)tj35=H>_$3e6A2zxuv}Jr!rV84VR#L zJJaQ1>}w3BC0Tj~Hl^a%psMI0gb} z5&|7H_<3F$v8`aCR`$14wZF2WR~A}f_Hm5k|AV#c&ZpEV+ul`^x@Glmw$kf$B5Feo zIdr^2FW1l31)o@^T+YyslPSG%WL9B?#GC#NQ_Da+0SKq*(r8m<K6}j6M!UF|$pC*&V8->G7!WKKh-hS1!3(|L{ z&YkTNS9{>FF=lf>_6uW&rfJ?m-N1k=Si@8Bl5yqO=i&-pK_@+iqbQmeN1u9AHMcHs zsIVn0aFdcw3!a(%(yMsb&f^~DCQS!_I!nLYvC+Mn!iB{6+v<4ijME%eQL{ybdMLF9 znbYXT4NcaHr-+bHyVsEvZoAGasQeRN-631&nk^aP735rHZ{$N^S6eh`7{^cQ^4)E;2n%4F?~NcbmA5@KyLeN zQ`o`r)Vw&&EEmXPpbdGULv3Eql|)vIjo^7L%Aq8mFBHb171KP?O{@7-;fVmEz#iZ) zP%&o-7%wE6`333Xc$hFWjL7sewvp#%=iWl@#Luc1quZVaC=dh-{!(;HCxW1jZp1s~ z^l}S-nBd6W-_Ea&Sx{+wn#mc@ogTJrr$3Vq7dE+AAJz&6c+I3HYRE-q;OhCgr#E1G zU}{e0BdfmKeSt17z3Z%}=6jV{xl`O$r}|%^ab6kAQdp|L3||sWw)%-?Ay(Qe-v!9} z$!VPYK6LUurhg&F3uH}Pw~*d|BEHVMl}3RMQoQyTH&v;mVOKcFCHW0#;4?~uCl1TA zw|cH{2WXH`LBhZ%8)O|XXS!d2Z%9RZ*b2FVm5r=vM3d9Leapb=)jd=An9w587VyyrJz%=YHd(a|`snLBDKpCNxu!dmN&_dd72H;`@$XlKDD}Yk({mC2+@(ZVfh1kODKUjfSeY^Y-={^ zyH0R(LLi%Oi4}%isPlv5DA9DHe(3G{GTll!nE8d*JKL=><^2`aqEIm;Aj?uWO{_Ie z>J<8pC3)&51~&AlRbjkCfVk;4-6@E{ZghSfIGA2jdn_QAUw=9V<0@esnHw!gt)}@% zc)J;CI5xIX;BpC>!^g4&7GE`w^yAdsRSJvb-xL_%=UNxgnJFH(M!GJ-_8?Z{Ls>^a zDtH0cyFox7jA3=wuC6fckJ<*18wli%`u{q6~k-`&vMiRRx0?3QElfu%>5Af!oU z_q&jIMl4;q010j>qvBv(Kss%sCR74{@hhfc7bmmJ^}gaM`2LjiL)5J|Za;o{`WX*o z*Cb=C+7{Aj^y`B`sMn*GCtz8GrgEmx(uZokeND{T{6XHn<47?B?C~2BD(<1$emTtB zD|qUo1?-7@;&=QdkVl25)xE}>o|2koq=@I$oXB%&Tk%vilArfmhMce^`kD&YAMPHj$@q`j2qi|&T z9qUR4T-iW-XJ|pp#BuDMl^@LRk&OE&~ay>=A3h~_-JX>4u|==?e_zG1?= zBzO-HANb{P8M1&FzcQpc7m%Es+~AtuTazd!66Gmh7%VfoksWyRrad=QPn=}5r#(F> zND76VqQN#bm!p?%0UA@tlHfj+&;$(is}+zjZp*{QY7|R}UDOttE^m_Sopzc!C)s=c zD=S(R#ER{h*^@OPj$S0IdoJcf?YraAi8Wz*&lXCgoGAt*L_<@y{ELdF&bdN-E;j4Y zi7fq;FylF}?OU@LeI|}^^9u*EOX;*8_Ee^jC)gB7+)sVLQTIQX@#chaEDdHJvY!%A z4|~o+{L%LSMe;jX95fdn*%`;0yyK?WXNp^MLY=(Bu{N^JE9u(jtW4<-nxC*jhl= z{vor4=&l1A&AGR_l^K=~PKk;l=!>hYSL$LO?$f+$oH1f{gxHW~AwD(l>7MwZ98%jr z#PEc8B&qS&DZ#FAqM!{e($>A~RcpAQIbg_vns7p^>_e=cY;-&O7MXkkm1dx5vIdcn z{ff7{iU<*qx1=G3dkiI$u#c88BaF5%WtV;^aX53#uP5Z9OjWRzl(=v}oA>8u^d$`H zPXgSH(VcI1-GuI|t?w54Ki;NBU2sndgE1nC=Tj{?^aKuvo+ZZ}UgFW#JGPtCiEGY4 zaU=6KP&LImqNJoWf^%i6~hZQ%pjfHE{V z2n-Jo1HW0uCKN%XvU1Y3d^6x_tGxvwR1e4b(2tQ((p#YBC*IxLcmB#;*>+aQ-u9Oc z*PTZCcvk~_EW`tp{g3BiI0-+*P$X1{=`qz}1evDWT+lV@B#kL37%d$;T9tkx^V~K3 zp-d`s=hU`5(qoaLLeHkz56(f{^T!&hzYbou^ORQXTg^~p@#y>G&A#lSoGyucCxVsl zx91@mm!$zD)B=}vQ?=0>f8D6)OQ>y8oC7!5IBAQNagovu*iMs@?PM)bL=MA6pmuTM zYeL1?3s<|Qu3Rg;b*9rbUgPJjtDBP}xc-9T5PfAuW@71>eZxT`o8lWaergpZwA&4* z2EOXu)4Xje6TsnJ7BrGV8c<_hQ&^p*Y8?ZfY3BIQdAD08!Cx+3;06<+4y;|L*GAx| z4Pay^ULsmu8!P)j242g(Gw(8`ShaP*<7`#ubW|J2QuP-vfWaE@J?g}Re7yYFf=M9f z0CsvrL4)KeN3?v*U%&~L;x7n(Jg_T2iX=QeDDbT3M-`zdtX8L@XCt9|u@4`{8Y{y1 zM!euswTsMBEl#r$lf5XaPn=)!!+F<}zT-|U%PSGp+f)y-R|3NHVz_?qQ+&jqKm^S4 z&t!kWWvNpbFTP-ZvN%R}%Px%ZUa2m6jc16K?d9sbq`#fdO|7|XV3zx6h39$OnA7Qs z;fLm^~wl1U9z~ZUd^1Nffdc0D$z+->M(_e=o{!CbLpD@e(Z{3PP{rkU?{Be+v0OU=* z+lhSuk-7c+-%P7_gKL%tzG{TS*CAY@#Gi!+`1sIkI+7H^eBxMP(HH?2BRuir?L{ko zomk0b6L4><)X$#=wnUt_uigT)*eE4TwRY- zU~=?SEU;vVSuq58KtC=7W=JUJ@fC7;r2OfRA5VgHdxhMNqemwh#MxhcL()9`q|m5uS5=w3OnNArvSX#4anxkfLo*nD$yLNyI$ zccsX#*R-JE>tbi9oN*}1)|nkO;%`uj?DhLu*f(9-IGxCou>1lwx9}?(T3Zph5MsY8 z>mn=8qKtyxi8hHI#(hz`ZKzpy&oN|s)y#fie&5H14mK|Os78wCnBN7AX3-6(D2sBv zA+DVHl%YQrYh5W^hd`tI7gnamkur_C#*ELDVwoH*Vhb&ocjqh%zM zHKx?0nKOh@l09`d+ECwtGbkPg0UyhUW4OU?OA%gtA>GqLci=XHicYz0&2-etETA(f zj8Up6w+mBG9Hz!DgREJ;O_!Lph%^PY%I}8-ibLye8lZvWe)Y3|IaQl-~U`MNa`Y664Y| zeG(xwp-T-C2WjH4sNV?L5nzTMR!juy6Ak7D3I5V9eKqU|*lGmOK+mLp zZy!*PKXekc+G?-Y3JM>OxvfzRNgk-CHwp&zEj^e1ZEyd_hdH)4tv7@J?S0>V8o-u2;fdYo)jtR2bwKfCVUW#! zaadK|Xd@^Q4`u4}YD(oL^St|(L7$P#ou2#otHTp5@|l;>q_hGeJGCO-cfPCOlceGW`|va z;!{W?9fI@gC=s@0)Qh-QCm&LmW+bWzkt4Fhbq}r~c3(Ov#Dn8(j%U1%C*cSFfJORo z8osz~eyf*93r(-F!Ah%E!}5c|5b_@%kessSS!Y{=y8CVGVU*JO+axLK6K6wc<4qANy3=|`)$ z1x-UZGIO>6C2PO?zVwUpxQ=~-wJmW|oCMP~ou?!TT9qVS`$3@XHIgUxfQMe8h$f$? z%Q$XTH-2C$&UPdoi(jpcWmj2qCToZg3USVKgxi2cYUa8lZ*4J;k@qjIu9N#I|FwAN zvN<*PUTh=Dy(jgpHlR${7>ecz($4l^Uj1lg%H zC^JjEx#fgJC@s_aZNPp4?6Iy35fnm8K^~S+557czsW%2klh5pzsp$IK_90I8Hbn4s z`uS@)D~+G;g7&Gt9aVCT*qOuL;7T}uLUCaToxhYp))5W|Im<_YG4IZkz!;Ld@K>;X zhu9Pb%_6>Ii-IZoHV&QH8^|5T>g^3@ffV)4J?s!y8x!uhZ=cZ!G8(Da1S>g^WC<0& zoKPXMM%4eWJ;UfFes%f!_OCtZFvU_)g{GW_`80 z^w1=l6o}&bm!Ro)O))Q+@;kNw>PEZeeo?fQz>fnf$w7@FVPNd#`)-AlAVJOv%oFIm znnEKpUQNAahi08*k>2sWVk7?X_GY0M1>8wKt0j7ds8V*%&SMluO+VrqAX`Y}t%Kpp z$dW@{`#BHgS0BpKPX?D7$b5hQj`=pmFPJh;}Cm|L?5_1tST4hD$76ht`4F3=BrX3d_U_w zZehoyb|ilPQ<3G9?+Nd8F5R9u@BqbfQ^f|@wmwmc3PcC{7eWiFlzj6$Z~Nkh9}Ja4 zKjs(27s5v5&{3H%6Tdb6unNVoQz$E92=v5WUrULBM$I=4rgW8%*>2H79Rf0MxK`dy zt%z8z9p?7q_}vWYpAAI$d?8q#dzMCNqor+$tt!Xrk$(&M4ezK)psCh)=~TpK2c6!t zl24`-0|3v!ihcgn&mm)-ePMuGk@08FH<*y8b&%ySN?-=mOhlJ_?HHGX!jqwRo>HUez%Zq$~PLLbwh8xf}Mne(=k?{^qzSTUBSSmh)=E0r*TZ``)M>vafAwEl1W=iiRg|M1Y?#C^|iSZL*AI*ts0?HNuKO$`m; zSkxh%@FBmdjFl9{#BG4>kPyh}qzNo724MDw<2~v{!!JgP33;-{16;^G>PA(lgyeOs zsLH+ZIS4&H9<4^m10l?Lp*!yGqM*_oMkf3>X7>9m{_b3^48LtW3j(WCZzPB-q)dtU zeS&`hr}E>}(b`=B#&fZjuh>|&PB%1` z^xao`^8sT^-RER9!K464?fTp>GWjNIC}{@!wBm}Qj>r&q#JWj3xpTFt=a>hAvN$085=SVJXXD#Xa`hWsXlN=+y7kBgLqS< zR}4wzgyLi9*r|qX;ks>Cw>s8c-x<(#k(U7zrCQ40784%54V4TRM#KmWLsnX@Uk?^t zTO@eR1o2`!$}ecKz8?GgI?UcQGdHgqL+g(TbZ43_QH~gJ{5|_${=-QBe&_#s@jsvi zh|`UOpZyI*dwnRVDohXUf~RQfBxo9KY_@D#-UA@X0Z3txTw*B#xOQ|m0CGOAQ(}Co ztV}wY?`@X0tjL=NSIre%wVWDyH&h7XIdN_;E_Bfcy>TL*|0^D-`I%t&n=nTxa8d-G z+IIH%qJfuDR0+q+>NaBS@ISx1OzpkkqS29YGFk~I6xG>247{2laFZjzqz8`x5$E>^x3B2+C}*I|ow;XS z$D--3R&eoI?^KMTa*FfD@G#PK`Ej*@6a@oqt8TH9oj&wh)Yj$*1Rwr63OsDW|2UK9 z>9nga=vv|C&DB3QhFz`aM}#11t*>{NE?0|G$y3gvb6>o&t@#wo_#@w~=t)hU+vg|M zbZdtO765c>d+NuaJ|iUwJgP&(?oL_8oa>R2l8?Pr|A1gK3pn)f&ayGDu-nG;xIUe! zm5^={4T49Rvb&+l3JXW`bN^*K|7G3(`|pbA`-R|KG*}EojNefM?r5Y7eQIdf(ahm! zuO|)cx?-@U1iA=JR3j`K@k9F<7ZEf8-sG269$qe9EC}h6gK0|}s%gw4a<@dD>xfQ7E zeaVTLtq>()-{{nWzr?6VjPmGU;6jg~J4UM^s@y-byQkF0kMiDqy{nW%YgF8FP~I!+ zspF!}yvv2aeC>i@jNRaP>&uO( z#0$#Qj5^44uvK4MFv}2qAG&D5I*1-OO0)OQI zT$U~%qwTvjoEEs_+>oDDTHH`dhoNhS>bgJQv1KE0G|^zMaPF$YKCwtE8z0KY>J3X} zDUCbAYI&r7E|XybQ~+SS~o+{Bd>S@~L{;F;+u z)qObAVyK-~x{Y*4Ybzb$;>F#ATeGCCSYD7OQ+xNgPISf5e^HyH!kt5!joEG&!Bh2K zWP5;+r=ZIRs9|KlHcos_&yqO3!|cruL((VQ%sb&_yW1J|K_`C2niJBx$>;`R?g& zafwsq6_~$&Fz_PBZ0IkejOKvd_ZAjqPUK6535!OWx3)~&vEfgWZMQX}dEcPGt?1UT zo{+7AR88O}W8xJ=zz3U)+ZI?6Q#yDn8&tqG7&KbTvg3uv6L!T83lIX!LA?g=2fF^QClcG`y3_e11PgV$n{y#KWfT!W0 z=$443aN9`z`PkT4H}(SjcvgN^|3I=(D09}8EwFBdjZ(McS}2oRY`P>^4@i*<@*b9pWCY2~P&OV9{{;(Y zC38uk^#@QxhLRuQHy29>RXa4}T{%xGS{q6EzK8P+PuVWm3WgkmEv{W${UWKth5{-v zDAm$~q3EfNE65iEi1juoIL~dAmB-sG6)2^%j ztw+CHMRz<{w3&{h)E9Tgp?ekF*TW3cZf%M(&<*P5D)ZT;^4wE^*m@UG<&d37Gnc^9 zTlb_>?hhem4Rl%B;6NStDSX}JfiA|HN z_*IjTbEegV&$ZmJ6tI7`F=Kl;V^S&{Me7G!Hlqb6zgv%+Jzii;^w-7pcy4ND;xEy7 zZ`Co(Ir!=xVDWw*u%D=zhGkcEP6p$$p6ptj{~XADr7tiU@w)23S7|Kyxq2MLTB>ZG zvhL-zDa251V@w9+WU|unxQ%Ka0!R@8sWl%6*{f!^65ep9_SUB+Xt(DZ>+5{lTnf4y z)~NAns;n*rCEa(B%`kBYMqV4bxZEZT;;ABYcwp6g&m7~RFl)n7tS(J=sg?@=hXcxw za^xYsIpZ_xCJ>&A4~jj}5&=yPX`sJ6Vo;}g^{>F^@88bm*3HRS?6KyN|SNA@q2@fMlv-WHQL=~vX9yCH}lx} zG+s_|SNq3=L!$Ri!IP4#eJ2AWYHF2QnF5CSWGIpI4Ygyx#ph6)8GVOS_G39>Z0bD7 zF3v>rw19Tk;f(fR*~g*jrE)254}WJxQ#FmFPgt2Of9?gFGCwFFsE{6{9&Zh7YGALb zt-)6|0fUP%o7+6YrNq>tW+w?k!n5afL@f8VJK^!%cy`+(#=;}-D1vu_U3ON&h87*v zs2#Id@u`W;AL7uuqhCVE7&RGZ34%W7_MYuFvCy&FUCDhRms%Gd{bMU+x?$5)L-}m- zMtzjxCqe!96|sNHyliuHaGY7?gG~9%tC>~t#Vty z9tLS3Jv|)&%f?BGU@zXcZg30C`u4;Zg5{^ZM1{UtiSTGuFeDmkT=e@}7!HD_HWqeK z#2cXj6U{;Zl21m-g@f=a?CWwqo9_Y5T6{BBzjrBeJsv7ZWdvg+IRUTMv9}GUm z+eqDXM?;)$U2mEys55^`KfVns+fEYe+xOkE_X#`St4>LF8U`D8cje55CKD8|?sGWr z_5X49C16RX>)SuGvZm5$T5NHdR$HajU6~n=nOQQ~l$r~brCBNN3kW#J%1KS9tgKup z)67h;G;ss0T*{4dBUe;JQW8W(1O&eKM>FTloH^h3{r_`aO&6mk@bbRz^E~%`KldX# zxb&;?@jCRJs)0B5(8mfyBa?>X3x0jBSHn-X>XH)I!bq0R*Cfzm&+u0Y-qj7K9cvFbesRAzL0tPZs9^tVjymW)W?1gK8neU}rGp42u5g+NRTNnKhETuIPB(4#I}Z zvF_`DN1Sj`T%N67=-H|3t~$Jr3;q>bvs*n>lDTre!qXRAe?yecLw{V+|JUPxRb`=@ zFLtj4cMJpv4KOoFeWMfuNP>%2w4<9Qmc+gJY*y5{!?IaD;@lWkPJ%-GM8MJIZc0GR zjBPErXgCJg*pW>&2u0}y5P}Mei*NapKpyA;nk1lE8AW9QQ6g(VIRhrLr*VaKXwpoz z(HE!Ms&lEHCEm)7HJ)1y9P|FXBW^m~87CaM-)?S@q;a?LwN>kfZ|QY=b%f$JySJ)5 z9hzaxOJ5u0XNJLWemKYV$JOkHJQif;S|hf*%dYS0E1cmlm(=;cv1)ZkjvGVMVL|kf z9T_~(hrf7ULF(x@zl`_&c%Rl)BC>uInd!O1XtTW}S?V(Uum_D(fiwrHqy$Fa_p@RK z>PKK|Ck>v}@)PVon`o30_3t1h^g<$5p6B<5(0=}F*5&APt+k_+OaBBL<5XBYv5Q;S zb2X@Z@R&88F zec&Ma5wQ~|E%#EAQ?-+hI@PBRJ~5X{vst-umKdQxw}n!m+yt$C6sHl`(XkVX#)~U| z)HxpiMEh#%h1c_Tm~Iz^zMjZlVq}JCu6IKJ>*H^UNU~!fKk|e+hi=mK|7K7Elr>ze zfhxR1*KVl+z+>D=(Vx8yHT!GVVj8C@Prm}m<{TgdyI-Evv zn|z8c;kF^v$%be4%(j~2TbmCB9gW91Tiu$_w~01Oh$C=a2=iiKEi(j!jHl|%1aEI_ z_)UNY#`L;}_l$(z*QXfjPX9R)f%j?TnqJ!pa2DSc!zp)SXBGEfX!GrZsHYF}STy~Z zc3&%lf3 zi(b8=TruNSN$Q|0JG>kQ<{uIMqauS-GNxz^?sOOA#(J5rOCKCJ7tZhRkmE6CO!2SJ zGl}~>>LO83k|!R2O$=K{FmJoeU7s1zxwCG3XPd9(0&$ra-ul+Ez2s#R8C-t%zc-KH z28C097JJd>@@8b2pyx+D zTW%MLsp0D7HUCsd$THxr07bsFHJDQc+M?!x8e<+{co)I~Z+)X2|85^{b}`WgJYQ}C zAw6Y~vK*abwlxRfQV9(JaDYBCh{!Jh5RkBjtweyEYX@1L#*nUsY(QG0IS!`V&6YMJ z9Jv2+OO7bDHJg6d0?bWlHA`6EXqmH6DEjdVcYU+{SPA>hH7Q4K(fV`_Xb-hj3-%H{@p7!ucKyC;jk=05^q=NGwYkt<=#xS?t)_F?Hy zq{+=EeoSpgFWp}5$H5)Xql3G4A?r@c*8$6eVh}oYh3l#}HH^($S9w=>{q%SGoCmuy zsjY=p0YA(weE!W~tR+)B=xzO_uWJ>`W6Ey;%zt$(A~w;)rkYMhfD@#fjy!f%S+3nwxnZ% zyh{5{mnv`jeWa|bK7w|qZLdDZPrc0fi5l0!ze`y#;-kwp5<2TGZ?WFmm%#fIlsY!w z_dh>}KcxeI+E#yvR{7wfQDaZ*y8)Y2V$DSc_JdKNeUTf0zF!aQmHAZwWqbe#;S-TEjaTa#=1A9dL23yfqk*782fgU zM;7ob(|cC8Y;LnXYO_lZBR;iQ@etii&Y$YA;Gz3;;G+Z+Xs7->vnbr8XR9X z_=LNPxa9tEP(Jsb+i>d-=|Bs4>)&C0=E7(OXvjb3CMdPpNt}9QIPa9Vw`L3tCz%7b zhQ{3{AGO1{njEd4CLP{9@T!tr#1S)diL^m`l1U00^&nC;|N3|s|e**%i<-VX9;9ZoUx?>h0zZZ@*Fy-PfFzoYZ2 zG&6Qjqo)2(>-_J}H?uG-CU1jIw4g4AK1?4udAH7DL!+rbd(r?RE8Y^d`sZO}sC7Uv zhx%5)DF3Em;iT8KN%XO@#y3_jqovXP9j?v6hrP`?X6W$nh=ZJ{7Z!~Tq$7g_Ai>wx zDE#RvaJgRt;W(9@?#e0uhj(4kdtc9XpSEH=zCIboneCMBXk$GvxZHYhGSb^Rf({_L z7KdZh)6vG|yLK`2a|(MBy(^co9m%h8&7P}U99Gm$h6uir+Q)uvbm>lgPt3Cn8vP>T%NS3TuIK|$yhl-iQ^zpZ-V%Pm8I6V+zl7vVF5C5~uMo{VFF~ipR|B|2gdGuRk{Lpk@A4GjTtVa4y}{GC};~;=$=C zi%V7kmFdka7&-EbcUYzBKE3L0g+-#O>D2bC`sjiy)J-m!hgq?% zd8_w#&vTbFyt%u%XjSCCOF&gBx9Qz=s{$KId}~^{zxw;w1n8ry)C$_(x{6(06^LIo zSn$%g`bb!`PF^O;n|38KV5R13ve%;1UW+TbK!(-&huR4o?c$4nCm9Y6UIFm&FNL&P z?A%gLF5;5;&=YlNdZB2YhG{E+kd~yLn?VloCZkReEJ(L7^6y_X8*XZiN-xo=6=7xK z4m6vA1j5je66vW7oQ*#jj1~- z$Q6R}9YmEGN!sWOMb2^zN&YBlYmmI*Phk^$JkRGVRs@y%sdlDqBV9f8!TqRsLN|TaRqhC#0Bm0i*u*qa?Bqs~; zOWDFhTq|)_T#k)J+aVadDWb`Lj=Z*S^Wx?2b$j)g&fmYMmo{e_#cs{r4Ny3efLxgqi2w=&F?{Zf?3qN&|7xZD;uUc|#TUz?R;=>fM){-;p zPTm2nB)9+fjaM5-hAx02cp&g@0SSl%U={?%;6TY@o&y3194K5pJ?A|Dj3R*27lQ#x z&C_XzH@hhT;yzyG%IXxgMm^@Fp1XHDr2LTh4mp>ImUXw{9mJif6JxyQwfyP%L{irUV*%^rEUx`C zJAP!u52~%J^9Pq-@%7?MaBt4P{`hV8>zezA|NqxF{_6vOYyUJgRa{a)VCa08-#+IQ zfj}>1pWk;iGdVF4dJpDK%zQ{D%&h{(KKmg+`5Quol0KZ~xFuuK64Hwve+4x*H_zE+ zVewVfa8sJbr%($!LP$u69lear7YmM!u`ih<@WT|rL?55|F=9ST=LkX+4#tLMkw&L( zVtN_irIvd~QR?Mx&69-N5_UR|P|<7L%qnmgLHo9)dqUuaKG?G_w`rojOlmu9Kqmj`MaC1ez?&)6FIT^v-bsR z?PpCOK-?v;z4OOkC$PM4@;`61bw_n?h7tH||NKt9Swq31MT?**ENu|WU?Y;iz|5Gu zysd!p2jISPgrr;;1E8DBsxB6Neidl%a)1oLXj0Ab$ELJ}F`nU_IlrWGoVn>F65=&i zhCkk$Ig)$t-n~~k9b@53$=zdfcG=i?Rp-l(gSYQ)-&Zuz_wrKbi$R)yo}?!&pxp11 zINPPpQTC_B+33*qJQNqJ_>MXzIoV^dFAch0qp;0*b^U}fLiwm@)x%DVO43igM>6zy z%PueuaLMYOO8NHY#<3@Nx`ebxZQXdQKmNXQgotSLC&uBAOZT6T{kC7|R`0%I|NU#0 zKr2_Sj0Y~C^QwR;VUPh@C1YtsvgLhR-%-2u*AUO1of{*SMnP(XE6HMX3*=-rFc$*S zgwplO&!2Zd91aJXk}KAeZf|#)1E!dTkUqt=A?`Q%1oiR7@!nl|0m2?ISk(57QLhJx zj4htW7`g32H?deo+cEGsH#b32TB$R9m+z2Epa2=eAKz9>(!<;S_ObrQL$L|EUDAq{ zSI-$U5syPcVi}eh=*nKV*%x*GL_v8eGNFdyG8Vd0v#*4aXCV=~fMU?I{bIn;LM$njNMZ@R-P0f+F{J9{NhArnk281GOTz;VAhK(>)M5}VqAzKGgJP@ zH3N^ko=@Fxt%)jX&--DAVEf4)!oIH6>B+{KCmKUcA;lm>`7N@J|@VM0_2md znqNQ_S_(_C{O@&7+q%fJh+O5J$04R1wp+QFPAZ_W%RhWlB|rYt5$|1bb~_!a?_~{F;Wt}=J-_*sp4FOIImGGF$mq5QPbTbkJz7)Gu^}* z#eWwu?7~i6OQJ$EPg&C$u01y_~lmys)_y58G3g4gK2YnJ_q2iM};d^dAvg)N4@^x5+ zCva*Z5+TBL5ov{pH#{6aeuMu31i`70%+NEF!%Yx8HfwG_U~Tju^=K%ZkQT8N0|qt5 zIOXo$r}7X=+v z&kZQ%%n%!4t~dnA46Dok^ZRVkCWjgE67KX}**9a!a`K(h796^{&x3|MlQ-F&xt%&j z*&duPnFh_&Jw;*tlrJp6-sKwP{tabF5H%nZ^ILD4R&m!E6HVnMz>tIIxX{-Of2z~6*wy0?x zj30I(TA2T=l|{=}U33&z$bSzcJ9Ct@L1%_v}9{hqtUm4($VL z)`~xU`k@)60L&?kX+!w@mC-?$r)a!jnkVpO1telrGAM-V4OWxRkJ+eAmp$8eK8XL%-^Jo$e|47KRf44%Km{lAOTqjYjr04~#2WT@#TpL4 zqgq{ye1H-XZXq@kR!%~9!6LNebGVftOJs|{%=MZmfxW=K(r}2DLtx~w zQGY(_>Nmb~r#yVUJp4jgCO+BmO2X&KJdQ6BvsX@hI{VT;ha{T@g)#ht*##RJh-RVw z+NK&dLZVgI&4@KWGhAe>WGzFC@ID4m6XaX%$-GqyQ#qeLZGmUh#~`8y7O96m zK$TNDh@e*6+M#87Gyha(otrSAbl~cGj}GRF?I$2` z+r*Fmm-k9IKD{m=?=z=t?^=^fPym|^Y%M}n9NL?b09A#BQsDnP313_sZ8{3K)G50o zjB!mhldlxBcLE)WsvYX}yfdtNtX`r2@*IEp1Y1tDl`Mq6I@0$LP96E;`N+bT-8Eu3 zyN1(c1sE?1^`F+kt%7vX(YxUdY_>T%$sxCUY$f-g5~PTBvm+xT>u2A-T>xD_pc&KI z>84KJCbjwP{Y zYE9!O-@SXcezvDa4@$=MaN?ymy#>CrJ_hH!d~F}QdE>YF&)@4#`)pP3IV$t7@l}v`v^|!ez%ZF&y;J{K?{!k$-`l4kusmQ3b$Mb1xG2F<+$x9w zp+3occojqkZEVX~_V)G~chk}|AGBXsAxZ;_j)y}!@Ejk+COIHRa-$u5;azwn%|xT{ zOwCvD@_w25Tr~-TPF4?a+KoQSxJ3^s^pZE6kDF%;5h>y1M{R!{*(!;X8G$)sUFT&)FLI9K+7EV`0M6r$!H*? z>lG1=c#;aX;_B7p&$!Bw$7%7CIW50uu)U^S$tzRE;L3g%2643%FLJURoBJKb)C}`- zjY*dJ^NA4ig8oi|UeYdSPkg8i&$?H%3qJ!z=8{;G9bC?d`3;6QpeeD5N43QePH*z*Zj`2cBDB zsf+LvWRHbaLqlhe+Kgnrjckl8huaQjY7m5@jd6{!mx&W8B&&THGONjrlZeSDEPm(A z&6105bv|HAm(C0(B*>$bzSfPn1CKrwivHWS%42}bsLJlBOK_(@(ZfQ*eMrGv>2=gp=3=0JNYCHyB{jJ z!!TDTq9cL}e-FYr@JQo3fI^U|ASK1pyq1#hL@t!58`IQnyopIkD@8mWPo35@UlfbO zapN1#QoH9p=)9!86XQwFsn>VlJx&Xl(_g&|HMidZh5ZIHN)P(9PV;qnJ29nac6(U_w`HPMx%VDSr+~M(4 zdzIfHf*D>~&PmzFcxlqF#WSdiw#lMQ9rMl6uiFPw&|Q9@INK5-;6;armuw*lSX{~^ zQ<06PVn)ruAb{cwzIeL1mg&rSWQ^%z7(|H6GdRwQ6w!?^^N}6x8255|z(^^oBF8IC zF)IC0Vw)6(drSax;h<-(&$E6ZVK;h_bM-;;Y{c8@(q%N}o|;SdLYl%^o+~|C`X2Pc zU39eOePhFYR%iPfA@+F6O1XCJT9$sWwfN~|KIx6)^*1Mb4Z_OqV09cSLH|+Z7GF<+ zjP@`f8SCj#s+Z!_e5@I3(=Osiy6KLJQu$(wU#Fk7dS2-Dyb#^aKScxd=U+BaM0TJ3 z3bAtVv#%SRJg_`LFV`?{r9?kWOMW^)%645uMs#i5T_Y9nwL9g*Z%$UKpC<_9XCK~jRUbj05T2_q-u6j z13pL^k;+MuPE{)y0IrVbaD_733F0^(L>FUnF*9nx?^=L}iry^!Vtj*Hv(?+6Jkf1$ z8Im)g-jQqBGY5i*a5e*OjjFDf7ZXIf;1Z}YJ3RB@!-w@5ddHeizbcdM_p z7yrLx(k=RVlIM0brqOh#uOZZ-m^FeDleDMVp}N!}P|-}+9L5-YMqjvYENI{(1e7el zN55s(Ei_k!C$OEmo~6t@L66;sG@o&5A6^nGGyywAh-hcW@%JuJU1UsOgm#4LG8$MD z3==hoI6|JFy!gvG`0L5|@|CXmt}n=JFLei+t{3x02dNhWHyeSMnKTEkz(JNh4@6x5 z)`@U{U6l7413W_~27G!NfekyUzjYTbGA@N;mZ7Yc#%StT6+UrEg0q@a(b1I$8J(9_ ziZN8hhZ*{}iZvtPa~rxlzFLKxc?FE^OM|?32aHAqgGr6rO-6M$ZY+htk+`c3igd(| z9guqC?`L^vRxL5@Q4fus3(L3N6Is@ho@H>5){gs53PF@QLen3;bs*3Xfcye0 zfDekap7G3g_ujq7rY?4^6E}&=Rm%c`gI^pp8Y7Ln<-+!gE4fePriv&aI##PER&utD z?J_YrF~#=6jm@i~%LR+7fa8Pnls(NU*Kjn5=#oQlK#EbwVL~lW!cgHMa^*yQmJJwK zX%}TU^*~EwJVwjv!Fg(8%1~-Jyo*0Lm65z_{j8H)5e~iznYd==P!zd*^l6sWYT*rn zBZ1#huzeKQ8^+QoJpJZsmLdW?JVQ_b_>%B};9$yah@>=RMBNX%A$zR%jzLDZFxU!# z%Pk{jO=50aex}wvd$GmJJo~|%%4=}D-yA^MMn61=;G!&Z3`XLn9BbVt3Rt(Njt$#) zyiBAPdpEa&-^yFfD9a3eAwUPe+hnIYtM^!MHB(D3@Vgu6E=#_BJ&E8Tb!Hd;R?7Tw z3l?u0Ran22VP6}DzW*MI=H|yLN|&sZuuA63JuMc+;&-`tnST^czO_d#j|q6xqtjlv z=(m49;Iou@C3^fV{A?+(gRG9dplbUt?h?Qz+jD zWr}Xd7pg!=UkUK#=V#~}73#eV&d%3>&Dgz9fsN)`NtF?wS{ zA}>2N_-v(-mo>e}_-w{OMe0uX`BeqfzOSl|+Uw7&0=IbWdantGa&wf$J5v^gp~M3J zwV%OM@avf9M_W*X?Fth>dGwI(-{027z3ZK%(GLdrY#+_Qw(}1Oat$1i68^L`_tr7U z3`Ca%ELVw~eS?Got2DHhE228wd^N`JF{9_lpn(#}uB_E~gpr=xv(;WeaVwf1;{tq4 zAU*7-hj!MRH*by)3XI6n6PI?D25kWP!}oFVytmFA3fO!o@k}Jptt_ch>-EFC{EQ$l z=>lCG()GyL#TsLZ>GLSIukEbp6O$iK3e~C7FB_7!v^319fP!g$l{qhcZ!vzE4c9>> ztPs*6t1&;+_O4B$B68d0?y)FQjjk?-Cuge&N0n_6d0`0dLHQ=HTInLM@i|bYaw=C7 z)CcirG91nCAkbdbj(8&@ql9qtvEtUN)0nPJ+y;>vMggNIf|nvB#{}h83sb`#WD7~b zKKhQ8>As4CM0n`<*4nVzD4KRVD7Ip;Ho`JqCX^o9;-Kd2YRXfT>|(!TK{PMbA5X%U z$JrGd5+6v*&=A>dYzSC%!!MIsvO7Zz56en z{VlgC*R2=E@`cu1ODAze=BRLcfxI&e>M0Z)fUqG=HRG|utRqZqdWskb1B(yBcsel3 z!T-&(@ZmhP%hXh>3f{*!L75R0rysrw;*p4iAf%||C5++0H5QD~!nihAOzo<+wx5TT}j_VGeKw%3M#gh#tMo&m)fBUGZ-+VyVB2^oJ$mAuni^U_Qc~(1J9z22 znSShI6_ugzhcZ>OJk9Z8Q^x9;IP1@+ZFw$9t4+;OjBGu}_U&V7gc&Y^=r$D6w4b${ zvxNseeH#)NhorhO^&7 zB(T>Ll=v!5gI!bR>5=9HU{$Zt4}08QZDzx;?PNXy@lNWjNF&ro4dZ)_N9x|a0}eI- z5({)N;w_A5aWK*d9DB*q_A~^U)-mcuGE2DRo`eWGAhfHV*BLCT+Fhr&H#FTxzYlol zzYqGCivQnxUJ)TCWMNf5;u~%5s5`b2L+8tbWVBbxyPtFYc3xGr@E8lKE}~VV-&-`K zcum`vtA4kn5nLY?4z;r#>Ln@b*}nC{6vhk*J?BBI$87VOMRv2-uDyzQplv<%etn-m z8(EPWU8jZ~v&S6%7GHzEXriUkntlEHhoH{@!qtPPBX~p(T(PT!@rXwsK0fB6%#S4p zP#39Ld_XkRDBwH;_M#LY#7<-G24Mrbi9|Y5SJdo!Z^4`6!$h%sTji;p6t8Kj5IVf z)H(%|W?I&tzK1ULlrD|Qr_k4^X-Sa6{8XLK3#+#_HFde%PK|6Q6)iJaawT2FD>2NElgX2}KSrBe@oH zocW#;4pDb@x}PYno0ZtF^7i!=`nC zwX1!h@tdiYXY-p$AhT+yk+~(4ur}GMJx+TvN2PZI{gM+?+f*+p_?__7BP)nF6GwC{ zj!1fEB-)!40`v=yw?x#U>Tk@vlc@Ffw<*!*Zg|NyjASEI2sC;&5gxNTh}CBVFt?LS zv$havf2+O}gOTpwe18Gb0QV61{(CKr!0}gF>O@wX@a-iX!bY(F zyNhRGm@wqT}WCe>y z0o1W_EO)HcS#ftVF%4UJsjqT%3=F^RZApP5-#L2CsZxL)x87?wYQh?>bj?73h5$mmHeR+_Ra^&mWkH$amyMu8 zu+A$-9vc7)2Db!xDitu%phNd)X}RSVq#p(q6ICtnI`SX!gYEfv+S)ll0q%m54Q zuc{QlA|=by9ZaHMwrrW&Mte&~8K9|kJbhmX+Bv=;`CpJaRivE*ME4|>+vWVa<1OZ+ zPaZ+Q8%B{JD#D6Kvob+9YBDqfGQi`-n3nvycc2?>BufDb3kGf<(YX)47wl-QDzQcS zw54)}qwz6QUQ6UeWbhPz6ezV~LN3Qi4>^lktmV>X1qia75r`+F@@5-H-I6 zV3kLX99hA9<-w4R=4;GQR5HqXg0;>v47dI^WCeRjssTQbnQFwi2bASRHg|~Yce7Fi zFyww~1LsMAq|RpgIiv@&IXfk8?4`?d<7_-|RASBnvZL7tcOzD#wxR^SR{Hp<4|bLv zoL&5ypS29kLNoB{K6h{z+(1nb8@&y_-~*9ji{r_XM5@ecpp{%k;uEE*M7JrFKIM*d z3UqQ7yXFpp_YfrV3IY<*C3x*@D>XLmOTwE#%*fikf)zsdzaFA4dYk;{i)5d?z)osP zU?*h;p$eW4jrm;MJv&Iq=s1P!pQpQc`e$olj5qWNR1p81-SxZMLvIDw@~AkXVM71O zTk%?qUUaUYf-UtDEFZ8xP|seB6a*z}2B@s3Ucc^_)MYp|)B;Tl!1Rs(9vX;&4+x-> zpp#(Zz#=6$2_i@$h`KR`K!XH@R`kM94GgZ{LW&&}!AJlZ9%aI56U=4-p0Gv%Te;0u z*BJdx7{XpiOF#72*z=Zm2+g}Wo9aNxk&be5nloNRQY=<~q^pc>1Vw^!c_r_i5eLEA zo3uBe4xDB)wQ>Lq=v-Vi!EMmE3*=7qV$jsC9|L>C+49YI@74ei(xs)st*EH4Ab6yY zMr3nH=CND8Fc}GBi0Wi538|dP!aI|)(T>&^;WrW2Ol{jcCQ{cm>d6X;*RQV-AHphSP!j(l!FN6pg|L< z)ydNT6EF7iu1&Dy^4`RWK}g>KQlrUIb!(^V{C6cY!5jFbLjBDFDPiUT0{07yBF zQI(u$BvRTya_4!|D`~Y(*bMwsHI+|TH`G`Hi=h;&6L&kyjUJ;J$~- zXPdG-Zwl4b)#*6>`Bn9|w+jo=>nAr)a@9@{@I~{%07lwIZ-__WLZ#l5nE+0{PK*{P zz3lMcC;|P2-c4^sTw^e8`qJJCTGzGpiK|)aU>w5MNE=c64L<}-&jqyQU#;O># z1LyE5=IJLPn(MJrDxd*$_YBYki18iW=+%)dlu~SK5^wcvD(g0J?07Suv{af-;<_I9ZMQYna;0MIgfczs1YX2)UVL_-C=o> zao7;cmKUb*AOdP_Nfoe&@wkkViA$dmbtb@NM>9i;+m6LG$APWBH(rvIGn(f=iKEEO z)0Fw_4Wqm_&dgisFz(dpzPi95AUg`X}o zeBOs<^}P=My*E#A-^qJikrsd%$EW*T6vtb({Z47w%pJ$bYiAK~VKlC^0!%$1{~L8T4SzzzlCc$hZgI zOSo6A%!4|}p6+cMxYuA!k*bdYuTxPGf0K7}x-0H6K5Q@{&mYrQ2fW^1M%RPrCPV_DFM)H!)pecvN=(U;@HGM!zLX zkLZi4iF7L8idLlOvx`QhCzusO6+TUCATPzJ3+98_6IR+#5df;P(#0V!ypX{jC70X) z@X~^UHD)#}@u3RL$HIrJNa|}M&BTAssWWG?-Xb`l-V0w--n{cT*?%Xuy#Jp7O#%Y* zpRBZ;n~R0;MDwgnNwI+&15Q|1f?)zT!mVJrOC+$5TLcHUII?D!2}4qEad$GO<7Zi= z5`QmzhZyIk@F*_|qY<1C;z>`yc6d+ZktVmI6!erprSimo?#S@PO||{P`edt#gOCy= z%?nHMNSEp3Q>!lH>d;Cwg6@uJPvg!R)(j18ge2}^j91dBp))qu2k$D*TT9r# z9h(-_F9DBc>8@1M8D;y&kNtA<4F#`+ewHDz>~6xyBJ{le7NJrXXs}iHzhxV}i--u0 zu0NJJr$^a)GumY%6e(;Fu_;dW8gC zz`o}_6c&xJHYAy4m1asfSg*Fpbvy^(-tsV*xzXM!xxfD+0C}pJ<)oR_MDxcU3#k{{ zK}X&f#(wC1G^K3#{`w4=`R(ALyMf>}N#e1a*q0kZMsg5%zYl^uc1D3=S?1Jw(F+1z z4I6d1WAtUwG|(s(KhXy0R;S^+(}&G(0(Osd%bP-*o&!(mwX=Cg1)~sn1s)W>0@MJ@ zV~7eCai_q@YobgY1cA9%{8R|Tz-6@rxHWx$gQ%v8`T-EVbu=|}k2a`H{^7TpUoV`b`=@uj*w*7=3{h=448o$2eQI%U7u0`|&8dd;+vw1AYT1@U98BW6(cO>7+_2LPh^-|l*iGcUf z<$Kje$$<4!xF#%dw5{WUVR#Z>o5_1nI(9VAPL-?Q%J$=7`~13}uzIl`EN1?cFHPWA zXMhyr{z0>8XCMmLwxi>Y1x4bJ5@UOX&51Y)JB})r?fln;{M!Q=Tdejlk}!yclt>?J zA^V)P{uLo>r1KRaT}a^Vf#~B>%AE1)M+RCzVQ~PkS!+JC@7-Jb*}aia^x6HXBvq>a z&F{aAv>S<>p&z7^Zd0xuZeC2^VAA3->vcRLP<4x;(1OZi(d?EBx7ly%CGBd|ks-4t zM`QAo+V|BVqbvA-4-hLWfJ*a~W|np2KVW?>EZPHp2xhe~PYvi1ldNq0$(nNO+4gdv zfa3c#7X>7NZWo|*^z31ZpHYEY58Af|oOHQc?o02db#`YxhOo5*Kw<}(!Be3@-JI$}ZJHs?JFJvxnw8s$X^jC!jbXW*1|m$|uHTP_(xbIP^>EV-f>ICsK0 z%8ASQ-O{UT1E5o2hRRwCr)4(clhgpu51D18$;*}iEs5tln2wQ@hj5@WabYGLtT-DT zt&+15gPUmSa0GZuC6G6<^;$tj3Ec-RQdA1t{LIS%??qHcsmG#(PCuLDXxc;O9CEkx_ z+yR;$CZF1L45V6gh(;wXSQ>@X&OG{`XQBn~PC?FCPVB!Z(5W(Da6*0p+>|lwnKvm z0Pj&ZP>PIEnWe8fdiXU+kNw-Q@V}&4e|jtKN8$dZ6*^a^?^v%<8~?+Ke?HBNdEnMY z7#`yf!oocm?FEUCuEgb;s|E}y_uq6h)`0V7yV4#M_WFTGZ&8+yfzBfx0LT!~OvMM? zq|CYZssxi^t{%V^{o(A1o_+#EfT*I;dKz7XbqjwUsxc}DcUj3nno z-$VIPZz0A=*v$+e;E^s9ey@Z58iJYYmL?HAiR(QtNMThl*(3&KiwhIGVD^H67?fBgw{7?HR7vB_9%Pt)`*+eT>z|~T&et$n_+m6V{IEv z4R5w65z^%=#g6ixRtc{dGeO}It-bcDT8LYdOUC3OvRne@ZKV`I>?abbiTRC&V|Vum zVLI^q)Q}B|Crh}uW>o+LQWctR%#c8M|<^)rPU#~HYP^{qDy+&0$p@v1b zXdA*KK;Vx0l}y%ZQ}yJaBpcaz=<3z0@v=Kp*|ZMMftDN7Q3^3DLN67D^j(Y3FNILi zgUb?Ta`iV7#9^SV0O|?v_D}B8*_Z*VUxTbT?t;`%u1pVAnL_NO$a6rCW8N*Yic=v!2UdL+u zyXS^*%?NIS`7x$+(RDjz2Xl8BYS6vz5t$?oX=pNKdw8STTgH4>IQ-8r_Z^{oL(091dCL?LDU@fj!{m?RCiAKQM0YHana6yW znx4vcc7j!D1OX|u07!l~5`_h06w4^PE{D5AWO>y^WC<)c*MR49v3F-M@_U4@vQH9hSG8X-)R=U(Y9R9 zJS~l`iC>|{*SAjZb!y|BQF#0$VUM_s^4czYtN!Ew1Ml|kJDr`4xtIEIsz*lF8ZnQO zEWU{W%4^4_Yxk1d>E!(ON(!NNM5uL&;fvIX;o^2^o}%YVqc4=)Gi+xQQ25^H^6-P! zWaRqajt)adT#oZ>ZBo8HUz9mWL=J%22hQY`mcYt9B8|^^59L2)p~k6reP)okH^E9n8y3o)?=5z5mKKS zrIG;o`1&S{RaJfW0r}XliPziPq!MI;n*G#=EIV~P||M;p6ijBMj2{GuiTZD3);IGqWiJ!AX0H~hPTWH0q2dY;c%4Gh~j*!N-%Ohk>eEQjEQ=@F2ZYZ)o6Qu zf*cK)X$Z6ivA5Lc34BLGxj0;}GAn?X=i-OFhMow*bMT&qqm!!l29f|^#A7k~L6b{66 zUUIPRGB9Gbu>vvM=(va5i3Hj^N_%`mMyP+00T5kj)?E2e z6u{CkQbTCAn}G)&X8CDRyZOVtFL{nf_OUWTd#*vzqY0$Br_PRiC$F5g%5Pls88bXK z)%=nj;oJAaZ!OZ<>{#w(+wP6H8dW_x+oRFkkh| zJ=cGo7#l=ps2%)UFqG~c|MNa;&IlYz7#H9k1A$%8H3D2U_T_pPc(_6Y{-&F+D2m=cr>6WnBNcl48KyJB}ILhkRYkhFcXw=)D@LQ zNq04JfE~iLhTyq6h?aZ&o0(>^PmdC&HrkA6jmk0<>8U~6NJa*JhMmK+7W9i9&ooY* zKmWum(?1DtP%8&ErB{Rgif1#G&5|D9(9mLzWq|itA)&OU$v4hokvx|4w!eM=8U6CY3P=Sg z`?*!~bUntOa4z+-H(GagaBz6lkh?znmZQR&E>Mk%Y)CAiZw>}SFLdSacx`X8A0=UW znf{+9Z3WcxAF{GC+j#Aq_N#s9wf3V?z1pKPPo#@Gp}RyDbp|g!{ZI(NHrv`xwQv+r zs`Fw%(MB;tasXitTwP&>7&HxNDX@c=}{w^dIYML8c}AZ0R(sd zEB6fy&8DXk%y~N6+OSUj9Pyd`sGFw>6g+7N0*zKfD9JYh^P#)RY6o{!%TvY$67_929YwRDvFFzQDzbpks?AW zQ)CP>MG`}Rkc7&ceDTQ3MQ$#+(g)CvY4!N-UXeSF-~}!`B9&ls>C-fDa7n zh29Y3o+ynSo9h#!H^I-bq5qlBI@IQE=upbh31E*l+Mcxayt(_3!Zs z7L_-)`gt1&Lz}3N4#5E8*Y{5j)?+V#R9p}3MBie25Re6ctxLmXb)Y51HcwUaFEW|^ zvn1G9cV1(6LFolhD$x$LEdmzZ{z;}ymQ?a-AGuTG0>3x*>^Xf3%fBsOb>brDbLsv# zO?l}Wie}H@*aA+0>cpvRHvw<{H>i&ahgxuAH)>3qH^9^im{=sx=(r(jSGy^%OCk9z zGW@>DBVih!|DKWZ^1Q9Qdl=vZhP*37eqX`@pKq#H8^w=X5D!D|a;%xTQOs&TmNkva zN^=ZrK16(N;3L1x*K2i=Jv1ntTu+PG*w^uRb55>3`-tz&xHxN~@Iur*QDk^ouB1cK z0JXN4F2?`pla3>u%ksDtjpCfRxy}5&WPefjX}$MMOy=$P%~xbeFQ+t%OLRk0@oPW& z<=43Ei-<9<>>rhPrOBd6S4ETpAVefBmKSh6y$;*wp6VQ$y~09%ng9Qvn`0uCpsJ-0 zDuUbuEPtIdoqA1|i8%Cdix|B%HO(K%k?+u{Xu2Q_dE`MQ(x7m4HNYB-!rnq9n0mfc zW+h_Zr8sG}Y4U__-a$KEOLV}{)fB=Z$4}GkOT&_Z{%GY1Psf-n9L!45eEV%iKMXqu9 z(^;a*KN;=1V??;F?{b%C4)Y&IkKbs>z1Re9v?02r!a6ES!2>{lcGk^wIC&ao` zqPma~4KIv0`N-DJ@_IjBzkXM*xyW9%g4urZv`O^e;yEA_uq9S78F+xG&uu6vQnf8u z!&&Zh>G%t#P$`AJK>c`xJG`ggDfiN3I7DzYPWFlI9B=V8VdB@sxIhn}1gsXin zN$v(jqwtkV0Y~_pBZ={sNumTE#ZI&x@&sw=I9I^$YHvV9QLK8*IcUy6NK%Z^YzH!; zO#1XjKug3^@}0vwLhLjhpKA(#pXK{YcC#7Tiq&Zb$?`^yon$+<(B}F3*f?89#r9p4 zY*48(6g~`Pdj+O`;^R7JV2h>wYw`UuHE7z5)=ckqjd5uHG|am{S1k1B%nf84@?2@Q zYOPG#Ty4b1@KF~!1!Ab|tBxFkYLwtJ9P9%~P9=>FD{FJHYx#Je14Ux#R`%%6z={e3p zhT{1;4AR^Bjn77vFOK#U`{fbyg5K-^dOTRhg%~te)bsQFteFe_!U`hJ_~H5K<;#~p zBm*7AX4H*LHpQU3bIT@l+a{riGAACv{Pj`?gMfWwL^p$-*=RE1j&X|pm{_bo023%k zA>P@ih0!~C<_$tV|5UfreOi-e|D0O9;{zwE%$qmwY;dc^q9iYJ(kg2rUv#I~FgRHU;XNFqP7V&u}8 z4a)=4!#$J46kSxUo<4R{r&VS6@0(B@xIB4toOnA9E{om_PD=vyTX(g~&CroYjw0sT zg`+B$!Q>D#O`MXAru1RZUQWga&aG>~_CvC5%uwyr87{R{YyXg4dtN#3clW`#@AJ}pvu@nGSGbN_D1pu&t^-qHOu4S9dhz0*L^bBgo7s&8 zYPvW;fb{D1;`x~66ckh$yGU$BP2_&{xT=)vceG3_@5JhDwNZTndQtt-jH3&~RFy-hd%>B8q&->Wb;@pg?9HPi<(!|tM{IsPt z@it@HV8MdLDkzlC!=RT3!wiV(;0C@~z|t`tkYup;jEr6z04~WY!Do-wjnp7m1d;g_ zJA{hQMIVm%aIdx}+z@ghKh~=Ss$$P4)w5IKQL0}K&?vU2^w>Wz5S;o572g(N(5?}4 zA^Tq*xwb=2_9n}Xt+QqHArcqd${V$=G-ca2Ufar{0OuKECnw*i>Cm)^S3X;Ak0&D# z2GvKVNlGutGWbcSral-+o3`+-d^i5Yi4z}c8X40J;C5Of|GC9h2$vSzV(8UaT>+bH z)@#$>W#)aR;kpHTIAU-IKc<@`OV%Cb7O%N)J^5f!LWmhZH&dXgWH(MKwf;$z7$;g< zZ0}bTYhR+YWZcnecg32$w{s5I>`ZzGDP=9u>`b(57h6vE?kZq%qA03u4#!q-+l+>LZ&oWM?QPRU z3N3Y}A5e^!q#c|M2)1rdLx?>`Un&{+oM-z76PuCyIQgg8ThaYr_Z#w{0;9JH2rzmx zRqOg+hk`;!e<|O!G+qAqbac8RlO`4Dc4pF^H)-FBFMc+qQFa~a52bOv-lh&(|IB?X zxz?lD^wMOPx|9?_gjCELl|N~046S)EJ}#>aaBCLl81!rY4w^dCUqa+kyq9<6W^T>+ zH!;@V=jElzjMGJJvZ$4q=}dzG<(y*;b$3mo&)|@9_L>E3@+kX-rulyito&c%xW_4! z#VTaxq2|6Af+E?HhUNaA$=Xj^O`+VkSaUEAS264B>#L`LIY)RESL6@*NKGr07O35e-k(Y8Npm#&oFDXnN{pE#k1lcArZCzb zgG|~cY>B0hmzUM1O^2&`&gJx?_`4jAnchsloW-3~Pgkwrw`a8L+D}jODE52dz>5AU zB|HO?q+al&6}i~WBOUoysG|~?>X))b<5cd?`G@H{;&zDzeE=@U=gNs-t4Ty}4ELnN_4rkgj zF9=c#?+Y!%WQ|4%9bLW+vg~l6&v>76v}UnFW&jl$+thMRy>XXDovws;sOZ!joywz9 zwE-fy!}aBiiJgovH4yS&9{|eZ^QS>v#U4|ps`=C-q9P+YJ={Z=xRYk^^YtBN-1fFR zNf%=tts2Rg>VG%auf&#{1h-70@q`w|^u)!<9S; z)dDdNtK|e$&5!5M#uMtL_TJ6vvkKYt*?xtu&O0}AVXdgFcV6vL-RRQ~)hD#REa!w# z2k8D3MXr~;x6blLK}R;HvHH}@6vw~dqV&t?5)*ftMD_e5;iZj4MyMt!gXf<_93^0# zg#o(a?zY0Z6ltPTRifP~sS>wTtrwm~Y@gan-YI?kc?pr~>%#Bag&T{4yl0WxX`xrM z+n>ctzkfaF{0%PPSeks4XjE2J*^Za(D;kXS@bG9TB6Kk}Y7Pbk1wEnp5;aGF!gq_o zQK?k6vH~2>-M$?|brC?XGSgTKoa{+)>1ka>AF85>yo3u1Msp|7Om}?RN`2(_$FFG@7#Pe2 zB*7h3R?g0hvd%;}xT=M+3{_6n7Y{QY$X!864x7RCq<;SSt0yLds6lf8?!dka*;1+P zGN^U*MrSt1ElRTS_RK%UD(mRE$N`9^Z_@f&psQaNWeH{()#!x1XKcJi;3#Or-W3Ni3arP^iP z3&Y|bE!S+-^rnYI9_TAuNF17hYDL>*H6XtW;}J<0dja>wxasT5A+!%xFz#GErOdeK z>dn$pZ%5rw-BT(+-+-dxmTEM@GL6Hoq-7M^1xj`Y_06^w&W`syN>=UL zK@%#oRem~DFCM^q_l?Do`WUcB9iMP=84@{+EBYnco_*4N`;dckza^8u_@OSE2TeTk zRD(Ied5NoXf8hckQs>V-J=u=d-0a-Wwrohi|0Q=d1!5I-CfV1AlV+;;7%Qrf#YnwH zii09~$LZ*dsI%)15$+1wNmMG8FO_4~l=bd3B5%E)SH}r2QGCBsEfkHw=_xi1|Z6T&Zr1E|@6vypC>TZ|PjVo@P4fKmBJvCqmaLgGs|7S$R|JiL`!V<4^!cj2 z#K=|CqCurT)Q2s|0<5EtkZZklvd~R;Pxh|9q{ejEkuQ{d8nfQ6LfgarUJ?DZz zka-;jbMZdFfM=GFFtLcSK;v`axVbS!d(9-rOE4WUw$p)9(K+T+nYw!KoLVT=8Pp;U zbHrozMO1WIQ0obkEk5iW=6^eSU@%3QJWc4G7W(U-@oVYvddRcszaV4G@FjPW@9=Wg z4egrxs3$WOa45G6Q&rQgPI%9lZE5oP4T)Th07Atb4_5Vf2G!gJoU4DCjn))#n$<%@ z?SZ!gAC5hcAo`Ag0^j>0=tyK_`C)^Kp!)?4xklt0?9S^cQ?3;qyt9z_Ko#4(@|Vhe^BLo|^OQxVe>!%5ef z!T0B5Mp45Tl9syu{(!ZP7G;}e%DX?3WUV#*M4ZI0c8wFbVQMK^2M`=XM?1^$EE?6w z6$)SKLs<`TUc7Eb22DUsb-Fj=OB|xju%>X{tqPVgLw~mwbSgtZ5!>#UJy0zPdU==w zvTl#}d|IIJE@iX#a-NHM8Y;$$bSd%qsr?H{%o>S^l2iGJl5>0EPqz=*g35XYLvKIF zT(cvpPR1$f^X@ZdGP!Pw0Pca=yCCEf-}vo_fj{05{1yLYblQ8f{)|dfJD#BDd>#>D zg#j}ip}u+`y@fp=8(Y!BnLHL&LkKw9YA|ZxHD&;e#srPhVE>q3n^-278>;48g~p=RFA}GIBO5{R8lpp2%rV^$MMjq1s-;z za=C`Y>zt}tK1O1hn+lX^Y8Pq^PPm6&j`1HXSLguB{TnWd7X09C$9JViH#ad&l5hql z0%mU!i-r8rR$aTuTyqZ}HHpNZj~6CQ0_nvx-Va7G7_*bOoD^wfR%CAkFAi_?H6}8f z$``B}tr-vADcKK;VwU)(D zb!ws?X0`L|n8t8Z>cc!)`rb5+bPF+!-xBTRAbGqlLHhz4JBv>Jo60P@Po6)X8g={9 z-coTP>T?m75BH@KI0{2pwX9q>2Z}ebHS>c?qYUDHWX|_fKO2pcUbyGWYUa>7AM4Z! zbwDiH!Bo5nGGHK%J#87-f0PsJxR%UQRJ}afr912Fo5gt^URqNub|S9X#x$b$xD{5? zCXzaPHjD4p#5I@36z?_gyI!Tpis))-R=Vo=YTE1j6&B4@;RVyk9%1U8M-)nBrAC9) zROvay^|~FiKIG-E_!Gb5`H_w|`+XVbbpQN{-oArOCg0+9bSP4LjmXc#NZq@SM?~np zY=CQ%p+DI_x+4}kT%S9FQs!BXp&(*WqK1h#f}s=}VYu&cJkZ%f0%Pq>?Tp<8Qrq+L z3h}qw;#^{p?kHSIJ>aRZ51U%k_V54kJI7)tRTm_eSR^9O)P8Wp- zHD*4g?B>mnhOQ)72=-7{l(jR_iAvh1GZgrR`P}!u(KqY>#hfj1=xFK9Kp%S7WF<7z zn8dSZ64=;&oY4!X$D;c)W2&(@vkbL@K%F-Ng8oHVo*%C#d0Z991EwtX7U@T~9HWYE5`z5O3W*I>>7rj z@z^mZ@*Kn^sF$80_pv)Pds_;Ubk{YbLHR6d+_O$bu6n1bV0z8*I008r-v~ovmE14V zO7p}3i{|2UEU-wSC-05gvx`@qRbHT$Cz8iArs4)p=+6#BN0b}A@{voPSfr|lx3Kmz zedG39-rSL|h~#U%f>?E&w1=YER^6%+7`e?;S`VgdB^^z~|7Yt>{GG|8Ac)Eg`6i`uE_odW9ks?8;A16$W z%khapY}!^^d?7Lp{=0JqK(4|8!=(sNx7UswTg4u^)0v>D5pc%(17K1ty4&!E+eAdE zM?E>mSDOa-2xNPS2KxfLzn{!0CQ2hqSrc5V;*MuI+Uq(Mq)Kn$P0)kZ#U_@55GfGi zj!&wCVK0_whM*C>L40&fx6WbpwB2kT@|%;c)a-e#nLagIuZsp^FEy|<&RdeL=rUEk zh9NNVW}uH-K|vHFlO?OZCObN=u@@<_hngp8iirv=c5+>F^$oVoXa@{Xe5%ur@9&o> z<_?LzyTkX`(W3uCw1$6IDmU0r)*Lgnu86miOx0+#)~jC9yzZo=Gc3751vD#AZC@c1 zN}1Oyr)-a^BI>hcV!EWGa!**ZINy6c+pj;k&Q)(_z~2woD3~Mc%-dI{-4~PfKrt%k zbb3`3_qAzf{4roefg#w-EM7%?(J8d4I5sP>m*G_g5#oQoJ*B!nTjDalV zT;zN7$^mFikRV#AGOlic6&4J)k#bCoinE%qIDrJwb6Uds;cUM)D+>b?Hp)L#i%NSI z#zR@;#!rlY5g{_{<>SJWkWFviVM1~=<^{o7#Joc47h^A=S7nRcZAR>Nhdx9aK=Mil zv?C5AzhYO193rV&sLpR%GqP8{?1E;2h!`3-o~m|M3zR7wp)x~LNp3Dp3`)%Hg#y=Z z3<^o(iIYw<5bZ9*bW*m~1A&!#XvT*D>Q_w|TEYBR3{XfGkEBkKY)R4U7PgPT8^wB7 zM~3WZv9=t#|KP!EIIRrqP zBFZa9JAJrhnmq>K;n*J2FzO>)5kYYwa=Gb8BdJ64%|$jM>oI2=q0EQK@QRamV0a{0 z4H1ULBTGJW-`zLj5!nwIEjc%dOpnM>O@+*EH!_L@F-E`r2>mk;bqj01X2$MSX-Cv7>$U6^lglE9lg+5j z{OJ_QJP9kt-e8rujjJlE(0lXch{f)06seW)o#wJfl#e?#K2O{FgLDr2G{rxVO--%s zcs{y9jbwO#P|3oMW~=^rXr8hj#-fakU~kG=3OaFy@i>XSe>3_=1m9zY7*c=v(%q*FC= zwU))eJJ>h|QRHi=mM|;Ag17*y?=w&xHaAxvZy|+R;lYuvH@rYP3!Uzz@c^`6$?VntTp)${mIg1kG9J6(1zNP5UFCEmVr!+cS zh%>$|kS0{Wth5Z*Rxq5Ltw;M_FQ>ud0vl3weQdT@8Bq}C2z~xeitI84NMf)ibYvSU z_-Q}ng|d}fl5;+4Sj6~Os=83}fg&ZgIqJgDd@H^m+k$((g9fC`#Nh~KStZ-4QV>gu z6>9IaaHdb6p$YiV1U3)ZehUoRZi+wD?KFJ{STjfi5g()u7ejOPlBd0v5a&;nD$Ya{ zJCy?g4u-!hel}d)o-ph>6?7o4A@!o$)Eb4#iYBqz;=mob7$AV=VK1x}@XW=S2{2UO z9r9>#vI(=3yH>iQDG5}Nb<&$ao#9(p9ikQ74n<~cbE!q@^xQE&w(Kr*@iTK)7h5%6 zr8kobaseM^4j-7>&~r7(VQD;~*p<`zESGC>s%o6FDrZ$s`)M0uP{qg+ED-2=iJI7M z|2_k*=D;`uI92BT^gDSurrT&~{GSj?Qyb)*|%gOQc3slbAHq_CTNvQDJ&HF0tE zi0NvZz zH{)|1qEr#8c&Z1Oq?q7^GXeA?49sdheuO0}p4lMSNBPhEgA`mvqeTeU(Txec}uib?zX zOs^-KT3bFtXx~jJw(G&55w~7f&+B-8bno70IF~3IcZ#^eOJ!!+WfAZts&2T>x0DJ% zHRx^36i7KZ>h94lfAEoNzU8EAjPpz$$2thnM3IUbQS>zg(dD47p7Wg!7cHB}F}I(O zYxy2!>Vn6lrOwX2oQ+LV>(n+m^*>DQ9%2TQ9MrAfLktxNlRmy zROMK`yf9Y$C{6%HxopG#c2_SAeMV}v&YM%xSW0!Z6Q{8G_F}z(<)^jgSie@5tXD2H zltaCfm)+3Day4|Gsp)I3zIyqDS+BdndT|LkY{USf_SRK;KU_}pC~KR_(gOk#2yM@I zjkLq+oYEEbPN{(gh{+m5TXAzdPGQ947uc^gtqSVj_A<5$xVBHyET#kGSo!H*@_~+g z2t1zO^VI=?@0iRK7$UcB`DW#b$sL8}@+@(zemtvqJp3n-G-ot@LKU6PTgCV77Me)! za?fW3c6#S?n;u2)Ltzq5txW%_s;X(GjFt*wiL>g?8O%Un(fwf;e*a2Nv!}LkHhb(| zJDvVI`ud$iO{V_f!SF_1rikBN1MO~YKJb>ee-S;3$4AF{LN-d!tAEr5ax1l6GXv$hry)i^ zQ3t*GWJ2$7%p-fv3!s@Tti`o-@GQE_rXQ}bah!-`o-!G}JL92;X$mx5Tf zM{fczTXCyLH()QSB`sU*o?-htEiegmik}^wkR2>^gP- zAL_@5Oi@!5I6QC?D}R2)m-@_+*!?(HIn$+^^J$FoXXfLKi~!Rhj!xvyS(kon z^aF~Y$$s&Nn2GYxI*v$4hN;1~(n7U|6I zNG;w_4TyQVzw$hyU(D`4lD-2+=7p$l%1g?!M%E9dV#bU)6$G8^u~`d%ptc{fv%xeA zG!+%rs5b}K%f$iC3l}a7M#gDA$1xBj=x-z*AGau;rL6HZeZ0jC^EjzO**7hdp)q6& z@q90In>|#cS3R`273XtJL4huvV`c~8)8$` zT+|EIe@tDblkqS#Ql$k(Eg-DOG#*{IE!A#BNmQaa?<0n}y3*U+!5ZM1vbn){#^qj? z1ZiRMdj@TRZNwZeRCSvsn1-=!Xst;(w4W3t-oU{7I9bZ(9VR$tsri_*cDYF-kOkFY zcADs^qH)eUoClU;&ND06$C868ii??N{HAP2%7E*0MKj`%XLZK!k>yB5CC1)Hgbd&oi!KbNLE6xkvud(EwdySktj^#=Wmq3%XG6Vhh{MXG;+ z7l`)SP9^yGvw?HlnxfvOq!-^|U!>h{(u8_zwG?_$h~{h(fMb#ypKOeTGM$Xe#Sjp;fAU7*BYP?%*^%Qt_pB zS<%#e38U2(-I54@7V~`425}w`Wf=3&nBMnp22wd6Nrv(jv3*I8%eFsG=cu|R)Q1gC za$ZaU^VFNoR)zdEMH%8wzh^I%r9Ja16u$QN9jR71xlhTtUc`SgBIN6Z0mJg9t}a>S z*K_yE^AsP1L4cRFUbyg3%@t5}wVcHm3q5IhRPl1|Ar0bXJ(;W+!|?-9OO0(8`asA$ zX99#j%Z%$L?GKV##cqEIU}+t_T-8JS2zS@5oj<+|I*aO1%Fj^s!VquNUWX1j&_VQs z+NZWquQTF#4bl*lGTXU|$j;#OFld3>5khTkL#ZO%qSm|7(j(~5&GLcK&<7vS>rkz@ zV&rCt^0?@JV9Q>NadqWPUUj_uM%d5cyEkNakh(gy!er5G-GM5`|_b3jot~aIX zr%4;r+s=^{7El!!`5`(boIbho8FwTz(u|l)R@f;j8%$!}RnR%YY2OUhwSIHnZ3age zN}7SV_mM(jH~E-C3WE4{SHn8hE$G0|y9qhlT(D9;!=5rBwPc(xjO=*UuDGBs7LsJR z1~ICM3{^OY*2$YC9iNMGi%p?DC1PuLdn>z>m$24i94QhZ9ff+y0VPwZ@=*4n*+zKn ztsdHLHFhh0j}3IMZI?&{3Sye$v{0NRPtm)Zlyq*r1S2DXtezm6ss}^73LrRBCE3d1 zUF!^PvdzVJQ}4O>YI~p`_3(}uEl2&r>upAKwr^23&5Dp|_n^_vIe-l6df6DHI z!EL_6Nfy1yHLU-bQh=rrF~u0vDkweU?*_ z1??ow?fz`Q*U$_8B78EM9!yesS5EFQMtLBTWpb8Ek zR2~m(1yftq$w9(oS@q9S0KBndAX>1cI*fNmVq<)U514X@_x&QFTVD`?ZFPs}tON`r za@>PEl*t%sWEgRz3cF_qJ>^$Pj)?|uz4fjlLn2V_=DUq&1l)Iqo+XN=k>AA}O4IZb z8-&q2OqoL-sp6o028Q(DwwBxEB+oNC7)&tR=?WExEmUZWQy0`ZX^MmqElv)diVLdt zXO;v-Z7{IH@xUAy+r<+cE<7=sRGDU_5jUE9;{N4kXp*w z;AB>1ra+z7bF|NdFlKPgL|?!kePIgybC?kbB4U|v!HpY|rX$E8Fo$h=akI{b%IN83 zo&H7N>DMp>{ECX77LhD}BvfoYsNC+cF)bo(;?XrGu;TY)GTJyOe^o{3EV1J9!8b6A z>gUTaD4U$Z3K8gd`SK{Mq{MksO5T^h zdZ#~s;=ky*K(KZZzNYSNikB$dA|XQHkU3+q9B|=7GOcCn^!&yi-89{BNzdbq zYE<-^l|EIBX6TiJInXNy4|RefzDES99UKMSV?{o!)%l70zX_-kC{NJ16f(y;!2RYo z{wYo;h(Er2_Qc@p>4VGNug?FhZuv^qcaxuap4U71<&)PI4X#@H@pp$mT6}=<-L}xe z2mAlFqu`5Ocizq~{aa>hc9`stx6a@+k8!TCzb`=dW>1`qFIl_xq0xcI6%W%7H1qbv zh^!4Iqd)bYUWUQLf|gK!P32q2WSWy)vTRZa)^WQf6+G~h=DEh&V-y@Jfxzo+Z!l(KNLIpU&WP*mOdt1enr7(}LO(T%5 zxB1f#+L@S>EWFCUC;a35mn6DduFHmzzM;PU{y=3{PmdG5H?#?w@)&^J2MO+l?3R?+ zl2?%>w?;hz$g7t$+Z8BUvzUF#4ZQs7UBtVLd=HiGH0UFbhUN4Gb;Llp*WX?+w9nJi ziT*+?wvrauV_3r~YEgMf74^l#sJjb6E=7Hh(1;>bv;NhKiHJ+EogT}Np!&73I!~qO z|M$w{9j{V#K5URCx<&*PXgl2tO8F#~x=L2pE*@A_r0fgt<2B7iKak44$})cu$W-0x z(C`u~UA^S2tQ1iTzqKNpw4Do~tt*6_rW! z$;jSJK0J${4}PoWY@Y=NXlA6&Wq*G34MDo$s z<3zcFI1!f`Ac-m|E$w{FRjO}<12oyJ#js62l_6i(Gy0_a>rWFW<`TarRDgiO!gH>x zG(Buh51|qMU@s?SPoDEzAF}EipA`>PUC+C-(#JMiUlwIcjCZ}Nwyy8#J>7qj=Ay!Sro_MX-0(>JChe2zH@ z&3_<}To?{nguNE2`qwquYku|e=o&&}tONVb$3_b?qVLA}`I(b$mzG{a$G15|42Mj# zaU*2fMCDatIze%UF>{8{s8mMNhrwcRqn~RSx^8Ubw}E)XA+D(`)`>n0O2J#{+jHA| zwMZ|FL|1EJ7zurt!i@1=XPQrKt{acdyE7^}xYmEPt0Ig~C!o%rWCC02$?k{^JQf(Z zkzsn50YvR)61}LX$d?*HJe^Oi8DFW74Mg6UDXMaKQLMjeVPW|Me&?%)mjl-%7O)W8 zn{jtl1Nua`n_$f{Wqp#!kOe`C;Wnfxn2C041v9*Zb8sz*j@GKkFJrSAFU=Yxmw~`4=m@kku~Hz# zUJ@P2MU&{cpz2Jb*MIY5n$2SM8+qhupKa>y>vN_b3kX>L^58p6UpCkv1xvM#gT#le zNKo&}cBg8u?qItz?@)RD8uoBtjWRA)E}!a)KAopc%Ev8|5&3kS zLF6u0d)&rKK}@57wrLujVQhfT`e61gUioKy$3K?N!LWGq&TnoVo;rE&#}&-u>z6Jk z0V^$;fM(%bJJmPQ+8T(AGkhRb6T)L;i&W>DZdO$l(1H09U!A+SJrBCq7>YCoiC(Sj zU^+}t+=(fjRN+hko61}F!kUGj|F*I9q^SU&EPzCZPL4ICsg?*<`fz)+RPfs)d3n`iLW=9e< z>IYX-xPE<7-};Qjyxmb&iPfX&N9hDKw87w1gk3&MMN2u*l$n!O|w7AfR5nzxJ^_8S> zs-xIeER9V6ok2Y)b zeIl~wV!?Lq>Zj*zo$604mb)880w-=c36(h3i&pvc_4a}%1hN5~dJsLIhX)w5>yLLV zqKbVgb)2Vt{TgCZot9JK$yU*NPm6GaTpnxJFE$mjmR@6#Uv#1a`&_W>H=o5XoA~SB zd$KO0V;dO?cGj82V^5!xcO47&|G>&YvaMzcF)yIocBo~awy>i z!Ss;rbQmKN1SfIeb<4xD9{UynTjrrj`ts`azVY$E+2~?aQel;KpdiM?VBSeH8pzF= zkpe45E)g2LO8ns1_ec5BhoN!e#R*C2sE4$z&qq=qBTLw$&*X7DIhYH@BMy|;6h_`G^@u7mAcFr`Kl!zk zJUN64jB~k+Ul$CvFS1?SS*7DWT_e7pRLUC{kL$r#^?+ka)C=F|K-I?3kE4uIhG|+-^neWD*;uVi8C!3?hS|c;QFPAi!-@ zFbIw4+~VOLn#e_HOj`#kajsF1po7EH)6>~z>3Nfoer~Ltpt?xbr+DG`Pt5K@Wymh8 zY=MuiGUPD#Yf(Yp4%Nw_8!-+Z)e%o)Z$y@k)5F7~vOW6)49r8cStvT3TELrF zK%xU@P8T@(b#n$D*tBHXrV^~fM$dirDknYgU}F7$ysVdJFY7l_6bgk_*afR#O{E;P zIN<%$2^ElWCKHgDR5GD$Y`T`JfATZ>Fv#X@rB}oFF&8>glOWL>55l(sx=lYA7Pbx# zV}~N|G{kxFPLB({Jw0uMy;i}{ql`Q4s$EAi^Lc{P$pM>;90u(K*&clo{{shxYB>pY zqr#DmL3*3g1UBJMDMf=7b?ubf#U*`^VTd|-Uvm#_I`eQSgg%S}j!G;<#pHK_f3NzAFACxkjqoH86L4!LD{5J48jG)9>2McntRS>L?nFH=rMkh>+K6+%;ewuREB#WqBmjU$G_%stcmJoSo$A zs5>HTAoM$iP}xp~b);-sawCB>1!7-B(iGy9G^=k! zxl2Z&feoxF46EMo-LLEY&6CXc;!x!Fr;mbl>(QBVErj4}7_Nt#!+USh3CIT37@SVM z6bc|TiH@6eP#?HWXiR{Pg6oWJsZEF4IAG-I2jk-GNozBj=>$Uyufu{AsLT=?WwLPk zK?eqT$q;#HC)02J(hVc9&&{f@hA6Os;IW`mU?nNwX5^v0(tMgrDaf0tpSQ@z zqq?1|^5e219Pf%d=f)5HSR4UfvN1J`-(vh8IfBXXAje5haOH_;q2~l1wvj zvz_*LsWs4u`qc<_2sE0B zE+s)Oz}R*)CJ$`yv*4#MB2f4XQdQgOt5ch<`vee+1(X)w5;KuBy1ASp^ag$*iH>wE zNw?W-UNzZWi^AhPvL>8o<#Ij!v^OQYL%oKgJ3^s+L`>SH3icnmm;U~XOCZD0HsFu zkMX9FwWL;|lum#%Nh*kiRf+u2kqruBt}tC_;G=BjOXjNZ@B$Lb^$qKmo==wzO_ux9 zDyj`OBubS63U^a`Inb^-+vvSex^-49ZNMWXsBt9|6fl-T=4)%TUbnJHlGi%U|6DG) zk`Yeys1&UvOZ5ge`4Cr`6brkK+XM_U>js5qlNgXm8bun5e9Wu3?wcAbLq_u4a?_=5 zMX8mN=W2yy9v-#)ZgKI88u|kM!=m?}f~qA%+-Ew_Qj;5pTR#to^v~O9l9Wt{iHRZ6 z@nl>%8z0Iu!kb2JldAHmQV$fow22R|-$sATmyAXl2<7E@T=${QxVXDDt@sf+Z`J<; zK>;$*?dRO&ni&;ADLOHMaOjD(_YBES(`Ih;P2>X4L&pSJJ(}z*FZb{%u&>8yi-fxnR zU-x0)!PsAl+P+zn+T`@+ov*9Dfa&5%dthv+!3+MaeXEz~%g{$Fs__Y0u8O(+4-{J{ zV*9I$qx&|rY%sw$$yEEy)P;@)cLfPNh#g`mU}g~%w@g*H2#x4NQr=}xMrIPywwvj* z4DRxpo0|!Z01%$w>os^rWE&{O}ZUam10BGM#{bld|-eb^P{kyEG}XXn*=BY{7xRU^<)yGoDbI zJ{8z4hNKexXQz~0wgYL(i9PLvmUM}bhK!+9DkU_6^cloAFjfi@=f2}^^n-qWx&9uS z&Jg${9qlZz_#t2Dup-^As8D=2*^KVYcC%!&g%B?ueZ-qgjDRpR%a;3qyqM^&gbpNo z(v&f8(pdkYhxgq$wE6Q+ewM5+%(dh+t@CSHFS+_?6?qJIGchr$T197{YlMj>9xnfW zVZoWxe{H=G1UdGykn2yVAFNdpr#^xjc9b}V2VpM=K6D)e2+vt-MIEwApSI>ixwO$b zz!#)08LtZ;1K*e~ax^AlwaLv7AAU(ch=340ON39~x!NscN4@Q0OLrv%5dhP=f^HiQ z@qoM@&lSfrHAq9qiGD07$f~D6`*q0v0H)@@q-`F>$pHR#6uKQT@u=CxVGwzbxphiW z%Vyy#omt}5zY+jrF1LM{sR#*6DzEWmwvHVKcxwcPA`%*fWS#Q57Eep3$J$7ApvwP~ zG=+vNljvkk2icj9*uo?_mCd_@J3=sxIj4;+VcmV-+ye9qK2w+csclsuds28I4bZ!a z&@-@)D5M1Ezla9F)^buH3^5}#f;4I-0ifZdN>P4f;j+{2YRVqlO^w3bbeg-5d{>8adh*9QX#HF$0PTTzD&KrHUQ=>~n<0u0oM+jX$(lLX?xf$sgf| zYXM4(Jc{sg=y{+M(1<0-O96IWWZ(=I&0;A?7|aF_2wU}KYpCoocDF~=HFx%4-Abn2 z9?LjuaHq#c-N%Xf8qe_S<&C6@fpK5A!{@P?@F*|p^|OZ9z7>JXQP(1?W1 zHums;0r_$CJ_@L5=>%k7QwUL@uL%yUlBudh`Oa2)rVV8!3Y2T2VBPJ=E`ZT;&fK-q z#U@5}j6RI!Jkf`tV<%|v79{x(NFBw4r94#)56(71>>!3OVH^S2WZ&dIWB!hi_cMnn zy8v)?j%sF(tMe$^o0)+%b;{p>^}jAYe&|LJ4vqc#%D+F^Pr=TYhqMpcAa&vuq1QM|^4nSpujN?yVtD_VJKaytCzUU!3I9bi@Y*; zJ}`Qh`Z$n1%x8fD{6ZJaK4-|LzlbB^*bDE zOm1c?AH_Cjj8_Xs3joIo=YHZ?)br}q|FuuwgpIo+95Opt2n&Qph|?C4U+H&i zb@N$%4r+SKN#Wt)>E#Gn{o_Mx^_P5Y?T{7tOEt15Pzf-b zwUvNnBEP~QMAfhIfd~qE_c94+yjU_JN~huh0tI0~-yLT7UG_HIU;#oSU0F1>EvBab zn&oTXrn27o!6mo<3v?_u4L!-9xw^|}tw~bGrb8PuHYm>K2C@{RghrTca*3e09;Cc} zoqBEAvSrQM*|OQA0@@#!1Fy^+`Ac|bd_xfO_??}dbV7}fCFyoaNiqS(ypia5wzL1^ zt#kmBd{yWmn!f}T5IPG{Pqux=%aSS>BwaeeI%3?bje|}ke*H=c=ZC!8xG;*L()9d)hlGIJgl z-*A2K{LT=kdc(e{vzu3vC%0e2>yCHtc3kw(fW?GJXyy=&#y*qVl4%{b-_BiHP%V3v zXTvbE^IwZ%bV0}{--a5FMfGV>Ys|)s<#<{Gn2MDGZg;!*64aMxR|Um}En#0g>zVIUzucYO+D*2ijy)tVHcSa5E>5Kky&9UH zIr99kB$DwBOthU)jLL{b1=mi?8Xs|O^r2gOjP|ercy%nKT?~W^=Q#Fo&FT72PS^kA zbUhJ(zP?SlJ-mZ!t#A5Rx!AmzdF81bD9HOixIw#r{wG9d_fIG1ywVQO{h>NE8GnuwQ)3i*NEKYzrQxeMo@WM@Lju0!2FVngP7Zo zZrW~d!pT*hiQj$vxBoi#?xD+b_Feg}p&KLL&0E|>c{g{pZ`qM2`;VtmcJ0ob)GyB4 z@y3=byDy*k{?k^Cx>7!F%2PRqnO4nzpx-)H?X!zNt;jY7?RJSeWIWfJ|GST&c7EuN z_kZ`=Z~vM3zTxZtp$_uDKmOaz`uFGmyLJD*AHVt6{~nv)Y{u`N{$GA<(lcIR4vJqB zN~2d1)3$Ga1Fa@_2lN?1<;T1m_`84psBKJp^_wL(9$dxtVG^%LqaUtK2|8z(IC(E7 zVNs&A;kZp!`WNu(z|e>3fBx1OW_~yj|G)hE_mscmY?hp~kLAO3MyxpapB{*v>>oVG zuoE5`)JLhL!TZoG@+Xhfl9NXcA!K+!@8GN7y>NHk2VLmDUvn{L6&|maA1e$$ef-Eu z^iwD9b0h!cts~#x69lY4U`^IktoY;i3~c*2%f(U|Tf>^kgZF=sgTk=dlMkLf`2cKWut-B)R3=gEc+@4=q|yikVsf8X~WL#Ou>3LTRP6dpmlUkt2V$mVzSn!L?o6 z(C>=ZXV!lc!ssrtvYPDUg*+UI{*KmS#y>sTyRNq2h~@%Hkm8%_|Mc^&o%=Lef;Oc5 z*kk8c#E>5UHFjJnKf?C0-GWnWijN;K3f^v>Op-Wac)r-xteG@v5xAhR0a4xN>KyQvPQi4&$n-$G^o_IvV!YtJ31v9TW(uXvi% zR_9#vK~-RPPNUiD@O#G%{`@$3z^AZ#jpyBs|A21{X8-W2%aFgjz}+2wi^GW6Iw1OX zpExvXPX{nPWDm!}YdWqgj@OL&K7ZcHRwv|Bfz*68AA!dC0{b#;lsoYAXd8xk($vjW zjR%i<9nkJ%M66HSWHGzJyOJCK&hRQqwAkm6VK8|Q`fE$`IWy;U`T4G@JFW!+wjz_m z6oCGAth;X5N!$1{FZ#Q6EoMEFEz;;k{OT`ar*t zSmvYX;>Ct;f7Ry5T1Wo(lLN@|PskUyr>( ztVop?a{oWhz67etYhC-Jo=RH>s#Z~vl4TuTH=_P2+3 zzr*uBPx@i?dWPqcaiYdW8-jszj2J*s7>=P0;Lc%`$~q;e9nczSG@1wd!O;LqHPuSJ z)Yv?;=n#IO+Yzk`*Ds8&;axFIeTy8v;-4K_zOETPsWI@7J?{;9ARuWO=m&ygEA^Z^ zI-qs2VC9Q7f?2XT7qk(myaPuO!8ie#WNS2UNyRpd|z0NY$n0WE$jF`z%qg(5gtGj@)G z6C7ns=cNX`B%|VSLWQJKB$+E>3-SmPrFdH(`6lpu7hN9zFK0FnQ*OCLHw$0yZnhCMHtR1bsLK&Frqy?8_i`^a2S#^bm?Yl6=90x*d z%~d%@8m)=l?~nFZ;PyoAw(QGKmzWoAKCHeA9<%o2sl$JO6@u^l z`-P~#DXeqfei*K_X8-cHH-ia4tomSb?Bk+@3LL#8aOclK_T(c47-PhQN2K(Xl}99i z@w&@ELx7W)Xl6D6S8ybU5?JoJL>QJZ8^^vExIB4zwHnL8s0$(GGz1cE47B26GFUs1CK#MiQKY+3iRRG%Q3k9&a!w4D( zLPif9XX^vpc(H0I?5gRGC_{|#y2YU_7Oj;GbiFzCnLsdfIcwk8(9oyEYS2DRnM)PR zQzhVTK-`|=b3?ibTp9S-<~A`oI#t55Fy>vA zDq7vEN>y4#IhP^;;A##9#|+UP!65NkkwaC^`qo$i@oGXvuKS*>rQ>1rc*%f5>acDa z?(|5|3`oEX0>%vw830pvV-8!1NK6|NN&BsA!lJ+aEOTZFIefrRdaC-_3HpO_%6;S; z_>{|I0OQKvO&s_hY>qywT&sbNd7o+c-#A74*s#@+DlUA*jd% z8H{?y0A_uaGlt)qQzUrT{RHC^{%Q{f5UhL1z?LecoI{4O$LI+dHI1ZAz}PFATpkF> zC|MK6UVyZtP?}CzU&q-PwdHkxk$Z_f@C}L_loAbK6!-F)PKCW;5g+Jc+1dr%A&r7E zw%4IAykhlmcKVgd!LX+f1!wH*t(Y-G5f&Z9wsr)=Su45U|EZIj6rwDq1QUc>A9k;n z7vdos+H0xaUGGC$THvKl4ef*tD?v!RLdXRO5_cI;cdVuapq1V1%hvqX)>d~VPr~9g zPBo28k2NU{Kc5_s1(t5%+T|vDtbS<8eN|HNltcj-ah_2K(@w-P3P>DNHm5rS?!w4OjFVG~Zg2Kk6&ctE0wz(#EW&|O zp)WWs!IA4VHWfyIVLbFr-C;r~`OHqH2Wl>=HTFuEgU?nq1 zBED4Nu7u|kFkx0uQ1a&n?&YxKFf=sO8PiB$&L*dl457h%juX#zME-09Y}+#!gV+}i z6kNRtqO{!CrUnICD*8S3;bn?soq32ZdD1wGShp@H1Bs3P)0@~?gif#V&D(mycGf#^o~#d^d&x)VQH*kDrzx6#hFU=(%q&5BuqPs{ zsgsi@-UirNUf8*2Dk*2Z+C{6NQE+YPHTe|yo-Gl;xP~H9@@(p|P7!NQV?sa@I2evz zE5bu-xfa;;K9%MULyM0l-K zCC8n8NUT=Fju|Pgk4KrKQ0FL=nqjk=yOad`cP9;foB&dX0!j=|GJt{{6@(Z*5%h(B z80>|6Mza|RF5h;7wmL;=*Er{|ki?t7MLi5FzSRmN!)m5oDtfnDIvqbh6a6sSQf;1~ z{wH>lgkn%a2ydj6IY9d%r_8v=VYZLe#mySrG>k{X?_WDQELYfO?rgLnk%>~3A(}=B z(3oqhDM;lTaE>yx>SByYbiTFw=f&`)9TC8H19o&tC>xL$?=7;8474Wl$ARXFBRASk z11~pz%EtJAt3@4ot&4wZ;{oA6pSz>d{f4&X!88BQuDSK=+R&X+TQ%yH2isNy%k zp3QxVM#Tby0#{@(oeyfF^CsLFfe4marTXvu*{$4{N}FPm7eM&xMp^y6DW)+tI$4pI zmsc!%10;}JDFjekWl&%Z3rzeQ!+2K=+#7^(iR)>n=uQqY2RzI<`v^N@uGQEd4@`+0 zbJitUxwSF=pUf7f+IaTeg6<&0SgvxwgAGGnw|C@reC3{qTFURR*qlD9h#W<6m1XR|1c*suyYA0H};cU*+apv zPhp5~?deaEU;IrR11~kmgS_m4W6#OKT{;is0vAL#lvLnHPB8)P8SKbYaDY$Nmkq-k zDR9@D|@`Xpr{9IOu;MwF?z8~)0-T*RluHkQaN6u{%VO0RLkX5mQ;4w@?tdd zXQ#Yp0hPS5&KE4wz&v^$vN*s+zDdzksTDo?UvydOQZz z%14bcN(5XjQd>eXw3}_f^iL^?w!5={m#QrRJ1$NX=|BkIF=XZ|p@6fxrd1p}*REyW z>EDd1_BFTSk^8CAXu@Mob?hRKjqo)8bLXBvJ{PRGR30A--y z8kevN1)W|Y{h$@_EIP+PEfx_gDd!Nl!x+#UODz&e$&&^}wv1<0&mJ-SZGA%M+8;#m zNapMV-TQl&?1-Fb-LyR+T#uED7^dj!JFJVgN9Lm64FMN&j;y9(;%B34o>8&}aN!tE zF1>^TYr|kbI|U|k$otpKn(2GM!Eohx$N*cogDF3@jWYL;HTRH$00^Amds1M^h`^M|Kn9UEN2dWV z(e%7`bn-aR59Yc>0;*Ju3T2V!O+L{P#*}K zU2tvgXn%F7y0S#)AH9Eo%5PJYyHmwPdPhxLC_@NXzJ35+uRie@8}>X02Y;dL3OIr( zXRMuhgHpAFpfClky+Q#Jh(ivFA>hXvh`qMm$K>=F_i!U;eX{s+}_caB05@7AmDFmUva6Uyf9DtIz*^S`PjSl854(s0c7 z_5v3Vo%;m~XRvz#ba4I-0!>tm@?c=J9)e?Z6r&b#fg1!=(Zh>y-7UWEY%6@7l^eVb zfS;$h)d;5!etS*si^e0zE2)uvBwY0Xj9c@NwJ=mIUb05Dkv}?4KiWe!NxTq`(GF=< z_wL}M-v&g9zC$++JmN-1M|W{7CKA+x#>j^|-eua`uI))t?q51{CA4+b3`4gcG7FSA z!mHH*IjjoaU4K=??p*3FZhvuLNV?X;T^w%Cm8OROLWat{sGA3sYv&Fm|v4&#~0)FmRZ8b|- zO+hHTnC_8)L)uNY3oYG+QNwT9i8}*;8#=rtQzRh2T`hlI+c$Y08Lt$c$f$Q!<~&s zUtqLypGSnFT(hhtxTlJ~x|nEphwREw&jJD*jK&N~W&c~t)O#On@`NQ=1b9COmL3Zl z1>IP1A`4?|Y<%7x_`qH=ENHXD$lzGXOPwO_hCNgGwq>bavK|7F2pk4Oim5l*VcXht z>M@&uA_L<-kSucYui~-vAe1jsi1T}BRZ`R_>ySipblxu@EiJ;qbbf3@!jPf6%mE-d zy@Yo{<9E-X@AIC$GbIe0g1Bx4xso01&G8Y0KXflj>O^uPe_Jg0@64(^_9K$NWe%o1 zFplvKbc>o>jykC6bozP*%8QNf+T5#oH$kaUG6}zcRPql_n8?>pDN&|%HQjlm-=W`~ zieafJM={2!3)Uj6vX&?hfHU}{;nc1g6ZCHQ{RbaMJqTB$+!KWBQMTRnSdWzPF??+) z_wc^RXHT6^Bo`1a%r3yhb5pn;Y%-x6O%<@TEyci*b?}Z7_kAkv?>7FR7CpNf&&Ol@QIaE*026g3kDhnif#krRm>3< ztB#>iT%f7gjD(Na42?mjzzz5Fp^NhE+d@ha72KRa>$Zr(1PZ~JcqNS(r(3d$X6B48 zDD^C^m@{U63{JO2$+^bJPB`F{S{ReDaE1gvDi7wfiJ-SB6ax}=J)Tx6!Y5#+DZa`c zj=StfQ@fMrj3YsxbTN(>=j2ahauWRih-)7VgYQUB#QB*wM-t+frC*trQMFAJFfGBw zJKp83ZZVafL<=>~pHgaMy^MPlaET{fnM-K!FSo+LKvH*j z&M?7mpPj7>*ax2n(S1FD9fGg*o@UZ0$nks&Zqq)>teUG;dnrJCh|x||fynbHT6VMu zH)@7G&IU||UToM=umuj5dli_Tq?x@SiEWDlwW$8efwi1n{?EWvciQuC=e(U#WH9Tr z8<1C~Qz%Hqb>ss<>t3<$5VDD!ssRk!T-|1lkws!>BfZ2%!W1DjG=9$*6ZfZ?jgE;q z#-8vipoCfp;xxnd82U^(70hauH5?!q#J`Bdh>-mlwsta^S}5pdDGf2(ISvf{mczIU z->|#e0}?b>&YKk5PC39c0LlMx%y=n(FdJo`?NxcqfG?|_kfGh#T72Dwn%1B_*>z8B z8VeLE3S|Hgw7ARc0T>uV2b=>XJK}5Ld6q~Gfe&?&)c+rgZ}6x5R%}JQ+??LzP5_P* zxCv-&5(OwBVxDwAq*&3%4`b9Y6$9)7!#0>oNw6DPk5ShesPceB>vf{vfiz&oU`oALiTs808m@*h{b zxOkT&NW-FBmgse})GCl-_VS(XqiAFXnixz{$=bCK6L4^GN5Cgc@F}>lUF73zf3Rf) zzsBz6*`Pu1T9{@jcr#V}f6hk1pY`uOJs%*$mcvU?irwJLW4JiQHmr7{)V z4}$@e>hU1&tYr<*^}XCX^tsvz;nP#za)(w`QjY+_pAutXX}@^{daW?p)@I_-bNs$tJz0tlBR!!~19X zq;f*QyYIv{bzj)-lB-n~=Q=&D`bh z%19^mb_A`CfGF&(keYzJc=_^&(AJEDZ(OOI^oWS6{*Cs446jKsYz_F*9S)Zz+*Pk= zIpfrzIYV_+ns(;-?6I_5SbNS_x)wrBT_-Q`*QkrmhHgJ0NzdO&b=>xo&r zpBfq*p+?gz@GGV7A3I-N&d};AJhUs_{~x}SG?exB6b>2N635g=FP;VQS`Td<~w$L@J+G1UJM{V zM@?7v2U}gUYBjXYITEG;J2=}kl?8ekoNe8{Qf}_qTkLMu%{{X_Eecac0#n`_E$aEO zc1=LGUM=jf-VxDZR$jo;uobEZ0a`zDxg61p3C%rF@cYny`=af6N&nde7x>1}dXHkI zV}4GI>`Yhln{;0bY@WsOm(lUexB#sKo*QX{;rtCNAu6gw6ry$Y= z@IeWQQB_O!J?UQ8GmtprR@U;bd*3?uIF_PNsN5gXKsQ+rCX-8_e4_#Kl8Yd?zogAK zDdO^(0JgBfV+&{mu%Z5C*lo`#-Jg%0hx>1gCs#-}1~#ZtZsqixStZ?=tL($-4}uPk z*Kg!SIaaM0I4TNM`0D`W>YNtU51n4vPmNzp5cO4B%CZg`BC+>BKK!;x<{0hq&2#pS&@(7d-<3ui@VkL8x4 zC@Pcq7GuRuRiSXRb6Uisq?xY=mM(ge>-ivOjnX9d4B8lIh(LXK=_}`|^6sF`Ot^%s zldy4CZVd?;mGmy1C#3)5^PtaL5Knff67i^W1n-b<-ZA9E=6Y3p$Bw`TeAf}`EoATZ zIyFNaZwiHH=qFxaf44kjoq>9a>-RHj#d~$8jp~SoI0Ih%jVf-^(PX82b7-$GeKk$p zu!6YkBENd%DdZhJz66%-Gn8ywDJ}?Fk~j+5bB{xu6F{DUw09-(7YqbJ&RYt zl+{pKuV|z93l2=Ge`fbuXBDs}-Gsumix)abjTOmi*VOA`5Cz-?|&!ar}3yH$zW;eFg;C8DCiqFd^0DCy4Xsu zR!pDS25DH19(u2TS?!W48Qq`vaV^NyZGr}-wACM9#L58$E9T9E=<2YHl`j1obh;@EiafV#uEc>J=bMU|*FlzgT{ZE0Ay}4X>AZlB zd$H$-`)Iu%o`4lp0J+$bi#kP=O!~XiQ1#G~5a3$WiDDVXPx1+L{O0o$*SMt4gVx(7B?UZ}R(Iq;-?wb?-WYO%{ZgT&Ex7aWk% zsd_iHao|Y7RjRvL<8>+^K!+^P&s`p_j@0JQYx*ji$^QSRsfEW|hGftC61I zoTK#_r7VPUj_3oU48ij`*n-vUgt)3PE4^c|;?bc)hjbPL^!Y6hx%aK!AP7gf={)ZA zeJi~Sf(Hd3d|_|Dq!#4PU>Dzx-^*wjL)~h zv36h}Jx|VfIL2A^w40T;PeozNdV$TTHxf;rVE{6QzdQ}Weg7+`SNy|$vHTn;sJCJX z*!S7yz2^L*9Z9h(jL4NsFD@aO-861iw6;+5!w{F>zlHG)%4l3vU>!GaMN5piOEu?l zzwbW=FBjx2t{r=YU2x*;S-p$jg;yt2tzV61Vwct)!p`l47LJa`$IXk}DPQNz+s(g> z=&xf0&-Cv*4zy;!{e_*h1JtPwRRF%RQ$(OWG61vK(O~C7P@V1(zv#$Z0ke8 zEaFayD$Y2KiXK(xWYDBjGob^%Nm0=R#9xM+PtL&mI-tnW9o3yW;_vCI1{}?+b#IAN zbwhQcVfCbqqa8a8vA;aIfN%H7``@>{H7d?}7gg2txI5^BN0pTy?W6lTSCFT(u^;|o zyJ`V*yz+ug@=HKYm608A;*IW9l(GKH!|q>8KO8jBP8#SPJ9Z5E1q|`EP=;8+S>L?rD@DpD znPO5{>eT9$J36IO@H}V5Zbj@nqTY<2^QDNEGa*j+Wjk)h#r@4xE!Z@+w_eo=QdqX# z2r1Sq!w)5C`{l#hrh@Fn`;sgnOk(V3^r;NDJxT8GEx^))m%gNVr^c{`epx+qe)n1^ zqN+g|9$4zTsLs2~c z@%|0}Zg>69ABoWKqp+n91Iy(`6L6RXHdPRqI%pba4}rt__$3KxWVGOy_VJC9%U28B zi#7=_J^$y&hC-MkQBNo283S@YAJ$5kNOHFJo0W=}(q+B=YdGsiMv+SYxEmn^Ji*&{ z0(@{!)l4|B48QnNO+}P5364s$kPt_?H7bPW7`Huy@t5tkQT=d4>gj-T*<(NiS{bH8 zw^O!2J)?G?G@l{a&ukeD9k=UpF{^BN ze7MwZ;z#VlL4B+Kh5L%zzHVJEF0BR?dcBx2XVc@;x=9h`{dd>60NmcYvkpTGFX=%U z2RD5T>X>OVLqH)2;ejNqODd}l4g&n?*wxrkYF_ zOm@<0Rsu!KpEqiBVIz~7Ft)4Hgi)G24Uqa9?bQUdv&)YFwFDe4`IPX}me08_0)9-I zZkPdwICcvEwl-1tDaW+LYg=ATV`P$YqaZSfbh?9MjiPc(lP@TX5Qo}Wo_4dt;d_a> z4Sc+v_+LfrgQ@9=!)haTO|y_!7!^6t`jS^l2cGVqHqO!;_}p_3nfOHSAgyMWf&G!X zN*X#S5dTq)tMrsqVHLL*Ye}Z=5>y^3zeVmeyTl1>Ao^)yU-E-@H$rF3 zSM?TM@=U|d4{5x#@Y}PSg_4?rZ_c@038uO=&yFlflnm}U*9pBY%n&!2Vacm`4dtFT z6L{e)WIdjb-R@7pEz>;??#FkB|9a#8NP*7KdY$cJt6>uVZ98^s?lnQeSLnfeY57mMVuS6-@#|OTZgQ$Bu{FR>=}m%+ zkTTKT^fxTPDqT(l$TFoVC4J$n4^PMh?6`+ye}Dg`__OMah{D9EFYQ=<_9z-J)=(%> zpzZB*)|vkGna79xd_{$_Po>xVHE{Xran97ETjZ+mUaf^FGC$@o+59f#bQ>??fRLRi z>=Lh(`Y8Je#|;t{Pk7~1&oZSaOx(8l(WERFC)o~PEw7!BR2{VOUB?TS3>2R4 zt5yk0TQg44IYPb3mjS31 z7Y}Dy>ODH$vPu`D!`jJkq@oF?Wn~ut6U7`=^6&uO3uG=IoKnt+dV$I;Wf> zW-*XQ>eK^1WqQm8ELd|0&i0KbPk&W;AmD1uhUzl&wO})-3B86A-4vy7~itos`kdElA*`k}Nj zy%pzcCt}fToGq!;-TYT%WJk}4{MKmYI&wP#X(Y@4w$&iyGfp{oJ3pG%68nh#*!RZJ zxzWqiJIoQT|GzTBDe+zC3uhJ&vf)h2i&HUpi4en(-ZpYBizsU4&-?;r8z; zQXuK;71uEYP0uw{dZsuJnledAl4r$Uw9UQq$v>0-GK|0U9J=UPyl@}z2!8?n+Cvof za0s2|xgCP%4yJz41A1ntpXzBKA@XAq&mXp)o&Ib1=Pk3~0G5f1Yn+=P2*Qp`|&wPOq)XLkwY8MuBhvT@i$j-hO(^_mq1x*(Z#IbXCGe# zngx1rwwA{NvS}lgfV!na*oBbAo;`c^t$K%Z!flm}A&X|bH)^Bk2Uzp1lZqt=0p^}R z@5Hjlx0qS16{(7Ar9d=UpWvr4A6K1hmYI<3mS6OxW^=sEW;5H3rHY%jZoSN@B1yl}_*H?hU%9oD3iwL%5ddK~qPaGwfJx4tV*QAHPOc0f-kru9wsRG4 z8vka6C~*=dgN0xM@71CYexcU@Px7~0I)OwJ zh@=zk>7(if9E0Q1n&)sdC88#}_*-w*{=D3d^r+)0fus1T#(K(dWt1zqIdJ;X+$!0o zn@TT+B_==5?tE46R?Sg!0Y9erK!{Zq%lAP?3yxd9{C$!>H7`ohkj zHqoQ{gNs1q!vIt(F5l&Om_H+c>)9C({5cyJa7u=gc>l__N=;J3C&#_$!`hLNQ_Dks zBu($PtG`i?Kq?0fMvM-x$Lb6SVKVHTtGrwEIoC4_3rH~gb`0%fXVcAzJ zEB#uaXr0?Udre9CHwX>H`= z9{!DQwD$bH*sG=JvGvTyp66~TlVU4@8M1Nb`r9p7xN6QnZ|pNpc&x^D6lcMRJo`)y|#3_d?m?XI zC-WnO&>&O^xHj@v$wum!sAv$~~Pclr`^N0Q!IDrBgtfSeI-A zsbSdF`T*qM-6IACj)o<@qRm41Si!da$X?NIs0I{;H%p@62pi5A0eL8A92^M+f$Sx| z)V>ut9!%?X;9XNIVz*{|0ud#KM7`Q;rnw)@Ydu>A_uabTa8){@xQ@71NDjrGAE>tJ zM#y$dFbk=mr1vCYfT4(A!*o=WCQGqtOEHq_Dxd|E&iA(C4nJMZdR3hRE4%&a&sp)P zG&0PBYdR)&^V9fSVy(BaOzxA{VIR0}d&m6oF=BC>keheQ#YpOt#ajW{?e==K z`q zVY7o_FpSE%b!)XJV~YQ;d@Cw9bimrhmne<#e+m*B;Lwh|=I^gxp=gmVW0HUyI1~W# zx5wzXJ)m+KcSg~OBlK?=I7YglkbcfUl)e=As%)-vlDZHQzHB2)TNw6=jEu!iiDKo) z8Wnkcck#SR>yR6Ve35=Xb1roum<_U2xd}NI9~e2jN#Ne-Lq_|8@uib#kmM@GY@M;oecHSDr z(>B>NSnYrZeizjwjNit4YB-vP_VAC>cqd&59k^=RDUCYnn`)L3M<6@mrovu`i(EM; zJJdj%I^UuFpy-*asL_>~vX<0BYU$-EG_ImqSIJ1PQ%*n#&v>OaamXxg2c!8!%+vbN zyt(Hs>hy_RB{vp5dQzQ#Fp>&tuvQaCSh-3AGj`I!dleoE?+3o4LNgzRlNV8{Q4Sy& zZ3_2;{5q0Y%vLyMoFjhI&677Dl-;#`E4Iin8?D0jS4lD5QuW?2{EcRW@My?uuDNXF z`c3XYXF`TRagcX9FIS(m+eZ=WPMJ;1CzCY)8VF{gNq#EZn~K?)yc(xw2J!_X`epCE zSmn0XT>@#EAnzOvv#?XoxyLZC2FW7x^5(Wn;=jXd?FBYuqCqy%{5Fw=ARC2hTp3<1 z#1IP#)>fd5e!s`ta*L;xjLAn_s;oJ*dn)=u2F^zNxX+t}{G8n)Hs%TIyL?4na{bqK z4VB6mMrG@{xCBEE$&b)ga!^9w=l@L!)pqg4y0Ik15Alh($=Am$Qnf#lLd^VkP@^N` zO4s$eZbNP}tBsJv>vhjA{Yq_gN?5v{OOERbjjbM^L08_-&Wz#BCY>(GrdrYRBBqqc{8?CMC#r$F15jVN)dChysNohhNh zv-svVlC9u#D0il;KyOkf_uSCXaA~Ky*nJTYg0_IOPH#>Y8~Sv@5?mh+!gP303?M;W zR11rh0jKovuTJ!Wn{5jj4%kdn4NC2SWw;{@t+A^nR1D880qJ>J0Y|v0r7!@H zJi{V~$-r8?4yL86U5T&&!;)fsS1Rlm!6CO#Rn#V#1;fCEeqE!JNY+Qh{PXw#P*ZcR zggcj^hh8+09yYRs9^qsH`vLLwiDUbpnQ+_N+o6Zxm*0klYOHs(u27^3mnlec1n%v6 z2-X>eCTf(y^M&ueKJPSG*xVVilU#H8ddi67@8`E+3*&%`H|=xrfsh~)e!pET{aRS6 za23b?S9@4GX*fw`xi)1Z7j$OtninhVsyOeJ#y#LPVphVve>f{m#Z;K!eM9bQQYbE^ zFUkH2X>?fF&f_HMew)jH9DdbUI0D9CmbM`bx@)}c|)i!f)^V0E` zD~ZE%`*9}5u%76)ZQC}Eza$%F`r@`gFn#)Ke%Mvz)*h2(L~w=99({-&`q<^buS^B+ zS%cJ6;J*&3xHh*^k*c&3G|?DpQGRyxMBmTe41bcn z*X24K1{trNB7_c5lhtohQ%(;clcM|v%$t=b-?S$4q@l%K2+`0Pn$zB0!z6XA){10R z;WRfTmufKJx*(>L(8+5Hy8EveF5b?6U3iaaPGVx(!>Bh}MhVB()a=Sl<$h(ivN^1f zGkh>WEqF|K$!28T2rrZ7p_jLGAx58;S)-@tO3IQaMMpy3S|6xK3*@7dqSn_u>1hNv zj4mjfjnP#5Hpq@m(AY)}O10Vc%sMM02j=&AS}TG4BW_R!f~_bjpSF9WO(J|(ldQg8 zb8Syi+FUlNcceF57VKWa?s?63rJ@fgIicLeS4ik>{E&KiOnnxKxPzN{S~L8NKg%zS z@xTR-jjrgG@|`s6jP@MKCW*&bbl36LOhx3{Jxeo*fL@e<8s&8<_#xZ!>{s$SW2<8C zu`;UTQG&SA>i$fk_y#F}Po}ku|MJ*leCp^oX)}izy@zu1T3EBb-NdH3^|4y(IdLqb z`Zg&u`7POzVcEE6iu=4?F&w@*l|;A@#8PqzBWo3;RAdg|ZQo<45RJmrq3} zgL>^l6I_Gci>(cItph<81cnBgs35qt#GYpR;Wbx}NbGAX0xh(pC<|_=3@?fQwWKsm z`K<*h<)DRzb(KpiK68)fr!BnDpVu6*N;-F@HFNK(X8cO&^jkyec&?UjrT6Fo=Eu0v zkd;zE#PIEP($gtBBf})IX~mIM(uRrSpN}mHE;N+5QxXhEzh)f-H{I{vO}_g6e*?F% zEt7$O0L~8Y+O-Q(wTe|sAy^wcY3T_%fN&gxt6iZ8vR{XmE>m4|bjlL|kTkR#fhA($OrA5N4u zNfc6a>x$kyl#@DD+8|37&H#YF0f?WQ^82(SE}DN_1e&|syNP1@><5z%T;e~ucxcDd zjZGo>=(dbZjs62F5 zMHgXkuR}YM=_s$%xVFOaT5dY5tehr9a$<2f+v6l$)M$ps2e9z@CdIXguPR@7)EkW} zn>}iFmTC6KlVk@%aS4ptbzJ`I_@Vl z(ge8-TNW^d+W>oN=Tz%^NU_>&zd91l#M*|PJx>%dN6P~8?0=q3$}?Wmn@3(gewE<# z&`D{FG+n(X)q;Y~rPVpfc1a0=BWA6cB*v(^)4d?4ti?pV7lHr08zGV&+rKxC(--vE z{#S8Vx$#AXLUD?iykbP1^*QHrYPa#;*<0d_x$#R2yoqMrxB;@K%@0E|{5{Fl(W&m1 zh}44|%??I}AUwE3evqIloHl>sGV$cC_DBYAxkd12yw6I}6SL**37cl4rx^kFJxqlb zt;>bW2w`luHnKg?d`-J>y~K4k0JY<1lHvaNt9+-cO4ACyWQ_GQK6`813Kj|2-86KF zG_=9aK31-ZMX>m;L&&r0S4=EO1YxJ5BCoPos<=rxz6EU{opIMZO-pED7$&orKM_;M z)Y%`Q(R(xN4FNs^%?xwXZqTl~xlxo;&Al;;l@gpZHap|HzV1P;BOjPCCk9Wq5#lv_ zZ>pkK=QUp2ub%j)*eT@`bfEX|rbA@;^oblbc&Et^aWwkwu_G^6O6dzP?SURDGpiC8 zc#h7NUt+Fo5yXQ)dV*0ru!Y9D*|I_7b16(!BRNs}?gk@k-Pidaua#9*b)f=nwyF|) zlf?Ia{+xfZd{wh(baqk3RNOR9qTG8kk$Y+23FVenaa~(ka?IEJMhji6!9W$3i89P- zdGlAHDu|E3LUOrWE8Ro8k>6 zBcq1=r0K70@Amm?KQ#1)YC&5QW8gt?Zxi5 zZ)s5iC276EcTZRg2m8f@=rE#V{LA(n4h_VNcXJzd3gRC++3GmswXm#pno$1Q!C*;z zD|4s7Ki|BB1LT#jj9D?TCMqBxwgm!RLgpKl(mCY#3#!MBqa)}Nml3N}lA50rjes(} z!#;+Ppgk9|9=qkACf}eA&9&BEHuSx_ zL4PhYb5=i->QHPuTrD9l>)paSS~OvillPU}RTgrO7Zvhi(RcDOSEj+-*?5Y((4i%y zLv~GA`E8}^xL{Y8G}L5-x2R>7I|0SnXew!(;3+U0eG) z4BHw~{@6o??~t`9yA|PP6Vy5mLa!D$?vVA=3llx-;YW3P$7sPmkBIRP*0z z$9PK6w<)=yQlz1sq>_tx{LWuTd1=c~Xcv@bS75c7OEhX}7V}BltU+2nv!dEKlZY7Q zx-{_jYZQh;1>YBe8otJo;qkVO%#f#}t02%}s9DsdrhDMA%SDQ6^m2Yv@0#na_-gwQ zt#jH;>p?DZjBKWO+%L*kXN1{poC|P5#_xt))D8Z6@_X31O;GOSk03>)YCrP%-1koF zuCL3~4r~=OQ|9`QXcSg~eVfDuznQ!IegSV3|De{Ctb}q^ufA>8;7QFMzbe7m=Z~-&a%S2 z|6#}@05Qx@iW#mx`Reu_D5$S;nGO;k4AkH^8UxqE2@l||&U7qr4VJr%n9m>y@}ang zVg(L3n1gUzyPh0Y?3zZU!gU&F4G%XW*^jzwcQ@OQIHCobvBnTmHK?y=t&#K6glS{Y;3^+myXOAAwOyxt!ngY zU7d_SthM21r`H=PUXX>$h})!U*9o92xd{SLTcS;w%EZ`}Ws*~h0H1%qpg@;$C2Sb@ zXcoW?kZ|4f!;3T^&Da!~BpoA%g!*j2hSt1r0a@~f8MsJ%0Vp5nO;*t8=g+8^lNO8Eyzl%U1?@UIO!!5?!mxGnrOiaiufeWVTzhy?AI(0rNEaQy1z zdTuV7J2iI$O*GMbgx~1s#V@PzggbmrvlcGOPQ*=88D;qW3{wBYXdOR`2y#v_(&ODY_>mbl{B3 zn3~ar6m_2AjHwf=|Y36%mu7m zray8O@rH-?dS@3td+>S+*1f1s`g3MDgUsB!dKq_;yjdiw43sTP^%oxrxUv5{so~ML zwiI@c$tUO+buy=pmQkVGq12aRsqei^w}%Nb?32licd9ooET)T;|83u_Met+0nr{y? zBd3&`avheT8rg7~A@mhoEx$1R#>rF&hy%J1vwYqc3b~8M`xVndDtYcYG!;6Mk!!Aq z@v479m3gO4ccl?gBfMV~vlxlZN-ob)zM8~hj_2hd$(MBL5Ww5Bavop^L*T#f=VFU} z>9#f@8vEB%mC@Ucqp`lMsqQ4P)!sU#Nm{vAcan8qm3ZLTpv}9sL5qZt5p8f7z#2rAL|Bk_5b>9V6d@M zy0Ud>*uOP$fBxoK?SuK!aCADT)#X%T@o;_0e9^{lB zb~P+HMyamJ5dL$O%ue78Ok+tvM^a@;sG9c-m_e-ZT2zU)l!;hW`z%s;B^RBVBu%hJ zRcK{cZDd+ar9qyrlO0fKkV5m(AP+HHc}KFp22_wgfnZhUO+!=C`uHf=$($TEWUSVq zt2-YaPhUqaFmE<%4h7jV8?B?E&y9TZTyxjl(!R=26R03N>=OvE7Nn;|yz44mFb@od zTjiV7YbDHhBZUf}5};ZjuCWG!aqe(Uk8rmyY#D0GNRd2QO6WeFxrym$^YYcJP2;$s zMc9vEKywTPnuZDMyBXp9BajXNh5}?=AADh9aeZW$3rB}60Ub`$`Kr5``q0wFt{R0@ z^IRYKk3pbotbHAG|Ni~7h_47NTj4gyGN=oJ5#i}V1Z(J!IsLPXu-zD4Bik}TGLIiW z1Z79{oP~gNwTtsGaBt^^pC!oqUZ4$3xS?tj66;RcSnzDN-YzJ;`%^g;OwHVaLZrMc-OXE+TTz}1!nA`gD z<;Kx9tZS;8t3sqqO4SIcX?$m2v6S!TO)^C*nwq4jk`|z3rZ?|)!{VhQyP8ef){MqU z8Vnaq0!lHvrVvdZs6(~gS&uYd&DJ(6%|SjpB*clhNz8I${8Jq8Ra`9a3ZXu@F9RT? zfJX{@g+%EtJ5TBRD#U&F@X-oS0N-(pJ zvym0|-a5^+tK6Hf>W|L&{#=8)bVvQJNpfuj<&DrzWN;pMuJJ5kSBmxPTH-t8=aP%+ zrK3OryNeedeh+8NyLpaNa=6jS&l}{5QV&+3;Y@R5(Nt)^ov~SpmW1;iojqOC*Q-1P z?CWRbq|#)#XH5v_JgWpU)H)F?IQ|Izl|VNKtA|L_-S|0H1!QcRxA11&%q}xr2H;1y^OZ|*iGs9 zjoQNIcdk_0VM92)L^s~C69YRwV^cr*R}K+;u-HcHYjEP@MdY@6**i`Q1E&k_^c0;Y zg`hi~^%Nchw|M^K$<1TijkO%(;Ha z=#QBl$!zh**z(G~oM&Q|oE2e&sQAOb-NVW?p<3a0Av&RnqR1P5l3PNvtn?-9&q`3W zvW`D{RkaCONv$71zcD>*l6TlwDknSNCYT73cez>8xkt?0BVGL3lKZH1Q+e=$ydgp5 zztB=E=eA44gA-5DKIS-`|0;$A&aVG0{?~eS6b#85K(nM7rN7}iPa;;dft-e~0oP#< zbQ!$1m{Z&UT9jsjE(=!=9n$(I&}R;`ynOk`6oB7$EjX&Ea5Wk_8p>GPX2V!m54l*e zGj=j+YHD@@b!2 zU)44=ILKg_eMY6amIC)ftsrvx&+F+iOnSIrB8^$OM>DT5*yV=DN1YBv}4!%C+uCJ8?nq zJA{&F)hDsIJb+|&Vj8Y?eglEb9DOLPYflMu5QM`_HJh|<%AeL0)j;{u9{*i4Xa%*Nj`YR6NREjCvy1YlB@1JkKM;vfm7({WK!S{g?3Vgcj zTj%`mRIqw-t-eiMK45@5W`C2atK?}EW?AvQ5nKeP))=}2LN#xLMGnTK)(*~Sb>O3 zGgf}EMYo&rDdnxin9?!2Pd*-g1705~#JVtC56tTBF*q$;{4COGUYu|hYUjoL`SQ$X z{|?dW68-A8R(q{oe4nz~Q6jVA(}YKh@}bb2^M=P81|#Z)!m<%=^QmnHvvmGnC9B~u z-O_8yPfwM?0HeOw6S&f|+!o5m93+yh?Z>H^zWTyfd4k}YPPmlOOpzbtpvzxL|=Bd1;S*_;* zCZwMT^}Vx z)-Ol4a=P3}ut!AGF;}5zN%3SG^Vv2-JO@UI)qNw0?4FcbIxI-hHZG#2X1g6Otk~kJ z(}!ALB|MX>PRH5fB=Fmut-MFLwBDZoc*NMLw>|{;+yCM*dcY(4n?II6kT>dF^bMq~ zt=*}3^X4ls)Ym1NQEk6X)KJfz0+*873%O3UqhCGoJz&f+v$tOlTrjR5E0*q0U#Rj* zTkk`@^m--40Q!3Ed7u&hHcksaaBKmRO5|3wVSHw7dW zbMec`w+#TlWOjLRa>=XS*Ew)N|KLEh%bI06#c=wTCMhpen!(h68uv-j7&5WCEtV|I zyNWY(mBTQL{SbQo7VK0iaXzXQ;AoF}ObFa&lBg>5bl0TDZDW->aHTz25Ap?d-jMAw ziC)4jm@zT#)OLy^kt9 zZ-hV~P4Y6<`&6S&JN^7K@TCL3-l)rewbM*78)Y*II?6A9Q{9!Rkl{cu#);~ZzLS&F z^q{6x;^OdvdWN71NMehCK~1L(3C5@~=h?jG_cfTh71u34J*%!pv~FFd^aGNZpv)So z^lkbkaaqd@W`M|$)+O%^gmIMpB)e=P4J(zQQ3v2~k!!LnM~ZdV%+W^KDzlO#u@uyS zrG>Ar=jxw=xl9k|6Asz#pYbk*mxu;ez82RO52uS}`BO?fI=7By`eY&&{d~8b0PTJ= zR(^n{a-3Y0T`KC^F-94}yj$(}d!7xp;QCgC>Ez{?3vGj!RSazIFTV;;vFhzeDvr%? z+)aMmgrKj;{m9zS4>Q&5qAFmSl{SYa`l^=q=L{am%Mvx6-Z6y89&#ouTl8FcZf zPvyth`ESiVRDssR3y!S&_rKe};sd~^PQBfqwg1ASQ=kt$H{)bWw@}~AECC1~Oou|x zD9hqcQF$Pmn7T{35+oHm3B)lf-=A2EW3Hl$_K5vs2vK3aa37LvkT~kF<3xg=XzaoJ z{1KF%9ib{JPp7BHD3zqnlz0n7@h;^vBliwYd{;6HtG>r_k9WQL^r1c9i(-RI&4*`U z-^Dpq>eS`Q9x}n7`7b>FjvKZdjpR0w$A9<EQk|ic!><3hDV#4lI8?&u;6g&7D5kmi|V30NuETS%x8dKUA&%s{b zW9G!Xj=h}0zsG$o6s_?}lAJ`=?k4G@;kCJ|{&RK)XDtvn=KPnJ{Xclh`^nSzm+Wl2 zGc6M0u%%4QkFv(#`2w1ZjSn=Bl4`Ep z!s%Y^FS2EzCqF56R}vN88PwtRyT|4x8E}-`VPODOn)jl1a%y2P;!c86MAHs?yTu=x z2ax2(L~z$`ht}%qN?Ku=`624hhz!$ASAl}ag(p6oLG?#9 z<#CwcsvDjpm1VfQXk7$hx0uGkaE*Qlgr0|@H!(Vm32WOx{?omkes$`<{`v<5j&&XA zzP>5rlGPJdfKqcR0CEMvlvVWKfHd<_Yu*XB=+uc~FL;ym;Jq9P9psa0S#m3Dp(Rbz zqy?r<4c27`2q7ED+BI_!Ot9M(ak~+#UAcO?c{qlyCcbIZ0yeEQX+J5LuP5{ghlyBo z2)Xo1U-lNnU+il#YOY@02;3|*H`=wax_~cI`4RohMvbB%ZLzbv^_O3M*{QHK6vzeU zMASOS+c(^iDL6lg1>`!U!K-K2%L~R#i7)}0z=5~N_CIxmfMVPVTDyL|n5Uk#93z%e za*;?wSB^Qzq?mfGAP=Gu)tiQctX!R4u6a=vU{*=cSbXIY@+`$KgCDI|>NrVux)5;n@pXOE}6fg8_8G}XNp`H%w!iwohD%k8V75DEyQ zn``DC(7-i)@7Eh}>8{klXT#Y&eI>LP%uAG?yK~Otn#bL6+i!cD1dt9?@@;pM>I{8m zi)r;f?mm0WQr!2cbdb$W;I=KU(!;$$MO+lR=Fe;;7P4HuNGCk+qB+q$RaXdZl5t`Mk03`0|5l% zxl$=fq7!pc@7KjQ`jOP#O0M4NdP*8GvjqF&$TprAGbX&qcVaT`)+3@{%htr<3jRt> ziqYo_#4@Jk<_`7|Mhj@@)jHP@yam#Mp$|d;1l|le8H9fYMRNLh+Xx|L>NuKsp{x#B zvCxcAI{T!GXCDSD4!AOH2{&?`SdPj5_I;ucOWE1HQI@?lvJ_3ShK2G)Yn%gGeTa7i zQo4q`)gVXbD7WLUpXEs^nP(TtC*0&eFJybK;(gwc1zXh??gZ0r)2c!EY$dSvUwCn3 z$f5SS`@0WE%xdkz6bE3T-GtbQ1SUpvt{z^Lk5p=Xpiw|300F6t1aU-WB=3RBQl=g^ zDe!Y>`+Uda0<~pjc=dZ^#jwuDm*+}3%S2mZC?ae-WQzI|wqpK~1;uI+r)y_<4_9@T zXX8?eGIUZtLFXQZ;d+=(ZtAzx(7$JDaO_C+UCDvbtpr`&+maW5l(tKk$o&_6?YHKu z8Or8)zt1;NW>9+OK^)S2twJ{?iT`|VhpiLhNYi(TDSY35w0(9BSu+{UlAxrKQz5Y( z-9@#vay=dl=J6@Iu(Ku#W2I~SKrKN?D%N$}&i!q2oM2S4iQ!+grB8BG%{K;1_pOek zUZwK)|D58ypRQ>}_5bEETJ5gckVmyWLBfS*ZnGn-vYj_n2X2SQG;*hl7r)|n?pegwZEvB-%N;_sBabAYVcm-(ug`we5!UWcvyl>Vs;xcFKdp7 zR--{|fdmB-8rILM$utUmQ!+*)AH9cv z)y>y0qKMsX%{^s-Dv&u2QW=1s26|r?;Qj(JS>;o|ca?Xyc`x35h|(YL2fI@e5e3kL z^Z7~XS?SVnVH`L9!1))2q0(yJCI-O5AvmX&ry7GWh3cS+tCBvw9$Ci=kPL%!GE(`zM@RiUP)l&`!HbDfMu~4S0 z-6%&~E4U@7@2Vx`Q~5Tvd7ZkB5){H^ikDjU3f#ag9T+X}tvI3aytj#W`sUznXJL z#JnW8@aNI}a^|w|#+e^cb@8uVY7`YCvJqsFbEW>} za8h_=Tz;c?HVAq-*Nj}``3H{Q_>**IJ@ACOK?adMhoF79SCbGhygfY8qlOXA5O?j3;bwTBOS*3WVeh+}RG#SsazK&3P>13Sg z)%Kep7>{LwUEQXOm3<3&Rh?hg7G_R{Z;->R5fy_jv;BWg7EHx)kc&5=11~?>K(?&& z)UUpn*)$tJIG>64l%(p^^pr+6>u$M@CuN!+djm1~<+i>&%6Km2^#{qPcc9hnbxTIq z(d3?KOR^uHddFm;mEaW@;>W+u!v`htE9R;vHoU4zk{nO+nZ#7zD>z8DA9*l&gcZql z7W+B$$8MU)8xJX08daBl)FWJZ(i_h=zsAg-L7m)V3om^aw(AIxe*V`-t>0VRaOZ!( z5dY0LgZ0a;SM~H%63&)=_0J{)RdFyd#}xY@fWnClkhX-TGQk<428M|7frDWRUA3xm2J?LRqU zk|&G;T;G*6p<{J>I^hlu4j8V* zO;f3o$P>tTpPu1Yeh6?)@QG054jiIdSf@_wxlSORVLT%0CQ+O>CwwZ{OU;rln0unq z8bJ$MZuTih_T5kRU3-j@as3$SIXBy7eHdbz1_P4UP>W62W4ZzIAbhu^oH8WzF znAkM+1eR$xrrYc)l%JKf&Xqis|0#BHWM6b^IyXc{0unM!_a@aLL}h}e?9ta%aAElGq+EAYA$xCZov$bWxM8@Xwz(v-7sW$qkP@mC zWLfM~rf4h!V#3HS`OHgeLSSgQ>VhpZFqoNnS>BQ-PFkrmC}^gs_@#Cm<$s1Z1~wJv zdn`2?7W?Jl%5oO@C8u*%wiVnLYhUb4`>_DFS)L;Nu5V4Z{=O9#X9kL(o_ssIn~fRP z7`fR*%wv^v*47#f{qciIkHMwu%8ZDzT>n4n)LxzW7T1+sr2HebOv2M}6~pojNw;Dv zb@pxeHa&c}&pr0e8WQ(S*z9QdSc;|nX1TY6=uCdkA2w?eF`T>M_gQ#bQmDG(Ut=M2 zzc(aVXgHIu;aFqVuR*lC5$e?Fpfr?-YSFG7aZo>2A6fDMe&M{p0c^_?kF3d5_>wnII^uIX_6U?h}zYpeHAgk7Ty z0IFJTEoxaFq!5q1bIm@Z{(fZ5Iy=gzn6&U8G6zLIyCo(yd6>U`nY4Nbvr~T!kv@%1 zS`DY&xqlh6ts}u_0p;|Wa)cF;^0Thj66`bFNV@d$a? z>3w_=;@D^ZQrPj@ey$YS_t;K9(W=9KRbolUj9e{YLSe5%y&w%uuq{!D3p;TZ?($|1 z|28FFsgq*n<$xJV6~dSLbwTJ$5*+%VM~rhU^=674^uSe{PWLgg4+HYJLzS)06qQs;&e~;vQl3Y zU4>{Fug7MXW*|YF?F;P1i%_CjpQUypD2H=6fdN8Q*>q67?hzt7yK$TNh&6Ho*NlwD zYPcgvv&4WT=ni;(;j=lXgkWJy^%cYc6E&Dls}ez(+JPnp%4P}J^H0A5kVi?x1xc3R zB%+#{ZKji`(Mf?w&-9o&0TrlM)8wx5pw`rinGYYnhcvL;$AX#Z-kfwrg;EfsBB6w7 zB5U%r9`eq!sG;zBKdc(aPVG+s&tYL$UF$1abkE?0P5hK?7AwMr+?+Itr{z}@?trUw zd{7I?7EI=ulErtZeF)OaP=#=5ntj8*m+Iavl)+7j0`4Psv=ChnY*bUSPw0%f!Q#`} z7u8CMYjwKV8=a9ajqc{IY4?@~w0ynhyPX}h7?d>w;L;F);A~eJyCywVa9xHRJAr5;{kHr23_S$#c`YfmsoPITlAhaeDFCG1;sug}J- zC2wB~L|eTp+9Qf4oRjg+=BY5z-i+sDdNSL0y!zDc#^xG>J!J+DCaDCyF45PKsnCZv z&j`O8Q2M>priO6F;f-%kj#OXJ1pQ59o}yJzbP}24@fXfv{Wq8RP$pGzTep+0KZt!c z>f9F_Q;FZ2a?kLQKPrLZV|ZMiSA(7XW+15d@dgmA16YiSljN)iY5<`aCOWLQ-keyH z8}$REX+?r8!GW5_DzhJXnf$7s^s!3%bfJr+^0u23(0jxlTVX>>kt))^lxwPR%hp05 z@P=`R&yhRg3%=u~)Vyq!9ebnt7f{%DKP(6v_g2lQS2VyIZRh@a<`8#id^)ZaY0c_A zM|c}w?d~$$`)6Pk&gT1sd?21~R=dT?ms;5EoY$8T45r&Lz*G|gW?A1CJ_;#}N401i_2sUNs zFcrJLsIoQOGqm&?9(Rl>-0eNc`N3JY6;}}cR)P6GWrxh})Zcq1-7xdQWXq3>`Nel@ z6YyUz8dzeD0a~1HQTIAFBr@>QI2vP4@;AqQkEJ$PRU&P8UeI*rYvJtNKhaF?TpTG- ztaR^>izh1z`rrd$$*%^rSKB$wOi?>+W>i)4iVv!kNxSlg(rEvm{{EU@p3$!VA?9E0-D=&gI_B>JMEA^n?oM^5OeK}@Npb9zp z`6156d4xG)eCztQn~f7I+ca9RaXi3;jcBg4lTvtShw76Pe8okLG8Z&S8PhN~Zu}!r zQ)c)j_e)1d{)qON==J(t7l>9^Ub0{LHAn4%fG9omn`$w#DdO6T);XuVq=s%jyxoEr zy1euOI%^ViSFHfTYTNBr1QL381gJ#*k@BVYO%ob8bVt58L9^P zGFQvls00%}kf!%`zIVO(cHtM7_eA>*UG)AcdRl~+y#8_edE5_}16~1d1=(RnfGX4l zz~u&+zRtc;SmHIE&xFM*>+Pm~Ke*n_m34A$;}-mdlnHWRRc>Wy${j4a%s}iOR9Lt$ z={!!}^l><#giRWWz0z3aTmF_kBjVLrpIFlU$MyguM|t~))^I_Df&T6yj$@B*mp+OT z>yN*3-QA>_l03rUb-EBqpLPwdR!RmxhU~+c)F^mwuwE_0sq9N9HYNj0&C&eL9nqI< zUS{B1uJNWq{)}wW$_%c_E&T3Po<5xu`RIVwMJZN(H!fdLw`keAbZ4GVCv8lmhw8E} zG)`a>K1-6q8*@UGH~(Nnxo?J*W?Y7wWl-zS@q=M_=Z^60BhSJq5vQovgAP|YeMIfa z^Y3|_ovU6)k(xt|<1r=>(2=Vj4JC9K#u`Az1oUCZI6v+O2>1u0213tRRbfxtuP=%k zuzdQq$rqb5n(sm#6kwCOEbL4*e^sk(m!#*7kK*W2=@Hm7)STu^Ek*Yoa34+k;`JaZ zP8@Jaabr5S&~IYvHrL04qF>@||7qj$6Xxue@(%2=3b5jjo+6Z#B0ahqOj-=URM?@{ zDtU;n-6UGGjkjp`YlQJS;?TCxWGZ!7Hz+B-$cUSjo@%=?(O(_;2|3YlnA9OKC4g)e zzHdx`0nE9h=;w}Ck+jLlclaS8Qe?5Ldm^L$ytB-5uMy2)3y>D}Tk`R)IY-yMSCM?F zk-~6iP&4EEj*t6)ZMzdGe7zZ~a06QLCZ>K>Qh0d78}`DTp@sN3LiANkT)y2|UFM)F zB7QRy9&dTOv2Gh9$xokYYPiMy()e*FrMN#1AdXhLwt3z6)A3&@?E zIpT8k!ClU#lN?&!{~vT5?>NX91#}#AW^bkjP~v`Zc5XH>_zHw$TKD$^6Nf-pR>WD* zP<10hGk@g;1qFrU2AelSV8pIS1nMyRQP;`cil;0(BTwDMj$opP!88_?)66Ls|6Boq zYrfVe5Xt@0(VeG`>%=uvGwiKRC$g-W8n*!miLKw+`nl#I$J@`?tw{AbK7TSpv;+ zy6}ujGpfGPZ9hp*NlDR~2fY#0&WO{|Brwgj!`>a!`EhbL9PF)(0jVa-yAxGvzYRzp z8rCZ|M&sU^bxh<5LpY+A*bYlrD-|HRc()ddWu3A?zBPsKs`M(*_qFcYA-a-KFn@UR0pQyB$VK=+6| zSoKqZyYw6lDb+ahJHs^?!;}<`B*%*aA&u^u7nVS8nF5)xs=a}Pz;dPjexHoG?)5gG z4??N7g5%J7yTx~fhUqkiy*NQk5a8}2hF#0?y*)61736we^%c-Ut!GPsi)>JQ%$Nt?jY#%y{f2wb>jp8FUPIW_%{^OGxE^7R zFyA0YT2ue%%d=z#yX@+y@MmG^^!+u=GHlJ5&|nsB*M*wXRAU zvgh`4ZEaQ88@e~8!1P3sV?31vX$-^fk-&^^oIY|{>Y2h$XbrXU*k8o@F6zq5ty#1C zU0LVLyh^Mm&po^oQ90q_46zL#G&)5TH`dB)?TefN@@@{6% zobF(ziPpKA%Z{J@Ra?Q{8}%&&Xa?7Tmca*u$sf0Hnb)mdJcznKvJ%RP39Qca6f#Y^ z?EC97A)b2Wu8bKtVXQR|d^@njbXWT=iB^%*i#*&7l82~X29w5BW{>9oL>*krJ;5Gb zwiv2F?+*Z6{FP8(f$ctX2h8Ek$YJ}a&chI6G1*%?Ojbr+&%QBE_JM8xu5jz*Dt3^C zz5ab(#AU1TPXKnd9obg3*mT-r#J`o}SVQnUl79_=j1695#SgeX&?iTg|59IRM-*es zU2+zz>cmunJI#5aDc&~ZN@N^^5~aEf*^9}#TJs~c)>VmWTOeWWqs*^>j%GBziQoSw zDCjfi2KrQoRK*688lYNQG2f5RUDLLQieC9#i9!`!%-x3zHFh=rd&8`qI1zggt?y{# z!iux4TJmk$#__)^$$r17Az>7LJABEcZxxHv>vc{wb$TITG-7Fe#l!=?bnV@lr__mU zuW7N~?r3;ACHCmB!lN5jr`mWoi$WgiPY7ACLY3w9bx0^%d5_Hqe^2u-&v+3kbECgs z#Q)@3tzYTv&Vy#PIo!3nZfni(O3+Ca;HH( z!P&lYg(nKcKBtG(BiKYSHA^!}GjubU8Spxg=nC3nw7^9EoP(@yEq3Gz6|E)3Oi;{x z7>JyPEd2N_d@_0FGCsvt&1O%AVujn5>Fb>oi>>RHQ|3$ZVtkm~s)*K4md*I9fP5R} zgZiBH5U}E7&-Ml3+Bv|jUZuW~WfvhC-^cxyu=EwAk(ke}$|E#r)Ldz| z%{-g;5==T!vF8v|Ge0;J6lj6!IN5hlGxiLH-0GzO+-V^%xy$UdFg|W> zya|(~Uxai+J??Ot$sxG9DUJ}iHlFa)lNu2khE*279q)=Tnr+rZ8J^%gjp~)Bqf|81 z4Fo+1$Ih`uZYsKv-{(V|&p?3H%=HC=zXit9RtG>*5EN~6T1(z?6Yz) zLKaZ~CSvHyHQu!=P3YcKARJf_#{Ahg?ZC_8P&&rILOLkHddPN9WE{vjdUM6at5>fk z@x*yW3gRM_nBLW4+X^$~mDu@VnR^LSY!3u^H*-@;)+pmrIhR1&;V61GhS^K4Zkm2v zm{*Q&1mLxH*!xRKT46nhI-syO8=asp2TgfihD1G%J5=lEsnQd956!tr#mh(BvceYV z*W#0SCqbCGIPF!he4$m0+ZR$%e-77prR9t$=EpbkHjD#q?ynvAwSzkzX@@OMCKv3t^%1{0MP5F<3Ktf3 zT<^?-*mKOACCVpfEArl4+1tw9^g19*-gxD{tYx@*7M>Qx9d>dQowZ;8dp)%$n3nR) z{u6fJIe6P92{MZ;mvp74b;)0n(^I|KEIm3m``r5IW*c*c*AcS!5B5$Kcf(AEEnP-a zSmrh4GtxYXwhims6N*oI$OS9HTkns_hbMNP&+2__I40`o`RcSA=AkJ^^MJ; znlQBG3Hb*_`5hGUa*_sBBI>P&A{T7;UK>sd_uc=tV)%yGkkc)1EjJc~rvlN)=0rZ4 zV^e_5HD8m+!o!XJkKJ;aWd(JZIt_4tw>U>u$@S|kHknD=EJD8e0Q&#*AP^uEss(O` zF7&GN@RIShZ2p5kpHGa2qHG|4Wk2C|gpK$@;IPjCaq|Qkx7+Co{AZC@)7Fmc3kz0S zGm)DN8#i+67GC2H|^2L!1gPn*ppguawG`^m{Am z?bt`MLlP!zwihlJdTY-E8@JU^3$RoL8}$Uw)yYZx=2_$|9M^gymGLUkjyWMVM(SJ zFgu-R5-M6HZv;Sf+spqPqZ>=nX?GP?McLaEo$z0OObBe%k7dcv>8W@Ae*|Dkqb1tV zf0NJp&v%O{C{J$#Ja4J*dYnT8NDepym@N@VaXpXNo0-Ob47Pmw6uY=<(5qrNRt{oJ zPSsCnWK)}g(|<&F{rVLLH3}{uAvil1XhXLFU3g@`k2oU06&B>+Tfiz1095Eyp+aiBXzq0j*uTb|)PGrN$>r6N2|JO^~zAH)QDq#m-TzMz0i&>2l8>kYM{UPg$U3 zW~aP@M2qQi^&u}y@PRsO#XNxV!;=vo|}+3Sy^i64{CC zzLw#2GB3ZpCTkS>6xAuq7{0*wf1h*yq`YKZkwdUkIehpe<)trOw-mc)k76@3ME=YD zk`P)D=0R+QVYPd!ubb0P`g@C!#@T!ky)JLo(T`tg-z7(388;fZpX@(>_uzi!3#kr| zIdh%;)4kvhGVk(D@3PEvL5%$6Xdn4zfqJ6L;n1t1!f{W`nH2U5XSK^`{}}UXV7s^2@GXlryHhA+Eww9*jda=u zQ|)$*z?wHqTIO`Tj=Zl&X2$kXQ8v;}=03&)URT63YkyRC-*3mL1JB4)#ofNMbnbE>RXI z%ffx^H#TtJVx8^8HrKs>vXiP?)!Gn9s*uhp)X+YIxd>8iXU_$2LU8kRLrbhf_ zj}BZdrD`pKQsJD%Ab~_nfW`mUp;|in8{gcsBq?AAnR+DL)z|24@1FLVGjVf04f6&U zZ_kfklp3>yoUVy*wI%$>=B&`QgKn+H<@#AK2)nwOX53bZc{|_0cAR-Eu5eP!2u&G0 zYYS`jFIFb->)-ON_t*&k{%T*IjUM1Mp8d}e`QLo|-;*?1qZH7HY<UV? z)^sztumj@i?%i1PBduU7(txBc4VvZ$jcaRi{rdI0=)+u5p>0UL&jEZ@7A}J+K#$Hu zr3DZjHVfIBG9pcu>JdPaEcWJTFGGdHW!wXXR+D2xnF%)-aa+9D}f_pZaCUYnr~_v2s43$h796Y z7XhRQq1AFGKL=k=`6vMvtQVm#t@Ha%ijEhJE}hFfNwCwwV?nO)4vGY12f9RC;=tZ|aP{5ro^^3tJyYXM+Df7^#rxuMF?l_+`(o9*YO zOryFE6S}xxv)!w_Ro&e)71hnF6HSPu!vX5nk9tHv*L>>=3>IOYd`*^Vw>hF;?GPHYu>S3Pc^I-^iS6kHL@SD8-G0*Nwct;B9%Od+$+wN$9C+8Mz! zcBE&9<^B>892>2tK9~Pi*>%YNR*ii%?X}dNdz+OWF*n6>u#tXnSihD{yI4v)S=I2( z4?W&jD1}u4Zdx#UW9#jnZ;#%)Nw`fbR9m_g*|euRc1$o5CoZWYy26{!UMpEIe6sFi z)RPQDN~vE(Trl`_q=hv(aMf;3+F5l3rU7k2H7hZ>q*dJiCobMRk*BM?6zw`74K*_2O&~aF4j{{|AoCFmy1b#__ut91438YxG)s@j zJ<3|RkmB`%JUn38ab^I!B=z)8Qg!MTAO$M55aa+!6bH$oS#QeLM2*4vb5XppCCtH&ac4|tw#P{XaF41zvG)_i=xx5N^ zt2%Xojs1-DysULJ_TU3=buW^vod`$u-nBHuktJ$2+#Q^m?5=?Xf2Qdq-+P zgLeYEZtGl(jOL&R=NvUlu-$X|Q2yB?WaJ{+7KYmwulq!Tex`r%n2qg7qYpiSxV?|I z%R%8{HQfm*sI*sddwx9b6Vg}qtlME;bcUaL^l}S)q?;XwjX$7%rBB}Q-afzSYG>M% zt$;Hs73L_UzkO;=AFcJ*rW0zHl&Al`QOhx#=Wa_by^q2D(Ko)S_=5i%c6edyH%#J| zH7v%^tAwAg@a7Sx`Hsbw2N(%?ff4&u{@)*<5_WK?`pFE|-gAg9=C*EH5UK)anfL2L zY?Wy{XG`9%9A*Rvn%Q)ED^|$%{i)4(- z&XwPE)|`rsC+^2oXW-PeCAgo@9ca8#vvDx!U?FYls-n+2pLvUd&wntJh2@to20W_e z>t6n8Mt;&ym!kH^A1_f$B(0q38Z!40D`AAEqL>z-eahAr%Kh2L`eVn$2iwIC6*-Tn zMJ{5$Q~OHdnHMs-;qCcK<(2+exm2d}V-6!S^cscW9wO>3*;_*IJp%KtU8Il3ZNkJ7 zk_OA`43V*49{v_kX(H{sZ1R^Bl8Psx{%UKJr_!QE&le~D>aA;xzQ22dp9-a z$45H~*>075deNfXx3zJ?L_Jo6-uy1%YF-^?%S=d)kGsrn?q>s1?CE%T>Ft70Mx?Ejkp(q9((vkl#7daf-0ViZy65! zQR?bG%vmBJ5wO)y>8Z|n|a_VTazTF5hp5oALpF@J=JKfR0gRC!pTH6~nIIEW4CjOo$=&9H&MZRhAFecXuWimWwi;S=aO zO2Z)bOD3aY{&hOKP_Z|e_==`V#y@few*%4Z@uo5Z|` z7>%bJh~FDYMWL#iR>1A$q(%6!`2eC~mI<=E^oMI{h=lzRz_L#Y{1C`eBM(bd)8h*@ zm%oNMxsmIkW#bkoO|gRf^HIB?ZAXJ4fXM8$O6{uAs42Ex7k}#RFdeEsD}54^GP_4a zzfR3JlvSk{jF)CX z_g=-jEy@hB#23U{;vF9Gbid@dN4nar(@f%{w=EjXd9mKCh0FE6sgtoDt`*qOw=v~( zIk zG}QpND3AWQuCJthjJ;AdpQl}PJ|Iz6 zS!q$7hY$4^N@KqoO=xyE<$sRD7!k`B8P5XwmPU^9QAAO>#qs!2(_U(C42jZp=~fJy z(rSNXgAf;o7?S(pePpe3OL6C_IgYxjxnQbleaT=aIp^jZX6Wt?4r)P3UWG9>8Z60C zV!N)QsYC;&$&f^+mH&=ADWy-vneEx&th)BEgs(DS3C!9G-_6HySx-ja#WM1CGDFmU zkIq$Hrx>T zFS?iuJK<{o;AY@+JznaNhlbt=v zzlB>z{g1i)-+c4`j_-T-RzWLubkf-%H~TcNit?gMi;SK?B*Xyr>z`nBW#Ct4e_g7(QIOdTo)^ z91C3y0%+gF*o)K4FP35o(V0y67()7ksRS6A5Rg5%;YsYOQ-LtP19~1d`D)5`Qy<7F z)M@+Jem`BR1TdQhiFpI6?nHK`c^w{-rUrL(hFa6uJ|C)SByWwzqU=DqV%=3W$*5b> zT{4CM-ri;-XJbRBg<%u`@Momt-FX>EbD|;~WV(t^ThyC2giJi%M-zEA2ia370!0xU z1$U#Jp4^k)rMuT3C?LdcZWz&xK}_+Ii5;jsVW9le^1JXTrDD0+z}?l({De*(A@v#e zA+r$x)XvrS2X3;vGBJa$?DV&L_8DvQm|J?Kh_LzGX?@7~6wpV_Ta#oF>pc)gJJ^~`)cMj4##%&^ z1#$T`uz}-rzdRB$Lu59gI&4Le7>D)qW3Uz&7;%eO!@KOCmg{|W}VX7bcqnU6AgeE~V|#+&eV=NnEL zSKu-W3VBtc1zEMOO&rc!UQ}|!-)c;JhrZ$R!`c+6)VB%djAC)>$ASRWct?-eDD&(m z-QdwWt36I*y-{dZZzyVwM#**JEcS@;73r3ijSiWCpG$5lsj}A~A&{?<F|yNzp%$O3L}s0yJo5_{8VMR3&58MH^Q?E|@(Nys~h{x9T?E zET<;V;~v|!MYgmQF#MFM?4<8O*?)fYTJcxi4CsM9`d#;o^VvQzHml@2anKt{p9%Xc zla*1jH>)R>vlm;trVAOI$~UqfRR}1>=~38%edx@~H~haU08MjF*L>H< z@XL`2a!D;#Js9jdeB*&XZ#aw+`*vH6fivaxmeMydH-hIISh3H|bXffD(Qnh^DH9=u zJlaxTNZwMyK{5~xY91|LWAu*<^Iy$Kn1XQT)9{0M?VRk<>M6>u%U|B%fBJ938qYlN zxyApKtzm=J8AH=uyA}d92o8>CqdhNQUkP*`vIE|z@(43iSJ2WW4`BkaM#On`)wHH- zE6h4C)$~_xDDd7L_9S--hc#9=P3xz8`pRj&>SAe(RL~()1cB;mcEpy%g@uKy_7pr& zvZl$MfL6HAR(J|R0Gr#2gSNJ7HDN?LZ5n+-RDH+YYC^HbVE5tfwh&G4>MmM6C}Np^ z;BUL*JQF}d+q9wjA|OG*xHn*1_C>+M6e0hh638JRa5(^Szv(Y4ixijNd#EY z6gtY+&~+vc8QL!zQdTSu0XRJ))rUaqPKcKhI85JoDKdRPGwO-6w0d@`WOPMi%)0koMhSO{UwvA2w7JbVN}p8OKpn z1gsz>$rcMLB4R@Y60m?sl@4i)ir6!XfYLIGh)4;&CqWSr5h5Ujnt%ZVgb+dqA^pB# z_CDwAd;dE7{^O%Q64CE_*IMuTHS1lwo_0hZT3vSdD==9l1@C(giM1&_`225JtJl5wVx8bH%f6J}(eP7ABw$t{ZG%=xS_ z#97SYrqZ+?%j?G1F$r26+yq)fEly9HX|Nu|!`3AyZs)J)*+0K93UP9|q9AzdvsJu% zCrM_hNF7kHg%_aI;3+`Hl^kujGy8Gs9Rok{mayhWzyGHj$-&zr7G1(b?|R5S(}Dcl z>vElZmA4~lZnSQaWm5!q&a;yCQgd(Yf5iVJrJi~$3LO`3OMO3hk{IE-A@$Rh5t2a# ziwY!5olP41gkN_U2FCI5(qfWE(5RYCz$4?A)Q;`t+EHAW59%!cD46ZaKZ(h-JmIRP zNWd`Slr3>cxhJGvPePOB98hszm@Ck=GlFm)NlX{j99CdT(f2>CZuT=@n{}_Qqx2Y- zoY(I7C7vNav{tVYP@|sScqS;Et=W-yxx@~6c}u>E08?y%=(eNTFRSggc=rhQB}8vQ zyBev7V$A37=vJ9qvJ29COJ{l0PxE;LDQKA!$eC9g2{LGy{Ufkt#NQS3ME28tU>pK6 zesy-SMx@{YI(6FkyV+-!dFzy1BA%F1rvCSq7>@AIyUpn<;p?-5NH>>tG&6kvzG)q@C5+u~ zn#p1C5Jj_HMwx9gqaAo}*Y=3NzA#5+S9O$BH~whc@S{%$-O`o$fBe*p<~KI<78H6D z8ge<|y#&GrstShA**Z$8KR;g5I|vl@%!uX_M`(XU%hTQ+KF&A=N;Yis!-lC`v28wE ztQvpPtE~QQ1I%4iX987BeetzJ=o2NSx+I4N>0z-7MzgiG@8th}Msr)E^i+2IZ4v=F z!TC%qkUvH~ik^t8!_?=^hL|^b2Y&p|(OCY^1B4AF;8WJrq;ZhE3|7r!(>B-^C zyM8u-rK8lz{M`SIs$d^Y_QT{A(A zOl_1~)djlG!l840x>OqP6Falld&}@FPF7KUNvh3 zuWfugaQCmNZNI+#OXJ{zgO{%C`2$+gnATFm(kbn{ced}q9sMO6Xq+Z0hSYPm6wuNx zmR_$&*n(g%F13SkAW0`Fr-T-6MNxxHYL3+J_C`+I7(Ujc{83@ndR2>B5x!{AqE{Rr z8wcKMm11tUuC7@c&((r#bq4~5cq7$Zk+l7X?2sE#9Vl~~x1pmQ#QAuf0icqYfhUpXz$^n`GW4V`O3Fk=RvFOzr|JFmcW&lUZYOh&>}0ZHLNpemgN zVuTZ1dkBcrJ$Xr4N%q%uD%@ni>yl0!>lK0Mzz1I>zG&HcNKIkXcy8f;@ppPa)@u=G zc+7>6{v{!cFA+|+-?#>{r${taXgg9#Nv8w*AMsI9ETI92RW|!+GWn5FS<>1%Ss1t$ zILoSlR_GNAPC^C7o7yL71_m0TrD^Hu1Y&vn8CtHX5gkzZAT>Ol5$TS{%X(RVt5FK* z2-)#1-bU!_5n0N|XUH?}RG|icmT7hC=&vEGO42FJHhkUiYZsciMYF(#d?TKf#R$dn zrxJUtvJE-WIDiYxQ`<;Anr7DPp$$eW&k!@iUo~(Pjlh6VyLQgcvzn!t*uqLDeVEffz4q*9d&In2oK7FCzTMDr)wEmnN@7OS$MEAFKXBbzs0Foy zf$(K9Nz@$=^&J-KGfHpuKVKcvI9osdu51)3Gvs>NyvX#XaU5Y+5oUW7FUB|}q(tT~ zc8d#lwi_CERqXK=BTzwYA_K(Zdd2$8eax@pi@07}lioQ;^1YinVM!r_5-ti4L7Ee= zPn#~YH^3~-7USP)nMtE39r}YqK4(!}@)Da#XNp7pP&pkBr}|rB@xbzbIs2IdRpJlL zjXdFfHyV~SIA@b=N^jj>kz$l@jIAlbN`fI~^la}Oqc&r5f@Wl3Fs-o!0waLtzz3Ba z2BnvFSiC1+X21D#evAQ6~IHpLF63f{$FWpT$O!1&JzUs4MN)+bitU zpEvizj~Dg#KO_zk<-0H|_iy-eDY+Ak{u3HA%bJs)*A7TQrDhAqqO)~8@X3Hif6}$IOri(x9P>plXX*DWO z1@f8ZRq{8oq57&;zdKVh=|WG#8-=UJvd=thD(xs!ypWu1HKwQ{+jr}Wm8`w}njOBR zXqCqZ;au)A&Jur%O}V!W0%Mig7N;Ia;pn6Zt`R0}_cP%}-{y&9#iN7CA357*qnkn% zhq5knL|s)=r*;PNyn}}$BoX8V1D4^`f7HwL<$dd1RY(6!VHi!XhrHXE=7D3=0p$sHx_TWHAl&Zs0dz8a;)trfEftiQCk;U}JSms%(%_9$vfgIEQu z()-8%`JmD-MoSvxZlxV*vJsAVzOOZScptBP^C|o9C;6ZM>|^xnx4(hcVP)8|`fqzn zE}D9D5Vkw1LMmCng=uy23Of*w#yKj=W@SZ}bonHEdeeRBF$BqlwW^0|OqCUDX3B7yuXs@_QL~$NE=~fO&=Ul=qZR+fyDEf*HaXr@75@2w&*=5;tnd9M@bl4 zlnT!eZe_7x{nZBUI8lc>ef!X$8$KKqpCd}S;&8^w*brH-Y>5u(r%3`E)Za`Jp47k> z8<7a*S+Z1)t!51Pz-2MRO-c3w8D?(d%t5NGAaZ2A38)Jezj6tGeo}E zLaj)s?q&E;-4PUoyfUdgi#*%lCtvvK8g|P)sKp^8Q z85FKYx$u;YnPY?KXpgGWT{cIb%ydYa7c`m)eo%)KflHi0Jy(&y-OG`&REWPFuI4uM zKpL-KA9?bwqXt7i0JsW~zQcaZH2P$vEvjAZ;^E8%V zV$a8v@*qPPZO@0uR~`z?COuRaa9xKc?<>UfJX$|sG%6;i`190ZgvK=t3!9JQcf7pV zyvMT9TkgBAE0gD76oNF1FmK@QMj&Sd4q*3qYh}t%#F52?qitz%zmAk|Bu3l6cBv8u zywS{Ze6jQ&xKP~~f3LtLwNYs4F385u@aJmb8?)Z#NM)|SGV%J>+0orYs@34&b|$`} zlg`t|6k#2oHqFZYr~~ug1Qm)^lCZ!99tFCemcSH4#?V^29j*V`o{SsMdbNzxLQm}p zhz1T_Yafx?!fZ#SYnn;_Rn^tCU=~v(n>B^DM|1wz{PN_#xBz9)abC*2+Mo@yG+$nv z#`9BJoRxb%Oc9FS@Q_WbJGZ2v@Drh|_hTQj=;f;K^C`bP})=tMPEJkJ45n@r&Auqy_n}Ieff*X^1{!l z3-_4ft2MkgBFm_=@1=h|iw!pWS*M&njEkDaWctaBidW{$^lko)S!rBK8@v1xnS$8U zFSU&4pIGs=S<})|VKh`3pkQruFwz(pw1;U8yDs6q7bl%~@k@_mDS%%R(nFj_&7An> zWs4byJC(gZk0M~?TY(nbPWrUtJM8_2XLf`bo z^6d`wMWY#i=`3yjR**BS;ma>mm3Yc#{jmdz3etm98EZx_u*f~TaEW}~X4IBLGco^c z(syaXS^6g*iu@1!iR)xBR=Hb2TTl7Xdq(~kJmvI@d(yW2&wl=&c7a^```hM$p@8r} z6*qNh=frOp^Uh6It2iOF!17*5cj>~_uU1@&{|J&Ex*twoGslfZe`ooEYXXQsTq4R* zF&PZa+ODp(kV2tQtK83i*nAv}v_oHp-;>Q;7;YiU)h>9ChuZ6|Ua)xSKEOcH0MVSU}RqUL>?XK#279@ zk?nb1Lbl?TSWk%swoUixLf&guT4D+(LM`Z28(KqdSV6tBvTYK$(S)DTHpdGdWXLn6 z+ZI%#xW&&h+-z+lEK9Gi?!HYXd?iCky0B{>c>p7{lU>+JNFVaRCEuTs_+R`tom zvkKs4qNz5$M-J}m8J<+H$2@?q^xYpfBqF{@3YudoyboBiX{n>A^h}VGt}GC+h@wKZf(DpQ?X#_? zUj>I%3a3sCw+JFB)HbPj+Kmupou37-=_+_VQyf3T$ab4+(~`xD`LL8J5}jBOGrmcAOUpriWFZIMOUcz|6f;P=!BQhV6K$LWE9fe%SY8 zFZtp0$#Pg8<2U%Joibze5=O4k^qHN3k_m0(Wy0zF_-+c$d(96zXWS~N>UC4L)75+00g!8Mc(;nJC;8C zwwaJ@_{4bT07{)cCI|bzyA#0dW!)oUAGNDXL9@FXmUT6H;%&Lg1Omw=!rScZeB`H} zhtd_fX*ZEW!uNCp_~UNg)juZfrJ2>uBRr#aXw4}c?i$^Qlh;T|zM@iBL`Y(E7Z zPP@|bX(!ogV0G=gsAs2J_8G=(BEr{y(!VuWNf%qgX8$pHODZi=i{!@m?P1}g(ZBs` z+MlZqKltyz%p^k7I(MQ*KYcy`>1uvJWH$Sy`J;_zDKP&y3ID#NS*tla{uqrB-5-j) z?qus0?e1PNEg75e$F;9w)AKIiqsPgs^{uFiO%KRa1M-X5us|YWdLN{RQ=|Za3l-VR z7~aHfwJytlw8FV?ZMuu>z|BA$*VV3q`Uvn$`iIRInh=9rHjE#L3~Idvxe4lEaZ|Z* z!PocLRVE6V%BgetWu&6D>6VUx&8cw17)t}`9pergyWHje*#2CyaH%-fZD*UzkVRy3 z8;?89QawmIGXZZHlI;@gVn#qT_9?sZ&#Sy6ut=+{3 z^Mr+Z@9SepxxEm0uy~zXQ`&Ow*1)RK#1YD?ZpYY5^-|i&^t|l|`;YXq-B{te+IL}z z?OB>QswjNWKm%#%$w?!@s_Ng7dN0eoWx<`u+w$`B$QhRibY$^r_d>be(*20GkULWg zU3G4DlC%m7pO!1jyO(hru`pb#boBAM3acN&O?_YKVT=rk)uA=aaTms^07keUW%eD; zWWD{;zaQDhb`2!WR7A?;Xy;C#=DPYTl>NW;`Tud5``;A*^F-aV3&0mB*Qj(jm2TPq z@Kg`+FLq$nG|`h=-cH_L(MAH+CKoeUDa~H5k*DE3Ch6yLR>z7lj9##-8Gd<0 zd;1Jbl73PS7>}370P^CtV~o0f0O3Mv>m+EriklA|f3a)RQwJxc!pw;wL`F=wL7)_- zjNpN7t8%>BdgNuR*cPO*weqv^R_uqnp{_uhVlMa&0_Mw7NIU{QNQNfLyoF$net^fhD^E%p56Fw5jYyhCBFY=(n#R~>L=?6X> zvoZwv87ieYG}i^^PTYDh8}JQ@9bukV_<{t9?JOJwA^NKtS#Dt4|GMb@YUGyLH}@!XvOd;`q*vGOy7 z$cnIfcA8}%YT7nIGbu++u^J<~(~hq(=VUy#-nPw&OgckQgA>pIkk`+L_|c5&-=c(A z3@!?u8Qn|HSK);ztb2}(@oErc50kHwr9>b>=Lyz}O=e_J_G&rI=F2jfM zjo5E!+yO;?#RvMMNmQXhTl1!$7%xQ-1K;dg;bzNh9QKo+3%1$QSzW(Nl+gxKmp{31bX}S+?SCy2gN9rG!CV zNpR6m4*i3m6_L%HehA?!7eZhc^f{|ZZP z5x?OE{M+z&SJb#Mvw&J2k1>d$9($e?FzgTKKj#t5Vij3?Zz8K-#NOiud;U;FIK4nu zGnBW|=`KQyWh&04^Tj^-*aX9Zb}8fceqy{avpha9E(}Jromva&rABzJnD2^bmMr`= z)~$xW-JQi`SJ~ThPue8QzO&DnL&EbwIl;xb)Hd%Ea5}kK5rX3K>=A7V2NR{)yX~ zCWMpwgLIi~5zS@0rsBzjo>+mq;KF~4e6g(3u100Kn^HKob~BP*+f;2x5g$ih+}n|p zcEw%7Ft1n}qh&6)hKLAVX{GX5SD>}Uu7Kfn(-L%(=p;U4Jb<4wi?&(L zi~1{Y$HCskQ_g{1QCb(;rGgtxD%MZb;zH*c*rE0~(e@&HhfX@oUcc+BYYdB_soX+6LxtCr z_wIoinI+gKwV%q?#nCv7I#^Jx22>+y1|Bj>xx`uSBZ)##3>3_E2{7Bzt_2^ipUI(N zYy{lB47d5U9WLnQwQ9}T+;k71hwx`i;7zcswk^{^<9tozP6$vuoh!4XjqMJi1LUa_ z+?|^`kd$dg&kxD;)Un|=Nz`fIcYzp@jt^F;1mR^yeD^G0SmI<}M0TtSA^ zz6&1Qyz)gJ%$lfHq594+3490VO@fQYyn;*h_l^C{QAR$>cKK_C(>A^jZe|-XY*U_2 zLp#ftOo3)W^WkrC>p^3pA}c{^ArrC%mjmK-su-=l#|(hzhx=c;X-NO9>>BY??isVw z6gHg#mqb+I9k|T`K1IEd-v|6^Ag~leY19I!;T5)`rVsFMp5O&RTTy0eVCDA)7gW^M zX~XQA$V!r$4TA;I1{xl~wo%gqJa-^rZx>AAM_$Igl1@fX;WsDPs1V&cPIN(UPfxa; z?F)IX|Lop`obLfgd3hy-qii_v_R3HW1Nq&R*R*JnYK~qHnn=Lwq<)k>3`RJmI`obZ zyCQzwKc3TWSCFOz9W+CNo1YFx>t6r*9?wmBJ>0L?*OH7vW(Pe}XWv+A>5 z`T+HkAaeIa^6sB5J4+MyDl~mnl2dZ#wnHOJcT7b1*b6Gx3$Yv?=j=Iuv#Q+th@(5q zTUXjG7JM}%OiziA?^vPi4Jh+<8ERAZ$viPkchk0AsuZzMiR#fQkJsMkjnB+Sf&{_y zOWBXb2ADmNO8)58yi-lEnVXk_o7Yk%H1#}i#l3@~amV2Q}Xc*5|no#bSeSqZXb`J=ditjtqOF@Cdi_UJ%oL;W-kOY&ze=%VI%opS1zT)dyIm{feIT}0Pj&51gO^vtZQS9Ztc%(`0Z`_;2jtO^tl|>;S&E+N=idre-g+}lJzJZKk6uSNWb9$Wu#rsneKP@Upj z)8hia3sq?eGcq>^m8Xot%70)=-MR)JQmrFgtBb$vBD0D(b~k_ZOh0_z)%%+)4P%bM z$60HCOVyb?x%IQ4i27^N{N{omzQq@h5OQs)N6S^$ZMF|xlLT8L`E_ELp6fZ8Mf$bM zirXuXEp#RQ;0^>!j-V)EtGA>)za2@QZ5ak87h^Vd6?S(kPA!?*Yw?p~(1i(K;E}_Y zb-AZnj@w0=lMnmud27|{eRSgxH%BES?58FAv9e6Zxh33l$Lrr3${5f5{%~wbc7N&5 zmE^)155ijxywON~z6?Q3Np-n|zv-55(CA%W-SlO1(gtklpAV!ut|uG{*G~I4OdpPb z-FSKEbyvN$bGhQk!uIM+>p{N%c+>2c3$y7Le{wfFF7m-*2@)UeLX>yPpc>Q{JC%o+4=ERq_vZ<@Z;p2Tz!<&Z~oxZx;HgoXW2S-R3 zn`h;BLJ7VRlnClI$+8*YRHi3Vr9+i$1QCOeVCSPZxY{dR3&@Xc1kXLhiUn zf8kS-+wZFBx@9){lq)(w#!rs&|HrCJ88DS4jz$q}#4GRT9WYP4g}N5OxYxjLxYJ!2 zTSW0*gU4=O2dR>o1!PgN~T%$ z_`KRVSXFJ93p~!9Bk5$8Ws2jH*27B1>0zlVb<{kTWO%XH1=pF ze~KrFjPAI;_ZV=G^np^N$FiTKzTws1sGJD3cD9+E91f1MlQ{NX?r8zeEf>{sw$Tt1 z80$jn%^rGoD#-+lw!@fCrW%CAvB$X`$Q%=TZNGE52zPenS>EGch9W%;jJTtPHg>RZ`tR|8^EE$CtT9U0}uL`Hi^6tFnl0I5xKbyZ_MFPg|n#Z8rQPP952 zgI32D%Ok6NqA~|N^?YcZ!@pSVX4H+c!K(m1Gs!@AO2eEe4ltDy1mUos@7uG(unOIy4)Q?j~ zbgI*Z_-5V$Mv~Eq#f1|UiXXgbCoTTtandEL654gElJW!-m%}`B*3$5=TBr(r2rYG6 z+u)eDs5$Y6ap5GBg*N%;bi&VULZ95Eezh;<9Z_s>M_%l-kZsm`bn@!I&h_(R!7zx-`;UCJuB`w}#!5NdQqU!7E$nFp+ycg2&? zzcCY|@%*$_cw^fi1ry=xPJ;w+ow{h5Dln?!lD3%sC2-vjSfx&k?9Mpnv7+^&k@s6h zO$m~IZq?(xa@?c#rcNHE5s_6IAaN4*bETm#cbi9fOtq)`NJOGDo+>nTvMDIVby001 z+Y*{&^oi}V&)!UvpWpEL>PFgv7}LYIuJ(`L=H7pgoOkm2*Mwi5FF>%q`z@mUIl*x& zg<0J8{eH_0L_0=+)PvU>p|#s?i;W1jBe(we<1g67yvJI{bP07o^MD%PNg?jFTDhQ7 zAEv{q9wB1N-ttjM>X1oQ8t<}73g;iwkVlOWXfpM21|4pdv95fR+>%FiiJq) zGIa9{rNebF{pJjL$z1)*dOR&JMsZ$eat>zlGdbQ?0Xm#&VYh$}&ZAzzSf2$^TuH~7 zK{mXS)4DWOdJhXc){k!vMuVDBt{w>|KMK!NSjXu^UG-S;``Dv@yUzZH<@&DZi1VR; zu_W{@dExpoYWXx^RC%Z)59BlDw1vMWHEDbUwvwNVwnrl_*rxBHs@GnTPRLAvG0>by zz>E|ezd5Y@`L3%=Az&~ngVYZ9>IhKh$}>3~Bh?BC@LycJR&Vbf=JQJFSdPZEJ?ouu z)dt{Np&0{kXvq`oPoYHgl-iV?#ZtrEb2O#7L^(CE0QgA=tysJ7C|gvLxP&6(Gbh?- zhNdQ58k)Q#BiF{Lxv`Kf_X2(1+>J&yImt9rNYv;%HHn`@lV4TmFdIMtRLRdREF~O} zgZfcOWgmB|B0z^fXbZsaI5F!q?pb0^L&_VDiNUn4)jQ@k7`6BzyhW6A@Q6fZx*znWwm+b%=Ptr7Y0$8p2Zn{ z@ufUY%m?`#IdLIY)}ycEp)OG2HDSe{1LRpo>qaqrQVnfJjF5HH=M~gx+dy;FglID!5Dy&VFWDKj zG!whW48y5@ynMzPz4vc?8u^!Q6MNcl5X21maqm51`*X#3_>=C4*i!{rDT&&W z)TLylU#yWrPKkdRM`#)St(7vmb%N@NRFz% zN%6*K%q>_0-GX;QL=e|{m*Da1P}4Z+d*EJc>-F{RP2_F2XJzqH&wexz9WKb7a;jFz zfag%4u9?dczFwiE#BMTzZm=|ZKwkU-+gbCX*sj{V2OBFvb~0@LR2Rc^>$WJbauv>~ z+hs|Opq4SO02{dwB3TKM(3a`d88>3U*decWg=Xm?!2ATjO0Z8m(<0%Ajkr<_zy@NS@L$pa^{-AAdDicSf_tgl6^GmC&Wy z>E|T~zr26qdSwCP-T57uio&!^OS4TomL!8&cM&OK>W@Q-68!Kx_c}5_Xr}9Kq7{o` zDB`Fk(1GC{gpMLf<}0#Gc5hWxcn#^sEdmWkCbQR-2)fE?H!f^R+CEf#Sr&z3IXC3T zK5Yq9U*%q&SrXmBI1r7De-wxRAxl!NjoE!db!8T=*J8O2mUl>6?lvP_JKh)ThHd#% z)?URq^gQZ}Sx=?yi!}4WbE%5fjQyYT{>23-w6sw=Bo^b_JJHdqYYOjHL;P)-U26;c z`U0EbL*}@ZP{;Mp5&rPB`RZZ~3h(-EmRI>3nNhp&O`CA9f->D_i*6XFWpN@vY5;A) z01NQTH6no{RmL=ykU_d!0i3rWH7Rm_PN${1tzSBe99Qs6%lm#2H9elu0>2e${c8B@ zKi`nQ|Cax-g8yH0wVRpdoGm*Dr&m)tH0R(A1Wf5N2XDBU*^9aN8&ZBqjtIN8=tM%? zZct6&$e}+E{Jr#7^P8?*So9Ax8V2Q^Gx-w7a>?mfA~d!*0nDftK|l)eyJpMW7=~AS zS*lkwK)-Ne8!o9A@PkUTQUa;#Vu<+Zj3@HyzNm^zJ1e|E9tB6E0HAX7clh`0f$peD zqFKQ_UN6m8-BsqmT9C+lcr*%=tnL^Y85wg9K~O4grrSAMSzTTX+$ZemaG282HK1}6 zL;~Ty&@70B*Qvdl7lVVUegqC26q$#j%EWd zLmi1bmoa}sR`eo?>g zU`qX}Y8Ta25jrM>6Imfhr#0hGa}DLD?Z@7erZNdS%AY#;sbvr~qT}qK;!?rsKGqLq z8aH3I2||M;S7;xQpdY}0CB{o5u<{=OzIJUSZ2Wk~Smau*>R03$rv}Y~bdd6(J&5(A z=P~>Rgy8p|9K7+?Bk>&mWbOcY2mE&$xB@RO6tIOxxW=l2;MUIeLGHp2!i~&nkbXzy zoq)_JGY#*}3U1t@BdZOk_$1kOh#cLK19gEM?^|h}c-Ma&U*AtLME0&;Cr|dfqDPFk zz0zmz9&x71P}UK1Y6OohEz`6F*Oy3~#NDOMxu5A~|bk(kh}fWD4n zv>~zHhpIm>Z1jY6ogdr}-D&bfa=26}1QCo1!_062-d)s;&ZWDLj~9~Te0rpM!ri+) zT&n*Z$MBfOMAC|Yu<3^p%h?3GSwG%Xan+<1t08G9D;^$&^)i1tyg7-4Vs&#P90I~* zgN)={&qbqYn~^#fto&BFeo6D^MH6GIZQS%d-9Hu|{y;75eGsvrlBY8x^f1lET->$d z=h3qD+QC(?P9^?wUgxz8{j z5Ub0&BMYG0fBt-b^?nff8Rv56hQaE8kYTT3;zGLA6nK~H(q)_Q|Ub@ zt#%F67&rji2Ok-%Nz3`ROmU>N2PQHJ>(yfw;j{dd79r(3LmmVfy`P~oHDW45IE!p9 zJt$y(rx%{Dns)Z+kZb}rcr7O9i>@qz2)G??z0Kc9nq}~1bVN( zHx%B#*Uh(kDgTe()sQfF@wXcH(}reo(Wn?vVrBmcIbVA zgsB5S1V8HMa?qTCYcvo}Yp5f%ptUd<@k{op?1KVK4XVYcML0~SxB>Q$C3q%bF6!+K zq>e$-h@5V<&j@V~w+S0Nqt-t_U4F&Q&eGP8O+?97H#VzA?GM(O1x4?vX^d45Q6Q^5oJvuMSd=eWs7pM?8pJ8ZV!gO zbNOJvy*JcI!c>RA$3;JbEdVp+ElN>Ia{-tPHnMUyBC_T&3xgMTUDko{Q}LvOba!fv zTTP8^f8*0Hx0)U`Wpdh^e8AR!p?V#J{jQy8j@GVyy2;_yRG3wo8pQ@OV94WvY1!Qg zcOyAQA%j!YmeJed&g8d+4~nymL}GW?tndt;HF*bIf`RJTC`KI^Da{8-Pltwgsj>Vw z$6w5=1x~;jGZT_&zJutUD!t5AWklwe*9@CNa?qqqpg9c1A<68~*an`ln~^)-9ad?8 zfxnjau!Pgby?ijzT)CkbQoIyvA||W>{I9W3B^Mfhmt`B-01paC4eRKldsrtr1Aeiq zb+qi)Ev1Lrp5Ag^=672*A1zrAF=c`LWxN8f@K;2=ugV$c1HQI3E_-as{+vxY_W3b@ z4K{r28N{!|8Vm||Tb!5_9B5V8Owhw|d(&TH+wDS4O2CX+Ry=-oC za~6P=pSs;>gWHft0-;&qFhNAKgkE-c8`-?V9Q$P?N@ zSEGWrn%D%{1`fcW3R

ju0~?FcfCKIr77FM7&$1AC`+*|ovs#a;Z}J~3{t)!)nzJ$ zq*@x&WKJ`PdE#lK`Vd}lmPbiawJU*P`e3Stw9n`n%xP}t;kVDm7T(aNI6!H+5rTY7 z<11IP#G!M8c*{be&_!-*N}wx_D=}5HMmkRlxOq!z!B#T+*ecM28lTR@GYQNvhPyMZ zhg;*Q?gu`tr0m82;bi`AUk1_w|9iJWU$!k_DZ~O%*lM%ag;1=WMhqz0(nigDXpg>&xq6&muQkYBHkBF?CdBFaga~zrXi(4!AphD4 z9n1U$h!*TW2$$6ke;LjICGzR~qz3}Q4d{Noi2chbpHFO~On%VP05Ei`(5rY|EE3qn zsSo_j?Wr*qTQZFaG{!sjma;I`fUToBFcj~!XaJ1A9?x37B$#AOT~w!gxcZl#{f@9@ zmMElrWglC_2mH@gvCi`48U^hqY&0f;hoH?;PGq!F;bY?{92AX>M%##kCs~wK#n_-N zoS`);wo;;!l~<06GC0PjfDbf@N~cVd(QxcVsuGSuf-) zganJ+rF?V^DM-PWihz7tbD*p@V7@x3CB1+aq5#vZi-y`18SaCe2nW~0{>o`rR(ggQ%KburX$5z?RH7_m9hhuaW$VNj+Y~)qVLPvdx0xXP@T^%;G{|HqiEIB?2)b z_31YsQg-IEI^>)05w8ELm#+$$>4prZIsx~TTJK;NELnlhGm;~k!;h%u;&jxLalse# zkkS*@P~kqze(~M>_uG+UM>v^%utDVj5jTF>*+vxJo947`Ilen-yha}~Hq`4ObS@Tz zVo}ZV;r^&+ZWZ(ms=4X8a4_7iY5b;!tMNI3#9bf#%+H{$B&u)g{0)6pD#1cX8{fVG znohXozJvLgp{GXBA*`ecy=dTBJko-NSCgNCx4qZ1WDC}w6`ggWtDcCv+jAM6yVimed#OERm(jAA zmp_g}UKY1JcT&uAGZcONulOydyKikA=f7IjNjllzGybH3Cp}+rVsD+-S7y`l=Py@{ zuU^_{y|#hT5L$GPB(CBgB8#i&A--$SWLB}CX}bf=iHhW*#U6gTQbZY&~CZ}~2g#S zD`9gG@7a{o1HSo+PcAIV_|%TaCW~;4#)wz^=G=a1&4uifi7Nmn#rj*IjJCe?FBiVN zpZAhP^S<$omiLYD%;6=;T=!{+-Y(bjG%cy*GD-3~WKIx)Ot0W}X~v+3Uf`Ox9Lqb+ z$4h55Zy5!a+`N_Fx|p->*|9P@ZN-#jMbzg82eJAxjV{uJz+j%6b-v)&aVN^4ixyNd z)5C=#`}I+)q?GF;NLQxL7lQFz?Mt_76Bc68r^v#BHerRF0&e+gk9)js%%KNs7FPGk z{~T4;PuG-%A}=SWtCYaBc1=tYW&El$Ry{wwxhu|~(i`aP<8)+54eM+AyC^$#Np6(x zT>Xc-x&K>z{SSUwsNC}IU$y-b`_09?zl|C;CArO;TcH3;hCiEzH)F}I*N$()0rBYQ zK{4JRNV`f;ZaMy(Hs_WTf5LjnTzLWGw=tHCBG3QEU|4!q{4JJZ(=b&MEXuBAC3|Wh z*v%|yV^mOOs6;3YmwYYqJ2gst_z>Em#tH#OwAs%^ZG{h3s>{^D1_vcm*w+#VA)o%B z{>N?JR~oQg0{IzDOMr>3Ob0RhE2N%UdmMW3a2%1<8_%$p!i|_sn;y~YGDU-frxnBe zC`qJ^SPDjEBGXC89UcNgs}~&ej#WOk@tYKn&k$zX%6%k}%C@OcTO~@`Ldu#->g0mW z{26V$*^@3hOYk8K2{6i@eN0^PA5t5Hq_N^Ns8evzF>p3MxYxX2{gcx9D4rW2>@ug@T=PlpZviQ$yN9#_&TCfJeMk##lO4d6 zg`>7E#ALZ!(vA|=(!#%<*!X}qd~;NTSf2LKX6}9Je3l!BrX~nkx7^$P77-j8@#ftX z8SUCLN3z!CA1Zdav-CYb{ENVN-eilIYT0GF(1CoiNN}Qn0F364vD07a@hz}QS_46R z!+9)F&(1dNwQv>F%n2d60s;GwHIFX$p$C=Br~|+kLbjWvC3tj4;8b6CK?Z|G5W*&n zEt3)EnbvcotHl>D%V${JvHpg27Dj@o8^Q-wxIy+Ggy%FkMh*p%xd+SN-?PH>x7Os#)>xTtiE|@N1HbH zAVR<$-r(UW4B~ZFiEri=^S1Am)h)Z<_Asbaa&>A={@&~1M|KuPjC_=_Mt+y|U`eiI zMF0hlZdcsszTW2QF7;g>NM{w7%1#vRZBivC(xZFqD_X9U_rhc!rv}1rRI?`Hw#zqe zCh#iujoHL;Yah+njyt7yS}%-QEqQPtG;M{`CKnerT4z>T_{SfJRarX_QKL;W&_EJH zW&Nx=FJ6e-{UVQUp76oC+`(L-4X8-uz+F2FM5@dyX{mWEbGYN0eBB2dRD_zn<9gC| zUW8_p>&h12=N6kjyFREI?VV(z%=@)s^a%?6B~Mn^<<93-rOp4gL;)72^3zejd59M^V+ul!(N=CjGPlt{j(gu9bs9^aFJCmiM`2B zBHJrG#0HOp7h5uya`5A2G9lC@Sx8?v(8VSO*L98i^s-o;Pb%gf-j8f!xY!J?^NwK^ zMa`czK}(iGdhVgl)6@nIA2a*%<$#kw=x)H`baDIa%IPqWV7x@Q6}-cK$6j;ZF@x%s z*S=GH9+f1F?(O|1j02y!fO_%be^4g>-#W{UeRcmI|NJD20QEQuq#e8e-V0Q^O&lcy zS9wp5^KP^Qb=#X><(%H43ypBO^XGYtG+k^+xHnu2pMhT*wKav@t zz%epV#Pr7??{lLgy*CRPA!jy4V6nFE^oWeQOvx1fBipGr7PuFLRtm8BWT^w&k!$Sl zZ6@}io{kbm!c{)i$A%dMaj=l@1O`y#>u9jW6+|K6EJ~XoCyUK%Z5JrJxZP}rrk(zQ z=~PCI zo0-Su`~A!LeW=2jF`kW~`P{?KHMJ76pW|A?Z8CJzU9<$VAL@p2eV+KtuN=ac_pD-B zOZ6QYH3-SvsR}cmy)6A2P&L!zdjRnZ(s!&}Ab5<9U7@cnwy9A40E;4x0YLcIm^_Fq zmKkFEf;$P=^)Y!~^yR#s9~btL{FI!cW|I!z)<_d>$y_!2c=!&{IK5!vdU_AFc+6=i}yv zq92CPasF+EBXLHPz}KSLUfS>`t|_vJT+Hh1CTbFPr42Z}@!}o}wg`0JGe6@BUsiSu z9o{=yNbKgl48;YAqjY4MjA&*@p3t$Kn}8?Jc7_@X){`wyh+DmU6DN9@PGx39!HGSS@`P{V#V6lBc)K^ESGfz2#hZoj^syJ*bGnc6+511W_TK2DO}MB zmnsz(;a#N()NjP`ssPp8jbB6XI*;#_31bI>fnU8s%;5uoY<jynQMP!p$EYtr9{Z9U(H&^@gzba4AS;rU~ZIA$G zhW2rDiy^=*zEO}7FQl}EP_dM$$;llBW2rUMay=&&%Yn}A1>%V_KH#e@qS`_n$uy<( zEmExbL?wLYTn-I*3U=&SX9;iAYXASYp8aosuM>9Yza~&y_-9U?DwxdRLT7T$f{4x& zp{7Yqpt$h?E$6|9VZdM;qof?2mKp)dvs=~KBS{~vxw2LQ|Nb)N&+kC=^JjO+y|E_-I`a9 z68l7_I|Yif(~od7liYIi@ro2)`94)mNjRe8Z(<{%0q<2NM~ zqo(42)^V*AOcbl#gv51FsmDi0S_92eLG5U)Y^Z)Sl~VQqCF(JO<@_Jo-ZQGHwQCz) zpn{@;ZVMtMc2E#36seJI!3rwUOHi6f2^~U75L8r@Y(+q7P!vQ;r1yj(AT>w}5Fj8z zgb)IRB&45(`+eRq&iQx7^L>9D{xD!xx#yhMyb6`jPdB9+-qi6?O9ZPY>Y%LH1*P%jXTKKCiy6irj?T|NeiFtpXpG7(IoP7 z!N^rwR|)4)fnst%iSv4QS3D$f>#Or$s9VIM2SF#Zog}ANcdCY{v+SPO<%{ZYBsusQ zymam?@J>H601&Qx&u*M(zb{Hehl3`Ncwl{pX@ajMx6-~c4bca_t zw3V@-e=o~*&)0ZhT!v0mjy&T}Z zm0V2UKBIY3zAUWtnba(iz|s3yG8(lb?Oa9@r#b=&9hn_j+!*=e_&ajv>a8cCWUD+E z>Jjfu4GOY;QFtbyt^nc1_EGs8I5<0Lg?LofB4#_#`d0p>l1GtJP0amnV3e!I|4vRVO$-6T24YF*aEAytSOQTV{y z!=JG$F2s{J{_FWnuoj|I4f;g>uP14HiG~M5j|#i^OO}oC@<<~9HFElKnrz9PK?9|A}KkAgV;F)57*b+l0Owh zTD?JPOIhwur%+*?VP%Bv01Q{FG03v|6@jT6QLd2r@?$f%&f1jyT>B936CTyu5zy69 zxX*z~OKO47?X`U^RB^$X9g$^TK>UoX8y7St`g)fyq(2JYY^82>lKkq*z1`Q0=)ZYb z)nqROwZpj-pID{U(Eg$HTe{i|+6=d<)@|pP>`)d0!+d5x1t0V&BeY+VS#uxZEA^ zmWAve%Tp>Mo1yznkD{%wv+Y#y%ZbM1mxF=`@sRIYH`ISJHhP7&Eb{6!H|FpC;AcDW z3zO9Te9-F0>dn%#_h%h?&prth)zK5!!6VZV@ry{E>u`&JvOos|udyp33ROrJW@rrA z70%Dm(PIgoRLnFJODhfu(ykM3|MpXQ39F^3yvtnp^@g!=^xJYc=a;f0JyZ1?FUV6% zpZqCY$s{cPQTxqWpD=fSVeQ|=54lciKc?F`yV|RD=a4hX>1KV;iJgI++54y=#5m-{ zkaeVH5A2{3?2sY;!#^N$%T;kOd7H`ZT?sg|zTT$GuHT-PnqE}B_rq|XpwD*B0FX-EZ1`AC9av5VM#=p$MaA5goK5S&!4WprCHxg zgFa9eAhi}X3MQ@%_1Hdfy9>z_y(nax64A167#GK+gE&~?&z0av3g=*Pa+;;t%>9v7 zTm3!^0s`3TKeXX#>@J621l!K`kJP8(+?{{%8fY(D;$%1Y4%vWKt>ZyN!#u+`IML(f zWM)7VpVthIJ*-w>c7bSPP+}*7>a5^jcw~CZr{29FyPj~ho z*nDWa)aFCR=XV@=c0lUT+@&MWd$*5AHb1w>osgm0ygiGUCG~u;H3Aj9*)#m2ax-8`P*Rm-pa zHGEQ+hWR*@RsVOm3SE+Qr|=%@I+i5;GXVO%xmy!QW4+#SpEWImWFoCtabtKt(4Ja8 zEQLvgBdK-(Xm(oN#&zG_O8!JH8L_1TzyxQg`R>|!k7s@Sivompv$AXPidQ=Gw~W#j zAe01J(*yeohr?=2-GZ+)GK1P6C#_SeZuWZ^!fEof?L&*8gX8bD>2lE4MF;YH3FPw1t;g& zLjC+hqg3Z_$#X5p;urfAzy5s=Bf-UPABw?puj<}_HmzXp2hTE1#DHDx_w(@`21yAm z@>ciDBM)ZIs!j2En<>=DipXy1GAupe6feV{W013Zqa26)^j<0%!+ijw)Az1gYBdC`g8KGILP~` z`*j?NmlP=>P7B4}sHQ%HPO{k<8y%RlFk2QP8TWU!eiwN|Bkwq)`t#`~z+uZHn@L_x zC(kWHakNnqvJI$OIKzO=9vL#_+K0gMQpBmN1z@odZUPAzXwCs3t5u+lb8Ma7^v2>1 zk_lXOiRW>91_621AqqB|>MZ=S+E!cESr<99AE|Vu;##K$`ptO-liiw|3eqF+Hu+~; zg1xBPxw*8l=&%f)`MaXW$h9cq<-rcuu%;CTsB@GM6rS#L=XA^9k^>x4biPYb%PG^l zZ!9g1CCn{~g5j}cwl~;U&iOzeni){N3 zWLtp)>zEo&w>!vYvHz_Oi(lLZ+59Vt03%@NGbtNsUk1xpmKbhfWij?XM?6At7c0#~ zQ4~)#JIwXtNmpXKr1b74BwW9VTjFEfZs^{5LvgJ@Eyr1-?TI^{2!eMMV|pQspm&4& zX_HzJ@()gKVYf?5)V11I8eixp6|&o=YF_)q`M(sr;<27mPWbUl$Q3-dQ5R?>PkSbd z+AKwE2EBz#FbJ-{t>>7=@G2v9o}WqAe<`m2wPUShDlBxXRqmKvmp=ra?>dLkaa*~c zx^fTj!?w{RD${i17sHOdNAzX-hQH*ZXPDn(&=NBXbH4S7If>_?u4B-@ zWXsHSeQ31e)}_MdOJkZ?8HgErjJBF-am5eZ>ZIO4K%5ZcK8Q#kj=`Qv##74KULWhP zPGV866jn9a!U!K=k%6}|GVArLbpc)aHvnE=8znlsW7kKXW49+z3&{6qSR#n%CowV9 zxe}y0CS+DPTY~trFk@z0RR8~-fd4?{|DQj;o%s0SKlAmoTnU>bG2qP+n?-@sDFFCZ zEFiESmP>(aO;F7QjQi+>I_};C9Jtwkzz78;Fb+7%!1C@!Q*Q6xWVOCCU>gX3{J0G? z=#x37K!cD+heQyU>064Anfr+B;PqO-Bo4$6xN!jjCUFB`syBB8Xt0}DNt$8R`lB@1 z@^mzkPKK4ZTwTgoR|7m!0BKB(;eSdcDA^0FCNsm$U^YhoiZQh0D^(TQj1J_qV>oRS z=-2!!0X#}~?Rpe>&fMUvK>b_)>k^dJqQ+t_qH2A|-oT||2|^d4J2rWf@H1NSrm1PF zs1=!fcY$qO>r7PPBR~BB4wnTk&1-tjLRogS290%c7-&rPnyvQ;_PST-VRlmtc*vOD z0PIzF+&_G~{*A+Quq9`Hs#Ex47W>#bNN9#|^!$x}JKGYW)G0_q;$^^ilDFmy2r@Uw z`J)FN0bLy7(NUCIF~HMY-6i4=eg5b3J|EJ#2ifg@{f|cE!xvamwKx|mpy3m~W|aRr zY&+2VGn{j$^g0e)oooP%mLACrGy1p&l`#TjgfdF)>uRU9CF4#-7ndf0<8O+H!b5J^I9;x#mry)KJ=6GJ}_r3H`ywEQFsG zD0Qzoa$@^%eLuG#Q91k~75m4^Ea44Kw_b#iseZy=M{_b#K-PpSYv)h}BKk4wwiYaI z&PiWoALQ7D4aNKRcWvHibz4cqhS*MpqbxEV`EmQ;b3w4Zx7oatVt~Bq*fzjglP%|_ z#7g^1ymT4$lvk))u-8*Wi!VlXPzD{sGNsLRDSc@J>=9zFZTWt4g0utFy5CeEP zADLZe2Xb+^WAT=+W*KWwsex4*V~oTjF{kNf_4Nm%; zFN%8{4Lv%KCElx1#X&ndbUVF}Sj$rWjdCy4UBhzWnKSo=X5o2HJpG_8uZZKD!9}kv z2``F+4cARI#X`!%DBR01_P~`k&bSfYbIzluoNhqp5F~(wDh3OP$_O421D3`pBMgI* zRf~jY%uAd%vyZAeOML#JyHZ|s`6#+55TgSr)seX)UM2Qb@uhD5XQ%awe&n=wOU;l$ z_Z%^xYEB}-OxAJfEC6a+#!Qc2(|^PGgy>_fs9m`%ZF#>k6KdHULdvVnH zkzmA4ztB?tz3um#@#y%Tlgt25Wg2Vsga>Lo82Z~R##p%u?c7PZ8_Np<*Xu(MP$9LJ z!rMe7emWj}m!P~kR{$g$)&B)7^8W?GJ`z7>=QiH&Si!^b(l1$xU@bcPglEAkm!!zO^*R&A>1s zSutkSw6aTGF$HSM(U%)QVaRzouPn|)SQXmUro3~!5`JjQuRf>q0!=6VkLkaIXkncv z*{SJxre%U%r2xEhJTi%iX7k%S-Zw?Y349JdP4JIAaRpDm{-E5p0Jf3q{RJMD^{h$o1L1ftY&`yh{ihnenc7_I zs^dh}&8s0wb^`qJtJq$pecM+<@0PFiS0p+ge)u($CctEPav24sT}HLj>>QKaT#|8k z+&LsT@LFf;(k%K3Fu~Id=<0&3QJQ~UC(>A7yYP;Njlp;yZd;{>gVA=vVdq+tdvuM6 zE2Q=dEYAdB^m8dcc4 zenVKAnx<)gPn`b z-jCU!#spxIQ7GVF^QLvh^NtBTEFOCGFD>!n)}J&IX}PF&Q3`Z{_!$>NnJ`OV7d`;K zEqkoM53aLl|HyjRxxkX^A#GV0b;r7W>%l~S1AqKP0G%=44ZvioXOS3i!^}*Q%offr zht^brc2<+M@$B&At(R_Z^lzSKB2WHlMuxVHAp^pD1f>RqWNL13?RW!uY+ZT?{gaj;qLX#4wtdm$w)6f~q7-1H=B=+W{b(X}eS}CTK>s z!*4v?tL~1M#6!;3t(VAe$yK(=c&@ zXY2}XZQgRw&1{6QEVRnZWpPyT*dH1kE>S2CEP;N3V!-9r%VI$Ps#V0^ND3_k0P1`+ zFw7l+AAC7>ZlM<50C-m9v!-Le3|KoJyuvWIXh zk;LIbFzDqVL!43W!hhUQznKrA_P-OI;ad5;)Nna1`fLWxon(|fWSp~bdHL*cy?5=! z5C=C@VB0=LV!p53)}a!n=YwUAeY_-UUai(LXJPcT$WzU+)2CLur*84L3;@W7s42?C zkMde%klzE(DxStC_iCuAiO~jdq=vPJCq*7fl;2Sdhv$jYyQ96b8a;29hAnYQ@eM<= zg_zTjE_fix8;CkeqI)qD2W<#@MHgL!3+FV2V1t$T?5ECqKHIf%PfzD|sRXl2iPSfe z$8>&vM`CaL9L!bvyvDs+Ti6;TrT%o9(L~%ZaK|Fr+Cw5T4UXLBW%3+*ws_fQb++_9 z33>Bnv7+hCO|(5TK}V6$cT=2Q^{^3&nYJ)dAWe(HbH^Em)9gsHhe2&1)~kA3-j5;| z0r4FsP?}*96CjO(+tg`9JSvYXXy!z@oGxu)zagIqN3U?SZxBc8YTom5@6;6Yc`J<1 zE|W|*6;m16@eRN`C?a|U?u?m{xw+-h$kiRH6Ds!TSzhU3E|f=S?*|$@$XuL-UCRRP zkgB-J;ZI|_R5cO|N2aoV1|YqwV6~zC-4PNe_7vf#`2z*JGDO+Z3^`&Zf};y z@=o>}i{iJuaK7$)`s-890tLPa7!K@wB?g3y3)ak=QH5MS4gq?co>9Y}tV41gV0F(0coRJbX1rUk#NBX$-2ySNC)pFR_2TbHJNoEu1(BvI!ElT~$z4|+r z7iL#^{pkv7Pa)K=dW2?MU7q9L0ZBNFRxe()55#4J1kI-8vW6% zIm3jPWt$sr(JT78QZvclDh9wfKa&gVAX>aHHj1u1SMIdm{Pm7a9h8^Mq^|-);qXT4 z-~^Z-vH#WE;7#JZ?}ZoX%0-oW1CYscW*w}rfukL4ePtwgZq_H5-b2_#1hje8As^<= zJ$G|Xr^WCLt4!kt(I<$`LDlMf865$Xs-crQ!R2ltZv=bM>9`m!YN*1_c16#|DI^r? z(QCwF%}{cfVbyQrL0JwYtIn*QnM`V#VK8#e33;Hag(BcPfuM+4SKx@W_ba+7;P#Zx zb*Y$BN-Z6*^Fevxsz-D{TWp3uW4$yNw08wgRPeIn(b-%&SZyj^CHorqAFH1a8x@R~ zVAgTX0@M5dyD2HUQ6!&eDXk>nfQ3J1{VW%UlId;GsBD{gg6w|GG7b;?bOBB1^GY}h%<-s^^xNsUq}r26Ef)3d9rdF zZko)bhZ+fZnWhJY8j}K-b#q6cj$%j`1HiB(@)m9)n=&^ymjzP;rBHxx$MM{apk;km ziCPsmfNrLQWvilzylgy0;Fp^bEDl8Qr@=K=u$^i6w)1)(?~>6EW^He`DNxfo_?qvdJueNd_l&=jlRa!hl!@Cu{8>qps}LkSn9fR2D~26a`n=m z8vp8a^xC&i@Q7nx|CUNv3mBz@X>o0y(ViGp?)tP|yI1Ff9X!2jE4;?`O^@NfexHp! z)fA+LIrpO|V0JX9xW&%QI~`HtW~I3g2cJ?Zunt=}YkC%SvVwWKa2R(MdmW1Hb1~ZA zJGrE`R1JDVAUdq6&^j3 z(Y&J{LhTh|kBx==^LTq(rPjcSW3mLnk2#iOTi@;X_ZQg#shfhyb2d=jwSPA3i(j-7 z<(;ONFO0>j{UfV8V_p7Gc-#H?Ah8mUkOC%AIB#mwPsgT3OJ~7Z@=0>rJ>hLMBWgoy z+TFqf`?vQWwr@R`E~9p6vG&UVC)475+A^yqP7E-q zbXZlw6Gg}J%lHQ4JE=NXBajI*-~Bm@&y~dhoDe*^s|p%}?`aI>AAU4;?XM!LT2qjl z;r?fP!Wyo=?JFpgJn(~`H4fWV zkstPoBX>h`{^->z__LY#uF=kck8{u8QGHd*=mZB}X<#y383Q7Q%ks+Lv@jB42F?V> zJ#&LP@}9ByB@E{>w68@Kj4_*1Uoyn!08woa$G_I@%|t##qR1I+F`!~>MUEBjM*Db0 z$bLX55zg_3_{f{E`ddcspUB z$;7DDPe+w_?#mC@Cxwrh-C;)R9{xG~u>Ie~cJDO;BLEzdf~-+*W%p$A%6cb6UAabE zfi-ZGDGjWl38U4fR4=DF?;p}~MF|=-eS6MI>C-d1?Shx96gYkz*^KTdoZ$qvyK2I| zbZN9j^1M6nmnW5v_`TwyDQ0`TJEm_DLaQ%7pmd=9?Svn1iGCF^O9~lCEFppszADcxCP7)@J7vxxUq9u(o*DGM-Ltnm3o0$u zIh=mF-wv%2p|B1rS)y%zvJw)>?m`;42wT^r(+n#JwgydfL~T#=)PCvDyAvOCqk>H< z_iStMs+xFe9LcNkCL~*3ygdpxGW4EmHStO}gxgiy2d9)1rj7j4tNXVa8eNBXutvPy z2V!o7_NQ%O+QDD&hVxL!NWn=UsAy9SSB$f&(b%f)M41l>|TrEI2e>PXd5h0!9W1<&nT4 zHFO%xzxW1=uUs$Y;9}XAKE)N_PQVJ5L!r!In!p{*$CBpG9M4S`2R`3DVT{j6Pu~be zn*{;qQ*(f*MyDO$=z=b?28zf_hO+b7rLVL{Hv^y(s3iERfsrcb7u6hUG_NGHQ3N#G zGubNB5vBg;$_#Dm5Z)Up3LKLT$4elbMoAnzf%FPDMBt{H2WQCnG)} z%@LWVo0oqHF#aM2;*&o#O$V+7{Oil+^oHRnXlkNtvvLT|Y76fD$!Luv0m)VS)o1L2m#;8EGf^ z>te&X}Jr+$t-!^m*&%F{iJhd-SZsO-0+lybA8lU&qw`;pDfn9y>L_4dUS$X9g z{)X1`wlK}W))BRlC0_lEiGkAw4XlREkFbOkBy1$qa&=M?`#zf z_td;$@bk~F5fYYcw+&6l9N+TDLY#>Gfn!CI+Mn%Nb4wc$(kM{6E)YfX`DK(QMl$n& zAu5UDKzjM40?_apg1}nn@XQ}sl!|idkVS87xuq|8y(!qTEPEFr(x3IDF}!m~^lpMc ziyj@`3`|;Jk;hvuq3;JQz5QHnG{g24qFGXBR4MI8`++EDBk5syvJ%xQSZFe6TS9 zAPShT1OSa;7J`M;ajuyU#56OmS?&#Ku6>DjEDlQD!~JxLoCz}gDFue2NiPO;S3E8jR2*%ZaCvuYXOhKvjCm|Zi`EL>k{5zS=&7O1FHo}l4aZ_CMW%#t?%y0YIJ;#I7Ux2+r*leErzV#2wso?I0_NL9`mlg}hvY%csuW|(ddCkZwSCaFbr(!kQt z1@b*+_@a2iVdrD;Y`r#z+962(kA)$Oi-V_3cTIa<>|*419BOzfiLOqQRJxab*j~o| zQ^fNZkNsnLak{vg8)iY#yqD8rJ|^&_9d>1rDoHZDcouU#e7VxQpw8L z;w23LxN%FPj=BW4F^e@EZB@|;!gkMC%Ij)-voAj6-i1l6`{AiRH~WO8Z0&LX(yaO# z;Zi{BD9v$Fop^3>dd2W&sPvt-zJMtNxUq;A4Ew1Oz6$jVeQv=n7`Dm`$g>R6W`*dt zVJCFz$I=rdSQcPN#{j|D-ym|+%y6X{KeN3PnugPRORX#MfqpHmGZ3U7gUVh5(a%(G z9bS0^ayA=n^bq;e1PY$~V2;a%`gv{xIlyo#tc&@WdRkO}N$`tcwYRcuJd>S|)>!~? zflfsVqJ(#JT^XPIL73@*_d}$U=me-J0%({9{W^Z%BVvR&UM&vR)?AIl6bko($EoEe zTlBhl#(zYy>@OQ8Op!-_pyIaT+O6$|})i;wIa3j1C^s@t#p}oLGKsX|E z1Mu@4AG5s9jq0T`quDvRxnh5!(E!lUrZ*M<0%p6m{8Zc~VH`2|a1XFS(H0-tW+PMlh!?Zz-jHLgC;~8Yna@(cxl)l4%u1V%-my3H6 z?OTcHH*uXhwib(8lLQ*T6ohK~@B!LDiRk#Z{uF-{18Kg9nm`%Uu~lWrv=mXwP`8=q z$f>RKd*bD1hUh)@OCa9Q3#^nFI`nPB=Ai*>)VXUbQf*yV>kc@@hV zQ@Z*9-)lJRwM!lz$?Wtp9Iu9oQl3!G&96r;{V^7a0^^?F3H;Zl9x47}#bYrEqW@{( ztj$03APORHS6op`tB3q1V%*4*1~;e{AB#1lwC2;$w;7nFE06m(LueYeS{WC7MZ$Vhpl5n$E?}<}On-OUTM8 zi^b2NNAfHA%DTYB=V6FNPu$Gw!RDXehuE5>(ad{WGF?{M;QXUh^i1E__aeiI+HhOn zbWsRMT*#@2JPtTrbU6Pk*Km8h`*xWVFBWhFk~G%AF@I10hUrULMo(9s{@lii;t}n4 z$Q~nq8GI_`(3QH+UXN{Z_$RZL4j6i4A zlS-rX_7k>iKj(@pzwCzHYhI|ueadB1VdL9?&`C2?F?EpQGLXr3W!GnZqC3FBYUN8T46&(}d8npC^)0jv^*LYL?f=#Kkb-C2GSIzcRX5pftK%B9^6k095GUBs#I9fm_3FP zbb>2v80&s%|C*qEakpS~w4ei2f(QJay1f&nJ^pF9EHHhEg9RBJ5u(^6{BYCEs>$n6 zgdJiHariVrc{xBXLRgjzj@XmPGATB(>Rpb3TA1H0$5*xqAn2Jny3Ku=&pRywTlpE) z5O;jgvL|Ydqc#C$-2ZxQ6oYlon5^D@_;8X)y6yHqcA`3VQMecYH(Dx85yw5rns~PpBcP#VshlGPG?e z2}i^sAzPt$?uEl^l@A;}quL~J`Qx!LIqQk?#3!+GFKs3olKo@a8LH}hUBD^V>RtM0 z^w5UL$rS_Bgvs#F1tTW?HFuf**UT5EuQvzWo3}?Hm)EO|Da#IRKt`3JIkF1$0V zHEUdk7Hc!Qb8yP&UMImV&07Kp?J2Dg2gblRX5jkU3*HIlNTu#5?w1Sox*dL#FN6ED zzDF;&iJ>!WEj7m4(t}boww`KYr3Dz2k%DT!{2d9A6q2z;9FR2h5uOP?TGRSoFjqa> z?W?7g$zAO5z9klGns+=vIzw~_dB{6jY}_w6Uj!C zj0A{iBe0a`ER5l5znA;Q&E>CvsWsM~ANfVDzV{y<9QO^!x2&AB26UO09m|W){H2O# z;9(^#!Nkq0K9_6fqD#GD-anqdi+>gd-9C!O)nuiIC1A9VK&z!1fV@S#6!* zZwUx%?FH+N%U!=H2!kRGt3fV#g`*B;Hp5S;-T>8It^uH^_|1XNz;B|#qQM2KT z50f|*DFC!RX^c$GY2*pOWywDjT_6k8c>*=#K078*U_GICyX^veK`*EcfFrjrSz;N% zm56Ed@2Af#Eh+W%gL;IdG${DR>gvO5K@)FdXdaPj3}SARF<|A6q71obKNj z2fjY|ap+I{vlw7*ZfiX7{we6-nT)^#L><@(cpYdv41r%27ZnX@ioTMlV!$11Jk7sqMpYTP0q~_Vt-zbI z`bh&^#>Wqb*Ux!`qo@CFUOiQxHP-w4)p{QzoG{=o!2rO7pVn<9m}NnSW%iVsKRmp> z6wB4lR6P7){7rwRR9j;tt^+%>?9E2;LpZ1ZE-;wkz}Tn=rBO=%m{|>frBe(5jiR;- zKJ*!^oz)ouF(F=ONU4x~2KxE4e|k*jb}~)+A$a)SUI^v(7;bqP#)16%>^#$YQs&H1`P}jz zc0sfyQm8Z{fN&-PeCnP_59b7D{HjSgZ6}Hb`)a8otE*lLau}EsF_TvL;MEzS*U46| z>2K5qe$|t_!WiQ;s#827*8zelquwFqJ~}~$8<5`cIw962EUx=(8^3nHfrHtr!@8|o z8IW9wF&*u%yvNk%Qo}VL-H0n>rKxZtD(&rfPHL$J%DGVLC2$=YRjO^+i9 z=^iA0Ff&K{&fi!nJ{9W zag;UO^>?6&%w%}lsmyMgtw?c(7_gQ>Lg05bI}pNDD$#g{hzk)2^Fczn$1D!5yT8^ADY5kxp?0{RzLaimqzDy}-OqE~bC2vom`f5Psa`Wi-drh`nc zT09p5^5DE4a@{~m=cf^uO$Mj$34gJ3HrY z!`WVC)=G5sA9tImTw)Fs>C)PfZZwZrhmehcg72ge5LJl59|fF1%X?RZUEQwh-A4AS zUo+Z#`)?i#(ZKf(rHylI$)!-+7LKsu8FOB+N=bHTTx~I~UGitL776a&C|?}H<)czr z_G3ZwhPMmDmc$;@3dM)jV_hjII>YT%glhJJn2PdPzepSE&ewXLvH9Pla`e9FgV zK$m+3GT_`-&2t36c)sdAfUi)Oa=Il6_weUTYkO^v`5yu8-&?Z44z}Jyd zo%=$GN7Wi`xu_4rQvPY)6w$k6hXyhrCQ3c27sL1hGKoqs8V!6VUx15FaEYqfQEF=bT{Y3T!PZ}|u;HAwACTPiT>X{+-xs)c zl0k+4!+6UF5E=jsuZ$2j0$!`zR4Z@llv1CjdF}X4=7}BzbDK=|ZUB%GJ!S=UMKzgR zmmS|feR$g=)t0h#e;1j9p@R#4PV$8U#oay_<_IAaY56UZs8FeLPB_2JrTcuBA&kGT zQ_8ceOqch5QJqqh6BLR zcyEvVLrpdERtR!8umrI0jfG%1jU+hU;UD44V7nWL1oqa}{>?bMb{acYJhrP~ln3SrEoXi4*lhKvll54eG<4 z2Ne_cMHmV%v4isdEy2pKHX?UvQe<3A&FLzQPAKEgzgS152OTrvE!z?KT#4tW?czG4j*> zBD>qWfhd(106;(q8DfCWc;K$}q5Z(bIy4UYMyzAm0Pvg51ERoe#c$!0k0inSt@rx? z&>$i2wc{~?VXzjT+f3G=v>nf_RX_$!&-|SE;ZqsouL_E%!v&$ZKF#eNK<2d9iH1t}9}_GV%@m zBIj9Y)DhF%Qv1k-x3t^%q~_X4%ZldCF#Bg(ul=F#+EHo8>v*w*$i;wUDm*L{pESMm z=iEzb@=Dct8;x$w_%kZ!l%@qQ!cM`>#DM*EY<5^A&kMQ5F`%V!~MwWkDb-RrB|DAlsaM9Au)o+Kn(rNd^4@c1a z6S{}M|71a#KxDj|XZ&M(w=@U@%;x{X zZ$2H~A?R$AvCn+VBgwg^@9ZdiMrYqN9gt-457P~zktRy<@LtINK>bHxEDhMhEI+0t zYbuCtO#9Z~_~vH`5slq%z6tV~>sKwnEk2f4%`w?V{4mal)%TGE#)8^F)tYzp1pL%0 z`Zs2~ltuKvaE{pkQll{corfYSC508zRTlq+RE~JMf_sx2$srb|bHDoM?;X=<3_4an zz|VV2y;vb+4zf&woNF!5nWh)lg=?EV9^;=p(jP473$gLD|9NkKw<)(ld*q3>Hb*a#)SX(3m6gu)8b2PTcAb>I{3;a9EbYXKMx zwhxnTy3qLjz-%HfVc07TU;Rd7!G-9If0`A2rYE-=#eLq)NuL>F$i&;7778tahGqgd z;5iAJ*eSlkUUq52yp}VQ*?_3)YN}MV#B9%zZ(eK*JLx4{j@Dl+p`R=dbJR#J1CxFc z9hbi`@9g3AYLafh(M~3Tm@HSPyU-xBc;qsu3)CqOc5ptuJ%a3T_l=2>=v<(PW3s?d zbdb2t$#m; z`ed`yCg|?%WciJzx*w& zGgFl3ytDpi1Bj3)P1T{WO~RpR*#zA>b`WY0w&9jiI8=@$1JfAZ+!yp)*~r_f2zH(@ zU!Fo@lF3{G5XDw3LnmICefUAPjCfB_I=**MMFs8Xrj>YXFe_5c4*yxO!Sl>k&mpCX z#WZAXHX+zHZJS<$v)rBsPgV}kKMnqVCPuR^36p(rH-|vJNhJV=uuX-CYKvI{K zm?)_i6+;76Uhj9VgC~B2o!oukZOj#0)npVN;_`_SgI5${JB~}!S~R!Jn8a~%jiTA2 z{mnrhFzt8opSD@S(v}TepzIr1_(;Xt-WG5$(Z1o8j*3{Os_}!MAt9 zeW%G@(yI1Dx84@j_l5T#-DF-O+q##;PJUT!I3UCS{Ek|9KHTekb#imF&Eu=LBW*in zVC=nC56Hjnp&oq8sjL}@dBt(V@TZL{j6^GEQCE4r%q{HZqxQ+f5}}|U;vfYKM-r@T z*)Nl-Htbz1Jj*G^5xR-X)lh0!;dpLFE19>TYy=r{u7$F@Abmxt1lB;#e)t{x(GViy zsa8Q@1*iZqQ+?zRU4XyVfyxO&s_Nx7k@N3X%iQl<)WCsZ(G40J_3?N5*xOlhnbdvy zW!Ng=4*|i7Rw@Zhen$^;Kb7lb5>_gLe@t?(RmP1=?{|Q!15TxdT|SXNib&!>{MtX9 z45;u2cqn2-Xwe75p%03B$+S)uV&D$jJ%6_-3Ga*+f;8f*?#Sjhq>mYPwmN!Q$R)u$ zTzr1SV#AWbqwk$z zY6T`a`afI%AC33?WsVIiPF)~B8UrdLa;_s=@Ri^I_XNANkTO9O57#rw5Ms$&fu$eWMm}?h z-}h4YI4_9Qu#d{AALwjL(>5g!CTF}9=RpfhUzxmG)PEA)Z#dq5Cf;hOzyP+=ks$^U zNcZGH355>7XY>`Di2N+WjSR;+7B3f7Dz5r~>}?hN#M0~rYl1y}JS|7g82?Nd65ktt z{psKZ=m@*oFQS!{M#3};Mik82hvT18tFKeasti7Nu^!l05DMD(OA1;3`T@Gb1+gpo z*^PV5r3YJS+!lBmpX8T}`npOKL?mgq!PM*%XQl+f&4ojfz{20UAxoq#tdL{oozdETG<$U~t+rL_(DN2N38$Z` z1f~2fuWdB(t9$j0(Rlj1U}?7b9r)mhoMefd%YJic^$oxRd$vAiIa3`>rg!c)_lDyY zuEyyKCutS(HubAN=bX~jsU+TE1@;1C>`PUurH{zHx_@$18Ep z7#UEwyVj))1p(JZuKYr+Ap~2-{hsY>-I@8;@WuOO0Lp-z{0 zYaJ9vy*OD?1bvJ0J5a3SnnM@xzprpok@R2;S%&tbj}xY59I1t^?Sq2BdAh?bz~q{{OlZig^{xAZX;3(=sFF)5p-O5HNMs{&`_}MNZwoO z6T7zNWg}V-_s&nS{@;wD{|Zwo0E3+WATS^%WnI5cuekDNBXDZpwrJjZ!#M6E`^OU< zh*b>U2dqa@Zv=UeJM|(GJN(H3Lw}eHxkk+6K*O;#FHrcr&KAfb(rkvw%dPtRFWD4usUsYL8Jx@4aVbv-{0VLNANq6g^wtScB;}& zQ2+n1_FiF4t!>-xP(-E;rlO+KoYMjVf+$rWQJIR0fQU*b3IZxkdQBptqN2otbO_26 zL`vvA2~8q3DnjT1LI@B-fRKdrHSm4Eb^N>kx7H41XCwz>JmI;Y`@XL8D$ioz8^&lv z?{Z8R5$yX+ir-dHlbtt$Cpua-l~Y(wKJwThGI>soZlkYiF$LeliNg(}e+QUr00-FsN~K_E=|ir#YZ7>9WjNFntu4TOb?&$#xp8RTJdFOiNXJM2#0TKv zC;|03-VQ>%dII4V2|J-`4(|S?lT+2pwi@=|!%YJ|o(-RqALo?rkL( zj0?t^bV};Sxl5}=Q9}XJQ2P<{Q2sb(kjaIto&bBLc6b@Uh_w*B0Wcdr71*J2ai7h2 z4O5TcCoZ(X!{>f(sYp3ca3J>2LPznMFeFmgBVbQ`P1pc{rvye28Iyn1dq4#Zx`R3_ z-LAcb4lqaT?Dhs+uFIV?`F+VsEaQoxdpWYq?ESW(hA*g|UIT|}$a zfrB#8(}Z7ovUmBQ{fG4!6>tr|P-haBmcLF4E#~Ie>K~^KC=SFZy-nhZOkJPXYH7W~ zjxL;)yXkh1)M?B|Rui}ML~gJCRRYJQEID~%+YIEMP^@RnkgBT-r`ttW?JU+~>!v3qM?NHqZwrnnO%m3KAi~O?47ke4RH-Kg1}@ zoJGn`P}`=<20!Gw=4})J9yw>u}wV`Mbg(Q_ocxM|19L0`ZCC9hb zG&2z!fC%4_6zA0#>{bM7?C?y?_(lCKbolJ+ctqsxAwh99ocT*t{0Feq=Mzv(vptZP z!AkrVb++yyJ%2(lmA*j4I&#R57%0|Th|Sh;R_ z%x;3}J(uM`>4e%=!RCsT<;Y!XNdfO~DYZ1%=pQSN*6wIf>%ysJws73%{>g0)YZg={ zF_U<#Ctf@)E^)`+3fF5Y)4ma9Bu&hd8uPslgki#HWO8By4ygg$a`T$hxTN$<4JLBJ zIIVpn74vZC3`R3W#82liK8)hzfkzzW$kUn??^Jb1Vo~m)4!vx>EjYvNMB(){bJIWQ zP*jo2=7sS7mv~aKicd7Ycjbl$l0FinWE2qpDQTNN0{M%iSxzp?GT7RMG%46bjAz_Q zJ16&Lr`zK86fe7l_E4X&9hx+`tlL3T=K12L>77MpkO#3Hb%D%mp)bBOR358BZK!(Z z^iN$U=YU-{+d<0-Ug{Ohzaa+zi~Fu>|LdDC%_fv0+ioKn$S8 zW9rtDphhYo8zb8X^Q_amD;Jc-x(W@=cZB@1^BbVYK$t6{?!#XhFl6!^tryEVHBdJ7 z4Sx?!&r;TS5z~4VvE%YkLdE`h_LjW_*xXqmW6diH9TvR1PDP`~rY-Iu%pbP>@8S`I zz^flbdwS*H6OSyhmv&U+v7F&;z=NSjg^u%GTs8S&O_ED6s0kDcR5t-d4Re`PJ{EDD z8O@3;S`pW8V1LHb#u{jJ?p;IIh{-2s8y$b>ypMa5F+XIym1(r*7fT+cur9kfmPoKa zXTohCa~8G=c`MOb)l~c`&mampQT&X}H0VkHv2$Khs#I7h7xhG1*hWBs0n&?qvhr6d zZ{e(PFB6^MRyG+i1vjnSLXXA6l|36gm-RC|!tiYi8HSCM;L*IKp)r#y@f8o4gVK;m zNU_n@E*@uu30gSlc)Sz{lQU=LC{t(-*QL%*EhlwhVsOt>EI+yPc*FMAdZ!-DU8}~J zu3^&h#1Z+^I)~B4;wc5?KHq$?W z#*&Y&s>X{EUefhLd&6YA6>yyM;C!AqmdQw_>Ltl>u!v-tWj;;?8t$c46*LM)S#AKl zvm3xq`hki8r;@Gx_^J>aC&_+AMUR-BT^+b!g7VE?d)L7Cyao*7(yUs}0$1XP*GvCI z<|8d6|0*rc+KxN%kgFcxyhC=7d1jkoDZjWB%@z0(*h`ro9z}^BOZdBhiAI)h7N=p{ zdu2*h5?C{VRUyQV>I4aH-woScq7YqFgUBK6>O9M=@r2I^?69t@>NeLB^Si@TzsSk2 zarrKiMT**~EKNOA^s%xKW(Cs+949EV5sKt=XAg*ntYM`&&v6G^#}w9o|Fie51R5xZ z+5_C8DTjj(oc}(0kn4OK(#gmDC>2L{71yLo1P^&8bbm|2G@i;K!10CZb{6!Wqc3G=bsq$@*5RPM)~CA0iRz9j5EYhq(sz6 ztF^nniGQ3D(bFuG0e>(Z@0VGvTr-4boxw*49B!}_S~=~D(XBVU^4*Xk_E~IX-45jg zdtoUE2bZeV%$QCI0Jv4s6V$DW+$;_ZxaH@m#f3!UOrfp8bw=VnR1Wr);rKpa=~V!H z2e33}qYOIpxi5nU5k)Zs<3wuMg$A?p0zbXi+r(zzhNVe(fav5(-hzaD5B6i6>CV$1s`>ks@?TdvpuyrEnZi z1wmC6qed0D)%-|07RO$WMw7l9-v!IA?5(r-|DV*Zum4ZAYg{t`Tm==bwDYzAuwrT` zQ*v#~V$F!Z3DC4fEfWBK=c$7U1)vRt&IUlus|+c;k)jNWfN|_%;Guubw*ORdW@gw? z<^|i`|8})v<{}{FTH87cGHr;+{c=pULn5*32b~1kE7z;fp`n1{o7;fzcl9<3dKkBB z!23``6wY(fMk4)5#{L8z(=!@O1+S3Ii>99F#QX(JEH*I383^|}C!Oh!6qTddK$hDaGOA{v!RE2-)0w;ZeU(`aqMMP8~QGSVdm@kW-k|cgIf2Pfm0hmJa7~^HytmM<~#vCE;hw` zIju#9VWT)aSVP9%3eHMLTc7W-OvgDxQsbwaY;L%4WO_5{kNf#99OQEK2MFNsu; zG?pwkRKFb?8Jy@%Fi7_P25@X2&uhT(B>k~owYT-_HTO}BaR zUWgUOc@t z@sY{nAL=x_iu{OvLY?3(3pChv3G8f0+kR9-w z3PO8iC;N)9p=ngf%|i>HwE^s({e*U{GUa1byYMKNaoIj9ZPOjkc(qbRjbHNTS69cO zH{6iPhx_q!s`aosO5zOKw>)h~M{YcTP7RvJf|}ZkE;@#~y6dykAXff(#9-{em18NL zEPDi-eexLFDH8AU#=DE1(l^+WUqn;Qj?`wvAHIw4>b3g(9al!?jtlh9MlY{C+iZJP z7Rd~J6vWe$uW+_879k(ef@!@TyzLP^j*z1k?!)~|lX=i=OH#eh)&;Nq=yFB#Cc-i~ zH8YD4k$==#FRm}gB{}QVZT4`Enf{nQ(zAE?Xud+GU)bE)Xl%z+MmgsVw@x-;MW@*C z?o;QeUL#68@K8b=kCeH4?fsm%;EPUJYWA7E`y39xc=Yha?4yI~kN(k(-EwF?eee5* zdas*bqexXLeSKYK5-vg0MS;GUDv?6B{CC(!Be+6D+6V=$5Rd1|`O~6?hA` zNK6_EF_X`CRI3g6Gd?izya~o>l3&)wdf%r!yNPY0TQ@i#+%`|K$Oku;2%dJdvzRcl zdMZQjP_$MyQgW!y|}q-r#N&&!%*a|?))ZNM#Nipt%A)@s91n<0c!h?$U9NbGI7G0R z6kz9VZ++g*S7jtmZ%F$GRYl~jJ!0gRq_jrxjy6{2al0vhxE^vjk)ce?K#R04^Dh9F;gvNo)HOt zkSEwj`)K^(Nuzz?*hDNBG)sCXdKOsU=)MI%>4(|Y)uk{K znGp2R_HE7iTbQH~k3VZBM7CFqztXX8@gQb0IA6hE4j5#;_aeZ8COsQb%P0Bzb@WJs zHGPtceA^Edk`BM(5Yh#;eZYomZt`=-aaEY8yVFe9lfqF}J*@;h_+_h_tN8Zh=)Ua`0>#0~cs>f-vi2QibGo0;yq>_gXktJQ8m(2zS8|7nNW@_-F)ZQ-M@i?GViBri z;Lg?T)1sQN2CL}qu07H#YBNtDWc2j|v0^9A=Y9%CyLNE;FFbliR5_$1JC8XR?f>UN zkh-yam~GK`4H0FW_&q<%NxYcH)C_8%fhZ~&AZFdY1B`=^O`-a{zR&-B;8Cdau3=@MFI+G-4_|Byna z^LZdp3Vc`4)APq-FxF|DSfR~bUJuBT{i#VG`eB>UnutQ#WWd0B$Nkk>-dC3>83t zjFXMOEyboFm>V<%uj|IZ6fS+>On#p(q4e68l}QUIZ1?%sauUWIbZj{&iJt2sHYIKV z)&*QAOii}~sp;uk=lhwk`jC4|Dp2TyZ)v#v|7g5#%GOwpiRb?_%-y<9{MH4)Xv0mM z6KIB15@dn0gyb+^ zgtUGyvU$8OKkLNx3i7>tZ)UEjoDwzyioXPC$rfOZlneFYSV#YS>mys1b3gj8f!!(= ztlBIhp&mLFL|g5%kpW(UAua}avSk9UkT3=S-onuV#;aggG{KIET-pFMf!|Kh=Co`X zrYrd5Ewq^}X6@;2Wcd!K>mg%id(CiBQ8jE!s zhOw@wVN#*L9ub)iqHYmoPnh`=mR=ETyHHu|A1org?y;D(lz3Y39Yse+K6HiNH|MBa ziE4E)-5-eVa59-Jl$gt)7<%pk_>7r*AoYkoQI-tt4n4j{#RM!#2WG@M=R+4|?swM^ z5=DQ>JB0TbJj4(#4Y)^&JEQKE;~XS@g=kyrRIBaIqL2D1bE)JgR((+_uAfy8Z;*UA z0Fq4c9)w(S56zLxwly(etabVz!Pu?Pcm?p?9)?67Du>L z#aOY2=;4Br_G0GAMsCZbpbNX+D%`!x?qYly{F8Z0CFIwA5&BBzh$x_P%Zq7B=9$;@mN7%w1o=1HDN@NsT zjdt1;7M9ZAvSDuCfA{E6X3;7ZY zv)&o{AMjCrB<%I*%*O>)1DXpUMKuPsW` zx$|9^>ZsH^yti+D!mFAv33vD7SE@iYLpA(Ibuxa@=v@_USB{)L&5k<0kLD)VKfkaKq2fIsUYrR+ZWO>laxb>& z7vm51rR)FE>s?Qe8+uj;jBi%xiVG{&)qXd96M2s~Um?$qix~bG z$f94cpsgUH5##kzQd8E=q0~$o&+(8l3&{H|XLW>?#DonGx z1d)3rl?XfP>CS&U;Wyxl@v4#(IZ*4bQ@z%|2VdXWzMtf2sCbkZgp&W~{Vk!Vs1zp& zDIAJNOQvby5{T&-2bD-iEjT9M5{cvP^7C&}`R)80I+u@m`z#RH2XXVvV@SMO*Jq14 zf?85VWM~-w?0n$St(#M%blH(;wqv0qZorMS3c`tb8#?#DD-3Pd%^$Y3T%P@KMs6)U zem>+NJxS`FU2-B4{jH0fCk$jQl!mK#ahl|an*h@o$;qd|Eo`kUFsRB$%m5o!e}MHd z!UX!kxvYFO>UQH8b*2gTAhMNxg7Pw8hpzTe0I0~&!Dxm>(x@v@Ru0IW6yTCO$IiP` zP-Dg&=ildS(Bbig~Z z!7QO~QT!qWgiNmL(ve;DmK!6T3+yltvZ_eVN)2XjzJIZ?*OAmwxu9flGvE^S{3|@8 z|Efe`)f_Ioei!w!y8^NNMau0gwww8$T4}CndzF08X%pjGz*CliG3j}<=y-N{lp|6c z6)O(sahj3U{D!qD%e%M=@g%*uC6p1jvU@lJls#3%cG;1*i?kgr*N)WDIcIo+{1J-ODF&!q`e_Mj6$GQSjUszD1qbosL=9fV zIH+w~ttPBn;ePyWf7@90<$vOn{#$_k-wmv z#E8#!D&D;AIgdbqQdTLTQ832b46H_B51JAB?&%!>;m_GJbEj>0TSc#O`N1~!rV_X8 zAl^EsJUSv`9{^I1KyTjk^qnP_WDtwEPIU*q^xwbAM4*3{jS)e$*!i&JkqZABD=RDM zu#m$3t%wFQ?2ikI(wq!c5u_$p1V@Z9&i$4Be698Y-m9>ktC2;nH+Wk>Be_bOF z8Fg=OHf`y0L!OMGOSCJ{B*}~fYqml%e=|r-TCZ-joa)0c%a(r^thsh=#{PgDS2d>P>*& z+xkz$)m?ealXMF-vjafM}tN zpmH_}LY@@(d95gBm$8W}sMLnISp!z^ccm(iuBob7UX<>D{rfi~M=`3N^s7xg^nE<3BD?Gi-}d5GY-Pq;c5XNQqrID@B);7LD&}5xB!%|8m35+Z@n-mI|>Pw0t!G>ZB$NTN*VRrUlzlq@0)P zr}9FGp2e8sz6 zpB?*Gay)&osrHXX<%-35Eb{KlATYdQEwpa59ASF;B#AHD<-?qL-S5$G@@rtr3yM&0 z{?ds=e;?=58Q_rkL921~qQB@-)K@`Pc#R5etj^m1JEKvc)<3jg>^62ueXlE?X9{_t zzRwVnU4o~BhZt>D-oZXaFQ~iOb$EHpTrqsJgQrr6PvCP)or0K^#nl5u#0!8zl90;sR;P`ih6Nh{6F&qgrJ34t9y1v|&ad(q3WYvRq`MZ`T-LTC@zG*@h$Z>WF)sN-U3KE3 z88cd;$GIZIVn+WFf<`HIQ{KWi$LcMzv|cuV_%Fi3?UKG?k`;O2Kr))!$Yl&jtRX`y zmypMDTjk59_(jYM?rcehCPo_Yt!tgJF)Fd=2v5Dge0@v%S$qH^{RB?`q!)wT9{`sI zmKYdiP@*!q2HNXw^T+ZO4R?3ux3pg7WQP$1u^n$ZjwCmAKr18Y$J8=#5uD)c9Y92@ zB51~rpcYzL7wZQck4{9mLzhLv1JLwVD>vBQ*)p%bUYq09feYm2Id^vZ?MOjQ*Rd3g zCogkPn0SJu^5-uM$sR2}+h*hfKd3XN6cDK76zQH4kCTJQk(oj5m}n)+k53%<$CL%m zHK1uuElau560d*=(4ek0FtY_~Ds&xe|8^MRvt7W`6@y75%qf3JM?@|yzejA~qMBXZ zD8<6TfRoxXtoV%R9O;-s3yd-Yzo=?h?{` zdfh+xdN^V&#ene$=tyqKX)_yjsmr=VU#oE@Hl!GMfaxaw!IE9 z-do5ts;WCeJUL__sUV6m8v)_MYViJ&FS^~=<@i3IKzXl7=kcG!E1iMQ({Lu&#_gGXtfA+r1!Tz7K>{1h)iP!icUL|8Eh?W~+-Mor-@kmQOzv?m^Bl-n{4F)JvoX^zJ=Dqn*~hMJd1C=a zdJNuqXO;I#Dv7d*%KYOp3e=oMXyn}3|Gz7-{?i}(pKIH+uG#)hMP1)r(~l5?fiuN7 zr9c?&=5-4~GcT+`c@$QVYDogA0fNGQjo+y*X4(E&H@QU=#mFcbt%>G_fR2L_=op<$ z@9UZ)fWRL*LNZzh%B;9!;jr(rm)-&((MR^b?)Eot-b~cz|Htlsb4LxdVa)#)xqd7R zjChjnBj;ojfT z7ajO@YTLVa@?qJ_xhj;M&$Rqm<29hUYUvA{1A_e+&g4yiOT^p};Nc_~aeZg#T6B@_ z?w2E;ewLVvSD@mRDX`4YuAK&#wwIzGh?x!`+Jocr21zXN@RcDVnVRdt>y@`8emc9M zGhXbFT3flc#E8t9LiZp(KwGAHByJd`f@$Ck`}{U~9t~HQ5zca%Wg}f$*vFPx z38`}-$7Wk-Q?(Z)AM-E#u9&_I>AV@*PmN4$79K=;zwE0QP10%A*k|-Lh!eoEPevxWH94YHy!6NX=P_AlNkt5dM=pM`Bc<;93iL*MMMO@)+ph+s&PrRkU*N+fNk_J}k ztVmT>!}MP!vd*XF*nW`h>;p`X`Y{Ves$S~&xo9MEKhqmOLYvTc64_}v=zC&s7U3-> z_#d_rJt9dJVz*q_a9V3_5A*g*);+U~a7!`M_-4;b${}wTTq~cq@}@|pn$UJw=3gph zp$7&kCrLSy%n@FFKBX&wG6(gWqCXwo!3@D}2;QV{z}JvlGo*c27b>1hdc-SfzkRw; zaBokqdCs>~2)%5EJv8OnFus2#*v|xgbg#@AGOLwV3)yz5+vo|KnUS0-JaBsU)lKj? z==*8u#JTy7-Dm6M+yDAGx?)>!xT0@^G5Tc^k7(xK!sRx?+RsYvF9oR;V8#W}k%Op4 z<;Dmln*0_)t5TOuL#n$OOZ->BzM0_0uj2EfA5|eL$|ob)MdsjwuU$+bj?=rN(15w4QMb z2l=$Zs&wPu%*|J-lao6qi_R2B4QfGVA5@2{ zbezlYY>9P~A~Wwl;ON193R zZ=r34#kZ~)sOy--kgsr#{`D zlgDbC2zu;kn#(TJ*S5FR&b!@L>2Wq4kc326<&3H`?O41|mydkRuI4+3o725@q9(0@ zhlM&7?31`fB#$3yVo;a|BSNUY;2|)wIo>LUSC9P(G3MH5t%UFTyGD+Gly=~aYk;jz zwxny@J;GYJT9H!VE>67kwr=9cr1VII`I<({ibO z!|wa2_{RxmO_w10&{d=(S-du;Pa269xDm(kK^%(uX9>*@|9+@v#4KL)MZtxbX6@5C zAG$aic~-D}F<&wrnDy5%jmpGC7;hGkX3?SfS8|v*0Tcu<8naytl3bagt;|Q)3n37v zil0asa_FUQV(mmDEG^??`9bw~T0?VaLvbQi@Qj-(%IJGt(w`BMFbgGpQA+%vbfB$x zev2TWR#Hs}Z5}6^CeF~zvW&NNmqbh^X!(crP>~b00p-vq%&s-zanjgH#u{7DNx5!X zChRJI8+#UdJaJq7*`pR>-e z>IG4MRCzj^CVEA;oJ;W*l_Cl&x0$%4;3UlgY4*udsi+TbmYeAwmXvHh24xyy<59cw zm$}Oh_RT_%KTsUtX2p)Y}nOa7(bL!tzp2s8}I z@PV1YGjE?j9w6R_toNsea!gfiA+P~_uNl`t$bW|x9tI|oowc%!3=H!#4v8_wYG|0$XN*RS077W^Oc$!qIrL5Y-!a13Y) zTi5#k#vOz3V!?q|n}HlKNc{Iy5{xAnr4a@2_b=)mgIPk{;@R1VC~H7+cWRK$0cx=I{Y-Hm4HI2aTTcCKg#&m>H3NRHx$DY6m&KN z-O9>a+VcrM8jywi29TQJJ03t2Ny4}bH1#9F_WXlsJYpm8{TTyy=LE8f~3d=(Tdpyz}(y%cvv=bg6uJOG>r-+ z@?$ffPXwxU>|7t{z7NC?(hjYS5c4zRgEW5O{Yl|p3?w0vF)o;b5juz20Dw*oZD4}a zAl?q#4fP!_un4cyJ$>Kn4=rq1jr^KiC|}Oji&-$8$3!>q_JYxa5YA4Zsh3FpQQjqZ zdw(kN`OuNNEHL)Ca__EW^O?9mebR$sdPKX?B;5R$!bkMnB*k~{-ko*1jRFlk6xAeH z!5I4bzW9A@TdZ)?S=!3#TY&XdQgmQZhw6CGOaXO+f6AY<8$NCk{dDq1V-OJw00i+Y z_=hhI-hDl8WjMBJJtv}u-r71T`0t`B><-7Vlq+a(aSmAykwP+lPm;KTqE6spoa||E zo35_G?f@RN?X=j$jUHV|>)kCvR|!mQC&hVIey$foxbMRd`$d^YGHdTGz1v$LfL@t6 z6;MO*?m_YCpIue&1!JF)Oi1q!Rxjk|M-G0iRs79O~sL$|-JJeOtZ z{wSxY6K#f~uIcfz7j7pL2kNdYvJMDbi-!2B9Fg-LO$DC~Jvd&mr2VC zv_-MU=KRQj%%o<%bKrh&3&X8+fdrSr_(ro}m|X;AeH=fxjetW!Anz%68=r{H&idS6 zc-Hu5OQ&sD5j^*>Qp%ywUQK>!t|}x&bo&@h`bT+CH!bQ*U!oX<5`H4`WcWN^gYj{f zvCXs5P^C?~!Sya=L*7~Bqd{Tq;>Uzyp-cRe16{;>+hN2fIL{L$4)~rr?LruJTY6Ov zxtR4r3IMp7AjtOCc#e}_UF^pfD_=P2hnlx&kcI0Z~J9fEqPwa+APzPjrdaWIK`>i%GaQ+lcRY1#=Ap>8x{ zE?#S_0T~5a3N~b@X~Ybc#!@r{I<$(Yn;YH_&)<0HO`tgJXOEr1FXKE-Ark$74Zx`V zDdQ7AE4a`TyL%gU?3$@veex*KpU~(pJsSFJn5UqIXze7)X0!%hZM$MfThq}(V)w)h zsk_S@SF!gud6&hKN+vD}#-&pLAdx>4xNwV=hAfFwFETq&rfjmK)4cOY_id<~yMFbY zVi9i9h>AbfUKFe8k^-b{K&pM$21cOncub%61Ko5MWWJ#J3_9>($yBJs71A35WfaFE ztj%DR;a?WWmdV!=9VNYcp2tNg{Po$q86 zM_H!xX^gdY=-};9M*5yRgVGP0Ur4b%KZU^aXr2F3w>Gba&0gVSLy#^nE3#`PbT5Nx!Eh55aSoaCis|X;2FNJQPbW@)=7K&|=Wmk*X+FGu5igwQ?o5OkjZpj0WfWZ_^i! z?4Mz|4l`X~7wamR8!!gg;i7VZSCafBqTyqu+uhJ(MD%-V9E_WbC($SK=>C)JJ5IcL zB9GtRN{&z$$qc}*m}F7@-Jdb)7wXi^>bRQK5j=K;uKVH8Nl-M!G8W=2o*-QFJ(z{0 zR;J_2ADFRaX`-C)MU+49eoc`?FC)~=T7?do9QG;3uMa}0vNZGD%ER7W0{j9OCJxdO z?QTE+{TAL`c*IX;<>*IT%ou&mYzGFt!o4^s>*3*X78?Ow;je}%?XTu%WTtQL6+~5J zkBq!Gnw9CPo7u3)(@3xvPXALu$za!2a!q0IyuREqx2zk+Cn5ek996C+J^TL4&4_dF zDa9oECic>TpMfX-7Np?dN$p?k&1$l;+-ZNVtpD!z3hWiJ79&VVRQ@2Y76(r# zUU6eqH+=Q6=KZ(_Jd~FR?pR{1;IE$uK5L3wKDV&UsuecYP567f7xeJ$!lqunJFXJM zIhKc)H4}o(nHm*DDu`2a`*YUpmniyT5IRlK zzZ~3#u&)NnShmn5J3&$aWHFh`oMT_jd@RPJYK|Iy|KumgvWH7dSmwr+x0)NjXXj+D zEbBR>ID3o+#dIELI?f@hE+~D=Ehs2BgQzv#-5VLb0YlJ8#Be}4gPdt&@_ z<)N8kk05~`GwPk0eMOY=iz3%xn2wc%NDuvv8H8;n{}qSvaKZu?(w!inDl3c*dzVZ$ z&Q;?2qB5BU!Fak@2H&z`Q>S;M6HPcjMoK;;QSegr{3RtNuyiuLBnXve&1z%dV+$Z~1@!RZB?7z*=K`HnMe{`{WN!?G5_gu7wzf6Sq-COvqAaABS4fMc(oq|WgqHmC61lP*M zv+Mn)JLjXi*#9EsmcVdO*@cZHa`8>gh+Yc5gd0X48!(51+AP_{U^ufD!VXCh#NLka ztX^2(WZkQT_in`gf3L{Sfq;kW*QNhk*_C-+0_~Q+!yMKjm{$w?0RWsQa`&9y36@~} z>!|6%{*8cM2H4?QXHNf)rr5uKOLoFX^@}7FsQkCEAz-3$*^5bCOloLjrknWkSC=_^ z>^um6U?L@g=13btpBZ-)?2KU|dwd?ze}C{)$0XND<%XLAbe~`4mO}(9v-TXc`<$oom+G(Vl_A zBS@>al1`3qNU+;}Lo0jrXI2;hxbVAJAmA7s3^u>LX+uoS5}vY5NL$?8*k$+SFaNsJ zPR(d~x!k`;nzB-G6FCD-=SeQ0YjdWH007m2xE)XsSK9d_F+S@U=s~CPH+Iu7Ilz;v51yJHo%K|;T6PpDh0(4v&xXbRreGU?T zLI05bTqia#xwFd)sker^e8ns~Sp1lvU?u@I8R1HN@JgJ{$SWRQCd}HUCT3==M3g7l zX}>Vo6%~u#Wk0X=0G4&;nf_zKmKa>6l6}L}3(bwjzo=^*vbd43{GAeEeZZh`n<8_k z=)DRhHk7Yvyc_kOcG#Of^aGPNN|9@?VZE6hRd}a!&8O-%=TdgMSp{QY1OT2secB!H zG3#12s-$iPM3;izijvO%dIBy5kz5Us%S}>pfCEm_PI={@BEY4A$Nldl=h`tOmBwI*JmXAR4TG_-^K*HY3;*0f$p1X5d;*Q1 zJl&%~OT#T$o2ZCXfrponw?hXLGv&_t#VU_n$-gCw6K*p>c)?5{^}iB58u4Fi1N}u( zxi6h-iqTD^!jK`{b2He2hv4DfIFZH|%nv^5Z_M37{5P)J50szU&2jg5WFDzop7;a{y+8vEv zitOumMkd)ClRH8kc6L?uv{x_A`KZRI0u#=vy!?qevPiraXxawhG8eqN8-KQKp#e#xx7qOR#=gskg zr=e}HxY5UfiDq<`S5Le6{Tk#-wM8~BiaX@UO)ggw$L3ttFG6W2RIng2Qoy1Vb-0ur z*Kt}fw5iM3^g!5l9i%ToS)p4B5YDd!3^Qg~i@Td2KCMB%4|*;-J;fIdvMn5UVlGaY z`ES|?%q+JMxRq>a)tEYE^4g?C3aFhL2PfwVr;~}j-%YD2KhNmwIIb(m`Fa40;wd6} z{<>*J%ajH7b!f&a?Xt1kIv)VTzg=+%7ChpYP?S*|8Ct&WxSVO9c;dYo{-Z!89DXl{ z`+9kBIdEjEwo)u%8xGfxj@682QfK=z>xbUXIpVYBk5}Hw^98!|&BmO5eq0;;exYu} zSrdB8oLBY2z0@~=ILZI_$U8d&?f7DN-d(x9?~hU$N?&Z|o4<6>WDaQ(TarIj^tWx( znmi9S6sybm`*~7~)GFT0`46d@3NUoTyGAP9~<^rkVX~xi68=15{9~p4jbn^S+%n&2^qPGZ#`s{=Pjzd zTz9I!d==BUav5j>jrx1m>ZoujD)YcvV-lGrm@iMJGGCtRonlpz4=GA;3))?3Qi8?r z=5kb#PzAIT`2?cJFZOZt@5F^ZuV7tjT@m9}xcfU-TDs^AR(``GN7lyhk>KxLCEf=r z(OB;hnqMooJmz?x{Grf*eE%g#0O}LNGZqfLtC0TgUa~X-MQa#!dxEkIt6i?{cpphm zM{^&ohXDyE=M^6rJxi%ME6Du%wZ|}HlpKJE$4$D4{h!lvZ3HAz^f>6DEK-fOIUAc@ z2eUFUa+jRF&Bhb58_G?^u5YjIJUmK%%h4iVI=*Fv%4QP6)P9-cQ0?637e1|G(O0V4 zHvk|4?cP;rmE8GcSv7oinvic3|P+5A|>}ik8U)c5_o0Thzw$ph7N{`g0*VC zsN`Mgf>$2SkBcXaCAugPg>rF^TLdPz&oO zS=EZ3nQEjJ1YVLnVGQSH3Y!BtqAWY>(`mfYhz`1($>C=nk-Wx6DPCo|=+m_V>q6Dd z9673`j=piFNOr^)XRBS;NHN;y=LD~D_T$r((+gkbN7W@Cciuby#GO4rjKT-ahr(z+qSEHIyWRZNP~98V&e@MS zZ)rYxA_CP2{*`jcE#L|nqvqTGa>XldhXDYLhEc{o;EoNM+a7TtE{JqczM5xSIb;>CN3Aj;aEh&;D)%8f^|+Y<#GlX7;f{K&+==k( zjJLi;9=LzV7uhEhy;v2W6;Q5fy=-z<2~`4{sy*!|h;FDAy{7fwW9*Ui_?j+829RnU zy+tJf<71+|3e50U=F{^FS4CIoy-&b8rv>w=w(TJ9A2S4ZEH5Abo1a+%TBOgY$hCMJ zfwo{$C^woq5G+*4F^c(>?3#}e{ion# zAY|%UezI7HPAx2{-wG_Xs*3*v4ZS-IH!W{O_KGCs$`fI24db56#fK7&+?}8g&a)1C znwazLz>3XoOHcN1lj47*)9d|?|J|#Ki|2o%vDdHmN&cwbeYWuWBeG|?mejglUugaI z@ES-!&BKyW;G_}wldxRLWC?`AD|6jR#C#tSF=V`p&R(F^kP*>9)36T)9E5}6Dfz`{ z0TZtZJm675sh-jI<&&>6{TBa!xU5fQO4YiBxocsM(2#XmyH5Ojw(ZWVa zSadt%2<@@FbtzY=87V0%O9NM|tv6HJ zM2Trj%k$j#qW1_q{+hV_PvWM(2O|n@Xdzy6j1gxZrtMVR6gCbT@LSE!6@9iYHJfB^ z?6#*&xoK^esaU4N=C8Isd4)Ksy7?8|s>sBTyLwUErw=1Zlsir)CtoC-=}o3tmg72x zGNWo16p}?Fmv`D=!Y#v?6k>j}po>2l$P2-KrI(QYo_}+=U^C*uAzjh#eozag;t0mx zjD>>DPi@@FR-owyUI6}6iH%pOT(OuGo=qMqa|+nR*A5r8#IP&A+hEFLg44nFAT|ul zr+hFi1KJ-Q0*dQm-7M~KxlNoaT4P3Rer~qrph{d6DiWWeHssNd|I>ZPL{R0mw{j@c zl|LR*S^05mF1ae-vJ{|#I^f}(7adtDG7-n~Os^Q@J6Xm%7TpH&ufN{(yHM2NqXYMO zhv3VvrPKDUh2Q9Xcjw*&icwko^;AaMkQ$|KNbS~XcHWR7E^)!LVbN>uNZA4KOap!) zp1rRk-%E*(`9V9bL{6Dx9ij5}qrJIs5CIW6HUXmSw*V0m0U5Lqfg^2@zb~2v70Tx3 z29imZ=_!Upoidj~vU<;z=~;EDGe!EGmL>%*bJbCd!A|mmBfPTO5a~T@h_6(9YFL?T zPrTEWI3B~+H9<`OVS-quhrKjHUEjXke#`_UgH+oRgMyfYwN*4V>*_nu;1j22bZ+sB zJ;PR~?=Q-Qr9mz8BlQ#jxv6s~)6o1Y!X^lATj9YzRY&k) z;0hBf1h~Nehqm($OS=EtKDMkZZQD?3s%1%KxrN46=4EMSxs}S&GIQoaMrLN^pwiq* zrD^5By#>vboMaB%sEDM9h=|C3zFybyd+z7?^SSTi`J?_yI^+wV&wHHb>rAgmx@8)} z4K2!NHdsXto?>#0MXtHPZ!{m0YKI6Ab`9z2?l)oQZ+^iH3Zv3Pdt5d0W#?7DN=yKEQv7M!92tbcnS

|IGx zr$4cla@c1yE=-^H^7q;4Z7%V?M}8LC1y0#h((` zrv;`&jr~*%RY|TBDhbLtO#Uy66aRyne9vt6@dfOsu!sE{S!nP4c znw$xLUTuDQ_l4`{n8{vo89*J%ZL!!08WSRZXmJ#u1K z?oxDu8o}iRUDh>CMgF?X9fQ zl9X?ZbUsSI??I;LcB$@0zHT;{?E6LO-EYI*2I#Xr1in?w ztVw!oFFO`e$h!%w#x|DM)K%E%NyfPed$^&D-k|RELaAOx@^I84W>9Zl=j@f8Wy&^c$80`{r`GOKD?itYKMwgr>|SLvbas0}z>(IKlJeMpq}jR(PKyil=R+D;O#ok zZv2RE+x$H6ioJ0)(#P~(LOUKdDpW23e~Da(7OBhAehN)<+N6D@!qgh+mD?IOncWz& zM>gB(1RK{)!l?yePD_@#s1GQ`_{SAg0Q5X+qP$jVmsKT1!&2uc7|?)4rv(xpb$*%7 z*?@-q*c`qfu5KHri?&=VPz#+c0?^UG#nKRfg(=P^?peB(Ko?)7LXxMAqVCJ?-Y0Ua zz6BekbWD{7A``CR-ABb!dX|nEq=7`O5Er1%(}_x6q{n^EBRNwOskYKN+|XusZp~jB z=})#j1@`c5zH%6Xc;qyO?eHjnEogZ{`LUEr95S{RdtRbL@MK_tdm!WQ2R2v#d&S}Z zT~j7q`Y%twmWdYbLLw3H&;l08RRE4d{$HzR#n4(161Y?W4j+(Q-+=nJXgmelf6W?n zuFPTWO1k^7-kCEyjA7e>Eh=e2qJ;o{ParrI0}Ss#pcRSQ%0l!*_w<@2?9eGRhDcSK zRfhou8laJy1HyjBs$uMSJo1l=GcbWh7IkrH#E4(ZC}X-WgLInCaaLoX zovaKmyu3xCZKhXVj3C7$%D?+oNMemvfik#hKs`6Q&;VF(0EhY}0KtJlZUlYk^+<-Y zQq=HPM2qrl5U8;+vxgf9Z@A-(r-U#6fU5-q6*}AAeT54B?K8MhEqHEMsw1I%sB=QA zX3m3w6FC7@u(Oc=rKyOhXR%6s1oG0kXP0Kx(R& zfIxUm2lFbNoNQi`srm=s-$oL|e`5|^6`H@y$ropvu%xCJLO1{fIzxJ1W8QoiU2ya2 zsAmnYuYuR!6xOyGt8`;-Y9n~%GjVq_4L$s9Er4oc=9MUdKJb374s)KArVU;4l2yXg z|Lu9~PN{)cWT0}#;qrfcO)E-9EcnZrWZH69HzzoIh)I|1Z%a6_8kB~S0wc|TJ1xQr z`(!1O@T$m#zq*zA&CMb5k3f^nofk=JsTJ7|PKunH9~-@;=MI!9wC!E0Y2s~dQs9>_ zP=g~(ez~2~o6yj%)LtDIHEUPI%UWI_%`PW{`-^0npxYKfP{4iBKS!+4^`vNcqVX`& zs3=bs1Q_NB*xD3I;$V?((_&xtMR&=%fJd^s6?*amDsAmVuHf&Rl-V2u2PexL0McDh z6VFZmVn=*DP*`8MQ^DM21*Tb>Np=gV73Z;|dB4Y3httn4i55C^ixxd}VfIlu_{atX z@pm<`!7#}kA%42kXVx7G90_O0-iWb%1B$wEVIi5KSaP7sYr_U+I(O2DxCE@T<)zHS zf&H%)6dz4B2;IFC-Iomi?DZV>sR9)kllpQV2aJ8xwJ?TuvmmYdseWPG^}HC4HNZM5P~t$({qKjU^> zRKQG>@rI$0SXns_gG~ztxSeuZcVAC$Zd((I=yt!TX!wM3*0vuqtY|&}(;qzaORb-OX9l}z@l4$&m8BiaTT?uWN5Y})-63$4{X0XI0|PX9s|<< zaJW0t9&D9&9<^u@;Uojp6xRt!&Lf$7bWa<87Zsm#xnCNDK_q+*WsG0|>>FR6T7=eZ z4z{k_mDG~kwt&j*ZnHf}po_>dmnCc>$tnalcU2V7Mse1@Kc{SEbua(SFXGPTL(baE z{P%N2{sB-upb%LHq>z@w0BE%HJf_+I#U# zFj8^O<>fE;U{$sLp@r1Vj8{nN7r63;m%vx~yM$Ei5)^v0tplBsl$6w0K%9JM4oFy3 z{5OHdK1Ql-%-}DMu>CMBc9W8isD-kpQo=$PV<->)iAVP=?YjJV`32BR7tKf+uNOlF ze>RB?;T82%dVmL7(rLsJtp%h%dq87TsopD&$?4Ax+&TDrT3*KJ&!0a_14+6={YW~6 zuvGR?((d<~!MZ*|{s3H0=n=Ce%PKU@tjv`9dTp7 z1L!H1$OLdYN<^9o=qXoQuoE=)P78=d5tG+4QZw)a%X2)^SJDaSyD9tm3~lWFsx7AP z_rr-q8j#@}5H!GfZf#j6J`za%eoc23sCF^}=(}dW(|~8C29L5XqK^*UAA2wamcqQUEZ(w1KO-ae_ok;I2+n}6d#hY z&`nl0iDLw0Kty3yH;Xam!Y2lMnoiS8a=-_!CPqdh4l-U<-sP@Rok@jBcrKv&4!vPd z1SfkR2t<^O^ae^Yi0h=>do8#*Yha=5bscu82XhIggkxJK6YhtUq)j)>4Lup*tt7WB z0y2I1S`YY2t_Vt->P=R%eXN#61yuJbrI6au)6Md=>B5KF7$D`YMx3pp_31hq@GJ1b zOjrxqy&fF2kxa~h!Q!*ynLA9cB)MQ0j!*e#Xzw7d)WtujNl6^So#s0gqj$|g^(Il^ z)1rteGaR}XV1IZ?_ejcLO27R(R=rCH@QMJwxt7}rdfEmds?;y=!l7&+@aED()xQ#% z8c0>w5Y;#sTXY^d5(f)f36ruTU?;cZme+u81GY%TQR`-^Ey2a5!Y)H2L~PuX@?D&S z@Z$BlmkZx^xX0|Y@qYguZP~O--bGjosE4LpP(>yu&)}2>k=v%Q{Y7mdWr~zLNbz+e zDUmG)YUcVvE9AJYteB7ld=<7|G^Z6T9daMw;FfL+of5m8?~$tOa{8I)bf1@bmptM+ z#s$`&q!i^%59F$2t^PKRliq7bz*BQ_`n4as1#9Qh!dUrfOt>f0t1vb{r%9i}UKS>| zs~LyVfFe22K5A!JAr*l-4zKur@$09J!VdD{05P7lE;n?$&Ce#d_#o&u-x8G&G`|i` z+Uf6-%6s{u<+FTF!T{TVpmf5CU+UzgIXB-wV^)>n@wk9e&6te?0(s_8K;r$tyry?4 zBG)Yt4&(0HMCV}Ag#MuVt@zNhxkL2eZHpK`_-kdd3C)3n&(; zLYI5=w9&WGbU+Y#rjGoY0!+)yD?W?>)#P~KueK!c%-feQRw2_90FW-vBE~*}G_ptg zN!z+LSS8L6lh4cw?oYod7xE6!B}2*l35do|b7JPsVOY1obL~1%#=UC6CjKRzVYo2; zNIAOtWu9tD5_2}`j-~o*q;tS~kHcY`DtmKkJ42cIkHubGuPJn36ZuHL4XX-c`O?}13mAWQGsve;UQ*h!xS^HsIS(o_!+kxrWowz7`a8t$^ z^JzWXBsJE}3cV4W_Ui$jVpH5rB-wqd<7MNQcml+!kg7ZHt)>Ixm$|x!88xaY|&88-~y#S#I1|9igDS zPTQIcCDm!wdyDtb^C2VjGTTh~i&QQ(PJ0yy2F=kJ%KR4j%66Xm8*@{Di)} zByjg=Zf=)8((M4FWTbXWq|6ixoXlbsZm8vQgOVHwQ_CawL1z3lVfmPf?0(Du=0<@$ z`AgE?v!Ub;OyydT&nyKCbi8v>xMF=LLj$b^*GqsZ$8VF1wCEvksasdfsYEH|mRnX( z-HlGGIgAJ6gOSfT@rpyD?_5X;dci*`4jwq}9ga|&wJ!Q3t@@7^;2vTWrZczwD2Wl* zaQgOC;?nh1bCDmwXAM21zl%R(lO(6lG!KPZeqGwVm}@L1bOTW`F7JXYV=OON##a2z z`T1vqs&XO~8k=!ULcA^Q1Gw4e6cgC~g2kR>%+?hL(*T*zRw;LM!_ntp9Q_qUCb(b-D_a2GYm(PxN|#(kqF^XpckRP#3KOq2aoZmXME z$F_}x;yY}`OBGWT*_0kp4&(%w&_M|p2AQ=6ZpvWp8z%AtLg!=}pF`GHC>X;dRqis? zpyF{Q;E=Syt5AROrNAqSO+MX9|Gy^=|GlNw|7oQIRO1-}E@L%lC0!jzeK45X4XR&B zR|f<|t3jVMf%tC3%vS(J6*RmRR0yd1K`R`BH(o16#1CIjMdXnusRIt5;Hk!b~QY-f{(Gj2fu$$@K4f7Z2Uk zFq1Q9&a46HFZ?_t2YQ&Yl0B#mx*Z&>Uzppv6raNkOy4rWkzN|Wak%$k_dPs32DxN;uU@%lkQv?#tf`%1Al}j&DXQcr9 z*4qDa2XQW0?Qi=6hY|7k28v!Dc9&!<>n_jAGzL+y@6W(qe9%{HN?sC=(E+@R<;()m zQ$4j{08n4$=-IT8O31q*$r{! zrNl@bKH4uBn@2?v{jLA0do?VcjsT@`@O-UxY$^g;YivQR&B%O$+Kd2V z=XM>(TtsPEVwuO^4Agd=q*Y?;TdQnHXb7rn4)nCjb_lVphcW1D7TMyTp$W**uli-M ztW80nUFZowCl7z8i*rXVTZN*hw~#el1I4o0x}YRC+kh!i#{B1RGc#Fpmkpj?zr26L zhUERvVqWR$o`3rIq^?sm$JoDN{8_`DP32(MW&vk+j@&-lsjjDMpB3HlZ2s#&e8i0k z2m6tChPQeXEh;g2LFvCoox!~`+C3vPd_MVW+|sCfWEsUFl!XuENcgf6Rasl^Y-Q(! zEi(s3?o{aWc4O5>wZ-N}Mmy50CmYnwb`y1`zN=6UDf9(}X84}AUQ*@zrU#Hpd&#E` zHB+E%GY16USl-{;ivpbrg|*^Wmg_)bAIc#Ae7}gfb%_ydd~F#2Bj857X*poR4}Oqf zaM=b-hDz>%>bn%m)sXDP9n223JR$7^Mre4}L3b{2{z*Qb!%t+RKj%4LYDHm-B! zZ-d!qea&r~m;r5c+-c7I??h`tlFA;)4rBu}ebqDO7?Cqh1hh)Sxh~s`r^41J^JVI@ z2SS&R&J0pSD0HECTLQ#au(6FFPObdjtViy;gE_iul|l3!wO0HvpnE{ih!rylSRVsa_NI*>fkAnno|D;VayOYWH2*h-yluPEdqfs^8`n%Zp4KG zjFnZAkFRnWl6b^(BAZAlOmy~tnFN^w&r|c~5&Vr07tLqhqhXi+@UMK59N4onqZ#|| zIr_CpXur}xQR_3_MM+=x*Ch5Sl4^%Ia{W$r_&tp8C`7oncj5tfGxN}B#h}Xp5w$zS zM|AN#Dx}K6SaCY&!C+GfUang!`(}h zkrccLRyWGHESX1FD1RZ%k%GIl;A=p^16u8|;@`?0N+J?b<_@H&l#hC|t`Yz06y%ao zHc_IwJ+>)!GU*Q(Y8z<&nX(CRt~Oj^3h;XUb)f+RD67JTv08BI0SrP+b)?Xo#aKJ&V4o^4M? zsYBT@N^jY|sGmIMRz9n8bVibUoYWu&NHnI$Vw{t0dRD(^2&mv!p^B0dVe?@C)fy_9 zwv-nVWf?EYG!CKyp9r&=#A6aMmbIPiJU9`Wq=&DZcumSuI|eKsx|(Q|VN69Rf9?p@ zi!Cr_lSOE1=v>y-aZKel@77Ss;UbT zjV<5(6|(~Id3o`fO>S9M#w6le#?bF3fN>7+n@m9UVzDu3%!C~R*iLJx60i9Tra5{} zNG3~$$p8%?-x0`0{B{fgfnZ6r{TNE76P4?i{qWo69j3-b>Ekr@a-#_bGEG<+MY0wX za3XH;qv!pYVd@(nm&WJ5w|ohY#A;~}d5J+POtj?ES0>og__0MUmf<%l-U$g29tjkO z0nG7H*-$qyt0$m6a0?0eBA+0|C% zdYdn!wE`&vGC?|19F?#3CGm;jQe6&=a|Tpz-DyPWv@`%vql+(qsZJDx(7$k}$}pED zx-21lPRTkR`8o0m^?WnJop8H~n^9)ng!n?T!JREW=jEmiQ)soX4u9vnwN(D}%-Q?- z!16`^_))Er?i*PjrOC<0`K_1zvs8|Qp&bXzuW2X5W1+B8NBjNQNXc3#bt5DBo8lyI z_qceVJ+rIId<2gc0Sd}G{wN@Gm{Tut#n&x40F_IH*E-Pb=wq%Yi>u6_J_Gkp>Y1zs z^A3u_v3JopppMA-mfTDyag)v&x}vyFw%tfW4q-UqjyV=36a9LOjyMzLJKrj zxjojF{;vwH2wN%Zl0*@dXp?ckojUAGQU&djyi&{gBfSjcqxMMCm~7w=D{ev|C+CF4 z{#Zxx5OGD_PET}n`mypsTcKd#WF|z7*c^|E)t}fV{T%C>Js(b{`l?1M+wz{%89)1u#KhJeWr7x0?wW6EC8R0PipAt3Y_Hr`NcLWr1b?6`Z}qy^8}K}lf2p0 z{DJ7ZVhtfs73b(mq7uK)aH;5_&Y&b&2>Jx=gvTRl@0?esec`=!h5T>_nZI%8le{#7 zqM-)|IT#x~(?CmkZ!R3rF@orb>V?zzVsed?3m``_B!HW|#8_Lig5h@RVk2cW5Q5kP zw$_|e0&T;FPw0R1FGxMD{U?Ep`98@1oalG^n$`7_Vb4}SeobKetlxPo9D+dGnz4)gife1GCP(<$|*7C25@j=n%`#1`w?QfWTyirG=1T%df zQa0(_c?!DO*_Y-gj8=<%BC^%{K zt$j4B8Dmc`_5n$cSLsL8YV3yQAp@)xIB5b2CBFdspSyZsH+A<5i6`#W693>GsA_iO za<+Q?*Sj!ge$d*t^b+()N}05>yz@(foy;ZCyOg`okyb8?@M|TCcBoN*Ji$-B721Xz zz@}5&(zb4~wvfV2 zx-Pa;x!INm2}0edV($Dq=Z^Xx@B7WcwR^iFPsJv4osvt{OL6_+kT*VPa*ii~Fe!hc z6NjC?;qTiXl-Ow_DUr)sosDMMho1FA%bKu6Cm^$|S~ za}_qMgYyTrk|E?!MbMa%OoU6pOsvM3OP-`i4lagP8++5c&op|fh{xFy;GeN&)9f#L z7|hgLH;jY-BI;iH#a3DM?*+)5a$JUY>>+|BPwZ%mkq6DgM;8L)|K@K|St1q30 zT`1T18>Tdw3Zt+r)*>!r?)+ptR`c$HoSuNL=2J@@tQCueRKYKBUFpfDX?@LyvhjqnrX`smJh8hg3btjh_H%!;u^7$g#Q*PT@)Dw_6vEA zRAJu{p7qMU88gMcL^S_Hc`5NJl68-zqmtVv$JzbKR1{O1+yE-Hfqoy9C!dK@%%79X(Hfm?@LjdUG^fm8QD}s(m=Cr7a!6a9yF(|Gm^Yuk9vHL1u!_e1z z6eAn_i|Qt5o>jmw1DobH+=U%OuLiq6^!m6OI&%za>qlS}IJ_LX9bP8 z3U)Pq7&d4#xu6O2K>vJjkAB5)Dp8J_`}*{k$aw0hyc6l{;ow7{vDSTFCs9Wfy8R_I zUnUmFMjtGd1I@Qb;ee-E(mcalreWnl#`N~?aDKIk(XKoQgmiPL_3gcf^XX>4mtJg& zZR(#n#9o>`J(L?r+`>yceCxr>EfXipz}s6PuZe#5{7x{RT-C;!hSfg7gyi;qtkc^x zdCXAVQXW;4;Rn64j2!(;9icy@6nIk!Ks0DbTSC1>9jm+p^fXm(NC4~}6hXnaMXM7m z`!@21j+a7?7RYs4LzR^53r=6tq_Ao))(x=+elVDK9$41&uwz(tx=hB8xW_Jt!L`bO zum#=Kq-$w1opxKgcgcX9V>4&Webrl_@wwNsbzz5C4k(5WV;Bq5fW2eGBGpZ7ZJ`>_V-;`o5e$hEuU*);@;RJ%)7Z0X@^Q(} zSg42V2lg4nCRWxv;V(c;`?Y#ysRDe7V6P<4@u4G3VA~_iC2j=$@iA$kV4Yw=6T#;& zowO}?WuNin;Nhn*Vq(ZSVm8Tuz=`o*1F9Wc3jtE*BrI=vkY|74ofiJYa;sbV5!jhN zi8cJy-tNxnlLZaK4S#M0O62+@cw^P7lyMv15y+j|zfk2(_J{S*BVM!R0S4_SEuQgW zzLB<_%!G%$SH*~20fN*f_<+p-Vn_C})JG9i82EtcD@buXPP82)qzhevQ|nKk4dF~t zAd8Z~E@&_C{#L0S@}}{M`))8T(!-OdPrDO5r_1**P!@*t74ZnQ-aPyAfmnw4`fX|_ zR{+(SIR&0Hw>>wDr?z47kFVv)$7w0;99Hi0kf&vzEfyN4aBTRy&OTemAe=eANq_9P zmyjZ3$jgCndsmRY1HVlJd5$V4}>vmte8;79^KpTZLx_L4EhIvEO$fow}4J znUcqFSBs~0Llo9g zblBVcHq4)jQPs^(&`!1{SbbXV1C2X*;P>O=(w%91zhT zB#~n}Sf7`$Tk}!2iQ0TS)Cm?i`1t&i!d12~+}qz6Qb^KJp^RFcc7Yf`OHe8xD<_6a zCA8ZWaes$WiegB*#E);xOBTt2Y>6zugqQBDMQQQxqf%z2rsn0#r@M9P{C`Pp;~q`x zqDeWyGS2Lh@+tVn?~^sgqxGxsbbq9pbsG{>l!XjQ%2qX@`bE59^ibbH!Tk+Pm6u=5 zHI+RJOugaN#%7vSt810;6FPTpCqJ2A-<|vf1IIbWdfbqdz;dER&q&xIUYZ|%3Ta#7 zoKkt;_uS!tUF|X#qN(bAUj!K)J8%(B5VxK84;{Xj$Qj>M(MZMoY?hQPkH$(BEkX-0 zj%c++9sZj#5z$5Jokq2_Vt)x|IHzr3YXZ#htB=xCLeA!d0~ zO%SF{@n{uKl3_U^d_}AYDJb-nQ~^;FlND%*$>PPnCy`gJT@u3Vdq36YzFF(nU(%v! z>Y!Rvh^}jSwcXPAt?=#i0qc@JuscQV66)s^uQ%70TdTT=3#w(Q_7R(VU(Xznci|W? z%N%a_2AOD?DV`ubRAIymE0gDInaP-=1yfK{>2LfTlB+Mqzas+&N zEvVi{e+?+>;YQHU;^V>Wi(nxD-&$yc$b-b~WHBHeF(3o5o7*RMuRsAU-eeGQjF30z zcZWT80H^o`Ao%qDoIgM@etII`e?nI{2RurhR*Vlcv=;1=S6Y2>I( z%9j_!Ud}wyMnl9gt>apmLPuwc$6vlbCTLBpEigW(Rt1VqD6w7Q865aQpwW)ej8lZc zYFI$zu>q|xUnQLrj3EvI0q%X5?(l1-Dt-p5#w)Xi7b0@VNdIhqadsvkc@fWZK_HlL zf-z*=&Sw*4=Rx&)1ny%{y)99A3n1T600d_xb1qa4bjfSF%&}+tvNOGZ5rk{9hEiHf zZWF+Xm#!-vB?f&g6qya`)_p2xI)Z~ipu+n@x}f@<6enU9+A48L)wwun)gkJShRQ80{&|-}MR5KL?Oj*kF9rdZh-i(TV z!RA!&4YMuJxXu^bCcT~=Pcq%u3dH`I7@WV3mSud}3o-C6)t61~gip%vN>i%8S7Wv} z4Tl^CB*y+^6QJ^*a32zWA8^O1w(DFRqy#+XUc`^wF<{m9D)63KVLAypBU7kU?6E%c z){ufOu)gdmYm9rn=;Zp@SGp|Wv*~!Fpaq`dGbQBK*cp90v-0s-e_$!jtuBJ8 z2MKNf^qUx-EoSZQE{MU=ImnW&E>nc;WQ(fsaX&-`DZFo*qb+2UP7WaZIS3ltM_mR0 z-H*LP_dRWr9MsX#F`o+ha!RL=m+u^{G+pdqMmx=H5>b>7WdwfRKk&AzQwaSw7|-YLA!#gfv-nM zrub5g@*@gBP0=#5pZJ=WG}SF~18a;M=ImFIw`Ls;guyXO0gAa5A|Qp<8D4SUD3Rcj zwpLO>jfQ#an&aQoNB4gdMZWz`KZ7M?kbURpa0nn)6)PR5>E^i^s9j!U2iN(ftGGH^ zTGkruQBTYXIio%Cs$CRO*WYT?K(BV)aN;u2-(|P;=rw=mIyZwC$HBDSh}CgHLr@vE zmMnLzs?hx&lRpyMVQs(wWBtud3Sy_-%@98OV*T*c3eLFilaTG02FV6bT-xe`8{D@q z8tPINn-eTFN)S#g)dMB2$)^iGUTIwO<4#L1#6S%+FHgCS*}WW0ul#AS&<{E35y5p! zXny_YH)$EIgZyf`qvreKSJQ);($m1p@N;BLC1BCIHzlJO6k0$nS!nOXB+xP27~#Zn zv`cNQK%%NJf4p%WnMY0R=p7Nob&sA z8T|d^uyeM!SIoV;bq%V70Cj-m8INYG&J4u)nVaX<1H+q-)~e~MHaN;PUDLsEO8U+`kmo-6t2rs`+*7m{O|UTj+^!QSbRR8m_}dZy#3udkBE}M*W!XO z38Ida<#Sk#=n1(xmkCMD1ZW6CPMl>#(UNKV-$j|V#jfQ|^y6n5U55d(&YV0r35l4C z%1{6;V`RjN)gb9SU$#GqUyBJA1d>@A}O!?K1%=~Q7|J+Aj*#~N-vmilS;TEE9(lF1g!a>=GdW6@neZf{Jx0IJ_^32Pe7 zq2^|(4Nlv^G>cDlZ|3ulq>-8x0*Q$O#=MV26foj^!Q6wfE>nCZj1}Gb{Zr=qh1hfH zkN%H__u(ZakG>;Jw@sIMj{OjZ$p{zyO^hnnUFs$q5F|lIQ+%)YhK(M6zGc;0XA>7c zXxk_1Sx@m!7wtcHOmW*FAY)vZ;St zHHsh z>Y`EDKeK@Qmf!yODF1(Q^{>nVmj3rFKp$YzZRpJX20Zp(N)q5=PiG@FKx1`m6a?t_ zH-gS3+;>Z2xTRo$!9aca;~}6K*Rv@*N@RP)gTWB5mR10$P~KBU6?|_X_D5HFBVe5yDQ8Pj3Ka6=2&H z0{jr39(6q^35ToT|A~|%4t>6w@SLuB`C>xJzMq+Ly!8~wf+8;=0?J#T%MO`}%UVkJ zuVG1J*-pTlHu&CFo}Qdhj1)@HAFx-8R67lsfMsNGV8t~|Hf)>Y=c%~EI6#tW`xC%e zAe;^?sNVw2fpXpZdiJ5vKd*ioc@o~j-%4i$=uv^UoPaU{jSVvKdInBOCc!3T<#ni5 z|2Wwmx+f8~X|~hqydb@j@V6vEKuLql{FZ7OKn~Jtf477SvOYOGB;0!AmGJ^``dDbe z828KZI^gsGkb5`g;K6fM>I_TNvlf{(?g4kQsNCg^o=4yrJeDP@VQo^9y}4^_cLSzr zk70R51szZC`u@z>U2>o##JfF2m;XwG_v$_KK#%E+U%Ts`nL|bC`*qH}Y#g=xHC0KZ zCpbRyonwB!bq}KwH0+!0Zklz%W}Y&t#$dE%ncU;m5-y81B2(%h58@fVx-UB!Qt z`FsQ{PlqXvUUfX1TN(eByP{VliNTyVa$Etv9<6t|s&@B>0%;M3@XOL_=HX&McE z6$2r2k8rU7&su`y&y4^e8lq?fw2S`=sO#CQ3jb)tbO#YVkP@pLR~{jS>2j9R5=eJb z%6EHy;Zgn5SPSgtX2i3mnAqZgz~r{OM{~W$lVKY{N!6_rW~VA5A+f=o@;UCam8lfw z*{k&vS5AfI0H(=XcR1S}gB~d*`I3Z9`bGLZR&AX%(cJzGS~7(9X!z`lxr#hc-6tKPV$rX6nw0IwktcRYA$lP&#ZgNgYBCDZAOmHJT`QKI!EKL)gr_%?)C8F z;ayvu67vV0AasO|Vf;m|3x4(`D^?G*eWUV?S|LfCn}1B{m^z1tDRAfQA?bDmd{t{| z^o@7dvz652!w+wk|K1+0p#L0mb6ouHRBXDh+#Y$0cabN~z=}m1C_D&48f9Rix5jUG z{Ax|RjIX!E7NU$3vgJEkYQpmvhcAzva?-utcz<7x%<6n5{I*;-aNdhh)K=mG@w9Ah zh^Akqh1I*ZtvwidD`!Mu@L|%;29ZH(P?|O5Y2N`Q_t;iX*~S;MX3_GOT0-6+u655p zPE|g&_b^f_A@A}&t2LkHeBXiN|_M4W>kRcPk;OZUb5gPuZ=;}`%?zSHIo)11%bh*kVrcvU5sCl#c<2Y+bNJ~_|cXPv!X~~xkK{t`a5gU8vyy}Ks z=fw!Jdm7aPNKKN@CV12vm|ed zJ}C?yDuTr}df`zKE)c2HEPT7jR2-`7!uGA5a^ZhfuCN{~O2aQa_vNI?zX%_HCmHU> zj4;uF1OO(TfXoVsgFckJywq*KJ;Aw_ruxuSG090Be9czhcPiVadHxsL&D92 z4I)Cp98pAL#Py9U*-oIYbr7y*;9cqre^u0~O$b+#GnYuRtwWh@5{nENgZo~zR?Lqk z8!%KrQ5;WYCm*?|O;ld$V(8sC7VTJJ;{;!wB_GRIfC!s-Z3e(ks~KtTz4kx3+yC>k zcIOw_-v4L|ffnlJ>iuT##*(s+f<6=gSN0>g2v`&Y-5o&K=uRD71L`kwHJwK$+;0d) zBj+1{&ir2&hd)j<{EmFWmQd+*>(*L+#1_*6@i9Ou@Q>(J}Bp$pC0H^bMpydFLfL(1S7u^O;jpG4O(bsZj((b&a7C;h{^j_BXt_ z40Ti_dgwsgI9X5`MZRCLzGxLKk?UbBJ4*{r7St3({V`VIhj0x^ zk9KA!hM}?VXwXLy1Hw}+-xqW%n1$k|{;AMW7%M5JfpZBq-&0uoxO-o@=fEmZ<#>Xl z97u1Zb`Qvheimrs3M6wepr`3IOV^c(_{SDmb~nfmlGBbcx3Z43)|8`ciCM;Nj>k15 z`gw`PD%PX)o<|R`F<%&^h$WTYx6beBTprVF{cI%!DR~rDH-)fn5&DOuPy$y%S3T)1 zEF%kycsmFL^qiol1Jg%A^;Pca+b6lDIWAlJqga7^OEGbnO!ASBP*Q?ZJhQE1`>kJH zm)A4G%!Cuc#x6J5ZOlSq&M5ZV!8EEJdcG$29q;F2VrtGdK2Z>WDOk_5gAf@E$p~|( z8!HKsf1SfdoT*g{u6rqrp;)7yB7EQ76D_J1WfXm}jo0FijK*4-h=LfUmbE4+OV={d zG11zNaz4+d$oQ6P&*P&*l<|pINwO@0Okha%K zUO&jNaI})Sp^k+3%$*6W#)h}fz`WhNeyB}ZC-y>PFb96Pk}zX5bU*kx%|wA$A5FpH zoPi{nUB<=fTs;g*K5>$@l#sJdioizs(%f$PU1htuf?D(8BUFI5A} zX0g2ypDVd7%7A7DDKbA(n@9Ln9IWUg*_zwOh0+|R*riBq^d8XdAHmo_s$Y8&=Yp5u z8ldLyzw2L7@0SyZF=5pimIQqFFn&BeISQ_;vyWCE4#P}(DrE!}db7sydTs8;OW)=% ze-D1M2dcYt|2$jI2V*EQpxs1B&qevxif1z`(S}pP2T{z3?)7b3`P@1iLFsJ8qJC>0 z+I*?Rr6V@}uZ66;;f1N>o<3|;dtakGT^sr{!MkIfYVo((CNIO!e`9uM-fQ=Lz)QWJ zcKepm8&j_RB=+c0%#GNb*Q#l>HQ^CQL`2geUZon_YE#{_J*9`vugAJ`tR9g^PL`xp zNDSK8&x^`rs?p)+E``{RZU*)DxP>rXt`~eYqxpY_D3vL`U3z30uk-M_cj$;^g2nBk zy}#KPy>DggsUj<%o&p0JoPaK$kIqjFV#1+Ivura0-YlP+uWldZwaR+HiNe#qCP?%=dn8<*-E zBOA8Oqsxx%J=gePbV2TNi)~OQ!=4?WapS{5fq_|FY@Fcx~@UdQtKe2`&M`O(LZexPA-gZ~G2?+ouNipuqUAfVG!4iib_n{{Yod>-X1# zyK)wSysJiDSTzJMwA1qUc(;flNOcyT_sH{%HSC%un3?r@cJdeW8cEHvwX!p-?ldp) z1!t&?gwuF-l;-ACLRPzu#Llh(@Fi(2p`L zQBTc&bI-NQ|1@+|&V^QLvF&$y4==;sD#QKG;cJexX^ggrCMP z3AWv93xn4TYvy-gLrl)dF1m7ZFI!vWTeK`kLnUG$KAIlN={XxF>fcq| z_27e^y>_y{f8bJYK?FEipwzCJKYqY@lZ4HJus;N!s`)Hj^uFHrG1>Fx2meZ!*S&1B zPZ74a)xzSW4+F$g%lq1}NyADK6Fsa%@q7(KJo8|pnwJETtI_!gjW| zeEIq?a+CATIYEDNQlify^0FN%{9AP8dKTvJg5?tGG9YsF@@&a5P8OnaFeSO&_iJ1P z{-1y;OxR|B7vohTPUW}(G6E9}Rv4)AJewzXUdLPob7uw zRd~#TLJouQgIdu?fxC0ji#*eRHROMU$qWP4TWO|m z1{u>n6XigevIiw0Z5_*T`@W*b0MY+{l$n6e?Z89IkP-$k0fCT#9B2$F827Z@-hpb% zA;hNI{3-D^*4QBWyJgjhy_6+JzWuDYob`^ZMXopn(K{}R4*Y6aapkR8pdxvLq*vhT zj&w)kr5XOCVh?Zdr!8C(<9Ij8#;kQ7P8DsBAQkwJlq!G%ek>8?=X{OlkAmuRvNP}V zPO7b6^M`toZOtsc$rnmbp}c%9OJ4J$UkUmQRfkw1JD=CDM00PItxTkdJY6XXRoFOT z{BS(8{7YJ}H7?H^EXOxbRNi&(%Eb2or1AEyhT!&`Y*AP%A|P^CX72;Iux(NMlX075HIweB-mUKAM;JcR;k5Cp(q7& z^u1xS6EzRWEa1PIfvfI$Za>=gi0phsR_;&HI6ahK;j1|FvAs&iv3!VU4tXN-4$+}3 zy?=oc z)a|#Qyn%O09`|^rggdlb6~P9e$Igpxd*viVTh0s9H+j1e@bAaD;YNXo@;iR_!SASF_*F-Dj ztcct)hfN=h9DgGAXsvwJF`{qVD43GM2kC#dRjwY&BfZ+_7gno4o};=LwGBxxQ^?n2 zTpLiC@)W9g-Od7ijK1BrcBZhb$|cV9?3GBT5N2a0doJ5b@rCw%fdy0M;w$+>CqLs_ z3!yUPjxIg?oMT1HX|aDN%J9~mic3RES-d^$Xy(BhN$*!Ix3F#A|9zk*@qYRpl_BPBRay|5LWwPug4l!!C;Sw zg*H3)AM6if_h%2Rgq7NAPOaVuADjL&v@Qo4!RFEmG}}M#qF}JONjKz_acrNkw+eIC z2*L>QVT^+CVHTacxH=Js9h`(dUAR$!Gt&sYEFqDjTo$L5folUOUN<;-dU=IpS_qbe zY%U;-OgH#fPE3!HOl1#u?Mj_Hh7~n=tuK0a%A$u!(yriibTWZ(&5D-GMPv{4C(_9) z`i6N6KYUfZ#jm2kq2vfcuQY4dJZ&4e`P+%Q`FIEm1TvdiphgMIodsHzNTyh=q+`=( zVi_Vm!rH{7G0gL{qQHk3j7Y}-y?zX!VP$KxZ_DasJZsA0r6EreJSu8fiWP)JQh9dn zx;E3imH<2#xz5Fg9i8RcD!s9hR>!Jl)E-_R{t%JK8-=^_PN_K5<-ac%R>n`$>ZGr~ zoF&AKQrh+J!5F)#IQgI?3gE@kFr&|jEc!j6K1l`Mvhhl|+Yok0X`H2c(xU3sAf{rx zW;>`5)p)x+B!+igd)T%(38$|~Nrr1`I8Rej?@iBVArHrh{eN)f;xq-)U&5}A=VAgf zU9{rRp{FEb6cOKB08?1{`^@>Cx}x8Hl}%uQtv1R)x}|$L#&nEw+EZMW&Lw_@3{ot_ zKc=oMC%^3XBy8DM2;HNsUVg>0SIsXVuZ3B{O*8 zvcRNn5{1A~pAl(aNC&_Bk9wKT+?&i4j@;~P1D9ncE_YX_i5*D;(yj{(9d z^e^1nvL(B39jO1&cED({4Rj^RBf#9cZ}QiTn`JXWDBY*Vf$*@&Ko3{f1Fs=#<3|e> zzMt$=u=&9GJliI^uH`g$czE~>t5Pk{q>n9F2MY7Mv=nifIY{nTIf<9q zzKsYQ=RV9iNL~@~5A$|r4SZQ*l_uH=26Mv0FGI64OV8BhkQok-z8G+QHi|~9Lnh=v zD=hU8N0ZTD^SbYM4U{7_=seTu5MkA5#<%$oC?nqyAOg!Da?=_AzIonFBH{t1v+PV4i*QK+)D4zeCf>nXc@i*O5`7H zyK7Zr(#aI12+}y7_pTJ|=;H}Y&HKEFwRnR21KFqJjU2keqIxEw;+)raB3$3}!)hw? zJlbOnop5@AO`QR41Z|zAG?Qo#b%|4aPaKTT1srxd&xRG z(obQgvR=*;REdvw;2rL4K9H7l4mu0@J%S6Q7IXpf0c+et88Eil%H@!A(U`j z>lfx|g}mex9~_QHnQQnz8r;&0!M2YMV#X-u`>MxZ+*!r3tc$?$QSzdj`^+e%miLqq z*Yz8Wtu)^?GKhq1PQ(y2LXDnB(4gH-4qPlP>00}kPq%^|yTw}o47%A=FA%89lzSNz zBSo{7nFuw$e-f@)`sDMRg2m1!jg3+e^yorJkJ%(eeGMZM^{o9cR(_x4;S)kP@5;Y} zLEMloKZ{`$r>jYpP}oxnx>?I*9cX2R2*uAJ zya2CB8 zG0^h$53Syq+<2vA1`mgH7L-%_TpmyaasH!{9Wd>D>HJnbpok~g@drlr zmKyPSd9!ltRtURvu=;`rndn4zuhC9QkZ36|bX?E3gaRW&-&q8{Al)n0zVnB`(W+n9 z`gy51LVd^8bUZG)TzmXGzjDXdKC64SB2({9=@~+}`XD!j=CdT}D9Pp4qja=Hg1)4u zq`$8S5rsEbK!X@WY)z}fk`HkYfqQ3Gk{e812ie)Tcg6U{plhX~OHs6kj^y(;(C1kF zZ+qph1c1tn{YG_S`i80P%dxqUxF9%TfAFaEOrArth|(DRaz;&JnEzDal%hvd@PnW+ zvW8I;r*wGC(VXx~E|F8$_%PZxUa$iL(LuTEP{3bfN!R+eM!pN=@HB_cNV+9z$IV5b7KKdINR+dp{=848^jtza8TP2I zjOcJAIl2eoJXYXGFj|_n4*RjN!|AxZ^!vR8 z!C_|86E;*>8`Q|rq#fjhq$65(MKS!K+4nTw5$=2toP#{j81JO^Uo}VBLcf7Aqu{v9 zuMh8~e$SHZ^4L>elAIMg?JY5jSLTZ)(if6Ds-w*Uuh6lOJrS^^FIs(QlLZobL}-|{ zXAjAI)K%E_N`et@yy!h0f`FC}A1p)EijQnzx_%m-M` zd;qH*onYslUal!O#!`aloei}^t<|f#yhypJjy_U(@C-gXSTkGg1GvPf5jDxpccvg} zn0jXX8zuJQ)$teaP?fs0hCfZV%>2a_?|kEO`kwO-98uCiCnkwW;$Rd|Mj4F|hN}Eh zjzL6@xNL^IQv-`WpLwugGotQ?6`7XG;-`DYV8S8{avNxIN(rq3TGT{yG$ZT!7RbKK zFBb*7%n3s_apXP;(D};NsZf+kj_|jxM9?T~L>*fWzfjx`00x1|mi3Te9I|z_z<9h> zgk3u-zeJW1sK$k%nHkHqvwPQWURMi=|9kxZuSJBlndASR#`H1<4!ZxdEYSdUGlAAT zAU#2MNZLr&!><6=766rH0IJWn-~k(iWc>1%Nzp&>D({NX2633*SLD^*pvT|8Kj9p< zDxTq07etAHx384ERTt`H1BeA1IrrZ>y;%iRgl67v$#Mnum;Xpo&bQkNDh#4`ogu)( z9=7ZS?){m9S~_rU1I(D~K^n3ZqCBZaV^6Y|nTC8IPz$vV1UOiptgfw+;2X}GH}c|6 zTQgIx!gB79K$k1(Sy7}&k&Ux`#DxDSn5b3;JWy7@#$3i!{rVH=El(S#Mt(_4LluV$ zfK;AN#l(H0PH@Czq&x^TTJ8&o3t<%CH0)D=Kh_=e+N&Y#Pe}^_*j0+d2HWH$_k!BvjJKP{?m8D&%YT1#k?c-vE^u1xX>X&5HDkphRA7M$o+O1nL0eDs> zbq9%?AFW8-pNleK{;SRgPnJ(j_>W{*%3N@oM2T+~95cEZbgRP8d2zf5*2QS&*`YP^ zDpH1Qja9Zi5oE43X>KAh^%7Pei_;01D1h5gB0WK70~-#YwGvd%XO6DZ7G2Lc-KA%8 zLSpXc4@NtdKTufM;BV`1sb`<@>ae&_Lq0v1^V$ayNN9#XGJZd_e2}T&GdJg@B3#=d;1GVD^hH)jq+-3ejwCFgDi38 zZ8N-+#G$1j;&7<>)Ok?70#OZMMgU_|6g?ht-}yxRk=B@4p3}{DMiVxVG&&9jaECSD z?rV?~ZdF~(16 z!Gv4>M=Yxqb^_G6)gg>T1@LQX!|PTM)wqK!hQPKsMDg7*TVhNDZCO6BrZAr2>wtl8 zFGQs59S(q0wn%SSCLAXV%*v`Z)&y-7Q<4ktTk7wwF0?$dEAVnz1>(L5B-Go1 z>qHJIEpVjT&u3D32tV+e?xe{#R>7xs@1zy$j<{*r4qHdf|CLp^!Jiqcz;97gQg6+a z3)+9F^SynEyUsi1;AM1*UUZ>55J-v_?n%tG=qOrpp(LDem(6?Y_u=AczRtj(WF^!x z4Oh(R*3eQlx%M2Qeg zX|=yy=iayL;b>L@SNl>1HW z`!x_FQHZ2`UP`{vx+M;Mi4FwHXe(a`ce=Cpk|gR;CmC7l?-*Dpf3K*l%j?-t_;{+W zIFYETe6|FE^mY&ikYRm?#UTrV7|Ca?J@#G5YbN4*t74N2rOnMjups(IhxlFM#!*MSnlKKakZ|#m4)-J2yTVhD{cY8BL@5 z-v-iGV{}hc*=b*4?biAPJ>)Ec5gX62=;>bbk+dMPy1+r^Gi;`>A;O;q#)6NH5-VV$ zVrUt;)D3sgGD*Z`z2t+{2Fz7086R-mKO)j*Lzj-BIoz>npJ>kZdhR3LeL+)pBAB3>D-Qdhu39thGw!KBiUiQ!EATuWuKjt&H1GxdLOPr7SJ5msty80&;{Ft<>S58*#C4VG3l!>7 z6Il4u5*Q6=h#PryBE@`8@MgEYHb97|mJs}xoZBTD4TlK1D8d=wUbrGX7*hVP9qpfE z&VT(<^knAyTI1S3f5BbZy$YZgq{n-AI{A^hT6^)f39VNE#*)E%B7kw;CWAmO%b$RC z@LIFVKYXmmz>yIk5{Q|=9^S8t2EZNeoY)+n>HqdnL)iPTXRE18x8jaA?ReELN2Hy< zc{6hL1vVrsksheC1^#&&(RAppc@sw0(#Apvz-f+Ur|(oFLD*)X`tLV$K_H;MuiJhV zX-k2J&4`i+gu=q#%-H`_txW&SYO_YS9)w!`F7)Z$!#wm|l?DuK2mJIrQs;iZ#|pmg z4NlA|(gcB09+`hG=H~3>%KMrMtN?04m$_zl$=|d3YPY51ETA~t2}$KO2+=5BZiBGQ z$VE9F7Sz}Ydc59lV(L#YsJnH=xxZp$Jf22eRh7h3V4UL7XD%2QKeG6)q=S(U0KO?S zb~FErxP6S2*q>x$)cHivB60O5w9t+%)D@An+WY(*!zXwZ znr=po)jvwnrj&G<8vWG!=p>L%v}N(LYg>(VTizw>f=`u$hggqJ&I+p4o#eTjCxg9> z*_95Cnm4at9AY>>h~`)f?aV?&?jKd=b0%wR0JzBvciM$>jKi(c0JdsCSmRbt6ym!1 zck@Rdq@QbHy%rg1XEdf#{nu~j29`Lo4!~S&}nddS2<5w#nEVn$!EXO6sk%td>)LPz~ORU0W zf4Pd(@A}=z%Xg4i<-!bBBP)c<#rXd^+2PtmO~{l1Fcish5Zh`MSwPaa*}%PQ!8P7F zA^SmiCKQPZlcQ-m7Tl}hom913unF7SoqFU?LL3TmTwbRBOdFtpTf(&qHe4M+3j8D+TED&`v1z^BzJq)4n z7iZr4U60?3o+fS0wQEbxWXrOgR02#GF3Q@EZhXwMJv6nFZll6hOty?o4iNsRG{9P) z!l-&*pV0Sn|GB|Mn9DAOKF_gL>oAIGYYyCP=whV3a^8*o=otA`IbX8*P2G%<@1D!s1uLf^+mH=|lYP+y3zMcVi#&@H2p{RaYmgV)u*RQfz|<;Fo5 zI$J3q!dxJa^&na6Pr&~|J|V^JmHmQ83x`a9DC_BmPbbKh6%xRb9Rn}sKB!TjY*A`% z!oWn--O*X~(o46xs?AO>pl}4^2_w7)I|j9c!2=yKOBnQ6Q$WA2`iQgSt{gBh7&ooX>| zB$TDvqV@pzqhu%+wJnY0>Dj>nZ=01vKO>GKpdnF4eq>{p@Log?nR=PX5swif-Uve@ zR(c79skwr8sy{~!JY->QVdefaBU%MYE%##9AOg2dECi{fG#JAGSS+W*yLNTBIH_!! zBZ`gZt6wi7w{?OORx_5QyPzW8N{BCC8Ek9eBKftn9gnRizJXysVX2NVncWpX0Ut`rN(!N!R#}wkGc*nl|ckdAZOGl~~7NF-*2gCCq zaH(DzUqKKhVYd?idK5jp!`5V^&#Pugay)mvi2=aJVw+xStFj`5?2@6#6XKc$>43eH z4@s#?dg}Pmim&S>Zg5!YzXAZKG^LPIF|6nEj1$qzep{8fn2Z)C&*D~QGy%S*l#aHk z3HYr8qKi?5-2cJ;*i5$nn{9sOZjE-lT)beI0%YW7Ny1OVxl%|%JYzh8A7gVL3OsY) zljI0z7BpXYuNw5Qh=Css2@0^Y0Co2LSuFjm1$9XUU^HEL zjdChbLBlq~XQl!@{rnU`i*(>SU(@32fD5i3XfXxZm@NUPJqAn$&$lEvEbWIr#Nh+X#sVXm>953O?c5DAY=sPfuKsy3Dqsaz=q&!(fE9WD6 zeuPl~&=Wst%!-D9`oHycxsU-j$w)|27U~LET~ze_2dDG8me)F*?FJgN1V1KAEH<04 zu?{c7BgXv}oi}P{9I}t+)X-^imW%qvwcZTz+r4EVU=*tgFbw+o>+7Bp)q^SA33 z910_9oO3b9JJ;74>SRg#Sc@jJjqGt)ExtN0`*?fQs(h5|Yjm$Gfi7GGadCB-Z#E#p zn??C`6So>BGz{2T7KKZcdFyAAXkLc9>(Bnm^56~!8-GGWs7h%KIYE%{so<#BP<#aV zG9f@_FG_sg(Y~?B%Zhlxa_cr}mDbwz46p1R`cg{YS5q8_KEFqF&fA|oTq0jTUHxk!*spj-7zlU$cO*7TTH+<& zvW=(8VV7b7&B?}e?hvRR@OUKmQmKv(K2-K1msG|cT*YKCX7)RvgLy5(Qg}62k!uI=Edu~*PY$H`_iW#mALnJg-1@> zHLW*%&~vLE^hzc1{MoD8DzUFVUHkOn#jAI&o_=AY_XyCzl8JVUAQ-D&T|FR^1P+=_ z7o;Qk07iF>Mu`otcJ>}-d)pM%?%7(1x=0+fD#{dJyd7c7I$W%Qhc=VtiHzrXn|s?Y zVNvPh7R3zZTE=creHMQ>gI26k-lu%J;1a0y5un*&tNin`Mt z;dW1ONvHqdL&#k6l1NODs?W>u+`F^JMqSTV`QJL#ocR?c$T7?q60`@P&+YwAF7KLu z-de)u$2anxCfF9sVvTJgvXaz(`g@eB|cAJUtpADIvPOg%g<0X*; z_eYnGpkb%ff3;3;(uyB_<4nm6KFnE4T@jBfzMFqo)JYZZbIchgnTyJ0OVv5%jggJW zJKr{yHabkgKCp~L!p{twDEh&DFTN|=Y-6f>pD=u{UMaj-qSWUTmJ$5XI7P{cbieQI z?-m$XCl&wk?E9ZTo-)eV8^@&|?0qyzStGnrpWq|oU>$DwqtSth1;NcSx6?rJt>|X^ z7{RHsXe>P1&C=BU`(WISgzfsX=z?SJGWIs@mPZ3`mwc|y#LDlrzcC9lTe1S&@A%bW z{}eP4S;q}hjaq@9M*}w~(VlQ6>g!_|y=Yf8mFqZ`tX?b!;*akG>*unMx2k5 zm`^Bc42Wtr&)Z|jk`Mx(WnTe&Kx8wib~cnYhmy8gc^Rwt6G2&je?-31%InR9YZKtL{8x|Yrb%x z7WizS=!!E3oQidYU9tePQ;K(78VFpvmy(M1Eu_UtCI)!Jl5R*ovh(d!EgF1>BnaPn z*2UvXJ27GA!=pK(R4OySFq7){c`gcg1hu^P$0>y&P0>Zuz2@Td$JYL2EB zV%TM@eWjARXBb|i-cc|vJ0Q49_dhFvL-yVDu!de{nQ%Yl+FAE0b{4}N8zN3m3sg?F zGfKZkah#M8e$j6S+m;LX<2~Pxjav3$OIGM!$u0_@6`UUG@FI5lw&?uWSuv$t8ZmYN zFz}@ds<@YZR0!q5y3rMO#}yWXoGMKhU||6A9SK-*m9tDnwu>AGp0fMeLR%}n-a6-^ zE9Qv7K0t7PHWj;lea>?b#0(l`J_BZ(Dig7bUSqqK3x69J529 z{1mtNA|VZq0Z2#xP-!9Sx6{|pxYOc(#<%TVCwoss|fS~xN1 z`<8!~5voRh8-VUs_sEd@A475n)`PZ1p97uA>H&ccS*-{C5{MguP)=p{0hluig=(eL znL!1CRAa@0aP#(JCyvwRBIXpIj+M><_tj(<#4!MUU0jcHhByBXdQ7fkJp#7&Hl~36 z4`@&N>&2fBsQktpv1&rfL)g{bk%ZE#0s(l$y%bH^(?1$3d*7o*REy8c^-_=!iO%=4!`PmrkexDX4V+7+KA@Nl}s5Cvw2@4Fc( zy|I8TvN-P=L^bY^)!+w~M1$n5tAO<2s9JGdfo>(xKO)*{AanR<8&U9_=;lUw7+pl00zk#jS+1}{ayLKtxK z{PA+$YBO)%y%iynS`jGJIDL}Yu77%502JzbaxL(Y2ZB-h8b!SCBGFu+_>5+&fZyA? zqQT0#HHnWo28*U6W^c3#0>XR`l*ukPmupn)z$QcngnC0h&w2i~l$U1jrOgknfdCK|}P-iT(?!bK0z zKalzh(XW)ww^964mq>nHm1wUdGFw(k=RDWd#hS!^b6 z_#RYT_N+zM3sg&y*yrv_pp9hNuK1`^1`W$c2T4e_{ORBNf@j#?f$6bBk%@Hlf(S!` z8b$07sIF`(d!qXO_SpU_#TvPj602|;sZTKu5-oix&NgzC9Bo^gS=d6(MPFC=89@9s!+R4`c` zf+w}GM+Ro;%fGM>i5or3lmk(Jg~sxy!i7(3e0Q|tIMp6`17w}6gkZ41_h#Kl)8JO- zHD_&E1{SogUFQ#U4wE-V@jK z>nA_8QD7%fry-#oaLcrPi3eLQT=*^i(A9?!`6jR(3NM`%cQfE~(^66ZSlWNwhyBJ< zT@kdhkd1FD>|8BLX9UIV5r~7vS?RJBTTnd}V+kn!m7y%E=?wA*lR4|8Ql@dHF z^+ti`h%!wyxl|`Je_3V1U{?@$&U-BFSMhJ9Uu1O(6lK`S?w_}C%J><*;mrG1L9lE* zW{d%OmG5F{A^8hfDt8Hs=dAU%&b)@p&^67gEwvX+*@$Qw%tkyZ&NrJiFK!~hvMFuC zRxVgK`4umRe$0vGLRpeJ$7>~1OuQlh-kq4@LOKy~{Fj%^=D@4(chW;^%dWL`v~^aJ zM6Wf?_J9i8bs?#+z#!;19HXLwFrE&O2f07|fu4b-GXGmBdaHa%-PHhegP>U*Wq)nf z@w0k6?PZcGVLEVKsz`fy>kwRCX z?94+iVq^4P0au?i%P+dS-uI&y&7&hmJ$QvWAao3ns?ifA;yoiKWXxMGF&H=)n7IpI z_ju2-r_EsFp3=GbTpZvPVnp)QTl<$Eo*+zIZJ{K8Hk6GQFYS5m_0>zZTuqMVa*SOP zC^d%zAc2ciup704Mu99r_M_O>1En@uMLgcmc6dBZ3~g;N(Hxr}s4gB(b$L#Fm0&4+YTCwbjN0 z!*%~87v3In;FzGX$qYK`;c5ulcK3w_Bw&0T3a*;}4jcn1GO0igLgoKtTE71pi8Q6B z15Fc3=#|F9=MR)$otIv?dGpeQJ%J^tkx`}bqhLYy?R>k*ifx;s4okEuY^9l^9txd* zNxG)*6ft+wQ)L)=$dP5uGqOz^{V!b;9afOUoLY>!a^Y;)%WMxbcJZ&@Qe``bi|&~0 zFta@;JN)V#bW?4YTLOW|k`-#4l;k{-UY!;m5Rg}Ef5I8B_6)!7e6T6-_$J92-3u+X zyBXjrs9_=9Et6do3=B!c_(ZZ_NK~K_22%8tDktxtFB@V)(Ehz$h%8%YQ0Zod+WPPe-E}5dx z9Rj)*0)21^zOUurgwh0F#npTl-!@fwYUR}xRosH0cuNa1D)@89IpKsa0AU6R#u%&y zWTw0y@m_!oC7##{I$>_!xs=K2NZ_CD_USx`~K+#eqkcKM$F zmFIBhhTub1QN*GH3L>|3Otwao)lvFUmuhpD#=IlZ#!K*#Z415)VIM!Z7*M}Z2bNaO z3~7+Fnkx;RPi5v4>n0|?m$P2SZu|pi32~Ya{?1}sSHu1t8QP98z8avvq11?3_e$L@ zT1X5=al6p@-;4FZ)_$_{JmqTt(xYqu5W5Mq2n3b|hV9~WRFo{Bw#$Q_VFPF!C@YGk ziHlJ#Qhr)ohs@|9uagMUs!-AqG9FOkCgq`kBk8$O?K;p2ef?dvbj-yzo=ric&6gvB zXHOZGPFO~h9d7t*298h9uez17V)4S}O2pn#UM%|0tT+B%^f8UM_2f=#8xz{7_lT(n zmS(p66G#->M>5`>6r2q}Z`!9F@Qb&&(UXFGE5GDwH){VRLZz*_-6s&~Ut7?{Lli)c ze90H@;pWY~8(_>QB|?Il0Blx9nO%7_gZRKQ{j_^T zIcR$@8XJh3yp9F%Z|1?X>SIKz`1GKEshihx#IBJXFls3%(k0&B%^UnGa3UT~9sJ^7 z(v)d26`(o>@J|CO=uj+~T>a&1-ITG7ycy>#cy^4`tAY7?T*VHLJ;as3#{3OcdkTaW zyX+P&!K6u#LOdlDTN9gm*m@+$yW3CM^S6-6aM7EA?CVWu^{+MUi@2NSWdZZ27ME#) z?o@D;)hyMzedA^Qe9ccpxN;}lsY27_>r@d6`QWiPv7HhW!D>^8jJIxc^UvhX_oW9} zX!QKBgwC)9t|p97iD?o64Ex0sNUrlFA!U-s13!A1F}F|-KDPZ^7sihet!GNB;-ycy zqDR=pX~->IyIT}1f@OQD+fJHi+W`g1)KbvHX#bHOevMzsDRUpi;OvyY_eSQwz;Rau z0T=ePVAEFB*L8`3ZLA*X4%xZWWh_9#RskytYmYe{yvH$t@WT)CA%c??4=8hgqBC9N z5@k=%DNPyQ7O~ZCTmXiIMeuWLe)b>-gsPK@jU%)#yO6>sUIKg==+edsFDCPpOQ3Lt zzOM7RGhiW?0T%3*wZh>-qDD=3p53LEsl(KUPv6*$y8il@DItg)1vqdLfXp<3vwc>n zxR5ZXBtp?+xh-|zJBT2PJd4OD4Q%__D@?Q!Uq_;g7*xL+AZ}5~e-9f$eqrZ2i`Xts z%bos!zQFefSD->0N`{LTpk223&Vljqqf0^qC(C@gzmqhRoRU_;cL^Q_&em+m00*kW z_j2eTbalX^KpTJDy}g@tB%{i1~BE$YSiu>?O7SbZX1+af!UAd|?pFSJU<&=U*n+o&F5 zbD-EDIS!i|9au9%6Nhrc8tE~|i@6E5C{@tpE3DIw4tu-|{SM>3DKArsY?RaR3--x2 z53vK(g@_0Qx=OadjYdOzWjYIxR0j46;#%Vrv}hC)PUQP1Z( z4ncN8Jzdrgt?OC%am@c?#LvK~e|K(f{E4l5Ex44xu5Z>fWA=Fzj|U$ zG&@mI752Py9QRKSp4RgKXtntFruffe{J(s8aQ52&>J*jKHN@nLPe&^=>0tpH#FPIp zpQ@pBa>AjiPJs}~6IdS)xfbT)f1s(n9q3Dan8wfZ^BRkn0Fn7Oi%*AIaz>*^!(ro4 zI3cIUdjtR5T5RI|*GnQjUQosLgjM~uf#X5D;^pE}@KqA#dqIqEG${EXQlg5Q8pP9|i%Ovvpq;qzDr)d-2h4hrbUT0W zEoa!>?Qf>Q`rE}aX)%{sj$IE@E0jo*<`#*54m{Uz@!C42t0VSh(CIC!gKNExpcB)N zl5GdP*RS~}K_DQx3VN*Fo)m#HR@=nZdz&aM%W4xXRj+M4#zLuaXC-4wg4Mg`whlHb zN@(riTfUqTV5f6gnqjk_j;OtY(QAtwH!|=L>61_o4Aag+*Z@pp1DPnYpKS=A=K%0< zowb`y$VO1d6=*Fifz$Oi+5`WdId^)#@zhqP8r$dMqf5#<_iTy6L-C=6nvHvmHTYIn zGL{BXg6}mMcVn|8w@<4kOxM46&PfZggFLFac?;UIi-j>dxuHyphtg@QFC@B2Uaqh5 zJ@r2Ku-Cf&#$hYrsui~}kg*ML1Vp!tC;!nd7tZ814^ftR9Y^sxMo#}cY;A1KvBfF{ z&)_BWt<(K1t?z-Yf%C#DK=GBhtzBCyRQ-F{w7_Q6y(_N0ruhVaHE2JXshPLcx|+qvrT~Bu z5C~|2ISrZ|CZP;L01i5|!E^Y{l1Q^XW7#~0T!BJM#;AXI-FI4Atji^8`I{L^kR1>% zqHE>;*S{U07A@Pi7Hr&WKeMHPVX)GUkmASt;wxk`&Ms3TWG_8zbnD#)olNeJ(1t|P z7ndDAGGNN$>zi@viMJr3vd*CLc*)Bk(ngR|1W#}HNd~AzQ+J zidIuNSbm(s81zc_o!CmnvNrFlm;)pPz`%g zbIPN-Wua+7uLG4g45%y^+oG^VUs%bT&5fiKKSZ)Cznm1m5plq)r&5IKKzQE8c{A?ZdDVC{mfOr9~=JY|0l z&!Ca<{CtqWX&aU~<3kS!7!+7@|^`3Sa%xrubq|FrvGLPEdLe-7)!c5*@!G!)< zH+|bA-IHv~`vwINWi*|BY^pC}m2Rbii)|jgbMOdz_z90wN);Ly7<;0dQI z!S3C3sjca!dwp%_`8T$OjND(ES0cY_-!-4!k3Tm3ML_Z23DxgXUBTb4@b5YL+O}0x zneNth^8QJiORvtv|1SAz1s5I>zrN47f0)Wr=bsQu=h0zbh9yR|<73A}W%(B&3)yd( zFfY1ZL`w{yewi=+x>XjFOfgmD9^yZ z>W)tDU@O z7-b`&AJvnA&zDTz|&)KOgNb^RMsb=QN7*z!G@O2M+(TeFE!evj>@uU$*J@rO4r7quGGeXxL|z81ZT z{_5Xzy6ocx^N-kFCC+}BzeOH@t(QH3NB$7hCHFgGG=E`EU?KaSao?816%Mez_mkn~ zir{UVTs1ztN{Tfy6tK4pP=z9-9=fU-vPO3Ag37p1xm&BOY7N35nLJMUMnAb4f{9wK z7Uyb>%U>Z`UcOI1RE6fIQ8pB=2*v<+J?603BdC3?y+iuj1D4coZ5$bk554xs|Ak3v z^MP`;obzj?ssA&IHI)+q+eSdQ>MIQxEb%n@_I>rx~YhXMd?)Wh)MF@!>qPQc6? z%k>}|sN^9ih7pz3v$=j+1|k_i+mjL~l<5Nl zGJ}(HV3i>-R3!32N``2LF1DIH-hhq=y0&%+0l^vd1~?Cr2~-h6$5!>A*cMCj74&Bb z8;?Wm$_rwjFz7FxN!QBJO_W@``t=(B^>u8TNNt9(3+iz zo?K_A&8AZq2c*dO|3llGhb6st|K609m8H#APP?6@rDlUU%Q-O&bfcrIoI`jo^w6lKS)zS z@WW?))>`lNddF@9T@j`Nob)@iT>+>*WW;qR^z<|hKw%N-fGMV*I&b53l^iem)2?o?)}nMdEM(T{!$QyVqhn{8G}KmxX|esLIPhy(pjg@o z_|GhjP*9Y)we-G^y>G}1#pK+TQ|)HBQQV#2vJn~CFHXphR|>ki2N^j}rMmzaDN|hv zJJ(w>{20*~dAOwxy)TyWNm$}1|GYlLsp6zK&gfY28H4rx*yq$L0dz_c}h zcMYpy5DcL?A_wy2hUkP?yR2LrtJ0mqfdeWW!sNE))`zya1Z)kB%(BlUhJBNUAd3kD z%)o6|GgeB&{2Nflij#^L&rKf7j(_APJr6zJ5;GG}kIGGWMyaOs&Jj>ocXL%$+om0( z2NI*N>|TGF$k}{MEniMUX3ddz(X_O&8`&i-vV-UOapa`vxSty1uq^?BXU1&odC_mI zdafhJrTgumUDIdAR>fW*&Io+$f=(M>;D;8Q2 zHZpfddit64{0`GU5`HTUez%vPkiDoU=i$0?neJr+85xHebC!9=xh5lzY$s;{kuxK5 z9&(G`3jJZuv80Gu=Wk7jOl(P+1-1jwq*$k}oW=Nn#_8VWp2Uv9NC=m?! zp@?*N%Rn+o8vQVt`^nc##iz6R;lq_pN7jyfp~#8={j9Fwy}gIbMOgKHamSpz^SMX^ z)$wL8MXC7QUkh1S+DlVRKyfAh|2fmodh^`Qps_uqbWNa~Pd#ij@3Y6+u)Z??RL?c=Y= zQJoT%P~Pf851Uq0Gb!F32deFE9m|hbsp88ENVJiKdhD2`pS9J74usw7$h5SWCP)GB zvfqBH162?(fNjC-7hv0nrbrX7k{|Kyk2vm&H!F+hEDfbcu$HWS(J_Do6#y2a*8>1z zy!W4SfAViwmAGVf#VK8bDz^zr!vLLTi9cn#zud6QW&Kod#0yp0Sj?Q*H2k&hLizfN zK($+%=gXLjl9**6rGROk^EUNIuu>Jg8Z`UZzCXsKA4ahSU14X(01kXr{iREEl~(9P zH1V#|S&UrWi7Ud~T+dA-DGQ6x>L5$bb07@|%jdS5`+9bRTZ7iK)0~5-R(5Zz^ZG+5 zdePOrDF(Ka2RXco0UPBFtC-S*R<|>XH3Xep&UCSVxTlYno=bM+D_A_Un(s%m@a0lZezSQdluoOq2(8G8R|HcpX zr$%uzq=MJBM4@LU)mPHQRsls1w2!>b;?phhET+pL%qG5VRMa+Qmf!=;OTnqY_VG?9 z3k%oDGS1Xz64~;TZqV>NdAaS95;i zEo9av>paMV^c!*FsPmMYjC0rW2UmR+cVC+}xM)Y5zBV@(0u^Zh?t{#^@DQptyQ`vp zVf$E8;CY%y9Kc4N9|cR=H2_l#bWN+f2Ig1271O*I%Kc4tE=ip&hfkHuZ*pl4G`pL9 z6cC6kuY6X5n=rsiTn{R1(2L@c+A(H|GybjfPc&U*ShAe*Tdr*+rA6+g2lHQY9`w4R z+ZRR{1k5Rb8At$woE4QQ*5RS>4J6XR8G&VR2STuS?h!S80mCYh$?-H|U{)kxFVKZY z-@ZVT#I*bq;S^>UUM-bR{y!E={^yVVN7-aa@qh)q9<@L(fLQ;R@j1|OZ7R?kVQ>T> zkaUKKfAG&^Ud!AaY-WsZ`}ZLL2|%aJh(5%Rh4=hk`U|aDJmgP}9NHtm?yF$XcOL$S zQBh2emzev?>%nD>OR1_L{<4H$&}L4aaxQt*Xuq^SMR7eR6B1CLOQ-@JJ2 zo}L_m%6>z}l>3&tY&m_wQGU5Vfd)&+&bz8kM-Se12YtD#stZt^8!gBHJv73$C;eX- zjglcz7AGafbt;;b5mLGgR6B7Rz=LQ5-cq1DX||_(R(BzV`>3**)Cv9pjWdx4hb7K;4k^8|cf7z1@Ab!oWhN*ti})m?l{C?jOCA zKS)4am&`sh&GBO!zjKB^6ni(`7U%avlUA{i*?wId>xrvMF< zH=tCoCmhhFm~tg^Fd#XJ_<1+bGg4(LGMQH5>rWMXSm$@+*U+2Q%>9aJx(A$i&>7Y5 znxQ164$ejg6mf>oiiiTAsP8-CIlSt5O_LCc6cm8RztI%U9{2%~->u8gnV2tauQ+GA z`=d9QP;3X)dXm_40N5FV;RBWe$^Hqp{qfMN>5k zP$*%NZsds21ls!n6lTZe^i^WJm*tp<4=aRi_s(^ zHpB19^QkbneAJH+MQIY2TPy>i5ehc9q<*A3VEGzBhv?lv)7?`CBR=NA`)*M#_&`~1 zf5pTwgYDy>J(31>5z6d|5Wjk(eSoUx`War61qc+~5~eL*Z~)%U13W#RRJ z+d+C?t$+Wm?i=ov?-nhrn0@i-ch``jIw{qr+B;ufSQP_-CDqQ16=h<7<{WaRU;lBpj zJ|fR-?&Hwl+du6^qEq?j168+Gw5^@E&4&9$W$G$(UnWpqp-(hfzT0`o>EfB^aiI*o zb?24DL7hprVzrUS-*E+)3k$6W%=&d2Pjr73zh5V(TYnFa3kS}j3%jsD!&OwLtQ6%9 zULn(-W%2xN#^JagzzUcdw;Uv7Vld^V4RuNAw^X7@<1(=&bZcYs3|=A^AX9JzPoK{g z@C^ib?d3%NwR}Q$;*XCO8KhQOPXH0`5J$mi4!$pDz|c}H5XcEr*l=G>%R%=@(RqfN z+H+HJEwyH4zqI#=T_E*{iTXgW1z#irq&vwLlzv9paO!W4fcgvr*fDeyOwCDGwRav$ zeBTtX0kiAuJxR>50bZ6DN>BJW9pCZId|GB9U6bVp?xnb1(cfqmZdGqd?X zIomuu%dd7IZ)cqMY52U&*<#9>nuJaOj0CidSnKKimH@3AcSX=_i+;PSlpUGbq5;<> zT(NJMrWU5^>8@LK!Z*17(fSJ0e0LFMythL+6x~;T( z(^@@^GVrQhEVb~^uFBvJ1{Y-Ci)2fX_3JHqYz|v{sX?FqHF${HYFT^Td*r6+B<>Gg z%pYAk8GAXSyOWuhk_Ecz!OHymHZEsKxbVvR>}{KkW4N+xkykRc`mk6u9$!J7#&0xv6EfU<8|T!B*u^0(F$ynQ34tysbbeeSv-{H5!}@xPo)RTQb@3KlEqxGN7Lh!qB(3 z-hMCYPVDu?$uF&9eH$~oHeflJeHFE~$$CV-lKMn?VtgVnx4pXyJg&JJr~2N>$yNdZ z<{R3q0fU1EZ0Xv0&-$y5ab0<+D!!|ZS{a_))V)ig7{kAl9}?@$8uFt=15tnurWIhl z%V9!Rpu5i+pgl#WtQjZWyEau7xSZG(nn_+*)eVy}`lnm*Xu-KQ+^|g7-g$$C+0Mu- zdhz}^#hXcon5XcCGX^b`x1M`nYyl*dnZc(e=>nTOP_SeocNqwak+A@cd0c-#2n3+I zPrN9Yjm;;Q+FrK-CiHX`Rb#ddU}uGk$4Jr=KgzpprvF6h>X6OnR{Yp+y8C}h zCV+eI5dvpN&>gm&2`_*b{>e0EA6KI=ZjFyVR4L-XK&w^AnL|b*_=%RcJdmIdDjC$UcZ;d zuBw$tk_hmE1~y9|byq2?#(Z9)O#nEII}NwbQhb?#DEC~lVp4c@iIM@>WB_`re`WfA zBZMqvvp~+XgAS&0-hQM1FyNkIUK9qk4?GmW@($6sh@6L$8LdvD}-VVlTM zw<&iuJ7puMx3_Pt_eJu%bI%dm)9;wdex%qfo3~s0*Vl|RBX5e6A>rE5v4%vQ1+nki zr_z;;h@YX0cm6uQkJl9fXGE*`69mPzs6L@gnG=<-3EM*qi+cVm!Beh;O_tVQ&~X1O*PmC6#@Uztwd3B)Ul|`+19zS_&;86| zg=zEd8*W!^{Wf}J>U??jKw$k`ELV&pjJw*^)~`yb^)2gp&ObxwB$HN!CsDU+t0RZx zD$Se)D`%oqmJjz8Q_*`^v@S2jrH|EEdIKzO2 z8=|qw=!nnD2cJLfjQ%#MR#Z2mm{e)u)kq%!^_b5$?OnRe8BV9qv-=XU+>c;7tmriksCjcQ z&SqjR70iPh(V_$r&JHNyy<#u*!JLD+nUwzL1SUnL5was565_x1!UHM7tC)U($OE`$ zqFNddfH8H@3^{2AL8SQ0k4*)hi2Kal@fq8lcn>|AK=%auD2sqU0JwUws*z$5mC@LL7+0ccnbw=+mj?3*7d(BnH4wO+k~#vk`R(V>t$qRsp*1 z336DXwJ@Y(wX~DJn~{^Ao_<`HpPs-f&_%^YD4B9@(e^W?*J9J>YYqoN(^wdQ(LM~T zXGAsgN)%BXU<&dBo4DWU()0X#BK=>rp$Dn6LdPjeBA4Z7KosBb|*D=AXg%cJetIwBh}!xay~J7|TE}T~Q&{IqNEpF>pQTnDOJ% zL9`9>cHF4s!g?!SJ0lK_e{6qMf#u&Jo`BTx99{?hm`8`zX8gY8-Q;k~jdBCxz~D4b zYPWE}2($l<$1E@Mtw|C%s4@ui>EUsn-?1z5yRG2JXDoupipfWzM)B>Xq8;pp$@HV= z!GLv_rER_Vp1f=FqQX_X2Y;u+PJG*$wxFoy3ZgJh3Iez3e&O?pwyEpP>JL`s%m; zRrdII3^c%i}&**yJ zuLi^RZw0rt0VG1kd@2Db*33Qx3eSMn?bQ%@Bi%KVZ0B1J_Ck<;73oFFtyu`<-~j}9 z_a_niT`z5A4p^{Wd4pEz=4EB9aevuLKdOKmoYNnNzS##1ZxMMPb;bg@#T`k!L-70( zz|mT|dYa_Y^+23gTzO>NZY8u-FEcDTrcJW9EE*?F0-#sEi=ze0K)T`)N|m(F1=&W{ zx(pstsDLGE83({kZ=Uy1PAM2Q$?L96b^cs=>MPt@N-DE0MW?7mJj8gu%Zgd4fyWO^ z;{e`Tdt!(KFhH9q7)FJ4!|IJ`{deyvd-rQOr6I=wm#_V0l#6 z+$`pL*2tD!%EG^8te~#Qj6qoubxX=vZ!>l@1auc=U1+;0a=1au<0g+C=9bh=28KS8 zTRAHIFb()9L_*;^0PsUPP)Xh0SpY3PO6t8mA-&`P?KU2$&!^rRG`CeyrO}gm6+6v! zH%a_9${95^R-diLDeDo!fh$2e^ss)x!xq_z@FO}@6^}`@`=w`TZdJ8$ zwg1yjlTaIWC^N$Htq*6#+U+qVy79J?bG1#2Sm9op(W8}ayuD+GwKGtEUK6?0%aW)s zbxjQ*pRY4MoQ`A9&vH~6eVjYi0nr{CdOOqWTVky3q}>~y!Y_bI6r;6rJRy6<4>*VHMnOMJVvi6(NVh8nP5bWBl1q#2+`zYqvs z35sotq?IdthLGE9ls*S-g1Y82O&tB)XyOFad#s=#2uzNfhCw;lW&uh-}Pm1 z$Z^X+4AD!M?FP*@g`$Q{S1FQ{W3OVvdlu*}x8_CnA zlZ{Haq8&r*{A$@072*`vH`3>2UQCE~);Q8w;_tCyzB+fY`ldBU{OlF0Lt0}P@ZKUgEMd831@8_s8%q8gyRv5a67?mZHwDv`_I%&eU8PO*Xbj z)YHQki(asH7ghZ!33?YRM>1{^)6GVXE93o?_-m9cca=L5p{ZP~!XQdrJ{*t*W z7h|aWJJs&qj!ACBwpV~;&hzc_~WytES0ufry! z2V`ezEF1`AkWa`bOAmp$^?Fhe=n9Qw+{TQ8@zv1qR^Bw~R2!`|eUIFR5M##}(0dKt zzC!GXV{(`&K>56yq;T50xJMv2wo@j`fA5pWG5Ej%1Yppp{R?8CoM(tu%_Xo=NV}WL z*lBwB7!N}t>0+|cONKsycKq}=;E(Cj2mb`xfulde?Us7y{~3$^-+y{dE$Qt4nx2Qo zy#l5V9=}};Dg)}~K(m7Z4+I@wnr_?zSXBWA*VQvLwSOKM({X7l=r_ltq1BBITY-t) z-mZus5ujQ84HEXuv5t9YSE()-2jCY5c$E4-2=W|HxqJYFWnC|N3Wl5yPocvGOk~iV z!e`1H7xelnN(%{fy2PhF>E;cy{=44huFozwZj|=!OLyA%AJ6L>?f}{Mh5j(&Dc;30 zcqh8a2a0*Tw^+5IQlJVJiNZhmIvY`W__W;*KpCxjK|_ysCjyh}6ZYV>jl3n3iGM}i zN&%C&-xtHwn~ARH{;I1x8S!_s;O|Mye>rv!CF5lJper|S>^G!wFAmOxSrIwWorwt* z!>L^gf7_97r8MkeAt4(NZ*n$ksnWAE3c^Pb@%jX~REDb91lW{Ax4ST_#*dHJWShcD<5@Q!D->);lZc>zWKr zZ$|G^U&SbRc%7}HE#mEf>4ucF{W5^XuR)cZUP5skSGcPfyzA` ziPu=ct(O4S2ksZSNZJbqXhB0{15Dnv)Ku9ul>xt_&*j4yrwWc({}b3R{Uv$Lz^TxDL3N1dZdl--k? z>GK_}#xYTYDO1s@7LztU`L@muvXZ{`f(CT!9jR=zb6o*lK!jdxC5W0Ch2nC*l=v@xBX<_G9BMySZbJ#u?L4<1Sz+bG}A##@PT zN;Z7_OlCb`7FP0JQEuz<;sE!{kwjZyl48+fDBBX@s~b zoptwn+tSS1z$rEbxDl}_Hj^_q3UJK&c;Oya-zW3qwEI@ex*1;*2WQmHjUQEHa#K(g zo9J`Br2)x6mt=bjHTl@~}EWl{gU4?6t?07>gP2>@`JxTS^ zoz_$7*yZps8(7EH=1C=X-mdv52|je62du@SG&0c5owZli490I{QmacsST?y?tf1y^ zVce=NzwhWULjSKQ1>=yI8wJc?fyfbe#q9=^C30FC<<>Ik*FGz44@%;rJ%aonXbX90 zRem~l#|7vP)pF%{T-3Bv(ed11Zm6RNMSY)=64zw}RPn)c!OwH6u{ddCOs^l%`r_e9 z>8$TfIF6V2NKt_bN{HqQ<9z8@y93@DwHTu|sv--U*}{rN7vK&lx3srS0~84WS`e$K z$&be^0ttVG2J-u;-ufN7QMGAEr}73HbynKp^;47lB&>23t0ACh^Yi5A0^_C=f6+H` zsnd0}y+4k2!{W6imY^>$?ZN4MZD>VZ*n+d!X}E{2_4>}VYg@*tovDL#TH#Pl!xKU2 z4?`ay&!76Jssdst8CXxdku%B4!m7#O# zmnEtT0Gc?d{c%WoKN|ImcDZAXz6>sQ+wyEBF5D<6&if(XTxIxj@ zANaGLN8?M@P1)M|OX8aA5aUj6&LMrz=lLalO6Nb63DkDW^_}acFMgfVAyPDn*}ki% zJ4a$p%q8(rI3=YR7K_>V|0)&-Suoqqt5Yr3nWC#YJxnMtAYl2Tb6o zLJQX%tg(5V(6LRv!tqu(C3yg?S2Kw9_T7eSZRv>(jRvrlp!M-GlwcjV=qU2dxaK1A zkajhu>^2um!04VVo%nDCJF+)+b359x4l-}a#I8s>j8cU-EGY06wRFDC}?7T2*8$X#aICr z&8--Ca)X#DZj0G&D!6A^;mhc*<3naNL@$Y$;Yan>+vC z2I-~n*K_ZGp%4H8xL+5ZqXN{f)f1PP#Rs+3m;Ighb{Bi7EIAqM2L06@1aJfX{m@nE zi&hRb4SS!~`ME9wZBqEA>;e5X8kx+_?&C)$8HgsoGOPo-a`o!6FC%tsH8nLuBCqh8 zm4M5KTrwmwZoVg4L_FHi|GQl{O0NRjyqCU}!_p7&I zt?c>u_4a7hk6erXa=-e8yp+qtMC2Dzi@Lqr&COM0MsbfFPF>bOF1a&q#}11;vL;;5 zO?Z4A{w#4FyVctJ)b zYkwzA+_Vt3Hxr=Y)wctU02yly^>Ss=^VW}u)Zk3S)my*QAFoo5Exrvf+hm<+81GiT zbz(TS_Gjr}q)HRwx{ad~-$re02w{GWQn9b#hW|V+6Eg7i6HAh5`Nb0CNya=7e7u_Q zPz)EUV)R~TLB$iNmTa;n)g0PO3tsZ-drwNwT&v2A z_Uqx*`2Qt0x{ua8X?*e9k*slJ@{RnsInDCjA!oMPuRnN-^0iT<+YW10iZgXu2TI@v zfl^Ky3Yhxix7ppYj2?a#q&(M`{f+%$-_Bb*^WixBnPHSqc_*bU3`w9Dn_Gj6`&o8a z*_(wmL`u34xr2YnBnlnJYfk;(RB@j4bgLbuMu7X{@UU$rvE-BW8#Fn)U8jrtUbkOb zA$5_~aQT$Ut2wewzKDEsh(6v24*d5Zgb!fmw|E8s7FH=QUgJLQMf z;F7z|SgMWo`>11|Ycfl$1M}h8QpZW^(Ft7az^lgW{tt%dIwvn%6(;UH^p6&x-lK@t ziM0jVoOd-S*cqoL$!pAEriC>5;07z<16Jtvag{~`=LSUx3fn;9mFNq|W)+ z&J?kK$f{Y1`FLmRT->#rZSii{^tSy`dQmPvXDU*x-^$M&Lt-iqi{Z7zMUt8PS`IAh zC~@%WK!4a}+Z;YJx&H~x7AWWm)Bt^ljJhLG1w@W&kLd*mFs1ycU%A~$bIEMIaW)q< zs4j;v+%95s5TnDOJ5)QxN6;M=4P@*Fis|*exPXF<4=URq&)wz42Yhw@bQt_gKGA@L zY@KlOZM>^EUvd;>E0!CiYvX{%{7`2BF8HSG^N&M2ww$nvQX{NDS2llie@)JFXl%&h zz5PD*aj?{S87OCQl&S(!2RDtz@Zy-sb16THwBw)bEt7&{fe2$-s@TNLTqQT}C)gBW zoR}Qv8E%AZ^cT}Dx4vDsQ8l|q0uSFX?sX=;tDd4Y`-Rb*(C>x|)`c$80oMa%uahb$ zE2<4>_`xu8FVG!5X!Jw)(Q=E!=Z{#sCcOO(JA@b&>tS&H)VUa7PW7X#XXgIjRep#Q zd=qHsB^IgA%3al;cEPknrtje)vl>~E-uZGOMVE&_1|!P8EaIDktk^}h5@OAskwTq~-h+SpSD~i$vnLWXu6;wZcY|16( z^n6H6F)FI}(U+O=nl8Vegy_m+Ek)7n3fCpTq4J})zNPHkOrU;itf3Z54E>RQQWm<>b0$sbQy9b_3CA@+OI88VqZ^7Od+sO?Z{npSmtfQHdT`Swx z>|Wi+Zw@WsA&HI*x2)J=dpcR)VGgp3G&4Lm3;sEYwG7dZ=~;*v=F4S^n*D+MZjcL zq-wcE)&^>%bJ$>mQsIHp>BmJ$Ic#fQ?GeyWHV+ASRK9(7;H~mz;aFS^!`9K10TI%k zvmoR5+bmf?u`*XrK5>QYeT!dY?JEDrNqV5o^1}a5LU#M56nqIp2>3~Z?ifA={q3;0 zl;2(fbRSE~u6>~6ss5YZ{<|db7!2?DRn-j`D$X5Y4W9uT_=ZpGX8dfpqCGMl8NH8tE@%=$%l2TohS za5fYLIPHCEJ^+Ah|E3%*NeTZ>L3DwcOb6nidXtuVJZPIJMgjnj_dqp%2bAyG*NyPf z)scZ;sP}m|q5K!%gSs4445ip@cS0|qI%5Qpqtd!KlJ8HlL_%o(K!zgGXkAdmT!SXC z2_2jGtG@z+d4D?)rrOwCzP`J^Exy)wRyLc9YtUQt&ZGu?d(@tTuR_g#KX1nYN2LzB zUAE4Ta=SBb&Jyew+`g`RHreY2IQNj-aRpXzWB%@UOlc5P4W@Z78X+C()L?=`$C0GL zp8?C#Fq7J#FB$c?A5YJh+r-ft$*oSWgX6~%dw7R{ropauw{GRY_=$@A|3n(-Z6%Sc2q}G=y+XU8yMIOQQ&cYM$_^Hi%gybaFE@9#vdJZje5!KHG z#ZjdfdWKUfqvQ@2^N&y`DHY__RRbEiU=#hStI=+X@(GoK3zP9z%wxAH@@5I>? zGvNXwA&^E45SzwD4@o7mTk;js3%|7QOL&4cSL*~pbv&=fAv0)km=oTw-b(n}{qhy( zXny`%s;obxqRF4LrJ|%hZlo3IdAd4hBb~u)b-)EAD{&-ijf!bw4pQmW$&j8u>c0BL zM*S5>pWJyc{QG^)-BG{2egV37Z~5U>7pfC2-nV3IYj(uT3iLs#(ofs+QoJ$wk`BRrrV9fQd3`Qkm63!&i`wYynGn(Cu z`Cs!#Eh^@WO-gJ9N3`ftAFnn|J|nzDe%?ecgc~J^?k8X2zx5g~6$E!|gLJHkW9viW z5U0qLmzSi+$)9{r-_oLn`6abJ?-(%2r|2}%Nis`>Yrd*eqS1^tldtAE ziQ?zjRAF4>aFpMoF6WnvQ{{Z^%sD$|n428g6AO2{QB1#WoK4WGSO>s*E&y7+cNjbCMs9qRG$k-egLur@vhIKnGInqHo zN_~hj7F0K-|D$#E8M>|b`ZH-kj^cumrt>_FLU6Y-GLTHWgN?NEQ zkW6l1Pr=xv#1d19=Gwv1F6NbrpZi7CwGtE$-`U-EMp1KYG|=Ds87A#u8;tV2ulc5q z1pm&${W1~Weh#V+ku*S&>6H{)HwGOwIIdyAM3#?<+DPWU1L(M0*$*2?&UX0)fp4SX z0AGg2YTcKVr)t|XzOOSX(kfA@eiuV_K!VnDROKYvD<8HNQ=+sf^$-V?&;4 zngl7e`@?!$fP+@YjCsu$KR#M*ZzpsnBdb&5te%btEtF}tn4TE+$ptxdQMzH+szJtR zct5ymR}Rh5L)!DXi%p0!*E=@+V6#31eDkw_(%T!%Gkw+`+hKMJW*Y5BN}mj>UCXsu zxZmJ!jd7tTYH42=%h}l4W=i0X5n8k2s3W7CvtWH{eP|L0orK;_>kwLnW ztE`NEA(6j2=dAP+im5FD&o;-b3)P<5W3Ff1ykEjFdtRTNm_Sd#TD(EliAL|=*p8yT zjPo{1Gj=2mon`|ICuSyU4}>7G+@1qNvAw$0VE~*T=AMG3G&>WI0kdMo48mn8Vj;H# z6%+S_x{O2RqpVFuv_LqGT7OrG+M@KGN%Iw`CFMb5ThC3)8o~C9HpB&KzFa#}fU z4rsG?hI#X2H_sp0pO(ur-K?y&DwIZ&RpqhT>u4dF7eRO8U^3~{8XoZhylGMTP2WQx zrOb*C$mA6f43EIUMVA`0sDD$9*#10}0NU5mgr!>O-;}?UT%QCK%c*eff)X zZg57BEsY-(m+Ie+NYbyG!4>p3vgx-MYGlW06zV1l1(CHI+IKs-TeSQttcRgJwe~$w z8VIn~^F_b0q4vwAFQTombT7fNkG2(@0#vq2E$pA)FoIV}0jg7)t9jrmiU_6$tt2nq zI>nZDqLuT2(EX*pT+URI^d|u%K#IN(MlMLo3nys3)9eTa<5o8dr5Rj+g<3!gRilTc zZH_QoBhMeyAcXbrg?t`M_0lh~p|5Y={f=3|@~~z+5WL`_DW;J9PfvC%2nnet>3i;mJzxnRisq9iyTfH$|NJk1R zHhB!cBk)@)+{SK;_<^jTK|#gOSyo9BZ`DCT!I$jW%`gL_`bB?(RA0N{aOkZO`5uTK zancd|ldp5wa$ag8w~E=6`Y??@JTL3#MU9Glq@M!0BAkLpfFuD&(0Ex8g}3*4n*XIQ z{)UYmRKRPHT-{J48Vv-~`a8!X9WF1o!@SN&)<@BK?Aw$u(HsuLp@-%S)782%Vy#5A za{XbuqEB7A)6d^9odQpO7|p&$;TD(sHb9R$6J3V)(WOXfF|>!h8>qXbx$PwMpR$Nr zkkEv2ml#uYkHa61ju&ah8i{{dJZDndlVCVu^rXSIH$HL1{c&WH9!#`&Q(ysmrSCKQ z4Odo{+Al8Wmh~^(dVL8L@!eRwXlcFkjQr|V^O-M@t-t3tjJ2?>GS3_-QP}ux>=?9=$CTXH9@!ka=@YG;LGzm^+swMfD*qDQ@^ zB1C$Lof;y&wlZlc5H`qqYT`9|usU{P;S;b%A~dmvGJq6?FV1Yj;Uhm1|yZSP->%bZnyOglJn)HES6_d z;X!673bwtCm}Ec~_I( zXX_MiP(7ERa;ZZ5k??SiTO8>9=lIj|cp-Fb*8!6%@=)+w{EuyCP)bBp+!oPEOc+Mb zCXrQQ2~Bubw7DUB&gPo70SStT=|{`FD`f3QoUGo<;v+gli=MZzhYVZHZsIJku`H&5Gy;y(=^eD_TKq$4q>xv-&@7Uc^d;P zwbS=K{#`5@JOf3Qn9mMyMgAU}6Z%1S?#mUsZQpYNlWMH{u$3qlJ3lm7>Aur3a=i1B=&$^Z6wKBfX~FF!rEN%5qYd9rPv4;jWr_O$!2RzK zgtVWjiy^lh-l?&)r-qKw>Hz`OfA``T6s-Q=e{ZOnL+77KN%~JxJyN-%+NJmaCj(#} z)%$n#KlF3YCY{NBssF)nEu=$}Ji8Lo@h;7GKSa{xPW~N|k&~0E-*sDO(B2X}0IlZz z7V6Y#=J|+q5^=TH4o?bA)3}qO*dV$6ScMqFOVt-3NE6j%ltG+DVT6(^duX|RbVbrapF7A{KMOwU?6?KC;Q*u>4z(-lbvF z>^Ge)PHd`#WK2{f?4OJFMu=TM34Rx4!D~MFjhS=VVOs0c9}0+Y5b`gijWZ( z3Z2d+9f0>fo7uU354L_jI#vIbRmA8TmLOZz62vgsP(seS#qgwdg|tAE=D&=8aL{@9 z^*48WQPA&6y$RzfG^b2!TZSZIuQ~6W7Oo)GJTdK))Od5hucm<*8EoP3@5pc2@@{iZ zK_w0vFIut0#D{cH?SzLQ9R^@A1k%A%DtPBS;b=SUzNoswPdtBX9Vz*fT-!!lvnrkq zARr{{HBHzjXe~|8_h5 zk8{R<`HJPhQHRYYyh%7D>CdG30rWK=*A)GI4xK?5LVrS=QpN_GLL{^CU}K1+Erz`L zowc>K5oCY!Vb4dm>JWe-M1|eX1bs;T@P(vT{vH+UL|TcQTgQ^}CMSa-9W5>AE3z^& zo|fu$tYNFz=l)cjqFna$Ow#v>8NBVR#BDL12_CQ-s9;@qthNhE@^wlw+rI9t<)`ZE z?F~PuV8xuXxPs;BLvNqs>{U}c7bA;5EJF<6`6C+YB<*$Y89!lJ`n03{XA|pnYgAI6 z!{n#2{_>-3KHRa?+v>sccTdKMZb9*6J6&q6F40ZH-=EW&&d!^^_vieJE8Bk~-HjD* z@rAuXbH+>4`pZsQ`KPBi88#GtAR)@%+dS=?R{|AK4@gI9AjS9qrI5EKew82B&*>`v zZZc*n6w)z65nsBLbz?uz)Z>@wDU63lqul)4uFzj=icOP^*B*SMH3g{LYJQAIuI2wk zOsG~h;(i`S9_#f=pFJ~)-EU2?{5Gfd1@Y}=_wj+}Ba0)fXJ!yC(XPeAtid0IM|dw!rfIsQJtn`X~%rEg0woFI`_&ZOwra za3S%cuzP@i2Q=wcw?`l(2`zBPxF#)_D(6j;+;VcA5O{+iO5g?|0S`^Uc?CElhUK_f z-BT18ZMSL7=q5-92OulE(o~o3GSqnA>7{@FTh|xKkN?x6)HZ&TN)gJDzolTz9znlRQ=R4pD@S zp-2al@?c1oR#M(@#C8GmCuQoh-P?J+k<;Iyk6m{8-XadZw>g<*t(5WiK(NTs-MO*b zI;2V;R;1~iB~)tpbDvCMM0;;SuAz<8%nKryo1WFdW^$!RTdJj^k=1dm&wSjym zrp?xjC$x~Fe4+5PbXnr__q!l=`1ZX}p_l4`;I6x+^2oCEh`W{f#AG2pMKQWgrw9IH zXMg6=DrVRaF2cKKrZdM`cyK46w_Cc(PCQ^2aM^UONTp6O>s>3CA)8Z8lJZ0V7%wT$ z6g~GWDR0SJJ}D0{BZYXvY41^kvoiBq*oIOU2|qi+H&&tBrbc7eM@fBYIVum?Vw`N$ zU=L);ZsYC3-@#LBGfeQ`j-q9Wi%g2jzkav>a@%uoDT1EpG?Utr3-?1VI&W!L0ICdN z*#RA8-=vBV`T@vAV6#iabB=i)SXyewvK=dIy?H?OCLtAz_?4>rdt-J3gJs zH9iNqm_r3G)0oT;M&O%_UNw_!^(;3%Ka(3Iw!rW$oV{1e4;a@+axKf9A0i(i{XOs}f~mPB6eW*bdTh<{L8lwbH?*c2Jp)F8+S>$l!$j7bTB>N0TO~xCk7`yI8=3UBYg%M|V`CN}QTmZiG;T}% z6uZ!Gig~9$sYG&kh=KXotrF-I-wh}*5O8k@X&?4K>njO5@MUA(+GXF;z(FN=tGkzN zf9u*TB+xl=ugU7Oa5aM5d#xGY( z-uin1>AA$!-DeL3!M0}ZgX~N=lavSe{sIy6V$!Sp;0oU@5A{Eka)OkA^bI%`(BuO& zNRp$i;MWAYPEO#NhQgRoYpAO$*Mh>m)!$Fw4<7n+n}qA1=Cc>lM~aL2#XN-phEa=h zG=;DQ8G+>iL{75JOe8j0U*3n8XXsDf)j(c(Ng3W)Rk0 z4o-h*(dDC&+;jokSv7Ghv?f$k@KO)gj~MRqGCu{mNU3^oO#Ey3kizwy#`Q?sN5CnY z0^e>w_1@RXI=mn-Vq!9HSADm}<Xzd#BS$_+Bld68{{R`lb|1c$XIP?w~s1@KjOx zr<;mCs4QK7azT@_zfgl)GnjhhNY%6_aYs|%pc+POp%2j`Eyk3PYtc#iEo!wGI6Nk4 zz3Y2>9ESkXDyIwtI=7qVoS??akwo zPS-x*<}@>&rqX7baw?Z;!fbJwsoWQ4#%aM)E605yr_|IGHFptaro|~!3R5ZAIq&m6pO-%ZG|7Iy>%Q*qa^2TOUN@81 zX!XYpe*Lx0jH-Z0XYn$5wYg0m9xU-RT%HHN0Vz5^XooEYLpy7*rJxO>_fVghxvE#Gt^i4Bw@0dLPNL0kAiRr2+GIFRT^J5r@DIp7??KQEKsFR#zH{1r(at#~)!!c~H=8 zyHp8!R=>yq$?0u7alQ?xNL!jRL-Rv*esrH$k*3`#Kk$_S@p}#B&DFj4P*4M16Cga9 z&JylYE~<2x8eJbRuY>)khRfZ61gMd~aS$hYuSZT_mH89O2A}Vd>!$2xT6Ka@+l~6T z$Pmean?aP}W$6a>Nx0sq@4PZ2hO``*8GoNEKEtZGy4(2M&Yxq?@-6$#pei*MQpqUD z>2rEXbxoym4_Qtk&ItURqJ&%h9fUaR;9eWXm1r7kxfoERFIT~S`0mHq9VMCyzq3$p zqp3QPNPN}JtfZumbw2;_fT=_(`Mo%=pH8+pik&YUpD~QcG*>FWzm(c`I}egE%je`n z%r8wJsOKk$4@9CvBYfsQk9ax%@NsH$M9*-Y{U;8+XB|Vo0$2pVdg zGg4GmL13$)X6ps6n?m)@#mdK9<63$y`Yg7W`|&SuFN#fT2#Fm#1JOg&=cOQbF@vjb z%zw}WH_=Hc)Z_He?8|;f2Uo4KeKtNUgLjHwE*LmE9*&06Vki|GL1qHuM_|~5nKGQK zf2!uSHZhrutH>pwNarB-fQ^kQ%s)6dr&eF|uxDh^%C9}`<#0{+Woe_M6C!*NBCCfEEs{;tztQN`}4vl?Sw`f%} zHNF!m`?@GSU6tNSZpHEPbW$i$Wx0)XzIUqEV!ZXkMWwF4!5=p#_SIUNN9Xxjsd!eJ zZl{G<5~+9*_I)cjIaUmrfq4M=cScYBS`PvPelM{!Pj>Zk*92;KJ#Cyty|m)=8W3~Q zyZCr@>*W6_JRZHMir5MpGAE6F^{1NP@9TU0PlQc%nvcVl{vlPLO!?P%dVO6hA$xh5 z!20TsVH+D8A+WT^YlFXpwaTXpOkm?NK3Yi|0t1 zRcydIa$uNC^Q{JvCx?+)?7lfac;k)$_3w3Q?P_RD9;660=1$1$Ba7^$papeN2y%S57;E zjbSaFZgQ?q9SA(F|4MCKap~L5fIwS4$JH!lr0-+?q`7r({h!iAW*bb%8)M1hnKY_t zL`7R7-y1f>;;55gL&JD=0&J*_EwOlKWI}4%D?U%Kmv)~?(fiVoq9^;5y*fqOfOiYk zqWVWS7VPrB`qyat5ARN?Cf{$lu}i%m784{RK1Z?a#hm0L*xA;6h?mzUa~dc!fem%* z!_Bnt{N9FtVoUIp}ITECULP5*J`gf%yT1oVT&a-%JGw z^rOxA>oLum4REF~E>kc;7H+7pj%Gq&{(*rvA?rWB@OHUw-J)PUA_YJSJ6fvykErdG z?W!ippe7Bru?;a)O}3G@U#BiMi__PD!oj0inH}CvD%hE|FB2r zzSUEF@!v zYm5V6&$*rQ5;bVdUfulmUKRvqTAXqZ5&;0bgr}WSICTlPryOQ8kG{R~{O^=7${O$R@X=Cr3aKxUkTsTj!2gdRF20Xcj? z8{GFle7zw;Mro}rLHT=mB?P`w155J=wh<_C(R!WA;k$T}Qk5prY)I~u0CxO2?R&-c zL5WhetM*a~NvqhbfO#AD9Z|CtC|B$f=#=h4Qy({}$rDfMF3t}QdYqb^n&E9K@_vj#yuR=4V`SXw?QCdyYm7Atr_3;b zA+KuUJluFW7>;UJ1hXS11PE{J(o-A-C{0AD$|VYMmw6|Y&*@$FSaamM347czXOj^D zSYu}pM)Y9obS0DAIJp?bWaUu__xOUR(}cA|!{i1Nf{vAu`i#KBqnaFQWa?tOg;wEk z@N?c_BmK^1;)ML`>#1YL53clNPYpTiEs1hKpTH+4zIYVTX`3MW`n9tCctg!<1$9|E zzSun8*t6JFPJDSWvF0JP6mM`!(YuuqjgYC6`8nu3h~Q3x=vGzrY0GQ#;c*R$RvN;i zUM?z>aHj^NATSmWP2)kY;%ZX!dP3yB#?*)le+HMgela1FxA&+z(9}_=N8?tg>}XMN zr9W(FfAJ#7ctBW8v=)`a3wgcvh1$+2_72^ojS@8nniXA$T2hnONedm=Ci7A$%WiD8yU>2M( z7g7}4xuz^1WiBHL55XzDDx$V$udt*@2a= zKD%?qOiCZSI@&k5e(KpZbCpW|x-}ps7vi+?$}kD_W@^mNQN&46UdOrE)%wyXh&5Cb zQBr^}Sv}uvJyWR?L2lU_W{%%${2a z5Kl~0yV}LkzmcQ7&tEhnE(#up7o8SQQIw_87A_9NCH~|3?kk-`GYJ`il}_`f5e72M z&1v3vty#v{!1o4Ai8RC%7x9^h20zm|u?gnCaW&k#jT=V}-f{cuVSv^djdo)m)M>RhWs4;6Z>eU-KwZrn3Uz30|=wDdKlA^NGN#w6` zqV76gT~-8{6TD~tyzyUeF0Q!9n*{?3s*Lp68~;vI8n3+80scsb@kQB3o#z)9#+<8l z+QAel@YuEuap6yB@mJjZPp|57;@{(6TgBXH11P94&@B;YjSj#TAQ4}GaRhty-gU@0 zQatdv(Z7C*Ekv6#CCUu#E{-kL&g{dMW?8fz!QSLaH_k2?odJZXQN*4E><0qF1;)b~ zgBgi8V*s&9{JB7;{RaWC|hk%IqDx&emCEa$toyzRjkB^yjT7bl!%rE;0HBRnh& z;0l`I8_#A;hEjTvdWoNjyvPk!GJ&}@=pb|iOBAgmh+o6sZXB|uZ|@|K#7ohp7&%@1*PGyEzGNX1ZdXm4bqE7mO_}5BD;n0w}NZ2tXAQS7-nhAL(}sj>$aQ zMwk)=!|q40=DA*X6e)RUl1KMuycMboRHu`|k39DvAb(yzH6i#4w#NE|xB+XnpOE>4 z-nL^e;Vy~d;C#Le62Ur8lkBgE{zjUHWDjda%1f3)y!5z8HNTV>H0BFtnly8Z%LfZ^(jjw{A{cfu{H!X8 z_ADA(IzDQSE%mHY%H#4MWe1>uA$eVx-s)Ks9~G2ro$)Iq;=8p6zW*~OKsQ}hS31no zQ`&txrPjDu=$^(l9@L|mbjC%ID?XPRThkWbWlY)M5xoBo;knG|_*1ezd{@8%3cnVb zZaTOf<~sZ%_Q(7(NF~r~QiousC;l7JI)eQHWa=EP@3vHKLxyRcGYG8Y#V-3=t;U@n z3gq{8t|7M6JfP`_f2zm6QsYL#@JxbsqhG7CTB0pDCp_2ke4oI$g9%7mizmN)`6ROr zTFUBRr*#v$yAQ&KKxgj+)q+|I#$=Z?H`LXA2}^sk5x)!8`VRa62j;hVf+3YDUk_v_ z-^={W<}K{KVDNXQPj@Erspui%>>}#$3*rjHJGfG_CL5}`019iu@RGj~o^E*M1ixSs zkTkclP2sJ+&_*O3M`t2eT@%5*k|>4+$)-1ef&|!;GB=W zbr0UsI}>71S)vY(n&b`Z#r>Uyh!(_!(5EvR(qp35AMY@8X}^8om*P63%nBmy=*~j) zqmA7JZYs3DHo0Hk*)u8-Tcg>I$-hR^9BWOi=h+C($ceNE{!{wS)11gG zjcs?#F2XVNcYDZd$1Z2!}2ny~w3cB5qieFNHf8-s+t@{q@@&9E*H&)Ap zf2@E?kNQ?)Y5mhj{MEl}SHN`fR%d!=5; z1p7WX*9QSrZilE0pkZodf+YY&e#9<;)8qcwmj|*T5eK#czX)9)J(RRcR`eXM-M*(% zjU>uC&AU=G!FXHp2+AO5?c#Qjp|89-2tx0Yil2v()`Lojy<(n}7uHB0kdA3VyxG{XIn+VjCcK&r|4CZqZKE!RAj)_3^J zp*?a*n&8>z4}4;(ecYauHn0>voN4b{00S$`=1;h5Ms{)AHa!B9j$wH|>UYtc{n zCdFZkyf}h(;zsw>zH|5ZH9l>eWi$74pP=uBo{9^37(QjgRh_vt8n@Ob!xft#+K;z{ zwT=|ro#SnB#mRgh>~CP_nCCbNf`ku8Jm8Un_A3RutKqZgW0=rHpljn)Q`TeAop_*+?Wo zoc8rW6Y%v-x?V}j@R6qaVjP9M0XjKg!}v#|;m!-&p3(Zw-ikXh{J96m5HO$DhEm?t%3Dl2j~RC?u+k zJbm?q+Z$<72tDCP7bOd*?;L`(@%O0bQH9+ff%w1p^Iu9TwOk01+7Gs9?GA1N5~S94 zzVrtvDqgr~3hu9yEbx;*h1&NYZ}6G6{1S|S2GZ4vs9&qfQey+PRB8X9tqXPul-Qia z{-`aK=&uJ1-Kh}AlX}k8n}J~&GBWUiP`-@BMrgaS3Yalp+FP_68!D1*EtfPKpq=XH z9GA3^EpIy0x*sN`ZAV+C_8x`6C`(5v5UmX1Z&h@ia^2k)IvGBsy38kCieJMc_Vcc9 zW3<)idbZJS&qP2U%_2&FiMFgB3to9yaRh6j>Zhz<-@U&uceq$Ll`SC~j zgNHR^Ea>S+hc@@zXgjd0N*eRl&*;@wJieT5nc=9YpaCZ+qQ3zj&I%`!$3ny*11X9; zoA$9cZ|0dz(CJDV7NWvbLHEM^4VZ~x(JM=t1tN`t<^4?_R634t=I3UqcP=yi&vQqK7(C`OMq(?iY<;&ah?njPn^3-Bl67+yi5SsbXeQnjHQ z*Y{@xY*q)a+?OU)7st3&qZ~ZY0Da5gsPD0jp8cqHxTVl9Lot(QLyEQ8o@(VSbc+;} zRG%Y1eUo$x;&v00_wg-7Xtbv9?BvwMjF~cV4`oo#N!bmgw2W(p&b~{^;)heu5$)Sf zt`qE2lfSiQ7aIb2u*O8#%{UQgcJRIpJZ1k-0eX51aP*K%w2ZH~yu>io4+x)u=T(pO z-eT@0RnRBj7E{g^k`aQhPV2?ovNqekC^&8nU&M|x??etRiZ z&XG!|0M7%b4Eku4nMcf!P<*;qQunHVFeGxO8vwVX1Z%s1!A)BF@+AnFjRevG$U> zwJ}?c4gdX533QI(Pn*iR_e!`Z;4S4Avpmwkd?4`nzdD&em-uK2xz-2&3G=CO|O!xOlsMK zrKDSF4t0iZM(UfHma65iB#DIi1#eU|LAduv5@nF^Oz&w|>Xje@p9{~^M`Z-KL&f2s zbQf6yqwl$KfE}EJcpgKj&9I?t^VK7vrcYx_i)af9KapsDAXTnMW`7*K=#6*4$51&d zICP9;p4GNJa5+9TwU^0Dpz*D;nflhvFC*h~2_17Y9(fQI6tAhb!awUN8}HhX)IQz6 zTkq@5oS3I(b#utU7HlcYIj+pAYv&X50E=~k=1RyD+-=)aRwsP7QESTjn2332hAC```L@zt9wSn2pUKZZ_PwpX^V z1u{t+2dMZ&p=Jm3OWoUN&N>MOYxL?QYTyo;J9}K$+xE$L^*NL05z%^a_i`#$b1C0$ zZW2Zmag2`ZCissVocZ8bO);%fNTKKN2#C>>;av08Z72ZdL$FW%=2oo=t3c426drcC&$DJHRXw;Nx1_ zKe@1AlyIk9Cs7qUUbk*W|KO*F456%qf5}$zvcehE;eiRg-q^-OKK-qF=Nh>m-*%3p zqagr)!ddbRD0^Zeh{V3x6GbT*9iq6B#(4CrQOkTt##95&726a*+(lCvkK7~30Np!| z4vsELd%LYO=R^WJVqPfIt5JPkROzidr^fz5p>l})()y%)D9;f(NMlV3>SxotmiQ(? z9)n^ZWdxCPi_~xf=W;_4cYt!CzAFoTlr|ejd(H~ru;6bbcT&jfqg8_~07ZKs7r|jH zp5x*--u-Ha>&G#ko(=TCy_%pSVVk^l1rB+fe>ihc}6NY!Ud61m@-q!*M;|&C}XPxVAbqm^gI>pONcuBLJK9zdu zpruU!XIp2?Pcrm)E%|_xL3~k!a*X9y434l+$BL|rUs`L5wm2U@L4CW&h@&6Ca-pst zuaEFZ$*`-j9S`eV?7b)$Ky{5Wso85E;nGWHo{!5i1>%|&owXffTRM){b?Xf_zrgx6 zt{vN_Ox(**>-p{bDDHj8ynE)G{vWsh3@!UNfz@*afF3EU9^Ct{X>t3f?DNk6V|wS$ z12Yu}02u9<5Ku|MKXLZar(Z28rHXe2>5+F1X`#bC)xi@-?3ZP~yR=TIWVmcFyJv{smi@!j^lydb=CEo?|Ni24N*;k+oD z!nqPwMIDy>I1$doPd3|;xbA7w;%L=*u+N8iEuWz4ewIqs{g`fKPd7T`XKF;-delb3 z*NbXHy|@jq30t79QEg&(NY_Oxn|MJRjBmY49MK~oWXYPICe4>(j^4Gt=XKF`Q{2$g zYm)6JJZ_X8D3%PL&Ml@jicyP2!aRtR_zLD>*IG({FgjRIX74(e<*5fTmcy0)@Z|rZof8AEm=*xfe{ou1X1hzQx1r8qIWa0F4Z8 z=*hl>-{C&XcH=iorPJT8{@Jo3KxU)A^KD0a11LE0kI5GD(CW9z=Nyaiq&KhHeXxzm zXFob07_Bp3vgV(Ks8fcE-WLB7rvK^9U!`t8mw0FkJG3-PE1TXazmgM$AUxP(|6d5x z>yFRZr{}b8=;Kox1i~aBmV*BJn9jfD91rz##VS-CkQszViZFRsDGnGIz^K2wjM@68 z#-Q(PZ}%CWNr1qzvOv+OTY)tU8>|*-(PQ)LU%hB+`9Rh3M^H#kR#pc?3B2VK0WC=x z@+m#FDQsv5JoWC#VF5a=#6}SQN-5oHp5E0URv2yzcXwpX$~Mu`E<*?jLG{EA8ap`?06m^B zZ#D0b`rxla1V}|zbXVh)iy;&p#}*^>=k%hHf=z7Qjm+oY-LAhnJ}ps-OdUa8=`WV3 zQPvek09Rryl$?0xn~^;C#5n!(03v3Y;Nwd?Q)yGR=YgzpH-U4bA|T=Zh4p3<-B+K3 z1tgr`b1q%85#QJ>cFa0j(nI1+d@JN#aENZ{W@H*gxEA<8l}_k$6Cum)+5+^(1P8Ab z<=sXYH93i~{PjZf4F%Hi*-)E(GsoDCvH7qp<=cYZ*5ZtS%YjD8gYqQKL>iITP$%yk z%$X8T_YA%|;g4>KreiGis|yKrDSqLNP6i)~gGZO4q_e6o$Ctk4>-FkY;F?+{`2=9+4sFK zHGlm!x4aT)rND6qX9d)CPCysB0j#3U(f@5rzbN}ct0xUS1H#V7;1tcwTmvm163yud z0?G8o1R1t!8{_N&>^l)wu~Y#ye`mxe$Ye$R_1E5f`wE31a`44?JPuNnNcY2*a+b3} z34Zi=tChWTAoNL1tR@81*KtLTm_a>x=i^?_D$R+=DJ1h7aDH5)&;j{MwNMp&T8ADK zU}46J@8OddpaTyb8x;gIGDrCm4vv}*hl3LrX-7h4G{bP@Gk4*u)EyKIB#T-Sc`CiW zmYBKVjrMh(s06>7qGxRNTvMlfz$gllZmx0&Pj4c#7C%nQ0tYp-G*3Yy=ap#GjjvRP==lGgUf7`71?h53GzaRYc_@~Rq6`vmc?%VsN(j6!> z=T(mvj_*Yt*l-DQIrI9~uMTXkZuDNmh$Nr<{`+SG8O2m~Iyp@9#p$`+mUwAK@z{Bq z9Y+Y|Ui*e>aDMO`>ghpZPcTm}k2i5)GQ{@f8XOW^`Cz3~0k&mmkHbuNS zww`s`wVy;$eTy8rAfn#?8&giq;TWB@I2S~vj?_8R%{){79lPEQbSVRmv9!w}Uvw)w zj4us@HMB&&N-T2y%knMu9*&R?zs=IO+zCrcPCh&XS5Lv6`#5xmCUDRg+L;|?U6B0- zswjcl&vJ5mA+Y*Di`+mzeR4xXF2t|K+hbZW9o$?m?&<6!DY$ zEz!*>uTw))Y~grW{^70L-v=(^+SJ+ium8>SUjf~!=nLlXxnUhOa+>iJg1`Nj0xDU?E`YQMJCxW! zKb`TO3!8Pju0!nX?YEXe{oo=KI9SZH1obS$hwew>dQ|vmW-eRnMq}X7l%(RC_RWY1 z*p%qWk^?cr!0f1MK2I@NJ%;yqb#sB!Apq;(JVAA@KI0W3#Qg1>6XS$=%5J`Je~oyc zw6+aj=#E{PK0cfoKUQc@@?wXKQyR0L$s$`4mE~cMUY+8@Q}#xIdE7IEQuHZF01OLX zrfastL=pIv^jrugp=`X!Fj8sEd~QH1sF2uH@F;5NLV$GtCO`Qmo|%j`PS&4_re$o|o9{Yj#w*{D7t)^sPunO_1}vj_t5`bdTpl9(hUO0^H)9M+Qyr=7IyIDhCtY9wARI_W$S^YuR7f-8K<8Ea4}TRtqhF%d}w77++vKLTUVU$AKT))$v2)_^Gel7+HF%lkpTl{Jr^JG8j@ zZ#HV%xY*e1lYNS5fqGmNjoFyNF@@>s>E%M&+S*Funa$teX}J)s?48^v~&j$As?stQ{3@J?Q$hy`tY2y4-Vd$M{1P_{4;pQ7f2B)%e4pKOaW zspii&JPTN)$yiZZX(9s$+3r!z+$Fal3iRCUmSPXAV{_{*NJ~aAlPb_R z20>MYPY;M-m@CT-c;}+RYU~6>yC)uZrv*!pt*zXtcxPphT>0yT-g|TTa?b}EH%hH6 zWM6G9rg=3i{^4HJ_G;PoTz}f(%$gk|3JVtPWm;Rir^qeSA5l=Fj4P_)r@AR3$mFTZ zb0fp)Z!6Rx*vU)i1qo;yM-S(#C^eH+Xjxnxi^%~we?3|wG=X)jtS7**h1-&%oJi91 zciEit9)iE^JZx1hIMr}JBwI+7ijTKoBn?{Pore{IrCLa@5~XwsW#7(2S#NR2&68?rYqGE%=pRLbVB}$^3FBIBCw+Nj=ZcDv zKc7CmwjcS>lg+h!8FzLRCl9mQ%}Wkz*>6N+o_F8oJ1g$HFkBB$UaE%l8t#_FFQzDV zPb~$YjlmAwt3bN75Y}aa#qf_4R*b;^?P4%6G6H}jDe2G*Ut$l0;0){;qQ40&zkh0% zCamGamG|ohv^gE2+jXo=S)yMeyS|3NT5CNcVlPChWb-dST$6IApVNEs2k?#3Av&Oqmn@HA z1D7i)I7lk14fPF7q_6G)x*O`W*EDYA48{6My%Ch(%0_9<0X)*Urt)G%q=TdXH4!bU zA=s=>J|~>d7k8}O9YQfoR6%GB733S+pg~eqrU27+P`>+yB;L#6xWiz**OO&rxmCxO zp57pI^ej1lzk!UG2FCoZ334LH+Y4o*QrRa3X&p){wnc(h=h`e$@~2s-{T3-FJH3x_ z;G85z&uwc9BHv(N$9lXPt8WTptD`iCXj2{*q9bN@Tuw8ts5msGQA3yErm%VWjB$9tfp1(WT4EGtHi7IDHG?hN&i z(~26{>8lGnQd*aA_5_RYGAOjpre>nw6>X;+8ZyUmeBbbgFq;9OV;13R0^DXDz3>jx zRieU#>EERkw;3U3uH_%}Z_F{x>mNb$Z(cA4L7RQ>cvKU0D*01$+MfdS-?JRH^l z!_LjkL11AGqh8{_Foa`}7CN`p9>pSoZzu8^4rmT0fOHI9GX~TeDHcLNPVLQwP(*tv z*vY=ux8>bFRU>C9YxON4uxW8Bc2Yt>XJ99zDAK50h*GJ9)mB%;@b9;@5S3%o+2zDs z`E)#X@~HYzm1~>)83zyXqpxtJHXTryNRKV|INDZlGjUlD-tp>_Jt0AOd`Zbxqx=Oa zD;JWLX|#1pc@?@?2E$TTT77V-bA8utL2lFQ@Yi^42@>H)VWHz_+`ulUTrczT@`@h5 z$MtjGH7Xh7?B9fZd0u=Q6RL!nqxV&@`csi(jiq=eGl}eJJ>3W16}p?3LmrPWE+~27 z_3RKaK{e9v9OKbdUO1EHHP5I_wc}D$jE)^J5yF{Agkg{N?+aE{B&w9zz4dgLY~y^s z^N|(}E>c`1yhcFtcEcZ+eKwc>8IGT3oD|hod~@x}qi(R7>qqUc3NLV2_YGrYw%PEe z@+HF6Wz_HHGIY70&5J#fy~O2vu8|y-|^a@6F{0jlXkdWvnN`mS+h0!#x*1Lyekh<{`a$D3;BLia%3B z>KRB1Zs?`UUv?!-*>;7PpJH%4;`mHKzb#bYif%q|Q{7UzkyS3mmDE0?>NMOz`@c~Z zhzf@ps-0GW0OQ*lRrJ}$wG3tsb8quQKr8q^9zPcx`rqchcbN1~AdubJkF&%y$ugi! zM8oL)AO8HbRjXb{Sgq^Yg#ssn0O^sn^GsUmJ09{|PjvL2xhH!4G0DFM3jx&a%0q`Yv07Q04y zh_WMwwkc_KV}%1dZeThy$})8M1NX;K zOCS0&z+4FKQtKd8btb_nx+##z55CFoef-^;3ai5ckK2vQ)cZvYd5eUTgyJU!x!6(0 zJI-nN97T;u^`_+3`MR^p5f-r;m)PpnB@Kz{oNOhv)ufOS!)Y0rnVB$bLt7gRi&lBC zgNLHj4zr26f>x46f_K%!fRSstwchE`mbpb03F^ZLjZR6@*!qBhhB9LxWYo`xiCuGF zAZts=ca1r>=oU#z`}Ui~RzbLBKAI-95C{bb!*PbOrW+2!uDqZq42ctl*ZPC#pN5mz z!lPCl^HlCg<2F)ZnsfhFO{^0kA1!9)<$hCX-zT3xsRJ)x@N1c@=I*_|r zX>#t8zk9UzZ7|9GGocNUycDh8y?D;rq!8rSq%iS|)PbGEAdeniD$*#(4l~f&6J$rzU3J|%?E*jfbn|8DXbWS~PE@T>GTRDwt@}u&g*PiVks7BDt(h&p)G?bfP zs`fhKK#4aL*HBKsih2$wG^%)LQU#Y^L|>tnob;LZ z@L@3;!yJ9{6tm3bSEQdJzstpbjlRl*whbLWq<^4MI_?&EK0;Lin*D4g?yb$Glf#L> z9P2l9>We(UcFT;1w}m)fGmjE0-gu-RTmzS2YSV5V9FX~N6giMyP6mT!0#mEdE2p;E zQXLHPQL*JYS$J_h&J?D8%R|LrCmY~iZz`HvQ=`22_Ri+vwq;X^!jY9)E^43@rGe%8drYo1RW)B zK;%+j2|&Ok<>j7tWE~*01Qtch)?-@&cfxk&_U;NKFiLD~l9Z#xhmH243WE!1gVp&Y z3%)=JD@tx+Qp1KB0xRd6(pSkQ8N}f19Gl85HeG3-d-TP%GTE72{_cs$7jzX)ARaHO zXQcUI;C*tQY|L|;+I{@tIa=%H<_DTvy2d4IR%T@(L7JG55j85re@}`htzuxdUtSuF z9z}bv9a|Slfz>elgKoc6^tGA>@e-ATZigSjj>~fJ00MRik?LC8q_?c~nVI^rrkksJ<_RV*Kpxqo~B(VUwIL=MGi{ zc|(RhKg}r-Vn7BGb3*i@}(OP-5UU znV322K+~*{{a1y@!b_Ay|LUl=-8S(jS)7SCzmIX~#mr{LG?lq8DcM!+W|*m9A~Kn5 zJ!L+6thk&w4o|h`oXbWhjt$J;|G6YT-Jg1+6;5gmjF&lBnVIORn$g6?_zHX=`K}%} zWQetzRPTm2^^P8_MJ$V26-#ynBRqoM#{pSFhk}d0i^lD{|cW~OeU|<&q>1^DM=GvfK zrru*EAkG~GyH{;>vSadmAh1c5Cpy4>BoFxeSaXvyahnc+;SUeJy7s;nU>E1izAfDW z9fG)f-C6{GScz{mwrNU&lX#i*4*ys4=}~(9H)DwwN}+q7G@^$uRIanicy;HI>vOu$ zqhh#$+XJ#uJ$7f+OnK24x zj5q%-n-mkM7a)uF6e?%vU^=`Pz07%995L>AO>NYykEaA^jGKCj+@c62F?VX4gl^_D_bmmX z!i&+VO3VAU)n5R)4=&KT0I8`6Y#kP`w;#b^7+q z0h9rzqIh-lC4rFur#`M6d6D3!oH>D=eBGR$mvg}I&~QR)4R6LRJ%@7#$2{*%sig7x zmEx?iD4-AcmIRm-YSCz^C|Tllwz(>(c@%Y}a+2 zP4DM4A*i%MyZw7AYSIgj?exiBZBr|VJMGZnRP5*t&YE(caK|wf)U;h(4Mfthjw|T5 zXM2R_wzdqKRthHWvl*$Q=(l~X%Lm8N?14VV{Rv!An-}(sH~P+L>(Ll}kLsdh72=Id zK?$|Nf}fxgoV$|SmO0?6Ttm@-P`csGwnkCA0D-L|FMq6fO0^3 zwX|5PTEaLo81}-23y|JQ&k8K0e+md|_w>m}{%JvbxH1xb+>9@We2(C6^I%eH(!tUm zJDVP$Gunt;HTG&}5cH1aWqXgM#6tXl>#MNbTJ`C;gdf!p+j1edEg7N8#!0-TYtO;a zLcgLz77XrOO}|Hc0wdLa%j{w}Ro}N3ec#lvnvrN!Xab{Kamtd|!)d?hTi{|f2}Q9- zW6{aLBDqA7+kk${8UFhd$ku<4-2sqW3hqwug^6!u#zLF>%x%m%6jI1BElVDsUNq#z z<;#9x3F}gS5-gNBBW^jiaG@{6+ROIywoxhXG!>r}U50PW(S9Xm#fSRqT$fJzS(s5%NsDbyvB%BuADp z>Wl}4)&0N>*XK9Jrv(isyCP93Hz9Vna4K1w3ep=gKuIsMM0;5{G~8A1m~u1@HU!R- zTYs7+?RF&&cb=fxK@9Gbj9FXCjr?m4N$$W})Err7TcjP|pGI!diCx`+FwCqJ9&S-6 z8PfDEHcwB3(aH#O7EF5`;e2sQyFgl5j!uAnVU$S8b;hSui?mt(by)0>eUTv1&NO3C z3j-YEyr=#GkN@pTKC(LC_3rw{@__;4ibeL5kVmnT&u;GrOi*sA3B^u!URn#o;>dHM zFs!Ar-f~+vgHQZtAM@rwST1Bn=?oi69N%SddE&uh6Z|ONor>NAWb!Ea2hAr;bL@|^;R_}mH)9p$2ft5+HTmu4x}ASBmwPw zU|7!zcN48aa9GaM3Q}+1?s~`AzWT>KSh$%GWOEf^T~Jl72jUMJVR*WqAye(jaL|2} zZ5Tb|H$xp8$7ZUC)1phz@dS#Jzo z9fk7x^+KW_5BjWMB0$WpSnq}!#vez2H%X#QXU*m~ln8K_)0{H1J}cg<)V#7m9^Fmy z4xmpdGW}6J(2=nC1vCBm;Y*1h&nbYjcmN-h8Qp1U-F4WV63CNHB|-6XQmQNZo)St& zVPq>td|MA`d+lU@uKl~}r?F3*XVV!mD|R9&Z$NwPJb48FfBa|v`-|6G@4Po?99I^) z0MIKcI?)VFZor&P0YQOb&wO!NOSb=cwYoCxy9JUf(S}?Ye-jHOmOH}gfk89Fsz!g< z?3cHwS=QN~UuEri^KJFH)U!b%)kCNT=ejZxI^aEI+Q7m>9MF@1z3ZS>q5+eRxD8G6 zrd^kp3?{cZD5n&WqDa5N^Sg$B>R4f^<68#(^>RmNAiZg+zWa{3s`mrYUtibw2zKSJ zhk{J>wn_Y3&BPNA?5eRNO`LRpWXc$5g16#GmWh|0b-TE&LZ9H@jb%67Q`Bz>e2Lz; zVljxfk^Jsjy-q0_s7L9w6m`_jxTu~imnJ%Euh_;EW1s(!?N-GKj=ire7%X-I9%}5w z_t;W?>mN*TD&DS-0#s71@&EZhycmlg3Cn}rP6CAdHm1pVWWj;Gv>Ip}YN)0bGWRkyL|bxT9B8pP+5+hn5HxQf zy-<0M5L^kKCF(ou8`8(vz8WkU<58Rq4e>6QuVvX51MIB_ma( zL!J*#+3Jco^$SV-}iG4 z--&0(E2bKpTRi%~PBt6va)R7kMW0Mi9)0AeSyoelLvVlZO1I)fW<({Y3^wSrC(E^s zN81eIVG3oY+WY@rd;k2+zbwKPQd%sL0FJkF=OBIyZyQ~}OD20WEeoBiy~0zy)qjb+?p1x-@*qnc?%ySkE7X0WEOFjuj-2En zF>}ULL}1XI3SGEu4poP2`A2HYt7q9lE9UaG-3))zQKpPV7cJ^sZL-+MRorm8g&p9w z9dcZ3i}&T~IVW62XLDyQ9-lm()k5<7$i*=EA)5m-uQ2jHSA3h1X+*on0}V?peiJ8L zJlGScfY*J-*Vg~F(P?Sn;GJk_EY0?D)GCP@Z`d3tmtcQQ1mcug``h8 zfY&9&SssL!;mrs%rcBoCPK0UrQwR0~cuCRM` ziuVL4Jvldc%gH3e$26ccq2$ev*m!_Yn>ZW)xF1c@uxj1AXZC82a zRvvxpM|e}O$nTwbHm=q5;tWM-pJxs0iLbh^^x2SyP-g8`bJVvoBb&s=jbY9Zh zNLR`^IYUAxPrjr;eAejt6YWU521%h9Q%`DZA@Pp^^`Tffc8D>IJ;PNkWwsTy(?M~bYJVLo~7i??-e9y~$hlL!Pe}f`xWJd%{ z7b^O^yylOq{O|ty=RN@2Nif$E{xtxU*=;0r-KdW_x|QhGclq(3^@9Zju z_41h+ryp~(#BUu$e=Sa)%Nn^x06ImcwMUy47Ut z&|Dx8R4S>Q-*w;~Z92ZybjM6mAwYHS9-;V_g?Y48iCBD25Bit})W-@mS$PjHz2}(e zPL>TjR-q7##fIg^;j8D1kWX0ZmNGk7VLf9zRP+Tk`k(RR|Go$RdIxI00EU=ZIp*t# zca2rtOr!kX!0ZS>@xnsYSt}?{^WQBpG_=pXGkYB2$#yR^tq5I-qke?vt*tg=jKzpN zZOHKm6)5y=1k!HDMH@f0J@7a+B|N#3SQ197?>Xv_e+1w2Yc1Le-4uqX;IcQG0Zbp3 z`JT?e0x>qKS~`#k$LDXm_dKS^LyOk+L)=Iv@?CAsO8}EZNU@c%+r}ZO-o?Y z0d)H83n=kqFcS{~i4ja`6MU^Os*Z%V0R^vnuMPxT@5cHMu@DFM)lfiGqs1yskeovGfm|8FB0bQ*H7=ZMSB;%z4}xBasH5kdcT^7 zojLcJ*~rptcrUrCbyyTLnB-Iy)KWE@TKrXC5CAB9;?&n0nWM=QEzS{p<%o}2i2FvP zp&av}%?opwYlZ%H`Huco*vxlPz2gun%x*+B*NACNrvnsfKr-_sCf4K*4!?6L?@($zv-w)Zk#3_V4iK{ug7I{OR}yhdsF=t)8Lvm zAA{NlAKT@DBC22b!)d0U#l#{=tmV0N*N{qq7z0H*F({wVh~^3Z%^x3|vwsT1wz=TK z3NCKTG(sYgj+woIN5>cd>0$)aoel>HIe%X031TDtv#@H1#N~H4fY^KEISCCEHOFwW zQVd?V5C&zwU%ZrKWHqU&y=13%pbL-xie299n2Ar8nC%8O17Gapu)7OGpUvI*Y5@=U zY0sC}kC+Ac2^&I!8{O5e^iDgJ$IHn|WstXgL!1F)f%p%q{b$CU(@jq)-t=e*H^dKf z$T51B$F`n1F=?iIwaJ_IWaxy&LAHO3PhV)8f1{cEW5sCf-(7@HvzdLHQ@D$&dSy`^ zDZG`W>$E38fdL))elwB|`zwblCC_V_k5A`yTaxz+oC>SQmhlZOD<8D!-oZIZsgB(_ zvi3D@#GLNa{5UB*hK#{2+h8c`sqx%sFjnpgk`(FfLL`|sD;GTFD%3b&YIzog&M=Wx zsR3xT4o%(q@c-xPXBxdals3(w;mKE3Jd}>z(i>1`FvU?Cn&{8;J7L>M-*ODp%G<6* zG|~^+ay@CuL($G-H0w1f-h;EA+ts>pdUqq{em(zL#++FMN*n|Z+FuclB7oeWQx2pl zUGzKpa9XPM9ntOyrrEpb{=g=^>a{x5uNH4-_NM!e0=&3k!@X>^hR9f`;61O zlhtJ{ulfP{mopZDUQx*V8@-`mwq(zI`o_8sq0$p&DN@zK(;{((1FxZb zonIc`*f?vE^se7ksuF07ESDxXK#|G@PIgupf)-W{E9wozwSenl$W4Ske*fRmui&K& zt2I@yCKzX~q~igJGFT1(kJ2#t!^84i`eK}>)Pn`J{@Ipks%v;;`>|`j?XPqYsNtnt za-~_~NL^9fa5RAqt55X4@n#yD@G>p!Suf`vfn7N}b3K4Lob<#9HiYusfcQpuq+0Qc ze>$^*YrFV^FI}9DSWd_^LM%1nt2R%ReN}b(r-ZP0ph`Z}7`Ydbg!I(LVEiWdxcVjLbKawShsXG*u6f;o4>ru)}-t zapRxv0U=cfurG>KtiTM^UubDtG}f&x0`=iR7@6hCmV93%FW~HHUbKpp8+-|+l%AF| zh<3ONw}8pPFt?G54g7ur*(E}LFpui0x@K$#CLNtARfQTuMJ;|SkoDh09?h@X8(8GR zKZ+?sf+qB6%1UQE-0n7kcW}PN6nGmK*{-tqq=WwJ3IDX7ShW3-Rg0GzogY-11#`8% zR6tg9jP=Gxiswf`VIk5UkAlLl$zPc`Mu*g$%IrCH%r`f{@6r?e)W_a(SeeTQBZR8i~*&$Q=CuyL=Nh7s2}2sHZo=;Wl+CqnPVi|;;J)-)!z zaRSIm8opGZ5PT7?4PxAYH_L=*7E-bn|U{r&xWNhHd_t*PZjut<9@j2Dy6lQ0Tp z&95&ii6!Wa0U1|>;WG{08wvO&X}egy94`|m*7^D~nE36y#)el}GY@%+@mZgI%-fg8 z{Ry-eW6IHmh;bz$9l;Y4cuC}I7Wmarq#YB{$h7d1fjn`NRx{4;1)bDERBD?;2G%Jj zxKiCIH+;&e$Y)H|t@PRAnTWa`KkQW^8Y5E5s!C|yjl7?qIXZv@|!@p@Mg6HN0B;7DI~Rwv1lmv z4nVg{Tf60S_H3s$f`?;_JA2`tlSqX$vE=c}E%w=w?<6Pvx+w8`)q*k27fT=>pEkz4 zkAj3WHjSp9DfIjxb4p_g-YD>UYFpmn6 zK@jT9%z;1g!M|NY|6&}|I22VAczF!4T(n6@?!5^#JFu+V-t7Q}=tWJBTY$bvbLTbd zrk`^IFiA$QqnpoUPDD`*KFOm%`F6|%QrEj**6RZQ&TQ;C27uNlzoC7hsVXG_HYC2> z=rv6>udyqB{8<;cE)>(cxG`X1-_9d=n4IK=+AzeSyi%Zsx%I`{*aEvc52SO8pT$0t zjMDpCnV5*H7snY4f|qPz0oVeU0a#9-)Q9XO&AUXYLuehmG)IWBy701KW)3ow= zw74;4D}yHq+>Y>42P=bG{Tq@{t=LBSP-6tvZ)7ooN#=2Yr8&_$3{l<484{zCnP|!G z=EdfbMsf2n7{N4A4yQLRl1epOvn7xCemAHKSb5leuXFf2;<*x?8|JFhC6$3BOtp!MXt}DIz;UZY5lCw4(SDOvFlYQ5@f2bi%tCa1bpTk^RUi;H@ZL zUWq#i2?_NE5ETAiQX|%9yv$;d$6_LpnDBQh6(;lj9S1{TXXNqHi@2%&Sd5H~J=TJ#Gzi1OhZ^xim_*c=Wxve6oCZ2! zAyFY?oJ(NM)Y`$X;k;$QA54EGmd~8fvCDqn&2i-+XLyp{nNbi|KS&ehq|Rh;H>cbr z;MDwa$Hj#N#zw@0#;M@g zi(oS%Dt4lAfmn&ufMP>LdD+VI{reyPERLH*M9{0%Wks-Ij*dEJ>KhpBcHEk3pLs=N zE?po={6IB9csd(voeU2aXa-_`qVhDRmzJv5tI6xVlh<41_kKv^Sv~%x(zXg0k8R}B zK`-_>jeM^6%Q!8MD@tC{mnGpPU~Utgv;g?xYUzDoV+Z!Ze&L6Og@q4^po}Tz9V9hS z;|oT^0GVlPtKZ-b%BoEO91Rsw^2Lf)r4|3hix(f7o7kbWehmpEsRJwl-Z7In>jdVX z*)C6AFXl+a#K|IxnQsfQZrp)M0e_*C05_`iNrY^V1CT@K{o3wtwyfQp`k-me z+AUF?-+OdCuSosD8{wVc_uf8-odgzOu*otQhOqhgdhur9!ap?N4C5g~h zI202xnK+Gfo*d7@F)#US!5?Pq$NI-AKtware9O9rus(h~_Zpo}&%0&} zK8xLsY*ul!&oMY0&TiTV-^DOxao(DB1yBqU^dY(*@>(OdN88MgxXr`bUtbV^lT`%X z4>aFE_k$`3jO4&aPb#14y5d+zyM|qLkU>4rg)6T!D;QnN$pQN)v;g^6lFgF-aJcb8 zv0ru2czOx*;U#pBZZcnoYD{x9R#N&p!6l-w_p_cH;7ca?YJ>Js$2U zh-(#;Yp%N;y|Q`cJ`~n4>^9m|US2LkAHf7|(>K(bLNaZ(q|h)6`0}W8l{x9uZozr0 z@BbJCWvrA@4$W;_dKP8z|)Tw=O_`LhtseT98+6Ti|1LU1RbP=qFG~VO@xeHh~ zNsPX}zA$X1{^aDOry>zNTI)@J|KR|5dRp2Q?&VfX=SsKL7lMQTl2TK17!otFSm-zt z6$5Fe->rHxJZ#QR+L?g{vk2?hhpesd^FA$;?-ly^Z@N4WWA?Ln%V|w^(;sf8f@xp; zRUco$RFU1nF7aVTzE#W~DE;KFR2Iv6`l#4OG7+PtX&H2B&(*R0+>oLCr^bhi3=H(w zOaXop4oSdEH!{1wqHm{94i08DXvc8+)?VN>(AVXv81*Awp#eOHG+MFprU;EjkII6p zWioj=Zy>o+f7d?5BgDROWv##78Fu1Gfz^4{W2r&q+jt|mi>=^$Xse0GZ`WX=vVsI; z=K(*h_I#!y5R#p>%b%y5C8e?QqzAwaQ<~e`A^{i*hzjp0tI?+&n6Dpn9+gn_Tz2qJ zft`JS0YC@|6#060-WArd8 zAnLN15i1K~wk+n@!9SMae~2%i{^@^5nUM-l2=(zgY|RvCA*uDitz7F3ld03*(Xj?P z8q~LDO06WTgQh{SaZ1PGf&_mnAMS3$ipu>rv;*gHoyfqzV3Y$UDwzlPxQ6R;QWL*F z;9fVn=0R)gjue2CSG^e=T;2ZhW275=C+xCS$Kuh)7p=4WLyK==SmC1?;4&H0S-|S+0aVBK!0LST(pFQaSmkv7pu4_L? zV66Mr-JQ@;p0&7pNOCE~c=)*`aj z9lq;Es$${Z;ngX-adzw4K`2|_F2t)*(DAnyR&S14yQKbREFByMOv@1*satE3GE&(b?SV)4FhB(lh`%jDLw>UA={B$0u5=Uj0 z87mRR3sbn~r~e`beR`Dx&i@jLZr%C}5|0MKAjWc+#j20svgoHE`6S&GO*sY46NP=y zv3#eMfb^`mcvtE!Ya5$7k@vNREl?01eH;Sq@yDycx{CzoB?74ew7x|e4(F1Dshm(m zG-YcOot~mzN)~@k4b*T}rxgWlc&k4z8D?>8cND8m2=% zg*F!BZ6+4%TJVvU!hX!ajdgM!_6I|UjvZUm4lwp-?OWV^?sRfC!GH`C$hnnh-4%2( zEX-saJ`GU%i!iI6yN6%(4A$$4_RZKoDzaCm%C?F2cahjl$pN#x*|Ms-Z9#cF=L(oJ z9g#cs5T@dU3@v{ScM$o#yVG^OVWG&HHXMd2p4$<%wvR z{}+-#v9Npy3M$0xfsj14DKz2}QyDhl28a3V$J^xnaoa0B1U^YOIbBpqCul;n16qpJ zImdLz;g0ruUel$$3j@=3+Xu)ew(^l3wvo1ZXH8FQUao{(P#p5#hMr8f2Q7|&z#%6D zsIj4XuY=HtKNhW7LHK3O1_i$=Ux|5!c@-MxGF zcX8_SWsuv)fm0y}oU1Ax6-N;*Gu$$M=iFRRn20_NnSvI`Vd%Zo@3uSSO1F>aQEo+l ziO0$Y@zC2iy06yTkkUfP^23AcwaP~YT}ji{e*-m{8{<-IQBhGCI!S+a)MW0vplt5Y z#)K#01mbAV-1rrFVOzvwUU6G4<>0t7`I$==$B}Z4zOg>k_-Yio41L$uIn2KRkukWi z+~4;-c{4Qt%UUK3oMAsY(2##;;qtCs=aKgp?;Qt&wRWF|n!rg9DDmS9ZLP%l8p<

v z=hM0zYtXHjr}Nd*wn2q)TOn|8L|xQCW@7>-NEO14c)@l~dSXi&^K!voLqgE|Sk$MS z-s6IVA@XvY@i5`ean?{kCz&@^x+wPu-x`(REPp+R#$&%Ed2+1w9jEH zMipJHXd*u)0}M!ssNW8>S|djnG{>L77k~AiHZdf#$kemUB;wp+qUwe`EXGw{NrQ zghaummFN?#IO9vGKN456IC=Hhv&Evh?}ly&&)TV~WBRl%B!Zv}kY|I$751p@A0Dvr zgeN@8fb^1j*+h~mQkKDi`>54xfvO4^6H+1K4JCGPJNb~V)1OrFrOh1xxX3ePtTmZ9txf8XVa=~Pu_ z!|bApc4BwJTtG)V$gibyt0&x{?Vu)s)Y0IrvNPcL!E{69d`*RaYW%UZ67kM4{7rvqe9iog2(mIB z%7Dv)q3~8-=z=;PA~{n$Q(~U|SyUDr6H;Qu((P{e3crfvUDc-DLW=XS?ueQjmM$qFGFE%(isTF z3-zaF3~II}7K@FasP(bXOP_~d@=SI+#ZIm@d?M4QV!_BFt!0sj9>j*vogU&og*^_e zyJuiF7cxWx(_&KN;km4){=SZA6iiM@dbz_-oeaezDK^mCXzIao76siUDwTPkrf=ck zgmGJWXOnzuY<+WPEpo?t7tT3bpGBhNe5_9UPud1++BMYOi3lE&mtli#AiMmHk@Z4YhXzx(H0O){oYus2^>YOi$0rM?g8Ur3M^zfd-+ieO^Qi9 zeKT5Bay56(VGOj`a2%xlc(r$w&SK%2gzov z#hZC2cdAZvQsV=Uz)#JU`EEPS`x~k&4)-MfP=@*!GyS#gpBY!IZnx%EWEEO8rY_f< z!QHG&8+We=rt3(1g6b4^3;2#w#t*EzA0!_D(GP+!jBQz=AepzB(euV(nB{lvSsd-8mw`r{8vZB!CJevMz?UE5Y zt(5fRNf>iKZslX;uY;IjK627-R?_pvpb8jGRP6p>>PKucOJ4r(j;>`C>yRp9x=Viuj=b^)g@1Pen6@z9Z8&@ik29S(A;#F^54mgxxQ779BQ6FEdP)y&7px&bo2?SlRo8MIw zbX>QrbUSbWPxr^#LE~X{8^8)*rf9tdl^(PQCcDKtDNR=v6HPs9*Q9_LvHqu`hDSaL zPvToeSi@vFS!6XAF(d>PK?VH#RqAsP^f9CmI}QU!Bko)xOsyANOecl{q&P@le3te23>=qY#nhY`b(u8^9w>2t@{lopX zN3gIG@i$}FgM%I^Z(*7|muUT_eqdzTO2k^Np@#}<)oJl2eLHwli(NYX;awt%^btB; z$b0^3c1nU1OgLEqmxytKS#IJSkA$;vS&q3AOrQm2U-?t;n0D zP5n13?70kGYLa@JyAMXn9&Os$wVRD{@6qsiAdwZl=KN)lA0}v3 zZF11pGC{*1qS`J#I)9D1t)eRmCXW=!8TW)Pt1q0aOsdyY4SOtj;HY61 zG+$@;bFSIgl;0Xj^g$HS_Ay%xKU6bzZ8td0k2>~@r{b0HS4Hx2)T_Vsr=`u?3MTq~ z503l!`bO)&qyP#4+08wG*V)e;1bhB|csJ*9MAN%UIr9Onlp1#w;r4Z4noeaJHn^B4^+Pgyf~`}xo{D9nW$)m`H07L5Ny*e zi0?Lpf&U%WI2x!az5rZXHmohl=~M6h@zcBi|NMF2dj%(jtmH*Be|)*#NB=Frl`y}7;-9I)1*tI7=toAd&3D{`1iKYwD2QK&MS)&nxAY z6&J65YS;t?gW6ofKLcbr84u~Bm-?aDgoM>^!j5Y_rI+LPI-EQSErUr6nb#125i}qZ z2o~NW-TlMEYrie-haQxcYNvo=OgCkvez8y88Xz5KGVwe~tcGV|@5}HnAFf^8Lf_84 zOa}fCZ>S(?<{qRU({mWAAgg517l1N36gpB-;X_dk)OlvJf+?({-AP0F!T4ghz1({N zru=q@`_`-tC$^ru7zU^*-ia{(BI8C}{1s3~vE?t6MjwtRO&hyo5N{ zGN|Io%%^38Q7oA;8UIADjFi4~J66v{8V{!G2Jx^Z?)NYPtIJ(qKt*7bNku`YBtVLM zo$Bu+vT-#W6cu1(v!hYdT0L*cJp{nVR;X`A!*tS4Hm6~xI`hNt8<;JW5~~GEXiu`y zzAkg#7;YIpmL3e_H**Z34C}eQP&l~hCVuJS1L6a$`7P$*x48y8cI4Ok>wtI~Iuf2I zXf7#bV!lV?XgBkagNce0Fxi(-Xl0%Ad3CC6)HLHDE9ZzIE?K{OSzbdiq>h#HQ5+j+ zBAR+ahSbZm)-2gpc*~b5sb+24QV2x$K~L0Hx_idvZn+6Pv@;E`b=~iN z*pH7Hw0%>&UN?~SX;-?WX^$x%{$ra(a+OXr)2=-@nSU&ik#9Kw&XF}usb1T2#Rp!kdiSv9`I;| zPXH_j{-_^G+m)LkIev05NTfQcu)C4iMjovwD&HPrpBpp0;4z?d$HI zcdt5#l>7|$jn`n{e!gqOqoU5}hzNeR$S&7-&mIV%ZQ7i-zVS%jxP$U@0lIz%JF3{= z7KUVzpo{yRKG6@j-)Y%6 z1YFotF_Fy+9F6!Mb5iOBZcBMQs!t!v{bJ*)NMWzV>W?qek9RK~T;v!FIjaxZ+7?ve zmUm9yw8Sp0k3s{gO$!3rfG0K@@QtfJ5=A4_Zi89NET!juvnXFzQ>qI~H$-+@{AeCl zjJ_%A_bW`ED}zvKIg{3i6CW7s{oue_jv@v#ANl(*#HAC6D&sNXvc_dlhiQ*@@xq{8 znd2P~Q}ax;9+1i8XZSLeJrTa0Y*Or(CrCsgwyPhrzE^A&$SOSRpv2*FzqRc<7(i-p zhO3t6gf^w;qP#Xya3$geN7k(+DNHSZL7FTuta#IN?`$JJp!FQX!3zFn@aU-jW7QMa z9<^O#u&bMC0kBX4+Du@1ytgio)o_=5JE{0f)(uedBD)1XX*QS0^fqzBsAZErYz2I& zj}4L9)M5@0#gw+zG65?mT-86RQbf9w^zgnhY&8@)kw@OpuBDAx1Ax~|iSRsG zr*V(ooB&E8|x5^OW5z{VGlN)cTM zEG9S*LJP0!!0+m|ef%ZxNl8$e+V#OdV@gTD-0Z9tV1ojwyLRpRR`lDtZzCeiAZ^3M zzd~Sq+*&}eiv~4^2O-cYcAW-hM}nDkTF-%ZZZ&udMLdhchcE7_`2LOy5RpCGD6fvK z;N;b-S9@CBLizYeBUC_yLW18NTlHFFgIdLhW( zmt23u{4D8~%WLJKdUU~)MrueqA1{7qYjS@Cw?*lv zPyU)Rdxnj98Exd`-H-pibrbJw2NjIz*#$Z1N3DUtatj{mvqxAU#q(k4CAt7Timl$; z@8V#yL$aASk?S4)Sv#1McF)%ZRrP^;=gDRH;iBnBPu(|RJC-e{@7}~~czyQs3fVR$ zB`9ekSbGoa)wK1mXZ6*ZSyiEkHaUqOL`TGQrR@j(2aRAiO5`L_BLa#9ZZ_!o>pl63 zQ7&jBHYn-aQMv6-^_`;PGWF~&h3WJqjuTihC9~!Fd7X)eFlL}U#wAJKr>x%id)F_A zt8UWn@;|>}5dSutvZ4prnX~!fhgJ7c;rkE5fB_luNn+;!3_*1Y(RWu?(XT${6k9)I z8oMY}IFMUPliMyfon zSncaQ^Ua*1EVb2CfmKY=*Vo}-6B+<(MPTp!P7s9?f#tp(SgrQJ74LLnfVwLHj;jnI z4S|;mIWAKCI#WO5UCn?pA5bfyg!#-XAajJv2?0RAe_Q)smeC>iFiNh!um*=tE3ay~ z+RX=pG*r18o0YfxBGi%hr|_53Ew;3BE^x_t5J7Zzq~$bLCb5E;LWy zK2WrT*8{#b5)r6ftf}idGnZlVZD6kkJ}=qQ{jZh5z>x-^tpGYpYny>VZ?4AVv2FWy ztqXkxRuB>NWYnGlhmE#C4PF~G!rHtg7*(+m0#v6q6k6512V${UdsNJn#OHw;j*P)z zm>_mBH7p{e;*wPd#v_u_)6)YupvAPOlnd(A#%enEY`6u;w!XkNvaA3-ht{`)l44am z=t@dnbtq=$jHbArD?^8uB#+%28}2*jqG9G_n?(EOoa)v6;SmI0BQy`Bv*GwMzl>7e zbYcKHJ$d^hG*0*%d`B64aQg_^-#DSW*gN@`h zxW~6O(ULd$|ByVl?yUY0etI$R8Bj+=Ph_4Wbw}X0Y0<6mLX#{(8`7P!@-OK)P%XV)2v zLib@W`*@@Zy!Yz{*lKJ7v(08*{)Y|sa`Vz4TP)b_Q z$F7zKj8{LEu9V96O{XRJ=z@`jgwAEs$DyT($#F)T9VB50f3$W^_Dk5Qbf#B`XFVIM?M$|W#qIu3tRdm zk#C+mu2kmSEWb}(wmHrg;GNOsO45Mea*==-#zeZPa3Ty`bJh1|s?DH^IkhmoFwJ zZc72&+}9fa#9EE&at%gFUuLb8g+WD0dmxQTBk6K8vFeE5?Ck8lMWmhG9oil29i2N7 z??3{8E~pezPdG)ai3K$^H5J5cwsD{CPW5=st0UOUR((6DfMIf;PM(Hel6yyxwxkrs zkA0Q`Xh7W*jhA{f>ZF519d{qUl(^7TeY5ql+R_gNqr2uN%z`OKWkd%nIbsVyp%*=2 z-SH1h@SAxfrO_}^MMWt)zC_p+*QYVI9VASA^~o^+Iv+EksU>i6s>*HsN1YY#jS;W8 zQE3T-s(#bh$FVn+tmOc-Dc?e@!xoG;xL~8y`5E+6JkAyM-Gv0($NnZ~>j5aJzrD2Z z0|mB2?^*YWIfvqcp#e#aZFXa-l9kx65{X0s)QIsU&c5+i!4|g^`@1*WCu_V;QhiED zo{LSJsB&1`-*=Kcyayee|`a(EuC0>?B5%u+GzymDl=baf=y?SVXhD;;lfC|hJ!UR3=wX*{0w zRG0L@9wN>Mm{G^r@lnK*((q`Xdopsg?c^EW$x-L}9lC^2=Gd)d1MrN}l)xzTH`DFD z381S?TTFLCxoDqX@K#==zHybLs&!%26qwk?%4YpqJ842O3qZN|ZO&nRJmPFhb8n<( z2}1KNAN%+>QrTj`cBRvHM2^&B#-WvKO2BbVg_Xp%Ba=IsdYy?u$EMq}tM2HqMGagF zs$xG&ek9y)1l@2zI{@q7ch;#(YERnF^=*xczK3Tw)$1E~Ssv)xasaeOD-|9e(rJ-zc zL_GpI>2qJ)-Z65N^#3lq&Ds66!o zy`s~yW{f=I>dXHSZ0&-c~15<|=V3dyLx#*s6$B;0v?km7-?b%($8T=}ppP%)VvYj}y#XuV)L zEK;%%3#;3;RZ-;m7Gr+sc3K2Mj?{)zc7eE39?u6R|5j7p^wW5(*@H>>HQFBF@p``* zzq|%;gE}BSG3uu80|1i9KH&d)CO@JN$q*W9I4RTS_g}3imIQ@czx=O5q5maNQ*nM? zZ#o6oW8{d!_$ReHD1ai82lFkt6?Mxagtkt-(29#sP9>ila$5BXF^0&tYjin{(~!aN z(`a`P5ZhL4A{rm8ot@pN3iu{vA3l6ou46jrZ_-#qKJbB#41)FQDH*q#Hl4nlrqn&? zwFY==U>n=PG!l&i?U2T2uHq{ZFT6-Z=K_yL^_+x z^dMKrj7HVpIy?)4Ew^g}o5w_Y(1^ zMDezw1O1E~7B_k6XushTmhG^reo|aHg#V?lsVcVUgXEGt*r8i&R7DOW?67bBdiF~u zE{{6f1{l^SXZRj4jwf0)T*o&aJdDnGBBT&c+*^2fuWFY}a+emob=E+ttK$d8Bvr&W z<}L(1vc&q(f1kP#r@D8vj<0=I|L7rPd>muQr7MZ5s_e=0V4hN5b8jKv0r*9pu4*d3 zTY5`a@l;h};!tScJntlNY$@7X>I%Cb0&FC0p3SK~;cVd1+#GPWSnVpjOL= zzNm+@ySrWMf{k|>9^fFn?QLMxO?3O#tw3)ZFd3uVfMNbfSpaCkQh+DForuZ{nxX>Qcn`r=KLTtVP=Ntd8I>2~epZP8>I35H874aV;>O_f&c-WQ&q zMjrr!Ic_bbV~ncI#s+J48s1j2WYh3s15;u7q^{55(#m^=S8A-j% zCLCSXau(}Fz0ZuxLH0Nc_#zM4boNL2Mlq?OduJXGh`P|nZ_7sSRfWF3j0<4brMc`x zKSCj7AAq;%a9=9#5z0At@Lbi-$?2or4xUwc7OH*J@AMN&wmD?;PPy#IU$&Q;3y9@W%ikX2qsL;Lbd8`2&RP|j;{0>- z!N9_OOv-=~5kkB>uy4+P-o23vN`Syz1aGXm7L&*7c5Hpl8M3lG~l9RIO5&*E1hQ`F5N zUovbS>CZ{m&A%Li@}g33ydvYP5~6P}x#tfPja`t_nfLr}&eYg9E|p0B_8On6BOnUi z3u_IA6(J4>ndBV9RzA;>f3SppWyj@VL58&=80xc%!89m;XlNr4lFjNydpH4?i^RML ztPKPXIdN*`2RKpS=}We-KNRC;KwmKY9%dQxc>d9#jUcwcfQNB7i!q#4vgHM3#n1gm z)%yqL{RHR!kDv8tv<+gvZ1Cb)#UKDK3#;ke>LPP_0TAHF|3QEcH3$%}wzZ<(z|AQd z6zJ~{`%&p3z#?{ok?v=^v;t6H>X-uv7{t{ScFSN$`*P&et7rT!?#twSY0_Gw0c?|%$yD` zoUNNsg)p&K&8!Ec)~uFP*2&hd#&MJdrv zn5{ey;Ij~+tr@>(?)F3s91V5LDY}0B?2N-fo-XCUpaq=D=k(mmdVHca@$pd^A$HQW z=+WeYqoCt^YfOY!_+v4=dl>lchNC--?GbNGt3Ho15RBed#doFMZ7R#iA2{0Qf0`5W zgo`IuvrVvBI@4R}Huw9+vql{3ZZ1_mroB$yiVtahtvb|sz2OlfclM~FW3#7^Znq*s zXrFv$o7yRqH&*^yTUDj~ z)TZOFl561N)cB?}2{mfzN`YU;cECR|@+5T{&sKQzf;EHzMy^jRc_ffJ0;7684m10! z{BMh+7({buS*}T<_GZbOAWe6CEsd017n{nxjk=}S+o{Z%-7Jv;&S6y~D4l~SI4*MM zbk$HF1>GsovsY1+8j%)U^(58rl*-1j&r{6lJ~7?Vo8*!bZtzxSp76A9XW}>0A3J9; zmoRg>C&$`41-DI*F32C}l2P!XP z^#~xp^II5CIGGyMNEA6@+!t$#3#4R=s~8WrN2W}j#SA6 z(mzLMpB+_U#vvV_xS$Mc3j=#s`vTBa@@ox&24k2W^&-0MFCH`gR_d)BGcxRyz%R{W zXP+Fzf%WYxu5lpt=085|>9W&z8ud*hQjlt~_5!$kYo-Fske>Vf&m-U9yU^*EY<&&Q zW?mI5j@_1aMHe-=_UL5OKJ$s~xA#oC6I`1U$yvlhq>-uG!U3|BSq|aFiMk?z|ZB{ceG`KtrS~;5VahZh+ zV!b#ikPp1ne#TB%sMm$~lci%3c%3$#l+lK(D$AKWl&+HliAQ>wlAi9ayeKW9K9{2+${||J}56G z`G#0w`LjR}Wv^^9a}vdOAA3Z=C4wztg!EP+_CM7&|VKNLELFcz9#g zM{xdt1_3?($~dSQp%pvI3f2T?0Iz;Bml+zW>IA(S(jAcMfsD_;lmc5ZsMh(r5IdO` z6Okt%jV}fQWKod`)~nJjmk;9gsO(E%_#o7F14oM$ad|pbOT+wWST zs>+ewxfMI{Ix2Fk=p3$?{5h)z-I5^y;gc575Gqh6w){Vwy$4j2>Dn&*qNC1?ql^`m zlI=L7(nLjyltjnE2uhGDQlcQDNHIu@{_;)e|_H;xcul_QQo8 zk|#pjzQxL_Dic*xY7F6rVjsZzDjb=ts>M8!0(HfmVu!I{69>Khg^PS1@73t5M~`OX z+Jo;nzok1Uwhe7iow!HCCb6V<3elatPttW0T=Nh5m$;1pfrrl+#;Zx_=_qV0Y@}yK zso(Y!PCow*OSluhS=#5w8yFb!Dn538{rFY-?Zs;VFD^S%Jj>gada{BOVYpqXC{9PJO(k=%Xi*byqfqv5Ben>l76{<>SqU+dCf}LR=A0BKno+J# ze1=hHItZ^kyfhn=oy?k(I`MG_F)RrJd)$KE0PZ1ld?A1dUy^?vt|6zJ1=80bDp=6~ z^?{TNE#m5^ed3GNzE9?@CQ3B;&<}x-3I4HMweUN8{Oh)R!3FEucJPqMrN3~?%=r>K z%oCsB0~90vXy1Sl=r(mH?R#RE_pV@n5}p6ec6@%fQ9Ztnwh2B0H!?JDSu z&$N#C81a#i)evd7Lop6^aZu8Sr$yw`%xYiSd`FhEKvy0>d4=tY|>CW~7U zNvl(;4sntJA+Ba-8Zc;lFqx!IcC@ptpw^I^8>xilaHr&aBWB=(2-;x?)a-o_Ov=!T z1LdSzuxGm1r{L;yxt%~kUirC3qc(&POP?p87XCO$vGBuPO?r2l`)Wpdub57Vy2Lia zeP@ypv8POQIc-4?<8XkOuhFZHO7bybemni>OE>&vn$knbq+jofqdtmb*WE!)`_M?1 zNCQIfE`ovO6k13VAfZfaju=u-HL*+jif{K`PQaIVcDpSWOMm=*b$NVWX3tFgM`F^) zT|o4~6@iX=v<2acBmp1+l-0s`WXAu|*#6)D`9CT!pXFq>>OGuPS)!zYUF0)BKw9y6 z`OcxykjpBe62LG@zJ2?)AuRb?@g}6MTEMA+8GQhs*pY0%_-FTb0 z=ugG7V50^tYxNQ^_?pYAnnWA3CdEZ}<5K1nBANeQBiWo`MI(fqG^vQwSKj^rC>c}VhM{Ik$A9o+W@r?2Xe>$G?N%Rn zmQ;j;1t>Y?h}4)zIU|0li`C|myRqXyvAety!aZ@Q1thqEa|qrJCVdK~6N3AtBs4dI z&dr>qajdKKripki9cC>n=9%PJG2K0je<5cDr`*BuxG4%55w}jp1z}z!H8n zw~J!T+QX8Z(2ekAoTbLEgQ#%~V+fw;$^g^w7prr)^z@29mdNG;g7}P;)@QhAb#gnR zBl0QYpTHX1LtVmeG@l&3y{EuEn~G5xIWb@6#*3%4K64If*NxgvXte>5sW7*}-Jj`- z@}45tkKB&(J%c_=>9j5n)$Up{UrbBbCU^%98?81gy6g-DXHFZArbKG2)zzHW^EbQ| z_I&}qTcr|!<(n#yEQ>kwIa@&SI>V|#*CtNJS#avFIP99cEeW!{4vu$s=BZKQR3H*L>5<25hnK}aJ(4k+Z(6+Tclp$%|DFqIFaPJ7`uU)v4vSMi(#ows()K* z6+e4H%&W#^sQ8SfALeW+<$9Q?628x$Q@?g%(NCCvGYwY0?_wscN(QZn=Bo?9-)!ak zIx0>8pnFP@No|T&ghzdY_jSk!C=!8%1{BsKwk_ z|2F)D&AKO|K<~oNsBNbu zD29!Tg*p5Zph}9Nd;J2$^(dleCUrM~^ouI$@V;~bti&+Ega7O#zf3eo5kXLlLV%93 zr>nFFFY*!+?A^++3DFcw9NRm43{f+m(Vi z2es*_C%d40{D?QCm++$tn^O`;`Vxx!1G z=yBTCR%7y|Kedyif`8as`cR9dyLJ@uS_YKUA=li6o052G16Z&&&YiXC zn(phG#^*?%hb=$d!op#rf6Mv4JpcdoE30M@XnbG1c#)o-{$^;%0dTcG5|pN=jMM&` zTXJZfDNKuAUM-!kT)nI_3JeS!o|nM<3qx}LiZ%z7{OqQ)Kx$r+)OIl-U`4V>B$`#+ zLU0tzCyS)oOr?t+3t*I6)o2wCYMsAkr@quG+%tW@In%g8-C~6ex{LILu z6qHgc865cQl4D!A`p|3x8$YZ%{yaT$2a~vPJK-N^_=|f6Zg#wUnI*wqxpJiwU*?x+ z5!jJ#sHl0$b(yWftmy`ThuZDE$b!V2k^y`s&PVL%npd|EY>=J{$A>_SpHU>t@`;1e}E_7 z_k4jBiu1+jS@mpUH(b-eFy~$C33RWmb~4HD`q7xNcl23;znqo3;@PsI{qrT?O_OFE zQj|S*9GGBz4`DPNR@I7q*)q4qv^z8`f!?%u9>Np)&OrH5^RGNm&5W&(QvBr}h4iUi?;g$#2+q5f+5o0WlTi zi)Uxv1@b0t4oAQ0KhOUr*9-eELq7=iu(9&W9 zpo;-)K1*=1c4g3W6UC!yB^Z0_-He^arUB(y8vt3D#qH89QpAw%kQv-vlMm8TvDa$J zy9|+lUdJK6Q~z@_A!7^nVy*7Y6Lva+8Gsep1m&Nt=L;wP_O;xMh<+OwcLji@gU>MnfculM16p%+?pab1Q_NHi5Hbgw%N2UI$f1>vNrXPsMp!plbWq zlE%ZeDXN{6^htW?7$$xnV#6R*-wb%Ppo^qIs~uQ&Q!g(x^>A>bV2)S=x48O1Wn_d8-cF!?FWjS6dkMMi(zx!| z;2Qi1jg>EG#Sx=HR#R{e%HC{iV+TFxNF#mz#WfZ~n8dmUDny9|;)H zpJ622hX5L<&!1mt-x%}`mZ5qqYYuQVd*8sn1$>3tM*(`}$~O}e>yMz(+Ay^is;#eI z!8o=)d1~r1q*hDmwj%J2-zDF_zXJxf%P2&)m4!u+=tNkUDYQ&G0@S(V^%$@mgd`Hi zu5Q@YR01rlt^(rv@>rbWZZ9I=)%jonEHK^gXBZ{$S)T z+uDeyy1VAR6{U9VVus{e8Zgv?cI?;zZQHi(CCw||Enn3~Qg2QM^s8@^`8j2!5Chv3Ck zA>5lEav9?60#vHBQWaI$^oA}0;@Y8N{t$j9WuWI!374;EI;8ee6fHXQ9xa}k)+u%f zT@&qFoYF()zmXomEVP1^UgD%fXvzs`Y1Hi@Dz{Y6gY{GKi)WNw1rhs;9dPFj9N8oo zzgNEh=h|xi0C~=Ku}I!8*;`Oou`m=}ig`PqVKx3$+XGiU$zR$@P_jlk*kBasS*k8o zmfE>@-kv$(*t-O1q~O3NgAzTfmXs=7)TTc56(KEq0lMooi1uXt#7K-OMf9 z7SRzyp7%9otZOyw=~LwcIR#{rFe@_2yt*FyIqTJ3t#j2;R|n!#r~2ovdQ1Cqu483fDVaN?NHG8mWkA7hxYdAE@n&g(zLosF59nz;vf%fh>D#9V zqG*ft^|MVrt~Md4aVo300#mV;=W;KC_vxvPL7W21)Ve+0~5yX>JV@UUT8U{q4r5Tc$q>B^I3X!cEg6AP$*P(2`44Q$jfu&Et8%dB+NpB+4? zPKwgjnNPE?=bfZQ^A&>HiiI@6gU(a2JJ#GD`R!f!UVB2UM3~0^MY61@>}=@S#C=vB z?}-aZ9H+A!fZCIycpHrbyhkWMVTRz@&F-+KXiHE%X1Nk$K-s@uUocqkVs zv$o@?rDRSeg#Bu}OzdC$9#sIE6@XGBQ+FPs@rWOd#! z*U|d#@*3@VhK|q8qoG{gY%OZv1dA*4T`pVh4k`HRKP#eNzT0xb4m?V)Oa=6lbH@+X9B>o< zxc}d;%SW>;m;#C@u(kUh+O+#L^bm$Q;5r#{?b@}{`{~HYH$+ePwsp)hgrb2sDE%sz=^ zVd2xihE?lJWdF>`$uaQ@RBRO<8}*>rJ|Xp#t9RB%+$X<@89^&~W^@!51*oasyJ-7WZKWi2)IfYO5Xb;iGPS?h-KLN=y~TF!>$WGS?xu2!B`QphG9F5gohb6Ec0rCzV6h( z5=L^GKS0Z=>$+@-Z;!KDEXO8|)r4+`V1a_x619((Cm=6|c>_Eoe(u&oLZ*JWr=g>6X)xaJ|)!RVjZWi>V`sXo<3psaEPv7=cPZH z#h;3?E3gVD3!{OPYEP`I&4fp^a(`88>n$%g`l0VB$0koshNpF>$zFh^1oRA~W36~I zIJmma&5#D9REb)R^Ir6xMdFZtS8zT%o^W4VJ`qb_+ty#|Ruj2RER?z0Fl1$F7j#rX z6W))M$XoattlSI`6B%%|VBLgzd$$=M9`dGUPWA?g!T&>F7&IEOrmYL3o;N8dIrAVVBNu=OmJlEkmy6@+ z3#AKZ6Rx(-75$W*CbMF%XycJd{^!yj=>$s*KkRwM9X|~(kLAp=pIq?M6{jXoo%F@KXSY^$jjIH+qux7{Z0ZR>p*+(%1=M~Bs4s!9FjPxgCn3*(t04ya43+lY90 zN>zA^{5#-j{7A5ah<48B8a98fhtv9KaBu0P@UZCZ%n$$D4wp64LzkVvcYB)NtEzl&8(v?ng1vaY!~ z4&gyqbK%vW0S1x08pgDF?`GlKPq8cX^%+T17>e0M^HAbG$qR#fp#S3A?5vxK|ITed3rjlF4T5ha}+ji049BDWjsBui+*f{A!3VJ z8pX<=T<>)_gZEHHp{o<1Wo+8nUg8cdf}&qG*&S-qF87nU%g~Y^a)jAoQcg_JG*2j4 z^*)!b1k5(|jF-3Kk$m{wyFAl!0|PKG#H&uTuXzjJUVqw9g`b+~a^!LO?+Qb_WhMA?T}{qm)Tm6c zj>v`+-zC2rM@Th3$zRiB!#9B_a^QkyzW$Bskd;+1Hscc#Q?FWNi29t4D&>W`DxbJD_nmy_AoHAac#G&HL<1q}TAu;M;?iV|>%_^cs7}sb z)iD=?6fXv%HQ;w`JSlM3BkKcQ@i4A*{H(k^!C+L!r(sGffUt@C>pLP)9oULIxk{(d z+yGxm5>s4N*b(C>E63AK!kfUI1uX%(`0BO@yrMZ&vyD`)f14T4)m^{x0ua-jD_yk9 z>Ds1qn*wH-r8SU_@6wAhQPOFL@z@(iy?rI4mJK4(p9^c7Pk8X%?dsACV|EXWIlC9u;Hy|o}TS{hS>c#p-MVQa(Jysu)WF5Cm= zKAr~ADiJb^S~ip&_hWi_kDa(8dq=M4j8C3dUX9opO5$uHK^N|xu`Jzz;3=EtR<5^( zfIQpgn`GdcwLY1OmqbB1`+Pj}t!lg0J)#+p?^7@UL^WkWeKn=sHFGy7P-c|I_S~kO z^IDsSA8=m(ySz+HmwPah=o4lxwo8C+0$mrI5>`D7`}->hz^K6x5E2%Zsj}yNWSf}3 z@w*y#PephWE}NOC&wG?1stg#CqHfK-d($5RmpNj~+ZpK7ncm<4 zJ%3B~W@c5Y&K72R3?fMM%W%S%>Z$7!B=lm6cpp-9gc_iBP(w)o@Lc&GsQtcL7P&nn zJj>Ijlvbj-FFDUop4mZMzrb&;;wTn>2ach`Cm1aX32`hd1J%_rYOFxEzy&bFOe70U z(smMde|nyh(}nLXky?Y&l{`sDxsPol#D0=qfsulaVC6Q#8kgNP^0@0=e6CXh;@%IL zvx0pS55g*X2@a2>34Fc;c|$ZIf(Z;J1PqQ`5Yx%&?@Ua|=D(y~3u2ew7GfWOCmde8 ziJ;LXq?}ufeK*T659-|91AOQ*QR+BgnLN=c2M#)gpf@DGy1U2r&g;%T{rm9v z5B{#fUIpA~FiX_|jPH-09_QW8`7ePd+C9s#`*7qh@+Z^6Mq zPq^oc=^W%(vk6d2FO~F&$mOf+D?myJY}=g-)bL^ST?rOmdiGt68XrxMn+Y5bG0F?!<-Ib0Rbj*41({}`17 z3$kpYLvm97mPj?dlpV33%ooXQyJ>_AqVxuiDIe?e2!L;XV?MgSf=3^eyZm z%{3$SG18zpy88JTQnkWeq_R!5)hV zdOUsNJpIzEjS%pkISLuXJzPq%tRxElnd9T?Yq^<1E~?p-Oc##Zb5VJ!x_;F`M2w$J*6 zY@l4$@yuio?)`CSJxckxOnjiqPj@|6t5^XxV_MdHKE|x$7)hZA<*GRIWs+wHwX-M= zl8>pzgjH<-KF+Vi_^IefU6yKZYZT%^??}ZquVO44d?3p+M~e*y9z2Rdv>NVvcc`0- zH_YhWjKR&jk)dZtcL?`l*8Cl}D|1u$^U1A{1CRdwl%`4T>1Zbk5qdCWLuI14Nquo7`fWjB=UJ18Ang50eVJDdsNc0r^yRIJVYpiu zCB8$aTlN-<+Q9@+*iBmZd*hkjVT<)|B}SS2(_Fjm-iT*^t%j7~JpC%C_hxQ)AY3wG zM8Px@ZNbL7CXh~(01H5~whfpQ>O*b^7*3gI7F+RCL> zvvsc=rs*Kq!Ea+k_BnC1Y+M9rMFH@;PZaxoQ7dr=J9cO>fqs%sPo?jABRu3k|7R}L z#YQ86mrL>_LE|JUYiIw|F5RF{xDOt*wJ|_%t0w$ZJ}Dhr(7I#^2xyl5>^ld{eVqDj zzJVY3|Kai^CG7bXewFYwEGHKiuS(X&f4&LjM-PG>5)7Y6VdL^;@=S}QG9r4p6t89d zyE=pIBh)`QxU5?r#smG0|DtSMi2s6s+`oUny79#HTWqZ#j4Lm@7%mTS;N!P?8Pk{r zZz9w(7foGxLB7C)`UeJrhPDu?=(mwAbRDCTuDGIw2OzK+T3#MDf|;{g2u$>A6k(yE zUjgU0>jh8OG-)$Njdb8+1IDbs00QEBy^W%0btPkE`D&S*Un4({!H}|pO>^r4Z89^< zRj{Ns?h`0vaZ7g`pk-=>DmqDk$Q>x0rL*x6uu_T8dI&5Sg%Pgw4b+Gq$Eb_C)Br;9 z!rS@loy_0+_ese3rXbdn3R2}6#X54-gM&fe!Wunq8y=v!Sn<~czs|cOX>QtAm(o<2 zNA0B6*Rp}6Fi*nAs()5oi|4f81`}ID@KBL*AkuFpB1==%Vt=ob8;q1bH<`FKc}~HT zN3jtqDVWnhxi-oySDO|SGUlxOyxeBAFGsXSi)8R_5uU<7lS&ETxa(y3$f{vpp~hL= zz6V;8{gk_z@7DCn#U{e9Df3Yj#*wsCy5NClM(!T*m#K%*gdH$?UKwHo%KS~p7r3gQ zpf%KOA?b#?JV85pWjpvaEF!)UO~w&Pp#$wpijQ~EJ#y+dn|5E4oSTgGGqh^bSkJL= zWd_&0C7*t`$ld>ME&!0D1rCOHaNDtGPor@3GA=^bCxoq#%jc~P`-zSCc!ll9P~2O)g#OG6q9eKw%3kjak3ok(LM_Boq(kgC`J7^{jFqhEd%=Zw7&k^x zncAGBlR-2ih7ZmVU)oMLL<*d^h`NRu@`QMTTN4-kXFZzfHDb_tHqhKSUPuTNS5Lk8?cY5uR@)8E-m`fP$ z@i4FL(qj1&r8Q8DLrW^y{*UCUN|Q9~Oy>^J#-qS?xMi`9YRa-NAn$J9g;iT*8dK2X zt+DJI5uOIe1xLVc!fy{riGW&wg_T+6gGJ#fE~da?vaUk^OdP(^2IldldC1KFWS$*TAEc7UVT1Cl0@V zVAPu3492Fd+)+SWshG&_9%@ejij?fu-a3y8O;r+O(WuAa-AFbed!96ZY7*IDTtz=n zBS9w>Zd5$CCT$wM_fEanAHcNDeQ}?9m;L!*<3JEL;kgi!zd(#=V$fMzrR9eL>dJFj1u9p$Xr%WqbL5(*_#Z^%d6C zt)#B@X$vqR-uS0AVOmOr3uFg*TQ5(RQW(5foi4m-=-hwf?$QL~NL+-yy6_~^+Y7H* zMlNVB`3=>gVh(L7@f;)Ner~_;JO~HzTKO5Q$Mw&t@n`N?6SuSZnWGs}X)x&0XBpGe zr1IVB@NyG;S})T}5_=QiAK_T)g)g|zfBLn4$AInsF6x7*n>TNMDZ;RC5o8B9MJ6Dg zAsJSKykW6c965GuB?P=foR5RfI?+F8pWnAI)CmDcm6Z>HJ)c4kY|wI2Qb5TA(H>fD zpz;5%jfq@5u{=7v7=I}-dDDaUcM$;h?9zsd!qBk>5a*QJl z(e4lDTQ|{QHQ{>O1s4b@d4;|#pv!NL)T%Bt(d1FS`YE-3fj4eo3#0V89^RU#F%Gi+ zKWJVy(tSRwkK7o^ti{Y?Mn^|$xbCdOlMVCeeoZ;9{FLmNV{rCLuowFdm{JG=)At0# zvtmZ5mIc(KDyQ@8zG3etUG29|$bJ!>IA*j4oLtxeg^Mgz(3~VF$E>>0oZFfnb&L^n zl`xJmkvazBsPnB%*j7ItZvTX~AHIU^#lC@T`f!KH{6P_{b6oD6-@#5pN{XBPDb*i6 z<;rwf88^o-Vs@G)pET1MqB9NRZJtCHsjSc<*6lrodBt5vny8Mp8#;FxvQzrv`G3dM z<6WP79QMlJ!Y}VUTQO9n+7S`OPA#|aKb!b?@htDqMKBgG%kDGS(RRg7#eVBlN$G^J zBqG}#5kS~HRIbjFq%11;`te!8XvbKFZ5go>ctcN*;;-7)-8?_%jP>3egs^q&*M99*wj$lqIoHRrB)>e=bF5tfHH$(?95(pnv7-R zrdNChil#%?_6!g|;oT=BxFB3-O5tXL;l4iW5~Lp{U-%AYd%ja)VvH#Sb+4C9vW?0zd6a8@f)HLy%prUUdJR6A5#)M9+O$!~;M=UT2uGc5_OH z?Eci?;14P{b<)8M6fs_01x$oh=UwAnXh6JAU-dqE4I)3o%_~}cid1Q6U#A2-Boa8g;i|;?Q{PaI70E^aTbxXAq#t5^Gs=ZtW_YK!;9uwJWYTf-u%%{8Lj#nqx|- zTN)p=Gs7vGwxvV$p(!AtU0oTPk5H6vlMt`;~LX(IGX?p_ThvnX!GVH=nms%g} z$Vec?0DKL44+11tL4fsmO)Jkl*f~a7E&ZwsPt;|5WsV-61MowQPG>J_=ybwN?2n>Y zV8#;)3LHe7S>{mu=N+$S(f@3COH02=2Bc+ZW@ZNVS6dDBk@RScgFE9dJ}k@JN+2CES|$L;DB+xgIi65RP4F$Cf}+># zT!0?62h3(T~h{1G!Y{<_IJEY(yss-q5{y^9K4-xj{=eNc3uo$c`OH zx0q$?X6t05YpR|sI!inzUHKmV9zERZk)yaW`#0o~(RT*9I0T?&8E7*RJIt4`AwAkpnbdqP~}mBEHtT zKvth;MqCNsx42eSky*qoXAgk#rOG07N*$XY^o;fm-wQ)bpZY~w#J#3)!mGsr zG)l47J54mzm?q@Cq8yX@72$Q~=kM&LM`>(GIgdCUljfeuG#;1!>zhs0 zilJ)SkZR@zATRWcjBX=!EK*tF|C00M4dMMH+8ebI-D7N?quV2* z4R;Ro8XDe5bnayDBkU3@D*T*mE$+`XR%Xxy$@DB4TSVvgXPFiit`e)tF*^|ni*1fPSWzE zO$<_1YsETQ3+Sv?0HQOYTW(m#4J*|bo(6vQ{<>67uuTmID>n^m{O|A@%Sk*Qz9rJnOFhT=iqxn{G zyX)3qlQ!^UE2Q&4LwK1-<(@t*=yu7Ydf*$#)}Ogo^@g%KUmbDOo~)se&3x%2x6jcf z!+&OCNUSR3Z0a{xRlp``W!qm=<0*5zgkPR2aBcS9jwbV6nr-stFT!dTcp~h*hYS*V z2l?BLH_SUo%VQiSy`;FY(TUY^8%H>+`U0iEC8s0ABZ_|#7 zigMSF?NCWd<6N5T&h9wq<4qf7J5iTq&TL5wnAF9fJNjCboFQEsQGu zf$AuHbZkyIvMPFGk9zPxDM!Fr72&Gk^6n3Yal(T@r*QZb-oW+&yP+z3jFd2S5B)4xDn?`5M5Rt zlJsc)QRLZs^j~-nB?oXJtQb|P=IZIuaiBwiO#GS-*@EE70`yRG7fk3Vv1%=~_HXLx z*v#%PO(I8?dDz{_w9R6jk}aV1Pk9;&PZd#E0Gs1ILY>dH32jM6n8p`VoBXaoviC&?^l!9C37nRn3$b(Fo@#))QnT_&Q(}Y_?x)#(s^WK7Mgb+;B z#P18Gf+$>fDef@$bq%!CLFiriKCg(l;ytiQdUKT}HD@4eMjZ6}(^sKK?pHju7qp~sJhQH{ILJ)O{MM*U`${|t%n}~CXP)1}exd-m;40{oGxx(e=VU%_{fBvYuEfF#Lmr9Q>I!_0bAId;$umLUzp!=$CSGM$+3|%@tw8D4ccWuNs+mVGC(gb@7dv4+RkTPHzvkM9 z6IJ)-8fI}B)+D{row}n+H{0?^1Mz`ydQ%|+z@1x|M-TMe=Kg-LC?mJC!(+xlGORO2 z23(a3kTWpIt-z)AVi&e6sJUIglA(g<73ClWe38%EH%2!+?5v#?K8_I6Bi~` zN8z&TW7fWO5ZSvVAEK{sTczXsM;Lj@UD%ui)bFcdhKmRH&64d}loaM-`81?2A~V~i z7l28ctRVD0yFWMy<^B8LioqZ{75;{rsttr#{pp=tMPCq|;Kv>Z5e%T{gY1>RKxk1n z{0Gn^Z>+apcr$>f0P^l#;Lm5}V+G<`r+Nzb>H7g=_HFBbS})=6uk>7wQyiO^=mB0$ zO(%OSz`6p4@k7T4yfA{(;E_H42cfKgNXHG-x|`5~M4|u!BMAgL%w~A) z|2;z&%+3(deL!$J8_ayzO1R>Ub&80HsHcH^KikYfJ|n#-)8^Y`aL4H1zJ>GMVo;!;9hPk|V+gmPcuQtkJ?8#~J?Ww+^Bz1M zu>~GfNm?}Kz?`QnRaz>)V3`x3sEDkHEMGL;SFZAu5d}?8^#|1>&XS{~5oc1F!w4Ga zT7Y~fiQghlUDQ<6KyZWqwx^-)PE3WobkS$}Z1bTBre-E$K;t3*>&?DvijIEGy5D0O zcYj3BYG;R^_oUN*Cgd!Z&yp6)(Xuyra_$HBId@8c-b1{P>bD+5R|Fs&W9>KX-ppL5 z5+6x*GnzCP!~!?-AZa;&2)vR{HvV;rxnT+Mi6U*tt<>Sb+$o;LW=h1~*S(*R?%`5) zEC!;9TZJC>q(wKo`s*KlUS4xd&yv2zX4dN(dYj#VwfrNyonKRGP_Iz~pS(eW@~)4Xglh=-|(Te*s7TVF+q}!C_csgxZr({{xx>+NFMv#z@m8v!-gWqntF}= z`aPQ=2v9E7ko;D(fuIJhw(Y_Bck22^P&vj0)LCTRIB?B{-d(u44gx}B{W1eIuTp%f zfb;kF7|UYo=j56iU@$G`i0eABFCt@8-L{a>4=gKhr5z&vnYn4&iK8l`>k6Y#R^w4_ zIlY?zK(gQ>??|#;cymna(`i{zRC)Rh*8Gue!C7^QH+T9)BPDk`2?DFwIbR4?Wo|~f z+~hDX^;#e^xp^L8^ya=Lv}ggu!kl}3qh!gK`{c$)hh7}J%8#J`bT^_KbHW1gPoIZggpa;(L9 zKwQ0eTHt3!NC*Zk z?}h0Q5F1FN%OuvN&C}nIge@0_6YUbH`_*%nGzY+`YN{UM%behHH_*uRG*vCc`f?K4 zBH@B{xKH?n3(0HuHU7gV7VKxeCj2ntAt52K>p&y$ATZ|B05kE%Oer~2LEm%fGB=lG z^#4-JhC^T%l0p#v6*>;^j@pyG&%p*Es0(z3FTWkSaCtGH?0>MbfB_;PhJi+}q-1q6 z2*j9Gz(X$(dT9YT@bw_PU#qN;%aw00rJMc0&;{-3a00_v-XD{C?GkJsmSH*a^_T1( z5VfO$9>Rfvne{`_U_%e5t$`DPe^$AfyBC*QPvj3-wYIjx<;}gDFo`)Gfukz73nx(% zDV@YY=KN&i=-}!yqEW6b2&*!0D4aZ2{sSpH-h%S-CVi|Cub5YoUumav|z>ZcgD z2GT3=?t)g92g%0yC*&{FAY0Hu?$0boGt(xkL2zR8dVRFCw6~;R^Zk@T$w(PxW4QJ#fD) z6S<@}XPL#0Vb_2>z&fvbOuEu%^9bKwi zEKzzk4WaW~KHdP`M&sMippH#dt?3e+-))q!;OZ%q!KmmR$;ckC80OUn>x_d`)s?!~ zhn;8iG$F0XTvY7!k;+tSG)s4;fJGRb0tMX~%ehe>(VIKv)YR0R7I)3tAx0m>w|Xmz zBH!kXWdB-Q(YvZa2m9)qJ^2t!bgG^&S-2%|Pv>%FYks!#jA_ko{J}RiIMsF}i)QG| zJHsnlO1PdS`Z>MV#%=82PR4-{FAZ48Lf08vyg-o`uMyFVhnh2cbjJu9mA8CGosBY0 z)pnDY(n-J^|EyVwv+1IfY^%7mf|M>f&CHl|RAEeV!DQYxkH}=ildRe2JToy*9mfhf zC9iXVvu%HtL2IcJxY?*uc|{Vek}R_ycSxEu0j7GdnMZkwNBa%>t7%@XZ=~m`%DoRT z4^Wz^&}Ld;;jH2uHOe=G%!%96aRNIwi~#SkPg74Ep-Mkvk`JiHP#fS(_0E^tjuSKc z?oXLcv6Gwz;&a}%3$nrwP|}9zflWwLrqXN_l~a>PbX+q@s4zQ#3czOod~d?RJAS(< zt)XfI|8{5CtV00qvv`-e?svV0ol{nUwg|=TY{43?9kmA#7yb;jiv(UiD?g76(YAhZ zwgX0-)id#diJq2deNxJQZA|1lGuK!cWb{5Lsuo4&6VteyCAI8|cp6Xt3)HdD@r?w+ z4+$t^WbADr`IBOKt^x`IT{w@xQi1^v?|4-!5kiLO|y1$!%2=Y=!;SI1#&PH9%`#<@i0BD1@85pbvtC01} z(R!dd!aE6giTwR?chrl*}mCw(cEmED|8j9o>QfLy;4Y=O0s_mGk{!rBxa*kf0I zxR=^^QgjxFlQ=xJA=UQ$mnI|o+6rqo8CmJ;!wsG!A_EWKnmjHo zSENfT=XHp>AVWp>X%ha^le>^L&}fo|@Ks1g0t*h=g@Yx5@)CLvwliP7?|2Zt;ysZd zcN>WD6(rhGdNd@G>^nGURBPGVseJKR9WU<1Rb_LTATktHtEc!aP9K|SN`2-hPZwMT zGxR{r)mnV8wLbmeoY^UK9omLMapF5Plj8iI@QFiiR*v{~{w)D^GT{*EPL44TvDk-= znpg!$VBgg08CfJP*}oiNBpJR`;JZ%Wi*g~6Esb2_c0vjs51T)<@+HK?5^DkF+AirX5>a4z03?8!43fD z@ij0FH+vAQAJ+1$gb`O0YWHLTFeyt%P?li$7dVCzQhugC66a(PLX&QB>#L_%bP9)S zf>RO>a*q41KA2PDgqMlL|Mj5)=t)jT!?MH)C%e zbSM($Hgg{&&Y5!VO6eNaGTWnK#DIyXquReYpbAK=Wg;`Zi<)l z$TbVD`EjsD4tCsvK7Zat^c<(3>YZjqXEQE_ZQAXAhAcu77W&=L9%tC+3NI-H^{X!Y zleKHXrwWrQG3?L|QibFTjcxJXW8X7RiJs5Sq!Jbma!Yfq>vB7$nzVS<_8MkV3{(hDxgnt75`(2Q_#c@I3uK+ z0wg+eLMM*#Dq>s(6IaCR9HrIs&$=x~FY!Se5$pZj@PXGYZfA82-tPe+jRYn=w}JVJ zBz^w8?L?`5+#` z1XS5V2zJk^NGJ%x{_H*WFz%-wLhiV{_ObTHc@Z$fzAze=CcVp11P5WC!tF_hc2PHi_D06=$k293 z6b^zPH0dE^tNb>Z_QHPGcUkPTs)6!ZX{;y~6@>L4W*_s>;r_K`wf2Hc`14TKQmo?z zfl4>{ay|aD+fU&0{U2XGMGFX1is68ISA>e3dHHf6Ywqy>cPlIU^(Aae3O6p1Kown& z3k=)^Zv~dOus%P5a|g~>THykd&vzxv8Cl7r9vw2VM9 z=z9?Gw(_B+AG&fTs4O-l*0WA!dd;eH6A6%T>%~tno7psiL$g4P*#dWG>ALH2N zz*7>m$U77P79Q*^4dsZl_Cy6QE_Eotx1qjWFeBx}IS!RKy0gt-qXRf*%G>raTtHsU z`Eac(i9C+_N|Kw(1@#c?=&fQ&ySTa<>i|19ypd*rQi=!AcnJi@b3uA>?|+3_1Yr>? z-vf%^sRQU6AGB*!5BlN=)}~BR%}A*O!gOja9xVH`u7BQ{vUw<*le?{VFaB)v)~?d} zzNDLrM6S>}{0|_Z{xLls&ZDbWmc}d)_?M?Al4tay5?p$VLmM_Fq(*OJMNA(!r1_Z^?M#jZ;Qu8&s#VVjL8vaZYfH|Lqjn2cNxh4k|_p9 zdD{{xwjHKb_UYVv@4`FqZ{aHNUsvd%g!PZ4rn}GF|EdjyfPY)DH0DPVwQ`pf?Mty+ zqw+0ODHZ>{?Nbk5p*!qVG<&P{U%X4D>;AdqFa2dnKm31j_9kFWp4-}R&{ntGR;s8t zAgNMCKtY@U38dYM2vjH_AhSv-APNRV2!w=g+d3emAcM*fDu>DIi3d zf(Z~ILQFyuLZ)xMad)46`d{bz&iD7auu!W6c%SvGb+3CUWEioyu`##jQ>ml=p$1&x z`1kvg9rx7q%(JaO08amJzdjiRdKUFz3`9Cb&lS4mTLzyDj*hO}^Ltmv*R9u|{`K_ zy=HEGZk3gUP#-$9eFXY-2-0HHmV5~yxU7}?5@HZfC^{VmV;@}0Bb$5R>~~hS+#|G! z!LS+#KnPW>Gp}LgBb?Qf^xTD?2j%mrWko*9KRBVaeFz!bkERMoIVLd8ATyPh;jjIL zcyQ+0pc3X_1BhA!7m!vEM7?s{q;20I%p3@PtL%A_KI7t_QcLq%FiSm^eA+Ir4`M1G zgkF_=vG7uJ)vq=fijwcFug^96^VnPEQyvqE(5$^)IG%Y7U7Wlybq0TEV%yH^nTCB9 zcccCB0|_@WwzgltkG(1|v<$k@o)5apXVV$ePDc6niYsW9n-F&Q?Xt41(Ka1Tn+DX; zm{KJis)Ynn6B(;juyp%R|AjUsXwEcd-KInHs~H>+d#gPaJxKx5wv*51?_Q|hKaq0u zzSi2(%dUrypeVLe7ttSm@SyIaD9)@!M=2+3AIM6ub>5pQ2t_*DDOi#C@K)3{o0D;v zDk(0+W#*SbXT12iX$}4^M>SdBzEECl-od|aYA88ck=l>jmZMs?y@T&cx7Gjr;?zR2 zm&(?x-TPpawyZNRTt1RF$RD*Xw(|EDrTE=qduu8RKPf$*^L?#)JO`Ucj^0b%KiE)n zhOl?w@J7z*RMjsb4?;zn0|!;U1vEbgBa73at4aMo$*em_B{BDf^D`g;4NS-F85Nw}!I)+Z4h}xX|1@8GiT{0vuRq}zm1m(>o=3IwcQM+| z!Hc%~CH$X2#uOOSxD}<74-1po&5v;l5E+VknlWX^gq%=RgJQBk?|5Fv#%a=CqBrI) zuT9V^n!`iH6zYA2N9x~?J9-%LmKQlQ`c?nr0s*xAo$Y^~y*vDRky@h*>qVf+4f z1S`&rB8f&R3_Upl_)_raFV4JMSo~j{R-}*k-~Ll&>3#R>ldvm$J`f!Dd~5U^_E8JO zwLG2xlGX0BgL@->x|wnFztabRSru4aQK1KJ=i39TsjTwPz|mVL_?7*0I}Kb5Q1F<3 zd|RCT6ZeyLG-%5{kY(YHFh@c9TA-}eaSi`10YNT6e5?2H{FE08Ns>Wj+D)!Q0UZ4e z3U+9Snt>^^NWXZ#_{dOx>kp4aW6z91KoNk(ZOFJFs#4{j%@vC2EvkpP%^OXLInf{Rz5k{M5-dIll+w@c^8>3MvUzU-}Eu=8Z9)W+vw7@i2JO? z5TVj|&qH)L*c@%HZ;oCZrTF>ais#(S8kX5${?bcKP?S$YU}x-fQU zdLI<1FRIxh-n_qvS{3q0Wd?oHJ`ZEc#Wwt+jT@)QJdCG5c9Vp~Gp&*O5j?D9N(+pY z*-fft!6h&3V`M_0JmTm`X&;Uqa0Cm!1Hr`%^K0i%9W) zP@~EC^F&q!atX&&!T)$IbmIztJaZ;Tt$r|{h+>Kpf}hcwnhqf0>25@Hojy8RcI)3^za6SJ&v-B2xZW@bEuQDyP%2_HE47hI=ok;rew2&`>%b}dNpc*&*~k!!;o8LEtluPEjbFnPRI$AZ_{ z2Y&%Xi>HTV+MMH^Y93l*Lgg)c2fYwQR&tE$6nkiYs;eoXD?;52)rE6y2Pl+mbE{WLnPO8PHs%e-j8uDd$Z9;xO`S=W3DN!&p@K4OS6I8E;xW!M^lHtasR#JQo2X6_@2Q51fFTq7wt`>R!N-0%67DJflCx!eHIt=8gpjYUD~>{JuDdF`5M1z^-1+fjX4|j zoLrOlvb^a${j}+&K52WyF~!jJg{kn)JfE}9KOJ@#ZEp)%oZ z`nSmM!iFgSL6QE?*ZBW=UxQ7NZ;JKv*{0FPr*t0O{MpsjQUCngmUW+*d3AaGkcGPw zs`~pK-WUOQY|B6QY;WJaeceaxp6esd|De~Nb$RusZ_G}WTuJ-;k(Hl)Z1$Dw;LL=a z;m`VBJ#M_nW!ExAi`I@B-#r>lZDrOl?f4=8<=5BCeA(Ok8wA$9otdd>AwJ)lk0dt6 z%U(xFz>I0n{{6C+-c4vUA5yy~Hge{dAY0m-HnF2#3@|fA|I*w%y=nwaEm*wmLefGh|Be{ zObW8eg7F|%(TP}E>$*FH!Gl7;psi>;vG4V3e><$$P_axzBxIFVR8(YBlZ>Z2M52X3 zbEW`7u0Mh<=bo2kqj8DKFF&qRdTVF&sL_lGhoj49`F^SK-yoB~uZ(g?nKkh3(XF#y z+%HRrdk4nUne+b~89Q!1@J6F(Qu>q+y>*R2^=4iWAAqq^m6V{I+7ke z=7wgP+7U6lv?QKC@2I;K_g-B0W;fpjs-=vG!qcK@P^H>WRx%J#-VtcE|J4{`EVLH#{8hus`6N zuFRb8+uq%lT3gPNu>B*7TwD1GXL2G+4X1WQ7qQ=}`>RG;j*lzwCDIEoV~fk9`k9jP zVX}l^Pqo=Exs?umXmbSQEkj?M7_;Uh7Tmo7@_1LlPVh*h**rjDX3OaUVO z2I8J=;F(7sBaqSSRQbqjAXOpWbI_#2=-)Ry+%-RM3qw87FhpRu9}f!r_?bHP?$}Wb zq;$}hQSq>G@7H$Ze^KSCDf#lx7yIG${Exp@od+Zhi0q!jz>zPE@3wPmB|(@RZZN7` z?~kFOPS8QWot?c~Q&U6i0tbubhL+#{`p=JnH2l_|S`R=V4rWbQdW64m3z$&-8S35} z0GPl>X)yKcJl7zodFcrxQ4WY8r~@Fr(9I!r{Z&qogQaT2lprcUASK_Hm3^9qgiu?y zy#EIZ@_4L{b$3p|*XuL6HLJU#y!h{kOeE+g=f7E=$ZkZ36+?apH26^O9SC^z!mZ|z-{1}zkIt%NnaqC`C|-&lA3hO@Aa9fB%^ z{c)~bg6oMXGrag{HM)fK8M%k*ipd}zBlm4}JXCm(!G2u4$qE;SSo+^1Raj+`&x_{` z1f!MGAw8T+#O zOl&~WM8e@N`;t&b%icjX*vTwGmX|17`Eb0RE#z!js4Y8g9w)a+j#yRK+;30i$H5P+ zdK}yFdy>O~t;|XU{hg~EUYc}y~lbKQ@ zNug1wgyvpshzQbOlJ?dB(Az)gcLjZLYsg%A!nw9Smv9;h?$@IY+UtE}SpbEVLuk|g zKBifnn5z`*RIXUwNhWbyE=Lz_#hT#h586O&xjAJ=fjdmMiKwY6>`bm5H*+jjK@1-JS7f25H5F|MUv)67&LO5O#C6~$K z!0%fEPT%R^m_GPhUhQCH-Jek%bsosJ0HKHlCG`8Ch9M(UC)Gur0qP5?3*m0VBlsC| z{Fi%Xf8k(-h;#v8^qbsVQd084TTB;e0O3P^#svihdXcA3-^Y*@&vk~tI*K-A5N%c4 zI!;<=xj$q4XiI}zHk*P$UzLcJSuwN_B5g<3?5$~NS`L?moN_#{YZvNEXJ_ZKptV*Z zL3S}d98wTgoG-AduZ*<9h3bAdh=kW-X6u3zhRoPr&L46w?f0zZ8KDcj3PbNxoG_MD zXUw|w>wo#=i#lRouh)7%uG&a`um!EIg<=>N9P+4@mzLFbs)*6+ z6FYk6vn6)fqoJpVEv6>pqFz!Yn4^T#ZgxJZBZLpZ6b#l@gm^7 z^ZZ|uT4-|%9lL|BJCV8tqDVteAugy`zfJH+YcLZ&SQ}NBR4}>OZuWdW{&?t3j^q0= zifLge1D_$oTQ^e7~&{QE)j&)cr1 z*cnf%tA;6DYwr2+=+YMBiCesLC5q8o<{j*xhfd}kbT+#b=>fu^h9ddR8LJuZNhz)* zwc5vlXcIshPe>g_ch79=D0$23L6@|)Daa18<7H^-G>a~o-!So++(5ow-X9r?vmJ>T zWj_--2R)Z^B1w08;;RZ3DaPm$npkdsC#339|G^UJiQ@9{g%|3O-T>QJH#NP7&%%%N zzK{PQ{2gmDVdSxFaa`fm`8A3Ks&>H&jfDL$AaJZ&^eEC-Bw{IcXrNx+}JyPjA zTHi%q-YLyvr96VD^*11`#Uc#Ph@ynGAX+z;@L8d=bIBJ`)WbAOM zBw!x?>*W1kPxRbBqEDaJWhU14x$}@uf*5dsfyh=U;G2MRy$cG)>7*FI_#vs$7zt@# zYIIK5j3d7MZ{NMEJ+ZYa?>{c~cj=+3Egy84u+!2zK_D2VfveTZv~mNc+YU}Eb-XL~GbFKLnhcq=9Y(QQiu|(hyYGF~drj_^-;>A+R zg1Ay!%V{OZBo>W9mvFzunBOkB#~BVOj&8JatJW6A#6R2loaiEYIesx@zh3_dTQ70R8!05=vk{cCXR=& zXDk&95{fqHd2H!Mbxm{gMue^5sx3ZnHeh1GM6EU$jm;#9h%LqnrsyGpR743Lcz1?A z_h>ch3p1|gYC1Yo9ZYul&C0=j;$B;&ksn(o3nW*XjZV%NFw$=V^m7)|te9NJON`+_ zPiBJ>Q0c@vZI0}#1E4r{ORdY3?F$%8G&U`^KG)yvF*B3ZtB)u-VFIZr+sHc}RE{Cb z)=kyt;m$mmtH*@!#X$oqkAO`BRy{jYdWft{LE3 z*HPnI$eK-jCsl~Q+KhZLE2IQ>$Fl`V2}dE0>h^EuZ?Ko+9jo@GsDcI-?8sprY~_s~ z8&xd4}*dBkw=w4W)?$r&9ITgSoB(iG^b7r%#OzC{m_#OgY9 zp+mfOdw05$e4+J@D6HYv1N-uoUV>{rJH&?4)ry$CnwM;;ce#55rT%5|t~xp-0cE<< z$-FzpcShrhVw}mR3tz=>GA6?^WEMP?5$C7AWNQWk!+L=;Wn>Jm*ic6vCg#;~?ytE% z6gWz$X<=Ve3_s>BTdS7R=@oK%cv zKr)Li$fc_4Q$#z7a>s}cq1NsUIOc-w&5-aE2)YRb@D(VeB&gaGxGm2v`E~)_=?Cd! zB-;-5zsD*r0WlM~Y-{_sXnQQdro&=p+n{qTBMIB4?odJw7YYGtgYc8QJth63se8Y{dV^E)dUWWTZ+&|ponG(`VmYYzrgh#N&_4W0B+z#-5U4cm5 zkJ-~&u)e(+wqC5nwqLTTkGwKHi>)(%y8JsJ(h(C4mm{*Qc2bHGAzM2XIMrpuFrzD) zl@r7XA3wc^$n0ux6<1Q59vN|VYwLg)_4jb}-Ktu8Pb4K=wSI@BY~W=q=z`Ix_s_4p z-+I`p6WyQjGETf{XOMOf66FG9Oq^7ZuX;tS-7m1@xwlE)^1&Q<7Yez{P`WnDF9_!} zF<;hEx7WR+$#gQ`+W3K`Ape>3xP+Sy2>^gQQM=?k^>!;43(Lv%*1~5Z6V^eAD#eL+ z1Q;w^9i1GvxbJIn$B^-dx)M)?93W{_0Sn=@9&hg9&;=v&S)1yb=N(oR<0dcp-ch4* zIzAM@a}{32o*}=RjUGPm=Ad)xXKD3x^-P^AKW?2R&Z^u{@@rCS%>$dYR>!|FW3*Te z1kKjaa}tL`2}D0mV!FDzmdKpgl|2w)VNQ+b;)7xz<~YY(5)o4#lrt5cj#8R%q|7)gTfkW-0FA<>~LSz5|9SR~vmVW8&ZGiaN zpb);TuF0qv7#MU;hdCmbfAZY9hX@+kxSIlvdqs-|EQOb)0e)U5CYBDpe!c8V5+w8? z_==wGI3#E5VrAfLXCG+zK%lFEXH1_#Y0jP3mIzt=+{yBB6ld_okxvFNb< z@}A5)4jsy#Zmxza{V%t7=)IY$1A{3!1-09*$~FR`m(VtJSwDxL!+$Y>yEoTt>u6ly zdH_er+>^tQ^>Plo9l!;7KWCNYhy4Ai6lQZoUTrQId?o}nsRfU@5sVKXQl8`zF?o!- zY~kglS!7#!$O@;nb~sv}rTNjsxpnhi+#`kT(UIUr&adt17C4LEC<_HTq~W1>0Uis# z=YB7*`)1y&Zy~MXoxqUiNmEGXg3VIn;~UYkS0w|puB{hxW$O=jDvmaex~7~qn>kTh z%WsH@pN@?nPFUm;4%uJ6AcnK43yfd%{_4z=%Cf*}PC##lGUuaoH+lwf4Gw?u%I~nC z)*E{jM*bL4K%ofJ-jw@o+7`#Fxb2fYE1gdj4+0-H;04+Qt+QDIeD}!Kknskd$}a#X z#m7&191Jhoz)PO1e;XlMr;c@!=HU4!Zh1HxT}z1XB83sNCHo4+sW0}ifleiuJ0v|n?*eUoeh>Qz~arO+gUeV#`W~80= zqc^?VIO@9*>()5Xxo5H}eADVFHORk~oPcF-;klxL`fpvfqBEF}pe8@jKQ?VKTSKEC z6B{UIEO`s5;j~l^dX{o7b)8)FDvYEz_T04i!3$QX=pR#$7u<18UpIBdr?4CeOQ$7YQB@BVSZ}w>|+gC zI(%XfeVGA}-O@COfLosij{N8}XzW_z)L>Dop9T+tVcPmV-+gUMdNEPJyZVJoFTg7q zH>fiJ7k=<5R0-%B!4HCK$FVxcLVa*jKoKDFk!nPdzbRrI`S)=1;T=|}ing5?k*gi` z)6im%`_n^%`y) z&z$*0XTb0T*bt~uIZf^}X_Z9`q9bhtd|At=XjUEJ4xyy@4)G4{U<(OZ3r$;0<9)7N zSUI!VnCEyW^{|FV3>!6)H-ZiI;&Tvs@%5|jQ=>+FzGidP^Eef(#B%2$Y*RKH<9g|r zj}`{|lD-z%$@^>_J1)Hq#cD+4Xf+NLO-KWB#a=`>apJ^p8XDMJu*LqpCLQGSXel9w zP)*k?7);tTO{)#rjq}-VDBJf>7hCZ6<;xkag;$b#JO}uqS#Lh;&()%*!H=p39h zDak3(EH_#hp#gSbdftc9rVN7|I$VL4%Q{*-9N zAq5hNFujv$Gi_oF`>(@@!>4b$m@G4x*(HWGh)*Lm$q_MeT>_?cQS~1nM*H3?%1l99 zs&h)FbJWpUEndBDhE`X#bH#%W&MGZ~nUuv#xTY_zBF1frW7+Zhg!T|S8_SRg%Tn;O}qf=@Uav?9Z@P@|@bBPFMBw8GC*e~UZMQXUC zV?`l0hx-Oj+aR+A+Z(l)lL~Yi^NH(hv;uEr6VA*5tv-%U9Zn3g;ht#IcJD?Txk}`= zj+vbb`Ud>hOV1f zMRiU|XP1P8jjUnR

UG@^a$36!XtLNK3#f8mNm10Q{VFq=I_H)Ocy1z>})bNF0yH zc+epg&0KE4w|XuOsWyUgCil{jnv3Q zsi?%r5P;@xHQe%1cs>qKx6lQc4a0U6g#pEFE@`G&qF%wCRBs*v@>b}Q1R+%$ z^~tB7>g_sk;O|dYIvbsQ;{w3I>6n;mOeP{t@9p)vX$5A^ej%~Z(TkC>CRc(?+s7ZC z&_j)kjO0o@5RhO$=$DuEfp5f(bF@`#L~{FnE`K*{S*qIBfKmT3{5A14F*#YoHKZT6 zvzo4~OXw%=-85rqT#^b)Ge5DRWDJ5kQg!CDxyo6ApmLam$;`;`Szv&o%3KmX)&7UJ zgsv5LY_ynhSoo^#&1Y(}Zr0R~wy{u$dP!V`j~}?UAx{cM{c#IoGlwiQ)w{&&d9I}f z3cFbobOk2Xijk4St`EJV@AfV^YU+3K^owz~DY3K{! zeija;Wj!w*b0LR#z-!XL;=Us_=AmYjInIh@vV@6oe*ttz0UL7j;D`X;9FNwuZm z{hK)KNH~#R8f3*~Y}-ix_BY$>a#GxIA&582jCc*(S>I>VGmo?Kc`El7UhAR45Pt#vh)sT2 z$%PF{3mnCB#UtZ%oV9U@u|PmQU-Z1JwIy7}(E}rWM|w8#CFh8XDfu9BA&qe~GaU1Y zP0||-(GeRVDWfbQVBBUj){-B59)fX#Na{~>m5~zGXukDu^5DC&B)Mga)439&bFJmUB!uIk(p&>(>pR`80R=P=XEj6Qfm2-!y zvO-A8(X8T+KzYiw{lkBd?;Xy;aQTtXol6ZEt?HRr>Ug+^}L`zz-DJOwS)t1es3S|9P1Iyy1)K^O_h!tc@nqO%>d$N<5-2 z>TMGXKRc))CTP?Zbb9E%C4Is>8(X)zkalKP|Midm53@vo&Tlhdx(=8~NR!ZadgZ^0 zOf&^idF$Xm_zA_KbgrV1#kTpwDsA-rPc|q7?e*973ELN*(>fz_0>;HD2#L$18J zxmhm>m=1dLP;0&I5ETaM>tf7$-K2nc(dg8!Pj<&%5hNxg_=G_1xEGX&NXEfJ?N}yy z)26yreov8Z3k|l#%z;7tICd7> zO_mwWRKmhBJEkyJ*>~$fU7f5y8-XT>3pEtIbH7`W{G&KMXzo0uq`G1+^F`~p>d%DE z^mNT$xYC&OS!A;9YQ+`F7Hq7dnZ6hyU5D3&<_TU(0y}cEG#b~YtPr;;eho+U9m+AW zW4%GkAALP=^@0v?vYa5|B25@c5VVPYdYV~!`i3?VUj z9xucEZn6tSjV!(Qa_xKl+=a}zP%w8X3wCbR8D)3bO>alZh$8}2((8f1#Sft^_nY3? zi$`LnZRuSl#*k67Okgk08ag>1rhQDU;Z=tC*i&njH$2)!JIhk*@`<}9=eIKG$=EBz zd_I0Zkd(Yxxiw_;^p9MVSu72x`rdFv>0LZF_yJ))sB&}VH31eWIairutrQ# za>bFtsfvlp2OOY|1YHw9SEl39@6X1j1ip@K`XsHKiLoOy@F4^q_4I_YaW{zq>hK?= z*V$$G)LeNh0NKJKFr0lFflUc0b4YB{c*8C|p^>_s4N>Ap8#4QAUviCv!*lvn|kqB--M9TRR~}+^m4{J^2qUp#ADIThw0bP z%&8~mNBlLxH#Es`W+B9Td0uiV@JZHY!hAtm{lA_sdV$uI9~8HYqE+$plVdM_y8mGs zv>N}hhyc>$A|122CW5tO3`VRaFfX9e0C-$`3c({G;p_txc|#vi1&?{#0|x?bM!>4% zvk&gjZJ8i+S;%bYSIXw)@*usSv4 z-^;%zL=63r;l__i2V~PLw9<9lRJ>_s+<-Fw5eoLE&SkAY5A&Nfoek@`X_Z*HnL)mc1%n=gOt&nG5mh6*eGxSeC=2E! z1~#27RfRUdYrs2(^j1;RJJslOPxVlQsjfX1Z1hnEUu2B|m;pslaI~#)p;>eS2}gX; z$+MnK1lsM#W1w-aY@_;YBC*f+xOccZ#2|_>!g#3W<1|(=F}bWTO!W+hoi-W#eGOuZ z>;quO4>LBb&^w#j67Ga{hDDDu(Vq;c6)b75yXt!~yv5N8a}|FyhqbNXjQD}zeerQ>zQTW4NHM;b3#RFk zn@r-bkvBuz6YhkZ(u_uGKMx^G7MKTkchW8IS1s;V$(i`95F@nWG@0GkWID=H1o$gS z-gn*g#;i#AmTaQ;O;SgyEdK&-B%w}rF8n4Rq4=TB=PACkq+n>1QnwsV$Hv%Co?13{ zeFQ;Fuz%lLhKWzPJXb=noS*#>r4vOW#5U9epc>X)@E5Z~dn@bNag@d&Ub0Rx0&PZB zF#YAhVfMKwn~a72?+OM6ry!Gh*PGYFSQz^g^{bOkgT44C+Xy5DwWaOVn*sFBB7k`f6?!s{C+vpF?k%z z{njsvf+t~Sl@6ECAujcv!=3VcUO8ZrSGzfnRd!z!p_NZuI+j39*@VJcD#xu^S9zx# zH82|1Jl6j=Z|Yx{f0QDXZ_m*y))c8`>e)n(HKa@DC%XoPip&FW#~|4oFo2S@Ye=N& za;)0y;heCUYaPM?ud}prZ44`Y1c_?lC#b+{jv}|qfK%JYW-}0GHvoPP8Xi_t5)%3b zDY$BzyV&751OnL&~4};nRb7!L2?xby9~Js5(M;m*itUz zoiSQOZldEk5>;merv$JU-E3*AiBOm`QMhpBfU%at&<&4RoTQ&P$ zd8B?1?T)cYtF6_bTT3yA%+Csmh=`a_!wsQ_(B1v24HU}}0XR5v`ShmC)~h#<6WT=? zgO#=B+{9Vaqh%uuWe@redXl(7ZAV=mPkJ^(a$;Z4qQaUp8c??k#5@+B0ODu&Qoj=3 zF(s!s` z#o!XfNdX>MX3fjk98=HGib8J7J-(5U*ANsd}fx`Hp+(jYRQ+mGJkj9%sA3WZlG-E1QOL7!A1RnFGV}&A^aFT z7wSJg0ik*oIgAshtq_0D4|&cUEyv9?XemR{V|=RR1rM+rpaYlo(&C%D1@(s|VnRr& zL;sknC6g?>&i=|`gN@iRkm$g;Vx-+rH55-eTTA-G*O=d31jmc6{@JzhP|De3v>tuK z9YKGC7~|C4C`72M2aMds<}=?d%uVGkpv@%}=KwwPnZ)}A;e96HK1qpvbN^c$7OXY-v0v=jp z;@L!)`%v$%co6Q0lQp#Ss^XoR70(@&?-uV!K|gxKSr%|mh8DsN4s{X2oCNf&)L&W^ zFJixgZ$aHkw}3KL^p3wYYX8%R$stHMPyg=F=68>mgL6-^8g1j+d%ySJBN_a`0C2Hh zi|RmF8b(GRo#-uf$>obQN0H=d1o+V5IS5Y_k)Z;sen$a z>`<_HT7tT?DS0~zv7JSna*mr#u!E}icyYf8s;?-8 z<+r~nh?&Bb`dkH>2oX#O$<8(C`oMQRv8M=R;w{F=FS}V+aCRc-axO8 z`s0QPxs0A#$ID{-i9PYA@2FOs3kA&7QIZxN&5MY-LdrDyj>DW_oO!laT_k{ z<5WPXLd_5J-ItENm#f0cjGes|{p<;|VrNoD!#Sx3F(8WK$+Aj$J&?6?~>-CRW>xKNdX9DhU?9he>B%ZpE5~*F8IL)?DF!|8_IQSvhvfiuH zl~uaL!fiq2?JW~v#8UY}umVf?O>wOF(ocsag8p#z&Of^)-{N_r&r^{5Vg8RxjvqZO zN*rB=Q9I<#j}i&p=N6{wgaz+lhJM;hPwC{8U;j|;G`GZ#K9O@uF`~r29T%cey5oo{ z$7p7bna(O@?ZTcm;R%U6uLL6zHp5~x84+R&naa)VzFQgxKBNwRZ$H~`F}k|(CpXK+ zACO}pr5}B4J_odRKp!NJ$f*xBL;fxSxvI+lhZ;8+X#z2iCJ;}8MTEi7qB|m#Y|G|% zyTOK@vInS2OcAJIEu;&;6#M$t-YX!&iU(hUU0}J2q|*VPtjY)>=X>7%cN!pse(U#;J%ju>y~|BKn-` zI@br9x+c(=J)Bq@hzS?3>YCA#7d4X4l79&BXM9p2t%&c6L;?ArB>S?np5rI1|Ou<@|BDnl{^dm!Q@fuCGn$S&ljx{a4(&>@dY zvqQX}N0b`yzRDcRqBOAJ5vL9s_o=hRHwBw#6B}~e0xN`*am4t>*|lY<#h{8r1;xLB zv>yRUvovjpByWbz^@QQfX7MI639kaQ?GpfWElrI{7}V0-3t5czH8XS^r_T+0P0W&f zkEwPGz`??cu6-AQUH+6sllST{e*_T&`?{43?5l2Y7hmkZ11| zNt~kaoC=q#I2(8~^jaQaJoVo8atx~q|rwaBrUN%)o6J>-wj7uX~JuJTIU zJ4@bM2l=SY$>ysvwx4V#4?i67(j&G)_6`88v#Mp zmyVTQcL&zGq+o0Ks7RMqhrmMy-AXZLXSvCrwKGJ;RT{r;yGt-7xRwMpOdaQZn z7mIWbGsT+V6ROnw!}OF^P-I(7Yy=B(=`FbldoL6mD3g!jJAu53s}oz{=I0u|13V&^ z=Z;m+m4yUmML(&n{bOY01)>&w`}VzZJlog)-vHX5um9gA?yv7ut%mUGckqQFX^jLy zAm=`v)^m8{BRHXfUuF*!Cq@1VHFg*gV7JwQAY=kEgbc( z6|V9da&3lE!GY*fd+z1IWmR?5>}&Zv-vCF~!e?q5;}Y!zZp%VyTmHa0(vl^o`DNz= zHk8)i;U2IfdU6j2Q4@CC1(_*ukG)(n4m${X1=>SpfyKg zLQKqKo6I^l@ydd3^``!KkCXRysb|atzIE6Y_HD|m5&+ZW%rGMo3~)Y!zkm5$t|M@U zJ{R6C0AEw^didK@5IURBoC2hVQGqGiH+QAierM`}3 z=oL8&?{>_b-Z7~+p%(2ec1@n$QrwQ&%Y6o%fMk?Tv8|C{wFZdE@a;Il9Pfr7UpoBd z)n;4}rbAH@g?Y~VHD^^K4UX61T`976%XpQw`=cz>k)?CIw=8_CktDsvIK*GVp6Cyr zJXumk(Z8GU)tt~Mc3N`p1}j$~{^w(l)sh-mmZygTe*0}g@kW7cZ|DP7vp1njafQ&P ziiHx1YT-KwD_#UrX-)fu$>%j~5_+~fhNU#Zr)IOr4eJ<3XJ?8Y7XJ3->xi2H+_wzJ z3tPZcq8^Ph42H}Dm^iKOv)u&Q+t~xzR4_UEI1S2li7dH-rSa0w(%zs;bJYz%$I!w^ z^wC%|M{S9!*Mh(Rf5`sken_EoV>WH67i=f@>|HT}b3Plq_ZV~Y`MmRCtp>_{xRGc4 z0TWK0*6*yQNC?QezIMj^0Jc56Kg_Ji#=x(o#ga54G(y( zL&N%9{J0ehT)Y=(U)@4a;Exg=gn@trj-m>sQthI#dnJy?@Yq3*wwuFNd8gTBqj!QQ zlHa;b4EiiYdtXfWwiK;44D}}5)xCpKg zoEKJ-#CWI%ET6836$jpajsj9F_2s}>STGDMw&tX>e273}T)YdynwRmT@U%R>HF4JB zUzMmgJya9&3Q@&k)Zhis8mKC^CULeDb3k$Zw2!|0nON@hp+=P&dTAUyHAP+}oUa_f z2?eD>7^j64?W~&-nooyR&WaYH{9NUe!P6E{k8Svz%^kv*cT^OD*adYrm+sd0Ru~DY zuj%x!qN6Q9Kch6)ZvQ5GV1MS_|7mvq%YP53K%Vtc)uc_}XW0V*|0Y1d>njZh$-@T) z7P8=kW>W91P`GpqsP1Pc8W7<^Kwu^MDDs{8=mcm}m!1H(hPWa`g_*i#YA%f+^h!=$Qmgji*t+(PfJacw)vC2`@ z1u&xa(+q$|g;rvT+XfYf39nOIjmoy(Q zKP`8N@)~~n!^|1~nN|%ZgXB4z#fc>UKH*qm^wXW7JFQ2ON4GdS4>NI_xlDllP}8A5 zAqqH1=j^sOEu=Ufuoh-22${d*g1Y!6bUvQY_WVW!!bDy)wvUR>$h}uL@a1dt6=GlU zzo{utC`>>8zXQXoD|LgxD5O zk{gS+#B`{gi5uFbiN>-OfY-g-rDG-EaHCxY#4qg@AydY)G`WQnkSAgFuy~A~FI5Sz#tv2L_N-R+JTy zB_l+LA%v{&+z;BX_3zvN_xtkWhXz9OjQc+4I@h_*IYF+v^`d5LgE?qOY;M9fCq}=K z#qRnndgF-|lusp95#ee^P(}2P5sVa8n--<(9f0Gn%T%rheZ2F zqoj7ji|LCgCiRg=eWQv{zJgRaz@8UYwS^2l=zP`=43U@`f73 zqRK5ox~1`8l(amqtLd4%S|hnHg6itc-;QI@%uaM2=`j zBs&9|kj`DTW-dAWmPls~aGL)sn1dI1Wp+Z-q;im6zN7jy^G0bLvU#unvUwxXEgvc_ z!jMw(Yabs$M%`?U#xH%4w0H;Um``W2vp1NTxf(;M)~I(I>OqZCu2+@cCueiJY_9h_ z3j;3?4F(fLP@Xe{iatx?5DY|9AP}FfH-r#eT3y(q@n;GY`?C6^NcuVbAEB-Df|$gq zZU|@3KX~xeV<60Dn6@^vJML1bmck#Zxi6i}pu{BA{h|?O_MmU4%P`{-#J7#RX{w(B zJdJ8t)y`JRxk+ns1~T&Vry`+T3og`(7p(>GVRn`+bCB-0rmrQdt9H+Ds)+jolZB95*b?hjz*SDAh=GBT|bP}y(L95eIlbmwOQ zq;yI>edS1gwT?tndQ@Mkr9U)3X|&{fX;NRHR@_XS9BRc}xRj;-_cKxxURqii*U2W# zCcnR;Gk>kSE9kTpkIMOhluLcv_l1=v1!`ZQYD!m&_i}0BHhRqMf{}_QN#gkx1L0z7 zVso6**tug*m!w=|=v8sA+W6VmjV>zM)cEGdcri;uB}qGx^rT?(m2Vo3^1pW6QRcMf z4EdK`yB)PEj=lTi?z2~OZyf(|{^>7w7(4!Z_sy@)=;wU>Zt#ZzTAk;c@JY#O?8cIZfs%-E)x8vc&A8|IqmPf?T_X{NwX}8Hk`yUfNurmGbWw`5 zH>fO}rsXa{<)L`LAx&f$kI|sf!f#j@Wc@BTIx&!a81MIx)yRCUsm0pajD^2R13tC!Q?yLpcpI=A%N}xF0*`7Z)t5e;EsXi1k z-gOy+PEOV)l4}lU6{dq*^F{gLZhi%Q9aPl|v#S1QXT%~vyzq&Qo)(+PMNzV;anbEUxj^La zwg^Tgbs5;FUq~Jj-5Z?A78TT!9<}nvUW65n_z%b;-HlQ*2c@Mq=>v<){5t~2x|xi9 z8sd~1uj$CmhHGd8`#K$JZ00k4M!iZ|XB#?=hAM{A)3Gpt4<#83dvwA55>^2^avW1h zRMfA?leOdBse;_DO?O6_^>%gi0;UzEVYE6o+(@w}&I+%dzBiZ4gWebwsW0u6^;t0# zd-GC3aR)6zJHLm<^c`&takI6v%~BU8*)?tLap^4O^{ohUe@9K1@y!Ard+>@6jS;{_Wc3Jhc|FL7od?PK1c3*}0WmCosO9FaRDpxNJ&uv&cStFilcVGF z{!XF9bl4|K5O?q(m1;q59u=F^MIAWsZ}LV{(`4)kD=Gp%#7#3#?Q^PQ4C<#(RZowH zu#+)knAe44o+$y_@&0APP)q#KXnTIDVx573E*Ua0CM{ED`6q+i?Zcp>Sl5$+hsnR#|I<(lmZplrC+`5~h?I#_L@H^&6*>^{JuUBz{m6@x|?eTFQaP%6FNMO~w z4LgkbOiM>a4?h+%T+T}Fv_(d@lu?sLBEmzfhoT2kJIVx4gC9zc`||05yosYt=@Y#} z>B*Okr=v(_+I@G1n~#Sj_#F7n-=&8gbX~j4Kh^)CqI!_-YUiI587&!wS8Ht^I>Gj` ze0inMHNAMmS6sfCuS+cTg;PR`WV$WPX<(^OlgPfU?(zDlhYYG}b0=dn&-!7UhQyS# z#vzrNkvGoF7w#F27B^FEZRU?=)wlR`#MDasI$v10@j8{gK{es&Il|0dv6Da?t_*KF zkp|WVxCY@Wa?eaVa~0WQ>Q-D3i{g1!GMdw?I8jm2G^%vq9;Wjb-0>RN$L9j!!9FwB zx60edO{>gyL>AQWqJjxcyR))*F>Eb3@u;i5o@IjnIQLl4c=E9q7W=sTARU*b?UtQN z|4zf`lDKFYJ$Q#qPLULEWmnqzX_sYifwRRp5K8?Alm1MZ{u$#yw3m!YK_&@D2Wgfx ztO9g?@tAB%FgoT5`PbCY2bVq5E@NSn@#!hplTsgtEcX#*lt(CqVPqMV=9`t$?B%KI z5h~n=PS{s&E!Zn2<9IFz#hF}5gVT!u6q z@mPdpTo6gx&jJl!z>pb3UIry)w*w6)*m^Vpdoo9uF;gu#@x3nvI!oTyjK|oM`t0U& zF3O4ChSAqr05L@cb{bkujorQF?JTtq1Ml?Gh6Qai4&(jMhC^^EE1!L(edZfc*R9qP zl}}j2verqJ@I4))P?A@yOR{hTjG{$&P>=bv$58m#mWdztaOYU&nroy_{OXC3)22$Y zmYMOR>!JcesSXucXT{ZKW+b|N?pa=Ed*!;b=~a^qQASN#N9c(JpPVoY`cU5$hEcx{ zMN)}zpvW&BgPbL+*_ejN4PA1uu&3M0woS@DSQm?NGPK3n80Y6j?!RHiP#gNz9o@n+f!%S?C7NeQ`JSn~BGm zJ>fohspLbk@|u9ZPhq~U=4G>ug+LZvmi{Grxz;>^@b`|;HZ;<%{|~-#;#Q^IM9JQO zl9}$S;TGA?q6ITkA*bzxAO@mjr8%e@Dj(dD*YG(v^7Vt>OL-vgcxBm$kN*3&Q##p@ zD?!R9wB$}eC3rt%fgbkML=?jmrkUu|+_0t-lb5p0zCZkqE=K$O!`VOAYJdFL5x>D3 zT}e(%f(4Yk)z=EwGCKYxc@kW;1-U3pG>CD!x?J%ae4#EGQJxs5z^w|`8vE@c=EC&b zg*P@e1vlH;*sLOdaOU)x<66-0j3P11?%O|f`|jQEbQQn<{_Rj?j_CT%UAtD2gU23U z#KIo>xW{7^Gb8l)L#6RHnRL<3))^C9uqaI?=_6*jqWr;5fw%JjLrz%8{ueV1$EoYh zxDID|`{2bE4}-+Vg_;@lrOjP@2e$u6<+8-$M1O5}cEg1Wse%y`#wc?zu0!I{%{|NC z7nDx-Ix9)eQL;pQe~_DX6)+d7Za1Oa*6isr8LMaffl z?-r)sGF#&x)pM?QC_g+q(19wDZm=JESg1v>^EAqjl-&t*FrP|}oVml@w3ynIFj&59 zGO8_wDUj@BSEwhARxcf7hEsV%nu(rXp;Vi;fF+NpCe6Z9E1pifPpVF-s!j26gW*W{ zTvkY%yI&_GNQW*Ko9`D)ltr-si(nj=p2GY+<&H0n-|Mm32=`(V*M;6k)=at1r1Xt9HJ632g5Su)(z3Qh{k|aW4`Y^{ zj`gmHs%Gd7U(wE-$ktPNKohu#Wu<(kAZ)8)WGr3KS?&@MIU*m{kWkW@_51iLLC~~f z=@$<4p~q5%jOXj$DC}g82XvX3cN^Pwh-gR3BS*`8Q)N2`DB|Y(NtVl^JNydH*`jW! zHn`c~hBzk)i+6unXQX8}+gmnec2u~DKJn9_31cRoTO1yxF`BuSH&&?Z6nTg(Hs&!* z+!KUXjcFYNcIp!mYEL7!`M*awTqq_0`()h|=;p6Y@BiHZ z{pDHh3aC0-M|){WyJX0gK9thgGxUr^Av73J-eXTMsz9oSB&4osGEhD<$UXV=w@J&> z=a)_Oj1z0M5bt(1mQ0>mlCR`rrCmQ)uFJkik5Yb&dyeNHy=; z(Agbiiqx*Eq)exc<;&$c_Y^Q9EeH0=e3)kz%orCM3r9`7gbYhrHOZBlF)q}5KVJYMEnTs%sP{Zuk^ zQEYl$o-uNG3;#$}nfrFH&bj(`g17myoeO^cIl7TqA5bnDJ}wS@AbD75meDk{qNd3{ z$#B#%Rp?bPxVM7zYya;_-`-8taJE67DszR?W%Zg6{@M zTsK@_wrNCqAm^M$sUUF=cZe@MVZD|ipSvp})0aDcdx>YqltZ##|B0O~ett~c z==n)o(+57~2Kh$)8m@AXjP7aB2+?K&{5!GV+@2N0Vm5S&4zv#C3|K!rQJX!ZlVJOh z8IUy2O`{bn9-vx2Q9aS=W1>iBgl7Qnp``5@IF&-_7LCpJ?|!YJ*f^m;VG@JKX3K^srffq zDw%?c9|{QAK;F1@Yhu5L?k{X}ir^RTkXAYLBq;N%I<#MQeX?)*D5*W55=gFi4C*;N zaSY`eheAsQ9B)sXojWmdnQvzz76$3eD_8{-w3*82FWYSLBh}PS-APVK*@;6}Xw2z? z(B8DFL!z3bC>tHm))PDqzrtJfVIrx)FsPRJFdo zxwH{^25BaPtR6*ioyfZa?Pl}IICOq4D~`N$Cm+31P0hwa98nT(utzvA7?B;uP+t_E zSCfT)b*j%PMz-18tl&JYZN>^UehnIQX=rF5x6cULg>^p08c}9?P1BHmIvKl7q0Mfc zb`=;o`^t;VS<$BBy4S%lp>B^z{ls%4nVY8qmod<|iQ9+(s7eVXVUp5xptcQ2(srVXFxOr>et=;Z#c?qqA3IMkzqPBvswx}Q&zqN_dvL{tnotdWd>%hQ3lN`_z z*i2a!1iN8S4U1z^BJdp+hm-M#$%Y}ZeOyHscO}_foa?ZXJT=~(j5#?wkCfQb&7Ebw zTwOIakCLrTE`pJB=gwh16W!i%6lf|)#!sG0gJAb+T;^{d7FI)8Rjm{=NQb28P^w{jHGw9Hwx1Lswl&a^Kb`|lZpXiXRx4gfQ9TohpR$_Q9ka2X!!Rfus(AkBXO+P(D#v9gEtsw_OA7nrTle)Bx?ntj^Cpn9^tNxFMz*FY8GDaL_ONmalhTKfI)!?jHmu774nUt5p1 zj*f86EQLf;w2)FUNEbwyv_KKsdU7DhmJ4KBjv{m;3e!gB{sG*I$ffKU8F!7GxJSlO zOTJM*rYAQIi)OJcnyIrvO6~nUY?`Ln^d1B_KsI?1fakm269r`yzZ~gIcSt5Z)al;6 zT$%uk4xSey1NGmk1orjcq`mkF-AL+p zN^Cu+lJ=qFLBYFP^UP?)d2}Gd;|K>=TT7R&ridilz8pS>fR$Q?GSI_-r8Vjn1o&a?YABHnzd_# z29r?WhA^e+=%_qfbS-uW%CzxADy1S-bbT^Mx@QPX;d2M)&p>**wlAofW+Kmb=UobKZHR!>jV~6ZI*RENEImuE@oS4+k8Pf5m zUwcNyq_<5S^rH!B<$b|7Mr|=ZFeXL*d{$GDuK6%I?zsieG=0={uF*ny?tkA*r!tNR zdupQBn2H)V={$-mkj%Xx9*R+``_uRhZAzE!#QUL-Zy;Wp^o97}2MHF>*!VKCOF)x? z>dzWSO?&k|RonVoLITK!*nDA$hPM>r-7H06Vc{QS#4FfTW$;FGS{FP306P@Qlgs6i zS&H)N>Mk#bb_i?_9yziK8P+!A^bS$RWR3z1jiplFUdHVV^l<)=fwjdM;NynS>H@j-9>rnO~MVSGR~ zHwnyWluX^J40hMt_l%C*w?i9I?U)&yAQ@N4B)a4>zwRru7q^ki4Qb}hy9D)D)&uX< zeqgwDYrB`>Sa%h(ICA~syksg+V&rxcpZ%526hX`$>G4P{iMMWSoK9q$YpVOgg$qrF zfheu^t14Q97}Qm(R+-dAfbBX#EtSD)j&((Min(afB2$3`h9dybEv~)!SKI<=a_iQ@K&E;K@4Dl4DPb2L5w+ zuDVv}cdBy#nu!y((4ts({qe?7>;JxjYxfkx>e`mr4a#X)^#0H>s7e0v<;f%TKv{IN+xQeq+z&%845?fEWt+G@vkgUL?~-u9#>A_PA~EObLdo|k7H zil?>$O&~lMed)@WXO)5ljuSkrgH~>u$yRN4Wa1Ixc7Og^UQQu@ad9 z`cUDO#BgoRHX8%6tHUt821JHRn+R4J?BUu(bz zN1}r6-STFECst>Y(2LZb_HP(5|GLI9JK%G@R0-TYxY|4PSaux~+blB~r0?l3zgR?{ z*iPsB-h0G?aVKC9%b%TJ(ZSvRNJ*sz2XEzc2*>DYaLrvN>V}7hn#0)Z$7lvCL8)U zfvuQqDq_|Md@G<9MEjzm8$>ZukTbSA1w$-u^hxc8Ew3$n1=O&|V*ub&2jIT%z>js7 zttA(MRd7;4OD$)#=(hlZ#+;m-hD%n!?B3fL2aEuID6(4iucBsQS4D9P>$yZWi^2g# zr7HGU)-##|jzDUoVZqE*2HvvQ0G$oFh=m77&oa0REhH8ckGOjaoO~>SV zU`gM(Qcxeu84(&?#em(ECa5JOk&6&_>=c!A ztGYS`14{n{jsfmTt$**-eDA}D4|hvpK-P*5D$D{KU8kd|8S!91KqX8cgb?vOD0hq3 zV)1LUmH*0^Sx#JLHC^_U`R(cH>B`Z$XTI@NK(fh`iZIy54I3_E=vI6AQoN0ctNz(` zaon;ml19)%00>xB9A%mx>iCeYQV4Bv`R@ku-z%l<(>E?$xIo6i&#P#0M(Bd9 zu6Ap$zEh=2RJD$mDjV%I+BCw0gUj}@tAYh1{hM-WCnL-{CsUtq_3gOCs~HcH!n*8` zu3*zd{>ohzvM1a@G-^v*HC^W!7&sYGAF_UNKgjvHDtWqU6KYDNA&swv{Z4M?QE z{|FW>Tv!w$?13n-A3WXz9_;(b={q`2sCz&cv_Bdh7m*ZMs!LF@@ZG?Bbt$BOA-d_* z{A>hK-67WjK=n3hdM+79cbtqmpF(Z{qF(^2teFvv2}8v?@5gviV~@HKy>d{_M%Yjg0zM;eM-UMQcPf zvAHcQdX`-Y^7hu_-2Nsj{t&pNc>iP_^lI2Z0hM|dTlB{5`eNJx(59Z0N#G_P>nK^x zadtfh3xLxRdLUiaHS{XscxbKCUJB82o=cilNcnV^hI2v{f#~R6;tFrpS_$}DAh2YQ z>rABfHN29^)sTf5s2s>u{5ql$RW8=OPGzT~HC_QFJ4l>7`k$F7m5iT`rfFYJOLGtD zn%x2em-cN0%m{pezxcOb1ECbIzr%+SwBViCdCIRonLX9Fd_QG*7u*rB%FO;t`)oMS zfWIk7H3mQ}YIGeN0^7Jy9R_gulQ4>BReXHB-ZGio2B3n5p+GY0lDdpZ;KTS-W*_JmYQv|iN$$o^`vT1T@$h9aEfaOkfgKKa7 zaX(9)M+%1Y8yU~C5a&Q@dHMo-N6|uCL>>0;3n)U$BQg$E2Q{OEq)m{>c~W%PT9I)e~#9fUALw;zRhA~7^}%pQmW{&o=6R;R%_(Ee=R2S$nEx#*I;{c*~&8Nj^z ziys&D!@lH-%9u54A-Okkr(Osw)I&|^--4!7apt+dc$fCshgu z8u1I2%_u^K)96TtvSdr_(R=yc9aHS6-g%VYFd&~@Oor=`HO-6J7EbJEG+1QS?0)tM zC_Yb5aAz?6q>!iLb@OlKGOx4`<#Ah@n0=YuC%EPSqAv)h^Tkq$ke`mxX}cxcNH4dM zamY-Z=niZP&WIImhbdP>1IXwRst>p+w9CkR&>(ikg^@$1NH1pTXZVWB(WO4qoy_wz z*SDok#-D&nQcW0I<0qY6;cvfPR1e7Y2M%8L9L%E4WQo8_nHqK--v(7aEY!32NwKG` zx|Y#@AdrPn5E%xfq(Q_KqSiZ^{-l1}MeoE~zS@*quReM51ml2!;gen>jz^J35#Y|* zj2yAOnuU`1yVjqmd9(qz@T7uDqWff9k0XGNH4JG-!C>dfS4RrIe`Y>xrtRWwj*gDg ztVsfh0Y8QqFM_~HgvO&3Tii9IJBZwVf8XMdKS({PGhI6u#|$ES=fz`;D1ncuGvdP2aaaelW`%1&I#Tg^7?y47TuW>jb-_o9<~o>V#0yp1NDh zqJ1-_Xyf8=v>*|;2oo0`{H&2;CNcbnh#dkk?rSd0Gg9tx^f-Zx(GDL*4i}{sIFMA( zb<3x?yW5`whanEHX_D91*H?-X3Oc^XghXVApi}Vb8lGCf&sM5gcBEfT^=E2sd1Ztm zx7U!SWBAiC7)J>?B{xdnIuM><9Oy=yf`vhj0#F4s_JVMt=UO+B-_a&#%+!_jol3BB z2w+%AeqY2|Po4xett#FRG$~|UDwQ^s++VKHRrROOrhYI^>@rgv11mc^P6M;b>6p%} zyHG|7a3O7>q|1`1!$~+snf`r|NkixK)3LE*_#udnQ%^$WRWWd{3atWKv68E%3j>i~ zoy*()IhEn*`swNS@?UEKzCbUkyHH|# z+Hmp5?Y^F^k);OStedaiy_UleSROB@z85{pnElZ)Eig>c_HOR(Hbq;hzoW*dHkE`h zXKZY&Qra5&uhMRaXxn!=u%nO71`DJezO;@#eZRC%rpqeQt#Nr05XL5*FuH4{(}feI z6O5V&$ttq@Omh;@V10(v734tJ=PR8wDS7t|Q?FbZSwnKaZ{XXoN4#&KKy26(dJrd? z>UxIV)6my8ChR?|Do+Jh-pIwspdqj{KNOZrrE8VyC~mxCW3yfK!K?Fm%Ri?wj=H|O zPS8oa{W68Hw1n)ZuFkXp03ga`fua2)_IZhBNAUNiK&S5&ob&e7@*_oC&+bBqh|D^a z@3wKT&gitx1S^kAk(~s(k`c~WKT`9xso%G3KI!@Zh~DAW8-4r=_G8QZQWz`TSkTP` z;}|HK4OWtscWBcZWc3)v+I8zzIbDcG^b1l+Kt{+?^;`$&Sy=}!cu-eYH!j;Of2(## zxgd6Pl&oUA?`$p-_)Q0(Z=kjx^9i_(+-b>-A#VuOrn?9vL!K;3su}wwwk~`6I$BV= z8R4&B)LX8NTJ4_Auj3^Y2kj)13jm}Ez91wq3D#3n@SA^FXBBg1cgwwd_b?71!6Zxy zeInYNdPc>-?Io`#_!~z@Fe$(ur;4HL;pjMHGWoaPjuSw}Xxz?qOh?s6OTi#7VF5ZE zno6jGg%WR9?X!L{=A@Zb^31bT&Z7!=IELzsV2^f>x+h^3FMbCj2Z=(<{RQWl3FjNy zs2wK-CCyA}s~pM2N{#r43g|Z%hS}o%Oqz@do{H_e>+O+OALKojEW0cxgT5O$E!HP0!oZf4}=Wt+Wm zp`MoS?psb!ipV+irz?tSiJ0_(r3HS-H}dDbeY$JVy-y=#H{fF`#;M-&pQ&ss$@A5d~0*oqkcop>Zn`h@S0u(2vyXpns6iWNS_XOm#DWq#pFms>!82ms!M;t0+D z#5fn+UxE@7IJQRQp+Y@n&9NcQ1N7nE{Zv6{PO02qIp`1h0+A^j?g2{E1TpE-^D?w+ z$*XjAgI;Rf;k+13BX`M!FpW!^Unn6{?5Gm=nz zO~xUyyJ}Xz0@KRLI6%VXcOhCr5%=}_H`izEye-skW0Ig>E@Y_24(j}wGKHdS5LyCC zj>kI7e1RK;lrkig3My8?Wtf)_V4KKjH1dETLFvH#6ZEaIrG!ETh9)s|$&jrW0)%z! z365PRHVmItvxz_G(09q+Re)X+k0CgK_s{nah3`<62;BZK+13h|KaW!$#ruDpw{8bM zhSCs4L8I-MPu~rDt%51q)7sO7Z;8hs_Xvw`9%cI5SsV6>-$V>Q$64!Es=YG7|K5D- zC>FRj8)>khBLNGmGwpe_+W+2aeTv!+{C~CouWi}lk6*D&;EXU%fBzv*=TQy7E=2sg z`i)ytdkdBuFxNpm)q^B7Bze2Kp|fj3FgyfoR>X%qxpmO;wN-`2vHo~JkEY;(XcB_w*%%E{4+iuB6w9UR6(^6k`Z=wKwt!+>Ejy{V=r82k*bkjXa{u-XnJH1XB44XKroo z|MNaTRzlhW$t;Wm1NQC{@Ml&qcfU^K>?gB=^iIAK1Xz$Eh3wnkAIEpLO@9awJ5{>W z;2P2+&+P3pKtX;P{iTA+-b5MpA&8)q4?CtEV@S>crRPKJ7M3WnTk%7XmMcz2YA`ZB z>yVHbsF;slbl$+Pr@CSj|NiQC9{=`F;U_Td+BD61=#(`7NHdSXG5_$EWE|0+WE}P- z%9y}{0H{G`$p?U_SB}yY3^I;t>mb1Y`@?8mtIQGWO#B*hhMPsqxzTgodffmY^Q zrzM5|fQ+Ah*Q&gR)lU4*uS2f7_JflUpq?BY8XUwiWF(5p%YU2A$o(3p^YKdP3_`IB z`}^6e)dpN!HXVk%JH%|fT@*sNb!r9ge%q`G1S7z|3~)Vt7aj?{KEN42eNSgc(blG6 z$b9RjpYLlcc@}l%_SH?K5sjzbT5ZjKH0c2FV30k8&}(1%Oe@9#iAEw3e0$1*&FpQ~ zsK)t`;wR6m*OOQ5cLnyk7luj=_T#5Ym{9 z2gEhJd|C3ZcVm~lA_F!5P)H!&o=tBedk3BZv1==5ugQ{+KOo}}{kjai>C)?dUTS#l z{`ZSZt8do0Gm2+JhT5Z`OG%YZOpH^s?c@SWYQLpP3#=%RBp-s{7x=Smq>aUqwiPZD zw8bCa2{d9V0G`%;HpcLwgbo;TA|Pco^tZuLClE4j3Hm)UF5;4C-CV@-9y;k-_$)f* z@syx5i5oS3LfSQCkpxU9aemg6nPwzJ+>dZhTR`YL)ZOwuU%vfHp`Yr;K>A?-I#PZI z(~NDQwx#L82_X5<4(7(Ey}Z#zy83?TjaRh6<5PqT<^O^fFk-Vy2;?^Hm)U?pA)E8} z$8p!(fX_Ol69L-_iNz<$y}#DC&j&39b62@9a&p4}lqKc@8ffE+BU_bvvy?LmcF|f> zXl)ty&Sxk}hmCJ_!*HrFUR6&5U(wBf*rzQiuZFg54VwTara)Ot^xedFVsHMFaeR6R zGSaL_5S%?gN~J8bfz@Fx$vrprp0aFx9cvGPdH^C&08wNdO~iuiRnPn-&;N9I z$nuA?<=cJvHyMGYQsw|-KrA)76)Er2;UXdmQ5J;!Fmn7vI_hI^5$;t?*C5E-8=+sq zZ_pg)n>p{sE%wj&`z8PDGo$6^FbvZs@I{;s1aoM~Ko@RZJ}7dOP$_19za^KYGcAiq z)huop|2_B8oNwIb_&O)t6ED2B%60Sk`MZX5H^gk-o^-0?arWlBH}r0OlQ9*&ysh2( zn}g@h?lrh&mD8d2!5r(pOy{DwwFg93zeWe|czOnOT>YSlF^I=> zlrN&=G1HLoycFR{n8}GKV=}(hrR>B1vD6$#k(?*IhV@i>WFOfcy;bO%Gq=Cw+zu<- z%Pe`x^Df0iVa&kp!i@z+`(ff?a8<(SSzbims7C2JTfb80wK3aQKtuJD&=csn8RQt6 zU}AX!kTqYy$2Y>o6BVZ`)*zic3-q6ZK#VQi>_M*3|L?G*HhypMlg1hGXfahWLs%jt zT0rx{fiGL6<>AFveMs1rJTpPCDLL@|5>Lo@Ldt_EufPw1l7fscurXRrZuTzKtD~|F zenDsTmq%^UQe>r^^Ai!aO$at#*zIS?QU3S`dQEMHW)mYJ(iLn}NpzhC#o>~?`?x#f z(a|+VO6J49`Rszp@C&__xp#(~mW+tQ?em=z_Su@-tv{8ipssimA^zn<)e1wed`mpn zx6QY=#>YCDkmN;R!;9HT#t+6CuOc@e-3-G!?Dc0$k8*gUKfY4!|M@;z_EzRU?RB{J z7-1;n)yVk8ix)A@N4hCr9!0NuVKjfJ`)ud;44!r1aaKXAaTMH5kK0mfRbP}_^io=O zWp1?dA~mJK{(s~+zHj{;nI1LAFhZN#=Dm^(2yq}X<56~rX{rr=!<{kjf3Bsk#0$l0I4C8AJt+@3NE8@cX4LH-*>p*KmWOWXsm$Gz z`!a2wAFt}ab$0Y;^`Tp+>%}H=pt#KepvD+i2@)k}Os?xBm-CLhweVZ7wX1efcn9Ky z!Bylz(XY`RSwF;uq-*Q*KdQY*w)01)@LwK|-|brd)?dYLh7~<~2fRw1XTw2MZb-eg zk%f{`0BUgz+JA(+iswM#$=`vuj*9DVxjJ;fvMI7a@4Mz_JZ@%J8?k-z5bdxNm!tNj z1LZ{gNYt!IYxuI7da=G+R!h#^j3-^H>?}W~i5v6G zs9`M=1ne*m0(d7NC!tL15L;kydv1Z{HsqK+tc~8o91n&13$Zvwfe1luJ+}zt753c~F z2O-WTww#4U+U(gQG65DQ8HW^L#BFJV*&QTrQ+x5Dp|>iq~?7k zDsj%oIEZQOZ+G$Eg#*gcMJKjmI_{TIXm|VDAq&Q1w1J7zLuhMdLJovJ6hIOHA!Amc z&>W{#%jVZ!yteMGNBrsp|FjaXp8xremKO@qZag}Qk*_>dWZHw>&NR2Ma9_d~O7w?N zV;|9_(T4Kob&#~uCw5cq-t0DSvU<~*`I=23U!c~HdV=682$hqQ>5Zl+5;KN5r|}pd z+mxrI-(k$JeD*p)PoQ;+*~sFOp<*6xDAbl1+C6i7M$z)7G_Z91ETB3fe3P1d;Xkj+ z8c->J^8P9Qk8|e`#^ld)dG#O5lMF=pArAn8YW_BTV+3N8-n$jwa-*GWr;(FFtiTneB>zc8!f}o5H#O@ z054LXW#WeXNMaf1)PosmqeiW22Z#;3#P#3M)bKU}9h0==Ot(GdOgNlWRv`m=cnyH=!SPdl7&DGEUS84~l+4 zN>}x!6-fJ?Fum|Fsh*V>nj~MPX1i{D)tey)abIS?)&AT2U$0ODG75w+12X0XDWH;1 zI0)VnEZ>&fZ!(AK^J2F5?cqXCYmCszK2??26g#c$bspJ(-nMqHURXAWP!6%y9raq2 z$?m^}WIy%g%_L|tbD4?mGXT$OEH2gGWta-u6ZU7^z5LR$6l%wZf4YjiuL%h;QWKJ) zt#~#@M%M%wzW?hM$l%*jUZ*eq_W6l6IrGA^gEt^XNW<*K*N0tm3{4`VG9K-~%>cF9 zZlr`r9U%65bL=Fma zkyNt+N)AN81j0FlOB*O@V5Ei;SV)3x3_8%7Yo`zQWxdjF{OyolJ-IM~DxEUTftG^Q z^;(WIjQ2Uv1d=~1K7<2D7o0lW_8RCAff+DuAWMe4C`g|GM3eDUQ%H2%KnE9KgCO^g z$^36%JP3G?D}}kg-Rj`^0tEkwVorYzJw_j?B)s|g0jzKG^|F;u{DujsLd>Aalyx~*eTsm@<1#ax| z5w(V!ug_IJnZyRF1v)KQaDP{VKOx7qWr|pN9b~7Y_B>tP6AlZ)0 zalzgtOO_aC`@>8(%_WXogWe8CiI4YKy(RhK%2c*HY5Jn;zz=b14Ux7Mm_O{hNTGQ-qa8{ZnK7-^#~ zat#UXvT}IlI8v+~_3}}I65Q&!6MLq9C}+DeKLcQ(-D(JJ5g_JR=~CP>tkufAH2RKN z;yrgTiC49O>r>Z$qqxW6-#Sj3bBwl7jRv-)7Bg z7v$N~%}0pc1=o*^H$Hr5SD^Xv2MH6I>cjq(O;&JtRUTbc-F}(f!zcA7#sH+2Gk!8+ zP}s_=EHK8KV6=QQo?ii~iZe5@=2MRTt{zWXe}3D2p8eUWdqptsAtm7ck^~mWuEuM( zElB#E$5x81WH2|73vHJUZaQ_iZ`L{&zU%wAa5)(89l(>qSMigb@CCbF?!)YaK<`Oa zh(D^7iP1DX(sw`R-8bVq;v;QO?Io2!rrn*P(@If(fedy>sn4dxA49c=`pcxzhH0Cu zc$=&fSJl0ikg8*+g>eq&xZ5<$FGa(gr>|*XcHs-aJhRjMD{858{-l37?ePoNG7k81 zL_J#ZxA0u!JN?;Fjk56e z9gr+R@>mg_qVV3sLj*sx1ulXDaJw0COOddw)4%RA%R zh$LsjwSGE8tRI}YeOd_?IJ|~>HSOr0k=VqE!(Ffo!N#lq%7?>4hH-lad{wnP%bfvD)o*(g zdZ=DV_ZC(@I{KJ+eWEKe<7=I*!veKe{&QbXX2x1;?NtEp&MeubETF zXf00){d}eSN&{mU4|MK?F-#=fUqIU56A(&?mWVh{0lRXmdc*~&D1-hIAzvgwJpsWz z%;e>Ats(QAa@q2Y_=(4pu(!v+vp3^HfjeVJ;7^M{(jvqN&rRinBn0Wu2}awV;hqXJ#@^HTVJ8lN z7Ah+cuLRg6yj6@=E|R$d1->Bi&(F#GdSKP}1h(zfQ9|A!PWe0z#|Y6JVrXRkKbaM= zA07ShO^;HT2aFEHs>4Dc;?xV`cHcKlZo~63)4r|hc0Zm5(Sws^H^G*reF*4+qm0fo zPYbvQE)D{29s2@G0Ye^$eOHX%w zthI3DAEJobz34`vuyC+mpMl7`MM2|`H?0Q3vJq82nT3ug<_>cAkTnc z);qWG(b4@KFnW%R+hnbP!tcL5YJd2l-2K~}jNK!vE=?g07eiWoh(7G|p{D1>&jvej zGl7>ay>Z{j#!F!%;z|#M;$QqkN5KXsWPZ9qT_CH!3S2gR2-Oxurul%k=R_(;^(j*5 zYL!>q^w~?xqd>w9j^3g|Ep`6g6T~jK3yFpra^QpLwuC5q=68Px*MFY?)>&a<}d2R3SXNbbSzn#*bPaFfmb3-RBQ6SW^ zqXt6K9tkT6Nb2Rxu0U&K3BZOfo9uu~sRh%o!T|iGg7neQ017gRPyw2ozdKJW@l_md ze-%j<2-XToq&TNFmm-Q3Y2yja#%vmMY0TL#-!$pbKlg@znEA`gPEgll9OzSj85;)X zcyM&x=46?5u{)vWBWUEQMLASxe3_hVy^)ecnC7${keO&+7e?>xJExoG=!K%C%iOpG zxf3S~+FAMnF-HOP2Uq-9re0XxUbYy9IpvjDS6i_rvNK|>h|*wk8USR6!PNn4Wr5i< z1&3K1m7KgsM~Osk8VGxiKx~85l!*8q>#EoUAwJ4hK^zPuECjwN@w~;Sz5lm^v7E}F zQT4GowY`o1c;Qv>!mIb+3*9?$^=oun98ZJ$$c$>;W4V$Hz6CL!J7y1%;Qdl< zEI)bEAKm#DamBNONLy%fg^7Ybf4$mJFaW9}kU2o^pJ)^0RO)San6=mQTDRZ4t>_bN zf5UltVR`sm%>sRsvAYXwp-dOWn>TOX#4)(PWMqI2vDCddG${E?x58adKKlO0bZKQC z(S2q%>4N4VYF+rOu84#*??1}yr%sf+eLQ6Pj!}`cKplO|e|cQUehrL)`Ku2I4>uub za|MFy1RV`*RsvDL`PVmyzChFuLyK&@k!MX5wGbKj=g*H4dAr&8S_(DYiL+8jgsVA` z=4Y>W3RaP??86ll*t8g0(}zy1aM-)K#O$-nLq@xuDW`yqt=Hi!Pl zv90=jFiR>PrG;Oep?v|eK&LYIPr7HWdwyK|SUEsZFvg|sPzbC`Mp;AHApQvlIZ0ov zBu#Z*=*;=bVIFW%kuBuYCW-b!b~<+9FIYWo*=bKD{^NTl)D?# zvK8hy%aPM zl_UkGf^ES|I(=^=uEB$o5Pmf-vvt8*$nho-b+LHAjIV;81t>op$?`GM2_|S(i^+>| zz|zRMDl@d}^!lu#nsJh*JI%=7;l74M$GfLjxot^7ymNtOnrCSDw)_w%&4)q-m0$IH`3U(7%+&nf@XILIQzZANT0Y(rE;PcNv{gMI{dhy<&Knl-nVrKl=p2+V(2 zrf@E4`kN*Iy8Iod24u$nMDVV;DFh#rFxB7Re>&P>UD*O#vkg#LPR$g8Q7rkWw&+iq zir9fjH-Y%7W>{u*&7{R2FVuRIfhlw)$`sJ#RwXoyZ6j#?oMu$kaELYjZBF`&k}Pwm zRD*ze4sUfIvD5{@E-%myG&)dCsI|Lh&9ql$&(R!P=SdF~15!cd0Xe zqs`lHaP?5bt*`o`N3SG`&u`=x0!#_(UdeSBTzi&cxu@fB+Vw^eA{59tw3#&GL@sN` zHP(#U*xDLQ2j#gi*P%XBNBk=-$9%(dzJT*ro=zD%03I6^Z%q!sB>AuTXo{38%ng~W ztS!o!_lD6$3+nab0z!jCtQ`fE+BjYi1{KNsJCM5m`4%;^i+`rw5IRFt2N|#~3LICG z3AUKH(0)vP!P`@XOP1u6AW^UM+ZNaqvrT9qmjgH(05hp~wkam$Wj*m5$drIbI^1%8}jq$Tp~3iJg(-06dqP{3Cd{tGFpX8jiPA#F<^C{b$9!tM~~{O z4=bz&n3g4<39I1>l~wT&-dImVd}ICbnMF{=CBRtj^}kjU?4jG9V-Bqj5j%RLs(`kEDT>6olss20BnyuTrJCluM(F!awQyw!R!FW?eE1iUWZWguxlJq`fqf z66SU|_1SKUCG_n8iFQIc&_v-PMaQ?deoIY#2v-WvHwDCNKV-DVoik6SWp74+n;b$WP3|S~xwFmUbw3(?B}<5f&8f@V%sK ztqxxWEvT0sfJ;{VBoEjWgz_qz3HBg51Lx~W3vr1FD&Lsxc87#7J8(FqRb;4Hh1g29 z6y^#wLf?)a49Zz2<1x-GXx<0ah!{~0iJ|H&(6|bfqIlWw;2l8mYYXPZ#IMF8*|((pJk2Pk;JfndRM+PvqvL#2owZ<%)wXaR=X z3847k0rax)`qT#JKf zC^KSB^iZB{Usx(ax8)xXePm6&Rbri1_^Sb92&Qg%;D=x!^St{}M(lEY%hDLvh~nt#ADpqO0;=JDLNKE05&I2tPRemY)nUEtY>`ck z`y%Mrq^71Gu~M}qg{vnBTqE8NV~gWp-$SGgs5Kcf6p*wl?Uh>hH8(;HH&t98bO(vv(|@De(A9EViH`j3kA-#_ zj~?7xL`i`_+De7g*HC(GiW*r3!oZIIL)v>lHJPpL-w!hCj0MKApn`B5Q500fMkhMf zfP&IRAUFspy>}8kjxrWtq=-mUX-X9pkQx*P=}IwDg9L~WAwUQLLXz*^0q6Lhb>8!T z*ZN$p1u9_jJp0-EF4z6LAgAo39az7@9f(i^!V}tqj}W58_U_%Qulf=8%W2g0oIx{y!a<2&7Yc0hORn-b62s*3rlT@x+6u_hRlMpqhX@A$Nk?1l=d-v{3~~Jr5?h`ZFcBCwwGr zdM3EdV^lY(dp>)fetXuld_#)KmZM~umjr>sfEMX&qqxKa>GGWI3{Rxy$ZUHi{7R{U z>nuX5#e{ujQaGwOM#@4HS$9Faf~KF5I#-@1et_v(?)+(Nc3>7x&B0BHWT`raJPit= z19RxinF+L<(a@WbdOt?@znSkq0ci)yV;06ukm!Ih_6gxaOBohmMhA5aZ2=V5CI*$s zs&Ywz?x^Vo#)OJBc)5fqn02s$LJA~W5PU&Z3m!u>MTnjP1mZx1Ef1g^*}^WjJZGj# zxm+pNzb*3z8B=ef*?+#n!dvqNvXLJCRG}SwACh?byq|%(;DGSKfRn|RJ>0fIMy3e> z`&X3UOh7dRI0*M0IKY0)aOx>lMQufx>Do43@$7F|9UA47#Y1F+3Y)cglRhQgvZ}5I z{m2*o2jQuKnVu2eFaAGj=6KhHsxtnp*3o3++}l)Tb*N`mtm1K-tey%uNoZ?q=Z^K% zzC@}3LdU?7w1&agGUM{(6VcY^(xrrv0i;TPl^nYGoCPg6+J9Lpxhq`pdUec9ZP4sz zMlDr%Q!DORO|!OFCYkI&$Cdxb=ZVp>=sH1_VScCuJd+?bSJ=hYsbe2g#l@eHY^+QYXZXiBAFHsu%IK{dsybQ8*ffo-hNyZt&T-bEj6La;ExO z&lshoIoq;KO$G*zpd8DBByRXG8e~V(ti!?KgGdD{2a=(9*VjM&Q|$j|wNO;Nrw65F7<0$*}LiWXiCgHfp#qemb_;B8|23uhw#xfqkW zBQ~pkYmiTV4uBX-*SQB2J!n*Nam@$R)MgokTf@=RNVv5!FZmRnjm5$Ov+A+#)>8yU zn>(3ot{f;;?m~!~T@KQ{_vkLE^hGxn!HUo()sUh!IuOP{wg&dVIN2g0PaPqSdWvwj|$t!j+#p8ADNyXG%6QqgkluJcy&3TT+wwLrL3&bWqtE z=ywu|h{7CKlk%CenJ^s(1gLkA1+#9Nl>UnP$t*dlW>Pj9>0gt5>Z~6wiR7IP9+&}1Uc>R6LW)fi zc_t15jZIMOYbvgGhU@_JN#>{;4H=A{ko8;*ClU;_Q=V3Ew&LcSERye$vUy&#a4H=} zH!kkQLk(f%cyVWT@h?pP*a@8v3#H!snuYIs0lGeDKFQ(JX!7J1hAe(tvRgDETyN(} zx(b z#Gk=OAjnzGd=fZ+{}Go}wp~!^RQVZ%)Eh%LvL&8F?w3AUy>iLb<1pt{B=UiG6%K}5z^w}E0`D}gfbiC&sbAnJL`88} zoClVS0oHnfW!C|y$%uZ~OkkJ;jB|7%fD#1)N}5{)NPU!Rah#&2w)t{@g{Xd&1mq?W z7t5(S5x3Kg{97TmpUTbwP*v~K*2GhhkQC(~b!g7%i$iBJM46kIr_zmzc&t6cGaI&& zqZDjgQo9^yA{HihgF_wn#_n&n2`s;z>!X&yJeQ+Eco?NP^VU`66xzu}KPsOW?i%Gu z^;6c-#Ha1_u*RwI=#d2^8*t~WZ7k~0uCstO5LT`Ffr$ZU=e*2XSPqzvM-jdegk__} z3V0Mz9BechR-_OzV|a}&k3l%Xj0`{(o>1EaYz zd7sFQCk2}^!6ftQw*PRI7yhc(st9EPU@JH9$7C8zeSaiRW z39rUS;4>DLsgYm|q+&&&iP8=hz)E{=0uFCYxTEQ-sVM=UqIYt1JbH7uu!WVVuT6%Bphg$d!c` z`X4?>?T6XKP5~3O009t*3RL@7Jbn20MHX@|;knU?i1!-=F{#YlD;dXrz4<-pvJ_!r z{!F)xcKahz4@Bwnfc5f$R1~3Q6ePf!0F%GsP6%SPK- z^_Fx;lKy+X<~J>XRSk3}mzL(RE%j;$x#YabeRthc$z%K+pDpJtEz9+KAgD{Z5WMm& zL0I?t50?@CO67Y6iAbM&cXtk)M^RGi$quE z%>O9RrNVyhS=Bc64kmrFcC~&gERE_fr>&$;e=LL@+tt)Q{l1CPda+1Z7|a93Bm|U3 zeBU!~Pq621;WsIw3yA+XXkG8DQ{-+Abw|~JH90w$$PZ_bGopCOgkkc>moLlGBVCkZoW9~8}W1dTPD@c5{YbLSNp8y>G zVm-^tF2SL%3IATe`0jFw-G_qlxRxL6-XHf=TyJg@Hs+s>!+QAE!Yi!ti?+aUNN_*M`Ae35&SYFR{mPUF=z*7d1r~D4 zNWTp6a~Q9rIfoc=Wj*d)Ul{=pMyAeif3oB5kQwIssHhC3L?Qq*DzkJY`GA}mHO#a= zRb0Pd_$`t8?)rXvy;sv(h9Ft-!*5~2!Bq#=r&S^cZ|8i!FH7eL#nig#s6lr>BM_a! zPu97i$J#zhp&C(cK>ql-{&~hSx#Z(K;X0QRi<3({wOTDB7gmDdeiXa(6s0Qh-~Ehhud z6MW7972==xyK2sDLCoA<@n6u-;Hzr9Cq8a=F)BEAj#Bq9NN{sP>n^{Z-9tWco=iiN zY8tc+P48fLgNY%;v_)xljHUk*fo#QftNs7&>evG>`kU+m$4pfWy6jxWHVzea{L$j{ z!plb=G|XFuUgSBUDGJ29!S9VdwQ+_vt}?GLEDRjFg^%z^96Ms9nR!+8BO#2@TAiAs z$R=XJCr+gsglsknbgzpc8>TYGE~-eu#eK$anFO_X+BZB)?>~mV%8+vhR|yBNg@jcH z+Enrfcvu6gA`%hvC@UuiMpc333$j+Q8-PE_dO~;OUD|uoT+%qd zr@DRa#8oK1>_bgBBw#{~W~4ntquYRHPC(uLIXe;QZj$)^g$)G#pnv}*klI~3_8KW$ z>QE&?7NxVS$#k41>b6ZxJX^9zI=}e-ci-jv5|*1ti=OZ6)e*8N>Vxj1xGA4Sm<7`! zFO`!KFAPSQUQkgx92A(Yn=XYj5XUOE*l6VR-F4fP7znBWr#A|yeL1D5-wiSrGz)x~ zw(tro4)2u?2XaMAj1$UY0m%t75iei*R?zI|%6BKTYA(802phZ=xb)^GqbOp=GhH7+ zai>`gC7jL6E)BQo9PuX8xzm^2UoDon2&bVCpW#F1cF%w?_b!rWqq+hMQEd>rT32KI z4L9qbIO%$(YhLTVxs^`P7wR_1dM`ygVf zP*(v&Ci<7;@J0Z#9xl2LqP1p85vjez?!({J8oTuMqk2LYy6VC{LPY+|Ex3g+xk@ZNuX*1KO~$%^ zJZA@Z8jj+CPSIY1@Zl7@*3K?HHYB0IFsdc;&V_uuv|rnu%74oZAW1th(g*;HwnJ?# zKy;%47QlKU0TN1zf@)D|6D`XV(DJUr7=;MG{ZD1-Al>!zHK7urdtRn2I}W;{MJJkd-mb+eLK^^wRdN3#8`O4DU0al zFawrb#kQlXhrw1?b4;V+&n+9d0cAZRP3r4cY-FP@K3bW%MziFb&xoo0F~q&) zxcVu@yPV8!XHpaA`PZerA6^xzeYdp!{Zpx<#HxAw9^t>JT!K?_zIy-{Afo@#t=$0R z=%(^4B2qd>4%UWO(d-3+X+XcE3kKc-Pc5aE361Ryl!4lDn4mc;eE3YOcO>z?Lf=0# zGt;Q+*(aAba!|;^PecOzJ=A?`;DAZJaTz8Hj#=VVH>Jf4DTM zNlk)XD-#L&`TH=*`{b&BfAjzPvy>s6uK6(gzJDQWp8qEeN7U|xO+s=AsXA1JJX}!v zHz-i%X|{xCB9MT%39@A5~xR6VuhW@;f+nQeH}(FaPT_dR|_8c_+%Em!pyHq&BfBB=dMK@v{qmTdVd{xZOc89X*R$G4L+Mf*>sNmxsh0bv zGJS1f0BvWzqT!I5bnM4!(ia!V#LnTPai&j(V>4+$GC%rK27Bkfp+wbYOS~^Zb)sDP~bF7Ov~`{mK3B z7Z6*jXCj>Asi@kw5mG%N0|7B#kfoWh%!ZGkh2y|K3Sn$&P?C+Zj`?r zl(X}0JW|MQ#cK)s3PN)V&I;p9+U#;LWCZ_ofMt5~!?)qrNk~M>M%Ku(f6DO8)J-3N z*C!pXmIys2FR6XH5i1zJwr(w8R#jWZF&?i^gX;=egv{(M@})O&Eo$WMjJ`+XFEf~^ zC@(o+HlpVIqz>tFzWu7W62;zbv&ls%;atq#lAJ;fWl#TmxrEpt3-ON~#SiVv&YWXv zzz)Q%Q+v=(&%QgQz&rwq?q^d{l!)B}+GPVg`KF+RAVDp|j2S4-@e&>5rDp%{K@;$U zdWYl6C=i3cygqvSF$ycyNh4mAvErT(_WV6?Ujg6>-hoeef3r`8^6iK(Cuif z+=raPC$!L+qPOxn8z5Cu$b@h_&Y90R-nB2gF2;8N^z6VO0kxIy7P4dA%(p9rKK4Gm zcPfLeG~&$_8IZ}br6@Jsw@5c=wh3~-9PV7$`8fA$5(>8fbh7DbcdeUE5KCCFMG8tZ zcrJ?%gJRNgO-6S3Zgx{5{lf%NonF=k;<3P(+ zc*f&}+xI*}=Fcr*5N^ZF{D(h!y-gTR0R z1BMDvaO#f)?Ix6(R)8yH8QzFsXDekb@H^il3YTs}>dh@xk@D}iv$pA1OV9=muO-z( z0jncYKCx)XMeB}iksVT22DTHbcf1Ar(ti2I*f@tbyRII#aM>jI*IR)T3>KSwq(4sw zAHyzru3hwd8Jipu*G@Ux{Cwn0vkeLQanA}GiyZ6J9_c&V*rc8%LS`uaxYLQ>YH&{3 zaJXCem;oy=)W6nyYPy}{RK}4EuAE~Q{U5hI6i7h3u*c6X;v#wkQ7AqjR4EECE>w~& z7XrqmhE;28eUG#YV`^Uw-3TF74D~97+#-BY?j?R0ibN0yTSoX*qBF*5q$$l zw;)khaZC}xq#fs)N4zD+>pa!6`)>Jby|uC-7!e<8lj(YV2>+3+sZvdP$DqH_oJYHj z(Hu-RY#ituDX98B_w5x?U|7@^WZgQ4rYs`+#|WhJmUe#K&qtIx#kN`$uvjd7gpf#j zo>K}_)l~&qit+=ij#~y$Vs! zEutS$Vy-(EKFjg*=CK^P-2mdoM-aIA1cxZa>ZT?W#x2-WtV#GLq=rJJtcxS6x#z*5 z<5@w@tH$yqOha1`=J4Otlq$Xp>CC}ID6|O|7d%`D?jZzjLTDR-a#?)gd9STn z;HmauVQ1sCqUqL_q5U;M)skp2&X2S%u+i=c#(~y3;RKmWdFsQAA? zfN+O0>%S`G&5+6p#ZFLpMZbE3Yo}jZ?YWc!SjFTnPc^5MT;afikqJL(UUzZsNwwTT z#d4?oOD7Ehd&OEPSuGK}bw#&m|a8afaT?_m9^U8>*we7ruJB!Dc31i%!~6cihE zeYkHpU~o+lIYcd#?IG_r9@i@)IXmZMkPHWW%$%SI^xZFpJj3}`&Bf-EF>J+D^K&vS zeoSwWzV@!;XKobemZn&4?wA3<0Ql% ztL&hEdz`2w1;-zR^5+hx%ahGysP?B}8M;L$G8-w?;hD4a%n9MySmfOl2SsV)v=+6V z9pi?DH^bAsGz^)#az+{lVozntvQCGB@MN@6s1kAr+gQZNePiyp5DXH1naqHU<0of`;z&LL!? zCkMf{fLidlTx%O!LZ^^+8r-^U^+N-(3AJ2*SGS`PcfeLKc0j+hLtY|N>XXMkLR$J=DmY}51kYiS>k70(z6 z`gjE;EEQXN`Ln6Elo=vNYP|n%6tR=tQkZlR?3AZA}LI z2*18*4zi`SxxUAe6Wb}Oah6OkzaEi_NZ|02RasbSdk@yQRx{ONvppkNfV-%vAA?-Pl(fRO`5L;bHUrVoMi^8K2&(_q2XXd}FHc zdheW&276+mZ<{LZ_P&kWTl#u_IcqS6^i*?bDZeP^ceWLD zW~coYw!}!Pb@YX#o6*_6)b{w8(F3+RI%?Av4oM(iX&pEbnPKi=#%cM;dye?afYguQ)Uwuu-`Yn|0T3W8Zv-4Wt9p zwWYm#b#W!5_I^e8Ltdiv)`7KJ55NaiZZ^{Hs>Cuk{@!pzhQLe-e$M(sdT}Er3Ot|Z zJl9lm&Lnq)G}>J65wFdIBG9RbQsD3bEfDORRIM~q39eIqSvHzy@G*2;G)*XOuR(yb zD#muM9YcW;kB4G$m&J-2cZt?|lFY9V(y1Y=16? zigN%UOGh0aaUt8IV#$8Ar3#DOhyoxy{84Cz8WPqhyp4d0F3JbR&)i5lpKWa2>0Br| zpgpIF!YT;ljHG&o4o=0;I=oD^Agy7YQZphJW{~tL21#Pjbv!$(7~rL`{|Q%4y&DAu z&11m>oe8>I`0_KUWRHVd3fh5b)YW2Tx;)i#AoEAa=K77~Dr+^h%<^sZW4PTu0w})_ zEZyB8#IV*C#mHG! zZqktZC59W(W>0*(Dh||-=&}w^x3xi&xHMHzP~Y^4B+%!uZXQ{1Fn3JojQ|jD9%!P#ZUjaaBl^Kj%%Qb}MD)-kTX`o43sg_boxu}W|D2sc^RaUw z2D6;n+4yD|liAO;c3;!xd#8bJS#*b3Z!H}ui=b>#7qeyZ zzADzTmf0pT&6-h9jZ>ozEfCal`|o#m2X1Ow&k5#jD=|s!Z!vHSiglsDwIJ&B&fHxI z6shc@!XW?Wr{u;b>)8(Vax$|bD zZAK4!7t(>RC(dF_uu5U zL-{d9GKT(B6u;sbX{sp8hO0xK)CqOkVKN+N0Xd+;P&?%aGI8SeNArz?Lm&OOYJ-ZY zungZn(o=T+`#UPE-atNc44U!tgq^)oVv<~VBIhBF5NaRz6@sNh5FOn~)J+dn2>>0% zRE)};us9&)_GT42mCNMt&&;bCWtY9W=;hOJe|u7c;9bn9H8PFexSo3_;Mfm|Hr^z@ z5=GhaCUK{KN{HmDK~&bK6d9S`n~J1rX0%fjuG z9=}T^LL7=vu3m!u8!1VgVXgG;>?&pcaIOoob320Y*CsKRupq@VClV|R<9mbY-GUj; zI;`S)OQ&JC;))Y(fj6M`4e?Z&m)h2_^(MSLoANh9=}CzTpnfz{xYnnYrKpW_iA_2? zdqs!jR-^b0UKtz$eJwL(YM@85`G#D@PGPbsNmdk33H_Q9sav}>#vE0S-AFyV3cDge+c*=9 zNgh8htjzt7-TLo8xITqWI?`QU?uWXE7xdQKR4T52wwsq0i-HC$)N_NB5J6{|OZkLD zX)NS_m!lvNA0^y)u44}UfsGT_LtMUT0fwIKs^$L@lf*uPi;|srLnS_w-y>zr_Ri#S z?}8_RU`P+f(2KsJ17@f5FlDX3I4Gd%F7D6hdx*f&&Dw1;?L)@A`(&TW5{B<%gAp0A zjF%a@deugOUZ5hD-qR@0OMX7S!Szc+7-3#hLD%9GLAdPv^5I$Ma(<#IcnL(5j)sEG zgvMvWoNGjX9?n(Uh^Qab6Ouek%((602x3l%AHoetO1`VXuVBMRw^ss>Zgt4wcoPzt zb80&`eC>v5vS&|jN~W=SwQ)Vu=i7Mi^xJUI>yqqj=Y~S!3af3Dvngi!ThY+>Q*G&M zR4{g6Ek|8p2+z+nDDP%1(J6``RB`J9-^^p)5L6WU<-;?JZ01N)rRY{v%+}7{Z>R5x z=I2L{VSBsiv}p`=x`bA_ky&hNG)W`TzX=`1>-p;=ycp`r;Umyc`pj4eE*|E83?T;l?Y}OJJzy9x z2(Ca4a3W37%V4adA-k|A!!3B_!+)!lH=pF7_Lr-QhE)+#tKrjIP?2p6i}v+5F6q{? z;-gd)Da?Q(Lz-CK#@F@gdb*pETgMBb5)C5yNnC;;Mz-dLoD}rlzQ#gKD`lxo?)TmE zxG#aYc6_cO9S6fvU}k;(?UC4z<7d;|%u6hs&?6yQWt~c5T^SS&FLfukju>u7)%*S~_$i=fXh1 z57;M_awx3=d0c=qcB>ptxhZ!y&8W+D^3Dmv8E(c)+A&4{vovDzn_XKIx3hL>^8v-X z#d!y}j#uqS+OzOfjrB&4a}XunfptZv*s+fF%g4b@CN!>%)@M++L}WD(cG9@d85JOO zv&NzCfjxohP2$-L&Dv#MNL=W+3*FLT9k{8MZy#!=n#}%GH&g4M*i80;E-MUJ2uDQ` z!B=WbO*;r|#cf%3O@u~+oEWYe4EcNT*cAG+y4`h+3F^C>U z|F?2}1ug`QE5HTkx`Wsq97n(+6_J{npykRDyxdZe{C62_A1Z(eM`K5fEv^Co@eU4! zY%ubi2P`(yn*rs)>1UZxnV*j2 zY{jojyJ}Bz=umNKFJSoMT84}7o@fiIfH2uhW{(t_QEIFYdX?fvelM-Jk+W*VyEASj z7Jhd+HB5Aym(=^5mbb&T+FOY4UWADb`zBAGDB!lawWwL%^UQF+dUmZ#iV54GG1@md zY@au^L=mhj9C9Fj&(C~m_f$}7q#3Cut!j_@6UcpAdJkWIT>FO06d`GPu~^YW(au{c zkYk!Vg@1~W1(z$5CLY`mN@bz?M4>LdI$UnooE;|UqPRcODs>-xQ|FF!$;&^cZprj~GPGD>kEA zPHyCKzP3Z;_9_#Ym_>GGF`0rymsrCAlbB3ym_ecDLHxu`!VGyMJBoYb_o?#VO*y%D z{WiQ69g$hGuoOoL8_0^R*;+UV()p*aHZ%pNf%m@JCMK$43{CDmvvy=)**HI(_@1y59OKV%9#mGHw}d4y>jb-vNejBN_A-R37Px)A(JQcngF=f z7c?6vB4+nLdd~~5|J#S)tM+OMw(xsz>_tC6RVc3(4QFJ!b4Lp4gK;j6Fsq0DE9BXR z5LQU(jKs65vtN?;;c%ogIiCSLF?k#dx&3$AiDtuu!%X`(9%x1_fM;OudNPIvY4V9i z@wp0d5JE8tGtn@zWyx{~oi01|dq*LVrY}|(dc)}7G^unuexg9RR15Fk2hreKF=HqZ z8FEC14B0EDvf&eCo9>sY%gK1MnD+S}_`Q_u^o^%7xmTl+jEWd!Zs(j+-1Imc0K9Pj zRq8>ZRaG#jnRIF!KH{^h>D$F@Dz9kolj7ZlX4R5qt-QZ@^2GH!6-{Q!ER9Ny-frfp zph^YgyYUomt$)lF_6IhcOx?!>_K zg#VQz>4D@ZZamlsOqW^;%TUNY8$S`|QwnxDLcGD~IZOL4N~r*#;}P+KX%FbTar@Ja zsLLaVSv?trBlx0*%Lm9NwVc=WoFg;k+o_6WGqYNcrCp%Bud3jM>rb1pCsw+a>I*cP z`$z4nyfpIEGWtRt_&f)Ap#VtTBDOk@&1gStM;uF5eq&<=x}%VB519tr?At(O&sBo1D(k zaV4aJq^zvUgb^|q9F;(SKh@zdf27v=)v^|87iTec@VvNFMos7C8G+jT@VR#|Cl0PS z_PqD2PDp)tIm*rfEq816tMOS1Sxepf>pQ2y?z@1^124#}NG1)dZ`1(U$+Z`G(D?$% zrMF;}iG7ove_O);<735Pu!}C!-U1!@zgHxZ=RxN_;^O{|Kpddcg(^j%^}{waMMZl# z{JNgRT5uOm|sADMW_g`0U8K6tjB|@Y{fR1I<&)?j0jAc`~U>m;1UR@ zY?q9d9KJd3Mds=(L(XEJWp39VS^T<=HvUjbN0PA-Gex(sq%Ej&4jE33Tb=19WMvsr zEGWv^xl1vXz!wZMO@TXgfEJ-fo|H&4db@*GOmZtNHha4vE<@$QWf!lxLH;VE(z6_# z?(jrQn|na;W-R|=(s_1mcoea=CZz)=LR*GMQrQDW z6iN>5{_ zHcLz?IYSNc%v9;~>EVND94BCe8*w&2;U~Iea8>2JNQ3r4B-VK{`#j7G>*098ZrV45 z<_B;ygO8+ZC_gl5^4U!j05c5oHDt(#dNZx*5wfsMpr4O?5Qx(uBxo>#|KGtd#oE9t z>}IqQp00}G7e30$jcrI8$yJzV6Cf_*{@PtT4VzBOGkA6W3DL8v$G4KRloLmm?i@n4 zGx!PcM4br+<+x}Jt+x0x+mf&&9rtta)mMc7660gSUP7~-nZI|L_*P`2^D)ldLWIe~vs%>^ek8NnxH*GsCGt+nc$g|Y8!~K)J4n(IPHVn>|#xiE?Z!aD6u5{hAeW$f~ z-bedX??VAgf*1Rh#Z*f&tgW#Z9wlk*CmGLqFNi%j(l5c{f%3b!ybCuA6PrIU(!8~M zv>kSax{Q*kt3`YvK#J>7Qnko)9PoVe^3rXWjp9Oon_Z$BwASUJEIz#0{s2ek_ds{sgUUPgHM{F*sWP*l0GvZ> zkCqLjJz5xRb(r0=EU^ET5yvaX&M4-j=>0CQ3{L zt!r&?y5`8`ielE9BlXLF%){NyBb~Xgw@didvYPAO`ktTRQP;=Za(iYuq`nxym5LaU z!_+VsY~n2ARstw$`RA&=eW7F4@ruoE)1N~`$H1m*ZSb|_66;=va<2WFu*xzkhDz9- zt|Oic`ku~Hf4#u7aLc05I{AP9Zmt{#F;sHwKQ2cOlGbdW{|2XjX0*v;JfcjnapvL* zcnKo+?%l(z!1LC1?$f9BShq9f6#DtnHXo!xppFH8_DEXVYn1DE05dUnTP$3MfJ(#E z$`0FCM;0Wp!)1QxWyA!E2DkK#Ex{6BG+7Rnn%N6X&hOB&zx*LKamfSA(~=vlpHTO_ z<6Ygn!y@0T1J^Q?;z^0W{||g~)wT}F(5}RPgm3Zg9BGw)oIYEo(Ri|lP&cwzp53MO zh3<_v6TAN;VVp@7h+#~1dJ<+@;CSw4dh@AtWqq#Ao|7Z7+F`lvmoS9nSQ{Kp`kG`A zaPdk2Fafm!>+#hdrtXhx zFdt1}74r;p@&le|sEnVM6B#QKPoJRO2pCBSYZ^bQs&W=TZsakuYT$_1Uv)S9we6H; zWQO22g#RwsN}(onjphz|A9Z-O2-BL(3tu$Wxg{REa-%NXsmuGTnRi%1kw*Z;o9i zFeuNNP=O~wJS5}XKmOl~XMTY-SEf6!TX>;yiQ1`)Dy;Tky?`W?$3XXX00Y>Qh*iiY zZ>$$Fnz1jiU|yK*+IJSd+y8~30hf2B%>V=5Y3fv9u6TtUj{g&8Nt(c8$aQ4NgYFUR zJu{uQPSNa#ipk0S7_Mka4dYK5@LbEYdetjb<=R&Hh zrczG3<3SNXF6=V5dvU(+<$7E+ETjopK5-SfdYU(-!~1J`6B(RcuwGyTZ-P0aS02l_ z$?^BrOX>|A-Fl7GMJV4GEKV9bf@!dS;)?D`q}_6Vs3p}Ht2HrYntR+29!hB*U&mQF zYbezXG!}EoSgBx(AgW{aa*33wH+$MT8@fnCUit#-IHv-UUht3EkLldqx#LYs125`Q z%p3o*(adw-?FCMk+U`}yuVl@Y-wRQ>FMdzZD8pJ+$GUEv2vbHbd%|*~2Yy>ZWnC9EB#Q<_8kHM^>=u>a=9F4|xOiuOe)5&t@nGy`4(J)`ggn!D~6cgvt zm7^hT$`3oVlP*b(Q#1$Nh5B6PcQ4}qvAD#Oi@z;EC;#h}n=wQ~!d`-F%ks{EGHI;M zer+%^G{=x?c8L|daixV5uaGKY<@jjvp|bn@R$P|h@{uVR+Y%2hEFIX7mB)9ETW2r9 zV0IISka%$vu zlBLHkEQ#M`Co&!GlI~pvUdCK&7{JcAlh1E$d_tc(UACwqh?D(;)hjeE3++6tbi~O; zXXGz4HJ1;Mj$!pL>^xqFT=Uk~OjobMS#S6?9tPfXp6ibf>PTjE)eLAW#3%sIM0{2~ zY{acclIpn(%S_W3(1BM*qL^yL*4}7uH`MK-oJ*+9HB%G3G02WlR-Q4Wg25B0HxfjZ1F@9ny4-bW_SKT7&}{21=?C7$BH?r!rE3D%F+eAl@ITtRi< z=QCP4I|oJU3*irQukUdd&Ko!eRnFhO^>(^Lf;@c;gEVue)riWuQj!6Yxvh~wXRePT zFa4Rxx*eX*TpRe+dPwsj4-C{BJ&#QfxD`)utCh4&rJ1*Q1LGC8&xcOO$ZCGgu)Xqs zIQ`}qqx}US2>ivtXe)D7`QbsEy$G1DVd$~%O_t$p$B>0_!%=vXLudD9 zQI?HNp*8eAOLn8k_sj$a57dFp%XEjDSJZZI9)8h&Yi+~qRm$!)Ni z<>lk9znxX@*xWz<`u%!IQL7mXACWOQz(s?@&8Ks*uBTt`cFk=suv@3tvmzL%Kb@-* zS`%xk*(b-D5F22nGY#pB#^&fanP7j-e3Re1p55;}=0Qt`tL#dm-8ASpQ zYO6eE$+>QzPaZC_OC`4Ly&c^uZtq3ZI2$B%a)0*h2|f_KMH!FgRxKPgG8E1i`(5$cVf-1pr*ZN9#m zRIS)_HEd)vzmbrrEX5M3`}}4~rMv~tT4UYwd;R&p=D%9H?)R>*Idc|QkXRerX;Hk! zRGqkv`T5ODvzegy58N`<()BL4IHYiZ5`#_iO#ka~B}<9whQ>7JwzmO?lyptBax}Eo zbL*J9VmKi)Rj|>_T1?9<*8ab3XP2*n>&=QKqJU}`@-Bf-Kr0b?{Hk&0v3xTNIxZk8 zg=)t?ab^C8HE;V;(CIAtz&KOn{fM{7V?%iiTJx`G(TNI0hv-;2Saq7U!@REDa``QB z4TJI3E3H+(h`3IFR$S7n+T?c-OPYQohfO1Q(lCbjvg+{sjUF8FVQXy8cW5OxH#WNu zn=bKyp!g|WYiY1`IA4JI&RGVCy#JEw8rSbS%Uk1|9W`d3NByA4K9a#UyP(-@)bQEZ z;A!$mo-D`VvDk86ylL1rzkz|L$~)hz=5778v^41F&wG7J%g#;?y_|h-XBu(vc(l3r zq(|BM<80Bl6GTHnRDn1>Q~5Z%{=pf&Db_PfNyS!NvTMd!>WXD-<5r(L<+J~2b194G zf3n=sb}QMlpH0jmEAU>F6Pf|WyO7lLN z-Xa1|=(;|a!pTO>2qCHAVA{BrRY^?5Zu@i5wH)9S=8WhV6 za93Bqotg-We)0LJs(voz)`0nwz)fCq?g7LRw>IuRq! ztH!qKDvWc2Y0&?xeR=R(X}3OZPn9)o_L1OcHuT2- zUW^@MgK>qM`j~SAew70wO1KA>_c=dz(gSjJ=D);$e?13op(07yE{JB!3R1i<*_|W> zGd@Y;6(P8}3sxh(=Tv)sra27IYKJ}n5kN!*>fRzSBb7Mz*Tm*>y)y;n98b!O)(8Ja z>KW?hdmMFqTm>r#ATD{HL;3*f^C$9INgmLU&9iTlhh=}F$+*nY4$kNcWZslDe4P!9 zYTH8Eb`b!A91ABrnKBo?2ol!s-o^j=`Q&5pOGmRDXv_r_BN3SfXsy|tQfR-d39hNZ z!xN;A2;*TI6DOx&)_&zwl7@Ek)$zZRqJxJ~{GgjIdMLAP(Sy#BMeiAG>^Q0GF{R~9 z-;o*X8}LF>sEWXq#ZZ-1v=`F7MZGYkvjlaJf5Ctpf$fJ88>lJ`D^IFjBOdi|ToD{# zlFol6;8IAUz0}>sKL#g2C=yK9q@TeOXK-RmdV@v9um{Z}={Bhmvyy)2@E<-rHe})M zUv6if(A&0aNoC(+<5;R8;iJQ~*=Oo4jchYbrY<+Yk5qP_QEA?k9UHr&iqOQ3dtxz`uc0(6XTp=i zVgebG*xYd9z>b8Xo=j!4J4!_#bJk64?U3Nbhk2`~dI+Ny;MHUCFz9=2s?OBer@r5_ zO2y_+w*3tMWXHv0e#X0fw7{Nhkt=wah`Kf$pscAI>B9X>H!Q`h`eI_PJ` zPXtM@9bVArak=j=rMbPDa?Bt|s+P9vX6P}`#m;WY4XE6KnUk7wmkr%>DL>+f0{{;I zME=~eZ0P&!STgsXxM||jpIPgFmp)uC2&2_?*0Cb2%U=buqU$~{tjh2|E)4NnCVY+j zN;69}-}su8x)}7x&>YCKM*ajk*bIweNT;`XQ!;bOfNNjrgwL(tH&EA_7P)W5kd-H< z0x*K*pQBqirX?h|gn_vmBdlFAb$1mBjUYo+0ssmB#z0++K=%TyWu*fL-Xh=u^oXQD zPH@x`rqZ5Qim99Wo}4Y~qrj}||N3}*|4);D7S9f6qGCO=CLUK8$X&6ii&3Nl3!xt% z-byBxeyOoLhdA2ZI~R`1gPOM_6qh_eHKl#~L=S-s)-hLj1RcS=KV4Y54;*&z9q*2o zlw0Z<`~Xx7IShdTMS>AP42y>%IfkA}3@n<`iLd|p6Xp-0dgjNh9g{Lu!%qb7N7*7T zhjL1==>a>fl@&{ITs*dJ*D2m#5?Y)+nB`df5lp4`hTbBp3;R-Y51bs5o-zo`x9s_= zMbE~xwBmSA4>RqUWmry$>YYjkopF~dXu+)h4`44cJjoiveCrM z)x*oe%^E_^L>wIS-yg%d@E7BjyQaF?AdTB?nIo;wd+J@9eK#}T?1JD@EThcQ=1kV; zahj*9=G8vFb`E9Z@^u5eBLA;13<9&q4GBK#mpOahQlF&E&T*+cEll7xq>N2}TpJ_N zAzG$w=rC03oJD8ECMLkKq@BlG#b!oqg=7ePQ^YDjK`)xM$>(;UW4;-dRWlN1`~2LW z-HhC!+G@!PRBFn_mjQFHMA+q(=^lzbU0X%V(}moe3E5 zyRJmCn$$YGt~#dBhZT6^UhYCUK{KM3W?(a(=s3fDn0I?TZ++8Ej(s05aawvPU5mW~ zLvX7#uW#AR1o3|?j4wYt6!0T*0>hC1jL0Xzr7q{zQE@?8RPu!X4Bu&4xq!!HJ&P&( zdVwpxs1ERmW~dT4fe6U(z|{YCw3)e>YP*oke*bX%r_Yzg+ZR)BE)%y zriP-JYX8rGOp`bWdK91vg1gtGzb2GaDsQe;csZ072LgF=4G_}bwYIjZecnkZj|W%5 z>i}TAhsaE}r70^L1>4WeKe8P^z(9yI2IxF#Z0Grc6)4F#M5Zpa+OvL5W5BEBmR5u4 z;g*bbE?f?ET!k#j6_sz9? zr~gbO-%s1tRbqbtBb1IkTEjPiOy*kXI!upLdjKD(~pdFHO48LESXDd}cjGl9yr{ zPBhHF-^Aw0m$`J%4Vx{9#0*QD$&< z%kUlT{MRlfm(Eh#Bvs?ho8H&4KRlMs7??~QE7|w_GN~+@QRmsE$7h9L<};NtB4f#d zxt}l!uWt#aR%H40PT!M!{nE1d^0Vr{rf1$~a4Z{R#y^@~&co}_o#t%{wIN^9-}pzhuY(e75(a*5qPSRG71^4$jSRNAMRc8~lsI-N}JxoG*W zzuj|Qn-03RkeSAq`E@LfAK>`Z(k^VSD*D}Ap{nS&%;@qHV^9iOW6JG*2vtQ%Q*OCn zdfIn$xqGQn*gEFQw;b&E($OdbU6Ed(Sq7b?NPMmicpc7WE{$S&1#!4}R&5zZ=vYTk zD8|NyD~p$IO3v_P%?=_~yzH=!`D|Cc@pv_m0=^jMuS%(GX@^ijrB1vUbQtWnm;iww-{s_ zi}-@-dRYayn_?BssX8J+wqJ^Uvd_%MR3_eS5mEX2w6b0UzW(wSkYQpF6Rx=e@WxNl zb)TeHC%`D1FNQ=SqGJs%#MQyV;u0gLXkD<`R<`1G-Wuxk^dfAQ_u`?_snt2N(f*_1 zt=N}-U%Tr^uyX%)8!V7ks~|EBP`RbKC5Tw4SV)3Lm~=j+^Xkxkht18-A3xXZ;{;X_4i_pY_Um>N{Kf zT?&>Tig{tVNsm1>d8&iol4K!T5uQ^nzl#)gmtx^#k>O&ezGKdI%Za4dPg?cM4(@v_ z(cGDUuMQ7Qoz3qY8JyiW!eqDbjklyer{KBbuvJ2J_NBFm@32AAvI;l{_5IV+m(Q%J zdq@5gd^py81f|zz(xdKQhHkq4PRi2h_@QpZJP~T^zd_ktJIW!C2l8AUURZYFl3rv% z?#!W26x@BiVUT!eUyKeXt5`gEwIxRalNx_5Au<*gsdt(ec`fo9vGUWKqj`MO=i5>h z()Z2U+r??iXMp#6f)G6ajJ!18;AQmk0(#7FZhe)6O|*#t-BI31@Rd-waC zLe+C&tq56vzn+?wK2+L-nRI+?LQ8zGsdn46{I=SfqD2tW{Z}R^1(3O~?&?r7G zt)IfmQ|mG2?>;u8)!>sPqhOq4UV~62uL~gi0k^OwCem7%4BG|Dh1&B!9plx(oaj6E-GY;1>7Z%*4{nf1^BgUqTVfjY%S}HB^E4}R(A3EdO7$BexCh+I*#7s`&N}q(SW|gHNi?7` zUi@v~@;jt~8ruOFu9HmP-UMEjIk+W$QoCBuXTRo~JvV;rbDaH~rLp~v(hI37qwDIn zUFHS>JjAu7p-^P3Ovd7;iozVoZ_VJ44hRCG03I6mnIen7|QzOFSX26iK9Ji3Ug8Kd%F~V%_()Zhy2wG56id5xE!fd zfrUcF9sz%Picf3Eo_+-_vX&IKoP4QsejRj6A5Z>{&F5}$-8=+6-|KLD3 zvo-h6@d^YA7nY4NNOsoFo=~#fI3ijNK<>XEtB&cnfx*x2#h51&*0b(e2V{lZV@k1P zYMCn=l&K5N41d|s9?A`l??+ZrCirlQK85N7hA^a}W6=BBXv(!b^T%o?^{B>p!lU&h z1u~WDoX+`Y&vcdYBG-DQ>NR~yE_1v35|}T2sd9Itn*)j@8A0c%JpU{*9cYdcYd?x> zO43YOt&T^!xNF`h7jg`r50n6c8#Hq67##LJWa~ z5R#DOcb|Z*(!TxszW;pu00Hw|=bU?7_kFGlwv}X4F~fpX*+-Eu!Biq~huj|ctYTcQ zvbQe`t5PSLGHl&m)z{IRGtAU9HC2AI$>$VsoiFu=4Q-3fw^crG=A8OHC(s$`oWME zi@qc23AK5Fy1O%lhviAb657X|)q0`xF$8h+<%6{2qV_`@i}72Yu;V#u=5gD;E>4LV zlRWydEvHeIp0PEfBDu}Od{96CP^l~oD_)5q`L(cSc)JcMUL)y^(d1p$x^(+$HI!^SdSN&OB4YV-J zFhGQ0i~)#l83u8VV?d%~MQ&h6U_JKU)ehC=S35Mx68T#^{gc}~jVm3|=XsyTi%{fU z*~*()Jr-<@eI%XObkRm`_TLMo)zY0Rn&BHU$W25R@CL5Nv!t=!wY=u5l=O)eFJjsy z6+K20tLG+SzFn{)Sj_L+yKfO|PFNH0WBvR%q;iJJ!j|;AckL>|#jj1bLZMj9c*i`v zb1Q$@jJLlu^+4Rg0fnBW+``yXNq%Rswnkb5&kw%iit=84{2PlmGxy}P>gvoa%{=i63j$zNm>T!khLt?VrU8O+(2bA6+#jX8Ux&t3pKw262BR_ ztRB&)N1-{>dLV{)@Tbdy(i_r4mty1Y#O&3j)%69hW?Vjik4s24sThOu`|QZn_N1%S ziNS4{oN~Ds5 z`|O~yt(OW#&10{uO84`4<;)xm;>Y-;bDfI%tjf!&BQGBMv?n$6+qv%^^)0t0Y@5xY zm!&!fHF$VZnP)i{MjrFW?$sO*|E6g^^Y)7W^d$PoU%s@v8L`T(c{A|8uhrin=4l?V z+Pin}!rIG3tS1Xa-$mo6@hY7ZmOK!H#13ZUuBq9nt8zN~a7uPV-~_+!@DEh?6XQ7 z!!cQMXhAV;^BcB&?+~FckTj}hlWiYA*yvek)fGNog_-EOJnxC@s@F~Ye|j!ms`xH? z@uG{-a+^_T2YfrVpch4_f5sK|9IJi%W()%h*WyUXeP*0yFM1SSb#3=}5J?n-LE2~^ z|K?0-T-JChC8%EKZHl&i6A`DOm_0(=D>l78ZvFgNScs!e2Q|j~xXJp8Lx<(_o&)!= zYURo(^^Yon-~KqnoQ`GeFal=X zWr~5jTAs54^8P#dsn$`bUSYnDpN$bp+IF73PYX|hM{_5J0-}jq`aKxCNkQrV>gS+Wo{mUYZBS@TK5S9N~ zA<($aQKIr7-O0GD{X>jIUu+Zxjc(aCb0Ud)lI;Vb=M2TN@D)`qt@653sJ;k;+U8Z# zi28ljw{7$uvE1*B2QdeHFmEb22x5^Ag-9DPZwee*X|k&hGjmTpzqOF}Wg3Msuz1M2 z{G0lLeGFXYw*EIYxMQ~*+Ni_~+EtM#s=cXyFiBQ^?a^XpPf^!z!H#0XWog=aVQDqD zuzLr{2d@jpeT(`|cy0{;L(tEy>F@i^--#0)lsv4XnyqPqFKJ=KI7wu-;(v4jZ7~ssJu~N!q}Qs?^m8S%8258mOdfP#%XME;yki7 z)BXDDfe`(*LzJzvX@ck9x9!rU3dB_t#_GL7_cZ69X_hA%e0}L2U-MuyFntcKY-A1Q zYCBN}S~J4uU@S=?2#hW_Im_V55^A5_v(`zI6@;4}u-@fkLRKYLv2=%)9weGI`2f}4 zf=PRiSpH`O11arCl{OBZ@wGfh-l*lsF8=NEjZe5g>>*?y+^_{hF7Lkcx}a=n@R{ zDHJcgAXyjhSKihUD`52ca(nR7FEFXJUKS>}OMu5zX3F0LW5!s42|GfT^%0F3de3Qq zlhdX5X^!)1#>CD^8KWwy_IL^<`BmolT#F{&u@^&n@jQW0yJZ>aUVXb&L;E?QC^oa& z7u#^#vGLn5te$(Zv%4r=%v~YdWhB2vaQdiHA)ELF<0y+xO&cj#N=Z7^KXU(tVmTou z4VEWEjqVzvKKz|t@Jftp%Jsp%XMkP|jbyaxt!+H>v$y7e&$4&c(&+0%f`+nah9Qnb z5f@yai8S_wm}Hvd;N%T^*OCaoz4>vD z?%MUjKh#4{NgA5+0Aw{3C9o6`_gKqbU3+B5A%LuapBJeJVvvBsn3x#xe=S;%et3I> zhHmuY4fd`~nTVwT#|kJ@PM#;!Z93+ftL^<0_*a^^*ej?c2EOsA2b8mjG*#67fO>0Y zOcua^>Id`N!3>Nt&<3IKuQ54gd4$b`-_zni+|JODL-$K(lF47sI^( z0dHQ^8TVl2lz3Ihwqani&Zm_OU3n<@(0z=31;3_vQd!F*fhKY$RQ+M!S7C>dCo3d9 z8KW#ez7(n!TF{kAvu z9LNtBEbRD_ef$ljZ~Q9kLCFIx){`aHd{U_R} z{J1g?QJA2OW#MATlBMdA>RJr3sf!on)i%0_jcOU2XHyw^cLO6DJ=di?=bcO*jqlc+ zd!4ggBdzpcZDBx4{`jk_^VZ8cb`7kkPOLi{l7o%I)%D>Cc2E04u?ckp6QQfG*sjRi<)+5h zEY`iouUNNIhWbKx#%uW2 znw+;}ruRs-TS%uf6fe>c2G3*3T!O(%5-1q&r~0%RmB~6o>qaks3SG-j0p+nG zr>}YZc+SYV8q*;kOcGHtaF%HG0@bqq{C5Jv5)8B#iDxk!{XWkf3fp<#?1uIMoeC<_ z!}!SikL@bp<#n#GRNn@{`}%yy;~n+k(jKW{Yi>933Qcc*Pu%^veLnRyKW->|c`Vrz zDBab{caH5T0v=MvtHrFMxI3R8)V%h|pJ)B|A3}>f3g|W_P!L(~Q3^z#n)k<{6AtoD z7Z>L5Dw_3>`wzUACMs`dbPwp`&V<#6fd-Nvwp<^i3Z)Sk)X$5$!|k&_t?bcMfRXJ8 zBcMFR00TN(m?D5NXN<#)m|Y3gbd^dF2AHG7M(xT&w_@fjUcdHECbV34XEHcvPu$UN z+0AG+AGjOWayt7Yl+^B?Y!W{HIA&{*?9)hmhmY+_%^Tv9s-u5ccJ)4U@GMvafKUEr zOl(+N59Xy;DHrq7C6POudHi7OHp?~9jOJ5|ySZ%6m#mBK1vXEjhCFI1yKc`{xl>hI zzQdrPjArxnvj(OLxEV#a+{Q2T39~0ya^|7$##X0r=i09C44^))j(xJsz+UrPgQ^_{ z_viYApIM&vvn5AWKB3wZ+HczR`^?whhyr3;GUHX+u_ZLM4DnFo>!5E+C<8`OXNnYV z<+zBS58XcJhjkqZTZ>)7$x>y1Ms3wI(@$gjUl}@pspe!KOqPnH1%AcL2G+7|Pdqu3 zf?sEP?soV^7LKOR?y+x(`aOKv#w@efyaIf7{q+ii3J3GP+L^^kG@e{KqX@t^%(#$p z$S_n&;bA&~=X%2nm7ttGecal3okJleTjRSk(tUGZt&fkYI%YN+TJ5Ww617XGwkqFj zv$2Cgj@h;YPiW60oY`XX;##J!*>}Zi!j2zQG-g%Gr99P2GwBu7z)_$tTI zb)fhNo9NZscTP(dlzk-c)5!DZ&p!;cd+_~d$O&VRraETgxuy(KP!dyNC2ZnS7d6BZ z4_uKx_kxNejo~%ZTO5P6tP`c3;A(cvtaND%-GfMOBXfiQ;nAbadefN=TQp3%8 z8FKYT&hv_)-q5WQmCPO2;;Z#}NO#MPwi?q)%R;I!iI}~dlFFI~Is4d;}&ca&PZyg&O zyQGjRm?-0yQTbutZqmC6?H?`du+){`2Gg6uKc6%M1DE#8ttyRJE7z417Vx;Is(u-% zg61&6&(eYh(ax65R>hluaR_nPLl+%E!_!#Q=7$)dcd;6S4V)r`bkvM;)>9jlSWfMQ z9;iJLM?V}D@_({2N}2EmLu*|%%nHqxV33m0g)R4TC1#6|UBsdlvOvpo|DO@L@;*%u zN}whUJiEE&P?lf(b-aa(!rNfI2QRP`17af&VP~7FvXNgS?9B{lYURzV1g$^Z&#s}N z%Ta#%NhD}4Esg|Xr+JaE8Rk=z(k!}2vB^9;e$2Fqp>~&(`D@G5gjdPjJL77{&k9Pb ztX3u!<#<)({Ql(%{raxY{p9WUyY(?`1^kYD89B4Ro-vDZ#JtN^XB`%8;l~udI;n;! zWVv(lE2?eIa_>7n@Scw`9(oXAkALuSWOdJ+gA?>!zhKiTMtcT4FXXJMw!D44MGz4q zHGZ>%7i6lbA-2!%W3S@t6JmRoRUayo^X2YtVFjb%HWqb!Q0jJQv(GSjEpWa_v!OHw zb)KQHgO_s*byG@x%lbGhJ7G!r4&q^pI(O%mo{9@s>BorAE@Q6@xr)c;2?VwctOFDD z+BWTrwY1-p2AuVi2HOS(1-}jCT&=D4H9A**EX1f4zd=`Lq_m!vzMZgtkF6s~@78(W zT$`^|b*fq02}Wn3A|<-R7d7B#jVewb{HRHP+%Vx6f0#Uak%N z-wv9KH$E9yPB)%A2b#YbOJYJf)acWpGUxk_PsIl_^=J3F|I|Dn6pZ0=5OHX zEm?ml6>3@;;q5-VAvh+HF!Q!0U0D3K7r>wsgMqrZJ_btFlpUtFIkt6;xEGr*HTaes zUp8OFe}yhE>d$Vv@#!02S`k)|^$@oWehh zNqIoBsDXgwdQ)=G`B3tvmap?0InR<(0*&d$^!vx8>!@?ioslPTU)yKM=#ULG}F9iCN%Th?Zu0 zv95joyA?Wv1%n%7!?rdR?a_&ieIzI51l`gw?S7K*kvGWnEs|J5T@aInEZ>Nu-Yx^~ zL4$WJf?G2#B|%d#QOGhYU*gA34*DoJROqgZ*2CXii_I0b43sBcOaG5R(Opy^*yer` zbg0nBHMaKgI_gEH7lN&~J${<+QY}5u$NM^)C%;&cFtSuEuL5J-Sm!+UO$C%fVhban zm*XzoeFV_@C{SGMqb-tq8}v#z6!r!}$Q)&DlLfln29k?WTAsO~Z6977&2Xj&u2Rd+ zf~eZVu{pX$k_wMd8F!DZ4PhN4yk*bY;epnbSnZuj`x9J^R=;81oU8K$5eJZhX`UDxhoE6Dq_fF;{h+W>*5C!EDr;aL}Y`A)&y=a^Q8^Xh2Cb z?EA=0>NwfKY8>D!$}!Gy^>2@N-_%xCp3r#4-x-DV{sLi+*Y2_yf2X>HncmEH(ZYw6 zD?`?sYnBY0508<}(O!(ng=lJKZguZ5=pBMRAuSG`I6JHL%V#d#_XiWO+!-(3^YR@Z zd~e=0QKWhapYJ+*WOqqz<4&=7*#;HJ80ZFHYOMFg_?_EK2=4og)F*5bn7j$nY!Z}P zl^yP%OGqA%S~%1b-sPFKyeIo%#4ml``Ra>@cAhxg@U8rFjV-j@ViW6-ooQpg4lU^u zJh0C`@)FAul#KaY;BV05#2hQ>4oP7bJEkXj#5@ae8YxY2POd#K9inMQ#_QNFuKk@g zu_l{xBySBDAAs+Wt6_$UgHE`}I;2B7>e%vLX}THYSQ6~Fn; z)I=iw3~_e4MxX5l&D}Li>ooY-SqexrwXXZ&>h!Xc!F zvT!h=UDWn@Buu`v{fbADA3vGXpT(#ZZg+I;-EF9OENuPpu;PxG;hrLphKxFlw`{t5 zx7l1+`O=QO+x6th+1wXvvA0Q^@Wy=QzMaYT3RTkhPK;UbP!y&Jddue2?3OXrExD2tz(yaT4^ok^(6-k(Q?TcAVUA(YX8Zq`tB{D+PHYd`P+E%ikzy&6(40&q^ zo@;SH3(F z^)f`N5u84)+^66EaV6ix*K5j@MyS0+*YTP((=7~|*hY1X z^0+2tmk0BbH4+AsZ^xP~kHkvzu$Dc09lSQNgJ0Yu)(elmsJmZbMY4LwQg}W%R)&z?_T;Wvrh1EXT9Eqo*f|?po4`z3{~FPozHZLnYUVQ^pZXXANprrHIgM<;# zD@aC6xsXEqc>ern(0CtjarXD4Q1d}DdRcb&e*_|$ON0oH16tLnIm7QUQh|H`+*aZ* zEm}Lnb%0TKGNk$c_^Mh83W}gSZUp!~L*WHwy&1xd#d^;5x4w(M$6&}s z_Mo{7U44yG&6)B-(3CiX1uDL6J*#ET!+&3N;?V3RLOY-Q^#~2c9-EMcijl{$($IUY<$z!GiC8P-JF5-8RZnMt^8bTDS zD(nYBSCQ0W{39GTIDcFTcy$*sJ|TK>S`PrKvnt1fEkuQv6Bjk$bB}I!SymIDaOq-o zw6-sC|C5BGi*~8j1@<}`_-~Tf6XJ`hjcb!;`*i^9Nol{H*LLs4pr5ni)UrlfVot4i zGu|wrDx+n&YG`VIsp?3$$k?qM%1gDWcBo0o7aEqlCn&b<4hBRRZ;Z(>G=)Hw^jYzR z9@`uX4#e*RLB_)Wu1pzkZ%9-ep2Ijq%pnWA!F+`zk!s4HFyy}t`;iaxWm$Ix#vzSg zCb%9C3A>_llf`2T4o$?I%)T}h?o?p$3t<~$X0QLQkpFx|s|wbU$%9^Y0dNZRIsll~ zq?{F+iuQ5RwMUn{Ars-txuu#e>sL2DhoIpx@Pc_ zja5)FZ4qMym*Z<}oI`D!x0HhmM*+otUF)d6k*I^%uO64)!U_@xRQliG3rj&gBy+o4 z+0m1bnCK+ix$b{$z1#E)S?cTb$03`Di-b>pur|X)+Lm=M!l)C+yD;mIlo<#K(6ss@ zp>G7rVj+Gxatn3aA>a=Fb6MTlXIpF=o9dTT+aWGbT&m4PjrHT*OK7Loc>Wg}83MX0>P1f1mQvS~ zmUPo3D{ioYpcSka!A9t^w|6PXL{c5X>yN;aGi)QWF@BBQ~(c`mOU9KNzef!mcTUx(t!hLjp z({jas6n}hsJ95*{zhW#t`gqoc?cXh!m9W2dwP~8ad-`U-e&2>y32a$UQouR(K!@Mm zs&$*21!Aqs@vf$)vt3O&#aZPoukyCJTBiLA|9Ws4GE_>5b()+*>zG(s%~fiGp{_0g zv`!j=ZCS2n;5z!Jw##s^QpcD~Ud~i9h!hrH)ym+MT*^Tz5kP_$AhYZhFh!5Cg$;iSXKizJL$Q1wvW6|ehv8CmtNIV8QYdq zcXm!RP?`Y0Dpp@Vs5ZBkcGRolz7yZt*ee5{K?w0`5Q%n6LkKi1@QW^YDOOyyRwsB+ za&oNp;#2A#(C%QvC+vD;n>OG-W7e>WWQApTCc8kKG9X&>n94lS8qv;swyiL#=lQz2 z5PvDbL|=_}DURF`?&*JHzS_FM!)nr~XY7#!LUqrdILz|69O@;KfqO3jOG)I(O7aDd zrKeXBML9w?4?6FFoaagG#Gbfb(DAj3a}1#4%;O-f+-B|zgh|@wP!;_pMs%3aJz^=V z^DBHsi5HMtsIL00kpFA5qH#UNy1XcU+11=f3E|uxuZjfyZp(^;se#2@DWTP`MOWo$ z`Z`%iNNgcirBHg4pTe4}FQM|qS&1%+ z^N7PQ`JpuL&QZzP>-WQEb?(h)C$O#$8T&1@cXH)7>ucYEPGF#M<}N2<7f& zn;KG>ueG<;sGMjk*n}|-9X)lMrlXiIm?Z9nMl4|I03o985DkYTymWGm55aX+#TDCr zPHoxlWt0SY(P-)F3R69x>Quhom~eU9sWj_h=-T?)oDa1|c$R?PuQ4A(+(oa-M`WeX zcq~FV_GUOMC!A6mPP-|CCRTVEB(`#_7S2&qC`Q4aH}7Hp>c-J7VB8PBMUm#_=KhMP zp+koJQWKGmcO8-p2Kj(wpAaNMie$zSp>TDfV*Ja>z<-tKGZ_NycdwFxyGrTd6+vy6 z5Hu_Xd65)E#~K2Cl@H3wOtyj;L6{$C{;}D_AvXWVC1qctKZB%>M$GJ_yuTE5hKmM7 zQuDZif&#=vYZd2T+WuI6>y4fZD0}|28t=r2$t9C_8J_w5=rQr?iaV*$%@jOwaOl() zqdsxKAo&380_XAAJ?BQOJ=UY7T-f8}#|6fjoCtnEjO0qs)zHyxzibc&Lu=gIl@Fxf z__*FM6%7!ACjpx;lO+YZ(o#zfQ$-bir*7nb?|^f`^Auc*>S4kQQrQ8Xm1&0oMfe-_ zK%+DL^b0>&=>|M9)yJ$y=xxw?D&a-Um=3$Wf|$tN-x_qAs1_xynDd8cKeOzj8l$ug zt0uPAAg&_zOf-8gD!oKuBsx(PyCObSWBd4r$inj+8>IUMW^0AZRJA*5xKe6_CP9Pb zXq{#2P?{7dO`){%erPTTfk9o6${7=0Yq^*lBb_)f;?S!n!qeN!ywyZUg7{(I*UCfN z+&b9V%pxz=_=&tjLku5|>{Tj1X>z4aY8bXr<|7^j~Re`Qc2+QL| zSYwLiWvv89U-KzVb6CZovJh|UeEoY_bg8*7Y~IPD*(w+HKw4OH@DBwIVXb}zA!hC5 z0D0nPeA(xlI_?D}_xm|Y5^sT64UDk)YtQpeliQ65~LAH{Kj|s~0kJZ#Bh=v>O-GcrXJ_iO^U(04!UKe67vrEq< z3ckq}1qL0TpmRMhD9&Xs@e|0(4E35q34{i$M}5!*BaW;#*t6()cEEChYl4$cNL=qH zdcoV{dV$@`C9W5CS2jE%x*BNgO9H;okte$fLe<#!OG<8N9kH`Jb;XunP_sMD+2I)r zF?yqyb$I(a76X`eilh1Ef-iu?cg5=UEXz|0#ho$H!$y&UrsV!E&Mue#H>~`JF-2Ix z#meFk+1+RQg^c9jEKZim{KL3-0OoX2C$1Olo>MpU)lxMb-9JymyOD!lJ~Md7$fO`= z?o1pm9+Az)aBt;s&|36*19Vz3GB|((FoDPTd>1xx;PDF26BEz3ZHO?g0Jq$28N&5V zk0IP$p?LloL%_K1Xccd@s{OWSFM&>LePVOWxLp`n>d#Br6u+`B8-2ZTvB_nXyd z)?xE%1Kg;be1Snbo9&-UDH%+96`fv>_oxp`8W3PLK3~qHpELyF(g4yGX~m(`ld|*| zw;cl3G&Fm(rwH@adav{)hg%bp6h&f3MvpN3wvQkrDLHvx+#4kA1LAJiv=Dt-+|(ih zZZy!Gg`Fejai(D{tmD9q@}1H_I+JKlp>93p0sW6%{S}@=n))oBw!?jKiVicm#f2GV z*v@us4deT>w7En=>qvS*m79o!vG(qZFv`Z>XwhrDLiBslhBs@BwZyhReZ;pchVqs9 z1F@!^vnW#me(<8c3`7x-1R5U{M$F9lK<2u}+!toiYV!&i7dW}+tuT7=n!>DE$CEh| zWIBC)yj`(4so+1D*b&p#A^p~#*bV8Q7%iw7>5IUo#N97b*zP{f=_%4-)c7<8Ki&*H zS&$KfehzGriTj>a5fLLl?&-%9g&zw`Iw;AA8`>+~ zFf{`s`Q_)2wTkXhCC<{@3hZ3;YY*Dk1l>k(u;p_DeF`g|b24TAQP<(ZcXdgoKK=Gv zr00v~Nf}N7(jIvF{Pyn4T1ccAEQO4v0A@?2sYgpEu9rdNGT@@2%>g(RQ!>5-o+-ia zNG|C`K(ZD%5_4tDqI57Nfj|b$e1dPHWM{87*9hsor1kWSs@XLyL;u(N!Cfew#P~>0 z%s4uY=TG1(Jlk*`I#v}iSnX;YJ2 z*seiK^{N;PEUjq1c&OoacD~0wTHSCCpE zhKsnazfN>oYW9j;ZXxlNF+~IUlETKT?{*Y%z%7>0Th-AS|QN&-#W{30(PE^ zYCVN57Qp2W#&LA#iSSn%?0BD}*LT0AqMI5dY2R}rLSfe%yaIs~`LdR{ z3OfNz^-9S}a|j6hSw{U zuh&zJ?p}@r(T+GcB-%BA^xcbDb#mW{2Tp^}HBztU-Z{EnW} z1tCjyL!cgpfsX6_?ldGF1J9J?TV5Lx22ql>d4E4} zHpAGt{{$>}DNFH93G9qyT5sawhQ>zyhhqU=QLdp8t$kO%`TH5*rca$S*CW&7 zUue>D!-)!cV{xJxK@4Gq*Ozbh-2ef30lb1PuEJ1K1a3Y&;ok>_n&G(}+Jv~HH+2cUB#xJhQ zR(rti{p4gjqA|X-ZEU1rbbNKii`#8M;GScKq0#5m8$H2#Zb1GtRR^uivDs~brmc-5`5fmf(z`3LT+`=7`HZ95U#BW8NH68;f5=Umhpg+^3H;*w z?V1VU5CrR_2DPRtW*A_vh$LH@`~6ybYQ};+?G3r&g|hc-Cp_8C${)(QJ|{fc4N^@L z9bouVU_&P%XQ!_j*tp$%xrl~EdtI(xy{Z)b($&r7e!cJAN$~#9l>-tKgq6MiPOla+ z<}suklEIOb-4nsO5fBt(gb*l!q@ZjA6!LsG{ZO;-?H~(=t6)8m$rf?DW%EyXdTucP z#@IM<(hj4;ECClI6tc?lFk_9fsu_F%J9(o3BhMECna%(2U@ zFL5!JTVf^s=45G-8>4G)U09ZwDPmo3Ii5M@j#bsRO4<`QDwUn@v(ia->d5ceTdK=t z--*CdQ`v##%z-*tGcl)C_;t+&fq7aPf5NIX#lcA==(D6@a+6zAPg{a4df$Wgp#-KZ zv{XSzV9AqS$lGRBiVYQ;Jx#El@{_G)`%9C?f(8EUBkw)jm&>juBnTRUDmE*IRTaM) z4;8IKY-B0?E9o9`1=SBZJ3mNY*c7M`^PgH1zX3L8-7klWGXDfo-aFpp8wa!x{1Py1 zZtX^ctU?-AQRacv3SqXmjct|yt2$X&0-~TO9SQ*ox&%C^DAemI_CrG74?kI#d=_Zswd7z;K zJ#PiR_B1{ryTQ*sf#rAUuwYa8Mjip!cqwHCe&|g05DhZPW+P&0 zTXsWcL^4z4DxF~V?jUe%rVc%tlkp z8?7{wp6ry=&*>Fwh|QF_N(DRC5jHMOdVuvJ4PiPba_JKmrKGh3%$}ESGK=e)>nunI zB?)eV_q~~kk8Fr1_flrb&^FM})O7GT3!+>;l1CCJ(!_}-oFk2>iD9rUl#^X7mvA=g z>x*_CvHjjegWVOS*^mIu(7<`5SqFhf$8NenT8@Z+H5DV8(wqP7^y&Jy2>&Heo75p0 z5))yD^)uK~-Vok7Q}YR*@4M<5d@+OH_JLH{;XrO{JW~JFix{vJWVxE^yj5bZudny0 zQ&oW(p>7(w1$eVa6%sn7AKbgY9D*NHq?c=bod6krF|-f?&t};gFm<}?hsVn{Pb$s7 zF_DUR)-Vr#0&Xm=r#E*-#Ub4h^SF|nDI!*=XBy{GLT3{16S!%m5UJ3ae&<& zCc$Uv>!hXaLyvm)SzK*&)wvy7#tjY6qp!T7HDc|T)3}}mD6a$ zZL?zOQX(eY7%r_L`!Z|t1?~y9If{JWR)%Y4VC&cT(g`AsWItdfsxWJ3pSG8!uL!c2 z6&fkr`Vw-JNX5>p@fohgW+zJDum&Y5L&8^s6V@Ui%Spb|u9_e(8BORDxS88ZMgnQH z!orqu6IKUSeV+B{Buu5)Gt-9{Hqnw&vrhK7wzl>W@%YJ;{hMM|y-f*~>oB)`C%fQmm*$OVZmA>EdZh~h!|hWkmE>$zpO*)yDMT@7+@j{Ks6`Phsv3y_bH0Y~mttOp zR6}^?>}8a^fZzq~3NiyoGh2o~-RaZ!Z}Td6FYxI4!6uPD*kbD?htR+f)ILc!fTjLW zKN$PkrlzKTRh2ayY==I$-)P;EwdKwP0e+MEJh(D_(%Rw5A7;89X0fpfu{fC}PDVZR z^%E0ib7q+gO#wo0{;RazjI#c%=5grg77yequ_$h{gxO>RW*I1?@lyL3aC6=oK&ceN z1!|H*S+00KEN0_q13QBU$g(&!&2laR<(V>8K z+%vX$A1j9ys7cG`1Ufaki3>|;g?u|EDKpR`HRu(VYeuLc1Skq)H*<}wbRZ508L})H zlR1JW;BSG&X&QYdtwPgw>2FsU+(7L@X1+i6wQy#n;CX)I%ryLZ+4rS*Jz2S7d#Ls_Nds)iHc!WG8!9fSrpHL{3N5+f_eHw`3lVoT0UAFM!@+t@@sr(r=)g3_ zPjoM_Hdep_1}D)tu7a^X!N_rfpgrX>ULnLc3VA( z&gQ_8Sh3NZk9K*Q^DAenf)d_fIuYQ@GPbFn`8LJ1P0M4pdh^mYqwEh%41 zQ%Gs4$V3*yCOQF>9#cVqqosFClg<@#SPw`)i1AY7iXrR1q_?;p_L5 zpQpLz%EH0G(qbs5`=J#N%q%`=%kBvf2t}m{zK}vP#mo5#0)U}bMyYBxfDQHSesunj z{O8Ubr_yhx78nBV5MgzjfAhZLu5)UL(%29vUQPf5lg!{t?}jNXcM{!EE+a<}Mlp|* zNrCDbDU^LLP+hK6k^*I#k!jb6akZo{9PI-(F5S{yCz;hV5`0V!7CcMuu<5#pBZQl$ z5)QS1R0+I}o`y*UP5{}sF&-;56|l!=!6pqyH$J(vYPav`>*vTMLA9o_w$=jn`9gjM zFcka)ladGMG~_SHA|Xvd3S+bFoTTQz5aQ_<1@EzkakvwKf!mNa77&JSnKk^=)SN%6 zvK$4EkjuYxK61qE4oBFM{vYFHD-@+X-Zt$}m9IedEH23r3x!f`C0%$gq&Mbqq6$&D zc^qWB`sNRUcpX|#p|$0PmiT(A__`Kn&Os7M{-E7#O4u~%;=iRqn98vMH)ZnsG6e29 z?heBt^{Y1ktBoU12m%lDZE$JRAA6+V4k{BQ?{adW%H`^MSxaS}`kuHlemo#c@EG$i zRb*uj|E@}eMK@?mGcxC+h$6!Xrlln=vs^8Bqu14BIsV5abP@wBnI6PXe}r? zKTs!`1H^&zC)57QjKh=Z{M&;~`(9}MHbnqGaan81l!N{tT9@cK$%v$GxfeAPCHU{Y zBJ;=$&c3z^aB6X&2MdS?xWC@s(9ceoo_(Mg?y546Lvo_#acD)GL#(YE5}Jl!f!P6*}6wv-UgdY zr+uhW)!~Td&9xjYq*Y=bhs+uBk4Z`;cB>rfegrC;LuO&J`srxEB8%{+EHksoV}xTf zVK34D_^RAC_;yC4Tn>o{nxpB1B0UhRaeD-{NM&^vW$|{arXfEVS2`%v`uaxv^gyOV zt`L;2ECbR z>H?%d@w8WJlI2FU3cR@?mAC5I@xp_3f#&%B`IawRoL%W9nHZ1Lq`Uu8O~)LSPSwfZ zm?&4r)C5FH@)fNCVQ7s1>I%rhrnG|IyZB1On^}qbuL$P@N1tR;nsQf(@;*h23l%F;s)|%iC^bc4}AT}eYtAh5La>>VC7KELMpxajP<3T(W z7|dWmoO7Kjgut+3^1oWF^0yTbUY5@>t`GPCE!7R6jtt3Vdl-zdF}G~D?+$nDD6q&4O( zEiG|LNi{lR^tNI?es3c7<4Eq|$!R?8Ja6B=G$USwc3Z$8a{)mD>|E0T05>a}8mj%QVrKU97dJu;;G{o8=AbM+4g1Ew&B5vLXx0bsitN%T8?ys{ce`4NC zS5Bd$>`Al|Z54`ZgPq#uafI+5*kq2NR3IZYkFzZw#F@t-`ZhF2kpkU&FK-)E2~x%i z5O43YQ?77`1AZyU%FKQ6AWn(c`1-b=g~Wc3ZHvQNCi-|~PkMdxq7!DhYXuP}Gzcad z+J=U)w*69sYnp^QQ+jo3+!9nV%2KdX4E)@gwH!gZfb#DOM_x*K_rlNxJt`oNdYQ9vA&O>-SL0vd zaq-d0&&FOJ3wOM(x?T{`?iF;b`SK^*;&KUa35 zl*O~&i*Ncos;b6c9sTE{Ql<1VGQ5zkYncz0c>Q#uBFDozQ5C|A1`)`h_{fXv4jHxWXPd;O0=fM5OjHAaE^7>}f77%rQ@8xZ^j${C4rnlzhfM`$%D4$cwcrY)4T<5>lns@vD~jGFA*a zH3)|Le&bRt(Dt&5>jgC@(C+JAMEPS<1Ag-G=qq>ycrY+)tm3l3&UA;B_csCHooOBi zwGyQ0O?ZQbF+8mea_0ey@FNLRq1T#>f5RFv9;qoxCmfmViiCHZA%g)WnX&wIsHs6u zx)zjgCXHsuSYB%O^S>@g`BS8y#SyG>oFMGg9)i6rF|t24VCL4Y`?=&Q-l9^L9ssEf>l5s>{4@gbA2uIju9kG zp99De3V15MKYfIB(<4E+3|1$|c|biBy^enUVeww)qw^4`A*pnTePd)WTNeCno;O@Y zoogwmv*p=>cI&f?YO5&xp{+ z{n$intNkj{d8~IQH2)z?DeilPT(%(Aw&nEX2_u`kD3X$SNu;ifGG8T<`1Wwf%KH&LG5TxF(XJS6rJ1R3gxVfh_ncese?6-$WL>=gnMB0(qK&R`y)W+{kI98@q$b+HK9H|{^C~ZWK3<6)9^H6q zD!Kj86ma@Y!Xy53I<1J=H6&Y^z0JnYg%Fe*G9CSZe=)L zR%UK;P2->i?neR{r13n_D|Uy26a6v6DtzGAk1tq!8F+4o%6I1bXU+R|&ulVHm z$g0TMC-1G={!`?CSfBoQ@yDeq-CuUr?eYF~&9=bHR;L$yv9Fm<4`?)X)l{jznMa|P zNL1O@IKNca2X?1rE%w8vc@dqAjdN`)sLqrnyeu0X?9}WMZT;nMxAo31F1mg(E!Cd< zGyxUhq@=?_q}oVEi9pUER0neglX>B;I8Eux8geL;*FZg2m z5r_8_;WdA9St_UHOeV&aYRjgK=bzX7-VZgK&nw^Oq(bXH8IkiH79yG$TykovEf6S@ zL2%m!2X$1E6CEW{Ho^#&C}PL^Riw8^jFc|@z>^_y6rAiCD4`#VD0jBqI(H{7@!4H8 zNRl_MSO+>P&(8TDMr7#t!ham{sjENnUKe%5jZBdknE$LdE;(a>bxiz zo)k7&F2E`dO^-FEjxdb5Jw4EUdxz+IXYd;3l~oD_Q2xN7YtN*U$TvZM zvKOmjgEEtclu$c-W#Nt|Ks$jbfBNi%NXqKy+cOC>14dL0G~MsJhc|M zprN*m!N)ObI1q0q6GcHJDj6;19CsSawg9XoZ|`NBsgsw5e9+YY+qDsJ9AW4t1!{jH zqY|u502Bequr-(5>ah);gB?8`D;prEz`hW&S6h)!pK+8Q?7-SId8hw(>s!e5R4zVb zI8dC17M>Fwh|X|JmMn>uJUe-a4qL@=ji7mh!%A*rOM=bPe>u80Zr^vIe18NJfZmw5+PQ%DA*WhYZTI;gg5b zIQ2oxbNYEZv)%Jv9Y&|LHWPn8c~4V@=R28L9D;5cR}QLanaIF_U*fcYUn+7fZPy)} zTv(l7{~t$ptH8uIAT+eZPSL4?wWFhkscH#KH<&?sk5!>)kBeXj5!s2MK{xXKKT%$I z=kFi7Xu;MLfYTSam581W54D#@%aE;RADM1@Dp$F{>H4E8W??}h(U#6-QZpr)G)X4e z8LEkcLbKUe!Dj3sI-)EF;dpCGSx|xfXOoe6q{8)sZoN_n7He@2x;3yRUhO3~I&|k8 zQ=su}_tKv9(0>g~=S`#0eFjTwd^Q2;CbnW}D$qWXQ^NUA$^9WY%C!gufj(p46!uZv z70UJkfmw1nDA0y5FUK5<_@-V2A{GFj~~u(eDhh@I$HIp`oFD;qV;qy1gO?P?=5!!o|*SdjZ5leS)cH zv8cW5pAEnA@0~xtb+>u2ueb+o9X>3yR*YKX%0Yt$R}RTr#-J^+&z3Af5iA)RRj2!r z$S8CRTO0jwJq9)?sDUk-xa5=+-3F%=O-Hk!rFx~TkjaY~xB0+<*$nFtX}*#rz$orW zC(`g28chUyanMDUc_pE005joa7!pn4FZ-4?>OM6#8Y75N6cIJ8`~ z%Dte3-*$6oqcd^Jv*&#CLD8=xR_S0+88$N%7=g$}rl#5MrR_Fb_le9_HV7Q)sFeYk zU$>Rhor@~{rryKfZ{i?EBPhL+#cJm+#C$oro0r~@)F8iT6r|CJqw z0M(N<=82Ov>B$b_lyDy%qLTLVFe*7wpP)O41@fj@(__eBtv7#VF&vvrASCheIY19B zcTU2#1&KSVf?wBp(}E_SL8pV*R*B_Ljs5g*7f(&5XO38ou!M}4(ec@X(5bf{M^cDL zcsfWT9i~fJ3OSiL=1M9gKRdbV?=#}T+XV|52{@t96YXG1UChAKqSegeEfJilD%baF z`;XD&*;TiE;F@8x**NHi#R;L2F^`O@d&)?~q%m5eoHWtVbN*qqz%l{qRdQgd6D~bH zonS}0XPjjG5`}Qh^bHP#wxX5(lh@Ia|3TqJr{p^1C7(9J>_iEw%hJUGa9o_{wnQDy zbhD^Yh*nbzM|ezDy#${%s3-5%BTHcOoelkXtf>vMI8Jm#hhRey9XGhf4%wt3`9{4z zfi<~kA_-}U`qPK#o%ia`-=hY#aVVGx478y` z9x&J_f73{oR6`#rIVHIJ{CBbwus%b}^(G$}=7ZNb_{ZsAKtU(s+ zQpsdU*Ia-c0R^$m{*7kLojFzLd>kI$wdk@UNEzaq@Itd6;l7!Yi@foR)sh%B{l&Z$l8uf- z%0eRCne7PSi|IG`&ilU>)MKA46qO@1qH5#%$5oO(zjywh@<&xBN>!9|A|=H}_~MW? zP6*Sa1}EdP71(o%% zPukefu$U9&7bTx6{CY1^teI}T{yZd9b>{E;3cZD#9da57|CP261_&mO3W94~mj3kj z;1W)lFuA5bL%9x^VQ`hHXY1JL?9wPnoowmkIXb_V|7)e?uNAYUUHWY>yCInjrb(Vj zM&`8wwjGy}spVYHhtp#vjdd`1{SX5bL$RBZ#e~To(=&cf9bI}(Kk02i z^uNfrgs2Y-7ty1}x(9jAckS>Leo}Tt;UXGE;np}!+3VeZv-4BKC))IZ?o}BM>y?il zRC*39MBf0q-dCV*eO$!oZOui>88CI!cfPgNd=LQQErVVG*1bV!#ksF^0dN!kMJSPE zr;?nf2X<4(|8Qu}i6wN376aI(%*gHKmQi8O&$4zF*o9{3Mm)GUp7LlLV(3( zGdN^qgOJllK*=F%)78_1ps90wFmzb#)3Xal8&RVjpliyWYvP1-3b6nMyk&q=IG|JD z=t>fh=`QMjdow7_t+MLBOX>qHT8A(cMcSv{xcvFbyim2uP+NPve-xRK{rjeG^9! z;$WXWF=b%d7#}p>rk}IYpOU9lRIe_gMa&=@NoKR0ew=!BO#j@HT6s0Qd3KcqYp{v$C&Rt=w~K zSAy=`U+h+XY4p{Pvp%W5dhD?Gx;>m-f1Le!&i1zAKXgClq@KO<)sKOhYrP)*@!hGz z_coa8+}(4dzFyFVWmA{V>$zV4z2vgWH(ep++wc`6z3`zS=W%p{m8)C-d2R+%uLH_~ zVj7q@px^^=Dnj*8?glN`fE?2$$BOm$HM`3il8sR!k!?10dmRn`I-`nA)TSHs$tTU* zh900%vQ5urYqKNtk>IRBiI9V;O7wEK8#WwhwA42IOz)%D?IMA)fd{@>4)1<7bT znXZgFQHdveP{_bjkVOsPL4CfUq7C|yGe!L&&h4cNzsYIax&7b$Oy8^j>1hJS0i8m2 zbiIj(dQ(LvC8{MO(n_f9UYWq1DxE~n=<)%7GJ3%5xm90IWm6h%|LRp8Qzn~}ItQ1* z`mgHRudI_m#C5UeIOK^rV>~oRx9s~{<*MU=rS-$6+qtjWvh8I`bW@2`%Ow;=Xn@?` z2qy&9sKdeuK_jKZX)K}RtduPxO(6g#Pd@0`SJM5AlF223Tzh47haz-@P(wMuz8#EG zrgZu`<6W5yUAN*T3ji%YFbQcw#wl#+_^D40^8x}jy1dKVt{vYo8FfTbH~*_!)@?+a zQDG~Ja4wu41==u~lt8f(F-PIJw?Woe!r!hXHX4oDNx%Qc*_#J6b*63O%ZsSP`c z&?QYcrH77SbMXFUGqc{}_pz#<&manwAq*fFdDB$fMA)1fLRdY`LNT-u`jJo8bI3zJ6Prd#LD4i|G$n!BOqg7sFg%DiKZs8)DIm~ ziB!b*8A81&Ss*W%_i7lr_hK&eK)tsxLw)=*GeQ5ChUtSa5CMrU*8D38rt|*d_lwp) z4~4oCx}p-P1M3;LA3C%!PH~ z8&|_VZ`y-H`|z+;Itl2|bko3nArp@3A`n(3o(^A3lGbFyl+gE2KbcMICI9_>{}yL&$^clh zSQiYGMRyhE6|*~5nqmUc!LJnWepgCYy54qy; z?4b@$AkiwV4vGQ$b1G=mwF1@9ObYy_E~PLR)!2FR3LcAzc9OA?6?6m~5RyUGrk}^d z^rG517z!W=rGtA?i=tkD=$qI+6??IhEZ#@D2Q3?fy0iUVI?=3a`+ZC~6A$p4Lj4xXWrVCXip~nMz zdK(e#)MJ%XbOJnd!*v9 zuV)uED#fGZ0G-s%mZ8qOJWowp-L+RLGj$K1oLmD?1nRNr3F>EHgJ7rcJ|T*nnP`|3 z^m9##Ofr7DmkArnO5$moxSmrVUYw}}B$@v{2IGBoHZo`r8a8Z*mA4fzd4pNJ+gOw^ zgP4Kc_wn}LvvFtoD`yzf`XnukX7PYO-M~k|8M8c5A5W~4hDRlq@y}tX41Qz|{!V=8 zppWOusu}!aB>GKPJz6$uxrpTiiv$rnZXRb2#kv&M0s`_IrpNkJTrNok@lBVU1V{Ue zmO&oC0trb_o3NlEp~2ZSgGQp?bS}?J-i#1YR2Bm&E-M_?6s5RKm;P)t_HQp|=p&wx zU6qTjjeKQ6`JRck>ag?(KypkKRvAv+AomXDF$#y(E4*PWwebrs&Q=K){XS;!i$8n; zp#eG}*qy_{3seE{MZbMl>K>pN91M*5ef;SAL_O^GhArxMX*9uu^#YgOD8=RMCr>T7 zXgGcFLu*IVNG6&8J{js4D`<)u_aYaIqJI?bl!_I5XP6O$KaDq?L!%!B)1Bt|$ zG!QXdSdqYU#_to(wX6IB$}TLuPD1|w5zeAY6A+W&(AtPR}bBvBLQMRp7Ljm@cH!%J|94F8-HG z{3SsuEJSfNN@P(r6(v5Pc9|c5O_nw9yyBO-cY&8iiD#Fb;MtaudkO`T+C-(+Z6v3^ z8>OsH&dy#6JLSrYiDwki@)-te&z1j|mGi!b%Pv%+51_)l1xmuclsf6ImopHs#K?Jk0P-k{%9!x*qN(iunQa4=Id2%Y_``L#^q{nfsNPB&-%8ZC;PtzS zsLQws4KY;}OmNR{!I|?RuWHIWzQ4%=COj%-fu+@HwI8q4X2ifgp@6~|(@jYV{oxJM z3{&|9E4ZLI1}arm<4?8|sn6bL+#1qC$&H0T#@{k4xu=rbgY1oYFiEMcGSkhkHrl^o zN>Pa&rFSm*7M8DK)Ik6WP^B9sJE%#=9yK&h`8AGNetrL{SuVh@CYzb{p*PI)K$DHc zbh1vvVn$>$fJ`Xy7NT|y=*p?k4(fxMd;06|N3UHTyy1YVVDC=MMm!t@T}Oczmx-z;^M=iY%AAUxJMucCzVrfj@o%DaP|y z8%)#`iES!!p?JlM8$!fNU#P1lqRqVZN*rf8PW6)CC#yCTn|2_0yTZg(Z-@R2#IDM? zVy1vP9BJbBQK7&~BjVM&@rXiG_A(7`W~h4Jd(*{^j0Ys1)VdhTR-zqjqrtLzR5gWL z@@fh_a~WPKFT1dr17lA~zg$Mm8kaCCMDWv)j3CoCLlOJ=-6QS*g$S#7r*sK_^+jAv z15`BcGLeupQwtgE44)Bkzo5=DR8aMfmeyR;Y+Au@x{`iA+RTgUiY5pRy^SatWx5$g z(@0o`2r&y<-jFd%_NuY?94os*P8v^Jpd`=uIVdaV&*qg`B7SD2tkGRzhB6DJ$kqa%m%toaJyVpjq8be*!Yrw4H&py zRnyVpa_xAF2fG_;ubw$x!*;Xe4#k@-y_YX(BxF6zTRr^j{;X+Xaawrc9^G zs8_OwbUdYwIrW{*KY6X5Y1$4ZwR89wQL~HA|7O{R+;O6<>jbFdFzJ7 z!J_eh1aZYoo-Bcvhp{>`reKLNJkWAS%VkJ1HvSv`Emj}E0jjb7~j_RQx;ZH=Z2RWA!qysU}&@~(V zF(OK2ynhP{ow<_a-zV|^;$jdk@h*~p3LvuihIkCz+}U!^NoxySBckcvFR_{Y@5?FG z%yspA^rm+Mi)fHTFBk$z8WX|*!CWUq=6Z}?jf&x~0Ch8mv3S$m1H`^3WEEBPU;p zq-So-);GN|Je4!Qm>H^X3oRHz`$wQQbUbPpc{Tt`fGL>$vNr~J6(M=Pr*tR_hE%Xy zGp(1+hb$~AaY2SUd@*%~$Jqj5YygBgd;PgLfLl(gE&OnvPv99%cU?`{I|( zAJqhG0D&nSnv_92gt6kux(*RD9wIOlK>P~>)eJTX+J*Up`{z&Sjn*V~x2=2osgr*^ zUm!LAcqjAJ^8aMlCSClx`QwfMsQB|mhRf2&*YhLj>--kH{mJsWZ?KFYL&yR+F-(HcRRc;N=y zmDB=t?_gk;mh-DI4f!+JZKneEnyjWm$I#;T7v3_}l|Ax9MNcyw3fC7p`{uRWK45|Z z!-$EOF>wD^AOH1B(NEr3sk3LG7?t7NF>)CbHS61>zL)1(zyu)^BFB3*3sPNY?aT0@ z%l>8QgsA=I2ko;I=hF1%=H^6d-x8bKLm`XWBu~Pl|5CJI};U zq-S~CetjA*y$*Y~=`{AI5a0Cr>r8-}guT7J?%9uQJu~K0-_CV@pM2LR)ce)I)Wx(>u~A=*xD`WUqU+-Tc?D zzU7sdqUN8+2-q$iVlivz$&KyARrAj>2UH(LIRY}`E8~ z?&Ur}MRk%IkDWB_~BG{E|I=Ljlkxzc_28Jmb zXNL;0NBbkNT|h%UJ>N4YDW88-D3a&INvSViQ`*I%c@Ophfr#dv)8_G^eEB)iD6gib zhKXuh3JdNj6z*xhuSn3Ja6J1-U@uVFr z^Y#s|g2^rH_Ot?}+?a*j6OeZ(DLNqpwk&=o3bpC`Fj)RmzBRBCNM&> ztQIfcvoUiA^RggjcQTqx!hooU;j=?I^PIG`!E@_D*q5htj42M1Dn(0h->N{fZg;Hv364Gwa~PIj4$p9~{fi3M--Iv)8nF zTyoEIR@JHJpMCg&h#|NzyF=&7o1TR1`SV*$aY#i06A^GsdyIylYs>8+*s#wWLS!;l zJjiU5g+cc3|NECy!Cv*0uW9=f<3}$`&lxSE0UL$mxiY#|1h&yhIuQ%`KgWCB8@+6bhI&Dh;{01>|VTdS+l=ZbJkm81;^4!%2+xOTCuGt zXsIEV_<(BC-AL7Tk%Z8VVStIiz%O17hm+)wj;w1xo~C}^O{8PDVN{u>qyI5^(Q4uw z-=uWL^bulRT^(j%!DnNvRy0$fh~ z82AETVE}2)w`cMui{o$Tp4>84`!ECCBs<@OorV}y z*bnXr9E0#UxB}XSRK`Y4no3YO18-Z@?Rj~<%?>?!?&-*!i+QR}N-pfi*^7k@5oP%S zP?U=b3%k?82X`J1CH+;el;@%jy74293xSOgnX_Y=vlMxv<_E&;DrLY7@xJ=Pj@K?z zURa4Z*R^fG z)O6HY!~(B@97IK=m3);;E4cKLeeJoG{_zE1<&&~%Upo{%a|0_1f){-S=aZE6!)z#{ zE*Di5GEs+XqQ0InO(1UFx|P@kjh2X1fUT)S(9v1(#~Hf~pENy63n7+Sp+PI-X3KFf z%Ga4vsgw!E!`&DQ$bqx`9CWW5iEB^II2rKx`i)EQvFgK}uRpc#?gWdA)}VLoJTN;_ zn1zMFsFj%rg}+j!Uaw~&i48c+XG8$9W*qo?uNaeE$gE$6vC@kbns4;r!JG>6*0k;V zP`>6|ZTVlMPW~zSAv-v~_DehJ@QQbp7cMxUC^R)SOZV63hgjtkSm;1BbMP`c2 zB?%v%wPc|Ucvx5g1Cf(x8m6CrCdnV3aO#5P`(CY~DC7ZFxO5x)E^SX#ofViN&=dL| zvz<-_HSiho|E8a0mdN$Yy~T5htDNxfIJ~z6o!+_3*P1<-{Uw|Qh;dYXxNi2&B6p5O z6=mMT=RgaYxh(zb1K2wF`Mk``L^sg>S1^+S6;_F?N0=Fkq13@iU9q6vaw_1i>2_#_ zYA~5B6j7Q)FY@tiJ|D_bg4G|InN4)Quu$Hpkq0(}bXnzogDjv++k|Y!ApGhezr6I` zYzhj0`OD0!kPXG{`OItAt}!n*K%Ylw7|Q};h1(urnRJzfR{1I$jbGVJe8tQ)PRmFf zKA$Y4xJk-r>oWg5yQ4?^+S!E;@v{wh`N(0{E(?HcR`LhfrL9Dn- zXt})Ot+(LY3nLBcmvQOYa}ZSp2Rj6F70>%DxbGTawMnjGPZ73!J z4@=+OTxm)K4`PPL-$lBBNQJgR;jqg)z~6xO<+|ByD+BM6qx*wd&wE9ixGgR&WzDv8 zQA^w;dFyK@N>6RE>m+jNVfq=5@#^_xHXS8eSy^GZ@YI>mi8IXA-lbm%5o4XChOWU^ znGX-`5;nj#OC}G{=qlK+wj1Uu|5@Y~X=O38il;ebVejat#Ds<`uD1gc#w7nn=CQ|V z-3M5V@bV$y_tIS{{EyK%mFCVQc`v`I^l=ck#_nJHR%f7Df*ECYypY9=2SCh6h}05H zx%@_k#f^QIXCF3~LlEGsmzF~oCWELd?fSLBk=$}Fm-ZwwZ0wK=Pv9Q-4|h09nCt6} zPG_lmVj9Xa!mP02@=g$^GtNOyo1;Mn1s3f+2OzCO6CHm;tg9GbQWsSVbwd|#sjdc; zm?0-Wlt-foE(v7Ls#-0&KI~EYolh`r_*lY)ZJYeHZ=;1(eYm|{<$tX>?(V2dN4tyI zF;U~9-|XsNVI_DhD4eK#Ea^_5K|yr{`b1!xz1^lBr|nTg_vSs&PDZ;Y3Pk7jMh!kW zog<*O9l1O55XB8ib+@EVH<%i&mQ^lIb!NtQN<$>ap+BTY;m~$o-*#p^h%W9DW>7~X znI3|K5Qro`hwq7=@&egXPH@YgPWz$$Le{0OXLthdaxC6`NYxzh&ztYzPFWJHu=0W`ItTN4g1!RpPlY9(|7!;B~g8WJbIyqYAavtZ9Sl5EE>C*lkYIF zQQ$_SH23x%Bm!u_I|e5lcJlQO7G0@h&tGWxV2H&}C*ODny9|OxoynXdaMks7Yp(8x6gcVgqchLMA^? zHe4rLHQ$z@niz98j`zi1mUt;UjXjfkbCFmzJFnD3W9|u!3*Bv7dYxRCchTsUR;us) zRbK-GfcAwDtE*!4DW@R{pBv(oGz%kg^+K--_3Vf}_Zb<09b5 zw5bEM-ET0V(FZ<%=)vi^&H^96ZdM(wiQ}qA?-+t2zI2lJN%K?NY#9k3l2>X9C*rtb zb$!%bRhhd$m-fDv@;AG3rF6>tw<`6^!O6CDB>h zQh!3G8dm38mJuj-N?cKNQ(-KGUd(?;V_N~UL)8vWMOkY67p?OOev<3RX)~(2?iIl45 z*9syz)2rYpVfu(qUdGR`!_WLEyn^EAYD-w;1p(_a_iW5@kLi$3JjItdWHmvyKRx;v zvyn+yb~;u$WikR84V?f+AnD>|7dx2%BV^Z!9kX(AIC#PySTYz4#r$@rlJA)LNRi za6NBgAF&Ji9TQy*&H5b8`;75YX70!-+dWaB&b}v4HrSl2A~)^jVYQB)wE;zL;LO3q z!Dx_)&|F5U4rL+Bl&q1wBQm%Ca41@jr;T4>wiU3-{X#vJ&mMkcETaJeR99CsB|Ew~ zF=|J0>ZA~Qg5)uw_z94?CA2_f#bXh71IVclZNSfki|Z_X{RB<($tw8|8iBMsrlIXd zxQ|oL@I66CNV3f(6gO%%H<_DF1kIw25cBTo9UUEQ*405c8K;bW@P6*!f6jh8M*v1s z59-1N!&wX3x#;-SO~+hf>~ps+5;TyTyg+rBnfN?8@fH)>%5$g2DdN)R*xX=m8rXrT z4RPNyYwC9q$>aB+cyKkt03B!g`t1sa8gf-PhzbTC--rjO^=>7&9l4KwvGIaH#s0;3 z4#q#lJ7zWl+K}1kQCJaIQy+bdTps%4dY;ZmiGls$p^4X+TEk-?=m zhbAV*n#*;%&Bi>Cgkhr2bu$c3QPpsi@VH^Tvn-M!kA5mW=OKcyRuJ1os)jl#g&lcFa z%8Af3vQ^2lRV3~(rMbQjXW`i8nnBZSFzyWxf~zu%ZS=BBuI#&1N4eA7MyNNmOy#Dp zSDld$XtoopUfAemu$KuxZBDf6#P_YD=|n~))4(R{x|oIut&vLHD(^L{^>7;4Stp;! zUTdQ(;Z_0cS0(o$>RUbZdx&;%_1oR}W_EwDkBZwh58nwXkF%l}d=%~BD3%pXj`gqT#>3F=XNyRc7G^!Y<^31EnB^d4!HRG2 z>i!aIgogSbF$6IK{DF<^_kM;^tfTz3%2ZLYE~rl4Za8`JBz)5_V^f7Prn3&sZ!-t_ zTDxd_KHNWV;h`ePb^K*%dF>)`Z)bZ2O$JwHA<;^(2>+0ORs9K1TE&9qJcWmV$Z@hB z3u_sypXAh8$axw*)el?^lkL1fqW4br>Ch)-9n6WVsr0h7Rf%k+>FNt}#{O3}X3O(K zN9`#Jbv^Pw{yu)>o|ADu^g3C+pKPmpM<(|2*pfV2lX@nOYY|tUL2H=HoH|3Qy2r#D zVrct?c%3YiIi%<7GisdDdnb$AzulS>lw3JRA+o_ni)! zIcDA~5Ec`vD+y%{-LRt`CnEhy))e&4Q!#zV=ox#|pr~?(Sykn#v_+SU9^Nsv`jt|* zu427^iY`jf?J&&B6v#;4SY;{XkB1F`E)pK(nJv>IxEF_vPjh$Nv|XVX5E+`B4N*ks z6({k+%q}R3Z6#W+_IAuHbKDnm%xY(HTtdt{4B1Ab8ecf%#nWG?k#*V(ScRmX!SA5H zi~L5CTwKZ@IbrM#^C>vvA}{|)GC9qM$iNj^SQiZGjl(C2dK>&`3sKKE8fu75+|9mG zWl%1<){yKh7r2j=htkB(D9@PQLo|+&d48qFBNFMQ3@h!{5c`^_MoSbrr@4r)pbLG? zV+V*SHwVJ(h%H|9<@5# z#Xok~K-ncc7)ZvDx1MtQVFwpCo_EfGpMUX3#<)S8GX8{-%ow+b`-x4tPy13RnD#e% z0e!y$=5AKvm?d-SM{{5wP3pz4l#?lgN1b0JxU?Id0xrg5@-%_`5^QTd(RGmE;W$aD zk$tH8L98hCcp^g3hV}6`snLx5*bW6+tiMs_4pgZ_h2w;53$w5Ss742bLDqz9D^$8k z*)a%jx!7>ym$;&9fmRHIo}+1}RzTt)i|6PoEjXqjE%YZf1fm)lz)<`aeD0!zHKF!a zg|O`=^wy9qbP{FlZ#)xMTpuW0*c9*(Ru=v zJ(qQsSm5!9s@G?@aGC7HdOUlGVPFGwEOgQ`loe3Ea44zCHo?~BKxufsrKZ;c;8V7) zoV3T(#s;Y%*s2C>87-y(UG+u-dKzuOgsOMNAtTfx2Juc#9fSOi%3`URnT4ob`^~7U z9}Fs_**6e&6|F>ZHTLkSM zUdw5_f1u-s`IIZB6N#vrGxYq}kDvY5@y9~l zhmGcviL1SF9Y7KKW@9W$31=4V*cc- zfBpHR#?6gC?2Jf`Yz|GNw?;MNriK4pZ%3 zIJo87T|5I$E;nP8(|t9LN7d6k{MW6P zk`G*K@rku!ej}wrrlB)s@y_AZ@4lCG?fBG%6>^+>S$BO7hqhLuMuLgNvGHR&4vBPfT z?8-@MxzZe3R-sqKtp)_{)MV>;R5v?r9Xets1IBh^32fam2_T^DZzj2vzKQbU-Y^c% zlc$Jtv+GSWOhAWho-Kv*{vVoSJrwg5-)5C612Z+ETqPpZ*%S%1~&y8IITT*U)N z_wh@3;UV-Rcc-(l>+-)G!fkZU7Gx>khcP|KZNz&wZks-`rP!$ zIQGQlgs~UOi4t7>Zr{-awNELzFx>yz z1#helnfeBvhASRkb@>k1*PupA$wf`yv9|fn)+FJ$badj><#j*m~AD$}QPQ>VJ)4z^35BqW$58-4u?YK5CpyDW3=TSjzRcd2i zI0z>|iRI}`FRPxfuDID23QopkS?*@wMKYlxE&*<9kt^ux)MfFp%vArVGJo!N|4q%D z#rBKd0hN@|doO5Rax?^ASxj%HOQlEU`iRb6#a-*ed(SkDo%eSInZFtmEtGzw6~E^$ zBZDG}4Eb8{xVA2{^&lstK4yPR>u9N;A-&%TW7#IH7vdWJAn;q`wLx-ny;tKx@yQGQ zrOzu<&#jiwiWgfJ6kA)KEuaUl*R_h~uAbj8+PW3ygP|eI5DD4}Hl*NW1cd8W3MDC|QcVnzI^6f{Gg<`BO|&#!shNuIeS z?s<`EB_ff?;}Wo}Jm^6oA3$pI1b!*Oz!n)np!T95Dh@}$_i{2z zO21KZIKWf9v}JJol;TP>KXAiB#U*8!BiQJp#aqgePBeN{&uMVYu_Q4ZY@ANq`YwbU z7YDs*3~vkk4cImN-+g>5pXIr4xzx9+JHkgEy-ySmQj4~5v)45X6&FdnnvQ~CbnsHb z-E#g9#-S|ZcKVUhU1LoVE(Pg#4N)%3r2#7%q(e@o6|dPF^F_DUQ^JB|Jr_S>myTl? zUVx4wo+7URmGZ7gI_{+J0zppGG6J2;*DDBg%gUa&2y}mF{=QVwmf*2MiHB-qlIwE; zvcs=^Jawji2LDX(FY?R|GXk#`)HKkbB9+rnAS=~Id!_IzRDZ~Gui&*!?W^e;QvEzJ zj~2JTt?EK!3+f<>ONQ>m<}No&v+DXUb)R;HXiq89qiK1yT3!F++E|zFB0b=Ui$)cy z=UY*d2aoKWp-3sHPD`zCILD6kyp54ot22X2*L_akanvv8Gd}4VP0sBG@7U?@mW6BY z0Ew6KoV&h<*L+-;G9?B*GOldmE|oxcuJfZ5te1Q|SmBrRB4}^h*&vUGa4Ne5 zyL!jv%ZkU`avo0~qaV59HL%fF|A#FQo8>Ygr7L-#y^0!WV@XGEbf0`$0r?U zZei`x3j$oz7quX(xlp{(N8F__clU8+tBb58g^Z0p1={p_91>xPcrXnlI;5!sN)=VU zQs5);q;Ya!oVjzObvH^PI}|rM3U40_HV+O7u}1<;TilRXAX&U`Bd@D@=2rI@Zo5i` zhX-2PPQ-MEZ>38G*TjO`@<;{fCx_OlGSg5TQ>`cPNxo~NpXq#qQEl5S zY+dVQ>WW2vmC9_?R+a_AriTmh%r;P)&6*}%C#}IZ);9?pk7LzD-NL)e~1F7KB z>t4;*@}m>*&qTpRpFbh&sDn75XXUftQuLj_@u;rnc;fbbjIN@2p=ko=iGvOEwTE_D zyTI;lTs_cZGLTXo2n+?aeNJw%f;C&@qxwU$OAHAg=vt^vj;`|c@!4sz$Dj~uvbCr- zvXp=t>e~{AD^<<7`kfA4=W%6FegsA7$;$Ui@FZ6|v+BWUur*uw&q3*tKu6-*+vpTg z43$Lh`({Pq=SZ7yi}7}qE9m_xZY^xnPZ4J>g2Hj~;1=k;%iJU(N(UQcS(0=o>#ygt zKAiU`w#~-a>zt0;nr4mEI74uRo5kyfIc3xY`Vq;b4s`#tn~iRSyU)*Uf4WM(Db;$p zoJ>mgqrQm!QX@F(XQh~$il-lO%hj3a^zQ!=9-KV>Re@q%cY-KkUIxB0rhBOj)N*^0 z$=~AQbc<+s!n&+gl9S~hOSwplkB+Js1o{eg<=wvGY+Go=u?XdQ?-`-*m7Q_zjwggC z>Kib_Mv`Q&75q)weZaH`0w=Eg6P)B6SOnFrgD0Xp3pENcY2h>dlfMBwBV~Leve7Z; zLS-a~No2!AeqH36NlN!EBU%V!KyCS$FqXjACJ^Xor%TXv_Q588h)DM}5C+0%;h+k= zl(3fs4h3YV1MVdkf^0Re{+hM=7OoyC6K{z%Ov-GL=cb#66D&DM5{b5Bs-v^|kz1Ti zbVaxaT+MV<#cqbBNY<)|IeC|NC}B^V50u|RKc}x8SZa7W z0Fk=qa&b%I0!tm4J)MiIpSlm8G;SaJ@xFXGxnw`@;B{hmdZWLezILjkj6g3qyj0RX zHjTd0>_vfMva$#IH*rPjPM;oi?#{t&iVs{MC{Xq;NNb1< z$stV+IPf|OvkJ<<)6m)mm?aM(H=yi{TTWt7u56^4m ze9Xc>9gfc8H@PmJw|#RZC~z?ZvXl{|WTRjXkcE0hY)|WPk!rs&hQ$RTWiSX;Ik>nk zxo-w8PVOb?_#A-Vg<`dfdLkZw8)8ix4t}c+@b>Hgjt}VEF4SKJ*~nUhN8(s|islJw zivnh5z&~*rVGOCiBWnlRemlT{`LZ)G^*05mvi}@c8VX!Jz_llAV92rsX7) zWZJe9vh2v_R9RPWF2>}^CJd0zP%cQpw=YcPHgwrn1y^we*f^ zrKGsH{5#yH@;FC`Fdl%XvE)5ZfB;_~$Ynsp z>FNC*OUA|Rjp?n9yElMGh)o*&8Xv1@O%7<@7V*N(g_ z1bevpAp7lfkX<1Y=qH?Vwj`pYszmzL?oc;|gL8-eiLkr78#LCC_!ZLj>Prcw9H#fJ z3XR)N?}M!#J22B!zMn6rUKtR(k-G}D-z{vL&`_^+lR6#Dl3KcGOhtBY*wbU?!sths zNTyQ`$SMKhaT%eoe259YAb0#au3n(;1UbRn={{TKB@b6#3AaPByhl#bSyl*@uEi0>}APpuauO1wlP#*9c(~alBA~z|C^?`O*$2$HxUL{FoYhIx!4J;MApkl3c zY&lA~Ui;9vVk64H7@lK34@cmE20aeN|4Z~mhwI>?+E)_RW$vKkFflx6 z+_Z3CQS48Itx;UU8=6`?vNnR?F6yA(f@8m zYJ(6klq@8x%*`BqkZj-%yS;3FvF zg)SjP9Ek>#uQYT;rsIl-2iD>Ju!uWsS-D(q-EfD;kWEm29MHX^3+$)583!a8;XN@` zgGK^9DEPwYGY&^sJF29}z;SvcbgbVmD^My&t@prblyWQI#UOUmX}|JwHj9`shv~SP zbYgn17PPcV50@`Y-Es|A51CUsZqcGe&Nt_{^Y^m@s|wV`Z4wgb91`egQ$@-+NG+#O zfU7N?Xb-xGn?y7KNb7+Yi>rr?){R@-)NWsL#oA;I71HPr%&KpP;Ds6i3e|D-9FcDp zE>5EB7X8--=}#}νULTkQBPZWCo9Y)RK#*hD>}=QWA?7jLNu2o5d?_wDHpOpCl% zx4c7tHlaguYZ5jt)KaffXV{Gs0SeikiY9S^E-)5!;)i1?@5&>OqY-e->5%$X6l`aA zNDuLm);vT>kIK&--U>0WCMUt2ex&5mGD5D3&8xcZwOmz}7t=JyA#D-*XwQyT`T5b# z_6YCTr>n%*_ASsV?CN4<>ZG^@T0K{wCqXQ2sO5uyOio{)b5Z)IlVgcL078u4Nw0)b zAAz3FX0L9E`M?dni;^?Wd`%pU4t2mnjm6NE9QV8;YESi7TPW|!RB?xOQ2Ky?3xG&> z`i^Dk$ffVT6?VoGIkXV2mlLvphIZf};p!_Omc!N8)^4F6amukDitOqJt$2FhagT)k zCOt*^Vwc!E-_Z(;5I=|WG}N1kg~IPmPw*Z2k0p+Py4`{E zLPATA$xd*0u(bR=8mP_V>QTS;WrS``2?Qp&6F=D6caW>Q(_&iVpQf_)Y0)%4?uiOD zcX;&-=8mhA)hIOCx0@>}Ahujs$7%5iyZ@H+0ACzwhX%m${d=ObJR|LV)Pg9hy4p<; z{fWimrHy;(edK9*=nC@_k19uyyzhc}qQ8BDK2g2k7B4LfVmZEQGQWtQYUvR;U0JCd z8QbcDBSiFE9qXVsTK34#rt)}*g&=_B| zR6%a|bQqu`#YuAu`zJuEu}af{d(*#hxn{s;5l?c~;8Ut=D+4l4`?& zcOH(99o!CZKpar_9{3rd zJ}-wCAQHvf9J3L*`5Liw#Vjf39%Cy<*1&=M0Q51f(-sYiRdIG@ee{I$e3>U2itM+C zO84euP$x!uarN&ybbi}6F?II(3#BGQx?lBI<9Mj}0{ryFR~ z3b@gzT|-8l#kNEq`mR#Zn>m{GCP$-kD2Hy%Ppn$BC|W>h5(Nf4pl`R9%LOjggEKAH z+xTgez9j@YFonn;TMZ``H&mfe43nD2lS236R zVO0{c8a;bTE8^M|abk=(i>rsM8hKC{yO#d)l<>U#Na{c_efC#9p*)Lwh2=ySSv_`?rU z6lnbv)=>>A2B^TEsLni$1a!Mc&N$#8mW@N^Y8e5`NRgy1mcp zD`m>J1&I_9=s;+xH9*1NNuxKUYBM#_dT(MCqm*rCrfs7J;}L%4e4`{UKzDPPt+=I` zL}bv79ouQJgXt$no;9fki8;8_DI+Isz}P8{d#IK^u$GvoF5VRJAd#?>PXmJcgv()@hmq*W8gG#@t%< zvciiK1%I`6^{zeZ(AJ>)E3}U@w*Sa2ZgAWPp9*7FsL-ch^QfjiCIONVCV>sWC~n$x zupHsyWzgtoQC(C;ze6SLjW_8RV{_A#Al6Bre5=g3IUQtQu~qh#doHpu9`tGODuBy? zo$*W;n}rP#5%d`e;^3aGuWhH zZL8)-2z>96kTyK-+T&YA{F36KaQxU3JRMhNn397wEwpO~h}IZLrMeQv&H|wy;65N| zjxT*cZ5{Rq3J%^_B9`Jdc7ic%)pqcYOUNDijQxxhM9hK5h_FMDHar}s3FdZ{NJrIW zL~1Ve{#%yWYiNQ3^ajTZ!y%n4S6@tbiIB<;Ezrr!TOZ%c_oBj{w>|Sb-6gpzw@vxR z+;J(oiv{hD?{g&N{WAROqkacsej@X@VY>#lHp_J`LkYCK+b5pdCT^zp&7t|!UI0x{ z8vTgNC(<8VK8rIB0%`)6)%U5z>bk5gh>54=;Zg4S+Q$JU2AXxoARu?$Ow7D<_~D_) zpOd6^{OPe;!@`oZiqYCd={S^(K9Q^cW*@`ywN=OGi~Oq!F(G5e#lx{Vxg4v$#KC_6 zHyI^Mfj?OM=_IaRqlv!8<3aL%EJ!qtR*)1m7O^FamDafqpVEFkRi)5S_f}9t70Lt$ zUW{4KiR0Rn9s2AJ-vY(uj@*s{n)(}3kxYZ4iN$ApY5f|5Z*g74D z6xZ)@JM_huJ)EsRG~eXrJ9>dlxv`58NKx1bL)Ix{(hmVR0!F&?fVzKbQ@l>=UX>L`8-$F+;PC5?sYnGgC=`a1MKh2H>M4JnD?J zOOmT7m+K{zd8#Lgh(o$T!?kjbUP*{(_sdYk=60o`+I3R;m8#Ur)EHqnvz&h>%5@Yr z1a|zFT_=p|SyTgvF^D_Uk8~_o{ouTOP`N*^QyqW>Wl82&o$SUwwTS-$%(@=3&#Z?L zG=hUCFJ`n{Ubvm^q7oZ8s8dK1J6-q&ZqEpXI#LlWGIBj(-QszR1_gULhSn)k zO>|jnL?Hk)hSd;_G|_ReU2s?TM#$F^qv?HD&#vAEsaEht9;3Kw8U{X47vtx*lrRRJ z>m3MVh+6o#Ie@DL6(a+z5R4O2GDkcpR&4pg$ScWR>_#blY{8U%z{$0C4EyIyud=Ww z_hh{s21zK}&(Oc{Uo;p%p*C}Pyj^TPaG0bM4sgdaaWp{U2h|m{guovfJ^uD>1 zjsEr}fu3nB3FKL1vU6ccpa2gbCV%kEqL5?vxPLnNDZOtXD-;~{a3lj}4fxyb^l@GP zclF0M1l(iDM=R9Io}ed%wjAp=1yhs9)a$e>W*BIi&Ja~(OqK=oV|K_5Gaj~9UD}ce zpqec-n42+pv*Da9g)-F;_bl==MC=0S zw^ecRD=7wL#GhQZx-(NadBN{gY|&x|h!Xc4BL@TP)TbZbSzRShdu`<@DW^$xZRPy* zBSl|)2=DDYY(l)c5V!>>*Ouu+8RkIc1^Wy;3{TWiru0X5>uVQ=0qUGr&Lv1gSbu(YhK{T5yk{>^?5h+b!V7258kHdlx#e3X-HGEi8q0=And z{%MVM+kZm^a|;_KDE;W!bBv7-PD|j32(96FoC-N!Q|9D&g0$3~oyxwTw(PneG2JkR z+2SPg>v4}JMnEp?QXi}CTGpJJsKt{(G_s#KpEPhq)AF?g0p@?0$G!`%bFyI;D~G4} zwRm*gKvRJ#UNz~0jqttqBno?4Ljtn6A-u@}6H*TO^_HpjpqpO^4ym0yv06!R@P&C~ z{)tIwF)J-!FfTbp&Z2hZ1fxVPQ8>m_pD6m+!rT1hr%Mc|W#OWJQW|U02h@}abPk8J z#jrJX@$nx-U_wg$19Z2~C=wro>d43zU49pj)eHOE?;ge%l;~=uqp1qlHM!xHxfE!f z9?)7>&eza}G(4*AZ!_>lo;S5y2UOPO^`kP+jX?sN%I8?n!UjcjxLa+CyE;?ICI2` zdkZZVvq?xID<&FEu){u(nWWgN{#alwRZ32C(%d_`VCMuj6 zlUV4P{gh7TJ(cz*TKsm&h!_5ho?WHdHn?Oi`0MEnZkc@w}IYj?nqAPM0j=|=6BKDc0)9kruIQZ ziKh3Rw3$92bz=5nXdi!U1s`I&9ANtcvNQdI2M!RiY+1Ij-{He|FuY<{NKNmZT}rHS z0QqZRtJs$HRcoe0(_J#F7hqWF3_g3=zqw@?bFUjtEJMdPx zuJdSLeV8Vy4uNvN`KY$`!euT!yYeDJ3bS|q%zXi*C_2Wlb2xe$E%li=DKL;#L~~=< z5`a2gUOTF$peja$k(Y=V1$IigHgi`yoh$g0D7^~H)*0B&Hm9sd2 z;#HBY1GF#~_Ap$eIaf_3#0!;I5+!P<58wGyin7SZ2q4Gm4#{C%Ia3T7!P5$jR|SpZ zRl%Zq6O9REV*>il4)Grz!(rwSwXD#S%Win@Wc?Xv@f!M0FV{Ay*ordK-=M0XvRp=Zg~Sk{n+19Oa3v zOMPUO^zU~MqzLBMT^wwf*k{+I{vhYX-wX>51bg$i;xhn&$n5ya-o0h>+^T$e0)6pG z0zHL5$Vi;*wrp8fxM`v9CEO+#J86zwf46zM@-di|@4X(_HS*8nCTaM+g~`V8`*%ZJ zqbT>z6h!@r9QBobO+q;>W8B!A)gaMM=eYt(|Ly(Y{_*shrS}6SD{i)D;M%RbGI8x9 zJ86w9^^pvJ?@Kpj^u8_G_Yd6gHmbzwI7|12QSPd`ljf?s^A0`Xq$2l}WM1fQ6WQq zz)pOYws_N%iRmz>R(WY#22?d~fnB}FR_(VFtG}mGh%L&!u%deQF!hg^n$3X~B_Sfi z;b1|V9_bE4q0+(zYItn6Lq`oKM+cSxivD-+KKT(Vi-05f7j?BTH)fwKR5*g##T>3^ z)-eV5F{H@SJM7aXJXZ!*_vqQv?yHyK>bF4vPl?`JVz+;}yqcvuSf1<;gtps z%VAPKg+DJ)%WcYU{JSw=DstNz{iD0z8~9MpF!p~2Yi={ZNtA;(&dsrTadFHm|AN{= zb**2?3mf?sS6@fkX0UME+qNJy_`G_)zWwr9aE{m`$yF-)_ix{B?60#`fkY434M)cz zVUKN_tH_s!u08%34E{yQ1h4nZB0A2JOaXGJeb(|y^CWi|PpIppC@VzrjT#TI4C-)j zxV8ics->lnZq|ka9)JZXwQ{iZ4WG|ct;cM#J4fv7XKmPQ3zgv@?IYb=tHmqj2k_<5 zQ26~P%@tT97bXpoxSjj_b4Rzlov?MAXPtA#hPnj>WfQYE^l7BeXS1(=>p3A0V<(I!v|h8m0OZw%&^Qdk zh7{)qG6YFLZi7Y=h`4ZV#QLA8gqknWQQSf)k|-mVen&Po$3oGroj^~B zt_&H9Dk(3&j9Y+Q0qW@mq_L^njBRum*h~aH<_qM5+iP$R?1X#}OAqzX)+aQY(mv5>O$G z3=2X;geWLMR)i2@2q7dPBq8g5LdUs(-2408`^z7iT52;s<9(m!d7lSdO$q(h0~$~! z&(AdB!g=|DAPX%323;&@>VT*exP~ONzEu=%Ay}rQ%Jk8vWQk@CkR6ET+l7= zPDMYfiaJ4h(uZAYONL;pcVFz#UiD-7r|vOAdk*d0arYB{Bz>Ox{=FnDh1luhr@REd z+r3s4q9%Ah{lt4cRc@mF?j2v;eFk^H;(e2VN#R$#_ny_(QJvfBHiI(rHSy=Uh+##sNE2Oc(;9GuE>iy>;J{?G#ns%hN@SxTrHoP3~) zV1VGh9Hce-9G;dF-$K@as(y)NY1TF%946W42Wq?-n)}^W?aMSGFg$!LvJS)F*jFQ= z)kU%ogomFD)25If$AdC<)q7%iChG1aNE`dm9att8PLue=2syn?qTWWNVln_HU@@D% z{GF;jopZmf*S{6`^dvvbj9t1~d{{{cd+UTY{(UMZd zo*m%nlRRh_%7c0$GA#nxxGE_nqs5rlbGA|}d|G47Z_#BP#f|o?8kju17~APH6uMYQ z=qIfvY3~pxB8yN6b2*Sbrsb(>qS-h22isD<`LjVSkNSUN5T#52P$QcG#*mM!k=_Rf zxyU+dlf`_rVw7zWsu$NStzCL!`MHIge;c^`>*jxcz9HuCoBw|A;)=FQuOyciEwj(L zm@$uh=di<|i(orr+fPXUuRdR}!0CtYzI^cd*>^9In_j+YSgd8y?Dtz{?jPN?RlAOS zL8xsV+4%n9{?hxELGMRW55XJ!{M;bx5T!E&VwvE=>({@TtJwWZn5~7J$rT1wg{;c` znX5j_MBvANgA3o6h6KRCiAspn7xl>f;mf~XLxK@wBx_H}jdLDoGy*ffbYOqQ+v7;( zh$f?%7>{Mf6(FTIBeSm~9WzHO;6f1v?Kg|3RBFy_{~MKxvee`+Ei1HdCQajEV7wCn_0)HBuXi zB4ldqlEq!jurCNfatkr941F_8&1-?{pPG_6maedU#C&|FuaxXDagOp@HF zSJ32v>oF_h6hUP1PShjVWILi(iyLz39uPG*0-uvVrvvO-!_B zZl*|iPpQ+DR9-)2J``xz`|&$Qx%e?AU>v(Djn=KRk;`7eGy z{o_7oD|&UhC}jS5WHj*kxq$@sr{QqnwtL?~x9M~$a{sCYP&-J+bcTSnOC7Rg->T~u zDgTxzBr;84@iGV)$k}BW1F%zjpmq>SL>uMvWvO*D8%|apgbOcSng#8t>&ruG45snH z&|O$ZHXE!HEgrOhlXTqhEx2&DTlK%-Su$-{zZ(ih3){@o0pUz?$7yiaY44K{>La{pznz#r1QFNF}_78fy{FkFOrM@H3YeuU8zT zkc=YdJ`aosf^vsUr_leHdpa%5-Q#TZt7!<+nf7G)h_cHG&o-`xZey#ykELJc#!q^j z6TdD*N)eb822y&hAl0^*s<@6c)v^x4$wMV+=R~}`^GChfg515M&!G^T==L&=G1KPI zZFG6j_^PCJe&IHYs1cmA!=lOS%IZu%8hSw<+Y`+Rb!xbkHPD1p?v2pDwmYZT6gI)_ z^oMC(v(+ochNBMu^>)A$*Y7Ue```chzkFVP@gdT>l*=dli(sU5Wf1N%Qu^=NZeW|r zQlLFWd!TGCNLg}*z-A;PON5G?iTngw{giFXisqml%3|u#<*W%P;L1)3u@Qt zs*%z+TkUqWrLhYne3pX`dRY8wU|;F`pGlZFws2rb{H}-4XUxFWv^2D|@XEp8FX8El z4uP0JwTedVL9rsVb&uuoL@o!-6Z8NEY>%DZQ%Irk>GTG^qGP}|lY?V>KH~It8u)4w z(GjGvT<4!T1L6=D0zI|I5{<@*LLv*)_g!B1+v_`CdF%VahKRl2j zoodpqAJvQH!tyB>J2an<^A(#o9D#L$V@S8ZG}u{l48t+u@o1c{Ojd(Q$&m0#p==u3 zh3@glX-P1DNGdj@us`Y~Qm;n^pPeqMDk9QlX-s-le!{jr^sKBbXU)@wOaMZyfDTq1 zX?(7AAKCoZ7w>vMf<#j3&X<&|gzSu&H!Z|UzMo-7#z&!k4PyH>%9Uo`wC*DP>$^+K zg59tBoY*j!KJwipV*zME>JwMds#x}u%vTW(r?w`rQoqSQK{Y6KB9ML!aHI6tDH zn|Kp(syUMs1V6JF#=f)}`+C`%#dFtL-0ql&wdI_0_OtV*#*e3BBg`QyftWDC@4yKkSFC|1vZK1bM)~d&PGbysu;*68dpB(f|yBkjTN!h z1wR4ARblXiHK4lLY&|-GseW^+gmF?l4ni%Qp`4tY<|$zAiR=mrpo8 z%-oANmgDn;#oA0?JE2O#4wZB1Oa`54k24bT)n}LnmJ#Pg_v{ah7xFt_$xJA2MxXwT z@)QeC>sd;tqWLV-OQr$E<4L7*PFc@Zta}iTWuX?|(fSxs*kQdlgRDwL^A)y2u+Aq4 zUL2f$Tm#l>RMyC_XQhY3|?$51;TJk9b>hLCNg+(xlsO zN=*EhPnjE~&lv2SohBnIk3SW!wM}Ow1))w9A)}u;p$p!s)5kNnTG&riW-3^nUBud; zqBoYe_(e>Hzi%1;oLFaD(J+5FpoCP4l=gBzxa4ya{NZP5C$f;z?q(k(l-bsV+izOy zet7Tb3X_h}Fe#xek=)&X;fL1%F2#*B3_rH(DKqhLnOy2p|B@_5TqDbE!G_Vf+w&|NO>H;evtvr2`9q zZ{l)+x>n0PxCUys{wP@2!0jsl4)oLi47Iz}jeZ9$S-cpk*P7CAQLk*fw{E=F4biVq zJG52xqpRZET-xm0T!cXMm17hc+oj1!$i^xC7i2VmYIeQ4)ll+225iFh;7Tyo;C0?O z&g3LWup7cNXWH@tuxX)&kLfoV@i-JqC=0X^Mq#p5Dyi-!g)h5@j|M+zK6#R)PS=_9 z%^be5R<95AphtmgNY6^GD|naIPeU_N1V;p!`B;-iBNZ@MV>(Yxo1MOvpX3cXN&(Ny zw@Tw>Tgq2Uq^3BMDmm4dNX*V=bVMg7Cx1-2;0so7io24IE zD_^TIIk1zbhWH!pgi^^VM)lxjWFMZ@pqXTa_wOO{TObV9pdT6-(iRj;%BNn+8b*p( zq{<@mIgvudYife!GscCjn)@0<@06^iPl`minq2Z|YGjG=?s6@#wY{edu1$4!FTs z`c&UqMt~C;D}6zH`$K43Bk|pi{E+>Rt|1?HCC7W>HzZ%X-Rem{>F0Xq_Lsi}*DHOJ z^B{Qnb`h#ACTyWHL(eLSklA&C#on7&PpB4DCXU8e9TO26vm*$*LaP*Cwrm^S5%-0< zr!LFWQq|`&@6R5~`W6qryhitN z-lW?CLKuw%fSV`DxI}ZBK1tt3#Z~K52tr!{FVj#yRnN=hiX`co(o+ryP;WR$w2E;y zIw(jXJg84<77b{OxT)GQ0Um*4e>%lN2T81Z(R`t?!=Rq%KZr9ra71*82iBdh!1*bY zm_{$RO~qtJj>HG+5l?lcP&P>T6EDa+YB~&ou%xg~NtqzjlxeYDq2r~<8PPHQjW6MUP+oab82)){2(m|Cz zr1y3Az~Lps!=DdtouN~}S~NIhtJFG@=#|*IqQTzK_ei|BdT;QuWz9-Ip&-EzG$mN7Jr%(d3>ea}Jmq*QLZUNN1F*qC zqN{PxqXf&cK~2v`1pBc`0Wv@Sz>rcHT;`&hXzO^-cOhkGhx7;fuuvo~6oM+bgJS1f zSZIDw&q0!2?`<^J7`*lBW>EpvMRFEY`^W+U8aL&-kS7pm)UQPaXex>%mKF%PJ$H@O zA3l6IBmG!3r_@6=F^}CWwGsx;GJ%}dE zLlUeLXConkV!yT?;GL)#N}DAh`L)Ua-E-+A!SYA<3YNy0yCu^Mj;S@!6j1-Kk)9Ei zpvEF7zLVFG@@i2wl-|%3JtV5I?h+I}cTa<3JhTUGo7TTb%ZTOa(Yb5=GVU*#8kkp5 z&Z=BEWva7`lJC;Z&$iYZ{^fhsZfs%@b>DO6$!80jW>HcH(+8ID!Cw@M-8K@@%Q+TF zI=10;&>oM;Uuq-rxg(|ZPdt`Ex0iBJ6UxC_;mY4d}`{Z4`WAn z&!0xmpglDX<{$%5T5kUX39De#kgz%zU?dyR*KD=Y{a$W@ciz22V*4Rv`XO#=)>LL4&ub?x5r# zVO_XoT8Wk}u;EZpMx!S$>d2ArW-;E!I7^4*7;iHs&2)n-n{wt7iS$FOgRo?<8R=o5 zoWWqyP!2)v1hg%`#pvZ(pqAVa$Qo#+B4#BXZ7Wc|mhtf$D$^g`9nsQFi)W9=_dULJ{xmK zC@SLmydiCt%S*i?#2{x1pP#y53a4|Ytq*$24rGfSMQXb{zDOO{={?y|oCQq??ZefB zXClJ7$I$xCqtD3+Ekyww`(VpQ(AQ-y8PM%#uk|}1URxHjU3k}#h2tJr4LqaEx)#JJCv7au6*L58K zs(5ka+CoHS5$-d)E-XrCju*&C?iNvmQ{%Ao=2&UV z&s+Z70oLDLm;4WQ!T8I!3avB5$F(JGA+wj%jsjff0Py34+WExE^>gQpq}IWO2Ols0 zeNWt@_#K2NDky}V@IE(cj;Mf=ÎSiE@gb?YU8#8LoYA}qS*VV#3HkYxzXGZqyu2DV4oE&pp7ESAcfg%x;dgqe4H}b=@v(TaUf-!5Fp3T8q=XEgaJ6b6 zRhjKGb`2R0g2^FsN+!UM^lAbEU>7shR4NBcore`l2SpB8fepCs>yXi=DGgk>>&_-; zexvw})Cd^C&G0iooVyspv(<(+dP8|et8y~M8VU?9eIVF9Bd_HPS-2!sm?{})gbGlY zGCL!lo$M<)O3TJ%kT3{?ehMc9R={v!Nl6Kk9R{>ZCm4mqWOSocrYg|3s3tx?T{JxT z*5ob-0-o8Ku(pKdlw=yt(15~V=LnC5&1Jgie|Jl;F5S?t%$a$XY_W2NUPm=5FeH#_ zqWfiFm}Ggya~_}W&OZ*Urf%V~ooXeu>un>6Vc(e79kLZ&k}ZqCPaa=HD{WsO{K9ga zoG|(Dg0lBvqy8Tr=k)(<#29#{h%jC3sD*lBp!OaLbP45b4SnjUWtnScP6f9k+jMWO z-LBlOgb_0FU4VJzYc#M-Z5V$4crE(jzAx3EtroAfJ6whsUhw+SS85xKuENv(*-}a5 z{$m|qh}HL4ezE=U%0-Bgsq%5B7*F<1;=I-MiF)ZdY9l@XnWPtf0QN zHOX@)Dz7+0bWl+tp*x#S`m1u)eVe$U!KF|-NJ)qHt;$70;zX}YRKXQM++<~EFV|-z zChmbZtY4oqWT>cE2eliGETj|!o&s=d^eI0NSpd*{xDbea0^oNsm9Wgq4~H7>8pbDB z{T}p!{Geb4W(}0l-9gJ6N|TTztJSsYnuY{oe|f}Nq!eq>V4Wf0MIVx*-nMNUoOlf^ zQ-M-btKccs8nx<^9CKTiW%bF02m@+7LEs}#^|?fDky_{edWFu^FgT1SNbz)I1k!)M zJHWylG6T05Y5T$o|>yz#-{`_L8_DzE?k!FdE~G;(VQ?IUnKI+Bp@)MkfEPaQaK?+?3G zfBbi0uf%jS!UQMreLXJGI~t@P4HLxTI~|d6ac;9Duh;W$Aa?}?xk$vOvnb1umI)Sq z0NfAWE8ri%XXnC`h&R~T&&azhTPdg`0i*bp&_QNgczYbq%g5)=yV5bLL&m4OGYzeA zR01Bum6(G5TE~YE4UvT0;oB#WzNhj7&02$X(xJj)zk$afK{H7}uYQyB> zT`&;09Io52=ZZ*~IiL+p8C1ZLdzBYGgH9y1x}0lR5`pPeqZe)xP7SZ6$nPJGpy?li z+B1QON@mfN-1tipev>(PG;P7UUcA$30r@0f++$LYY%Rcy|I|Qup75>X2KKtL45zTs z?HRN~=(4mC<>W2Wv=IM~_jtOI_h9~C@AtoZvT*nBkV77(XwP-zf*-&A9a6LM+Mv*M ztJ?{`lWAa4)Ujho2n_GsLq^NHE1QX{sV&WVj8;=lXi;a;lO6}>ARTc-*Wp6z{%%$l z?TOy4+6?x4eap${I+08R5UgGbNbNkM@77y(#b@LorPh#GXGoELSB$^*rxt+c1CQfb z9YybB=D6CukJ(tmF!~X~FlFT%jA2Gp)jq#ri6ln@{GqnO$zaX`IpBrx6gqrC~28l??w!9(rVB|MZaA<-pr0Ef?1Cpb}9gqKVop-(=`K`i#iWXS7x+u+40xn>mLDLRiD?yR@`@S6nE!V~+Sx7>w z{x>ZMV#<-*YN=e~(Ya&RI8X>C0w;L%CzDvYB$(vsXf;-a+DnSTc{v3fTUJ6-A8pVp zen%QxW$t8|PRCKp#ClPC0APm+pSB`kNb--XY^nF#OqgJF8Bi~n-I z0)(XOY*PSJzYMYd`*^e}xmCg66x(&pfz+|f1A?c=QZ;8f8o-6`*{4302E1|3h#Pn~ zsc{0N-F$~DYk1XBKIs)CK?#TX2GCRk3JzEsV21MYXuWJ~Or$mXau7@RKMf4CN z@HxiI@z6h8#+9qVHHnpGMc<=y9Fm)7$PD3I2HBi-d^BKiTmqFO<)})X1xOS-Bz?*Qo?n zLfGTn`s%7%fe)xIBNdK6J>cERI`#2#3!gIZ$hIY0ru?voQr&+^z0p76!-sW`FpGvA z=aZ86RR_9{#)s77G&PI(+wj;GjR|RD&#^K@r^kzyN3i6)YeMGS;Tk#p|F!PA0 zACFt^i`KGtH>R$?AHVYRpH|nSvM@U@VAlB2i8uM{LY%Vr>kG-Nl+TZB@DmstT`$qT zT(Axo`9rPpT+B}Ho^Hg78uZ80kuM(RK8$yz1s~`hW#G-B-ZX6E%)lLEAw7oiW@jh! z*yxbJi1&7esX+8o)VE*d7KUv&l;9K?i430CS4ga8gckCPKiG<> zOEsQlH5WB$Kqm0M;YSqLL1shxSBdb7s8VF!bKd2mT+mm!ZB!EbRm*ix_b2RCaTL?N zOCSL9K1{s%>MqJ`wqzNuG*=r>c0SsLG_n4{Heb73lYUdSwJJ*UdQ0U;(UjQ6P9Ry( z&|SJXEqQRj(b;hcgId=-Y>QroT6JSEv!0-CiZrzEQl}Inrf@VySL}UtKDo8yWa@Ba z_4_kibgrQ%ZQCnSNDGNv+N2s5Dpk1FL=tCod<#yN+Hj}g_{RJ+A>I_1$ab6KS+$uR zva5gDEAh1mANDmngHrpMgc(?xb3^j$jkfi{qwyJ>0@>N~#L;Vtxrb#}SK*oWJ``m0 z8%g#nq7<2TInozG@{-&)^ys`{M2A$c+qaoNQcV|6x|__Ki}Fn-@n`br$t^UKA52pG z8df$3k8mH%Jh*$+lhh75nH#l7Sf#>VeQ|0_P256h+K&!ge`fwWyQ3t=W<2_fF3kIk zvle35!zRUreX@$aF{Ipo4aa3eK+;`v>H)}TdP*SY-o!#&YE#}-8$F*e*wn!Y=BWcLaT3q%Qif*+4=5F?3{3&6=Qg~ zZ`HK^;UN4BKxY0hu#lNcqcDG60h=(q$%1$g()WY_LY(1=b}DBdoZNXJ3WASCMdcXP z$CHL{Mftjbp}P~RPz?$x&4yMDut3X)WEZd^I|IIl2-4Q!)8Z?-+fJI+1d zOvKf;d<*o$5o@f@mV3+W|SF5Ig~*Vt#@rY`}OtWGryYE|#Iu z9_rNMD;@F2j%|ZCY}&LOsghwD4d_+JP$w{o>IT^Xp2s5A-lFbBC{(A0u5b5; zv(aW9(&QKaV!!02C~x3myRFy^ZaFI&#)r%njUb`8o-sqn)p zoeXQqIBaoM*wi|BnTL(n(WH33XkenNI^iBtI*+b9H(d+vqLI}U0ih3t7?O%4*O{el z0$V|kV^A^g$(AOcZLwU=Qc>l3iv(8-vlxQs(^yyY*c7X`KjH} z)7TWMS=g^I{OvT0u(&g5$z;D|il>RgVC&c|J>N35#rcW;?TS-|kqj?|9j;Gv5QB)5 z`fjwC{==Dv?lfoK;+-85%|mgXi)?9{M^V?01Ap#olw~gk;S`BzyEHs1RGyDT>Gb;AG zCLgD=wY0M0wPiI6i?eo44iA17=^q&So!pa^f#hg!^tLrdyef(brQf8ecdkmvCpot< zOI{?eX;*$!vpu=LIM9N==_m;=xF}u7TaY)1Dt$tK3Tw;#{l>a&_Sv}f)>ls~!n~gq zR>JErjIBU@Kjps4_i#WW+LVqiQY~B6f92*3$`|YEP=_%yo<@?GmoZ)|h38461RVID z9t1DFq)EenpBu`zKR{G%xYC8Wn(nt>s7YEoGJfU!m&mHwxk#2_xO83IV#=>7!6nn( zS5HFydeN8Ftw%DsZz>)({jKA&xbQ*E>7PHdx8wA!Dj=s4p|b+RR_~v1=|$fT*n&e( z`&+}(+rKG3W?`W=b%m^Y$i&lPo8PXcGJSCt^)K6Yzwd2O$C_b5%;@&KOzG^9Vr+nlxv=(V1!{gSk}GM+ z>#oQzR39s?2w%e!?s8!KICnk{@fVr!=DM^9dDIXW^;euw%KNouV? zrED^@Tey*WpIbF+vf@7n=l0r`|H--i>k1@f6>KDr|7H^a`vKLlKMXooBvcRxnr}C< z#b6+#jdzjK_Y7hsvIGE01D;_rt-;D7Zuu4cr6)q?QEwv<{nN%1697W;0~Q0q>7&IGg7-E;6k)M z2YnqW2SqECp-&f56b+ioJ$5_^kXu|s7J%~>svQEFtViK>K*Rx}TK@$lklmh2KXv`&^=Mcoq3A91mph3&w;Lb~mjdjJj1x5*0<1%~z@+_C&0N@U| zAYgA708b+dpwkT+th9o+?E+xPhIAS}Mc@i`O~A9o8ae6=Srf~045zB~Nh)<(I;MYG z=a>`Z=hbWj(?54 z0jL20yY)2XS5UikjO}O3G@3XChNxFTwMJD-j&Tg!lvan%LY5Qg8)vHjahg}gB8`a4 zGgc|$hc$KlG~Zj=v_Ky@n3lKBSQTIeBe#4MSjyFu2`>k@)>Bs5q z%Po$tNPKOEIc;6MHvOv^&)Z+|77vL^RmuV@vPu{ed7SLNagItkJ9CC>nf~ohXvWnB zLQ93WBC`wE`)02L79kU-uF3asB41Bgz|&SW=oIZog?2t85$1DC?i@M$jr#_tCT5Af zd#J6bMAL7Xmv>J&C12}=r>8C|#+!~GNBd8?L+$ep-f@OJvMwZ~v!B-MYU3()!W;Hv zsmU?&Mf|S=@A)_~d}<1%!hQMVJ=yBdBuA#MHY&+PH%&Qdxznb%SJi(X%&1$+TefiE zTr)O&N%sd|nte87*NfFnvXm{}uRH6Lh9C!`}Cb6e#6TO8La++!ktW4 z_15FRI4ZKErZ4%Ko72j3Ee2`7!_kvZ4E=FJE>$;Wx|uyD5sOZdO_=fHkfo^e@sJqx zNTB+vJ!{UEO88yCk828C+9?{!%oV(hLVDIy%qM|%u$|&VqS?LtC4qKH@@-bO`s<3m zC9<{`iRF7=EA|)tW9?+l@E}SOlTQYRHZFS88gyKUT611$K5%Aa*$drDlm9(qFIzLn03jRkQxLq>5v z>|^;+4gyEa+bgE`sjmP)6dr!gzr1S_i)3cGY#O20y-;^=qv>73?{ao^iYvaVir zHdSEU9YQFI&rC!Qsrw3!M{35&@7=2oxK!EqB^ueDWrEdr!%sA-+$^+j>Zq-4c;n*l zOiriuzMF(JlHhMWp(JXLvf{;57kA!`6Oivad81YtW8?&)U$190shvt!t&24HmWIu4 z!MGHLaNq9|-iiDSq|>~`XPE1Shh8-+HnBq~Y2H!;rtXuAoYB%+FHER44YAAjJwysV zbmAUp-fqL6%o0fLD;z1BGs5kWT!y3J9|UInKe&h)J@Nkt1s|hNG<~(%DF_`K-)KG- zeb;ZcW;op!uRWehT-Aw)wlF{GeO&L-d$JWa;<==0Ys=F`H%RnWQ5IR5N#jbqVlCye zuX^l2*ZUje@}Jw}zy1AB-4!a)7q~y=Ai?LkV_HU^+cID=j#)2~X$I0Ng*Oe0RO<0h z*Cba!S&xcRpn-Ka5~7AB0xK~PP?Sora1_Bn%RAiw>?r|)Eo*ZzjVnz6M=?OzT<~CR zhLc~fSkf0M&@SwNt@&{P@bTbD_A@fTxR}-AG%>$H**-08Av+}QXveVht?_|_@oKdi zE=-Yi0<6Sp&!;&Hs$zBfz>xHl+Rr2na_J#pt?2X!HH4$eg?zyXg5Nj?upg_T3@`&o z?G;X=Ikl7BCvsBXDto8vX`xvzgk>9+WS!CcTO^Z=nQ+7>EWc28MkO-ZDuj9_BpSf~ z&pgQturfi^p{c;_yv%8N&rDm-e>O)2^1&q;ci49ZU}Qn!tCcg=bsg_z*<|%* zYqwMY0QQ**0nZ0J3fLyN&`J!$g}JLkY+XG9i>D#*Pm5@((lx6A6d z=o~<%7t>`z6xW*_Mga_xtCtxN`GI4?fs8&L7l2ziNcmH~Uvokm-4Sc-If(dVS`H1MD{#vMs4h{GG7QFe zylCkk05VZc(BXWr5=}i|-CzcRqcB)AmyyyMU~}|5qeIXvqM}K~Bn%pd11OiX*hOLE zY4KqTO=D?*;IbmV`X)T5g(<7Bx}L#>eEjfw$fzbx0DYuB)n^h(B2XLN5Wga z2sPFq+G5l`z&l*CU?%e=DDn?8PjA15N8Q5ZpW&_B37}~n&NAG3Cr7=AI%5X+(8=@4 zs(Y73_e9C36Wz1XWfwBKpWg6KIJmX$3lYOZ+h9Ig>52HLx>EmHWKP0j{^*5-j@!2` zf1A~GVOgi4_<}OwVPnwIeS@B?Z5M^L{?+}*(fJEYX35ghu`B^SrJ1)j&Y{bAaf9 zebdS*zo$xaz~eoS{R(vwCE?js1a_qOrv$H%>uXxhite*dlG$ReJX{NdWs#_i-9CH3oHKi!C~o*7_W1CDSX1W*vUcE4VXz5 zzkKOwXV=4y8x=n22L8UTq{q7w2*`!+4z?rOcZLNwwW~6*4I9&MDHubBfCDLEUtzH6 zN<{Z5zx1t7%eg*3Tk`6&?A>w^MvnTlK78Qqdt#$Do5jr|McwP81x+z#t34t!LspU1 zN0RREc^xulp6nxJeTa(lo8Xrn#r>jjurpX4sY@C9@gtk6NWWAkc`pUXN{w@K%>=ZdsHGqxD(Wp)bI`L+qqUYr#Ym+lu6nn5J^ znEUrpc+7$}w)uH>c!@*H=E}SqLv9@`Wpr#{I*J9=IpVU-{)pL zK4n(vGTUR*+~mxKallBO`^&*KO7H0kDqw}B>=yT|Pb}-YP!jwo^m;5}lFi)ys@i$R z^Lw*N3;(l0{+kQr@HhLXrC=Yg-?IuxTBz<+q#TTGz=bjf7XyPS39Gy$1fBqXkK5p6 z;QPM=2InZ_%m5YKFJHP}{;q~5s344J4b5s0iENz=x;`q5q7z_aGVm)}ZeXTyELK!# z4?~axriF3~S)K$y;_!FCZC(-wsC{K|{&!F};FfaKD%C(TNKb7}DSyWA)7+vG`x{<0 zTPH(@ynqxEY0-Hpz&Q%!EE4&+BaPZWmTm;Ml7GE+^4L?rDOJVZLQ1rT;XO7%QCH(>dTNl z1|rps#urwl{R{2yEC^@;{Iip4RD`zHR-igvz8p@pMq`4$ZsM*TG!6U=c?Y`l6d->B z9JAvMcHq;rc0K7FI?%*8;RpkUmvr951Q0(Tb0;p90totNtjAUljjAY zc27$^oed@>U1o_NWSsnt^grPs7H z0$hhMG927n(3B4g+?>t^I>2R|rF7wT%$NgO^7PiNTZKk%Vw=!bdwQnrH6&n66-z>$ zRLRNddXr&FZ)V}IQYZkfLYtDOMG&xUHkL@lj1gEOvDnIh{?73il5T1(aiISz#19P& zYiJ!kN@cAgNQ8z<;JXHfdIL_w5ZF#|^^;8KT7 zePG>ywFwMhK!Xj8)V_LTr`G;RmQCt@zvoN}5*Hb5#TrMVGpiM7WB*^<;k3}%Vv<=W zEdUYGfg1bG;2Zc6yqJfD10nzfP+Xm&GJ#teQd7lSmY$y{)@1ng|7}M3$vpdnu=thP z?qwGej>}^nM9KO*v#aNJM#bF4(2j#YTc6;0P_J5x*nBU-q0&2{ck*?s^7Q@$->dx2 z9S-Ewk&uRV!OBMcPwY3A5cGw;vqUepi$K;vvnR+~0aeiwp@CRULF0FOmMvJe@a%_0 zcXGrzc#k1WI(vXh&R zmapY**7P(Cr$jgql@n{}hqnzFFQqzY7coMvyggpmCvVipzQ0DxO?1ZXT&0aPpwj(B zTQb8ZJzJjnx>NJg&A}ybV==0nt~^c=f~B9zX7^qW-4!@zWtZO5a%6lG zqy3G)`tolrhE>;oEW380Z#!xHv)P6#HWWpD0zPeEH@HY7H~A8^(C*=We&$#5?yk^u zw37914<3C}QK+6%n)~LPHp%V`&clX~G(#X-dqRz)H4Jw!4;9?Zj#O5zVNr5dR}KN? z{>F0Q@Hh%v39M!3bY(0Pzrnil>^m*q7BIiacdJ_-tGKUcamP9HI_)wwKpJ5fRTt{K zmr4lkc7XvIpXF6TX7wqxCNB;qEyF7;J}txG<<$#rnRT;-T6ww;UdT}J1iz;CC@w}Y;+}tI8eEmz_uE0RsNA!Fwf#_r>459UN${KuBFP8yb(bT(Q zIQgaQ9ss&Rre{uuJcaZQ5PUTCZIF=Vpz~9yKFSQ>SBW99HWWiQ0VWwsVkLaC-& z^zk6BdS=UADsIZ#3}gn8iCD5z1`ry7+cz=MSK>}5O|xD7syrrAOJyFK;p7wMUgPvq zmL!P8WlA!J>x3pCG6(e1juA9MlHo~;PcY>%gFy3n5h-O@rWnr9>flE^2E|1U3BG^i z<$>jnnj1#Yyq{M0`GX`9d1l`CQn_&y}xY{n*HrpCpums%${+SrNj5@pDg5gh1BB_ ztxdD#UpFyjB*J$zd!8`0!85ish}_z;h0A@PB|3NI>;4P-6MB_*X!d-Hn&h2;*EI@f zlhKy)^ho6s`60OGsLEMpj4LWwrCdcxA+izk8gT{5|4f)5<8@N#$Tth6n z`cw6)OE=r@tqULfq6HJ?-l!Z1IhSBDMLEuQpYQ&O_cc94tjsl7|TgoYLuWjR8loW%CuKtU@VpBNu*HJ?N?1vxE_$t?bkG7}p>gZ=nRCX~$*Smd|H|$X=X& zwd}*}&9U*TlpWCfX9brt8%Kj;`I{(ZS?(_d7sRjo*4^WMg}oxahv?A~M?)4ojc&*g zUST>t`z$yZI~ij~G`|@STQ||JS>tFqJg|tS&m6M4l!}EFj+Z5wv?7V@#+0}VF^T_e!sqW<{*i__i-AVJ#ir`azV0=1 zC3hzyr`jsvslp-cinRGhrh-dp?)hy1OF?0C^#NTOS&6PZ`87L+sTsh|Uf8K1?%`s^ z?4$e^yTwGliM9d)l{EVZ@o`he-KG4!aD3ft^;DnTG4uGQ_ri+BXwhto)5wRw2SM9o zqPO;qz95o(jc%H3i=^6MiB;ZwKgvHQ-}{=b(K*@YRP|q|1K2u`}l485NJ2x!mW0`fYm-{E?3l%;RpwXM!Uhzx(lhMn;tVLw9 ztzv#_b)TGw8-MclIOtWpWv<|CB#zI7+I{+!*@X_ZMo(67>5t3te}7LUj$wv1{Xp_M z4{E2r*EC>uFz`K zBJz7I$2{Fu0D7PTDFvw|JK?xM8=j@ACvK_E=s**RW!XTVsky|g!<^Ff6TvV51m~?e zl$S37ZVkgaUCodtkjQO4wiljw;o_HY*e@-;if95i!aUmexij)QhDhdvrls}cnw86z zErYxfgo(G)*V$LDLBodp{CtN{?y&f0AUJ~y&)RzLfRh1g=FHRM`hjt6Gr^PC#`Ei! zju~3$R7#qk8W$+B$_@Xp0I>=Z4C6{bwqr1gbg2OhqOnwROj^2|J;!3z24lU=ZCC&o z6`1wJVy#v4?I`4m28hqW80hzFxnBL?8+c5oD0qByP+B~{<2ONqG2f2Bp|r@*WUJ0$ zx$;#ac?1Wbux=(=3g*lhe+lz5Lo1H__@qyh83$S>cx)*-j}rL|AGU7B z$B0$+>CBSQtZe(cn79>J19}{oOj!g=o}Hl64~Zj)*-hIz#vdm1Z)JYdfVlJ-S(!Xl zXYq)ms8eq%jTO59RoF_-ute=muw*9~53PQ!eS3(KEtn-5d$#VGeayK8)ky4D^7@*n zb1keDuX%OFPGcu{#MteZZ!H?Q9~@7r|3t7c`kHgCx{;c%s^K&Jmo8`5vNd?;3LQ9q9#0 z$Q3?w-z8Cdi<6=YTYb;KCYDo<@{(EPWr65 zMmL`RC64bNxdFZ&<66MBSS<-T!<|JJ4%P+O>=3bZbY_`?&jY?{)U?tBiXUcQ*X& z?e-H9W4kFFzIxYZp~>(EYVLizjqQ_4O;M}owY;tew!^FZ^Bj40po#piW&isxop)Aj znDg%=OA&wpsUl`c=~Mz-aT(f|b7l^-dyPJ{@7*aKXm{Fis2Z$(mqPE`uhxK(Ro3h_ zz9t`Ev|Sepw=;$I6}ZoZirIpFmW$@4Euu|CkaeM0c@wg4<7b1RYH2}l6-R)M=O1$# zM9nU~p=zL>UJX_IME1^uqImY4uCVlbrUwk7en$Dxr&Qo5G7oyvG}IrRoHlQ<5d&k-L)l#BOV*RTHY;*4dy%Yg zrhk%J`Q)}Q$}3VEI8L8%RsIn@CC^N@+xp(RX*CR{2Q68$WUk;m?k~wj`#X2;+`!_6GuMKi zsEaY3uG>kWca?-tKlOY1F(#@;k>4X1^YkPa5LPfL2cZ6~(9Le8@yuN$cLPQJ#t%*o zaR2B6#~{6?Ksbti0ymzza@)D-A6D?Piw%#7EDv(0SsgcCU8AT|pnx<)R-6VO^E3SRu?2q^ouI}@njv+5+SpMgA8RQIXPtW?(`D+FvfR*sZoVd{l zJ@6p(lEH|lBD?*0)=7d`dc?!D^%hPGeyy|092k!5WSScy@m|G}p!E?*n4n3vxUAO? zf3axXJ-~(d?R~vqGlfb{>Jo{KD>El0fzj=xDXc+6%!KA|w)WRD{#pl)%rt}{eQUo6 zNkUGl#7})KIc`t7EBQZPhI9(rv_0Kk?*8r0!@^ZKLz=f!iSZ4%^ zTQTHNxk53-RMF8vd1d~LA+Pj6hCx+3bBj6B8g8=|nly;?#N)=loc>UCtBE?KDS19$ zoL9=Cbm~*E7%KLA8`;`R`cs!)r2egmRIE9gGKGGM-%=oXq7+_JT1)$s(H-z8)u3R% zwLy1c?BBYPS8*yaG8n0jtY|g+{Wb?HI|v$?jk;#?j6Pv~+LUI$&0FCm)4cQN-$kNY zD=y_xHZjq!>y{6q(RIwCerSTZCLinUE|gXbnguk=Hf|g?_};7#AArqlhWg3Q@P{v- zG4l!81gbmFKhQm?u%BN)YQp#=--05)u zO;`|s8vQ$>)KzAmioC+jnouq)(uwr%;f$i@e^NUn-5He!Q@W}Kdcu&ro0F|O=sD`w zf*bVh+>cM2@CH!%xL2y@?h{kdUb)KYZwN_@zHG;FL1F}*#`2As1GPrA!_r8WUR5WU zYfoJ$kZ|c2FfSu|T{jgke&NbiQFn8kvQ-mzg|3$8;!&;G*cbHdk+%ajM@}(Wyd4;& zgOz6mJLB={@0V&@ozwA?%K5>J#kw>tQj8G07hE<8Rhyhb=9R?n@*Ts?`amxV!DS zSLJ)tO8olZYR6ZTlp|h_8be(6x`n>@jd{i%HDf0mR53oi$?S#KLSFLQs{R=Nu;HNA z>?oU_@YAeyYCD;I#rm$^R!z+QCbe^5BkY~4fnnCT+k`Z8oTGF3DrU?7g97Z8 zG{&Ij(F`Rp)}UUP@|lyFEFLYRc>tK`G`hSe^Zx)V{Du**_`S6rGYMQwIuO_gBP9=F zqG!ztw|I&q10ZpKt)boKNc4rUYxB4Q3d-IU~B(@7e!!mLL3-79ohB{4n3z`3VLGzqRmmCR^?;wNp?@T z`D#If$wFvG5Q1MjVyrytVEqE%-r$_iVd}g#Ul9&LF;eo}UHz$vYPTRaV+-#Ekd3$* zWDU>eQ`UckC5IIReABsRkP`gbLA-PTs&vU?)j`{ z>`SDRmu&N2%a~^sAJf)(^Oljb6IqOq&ivQe_+JaPe~e@5_$aRAY=O@`R)nzF!QZ_`FCKv&H+76 zF@veFB3sH*f;f`QgUCK>qK7)6%I-h~%d%wg;$T`0Td1@P^+U*CuQ*?s=2k|I#2$7qdWdxmgSpL7qsi_M5^oyn^+ zk5EgRggK`62y7?>~UgtSo9{V zAM_%)Oqh3fyvwt5;+f2KV(Lq2L9a7qz~-2PHb;_wq!aOoinL)Xw55rfl~BGxE=xKM z&vVEAZJg%jY_k*#JG$m*aeJv|+lt!c(L>7W(q}onTl1hTxdKdU==WQ~w6};U`Y+vW zS8ghzBv>olM8~oi;(2Mv_2_kHs=|qZJxhjs!fV>iduXY*P=x(LLSE_#>2$`q_m(}f z0m&+v^p#-f-6a|0mnNZ_Q{J77q<>er`a44Xk%+1#(9WhI---Dx^Q-7DZ|5Ty4diG1 zt$CpLuru%2rAy?$ud~T+4UFSV&r*YJG8=zgVtCnk45I%|WYnkWypzRZ{ra=}(>;P; z;!SQSZN8aBM5OV#*NV$L*q?@ElJ9rc#bW+Gs=UNXNZV*oHgW@tU)jz66L%Y~Tx)$l zMcK>gk-Uxnazg@#@3^pF$zE~8dc@ABc(Y_~RmdyY>P0Nln}+B@3Jmf)B5;AKFVj*Z zoGa71-VHu?#`h!^M7{e#9K!UYF;|Hf)NL+T2p?`q-j2vaVX9Y>u%;ESGCC!H^HDFA z@YWww?1%0O>Y7p#rg-;cx|#OM57Nh~0mW4=eMHpz^66d;T_n#{+oqN&)u&#MsEq_x z5)B`t$Rokl(CxF+=GWHXRe1FWs}{ zNV#j(>hgt(f;wJCuS0C!>BDm(A4NI5jI_b_3!lrE?n#b3=Ltt_i@a~addVbGZXzu& zMU)lY?ym?k4?*xY2Jihg{*r#VlJkB=Zg94bi;q_QHh8LH`WkBvWFa3egCIWZ!{Xrx zk5f~u(5Kbf-EwN&93<)Kt>}Qhsq%p4Rqvs45+IU@Q@;p-juBX2LWhu@=TAIHM) z+`=2bNvlx4!d{sv4^#<+{`T+uXdT;UVr;bP^%CCV=w9wT3TI4Fwe3o-;a|%xm~YZd z{CuX#GbB5Y1Pw>N#>u$CJ5ptz8(nC&2uxzkfI&$#*!V1Jt#G}qiMV+2VxlT6)<@$8 z4!)&-(l^aM=0DHR|7Ww_|L1A!hqwM*_t&z)(MiN(hPEJGt3&M#)`e)Iq>%r@VQo>x zvOn$y&KH2#CNPT~eb;X%*d+)r@hpj>omlW)qyb#zX{cJlLK;AuQ=9fzp37~wLKcMa zUB?qfNnP}5?hi{MGtJrnnuXhWsC_1|$-|DlXQwwkgR$$y7YbGtr*WOf-9_6Zz^45R zv~%aqWt|m3#+Jtu0}J?J{NO$F`RcyQyKlutIhJM?sE6fm4uWEqe}*!;ZRTik7bE zF>A^BtbAhedX&mY`P3a(fGctYjG?*IvjOKBKaA<{Yg;zN7^g4f3lQRn>Jv za@u;w3>LYcxw1+%e40oW=v9|v^0qz?0MP<^jIcSs9Bma$*FMod(iHBaHDBUGLJf;BI74NO$D_CT8= zMe@q4R^vC%3-4k!yG_(y+51N4^nz(s5={OeCoP1$I=sS?^79lQkrgtdDjDal7opL| zHMg5(NnbTl@Y}n449;wR>k7+zjH_8Pg?;+^{0OBr|UJLxi0#Q8}-h z(Ru7*Lus_ZqjJX`7f<3AwvgJ*#fuM;L%x@YMY~b<9q+imM_9O114$Y?XKM%ZCxH{{ zj0$XU4B}iZcI|r#@m#`hwL`z$>ytU`08ZG^z>3zFjAR~Cr1GKf8jyt95}r20x5o}e zqqloG3N;!KYfY(GefiAC-u5hE8QSsHi>(+jKVN$8@v;4#F2}mKkGEUd_g<`W_YJ)C zcYztS(;WNgMQWjdzje3lcl9dwYolpdu0_8W9Uv6T=0fSBAr1xhQpmkXO&#Yh)Wwc_ z(niHOqW11W9ynYNGe7hswc1mT>s#Xvuk8ugs8FO1K`(|II$D z$(v@DT8yQ8#@s~8G?wZwOzjs!p84paGIAN^yj@Rk&$Z(YEg*e~pzo=&_Q$2K+p$)yEDUaewW%1EN(%&KxHYc{mNIq%|4UH_Z+9_e%O)n)Dp**8X8Bw18V?n{5 zNi{dBMTHp@F5Y@DX5cXziyK2t2hg}gSnGp=|Q#21RNB&=)CX2e}{tH0bB3purFnr9~)^;JZ z*(mXsH_QGAnioyfg$nJ;% zKVfhIj4pqZ`dVIC=|TCyy_LtubS3ZEg84lA*_@75Zb2}vNwRq(YV`ZNn^npNHwTKE zp}8-4!XN=7PmL{OA&HIGs^k5G5M0!9LXnNjumniO=0o|h@~M3xQQ8TLa6Y7B;nl`a z|C7@$3#WILDr1R~OEF-X2Z|ZB3h?0o5Omm>&-aZHw6~enNqDxTK>Mf+Cq>F1HNJZs zRg<-j{D#;U7NuSe9as0v60r*vF92l;K|u8pAKpyR= zUz2E$oy?*)95!ZhR+&49>I`pc3aC}gutqY{v6D2e_lyr-(`!T0;+gdd zM4LS9gX<4am2eFC@th6>=1=g~t9S@K$QwxaZu%0EljqA%Fu9?fny#WiJ*G^|DNuKN zcvOTAZIV0`r0uzO&MzbCrl$vEJfw~ z?8w22{Rri9hEsv#eQt8+QijH=)VhF31R8bF-OO|_gz|w+RQ7-~SpHC9`QLzQpDJbz zYaU8{qRRI^J;5hNPy+4sMKr9(lMs?0rvc3{acD4N1-KSV32I?v1_OXG1kfrY%G7lB zfB`|X-4DMFEGkGeWJDAjcr&9xx4SMBTs|-#YU3X#*bFX!szE~hQa_A(<$3Q0mAi|H z3oVg?h^jkjSOnnwIy;)iM|mh^GCRjTrq-I zEP`J7?xi3?dVZGf6-PwfV=Lc0bz5B$yC~-!3|72oiO-C`J-IU7kZtW|wugFoNw;L+ z&XD~%(S5L7)1`DE!(x@6Dt}0?EYTuTJ8y?8C)RIH@={Ozn(LR#4INX^;%=6pgIYyW z$FW7U1<<4_xh2LEwMet&Q{y-jb@adCY*2DWT=k?*Oy!q`W}5`&kvH0J#ws! zM^`X)Af-OrhQdqJo^PluMXYanfm}14*L4n#ss4D6En=y>)?L?pG5ir6-R52H_TO{PP<5iO|w*dxzf6OtXq znK|Wf1H$R6ttjhdYi{Xt2covg)menJ=I)}c)>JlkTntEUw6(H)4B1R8IZC5n{m$sx zH!yl>`O52SJ2$Qxc@9tR_&{&cjZw^$*r3*xMa;CQj82n==d4|NuWN{sd&v!s5#=YP ziQ;`MTSAsTUAkL*0AUA`5J;%1M`3eJT<^vpnUDKM_eK(@cdbL&$}f$zk!hXH@-sA# z=~xOcpD`@081m{$F2}8XNBHz)ME)(vz}hL;ydQOHg6Gp6Eprl_r@K(fI@`$8;d$nQ z3YNKe)^cwo_|0h&p+i%?X^*{LKv} z^Uu53J~uFGqG`K?*=E^3YAR@#(u}L6#H(Y<<4ZX9&uDE|4&AiCv>qrk&S4YIHBGm* zY&mC9q-GEtrk*o3^QqW*sf%SRt>69$aBO99FY?URc=+|-b-gF$CbgjItb9H+DJKBY z(d9rmY&u-v(oa7%83WT0KiZ8;D~)ET$#mXkC*?^w<@jx(xr*vU>61ji!G^;&| zCge{OdifDmI3q;KRW#i<(&_ZGrT>gRcLlC=Va>F%@0dzmFcWi7BpU-dbR9nowieh2 z7x*bC=NT}(1K29)JMb#E%uZ0p4&ar*oHFORrq3UtrACPct2qo=3G+`dW)$4*ecCs*SP$!29A+z-s7K+b9Hm->p9$ zLgYbl8;(GbuC6W=2Ljjoap|tFPb_G?LI#Qd4`GB0yuZ}0)YoQTkwx0*3{Bc-m$rZ? zX`*xFtw#&omkd5Xm%Gv>9czNBnxz9BNrr4UZS>pTiwHQt6eteZeuBTlX&_x?!Dsz@ zj3YtWFNmn=y1Kd0E6?*TL5?wn{BGRXx!Jm- z4XnxLK)|}B0rCML$={SOGZAhV0*SdU)ckVKHc(`%JG83oT6p*hXy@L&9c{sN+?zup zIJwC51A^bcZ%&mAgrQ<=%($B2uyGiixxS5o&L_d^1#f!>dFUUg{hdq4jL$n%Yc5a} z_4+@YS`Gc98RGiR@^#SSy>|P$e^arUW%{nCQ%L~C94XTGBxM<&*H(P=N z#mxq0?Z-4G3H)&d-rT7CbGh@lDN}l1aLf46m@ z)?prG0t`<3y$$)6fE$J1L_7b-MReEn-b|#GCa;xqC7Q%PJ44X|+5ZwhMWm0$;xmU` z-Z%7@q053se~fx!+h5DrsZpk0Rq#~Ctc?{w9LX3F3D!RIH2ak6|LT+s%<$~K%{IJ3 zakE!n z9LqaE^LB{t$5@Ytjk^z5?oHACZ0_)yJ=6nJjihdG#E*-~RsqT=&yYj^a7w=jvcY9mtx+2!Gzj^`W=fje^KFOtN zUs33(W#N?jbX(TYkl?_#ZYzgjenV~_EgS^v)%b=uYfms_&MSQgYc~li`jDpvH{jRwEZanF+Kpu#k)4hy-$`Q*P5j+-B|m5ajozYZxt z5U1KwT2$&I&m0_;Td=NFq_eG;zxAe0%Yzg=H@=iKqLQgC0r_W}*v*w(SD+euUw9iQ zcgH;2qo2ZdfVo|xY+_jmoI9rP%PZ)NuH382T|9!jc0;h1rNEUqxsh+tS1l5p>5{xp zn~M&(T6J@S^<>`!-uFH zuVBtj&E%bc&xP^}uju;A=eH$Tdslr)h#%$N`Y~R26}FE%nu*50=JPAJ-#xvGcTAa5 zRpygs_ZdxIOKe*n7nADg>6KDQ(7PENOgip-kgV4DN=I1U6^(V(HU^_!11dh=^JyqZlpve&E-? zo-gdN^mOTU{n2dM6I**7ievgr&MRV=WgGoYGxikXJ`abqHwNJmTxE|MyA|)|YNhRS zDjRC!7Si#T73x2h5tEL0iUaVLC%1dSQzlry6JYkon?j^tP4KP5gbI%9?_$GjBIa*b zU5abk?SDx<(fBg*`XpB7XS!JCmAXRfyMoRdz}&LL%bTD$pbP&pcvb7T#W!S9OwyV_ zH-P^MfKF6%yx4@V-dy^}7wLh%3STWxI<%oI4 zinsnW6NCdna)%rLHQ7oFmU95sK(-rH4KiDe9GEH(&a5yWiknSlgW~EF!TBGwX9)W% zfOKC6GK-zu5A7@Q)So(UZ!=*4GLE5YAW2tLlkG8a9T#5I$?GU!4<6nl|30;^Wtjijly zt5+|a-csssl~mRlBC^ReWBxPz&T^exi_(w_%DSdvgK^95giZ~Ed)A@so@9L~;xB0o z=oEmmIN9GV#(_^7iZiB7fL7Rs?0LIP9=bA_A9Iojp?h3hT?c2vJ#4$9bfac#Y!M} zhjvmeaMJOvU;bW3vGF#D08x;0pl237i~KOA=G>!sYljq`Q*4gL2%0P{qXJb5Acp5a zeUiM$d7?miM6n73T?!WZ+fS?L^7>v%FM2)Xw_PmhxzeUzMNZwp#@PJ8 zX7mR3U1#7Gg{EFKGmC~>Hncq#`32u5mGhWoHbIO`QV4^$W0KdRLDYjj@N+Tp9U*aK z<^w9`Ym`h81Tvi*5h z-YXkhl*{L(G@T}?vYHrdWLD30Rez!|(pIj1lVth8lU8Pe0v`obLz5Vl@P?c%6P-R7 zct`-7LfQ6BD#A?5$SF>r)xY>Mf3Ol+=Bm)>OQt7G^`}?NW6A3H-^^smQ(-Xk%jJC? zR9w-aDmacn&Or?EDkgy&$U2`DMRots% zx9@Rm^bXGTK@QW0O`$lXwh8gFLZn_>wDn*J?$>F96=9l$t^^O}M|BSWd=6hC{F8sz z2R2L_JKLvT43-ZtyFDzYRK^{2Cn-1~$1is)4$l6p9Yd{k6UNG6x)5Axy(#+^-~aXm z;cChurTJd2ncSVhsQFaHSpvDUMD_t?2PELh<9NmEO35|P(&&R_{W32ap}&OJDcMHz zABu(tY;xXCdh-W4<7MmQ>8z#P;4-@oa_vgKupm#sLZ+kvVu#)xRavlm7~LaqEA5#^ zNJP(C!>D1}%8oYU`>)MPr;fARs+f;-H!}B%V`-!OE8)bTfv^R=W0#r)smN|YTVE93 z%i5?qi{CE9air4}+Lib-S3Dh_hp)M*Stqlx%D#QW-gam~kL5d7jCiSJI+#BBo}BZm(o>EL?ng%1B~pCLTUFmAivjo2NxiPrW8BXo#2+V>MR~K21n6G6i0C0t%5F!& z4-X3CpDZ%U_Le3Al>pF9y+Lp$ckb!o5Ec6b>r}KaQc{!>_qI}KMcnPE{dSh+smA&~ zHo}r_u<(Q}N_bbY)D>tkI!=gXThDSxo+B?|mC8J&4g}Mr(SbH@1Dx)>%?vG%G>h%? zvUq{Gx;seYw%G%Ikd&|Ax(6}+$=EY}s){E#&x$NllVj<2w7%i91V>Fd)yeY;>T9Dp zsdy&pcKsGX)r4fH_QZRIGU=W^ym$M&f$xhA3S^iVA*H(ZvQ~4{P3PZvD@uBgwL_vz z_E6#8b0S$YsvTuuOE>c)8yq zmN&3c4cn*?Z?wG}wJG$fD+mmggkc497!1J?;S?We-?IQH4nsBT?0xUJ4RN^|-fXTj;E+;XP2?Y{ls*n(0?CDcPC@)8xbJ zOqEZ$gKtw@=0MFvNBjJu0-_Zj&Qv}%-(DXKu9kskEHy1GuHjNK$?8SXx?%C%0K z9GaP-1i^qSz9h18vQa(DlrXq)HbG+b3|Zni5BlI$7My}vhA`&QZEYpu5HCJrFdFz@ zKc5GlTCa7?PTyS?nLc*23?MX%ACRA)-?~+nA~DJ#AG91m)u4;C=d?Bwjd#J`Uaytq;2@i#{q49r&@;%J?CXcM$RdVukX0cRko&FNSV_H%k)h;$Z#SaTpa9 zWpg+6a<)Hy89Nt;i&v~~If~s8n-8`(pQz(myn~M3nruCdmP;;UFM^0v=FOvByVNv}$fRe-(bB^Ej zG00=4KYtEEvz^2>Hav_)Ug{ub-{+(%GcrO0T18=XX|pBf5H#Df=NDL>ywjJ{T5jc= zU6tEKs=3f$|H*(B8&f-A?imB4w_E&()t>4#Ja-JYdB$$~jzmv0(SlaZ;7|GaKFoGj zMH9q8@G!5U7m01FMXPlaqBI>G_w)Rp1*{1o!TJ?)-h6f{E~eUcUoX%R}3QBg^Q;7RkzY5 zSJ;8kgNvmLDl=LdE_GwQ=TD4%pAuLbkoUw)ztEO1fNqFNSQUMgs)Ua?iuLduh!_-Q z{|i#Frv@`-ri3Zzddtc_q8yu&Q`xm|*rSg*JlgB#&+09jzTwAYZQ4c|?qi|H zj?@E>j4LP1V_}|Y%9`)h7NVxpk>uVi60oC8#8$vHxZZ}6zIiR(K*hkF?*qGUdj?eN`OwssH9;MyI0 z2wJr`Mh6Ps{-E*};V-E^rGiGDLdl-wDnEQ}s~{cf6%WK(_sKqsEifEX>E^U(8Z+Wg zkt;GdW8^`PSz8q0NiGmsavRx`*u6tv!EMTE9>4Ox0B zmCQEnmZ#(HtI>~1ZNjLFPhwQvj^CNR$pMfwGfGOQG`Yi^`Q= z8;iK_^5oQAEbfY;i*rkBVsfLjm#9~8ca#wn!3pwnjhl7vHSxPW!wGkI6#KDIdRTjH z@(tq4@U)49QlSiCXR1|aT@X|aH?BQq5sXN@vfIFAizO%I#bxbduw~M3e|?|6cM|(H zI1-^*uCzGO#X)Y2w#}~|bf;oG$p3D_RZgDy^F5>{vTv&?67fgLo=Ty&URUrR!R1C6T^b7by z?`EIgZWJG;ahm?4#(TcDR{nUS^DXg)BO4rmbYOlR`YGswC~gBOfUBT>f0)vJGImy) zam)pP((HjAvgwZlodUJBfT6rpv=Dl}=b91p3RE=BP~!Uj!O@plHIEdvVPhtF2-2#U+9lBDr$=kV z1!lh_vu@tJSxr2RH{Rhn7m8}~0yzveAm!qgsqh~8 zb;MxIbm*ueDaAc`dn%@uKW_Z2HtLSN9%D#QwsghgFpj4Or*Z;+-qZt4bv>Zp!;#&< z+d`iW4us->p}HU&_zci-GJiaizjklX0%E2~IZ!K$tXt6BP4vdUX1yYE&t zr}{I0tK8nRs&Dp4C4J4I6-Qxf0cE`ynr%CaSBfeNKyTghe=zt+`5)tNcIJ7N26_1D zt%Ch-@z%%Bq(Rc)EnG`>Z-`Wlcq@@ruBy&y^h_X5hbBjKM$hEhX)?XcOL>9P%AIYL+#NRk=j%^N$24woGryM6eUm_`d3hJ^D@+4(*5^BFVA2cA_4?PeFAGbqmc=+dbJ&WrObu6f|`BU;7p5Ew$$x$Ar$d(yVFKWXc|UNT`TUP=l-&@T?T zfjBp)EZO=jZ;+5w-nq76$&e7UbRuC=0~_nO)qVnL$R85D;|3JD)hbeGPLQx;b3KPa>I z*O%epnx&W)ITV+@iK@AMo>?tpm#Z^ag{YJz9+YztM{iQ$!3N?;gvyOA+xt8emcdTD zu%>(T4-=*mGnPjVw8%<(AZsfwqdrNo{Vy#5(S^!t_{aef1VVUIpR%f+z|*#X^AbI( zP7&$Ikv$+#XqBEnj=Ox^Wy5%|6Gld91@I5am4gDXOVPVIVc1>+`c`X&uo7`^XOKlOybb3 zw}#D3`Yx-0@2TGV`{whC8F`cq+I~Qi6FN8@EsUM|5UJ+I4oZ3QEKr%;Gt+WFh`#=7 zyF-uhy*&N_o;zZK%W~`=aVhUnJ4D$3b#G(@^Qb&FRAz2UK%!R7(fwSWD}MzeK+uwG znGQ4@*-Q-CJbr#6*H2;d^^yMbld6-y%dLA*UqR|D|FW{2S7jx+$@<`>)%?^$%!8_i zld_|#hA~f3+H2*oy}kRTA5r*I6QbI)9PP~#D=X}^Ju}KgI!EkpFTuL@o;kV!kbL{8 z!NP+36^tO z&#@7x5dNJQ|Fo>^&9l8ff_PQN7WCU`<89kyN`n#b8;Gl;iF)W#_wDm#O+(-#2r?kR zDIZv?XQ#%c(3ip`(AS{7I?!w(4%83QpH5H4f(k=*-)%p75hBN_01+}fi*Cy+|Lrh? zHzMT5jtV39*wAduHj@GS3qkOi8iP5Iv?t6Js;-RS&xscN9op43zESx|fdvkUx$4!7 zf`7Q%cmsHOHujzRB5geI8ABWudx9{3mc#r?MEoQ2n{BxN$ByyiT5K$!pl1dWbRnZ1 zo-4dbR)I?)znWX~{+XzZp`8v6CL`CqEr*%n8B99Sse>Ts1VQT%)bwn};-C8g!CFHU zFEaCwE>tx%9QjAuGG@qwF%K+YtQQlMTZrU3`C}W-OodB!(E)b74ess>#~3sdRfJP3 z{Jj62Qu}Y}Nq@w<(+X(fRmzRYwKH-M90|r+#>^d7j`=Zqh33APeGX+Hm@Amiuf;e> zJN;{B0;vA#7`_(=95+VarrhGaKnywC*5Mo{@P!!37*Q^1lVv2$y6G_WxPDvZ3$$#8 zOE6H}RP1Ui-!_r$`O#?FkiQE+yn{J<$$JA)`Cauw^$KMP3AUW6e?bdl>=3DOn3ar; zB5?udJ}Cz%c0mxIZmyYO3Q(3fSu)?L6f5JP z_RJh^>t=6F^KOg?!8c6fJNfL`Z#igrtNpmZv9Z@S;IQ6Gd+S^N_9&cU9PP+EG{$cjP#XV! zx+fG~G)1|`SiZA9ftVmxPBhtyjx_JY>f|(6g+~p&w^w4M}X7~*u zUOyB4A&R~i+n1gMmMroHCO8SoVziC*WB0p7Sd)1i zk9E@iuE~LzETy@XviSJ=0`$gUUiOEp2}(+=O=_)8;lz-uiV=7m;k-%c*jkK&?;Rbr z>b@@9wxgSHyGHVK51){lAqVDIA2;M02jW4cV`;Ri>%5~Hd$_AH4KMA=bINmFU#eZ= z57*4!)Qj*$e7sbd=ZGL*7kHs1*6|wWlv3j+ESi?r#qj^$qDy%cgNhzg=B*-UwzXNh zIB)cJa#Yh89DeouRgoUNJR$t*WG}u{Bl#BJ>xy8E`Jj}X7vYpq+u|Tt+l1jcx;a)A zk=W7tGXj5g$QyI}7TG86tw407ZTDNRk{SQv3z*5LUL)nPJz*5jUIrdi6e#m3XXz6> zLuahXxL|&hcrE5TY3&2ugXHHh+vKdg^;LaIM7GUj&$K$2Hae{)lI)mZ{U!_>k2^=2 z{-(K8MF|)RLQ!vi`8s{_;c@f#rAI9ettrp;-Y&Z%IzEQl#4~SL!^%FOqA9$B92hV@yX5ATE`Ann{j3qn>sZ6+YEgea0D)1N?uVV-=^In>h5Cx$QWKU^jLU zit`GtsGoj2#u2>Vb1QoE6*G*6Z3uwkSSkWS8vVBG(Vx#(PZ5v}Nt|0Kwlzu?QMt;WQT?KDcmp9aVY!A1ti-p$W&+!&(f;;u}Ba-1+@} zjf0KM=3UC(SYLnCXCzfAfZ|d~*U`~cMW1=?50EP50{y1V&f9!(y-26zK zzr^TL$VNF{qyx?7Z^d-`6On-6g5qWqIiR?Iz+rX~XKZf`P*P#Y*CBJj>Jy3s{Xu%N z9K9d*-(3;zeWkxyl(k>&sw+9MV4B2=ZdIHnCjdh?>X+bV@RI^R2{=n`a;zW z<|LEi6v;AsqaSOf{E-o0e;rN>Svu$@g(;N)dq=v+>gn|zOdK~E{ZyW>0@ZO)bpm|) z*MGrNeNizR+788)(Ig-DI}R!)D^~RhSxY-onPLZ8tMo<{AXTC2S=S%r4rUcs+kd=- zpv-YGWCQxtFpc+|d;>M=aibv^yDTcUkamnIty6#T@6q6JFY2bcz6~xITgyHWJ*imk zWTFGSu7yo=99Nr|%>CDd4+?ShA)B8M4`3Rf%0+T_*9#S&aT_LB?ubELA}@2hWk9SV;bMi>G|;L^Axyq16$3d=M{!B|*yfF5BZvNKi0ga08IuLE060Dj%RA_NSn z)a;)NAFdp4YYX{eu%JN9Cj7I^ovo>aszJd0ttf)@SKn+iviN~Ohm^>yFq;OgRg0kh zv0R{Vjh)VffphQ3LCZzs$EucM!&uyi36b5T>1~e)B;i~s`iG$xC%xvU&5y~57daVP z=?E8s^muN~c}N9{1ByFmn%|oiMqu&R?UN=zCoxa7SKe>=zr% z3=+m?lC9JzFJa=Q#V-RA?l(n#QXLWx;RoJ%z|TfuJ?mesVJAM?aewXwrtf*reK0|{YC{-^z(QsK1uh%%aDZbqfj+rDFO_|;kN07vnK@}%q<@?zDFWj z!`0zqwvo#_$}X$z=)+3JK6zBa(3N$>j~4L8R}ZD)+=3W)obJ=H}CaJ zUoR4Qm#Jd~x2|06vfmncF#Yw+$%)9!{88cBGgchgBah10d7}d(?MOXn9Nj?zzu;~c$u?-YFmn_N`}YvS_NQp%qi>w8Ti|L)P05To`oB(s> z9q^Tjjn^bDMyar}1YR$CgpoBkC&W@ejAeE7{!I>fWiu=_RJJ%mW}LxwM9M&GjityQ zf$WzKpR`P!iipui4Cj8r9N)E?x>H`kU#YuPn|Vdj9oZXOu9A z%|=+nrKahe7cY8T;aqp9`eS6LY|T~f@Rq_SsK1n6q=^R>WtymwV79O#>K>KXxQ!>D^4(X*!RjbBjtUKpdkXE^GvOgv!yV^ij&!{TN?m6CN#w%qVC}nM z`l4Y^Hf&^Rkf9*0kJSoM*8WJF%8mR(JyzPKL9e5Grg82a#;#d0kvOpqBiuct7sJZs zM6+O<%e5_wH!+g=*Qu8zt5z4tc1iAZmAOzR&{q0mTP}?xIN-{q-$|Vngv>hi$w5Q) zp|Br}T&mD^+D7T~^Ef_$liK|i+nBjwty_B!H=~rEFdco(`lBA^=^?Se;l6cB(loJc z@YM4-vG3um0p9QR3XwOkVF>>q5>iUN{b4J z^iHS=E`%URF98BXr345eQj(B}__vvSSjr-d0xX7i{!+HT2{#O^*J&l+(m-pi)8+gt|ud!*b zOWE6!UScHMCl_r-!zn>86;V@v4QZZ?VocA$&T{ooPR3E zL-a!>l96Mi^Z=C)5`#Bp(4T+gAa$7`se*##jdz>ptwVL~zA+LCv7{{Pwn$QT&cMRm zhy9NqQN+r&$aWcNW4>@uheYPl-KM#Cg@33hIfvEdK&)hNAY8MVP2_iN@87+~yRg^z zjB>uZqoIsCGh;U%wzQTwnbWo{nA1wRb91j6v;OM-d9xXn)KM2g9wYp=lgE_cyT2ci z@|K#BN|+AR2M7l)Z#Zba(Y-HxKd{r@%FU)QmX-u-y5URwbo6nd+~8VG3(QI7CkUwP zwxz7{GH>$r8_-L)2n?}4N&H%i%Az)W4>+DeF5M#3Yh&aHhy#4Na6TT3uC%ELJ49l) zuz7!xjL`p+!u;^>Dkm6j{{3+Kx7*%-`RV_pFu@=E%mPi!-x|pD4N%{J5Er^W7wG}} zd(xGS&^^$rF1GF!UN;T@HW!8l2f8=}+GzfO0AN}lf{JVkJUF+ge+!&XB>gfY84^PN zm&o%gyMg}6Wffy8M*dvQq`#Y3S04<>L>kc94eRg2lDA+z>fxMrTVR?2)O?A_XOnKO zuF$$;ch8syYRg8mBAnua#XJX)aimvwGNVJ1(j%6?kh3G^<(Y+mw!>}VN$ULs)uqh& z!H47SjgmUE2cRb4$YGtsj2O&IujHUxC!fsmI4PZm+SmQQ#!t%vS{ra)fri;nfrEgV zbFE+y1Y*EH0sII=o5%>a+I#$K+fFI6F#duc{58FPBK@t*t@^*Hgo=60DXHo-qzMOz z(lxnbo|+(>DF#V72mJ+o0}KLf$o2N^>|FRwj-b`PFuE+$uNeEozsG`slPV0A$@Ong zcCJ&qY=QEq*Y*P#eo}wtXJH!p7%ToNk#>g-s5QCH(e*SYn zmV{}g4SH-QO@g$j%>R0$0D@HP^+d{9kFN9s`n2YnlqGMO__OF~)M)K}XFBV>{gOZn zJ>eDTT+47gR+~}f?pl`1>%6`vm+(!P#uq-27LEW&fsWVJwO-QqAI*r$Ne@x~%$C<% zAkXJyd~Cu%yR;J0AV|T?k?>5#1wkZef04a55>M0nUh{>$8zCt@TFtqb&4I<3= z`h%7o8);%W8SxsY#tF_`aaBud`r^i_<^Vz4cytUeC#1h!WtS?B7jic^0&dcZI%4B@ zMzwx;Rj4;Lh;WSxDXcem>Z8klQ5LRznCz1Nrj9*f?lQ74KHNE5IWe67&JXeAsY4DU-LW)k3G0z{0h@(cpUj&M58!Y;$JGJQ%Q3s$-n!e!dGJ54jFsj(%(#`|FEvYYGM5A(~ z5L^e@b+O|_nIN#j;m%f}?+yZov1Kei?GYtVYi2JQfg+m`s%aM~w%T6Y(^GRt(^%oe zoD{n;|4(HiKo6wIY@M*pnfP9$CP~$FbQbw-UzAGhd1uDh!e*%KS`%zy=k6R5o+PX^ zo;#S!u1|^de<4*gLBtQxPYHwTS@8N!_EQZ>D&um5$`{o8bcChcnB?e%8ityw=}lM_ z?NfO4_wp7WCinI|l(&3&=mm%0dAWlu0YKc`FHRpu4+STF$Nw0h?PU05aEqhZfeP4> z@a_1N*RRumZ&$HiXcKh3lBQ1$@J4U-cMHNb_D$cM(?LxH-hTX{D2wq9rBxCtJ}gSwb-JrBOs2N`tSD3WH+d=%7fH zikqir+c^BwiUn`&`qI^JdO!23vUumquu}2gT&VeV+nx?S+Kfz*9|m4A zfnlw~wja@P-Hcoe@k%(0bcumN3Hv;I`~bUX(h!yVPfs4C7FJg~=6y63JVizioj=GH zX|82F(Yqht!unv){UNiOd|j0Noe^dka>_y3W@KsUxzF4n&~XPrEDA>C`(8Bg%)MgY zFtx^IonCs6FzCGz83d-7joeEDcj4D)QeL;eZXTgbG#iSgF5RCDb6N!Q_wUea{csJ) z71ZZx^{MEueFI){kk(ZGX?Yjnq>|k1_9yKN^^J0}Te>9g$fD%v7v~mHnc$Tr0qtDr z2~K~|hp_<~13d1Bdbh${K;b;}fZagVf9RYuJ`-4mietta&-)Z&{b9s8`)$1V$tTF# z#wX-`*g(B&egnoIq-A9{LIWUW+tk!_?tpxCBHH8j?ZZEPt`e!g0=uBr96+PidAHsE zxS~}o=j$)F1sK6L`YY0^?v9k7LjXQu`22O4?rxek^Hl?Pa9v;kl&P(Ymaw(4^+r4i zt_7#BtCcI?k;Kl6F4GG$Z@j}B&eSrnT-V?J{VBNH5O>VQqK&Bpc#Ov$%uy(RV2ISZ zQlRg#_a%XR&@EmQ^QK_vdJFAP7#d(@H;7>2RN1$-*O?*)Y7xb)R=fJUy4ZzRaoc}O zIC0~`@7w>nfAQ3V%_kZ*^hMYuB=)&Yho6&ZUH7 zFzJCIZ*HbMS`c3S&f`!QeFMSFpiTX%r&_W1HFmxvx|CW1dP|_DUSs*KuC5cQ1i8rV z<@2h+^HxyPFEohwioUNwE({w{Gt75QgL|`Xw?5woJIdAN z)!C3{;B;Nv6R9PrfF5wQZM`qHMa-KQMIX6iPdV+5pE-S!g*L1oBj!|0-Jc>>L5D=Q zaapUpmvT3jG^?Hu`26mS+3sWTk9CBwB>#HdsLWC@Tc}@l+Tj@6 z7TTO%9PT}*v$0?Zt(!O-tZ$KxFG0^<|AKH)wYrXqsca0fwZa4}mQ?{SM(ewNVDpnb zG5zO&S?fD)XWNiZ3_9fY%1PrD6PYr}0V9ooWoa*TWL?{l6sg+Ij4%+|>lfd4o%75o zvu0lOx#**o&sKZN>d58U=rKo?ETk)y5@GDkHD@?HcM+tVA8nXT0M!%Gnbi-zv#{ zF}GcIJZL1-aflXzucY?h;V&+uVfLddMYxpaWi-aVqoR`f^`4!cFA>cNV-RocRgGzZ z;bkj@kBcJdy=+}I!DA$9U^^$B`Z+^9`NMMc68W6W8GCrRZ_nM9vZ>GrBmVG@2+7!{ zd;IwM$AWEtX%50QF2Y#Yqk_ip#rwmDw#BLj2?~bbJ52ROa~9=`E00_s>Sea_Ca`tu{C=2& zezlZo+`H1{gY8?pG2UE_#e}8R{^g_(ktHi}=MxY)w+Or#yp-x5f=S~*K>()r?~B}q zHs4jx1V=07Io@*P)o)0wgEi6ed>+opFjKP6FV{fY`xuX=+sDLf6g!o8alb#Ag_{d3 zEa2A6+|J(S3mw<5>8ok0%!|95qVsSvnZo+7tAag?j`K`F?b&BWyubFFtYOA zQh02KzPUpHq2A-+aBlf%;2GLu4 z(CeFzz?oK)DK;Y}CEJG$PHP*x1j}~M_Ab8`s3cpv(BF;9mf8AyvnIwZTBq(ZByKuS z@6cX~rBlB%>Jv73TWm=s)R9YODa1>Y<*9VHcS{z=@670dhRrc9z8+zy;%|>uVxzce z=`#u{#KIxBBMA{9$%n@?GoQ{^y~;Hue)2hi!yi%C#22&&s>{mngI(etDQX+19J;|v zxHJ@hO?!6j<(E6IqOF?So`@slw--^9xor2WoTc2wlkb<#8JyC-x!9qCpPIES`^b@l zxh!WPEopz%9?p;dV4?9`qqO_AljlHwK5kzuRxPf?dO#Fqu4h-k>g50fy!QiwWDX}B zBQnut5h>hsCe{VTw1%2CN;yHU-`E(K!QMOyqm#nr{p74{GFXbS@RYF&S?&-NfU9X? z`x?5NU4qA|iaZ$3{w=%vO8S|KGgM!hw#;=*d==$&Ru8z17OItM0tV}U9#;RSzxMpV zUl)Pt3w%)9Q$>><(7C-)(1QcNLy*6W={Gr(+00J&3@TlyXs`qn&GnQl)OP!9Bpwzx za}{bjLzEF;2N8FLe#ws+3{CwIhn@=1hR%UZ732g=W^w4EQ?FgfWjGw^xUr)n`7IYQ zts$rm)ODy`kC=UXq7Vl`SJN~)O#m-g@UAkXncpq)wN27tWN$7Od5NkpR-x)=hVTTuk9BP}q)`AEf!fT9rDnL!3vtX)? zv|4I)LJR_%H*n@@kndI)GW$lKRh687_bsBEpEU9b)H@HG5a;_5qUM%)|M6hb#$Pb4 zID=sUuuy{3Rr|>j2pVzcTWypRYJ z_J(!M92t)IJ)8mh!l2VXd5t%_rCAe~JF&pddRC9esHhBk0#a>Ckn4}nG0=msm6@&H z6kjw$o#X_fpkV7M!QArbg|#HcrLnW3DtJSGu0`mN&p9I*p-($QtHDTeQ|go*1l4@| zdkX~pD^$JOZy|B|G-GHXsijfaaKD2fYrBUxcoUQDU1I&!Fn=k`X4!s_Tzi=+Fp8VD zqb4CLa1HF?Q(Fge&j&Hm|d_URTt#0Y%Rcr(hvqKY6rQy2-n zmAC3Dvn&=BnNnW53nj|R(z{V4FMj7W((w_l=@aMuxx3Yls%&s{Rr&1%<2g40?Crlk zYi#<#d5I>Z`O`17`h7%MFwsT@sypvgrB*t7<2Q7sq*_>2$w|U^j?2c^=}u{d)bl0# z3*V3Sr_W>=58-5MnrDD7)gbt7wtW4_R-=;-BEa=hmqOg^g5GUsusZzI@dQj(r8+C|Kh%Qo7(r!g}-5-vYKM!*+#tN0)ptH{(kdW1az5%pQlqA%3*bw94V`|_7h zf1Z0$6ZPAKmnb}6C5L}wb5AZK58t#9$E6P}v_-%U_3*7Wb(+Is7&a3nSkAL>isOfB z_lrZMR*w3}n(^TWrWyW<&!mwBw_qU?;ZpuY#=av$Bwj1oX>XXowoDa;+F?8vOl|~z zD9M+>39)ZKc)yPGWDS}fN^x&kb?d~w>KK9z(E{|PucQYG0 z@IFDmSt4$5G>^vFj1z5jR?PPg&8MrLn97KnDG67;bVE3EC``rsieMrvx^O36P^~Kl zSr`ZbG(1*2P*(M21(Rpyn48aY`8fV{`tW$L#*3wg@e8Xr!6G{}VRDo7-2-!XQl98J za&=SN^`DO|MdAu^ahE=bpFV9atxG@9HZPPD(EO*-1}ex`1HrNw(;oxcqFo%lr6u!TET(^ zh(pWAcd0E7xiyyL+(g-T`ZvcAqfOHt@LSCFH#)!)+HoZoEbHTQH5bcXlY|fTC_U`* z7$?kkY`HlvlPupiDS#e$V0WNSc^t`b=vy7js)on%bnCOx8WUHCJ~8m*V$f<-J?_B0 zxbujgx2NDzDQf?iAci|=)?#yNG~Al|;^y|K-nWyFdbiW-323I4cjbUlvGeF1Yk|#2 zg^eSp3qr~{Gm9ru_Y*FF`d~|H;&X{VXEMtllvteWl{zOpMsp;HJZ)e>5IC>z){e;P!mss zvI{B&^oZmL8z1vwsi-p5%z+;MNX@Cg6Us#TyO$=bORhE!<&I~kj)u!h{t?7~_vU)N z(lm1UQ_2P9ZV^C=&$`9ukEaOKX1<0w3mzva76`8|I3bGimlI!FOt_+AqHq!5b(*z2 zdMz%kO;D^cPPjxb7!0hqBamXkN78qeNG=dx&I|r-w-vmxIrmzV`bSIs|!!g2@jYY4OsB3z1GJV3Tk+pk95vnISAh9dqwP!3Z4e6iNfe$B`j0- zIfqRrL7Q0ga&-DQZuKjSOQ1^PMD%!z*O*HGGy#xqdX{&9i+8~xp`HOIq62HZ0P|Q` zaqoP|Ia|%hq9CeJCna?kPE!TeD3Sg87fPhZUiv>BcK+jP4?f`kF#!E(1f7#1c7^TQ zDp&4tb|19{C?6Z3&5t)gZ*y}cplj=!o%N*sPhdR(hS)w+6*1_jdHV0r<^ESmrJLQ5 zuH3}KfBXs`LNj0Ah(k4?3RJc+pF~3xc!-Yk`7;69xw5a(4rqE<2QGz4CJcqbjumDn5D^&V|Y3mX+1j15+I zp)@i}^MgglGn8Ul=LQHF9-hIl=Ym{5AngLDNKkQ_fj*#GLg0Ybf}qkt8%Hr{luQWILk23@$YBIV)mNAhl~FGzQpg!03f& z9P~=Db@JZ`xh@QsnH&6CquCS`?cp+b2;eBnV0Kb2X(t_o_=cjf z$=2AwVn<<7#%qcW2;2PHPW8TJ*z5ijn1zOFf)oIV(?7t9_|(YOjYm~Fa;L*7PuhY54@=WN)AE3z5qP&c#D({ol?vTDem>uqNwXvt%{&ZE^L@*_JQS^0u7ia&9Sq-Lz}Y1;o_7kn8*akO~2$voFT-s{DUGzb^om zra?~^N0~#cW6;D2INJhiNPOk^U15`78>wDyvK@l z=Y$@Z#GnUY4h8JpUsmEcP&}fD5!P+Oj1OOlywl%d^IHa^kl8yhH|TFe@;#@#YCIVn z%hOEc{P;wWe@R2>`~6du9DeFa2Jwu1(qw3Hiktn6N>(1cEqUObXCwQI&LbOp?#>n? z_DeEp?3CcSRi@;@Ico6ey^^E0J+uOyup(TP-o^5T;NQ#%I}ukO;~?$ewW z+^YWQoxQK^j?Gi9se0}^Znu_~ckOv1c9U6x>09RJC-VESTQWAi*>gAIN@n~HCgELM zPCmM(?P%qdol$j*sxC(KMOwh!?q)B~u;BYjz8at+q=D3B(vMHXkYqJ8bMTU7+Qqk=Ze78s zjZrb1)y5Ho8l39R!z%3xM8a3K;V&cns+b?%W2D>hM=V4<#7S#)E^nJ1M!#q2f-mc` zTdjyoLjbB7qmKqbs_ql-#|QK4Bx6ODaL2e&S=F5bzVd(0c?{9i>6*_P{*ID?hw*oV z?6Z8zeW4_*-eGD?Ie|#(dgUJ(zrg(~*m&8+A~1LfKARPc=Ul3)wp&T;0`!OIE5D_FqxGuVHh3ru4 z*ptEf5g(WE+X}PuCF#zGqA%s-`#)P$wMs&w)`Va=4&zZeAjm>rCeI_eh{B zkp#!)cN4o_Ob5Z{F*>@s)?J50*}HWoUmDS+dhT8p*do=1J@?uc9#=(QBO5vkl1Fxb zE~w^jzCKjO&Uv3)U(B3$c_of@LjFB^uqiLU67q4k+)=k%U3z?K#it>UKRhKh<<=LRO8@(C&xg>sDrZSH= z{E9q|Qb@MqR-&JI6QfANd2hn|;U9l#$DN2x=93N!7He;jL~52S>l)Vn7{pAT{NA4K zrZV=$SLt!+Q7hiZxsXfFI{YI{Evo%oN+)U&ECMpZ5+>ib1H2B4+<`w%lox?VVKl9H z`m@SDSww-lbXSex4qSRjf`*!0yIOk9L8~y9`uu*}ouJnj`bvhTIUVSTS`A!7oFbXO zC#*!ygj>&O+zGiBZ6)2~`&1n2eykM6khG1_!(blsCziM!<=Be{uK|eujXw`7=dnMc zU#V~Nd@aA7(}l90V0iDW-*Gsgah8rt@~l&mg|~ZSYCbG~c^uZL(w*yo_}IQgl=^~J zBnUbgYR(BOvl_j$8siDtu^{cq*F{fT#ZvKum^=8{kqUN zIQiUyKhlb5mALcSb948-vzfMe8o_#}14iZ7r>dPW0~YllXT>s>nwcmZ1iPiJP}BN} zJ-d=z+>(M81!tG8HB%Uf6z32z=(#*BGYP~`Z&!U{lT67{!xYkLenD|SF>lgsh(d8s z^M#K!$>vz)V70+Zo>7j_7u*=K@|7ph;*49)(>0dGrVL0LWQn zZ5+~ZeQ)L}I@}+;Bq?~rU%&L6kubmIsV{M|)eJlR&J0~C=qKp!5`*@fvOE3Jz%i^aiNLs!*J-fKL30eTt6ZD`%g)o*7&5=5pt1kj=>>#B0Yw}%Js1%(On^(GF zk`U+;7K*KJ2dfoAO$m0VbT~}+EiOL7na+F-FL3(QCae#@7V5u=XW#?bcTxmrt~K-w2zE>eAp z=bSVfRrNI*(E=tZBjSyVqj14902g70`Y-$thWi3LC=KDda?{t~b!TL07&?S9({UHz z)1e3cldW4*@tusl-Yz5es21R1-6x07(5Sx~GBM-Ep~-?}DF$lFNpaci>MEDnIa$te z(GU&H6fFXq^Ddo(1%Kc(O2e?0AFhf)V}Sb?=e+m$U6<<4cqY{-t#O$|v@=+55QFyG zk%HHke^50jBR1^tSm7!8x%7fo&vVPxRl&Z<#&mDI)!2yD^7z+zofgv^YG_q9m+hf&m!LEAmOFVS05N* zRVPn0m7rJz5W-GX4%xr4$|$DYl5X&oU?-z{t>2|Z(ScI8c>JrD#ht#-z8-mFdGpgz zE@-7Jgxcz*{y&T7P4?;%%?Hz{h&l&l+}MJ;g6W4mmig+3&c2GDx(+6-W@^&@CqwwR zwm*u?&=5U$ni{ZB8Voim4c0hByizDU-mZ-n>tQ;)m32~DKp3rl`=N^sB*$_wx2@nO z^pVcYciB%#R|Ii|WOHtJ|2rMLDQz#If?D>9f$v{kK%f4+VyL&qyzND?kf~=S^S3c4 zbd|bqOz%EL4y-n?9DLun>&A(Z#Ix}jl4L& zl1jOPSyhyBK056+Tod`(WThs;f%fQ=&zc`DO4>J_4GEH`IfhwYr#I-k{7wESnrQbKm|@>dhob1z5B zN&7NlMRBb>bg%oXA_NgLkMQIcDsY8vE`fy$@E2+q7=g9r$)*z}WG!>N?}gP8yH~3P zALOsk&fZj3q!&ymvIEOL%Wt8^93-rMJqi8EcX23a3v_E|B&-;iD9X_PJC>((vIJ5YrYbk%fDk*UZ9hGJ) z?SCiq869996{M&S?L8^@$Cmo1&NgjV_B59wpHdUL9K}dl!X?jIH(FcLw&!O^GM$(Q zT=eHl@|OrB2!-ck0?k5Qm#_1}^NqLSg=d`?+O$0~3+U(N?rzDEdzr6id+c&O-Ld>K z88z|8w*_7ISZVJPNAKtb^aMYtda_03nXDdr07%;Gi4aMUGhH#a7AY{)EkTsxn!v_n`uVWIp2& zD^@zD?zRi$Lg$?!c&Lw=7w zW0LaYB)!83;mh%25RP(mxyZH!7E&FX9OUC2T140w%j>9%f6OJ_xV~&%V0#tNL3Sz@ z9|_nt)i^P6WI3XGc0@jp&3SekzBes*r&r^i!T5=QthHzI5p0R&M$M`4FeW=pRw7oq zj*`OlS1a(2z6*PKjb_T!Nv4hlk99>g-$?QaKd{WD>5G7^gEg`K<8lr7YXJT=drDV< zbGLtq;gIuziPiDngqu1F<|b*3!AGVAA7$(7H2KdKmEWMNScCCe21EJ|fh}DPPm&b~ z7L%ng6fGXUGhFA#6NOE(CCk+2UwTxzh{Zmr9wEAmUa6xMOAo}8_4LWqrPAhYS*7cH zxCY^iL;t_PJ1)gN`JWHEePnY_)+eLY(|<}d$`Q{%2NDxRVt8QE5MMW)>OkkV{{}r+ z2UGnjWuui4R z^7K0(LI4bfQ%z4k}&cpohA8J*M>=)vNmA9T*vRtoy|Rkb^4 zb#V0E9B!_p3T{e_8iHu1k+0MKd~M$Ld8Bw82%&lBjv z)}F&TDCIi8o*?&5)l491pezpLgv20oow;m7as@Usseit6p$}j-zPI@wgLzWe{T19& z1_$ZNap(;}WS0iKmPrm7p9lSPIsw{()qddj0KLl))D06g^k+us&XJlygiHl$0LM%M)t;f9ql07Xt`q>UBI4hJ4hApj^?oc_89$Vz7* zPJx|fS;Qz0PxvoS-bClxit*iv#LmDf%6CDS4NI4wYrs4XAu=R22K07|>V*+|fqp8o z=ZEFFr*w8n3w5@HGZL|U5E%i%#h4X*rz(!)sL>60J=NmS^L?sO|0v$x9WRg4y}9)5 zd$jyEFeZKMq_U~sl~vs1`J8v?bnlfbf-t%}0a&>u0OEDnO$-A5BCwrUkI(q%^KezXkABA>;OCgaiF>tiDaa|i zp|E~bOxpS4kDK3j$0pF`+${ll_Ka~hwi25k4os>Db2e4_R?37|>Q7s#6yiI(jy{xs^3Rz< z0d?+M_g2{+Wis;)YMbeo?#mcp65@R+|Kd|a+mOOR6_L)-?|kIY;P4dxj=>$6{fC1A zeCpbCHEa(uz@eS1Pch6zBmGz}8iX_Vklz~93&po0HAyQTVo*+HQ!8vkW#xv=@79_f zeE)dxF!;HN&565o6Q3B$#I2p+I%?5aIGDHOke{g*BZEQFm2J-+kS*6l8(bZP6B9 zMt;dC?X!x8%ZuvsiY{Mf#smhZT`>{T;eF)n37OYOn z?Guy*d@ifcthf2Awwii}@~8suF9(vTG`~S~Xi7fHXi(YzA1WG_U@}Pe7OCb2%wb%J z(rR(Bj-!OK;QG${*CIy>1OTGF){Gtdphzl_JfkKh%0xhjQivq^A-`;zxrsZhSVh- z#r{^^eg6HDs*K=yCDx(jX?or01<7vhY2CB<@(Zob+q694UbfYQSdM+Ll~?;xvYJ@ZiUM|g)8Lqcc5P58d%DI?b{ zBo9fj3U(A;)}3Kr<_Oba99oJbB%)oUo;cd&{geFf>Z1vL0bj&lI0P!-FSfxh8vMOQ{Co@I*uuV&fv zkx_{ZE#Y;TZTr&(cah_d9#wN4)2ZH<{KuVbz5NfDmhL@4Ua$9@+?y9Z9uGg!`EXn94)Ii-@DO$962OxjP-dVJgie1 z*mA`7iU0{zAgVb$7DrS81eV7HH>oIWV)Nber5e*yNRDu>|AaL}P92L0OY zxB>*G`;o$bDjk&OzFkbn(garKlk1X<7?cXo)o+W6wn9CgYMTZtL~0*MBX@^-5giES zl)%vz8%N)bznEYU!~lWw^yT63wOaDZ?Us9F zUz|46)wrPh{5uF5Ue`2`RkRP7j%OuOm*8-)S_UN+00?vM(?(Ft42pE22N4F0*@6;G z2<5bZ)!{YF!;5~FfLUcd-Ub5F{y#22O_!v7r?L?gB`}uhE|WXK${+Yy_{$xja}|pL zPc7)T3CYAU9BWa!?sIRPQjUhg^x<(Vuq9=@ju3?4^NEt^M*A=`VaPa16g*@v0sU-b zpd_Jtb2%v+pmnCG;k0K+@Ug~99$K)*{6Qj&oE?h=+o09{4UwGxsj}vg|*eo|F;IkP-)<4VF~F2(skB5K31kY_YsUId1H$OI#%d$5>~>Ko1l%?*l*< zs2kU2qx376)6i=nEx2yHV40#RDAS_)zPL@ z+`wD`O>-3=H9dY*QOTp;*DJv;qL>yiVD6#t1{PO0;K2yTi9wb6$7t_+)LnKCz0#dG zM_=m?BYpyag9hLDoT&0d+^&U5)1v~ zv`_GGPH>n7-BqI;UC3%yQ+}<(gyxEagJ)4yDX~dzOtQq z7GL*ph!=}2iacV;h4pDi_ai(TCi;Thln}4`gPR@>g&I!nD3E zW3CeNrPXTqKCfBBi=)iz@U`O!A=G1c{dp2{3zTQe$R zU(Q&LPMXV)xJ`C$^Tr@YDFOaUNr&F(G8jCUpWCSw%vIP&hq<2*g(W4kFn?PulQWaJ1@qjX z{to#JOpKn?mlX-Qa_oF`T-t9A-7>JfnW8Tr1terf>Ck=SZt*JB(LXn}np7DuYA4Z9 zlXxf$ynOxQJJXRR1GgJ0U-$$rB+wZULjHwy?A60uhrB27a+CxTSywgYmi>BEPIPY& z0J`Q#E-E4B)3*ft)HvIj@*~XoLqw-CGr_|}w;i;a_&Bm1?@DmQnI)ELDecXH63&Ty zn0Kwfxk@U~1Ly$}leak;Gzl0dvwG!ewJ8a)HA}phWrv4_HL^EA2Bk}2ACN1bAP8u8 z5Tbd-EIpGI^N2I0@sab0R?v}xtwJ4Mz4Cr;<1YmHPtLp;0*2tv74-yl*|n$E57&S4O$JtiSrDHTSVPa(zV9K zrTUtq%z^s1lWb|K;hY`HuPKc{W&9hyjUZel|WsGRK3l ztAC3rVZB)DO3sd7+aroP>a70L_ibNr;CIQKUtCQs09oBMXh}c~vroZTz8Ux{S%bunjsJ=P>HsRuz?pyf$MUkz23 zH68d&sTz2xGxhdCJ-xkR(Cqwtz_RZgIc%Nny1K6T7le*3jtfgXUVRts5LDEy+*4dfw7X!vHdY&ud+ zpozk<%?!}Oq6j>i%ocKx~JHn11m0MIAwnIOGKUkOPI=4pe!Xa$n zzueERqm^FZ9hr4Wn==ZKMG~a;O*QY;bo<{$tLQ}4DdVs+)%HT0LE0K36vN{z?c^ZtSneK&}$vEgi zS67!|KYWzGy$f^X(Tm;B5Yf@)9Y)TYBG;861KfK)N3EhKJ8AL3)euYc|-^)#q7C`%K^2G`DZl{&Z zm!vIIncI?p1G@l2`AHS$(e9m+uip0kLz>?PDE(IvWH8B@?Mr_OV+><4`c^W8uTx`e zZns|G_)X0H6%23H&irt3_du|-kxN|C-(88xcXrs-j2i=o8J3^1`JSaV3!lAjd+kB8 z(p#2Hwz=F}4-6(9LPYwNub@Qt|tNy+RTB0-*3ywKr-ub`RP}9}a zwL7c(o?h{$KUg(ShDg`6mfu^%8DhqL+A+9gFr`kex%ZqB{3?B_N4M%@u^WrA*arUeH`y_YV%c zv3n}>-woKv*=x^ujjh z7`uhC@5WVn6--^-0#vPKw;PdgpkUdTBfGz*JIBb0{h3TKN>`~)rP-a=$2yV4p`SSR zn7Sv%^_!kdC3gEqnf`C%jY6d~Pt0irZeQEwMvcA(t}j`_a1nF68e9x~I{J;(6hG%B`!4&&nTt56|jyv{# zHjJ_3dV^zeN^r`Zsz`tHc5>f+1*s{u%%qUYj!)$VHJX;eJn7gZBZPyY$uu)+%7TZX zaAvxX_#v-OrF+fme0#4pe94k;q~Vtmbi-MSe3ag>rz6~w^@6!jzqs_5SEzJ}%FT%I z`c!#m#F(Q0R7XhzNQcmjEH*%k_-Pf8Tu@QE7JVc{xr9?3Jt}97&l_#7o&lc87_3MG z`qsV=KwF<(LbUMATVeE&(5;G@)1gY!Jdq~=x=Rc}p8Pl2}#zsDE6YTfxdAh9(C!5rnM2<5Vw)d#Sq--_D%Riw{SL3S0*LUtK1f?9dV_?EI{tgSJMn zybLO_Z@I~=7=ui7;d4i&utwWvi}8{}MY`yc?NQE z4X7zO%@+W}Ys!Y}hle5+onZQf-^GI$%;@or7~?V(l2Z<{cm5hIoAzB?mh`-C5N#5! zMSr9!2BkPH=|fFXj_ozKJ{%6g+mgf2No?;s zvbJ897*`%<7A>>SD>sdPU^0>BcNzr$S#^_Q5OCr%D`y;b1`nSboikM-{0ZGhb%C3l zGj3cG`oCCv&!8svf9*F`tYBG+NYmv~EEEe!4G|kE3MxtyhzimL0V#nL6%`RADot7< zqJq+;6G%{+f)JDvdVmN4LWm)xxAVib=bUHuf6tuRGtV1G9h_l=;r`xN`CQ?mG0cq7 za}oJ{yp|6#o!o|~dqazO(LHDz_Ko-#c31`w=Y!D7m&He~h&Lz|X{t_DoynKI)rf`m zv1}u)h9{cayJmiKzT?sK zTFKKsae5PVC@_7Nnm~C$kE=+y(sJa8iFJO%{;yN_`dGRo5YUj9MKx+*yQCgpl6!40 zv_#EE6b8QJ#oko4Rt(YHpzi&oT%7pPwd_jCE>P)vJagmdPyc5_a5GC{Si=NnIYh$Q z$j+-iPYCS_`WRh9w%JdnEQ`@PjZL73_S^Cfq#m^3@k>yaTZy_{?w7p3N&Thpr`VZ=oP0uQCKEs|+ z|LX5UaQ1xJEq0Bb-6a%_?inZbCJEjrrtl~forwbOS2jC(H#$Nv2c8<3np5NV;%mzo z5f?D{OX92FV{t=>=rUGKucbv1h2d|h9p`k{9Mp>*tSi?Wyh!rZyRG@{)&==XE8Q1W zj&wXIgqf00NoD&3;~&FV^>THJ90#+h7Y(BkQuG^GyqoZoOx#!UQ?EkWCy^E@|;f&qWnP&r@U}7>u)=SfFJ*S4D%Yi&IXga{1+D?2Ch#@VzkWMGb9Tr|JKYK>6R_ zqq|ng*8?K|qC}RrBsES9t58?!iV#s_0@cu@DHV1)Z0MgIm-KhXM^YWz0 zx=X)mJ?R!37Dq}$;@z8`w|gvs)Ya92{Yj0;b$^;*M%*orvux#d0il>`)8voJ?+ugj zr?bb96p}v`aEcZObEc*kpe_fp+kXPq*_PPQ`2b5Re64Bq%*Q-6C_oAJ`2110oGU{w z?@v;LZub`-F*OdGbB5F}!uX9M{Fw~{!+%oa{|YFZ_nf~oe`{LE8>U23WlrdbhzJOT zGugfjk`X!pE-H$Hb$%6TgrBL9S zqTFT6A%Q451$b8@y>kO%J6Ndyv;)-1ykLkptoPTNjz~Mr_c>sw>A@~MxHlmX*NExe zmLojeJGao`S^ld-ZV(8Q$+VU_Nd+ReAoj#=d@`Lk(K6GpP@SFa))_tkK2Of%Cj%-S=!m`3=sb zpN#Qm^g%GH09yqS)GL^e3P4i4jzy9PKgdw>>a3r;9O>)VnhcX zOQld0fXJNNuZ$U)zf-~zaXOhp@kcee;Bp22DBxRKt4ZGJp}rwH;QWaqg<>Je+(Vg( zfWbWI63)cCx5JOq_Sk-Bm|@O{48AT~@zreP()xYWyma+7kXM54x-^Rd>?Gqdc2J&@ z!MIBcdaLPmRv7u|FqH_sr-jMnGbg%c2ypWkt(Ka81|hCBt*U9R+X(w6(oFP5aIu>% zz_|`|V+CWEoaYXOm*HatZojs|vKcFPL?o3`^fUFl(TQ~mjy4Hxt;nf-bNdGirH3ag zua94;JLy8Qealu^sk6l|y0yHgXs;9IeR9)FyW;(KUEtb-A;s&4wcAoiZ?`Rx&o6E60MG?zn|%{~OhM(cmloKNE!e9a2t$pg7B zUo5oVuu%P2*wT$$a9XzRaCqtGGk(xh`CBr(xL5Au_*b|!w|Bl}{B#bs{7QJ)#4T5Y z#YJE%G&$sf!gg%MrX#ujgln>TZf2&Kk`jvX{&lPF)nof(r`v9Y9?X_O6S>`adYc zg1WMEnteBY;Xu0gr=TmZ2Vz&fESk0_C$bwNHSWypOldP=+rE4vQB8GC${rd<)71g_ z%yS6-$MB7 z?X73aLXcg}#-$&z`{_q)j6p-FG|yeKYn7X&o;0LWhTajn&;q-ODt!I}wRUR^)SRB~ zY5G{}f}i~5$rjDl2p0-v&)@iYuNMuVw$8HYzoE*k^nn>?i`RL<%=37tEBb3dSvMo! zhU%UGntgclE4(hml#H|#LJaHWyGuMfkW9r8Z`g;)^7OANctl%``1;#*XBve5d%iL* z5i#G$=p~Twv8I$VL=EF*0nv8-ldc%vJ*9qh6{AC^sQ5r+$o6ZfQP=I6;K(JwuEjqC?v!&#N%!57*>{H*07t$;oVqry(((NLjqTm~6xt0UhvUiFwzp~&P#UUU2 zGNTleM6A6Z>&`K@Yv^|WE^-;P>{3zsPG4g!w$}IW_~$a~RuAVNAo@9*cEdD08@38F zXg0TzM^jXBG66$)A}Qe=zGq=ALPgWJdnmAHrRWO&l&;I>f6!*^D?j+RVSkHr+}G}x zUujvfX%N$-Ztj4Qma9buSH-hr?6ko~4d*)_%1T#!QaWKbXb*=M4K!plZ>*8q@1K@X zbd=Dwd^ZuUd9tYmB%a_)N*rI_Wx@)QcjdIgV~cyzUEg@J)nWj_?zmMx=Fh;&)LrKt2UFg(U#ooEgw<) zzQ%H|;0~Oe)Z1NGlKD3&q<6q-mU4gI^xFaXOQ{19HNppx7xTRUoUp~zp@5Q6Rp~r) zcYLsNE^2GnA`)Sh01%s=3AF}`T4#%xbp3ADkhH!{yj7GJI@hv{ zC^wx*v@O$=KH%UqF?~o9fF*>bHpebnkx5G;cK;6++EIurwBjhI=v3=-OZOBr(P!81ZRO662QI=a9vc%gazmQdq}2k5wU+uE#v z_?0wm?Fa*^U@)$2B4a)wObdVzJm2#}hZoux$P0JhE;KK|7JGn&we#obgkHt-RLf`E zfep$m1E8A4LL8{`X`LNc)2YiEC>gE{eMzP-fh^LUAfx}%n1Os2_OtxY0PrY?L@)g> zMX|ScQn1X5GEy3(loSAygX5?p@aNmJQ3Gp9wRZ3m!d)bD6oF1}kYp4ohf*;kKf%Q5 z@HN-(@s4#uY6&vU^%{CsY!^UO(}vrj;e|YPI2MH}r!&QNR9Xw@JOZ)o^I1=CS>sPW zn@P}w3%LFfQfp!kAJjr|hY7=IqJqAt z3p$BHsd8Koe%iLqHSFg!lNs{!NOd2|v=ZzqHhoKZ7a*Fg^b{9DsXaXgx$3Ox=|H0A zv@ocQu7j~7+3AXjRN{H_&|zGqBu)bw(HY8QBn4o!p(K=(3@QFn1y-H^+YR+~<{?be zJr@xpGXtOHBum?=Y!&49Xq3{@Z35}=qN;V3R?l6g7 z-a%uf5E%iz?@VO3ZmimV9T$a---3_&R`q;}=oO9?IN)uCULuR;Uwc~8CiTAi%7(l} zQ*%iyD7PLI3h`#L*U9#UVIMMJH4Q-7EHFgsL9U-aTc|wpcW43i3O&|$_0l@U1m97Q zx41Lrgu_8~Srj+vu6<`n2fCi$y|$vqUW$IQ^;yT61$?R91t&~Qt3fw+Menh+kR#hb zd*bqkC;eA5qXBC&ZNxL@3}MQC%7Ien75Q~VVqM>gB-q=`lPb?R&%1a!Ew_maw{9B^ z#2??{XJz}<%halgx&b4|b5%)jNoN$F* zoJw>%rFsV}($FzdRd2HQcwg*uuFWd{@IEtq1?*a(JG1sjPN)HQ0M)Smx@djG3@GgH z3W*17rk^844$iMbfiJl>!r63lTp-hiWriXeoJ1$wIzavzZ*{=amT`erjD~qLNSmN` z`mh(xeChe*8paz01hQ57z7n4BHU7i0O4#nI~NMUG3LX=QoBw^`lFrs{YD{PC}>9+f6#U&JBlj9F=wb46lC6yCpwi z4u<-;H6)`zi_$EqLP)1EGhhe>G`M@!!IQCRg|@wD^^poqlkNytf@=T}m&*Pz_og7P zlv62}o`)1roWm0IqnmCjSt_?$V|G-?k_Xy8;LXnGpZna)ZRPKW4m%N(+=q^-)_+0# z#F0?F@^i3iU3OwY*!shp=my(s;*Fm(k|b)7dheVJFe%%(Q=BJxn~|DPTUl(0rhcN9 z{@f8V;ApNF>Ju0egCq^T4?QFKZkaQ;2GVzz!CwZ6ht2@mO;vBf+boCu@o4ws=JYJP z&QhB?2pH=+1@0lFFnn5zGFE74=vVt1jyb*-)N;N#GGWaZUeE&2AzMJv=D{P-?kTn+ za7_w-c8AQUAc(0?GEP5BkuFC|#a5?Sd%hwvrh8-`;!R&IiS={2>%u)de<_w3c0xVp z!YRc;RCw@NOii3vjsJ-DRufH)ovV7vO(=&789asNDAiqV=4I5a-p(8K{{du{WRYlC z@0OW~GF{;|>Av0hmm7`auGu$}$kNZCe9SdWL_K`ze07WKi0@R*^l^7ukCSUpH16@& zy4Nyvf{r^rT2n4M4pM651lq1J^Ignil!TPQr!%r1%#=|FhupCeVs-}`7dcBL|5Z?w zXzp$#>gtqU?k+sWl`n@F zO$x|oVByiv-qnSk5hJs==WTOs2bYn4$4pbsO%bHNJv+tYv3*?SLai z*Id$>a5%okp{1b(`#t$fU@rRjVBnsj(96=#fm)jDA+I;<|$qo+^k`D`G-Sp?xYMoHAMaliP`dIf?Oqj ztFJ#i2;>dO=q^pFs{}D4x*xZGXm&ptY5CSDPv*NWJ9#->t=%Mw#h z{!~g2{i@YeBWTI|i%* zofDtktP+E5F3<~QT9PE-X?@KT)Z?n8e5G8MlBz-8?ow%LH}S)WW9?@t2ZUzA*OkL? zLIq6>=3|zqKxXb&q1W?B`5fsMfD%OUg4;Ef)*04_KlBUFFg59IpMUeiIQ+F`^a~ml zWU6ZH*|1keubnqJi5){k4gbZ@33aCsYL>%vhjja()Z1OA_uKnklT zw=c%p)(>=%+CI&0q+!acoDmWnURIwFG#zvV3SAG`o6-h?C`vE@h_0d3={_N|S!N_0 zCdIA9n>No3nhe_5R!s`4_C1=;`Ke@Pj-}g^L{~eNB-MN@Kxb$|iUGH3IlF9-Y1v&` zSfs9phw;T#)Ntuyx9?jmZWS)goXPaeIii;HX) z^7&xFE1V{P$^%g?L8JT;S)eE|9II84?kcMufCYr5*w|2M*onU@_-uZOIgf{B==sa5 z@ZrW#XKgZ$R5d&w&nwMuO(SE^c*q8JuzXm)IC#1N zTeRzU2v|N8pVi7-etnpG6&!X)b;d*T{Oyr-v)(zGz$N`t7XSf(`Wl7~N-F*1Rtu&q zVPH+B=C1dK5tTCR-xa}Z5y6Ob{KWuS;+|0JS`Cv&U^!)c*Zmq3+>#KCmn4)Tb z9*@0~8j$lfq~5P1bnw^P%RFwkc1~X8q~Te|xs~`MgTtRzAZOZ?0Gz{iY;AMw&SnO? z=oi2IqbGQEnGD06jrtt-K0kU>*KMaSd^GKNUUYYFRL+Iu$}ik839&8cprOMn2lHzO ztMlf znSQL*l@0j;ch6Jr_PIpo85lbBYNDv=gC(CTvr(NMk;xyT`KYe+Tv&23{_uD{<7p!Q zyL?8x1Tu(N8fQ37r{eRJo?b}7x|lY}^uNrtEB+WQ?^-9>_;wcNg0|n=q&WSZA0=t( zmwdHgx6f^Z%?FbdpOw_s7L_`epWUw;cICtB*j-s8OSm{OReD!yevK62NR4pPlAp&s z6>5ycO?a)4sctPEejh;RBdNE9mhE~ceIHOkxp*j0q&w+fZr^WEI@NyPj=x=_&DRWVCB1%Q&h{pgteyiiVH0Oj~K%vL4O+QPgK4BzS7auy6mx6 zXFwk!B?ow=1I^91@-*{X6IC@G{J2VY*=45)KQ(WM38N{D zo~ZD{edNaBo3MaK;!4gPd3XoUY?^r#9uAs9{ysUL=$j#Bt&w>tKYol|uborxZ5cCH zavV3h1D2PMPQfQ*no1`^lSxO&Pym>1wg>O`uaHl~O^j}<6JxTW7xPPfTY~jgEP5Ix z_tAWIE%`vBxL+)ua#Y$K-*wI0wd1k2jB~;Ij+Jcjvv|<_IB4+^0X(a@0Ty* zQL?L#^zDxc^IymJwR&!9v|m6(xu$TS>s!_N%F0Ya)}yU+4S3+ljg~7oNedjkdAf{4 zBaLLM-<*BNSESQ9zl^lh-`i#xm$LbEbJTrkz$c7ICXz{>q3plv;B9qEF?iXO)^`?l z;;fo(n&hYv>GF4N+;&E(m2j4^Z);au`W7SfFG<7JK?gaEiyv793NO(CYkoD+e~83F z+l1I3pT04bJInxhiEo9wK+4-sJEZW~rx%xy;A@2Sl6~tn^#|~~g(*;3OM(AZ_hm&y z(|)X2@b$}Z*xPGVcM9{ATe;^1eNCqNjYf6 z2j47AIl-qlOjY~1-m1E&S%aI!Cme&lXY#yrxA!HjHyqLq_)HFXC9F;K*Xr(S$Mx{3 zY5}2w+brY(`U*tHi64bsaYat59`aGdDi_?c&w@x717Jccsp+ ze9Hg&9_--{@5i)mQDxOh_4A6pMe4v{^3OI=2ox*FS30TPX;r8RjwZitY~xGpnlyph z{>l?<#0@r!Ba@|5C42t#p3k zE$XfWg)}di{RnnbTQ&{2{9hT7`@ZYc)qm&DH()W~(wU2_1BIABm~Z6hGEE4?-Ek!( z_sV|Az0=vC496SrT>?=u7XVM(&u<;%puYZJ$P{|@`hT7tP5gsrIBp@2lIY#1V9r-N zgYM?E6@1M=-mpKp&sVw?t(3DKJhiI z3P~7>BuZx3;vT%xjB?Zh8;p{9%o+xYcJ10_VdffuD~#r#=f=QC2zy8z8IxS+l2J7j zC~llkyMiWxIbe1;x=vjZi5c-b8pTI*!^eY!oIaVklNpMfL#d>cw!NT=#eEtr+)Z>- z>lOxeVrOz?6~2+$GJhmjorYZRQ)$7Z2W`n=v1s#=&>o+1l{N;u$|cQL+$24$%i19f zyJ^hLtF%(n<%w@bf*-hN{Qn$HPZ*~DT1A#hhEOOpJENJ)6mEMfDw|M<=S^^48&acVn_ zURt2Xrbd{F_!bNc2JG(#;dEd1SBu|DhHR7XfACPh#Zt_>MU|z$HwIjVu`&$3=V!em22y)|e_zdGLN;*A1 z0lE)=YMA*1MFUqBLweLzZUPszawZtTP)(w4SZ9qqe7La4whl~>FM<5KQ~z+YH3U)% zKE?H|o9PWSE_Z=#e%}G#zJ+?~VyljHWzX1Oc)ee!Z{0du4-OK2YTG8dn42h?{QS(6 zUm7=m&f^Daetp`q_SvULMK@nf%w4v2DAaZe%NPiT7Yt8wY)L^&Afp_33485nZS!tF z&6Ph77rDiytZ?|eNp}*R;(d)Ae|YVz$Icr9kJ#0mQ%&PkADAyS=aOQF{8tMoJKnBq zY;E$|?>eTf97M3&1itVaEZRf26&ByKFU77k8aih(a`2}`@S8R%Qi=-ey$S7Q7?WyW z&NqZk2Vq`|*ApeRrtOa_R(=d5Smd0+9$(u3VH45;Sg=?V9aW0jUcs3zJ&`UzhUjxn5z{rx$!0EW`~=oBRB;22)cL z+oheZui9_U$v)KJHthB5kTu|1r6vO2ijXsGRkub&}1P? z{w~50_Hl}IBB@U^S8ke`+`Z&(6eVr>nB;n}@OJmKJWLxGfMOEB7AeUCaWG&5 zIlYmS!A3JTCi5**`i#Sj`!nAv86O*5#!CCak3E?O&^-Srp0f1sqU8iu{eKiw8{mAuXWSEr-Ub z;2iO9HCEK4Cy%(uLEwHgO%r~LqiIlT$=t)Y&AB0;`vIw&)(HS~n5xqLf^T!%iIsQW zjbGe>F{WeIxZpI8!E)M#TGSS(vGLhzMeg^XTIuSj9(MFHNQd}~eg238c_L`jK)9PDFLw*PuTVi6@Lltv+C>3l*R$&8J2Jhf$w}+O?*2&|R)qg4aOIz5rkxq5ulg*8KA;&}XWhe@Sxj4c8-> z6pCaeq?pncwss~gDyPWFjFRQFcGB?hOzBMVL|Dx!1+#LI%W#?4$y<9bNeuN5YnC7r zg2HuAxjm)dzhj;^K@a~+NA@kbpKxO)FM;aEyelPRe7QgNDe#)H^u34clJd@DLbu!D z?mpxkEIZ*ilB!&NpsOQnP(8WGg3v6=`-IH&ZcV*)x00TY=Z0z96MRD}ti%_sj~ma5 z5Qob5GbqEgzn$YZO}xd7DEzD+K@hH-Gb2CZd`KOF zHQ7;%1nKt6WfxK+zH{KCX1Agr6P@yZ;7^a*dt|`cH|LQ~?=nkq|9Fc(ojW1_Wtb`@ zJ*H2E>_O+ufuV7bP5F})sREgp6Q+%UwvxyQ zr0W%AVOowVo^Jqe6vmQcVx_e3%r1Z-n7QLzPmk(I$z*)x@j9Y4-Orc)kv|FY-tNdq5v5M49;hhHfa_S=6bbK-cj(BLDHyJW+>0j8dvbhyXp4E z@$7#-`u@9<^WTh}|KX1}HTHd6^Y4Q*T(SuH+_AU^lLi1o?MDYe>gr`m?eag}JiS%$ z0=p#uqX0PIe}=#oy91E=#>VCUM{jYExpMjmOb>yBjY}8TQF)*sXkp5afG+YHR*ev9 z!>iWc0>dZIv7rR@x0QFsfC#)ORzSjg@?5t(UTPDU@jBY6FWWi(gQ#fk_NZxF}E(%#;U8o9DN--S$h75~|E=)y+uO z`LLU?PUggpx~b_%NuapOtQrG;2Nm;i@zK~puo4_d3UMxUL(!)NVh1GzafU03<8F!O zh(b_*@mc+nrw**9=|<)@#%?T?VTCUH4=eA9*3!}f(i4hJqvV$sY)NsbOHz^Y6AT~I zTZ&r&3P}X!1OB2)bYYz98rAD7BtA*^uUQSb!)Z5E;5AY3!kWqtANE1&_bvJw;SH*w zDQYzKaI;65C)=bYT-~?U6;a68`e*~Au;m%Wt(ja$tpnH03J8*Mjha@15sAp*C@%@J zQ0!m^C*pd#RMgei|KxjbJJ?~6?iaw5Y1>KiGZ4;2VTl2Mq>jTK5WN{?>X#g!|+SFyP-^ba16Yz=AE)~tdr zNvO3E5Wo5oHvd}Ivm*31Tc}Ud8INebGaAniJ8bMm))_vId)M$325Uz#( zc~jPg#MQu>BB|8Wi!h2q{Jud$r)jhPH~rH`A@x04x80hw2L&+Aa%7mJ5;-&CK-V)) zQ~25g8}_45&g%yo_YZkoWWX^_^a@Rhd?xm3brtmH@t*iJ$!L$0UO{kDWAED<}>5FcfN zS3y5!U%_lu50ef~zfwc84x6UDIcaeRvokO-DNZl$B&E8qs#_Q zj2-%cgvn)B?~mwpvCgfn+d(?9t3i!FAf*x~!jq$D*F^zDo6stgO=TxAm@s{%OY^AC z#ORvs+j9&L z)4!eulF~Gr97e0wr@E>|=(D<5E#`i3ciI<{F>gbJ7Sjv&l^=F3GA>Zf z$iSa@(9DN<1bU0>lsE7Ea1Z^)Y31eI9_AzQ8?4EjU9r4?NBzzQvX9`47QiQGa1iOrt=HBrRoJ=j zZolg(-U;89#E3xmSJQVtkOz|Q>tR0s;z@!pD9U^&ndf!Am3aLDn!$#KM}&EXIoqgv zxkK)+mGjDfBM(sUXn#v2`pe^&|BSItdcP1=s&^NYy6c=kg; z&E2p9*&1U`N>^u)>{F}o!W{|qUFP{u|NP#Z*8bc1c}Dy-tgLyCTp6+v)b@6}Wgq*8 zxkIiZjW$qUMjF$6Lm5%Fr+~KU(bHQaXBg$Ajz4U9&Q#gC!@d$ytFHQS+&KtnM%8HL_p@9doM5I&SiM|m#HR( zrF1RL$r-eW+**{j~S0X_H>ff)CaXYH6N8peoqI?)(+a{Yci`3Kt?@b zRUPBm!+KllG6@!he;z%E150<`?+f@sf%8X!*}=*L|1c?^O~%LoBniZVV?$qW7hGTq zUFKlT^r0rYIl_EAxMGqaX%Ej($CI{-E7^$aZ7F({h}X9(QcKKf2z;%$-Nf;Q+ec1QP zOaFi9;r|b}&i~JK^nWNqHqM{4mqBV}+#&_Yp9d!-4vPiDnmr%auX}n_Uoc}&0u*hK z%z6+Gq@yn(gkcp{ljJ2`7{_OGiU2Gdau00O)oN;J;Mmob*K)d+fO);cImif!u9j|? zm_-6FCx4<(+Kb65Kze{c7KsL(3@!xn=z&yEJUuOe+kOUa5~BIWZ7DLU5x+q2B;(guGYpZ%R$fg)Ty9|II05=?XZ_BB6axu-Ma z7NSlPAK4hXw^u7r0xly$6L)I<*VuS?t*hD4<4qK7TP$sYGvCeNstoqi*rCrRn%a~r zNzr*%_8)1zIuwudA5z3Ak?F5Ryl~oGXePM%8i7^BYa)fgnwjT78DM^b9bpnkgl4Y4 zbyT{s?Y5eEvKxjq=4PtSS5l(e2iUXVfzoOu0&1*lkBg@;{Dg7R&lW-Yx$2o!88{rh zVdC#(Rm|K3FAv5KH$!HSt7ICtOZ#z<&@-zAr^ zrnG4bcqj3!#hJNQn!&5&GZBql94-lm4JV~U^SLadZXaiQBA#zMgcdU)_b9cK29SII z4)au0*6DOcX<#v$MPwX<7%LOJZsV55ytCbR5nF*8)}f|U>uw7zSU(u@fMsbbMjwju zjOnp~juBtVjv5wbvot?^4&w1)caWdw4Yv6d?E?Cfq_F+T%NuB%u)d{mcOju896z5{ zVwSXN{nhj_A*xk!NB-CT6SaJ`aCEgfp*3R0`j7^(NE`TUlyU(cllv^4$WbYUU(+iI zUmNpCv+Y91!K?T-q8LT*(VoDvAFK{Z~K1DUA=fM zg5Qo#ock>O6pxk6&HnNI%d_HvoFU1=al#KBhseZw>DduH60?#dpI5?srk9UGBfi_r ziK12?N^L^q>Rz^?mZrP=V0P9VFg})z)A=#CXQ+h|O0at5us)20~#|pK_ z6i~2hW>H_-ljjx=&m<{$!~G|vlif$OG0jUo3(hDNdR7l#nu6IZm6{ftF$AfXB<8d4 zVNqX>6AIsCI#;O-rcN-Ngd;&|{ryqN1zGUY^rZccCn|k?d&wst_n;=MPvV|X+0w@i zvpb}uW9K_Fs|8X~tEX=U}ua^$G-YIR?15P7-m(^|c9M81nnS|H9B)#ABAq`oEI zFb&$;kT}7g4?qsizvzD8Pxw;WUNcgH)^A3=98PYs9~XS>8tiip z>sfDCxRuYCoP-=Dyl3sdI>+!I6qo3H$Bek)Jskv3l?LBC7Wy2t>Gb+-OqWI=^t|MS zc>b99^|a??V#`8N=EK9IP;DuSYR6m3aT!>dN`OGB}W(lUO z8jvq9QxAJ9TT5Q@gfDIuyIfPlyGl8}I;0QJV5;p`7knL+)Cv9-shPVzaTTvEfrKKP zL$}&c#=i}Y33rsI^~f7%&W>DzdZ;Im;%Ks&=RMT47=zDdr=K2_CnO|%HnS=5;IhAv zMnIl8Dz@ml!1}k#q1F7-Fw;kf7o_7irP+sWhjMdZ0`G$b7jFt{GW7Odt%Bj^1s@X% zhNG2pGBmSlx%8MSxUnbknjQJXE^~6v&Dvz_KNGBSC_zLFqOxY^M>&^ItYy~l&Go=q z7>AZ_B)uWbUGFHmz>B)vG{AZcA2bu)IJ`6X3&tkbrgvF2{Gk5#(3_N{Hj5 zsC{HJBkr+1ll?~3V zvqaE#3a0R-Fwg#(Ev6H%XFq9&xy_)gRZgpOgdS26?n-9up(~3+Nzaav6_q#{t2#W( zBttKKc0VaPRY!|Jxc8A=#)O+t?D7Z)gBzhY|U)w;oBU@h`uR*g2C_ogwN5I8<47!`Ee=i@<=;OPwOhND*4?F1=#N=%K*gd z8MOt08sLXU^Y)u*e#H2%T9NyexjLEdOP$n`YpCa%J->M zD&!BxSHTUYkoxjrGn{>@9_(&s^h^PM{%^!O|R0ivrD0n6usY5Xr#z!%P-WTTghDXA|zzw zBxu`)4Fy_0kEe4iVIlw?Z_}w<4p9StXMqev*J2RHW^y#k9*Y4(%#eF|g*5=~{{oA2 zZ)VXVBEh(c=E4yr!^4L+b^H1Sj4!x}X%jKjme2-j%hdK{J#p7jS7I zo&Ae)`e&fY(GKz^sY$Yek}@N@YS@hc9<`t?i)?ZKsIEX!=sq`N(bLC?%V(E%`$ z;-@1THrj^&{rCajA+*%~-qc1$tb`u|HymetP?Kj4Q&i%fJ^RC)&)uORcX0(D_CBVl zVV-*{s<)_m*sVN;f^V_9X!=NI;3AU&4F-`ha%-8|b=h@c5G|+-E$UT4!^=KB*Lh$T}A4VgV zR<-^{u1cy!NBs4-e-5TtvCY-66h1~6;fP^HaNfk~E`>+{t9>P;mZr`EuG#D@y|v>7 zJw8Wf2CH9gMGc!WvafT8&Q_Z%CtIhsg|($h5A|TksC7t1*EME`-Qs3K(kNNqaeBWe z>d%gi;j`NA5U<_j+p8^s2yAaoScW7%j)pI!X|I}i-4)v4PTAD5Z}(%HZNWEJpg}xu zDTFs5ZsJwlWK^+Ss{uN!zX%S3zTCR7sP!F7?Y`U%e&6N39QSq(KTB{Dk3$pCh zSA%zi+=jiQL-rR7t51Q833p+D>17b^>S?;(^SpgSu`xR~4>aHVSj#F{isd)#3A~n_|itt%s>Oq4GB_ex#8vM9F1%wfQW6^V8K-N^y)>Gk@fkrJ5i2&}}K` zLrpR(L_i5#xexwSofpaQq`070v-;b+h;ITLmO;E47J?7)Q4YMC&*-DyWfC}E6oB2^ zeLyPVH0Y{U_>HYu*Sr4}@x$B1JbyU*BWLv`?_ydB>}^@}tl`&T^1kIMXv}F1)3;G- zJ>1ASK0Bl$yq9`I(Lc=ZPd|sBE-%?DPn!Nba0ypm%Nn<`U3DtbIWBi_ z?Oy0OKZ;jieCx_9rLHbVAEx7ZX6?h{6dvOJa6x``LZ+jlsb1FLA-PBHji+qf*nX+R z6!2_QjrMk@uRRfXtc&ZtfV^&?eO*L*VFvx=(8TjO8hMn#`QEO;FX2v@tzC2F`w5c6 z!?I!QekT;y#HBX}OG9FIx7=7U;CU2;%bucypTQ5RLRtR>JJ#7Le)a3W_U%YBI zGh`EuDlO$v+w`g3Txp!dW#VJrZMpEo0Y8{%)#-k{{r9h>g)y-W*ob#YrM?qt<5gJ{ zzfzc;(aAT}OT3VFlG(Z04U_5Eo5o#w!tJ0M$btH3R-EZB-Lh3;uw_qm~0C_wRcL0JdJH=-cAdw*88OS$_=?tVi~+` z__cF&HFv?N1E02L(xp|66^g_M&xm6f`K0+cA(yvp`x<)8z-{r|y=8tf-b)VAQx^7l z7QT}$TrDDa`$WzGd!cZyuL>6N`m^x;CE#_nb9UwG%{P)SWD&lIsQhBo5=fh|{O(9K9LUK+ zwbBCx`>2($b9Kk6y(>gGN&_8pSaiR@8y{A_T=~i@KY66e_?$9^4R8$;${>LeKF!e9 z)KUwe=lYer%RJFNBdEUmXou9b2&%^^EP+ikRu2FkTVgTnAxS}}zg03;P*D7Od(!64&9dMK*+_S2{Y|mNi*D?>%Unr^ zcVs*-#aA_+64rk!MTMg}SRJmG&n*QHuA_U{)o<7}j+Lc`LwA-fs;Oj!;!yQMjNz2C zV`Xq#vd%IH5}>mI-Kv(;apb)+zQJS)c@BIw<0TLC6aVAI|9>wDtvZCAqLSABIj7jm z-wZO-vf=Aor2m`+@euVe@GykQIj|*=i69lo#l3y<4dhS2GW)YZTTF3{T)~3M7RjLR zF7Fdq`)xzu$bE_<@yt-uDi${a6fM#Ge@~&Z(iS@N*O+u<`mTwEX|phf1;qi#R%6&~ zNy}2mza)S^hlb(y^^ju)cIk)Uy6}>r<}nGHxa}X(h=wZ~1Y~~G1M4DEfwHn!v<%VEK#4&bOT*LM%HZwQyBLj+z zU{M2lMC_nmhm9I*2TFrMGa|)T%6CAEiP9a(6oQ-kfen zjZI4?fAhh--R`=XF!`kQM}X#Is}qRhJJd<&JuSO-Q#Q{Q&GMmW^9Og) zGq7ypzC@SLzi|`$os5&?(_L?uE^QkkzDS|Jw=tE5>X_FKW?gxbmN_AtQ0F_CX3X0` z>cd-Q!!D~JUWr}BBxS1`%TFci@%f=HjeMJ+L+ASC`}RRA+|nGN&%#eEq<=nKTZIR$ zxXI8TTO9ok)h=Z^l^~y0oR^uNvj2Zrd(WsQ-)`GGii#DiAkyrpfT$>)L`46nDA*{{ zqJs3^TM|2B0SkiEpn&uyy(B@AUP2E&LVy4vgb8Q8=${TrUJr<=Q&x8sSQ_9pRV9uAUArL|_$yEI65>lY-5D0w$z%NGCs=1N%Y zr{Dk!Z_YvcqU$)Y!fZsESV%7d2Dk%L!1#2 z8D9ZA?bIzO$4qHd*#_f23_~D+f;d{YijMZ#rRR+v4AG@S9pSRfMnT|d27LixRiigV zDvE819B6frvpxi~zu(=|@FZr)LG!8m>?{`IIzh5^=##l8kTp*HVQW_Fq{)^JyX*8KIYw0Pho zR>|RYu-%*54fCoGVaI*1dqku%N`!fmz8d*NB`;XD^cyop8P8lRCk|G~5`7}pIoVh+ z%oeE-LREPMl%yex3U3$mg-lga`C!8j0hkYj%1w^ax@T`BPubSJ`!*@nyERxszruNj z%IqQoOr3yry-%+0ezRfyZ&zGTvrh|temz8{ScMp4rDBP5KYI^_?%_UlX<$^n&@lo9pf zSf9NRT{Q%c9qtJ_dC}?c-n_FGw|^{XH0>I!x>>1B7tCiMx%^{E9!2Xam0vpy$+kwQ z6@_SSJ5%qibRTmY=eg38IfN*4_twyHUF>ZMDOR07$*I0|1mlB>*7dJ#~sNiYvGH^r|9V0P`;CU{zc zC)Z-Xw$x) zf8LEwPfeV^VQi{lJn#TE>{K?Q>Cy2mF}uU&&QFfn6k$@T?(hvq>-?hxgZFYvi}j2O zchQ9P#v{W?w~3}0;e?65WxKYWUweoLQ4}kOkbN70={tZO$RX*{k;!<`3e2uN|ix8GbYMV9IY&u zu^h4yUpT{4?9}uYA9nCQhw*;J^7$Z;8k zWG3A`C3aG>mT`&CZqUPdG>~n%v_3ee85sKIdHoIcJ>;k3(r4w+)PyX*xy463P99E= z>=Y~r9eIzI4F0)EVcq1uJ7@~eYW4TkCHts5$mZwS3Tu71a#T)mp)7V)#F}Fr8B)7V z{TbtIC)d0T?Il>z8SOTPjZ7IzCF&r!sG)a3u`KbhLW zmk#_p-AaN?1i6bLJFQmytNwV+^vXd1Etm_28;7oibWlN7V4~5#!j1p3v{x@&ps85B zHm-`hFMn9~clGw^>ZP}*#CLCgo}7fSUa5fpQG$asI`N-u(cmbNm@POhxzVAa$(q`r z5TonhLRVj3zqTTH_BI;$7>K-4(bWQeWjk?c$#*|0$tLdGqbXd6mG>+{RXuDRQ#1m9GT8H5Zlav&Y7`!W#Op%Dxn$6g6Bd)Ugfvty2 zF4e1JK?L6$2OWS#Y!ASy7MNiLrC_dSUM1I0Bm#gY4rh*!q4j^e8i;x!Pzqevd1yL~v!}-z?|^BzNX9ZuS>Nz{kK))f5=P1*`HKz^>Lq zes|~t3)f#Vm~DjS(GFBzn3V?y`6LkYP^O(}q+=9VyxTi6y)lrJ;GB;`Z)Sac<483A zkrc^R8n!V=yhp$01HTx~EpHC>4^6IpwrFy#aokePjW_GQ+y@-e>$lFQMB5J?yxc6| zvED~fIB^7S51sqownNRvqPu`~v3WVx#=_Fhf9B~?vbc5V8jxv7z?YiOzdulAq?&Yw zw(E6z_Zalkt;+Z7KXbmxUTR`BuR4K}otwJ`L2~!A}v5G3Vpv;I726Dg7zh z?jfHclm*2+1VQ@-1g7lh&tJqQONTb9_D`L&{u%s$FPZYD$|B!nAmWX}!{;|oN)#*q zmRI3t^`fSFn?omt33dL)zZr*`beeG5-qAugN#)Y$x8mAI4yp(2Y+{D&#yxbG=IMi=-5uCQ9MEJT#(Hc>iO8snH4WM3p9x{D;p2l zX=)X#S4=Vn#Rqtqrm@HdAZ^&QzQrup1t0c(U)$`dk6-x5I)rKRCaso?6xaqa7y+Mu zdGp6sxXP)J%jH4^!?hU8i;2iblc=H5HKv#%x2M-jCz(bzMX^+CIcpNRDJ3UN4=KkKTXahiZx0cCWypWZ>LrfXO~LY!$$J{j z=J$Lc>mgeAz}REamhK=qG{k^ZEc@W-O?^Nz2sY4Xn}lJ?$wro+OV|G2?-PI+WV*_uM^Y&E237wLuB%x!9Tip zF-qN4<$xo8HXS>W;pL1kl&@Skssok(5Y%ub;b1p%e53*q8eUTN{y38 zp#0Fes98dm=az{NRM(%wtpjFJeW5G2>MCLknZ!E~bVh}bqV9Q&c|}llEn@b4kwd&` z9J2_C%8$W(R(Z9@MQD_-k8xjAZ3-&b#|+K2*h)rdigzmJe1%^SKH_8E#q4d*pY*hv zU?|Ci_1rQnBa}!ga~0l3PuhOJPtUuH>6*xm-XLhJ{|Xbf(M?>%wiKsWx-nDtSF__~ zq|F3?8z;xTQp_9q7O;Pon!urMU^pP>`)!}EZTJBa>JLHg^(w26k8S=U)OHf;eEaYw zZ2N;PQnScL%QAo`iQJL+7M?jf-pkI%U3{E*&8zG)xxwU~m%ma>cVFdLR8Q%hw;1^t5>3COxsgBVrVio@_-vh^H%S96Z z&WQfrO#WVLPOSu(HV}4kxrc<9UV=6sn5@@Ah7s%$NMD}eK51!@0#?94ZrSL*KcTek zpXY7;WWTQ3l0@Cnp$Nfin#cj}P+e?3EJAxq9J^n%U~x*u~DwgcR3Itvg; z@~un=sOLmpB(E%zEg*66>}}=fc{Wx^UQIcJfF&ka?yTN7)dsmqtgM690iSVVxCiW^AYx?o3^N4bxo*S*|@xIv!_ZP|x?0MWW4?lQuDPu!6C7 z`VfeaRSEju5_k9*_hntZ!5%~0kMvKP(Wq26&TN$Os3;yL4_PpdA_*&(L?sHGj-T5i zJJrH^?^c1HtWqzI)m1r-+x9~7(_MklG4-^D#`5es#L3=$LnYG147>*}h8n0qc|5sS z>b|b0{MJc1tb8LuG^8RdD@vvLQZ2X6UUyXCjlv{*6UEE!CH=cxys~Vh!t&Pl=V9E*JBA$iUKNFuEde2YwI6BSZF7jz?{NyhEUbjLP|VrTYr4Z< zkTdLV{|xzHPV!>{R~P>+@ZrkjO zx$tDSNW6#YPxl{;V;@XU$r(Q4ffam!ODX^~sLE{ynxE#4Lz4_WxJC&z$? z{qT$|WjErVoF9F1B!O|_k~5$6Pm(Y>XtF}V(yinOa_@$BjZ zyvfiBlmHx@c80G#s@;%Yb!5Y-3>d*?M`J8S7wt&*DX;9=Quj9WER8$Z2_tYWwM8)5 zHu#BDQ{_&ZMXM9-^63fcifQ&REmyWPw!fO9eVsVEQUylf^Oa2Z*-JcGw_bhH<0LCg z@Hxo6$*kUlQl$;d!cyH@`p@Z?1L(|iQq+$6UIj`cn&M`2JK6zE(Kj+fKHxQ6IfjfH zzT#(N)>ECtyHFpPeA7yfa*Q(a+KgU4ijB?_lyZ94<58&52%K%|J|U=UwaoNLZ7%6r zUiT8c7^Wgo>vI}L4>yKDPL}_EvOj6r$`Iy33-?AQ3I!zQZ=^$LUV6U%0`r1}@C2x= zP-;LZY^eI{kPm8}u&hHF^1y+&6Cm$E{8TmlA4tu0^$|PL@+Uq9% z4 z&-F9B5PUiemBU1r3q(g6fEQSgsyT!S3>{yb3lz%8%LB>LpG2)cs{|yF=K^@=s^=Cm z4^rZjiOcL!A_++Y%YQUnme2sQRgK>XBDClCRt?G}W3x4xbK)?rhM=wcQue(P@O4B^%Y4wzi^7+eH2*Ef{l*GS}jr=_o@38@r z5I7zVB;Fte^$#XTXL*!zZd7=Rw#;q+m8VPa$fRh_7u|#2@wM6m$9vTRH~QX-=8toqjc0u?N@4eO_|A z%9ViP4H1CEBea4lqP@_A1GeU#he5}1n8EGq4{Tvr-Yb`5#1@>_EsCitUh3ce+q+l$ z4CSLi+HUiRKD+&*83i)bbFOeCd_3UNeAwLgAII6BvsC5gafZ1gre^y_cfP^R$tpSq zoIR@lWBLrNZU@w%T(0}LD$MP*(CLUG(Im$D!NAp@c07)Q&Lvjan>A;q(mzWa4`ZA~ z&Yu$5M9HPaELeO=I9!U_L5hyNRKS0ko#oToaF{Hfb#5wM@~nE6{(Wp*;(4ReX$wB= zl-1Uu<70@nL7&atkVho;FqlHjpaSDB(I#6z-VnnKg#>)H3Q|16ND~i9Ovb_&s>%PX z`Q8D)wqxq{%j<({Qn5x(qQO{P<=)vf1kx4l0`zp-T;$ejc*7$DUJM3pUo3y*+sBgO z-?xoVTR9k3MT~3$B(@NV&z#TZcKIK90K$@2NNe7bQ?0FIM{2I0?lVmojhe}HjhYOG zc3b}fkmL&Nz-y+baCnwT3V1F>Fqr3=4v8EzucO3uK%Y`KMJvE}O%Z$HrJ^lf3TUEa zn!L4kXUVJBTHAll={yxgOnOBx2UB9@?3=>kGkYe|hG3!xukI&v)=s|=lzcX`h@#0R zmF6-D>34C}Y-91liX0(p`bO#!=~aOslyeCPGpgzG#;4S?&$Un4hOmYRJTHxi>lrp! z?;q-|)cY&FA2d9n#WURVtnJ33^ZkoTf3J>_|o!` zk~8Auy9#^5BzCm~`c%rk@hDOfpQ4?K;GLP}=avTF+KwZ@Iz}29l4Ba9IbeQ@R&sygxvwx02ZS^3uNg+|i6UoB%(LKKOv;8wDpL}|~V34Z423TqcJf%FI1 zijexngV=UI?pL>&d@1yAovlN!BZ8yD$oUg@e+nF#&|Akgytr4&vIyWFj<*zSsg*IJ zuY8UpcQ?fMh{#AfxGva8w2N1Saxr0sJch=V?I)vE3Ln>!o5JrQ^G4M4Z%pW1u_&VS z?51v_{^<5EA|Cf2>8+3V5b)I3LXgWNpbdwH)JuZf_J}<>CCb|c1w$W8w^25m!wwbt zN;B1lMy9sRS2kf`yjGtz5Dz{IrJe$fSK+-sJQj4df~Q}Rre5lKR>m0iPXvC@oN_?R z_gMBhf>1YDs_wi;ZcyV=Z;9DZ#Y3W?lrQFA7l72)-Ce4mNfdV1T>Hi2C_djDb|u=V z%MTMH%1}|}u6gV9k$rWt5RvM=QuR|zTj$*O_!--SXxa8MKTVXCRYTq_Vv+9VV)fcB z26BhaQ8{HR%63A3?K{Vh>iIcf3PLYzEX?d8(3To_c4(Oaq?jgBl28X2WHdFInI~MAd_nq}Kk3dd4QKSUWj{ zkM#Ebz*@0^)G!Hf57LSY@cRu1Q*zbjr-ILv4LxO{mdh2P3Iu8@+Lp>;`Z<8)XC27B zrVmH-s_B(QdU#OaR!gRb-2Zjl``=Fd|LMzLI=AlZ`CEJG%jq{Z3Pl5x$dEtLoOh$w zkJ+q@OM(^mpp(L%ioUy=y|8LwUHFtAx*o)mo^AjPuhp*7DQ?3|__kWg5T$!47uZ=lZ5&5shJMhjzAc|&rTjv!N6rKvcgID8U8~TCo4<8Hupt`U&`J$Y+yzDR^i(J!p za2npmh@dYBy&+ts)CqR7)LGL(VqAj02kXt$kI!pk~Ut=CHl9-QvL(pBV!I2x_2GS%}i{`{T{U?;k7`PZj zgFOsh29RM&dBak%$=BA>+V_m%x9rlFvj$@2Rk!EIxpw2~dG%5J7!ASC&u25U!@9MO z9q41})5Ni(ME*6JN+F7Ou^W{qI*gBChSL;1F{Cjl7$yMSUB9h3qz2Sf`CsanarH?4 zhFu7>0qP+F(aOJi`Kf0M3-(ixp*-R6Ex*&r>rt)OKTS(YgQfDM@4`S8P`6)4uiHd! zHBxjyNX+R~+^`Sr0Wrr0mFrvJ^(-foIb;+qV89bIdET7 zbu^$Hr96pv%qqaaoa<+WEhH(Y68$yDd~`j#)RU{fk7@&(!={HtwV&e(T+(E)4!X8> z{dwKjRPL8+O{R@)v)V-2QQUIqs6lRy-uh?z(T01iOWXuC9ypHQE@KsmmKeW^`B>`5 zGPi`pn10wHUXp|%V~8fhRbQY7;&2ART_am(dyP6N4iPgkiqIW9Ozl45HH94>D6XO7 zwKWjA#@wW4MP;k1dRleJ_ryLcJD#Itv9(;~>brFKuUYimu6^C_Q9q!Uq5i?bQDxd7+4A#%nC|R6 zMK+3{M!gV2I6a%4d3Z_xAA`LDz%5{py7R7wpDK}k{m zz$b~8*iOM@9?l8>p?52umw>#5=O+!y@XPkRNtYgUIeSR^Oq=I>>!!QnO_!If6}$2i zk0H;5nmD3^-(#9nTn?Nft7;~x4(VPb8caDS2Fm4X91+2`h(2RjoOshA*5T2puRSR> ze^f5fLFd!_XVGD|;}SVvy0UaNmK&df9T!Jk8vZs)VgpIVZWwQhp1-FlNn;MsHJfPKqM*+{v;QD z6TzJsJAcez+T-Zu3juBsTNiCf3{`Ix7(-R_F8XxRvNllgSoGkZJD=Vw=c?2J8c9_DzSn(Pr#-Yyq*sI3zakvarfIZfogfO0D!)7s^^O{Tvi9x0^HNVsJrbhGT^M1}Q@B}`y2~~Kgb`Itzls6Z*TlYpceZ=`P4?)TX#U0CRd+nEWm8hSI zlgfne7|?pxUs4-1>ff$x{dJ}*Le34U`pMU%B%rX~j(SlR_41l>`V`5@&U4i=yaC1Y?-ENvso4&@1e|-Uf;v%`yOVIodPa{%=$7y&zY(&{t@g9#fFp0iDU5slQXa;kwA{hhx2N!szp#E0Iy&n{ zFDoJb{+-+%G2~ta%zII{jlAYMs{wdu`vo<}ecNIi3i+;JY#>`@$Lc2<59aYGhWzsO zzKTIVly%M|32*S3U&IfDvn#>1--b6xXb$b{=x}D7TS0j75xw9nf45D_LiPWTB5WrX zKCtd@b#@J1rw4&F0wLV?7xTi6kzWBiysr!kX0}P}YJ;`U%r89C!6$}dIc*+mAp*L* za0jHry(}@barnmW=iPaI9{wZkEx-)0D%}S`4T#y>kVmH3>meDY*+xRnwd>b4XRe8j zf^1}MKKffGquL^O17zA)3-YL`=?Ymhx*Z_9jFA4tl%r^VvnJ6TVXwvIj}w7XF|se8 z;=*sLoUzuc@DAjRtff<%ows^}OJi8oYWSj?zPvi|aF$@u?p|Ub8k$|VIPKf_)67cSNB9bEK zrrUMbA#we^;KVE(wxcf%jKpca^K1fc5upq|hl*c)EkjxsmS!%jX7Zl=K(Gg0s1+Ce zm#7{v|ACrGVIG6lY!RhY@HAkgdho#CO*RJeUnJxn%U` z&z?y16%|*<&qQ4vow)=ab*wDLx{Sr);89|be!v1XrKH2GgB2bj)D?oO2YZXckKfWf z+~u(i<8kIaP5WJXoO_q^iLf7fs@oxtWJR3NG4n;~rk9z4^T>QNYQ0`u;)~O6$_8mW zT`C+5x}sTgcP4oe?|;5Jg8CJ#!=b@}6bueuNjRM0f_eR`tssS(OpiS$66faT=tf{3 z+1gTsZv*=@-Qw)G!Y#=)?i*VbDoNmIM#L;eUZ0DE$PHLS+ig?61MwD zUP9U^`$Cbu-T8>ygGLt6iNU1sq%eyoH;=QQF)h_v$SyoTURes&QmR+bLoW6GrJi0q zvh<_yO}x>i2>~X5IRsKe9`*m=_vo-X;_iF$om}|U-mt=`+7}@Ylc#nU$r;QCs#Lvl z@&7d#d2?!ZBsu@kMuQt02cp8!H~0k=vx>51oX@i{9fgdIzSU`N1fwNs2i3$FXL;{7 zm7Cjk?#|=i)a{lfkXj&+5OF04L@DnjKQkL4g$JOO(r1G_g*&kNvaBb;7b$ethP;$w z(XFvlT|}l}L9AE3r3cj?5G|X`)$XtzqvZc|OLynRezVRbD-QE2sY46%3^@;$LRbnd zeX>@Y%O9D!t&FEpk{l3SZ5Ia`+}F9pMhchj)tnZ6xPSed=0-!Zn$?{bJ+eOM%`1sd z%ZAP%#X~q?^54!~88~yTSD5$fjM!q%Ls=1sIV)p2n@BuNVpN1F-z;>@(%K}eo@o(7 zuFJH-Ef2xA2b}z%aVr1S`G;BLt9z>b_pcR-djyxf1VFHB8TT5#M5}B|cx>`kL{`iT zU+yijqk5O|hW9esS;n^g)p0syt@?@T@!rilG6br)3Z+T&?!cP70HQ~(!XiLN+(EYH z)9Pn3r2{Mxq>F5YW2wZh2yDJ|M)|8W-?n>s-glgFrM|G8@hUHsM$c&PwBTL+C*`$d z6Mj}a#EvkyoKU-II)T|{2Hr0yrB|kprUhL^?cX4iUxx62fYT;Aw z5CA>A(Yk01A)FJ#s3;oGno2xYI8}^L{;pu$;M>wI3-_9yuwAr>o@ZC83GEqP zcCzHQGl_(rAhh}9ftxT5x9qjY59*fjpKUOy3g}64L`_t}!t2>#)hiH4O;_5u15#)S zCXVge>?WhS+G3{T9L<;k{c`qlzqW!P^o{Tx{cO6a$-AB0#`;somCS*sz%CI?4rR3!U z8lFDSA6-|-5d{-cj1Sa>r)tS}n7T?0=NS7dVc7Zi*_gF#{Gn&>Mv*3C9FuwIYcp%V zPqyl={I>&7S0O?*OVH3#n_^;R$F@BvOeC(orVl?DjJEW^^g04OT1Bg7eB2;}Kgpo0KP0CC7dlQDAVDphi#G3--qgg<)Hmahv6RDxx22oi*> z(5`&Ym@HeF3q|~haX{&qS6Ni>1@xiXcB&)$<6DuBF7PDL4I2`??vd)+Epwv}!2}uw zz5<$>5j7wLs}lB22;%5`(4SH|9W_Wpi!NnD?TlGw<)N}@aJ%$L0!rs+t?BaA25o!v z+%P+z8Vrb)Gm-JKC6qS-|5nXbADkVtpJ!S-iXROiA|Q{zC|nZqjo4rj9HIgmKY&F5 zk=cp7Z6V^Wl(ZFg6{v@emA~^vN|_h=a|r{o+E_3}1&Y7G-;W`WKw=)a9)JOMz2>NQ zi}IiDmy8U_gUg{t3I+_T@bEJCWlAPD*ODB3?E7(B%9LC65C<%rH$Xne#d+7Kzda<4 zjz;RIS>$SQ8Y!3Lm}4SwE{j(M8;{Z*jOPU!p>)b7IQ+>}Je$UTzBMP*6LG9v5&yFhU21xk=(a38tGl2Sdg3$Az;T$!xXTpG=*R`U=w=tu56z#t> z*f~oHc#$Q3-NW<_tvz`8^Y3T3@ONZcKs$< zmD;?K+lV(md($_%r8XeX_wuTVevCVJ;QKqtm%cSD1(Xm!_t32+LzZgUfQ{n`chH3V z_PSh2ogI%H3lCvx#yfEWxGAG~6}8&v>BQV~ zG5KZ94wd_=tBE|!Y@fVhtF15E)_^Fx1w(ci)#RSlpH4GCb6d@EP+W{ltZ^qVSYnpl z5NvBT6=C&FuY;_&<5Ppo_80pCqQ}K2PsGH#6<(hDBo@s{g$t(txhXzX=43GS-goZC zG2&z40`3=%NDivHj1zNAFeztTczB#UMSnOB_5zZ)kKWS@(lQwtq0`WfCpK8ps=+!e zK2btrGNK26Or$p~t=ZsRh_gzF#b9ToV#_r)VqyH6_8^vJ&!WNm(xN)? z*|HLVsl=TV2qYxZ$@@^x?w2@dp@>rP_=O?W&DU*)mmb>~f7p)M?G#XXbMhd8m&@;y zpR4M!D`7|o1P6|hr!XlcAtkf(4L7=jo)|&X2qS_kCmyS!ZUldDWQg~{pEIS0y-r!K zaewjk)Y?x^4|a2$yH$f4!qr_Z(smkZq2%{{tXd1I+x@-FIq@@w{2gWq9Ff)>omL~YxwS-N?o;UChMHa{pbUba*ei0~$(7gug} zMsATj*~N*=8VtC3)-O?1I`6V36qz*_W4g(8)Coy_uM+TB@^VyKnTp9h|3H2R>i%4` z>f)#>7kPJrBSjIf#AfW%qypQ@31a-t3mT{$TF7L1f-jp+pORO?)De(eg`~ir;oVf| zYkBo0_Wa256(X9$RBiS6&MXvQqlP0t{5kn(w|P-PMjlBM$8X+7^KEOPUdi%i%GQ;j z9;}do)WNeuPjn3=TFHy-w}Nm(z-kv6>;YEGeI6?gt_RLu*WGlD_(%U^!TSGL;$pQQ zsIB>1D5atIrv>5*N-e*}B1uW)f9`@zyj7VFXt+Yhcdf!}HbM68-3x)-Ik5*)10q62 zAsVw8DG-R>_5wN?;?i*JRRY5(owO8>!mJKHji4MmNKH`^6JQ8{=LK6L%wgE0 zkVhioI@=)yAlU)}S=0%L!0Pl;{0xvviB0+`ozurfkwLFPsZDj-3~?+Q(q*7&bX(F7 znxwXcRyM8##|#_spxQCNbBAbx_aw*KdhWnPEf;oWCpmPo`4i;PmoHzEye9#WfK$v< zLTQ(IJtSP8ch#|gs<~e$_9LR(aYT9aQMOtVf~OB2=87_%`i({$>?||0r?twT({^Ox z=Td13@gq#M;eQmm_!EN43zi;sG!qaQJ)A_~v-3TWjQBOz@-0h3sr+2wC;=|M`eUsd zifmux9y-J8#?LRaT)}u4I0AfHPyY+u@{KS8h>76gN_$lAfOrDQGz0?3duovL*}dL# zXf*CqHj(Rtvff3=-nDqgRcEMt2^UGvA1Y@wkYCK>hPYRvBpxxO{6WU;2G9<+(JIeRXPr7R|Pn1_Hoki5tyb}VtFQP}ByRhw{-AF(lf z>Z1lVJ_}hm%($?03lk-{Zi2M!)*tnr(&_bz`PGh}w3^AiHKg)pnBLts7mG|XC|F#osuj^HpsLv!c zI&5#1kH|%{_~vz-FRfc-VFzQbUsvDW`dz;ARW4Zg2X<~LS+O|T?T{=qcqzp5CEoR|nbvOCHMr}(*ce}z57*@^Rvk&I zhE?>9(o{Vs;WG0S{)K zoc`EuOe18^eH249TjoF!axaH}Jshg~QDPX&DN~C~_Q!2q2Kws)&gWL)ycLy`9U0Bn za_3gOVfcQhmRDW+h6J|+f)zuRt`1uUJl~&R^wMFqH@ebe-;m&f39Y8|BlzrIiL5vm z$2B9I{1iAMH6yz-PF1VuS~Gk$BnS&+;eA=eVBsyk&1>P1V+B1p zmGQPB`*IEp}*DT^-c&%+iRB@y#k#?p{~kZ^{&?5_&$02NN2y97}LR};k5H3rhLqS z;1sCF3I8vL_>R1{*NVU0F}f%&yZD{F7GlaTnu9aziUihV%0>61MOR7=ZLvb6G7gFJ zC-opT^P(qXckx`9#*ACk^QnsoSp&@pT;IvyUBoyx7Y%|Z@S!V32UTvMQljYA{yk#T z)rKbNmlnORPF~~sh$o`8mv6+D1^^WmgLz`O{PeAmLHbf)0PN{O8dWHKd$-v0eM*eto)OW^ zQ`%X`7;ed~T|0^Y=uNcNwyWERRSC*DPd(Z2;8(=+8xO=UuUr~vK!4}Y%ptObavJ#C zWN6VL3GsmnzB@18U5Nryo~N_XbM0on{I;L5^526tt#%34(`EUAIKC?%@rjT4s*D-P zbSr(oj1cmH0$V0HquQVtk<-@BV<7B>bHj7V=yo`ID6wHNkH?Az)o$SPZ}<8CK1=)G zRKZ4>okg4e9_<4gW%(L5)QTOeZQE)%Az*EHo^H{hInpBq3&w+D_glE^ZIr6B($G>Pni zFX82e@)y~KOa4}H?ntu>lRbp#p3zuzBiTAS0`jBxm|~E6tIg0euk{1lGO?=7d(C7U!s&?eos%&Zpi*CJ0aaPTOvIo01`MG zoPqR|-39(e`p1$I6@K3?X?u1Of6hS!fG=EBAT>=2z};zH0<^ck|ExrRgN+W^&-$St> z4kWD=lvf<9<;2yWc%usSEHEMc+=kSFgl4mpr+#czlbjWYWmVCCx%sz*p}Vmq{H~!O0!|KYFerl5=da z%VnrFkc4#a8{ro|nZ12lqEjKss#mkg8NPqLA;n&PX(nJ7{;-uk$I7f&DB@z)R(6ym zD(e(E`93P-qNsBw_0-TUSJv(@W!6#xv+CUCxhU;9^|tK_h{0TT!MuHBhQpJZDxG_h zca+onkc0T9tVTD3ZsW9Xiq|}xnr6>@$NBaeuYthtpuDR;uY+tgSj41=iQaiNI;miH z`vD<(nm%L#`_dIf`t?jT??(6I!8@3D86(H95<7h(M<~+>%$c;{TT~C1FEVBCyk0*H z-nyhwtpKp(m#54~R?Y0sgOP}Cl=s|y)2i!TB|@A07AQnd|D1zwSzqn$C?UT>Q%gLA zl{Vf-IW7p}il?y?1IH4x6O-)I2=2SGOFR}}G0MCIwxUDniy1!ouh!GfjOc zA6e!mf3(`iW%Ph<>2Pqv!y&hbnzE2IuC|TW+V(@OS5|hO?R^_Xmvpwel>r?jCnM~N z-SPxkS~+^@U+3Ltf2kKX6Y}!}ILM5qjS%%idZy*P2AX{av~QyTc+Lx z@;0WUZ$-SAO&QDD`p!RltP=HQg3;(*kw2&Op&;v|*Dce&koqP;z)Rzy6-#%o>PCI7 zSm!G7tSWO2x~nro=6%m0tNjE<#B%M;OYF9d>32r9I-#x#>-QN>o-d zHloO!m;X7XyzC%edL*>4`00yJdlGStCXX#awzjr_Gp}8e5Z(>#UQA<1JI%C`C-#;3 zul@A#`x$NeZCJAQ7E|W7#CGE#ulFAQ(m@SU9zQ-F%(7uzw~Lq(FTqrKr9H=$;v<`_ zhu<aB8d=TFK2B>7E?x{7)Ee<`U$QaHW30TS$8T>OEJsu<_Du@{YA{ ziNau7t%LfqizQU_0JX}!&0SchB5TWbKj9oxZfq@gQ~*R;~{ z@|Rw&X-jO4lc>FMwuNk>aeWl`(0kP{H|xEjy*tAKkIB*JJb$-}w^<6Eco+4qQw}j( zUzfHUo0On?gpFsRcX&F)H>R$F;w+K{FY{U=eP=x|1(;-eZ$qb!x*cxUptKid=@qg! zB-Y5L_jwgju{qkXV~azTC)sde&x!%h(ofxnaYG zo}>z|f;x{xqp=ARO?8%LHbbNJ(=fO;k!5;W-3|%yTosw#O*l1seZ$pCplc!Eh+? zB##)d%&H=?%;RfojAma`g{|$}|@k9ry0r=UuO#-hH36YqihK5t)j) zTYVqd_XmXVSXAy`z3Ie6>aGCu;qSes;GRB6EhVvIU3)$C) zw+6IvBdrj$%Z#h+X#D6Bxb-e96nq>-skqQQwpp@QpPlQ=Wq(&+-`)S}(8Y8X&MPE( zh_)FG`W0p0z75urdlXH9rS zbyj#+e0~4K-cRnj(%g_QG3p-{TCg@-e>T29WXDCADe-m9-xDpl3?ZK`e#9NoOG&+} zMbd(pK!5&n!{3Ui!i{S{YM#QlNz*ZWYRbaw_)!L1 zH`pgn^qV$iJ8tgv7+Di{x3ct51M@Q0rPU;V@}=5I{-s8xmnXK#ifi55_fgB!Kg!nO zLc_;Ug*Q7A(-W*Rq>9aik7BVZ$K8Hq>uozHXJ}}4U~98ReU@d9n2vGVtF04tle(jf zU&uOEtoPY(%pUH+pg0+?Zdj}t{wsc)A~QfmOj+N|IG5_6#g{XCKwW~LZXth0N8#TY zSG>USsP{Rg)kIa|4=)-;DO65W82l)fZrNa5vE{nUx1o3QNS!`hVBU$f)R(&^2Zm2^{ z*fzqp*fN%4Ft?UJ)ubd>5>&cgWG6c^|AU63XY#lCXwKvZd`e5SCf(DKYCWOLVB?IJ zNWMJG=Z7NxBv)Ais+wvfJLSG2H^bPe49}pqcZ{#mB?PT~OVKY89|_c*)5U9mML^G? z#igw0CYKE6ev%?}rdXD{wW&M&S`;P=Xcyd25zc<+$x;?^?*9HtU27ebozfdjm2Rt7 z&NN<$yUKAR#$UBc?B+-hnH|n_4ozpB5o<%0vEF(22NXWvFrHXVZTb}N5uJ8Iv{L-4 z!M1;*ZfyD(Wb1gjzB4-l0WN$Wd`|At?BhhvFxUIZiXJ4RI1KneppPT$5G^>s#*+WI z_#-9s$@FV^YyPgzn8@54RR@C=N;nTfaCQvS*)Y=$6I?UAFl(Yrf3ANA4@o3Ow$8nk zt@5`alfRo}wM zHPkZj9iY{dg+Tsz+w6;Gh8;LD505{-ETc_+)~bM&iL2hA*THb~22;5Z5r|yi?=_H` zKSIM*N8R=p2Oz)IaU38fe|ackbs#sZLsqM^a=-3Xq7aBTzr3{d;MIca++NDVyrzp1Imjpvc_{~t!I+KOG z(@^(rmf9ujuuax1Q>-=Xn{iBT=F@TJlY~;RB5@UrlcP|a#C8REA-5c1qHeHVmQ8d00fA0L*8fPo?q(U6SHE~ zd*k<3eCh|sy zp)a$)*f;m%D`;3=sxDi`YupCrUh2)EWmirff2Em;D(dc742Pm7-aj;|PAp?`(n<`b ze@Saoi(f}6G|Dpfe#(RGn^C#HS7V+up8E^y+cV$O{L2lM8<2&sq8JBc+HAH8BCs6K zrb1cR0_g@9Sbva=8Tp&6-*DYNc&>TjPbVCZAL+)B2L<~tsTr&>KaL-vf*z^Z9tLQj5GLQQyLg_B}IcF_qnVeYU5X9I#JVwZoX#Ny>NIi7$)b z1fCpG>q4Vu@Np-D-}2s{z;1jZtrsge81wTA23s$77#rz07sTgGKiZ@fHEH6w^>cH_ zc=DY~GrzTr2TeW)m@{C!NN4K%FS}m9%btCLEB#iQt@tJg8SFA~_`n5tocf)T8x0W^ z%J=$B3fiHAh@ea8_d&ZPXt6a*dn>c1mRCk5=KXLhl$sZ_uXl&t_cuDyN&6#j0*S|I z&Qp@V1-?hTd}{f;lD&FY=854NjIDvtc>PV0`b=$en^tznCg9e)1Ha5t%{BJeqe{k#l57*pH;Be?xW&6p0w)sGe zg${Nz@cd)J9~&wjZ8&cYma%2BogpoY>arwmd%l#J^l#VRx}$5e>5qOY;7kT1%5x<- zcVDN@E%Pi~LVkhW(yw3EK;z-{e)mlbojgwZAV z_1x!Y*UJI3Ag=W)LKmAn2{#7cNmXSJEzy86<%p2P7{bp{Jn0lKtJaoN_`r&4K4p1T z0je`pfTIQmf8G+-Z$!F;8fE8DrvohUG_tVewbH{ZE@cG> z#f2^v?K3Cu7TzZ1;3a#3(O6MW$7EZNQYZV)4MN7^%58Y7GSW{LM_!+^`ZpZp3yUst z7Hb9*PNTKnEvh@`g%?5h=zsKX+!BH)-R@<`dbFeJaLL#45b%bL3qHeap*$=Y~m~Ys6C~aXvnSOqV z9Dd{HQK)@vklOaNFx8tUJ}`E}i=9g7^Dn|MeEZY8S>x)M)|!DpK8?P35Z-|iP;{Nt zd>I~rJYNa_G*_WAc0yE7`5``^;}q@flw6&=;l;TwCYrg)j%?F$b?o40e-l69%{^Y) z{Hu|(6W_Ey^76S(b3YYLc;ACo*Avc?!JyL_yed2jV=rB$oE4>I$Tv_n1LjCbx;5tw2GXDvP1 ziFd*tdHl8{_VZbV->IEbR_7^b)!Rz4AR1~?db|hmRNY*cn1|f)`U7P4``K@%KZ~t z_L`I2?in^{siLS#>5@ZYZF%80%7=I7Lrb7L3b0o9C!53-mk$X%n`ec@oUQ4*wU9Wf zmD9*I?$trvQ$Iidoo)5!z&AZA;sb!Z zygU#C?wvn?6eI+wR$+G-gE6JExU_HIaHel91)^Ju6M7RgExI=Y z8weGt(&Z`&F2l9VSk#xV#TURA{9lfluS)>U(V<}C%A?p0FjQsD5*JhM3LYbAR4^MI z3Sjz3LL+%OeNg=JaN&D@_?GI@)_Kx-0X=ONo3TmbSUs|^i@onlb_^loBlP@s*Q%H3L$ZYI^34yJqu?@8E42Us@2k= z;G&Y=4S9XXf}WwHZT!kygteVuEF|?w)8CyFXQ>wIB-k#-#>Xi>5L>Ufx4S8c=MM&i zOGiP`{|n`jw?XOO1FubLqplgI(%bC(lVlVoNN1(~B+3C7+7ZskPZ4Nt!+YwFi4AWh zBhQc3(tn_)p%O{CC$n@_K>^ib&i5RPvDwn3ay$n)M4FeiCTJS`>(fcP@1Y_vXf=e~F!oqYAo5n42UYe$y+8g-iDNs{Qed3$o$ zod*5b>K@imR|p;y+3HC?k0hIK zqiAIXr}+y9JilWaXWFbguHkOQQalT-j-3^F9`u~bCQhV#@sr);fVWpwj+WgJ4@gQa z8R2iGp)bB4`*9v{4ISDD#84>nfdbRiKsBugg@=E+C#DaD5BGx8ASqWeo*@t8|? z5(a z@>r^x4{koD5U$^#cBR>ta^bPa+66`nqP>rG!u+4-v}JXi{p6-}#(@&f4caN3tC&oFYSvZ0hleBTSE`BxpqXF<)3q-m zTLXHjHgl2Eg%~pjiAprWx&~qWX3H#St8`3s)sx(OiBMawWj2hINx<-DkdO=hz9&P# zNoWY-jQF&(mAB}mP$c&ndJ8@{n~cryM>}0buq9H25iUfsNrr9crGfV{d0+_SLxCdu z@S$Ic>?%{SAQ{14sxBuj#WjN%id2li!+aqAd`2Qu>HhEm@Q;qivG)FJEc@Rp(SIpS z|Hm)?gC^+1{x!gagZU60s z!ZtC&z#tb!08$k=YG}9`yztUTM|Z|d^NV$&#uVj%rC5mU5T3Mep3*%ia>=DcgYU)n z*q8zRrr8hR^>=Q&2R}+gqcHO^ly=;p@Zgz~=a*;Ci<3IYf#9M3#d11mz5=cl_4d)> z0&Y2Lt}MKmM8Ap!6U^lRW(2_)0JO61pCO8#UC!&WB`y-tVO3F6KLT_vC(5tj^mw|J z3XgXE@z4f(<#KIj3Um1RJ>&&Yuv0wA`0n))qV*VERBgV^x1X2)GVJ2P+i_jRCE%oY zFT<{%duxv^;Dj<^Q!S@uSDhzvok|KD2_E;(yPS}e{uTD%N9ss&Y&AKi`0Ry)^bBiN zFvUsuMJ33wEycS@Yna)0PD7g))!cWQ5cv3V)Y&bN?(RJ8=dTO2>;3)TBTxx8I*%$2 zoH3tE5;rRB8&E>*ylt>A$l&OmpqCY_*1w3S{H}8rzdiIG-TFPyg<$@kde}iSyxqq5 zmayTMG4(Z0yMofLv%1%HZy}M;taIKDmCGyGq_YBehSmjBd-2&WrkldlclIjD!{1T= zdR371V6f~n3ioQ3`l@7p!7Bp39!nHelinCTqDMgSBF`V5)m+w4E6$vADPZ_@WZ2ib zqM47#h6cxP+8W9-ylvj5SLR0Q1D>}cPK~7AIbZ65qkeM;lblz8K!*r!5`^DnLf|d} zEB;s_wff~R!z0u=nv87>H0n8sZ}cWHXk&8o?^F z!LIA3_FJ$DIkKGZl(MnA<{B%2n%OSxJgK7{*Vj%`X%m;ce^3_Q1U5Ez`G*vMLqmS= zcn&jaqG*f(iK1zLX7v?3@)rgG&a%p@6F5GDUOooSw%osnN(Y06ypkD{@DHpjV;$ik z{MtW$aDB7NN3i}J>+H4zTW=r$zGBEVo&aoqaUa+q9+41b4yx8 zcCLCFz0w1rIQc;?L1z+$N%ej6R@2@L0 z=!If2>B=k@X8~U34&iD1E~^=QSd~mWzF$&T&F7Lv{sbbdXpAIH_n=X}jDuJo>?13B zu&ARia~*$Qz)DZIozVKzLhI0GYbqT21plx-CN6p~@ z$ol@rO-dPpxMj8eeNd8DaC4f|Kpbf!GbQ7)Tc+`m#U|3oKnjt?hhisQSO5Ssk|N6D zAfo1)LLp$G$njMY3r1egy5mDXNM;?^0(>HhK#h2z?Vq@1&=v51dnJ zkD0_h7koI>mP4up*J$Ugdvvj_tYy(g4al*BClxJ0QfsC^W@fR1z%g;4+VUm+=T~H< z0hQBj6)vuhk6*pK1*2wwvt-bS?gQ29f2+L%J=;No**~#9A4$F2LDFg? z2i^rn_p}R!M`*%D5(8U;B}~>Wj~nErbs@;%I#dS%lYl^k4bOVx#j6Kiw!9XUfE(QH zTcv!>rEH;tX=`GpnHJ!d3{4YJhmxvGyOQi7MajKnoTuS30}Yh)hRk>|Jop3bS?PlZ za`JbE8d;NaDAy8=lk$`P!dT}G1~GKn}igyJ)d|<6InzZK8!NTev4$qAQ>(<)aDJ3)x-lrW} zGw`zYBk26vvX4xUX|O{MoabVa@`HjUH@ShqR|vzlYCqA&nCY&n`;zA=awH9(uVD?R zIOmIr#1p1r-PA|v^?9*gHYn#_If`Tb1E>eCWue2NPLQ#Vc-qGRe$NQX!D$q`85K`* z-&@t){wOS?3Vi8fG+Umblb~I_eC}x^hJM!h;@>GR-I3UG*}E3cnTxUI(I?LlLr(aw zmN#6lvjIClo`DBjsbM5#YOw_UW>GFXnF!E{r{%7~H(^-if=A=Mitg zg+!z*8UTz<+*Mm4DS>JpHaLUp_?mmgTUjP`O*?0|%ichk#AHToNPHE!yvn(!@b`R5 z)U3qLTv9yd#;QMJKr;pbInlG09St#)OeVyQz1lvf_I7jhCZel!So;H>yaV&-u`21) z&J&6S1Z`K?lOE5IQ@t0*URJ6i9${%J4{d1bXISRYg-x#^HRAU^*N1B)@5d8%r;7Y> zV_Sg6o8cpSB;~YCX=f{Z(!7ULx?QOG#eZhe6e>Bl9L#o6#|mXBLCMMS!)i7%BYV8D zwoSe?n5*u^7m;j)lizb)*8?@8^s-Ax%ju~5zD$Un)$ecT2GXgHC^Kc?J_yZyrls)l z9WT54IRPX_myy$hhe4diXwJf zSQRbvre36~LntaH_Pe#-3tDW!L}Yj3CZdEh2nB$dyUoAyht}N+>-YIa!~F(>Ad2uh zqC4xZ7Y)j~zVdQ*Ze%qoCnw+q5QitjrsKaI3X#w05g2}29@SjVJGocqRf#0vchq1q zV$nT?^0Z&FGg+^}zew^ju+zT?l624*wc+z6ch=qQ^>m9fHLURI?C^Yw39<=k)zEh2 zxKgB4u!K;rIWs<_OMDqY;2l#39cJUIkdc%4Z{j~Auj0A+NL^W*bB6mQo9~Q<{ppjj z)c*q?`DHFu_fk?vpczL=f!a&fWm81vu#qj&6^p{Yo{n6Vi6Woxg3ePeT)mMK>ENsdpZ}r&%Kg9*MvmYH+?n9X ze7sI(T7tzdZ|BbWnx+mIJEJDZrbr1KBIY0vk{pZgEzc4aQzMO~^Kp<_L!2+x4eb=# z#aH#DJDWx$2RJbT8VL9Lm#rYrBEv`;ju#3@OJ+6R0{*P7nAtp4IQ-N`xNo872HJ_m z$&w!3OTYB%)!*#1cHBGZJ`Y0M)-iY{oWA(M=~WuBj|QRVi}r0@rS0TqdXg|{tUF?Rw5;y^-ehCw7%u zg`Cr1A{N0JkgK!bN1|mG?eW3njqSV(r?)+t6PtfxYkt(s6S10>T8sIw%9QuQq636C zf88s1)T401JtaRGDEu02T9_uj)O53YUlBd@d zamQN&m#l0#%2D-M}j2h5{+(|72F$g6RBL^MT-|k!;Y+QDRc=v&gxD! zJk8_Q2*)$`p}vW4u=aFaX$jc3BSrMrzK2zN2xE1N>XyT@rtz=Wsad(n@U%S!_PD1% ztOip=f8Ucljg-`Mk3%3hX3IMjfnHWp$BJl;t{Haml}rdJ#oc}kH6M8 zm_0WDxlOtm9y~q+uB8`4?Mo4pHyotYpOx2-o=K#%Yv^4^80{cDf!?q)G~{U}rSpDC zI~dW~Lev(bRfh9q`PzmW*M-PQ>oO2g6UfqgQPyJhdyZ(`Jzp2H05a|ItANFy`H<ap+E90Njx?H)19OzpzWg;hv%vP48~4FgAZ zSx8yZZ@LG;E92`78Qo;lSTBtL1MUb8;Pk>^(N4jTeI(uA`!eUu9r=th>sFH)x~-L< z{CBTi9cBF;@xF&moAPHGo=iVH(<>@UGWt8wU@KdxkkV$8R6FzT8f|qaT0W?*eaiVV zoNIBMtN#;u6!Imhwfr~gu99gn+uNcC707LH<8{sHV*}tnIc2TM10_zDvlhi`{4Pt# z-DcvzUW5voG7jnW3CdA<`Cy>aMkF535QE;TZm!}t+4ijvueYUa>EnUuw1jKo6rG%1 zBC`h0d`2r*Zpj~|HOk^6kJi2_k%cf((%p!*Nf1NpdvjofWmU(#+dG1Bwn^IgVQlXs z-@#Xsq|5p_dL_PaFz0pOM~cjQaU%y|wU&wK=9_E4z~#r=hkJ1q(GN>&r*@#3i8<20 z10cSTw*xK7!H==w7?`{^9>4y}YzkVG=HLp3-?6?a4;%bqrxr^*;CRsJd#C$N+_Aop zY%hG2aF{9U2+dzAWkaWn{H_a(MK`!6}0=`?Y;IzlOGbO{g(oy$<6yfXSAIa z{(Dy@geN57HGp!_Tz%O|DNAyyjYI@<)&I-cb#jZCUx9X7*T^x_cP32X@!)9*W(I%( zP94njk@6@%tP5mTinyPx1|Eos@vmnG2NBSUkJC%byhkN8-P)7*?mihfCU!vxz%9I8 zk-Z^bK=E6o|H0tb?trA@1^6)k$?DKFgsE64oLnGT+ZWjZjjAK_pnor3E9cK!aO~8Z znG^&`!u0FoNkjJ?)b7&g}QHTCM?h8_3BRsEb@86!721oT_Y)(#` z*wMeAb75ulgkd0AJK{(5LC*(z87*Z6Cwji+fh2;pvZmEtO4hey9RfBRsvW!=W!16= zh>0Evee+Dz){)_UEKi_hR_lB$Z{Lm&Gn8@~)Mi>{kOIp%p{w@+uKhyMLOG@8v0qAA zTKr5|L|&4WwF_Bq!nj$%2NZo~Q?x0vbQw3g70PwEW}hACw_;?jN9b0~C{mj;#*JSUm_{9{6KaF4n?FU}#pMNiZtT*JrlFQ1=N zdTJg0Y(PThO5F_P?YcwR=)i!aGo3uAb8XN!w>y~;*XJIjxe8Yo_g!0&1O1$iSMXV^ zPvHkwkrrytqi7krF9SXjpNcv5Bllv1CS9fP$floQcu&xtQ7^gh?MS@>P~L#c6%v&* zIg80_ynbI1i|esy zvlq|}c}@X-WnGEMZQEBx;iI>Gq*C-UX$LJwEr$nco3+!PDl0)I5q_vt*)*s-Orywj zSQwx8`t0jeuU_WI7Tv?N12WDfps{vq7HCTxa6d&)F+$wj;GOkGFZ9NbU~14sHuGL> z(7HW$ZfQM0D1f)Ynp3l_7(5K*&lhUT_LCTqt!FsFIbM**p?1nIWMTDw;9#2fAks%{ zF!vpWhHj{p++K5#)1}vQWERZqafTFuTau=ma97{4tHops5;CE zC}7a(he$R z(di3x{=kcT-=en(wd`!-=F0qB-r(9s zk<1ZK{JVw%p4N5aGWcl*ZdiXqUlt61lTY3P%x5@jricf-BRmiH`?lUrNLUy_cjADIyta+=d_SpYS-lUd!eRb zlJa~?S3pi(5$WLwgQ9N=r(64uvbFL$P#LuxQbn2RAHe{{sg_`^&Bzll zmkCg?tpb^`m9ciEL0|KK_COTRa*O^x&B(74x~*eCQv-UPcf@7BKKF5xpea+KqfW7#7 zQ{S5xT|)B|UistDjh*am)w1zp1T|_F1RR^2aw{3o8N34@i3zsQKe0Ek4qx`mm*Pp5 z^qEPGoU|z+LzBY5tu8xRW0$nDCnhqz!~xuCjK(q!!xYI08MZQB_Peed5B!(s_kS>< z`VR-v@$i3#V_aNnYu@h%0Jr@z+lHl_=W4P)IBs>O2*LO?eH?2YZ8`RE8PpF>(zk<+ zuptcqKvg^dC@U*>z&4-n2wpv~;h10;1Q7wiKR9|OoK*i0-*`vjD&W?VRcam-&&UB+ z4JS7Pjmsx-KZu-&OiDfkwk>FjO4Hr z!Upbc1scuG7B*`6UwZK0m zoUr%HkJyu+Jhst=gDyHZhA#Bx2FW?(oQt~E_NvTe5Cd^E8d%yj*ow_5=vMh}80)xE=K6eiYSU4v(L zUsl(t6>OaexRTfwc#3fM?fMI@us1%h>hn*_IW+3(U?mz6{IVYU^bVt`kz?~6o^=71 zWo6S$AGK?o?IeWsGzxw@Rw$gN_KMI}2h2P3BTJlDwIAzoH*eO9aM28pX<3a=zx~MB zW;N%DmZx}_u8=_Zq4%<>7S0TuIA#vYyTn5o-!#{)>cxhvY#SfB;BG@af9m+UwO59c zjAkyTC0rR+*2i6CH?p?=&?~;rk5;mmjpS3fdeI$De2ma`Pr`(#SqUjV0W?l)3X5Bk zRn{p7`zaiN?%Rx|z|PFcpaJQN!b+%cKgf}e4(kgn4y>DLzev{WT&((YFU3Yu>ymuw z5HXQmCsYk;nAZj`K2a=1O8QvNvrkSPRZ%vjPdk+FoGd!2Ry^-sHwe-s2q3^X253CyoL^Pqk8GCa=4)>L5RaZ~J^lBmnwFbe@7`2-d8qY~ zqhoV<41EV~MUjvFAIB=ROFunXeY$b>>C-iJ2XEilGk);Ti*GuIF#*LwNnpqH$U)&% zrJu7(Zv+j}x4-Ms=soN#TEWq5T)Tt%V&E%u1gMUHtZ&rnsHFDKQ>c<|0sE%d-#qU{ z8HVCEWCD#kQZ~tGxg6@}R$s4MJ*)%g zwE|h38RQ?M2MFX))3M1@$UbUwS<#1}e8tlH!;^WEBol*a5%>AwN@`y<3%@v41}yTB zu5yhIxRBLAa>NhkqrvPb+2;`r4K z`v!|9X@@TKqADoAvODSAr)`$H0V<0}D=Fba!-JPd(g_b^4HaW#Gz#z3-Vjk4M%04{ zHUgTm^0ay)gym>I!X<6~K^ZHD?hoG+5MM)Addf)l@L&bAH4m4;h6@5#jFKR@5xOlo z=fm7&o+c{f-lSbR&+duu-!(obW=+)D&)fk6ri*s+Usz#J0B*xb&Ia<(_?D&nuT~@xHY4f+s98x)a*^#ura0HUS_iRMQPROQuh0jvIZT+ zjw*^9uRoizwKzTRGxain%)6BBRvmHexj7RJrq zyK(7}B1*p=ck0~#=y9jJU}nRUrN{UC27iEwDVwGKCo>;UFSCW`$CQxUKyR?ex`aZ# z!*w8PA8hhe06fSy1p0l-u)qymlp*l&3lQU4=I{!7=fDrP+7g{OLQK_EYHWi_gO)J8)bjxR(iDW0Og2<4sy%j-T9t;GnJU7keHYYxfP>g$MosFOHbp)CPCYhCcnhq z=L_!(2FE=Nm@T9v1NXr^h0TM9Q?{P3-#4WRZfHiqU}Sl)Yt`wOF24=`{$XIKSuzlT zJDg)d=ZhY@nP}?k>dH3?Xej7nbWqTyn1KPuTT)n8>Q$rFuaF%_V!htXj3z97>zi9( zUi)cjtVk29#avCvpE0WbHDj(Rap~;w?*#M2WUu|5h<*#C0_k+l9P z^FXtYMOu5qcKvX%#6YQn?p>>yaaQAVvQ;N*F>}8L=~x0}u&?x^8;Pq&qWGo?yzVUq z|8n$T%s9;jX6@w1!=HT>{9JVwzxHtHrpPNf0WRV|&@b`3Qts6c37lw}PdOA6# zVTIvuWGg~wR0V19H$U{XoIEuA-RAfT&IcYSj3x_-_BnNbmJ%Ozku4t~jA1x6bBQKnoBc5zP@KG>BPriJ~MfpG`_e?=QVCyP0kl*D1F_AN#o!&3v3_JBZbBw+kt4zm6q zc%>Xa3RZxk+`wC*Df@GXr%BX+-()(4z~Wo7qx$m8BJ@x5{hO&#QRJw@_@o!`8IoJO zXy2hS2;*k{iutTu*IVSK8bYF<=agid9nsiZrEI57!ox}m4`h%~DQ0Uxnu=fWQTJY3w?vR4yht2ZTB4OQuiim^13KiplUg z2{{T@JzR;sh>{2`4j)C`pO=Erx9{#Is)-FDR2&4|&vl-QyXg&UyDe$3&auPpQsY{W zOU4T@k#T2@M^to}npk`q+aXHot*sO}f|`*e>*<&W>)dFO2xa|-s489$+CCtAaVCEt z_DX*cflw0Vb3G0>vWRmWrVfw8y?=}^j$7S8wkV85r#JhwjTgR16uHft{MCxBOK8v% zo=>gso!ybf_w%>WL4J(0SMo(aRF^qq`%{{X*zNWxodCzk+}F+m%InW2TKt(pCbvJs zOy&nj=?eA%crnU#RinG6?7lM;OO;%Ur@3v>4^2+)sKVIB@%W+0D&Y)O^K*?zn4!ZD ztEWYy?Z=F!xyCuf8)_J`eTtp-FU_gI6E|d`Dpa=hLBGoDQpU?iUKW1LD`?sSK+{iB_&OHCvjI@n1P+L(BF8Jk^xm7Ws^tS9M za;*X=@mlWO7D1e-67VrUUv3+C?!2`xQt8wFym{ZH6{qkN9Tx=E=!q2b755ifwYGj#V&(KjGhnUW&4XHw z{)D0zWa&PZ>l+Mom4*hgp4|aNF|9BlizQ4^nI~OuYCN*m6X=(HQ}(S(F!Q8}%p9AO zA|FPYM}C}Psbzc_@`B(8#CS)bh_r!Bn1i%ptk=)xMzOG{R;>5(RRWn5u&Hh&PF{u6%KWka;#c zMikrAMyqj+_U$ce8bVlMqsY4}@EFy8QH^b)9qz7)Mt$Sa6f?9Y5)L7Ic;*$l`}0O&jixaIGBkh?bZ~?k7^QRqNK`Yk57fmRseXXOyuMt0pDI2;@YrYz??Cm)I(% zx{go3qtSB+`k7mfHa$==xs)f@I2U=N{vKb%oCqXw$G=R`*BzdJ;}!hnwpZ!R*JUz; zmvQ0#vcH<~777uz7^r7`HFRAWh47c3?5Yr{qcFPCs**?PVc3t9W%b9(hTp3Btc@}Z zYuKS;jDyxLW&$n?1v&$vnC>LNWc0LBx%Zu-65s}j8}_wo(oo;d^PPGb+GKA)9}a4SLKsog*p7N-~fLZ#AeT7dt$h? z>t&8@GHvczHt}@3bncG56_fRLcs4@XrY&bcU*`$O%`n=B7G!gcQ)Snqka5Fj>fd1p z5y)jB~v9i*31hfw?@O=Dka1jTY$#ilk)bemr=DLLXQi65BJqGzEZ%`w~ zK~~g^jDGD$2q8?B6^U~1OZMdngOA`3aD&Yad+4ypw|(2}zLJsHf$sURgd4QFpwNKHzLzCh7Jh=8Oow}kBKAH%Rav_ZLMf&!x*1o*nU6DF8Jg$3>Yy4R(bvsH$h5MjPyT#t;)XLjh{!mBtBh#b?g`AN zqQrm1EQ$4Eoc(L_I!EFKROSXg*}XOpDjj*jzZuCu1L9dvi2N-|H2?6Gtwh6f^_JWh zOI_KM@0$XY=>K$R%U}OreegB1Q(v*%7p-3E(!em4ZBTH6w{3kKkQMaf01(B=)dasE z9(2h7I54zZetwvV*cwAHoeKo2$bJCm+FM_D2sT;6yp;QxTL$|k$ zn=yS{**k7YYkL;h?~mc4{_Qx69ye_K?%qa}ZwY>=KOCYAI6$LA-RdHRl7tA2&)w6T z7smQazvsI{me!r1YEu0l?ZP9@0s2cm|z`eT}v{SilHRk1)A2a-@}E zqJ@*8AnX#rH}v~4o~VCKK+W1cZ==MI^sjpWuejXv+JB9?28mH7v==(G{oLa*tF4t(xmfY1O}9x4;}&RhqGGr~^FE4ppoBjt0Hh*yQB( zO2ulAYjNdJ9OL#jDcsqIRy^&9mq6t=mmQ)Ra2@gM>oIUikbpR6iJy$M5LYDrj_Kb+51-RoVZ4!eHe;3*L+bb^yzn^AUBxI<7s?YM*to0~YzcgW&5~_&i_VJJ96w8Obk4<4IccLaOrxS1rUafqQ%#9QnOQ zlNsw^wZ*Et=&;U?fB_!2>$c1u<0G>3`!qMO&2|Krz$9HS+LE6=Z7lxgXlU41MaLTZ z5Tc7cXqDXoiTj~(W>pc}tAjPSAj|$bDJ%C3w6^jyeb~`R{Z*ZBQv?Dw%F1y+%lHMa zLvxay#dFmeX?UjOH`&BeBVuS#3a>4%m?V7?nhE)vn5XJ7{EVrDDrInx_;5e+{mzc3 zcp`0`D}^%~QRZ@&Amau=w#V#Q50AacO!UKwzO+4lfI*2WAG(oo4;|Om}OaEdd%tvl#huLQcKhe4d348)UWj zu&$nDFKEqKoY)bD@OO@`*Rx1>j*1!`s+5ah1w+?2qL6`Dn&~>GL&f5adXJCK6`lzM zZD2MeZ1_i-HeQf>#-(FYuIY9bsXP8F?CD&W3Act{h3+eiz`Rb3km1R^9Pv{JU_K5i z`PFvV8A=j)`XtUgF(JkM>P(n|y(z{=YwzBzO479X;qZgcry3_g7r$-=3_!0Au-F-A zcK!F(c$yuRxj5o3sO^n3(#i5M7x1!>IIcqjRh_wCEqNhW4nAkURIAV3@#?mH_n%se z|Lr@muvX0w&OlKuahX}z{C>37u{fwW{e;*o-WrUAC7f(6D}Grpb&VarLb|JaljL`uJNgg_7a3w zvD$by^go|78~sfgL7HQ7@3w9P9sM6&2> zf@#^orT66XuHBmJSXpCW6sYrePj|7YL~V@2IkB__1VW6m&mhzU!4s3qz2TCm@~F(@ z&RTcjf|wP#pGLK?)2m=4WA4>SktL_6)~O+_+WFzNOy(avKBp07r9QRsC~l`F6it zl)WVd<#MYKuVyv|yv~ku9WKQSJ6kYCC*4G!6&b#2fzMFs$DlEq#O=`eb5+TV8cj}} zEn^YM3Jx2L!Wt=BkUuTFHsR7S{rD~py?i>A;fNL>jk7-AD=4XYhfgPG-c!jizB@Vi zxhaVqA%2$T>ALP?9%-Yh1_3)nHLhT^N?@|Qk>qYplv`gL+PhuYgjndNteB5cn4RuD zgT8jPkGAH5;}0j(`kcPK>EU|EBPn}*tejk~&zeBONUH#p!I!DXDKJe;uR1YBiDwEvbzN3zan$RyuC!V=+yk<%chvFQ##_l0YrNK;YJ3CE%`3|2w$#m1!?7XnPmaC}b8XU7Q$AP01J<)V% zt^FVj<*3WyE`~)kk1)bavoSv_`lx}JUDO8#~!W(&CMH;T9#8QZgT$Gj_!yNm9oc|B zF%;?3&jW=Kh%wsTu8y{1B*yKwViAE7o-}DCIH|uURET zMv|p=WPdsEJVT~Bd^;I^SXgf|a$y+RYI^O(#?{gx&YGPYQxbnreh_n8%7+zg?SlIqxX;KlOFrr;g-^p3%C-5q?8ZtZ=4CZV_w`6&pbx{!y8W^A$}h zbZL*q=KD~kzK5)q4{rB!i0!a@6BWmn1Ij62ir51IZL_;DQYiQmapd?2>1l(}0!_B&hBIP8`KHe&} z|H)|&)gf*ZI{_CEk;m~$8S@>hvEdr9YOdIbB0DRob|nELjJl!Z05!s)O9^f%{Y=<1 z;%r7GN^k{m4IIh$t2;50`*RH~F^od})xpd~vA_)O$S-kB6_!@!N9me@&W`Z^hqX5W zOEUl4M;~>{X?2=PE4QYatXybySEea*+B7qF#mdUe1$RXhY?`t%wX&pgp|Uhra>WI~ zGR53TO;J%X5y=G+5Rm;mI`8}be)GG|`JeY(*YV=w0(^LQl<)Vm-k$dm z4)^ePfiCXTw0v|H?B(6)8wKLo7cx1Gy{K=SOB+PVWc8{af5JHi=J-Z(=Hsf4y#D<& zXK1mOp3oqEpph?2iVLVi9SGrj*}DYID3QnVPoL@=x~!QQ^8P06$PFv|VEV|2uW2aS zo|(*?+TkM#edYB6{hTs6_4_m*jl|>B-q(}#1<7quz^rT4s=$H8%*Qh-1B3pve5qrQ@rYfu`6S!53ti!Up9l-=Erd&nntCu18q|n0VeAn9+qb>~BG1qO0e%mCp=j zH5v=quAa4Km9kfyOpo@!+rsbh7KaI^$!6f#RS*{9X^R=+Yg^J+>XQZ?zWrW|pC&5R z;E=(^dnIg33;r?A4`U2{@fHAU1qvmG)CXh-V|^QsK$Oza?OtY8U74#kLp8WIKWaz& zmw1Pml2?<=GF=@brNrz4U9x9WYEzfSuwLRuhxpV!M=5k*y66qZ|1%9wNC9 z%3S~}8)*w@#Sai!hpM`&k;Of{_uLssO0-=iy}p^vow2&Z9eMn&<~*HauBdty>`!mH z;a9tNyFA&~<)7@o< zr&E4E^2ER5_T>g0yc4=3->~`ToiA?IiXtKjI@*CA=;kj5lz6LN?$_QcUL0GELES`5 z_7(cmAO31%yx1K(A4g_Bf(042>8)cqmYC{9w)|p&vyay#iEvbX2)G0}}6!zz29z=jxHP z#hE0Pc?QS3A(Z$#5B(TEOub-jG9Esw5eK7RK@kb0D^IhP8mQrE`I;_#+{{jY)DYhx znfl=YbInG-_IQg?+&1wuJiJiwiSfA$InrJgO>5tpmVWPmVN(BD`XnHC%pM$Zx{a+R9NdQzplz32A^eevh)8FbW-_wLG+uVlK-PL$^@i`OYn@@dhu z;G1HO1!~5Nt~%BUM^*Ic&-8ERecaN;vJ*Y1Nz<2ZrW^s8)?INe5o%#s3p%CCcDyy= zR$P(Y->&Tc>%Hl#!u2ntZMrnDeucWmqVP1LiWkQIYQpQ5~dxiQ}M>FMbJ0VHl! zA45=(ddq{sz~WpCY0a`1UblDyY`eI_jeNn&zEUvh_;=ZGF%h6cD(vSW8J*pExxYa8V)tfIO|(Dvq3> z6}gtEj-|lkcLGv2-uF^HL!RE6YtQ7D5z5`W5>C9Gma@?p2Y*fIE(PIwXC71lHs;1U zaC)3}E~AIXFkM*uJ`|7WOcm*otw+YEoUxU$^+}`-CjonUorH+gqaUNQKb0oE(g5k& z!iXe>UZR@sL$dQEhi@0(ya=Oa%urpbJl$TY9loQ;d840qU-#NHN=7Sa?lw0a@q}Pu zs7f1k`TaGJ@>TxUq1wckT~F2-WDJJ(qewUEg2T*qj9$GbARUL^Bl)@)ZEhEIF!p{< z&bKP?`q10TmSoEp@{1QO=eEL92ayj7eCZySDU@Ep3e#DMFYP^DKlDkb2&(0Cu>wh$ zd=8~xV{0EjRfQLjeJ7bj{fs7uxVOF`g5ND_(lMEMU*1-n{%gQ>E##{6F)Jm_l>Jw= zzp=hoy1UpahbA>hIiOvdqu2;YC;A=Znyri+@}wg5v9Vr=KoYKzhIP^NN6D)QM7ASR z<;xbQ{S$A09uI+CM-U7uy0kXll=|ySVpPL%qVyiAj7beLjXd;t1E0>GiQKrkQrpsY zuRI5GP=1+aF6N1ZOeEilUhwJA!) zA0VsQZ#}D+$f=ojnXUV^nLyLmBc}86O0^Mv&xY=2@28Y=tJFa{COU0?gJbMFW3z05 zZaJenyNY?Z$Z$2egu%Wa*V!ii@J`}?43~7X`Dnb^wUHni#cQ&2`o5Nbwa-Sw-~^>a z=*%|P6Bz;~vaiiRJWHF7XJdsNcr2>_x`RYWM!wvj?(xgo*R-9EK}xp~$|SPq5?Z~H zkrrzNPVJPJ9I2NP=>5dbYTOM3pD3MEYt%SDolqYdi-$effpMYhPa znOQ%m9XlGmP*Dv1cAHWjJsd;gud`CR4G9*KvF@XidaN(Up#0m2|69(N3Fh%#Cy-*V z)Cl;?^((^?4zHFw1@yJR8yKemg<`uQJ|}YGD5w}h>w?*K#71wh10n-JbUBCneF3LEsc`g!yaPy||a>_v%r;?HA%*_mHG8 zB1_NT!YL>JHlKc$EmDVm9kS6vP3;7{Hr@qe4x0xbInUCOO^a<2?1+2%r zc0!(|0eXHw2lcGE?%W&g?fxsi*8EU>l@QZ5rU86ZMi)B@#cPI&%OBO zncLTLJzFV5?7gI}Vp!+C=!qPz>Wrhs>`Mb6wA-I}g*}Dyte?CC9lp#OY%hrGoUZ@M z&*-qK(ay>5Iy1O<0G`iMAkDniG;W|&@+QCd<^0&FC!M{_Jl)4Ao+_ExYUma{6s3-f zX?UuGsKx|t#==eXbvHnF?BJnk`Wq463(m)F-9j*Sqt0E1lhR*0{Pg%+03wKDPzzmd zT7fKN{j}%nD9(38K?;u4i^xHD+eQ0wL1xE*R|zm%+rN!{`Qf*gKm60~|F55AIy@#r zw=N+ez;fCnJ#nkODu}#MY)HIObz@!HE|u-=2bV$RzO~u4wlTP6}%+7G%9p+izpDu7i|x0fl=FB1?D^yoh^DLd*qQ9X^?7Z{xU? zZ)b}dzh|HtIvhq;0kt_-c97g$x@hg3?A8(flZr0&u_9mjKA?huvR1dz_w;@!;Zo$H zPl_&xB1SpV6$8DNCiGYgM;i+0RH7(+>lD0%aW9Prljff}|nrCAyqgma|w z*#SCX#guO(gR}b82{KdSH$3^zgm7Oh#~n3j4?H^=pHN1qZ)eoAxISruuG>}mHN&Z& zZsrdnlYEs^oMe(gmHA8JNVPB2THXoF;^}%sQ8P14!`HT~_G!L#r~N>b;|M5g%A}$e_Tzh|gh11~6zOIzv!hZ9QD*m}jH0 zO}kWhId77A$>oDV`QUv{&V@Dd^@n2I^W+%P@XAIkoUnCCj;)&~pcU%`T#bU`+z|(f z9R~|tXhEwQLy)tojl%i?WCnxb8X~%bfvJk3TyH18?ShIg>ibD}tE>*S^hG>JkV;0b2FzF*qJemC(?aR`W$na@-Q7 ziy!;<%5+?#7)%g}jpK1d9>3z^S0V zWtHCB@}B$P)aun%(R?X z*4Lo7iM%kzUMqjY(^gDk4U?rM++lZX`c{539mLNASz#?(Z_|%e4E`{Q)Y7(Nggf20zHM}YoPAFoz8BOw_aRTSIY=Sj5=fV7j1vWS$j z(Jj7R{_Wxjf>ssCm!T3WnaWpNs_Sj$Eezy1iOWVahYs8Fdl+m&Jm+Wa18g#qz%=;M z)im*C{8F&({GD^;@v*)3YN7nMTC~Xi;n)3mdcZ)u{QkP4frmHOIqOrr7sZht1tSxg zPuK0V4IXfqMYmYlzMh`p&tKO`c$bEqUzyvi%&^HEtA0WD_iwpl?CUUsOWzu(asK%p z9sH>|{@%4_Z9^U70r=D^DlT)rX6D2#+7MrGbHyLHagO8qlB~tAo9QdY0?;CN=?KMs zGyo|+e2rn#xH}^AWu^pqV?$c`VZ_ac5`?u#-|>gBUf=rRnS!VNzPB)5x{%+dU0)Ul z#@*}_T)NgiIDRroEp*1sq>A@CFwB`BzVB>&T$%wvC(}@KUvoH)4ZT1Zn~t>(c@DoK z6AKr5F4vmPk#ipp#7P&Ct8KlENA1mdv2=9(nCoOeuBuLxSx#}PP?T&&WuX$9Ae)fk zoJYkVWvurVOVe2qFR*z~f+IDTQa}wv~QVHN=)cs&c+j@;my};{lWgKO_u+ z^6sr1K=(jgE_72KUlpY+Cd)hcn9@o7a1E@i4?SmG)ia zN%L}khLh<&aaxc+KKn%YRHD*Ou8^Ukto45qaYci{lpchGmJDf&E8Ntd`Q+y&YYDmM zJBerAjxqaA;~Yy%t1NV+jcT=Fofr(uGzCvLdO&5FKReNWjwN?ij3hoYY0GtYsG63$ z`f$)9#b7ikKmGj^hJhfiO&U0T*3dVao?rlgg$*r3`?)8qH!{wrqG_>{wdl#!gtYuR zJVO(ucd{?-ZtLVI3WD-KFywr|miJxV=KE0~e%cwUI(7&vUb|d3D7KlPXdg%E@|*c8 zhHgXhXCUNf1r6R^dY=`@5V^yGe8tj2Cg)fSrz{(6fzJNPhAel7H+a1GTKFGL*uQ`A z|9(ICSK)t!o8{fhQz`3h7R~{-;4F>mJZOp5E{&gTS-TPdYbVQNm5sNxU_UkvRR#VH z-338F8NN8`8oox-G2M6%-j1B!!^W}^j%ZCUXlOo0zM`=;6(&l(_T)cW4jKvs z_80FpBAqV)_@W3A4GslTZG;1Y5Vv71y{d-g^TgPR5zVb;keabSKXM+yGq7$t0-ehb z(&8<6^rc}Fumm#ok;kyf|7qr)$kw&gd)2aNUE@)8-O91Jc8*v&=kon~JHwg6ZRuOxl;8c&aBm)3f5g59mcXrIQX zCr@-z{TUwkeyy(y0$SuPV##|C+}>Ezo7bAhzYRfrgIJ>dX`~_E$5puX{v~pssm&;~ z#G@Yu4N{Ult-lhou4~fq9Z19v8QJ9f@{uol-G>S_^^GCwaf+dDV{3%45JBTHLcIGV z^Ep^GZ)ox~+qX&E!5wjeJ|38)mZmzkQ5k0mIHCq_mFRH816ta1f2q!LZQij@e}FD1 zl-r#2+>^9>vJIbt)VLUYxm{PJtB?D-n)#~&Io&MAUMm?zHF8ne>+t35;b?onHO7sm z=p0I|B731O2CD?B3#Bh0@vt%jk?LE~cmA8h3X1J;d8|*Y+_4r}Et5EkdEP;i=1B+@ zEnYb(Fp@PS27Su!o?9SBo`*epx<)YEmNE_U>~8j{AH_rwhUW#lxuYBVv9?|#Gz_1< z`skz13orULXC1#IYn@Um>%WGiP4S0$Z;XsJi<=IQrt8GMc(Q4-XxKj0w}}P~8|Dpz zWZ9kZXlJNxbk@r5RJ~2~W7%kBrTb`%+O75@f_2$r{$iAh2*QF z*e9Cfru@~G`mw{i8wA3lb-k}B{!L&Z&I*EpT1ar0k*|G&zQ&xn;@o`HwQl`=dhT8< zCf4L)9o3}6&G@N*OdEuy5Qe|=opnr+z31FC>(HE|JU}@msNV=M#E_V=Mwm5Wid^sS zaiolx(>;6z*){=AMA}R>z0>ONEEO;-BY(~a7j1U(6(loVHW?J4p(cC#_4xwCF!dhq zSIF0h2mv|+;tszLnw-6J9J1)-PDl#jSKgD>%y!YYG~1!M3Q*AHCP3h4gKdUnU)kkL zwFe%sZZK+T;V3-{wS~jv;94XM3;MulI7+u&3f5`W8eQCcRx~cP=dvT0t;!=QV6)le zAG`K{{iJ-AOVp3c`QH`isYpu@12n=}ITn@%s{J;7`)SGKEVU42i)84^vGHWc%B6-U z#0Zj?ESAbq-$bY0s{WS4vRo{mXclbf0qu=$nC+IetEqP0>P&Hkkzt1x36oTZ0zcr2 zGwVR^@zD+9(-Ve|L|<=It>C9GTIJEd2=4u(KNw^(ZX0Cn?@=;j=*J~^Jb+huHp&tP zV(0jGv_pWgo%-S*_bcN)hsi0@x&v~rQJyAPbqy{y=(jqBqb|+N&AeGU;qt)SL{@1S;exau_Gx%)x5Q0`z&q;&L z8F+U+=nd#j2!sUP5#vT>d>A@VM5zT5pHk4(a%&sEraUOw6VGTU_G2LnH?Vq#>))7B1%5-%QVdvf-Np&#wH>z4oD>KI+60S!MxilQft9; zz}t&2O1go-H z1?>7*&{Ii-8uA)Q1>cp+e<($K7g;;ZT65xtomFa>A06WeixYYVDfjX8jmh|Hrgwp_ zC*42tVpB&Q{YDsmLln>ipu5}NGE%EHusxuh?qB@mR+X~}KK)&qs`<2X)o;)6q-Ov3 zu?_4UhGwpvHlx1NqJl-6>}NYx#wlbk#aY+d1!jlzJ=qa&fTMFLd4o! z==QlZJMNJV_1mahZ}j9VGw23Q$05T@x1HUoxG(pOpbH8%lXgBnC$Sp;Lc3S%_VO59 z{mRIA%&9i&QE>u29GyY*Ee~s>-`tB$TD0kk5zE{Hn-@RJ2&r-K`XF;EdThqHf#P#k zGh}c!lnCG`QId%_Kn;+)K zbfG#%ObR?aMN=zbrFISH_jda3^KTQR5~08BC`a6vE`Z*8xKvLt()`YGw*lMsb0%GO z?V^IH^je~F2qAD=R?zT|(lGq&*e=sN2--2=YcyHLwQefE8m*k)h+X^v9Gd>Lb9MXQ zfy$$|CmpBm|6>#Wub%|{{@*CZqv+!+{w*bx;l?D6C0D6 z`pHb_BdWdP5SV$P^8-9vrG7R{!O~pozkY&B#}xtAX`tOGx*X;keeGJ_e%rj6NJ}uc z0cGpLPhBGNc$X!)X|~%ElWb-$HOk~8&q@hCSHtv|2Zko*_CW0y%hD2xACiS}4Ieom zzuq%6?7*F0c*E`?OJ;(4qiQuGs@Q->2l;cg6Wto7T>$IvM>RY-(d7NSaUMUN$=WR# z{kl62+hqy4LQjCm@2UXitDFKQ(uB=eB1?P!{JIj)VV4;_w3;uLc&6N`aA4J(ADfNO z;iUnDnM_B~UfCxpLJuBeJiTv~Qu@R2G+Qur-xxQ=Jy(KIHhxhJa>tW0Yug@4|N8F8 zoe4X8gUY%8%7;_of^)MKc>vWmGIJp0vkj^chC1UVGYa%Tm_Myb5UWAZ(i&6`NXfLOyyn&*j~9eifynrg`(+^ zPsp3d8DPOridrHF_K| zr|YZMtl2%LJ+%FCiM%4{gWS^y#v(#{IFk6M0%}Fm<=Zw?M<)L%N0dQV8ly?aO;gNs zn2M$vv_7M~sg{?o@88#+zY8Msv4u1FX7j2+CVHnhX;hi2!{ggnjW01lbJ|?DQfixg zi1{tv#$r#T`X^46XT|%h5WLqOx3)0IY3Gvj))a)Rk;dcm4NqDk)wJXA3*O!n2-o|# z%UQ?u-VemofzFoy6J^z>nA7d-XW_QaUFbFhcX>R|gg7v#9sm89zUo+3#4It~+U=YI zb!EZ=EI8k`wylP=kY&v1;2^jdSp-#Z(^M9Fv5FrWL9Mvb^uOba^1lkqyqCj;|Ic?< zHWvQ(IDSi8`P#(8Rp?}E5Pk%cPlI+HEWNKvtO}&6e&L8zu2*>i-e|c1T&g}C)#duk z%aIVR3@;GqgcilwEQwNsfLf2G4z4w24Wm6wm+B#oGc<(c$NRK!4m{v4#Xs7GS@2;O z5U2Do>Gq~KC}7U(tiW2eOpJy#cfM0Dy>_+G>`&9 zXX{!WcRHq8AK^_rBX(a0_!P4YKC7&(>|-P)VJ>1#6E7m|7v6U#j!m=h)Bb8?nRsGu zO7L2KnH6{rPjZ_pS4|vrjMq#o!3r+`o`EGc5$zf6vcEj7lJDaK!_oQ%fkUAeSvZ?O zbV$_LBVcy_B@SieCkn~c7{-m=IdoFbw?kHw@KnQbl0`zE9a?;f#y{nfZr|dLOpgvx^p%~4>(v3|D-?v<7XXdB`zs=Ltc zqk=eG$jzt`JFLpsM}1l2bWseEuDSmj-;tU9?j0eX8@?(HKiqz=qb(`W&E0w~T2|2z z&l`UoESNwl#V&x3xW?WqY=Y}`_B`ZjdCHI+ZOP9CvbZTY1cIe9B4jVbWoJ;nRLL`S z1{SNHGuJf@)hI(~`0{IfdiWzjn#<3mF3EFWM^k0Zj&pNIoroK$!R6PEzt$0)o#OCj z%ufEIyR9Ls_OGf1(L1qeWIXXEI&BE0`8;O>4SIu=emjjG8SDvGq@Qc|bz$B1ZqfVmx0ffaj)KBApu^%%G{-Cyy)D2d{q<4OMz`21 zAi_Th-f>>GuFO%MCtY02YEK!LFY-iJ3X>-s5gC~AOYT`O9m3=mYg$IKC)Ki*(uX|6 z1HL|U_MLf0(#coWCm$D|Ti+bM@8sKaFiJD^giycL#s_6b*pw77Er@eH6Uuw@O#SJM zGF1CVMy{v+T3l}7?7aJ*_W zkL3=xCV7=oF3T=cR6Z>@QJN#xVti2+Ks|?8?@+hwom+p-tKa*-&PV=v;J-$iO&7Rke155teuajpWbI_7-!J`6;mud3JMt) zw|SPV?4YM*FBO16;w;catwGb{W=eo8lw=CbPKFk3GH5dbz?}KS85jV`hS!kD33T56 zPjn{h^s>D%ZRs?luWh`zf6qid7j^zY+9eB(U_9;i)J{&YX(kw>B$o|?B1QKA!sHR0 z%Eh3d;{Cox`9?f`D|sgz0yWr5(-KE?XkA7w=(V;4e0}}uG8q{>$AXlJhF_ots6LAU znX`kG_Z#qD(tCMTn0GF@N3S&*-Pz!e$|+<3!}~>blyGg(QFnW~WGOSEOtZXis=wE; zflU*)qe*kA%+B%X;(Fagj=Y_wvY45~@q=mvK@DB##Ef9A?7iBlG}PJk!3lv@{W%L& z&VoLJ&COq0S~prfJUPjEbk)@)2!DIUwc!3Pd+e&rOU-Y{^kCHJHk87KIIZdEA3u2z zfJrNxtlJ0)u(I$1U#{p&%YK~Cgd zj2{esq(SF~sKJ`YEBLl%Hl`tq`(vSGaVh|~rmcyCSo0H1WsMd@Ylv^W)ydtaA<96a z;trYjk5(thH_ic$lXh?p50@<8MQ4sfwt4ExH@BOg1P_UrPp`$lSR#_lv>V> zU8cndVrITjdhyw&ABl8qu+*q=@rG4R6Khz2Qz7Zeip(FASozjTs&1*v-)l5L1?C($CjT z$+c{4NzSCJL9C0PgfluOSDVgTSWR0z47WHJqVZ|*#;CGTVH&tj56pj^c?{c}a->ot8pg@+V zfzNUPF=6`~CF1{-d&tnUBgh&PH=i*v+@%4NV1>!rmALC}BKz$t?xlyxht+m`dD-f7 z{kZ?9yqg#EnqP)vG>`qYpceVtPlvp7RQ!XE9N+#)vqx2x|E2L(s@BT+&le4;Ky5Z{ zRGL7q>(HOU3-k6NBmVRUv19R2WwEnMQNL%^4VOO8Dz^sv$GcAc=k1Nn;qui>FSDPP zpHXTddJhLQ9yoO@m1V<00Ly%>-U))fhS*@5>ifF7)s4xpg&jaCfp-chES!GpOFaUl ze}02WjkIiITi`h$9pCr5QY{FM5?)WC+7nangj*tz4hAf<4)DXLjyrrsZxMPqJn7os zV4d0;J;VCrf3By8JrWALsz3(k_&!(aWl~ur5^PDXQpY0v4W?^y@)|_Wk&l8n?xn7M zG!`HIS{&?=ish$+7f;;S#QCLsnA0gAALexMHzd(r&g44-n#p!>ge_Jr6jMLEB5@6b z@KYjkWCms>L?+4SE?Gp7Vy*Q=ESf35Z7zG2x-nb*TEzeqcQ^C}G2PE*97Fa?2;!tR-%I0+cRg$2o!He2 z(F(p{U)-$dMp3L^v_w%+WA~o~t!>hbdb6=!tM+-<S{6CKLybODk_Qw~^Mh_D`bmo`W~vtU-U`?V5ATJz0z;%Xh7=i;U*j+Ie6nR#bj> zJ5EG1|=0LZ7E7}gvIR_BF8cn0j z4vcPOUOOmrAnb`xi?r14;%tF&>Km{W?rv_*MLkabsc`A_XaXGGz-i@hz38{F{S;hu zKZ)}xK;Kqb^cuR|h)0hZm$9O2qR{*5bZL}%QQfYSc*iEXla4|$6*~MFd#DLDJ1=rh zA|WlaG~QP}QLN`WLj}(Ng>zY^VOg?Baf~-9DJi48%ljQhgHcCwD4DEZi%Uw@jp{Gz zJOQ`Fo8EBiz|H^KUjMVt{_h`D#+-1yx%9U5z9?M))R#TyvKctryUK?ZS`XEW8HL-`@|IN6)(B z^erK0qxTTwq#v- zTnCrG%EoTD&mfMnWApsQN2!53U6r47MQZb};Cd=!b%+pq^<2Fkc}kx<+PaUSpt_WVM zbUw-`3H{P#aJoTya_hQzaa!CSfCZf(dup=WKU zZ+vSO@7>Hu?fbyHjFD{7Xbao*F=8hn`XOw%)P2ey8Je?WxaNxXlS+OpF_*@zekwIP z`=H`Q=6YGoC_FGCO1l8(gm(85N(-XO1mnAGh&=wq!&6bBy6QnO0`D?1(iuk&-o!kYmx@oDhn zdbhH^bI~EbBQHlb=hV~w{3amL1VkEVYhQ4KU%m1Dk;$})X!r;{BS%?3qA|U3*r0k8 zZ6_UKBPzY>pX{;-Nn(?f98;KOH~$JAfvbO;j&&<4no1a7>ZSlWODME?FVa(mf(HVw zOF{9k$%xnDwTRI*6koMOpi4DDf9hMP-YvRQ}k zKR;`#--MFy4TG(`Xqu_A|M{7?c5Q9#rn(6Vse6?j9s!4csX7KQozKgfMS9^fW_3=)V^8=MmsR!FO z&}Q7}Z)fsbrObu4bi(@A!!C-s_ifauCx=RD&DZY7MwjN>6=D-y_9?qfQ*I6ZA&BWG zZc&gT?THODKTsORhe4hhqW&69b#re;w%7Mnm~UMt)MT&#|L<#<=2nA2+UHZT-PuR413K`{7mtg9$Hs@wy>6U3)`6gh~Il@IVa zu#%0f6y|-U?iOuAuPq0^-DApJOzKk1M=E4*!o$mt8ttw-TI%ooO zOGr$u4Lm{VdZIpAFd_61l5PcNE@ZCrD*5XqK8eJ1ZM^RuJ;@+Ko+cgCi!w7k&A zrX6&?1JIJ(8snYiy~4C%IyZS@C{B-TGi3+WYsTg`eS5H{x?k@_hO^!$`p>$9=gAFA(IR%JWIZ|^F&TI(H?26ds)|^r?d!A7D z9vZnXg%)`kwJ2eoP+P25@Qr=ni4O-KH{Uu8+!^{1?trsBO4vAZ-TWwVB<7jJLcY1- z3VkS}Y46Yi00cHeflxob8lW?s`j*yWT}7{l6ZfU4DW>Fd+6MwPxb7sHsZWtRPMKEA z{7irUd-aT5J7OnGWtA;lo;x5q%Xx}d5I5GvJ`(0aUyL^Sa7UHbnC$J!E)cpkM~&!! zn#V8h|64M35tMrdf|PU9K^d^SlOm;vC~81*Dc1ZmhG0+XD|22TIHf^1iw?6bAwAquO5ZvYk5QdH*q@x&-H8EtwjCTBbcpcyvKfY0=1|0QW*N9b$v1-dvqK0A& zL>qzsi%>F({hnks%NkN{{;jX7)(+Tj!pOILu?RD6aeF}C`|P-Ez&qu<%&y^k6=S$G ze5Q?2*?oQ9d3*2;4m;>jbDD7EMD2^Z$GQ_-w-_ql@Mm4p-QEW7$ndRAmTY0cw1svR7{q0Fg>aNp;n5uVLbqDD$<5=~9<3!h|x9&qN_?ArQRu_=C(Gp8s zghgCvi8`lA-*z_tkHhJi!&-OweUs8-1dYoFC>o=YbqoTOdvnS{x+rQeY^Gasy4UXj z#aAd;)p_4i2>_tQZ*h6+dQXzR1?9!Jhqa0BiB3&H*$2{8#=0NT=OroaH=Vrglg%4H z@A9#?(-M*Ix=Y$mS!Uzt-($`xe3W-+qr1lM{VWmAB+=?KeSRadQxsNb{jdMJ0&Bf< z#H}m1SNB_JgfU=We*_@H<$(pH3aF1#y|9vj$v|yRM9!hnB)^C7i~JAa){_|w+lWuQ z_x!!srhecNhQnmp<`lnH{#u^oSkiixc^XMh17ti1IIXHL9Uw2u-~ctWk-3$PAb6q` zW~&G(T(aULrsmo{|6EW1wuG0C__y1?t?18Jdb~=(H9HkK7b7AzE)@~pst9c z`PQw~!8h6BM-as4(}25?Ch&mlcM)mp-+LIa7JqhJIR>&eFQ$qo79z~IiBG-&-xq?Y zg<8rS_SA#KUfpYwq*uk#+)JeBXNr00&Jp6%kx-pY?AK=*`|_r&s#6b}&~{Z-ApTN! zKglTHm(m_?m;=w_4qbWNpH}$9(FXoH+jYaC5(uXMyzQgL;*(R56nlBtC8xl%}6 zgn0^0fl<$Capcu_ky)2OQ|wwB%`mSsZ8J-mNgtK$ zOFg7-7VM38g6EbzNp>DW6<+=tXx8^m*56yj&IEtSM6ytCrp%n`3 zUu*e9Zuna&yS=@=Au(^mmYkqjuM9SaW3u5R?VMlc?P&3O^t;QOh25vQ3EIgh zY37qMVzO}C)Y=!!cc*4%+)em`bJY#8a3mf3c0p@H#5P*yk41mfyf`ie2|ptp=Hj-b zki)FQ5*}ALDhqRmPkYhzPY`{&9j_n7DZ6>$u-KarZ3IIDmrV>#*X8} zUaMlZ^jv(5#n!h%mllL}v-tT0<2fXJg}rUJNrcU8=ak>Ag>2UOBXd>w4Px6&zdU2U z<O-+qZNhOP#>?-k3O@u`>MZ)%w#3IrRwG@31a@7?~T>Ba3MySj)2et=K&v9-z#o_rrcp1YVtEZjpEvxU=o+aJQ?GMqPFq>IDRHN`7;-O6 z=Dd47ylz+@4tZhu6ygwSo6~JR7ZJR%z4sl)8~bQ>7m@zp$ki*AeR_2bTS**_>v_br zZoT{k40&O->0bL=f5vH8ydAgPmWGM8*4M7+9OakY{V}Z8sjnNxFnz!E63XUqO0Um@ z3}>qcd?Mkga>uyat@_i1{QKI0nVdCYHivUBBtG3uH|SOHTVo;~3voji8lTo^xK-3B z>|K;A_hsLiKA6)a*qCcWs#nJ5*w=g|Y4pxXew#~D%G9M@Q=7~pNS9+dSFSJgBvpR?5h{Ny^R%zZ#fF9hlE4Cb1bIejJq^dqomKj;tw9yM7L7gq`@4SQ zGl5WVqMz06Y8n|B04^dd6gfYmsRPD}`Iz$&vTvNa<)!EV={kY=kJa)2_%uKxrC4>z zwjtjdwZq)0QQKl4^nCI2^HWP?GMS?&uxs(Rb^#a`CIIpCxAbS(q}tk5jl={g5c~Nw zOtn!>fAKe9y^i^p)6o3PrllqAw5fr|y!uaz`KH${M+>nye+kHu4zpBgd=o9OxnM!b zkN>n&Po=RpM|gNFk8hKir}DC}5cW(&;U5RL${Vkq5c$`j0pCvg8m2rC9CA=v$=2`V zKxJL63fy_MlSixitnXhpD@f?`TDQBZ{aQ{xxScUoQYik)-qGeI=%Uw5o4p?9Xpt!4|l-6x`6mo;vL$%QwRgdE|QCbJMO zqH~;t8Y=pd5iq*(0lX@{Erv{vZl7Ef-U@odbJpLs8*t1?#YGTxwVZSW_(9D z=a}J?mEz`ge!12!3hO;8HvE^x8|n^Ke4H0K@ae&Pp-lRYzldaknN2Wd>MA0Ex=xWZ z%gI|hv?rj`W9olvBzLi2J#yK;WS8nq$F>NPbU;sR)na)6HD0#NCIkvVJp$qx+Vt;i zI+(L8x!@&%^S?W7Mw-bm1z1T)TPU50`?lZrKbD}^sT}$0p zP}pht{Lr{GQZ}lvfYGXfSsGp=@w7?p8@E)DjyQ`R!d%*c4!1`n1qRc&^&)K{n$5}C zU8nO2dfEEIT0B{Ft@+B&+@9F)TLP@BTFgtZX5HI+c(2cJUzoJ`HE&7`!)X6uwa0O2 z|9#a5)kNph+!MW(gqJHXKIZEVuxF|>Kc&uUU4@+?TfChOe%F%TYIfg;oErRSbLOBRiA2AbH+jv|#^b))a4z5dI3lJttLQR*^?lVd(e(=WNN3E2RQTvU zieSG<%md%G(Uh_l=5>r@IQRbE6Lg{V=39?bxTv_m@bQ^koxjB4x+8><=mUdQHrX$0 z?aaaY(=A%-S)O)+KlOQ}<+JE13t5amT{{u|f3fx^U`<_H`*_q^+gjzewxTk9x3*HN zf;bT7wAU)N3W#t(Wr#`@Q6LOL7&6d$TUrrPii*mRwy4O7fQS&LBv=Yjk)#YEj0qSZ zkN^Qf5;Fhy3DtUghwpyB=YJlaa1zMb=j>q(?|S!I31N|E;#`d}{Tg$ zi4IA7E1IMw^?F^cUlS}Fd|hdCG5QHbCu#yr&k*C%G@1%`v#EfZqruC=+Di|-dx$>4 zj4ik5kePjoj?vbbtjuuAnp*kWn{pnveQqyT9Y$2< zM0IoTmH+!3@|S{Os$h6E$ad3lz<&AA>pn?KQEZCMisK0h@21_3KrN=4vJfupL)p~~ zEk#}Y8TDhCR1n80hCf2}RV&{A#{kgHApb({H8J|?soo%t+`>F#IjVEw0o?%8%9aj= zfc*I&#xv@Modc}=ohXp*xB7}ba`O)!D2P4BgLzv!RAEJrzt}$g`7FO~SK%?7pKkX3 zZGgY+ku*+MzUz+DgfMK5D~82)+r6JhBU+vK4NG9;e&s~v+} zn2(*9)N_Y_-JAOhSv^v`Z|RoJXF6lCflp65KcVM8!z>Kcd+HuAC6Vy}2mWXU?w~%4 zm&DSd!*8n^F>y`Yo3F{gt$Azs%hxDMtyvIjdw^%nx=896>EPd`1g+tK!=!QG=@b}K ziw&MN*T2$|XW5<6o#r4&5^oT9QG-5Tv8y+D>Cl&0!XtWOcs0d7SK2eQ5T`Ftr+sTw z+$kr%7){fdOsDloAa!cRx^Uj! zUk_x2yA3}E#gYqKms~3zx%TuVE8_F;hx=|;<{#cdyBus6_;;_IXZCxUdD6-U4Z8-5 zHhebvp=T>YNAId!@Z2D7^V}@7ze@A9aWG`WIeaI&6f0r|3+^!YMn<2OJ7s;Bc?c-K zp5ZI=gu*L$Z?1Hldr!Tm-Cjd*>(BCHhCE&u8{1f)=b&0>eDy`33&$75{ zpLmp&rx<9zxLsqr7wnX`Hu#iFYyzhftukUtSXispB9UG}t?6f@dbpXWyHEYTk{Wbi zSH~Ycru@Fs_vQEh@w{)(MjFW5v%jMx;GN~DgplGdpMW7SUJPtlnnodvY(xPZi@N0a zsZ+Pi$q13aRN7(mO5oCaIH3m$aiS`F-vp+{DZn-O+SU|$`J=tTpMl$UJSu8cT2s2& zvVOT$rJK)=bk^J!*3!mR=mAt?k{iH-PXs{A8&yQ1YKh6Q>zD6{cQFlM-G@(h7}{5k z*V$d|yy1BDyW?-YRln=tvrvrcAj+|C*L0ui*Wx9=1uN4df}xvMo?aI*WgQC<9qp}2 zVb#~4dF38Ha)mKxx6P{)bEBtXt9RK_$z=nft9kg={B618t63ASwN1W0>DH>tJvisL zzmdru|P_4uiUy=3WKr+nPl>VUsn zyk(uW9Uthp(aq^YV};WIA=np7R@i%7cBFL3)(oh7NT{;S+^os0p4VPp1P{VLH;vLC zgZxJ)zcliTAKql_TqLv%o_+PhBEw&9ZPgm-YFDA!8)ZHFttTpGastX5;au$!_c!7?_OW%B_S8E$F2) z-}w-h_7s|57mcdFPuAbc4SfTk_sf5b3PQZ;<8IceZzI<#xH?x;y!BcAJ;$q_6ZI8) zf2D0CEMyoqqKq|#&#FdWNqd%bv39--EaD!j&Ky-%?$@v<; zZ1c^6CrKy6a^9bKr&hiyd`XPrA^T>J|2>I%TsZgT<);IVzr%c{F}ukS8Nd1KKMmhM zcKnMG@37u*bUE_Qy9%g11WpF1+=myJBB-*HY9M~eqA`!twczGEHn+}`f8w6r`TJhe z737%}AQBkiBfXU;66^ldLUcsV<5$w!+PqL8Jac^4=Ts7LcqJA!R8>ygh62_b3bD#u zl1U>6u$sfDX+XW0z9vj^tL{K%Xu;JndGZGSuTF1t;KSAOdyd50`vPyxwr|1e0byvw zxL`av^bV$?;#*lyfL}xNX^nN1rpFdFQ-Th?(SpS@!rsVm`*DNrQ_tKv`;7U5W7l(V zCxZh=7n=85sfHb`hQ54M+<#CJL->H#w9^jo#B<3yT*qqYlJ>OJVdm24{flwMBBCk- zd-X|B%P{Y)D7Wtp(|y0{XtCYjv2D_#75nPYZ>LkZK@?V@wcPRmS2pW`DjQV2gJO~< zkD_!lNi(5N8qdhpSLNrPMNIQQJ!1C8;Ww-4_3zZUKHLF(zI^O?)!GBT17Uha5IcL@ ztAkhPHx!iHv6iG2tst*U<0;O8vRQp+CFO%LVV@^nek1Ph7AIBjTKx2K7b`DS#=g9m zU;f)aoxdcH0sjP%tpJtn4+XLr@#HRryDbBS3me`}qX4=t>TX$?vg0TKRU77S0prLN ztj5gm!Mp-Mi2&w5Ufbn^Io+eWGJwVAI0MbedM%Qhmw#~6B6!A?ZzoT(NpG5p3FZ29 z?qVb*8@>>j0@>c?$-$RdZ{%^s5&ERp({_C0T)FGbXM1e)pMl-p>o*U4CM+rEmw*FH=iU30@>>!zzQuSv7KN4_ z)_CQuj)@0@>rNUoM=iA6&ARktUCh3ome>R@jC;r28ta)NUygsr?E@2(XT_iOt@F7G zMTo#+UYnShqgg|B{a&!9JklC9w3_W_6dg*oLP>MtJ4ECU6e~P123ZO9zSGi1Rh!L~ zx7{|p?ny2Qxy>g(Tlk;+Pm|&?Y$1)STrqK}NV{*E^Ypz-(`H-VYT>T26c{a0(bqgY ztiJ-NTAS8WbHAYzy6XI)PYt*b{{M9^mq??e^uOdR+S*dcKZ$CeLSr&E^2 z=$+hu%p0B8m7C+iF4X|ch~$CrSPH2;$`Ums<$0odSQrI^W7?&!Pprd+IS zC-xk}T-Kro?M_r2Xps4H$fpi5Bf7X&hZk<~jILJXz+dAu#4q)}_91ziPcn8WD5Rmf zTV3wQG=GyABlEL085yJFHn(GlgbTogdub?BTa2sry!^p`I{zcOGWX@y4 zB@g zKz&aaqxmly?U?c`Bs*O{h`J-)P%}YbTb9GHRj!=yYl9Ay@!F7vL>&Qc-EzVsqj_l> zou!{bS-PCR6Ub~3#)O{cn5YHb=A(BoW$Oc7jsE+xsvM6PDKaevwUEvBcq^@G@NnND zN?7`Z&2Di{U(+0-;m$cieoK(C=zuWWD&;Bn)>P30-ET5yi!{7ZU~xDufslLko8-uM zHZy`HvHgnA^UcHGGOi&T%}{PqHFw3(O3}`_kL~9&6b*2sjN~S0I3y;sas#^_v@SdpE zcRkTR)x4E;|kgj;5y1sF->uy@pp2k zhWC36c_9|yi+=w>nU78Q)5?44uP^^6IDe4-JFBpBaGCWC3KgonpSxscE|+DQR#mkd zQMf-!T~HqS37=&%reQO07+Q|V^`$Wg)Yc+$035`dOS(Q>^%G)BgW$&@=*pkWY>*lo zp<_fkO1I%6BTd}ENUrBoxfz%YZWsp{zWH5HkRc1-P(&wZqwGpk)@(hW(F*Ec_Y>))aR|N5b7^G&6?+C-${&UaJ+FVN#R8L zIn1qXqdh2g+H8Sxn(x6rT6X`PfKzXv`SOos`@iL0pYvyWLL^$hoPslY_!KWYb| z@%<+TKh?BmV<%acpJ*S`46mi_DBkg%Y`E@-FSk?a`w@11^SOG#iO&Idb3c9UU{+Jb zNy`(Gx4K3vx~7iSZc5l#<0HAc74VtF(>F9LEl zydHDMRXcyy#z&3j1@lN)CCPmxgUm1!yf8nB?B1p-P z(0*R9<4o&20nO{@T|E96Y~UD`rga|u)2ROYw=bl71m@oQTR*kg>pSe1%*;~AmT6|h z@jW0?u;gQ)UG;+5EI^PHY6a$L=@{KG>hdWKhF9FV83lp>APGxg;F4=x@t;*V-4ECR zza@m&572Qoz*k##kI7u!*wJwq$8uxMn#!%)Cp}z=>H*+#wRq>)_F`2hM&F5Y6Rqc> zj_T4DHurF^>xv=J{!o>TRKSC6qWHxOJ+q?Ua{g}$Lv%Bt(39?)`Aym51nW zR!n?+CEcBS4$y)dm2NGOfJiuVH{6{@&+lxNTS@C)9a=#eT7jw=`Qbns+2Ga^Ty`QE znWPidu4xBEux_QrInj==XYWwoRvPppDOO#f9=Wlt7+J< ztYNQp%wCTQkCyoF+1Ln6ezaq@;n*9+6-4Wa2Nl6LXGeywTn>HrG1@asS9{|L_t}|` zy6c4nS?2^wO8uo}He%|!{r#Q}p;7x^r~3+guROcr6n**qiTQW+X^EDtCBj2jOKi4B z+%D0+HTX?(tQCsy!@5yt+xF$S;3nyBowx6u^>b0{=RSA1XxMY58zi^;zJn{P%^MFN z8Lj?If8CC4=U3-7DZGr$G;lLFhvv<#$ESBF$JtL|3)B&N@mse^onQ8?MbPKxKR1IY z|8FDtI}!BOtq)M3oXQTAwdS|Zc`!k`*W zj(Px)0C%Kyx!fc-fddtW!3?Snj7A=+ZtzZr>UhE>KD_@}NKVWC1cHrnVTJL`1}gD? zh}gvW&n}X??R9Z|!+lMggI#xi9YqLq^b&pINMWbNx_>R&WmojAtbOiYX=&I{^|woDutxRb)XyLD3sW!kGiV|Fg%!O6bM48x}(x6bo77PQ?;B3q9JXD-m%w;j7kK*M`V zumQK@?BeJ*L3Z)L^r8W^lVAFoTO6yh+DrtmdhtWwK;muw z3FF^zU}@G16lK5XnbygHsWRSPgxYFvzw8jO(2>N?OOk*?{92<-#{p^Ek>J_G3oA_V zUt~?g@Z6t3=JT#OV*9b=Z<9TSzt{oz1QuCrkzlT0pmaoe=!Ov>%v#>iVrgG2{|4Q# z1c6TEkJV@L=wxBq{2WG{x&xOSa z(1zl+jY=GI0u*`D6G4^orlieLM?U@L)XhM8MSi?ps`}nL+BV0q_&&qUbdOikglZm0 zy1TwY755H(i-vu4>){l2OqN%`k|Po->DE13bSr*pfDBVM}KufQ^JwK4fp>w z-tCvV1F`~7#o74_0UJ*rqhf7x+pLx>Xs?_@U90ZwGM@3nJUgCqm1d_n*LN~o^Dghr zG*Qvso4If2?`zNxe%?o3JNj1z`t&IYC3uatZPVrgSFXy_r6j%MY6*2$M&Xh5`a7|? zPpaOYy?Wz;aX9wG#+S*e9i{ss{vY;mbcO)rc@%8tvekfH5<@`$x}nH=K6zDRztE6~ z0+0pHXlXQo{&=zqza&W@FI>^sUgViNVJ81QTKjyC(QUv~0kmzz&`#@(d^M7bemZP& z!c3dV!6vY;5qMe*yl<47#FtWcbo5bTp72_$3EH8fGu%QYH+E&6J$Zed?YquTm)&_M z^1+0*^43SV&qU1oUZQ*ae50(>N!AeFQbQa6U=QD?ysR?`x9g+SezE)kX~(uy(J!U3 z!3C86DV6Pm?1y{byF#-no}KuxyS(kayoir{`*-PQBY)*C=ZN~+lkz2{`&&g%sGG9i zpE$oWqC?J{xO#Y<{=(k#A74zo@m>J?Fy^O&!glj^nN#J_??`+4aPR1^-tNCN|1*?vn1 z>Sg`|+nw%08vMVn#^^1GAq;{7-+5S5Qv0#0d@M+4< z!wvf{7kztjZPZ5}o6l|%Elhl0okeIBeAyLCC|a<#vkC9s`ZFQhTQD%XfJB=7wmHf% z9dZ(3t77Goq7tJY)k^`;1j>OH%lH!mFdWTk`Mh1d33 zezom<&ZievHOHZTI&<^tvoyQ++o!9&(!Q+-{NjTRVfX&(kn_#<%ZKRG#M|-S#tJIc zhttG3wmi2)qR?Du6KK(sV!D)4&yrTEYVTK&9alIhjZNeQGiVTO8ixjnszD2R)_~~Y z!%rZy)Sf0dg+~ib;7Q%{GIUMBw84ecB`B2j_G&zun&Dk7c99R9SH?z6;)HbZouhfO zk~Z#nY~)nFGg%>~4|?})^B8(K{dAPWzA(fy>NLxYoMxpCF2>L4UW*}bmwYMB(N`w4 z$DXzn8%~h5&mJ4|B;3l;dD+E+(F;CV<=)fU;m52^96j9#k1ssMENdZK%g^b;?=zU) zm^+S!oiv3GPIuiYQzqFMN7h8FsLZ*Vu_e!#_mlJ=6L&_?wNaTdHVDtT+;^(6zVwLez!ts7H99WbRR3E@>h+Kw(g^Sg@L) z@lt^NB(PGcgj2vKw@{vszRL$vlu$J#7E_6MJX#q@gP_2&;Oo|n)wezGO-)T@N-d_q zoh%+{l>BV-V7pvi#58W?FKqRRd?6?_-h=6v0g?+2@*zt6Xm!%Edbrbac>5S5X83Oq zMpJ}2#sfj8dk)jR(sg>y*axP|)sU~Xowm@5h-%8^b8UF#La$U8qt`!gX=XMu`t|2; z{>AShP4E3zP^d1j7ZV`*r#6n?z`Di~Yq$&8Hzl$?YcVY)TsXFicnF&wjSt?40SjEPEy zI`v%Y?)pHH{)7lC)lJMsxJpEP30|qQ8HQRwh-arV@77ToGsEb)w2kRbao;pKksM{{zRU(B zr{FV~G;2<-fc1JkTurCbI(0*j)_9C=_UY|eT{L(N|G=iUsi43Ly$ZR|(2~i?)JeVz zGq`LTs;?k}DXtRArW%TbL@SwqgD+Lhajmg!(;=*L1vZM5=WW+m-}HTc{x5zHx%B-$ z@taTj@=_ROXC3pP1Oma$eIYU`>VXBDtIXpT3lEbYt_(fI13nE07R-Eu_OD_vLcmIB zZ_(jWI`iNZWhFI)Cp2owTl2maGH;I+%mFKK%t|R&ZxwdOIj>& zcO^qZKf;JC1X7Q7%q}Vkyu-~c}#x8t~z2gm1 z9Aqkij4YAt1RM3>6u?zxv`3Fv;1d(!PQq}n+S5BW-)2#0){yJEwKDP%tuH<_id)_n zozF5~CMZ@)&1MPw6629{HKSqrT&bM#$K0zi|1Tf^ZU7A)lTV=A>0l;;N3dX|!HB=>=^X=hQ@{v{1_3Ri?`k1z zq&q3gPmQck2kBk3T{g&-(pFrpOc*!oPVX}ojWR=3Y^m_MSZ`#Tv-Uk-`!8C3>4e{Y z^#kyB+I~z=#lo_Fu-IwU7!Qv!$wM|9@g>C4O) zI(fu>C0cl-$ebEiQ2?-6COOJY>4w;tdi~#h7XA*0%5U)f6GWNl=wQ_ue0NRO@G&*p z0fXFnj1eu^=n=HSY9C`}EDz_=OXb*8=(IjT?fDK|A6ZtfLbH280M^D!#2u4ROi!x{ z7wx;6SUTHF;Abq2m?qftP_(53t zZfvCkqE)`payXopB&i}LtYI|nDDUCJhb;-22GeaLFEbzSh{JDWp}|WazBGT($zv{A ztJU%%ym2FMkcZ7lKf%Qq{RGr@x)NEB5ZdoLfF zWjqyf>2`LZReF#dD42MhxB>jRn~<>jITxTtrMvM$x0$~Bk1xTVK}g>w{TO7Iizv*@ zsqH}Z_vs&F)V8!bImeZzk&^Z0feGi&`-5Ki1Twb|y(>YqojG&nYX~S)FUWoPb)P$D ztByelph^MEDR8B?<1B@JH>`(GS0062U0p4vUNk!NofrQ*(WK{wYbC?%bJ1}l65UL9-L~gU+`Q%&wR$cAbXI2E( zT^YzxTbul%0k`9&e5qttAX-VGP|$sCV!31&9vzn~T9_F_vufdTeP4O0w>(YErIvfg z6TJAB3hRC!s30Ymz4E?WvRJ z&f?{5=s=r=0&j5rA%Q$#^NZX6=ab*3+3`3;VSgAE(24AjO$r2(`|sgs-AU5QRD4+I5o zo*zm7bF`e#i^z%|nQt*Xn$6S=K6*VXSlD9(5oQGps$+C##(LJb=Gq4$mH|-nNQ4Yh zT&Jq9Td>o!%`KCg&W*lf8vL?TU0c$9jc!3UjEUQMaGiQA4>5kHT_zlO+j2eqh;xG= zFUWpYiFa~Rk`L6MQDShWdL^{ox@9r~Yp{LSy%aa-F^-iSS81hWIWeXT!t8!wM1e z2!v)PGh>;$R#pQHj2RxtLG|+#g+E3in0^@C*h+i&K&C(Ys8LSgb^XBqnKD2rmy?C(4qC=@<-m^jk>hFD%Pc zUG!wctxMqek}W~ej`To9s;s^kE^3-^7az4%v^GUF6m9V7sJvIzHTZVjd zz^qN&iQg?$Ar6!*gmb#zR$Q!^dDwmeqADY(r`y5*4DBIW)Blqoi_u3O1>VEZm(z%a zGdMU1Bd($09*f0_C3vtjxN;+pr3s{^fGr4zd2~W6J#roc=FeGRL5Jnz^xe2pNr8+I z4q;UZSh|Do)`8sHD{4z8e4{Il)V#bhGc5-n4Fj(T7n$373KDY)XkY~z=c$HHIJ$nS zIx}_}c!DsJLKi+=+1uh&4t9&f8XecDlkCDJbvrSSm6oLxfL0exubA>+De;*v%Tn>k zm#y<7OMqW$PoBc!JAQN_Vv-Z)Qs%*GXTcIKK^g9!kVM!ah$QY{RxM63j7tv&mP8rH z=( zL{=8=k@0+>iRa0kPryk;6@EHDNEuxS;J{4?Qj(p)? zAJ4IegUIL_PFD2vP`P6AabSNEc_N?WCw?$uco6&P#D0dv2w0{>2qe+cheF&`f!wiW zJ)e$vB*>yNoWUbn_3M@9h)WswXHB^=Vq|n5DY~ccMrg8&hlk({dM;+8F|pd%JLP1I z$Ee~Vcj-|s>>VH9I(pxwZ7@^4FLVZ!AS#yt_?ZA928NufcWrSBr%)3dDX$@q6VrVs ztgzHJl(sr0Cgu>&obb?igm+Ro<(IDZ=exP2kfGL5f1!>?p2OXaEhHWeFTlDulSPkf z(xXkTc_aHCsFcCTk{!0T@ktO(HD3-7ag;pr(-!gO8BfTwWaBZY-qSA>&jT4SE76(p z+cUM!82$AZAKv$%aaJX`p(jbq?VX8lmrJWMyb7Y}+&xVVGfsa;A z4^-3}XYsVU*&N)p7HocltRc=csZkzlGhJrd+x;I|)dEdd;!RL@j7lCvHw1en2-Soq zqs&>VuA{~9?QE9YRBgPUmAcYHT6x+5+F_)UE27N~%g#Dup5 z-qvSR-CeHD-NPlf+6V5Ydg&>-Y+F5A+Cd>shSALK9zYb5YVf~-V^AKuOyC+abpXN` zr9>3(?Cfld!PK1;;{qj60ct?if_BUN6FR-nP!)@XMW71_$lnkGA*dW<W=JI zi{v_UAb~BWxMO4c3UOd323H>p0h5_Z;>pA1i^|I;qDYY3eTTdn^yLDZ906hUJIf0+ zRKi)wcv1=`xq3J6C>co4QkZ3cumQZ1Aecb4i7B%R&a!gcI6jWRbs)hyd;tS>Z zaHvtk2Y6>~HpOC!J)7<}@>fXztB>we2;sk?$E@U;$}mB;u|6R5FUU2AQr0It@m;|v zw+kU*A@W2erWgjcatJSiC{NnWbESy=3Y50Sa$^ykXA?Ikn4MmZHZ7H*u-_iBtt~A9 zK6OpPtZbeqKITc@4^MR?0??gf>#xYfswQEQC=(|f$1o4n*1KlP8^rGAaIyPT#99l` zbXiUEqy6~Lq_m5h%+@-o?MNV7v5qdlkqJ*dt;2fRmK zgkWk?>VqtYa&28XO#*182NG@+TZpm}H?mdVr@P%;L?GF2V-BEm2|+yZIla+P5ZuU{X zK@J=>8e-G?wDgyFOS$3F4sfYP8%|IOa?O8p#8|ZfyRvTw_%mU=MZ^Q%ixV^04$Z-V znPiFGx)$+FOz_ko8Gz13aJlYD`CSP}s)tAPJ7k3nd3}8l56naeVQA4b!#0Lt7%A@= z{d8wF01+(oFNEn3umL2%21rek zkNnlFWKp^xVsz`>=uI>$5NrltwTh(bAUkcSxt!0-!YgzP-iz|1(D!F%v1 zDhimON~OP;(LlySXslBp&%#Kh`zzY$VEwO@Vh;1VPfEx!_pbNr(~aZdG)*U2!p(oW zi^!cjChSKP;56gEmJMNS#LWf@dbf4IK+VAiDE@Bi_7jYBW9Nmz$m(_y0CoT+xX2}wyc028uN zS7oPbN3n)cldB}z#Z6StjY@Wj9p@s$@)5ZJ4T61p5U8eR)>l*Lz^N)j06^4`Xt391 z9MIgl;b>6bPV&IRRZ>v|UhWF@P24g`rQun4K(V4{3r(LB8)&=JOHUu`RA10X2%0$^ zUZ%dSfs4S@$)*-4*?8+fwx2$s`kJIZoV@QiJ>HK6fSQmmpes4oZ#of}bud?RmkWbL2`Lb=mO=TB%Gwji?Fr;yWAVQ~`0bI5(DH3SZZ>*y zp9kgG0NJ7Sfy0S}fL}V_W;7q-`o9e@Qopod@;)!)G8v@$NkE-c{nQ~WFe3&6# z%z+XnI&Nj^EBOu}bjfRRd(pbvy-T{DB=?AsC;K>&h%MSi&&(g_)%7CCXI;|lNeIwz zr7!|#A1xT;?-N8#Qi2E8$>bg~Ib0{oa&>I6sjbT{&Pg@N`Dxj{!Sh|7Yy+tI0w(eN zZ8TJz&{-*a7o%$jq6We!_dr&hVpX2F-jA4ACGeQQw9zZCNvbM&Cm_$Z-_0@FjW=jT z*7fzNOt?An2|a6I>ch&chDi?RykYz}tg_?#xhIaq(*x@veGy-u0~-W*!-W##!(G3n zI4k=5uJ_qkd+UX|F?68fj;H%sQQbaba-?TyQZTaSVr2XMUd+1}{qW+XF=1CwIsaTM z_+X};4%RW^nVK-W)TE>%JPyYj;wH81gf$2RWD1M)35CLSVgQ9ei1<{fL2fIE0XDo0 zt|KF}05RuwTU)RRVFLg&j14p4{YY~F^9NXQ!5H_O>^&CARVN-;=n)~+HweMz-sOS*9yDy>g=L*N=(%JrI3Qs6#8VIR)kjylgeGi&OHvXfn>*eWYeD;4Nm z`t+n1n~mRWwtBU9pO=`<@lo{h4E}s~blzNz%rK>b6qjuW9NXg6Bd%Aa5&6C=tknay zZo{ubjG3?P2WqmiWmdwUEES`bTJVUC4D|nk!aRMU9_KAV?*ZumA;Iah9>xpgexe%( zLBtViMnHmgijKQP2f9;fk7Wk)5ZOyf=_-O#CJtba@tUla_h5iOz`zEy(6`KjSO$Wm z|6vrTt@ttffJJh3Ho0FYEJt8K-iopAZxELQau+*uOC(~l0l>k*1LK&uL{Cpo3%My6 zOiO9mS?EhVUI?sb{_Z&(GIA&to}YqE?(!m!r<3k1hV!QO5fgEaym))AFpx(#Fv1~4 zEnPPXU+wSIvNL%8HUfdm_2bUG3lzS1VA1QKT|@K2fP5)O&|Z$Qt9h9!SRlTJc1&6LVt@wA+D8ZwRweG&!_;C+$>Q-dXG^-h~dNY5@<2lFu6aN}K= z!C*)pl(|WMEh?N$%>nlr<1YCRQytZqu=Az4zGy(3CNUzwl}Mn@MeBslYlAh}HtKAc zIU2;gW%37CuU-w-eZPaumh{XvK(UBge9f9M2cOEbr0Yc1R#q?J(`FxUBv*rxEg0(o zo@BxebUh+i0O9~K^iu$U09C?i5{!XuJW??BY+rQr0~=9RGHGT7`5tVq`^EmQw>+k4 zop>^-siw@q1l>03dA+J!PkrLLgl9y$fcFY^2>*eUDN2kmtIl z2X+Y_8oF?JCc4d)dX#{SUmEE8(PN&R1fGAm$xsCaX9PVqQ!RKKyy~{gWOw^{gOoLA z0c(sTy%n@A%bOWkn{e1X8eU^la5+l0@Dsp*n*r2#*iw zfq{xu!kJab0O1~B`d^kK**3_e27qp{OIsBP%SJuWB4WryZU{&K3InX1s;UGYuoD2K z1w>*)cvNjXxeOUh$u6Oz2D>F49fu2#$D3|g1y~rvKx>33bmc}(d7$D0V3YA8Lh#-G z_(4J4PjIW&Hi2(ZEXOm4SI|<%n~LQtXcIdFf+{1(yZh;jfs;UoxU!8Z7p6@zfQ|~r zf^iK=VF@?rgM;o^9w+4qAFM9t(OBaI*H}5BI|*~MTx1;1F2krEgwAjD7l!hT%CC%7 zn#SNMBvgzj_J!Y)Y4852M_+zhR`kL#q!c0o0HNw+*?=fD!0s6pEU=@+{WrwyV*>+0 zgd<9WXmyJ2AXM?ZTyi;;{5(qw-k$U{iNoooa zYI7^Q>?pG_o8 zkLbF{6}w&6lO>orx^Y}=a3BMQfu4-aw~>RPsVHdcUB!1<1SCFokkE_DTqL&;nS+Fq6!XOx223@iQvc+0QQ*3Hikg;J7qQ|%O9}hW232{7eXyypD+8DSsimTD z77l?c34c+b5PrtLJ^0)|WJP%;C84r+GPHmUh`@ooEg)Q&DPKgAm=!EAaU2Ao?8DC{ zb2l@wX&gi`uskpHpg+%DHBjjw=#^UXO$4od^?(o5K0Vp+qS*o0jnpbSJrKd7ATGtN zr@K2K;l|2-Kv)}BS+Hg;AW)Gz3Sxl#OoL9CIc(TKr54!;Xfg?F9qdgPvat>q?y=Tl zXWzy8@{m16P_b;e4e(tN#uS`IF{2#=_A(PpgJPIXXm5fylrOQ#k5q9AgzJiLlgv4? z0K`C@nCYIko{g+lht5Xs#>5Yb7?S1|p#yc>U@+3m5Zt0#S*{3as?h7}My|^?mi70I zvqEjCk;Npb`f;MWF)J4H2o1x~a-sNgq^ws%bry^qxzBA8-*sSK8svUK*v z!>3a%M49>1cSKpD^5nL2uqL0Y&d+2W@IVBO$DCWHY0ym%_fIb`nTRpQcmg4q2l@;k z;rC5sD+8zium=J}h)NjXp?A-^*&)D^2#IS07GEwo84~ITg$xJb8FWi6g5x7GA%K+J zF@@eh;D4XCl22CFaSf(0gsF8x%s!7r>pUUTjo+*5ew1#)u%84n@53xD-Uz+~Cj4m= zUQnD|Q9)F+LILJl()t7JVP2M?Rxk?t^mv1M%Vd3dM()~qjjoMeWZ+j zsI)sT?TYQC=Uf1pEoQ<3L^^gFy7^wqIK@hQh=BpGtq-!DD%Q*rcJnUK%{!Bd%G}0) z78F%BgQ)i1c9T~-#B|BSS}7j;;s*rRt7)VDRLffa4XRGIfAIY7{&ChdA9a6oq{dG4 z@N5^Rm=q*0iPi{=^Guj&=*HcZgczsYp@M$q@E12C24>ya*)%KXejtuzC5UdkpNA|T zn=Oi0|NG^G1T_gvbw}>Ao;aG2VwsDm@f{e`g(8+VQdfF2~FdO839x^-)W=B&r$ zwpGz*zT}GACY{;qj1FVL`qNpOws8K0VP>Il;a-kjkWOg60H<`A5%5qTvEysmcj8W2 zZlJVa&+<`4c8FprR})j9A3E7Cvty74-JbdcLq67j~|zy$kG@$Lj5K!J-AK zQBE#<5v1cqkfJO-nddCo3Z$wiA19YI49?f;d%NFxigAF5ceXxt1jJVWmRCxhW9eY+`C!X)d9KlRH_e@n9$r zo2+CuBzlquN#pyh)Toolpc~>akPF`QB7!JemTEWxObq} z)!A;Xt+tTXB+r?x*P2UgunInY2mqhU|^axztNRg_GseGHmR9j(3X{&RZE} zcG{Lbo0@5g^QM51fJ1Hh0lhVXSbLlid^T1YQpj9WG0O<`Gk~m^* zZ~%Z#U2B=4<+K}IAHz2F>SaMZ5mVd_Ah5q; z5AjZdJ|^tK`b>-Fo$~UzXxAKt{;{fqxd+5bP$;_-lR44G!eOOsV;1odz}Lc8tgvN| zo;?=6Te-(a5?o8}z}31vo)94TgS*j~vLCWv_zrm9&6*Lsg;`=bESa2sT!R-F1UYgV zH$cH`-Y-zXgEq12NSx5tvkEB!rpU2ry3t zABzOf3h$nC%=2o=V9}9Q7@y`LKA^?$u+EfTQleqfvg>3nT zWieGtCcNn#8E%;7TPV@&pqq z#gFAEvWTOK$P}9}caVKK8nYf5xP{Kl0m=5bsxWHoBnB7=-Wrx{TnQ^x-24al4raZJ zday;$*aD}>Tz!-n@>nr&?8JsvaX?l4`4htVUq}zsuaKLBaMKC9ahK}!u zcBSv;fL!CAAbL~mQI%qJ>$?ZAI~BJoN(>yXJFEf=3$f`kx(Pdl`}7B@qn@gjZ6d9p zpdf=>tgGWQm@rZf5n?Fu*!UeW?)BRZBj?GgFIdSHE^5KT)9X3J5&YmR3$B}cQVHpC zDv&2oR(psE!gPkItOnQ zx+wkFiW3j84z7N@ld(Uf68ZhW`NZioxIo3lTs;QTbIt)aa{H96_9to1@RV#Scpav# zGfR}hXoztuG2XNVJ4h&GoyhJ||G{on0nHuIQ@!IGMEL(1#UX=Hwvl7fq|ZC;Prq~9 zBS8?~4T+voLZym2SgjioeuYpbrlzL@c86*KPlf6d4Ztq8`c5U#TY66hR&xsrOI%rU zSH#QBpSMY9sfi^bQGKv8#HR!(U6~e60rrc>1bxtWSoK1aHLegN`xj}NbmNT8BID4d zEj@Ff`%wgfm6`(FReYel5jrZ*oIGnDSOrk*&Vim=bJ(U^0Oq%{xnqSqL7Hp0wtnEi zRAm{cNHO(Gbz~l90wcM*N#r@${A@&1Stz#$(K*0xIh#y4e^~Z6V+92)yRpBGf+_jsFN({vso3}t zf3_kU;BvhtdK*^IvzF&)=ih+tEu)hZ)YKjNTQ;fn>0~~%d-7E|Pq6AzU1jm_vFP`$Jar%e1cGO-l zEJ<&I4+(c#&W+ZC4Z$!|od>v`h)s#CJvzd$c&tW5CtzhI@Ss4GK=xOd86INpNI>bK zRFfD53Mc^hd+7BT4katgJ2WZ0GDB&xNbnf2d9mVa4qVFbe6%w&zq7Y8oYvXU^(X}K zn`Kni20l06*^g(19{~K9;%|_$6%Y{r`XeUB66}JX4oWC2xTEh{f)Go#WX=0 zIi#?HCS~{m5y@nwEc8?#E12BEn()HG5-rQ9rKKQVOTn}!V6RK`DhfI;M?Pyc(_?0% z<@5_zP?Wym#x>qbkeTPE4=gEE87{-qfEocQ_EXQd?Puw8h*Jrz`wNn_hHsJHn{hZh zX`dzZA+VzW#u$Zjb!Lj9j5Ju)AblV2s&w!S#Y@9i8GlVI!)sNm>G6$W82K8I*NB$# z%bLPNCeGhMjaeAk-`eC8Y8LpW82&#D%pY%tcH97URWsS(lg4xtF%Gc+=S92aU`GuZ z;vw+n!pQRTj(PK}XBW4?fTC5{1$>5J2m%M{Cf~S+Iu{!H#kNO4DP!%je%u~R1YO$cz zf$9;CN5$*VaSWtGJ*3jBu6b>nL~LD=A@2X9>`UNc&j0`S zvKEWAmM9flcZBQG)ghRks{3HbS+&FPT=};kTD8T2TSS8Cb2h z1KG(MjUtTd(ay}&KFtC#DVE@=C3Ezi6UELwhb|-rq!0+q5k|G}!`lO6O_tJ>GL4vh zg4}&{xv^@?n9&18Jrp0}>fTruRlYmUcJ}NzF75n&x?|t`Uel}zxv{g6vxMp;+#q3T z4_jl&SU60AOldQsT9GY;wLpT2=u2WcG>(OaH#mV~|3cemmyBqKe!S;@etcx5{4w&M zF0}&P3r2lIHdGBsQ{7SIFXFW_h&M5y<*d_VV_{vP&%{1{f2sL3faa{nC^D z$Hy0`+IQSz7-9EhQg_`Eq+AP9 zymq+f^l6RKg*^rxZWcbHlKn%@z7w$T@||YHZ@;mu=xs4~&#~APWa;Vb%XChov$r)h zNA=1DjrWKwh8ViBH^?)$u^FkxWDv)rz|)jHp{1Km;8nw&8wDa2rRYlj^Pe{ZSO4ZQ zKCUB`(0D@bSu(403a04BtY23$aGFWsRISe-hy$Xd4H=SB#i{nPtD=JC5PutH>ygbo zo83c4Sk92zh%6f$L5}-zfxj5uC6S#N>?-b^gl4vGWr5qNQ|lK4xX5M@?n#;~tfr0b zOCq|1LDr<`CfhARXFw91`S|gI>DnKD{gW!gMlvFE9>zK^;V#~k($n;6eXnTWpQi7# zjPZG|;M6Ba=QfU&a7KM7{gh7CytlK-!tRs_Td?FB%!5(X%yDAoqgW7aaLYZ$9dsbKzpxQsAJv*B6A90>6k;+$SGy}kn0{4UHLZW%RnnjOn#Y@T8@sC+~2H?IU^{J|!`72Zds{@^p}C zaz(t3ihV&M;@AhXI@IoL7})>t^m0@)|7_ zTv}8?vH2n$Rv?cM==>Upw259Db@+B zpc0Y}26gy6EsN3U#DwO!HG`=A?raE(EjL0t2$`RjpmmR7my+*sz`9JMJJ7E-al^o2 z83h#=Q|aQNjy;Vdgd7WQF2nCeOga=3wO!i0SK$Q@S=6EX`F(kmiC+bG>L*STKl8!) z1ZCM=4Uf6|Xgi&+(IvUjq#!+Ij1INz~Yt8lCg`8V;3x6*>VLxW}4a3cweW}631@g#vfsf(8hnONB#!h zoHF`&(}e4f_dPd-}>iQf7N1NOyuG4DdL*-;d<%`C3}*Eu$w_pCYcQOSUx)mpca$Z z2BBi?*LR8DG2;G-tuX}y;;^dZZZuj{#7*8em6vlrZ(v;`DQxGBw5lb~<*0vr;Sa&8 zE}LcKl*@M}TetqIn{2blJI+~1MCiS)dkTve1X7b~ZC(5b%>yU`=)57zljEdhy}Jk+ zhc3qpT2iH^Es_o1*+;P@$Mi3b0Z`X7e))3^wBPUuV+yeT_|pIUC?ET&q-i3v?7GlH z3qYPg?2ZtyjVYTTw2`AVObVHkfC6NKfQKAe<69`vvcHx%A<&SIW=(1?VI$<(I#ki;jY_eaFQNR~?ys%E>xt#Y^fQsi0BG#A(?p zI53;%3{fO?Co(FcHtH#?_j^%)XiAljZk*`Qk)3n859m^Dnu>}|0^Mmh`aizJ)W=8X z9sg&~pZnZvon+C7AGLDmwY^7WKenBc<6xtd>S#VBgN$xWEk8EpviXoP$Z_h?1WU94 z5CC$6r~)V0JqCF3f}0G?u+SC>cSJ3cNcj{Ff0Bn}aRdA+AzO35rh!-&N|mut9@tHc z??vBVH2*Gx81;B4!8&V-g0b24#efmv;J%*3Dw{nTSQbR_9EYv3%& zf4mJB_oq+M!2kI6r!P!(AEFTGUxaUDCBbh)?i@<1CUl}i;m{>|+E9gfgTP$T^Jf-f zn%0{UUNCr@tcB5|Dc$e@liGJ^L{#vi6rg-wfy2ncMNEcLuV_vIPFu4l6WNa(SAzT6 zT<7Dfe+b^-vb)hcE_jys$n7btqO4+EkEp_Ut)rc)-B_GNRTsU+k%Wq|JeUMd@YqU* z&=X6=Yxm(+_f8&;Io`frHfL_okhhoF6&WILa^oH?69L6^aoKJUY8fy(;ItLSr!Z- z(eR1thct#NYe$EKyY8HJ5!6uu_ntIL$c_D5>WgPII~PgT+9X>y8u#G@sz(eo6&J)Z zBpkDDs=sKnXMeUjA5shJpZ;w%KL7O8;uoJhK1akp6WY*PTX)e4p+q1?Q;7O|?sOA` zpLs1&eh?*`=ADNW?e00mbK~R#?_v~yu+C>Ng_)Bc#EhDvI5q)xh-$Pbjv4?7#l9i# zq?)v_p3C3SM@yr3+;4U=6;%11n^h53~?Pbtkb@eBEB_RvYkI zD?|>57x4QNZvb9PzaVYjawzXbtr4ICk@|Gs_|tud_?QOlbHEzzLjA?$68NM#@YL%~ zI*cniC@mdizNkZ+7UMcvk`2I>3F$Q36CGJBT?0B9BT;YU%aJ%q8>Ju?N3KMgqcq6|l4FC40`JC4tycwEP z=%rnFEH$D@?w|MlA=UbS4j*(QPP&&$BxQd+F^Y=V2t!7B?-r1S!}Lin=tP<0;iRY$ zKXc?#1t+5_3$*%(bbr}g?_Ff}a3EW4lQdVH1BK@bK2YEl?g%IwY}hSQugr^#`_zKC zjogpv3t=x;0-8$H=EztpxvYk~tBQ&zZ3CYe9U#Hi9aHOttUqBSLL_#Z>g=^r@HOF2 zY`SVdYxT(Bsc`&SJy_$aEnb#nQJ97@F-9kwzNamS#ao;a^B+5@$mrQW7v>*d{c|UI ztwYz+RLNcev}uG;o`HzkTT2rSb=f!3y3f$PbY145bR@<6oz29OOo2g@w|GHA=VM@9-mDTI@2(&T_Ih6eRlvqkYoHQkCl$mZ8_0uqFa94R66^z^OByz zp^@U)fC(9ehfztJh;b`ud31KaR^g4g6v|g=OOVIO@mc&}&SG6^dv{Vwa(5(#D@~hBgscJ?1+cy`;3WB`ozft*1qFI+%Z4xm?Z?XK>&jUeDvoSlyee} ziQOXXP7BFW++^a0Uhdb;lg8|eRL;`87rKP3Be-JsAI*~0?8~M6XiqyOyI#t2%?X0Anwy-0N z(R=LCd%)alk2fVgB;qYQnb}O`*BTk|7eLfTrZm@6%*YlWrdg~vDKf4f$|Jn-*y5f> zfk+2|OMO%n9;~`}Q)rK0*?N~1G8LFyVJZTlMb1Gq$z@7w`nuHIu+|~7Su`jiu6%uY zrt2mIHl>OPK`DCX83cxDCc>jJ!^8E?^_LmJ8^k&Cd{g=K@*dNV(e2O0LDWeZ`OLBY zeck>B)qb%zpuQDimTU|ZAj*yGgY^ov)H|baM2X!3+N2<96sLigFsKt9m;gn@N9NnI zs~gB@q7p^Q<1Llib*@m#<}*Zu4zpi%Lif)>Y8ZEOylFiPZ4;B#A7B21oOeX-kr}yGl(P_mH6ucc?56aY<-C!x`d?$s7ro*{jW&&<5@{-(!sV#NR46p` z)uvG0$vXUlI^?OW0Mt{9XksG98HDzWh!4aHr3#`-1)>sjmq*b5Nj}Bfqz=>usUD4C@{66|i`-(l6NJ{FDQU>H`E@bHK+3Pn^S=N56 zD4$*qRX$zRfAG=B2?pz+I1$^49F)DGs(nwe1^{p zJM#^99%g0=66zY0bDImyIuypwBDt8FQE4>UZE>%APeJP*h~wBXgQDUDD7_22QIHJ- z7+Iv+ZI4!>B6dc*e7tJEMBO>Zh#vcq-+MF*i`UqrdATwBk3DTG zSbPJTDYO8g6OG(Ox^vpXX(QoDK5c6NcK1zA^bw7D##3HRUHw{_#(+%hL{?vqW~p%n z^h^e|qN0L0@|`h3XlBIB2m(*ivZZILr~KpV1A=J!-?wVTc@zMfAbT++fOKJoMF$Y( zRzHy32+MSLxN(`>txv|21Y{Y``vF3eK zF7E9w!u9LdyZk6qzkWbmCgpy!Hj8dbq;!MQ9Gy#|erb$c5*{W`zSS8&kG^c8wU^-h zL{Yh2VpF&?70NLZvyuw#rNpOblr=(39WxRQR3JB>Dx6JjMztDsm-ls>b%sNSR_kZE z^69ym{~vaDS5$S6M0IC61mr$+&&U^z<`L0g%N4vyWEy&HPCam5A4!ztu9eHh;NZU{&qQOmObhXdkB@f>L~Y6l$83g( zq9dcnk$Jc_61{{a)}bv=Z85wro5>(%a;d)v#Q5I15jBz{ z0X<_O>r%oj4L|P!zRbq>8aMf$G{BO zWQM}%Ed@sRmvg-ywW_h`ock0eN3Bk=p@=75zmr)B9KMnBF$0{S4fSejG6p6lC+!}b zx%&I^tFp(R{P^9k_h-!fS>t5-t)FzyI?vqRQ+q$a@niPAMu>c%&V{Rv^xqavaNgbO0%t7 zw5;y#{39qRY>>^L#ZsSjCbikP@0X{GxvIN&@0KM=sN(bl%1B#yNSr&jjuhh2)6kY4 zQ<u*!zNf8$?2-*TMir70ypl_~^RwzSBT zE?&6M@6+7lu_q*8vJs7Hk}vxMggCOIK30K~#j-qf=yLrK`Y=fc?CtLwHRZT-Wl5Ow zz?3CPQwD=&@7%xtGL8M>#fy-*r%$(&4(!?EZ*A^UJG3;-KPAQRDawo6E7eP67Uv$;|$KPs+zRpGE`_@MnJ$pt{_mMoG zYJMv=w$N_uPD1o*lKR;Dhs#MALz*mW^Zr$8QB%dqBDH;%`4=mbAA90wqo0NM`G~QF zfkTu=E8$0M;GZjbdjExYKmPMu|MJxjnmEFN^LC0_`@w`Ysii6vd+U}S2O|#@V>|`J zSm=lZ^I+sMx<+N>X?0yd+yfJeFgYmiY}EN1g0dhckfb z>*md~ws*3#3zdC_TMhYkuGCtRxDGW>*5>IS^FrbjEm^xl9z4YDnwXe`#6?6bA|0@^ zOEU7+)zHl~tpDh^iA#TUv9v8j)}c9v9CA4(MuD_?<;qL44zC~V2svbIypm*VV-q~c zc9&wsrUu=_)Fmvj!J@{%j)}S>iz+DfR*SeAqk=vK(lv;DWxKAsmIup5x!0MPH&Pn= zGo_xr$({K5Wluo|*H_Z7qMAMNe=pV#s|`&54MzLtZ~a!(-tVPpI_0@uO0sm;WHmdI z4!F6c)eo^)x}>Uevn6CnNrv}AWGMqBav^azZHIIKZNsvwR;32o&n+lYOip6f3x%#h%=$o~P)ned?38Qh{MS{V4vl1v6;JePjQRHV z6)FybJvop5MbAls;V zI0g<5_ntq0u0FYD-8!<3&7Gm1#sr*yLOyh8rM&#!Ip?>1J$=rh!-uaOIC${NwQH-- z7gmnM^lK_A2G0WBSuBzv;14ZPJ(l$m1NrOj{`Q%V|GRg`UPLHvyM6n1yz`#0wUzVt zY2J*8D4vYF*4R^rQ-iVG<)YAK)H`lHt5IPYRmkcw6vD^OyiO_gkQ1jf*GgJBAP zae_;>6pV6H?A|^(8|F;c9QcEXtK}24;kW0nY595-j|uQ|+$PtGo&hQvq}6$48`ACS zlcmy~G8?Z12jARs=G&4kx78aqJmk^liYxEBdiCn=efx~;?d?+*SGQ=ZG-IIgP5j9{ z0yd3lbG`8G%TYdeqy4M(D>}Z|xN)ORQIO>5JAZ{i;u>lhNKm;g-k_oFNZT7qn6Csxt;AKMc?VkreyYJcSu!d(RC(_;oCT0U*tGW z%vDyH$SDbx<6<^4{oT8FK5dKOxP@=$(q0lR0rGugIBal-k(oO=^6PJRg(xX0$=-2&-0|k|ndKx{+GW}0%a=>? z{QOd4E1fu%Ud@78n+q)7DZcZQp3Rz=C>ZBR^o=d~n_ZEjbh5g9*^S*3na-2VUXD*C z1YMLqxWrW2=p%g-iCOKl6!LC} znS0&jg}D4TuaYVmo6pwGWESj?vL~5>s!v8IkUCj%cnLge{y0h?Ad(TF8Bze!-o%dxMns= zKKNPMA{?G!aB$GE=Llk3c3UTu0H4V>XSS`c1FW=}$ries-RW#_eD6$tJ zq4xQeHTwR})aGBmo*ogQ-y(w?uLiB8SduewY6c> z^v{34_~-M5kEbszY8#h6X+G|bV?YmW+qNx5nA<;Cz0Wgy_@?ZpB|$e9F4Q*RX$;Js zdY@Xa-z~lmpZd(r=*W>s?CZD-7cZ)xxZXc)e&mbx1*#{?7Lg`}TT5X*z3FTmiM0HV zt3h2t40ScB;enO^R9;n+omH$x#fDifX>xs)q)F#;lA@xbc|~9ki^}piGtc^ZOy6%e z5k69O#?lJv3$ymJ`jMY3Cs9TRgX7AdX~Fx2#9^&=hu}??R8mqRas*!aj(*uij+wqa z)|%ptfkS>pg`ITg3Bv$2PHHB_`ER$L~HzE+Z9B#ju@KMsqu#u+DWDIxpy(qC-}gjL&p!7J$B5r>wJ0h zOUX5gij@!c|6*ct6?Ul3JR)g>$63=yoAfQsB0T$Vv*bVc;M4U0Yfe{&cc>IqZy_V&&{Cfo&!wR)WD&1-j0?s{C(;}toXdreG&!CDh zW;j%1TVPB3_~aO+c5$5d_RSuV9Qk(YeaZ@-C|vy4hSVFXYJS7_;XdY{8b{Wa=Fph& zhoF&Byaisis=E4Xbwi9zd+^|ap5?X3NOvr#`3A?ne1ZAqX>Ax3t1^4h`bXUKhK9pe z!o$z!(GSM!89=ubF5q@{zHCUhlc5F@lc*MNIB^n<)CdEcjT_$gSXN8w>+AaZ4yNb*9;|9{X6l=9X6eQTdf?*VbU-IR-@t zcLxRrF0;ew5Zk!*DVcdJg!BAkd7eEDR*ScNDZeCzM>FCbV>938>m%#LFnMMoR_(}G zMWpJ!88c?cJ-sef@*lBUtZ@0%nl&$wkl^y0j9+!;KB{EDTfnYwL@d7J+JDf+<$H|Fj zdvqrQsf+qq-ircySWu{P{8fDgy?mcRPf3b}Dr&DL3uYis`CVWKI^MB&&R)pMQcKw> z(;~=D)$tFRx-ikdRh0sI;8;CN=*-df!Dg(t#4Xmx&Z}=~@+(v0P_vwTS-j1(OC@ZI zJKH0mp0Zf*I&dcG+kWtp19KUzwLz_mju4xuE@h=sj9R1?OnTY1!Ea`*qofzxm zrd99te~)a#(Vma zDpy&rdOT)JYhV9n7J>u2HgA}lb0k3Fa>4MsSw!4#ZlOK-ea4nG&pnhnJXyn-vK5uJln`T`hMbwv0 z1tak-v`L}-f&^E!C8OxRNkVRQvb8u^T(sB6Y zBbG%$WL@2JqeDGOu;=uQ3PWp_QIol$ViK#@tnsZ~q^zpSMPr&Jed1iel1&4@U#~8VVh1~261`77nWf_^f0_z? zIWIWF(il9E0A&J@WrP8y3BFAk~6-rtT;q1NskaF#AA5K*E(+r~>k2D#ZiiwGJ)FwOS^`Cr-=6v7u4qr1rv7W{Cg?|XNqEnl_ z20h0M&(D>1&GQooJpY=P_{A5Of(laJm$H4n;$+uO80vUfHiZkc9B-i7pS~zytmj*f zc0r~2!Goc)t1wYHDTaj*J;tJ9nZo1BrTLmYY1OhM8Qp+OvijGQNmVpjO|fI9G|xw( zh-c<>)LxM9SkPg~k6ovm=e&fy^OI{?ddQ=<>2(_gUM$_2qf3AY^SXEw!m)M!n_Ao3 zJ-Q;0UdK0@$%l7yxiZ@3DEfTf{5pLtr~O4_0wgonSaET2E0-_dVSU~CTyjqLYIugk z8#gv1x6^T-6bd4H$67=8bw4PT=jK^U==l!4PhES+1q9bgY?-hkTSrGj+!G3h6a*pn z>^Cj-$f3KSy?zOcNIzckBa>Y}_gss*iWwhSr5(S!)~u!P(XU_I+S?yuQiXe;duJCE znC}UfuSHO8l<%|4Y;^Cs9o~(@DeEH?EiYfadgtKYP0w6(=tZ$-rq5*{@Z@^*R3S|9 zhdJ~kRCB;J8{sxCnV$dU#Wdg(>WGZkJpg4=$VrCoc2dSUZg*I8=@itUI*A=7#>OXK z$J{xXaL%oK#-SZMc62!XKD@zau=9_fR_y3=$t$01F7cm@IxC-1+*2e?yIXJWd#`a! zBWf?I)Pc|P#wFn&ZSd3c_{S$p-{HWIG^%jav1g2^h*3JuTXpWh{{6okZhiFj)X5KT z9-FD3Si0le?(XiNwNDT`yGm|_%<6Doxqjo@&d&kVp@(my=uR4Zz9z-(>_=g3!90^84@KtUH{xOC@sMT26}u zD?SM>dY!-cX!XmNH_n|q7m7R!u@74Kg<60E%o?2FT8Q&EHafNB$yd+_%AbTuAO})$Y)X`i@jKv@&o_J& zGl`~0{rPfxYi?{g^P5NX$jC^Tv)MYT#A@8(LDY3NWyt3DGPWL3b8erEZhcl2ytP4! zo?sNP1sQADx+8Tx@4WH7Umss%m-H2@PQ6d9^$(6e&74A4n%g~cmv<#TJ8tM-8qhN_ zl-W?>g;6Krvd^A+P0j~ne_mEH{nUz$8_6N+VkR$utA_-QKG+8*C~H%nY!(t1D51JL zB*TdB1r%=+eCkT&^rIgc(R-MqOahB9Q5iSd3{jNBk$acc>k z6~6i6Cbg)&@Ph7LSzmwq{r!MVNcFZCCB?_<&ei;I+Nud@+eU|mG{^L}9=Yq-^Y7fb z190a1l1&&bXp4;;*5jFZlC>(~`mJD@owhI<0))RO3hijU9z?>o`ps`j^5J`M{==E; zdlOESMa;|Srw_hzt4&HNur;ZviWwdCk<|3Me`MOpAZy%S4utn^WSMgW3Ez|peCrJfzf);Pk4N>-Z|H^WxsPlZKg5^d44!3sDBdN~!|VBumciW@{CCw=ReED~nxouspy<5*OcdV=Z*rRjn)g^GY*)^30jDeFua<(PivZ#I9(vh| zQaX2`ylG-qR@Qxfc1}(X&$nZ66FbML89K18?_TGZg@pWgB`oY*z7T$k`*^x^O~B#}gXnEsdZXf5 zxCQl&&O|?**oaSDoK&kU=dVOW$5(p+a#1Z8;%p~>`J7!i)kBq15wjLk-G~_=ohUuN zTiNpVR3{3P+G%2@FGn=KeH(J=(hZY`C%S+e)c`KpT1Q>#`WOPEGeguqHj_z3>v#9d zo7)bj9y@DePwc0=Cb7@_#ROT+4<0Cyjf^5$_~D`lH;mLU>%R2ZZDfj3c7CS@Hnnun zlJeBntTv|vwDx?Q-HUSb2?-@p3S4*x-Hrd3eM+@MSh7wL2(&zW25yA&&&*qrN0Jcz;E zX1IrtJ@N)I+7vzdK?4Kl17*)(q&X<6_<-xb{Yi&1>IRt2mx%6#=4l;Vyejf}sRx{$ z$K*R3GgEr=czXkvY^qWkf_j&X0qXR|wJTTNA2*HhXs|Jpr%KFhYikR8wmU)Z>amf` zpX8FKHf@hb?tudbE{BD+tFqc$?j$G6zl*6J>&>kg)WjGpkB!R8i6|1Bu2**f^q2h% z;C$q~=G>n+NKWZ+)gP&+iaoy!PHnch-q;f+TRLqM=)8_xW}nzLKW5px+ge5e5F%mQT#X0JvgU8&=uIh61s2- zC&$NQ^xeAkw|cIwDG(SSZ4tU6&fv|XUxA{Zyd(?gWphYPiiJ)}J$}=J|tO6wWTaMBd7f5sl2MdNN9zt{1B(#-B3cle82^+EU6Hfiec zO?D4_WKLhf7*TtUty>L@{5A-k)d~s<_QH(S?Qh)FuUxrO+r|NNmC2(f>LBB_b#O>u zduqNt;mm>fuH%HQHM!N#DH0^EOFH&D+-~8y*|SQ#bDA9y{8t_8k%-?HlT7SXOu^1P zL?&B_b6v92T-tW)Usp@zY5OH^cd^e=O3@r?4UMnwdH=AY4v8v5%XbI`%vh8KURe%+ z%2u|5XKu64`_!T|`_~UnzY|+)d}F~H!*9~P4m#z_2Va|o41oZr%rI{v-kIRX5p3b- zUOHyB^?RiOqA$(bJafj`ZA)Qfd!rh%S>MW8n0sqhOd5JS1L6Ul${uM{npng%nLh(5 zvO@k~h)uHJ&}(bEmOSs#G4`b6sJE1geFV6=lD;RGJ`WMf!-gtvRaFK z5jsk%ow`x3B{0>G)_gsvwogZxW>tCl7w6pO(ScG0eM{YdMV0rStN{HZuH}b12}e`wSlc4(^9CSh+J^5p?tO;^Oa0yj8%VA+?HEgil&# zH_0tm<9v^{6

(Z~&o>NQXG$(vTdnyMaJbsu7bNM6iF`^y+(bsc_abb;oH1-?< zL>IqfUZD4S6_v1Y`KO;_np@@m+RG;%PD^XHFh%&KH}Eow+A)8y!Bj{Mhd)N$wfxw}ikAI>$(ZEpTcfH-&v5HS(~p`#8eAtY&W%lCvuEZqMp!yys? z3}Zuu=7v^^qpfY#wT*UqmM1EroAGQgQwX)BO5i5cisBLy8&AD?bU*MQf}ND#hb^U9 z7-Dc!@7_73=@$2Y`87FjV7J!b z%Wd(_$aGWkhpw&AzxLAu-#p||o72+MZHTeE42Dt<2hmWL1e_4R3fYqS340Rh!nJGH zzFOVxs=0dAs!9+et%7l8uLOJeTo{hL0pSw*uJ0g1+wU{x!(}iprNvyia(hF$+BPC2 z1+D?GigeT0Et+7B34_@UU~xDCP;MuOuA>r2kS@dnslH$*6t{MMD zc@TVvD00{TEGM@U8-Wn8ESYnjtp@0iz`dmCa;=o)#Kik++t)ibWjQY3-G6N#_G}&? zidrOh8Hxj5nm#C1((LNxtJM;!?;v`wS*0$&e+DsXOwi%bf#Pjv0Eh0@}{UYAqy_{>~guGbjWb-8K*w!^Eh94 zld!Iffq`Ua=Tv0;LRP~RHaRZ;{eNtk^IL!Y`!4wW^PgFq{!WxqN`Z@k1VVO1FmX`8 zMY1NKjy{M0Up(eOCPKu9;sk1`I~7K2 z`Re(%?yADx-|A5Sv^FiEY-qh7!ts!xW=zWH8K;v3UYu2tH5o%tLX`}~xmE@j`OYbt*MA8dH`dLxY z8acTlr=|>t#+u5?d3A~>ot)||=~^eQ$yiYMwc@Id*JTMILOchW4Dp&{&Y!1ifBe2k z{@<#x)4OayUQUeW?*%BvDo!QF&tpXW~YSOV?IaaSDm?!Bx>5G1LgmzfYB z?=h6C0W9|9;)6y;Vqqc!hPNgIZ4wWUin_PMe<45XJ`^RD*jDcc+%?NJbYcoKY|4-pC&Fu& zcfjdhywE?%3xM;PK4*#A+je$$w{0^;&>Yq$M;dKYpxr_F7=4Hq^HtO=n1^^3RaQ>! z2&<;9&Lj5}eTs^6^p4KIeexjjY<)f8+(1MlVA#w1Ehpw4Hea$}fz{!fWF%O~r@Glp+N;q?&}!ZN zMbd)Y<}SBxsJx?Z-MZE4k(=(=tX|D&Yi<4J^8Cq01E2+*9p;{QUh`#7E-ZGgRgr$>=4A_A@T~SJpJX@InBBN zAdhD@>if^fa-6dYfd3>ynlXzQQw$k~mSYMSMiD+hf(XXP$7ff0<`kB4)kY6o8w46p zCcGCT%fE>^v@m| zHL30Yt~uDg8wD2@Qs@Y8V}QWf!KQLI$rem-&WTF+=qgI6R(KjJns!| zK8cHyg^!73{T_QN5PM{gLjG}e-H^-S;R%x%K0jl3BtT^->_k||-r|dK_&-T;aZi;` zj1G3~*3g(=VzPK}NJdM=kWiJBJ`vX$t)wd-Fa9B=9yEbL ziX@lf{PXKv&lJpMU>mbGEtx|c{xZ*-x$`i>EtoXe`A=0;=zdBgkRdrrGjHA63gRnX z5zxP2^vIG`*a4#-3g_xu;-NxC*u=>|R^-N}2yC#_hjBh-ThlQsV2Y!}u$`{E56chi z-5UfgmO6oufKvUUWGRo9$vgJ&$x%2`MoP-uBW#_K*7^AZ&)39fze820?*9nY^x^VU znCpzgm1>`3&)a97{p#PG*!D6kgnnj6Yb(-lPwmXt_Mh0D2|b8a zWdw>!sRb&ap}JA$+32A{*gQ?r=T=shF{tl{0mQL+ZKu?JtBNNFFA?$lSPY)4517y^ zrEN-qn}mba+v*CgT$nF+Sd>%3A#^(I^sz64r5Z211t>3P_?oEYe?hLbjK);mxY2|)h*G|h<_cdVDmgaT)vbMoLfKF6 zPim~R!W*AuHWJPsRMag%`rSx!v)g=cXqd`}6Z{uKq1~(dM-PJlu+siL)N-QF*4%!` z9AFl5`PqAzb0yW|tV+YKQq76#HpLDKF2`ywVDDhzg4t)6eO8WcK54M>-^R9W-VTES zv0SO#ZoFe2q)!Pbsv>&@Pi843W@m4b70Rz&`@Q$kQ1BT$E?l?}uZR_#ZSz(fIEa{q zU*&_WqMADg-6uvmrr&Ee%k?l}f%;N9-jNSv>yHd4E06_WF0fj=YSj#%#$YLl4Jieq z3=vxpK`?PJOr36bgj4_`hYJ+8nni3dCDe-NrKzu!j(#2}W&&NbDJu?^p|4(e>iMdKPEbwjoBImq>eQgQ z;7p4(R&#vZuR0qtbjdxO*Lzje-QTnpV*Q-LH z!9tb1AjH)X#sr;_boC-+Ebx&#-@ArhyT*VwD9!psEb5QJSH@9X99*}q$N}UF)t_#h zf8PJ%#jS>Z%1y+q+tV%5nCX%W4aUCb%zeL>Puq6x`x!PmkdwhZtmPnH1g&OBKm?q&t0npdw931!tN6T@Kc$AnjR)xa!B-|rh3(YchTrm9-c z)dV(dI&ws7TSb^tggk(H{S>tvUi_NnzT_JxdYM z*Unn5tfJBlbaL5Z2?FGcp|!a974!!>b)vS>ec9lEa;&>HN#7DV&@uHPEd!_tz#d*@ z#U&){Op71^rBe18JXx9r%ZG|PX1%2)-+bNLwaS6HaF16YE%wYME8c==x%9VXMQ0#n zA(Cs6l@p2_n@R0yN@4`9iP#n=MDTwJkqlOU!-ibGs2D^AleqH9{ZEUYa{5n# z%o05f@CKa&sZfppq);Ntx_O&2MY-Ff4<()3A>|1blUu$VICY7WABffcuD@D(a{^WM zW}T)Wl+@&T^McgtA?B^-5P)H2Wo4`9vhvT1Cm+Rsmu&~%2cC?GH`3herY@YgglT<+ z?f^2l*&Wfm+)325v4C&KwB#EE?nwGxVxQ|<5=a4DH$PDl1FTI}zQyl=V@+Fos#+4Z zUr9*ukYwgkN2_C=J7&VgOzD^-SYLJi4aI_BA_&X?3C)(Lzaz_=(gJv$DlT^gZLw@bqneDJ z4lwcH!GmpX>bZXJA@nr@C$%|zs078?Z_{QGc=+-aD*%@!LYn|3Xak}-QfsmOmCND4 zr_EY8sOF@$IO-&i<~i^Vxfr1?2J2YVWR1#Ti^8>@NQt#(IW>bq3Sj0$BoF2(VE&IR z+))Oq?ZpWh<}7FG&VntX_o*;@`ep3?rT=#dD*Eo}KVCsb)-?pjGkb~JhloZ%sO}}p z!y^{XY|8h|RB>pC1AjIBUIiErk&8v%=(fKBrTE5XbI3DTEY=OWKQeOH3N34sM{NMo z=5(iD3Hm8WaZQK}38IYIhuvZm>Otd8^6b#?80{B`3%kn)i zNjxVt2obhwi@k5kK8TkJ%L*fCg>DzypYi{00d}qd?h@QQ_Guuz)iv0 z$(M4MO0Ig~3SyW`Q1<|~(V}BHnwgf9bC|F{5ccT=Q(R6)!b-5#q?uPI*YS>>FLXY% zB^^E)hS!CfYE!#50a4o67@+n$(_#VtfdR$RRR5oLSG{}YztQUXTvPdc9LxzoWDD1G%a&ie=wlj7=FFDT zR9op_vOxM2QU~N}qp#k#xrhgtFhJ3^%6dpNl}it=gloQHurAXfD~%@Y~xJz;q#l zPzn6A6x-jx1=nMxXJ%%$zvx+y2N#GBeC8FKCg?g7(daZSzX#ZCjWWUaLs<*|hI7Mg zkdwqKf*&^*jkQ4Pt0R7Su{`(z zQJp=JL%_z0rmel}{`0Kk^52~>Fh(0)a}st>xsv1K-=^(c@NT9X)fh=i+t^XqCqg?K z*{raRWs6?hXPl;c4}Um1r8N&Xb-5tr`#$w_`;Q`W8`&xsvdJnC-R{7?;G_xQ+N^uS z-ZCQcd%-mpU5|V*vl|Q+82x3PLOu$|_N&%g(%|L~{uM9*q?P zxj+1epl$BI3);TRg)4#mMX&-EDL!$_eQ~0Cq$5TFu0u5*?P?DmncRRPH_eA{TOC2n zig98WY_U<;w8>=7`9kG*^tvT-9=LYw)f(-L6$Mm`ZJGPE{D~hEQSv#wy8l>tg{rC% z;wK(Q9Z?>fJEa~47Z^C)A$RH4YLr#TE99QOT2hLpwRrqjr_6#-24?xCSqhOj6DYGH z!5QbqzpXUsUJNO{GY zHKl3anPeeLr=Z>rtN$6O*q);1ht~8q=cv62IJm9g#tSFV)asmo4TGJrbX!=9_(t2G zNXvM%&EEY8slw(jL4(UB{L_-_=J2i7kaa79Di&+17U}v8Nb;JrDUXWD*BM;m0(~7;<7wr$GZV@6<>AN$tGDlZkrdaVG2kaLv7HBs$DOWc2>{-b3f(m;)dRiq_fz zHEe7=Mf7LD2$G&Wc_Jmq2Egzjq7bFV8IQoPR{dmbV6co{G(Szz``raOJp#f@E*P~% zP5O;c%K++ilg&rb9{J$4JxqJyPd^#60;F}ckN0@)Ku3X<9^PKg6zuS^_oKjO2;tP) zj!Qt>*S^sPrdfLH0Fo0Bb6WBpO;H-2hs!NOlo1ci`=#h{o7;Z_a{tY}m3LF%dz<}u zFT!FrA<046_n`5tg-2hD!)w;%RR_0vM4K6)=>_&)9RbkW3_$;v($z?j5)+G+h{zx( zw7#Xf$w0(%FJJ!5=_uhOss&W+r~P9&Hq|SH01OphT!ED1ouxSqu@^<=a~H>ltDAOF z#hUGOV*ja*RTKaB0^)ZJ6+Ev!F>F~K0p-R@ZEfuvoEKL&?1ZoA8zNu`kY8H8f@HH? zJ;L5`8Pj^ym&?ko+_0fqIUWFnW2VMR%K;qH0E31_V=jb18HB+NrW{HtNFGHgEyCNv z5J4O=CukbiHA-vyd8G`0pPrUx%=#wjl3rX9%V^uSGa_o|HyLP=wuO)Wv|zz)w7Oko zp-?%T(RhTY+Lcy-Lr;9Rm%bB59kSN{qPr2eC7iu3Z3EaaEEOW>AVR`a#VdR9qGti; z0E9Io0z!oFzUq1hC`#HSRa7^or78cIY5s0|)a2JC{vD86qCHW{^~iw(A#8^Or~G#l z!XEUYg?6Q9VW`bG)e(cF(mi?(SF1p>KfZ8>ER!od`&G$5L~f^)|BKsB%E?iOTDVsE z1mXmY@JmSrfcAjblmopHBix2Rymb@+0_B1UQ4Itp6F+TM?(-85O&01CgdcQ@K23E% z)CaNHwS>>@#2gpewgcDWz^Xrpu!J7iYFSJ#60^!FU>xm^3Drb$17!nRmaI(m^a2R= z{8(x~AD)ybJs*y*l|P)e^3A4JZlCi8ptK;~)q2~TKzoW>jhu=s!!p^sTNOU%QF5RW@Fo)F)Fn*l6( zK2C^QZGo6rBS7YQds35&FMt1@xjoi4M;(6k%r$92!Ne zB%1Ts6~7zoFIy{(Qk;xR!A1jpB11=h3LcmD`kd{0@?Y4lCk%M|!Sf#wsj*LT+{#38 z<(zZ2sSkQv&@7Ns+kxB#`eo!U(mLkQyPm8s!5}Y!>sD4)MmDxh%hnoMu@7o#5ps*% zdr1v%EC(=3=@UikXzzHd;Iq9TV^h~&R>Q;?bAuXy1CT>MQj&+nekEr?SL#R(5=8R< zWA7~ks@}IPP}^;869Eys0YyMU5DNoA5$RNs7Lbw#V~c@+1%i}_fRfS;ZWU0G66utX z7U_mJCdR$zY@g@8`@Y`&;of5{*7~pCZ^oEojJbU0x$TaGu%}|{tXXTt%(;xGnAA+! zE&cXnNZbX!X8Hl$lnKWl2TzpP+s_znRUQ9;9xE6&%NuH@Z~}}=0Auzc>(UP()yE3v zqLkXYdiCmBwGSZW<_eEjaBl!78wIuU*m7NImiNG1(6_GBPkC`eaJ-*;8+M6-ulTgO z9khI*oTcjtkx@vojHzyK^fL2?>!_OLt;?6EWF>I8**W|>(3ZDc;`cyi2~HO#X|e&F zYY#a^k8s+kLcBmm3p&hT2OyTs^JTshB~*ViAZZVPUez)Fq+zpCjYq~ybfR%+G6VJ4TiiOu9& z9hNg)X!Gpyii!{blUnIFmx7}f$-p)+UXs+^^t2_(xz6Xvl-0LA{@PHXL*5XuMSJ2q*KLuWU%wT-HD*>jyxzWlF9o$FF>~y$3Y-t!vi2>`5Z#Y7YvT z%HZntgJGu`rr+g#v%G#@*LSZHj%~MoD=J$>gvuD(QOg2%_-a5{{zPVY-mF>qmw^rmY5oyuD+ z>1Zv3uW1wy1de&;q@@o4MtR+^MauQZM&JX<4rI} zBQpEXKY6{-0U*ZnVTV}CyHc4DIbUdI%qwg$AbMsuJ{B;Hk}MCP|7n4OL{nW%wC{C{ zcy@gIwk*DLLobP(_$}OYaWgx6A*hz(nwB^Zfm3cFwj9LUmAP;sHq#xJ0FcrI%h=VV z0d7P0;NMIS7xI9uKC0os#F^|*D=3@?n_8ef2^bB{mDN|jJb7Hm9}VkZyE`7$1W2oi zzC0AC%}PMOi>XubAm5!k zcQC=hG0)D{lM=avDK!NBk)B6IMI%SP^vd_65owO=aP#(73H;!i7=pSh>-orOh_fvr z0h;fuRaXVLopD0n8PL?c_0(&lS139ckjfq zX0F$Mb9=Sx^v$NH8SOzu7S|-4I1mf4G+PO*_d0U)XaPpZUy3zfdepkXsNyNkur2=> zUJOPPIsyfLhyIKKfo|`JJ?dZftQGTKeLZqV?0JV{Pk606I0>22a{Gs5EkN?GkPKIg z(aL|>l|*P2ss%@l?9i~VBIv92_=}@7kNe!OR?5luJCw3TI4C&LHTI)A`ybqhDB#Me zz%aa1-d!5BQ!2AH1unD^RrrzJI5D+%tiSeZ^|e`lwZ0hcWBoWETx-Wkx8E!M1jIbY#`N$55j%#mX$pX2q@xV)#x1xPbQe zmL^@dK2){J;5MHAH>_1_)28q8j$dQ13%0|4*-?AeK~PTsGRg+pV8V@y{C4|XXD(pX zWyu!Cp?y{(1BM{uxaagVU5!y> zNx(59#c3%svyS`)^vxQBN!o|8TB1YR{pqVuU(0PO;tYJM!`xwNfH*oemSNflAcw@M zgbT>4o)nn}9onYC`thG;K!CyBR9u6nRfd1kz?BE-nn1hox^)Kt0B>iP8+OIPp8BWY zqmAC?FC+IX4H-{;myRo{(ohRudvvFWI`Cr<@B(I<*;8IUR@Bqai@GwrY9&So&pn~= zoioTWY0xQo@5E#O_K+9IE%~kN@sB?MG+;)F=h;%272Wh&6tiv@cXw?8GDbi(Ymt(h zoGe&ycj~L{_c5t@pj-4=CQ`DgsV`Id_J-sP{Dw!f9DEz-)b-&?o)oQmN`GGU*gP} zHUEppejSZK)}mW@V5lDo2N|(4nCy$1-xEx_Lcrur@MuD%KA1FaIqTM~dy1i&ZGr~s z%9f%~kLLF{0XouSlAs7vdlfw*aTm;9eR)Ya*z_F*{2joR%HcGaiW^Q$(*Q2}(@$yY zP|Isl`+8P;Wm2(`K8k1^NO1k=yz77!8UygY2zn(J5CPIU`H!#WI-gKh4n@wC z3$ySNFx6R*+kwleHW&uZd#Nor&SuN@pIx=x(wB{~ZvZM(eXAw+%$R-A9enFSq z_~=~+SVBC`%ah2mK{0bc$g*eqD0_lZn1WX|8lxU)U-En6crj-Tx3#0P!+IGX%ls^> zJ*mNVscRuptw>k|TM|^WtHwP9fdib!`f;3++7c8yA(zXc+rKpH7%==WOlx%*2 z2mk7 zg3<)Yo~3YWRoL`Y4MuzPH2$RSZEkLcxZ+B2tGO~Z)MiB9LIZOQA$`tJR4d{;f?$@e z+ugg89cl+HsBZGeX>$gPiVsLjDl+z}XHL8&o8x{=vp*mM4ep048%^IOpynmW8U|7IrC>lk% zrocTopx`K_DRJS9!?(4-9Y?oYN?c z><=(<2_$Vz&aRH0@{VR5!pl+i`TF|OeqJ$$DfnQ) zq-KGNmba2PiR5FW2`c}nWhDw!y~Grdl2Fi!>ox?Mk9ocarj57Tj2p%9(oFD2S*Y+(lWT^r%zK8Aba4z zUkcG$tI@6iZA3E0U7InQ2w9kdZ7vcW?{%JV(Dopy4d6oA-lzRm0?KBZ+y$W$vP1;d z?JWi1EAF;^FT!vk?xils|Bf}8L#p9Qp0{sH1S?eph%a|8$M+D3rlx`u$N4|4T5Y%g`2fO%nvSrQdR=$(qNdVI1W{XD8z_vl?^-hytNcS6SMe< zQwXH8#`AoiHVmWcCZ@T7F|-v{(zjcJV@?5In7s8wZ}G#%><$4Qo|wIYEV$P@EIv;+G z!$46B*FS&@p>_7G&TOBu|8qwHaggMQvimzE(%cEQ3y~%U@?)()3XZfi&NxMJy>a}aFuoWpLdpezuPZjJ^f#ef|3?ry# zFcsXl3ai_UI&uB*c;y??p*OHFRiP=r5Q7Ypy)Sy_X9z(mKE-I*CZ{^jjF~Vl@t*{8;A@`t^Y}Y&Z~yu!)OZ4814`d4vcWEyMV{*$|g^2@_Luv92bg z{hBpv#5A!`M0X;j;7jk58|i8{q;i95@m0CFu?eX|EnBlRALBZrH3JTryyfyO;JZT| zjY$0yS|k1kuC4*BLYCYuR^4wm`pAuaxg3hyjzU9siEenD|Au+b2J*0d(N*+_WSoBG zlvoCsyFG)77!S71M1Va_6T;mq1Kz02$cllz8b&G?dRi@>`1?0QtCP2@Bl%?Dq9`i8 z9&SH<*@=aJ-vS(4Q;x89=4EI;Vcg+d?%?3S-XDQhd9-fj_}BPR5@iyG5!}56rG<0-m$qLXTc=5qTNd%7xqjBA#!JZzItA7pj z=O*I21@KVdjxXF@zdG3>YWHv(SAjd^fX8v7=9O@L(lZphw?Q0*B5Fb+_?l>dDrmdy za&#>eXIlAd!e`q;UQ>UsdtOn>X6Ro4wU^WsN8tuRMwomeBn6>YB^&mP+-}BUNXio_ z&w4QDs|Nd5^&8U9amvdgrFO#468B>6P=X4|B&yE5jvPKr(Raj_16B!nm~w_gFV{+Z zQtio|4IxSik~GLR?P)sk5~nxR$PlW=&^{(`+#gN`zyT7wkUnbK0x*~BBc#eOyNB(G zN}6n{DP{l#k9CfK@AjZr#Q0}MT=kvZ&b{;Zoz~YR$lpEmkBf>*Lbh_;nQ7rdU*zaw z6OGbHb=Gk=TQn)^@-XRD9e(0X#=5h?;DlMQp0qfaK+3lV=!y_oG)(r}e zp%A{$!Zx6TxMp?~hx=4JtO{j;GLojao>8!(LAAo%oj68oCdm457q$RSJX4PDOrQJs>MVz|Ktq0C~Z&a()Rv zC|fOn!3)r+^FS1?V1rl_L!xk)9PeLscC>%XGMKOzTRP4`{siESY6F>D`$P@&IMF!A z&gFsP9s%GLD^!<5U?`X4|K!Q-e`dA6_x^{h7WC{{K8A(jn&1R3MQzrtrg`=(5G$(+ z6-4@NX(L^cgUo8Mz8aJeZ$oWyTf3dqOYPTDGN*7p>0Y#5xoeCPX(9diweFC%~R4w-d=&Cjlh!#5(6qp+MHW$PI; zAi~JY%e&|e|Id?PO-+p7K#1keUN)``nko&O>x~lru$LVi>JaD1L7jNuz=6;{IBcF# zsDE(pUR~$lRtV7GZdlV91=TT-g{Qqk=bJyQFX;N)pue5++CPJ@@9%@JD$K=n0d53v zp*fK`e7$2_PysDENj#C(V62GXJl;{=*wEl^oAc>BhQn1I#oN}cdmVWnWBsp@BA0OL z9>V%S{nIDh$CY_g{z7Al^qXU{0U|b>o9?X6;Gm#rjcn!uCXB;j%J69H0+b)UWxC<= zIS4j}bs2;8yi0Cb)fpvlV~P0EB}?3L&^nP-3edmsad}Lr8u)!MC^!+4W`EjTdMDm= zW5hY`Jj~Kg@!x}N`QBwFgQ6*_u5#1w@iu4xSlPSn3oZxi9P6}kB~+!mv>F+4CBYg{ z1Usqd^^L_TV-=8i&BPmYIA!hlK~$TOoxZRl92qYmJL@#2V`$U^J{j4Y{# z*Mo$)d3<#^_U)ZSNiVL6>|s>;!4V-(7u`pI50R4#fgoSpy%^|NBwHgG#Mdy~MPSC; zr@(KrK|A5w7b?-zlIdY`4#a*k3a05++7@eg(-Ehw=Mt?f~ za01pq{@5#AjFyDB1f=stVcKFb3_V5zj`=m&PCDjcx*SIjAKpHyr_6D(zOk_thVx#T zxxG-A?l@|7*T=^|ehNPXWq=0$vTkhrH_-79)|xXGG*}wX9o8Y|8;oeqF zh4@++e*&l)QOs9CSxEk0iPZ$kuU>s=@SeVm|3ZXEXhDbK>|}w#K!&TJG7O187UZx! z?wdAkl9<E;pB`T0Sem7-DsTyY+aXaMzwa_AAHt#Ub#3kr>VeYJaQ>8zd<2v@)47eY?lH$X`){r6i?5ZF5QuJRPNQz6qa%6DH& zcfIn6Sp}+vW?DGOmlMx_y4;|IVbm4dFh~=m5rCSB(8zq?ZC+e)8xeJoTpFY;av@J) zD=rC6FH9U8=33^_kd6T=Zq9gG3|&=OJe$m1A;>}qwD8KA&@FKRfB7mufBjEf{ri~X z(dx&)k0aISl5sF|O#)yNWy-Ot^B{U-ZXBcncd&dhGipn++ilesf?iErxBw6JqVl2O z)|04kaenNE&8T|nh3L$~Rv%+gw2iX|j5{IFa&xUeEQy+>+Nnd_P#mnQsWQ_BeTb!t zWvQ|%*}Dg7t^jS)jqcY#6eCNu=3A#3(HE;yJL_k5jCo2I1 zSJddMni>@WQJ~Pxs@A}?8R%|LTPatKde9h!)GT)(xHE*fXny3Xn}<31mDt^?$K>SP zsNxs*5-2p53}~3CoM@5{oygL! z`)2}3!4{T_QF(u&DymHS}TRaQ222V@RVpoStxT;eE?s(N8TOi64`u4(P9tRyZ39b zZ=bs#Oeopc!zCX5=b*Nd$n69M?V4_g-%ac*(_LQQ>xC2`Te?bEcswZgnr!R$G#O-k zLk9Nf__9&lZQeW>XY9^v;GM}+ynQag^xVRVm38nQbIXB8>NB9>zd)kS{~n2YVQ|b# zz#qjgfOH(D4t3KBm~#I(&=^>uJRpyGFFwPU(aA&yK=zrOz&KHa%@`>oj5Z>ZHHxkE-k9{J zE{K!{dIYE17?5ulL{aj#xhuy9GIhWjd+}vI1QUgrtxTX^xahn$P2*7wMZ$)|cn$;G z*Ps_~e7!v9A=^7Fu^T8au*W?B0?#=xQ#{+?q=+pat~mp+|avX`>BiC zoM&u2`3!55hTD#y=#drKn2PCL5Lv1s!K!2nBe>AEFsif#ywIh%s-Jix)L-ET3hB%q zmB0S_OO8S0XVv`*uy0FgRA7S=A}g6gMR$jwfCXfsEuaRHGI0Hiq0=VSKP1SiAOr^TeqaTH^-XECwL=s>U7WUr!Yn z-9~FdWE1>g*hTSF%I10t0dhlJ0ngO;&6~r{Gq`qv{o7PZCAR) zklGbp^Zf$@heO@|_NaYGpfCW0$w?bfsegnjJOFrZaqu7v)thiZLS!!;f8LQzE|xGK zamztOf(g~8NB*7@Hg{ply9$`#H&76GoX$8y! zuHnU(5ofk?bNAddUm2%ekYS{O_l5Ri*1JK7lJ-Fe!#Tn6fToGL#}q3uV?DK>rtiOG zgeNbPz{kOn(={NID+6RQvmIbG9V^s+z39}w)Urur!-U``W7acbvzObth)b7&Lx$59nd)*cG1xyP&uP< z_YlS}`ltcrMeQgy(WBx0e2*)xw{Ne)P1HnS@&lP{#z+Og{9d9B8R?_v-kmf)X4F|L zlEoBNTfMciFnJ=b^nXzfS7KST=rM2~s|s(dAOKzp=)|rI0)3E})ZZ>BmpWTqjT{_GTm`=b9*GX7|9aWrMzBH=%Re-D(_6$ z9EON6>rISYjE)P7QTp8S}i<(yvc1k0k z_JG_asiC29B^B0l<-sotacWg#O26u;(A*#6h~BX(4H08ZK=KT1yO&e5j2Vu3?G4KZ zQyx5cpjg$2FyPam-L+z3a6=RSZ;zEZa{F)187A3n+PoPwXYXcG0`JM)EF?rkXM~QF zwmyQ(;hu_79BV%SF?U@m&M7m6$QN0;Sjg*dKry&$i^R zmm4aaZF|<5c2o&z>?7|G}+k=!6%h=7ro;!?PIsSvktKqCHq5_4$v3&VQ{|JY6I^41k=SBx=Nbw+3re} zHWWu6`VmyAG!~NL<9-#Ac|2`H5(Z|gVmE3EKf@dY^|_bgd4a=_eN=XE7U3 zv@D8w@nS!Kq6K$7@I)~Nz8F5!v@r1aO1L=51MrD#Xv4@1oO=_J;KQY$dpH4;WGy`P z04@|>70qOBF>LS^uw-6Oa{+tS6S#MV zqA8nUN0#c^38|p-Dl&|#+@Oal{2xjaN9LH(7u9zd0reAKK(=Z^vJPA?)Zj>s^45;x3-3n4YA1$Ex& zCWB?ei%1QQn!70Rz-O8$4s_OyDm-3z-RjwwE(jO?8x3o zE`K^;ui%c^@{^}Hh99l4_>569!)1r<%v*s6_x0hQjeJ~yn>AvoN2M}Jt6hx|pIp<^ z#-D-sh6Mr)e%jv&4Ih|YQyvDs{V|33 zF>cm+^LvI;jWHs)s@-VJ#1IX=%T!x^uo;QUP@p;Ql>{Q~6(e!Df{VwY^eI{x;Z`5#*<6tjRqd1pRdAL&9^Jr`_|Fib69JtH<0cG6i^&lNPoP+5B>+5x zi2|@}M^x6-b^JH*OU||taKlXhTiwH$a?wAnn0!KAgQ{iYQ!3Z8?QBRy{*2`S5W5Ewsi60ldZWVrh{dybc(Yo=23_OGq`_I zBHjKEeOMz}&1kZrai#$VaASX_(BRUY7}1fd)W>Xn)YT6=HVt6s=;Rr5*OIU!y<7%~V;~ zuK@&t(9VVm(Cul@)J&UsSoYci;H_BG1Ov|#cWaG>*56M zW+!(=8Tx@Ca-SladK6Q)SafNyKc6TX5^JlnG5^9C*#&r&jyUi@PnyP|?kXHE0pqC< z=ZVL9;5Ksrgf)P1&kfmBGlj1<}3ujZMJOidiJ?J4P zmV99UAabtB((EX&7EwOx-4e-w!>RyhAldq1e(VV(yIvE-Pb;2rcz-uh6ow$o1i(Mk zZEzYhbWJNjYYsyhvSq*nxHpE%LH_gQ4i5|uy={lIF|-}-873O)rJ1a^d%}&PCrov* znFbt1lr#&lRQxVjSmp;V{UiodgPx)jrexFS_?u^CrY2 zP4DI(2M7%;B3LH(^KrYOlSDYG9{?j(mcULij4Nzh4S&wee}vSX%=5W_Z~?TM{!@4b z7oUgyD#~LWyj)(+Hb5mfvEDNGr4-~oA3pEwm8oi&RY#9>7NW9&J#7IeN8$ z`jNlmd4Xi-!3Kh@k(|%)uwwrXQL9=KC+D)7M+(r*(5sbWk8qh4!mn(V3CY7;Yu#w@ zt+U{bld&>RZy|`1#8*Skz)$m~UXh)@Q3D$arwjVquyBLB;x^{?)l zK!npWQ+Qx+sBj#5NgEI{T#X@-KP)lFO@4g&F%C#pT1q1Pin}o_lL4C~H>?s3Jv&!A zM&=;OGC$`T)%kf440#&Ak?$_rJ3Zd>>&yS`cXyj%F-bu?8X~gw$jb@G34|2-TDF9>@>p@8dfizG1p5zV8`%2c52lYKo-2z!f+Nv2XnW`uqpvC|0kGyU zw@ZO@NmgSaYF}(+q_*Dgg|ySqF-&Vde@I zmqk1oR#DKZPTbdF%a&(>fHI38eL0S~z*$CnZS8}|2Nn8ep+0Zz*T89w21Jkx1X1NT zFZLYnL*+*Qbzn6j4X4yr7vhmjtfC=v$YTvc#C5cVUaJ+rkR*m7;cl(gAk#PH+Km}| ziPix6>Lnxj9Ati=)}}h=tG#-whm}0t-J1)P+Vq(EF?l8Pw}G~JV6+NR8jcMHm9iE? zIzQPfKZ2Z=;J)9lg|vpOLBX{Yc>Ca!zkGMs*46PqfU1yP-0)||@J$Tw02n2ElSw-i zEK`lV;6x-PDOuZJ2P`@2*wu3sKX3?{-y598h+qih?*L*o;DdGP-Voc~k&WQe1J7DT z*T1nahl)P^^Sbii5N~l!T<%p5vI?ITx1oh3sF?Ovf#Y4eDmPzNL;y88D$V)V>VnKw z3C*>`GwVgjo!Ki0I!zCd9%T?g29`>faT$LW1Hu@dCSr$HqWl7`QUxF7U5z}UO93S^D{zn@W*b&@p4BJIl+TW=&~f?aV; zWV=!ONMjD3ms6|}RTw0vj@@OVgsLP^tK~X*aG!G8@oytp#*dBcFkuf*8%w0EfQWcf z6vI$uhc{Vu!ybXArmj9-oN5R6go)^(LEOA$%R3Vd7y_V#(CKN7g^L2BT#QMxWD4*^ zGLIeNhwB`L6H3Ma`C9MUSVl1bosZEj_#9-R2^vJJi>wN2J)0$8hrgWXBbV=HvNFYd za76N)hnt&Ru;eM#!^e*oBZ&~RUk1bRu!7wjfchbERm`CY+`LGwo4vyE3GsmRUDunE zG6AQ;+Oh(rpuDU2#3ryd@zkU)5?uV$ioGb@^n0z?iw7Y$-=Ym${(hx<2MQOQvH+wy z`5Y9k#oPR+!~j5PETr|I=mF4dj@Tai#*VNRc0x7<)25}T6o5fiEwgIYn>Ts1;S`)v zMQ~M-!&6?qoHD#|(iIB>&_euZZIP?b3eGcs5n?kV!4-Q+lu-Hv6o19&6E=-|*%$V% z2lxdc)@aR4;XI_ho{Z%NT4)L;?;1lR@xv1f!8{lgnNJ0yR#6ORH8cI&US4rJ{un(inR8fQ4%h zZW0hkR2eRVmjqF~EmnZ=ku#%TFk-BjgPaU}8f!7qK=Bh4A77%zK8<66W&Hccep z#wbCXn9o1RvOlRo93yG}-EaIKS4Rs!)iHI_r` z75>@VvSo7&I>%LwpqQ)4bJWa1sx-CLPaXWc+{g3EX-2RKC6Gs8*W(9bGb$Dt)Qgzh z#vtCP$CX2O99U9P@@U!78Qnio#bj^c!MDfHG9J95lm(+GSkVMv`?8fMp3(^bd`K)-geJ-sDhAm4A$lo|fu$t>AzD%+V*qUCvAF}fR zbb?5uMX=*|4_+f=vg)fS`O(<}oUY=9Zk1F~GoY!a#6<9KM z{grTJ({mQlsiL1@JMe0y4)tJ1k9NQ92WT*1yE+G));wIQP<+A}{VWsTmRa+M^u~me z?Q`*MSyqfWsE$0&X~^=%ssQv0iaB6v^E}cWa&h|y2oqm|IlV{LKfb1a{f*=JOVm+d zSPZzV31A7dhk58KX=4%nCU&5LQ6H)hBMC)4qJtl1(oa!JpuBNxyD*$K+K+f_e0q62 zs>qHnrU<71z)c~UvrFoL8bu0*^Z4cpefQ6#@ABIO5G)0{bd)G-X5T@C7^pNO{v3WR zcw$d5fyIv{>=`Kw7&F|)Wf7rK`vl@bV;AEIybMHtp%1yWYYVysY)9hBx=q228@4tG z?atK;CVd|@|2(|se~%a9q3oH8263ep-Z^XBa(iEzRWWZrHEG!#?9%!H2FEY>Js6Ca znO2NFXFW6*$`QjD86;XIHoW+EEr8av-**pqfs3nXIa3Vk6X)_3^Dz!;T7A{t-Y5cp z^n;|Z@b^qp{LM+1-(dKTbtE2W4l>;#hC2p5PR$vxfPXZzYyU$*g%_EYk8Y5Nj?U!} z51U}k)_LFm{x*Kkxte$zT4=U#qT<_jLdgfQ_NEOR?B`$^&<{PG;~&2tVBkd-tRqAK zwBKgKVK@l~f&3Zxq2Msxv_yd+|F6cu=e-oSIhK!iVfb!sy~oE zX{O;^?t>l+T)%&L8w-EW9GQ3;r+}y_M4U-ikE3{FR%Nkq8GzEC?_u+Qeh)7&62dB} zG}K~aX?pgjo7^?MVfV7+Up~wKgOHwrvFF( z)hu8AdlSKbc@^txkN5n`Z~X2T*4H2JshN%@E)oSiEQlG8L(ID2Qq7Lhwv_6AA$g9}-@G6Bb-yJDMIQdGPsiCi zkAC;b->ljDyK&xak;}j1fvs|w_B)y69)?-JldfK5Tl8C*(*J$%U!RKq`^JC$T>kH< z|N6Q7-yQ$;bNT-p+<5YSwHY_)F{(*a>MOy_a)e8q8?S*1>ds7nXLx z<->PhlYjYeTez^a(j10YXCMRPgoO`#8SkCLuQ8WhiQUMeEVcum?qDj3qDtaWs8Yg? z`McVlM^`UL^_UzFxc9M2iY4>J>weU>or7xS-~W4|fx>(XNbIqEH5aGWf;CrBd;7)p z>t}Z`K?SJgY4mP$C%7mAG!RAsDRGEhIguNDT-VsoSOTzQ(Zt5uwk6D3s0;t`U$vBy z5(_3H&am(k(DmVmPqtRx3*jp>O^;yGAP7h9PN4f>nHjQ}rJ@CT2vPwD?U{mIHneFB*9Uc)#57@K}y1rRrE|Pe0f$+H9s>$`(MIKNQtBG^CrQ zc6$lLF>ne`*rs@_{Z+B>)G!=ddLX#n;Ik6UWr0@4KbpvRthO_JkqmE4nfav`wlLI?SzLgQPP8WTwV9H2j>^nmgA) z1Ixq*V}aVi3DA}45NA{uJtwaA_@%(OpJYnszVtRT$^u4O|BgN(46r zmYJ*39Am~UEG&$l@Bg>Nq4%ErTx#2oAE4V4q8P_!$#r9CYq_XJhR5NA;Y&^5QW7uHpPo}J-?u^D~uF)3y)M>gE!gcl)QmIEW>=c8!V z)O-vWwaTKOEk6+YFzv(DX$KEj-C+FaAp> z5}z6$U}G)N-SHjgViLiM+6 zbJ~s!Fk%-fbff~9+ch3Mu@yfju+sbRy2oHD$LO=4v7HE!v97-=tsfmEU_JsCH9Eq_ zFjDoy(o$v>5%^z-CF6V7)?IYt0NK?V;*QBMFNl8&58Q z>SC+{I#z;-0#E2{M4?)8sZ+X8s?M`0U|1X?9>X&2z;v2=@O8dkT)E)FY#&T9Z!PEL z!2Y#}SJS518dsWPHZXhE6PavvjCaKp?;(zW;S{6s^8>h4cm20ToIm0uj}Ejum~=WH zUbjeoybT_W9N1OZXO3O6wQ!Ay(2&hXWU9p!AX|20>#%W zD-Z15yO&$%<0gvc*!{8X48)3@_#AG?1Ps}s`1&{z^~~}8(7o1QV44F2x%SQ>)*GOT zocEYAW$}#z)`-2%IL^X=Y!|{O`A{2Z7ZG6eOuQgUh=}u_UYcXXeJ0};lcMP@;~f5t zeI;st@;WN0!Tt^D=wa;V>ua4#K6fnU`58w`Ysa7e_@e*Y|BrP5j(ZRT#JZX*e>ws1 zc#YlPEKpu^8rHs!Zw~PI@MRc(l7Xf4by_9QeGlsUAyJy+q;t)XRzfz2$lX1x#n?89 zWFvgogy;uhN|w+1MK{c0WTjI-aBal}Ib_Dbc~cWn+4!M2(wCWmN}i!1iwsreze=i^19O>Z1n zGoPb&^C#p9HAg_m(7v${D5on7|qAd3RQ$~A(Uou zrJO^k#%|$Pyj%C~F|B;$_?G?d4=%voE46TElFvvyux25|1j-cEzraP6@3WiQV~jYM zu3W}EoE2_`H>rlgZCY{Tz+=%yQy^&vKsZ=_zzbl|&G?$jb7J|{!HUPJlfG5BX0D_V z=xOrdNIo#pEfl%y>+4@bpu8A_4Ng>IjBqeygPUmpyy_mPSZe!s)NixxbjWs!c)r#{ z$m&8y0$YJu$_n+LkIVlO!#^L_q+K{RH_gGlD=4b}{CPslwDl-tRZD@F9aS_glGjXE zv1<31e?D~p>SLz549DT3T&td@g^1wKBD(oL{k8ZECRJ2RxM2w|R)X!M3-=o1B|l8j z_DuyW*j&+c%KO$Cbw zERbf{`&_X)5*ml4&U~0ct6`iVU9}Ne6w2vwowCsDJb?%HWcPbV!N! zuFbgr0DZ7JY`MG+KB891PEmhJLohBgNX4!#jxNgm{p@H_krA*`An-kHGE9#5}0(_Wt>A%RJ+t4~@q#zYY_hY`Jpi7U`}&#_#Gc zh$-r7$Wp_+pc5oCSqq5P{WK}naL`&pBYLW%arf?u>urw&C}4|d5Vn;Vurd^LYph|JZaZErC0 z7=MkD;$AFCT+k_5WnW(L0>q=FQtCHw*K{PVy;&Xe#maM#Y2uT}{0bMlxFQ2x-dcCDfKUtc; z{q8d22?>|V9%5GW8oAzA{<#;FE$OJZqDu8p*PoD(cyuI24tKWp@*Ixq@T1ucM)Cqg zCkt+zas?-1PE$dWk%%q}Az6QZj3oXY6%m~{X0p$Sdd7*x5R6#acwt_1IShm+(#k5+ zE-NLBJKz6!{>{Hko~I6>=(~RbMf?JL7ZDtT9LA>nVA*Yd-+J!lFZs+I#wBOKrtzq( z!ZAmya`j-L5!w=j>f^7`3Cgr*3V>F&7Pkb$rGg?dZ*uf4!gB2Eh%3T~LCXF^NX?w& z-GlQ}ELb9nU2Uh!+VLUsWAzlfqrN@V3j)2-zlazm3vupN)3+PVg9$H#z03M%Qd_D( zH;Lv!v=uOEdW1)sjhwHa*c>1Vx0m7S@H=0bgH**UGbxXtq}WkJeJIMP3Pv3lM6b;p zkhpIt#3oANTVy-DZ_4TAX_=rjuDDF!qop3#@^fTc(#T%qq0J1)nbsQ6Ux{0=*o_w z_4kfsKbDcl4Nx-47m>ESfL^+Z%&~P$&6(OZRo78gMR_vR>wuB@$*6B#!}EfLzrs)3 z7Eo&n!x%9|7;SYtmCY6QbDak=l_Tav0ipaYPyMD@zPQjwWe>etb03lk_*EY_#U2K| z3Q<`jq#QgyjT5@mDCj3NxVrJ^%4C$m6+Rvx!w+$cd1kp5?gdr3(LBi0T|zk{bLqR= zwLXmr!P3mzhovW6dHwl+uLSOG==ur6a-?TN0M^ zkSTNBFGsbKZRu+_Hlp zO8B9i$j9|5K{HIq03LoCwfU$p6nZ&8*523&%NW`fn+&Iz6Tm^}ir`G~L3l#{0JF$t zAg?$Bpb^3A^(M|w4S#!f2XbW$HDrRAFlp;zMbL{9oJg}4{(8g_C+H%P^^d@RDIJr2 zHS|cG=#{|_+DnBIs62jPzUAXBY88dJ-{4&X4aD^??bfqhE+_!x5>2$(l;XG3W2dJCE!t~5y))oD&{Nvhp@*>bnBE*j30rLXKq&m6 zpPQSDak^sCEO~Dq%9|I<_Wr7qd|n9+?`{>_!G>ran$02vXpWXp)9eWqj*Gs}4s}tg zt&a~gA&%gp`MGgxOVako6EZTfrHOFCGML^2p*$~jo0+Uu=h~2M3ybPVNlo7pt*NV{ z8V!Flx))yp`Oh}wcHpcegBU75CsZ!Y)+QO8q)y;tufa(4?z3)u@4gt)veB|!lpiBa zhlFoZKZ`8CoJ#wc-aa~g`{(nYjrw$eP8ufon${PfbYZj`F0VdQ|K^U1J%pi4ZQ6(6 zPkz_sc|h66*2J+?4!P&6$gn)F!ssY02LnZiM|#8U9I2mCv<~smPuxt)47OZ*;cu10 zn^1dmE4>uYA~snS6;^OGju9|J7fQ-fg|Gn76l~iqQog>NQ%8j+khBQVj!bZ^d6$>S z$;mOoB2Ixe5ENx6;o*V4?G?en%Nv(7eYz`h8d!XFXsI3HjuL2d0aaaZs1Mc-9}{SP z!q@ivRji_frk;;gSM*W6Q!GPiRW}#KCy8Bu{AxNCn}nBz#OjD*L7Lro32?Y5%L=MlLCpV9bar;C z;`38`D^S97f{PaIyVS*fw~eJ3wj&d*7AB3xRB-72%|Jl`JIGhTci*3&)gAUz^&QaT zYp&J;TK%#0(zH6_~?oeMA;+E#3`CSF~dk zsg?ml+#r)`)R*RPsHo#j8x;4yWa>eUJz&PlXO_yc)x~Hj(s2?d04*e7yWkvMT`GL@ z69RDrn(<6xW_D9#r=k_b+s(}<1I`t8TY|7kNLFoZxxRzXNd3lq_*{#&@-A~qc)Rcr z%bmW_synC#+=T6HGZK;53f*HeBGt2M2{%LYb=o1L+(b=;)m22~DKj;PvJsATTPpaPoz3`0Pwce4_OP z!!oXA*Pb2jw~#k8$p{6Dn8iI&)OyUjGKSWAhrLN(N40Rf@rm-HHQ(yYQ%8SYXIepQ zoy~ZPppB53&NeV0=mb6_n~9hk{TyVtdCi z3d7pNV+cx8L$-auwiI3-(tU}{QU{XKJM-g^v@5q3Ga~HLb>g<{|FQyTd^ONJ19X)t z6g~}K&f54IZki|5ilykgZAq^NIbchuF-J$`8yQY#2p?ExAmu2?<^KTw(Jxozp?Du*gQx^NiLj&q`kO5bLCLhq8jhovH7=Q3rJ z6A-w1**O{DHfz6XMTwAHx{6k+r32#nhRIX_`_n7r`S@9-IHunei%AIeY2Wb~K1XYS z>fDKG6kM~0mzNh1g*1C(5@O##8n#}>Bm72}K6@X>RimJ+6^E@X1<9u6!$;A9vcC?*?dYd2q*HA`&WBB~8S%i=Isn3zC;-x0o<5;uLn5-7J} z29u`L(Ze+xuxC)U0(iFS@;d09?(AgbRP?=2Zu|pXh`stNX!=wVRobxK=hQ=etG8y` zwr#DMW8f?o=y#x|A$(DFY!vav5e81zs1}gXJ&r-b87$80L|8J&w{^K1E)@q3BHF-o zhK_!Gg7l@bN(=FthGiG;oD;=N0ElqN)jIgTuR{$e-`s~AY{MRcc5oAc6R0JtWL9#a zSwvxbAyK8+zn=%FFy@a@l^K;NfYy=vgf0d(;w6uBLvDg^%pDeAdov!(0E0#oXg?v9 z2dZN>xOtQ$M4df+d{%-48SK*i1toed=1}kDcu&2!NSk&;lTxL&ET!QO3nXVetll0- zx0oZ|qV5zeVNg3?jQW>n`dBu9qyp)lsTGB>8tAXPMjs||I6qT`@;Repap6+IF=>4H z&eT5ES)8>RGsl;ph95w!w~n0?eYp$vSg?@*wU&MBm3;ug! zG-=YQB=EB)Mt^$Q0DuQ$COtn_g5d|`E@(#4=J=uHU$~GH^#mVkHC;B7CK!_(0@S(O zu>Q^=nP4fV*Gs6|s!$eqU+W2Tkr|@{UaJh9m0)qCd3xLPcMmngi0SMl$XKN2vj7X8mT&BJg zxd^Z<8{7kv9Fy>K-?bRCE~GvpeEL1T;E6gJeC(YFHBsrql%1VWW>dREZvarjV)~J| zfsiScYV>1}L_z-msA_&FV`v&0eOidHcxt!aS(qU*5WF7EL(LyJlf{X;NBt!U>lq-| zX7>5}T2Xsrw6~qQB-BVZ(N+!MAw0%jbyC7^xz@J+-dj39G{K?4u~e36k$pno229D< z;()qD&h=%0s+znx2HJ#z4je&IcY@6oJMrSl<2Ej#wz#T{*Ort1Z zH0!WOcLCLV=(k?O{bB29vUq$tXJe-8!Q70%!eADq z2biixW^p)RW0(je7iS(QUl^3hhP5^8O6#wCafmh$jNZwGmh}@K1aWKc96r};ig_*F zkb)w5LTlF0M(j*rb!sS>TvY0imFn&iE9Mb!M%nv$wWVm-fEjqX?E`; zj{d~3lyLfpm@tHVXh#o3qq7XyI^6D4K^$XGMVl$p%BbwpD0t^EOddH_7%%Dr-jj|OTp+7#Dm`)Wnxvo$(p6{7 zN7+zdZ>0xbL*Ut7@1pxq1z&L3Hw7v;Ml7&Af|>d@&^#5JOQ?RP{(S_~up5LaF$oJJ zu;CNe;j_UVV1(IjV;u07#o$IPfHcHBBN4yJhmnn0OH5zR_5j)yrY2Ey;?N40-kMH> zEhtvaSxY^Isgy+dmcwv$nl}J?L)5KAR!C*pn%ewOH|z#1t=;$x&rmw3X6^>aq=v4| zuuxi7U1_8G+GK56M$MFgPc+x?__+YibpB)h`pM>fnnwr2x0)b;Opgt$6)as0R0YT`^mLEp3UyB_~ zqQh-p0OVk{ulgD%oDnkRigVd$R%&l?--G88}{}fWUI0$-}3B z*HcG+Y*%}92@cYZB3`U5w&5P) zf*Cvsg`IGy>-LbKX`e+7U}x$Y42Tj-^-y^#C69DvQ4QGyor=rli$rH34>Om2W`qs_ zZjevD0IVX*cjHE_NlTL6j}x(K*tXQU1^HXGc@2N~?;2Tc%@$GWbC_Vtqhv zbyfy)2{n3nH{gAcKO~4&=fI z*d=Jmq5)~NHxAI5@1P8|iH+^O+&I*^tFcc06|>Dmq&NwL;f~=s(RGVB&)2*NnZZ-E zbJr_31^5>43<=v%6z7qn22UoYwBoaBV^lI!peUuRxTVks6)_EXtE1th^{rfi*se z`106+Y>zTvf=!r4&Cx(@pxP3QOM{1WbwZ_K8;RYOAt`Lc2M?{V@ z*nLu+_!-ontx)+q!#)#(n7_CS{Z%?{b+lpywsm#E3(bu^8$NqUPqhRbPx=|f?fwnAfl(nTj%pfv=% zn3({BPX#*{L%J9bC12{&Y3@f*IK^!;F*+pY;Pwp|J-2vbFz{krts|!{h^>3v*>1u4 zG{ebGw`ZQ%b3%f^qef;LBq6{HqEapd80xhGRldyA)`DLPn9=2)rQ z(WmYTqBkE!{$YOc%9@%fZ}|H=oGm{3sg@?_*Q6T67I$|xIp4|UFSb$ckKwfY`YS@> zq)E)T;Npjt&4&jeiBo89EAF8El&c&|5Bq9AB$+iUK9=QlRXq0+iINQPKEzu4@qgDr zsH!Or>HV~sGd;$_uz#=t#m-$kOdr#S0dE0cG$6%<$X*9p?z!W|Gi+E<`6Lc!lihYO zF=nHR)^15&Hv;G}O#C+>r!tOFR{M@t{>y5#6qj+cWIqnJsPUd;Y2J%5tccBqcu-FCoOxOnU9Hc2S6pb9J+ zGREW0+Lp#fAoBWEs~)N3E?{Qff?YfB5>d^%6?H(S1;5nYx*@Y)452$%Ymn0GYGkd7 z)o1P~L{iG<%$|K07|c*er@5f%m~D9RR`fB!y^XlH>yHcd-uFMvldTt`SZ0Y0-gh59 zY>C2pv%@H9*sA>;-y37M7MzfJ^=Vb>0~F$$S|KTd1$kuAUa8bn19Ss1Lt-$iYHqw+ zsM!vwughsu2=BT`0tzHWz%C3LIw=&xoDIrrV3kapQ%>&h!XK?q8%W>wy4hqX1k1?o z!b+ticEj?PwuqC^G48l6eqi0^&Cia6N^`|L5h{Gsu^2dESika~^{^vN$SsNsPYlb# zsZ&Q~_^u4`>K$f05R@-jh)sE+U}8kc_;9ZMSdx?UN8<#*jI_D+>v-EQM*Ke$s@4L$ zCUp2VMOqIvjxfr=jluV#`%1*88x_{w*#UoXgOOP#aq32KD^F-Oxkk)*Cb#e+{BGG0 z!Q&&_(Q&-NpjuOO80+ETCTv6G*N7ZF)XT@9K9UK}6@$MMpE z_uhrMbJPa-O@Dk5HT2#%ut2^;bMv-s4=9j>t*AY5fn&;8u@bzS#$*CK8O zqqDJ>iFt+XVAa+4@85qRO#?8I3Dh}PJuFB|w6v=oJV3|=Ai;&ziF8_QB}+``O>=xD zlnar}Q>Ra#2VjzzyUJOkQl}W*i}GOPg3h_)iPU)(K6yf0MEBTEY1pq|4l;^=+O$8d z=EAMY{(>rk-<;*RKJg2W9a@PPOIGGB2isSB_tq09ez@_JEC zt9|OirAyiSGm9H(z#9z(cRx4ENsEjrMgQjMg1UAJ46Q}b40qduzCxA_>$wP2xt&%G zNce(4P)Tudi7&vlDxXP5*600z##9PsYMYoWA=X0RzW<=~L3X!|6vGkk;bE;oT3V@9 ztzJX!0xE8aqU=}X`8@X1;Sl8^##>PfLK6F$;|*cL>;ZF!d_t;h-n)12nGCa(2$duT zUR)Z|vmJ0Wfp46M^z3xYWP#5^>!mJ76?DEV)N93kjr$cA!kAn;hH~|>bJco< z;p+u!sy`;a^mVa$_}+jDvWKphE|1R1=TSO~bSm)|+CglTG%W{Q?vIVT31q<*_p_LT z4MXxB9LtbB>h5q5BHr?Jc|;rjIdg2A4mLD26c9_G=C|O67b+-}B|3x9mqhxhYK_!+ zTSF^5FK^qSq3*sEw3jymJc_g)-sMJdu>Hty6P7jiNwlI4&}nkAY~pP1rlNti_Qi(D zTPVTsQiR@Nh4icTKIjN`DIB)|3_!y1J9ZhqkdLd1T9+S~Ux5NK4|ly7;8xA1IiLbI zU~uV23UP$G1ecLWZY1hw_+NVxRTJ9iPf1%pf1i+J3oPY4Mkg_a$b;%zZ&v!e>jj*V z75Be7%ntGKnON|X3347P9!5}?JbLWOwL>+AWh&J@4OHc3STgvp4#jA^ zql$cKysag7XQ>nQ@U7gTqj}A{%)=2f(@kvl!HolpaT?)A|3y>d+0J*Zv}|v?ZuTQ7`STQ+Wr> zO-6od@5$C=fWzKaF!0=`evm%fWm?g&W(muxD>9b;)!5@ZLTm}K`gw#=9@ zLmgN5>t6WdYNG>Wi`Q9VwluW2{;Riq_@zCXQ*`l^yxNOq(HNm&jfxC08pPQ8cDtCo&-t)0o5ogo#4ms&UgBJs z{x8Iwi04_!)}=+-#uylR;m(=;s$T@jw+(pKeO=-)KDld~7OMe?D_ih$y1+;YyL?#+ z5V^P2+c$5_0Wt<1KVxoTA!Uu^0lVema}$huVKGqVGP({R9m#jLcye(*Z1tAK6hf&* zfw=+u_6*J9TJ%E4b+}7QO9vuBm~{&`X{_|U-GxphUy+uO2*8`_W1jN$WjcJgR^Fj3 zCbj`Cr`(s1F$9XNfgWku_q>KBu{dAK-N~UiEx8m4&qhpITnt8PWugg+-}0M%9XhzK zsYv+q(}o=$iZ0thoV^)7=&aa`jF;{5mfhvWQ$do@NGxPj(GT6kAX{RpPZ`3}@t$l> z`ly0aMvNs{MZR~HN*rz;)!cIJaXSSkuV{py1cW3dw63hd zUTeDio(a3qKc9yTLTh+iWAfg@AAb0Ok$5Q}U_P{JhNlD?SNu> zc|)d8+l+2~pu7crh8QWpBw=m(#L}qUItp-o`DWz5tuc*Rl=`Ue+H+g%jtA zyN0`<{#xt5e^!RsMfyPBJvqL!SLu>#$toSm9Xu^8Z?1qd0%X#3%JU&yo8?)a>8vGi zCqTKV*V?tj=*}+ji_f0$FfqTnJk2SZ9RfoO0T|awyzRki*&^DjSGy|1H{)*b z3)mmSa6|-wEgSViyw4_>#c-!5Td2XoPxfh&>TWZ$avkfoJ~d2pwr4M2aJwiNgc>>V zRL?>r?=N2LpjH9ET@u2e4BE2ktK*n5QZLGyMrDMnzMT=?bq1Y~?;j5o{qmgl?Yg>R z&=1S+K-Jz2OAsH?fu-=y883R;2Xc~!G#MZ?wEOqh(&b@p$QZwz!C*)pQvoD0AJREQ zTSz+Ffk@_(A`&do%8!?DnW^KThE=|BG8P!WK;#X^1%OgZ7QGWeme`P#6NnieNd3iR zWMqQa5(_$visg_I-*O<6ZD^#@l}GXnAkR$0Yo~ie?$G^tF_Y&;N8>6aPW;db8w@Ws zNm5aetSIkW=j4>skBeTns1UA|m4d=`Z3k)comF!MD-liLqqhild@JNiV+^8U6yg1& z5chtgqdF3*G9Ar+&-=`~N~YlDXs-i08@W)RN+6)-yTCDaUqLz|;?Ci%?LL<^I+FbQ z?IQEGF^RikD^_qmI>XRj0`Y%1ormr=sB<++D}hgc*3Sxs(_eWwn%7ldG1D#bY`tg|!1J%)@1P`QyO+)vHaj`r9?=MD%zb;8I1`7W_7lHZ-Rr zM~)QMH(QFsY3Txn?+Pi+WL8K)d-Y2}Zz+6sC-L!7wdC8nPvMyx%n5ac`#$xcxpAL? zj?MyN*UPd6ra`fIFS4T?Zoi*<+?RN!Hj_s&Rvz9N?_34*f` zov2l#yYGa^vnTM*ng?gdsN!y*VNyRWS+5ogeL`DzcS=S!UQHlMpB^U003;t@CIAmq zDKdtTn7dJg5WC4-b{xQnyqfv2)uP=CL)n-#?&qJ&dyVk3J2{C+m&!v``09W$;|w#H z2~7j|GX?N%k>=I_7H*m@Ue4dW<1+@K6FX`7k6e#uPR8Ju7IUourII-$`Dy4vDQ~w` znRAYnjiH-JTV5?PyrR*KCiKqXpQ$`f)d{K|jO2c5QkyyDLjCKLb@eZ=j|$DL7!-Auv} zET8%D2tYOYBNXU_H;}>H8PJ9>#x1R{tCM)}67oSQBq@p1;iUT%w!;ZR$&UxLkQ?N4N9ohS+S6zkgcA(Dk6ZVyw^8rDJk2aX?f<9$)OLR zZzGELS1$OruzFkEmk7>FVa47?G1J`@wiFpcd(F^bDJyFal`2u1g%LtPe1(9;=HLwj?)3y+N6YO$2Q{i#ddn zk!4-Fgj#ic(r*#a<96D{Wn>r$OrM21(_q>4SS|~#l+-HdMX3Pwg4iiI*7;DS!Vmr- zt#uU!v9ap$NRcBhZZ#wk=8nno<;#Zwi{x=v1THm@9pBM?&W>4e1(4 z?%*`Uip2pbeA8^X33rTl(~~=^jZb6IKqN(R=p;)cNq&PZ{JGt&qz}=xPfY9Lma=Ii zlK=8cAU8qPQ=UIW83TGMt79r#W?-gDf|#Nr<;NeJhAiS@vPPxczkJgRgl8WPXEg5= zj85{0Q14PjY-P)e>9r0Z8ZXU|bRmHqd?C#V1wX_0NI(}HMC}ZgneE&eNI(caRW|sk zTIe@}pbovvuPY+*2FC(Qx-QCf%Mc8IEa$K1gc96DkDOKDuthP@j1<(|c;;H2V4+pe zD=8ui_ryuOyouCb$s${i=bg2yAgzl4a`xzhQ->QU%Nh3yYxkqchV{P2-~vSz6%|#j z(Ft1%0z08I6_Xw?TV+$~9>}#`E-ni9#=KbS%_3$(?qZbc`OC`4sPn?AptKwHwYB12 zbZD!YpZfX?D@t($>l|Jr#Uad+piZrVVn%a{es_uKPH`$pUNUD=JZHZ4_yU6OS~OON z$k&j;Q`Q7-JwF73!OPXQaOk`G!5bI$16)n$ z7E!luod=vP$?ffKwJT8WI;hODsa#RF3S&8JpC&k9*UOqc=B3UH^vT88_j!bHZ#m+s zE45_F47l?okYSXhel(;<T@l#0 z%~9h8CN02l&=fgkol>XrbeA})Cu}X59vJZJ-D^3)xRmciH>j}aXQL1Ne?T0`0G3(s zE*;K~khV73%!Q*m1w&{TVxq`%83m6|w^?{?JIFLsORe7{c02RvqHH~OycE-a@-Z65 z98F};V+>bnR3Ne2(Ph(Ko98sS_fMu_mz(rRrU%yX0;Kl^XeV)1MbQEn!7MCm2oE0P zIBBgMri%6>*qtJ)3T(ZD4omw*g3Y>jqZ9 zQ}KD0uC8uAnY%+V)lgTDfI(zA8zW*6J`t>HK{4)l@3iOmtN&>yGVm|~s=rM@PlGBy zh9X*}lTA(nKdN%3==>8k80Zp!Ja?Acugf3b-Q!^UDr$u#W!%WATVgTX#9}u_ifd z>mrD@fBf;s#fAh3;>&xNE-xs)cLIXRgkOcVIS}**tCULFxe)Tvja^ zz6wrEAEpG0V4NIu@~1ayJ4;~6Emc>^<`3}^>+Es>*9EJqyj;4hYQFf{fz+b5%WHDG)+fTbg2y zaiBa&V(O~M%n8tPUIHQ3ny*LUIa~ju3Oj+c9^ z6+&D7uHTmD+}-4_4sOlzPix?7Krvaiz@}v(TNab|`hrxeT4VSCFph`Goi9EU9pFOB zQ|#z$L9j>-Qza4_T=6FqgwHCm4Hw+o2>2?@E%m4oD@NDHd=Y~;Ma>YVAQuQ$zmf$pb5-gB8QilA-Uq%&~i>^x$7&RUpex!C? zNY1kA447F~Hp9f=!sYDG`H{yks)I1K&o#VWnImnJDR{|H&*;Djm;fbjd(r`=q0;Iz z&t5H9wU&)PjQ(a_0fjxCMtxa;t1khIygKz7E|7A#=57g9h<|i^s=%bFF&bqCp2D~R zhf#mr3QFuoa%|<0L?F@4Ps5-X2B@Uquaz!k# zHKfE+T-x&9Bz)W3@(Idq0=ruxUf+W~#+7ILjhhQL$<_4S>{Vi< zfV`TzC#M5uIeGA>G^Ai?6w4X+JqvRhHgDOoPsU{;h(1*O`6}$1ni`G;=2YyAy@WQn zlAct-x8=qDXaq2S7k&q}?f@Swr(0AYB+&Yd;qEbvQ9<&#=yT}@^p@%gXMObkt80+Z zD|`H7rTm{s1Ym{}HVeH32cF}H#SZ)rnIc?VOa%_gfXA3LapJP1DstK|6EL~x$juQg z0LvCqn4s|oU{!a@Zjvy7z;}tKJ#G$pLkNHea6L-x% zPe#20*^6LC01M@(^&a1JNri2;dv{Zbx8$qsYFj1=Zdb{{O~j{C0HxjHDV9$_%|af( zlZ;Db!b_ilZdi51g;af`qqC2M)Ijv_Pi8b(_@Pz-#J2Ks!VTy>_UwI+)LqD+Mt7CV z`_O$gS&(?(SSf^U#bp9f1P#Ft`{4PN+M&Vj?DLl{S-P6~jr%!d+LBne6rmOS`~fXc zZ9w{DcJEsCUL^?^6X zuln;PgrkSqwA=(#NaG*&$dh$B2Q#b1Zg_!A`^{BOH12a zSOYk%0JtO)x>?JvnEbKY*Qvnhi}rwFbOTqj-)=$;8Xi4%jO#RG)};%3CR{v3N0@-^ zkHF-kiEgox3`RcGX1rnN(7AH~&6rz`9zAN%Y@j_yXH?fn0c@sVlUN4KxMAuHfg&)I`UqHapYV|;We*}O~Rcq{87!+q< zIY(qSXsNfcoT{ZUi==MKiN$0s3>LGA90{RZ9Gwli3jf}*wSp{2AOJFUvAv_pl4GEg$HkW{8%Tm`u%j{{2 zo#{IFRw#m&Qa90Occ?{XT`h!Q38cRF(aI-xG)w_Joh4n!$zgLhRL6iFoG=b&tJd{Pt`2mVpICIjR0)GY{^xLIg_kGIx->9VYiqyBIA+tVX3d9E_hf(s$pg!VaodNN(UlaJJQ^=?S+b&Ei z(9{usLlEeLD4C4K4uRo6q0<^YV^E_*sk#|kQ`*zUs?`}+1|u}sd6WR4%Ef8tBnp>yOe@I)(8}i}hMAnz7J*^^%Z@}&87Pm+i;;qM zqotVoIDA-t`_0U9$idz|f{6PXF!S~}OGWCazIx($4(={NO&}`vi zUziW}_vB|*KFJdFOo4WrkzI+@H4kn42GK6okE@=HR%NA+X;Ja@UcS4XuomyDxWfuS zbMuJLK#ssC2o9~peXgq#KYRH-<4rN2L);pjBW9aT7l??^UaJ5&KM$Fon2Z;SBPjGr zEi`d&%VwO8-;+7$k6_zeCPSAeB?Q$L(ClX@=Zag>+lB^4P$)8ft0ISL&PD4fWzqQ zgb3Ik-S?Y-0LFOA_u8VE-`$j6n49a54(FWx;%`z0T7VH=NF7zM2z?u-UF$GJf%3;@daK7B{q5}xp{1{`Az<_#z zl$@I}JN!C5o3N()FTx`1h19!~@wS(^5R`2|>gK^2H*VafUAuN&zt}(;qV+zMr@$E@ zib-mf&{Zx)v@A%Bh|U7Q_ibJJnX_jx%hum5#niw+5(EluSb11ML^I*A(L|%VQ1D5O zcL<6wV~o1jl>v$>AFA#^C$MY5XsD8}Ot>NQp}RYnxZrso<;O;-Bg4DE(i*Z8hvA!N za)q>~^+Aa@1Sjz*gSwF2a(hKzHyN!6~bIpH1+IDD9=pw|n-ta>0Z z33MW&C(#c#u(+sifXNe*?#dZ_o(F57>r4#53~W~vItDJE3KjIeW6FjOg5<(S(kc6d zi8V)9EyIQHd5oP^0yKttrxbdxh0@Y7)iz9n|T;R08s6l_2p`j`Vi z>y}%DF)Ufg3SHstvk*o#XC7$5ls3=^?Fsp5ByN6GA$Ar-8L3WErZgP;{J2A%QtOWF z(d=xtTnzsl`<@m%FYBAm1U1H7SD<;x%9pXNfD=MA!p9c~bgt`#!9-qO8VWiodK*Bm z&iB!%Nr=_}0-fS{p1)56Lam)u2^lFzsjPK-7T+W}`=HqhrZpb=5x zm?qLJbGXTD*Vf)u?f_rYeCRD{f9_jNNI6t&Bj;k{v6#4e*a;oT6r9BGfZsWgkkuUC zfZ^i=Xs-+PgdqjVeofq86S)Q2Kn9<&q2iY$tR|b?-CC1v@~0hG30VXVSB!Y|fbdZa zqtN^zJShEm_hmi$9{$9+yAiq?o+bX4?$z)spkldLUPFiK())|)g9{-$-5it0keoMf z4GdPLx85k$9>ASBA0CD2&^#Fq?)#!URNjc`BxO3*R{_kT?sTWiL;nF?=2f>OY>EY# z2XF&Xu4$unKP<+yixijlO15Y3vRlyV3`E{3%j=X6eq8}3jm*b|`e_@W%Dx(@KB2jD zOTk+y-@I`zgPAP~aWlY=@E0nhq& zXNl3{G||}7yLZ>Dn|}^EQ&u8-L%l1?b6IBYP2duF*o=WrM7K_STAZvFNK0!XrJI2& zz3|+j88c@tLR1a~^Y|t(O6tU^kZ1~P6G=1_2BLOcN6Ch6irB9}rpV{nB{u_zE;)d8CcQC8m2x5AuO+&v z%%decfT<1>p_{jET|uo%Ij+*#SK|PO0tqfrhbrQ%#bnnYA4W60Is8gl2!(S8JkZl? zk9E=j4qJzFMMkE6InE~Tnkz6e=!W}Cz%X^UqGQiPSo~q*YC0q&q{l4zW9g_X`>^+# zZ-==82&r)ubW+|rDv64S-2V#>W1KKGJ{`*)Mz%4FOIG^BA~Jjr6GY7#-KQW9pmC-^ zdp{d;r8;jRTYo@P5VprST!wfE{i-uk;k`*DhQXxW0--arcbmSxI7o9Z%7bGZe7G*EOR53WCI_9Mksofbf#H#y7m zFg#KxeVonNXwcv&)+Wq{?~5_Ug9kc6@)Hb3b~C8KGX~x4I{Q4>UDTnW{sM{L1L#u} zN6)j2sP-+1P!&HLRpR#O%>{i-#%_`4R)83@g^!mBqJ@+SxH4rOSE#C*;P2u5&c{iI zmWbwr-m$a9WXQWF200D{)?m%glII+pQT)AB!W#vu2u}WwOyjM6>gPNhMUi9Ei@X=k znI>@%1#hGPkY9|8AdxwQkvtjZxAUiHUR-vMc9-tWyQlvuxlIsM{=B&Y2ZqQ~35SGl z&&9i;(J{vrcCE_9Oppr8DyVr)1yJIl17y2Ek=YK^negN7W0Lvucp|sGD&nHRdK$-( zXO09%oQp)g4$TSO0zbO7y;+aA5`ayj7PQ1z5)|bN*YJxM_ckfd1+xRzS%Ql~vktU= z5ShSzcJAs!dAEh3!2#P)?b&TCKve&*kO&S92896HTNI8t{79!@JKR=Yg9BBXW(u42 z0K9S&S;r_DWHl>9Z$DXr?}iQAZt;0es6S;7yip@dOqK>e(tRNbT+I3K@Fiq;Ixo=6 zLG-b+pUw%U(H454^=wOefm~q1jd+;DClRDb7JJ9Q(-ul}S%|Y!(b+hxuEPu!ddJ)C z7J7K5P+=D!Ak@P8fQE(=4Qyy-SX-&CMkj~b`t|WQI8buaeT5aDYqzYMqOuUP(gT{C zo9Vm;QmA#w+M}md3OExD`9aTlCnDB^kJ)C#&IqI^;64_&a=Tj(BCXoVse>jTMR+Yn z!33bim+p~5awqhv7od!!!V5jvBZVZw%5ZW5TF=TR(HcyuVOM7Fs5(|Ig6G@NyMUwd z%#=!L5aNWpAk`CThY(;T;3_1{95L;Z#nj$L*xr!PKk;TCU4y(IL>37y8PR0B3-`V16V=FCVW1!rTCa0NMXov*V7xi#MV4#Fo?$U9a)-4r)tHY<8Q6n4`asWL zyeJMC=LH}Tr;*`6dWmd@R2b!V4qTVjj>tk~aE%C8T#m(MGE_OzPMN2DaCywSWte>^ zlgJKP;T*uHd9YC0M{m#_S3-t@0e1E_D9l9LX+Gc&>kzURbdW<9DBsU@yb;GuO@% zJx#tAYb(^hglL6I^BOeaJBjU?lIOraq=FxWlWH71BHTBLz!SgymJr=Xw@oz)`Z z5XkGb2i1B0;S3G3zX?GNnPOI73N}|yzwaf_fkB7IoB;K97VBqYzb2EW)qi!*dP5cHPkNL=&pWCO0HxXtE(Uw4YmecT-8hc~|(mPjH3Hk|=Z?tU>Upx#)cU zIP|l@DIdJ94T!i)dIUZ?0fygn{ZsT{cQnubZJt0#NQUPB$dT}S8gONuIlcqrO!E@Yko0^$l%(Ip7}CiMQ$-#rI7hky8Kv1RR5)Rl#~0m zbnL}uXXh?0@1qwb#Kai7ouoGa?Uqb3$(;IHPecUA`Lh@rm+_;Tn(XS966gvmf0;5R zWno#a4|vTyqHzS8mYlfq3v5X2zMLlSz^2 z?${_o+JB*2uXw)Z8KxumknUl)mLA?4!r519?m#*jNe%)ta#Q%7*;EW}p9>5WnUn~a zS6sun;Rad|fY8XxAaG!X%ZByq^AOI>Nv4Q4u(<<9rA{y!OUaC@$^M0SMKOL}NWO>E z$ABu*YaOC{A*t2Z>9y|6${A^uzH+r~=;OCKb#KZe>BAumN4n6)Kd0(W zXn|~|m$&qjE_Ab>o9OcL+9Fhr7#O! zgsI5oar7g$b{NW^l+(1Hes7(oCkYq>SD{NOu#U?VBLxoi2Xm*e2BHF5T>y_pV(uD* zga3l-eyHPeO)hXdQsk&UID^{pP`d@9kJIG(&)qe#VlE`$L{naBU>uD!CgA!*ewT}8 z$}Pql@r~^;q9o#DDpG}@mi2|N5j~&^!Eetc4FEWN;Sd@;<8+~tkR)ED=9qtAk8<7> zHf!XEo`XK>+VLANP%{+A3z0Z7yOrT(91qciCQ*8WwZ3B>23C)-n43(qLf&Lti(o@T zepvuWXA$xf$Czq#Y!BBFq!J)-vHfu47-?r9N?>Jp&6AEM!QM~?INJ9{WLr>SL6)yl z#O+ik$9*sZ)UK1;An=b;R~z~1h>#*8*C#~DbowFD zk*wKCrAD%~)uQ`)V-Pa)q4iyo-CGh@i=68Ma=WtqUI1RjK@ta}Pl=jKzm*M$*OtF; z&Pf0KVefkMVizs@+r$3HKh_j?!G1R_yJ%aCf1c4Q47B1;kMEefP8l|YPB04ll ziwTYgTD7$x{X($9wL|%<1wph2K=WEzDyW45nu{LRR)IQT~oJy?3v8MMcG+PphYw*J@ooJu^#7iJ;(MvjYcY@UQEcnJODLZoHli1G_+V z%mRai{cLRP61!Q0ZJB6wF!S_OQT6yV_44J*+oDT$ySuOAl$KtPiIE;ZVS<^Xqa0ig zr$=s=@IHC+EqVB+jEak>$X3nmv zs(MylE`Th+ue!SWT}z8!Y-}v~5FQ?$t)lAGL}R~L;JAPPR3RauZ85sn4PrLL=$^4k z6129q-XG#;K-~bQ2D}4A+ZTgw@W+Rt| z+>x9DY{b#}2PnSv1G%yb<2U4^gWx*AF_KR-@P}&9+&pJEIk@(ozDS%PK!~b2`e0;1q zp9WE?#((;z^NPf2lkS4CE6^3TV_~<$bi7+X*!P49e@y6v+G~itbEdMBdam$qi`l# z+&5f*=3}2KJbU)+p|(!46^x90q4~%`HF>DV4cMZLtn6<6FF$p7!Mw5jF+K?C8you} zw-TRm20Vx#-l-3|0RV-n;kiG?W`7I2t)UWY3@f=95^{R&)m{Jj8_O#z&wysQNT(0c z!%Y{@cnkTLjDo`6krQ}$f&G)QN;I#t(4Jik4V4@D;@xTdV>xtxTK6)|NotRU#apna zk4IKTVeOcE-s>=K=p3{-U5kz`7&2wEjJB0M6-+}t7gtg6aJ;mhwhT;o61`T6=67Ao!eR!4X4g0Y_e z*PkBy7>s;*Uck5i^q9|SJ(gy(y5 z^6xeIo}Bz^O}-~5|5}p&Pm_}&+t#zg5Y6}=L#^Ewr^fLt;~xu47UH+{2uS+H&1 z$ajwM(N}FO{A%PIzyINpmH7Pe`%?dXJ$SxvuD>tH_v7^U_4tnfBA)41^~V3O=XUZ_ Oo7Znui(R|x=>G#?IKPJg literal 0 HcmV?d00001 diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index 80561c27..c2b27e2a 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -13,12 +13,16 @@ docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingf -To use GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) . We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. - -To use TGI on RoCm-enabled AMD GPUs (only MI210 and MI250 are tested), please use the image `ghcr.io/huggingface/text-generation-inference:1.2+rocm` instead. For details about the usage on RoCm, please refer to the [Supported Hardware section](./supported_models#supported-hardware) and [AMD documentation](https://rocm.docs.amd.com/en/latest/deploy/docker.html). +To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. +TGI also supports ROCm-enabled AMD GPUs (only MI210 and MI250 are tested), details are available in the [Supported Hardware section](./supported_models#supported-hardware) and [AMD documentation](https://rocm.docs.amd.com/en/latest/deploy/docker.html). To launch TGI on ROCm GPUs, please use instead: + +```bash +docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --device=/dev/kfd --device=/dev/dri --group-add video --ipc=host --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2-rocm --model-id $model +``` + Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section, where we show examples with utility libraries and UIs. Below you can see a simple snippet to query the endpoint. diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md index d7d45b70..34775139 100644 --- a/docs/source/supported_models.md +++ b/docs/source/supported_models.md @@ -41,7 +41,11 @@ text-generation-launcher --model-id TGI optimized models are supported on NVIDIA [A100](https://www.nvidia.com/en-us/data-center/a100/), [A10G](https://www.nvidia.com/en-us/data-center/products/a10-gpu/) and [T4](https://www.nvidia.com/en-us/data-center/tesla-t4/) GPUs with CUDA 11.8+. Note that you have to install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) to use it. For other NVIDIA GPUs, continuous batching will still apply, but some operations like flash attention and paged attention will not be executed. -TGI also has support of RoCm-enabled AMD Instinct MI210 and MI250 GPUs, with paged attention and flash attention v2 support. The following features are missing from the RoCm version of TGI: quantization and flash [layer norm kernel](https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm). +TGI also has support of ROCm-enabled AMD Instinct MI210 and MI250 GPUs, with paged attention and flash attention v2 support. The following features are currently not supported in the ROCm version of TGI, and the supported may be extended in the future: +* Quantization (GPTQ, AWQ, etc.) +* Flash [layer norm kernel](https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm) +* Kernel for slinding window attention (Mistral) TGI is also supported on the following AI hardware accelerators: - *Habana first-gen Gaudi and Gaudi2:* check out this [example](https://github.com/huggingface/optimum-habana/tree/main/text-generation-inference) how to serve models with TGI on Gaudi and Gaudi2 with [Optimum Habana](https://huggingface.co/docs/optimum/habana/index) +* *AWS Inferentia2:* check out this [guide](https://github.com/huggingface/optimum-neuron/tree/main/text-generation-inference) on how to serve models with TGI on Inferentia2. From 3238c49121b02432bf2938c6ebfd44f06c5adc2f Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 5 Dec 2023 14:42:55 +0100 Subject: [PATCH 488/793] Add a stale bot. (#1313) --- .github/workflows/stale.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..a5e50a79 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,14 @@ +name: 'Close stale issues and PRs' +on: + schedule: + - cron: '30 1 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' + days-before-stale: 30 + days-before-close: 5 From 9ecfa16b12c13ac6ed136929258829208ed8afc5 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 11 Dec 2023 12:46:30 +0100 Subject: [PATCH 489/793] Speculative (#1308) --- Cargo.lock | 66 +-- docs/source/basic_tutorials/launcher.md | 8 + .../test_flash_medusa_all_params.json | 98 +++++ .../test_flash_medusa_load.json | 414 ++++++++++++++++++ .../test_flash_medusa_simple.json | 103 +++++ integration-tests/models/test_flash_medusa.py | 59 +++ .../models/test_flash_mistral.py | 4 +- launcher/src/main.rs | 15 + load_tests/common.js | 21 +- load_tests/tgi.js | 4 +- proto/generate.proto | 39 +- router/client/src/client.rs | 4 +- router/client/src/lib.rs | 8 +- router/src/infer.rs | 94 ++-- router/src/queue.rs | 61 ++- router/src/server.rs | 1 + server/Makefile-vllm | 29 +- server/tests/models/test_bloom.py | 4 +- server/tests/models/test_causal_lm.py | 4 +- server/tests/models/test_seq2seq_lm.py | 4 +- server/text_generation_server/cli.py | 28 +- .../text_generation_server/models/__init__.py | 36 +- .../models/causal_lm.py | 19 +- .../models/flash_causal_lm.py | 188 +++++--- .../models/flash_llama.py | 13 + .../models/flash_mistral.py | 63 ++- .../models/idefics_causal_lm.py | 16 +- server/text_generation_server/models/model.py | 7 + .../models/seq2seq_lm.py | 18 +- server/text_generation_server/models/types.py | 40 +- server/text_generation_server/server.py | 6 +- server/text_generation_server/utils/medusa.py | 51 +++ .../text_generation_server/utils/speculate.py | 12 + server/text_generation_server/utils/tokens.py | 94 +++- 34 files changed, 1349 insertions(+), 282 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_load.json create mode 100644 integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_simple.json create mode 100644 integration-tests/models/test_flash_medusa.py create mode 100644 server/text_generation_server/utils/medusa.py create mode 100644 server/text_generation_server/utils/speculate.py diff --git a/Cargo.lock b/Cargo.lock index fa1a45a7..93e13056 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,30 +60,30 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "a3a318f1f38d2418400f8209655bfd825785afd25aa30bb7ba6cc792e4596748" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -333,9 +333,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.10" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" dependencies = [ "clap_builder", "clap_derive", @@ -343,9 +343,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.9" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" dependencies = [ "anstream", "anstyle", @@ -392,9 +392,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" @@ -549,9 +549,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" dependencies = [ "powerfmt", ] @@ -1218,9 +1218,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lock_api" @@ -1612,9 +1612,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.60" +version = "0.10.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" +checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -1644,9 +1644,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.96" +version = "0.9.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" +checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" dependencies = [ "cc", "libc", @@ -2309,15 +2309,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2567,9 +2567,9 @@ dependencies = [ [[package]] name = "slotmap" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e08e261d0e8f5c43123b7adf3e4ca1690d655377ac93a03b2c9d3e98de1342" +checksum = "dbff4acf519f630b3a3ddcfaea6c06b42174d9a44bc70c620e9ed1649d58b82a" dependencies = [ "version_check", ] @@ -3835,18 +3835,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.26" +version = "0.7.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.26" +version = "0.7.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" dependencies = [ "proc-macro2", "quote", diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index 62abe8c6..9590e463 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -67,6 +67,14 @@ Options: - bitsandbytes-nf4: Bitsandbytes 4bit. Can be applied on any model, will cut the memory requirement by 4x, but it is known that the model will be much slower to run than the native f16 - bitsandbytes-fp4: Bitsandbytes 4bit. nf4 should be preferred in most cases but maybe this one has better perplexity performance for you model +``` +## SPECULATE +```shell + --speculate + The number of input_ids to speculate on If using a medusa model, the heads will be picked up automatically Other wise, it will use n-gram speculation which is relatively free in terms of compute, but the speedup heavily depends on the task + + [env: SPECULATE=] + ``` ## DTYPE ```shell diff --git a/integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_all_params.json b/integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_all_params.json new file mode 100644 index 00000000..d8a298eb --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_all_params.json @@ -0,0 +1,98 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 338, + "logprob": -10.0078125, + "text": "is" + }, + { + "id": 21784, + "logprob": -15.515625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -2.8847656, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -4.140625, + "text": "?" + } + ], + "seed": 0, + "tokens": [ + { + "id": 13, + "logprob": -1.1582031, + "special": false, + "text": "\n" + }, + { + "id": 2772, + "logprob": -0.23083496, + "special": false, + "text": "De" + }, + { + "id": 1022, + "logprob": 0.0, + "special": false, + "text": "ep" + }, + { + "id": 6509, + "logprob": 0.0, + "special": false, + "text": " learning" + }, + { + "id": 29892, + "logprob": -0.61816406, + "special": false, + "text": "," + }, + { + "id": 607, + "logprob": -0.7089844, + "special": false, + "text": " which" + }, + { + "id": 508, + "logprob": -1.7724609, + "special": false, + "text": " can" + }, + { + "id": 367, + "logprob": 0.0, + "special": false, + "text": " be" + }, + { + "id": 5545, + "logprob": 0.0, + "special": false, + "text": " considered" + }, + { + "id": 408, + "logprob": -0.3869629, + "special": false, + "text": " as" + } + ] + }, + "generated_text": "What is Deep Learning?\nDeep learning, which can be considered as" +} diff --git a/integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_load.json b/integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_load.json new file mode 100644 index 00000000..413af1d7 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_load.json @@ -0,0 +1,414 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -10.734375, + "text": "What" + }, + { + "id": 338, + "logprob": -1.5488281, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.2890625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.2753906, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.48046875, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.1845703, + "special": false, + "text": "\n" + }, + { + "id": 2772, + "logprob": -0.5727539, + "special": false, + "text": "De" + }, + { + "id": 1022, + "logprob": -0.00010967255, + "special": false, + "text": "ep" + }, + { + "id": 6509, + "logprob": -0.1239624, + "special": false, + "text": " learning" + }, + { + "id": 338, + "logprob": -0.04510498, + "special": false, + "text": " is" + }, + { + "id": 263, + "logprob": -0.018295288, + "special": false, + "text": " a" + }, + { + "id": 11306, + "logprob": -0.45922852, + "special": false, + "text": " subset" + }, + { + "id": 310, + "logprob": -0.00020992756, + "special": false, + "text": " of" + }, + { + "id": 4933, + "logprob": -0.0046539307, + "special": false, + "text": " machine" + }, + { + "id": 6509, + "logprob": -0.00025844574, + "special": false, + "text": " learning" + } + ] + }, + "generated_text": "\nDeep learning is a subset of machine learning" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -10.734375, + "text": "What" + }, + { + "id": 338, + "logprob": -1.5488281, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.2890625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.2724609, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.47729492, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.1826172, + "special": false, + "text": "\n" + }, + { + "id": 2772, + "logprob": -0.56689453, + "special": false, + "text": "De" + }, + { + "id": 1022, + "logprob": -0.000108003616, + "special": false, + "text": "ep" + }, + { + "id": 6509, + "logprob": -0.1239624, + "special": false, + "text": " learning" + }, + { + "id": 338, + "logprob": -0.044433594, + "special": false, + "text": " is" + }, + { + "id": 263, + "logprob": -0.018295288, + "special": false, + "text": " a" + }, + { + "id": 11306, + "logprob": -0.45922852, + "special": false, + "text": " subset" + }, + { + "id": 310, + "logprob": -0.0002104044, + "special": false, + "text": " of" + }, + { + "id": 4933, + "logprob": -0.004711151, + "special": false, + "text": " machine" + }, + { + "id": 6509, + "logprob": -0.00025892258, + "special": false, + "text": " learning" + } + ] + }, + "generated_text": "\nDeep learning is a subset of machine learning" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -10.734375, + "text": "What" + }, + { + "id": 338, + "logprob": -1.5488281, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.2890625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.2724609, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.47729492, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.1826172, + "special": false, + "text": "\n" + }, + { + "id": 2772, + "logprob": -0.56689453, + "special": false, + "text": "De" + }, + { + "id": 1022, + "logprob": -0.000108003616, + "special": false, + "text": "ep" + }, + { + "id": 6509, + "logprob": -0.1239624, + "special": false, + "text": " learning" + }, + { + "id": 338, + "logprob": -0.044433594, + "special": false, + "text": " is" + }, + { + "id": 263, + "logprob": -0.018295288, + "special": false, + "text": " a" + }, + { + "id": 11306, + "logprob": -0.45922852, + "special": false, + "text": " subset" + }, + { + "id": 310, + "logprob": -0.0002104044, + "special": false, + "text": " of" + }, + { + "id": 4933, + "logprob": -0.004711151, + "special": false, + "text": " machine" + }, + { + "id": 6509, + "logprob": -0.00025892258, + "special": false, + "text": " learning" + } + ] + }, + "generated_text": "\nDeep learning is a subset of machine learning" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -10.734375, + "text": "What" + }, + { + "id": 338, + "logprob": -1.5488281, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.2890625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.2724609, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.47729492, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.1826172, + "special": false, + "text": "\n" + }, + { + "id": 2772, + "logprob": -0.56689453, + "special": false, + "text": "De" + }, + { + "id": 1022, + "logprob": -0.000108003616, + "special": false, + "text": "ep" + }, + { + "id": 6509, + "logprob": -0.1239624, + "special": false, + "text": " learning" + }, + { + "id": 338, + "logprob": -0.044433594, + "special": false, + "text": " is" + }, + { + "id": 263, + "logprob": -0.018295288, + "special": false, + "text": " a" + }, + { + "id": 11306, + "logprob": -0.45922852, + "special": false, + "text": " subset" + }, + { + "id": 310, + "logprob": -0.0002104044, + "special": false, + "text": " of" + }, + { + "id": 4933, + "logprob": -0.004711151, + "special": false, + "text": " machine" + }, + { + "id": 6509, + "logprob": -0.00025892258, + "special": false, + "text": " learning" + } + ] + }, + "generated_text": "\nDeep learning is a subset of machine learning" + } +] diff --git a/integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_simple.json b/integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_simple.json new file mode 100644 index 00000000..15754b14 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_medusa/test_flash_medusa_simple.json @@ -0,0 +1,103 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1724, + "logprob": -10.734375, + "text": "What" + }, + { + "id": 338, + "logprob": -1.5488281, + "text": "is" + }, + { + "id": 21784, + "logprob": -9.2890625, + "text": "Deep" + }, + { + "id": 29257, + "logprob": -1.2753906, + "text": "Learning" + }, + { + "id": 29973, + "logprob": -0.48046875, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -1.1845703, + "special": false, + "text": "\n" + }, + { + "id": 2772, + "logprob": -0.5727539, + "special": false, + "text": "De" + }, + { + "id": 1022, + "logprob": -0.000108122826, + "special": false, + "text": "ep" + }, + { + "id": 6509, + "logprob": -0.1239624, + "special": false, + "text": " learning" + }, + { + "id": 338, + "logprob": -0.044433594, + "special": false, + "text": " is" + }, + { + "id": 263, + "logprob": -0.01852417, + "special": false, + "text": " a" + }, + { + "id": 11306, + "logprob": -0.45922852, + "special": false, + "text": " subset" + }, + { + "id": 310, + "logprob": -0.0002104044, + "special": false, + "text": " of" + }, + { + "id": 4933, + "logprob": -0.004787445, + "special": false, + "text": " machine" + }, + { + "id": 6509, + "logprob": -0.00026226044, + "special": false, + "text": " learning" + } + ] + }, + "generated_text": "\nDeep learning is a subset of machine learning" +} diff --git a/integration-tests/models/test_flash_medusa.py b/integration-tests/models/test_flash_medusa.py new file mode 100644 index 00000000..003409b0 --- /dev/null +++ b/integration-tests/models/test_flash_medusa.py @@ -0,0 +1,59 @@ +import pytest + + +@pytest.fixture(scope="module") +def flash_medusa_handle(launcher): + with launcher("FasterDecoding/medusa-vicuna-7b-v1.3", num_shard=2) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def flash_medusa(flash_medusa_handle): + await flash_medusa_handle.health(300) + return flash_medusa_handle.client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_medusa_simple(flash_medusa, response_snapshot): + response = await flash_medusa.generate( + "What is Deep Learning?", max_new_tokens=10, decoder_input_details=True + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_medusa_all_params(flash_medusa, response_snapshot): + response = await flash_medusa.generate( + "What is Deep Learning?", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + stop_sequences=["test"], + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + decoder_input_details=True, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_medusa_load(flash_medusa, generate_load, response_snapshot): + responses = await generate_load(flash_medusa, "What is Deep Learning?", max_new_tokens=10, n=4) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]), f"{[r.generated_text for r in responses]}" + assert responses[0].generated_text == '\nDeep learning is a subset of machine learning' + + assert responses == response_snapshot diff --git a/integration-tests/models/test_flash_mistral.py b/integration-tests/models/test_flash_mistral.py index 63cb09b5..7d21afd9 100644 --- a/integration-tests/models/test_flash_mistral.py +++ b/integration-tests/models/test_flash_mistral.py @@ -21,6 +21,7 @@ async def test_flash_mistral(flash_mistral, response_snapshot): ) assert response.details.generated_tokens == 10 + assert response.generated_text == ": Let n = 10 - 1" assert response == response_snapshot @@ -55,6 +56,7 @@ async def test_flash_mistral_load(flash_mistral, generate_load, response_snapsho ) assert len(responses) == 4 - assert all([r.generated_text == responses[0].generated_text for r in responses]) + assert all([r.generated_text == responses[0].generated_text for r in responses]), f"{[r.generated_text for r in responses]}" + assert responses[0].generated_text == ": Let n = 10 - 1" assert responses == response_snapshot diff --git a/launcher/src/main.rs b/launcher/src/main.rs index b4fc86b7..4e230205 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -155,6 +155,13 @@ struct Args { #[clap(long, env, value_enum)] quantize: Option, + /// The number of input_ids to speculate on + /// If using a medusa model, the heads will be picked up automatically + /// Other wise, it will use n-gram speculation which is relatively free + /// in terms of compute, but the speedup heavily depends on the task. + #[clap(long, env)] + speculate: Option, + /// The dtype to be forced upon the model. This option cannot be used with `--quantize`. #[clap(long, env, value_enum)] dtype: Option, @@ -375,6 +382,7 @@ fn shard_manager( model_id: String, revision: Option, quantize: Option, + speculate: Option, dtype: Option, trust_remote_code: bool, uds_path: String, @@ -432,6 +440,11 @@ fn shard_manager( shard_args.push(quantize.to_string()) } + if let Some(speculate) = speculate { + shard_args.push("--speculate".to_string()); + shard_args.push(speculate.to_string()) + } + if let Some(dtype) = dtype { shard_args.push("--dtype".to_string()); shard_args.push(dtype.to_string()) @@ -882,6 +895,7 @@ fn spawn_shards( let shutdown_sender = shutdown_sender.clone(); let otlp_endpoint = args.otlp_endpoint.clone(); let quantize = args.quantize; + let speculate = args.speculate; let dtype = args.dtype; let trust_remote_code = args.trust_remote_code; let master_port = args.master_port; @@ -896,6 +910,7 @@ fn spawn_shards( model_id, revision, quantize, + speculate, dtype, trust_remote_code, uds_path, diff --git a/load_tests/common.js b/load_tests/common.js index be812e9b..5d71abea 100644 --- a/load_tests/common.js +++ b/load_tests/common.js @@ -7,7 +7,9 @@ const seed = 0; const host = __ENV.HOST || '127.0.0.1:8000'; const timePerToken = new Trend('time_per_token', true); -const throughput = new Counter('tokens_per_s'); +const tokens = new Counter('tokens'); +const new_tokens = new Counter('new_tokens'); +const input_tokens = new Counter('input_tokens'); randomSeed(seed); // const shareGPT = JSON.parse(open("ShareGPT_V3_unfiltered_cleaned_split.json")) @@ -19,7 +21,7 @@ export function get_options(reference_latency_ms){ thresholds: { http_req_failed: ['rate==0'], time_per_token: [{ - threshold: `p(50)<${3 * reference_latency_ms}`, + threshold: `p(50)<${5 * reference_latency_ms}`, abortOnFail: true, delayAbortEval: '10s' }], @@ -28,7 +30,7 @@ export function get_options(reference_latency_ms){ load_test: { executor: 'constant-arrival-rate', duration: '60s', - preAllocatedVUs: 100, + preAllocatedVUs: 10, rate: 10, timeUnit: '1s', }, @@ -48,17 +50,22 @@ export function run(host, generate_payload, max_new_tokens) { return; } + check(res, { 'Post status is 200': (r) => res.status === 200, }); - const n_tokens = max_new_tokens; - const timings = res.timings.duration; + const duration = res.timings.duration; if (res.status === 200) { - const latency_ms_per_token = timings / n_tokens; + const body = res.json(); + const n_tokens = body.details.tokens.length; + const latency_ms_per_token = duration / n_tokens; timePerToken.add(latency_ms_per_token); const latency_in_s = latency_ms_per_token / 1000; const individual_throughput = 1 / latency_in_s; - throughput.add(individual_throughput); + const _input_tokens = body.details.prefill.length; + tokens.add(n_tokens + _input_tokens); + input_tokens.add(_input_tokens); + new_tokens.add(n_tokens); } } diff --git a/load_tests/tgi.js b/load_tests/tgi.js index 93a0e278..1db4ab6f 100644 --- a/load_tests/tgi.js +++ b/load_tests/tgi.js @@ -1,13 +1,13 @@ import { get_options, run } from "./common.js"; -const reference_latency_ms = 30; +const reference_latency_ms = 70; const host = __ENV.HOST || '127.0.0.1:8000'; const max_new_tokens = 50; function generate_payload(gpt){ const input = gpt["conversations"][0]["value"]; - return {"inputs": input, "parameters": {"max_new_tokens": max_new_tokens, "temperature" : 0.5}} + return {"inputs": input, "parameters": {"max_new_tokens": max_new_tokens, "decoder_input_details": true}} } export const options = get_options(reference_latency_ms); diff --git a/proto/generate.proto b/proto/generate.proto index c873e661..19ec059b 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package generate.v1; +package generate.v2; service TextGenerationService { /// Model Info @@ -32,6 +32,7 @@ message InfoResponse { string dtype = 2; string device_type = 3; optional uint32 window_size = 4; + uint32 speculate = 5; } /// Empty request @@ -135,43 +136,27 @@ message GeneratedText { optional uint64 seed = 4; } -message PrefillTokens { - /// Prefill Token IDs +message Tokens { + /// Token IDs repeated uint32 ids = 1; - /// Prefill Logprobs + /// Logprobs repeated float logprobs = 2; - /// Prefill tokens + /// tokens repeated string texts = 3; -} - -message TopTokens { - /// Top Token IDs - repeated uint32 ids = 1; - /// Top Logprobs - repeated float logprobs = 2; - /// Top Token Texts - repeated string texts = 3; - /// If the tokens are special - repeated bool is_special = 6; + /// special + repeated bool is_special = 4; } message Generation { /// Request ID uint64 request_id = 1; /// Prefill tokens (optional) - PrefillTokens prefill_tokens = 2; - /// Token ID - uint32 token_id = 3; - /// Logprob - float token_logprob = 4; - /// Text - string token_text = 5; - /// Is it a special token - bool token_is_special = 6; + Tokens prefill_tokens = 2; + Tokens tokens = 3; /// Complete generated text - optional GeneratedText generated_text = 7; + optional GeneratedText generated_text = 4; /// Top tokens - TopTokens top_tokens = 8; + repeated Tokens top_tokens = 5; } message FilterBatchRequest { diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 341e70fd..1560f19c 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -1,6 +1,6 @@ /// Single shard Client -use crate::pb::generate::v1::text_generation_service_client::TextGenerationServiceClient; -use crate::pb::generate::v1::*; +use crate::pb::generate::v2::text_generation_service_client::TextGenerationServiceClient; +use crate::pb::generate::v2::*; use crate::Result; use grpc_metadata::InjectTelemetryContext; use std::cmp::min; diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index f334be21..c38b931b 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -6,11 +6,11 @@ mod pb; mod sharded_client; pub use client::Client; -pub use pb::generate::v1::HealthResponse; -pub use pb::generate::v1::InfoResponse as ShardInfo; -pub use pb::generate::v1::{ +pub use pb::generate::v2::HealthResponse; +pub use pb::generate::v2::InfoResponse as ShardInfo; +pub use pb::generate::v2::{ Batch, CachedBatch, FinishReason, GeneratedText, Generation, NextTokenChooserParameters, - PrefillTokens, Request, StoppingCriteriaParameters, + Request, StoppingCriteriaParameters, Tokens, }; pub use sharded_client::ShardedClient; use thiserror::Error; diff --git a/router/src/infer.rs b/router/src/infer.rs index aa6dc664..2e199ce2 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -9,7 +9,7 @@ use std::sync::{ Arc, }; use text_generation_client::{ - Batch, CachedBatch, ClientError, GeneratedText, Generation, PrefillTokens, ShardedClient, + Batch, CachedBatch, ClientError, GeneratedText, Generation, ShardedClient, Tokens, }; use thiserror::Error; use tokio::sync::mpsc::error::SendError; @@ -50,10 +50,11 @@ impl Infer { max_concurrent_requests: usize, requires_padding: bool, window_size: Option, + speculate: u32, generation_health: Arc, ) -> Self { // Infer shared state - let queue = Queue::new(requires_padding, 16, window_size); + let queue = Queue::new(requires_padding, 16, window_size, speculate); let shared = Arc::new(Shared { batching_task: Notify::new(), }); @@ -523,50 +524,63 @@ fn send_responses( } // Create last Token - let token = Token { - id: generation.token_id, - text: generation.token_text, - logprob: generation.token_logprob, - special: generation.token_is_special, - }; - - // generation.top_tokens - - let mut top_tokens = Vec::new(); - if let Some(top_tokens_) = generation.top_tokens { - top_tokens.extend( + let tokens_ = generation.tokens.expect("Non empty tokens in generation"); + let n = tokens_.ids.len(); + metrics::histogram!("tgi_request_skipped_tokens", (n - 1) as f64); + let mut iterator = tokens_ + .ids + .into_iter() + .zip(tokens_.logprobs.into_iter()) + .zip(tokens_.texts.into_iter()) + .zip(tokens_.is_special.into_iter()) + .enumerate() + .peekable(); + while let Some((i, (((id, logprob), text), special))) = iterator.next() { + let token = Token { + id, + text, + logprob, + special, + }; + let top_tokens = if let Some(top_tokens_) = generation.top_tokens.get(i) { top_tokens_ .ids - .into_iter() - .zip(top_tokens_.logprobs.into_iter()) - .zip(top_tokens_.texts.into_iter()) - .zip(top_tokens_.is_special.into_iter()) - .map(|(((id, logprob), text), special)| Token { + .iter() + .zip(top_tokens_.logprobs.iter()) + .zip(top_tokens_.texts.iter()) + .zip(top_tokens_.is_special.iter()) + .map(|(((&id, &logprob), text), &special)| Token { id, - text, + text: text.to_string(), logprob, special, - }), - ) + }) + .collect() + } else { + vec![] + }; + match (&generation.generated_text, iterator.peek()) { + (Some(generated_text), None) => { + // Generation has ended + stopped = true; + // Send message + entry.response_tx.send(Ok(InferStreamResponse::End { + token, + top_tokens, + generated_text: generated_text.clone(), + queued: entry.queue_time, + start: entry.batch_time.unwrap(), + }))?; + } + _ => { + // Send message + entry + .response_tx + .send(Ok(InferStreamResponse::Intermediate { token, top_tokens }))?; + } + } } - if let Some(generated_text) = generation.generated_text { - // Generation has ended - stopped = true; - // Send message - entry.response_tx.send(Ok(InferStreamResponse::End { - token, - top_tokens, - generated_text, - queued: entry.queue_time, - start: entry.batch_time.unwrap(), - }))?; - } else { - // Send message - entry - .response_tx - .send(Ok(InferStreamResponse::Intermediate { token, top_tokens }))?; - } Ok(stopped) } @@ -591,7 +605,7 @@ fn send_errors(error: ClientError, entries: &mut IntMap) { #[derive(Debug)] pub(crate) enum InferStreamResponse { // Optional first message - Prefill(PrefillTokens), + Prefill(Tokens), // Intermediate messages Intermediate { token: Token, diff --git a/router/src/queue.rs b/router/src/queue.rs index bbb8db0e..106cacc4 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -34,7 +34,12 @@ pub(crate) struct Queue { } impl Queue { - pub(crate) fn new(requires_padding: bool, block_size: u32, window_size: Option) -> Self { + pub(crate) fn new( + requires_padding: bool, + block_size: u32, + window_size: Option, + speculate: u32, + ) -> Self { // Create channel let (queue_sender, queue_receiver) = mpsc::unbounded_channel(); @@ -43,6 +48,7 @@ impl Queue { requires_padding, block_size, window_size, + speculate, queue_receiver, )); @@ -91,9 +97,10 @@ async fn queue_task( requires_padding: bool, block_size: u32, window_size: Option, + speculate: u32, mut receiver: mpsc::UnboundedReceiver, ) { - let mut state = State::new(requires_padding, block_size, window_size); + let mut state = State::new(requires_padding, block_size, window_size, speculate); while let Some(cmd) = receiver.recv().await { match cmd { @@ -136,10 +143,18 @@ struct State { /// Sliding window window_size: Option, + + /// Speculation amount + speculate: u32, } impl State { - fn new(requires_padding: bool, block_size: u32, window_size: Option) -> Self { + fn new( + requires_padding: bool, + block_size: u32, + window_size: Option, + speculate: u32, + ) -> Self { Self { entries: VecDeque::with_capacity(128), next_id: 0, @@ -147,6 +162,7 @@ impl State { requires_padding, block_size, window_size, + speculate, } } @@ -229,7 +245,7 @@ impl State { } if prefill_tokens > prefill_token_budget - || (prefill_tokens + decode_tokens) > token_budget + || (prefill_tokens + decode_tokens + self.speculate) > token_budget { // Entry is over budget // Add it back to the front @@ -359,7 +375,7 @@ mod tests { #[test] fn test_append() { - let mut state = State::new(false, 1, None); + let mut state = State::new(false, 1, None, 0); let (entry, _guard) = default_entry(); assert_eq!(state.next_id, 0); @@ -375,7 +391,7 @@ mod tests { #[test] fn test_next_batch_empty() { - let mut state = State::new(false, 1, None); + let mut state = State::new(false, 1, None, 0); assert!(state.next_batch(None, 1, 1).is_none()); assert!(state.next_batch(Some(1), 1, 1).is_none()); @@ -383,7 +399,7 @@ mod tests { #[test] fn test_next_batch_min_size() { - let mut state = State::new(false, 1, None); + let mut state = State::new(false, 1, None, 0); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); state.append(entry1); @@ -415,7 +431,7 @@ mod tests { #[test] fn test_next_batch_token_budget() { - let mut state = State::new(false, 1, None); + let mut state = State::new(false, 1, None, 0); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); state.append(entry1); @@ -448,14 +464,14 @@ mod tests { #[tokio::test] async fn test_queue_append() { - let queue = Queue::new(false, 1, None); + let queue = Queue::new(false, 1, None, 0); let (entry, _guard) = default_entry(); queue.append(entry); } #[tokio::test] async fn test_queue_next_batch_empty() { - let queue = Queue::new(false, 1, None); + let queue = Queue::new(false, 1, None, 0); assert!(queue.next_batch(None, 1, 1).await.is_none()); assert!(queue.next_batch(Some(1), 1, 1).await.is_none()); @@ -463,7 +479,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_min_size() { - let queue = Queue::new(false, 1, None); + let queue = Queue::new(false, 1, None, 0); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); queue.append(entry1); @@ -496,7 +512,7 @@ mod tests { #[tokio::test] async fn test_queue_next_batch_token_budget() { - let queue = Queue::new(false, 1, None); + let queue = Queue::new(false, 1, None, 0); let (entry1, _guard1) = default_entry(); let (entry2, _guard2) = default_entry(); queue.append(entry1); @@ -519,9 +535,28 @@ mod tests { assert_eq!(batch.size, 2); } + #[tokio::test] + async fn test_queue_next_batch_token_speculate() { + let queue = Queue::new(false, 1, None, 2); + let (entry1, _guard1) = default_entry(); + let (entry2, _guard2) = default_entry(); + queue.append(entry1); + queue.append(entry2); + + // Budget of 1 is not enough + assert!(queue.next_batch(None, 1, 1).await.is_none()); + + let (entries, batch, _) = queue.next_batch(None, 6, 6).await.unwrap(); + assert_eq!(entries.len(), 2); + assert!(entries.contains_key(&0)); + assert!(entries.contains_key(&1)); + assert_eq!(batch.id, 0); + assert_eq!(batch.size, 2); + } + #[tokio::test] async fn test_queue_next_batch_dropped_receiver() { - let queue = Queue::new(false, 1, None); + let queue = Queue::new(false, 1, None, 0); let (entry, _) = default_entry(); queue.append(entry); diff --git a/router/src/server.rs b/router/src/server.rs index f254afd8..5f41fd5e 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -596,6 +596,7 @@ pub async fn run( max_concurrent_requests, shard_info.requires_padding, shard_info.window_size, + shard_info.speculate, generation_health, ); diff --git a/server/Makefile-vllm b/server/Makefile-vllm index ddb648ea..c9c1d520 100644 --- a/server/Makefile-vllm +++ b/server/Makefile-vllm @@ -1,22 +1,25 @@ -build-vllm-cuda: REPOSITORY=https://github.com/vllm-project/vllm.git -build-vllm-cuda: VLLM_COMMIT=f8a1e39fae05ca610be8d5a78be9d40f5274e5fc -build-vllm-cuda: BRANCH=main -build-vllm-cuda: build-vllm +vllm-cuda: + # Clone vllm + pip install -U ninja packaging --no-cache-dir + git clone https://github.com/vllm-project/vllm.git vllm + +build-vllm-cuda: vllm-cuda + cd vllm && git fetch && git checkout f8a1e39fae05ca610be8d5a78be9d40f5274e5fc + cd vllm && python setup.py build -build-vllm-rocm: REPOSITORY=https://github.com/fxmarty/vllm-public.git -build-vllm-rocm: VLLM_COMMIT=ad9b7c4095ef54419a0533d254f2ad84bd2dfcae -build-vllm-rocm: BRANCH=rotary-no-positions-split-cos-sin -build-vllm-rocm: build-vllm +install-vllm-cuda: build-vllm-cuda + pip uninstall vllm -y || true + cd vllm && python setup.py install -vllm: +vllm-rocm: # Clone vllm pip install -U ninja packaging --no-cache-dir - git clone --single-branch --branch $(BRANCH) $(REPOSITORY) vllm + git clone https://github.com/fxmarty/vllm-public.git vllm -build-vllm: vllm - cd vllm && git fetch && git checkout $(VLLM_COMMIT) +build-vllm-rocm: vllm-rocm + cd vllm && git fetch && git checkout ad9b7c4095ef54419a0533d254f2ad84bd2dfcae cd vllm && python setup.py build -install-vllm: build-vllm +install-vllm-rocm: build-vllm-rocm pip uninstall vllm -y || true cd vllm && python setup.py install diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 71013cb6..1990ef8b 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -133,8 +133,8 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): ) assert all([generation.generated_text is None for generation in generations]) assert all([len(generation.prefill_tokens) == 1 for generation in generations]) - assert all([generation.token_id.item() == 10264 for generation in generations]) - assert all([generation.token_text == "Test" for generation in generations]) + assert all([token_id.item() == 10264 for generation in generations for token_id in generation.tokens.token_ids]) + assert all([token_text == "Test" for generation in generations for token_text in generation.tokens.texts]) assert generations[0].request_id == 0 diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 0f9dab2c..f105ce6f 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -129,8 +129,8 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): ) assert all([generation.generated_text is None for generation in generations]) assert all([len(generation.prefill_tokens) == 1 for generation in generations]) - assert all([generation.token_id.item() == 13 for generation in generations]) - assert all([generation.token_text == "." for generation in generations]) + assert all([token_id.item() == 13 for generation in generations for token_id in generation.tokens.token_ids]) + assert all([token_text == "." for generation in generations for token_text in generation.tokens.texts]) assert generations[0].request_id == 0 diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 299340f8..d553067e 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -151,8 +151,8 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) ) assert all([generation.generated_text is None for generation in generations]) assert all([len(generation.prefill_tokens) == 1 for generation in generations]) - assert all([generation.token_id.item() == 259 for generation in generations]) - assert all([generation.token_text == " " for generation in generations]) + assert all([token_id.item() == 259 for generation in generations for token_id in generation.tokens.token_ids]) + assert all([token_text == " " for generation in generations for token_text in generation.tokens.texts]) assert generations[0].request_id == 0 diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 3abe86af..cb151173 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -32,6 +32,7 @@ def serve( revision: Optional[str] = None, sharded: bool = False, quantize: Optional[Quantization] = None, + speculate: Optional[int] = None, dtype: Optional[Dtype] = None, trust_remote_code: bool = False, uds_path: Path = "/tmp/text-generation-server", @@ -81,7 +82,7 @@ def serve( "Only 1 can be set between `dtype` and `quantize`, as they both decide how goes the final model." ) server.serve( - model_id, revision, sharded, quantize, dtype, trust_remote_code, uds_path + model_id, revision, sharded, quantize, speculate, dtype, trust_remote_code, uds_path ) @@ -116,7 +117,7 @@ def download_weights( logger.info("Files are already present on the host. " "Skipping download.") return # Local files not found - except (utils.LocalEntryNotFoundError, FileNotFoundError): + except (utils.LocalEntryNotFoundError, FileNotFoundError, utils.EntryNotFoundError): pass is_local_model = (Path(model_id).exists() and Path(model_id).is_dir()) or os.getenv( @@ -137,6 +138,29 @@ def download_weights( except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError): pass + try: + import json + medusa_head = hf_hub_download(model_id, revision=revision, filename="medusa_lm_head.pt") + if auto_convert: + medusa_sf = Path(medusa_head[:-len(".pt")] + ".safetensors") + if not medusa_sf.exists(): + utils.convert_files([Path(medusa_head)], [medusa_sf], []) + medusa_config = hf_hub_download(model_id, revision=revision, filename="config.json") + with open(medusa_config, "r") as f: + config = json.load(f) + + model_id = config["base_model_name_or_path"] + revision = "main" + try: + utils.weight_files(model_id, revision, extension) + logger.info(f"Files for parent {model_id} are already present on the host. " "Skipping download.") + return + # Local files not found + except (utils.LocalEntryNotFoundError, FileNotFoundError, utils.EntryNotFoundError): + pass + except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError): + pass + # Try to download weights from the hub try: filenames = utils.weight_hub_files(model_id, revision, extension) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index ab3b25b7..27e3897d 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -6,6 +6,7 @@ from transformers.models.auto import modeling_auto from typing import Optional +from text_generation_server.utils.speculate import get_speculate, set_speculate from text_generation_server.models.model import Model from text_generation_server.models.causal_lm import CausalLM from text_generation_server.models.flash_causal_lm import FlashCausalLM @@ -77,12 +78,12 @@ if MISTRAL: __all__.append(FlashMistral) - def get_model( model_id: str, revision: Optional[str], sharded: bool, quantize: Optional[str], + speculate: Optional[int], dtype: Optional[str], trust_remote_code: bool, ) -> Model: @@ -97,6 +98,11 @@ def get_model( else: raise RuntimeError(f"Unknown dtype {dtype}") + if speculate is not None: + set_speculate(speculate) + else: + set_speculate(0) + if "facebook/galactica" in model_id: return GalacticaSharded( model_id, @@ -131,6 +137,33 @@ def get_model( config_dict, _ = PretrainedConfig.get_config_dict( model_id, revision=revision, trust_remote_code=trust_remote_code ) + + use_medusa = None + if "medusa_num_heads" in config_dict: + use_medusa = model_id + medusa_config = config_dict + model_id = config_dict["base_model_name_or_path"] + revision = "main" + speculate_medusa = config_dict["medusa_num_heads"] + if speculate is not None: + if speculate > speculate_medusa: + raise RuntimeError("Speculate is set to `{speculate}` but this medusa models only has `{speculate_medusa}` heads, please make them match") + else: + set_speculate(speculate) + else: + set_speculate(speculate_medusa) + + config_dict, _ = PretrainedConfig.get_config_dict( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) + method = "medusa" + else: + method = "n-gram" + + speculate = get_speculate() + if speculate > 0: + logger.info(f"Using speculation {method} with {speculate} input ids.") + model_type = config_dict["model_type"] if model_type == "gpt_bigcode": @@ -206,6 +239,7 @@ def get_model( quantize=quantize, dtype=dtype, trust_remote_code=trust_remote_code, + use_medusa=use_medusa ) elif sharded: raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format("Sharded Llama")) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 8056a8ec..c571a022 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -10,10 +10,9 @@ from text_generation_server.models import Model from text_generation_server.models.types import ( Batch, - PrefillTokens, + Tokens, Generation, GeneratedText, - TopTokens, ) from text_generation_server.pb import generate_pb2 from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -676,8 +675,8 @@ def generate_token( clean_up_tokenization_spaces=False, skip_special_tokens=False, ) - prefill_tokens = PrefillTokens( - prefill_token_ids, prefill_logprobs, prefill_texts + prefill_tokens = Tokens( + prefill_token_ids, prefill_logprobs, prefill_texts, is_special=[] ) else: prefill_tokens = None @@ -691,7 +690,7 @@ def generate_token( special_toptokens = [ token_id in self.all_special_ids for token_id in top_token_ids ] - top_tokens = TopTokens( + top_tokens = Tokens( top_token_ids, top_token_logprobs, toptoken_texts, @@ -703,10 +702,12 @@ def generate_token( generation = Generation( request.id, prefill_tokens, - next_token_id_squeezed, - next_token_logprob, - next_token_text, - next_token_id_squeezed.item() in self.all_special_ids, + Tokens( + [next_token_id_squeezed], + [next_token_logprob], + [next_token_text], + [next_token_id_squeezed.item() in self.all_special_ids], + ), generated_text, top_tokens, ) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index f1a4854f..79344ea1 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -11,13 +11,13 @@ from transformers import PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type, Union, Dict -from text_generation_server.models import Model +from text_generation_server.models import Model +from text_generation_server.utils.speculate import get_speculate from text_generation_server.models.types import ( Batch, - PrefillTokens, + Tokens, Generation, GeneratedText, - TopTokens, ) from text_generation_server.models.cache_manager import ( get_cache_manager, @@ -41,6 +41,7 @@ class FlashCausalLMBatch(Batch): # Decoder values input_ids: torch.Tensor position_ids: torch.Tensor + speculative_ids: torch.Tensor # Flash Attention values @@ -120,6 +121,7 @@ def from_pb( )["input_ids"] position_ids = [] + speculative_ids = [] cu_seqlen_prefill = [0] needed_blocks_slots = [] start_slots = [] @@ -163,6 +165,8 @@ def from_pb( input_length = len(tokenized_input) input_lengths.append(input_length) + + prefix_offsets.append(input_length - 5) read_offsets.append(input_length) @@ -186,7 +190,8 @@ def from_pb( # Paged attention # Remove one as the first token des not have a past - total_tokens = input_length + max_new_tokens - 1 + speculative_length = get_speculate() + total_tokens = input_length + max_new_tokens - 1 + speculative_length needed_blocks = math.ceil(total_tokens / BLOCK_SIZE) blocks += needed_blocks needed_blocks_slots.append((needed_blocks, total_tokens)) @@ -224,7 +229,7 @@ def from_pb( cumulative_max_length += total_tokens max_seqlen = max(max_seqlen, input_length) max_blocks = max(max_blocks, needed_blocks) - max_length = max(max_length, input_length + max_new_tokens) + max_length = max(max_length, input_length + max_new_tokens + speculative_length) next_token_chooser = HeterogeneousNextTokenChooser.from_pb( next_token_chooser_parameters, dtype, device @@ -255,7 +260,6 @@ def from_pb( cu_seqlen_prefill = torch.tensor( cu_seqlen_prefill, device=device, dtype=torch.int32 ) - position_ids = position_ids.to(device) slot_indices = slot_indices.to(device) input_ids = torch.tensor(input_ids, dtype=torch.int64, device=device) @@ -309,6 +313,7 @@ def from_pb( top_n_tokens_tensor=top_n_tokens_tensor, blocks=blocks, max_blocks=max_blocks, + speculative_ids=None, ) @tracer.start_as_current_span("filter") @@ -419,6 +424,7 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": slots = self.slots[slot_filtering_indices] next_token_chooser = self.next_token_chooser.filter(indices) top_n_tokens_tensor = self.top_n_tokens_tensor[indices] + speculative_ids = self.speculative_ids[indices] if self.speculative_ids is not None else None start_slots = torch.tensor(start_slots, dtype=torch.int64) @@ -454,6 +460,7 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": top_n_tokens_tensor=top_n_tokens_tensor, blocks=blocks, max_blocks=max_blocks, + speculative_ids=speculative_ids, ) @classmethod @@ -473,6 +480,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch total_batch_size += len(b) total_slots += len(b.slots) blocks += b.blocks + speculative_length = b.speculative_ids.shape[1] if b.speculative_ids is not None else 0 max_blocks = max(max_blocks, b.max_blocks) max_seqlen = max(max_seqlen, b.max_seqlen) max_length = max( @@ -480,6 +488,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch max( input_length + stopping_criteria.max_new_tokens + + speculative_length - stopping_criteria.current_tokens for input_length, stopping_criteria in zip( b.input_lengths, b.stopping_criterias @@ -577,6 +586,8 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch device=batches[0].next_token_chooser.device, ) + speculative_ids = torch.cat([b.speculative_ids for b in batches], dim=0) if batches[0].speculative_ids is not None else None + # Needed to avoid dropping blocks when the batches will go out of scope for b in batches: b.block_tables = None @@ -611,6 +622,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch top_n_tokens_tensor=top_n_tokens_tensor, blocks=blocks, max_blocks=max_blocks, + speculative_ids=speculative_ids ) def __del__(self): @@ -714,16 +726,55 @@ def warmup(self, batch: FlashCausalLMBatch): def forward(self, batch: FlashCausalLMBatch) -> Tuple[torch.Tensor, torch.Tensor]: # Model Forward + if batch.speculative_ids is not None: + input_ids=batch.input_ids + position_ids=batch.position_ids + cu_seqlen_prefill=batch.cu_seqlen_prefill + kv_cache=get_cache_manager().kv_cache + block_tables=batch.block_tables_tensor + slots=batch.slots[batch.slot_indices] + input_lengths=batch.input_lengths_tensor + max_s=batch.max_seqlen + lm_head_indices=batch.prefill_head_indices + + speculative_ids = batch.speculative_ids + + B, speculative_length = speculative_ids.shape + new_length = speculative_length + 1 + new_input_ids = torch.cat([input_ids.unsqueeze(-1), speculative_ids], dim=1).reshape(-1) + arange = torch.arange(new_length, device=position_ids.device).unsqueeze(0) + arange_int = arange.to(dtype=torch.int32) + new_position_ids = (position_ids.unsqueeze(-1).expand(B, new_length) + arange).view(-1) + slots = (slots.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1) + input_lengths = (input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1) + + # Add Copy the block tables for all members + block_tables = block_tables.unsqueeze(1).expand(B, new_length, -1).reshape(B* new_length, -1).contiguous() + max_s = max_s + speculative_length + + input_ids = new_input_ids + position_ids = new_position_ids + else: + input_ids=batch.input_ids + position_ids=batch.position_ids + cu_seqlen_prefill=batch.cu_seqlen_prefill + kv_cache=get_cache_manager().kv_cache + block_tables=batch.block_tables_tensor + slots=batch.slots[batch.slot_indices] + input_lengths=batch.input_lengths_tensor + max_s=batch.max_seqlen + lm_head_indices=batch.prefill_head_indices + return self.model.forward( - input_ids=batch.input_ids, - position_ids=batch.position_ids, - cu_seqlen_prefill=batch.cu_seqlen_prefill, - kv_cache=get_cache_manager().kv_cache, - block_tables=batch.block_tables_tensor, - slots=batch.slots[batch.slot_indices], - input_lengths=batch.input_lengths_tensor, - max_s=batch.max_seqlen, - lm_head_indices=batch.prefill_head_indices, + input_ids=input_ids, + position_ids=position_ids, + cu_seqlen_prefill=cu_seqlen_prefill, + kv_cache=kv_cache, + block_tables=block_tables, + slots=slots, + input_lengths=input_lengths, + max_s=max_s, + lm_head_indices=lm_head_indices, ) @tracer.start_as_current_span("generate_token") @@ -752,21 +803,32 @@ def generate_token( del batch raise e + if isinstance(out, tuple): + out, speculative_logits = out + else: + speculative_logits = None + + if prefill: next_token_logits = ( out[batch.prefill_next_token_indices] if prefill_logprobs else out ) + if speculative_logits is not None: + speculative_logits = ( + speculative_logits[batch.prefill_next_token_indices] if prefill_logprobs else speculative_logits + ) else: next_token_logits = out - next_input_ids, next_token_logprobs, logprobs = batch.next_token_chooser( - batch.all_input_ids_tensor[:, : batch.max_seqlen], next_token_logits + next_input_ids, next_token_logprobs, logprobs, accepted_ids, speculative_ids = batch.next_token_chooser( + batch.all_input_ids_tensor[:, : batch.max_seqlen], next_token_logits, get_speculate(), batch.speculative_ids, speculative_logits ) batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( batch.top_n_tokens, batch.top_n_tokens_tensor, logprobs ) + speculative_length = 0 if speculative_ids is None else speculative_ids.shape[1] if prefill: if len(batch) > 1 and prefill_logprobs: # We create the prefill_tokens_indices tensor that will be used to gather prefill logprobs @@ -792,6 +854,7 @@ def generate_token( iterator = zip( batch.input_lengths, batch.all_input_ids, + accepted_ids ) # We do two for loops as the first one can run completely asynchronously from the GPU while for the second @@ -799,9 +862,11 @@ def generate_token( # It is faster if we delay this sync for the maximum amount of time # For each member of the batch + index = 0 for i, ( input_length, all_input_ids, + n_accepted_ids ) in enumerate(iterator): # Indexing metadata start_index = cumulative_length @@ -830,15 +895,18 @@ def generate_token( start_index + 1 : start_index + out_length ] - batch.all_input_ids_tensor[i, input_length] = next_input_ids[i] + for j in range(n_accepted_ids): + batch.all_input_ids_tensor[i, input_length + j] = next_input_ids[index] + index += 1 cumulative_length += input_length - # Set values in batch - batch.input_ids = next_input_ids - batch.position_ids = next_position_ids + 1 - batch.input_lengths_tensor += 1 - batch.slot_indices += 1 + + batch.input_ids = next_input_ids[accepted_ids.cumsum(dim=-1) - 1] + batch.speculative_ids = speculative_ids + batch.position_ids = next_position_ids + accepted_ids + batch.input_lengths_tensor += accepted_ids + batch.slot_indices += accepted_ids if prefill and prefill_logprobs: # Get prefill logprobs @@ -851,7 +919,7 @@ def generate_token( # GPU <-> CPU sync next_token_logprobs = next_token_logprobs.tolist() - next_token_ids = batch.input_ids.tolist() + next_token_ids = next_input_ids.tolist() # Zipped iterator iterator = zip( @@ -864,13 +932,13 @@ def generate_token( batch.next_token_chooser.do_sample, batch.next_token_chooser.seeds, batch.top_n_tokens, - next_token_ids, - next_token_logprobs, + accepted_ids, batch_top_token_ids, batch_top_token_logprobs, ) # For each member of the batch + index = 0 for i, ( request, input_length, @@ -881,29 +949,43 @@ def generate_token( do_sample, seed, top_n_tokens, - next_token_id, - next_token_logprob, + n_accepted_ids, top_token_ids, top_token_logprobs, ) in enumerate(iterator): # Append next token to all tokens - all_input_ids.append(next_token_id) + next_token_texts = [] + left = 0 + before = stopping_criteria.current_tokens + + current_stopped = False + for j in range(index, index + n_accepted_ids): + # Generated token + next_token_id = next_token_ids[j] + all_input_ids.append(next_token_id) + next_token_text, prefix_offset, read_offset = self.decode_token( + all_input_ids, + prefix_offset, + read_offset, + ) + next_token_texts.append(next_token_text) - # Generated token - next_token_text, prefix_offset, read_offset = self.decode_token( - all_input_ids, - prefix_offset, - read_offset, - ) + stop, reason = stopping_criteria( + next_token_id, + next_token_text, + ) - # Evaluate stopping criteria - stop, reason = stopping_criteria( - next_token_id, - next_token_text, - ) + if stop: + left = index + n_accepted_ids - j - 1 + current_stopped = True + break + else: + current_stopped = False + stopped = stopped and current_stopped - if not stop: - stopped = False + _next_token_ids = next_token_ids[index: index+n_accepted_ids - left] + _next_token_logprobs = next_token_logprobs[index: index+n_accepted_ids - left] + index += n_accepted_ids # Shard generations # All generations will be appended in the rust sharded client @@ -943,8 +1025,9 @@ def generate_token( clean_up_tokenization_spaces=False, skip_special_tokens=False, ) - prefill_tokens = PrefillTokens( - prefill_token_ids, request_prefill_logprobs, prefill_texts + + prefill_tokens = Tokens( + prefill_token_ids, request_prefill_logprobs, prefill_texts, is_special = [] ) else: prefill_tokens = None @@ -958,7 +1041,7 @@ def generate_token( special_toptokens = [ token_id in self.all_special_ids for token_id in top_token_ids ] - top_tokens = TopTokens( + top_tokens = Tokens( top_token_ids, top_token_logprobs, toptoken_texts, @@ -970,10 +1053,12 @@ def generate_token( generation = Generation( request.id, prefill_tokens, - next_token_id, - next_token_logprob, - next_token_text, - next_token_id in self.all_special_ids, + Tokens( + _next_token_ids, + _next_token_logprobs, + next_token_texts, + [nid in self.all_special_ids for nid in _next_token_ids], + ), generated_text, top_tokens, ) @@ -981,7 +1066,9 @@ def generate_token( generations.append(generation) # Update values - batch.input_lengths[i] = input_length + 1 + batch.input_lengths[i] = input_length + n_accepted_ids.item() + if batch.input_lengths[i] > batch.max_seqlen: + batch.max_seqlen = batch.input_lengths[i] batch.prefix_offsets[i] = prefix_offset batch.read_offsets[i] = read_offset batch.all_input_ids[i] = all_input_ids @@ -994,6 +1081,5 @@ def generate_token( batch.prefill_cu_outlens = None batch.prefill_head_indices = None batch.prefill_next_token_indices = None - batch.max_seqlen = batch.max_seqlen + 1 return generations, batch diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index d2ed0b15..3a84b1b6 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -28,6 +28,7 @@ def __init__( quantize: Optional[str] = None, dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, + use_medusa: Optional[str] = None, ): self.process_group, rank, world_size = initialize_torch_distributed() if torch.cuda.is_available(): @@ -66,6 +67,18 @@ def __init__( weights._set_gptq_params(model_id) model = FlashLlamaForCausalLM(config, weights) + if use_medusa: + from text_generation_server.utils.medusa import MedusaModel + from huggingface_hub import hf_hub_download + import json + medusa_config = hf_hub_download(use_medusa, revision=revision, filename="config.json") + with open(medusa_config, "r") as f: + config = json.load(f) + medusa_head = hf_hub_download(use_medusa, revision=revision, filename="medusa_lm_head.pt") + medusa_sf = medusa_head[:-len(".pt")] + ".safetensors" + weights = Weights([medusa_sf], device, dtype, process_group=self.process_group) + lm_head = model.lm_head + model.lm_head = MedusaModel(config, weights, lm_head) torch.distributed.barrier(group=self.process_group) super(FlashLlama, self).__init__( diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py index 919e4625..e103d9fc 100644 --- a/server/text_generation_server/models/flash_mistral.py +++ b/server/text_generation_server/models/flash_mistral.py @@ -21,6 +21,7 @@ FlashMistralForCausalLM, MistralConfig, ) +from text_generation_server.utils.speculate import get_speculate from text_generation_server.utils import ( initialize_torch_distributed, weight_files, @@ -132,7 +133,8 @@ def from_pb( # Paged attention # Remove one as the first token des not have a past - total_tokens = input_length + max_new_tokens - 1 + speculative_length = get_speculate() + total_tokens = input_length + max_new_tokens - 1 + speculative_length # Needed blocks can not go over SLIDING_WINDOW_BLOCKS needed_blocks = min( @@ -183,7 +185,7 @@ def from_pb( cumulative_max_length += total_tokens max_seqlen = max(max_seqlen, input_length) max_blocks = max(max_blocks, needed_blocks) - max_length = max(max_length, input_length + max_new_tokens) + max_length = max(max_length, input_length + max_new_tokens + speculative_length) next_token_chooser = HeterogeneousNextTokenChooser.from_pb( next_token_chooser_parameters, dtype, device @@ -272,6 +274,7 @@ def from_pb( blocks=blocks, max_blocks=max_blocks, prefill_cache_indices=prefill_cache_indices, + speculative_ids=None ) @@ -340,17 +343,55 @@ def batch_type(self) -> Type[FlashMistralBatch]: def forward(self, batch: FlashMistralBatch) -> Tuple[torch.Tensor, torch.Tensor]: # Model Forward + if batch.speculative_ids is not None: + input_ids=batch.input_ids + position_ids=batch.position_ids + cu_seqlen_prefill=batch.cu_seqlen_prefill + kv_cache=get_cache_manager().kv_cache + block_tables=batch.block_tables_tensor + slots=batch.slots[batch.slot_indices] + input_lengths=batch.input_lengths_tensor + max_s=batch.max_seqlen + lm_head_indices=batch.prefill_head_indices + + speculative_ids = batch.speculative_ids + + B, speculative_length = speculative_ids.shape + new_length = speculative_length + 1 + new_input_ids = torch.cat([input_ids.unsqueeze(-1), speculative_ids], dim=1).reshape(-1) + arange = torch.arange(new_length, device=position_ids.device).unsqueeze(0) + arange_int = arange.to(dtype=torch.int32) + new_position_ids = (position_ids.unsqueeze(-1).expand(B, new_length) + arange).view(-1) + slots = (slots.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1) + input_lengths = (input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1) + + # Add Copy the block tables for all members + block_tables = block_tables.unsqueeze(1).expand(B, new_length, -1).reshape(B* new_length, -1).contiguous() + max_s = max_s + speculative_length + + input_ids = new_input_ids + position_ids = new_position_ids + else: + input_ids=batch.input_ids + position_ids=batch.position_ids + cu_seqlen_prefill=batch.cu_seqlen_prefill + kv_cache=get_cache_manager().kv_cache + block_tables=batch.block_tables_tensor + slots=batch.slots[batch.slot_indices] + input_lengths=batch.input_lengths_tensor + max_s=batch.max_seqlen + lm_head_indices=batch.prefill_head_indices logits = self.model.forward( - input_ids=batch.input_ids, - position_ids=batch.position_ids, - cu_seqlen_prefill=batch.cu_seqlen_prefill, - kv_cache=get_cache_manager().kv_cache, - block_tables=batch.block_tables_tensor, - slots=batch.slots[batch.slot_indices], - input_lengths=batch.input_lengths_tensor, - max_s=batch.max_seqlen, + input_ids=input_ids, + position_ids=position_ids, + cu_seqlen_prefill=cu_seqlen_prefill, + kv_cache=kv_cache, + block_tables=block_tables, + slots=slots, + input_lengths=input_lengths, + max_s=max_s, prefill_cache_indices=batch.prefill_cache_indices, - lm_head_indices=batch.prefill_head_indices, + lm_head_indices=lm_head_indices, ) if batch.prefill_cache_indices is not None: batch.prefill_cache_indices = None diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py index dcad1fa9..2f4bb139 100644 --- a/server/text_generation_server/models/idefics_causal_lm.py +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -20,7 +20,7 @@ from text_generation_server.models import Model from text_generation_server.models.types import ( Batch, - PrefillTokens, + Tokens, Generation, GeneratedText, ) @@ -791,8 +791,8 @@ def generate_token( clean_up_tokenization_spaces=False, skip_special_tokens=False, ) - prefill_tokens = PrefillTokens( - prefill_token_ids, prefill_logprobs, prefill_texts + prefill_tokens = Tokens( + prefill_token_ids, prefill_logprobs, prefill_texts, is_special=[] ) else: prefill_tokens = None @@ -802,10 +802,12 @@ def generate_token( generation = Generation( request.id, prefill_tokens, - next_token_id_squeezed, - next_token_logprob, - next_token_text, - next_token_id_squeezed.item() in self.all_special_ids, + Tokens( + [next_token_id_squeezed], + [next_token_logprob], + [next_token_text], + [next_token_id_squeezed.item() in self.all_special_ids], + ), generated_text, top_tokens, ) diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 17d2ea9b..8552960d 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -6,6 +6,7 @@ from transformers import PreTrainedTokenizerBase, PretrainedConfig from text_generation_server.models.types import Batch, Generation +from text_generation_server.utils.speculate import get_speculate from text_generation_server.pb.generate_pb2 import InfoResponse B = TypeVar("B", bound=Batch) @@ -22,6 +23,7 @@ def __init__( rank: int = 0, world_size: int = 1, sliding_window: Optional[int] = None, + speculate: Optional[int] = None, ): self.model = model.eval() self.tokenizer = tokenizer @@ -33,6 +35,10 @@ def __init__( self.world_size = world_size self.sliding_window = sliding_window + if speculate is None: + speculate = get_speculate() + self.speculate = speculate + self.has_position_ids = ( inspect.signature(model.forward).parameters.get("position_ids", None) is not None @@ -50,6 +56,7 @@ def info(self) -> InfoResponse: dtype=str(self.dtype), device_type=self.device.type, window_size=self.sliding_window, + speculate=self.speculate ) @property diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index d4d3cd19..279b5505 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -11,8 +11,7 @@ GeneratedText, Batch, Generation, - PrefillTokens, - TopTokens, + Tokens, ) from text_generation_server.pb import generate_pb2 from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling @@ -733,10 +732,11 @@ def generate_token( # Prefill if stopping_criteria.current_tokens == 1 and request.prefill_logprobs: - prefill_tokens = PrefillTokens( + prefill_tokens = Tokens( [self.tokenizer.bos_token_id], [float("nan")], [self.tokenizer.bos_token], + [False] ) else: prefill_tokens = None @@ -750,7 +750,7 @@ def generate_token( special_toptokens = [ token_id in self.all_special_ids for token_id in top_token_ids ] - top_tokens = TopTokens( + top_tokens = Tokens( top_token_ids, top_token_logprobs, toptoken_texts, @@ -762,10 +762,12 @@ def generate_token( generation = Generation( request.id, prefill_tokens, - next_token_id_squeezed, - next_token_logprob, - next_token_text, - next_token_id_squeezed.item() in self.all_special_ids, + Tokens( + [next_token_id_squeezed], + [next_token_logprob], + [next_token_text], + [next_token_id_squeezed.item() in self.all_special_ids], + ), generated_text, top_tokens, ) diff --git a/server/text_generation_server/models/types.py b/server/text_generation_server/models/types.py index 0e27680d..87c03d63 100644 --- a/server/text_generation_server/models/types.py +++ b/server/text_generation_server/models/types.py @@ -58,33 +58,15 @@ def to_pb(self) -> generate_pb2.GeneratedText: @dataclass -class PrefillTokens: - token_ids: List[int] - logprobs: List[float] - texts: List[str] - - def to_pb(self) -> generate_pb2.PrefillTokens: - return generate_pb2.PrefillTokens( - ids=self.token_ids, logprobs=self.logprobs, texts=self.texts - ) - - def __len__(self): - return len(self.token_ids) - - -@dataclass -class TopTokens: +class Tokens: token_ids: List[int] logprobs: List[float] texts: List[str] is_special: List[bool] - def to_pb(self) -> generate_pb2.TopTokens: - return generate_pb2.TopTokens( - ids=self.token_ids, - logprobs=self.logprobs, - texts=self.texts, - is_special=self.is_special, + def to_pb(self) -> generate_pb2.Tokens: + return generate_pb2.Tokens( + ids=self.token_ids, logprobs=self.logprobs, texts=self.texts, is_special=self.is_special ) def __len__(self): @@ -94,14 +76,11 @@ def __len__(self): @dataclass class Generation: request_id: int - prefill_tokens: Optional[PrefillTokens] - token_id: int - token_logprob: float - token_text: str - token_is_special: bool + prefill_tokens: Optional[Tokens] + tokens: Tokens generated_text: Optional[GeneratedText] # Optional for now, since it's not yet supported for every model. - top_tokens: Optional[TopTokens] + top_tokens: Optional[List[Tokens]] def to_pb(self) -> generate_pb2.Generation: return generate_pb2.Generation( @@ -109,10 +88,7 @@ def to_pb(self) -> generate_pb2.Generation: prefill_tokens=self.prefill_tokens.to_pb() if self.prefill_tokens is not None else None, - token_id=self.token_id, - token_logprob=self.token_logprob, - token_text=self.token_text, - token_is_special=self.token_is_special, + tokens=self.tokens.to_pb(), generated_text=self.generated_text.to_pb() if self.generated_text is not None else None, diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index fa831682..ebe066e3 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -132,6 +132,7 @@ def serve( revision: Optional[str], sharded: bool, quantize: Optional[str], + speculate: Optional[int], dtype: Optional[str], trust_remote_code: bool, uds_path: Path, @@ -141,6 +142,7 @@ async def serve_inner( revision: Optional[str], sharded: bool = False, quantize: Optional[str] = None, + speculate: Optional[int] = None, dtype: Optional[str] = None, trust_remote_code: bool = False, ): @@ -157,7 +159,7 @@ async def serve_inner( try: model = get_model( - model_id, revision, sharded, quantize, dtype, trust_remote_code + model_id, revision, sharded, quantize, speculate, dtype, trust_remote_code ) except Exception: logger.exception("Error when initializing model") @@ -205,5 +207,5 @@ async def serve_inner( await server.stop(0) asyncio.run( - serve_inner(model_id, revision, sharded, quantize, dtype, trust_remote_code) + serve_inner(model_id, revision, sharded, quantize, speculate, dtype, trust_remote_code) ) diff --git a/server/text_generation_server/utils/medusa.py b/server/text_generation_server/utils/medusa.py new file mode 100644 index 00000000..029de122 --- /dev/null +++ b/server/text_generation_server/utils/medusa.py @@ -0,0 +1,51 @@ +import torch +from dataclasses import dataclass +from text_generation_server.utils.layers import TensorParallelHead, FastLinear + +@dataclass +class Output: + logits: torch.FloatTensor = None + speculative_logits: torch.FloatTensor = None + + +class ResBlock(torch.nn.Module): + def __init__(self, config, prefix, weights): + super().__init__() + self.linear = FastLinear.load(config, prefix=f"{prefix}.linear", weights=weights, bias=True) + self.act = torch.nn.SiLU() + + def forward(self, x): + return x + self.act(self.linear(x)) + + +class MedusaModel(torch.nn.Module): + def __init__( + self, + config, + weights, + lm_head + ): + super().__init__() + self.heads = torch.nn.ModuleList( + [MedusaHead(config, prefix=f"{i}", weights=weights) for i in range(config["medusa_num_heads"])] + ) + self.lm_head = lm_head + + def forward(self, x): + logits = self.lm_head(x) + speculative_logits = torch.stack([head(x) for head in self.heads], dim=1) + return logits, speculative_logits + + +class MedusaHead(torch.nn.Module): + def __init__(self, config, prefix, weights): + super().__init__() + self.blocks = torch.nn.ModuleList([ResBlock(config, prefix=f"{prefix}.{i}", weights=weights) for i in range(config["medusa_num_layers"])]) + n = len(self.blocks) + self.out = FastLinear.load(config, prefix=f"{prefix}.{n}", weights=weights, bias=False) + + def forward(self, x): + for block in self.blocks: + x = block(x) + x = self.out(x) + return x diff --git a/server/text_generation_server/utils/speculate.py b/server/text_generation_server/utils/speculate.py new file mode 100644 index 00000000..38a91972 --- /dev/null +++ b/server/text_generation_server/utils/speculate.py @@ -0,0 +1,12 @@ + +SPECULATE = None + +def get_speculate() -> int: + global SPECULATE + return SPECULATE + +def set_speculate(speculate: int): + global SPECULATE + SPECULATE = speculate + + diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 0ff07417..a34c5afc 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -16,7 +16,6 @@ from text_generation_server.utils.watermark import WatermarkLogitsProcessor from transformers import PreTrainedTokenizerBase, RepetitionPenaltyLogitsProcessor - class NextTokenChooser: def __init__( self, @@ -146,6 +145,20 @@ def from_pb( pb.ignore_eos_token, ) +def create_n_gram_speculation(input_ids: torch.Tensor, next_ids: torch.Tensor, accepted_ids: torch.Tensor, speculate: int, verbose: bool): + # Very trivial approach, find first match in the string. + # This is much less refined than actual n-gram but seems to work + # relatively OK in grounded mode and is by far much faster with + # much less worst case complexity as everything happens on device. + B = accepted_ids.shape[0] + device = input_ids.device + seeds = next_ids[accepted_ids.cumsum(dim=-1) -1 ] + indices = (input_ids == seeds.unsqueeze(-1)).max(dim=1).indices + 1 + all_indices = indices.unsqueeze(-1).expand(B, speculate) + torch.arange(speculate, device=device) + all_indices = torch.clamp(all_indices, max=input_ids.shape[1] - 1) + + speculative_ids = input_ids.gather(dim=-1, index=all_indices) + return speculative_ids class HeterogeneousNextTokenChooser: def __init__( @@ -215,20 +228,79 @@ def __init__( self.dtype = dtype self.device = device - def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor): - if self.watermark_processor is not None: - scores = self.watermark_processor(input_ids, scores) - if self.repetition_processor is not None: - scores = self.repetition_processor(input_ids, scores) - - for warper in self.warpers: - scores = warper(input_ids, scores) + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor, speculate: int, speculated_ids: Optional[torch.Tensor] = None, speculative_scores: Optional[torch.Tensor] = None, verbose=False): + if speculated_ids is not None: + B = scores.shape[0] // (speculated_ids.shape[1] + 1) + S = speculated_ids.shape[1] + 1 + scores = scores.view(B, S, -1) + else: + B = scores.shape[0] + S = 1 + scores = scores.view(B, S, -1) + + next_ids = torch.zeros((B, S), device=scores.device, dtype=torch.long) + for j in range(S): + _scores = scores[:, j] + if self.watermark_processor is not None: + _scores = self.watermark_processor(input_ids, _scores) + if self.repetition_processor is not None: + _scores = self.repetition_processor(input_ids, _scores) + + for warper in self.warpers: + _scores = warper(input_ids, _scores) + + + _next_ids = self.choice(_scores) + scores[:, j] = _scores + next_ids[:, j] = _next_ids + next_ids = next_ids.view(B*S) + scores = scores.view( B* S, -1) + + if speculated_ids is not None: + accepted_ids = [] + B = next_ids.shape[0] // (speculated_ids.shape[1] + 1) + S = speculated_ids.shape[1] + 1 + indices = [] + for i in range(B): + _next_ids = next_ids[i*S: (i + 1)*S] + _speculated_ids = speculated_ids[i] + validate_speculative = _next_ids[:-1] == _speculated_ids + index = i * S + accepted = 1 + # First is always valid + indices.append(index) + for valid in validate_speculative.tolist(): + if valid: + index += 1 + accepted += 1 + indices.append(index) + else: + break + accepted_ids.append(accepted) + + accepted_ids = torch.tensor(accepted_ids, device=input_ids.device, dtype=input_ids.dtype) + next_ids = next_ids[indices] + scores = scores[indices] + indices = torch.arange(B, device=input_ids.device) * S + if speculative_scores is not None: + speculative_scores = speculative_scores[indices + accepted_ids - 1] + else: + accepted_ids = torch.ones_like(next_ids) - next_ids = self.choice(scores) logprobs = torch.log_softmax(scores, -1) next_logprobs = torch.gather(logprobs, 1, next_ids.view(-1, 1)).view(-1) - return next_ids, next_logprobs, logprobs + if speculate > 0: + if speculative_scores is not None: + # Medusa provided some scores + speculative_ids = Greedy()(speculative_scores) + else: + # n-gram + speculative_ids = create_n_gram_speculation(input_ids, next_ids, accepted_ids, speculate, verbose) + else: + speculative_ids = None + + return next_ids, next_logprobs, logprobs, accepted_ids, speculative_ids def filter(self, indices): if self.watermark_processor is not None: From 3a521c92b3b5843ee6bacefafbc2ad5822744b1a Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 11 Dec 2023 14:43:40 +0100 Subject: [PATCH 490/793] feat: mixtral (#1328) --- Dockerfile | 9 +- router/src/server.rs | 5 + server/Makefile | 3 + .../text_generation_server/models/__init__.py | 27 +- .../custom_modeling/flash_llama_modeling.py | 85 +-- .../custom_modeling/flash_mistral_modeling.py | 82 +- .../custom_modeling/flash_mixtral_modeling.py | 708 ++++++++++++++++++ .../flash_santacoder_modeling.py | 1 - .../models/flash_mistral.py | 59 +- .../models/flash_mixtral.py | 26 + server/text_generation_server/utils/layers.py | 185 +++-- 11 files changed, 959 insertions(+), 231 deletions(-) create mode 100644 server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py create mode 100644 server/text_generation_server/models/flash_mixtral.py diff --git a/Dockerfile b/Dockerfile index 02540f81..b6c5b2ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -154,6 +154,11 @@ COPY server/Makefile-vllm Makefile # Build specific version of vllm RUN make build-vllm-cuda +# Build megablocks +FROM kernel-builder as megablocks-builder + +RUN pip install git+https://github.com/OlivierDehaene/megablocks@181709df192de9a941fdf3a641cdc65a0462996e + # Text Generation Inference base image FROM nvidia/cuda:12.1.0-base-ubuntu20.04 as base @@ -175,8 +180,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins curl \ && rm -rf /var/lib/apt/lists/* -# Copy conda with PyTorch installed -COPY --from=pytorch-install /opt/conda /opt/conda +# Copy conda with PyTorch and Megablocks installed +COPY --from=megablocks-builder /opt/conda /opt/conda # Copy build artifacts from flash attention builder COPY --from=flash-att-builder /usr/src/flash-attention/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages diff --git a/router/src/server.rs b/router/src/server.rs index 5f41fd5e..fe1b8309 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -629,6 +629,9 @@ pub async fn run( // Batch size buckets let batch_size_matcher = Matcher::Full(String::from("tgi_batch_next_size")); let batch_size_buckets: Vec = (0..1024).map(|x| (x + 1) as f64).collect(); + // Speculated tokens buckets + let skipped_matcher = Matcher::Full(String::from("tgi_request_skipped_tokens")); + let skipped_buckets: Vec = (0..shard_info.speculate + 1).map(|x| x as f64).collect(); // Prometheus handler let builder = PrometheusBuilder::new() @@ -641,6 +644,8 @@ pub async fn run( .set_buckets_for_metric(max_new_tokens_matcher, &max_new_tokens_buckets) .unwrap() .set_buckets_for_metric(batch_size_matcher, &batch_size_buckets) + .unwrap() + .set_buckets_for_metric(skipped_matcher, &skipped_buckets) .unwrap(); let prom_handle = builder .install_recorder() diff --git a/server/Makefile b/server/Makefile index 2810a528..b1926828 100644 --- a/server/Makefile +++ b/server/Makefile @@ -16,6 +16,9 @@ gen-server: find text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; touch text_generation_server/pb/__init__.py +install-megablocks: + pip install git+https://github.com/OlivierDehaene/megablocks@181709df192de9a941fdf3a641cdc65a0462996e + install: gen-server pip install pip --upgrade pip install -r requirements_cuda.txt diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 27e3897d..0172d32c 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -1,4 +1,3 @@ -import os import torch from loguru import logger @@ -78,6 +77,18 @@ if MISTRAL: __all__.append(FlashMistral) +MIXTRAL = True +try: + from text_generation_server.models.flash_mixtral import FlashMixtral +except ImportError as e: + logger.warning(f"Could not import Mixtral model: {e}") + MIXTRAL = False + +if MIXTRAL: + __all__.append(FlashMixtral) + + + def get_model( model_id: str, revision: Optional[str], @@ -141,7 +152,6 @@ def get_model( use_medusa = None if "medusa_num_heads" in config_dict: use_medusa = model_id - medusa_config = config_dict model_id = config_dict["base_model_name_or_path"] revision = "main" speculate_medusa = config_dict["medusa_num_heads"] @@ -292,7 +302,18 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) - raise NotImplementedError("Mistral model requires flash attention v2") + raise NotImplementedError("Mistral models requires flash attention v2") + + if model_type == "mixtral": + if MIXTRAL: + return FlashMixtral( + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, + ) + raise NotImplementedError("Mixtral models requires flash attention v2, stk and megablocks") if model_type == "opt": return OPTSharded( diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index 4aeb447d..d06b87eb 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -34,14 +34,8 @@ PositionRotaryEmbedding, TensorParallelHead, get_linear, + FastRMSNorm ) -from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM - -if IS_CUDA_SYSTEM: - import dropout_layer_norm -elif IS_ROCM_SYSTEM: - from vllm import layernorm_ops - class LlamaConfig(PretrainedConfig): def __init__( @@ -95,75 +89,6 @@ def __init__( ) -class LlamaRMSNorm(nn.Module): - def __init__(self, prefix, weights, eps=1e-6): - """ - LlamaRMSNorm is equivalent to T5LayerNorm - """ - super().__init__() - - weight = weights.get_tensor(f"{prefix}.weight") - self.weight = nn.Parameter(weight) - self.variance_epsilon = eps - - def forward(self, hidden_states, residual=None): - if hidden_states.shape[-1] > 8192: - if residual is not None: - hidden_states += residual - residual = hidden_states - - hidden_states = hidden_states.to(torch.float32) - variance = hidden_states.pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * torch.rsqrt( - variance + self.variance_epsilon - ) - - # convert into half-precision if necessary - if self.weight.dtype in [torch.float16, torch.bfloat16]: - hidden_states = hidden_states.to(self.weight.dtype) - - return self.weight * hidden_states, residual - elif IS_CUDA_SYSTEM: - # faster post attention rms norm - normed_hidden_states, res, *rest = dropout_layer_norm.dropout_add_ln_fwd( - hidden_states, - residual, - self.weight, - None, - None, - None, - None, - None, - 0.0, - self.variance_epsilon, - 1.0, - 0, - None, - False, - True, # Activate RMSNorm - ) - if res is None: - res = hidden_states - - return normed_hidden_states, res - elif IS_ROCM_SYSTEM: - # We use VLLM RMSNorm kernel that can be compiled for RoCm, instead of Flash Attention ones that can not. - if residual is not None: - hidden_states += residual - residual = hidden_states - - out = torch.empty_like(hidden_states) - layernorm_ops.rms_norm( - out, - hidden_states, - self.weight.data, - self.variance_epsilon, - ) - return out, residual - else: - raise ValueError("Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") - - def load_attention(config, prefix, weights): if config.num_attention_heads != config.num_key_value_heads: return _load_gqa(config, prefix, weights) @@ -363,10 +288,8 @@ def __init__(self, layer_id, config, weights): ) self.mlp = LlamaMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) - self.input_layernorm = LlamaRMSNorm( - prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps - ) - self.post_attention_layernorm = LlamaRMSNorm( + self.input_layernorm = FastRMSNorm.load(prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps) + self.post_attention_layernorm = FastRMSNorm.load( prefix=f"{prefix}.post_attention_layernorm", weights=weights, eps=config.rms_norm_eps, @@ -430,7 +353,7 @@ def __init__(self, config, weights): for layer_id in range(config.num_hidden_layers) ] ) - self.norm = LlamaRMSNorm( + self.norm = FastRMSNorm.load( prefix="model.norm", weights=weights, eps=config.rms_norm_eps ) diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index 959949f0..4e56b188 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -35,13 +35,9 @@ PositionRotaryEmbedding, TensorParallelHead, get_linear, + FastRMSNorm ) -from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM -if IS_CUDA_SYSTEM: - import dropout_layer_norm -elif IS_ROCM_SYSTEM: - from vllm import layernorm_ops if not HAS_FLASH_ATTN_V2_CUDA and not HAS_FLASH_ATTN_V2_ROCM: raise ImportError("Mistral model requires flash attn v2") @@ -100,76 +96,6 @@ def __init__( **kwargs, ) - -class MistralRMSNorm(nn.Module): - def __init__(self, prefix, weights, eps=1e-6): - """ - LlamaRMSNorm is equivalent to T5LayerNorm - """ - super().__init__() - - weight = weights.get_tensor(f"{prefix}.weight") - self.weight = nn.Parameter(weight) - self.variance_epsilon = eps - - def forward(self, hidden_states, residual=None): - if hidden_states.shape[-1] > 8192: - if residual is not None: - hidden_states += residual - residual = hidden_states - - hidden_states = hidden_states.to(torch.float32) - variance = hidden_states.pow(2).mean(-1, keepdim=True) - hidden_states = hidden_states * torch.rsqrt( - variance + self.variance_epsilon - ) - - # convert into half-precision if necessary - if self.weight.dtype in [torch.float16, torch.bfloat16]: - hidden_states = hidden_states.to(self.weight.dtype) - - return self.weight * hidden_states, residual - elif IS_CUDA_SYSTEM: - # faster post attention rms norm - normed_hidden_states, res, *rest = dropout_layer_norm.dropout_add_ln_fwd( - hidden_states, - residual, - self.weight, - None, - None, - None, - None, - None, - 0.0, - self.variance_epsilon, - 1.0, - 0, - None, - False, - True, # Activate RMSNorm - ) - if res is None: - res = hidden_states - - return normed_hidden_states, res - elif IS_ROCM_SYSTEM: - # We use VLLM RMSNorm kernel that can be compiled for RoCm, instead of Flash Attention ones that can not. - if residual is not None: - hidden_states += residual - residual = hidden_states - - out = torch.empty_like(hidden_states) - layernorm_ops.rms_norm( - out, - hidden_states, - self.weight.data, - self.variance_epsilon, - ) - return out, residual - else: - raise ValueError("Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") - - def load_attention(config, prefix, weights): if config.num_attention_heads != config.num_key_value_heads: return _load_gqa(config, prefix, weights) @@ -371,10 +297,10 @@ def __init__(self, layer_id, config, weights): ) self.mlp = MistralMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) - self.input_layernorm = MistralRMSNorm( + self.input_layernorm = FastRMSNorm.load( prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps ) - self.post_attention_layernorm = MistralRMSNorm( + self.post_attention_layernorm = FastRMSNorm.load( prefix=f"{prefix}.post_attention_layernorm", weights=weights, eps=config.rms_norm_eps, @@ -440,7 +366,7 @@ def __init__(self, config, weights): for layer_id in range(config.num_hidden_layers) ] ) - self.norm = MistralRMSNorm( + self.norm = FastRMSNorm.load( prefix="model.norm", weights=weights, eps=config.rms_norm_eps ) diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py new file mode 100644 index 00000000..66753d5a --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py @@ -0,0 +1,708 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.distributed + +import numpy as np + +from torch import nn +from transformers.activations import ACT2FN +from transformers.configuration_utils import PretrainedConfig +from typing import Optional, List, Tuple + +from text_generation_server.utils import paged_attention, flash_attn +from text_generation_server.utils.flash_attn import HAS_FLASH_ATTN_V2_ROCM, HAS_FLASH_ATTN_V2_CUDA +from text_generation_server.utils.layers import ( + FastLinear, + FastRMSNorm, + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, + PositionRotaryEmbedding, + TensorParallelHead, + get_linear, +) + +if not HAS_FLASH_ATTN_V2_CUDA and not HAS_FLASH_ATTN_V2_ROCM: + raise ImportError("Mixtral model requires flash attn v2") + +try: + import megablocks.ops as ops +except ImportError: + raise ImportError("Mixtral model requires megablocks to be installed") + +try: + import stk +except ImportError: + raise ImportError("Mixtral model requires stk to be installed") + + +class MixtralConfig(PretrainedConfig): + model_type = "mixtral" + + def __init__( + self, + vocab_size=32000, + hidden_size=4096, + intermediate_size=14336, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=8, + hidden_act="silu", + max_position_embeddings=4096 * 32, + initializer_range=0.02, + rms_norm_eps=1e-05, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000.0, + sliding_window=4096, + num_experts_per_tok=2, + num_local_experts=8, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.sliding_window = sliding_window + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + self.num_experts_per_tok = num_experts_per_tok + self.num_local_experts = num_local_experts + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +def promote_scalar(x: torch.Tensor) -> torch.Tensor: + return x.view(1) if len(x.size()) == 0 else x + + +def load_attention(config, prefix, weights): + if config.num_attention_heads != config.num_key_value_heads: + return _load_gqa(config, prefix, weights) + else: + return TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + dim=0, + weights=weights, + bias=False, + ) + + +def _load_gqa(config, prefix: str, weights): + assert config.hidden_size % config.num_attention_heads == 0 + assert config.num_attention_heads % weights.process_group.size() == 0 + + weight = weights.get_multi_weights_col( + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + quantize=config.quantize, + dim=0, + ) + + if config.quantize not in ["gptq", "awq"]: + weight = weight.to(dtype=weights.dtype).to(device=weights.device) + + head_size = config.hidden_size // config.num_attention_heads + num_heads = config.num_attention_heads // weights.process_group.size() + num_key_value_heads = config.num_key_value_heads // weights.process_group.size() + assert list(weight.shape) == [ + (num_heads + 2 * num_key_value_heads) * head_size, + config.hidden_size, + ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}" + + return TensorParallelColumnLinear( + get_linear(weight, bias=None, quantize=config.quantize) + ) + + +def _load_experts(config, prefix, mat, weights): + if config.quantize is not None: + raise NotImplementedError("Mixtral does not support weight quantization yet.") + + assert mat in ["w1", "w2", "w3"] + + world_size = weights.process_group.size() + rank = weights.process_group.rank() + + assert ( + config.intermediate_size % world_size == 0 + ), f"The chosen size {config.intermediate_size} is not compatible with sharding on {world_size} shards" + + block_size = config.intermediate_size // world_size + start = rank * block_size + stop = (rank + 1) * block_size + + tensor = torch.empty((config.num_local_experts * block_size, config.hidden_size), + dtype=weights.dtype, + device=weights.device) + + for i in range(config.num_local_experts): + slice_ = weights._get_slice(f"{prefix}.{i}.{mat}.weight") + + if mat == "w2": + expert_slice = slice_[:, start:stop].t().contiguous() + else: + expert_slice = slice_[start:stop] + tensor[i * block_size:(i + 1) * block_size] = expert_slice.to(dtype=weights.dtype).to(device=weights.device) + return tensor + + +class MixtralAttention(torch.nn.Module): + def __init__( + self, + prefix: str, + config, + weights, + ): + super().__init__() + self.max_past = ( + config.sliding_window if config.sliding_window is not None else 0 + ) + self.num_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.num_heads + + self.rotary_emb = PositionRotaryEmbedding.static( + config=config, + dim=self.head_size, + base=config.rope_theta, + device=weights.device, + ) + + self.softmax_scale = self.head_size ** -0.5 + + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) + self.num_heads = self.num_heads // weights.process_group.size() + self.num_key_value_heads = ( + config.num_key_value_heads // weights.process_group.size() + ) + + self.query_key_value = load_attention(config, prefix, weights) + + self.o_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.o_proj", + weights=weights, + bias=False, + ) + self.num_groups = self.num_heads // self.num_key_value_heads + self.kv_head_mapping = torch.arange( + 0, self.num_key_value_heads, dtype=torch.int32, device=weights.device + ).repeat_interleave(self.num_groups) + + def forward( + self, + hidden_states, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ): + qkv = self.query_key_value(hidden_states) + query, kv = qkv.split( + [ + self.head_size * self.num_heads, + 2 * self.head_size * self.num_key_value_heads, + ], + dim=1, + ) + query = query.view(-1, self.num_heads, self.head_size) + kv = kv.view(-1, 2, self.num_key_value_heads, self.head_size) + + self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin) + + if prefill_cache_indices is not None: + kv_to_cache = kv[prefill_cache_indices] + else: + kv_to_cache = kv + + paged_attention.reshape_and_cache( + kv_to_cache[:, 0], kv_to_cache[:, 1], kv_cache[0], kv_cache[1], slots + ) + + # output tensor + attn_output = torch.empty_like(query) + + # Prefill + if cu_seqlen_prefill is not None: + # flash attention + flash_attn.attention( + query, + torch.select(kv, dim=1, index=0), + torch.select(kv, dim=1, index=1), + attn_output, + cu_seqlen_prefill, + max_s, + self.softmax_scale, + window_size_left=self.max_past, + ) + # Decode + else: + paged_attention.attention( + attn_output, + query, + kv_cache[0], + kv_cache[1], + self.kv_head_mapping, + self.softmax_scale, + block_tables, + input_lengths, + max_s, + ) + + return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size)) + + +@torch.jit.script +def select_experts(gate_logits: torch.Tensor, top_k: int): + # all_probs: (sequence_length, n_experts) and upcast for softmax + all_probs = torch.nn.functional.softmax(gate_logits, dim=1, dtype=torch.float) + # weights, selected_experts: (sequence_length, top-k) + weights, selected_experts = torch.topk(all_probs, top_k, dim=-1) + weights /= weights.sum(dim=-1, keepdim=True) + weights = weights.view(-1) + selected_experts = selected_experts.view(-1) + + return selected_experts, weights + + +@torch.jit.script +def round_up(x: torch.Tensor, value: int): + return torch.div(x + (value - 1), value, rounding_mode="trunc") * value + + +class BlockSparseMoE(nn.Module): + """ + Built on the paper and library Megablocks as described in + https://arxiv.org/abs/2211.15841. This implementation is + strictly equivalent to standard MoE with full capacity (no + dropped tokens). It's faster since it formulates MoE operations + in terms of block-sparse operations to accomodate imbalanced + assignments of tokens to experts, whereas standard MoE either + (1) drop tokens at the cost of reduced performance or (2) set + capacity factor to number of experts and thus waste computation + and memory on padding. + """ + + def __init__(self, prefix, config: MixtralConfig, weights): + super().__init__() + self.hidden_dim = config.hidden_size + self.ffn_dim = config.intermediate_size // weights.process_group.size() + self.num_experts = config.num_local_experts + self.top_k = config.num_experts_per_tok + + act = config.hidden_act + if "gelu" in act: + self.act = lambda x: torch.nn.functional.gelu( + x, + approximate="tanh" + if act in ["gelu_fast", "gelu_pytorch_tanh"] + else "none", + ) + elif "silu" in act: + self.act = torch.nn.functional.silu + else: + self.act = ACT2FN[act] + + # gating + self.gate = FastLinear.load(config, f"{prefix}.gate", weights, bias=False) + + # merged expert weights, all of size (n_experts * ffn_dim, hidden_dim) + self.w1 = _load_experts(config, f"{prefix}.experts", "w1", weights).t() + self.w2 = _load_experts(config, f"{prefix}.experts", "w2", weights) + self.w3 = _load_experts(config, f"{prefix}.experts", "w3", weights).t() + + self.offsets = None + self.offsets_block_rows = 0 + + self.process_group = weights.process_group + + # Calculate the number of bits needed to represent the expert indices + # so that we can pass it to radix sort. + self.sort_end_bit = max(int(np.ceil(np.log2(self.num_experts))), 1) + self.blocking = 128 + self.quantize_scatter_num_bits = -1 + + def topology(self, x: torch.Tensor, padded_bins: torch.Tensor): + padded_tokens, _ = x.size() + assert padded_tokens % self.blocking == 0 + assert self.ffn_dim % self.blocking == 0 + + # Offsets for the sparse matrix. All rows have the + # same number of nonzero blocks dictated by the + # dimensionality of a single expert. + block_rows = padded_tokens // self.blocking + blocks_per_row = self.ffn_dim // self.blocking + if self.offsets is None or block_rows > self.offsets_block_rows: + self.offsets = torch.arange( + 0, + block_rows * blocks_per_row + 1, + blocks_per_row, + dtype=torch.int32, + device=x.device, + ) + self.offsets_block_rows = block_rows + offsets = self.offsets + else: + offsets = self.offsets[:block_rows] + + # Indices for the sparse matrix. The indices for + # the intermediate matrix are dynamic depending + # on the mapping of tokens to experts. + column_indices = ops.topology(padded_bins, self.blocking, block_rows, + blocks_per_row) + + # For now, use meta init to save the device memory. + data = torch.empty( + column_indices.numel(), + self.blocking, + self.blocking, + dtype=x.dtype, + device="meta", + ) + shape = (padded_tokens, self.ffn_dim * self.num_experts) + row_indices = stk.ops.row_indices(shape, data, offsets, column_indices) + return stk.Matrix( + shape, + data, + row_indices, + column_indices, + offsets, + False, + False, + False, + ) + + def indices_and_padded_bins(self, selected_experts: torch.Tensor): + # Sort the expert ids to produce the scatter/gather + # indices for the permutation. + # selected_experts = selected_experts.int() + + # returns bin_ids == num of experts for this sequence ? == unique selected experts? + # and indices == how to sort tokens? + bin_ids, indices = ops.sort(selected_experts, self.sort_end_bit) + # bin_ids => [0, 0, 0, 2, 2, ...] => [num_tokens * top_k] + # indices => [14, 32, 33, ...] => [num_tokens * top_k] + + # Histogram the expert ids to identify the number of + # tokens routed to each expert. + tokens_per_expert = ops.histogram(selected_experts, self.num_experts) + # tokens_per_expert => [3, 0, 2, ...] => [num_experts] + + # Round the token counts up to the block size used in + # the matrix muliplications. Caculate the starting + # position of each bin. + + # List of size num_experts + padded_tokens_per_expert = round_up(tokens_per_expert, + self.blocking) + # padded_tokens_per_expert => [128, O, 128, ...] + + # Cumulative selected experts per token + padded_bins = ops.inclusive_cumsum(padded_tokens_per_expert, 0) + padded_bins = promote_scalar(padded_bins) + # padded_bins => [128, 128, 256, ...] + + # Calculate the bin bounds for the sorted tokens. + bins = ops.inclusive_cumsum(tokens_per_expert, 0) + bins = promote_scalar(bins) + # bins => [3, 3, 5, ...] + + return indices, bin_ids, bins, padded_bins, tokens_per_expert + + @torch.inference_mode() + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + x: (sequence_length, model_dim) + gate_logits: (sequence_length, n_experts) + """ + # optional reshape + input_shape = x.shape + x = x.view(-1, input_shape[-1]) + + # gate_logits: (sequence_length, n_experts) + gate_logits = self.gate(x) + selected_experts, weights = select_experts(gate_logits, self.top_k) + + ( + indices, + bin_ids, + bins, + padded_bins, + _, + ) = self.indices_and_padded_bins(selected_experts) + + # Permute tokens and pad to prepare expert computation + # (top_k * sequence_length + padding, model_dim) + x = ops.padded_gather(x, indices, bin_ids, bins, padded_bins, + self.top_k) + + # Create the sparse matrix topology + with torch.no_grad(): + topo = self.topology(x, padded_bins) + + # Perform the expert computation + # First Dense x Dense -> Sparse for w1 and w3, + # (top_k * sequence_length + padding, ffn_dim * n_experts) + x = stk.Matrix( + topo.size(), + self.act(stk.ops.sdd(x, self.w1, topo).data) * + stk.ops.sdd(x, self.w3, topo).data, + topo.row_indices, + topo.column_indices, + topo.offsets, + topo.column_indices_t, + topo.offsets_t, + topo.block_offsets_t, + ) + + # Then Sparse x Dense -> Dense for w2 + # (top_k * sequence_length + padding, model_dim) + x = stk.ops.dsd(x, self.w2) + + # Permute back and remove padding + # (sequence_length, model_dim) + x = ops.padded_scatter( + x, + indices, + bin_ids, + weights, + bins, + padded_bins, + self.top_k, + self.quantize_scatter_num_bits, + ).view(*input_shape) + + if self.process_group.size() > 1: + torch.distributed.all_reduce(x, group=self.process_group) + + return x.view(*input_shape) + + +class MixtralLayer(nn.Module): + def __init__(self, layer_id, config, weights): + super().__init__() + prefix = f"model.layers.{layer_id}" + + self.self_attn = MixtralAttention( + prefix=f"{prefix}.self_attn", config=config, weights=weights + ) + self.block_sparse_moe = BlockSparseMoE(f"{prefix}.block_sparse_moe", config, weights) + + self.input_layernorm = FastRMSNorm.load( + prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps + ) + self.post_attention_layernorm = FastRMSNorm.load( + prefix=f"{prefix}.post_attention_layernorm", + weights=weights, + eps=config.rms_norm_eps, + ) + + def forward( + self, + hidden_states, + residual, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ): + normed_hidden_states, res = self.input_layernorm(hidden_states, residual) + + # Self Attention + attn_output = self.self_attn( + normed_hidden_states, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ) + + # faster post attention rms norm + normed_attn_res_output, attn_res = self.post_attention_layernorm( + attn_output, res + ) + + block_sparse_moe_output = self.block_sparse_moe(normed_attn_res_output) + + return block_sparse_moe_output, attn_res + + +class MixtralModel(torch.nn.Module): + def __init__(self, config, weights): + super().__init__() + + self.embed_tokens = TensorParallelEmbedding( + prefix="model.embed_tokens", weights=weights + ) + + self.layers = nn.ModuleList( + [ + MixtralLayer( + layer_id, + config, + weights, + ) + for layer_id in range(config.num_hidden_layers) + ] + ) + self.norm = FastRMSNorm.load( + prefix="model.norm", weights=weights, eps=config.rms_norm_eps + ) + + self.head_size = self.layers[0].self_attn.head_size + self.num_heads = self.layers[0].self_attn.num_heads + self.num_key_value_heads = self.layers[0].self_attn.num_key_value_heads + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlen_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + prefill_cache_indices: Optional[torch.Tensor], + ) -> torch.Tensor: + hidden_states = self.embed_tokens(input_ids) + + # Get rotary cos and sin for this forward + # Avoid to index in each layer + cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin( + position_ids, max_s, hidden_states.dtype + ) + + residual = None + for i, layer in enumerate(self.layers): + hidden_states, residual = layer( + hidden_states, + residual, + cos, + sin, + cu_seqlen_prefill, + kv_cache[i], + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ) + + hidden_states, _ = self.norm(hidden_states, residual) + + return hidden_states + + +class FlashMixtralForCausalLM(torch.nn.Module): + def __init__(self, config, weights): + super().__init__() + + self.model = MixtralModel(config, weights) + self.lm_head = TensorParallelHead.load( + config, + prefix="lm_head", + weights=weights, + ) + self.max_past = config.sliding_window + if self.max_past is None: + raise ValueError("max_past cannot be None") + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlen_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + prefill_cache_indices: Optional[torch.Tensor], + lm_head_indices: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if prefill_cache_indices is not None: + # Slots also need to be sliced as it has the same size as the whole kv tensor + slots = slots[prefill_cache_indices] + else: + # Clamp in decode mode as paged attention requires clamped values whereas the flash attention + # kernel requires the true values + max_s = min(self.max_past, max_s) + input_lengths = torch.clamp(input_lengths, max=self.max_past) + + hidden_states = self.model( + input_ids, + position_ids, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, + ) + if lm_head_indices is not None: + hidden_states = hidden_states[lm_head_indices] + logits = self.lm_head(hidden_states) + return logits diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index c3c7617a..cd93d32a 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -6,7 +6,6 @@ from typing import Optional, List, Tuple from text_generation_server.utils import paged_attention, flash_attn -from text_generation_server.utils.flash_attn import attention from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py index e103d9fc..5ce37164 100644 --- a/server/text_generation_server/models/flash_mistral.py +++ b/server/text_generation_server/models/flash_mistral.py @@ -8,14 +8,13 @@ from opentelemetry import trace from transformers import PreTrainedTokenizerBase from transformers.models.llama import LlamaTokenizerFast -from typing import Optional, Tuple, Type +from typing import Optional, Tuple, Type, List from text_generation_server.pb import generate_pb2 from text_generation_server.models import FlashCausalLM from text_generation_server.models.flash_causal_lm import FlashCausalLMBatch, BLOCK_SIZE from text_generation_server.models.cache_manager import ( get_cache_manager, - set_cache_manager, ) from text_generation_server.models.custom_modeling.flash_mistral_modeling import ( FlashMistralForCausalLM, @@ -46,11 +45,11 @@ class FlashMistralBatch(FlashCausalLMBatch): @classmethod def from_pb( - cls, - pb: generate_pb2.Batch, - tokenizer: PreTrainedTokenizerBase, - dtype: torch.dtype, - device: torch.device, + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, + device: torch.device, ) -> "FlashCausalLMBatch": global SLIDING_WINDOW global SLIDING_WINDOW_BLOCKS @@ -100,12 +99,12 @@ def from_pb( # Parse batch for i, (r, tokenized_input) in enumerate( - zip(pb.requests, batch_tokenized_inputs) + zip(pb.requests, batch_tokenized_inputs) ): # request id -> idx in list mapping requests_idx_mapping[r.id] = i - tokenized_input = tokenized_input[-r.truncate :] + tokenized_input = tokenized_input[-r.truncate:] input_length = len(tokenized_input) input_lengths.append(input_length) @@ -278,14 +277,16 @@ def from_pb( ) -class FlashMistral(FlashCausalLM): +class BaseFlashMistral(FlashCausalLM): def __init__( - self, - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - dtype: Optional[torch.dtype] = None, - trust_remote_code: bool = False, + self, + config_cls, + model_cls, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, ): global SLIDING_WINDOW global SLIDING_WINDOW_BLOCKS @@ -305,7 +306,7 @@ def __init__( trust_remote_code=trust_remote_code, ) - config = MistralConfig.from_pretrained( + config = config_cls.from_pretrained( model_id, revision=revision, trust_remote_code=trust_remote_code ) config.quantize = quantize @@ -321,10 +322,10 @@ def __init__( if config.quantize in ["gptq", "awq"]: weights._set_gptq_params(model_id) - model = FlashMistralForCausalLM(config, weights) + model = model_cls(config, weights) torch.distributed.barrier(group=self.process_group) - super(FlashMistral, self).__init__( + super(BaseFlashMistral, self).__init__( model=model, tokenizer=tokenizer, num_layers=len(model.model.layers), @@ -396,3 +397,23 @@ def forward(self, batch: FlashMistralBatch) -> Tuple[torch.Tensor, torch.Tensor] if batch.prefill_cache_indices is not None: batch.prefill_cache_indices = None return logits + + +class FlashMistral(BaseFlashMistral): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, + ): + super(FlashMistral, self).__init__( + config_cls=MistralConfig, + model_cls=FlashMistralForCausalLM, + model_id=model_id, + revision=revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code + ) diff --git a/server/text_generation_server/models/flash_mixtral.py b/server/text_generation_server/models/flash_mixtral.py new file mode 100644 index 00000000..c45ae50f --- /dev/null +++ b/server/text_generation_server/models/flash_mixtral.py @@ -0,0 +1,26 @@ +import torch + +from typing import Optional + +from text_generation_server.models.flash_mistral import BaseFlashMistral +from text_generation_server.models.custom_modeling.flash_mixtral_modeling import MixtralConfig, FlashMixtralForCausalLM + + +class FlashMixtral(BaseFlashMistral): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, + ): + super(FlashMixtral, self).__init__( + config_cls=MixtralConfig, + model_cls=FlashMixtralForCausalLM, + model_id=model_id, + revision=revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code + ) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index a93ccd0e..d533016d 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -18,7 +18,7 @@ from accelerate import init_empty_weights from text_generation_server.utils.gptq.quant_linear import QuantLinear -from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM +from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM HAS_AWQ = True try: @@ -43,16 +43,18 @@ elif CAN_EXLLAMA: try: if V2: - from text_generation_server.utils.gptq.exllamav2 import (QuantLinear as ExllamaQuantLinear, - create_exllama_buffers, - set_device, + from text_generation_server.utils.gptq.exllamav2 import (QuantLinear as ExllamaQuantLinear, + create_exllama_buffers, + set_device, ) + HAS_EXLLAMA = "2" else: from text_generation_server.utils.gptq.exllama import (Ex4bitLinear as ExllamaQuantLinear, - create_exllama_buffers, - set_device, - ) + create_exllama_buffers, + set_device, + ) + HAS_EXLLAMA = "1" except ImportError: @@ -112,7 +114,7 @@ def load_conv2d(cls, prefix, weights, in_channels, out_channels, kernel_size, st @classmethod def load_conv2d_no_bias( - cls, prefix, weights, in_channels, out_channels, kernel_size, stride + cls, prefix, weights, in_channels, out_channels, kernel_size, stride ): weight = weights.get_tensor(f"{prefix}.weight") with init_empty_weights(): @@ -136,9 +138,9 @@ def load_conv2d_no_bias( class FastLinear(nn.Module): def __init__( - self, - weight, - bias, + self, + weight, + bias, ) -> None: super().__init__() self.weight = nn.Parameter(weight) @@ -162,9 +164,9 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: class EETQLinear(nn.Module): def __init__( - self, - weight, - bias, + self, + weight, + bias, ) -> None: super().__init__() device = weight.device @@ -183,13 +185,13 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: class Linear8bitLt(nn.Module): def __init__( - self, - weight, - bias, - has_fp16_weights=True, - memory_efficient_backward=False, - threshold=0.0, - index=None, + self, + weight, + bias, + has_fp16_weights=True, + memory_efficient_backward=False, + threshold=0.0, + index=None, ): super().__init__() assert ( @@ -526,9 +528,12 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: try: if IS_CUDA_SYSTEM: import dropout_layer_norm + elif IS_ROCM_SYSTEM: + from vllm import layernorm_ops else: dropout_layer_norm = None + class FastLayerNorm(nn.LayerNorm): def forward(self, hidden_states, residual=None): if hidden_states.shape[-1] > 8192 or IS_ROCM_SYSTEM: @@ -563,10 +568,81 @@ def forward(self, hidden_states, residual=None): residual = hidden_states return normed_hidden_states, residual + + + class FastRMSNorm(nn.Module): + def __init__(self, weight: torch.Tensor, eps: float): + super().__init__() + + self.weight = nn.Parameter(weight) + self.variance_epsilon = eps + + @classmethod + def load(cls, prefix, weights, eps=1e-6): + weight = weights.get_tensor(f"{prefix}.weight") + return cls(weight, eps) + + def forward(self, hidden_states, residual=None): + if hidden_states.shape[-1] > 8192: + if residual is not None: + hidden_states += residual + residual = hidden_states + + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt( + variance + self.variance_epsilon + ) + + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states, residual + elif IS_CUDA_SYSTEM: + # faster post attention rms norm + normed_hidden_states, res, *rest = dropout_layer_norm.dropout_add_ln_fwd( + hidden_states, + residual, + self.weight, + None, + None, + None, + None, + None, + 0.0, + self.variance_epsilon, + 1.0, + 0, + None, + False, + True, # Activate RMSNorm + ) + if res is None: + res = hidden_states + + return normed_hidden_states, res + elif IS_ROCM_SYSTEM: + # We use VLLM RMSNorm kernel that can be compiled for RoCm, instead of Flash Attention ones that can not. + if residual is not None: + hidden_states += residual + residual = hidden_states + + out = torch.empty_like(hidden_states) + layernorm_ops.rms_norm( + out, + hidden_states, + self.weight.data, + self.variance_epsilon, + ) + return out, residual + else: + raise ValueError( + "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") + except ImportError: pass - try: if IS_CUDA_SYSTEM: from flash_attn.layers.rotary import RotaryEmbedding @@ -574,12 +650,14 @@ def forward(self, hidden_states, residual=None): elif IS_ROCM_SYSTEM: from vllm import pos_encoding_ops + def _create_inv_freq(dim, base, device): inv_freq = 1.0 / ( - base ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) + base ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) ) return inv_freq + def _get_rope_config(config): if os.getenv("ROPE_SCALING", None) is not None: rope_scaling = { @@ -589,6 +667,7 @@ def _get_rope_config(config): return rope_scaling return getattr(config, "rope_scaling", None) + class PositionRotaryEmbedding(nn.Module): def __init__(self, inv_freq, scaling_factor): super().__init__() @@ -606,12 +685,12 @@ def forward(self, query: torch.Tensor, key: torch.Tensor, cos: torch.Tensor, sin if IS_CUDA_SYSTEM: rotary_dim = cos.shape[-1] q1 = query[..., :rotary_dim] - q2 = query[..., rotary_dim : 2 * rotary_dim] + q2 = query[..., rotary_dim: 2 * rotary_dim] rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False) k1 = key[..., :rotary_dim] - k2 = key[..., rotary_dim : 2 * rotary_dim] + k2 = key[..., rotary_dim: 2 * rotary_dim] rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False) elif IS_ROCM_SYSTEM: @@ -630,7 +709,8 @@ def forward(self, query: torch.Tensor, key: torch.Tensor, cos: torch.Tensor, sin True ) else: - raise ValueError("Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") + raise ValueError( + "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") @classmethod def static(cls, config, dim, base, device): @@ -713,9 +793,9 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): # Reset the tables if the sequence length has changed, # or if we're on a new device (possibly due to tracing for instance) if ( - seqlen > self._seq_len_cached - or self._cos_cached.device != device - or self._cos_cached.dtype != dtype + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype ): self._seq_len_cached = seqlen t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) @@ -729,7 +809,7 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): self._sin_cached = torch.sin(freqs).to(dtype) def get_cos_sin( - self, position_ids: torch.Tensor, max_s: int, dtype: torch.dtype + self, position_ids: torch.Tensor, max_s: int, dtype: torch.dtype ): """ Return cos and sin for the asked position ids @@ -747,6 +827,7 @@ def get_cos_sin( # Note: this unsqueeze is not necessary on RoCm + VLLM ROPE implementation, but we leave it as is to avoid yet an other controlflow. return cos.unsqueeze(1), sin.unsqueeze(1) + class DynamicPositionRotaryEmbedding(PositionRotaryEmbedding): def __init__(self, dim, max_position_embeddings, base, device, scaling_factor): inv_freq = _create_inv_freq(dim, base, device) @@ -755,18 +836,18 @@ def __init__(self, dim, max_position_embeddings, base, device, scaling_factor): self.max_position_embeddings = max_position_embeddings self.base = base - def _update_cos_sin_cache(self, dtype, device, seqlen): + def _update_cos_sin_cache(self, dtype, device, seqlen): # Reset the tables if the sequence length has changed, # or if we're on a new device (possibly due to tracing for instance) if ( - seqlen > self._seq_len_cached - or self._cos_cached.device != device - or self._cos_cached.dtype != dtype + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype ): if seqlen > self.max_position_embeddings: newbase = self.base * ( - (self.scaling_factor * seqlen / self.max_position_embeddings) - - (self.scaling_factor - 1) + (self.scaling_factor * seqlen / self.max_position_embeddings) + - (self.scaling_factor - 1) ) ** (self.dim / (self.dim - 2)) self.inv_freq = _create_inv_freq( self.dim, newbase, self.inv_freq.device @@ -783,8 +864,11 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): # Inverse dim formula to find dim based on number of rotations import math + + def find_correction_dim(num_rotations, dim, base=10000, max_position_embeddings=2048): - return (dim * math.log(max_position_embeddings/(num_rotations * 2 * math.pi)))/(2 * math.log(base)) + return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / (2 * math.log(base)) + # Find dim range bounds based on rotations def find_correction_range(low_rot, high_rot, dim, base=10000, max_position_embeddings=2048): @@ -792,7 +876,8 @@ def find_correction_range(low_rot, high_rot, dim, base=10000, max_position_embed low_rot, dim, base, max_position_embeddings)) high = math.ceil(find_correction_dim( high_rot, dim, base, max_position_embeddings)) - return max(low, 0), min(high, dim-1) # Clamp values just in case + return max(low, 0), min(high, dim - 1) # Clamp values just in case + def linear_ramp_mask(min, max, dim): if min == max: @@ -802,13 +887,16 @@ def linear_ramp_mask(min, max, dim): ramp_func = torch.clamp(linear_func, 0, 1) return ramp_func + def get_mscale(scale=1): if scale <= 1: return 1.0 return 0.1 * math.log(scale) + 1.0 + class YarnPositionRotaryEmbedding(PositionRotaryEmbedding): - def __init__(self, dim, max_position_embeddings, base, device, scaling_factor,*, extrapolation_factor, attn_factor, beta_fast, beta_slow): + def __init__(self, dim, max_position_embeddings, base, device, scaling_factor, *, extrapolation_factor, + attn_factor, beta_fast, beta_slow): inv_freq = _create_inv_freq(dim, base, device) super().__init__(inv_freq, scaling_factor) self.dim = dim @@ -818,15 +906,16 @@ def __init__(self, dim, max_position_embeddings, base, device, scaling_factor,*, self.attn_factor = attn_factor self.beta_fast = beta_fast self.beta_slow = beta_slow - self.mscale = float(get_mscale(self.scaling_factor) * self.attn_factor) # Get n-d magnitude scaling corrected for interpolation + self.mscale = float(get_mscale( + self.scaling_factor) * self.attn_factor) # Get n-d magnitude scaling corrected for interpolation def _update_cos_sin_cache(self, dtype, device, seqlen): # Reset the tables if the sequence length has changed, # or if we're on a new device (possibly due to tracing for instance) if ( - seqlen > self._seq_len_cached - or self._cos_cached.device != device - or self._cos_cached.dtype != dtype + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype ): if seqlen > self.max_position_embeddings: inv_freq_extrapolation = _create_inv_freq( @@ -834,13 +923,15 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): ) freqs = 1.0 / inv_freq_extrapolation inv_freq_interpolation = 1.0 / (self.scaling_factor * freqs) - low, high = find_correction_range(self.beta_fast, self.beta_slow, self.dim, self.base, self.max_position_embeddings) - inv_freq_mask = (1 - linear_ramp_mask(low, high, self.dim // 2).float().to(device)) * self.extrapolation_factor # Get n-d rotational scaling corrected for extrapolation + low, high = find_correction_range(self.beta_fast, self.beta_slow, self.dim, self.base, + self.max_position_embeddings) + inv_freq_mask = (1 - linear_ramp_mask(low, high, self.dim // 2).float().to( + device)) * self.extrapolation_factor # Get n-d rotational scaling corrected for extrapolation inv_freq = inv_freq_interpolation * (1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask self.inv_freq = inv_freq - self.mscale = float(get_mscale(self.scaling_factor) * self.attn_factor) # Get n-d magnitude scaling corrected for interpolation - + self.mscale = float(get_mscale( + self.scaling_factor) * self.attn_factor) # Get n-d magnitude scaling corrected for interpolation self._seq_len_cached = seqlen t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) From 72ee382dedb9d2988b82ddca1e6d8933088ec707 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:49:52 +0100 Subject: [PATCH 491/793] chore: formatting --- integration-tests/conftest.py | 13 +- integration-tests/models/test_flash_medusa.py | 12 +- .../models/test_flash_mistral.py | 4 +- integration-tests/models/test_idefics.py | 4 +- server/tests/models/test_bloom.py | 16 +- server/tests/models/test_causal_lm.py | 16 +- server/tests/models/test_seq2seq_lm.py | 16 +- server/text_generation_server/cli.py | 38 +++- .../text_generation_server/models/__init__.py | 13 +- server/text_generation_server/models/bloom.py | 6 +- .../models/causal_lm.py | 21 +- .../custom_modeling/flash_llama_modeling.py | 11 +- .../custom_modeling/flash_mistral_modeling.py | 9 +- .../custom_modeling/flash_mixtral_modeling.py | 182 ++++++++-------- .../idefics_image_processing.py | 6 +- .../custom_modeling/idefics_modeling.py | 14 +- .../custom_modeling/idefics_processing.py | 2 +- .../models/flash_causal_lm.py | 124 ++++++----- .../models/flash_llama.py | 15 +- .../models/flash_mistral.py | 107 +++++---- .../models/flash_mixtral.py | 19 +- .../models/idefics_causal_lm.py | 13 +- server/text_generation_server/models/model.py | 2 +- .../models/seq2seq_lm.py | 10 +- server/text_generation_server/models/types.py | 5 +- server/text_generation_server/server.py | 12 +- .../utils/flash_attn.py | 10 +- .../utils/gptq/exllamav2.py | 132 ++++++----- server/text_generation_server/utils/layers.py | 206 ++++++++++-------- server/text_generation_server/utils/medusa.py | 28 ++- .../utils/paged_attention.py | 35 +-- server/text_generation_server/utils/peft.py | 4 +- .../text_generation_server/utils/speculate.py | 5 +- server/text_generation_server/utils/tokens.py | 42 +++- .../text_generation_server/utils/weights.py | 7 +- update_doc.py | 6 +- 36 files changed, 715 insertions(+), 450 deletions(-) diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 35c8faae..4cb4ca59 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -25,6 +25,7 @@ class ResponseComparator(JSONSnapshotExtension): rtol = 0.2 + def serialize( self, data, @@ -69,7 +70,9 @@ def eq_prefill_token(prefill_token: InputToken, other: InputToken) -> bool: prefill_token.id == other.id and prefill_token.text == other.text and ( - math.isclose(prefill_token.logprob, other.logprob, rel_tol=self.rtol) + math.isclose( + prefill_token.logprob, other.logprob, rel_tol=self.rtol + ) if prefill_token.logprob is not None else prefill_token.logprob == other.logprob ) @@ -153,6 +156,7 @@ class GenerousResponseComparator(ResponseComparator): # Needed for GPTQ with exllama which has serious numerical fluctuations. rtol = 0.75 + class LauncherHandle: def __init__(self, port: int): self.client = AsyncClient(f"http://localhost:{port}") @@ -198,6 +202,7 @@ def _inner_health(self) -> bool: def response_snapshot(snapshot): return snapshot.use_extension(ResponseComparator) + @pytest.fixture def generous_response_snapshot(snapshot): return snapshot.use_extension(GenerousResponseComparator) @@ -219,7 +224,7 @@ def local_launcher( quantize: Optional[str] = None, trust_remote_code: bool = False, use_flash_attention: bool = True, - dtype: Optional[str] = None + dtype: Optional[str] = None, ): port = random.randint(8000, 10_000) master_port = random.randint(10_000, 20_000) @@ -282,7 +287,7 @@ def docker_launcher( quantize: Optional[str] = None, trust_remote_code: bool = False, use_flash_attention: bool = True, - dtype: Optional[str] = None + dtype: Optional[str] = None, ): port = random.randint(8000, 10_000) @@ -335,7 +340,7 @@ def docker_launcher( ], volumes=volumes, ports={"80/tcp": port}, - shm_size="1G" + shm_size="1G", ) yield ContainerLauncherHandle(client, container.name, port) diff --git a/integration-tests/models/test_flash_medusa.py b/integration-tests/models/test_flash_medusa.py index 003409b0..a0ce0570 100644 --- a/integration-tests/models/test_flash_medusa.py +++ b/integration-tests/models/test_flash_medusa.py @@ -50,10 +50,16 @@ async def test_flash_medusa_all_params(flash_medusa, response_snapshot): @pytest.mark.asyncio @pytest.mark.private async def test_flash_medusa_load(flash_medusa, generate_load, response_snapshot): - responses = await generate_load(flash_medusa, "What is Deep Learning?", max_new_tokens=10, n=4) + responses = await generate_load( + flash_medusa, "What is Deep Learning?", max_new_tokens=10, n=4 + ) assert len(responses) == 4 - assert all([r.generated_text == responses[0].generated_text for r in responses]), f"{[r.generated_text for r in responses]}" - assert responses[0].generated_text == '\nDeep learning is a subset of machine learning' + assert all( + [r.generated_text == responses[0].generated_text for r in responses] + ), f"{[r.generated_text for r in responses]}" + assert ( + responses[0].generated_text == "\nDeep learning is a subset of machine learning" + ) assert responses == response_snapshot diff --git a/integration-tests/models/test_flash_mistral.py b/integration-tests/models/test_flash_mistral.py index 7d21afd9..ace3328b 100644 --- a/integration-tests/models/test_flash_mistral.py +++ b/integration-tests/models/test_flash_mistral.py @@ -56,7 +56,9 @@ async def test_flash_mistral_load(flash_mistral, generate_load, response_snapsho ) assert len(responses) == 4 - assert all([r.generated_text == responses[0].generated_text for r in responses]), f"{[r.generated_text for r in responses]}" + assert all( + [r.generated_text == responses[0].generated_text for r in responses] + ), f"{[r.generated_text for r in responses]}" assert responses[0].generated_text == ": Let n = 10 - 1" assert responses == response_snapshot diff --git a/integration-tests/models/test_idefics.py b/integration-tests/models/test_idefics.py index 5a81a4f0..7e1d3e11 100644 --- a/integration-tests/models/test_idefics.py +++ b/integration-tests/models/test_idefics.py @@ -3,7 +3,9 @@ @pytest.fixture(scope="module") def idefics_handle(launcher): - with launcher("HuggingFaceM4/idefics-9b-instruct", num_shard=2, dtype="float16") as handle: + with launcher( + "HuggingFaceM4/idefics-9b-instruct", num_shard=2, dtype="float16" + ) as handle: yield handle diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index 1990ef8b..d9a33795 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -133,8 +133,20 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): ) assert all([generation.generated_text is None for generation in generations]) assert all([len(generation.prefill_tokens) == 1 for generation in generations]) - assert all([token_id.item() == 10264 for generation in generations for token_id in generation.tokens.token_ids]) - assert all([token_text == "Test" for generation in generations for token_text in generation.tokens.texts]) + assert all( + [ + token_id.item() == 10264 + for generation in generations + for token_id in generation.tokens.token_ids + ] + ) + assert all( + [ + token_text == "Test" + for generation in generations + for token_text in generation.tokens.texts + ] + ) assert generations[0].request_id == 0 diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index f105ce6f..8b45e781 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -129,8 +129,20 @@ def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): ) assert all([generation.generated_text is None for generation in generations]) assert all([len(generation.prefill_tokens) == 1 for generation in generations]) - assert all([token_id.item() == 13 for generation in generations for token_id in generation.tokens.token_ids]) - assert all([token_text == "." for generation in generations for token_text in generation.tokens.texts]) + assert all( + [ + token_id.item() == 13 + for generation in generations + for token_id in generation.tokens.token_ids + ] + ) + assert all( + [ + token_text == "." + for generation in generations + for token_text in generation.tokens.texts + ] + ) assert generations[0].request_id == 0 diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index d553067e..373867c7 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -151,8 +151,20 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch) ) assert all([generation.generated_text is None for generation in generations]) assert all([len(generation.prefill_tokens) == 1 for generation in generations]) - assert all([token_id.item() == 259 for generation in generations for token_id in generation.tokens.token_ids]) - assert all([token_text == " " for generation in generations for token_text in generation.tokens.texts]) + assert all( + [ + token_id.item() == 259 + for generation in generations + for token_id in generation.tokens.token_ids + ] + ) + assert all( + [ + token_text == " " + for generation in generations + for token_text in generation.tokens.texts + ] + ) assert generations[0].request_id == 0 diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index cb151173..1d67d7eb 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -77,12 +77,24 @@ def serve( # Downgrade enum into str for easier management later on quantize = None if quantize is None else quantize.value dtype = None if dtype is None else dtype.value - if dtype is not None and quantize not in {None, "bitsandbytes", "bitsandbytes-nf4", "bitsandbytes-fp4"}: + if dtype is not None and quantize not in { + None, + "bitsandbytes", + "bitsandbytes-nf4", + "bitsandbytes-fp4", + }: raise RuntimeError( "Only 1 can be set between `dtype` and `quantize`, as they both decide how goes the final model." ) server.serve( - model_id, revision, sharded, quantize, speculate, dtype, trust_remote_code, uds_path + model_id, + revision, + sharded, + quantize, + speculate, + dtype, + trust_remote_code, + uds_path, ) @@ -140,12 +152,17 @@ def download_weights( try: import json - medusa_head = hf_hub_download(model_id, revision=revision, filename="medusa_lm_head.pt") + + medusa_head = hf_hub_download( + model_id, revision=revision, filename="medusa_lm_head.pt" + ) if auto_convert: - medusa_sf = Path(medusa_head[:-len(".pt")] + ".safetensors") + medusa_sf = Path(medusa_head[: -len(".pt")] + ".safetensors") if not medusa_sf.exists(): utils.convert_files([Path(medusa_head)], [medusa_sf], []) - medusa_config = hf_hub_download(model_id, revision=revision, filename="config.json") + medusa_config = hf_hub_download( + model_id, revision=revision, filename="config.json" + ) with open(medusa_config, "r") as f: config = json.load(f) @@ -153,10 +170,17 @@ def download_weights( revision = "main" try: utils.weight_files(model_id, revision, extension) - logger.info(f"Files for parent {model_id} are already present on the host. " "Skipping download.") + logger.info( + f"Files for parent {model_id} are already present on the host. " + "Skipping download." + ) return # Local files not found - except (utils.LocalEntryNotFoundError, FileNotFoundError, utils.EntryNotFoundError): + except ( + utils.LocalEntryNotFoundError, + FileNotFoundError, + utils.EntryNotFoundError, + ): pass except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError): pass diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 0172d32c..f2ef55ce 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -88,7 +88,6 @@ __all__.append(FlashMixtral) - def get_model( model_id: str, revision: Optional[str], @@ -157,7 +156,9 @@ def get_model( speculate_medusa = config_dict["medusa_num_heads"] if speculate is not None: if speculate > speculate_medusa: - raise RuntimeError("Speculate is set to `{speculate}` but this medusa models only has `{speculate_medusa}` heads, please make them match") + raise RuntimeError( + "Speculate is set to `{speculate}` but this medusa models only has `{speculate_medusa}` heads, please make them match" + ) else: set_speculate(speculate) else: @@ -249,7 +250,7 @@ def get_model( quantize=quantize, dtype=dtype, trust_remote_code=trust_remote_code, - use_medusa=use_medusa + use_medusa=use_medusa, ) elif sharded: raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format("Sharded Llama")) @@ -313,7 +314,9 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) - raise NotImplementedError("Mixtral models requires flash attention v2, stk and megablocks") + raise NotImplementedError( + "Mixtral models requires flash attention v2, stk and megablocks" + ) if model_type == "opt": return OPTSharded( @@ -354,7 +357,7 @@ def get_model( raise ValueError("awq quantization is not supported for AutoModel") elif (quantize == "bitsandbytes-fp4") or (quantize == "bitsandbytes-nf4"): raise ValueError("4bit quantization is not supported for AutoModel") - elif (quantize == "eetq"): + elif quantize == "eetq": raise ValueError("Eetq quantization is not supported for AutoModel") if model_type in modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: return CausalLM( diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index 8e8daad3..c3876023 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -74,7 +74,11 @@ def __init__( torch.distributed.barrier(group=self.process_group) filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights( - filenames, device=device, dtype=dtype, process_group=self.process_group, prefix="transformer", + filenames, + device=device, + dtype=dtype, + process_group=self.process_group, + prefix="transformer", ) if config.quantize == "gptq": weights._set_gptq_params(model_id) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index c571a022..b771264b 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -510,7 +510,11 @@ def __init__( load_in_8bit=quantize == "bitsandbytes", trust_remote_code=trust_remote_code, ) - if torch.cuda.is_available() and torch.cuda.device_count() == 1 and quantize != "bitsandbytes": + if ( + torch.cuda.is_available() + and torch.cuda.device_count() == 1 + and quantize != "bitsandbytes" + ): model = model.cuda() if tokenizer.pad_token_id is None: @@ -676,7 +680,10 @@ def generate_token( skip_special_tokens=False, ) prefill_tokens = Tokens( - prefill_token_ids, prefill_logprobs, prefill_texts, is_special=[] + prefill_token_ids, + prefill_logprobs, + prefill_texts, + is_special=[], ) else: prefill_tokens = None @@ -703,11 +710,11 @@ def generate_token( request.id, prefill_tokens, Tokens( - [next_token_id_squeezed], - [next_token_logprob], - [next_token_text], - [next_token_id_squeezed.item() in self.all_special_ids], - ), + [next_token_id_squeezed], + [next_token_logprob], + [next_token_text], + [next_token_id_squeezed.item() in self.all_special_ids], + ), generated_text, top_tokens, ) diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py index d06b87eb..3b424f80 100644 --- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py @@ -34,9 +34,10 @@ PositionRotaryEmbedding, TensorParallelHead, get_linear, - FastRMSNorm + FastRMSNorm, ) + class LlamaConfig(PretrainedConfig): def __init__( self, @@ -202,7 +203,7 @@ def forward( ) query = query.view(-1, self.num_heads, self.head_size) kv = kv.view(-1, 2, self.num_key_value_heads, self.head_size) - + self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin) paged_attention.reshape_and_cache( @@ -237,7 +238,7 @@ def forward( input_lengths, max_s, ) - + return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size)) @@ -288,7 +289,9 @@ def __init__(self, layer_id, config, weights): ) self.mlp = LlamaMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) - self.input_layernorm = FastRMSNorm.load(prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps) + self.input_layernorm = FastRMSNorm.load( + prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps + ) self.post_attention_layernorm = FastRMSNorm.load( prefix=f"{prefix}.post_attention_layernorm", weights=weights, diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index 4e56b188..525bf6bc 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -27,7 +27,11 @@ from typing import Optional, List, Tuple from text_generation_server.utils import paged_attention, flash_attn -from text_generation_server.utils.flash_attn import attention, HAS_FLASH_ATTN_V2_ROCM, HAS_FLASH_ATTN_V2_CUDA +from text_generation_server.utils.flash_attn import ( + attention, + HAS_FLASH_ATTN_V2_ROCM, + HAS_FLASH_ATTN_V2_CUDA, +) from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -35,7 +39,7 @@ PositionRotaryEmbedding, TensorParallelHead, get_linear, - FastRMSNorm + FastRMSNorm, ) @@ -96,6 +100,7 @@ def __init__( **kwargs, ) + def load_attention(config, prefix, weights): if config.num_attention_heads != config.num_key_value_heads: return _load_gqa(config, prefix, weights) diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py index 66753d5a..6f5edca2 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py @@ -29,7 +29,10 @@ from typing import Optional, List, Tuple from text_generation_server.utils import paged_attention, flash_attn -from text_generation_server.utils.flash_attn import HAS_FLASH_ATTN_V2_ROCM, HAS_FLASH_ATTN_V2_CUDA +from text_generation_server.utils.flash_attn import ( + HAS_FLASH_ATTN_V2_ROCM, + HAS_FLASH_ATTN_V2_CUDA, +) from text_generation_server.utils.layers import ( FastLinear, FastRMSNorm, @@ -59,28 +62,28 @@ class MixtralConfig(PretrainedConfig): model_type = "mixtral" def __init__( - self, - vocab_size=32000, - hidden_size=4096, - intermediate_size=14336, - num_hidden_layers=32, - num_attention_heads=32, - num_key_value_heads=8, - hidden_act="silu", - max_position_embeddings=4096 * 32, - initializer_range=0.02, - rms_norm_eps=1e-05, - use_cache=True, - pad_token_id=None, - bos_token_id=1, - eos_token_id=2, - pretraining_tp=1, - tie_word_embeddings=False, - rope_theta=10000.0, - sliding_window=4096, - num_experts_per_tok=2, - num_local_experts=8, - **kwargs, + self, + vocab_size=32000, + hidden_size=4096, + intermediate_size=14336, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=8, + hidden_act="silu", + max_position_embeddings=4096 * 32, + initializer_range=0.02, + rms_norm_eps=1e-05, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000.0, + sliding_window=4096, + num_experts_per_tok=2, + num_local_experts=8, + **kwargs, ): self.vocab_size = vocab_size self.max_position_embeddings = max_position_embeddings @@ -166,16 +169,18 @@ def _load_experts(config, prefix, mat, weights): rank = weights.process_group.rank() assert ( - config.intermediate_size % world_size == 0 + config.intermediate_size % world_size == 0 ), f"The chosen size {config.intermediate_size} is not compatible with sharding on {world_size} shards" block_size = config.intermediate_size // world_size start = rank * block_size stop = (rank + 1) * block_size - tensor = torch.empty((config.num_local_experts * block_size, config.hidden_size), - dtype=weights.dtype, - device=weights.device) + tensor = torch.empty( + (config.num_local_experts * block_size, config.hidden_size), + dtype=weights.dtype, + device=weights.device, + ) for i in range(config.num_local_experts): slice_ = weights._get_slice(f"{prefix}.{i}.{mat}.weight") @@ -184,16 +189,18 @@ def _load_experts(config, prefix, mat, weights): expert_slice = slice_[:, start:stop].t().contiguous() else: expert_slice = slice_[start:stop] - tensor[i * block_size:(i + 1) * block_size] = expert_slice.to(dtype=weights.dtype).to(device=weights.device) + tensor[i * block_size : (i + 1) * block_size] = expert_slice.to( + dtype=weights.dtype + ).to(device=weights.device) return tensor class MixtralAttention(torch.nn.Module): def __init__( - self, - prefix: str, - config, - weights, + self, + prefix: str, + config, + weights, ): super().__init__() self.max_past = ( @@ -210,7 +217,7 @@ def __init__( device=weights.device, ) - self.softmax_scale = self.head_size ** -0.5 + self.softmax_scale = self.head_size**-0.5 if self.num_heads % weights.process_group.size() != 0: raise ValueError( @@ -219,7 +226,7 @@ def __init__( ) self.num_heads = self.num_heads // weights.process_group.size() self.num_key_value_heads = ( - config.num_key_value_heads // weights.process_group.size() + config.num_key_value_heads // weights.process_group.size() ) self.query_key_value = load_attention(config, prefix, weights) @@ -236,17 +243,17 @@ def __init__( ).repeat_interleave(self.num_groups) def forward( - self, - hidden_states, - cos, - sin, - cu_seqlen_prefill, - kv_cache, - block_tables, - slots, - input_lengths, - max_s, - prefill_cache_indices, + self, + hidden_states, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, ): qkv = self.query_key_value(hidden_states) query, kv = qkv.split( @@ -399,8 +406,9 @@ def topology(self, x: torch.Tensor, padded_bins: torch.Tensor): # Indices for the sparse matrix. The indices for # the intermediate matrix are dynamic depending # on the mapping of tokens to experts. - column_indices = ops.topology(padded_bins, self.blocking, block_rows, - blocks_per_row) + column_indices = ops.topology( + padded_bins, self.blocking, block_rows, blocks_per_row + ) # For now, use meta init to save the device memory. data = torch.empty( @@ -444,8 +452,7 @@ def indices_and_padded_bins(self, selected_experts: torch.Tensor): # position of each bin. # List of size num_experts - padded_tokens_per_expert = round_up(tokens_per_expert, - self.blocking) + padded_tokens_per_expert = round_up(tokens_per_expert, self.blocking) # padded_tokens_per_expert => [128, O, 128, ...] # Cumulative selected experts per token @@ -484,8 +491,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # Permute tokens and pad to prepare expert computation # (top_k * sequence_length + padding, model_dim) - x = ops.padded_gather(x, indices, bin_ids, bins, padded_bins, - self.top_k) + x = ops.padded_gather(x, indices, bin_ids, bins, padded_bins, self.top_k) # Create the sparse matrix topology with torch.no_grad(): @@ -496,8 +502,8 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # (top_k * sequence_length + padding, ffn_dim * n_experts) x = stk.Matrix( topo.size(), - self.act(stk.ops.sdd(x, self.w1, topo).data) * - stk.ops.sdd(x, self.w3, topo).data, + self.act(stk.ops.sdd(x, self.w1, topo).data) + * stk.ops.sdd(x, self.w3, topo).data, topo.row_indices, topo.column_indices, topo.offsets, @@ -537,7 +543,9 @@ def __init__(self, layer_id, config, weights): self.self_attn = MixtralAttention( prefix=f"{prefix}.self_attn", config=config, weights=weights ) - self.block_sparse_moe = BlockSparseMoE(f"{prefix}.block_sparse_moe", config, weights) + self.block_sparse_moe = BlockSparseMoE( + f"{prefix}.block_sparse_moe", config, weights + ) self.input_layernorm = FastRMSNorm.load( prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps @@ -549,18 +557,18 @@ def __init__(self, layer_id, config, weights): ) def forward( - self, - hidden_states, - residual, - cos, - sin, - cu_seqlen_prefill, - kv_cache, - block_tables, - slots, - input_lengths, - max_s, - prefill_cache_indices, + self, + hidden_states, + residual, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + prefill_cache_indices, ): normed_hidden_states, res = self.input_layernorm(hidden_states, residual) @@ -615,16 +623,16 @@ def __init__(self, config, weights): self.num_key_value_heads = self.layers[0].self_attn.num_key_value_heads def forward( - self, - input_ids: torch.Tensor, - position_ids: torch.Tensor, - cu_seqlen_prefill: Optional[torch.Tensor], - kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], - block_tables: torch.Tensor, - slots: torch.Tensor, - input_lengths: torch.Tensor, - max_s: int, - prefill_cache_indices: Optional[torch.Tensor], + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlen_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + prefill_cache_indices: Optional[torch.Tensor], ) -> torch.Tensor: hidden_states = self.embed_tokens(input_ids) @@ -670,17 +678,17 @@ def __init__(self, config, weights): raise ValueError("max_past cannot be None") def forward( - self, - input_ids: torch.Tensor, - position_ids: torch.Tensor, - cu_seqlen_prefill: Optional[torch.Tensor], - kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], - block_tables: torch.Tensor, - slots: torch.Tensor, - input_lengths: torch.Tensor, - max_s: int, - prefill_cache_indices: Optional[torch.Tensor], - lm_head_indices: Optional[torch.Tensor] = None, + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlen_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + prefill_cache_indices: Optional[torch.Tensor], + lm_head_indices: Optional[torch.Tensor] = None, ) -> torch.Tensor: if prefill_cache_indices is not None: # Slots also need to be sliced as it has the same size as the whole kv tensor diff --git a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py index 4760ae6f..e323d365 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py +++ b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py @@ -198,7 +198,9 @@ def fetch_images(self, image_url_or_urls: Union[str, List[str]]): image = image_url_or_urls if image.startswith("http://") or image.startswith("https://"): - response = requests.get(image_url_or_urls, stream=True, headers=headers, timeout=(1, 5)) + response = requests.get( + image_url_or_urls, stream=True, headers=headers, timeout=(1, 5) + ) response.raise_for_status() content = response.content elif image.startswith("data:"): @@ -213,7 +215,7 @@ def fetch_images(self, image_url_or_urls: Union[str, List[str]]): image = Image.open(BytesIO(content)) # image.verify() except Exception: - raise ValueError(f"Could not load image from url {image_url_or_urls}") + raise ValueError(f"Could not load image from url {image_url_or_urls}") return image else: raise ValueError( diff --git a/server/text_generation_server/models/custom_modeling/idefics_modeling.py b/server/text_generation_server/models/custom_modeling/idefics_modeling.py index 946f7683..555bf5af 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_modeling.py +++ b/server/text_generation_server/models/custom_modeling/idefics_modeling.py @@ -62,6 +62,7 @@ elif IS_ROCM_SYSTEM: from vllm import layernorm_ops + @dataclass class BaseModelOutputWithPastImage(BaseModelOutputWithPast): image_hidden_states: Optional[torch.FloatTensor] = None @@ -431,7 +432,9 @@ def forward(self, hidden_states, residual=None): return out else: - raise ValueError("Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") + raise ValueError( + "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction." + ) # this was adapted from LlamaMLP @@ -613,8 +616,13 @@ def forward( query_shape = query_states.shape key_shape = key_states.shape - self.rotary_emb(query_states.view(-1, *query_shape[2:]), key_states.reshape(-1, *key_shape[2:]), cos, sin) - + self.rotary_emb( + query_states.view(-1, *query_shape[2:]), + key_states.reshape(-1, *key_shape[2:]), + cos, + sin, + ) + query_states = query_states.view(query_shape) key_states = key_states.view(key_shape) diff --git a/server/text_generation_server/models/custom_modeling/idefics_processing.py b/server/text_generation_server/models/custom_modeling/idefics_processing.py index 98e43a27..beca864b 100644 --- a/server/text_generation_server/models/custom_modeling/idefics_processing.py +++ b/server/text_generation_server/models/custom_modeling/idefics_processing.py @@ -112,6 +112,7 @@ def is_url(string): result = urlparse(string) return all([result.scheme, result.netloc]) + def is_image(string): """Checks if the passed string contains a valid url and nothing else. e.g. if space is included it's immediately invalidated the url""" @@ -344,7 +345,6 @@ def image_tokens(last_was_image): image_objects = self.image_processor(image_objects, transform=transform) - text_encoding = self.tokenizer( text=full_text, add_special_tokens=False, diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 79344ea1..14d30635 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -11,7 +11,7 @@ from transformers import PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type, Union, Dict -from text_generation_server.models import Model +from text_generation_server.models import Model from text_generation_server.utils.speculate import get_speculate from text_generation_server.models.types import ( Batch, @@ -165,8 +165,6 @@ def from_pb( input_length = len(tokenized_input) input_lengths.append(input_length) - - prefix_offsets.append(input_length - 5) read_offsets.append(input_length) @@ -229,7 +227,9 @@ def from_pb( cumulative_max_length += total_tokens max_seqlen = max(max_seqlen, input_length) max_blocks = max(max_blocks, needed_blocks) - max_length = max(max_length, input_length + max_new_tokens + speculative_length) + max_length = max( + max_length, input_length + max_new_tokens + speculative_length + ) next_token_chooser = HeterogeneousNextTokenChooser.from_pb( next_token_chooser_parameters, dtype, device @@ -424,7 +424,9 @@ def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch": slots = self.slots[slot_filtering_indices] next_token_chooser = self.next_token_chooser.filter(indices) top_n_tokens_tensor = self.top_n_tokens_tensor[indices] - speculative_ids = self.speculative_ids[indices] if self.speculative_ids is not None else None + speculative_ids = ( + self.speculative_ids[indices] if self.speculative_ids is not None else None + ) start_slots = torch.tensor(start_slots, dtype=torch.int64) @@ -480,7 +482,9 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch total_batch_size += len(b) total_slots += len(b.slots) blocks += b.blocks - speculative_length = b.speculative_ids.shape[1] if b.speculative_ids is not None else 0 + speculative_length = ( + b.speculative_ids.shape[1] if b.speculative_ids is not None else 0 + ) max_blocks = max(max_blocks, b.max_blocks) max_seqlen = max(max_seqlen, b.max_seqlen) max_length = max( @@ -586,7 +590,11 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch device=batches[0].next_token_chooser.device, ) - speculative_ids = torch.cat([b.speculative_ids for b in batches], dim=0) if batches[0].speculative_ids is not None else None + speculative_ids = ( + torch.cat([b.speculative_ids for b in batches], dim=0) + if batches[0].speculative_ids is not None + else None + ) # Needed to avoid dropping blocks when the batches will go out of scope for b in batches: @@ -622,7 +630,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch top_n_tokens_tensor=top_n_tokens_tensor, blocks=blocks, max_blocks=max_blocks, - speculative_ids=speculative_ids + speculative_ids=speculative_ids, ) def __del__(self): @@ -727,43 +735,54 @@ def warmup(self, batch: FlashCausalLMBatch): def forward(self, batch: FlashCausalLMBatch) -> Tuple[torch.Tensor, torch.Tensor]: # Model Forward if batch.speculative_ids is not None: - input_ids=batch.input_ids - position_ids=batch.position_ids - cu_seqlen_prefill=batch.cu_seqlen_prefill - kv_cache=get_cache_manager().kv_cache - block_tables=batch.block_tables_tensor - slots=batch.slots[batch.slot_indices] - input_lengths=batch.input_lengths_tensor - max_s=batch.max_seqlen - lm_head_indices=batch.prefill_head_indices + input_ids = batch.input_ids + position_ids = batch.position_ids + cu_seqlen_prefill = batch.cu_seqlen_prefill + kv_cache = get_cache_manager().kv_cache + block_tables = batch.block_tables_tensor + slots = batch.slots[batch.slot_indices] + input_lengths = batch.input_lengths_tensor + max_s = batch.max_seqlen + lm_head_indices = batch.prefill_head_indices speculative_ids = batch.speculative_ids - B, speculative_length = speculative_ids.shape + B, speculative_length = speculative_ids.shape new_length = speculative_length + 1 - new_input_ids = torch.cat([input_ids.unsqueeze(-1), speculative_ids], dim=1).reshape(-1) + new_input_ids = torch.cat( + [input_ids.unsqueeze(-1), speculative_ids], dim=1 + ).reshape(-1) arange = torch.arange(new_length, device=position_ids.device).unsqueeze(0) arange_int = arange.to(dtype=torch.int32) - new_position_ids = (position_ids.unsqueeze(-1).expand(B, new_length) + arange).view(-1) + new_position_ids = ( + position_ids.unsqueeze(-1).expand(B, new_length) + arange + ).view(-1) slots = (slots.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1) - input_lengths = (input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1) + input_lengths = ( + input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int + ).view(-1) # Add Copy the block tables for all members - block_tables = block_tables.unsqueeze(1).expand(B, new_length, -1).reshape(B* new_length, -1).contiguous() + block_tables = ( + block_tables.unsqueeze(1) + .expand(B, new_length, -1) + .reshape(B * new_length, -1) + .contiguous() + ) max_s = max_s + speculative_length input_ids = new_input_ids position_ids = new_position_ids else: - input_ids=batch.input_ids - position_ids=batch.position_ids - cu_seqlen_prefill=batch.cu_seqlen_prefill - kv_cache=get_cache_manager().kv_cache - block_tables=batch.block_tables_tensor - slots=batch.slots[batch.slot_indices] - input_lengths=batch.input_lengths_tensor - max_s=batch.max_seqlen - lm_head_indices=batch.prefill_head_indices + input_ids = batch.input_ids + position_ids = batch.position_ids + cu_seqlen_prefill = batch.cu_seqlen_prefill + kv_cache = get_cache_manager().kv_cache + block_tables = batch.block_tables_tensor + slots = batch.slots[batch.slot_indices] + input_lengths = batch.input_lengths_tensor + max_s = batch.max_seqlen + lm_head_indices = batch.prefill_head_indices return self.model.forward( input_ids=input_ids, @@ -808,20 +827,31 @@ def generate_token( else: speculative_logits = None - if prefill: next_token_logits = ( out[batch.prefill_next_token_indices] if prefill_logprobs else out ) if speculative_logits is not None: speculative_logits = ( - speculative_logits[batch.prefill_next_token_indices] if prefill_logprobs else speculative_logits + speculative_logits[batch.prefill_next_token_indices] + if prefill_logprobs + else speculative_logits ) else: next_token_logits = out - next_input_ids, next_token_logprobs, logprobs, accepted_ids, speculative_ids = batch.next_token_chooser( - batch.all_input_ids_tensor[:, : batch.max_seqlen], next_token_logits, get_speculate(), batch.speculative_ids, speculative_logits + ( + next_input_ids, + next_token_logprobs, + logprobs, + accepted_ids, + speculative_ids, + ) = batch.next_token_chooser( + batch.all_input_ids_tensor[:, : batch.max_seqlen], + next_token_logits, + get_speculate(), + batch.speculative_ids, + speculative_logits, ) batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( @@ -851,11 +881,7 @@ def generate_token( stopped = True # Zipped iterator - iterator = zip( - batch.input_lengths, - batch.all_input_ids, - accepted_ids - ) + iterator = zip(batch.input_lengths, batch.all_input_ids, accepted_ids) # We do two for loops as the first one can run completely asynchronously from the GPU while for the second # one, we need to first do a GPU <-> CPU sync @@ -863,11 +889,7 @@ def generate_token( # For each member of the batch index = 0 - for i, ( - input_length, - all_input_ids, - n_accepted_ids - ) in enumerate(iterator): + for i, (input_length, all_input_ids, n_accepted_ids) in enumerate(iterator): # Indexing metadata start_index = cumulative_length end_index = cumulative_length + input_length @@ -901,7 +923,6 @@ def generate_token( cumulative_length += input_length - batch.input_ids = next_input_ids[accepted_ids.cumsum(dim=-1) - 1] batch.speculative_ids = speculative_ids batch.position_ids = next_position_ids + accepted_ids @@ -983,8 +1004,10 @@ def generate_token( current_stopped = False stopped = stopped and current_stopped - _next_token_ids = next_token_ids[index: index+n_accepted_ids - left] - _next_token_logprobs = next_token_logprobs[index: index+n_accepted_ids - left] + _next_token_ids = next_token_ids[index : index + n_accepted_ids - left] + _next_token_logprobs = next_token_logprobs[ + index : index + n_accepted_ids - left + ] index += n_accepted_ids # Shard generations @@ -1027,7 +1050,10 @@ def generate_token( ) prefill_tokens = Tokens( - prefill_token_ids, request_prefill_logprobs, prefill_texts, is_special = [] + prefill_token_ids, + request_prefill_logprobs, + prefill_texts, + is_special=[], ) else: prefill_tokens = None diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 3a84b1b6..2415a245 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -71,12 +71,19 @@ def __init__( from text_generation_server.utils.medusa import MedusaModel from huggingface_hub import hf_hub_download import json - medusa_config = hf_hub_download(use_medusa, revision=revision, filename="config.json") + + medusa_config = hf_hub_download( + use_medusa, revision=revision, filename="config.json" + ) with open(medusa_config, "r") as f: config = json.load(f) - medusa_head = hf_hub_download(use_medusa, revision=revision, filename="medusa_lm_head.pt") - medusa_sf = medusa_head[:-len(".pt")] + ".safetensors" - weights = Weights([medusa_sf], device, dtype, process_group=self.process_group) + medusa_head = hf_hub_download( + use_medusa, revision=revision, filename="medusa_lm_head.pt" + ) + medusa_sf = medusa_head[: -len(".pt")] + ".safetensors" + weights = Weights( + [medusa_sf], device, dtype, process_group=self.process_group + ) lm_head = model.lm_head model.lm_head = MedusaModel(config, weights, lm_head) diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py index 5ce37164..0fad5aa8 100644 --- a/server/text_generation_server/models/flash_mistral.py +++ b/server/text_generation_server/models/flash_mistral.py @@ -45,11 +45,11 @@ class FlashMistralBatch(FlashCausalLMBatch): @classmethod def from_pb( - cls, - pb: generate_pb2.Batch, - tokenizer: PreTrainedTokenizerBase, - dtype: torch.dtype, - device: torch.device, + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, + device: torch.device, ) -> "FlashCausalLMBatch": global SLIDING_WINDOW global SLIDING_WINDOW_BLOCKS @@ -99,12 +99,12 @@ def from_pb( # Parse batch for i, (r, tokenized_input) in enumerate( - zip(pb.requests, batch_tokenized_inputs) + zip(pb.requests, batch_tokenized_inputs) ): # request id -> idx in list mapping requests_idx_mapping[r.id] = i - tokenized_input = tokenized_input[-r.truncate:] + tokenized_input = tokenized_input[-r.truncate :] input_length = len(tokenized_input) input_lengths.append(input_length) @@ -184,7 +184,9 @@ def from_pb( cumulative_max_length += total_tokens max_seqlen = max(max_seqlen, input_length) max_blocks = max(max_blocks, needed_blocks) - max_length = max(max_length, input_length + max_new_tokens + speculative_length) + max_length = max( + max_length, input_length + max_new_tokens + speculative_length + ) next_token_chooser = HeterogeneousNextTokenChooser.from_pb( next_token_chooser_parameters, dtype, device @@ -273,20 +275,20 @@ def from_pb( blocks=blocks, max_blocks=max_blocks, prefill_cache_indices=prefill_cache_indices, - speculative_ids=None + speculative_ids=None, ) class BaseFlashMistral(FlashCausalLM): def __init__( - self, - config_cls, - model_cls, - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - dtype: Optional[torch.dtype] = None, - trust_remote_code: bool = False, + self, + config_cls, + model_cls, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, ): global SLIDING_WINDOW global SLIDING_WINDOW_BLOCKS @@ -345,43 +347,54 @@ def batch_type(self) -> Type[FlashMistralBatch]: def forward(self, batch: FlashMistralBatch) -> Tuple[torch.Tensor, torch.Tensor]: # Model Forward if batch.speculative_ids is not None: - input_ids=batch.input_ids - position_ids=batch.position_ids - cu_seqlen_prefill=batch.cu_seqlen_prefill - kv_cache=get_cache_manager().kv_cache - block_tables=batch.block_tables_tensor - slots=batch.slots[batch.slot_indices] - input_lengths=batch.input_lengths_tensor - max_s=batch.max_seqlen - lm_head_indices=batch.prefill_head_indices + input_ids = batch.input_ids + position_ids = batch.position_ids + cu_seqlen_prefill = batch.cu_seqlen_prefill + kv_cache = get_cache_manager().kv_cache + block_tables = batch.block_tables_tensor + slots = batch.slots[batch.slot_indices] + input_lengths = batch.input_lengths_tensor + max_s = batch.max_seqlen + lm_head_indices = batch.prefill_head_indices speculative_ids = batch.speculative_ids - B, speculative_length = speculative_ids.shape + B, speculative_length = speculative_ids.shape new_length = speculative_length + 1 - new_input_ids = torch.cat([input_ids.unsqueeze(-1), speculative_ids], dim=1).reshape(-1) + new_input_ids = torch.cat( + [input_ids.unsqueeze(-1), speculative_ids], dim=1 + ).reshape(-1) arange = torch.arange(new_length, device=position_ids.device).unsqueeze(0) arange_int = arange.to(dtype=torch.int32) - new_position_ids = (position_ids.unsqueeze(-1).expand(B, new_length) + arange).view(-1) + new_position_ids = ( + position_ids.unsqueeze(-1).expand(B, new_length) + arange + ).view(-1) slots = (slots.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1) - input_lengths = (input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1) + input_lengths = ( + input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int + ).view(-1) # Add Copy the block tables for all members - block_tables = block_tables.unsqueeze(1).expand(B, new_length, -1).reshape(B* new_length, -1).contiguous() + block_tables = ( + block_tables.unsqueeze(1) + .expand(B, new_length, -1) + .reshape(B * new_length, -1) + .contiguous() + ) max_s = max_s + speculative_length input_ids = new_input_ids position_ids = new_position_ids else: - input_ids=batch.input_ids - position_ids=batch.position_ids - cu_seqlen_prefill=batch.cu_seqlen_prefill - kv_cache=get_cache_manager().kv_cache - block_tables=batch.block_tables_tensor - slots=batch.slots[batch.slot_indices] - input_lengths=batch.input_lengths_tensor - max_s=batch.max_seqlen - lm_head_indices=batch.prefill_head_indices + input_ids = batch.input_ids + position_ids = batch.position_ids + cu_seqlen_prefill = batch.cu_seqlen_prefill + kv_cache = get_cache_manager().kv_cache + block_tables = batch.block_tables_tensor + slots = batch.slots[batch.slot_indices] + input_lengths = batch.input_lengths_tensor + max_s = batch.max_seqlen + lm_head_indices = batch.prefill_head_indices logits = self.model.forward( input_ids=input_ids, position_ids=position_ids, @@ -401,12 +414,12 @@ def forward(self, batch: FlashMistralBatch) -> Tuple[torch.Tensor, torch.Tensor] class FlashMistral(BaseFlashMistral): def __init__( - self, - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - dtype: Optional[torch.dtype] = None, - trust_remote_code: bool = False, + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, ): super(FlashMistral, self).__init__( config_cls=MistralConfig, @@ -415,5 +428,5 @@ def __init__( revision=revision, quantize=quantize, dtype=dtype, - trust_remote_code=trust_remote_code + trust_remote_code=trust_remote_code, ) diff --git a/server/text_generation_server/models/flash_mixtral.py b/server/text_generation_server/models/flash_mixtral.py index c45ae50f..6f77a658 100644 --- a/server/text_generation_server/models/flash_mixtral.py +++ b/server/text_generation_server/models/flash_mixtral.py @@ -3,17 +3,20 @@ from typing import Optional from text_generation_server.models.flash_mistral import BaseFlashMistral -from text_generation_server.models.custom_modeling.flash_mixtral_modeling import MixtralConfig, FlashMixtralForCausalLM +from text_generation_server.models.custom_modeling.flash_mixtral_modeling import ( + MixtralConfig, + FlashMixtralForCausalLM, +) class FlashMixtral(BaseFlashMistral): def __init__( - self, - model_id: str, - revision: Optional[str] = None, - quantize: Optional[str] = None, - dtype: Optional[torch.dtype] = None, - trust_remote_code: bool = False, + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, ): super(FlashMixtral, self).__init__( config_cls=MixtralConfig, @@ -22,5 +25,5 @@ def __init__( revision=revision, quantize=quantize, dtype=dtype, - trust_remote_code=trust_remote_code + trust_remote_code=trust_remote_code, ) diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py index 2f4bb139..86389ad2 100644 --- a/server/text_generation_server/models/idefics_causal_lm.py +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -792,7 +792,10 @@ def generate_token( skip_special_tokens=False, ) prefill_tokens = Tokens( - prefill_token_ids, prefill_logprobs, prefill_texts, is_special=[] + prefill_token_ids, + prefill_logprobs, + prefill_texts, + is_special=[], ) else: prefill_tokens = None @@ -803,10 +806,10 @@ def generate_token( request.id, prefill_tokens, Tokens( - [next_token_id_squeezed], - [next_token_logprob], - [next_token_text], - [next_token_id_squeezed.item() in self.all_special_ids], + [next_token_id_squeezed], + [next_token_logprob], + [next_token_text], + [next_token_id_squeezed.item() in self.all_special_ids], ), generated_text, top_tokens, diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index 8552960d..dfb21dcb 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -56,7 +56,7 @@ def info(self) -> InfoResponse: dtype=str(self.dtype), device_type=self.device.type, window_size=self.sliding_window, - speculate=self.speculate + speculate=self.speculate, ) @property diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 279b5505..a85ef58e 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -736,7 +736,7 @@ def generate_token( [self.tokenizer.bos_token_id], [float("nan")], [self.tokenizer.bos_token], - [False] + [False], ) else: prefill_tokens = None @@ -763,10 +763,10 @@ def generate_token( request.id, prefill_tokens, Tokens( - [next_token_id_squeezed], - [next_token_logprob], - [next_token_text], - [next_token_id_squeezed.item() in self.all_special_ids], + [next_token_id_squeezed], + [next_token_logprob], + [next_token_text], + [next_token_id_squeezed.item() in self.all_special_ids], ), generated_text, top_tokens, diff --git a/server/text_generation_server/models/types.py b/server/text_generation_server/models/types.py index 87c03d63..f85f27e5 100644 --- a/server/text_generation_server/models/types.py +++ b/server/text_generation_server/models/types.py @@ -66,7 +66,10 @@ class Tokens: def to_pb(self) -> generate_pb2.Tokens: return generate_pb2.Tokens( - ids=self.token_ids, logprobs=self.logprobs, texts=self.texts, is_special=self.is_special + ids=self.token_ids, + logprobs=self.logprobs, + texts=self.texts, + is_special=self.is_special, ) def __len__(self): diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index ebe066e3..75dba972 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -159,7 +159,13 @@ async def serve_inner( try: model = get_model( - model_id, revision, sharded, quantize, speculate, dtype, trust_remote_code + model_id, + revision, + sharded, + quantize, + speculate, + dtype, + trust_remote_code, ) except Exception: logger.exception("Error when initializing model") @@ -207,5 +213,7 @@ async def serve_inner( await server.stop(0) asyncio.run( - serve_inner(model_id, revision, sharded, quantize, speculate, dtype, trust_remote_code) + serve_inner( + model_id, revision, sharded, quantize, speculate, dtype, trust_remote_code + ) ) diff --git a/server/text_generation_server/utils/flash_attn.py b/server/text_generation_server/utils/flash_attn.py index aca95e11..3237df82 100644 --- a/server/text_generation_server/utils/flash_attn.py +++ b/server/text_generation_server/utils/flash_attn.py @@ -51,7 +51,9 @@ ) from e elif IS_ROCM_SYSTEM: for idx in range(torch.cuda.device_count()): - if "MI210" not in torch.cuda.get_device_name(idx) and "MI250" not in torch.cuda.get_device_name(idx): + if "MI210" not in torch.cuda.get_device_name( + idx + ) and "MI250" not in torch.cuda.get_device_name(idx): raise ImportError( f"AMD GPU {torch.cuda.get_device_name(idx)} does not support flash-attention" ) @@ -91,8 +93,10 @@ def attention( ) elif HAS_FLASH_ATTN_V2_ROCM: if window_size_left != -1: - raise ValueError(f"RoCm version of Flash Attention v2 does not support window attention (window_size_left != -1, got window_size_left={window_size_left}).") - + raise ValueError( + f"RoCm version of Flash Attention v2 does not support window attention (window_size_left != -1, got window_size_left={window_size_left})." + ) + # RoCm flash API does not take the window_size_left and window_size_right arguments. return flash_attn_2_cuda.varlen_fwd( q, diff --git a/server/text_generation_server/utils/gptq/exllamav2.py b/server/text_generation_server/utils/gptq/exllamav2.py index 1945338b..f820f0d9 100644 --- a/server/text_generation_server/utils/gptq/exllamav2.py +++ b/server/text_generation_server/utils/gptq/exllamav2.py @@ -11,40 +11,44 @@ try: from exllamav2_kernels import make_q_matrix, gemm_half_q_half except ImportError: - logger.error('exllamav2_kernels not installed.') + logger.error("exllamav2_kernels not installed.") raise # Dummy tensor to pass instead of g_idx since there is no way to pass "None" to a C++ extension none_tensor = torch.empty((1, 1), device="meta") + def ext_gemm_half_q_half(x, q_handle, q4_width, force_cuda): """Matrix multiplication, returns x @ q4""" output_shape = x.shape[:-1] + (q4_width,) x = x.view(-1, x.shape[-1]) - output = torch.empty((x.shape[0], q4_width), dtype = torch.half, device = x.device) + output = torch.empty((x.shape[0], q4_width), dtype=torch.half, device=x.device) gemm_half_q_half(x, q_handle, output, force_cuda) return output.view(output_shape) + def ext_make_q_matrix(w: dict, temp_dq, key: str = None): """ - Create Q matrix + Create Q matrix """ # EXL2 - # won't work as the moment because the tensors are not the same. + # won't work as the moment because the tensors are not the same. if "q_weight" in w: w["q_scale_max"] /= 256 w["q_perm"] = w["q_perm"].short() w["q_invperm"] = w["q_invperm"].short() - return make_q_matrix(w["q_weight"], - w["q_perm"], - w["q_invperm"], - w["q_scale"], - w["q_scale_max"], - w["q_groups"], - none_tensor, - none_tensor, - none_tensor, - temp_dq) + return make_q_matrix( + w["q_weight"], + w["q_perm"], + w["q_invperm"], + w["q_scale"], + w["q_scale_max"], + w["q_groups"], + none_tensor, + none_tensor, + none_tensor, + temp_dq, + ) # GPTQ elif "qweight" in w: if w["scales"].dtype == torch.float: @@ -52,31 +56,40 @@ def ext_make_q_matrix(w: dict, temp_dq, key: str = None): # GPTQ with g_idx (act_order) if w.get("g_idx", None) is not None and not (w["g_idx"] == 0).all().item(): - w["q_perm"] = torch.empty((w["qweight"].shape[0] * 8,), dtype = torch.short, device = w["qweight"].device) + w["q_perm"] = torch.empty( + (w["qweight"].shape[0] * 8,), + dtype=torch.short, + device=w["qweight"].device, + ) w["q_invperm"] = torch.empty_like(w["q_perm"]) # make_q4 segfaults if g_idx is not on cpu in the act-order case. In the non act-order case, None needs to be passed for g_idx. - return make_q_matrix(w["qweight"], - w["q_perm"], - w["q_invperm"], - none_tensor, - none_tensor, - none_tensor, - w["qzeros"], - w["scales"], - w["g_idx"].cpu(), - temp_dq) + return make_q_matrix( + w["qweight"], + w["q_perm"], + w["q_invperm"], + none_tensor, + none_tensor, + none_tensor, + w["qzeros"], + w["scales"], + w["g_idx"].cpu(), + temp_dq, + ) # GPTQ without g_idx else: - return make_q_matrix(w["qweight"], - none_tensor, - none_tensor, - none_tensor, - none_tensor, - none_tensor, - w["qzeros"], - w["scales"], - none_tensor, - temp_dq) + return make_q_matrix( + w["qweight"], + none_tensor, + none_tensor, + none_tensor, + none_tensor, + none_tensor, + w["qzeros"], + w["scales"], + none_tensor, + temp_dq, + ) + DEVICE = None FIXED_BYTES = 0 @@ -106,14 +119,15 @@ def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): super().__init__() if bits != 4: raise ValueError( - f"Exllamav2 kernel supports only bits=4, requested bits={bits}. Something is wrong in the model initialization.") + f"Exllamav2 kernel supports only bits=4, requested bits={bits}. Something is wrong in the model initialization." + ) self.q_handle = None self.q_tensors = None self.bits = bits - self.maxq = 2 ** self.bits - 1 + self.maxq = 2**self.bits - 1 self.infeatures = qweight.shape[0] // self.bits * 32 self.outfeatures = qweight.shape[1] - self.padding = - self.outfeatures % 32 + self.padding = -self.outfeatures % 32 self.outfeatures = self.outfeatures + self.padding self.device = qweight.device @@ -128,9 +142,12 @@ def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): outfeatures = self.outfeatures assert qweight.shape == (infeatures // 32 * self.bits, outfeatures) assert infeatures % self.group_size == 0 - assert qzeros.shape == (infeatures // self.group_size, outfeatures // 32 * self.bits) + assert qzeros.shape == ( + infeatures // self.group_size, + outfeatures // 32 * self.bits, + ) assert scales.shape == (infeatures // self.group_size, outfeatures) - assert g_idx.shape == (infeatures, ), f"{g_idx.shape}, {infeatures}" + assert g_idx.shape == (infeatures,), f"{g_idx.shape}, {infeatures}" global FIXED_BYTES, LAYERS FIXED_BYTES = max(FIXED_BYTES, self.scratch_space_fixed()) @@ -140,33 +157,31 @@ def post_init(self, temp_dq): assert self.qweight.device.type == "cuda" assert self.qweight.device.index is not None self.q_tensors = { - "qweight":self.qweight, - "qzeros":self.qzeros, - "scales":self.scales, - "g_idx":self.g_idx + "qweight": self.qweight, + "qzeros": self.qzeros, + "scales": self.scales, + "g_idx": self.g_idx, } temp_dq = temp_dq.get_scratch_slice(self.temp_dq_size()) - self.q_handle = ext_make_q_matrix( - self.q_tensors, temp_dq - ) - - def forward(self, x, force_cuda = False): + self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq) + + def forward(self, x, force_cuda=False): output = ext_gemm_half_q_half(x, self.q_handle, self.outfeatures, force_cuda) if self.bias is not None: output.add_(self.bias) return output - + def temp_dq_size(self): return self.infeatures * self.outfeatures * 2 + 128 - + def temp_fwd_size(self, max_input_len, max_batch_size): return self.outfeatures * max_input_len * max_batch_size * 4 + 128 - + def scratch_space_fixed(self, max_input_len=4096, max_batch_size=16): return self.temp_dq_size() + self.temp_fwd_size(max_input_len, max_batch_size) - - + + class ExLlamaV2DeviceTensors: device_idx: int @@ -177,13 +192,16 @@ class ExLlamaV2DeviceTensors: def __init__(self, device, scratch_bytes): self.device = device self.scratch_bytes = scratch_bytes - + def prepare(self): - self.scratch = torch.empty((self.scratch_bytes // 2,), dtype = torch.half, device = self.device) + self.scratch = torch.empty( + (self.scratch_bytes // 2,), dtype=torch.half, device=self.device + ) def get_scratch_slice(self, size_bytes): - if self.scratch is None: self.prepare() + if self.scratch is None: + self.prepare() size_bytes = ((size_bytes + 127) // 128) * 128 size_half = size_bytes // 2 diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index d533016d..77e2fdb6 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -35,7 +35,9 @@ CAN_EXLLAMA = major >= 8 V2 = os.getenv("EXLLAMA_VERSION", "2") == "2" if V2 and int(os.getenv("WORLD_SIZE", "1")) > 1: - logger.warning("Disabling exllama v2 and using v1 instead because there are issues when sharding") + logger.warning( + "Disabling exllama v2 and using v1 instead because there are issues when sharding" + ) V2 = False if os.getenv("DISABLE_EXLLAMA") == "True": @@ -43,17 +45,19 @@ elif CAN_EXLLAMA: try: if V2: - from text_generation_server.utils.gptq.exllamav2 import (QuantLinear as ExllamaQuantLinear, - create_exllama_buffers, - set_device, - ) + from text_generation_server.utils.gptq.exllamav2 import ( + QuantLinear as ExllamaQuantLinear, + create_exllama_buffers, + set_device, + ) HAS_EXLLAMA = "2" else: - from text_generation_server.utils.gptq.exllama import (Ex4bitLinear as ExllamaQuantLinear, - create_exllama_buffers, - set_device, - ) + from text_generation_server.utils.gptq.exllama import ( + Ex4bitLinear as ExllamaQuantLinear, + create_exllama_buffers, + set_device, + ) HAS_EXLLAMA = "1" @@ -114,7 +118,7 @@ def load_conv2d(cls, prefix, weights, in_channels, out_channels, kernel_size, st @classmethod def load_conv2d_no_bias( - cls, prefix, weights, in_channels, out_channels, kernel_size, stride + cls, prefix, weights, in_channels, out_channels, kernel_size, stride ): weight = weights.get_tensor(f"{prefix}.weight") with init_empty_weights(): @@ -138,9 +142,9 @@ def load_conv2d_no_bias( class FastLinear(nn.Module): def __init__( - self, - weight, - bias, + self, + weight, + bias, ) -> None: super().__init__() self.weight = nn.Parameter(weight) @@ -164,9 +168,9 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: class EETQLinear(nn.Module): def __init__( - self, - weight, - bias, + self, + weight, + bias, ) -> None: super().__init__() device = weight.device @@ -185,13 +189,13 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: class Linear8bitLt(nn.Module): def __init__( - self, - weight, - bias, - has_fp16_weights=True, - memory_efficient_backward=False, - threshold=0.0, - index=None, + self, + weight, + bias, + has_fp16_weights=True, + memory_efficient_backward=False, + threshold=0.0, + index=None, ): super().__init__() assert ( @@ -325,7 +329,9 @@ def get_linear(weight, bias, quantize): ) if use_exllama: - linear = ExllamaQuantLinear(qweight, qzeros, scales, g_idx, bias, bits, groupsize) + linear = ExllamaQuantLinear( + qweight, qzeros, scales, g_idx, bias, bits, groupsize + ) else: linear = QuantLinear( qweight, @@ -533,7 +539,6 @@ def forward(self, input: torch.Tensor) -> torch.Tensor: else: dropout_layer_norm = None - class FastLayerNorm(nn.LayerNorm): def forward(self, hidden_states, residual=None): if hidden_states.shape[-1] > 8192 or IS_ROCM_SYSTEM: @@ -569,7 +574,6 @@ def forward(self, hidden_states, residual=None): return normed_hidden_states, residual - class FastRMSNorm(nn.Module): def __init__(self, weight: torch.Tensor, eps: float): super().__init__() @@ -601,7 +605,11 @@ def forward(self, hidden_states, residual=None): return self.weight * hidden_states, residual elif IS_CUDA_SYSTEM: # faster post attention rms norm - normed_hidden_states, res, *rest = dropout_layer_norm.dropout_add_ln_fwd( + ( + normed_hidden_states, + res, + *rest, + ) = dropout_layer_norm.dropout_add_ln_fwd( hidden_states, residual, self.weight, @@ -638,7 +646,8 @@ def forward(self, hidden_states, residual=None): return out, residual else: raise ValueError( - "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") + "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction." + ) except ImportError: pass @@ -650,14 +659,12 @@ def forward(self, hidden_states, residual=None): elif IS_ROCM_SYSTEM: from vllm import pos_encoding_ops - def _create_inv_freq(dim, base, device): inv_freq = 1.0 / ( - base ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) + base ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) ) return inv_freq - def _get_rope_config(config): if os.getenv("ROPE_SCALING", None) is not None: rope_scaling = { @@ -667,7 +674,6 @@ def _get_rope_config(config): return rope_scaling return getattr(config, "rope_scaling", None) - class PositionRotaryEmbedding(nn.Module): def __init__(self, inv_freq, scaling_factor): super().__init__() @@ -680,17 +686,23 @@ def __init__(self, inv_freq, scaling_factor): self.scaling_factor = scaling_factor self.dynamic_args = None - def forward(self, query: torch.Tensor, key: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor): + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + ): # Such controlflows may add some overhead. if IS_CUDA_SYSTEM: rotary_dim = cos.shape[-1] q1 = query[..., :rotary_dim] - q2 = query[..., rotary_dim: 2 * rotary_dim] + q2 = query[..., rotary_dim : 2 * rotary_dim] rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False) k1 = key[..., :rotary_dim] - k2 = key[..., rotary_dim: 2 * rotary_dim] + k2 = key[..., rotary_dim : 2 * rotary_dim] rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False) elif IS_ROCM_SYSTEM: @@ -700,17 +712,11 @@ def forward(self, query: torch.Tensor, key: torch.Tensor, cos: torch.Tensor, sin head_size = query.shape[-1] # Inplace operation, updating query and key. - pos_encoding_ops.rotary_embedding( - query, - key, - head_size, - cos, - sin, - True - ) + pos_encoding_ops.rotary_embedding(query, key, head_size, cos, sin, True) else: raise ValueError( - "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction.") + "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction." + ) @classmethod def static(cls, config, dim, base, device): @@ -732,15 +738,16 @@ def static(cls, config, dim, base, device): elif rope_scaling["type"] == "yarn": return YarnPositionRotaryEmbedding( dim=2 * inv_freq.shape[0], - max_position_embeddings=rope_scaling["original_max_position_embeddings"], + max_position_embeddings=rope_scaling[ + "original_max_position_embeddings" + ], base=10000.0, device=inv_freq.device, scaling_factor=scaling_factor, extrapolation_factor=1, attn_factor=1, beta_fast=32, - beta_slow=1 - + beta_slow=1, ) else: raise NotImplementedError( @@ -773,15 +780,16 @@ def load(cls, config, prefix, weights): elif rope_scaling["type"] == "yarn": return YarnPositionRotaryEmbedding( dim=2 * inv_freq.shape[0], - max_position_embeddings=rope_scaling["original_max_position_embeddings"], + max_position_embeddings=rope_scaling[ + "original_max_position_embeddings" + ], base=10000.0, device=inv_freq.device, scaling_factor=scaling_factor, extrapolation_factor=1, attn_factor=1, beta_fast=32, - beta_slow=1 - + beta_slow=1, ) else: raise NotImplementedError( @@ -793,9 +801,9 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): # Reset the tables if the sequence length has changed, # or if we're on a new device (possibly due to tracing for instance) if ( - seqlen > self._seq_len_cached - or self._cos_cached.device != device - or self._cos_cached.dtype != dtype + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype ): self._seq_len_cached = seqlen t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) @@ -809,7 +817,7 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): self._sin_cached = torch.sin(freqs).to(dtype) def get_cos_sin( - self, position_ids: torch.Tensor, max_s: int, dtype: torch.dtype + self, position_ids: torch.Tensor, max_s: int, dtype: torch.dtype ): """ Return cos and sin for the asked position ids @@ -827,7 +835,6 @@ def get_cos_sin( # Note: this unsqueeze is not necessary on RoCm + VLLM ROPE implementation, but we leave it as is to avoid yet an other controlflow. return cos.unsqueeze(1), sin.unsqueeze(1) - class DynamicPositionRotaryEmbedding(PositionRotaryEmbedding): def __init__(self, dim, max_position_embeddings, base, device, scaling_factor): inv_freq = _create_inv_freq(dim, base, device) @@ -840,14 +847,14 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): # Reset the tables if the sequence length has changed, # or if we're on a new device (possibly due to tracing for instance) if ( - seqlen > self._seq_len_cached - or self._cos_cached.device != device - or self._cos_cached.dtype != dtype + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype ): if seqlen > self.max_position_embeddings: newbase = self.base * ( - (self.scaling_factor * seqlen / self.max_position_embeddings) - - (self.scaling_factor - 1) + (self.scaling_factor * seqlen / self.max_position_embeddings) + - (self.scaling_factor - 1) ) ** (self.dim / (self.dim - 2)) self.inv_freq = _create_inv_freq( self.dim, newbase, self.inv_freq.device @@ -861,24 +868,28 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): self._cos_cached = torch.cos(freqs).to(dtype) self._sin_cached = torch.sin(freqs).to(dtype) - # Inverse dim formula to find dim based on number of rotations import math - - def find_correction_dim(num_rotations, dim, base=10000, max_position_embeddings=2048): - return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / (2 * math.log(base)) - + def find_correction_dim( + num_rotations, dim, base=10000, max_position_embeddings=2048 + ): + return ( + dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi)) + ) / (2 * math.log(base)) # Find dim range bounds based on rotations - def find_correction_range(low_rot, high_rot, dim, base=10000, max_position_embeddings=2048): - low = math.floor(find_correction_dim( - low_rot, dim, base, max_position_embeddings)) - high = math.ceil(find_correction_dim( - high_rot, dim, base, max_position_embeddings)) + def find_correction_range( + low_rot, high_rot, dim, base=10000, max_position_embeddings=2048 + ): + low = math.floor( + find_correction_dim(low_rot, dim, base, max_position_embeddings) + ) + high = math.ceil( + find_correction_dim(high_rot, dim, base, max_position_embeddings) + ) return max(low, 0), min(high, dim - 1) # Clamp values just in case - def linear_ramp_mask(min, max, dim): if min == max: max += 0.001 # Prevent singularity @@ -887,16 +898,25 @@ def linear_ramp_mask(min, max, dim): ramp_func = torch.clamp(linear_func, 0, 1) return ramp_func - def get_mscale(scale=1): if scale <= 1: return 1.0 return 0.1 * math.log(scale) + 1.0 - class YarnPositionRotaryEmbedding(PositionRotaryEmbedding): - def __init__(self, dim, max_position_embeddings, base, device, scaling_factor, *, extrapolation_factor, - attn_factor, beta_fast, beta_slow): + def __init__( + self, + dim, + max_position_embeddings, + base, + device, + scaling_factor, + *, + extrapolation_factor, + attn_factor, + beta_fast, + beta_slow, + ): inv_freq = _create_inv_freq(dim, base, device) super().__init__(inv_freq, scaling_factor) self.dim = dim @@ -906,16 +926,17 @@ def __init__(self, dim, max_position_embeddings, base, device, scaling_factor, * self.attn_factor = attn_factor self.beta_fast = beta_fast self.beta_slow = beta_slow - self.mscale = float(get_mscale( - self.scaling_factor) * self.attn_factor) # Get n-d magnitude scaling corrected for interpolation + self.mscale = float( + get_mscale(self.scaling_factor) * self.attn_factor + ) # Get n-d magnitude scaling corrected for interpolation def _update_cos_sin_cache(self, dtype, device, seqlen): # Reset the tables if the sequence length has changed, # or if we're on a new device (possibly due to tracing for instance) if ( - seqlen > self._seq_len_cached - or self._cos_cached.device != device - or self._cos_cached.dtype != dtype + seqlen > self._seq_len_cached + or self._cos_cached.device != device + or self._cos_cached.dtype != dtype ): if seqlen > self.max_position_embeddings: inv_freq_extrapolation = _create_inv_freq( @@ -923,15 +944,26 @@ def _update_cos_sin_cache(self, dtype, device, seqlen): ) freqs = 1.0 / inv_freq_extrapolation inv_freq_interpolation = 1.0 / (self.scaling_factor * freqs) - low, high = find_correction_range(self.beta_fast, self.beta_slow, self.dim, self.base, - self.max_position_embeddings) - inv_freq_mask = (1 - linear_ramp_mask(low, high, self.dim // 2).float().to( - device)) * self.extrapolation_factor # Get n-d rotational scaling corrected for extrapolation - inv_freq = inv_freq_interpolation * (1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask + low, high = find_correction_range( + self.beta_fast, + self.beta_slow, + self.dim, + self.base, + self.max_position_embeddings, + ) + inv_freq_mask = ( + 1 + - linear_ramp_mask(low, high, self.dim // 2).float().to(device) + ) * self.extrapolation_factor # Get n-d rotational scaling corrected for extrapolation + inv_freq = ( + inv_freq_interpolation * (1 - inv_freq_mask) + + inv_freq_extrapolation * inv_freq_mask + ) self.inv_freq = inv_freq - self.mscale = float(get_mscale( - self.scaling_factor) * self.attn_factor) # Get n-d magnitude scaling corrected for interpolation + self.mscale = float( + get_mscale(self.scaling_factor) * self.attn_factor + ) # Get n-d magnitude scaling corrected for interpolation self._seq_len_cached = seqlen t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype) diff --git a/server/text_generation_server/utils/medusa.py b/server/text_generation_server/utils/medusa.py index 029de122..634119cb 100644 --- a/server/text_generation_server/utils/medusa.py +++ b/server/text_generation_server/utils/medusa.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from text_generation_server.utils.layers import TensorParallelHead, FastLinear + @dataclass class Output: logits: torch.FloatTensor = None @@ -11,7 +12,9 @@ class Output: class ResBlock(torch.nn.Module): def __init__(self, config, prefix, weights): super().__init__() - self.linear = FastLinear.load(config, prefix=f"{prefix}.linear", weights=weights, bias=True) + self.linear = FastLinear.load( + config, prefix=f"{prefix}.linear", weights=weights, bias=True + ) self.act = torch.nn.SiLU() def forward(self, x): @@ -19,15 +22,13 @@ def forward(self, x): class MedusaModel(torch.nn.Module): - def __init__( - self, - config, - weights, - lm_head - ): + def __init__(self, config, weights, lm_head): super().__init__() self.heads = torch.nn.ModuleList( - [MedusaHead(config, prefix=f"{i}", weights=weights) for i in range(config["medusa_num_heads"])] + [ + MedusaHead(config, prefix=f"{i}", weights=weights) + for i in range(config["medusa_num_heads"]) + ] ) self.lm_head = lm_head @@ -40,9 +41,16 @@ def forward(self, x): class MedusaHead(torch.nn.Module): def __init__(self, config, prefix, weights): super().__init__() - self.blocks = torch.nn.ModuleList([ResBlock(config, prefix=f"{prefix}.{i}", weights=weights) for i in range(config["medusa_num_layers"])]) + self.blocks = torch.nn.ModuleList( + [ + ResBlock(config, prefix=f"{prefix}.{i}", weights=weights) + for i in range(config["medusa_num_layers"]) + ] + ) n = len(self.blocks) - self.out = FastLinear.load(config, prefix=f"{prefix}.{n}", weights=weights, bias=False) + self.out = FastLinear.load( + config, prefix=f"{prefix}.{n}", weights=weights, bias=False + ) def forward(self, x): for block in self.blocks: diff --git a/server/text_generation_server/utils/paged_attention.py b/server/text_generation_server/utils/paged_attention.py index 57a59599..4b12744c 100644 --- a/server/text_generation_server/utils/paged_attention.py +++ b/server/text_generation_server/utils/paged_attention.py @@ -7,23 +7,26 @@ _PARTITION_SIZE = 512 -def reshape_and_cache(key: torch.Tensor, value: torch.Tensor, key_cache: torch.Tensor, value_cache: torch.Tensor, - slots: torch.Tensor): - cache_ops.reshape_and_cache( - key, value, key_cache, value_cache, slots - ) +def reshape_and_cache( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slots: torch.Tensor, +): + cache_ops.reshape_and_cache(key, value, key_cache, value_cache, slots) def attention( - out: torch.Tensor, - query: torch.Tensor, - key_cache: torch.Tensor, - value_cache: torch.Tensor, - kv_head_mapping: torch.Tensor, - softmax_scale: float, - block_tables: torch.Tensor, - input_lengths: torch.Tensor, - max_s: int, + out: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + kv_head_mapping: torch.Tensor, + softmax_scale: float, + block_tables: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, ): # Adapted from: https://github.com/vllm-project/vllm/blob/f8a1e39fae05ca610be8d5a78be9d40f5274e5fc/vllm/model_executor/layers/attention.py # Copyright 2023 The vLLM team. All rights @@ -45,9 +48,7 @@ def attention( # value_cache => [num_blocks, num_heads, head_size, block_size] block_size = value_cache.shape[3] num_seqs, num_heads, head_size = query.shape - max_num_partitions = ( - (max_s + _PARTITION_SIZE - 1) // - _PARTITION_SIZE) + max_num_partitions = (max_s + _PARTITION_SIZE - 1) // _PARTITION_SIZE # NOTE(woosuk): We use a simple heuristic to decide whether to use # PagedAttention V1 or V2. If the number of partitions is 1, we use # V1 to avoid the overhead of reduction. Also, if the number of diff --git a/server/text_generation_server/utils/peft.py b/server/text_generation_server/utils/peft.py index d37e8940..45e23320 100644 --- a/server/text_generation_server/utils/peft.py +++ b/server/text_generation_server/utils/peft.py @@ -38,7 +38,9 @@ def download_and_unload_peft(model_id, revision, trust_remote_code): os.makedirs(model_id, exist_ok=True) cache_dir = model_id logger.info(f"Saving the newly created merged model to {cache_dir}") - tokenizer = AutoTokenizer.from_pretrained(base_model_id, trust_remote_code=trust_remote_code) + tokenizer = AutoTokenizer.from_pretrained( + base_model_id, trust_remote_code=trust_remote_code + ) model.save_pretrained(cache_dir, safe_serialization=True) model.config.save_pretrained(cache_dir) tokenizer.save_pretrained(cache_dir) diff --git a/server/text_generation_server/utils/speculate.py b/server/text_generation_server/utils/speculate.py index 38a91972..a1b37a34 100644 --- a/server/text_generation_server/utils/speculate.py +++ b/server/text_generation_server/utils/speculate.py @@ -1,12 +1,11 @@ - SPECULATE = None + def get_speculate() -> int: global SPECULATE return SPECULATE + def set_speculate(speculate: int): global SPECULATE SPECULATE = speculate - - diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index a34c5afc..0d208104 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -16,6 +16,7 @@ from text_generation_server.utils.watermark import WatermarkLogitsProcessor from transformers import PreTrainedTokenizerBase, RepetitionPenaltyLogitsProcessor + class NextTokenChooser: def __init__( self, @@ -145,21 +146,31 @@ def from_pb( pb.ignore_eos_token, ) -def create_n_gram_speculation(input_ids: torch.Tensor, next_ids: torch.Tensor, accepted_ids: torch.Tensor, speculate: int, verbose: bool): + +def create_n_gram_speculation( + input_ids: torch.Tensor, + next_ids: torch.Tensor, + accepted_ids: torch.Tensor, + speculate: int, + verbose: bool, +): # Very trivial approach, find first match in the string. # This is much less refined than actual n-gram but seems to work # relatively OK in grounded mode and is by far much faster with # much less worst case complexity as everything happens on device. B = accepted_ids.shape[0] device = input_ids.device - seeds = next_ids[accepted_ids.cumsum(dim=-1) -1 ] + seeds = next_ids[accepted_ids.cumsum(dim=-1) - 1] indices = (input_ids == seeds.unsqueeze(-1)).max(dim=1).indices + 1 - all_indices = indices.unsqueeze(-1).expand(B, speculate) + torch.arange(speculate, device=device) + all_indices = indices.unsqueeze(-1).expand(B, speculate) + torch.arange( + speculate, device=device + ) all_indices = torch.clamp(all_indices, max=input_ids.shape[1] - 1) speculative_ids = input_ids.gather(dim=-1, index=all_indices) return speculative_ids + class HeterogeneousNextTokenChooser: def __init__( self, @@ -228,7 +239,15 @@ def __init__( self.dtype = dtype self.device = device - def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor, speculate: int, speculated_ids: Optional[torch.Tensor] = None, speculative_scores: Optional[torch.Tensor] = None, verbose=False): + def __call__( + self, + input_ids: torch.Tensor, + scores: torch.Tensor, + speculate: int, + speculated_ids: Optional[torch.Tensor] = None, + speculative_scores: Optional[torch.Tensor] = None, + verbose=False, + ): if speculated_ids is not None: B = scores.shape[0] // (speculated_ids.shape[1] + 1) S = speculated_ids.shape[1] + 1 @@ -249,12 +268,11 @@ def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor, speculate: int for warper in self.warpers: _scores = warper(input_ids, _scores) - _next_ids = self.choice(_scores) scores[:, j] = _scores next_ids[:, j] = _next_ids - next_ids = next_ids.view(B*S) - scores = scores.view( B* S, -1) + next_ids = next_ids.view(B * S) + scores = scores.view(B * S, -1) if speculated_ids is not None: accepted_ids = [] @@ -262,7 +280,7 @@ def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor, speculate: int S = speculated_ids.shape[1] + 1 indices = [] for i in range(B): - _next_ids = next_ids[i*S: (i + 1)*S] + _next_ids = next_ids[i * S : (i + 1) * S] _speculated_ids = speculated_ids[i] validate_speculative = _next_ids[:-1] == _speculated_ids index = i * S @@ -278,7 +296,9 @@ def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor, speculate: int break accepted_ids.append(accepted) - accepted_ids = torch.tensor(accepted_ids, device=input_ids.device, dtype=input_ids.dtype) + accepted_ids = torch.tensor( + accepted_ids, device=input_ids.device, dtype=input_ids.dtype + ) next_ids = next_ids[indices] scores = scores[indices] indices = torch.arange(B, device=input_ids.device) * S @@ -296,7 +316,9 @@ def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor, speculate: int speculative_ids = Greedy()(speculative_scores) else: # n-gram - speculative_ids = create_n_gram_speculation(input_ids, next_ids, accepted_ids, speculate, verbose) + speculative_ids = create_n_gram_speculation( + input_ids, next_ids, accepted_ids, speculate, verbose + ) else: speculative_ids = None diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index f3344988..802c1a90 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -16,7 +16,7 @@ def __init__( dtype, process_group, aliases: Optional[Dict[str, List[str]]] = None, - prefix: Optional[str] = None + prefix: Optional[str] = None, ): routing = {} for filename in filenames: @@ -213,7 +213,8 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): bits, groupsize = self._get_gptq_params() from text_generation_server.utils.layers import HAS_EXLLAMA - use_exllama = bits==4 and HAS_EXLLAMA and quantize == "gptq" + + use_exllama = bits == 4 and HAS_EXLLAMA and quantize == "gptq" weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) else: w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] @@ -283,7 +284,7 @@ def get_multi_weights_row(self, prefix: str, quantize: str): if use_exllama: qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) scales = self.get_sharded(f"{prefix}.scales", dim=0) - g_idx = self.get_sharded(f"{prefix}.g_idx", dim= 0) + g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) g_idx = g_idx - g_idx[0] else: # The triton kernel reorders the scales/zero points instead of the weight/activation. diff --git a/update_doc.py b/update_doc.py index 6206e211..6127418c 100644 --- a/update_doc.py +++ b/update_doc.py @@ -21,14 +21,14 @@ def main(): block = [] for line in lines: if line.startswith(" -") or line.startswith(" -"): - rendered_block = '\n'.join(block) + rendered_block = "\n".join(block) if header: final_doc += f"## {header}\n```shell\n{rendered_block}\n```\n" else: final_doc += f"```shell\n{rendered_block}\n```\n" block = [] tokens = line.split("<") - if len(tokens)>1: + if len(tokens) > 1: header = tokens[-1][:-1] else: header = line.split("--")[-1] @@ -36,7 +36,7 @@ def main(): block.append(line) - rendered_block = '\n'.join(block) + rendered_block = "\n".join(block) final_doc += f"## {header}\n```shell\n{rendered_block}\n```\n" block = [] From d0841cc8eb2c27bf96c96aa78c42014ac7a9928b Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 11 Dec 2023 14:55:03 +0100 Subject: [PATCH 492/793] v1.3.0 --- Cargo.lock | 134 +++++++++--------- Cargo.toml | 2 +- README.md | 6 +- docs/openapi.json | 2 +- .../basic_tutorials/gated_model_access.md | 2 +- docs/source/quicktour.md | 6 +- integration-tests/pyproject.toml | 2 +- server/pyproject.toml | 2 +- 8 files changed, 78 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93e13056..105aefe1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,9 +40,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" dependencies = [ "anstyle", "anstyle-parse", @@ -69,9 +69,9 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3a318f1f38d2418400f8209655bfd825785afd25aa30bb7ba6cc792e4596748" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ "windows-sys 0.52.0", ] @@ -128,7 +128,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -139,7 +139,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -362,7 +362,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -813,7 +813,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -994,9 +994,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -1174,9 +1174,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" @@ -1195,9 +1195,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libm" @@ -1336,7 +1336,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -1387,9 +1387,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "log", @@ -1415,7 +1415,7 @@ checksum = "531c82a934da419bed3da09bd87d6e98c72f8d4aa755427b3b009c2b8b8c433c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -1584,9 +1584,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "onig" @@ -1633,7 +1633,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -1849,7 +1849,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -1872,9 +1872,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" [[package]] name = "powerfmt" @@ -1895,7 +1895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -1968,7 +1968,7 @@ dependencies = [ "prost 0.12.3", "prost-types", "regex", - "syn 2.0.39", + "syn 2.0.40", "tempfile", "which", ] @@ -1996,7 +1996,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -2245,9 +2245,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.6" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", "getrandom", @@ -2278,7 +2278,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.39", + "syn 2.0.40", "walkdir", ] @@ -2309,9 +2309,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.26" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9470c4bf8246c8daf25f9598dca807fb6510347b1e1cfa55749113850c79d88a" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.1", "errno", @@ -2334,12 +2334,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.9" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.6", + "ring 0.17.7", "rustls-webpki", "sct", ] @@ -2359,7 +2359,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.6", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -2371,9 +2371,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "same-file" @@ -2405,7 +2405,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.6", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -2455,7 +2455,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -2649,7 +2649,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -2665,9 +2665,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "13fa70a4ee923979ffb522cacce59d34421ebdea5625e1073c4326ef9d2dd42e" dependencies = [ "proc-macro2", "quote", @@ -2754,7 +2754,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "1.2.0" +version = "1.3.0" dependencies = [ "average", "clap", @@ -2775,7 +2775,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.2.0" +version = "1.3.0" dependencies = [ "futures", "grpc-metadata", @@ -2791,7 +2791,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.2.0" +version = "1.3.0" dependencies = [ "clap", "ctrlc", @@ -2807,7 +2807,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.2.0" +version = "1.3.0" dependencies = [ "async-stream", "axum", @@ -2857,7 +2857,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -2952,9 +2952,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" dependencies = [ "backtrace", "bytes", @@ -2987,7 +2987,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -3102,7 +3102,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -3175,7 +3175,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -3272,9 +3272,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" @@ -3293,9 +3293,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" [[package]] name = "unicode-ident" @@ -3362,7 +3362,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.21.9", + "rustls 0.21.10", "rustls-webpki", "serde", "serde_json", @@ -3415,7 +3415,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] @@ -3511,7 +3511,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", "wasm-bindgen-shared", ] @@ -3545,7 +3545,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3572,7 +3572,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.6", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -3835,22 +3835,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.28" +version = "0.7.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" +checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.28" +version = "0.7.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" +checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.40", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f594e0c6..c45b11eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.2.0" +version = "1.3.0" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index 616a8d60..d99b7306 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ For a detailed starting guide, please see the [Quick Tour](https://huggingface.c model=HuggingFaceH4/zephyr-7b-beta volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3 --model-id $model ``` And then you can make requests like @@ -76,7 +76,7 @@ curl 127.0.0.1:8080/generate \ **Note:** To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. -**Note:** TGI supports AMD Instinct MI210 and MI250 GPUs. Details can be found in the [Supported Hardware documentation](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2-rocm --model-id $model` instead of the command above. +**Note:** TGI supports AMD Instinct MI210 and MI250 GPUs. Details can be found in the [Supported Hardware documentation](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3-rocm --model-id $model` instead of the command above. To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli): ``` @@ -106,7 +106,7 @@ model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= -docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2 --model-id $model +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3 --model-id $model ``` ### A note on Shared Memory (shm) diff --git a/docs/openapi.json b/docs/openapi.json index 6a15ae66..153630c0 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.2.0" + "version": "1.3.0" }, "paths": { "/": { diff --git a/docs/source/basic_tutorials/gated_model_access.md b/docs/source/basic_tutorials/gated_model_access.md index e1abccac..1437717f 100644 --- a/docs/source/basic_tutorials/gated_model_access.md +++ b/docs/source/basic_tutorials/gated_model_access.md @@ -19,6 +19,6 @@ docker run --gpus all \ --shm-size 1g \ -e HUGGING_FACE_HUB_TOKEN=$token \ -p 8080:80 \ - -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2 \ + -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3 \ --model-id $model ``` diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index c2b27e2a..03ea03bc 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -8,7 +8,7 @@ Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/ model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3 --model-id $model ``` @@ -20,7 +20,7 @@ To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://d TGI also supports ROCm-enabled AMD GPUs (only MI210 and MI250 are tested), details are available in the [Supported Hardware section](./supported_models#supported-hardware) and [AMD documentation](https://rocm.docs.amd.com/en/latest/deploy/docker.html). To launch TGI on ROCm GPUs, please use instead: ```bash -docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --device=/dev/kfd --device=/dev/dri --group-add video --ipc=host --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.2-rocm --model-id $model +docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --device=/dev/kfd --device=/dev/dri --group-add video --ipc=host --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3-rocm --model-id $model ``` Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section, where we show examples with utility libraries and UIs. Below you can see a simple snippet to query the endpoint. @@ -91,7 +91,7 @@ curl 127.0.0.1:8080/generate \ To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. ```bash -docker run ghcr.io/huggingface/text-generation-inference:1.2 --help +docker run ghcr.io/huggingface/text-generation-inference:1.3 --help ``` diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 75411131..9457efbc 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-integration-tests" -version = "1.2.0" +version = "1.3.0" description = "Text Generation Inference integration tests" authors = ["Nicolas Patry "] diff --git a/server/pyproject.toml b/server/pyproject.toml index b96c1204..8ca1a5c5 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.2.0" +version = "1.3.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From ec6d4592d59c136ea1d2cac3b4e2a7ea004a0ce3 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 11 Dec 2023 16:46:44 +0100 Subject: [PATCH 493/793] v1.3.1 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- docs/openapi.json | 2 +- integration-tests/pyproject.toml | 2 +- server/pyproject.toml | 2 +- .../models/custom_modeling/flash_mistral_modeling.py | 5 ++++- .../models/custom_modeling/flash_mixtral_modeling.py | 7 +++++-- 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 105aefe1..6ae06601 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2754,7 +2754,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "1.3.0" +version = "1.3.1" dependencies = [ "average", "clap", @@ -2775,7 +2775,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.3.0" +version = "1.3.1" dependencies = [ "futures", "grpc-metadata", @@ -2791,7 +2791,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.3.0" +version = "1.3.1" dependencies = [ "clap", "ctrlc", @@ -2807,7 +2807,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.3.0" +version = "1.3.1" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index c45b11eb..b22fccf8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.3.0" +version = "1.3.1" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index 153630c0..6c372148 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.3.0" + "version": "1.3.1" }, "paths": { "/": { diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 9457efbc..ae5876b0 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-integration-tests" -version = "1.3.0" +version = "1.3.1" description = "Text Generation Inference integration tests" authors = ["Nicolas Patry "] diff --git a/server/pyproject.toml b/server/pyproject.toml index 8ca1a5c5..b2aa4dc3 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.3.0" +version = "1.3.1" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index 525bf6bc..5a4f5be0 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -391,6 +391,7 @@ def forward( slots: torch.Tensor, input_lengths: torch.Tensor, max_s: int, + true_max_s: int, prefill_cache_indices: Optional[torch.Tensor], ) -> torch.Tensor: hidden_states = self.embed_tokens(input_ids) @@ -398,7 +399,7 @@ def forward( # Get rotary cos and sin for this forward # Avoid to index in each layer cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin( - position_ids, max_s, hidden_states.dtype + position_ids, true_max_s, hidden_states.dtype ) residual = None @@ -449,6 +450,7 @@ def forward( prefill_cache_indices: Optional[torch.Tensor], lm_head_indices: Optional[torch.Tensor] = None, ) -> torch.Tensor: + true_max_s = max_s if prefill_cache_indices is not None: # Slots also need to be sliced as it has the same size as the whole kv tensor slots = slots[prefill_cache_indices] @@ -467,6 +469,7 @@ def forward( slots, input_lengths, max_s, + true_max_s, prefill_cache_indices, ) if lm_head_indices is not None: diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py index 6f5edca2..76ebc6b8 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py @@ -401,7 +401,7 @@ def topology(self, x: torch.Tensor, padded_bins: torch.Tensor): self.offsets_block_rows = block_rows offsets = self.offsets else: - offsets = self.offsets[:block_rows] + offsets = self.offsets[: block_rows + 1] # Indices for the sparse matrix. The indices for # the intermediate matrix are dynamic depending @@ -632,6 +632,7 @@ def forward( slots: torch.Tensor, input_lengths: torch.Tensor, max_s: int, + true_max_s: int, prefill_cache_indices: Optional[torch.Tensor], ) -> torch.Tensor: hidden_states = self.embed_tokens(input_ids) @@ -639,7 +640,7 @@ def forward( # Get rotary cos and sin for this forward # Avoid to index in each layer cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin( - position_ids, max_s, hidden_states.dtype + position_ids, true_max_s, hidden_states.dtype ) residual = None @@ -690,6 +691,7 @@ def forward( prefill_cache_indices: Optional[torch.Tensor], lm_head_indices: Optional[torch.Tensor] = None, ) -> torch.Tensor: + true_max_s = max_s if prefill_cache_indices is not None: # Slots also need to be sliced as it has the same size as the whole kv tensor slots = slots[prefill_cache_indices] @@ -708,6 +710,7 @@ def forward( slots, input_lengths, max_s, + true_max_s, prefill_cache_indices, ) if lm_head_indices is not None: From 82670d97864d45e0e3d0654143a4aaebbc98f16b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 12 Dec 2023 17:55:03 +0100 Subject: [PATCH 494/793] feat: add quant to mixtral (#1337) --- .../custom_modeling/flash_mistral_modeling.py | 4 +- .../custom_modeling/flash_mixtral_modeling.py | 175 ++++++++++++++++-- .../models/flash_mistral.py | 34 ++-- server/text_generation_server/utils/layers.py | 6 +- 4 files changed, 184 insertions(+), 35 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index 5a4f5be0..b97866f7 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -434,8 +434,6 @@ def __init__(self, config, weights): weights=weights, ) self.max_past = config.sliding_window - if self.max_past is None: - raise ValueError("max_past cannot be None") def forward( self, @@ -454,7 +452,7 @@ def forward( if prefill_cache_indices is not None: # Slots also need to be sliced as it has the same size as the whole kv tensor slots = slots[prefill_cache_indices] - else: + elif self.max_past is not None: # Clamp in decode mode as paged attention requires clamped values whereas the flash attention # kernel requires the true values max_s = min(self.max_past, max_s) diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py index 76ebc6b8..ff2ed9fd 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py @@ -365,9 +365,9 @@ def __init__(self, prefix, config: MixtralConfig, weights): self.gate = FastLinear.load(config, f"{prefix}.gate", weights, bias=False) # merged expert weights, all of size (n_experts * ffn_dim, hidden_dim) - self.w1 = _load_experts(config, f"{prefix}.experts", "w1", weights).t() + self.w1 = _load_experts(config, f"{prefix}.experts", "w1", weights) self.w2 = _load_experts(config, f"{prefix}.experts", "w2", weights) - self.w3 = _load_experts(config, f"{prefix}.experts", "w3", weights).t() + self.w3 = _load_experts(config, f"{prefix}.experts", "w3", weights) self.offsets = None self.offsets_block_rows = 0 @@ -467,8 +467,7 @@ def indices_and_padded_bins(self, selected_experts: torch.Tensor): return indices, bin_ids, bins, padded_bins, tokens_per_expert - @torch.inference_mode() - def forward(self, x: torch.Tensor) -> torch.Tensor: + def sparse_forward(self, x: torch.Tensor) -> torch.Tensor: """ x: (sequence_length, model_dim) gate_logits: (sequence_length, n_experts) @@ -502,8 +501,8 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # (top_k * sequence_length + padding, ffn_dim * n_experts) x = stk.Matrix( topo.size(), - self.act(stk.ops.sdd(x, self.w1, topo).data) - * stk.ops.sdd(x, self.w3, topo).data, + self.act(stk.ops.sdd(x, self.w1.t(), topo).data) + * stk.ops.sdd(x, self.w3.t(), topo).data, topo.row_indices, topo.column_indices, topo.offsets, @@ -534,6 +533,156 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return x.view(*input_shape) + def dense_forward(self, x: torch.Tensor) -> torch.Tensor: + """ + x: (sequence_length, model_dim) + gate_logits: (sequence_length, n_experts) + """ + # optional reshape + input_shape = x.shape + x = x.view(-1, input_shape[-1]) + + # gate_logits: (sequence_length, n_experts) + gate_logits = self.gate(x) + # all_probs: (sequence_length, n_experts) and upcast for softmax + all_probs = torch.nn.functional.softmax(gate_logits, dim=1, dtype=torch.float) + + if self.top_k < self.num_experts: + _, not_selected_experts = torch.topk( + all_probs, + self.num_experts - self.top_k, + largest=False, + sorted=False, + dim=1, + ) + # Mask not selected experts + all_probs.scatter_(1, not_selected_experts, 0) + + # Re-normalize + weights = all_probs / all_probs.sum(dim=1, keepdim=True) + + # Expand to [num_experts, sequence_length, model_dim] + x = x.view(1, -1, input_shape[-1]).expand(self.num_experts, -1, input_shape[-1]) + + # Permute to [num_experts, model_dim, ffn_dim] + w1 = self.w1.view(self.num_experts, self.ffn_dim, self.hidden_dim).permute( + 0, 2, 1 + ) + w3 = self.w3.view(self.num_experts, self.ffn_dim, self.hidden_dim).permute( + 0, 2, 1 + ) + + inter = self.act(torch.bmm(x, w1)) * torch.bmm(x, w3) + + out = torch.bmm( + inter, self.w2.view(self.num_experts, self.ffn_dim, self.hidden_dim) + ) + # Mask not selected experts + out *= weights.t().view(self.num_experts, -1, 1) + + # Sum experts + out = out.sum(0) + + # Reduce sum + if self.process_group.size() > 1: + torch.distributed.all_reduce(out, group=self.process_group) + + return out + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if len(x) > 256: + return self.sparse_forward(x) + # This is faster when there is not a lot of tokens + return self.dense_forward(x) + + +class DenseMoE(nn.Module): + def __init__(self, prefix, config: MixtralConfig, weights): + super().__init__() + self.hidden_dim = config.hidden_size + self.ffn_dim = config.intermediate_size // weights.process_group.size() + self.num_experts = config.num_local_experts + self.top_k = config.num_experts_per_tok + + act = config.hidden_act + if "gelu" in act: + self.act = lambda x: torch.nn.functional.gelu( + x, + approximate="tanh" + if act in ["gelu_fast", "gelu_pytorch_tanh"] + else "none", + ) + elif "silu" in act: + self.act = torch.nn.functional.silu + else: + self.act = ACT2FN[act] + + # gating + self.gate = FastLinear.load(config, f"{prefix}.gate", weights, bias=False) + + self.w1 = [ + TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.experts.{i}.w1", weights=weights, bias=False + ) + for i in range(self.num_experts) + ] + self.w3 = [ + TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.experts.{i}.w3", weights=weights, bias=False + ) + for i in range(self.num_experts) + ] + self.w2 = [ + TensorParallelRowLinear.load( + config, prefix=f"{prefix}.experts.{i}.w2", weights=weights, bias=False + ) + for i in range(self.num_experts) + ] + + self.process_group = weights.process_group + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + x: (sequence_length, model_dim) + gate_logits: (sequence_length, n_experts) + """ + # optional reshape + input_shape = x.shape + x = x.view(-1, input_shape[-1]) + + # gate_logits: (sequence_length, n_experts) + gate_logits = self.gate(x) + # all_probs: (sequence_length, n_experts) and upcast for softmax + all_probs = torch.nn.functional.softmax(gate_logits, dim=1, dtype=torch.float) + + if self.top_k < self.num_experts: + _, not_selected_experts = torch.topk( + all_probs, + self.num_experts - self.top_k, + largest=False, + sorted=False, + dim=1, + ) + # Mask not selected experts + all_probs.scatter_(1, not_selected_experts, 0) + + # Re-normalize + weights = all_probs / all_probs.sum(dim=1, keepdim=True) + + # Final output tensor + out = x.new_zeros(x.shape[0], self.hidden_dim) + for i in range(self.num_experts): + h = self.act(self.w1[i](x)) * self.w3[i](x) + h = self.w2[i](h, reduce=False) + # Add expert output to out with masking + out += h * weights[:, i].view(-1, 1) + + # Reduce sum + if self.process_group.size() > 1: + torch.distributed.all_reduce(out, group=self.process_group) + + return out + class MixtralLayer(nn.Module): def __init__(self, layer_id, config, weights): @@ -543,9 +692,9 @@ def __init__(self, layer_id, config, weights): self.self_attn = MixtralAttention( prefix=f"{prefix}.self_attn", config=config, weights=weights ) - self.block_sparse_moe = BlockSparseMoE( - f"{prefix}.block_sparse_moe", config, weights - ) + + moe_cls = BlockSparseMoE if config.quantize is None else DenseMoE + self.moe = moe_cls(f"{prefix}.block_sparse_moe", config, weights) self.input_layernorm = FastRMSNorm.load( prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps @@ -591,9 +740,9 @@ def forward( attn_output, res ) - block_sparse_moe_output = self.block_sparse_moe(normed_attn_res_output) + moe_output = self.moe(normed_attn_res_output) - return block_sparse_moe_output, attn_res + return moe_output, attn_res class MixtralModel(torch.nn.Module): @@ -675,8 +824,6 @@ def __init__(self, config, weights): weights=weights, ) self.max_past = config.sliding_window - if self.max_past is None: - raise ValueError("max_past cannot be None") def forward( self, @@ -695,7 +842,7 @@ def forward( if prefill_cache_indices is not None: # Slots also need to be sliced as it has the same size as the whole kv tensor slots = slots[prefill_cache_indices] - else: + elif self.max_past is not None: # Clamp in decode mode as paged attention requires clamped values whereas the flash attention # kernel requires the true values max_s = min(self.max_past, max_s) diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py index 0fad5aa8..abe07c30 100644 --- a/server/text_generation_server/models/flash_mistral.py +++ b/server/text_generation_server/models/flash_mistral.py @@ -136,9 +136,9 @@ def from_pb( total_tokens = input_length + max_new_tokens - 1 + speculative_length # Needed blocks can not go over SLIDING_WINDOW_BLOCKS - needed_blocks = min( - math.ceil(total_tokens / BLOCK_SIZE), SLIDING_WINDOW_BLOCKS - ) + needed_blocks = math.ceil(total_tokens / BLOCK_SIZE) + if SLIDING_WINDOW_BLOCKS is not None: + needed_blocks = min(needed_blocks, SLIDING_WINDOW_BLOCKS) blocks += needed_blocks needed_blocks_slots.append((needed_blocks, total_tokens)) @@ -152,12 +152,13 @@ def from_pb( slot_indices.append(request_slot_indices) # Create tensor to slice into the kv tensor in prefill - request_prefill_cache_indices = torch.arange( - cumulative_length + max(0, input_length - SLIDING_WINDOW), - cumulative_length + input_length, - dtype=torch.int64, - ) - prefill_cache_indices.append(request_prefill_cache_indices) + if SLIDING_WINDOW is not None: + request_prefill_cache_indices = torch.arange( + cumulative_length + max(0, input_length - SLIDING_WINDOW), + cumulative_length + input_length, + dtype=torch.int64, + ) + prefill_cache_indices.append(request_prefill_cache_indices) all_prefill_logprobs = all_prefill_logprobs and r.prefill_logprobs no_prefill_logprobs = no_prefill_logprobs and not r.prefill_logprobs @@ -209,12 +210,14 @@ def from_pb( input_ids = np.concatenate(all_input_ids, dtype=np.int64) position_ids = torch.cat(position_ids) slot_indices = torch.cat(slot_indices) - prefill_cache_indices = torch.cat(prefill_cache_indices) + if SLIDING_WINDOW is not None: + prefill_cache_indices = torch.cat(prefill_cache_indices) else: input_ids = all_input_ids[0] position_ids = position_ids[0] slot_indices = slot_indices[0] - prefill_cache_indices = prefill_cache_indices[0] + if SLIDING_WINDOW is not None: + prefill_cache_indices = prefill_cache_indices[0] cu_seqlen_prefill = torch.tensor( cu_seqlen_prefill, device=device, dtype=torch.int32 @@ -222,7 +225,9 @@ def from_pb( position_ids = position_ids.to(device) slot_indices = slot_indices.to(device) - prefill_cache_indices = prefill_cache_indices.to(device) + prefill_cache_indices = ( + prefill_cache_indices.to(device) if SLIDING_WINDOW is not None else None + ) input_ids = torch.tensor(input_ids, dtype=torch.int64, device=device) input_lengths_tensor = torch.tensor( input_lengths, dtype=torch.int32, device=device @@ -314,8 +319,9 @@ def __init__( config.quantize = quantize # Set context windows - SLIDING_WINDOW = config.sliding_window - SLIDING_WINDOW_BLOCKS = math.ceil(config.sliding_window / BLOCK_SIZE) + if config.sliding_window is not None: + SLIDING_WINDOW = config.sliding_window + SLIDING_WINDOW_BLOCKS = math.ceil(config.sliding_window / BLOCK_SIZE) torch.distributed.barrier(group=self.process_group) diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 77e2fdb6..011a9382 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -64,8 +64,6 @@ except ImportError: pass -from typing import Optional - HAS_EETQ = False try: from EETQ import quant_weights, w8_a16_gemm @@ -489,9 +487,9 @@ def load(cls, config, prefix: str, weights, bias: bool): process_group=weights.process_group, ) - def forward(self, input: torch.Tensor) -> torch.Tensor: + def forward(self, input: torch.Tensor, reduce: bool = True) -> torch.Tensor: out = super().forward(input) - if self.process_group.size() > 1: + if self.process_group.size() > 1 and reduce: torch.distributed.all_reduce(out, group=self.process_group) return out From 88aae2595d9a2011a58956391848cde869479653 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Tue, 12 Dec 2023 18:10:22 +0100 Subject: [PATCH 495/793] v1.3.2 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- docs/openapi.json | 2 +- integration-tests/pyproject.toml | 2 +- server/pyproject.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ae06601..162c68c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2754,7 +2754,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "1.3.1" +version = "1.3.2" dependencies = [ "average", "clap", @@ -2775,7 +2775,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.3.1" +version = "1.3.2" dependencies = [ "futures", "grpc-metadata", @@ -2791,7 +2791,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.3.1" +version = "1.3.2" dependencies = [ "clap", "ctrlc", @@ -2807,7 +2807,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.3.1" +version = "1.3.2" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index b22fccf8..df728d27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.3.1" +version = "1.3.2" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index 6c372148..d70023c6 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.3.1" + "version": "1.3.2" }, "paths": { "/": { diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index ae5876b0..2747473c 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-integration-tests" -version = "1.3.1" +version = "1.3.2" description = "Text Generation Inference integration tests" authors = ["Nicolas Patry "] diff --git a/server/pyproject.toml b/server/pyproject.toml index b2aa4dc3..d519de63 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.3.1" +version = "1.3.2" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 28821bfd5dfc0fe719d4013c1825094f483b7085 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:19:19 +0100 Subject: [PATCH 496/793] fix: default max_new_tokens to 100 --- docs/openapi.json | 2 +- router/src/lib.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index d70023c6..92faf1ed 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -523,7 +523,7 @@ "max_new_tokens": { "type": "integer", "format": "int32", - "default": "null", + "default": "20", "example": "20", "nullable": true, "minimum": 0 diff --git a/router/src/lib.rs b/router/src/lib.rs index b547dc15..898fcd04 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -107,7 +107,7 @@ pub(crate) struct GenerateParameters { #[schema(default = "false", example = true)] pub do_sample: bool, #[serde(default = "default_max_new_tokens")] - #[schema(nullable = true, default = "null", example = "20")] + #[schema(nullable = true, default = "100", example = "20")] pub max_new_tokens: Option, #[serde(default)] #[schema(nullable = true, default = "null", example = false)] @@ -141,7 +141,7 @@ pub(crate) struct GenerateParameters { } fn default_max_new_tokens() -> Option { - None + Some(100) } fn default_parameters() -> GenerateParameters { From 44b267ab22d097c2c9a0165857813faaaf4cd53d Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:02:16 +0100 Subject: [PATCH 497/793] fix: fix gpt-q params loading --- server/text_generation_server/models/bloom.py | 2 +- .../text_generation_server/models/flash_llama.py | 2 +- .../text_generation_server/models/flash_mistral.py | 2 +- server/text_generation_server/models/flash_neox.py | 2 +- server/text_generation_server/models/flash_rw.py | 2 +- .../models/flash_santacoder.py | 2 +- server/text_generation_server/models/galactica.py | 2 +- server/text_generation_server/models/gpt_neox.py | 2 +- server/text_generation_server/models/mpt.py | 2 +- server/text_generation_server/models/opt.py | 2 +- server/text_generation_server/utils/weights.py | 14 ++++++++++---- 11 files changed, 20 insertions(+), 14 deletions(-) diff --git a/server/text_generation_server/models/bloom.py b/server/text_generation_server/models/bloom.py index c3876023..fed5e6f3 100644 --- a/server/text_generation_server/models/bloom.py +++ b/server/text_generation_server/models/bloom.py @@ -81,7 +81,7 @@ def __init__( prefix="transformer", ) if config.quantize == "gptq": - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) model = BloomForCausalLM(config, weights) diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 2415a245..8a3bccdd 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -64,7 +64,7 @@ def __init__( filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights(filenames, device, dtype, process_group=self.process_group) if config.quantize in ["gptq", "awq"]: - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) model = FlashLlamaForCausalLM(config, weights) if use_medusa: diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py index abe07c30..8c6cb025 100644 --- a/server/text_generation_server/models/flash_mistral.py +++ b/server/text_generation_server/models/flash_mistral.py @@ -328,7 +328,7 @@ def __init__( filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights(filenames, device, dtype, process_group=self.process_group) if config.quantize in ["gptq", "awq"]: - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) model = model_cls(config, weights) diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py index 58f345a9..80f8804d 100644 --- a/server/text_generation_server/models/flash_neox.py +++ b/server/text_generation_server/models/flash_neox.py @@ -53,7 +53,7 @@ def __init__( filenames, device=device, dtype=dtype, process_group=self.process_group ) if config.quantize == "gptq": - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) model = FlashGPTNeoXForCausalLM(config, weights) diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py index 195b3883..dfab8888 100644 --- a/server/text_generation_server/models/flash_rw.py +++ b/server/text_generation_server/models/flash_rw.py @@ -62,7 +62,7 @@ def __init__( config.quantize = quantize if config.quantize == "gptq": - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) model = FlashRWForCausalLM(config, weights) diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py index 29505902..22171ec0 100644 --- a/server/text_generation_server/models/flash_santacoder.py +++ b/server/text_generation_server/models/flash_santacoder.py @@ -63,7 +63,7 @@ def __init__( aliases={"transformer.wte.weight": ["lm_head.weight"]}, ) if config.quantize == "gptq": - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) model = FlashSantacoderForCausalLM(config, weights) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index b296c96e..42ff1c80 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -199,7 +199,7 @@ def __init__( filenames, device=device, dtype=dtype, process_group=self.process_group ) if config.quantize == "gptq": - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) model = OPTForCausalLM(config, weights) diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py index d4c64dfe..45df4839 100644 --- a/server/text_generation_server/models/gpt_neox.py +++ b/server/text_generation_server/models/gpt_neox.py @@ -57,7 +57,7 @@ def __init__( filenames, device=device, dtype=dtype, process_group=self.process_group ) if config.quantize == "gptq": - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) model = GPTNeoxForCausalLM(config, weights) diff --git a/server/text_generation_server/models/mpt.py b/server/text_generation_server/models/mpt.py index 19de497c..e419467f 100644 --- a/server/text_generation_server/models/mpt.py +++ b/server/text_generation_server/models/mpt.py @@ -81,7 +81,7 @@ def __init__( filenames = weight_files(model_id, revision=revision, extension=".safetensors") weights = Weights(filenames, device, dtype, process_group=self.process_group) if config.quantize == "gptq": - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) config.quantize = quantize model = MPTForCausalLM(config, weights) diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py index b2b87246..58fb212f 100644 --- a/server/text_generation_server/models/opt.py +++ b/server/text_generation_server/models/opt.py @@ -55,7 +55,7 @@ def __init__( filenames, device=device, dtype=dtype, process_group=self.process_group ) if config.quantize == "gptq": - weights._set_gptq_params(model_id) + weights._set_gptq_params(model_id, revision) model = OPTForCausalLM(config, weights) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 802c1a90..67fda511 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -327,13 +327,15 @@ def _get_gptq_params(self) -> Tuple[int, int]: return bits, groupsize - def _set_gptq_params(self, model_id): + def _set_gptq_params(self, model_id, revision): filename = "config.json" try: if os.path.exists(os.path.join(model_id, filename)): filename = os.path.join(model_id, filename) else: - filename = hf_hub_download(model_id, filename=filename) + filename = hf_hub_download( + model_id, filename=filename, revision=revision + ) with open(filename, "r") as f: data = json.load(f) self.gptq_bits = data["quantization_config"]["bits"] @@ -344,7 +346,9 @@ def _set_gptq_params(self, model_id): if os.path.exists(os.path.join(model_id, filename)): filename = os.path.join(model_id, filename) else: - filename = hf_hub_download(model_id, filename=filename) + filename = hf_hub_download( + model_id, filename=filename, revision=revision + ) with open(filename, "r") as f: data = json.load(f) self.gptq_bits = data["bits"] @@ -355,7 +359,9 @@ def _set_gptq_params(self, model_id): if os.path.exists(os.path.join(model_id, filename)): filename = os.path.join(model_id, filename) else: - filename = hf_hub_download(model_id, filename=filename) + filename = hf_hub_download( + model_id, filename=filename, revision=revision + ) with open(filename, "r") as f: data = json.load(f) self.gptq_bits = data["w_bit"] From 50b495f3d82097a6ed5f6138f92b6006e5471884 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 14 Dec 2023 15:59:38 +0100 Subject: [PATCH 498/793] feat: add more latency metrics in forward (#1346) --- benchmark/src/generation.rs | 2 +- proto/generate.proto | 14 +++++ router/client/src/client.rs | 56 +++++++++++++++++-- router/client/src/sharded_client.rs | 51 +++++++++++------ router/src/infer.rs | 17 +++++- router/src/validation.rs | 4 +- server/tests/models/test_bloom.py | 32 +++++------ server/tests/models/test_causal_lm.py | 34 +++++------ server/tests/models/test_santacoder.py | 8 +-- server/tests/models/test_seq2seq_lm.py | 28 +++++----- .../models/causal_lm.py | 17 ++++-- .../models/flash_causal_lm.py | 23 +++++--- .../models/idefics_causal_lm.py | 21 +++---- server/text_generation_server/models/model.py | 4 +- .../models/seq2seq_lm.py | 16 ++++-- server/text_generation_server/server.py | 17 +++++- server/text_generation_server/utils/tokens.py | 2 +- 17 files changed, 238 insertions(+), 108 deletions(-) diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs index 67afa04e..ea7c9778 100644 --- a/benchmark/src/generation.rs +++ b/benchmark/src/generation.rs @@ -163,7 +163,7 @@ async fn prefill( // Run prefill let start_time = Instant::now(); - let (_, decode_batch) = client.prefill(batch.clone()).await?; + let (_, decode_batch, _) = client.prefill(batch.clone()).await?; // Get latency let latency = start_time.elapsed(); diff --git a/proto/generate.proto b/proto/generate.proto index 19ec059b..02f3b2e8 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -182,6 +182,12 @@ message PrefillResponse { repeated Generation generations = 1; /// Next batch (cached) optional CachedBatch batch = 2; + /// Forward elapsed time in nanoseconds + uint64 forward_ns = 3; + /// Decode elapsed time in nanoseconds + uint64 decode_ns = 4; + /// Total elapsed time in nanoseconds + uint64 total_ns = 5; } message DecodeRequest { @@ -194,6 +200,14 @@ message DecodeResponse { repeated Generation generations = 1; /// Next batch (cached) optional CachedBatch batch = 2; + /// Forward elapsed time in nanoseconds + uint64 forward_ns = 3; + /// Decode elapsed time in nanoseconds + uint64 decode_ns = 4; + /// Total elapsed time in nanoseconds + uint64 total_ns = 5; + /// Concatenate elapsed time in nanoseconds + optional uint64 concat_ns = 6; } message WarmupRequest { diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 1560f19c..898e2b11 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -4,6 +4,7 @@ use crate::pb::generate::v2::*; use crate::Result; use grpc_metadata::InjectTelemetryContext; use std::cmp::min; +use std::time::Duration; use tonic::transport::{Channel, Uri}; use tracing::instrument; @@ -157,10 +158,14 @@ impl Client { pub async fn prefill( &mut self, batch: Batch, - ) -> Result<(Vec, Option)> { + ) -> Result<(Vec, Option, PrefillTimings)> { let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }).inject_context(); let response = self.stub.prefill(request).await?.into_inner(); - Ok((response.generations, response.batch)) + Ok(( + response.generations, + response.batch, + PrefillTimings::new(response.forward_ns, response.decode_ns, response.total_ns), + )) } /// Generate one token for each request in the given cached batches @@ -171,9 +176,52 @@ impl Client { pub async fn decode( &mut self, batches: Vec, - ) -> Result<(Vec, Option)> { + ) -> Result<(Vec, Option, DecodeTimings)> { let request = tonic::Request::new(DecodeRequest { batches }).inject_context(); let response = self.stub.decode(request).await?.into_inner(); - Ok((response.generations, response.batch)) + Ok(( + response.generations, + response.batch, + DecodeTimings::new( + response.concat_ns, + response.forward_ns, + response.decode_ns, + response.total_ns, + ), + )) + } +} + +pub struct PrefillTimings { + pub forward: Duration, + pub decode: Duration, + pub total: Duration, +} + +impl PrefillTimings { + fn new(forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self { + Self { + forward: Duration::from_nanos(forward_ns), + decode: Duration::from_nanos(decode_ns), + total: Duration::from_nanos(total_ns), + } + } +} + +pub struct DecodeTimings { + pub concat: Option, + pub forward: Duration, + pub decode: Duration, + pub total: Duration, +} + +impl DecodeTimings { + fn new(concat_ns: Option, forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self { + Self { + concat: concat_ns.map(|v| Duration::from_nanos(v)), + forward: Duration::from_nanos(forward_ns), + decode: Duration::from_nanos(decode_ns), + total: Duration::from_nanos(total_ns), + } } } diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index b4bdcd42..6c5da3c7 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -1,3 +1,4 @@ +use crate::client::{DecodeTimings, PrefillTimings}; /// Multi shard Client use crate::{Batch, CachedBatch, Client, Generation, HealthResponse, ShardInfo}; use crate::{ClientError, Result}; @@ -116,49 +117,63 @@ impl ShardedClient { /// /// Returns Generation for each request in batch /// and the next cached batch - #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))] + #[instrument(skip_all, fields(id = & batch.id, size = & batch.size))] pub async fn prefill( &mut self, batch: Batch, - ) -> Result<(Vec, Option)> { + ) -> Result<(Vec, Option, PrefillTimings)> { let futures: Vec<_> = self .clients .iter_mut() .map(|client| Box::pin(client.prefill(batch.clone()))) .collect(); - let results: Result, Option)>> = + let results: Result, Option, PrefillTimings)>> = join_all(futures).await.into_iter().collect(); - merge_generations(results?) + let mut results = results?; + + let (mut generations, next_batch, mut timings) = + results.pop().ok_or(ClientError::EmptyResults)?; + + // Merge generations from different model shards + for (mut shard_generations, _, shard_timings) in results.into_iter() { + generations.append(&mut shard_generations); + // Return the timings of the slowest shard + if shard_timings.total > timings.total { + timings = shard_timings; + } + } + Ok((generations, next_batch, timings)) } /// Generate one token for each request in the given cached batches /// /// Returns Generation for each request in batches /// and the next cached batch - #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::()))] + #[instrument(skip_all, fields(size = batches.iter().map(| batch | {batch.size}).sum::< u32 > ()))] pub async fn decode( &mut self, batches: Vec, - ) -> Result<(Vec, Option)> { + ) -> Result<(Vec, Option, DecodeTimings)> { let futures: Vec<_> = self .clients .iter_mut() .map(|client| Box::pin(client.decode(batches.clone()))) .collect(); - let results: Result, Option)>> = + let results: Result, Option, DecodeTimings)>> = join_all(futures).await.into_iter().collect(); - merge_generations(results?) - } -} + let mut results = results?; -/// Merge generations from the different model shards -fn merge_generations( - mut results: Vec<(Vec, Option)>, -) -> Result<(Vec, Option)> { - let (mut generations, next_batch) = results.pop().ok_or(ClientError::EmptyResults)?; + let (mut generations, next_batch, mut timings) = + results.pop().ok_or(ClientError::EmptyResults)?; - for (mut shard_generations, _) in results.into_iter() { - generations.append(&mut shard_generations); + // Merge generations from different model shards + for (mut shard_generations, _, shard_timings) in results.into_iter() { + generations.append(&mut shard_generations); + // Return the timings of the slowest shard + if shard_timings.total > timings.total { + timings = shard_timings; + } + } + Ok((generations, next_batch, timings)) } - Ok((generations, next_batch)) } diff --git a/router/src/infer.rs b/router/src/infer.rs index 2e199ce2..bf5920da 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -379,15 +379,20 @@ async fn prefill( metrics::increment_counter!("tgi_batch_inference_count", "method" => "prefill"); match client.prefill(batch).await { - Ok((generations, next_batch)) => { + Ok((generations, next_batch, timings)) => { // Update health generation_health.store(true, Ordering::SeqCst); + + let start_filtering_time = Instant::now(); // Send generated tokens and filter stopped entries filter_send_generations(generations, entries); // Filter next batch and remove requests that were stopped let next_batch = filter_batch(client, next_batch, entries).await; + metrics::histogram!("tgi_batch_forward_duration", timings.forward.as_secs_f64(), "method" => "prefill"); + metrics::histogram!("tgi_batch_decode_duration", timings.decode.as_secs_f64(), "method" => "prefill"); + metrics::histogram!("tgi_batch_filter_duration", start_filtering_time.elapsed().as_secs_f64(), "method" => "prefill"); metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "prefill"); metrics::increment_counter!("tgi_batch_inference_success", "method" => "prefill"); next_batch @@ -416,15 +421,23 @@ async fn decode( metrics::increment_counter!("tgi_batch_inference_count", "method" => "decode"); match client.decode(batches).await { - Ok((generations, next_batch)) => { + Ok((generations, next_batch, timings)) => { // Update health generation_health.store(true, Ordering::SeqCst); + + let start_filtering_time = Instant::now(); // Send generated tokens and filter stopped entries filter_send_generations(generations, entries); // Filter next batch and remove requests that were stopped let next_batch = filter_batch(client, next_batch, entries).await; + if let Some(concat_duration) = timings.concat { + metrics::histogram!("tgi_batch_concat_duration", concat_duration.as_secs_f64(), "method" => "decode"); + } + metrics::histogram!("tgi_batch_forward_duration", timings.forward.as_secs_f64(), "method" => "decode"); + metrics::histogram!("tgi_batch_decode_duration", timings.decode.as_secs_f64(), "method" => "decode"); + metrics::histogram!("tgi_batch_filter_duration", start_filtering_time.elapsed().as_secs_f64(), "method" => "decode"); metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "decode"); metrics::increment_counter!("tgi_batch_inference_success", "method" => "decode"); next_batch diff --git a/router/src/validation.rs b/router/src/validation.rs index 1b47fc97..90dc3741 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -540,7 +540,7 @@ mod tests { let max_stop_sequence = 3; let max_top_n_tokens = 4; let max_input_length = 5; - let max_total_tokens = 6; + let max_total_tokens = 106; let workers = 1; let validation = Validation::new( workers, @@ -600,7 +600,7 @@ mod tests { let max_stop_sequences = 3; let max_top_n_tokens = 4; let max_input_length = 5; - let max_total_tokens = 6; + let max_total_tokens = 106; let workers = 1; let validation = Validation::new( workers, diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py index d9a33795..66df708a 100644 --- a/server/tests/models/test_bloom.py +++ b/server/tests/models/test_bloom.py @@ -103,7 +103,7 @@ def test_causal_lm_batch_type(default_bloom): def test_causal_lm_generate_token(default_bloom, default_bloom_batch): sequence_length = len(default_bloom_batch.all_input_ids[0]) - generations, next_batch = default_bloom.generate_token(default_bloom_batch) + generations, next_batch, _ = default_bloom.generate_token(default_bloom_batch) assert len(generations) == len(default_bloom_batch) assert isinstance(next_batch, CausalLMBatch) @@ -153,10 +153,10 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch): def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch): next_batch = default_bloom_batch for _ in range(default_bloom_batch.stopping_criterias[0].max_new_tokens - 1): - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert len(generations) == len(default_bloom_batch) - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 @@ -178,10 +178,10 @@ def test_causal_lm_generate_token_completion_multi( for i in range( default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 1 ): - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert len(generations) == len(default_multi_requests_bloom_batch) - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert next_batch is not None assert len(generations) == 2 @@ -201,10 +201,10 @@ def test_causal_lm_generate_token_completion_multi( for _ in range( stopping_criterias[0].max_new_tokens - stopping_criterias[1].max_new_tokens - 1 ): - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 @@ -224,11 +224,11 @@ def test_batch_concatenate( default_bloom, default_bloom_batch, default_multi_requests_bloom_batch ): next_batch_0 = default_bloom_batch - _, next_batch_0 = default_bloom.generate_token(next_batch_0) - _, next_batch_0 = default_bloom.generate_token(next_batch_0) + _, next_batch_0, _ = default_bloom.generate_token(next_batch_0) + _, next_batch_0, _ = default_bloom.generate_token(next_batch_0) next_batch_1 = default_multi_requests_bloom_batch - _, next_batch_1 = default_bloom.generate_token(next_batch_1) + _, next_batch_1, _ = default_bloom.generate_token(next_batch_1) # Clone past_key_values before concatenating to compare after, # because they are removed from the concatenated batches @@ -288,10 +288,10 @@ def test_batch_concatenate( for _ in range( default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 2 ): - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert next_batch is not None assert len(generations) == 3 @@ -313,10 +313,10 @@ def test_batch_concatenate( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 2 ): - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert next_batch is not None assert len(generations) == 2 @@ -337,10 +337,10 @@ def test_batch_concatenate( - default_multi_requests_bloom_batch.stopping_criterias[1].max_new_tokens - 4 ): - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_bloom.generate_token(next_batch) + generations, next_batch, _ = default_bloom.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py index 8b45e781..250fa354 100644 --- a/server/tests/models/test_causal_lm.py +++ b/server/tests/models/test_causal_lm.py @@ -99,7 +99,9 @@ def test_causal_lm_batch_type(default_causal_lm): def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch): sequence_length = len(default_causal_lm_batch.all_input_ids[0]) - generations, next_batch = default_causal_lm.generate_token(default_causal_lm_batch) + generations, next_batch, _ = default_causal_lm.generate_token( + default_causal_lm_batch + ) assert len(generations) == len(next_batch) assert isinstance(next_batch, CausalLMBatch) @@ -151,10 +153,10 @@ def test_causal_lm_generate_token_completion( ): next_batch = default_causal_lm_batch for _ in range(default_causal_lm_batch.stopping_criterias[0].max_new_tokens - 1): - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 @@ -174,10 +176,10 @@ def test_causal_lm_generate_token_completion_multi( for i in range( default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 1 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert next_batch is not None assert len(generations) == 2 @@ -200,10 +202,10 @@ def test_causal_lm_generate_token_completion_multi( for _ in range( stopping_criterias[0].max_new_tokens - stopping_criterias[1].max_new_tokens - 1 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 @@ -222,11 +224,11 @@ def test_batch_concatenate( default_causal_lm, default_causal_lm_batch, default_multi_requests_causal_lm_batch ): next_batch_0 = default_causal_lm_batch - _, next_batch_0 = default_causal_lm.generate_token(next_batch_0) - _, next_batch_0 = default_causal_lm.generate_token(next_batch_0) + _, next_batch_0, _ = default_causal_lm.generate_token(next_batch_0) + _, next_batch_0, _ = default_causal_lm.generate_token(next_batch_0) next_batch_1 = default_multi_requests_causal_lm_batch - _, next_batch_1 = default_causal_lm.generate_token(next_batch_1) + _, next_batch_1, _ = default_causal_lm.generate_token(next_batch_1) # Clone past_key_values before concatenating to compare after, # because they are removed from the concatenated batches @@ -285,10 +287,10 @@ def test_batch_concatenate( for _ in range( default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert next_batch is not None assert len(generations) == 3 @@ -311,10 +313,10 @@ def test_batch_concatenate( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert next_batch is not None assert len(generations) == 2 @@ -333,10 +335,10 @@ def test_batch_concatenate( - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 4 ): - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_causal_lm.generate_token(next_batch) + generations, next_batch, _ = default_causal_lm.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py index fceec560..1e40e766 100644 --- a/server/tests/models/test_santacoder.py +++ b/server/tests/models/test_santacoder.py @@ -55,10 +55,10 @@ def test_santacoder_generate_token_completion(default_santacoder, default_pb_bat next_batch = batch for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): - generations, next_batch = default_santacoder.generate_token(next_batch) + generations, next_batch, _ = default_santacoder.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_santacoder.generate_token(next_batch) + generations, next_batch, _ = default_santacoder.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 @@ -83,10 +83,10 @@ def test_fim_santacoder_generate_token_completion( next_batch = batch for _ in range(batch.stopping_criterias[0].max_new_tokens - 1): - generations, next_batch = default_santacoder.generate_token(next_batch) + generations, next_batch, _ = default_santacoder.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_santacoder.generate_token(next_batch) + generations, next_batch, _ = default_santacoder.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py index 373867c7..735ab5eb 100644 --- a/server/tests/models/test_seq2seq_lm.py +++ b/server/tests/models/test_seq2seq_lm.py @@ -103,7 +103,7 @@ def test_seq2seq_lm_batch_type(default_seq2seq_lm): def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch): sequence_length = len(default_seq2seq_lm_batch.input_ids[0]) - generations, next_batch = default_seq2seq_lm.generate_token( + generations, next_batch, _ = default_seq2seq_lm.generate_token( default_seq2seq_lm_batch ) @@ -173,10 +173,10 @@ def test_seq2seq_lm_generate_token_completion( ): next_batch = default_seq2seq_lm_batch for _ in range(6): - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 @@ -191,10 +191,10 @@ def test_seq2seq_lm_generate_token_completion_multi( next_batch = default_multi_requests_seq2seq_lm_batch for i in range(4): - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None assert len(generations) == 2 @@ -207,10 +207,10 @@ def test_seq2seq_lm_generate_token_completion_multi( next_batch = next_batch.filter([next_batch.requests[0].id]) - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 @@ -228,11 +228,11 @@ def test_batch_concatenate( default_multi_requests_seq2seq_lm_batch, ): next_batch_0 = default_seq2seq_lm_batch - _, next_batch_0 = default_seq2seq_lm.generate_token(next_batch_0) - _, next_batch_0 = default_seq2seq_lm.generate_token(next_batch_0) + _, next_batch_0, _ = default_seq2seq_lm.generate_token(next_batch_0) + _, next_batch_0, _ = default_seq2seq_lm.generate_token(next_batch_0) next_batch_1 = default_multi_requests_seq2seq_lm_batch - _, next_batch_1 = default_seq2seq_lm.generate_token(next_batch_1) + _, next_batch_1, _ = default_seq2seq_lm.generate_token(next_batch_1) # Copy hidden state because it is removed from the concatenated branches next_batch_0_encoder_last_hidden_state = next_batch_0.encoder_last_hidden_state @@ -324,10 +324,10 @@ def test_batch_concatenate( ) for _ in range(3): - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert len(generations) == len(next_batch) - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None assert len(generations) == 3 @@ -342,7 +342,7 @@ def test_batch_concatenate( [next_batch.requests[0].id, next_batch.requests[1].id] ) - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert next_batch is not None assert len(generations) == 2 @@ -352,7 +352,7 @@ def test_batch_concatenate( next_batch = next_batch.filter([next_batch.requests[1].id]) - generations, next_batch = default_seq2seq_lm.generate_token(next_batch) + generations, next_batch, _ = default_seq2seq_lm.generate_token(next_batch) assert next_batch is None assert len(generations) == 1 diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index b771264b..7b10256c 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -1,6 +1,5 @@ -from text_generation_server.utils.tokens import batch_top_tokens import torch -import inspect +import time from dataclasses import dataclass from opentelemetry import trace @@ -8,6 +7,7 @@ from typing import Optional, Tuple, List, Type, Dict from text_generation_server.models import Model +from text_generation_server.utils.tokens import batch_top_tokens from text_generation_server.models.types import ( Batch, Tokens, @@ -564,7 +564,8 @@ def forward( @tracer.start_as_current_span("generate_token") def generate_token( self, batch: CausalLMBatch - ) -> Tuple[List[Generation], Optional[CausalLMBatch]]: + ) -> Tuple[List[Generation], Optional[CausalLMBatch], Tuple[int, int]]: + start = time.time_ns() # slice the attention mask to the correct shape attention_mask = batch.attention_mask[:, : -batch.padding_right_offset] @@ -585,6 +586,8 @@ def generate_token( torch.log_softmax(logits[:, -1], -1), ) + start_decode = time.time_ns() + # Zipped iterator iterator = zip( batch.requests, @@ -731,7 +734,9 @@ def generate_token( # We finished all generations in the batch; there is no next batch if stopped: - return generations, None + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, None, (forward_ns, decode_ns) # Slice unused values from prefill batch.input_ids = batch.input_ids[:, :1] @@ -747,4 +752,6 @@ def generate_token( # Update past key values batch.past_key_values = past - return generations, batch + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, batch, (forward_ns, decode_ns) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 14d30635..930082cd 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -1,6 +1,6 @@ import math +import time import itertools -from text_generation_server.utils.tokens import batch_top_tokens import torch import torch.distributed @@ -9,9 +9,10 @@ from dataclasses import dataclass from opentelemetry import trace from transformers import PreTrainedTokenizerBase -from typing import Optional, Tuple, List, Type, Union, Dict +from typing import Optional, Tuple, List, Type, Dict from text_generation_server.models import Model +from text_generation_server.utils.tokens import batch_top_tokens from text_generation_server.utils.speculate import get_speculate from text_generation_server.models.types import ( Batch, @@ -689,7 +690,7 @@ def warmup(self, batch: FlashCausalLMBatch): self.dtype, self.device, ) - _, batch = self.generate_token(batch) + _, batch, _ = self.generate_token(batch) except torch.cuda.OutOfMemoryError as e: raise RuntimeError( f"Not enough memory to handle {len(batch.input_ids)} prefill tokens. " @@ -799,7 +800,8 @@ def forward(self, batch: FlashCausalLMBatch) -> Tuple[torch.Tensor, torch.Tensor @tracer.start_as_current_span("generate_token") def generate_token( self, batch: FlashCausalLMBatch - ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch]]: + ) -> Tuple[List[Generation], Optional[FlashCausalLMBatch], Tuple[int, int]]: + start = time.time_ns() prefill = batch.cu_seqlen_prefill is not None prefill_logprobs = batch.prefill_next_token_indices is not None @@ -941,6 +943,8 @@ def generate_token( # GPU <-> CPU sync next_token_logprobs = next_token_logprobs.tolist() next_token_ids = next_input_ids.tolist() + accepted_ids = accepted_ids.tolist() + start_decode = time.time_ns() # Zipped iterator iterator = zip( @@ -977,7 +981,6 @@ def generate_token( # Append next token to all tokens next_token_texts = [] left = 0 - before = stopping_criteria.current_tokens current_stopped = False for j in range(index, index + n_accepted_ids): @@ -1092,7 +1095,7 @@ def generate_token( generations.append(generation) # Update values - batch.input_lengths[i] = input_length + n_accepted_ids.item() + batch.input_lengths[i] = input_length + n_accepted_ids if batch.input_lengths[i] > batch.max_seqlen: batch.max_seqlen = batch.input_lengths[i] batch.prefix_offsets[i] = prefix_offset @@ -1102,10 +1105,14 @@ def generate_token( if stopped: del batch # No need to return a batch if we know that all requests stopped - return generations, None + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, None, (forward_ns, decode_ns) batch.prefill_cu_outlens = None batch.prefill_head_indices = None batch.prefill_next_token_indices = None - return generations, batch + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, batch, (forward_ns, decode_ns) diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py index 86389ad2..2f28688d 100644 --- a/server/text_generation_server/models/idefics_causal_lm.py +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -1,17 +1,11 @@ import torch -import inspect -import re -from io import BytesIO -import base64 -from PIL import Image -import re +import time from dataclasses import dataclass from opentelemetry import trace from transformers import ( AutoProcessor, AutoTokenizer, - AutoModelForCausalLM, PreTrainedTokenizerBase, ProcessorMixin, ) @@ -670,7 +664,8 @@ def forward( @tracer.start_as_current_span("generate_token") def generate_token( self, batch: IdeficsCausalLMBatch - ) -> Tuple[List[Generation], Optional[IdeficsCausalLMBatch]]: + ) -> Tuple[List[Generation], Optional[IdeficsCausalLMBatch], Tuple[int, int]]: + start = time.time_ns() # slice the attention mask to the correct shape attention_mask = batch.attention_mask[:, : -batch.padding_right_offset] if batch.input_ids.size(1) == 1: @@ -699,6 +694,8 @@ def generate_token( # Hardcoded remove image tokens logits[:, 32000:32001] = torch.finfo(logits.dtype).min + start_decode = time.time_ns() + # Results generations: List[Generation] = [] stopped = True @@ -827,7 +824,9 @@ def generate_token( # We finished all generations in the batch; there is no next batch if stopped: - return generations, None + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, None, (forward_ns, decode_ns) # Slice unused values from prefill batch.input_ids = batch.input_ids[:, :1] @@ -847,4 +846,6 @@ def generate_token( batch.past_key_values = past batch.image_hidden_states = image_hidden_states - return generations, batch + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, batch, (forward_ns, decode_ns) diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index dfb21dcb..cb358672 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -65,7 +65,9 @@ def batch_type(self) -> Type[B]: raise NotImplementedError @abstractmethod - def generate_token(self, batch: B) -> Tuple[List[Generation], Optional[B]]: + def generate_token( + self, batch: B + ) -> Tuple[List[Generation], Optional[B], Tuple[int, int]]: raise NotImplementedError def warmup(self, batch: B) -> Optional[int]: diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index a85ef58e..f2e4cec6 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -1,11 +1,12 @@ -from text_generation_server.utils.tokens import batch_top_tokens import torch +import time from dataclasses import dataclass from opentelemetry import trace from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, PreTrainedTokenizerBase from typing import Optional, Tuple, List, Type, Dict +from text_generation_server.utils.tokens import batch_top_tokens from text_generation_server.models import Model from text_generation_server.models.types import ( GeneratedText, @@ -613,7 +614,8 @@ def forward( @tracer.start_as_current_span("generate_token") def generate_token( self, batch: Seq2SeqLMBatch - ) -> Tuple[List[Generation], Optional[Seq2SeqLMBatch]]: + ) -> Tuple[List[Generation], Optional[Seq2SeqLMBatch], Tuple[int, int]]: + start = time.time_ns() if batch.decoder_attention_mask is not None: # slice to the correct shape decoder_attention_mask = batch.decoder_attention_mask[ @@ -644,6 +646,8 @@ def generate_token( torch.log_softmax(logits[:, -1], -1), ) + start_decode = time.time_ns() + # Finished requests generations: List[Generation] = [] stopped = True @@ -788,7 +792,9 @@ def generate_token( # We finished all generations in the batch; there is no next batch if stopped: - return generations, None + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, None, (forward_ns, decode_ns) # We don't need input_ids after the prefill forward batch.input_ids = None @@ -799,4 +805,6 @@ def generate_token( batch.decoder_attention_mask[:, -batch.padding_right_offset] = 1 batch.padding_right_offset -= 1 - return generations, batch + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, batch, (forward_ns, decode_ns) diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index 75dba972..a65138c9 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -1,6 +1,7 @@ import asyncio import os import torch +import time from grpc import aio from loguru import logger @@ -76,6 +77,7 @@ async def Warmup(self, request, context): ) async def Prefill(self, request, context): + start = time.time_ns() if ( self.model.batch_type == IdeficsCausalLMBatch ): # Hack, i would rather use kwargs in the `from_pb` call @@ -91,15 +93,19 @@ async def Prefill(self, request, context): request.batch, self.model.tokenizer, self.model.dtype, self.model.device ) - generations, next_batch = self.model.generate_token(batch) + generations, next_batch, timings = self.model.generate_token(batch) self.cache.set(next_batch) return generate_pb2.PrefillResponse( generations=[generation.to_pb() for generation in generations], batch=next_batch.to_pb() if next_batch else None, + forward_ns=timings[0], + decode_ns=timings[1], + total_ns=time.time_ns() - start, ) async def Decode(self, request, context): + start = time.time_ns() if len(request.batches) == 0: raise ValueError("Must provide at least one batch") @@ -114,16 +120,23 @@ async def Decode(self, request, context): raise ValueError("All batches are empty") if len(batches) > 1: + start_concat = time.time_ns() batch = self.model.batch_type.concatenate(batches) + concat_ns = time.time_ns() - start_concat else: batch = batches[0] + concat_ns = None - generations, next_batch = self.model.generate_token(batch) + generations, next_batch, timings = self.model.generate_token(batch) self.cache.set(next_batch) return generate_pb2.DecodeResponse( generations=[generation.to_pb() for generation in generations], batch=next_batch.to_pb() if next_batch else None, + concat_ns=concat_ns, + forward_ns=timings[0], + decode_ns=timings[1], + total_ns=time.time_ns() - start, ) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 0d208104..ff0556df 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -92,7 +92,7 @@ def from_pb( class StopSequenceCriteria: def __init__(self, stop_sequence: str): stop_sequence = re.escape(stop_sequence) - self.regex = re.compile(f".*{stop_sequence}$") + self.regex = re.compile(f"{stop_sequence}$") def __call__(self, output: str) -> bool: if self.regex.findall(output): From 773aabdda6197cae3d2092f1cd6d9ce08d649185 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:04:26 +0100 Subject: [PATCH 499/793] fix: fix triton OutOfResources import --- server/text_generation_server/utils/gptq/custom_autotune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/gptq/custom_autotune.py b/server/text_generation_server/utils/gptq/custom_autotune.py index 17dff02e..589d89ef 100644 --- a/server/text_generation_server/utils/gptq/custom_autotune.py +++ b/server/text_generation_server/utils/gptq/custom_autotune.py @@ -90,7 +90,7 @@ def kernel_call(): return triton.testing.do_bench( kernel_call, percentiles=(0.5, 0.2, 0.8), rep=40 ) - except triton.compiler.OutOfResources: + except triton.OutOfResources: return (float("inf"), float("inf"), float("inf")) def run(self, *args, **kwargs): From 083c2de9f83c6070116a386d5c87e7b6c7538313 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:45:47 +0100 Subject: [PATCH 500/793] fix: fix quant linear autotune --- server/text_generation_server/utils/gptq/custom_autotune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/gptq/custom_autotune.py b/server/text_generation_server/utils/gptq/custom_autotune.py index 589d89ef..1eb40f1e 100644 --- a/server/text_generation_server/utils/gptq/custom_autotune.py +++ b/server/text_generation_server/utils/gptq/custom_autotune.py @@ -88,7 +88,7 @@ def kernel_call(): # In testings using only 40 reps seems to be close enough and it appears to be what PyTorch uses # PyTorch also sets fast_flush to True, but I didn't see any speedup so I'll leave the default return triton.testing.do_bench( - kernel_call, percentiles=(0.5, 0.2, 0.8), rep=40 + kernel_call, quantiles=(0.5, 0.2, 0.8), rep=40 ) except triton.OutOfResources: return (float("inf"), float("inf"), float("inf")) From 80a69204c1752fb8d754405b1d63cd345fa85e3e Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:01:43 +0100 Subject: [PATCH 501/793] fix: slice stopping criteria buffer --- server/text_generation_server/utils/tokens.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index ff0556df..53722fec 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -112,7 +112,7 @@ def __init__( self.stop_sequence_criterias = stop_sequence_criterias self.max_new_tokens = max_new_tokens self.current_tokens = 0 - self.current_output = "" + self.current_output = "test" self.ignore_eos_token = ignore_eos_token def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[str]]: @@ -124,6 +124,10 @@ def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[st return True, FinishReason.FINISH_REASON_EOS_TOKEN self.current_output += last_output + # There is no need to keep an output that is too long + if len(self.current_output) > 300: + # Slice to -200 to avoid doing it all the time + self.current_output = self.current_output[-200:] for stop_sequence_criteria in self.stop_sequence_criterias: if stop_sequence_criteria(self.current_output): return True, FinishReason.FINISH_REASON_STOP_SEQUENCE From 9b78a6eee32c76316d55bbb86c507625bf79ac40 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:04:58 +0100 Subject: [PATCH 502/793] fix: only keep stop sequence buffer if we have some --- server/text_generation_server/utils/tokens.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 53722fec..04cc8d97 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -112,7 +112,7 @@ def __init__( self.stop_sequence_criterias = stop_sequence_criterias self.max_new_tokens = max_new_tokens self.current_tokens = 0 - self.current_output = "test" + self.current_output = "" self.ignore_eos_token = ignore_eos_token def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[str]]: @@ -123,14 +123,15 @@ def __call__(self, last_token: int, last_output: str) -> Tuple[bool, Optional[st if not self.ignore_eos_token and last_token == self.eos_token_id: return True, FinishReason.FINISH_REASON_EOS_TOKEN - self.current_output += last_output - # There is no need to keep an output that is too long - if len(self.current_output) > 300: - # Slice to -200 to avoid doing it all the time - self.current_output = self.current_output[-200:] - for stop_sequence_criteria in self.stop_sequence_criterias: - if stop_sequence_criteria(self.current_output): - return True, FinishReason.FINISH_REASON_STOP_SEQUENCE + if self.stop_sequence_criterias: + self.current_output += last_output + # There is no need to keep an output that is too long + if len(self.current_output) > 300: + # Slice to -200 to avoid doing it all the time + self.current_output = self.current_output[-200:] + for stop_sequence_criteria in self.stop_sequence_criterias: + if stop_sequence_criteria(self.current_output): + return True, FinishReason.FINISH_REASON_STOP_SEQUENCE return False, None From 37555cf4e8eef8b3cbae25f749ce658a47c9f61a Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 15 Dec 2023 01:18:39 +0100 Subject: [PATCH 503/793] fix: max_past default value must be -1, not 0 (#1348) --- .../models/custom_modeling/flash_mistral_modeling.py | 2 +- .../models/custom_modeling/flash_mixtral_modeling.py | 2 +- server/text_generation_server/utils/flash_attn.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index b97866f7..afeaf7e5 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -149,7 +149,7 @@ def __init__( ): super().__init__() self.max_past = ( - config.sliding_window if config.sliding_window is not None else 0 + config.sliding_window if config.sliding_window is not None else -1 ) self.num_heads = config.num_attention_heads self.hidden_size = config.hidden_size diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py index ff2ed9fd..35bb3735 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py @@ -204,7 +204,7 @@ def __init__( ): super().__init__() self.max_past = ( - config.sliding_window if config.sliding_window is not None else 0 + config.sliding_window if config.sliding_window is not None else -1 ) self.num_heads = config.num_attention_heads self.hidden_size = config.hidden_size diff --git a/server/text_generation_server/utils/flash_attn.py b/server/text_generation_server/utils/flash_attn.py index 3237df82..02f01e65 100644 --- a/server/text_generation_server/utils/flash_attn.py +++ b/server/text_generation_server/utils/flash_attn.py @@ -72,6 +72,9 @@ def attention( softmax_scale, window_size_left=-1, ): + if window_size_left <= 0 and window_size_left != -1: + raise ValueError("`window_size_left` must be > 0 or -1") + if HAS_FLASH_ATTN_V2_CUDA: return flash_attn_2_cuda.varlen_fwd( q, From f3aea78fb642967838e7b5b1940a25fe67f4f7a9 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 15 Dec 2023 01:20:42 +0100 Subject: [PATCH 504/793] v1.3.3 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- docs/openapi.json | 2 +- integration-tests/pyproject.toml | 2 +- server/pyproject.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 162c68c2..da609e53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2754,7 +2754,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "1.3.2" +version = "1.3.3" dependencies = [ "average", "clap", @@ -2775,7 +2775,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.3.2" +version = "1.3.3" dependencies = [ "futures", "grpc-metadata", @@ -2791,7 +2791,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.3.2" +version = "1.3.3" dependencies = [ "clap", "ctrlc", @@ -2807,7 +2807,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.3.2" +version = "1.3.3" dependencies = [ "async-stream", "axum", diff --git a/Cargo.toml b/Cargo.toml index df728d27..12b02579 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.3.2" +version = "1.3.3" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index 92faf1ed..62751928 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.3.2" + "version": "1.3.3" }, "paths": { "/": { diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 2747473c..64723ae6 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-integration-tests" -version = "1.3.2" +version = "1.3.3" description = "Text Generation Inference integration tests" authors = ["Nicolas Patry "] diff --git a/server/pyproject.toml b/server/pyproject.toml index d519de63..ad3d5255 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.3.2" +version = "1.3.3" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] From 9b56d3fbf5d90f5654262a71b5e557da6766b7ad Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 15 Dec 2023 12:52:24 +0100 Subject: [PATCH 505/793] feat: relax mistral requirements (#1351) Close #1253 Close #1279 --- .github/workflows/build.yaml | 78 +- server/poetry.lock | 1095 +++++++++-------- server/pyproject.toml | 8 +- server/requirements_cuda.txt | 20 +- server/requirements_rocm.txt | 18 +- .../text_generation_server/models/__init__.py | 44 +- .../custom_modeling/flash_mistral_modeling.py | 9 - .../custom_modeling/flash_mixtral_modeling.py | 20 +- 8 files changed, 671 insertions(+), 621 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 395a0b6a..066ea889 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -146,11 +146,50 @@ jobs: cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=min + integration-tests: + concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + needs: + - start-runner + - build-and-push-image # Wait for the docker image to be built + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + env: + DOCKER_VOLUME: /cache + steps: + - uses: actions/checkout@v2 + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4.4.1 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Tailscale + uses: tailscale/github-action@7bd8039bf25c23c4ab1b8d6e2cc2da2280601966 + with: + authkey: ${{ secrets.TAILSCALE_AUTHKEY }} + - name: Prepare disks + run: | + sudo mkfs -t ext4 /dev/nvme1n1 + sudo mkdir ${{ env.DOCKER_VOLUME }} + sudo mount /dev/nvme1n1 ${{ env.DOCKER_VOLUME }} + - name: Install + run: | + make install-integration-tests + - name: Run tests + run: | + export DOCKER_IMAGE=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }} + export HUGGING_FACE_HUB_TOKEN=${{ secrets.HUGGING_FACE_HUB_TOKEN }} + pytest -s -vv integration-tests + build-and-push-image-rocm: concurrency: group: ${{ github.workflow }}-build-and-push-image-rocm-${{ github.head_ref || github.run_id }} cancel-in-progress: true - needs: start-runner # required to start the main job when the runner is ready + needs: + - start-runner + - build-and-push-image # Wait for the main docker image to be built + - integration-tests # Wait for the main integration-tests runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner permissions: contents: write @@ -235,43 +274,6 @@ jobs: cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache-rocm,mode=min cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache-rocm,mode=min - integration-tests: - concurrency: - group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - needs: - - start-runner - - build-and-push-image # Wait for the docker image to be built - - build-and-push-image-rocm - runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner - env: - DOCKER_VOLUME: /cache - steps: - - uses: actions/checkout@v2 - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4.4.1 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Tailscale - uses: tailscale/github-action@7bd8039bf25c23c4ab1b8d6e2cc2da2280601966 - with: - authkey: ${{ secrets.TAILSCALE_AUTHKEY }} - - name: Prepare disks - run: | - sudo mkfs -t ext4 /dev/nvme1n1 - sudo mkdir ${{ env.DOCKER_VOLUME }} - sudo mount /dev/nvme1n1 ${{ env.DOCKER_VOLUME }} - - name: Install - run: | - make install-integration-tests - - name: Run tests - run: | - export DOCKER_IMAGE=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:sha-${{ env.GITHUB_SHA_SHORT }} - export HUGGING_FACE_HUB_TOKEN=${{ secrets.HUGGING_FACE_HUB_TOKEN }} - pytest -s -vv integration-tests - stop-runner: name: Stop self-hosted EC2 runner needs: diff --git a/server/poetry.lock b/server/poetry.lock index 48ae40fe..3326d0aa 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -2,115 +2,117 @@ [[package]] name = "accelerate" -version = "0.20.3" +version = "0.25.0" description = "Accelerate" optional = true -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" files = [ - {file = "accelerate-0.20.3-py3-none-any.whl", hash = "sha256:147183e7a2215f7bd45a7af3b986a963daa8a61fa58b0912b9473049e011ad15"}, - {file = "accelerate-0.20.3.tar.gz", hash = "sha256:79a896978c20dac270083d42bf033f4c9a80dcdd6b946f1ca92d8d6d0f0f5ba9"}, + {file = "accelerate-0.25.0-py3-none-any.whl", hash = "sha256:c7bb817eb974bba0ff3ea1ba0f24d55afb86d50e3d4fe98d6922dc69cf2ccff1"}, + {file = "accelerate-0.25.0.tar.gz", hash = "sha256:ecf55b0ab278a1dac8539dde0d276977aff04683f07ede73eaf02478538576a1"}, ] [package.dependencies] +huggingface-hub = "*" numpy = ">=1.17" packaging = ">=20.0" psutil = "*" pyyaml = "*" -torch = ">=1.6.0" +safetensors = ">=0.3.1" +torch = ">=1.10.0" [package.extras] -dev = ["black (>=23.1,<24.0)", "datasets", "deepspeed", "evaluate", "hf-doc-builder (>=0.3.0)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "rich", "ruff (>=0.0.241)", "scikit-learn", "scipy", "tqdm", "transformers", "urllib3 (<2.0.0)"] +dev = ["bitsandbytes", "black (>=23.1,<24.0)", "datasets", "deepspeed", "evaluate", "hf-doc-builder (>=0.3.0)", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "rich", "ruff (>=0.0.241)", "scikit-learn", "scipy", "timm", "tqdm", "transformers", "urllib3 (<2.0.0)"] quality = ["black (>=23.1,<24.0)", "hf-doc-builder (>=0.3.0)", "ruff (>=0.0.241)", "urllib3 (<2.0.0)"] rich = ["rich"] sagemaker = ["sagemaker"] -test-dev = ["datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "tqdm", "transformers"] +test-dev = ["bitsandbytes", "datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "timm", "tqdm", "transformers"] test-prod = ["parameterized", "pytest", "pytest-subtests", "pytest-xdist"] -test-trackers = ["comet-ml", "tensorboard", "wandb"] -testing = ["datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "tqdm", "transformers"] +test-trackers = ["comet-ml", "dvclive", "tensorboard", "wandb"] +testing = ["bitsandbytes", "datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "timm", "tqdm", "transformers"] [[package]] name = "aiohttp" -version = "3.9.0" +version = "3.9.1" description = "Async http client/server framework (asyncio)" optional = true python-versions = ">=3.8" files = [ - {file = "aiohttp-3.9.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6896b8416be9ada4d22cd359d7cb98955576ce863eadad5596b7cdfbf3e17c6c"}, - {file = "aiohttp-3.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1736d87dad8ef46a8ec9cddd349fa9f7bd3a064c47dd6469c0d6763d3d49a4fc"}, - {file = "aiohttp-3.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c9e5f4d7208cda1a2bb600e29069eecf857e6980d0ccc922ccf9d1372c16f4b"}, - {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8488519aa05e636c5997719fe543c8daf19f538f4fa044f3ce94bee608817cff"}, - {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ab16c254e2312efeb799bc3c06897f65a133b38b69682bf75d1f1ee1a9c43a9"}, - {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a94bde005a8f926d0fa38b88092a03dea4b4875a61fbcd9ac6f4351df1b57cd"}, - {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b777c9286b6c6a94f50ddb3a6e730deec327e9e2256cb08b5530db0f7d40fd8"}, - {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:571760ad7736b34d05597a1fd38cbc7d47f7b65deb722cb8e86fd827404d1f6b"}, - {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:deac0a32aec29608eb25d730f4bc5a261a65b6c48ded1ed861d2a1852577c932"}, - {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:4ee1b4152bc3190cc40ddd6a14715e3004944263ea208229ab4c297712aa3075"}, - {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:3607375053df58ed6f23903aa10cf3112b1240e8c799d243bbad0f7be0666986"}, - {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:65b0a70a25456d329a5e1426702dde67be0fb7a4ead718005ba2ca582d023a94"}, - {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a2eb5311a37fe105aa35f62f75a078537e1a9e4e1d78c86ec9893a3c97d7a30"}, - {file = "aiohttp-3.9.0-cp310-cp310-win32.whl", hash = "sha256:2cbc14a13fb6b42d344e4f27746a4b03a2cb0c1c3c5b932b0d6ad8881aa390e3"}, - {file = "aiohttp-3.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ac9669990e2016d644ba8ae4758688534aabde8dbbc81f9af129c3f5f01ca9cd"}, - {file = "aiohttp-3.9.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f8e05f5163528962ce1d1806fce763ab893b1c5b7ace0a3538cd81a90622f844"}, - {file = "aiohttp-3.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4afa8f71dba3a5a2e1e1282a51cba7341ae76585345c43d8f0e624882b622218"}, - {file = "aiohttp-3.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f929f4c9b9a00f3e6cc0587abb95ab9c05681f8b14e0fe1daecfa83ea90f8318"}, - {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28185e36a78d247c55e9fbea2332d16aefa14c5276a582ce7a896231c6b1c208"}, - {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a486ddf57ab98b6d19ad36458b9f09e6022de0381674fe00228ca7b741aacb2f"}, - {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70e851f596c00f40a2f00a46126c95c2e04e146015af05a9da3e4867cfc55911"}, - {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5b7bf8fe4d39886adc34311a233a2e01bc10eb4e842220235ed1de57541a896"}, - {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c67a51ea415192c2e53e4e048c78bab82d21955b4281d297f517707dc836bf3d"}, - {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:694df243f394629bcae2d8ed94c589a181e8ba8604159e6e45e7b22e58291113"}, - {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3dd8119752dd30dd7bca7d4bc2a92a59be6a003e4e5c2cf7e248b89751b8f4b7"}, - {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:eb6dfd52063186ac97b4caa25764cdbcdb4b10d97f5c5f66b0fa95052e744eb7"}, - {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:d97c3e286d0ac9af6223bc132dc4bad6540b37c8d6c0a15fe1e70fb34f9ec411"}, - {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:816f4db40555026e4cdda604a1088577c1fb957d02f3f1292e0221353403f192"}, - {file = "aiohttp-3.9.0-cp311-cp311-win32.whl", hash = "sha256:3abf0551874fecf95f93b58f25ef4fc9a250669a2257753f38f8f592db85ddea"}, - {file = "aiohttp-3.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:e18d92c3e9e22553a73e33784fcb0ed484c9874e9a3e96c16a8d6a1e74a0217b"}, - {file = "aiohttp-3.9.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:99ae01fb13a618b9942376df77a1f50c20a281390dad3c56a6ec2942e266220d"}, - {file = "aiohttp-3.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:05857848da443c8c12110d99285d499b4e84d59918a21132e45c3f0804876994"}, - {file = "aiohttp-3.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:317719d7f824eba55857fe0729363af58e27c066c731bc62cd97bc9c3d9c7ea4"}, - {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1e3b3c107ccb0e537f309f719994a55621acd2c8fdf6d5ce5152aed788fb940"}, - {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45820ddbb276113ead8d4907a7802adb77548087ff5465d5c554f9aa3928ae7d"}, - {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:05a183f1978802588711aed0dea31e697d760ce9055292db9dc1604daa9a8ded"}, - {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a4cd44788ea0b5e6bb8fa704597af3a30be75503a7ed1098bc5b8ffdf6c982"}, - {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:673343fbc0c1ac44d0d2640addc56e97a052504beacd7ade0dc5e76d3a4c16e8"}, - {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7e8a3b79b6d186a9c99761fd4a5e8dd575a48d96021f220ac5b5fa856e5dd029"}, - {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6777a390e41e78e7c45dab43a4a0196c55c3b8c30eebe017b152939372a83253"}, - {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7ae5f99a32c53731c93ac3075abd3e1e5cfbe72fc3eaac4c27c9dd64ba3b19fe"}, - {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:f1e4f254e9c35d8965d377e065c4a8a55d396fe87c8e7e8429bcfdeeb229bfb3"}, - {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11ca808f9a6b63485059f5f6e164ef7ec826483c1212a44f268b3653c91237d8"}, - {file = "aiohttp-3.9.0-cp312-cp312-win32.whl", hash = "sha256:de3cc86f4ea8b4c34a6e43a7306c40c1275e52bfa9748d869c6b7d54aa6dad80"}, - {file = "aiohttp-3.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca4fddf84ac7d8a7d0866664936f93318ff01ee33e32381a115b19fb5a4d1202"}, - {file = "aiohttp-3.9.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f09960b5bb1017d16c0f9e9f7fc42160a5a49fa1e87a175fd4a2b1a1833ea0af"}, - {file = "aiohttp-3.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8303531e2c17b1a494ffaeba48f2da655fe932c4e9a2626c8718403c83e5dd2b"}, - {file = "aiohttp-3.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4790e44f46a4aa07b64504089def5744d3b6780468c4ec3a1a36eb7f2cae9814"}, - {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1d7edf74a36de0e5ca50787e83a77cf352f5504eb0ffa3f07000a911ba353fb"}, - {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94697c7293199c2a2551e3e3e18438b4cba293e79c6bc2319f5fd652fccb7456"}, - {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a1b66dbb8a7d5f50e9e2ea3804b01e766308331d0cac76eb30c563ac89c95985"}, - {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9623cfd9e85b76b83ef88519d98326d4731f8d71869867e47a0b979ffec61c73"}, - {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f32c86dc967ab8c719fd229ce71917caad13cc1e8356ee997bf02c5b368799bf"}, - {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f50b4663c3e0262c3a361faf440761fbef60ccdde5fe8545689a4b3a3c149fb4"}, - {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dcf71c55ec853826cd70eadb2b6ac62ec577416442ca1e0a97ad875a1b3a0305"}, - {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:42fe4fd9f0dfcc7be4248c162d8056f1d51a04c60e53366b0098d1267c4c9da8"}, - {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:76a86a9989ebf82ee61e06e2bab408aec4ea367dc6da35145c3352b60a112d11"}, - {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f9e09a1c83521d770d170b3801eea19b89f41ccaa61d53026ed111cb6f088887"}, - {file = "aiohttp-3.9.0-cp38-cp38-win32.whl", hash = "sha256:a00ce44c21612d185c5275c5cba4bab8d7c1590f248638b667ed8a782fa8cd6f"}, - {file = "aiohttp-3.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:d5b9345ab92ebe6003ae11d8092ce822a0242146e6fa270889b9ba965457ca40"}, - {file = "aiohttp-3.9.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98d21092bf2637c5fa724a428a69e8f5955f2182bff61f8036827cf6ce1157bf"}, - {file = "aiohttp-3.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:35a68cd63ca6aaef5707888f17a70c36efe62b099a4e853d33dc2e9872125be8"}, - {file = "aiohttp-3.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d7f6235c7475658acfc1769d968e07ab585c79f6ca438ddfecaa9a08006aee2"}, - {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db04d1de548f7a62d1dd7e7cdf7c22893ee168e22701895067a28a8ed51b3735"}, - {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:536b01513d67d10baf6f71c72decdf492fb7433c5f2f133e9a9087379d4b6f31"}, - {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c8b0a6487e8109427ccf638580865b54e2e3db4a6e0e11c02639231b41fc0f"}, - {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7276fe0017664414fdc3618fca411630405f1aaf0cc3be69def650eb50441787"}, - {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23170247ef89ffa842a02bbfdc425028574d9e010611659abeb24d890bc53bb8"}, - {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b1a2ea8252cacc7fd51df5a56d7a2bb1986ed39be9397b51a08015727dfb69bd"}, - {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d71abc15ff7047412ef26bf812dfc8d0d1020d664617f4913df2df469f26b76"}, - {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:2d820162c8c2bdbe97d328cd4f417c955ca370027dce593345e437b2e9ffdc4d"}, - {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:2779f5e7c70f7b421915fd47db332c81de365678180a9f3ab404088f87ba5ff9"}, - {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:366bc870d7ac61726f32a489fbe3d1d8876e87506870be66b01aeb84389e967e"}, - {file = "aiohttp-3.9.0-cp39-cp39-win32.whl", hash = "sha256:1df43596b826022b14998f0460926ce261544fedefe0d2f653e1b20f49e96454"}, - {file = "aiohttp-3.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:9c196b30f1b1aa3363a69dd69079ae9bec96c2965c4707eaa6914ba099fb7d4f"}, - {file = "aiohttp-3.9.0.tar.gz", hash = "sha256:09f23292d29135025e19e8ff4f0a68df078fe4ee013bca0105b2e803989de92d"}, + {file = "aiohttp-3.9.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1f80197f8b0b846a8d5cf7b7ec6084493950d0882cc5537fb7b96a69e3c8590"}, + {file = "aiohttp-3.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c72444d17777865734aa1a4d167794c34b63e5883abb90356a0364a28904e6c0"}, + {file = "aiohttp-3.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b05d5cbe9dafcdc733262c3a99ccf63d2f7ce02543620d2bd8db4d4f7a22f83"}, + {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c4fa235d534b3547184831c624c0b7c1e262cd1de847d95085ec94c16fddcd5"}, + {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:289ba9ae8e88d0ba16062ecf02dd730b34186ea3b1e7489046fc338bdc3361c4"}, + {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bff7e2811814fa2271be95ab6e84c9436d027a0e59665de60edf44e529a42c1f"}, + {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81b77f868814346662c96ab36b875d7814ebf82340d3284a31681085c051320f"}, + {file = "aiohttp-3.9.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b9c7426923bb7bd66d409da46c41e3fb40f5caf679da624439b9eba92043fa6"}, + {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8d44e7bf06b0c0a70a20f9100af9fcfd7f6d9d3913e37754c12d424179b4e48f"}, + {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22698f01ff5653fe66d16ffb7658f582a0ac084d7da1323e39fd9eab326a1f26"}, + {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ca7ca5abfbfe8d39e653870fbe8d7710be7a857f8a8386fc9de1aae2e02ce7e4"}, + {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:8d7f98fde213f74561be1d6d3fa353656197f75d4edfbb3d94c9eb9b0fc47f5d"}, + {file = "aiohttp-3.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5216b6082c624b55cfe79af5d538e499cd5f5b976820eac31951fb4325974501"}, + {file = "aiohttp-3.9.1-cp310-cp310-win32.whl", hash = "sha256:0e7ba7ff228c0d9a2cd66194e90f2bca6e0abca810b786901a569c0de082f489"}, + {file = "aiohttp-3.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:c7e939f1ae428a86e4abbb9a7c4732bf4706048818dfd979e5e2839ce0159f23"}, + {file = "aiohttp-3.9.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:df9cf74b9bc03d586fc53ba470828d7b77ce51b0582d1d0b5b2fb673c0baa32d"}, + {file = "aiohttp-3.9.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ecca113f19d5e74048c001934045a2b9368d77b0b17691d905af18bd1c21275e"}, + {file = "aiohttp-3.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8cef8710fb849d97c533f259103f09bac167a008d7131d7b2b0e3a33269185c0"}, + {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bea94403a21eb94c93386d559bce297381609153e418a3ffc7d6bf772f59cc35"}, + {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91c742ca59045dce7ba76cab6e223e41d2c70d79e82c284a96411f8645e2afff"}, + {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6c93b7c2e52061f0925c3382d5cb8980e40f91c989563d3d32ca280069fd6a87"}, + {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee2527134f95e106cc1653e9ac78846f3a2ec1004cf20ef4e02038035a74544d"}, + {file = "aiohttp-3.9.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11ff168d752cb41e8492817e10fb4f85828f6a0142b9726a30c27c35a1835f01"}, + {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b8c3a67eb87394386847d188996920f33b01b32155f0a94f36ca0e0c635bf3e3"}, + {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c7b5d5d64e2a14e35a9240b33b89389e0035e6de8dbb7ffa50d10d8b65c57449"}, + {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:69985d50a2b6f709412d944ffb2e97d0be154ea90600b7a921f95a87d6f108a2"}, + {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:c9110c06eaaac7e1f5562caf481f18ccf8f6fdf4c3323feab28a93d34cc646bd"}, + {file = "aiohttp-3.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d737e69d193dac7296365a6dcb73bbbf53bb760ab25a3727716bbd42022e8d7a"}, + {file = "aiohttp-3.9.1-cp311-cp311-win32.whl", hash = "sha256:4ee8caa925aebc1e64e98432d78ea8de67b2272252b0a931d2ac3bd876ad5544"}, + {file = "aiohttp-3.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:a34086c5cc285be878622e0a6ab897a986a6e8bf5b67ecb377015f06ed316587"}, + {file = "aiohttp-3.9.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f800164276eec54e0af5c99feb9494c295118fc10a11b997bbb1348ba1a52065"}, + {file = "aiohttp-3.9.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:500f1c59906cd142d452074f3811614be04819a38ae2b3239a48b82649c08821"}, + {file = "aiohttp-3.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0b0a6a36ed7e164c6df1e18ee47afbd1990ce47cb428739d6c99aaabfaf1b3af"}, + {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69da0f3ed3496808e8cbc5123a866c41c12c15baaaead96d256477edf168eb57"}, + {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:176df045597e674fa950bf5ae536be85699e04cea68fa3a616cf75e413737eb5"}, + {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b796b44111f0cab6bbf66214186e44734b5baab949cb5fb56154142a92989aeb"}, + {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f27fdaadce22f2ef950fc10dcdf8048407c3b42b73779e48a4e76b3c35bca26c"}, + {file = "aiohttp-3.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcb6532b9814ea7c5a6a3299747c49de30e84472fa72821b07f5a9818bce0f66"}, + {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:54631fb69a6e44b2ba522f7c22a6fb2667a02fd97d636048478db2fd8c4e98fe"}, + {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4b4c452d0190c5a820d3f5c0f3cd8a28ace48c54053e24da9d6041bf81113183"}, + {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:cae4c0c2ca800c793cae07ef3d40794625471040a87e1ba392039639ad61ab5b"}, + {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:565760d6812b8d78d416c3c7cfdf5362fbe0d0d25b82fed75d0d29e18d7fc30f"}, + {file = "aiohttp-3.9.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:54311eb54f3a0c45efb9ed0d0a8f43d1bc6060d773f6973efd90037a51cd0a3f"}, + {file = "aiohttp-3.9.1-cp312-cp312-win32.whl", hash = "sha256:85c3e3c9cb1d480e0b9a64c658cd66b3cfb8e721636ab8b0e746e2d79a7a9eed"}, + {file = "aiohttp-3.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:11cb254e397a82efb1805d12561e80124928e04e9c4483587ce7390b3866d213"}, + {file = "aiohttp-3.9.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8a22a34bc594d9d24621091d1b91511001a7eea91d6652ea495ce06e27381f70"}, + {file = "aiohttp-3.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:598db66eaf2e04aa0c8900a63b0101fdc5e6b8a7ddd805c56d86efb54eb66672"}, + {file = "aiohttp-3.9.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c9376e2b09895c8ca8b95362283365eb5c03bdc8428ade80a864160605715f1"}, + {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41473de252e1797c2d2293804e389a6d6986ef37cbb4a25208de537ae32141dd"}, + {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c5857612c9813796960c00767645cb5da815af16dafb32d70c72a8390bbf690"}, + {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ffcd828e37dc219a72c9012ec44ad2e7e3066bec6ff3aaa19e7d435dbf4032ca"}, + {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:219a16763dc0294842188ac8a12262b5671817042b35d45e44fd0a697d8c8361"}, + {file = "aiohttp-3.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f694dc8a6a3112059258a725a4ebe9acac5fe62f11c77ac4dcf896edfa78ca28"}, + {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:bcc0ea8d5b74a41b621ad4a13d96c36079c81628ccc0b30cfb1603e3dfa3a014"}, + {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:90ec72d231169b4b8d6085be13023ece8fa9b1bb495e4398d847e25218e0f431"}, + {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:cf2a0ac0615842b849f40c4d7f304986a242f1e68286dbf3bd7a835e4f83acfd"}, + {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:0e49b08eafa4f5707ecfb321ab9592717a319e37938e301d462f79b4e860c32a"}, + {file = "aiohttp-3.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2c59e0076ea31c08553e868cec02d22191c086f00b44610f8ab7363a11a5d9d8"}, + {file = "aiohttp-3.9.1-cp38-cp38-win32.whl", hash = "sha256:4831df72b053b1eed31eb00a2e1aff6896fb4485301d4ccb208cac264b648db4"}, + {file = "aiohttp-3.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:3135713c5562731ee18f58d3ad1bf41e1d8883eb68b363f2ffde5b2ea4b84cc7"}, + {file = "aiohttp-3.9.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cfeadf42840c1e870dc2042a232a8748e75a36b52d78968cda6736de55582766"}, + {file = "aiohttp-3.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:70907533db712f7aa791effb38efa96f044ce3d4e850e2d7691abd759f4f0ae0"}, + {file = "aiohttp-3.9.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cdefe289681507187e375a5064c7599f52c40343a8701761c802c1853a504558"}, + {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7481f581251bb5558ba9f635db70908819caa221fc79ee52a7f58392778c636"}, + {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:49f0c1b3c2842556e5de35f122fc0f0b721334ceb6e78c3719693364d4af8499"}, + {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0d406b01a9f5a7e232d1b0d161b40c05275ffbcbd772dc18c1d5a570961a1ca4"}, + {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d8e4450e7fe24d86e86b23cc209e0023177b6d59502e33807b732d2deb6975f"}, + {file = "aiohttp-3.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c0266cd6f005e99f3f51e583012de2778e65af6b73860038b968a0a8888487a"}, + {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab221850108a4a063c5b8a70f00dd7a1975e5a1713f87f4ab26a46e5feac5a0e"}, + {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c88a15f272a0ad3d7773cf3a37cc7b7d077cbfc8e331675cf1346e849d97a4e5"}, + {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:237533179d9747080bcaad4d02083ce295c0d2eab3e9e8ce103411a4312991a0"}, + {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f"}, + {file = "aiohttp-3.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04fa38875e53eb7e354ece1607b1d2fdee2d175ea4e4d745f6ec9f751fe20c7c"}, + {file = "aiohttp-3.9.1-cp39-cp39-win32.whl", hash = "sha256:82eefaf1a996060602f3cc1112d93ba8b201dbf5d8fd9611227de2003dddb3b7"}, + {file = "aiohttp-3.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:9b05d33ff8e6b269e30a7957bd3244ffbce2a7a35a81b81c382629b80af1a8bf"}, + {file = "aiohttp-3.9.1.tar.gz", hash = "sha256:8fc49a87ac269d4529da45871e2ffb6874e87779c3d0e2ccd813c0899221239d"}, ] [package.dependencies] @@ -180,13 +182,13 @@ files = [ [[package]] name = "bitsandbytes" -version = "0.41.2.post2" +version = "0.41.3.post2" description = "k-bit optimizers and matrix multiplication routines." optional = true python-versions = "*" files = [ - {file = "bitsandbytes-0.41.2.post2-py3-none-any.whl", hash = "sha256:98e5e1979aea3d481ed06181c689f3a154d7f5dc1af770c5173485bc54cf7b72"}, - {file = "bitsandbytes-0.41.2.post2.tar.gz", hash = "sha256:d374da4700651f36a285ed53e012ee527736109614e3f5c0249985d41027136d"}, + {file = "bitsandbytes-0.41.3.post2-py3-none-any.whl", hash = "sha256:ceb301a3d4e6bf52bdad8d09f3064ac194bdfdeae535994c0315bd2ef7639cca"}, + {file = "bitsandbytes-0.41.3.post2.tar.gz", hash = "sha256:7d25a51fb3b74b58e569473f8b70a5239124c0593dc053479c41cf2cd6730502"}, ] [[package]] @@ -326,20 +328,20 @@ files = [ [[package]] name = "datasets" -version = "2.14.7" +version = "2.15.0" description = "HuggingFace community-driven open-source library of datasets" optional = true python-versions = ">=3.8.0" files = [ - {file = "datasets-2.14.7-py3-none-any.whl", hash = "sha256:1a64041a7da4f4130f736fc371c1f528b8ddd208cebe156400f65719bdbba79d"}, - {file = "datasets-2.14.7.tar.gz", hash = "sha256:394cf9b4ec0694b25945977b16ad5d18d5c15fb0e94141713eb8ead7452caf9e"}, + {file = "datasets-2.15.0-py3-none-any.whl", hash = "sha256:6d658d23811393dfc982d026082e1650bdaaae28f6a86e651966cb072229a228"}, + {file = "datasets-2.15.0.tar.gz", hash = "sha256:a26d059370bd7503bd60e9337977199a13117a83f72fb61eda7e66f0c4d50b2b"}, ] [package.dependencies] aiohttp = "*" dill = ">=0.3.0,<0.3.8" fsspec = {version = ">=2023.1.0,<=2023.10.0", extras = ["http"]} -huggingface-hub = ">=0.14.0,<1.0.0" +huggingface-hub = ">=0.18.0" multiprocess = "*" numpy = ">=1.17" packaging = "*" @@ -355,15 +357,15 @@ xxhash = "*" apache-beam = ["apache-beam (>=2.26.0,<2.44.0)"] audio = ["librosa", "soundfile (>=0.12.1)"] benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "black (>=23.1,<24.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "rarfile (>=4.0)", "ruff (>=0.0.241)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] +dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "black (>=23.1,<24.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "rarfile (>=4.0)", "ruff (>=0.0.241)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "typing-extensions (>=4.6.1)", "zstandard"] docs = ["s3fs", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos", "torch", "transformers"] -jax = ["jax (>=0.2.8,!=0.3.2,<=0.3.25)", "jaxlib (>=0.1.65,<=0.3.25)"] +jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"] quality = ["black (>=23.1,<24.0)", "pyyaml (>=5.3.1)", "ruff (>=0.0.241)"] s3 = ["s3fs"] tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos"] tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "zstandard"] +tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "typing-extensions (>=4.6.1)", "zstandard"] torch = ["torch"] vision = ["Pillow (>=6.2.1)"] @@ -441,72 +443,88 @@ typing = ["typing-extensions (>=4.8)"] [[package]] name = "frozenlist" -version = "1.4.0" +version = "1.4.1" description = "A list-like structure which implements collections.abc.MutableSequence" optional = true python-versions = ">=3.8" files = [ - {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab"}, - {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559"}, - {file = "frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c"}, - {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b"}, - {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea"}, - {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326"}, - {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963"}, - {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300"}, - {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b"}, - {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8"}, - {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb"}, - {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9"}, - {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62"}, - {file = "frozenlist-1.4.0-cp310-cp310-win32.whl", hash = "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0"}, - {file = "frozenlist-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956"}, - {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95"}, - {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3"}, - {file = "frozenlist-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc"}, - {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839"}, - {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c"}, - {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f"}, - {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b"}, - {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b"}, - {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472"}, - {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01"}, - {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f"}, - {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467"}, - {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb"}, - {file = "frozenlist-1.4.0-cp311-cp311-win32.whl", hash = "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431"}, - {file = "frozenlist-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1"}, - {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3"}, - {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503"}, - {file = "frozenlist-1.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9"}, - {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf"}, - {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2"}, - {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc"}, - {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672"}, - {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919"}, - {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc"}, - {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79"}, - {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e"}, - {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781"}, - {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8"}, - {file = "frozenlist-1.4.0-cp38-cp38-win32.whl", hash = "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc"}, - {file = "frozenlist-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7"}, - {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf"}, - {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963"}, - {file = "frozenlist-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f"}, - {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1"}, - {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef"}, - {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87"}, - {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6"}, - {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087"}, - {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3"}, - {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d"}, - {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2"}, - {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a"}, - {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3"}, - {file = "frozenlist-1.4.0-cp39-cp39-win32.whl", hash = "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f"}, - {file = "frozenlist-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167"}, - {file = "frozenlist-1.4.0.tar.gz", hash = "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"}, + {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"}, + {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"}, + {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"}, + {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"}, + {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"}, + {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"}, + {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"}, + {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"}, + {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"}, + {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"}, + {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"}, + {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"}, + {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"}, + {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"}, + {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"}, + {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"}, + {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"}, + {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"}, + {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"}, + {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"}, + {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"}, + {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"}, + {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"}, + {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"}, + {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"}, + {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"}, + {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, ] [[package]] @@ -550,13 +568,13 @@ tqdm = ["tqdm"] [[package]] name = "googleapis-common-protos" -version = "1.61.0" +version = "1.62.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.61.0.tar.gz", hash = "sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b"}, - {file = "googleapis_common_protos-1.61.0-py2.py3-none-any.whl", hash = "sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0"}, + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, ] [package.dependencies] @@ -584,166 +602,166 @@ testing = ["protobuf (>=4.21.9)"] [[package]] name = "grpcio" -version = "1.59.3" +version = "1.60.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.59.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:aca028a6c7806e5b61e5f9f4232432c52856f7fcb98e330b20b6bc95d657bdcc"}, - {file = "grpcio-1.59.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:19ad26a7967f7999c8960d2b9fe382dae74c55b0c508c613a6c2ba21cddf2354"}, - {file = "grpcio-1.59.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:72b71dad2a3d1650e69ad42a5c4edbc59ee017f08c32c95694172bc501def23c"}, - {file = "grpcio-1.59.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c0f0a11d82d0253656cc42e04b6a149521e02e755fe2e4edd21123de610fd1d4"}, - {file = "grpcio-1.59.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60cddafb70f9a2c81ba251b53b4007e07cca7389e704f86266e22c4bffd8bf1d"}, - {file = "grpcio-1.59.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6c75a1fa0e677c1d2b6d4196ad395a5c381dfb8385f07ed034ef667cdcdbcc25"}, - {file = "grpcio-1.59.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e1d8e01438d5964a11167eec1edb5f85ed8e475648f36c834ed5db4ffba24ac8"}, - {file = "grpcio-1.59.3-cp310-cp310-win32.whl", hash = "sha256:c4b0076f0bf29ee62335b055a9599f52000b7941f577daa001c7ef961a1fbeab"}, - {file = "grpcio-1.59.3-cp310-cp310-win_amd64.whl", hash = "sha256:b1f00a3e6e0c3dccccffb5579fc76ebfe4eb40405ba308505b41ef92f747746a"}, - {file = "grpcio-1.59.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:3996aaa21231451161dc29df6a43fcaa8b332042b6150482c119a678d007dd86"}, - {file = "grpcio-1.59.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:cb4e9cbd9b7388fcb06412da9f188c7803742d06d6f626304eb838d1707ec7e3"}, - {file = "grpcio-1.59.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8022ca303d6c694a0d7acfb2b472add920217618d3a99eb4b14edc7c6a7e8fcf"}, - {file = "grpcio-1.59.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b36683fad5664283755a7f4e2e804e243633634e93cd798a46247b8e54e3cb0d"}, - {file = "grpcio-1.59.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8239b853226e4824e769517e1b5232e7c4dda3815b200534500338960fcc6118"}, - {file = "grpcio-1.59.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0511af8653fbda489ff11d542a08505d56023e63cafbda60e6e00d4e0bae86ea"}, - {file = "grpcio-1.59.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e78dc982bda74cef2ddfce1c91d29b96864c4c680c634e279ed204d51e227473"}, - {file = "grpcio-1.59.3-cp311-cp311-win32.whl", hash = "sha256:6a5c3a96405966c023e139c3bcccb2c7c776a6f256ac6d70f8558c9041bdccc3"}, - {file = "grpcio-1.59.3-cp311-cp311-win_amd64.whl", hash = "sha256:ed26826ee423b11477297b187371cdf4fa1eca874eb1156422ef3c9a60590dd9"}, - {file = "grpcio-1.59.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:45dddc5cb5227d30fa43652d8872dc87f086d81ab4b500be99413bad0ae198d7"}, - {file = "grpcio-1.59.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:1736496d74682e53dd0907fd515f2694d8e6a96c9a359b4080b2504bf2b2d91b"}, - {file = "grpcio-1.59.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:ddbd1a16138e52e66229047624de364f88a948a4d92ba20e4e25ad7d22eef025"}, - {file = "grpcio-1.59.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fcfa56f8d031ffda902c258c84c4b88707f3a4be4827b4e3ab8ec7c24676320d"}, - {file = "grpcio-1.59.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2eb8f0c7c0c62f7a547ad7a91ba627a5aa32a5ae8d930783f7ee61680d7eb8d"}, - {file = "grpcio-1.59.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8d993399cc65e3a34f8fd48dd9ad7a376734564b822e0160dd18b3d00c1a33f9"}, - {file = "grpcio-1.59.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c0bd141f4f41907eb90bda74d969c3cb21c1c62779419782a5b3f5e4b5835718"}, - {file = "grpcio-1.59.3-cp312-cp312-win32.whl", hash = "sha256:33b8fd65d4e97efa62baec6171ce51f9cf68f3a8ba9f866f4abc9d62b5c97b79"}, - {file = "grpcio-1.59.3-cp312-cp312-win_amd64.whl", hash = "sha256:0e735ed002f50d4f3cb9ecfe8ac82403f5d842d274c92d99db64cfc998515e07"}, - {file = "grpcio-1.59.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:ea40ce4404e7cca0724c91a7404da410f0144148fdd58402a5942971e3469b94"}, - {file = "grpcio-1.59.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:83113bcc393477b6f7342b9f48e8a054330c895205517edc66789ceea0796b53"}, - {file = "grpcio-1.59.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:73afbac602b8f1212a50088193601f869b5073efa9855b3e51aaaec97848fc8a"}, - {file = "grpcio-1.59.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:575d61de1950b0b0699917b686b1ca108690702fcc2df127b8c9c9320f93e069"}, - {file = "grpcio-1.59.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd76057b5c9a4d68814610ef9226925f94c1231bbe533fdf96f6181f7d2ff9e"}, - {file = "grpcio-1.59.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:95d6fd804c81efe4879e38bfd84d2b26e339a0a9b797e7615e884ef4686eb47b"}, - {file = "grpcio-1.59.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0d42048b8a3286ea4134faddf1f9a59cf98192b94aaa10d910a25613c5eb5bfb"}, - {file = "grpcio-1.59.3-cp37-cp37m-win_amd64.whl", hash = "sha256:4619fea15c64bcdd9d447cdbdde40e3d5f1da3a2e8ae84103d94a9c1df210d7e"}, - {file = "grpcio-1.59.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:95b5506e70284ac03b2005dd9ffcb6708c9ae660669376f0192a710687a22556"}, - {file = "grpcio-1.59.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:9e17660947660ccfce56c7869032910c179a5328a77b73b37305cd1ee9301c2e"}, - {file = "grpcio-1.59.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:00912ce19914d038851be5cd380d94a03f9d195643c28e3ad03d355cc02ce7e8"}, - {file = "grpcio-1.59.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e58b3cadaa3c90f1efca26ba33e0d408b35b497307027d3d707e4bcd8de862a6"}, - {file = "grpcio-1.59.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d787ecadea865bdf78f6679f6f5bf4b984f18f659257ba612979df97a298b3c3"}, - {file = "grpcio-1.59.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0814942ba1bba269db4e760a34388640c601dece525c6a01f3b4ff030cc0db69"}, - {file = "grpcio-1.59.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fb111aa99d3180c361a35b5ae1e2c63750220c584a1344229abc139d5c891881"}, - {file = "grpcio-1.59.3-cp38-cp38-win32.whl", hash = "sha256:eb8ba504c726befe40a356ecbe63c6c3c64c9a439b3164f5a718ec53c9874da0"}, - {file = "grpcio-1.59.3-cp38-cp38-win_amd64.whl", hash = "sha256:cdbc6b32fadab9bebc6f49d3e7ec4c70983c71e965497adab7f87de218e84391"}, - {file = "grpcio-1.59.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:c82ca1e4be24a98a253d6dbaa216542e4163f33f38163fc77964b0f0d255b552"}, - {file = "grpcio-1.59.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:36636babfda14f9e9687f28d5b66d349cf88c1301154dc71c6513de2b6c88c59"}, - {file = "grpcio-1.59.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5f9b2e591da751ac7fdd316cc25afafb7a626dededa9b414f90faad7f3ccebdb"}, - {file = "grpcio-1.59.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a93a82876a4926bf451db82ceb725bd87f42292bacc94586045261f501a86994"}, - {file = "grpcio-1.59.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce31fa0bfdd1f2bb15b657c16105c8652186eab304eb512e6ae3b99b2fdd7d13"}, - {file = "grpcio-1.59.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:16da0e40573962dab6cba16bec31f25a4f468e6d05b658e589090fe103b03e3d"}, - {file = "grpcio-1.59.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1d1a17372fd425addd5812049fa7374008ffe689585f27f802d0935522cf4b7"}, - {file = "grpcio-1.59.3-cp39-cp39-win32.whl", hash = "sha256:52cc38a7241b5f7b4a91aaf9000fdd38e26bb00d5e8a71665ce40cfcee716281"}, - {file = "grpcio-1.59.3-cp39-cp39-win_amd64.whl", hash = "sha256:b491e5bbcad3020a96842040421e508780cade35baba30f402df9d321d1c423e"}, - {file = "grpcio-1.59.3.tar.gz", hash = "sha256:7800f99568a74a06ebdccd419dd1b6e639b477dcaf6da77ea702f8fb14ce5f80"}, + {file = "grpcio-1.60.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:d020cfa595d1f8f5c6b343530cd3ca16ae5aefdd1e832b777f9f0eb105f5b139"}, + {file = "grpcio-1.60.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b98f43fcdb16172dec5f4b49f2fece4b16a99fd284d81c6bbac1b3b69fcbe0ff"}, + {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:20e7a4f7ded59097c84059d28230907cd97130fa74f4a8bfd1d8e5ba18c81491"}, + {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452ca5b4afed30e7274445dd9b441a35ece656ec1600b77fff8c216fdf07df43"}, + {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43e636dc2ce9ece583b3e2ca41df5c983f4302eabc6d5f9cd04f0562ee8ec1ae"}, + {file = "grpcio-1.60.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e306b97966369b889985a562ede9d99180def39ad42c8014628dd3cc343f508"}, + {file = "grpcio-1.60.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f897c3b127532e6befdcf961c415c97f320d45614daf84deba0a54e64ea2457b"}, + {file = "grpcio-1.60.0-cp310-cp310-win32.whl", hash = "sha256:b87efe4a380887425bb15f220079aa8336276398dc33fce38c64d278164f963d"}, + {file = "grpcio-1.60.0-cp310-cp310-win_amd64.whl", hash = "sha256:a9c7b71211f066908e518a2ef7a5e211670761651039f0d6a80d8d40054047df"}, + {file = "grpcio-1.60.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:fb464479934778d7cc5baf463d959d361954d6533ad34c3a4f1d267e86ee25fd"}, + {file = "grpcio-1.60.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4b44d7e39964e808b071714666a812049765b26b3ea48c4434a3b317bac82f14"}, + {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:90bdd76b3f04bdb21de5398b8a7c629676c81dfac290f5f19883857e9371d28c"}, + {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91229d7203f1ef0ab420c9b53fe2ca5c1fbeb34f69b3bc1b5089466237a4a134"}, + {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b36a2c6d4920ba88fa98075fdd58ff94ebeb8acc1215ae07d01a418af4c0253"}, + {file = "grpcio-1.60.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:297eef542156d6b15174a1231c2493ea9ea54af8d016b8ca7d5d9cc65cfcc444"}, + {file = "grpcio-1.60.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:87c9224acba0ad8bacddf427a1c2772e17ce50b3042a789547af27099c5f751d"}, + {file = "grpcio-1.60.0-cp311-cp311-win32.whl", hash = "sha256:95ae3e8e2c1b9bf671817f86f155c5da7d49a2289c5cf27a319458c3e025c320"}, + {file = "grpcio-1.60.0-cp311-cp311-win_amd64.whl", hash = "sha256:467a7d31554892eed2aa6c2d47ded1079fc40ea0b9601d9f79204afa8902274b"}, + {file = "grpcio-1.60.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:a7152fa6e597c20cb97923407cf0934e14224af42c2b8d915f48bc3ad2d9ac18"}, + {file = "grpcio-1.60.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:7db16dd4ea1b05ada504f08d0dca1cd9b926bed3770f50e715d087c6f00ad748"}, + {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:b0571a5aef36ba9177e262dc88a9240c866d903a62799e44fd4aae3f9a2ec17e"}, + {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fd9584bf1bccdfff1512719316efa77be235469e1e3295dce64538c4773840b"}, + {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6a478581b1a1a8fdf3318ecb5f4d0cda41cacdffe2b527c23707c9c1b8fdb55"}, + {file = "grpcio-1.60.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:77c8a317f0fd5a0a2be8ed5cbe5341537d5c00bb79b3bb27ba7c5378ba77dbca"}, + {file = "grpcio-1.60.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1c30bb23a41df95109db130a6cc1b974844300ae2e5d68dd4947aacba5985aa5"}, + {file = "grpcio-1.60.0-cp312-cp312-win32.whl", hash = "sha256:2aef56e85901c2397bd557c5ba514f84de1f0ae5dd132f5d5fed042858115951"}, + {file = "grpcio-1.60.0-cp312-cp312-win_amd64.whl", hash = "sha256:e381fe0c2aa6c03b056ad8f52f8efca7be29fb4d9ae2f8873520843b6039612a"}, + {file = "grpcio-1.60.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:92f88ca1b956eb8427a11bb8b4a0c0b2b03377235fc5102cb05e533b8693a415"}, + {file = "grpcio-1.60.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:e278eafb406f7e1b1b637c2cf51d3ad45883bb5bd1ca56bc05e4fc135dfdaa65"}, + {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:a48edde788b99214613e440fce495bbe2b1e142a7f214cce9e0832146c41e324"}, + {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de2ad69c9a094bf37c1102b5744c9aec6cf74d2b635558b779085d0263166454"}, + {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:073f959c6f570797272f4ee9464a9997eaf1e98c27cb680225b82b53390d61e6"}, + {file = "grpcio-1.60.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c826f93050c73e7769806f92e601e0efdb83ec8d7c76ddf45d514fee54e8e619"}, + {file = "grpcio-1.60.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9e30be89a75ee66aec7f9e60086fadb37ff8c0ba49a022887c28c134341f7179"}, + {file = "grpcio-1.60.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b0fb2d4801546598ac5cd18e3ec79c1a9af8b8f2a86283c55a5337c5aeca4b1b"}, + {file = "grpcio-1.60.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:9073513ec380434eb8d21970e1ab3161041de121f4018bbed3146839451a6d8e"}, + {file = "grpcio-1.60.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:74d7d9fa97809c5b892449b28a65ec2bfa458a4735ddad46074f9f7d9550ad13"}, + {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:1434ca77d6fed4ea312901122dc8da6c4389738bf5788f43efb19a838ac03ead"}, + {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e61e76020e0c332a98290323ecfec721c9544f5b739fab925b6e8cbe1944cf19"}, + {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675997222f2e2f22928fbba640824aebd43791116034f62006e19730715166c0"}, + {file = "grpcio-1.60.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5208a57eae445ae84a219dfd8b56e04313445d146873117b5fa75f3245bc1390"}, + {file = "grpcio-1.60.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:428d699c8553c27e98f4d29fdc0f0edc50e9a8a7590bfd294d2edb0da7be3629"}, + {file = "grpcio-1.60.0-cp38-cp38-win32.whl", hash = "sha256:83f2292ae292ed5a47cdcb9821039ca8e88902923198f2193f13959360c01860"}, + {file = "grpcio-1.60.0-cp38-cp38-win_amd64.whl", hash = "sha256:705a68a973c4c76db5d369ed573fec3367d7d196673fa86614b33d8c8e9ebb08"}, + {file = "grpcio-1.60.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:c193109ca4070cdcaa6eff00fdb5a56233dc7610216d58fb81638f89f02e4968"}, + {file = "grpcio-1.60.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:676e4a44e740deaba0f4d95ba1d8c5c89a2fcc43d02c39f69450b1fa19d39590"}, + {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5ff21e000ff2f658430bde5288cb1ac440ff15c0d7d18b5fb222f941b46cb0d2"}, + {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c86343cf9ff7b2514dd229bdd88ebba760bd8973dac192ae687ff75e39ebfab"}, + {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fd3b3968ffe7643144580f260f04d39d869fcc2cddb745deef078b09fd2b328"}, + {file = "grpcio-1.60.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:30943b9530fe3620e3b195c03130396cd0ee3a0d10a66c1bee715d1819001eaf"}, + {file = "grpcio-1.60.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b10241250cb77657ab315270b064a6c7f1add58af94befa20687e7c8d8603ae6"}, + {file = "grpcio-1.60.0-cp39-cp39-win32.whl", hash = "sha256:79a050889eb8d57a93ed21d9585bb63fca881666fc709f5d9f7f9372f5e7fd03"}, + {file = "grpcio-1.60.0-cp39-cp39-win_amd64.whl", hash = "sha256:8a97a681e82bc11a42d4372fe57898d270a2707f36c45c6676e49ce0d5c41353"}, + {file = "grpcio-1.60.0.tar.gz", hash = "sha256:2199165a1affb666aa24adf0c97436686d0a61bc5fc113c037701fb7c7fceb96"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.59.3)"] +protobuf = ["grpcio-tools (>=1.60.0)"] [[package]] name = "grpcio-reflection" -version = "1.59.3" +version = "1.60.0" description = "Standard Protobuf Reflection Service for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-reflection-1.59.3.tar.gz", hash = "sha256:5403c5a738c6eec4bb4080da77e450312dace41a86e292ac37cfd007b2657d4e"}, - {file = "grpcio_reflection-1.59.3-py3-none-any.whl", hash = "sha256:526064089d71cddce7244a83059f410fac6ccd30344a624c10f54420d417b3d2"}, + {file = "grpcio-reflection-1.60.0.tar.gz", hash = "sha256:3f6c0c73ba8f20d1420c5e72fc4dd0389fac346ed8fb32a28e6e1967b44fff35"}, + {file = "grpcio_reflection-1.60.0-py3-none-any.whl", hash = "sha256:f7a347ebd6cecf347fc836fd520fd1f0b3411912981649c7fb34d62a3a15aa4e"}, ] [package.dependencies] -grpcio = ">=1.59.3" +grpcio = ">=1.60.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-status" -version = "1.59.3" +version = "1.60.0" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.59.3.tar.gz", hash = "sha256:65c394ba43380d6bdf8c04c61efc493104b5535552aed35817a1b4dc66598a1f"}, - {file = "grpcio_status-1.59.3-py3-none-any.whl", hash = "sha256:2fd2eb39ca4e9afb3c874c0878ff75b258db0b7dcc25570fc521f16ae0ab942a"}, + {file = "grpcio-status-1.60.0.tar.gz", hash = "sha256:f10e0b6db3adc0fdc244b71962814ee982996ef06186446b5695b9fa635aa1ab"}, + {file = "grpcio_status-1.60.0-py3-none-any.whl", hash = "sha256:7d383fa36e59c1e61d380d91350badd4d12ac56e4de2c2b831b050362c3c572e"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.59.3" +grpcio = ">=1.60.0" protobuf = ">=4.21.6" [[package]] name = "grpcio-tools" -version = "1.59.3" +version = "1.60.0" description = "Protobuf code generator for gRPC" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-tools-1.59.3.tar.gz", hash = "sha256:cd160ac4281cd1ae77a2c880377a7728349340b4c91e24285037b57c18e9f651"}, - {file = "grpcio_tools-1.59.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:17017fe74734c158e0f93817f1ff17aeda37d0f105ed6e63b12c26b66743a7a8"}, - {file = "grpcio_tools-1.59.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:ac1013e4f84ffd15c45ead6d19c9d188b76c14466a799aa9c338ce3b9ebf6dcc"}, - {file = "grpcio_tools-1.59.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0a5d760619305eb51a8719ce9c081398f145a46bc7c86a6e2cebe0648a21f40c"}, - {file = "grpcio_tools-1.59.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de3d9649b7a3091ec785a67d5bf006584440f03896ee52259c6d9ff412d08afb"}, - {file = "grpcio_tools-1.59.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21868aa510317d3f39e5de40208ffb8ab1beb1cbcab333317939b59a9b5db055"}, - {file = "grpcio_tools-1.59.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0b116a888580317e421358f26bfaeec47d6f73079e8a47bad60d1f9f7b30f2a5"}, - {file = "grpcio_tools-1.59.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6bd4a72c27abda191e2360b2b720ada1880aba96a63604a6f9d7c37bb3bbf5c4"}, - {file = "grpcio_tools-1.59.3-cp310-cp310-win32.whl", hash = "sha256:d70cad744e92c7576c226a72998ae8dd59240c942f73798bbde40284eb9eb991"}, - {file = "grpcio_tools-1.59.3-cp310-cp310-win_amd64.whl", hash = "sha256:2b8a4aca0c11f2a8b3bfe103362984bdc427ab762428446ef2e12922fd48ee10"}, - {file = "grpcio_tools-1.59.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:b4418b78408ff56ee70a0b14484c07f5e48c2e6f4fa7be390d486a686d0cd6e4"}, - {file = "grpcio_tools-1.59.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:58de83ced4f86458f45288a5f76d9765dc245a9ce4e783a194decccc7e0674ea"}, - {file = "grpcio_tools-1.59.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:76b0cdcbcb38722840d3eaff6439ddb4b8f0210c6718553d7b7c911834b10e60"}, - {file = "grpcio_tools-1.59.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0cacf59513b100bfb3d8de51ba43db6140aa9bcb7bba872badb48acb430c002"}, - {file = "grpcio_tools-1.59.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:019fdd986c80b13187574c291df5054f241bdbe87dbc86e4cee73ffa28328647"}, - {file = "grpcio_tools-1.59.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ff304b9d6c23d8e2ecc860bebac1ec6768a2d920985bcea9ce4a7aaeeea44f76"}, - {file = "grpcio_tools-1.59.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ca286affe613beaf2d5a6b8bd83203dcf488917194b416da48aa849047b5f081"}, - {file = "grpcio_tools-1.59.3-cp311-cp311-win32.whl", hash = "sha256:8f69141ff370729ceaad0286b8c6e15352c9bb39aa8f18c0500ce3d0238c2981"}, - {file = "grpcio_tools-1.59.3-cp311-cp311-win_amd64.whl", hash = "sha256:05ec4ffe16b6eab12813476e6d7465a0027bee33999d4776ae1d9c0664d0fc54"}, - {file = "grpcio_tools-1.59.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:21d976419630f72a7cefebe7dcfc451b33d70c805a43ff5a60c43367813f0527"}, - {file = "grpcio_tools-1.59.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:396106f92ea6ab2157535e1a009bac99aa15680ca8addbc8e7c1a4d3f5b1fb2c"}, - {file = "grpcio_tools-1.59.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:4f064483e0046a4a193d6c67b26ea0f61737e8232ff61636a7fa0bc5244458be"}, - {file = "grpcio_tools-1.59.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6dc6da8e3780df25095c1952f45c334e1554f25b991ffe75dbf0408795d27a0"}, - {file = "grpcio_tools-1.59.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87111be05c1a159ce3fffbfad86ff69fd4bf1702cde127eb698d8e8c3a018ab6"}, - {file = "grpcio_tools-1.59.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:83453a13c2120238eb7fb993b03b370496e76071a7b45c816aa682d9226d29c1"}, - {file = "grpcio_tools-1.59.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4384b29d8e126bc6e24a5efd9d60a2a2015867c7109fa67ff2ed274b3f4a05c5"}, - {file = "grpcio_tools-1.59.3-cp312-cp312-win32.whl", hash = "sha256:ce1372c9acde9d74c7e54858598ac0c5203dd3ec24b9085f7a8b2f33cc156736"}, - {file = "grpcio_tools-1.59.3-cp312-cp312-win_amd64.whl", hash = "sha256:84179e3a7c9067e993700b3255f2adc10e9b16e8dd28525d1dd1a02b9ca603ee"}, - {file = "grpcio_tools-1.59.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:592208a9a02df75993cc4dba111d2b81f0e6a3f3837856be239e1aceb6651f31"}, - {file = "grpcio_tools-1.59.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:1abe30ce770ac4fed966d017771fa7f8ced6a279de7ce68023e2c07f07911e76"}, - {file = "grpcio_tools-1.59.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:4dce57668e2aa8c3bb0b2a0bb766a2654ee0f4d8d31e02a6001e98af18569285"}, - {file = "grpcio_tools-1.59.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acaefd3c362250a02cc93fc1b5440e3cb30ab8d7fd81594b2975ea19f123aae3"}, - {file = "grpcio_tools-1.59.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76542e1c11e3f2c22c19cff6b3233caab35924fad1f685ce63184d31b4050fa8"}, - {file = "grpcio_tools-1.59.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:64fd1efb23da054f61aca2346c5139f317b7f4c545f6dbda5ec246f281af8e86"}, - {file = "grpcio_tools-1.59.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ea6454acde508c9a62dab3f59e98b32e32b26aa60df20080982503bb7db51304"}, - {file = "grpcio_tools-1.59.3-cp37-cp37m-win_amd64.whl", hash = "sha256:a048d4bde526f3c6e364abea2c3a481f3bbebc4bfa7fdcfcc3e5ee4f8ab9c4c5"}, - {file = "grpcio_tools-1.59.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:b4db59a62d31c98105af08b1bfb8878c239e4cf31088f2d9864756cdfec67746"}, - {file = "grpcio_tools-1.59.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:05ac0f6683759e5508081c09af26cb6cc949c2c54d75ff8b76344709f78dda53"}, - {file = "grpcio_tools-1.59.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:a031e1ced828a00f1eb59db5f5d4dd39d3bd6a7df8187f84830d4a32a1bbc686"}, - {file = "grpcio_tools-1.59.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f48b4409b306675b7673dad725c9cf3234bf05623bf8a193ad14af39d0368ba6"}, - {file = "grpcio_tools-1.59.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36524d97cc936767a69815b90be76a1420b3218a7724ce69cde6ece794e72a17"}, - {file = "grpcio_tools-1.59.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7cc7e8b893a6c37a8a28735fede1aa40275988a668d1e22c5f234938a77d811d"}, - {file = "grpcio_tools-1.59.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:917be645a71cf9592d2f5a3589c20c074a6539954017e8e2dca5e8ba13025625"}, - {file = "grpcio_tools-1.59.3-cp38-cp38-win32.whl", hash = "sha256:a1394b7a65d738ee0ce4eac1fc95158dd9c97b5c3f690d259e6ee0bf131697de"}, - {file = "grpcio_tools-1.59.3-cp38-cp38-win_amd64.whl", hash = "sha256:007745bd3c5a788dcb73eeb6cd773613a834bd2442e7d062dcafe46dbd4bb5f6"}, - {file = "grpcio_tools-1.59.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:102b5f14a500dbb766f24a96884d9572a3ea7a56d69695461100fb71ec922ef6"}, - {file = "grpcio_tools-1.59.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:46c384a0e30a8422a3e2c5530b3cd69b652dd659549907e2eaac7ca4e0ab614d"}, - {file = "grpcio_tools-1.59.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:ee013da4f5a4ef50fdeca372470733bc402677a4dc0023ee94bf42478b5a620d"}, - {file = "grpcio_tools-1.59.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7883ce3d532c09f29c016fdac107f9a3dc43f9e6b60faf8b91fcba21824269"}, - {file = "grpcio_tools-1.59.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2861e4814ebc147854c2246092c433931f4c15f3c8105ae8716b1e282313a5ae"}, - {file = "grpcio_tools-1.59.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d93590a6a82469f3e58e39692230d99c43a39b215cb581e072dcd52286471152"}, - {file = "grpcio_tools-1.59.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f8396183e6e0a16178773963dc21b58c0c532783476fda314198a9e42f57af7d"}, - {file = "grpcio_tools-1.59.3-cp39-cp39-win32.whl", hash = "sha256:6747b1d82d08e0f5e1a6438532343a1c5504147d1a199c5756e702e5f916de4c"}, - {file = "grpcio_tools-1.59.3-cp39-cp39-win_amd64.whl", hash = "sha256:3a560dcb176dd42c37af5d37299e318341a572547e32b942247daa834d2164c0"}, + {file = "grpcio-tools-1.60.0.tar.gz", hash = "sha256:ed30499340228d733ff69fcf4a66590ed7921f94eb5a2bf692258b1280b9dac7"}, + {file = "grpcio_tools-1.60.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:6807b7a3f3e6e594566100bd7fe04a2c42ce6d5792652677f1aaf5aa5adaef3d"}, + {file = "grpcio_tools-1.60.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:857c5351e9dc33a019700e171163f94fcc7e3ae0f6d2b026b10fda1e3c008ef1"}, + {file = "grpcio_tools-1.60.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:ec0e401e9a43d927d216d5169b03c61163fb52b665c5af2fed851357b15aef88"}, + {file = "grpcio_tools-1.60.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e68dc4474f30cad11a965f0eb5d37720a032b4720afa0ec19dbcea2de73b5aae"}, + {file = "grpcio_tools-1.60.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbf0ed772d2ae7e8e5d7281fcc00123923ab130b94f7a843eee9af405918f924"}, + {file = "grpcio_tools-1.60.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c771b19dce2bfe06899247168c077d7ab4e273f6655d8174834f9a6034415096"}, + {file = "grpcio_tools-1.60.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e5614cf0960456d21d8a0f4902e3e5e3bcacc4e400bf22f196e5dd8aabb978b7"}, + {file = "grpcio_tools-1.60.0-cp310-cp310-win32.whl", hash = "sha256:87cf439178f3eb45c1a889b2e4a17cbb4c450230d92c18d9c57e11271e239c55"}, + {file = "grpcio_tools-1.60.0-cp310-cp310-win_amd64.whl", hash = "sha256:687f576d7ff6ce483bc9a196d1ceac45144e8733b953620a026daed8e450bc38"}, + {file = "grpcio_tools-1.60.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:2a8a758701f3ac07ed85f5a4284c6a9ddefcab7913a8e552497f919349e72438"}, + {file = "grpcio_tools-1.60.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:7c1cde49631732356cb916ee1710507967f19913565ed5f9991e6c9cb37e3887"}, + {file = "grpcio_tools-1.60.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:d941749bd8dc3f8be58fe37183143412a27bec3df8482d5abd6b4ec3f1ac2924"}, + {file = "grpcio_tools-1.60.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ee35234f1da8fba7ddbc544856ff588243f1128ea778d7a1da3039be829a134"}, + {file = "grpcio_tools-1.60.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8f7a5094adb49e85db13ea3df5d99a976c2bdfd83b0ba26af20ebb742ac6786"}, + {file = "grpcio_tools-1.60.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:24c4ead4a03037beaeb8ef2c90d13d70101e35c9fae057337ed1a9144ef10b53"}, + {file = "grpcio_tools-1.60.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:811abb9c4fb6679e0058dfa123fb065d97b158b71959c0e048e7972bbb82ba0f"}, + {file = "grpcio_tools-1.60.0-cp311-cp311-win32.whl", hash = "sha256:bd2a17b0193fbe4793c215d63ce1e01ae00a8183d81d7c04e77e1dfafc4b2b8a"}, + {file = "grpcio_tools-1.60.0-cp311-cp311-win_amd64.whl", hash = "sha256:b22b1299b666eebd5752ba7719da536075eae3053abcf2898b65f763c314d9da"}, + {file = "grpcio_tools-1.60.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:74025fdd6d1cb7ba4b5d087995339e9a09f0c16cf15dfe56368b23e41ffeaf7a"}, + {file = "grpcio_tools-1.60.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:5a907a4f1ffba86501b2cdb8682346249ea032b922fc69a92f082ba045cca548"}, + {file = "grpcio_tools-1.60.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:1fbb9554466d560472f07d906bfc8dcaf52f365c2a407015185993e30372a886"}, + {file = "grpcio_tools-1.60.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f10ef47460ce3c6fd400f05fe757b90df63486c9b84d1ecad42dcc5f80c8ac14"}, + {file = "grpcio_tools-1.60.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:321b18f42a70813545e416ddcb8bf20defa407a8114906711c9710a69596ceda"}, + {file = "grpcio_tools-1.60.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:081336d8258f1a56542aa8a7a5dec99a2b38d902e19fbdd744594783301b0210"}, + {file = "grpcio_tools-1.60.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:addc9b23d6ff729d9f83d4a2846292d4c84f5eb2ec38f08489a6a0d66ac2b91e"}, + {file = "grpcio_tools-1.60.0-cp312-cp312-win32.whl", hash = "sha256:e87cabac7969bdde309575edc2456357667a1b28262b2c1f12580ef48315b19d"}, + {file = "grpcio_tools-1.60.0-cp312-cp312-win_amd64.whl", hash = "sha256:e70d867c120d9849093b0ac24d861e378bc88af2552e743d83b9f642d2caa7c2"}, + {file = "grpcio_tools-1.60.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:559ce714fe212aaf4abbe1493c5bb8920def00cc77ce0d45266f4fd9d8b3166f"}, + {file = "grpcio_tools-1.60.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:7a5263a0f2ddb7b1cfb2349e392cfc4f318722e0f48f886393e06946875d40f3"}, + {file = "grpcio_tools-1.60.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:18976684a931ca4bcba65c78afa778683aefaae310f353e198b1823bf09775a0"}, + {file = "grpcio_tools-1.60.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5c519a0d4ba1ab44a004fa144089738c59278233e2010b2cf4527dc667ff297"}, + {file = "grpcio_tools-1.60.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6170873b1e5b6580ebb99e87fb6e4ea4c48785b910bd7af838cc6e44b2bccb04"}, + {file = "grpcio_tools-1.60.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fb4df80868b3e397d5fbccc004c789d2668b622b51a9d2387b4c89c80d31e2c5"}, + {file = "grpcio_tools-1.60.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dba6e32c87b4af29b5f475fb2f470f7ee3140bfc128644f17c6c59ddeb670680"}, + {file = "grpcio_tools-1.60.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f610384dee4b1ca705e8da66c5b5fe89a2de3d165c5282c3d1ddf40cb18924e4"}, + {file = "grpcio_tools-1.60.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:4041538f55aad5b3ae7e25ab314d7995d689e968bfc8aa169d939a3160b1e4c6"}, + {file = "grpcio_tools-1.60.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:2fb4cf74bfe1e707cf10bc9dd38a1ebaa145179453d150febb121c7e9cd749bf"}, + {file = "grpcio_tools-1.60.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2fd1671c52f96e79a2302c8b1c1f78b8a561664b8b3d6946f20d8f1cc6b4225a"}, + {file = "grpcio_tools-1.60.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd1e68c232fe01dd5312a8dbe52c50ecd2b5991d517d7f7446af4ba6334ba872"}, + {file = "grpcio_tools-1.60.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17a32b3da4fc0798cdcec0a9c974ac2a1e98298f151517bf9148294a3b1a5742"}, + {file = "grpcio_tools-1.60.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9970d384fb0c084b00945ef57d98d57a8d32be106d8f0bd31387f7cbfe411b5b"}, + {file = "grpcio_tools-1.60.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5ce6bbd4936977ec1114f2903eb4342781960d521b0d82f73afedb9335251f6f"}, + {file = "grpcio_tools-1.60.0-cp38-cp38-win32.whl", hash = "sha256:2e00de389729ca8d8d1a63c2038703078a887ff738dc31be640b7da9c26d0d4f"}, + {file = "grpcio_tools-1.60.0-cp38-cp38-win_amd64.whl", hash = "sha256:6192184b1f99372ff1d9594bd4b12264e3ff26440daba7eb043726785200ff77"}, + {file = "grpcio_tools-1.60.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:eae27f9b16238e2aaee84c77b5923c6924d6dccb0bdd18435bf42acc8473ae1a"}, + {file = "grpcio_tools-1.60.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:b96981f3a31b85074b73d97c8234a5ed9053d65a36b18f4a9c45a2120a5b7a0a"}, + {file = "grpcio_tools-1.60.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:1748893efd05cf4a59a175d7fa1e4fbb652f4d84ccaa2109f7869a2be48ed25e"}, + {file = "grpcio_tools-1.60.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a6fe752205caae534f29fba907e2f59ff79aa42c6205ce9a467e9406cbac68c"}, + {file = "grpcio_tools-1.60.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3456df087ea61a0972a5bc165aed132ed6ddcc63f5749e572f9fff84540bdbad"}, + {file = "grpcio_tools-1.60.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f3d916606dcf5610d4367918245b3d9d8cd0d2ec0b7043d1bbb8c50fe9815c3a"}, + {file = "grpcio_tools-1.60.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fc01bc1079279ec342f0f1b6a107b3f5dc3169c33369cf96ada6e2e171f74e86"}, + {file = "grpcio_tools-1.60.0-cp39-cp39-win32.whl", hash = "sha256:2dd01257e4feff986d256fa0bac9f56de59dc735eceeeb83de1c126e2e91f653"}, + {file = "grpcio_tools-1.60.0-cp39-cp39-win_amd64.whl", hash = "sha256:1b93ae8ffd18e9af9a965ebca5fa521e89066267de7abdde20721edc04e42721"}, ] [package.dependencies] -grpcio = ">=1.59.3" +grpcio = ">=1.60.0" protobuf = ">=4.21.6,<5.0dev" setuptools = "*" @@ -781,18 +799,18 @@ files = [ [[package]] name = "huggingface-hub" -version = "0.16.4" +version = "0.19.4" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" files = [ - {file = "huggingface_hub-0.16.4-py3-none-any.whl", hash = "sha256:0d3df29932f334fead024afc7cb4cc5149d955238b8b5e42dcf9740d6995a349"}, - {file = "huggingface_hub-0.16.4.tar.gz", hash = "sha256:608c7d4f3d368b326d1747f91523dbd1f692871e8e2e7a4750314a2dd8b63e14"}, + {file = "huggingface_hub-0.19.4-py3-none-any.whl", hash = "sha256:dba013f779da16f14b606492828f3760600a1e1801432d09fe1c33e50b825bb5"}, + {file = "huggingface_hub-0.19.4.tar.gz", hash = "sha256:176a4fc355a851c17550e7619488f383189727eab209534d7cef2114dae77b22"}, ] [package.dependencies] filelock = "*" -fsspec = "*" +fsspec = ">=2023.5.0" packaging = ">=20.9" pyyaml = ">=5.1" requests = "*" @@ -800,26 +818,27 @@ tqdm = ">=4.42.1" typing-extensions = ">=3.7.4.3" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "mypy (==1.5.1)", "numpy", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.1.3)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "mypy (==1.5.1)", "numpy", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.1.3)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +docs = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "hf-doc-builder", "jedi", "mypy (==1.5.1)", "numpy", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.1.3)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)", "watchdog"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -inference = ["aiohttp", "pydantic"] -quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] +inference = ["aiohttp", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)"] +quality = ["mypy (==1.5.1)", "ruff (>=0.1.3)"] tensorflow = ["graphviz", "pydot", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic (>1.1,<2.0)", "pydantic (>1.1,<3.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] torch = ["torch"] -typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] @@ -1436,36 +1455,36 @@ files = [ [[package]] name = "pandas" -version = "2.1.3" +version = "2.1.4" description = "Powerful data structures for data analysis, time series, and statistics" optional = true python-versions = ">=3.9" files = [ - {file = "pandas-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:acf08a73b5022b479c1be155d4988b72f3020f308f7a87c527702c5f8966d34f"}, - {file = "pandas-2.1.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3cc4469ff0cf9aa3a005870cb49ab8969942b7156e0a46cc3f5abd6b11051dfb"}, - {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35172bff95f598cc5866c047f43c7f4df2c893acd8e10e6653a4b792ed7f19bb"}, - {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59dfe0e65a2f3988e940224e2a70932edc964df79f3356e5f2997c7d63e758b4"}, - {file = "pandas-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0296a66200dee556850d99b24c54c7dfa53a3264b1ca6f440e42bad424caea03"}, - {file = "pandas-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:465571472267a2d6e00657900afadbe6097c8e1dc43746917db4dfc862e8863e"}, - {file = "pandas-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04d4c58e1f112a74689da707be31cf689db086949c71828ef5da86727cfe3f82"}, - {file = "pandas-2.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7fa2ad4ff196768ae63a33f8062e6838efed3a319cf938fdf8b95e956c813042"}, - {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4441ac94a2a2613e3982e502ccec3bdedefe871e8cea54b8775992485c5660ef"}, - {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5ded6ff28abbf0ea7689f251754d3789e1edb0c4d0d91028f0b980598418a58"}, - {file = "pandas-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca5680368a5139d4920ae3dc993eb5106d49f814ff24018b64d8850a52c6ed2"}, - {file = "pandas-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:de21e12bf1511190fc1e9ebc067f14ca09fccfb189a813b38d63211d54832f5f"}, - {file = "pandas-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a5d53c725832e5f1645e7674989f4c106e4b7249c1d57549023ed5462d73b140"}, - {file = "pandas-2.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7cf4cf26042476e39394f1f86868d25b265ff787c9b2f0d367280f11afbdee6d"}, - {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72c84ec1b1d8e5efcbff5312abe92bfb9d5b558f11e0cf077f5496c4f4a3c99e"}, - {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f539e113739a3e0cc15176bf1231a553db0239bfa47a2c870283fd93ba4f683"}, - {file = "pandas-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fc77309da3b55732059e484a1efc0897f6149183c522390772d3561f9bf96c00"}, - {file = "pandas-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:08637041279b8981a062899da0ef47828df52a1838204d2b3761fbd3e9fcb549"}, - {file = "pandas-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b99c4e51ef2ed98f69099c72c75ec904dd610eb41a32847c4fcbc1a975f2d2b8"}, - {file = "pandas-2.1.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7ea8ae8004de0381a2376662c0505bb0a4f679f4c61fbfd122aa3d1b0e5f09d"}, - {file = "pandas-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcd76d67ca2d48f56e2db45833cf9d58f548f97f61eecd3fdc74268417632b8a"}, - {file = "pandas-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1329dbe93a880a3d7893149979caa82d6ba64a25e471682637f846d9dbc10dd2"}, - {file = "pandas-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:321ecdb117bf0f16c339cc6d5c9a06063854f12d4d9bc422a84bb2ed3207380a"}, - {file = "pandas-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:11a771450f36cebf2a4c9dbd3a19dfa8c46c4b905a3ea09dc8e556626060fe71"}, - {file = "pandas-2.1.3.tar.gz", hash = "sha256:22929f84bca106921917eb73c1521317ddd0a4c71b395bcf767a106e3494209f"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bdec823dc6ec53f7a6339a0e34c68b144a7a1fd28d80c260534c39c62c5bf8c9"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:294d96cfaf28d688f30c918a765ea2ae2e0e71d3536754f4b6de0ea4a496d034"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b728fb8deba8905b319f96447a27033969f3ea1fea09d07d296c9030ab2ed1d"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00028e6737c594feac3c2df15636d73ace46b8314d236100b57ed7e4b9ebe8d9"}, + {file = "pandas-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:426dc0f1b187523c4db06f96fb5c8d1a845e259c99bda74f7de97bd8a3bb3139"}, + {file = "pandas-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:f237e6ca6421265643608813ce9793610ad09b40154a3344a088159590469e46"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7d852d16c270e4331f6f59b3e9aa23f935f5c4b0ed2d0bc77637a8890a5d092"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7d5f2f54f78164b3d7a40f33bf79a74cdee72c31affec86bfcabe7e0789821"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa6e92e639da0d6e2017d9ccff563222f4eb31e4b2c3cf32a2a392fc3103c0d"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d797591b6846b9db79e65dc2d0d48e61f7db8d10b2a9480b4e3faaddc421a171"}, + {file = "pandas-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2d3e7b00f703aea3945995ee63375c61b2e6aa5aa7871c5d622870e5e137623"}, + {file = "pandas-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc9bf7ade01143cddc0074aa6995edd05323974e6e40d9dbde081021ded8510e"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:482d5076e1791777e1571f2e2d789e940dedd927325cc3cb6d0800c6304082f6"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a706cfe7955c4ca59af8c7a0517370eafbd98593155b48f10f9811da440248b"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0513a132a15977b4a5b89aabd304647919bc2169eac4c8536afb29c07c23540"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f17f2b6fc076b2a0078862547595d66244db0f41bf79fc5f64a5c4d635bead"}, + {file = "pandas-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:45d63d2a9b1b37fa6c84a68ba2422dc9ed018bdaa668c7f47566a01188ceeec1"}, + {file = "pandas-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f69b0c9bb174a2342818d3e2778584e18c740d56857fc5cdb944ec8bbe4082cf"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3f06bda01a143020bad20f7a85dd5f4a1600112145f126bc9e3e42077c24ef34"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab5796839eb1fd62a39eec2916d3e979ec3130509930fea17fe6f81e18108f6a"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbaf9e8d3a63a9276d707b4d25930a262341bca9874fcb22eff5e3da5394732"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ebfd771110b50055712b3b711b51bee5d50135429364d0498e1213a7adc2be8"}, + {file = "pandas-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8ea107e0be2aba1da619cc6ba3f999b2bfc9669a83554b1904ce3dd9507f0860"}, + {file = "pandas-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:d65148b14788b3758daf57bf42725caa536575da2b64df9964c563b015230984"}, + {file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"}, ] [package.dependencies] @@ -2171,56 +2190,117 @@ files = [ [[package]] name = "tokenizers" -version = "0.13.3" -description = "Fast and Customizable Tokenizers" +version = "0.15.0" +description = "" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, - {file = "tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee0b1b311d65beab83d7a41c56a1e46ab732a9eed4460648e8eb0bd69fc2d059"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ef4215284df1277dadbcc5e17d4882bda19f770d02348e73523f7e7d8b8d396"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4d53976079cff8a033f778fb9adca2d9d69d009c02fa2d71a878b5f3963ed30"}, - {file = "tokenizers-0.13.3-cp310-cp310-win32.whl", hash = "sha256:1f0e3b4c2ea2cd13238ce43548959c118069db7579e5d40ec270ad77da5833ce"}, - {file = "tokenizers-0.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:89649c00d0d7211e8186f7a75dfa1db6996f65edce4b84821817eadcc2d3c79e"}, - {file = "tokenizers-0.13.3-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:56b726e0d2bbc9243872b0144515ba684af5b8d8cd112fb83ee1365e26ec74c8"}, - {file = "tokenizers-0.13.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:cc5c022ce692e1f499d745af293ab9ee6f5d92538ed2faf73f9708c89ee59ce6"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f55c981ac44ba87c93e847c333e58c12abcbb377a0c2f2ef96e1a266e4184ff2"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f247eae99800ef821a91f47c5280e9e9afaeed9980fc444208d5aa6ba69ff148"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b3e3215d048e94f40f1c95802e45dcc37c5b05eb46280fc2ccc8cd351bff839"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ba2b0bf01777c9b9bc94b53764d6684554ce98551fec496f71bc5be3a03e98b"}, - {file = "tokenizers-0.13.3-cp311-cp311-win32.whl", hash = "sha256:cc78d77f597d1c458bf0ea7c2a64b6aa06941c7a99cb135b5969b0278824d808"}, - {file = "tokenizers-0.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:ecf182bf59bd541a8876deccf0360f5ae60496fd50b58510048020751cf1724c"}, - {file = "tokenizers-0.13.3-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:0527dc5436a1f6bf2c0327da3145687d3bcfbeab91fed8458920093de3901b44"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07cbb2c307627dc99b44b22ef05ff4473aa7c7cc1fec8f0a8b37d8a64b1a16d2"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4560dbdeaae5b7ee0d4e493027e3de6d53c991b5002d7ff95083c99e11dd5ac0"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64064bd0322405c9374305ab9b4c07152a1474370327499911937fd4a76d004b"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8c6e2ab0f2e3d939ca66aa1d596602105fe33b505cd2854a4c1717f704c51de"}, - {file = "tokenizers-0.13.3-cp37-cp37m-win32.whl", hash = "sha256:6cc29d410768f960db8677221e497226e545eaaea01aa3613fa0fdf2cc96cff4"}, - {file = "tokenizers-0.13.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fc2a7fdf864554a0dacf09d32e17c0caa9afe72baf9dd7ddedc61973bae352d8"}, - {file = "tokenizers-0.13.3-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:8791dedba834c1fc55e5f1521be325ea3dafb381964be20684b92fdac95d79b7"}, - {file = "tokenizers-0.13.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:d607a6a13718aeb20507bdf2b96162ead5145bbbfa26788d6b833f98b31b26e1"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3791338f809cd1bf8e4fee6b540b36822434d0c6c6bc47162448deee3f77d425"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2f35f30e39e6aab8716f07790f646bdc6e4a853816cc49a95ef2a9016bf9ce6"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310204dfed5aa797128b65d63538a9837cbdd15da2a29a77d67eefa489edda26"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0f9b92ea052305166559f38498b3b0cae159caea712646648aaa272f7160963"}, - {file = "tokenizers-0.13.3-cp38-cp38-win32.whl", hash = "sha256:9a3fa134896c3c1f0da6e762d15141fbff30d094067c8f1157b9fdca593b5806"}, - {file = "tokenizers-0.13.3-cp38-cp38-win_amd64.whl", hash = "sha256:8e7b0cdeace87fa9e760e6a605e0ae8fc14b7d72e9fc19c578116f7287bb873d"}, - {file = "tokenizers-0.13.3-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:00cee1e0859d55507e693a48fa4aef07060c4bb6bd93d80120e18fea9371c66d"}, - {file = "tokenizers-0.13.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a23ff602d0797cea1d0506ce69b27523b07e70f6dda982ab8cf82402de839088"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70ce07445050b537d2696022dafb115307abdffd2a5c106f029490f84501ef97"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:280ffe95f50eaaf655b3a1dc7ff1d9cf4777029dbbc3e63a74e65a056594abc3"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97acfcec592f7e9de8cadcdcda50a7134423ac8455c0166b28c9ff04d227b371"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd7730c98a3010cd4f523465867ff95cd9d6430db46676ce79358f65ae39797b"}, - {file = "tokenizers-0.13.3-cp39-cp39-win32.whl", hash = "sha256:48625a108029cb1ddf42e17a81b5a3230ba6888a70c9dc14e81bc319e812652d"}, - {file = "tokenizers-0.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783"}, - {file = "tokenizers-0.13.3.tar.gz", hash = "sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e"}, + {file = "tokenizers-0.15.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:cd3cd0299aaa312cd2988957598f80becd04d5a07338741eca076057a2b37d6e"}, + {file = "tokenizers-0.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a922c492c721744ee175f15b91704be2d305569d25f0547c77cd6c9f210f9dc"}, + {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:331dd786d02fc38698f835fff61c99480f98b73ce75a4c65bd110c9af5e4609a"}, + {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88dd0961c437d413ab027f8b115350c121d49902cfbadf08bb8f634b15fa1814"}, + {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6fdcc55339df7761cd52e1fbe8185d3b3963bc9e3f3545faa6c84f9e8818259a"}, + {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1480b0051d8ab5408e8e4db2dc832f7082ea24aa0722c427bde2418c6f3bd07"}, + {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9855e6c258918f9cf62792d4f6ddfa6c56dccd8c8118640f867f6393ecaf8bd7"}, + {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de9529fe75efcd54ba8d516aa725e1851df9199f0669b665c55e90df08f5af86"}, + {file = "tokenizers-0.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8edcc90a36eab0705fe9121d6c77c6e42eeef25c7399864fd57dfb27173060bf"}, + {file = "tokenizers-0.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ae17884aafb3e94f34fb7cfedc29054f5f54e142475ebf8a265a4e388fee3f8b"}, + {file = "tokenizers-0.15.0-cp310-none-win32.whl", hash = "sha256:9a3241acdc9b44cff6e95c4a55b9be943ef3658f8edb3686034d353734adba05"}, + {file = "tokenizers-0.15.0-cp310-none-win_amd64.whl", hash = "sha256:4b31807cb393d6ea31926b307911c89a1209d5e27629aa79553d1599c8ffdefe"}, + {file = "tokenizers-0.15.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:af7e9be8c05d30bb137b9fd20f9d99354816599e5fd3d58a4b1e28ba3b36171f"}, + {file = "tokenizers-0.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c3d7343fa562ea29661783344a2d83662db0d3d17a6fa6a403cac8e512d2d9fd"}, + {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:32371008788aeeb0309a9244809a23e4c0259625e6b74a103700f6421373f395"}, + {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9db64c7c9954fbae698884c5bb089764edc549731e5f9b7fa1dd4e4d78d77f"}, + {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dbed5944c31195514669cf6381a0d8d47f164943000d10f93d6d02f0d45c25e0"}, + {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aab16c4a26d351d63e965b0c792f5da7227a37b69a6dc6d922ff70aa595b1b0c"}, + {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3c2b60b12fdd310bf85ce5d7d3f823456b9b65eed30f5438dd7761879c495983"}, + {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0344d6602740e44054a9e5bbe9775a5e149c4dddaff15959bb07dcce95a5a859"}, + {file = "tokenizers-0.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4525f6997d81d9b6d9140088f4f5131f6627e4c960c2c87d0695ae7304233fc3"}, + {file = "tokenizers-0.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:65975094fef8cc68919644936764efd2ce98cf1bacbe8db2687155d2b0625bee"}, + {file = "tokenizers-0.15.0-cp311-none-win32.whl", hash = "sha256:ff5d2159c5d93015f5a4542aac6c315506df31853123aa39042672031768c301"}, + {file = "tokenizers-0.15.0-cp311-none-win_amd64.whl", hash = "sha256:2dd681b53cf615e60a31a115a3fda3980e543d25ca183797f797a6c3600788a3"}, + {file = "tokenizers-0.15.0-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:c9cce6ee149a3d703f86877bc2a6d997e34874b2d5a2d7839e36b2273f31d3d9"}, + {file = "tokenizers-0.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a0a94bc3370e6f1cc8a07a8ae867ce13b7c1b4291432a773931a61f256d44ea"}, + {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:309cfcccfc7e502cb1f1de2c9c1c94680082a65bfd3a912d5a5b2c90c677eb60"}, + {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8413e994dd7d875ab13009127fc85633916c71213917daf64962bafd488f15dc"}, + {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0ebf9430f901dbdc3dcb06b493ff24a3644c9f88c08e6a1d6d0ae2228b9b818"}, + {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10361e9c7864b22dd791ec5126327f6c9292fb1d23481d4895780688d5e298ac"}, + {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:babe42635b8a604c594bdc56d205755f73414fce17ba8479d142a963a6c25cbc"}, + {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3768829861e964c7a4556f5f23307fce6a23872c2ebf030eb9822dbbbf7e9b2a"}, + {file = "tokenizers-0.15.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9c91588a630adc88065e1c03ac6831e3e2112558869b9ebcb2b8afd8a14c944d"}, + {file = "tokenizers-0.15.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:77606994e793ca54ecf3a3619adc8a906a28ca223d9354b38df41cb8766a0ed6"}, + {file = "tokenizers-0.15.0-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:6fe143939f3b596681922b2df12a591a5b010e7dcfbee2202482cd0c1c2f2459"}, + {file = "tokenizers-0.15.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:b7bee0f1795e3e3561e9a557061b1539e5255b8221e3f928f58100282407e090"}, + {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5d37e7f4439b4c46192ab4f2ff38ab815e4420f153caa13dec9272ef14403d34"}, + {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caadf255cf7f951b38d10097836d1f3bcff4aeaaffadfdf748bab780bf5bff95"}, + {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05accb9162bf711a941b1460b743d62fec61c160daf25e53c5eea52c74d77814"}, + {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26a2ef890740127cb115ee5260878f4a677e36a12831795fd7e85887c53b430b"}, + {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e54c5f26df14913620046b33e822cb3bcd091a332a55230c0e63cc77135e2169"}, + {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669b8ed653a578bcff919566631156f5da3aab84c66f3c0b11a6281e8b4731c7"}, + {file = "tokenizers-0.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0ea480d943297df26f06f508dab6e012b07f42bf3dffdd36e70799368a5f5229"}, + {file = "tokenizers-0.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc80a0a565ebfc7cd89de7dd581da8c2b3238addfca6280572d27d763f135f2f"}, + {file = "tokenizers-0.15.0-cp37-none-win32.whl", hash = "sha256:cdd945e678bbdf4517d5d8de66578a5030aeefecdb46f5320b034de9cad8d4dd"}, + {file = "tokenizers-0.15.0-cp37-none-win_amd64.whl", hash = "sha256:1ab96ab7dc706e002c32b2ea211a94c1c04b4f4de48354728c3a6e22401af322"}, + {file = "tokenizers-0.15.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:f21c9eb71c9a671e2a42f18b456a3d118e50c7f0fc4dd9fa8f4eb727fea529bf"}, + {file = "tokenizers-0.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a5f4543a35889679fc3052086e69e81880b2a5a28ff2a52c5a604be94b77a3f"}, + {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f8aa81afec893e952bd39692b2d9ef60575ed8c86fce1fd876a06d2e73e82dca"}, + {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1574a5a4af22c3def93fe8fe4adcc90a39bf5797ed01686a4c46d1c3bc677d2f"}, + {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c7982fd0ec9e9122d03b209dac48cebfea3de0479335100ef379a9a959b9a5a"}, + {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d16b647032df2ce2c1f9097236e046ea9fedd969b25637b9d5d734d78aa53b"}, + {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b3cdf29e6f9653da330515dc8fa414be5a93aae79e57f8acc50d4028dd843edf"}, + {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7286f3df10de840867372e3e64b99ef58c677210e3ceb653cd0e740a5c53fe78"}, + {file = "tokenizers-0.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aabc83028baa5a36ce7a94e7659250f0309c47fa4a639e5c2c38e6d5ea0de564"}, + {file = "tokenizers-0.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:72f78b0e0e276b1fc14a672fa73f3acca034ba8db4e782124a2996734a9ba9cf"}, + {file = "tokenizers-0.15.0-cp38-none-win32.whl", hash = "sha256:9680b0ecc26e7e42f16680c1aa62e924d58d1c2dd992707081cc10a374896ea2"}, + {file = "tokenizers-0.15.0-cp38-none-win_amd64.whl", hash = "sha256:f17cbd88dab695911cbdd385a5a7e3709cc61dff982351f5d1b5939f074a2466"}, + {file = "tokenizers-0.15.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:3661862df7382c5eb23ac4fbf7c75e69b02dc4f5784e4c5a734db406b5b24596"}, + {file = "tokenizers-0.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3045d191dad49647f5a5039738ecf1c77087945c7a295f7bcf051c37067e883"}, + {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a9fcaad9ab0801f14457d7c820d9f246b5ab590c407fc6b073819b1573097aa7"}, + {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79f17027f24fe9485701c8dbb269b9c713954ec3bdc1e7075a66086c0c0cd3c"}, + {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:01a3aa332abc4bee7640563949fcfedca4de8f52691b3b70f2fc6ca71bfc0f4e"}, + {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05b83896a893cdfedad8785250daa3ba9f0504848323471524d4783d7291661e"}, + {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbbf2489fcf25d809731ba2744ff278dd07d9eb3f8b7482726bd6cae607073a4"}, + {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab806ad521a5e9de38078b7add97589c313915f6f5fec6b2f9f289d14d607bd6"}, + {file = "tokenizers-0.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a522612d5c88a41563e3463226af64e2fa00629f65cdcc501d1995dd25d23f5"}, + {file = "tokenizers-0.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e58a38c4e6075810bdfb861d9c005236a72a152ebc7005941cc90d1bbf16aca9"}, + {file = "tokenizers-0.15.0-cp39-none-win32.whl", hash = "sha256:b8034f1041fd2bd2b84ff9f4dc4ae2e1c3b71606820a9cd5c562ebd291a396d1"}, + {file = "tokenizers-0.15.0-cp39-none-win_amd64.whl", hash = "sha256:edde9aa964145d528d0e0dbf14f244b8a85ebf276fb76869bc02e2530fa37a96"}, + {file = "tokenizers-0.15.0-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:309445d10d442b7521b98083dc9f0b5df14eca69dbbfebeb98d781ee2cef5d30"}, + {file = "tokenizers-0.15.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d3125a6499226d4d48efc54f7498886b94c418e93a205b673bc59364eecf0804"}, + {file = "tokenizers-0.15.0-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ed56ddf0d54877bb9c6d885177db79b41576e61b5ef6defeb579dcb803c04ad5"}, + {file = "tokenizers-0.15.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b22cd714706cc5b18992a232b023f736e539495f5cc61d2d28d176e55046f6c"}, + {file = "tokenizers-0.15.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fac2719b1e9bc8e8e7f6599b99d0a8e24f33d023eb8ef644c0366a596f0aa926"}, + {file = "tokenizers-0.15.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:85ddae17570ec7e5bfaf51ffa78d044f444a8693e1316e1087ee6150596897ee"}, + {file = "tokenizers-0.15.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:76f1bed992e396bf6f83e3df97b64ff47885e45e8365f8983afed8556a0bc51f"}, + {file = "tokenizers-0.15.0-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:3bb0f4df6dce41a1c7482087b60d18c372ef4463cb99aa8195100fcd41e0fd64"}, + {file = "tokenizers-0.15.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:22c27672c27a059a5f39ff4e49feed8c7f2e1525577c8a7e3978bd428eb5869d"}, + {file = "tokenizers-0.15.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78104f5d035c9991f92831fc0efe9e64a05d4032194f2a69f67aaa05a4d75bbb"}, + {file = "tokenizers-0.15.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a40b73dc19d82c3e3ffb40abdaacca8fbc95eeb26c66b7f9f860aebc07a73998"}, + {file = "tokenizers-0.15.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d801d1368188c74552cd779b1286e67cb9fd96f4c57a9f9a2a09b6def9e1ab37"}, + {file = "tokenizers-0.15.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82641ffb13a4da1293fcc9f437d457647e60ed0385a9216cd135953778b3f0a1"}, + {file = "tokenizers-0.15.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:160f9d1810f2c18fffa94aa98bf17632f6bd2dabc67fcb01a698ca80c37d52ee"}, + {file = "tokenizers-0.15.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:8d7d6eea831ed435fdeeb9bcd26476226401d7309d115a710c65da4088841948"}, + {file = "tokenizers-0.15.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f6456bec6c557d63d8ec0023758c32f589e1889ed03c055702e84ce275488bed"}, + {file = "tokenizers-0.15.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1eef39a502fad3bf104b9e1906b4fb0cee20e44e755e51df9a98f8922c3bf6d4"}, + {file = "tokenizers-0.15.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1e4664c5b797e093c19b794bbecc19d2367e782b4a577d8b7c1821db5dc150d"}, + {file = "tokenizers-0.15.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:ca003fb5f3995ff5cf676db6681b8ea5d54d3b30bea36af1120e78ee1a4a4cdf"}, + {file = "tokenizers-0.15.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7f17363141eb0c53752c89e10650b85ef059a52765d0802ba9613dbd2d21d425"}, + {file = "tokenizers-0.15.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:8a765db05581c7d7e1280170f2888cda351760d196cc059c37ea96f121125799"}, + {file = "tokenizers-0.15.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:2a0dd641a72604486cd7302dd8f87a12c8a9b45e1755e47d2682733f097c1af5"}, + {file = "tokenizers-0.15.0-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0a1a3c973e4dc97797fc19e9f11546c95278ffc55c4492acb742f69e035490bc"}, + {file = "tokenizers-0.15.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4fab75642aae4e604e729d6f78e0addb9d7e7d49e28c8f4d16b24da278e5263"}, + {file = "tokenizers-0.15.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65f80be77f6327a86d8fd35a4467adcfe6174c159b4ab52a1a8dd4c6f2d7d9e1"}, + {file = "tokenizers-0.15.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a8da7533dbe66b88afd430c56a2f2ce1fd82e2681868f857da38eeb3191d7498"}, + {file = "tokenizers-0.15.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa8eb4584fc6cbe6a84d7a7864be3ed28e23e9fd2146aa8ef1814d579df91958"}, + {file = "tokenizers-0.15.0.tar.gz", hash = "sha256:10c7e6e7b4cabd757da59e93f5f8d1126291d16f8b54f28510825ef56a3e5d0e"}, ] +[package.dependencies] +huggingface_hub = ">=0.16.4,<1.0" + [package.extras] -dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +dev = ["tokenizers[testing]"] +docs = ["setuptools_rust", "sphinx", "sphinx_rtd_theme"] testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] [[package]] @@ -2236,31 +2316,31 @@ files = [ [[package]] name = "torch" -version = "2.1.1" +version = "2.1.2" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" optional = true python-versions = ">=3.8.0" files = [ - {file = "torch-2.1.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:5ebc43f5355a9b7be813392b3fb0133991f0380f6f0fcc8218d5468dc45d1071"}, - {file = "torch-2.1.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:84fefd63356416c0cd20578637ccdbb82164993400ed17b57c951dd6376dcee8"}, - {file = "torch-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:0a7a9da0c324409bcb5a7bdad1b4e94e936d21c2590aaa7ac2f63968da8c62f7"}, - {file = "torch-2.1.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:1e1e5faddd43a8f2c0e0e22beacd1e235a2e447794d807483c94a9e31b54a758"}, - {file = "torch-2.1.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:e76bf3c5c354874f1da465c852a2fb60ee6cbce306e935337885760f080f9baa"}, - {file = "torch-2.1.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:98fea993639b0bb432dfceb7b538f07c0f1c33386d63f635219f49254968c80f"}, - {file = "torch-2.1.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:61b51b33c61737c287058b0c3061e6a9d3c363863e4a094f804bc486888a188a"}, - {file = "torch-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:1d70920da827e2276bf07f7ec46958621cad18d228c97da8f9c19638474dbd52"}, - {file = "torch-2.1.1-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:a70593806f1d7e6b53657d96810518da0f88ef2608c98a402955765b8c79d52c"}, - {file = "torch-2.1.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:e312f7e82e49565f7667b0bbf9559ab0c597063d93044740781c02acd5a87978"}, - {file = "torch-2.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:1e3cbecfa5a7314d828f4a37b0c286714dc9aa2e69beb7a22f7aca76567ed9f4"}, - {file = "torch-2.1.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:9ca0fcbf3d5ba644d6a8572c83a9abbdf5f7ff575bc38529ef6c185a3a71bde9"}, - {file = "torch-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:2dc9f312fc1fa0d61a565a0292ad73119d4b74c9f8b5031b55f8b4722abca079"}, - {file = "torch-2.1.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:d56b032176458e2af4709627bbd2c20fe2917eff8cd087a7fe313acccf5ce2f1"}, - {file = "torch-2.1.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:29e3b90a8c281f6660804a939d1f4218604c80162e521e1e6d8c8557325902a0"}, - {file = "torch-2.1.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:bd95cee8511584b67ddc0ba465c3f1edeb5708d833ee02af1206b4486f1d9096"}, - {file = "torch-2.1.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b31230bd058424e56dba7f899280dbc6ac8b9948e43902e0c84a44666b1ec151"}, - {file = "torch-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:403f1095e665e4f35971b43797a920725b8b205723aa68254a4050c6beca29b6"}, - {file = "torch-2.1.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:715b50d8c1de5da5524a68287eb000f73e026e74d5f6b12bc450ef6995fcf5f9"}, - {file = "torch-2.1.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:db67e8725c76f4c7f4f02e7551bb16e81ba1a1912867bc35d7bb96d2be8c78b4"}, + {file = "torch-2.1.2-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:3a871edd6c02dae77ad810335c0833391c1a4ce49af21ea8cf0f6a5d2096eea8"}, + {file = "torch-2.1.2-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:bef6996c27d8f6e92ea4e13a772d89611da0e103b48790de78131e308cf73076"}, + {file = "torch-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:0e13034fd5fb323cbbc29e56d0637a3791e50dd589616f40c79adfa36a5a35a1"}, + {file = "torch-2.1.2-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:d9b535cad0df3d13997dbe8bd68ac33e0e3ae5377639c9881948e40794a61403"}, + {file = "torch-2.1.2-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:f9a55d55af02826ebfbadf4e9b682f0f27766bc33df8236b48d28d705587868f"}, + {file = "torch-2.1.2-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:a6ebbe517097ef289cc7952783588c72de071d4b15ce0f8b285093f0916b1162"}, + {file = "torch-2.1.2-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:8f32ce591616a30304f37a7d5ea80b69ca9e1b94bba7f308184bf616fdaea155"}, + {file = "torch-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e0ee6cf90c8970e05760f898d58f9ac65821c37ffe8b04269ec787aa70962b69"}, + {file = "torch-2.1.2-cp311-none-macosx_10_9_x86_64.whl", hash = "sha256:76d37967c31c99548ad2c4d3f2cf191db48476f2e69b35a0937137116da356a1"}, + {file = "torch-2.1.2-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:e2d83f07b4aac983453ea5bf8f9aa9dacf2278a8d31247f5d9037f37befc60e4"}, + {file = "torch-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:f41fe0c7ecbf903a568c73486139a75cfab287a0f6c17ed0698fdea7a1e8641d"}, + {file = "torch-2.1.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:e3225f47d50bb66f756fe9196a768055d1c26b02154eb1f770ce47a2578d3aa7"}, + {file = "torch-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33d59cd03cb60106857f6c26b36457793637512998666ee3ce17311f217afe2b"}, + {file = "torch-2.1.2-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:8e221deccd0def6c2badff6be403e0c53491805ed9915e2c029adbcdb87ab6b5"}, + {file = "torch-2.1.2-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:05b18594f60a911a0c4f023f38a8bda77131fba5fd741bda626e97dcf5a3dd0a"}, + {file = "torch-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:9ca96253b761e9aaf8e06fb30a66ee301aecbf15bb5a303097de1969077620b6"}, + {file = "torch-2.1.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d93ba70f67b08c2ae5598ee711cbc546a1bc8102cef938904b8c85c2089a51a0"}, + {file = "torch-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:255b50bc0608db177e6a3cc118961d77de7e5105f07816585fa6f191f33a9ff3"}, + {file = "torch-2.1.2-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6984cd5057c0c977b3c9757254e989d3f1124f4ce9d07caa6cb637783c71d42a"}, + {file = "torch-2.1.2-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:bc195d7927feabc0eb7c110e457c955ed2ab616f3c7c28439dd4188cf589699f"}, ] [package.dependencies] @@ -2309,53 +2389,52 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.33.3" +version = "4.36.1" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" optional = false python-versions = ">=3.8.0" files = [ - {file = "transformers-4.33.3-py3-none-any.whl", hash = "sha256:7150bbf6781ddb3338ce7d74f4d6f557e6c236a0a1dd3de57412214caae7fd71"}, - {file = "transformers-4.33.3.tar.gz", hash = "sha256:8ea7c92310dee7c63b14766ce928218f7a9177960b2487ac018c91ae621af03e"}, + {file = "transformers-4.36.1-py3-none-any.whl", hash = "sha256:0e309d03634885f02d46801ec4f2c3fc1d614a5b9ebde608181f3e842bac53b8"}, + {file = "transformers-4.36.1.tar.gz", hash = "sha256:28e55952d9bed68f06cf45a3d29cc480679b528afe944e68f8cf6c799e428759"}, ] [package.dependencies] filelock = "*" -huggingface-hub = ">=0.15.1,<1.0" +huggingface-hub = ">=0.19.3,<1.0" numpy = ">=1.17" packaging = ">=20.0" pyyaml = ">=5.1" regex = "!=2019.12.17" requests = "*" safetensors = ">=0.3.1" -tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" +tokenizers = ">=0.14,<0.19" tqdm = ">=4.27" [package.extras] -accelerate = ["accelerate (>=0.20.3)"] -agents = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.10,!=1.12.0)"] -all = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] +accelerate = ["accelerate (>=0.21.0)"] +agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.10,!=1.12.0)"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] codecarbon = ["codecarbon (==1.2.0)"] -deepspeed = ["accelerate (>=0.20.3)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow (<10.0.0)", "accelerate (>=0.20.3)", "beautifulsoup4", "black (>=23.1,<24.0)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (>=0.0.241,<=0.0.259)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -docs = ["Pillow (<10.0.0)", "accelerate (>=0.20.3)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] +deepspeed = ["accelerate (>=0.21.0)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.21.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.14,<0.19)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] docs-specific = ["hf-doc-builder"] -fairscale = ["fairscale (>0.3)"] flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] ftfy = ["ftfy"] -integrations = ["optuna", "ray[tune]", "sigopt"] +integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] modelcreation = ["cookiecutter (==1.7.3)"] natten = ["natten (>=0.14.6)"] onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "black (>=23.1,<24.0)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (>=0.0.241,<=0.0.259)", "urllib3 (<2.0.0)"] -ray = ["ray[tune]"] +quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (==0.1.5)", "urllib3 (<2.0.0)"] +ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] @@ -2363,18 +2442,18 @@ serving = ["fastapi", "pydantic (<2)", "starlette", "uvicorn"] sigopt = ["sigopt"] sklearn = ["scikit-learn"] speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (>=23.1,<24.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx"] -tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.15)", "tensorflow-text (<2.15)", "tf2onnx"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "tensorboard", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] timm = ["timm"] -tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -torch = ["accelerate (>=0.20.3)", "torch (>=1.10,!=1.12.0)"] +tokenizers = ["tokenizers (>=0.14,<0.19)"] +torch = ["accelerate (>=0.21.0)", "torch (>=1.10,!=1.12.0)"] torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torch-vision = ["Pillow (<10.0.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.15.1,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.10,!=1.12.0)", "tqdm (>=4.27)"] +torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] +torchhub = ["filelock", "huggingface-hub (>=0.19.3,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "tqdm (>=4.27)"] video = ["av (==9.2.0)", "decord (==0.6.0)"] -vision = ["Pillow (<10.0.0)"] +vision = ["Pillow (>=10.0.1,<=15.0)"] [[package]] name = "triton" @@ -2423,13 +2502,13 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6. [[package]] name = "typing-extensions" -version = "4.8.0" +version = "4.9.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, - {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] [[package]] @@ -2671,101 +2750,101 @@ files = [ [[package]] name = "yarl" -version = "1.9.3" +version = "1.9.4" description = "Yet another URL library" optional = true python-versions = ">=3.7" files = [ - {file = "yarl-1.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:32435d134414e01d937cd9d6cc56e8413a8d4741dea36af5840c7750f04d16ab"}, - {file = "yarl-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9a5211de242754b5e612557bca701f39f8b1a9408dff73c6db623f22d20f470e"}, - {file = "yarl-1.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:525cd69eff44833b01f8ef39aa33a9cc53a99ff7f9d76a6ef6a9fb758f54d0ff"}, - {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc94441bcf9cb8c59f51f23193316afefbf3ff858460cb47b5758bf66a14d130"}, - {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e36021db54b8a0475805acc1d6c4bca5d9f52c3825ad29ae2d398a9d530ddb88"}, - {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0f17d1df951336a02afc8270c03c0c6e60d1f9996fcbd43a4ce6be81de0bd9d"}, - {file = "yarl-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5f3faeb8100a43adf3e7925d556801d14b5816a0ac9e75e22948e787feec642"}, - {file = "yarl-1.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aed37db837ecb5962469fad448aaae0f0ee94ffce2062cf2eb9aed13328b5196"}, - {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:721ee3fc292f0d069a04016ef2c3a25595d48c5b8ddc6029be46f6158d129c92"}, - {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b8bc5b87a65a4e64bc83385c05145ea901b613d0d3a434d434b55511b6ab0067"}, - {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:dd952b9c64f3b21aedd09b8fe958e4931864dba69926d8a90c90d36ac4e28c9a"}, - {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:c405d482c320a88ab53dcbd98d6d6f32ada074f2d965d6e9bf2d823158fa97de"}, - {file = "yarl-1.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9df9a0d4c5624790a0dea2e02e3b1b3c69aed14bcb8650e19606d9df3719e87d"}, - {file = "yarl-1.9.3-cp310-cp310-win32.whl", hash = "sha256:d34c4f80956227f2686ddea5b3585e109c2733e2d4ef12eb1b8b4e84f09a2ab6"}, - {file = "yarl-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:cf7a4e8de7f1092829caef66fd90eaf3710bc5efd322a816d5677b7664893c93"}, - {file = "yarl-1.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d61a0ca95503867d4d627517bcfdc28a8468c3f1b0b06c626f30dd759d3999fd"}, - {file = "yarl-1.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:73cc83f918b69110813a7d95024266072d987b903a623ecae673d1e71579d566"}, - {file = "yarl-1.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d81657b23e0edb84b37167e98aefb04ae16cbc5352770057893bd222cdc6e45f"}, - {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a1a8443091c7fbc17b84a0d9f38de34b8423b459fb853e6c8cdfab0eacf613"}, - {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe34befb8c765b8ce562f0200afda3578f8abb159c76de3ab354c80b72244c41"}, - {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c757f64afe53a422e45e3e399e1e3cf82b7a2f244796ce80d8ca53e16a49b9f"}, - {file = "yarl-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72a57b41a0920b9a220125081c1e191b88a4cdec13bf9d0649e382a822705c65"}, - {file = "yarl-1.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:632c7aeb99df718765adf58eacb9acb9cbc555e075da849c1378ef4d18bf536a"}, - {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b0b8c06afcf2bac5a50b37f64efbde978b7f9dc88842ce9729c020dc71fae4ce"}, - {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1d93461e2cf76c4796355494f15ffcb50a3c198cc2d601ad8d6a96219a10c363"}, - {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:4003f380dac50328c85e85416aca6985536812c082387255c35292cb4b41707e"}, - {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4d6d74a97e898c1c2df80339aa423234ad9ea2052f66366cef1e80448798c13d"}, - {file = "yarl-1.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b61e64b06c3640feab73fa4ff9cb64bd8182de52e5dc13038e01cfe674ebc321"}, - {file = "yarl-1.9.3-cp311-cp311-win32.whl", hash = "sha256:29beac86f33d6c7ab1d79bd0213aa7aed2d2f555386856bb3056d5fdd9dab279"}, - {file = "yarl-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:f7271d6bd8838c49ba8ae647fc06469137e1c161a7ef97d778b72904d9b68696"}, - {file = "yarl-1.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:dd318e6b75ca80bff0b22b302f83a8ee41c62b8ac662ddb49f67ec97e799885d"}, - {file = "yarl-1.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c4b1efb11a8acd13246ffb0bee888dd0e8eb057f8bf30112e3e21e421eb82d4a"}, - {file = "yarl-1.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c6f034386e5550b5dc8ded90b5e2ff7db21f0f5c7de37b6efc5dac046eb19c10"}, - {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd49a908cb6d387fc26acee8b7d9fcc9bbf8e1aca890c0b2fdfd706057546080"}, - {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa4643635f26052401750bd54db911b6342eb1a9ac3e74f0f8b58a25d61dfe41"}, - {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e741bd48e6a417bdfbae02e088f60018286d6c141639359fb8df017a3b69415a"}, - {file = "yarl-1.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c86d0d0919952d05df880a1889a4f0aeb6868e98961c090e335671dea5c0361"}, - {file = "yarl-1.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d5434b34100b504aabae75f0622ebb85defffe7b64ad8f52b8b30ec6ef6e4b9"}, - {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79e1df60f7c2b148722fb6cafebffe1acd95fd8b5fd77795f56247edaf326752"}, - {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:44e91a669c43f03964f672c5a234ae0d7a4d49c9b85d1baa93dec28afa28ffbd"}, - {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:3cfa4dbe17b2e6fca1414e9c3bcc216f6930cb18ea7646e7d0d52792ac196808"}, - {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:88d2c3cc4b2f46d1ba73d81c51ec0e486f59cc51165ea4f789677f91a303a9a7"}, - {file = "yarl-1.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cccdc02e46d2bd7cb5f38f8cc3d9db0d24951abd082b2f242c9e9f59c0ab2af3"}, - {file = "yarl-1.9.3-cp312-cp312-win32.whl", hash = "sha256:96758e56dceb8a70f8a5cff1e452daaeff07d1cc9f11e9b0c951330f0a2396a7"}, - {file = "yarl-1.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:c4472fe53ebf541113e533971bd8c32728debc4c6d8cc177f2bff31d011ec17e"}, - {file = "yarl-1.9.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:126638ab961633f0940a06e1c9d59919003ef212a15869708dcb7305f91a6732"}, - {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c99ddaddb2fbe04953b84d1651149a0d85214780e4d0ee824e610ab549d98d92"}, - {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dab30b21bd6fb17c3f4684868c7e6a9e8468078db00f599fb1c14e324b10fca"}, - {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:828235a2a169160ee73a2fcfb8a000709edf09d7511fccf203465c3d5acc59e4"}, - {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc391e3941045fd0987c77484b2799adffd08e4b6735c4ee5f054366a2e1551d"}, - {file = "yarl-1.9.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51382c72dd5377861b573bd55dcf680df54cea84147c8648b15ac507fbef984d"}, - {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:28a108cb92ce6cf867690a962372996ca332d8cda0210c5ad487fe996e76b8bb"}, - {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8f18a7832ff85dfcd77871fe677b169b1bc60c021978c90c3bb14f727596e0ae"}, - {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:7eaf13af79950142ab2bbb8362f8d8d935be9aaf8df1df89c86c3231e4ff238a"}, - {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:66a6dbf6ca7d2db03cc61cafe1ee6be838ce0fbc97781881a22a58a7c5efef42"}, - {file = "yarl-1.9.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1a0a4f3aaa18580038cfa52a7183c8ffbbe7d727fe581300817efc1e96d1b0e9"}, - {file = "yarl-1.9.3-cp37-cp37m-win32.whl", hash = "sha256:946db4511b2d815979d733ac6a961f47e20a29c297be0d55b6d4b77ee4b298f6"}, - {file = "yarl-1.9.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2dad8166d41ebd1f76ce107cf6a31e39801aee3844a54a90af23278b072f1ccf"}, - {file = "yarl-1.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:bb72d2a94481e7dc7a0c522673db288f31849800d6ce2435317376a345728225"}, - {file = "yarl-1.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9a172c3d5447b7da1680a1a2d6ecdf6f87a319d21d52729f45ec938a7006d5d8"}, - {file = "yarl-1.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2dc72e891672343b99db6d497024bf8b985537ad6c393359dc5227ef653b2f17"}, - {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8d51817cf4b8d545963ec65ff06c1b92e5765aa98831678d0e2240b6e9fd281"}, - {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53ec65f7eee8655bebb1f6f1607760d123c3c115a324b443df4f916383482a67"}, - {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cfd77e8e5cafba3fb584e0f4b935a59216f352b73d4987be3af51f43a862c403"}, - {file = "yarl-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e73db54c967eb75037c178a54445c5a4e7461b5203b27c45ef656a81787c0c1b"}, - {file = "yarl-1.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09c19e5f4404574fcfb736efecf75844ffe8610606f3fccc35a1515b8b6712c4"}, - {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6280353940f7e5e2efaaabd686193e61351e966cc02f401761c4d87f48c89ea4"}, - {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c25ec06e4241e162f5d1f57c370f4078797ade95c9208bd0c60f484834f09c96"}, - {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:7217234b10c64b52cc39a8d82550342ae2e45be34f5bff02b890b8c452eb48d7"}, - {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4ce77d289f8d40905c054b63f29851ecbfd026ef4ba5c371a158cfe6f623663e"}, - {file = "yarl-1.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5f74b015c99a5eac5ae589de27a1201418a5d9d460e89ccb3366015c6153e60a"}, - {file = "yarl-1.9.3-cp38-cp38-win32.whl", hash = "sha256:8a2538806be846ea25e90c28786136932ec385c7ff3bc1148e45125984783dc6"}, - {file = "yarl-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:6465d36381af057d0fab4e0f24ef0e80ba61f03fe43e6eeccbe0056e74aadc70"}, - {file = "yarl-1.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2f3c8822bc8fb4a347a192dd6a28a25d7f0ea3262e826d7d4ef9cc99cd06d07e"}, - {file = "yarl-1.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7831566595fe88ba17ea80e4b61c0eb599f84c85acaa14bf04dd90319a45b90"}, - {file = "yarl-1.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ff34cb09a332832d1cf38acd0f604c068665192c6107a439a92abfd8acf90fe2"}, - {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe8080b4f25dfc44a86bedd14bc4f9d469dfc6456e6f3c5d9077e81a5fedfba7"}, - {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8535e111a064f3bdd94c0ed443105934d6f005adad68dd13ce50a488a0ad1bf3"}, - {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0d155a092bf0ebf4a9f6f3b7a650dc5d9a5bbb585ef83a52ed36ba46f55cc39d"}, - {file = "yarl-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:778df71c8d0c8c9f1b378624b26431ca80041660d7be7c3f724b2c7a6e65d0d6"}, - {file = "yarl-1.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9f9cafaf031c34d95c1528c16b2fa07b710e6056b3c4e2e34e9317072da5d1a"}, - {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ca6b66f69e30f6e180d52f14d91ac854b8119553b524e0e28d5291a724f0f423"}, - {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0e7e83f31e23c5d00ff618045ddc5e916f9e613d33c5a5823bc0b0a0feb522f"}, - {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:af52725c7c39b0ee655befbbab5b9a1b209e01bb39128dce0db226a10014aacc"}, - {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0ab5baaea8450f4a3e241ef17e3d129b2143e38a685036b075976b9c415ea3eb"}, - {file = "yarl-1.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6d350388ba1129bc867c6af1cd17da2b197dff0d2801036d2d7d83c2d771a682"}, - {file = "yarl-1.9.3-cp39-cp39-win32.whl", hash = "sha256:e2a16ef5fa2382af83bef4a18c1b3bcb4284c4732906aa69422cf09df9c59f1f"}, - {file = "yarl-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:d92d897cb4b4bf915fbeb5e604c7911021a8456f0964f3b8ebbe7f9188b9eabb"}, - {file = "yarl-1.9.3-py3-none-any.whl", hash = "sha256:271d63396460b6607b588555ea27a1a02b717ca2e3f2cf53bdde4013d7790929"}, - {file = "yarl-1.9.3.tar.gz", hash = "sha256:4a14907b597ec55740f63e52d7fee0e9ee09d5b9d57a4f399a7423268e457b57"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, ] [package.dependencies] @@ -2782,4 +2861,4 @@ torch = ["torch"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "cd3fb4b4e4aaf100f6015afa8c9adc28c22e6c0b48752452892dc3d004c1562a" +content-hash = "d314f7dc9ea4d4e7581552f340d9171f24f709bf98de8b8c01c449c23026e7a3" diff --git a/server/pyproject.toml b/server/pyproject.toml index ad3d5255..9bec2738 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -15,7 +15,7 @@ grpcio-status = "^1.51.1" grpcio-reflection = "^1.51.1" grpc-interceptor = "^0.15.0" typer = "^0.6.1" -accelerate = { version = "^0.20.0", optional = true } +accelerate = { version = "^0.25.0", optional = true } bitsandbytes = { version = "^0.41.1", optional = true } safetensors = "^0.3.2" loguru = "^0.6.0" @@ -24,9 +24,9 @@ opentelemetry-exporter-otlp = "^1.15.0" opentelemetry-instrumentation-grpc = "^0.36b0" hf-transfer = "^0.1.2" sentencepiece = "^0.1.97" -tokenizers = "^0.13.3" -huggingface-hub = "^0.16.4" -transformers = "^4.32.1" +tokenizers = "^0.15.0" +huggingface-hub = "^0.19.3" +transformers = "^4.36.1" einops = "^0.6.1" texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } diff --git a/server/requirements_cuda.txt b/server/requirements_cuda.txt index bc1b8891..694242e1 100644 --- a/server/requirements_cuda.txt +++ b/server/requirements_cuda.txt @@ -1,5 +1,5 @@ backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13" -bitsandbytes==0.41.2.post2 ; python_version >= "3.9" and python_version < "3.13" +bitsandbytes==0.41.3.post2 ; python_version >= "3.9" and python_version < "3.13" certifi==2023.11.17 ; python_version >= "3.9" and python_version < "3.13" charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "3.13" click==8.1.7 ; python_version >= "3.9" and python_version < "3.13" @@ -8,14 +8,14 @@ deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13" einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13" filelock==3.13.1 ; python_version >= "3.9" and python_version < "3.13" fsspec==2023.10.0 ; python_version >= "3.9" and python_version < "3.13" -googleapis-common-protos==1.61.0 ; python_version >= "3.9" and python_version < "3.13" +googleapis-common-protos==1.62.0 ; python_version >= "3.9" and python_version < "3.13" grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13" -grpcio-reflection==1.59.3 ; python_version >= "3.9" and python_version < "3.13" -grpcio-status==1.59.3 ; python_version >= "3.9" and python_version < "3.13" -grpcio==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +grpcio-reflection==1.60.0 ; python_version >= "3.9" and python_version < "3.13" +grpcio-status==1.60.0 ; python_version >= "3.9" and python_version < "3.13" +grpcio==1.60.0 ; python_version >= "3.9" and python_version < "3.13" hf-transfer==0.1.4 ; python_version >= "3.9" and python_version < "3.13" -huggingface-hub==0.16.4 ; python_version >= "3.9" and python_version < "3.13" -idna==3.4 ; python_version >= "3.9" and python_version < "3.13" +huggingface-hub==0.19.4 ; python_version >= "3.9" and python_version < "3.13" +idna==3.6 ; python_version >= "3.9" and python_version < "3.13" loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" numpy==1.26.2 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" @@ -37,11 +37,11 @@ safetensors==0.3.3 ; python_version >= "3.9" and python_version < "3.13" scipy==1.11.4 ; python_version >= "3.9" and python_version < "3.13" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" setuptools==69.0.2 ; python_version >= "3.9" and python_version < "3.13" -tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" +tokenizers==0.15.0 ; python_version >= "3.9" and python_version < "3.13" tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" -transformers==4.33.3 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.36.1 ; python_version >= "3.9" and python_version < "3.13" typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" -typing-extensions==4.8.0 ; python_version >= "3.9" and python_version < "3.13" +typing-extensions==4.9.0 ; python_version >= "3.9" and python_version < "3.13" urllib3==2.1.0 ; python_version >= "3.9" and python_version < "3.13" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32" wrapt==1.16.0 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/requirements_rocm.txt b/server/requirements_rocm.txt index 5a321834..e0495fde 100644 --- a/server/requirements_rocm.txt +++ b/server/requirements_rocm.txt @@ -7,14 +7,14 @@ deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13" einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13" filelock==3.13.1 ; python_version >= "3.9" and python_version < "3.13" fsspec==2023.10.0 ; python_version >= "3.9" and python_version < "3.13" -googleapis-common-protos==1.61.0 ; python_version >= "3.9" and python_version < "3.13" +googleapis-common-protos==1.62.0 ; python_version >= "3.9" and python_version < "3.13" grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13" -grpcio-reflection==1.59.3 ; python_version >= "3.9" and python_version < "3.13" -grpcio-status==1.59.3 ; python_version >= "3.9" and python_version < "3.13" -grpcio==1.59.3 ; python_version >= "3.9" and python_version < "3.13" +grpcio-reflection==1.60.0 ; python_version >= "3.9" and python_version < "3.13" +grpcio-status==1.60.0 ; python_version >= "3.9" and python_version < "3.13" +grpcio==1.60.0 ; python_version >= "3.9" and python_version < "3.13" hf-transfer==0.1.4 ; python_version >= "3.9" and python_version < "3.13" -huggingface-hub==0.16.4 ; python_version >= "3.9" and python_version < "3.13" -idna==3.4 ; python_version >= "3.9" and python_version < "3.13" +huggingface-hub==0.19.4 ; python_version >= "3.9" and python_version < "3.13" +idna==3.6 ; python_version >= "3.9" and python_version < "3.13" loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" numpy==1.26.2 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" @@ -36,11 +36,11 @@ safetensors==0.3.3 ; python_version >= "3.9" and python_version < "3.13" scipy==1.11.4 ; python_version >= "3.9" and python_version < "3.13" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" setuptools==69.0.2 ; python_version >= "3.9" and python_version < "3.13" -tokenizers==0.13.3 ; python_version >= "3.9" and python_version < "3.13" +tokenizers==0.15.0 ; python_version >= "3.9" and python_version < "3.13" tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" -transformers==4.33.3 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.36.1 ; python_version >= "3.9" and python_version < "3.13" typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" -typing-extensions==4.8.0 ; python_version >= "3.9" and python_version < "3.13" +typing-extensions==4.9.0 ; python_version >= "3.9" and python_version < "3.13" urllib3==2.1.0 ; python_version >= "3.9" and python_version < "3.13" win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32" wrapt==1.16.0 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index f2ef55ce..1bbff16a 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -55,10 +55,14 @@ FlashSantacoderSharded, ) from text_generation_server.models.idefics import IDEFICSSharded + from text_generation_server.models.flash_mistral import FlashMistral + from text_generation_server.models.flash_mixtral import FlashMixtral + from text_generation_server.utils.flash_attn import HAS_FLASH_ATTN_V2_CUDA except ImportError as e: logger.warning(f"Could not import Flash Attention enabled models: {e}") FLASH_ATTENTION = False + HAS_FLASH_ATTN_V2_CUDA = False if FLASH_ATTENTION: __all__.append(FlashNeoXSharded) @@ -66,25 +70,7 @@ __all__.append(FlashSantacoderSharded) __all__.append(FlashLlama) __all__.append(IDEFICSSharded) - -MISTRAL = True -try: - from text_generation_server.models.flash_mistral import FlashMistral -except ImportError as e: - logger.warning(f"Could not import Mistral model: {e}") - MISTRAL = False - -if MISTRAL: __all__.append(FlashMistral) - -MIXTRAL = True -try: - from text_generation_server.models.flash_mixtral import FlashMixtral -except ImportError as e: - logger.warning(f"Could not import Mixtral model: {e}") - MIXTRAL = False - -if MIXTRAL: __all__.append(FlashMixtral) @@ -295,7 +281,9 @@ def get_model( ) if model_type == "mistral": - if MISTRAL: + if (config_dict["sliding_window"] is None and FLASH_ATTENTION) or ( + config_dict["sliding_window"] > 0 and HAS_FLASH_ATTN_V2_CUDA + ): return FlashMistral( model_id, revision, @@ -303,10 +291,11 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) - raise NotImplementedError("Mistral models requires flash attention v2") if model_type == "mixtral": - if MIXTRAL: + if (config_dict["sliding_window"] is None and FLASH_ATTENTION) or ( + config_dict["sliding_window"] > 0 and HAS_FLASH_ATTN_V2_CUDA + ): return FlashMixtral( model_id, revision, @@ -314,9 +303,6 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) - raise NotImplementedError( - "Mixtral models requires flash attention v2, stk and megablocks" - ) if model_type == "opt": return OPTSharded( @@ -348,17 +334,17 @@ def get_model( raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format("Idefics")) if sharded: - raise ValueError("sharded is not supported for AutoModel") + raise NotImplementedError("sharded is not supported for AutoModel") if quantize == "gptq": - raise ValueError( + raise NotImplementedError( "gptq quantization is not supported for AutoModel, you can try to quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" ) if quantize == "awq": - raise ValueError("awq quantization is not supported for AutoModel") + raise NotImplementedError("awq quantization is not supported for AutoModel") elif (quantize == "bitsandbytes-fp4") or (quantize == "bitsandbytes-nf4"): - raise ValueError("4bit quantization is not supported for AutoModel") + raise NotImplementedError("4bit quantization is not supported for AutoModel") elif quantize == "eetq": - raise ValueError("Eetq quantization is not supported for AutoModel") + raise NotImplementedError("Eetq quantization is not supported for AutoModel") if model_type in modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: return CausalLM( model_id, diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index afeaf7e5..c85624f3 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -27,11 +27,6 @@ from typing import Optional, List, Tuple from text_generation_server.utils import paged_attention, flash_attn -from text_generation_server.utils.flash_attn import ( - attention, - HAS_FLASH_ATTN_V2_ROCM, - HAS_FLASH_ATTN_V2_CUDA, -) from text_generation_server.utils.layers import ( TensorParallelRowLinear, TensorParallelColumnLinear, @@ -43,10 +38,6 @@ ) -if not HAS_FLASH_ATTN_V2_CUDA and not HAS_FLASH_ATTN_V2_ROCM: - raise ImportError("Mistral model requires flash attn v2") - - class MistralConfig(PretrainedConfig): model_type = "mistral" diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py index 35bb3735..b468d09b 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py @@ -27,12 +27,9 @@ from transformers.activations import ACT2FN from transformers.configuration_utils import PretrainedConfig from typing import Optional, List, Tuple +from loguru import logger from text_generation_server.utils import paged_attention, flash_attn -from text_generation_server.utils.flash_attn import ( - HAS_FLASH_ATTN_V2_ROCM, - HAS_FLASH_ATTN_V2_CUDA, -) from text_generation_server.utils.layers import ( FastLinear, FastRMSNorm, @@ -44,18 +41,13 @@ get_linear, ) -if not HAS_FLASH_ATTN_V2_CUDA and not HAS_FLASH_ATTN_V2_ROCM: - raise ImportError("Mixtral model requires flash attn v2") - -try: - import megablocks.ops as ops -except ImportError: - raise ImportError("Mixtral model requires megablocks to be installed") - +HAS_MEGABLOCKS = True try: import stk + import megablocks.ops as ops except ImportError: - raise ImportError("Mixtral model requires stk to be installed") + logger.warning("Mixtral: megablocks is not installed") + HAS_MEGABLOCKS = False class MixtralConfig(PretrainedConfig): @@ -590,7 +582,7 @@ def dense_forward(self, x: torch.Tensor) -> torch.Tensor: return out def forward(self, x: torch.Tensor) -> torch.Tensor: - if len(x) > 256: + if len(x) > 256 and HAS_MEGABLOCKS: return self.sparse_forward(x) # This is faster when there is not a lot of tokens return self.dense_forward(x) From 1b1bfa49b04448ab1afac2d9bf790fff7bd1871b Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 15 Dec 2023 14:56:17 +0100 Subject: [PATCH 506/793] fix: fix logic if sliding window key is not present in config (#1352) --- server/text_generation_server/models/__init__.py | 14 ++++++++------ .../custom_modeling/flash_mistral_modeling.py | 2 +- .../custom_modeling/flash_mixtral_modeling.py | 2 +- server/text_generation_server/models/model.py | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 1bbff16a..39d1d58e 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -281,9 +281,10 @@ def get_model( ) if model_type == "mistral": - if (config_dict["sliding_window"] is None and FLASH_ATTENTION) or ( - config_dict["sliding_window"] > 0 and HAS_FLASH_ATTN_V2_CUDA - ): + sliding_window = config_dict.get("sliding_window", -1) + if ( + (sliding_window is None or sliding_window == -1) and FLASH_ATTENTION + ) or HAS_FLASH_ATTN_V2_CUDA: return FlashMistral( model_id, revision, @@ -293,9 +294,10 @@ def get_model( ) if model_type == "mixtral": - if (config_dict["sliding_window"] is None and FLASH_ATTENTION) or ( - config_dict["sliding_window"] > 0 and HAS_FLASH_ATTN_V2_CUDA - ): + sliding_window = config_dict.get("sliding_window", -1) + if ( + (sliding_window is None or sliding_window == -1) and FLASH_ATTENTION + ) or HAS_FLASH_ATTN_V2_CUDA: return FlashMixtral( model_id, revision, diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index c85624f3..0fc4e1b3 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -60,7 +60,7 @@ def __init__( pretraining_tp=1, tie_word_embeddings=False, rope_theta=10000.0, - sliding_window=4096, + sliding_window=None, **kwargs, ): self.vocab_size = vocab_size diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py index b468d09b..61488ec4 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py @@ -72,7 +72,7 @@ def __init__( pretraining_tp=1, tie_word_embeddings=False, rope_theta=10000.0, - sliding_window=4096, + sliding_window=None, num_experts_per_tok=2, num_local_experts=8, **kwargs, diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py index cb358672..cec9eafa 100644 --- a/server/text_generation_server/models/model.py +++ b/server/text_generation_server/models/model.py @@ -33,7 +33,7 @@ def __init__( self.device = device self.rank = rank self.world_size = world_size - self.sliding_window = sliding_window + self.sliding_window = sliding_window if sliding_window != -1 else None if speculate is None: speculate = get_speculate() From 8428ed10112d6d9785231ba64bd38a374c936e89 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 18 Dec 2023 10:20:08 +0100 Subject: [PATCH 507/793] fix: fix offline (#1341) (#1347) @oOraph --------- Signed-off-by: Raphael Glon Co-authored-by: Raphael Glon --- router/src/validation.rs | 7 ++ server/tests/utils/test_hub.py | 59 ++++++++++- server/text_generation_server/utils/hub.py | 112 ++++++++++++++------- 3 files changed, 142 insertions(+), 36 deletions(-) diff --git a/router/src/validation.rs b/router/src/validation.rs index 90dc3741..64f25c82 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -556,6 +556,7 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_p: Some(1.0), + max_new_tokens: Some(5), ..default_parameters() }, }) @@ -570,6 +571,7 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_p: Some(0.99), + max_new_tokens: Some(5), ..default_parameters() }, }) @@ -584,6 +586,7 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_p: None, + max_new_tokens: Some(5), ..default_parameters() }, }) @@ -616,6 +619,7 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_n_tokens: Some(5), + max_new_tokens: Some(5), ..default_parameters() }, }) @@ -630,6 +634,7 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_n_tokens: Some(4), + max_new_tokens: Some(5), ..default_parameters() }, }) @@ -641,6 +646,7 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_n_tokens: Some(0), + max_new_tokens: Some(5), ..default_parameters() }, }) @@ -652,6 +658,7 @@ mod tests { inputs: "Hello".to_string(), parameters: GenerateParameters { top_n_tokens: None, + max_new_tokens: Some(5), ..default_parameters() }, }) diff --git a/server/tests/utils/test_hub.py b/server/tests/utils/test_hub.py index fac9a64d..5438c153 100644 --- a/server/tests/utils/test_hub.py +++ b/server/tests/utils/test_hub.py @@ -1,5 +1,13 @@ +import os +import requests +import tempfile + import pytest +import huggingface_hub.constants +from huggingface_hub import hf_api + +import text_generation_server.utils.hub from text_generation_server.utils.hub import ( weight_hub_files, download_weights, @@ -10,6 +18,52 @@ ) +@pytest.fixture() +def offline(): + current_value = text_generation_server.utils.hub.HF_HUB_OFFLINE + text_generation_server.utils.hub.HF_HUB_OFFLINE = True + yield "offline" + text_generation_server.utils.hub.HF_HUB_OFFLINE = current_value + + +@pytest.fixture() +def fresh_cache(): + with tempfile.TemporaryDirectory() as d: + current_value = huggingface_hub.constants.HUGGINGFACE_HUB_CACHE + huggingface_hub.constants.HUGGINGFACE_HUB_CACHE = d + text_generation_server.utils.hub.HUGGINGFACE_HUB_CACHE = d + os.environ['HUGGINGFACE_HUB_CACHE'] = d + yield + huggingface_hub.constants.HUGGINGFACE_HUB_CACHE = current_value + os.environ['HUGGINGFACE_HUB_CACHE'] = current_value + text_generation_server.utils.hub.HUGGINGFACE_HUB_CACHE = current_value + + +@pytest.fixture() +def prefetched(): + model_id = "bert-base-uncased" + huggingface_hub.snapshot_download( + repo_id=model_id, + revision="main", + local_files_only=False, + repo_type="model", + allow_patterns=["*.safetensors"] + ) + yield model_id + + +def test_weight_hub_files_offline_error(offline, fresh_cache): + # If the model is not prefetched then it will raise an error + with pytest.raises(EntryNotFoundError): + weight_hub_files("gpt2") + + +def test_weight_hub_files_offline_ok(prefetched, offline): + # If the model is prefetched then we should be able to get the weight files from local cache + filenames = weight_hub_files(prefetched) + assert filenames == ['model.safetensors'] + + def test_weight_hub_files(): filenames = weight_hub_files("bigscience/bloom-560m") assert filenames == ["model.safetensors"] @@ -33,8 +87,11 @@ def test_download_weights(): assert files == local_files -def test_weight_files_error(): +def test_weight_files_revision_error(): with pytest.raises(RevisionNotFoundError): weight_files("bigscience/bloom-560m", revision="error") + + +def test_weight_files_not_cached_error(fresh_cache): with pytest.raises(LocalEntryNotFoundError): weight_files("bert-base-uncased") diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index 23743c9b..019d4855 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -6,24 +6,29 @@ from pathlib import Path from typing import Optional, List -from huggingface_hub import HfApi, hf_hub_download +from huggingface_hub import file_download, hf_api, HfApi, hf_hub_download from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE from huggingface_hub.utils import ( LocalEntryNotFoundError, EntryNotFoundError, - RevisionNotFoundError, # Import here to ease try/except in other part of the lib + RevisionNotFoundError, # noqa # Import here to ease try/except in other part of the lib ) WEIGHTS_CACHE_OVERRIDE = os.getenv("WEIGHTS_CACHE_OVERRIDE", None) +HF_HUB_OFFLINE = os.environ.get("HF_HUB_OFFLINE", "0").lower() in ["true", "1", "yes"] -def weight_hub_files( - model_id: str, revision: Optional[str] = None, extension: str = ".safetensors" -) -> List[str]: - """Get the weights filenames on the hub""" - api = HfApi() - info = api.model_info(model_id, revision=revision) - filenames = [ +def _cached_weight_files(model_id: str, revision: Optional[str], extension: str) -> List[str]: + """Guess weight files from the cached revision snapshot directory""" + d = _get_cached_revision_directory(model_id, revision) + if not d: + return [] + filenames = _weight_files_from_dir(d, extension) + return filenames + + +def _weight_hub_files_from_model_info(info: hf_api.ModelInfo, extension: str) -> List[str]: + return [ s.rfilename for s in info.siblings if s.rfilename.endswith(extension) @@ -33,24 +38,26 @@ def weight_hub_files( and "training" not in s.rfilename ] - if not filenames: - raise EntryNotFoundError( - f"No {extension} weights found for model {model_id} and revision {revision}.", - None, - ) +def _weight_files_from_dir(d: Path, extension: str) -> List[str]: + # os.walk: do not iterate, just scan for depth 1, not recursively + # see _weight_hub_files_from_model_info, that's also what is + # done there with the len(s.rfilename.split("/")) == 1 condition + root, _, files = next(os.walk(str(d))) + filenames = [f for f in files + if f.endswith(extension) + and "arguments" not in f + and "args" not in f + and "training" not in f] return filenames -def try_to_load_from_cache( - model_id: str, revision: Optional[str], filename: str -) -> Optional[Path]: - """Try to load a file from the Hugging Face cache""" +def _get_cached_revision_directory(model_id: str, revision: Optional[str]) -> Optional[Path]: if revision is None: revision = "main" - object_id = model_id.replace("/", "--") - repo_cache = Path(HUGGINGFACE_HUB_CACHE) / f"models--{object_id}" + repo_cache = Path(HUGGINGFACE_HUB_CACHE) / Path( + file_download.repo_folder_name(repo_id=model_id, repo_type="model")) if not repo_cache.is_dir(): # No cache for this model @@ -74,8 +81,42 @@ def try_to_load_from_cache( # No cache for this revision and we won't try to return a random revision return None + return snapshots_dir / revision + + +def weight_hub_files( + model_id: str, revision: Optional[str] = None, extension: str = ".safetensors" +) -> List[str]: + """Get the weights filenames on the hub""" + api = HfApi() + + if HF_HUB_OFFLINE: + filenames = _cached_weight_files(model_id, revision, extension) + else: + # Online case, fetch model info from the Hub + info = api.model_info(model_id, revision=revision) + filenames = _weight_hub_files_from_model_info(info, extension) + + if not filenames: + raise EntryNotFoundError( + f"No {extension} weights found for model {model_id} and revision {revision}.", + None, + ) + + return filenames + + +def try_to_load_from_cache( + model_id: str, revision: Optional[str], filename: str +) -> Optional[Path]: + """Try to load a file from the Hugging Face cache""" + + d = _get_cached_revision_directory(model_id, revision) + if not d: + return None + # Check if file exists in cache - cached_file = snapshots_dir / revision / filename + cached_file = d / filename return cached_file if cached_file.is_file() else None @@ -84,13 +125,14 @@ def weight_files( ) -> List[Path]: """Get the local files""" # Local model - if Path(model_id).exists() and Path(model_id).is_dir(): - local_files = list(Path(model_id).glob(f"*{extension}")) + d = Path(model_id) + if d.exists() and d.is_dir(): + local_files = _weight_files_from_dir(d, extension) if not local_files: raise FileNotFoundError( f"No local weights found in {model_id} with extension {extension}" ) - return local_files + return [Path(f) for f in local_files] try: filenames = weight_hub_files(model_id, revision, extension) @@ -138,33 +180,33 @@ def download_weights( ) -> List[Path]: """Download the safetensors files from the hub""" - def download_file(filename, tries=5, backoff: int = 5): - local_file = try_to_load_from_cache(model_id, revision, filename) + def download_file(fname, tries=5, backoff: int = 5): + local_file = try_to_load_from_cache(model_id, revision, fname) if local_file is not None: - logger.info(f"File {filename} already present in cache.") + logger.info(f"File {fname} already present in cache.") return Path(local_file) - for i in range(tries): + for idx in range(tries): try: - logger.info(f"Download file: {filename}") - start_time = time.time() + logger.info(f"Download file: {fname}") + stime = time.time() local_file = hf_hub_download( - filename=filename, + filename=fname, repo_id=model_id, revision=revision, - local_files_only=False, + local_files_only=HF_HUB_OFFLINE, ) logger.info( - f"Downloaded {local_file} in {timedelta(seconds=int(time.time() - start_time))}." + f"Downloaded {local_file} in {timedelta(seconds=int(time.time() - stime))}." ) return Path(local_file) except Exception as e: - if i + 1 == tries: + if idx + 1 == tries: raise e logger.error(e) logger.info(f"Retrying in {backoff} seconds") time.sleep(backoff) - logger.info(f"Retry {i + 1}/{tries - 1}") + logger.info(f"Retry {idx + 1}/{tries - 1}") # We do this instead of using tqdm because we want to parse the logs with the launcher start_time = time.time() From d077150eb7b9884f33e60abe99fe6359bf6e3e64 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 18 Dec 2023 16:07:05 +0100 Subject: [PATCH 508/793] fix: fix gpt-q with groupsize = -1 (#1358) --- proto/generate.proto | 3 ++ router/client/src/client.rs | 8 +++- server/text_generation_server/server.py | 41 +++++++++++-------- .../utils/gptq/exllama.py | 11 +---- .../utils/gptq/exllamav2.py | 13 +----- .../text_generation_server/utils/weights.py | 12 +++--- 6 files changed, 43 insertions(+), 45 deletions(-) diff --git a/proto/generate.proto b/proto/generate.proto index 02f3b2e8..1f30df38 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -213,6 +213,9 @@ message DecodeResponse { message WarmupRequest { /// Batch to warmup on Batch batch = 1; + uint32 max_input_length = 2; + uint32 max_prefill_tokens = 3; + uint32 max_total_tokens = 4; } /// Empty response diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 898e2b11..4723d664 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -145,7 +145,13 @@ impl Client { max_tokens: 0, }; - let request = tonic::Request::new(WarmupRequest { batch: Some(batch) }).inject_context(); + let request = tonic::Request::new(WarmupRequest { + batch: Some(batch), + max_input_length, + max_prefill_tokens, + max_total_tokens, + }) + .inject_context(); let response = self.stub.warmup(request).await?.into_inner(); Ok(response.max_supported_total_tokens) } diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index a65138c9..d5adbd32 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -19,9 +19,16 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): - def __init__(self, model: Model, cache: Cache, server_urls: List[str]): + def __init__( + self, + model: Model, + cache: Cache, + quantize: Optional[str], + server_urls: List[str], + ): self.cache = cache self.model = model + self.quantize = quantize self.server_urls = server_urls # For some reason, inference_mode does not work well with GLOO which we use on CPU if model.device.type == "cuda": @@ -56,6 +63,21 @@ async def FilterBatch(self, request, context): return generate_pb2.FilterBatchResponse(batch=filtered_batch.to_pb()) async def Warmup(self, request, context): + if self.quantize == "gptq": + try: + # When using GPTQ, Exllama kernels need some global kernels + # For which we have the finale shapes only after the model has loaded + # This will allocate those buffers. + from text_generation_server.utils.layers import ( + create_exllama_buffers, + set_device, + ) + + set_device(self.model.device) + create_exllama_buffers(request.max_prefill_tokens) + except ImportError: + pass + if ( self.model.batch_type == IdeficsCausalLMBatch ): # Hack, i would rather use kwargs in the `from_pb` call @@ -184,21 +206,6 @@ async def serve_inner( logger.exception("Error when initializing model") raise - if quantize == "gptq": - try: - # When using GPTQ, Exllama kernels need some global kernels - # For which we have the finale shapes only after the model has loaded - # This will allocate those buffers. - from text_generation_server.utils.layers import ( - create_exllama_buffers, - set_device, - ) - - set_device(model.device) - create_exllama_buffers() - except ImportError: - pass - server = aio.server( interceptors=[ ExceptionInterceptor(), @@ -206,7 +213,7 @@ async def serve_inner( ] ) generate_pb2_grpc.add_TextGenerationServiceServicer_to_server( - TextGenerationService(model, Cache(), server_urls), server + TextGenerationService(model, Cache(), quantize, server_urls), server ) SERVICE_NAMES = ( generate_pb2.DESCRIPTOR.services_by_name["TextGenerationService"].full_name, diff --git a/server/text_generation_server/utils/gptq/exllama.py b/server/text_generation_server/utils/gptq/exllama.py index 7353afb5..32f817db 100644 --- a/server/text_generation_server/utils/gptq/exllama.py +++ b/server/text_generation_server/utils/gptq/exllama.py @@ -37,19 +37,12 @@ def set_device(device): DEVICE = device -def create_exllama_buffers(): +def create_exllama_buffers(max_total_tokens: int): global MAX_DQ, MAX_INNER, ACT_ORDER, DEVICE, TEMP_STATE, TEMP_DQ assert DEVICE is not None, "call set_device first" - if ACT_ORDER: - # TODO: this should be set to rust side `max_total_tokens`, but TGI - # does not offer an API to expose this variable to python, as this variable - # is handled by the client but it appears the model is initialized by the server. - # An alternative could be to initialize the buffers during warmup. - # Dummy - max_total_tokens = 2048 - else: + if not ACT_ORDER: max_total_tokens = 1 # This temp_state buffer is required to reorder X in the act-order case. diff --git a/server/text_generation_server/utils/gptq/exllamav2.py b/server/text_generation_server/utils/gptq/exllamav2.py index f820f0d9..dd41b269 100644 --- a/server/text_generation_server/utils/gptq/exllamav2.py +++ b/server/text_generation_server/utils/gptq/exllamav2.py @@ -101,7 +101,7 @@ def set_device(device): DEVICE = device -def create_exllama_buffers(): +def create_exllama_buffers(max_total_tokens: int): global FIXED_BYTES, LAYERS, DEVICE temp_dq = ExLlamaV2DeviceTensors(DEVICE, FIXED_BYTES) @@ -138,17 +138,6 @@ def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize): self.bias = bias if bias is not None else None self.group_size = groupsize - infeatures = self.infeatures - outfeatures = self.outfeatures - assert qweight.shape == (infeatures // 32 * self.bits, outfeatures) - assert infeatures % self.group_size == 0 - assert qzeros.shape == ( - infeatures // self.group_size, - outfeatures // 32 * self.bits, - ) - assert scales.shape == (infeatures // self.group_size, outfeatures) - assert g_idx.shape == (infeatures,), f"{g_idx.shape}, {infeatures}" - global FIXED_BYTES, LAYERS FIXED_BYTES = max(FIXED_BYTES, self.scratch_space_fixed()) LAYERS.append(self) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 67fda511..a2cca2ea 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -281,17 +281,17 @@ def get_multi_weights_row(self, prefix: str, quantize: str): else: logger.info(f"Using exllama kernels v{HAS_EXLLAMA}") - if use_exllama: + if use_exllama and groupsize != -1: qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) scales = self.get_sharded(f"{prefix}.scales", dim=0) - g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) - g_idx = g_idx - g_idx[0] else: - # The triton kernel reorders the scales/zero points instead of the weight/activation. - # Thus, each rank needs the full qzeros/scales. qzeros = self.get_tensor(f"{prefix}.qzeros") scales = self.get_tensor(f"{prefix}.scales") - g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) + + g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) + + if use_exllama: + g_idx = g_idx - g_idx[0] weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) elif quantize == "awq": From eb8923a97e5dbc3263699b0f144148fb23cd4988 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 20 Dec 2023 15:37:14 +0100 Subject: [PATCH 509/793] Peft safetensors. (#1364) Works by removing adapter_model.safetensors from being detected as the core model file (which skips the real peft detection). # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/utils/hub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index 019d4855..62afff0c 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -48,6 +48,7 @@ def _weight_files_from_dir(d: Path, extension: str) -> List[str]: if f.endswith(extension) and "arguments" not in f and "args" not in f + and "adapter" not in f and "training" not in f] return filenames From 987c959f73f088885ffae9303b7e385082dcd198 Mon Sep 17 00:00:00 2001 From: regisss <15324346+regisss@users.noreply.github.com> Date: Thu, 21 Dec 2023 11:05:35 +0100 Subject: [PATCH 510/793] docs: Change URL for Habana Gaudi support in doc (#1343) --- docs/source/supported_models.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md index 34775139..0708c729 100644 --- a/docs/source/supported_models.md +++ b/docs/source/supported_models.md @@ -39,7 +39,7 @@ text-generation-launcher --model-id ## Supported Hardware -TGI optimized models are supported on NVIDIA [A100](https://www.nvidia.com/en-us/data-center/a100/), [A10G](https://www.nvidia.com/en-us/data-center/products/a10-gpu/) and [T4](https://www.nvidia.com/en-us/data-center/tesla-t4/) GPUs with CUDA 11.8+. Note that you have to install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) to use it. For other NVIDIA GPUs, continuous batching will still apply, but some operations like flash attention and paged attention will not be executed. +TGI optimized models are supported on NVIDIA [A100](https://www.nvidia.com/en-us/data-center/a100/), [A10G](https://www.nvidia.com/en-us/data-center/products/a10-gpu/) and [T4](https://www.nvidia.com/en-us/data-center/tesla-t4/) GPUs with CUDA 11.8+. Note that you have to install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) to use it. For other NVIDIA GPUs, continuous batching will still apply, but some operations like flash attention and paged attention will not be executed. TGI also has support of ROCm-enabled AMD Instinct MI210 and MI250 GPUs, with paged attention and flash attention v2 support. The following features are currently not supported in the ROCm version of TGI, and the supported may be extended in the future: * Quantization (GPTQ, AWQ, etc.) @@ -47,5 +47,5 @@ TGI also has support of ROCm-enabled AMD Instinct MI210 and MI250 GPUs, with pag * Kernel for slinding window attention (Mistral) TGI is also supported on the following AI hardware accelerators: -- *Habana first-gen Gaudi and Gaudi2:* check out this [example](https://github.com/huggingface/optimum-habana/tree/main/text-generation-inference) how to serve models with TGI on Gaudi and Gaudi2 with [Optimum Habana](https://huggingface.co/docs/optimum/habana/index) +- *Habana first-gen Gaudi and Gaudi2:* check out this [repository](https://github.com/huggingface/tgi-gaudi) to serve models with TGI on Gaudi and Gaudi2 with [Optimum Habana](https://huggingface.co/docs/optimum/habana/index) * *AWS Inferentia2:* check out this [guide](https://github.com/huggingface/optimum-neuron/tree/main/text-generation-inference) on how to serve models with TGI on Inferentia2. From 564199bab3d5151bb9b02c0a6fb08f4e1d58ff1c Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 21 Dec 2023 17:25:22 +0100 Subject: [PATCH 511/793] feat: update exllamav2 kernels (#1370) Co-authored-by: Nicolas Patry --- .../exllamav2_kernels/config.h | 2 + .../exllamav2_kernels/cuda/q_gemm.cu | 98 +++--- .../exllamav2_kernels/cuda/q_gemm.cuh | 5 +- .../exllamav2_kernels/cuda/q_gemm_kernel.cuh | 317 +++++++++++------- .../cuda/q_gemm_kernel_gptq.cuh | 142 +++++--- .../exllamav2_kernels/cuda/q_matrix.cu | 70 ++-- .../exllamav2_kernels/cuda/q_matrix.cuh | 4 +- .../exllamav2_kernels/cuda/quant/qdq_util.cuh | 2 + .../exllamav2_kernels/cuda/util.cuh | 12 + .../exllamav2_kernels/ext.cpp | 5 + server/tests/utils/test_hub.py | 8 +- .../flash_santacoder_modeling.py | 2 +- .../utils/gptq/exllamav2.py | 33 ++ server/text_generation_server/utils/hub.py | 32 +- server/text_generation_server/utils/layers.py | 6 +- server/text_generation_server/utils/log.py | 6 + .../text_generation_server/utils/weights.py | 36 +- 17 files changed, 525 insertions(+), 255 deletions(-) create mode 100644 server/text_generation_server/utils/log.py diff --git a/server/exllamav2_kernels/exllamav2_kernels/config.h b/server/exllamav2_kernels/exllamav2_kernels/config.h index 86baaf41..32a1a37d 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/config.h +++ b/server/exllamav2_kernels/exllamav2_kernels/config.h @@ -2,6 +2,7 @@ #define _config_h #define MAX_Q_GEMM_ROWS 50 +#define MAX_Q_GEMM_WEIGHTS 4 // must be <= MAX_Q_GEMM_ROWS #define QMODE_2BIT 1 #define QMODE_3BIT 1 @@ -10,4 +11,5 @@ #define QMODE_6BIT 0 #define QMODE_8BIT 0 + #endif diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu index 351b9cd5..b4e4cf22 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu @@ -10,16 +10,19 @@ #include "quant/qdq_6.cuh" #include "quant/qdq_8.cuh" -#define BLOCK_KN_SIZE 128 -#define BLOCK_M_SIZE_MAX 8 -#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32) +#define GPTQ_BLOCK_KN_SIZE 128 +#define GPTQ_BLOCK_M_SIZE_MAX 8 +#define GPTQ_MAX_GROUPS_IN_BLOCK (GPTQ_BLOCK_KN_SIZE / 32) + +#define EXL2_BLOCK_KN_SIZE 64 +#define EXL2_BLOCK_M_SIZE_MAX 8 +#define EXL2_MAX_GROUPS_IN_BLOCK (EXL2_BLOCK_KN_SIZE / 32) + #define CLEAR_N_SIZE 256 #include "q_gemm_kernel.cuh" #include "q_gemm_kernel_gptq.cuh" -#include "compat_gemm.cuh" - void gemm_half_q_half_cuda_part ( const half* a, @@ -29,20 +32,23 @@ void gemm_half_q_half_cuda_part int size_n, int size_k, int m_count, - bool clear + bool clear, + const half* r_weights, + int r_weights_stride, + bool mul_r_weights ) { if (!b->is_gptq) { dim3 blockDim, gridDim; - blockDim.x = BLOCK_KN_SIZE; + blockDim.x = EXL2_BLOCK_KN_SIZE; blockDim.y = 1; blockDim.z = 1; - gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4); + gridDim.x = DIVIDE(size_n, EXL2_BLOCK_KN_SIZE * 4); gridDim.y = DIVIDE(size_m, m_count); - gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE); + gridDim.z = DIVIDE(size_k, EXL2_BLOCK_KN_SIZE); - fp_gemm_half_q_half_kernel kernel = pick_gemm_half_q_half_kernel(true, m_count); + fp_gemm_half_q_half_kernel kernel = pick_gemm_half_q_half_kernel(m_count, r_weights != NULL, mul_r_weights); kernel<<>> ( @@ -55,7 +61,7 @@ void gemm_half_q_half_cuda_part size_n, size_k, b->groups, - b->groupsize, + b->cuda_q_group_map, b->cuda_q_perm, b->rows_8, b->rows_6, @@ -63,24 +69,27 @@ void gemm_half_q_half_cuda_part b->rows_4, b->rows_3, b->rows_2, - clear + clear, + r_weights, + r_weights_stride ); } else { dim3 blockDim, gridDim; - blockDim.x = BLOCK_KN_SIZE; + blockDim.x = GPTQ_BLOCK_KN_SIZE; blockDim.y = 1; blockDim.z = 1; - gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4); + gridDim.x = DIVIDE(size_n, GPTQ_BLOCK_KN_SIZE * 4); gridDim.y = DIVIDE(size_m, m_count); - gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE); + gridDim.z = DIVIDE(size_k, GPTQ_BLOCK_KN_SIZE); - fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count); + fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(m_count, r_weights != NULL, mul_r_weights); -// DBGX((uint64_t) b->cuda_q_perm); -// DBGI(b->rows_4); -// DBGI(b->height); +// DBGX((uint64_t) r_weights); +// if (r_weights) +// print_global_mem(r_weights, 1, 1, 1); +// DBGI(r_weights_stride); kernel<<>> ( @@ -93,10 +102,12 @@ void gemm_half_q_half_cuda_part size_n, size_k, b->groups, - b->groupsize, + b->gptq_groupsize, b->cuda_q_perm, b->rows_4, - clear + clear, + r_weights, + r_weights_stride ); } } @@ -112,13 +123,14 @@ void gemm_half_q_half_cuda int size_k, bool clear, half* temp_dq, - bool force_cuda + bool force_cuda, + const half* r_weights, + const int r_weights_stride, + bool mul_r_weights ) { if (size_m > MAX_Q_GEMM_ROWS && !force_cuda) { - //printf("cublas\n"); - // Reconstruct FP16 matrix, then cuBLAS if (!temp_dq) temp_dq = b->temp_dq; @@ -139,12 +151,12 @@ void gemm_half_q_half_cuda //const float alpha = 1.0f; //const float beta = clear ? 0.0f : 1.0f; //cublasSgemmEx(cublas_handle, - // CUBLAS_OP_N, - // CUBLAS_OP_N, - // size_n, size_m, size_k, - // &alpha, temp_dq, CUDA_R_16F, size_n, - // a, CUDA_R_16F, size_k, - // &beta, c, CUDA_R_16F, size_n); + // CUBLAS_OP_N, + // CUBLAS_OP_N, + // size_n, size_m, size_k, + // &alpha, temp_dq, CUDA_R_16F, size_n, + // a, CUDA_R_16F, size_k, + // &beta, c, CUDA_R_16F, size_n); //const float alpha = 1.0f; //const float beta = clear ? 0.0f : 1.0f; @@ -158,24 +170,21 @@ void gemm_half_q_half_cuda } else { - //printf("cuda\n"); - // Quantized matmul - //if (clear) clear_tensor_cuda(c, size_m, size_n); - - int max_chunks = size_m / BLOCK_M_SIZE_MAX; - int last_chunk = max_chunks * BLOCK_M_SIZE_MAX; + int block_m_size_max = b->is_gptq ? GPTQ_BLOCK_M_SIZE_MAX : EXL2_BLOCK_M_SIZE_MAX; + int max_chunks = size_m / block_m_size_max; + int last_chunk = max_chunks * block_m_size_max; int last_chunk_size = size_m - last_chunk; if (max_chunks) { - gemm_half_q_half_cuda_part(a, b, c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX, clear); + gemm_half_q_half_cuda_part(a, b, c, last_chunk, size_n, size_k, block_m_size_max, clear, r_weights, r_weights_stride, mul_r_weights); } if (last_chunk_size) { - gemm_half_q_half_cuda_part(a + last_chunk * size_k, b, c + last_chunk * size_n, last_chunk_size, size_n, size_k, last_chunk_size, clear); + gemm_half_q_half_cuda_part(a + last_chunk * size_k, b, c + last_chunk * size_n, last_chunk_size, size_n, size_k, last_chunk_size, clear, r_weights, r_weights_stride, mul_r_weights); } } } @@ -201,11 +210,10 @@ void clear_tensor_cuda int size_n ) { - return; - dim3 blockDim, gridDim; - blockDim.x = CLEAR_N_SIZE; - blockDim.y = 1; - gridDim.x = DIVIDE(size_n / 8, CLEAR_N_SIZE); - gridDim.y = size_m; - clear_kernel<<>>(c, size_m, size_n); +// dim3 blockDim, gridDim; +// blockDim.x = CLEAR_N_SIZE; +// blockDim.y = 1; +// gridDim.x = DIVIDE(size_n / 8, CLEAR_N_SIZE); +// gridDim.y = size_m; +// clear_kernel<<>>(c, size_m, size_n); } diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cuh index c69f1a70..b643f915 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cuh +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cuh @@ -20,7 +20,10 @@ void gemm_half_q_half_cuda int size_k, bool clear = false, half* reconstruct = NULL, - bool force_cuda = false + bool force_cuda = false, + const half* r_weights = NULL, + const int r_weights_stride = 0, + bool mul_r_weights = false ); void clear_tensor_cuda diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel.cuh index 0b899a84..9cd2ba01 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel.cuh +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel.cuh @@ -1,8 +1,5 @@ #include "compat.cuh" -#include -#include - __forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h) { half2 result = {}; @@ -60,6 +57,47 @@ __forceinline__ __device__ float dot22_32_f(half2(&dq)[16], const half* a_ptr, c return fma(result_f, qs_f, g_result); } +__forceinline__ __device__ half dot22_8_h(half2(&dq)[4], const half* a_ptr, const half g_result, const half qs_h) +{ + // Use FP32 accumulator to avoid potential overflow since unscaled weights are in the range -128..127 + + float result = {}; + #pragma unroll + for (int i = 0; i < 4; i++) + { + half2 w01 = dq[i]; + float w0 = __low2float(w01); + float w1 = __high2float(w01); + float x0 = __half2float(*a_ptr++); + float x1 = __half2float(*a_ptr++); + result = fma(w0, x0, result); + result = fma(w1, x1, result); + } + float qs = __half2float(qs_h); + result *= qs; + half result_h = __float2half_rn(result); + return __hadd(result_h, g_result); +} + +__forceinline__ __device__ half dot22_16_h(half2(&dq)[8], const half* a_ptr, const half g_result, const half qs_h) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result); + half result_h = __hadd(__low2half(result), __high2half(result)); + return __hfma(result_h, qs_h, g_result); +} + +__forceinline__ __device__ half dot22_32_h(half2(&dq)[16], const half* a_ptr, const half g_result, const half qs_h) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result); + half result_h = __hadd(__low2half(result), __high2half(result)); + return __hfma(result_h, qs_h, g_result); +} typedef void (*fp_gemm_half_q_half_kernel) @@ -73,7 +111,7 @@ typedef void (*fp_gemm_half_q_half_kernel) const int, const int, const int, - const int, + const uint16_t*, const uint16_t*, const int, const int, @@ -81,10 +119,12 @@ typedef void (*fp_gemm_half_q_half_kernel) const int, const int, const int, - const bool + const bool, + const half*, + const int ); -template +template __global__ void gemm_half_q_half_kernel ( const half* __restrict__ a, @@ -96,7 +136,7 @@ __global__ void gemm_half_q_half_kernel const int size_n, const int size_k, const int groups, - const int groupsize, + const uint16_t* __restrict__ b_q_group_map, const uint16_t* __restrict__ b_q_perm, const int rows_8, const int rows_6, @@ -104,7 +144,9 @@ __global__ void gemm_half_q_half_kernel const int rows_4, const int rows_3, const int rows_2, - const bool clear + const bool clear, + const half* r_weights, + const int r_weights_stride ) { MatrixView_half a_(a, size_m, size_k); @@ -115,18 +157,34 @@ __global__ void gemm_half_q_half_kernel // Block - int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + int offset_n = blockIdx.x * EXL2_BLOCK_KN_SIZE * 4; int offset_m = blockIdx.y * m_count; - int offset_k = blockIdx.z * BLOCK_KN_SIZE; + int offset_k = blockIdx.z * EXL2_BLOCK_KN_SIZE; - int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n); + int end_n = min(offset_n + EXL2_BLOCK_KN_SIZE * 4, size_n); int end_m = min(offset_m + m_count, size_m); - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + int end_k = min(offset_k + EXL2_BLOCK_KN_SIZE, size_k); int n = offset_n + t * 4; + // Read weights + + half_uint16 weights[MAX_Q_GEMM_WEIGHTS]; + if constexpr (use_r_weights) + { + uint16_t any_w = 0; + const half* w_ptr = r_weights; + for (int m = 0; m < m_count; ++m) + { + weights[m].as_half = *w_ptr; + w_ptr += r_weights_stride; + any_w |= weights[m].as_uint16; + } + if (!any_w) return; // Early exit if all weights are zero -- does not zero output (!!!) + } + // Preload block_a - __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + __shared__ half block_a[m_count][EXL2_BLOCK_KN_SIZE]; if (offset_k + t < end_k) { @@ -135,6 +193,7 @@ __global__ void gemm_half_q_half_kernel const half* a_ptr = a_.item_ptr(offset_m + m, 0); half* block_a_ptr = block_a[m]; half a0 = a_ptr[b_q_perm[offset_k + t]]; +// half a0 = a_ptr[offset_k + t]; block_a_ptr[t] = a0; } } @@ -153,14 +212,19 @@ __global__ void gemm_half_q_half_kernel // Find initial group - int group = offset_k / groupsize; + //int group = offset_k / groupsize; + int group = b_q_group_map[offset_k * 2]; + +// if (offset_m == 0 && t == 0) +// DBGI2(offset_k, group); // Preload scales - float scales[MAX_GROUPS_IN_BLOCK][4]; + half scales[EXL2_MAX_GROUPS_IN_BLOCK][4]; - int groups_in_block = DIVIDE((end_k - offset_k), groupsize); - for (int g = 0; g < groups_in_block; g++) + //int groups_in_block = DIVIDE((end_k - offset_k), groupsize); + int temp_k = offset_k; + for (int g = 0; temp_k < end_k; g++) { int qscales[4]; b_q_scale_.item4(qscales, group + g, n); @@ -168,11 +232,12 @@ __global__ void gemm_half_q_half_kernel qscales[1]++; qscales[2]++; qscales[3]++; - float maxscale = __half2float(b_q_scale_max[group + g]); - scales[g][0] = __int2float_rn(qscales[0] * qscales[0]) * maxscale; - scales[g][1] = __int2float_rn(qscales[1] * qscales[1]) * maxscale; - scales[g][2] = __int2float_rn(qscales[2] * qscales[2]) * maxscale; - scales[g][3] = __int2float_rn(qscales[3] * qscales[3]) * maxscale; + half maxscale = b_q_scale_max[group + g]; + scales[g][0] = __hmul(__int2half_rn(qscales[0] * qscales[0]), maxscale); + scales[g][1] = __hmul(__int2half_rn(qscales[1] * qscales[1]), maxscale); + scales[g][2] = __hmul(__int2half_rn(qscales[2] * qscales[2]), maxscale); + scales[g][3] = __hmul(__int2half_rn(qscales[3] * qscales[3]), maxscale); + temp_k += b_q_group_map[temp_k * 2 + 1]; } // a, b offset @@ -193,20 +258,20 @@ __global__ void gemm_half_q_half_kernel const uint32_t* b_ptr = b_q_weight + qk * size_n + n; const half* a_ptr = &block_a[0][0]; - int a_stride = BLOCK_KN_SIZE; + int a_stride = EXL2_BLOCK_KN_SIZE; // Initial group int scales_idx = 0; - float qs_f0 = scales[scales_idx][0]; - float qs_f1 = scales[scales_idx][1]; - float qs_f2 = scales[scales_idx][2]; - float qs_f3 = scales[scales_idx][3]; - int nextgroup = offset_k + groupsize; + half qs_h0 = scales[scales_idx][0]; + half qs_h1 = scales[scales_idx][1]; + half qs_h2 = scales[scales_idx][2]; + half qs_h3 = scales[scales_idx][3]; + int nextgroup = offset_k + b_q_group_map[offset_k * 2 + 1]; // Column result - float block_c[m_count][4] = {}; + half block_c[m_count][4] = {}; // Dequantize groups @@ -218,11 +283,11 @@ __global__ void gemm_half_q_half_kernel { group++; scales_idx++; - qs_f0 = scales[scales_idx][0]; - qs_f1 = scales[scales_idx][1]; - qs_f2 = scales[scales_idx][2]; - qs_f3 = scales[scales_idx][3]; - nextgroup += groupsize; + qs_h0 = scales[scales_idx][0]; + qs_h1 = scales[scales_idx][1]; + qs_h2 = scales[scales_idx][2]; + qs_h3 = scales[scales_idx][3]; + nextgroup += b_q_group_map[k * 2 + 1]; } #pragma unroll @@ -240,10 +305,11 @@ __global__ void gemm_half_q_half_kernel for (int m = 0; m < m_count; m++) { - block_c[m][0] = dot22_8_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); - block_c[m][1] = dot22_8_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); - block_c[m][2] = dot22_8_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); - block_c[m][3] = dot22_8_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + if constexpr (use_r_weights) { if (!weights[m].as_uint16) continue; } + block_c[m][0] = dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_h0); + block_c[m][1] = dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_h1); + block_c[m][2] = dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_h2); + block_c[m][3] = dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_h3); } a_ptr += 8; } @@ -256,11 +322,11 @@ __global__ void gemm_half_q_half_kernel { group++; scales_idx++; - qs_f0 = scales[scales_idx][0]; - qs_f1 = scales[scales_idx][1]; - qs_f2 = scales[scales_idx][2]; - qs_f3 = scales[scales_idx][3]; - nextgroup += groupsize; + qs_h0 = scales[scales_idx][0]; + qs_h1 = scales[scales_idx][1]; + qs_h2 = scales[scales_idx][2]; + qs_h3 = scales[scales_idx][3]; + nextgroup += b_q_group_map[k * 2 + 1]; } #pragma unroll @@ -279,10 +345,11 @@ __global__ void gemm_half_q_half_kernel for (int m = 0; m < m_count; m++) { - block_c[m][0] = dot22_16_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); - block_c[m][1] = dot22_16_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); - block_c[m][2] = dot22_16_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); - block_c[m][3] = dot22_16_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + if constexpr (use_r_weights) { if (!weights[m].as_uint16) continue; } + block_c[m][0] = dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_h0); + block_c[m][1] = dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_h1); + block_c[m][2] = dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_h2); + block_c[m][3] = dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_h3); } a_ptr += 16; } @@ -295,11 +362,11 @@ __global__ void gemm_half_q_half_kernel { group++; scales_idx++; - qs_f0 = scales[scales_idx][0]; - qs_f1 = scales[scales_idx][1]; - qs_f2 = scales[scales_idx][2]; - qs_f3 = scales[scales_idx][3]; - nextgroup += groupsize; + qs_h0 = scales[scales_idx][0]; + qs_h1 = scales[scales_idx][1]; + qs_h2 = scales[scales_idx][2]; + qs_h3 = scales[scales_idx][3]; + nextgroup += b_q_group_map[k * 2 + 1]; } #pragma unroll @@ -320,10 +387,11 @@ __global__ void gemm_half_q_half_kernel for (int m = 0; m < m_count; m++) { - block_c[m][0] = dot22_32_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); - block_c[m][1] = dot22_32_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); - block_c[m][2] = dot22_32_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); - block_c[m][3] = dot22_32_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + if constexpr (use_r_weights) { if (!weights[m].as_uint16) continue; } + block_c[m][0] = dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_h0); + block_c[m][1] = dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_h1); + block_c[m][2] = dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_h2); + block_c[m][3] = dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_h3); } a_ptr += 32; } @@ -337,11 +405,11 @@ __global__ void gemm_half_q_half_kernel { group++; scales_idx++; - qs_f0 = scales[scales_idx][0]; - qs_f1 = scales[scales_idx][1]; - qs_f2 = scales[scales_idx][2]; - qs_f3 = scales[scales_idx][3]; - nextgroup += groupsize; + qs_h0 = scales[scales_idx][0]; + qs_h1 = scales[scales_idx][1]; + qs_h2 = scales[scales_idx][2]; + qs_h3 = scales[scales_idx][3]; + nextgroup += b_q_group_map[k * 2 + 1]; } #pragma unroll @@ -358,10 +426,11 @@ __global__ void gemm_half_q_half_kernel for (int m = 0; m < m_count; m++) { - block_c[m][0] = dot22_8_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); - block_c[m][1] = dot22_8_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); - block_c[m][2] = dot22_8_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); - block_c[m][3] = dot22_8_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + if constexpr (use_r_weights) { if (!weights[m].as_uint16) continue; } + block_c[m][0] = dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_h0); + block_c[m][1] = dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_h1); + block_c[m][2] = dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_h2); + block_c[m][3] = dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_h3); } a_ptr += 8; } @@ -374,11 +443,11 @@ __global__ void gemm_half_q_half_kernel { group++; scales_idx++; - qs_f0 = scales[scales_idx][0]; - qs_f1 = scales[scales_idx][1]; - qs_f2 = scales[scales_idx][2]; - qs_f3 = scales[scales_idx][3]; - nextgroup += groupsize; + qs_h0 = scales[scales_idx][0]; + qs_h1 = scales[scales_idx][1]; + qs_h2 = scales[scales_idx][2]; + qs_h3 = scales[scales_idx][3]; + nextgroup += b_q_group_map[k * 2 + 1]; } #pragma unroll @@ -397,10 +466,11 @@ __global__ void gemm_half_q_half_kernel for (int m = 0; m < m_count; m++) { - block_c[m][0] = dot22_32_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); - block_c[m][1] = dot22_32_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); - block_c[m][2] = dot22_32_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); - block_c[m][3] = dot22_32_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + if constexpr (use_r_weights) { if (!weights[m].as_uint16) continue; } + block_c[m][0] = dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_h0); + block_c[m][1] = dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_h1); + block_c[m][2] = dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_h2); + block_c[m][3] = dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_h3); } a_ptr += 32; } @@ -413,15 +483,15 @@ __global__ void gemm_half_q_half_kernel { group++; scales_idx++; - qs_f0 = scales[scales_idx][0]; - qs_f1 = scales[scales_idx][1]; - qs_f2 = scales[scales_idx][2]; - qs_f3 = scales[scales_idx][3]; - nextgroup += groupsize; + qs_h0 = scales[scales_idx][0]; + qs_h1 = scales[scales_idx][1]; + qs_h2 = scales[scales_idx][2]; + qs_h3 = scales[scales_idx][3]; + nextgroup += b_q_group_map[k * 2 + 1]; } #pragma unroll - for (int j = 0; j < 2; j++) + for (int j = 0; j < 1; j++) { int4 load_int4[1]; load_int4[0] = *((int4*) b_ptr); b_ptr += size_n; @@ -434,15 +504,16 @@ __global__ void gemm_half_q_half_kernel for (int m = 0; m < m_count; m++) { - block_c[m][0] = dot22_16_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0); - block_c[m][1] = dot22_16_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1); - block_c[m][2] = dot22_16_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2); - block_c[m][3] = dot22_16_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3); + if constexpr (use_r_weights) { if (!weights[m].as_uint16) continue; } + block_c[m][0] = dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_h0); + block_c[m][1] = dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_h1); + block_c[m][2] = dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_h2); + block_c[m][3] = dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_h3); } a_ptr += 16; } - k += 32; + k += 16; } // Accumulate column sums in c @@ -450,38 +521,60 @@ __global__ void gemm_half_q_half_kernel for (int m = 0; m < m_count; m++) { half2* out = (half2*)c_.item_ptr(offset_m + m, n); - half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1])); - half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3])); + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]); + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]); + + if constexpr (mul_r_weights) + { + half2 w_mul2 = __half2half2(weights[m].as_half); + result01 = __hmul2(result01, w_mul2); + result23 = __hmul2(result23, w_mul2); + } + atomicAdd(out , result01); atomicAdd(out + 1, result23); +// *out = result01; +// *(out + 1) = result23; } } -fp_gemm_half_q_half_kernel pick_gemm_half_q_half_kernel(bool first_block, const int m_count) +template +struct map_m_count_exl2 { + static constexpr fp_gemm_half_q_half_kernel pick_gemm_half_q_half_kernel(const int m_count) + { + #if EXL2_BLOCK_M_SIZE_MAX >= 1 + if (m_count == 1) return gemm_half_q_half_kernel<1, use_r_weights, mul_r_weights>; + #endif + #if EXL2_BLOCK_M_SIZE_MAX >= 2 + if (m_count == 2) return gemm_half_q_half_kernel<2, use_r_weights, mul_r_weights>; + #endif + #if EXL2_BLOCK_M_SIZE_MAX >= 3 + if (m_count == 3) return gemm_half_q_half_kernel<3, use_r_weights, mul_r_weights>; + #endif + #if EXL2_BLOCK_M_SIZE_MAX >= 4 + if (m_count == 4) return gemm_half_q_half_kernel<4, use_r_weights, mul_r_weights>; + #endif + #if EXL2_BLOCK_M_SIZE_MAX >= 5 + if (m_count == 5) return gemm_half_q_half_kernel<5, use_r_weights, mul_r_weights>; + #endif + #if EXL2_BLOCK_M_SIZE_MAX >= 6 + if (m_count == 6) return gemm_half_q_half_kernel<6, use_r_weights, mul_r_weights>; + #endif + #if EXL2_BLOCK_M_SIZE_MAX >= 7 + if (m_count == 7) return gemm_half_q_half_kernel<7, use_r_weights, mul_r_weights>; + #endif + #if EXL2_BLOCK_M_SIZE_MAX >= 8 + if (m_count == 8) return gemm_half_q_half_kernel<8, use_r_weights, mul_r_weights>; + #endif + return NULL; + } +}; + +fp_gemm_half_q_half_kernel pick_gemm_half_q_half_kernel(const int m_count, bool r_weights, bool mul_r_weights) { - #if BLOCK_M_SIZE_MAX >= 1 - if (m_count == 1) return gemm_half_q_half_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 2 - if (m_count == 2) return gemm_half_q_half_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 3 - if (m_count == 3) return gemm_half_q_half_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 4 - if (m_count == 4) return gemm_half_q_half_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 5 - if (m_count == 5) return gemm_half_q_half_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 6 - if (m_count == 6) return gemm_half_q_half_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 7 - if (m_count == 7) return gemm_half_q_half_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 8 - if (m_count == 8) return gemm_half_q_half_kernel; - #endif + if (!r_weights && !mul_r_weights) return map_m_count_exl2::pick_gemm_half_q_half_kernel(m_count); + if (!r_weights && mul_r_weights) return map_m_count_exl2::pick_gemm_half_q_half_kernel(m_count); + if ( r_weights && !mul_r_weights) return map_m_count_exl2< true, false>::pick_gemm_half_q_half_kernel(m_count); + if ( r_weights && mul_r_weights) return map_m_count_exl2< true, true>::pick_gemm_half_q_half_kernel(m_count); return NULL; } diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh index ebaa42d0..74b0db2b 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh @@ -18,6 +18,15 @@ __forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr) return __half2float(__low2half(result)) + __half2float(__high2half(result)); } +__forceinline__ __device__ half2 dot22_8_h2(half2(&dq)[4], const half* a_ptr) +{ + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; + #pragma unroll + for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result); + return result; +} + typedef void (*fp_gemm_half_q_half_gptq_kernel) ( const half*, @@ -32,10 +41,12 @@ typedef void (*fp_gemm_half_q_half_gptq_kernel) const int, const uint16_t*, const int, - const bool + const bool, + const half*, + const int ); -template +template __global__ void gemm_half_q_half_gptq_kernel ( const half* __restrict__ a, @@ -50,7 +61,9 @@ __global__ void gemm_half_q_half_gptq_kernel const int groupsize, const uint16_t* __restrict__ b_q_perm, const int rows_4, - const bool clear + const bool clear, + const half* r_weights, + const int r_weights_stride ) { MatrixView_half a_(a, size_m, size_k); @@ -62,19 +75,35 @@ __global__ void gemm_half_q_half_gptq_kernel // Block - int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + int offset_n = blockIdx.x * GPTQ_BLOCK_KN_SIZE * 4; int offset_m = blockIdx.y * m_count; - int offset_k = blockIdx.z * BLOCK_KN_SIZE; + int offset_k = blockIdx.z * GPTQ_BLOCK_KN_SIZE; - int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n); + int end_n = min(offset_n + GPTQ_BLOCK_KN_SIZE * 4, size_n); int end_m = min(offset_m + m_count, size_m); - int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + int end_k = min(offset_k + GPTQ_BLOCK_KN_SIZE, size_k); int n = offset_n + t * 4; + // Read weights + + half_uint16 weights[MAX_Q_GEMM_WEIGHTS]; + if constexpr (use_r_weights) + { + uint16_t any_w = 0; + const half* w_ptr = r_weights; + for (int m = 0; m < m_count; ++m) + { + weights[m].as_half = *w_ptr; + w_ptr += r_weights_stride; + any_w |= weights[m].as_uint16; + } + if (!any_w) return; // Early exit if all weights are zero -- does not zero output (!!!) + } + // Preload block_a - __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + __shared__ half block_a[m_count][GPTQ_BLOCK_KN_SIZE]; if (offset_k + t < end_k) { @@ -113,16 +142,16 @@ __global__ void gemm_half_q_half_gptq_kernel const uint32_t* b_ptr = b_q_weight + qk * size_n + n; const half* a_ptr = &block_a[0][0]; - int a_stride = BLOCK_KN_SIZE; + int a_stride = GPTQ_BLOCK_KN_SIZE; // Initial group int zeros[4]; - float scales[4]; + half2 scales[4]; half2 z1z16[4][2]; half2 y1y16[4][2]; b_gptq_qzeros_.item4(zeros, group, n); - b_gptq_scales_.item4_f(scales, group, n); + b_gptq_scales_.item4_h2(scales, group, n); dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); @@ -132,7 +161,7 @@ __global__ void gemm_half_q_half_gptq_kernel // Column result - float block_c[m_count][4] = {}; + half2 block_c[m_count][4] = {}; // Dequantize and multiply @@ -144,7 +173,7 @@ __global__ void gemm_half_q_half_gptq_kernel group++; nextgroup += groupsize; b_gptq_qzeros_.item4(zeros, group, n); - b_gptq_scales_.item4_f(scales, group, n); + b_gptq_scales_.item4_h2(scales, group, n); dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); @@ -166,10 +195,11 @@ __global__ void gemm_half_q_half_gptq_kernel #pragma unroll for (int m = 0; m < m_count; m++) { - block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]); - block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]); - block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]); - block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]); + if constexpr (use_r_weights) { if (!weights[m].as_uint16) continue; } + block_c[m][0] = __hfma2(dot22_8_h2(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]); + block_c[m][1] = __hfma2(dot22_8_h2(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]); + block_c[m][2] = __hfma2(dot22_8_h2(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]); + block_c[m][3] = __hfma2(dot22_8_h2(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]); } b_ptr += size_n; @@ -182,38 +212,62 @@ __global__ void gemm_half_q_half_gptq_kernel for (int m = 0; m < m_count; m++) { half2 *out = (half2*) c_.item_ptr(offset_m + m, n); - half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1])); - half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3])); + half result0 = __hadd(__low2half(block_c[m][0]), __high2half(block_c[m][0])); + half result1 = __hadd(__low2half(block_c[m][1]), __high2half(block_c[m][1])); + half result2 = __hadd(__low2half(block_c[m][2]), __high2half(block_c[m][2])); + half result3 = __hadd(__low2half(block_c[m][3]), __high2half(block_c[m][3])); + half2 result01 = __halves2half2(result0, result1); + half2 result23 = __halves2half2(result2, result3); + + if constexpr (mul_r_weights) + { + half2 w_mul2 = __half2half2(weights[m].as_half); + result01 = __hmul2(result01, w_mul2); + result23 = __hmul2(result23, w_mul2); + } + atomicAdd(out , result01); atomicAdd(out + 1, result23); } } -fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count) +template +struct map_m_count_gptq { + static constexpr fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(int m_count) + { + #if GPTQ_BLOCK_M_SIZE_MAX >= 1 + if (m_count == 1) return gemm_half_q_half_gptq_kernel<1, use_r_weights, mul_r_weights>; + #endif + #if GPTQ_BLOCK_M_SIZE_MAX >= 2 + if (m_count == 2) return gemm_half_q_half_gptq_kernel<2, use_r_weights, mul_r_weights>; + #endif + #if GPTQ_BLOCK_M_SIZE_MAX >= 3 + if (m_count == 3) return gemm_half_q_half_gptq_kernel<3, use_r_weights, mul_r_weights>; + #endif + #if GPTQ_BLOCK_M_SIZE_MAX >= 4 + if (m_count == 4) return gemm_half_q_half_gptq_kernel<4, use_r_weights, mul_r_weights>; + #endif + #if GPTQ_BLOCK_M_SIZE_MAX >= 5 + if (m_count == 5) return gemm_half_q_half_gptq_kernel<5, use_r_weights, mul_r_weights>; + #endif + #if GPTQ_BLOCK_M_SIZE_MAX >= 6 + if (m_count == 6) return gemm_half_q_half_gptq_kernel<6, use_r_weights, mul_r_weights>; + #endif + #if GPTQ_BLOCK_M_SIZE_MAX >= 7 + if (m_count == 7) return gemm_half_q_half_gptq_kernel<7, use_r_weights, mul_r_weights>; + #endif + #if GPTQ_BLOCK_M_SIZE_MAX >= 8 + if (m_count == 8) return gemm_half_q_half_gptq_kernel<8, use_r_weights, mul_r_weights>; + #endif + return NULL; + } +}; + +fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(const int m_count, bool r_weights, bool mul_r_weights) { - #if BLOCK_M_SIZE_MAX >= 1 - if (m_count == 1) return gemm_half_q_half_gptq_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 2 - if (m_count == 2) return gemm_half_q_half_gptq_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 3 - if (m_count == 3) return gemm_half_q_half_gptq_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 4 - if (m_count == 4) return gemm_half_q_half_gptq_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 5 - if (m_count == 5) return gemm_half_q_half_gptq_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 6 - if (m_count == 6) return gemm_half_q_half_gptq_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 7 - if (m_count == 7) return gemm_half_q_half_gptq_kernel; - #endif - #if BLOCK_M_SIZE_MAX >= 8 - if (m_count == 8) return gemm_half_q_half_gptq_kernel; - #endif + if (!r_weights && !mul_r_weights) return map_m_count_gptq::pick_gemm_half_q_half_gptq_kernel(m_count); + if (!r_weights && mul_r_weights) return map_m_count_gptq::pick_gemm_half_q_half_gptq_kernel(m_count); + if ( r_weights && !mul_r_weights) return map_m_count_gptq< true, false>::pick_gemm_half_q_half_gptq_kernel(m_count); + if ( r_weights && mul_r_weights) return map_m_count_gptq< true, true>::pick_gemm_half_q_half_gptq_kernel(m_count); return NULL; } diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu index 6aed7470..ae08cc1f 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu @@ -57,6 +57,7 @@ QMatrix::QMatrix uint32_t* _q_scale, half* _q_scale_max, uint16_t* _q_groups, + uint16_t* _q_group_map, uint32_t* _gptq_qzeros, half* _gptq_scales, @@ -80,13 +81,17 @@ QMatrix::QMatrix cuda_q_scale = _q_scale; cuda_q_scale_max = _q_scale_max; cuda_q_groups = _q_groups; + cuda_q_group_map = _q_group_map; cuda_gptq_qzeros = _gptq_qzeros; cuda_gptq_scales = _gptq_scales; is_gptq = (_gptq_qzeros != NULL); - groupsize = 1; - while (groupsize * groups < height) groupsize *= 2; + if (is_gptq) + { + gptq_groupsize = 1; + while (gptq_groupsize * groups < height) gptq_groupsize *= 2; + } // Create group map @@ -102,15 +107,26 @@ QMatrix::QMatrix uint16_t* cpu_q_groups = (uint16_t*)calloc(groups * 2, sizeof(uint16_t)); cudaMemcpy(cpu_q_groups, cuda_q_groups, groups * 2 * sizeof(uint16_t), cudaMemcpyDeviceToHost); + int row = 0; for (int i = 0; i < groups; i++) { int bits = cpu_q_groups[i * 2]; - if (bits == 8) rows_8 += groupsize; - if (bits == 6) rows_6 += groupsize; - if (bits == 5) rows_5 += groupsize; - if (bits == 4) rows_4 += groupsize; - if (bits == 3) rows_3 += groupsize; - if (bits == 2) rows_2 += groupsize; + + int rows; + if (i < groups - 1) + { + int qrows = cpu_q_groups[i * 2 + 3] - cpu_q_groups[i * 2 + 1]; + rows = qrows * 32 / bits; + } + else rows = height - row; + + if (bits == 8) rows_8 += rows; + if (bits == 6) rows_6 += rows; + if (bits == 5) rows_5 += rows; + if (bits == 4) rows_4 += rows; + if (bits == 3) rows_3 += rows; + if (bits == 2) rows_2 += rows; + row += rows; } free(cpu_q_groups); @@ -138,6 +154,13 @@ QMatrix::QMatrix } } +// DBGI(rows_8); +// DBGI(rows_6); +// DBGI(rows_5); +// DBGI(rows_4); +// DBGI(rows_3); +// DBGI(rows_2); + // Shuffle quantized data dim3 blockDim, gridDim; @@ -283,10 +306,10 @@ __global__ void reconstruct_kernel const uint16_t* __restrict__ b_q_perm, const uint32_t* __restrict__ b_q_scale, const half* __restrict__ b_q_scale_max, - //const uint16_t* __restrict__ b_q_groups, + const uint16_t* __restrict__ b_q_group_map, const int size_k, const int size_n, - const int groupsize, + //const int groupsize, const int groups, half* __restrict__ b, const int rows_8, @@ -317,7 +340,8 @@ __global__ void reconstruct_kernel // Find initial group - int group = offset_k / groupsize; + // int group = offset_k / groupsize; + int group = b_q_group_map[offset_k * 2]; int pre_rows_8 = min(rows_8, offset_k); int pre_rows_6 = offset_k > rows_8 ? min(rows_6, offset_k) - rows_8 : 0; @@ -337,7 +361,7 @@ __global__ void reconstruct_kernel half qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); half2 qs_h2 = __halves2half2(qs_h, qs_h); - int nextgroup = offset_k + groupsize; + int nextgroup = offset_k + b_q_group_map[offset_k * 2 + 1]; int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); int k = offset_k; @@ -347,7 +371,7 @@ __global__ void reconstruct_kernel while (k < rows_8 && k < end_k) { - if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += b_q_group_map[k * 2 + 1]; qs_h2 = __halves2half2(qs_h, qs_h); } for (int p = 0; p < 4; p++) { half2 dq[4]; @@ -363,7 +387,7 @@ __global__ void reconstruct_kernel while (k < rows_6 && k < end_k) { - if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += b_q_group_map[k * 2 + 1]; qs_h2 = __halves2half2(qs_h, qs_h); } for (int p = 0; p < 2; p++) { half2 dq[8]; @@ -380,7 +404,7 @@ __global__ void reconstruct_kernel while (k < rows_5 && k < end_k) { - if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += b_q_group_map[k * 2 + 1]; qs_h2 = __halves2half2(qs_h, qs_h); } for (int p = 0; p < 1; p++) { half2 dq[16]; @@ -399,7 +423,7 @@ __global__ void reconstruct_kernel while (k < rows_4 && k < end_k) { - if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += b_q_group_map[k * 2 + 1]; qs_h2 = __halves2half2(qs_h, qs_h); } for (int p = 0; p < 4; p++) { half2 dq[4]; @@ -414,7 +438,7 @@ __global__ void reconstruct_kernel while (k < rows_3 && k < end_k) { - if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += b_q_group_map[k * 2 + 1]; qs_h2 = __halves2half2(qs_h, qs_h); } for (int p = 0; p < 1; p++) { half2 dq[16]; @@ -431,8 +455,8 @@ __global__ void reconstruct_kernel while (k < rows_2 && k < end_k) { - if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); } - for (int p = 0; p < 2; p++) + if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += b_q_group_map[k * 2 + 1]; qs_h2 = __halves2half2(qs_h, qs_h); } + for (int p = 0; p < 1; p++) { half2 dq[8]; uint32_t q_0 = *b_ptr; b_ptr += size_n; @@ -441,7 +465,7 @@ __global__ void reconstruct_kernel half* dqh = (half*) dq; for (int j = 0; j < 16; j++) b_.set(perm[lk++], n, dqh[j]); } - k += 32; + k += 16; } } @@ -461,10 +485,10 @@ void QMatrix::reconstruct(half* out) cuda_q_perm, cuda_q_scale, cuda_q_scale_max, - //cuda_q_groups, + cuda_q_group_map, height, width, - groupsize, + //groupsize, groups, out, rows_8, @@ -487,7 +511,7 @@ void QMatrix::reconstruct(half* out) //const uint16_t* __restrict__ b_q_groups, height, width, - groupsize, + gptq_groupsize, groups, out, rows_4 diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cuh index dda83a4f..d36b8d66 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cuh +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cuh @@ -18,7 +18,7 @@ public: int height; int width; int groups; - int groupsize; + int gptq_groupsize; int rows_8; int rows_6; @@ -33,6 +33,7 @@ public: uint32_t* cuda_q_scale = NULL; half* cuda_q_scale_max = NULL; uint16_t* cuda_q_groups = NULL; + uint16_t* cuda_q_group_map = NULL; uint32_t* cuda_gptq_qzeros = NULL; half* cuda_gptq_scales = NULL; @@ -53,6 +54,7 @@ public: uint32_t* _q_scale, half* _q_scale_max, uint16_t* _q_groups, + uint16_t* _q_group_map, uint32_t* _gptq_qzeros, half* _gptq_scales, diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_util.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_util.cuh index 71657191..cac9df9c 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_util.cuh +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/quant/qdq_util.cuh @@ -7,6 +7,7 @@ union half2_uint32 half2 as_half2; __device__ half2_uint32(uint32_t val) : as_uint32(val) {} __device__ half2_uint32(half2 val) : as_half2(val) {} + __device__ half2_uint32() : as_uint32(0) {} }; union half_uint16 @@ -15,6 +16,7 @@ union half_uint16 half as_half; __device__ half_uint16(uint16_t val) : as_uint16(val) {} __device__ half_uint16(half val) : as_half(val) {} + __device__ half_uint16() : as_uint16(0) {} }; // Max_scale premultiplied by 1/256 diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/util.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/util.cuh index 06a58d18..f56eda79 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/util.cuh +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/util.cuh @@ -1,3 +1,11 @@ +#ifndef _util_cuh +#define _util_cuh + +#include +#include +#include +#include +#include #define DIVIDE(x, size) (((x) + (size) - 1) / (size)) @@ -40,3 +48,7 @@ inline void gpu_assert(cudaError_t code, const char *file, int line, bool abort= if (abort) exit(code); } } + +void print_global_mem(const half* ptr, int rows, int columns, int stride); + +#endif \ No newline at end of file diff --git a/server/exllamav2_kernels/exllamav2_kernels/ext.cpp b/server/exllamav2_kernels/exllamav2_kernels/ext.cpp index 5e52e6ab..ff4e1851 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/ext.cpp +++ b/server/exllamav2_kernels/exllamav2_kernels/ext.cpp @@ -31,6 +31,7 @@ uintptr_t make_q_matrix torch::Tensor q_scale, torch::Tensor q_scale_max, torch::Tensor q_groups, + torch::Tensor q_group_map, torch::Tensor gptq_qzeros, torch::Tensor gptq_scales, torch::Tensor gptq_g_idx, @@ -43,6 +44,7 @@ uintptr_t make_q_matrix TORCH_CHECK_DTYPE_OPT(q_scale, kInt); TORCH_CHECK_DTYPE_OPT(q_scale_max, kHalf); TORCH_CHECK_DTYPE_OPT(q_groups, kShort); + TORCH_CHECK_DTYPE_OPT(q_group_map, kShort); TORCH_CHECK_DTYPE_OPT(gptq_qzeros, kInt); TORCH_CHECK_DTYPE_OPT(gptq_scales, kHalf); TORCH_CHECK_DTYPE_OPT(gptq_g_idx, kInt); @@ -83,12 +85,15 @@ uintptr_t make_q_matrix q_scale.device().is_meta() ? NULL : (uint32_t*) q_scale.data_ptr(), q_scale_max.device().is_meta() ? NULL : (half*) q_scale_max.data_ptr(), q_groups.device().is_meta() ? NULL : (uint16_t*) q_groups.data_ptr(), + q_group_map.device().is_meta() ? NULL : (uint16_t*) q_group_map.data_ptr(), gptq_qzeros.device().is_meta() ? NULL : (uint32_t*) gptq_qzeros.data_ptr(), gptq_scales.device().is_meta() ? NULL : (half*) gptq_scales.data_ptr(), gptq_g_idx.device().is_meta() ? NULL : (uint32_t*) gptq_g_idx.data_ptr(), (half*) temp_dq.data_ptr() ); + if (m->failed) throw std::runtime_error("CUDA out of memory"); + return reinterpret_cast (m); } diff --git a/server/tests/utils/test_hub.py b/server/tests/utils/test_hub.py index 5438c153..49549893 100644 --- a/server/tests/utils/test_hub.py +++ b/server/tests/utils/test_hub.py @@ -32,10 +32,10 @@ def fresh_cache(): current_value = huggingface_hub.constants.HUGGINGFACE_HUB_CACHE huggingface_hub.constants.HUGGINGFACE_HUB_CACHE = d text_generation_server.utils.hub.HUGGINGFACE_HUB_CACHE = d - os.environ['HUGGINGFACE_HUB_CACHE'] = d + os.environ["HUGGINGFACE_HUB_CACHE"] = d yield huggingface_hub.constants.HUGGINGFACE_HUB_CACHE = current_value - os.environ['HUGGINGFACE_HUB_CACHE'] = current_value + os.environ["HUGGINGFACE_HUB_CACHE"] = current_value text_generation_server.utils.hub.HUGGINGFACE_HUB_CACHE = current_value @@ -47,7 +47,7 @@ def prefetched(): revision="main", local_files_only=False, repo_type="model", - allow_patterns=["*.safetensors"] + allow_patterns=["*.safetensors"], ) yield model_id @@ -61,7 +61,7 @@ def test_weight_hub_files_offline_error(offline, fresh_cache): def test_weight_hub_files_offline_ok(prefetched, offline): # If the model is prefetched then we should be able to get the weight files from local cache filenames = weight_hub_files(prefetched) - assert filenames == ['model.safetensors'] + assert filenames == ["model.safetensors"] def test_weight_hub_files(): diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index cd93d32a..22d03adf 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -71,7 +71,7 @@ def _load_multi_mqa_gptq( g_idx = weights.get_tensor(f"{prefix}.c_attn.g_idx") g_idx = g_idx.to(device=weights.device) - bits, groupsize = weights._get_gptq_params() + bits, groupsize, _ = weights._get_gptq_params() from text_generation_server.utils.layers import HAS_EXLLAMA diff --git a/server/text_generation_server/utils/gptq/exllamav2.py b/server/text_generation_server/utils/gptq/exllamav2.py index dd41b269..a24e834b 100644 --- a/server/text_generation_server/utils/gptq/exllamav2.py +++ b/server/text_generation_server/utils/gptq/exllamav2.py @@ -27,6 +27,32 @@ def ext_gemm_half_q_half(x, q_handle, q4_width, force_cuda): return output.view(output_shape) +# Group map needed for irregular group sizes + + +def make_group_map(q_groups, num_qrows): + + gr = q_groups.tolist() + group_map = [] + num_groups = len(gr) // 2 + + for i in range(num_groups): + bits = gr[i * 2] + if i < num_groups - 1: + qrows = gr[i * 2 + 3] - gr[i * 2 + 1] + else: + qrows = num_qrows - gr[i * 2 + 1] + rows = qrows * 32 // bits + for j in range(rows): + group_map += [i] + group_map += [rows - j] + + return torch.tensor(group_map, dtype=torch.short, device=q_groups.device) + + +# Create Q matrix + + def ext_make_q_matrix(w: dict, temp_dq, key: str = None): """ Create Q matrix @@ -37,6 +63,10 @@ def ext_make_q_matrix(w: dict, temp_dq, key: str = None): w["q_scale_max"] /= 256 w["q_perm"] = w["q_perm"].short() w["q_invperm"] = w["q_invperm"].short() + + if "q_group_map" not in w: + w["q_group_map"] = make_group_map(w["q_groups"], w["q_weight"].shape[0]) + return make_q_matrix( w["q_weight"], w["q_perm"], @@ -44,6 +74,7 @@ def ext_make_q_matrix(w: dict, temp_dq, key: str = None): w["q_scale"], w["q_scale_max"], w["q_groups"], + w["q_group_map"], none_tensor, none_tensor, none_tensor, @@ -70,6 +101,7 @@ def ext_make_q_matrix(w: dict, temp_dq, key: str = None): none_tensor, none_tensor, none_tensor, + none_tensor, w["qzeros"], w["scales"], w["g_idx"].cpu(), @@ -84,6 +116,7 @@ def ext_make_q_matrix(w: dict, temp_dq, key: str = None): none_tensor, none_tensor, none_tensor, + none_tensor, w["qzeros"], w["scales"], none_tensor, diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index 62afff0c..deb1a941 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -18,7 +18,9 @@ HF_HUB_OFFLINE = os.environ.get("HF_HUB_OFFLINE", "0").lower() in ["true", "1", "yes"] -def _cached_weight_files(model_id: str, revision: Optional[str], extension: str) -> List[str]: +def _cached_weight_files( + model_id: str, revision: Optional[str], extension: str +) -> List[str]: """Guess weight files from the cached revision snapshot directory""" d = _get_cached_revision_directory(model_id, revision) if not d: @@ -27,7 +29,9 @@ def _cached_weight_files(model_id: str, revision: Optional[str], extension: str) return filenames -def _weight_hub_files_from_model_info(info: hf_api.ModelInfo, extension: str) -> List[str]: +def _weight_hub_files_from_model_info( + info: hf_api.ModelInfo, extension: str +) -> List[str]: return [ s.rfilename for s in info.siblings @@ -44,21 +48,27 @@ def _weight_files_from_dir(d: Path, extension: str) -> List[str]: # see _weight_hub_files_from_model_info, that's also what is # done there with the len(s.rfilename.split("/")) == 1 condition root, _, files = next(os.walk(str(d))) - filenames = [f for f in files - if f.endswith(extension) - and "arguments" not in f - and "args" not in f - and "adapter" not in f - and "training" not in f] + filenames = [ + f + for f in files + if f.endswith(extension) + and "arguments" not in f + and "args" not in f + and "adapter" not in f + and "training" not in f + ] return filenames -def _get_cached_revision_directory(model_id: str, revision: Optional[str]) -> Optional[Path]: +def _get_cached_revision_directory( + model_id: str, revision: Optional[str] +) -> Optional[Path]: if revision is None: revision = "main" repo_cache = Path(HUGGINGFACE_HUB_CACHE) / Path( - file_download.repo_folder_name(repo_id=model_id, repo_type="model")) + file_download.repo_folder_name(repo_id=model_id, repo_type="model") + ) if not repo_cache.is_dir(): # No cache for this model @@ -86,7 +96,7 @@ def _get_cached_revision_directory(model_id: str, revision: Optional[str]) -> Op def weight_hub_files( - model_id: str, revision: Optional[str] = None, extension: str = ".safetensors" + model_id: str, revision: Optional[str] = None, extension: str = ".safetensors" ) -> List[str]: """Get the weights filenames on the hub""" api = HfApi() diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 011a9382..6648b55a 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -19,6 +19,7 @@ from text_generation_server.utils.gptq.quant_linear import QuantLinear from text_generation_server.utils.import_utils import IS_CUDA_SYSTEM, IS_ROCM_SYSTEM +from text_generation_server.utils.log import log_once HAS_AWQ = True try: @@ -35,10 +36,11 @@ CAN_EXLLAMA = major >= 8 V2 = os.getenv("EXLLAMA_VERSION", "2") == "2" if V2 and int(os.getenv("WORLD_SIZE", "1")) > 1: - logger.warning( + V2 = False + log_once( + logger.warning, "Disabling exllama v2 and using v1 instead because there are issues when sharding" ) - V2 = False if os.getenv("DISABLE_EXLLAMA") == "True": HAS_EXLLAMA = False diff --git a/server/text_generation_server/utils/log.py b/server/text_generation_server/utils/log.py new file mode 100644 index 00000000..d831fa76 --- /dev/null +++ b/server/text_generation_server/utils/log.py @@ -0,0 +1,6 @@ +from functools import lru_cache + + +@lru_cache(10) +def log_once(log, msg:str): + log(msg) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index a2cca2ea..ee1899ab 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -6,6 +6,7 @@ from loguru import logger from huggingface_hub import hf_hub_download import json +from text_generation_server.utils.log import log_once class Weights: @@ -161,7 +162,7 @@ def get_weights_col_packed_qkv(self, prefix: str, quantize: str): else: g_idx = None - bits, groupsize = self._get_gptq_params() + bits, groupsize, _ = self._get_gptq_params() weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) else: slice_ = self._get_slice(f"{prefix}.weight") @@ -211,10 +212,10 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): else: g_idx = None - bits, groupsize = self._get_gptq_params() + bits, groupsize, desc_act = self._get_gptq_params() from text_generation_server.utils.layers import HAS_EXLLAMA - use_exllama = bits == 4 and HAS_EXLLAMA and quantize == "gptq" + use_exllama = bits == 4 and HAS_EXLLAMA and quantize == "gptq" and not desc_act weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) else: w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] @@ -240,11 +241,15 @@ def get_tensor_shard(self, var, dim): def get_multi_weights_row(self, prefix: str, quantize: str): if quantize == "gptq": use_exllama = True - bits, groupsize = self._get_gptq_params() + bits, groupsize, desc_act = self._get_gptq_params() if bits != 4: use_exllama = False + if desc_act: + log_once(logger.warning, "Disabling exllama because desc_act=True") + use_exllama = False + if self.process_group.size() > 1: g_idx = self.get_tensor(f"{prefix}.g_idx") if g_idx is not None: @@ -274,12 +279,18 @@ def get_multi_weights_row(self, prefix: str, quantize: str): if use_exllama: if not HAS_EXLLAMA: if CAN_EXLLAMA: - logger.warning( + log_once( + logger.warning, "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True" ) use_exllama = False else: - logger.info(f"Using exllama kernels v{HAS_EXLLAMA}") + log_once( + logger.info, + f"Using exllama kernels v{HAS_EXLLAMA}" + ) + + g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) if use_exllama and groupsize != -1: qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) @@ -288,14 +299,12 @@ def get_multi_weights_row(self, prefix: str, quantize: str): qzeros = self.get_tensor(f"{prefix}.qzeros") scales = self.get_tensor(f"{prefix}.scales") - g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) - if use_exllama: g_idx = g_idx - g_idx[0] weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) elif quantize == "awq": - bits, groupsize = self._get_gptq_params() + bits, groupsize, _ = self._get_gptq_params() try: qweight = self.get_sharded(f"{prefix}.qweight", dim=0) @@ -314,18 +323,20 @@ def get_multi_weights_row(self, prefix: str, quantize: str): weight = self.get_sharded(f"{prefix}.weight", dim=1) return weight - def _get_gptq_params(self) -> Tuple[int, int]: + def _get_gptq_params(self) -> Tuple[int, int, int]: try: bits = self.get_tensor("gptq_bits").item() groupsize = self.get_tensor("gptq_groupsize").item() + desc_act = False except (SafetensorError, RuntimeError) as e: try: bits = self.gptq_bits groupsize = self.gptq_groupsize + desc_act = getattr(self, "gptq_desc_act", False) except Exception: raise e - return bits, groupsize + return bits, groupsize, desc_act def _set_gptq_params(self, model_id, revision): filename = "config.json" @@ -340,6 +351,7 @@ def _set_gptq_params(self, model_id, revision): data = json.load(f) self.gptq_bits = data["quantization_config"]["bits"] self.gptq_groupsize = data["quantization_config"]["group_size"] + self.gptq_desc_act = data["quantization_config"]["desc_act"] except Exception: filename = "quantize_config.json" try: @@ -353,6 +365,7 @@ def _set_gptq_params(self, model_id, revision): data = json.load(f) self.gptq_bits = data["bits"] self.gptq_groupsize = data["group_size"] + self.gptq_desc_act = data["desc_act"] except Exception: filename = "quant_config.json" try: @@ -366,5 +379,6 @@ def _set_gptq_params(self, model_id, revision): data = json.load(f) self.gptq_bits = data["w_bit"] self.gptq_groupsize = data["q_group_size"] + self.gptq_desc_act = data["desc_act"] except Exception: pass From 529d7c259128f025f34c3493ba8353f43db9762c Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 21 Dec 2023 17:29:23 +0100 Subject: [PATCH 512/793] Fix local load for peft (#1373) local directory overloaded still needs the directory to locate the weights files correctly. # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/tests/utils/test_hub.py | 10 +++++++++- server/text_generation_server/utils/hub.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/server/tests/utils/test_hub.py b/server/tests/utils/test_hub.py index 49549893..721820f5 100644 --- a/server/tests/utils/test_hub.py +++ b/server/tests/utils/test_hub.py @@ -61,7 +61,15 @@ def test_weight_hub_files_offline_error(offline, fresh_cache): def test_weight_hub_files_offline_ok(prefetched, offline): # If the model is prefetched then we should be able to get the weight files from local cache filenames = weight_hub_files(prefetched) - assert filenames == ["model.safetensors"] + root = None + assert len(filenames) == 1 + for f in filenames: + curroot, filename = os.path.split(f) + if root is None: + root = curroot + else: + assert root == curroot + assert filename == "model.safetensors" def test_weight_hub_files(): diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py index deb1a941..b56484f6 100644 --- a/server/text_generation_server/utils/hub.py +++ b/server/text_generation_server/utils/hub.py @@ -49,7 +49,7 @@ def _weight_files_from_dir(d: Path, extension: str) -> List[str]: # done there with the len(s.rfilename.split("/")) == 1 condition root, _, files = next(os.walk(str(d))) filenames = [ - f + os.path.join(root, f) for f in files if f.endswith(extension) and "arguments" not in f From 630800eed37b15c4b0c9eb8e6ab47212026720f7 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Fri, 22 Dec 2023 15:46:04 +0100 Subject: [PATCH 513/793] v1.3.4 --- Cargo.lock | 161 ++++++++---------- Cargo.toml | 2 +- docs/openapi.json | 2 +- integration-tests/pyproject.toml | 2 +- server/pyproject.toml | 2 +- server/text_generation_server/utils/layers.py | 2 +- server/text_generation_server/utils/log.py | 2 +- .../text_generation_server/utils/weights.py | 11 +- 8 files changed, 86 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da609e53..27c345b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "59d2a3357dde987206219e78ecfbbb6e8dad06cbb65292758d3270e6254f7355" [[package]] name = "arc-swap" @@ -128,18 +128,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -362,7 +362,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -426,9 +426,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5" dependencies = [ "cfg-if", "crossbeam-utils", @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -447,22 +447,21 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" dependencies = [ "cfg-if", ] @@ -813,7 +812,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -963,11 +962,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1023,9 +1022,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -1038,7 +1037,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2", "tokio", "tower-service", "tracing", @@ -1240,9 +1239,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "mach2" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ "libc", ] @@ -1312,9 +1311,9 @@ dependencies = [ [[package]] name = "metrics-exporter-prometheus" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" +checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" dependencies = [ "base64 0.21.5", "hyper", @@ -1336,7 +1335,7 @@ checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -1415,7 +1414,7 @@ checksum = "531c82a934da419bed3da09bd87d6e98c72f8d4aa755427b3b009c2b8b8c433c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -1612,9 +1611,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.61" +version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" +checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -1633,7 +1632,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -1644,9 +1643,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.97" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -1849,7 +1848,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -1866,9 +1865,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" [[package]] name = "portable-atomic" @@ -1895,7 +1894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -1924,9 +1923,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" dependencies = [ "unicode-ident", ] @@ -1968,7 +1967,7 @@ dependencies = [ "prost 0.12.3", "prost-types", "regex", - "syn 2.0.40", + "syn 2.0.42", "tempfile", "which", ] @@ -1996,7 +1995,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -2192,9 +2191,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "base64 0.21.5", "bytes", @@ -2278,7 +2277,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.40", + "syn 2.0.42", "walkdir", ] @@ -2455,7 +2454,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -2580,16 +2579,6 @@ version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.5.5" @@ -2649,7 +2638,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -2665,9 +2654,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.40" +version = "2.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13fa70a4ee923979ffb522cacce59d34421ebdea5625e1073c4326ef9d2dd42e" +checksum = "5b7d0a2c048d661a1a59fcd7355baa232f7ed34e0ee4df2eef3c1c1c0d3852d8" dependencies = [ "proc-macro2", "quote", @@ -2754,7 +2743,7 @@ dependencies = [ [[package]] name = "text-generation-benchmark" -version = "1.3.3" +version = "1.3.4" dependencies = [ "average", "clap", @@ -2775,7 +2764,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.3.3" +version = "1.3.4" dependencies = [ "futures", "grpc-metadata", @@ -2791,7 +2780,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.3.3" +version = "1.3.4" dependencies = [ "clap", "ctrlc", @@ -2807,7 +2796,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.3.3" +version = "1.3.4" dependencies = [ "async-stream", "axum", @@ -2842,22 +2831,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -2872,9 +2861,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", @@ -2894,9 +2883,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -2952,9 +2941,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.35.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -2964,7 +2953,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.5", + "socket2", "tokio-macros", "windows-sys 0.48.0", ] @@ -2987,7 +2976,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -3102,7 +3091,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -3175,7 +3164,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -3415,7 +3404,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] @@ -3511,7 +3500,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", "wasm-bindgen-shared", ] @@ -3545,7 +3534,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3835,22 +3824,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.30" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.30" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.40", + "syn 2.0.42", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 12b02579..24055497 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "1.3.3" +version = "1.3.4" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/docs/openapi.json b/docs/openapi.json index 62751928..df2d427f 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -10,7 +10,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0" }, - "version": "1.3.3" + "version": "1.3.4" }, "paths": { "/": { diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index 64723ae6..f6929587 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-integration-tests" -version = "1.3.3" +version = "1.3.4" description = "Text Generation Inference integration tests" authors = ["Nicolas Patry "] diff --git a/server/pyproject.toml b/server/pyproject.toml index 9bec2738..6e9be43e 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.3.3" +version = "1.3.4" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 6648b55a..d4fa2559 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -39,7 +39,7 @@ V2 = False log_once( logger.warning, - "Disabling exllama v2 and using v1 instead because there are issues when sharding" + "Disabling exllama v2 and using v1 instead because there are issues when sharding", ) if os.getenv("DISABLE_EXLLAMA") == "True": diff --git a/server/text_generation_server/utils/log.py b/server/text_generation_server/utils/log.py index d831fa76..b1456f1e 100644 --- a/server/text_generation_server/utils/log.py +++ b/server/text_generation_server/utils/log.py @@ -2,5 +2,5 @@ @lru_cache(10) -def log_once(log, msg:str): +def log_once(log, msg: str): log(msg) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index ee1899ab..c4e82a6d 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -215,7 +215,9 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): bits, groupsize, desc_act = self._get_gptq_params() from text_generation_server.utils.layers import HAS_EXLLAMA - use_exllama = bits == 4 and HAS_EXLLAMA and quantize == "gptq" and not desc_act + use_exllama = ( + bits == 4 and HAS_EXLLAMA and quantize == "gptq" and not desc_act + ) weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) else: w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] @@ -281,14 +283,11 @@ def get_multi_weights_row(self, prefix: str, quantize: str): if CAN_EXLLAMA: log_once( logger.warning, - "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True" + "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True", ) use_exllama = False else: - log_once( - logger.info, - f"Using exllama kernels v{HAS_EXLLAMA}" - ) + log_once(logger.info, f"Using exllama kernels v{HAS_EXLLAMA}") g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) From 3f9b3f4539a25e64071527d5b6e4306644dd46a8 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Tue, 9 Jan 2024 14:28:55 +0100 Subject: [PATCH 514/793] docs: update required CUDA version to 12.2 --- README.md | 2 +- docs/source/quicktour.md | 2 +- docs/source/supported_models.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d99b7306..1b3041d3 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ curl 127.0.0.1:8080/generate \ -H 'Content-Type: application/json' ``` -**Note:** To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. +**Note:** To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 12.2 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. **Note:** TGI supports AMD Instinct MI210 and MI250 GPUs. Details can be found in the [Supported Hardware documentation](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3-rocm --model-id $model` instead of the command above. diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index 03ea03bc..e9a33f04 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -13,7 +13,7 @@ docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingf -To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 11.8 or higher. +To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 12.2 or higher. diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md index 0708c729..dce4f2f9 100644 --- a/docs/source/supported_models.md +++ b/docs/source/supported_models.md @@ -39,7 +39,7 @@ text-generation-launcher --model-id ## Supported Hardware -TGI optimized models are supported on NVIDIA [A100](https://www.nvidia.com/en-us/data-center/a100/), [A10G](https://www.nvidia.com/en-us/data-center/products/a10-gpu/) and [T4](https://www.nvidia.com/en-us/data-center/tesla-t4/) GPUs with CUDA 11.8+. Note that you have to install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) to use it. For other NVIDIA GPUs, continuous batching will still apply, but some operations like flash attention and paged attention will not be executed. +TGI optimized models are supported on NVIDIA [A100](https://www.nvidia.com/en-us/data-center/a100/), [A10G](https://www.nvidia.com/en-us/data-center/products/a10-gpu/) and [T4](https://www.nvidia.com/en-us/data-center/tesla-t4/) GPUs with CUDA 12.2+. Note that you have to install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) to use it. For other NVIDIA GPUs, continuous batching will still apply, but some operations like flash attention and paged attention will not be executed. TGI also has support of ROCm-enabled AMD Instinct MI210 and MI250 GPUs, with paged attention and flash attention v2 support. The following features are currently not supported in the ROCm version of TGI, and the supported may be extended in the future: * Quantization (GPTQ, AWQ, etc.) From 564f2a3b755edee70518cb5ab30b1ffda717e105 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Tue, 9 Jan 2024 15:21:00 +0100 Subject: [PATCH 515/793] fix: fix local loading for .bin models (#1419) --- server/text_generation_server/cli.py | 2 +- server/text_generation_server/utils/peft.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 1d67d7eb..403f46e7 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -198,7 +198,7 @@ def download_weights( if not extension == ".safetensors" or not auto_convert: raise e - else: + elif (Path(model_id) / "adapter_config.json").exists(): # Try to load as a local PEFT model try: utils.download_and_unload_peft( diff --git a/server/text_generation_server/utils/peft.py b/server/text_generation_server/utils/peft.py index 45e23320..48ca264b 100644 --- a/server/text_generation_server/utils/peft.py +++ b/server/text_generation_server/utils/peft.py @@ -10,8 +10,7 @@ def download_and_unload_peft(model_id, revision, trust_remote_code): torch_dtype = torch.float16 - logger.info("Peft model detected.") - logger.info("Loading the model it might take a while without feedback") + logger.info("Trying to load a Peft model. It might take a while without feedback") try: model = AutoPeftModelForCausalLM.from_pretrained( model_id, @@ -28,7 +27,7 @@ def download_and_unload_peft(model_id, revision, trust_remote_code): trust_remote_code=trust_remote_code, low_cpu_mem_usage=True, ) - logger.info(f"Loaded.") + logger.info("Peft model detected.") logger.info(f"Merging the lora weights.") base_model_id = model.peft_config["default"].base_model_name_or_path From 91d72675342e34c314a0d7cc9bb9ca9d8f5aa295 Mon Sep 17 00:00:00 2001 From: "R. P. Ruiz" <42214371+deepily@users.noreply.github.com> Date: Tue, 9 Jan 2024 10:19:31 -0500 Subject: [PATCH 516/793] Fix missing make target platform for local install: 'install-flash-attention-v2' (#1414) --- server/text_generation_server/utils/flash_attn.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/flash_attn.py b/server/text_generation_server/utils/flash_attn.py index 02f01e65..48f8ef70 100644 --- a/server/text_generation_server/utils/flash_attn.py +++ b/server/text_generation_server/utils/flash_attn.py @@ -23,10 +23,15 @@ try: import flash_attn_2_cuda except ImportError: + architecture_suffix = "" + if IS_CUDA_SYSTEM: + architecture_suffix = "-cuda" + elif IS_ROCM_SYSTEM: + architecture_suffix = "-rocm" raise ImportError( "Flash Attention V2 is not installed.\n" "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) " - "or install flash attention v2 with `cd server && make install install-flash-attention-v2`" + f"or install flash attention v2 with `cd server && make install install-flash-attention-v2{architecture_suffix}`" ) if not (is_sm8x or is_sm90): raise ImportError( From fbeb1c44759fac1a249a9be8b47ad641438e3559 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Wed, 10 Jan 2024 16:35:54 +0100 Subject: [PATCH 517/793] fix: follow base model for tokenizer in router (#1424) Close #1422 --- Cargo.lock | 4 + router/Cargo.toml | 2 +- router/src/main.rs | 338 ++++++++++++++++++++++++--------------------- 3 files changed, 185 insertions(+), 159 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27c345b3..f826ea34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -950,13 +950,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b780635574b3d92f036890d8373433d6f9fc7abb320ee42a5c25897fc8ed732" dependencies = [ "dirs 5.0.1", + "futures", "indicatif", "log", "native-tls", + "num_cpus", "rand", + "reqwest", "serde", "serde_json", "thiserror", + "tokio", "ureq", ] diff --git a/router/Cargo.toml b/router/Cargo.toml index 55af635a..5ccdb0cd 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -21,6 +21,7 @@ axum-tracing-opentelemetry = "0.14.1" text-generation-client = { path = "client" } clap = { version = "4.4.5", features = ["derive", "env"] } futures = "0.3.28" +hf-hub = { version = "0.3.0", features = ["tokio"] } metrics = "0.21.1" metrics-exporter-prometheus = { version = "0.12.1", features = [] } nohash-hasher = "0.2.0" @@ -41,7 +42,6 @@ tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] } utoipa = { version = "3.5.0", features = ["axum_extras"] } utoipa-swagger-ui = { version = "3.1.5", features = ["axum"] } ngrok = { version = "0.13.1", features = ["axum"], optional = true } -hf-hub = "0.3.1" init-tracing-opentelemetry = { version = "0.14.1", features = ["opentelemetry-otlp"] } [build-dependencies] diff --git a/router/src/main.rs b/router/src/main.rs index d90632ef..4637c77c 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -1,19 +1,22 @@ -/// Text Generation Inference webserver entrypoint use axum::http::HeaderValue; use clap::Parser; +use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo}; +use hf_hub::{Repo, RepoType}; use opentelemetry::sdk::propagation::TraceContextPropagator; use opentelemetry::sdk::trace; use opentelemetry::sdk::trace::Sampler; use opentelemetry::sdk::Resource; use opentelemetry::{global, KeyValue}; use opentelemetry_otlp::WithExportConfig; +/// Text Generation Inference webserver entrypoint +use std::fs::File; +use std::io::BufReader; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path; -use std::time::Duration; use text_generation_client::{ClientError, ShardedClient}; use text_generation_router::{server, HubModelInfo}; use thiserror::Error; -use tokenizers::{FromPretrainedParameters, Tokenizer}; +use tokenizers::Tokenizer; use tower_http::cors::AllowOrigin; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; @@ -69,7 +72,8 @@ struct Args { ngrok_edge: Option, } -fn main() -> Result<(), RouterError> { +#[tokio::main] +async fn main() -> Result<(), RouterError> { // Get args let args = Args::parse(); // Pattern match configuration @@ -98,6 +102,9 @@ fn main() -> Result<(), RouterError> { ngrok_edge, } = args; + // Launch Tokio runtime + init_logging(otlp_endpoint, json_output); + // Validate args if max_input_length >= max_total_tokens { return Err(RouterError::ArgumentValidation( @@ -141,146 +148,158 @@ fn main() -> Result<(), RouterError> { // This will only be used to validate payloads let local_path = Path::new(&tokenizer_name); let local_model = local_path.exists() && local_path.is_dir(); - let tokenizer = if local_model { + + let (tokenizer, model_info) = if local_model { + // Get Model info + let model_info = HubModelInfo { + model_id: tokenizer_name.clone(), + sha: None, + pipeline_tag: None, + }; + // Load local tokenizer - Tokenizer::from_file(local_path.join("tokenizer.json")).ok() + let tokenizer = Tokenizer::from_file(local_path.join("tokenizer.json")).ok(); + + (tokenizer, model_info) } else { - // Download and instantiate tokenizer - // We need to download it outside of the Tokio runtime - let params = FromPretrainedParameters { - revision: revision.clone().unwrap_or("main".to_string()), - auth_token: authorization_token.clone(), - ..Default::default() + let mut builder = ApiBuilder::new() + .with_progress(false) + .with_token(authorization_token); + + if let Some(cache_dir) = std::env::var("HUGGINGFACE_HUB_CACHE").ok() { + builder = builder.with_cache_dir(cache_dir.into()); + } + + if revision.is_none() { + tracing::warn!("`--revision` is not set"); + tracing::warn!("We strongly advise to set it to a known supported commit."); + } + + let api = builder.build().unwrap(); + let api_repo = api.repo(Repo::with_revision( + tokenizer_name.clone(), + RepoType::Model, + revision.clone().unwrap_or("main".to_string()), + )); + + // Get Model info + let model_info = get_model_info(&api_repo).await.unwrap_or_else(|| { + tracing::warn!("Could not retrieve model info from the Hugging Face hub."); + HubModelInfo { + model_id: tokenizer_name.to_string(), + sha: None, + pipeline_tag: None, + } + }); + + let tokenizer = match api_repo.get("tokenizer.json").await { + Ok(tokenizer_filename) => Tokenizer::from_file(tokenizer_filename).ok(), + Err(_) => get_base_tokenizer(&api, &api_repo).await, }; - Tokenizer::from_pretrained(tokenizer_name.clone(), Some(params)).ok() + + (tokenizer, model_info) }; - // Launch Tokio runtime - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()? - .block_on(async { - init_logging(otlp_endpoint, json_output); + if tokenizer.is_none() { + tracing::warn!("Could not find a fast tokenizer implementation for {tokenizer_name}"); + tracing::warn!("Rust input length validation and truncation is disabled"); + } + + // if pipeline-tag == text-generation we default to return_full_text = true + let compat_return_full_text = match &model_info.pipeline_tag { + None => { + tracing::warn!("no pipeline tag found for model {tokenizer_name}"); + false + } + Some(pipeline_tag) => pipeline_tag.as_str() == "text-generation", + }; + + // Instantiate sharded client from the master unix socket + let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path) + .await + .map_err(RouterError::Connection)?; + // Clear the cache; useful if the webserver rebooted + sharded_client + .clear_cache(None) + .await + .map_err(RouterError::Cache)?; + // Get info from the shard + let shard_info = sharded_client.info().await.map_err(RouterError::Info)?; - if tokenizer.is_none() { + // Warmup model + tracing::info!("Warming up model"); + let max_supported_batch_total_tokens = match sharded_client + .warmup( + max_input_length as u32, + max_batch_prefill_tokens, + max_total_tokens as u32, + ) + .await + .map_err(RouterError::Warmup)? + { + // Older models do not support automatic max-batch-total-tokens + None => { + let max_batch_total_tokens = max_batch_total_tokens + .unwrap_or(16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens))); + tracing::warn!("Model does not support automatic max batch total tokens"); + max_batch_total_tokens + } + // Flash attention models return their max supported total tokens + Some(max_supported_batch_total_tokens) => { + // Warn if user added his own max-batch-total-tokens as we will ignore it + if max_batch_total_tokens.is_some() { tracing::warn!( - "Could not find a fast tokenizer implementation for {tokenizer_name}" + "`--max-batch-total-tokens` is deprecated for Flash \ + Attention models." ); - tracing::warn!("Rust input length validation and truncation is disabled"); + tracing::warn!( + "Inferred max batch total tokens: {max_supported_batch_total_tokens}" + ); + } + if max_total_tokens as u32 > max_supported_batch_total_tokens { + return Err(RouterError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_supported_batch_total_tokens}"))); } - // Get Model info - let model_info = match local_model { - true => HubModelInfo { - model_id: tokenizer_name.clone(), - sha: None, - pipeline_tag: None, - }, - false => get_model_info(&tokenizer_name, revision, authorization_token) - .await - .unwrap_or_else(|| { - tracing::warn!("Could not retrieve model info from the Hugging Face hub."); - HubModelInfo { - model_id: tokenizer_name.to_string(), - sha: None, - pipeline_tag: None, - } - }), - }; - - // if pipeline-tag == text-generation we default to return_full_text = true - let compat_return_full_text = match &model_info.pipeline_tag { - None => { - tracing::warn!("no pipeline tag found for model {tokenizer_name}"); - false - } - Some(pipeline_tag) => pipeline_tag.as_str() == "text-generation", - }; - - // Instantiate sharded client from the master unix socket - let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path) - .await - .map_err(RouterError::Connection)?; - // Clear the cache; useful if the webserver rebooted - sharded_client - .clear_cache(None) - .await - .map_err(RouterError::Cache)?; - // Get info from the shard - let shard_info = sharded_client.info().await.map_err(RouterError::Info)?; - - // Warmup model - tracing::info!("Warming up model"); - let max_supported_batch_total_tokens = match sharded_client - .warmup(max_input_length as u32, max_batch_prefill_tokens, max_total_tokens as u32) - .await - .map_err(RouterError::Warmup)? - { - // Older models do not support automatic max-batch-total-tokens - None => { - let max_batch_total_tokens = max_batch_total_tokens.unwrap_or( - 16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens)), - ); - tracing::warn!("Model does not support automatic max batch total tokens"); - max_batch_total_tokens - } - // Flash attention models return their max supported total tokens - Some(max_supported_batch_total_tokens) => { - // Warn if user added his own max-batch-total-tokens as we will ignore it - if max_batch_total_tokens.is_some() { - tracing::warn!( - "`--max-batch-total-tokens` is deprecated for Flash \ - Attention models." - ); - tracing::warn!( - "Inferred max batch total tokens: {max_supported_batch_total_tokens}" - ); - } - if max_total_tokens as u32 > max_supported_batch_total_tokens { - return Err(RouterError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_supported_batch_total_tokens}"))); - } - - max_supported_batch_total_tokens - } - }; - tracing::info!("Setting max batch total tokens to {max_supported_batch_total_tokens}"); - tracing::info!("Connected"); - - let addr = match hostname.parse() { - Ok(ip) => SocketAddr::new(ip, port), - Err(_) => { - tracing::warn!("Invalid hostname, defaulting to 0.0.0.0"); - SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port) - } - }; - - // Run server - server::run( - model_info, - shard_info, - compat_return_full_text, - max_concurrent_requests, - max_best_of, - max_stop_sequences, - max_top_n_tokens, - max_input_length, - max_total_tokens, - waiting_served_ratio, - max_batch_prefill_tokens, - max_supported_batch_total_tokens, - max_waiting_tokens, - sharded_client, - tokenizer, - validation_workers, - addr, - cors_allow_origin, - ngrok, - ngrok_authtoken, - ngrok_edge, - ) - .await?; - Ok(()) - }) + max_supported_batch_total_tokens + } + }; + tracing::info!("Setting max batch total tokens to {max_supported_batch_total_tokens}"); + tracing::info!("Connected"); + + let addr = match hostname.parse() { + Ok(ip) => SocketAddr::new(ip, port), + Err(_) => { + tracing::warn!("Invalid hostname, defaulting to 0.0.0.0"); + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port) + } + }; + + // Run server + server::run( + model_info, + shard_info, + compat_return_full_text, + max_concurrent_requests, + max_best_of, + max_stop_sequences, + max_top_n_tokens, + max_input_length, + max_total_tokens, + waiting_served_ratio, + max_batch_prefill_tokens, + max_supported_batch_total_tokens, + max_waiting_tokens, + sharded_client, + tokenizer, + validation_workers, + addr, + cors_allow_origin, + ngrok, + ngrok_authtoken, + ngrok_edge, + ) + .await?; + Ok(()) } /// Init logging using env variables LOG_LEVEL and LOG_FORMAT: @@ -339,30 +358,8 @@ fn init_logging(otlp_endpoint: Option, json_output: bool) { } /// get model info from the Huggingface Hub -pub async fn get_model_info( - model_id: &str, - revision: Option, - token: Option, -) -> Option { - let revision = match revision { - None => { - tracing::warn!("`--revision` is not set"); - tracing::warn!("We strongly advise to set it to a known supported commit."); - "main".to_string() - } - Some(revision) => revision, - }; - - let client = reqwest::Client::new(); - // Poor man's urlencode - let revision = revision.replace('/', "%2F"); - let url = format!("https://huggingface.co/api/models/{model_id}/revision/{revision}"); - let mut builder = client.get(url).timeout(Duration::from_secs(5)); - if let Some(token) = token { - builder = builder.bearer_auth(token); - } - - let response = builder.send().await.ok()?; +pub async fn get_model_info(api: &ApiRepo) -> Option { + let response = api.info_request().send().await.ok()?; if response.status().is_success() { let hub_model_info: HubModelInfo = @@ -379,6 +376,31 @@ pub async fn get_model_info( } } +/// get base tokenizer +pub async fn get_base_tokenizer(api: &Api, api_repo: &ApiRepo) -> Option { + let config_filename = api_repo.get("config.json").await.ok()?; + + // Open the file in read-only mode with buffer. + let file = File::open(config_filename).ok()?; + let reader = BufReader::new(file); + + // Read the JSON contents of the file as an instance of `User`. + let config: serde_json::Value = serde_json::from_reader(reader).ok()?; + + if let Some(serde_json::Value::String(base_model_id)) = config.get("base_model_name_or_path") { + let api_base_repo = api.repo(Repo::with_revision( + base_model_id.to_string(), + RepoType::Model, + "main".to_string(), + )); + + let tokenizer_filename = api_base_repo.get("tokenizer.json").await.ok()?; + Tokenizer::from_file(tokenizer_filename).ok() + } else { + None + } +} + #[derive(Debug, Error)] enum RouterError { #[error("Argument validation error: {0}")] From da27fbdfdbaaab4157e0aa6114c16503471c499c Mon Sep 17 00:00:00 2001 From: PYNing <540439329@qq.com> Date: Thu, 11 Jan 2024 01:36:20 +0800 Subject: [PATCH 518/793] Fix local load for Medusa (#1420) # What does this PR do? Close #1418 Close #1415 ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/cli.py | 29 +++++++++++++++++++ .../models/flash_llama.py | 25 +++++++++++----- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 403f46e7..99be6c7e 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -198,6 +198,35 @@ def download_weights( if not extension == ".safetensors" or not auto_convert: raise e + elif (Path(model_id) / "medusa_lm_head.pt").exists(): + # Try to load as a local Medusa model + try: + import json + + medusa_head = Path(model_id) / "medusa_lm_head.pt" + if auto_convert: + medusa_sf = Path(model_id) / "medusa_lm_head.safetensors" + if not medusa_sf.exists(): + utils.convert_files([Path(medusa_head)], [medusa_sf], []) + medusa_config = Path(model_id) / "config.json" + with open(medusa_config, "r") as f: + config = json.load(f) + + model_id = config["base_model_name_or_path"] + revision = "main" + try: + utils.weight_files(model_id, revision, extension) + logger.info( + f"Files for parent {model_id} are already present on the host. " + "Skipping download." + ) + return + # Local files not found + except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError): + pass + except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError): + pass + elif (Path(model_id) / "adapter_config.json").exists(): # Try to load as a local PEFT model try: diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 8a3bccdd..7be61906 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -71,15 +71,26 @@ def __init__( from text_generation_server.utils.medusa import MedusaModel from huggingface_hub import hf_hub_download import json - - medusa_config = hf_hub_download( - use_medusa, revision=revision, filename="config.json" - ) + import os + from pathlib import Path + + is_local_model = (Path(use_medusa).exists() and Path(use_medusa).is_dir()) or os.getenv( + "WEIGHTS_CACHE_OVERRIDE", None + ) is not None + + if not is_local_model: + medusa_config = hf_hub_download( + use_medusa, revision=revision, filename="config.json" + ) + medusa_head = hf_hub_download( + use_medusa, revision=revision, filename="medusa_lm_head.pt" + ) + else: + medusa_config = str(Path(use_medusa) / "config.json") + medusa_head = str(Path(use_medusa) / "medusa_lm_head.pt") + with open(medusa_config, "r") as f: config = json.load(f) - medusa_head = hf_hub_download( - use_medusa, revision=revision, filename="medusa_lm_head.pt" - ) medusa_sf = medusa_head[: -len(".pt")] + ".safetensors" weights = Weights( [medusa_sf], device, dtype, process_group=self.process_group From ac08b4ef9cecfa089a6cba6fc2474eef49f42dc0 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 11 Jan 2024 19:01:43 +0100 Subject: [PATCH 519/793] Return prompt vs generated tokens. (#1436) # What does this PR do? Fixes #637 Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- router/src/infer.rs | 15 +++++++++++++-- router/src/server.rs | 8 +++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index bf5920da..b4094c1b 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -90,6 +90,7 @@ impl Infer { ) -> Result< ( OwnedSemaphorePermit, + u32, UnboundedReceiverStream>, ), InferError, @@ -114,6 +115,7 @@ impl Infer { // MPSC channel to communicate with the background batching task let (response_tx, response_rx) = mpsc::unbounded_channel(); + let input_length = valid_request.input_length; // Append the request to the queue self.queue.append(Entry { @@ -130,7 +132,11 @@ impl Infer { self.shared.batching_task.notify_one(); // Return stream - Ok((permit, UnboundedReceiverStream::new(response_rx))) + Ok(( + permit, + input_length, + UnboundedReceiverStream::new(response_rx), + )) } /// Add a new request to the queue and return a InferResponse @@ -142,7 +148,7 @@ impl Infer { let use_top_tokens = request.parameters.top_n_tokens.is_some_and(|x| x > 0); // Create stream and keep semaphore permit as long as generate lives - let (_permit, mut stream) = self.generate_stream(request).await?; + let (_permit, _input_length, mut stream) = self.generate_stream(request).await?; // Return values let mut result_prefill = Vec::new(); @@ -196,6 +202,7 @@ impl Infer { { Ok(InferResponse { prefill: result_prefill, + _input_length, tokens: result_tokens, generated_text, queued, @@ -636,6 +643,10 @@ pub(crate) enum InferStreamResponse { #[derive(Debug)] pub(crate) struct InferResponse { + /// input_length is the input as perceived by the rust tokenizer in the + /// validation pathway. It is redundant with prefill.len() but prefill + /// has data only if the user asked for it. This will always be filled. + pub(crate) _input_length: u32, pub(crate) prefill: Vec, pub(crate) tokens: Vec, pub(crate) generated_text: GeneratedText, diff --git a/router/src/server.rs b/router/src/server.rs index fe1b8309..3db5c7cd 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -170,6 +170,7 @@ async fn generate( }; // Token details + let input_length = response._input_length; let details = match details { true => { // convert best_of_responses @@ -257,6 +258,11 @@ async fn generate( "x-time-per-token", time_per_token.as_millis().to_string().parse().unwrap(), ); + headers.insert("x-prompt-tokens", input_length.into()); + headers.insert( + "x-generated-tokens", + response.generated_text.generated_tokens.into(), + ); // Metrics metrics::increment_counter!("tgi_request_success"); @@ -378,7 +384,7 @@ async fn generate_stream( } else { match infer.generate_stream(req).instrument(info_span!(parent: &span, "async_stream")).await { // Keep permit as long as generate_stream lives - Ok((_permit, mut response_stream)) => { + Ok((_permit, _input_length, mut response_stream)) => { // Server-Sent Event stream while let Some(response) = response_stream.next().await { match response { From 0eabc83541225979209ff7183b4b4442e47adf92 Mon Sep 17 00:00:00 2001 From: drbh Date: Tue, 16 Jan 2024 05:07:41 -0500 Subject: [PATCH 520/793] feat: supports openai chat completions API (#1427) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds support to make TGI a drop in replacement for OpenAI clients by exposing the same HTTP interface. Notes - TGI inits a single model at startup so the `model` field is unused in HTTP requests. - `max_tokens` and `stream` should work as expected but other params may be (unimplemented or not supported) General approach - fetch the `tokenizer_config` at startup from the hub - pass `tokenizer_config` into `Infer` so we have it at request time - use the `chat_template` on the config to format chat request - parse jinja template and render chat string - pass inputs into existing generate function - wrap generation output in expected structure before returning # How to test ### Streaming curl ```bash curl localhost:3000/v1/chat/completions \ -X POST \ -d '{ "model": "tgi", "messages": [ { "role": "system", "content": "You are a helpful assistant." }, { "role": "user", "content": "What is deep learning?" } ], "stream": true, "max_tokens": 20 }' \ -H 'Content-Type: application/json' ``` It is also possible to use the `openai` python library and change the base url ### 🌊 STREAMING REQUEST ```python from openai import OpenAI # init the client but point it to TGI client = OpenAI( base_url="http://localhost:3000/v1", api_key="not needed for a local LLM" ) chat_completion = client.chat.completions.create( model="tgi", messages=[ {"role": "system", "content": "You are a helpful assistant." }, {"role": "user", "content": "What is deep learning?"} ], stream=True ) # iterate and print stream for message in chat_completion: print(message) # ChatCompletionChunk(id='', choices=[Choice(delta=ChoiceDelta(content=' that', function_call=None, role='assistant', tool_calls=None), finish_reason=None, index=2, logprobs=None)], created=1704486761, model='', object='text_completion', system_fingerprint='') ``` ### 🚗 SYNCHRONOUS REQUEST ```python from openai import OpenAI # init the client but point it to TGI client = OpenAI( base_url="http://localhost:3000/v1", api_key="not needed for a local LLM" ) chat_completion = client.chat.completions.create( model="tgi", messages=[ {"role": "system", "content": "You are a helpful assistant." }, {"role": "user", "content": "What is deep learning?"} ], stream=False ) print(chat_completion) # ChatCompletion(id='', choices=[Choice(finish_reason=None, index=0, logprobs=None, message=ChatCompletionMessage(content='\nDeep learning is a new field of research that has been gaining traction in the last ...', role='assistant', function_call=None, tool_calls=None))], created=1704486762, model='', object='text_completion', system_fingerprint='', usage=CompletionUsage(completion_tokens=100, prompt_tokens=76, total_tokens=176)) ``` ## How to run dev ```bash cd text-generation-inference/server MASTER_ADDR=127.0.0.1 MASTER_PORT=5555 text-generation-server serve --trust-remote-code gpt2 ``` ***note many of the existing `chat_templates` use non standard `jinja` (ie. adding a `raise` to the template) which will throw an error when parsing; hence using `upstage/SOLAR-10.7B-Instruct-v1.0` since it has a valid template ```bash cd text-generation-inference/router cargo run -- --tokenizer-name upstage/SOLAR-10.7B-Instruct-v1.0 ``` trigger ```bash curl localhost:3000/v1/chat/completions \ -X POST \ -d '{ "model": "gpt-3.5-turbo", "messages": [ { "role": "system", "content": "You are a helpful assistant." }, { "role": "user", "content": "What is the IP address of the Google DNS servers?" } ], "stream": true, "max_tokens": 20, "logprobs": true }' \ -H 'Content-Type: application/json' ``` ^ supports `stream: true` and `stream: false` requests --- Cargo.lock | 39 ++++--- router/Cargo.toml | 2 + router/src/infer.rs | 51 ++++++--- router/src/lib.rs | 224 ++++++++++++++++++++++++++++++++++++++- router/src/main.rs | 116 +++++++++++++++----- router/src/server.rs | 187 ++++++++++++++++++++++++++++++-- router/src/validation.rs | 2 +- 7 files changed, 557 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f826ea34..3baff665 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -773,9 +773,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -783,9 +783,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" @@ -800,15 +800,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -817,21 +817,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1373,6 +1373,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "minijinja" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "208758577ef2c86cf5dd3e85730d161413ec3284e2d73b2ef65d9a24d9971bcb" +dependencies = [ + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2807,10 +2816,12 @@ dependencies = [ "axum-tracing-opentelemetry", "clap", "futures", + "futures-util", "hf-hub", "init-tracing-opentelemetry", "metrics", "metrics-exporter-prometheus", + "minijinja", "ngrok", "nohash-hasher", "opentelemetry", diff --git a/router/Cargo.toml b/router/Cargo.toml index 5ccdb0cd..f6f16dae 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -43,6 +43,8 @@ utoipa = { version = "3.5.0", features = ["axum_extras"] } utoipa-swagger-ui = { version = "3.1.5", features = ["axum"] } ngrok = { version = "0.13.1", features = ["axum"], optional = true } init-tracing-opentelemetry = { version = "0.14.1", features = ["opentelemetry-otlp"] } +minijinja = "1.0.10" +futures-util = "0.3.30" [build-dependencies] vergen = { version = "8.2.5", features = ["build", "git", "gitcl"] } diff --git a/router/src/infer.rs b/router/src/infer.rs index b4094c1b..6de07982 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -1,8 +1,10 @@ /// Batching and inference logic use crate::validation::{Validation, ValidationError}; +use crate::HubTokenizerConfig; +use crate::{ChatRequest, GenerateRequest, GenerateStreamResponse, PrefillToken}; use crate::{Entry, Queue, Token}; -use crate::{GenerateRequest, PrefillToken}; use futures::future::try_join_all; +use minijinja::{Environment, ErrorKind, Template}; use nohash_hasher::IntMap; use std::sync::{ atomic::{AtomicBool, Ordering}, @@ -13,7 +15,7 @@ use text_generation_client::{ }; use thiserror::Error; use tokio::sync::mpsc::error::SendError; -use tokio::sync::{mpsc, Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; +use tokio::sync::{mpsc, Notify, Semaphore, TryAcquireError}; use tokio::time::Instant; use tokio_stream::wrappers::UnboundedReceiverStream; use tokio_stream::StreamExt; @@ -30,6 +32,8 @@ pub struct Infer { shared: Arc, /// Inference limit limit_concurrent_requests: Arc, + /// Chat template + template: Option>, } /// Infer shared state @@ -52,6 +56,7 @@ impl Infer { window_size: Option, speculate: u32, generation_health: Arc, + tokenizer_config: HubTokenizerConfig, ) -> Self { // Infer shared state let queue = Queue::new(requires_padding, 16, window_size, speculate); @@ -74,11 +79,21 @@ impl Infer { // Inference limit with a semaphore let semaphore = Arc::new(Semaphore::new(max_concurrent_requests)); + let template = tokenizer_config.chat_template.map(|t| { + let env = Box::new(Environment::new()); + let template_str = t.into_boxed_str(); + // leaking env and template_str as read-only, static resources for performance. + Box::leak(env) + .template_from_str(Box::leak(template_str)) + .unwrap() + }); + Self { validation, queue, shared, limit_concurrent_requests: semaphore, + template, } } @@ -87,14 +102,7 @@ impl Infer { pub(crate) async fn generate_stream( &self, request: GenerateRequest, - ) -> Result< - ( - OwnedSemaphorePermit, - u32, - UnboundedReceiverStream>, - ), - InferError, - > { + ) -> Result { // Limit concurrent requests by acquiring a permit from the semaphore let permit = self .clone() @@ -139,6 +147,20 @@ impl Infer { )) } + /// Apply the chat template to the chat request + #[instrument(skip_all)] + pub(crate) fn apply_chat_template(&self, chat: ChatRequest) -> Result { + self.template + .as_ref() + .ok_or_else(|| InferError::TemplateError(ErrorKind::TemplateNotFound.into()))? + .render(chat) + .map_err(|e| { + metrics::increment_counter!("tgi_request_failure", "err" => "template"); + tracing::error!("{e}"); + InferError::TemplateError(e) + }) + } + /// Add a new request to the queue and return a InferResponse #[instrument(skip_all)] pub(crate) async fn generate( @@ -550,9 +572,9 @@ fn send_responses( let mut iterator = tokens_ .ids .into_iter() - .zip(tokens_.logprobs.into_iter()) - .zip(tokens_.texts.into_iter()) - .zip(tokens_.is_special.into_iter()) + .zip(tokens_.logprobs) + .zip(tokens_.texts) + .zip(tokens_.is_special) .enumerate() .peekable(); while let Some((i, (((id, logprob), text), special))) = iterator.next() { @@ -665,6 +687,8 @@ pub enum InferError { ValidationError(#[from] ValidationError), #[error("Incomplete generation")] IncompleteGeneration, + #[error("Template error: {0}")] + TemplateError(#[from] minijinja::Error), } impl InferError { @@ -674,6 +698,7 @@ impl InferError { InferError::Overloaded(_) => "overloaded", InferError::ValidationError(_) => "validation", InferError::IncompleteGeneration => "incomplete_generation", + InferError::TemplateError(_) => "template_error", } } } diff --git a/router/src/lib.rs b/router/src/lib.rs index 898fcd04..f6f8276f 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -5,12 +5,21 @@ mod queue; pub mod server; mod validation; -use infer::Infer; +use infer::{Infer, InferError, InferStreamResponse}; use queue::{Entry, Queue}; use serde::{Deserialize, Serialize}; +use tokio::sync::OwnedSemaphorePermit; +use tokio_stream::wrappers::UnboundedReceiverStream; use utoipa::ToSchema; use validation::Validation; +/// Type alias for generation responses +pub(crate) type GenerateStreamResponse = ( + OwnedSemaphorePermit, + u32, // input_length + UnboundedReceiverStream>, +); + /// Hub type #[derive(Clone, Debug, Deserialize)] pub struct HubModelInfo { @@ -20,6 +29,19 @@ pub struct HubModelInfo { pub pipeline_tag: Option, } +#[derive(Clone, Deserialize, Default)] +pub struct HubTokenizerConfig { + #[serde(default)] + pub chat_template: Option, +} + +impl HubTokenizerConfig { + pub fn from_file(filename: &str) -> Self { + let content = std::fs::read_to_string(filename).unwrap(); + serde_json::from_str(&content).unwrap_or_default() + } +} + #[derive(Clone, Debug, Serialize, ToSchema)] pub struct Info { /// Model info @@ -152,7 +174,7 @@ fn default_parameters() -> GenerateParameters { top_k: None, top_p: None, typical_p: None, - do_sample: false, + do_sample: true, max_new_tokens: default_max_new_tokens(), return_full_text: None, stop: Vec::new(), @@ -165,6 +187,193 @@ fn default_parameters() -> GenerateParameters { } } +#[derive(Clone, Deserialize, Serialize)] +pub(crate) struct ChatCompletion { + pub id: String, + pub object: String, + pub created: u64, + pub model: String, + pub system_fingerprint: String, + pub choices: Vec, + pub usage: Usage, +} + +#[derive(Clone, Deserialize, Serialize)] +pub(crate) struct ChatCompletionComplete { + pub index: u32, + pub message: Message, + pub logprobs: Option>, + pub finish_reason: String, +} + +#[derive(Clone, Deserialize, Serialize)] +pub(crate) struct Usage { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, +} + +impl ChatCompletion { + pub(crate) fn new( + model: String, + system_fingerprint: String, + output: String, + created: u64, + details: Details, + return_logprobs: bool, + ) -> Self { + Self { + id: String::new(), + object: "text_completion".into(), + created, + model, + system_fingerprint, + choices: vec![ChatCompletionComplete { + index: 0, + message: Message { + role: "assistant".into(), + content: output, + }, + logprobs: return_logprobs + .then(|| details.tokens.iter().map(|t| t.logprob).collect()), + finish_reason: details.finish_reason.to_string(), + }], + usage: Usage { + prompt_tokens: details.prefill.len() as u32, + completion_tokens: details.generated_tokens, + total_tokens: details.prefill.len() as u32 + details.generated_tokens, + }, + } + } +} + +#[derive(Clone, Deserialize, Serialize)] +pub(crate) struct ChatCompletionChunk { + pub id: String, + pub object: String, + pub created: u64, + pub model: String, + pub system_fingerprint: String, + pub choices: Vec, +} + +#[derive(Clone, Deserialize, Serialize)] +pub(crate) struct ChatCompletionChoice { + pub index: u32, + pub delta: ChatCompletionDelta, + pub logprobs: Option, + pub finish_reason: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub(crate) struct ChatCompletionDelta { + pub role: String, + pub content: String, +} + +impl ChatCompletionChunk { + pub(crate) fn new( + model: String, + system_fingerprint: String, + delta: String, + created: u64, + index: u32, + logprobs: Option, + finish_reason: Option, + ) -> Self { + Self { + id: String::new(), + object: "text_completion".to_string(), + created, + model, + system_fingerprint, + choices: vec![ChatCompletionChoice { + index, + delta: ChatCompletionDelta { + role: "assistant".to_string(), + content: delta, + }, + logprobs, + finish_reason, + }], + } + } +} + +fn default_request_messages() -> Vec { + vec![Message { + role: "user".to_string(), + content: "My name is David and I".to_string(), + }] +} + +#[derive(Clone, Deserialize, ToSchema, Serialize)] +pub(crate) struct ChatRequest { + /// UNUSED + #[schema(example = "bigscience/blomm-560m")] + /// ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API. + pub model: String, /* NOTE: UNUSED */ + + /// A list of messages comprising the conversation so far. + #[serde(default = "default_request_messages")] + pub messages: Vec, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, + /// decreasing the model's likelihood to repeat the same line verbatim. + #[serde(default)] + pub frequency_penalty: Option, + + /// UNUSED + /// Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens + /// (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, + /// the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, + /// but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should + /// result in a ban or exclusive selection of the relevant token. + #[serde(default)] + pub logit_bias: Option>, + + /// Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each + /// output token returned in the content of message. + #[serde(default)] + pub logprobs: Option, + + /// UNUSED + /// An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with + /// an associated log probability. logprobs must be set to true if this parameter is used. + #[serde(default)] + pub top_logprobs: Option, + + /// The maximum number of tokens that can be generated in the chat completion. + #[serde(default)] + pub max_tokens: Option, + + /// UNUSED + /// How many chat completion choices to generate for each input message. Note that you will be charged based on the + /// number of generated tokens across all of the choices. Keep n as 1 to minimize costs. + #[serde(default)] + pub n: Option, + + /// UNUSED + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, + /// increasing the model's likelihood to talk about new topics + #[serde(default)] + pub presence_penalty: Option, + + #[serde(default = "bool::default")] + pub stream: bool, + + #[schema(nullable = true, example = 42)] + pub seed: Option, +} + +#[derive(Clone, Deserialize, ToSchema, Serialize)] +pub(crate) struct Message { + #[schema(example = "user")] + pub role: String, + #[schema(example = "My name is David and I")] + pub content: String, +} + #[derive(Clone, Debug, Deserialize, ToSchema)] pub(crate) struct GenerateRequest { #[schema(example = "My name is Olivier and I")] @@ -227,6 +436,16 @@ pub(crate) enum FinishReason { StopSequence, } +impl std::fmt::Display for FinishReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FinishReason::Length => write!(f, "length"), + FinishReason::EndOfSequenceToken => write!(f, "eos_token"), + FinishReason::StopSequence => write!(f, "stop_sequence"), + } + } +} + #[derive(Serialize, ToSchema)] pub(crate) struct BestOfSequence { #[schema(example = "test")] @@ -279,6 +498,7 @@ pub(crate) struct StreamDetails { #[derive(Serialize, ToSchema)] pub(crate) struct StreamResponse { + pub index: u32, pub token: Token, #[serde(skip_serializing_if = "Vec::is_empty")] pub top_tokens: Vec, diff --git a/router/src/main.rs b/router/src/main.rs index 4637c77c..f5d44305 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -8,13 +8,12 @@ use opentelemetry::sdk::trace::Sampler; use opentelemetry::sdk::Resource; use opentelemetry::{global, KeyValue}; use opentelemetry_otlp::WithExportConfig; -/// Text Generation Inference webserver entrypoint use std::fs::File; use std::io::BufReader; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path; use text_generation_client::{ClientError, ShardedClient}; -use text_generation_router::{server, HubModelInfo}; +use text_generation_router::{server, HubModelInfo, HubTokenizerConfig}; use thiserror::Error; use tokenizers::Tokenizer; use tower_http::cors::AllowOrigin; @@ -55,6 +54,8 @@ struct Args { #[clap(default_value = "bigscience/bloom", long, env)] tokenizer_name: String, #[clap(long, env)] + tokenizer_config_path: Option, + #[clap(long, env)] revision: Option, #[clap(default_value = "2", long, env)] validation_workers: usize, @@ -92,6 +93,7 @@ async fn main() -> Result<(), RouterError> { port, master_shard_uds_path, tokenizer_name, + tokenizer_config_path, revision, validation_workers, json_output, @@ -149,40 +151,64 @@ async fn main() -> Result<(), RouterError> { let local_path = Path::new(&tokenizer_name); let local_model = local_path.exists() && local_path.is_dir(); - let (tokenizer, model_info) = if local_model { - // Get Model info - let model_info = HubModelInfo { - model_id: tokenizer_name.clone(), - sha: None, - pipeline_tag: None, - }; + // Load tokenizer config + // This will be used to format the chat template + let local_tokenizer_config_path = + tokenizer_config_path.unwrap_or("tokenizer_config.json".to_string()); + let local_tokenizer_config = Path::new(&local_tokenizer_config_path).exists(); - // Load local tokenizer - let tokenizer = Tokenizer::from_file(local_path.join("tokenizer.json")).ok(); - - (tokenizer, model_info) - } else { + // Shared API builder initialization + let api_builder = || { let mut builder = ApiBuilder::new() .with_progress(false) .with_token(authorization_token); - if let Some(cache_dir) = std::env::var("HUGGINGFACE_HUB_CACHE").ok() { + if let Ok(cache_dir) = std::env::var("HUGGINGFACE_HUB_CACHE") { builder = builder.with_cache_dir(cache_dir.into()); } - if revision.is_none() { - tracing::warn!("`--revision` is not set"); - tracing::warn!("We strongly advise to set it to a known supported commit."); + builder + }; + + // Decide if we need to use the API based on the revision and local path + let use_api = revision.is_some() || !local_path.exists() || !local_path.is_dir(); + + // Initialize API if needed + let api = if use_api { + tracing::info!("Using the Hugging Face API"); + match api_builder().build() { + Ok(api) => Some(api), + Err(_) => { + tracing::warn!("Unable to build the Hugging Face API"); + None + } } + } else { + None + }; + + // Load tokenizer and model info + let (tokenizer, model_info) = if local_model { + let tokenizer = Tokenizer::from_file(local_path.join("tokenizer.json")).ok(); + let model_info = HubModelInfo { + model_id: tokenizer_name.to_string(), + sha: None, + pipeline_tag: None, + }; - let api = builder.build().unwrap(); + (tokenizer, model_info) + } else if let Some(api) = api.clone() { let api_repo = api.repo(Repo::with_revision( - tokenizer_name.clone(), + tokenizer_name.to_string(), RepoType::Model, - revision.clone().unwrap_or("main".to_string()), + revision.clone().unwrap_or_else(|| "main".to_string()), )); - // Get Model info + let tokenizer = match api_repo.get("tokenizer.json").await { + Ok(tokenizer_filename) => Tokenizer::from_file(tokenizer_filename).ok(), + Err(_) => get_base_tokenizer(&api, &api_repo).await, + }; + let model_info = get_model_info(&api_repo).await.unwrap_or_else(|| { tracing::warn!("Could not retrieve model info from the Hugging Face hub."); HubModelInfo { @@ -192,12 +218,33 @@ async fn main() -> Result<(), RouterError> { } }); - let tokenizer = match api_repo.get("tokenizer.json").await { - Ok(tokenizer_filename) => Tokenizer::from_file(tokenizer_filename).ok(), - Err(_) => get_base_tokenizer(&api, &api_repo).await, - }; - (tokenizer, model_info) + } else { + // No API and no local model + return Err(RouterError::ArgumentValidation( + "No local model found and no revision specified".to_string(), + )); + }; + + // Load tokenizer config if found locally, or check if we can get it from the API if needed + let tokenizer_config = if local_tokenizer_config { + tracing::info!("Using local tokenizer config"); + HubTokenizerConfig::from_file(&local_tokenizer_config_path) + } else if let Some(api) = api { + tracing::info!("Using the Hugging Face API to retrieve tokenizer config"); + get_tokenizer_config(&api.repo(Repo::with_revision( + tokenizer_name.to_string(), + RepoType::Model, + revision.unwrap_or_else(|| "main".to_string()), + ))) + .await + .unwrap_or_else(|| { + tracing::warn!("Could not retrieve tokenizer config from the Hugging Face hub."); + HubTokenizerConfig::default() + }) + } else { + tracing::warn!("Could not find tokenizer config locally and no revision specified"); + HubTokenizerConfig::default() }; if tokenizer.is_none() { @@ -297,6 +344,7 @@ async fn main() -> Result<(), RouterError> { ngrok, ngrok_authtoken, ngrok_edge, + tokenizer_config, ) .await?; Ok(()) @@ -401,6 +449,20 @@ pub async fn get_base_tokenizer(api: &Api, api_repo: &ApiRepo) -> Option Option { + let tokenizer_config_filename = api_repo.get("tokenizer_config.json").await.ok()?; + + // Open the file in read-only mode with buffer. + let file = File::open(tokenizer_config_filename).ok()?; + let reader = BufReader::new(file); + + // Read the JSON contents of the file as an instance of 'HubTokenizerConfig'. + let tokenizer_config: HubTokenizerConfig = serde_json::from_reader(reader).ok()?; + + Some(tokenizer_config) +} + #[derive(Debug, Error)] enum RouterError { #[error("Argument validation error: {0}")] diff --git a/router/src/server.rs b/router/src/server.rs index 3db5c7cd..fe1827c4 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -2,10 +2,11 @@ use crate::health::Health; use crate::infer::{InferError, InferResponse, InferStreamResponse}; use crate::validation::ValidationError; +use crate::HubTokenizerConfig; use crate::{ - BestOfSequence, CompatGenerateRequest, Details, ErrorResponse, FinishReason, - GenerateParameters, GenerateRequest, GenerateResponse, HubModelInfo, Infer, Info, PrefillToken, - StreamDetails, StreamResponse, Token, Validation, + BestOfSequence, ChatCompletion, ChatCompletionChunk, ChatRequest, CompatGenerateRequest, + Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest, GenerateResponse, + HubModelInfo, Infer, Info, PrefillToken, StreamDetails, StreamResponse, Token, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; @@ -343,6 +344,21 @@ async fn generate_stream( HeaderMap, Sse>>, ) { + let on_message_callback = |stream_token: StreamResponse| { + let event = Event::default(); + event.json_data(stream_token).unwrap() + }; + let (headers, response_stream) = + generate_stream_internal(infer, Json(req), on_message_callback).await; + let sse = Sse::new(response_stream).keep_alive(KeepAlive::default()); + (headers, sse) +} + +async fn generate_stream_internal( + infer: Infer, + Json(req): Json, + on_message_callback: impl Fn(StreamResponse) -> Event, +) -> (HeaderMap, impl Stream>) { let span = tracing::Span::current(); let start_time = Instant::now(); metrics::increment_counter!("tgi_request_count"); @@ -385,8 +401,10 @@ async fn generate_stream( match infer.generate_stream(req).instrument(info_span!(parent: &span, "async_stream")).await { // Keep permit as long as generate_stream lives Ok((_permit, _input_length, mut response_stream)) => { + let mut index = 0; // Server-Sent Event stream while let Some(response) = response_stream.next().await { + index += 1; match response { Ok(response) => { match response { @@ -401,13 +419,14 @@ async fn generate_stream( // StreamResponse let stream_token = StreamResponse { + index, token, top_tokens, generated_text: None, details: None, }; - - yield Ok(Event::default().json_data(stream_token).unwrap()) + let event = on_message_callback(stream_token); + yield Ok(event); } // Yield event for last token and compute timings InferStreamResponse::End { @@ -463,13 +482,16 @@ async fn generate_stream( tracing::info!(parent: &span, "Success"); let stream_token = StreamResponse { + index, token, top_tokens, generated_text: Some(output_text), details }; - yield Ok(Event::default().json_data(stream_token).unwrap()); + + let event = on_message_callback(stream_token); + yield Ok(event); break; } } @@ -500,7 +522,154 @@ async fn generate_stream( } }; - (headers, Sse::new(stream).keep_alive(KeepAlive::default())) + (headers, stream) +} + +/// Generate tokens +#[utoipa::path( + post, + tag = "Text Generation Inference", + path = "/v1/chat/completions", + request_body = ChatRequest, + responses( + (status = 200, description = "Generated Text", body = GenerateResponse), + (status = 424, description = "Generation Error", body = ErrorResponse, + example = json ! ({"error": "Request failed during generation"})), + (status = 429, description = "Model is overloaded", body = ErrorResponse, + example = json ! ({"error": "Model is overloaded"})), + (status = 422, description = "Input validation error", body = ErrorResponse, + example = json ! ({"error": "Input validation error"})), + (status = 500, description = "Incomplete generation", body = ErrorResponse, + example = json ! ({"error": "Incomplete generation"})), + ) + )] +#[instrument( + skip_all, + fields( + // parameters = ? req.parameters, + total_time, + validation_time, + queue_time, + inference_time, + time_per_token, + seed, + ) + )] +async fn chat_completions( + Extension(infer): Extension, + Extension(info): Extension, + Json(req): Json, +) -> Result)> { + metrics::increment_counter!("tgi_request_count"); + + let stream = req.stream; + let max_new_tokens = req.max_tokens.or(Some(100)); + let repetition_penalty = req + .frequency_penalty + // rescale frequency_penalty from (-2.0, 2.0) to (0.0, 4.0) + .map(|x| x + 2.0); + let logprobs = req.logprobs.unwrap_or(false); + let seed = req.seed; + + // apply chat template to flatten the request into a single input + let inputs = match infer.apply_chat_template(req) { + Ok(inputs) => inputs, + Err(err) => { + metrics::increment_counter!("tgi_request_failure", "err" => "validation"); + tracing::error!("{err}"); + return Err(( + StatusCode::UNPROCESSABLE_ENTITY, + Json(ErrorResponse { + error: err.to_string(), + error_type: err.error_type().to_string(), + }), + )); + } + }; + + // build the request passing some parameters + let generate_request = GenerateRequest { + inputs: inputs.to_string(), + parameters: GenerateParameters { + best_of: None, + temperature: None, + repetition_penalty, + top_k: None, + top_p: None, + typical_p: None, + do_sample: true, + max_new_tokens, + return_full_text: None, + stop: Vec::new(), + truncate: None, + watermark: false, + details: true, + decoder_input_details: true, + seed, + top_n_tokens: None, + }, + }; + + // static values that will be returned in all cases + let model_id = info.model_id.clone(); + let system_fingerprint = format!("{}-{}", info.version, info.docker_label.unwrap_or("native")); + + // switch on stream + if stream { + // pass this callback to the stream generation and build the required event structure + let on_message_callback = move |stream_token: StreamResponse| { + let event = Event::default(); + + let current_time = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_else(|_| std::time::Duration::from_secs(0)) + .as_secs(); + + event + .json_data(ChatCompletionChunk::new( + model_id.clone(), + system_fingerprint.clone(), + stream_token.token.text, + current_time, + stream_token.index, + logprobs.then_some(stream_token.token.logprob), + stream_token.details.map(|d| d.finish_reason.to_string()), + )) + .map_or_else( + |e| { + println!("Failed to serialize ChatCompletionChunk: {:?}", e); + Event::default() + }, + |data| data, + ) + }; + + let (headers, response_stream) = + generate_stream_internal(infer, Json(generate_request), on_message_callback).await; + let sse = Sse::new(response_stream).keep_alive(KeepAlive::default()); + Ok((headers, sse).into_response()) + } else { + let (headers, Json(generation)) = + generate(Extension(infer), Json(generate_request)).await?; + + let current_time = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_else(|_| std::time::Duration::from_secs(0)) + .as_secs(); + + // build the complete response object with the full text + let response = ChatCompletion::new( + generation.generated_text, + model_id, + system_fingerprint, + current_time, + generation.details.unwrap(), + logprobs, + ); + + // wrap generation inside a Vec to match api-inference + Ok((headers, Json(response)).into_response()) + } } /// Prometheus metrics scrape endpoint @@ -538,6 +707,7 @@ pub async fn run( ngrok: bool, ngrok_authtoken: Option, ngrok_edge: Option, + tokenizer_config: HubTokenizerConfig, ) -> Result<(), axum::BoxError> { // OpenAPI documentation #[derive(OpenApi)] @@ -604,6 +774,7 @@ pub async fn run( shard_info.window_size, shard_info.speculate, generation_health, + tokenizer_config, ); // Duration buckets @@ -693,6 +864,7 @@ pub async fn run( .route("/info", get(get_model_info)) .route("/generate", post(generate)) .route("/generate_stream", post(generate_stream)) + .route("/v1/chat/completions", post(chat_completions)) // AWS Sagemaker route .route("/invocations", post(compat_generate)) // Base Health route @@ -822,6 +994,7 @@ impl From for (StatusCode, Json) { InferError::Overloaded(_) => StatusCode::TOO_MANY_REQUESTS, InferError::ValidationError(_) => StatusCode::UNPROCESSABLE_ENTITY, InferError::IncompleteGeneration => StatusCode::INTERNAL_SERVER_ERROR, + InferError::TemplateError(_) => StatusCode::UNPROCESSABLE_ENTITY, }; ( diff --git a/router/src/validation.rs b/router/src/validation.rs index 64f25c82..370e9588 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -376,7 +376,7 @@ type TokenizerRequest = ( Span, ); -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct ValidGenerateRequest { pub inputs: String, pub input_length: u32, From 3ccb3bb0b54a6ea51d7b8c3dbf8551f4247899e8 Mon Sep 17 00:00:00 2001 From: drbh Date: Thu, 18 Jan 2024 06:31:56 -0500 Subject: [PATCH 521/793] feat: support raise_exception, bos and eos tokens (#1450) This PR adds support to handle the custom jinja function `raise_exception` and passes the `bos` and `eos` tokens into the template Additionally this PR adds 3 tests to validate and show examples of what can and cannot be parsed currently. ```bash cargo test --package text-generation-router --lib -- infer::tests --nocapture # Finished test [unoptimized + debuginfo] target(s) in 7.82s # Running unittests src/lib.rs (target/debug/deps/text_generation_router-18a0bbf99c2ca1b4) # running 3 tests # test infer::tests::test_chat_template_valid_with_raise ... ok # test infer::tests::test_chat_template ... ok # test infer::tests::test_chat_template_invalid_with_raise ... ok # test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 15 filtered out; finished in 0.00s ``` --- router/src/infer.rs | 247 +++++++++++++++++++++++++++++++++++++++++-- router/src/lib.rs | 10 +- router/src/server.rs | 8 +- 3 files changed, 249 insertions(+), 16 deletions(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index 6de07982..8a9875eb 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -1,8 +1,9 @@ /// Batching and inference logic use crate::validation::{Validation, ValidationError}; -use crate::HubTokenizerConfig; -use crate::{ChatRequest, GenerateRequest, GenerateStreamResponse, PrefillToken}; -use crate::{Entry, Queue, Token}; +use crate::{ + ChatTemplateInputs, Entry, GenerateRequest, GenerateStreamResponse, HubTokenizerConfig, + Message, PrefillToken, Queue, Token, +}; use futures::future::try_join_all; use minijinja::{Environment, ErrorKind, Template}; use nohash_hasher::IntMap; @@ -32,8 +33,12 @@ pub struct Infer { shared: Arc, /// Inference limit limit_concurrent_requests: Arc, - /// Chat template - template: Option>, + /// Chat template (template, bos_token, eos_token) + template: ( + Option>, + Option, + Option, + ), } /// Infer shared state @@ -42,6 +47,11 @@ struct Shared { batching_task: Notify, } +/// Raise a exception (custom function) used in the chat templates +fn raise_exception(err_text: String) -> Result { + Err(minijinja::Error::new(ErrorKind::SyntaxError, err_text)) +} + impl Infer { #[allow(clippy::too_many_arguments)] pub(crate) fn new( @@ -80,20 +90,28 @@ impl Infer { let semaphore = Arc::new(Semaphore::new(max_concurrent_requests)); let template = tokenizer_config.chat_template.map(|t| { - let env = Box::new(Environment::new()); + let mut env = Box::new(Environment::new()); let template_str = t.into_boxed_str(); + env.add_function("raise_exception", raise_exception); // leaking env and template_str as read-only, static resources for performance. Box::leak(env) .template_from_str(Box::leak(template_str)) .unwrap() }); - + let eos_token = tokenizer_config + .eos_token + .map_or_else(String::new, |t| t) + .into(); + let bos_token = tokenizer_config + .bos_token + .map_or_else(String::new, |t| t) + .into(); Self { validation, queue, shared, limit_concurrent_requests: semaphore, - template, + template: (template, eos_token, bos_token), } } @@ -149,11 +167,16 @@ impl Infer { /// Apply the chat template to the chat request #[instrument(skip_all)] - pub(crate) fn apply_chat_template(&self, chat: ChatRequest) -> Result { - self.template + pub(crate) fn apply_chat_template(&self, messages: Vec) -> Result { + let (template, bos_token, eos_token) = &self.template; + template .as_ref() .ok_or_else(|| InferError::TemplateError(ErrorKind::TemplateNotFound.into()))? - .render(chat) + .render(ChatTemplateInputs { + messages, + eos_token: eos_token.as_deref(), + bos_token: bos_token.as_deref(), + }) .map_err(|e| { metrics::increment_counter!("tgi_request_failure", "err" => "template"); tracing::error!("{e}"); @@ -702,3 +725,205 @@ impl InferError { } } } + +// tests +#[cfg(test)] +mod tests { + use crate::infer::raise_exception; + use crate::ChatTemplateInputs; + use crate::Message; + use minijinja::Environment; + + #[test] + fn test_chat_template() { + let env = Environment::new(); + + let source = r#" + {% for message in messages %} + {% if message['role'] == 'system' %} + {% if message['content']%} + {{'### System:\n' + message['content']+'\n\n'}} + {% endif %} + {% elif message['role'] == 'user' %} + {{'### User:\n' + message['content']+'\n\n'}} + {% elif message['role'] == 'assistant' %} + {{'### Assistant:\n' + message['content']}} + {% endif %} + {% if loop.last and add_generation_prompt %} + {{ '### Assistant:\n' }} + {% endif %} + {% endfor %}"#; + + // trim all the whitespace + let source = source + .lines() + .map(|line| line.trim()) + .collect::>() + .join(""); + + let tmpl = env.template_from_str(&source); + + let chat_template_inputs = ChatTemplateInputs { + messages: vec![ + Message { + role: "user".to_string(), + content: "Hi!".to_string(), + }, + Message { + role: "assistant".to_string(), + content: "Hello how can I help?".to_string(), + }, + Message { + role: "user".to_string(), + content: "What is Deep Learning?".to_string(), + }, + Message { + role: "assistant".to_string(), + content: "magic!".to_string(), + }, + ], + bos_token: Some("[BOS]"), + eos_token: Some("[EOS]"), + }; + + let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); + + assert_eq!( + result, + r#"### User: +Hi! + +### Assistant: +Hello how can I help?### User: +What is Deep Learning? + +### Assistant: +magic!"# + ); + } + + #[test] + fn test_chat_template_invalid_with_raise() { + let mut env = Environment::new(); + env.add_function("raise_exception", raise_exception); + + let source = r#" + {{ bos_token }} + {% for message in messages %} + {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %} + {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }} + {% endif %} + {% if message['role'] == 'user' %} + {{ '[INST] ' + message['content'] + ' [/INST]' }} + {% elif message['role'] == 'assistant' %} + {{ message['content'] + eos_token}} + {% else %} + {{ raise_exception('Only user and assistant roles are supported!') }} + {% endif %} + {% endfor %}"#; + + // trim all the whitespace + let source = source + .lines() + .map(|line| line.trim()) + .collect::>() + .join(""); + + let tmpl = env.template_from_str(&source); + + let chat_template_inputs = ChatTemplateInputs { + messages: vec![ + Message { + role: "user".to_string(), + content: "Hi!".to_string(), + }, + Message { + role: "user".to_string(), + content: "Hi again!".to_string(), + }, + Message { + role: "assistant".to_string(), + content: "Hello how can I help?".to_string(), + }, + Message { + role: "user".to_string(), + content: "What is Deep Learning?".to_string(), + }, + Message { + role: "assistant".to_string(), + content: "magic!".to_string(), + }, + ], + bos_token: Some("[BOS]"), + eos_token: Some("[EOS]"), + }; + + let result = tmpl.unwrap().render(chat_template_inputs); //.err().unwrap(); + + match result { + Ok(_) => panic!("Should have failed"), + Err(e) => { + assert_eq!( + e.detail().unwrap(), + "Conversation roles must alternate user/assistant/user/assistant/..." + ); + } + } + } + + #[test] + fn test_chat_template_valid_with_raise() { + let mut env = Environment::new(); + env.add_function("raise_exception", raise_exception); + + let source = r#" + {{ bos_token }} + {% for message in messages %} + {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %} + {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }} + {% endif %} + {% if message['role'] == 'user' %} + {{ '[INST] ' + message['content'] + ' [/INST]' }} + {% elif message['role'] == 'assistant' %} + {{ message['content'] + eos_token}} + {% else %} + {{ raise_exception('Only user and assistant roles are supported!') }} + {% endif %} + {% endfor %}"#; + + // trim all the whitespace + let source = source + .lines() + .map(|line| line.trim()) + .collect::>() + .join(""); + + let tmpl = env.template_from_str(&source); + + let chat_template_inputs = ChatTemplateInputs { + messages: vec![ + Message { + role: "user".to_string(), + content: "Hi!".to_string(), + }, + Message { + role: "assistant".to_string(), + content: "Hello how can I help?".to_string(), + }, + Message { + role: "user".to_string(), + content: "What is Deep Learning?".to_string(), + }, + Message { + role: "assistant".to_string(), + content: "magic!".to_string(), + }, + ], + bos_token: Some("[BOS]"), + eos_token: Some("[EOS]"), + }; + + let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); + assert_eq!(result, "[BOS][INST] Hi! [/INST]Hello how can I help?[EOS][INST] What is Deep Learning? [/INST]magic![EOS]"); + } +} diff --git a/router/src/lib.rs b/router/src/lib.rs index f6f8276f..983079d6 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -31,8 +31,9 @@ pub struct HubModelInfo { #[derive(Clone, Deserialize, Default)] pub struct HubTokenizerConfig { - #[serde(default)] pub chat_template: Option, + pub bos_token: Option, + pub eos_token: Option, } impl HubTokenizerConfig { @@ -366,6 +367,13 @@ pub(crate) struct ChatRequest { pub seed: Option, } +#[derive(Clone, Serialize, Deserialize)] +pub(crate) struct ChatTemplateInputs<'a> { + messages: Vec, + bos_token: Option<&'a str>, + eos_token: Option<&'a str>, +} + #[derive(Clone, Deserialize, ToSchema, Serialize)] pub(crate) struct Message { #[schema(example = "user")] diff --git a/router/src/server.rs b/router/src/server.rs index fe1827c4..530a935b 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -2,11 +2,11 @@ use crate::health::Health; use crate::infer::{InferError, InferResponse, InferStreamResponse}; use crate::validation::ValidationError; -use crate::HubTokenizerConfig; use crate::{ BestOfSequence, ChatCompletion, ChatCompletionChunk, ChatRequest, CompatGenerateRequest, Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest, GenerateResponse, - HubModelInfo, Infer, Info, PrefillToken, StreamDetails, StreamResponse, Token, Validation, + HubModelInfo, HubTokenizerConfig, Infer, Info, PrefillToken, StreamDetails, StreamResponse, + Token, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; @@ -572,7 +572,7 @@ async fn chat_completions( let seed = req.seed; // apply chat template to flatten the request into a single input - let inputs = match infer.apply_chat_template(req) { + let inputs = match infer.apply_chat_template(req.messages) { Ok(inputs) => inputs, Err(err) => { metrics::increment_counter!("tgi_request_failure", "err" => "validation"); @@ -659,9 +659,9 @@ async fn chat_completions( // build the complete response object with the full text let response = ChatCompletion::new( - generation.generated_text, model_id, system_fingerprint, + generation.generated_text, current_time, generation.details.unwrap(), logprobs, From becd09978cc1f2651ff9062ab5a64c59fd64b2ef Mon Sep 17 00:00:00 2001 From: drbh Date: Mon, 22 Jan 2024 09:22:54 -0500 Subject: [PATCH 522/793] chore: bump rust version and annotate/fix all clippy warnings (#1455) This PR just bumps the latest rust version and makes clippy happy ```bash cargo clippy --all -- -D warnings # Finished dev [unoptimized + debuginfo] target(s) in 0.10s ``` --- Cargo.toml | 1 + benchmark/src/app.rs | 2 +- launcher/src/main.rs | 2 ++ router/client/src/client.rs | 2 +- router/client/src/sharded_client.rs | 2 ++ rust-toolchain.toml | 5 ++++- 6 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 24055497..80e6e145 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "router/grpc-metadata", "launcher" ] +resolver = "2" [workspace.package] version = "1.3.4" diff --git a/benchmark/src/app.rs b/benchmark/src/app.rs index 49654c1b..b27c56b4 100644 --- a/benchmark/src/app.rs +++ b/benchmark/src/app.rs @@ -466,7 +466,7 @@ fn latency_paragraph<'a>(latency: &mut Vec, name: &'static str) -> Paragrap let latency_percentiles = crate::utils::percentiles(latency, &[50, 90, 99]); // Latency p50/p90/p99 texts - let colors = vec![Color::LightGreen, Color::LightYellow, Color::LightRed]; + let colors = [Color::LightGreen, Color::LightYellow, Color::LightRed]; for (i, (name, value)) in latency_percentiles.iter().enumerate() { let span = Line::from(vec![Span::styled( format!("{name}: {value:.2} ms"), diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 4e230205..c5553000 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -53,6 +53,8 @@ impl std::fmt::Display for Quantization { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // To keep in track with `server`. match self { + #[allow(deprecated)] + // Use `eetq` instead, which provides better latencies overall and is drop-in in most cases Quantization::Bitsandbytes => { write!(f, "bitsandbytes") } diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 4723d664..023c5671 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -224,7 +224,7 @@ pub struct DecodeTimings { impl DecodeTimings { fn new(concat_ns: Option, forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self { Self { - concat: concat_ns.map(|v| Duration::from_nanos(v)), + concat: concat_ns.map(Duration::from_nanos), forward: Duration::from_nanos(forward_ns), decode: Duration::from_nanos(decode_ns), total: Duration::from_nanos(total_ns), diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index 6c5da3c7..f0e65ce5 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -127,6 +127,7 @@ impl ShardedClient { .iter_mut() .map(|client| Box::pin(client.prefill(batch.clone()))) .collect(); + #[allow(clippy::type_complexity)] let results: Result, Option, PrefillTimings)>> = join_all(futures).await.into_iter().collect(); let mut results = results?; @@ -159,6 +160,7 @@ impl ShardedClient { .iter_mut() .map(|client| Box::pin(client.decode(batches.clone()))) .collect(); + #[allow(clippy::type_complexity)] let results: Result, Option, DecodeTimings)>> = join_all(futures).await.into_iter().collect(); let mut results = results?; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 2db1883c..313c018c 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,6 @@ [toolchain] -channel = "1.70.0" +# Released on: 28 December, 2023 +# Branched from master on: 10 November, 2023 +# https://releases.rs/docs/1.75.0/ +channel = "1.75.0" components = ["rustfmt", "clippy"] \ No newline at end of file From 98e5faff9daec6170cc2b0f963f2d73cf846b341 Mon Sep 17 00:00:00 2001 From: drbh Date: Mon, 22 Jan 2024 10:29:01 -0500 Subject: [PATCH 523/793] feat: conditionally toggle chat on invocations route (#1454) This PR adds support for reading the `OAI_ENABLED` env var which will changes the function called when the `/invocations` is called. If `OAI_ENABLED=true` the `chat_completions` method is used otherwise it defaults to `compat_generate`. example running the router ```bash OAI_ENABLED=true \ cargo run -- \ --tokenizer-name mistralai/Mistral-7B-Instruct-v0.2 ``` example request ```bash curl localhost:3000/invocations \ -X POST \ -d '{ "model": "tgi", "messages": [ { "role": "user", "content": "What is the IP address of the Google DNS servers?" } ], "stream": false, "max_tokens": 20, "logprobs": true, "seed": 0 }' \ -H 'Content-Type: application/json' | jq ``` **please let me know if any naming changes are needed or if any other routes need similar functionality. --- router/src/main.rs | 4 ++++ router/src/server.rs | 32 ++++++++++++++++++++------------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/router/src/main.rs b/router/src/main.rs index f5d44305..bf987eb6 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -71,6 +71,8 @@ struct Args { ngrok_authtoken: Option, #[clap(long, env)] ngrok_edge: Option, + #[clap(long, env, default_value_t = false)] + chat_enabled_api: bool, } #[tokio::main] @@ -102,6 +104,7 @@ async fn main() -> Result<(), RouterError> { ngrok, ngrok_authtoken, ngrok_edge, + chat_enabled_api, } = args; // Launch Tokio runtime @@ -345,6 +348,7 @@ async fn main() -> Result<(), RouterError> { ngrok_authtoken, ngrok_edge, tokenizer_config, + chat_enabled_api, ) .await?; Ok(()) diff --git a/router/src/server.rs b/router/src/server.rs index 530a935b..cf1d94a6 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -708,6 +708,7 @@ pub async fn run( ngrok_authtoken: Option, ngrok_edge: Option, tokenizer_config: HubTokenizerConfig, + chat_enabled_api: bool, ) -> Result<(), axum::BoxError> { // OpenAPI documentation #[derive(OpenApi)] @@ -856,25 +857,32 @@ pub async fn run( docker_label: option_env!("DOCKER_LABEL"), }; - // Create router - let app = Router::new() - .merge(SwaggerUi::new("/docs").url("/api-doc/openapi.json", ApiDoc::openapi())) - // Base routes + // Configure Swagger UI + let swagger_ui = SwaggerUi::new("/docs").url("/api-doc/openapi.json", ApiDoc::openapi()); + + // Define base and health routes + let base_routes = Router::new() .route("/", post(compat_generate)) .route("/info", get(get_model_info)) .route("/generate", post(generate)) .route("/generate_stream", post(generate_stream)) .route("/v1/chat/completions", post(chat_completions)) - // AWS Sagemaker route - .route("/invocations", post(compat_generate)) - // Base Health route .route("/health", get(health)) - // Inference API health route - .route("/", get(health)) - // AWS Sagemaker health route .route("/ping", get(health)) - // Prometheus metrics route - .route("/metrics", get(metrics)) + .route("/metrics", get(metrics)); + + // Conditional AWS Sagemaker route + let aws_sagemaker_route = if chat_enabled_api { + Router::new().route("/invocations", post(chat_completions)) // Use 'chat_completions' for OAI_ENABLED + } else { + Router::new().route("/invocations", post(compat_generate)) // Use 'compat_generate' otherwise + }; + + // Combine routes and layers + let app = Router::new() + .merge(swagger_ui) + .merge(base_routes) + .merge(aws_sagemaker_route) .layer(Extension(info)) .layer(Extension(health_ext.clone())) .layer(Extension(compat_return_full_text)) From 82f87ada6f08114ae198abb0829d087f311cf5bc Mon Sep 17 00:00:00 2001 From: Jacob Keisling Date: Tue, 23 Jan 2024 08:55:05 -0600 Subject: [PATCH 524/793] Disable `decoder_input_details` on OpenAI-compatible chat streaming, pass temp and top-k from API (#1470) This PR makes some minor tweaks to the new OpenAI-compatible chat endpoint #1427 in `GenerateParameters`: - Disables `decoder_input_details` when streaming is enabled. This was causing all streaming chat requests to fail before, since [`decoder_input_details`==true is not enabled when streaming tokens](https://github.com/huggingface/text-generation-inference/blob/98e5faff9daec6170cc2b0f963f2d73cf846b341/router/src/validation.rs#L406). - Passes through `temperature` and `top_p` hyperparameters from the API request to `GenerateParameters` ## Testing ```bash curl localhost:8080/v1/chat/completions \ -X POST \ -d '{ "model": "", "messages": [ { "role": "system", "content": "You are a helpful assistant." }, { "role": "user", "content": "What is deep learning?" } ], "stream": true, "max_tokens": 20 }' \ -H 'Content-Type: application/json' ``` Should work correctly. Currently, most recent release from `main` returns error: ``` data:{"error":"Input validation error: `decoder_input_details` == true is not supported when streaming tokens","error_type":"validation"} ``` It's my first time contributing to this project, so I could be missing something. Would especially appreciate @drbh's eyes on this one --- router/src/lib.rs | 12 ++++++++++++ router/src/server.rs | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index 983079d6..894ab466 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -365,6 +365,18 @@ pub(crate) struct ChatRequest { #[schema(nullable = true, example = 42)] pub seed: Option, + + /// What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while + /// lower values like 0.2 will make it more focused and deterministic. + /// + /// We generally recommend altering this or `top_p` but not both. + #[serde(default)] + pub temperature: Option, + + /// An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the + /// tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. + #[serde(default)] + pub top_p: Option, } #[derive(Clone, Serialize, Deserialize)] diff --git a/router/src/server.rs b/router/src/server.rs index cf1d94a6..aa1ad202 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -592,10 +592,10 @@ async fn chat_completions( inputs: inputs.to_string(), parameters: GenerateParameters { best_of: None, - temperature: None, + temperature: req.temperature, repetition_penalty, top_k: None, - top_p: None, + top_p: req.top_p, typical_p: None, do_sample: true, max_new_tokens, @@ -604,7 +604,7 @@ async fn chat_completions( truncate: None, watermark: false, details: true, - decoder_input_details: true, + decoder_input_details: !stream, seed, top_n_tokens: None, }, From 7e542d4d05513575f9eb950f961c5b4c574c9c29 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 24 Jan 2024 13:08:41 +0100 Subject: [PATCH 525/793] Fixing non divisible embeddings. (#1476) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../test_idefics/test_idefics.json | 55 +++-- .../test_idefics/test_idefics_load.json | 214 +++++++++--------- server/tests/utils/test_layers.py | 64 ++++++ server/text_generation_server/utils/layers.py | 4 +- .../text_generation_server/utils/weights.py | 2 +- 5 files changed, 199 insertions(+), 140 deletions(-) create mode 100644 server/tests/utils/test_layers.py diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json index 2c5d05f6..90fb6dcc 100644 --- a/integration-tests/models/__snapshots__/test_idefics/test_idefics.json +++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics.json @@ -11,92 +11,92 @@ }, { "id": 4911, - "logprob": -5.7851562, + "logprob": -6.9765625, "text": "User" }, { "id": 29901, - "logprob": -0.006996155, + "logprob": -0.0059432983, "text": ":" }, { "id": 32000, - "logprob": -0.81347656, + "logprob": -0.8408203, "text": "" }, { "id": 32001, - "logprob": -6.687641e-05, + "logprob": -9.906292e-05, "text": "" }, { "id": 32000, - "logprob": -3.5762787e-07, + "logprob": -2.3841858e-07, "text": "" }, { "id": 1815, - "logprob": -4.2148438, + "logprob": -4.1679688, "text": "Can" }, { "id": 366, - "logprob": -0.014137268, + "logprob": -0.014099121, "text": "you" }, { "id": 2649, - "logprob": -4.4335938, + "logprob": -4.4609375, "text": "tell" }, { "id": 592, - "logprob": -0.2919922, + "logprob": -0.29882812, "text": "me" }, { "id": 263, - "logprob": -4.2070312, + "logprob": -4.1445312, "text": "a" }, { "id": 1407, - "logprob": -9.421875, + "logprob": -9.3828125, "text": "very" }, { "id": 3273, - "logprob": -1.8720703, + "logprob": -1.9736328, "text": "short" }, { "id": 5828, - "logprob": -0.26489258, + "logprob": -0.2800293, "text": "story" }, { "id": 2729, - "logprob": -3.7441406, + "logprob": -3.5625, "text": "based" }, { "id": 373, - "logprob": -0.0005393028, + "logprob": -0.0006427765, "text": "on" }, { "id": 278, - "logprob": -0.140625, + "logprob": -0.13952637, "text": "the" }, { "id": 1967, - "logprob": -0.06756592, + "logprob": -0.068115234, "text": "image" }, { "id": 29973, - "logprob": -0.15454102, + "logprob": -0.16357422, "text": "?" } ], @@ -104,25 +104,25 @@ "tokens": [ { "id": 32002, - "logprob": -0.0019140244, + "logprob": -0.0026474, "special": true, "text": "" }, { "id": 29871, - "logprob": -8.404255e-05, + "logprob": -8.547306e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -1.7642975e-05, + "logprob": -1.7881393e-05, "special": false, "text": "\n" }, { "id": 7900, - "logprob": -2.9802322e-06, + "logprob": -3.0994415e-06, "special": false, "text": "Ass" }, @@ -140,30 +140,29 @@ }, { "id": 319, - "logprob": -0.91064453, + "logprob": -0.92529297, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.2412109, + "logprob": -1.1269531, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.0002439022, + "logprob": -0.00029492378, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1630859, + "logprob": -1.1855469, "special": false, "text": " stands" } - ], - "top_tokens": null + ] }, "generated_text": " \nAssistant: A rooster stands" } diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json index f258e38d..21d6161b 100644 --- a/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json +++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics_load.json @@ -12,92 +12,92 @@ }, { "id": 4911, - "logprob": -5.7851562, + "logprob": -6.9804688, "text": "User" }, { "id": 29901, - "logprob": -0.006996155, + "logprob": -0.006122589, "text": ":" }, { "id": 32000, - "logprob": -0.81347656, + "logprob": -0.8417969, "text": "" }, { "id": 32001, - "logprob": -6.687641e-05, + "logprob": -9.918213e-05, "text": "" }, { "id": 32000, - "logprob": -3.5762787e-07, + "logprob": -2.3841858e-07, "text": "" }, { "id": 1815, - "logprob": -4.2148438, + "logprob": -4.1679688, "text": "Can" }, { "id": 366, - "logprob": -0.014137268, + "logprob": -0.014091492, "text": "you" }, { "id": 2649, - "logprob": -4.4335938, + "logprob": -4.4726562, "text": "tell" }, { "id": 592, - "logprob": -0.2919922, + "logprob": -0.2998047, "text": "me" }, { "id": 263, - "logprob": -4.2070312, + "logprob": -4.15625, "text": "a" }, { "id": 1407, - "logprob": -9.421875, + "logprob": -9.3828125, "text": "very" }, { "id": 3273, - "logprob": -1.8720703, + "logprob": -1.9716797, "text": "short" }, { "id": 5828, - "logprob": -0.26489258, + "logprob": -0.27734375, "text": "story" }, { "id": 2729, - "logprob": -3.7441406, + "logprob": -3.5605469, "text": "based" }, { "id": 373, - "logprob": -0.0005393028, + "logprob": -0.00064468384, "text": "on" }, { "id": 278, - "logprob": -0.140625, + "logprob": -0.14160156, "text": "the" }, { "id": 1967, - "logprob": -0.06756592, + "logprob": -0.06915283, "text": "image" }, { "id": 29973, - "logprob": -0.15454102, + "logprob": -0.16381836, "text": "?" } ], @@ -105,19 +105,19 @@ "tokens": [ { "id": 32002, - "logprob": -0.0019140244, + "logprob": -0.0026664734, "special": true, "text": "" }, { "id": 29871, - "logprob": -8.392334e-05, + "logprob": -8.583069e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -1.7881393e-05, + "logprob": -1.8119812e-05, "special": false, "text": "\n" }, @@ -135,36 +135,35 @@ }, { "id": 29901, - "logprob": -3.0994415e-06, + "logprob": -3.2186508e-06, "special": false, "text": ":" }, { "id": 319, - "logprob": -0.9057617, + "logprob": -0.9301758, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.2294922, + "logprob": -1.1279297, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.00024533272, + "logprob": -0.0002939701, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1640625, + "logprob": -1.1865234, "special": false, "text": " stands" } - ], - "top_tokens": null + ] }, "generated_text": " \nAssistant: A rooster stands" }, @@ -181,92 +180,92 @@ }, { "id": 4911, - "logprob": -5.7773438, + "logprob": -6.9804688, "text": "User" }, { "id": 29901, - "logprob": -0.0070114136, + "logprob": -0.006122589, "text": ":" }, { "id": 32000, - "logprob": -0.8208008, + "logprob": -0.8417969, "text": "" }, { "id": 32001, - "logprob": -6.699562e-05, + "logprob": -9.942055e-05, "text": "" }, { "id": 32000, - "logprob": -3.5762787e-07, + "logprob": -2.3841858e-07, "text": "" }, { "id": 1815, - "logprob": -4.2265625, + "logprob": -4.1679688, "text": "Can" }, { "id": 366, - "logprob": -0.014175415, + "logprob": -0.014091492, "text": "you" }, { "id": 2649, - "logprob": -4.4296875, + "logprob": -4.4726562, "text": "tell" }, { "id": 592, - "logprob": -0.29516602, + "logprob": -0.2998047, "text": "me" }, { "id": 263, - "logprob": -4.2109375, + "logprob": -4.15625, "text": "a" }, { "id": 1407, - "logprob": -9.4296875, + "logprob": -9.3828125, "text": "very" }, { "id": 3273, - "logprob": -1.8720703, + "logprob": -1.9716797, "text": "short" }, { "id": 5828, - "logprob": -0.26879883, + "logprob": -0.27734375, "text": "story" }, { "id": 2729, - "logprob": -3.7675781, + "logprob": -3.5605469, "text": "based" }, { "id": 373, - "logprob": -0.0005354881, + "logprob": -0.0006451607, "text": "on" }, { "id": 278, - "logprob": -0.13671875, + "logprob": -0.14160156, "text": "the" }, { "id": 1967, - "logprob": -0.06719971, + "logprob": -0.06915283, "text": "image" }, { "id": 29973, - "logprob": -0.15551758, + "logprob": -0.16381836, "text": "?" } ], @@ -274,19 +273,19 @@ "tokens": [ { "id": 32002, - "logprob": -0.0019130707, + "logprob": -0.0026664734, "special": true, "text": "" }, { "id": 29871, - "logprob": -8.392334e-05, + "logprob": -8.571148e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -1.7881393e-05, + "logprob": -1.8119812e-05, "special": false, "text": "\n" }, @@ -310,30 +309,29 @@ }, { "id": 319, - "logprob": -0.9013672, + "logprob": -0.9301758, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.2324219, + "logprob": -1.1279297, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.0002477169, + "logprob": -0.0002939701, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1660156, + "logprob": -1.1865234, "special": false, "text": " stands" } - ], - "top_tokens": null + ] }, "generated_text": " \nAssistant: A rooster stands" }, @@ -350,92 +348,92 @@ }, { "id": 4911, - "logprob": -5.7773438, + "logprob": -6.9804688, "text": "User" }, { "id": 29901, - "logprob": -0.0070114136, + "logprob": -0.006122589, "text": ":" }, { "id": 32000, - "logprob": -0.8208008, + "logprob": -0.8417969, "text": "" }, { "id": 32001, - "logprob": -6.699562e-05, + "logprob": -9.918213e-05, "text": "" }, { "id": 32000, - "logprob": -3.5762787e-07, + "logprob": -2.3841858e-07, "text": "" }, { "id": 1815, - "logprob": -4.2265625, + "logprob": -4.1679688, "text": "Can" }, { "id": 366, - "logprob": -0.014175415, + "logprob": -0.014091492, "text": "you" }, { "id": 2649, - "logprob": -4.4296875, + "logprob": -4.4726562, "text": "tell" }, { "id": 592, - "logprob": -0.29516602, + "logprob": -0.2998047, "text": "me" }, { "id": 263, - "logprob": -4.2109375, + "logprob": -4.15625, "text": "a" }, { "id": 1407, - "logprob": -9.4296875, + "logprob": -9.3828125, "text": "very" }, { "id": 3273, - "logprob": -1.8720703, + "logprob": -1.9716797, "text": "short" }, { "id": 5828, - "logprob": -0.26879883, + "logprob": -0.27734375, "text": "story" }, { "id": 2729, - "logprob": -3.7675781, + "logprob": -3.5605469, "text": "based" }, { "id": 373, - "logprob": -0.0005354881, + "logprob": -0.00064468384, "text": "on" }, { "id": 278, - "logprob": -0.13671875, + "logprob": -0.14160156, "text": "the" }, { "id": 1967, - "logprob": -0.06719971, + "logprob": -0.06915283, "text": "image" }, { "id": 29973, - "logprob": -0.15551758, + "logprob": -0.16381836, "text": "?" } ], @@ -443,19 +441,19 @@ "tokens": [ { "id": 32002, - "logprob": -0.001912117, + "logprob": -0.0026664734, "special": true, "text": "" }, { "id": 29871, - "logprob": -8.392334e-05, + "logprob": -8.59499e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -1.7762184e-05, + "logprob": -1.8119812e-05, "special": false, "text": "\n" }, @@ -479,30 +477,29 @@ }, { "id": 319, - "logprob": -0.9013672, + "logprob": -0.9301758, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.2324219, + "logprob": -1.1279297, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.0002477169, + "logprob": -0.0002939701, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1660156, + "logprob": -1.1865234, "special": false, "text": " stands" } - ], - "top_tokens": null + ] }, "generated_text": " \nAssistant: A rooster stands" }, @@ -519,92 +516,92 @@ }, { "id": 4911, - "logprob": -5.7773438, + "logprob": -6.9804688, "text": "User" }, { "id": 29901, - "logprob": -0.0070114136, + "logprob": -0.006122589, "text": ":" }, { "id": 32000, - "logprob": -0.8208008, + "logprob": -0.8417969, "text": "" }, { "id": 32001, - "logprob": -6.699562e-05, + "logprob": -9.942055e-05, "text": "" }, { "id": 32000, - "logprob": -3.5762787e-07, + "logprob": -2.3841858e-07, "text": "" }, { "id": 1815, - "logprob": -4.2265625, + "logprob": -4.1679688, "text": "Can" }, { "id": 366, - "logprob": -0.014175415, + "logprob": -0.014091492, "text": "you" }, { "id": 2649, - "logprob": -4.4296875, + "logprob": -4.4726562, "text": "tell" }, { "id": 592, - "logprob": -0.29516602, + "logprob": -0.2998047, "text": "me" }, { "id": 263, - "logprob": -4.2109375, + "logprob": -4.15625, "text": "a" }, { "id": 1407, - "logprob": -9.4296875, + "logprob": -9.3828125, "text": "very" }, { "id": 3273, - "logprob": -1.8720703, + "logprob": -1.9716797, "text": "short" }, { "id": 5828, - "logprob": -0.26879883, + "logprob": -0.27734375, "text": "story" }, { "id": 2729, - "logprob": -3.7675781, + "logprob": -3.5605469, "text": "based" }, { "id": 373, - "logprob": -0.0005354881, + "logprob": -0.0006451607, "text": "on" }, { "id": 278, - "logprob": -0.13671875, + "logprob": -0.14160156, "text": "the" }, { "id": 1967, - "logprob": -0.06719971, + "logprob": -0.06915283, "text": "image" }, { "id": 29973, - "logprob": -0.15551758, + "logprob": -0.16381836, "text": "?" } ], @@ -612,19 +609,19 @@ "tokens": [ { "id": 32002, - "logprob": -0.001912117, + "logprob": -0.0026664734, "special": true, "text": "" }, { "id": 29871, - "logprob": -8.392334e-05, + "logprob": -8.571148e-05, "special": false, "text": " " }, { "id": 13, - "logprob": -1.7762184e-05, + "logprob": -1.8119812e-05, "special": false, "text": "\n" }, @@ -648,30 +645,29 @@ }, { "id": 319, - "logprob": -0.9013672, + "logprob": -0.9301758, "special": false, "text": " A" }, { "id": 696, - "logprob": -1.2324219, + "logprob": -1.1279297, "special": false, "text": " ro" }, { "id": 15664, - "logprob": -0.0002477169, + "logprob": -0.0002939701, "special": false, "text": "oster" }, { "id": 15028, - "logprob": -1.1660156, + "logprob": -1.1865234, "special": false, "text": " stands" } - ], - "top_tokens": null + ] }, "generated_text": " \nAssistant: A rooster stands" } diff --git a/server/tests/utils/test_layers.py b/server/tests/utils/test_layers.py new file mode 100644 index 00000000..0a9fecd1 --- /dev/null +++ b/server/tests/utils/test_layers.py @@ -0,0 +1,64 @@ +import torch +from text_generation_server.utils.layers import ( + TensorParallelEmbedding, +) + +class ProcessGroup: + def __init__(self, rank: int, world_size: int): + self._rank = rank + self.world_size = world_size + + def size(self)->int: + return self.world_size + + def rank(self)->int: + return self._rank + +class Weights: + def __init__(self, rank: int, world_size: int, vocab_size: int, hidden_dim: int): + self.weight = torch.arange(vocab_size*hidden_dim).float().view(vocab_size, hidden_dim) + self.process_group = ProcessGroup(rank, world_size) + + + def get_partial_sharded(self, name:str, dim: int): + assert dim == 0 + + rank = self.process_group.rank() + world_size = self.process_group.size() + size = self.weight.shape[dim] + + block_size = (size + world_size - 1) // world_size + start = rank * block_size + stop = (rank + 1) * block_size + return self.weight[start:stop] + + def get_shape(self, name: str): + return self.weight.shape + +def test_weight_hub_files_offline_error(): + + vocab_size= 17 + weights = Weights(rank=0, world_size=1, vocab_size = vocab_size,hidden_dim = 256) + embeddings = TensorParallelEmbedding("", weights) + + input_ids = torch.arange(vocab_size) + output = embeddings.forward(input_ids) + assert embeddings.min_id == 0 + assert embeddings.max_id == 17 + torch.testing.assert_close(output, torch.arange(256 * 17).float().view(17, 256)) + + weights_0_2 = Weights(rank=0, world_size=2, vocab_size = vocab_size,hidden_dim = 256) + weights_1_2 = Weights(rank=1, world_size=2, vocab_size = vocab_size,hidden_dim = 256) + embeddings_0_2 = TensorParallelEmbedding("", weights_0_2, reduce=False) + assert embeddings_0_2.min_id == 0 + assert embeddings_0_2.max_id == 9 + torch.testing.assert_close(embeddings_0_2.weight , torch.cat([torch.arange(9 * 256), torch.zeros(256)], dim=0).view(10, 256).float()) + embeddings_1_2 = TensorParallelEmbedding("", weights_1_2, reduce=False) + assert embeddings_1_2.min_id == 9 + assert embeddings_1_2.max_id == 17 + torch.testing.assert_close(embeddings_1_2.weight , torch.cat([torch.arange(8 * 256) + 9 * 256, torch.zeros(256)], dim=0).view(9, 256).float()) + output_tp_0 = embeddings_0_2.forward(input_ids) + output_tp_1 = embeddings_1_2.forward(input_ids) + + torch.testing.assert_close(output, output_tp_0 + output_tp_1) + diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index d4fa2559..5a0de0d7 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -507,10 +507,10 @@ def __init__(self, prefix: str, weights, reduce=True): world_size = process_group.size() rank = process_group.rank() - block_size = num_embeddings // world_size + block_size = (num_embeddings + world_size - 1) // world_size self.min_id = rank * block_size self.max_id = min(num_embeddings, (rank + 1) * block_size) - self.null_idx = block_size + self.null_idx = weight.shape[0] # Usually block_size, might be less in non even vocab_size. self.process_group = weights.process_group self.reduce = reduce diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index c4e82a6d..186733f3 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -92,7 +92,7 @@ def get_partial_sharded(self, tensor_name: str, dim: int): rank = self.process_group.rank() size = slice_.get_shape()[dim] - block_size = size // world_size + block_size = (size + world_size - 1) // world_size start = rank * block_size stop = (rank + 1) * block_size From 7872b8c55b6cdbf97e30ba6e4cd700f2de7e9bc4 Mon Sep 17 00:00:00 2001 From: drbh Date: Wed, 24 Jan 2024 11:41:28 -0500 Subject: [PATCH 526/793] Add messages api compatibility docs (#1478) This PR adds a new page to the docs that describes the Messages API and how to use it. Additionally this page will contain cloud provider specific information for enabling and using this feature. This PR includes a SageMaker example/information. --- docs/source/_toctree.yml | 2 + docs/source/messages_api.md | 134 ++++++++++++++++++++++++++++++++++++ router/src/main.rs | 6 +- router/src/server.rs | 4 +- 4 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 docs/source/messages_api.md diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 6fa50a6a..d57a594d 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -7,6 +7,8 @@ title: Installation - local: supported_models title: Supported Models and Hardware + - local: messages_api + title: Messages API title: Getting started - sections: - local: basic_tutorials/consuming_tgi diff --git a/docs/source/messages_api.md b/docs/source/messages_api.md new file mode 100644 index 00000000..899de865 --- /dev/null +++ b/docs/source/messages_api.md @@ -0,0 +1,134 @@ +# Messages API + +_Messages API is compatible to OpenAI Chat Completion API_ + +Text Generation Inference (TGI) now supports the Message API which is fully compatible with the OpenAI Chat Completion API. This means you can use OpenAI's client libraries to interact with TGI's Messages API. Below are some examples of how to utilize this compatibility. + +## Making a Request + +You can make a request to TGI's Messages API using `curl`. Here's an example: + +```bash +curl localhost:3000/v1/chat/completions \ + -X POST \ + -d '{ + "model": "tgi", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What is deep learning?" + } + ], + "stream": true, + "max_tokens": 20 +}' \ + -H 'Content-Type: application/json' +``` + +## Streaming + +You can also use OpenAI's Python client library to make a streaming request. Here's how: + +```python +from openai import OpenAI + +# init the client but point it to TGI +client = OpenAI( + base_url="http://localhost:3000/v1", + api_key="-" +) + +chat_completion = client.chat.completions.create( + model="tgi", + messages=[ + {"role": "system", "content": "You are a helpful assistant." }, + {"role": "user", "content": "What is deep learning?"} + ], + stream=True +) + +# iterate and print stream +for message in chat_completion: + print(message) +``` + +## Synchronous + +If you prefer to make a synchronous request, you can do so like this: + +```python +from openai import OpenAI + +# init the client but point it to TGI +client = OpenAI( + base_url="http://localhost:3000/v1", + api_key="-" +) + +chat_completion = client.chat.completions.create( + model="tgi", + messages=[ + {"role": "system", "content": "You are a helpful assistant." }, + {"role": "user", "content": "What is deep learning?"} + ], + stream=False +) + +print(chat_completion) +``` + +## Cloud Providers + +TGI can be deployed on various cloud providers for scalable and robust text generation. One such provider is Amazon SageMaker, which has recently added support for TGI. Here's how you can deploy TGI on Amazon SageMaker: + +## Amazon SageMaker + +To enable the Messages API in Amazon SageMaker you need to set the environment variable `MESSAGES_API_ENABLED=true`. + +This will modify the `/invocations` route to accept Messages dictonaries consisting out of role and content. See the example below on how to deploy Llama with the new Messages API. + +```python +import json +import sagemaker +import boto3 +from sagemaker.huggingface import HuggingFaceModel, get_huggingface_llm_image_uri + +try: + role = sagemaker.get_execution_role() +except ValueError: + iam = boto3.client('iam') + role = iam.get_role(RoleName='sagemaker_execution_role')['Role']['Arn'] + +# Hub Model configuration. https://huggingface.co/models +hub = { + 'HF_MODEL_ID':'HuggingFaceH4/zephyr-7b-beta', + 'SM_NUM_GPUS': json.dumps(1), + 'MESSAGES_API_ENABLED': True +} + +# create Hugging Face Model Class +huggingface_model = HuggingFaceModel( + image_uri=get_huggingface_llm_image_uri("huggingface",version="1.4.0"), + env=hub, + role=role, +) + +# deploy model to SageMaker Inference +predictor = huggingface_model.deploy( + initial_instance_count=1, + instance_type="ml.g5.2xlarge", + container_startup_health_check_timeout=300, + ) + +# send request +predictor.predict({ +"messages": [ + {"role": "system", "content": "You are a helpful assistant." }, + {"role": "user", "content": "What is deep learning?"} + ] +}) +``` \ No newline at end of file diff --git a/router/src/main.rs b/router/src/main.rs index bf987eb6..b6190908 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -72,7 +72,7 @@ struct Args { #[clap(long, env)] ngrok_edge: Option, #[clap(long, env, default_value_t = false)] - chat_enabled_api: bool, + messages_api_enabled: bool, } #[tokio::main] @@ -104,7 +104,7 @@ async fn main() -> Result<(), RouterError> { ngrok, ngrok_authtoken, ngrok_edge, - chat_enabled_api, + messages_api_enabled, } = args; // Launch Tokio runtime @@ -348,7 +348,7 @@ async fn main() -> Result<(), RouterError> { ngrok_authtoken, ngrok_edge, tokenizer_config, - chat_enabled_api, + messages_api_enabled, ) .await?; Ok(()) diff --git a/router/src/server.rs b/router/src/server.rs index aa1ad202..ff48b4f0 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -708,7 +708,7 @@ pub async fn run( ngrok_authtoken: Option, ngrok_edge: Option, tokenizer_config: HubTokenizerConfig, - chat_enabled_api: bool, + messages_api_enabled: bool, ) -> Result<(), axum::BoxError> { // OpenAPI documentation #[derive(OpenApi)] @@ -872,7 +872,7 @@ pub async fn run( .route("/metrics", get(metrics)); // Conditional AWS Sagemaker route - let aws_sagemaker_route = if chat_enabled_api { + let aws_sagemaker_route = if messages_api_enabled { Router::new().route("/invocations", post(chat_completions)) // Use 'chat_completions' for OAI_ENABLED } else { Router::new().route("/invocations", post(compat_generate)) // Use 'compat_generate' otherwise From 86c8335f1b280f6514ae0a6d22754e503cf3d414 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 25 Jan 2024 14:19:03 +0100 Subject: [PATCH 527/793] Add a new `/tokenize` route to get the tokenized input (#1471) # What does this PR do? Ideally this is done client side, but this is a recurring request, therefore we implemented it. - Runs only if rust tokenizer is present (not encumbering the main inference pipeline is important). - Returns simple results, ID, text (gotten with offsets from the original string) and offsets (so users can do things like highlighting text). Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- docs/openapi.json | 884 +-------------------------------------- router/src/infer.rs | 22 + router/src/lib.rs | 12 + router/src/server.rs | 55 ++- router/src/validation.rs | 45 +- 5 files changed, 115 insertions(+), 903 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index df2d427f..4454259b 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1,883 +1 @@ -{ - "openapi": "3.0.3", - "info": { - "title": "Text Generation Inference", - "description": "Text Generation Webserver", - "contact": { - "name": "Olivier Dehaene" - }, - "license": { - "name": "Apache 2.0", - "url": "https://www.apache.org/licenses/LICENSE-2.0" - }, - "version": "1.3.4" - }, - "paths": { - "/": { - "post": { - "tags": [ - "Text Generation Inference" - ], - "summary": "Generate tokens if `stream == false` or a stream of token if `stream == true`", - "description": "Generate tokens if `stream == false` or a stream of token if `stream == true`", - "operationId": "compat_generate", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CompatGenerateRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Generated Text", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GenerateResponse" - } - }, - "text/event-stream": { - "schema": { - "$ref": "#/components/schemas/StreamResponse" - } - } - } - }, - "422": { - "description": "Input validation error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Input validation error" - } - } - } - }, - "424": { - "description": "Generation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Request failed during generation" - } - } - } - }, - "429": { - "description": "Model is overloaded", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Model is overloaded" - } - } - } - }, - "500": { - "description": "Incomplete generation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Incomplete generation" - } - } - } - } - } - } - }, - "/generate": { - "post": { - "tags": [ - "Text Generation Inference" - ], - "summary": "Generate tokens", - "description": "Generate tokens", - "operationId": "generate", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GenerateRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Generated Text", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GenerateResponse" - } - } - } - }, - "422": { - "description": "Input validation error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Input validation error" - } - } - } - }, - "424": { - "description": "Generation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Request failed during generation" - } - } - } - }, - "429": { - "description": "Model is overloaded", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Model is overloaded" - } - } - } - }, - "500": { - "description": "Incomplete generation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Incomplete generation" - } - } - } - } - } - } - }, - "/generate_stream": { - "post": { - "tags": [ - "Text Generation Inference" - ], - "summary": "Generate a stream of token using Server-Sent Events", - "description": "Generate a stream of token using Server-Sent Events", - "operationId": "generate_stream", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GenerateRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Generated Text", - "content": { - "text/event-stream": { - "schema": { - "$ref": "#/components/schemas/StreamResponse" - } - } - } - }, - "422": { - "description": "Input validation error", - "content": { - "text/event-stream": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Input validation error" - } - } - } - }, - "424": { - "description": "Generation Error", - "content": { - "text/event-stream": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Request failed during generation" - } - } - } - }, - "429": { - "description": "Model is overloaded", - "content": { - "text/event-stream": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Model is overloaded" - } - } - } - }, - "500": { - "description": "Incomplete generation", - "content": { - "text/event-stream": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "Incomplete generation" - } - } - } - } - } - } - }, - "/health": { - "get": { - "tags": [ - "Text Generation Inference" - ], - "summary": "Health check method", - "description": "Health check method", - "operationId": "health", - "responses": { - "200": { - "description": "Everything is working fine" - }, - "503": { - "description": "Text generation inference is down", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - }, - "example": { - "error": "unhealthy", - "error_type": "healthcheck" - } - } - } - } - } - } - }, - "/info": { - "get": { - "tags": [ - "Text Generation Inference" - ], - "summary": "Text Generation Inference endpoint info", - "description": "Text Generation Inference endpoint info", - "operationId": "get_model_info", - "responses": { - "200": { - "description": "Served model info", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Info" - } - } - } - } - } - } - }, - "/metrics": { - "get": { - "tags": [ - "Text Generation Inference" - ], - "summary": "Prometheus metrics scrape endpoint", - "description": "Prometheus metrics scrape endpoint", - "operationId": "metrics", - "responses": { - "200": { - "description": "Prometheus Metrics", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - } - } - } - }, - "components": { - "schemas": { - "BestOfSequence": { - "type": "object", - "required": [ - "generated_text", - "finish_reason", - "generated_tokens", - "prefill", - "tokens" - ], - "properties": { - "finish_reason": { - "$ref": "#/components/schemas/FinishReason" - }, - "generated_text": { - "type": "string", - "example": "test" - }, - "generated_tokens": { - "type": "integer", - "format": "int32", - "example": 1, - "minimum": 0 - }, - "prefill": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PrefillToken" - } - }, - "seed": { - "type": "integer", - "format": "int64", - "example": 42, - "nullable": true, - "minimum": 0 - }, - "tokens": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Token" - } - }, - "top_tokens": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Token" - } - } - } - } - }, - "CompatGenerateRequest": { - "type": "object", - "required": [ - "inputs" - ], - "properties": { - "inputs": { - "type": "string", - "example": "My name is Olivier and I" - }, - "parameters": { - "$ref": "#/components/schemas/GenerateParameters" - }, - "stream": { - "type": "boolean", - "default": "false" - } - } - }, - "Details": { - "type": "object", - "required": [ - "finish_reason", - "generated_tokens", - "prefill", - "tokens" - ], - "properties": { - "best_of_sequences": { - "type": "array", - "items": { - "$ref": "#/components/schemas/BestOfSequence" - }, - "nullable": true - }, - "finish_reason": { - "$ref": "#/components/schemas/FinishReason" - }, - "generated_tokens": { - "type": "integer", - "format": "int32", - "example": 1, - "minimum": 0 - }, - "prefill": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PrefillToken" - } - }, - "seed": { - "type": "integer", - "format": "int64", - "example": 42, - "nullable": true, - "minimum": 0 - }, - "tokens": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Token" - } - }, - "top_tokens": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Token" - } - } - } - } - }, - "ErrorResponse": { - "type": "object", - "required": [ - "error", - "error_type" - ], - "properties": { - "error": { - "type": "string" - }, - "error_type": { - "type": "string" - } - } - }, - "FinishReason": { - "type": "string", - "enum": [ - "length", - "eos_token", - "stop_sequence" - ] - }, - "GenerateParameters": { - "type": "object", - "properties": { - "best_of": { - "type": "integer", - "default": "null", - "example": 1, - "nullable": true, - "minimum": 0, - "exclusiveMinimum": 0 - }, - "decoder_input_details": { - "type": "boolean", - "default": "true" - }, - "details": { - "type": "boolean", - "default": "true" - }, - "do_sample": { - "type": "boolean", - "default": "false", - "example": true - }, - "max_new_tokens": { - "type": "integer", - "format": "int32", - "default": "20", - "example": "20", - "nullable": true, - "minimum": 0 - }, - "repetition_penalty": { - "type": "number", - "format": "float", - "default": "null", - "example": 1.03, - "nullable": true, - "exclusiveMinimum": 0 - }, - "return_full_text": { - "type": "boolean", - "default": "null", - "example": false, - "nullable": true - }, - "seed": { - "type": "integer", - "format": "int64", - "default": "null", - "example": "null", - "nullable": true, - "minimum": 0, - "exclusiveMinimum": 0 - }, - "stop": { - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "photographer" - ], - "maxItems": 4 - }, - "temperature": { - "type": "number", - "format": "float", - "default": "null", - "example": 0.5, - "nullable": true, - "exclusiveMinimum": 0 - }, - "top_k": { - "type": "integer", - "format": "int32", - "default": "null", - "example": 10, - "nullable": true, - "exclusiveMinimum": 0 - }, - "top_n_tokens": { - "type": "integer", - "format": "int32", - "default": "null", - "example": 5, - "nullable": true, - "minimum": 0, - "exclusiveMinimum": 0 - }, - "top_p": { - "type": "number", - "format": "float", - "default": "null", - "example": 0.95, - "nullable": true, - "maximum": 1, - "exclusiveMinimum": 0 - }, - "truncate": { - "type": "integer", - "default": "null", - "example": "null", - "nullable": true, - "minimum": 0 - }, - "typical_p": { - "type": "number", - "format": "float", - "default": "null", - "example": 0.95, - "nullable": true, - "maximum": 1, - "exclusiveMinimum": 0 - }, - "watermark": { - "type": "boolean", - "default": "false", - "example": true - } - } - }, - "GenerateRequest": { - "type": "object", - "required": [ - "inputs" - ], - "properties": { - "inputs": { - "type": "string", - "example": "My name is Olivier and I" - }, - "parameters": { - "$ref": "#/components/schemas/GenerateParameters" - } - } - }, - "GenerateResponse": { - "type": "object", - "required": [ - "generated_text" - ], - "properties": { - "details": { - "allOf": [ - { - "$ref": "#/components/schemas/Details" - } - ], - "nullable": true - }, - "generated_text": { - "type": "string", - "example": "test" - } - } - }, - "Info": { - "type": "object", - "required": [ - "model_id", - "model_dtype", - "model_device_type", - "max_concurrent_requests", - "max_best_of", - "max_stop_sequences", - "max_input_length", - "max_total_tokens", - "waiting_served_ratio", - "max_batch_total_tokens", - "max_waiting_tokens", - "validation_workers", - "version" - ], - "properties": { - "docker_label": { - "type": "string", - "example": "null", - "nullable": true - }, - "max_batch_total_tokens": { - "type": "integer", - "format": "int32", - "example": "32000", - "minimum": 0 - }, - "max_best_of": { - "type": "integer", - "example": "2", - "minimum": 0 - }, - "max_concurrent_requests": { - "type": "integer", - "description": "Router Parameters", - "example": "128", - "minimum": 0 - }, - "max_input_length": { - "type": "integer", - "example": "1024", - "minimum": 0 - }, - "max_stop_sequences": { - "type": "integer", - "example": "4", - "minimum": 0 - }, - "max_total_tokens": { - "type": "integer", - "example": "2048", - "minimum": 0 - }, - "max_waiting_tokens": { - "type": "integer", - "example": "20", - "minimum": 0 - }, - "model_device_type": { - "type": "string", - "example": "cuda" - }, - "model_dtype": { - "type": "string", - "example": "torch.float16" - }, - "model_id": { - "type": "string", - "description": "Model info", - "example": "bigscience/blomm-560m" - }, - "model_pipeline_tag": { - "type": "string", - "example": "text-generation", - "nullable": true - }, - "model_sha": { - "type": "string", - "example": "e985a63cdc139290c5f700ff1929f0b5942cced2", - "nullable": true - }, - "sha": { - "type": "string", - "example": "null", - "nullable": true - }, - "validation_workers": { - "type": "integer", - "example": "2", - "minimum": 0 - }, - "version": { - "type": "string", - "description": "Router Info", - "example": "0.5.0" - }, - "waiting_served_ratio": { - "type": "number", - "format": "float", - "example": "1.2" - } - } - }, - "PrefillToken": { - "type": "object", - "required": [ - "id", - "text", - "logprob" - ], - "properties": { - "id": { - "type": "integer", - "format": "int32", - "example": 0, - "minimum": 0 - }, - "logprob": { - "type": "number", - "format": "float", - "example": -0.34, - "nullable": true - }, - "text": { - "type": "string", - "example": "test" - } - } - }, - "StreamDetails": { - "type": "object", - "required": [ - "finish_reason", - "generated_tokens" - ], - "properties": { - "finish_reason": { - "$ref": "#/components/schemas/FinishReason" - }, - "generated_tokens": { - "type": "integer", - "format": "int32", - "example": 1, - "minimum": 0 - }, - "seed": { - "type": "integer", - "format": "int64", - "example": 42, - "nullable": true, - "minimum": 0 - } - } - }, - "StreamResponse": { - "type": "object", - "required": [ - "token" - ], - "properties": { - "details": { - "allOf": [ - { - "$ref": "#/components/schemas/StreamDetails" - } - ], - "default": "null", - "nullable": true - }, - "generated_text": { - "type": "string", - "default": "null", - "example": "test", - "nullable": true - }, - "token": { - "$ref": "#/components/schemas/Token" - }, - "top_tokens": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Token" - } - } - } - }, - "Token": { - "type": "object", - "required": [ - "id", - "text", - "logprob", - "special" - ], - "properties": { - "id": { - "type": "integer", - "format": "int32", - "example": 0, - "minimum": 0 - }, - "logprob": { - "type": "number", - "format": "float", - "example": -0.34, - "nullable": true - }, - "special": { - "type": "boolean", - "example": "false" - }, - "text": { - "type": "string", - "example": "test" - } - } - } - } - }, - "tags": [ - { - "name": "Text Generation Inference", - "description": "Hugging Face Text Generation Inference API" - } - ] -} +{"openapi":"3.0.3","info":{"title":"Text Generation Inference","description":"Text Generation Webserver","contact":{"name":"Olivier Dehaene"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0"},"version":"1.3.4"},"paths":{"/":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens if `stream == false` or a stream of token if `stream == true`","description":"Generate tokens if `stream == false` or a stream of token if `stream == true`","operationId":"compat_generate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CompatGenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateResponse"}},"text/event-stream":{"schema":{"$ref":"#/components/schemas/StreamResponse"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/generate":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens","description":"Generate tokens","operationId":"generate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateResponse"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/generate_stream":{"post":{"tags":["Text Generation Inference"],"summary":"Generate a stream of token using Server-Sent Events","description":"Generate a stream of token using Server-Sent Events","operationId":"generate_stream","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/StreamResponse"}}}},"422":{"description":"Input validation error","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/health":{"get":{"tags":["Text Generation Inference"],"summary":"Health check method","description":"Health check method","operationId":"health","responses":{"200":{"description":"Everything is working fine"},"503":{"description":"Text generation inference is down","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"unhealthy","error_type":"healthcheck"}}}}}}},"/info":{"get":{"tags":["Text Generation Inference"],"summary":"Text Generation Inference endpoint info","description":"Text Generation Inference endpoint info","operationId":"get_model_info","responses":{"200":{"description":"Served model info","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Info"}}}}}}},"/metrics":{"get":{"tags":["Text Generation Inference"],"summary":"Prometheus metrics scrape endpoint","description":"Prometheus metrics scrape endpoint","operationId":"metrics","responses":{"200":{"description":"Prometheus Metrics","content":{"text/plain":{"schema":{"type":"string"}}}}}}},"/tokenize":{"post":{"tags":["Text Generation Inference"],"summary":"Tokenize inputs","description":"Tokenize inputs","operationId":"tokenize","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TokenizeRequest"}}},"required":true},"responses":{"200":{"description":"Tokenized ids","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TokenizeResponse"}}}},"404":{"description":"No tokenizer found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"No fast tokenizer available"}}}}}}},"/v1/chat/completions":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens","description":"Generate tokens","operationId":"chat_completions","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatCompletionChunk"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}}},"components":{"schemas":{"BestOfSequence":{"type":"object","required":["generated_text","finish_reason","generated_tokens","prefill","tokens"],"properties":{"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_text":{"type":"string","example":"test"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"prefill":{"type":"array","items":{"$ref":"#/components/schemas/PrefillToken"}},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}},"top_tokens":{"type":"array","items":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}}},"CompatGenerateRequest":{"type":"object","required":["inputs"],"properties":{"inputs":{"type":"string","example":"My name is Olivier and I"},"parameters":{"$ref":"#/components/schemas/GenerateParameters"},"stream":{"type":"boolean","default":"false"}}},"Details":{"type":"object","required":["finish_reason","generated_tokens","prefill","tokens"],"properties":{"best_of_sequences":{"type":"array","items":{"$ref":"#/components/schemas/BestOfSequence"},"nullable":true},"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"prefill":{"type":"array","items":{"$ref":"#/components/schemas/PrefillToken"}},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}},"top_tokens":{"type":"array","items":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}}},"ErrorResponse":{"type":"object","required":["error","error_type"],"properties":{"error":{"type":"string"},"error_type":{"type":"string"}}},"FinishReason":{"type":"string","enum":["length","eos_token","stop_sequence"]},"GenerateParameters":{"type":"object","properties":{"best_of":{"type":"integer","default":"null","example":1,"nullable":true,"minimum":0,"exclusiveMinimum":0},"decoder_input_details":{"type":"boolean","default":"true"},"details":{"type":"boolean","default":"true"},"do_sample":{"type":"boolean","default":"false","example":true},"max_new_tokens":{"type":"integer","format":"int32","default":"100","example":"20","nullable":true,"minimum":0},"repetition_penalty":{"type":"number","format":"float","default":"null","example":1.03,"nullable":true,"exclusiveMinimum":0},"return_full_text":{"type":"boolean","default":"null","example":false,"nullable":true},"seed":{"type":"integer","format":"int64","default":"null","example":"null","nullable":true,"minimum":0,"exclusiveMinimum":0},"stop":{"type":"array","items":{"type":"string"},"example":["photographer"],"maxItems":4},"temperature":{"type":"number","format":"float","default":"null","example":0.5,"nullable":true,"exclusiveMinimum":0},"top_k":{"type":"integer","format":"int32","default":"null","example":10,"nullable":true,"exclusiveMinimum":0},"top_n_tokens":{"type":"integer","format":"int32","default":"null","example":5,"nullable":true,"minimum":0,"exclusiveMinimum":0},"top_p":{"type":"number","format":"float","default":"null","example":0.95,"nullable":true,"maximum":1,"exclusiveMinimum":0},"truncate":{"type":"integer","default":"null","example":"null","nullable":true,"minimum":0},"typical_p":{"type":"number","format":"float","default":"null","example":0.95,"nullable":true,"maximum":1,"exclusiveMinimum":0},"watermark":{"type":"boolean","default":"false","example":true}}},"GenerateRequest":{"type":"object","required":["inputs"],"properties":{"inputs":{"type":"string","example":"My name is Olivier and I"},"parameters":{"$ref":"#/components/schemas/GenerateParameters"}}},"GenerateResponse":{"type":"object","required":["generated_text"],"properties":{"details":{"allOf":[{"$ref":"#/components/schemas/Details"}],"nullable":true},"generated_text":{"type":"string","example":"test"}}},"Info":{"type":"object","required":["model_id","model_dtype","model_device_type","max_concurrent_requests","max_best_of","max_stop_sequences","max_input_length","max_total_tokens","waiting_served_ratio","max_batch_total_tokens","max_waiting_tokens","validation_workers","version"],"properties":{"docker_label":{"type":"string","example":"null","nullable":true},"max_batch_total_tokens":{"type":"integer","format":"int32","example":"32000","minimum":0},"max_best_of":{"type":"integer","example":"2","minimum":0},"max_concurrent_requests":{"type":"integer","description":"Router Parameters","example":"128","minimum":0},"max_input_length":{"type":"integer","example":"1024","minimum":0},"max_stop_sequences":{"type":"integer","example":"4","minimum":0},"max_total_tokens":{"type":"integer","example":"2048","minimum":0},"max_waiting_tokens":{"type":"integer","example":"20","minimum":0},"model_device_type":{"type":"string","example":"cuda"},"model_dtype":{"type":"string","example":"torch.float16"},"model_id":{"type":"string","description":"Model info","example":"bigscience/blomm-560m"},"model_pipeline_tag":{"type":"string","example":"text-generation","nullable":true},"model_sha":{"type":"string","example":"e985a63cdc139290c5f700ff1929f0b5942cced2","nullable":true},"sha":{"type":"string","example":"null","nullable":true},"validation_workers":{"type":"integer","example":"2","minimum":0},"version":{"type":"string","description":"Router Info","example":"0.5.0"},"waiting_served_ratio":{"type":"number","format":"float","example":"1.2"}}},"PrefillToken":{"type":"object","required":["id","text","logprob"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"logprob":{"type":"number","format":"float","example":-0.34,"nullable":true},"text":{"type":"string","example":"test"}}},"StreamDetails":{"type":"object","required":["finish_reason","generated_tokens"],"properties":{"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0}}},"StreamResponse":{"type":"object","required":["index","token"],"properties":{"details":{"allOf":[{"$ref":"#/components/schemas/StreamDetails"}],"default":"null","nullable":true},"generated_text":{"type":"string","default":"null","example":"test","nullable":true},"index":{"type":"integer","format":"int32","minimum":0},"token":{"$ref":"#/components/schemas/Token"},"top_tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}},"Token":{"type":"object","required":["id","text","logprob","special"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"logprob":{"type":"number","format":"float","example":-0.34,"nullable":true},"special":{"type":"boolean","example":"false"},"text":{"type":"string","example":"test"}}}}},"tags":[{"name":"Text Generation Inference","description":"Hugging Face Text Generation Inference API"}]} diff --git a/router/src/infer.rs b/router/src/infer.rs index 8a9875eb..5f078ba0 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -165,6 +165,28 @@ impl Infer { )) } + /// Tokenizer the input + #[instrument(skip_all)] + pub(crate) async fn tokenize( + &self, + request: GenerateRequest, + ) -> Result, InferError> { + // Tokenize request + let inputs = request.inputs; + let truncate = request.parameters.truncate; + let encoding = self + .validation + .tokenize(inputs, truncate) + .await + .map_err(|err| { + tracing::error!("Tokenization {err}"); + err + })?; + + // Return Encoding + Ok(encoding.map(|(encoding, _)| encoding)) + } + /// Apply the chat template to the chat request #[instrument(skip_all)] pub(crate) fn apply_chat_template(&self, messages: Vec) -> Result { diff --git a/router/src/lib.rs b/router/src/lib.rs index 894ab466..2bfbbacd 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -444,6 +444,18 @@ pub struct Token { special: bool, } +#[derive(Debug, Serialize, ToSchema)] +pub struct SimpleToken { + #[schema(example = 0)] + id: u32, + #[schema(example = "test")] + text: String, + #[schema(example = 0)] + start: usize, + #[schema(example = 2)] + stop: usize, +} + #[derive(Serialize, ToSchema)] #[serde(rename_all(serialize = "snake_case"))] pub(crate) enum FinishReason { diff --git a/router/src/server.rs b/router/src/server.rs index ff48b4f0..c5ca4665 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -5,8 +5,8 @@ use crate::validation::ValidationError; use crate::{ BestOfSequence, ChatCompletion, ChatCompletionChunk, ChatRequest, CompatGenerateRequest, Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest, GenerateResponse, - HubModelInfo, HubTokenizerConfig, Infer, Info, PrefillToken, StreamDetails, StreamResponse, - Token, Validation, + HubModelInfo, HubTokenizerConfig, Infer, Info, PrefillToken, SimpleToken, StreamDetails, + StreamResponse, Token, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; @@ -532,7 +532,7 @@ async fn generate_stream_internal( path = "/v1/chat/completions", request_body = ChatRequest, responses( - (status = 200, description = "Generated Text", body = GenerateResponse), + (status = 200, description = "Generated Text", body = ChatCompletionChunk), (status = 424, description = "Generation Error", body = ErrorResponse, example = json ! ({"error": "Request failed during generation"})), (status = 429, description = "Model is overloaded", body = ErrorResponse, @@ -672,6 +672,52 @@ async fn chat_completions( } } +/// Tokenize inputs +#[utoipa::path( + post, + tag = "Text Generation Inference", + path = "/tokenize", + request_body = TokenizeRequest, + responses( + (status = 200, description = "Tokenized ids", body = TokenizeResponse), + (status = 404, description = "No tokenizer found", body = ErrorResponse, + example = json ! ({"error": "No fast tokenizer available"})), + ) + )] +#[instrument(skip_all)] +async fn tokenize( + Extension(infer): Extension, + Json(req): Json, +) -> Result)> { + let input = req.inputs.clone(); + let encoding = infer.tokenize(req).await?; + if let Some(encoding) = encoding { + let tokens: Vec = encoding + .get_ids() + .iter() + .zip(encoding.get_offsets()) + .map(|(&id, &(start, stop))| { + let text: String = input.chars().skip(start).take(stop - start).collect(); + SimpleToken { + id, + text, + start, + stop, + } + }) + .collect(); + Ok(Json(tokens).into_response()) + } else { + Err(( + StatusCode::NOT_FOUND, + Json(ErrorResponse { + error: "No fast tokenizer or tokenizer.json for this model".to_string(), + error_type: "no fast tokenizer".to_string(), + }), + )) + } +} + /// Prometheus metrics scrape endpoint #[utoipa::path( get, @@ -719,6 +765,8 @@ pub async fn run( compat_generate, generate, generate_stream, + chat_completions, + tokenize, metrics, ), components( @@ -867,6 +915,7 @@ pub async fn run( .route("/generate", post(generate)) .route("/generate_stream", post(generate_stream)) .route("/v1/chat/completions", post(chat_completions)) + .route("/tokenize", post(tokenize)) .route("/health", get(health)) .route("/ping", get(health)) .route("/metrics", get(metrics)); diff --git a/router/src/validation.rs b/router/src/validation.rs index 370e9588..750b98e5 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -70,12 +70,11 @@ impl Validation { } #[instrument(skip(self, inputs))] - async fn validate_input( + pub async fn tokenize( &self, inputs: String, truncate: Option, - max_new_tokens: Option, - ) -> Result<(String, usize, u32), ValidationError> { + ) -> Result, ValidationError> { // If we have a fast tokenizer if let Some(sender) = &self.sender { // Create response channel @@ -88,7 +87,24 @@ impl Validation { // Await on response channel // Unwrap is safe here - let (inputs, input_length) = response_receiver.await.unwrap()?; + let encoding = response_receiver.await.unwrap()?; + Ok(Some(encoding)) + } else { + Ok(None) + } + } + + #[instrument(skip(self, inputs))] + async fn validate_input( + &self, + inputs: String, + truncate: Option, + max_new_tokens: Option, + ) -> Result<(String, usize, u32), ValidationError> { + // If we have a fast tokenizer + if let Some((encoding, inputs)) = self.tokenize(inputs.clone(), truncate).await? { + // Create response channel + let input_length = encoding.len(); // Get total tokens let max_new_tokens: u32 = if let Some(max_new_tokens) = max_new_tokens { @@ -343,36 +359,31 @@ fn tokenizer_worker(tokenizer: Tokenizer, mut receiver: mpsc::UnboundedReceiver< /// Get input length and optionally truncate it fn prepare_input( - inputs: String, + mut inputs: String, truncate: Option, tokenizer: &Tokenizer, -) -> Result<(String, usize), ValidationError> { +) -> Result<(tokenizers::Encoding, String), ValidationError> { // Get the number of tokens in the input let mut encoding = tokenizer .encode(inputs.clone(), true) .map_err(|err| ValidationError::Tokenizer(err.to_string()))?; // Optionally truncate - let (inputs, input_length) = match truncate { - // Truncate is some and < encoding length - Some(truncate) if truncate < encoding.len() => { - // truncate encoding and decode new inputs + if let Some(truncate) = truncate { + if truncate < encoding.len() { encoding.truncate(truncate, 0, TruncationDirection::Left); - let inputs = tokenizer + inputs = tokenizer .decode(encoding.get_ids(), false) .map_err(|err| ValidationError::Tokenizer(err.to_string()))?; - (inputs, encoding.len()) } - // Nothing to do - _ => (inputs, encoding.len()), - }; + } - Ok((inputs, input_length)) + Ok((encoding, inputs)) } type TokenizerRequest = ( (String, Option), - oneshot::Sender>, + oneshot::Sender>, Span, ); From 7e2a7433d3584a5a68dbf3e71def4323079f2c26 Mon Sep 17 00:00:00 2001 From: drbh Date: Thu, 25 Jan 2024 09:37:53 -0500 Subject: [PATCH 528/793] feat: adds phi model (#1442) This PR adds basic modeling for phi-2 run ```bash text-generation-server \ serve \ microsoft/phi-2 \ --revision 834565c23f9b28b96ccbeabe614dd906b6db551a ``` test ```bash curl -s localhost:3000/generate \ -X POST \ -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ -H 'Content-Type: application/json' | jq . # { # "generated_text": "\nDeep learning is a subset of machine learning that uses artificial neural networks to learn from data. These" # } ``` notes - recently (~1 day ago) the Phi weights and model were updated to accommodate adding [GQA/MQA attention to the model.](https://github.com/huggingface/transformers/pull/28163) This impl expects the original model format so a fixed revision is required at the moment. - this PR only includes a basic implementation of the model and can later be extended for support Flash and Sharded versions as well as make use of better optimization --- .../test_flash_phi/test_flash_phi.json | 84 ++++ .../test_flash_phi_all_params.json | 60 +++ .../test_flash_phi/test_flash_phi_load.json | 338 +++++++++++++++ integration-tests/models/test_flash_phi.py | 65 +++ server/pyproject.toml | 2 +- .../text_generation_server/models/__init__.py | 34 ++ .../custom_modeling/flash_phi_modeling.py | 400 ++++++++++++++++++ .../models/custom_modeling/phi_modeling.py | 308 ++++++++++++++ .../models/flash_phi.py | 102 +++++ server/text_generation_server/models/phi.py | 63 +++ 10 files changed, 1455 insertions(+), 1 deletion(-) create mode 100644 integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi.json create mode 100644 integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_load.json create mode 100644 integration-tests/models/test_flash_phi.py create mode 100644 server/text_generation_server/models/custom_modeling/flash_phi_modeling.py create mode 100644 server/text_generation_server/models/custom_modeling/phi_modeling.py create mode 100644 server/text_generation_server/models/flash_phi.py create mode 100644 server/text_generation_server/models/phi.py diff --git a/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi.json b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi.json new file mode 100644 index 00000000..51d969b2 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi.json @@ -0,0 +1,84 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 14402, + "logprob": null, + "text": "Test" + }, + { + "id": 2581, + "logprob": -11.6171875, + "text": " request" + } + ], + "seed": null, + "tokens": [ + { + "id": 25, + "logprob": -2.3203125, + "special": false, + "text": ":" + }, + { + "id": 1391, + "logprob": -0.98779297, + "special": false, + "text": " {" + }, + { + "id": 25927, + "logprob": -0.76660156, + "special": false, + "text": "request" + }, + { + "id": 92, + "logprob": -0.7246094, + "special": false, + "text": "}" + }, + { + "id": 4943, + "logprob": -0.41333008, + "special": false, + "text": "\")" + }, + { + "id": 198, + "logprob": -0.11785889, + "special": false, + "text": "\n" + }, + { + "id": 50280, + "logprob": -0.97265625, + "special": false, + "text": " " + }, + { + "id": 26209, + "logprob": -1.4414062, + "special": false, + "text": "response" + }, + { + "id": 796, + "logprob": -0.0569458, + "special": false, + "text": " =" + }, + { + "id": 2116, + "logprob": -1.1533203, + "special": false, + "text": " self" + } + ], + "top_tokens": null + }, + "generated_text": ": {request}\")\n response = self" +} diff --git a/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json new file mode 100644 index 00000000..221ff13d --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json @@ -0,0 +1,60 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "stop_sequence", + "generated_tokens": 6, + "prefill": [ + { + "id": 14402, + "logprob": null, + "text": "Test" + }, + { + "id": 2581, + "logprob": -11.6171875, + "text": " request" + } + ], + "seed": 0, + "tokens": [ + { + "id": 284, + "logprob": -0.19421387, + "special": false, + "text": " to" + }, + { + "id": 3758, + "logprob": -0.62597656, + "special": false, + "text": " send" + }, + { + "id": 1366, + "logprob": -0.87060547, + "special": false, + "text": " data" + }, + { + "id": 625, + "logprob": -0.88427734, + "special": false, + "text": " over" + }, + { + "id": 257, + "logprob": -1.0830078, + "special": false, + "text": " a" + }, + { + "id": 3127, + "logprob": -1.9462891, + "special": false, + "text": " network" + } + ], + "top_tokens": null + }, + "generated_text": "Test request to send data over a network" +} diff --git a/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_load.json b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_load.json new file mode 100644 index 00000000..62f7fd32 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_load.json @@ -0,0 +1,338 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 14402, + "logprob": null, + "text": "Test" + }, + { + "id": 2581, + "logprob": -11.6171875, + "text": " request" + } + ], + "seed": null, + "tokens": [ + { + "id": 25, + "logprob": -2.3203125, + "special": false, + "text": ":" + }, + { + "id": 1391, + "logprob": -0.98779297, + "special": false, + "text": " {" + }, + { + "id": 25927, + "logprob": -0.7729492, + "special": false, + "text": "request" + }, + { + "id": 92, + "logprob": -0.7241211, + "special": false, + "text": "}" + }, + { + "id": 4943, + "logprob": -0.4091797, + "special": false, + "text": "\")" + }, + { + "id": 198, + "logprob": -0.119018555, + "special": false, + "text": "\n" + }, + { + "id": 50280, + "logprob": -0.9707031, + "special": false, + "text": " " + }, + { + "id": 26209, + "logprob": -1.4414062, + "special": false, + "text": "response" + }, + { + "id": 796, + "logprob": -0.056854248, + "special": false, + "text": " =" + }, + { + "id": 2116, + "logprob": -1.1533203, + "special": false, + "text": " self" + } + ], + "top_tokens": null + }, + "generated_text": ": {request}\")\n response = self" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 14402, + "logprob": null, + "text": "Test" + }, + { + "id": 2581, + "logprob": -11.6171875, + "text": " request" + } + ], + "seed": null, + "tokens": [ + { + "id": 25, + "logprob": -2.3203125, + "special": false, + "text": ":" + }, + { + "id": 1391, + "logprob": -0.98779297, + "special": false, + "text": " {" + }, + { + "id": 25927, + "logprob": -0.7729492, + "special": false, + "text": "request" + }, + { + "id": 92, + "logprob": -0.7241211, + "special": false, + "text": "}" + }, + { + "id": 4943, + "logprob": -0.4091797, + "special": false, + "text": "\")" + }, + { + "id": 198, + "logprob": -0.119018555, + "special": false, + "text": "\n" + }, + { + "id": 50280, + "logprob": -0.9707031, + "special": false, + "text": " " + }, + { + "id": 26209, + "logprob": -1.4414062, + "special": false, + "text": "response" + }, + { + "id": 796, + "logprob": -0.056854248, + "special": false, + "text": " =" + }, + { + "id": 2116, + "logprob": -1.1533203, + "special": false, + "text": " self" + } + ], + "top_tokens": null + }, + "generated_text": ": {request}\")\n response = self" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 14402, + "logprob": null, + "text": "Test" + }, + { + "id": 2581, + "logprob": -11.6171875, + "text": " request" + } + ], + "seed": null, + "tokens": [ + { + "id": 25, + "logprob": -2.3203125, + "special": false, + "text": ":" + }, + { + "id": 1391, + "logprob": -0.98779297, + "special": false, + "text": " {" + }, + { + "id": 25927, + "logprob": -0.7729492, + "special": false, + "text": "request" + }, + { + "id": 92, + "logprob": -0.7241211, + "special": false, + "text": "}" + }, + { + "id": 4943, + "logprob": -0.4091797, + "special": false, + "text": "\")" + }, + { + "id": 198, + "logprob": -0.119018555, + "special": false, + "text": "\n" + }, + { + "id": 50280, + "logprob": -0.9707031, + "special": false, + "text": " " + }, + { + "id": 26209, + "logprob": -1.4414062, + "special": false, + "text": "response" + }, + { + "id": 796, + "logprob": -0.056854248, + "special": false, + "text": " =" + }, + { + "id": 2116, + "logprob": -1.1533203, + "special": false, + "text": " self" + } + ], + "top_tokens": null + }, + "generated_text": ": {request}\")\n response = self" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 14402, + "logprob": null, + "text": "Test" + }, + { + "id": 2581, + "logprob": -11.6171875, + "text": " request" + } + ], + "seed": null, + "tokens": [ + { + "id": 25, + "logprob": -2.3203125, + "special": false, + "text": ":" + }, + { + "id": 1391, + "logprob": -0.98779297, + "special": false, + "text": " {" + }, + { + "id": 25927, + "logprob": -0.7729492, + "special": false, + "text": "request" + }, + { + "id": 92, + "logprob": -0.7241211, + "special": false, + "text": "}" + }, + { + "id": 4943, + "logprob": -0.4091797, + "special": false, + "text": "\")" + }, + { + "id": 198, + "logprob": -0.119018555, + "special": false, + "text": "\n" + }, + { + "id": 50280, + "logprob": -0.9707031, + "special": false, + "text": " " + }, + { + "id": 26209, + "logprob": -1.4414062, + "special": false, + "text": "response" + }, + { + "id": 796, + "logprob": -0.056854248, + "special": false, + "text": " =" + }, + { + "id": 2116, + "logprob": -1.1533203, + "special": false, + "text": " self" + } + ], + "top_tokens": null + }, + "generated_text": ": {request}\")\n response = self" + } +] diff --git a/integration-tests/models/test_flash_phi.py b/integration-tests/models/test_flash_phi.py new file mode 100644 index 00000000..6391f2a1 --- /dev/null +++ b/integration-tests/models/test_flash_phi.py @@ -0,0 +1,65 @@ +import pytest + + +@pytest.fixture(scope="module") +def flash_phi_handle(launcher): + with launcher("microsoft/phi-2", num_shard=1) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def flash_phi(flash_phi_handle): + await flash_phi_handle.health(300) + return flash_phi_handle.client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_phi(flash_phi, response_snapshot): + response = await flash_phi.generate( + "Test request", max_new_tokens=10, decoder_input_details=True + ) + + assert response.details.generated_tokens == 10 + assert response.generated_text == ": {request}\")\n response = self" + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_phi_all_params(flash_phi, response_snapshot): + response = await flash_phi.generate( + "Test request", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + stop_sequences=["network"], + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + decoder_input_details=True, + seed=0, + ) + + assert response.details.generated_tokens == 6 + assert response.generated_text == "Test request to send data over a network" + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_phi_load(flash_phi, generate_load, response_snapshot): + responses = await generate_load( + flash_phi, "Test request", max_new_tokens=10, n=4 + ) + + assert len(responses) == 4 + assert all( + [r.generated_text == responses[0].generated_text for r in responses] + ), f"{[r.generated_text for r in responses]}" + assert responses[0].generated_text == ": {request}\")\n response = self" + + assert responses == response_snapshot diff --git a/server/pyproject.toml b/server/pyproject.toml index 6e9be43e..d1452678 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -26,7 +26,7 @@ hf-transfer = "^0.1.2" sentencepiece = "^0.1.97" tokenizers = "^0.15.0" huggingface-hub = "^0.19.3" -transformers = "^4.36.1" +transformers = "^4.37.1" einops = "^0.6.1" texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 39d1d58e..679e1e2f 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -18,6 +18,7 @@ from text_generation_server.models.santacoder import SantaCoder from text_generation_server.models.t5 import T5Sharded from text_generation_server.models.gpt_neox import GPTNeoxSharded +from text_generation_server.models.phi import Phi # The flag below controls whether to allow TF32 on matmul. This flag defaults to False # in PyTorch 1.12 and later. @@ -57,6 +58,7 @@ from text_generation_server.models.idefics import IDEFICSSharded from text_generation_server.models.flash_mistral import FlashMistral from text_generation_server.models.flash_mixtral import FlashMixtral + from text_generation_server.models.flash_phi import FlashPhi from text_generation_server.utils.flash_attn import HAS_FLASH_ATTN_V2_CUDA except ImportError as e: @@ -72,6 +74,7 @@ __all__.append(IDEFICSSharded) __all__.append(FlashMistral) __all__.append(FlashMixtral) + __all__.append(FlashPhi) def get_model( @@ -227,6 +230,37 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) + + elif model_type == "phi": + if FLASH_ATTENTION: + return FlashPhi( + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, + use_medusa=use_medusa, + ) + else: + return CausalLM( + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, + ) + + elif model_type == "phi-msft": + if FLASH_ATTENTION: + raise NotImplementedError("Legacy phi-msft is not supported with Flash Attention") + else: + return Phi( + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, + ) elif model_type == "llama" or model_type == "baichuan": if FLASH_ATTENTION: diff --git a/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py new file mode 100644 index 00000000..d103973f --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py @@ -0,0 +1,400 @@ +import torch +import torch.distributed + +from torch import nn +from transformers.activations import ACT2FN +from transformers.configuration_utils import PretrainedConfig +from typing import Optional, List, Tuple + +from text_generation_server.utils import paged_attention, flash_attn +from text_generation_server.utils.layers import ( + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, + PositionRotaryEmbedding, + TensorParallelHead, + get_linear, + FastLayerNorm, +) + +class PhiConfig(PretrainedConfig): + def __init__( + self, + vocab_size=51200, + hidden_size=2560, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=32, + hidden_act="gelu_fast", # llama uses silu + layer_norm_eps=1e-05, # rms in llama, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + tie_word_embeddings=False, + rope_theta=10000.0, + resid_pdrop=0.1, # llama doesn't have this + partial_rotary_factor=0.5, # important difference between llama and phi + **kwargs, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.layer_norm_eps = layer_norm_eps + self.rope_theta = rope_theta + self.resid_pdrop = resid_pdrop + self.partial_rotary_factor = partial_rotary_factor + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + +# this is the same as llama except for Phi uses bias=True +def load_attention(config, prefix, weights): + if config.num_attention_heads != config.num_key_value_heads: + return _load_gqa(config, prefix, weights) + else: + return TensorParallelColumnLinear.load_multi( + config, + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + dim=0, + weights=weights, + bias=True, + ) + +def _load_gqa(config, prefix: str, weights): + assert config.hidden_size % config.num_attention_heads == 0 + assert config.num_attention_heads % weights.process_group.size() == 0 + + weight = weights.get_multi_weights_col( + prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"], + quantize=config.quantize, + dim=0, + ) + + if config.quantize not in ["gptq", "awq"]: + weight = weight.to(dtype=weights.dtype).to(device=weights.device) + + head_size = config.hidden_size // config.num_attention_heads + num_heads = config.num_attention_heads // weights.process_group.size() + num_key_value_heads = config.num_key_value_heads // weights.process_group.size() + assert list(weight.shape) == [ + (num_heads + 2 * num_key_value_heads) * head_size, + config.hidden_size, + ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}" + + # this is the same as llama except for Phi uses bias=True + return TensorParallelColumnLinear( + get_linear(weight, bias=True, quantize=config.quantize) + ) + +class FlashPhiAttention(torch.nn.Module): + def __init__( + self, + prefix: str, + config, + weights, + ): + super().__init__() + self.num_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.head_size = self.hidden_size // self.num_heads + + self.softmax_scale = self.head_size**-0.5 + self.rotary_dim = int(config.partial_rotary_factor * self.head_size) + + self.rotary_emb = PositionRotaryEmbedding.static( + config=config, + dim=self.rotary_dim, + base=config.rope_theta, + device=weights.device, + ) + + if self.num_heads % weights.process_group.size() != 0: + raise ValueError( + f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " + f"and `num_shards`: {weights.process_group.size()}" + ) + + self.num_heads = self.num_heads // weights.process_group.size() + self.num_key_value_heads = ( + config.num_key_value_heads // weights.process_group.size() + ) + + self.query_key_value = load_attention(config, prefix, weights) + + # in llama the dense layer is called "o_proj" and has bias=False + self.dense = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.dense", + weights=weights, + bias=True, + ) + self.num_groups = self.num_heads // self.num_key_value_heads + self.kv_head_mapping = torch.arange( + 0, self.num_key_value_heads, dtype=torch.int32, device=weights.device + ).repeat_interleave(self.num_groups) + + def forward( + self, + hidden_states, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + ): + # Compute query, key, value and split + qkv = self.query_key_value(hidden_states) + query, kv = qkv.split( + [ + self.head_size * self.num_heads, + 2 * self.head_size * self.num_key_value_heads, + ], + dim=1, + ) + + # Reshape query and key for rotary embeddings + query = query.view(-1, self.num_heads, self.head_size) + kv = kv.view(-1, 2, self.num_key_value_heads, self.head_size) + + # NOTE: this is the main difference between Llama and Phi + # in llama the rotary embeddings are applied to the whole query and key. + # Phi uses PARTIAL rotary embeddings, which are applied to the first 32 dimensions + # + # Apply partial positional embeddings in place + self.rotary_emb( + query[:, :, :self.rotary_dim], kv[:, 0, :, :self.rotary_dim], + cos, sin + ) + + # Reshape key and value and cache + paged_attention.reshape_and_cache( + kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots + ) + + # output tensor + attn_output = torch.empty_like(query) + + # Prefill + if cu_seqlen_prefill is not None: + flash_attn.attention( + query, + torch.select(kv, dim=1, index=0), + torch.select(kv, dim=1, index=1), + attn_output, + cu_seqlen_prefill, + max_s, + self.softmax_scale, + ) + # Decode + else: + paged_attention.attention( + attn_output, + query, + kv_cache[0], + kv_cache[1], + self.kv_head_mapping, + self.softmax_scale, + block_tables, + input_lengths, + max_s, + ) + + return self.dense(attn_output.view(-1, self.num_heads*self.head_size)) + +class PhiMLP(nn.Module): + def __init__(self, prefix, config, weights): + super().__init__() + act = config.hidden_act + self.act = ( + ACT2FN[act] + if "gelu" not in act + else lambda x: torch.nn.functional.gelu( + x, + approximate="tanh" + if act in ["gelu_fast", "gelu_pytorch_tanh"] + else "none", + ) + ) + + # llama weights are up_proj and down_proj and bias=False + self.up_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.fc1", + weights=weights, + bias=True, + ) + self.down_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.fc2", + weights=weights, + bias=True, + ) + + def forward(self, hidden_states): + # NOTE: Llama requires the gate up states to an intermediate size + # Phi does not and we can avoid the `view` operation + return self.down_proj(self.act(self.up_proj(hidden_states))) + + +class FlashPhiLayer(nn.Module): + def __init__(self, layer_id, config, weights): + super().__init__() + prefix = f"model.layers.{layer_id}" + self.self_attn = FlashPhiAttention( + prefix=f"{prefix}.self_attn", config=config, weights=weights + ) + self.mlp = PhiMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) + self.input_layernorm = FastLayerNorm.load( + prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.layer_norm_eps + ) + self.resid_dropout = torch.nn.Dropout(config.resid_pdrop) + + def forward( + self, + hidden_states, + residual, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + ): + hidden_states, res = self.input_layernorm(hidden_states, residual) + # Self Attention + attn_output = self.self_attn( + hidden_states, + cos, + sin, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + ) + + hidden_states = self.resid_dropout(attn_output).add(self.resid_dropout(self.mlp(hidden_states))) + + return hidden_states, res + +class FlashPhiModel(torch.nn.Module): + def __init__(self, config, weights): + super().__init__() + + process_group = weights.process_group + self.tp_rank = process_group.rank() + self.tp_world_size = process_group.size() + self.embed_tokens = TensorParallelEmbedding( + prefix="model.embed_tokens", weights=weights + ) + self.layers = nn.ModuleList( + [ + FlashPhiLayer( + layer_id, + config, + weights, + ) + for layer_id in range(config.num_hidden_layers) + ] + ) + self.gradient_checkpointing = False + + self.head_size = self.layers[0].self_attn.head_size + self.num_heads = self.layers[0].self_attn.num_heads + self.num_key_value_heads = self.layers[0].self_attn.num_key_value_heads + + self.norm = FastLayerNorm.load( + prefix="model.final_layernorm", + weights=weights, + eps=config.layer_norm_eps, + ) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlen_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + ) -> torch.Tensor: + hidden_states = self.embed_tokens(input_ids) + + # Get rotary cos and sin for this forward + # Avoid to index in each layer + cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin( + position_ids, max_s, hidden_states.dtype + ) + + residual = None + for i, layer in enumerate(self.layers): + hidden_states, residual = layer( + hidden_states, + residual, + cos, + sin, + cu_seqlen_prefill, + kv_cache[i], + block_tables, + slots, + input_lengths, + max_s, + ) + + hidden_states, _ = self.norm(hidden_states, residual) + + return hidden_states + +class FlashPhiForCausalLM(torch.nn.Module): + def __init__(self, config, weights): + super().__init__() + + self.model = FlashPhiModel(config, weights) + self.lm_head = TensorParallelHead.load( + config, + prefix="lm_head", + weights=weights, + ) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlen_prefill: Optional[torch.Tensor], + kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], + block_tables: torch.Tensor, + slots: torch.Tensor, + input_lengths: torch.Tensor, + max_s: int, + lm_head_indices: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + hidden_states = self.model( + input_ids, + position_ids, + cu_seqlen_prefill, + kv_cache, + block_tables, + slots, + input_lengths, + max_s, + ) + if lm_head_indices is not None: + hidden_states = hidden_states[lm_head_indices] + + return self.lm_head(hidden_states) diff --git a/server/text_generation_server/models/custom_modeling/phi_modeling.py b/server/text_generation_server/models/custom_modeling/phi_modeling.py new file mode 100644 index 00000000..f9999537 --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/phi_modeling.py @@ -0,0 +1,308 @@ +# imlementation of the PhiModel and PhiForCausalLM classes + +import torch +import torch.distributed + +import math +from torch import nn +from typing import Optional, List, Tuple, Any +from transformers.configuration_utils import PretrainedConfig +from transformers.modeling_outputs import CausalLMOutputWithPast + +from text_generation_server.utils.layers import ( + TensorParallelRowLinear, + TensorParallelColumnLinear, + TensorParallelEmbedding, + TensorParallelHead, + FastLinear, +) + + +# PhiConfig is the configuration class for the PhiModel. +class PhiConfig(PretrainedConfig): + def __init__( + self, + vocab_size=51200, + n_positions=2048, + n_embd=2560, + n_layer=32, + n_inner=None, + n_head=32, + rotary_dim=32, + layer_norm_epsilon=1e-5, + tie_word_embeddings=False, + pad_vocab_size_multiple=64, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + no_bias=False, + **kwargs, + ): + self.vocab_size = vocab_size + self.n_positions = n_positions + self.n_embd = n_embd + self.n_layer = n_layer + self.n_inner = n_inner + self.n_head = n_head + self.rotary_dim = rotary_dim + + self.layer_norm_epsilon = layer_norm_epsilon + self.tie_word_embeddings = tie_word_embeddings + self.pad_vocab_size_multiple = pad_vocab_size_multiple + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + self.no_bias = no_bias + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + +# RotaryEmbedding is a class that implements the rotary embedding. +class RotaryEmbedding(nn.Module): + def __init__(self, dim, max_seq_len): + super().__init__() + inv_freq = [ + 1.0 / 10000.0 ** (i / dim) + for i in range(0, dim, 2) + ] + inv_freq_len = len(inv_freq) + inv_freq = torch.tensor(inv_freq).view(1, inv_freq_len) + t = torch.arange(0, max_seq_len, dtype=torch.float).view(max_seq_len, 1) + freqs = t.matmul(inv_freq) + self.sin = freqs.sin() + self.cos = freqs.cos() + + def apply_rotary_emb_qkv(self, qkv, seqlen_offset): + b_size, seqlen, three, _, _headdim = qkv.shape + if three != 3: + raise Exception("unexpected shape for qkv") + _, rotary_dim = self.cos.shape + rotary_dim = rotary_dim * 2 + q_rot = qkv[:, :, 0, :, :rotary_dim] + q_pass = qkv[:, :, 0, :, rotary_dim:] + k_rot = qkv[:, :, 1, :, :rotary_dim] + k_pass = qkv[:, :, 1, :, rotary_dim:] + q12 = torch.chunk(q_rot, 2, dim=-1) + k12 = torch.chunk(k_rot, 2, dim=-1) + q1, q2 = q12[0], q12[1] + k1, k2 = k12[0], k12[1] + c = self.cos.narrow(0, seqlen_offset, seqlen).unsqueeze(1) + s = self.sin.narrow(0, seqlen_offset, seqlen).unsqueeze(1) + q_rot = torch.cat( + [ + q1 * c - q2 * s, + q1 * s + q2 * c, + ], + dim=-1, + ) + k_rot = torch.cat( + [ + k1 * c - k2 * s, + k1 * s + k2 * c, + ], + dim=-1, + ) + q = torch.cat([q_rot, q_pass], dim=-1) + k = torch.cat([k_rot, k_pass], dim=-1) + v = qkv[:, :, 2] + return q, k, v + + +# PhiCausalLMHead is the head of the PhiModel. It is a linear layer with a layer norm. +class PhiCausalLMHead(nn.Module): + def __init__(self, config, weights): + super().__init__() + self.ln = nn.LayerNorm.load( + prefix="lm_head.ln", + weights=weights, + eps=config.layer_norm_epsilon, + ) + self.linear = TensorParallelHead.load( + config=config, prefix="lm_head.linear", weights=weights + ) + + def forward(self, hidden_states): + hidden_states = self.ln(hidden_states) + hidden_states = self.linear(hidden_states) + return hidden_states + +# PhiMHA is a multi-head attention layer. This layer uses an attention mask to prevent tokens from attending to subsequent tokens. +class PhiMHA(nn.Module): + def __init__(self, prefix, config, weights): + super().__init__() + self.Wqkv = TensorParallelColumnLinear.load( + config, prefix=f"{prefix}.Wqkv", weights=weights, bias=not config.no_bias + ) + self.out_proj = TensorParallelRowLinear.load( + config, + prefix=f"{prefix}.out_proj", + weights=weights, + bias=not config.no_bias, + ) + self.op_size = config.n_embd + self.head_dim = int(config.n_embd / config.n_head) + self.num_heads = config.n_head + self.rotary_emb = RotaryEmbedding( + config.rotary_dim, + config.n_positions, + ) + self.softmax_scale = 1.0 / math.sqrt(self.head_dim) + + def forward( + self, + hidden_states, + past_kv_cache, + attention_mask=None, + ): + b_size, seq_len, _n_embd = hidden_states.shape + qkv = self.Wqkv(hidden_states) + qkv = qkv.view(b_size, seq_len, 3, self.num_heads, self.head_dim) + seqlen_offset = 0 if past_kv_cache is None else past_kv_cache[0].shape[1] + q, k, v = self.rotary_emb.apply_rotary_emb_qkv(qkv, seqlen_offset) + + # if there is a kv_cache, then we need to concatenate + if past_kv_cache is not None: + prev_k, prev_v = past_kv_cache + k = torch.cat([prev_k, k], dim=1) + v = torch.cat([prev_v, v], dim=1) + + past_kv_cache = [k, v] + attn_weights = torch.einsum('bthd,bshd->bhts', q, k * self.softmax_scale) + + if attention_mask is not None: + seqlen_k = k.shape[1] + seqlen_q = q.shape[1] + causal_mask = torch.triu(torch.full((seqlen_q, seqlen_k), -10000.0, device=attn_weights.device), 1) + attn_weights = attn_weights + causal_mask.to(dtype=attn_weights.dtype) + + attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1) + attn_output = attn_weights.matmul(v.transpose(1, 2)).squeeze(0) + attn_output = attn_output.view((b_size, self.num_heads, seq_len, self.head_dim)).transpose(1, 2).flatten(-2) + return self.out_proj(attn_output), past_kv_cache + +# PhiMLP is a multi-layer perceptron. It contains two linear layers with a gelu activation function. +class PhiMLP(nn.Module): + def __init__(self, prefix, config, weights): + super().__init__() + + self.n_inner = config.n_inner + self.fc1 = FastLinear.load( + config=config, + prefix=f"{prefix}.fc1", + weights=weights, + bias=False, + ) + self.fc2 = FastLinear.load( + config=config, + prefix=f"{prefix}.fc2", + weights=weights, + bias=False, + ) + self.activation = torch.nn.functional.gelu + + def forward(self, hidden_states): + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + +# PhiBlock is a single transformer block. It contains a layer norm, a multi-head attention layer and an multi-layer perceptron. +class PhiBlock(nn.Module): + def __init__(self, layer_id, config, weights): + super().__init__() + self.layer_id = layer_id + self.layer_norm = nn.LayerNorm.load(prefix=f"{layer_id}.ln", weights=weights, eps=config.layer_norm_epsilon) + self.mixer = PhiMHA(prefix=f"{layer_id}.mixer", config=config, weights=weights) + self.mlp = PhiMLP(prefix=f"{layer_id}.mlp", config=config, weights=weights) + + def forward( + self, + hidden_states, + kv_cache, + attention_mask, + ): + residual = hidden_states + hidden_states = self.layer_norm(hidden_states) + attn_outputs, past_kv_cache = self.mixer(hidden_states, kv_cache, attention_mask) + feed_forward_hidden_states = self.mlp(hidden_states) + out = attn_outputs + feed_forward_hidden_states + residual + return out, past_kv_cache + +# PhiModel implements the embedding layer and the transformer blocks. +class PhiModel(nn.Module): + def __init__(self, config, weights): + super().__init__() + self.tp_rank = weights.process_group.rank() + self.tp_world_size = weights.process_group.size() + self.embed_tokens = TensorParallelEmbedding( + prefix="transformer.embd.wte", weights=weights + ) + self.blocks = nn.ModuleList( + [PhiBlock(f"transformer.h.{layer_id}", config, weights) for layer_id in range(config.n_layer)] + ) + + def forward( + self, + input_ids: torch.LongTensor, + past_key_values: Optional[List[Tuple[torch.FloatTensor]]] = None, + attention_mask: Optional[torch.ByteTensor] = None, + return_dict: Optional[bool] = None, + use_cache: Optional[bool] = None, + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + hidden_states = self.embed_tokens(input_ids) + seq_len = hidden_states.shape[1] + mask = None if seq_len <= 1 else attention_mask + + past_key_values = [None] * len(self.blocks) if past_key_values is None else past_key_values + + for index, block in enumerate(self.blocks): + hidden_states, new_key_values = block(hidden_states, past_key_values[index], mask) + past_key_values[index] = new_key_values + + return hidden_states, past_key_values + +# PhiForCausalLM wraps the PhiModel and PhiCausalLMHead together and returns a CausalLMOutputWithPast object. +class PhiForCausalLM(torch.nn.Module): + def __init__(self, config, weights): + super().__init__() + self.model = PhiModel(config, weights) + self.lm_head = PhiCausalLMHead(config, weights) + + def forward( + self, + input_ids: torch.LongTensor, + past_key_values: Optional[List[Tuple[torch.FloatTensor]]] = None, + attention_mask: Optional[torch.ByteTensor] = None, + return_dict: Optional[bool] = None, + use_cache: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, + ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]: + model_output = self.model( + input_ids, past_key_values, attention_mask, return_dict, use_cache + ) + logits = self.lm_head(model_output[0]) + + loss = None + if labels is not None: + loss = nn.CrossEntropyLoss()( + logits[:, :-1].view(-1, logits.size(-1)), + labels[:, 1:].view(-1) + ) + + if not return_dict: + return ((loss,) + (logits,) + model_output[1:]) if loss is not None else (logits,) + model_output[1:] + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=model_output[1], + hidden_states=None, + attentions=None, + ) + + diff --git a/server/text_generation_server/models/flash_phi.py b/server/text_generation_server/models/flash_phi.py new file mode 100644 index 00000000..1c49f2a9 --- /dev/null +++ b/server/text_generation_server/models/flash_phi.py @@ -0,0 +1,102 @@ +import torch +import torch.distributed + +from opentelemetry import trace +from transformers import AutoConfig, AutoTokenizer +from typing import Optional + +from text_generation_server.models import FlashCausalLM +from text_generation_server.models.custom_modeling.flash_phi_modeling import ( + FlashPhiForCausalLM, + PhiConfig, +) +from text_generation_server.utils import ( + initialize_torch_distributed, + weight_files, + Weights, +) + +tracer = trace.get_tracer(__name__) + + +class FlashPhi(FlashCausalLM): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, + use_medusa: Optional[str] = None, + ): + self.process_group, rank, world_size = initialize_torch_distributed() + if torch.cuda.is_available(): + device = torch.device(f"cuda:{rank}") + dtype = torch.float16 if dtype is None else dtype + else: + raise NotImplementedError("FlashPhi is only available on GPU") + + tokenizer = AutoTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + + config = PhiConfig.from_pretrained( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) + config.quantize = quantize + + torch.distributed.barrier(group=self.process_group) + + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights(filenames, device, dtype, process_group=self.process_group) + if config.quantize in ["gptq", "awq"]: + weights._set_gptq_params(model_id, revision) + + model = FlashPhiForCausalLM(config, weights) + if use_medusa: + from text_generation_server.utils.medusa import MedusaModel + from huggingface_hub import hf_hub_download + import json + import os + from pathlib import Path + + is_local_model = (Path(use_medusa).exists() and Path(use_medusa).is_dir()) or os.getenv( + "WEIGHTS_CACHE_OVERRIDE", None + ) is not None + + if not is_local_model: + medusa_config = hf_hub_download( + use_medusa, revision=revision, filename="config.json" + ) + medusa_head = hf_hub_download( + use_medusa, revision=revision, filename="medusa_lm_head.pt" + ) + else: + medusa_config = str(Path(use_medusa) / "config.json") + medusa_head = str(Path(use_medusa) / "medusa_lm_head.pt") + + with open(medusa_config, "r") as f: + config = json.load(f) + medusa_sf = medusa_head[: -len(".pt")] + ".safetensors" + weights = Weights( + [medusa_sf], device, dtype, process_group=self.process_group + ) + lm_head = model.lm_head + model.lm_head = MedusaModel(config, weights, lm_head) + + torch.distributed.barrier(group=self.process_group) + super(FlashPhi, self).__init__( + model=model, + tokenizer=tokenizer, + num_layers=len(model.model.layers), + num_kv_heads=model.model.num_key_value_heads, + head_size=model.model.head_size, + dtype=dtype, + device=device, + rank=rank, + world_size=world_size, + ) diff --git a/server/text_generation_server/models/phi.py b/server/text_generation_server/models/phi.py new file mode 100644 index 00000000..d477478a --- /dev/null +++ b/server/text_generation_server/models/phi.py @@ -0,0 +1,63 @@ +import torch +import torch.distributed + +from transformers import AutoConfig, AutoTokenizer +from typing import Optional, List, Tuple + +from text_generation_server.models import CausalLM +from text_generation_server.models.custom_modeling.phi_modeling import PhiConfig, PhiForCausalLM +from text_generation_server.utils import ( + initialize_torch_distributed, + weight_files, + Weights, +) + +class Phi(CausalLM): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, + ): + self.process_group, _rank, _world_size = initialize_torch_distributed() + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.float16 if dtype is None else dtype + else: + if quantize: + raise ValueError("quantization is not available on CPU") + + device = torch.device("cpu") + dtype = torch.float32 if dtype is None else dtype + + tokenizer = AutoTokenizer.from_pretrained( + model_id, + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + config = PhiConfig.from_pretrained( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) + + tokenizer.bos_token_id = config.bos_token_id + tokenizer.eos_token_id = config.eos_token_id + tokenizer.pad_token = tokenizer.eos_token + + config.quantize = quantize + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights(filenames, device, dtype, process_group=self.process_group) + model = PhiForCausalLM(config, weights) + torch.distributed.barrier(group=self.process_group) + super(CausalLM, self).__init__( + model=model, + tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, + device=device, + ) + From 9c320e260b404c0863358bf0b3bff5bd06cb8745 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 25 Jan 2024 18:16:03 +0100 Subject: [PATCH 529/793] fix: read stderr in download (#1486) #1186 --- launcher/src/main.rs | 48 +++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index c5553000..e635a721 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -4,7 +4,7 @@ use nix::unistd::Pid; use serde::Deserialize; use std::env; use std::ffi::OsString; -use std::io::{BufRead, BufReader, Lines, Read}; +use std::io::{BufRead, BufReader, Lines}; use std::os::unix::process::{CommandExt, ExitStatusExt}; use std::path::Path; use std::process::{Child, Command, ExitStatus, Stdio}; @@ -489,6 +489,9 @@ fn shard_manager( // Safetensors load fast envs.push(("SAFETENSORS_FAST_GPU".into(), "1".into())); + // Disable progress bar + envs.push(("HF_HUB_DISABLE_PROGRESS_BARS".into(), "1".into())); + // Enable hf transfer for insane download speeds let enable_hf_transfer = env::var("HF_HUB_ENABLE_HF_TRANSFER").unwrap_or("1".to_string()); envs.push(( @@ -573,6 +576,13 @@ fn shard_manager( thread::spawn(move || { log_lines(shard_stdout_reader.lines()); }); + // We read stderr in another thread as it seems that lines() can block in some cases + let (err_sender, err_receiver) = mpsc::channel(); + thread::spawn(move || { + for line in shard_stderr_reader.lines().flatten() { + err_sender.send(line).unwrap_or(()); + } + }); let mut ready = false; let start_time = Instant::now(); @@ -580,13 +590,6 @@ fn shard_manager( loop { // Process exited if let Some(exit_status) = p.try_wait().unwrap() { - // We read stderr in another thread as it seems that lines() can block in some cases - let (err_sender, err_receiver) = mpsc::channel(); - thread::spawn(move || { - for line in shard_stderr_reader.lines().flatten() { - err_sender.send(line).unwrap_or(()); - } - }); let mut err = String::new(); while let Ok(line) = err_receiver.recv_timeout(Duration::from_millis(10)) { err = err + "\n" + &line; @@ -782,6 +785,9 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L // Copy current process env let mut envs: Vec<(OsString, OsString)> = env::vars_os().collect(); + // Disable progress bar + envs.push(("HF_HUB_DISABLE_PROGRESS_BARS".into(), "1".into())); + // If huggingface_hub_cache is set, pass it to the download process // Useful when running inside a docker container if let Some(ref huggingface_hub_cache) = args.huggingface_hub_cache { @@ -832,12 +838,20 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L } }; - // Redirect STDOUT to the console - let download_stdout = download_process.stdout.take().unwrap(); - let stdout = BufReader::new(download_stdout); + let download_stdout = BufReader::new(download_process.stdout.take().unwrap()); + + thread::spawn(move || { + log_lines(download_stdout.lines()); + }); + let download_stderr = BufReader::new(download_process.stderr.take().unwrap()); + + // We read stderr in another thread as it seems that lines() can block in some cases + let (err_sender, err_receiver) = mpsc::channel(); thread::spawn(move || { - log_lines(stdout.lines()); + for line in download_stderr.lines().flatten() { + err_sender.send(line).unwrap_or(()); + } }); loop { @@ -848,12 +862,10 @@ fn download_convert_model(args: &Args, running: Arc) -> Result<(), L } let mut err = String::new(); - download_process - .stderr - .take() - .unwrap() - .read_to_string(&mut err) - .unwrap(); + while let Ok(line) = err_receiver.recv_timeout(Duration::from_millis(10)) { + err = err + "\n" + &line; + } + if let Some(signal) = status.signal() { tracing::error!( "Download process was signaled to shutdown with signal {signal}: {err}" From 17b7b75e652394379931c058a8c2db3a000b4225 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 26 Jan 2024 10:13:23 +0100 Subject: [PATCH 530/793] Update the docs --- README.md | 2 +- docs/source/supported_models.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1b3041d3..73356f28 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ Be aware that the official Docker image has them enabled by default. ## Optimized architectures -TGI works out of the box to serve optimized models in [this list](https://huggingface.co/docs/text-generation-inference/supported_models). +TGI works out of the box to serve optimized models for all modern models. They can be found in [this list](https://huggingface.co/docs/text-generation-inference/supported_models). Other architectures are supported on a best-effort basis using: diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md index dce4f2f9..004790ab 100644 --- a/docs/source/supported_models.md +++ b/docs/source/supported_models.md @@ -19,7 +19,9 @@ The following models are optimized and can be served with TGI, which uses custom - [MPT](https://huggingface.co/mosaicml/mpt-30b) - [Llama V2](https://huggingface.co/meta-llama) - [Code Llama](https://huggingface.co/codellama) -- [Mistral](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) +- [Mistral](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) +- [Mixtral](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1) +- [Phi](https://huggingface.co/microsoft/phi-2) If the above list lacks the model you would like to serve, depending on the model's pipeline type, you can try to initialize and serve the model anyways to see how well it performs, but performance isn't guaranteed for non-optimized models: From 13dd8e2361759f977832a41e7eeff0caf1781b6b Mon Sep 17 00:00:00 2001 From: drbh Date: Fri, 26 Jan 2024 04:41:39 -0500 Subject: [PATCH 531/793] fix: show warning with tokenizer config parsing error (#1488) This tiny PR just prints the parsing error when a tokenizer config fails to load. This is helpful when a chat_template wont load due to formatting issues https://github.com/huggingface/text-generation-inference/pull/1427#issuecomment-1909226388 --- router/src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/router/src/main.rs b/router/src/main.rs index b6190908..495fd5bc 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -462,7 +462,12 @@ pub async fn get_tokenizer_config(api_repo: &ApiRepo) -> Option Date: Fri, 26 Jan 2024 10:41:58 +0100 Subject: [PATCH 532/793] fix: launcher doc typos (#1473) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Andres Restrepo --- docs/source/basic_tutorials/launcher.md | 4 ++-- launcher/src/main.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index 9590e463..bafe3669 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -60,9 +60,9 @@ Options: [env: QUANTIZE=] Possible values: - - awq: 4 bit quantization. Requires a specific GTPQ quantized model: https://hf.co/models?search=awq. Should replace GPTQ models whereever possible because of the better latency + - awq: 4 bit quantization. Requires a specific AWQ quantized model: https://hf.co/models?search=awq. Should replace GPTQ models wherever possible because of the better latency - eetq: 8 bit quantization, doesn't require specific model. Should be a drop-in replacement to bitsandbytes with much better performance. Kernels are from https://github.com/NetEase-FuXi/EETQ.git - - gptq: 4 bit quantization. Requires a specific GTPQ quantized model: https://hf.co/models?search=gptq. text-generation-inference will use exllama (faster) kernels whereever possible, and use triton kernel (wider support) when it's not. AWQ has faster kernels + - gptq: 4 bit quantization. Requires a specific GTPQ quantized model: https://hf.co/models?search=gptq. text-generation-inference will use exllama (faster) kernels wherever possible, and use triton kernel (wider support) when it's not. AWQ has faster kernels - bitsandbytes: Bitsandbytes 8bit. Can be applied on any model, will cut the memory requirement in half, but it is known that the model will be much slower to run than the native f16 - bitsandbytes-nf4: Bitsandbytes 4bit. Can be applied on any model, will cut the memory requirement by 4x, but it is known that the model will be much slower to run than the native f16 - bitsandbytes-fp4: Bitsandbytes 4bit. nf4 should be preferred in most cases but maybe this one has better perplexity performance for you model diff --git a/launcher/src/main.rs b/launcher/src/main.rs index e635a721..09657c91 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -21,16 +21,16 @@ mod env_runtime; #[derive(Clone, Copy, Debug, ValueEnum)] enum Quantization { - /// 4 bit quantization. Requires a specific GTPQ quantized model: + /// 4 bit quantization. Requires a specific AWQ quantized model: /// https://hf.co/models?search=awq. - /// Should replace GPTQ models whereever possible because of the better latency + /// Should replace GPTQ models wherever possible because of the better latency Awq, /// 8 bit quantization, doesn't require specific model. /// Should be a drop-in replacement to bitsandbytes with much better performance. /// Kernels are from https://github.com/NetEase-FuXi/EETQ.git Eetq, /// 4 bit quantization. Requires a specific GTPQ quantized model: https://hf.co/models?search=gptq. - /// text-generation-inference will use exllama (faster) kernels whereever possible, and use + /// text-generation-inference will use exllama (faster) kernels wherever possible, and use /// triton kernel (wider support) when it's not. /// AWQ has faster kernels. Gptq, From b95732180dc52be869e8c3e752a9c54608a6c7a5 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 26 Jan 2024 14:00:29 +0100 Subject: [PATCH 533/793] Reinstate exl2 with tp (#1490) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- ...t_flash_starcoder_gptq_default_params.json | 26 +++++++++---------- .../utils/gptq/exllamav2.py | 4 +++ server/text_generation_server/utils/layers.py | 12 ++++----- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json index 5598a2ad..1ace3814 100644 --- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json +++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json @@ -16,52 +16,52 @@ }, { "id": 21017, - "logprob": -9.09375, + "logprob": -9.0859375, "text": "ometric" }, { "id": 81, - "logprob": -0.25976562, + "logprob": -0.25830078, "text": "_" }, { "id": 6009, - "logprob": -2.2148438, + "logprob": -2.1875, "text": "mean" }, { "id": 26, - "logprob": -0.3010254, + "logprob": -0.30004883, "text": "(" }, { "id": 62, - "logprob": -5.6757812, + "logprob": -5.6171875, "text": "L" }, { "id": 44, - "logprob": -3.0898438, + "logprob": -3.078125, "text": ":" }, { "id": 1682, - "logprob": -0.6791992, + "logprob": -0.68066406, "text": " List" }, { "id": 77, - "logprob": -0.38891602, + "logprob": -0.38745117, "text": "[" }, { "id": 1808, - "logprob": -0.92041016, + "logprob": -0.9453125, "text": "float" }, { "id": 10794, - "logprob": -2.5390625, + "logprob": -2.5371094, "text": "]):" } ], @@ -69,7 +69,7 @@ "tokens": [ { "id": 284, - "logprob": 0.0, + "logprob": -0.051635742, "special": false, "text": "\n " }, @@ -81,7 +81,7 @@ }, { "id": 11665, - "logprob": -1.6005859, + "logprob": -1.2236328, "special": false, "text": " reduce" }, @@ -159,7 +159,7 @@ }, { "id": 203, - "logprob": -0.11968994, + "logprob": -0.12695312, "special": false, "text": "\n" }, diff --git a/server/text_generation_server/utils/gptq/exllamav2.py b/server/text_generation_server/utils/gptq/exllamav2.py index a24e834b..2b897f25 100644 --- a/server/text_generation_server/utils/gptq/exllamav2.py +++ b/server/text_generation_server/utils/gptq/exllamav2.py @@ -185,6 +185,10 @@ def post_init(self, temp_dq): "g_idx": self.g_idx, } temp_dq = temp_dq.get_scratch_slice(self.temp_dq_size()) + + # We NEED to keep a pointer on Python side, otherwise the garbage collector will mess with us, + # and `Memory access fault by GPU node-2` will EAT you. + self.temp_dq = temp_dq self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq) def forward(self, x, force_cuda=False): diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 5a0de0d7..c9393d99 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -35,12 +35,12 @@ HAS_EXLLAMA = False CAN_EXLLAMA = major >= 8 V2 = os.getenv("EXLLAMA_VERSION", "2") == "2" -if V2 and int(os.getenv("WORLD_SIZE", "1")) > 1: - V2 = False - log_once( - logger.warning, - "Disabling exllama v2 and using v1 instead because there are issues when sharding", - ) +# if V2 and int(os.getenv("WORLD_SIZE", "1")) > 1: +# V2 = False +# log_once( +# logger.warning, +# "Disabling exllama v2 and using v1 instead because there are issues when sharding", +# ) if os.getenv("DISABLE_EXLLAMA") == "True": HAS_EXLLAMA = False From ac49972752bd9ea4021b3aeb0ac40a4c70e6eeea Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 26 Jan 2024 14:05:02 +0100 Subject: [PATCH 534/793] Add sealion mpt support (#1477) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Choon Meng Tan Co-authored-by: David Ong Tat-Wee <13075447+ongtw@users.noreply.github.com> --- .../models/custom_modeling/mpt_modeling.py | 81 ++++++++++++++----- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/mpt_modeling.py b/server/text_generation_server/models/custom_modeling/mpt_modeling.py index 5ccf796d..1a9aef74 100644 --- a/server/text_generation_server/models/custom_modeling/mpt_modeling.py +++ b/server/text_generation_server/models/custom_modeling/mpt_modeling.py @@ -28,7 +28,6 @@ def load_col(config, prefix, weights, bias): - assert bias == False, NotImplementedError assert config.quantize != "gptq", NotImplementedError slice_ = weights._get_slice(f"{prefix}.weight") rank = weights.process_group.rank() @@ -45,7 +44,26 @@ def load_col(config, prefix, weights, bias): if weight.dtype != torch.int32: weight = weight.to(dtype=weights.dtype) weight = weight.to(device=weights.device) - bias = None + + if bias: + bias_slice_ = weights._get_slice(f"{prefix}.bias") + bias_rank = weights.process_group.rank() + bias_size = weights.process_group.size() + + bias_h = bias_slice_.get_shape() + bias_h = bias_h[0] + bias_block_size = bias_h // bias_size + + bias_q_part = bias_slice_[bias_rank * bias_block_size : (bias_rank + 1) * bias_block_size] + bias_k_part = bias_slice_[bias_h + bias_rank * bias_block_size : bias_h + (bias_rank + 1) * bias_block_size] + bias_v_part = bias_slice_[2 * bias_h + bias_rank * bias_block_size : 2 * bias_h + (bias_rank + 1) * bias_block_size] + + bias = torch.cat([bias_q_part, bias_k_part, bias_v_part], dim=0) + if bias.dtype != torch.int32: + bias = bias.to(dtype=weights.dtype) + bias = bias.to(device=weights.device) + else: + bias = None linear = get_linear(weight, bias, config.quantize) return TensorParallelColumnLinear(linear) @@ -330,7 +348,12 @@ def __init__( config, prefix=f"{prefix}.Wqkv", weights=weights, bias=not config.no_bias ) if self.qk_ln: - raise NotImplementedError("qk_ln is not supported") + bias = not config.no_bias + hidden_size = config.d_model + head_dim = hidden_size // self.n_heads + + self.q_ln = LPLayerNorm(d_model, bias=bias, prefix=f"{prefix}.q_ln", weights=weights) + self.k_ln = LPLayerNorm(self.n_heads * head_dim, prefix=f"{prefix}.k_ln", weights=weights) if self.attn_impl == "flash": self.attn_fn = flash_attn_fn elif self.attn_impl == "triton": @@ -581,12 +604,20 @@ def __init__(self, config, prefix, weights): f"""Not implemented attn {config.attn_config["attn_type"]}""" ) resid_pdrop = config.resid_pdrop - self.norm_1 = nn.LayerNorm.load_no_bias( - prefix=f"{prefix}.norm_1", weights=weights, eps=EPS - ) - self.norm_2 = nn.LayerNorm.load_no_bias( - prefix=f"{prefix}.norm_2", weights=weights, eps=EPS - ) + if config.no_bias: + self.norm_1 = nn.LayerNorm.load_no_bias( + prefix=f"{prefix}.norm_1", weights=weights, eps=EPS + ) + self.norm_2 = nn.LayerNorm.load_no_bias( + prefix=f"{prefix}.norm_2", weights=weights, eps=EPS + ) + else: + self.norm_1 = nn.LayerNorm.load( + prefix=f"{prefix}.norm_1", weights=weights, eps=EPS + ) + self.norm_2 = nn.LayerNorm.load( + prefix=f"{prefix}.norm_2", weights=weights, eps=EPS + ) self.attn = MultiheadAttention(config, prefix=f"{prefix}.attn", weights=weights) self.ffn = MPTMLP(config, prefix=f"{prefix}.ffn", weights=weights) self.resid_attn_dropout = nn.Dropout(resid_pdrop) @@ -635,6 +666,9 @@ def __init__( elementwise_affine=True, device=None, dtype=None, + bias: Optional[bool] = True, + prefix=None, + weights=None, ): super().__init__( normalized_shape=normalized_shape, @@ -642,7 +676,14 @@ def __init__( elementwise_affine=elementwise_affine, device=device, dtype=dtype, + bias=bias, ) + if weights is not None: + self.weight = nn.Parameter(weights.get_sharded(f"{prefix}.weight", dim=0)) + if bias: + self.bias = nn.Parameter(weights.get_sharded(f"{prefix}.bias", dim=0)) + self.normalized_shape = self.weight.shape + def forward(self, x): module_device = x.device @@ -755,20 +796,23 @@ def __init__(self, config, weights): ) self.wte = TensorParallelEmbedding("transformer.wte", weights) + if not self.alibi: - # self.wpe = torch.nn.Embedding( - # config.max_seq_len, config.d_model, device=config.init_device - # ) - raise RuntimeError("no alibi no supported") + self.wpe = TensorParallelEmbedding("transformer.wpe", weights) self.blocks = nn.ModuleList( [ MPTBlock(config, prefix=f"transformer.blocks.{i}", weights=weights) for i in range(config.n_layers) ] ) - self.norm_f = nn.LayerNorm.load_no_bias( - prefix="transformer.norm_f", weights=weights, eps=EPS - ) + if config.no_bias: + self.norm_f = nn.LayerNorm.load_no_bias( + prefix="transformer.norm_f", weights=weights, eps=EPS + ) + else: + self.norm_f = nn.LayerNorm.load( + prefix="transformer.norm_f", weights=weights, eps=EPS + ) self.is_causal = not self.prefix_lm self._attn_bias_initialized = False self.attn_bias = None @@ -787,8 +831,9 @@ def __init__(self, config, weights): if config.verbose: warnings.warn(f"Removing bias ({module.bias}) from {module}.") module.register_parameter("bias", None) - if config.verbose and config.verbose > 2: - print(self) + if hasattr(self.config, "verbose"): + if config.verbose and config.verbose > 2: + print(self) if "verbose" not in self.config.init_config: self.config.init_config["verbose"] = self.config.verbose if self.config.init_config["verbose"] > 1: From 4c7315dde5a7883562c004f6a2ad69eb6ed16319 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 26 Jan 2024 14:06:27 +0100 Subject: [PATCH 535/793] Trying to fix that flaky test. (#1491) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- router/src/lib.rs | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index 2bfbbacd..6c16c4b3 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -548,26 +548,12 @@ pub(crate) struct ErrorResponse { #[cfg(test)] mod tests { - use std::io::Write; use tokenizers::Tokenizer; pub(crate) async fn get_tokenizer() -> Tokenizer { - let filename = std::path::Path::new("tokenizer.json"); - if !filename.exists() { - let content = reqwest::get("https://huggingface.co/gpt2/raw/main/tokenizer.json") - .await - .unwrap() - .bytes() - .await - .unwrap(); - let tmp_filename = "tokenizer.json.temp"; - let mut file = std::fs::File::create(tmp_filename).unwrap(); - file.write_all(&content).unwrap(); - // Re-check if another process has written this file maybe. - if !filename.exists() { - std::fs::rename(tmp_filename, filename).unwrap() - } - } - Tokenizer::from_file("tokenizer.json").unwrap() + let api = hf_hub::api::sync::Api::new().unwrap(); + let repo = api.model("gpt2".to_string()); + let filename = repo.get("tokenizer.json").unwrap(); + Tokenizer::from_file(filename).unwrap() } } From 50a20a83d727de52f609e9cd51d3a85c8a4ba110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Restrepo?= Date: Fri, 26 Jan 2024 08:10:07 -0500 Subject: [PATCH 536/793] fix: launcher doc typos (#1462) # What does this PR do? fixes launcher doc typos Fixes # (issue) ## Before submitting - [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? @OlivierDehaene OR @Narsil Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. Co-authored-by: Nicolas Patry From ebecc06161d3399aa1dace7be1a7a86efec85f8d Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 26 Jan 2024 16:07:31 +0100 Subject: [PATCH 537/793] Update the docs to include newer models. (#1492) --- docs/openapi.json | 2 +- router/src/lib.rs | 31 +++++++++++++++++++++++++------ router/src/server.rs | 22 +++++++++++++++------- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 4454259b..9a9ed116 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"title":"Text Generation Inference","description":"Text Generation Webserver","contact":{"name":"Olivier Dehaene"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0"},"version":"1.3.4"},"paths":{"/":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens if `stream == false` or a stream of token if `stream == true`","description":"Generate tokens if `stream == false` or a stream of token if `stream == true`","operationId":"compat_generate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CompatGenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateResponse"}},"text/event-stream":{"schema":{"$ref":"#/components/schemas/StreamResponse"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/generate":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens","description":"Generate tokens","operationId":"generate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateResponse"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/generate_stream":{"post":{"tags":["Text Generation Inference"],"summary":"Generate a stream of token using Server-Sent Events","description":"Generate a stream of token using Server-Sent Events","operationId":"generate_stream","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/StreamResponse"}}}},"422":{"description":"Input validation error","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/health":{"get":{"tags":["Text Generation Inference"],"summary":"Health check method","description":"Health check method","operationId":"health","responses":{"200":{"description":"Everything is working fine"},"503":{"description":"Text generation inference is down","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"unhealthy","error_type":"healthcheck"}}}}}}},"/info":{"get":{"tags":["Text Generation Inference"],"summary":"Text Generation Inference endpoint info","description":"Text Generation Inference endpoint info","operationId":"get_model_info","responses":{"200":{"description":"Served model info","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Info"}}}}}}},"/metrics":{"get":{"tags":["Text Generation Inference"],"summary":"Prometheus metrics scrape endpoint","description":"Prometheus metrics scrape endpoint","operationId":"metrics","responses":{"200":{"description":"Prometheus Metrics","content":{"text/plain":{"schema":{"type":"string"}}}}}}},"/tokenize":{"post":{"tags":["Text Generation Inference"],"summary":"Tokenize inputs","description":"Tokenize inputs","operationId":"tokenize","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TokenizeRequest"}}},"required":true},"responses":{"200":{"description":"Tokenized ids","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TokenizeResponse"}}}},"404":{"description":"No tokenizer found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"No fast tokenizer available"}}}}}}},"/v1/chat/completions":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens","description":"Generate tokens","operationId":"chat_completions","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatCompletionChunk"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}}},"components":{"schemas":{"BestOfSequence":{"type":"object","required":["generated_text","finish_reason","generated_tokens","prefill","tokens"],"properties":{"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_text":{"type":"string","example":"test"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"prefill":{"type":"array","items":{"$ref":"#/components/schemas/PrefillToken"}},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}},"top_tokens":{"type":"array","items":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}}},"CompatGenerateRequest":{"type":"object","required":["inputs"],"properties":{"inputs":{"type":"string","example":"My name is Olivier and I"},"parameters":{"$ref":"#/components/schemas/GenerateParameters"},"stream":{"type":"boolean","default":"false"}}},"Details":{"type":"object","required":["finish_reason","generated_tokens","prefill","tokens"],"properties":{"best_of_sequences":{"type":"array","items":{"$ref":"#/components/schemas/BestOfSequence"},"nullable":true},"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"prefill":{"type":"array","items":{"$ref":"#/components/schemas/PrefillToken"}},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}},"top_tokens":{"type":"array","items":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}}},"ErrorResponse":{"type":"object","required":["error","error_type"],"properties":{"error":{"type":"string"},"error_type":{"type":"string"}}},"FinishReason":{"type":"string","enum":["length","eos_token","stop_sequence"]},"GenerateParameters":{"type":"object","properties":{"best_of":{"type":"integer","default":"null","example":1,"nullable":true,"minimum":0,"exclusiveMinimum":0},"decoder_input_details":{"type":"boolean","default":"true"},"details":{"type":"boolean","default":"true"},"do_sample":{"type":"boolean","default":"false","example":true},"max_new_tokens":{"type":"integer","format":"int32","default":"100","example":"20","nullable":true,"minimum":0},"repetition_penalty":{"type":"number","format":"float","default":"null","example":1.03,"nullable":true,"exclusiveMinimum":0},"return_full_text":{"type":"boolean","default":"null","example":false,"nullable":true},"seed":{"type":"integer","format":"int64","default":"null","example":"null","nullable":true,"minimum":0,"exclusiveMinimum":0},"stop":{"type":"array","items":{"type":"string"},"example":["photographer"],"maxItems":4},"temperature":{"type":"number","format":"float","default":"null","example":0.5,"nullable":true,"exclusiveMinimum":0},"top_k":{"type":"integer","format":"int32","default":"null","example":10,"nullable":true,"exclusiveMinimum":0},"top_n_tokens":{"type":"integer","format":"int32","default":"null","example":5,"nullable":true,"minimum":0,"exclusiveMinimum":0},"top_p":{"type":"number","format":"float","default":"null","example":0.95,"nullable":true,"maximum":1,"exclusiveMinimum":0},"truncate":{"type":"integer","default":"null","example":"null","nullable":true,"minimum":0},"typical_p":{"type":"number","format":"float","default":"null","example":0.95,"nullable":true,"maximum":1,"exclusiveMinimum":0},"watermark":{"type":"boolean","default":"false","example":true}}},"GenerateRequest":{"type":"object","required":["inputs"],"properties":{"inputs":{"type":"string","example":"My name is Olivier and I"},"parameters":{"$ref":"#/components/schemas/GenerateParameters"}}},"GenerateResponse":{"type":"object","required":["generated_text"],"properties":{"details":{"allOf":[{"$ref":"#/components/schemas/Details"}],"nullable":true},"generated_text":{"type":"string","example":"test"}}},"Info":{"type":"object","required":["model_id","model_dtype","model_device_type","max_concurrent_requests","max_best_of","max_stop_sequences","max_input_length","max_total_tokens","waiting_served_ratio","max_batch_total_tokens","max_waiting_tokens","validation_workers","version"],"properties":{"docker_label":{"type":"string","example":"null","nullable":true},"max_batch_total_tokens":{"type":"integer","format":"int32","example":"32000","minimum":0},"max_best_of":{"type":"integer","example":"2","minimum":0},"max_concurrent_requests":{"type":"integer","description":"Router Parameters","example":"128","minimum":0},"max_input_length":{"type":"integer","example":"1024","minimum":0},"max_stop_sequences":{"type":"integer","example":"4","minimum":0},"max_total_tokens":{"type":"integer","example":"2048","minimum":0},"max_waiting_tokens":{"type":"integer","example":"20","minimum":0},"model_device_type":{"type":"string","example":"cuda"},"model_dtype":{"type":"string","example":"torch.float16"},"model_id":{"type":"string","description":"Model info","example":"bigscience/blomm-560m"},"model_pipeline_tag":{"type":"string","example":"text-generation","nullable":true},"model_sha":{"type":"string","example":"e985a63cdc139290c5f700ff1929f0b5942cced2","nullable":true},"sha":{"type":"string","example":"null","nullable":true},"validation_workers":{"type":"integer","example":"2","minimum":0},"version":{"type":"string","description":"Router Info","example":"0.5.0"},"waiting_served_ratio":{"type":"number","format":"float","example":"1.2"}}},"PrefillToken":{"type":"object","required":["id","text","logprob"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"logprob":{"type":"number","format":"float","example":-0.34,"nullable":true},"text":{"type":"string","example":"test"}}},"StreamDetails":{"type":"object","required":["finish_reason","generated_tokens"],"properties":{"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0}}},"StreamResponse":{"type":"object","required":["index","token"],"properties":{"details":{"allOf":[{"$ref":"#/components/schemas/StreamDetails"}],"default":"null","nullable":true},"generated_text":{"type":"string","default":"null","example":"test","nullable":true},"index":{"type":"integer","format":"int32","minimum":0},"token":{"$ref":"#/components/schemas/Token"},"top_tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}},"Token":{"type":"object","required":["id","text","logprob","special"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"logprob":{"type":"number","format":"float","example":-0.34,"nullable":true},"special":{"type":"boolean","example":"false"},"text":{"type":"string","example":"test"}}}}},"tags":[{"name":"Text Generation Inference","description":"Hugging Face Text Generation Inference API"}]} +{"openapi":"3.0.3","info":{"title":"Text Generation Inference","description":"Text Generation Webserver","contact":{"name":"Olivier Dehaene"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0"},"version":"1.3.4"},"paths":{"/":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens if `stream == false` or a stream of token if `stream == true`","description":"Generate tokens if `stream == false` or a stream of token if `stream == true`","operationId":"compat_generate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CompatGenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateResponse"}},"text/event-stream":{"schema":{"$ref":"#/components/schemas/StreamResponse"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/generate":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens","description":"Generate tokens","operationId":"generate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateResponse"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/generate_stream":{"post":{"tags":["Text Generation Inference"],"summary":"Generate a stream of token using Server-Sent Events","description":"Generate a stream of token using Server-Sent Events","operationId":"generate_stream","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/StreamResponse"}}}},"422":{"description":"Input validation error","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/health":{"get":{"tags":["Text Generation Inference"],"summary":"Health check method","description":"Health check method","operationId":"health","responses":{"200":{"description":"Everything is working fine"},"503":{"description":"Text generation inference is down","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"unhealthy","error_type":"healthcheck"}}}}}}},"/info":{"get":{"tags":["Text Generation Inference"],"summary":"Text Generation Inference endpoint info","description":"Text Generation Inference endpoint info","operationId":"get_model_info","responses":{"200":{"description":"Served model info","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Info"}}}}}}},"/metrics":{"get":{"tags":["Text Generation Inference"],"summary":"Prometheus metrics scrape endpoint","description":"Prometheus metrics scrape endpoint","operationId":"metrics","responses":{"200":{"description":"Prometheus Metrics","content":{"text/plain":{"schema":{"type":"string"}}}}}}},"/tokenize":{"post":{"tags":["Text Generation Inference"],"summary":"Tokenize inputs","description":"Tokenize inputs","operationId":"tokenize","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Tokenized ids","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TokenizeResponse"}}}},"404":{"description":"No tokenizer found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"No fast tokenizer available"}}}}}}},"/v1/chat/completions":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens","description":"Generate tokens","operationId":"chat_completions","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatCompletionChunk"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}}},"components":{"schemas":{"BestOfSequence":{"type":"object","required":["generated_text","finish_reason","generated_tokens","prefill","tokens"],"properties":{"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_text":{"type":"string","example":"test"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"prefill":{"type":"array","items":{"$ref":"#/components/schemas/PrefillToken"}},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}},"top_tokens":{"type":"array","items":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}}},"ChatCompletion":{"type":"object","required":["id","object","created","model","system_fingerprint","choices","usage"],"properties":{"choices":{"type":"array","items":{"$ref":"#/components/schemas/ChatCompletionComplete"}},"created":{"type":"integer","format":"int64","example":"1706270835","minimum":0},"id":{"type":"string"},"model":{"type":"string","example":"mistralai/Mistral-7B-Instruct-v0.2"},"object":{"type":"string"},"system_fingerprint":{"type":"string"},"usage":{"$ref":"#/components/schemas/Usage"}}},"ChatCompletionChoice":{"type":"object","required":["index","delta"],"properties":{"delta":{"$ref":"#/components/schemas/ChatCompletionDelta"},"finish_reason":{"type":"string","nullable":true},"index":{"type":"integer","format":"int32","minimum":0},"logprobs":{"type":"number","format":"float","nullable":true}}},"ChatCompletionChunk":{"type":"object","required":["id","object","created","model","system_fingerprint","choices"],"properties":{"choices":{"type":"array","items":{"$ref":"#/components/schemas/ChatCompletionChoice"}},"created":{"type":"integer","format":"int64","example":"1706270978","minimum":0},"id":{"type":"string"},"model":{"type":"string","example":"mistralai/Mistral-7B-Instruct-v0.2"},"object":{"type":"string"},"system_fingerprint":{"type":"string"}}},"ChatCompletionDelta":{"type":"object","required":["role","content"],"properties":{"content":{"type":"string","example":"What is Deep Learning?"},"role":{"type":"string","example":"user"}}},"ChatRequest":{"type":"object","required":["model"],"properties":{"frequency_penalty":{"type":"number","format":"float","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.","example":"1.0","nullable":true},"logit_bias":{"type":"array","items":{"type":"number","format":"float"},"description":"UNUSED\nModify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens\n(specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically,\nthe bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model,\nbut values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should\nresult in a ban or exclusive selection of the relevant token.","nullable":true},"logprobs":{"type":"boolean","description":"Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each\noutput token returned in the content of message.","example":"false","nullable":true},"max_tokens":{"type":"integer","format":"int32","description":"The maximum number of tokens that can be generated in the chat completion.","example":"32","nullable":true,"minimum":0},"messages":{"type":"array","items":{"$ref":"#/components/schemas/Message"},"description":"A list of messages comprising the conversation so far."},"model":{"type":"string","description":"UNUSED\nID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.","example":"mistralai/Mistral-7B-Instruct-v0.2"},"n":{"type":"integer","format":"int32","description":"UNUSED\nHow many chat completion choices to generate for each input message. Note that you will be charged based on the\nnumber of generated tokens across all of the choices. Keep n as 1 to minimize costs.","example":"2","nullable":true,"minimum":0},"presence_penalty":{"type":"number","format":"float","description":"UNUSED\nNumber between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far,\nincreasing the model's likelihood to talk about new topics","example":0.1,"nullable":true},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"stream":{"type":"boolean"},"temperature":{"type":"number","format":"float","description":"What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while\nlower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.","example":1.0,"nullable":true},"top_logprobs":{"type":"integer","format":"int32","description":"UNUSED\nAn integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with\nan associated log probability. logprobs must be set to true if this parameter is used.","example":"5","nullable":true,"minimum":0},"top_p":{"type":"number","format":"float","description":"An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the\ntokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.","example":0.95,"nullable":true}}},"CompatGenerateRequest":{"type":"object","required":["inputs"],"properties":{"inputs":{"type":"string","example":"My name is Olivier and I"},"parameters":{"$ref":"#/components/schemas/GenerateParameters"},"stream":{"type":"boolean","default":"false"}}},"Details":{"type":"object","required":["finish_reason","generated_tokens","prefill","tokens"],"properties":{"best_of_sequences":{"type":"array","items":{"$ref":"#/components/schemas/BestOfSequence"},"nullable":true},"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"prefill":{"type":"array","items":{"$ref":"#/components/schemas/PrefillToken"}},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}},"top_tokens":{"type":"array","items":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}}},"ErrorResponse":{"type":"object","required":["error","error_type"],"properties":{"error":{"type":"string"},"error_type":{"type":"string"}}},"FinishReason":{"type":"string","enum":["length","eos_token","stop_sequence"],"example":"Length"},"GenerateParameters":{"type":"object","properties":{"best_of":{"type":"integer","default":"null","example":1,"nullable":true,"minimum":0,"exclusiveMinimum":0},"decoder_input_details":{"type":"boolean","default":"true"},"details":{"type":"boolean","default":"true"},"do_sample":{"type":"boolean","default":"false","example":true},"max_new_tokens":{"type":"integer","format":"int32","default":"100","example":"20","nullable":true,"minimum":0},"repetition_penalty":{"type":"number","format":"float","default":"null","example":1.03,"nullable":true,"exclusiveMinimum":0},"return_full_text":{"type":"boolean","default":"null","example":false,"nullable":true},"seed":{"type":"integer","format":"int64","default":"null","example":"null","nullable":true,"minimum":0,"exclusiveMinimum":0},"stop":{"type":"array","items":{"type":"string"},"example":["photographer"],"maxItems":4},"temperature":{"type":"number","format":"float","default":"null","example":0.5,"nullable":true,"exclusiveMinimum":0},"top_k":{"type":"integer","format":"int32","default":"null","example":10,"nullable":true,"exclusiveMinimum":0},"top_n_tokens":{"type":"integer","format":"int32","default":"null","example":5,"nullable":true,"minimum":0,"exclusiveMinimum":0},"top_p":{"type":"number","format":"float","default":"null","example":0.95,"nullable":true,"maximum":1,"exclusiveMinimum":0},"truncate":{"type":"integer","default":"null","example":"null","nullable":true,"minimum":0},"typical_p":{"type":"number","format":"float","default":"null","example":0.95,"nullable":true,"maximum":1,"exclusiveMinimum":0},"watermark":{"type":"boolean","default":"false","example":true}}},"GenerateRequest":{"type":"object","required":["inputs"],"properties":{"inputs":{"type":"string","example":"My name is Olivier and I"},"parameters":{"$ref":"#/components/schemas/GenerateParameters"}}},"GenerateResponse":{"type":"object","required":["generated_text"],"properties":{"details":{"allOf":[{"$ref":"#/components/schemas/Details"}],"nullable":true},"generated_text":{"type":"string","example":"test"}}},"Info":{"type":"object","required":["model_id","model_dtype","model_device_type","max_concurrent_requests","max_best_of","max_stop_sequences","max_input_length","max_total_tokens","waiting_served_ratio","max_batch_total_tokens","max_waiting_tokens","validation_workers","version"],"properties":{"docker_label":{"type":"string","example":"null","nullable":true},"max_batch_total_tokens":{"type":"integer","format":"int32","example":"32000","minimum":0},"max_best_of":{"type":"integer","example":"2","minimum":0},"max_concurrent_requests":{"type":"integer","description":"Router Parameters","example":"128","minimum":0},"max_input_length":{"type":"integer","example":"1024","minimum":0},"max_stop_sequences":{"type":"integer","example":"4","minimum":0},"max_total_tokens":{"type":"integer","example":"2048","minimum":0},"max_waiting_tokens":{"type":"integer","example":"20","minimum":0},"model_device_type":{"type":"string","example":"cuda"},"model_dtype":{"type":"string","example":"torch.float16"},"model_id":{"type":"string","description":"Model info","example":"bigscience/blomm-560m"},"model_pipeline_tag":{"type":"string","example":"text-generation","nullable":true},"model_sha":{"type":"string","example":"e985a63cdc139290c5f700ff1929f0b5942cced2","nullable":true},"sha":{"type":"string","example":"null","nullable":true},"validation_workers":{"type":"integer","example":"2","minimum":0},"version":{"type":"string","description":"Router Info","example":"0.5.0"},"waiting_served_ratio":{"type":"number","format":"float","example":"1.2"}}},"Message":{"type":"object","required":["role","content"],"properties":{"content":{"type":"string","example":"My name is David and I"},"role":{"type":"string","example":"user"}}},"PrefillToken":{"type":"object","required":["id","text","logprob"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"logprob":{"type":"number","format":"float","example":-0.34,"nullable":true},"text":{"type":"string","example":"test"}}},"SimpleToken":{"type":"object","required":["id","text","start","stop"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"start":{"type":"integer","example":0,"minimum":0},"stop":{"type":"integer","example":2,"minimum":0},"text":{"type":"string","example":"test"}}},"StreamDetails":{"type":"object","required":["finish_reason","generated_tokens"],"properties":{"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0}}},"StreamResponse":{"type":"object","required":["index","token"],"properties":{"details":{"allOf":[{"$ref":"#/components/schemas/StreamDetails"}],"default":"null","nullable":true},"generated_text":{"type":"string","default":"null","example":"test","nullable":true},"index":{"type":"integer","format":"int32","minimum":0},"token":{"$ref":"#/components/schemas/Token"},"top_tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}},"Token":{"type":"object","required":["id","text","logprob","special"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"logprob":{"type":"number","format":"float","example":-0.34,"nullable":true},"special":{"type":"boolean","example":"false"},"text":{"type":"string","example":"test"}}},"TokenizeResponse":{"type":"array","items":{"$ref":"#/components/schemas/SimpleToken"}}}},"tags":[{"name":"Text Generation Inference","description":"Hugging Face Text Generation Inference API"}]} \ No newline at end of file diff --git a/router/src/lib.rs b/router/src/lib.rs index 6c16c4b3..fc5670a0 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -188,18 +188,20 @@ fn default_parameters() -> GenerateParameters { } } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Deserialize, Serialize, ToSchema)] pub(crate) struct ChatCompletion { pub id: String, pub object: String, + #[schema(example = "1706270835")] pub created: u64, + #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")] pub model: String, pub system_fingerprint: String, pub choices: Vec, pub usage: Usage, } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Deserialize, Serialize, ToSchema)] pub(crate) struct ChatCompletionComplete { pub index: u32, pub message: Message, @@ -248,17 +250,19 @@ impl ChatCompletion { } } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Deserialize, Serialize, ToSchema)] pub(crate) struct ChatCompletionChunk { pub id: String, pub object: String, + #[schema(example = "1706270978")] pub created: u64, + #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")] pub model: String, pub system_fingerprint: String, pub choices: Vec, } -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Deserialize, Serialize, ToSchema)] pub(crate) struct ChatCompletionChoice { pub index: u32, pub delta: ChatCompletionDelta, @@ -266,9 +270,11 @@ pub(crate) struct ChatCompletionChoice { pub finish_reason: Option, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] pub(crate) struct ChatCompletionDelta { + #[schema(example = "user")] pub role: String, + #[schema(example = "What is Deep Learning?")] pub content: String, } @@ -311,7 +317,7 @@ fn default_request_messages() -> Vec { #[derive(Clone, Deserialize, ToSchema, Serialize)] pub(crate) struct ChatRequest { /// UNUSED - #[schema(example = "bigscience/blomm-560m")] + #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")] /// ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API. pub model: String, /* NOTE: UNUSED */ @@ -322,6 +328,7 @@ pub(crate) struct ChatRequest { /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, /// decreasing the model's likelihood to repeat the same line verbatim. #[serde(default)] + #[schema(example = "1.0")] pub frequency_penalty: Option, /// UNUSED @@ -336,28 +343,33 @@ pub(crate) struct ChatRequest { /// Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each /// output token returned in the content of message. #[serde(default)] + #[schema(example = "false")] pub logprobs: Option, /// UNUSED /// An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with /// an associated log probability. logprobs must be set to true if this parameter is used. #[serde(default)] + #[schema(example = "5")] pub top_logprobs: Option, /// The maximum number of tokens that can be generated in the chat completion. #[serde(default)] + #[schema(example = "32")] pub max_tokens: Option, /// UNUSED /// How many chat completion choices to generate for each input message. Note that you will be charged based on the /// number of generated tokens across all of the choices. Keep n as 1 to minimize costs. #[serde(default)] + #[schema(nullable = true, example = "2")] pub n: Option, /// UNUSED /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, /// increasing the model's likelihood to talk about new topics #[serde(default)] + #[schema(nullable = true, example = 0.1)] pub presence_penalty: Option, #[serde(default = "bool::default")] @@ -371,11 +383,13 @@ pub(crate) struct ChatRequest { /// /// We generally recommend altering this or `top_p` but not both. #[serde(default)] + #[schema(nullable = true, example = 1.0)] pub temperature: Option, /// An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the /// tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. #[serde(default)] + #[schema(nullable = true, example = 0.95)] pub top_p: Option, } @@ -458,6 +472,7 @@ pub struct SimpleToken { #[derive(Serialize, ToSchema)] #[serde(rename_all(serialize = "snake_case"))] +#[schema(example = "Length")] pub(crate) enum FinishReason { #[schema(rename = "length")] Length, @@ -518,6 +533,10 @@ pub(crate) struct GenerateResponse { pub details: Option

, } +#[derive(Serialize, ToSchema)] +#[serde(transparent)] +pub(crate) struct TokenizeResponse(Vec); + #[derive(Serialize, ToSchema)] pub(crate) struct StreamDetails { #[schema(example = "length")] diff --git a/router/src/server.rs b/router/src/server.rs index c5ca4665..998d6265 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -3,10 +3,10 @@ use crate::health::Health; use crate::infer::{InferError, InferResponse, InferStreamResponse}; use crate::validation::ValidationError; use crate::{ - BestOfSequence, ChatCompletion, ChatCompletionChunk, ChatRequest, CompatGenerateRequest, - Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest, GenerateResponse, - HubModelInfo, HubTokenizerConfig, Infer, Info, PrefillToken, SimpleToken, StreamDetails, - StreamResponse, Token, Validation, + BestOfSequence, ChatCompletion, ChatCompletionChoice, ChatCompletionChunk, ChatCompletionDelta, + ChatRequest, CompatGenerateRequest, Details, ErrorResponse, FinishReason, GenerateParameters, + GenerateRequest, GenerateResponse, HubModelInfo, HubTokenizerConfig, Infer, Info, Message, + PrefillToken, SimpleToken, StreamDetails, StreamResponse, Token, TokenizeResponse, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; @@ -677,7 +677,7 @@ async fn chat_completions( post, tag = "Text Generation Inference", path = "/tokenize", - request_body = TokenizeRequest, + request_body = GenerateRequest, responses( (status = 200, description = "Tokenized ids", body = TokenizeResponse), (status = 404, description = "No tokenizer found", body = ErrorResponse, @@ -688,7 +688,7 @@ async fn chat_completions( async fn tokenize( Extension(infer): Extension, Json(req): Json, -) -> Result)> { +) -> Result, (StatusCode, Json)> { let input = req.inputs.clone(); let encoding = infer.tokenize(req).await?; if let Some(encoding) = encoding { @@ -706,7 +706,7 @@ async fn tokenize( } }) .collect(); - Ok(Json(tokens).into_response()) + Ok(Json(TokenizeResponse(tokens))) } else { Err(( StatusCode::NOT_FOUND, @@ -774,10 +774,18 @@ pub async fn run( Info, CompatGenerateRequest, GenerateRequest, + ChatRequest, + Message, + ChatCompletionChoice, + ChatCompletionDelta, + ChatCompletionChunk, + ChatCompletion, GenerateParameters, PrefillToken, Token, GenerateResponse, + TokenizeResponse, + SimpleToken, BestOfSequence, Details, FinishReason, From 650fea18341a009e86b30bb38d650dfd342029c4 Mon Sep 17 00:00:00 2001 From: fxmarty <9808326+fxmarty@users.noreply.github.com> Date: Fri, 26 Jan 2024 16:27:44 +0100 Subject: [PATCH 538/793] GPTQ support on ROCm (#1489) Tested with ``` CUDA_VISIBLE_DEVICES=0 text-generation-launcher --model-id TheBloke/Llama-2-7B-Chat-GPTQ --quantize gptq EXLLAMA_VERSION=1 CUDA_VISIBLE_DEVICES=0 text-generation-launcher --model-id TheBloke/Llama-2-7B-Chat-GPTQ --quantize gptq CUDA_VISIBLE_DEVICES="0,1" text-generation-launcher --model-id TheBloke/Llama-2-7B-Chat-GPTQ --quantize gptq ``` all with good and identical results on MI210. --------- Co-authored-by: Felix Marty Co-authored-by: OlivierDehaene Co-authored-by: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> --- .gitignore | 10 +++++++ Dockerfile_amd | 24 +++++++++++++-- docs/source/supported_models.md | 4 +-- .../{cuda_compat.cuh => cu_compat.cuh} | 6 ++-- .../exllama_kernels/cuda_func/q4_matmul.cu | 7 +++-- .../exllama_kernels/hip_compat.cuh} | 29 ++++++++++++++----- .../exllama_kernels/exllama_kernels/util.cuh | 4 +++ server/exllamav2_kernels/setup.py | 11 +++++++ .../utils/gptq/exllamav2.py | 5 +--- server/text_generation_server/utils/layers.py | 2 +- 10 files changed, 80 insertions(+), 22 deletions(-) rename server/exllama_kernels/exllama_kernels/{cuda_compat.cuh => cu_compat.cuh} (91%) rename server/{exllamav2_kernels/exllamav2_kernels/cuda/compat_gemm.cuh => exllama_kernels/exllama_kernels/hip_compat.cuh} (68%) diff --git a/.gitignore b/.gitignore index 20c9baee..1f9ba162 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,13 @@ target router/tokenizer.json *__pycache__* + +# ROCm auto-generated files +*.hip +server/exllamav2_kernels/exllamav2_kernels/hip/ +server/exllama_kernels/exllama_kernels/hip/ +server/exllama_kernels/exllama_kernels/hip_func/ +*_hip.cuh +server/exllama_kernels/exllama_kernels/hip_buffers.cuh +server/exllama_kernels/exllama_kernels/exllama_ext_hip.cpp + diff --git a/Dockerfile_amd b/Dockerfile_amd index dd331a5d..d2b6f897 100644 --- a/Dockerfile_amd +++ b/Dockerfile_amd @@ -75,8 +75,8 @@ RUN chmod +x ~/mambaforge.sh && \ mamba init && \ rm ~/mambaforge.sh -# Install PyTorch nightly (2.2.0.dev2023) compiled against RoCm 5.7, as VLLM can not be compiled with RoCm 5.6. -RUN pip install --pre torch==2.2.0.dev20231106 --index-url https://download.pytorch.org/whl/nightly/rocm5.7 +# Install PyTorch 2.2 RC compiled against RoCm 5.7, as VLLM can not be compiled with RoCm 5.6. +RUN pip install torch --index-url https://download.pytorch.org/whl/test/rocm5.7/ FROM base AS kernel-builder @@ -104,6 +104,20 @@ WORKDIR /usr/src COPY server/custom_kernels/ . RUN PYTORCH_ROCM_ARCH=gfx90a python setup.py build +# Build exllama kernels +FROM kernel-builder as exllama-kernels-builder +WORKDIR /usr/src +COPY server/exllama_kernels/ . + +RUN PYTORCH_ROCM_ARCH="gfx90a" python setup.py build + +# Build exllama v2 kernels +FROM kernel-builder as exllamav2-kernels-builder +WORKDIR /usr/src +COPY server/exllamav2_kernels/ . + +RUN PYTORCH_ROCM_ARCH="gfx90a" python setup.py build + FROM base as base-copy # Text Generation Inference base env @@ -120,6 +134,12 @@ COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86 # Copy build artifacts from custom kernels builder COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages +# Copy build artifacts from exllama kernels builder +COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages + +# Copy build artifacts from exllamav2 kernels builder +COPY --from=exllamav2-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages + # Install flash-attention dependencies RUN pip install einops --no-cache-dir diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md index 004790ab..df5102c2 100644 --- a/docs/source/supported_models.md +++ b/docs/source/supported_models.md @@ -43,8 +43,8 @@ text-generation-launcher --model-id TGI optimized models are supported on NVIDIA [A100](https://www.nvidia.com/en-us/data-center/a100/), [A10G](https://www.nvidia.com/en-us/data-center/products/a10-gpu/) and [T4](https://www.nvidia.com/en-us/data-center/tesla-t4/) GPUs with CUDA 12.2+. Note that you have to install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) to use it. For other NVIDIA GPUs, continuous batching will still apply, but some operations like flash attention and paged attention will not be executed. -TGI also has support of ROCm-enabled AMD Instinct MI210 and MI250 GPUs, with paged attention and flash attention v2 support. The following features are currently not supported in the ROCm version of TGI, and the supported may be extended in the future: -* Quantization (GPTQ, AWQ, etc.) +TGI also has support of ROCm-enabled AMD Instinct MI210 and MI250 GPUs, with paged attention, GPTQ quantization, flash attention v2 support. The following features are currently not supported in the ROCm version of TGI, and the supported may be extended in the future: +* Loading [AWQ](https://huggingface.co/docs/transformers/quantization#awq) checkpoints. * Flash [layer norm kernel](https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm) * Kernel for slinding window attention (Mistral) diff --git a/server/exllama_kernels/exllama_kernels/cuda_compat.cuh b/server/exllama_kernels/exllama_kernels/cu_compat.cuh similarity index 91% rename from server/exllama_kernels/exllama_kernels/cuda_compat.cuh rename to server/exllama_kernels/exllama_kernels/cu_compat.cuh index 8dfa25de..c5258813 100644 --- a/server/exllama_kernels/exllama_kernels/cuda_compat.cuh +++ b/server/exllama_kernels/exllama_kernels/cu_compat.cuh @@ -43,12 +43,12 @@ __device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val) // -#if defined(__CUDA_ARCH__) -#if __CUDA_ARCH__ < 700 +#if defined(__CUDA_ARCH__) || defined(USE_ROCM) +#if __CUDA_ARCH__ < 700 || defined(USE_ROCM) __device__ __forceinline__ void atomicAdd(half* address, half val) { atomicAdd_half(address, val); } -#if __CUDA_ARCH__ < 600 +#if __CUDA_ARCH__ < 600 || defined(USE_ROCM) __device__ __forceinline__ void atomicAdd(half2* address, half2 val) { atomicAdd_half2(address, val); } #endif diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu index 60dc4c9d..61380f42 100644 --- a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu @@ -2,8 +2,11 @@ #include "column_remap.cuh" #include "../util.cuh" #include "../matrix.cuh" -#include "../cuda_compat.cuh" +#include "../cu_compat.cuh" #include "../cuda_buffers.cuh" +#if defined(USE_ROCM) +#include "../hip_compat.cuh" +#endif const int THREADS_X = 32; // Block size and thread count along columns in w and out const int THREADS_Y = 1; // Block size and thread count along rows in x and out @@ -128,7 +131,7 @@ __global__ void q4_matmul_kernel if constexpr (use_half2) { - half result = __hadd(acc.x, acc.y); + half result = __hadd(__low2half(acc), __high2half(acc)); atomicAdd(out_.item_ptr(x_row, w_column), result); } else diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/compat_gemm.cuh b/server/exllama_kernels/exllama_kernels/hip_compat.cuh similarity index 68% rename from server/exllamav2_kernels/exllamav2_kernels/cuda/compat_gemm.cuh rename to server/exllama_kernels/exllama_kernels/hip_compat.cuh index 19b1e4a6..4f2a7ae7 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/compat_gemm.cuh +++ b/server/exllama_kernels/exllama_kernels/hip_compat.cuh @@ -1,12 +1,23 @@ -#ifndef _compat_gemm_cuh -#define _compat_gemm_cuh +// Adapted from turboderp exllama: https://github.com/turboderp/exllama -#if defined(USE_ROCM) +#ifndef _hip_compat_cuh +#define _hip_compat_cuh -// For some reason this include is not present anywhere in exllama_v2 codebase, but it is required -// for symbols as hipblasHalf. -#include +// Workaround for a bug in hipamd, backported from upstream, this is fixed in ROCm 5.6. +__device__ __forceinline__ __half __compat_hrcp(__half x) { + return __half_raw{ + static_cast<_Float16>(__builtin_amdgcn_rcph(static_cast<__half_raw>(x).data))}; +} + +__device__ __forceinline__ __half2 __compat_h2rcp(__half2 x) { + return _Float16_2{static_cast<_Float16>(__builtin_amdgcn_rcph(x.x)), + static_cast<_Float16>(__builtin_amdgcn_rcph(x.y))}; +} + +#define hrcp __compat_hrcp +#define h2rcp __compat_h2rcp +// Automatic conversion of hipblasHgemm doesn't convert half to hipblasHalf. __host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle, hipblasOperation_t transA, hipblasOperation_t transB, @@ -31,8 +42,10 @@ __host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t #define hipblasHgemm __compat_hipblasHgemm // Previous version of PyTorch were converting to rocBLAS instead of hipBLAS. +#define rocblas_handle hipblasHandle_t #define rocblas_operation_none HIPBLAS_OP_N +#define rocblas_get_stream hipblasGetStream +#define rocblas_set_stream hipblasSetStream #define rocblas_hgemm __compat_hipblasHgemm -#endif -#endif +#endif \ No newline at end of file diff --git a/server/exllama_kernels/exllama_kernels/util.cuh b/server/exllama_kernels/exllama_kernels/util.cuh index 2839b10f..7b397573 100644 --- a/server/exllama_kernels/exllama_kernels/util.cuh +++ b/server/exllama_kernels/exllama_kernels/util.cuh @@ -8,7 +8,11 @@ #include #include +#if defined(USE_ROCM) +#define cudaUnspecified hipErrorUnknown +#else #define cudaUnspecified cudaErrorApiFailureBase +#endif // React to failure on return code != cudaSuccess diff --git a/server/exllamav2_kernels/setup.py b/server/exllamav2_kernels/setup.py index 518db1df..4a16b546 100644 --- a/server/exllamav2_kernels/setup.py +++ b/server/exllamav2_kernels/setup.py @@ -1,5 +1,15 @@ from setuptools import setup from torch.utils.cpp_extension import BuildExtension, CUDAExtension +import torch + +extra_cuda_cflags = ["-lineinfo", "-O3"] + +if torch.version.hip: + extra_cuda_cflags += ["-DHIPBLAS_USE_HIP_HALF"] + +extra_compile_args = { + "nvcc": extra_cuda_cflags, +} setup( name="exllamav2_kernels", @@ -11,6 +21,7 @@ "exllamav2_kernels/cuda/q_matrix.cu", "exllamav2_kernels/cuda/q_gemm.cu", ], + extra_compile_args=extra_compile_args, ) ], cmdclass={"build_ext": BuildExtension}, diff --git a/server/text_generation_server/utils/gptq/exllamav2.py b/server/text_generation_server/utils/gptq/exllamav2.py index 2b897f25..80836a95 100644 --- a/server/text_generation_server/utils/gptq/exllamav2.py +++ b/server/text_generation_server/utils/gptq/exllamav2.py @@ -1,12 +1,9 @@ # Adapted from turboderp exllama: https://github.com/turboderp/exllamav2 -from logging import getLogger - import torch import torch.nn as nn -import math -logger = getLogger(__name__) +from loguru import logger try: from exllamav2_kernels import make_q_matrix, gemm_half_q_half diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index c9393d99..6ddfd6f4 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -33,7 +33,7 @@ major = 1 HAS_EXLLAMA = False -CAN_EXLLAMA = major >= 8 +CAN_EXLLAMA = major >= 8 or IS_ROCM_SYSTEM V2 = os.getenv("EXLLAMA_VERSION", "2") == "2" # if V2 and int(os.getenv("WORLD_SIZE", "1")) > 1: # V2 = False From d9758851be6279444a81901b54e7f55af771f6ad Mon Sep 17 00:00:00 2001 From: drbh Date: Fri, 26 Jan 2024 12:01:33 -0500 Subject: [PATCH 539/793] feat: add tokenizer-config-path to launcher args (#1495) This PR adds the `tokenizer-config-path` to the launcher and passes it to the router Fixes: https://github.com/huggingface/text-generation-inference/pull/1427 --- docs/source/basic_tutorials/launcher.md | 8 ++++++++ launcher/src/main.rs | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index bafe3669..712b4fc4 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -354,6 +354,14 @@ Options: [env: NGROK_EDGE=] +``` +## TOKENIZER_CONFIG_PATH +```shell + --tokenizer-config-path + The path to the tokenizer config file. This path is used to load the tokenizer configuration which may include a `chat_template`. If not provided, the default config will be used from the model hub + + [env: TOKENIZER_CONFIG_PATH=] + ``` ## ENV ```shell diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 09657c91..f0e45141 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -368,6 +368,11 @@ struct Args { #[clap(long, env)] ngrok_edge: Option, + /// The path to the tokenizer config file. This path is used to load the tokenizer configuration which may + /// include a `chat_template`. If not provided, the default config will be used from the model hub. + #[clap(long, env)] + tokenizer_config_path: Option, + /// Display a lot of information about your runtime environment #[clap(long, short, action)] env: bool, @@ -1016,6 +1021,12 @@ fn spawn_webserver( args.model_id, ]; + // Tokenizer config path + if let Some(ref tokenizer_config_path) = args.tokenizer_config_path { + router_args.push("--tokenizer-config-path".to_string()); + router_args.push(tokenizer_config_path.to_string()); + } + // Model optional max batch total tokens if let Some(max_batch_total_tokens) = args.max_batch_total_tokens { router_args.push("--max-batch-total-tokens".to_string()); From c2d4a3b5c7bb6a8367c00f7c797bf87f4b2fcef9 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 26 Jan 2024 19:04:57 +0100 Subject: [PATCH 540/793] v1.4.0 (#1494) --- .github/workflows/delete_doc_comment.yml | 12 - Cargo.lock | 420 +++--- Cargo.toml | 2 +- README.md | 6 +- docs/openapi.json | 1294 ++++++++++++++++- .../basic_tutorials/gated_model_access.md | 2 +- docs/source/quicktour.md | 6 +- integration-tests/models/test_flash_phi.py | 8 +- integration-tests/pyproject.toml | 2 +- server/poetry.lock | 1193 ++++++++------- server/pyproject.toml | 2 +- server/requirements_cuda.txt | 18 +- server/requirements_rocm.txt | 18 +- server/tests/utils/test_layers.py | 37 +- server/text_generation_server/cli.py | 2 +- .../text_generation_server/models/__init__.py | 6 +- .../custom_modeling/flash_phi_modeling.py | 30 +- .../models/custom_modeling/mpt_modeling.py | 27 +- .../models/custom_modeling/phi_modeling.py | 62 +- .../models/flash_llama.py | 12 +- .../models/flash_phi.py | 12 +- server/text_generation_server/models/phi.py | 7 +- server/text_generation_server/utils/layers.py | 4 +- 23 files changed, 2333 insertions(+), 849 deletions(-) delete mode 100644 .github/workflows/delete_doc_comment.yml diff --git a/.github/workflows/delete_doc_comment.yml b/.github/workflows/delete_doc_comment.yml deleted file mode 100644 index 1cad807b..00000000 --- a/.github/workflows/delete_doc_comment.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Delete doc comment - -on: - pull_request: - types: [ closed ] - - -jobs: - delete: - uses: huggingface/doc-builder/.github/workflows/delete_doc_comment_trigger.yml@main - with: - pr_number: ${{ github.event.number }} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 3baff665..7fdf301a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "once_cell", @@ -40,9 +40,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.76" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59d2a3357dde987206219e78ecfbbb6e8dad06cbb65292758d3270e6254f7355" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "arc-swap" @@ -128,18 +128,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -261,9 +261,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bitflags" @@ -273,9 +273,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "block-buffer" @@ -310,6 +310,38 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cassowary" version = "0.3.0" @@ -333,9 +365,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.11" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -343,9 +375,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -362,7 +394,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -379,15 +411,15 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", "unicode-width", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -408,9 +440,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -426,45 +458,37 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.16" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", ] [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crossterm" @@ -472,7 +496,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "crossterm_winapi", "libc", "mio", @@ -503,12 +527,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e95fbd621905b854affdc67943b043a0fbb6ed7385fd5a25650d19a8a6cfdf" +checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" dependencies = [ "nix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -548,9 +572,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", ] @@ -758,9 +782,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -789,9 +813,9 @@ checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -812,7 +836,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -866,9 +890,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -893,9 +917,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -939,9 +963,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hf-hub" @@ -1183,9 +1207,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1198,9 +1222,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libm" @@ -1214,16 +1238,16 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "libc", "redox_syscall", ] [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -1289,18 +1313,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "metrics" @@ -1319,7 +1334,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "hyper", "indexmap 1.9.3", "ipnet", @@ -1333,13 +1348,13 @@ dependencies = [ [[package]] name = "metrics-macros" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" +checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1375,9 +1390,9 @@ dependencies = [ [[package]] name = "minijinja" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "208758577ef2c86cf5dd3e85730d161413ec3284e2d73b2ef65d9a24d9971bcb" +checksum = "6fe0ff215195a22884d867b547c70a0c4815cbbcc70991f281dca604b20d10ce" dependencies = [ "serde", ] @@ -1411,9 +1426,9 @@ dependencies = [ [[package]] name = "monostate" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e404e13820ea0df0eda93aa294e0c80de76a0daa6bec590d376fbec6d7810394" +checksum = "878c2a1f1c70e5724fa28f101ca787b6a7e8ad5c5e4ae4ca3b0fa4a419fa9075" dependencies = [ "monostate-impl", "serde", @@ -1421,13 +1436,13 @@ dependencies = [ [[package]] name = "monostate-impl" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531c82a934da419bed3da09bd87d6e98c72f8d4aa755427b3b009c2b8b8c433c" +checksum = "f686d68a09079e63b1d2c64aa305095887ce50565f00a922ebfaeeee0d9ba6ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1510,7 +1525,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "libc", ] @@ -1587,9 +1602,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -1624,11 +1639,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.62" +version = "0.10.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "foreign-types", "libc", @@ -1645,7 +1660,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1656,9 +1671,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.98" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -1846,22 +1861,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1878,9 +1893,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "portable-atomic" @@ -1902,12 +1917,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -1936,9 +1951,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -1980,7 +1995,7 @@ dependencies = [ "prost 0.12.3", "prost-types", "regex", - "syn 2.0.42", + "syn 2.0.48", "tempfile", "which", ] @@ -2008,7 +2023,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -2038,9 +2053,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2081,7 +2096,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e2e4cd95294a85c3b4446e63ef054eea43e0205b1fd60120c16b74ff7ff96ad" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cassowary", "crossterm", "indoc", @@ -2103,9 +2118,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" dependencies = [ "either", "rayon-core", @@ -2124,9 +2139,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -2154,13 +2169,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.5", "regex-syntax 0.8.2", ] @@ -2175,9 +2190,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -2208,7 +2223,7 @@ version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -2290,7 +2305,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.42", + "syn 2.0.48", "walkdir", ] @@ -2321,11 +2336,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -2362,7 +2377,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", ] [[package]] @@ -2398,11 +2413,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2446,35 +2461,38 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +dependencies = [ + "serde", +] [[package]] name = "serde" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -2483,9 +2501,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" dependencies = [ "itoa", "serde", @@ -2588,9 +2606,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "socket2" @@ -2651,7 +2669,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -2667,9 +2685,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.42" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d0a2c048d661a1a59fcd7355baa232f7ed34e0ee4df2eef3c1c1c0d3852d8" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -2684,16 +2702,16 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.29.11" +version = "0.30.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" +checksum = "1fb4f3438c8f6389c864e61221cbc97e9bca98b4daf39a5beb7bea660f528bb2" dependencies = [ "cfg-if", "core-foundation-sys", "libc", "ntapi", "once_cell", - "winapi", + "windows", ] [[package]] @@ -2743,20 +2761,20 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "text-generation-benchmark" -version = "1.3.4" +version = "1.4.0" dependencies = [ "average", "clap", @@ -2777,7 +2795,7 @@ dependencies = [ [[package]] name = "text-generation-client" -version = "1.3.4" +version = "1.4.0" dependencies = [ "futures", "grpc-metadata", @@ -2793,7 +2811,7 @@ dependencies = [ [[package]] name = "text-generation-launcher" -version = "1.3.4" +version = "1.4.0" dependencies = [ "clap", "ctrlc", @@ -2809,7 +2827,7 @@ dependencies = [ [[package]] name = "text-generation-router" -version = "1.3.4" +version = "1.4.0" dependencies = [ "async-stream", "axum", @@ -2846,22 +2864,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -2991,7 +3009,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -3049,7 +3067,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64 0.21.5", + "base64 0.21.7", "bytes", "futures-core", "futures-util", @@ -3078,7 +3096,7 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.5", + "base64 0.21.7", "bytes", "h2", "http", @@ -3106,7 +3124,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -3135,7 +3153,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "bytes", "futures-core", "futures-util", @@ -3179,7 +3197,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -3297,9 +3315,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -3361,7 +3379,7 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8cdd25c339e200129fe4de81451814e5228c9b771d57378817d6117cc2b3f97" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "flate2", "log", "native-tls", @@ -3419,7 +3437,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] @@ -3452,11 +3470,14 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "8.2.6" +version = "8.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1290fd64cc4e7d3c9b07d7f333ce0ce0007253e32870e632624835cc80b83939" +checksum = "e27d6bdd219887a9eadd19e1c34f32e47fa332301184935c6d9bca26f3cca525" dependencies = [ "anyhow", + "cargo_metadata", + "cfg-if", + "regex", "rustc_version", "rustversion", "sysinfo", @@ -3496,9 +3517,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3506,24 +3527,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -3533,9 +3554,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3543,28 +3564,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -3629,6 +3650,25 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -3854,7 +3894,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 80e6e145..a328a368 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.3.4" +version = "1.4.0" edition = "2021" authors = ["Olivier Dehaene"] homepage = "https://github.com/huggingface/text-generation-inference" diff --git a/README.md b/README.md index 73356f28..5fdb9f14 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ For a detailed starting guide, please see the [Quick Tour](https://huggingface.c model=HuggingFaceH4/zephyr-7b-beta volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.4 --model-id $model ``` And then you can make requests like @@ -76,7 +76,7 @@ curl 127.0.0.1:8080/generate \ **Note:** To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 12.2 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar. -**Note:** TGI supports AMD Instinct MI210 and MI250 GPUs. Details can be found in the [Supported Hardware documentation](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3-rocm --model-id $model` instead of the command above. +**Note:** TGI supports AMD Instinct MI210 and MI250 GPUs. Details can be found in the [Supported Hardware documentation](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.4-rocm --model-id $model` instead of the command above. To see all options to serve your models (in the [code](https://github.com/huggingface/text-generation-inference/blob/main/launcher/src/main.rs) or in the cli): ``` @@ -106,7 +106,7 @@ model=meta-llama/Llama-2-7b-chat-hf volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run token= -docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3 --model-id $model +docker run --gpus all --shm-size 1g -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.4 --model-id $model ``` ### A note on Shared Memory (shm) diff --git a/docs/openapi.json b/docs/openapi.json index 9a9ed116..da3969df 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1 +1,1293 @@ -{"openapi":"3.0.3","info":{"title":"Text Generation Inference","description":"Text Generation Webserver","contact":{"name":"Olivier Dehaene"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0"},"version":"1.3.4"},"paths":{"/":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens if `stream == false` or a stream of token if `stream == true`","description":"Generate tokens if `stream == false` or a stream of token if `stream == true`","operationId":"compat_generate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CompatGenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateResponse"}},"text/event-stream":{"schema":{"$ref":"#/components/schemas/StreamResponse"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/generate":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens","description":"Generate tokens","operationId":"generate","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateResponse"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/generate_stream":{"post":{"tags":["Text Generation Inference"],"summary":"Generate a stream of token using Server-Sent Events","description":"Generate a stream of token using Server-Sent Events","operationId":"generate_stream","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/StreamResponse"}}}},"422":{"description":"Input validation error","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"text/event-stream":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}},"/health":{"get":{"tags":["Text Generation Inference"],"summary":"Health check method","description":"Health check method","operationId":"health","responses":{"200":{"description":"Everything is working fine"},"503":{"description":"Text generation inference is down","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"unhealthy","error_type":"healthcheck"}}}}}}},"/info":{"get":{"tags":["Text Generation Inference"],"summary":"Text Generation Inference endpoint info","description":"Text Generation Inference endpoint info","operationId":"get_model_info","responses":{"200":{"description":"Served model info","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Info"}}}}}}},"/metrics":{"get":{"tags":["Text Generation Inference"],"summary":"Prometheus metrics scrape endpoint","description":"Prometheus metrics scrape endpoint","operationId":"metrics","responses":{"200":{"description":"Prometheus Metrics","content":{"text/plain":{"schema":{"type":"string"}}}}}}},"/tokenize":{"post":{"tags":["Text Generation Inference"],"summary":"Tokenize inputs","description":"Tokenize inputs","operationId":"tokenize","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenerateRequest"}}},"required":true},"responses":{"200":{"description":"Tokenized ids","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TokenizeResponse"}}}},"404":{"description":"No tokenizer found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"No fast tokenizer available"}}}}}}},"/v1/chat/completions":{"post":{"tags":["Text Generation Inference"],"summary":"Generate tokens","description":"Generate tokens","operationId":"chat_completions","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatRequest"}}},"required":true},"responses":{"200":{"description":"Generated Text","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ChatCompletionChunk"}}}},"422":{"description":"Input validation error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Input validation error"}}}},"424":{"description":"Generation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Request failed during generation"}}}},"429":{"description":"Model is overloaded","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Model is overloaded"}}}},"500":{"description":"Incomplete generation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ErrorResponse"},"example":{"error":"Incomplete generation"}}}}}}}},"components":{"schemas":{"BestOfSequence":{"type":"object","required":["generated_text","finish_reason","generated_tokens","prefill","tokens"],"properties":{"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_text":{"type":"string","example":"test"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"prefill":{"type":"array","items":{"$ref":"#/components/schemas/PrefillToken"}},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}},"top_tokens":{"type":"array","items":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}}},"ChatCompletion":{"type":"object","required":["id","object","created","model","system_fingerprint","choices","usage"],"properties":{"choices":{"type":"array","items":{"$ref":"#/components/schemas/ChatCompletionComplete"}},"created":{"type":"integer","format":"int64","example":"1706270835","minimum":0},"id":{"type":"string"},"model":{"type":"string","example":"mistralai/Mistral-7B-Instruct-v0.2"},"object":{"type":"string"},"system_fingerprint":{"type":"string"},"usage":{"$ref":"#/components/schemas/Usage"}}},"ChatCompletionChoice":{"type":"object","required":["index","delta"],"properties":{"delta":{"$ref":"#/components/schemas/ChatCompletionDelta"},"finish_reason":{"type":"string","nullable":true},"index":{"type":"integer","format":"int32","minimum":0},"logprobs":{"type":"number","format":"float","nullable":true}}},"ChatCompletionChunk":{"type":"object","required":["id","object","created","model","system_fingerprint","choices"],"properties":{"choices":{"type":"array","items":{"$ref":"#/components/schemas/ChatCompletionChoice"}},"created":{"type":"integer","format":"int64","example":"1706270978","minimum":0},"id":{"type":"string"},"model":{"type":"string","example":"mistralai/Mistral-7B-Instruct-v0.2"},"object":{"type":"string"},"system_fingerprint":{"type":"string"}}},"ChatCompletionDelta":{"type":"object","required":["role","content"],"properties":{"content":{"type":"string","example":"What is Deep Learning?"},"role":{"type":"string","example":"user"}}},"ChatRequest":{"type":"object","required":["model"],"properties":{"frequency_penalty":{"type":"number","format":"float","description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.","example":"1.0","nullable":true},"logit_bias":{"type":"array","items":{"type":"number","format":"float"},"description":"UNUSED\nModify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens\n(specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically,\nthe bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model,\nbut values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should\nresult in a ban or exclusive selection of the relevant token.","nullable":true},"logprobs":{"type":"boolean","description":"Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each\noutput token returned in the content of message.","example":"false","nullable":true},"max_tokens":{"type":"integer","format":"int32","description":"The maximum number of tokens that can be generated in the chat completion.","example":"32","nullable":true,"minimum":0},"messages":{"type":"array","items":{"$ref":"#/components/schemas/Message"},"description":"A list of messages comprising the conversation so far."},"model":{"type":"string","description":"UNUSED\nID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.","example":"mistralai/Mistral-7B-Instruct-v0.2"},"n":{"type":"integer","format":"int32","description":"UNUSED\nHow many chat completion choices to generate for each input message. Note that you will be charged based on the\nnumber of generated tokens across all of the choices. Keep n as 1 to minimize costs.","example":"2","nullable":true,"minimum":0},"presence_penalty":{"type":"number","format":"float","description":"UNUSED\nNumber between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far,\nincreasing the model's likelihood to talk about new topics","example":0.1,"nullable":true},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"stream":{"type":"boolean"},"temperature":{"type":"number","format":"float","description":"What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while\nlower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.","example":1.0,"nullable":true},"top_logprobs":{"type":"integer","format":"int32","description":"UNUSED\nAn integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with\nan associated log probability. logprobs must be set to true if this parameter is used.","example":"5","nullable":true,"minimum":0},"top_p":{"type":"number","format":"float","description":"An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the\ntokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.","example":0.95,"nullable":true}}},"CompatGenerateRequest":{"type":"object","required":["inputs"],"properties":{"inputs":{"type":"string","example":"My name is Olivier and I"},"parameters":{"$ref":"#/components/schemas/GenerateParameters"},"stream":{"type":"boolean","default":"false"}}},"Details":{"type":"object","required":["finish_reason","generated_tokens","prefill","tokens"],"properties":{"best_of_sequences":{"type":"array","items":{"$ref":"#/components/schemas/BestOfSequence"},"nullable":true},"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"prefill":{"type":"array","items":{"$ref":"#/components/schemas/PrefillToken"}},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0},"tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}},"top_tokens":{"type":"array","items":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}}},"ErrorResponse":{"type":"object","required":["error","error_type"],"properties":{"error":{"type":"string"},"error_type":{"type":"string"}}},"FinishReason":{"type":"string","enum":["length","eos_token","stop_sequence"],"example":"Length"},"GenerateParameters":{"type":"object","properties":{"best_of":{"type":"integer","default":"null","example":1,"nullable":true,"minimum":0,"exclusiveMinimum":0},"decoder_input_details":{"type":"boolean","default":"true"},"details":{"type":"boolean","default":"true"},"do_sample":{"type":"boolean","default":"false","example":true},"max_new_tokens":{"type":"integer","format":"int32","default":"100","example":"20","nullable":true,"minimum":0},"repetition_penalty":{"type":"number","format":"float","default":"null","example":1.03,"nullable":true,"exclusiveMinimum":0},"return_full_text":{"type":"boolean","default":"null","example":false,"nullable":true},"seed":{"type":"integer","format":"int64","default":"null","example":"null","nullable":true,"minimum":0,"exclusiveMinimum":0},"stop":{"type":"array","items":{"type":"string"},"example":["photographer"],"maxItems":4},"temperature":{"type":"number","format":"float","default":"null","example":0.5,"nullable":true,"exclusiveMinimum":0},"top_k":{"type":"integer","format":"int32","default":"null","example":10,"nullable":true,"exclusiveMinimum":0},"top_n_tokens":{"type":"integer","format":"int32","default":"null","example":5,"nullable":true,"minimum":0,"exclusiveMinimum":0},"top_p":{"type":"number","format":"float","default":"null","example":0.95,"nullable":true,"maximum":1,"exclusiveMinimum":0},"truncate":{"type":"integer","default":"null","example":"null","nullable":true,"minimum":0},"typical_p":{"type":"number","format":"float","default":"null","example":0.95,"nullable":true,"maximum":1,"exclusiveMinimum":0},"watermark":{"type":"boolean","default":"false","example":true}}},"GenerateRequest":{"type":"object","required":["inputs"],"properties":{"inputs":{"type":"string","example":"My name is Olivier and I"},"parameters":{"$ref":"#/components/schemas/GenerateParameters"}}},"GenerateResponse":{"type":"object","required":["generated_text"],"properties":{"details":{"allOf":[{"$ref":"#/components/schemas/Details"}],"nullable":true},"generated_text":{"type":"string","example":"test"}}},"Info":{"type":"object","required":["model_id","model_dtype","model_device_type","max_concurrent_requests","max_best_of","max_stop_sequences","max_input_length","max_total_tokens","waiting_served_ratio","max_batch_total_tokens","max_waiting_tokens","validation_workers","version"],"properties":{"docker_label":{"type":"string","example":"null","nullable":true},"max_batch_total_tokens":{"type":"integer","format":"int32","example":"32000","minimum":0},"max_best_of":{"type":"integer","example":"2","minimum":0},"max_concurrent_requests":{"type":"integer","description":"Router Parameters","example":"128","minimum":0},"max_input_length":{"type":"integer","example":"1024","minimum":0},"max_stop_sequences":{"type":"integer","example":"4","minimum":0},"max_total_tokens":{"type":"integer","example":"2048","minimum":0},"max_waiting_tokens":{"type":"integer","example":"20","minimum":0},"model_device_type":{"type":"string","example":"cuda"},"model_dtype":{"type":"string","example":"torch.float16"},"model_id":{"type":"string","description":"Model info","example":"bigscience/blomm-560m"},"model_pipeline_tag":{"type":"string","example":"text-generation","nullable":true},"model_sha":{"type":"string","example":"e985a63cdc139290c5f700ff1929f0b5942cced2","nullable":true},"sha":{"type":"string","example":"null","nullable":true},"validation_workers":{"type":"integer","example":"2","minimum":0},"version":{"type":"string","description":"Router Info","example":"0.5.0"},"waiting_served_ratio":{"type":"number","format":"float","example":"1.2"}}},"Message":{"type":"object","required":["role","content"],"properties":{"content":{"type":"string","example":"My name is David and I"},"role":{"type":"string","example":"user"}}},"PrefillToken":{"type":"object","required":["id","text","logprob"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"logprob":{"type":"number","format":"float","example":-0.34,"nullable":true},"text":{"type":"string","example":"test"}}},"SimpleToken":{"type":"object","required":["id","text","start","stop"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"start":{"type":"integer","example":0,"minimum":0},"stop":{"type":"integer","example":2,"minimum":0},"text":{"type":"string","example":"test"}}},"StreamDetails":{"type":"object","required":["finish_reason","generated_tokens"],"properties":{"finish_reason":{"$ref":"#/components/schemas/FinishReason"},"generated_tokens":{"type":"integer","format":"int32","example":1,"minimum":0},"seed":{"type":"integer","format":"int64","example":42,"nullable":true,"minimum":0}}},"StreamResponse":{"type":"object","required":["index","token"],"properties":{"details":{"allOf":[{"$ref":"#/components/schemas/StreamDetails"}],"default":"null","nullable":true},"generated_text":{"type":"string","default":"null","example":"test","nullable":true},"index":{"type":"integer","format":"int32","minimum":0},"token":{"$ref":"#/components/schemas/Token"},"top_tokens":{"type":"array","items":{"$ref":"#/components/schemas/Token"}}}},"Token":{"type":"object","required":["id","text","logprob","special"],"properties":{"id":{"type":"integer","format":"int32","example":0,"minimum":0},"logprob":{"type":"number","format":"float","example":-0.34,"nullable":true},"special":{"type":"boolean","example":"false"},"text":{"type":"string","example":"test"}}},"TokenizeResponse":{"type":"array","items":{"$ref":"#/components/schemas/SimpleToken"}}}},"tags":[{"name":"Text Generation Inference","description":"Hugging Face Text Generation Inference API"}]} \ No newline at end of file +{ + "openapi": "3.0.3", + "info": { + "title": "Text Generation Inference", + "description": "Text Generation Webserver", + "contact": { + "name": "Olivier Dehaene" + }, + "license": { + "name": "Apache 2.0", + "url": "https://www.apache.org/licenses/LICENSE-2.0" + }, + "version": "1.4.0" + }, + "paths": { + "/": { + "post": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Generate tokens if `stream == false` or a stream of token if `stream == true`", + "description": "Generate tokens if `stream == false` or a stream of token if `stream == true`", + "operationId": "compat_generate", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CompatGenerateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Generated Text", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenerateResponse" + } + }, + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/StreamResponse" + } + } + } + }, + "422": { + "description": "Input validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Input validation error" + } + } + } + }, + "424": { + "description": "Generation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Request failed during generation" + } + } + } + }, + "429": { + "description": "Model is overloaded", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Model is overloaded" + } + } + } + }, + "500": { + "description": "Incomplete generation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Incomplete generation" + } + } + } + } + } + } + }, + "/generate": { + "post": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Generate tokens", + "description": "Generate tokens", + "operationId": "generate", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenerateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Generated Text", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenerateResponse" + } + } + } + }, + "422": { + "description": "Input validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Input validation error" + } + } + } + }, + "424": { + "description": "Generation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Request failed during generation" + } + } + } + }, + "429": { + "description": "Model is overloaded", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Model is overloaded" + } + } + } + }, + "500": { + "description": "Incomplete generation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Incomplete generation" + } + } + } + } + } + } + }, + "/generate_stream": { + "post": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Generate a stream of token using Server-Sent Events", + "description": "Generate a stream of token using Server-Sent Events", + "operationId": "generate_stream", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenerateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Generated Text", + "content": { + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/StreamResponse" + } + } + } + }, + "422": { + "description": "Input validation error", + "content": { + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Input validation error" + } + } + } + }, + "424": { + "description": "Generation Error", + "content": { + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Request failed during generation" + } + } + } + }, + "429": { + "description": "Model is overloaded", + "content": { + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Model is overloaded" + } + } + } + }, + "500": { + "description": "Incomplete generation", + "content": { + "text/event-stream": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Incomplete generation" + } + } + } + } + } + } + }, + "/health": { + "get": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Health check method", + "description": "Health check method", + "operationId": "health", + "responses": { + "200": { + "description": "Everything is working fine" + }, + "503": { + "description": "Text generation inference is down", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "unhealthy", + "error_type": "healthcheck" + } + } + } + } + } + } + }, + "/info": { + "get": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Text Generation Inference endpoint info", + "description": "Text Generation Inference endpoint info", + "operationId": "get_model_info", + "responses": { + "200": { + "description": "Served model info", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Info" + } + } + } + } + } + } + }, + "/metrics": { + "get": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Prometheus metrics scrape endpoint", + "description": "Prometheus metrics scrape endpoint", + "operationId": "metrics", + "responses": { + "200": { + "description": "Prometheus Metrics", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/tokenize": { + "post": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Tokenize inputs", + "description": "Tokenize inputs", + "operationId": "tokenize", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GenerateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Tokenized ids", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TokenizeResponse" + } + } + } + }, + "404": { + "description": "No tokenizer found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "No fast tokenizer available" + } + } + } + } + } + } + }, + "/v1/chat/completions": { + "post": { + "tags": [ + "Text Generation Inference" + ], + "summary": "Generate tokens", + "description": "Generate tokens", + "operationId": "chat_completions", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Generated Text", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatCompletionChunk" + } + } + } + }, + "422": { + "description": "Input validation error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Input validation error" + } + } + } + }, + "424": { + "description": "Generation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Request failed during generation" + } + } + } + }, + "429": { + "description": "Model is overloaded", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Model is overloaded" + } + } + } + }, + "500": { + "description": "Incomplete generation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + }, + "example": { + "error": "Incomplete generation" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "BestOfSequence": { + "type": "object", + "required": [ + "generated_text", + "finish_reason", + "generated_tokens", + "prefill", + "tokens" + ], + "properties": { + "finish_reason": { + "$ref": "#/components/schemas/FinishReason" + }, + "generated_text": { + "type": "string", + "example": "test" + }, + "generated_tokens": { + "type": "integer", + "format": "int32", + "example": 1, + "minimum": 0 + }, + "prefill": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PrefillToken" + } + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42, + "nullable": true, + "minimum": 0 + }, + "tokens": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + }, + "top_tokens": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + } + } + } + }, + "ChatCompletion": { + "type": "object", + "required": [ + "id", + "object", + "created", + "model", + "system_fingerprint", + "choices", + "usage" + ], + "properties": { + "choices": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ChatCompletionComplete" + } + }, + "created": { + "type": "integer", + "format": "int64", + "example": "1706270835", + "minimum": 0 + }, + "id": { + "type": "string" + }, + "model": { + "type": "string", + "example": "mistralai/Mistral-7B-Instruct-v0.2" + }, + "object": { + "type": "string" + }, + "system_fingerprint": { + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/Usage" + } + } + }, + "ChatCompletionChoice": { + "type": "object", + "required": [ + "index", + "delta" + ], + "properties": { + "delta": { + "$ref": "#/components/schemas/ChatCompletionDelta" + }, + "finish_reason": { + "type": "string", + "nullable": true + }, + "index": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "logprobs": { + "type": "number", + "format": "float", + "nullable": true + } + } + }, + "ChatCompletionChunk": { + "type": "object", + "required": [ + "id", + "object", + "created", + "model", + "system_fingerprint", + "choices" + ], + "properties": { + "choices": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ChatCompletionChoice" + } + }, + "created": { + "type": "integer", + "format": "int64", + "example": "1706270978", + "minimum": 0 + }, + "id": { + "type": "string" + }, + "model": { + "type": "string", + "example": "mistralai/Mistral-7B-Instruct-v0.2" + }, + "object": { + "type": "string" + }, + "system_fingerprint": { + "type": "string" + } + } + }, + "ChatCompletionDelta": { + "type": "object", + "required": [ + "role", + "content" + ], + "properties": { + "content": { + "type": "string", + "example": "What is Deep Learning?" + }, + "role": { + "type": "string", + "example": "user" + } + } + }, + "ChatRequest": { + "type": "object", + "required": [ + "model" + ], + "properties": { + "frequency_penalty": { + "type": "number", + "format": "float", + "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.", + "example": "1.0", + "nullable": true + }, + "logit_bias": { + "type": "array", + "items": { + "type": "number", + "format": "float" + }, + "description": "UNUSED\nModify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens\n(specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically,\nthe bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model,\nbut values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should\nresult in a ban or exclusive selection of the relevant token.", + "nullable": true + }, + "logprobs": { + "type": "boolean", + "description": "Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each\noutput token returned in the content of message.", + "example": "false", + "nullable": true + }, + "max_tokens": { + "type": "integer", + "format": "int32", + "description": "The maximum number of tokens that can be generated in the chat completion.", + "example": "32", + "nullable": true, + "minimum": 0 + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Message" + }, + "description": "A list of messages comprising the conversation so far." + }, + "model": { + "type": "string", + "description": "UNUSED\nID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.", + "example": "mistralai/Mistral-7B-Instruct-v0.2" + }, + "n": { + "type": "integer", + "format": "int32", + "description": "UNUSED\nHow many chat completion choices to generate for each input message. Note that you will be charged based on the\nnumber of generated tokens across all of the choices. Keep n as 1 to minimize costs.", + "example": "2", + "nullable": true, + "minimum": 0 + }, + "presence_penalty": { + "type": "number", + "format": "float", + "description": "UNUSED\nNumber between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far,\nincreasing the model's likelihood to talk about new topics", + "example": 0.1, + "nullable": true + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42, + "nullable": true, + "minimum": 0 + }, + "stream": { + "type": "boolean" + }, + "temperature": { + "type": "number", + "format": "float", + "description": "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while\nlower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.", + "example": 1.0, + "nullable": true + }, + "top_logprobs": { + "type": "integer", + "format": "int32", + "description": "UNUSED\nAn integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with\nan associated log probability. logprobs must be set to true if this parameter is used.", + "example": "5", + "nullable": true, + "minimum": 0 + }, + "top_p": { + "type": "number", + "format": "float", + "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the\ntokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.", + "example": 0.95, + "nullable": true + } + } + }, + "CompatGenerateRequest": { + "type": "object", + "required": [ + "inputs" + ], + "properties": { + "inputs": { + "type": "string", + "example": "My name is Olivier and I" + }, + "parameters": { + "$ref": "#/components/schemas/GenerateParameters" + }, + "stream": { + "type": "boolean", + "default": "false" + } + } + }, + "Details": { + "type": "object", + "required": [ + "finish_reason", + "generated_tokens", + "prefill", + "tokens" + ], + "properties": { + "best_of_sequences": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BestOfSequence" + }, + "nullable": true + }, + "finish_reason": { + "$ref": "#/components/schemas/FinishReason" + }, + "generated_tokens": { + "type": "integer", + "format": "int32", + "example": 1, + "minimum": 0 + }, + "prefill": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PrefillToken" + } + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42, + "nullable": true, + "minimum": 0 + }, + "tokens": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + }, + "top_tokens": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + } + } + } + }, + "ErrorResponse": { + "type": "object", + "required": [ + "error", + "error_type" + ], + "properties": { + "error": { + "type": "string" + }, + "error_type": { + "type": "string" + } + } + }, + "FinishReason": { + "type": "string", + "enum": [ + "length", + "eos_token", + "stop_sequence" + ], + "example": "Length" + }, + "GenerateParameters": { + "type": "object", + "properties": { + "best_of": { + "type": "integer", + "default": "null", + "example": 1, + "nullable": true, + "minimum": 0, + "exclusiveMinimum": 0 + }, + "decoder_input_details": { + "type": "boolean", + "default": "true" + }, + "details": { + "type": "boolean", + "default": "true" + }, + "do_sample": { + "type": "boolean", + "default": "false", + "example": true + }, + "max_new_tokens": { + "type": "integer", + "format": "int32", + "default": "100", + "example": "20", + "nullable": true, + "minimum": 0 + }, + "repetition_penalty": { + "type": "number", + "format": "float", + "default": "null", + "example": 1.03, + "nullable": true, + "exclusiveMinimum": 0 + }, + "return_full_text": { + "type": "boolean", + "default": "null", + "example": false, + "nullable": true + }, + "seed": { + "type": "integer", + "format": "int64", + "default": "null", + "example": "null", + "nullable": true, + "minimum": 0, + "exclusiveMinimum": 0 + }, + "stop": { + "type": "array", + "items": { + "type": "string" + }, + "example": [ + "photographer" + ], + "maxItems": 4 + }, + "temperature": { + "type": "number", + "format": "float", + "default": "null", + "example": 0.5, + "nullable": true, + "exclusiveMinimum": 0 + }, + "top_k": { + "type": "integer", + "format": "int32", + "default": "null", + "example": 10, + "nullable": true, + "exclusiveMinimum": 0 + }, + "top_n_tokens": { + "type": "integer", + "format": "int32", + "default": "null", + "example": 5, + "nullable": true, + "minimum": 0, + "exclusiveMinimum": 0 + }, + "top_p": { + "type": "number", + "format": "float", + "default": "null", + "example": 0.95, + "nullable": true, + "maximum": 1, + "exclusiveMinimum": 0 + }, + "truncate": { + "type": "integer", + "default": "null", + "example": "null", + "nullable": true, + "minimum": 0 + }, + "typical_p": { + "type": "number", + "format": "float", + "default": "null", + "example": 0.95, + "nullable": true, + "maximum": 1, + "exclusiveMinimum": 0 + }, + "watermark": { + "type": "boolean", + "default": "false", + "example": true + } + } + }, + "GenerateRequest": { + "type": "object", + "required": [ + "inputs" + ], + "properties": { + "inputs": { + "type": "string", + "example": "My name is Olivier and I" + }, + "parameters": { + "$ref": "#/components/schemas/GenerateParameters" + } + } + }, + "GenerateResponse": { + "type": "object", + "required": [ + "generated_text" + ], + "properties": { + "details": { + "allOf": [ + { + "$ref": "#/components/schemas/Details" + } + ], + "nullable": true + }, + "generated_text": { + "type": "string", + "example": "test" + } + } + }, + "Info": { + "type": "object", + "required": [ + "model_id", + "model_dtype", + "model_device_type", + "max_concurrent_requests", + "max_best_of", + "max_stop_sequences", + "max_input_length", + "max_total_tokens", + "waiting_served_ratio", + "max_batch_total_tokens", + "max_waiting_tokens", + "validation_workers", + "version" + ], + "properties": { + "docker_label": { + "type": "string", + "example": "null", + "nullable": true + }, + "max_batch_total_tokens": { + "type": "integer", + "format": "int32", + "example": "32000", + "minimum": 0 + }, + "max_best_of": { + "type": "integer", + "example": "2", + "minimum": 0 + }, + "max_concurrent_requests": { + "type": "integer", + "description": "Router Parameters", + "example": "128", + "minimum": 0 + }, + "max_input_length": { + "type": "integer", + "example": "1024", + "minimum": 0 + }, + "max_stop_sequences": { + "type": "integer", + "example": "4", + "minimum": 0 + }, + "max_total_tokens": { + "type": "integer", + "example": "2048", + "minimum": 0 + }, + "max_waiting_tokens": { + "type": "integer", + "example": "20", + "minimum": 0 + }, + "model_device_type": { + "type": "string", + "example": "cuda" + }, + "model_dtype": { + "type": "string", + "example": "torch.float16" + }, + "model_id": { + "type": "string", + "description": "Model info", + "example": "bigscience/blomm-560m" + }, + "model_pipeline_tag": { + "type": "string", + "example": "text-generation", + "nullable": true + }, + "model_sha": { + "type": "string", + "example": "e985a63cdc139290c5f700ff1929f0b5942cced2", + "nullable": true + }, + "sha": { + "type": "string", + "example": "null", + "nullable": true + }, + "validation_workers": { + "type": "integer", + "example": "2", + "minimum": 0 + }, + "version": { + "type": "string", + "description": "Router Info", + "example": "0.5.0" + }, + "waiting_served_ratio": { + "type": "number", + "format": "float", + "example": "1.2" + } + } + }, + "Message": { + "type": "object", + "required": [ + "role", + "content" + ], + "properties": { + "content": { + "type": "string", + "example": "My name is David and I" + }, + "role": { + "type": "string", + "example": "user" + } + } + }, + "PrefillToken": { + "type": "object", + "required": [ + "id", + "text", + "logprob" + ], + "properties": { + "id": { + "type": "integer", + "format": "int32", + "example": 0, + "minimum": 0 + }, + "logprob": { + "type": "number", + "format": "float", + "example": -0.34, + "nullable": true + }, + "text": { + "type": "string", + "example": "test" + } + } + }, + "SimpleToken": { + "type": "object", + "required": [ + "id", + "text", + "start", + "stop" + ], + "properties": { + "id": { + "type": "integer", + "format": "int32", + "example": 0, + "minimum": 0 + }, + "start": { + "type": "integer", + "example": 0, + "minimum": 0 + }, + "stop": { + "type": "integer", + "example": 2, + "minimum": 0 + }, + "text": { + "type": "string", + "example": "test" + } + } + }, + "StreamDetails": { + "type": "object", + "required": [ + "finish_reason", + "generated_tokens" + ], + "properties": { + "finish_reason": { + "$ref": "#/components/schemas/FinishReason" + }, + "generated_tokens": { + "type": "integer", + "format": "int32", + "example": 1, + "minimum": 0 + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42, + "nullable": true, + "minimum": 0 + } + } + }, + "StreamResponse": { + "type": "object", + "required": [ + "index", + "token" + ], + "properties": { + "details": { + "allOf": [ + { + "$ref": "#/components/schemas/StreamDetails" + } + ], + "default": "null", + "nullable": true + }, + "generated_text": { + "type": "string", + "default": "null", + "example": "test", + "nullable": true + }, + "index": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "token": { + "$ref": "#/components/schemas/Token" + }, + "top_tokens": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Token" + } + } + } + }, + "Token": { + "type": "object", + "required": [ + "id", + "text", + "logprob", + "special" + ], + "properties": { + "id": { + "type": "integer", + "format": "int32", + "example": 0, + "minimum": 0 + }, + "logprob": { + "type": "number", + "format": "float", + "example": -0.34, + "nullable": true + }, + "special": { + "type": "boolean", + "example": "false" + }, + "text": { + "type": "string", + "example": "test" + } + } + }, + "TokenizeResponse": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SimpleToken" + } + } + } + }, + "tags": [ + { + "name": "Text Generation Inference", + "description": "Hugging Face Text Generation Inference API" + } + ] +} \ No newline at end of file diff --git a/docs/source/basic_tutorials/gated_model_access.md b/docs/source/basic_tutorials/gated_model_access.md index 1437717f..060d177d 100644 --- a/docs/source/basic_tutorials/gated_model_access.md +++ b/docs/source/basic_tutorials/gated_model_access.md @@ -19,6 +19,6 @@ docker run --gpus all \ --shm-size 1g \ -e HUGGING_FACE_HUB_TOKEN=$token \ -p 8080:80 \ - -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3 \ + -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.4 \ --model-id $model ``` diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md index e9a33f04..78ebb8e2 100644 --- a/docs/source/quicktour.md +++ b/docs/source/quicktour.md @@ -8,7 +8,7 @@ Let's say you want to deploy [Falcon-7B Instruct](https://huggingface.co/tiiuae/ model=tiiuae/falcon-7b-instruct volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run -docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3 --model-id $model +docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.4 --model-id $model ``` @@ -20,7 +20,7 @@ To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://d TGI also supports ROCm-enabled AMD GPUs (only MI210 and MI250 are tested), details are available in the [Supported Hardware section](./supported_models#supported-hardware) and [AMD documentation](https://rocm.docs.amd.com/en/latest/deploy/docker.html). To launch TGI on ROCm GPUs, please use instead: ```bash -docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --device=/dev/kfd --device=/dev/dri --group-add video --ipc=host --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.3-rocm --model-id $model +docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --device=/dev/kfd --device=/dev/dri --group-add video --ipc=host --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:1.4-rocm --model-id $model ``` Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section, where we show examples with utility libraries and UIs. Below you can see a simple snippet to query the endpoint. @@ -91,7 +91,7 @@ curl 127.0.0.1:8080/generate \ To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more. ```bash -docker run ghcr.io/huggingface/text-generation-inference:1.3 --help +docker run ghcr.io/huggingface/text-generation-inference:1.4 --help ``` diff --git a/integration-tests/models/test_flash_phi.py b/integration-tests/models/test_flash_phi.py index 6391f2a1..0987b3a1 100644 --- a/integration-tests/models/test_flash_phi.py +++ b/integration-tests/models/test_flash_phi.py @@ -21,7 +21,7 @@ async def test_flash_phi(flash_phi, response_snapshot): ) assert response.details.generated_tokens == 10 - assert response.generated_text == ": {request}\")\n response = self" + assert response.generated_text == ': {request}")\n response = self' assert response == response_snapshot @@ -52,14 +52,12 @@ async def test_flash_phi_all_params(flash_phi, response_snapshot): @pytest.mark.asyncio @pytest.mark.private async def test_flash_phi_load(flash_phi, generate_load, response_snapshot): - responses = await generate_load( - flash_phi, "Test request", max_new_tokens=10, n=4 - ) + responses = await generate_load(flash_phi, "Test request", max_new_tokens=10, n=4) assert len(responses) == 4 assert all( [r.generated_text == responses[0].generated_text for r in responses] ), f"{[r.generated_text for r in responses]}" - assert responses[0].generated_text == ": {request}\")\n response = self" + assert responses[0].generated_text == ': {request}")\n response = self' assert responses == response_snapshot diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index f6929587..f0c5add9 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-integration-tests" -version = "1.3.4" +version = "1.4.0" description = "Text Generation Inference integration tests" authors = ["Nicolas Patry "] diff --git a/server/poetry.lock b/server/poetry.lock index 3326d0aa..64b1b74f 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -153,21 +153,22 @@ files = [ [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = true python-versions = ">=3.7" files = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] +dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "backoff" @@ -328,20 +329,21 @@ files = [ [[package]] name = "datasets" -version = "2.15.0" +version = "2.16.1" description = "HuggingFace community-driven open-source library of datasets" optional = true python-versions = ">=3.8.0" files = [ - {file = "datasets-2.15.0-py3-none-any.whl", hash = "sha256:6d658d23811393dfc982d026082e1650bdaaae28f6a86e651966cb072229a228"}, - {file = "datasets-2.15.0.tar.gz", hash = "sha256:a26d059370bd7503bd60e9337977199a13117a83f72fb61eda7e66f0c4d50b2b"}, + {file = "datasets-2.16.1-py3-none-any.whl", hash = "sha256:fafa300c78ff92d521473a3d47d60c2d3e0d6046212cc03ceb6caf6550737257"}, + {file = "datasets-2.16.1.tar.gz", hash = "sha256:ad3215e9b1984d1de4fda2123bc7319ccbdf1e17d0c3d5590d13debff308a080"}, ] [package.dependencies] aiohttp = "*" dill = ">=0.3.0,<0.3.8" +filelock = "*" fsspec = {version = ">=2023.1.0,<=2023.10.0", extras = ["http"]} -huggingface-hub = ">=0.18.0" +huggingface-hub = ">=0.19.4" multiprocess = "*" numpy = ">=1.17" packaging = "*" @@ -357,15 +359,15 @@ xxhash = "*" apache-beam = ["apache-beam (>=2.26.0,<2.44.0)"] audio = ["librosa", "soundfile (>=0.12.1)"] benchmarks = ["tensorflow (==2.12.0)", "torch (==2.0.1)", "transformers (==4.30.1)"] -dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "black (>=23.1,<24.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "pyyaml (>=5.3.1)", "rarfile (>=4.0)", "ruff (>=0.0.241)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "typing-extensions (>=4.6.1)", "zstandard"] +dev = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "ruff (>=0.1.5)", "s3fs", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "torch (>=2.0.0)", "transformers", "typing-extensions (>=4.6.1)", "zstandard"] docs = ["s3fs", "tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos", "torch", "transformers"] jax = ["jax (>=0.3.14)", "jaxlib (>=0.3.14)"] metrics-tests = ["Werkzeug (>=1.0.1)", "accelerate", "bert-score (>=0.3.6)", "jiwer", "langdetect", "mauve-text", "nltk", "requests-file (>=1.5.1)", "rouge-score", "sacrebleu", "sacremoses", "scikit-learn", "scipy", "sentencepiece", "seqeval", "six (>=1.15.0,<1.16.0)", "spacy (>=3.0.0)", "texttable (>=1.6.3)", "tldextract", "tldextract (>=3.1.0)", "toml (>=0.10.1)", "typer (<0.5.0)"] -quality = ["black (>=23.1,<24.0)", "pyyaml (>=5.3.1)", "ruff (>=0.0.241)"] +quality = ["ruff (>=0.1.5)"] s3 = ["s3fs"] tensorflow = ["tensorflow (>=2.2.0,!=2.6.0,!=2.6.1)", "tensorflow-macos"] tensorflow-gpu = ["tensorflow-gpu (>=2.2.0,!=2.6.0,!=2.6.1)"] -tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch", "transformers", "typing-extensions (>=4.6.1)", "zstandard"] +tests = ["Pillow (>=6.2.1)", "absl-py", "apache-beam (>=2.26.0,<2.44.0)", "elasticsearch (<8.0.0)", "faiss-cpu (>=1.6.4)", "jax (>=0.3.14)", "jaxlib (>=0.3.14)", "joblib (<1.3.0)", "joblibspark", "librosa", "lz4", "py7zr", "pyspark (>=3.4)", "pytest", "pytest-datadir", "pytest-xdist", "rarfile (>=4.0)", "s3fs (>=2021.11.1)", "soundfile (>=0.12.1)", "sqlalchemy (<2.0.0)", "tensorflow (>=2.3,!=2.6.0,!=2.6.1)", "tensorflow-macos", "tiktoken", "torch (>=2.0.0)", "transformers", "typing-extensions (>=4.6.1)", "zstandard"] torch = ["torch"] vision = ["Pillow (>=6.2.1)"] @@ -767,34 +769,94 @@ setuptools = "*" [[package]] name = "hf-transfer" -version = "0.1.4" +version = "0.1.5" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "hf_transfer-0.1.4-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:6ff5fbde30a5bed35ef8f0d4ba78bde9f6d60a233dbff78a0e4035d6e6f71e4c"}, - {file = "hf_transfer-0.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1c5c20f76e7f3451cff476b85c55dcb8566ebc94a596cb9eb39c0bb75db8675"}, - {file = "hf_transfer-0.1.4-cp310-none-win_amd64.whl", hash = "sha256:84c3ce20c68863a7d998711b98726ba9ae8f2e3fc0d685bc2c9ac9833c0f4048"}, - {file = "hf_transfer-0.1.4-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:dab1cf4e2e6fcb963fe0e48e6b5e3a95cf65ee376c7b6618a05dbb2ef0dde183"}, - {file = "hf_transfer-0.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63c9c7aef90facf45391c86131ed00e74333637735cfec52da4f5170004d0b3f"}, - {file = "hf_transfer-0.1.4-cp311-none-win_amd64.whl", hash = "sha256:eca1fe6ae145e88455d0a174248080498cea52ad45cee50702070b47dffa421f"}, - {file = "hf_transfer-0.1.4-cp312-cp312-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:d07c0d26b5c01ad50d22ddcff7d30c4e8cbb823565b7f61e0ddb35f7faeae415"}, - {file = "hf_transfer-0.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b9cf169c3c64883b07f7ded5e3f14ae1d437eb77448738b88c923fc5597c47"}, - {file = "hf_transfer-0.1.4-cp312-none-win_amd64.whl", hash = "sha256:6b8518b9ebb85b0238745be81f7b88383c7ea216dd8407d46444bcc7806dc0ef"}, - {file = "hf_transfer-0.1.4-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:ea32e9f91de3f2dad3567577c293f2e81a9309e680def4712ec0c4ea49be6833"}, - {file = "hf_transfer-0.1.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e81a10dbf2ac534083da06c200456b5d10ba7a1e8c4c5c48f7ea1ca4cf6af474"}, - {file = "hf_transfer-0.1.4-cp37-none-win_amd64.whl", hash = "sha256:97555bbff69a0459712e5d25d659c0dc74cb8f9726562ca66241f1e1b081f6a9"}, - {file = "hf_transfer-0.1.4-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:38bce7a511952e1b804168e956cd3a3b1ff7e38828259c3cdae27614060b90c5"}, - {file = "hf_transfer-0.1.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1977e94e8c8fc8a0e9ce74a651d4694629e526da246a492855fcfb710aa489"}, - {file = "hf_transfer-0.1.4-cp38-none-win_amd64.whl", hash = "sha256:6ca2d2c40e5e94c5de7e502037ad23ac1d803a2a12760b15b3e3f88c616202bd"}, - {file = "hf_transfer-0.1.4-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:c04a93acb58e50b8da1e2258185e54f6bf48ba24bf95e470310178b7047c1017"}, - {file = "hf_transfer-0.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3028a807363e0b2c64985c44732ba4ab187a569f013367d2115a6e09ae95031"}, - {file = "hf_transfer-0.1.4-cp39-none-win_amd64.whl", hash = "sha256:dc9c7c1d0d79fc06baf86d41620623bb6bb2736755329ea6b1ec5faf71e3e36b"}, - {file = "hf_transfer-0.1.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a466ae2b11d72df9e0005eb8ff7f537d5460c98b64fb6e49f3076ee14040dcf"}, - {file = "hf_transfer-0.1.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb34a023276936d4716112e17daea4ff98afc35b6113dd0f0383710dc208c058"}, - {file = "hf_transfer-0.1.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0647b84d7ff0eee1de6479179a5d43d0695001733f17eecc00153f0f8ab1ac"}, - {file = "hf_transfer-0.1.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27d0bc1f8b79a6d65751efbce7eb02d2c1bd7e4de1a46aac18995461590ce4dd"}, - {file = "hf_transfer-0.1.4.tar.gz", hash = "sha256:687e090639cd52a48dedbfaa9e455a2c99c5169ece3d911f95983b1d4d4c84ed"}, + {file = "hf_transfer-0.1.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c2953fe35e8a9074507ef77d3c29ec511eead8030d25f5a228a3c03dcda723df"}, + {file = "hf_transfer-0.1.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bc48a561a7e42a5ebfb638fb9c154c4c10fa39e878ce8fb9f9db12f98f24665d"}, + {file = "hf_transfer-0.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ccb2b43e1441273aedc83ef9b2419e02028686c9ffcdf0a2bd195657917e24a"}, + {file = "hf_transfer-0.1.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:606bba2d616123b89b749fef1e138118cdf3d84380a6a4fcfe91e1890731ea44"}, + {file = "hf_transfer-0.1.5-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5b870e1c2d6a87d1e5db890747a2d69712f1cbbc91e64f144e066a9fda16b38"}, + {file = "hf_transfer-0.1.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c82a9e058d77ac31cbc2d0f9be8011c8e0a2de787c1752225687c54eec00226"}, + {file = "hf_transfer-0.1.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d3eb58ac583ccf03954bef830ea70a4e02271195f24884723b499a6577ffaf64"}, + {file = "hf_transfer-0.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9fc4b5634a3a61635a8f308eba5df1336bf996b5adc12dc74283743b5bf8fcc"}, + {file = "hf_transfer-0.1.5-cp310-none-win32.whl", hash = "sha256:9153d589ced01668d7bd9a6a5ead3306d91ded5ebef5cd97185dcd51884d23a2"}, + {file = "hf_transfer-0.1.5-cp310-none-win_amd64.whl", hash = "sha256:f5a48f0a606e5278117c130bc85849008f00d50a8efcc5a5e9c9b106a96341f5"}, + {file = "hf_transfer-0.1.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:130966ca4f1266bfb9e13a4b6c40170115a2b450255b7c08ef0de85f04f778ef"}, + {file = "hf_transfer-0.1.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5c5efdbefcc35425933d320b5188485b3db080508c290748ca1fa5864da1347f"}, + {file = "hf_transfer-0.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2cbec42ed94b02d6c21f5fe78c6a65f82703d375dae9448a5efda5c386d2330"}, + {file = "hf_transfer-0.1.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72719931d9f02c1aba13526f431f69cd62a4fc0f7634126c2d1e64d8734760aa"}, + {file = "hf_transfer-0.1.5-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc404b4dc9b5a873bd29d2e95774d17f3b6ff38d5a19bfe34b549c3c99819cec"}, + {file = "hf_transfer-0.1.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e9988b785fd9bc8cd77819675a2744eb3b6a02edfb820c6b5012b861e4c20da"}, + {file = "hf_transfer-0.1.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e7524c4137646ed3471d5b6fdf7e1c6b7d3d43394eeeb595018e32f233019ed"}, + {file = "hf_transfer-0.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d96f414a0fd7b7fb57e359a79ad70a5ba2357bb91375dccc1a285edcc296d35d"}, + {file = "hf_transfer-0.1.5-cp311-none-win32.whl", hash = "sha256:980319ef96fda5abbb7c03ec3803a251f95ed3a9b50f84db25d948994ff6dc34"}, + {file = "hf_transfer-0.1.5-cp311-none-win_amd64.whl", hash = "sha256:885d89c59dd54b687c74a88dad76006c62be4ad11ee1fecea25582d854434b6e"}, + {file = "hf_transfer-0.1.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3a7d71529550eeba0525cec2155f12e04aab9d57eb3e15015768d222ac80743f"}, + {file = "hf_transfer-0.1.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9688be93d9aab0951cedde7ae374e2e77483d13d2f259512a1f94f30723e5060"}, + {file = "hf_transfer-0.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e8584fdd0435d75f8e340605ef3c3d6a5f18f09b75da9bd22fcf0106942b178"}, + {file = "hf_transfer-0.1.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8054e534ec7668fe7d6f9ca0764f1f92e16a40fdd9dd54f4154c5ee6200a00ec"}, + {file = "hf_transfer-0.1.5-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a29736f0d2e843db59898ce1e26e22b477a3f5f60a508e897daf0cfc49fe307"}, + {file = "hf_transfer-0.1.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c15a8fec25e93284d4ffb73000c47f909bb261eb0f8d32886db5f1e5ab5f07de"}, + {file = "hf_transfer-0.1.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad913daff9f602e0ae13cfa79ba265b1db01255e5784c2469619c70231f49088"}, + {file = "hf_transfer-0.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9bce65b7cd31ef883d67c8ab733c303231bd8b4c4d3370524405f6b416a9bc2a"}, + {file = "hf_transfer-0.1.5-cp312-none-win32.whl", hash = "sha256:40842f3b35ceaa99bb6029ab3d1c2cc4b948a19d0b5a2884f8516b744f52a934"}, + {file = "hf_transfer-0.1.5-cp312-none-win_amd64.whl", hash = "sha256:08491bcbd5eefbc0f33cf958671e24fb5d5b72e6e054448cac3b01dfc373dc76"}, + {file = "hf_transfer-0.1.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d721c3354092797531056757cdbe94e24ec898f5c935dd8f3a50b47083e6ea6"}, + {file = "hf_transfer-0.1.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b0192ff102c5b74eef7eb11e425fe0e4a3445dcb82b5ab1fab5b8d619c7caa45"}, + {file = "hf_transfer-0.1.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:95c094afee2cde827081d1b543879e64bb5066a12aba0288d8c479102cfa7a7f"}, + {file = "hf_transfer-0.1.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5758faa7be41636ac78199fda37b0b4cbd2d9a1dc79c777de3524429fc573f65"}, + {file = "hf_transfer-0.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3318bf2436afb24b50a58f668eaaacbee89e23fc00f19e9d2714a9155160098"}, + {file = "hf_transfer-0.1.5-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72fddff6f76f51145635adde4ba59a3c9e4fe97479f750712770286689acece4"}, + {file = "hf_transfer-0.1.5-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bf2f873d0d386a15f79e9730e27a5dbf7e3a52b9de120a7166a254d983eeb4da"}, + {file = "hf_transfer-0.1.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:95244c6c57360b9750cf6603522a8e1b4c42c8348e962efa62aa11d83e4aa6a6"}, + {file = "hf_transfer-0.1.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e58926e22651924510109aa9b37baeaf0a6ae2014774746bc43e7d92e0aaf3f0"}, + {file = "hf_transfer-0.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8182241523493dbc6b108e265c5551b98d8f75c7e3a5bd84f5bf9c1db9729cbf"}, + {file = "hf_transfer-0.1.5-cp37-none-win32.whl", hash = "sha256:a236db066bd017d9a2a543b7414dbcc3fc0df064c3aafd4831ab6b8dcbf1cec2"}, + {file = "hf_transfer-0.1.5-cp37-none-win_amd64.whl", hash = "sha256:af0d23d84fe2326d309b94d7c9ee5a6987fc8005839dd4efff2e4468df1a9c28"}, + {file = "hf_transfer-0.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffc4ea668aa8f35895d1373fc4b1f9544723aa6470b7c21619ed4011d51dc462"}, + {file = "hf_transfer-0.1.5-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c03105c8854302aa0b29f6ae5c180ce07f63e6895c46efde7eea2aeb4429229d"}, + {file = "hf_transfer-0.1.5-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cac2abda46b1aac20e75142b84c170af1f8f387ed35ce53a3856148d362c1a26"}, + {file = "hf_transfer-0.1.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2203860a8affb3bcbcbbb009f592f8591543cf3de8b85b5dccf3e65749d8724"}, + {file = "hf_transfer-0.1.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:34b0b043baf351d087b39ceae299fdc25aa2c5945be843b1878ec6f563a99a51"}, + {file = "hf_transfer-0.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf9e6c5282cf45847c1594d6736b3dfe0c42ec97fc624a70f8c2404c050e0a00"}, + {file = "hf_transfer-0.1.5-cp38-none-win32.whl", hash = "sha256:ea5f975016cca9bf36a76de273f0e477633b0d77dcbbb793260e7b625fb3dc86"}, + {file = "hf_transfer-0.1.5-cp38-none-win_amd64.whl", hash = "sha256:14c9094353e9f9ed4b834b0f581bd531875fccfac5fd951e49b2ab16f1a267c0"}, + {file = "hf_transfer-0.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7e4d25e66695f39d731c4129ce030b24456727df4ddd34febcef559109e4907b"}, + {file = "hf_transfer-0.1.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0ffe2399e3ac162c91d408fcb8be5b38893539ddaaecc334faebfd54a98cdd63"}, + {file = "hf_transfer-0.1.5-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:59888b7bf7a56b94af1755f47a4415c3e32df1c88f55261ff62df0657bd6483a"}, + {file = "hf_transfer-0.1.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2bd88ebe18d87aaf7acf1641127efffb6d082e566d4f21f0bcbe67e4192c2964"}, + {file = "hf_transfer-0.1.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2536d5420d1e7e7751aed592c6e59af59c4ceccb8d5e36f2f7a5707f7218efc"}, + {file = "hf_transfer-0.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2c39584bed7a880d2d5abd0396615d9a3f382009a047d6f70646c57feb27209"}, + {file = "hf_transfer-0.1.5-cp39-none-win32.whl", hash = "sha256:08ce2e01f4057f8e23c2665f1cfb90c3d1f4c93097e99b35907cb1ddadbe4184"}, + {file = "hf_transfer-0.1.5-cp39-none-win_amd64.whl", hash = "sha256:e10812129996981ee100f943c74963d801187c6048269a81879532baf1b32931"}, + {file = "hf_transfer-0.1.5-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8747d2b7ae6e8dcf44070ab44494d9d0f4d6a71d10888dce0a72e62a029e65eb"}, + {file = "hf_transfer-0.1.5-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6492086933e8c4d62e4910af952423fb4fff86c18afff8ece81f228c063f9556"}, + {file = "hf_transfer-0.1.5-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d173dc9fbbff38a3e67e3a21568f67d05427c4208ce77106e1822574a68ee27"}, + {file = "hf_transfer-0.1.5-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2aebcee180cf2731404bdf497da3a4683f5cac5f0b71aced8af5936c7d8283c"}, + {file = "hf_transfer-0.1.5-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e724c4e74763380143526e98b323aeb489fd0b076e70e994454519f35b502b1"}, + {file = "hf_transfer-0.1.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f5533bc5d061595b3f5ce66509b34da3ba51aa0014b02356ca28fecc1251c2f"}, + {file = "hf_transfer-0.1.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3332cbae4128639f2985be2197125e5f7e9577bf82c7fdad773e5740bb17b296"}, + {file = "hf_transfer-0.1.5-pp37-pypy37_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb8a7b55da901a246a2607ccda7dd056e2e594e05e0dde91206f5abae0a4ce3b"}, + {file = "hf_transfer-0.1.5-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b79ad597b1571b162938bbc41d0d01a8788f087f848283723bf42532ac44163f"}, + {file = "hf_transfer-0.1.5-pp37-pypy37_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:069026e38fc8786a91dac0de5e48a1acd6ac8bb59b9a02049fa73ce88303d468"}, + {file = "hf_transfer-0.1.5-pp37-pypy37_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4825a78fd9775b51e089f2d071abf7f3e6877be10d1fc2a0c245862bdc94f89a"}, + {file = "hf_transfer-0.1.5-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:930ca177ce417190283a07738449d08c413a9991338115e8619a1597b813d981"}, + {file = "hf_transfer-0.1.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d89d9bae3f5124519efea5b66b33bca68312d7e0a001313b703d710bddc3b317"}, + {file = "hf_transfer-0.1.5-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:961f2936f21ea8513b89d3025a83349317284a37748dccc0beca62be2e94472c"}, + {file = "hf_transfer-0.1.5-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f3ef2163503d51d7b0199c5ae95934ebfbae00bc7e1ca2e5fef0230f13ab90d"}, + {file = "hf_transfer-0.1.5-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:572655ece4259d5d46102bf56276fa02a0df5912dedbd13e816e4f3f104db733"}, + {file = "hf_transfer-0.1.5-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b883ea8289e1d734d55d873f0e34c8d5304a4f24f18a5cc1b4d3d9b6df608b58"}, + {file = "hf_transfer-0.1.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a9b86d0d432c9a1c76e29d5a5f233f248ddf9912e1219a3c3b2bc43809980db"}, + {file = "hf_transfer-0.1.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:770e58593d15573c5ff47e7dff22ccf7b33ca6580e1358af83dab43078b835bc"}, + {file = "hf_transfer-0.1.5-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1088fcca432486e50142dec893c5ddcc5a32ef7e71b53c5d25b321744b4cd6a4"}, + {file = "hf_transfer-0.1.5-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d7f850c6f5937402c91c0b999792dad66e476165478031e450797063a8cce5c"}, + {file = "hf_transfer-0.1.5-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:761559f1cb743da773ef831a8c19fc3485e1ceb3cbc9a41135a14d0f4ec53a6d"}, + {file = "hf_transfer-0.1.5-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc60ef0efff59b9a65365bc356f5c34a497d0b84df5887c2348855a15911a12d"}, + {file = "hf_transfer-0.1.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdf4149980b75cfb6c129eef142df6247d49c6820b088661985944c617abc1ff"}, + {file = "hf_transfer-0.1.5.tar.gz", hash = "sha256:762202a02627bf9eac438e1f0e12ab13b46af06ba88c2c317042597b4fbbbf73"}, ] [[package]] @@ -854,13 +916,13 @@ files = [ [[package]] name = "jinja2" -version = "3.1.2" +version = "3.1.3" description = "A very fast and expressive template engine." optional = true python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, ] [package.dependencies] @@ -889,71 +951,71 @@ dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils [[package]] name = "markupsafe" -version = "2.1.3" +version = "2.1.4" description = "Safely add untrusted strings to HTML/XML markup." optional = true python-versions = ">=3.7" files = [ - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, - {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, - {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, - {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, - {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, - {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, - {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de8153a7aae3835484ac168a9a9bdaa0c5eee4e0bc595503c95d53b942879c84"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e888ff76ceb39601c59e219f281466c6d7e66bd375b4ec1ce83bcdc68306796b"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0b838c37ba596fcbfca71651a104a611543077156cb0a26fe0c475e1f152ee8"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac1ebf6983148b45b5fa48593950f90ed6d1d26300604f321c74a9ca1609f8e"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbad3d346df8f9d72622ac71b69565e621ada2ce6572f37c2eae8dacd60385d"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d5291d98cd3ad9a562883468c690a2a238c4a6388ab3bd155b0c75dd55ece858"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a7cc49ef48a3c7a0005a949f3c04f8baa5409d3f663a1b36f0eba9bfe2a0396e"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b83041cda633871572f0d3c41dddd5582ad7d22f65a72eacd8d3d6d00291df26"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-win32.whl", hash = "sha256:0c26f67b3fe27302d3a412b85ef696792c4a2386293c53ba683a89562f9399b0"}, + {file = "MarkupSafe-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:a76055d5cb1c23485d7ddae533229039b850db711c554a12ea64a0fd8a0129e2"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9e9e3c4020aa2dc62d5dd6743a69e399ce3de58320522948af6140ac959ab863"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0042d6a9880b38e1dd9ff83146cc3c9c18a059b9360ceae207805567aacccc69"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d03fea4c4e9fd0ad75dc2e7e2b6757b80c152c032ea1d1de487461d8140efc"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ab3a886a237f6e9c9f4f7d272067e712cdb4efa774bef494dccad08f39d8ae6"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abf5ebbec056817057bfafc0445916bb688a255a5146f900445d081db08cbabb"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e1a0d1924a5013d4f294087e00024ad25668234569289650929ab871231668e7"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e7902211afd0af05fbadcc9a312e4cf10f27b779cf1323e78d52377ae4b72bea"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c669391319973e49a7c6230c218a1e3044710bc1ce4c8e6eb71f7e6d43a2c131"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-win32.whl", hash = "sha256:31f57d64c336b8ccb1966d156932f3daa4fee74176b0fdc48ef580be774aae74"}, + {file = "MarkupSafe-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:54a7e1380dfece8847c71bf7e33da5d084e9b889c75eca19100ef98027bd9f56"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a76cd37d229fc385738bd1ce4cba2a121cf26b53864c1772694ad0ad348e509e"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:987d13fe1d23e12a66ca2073b8d2e2a75cec2ecb8eab43ff5624ba0ad42764bc"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5244324676254697fe5c181fc762284e2c5fceeb1c4e3e7f6aca2b6f107e60dc"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78bc995e004681246e85e28e068111a4c3f35f34e6c62da1471e844ee1446250"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4d176cfdfde84f732c4a53109b293d05883e952bbba68b857ae446fa3119b4f"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f9917691f410a2e0897d1ef99619fd3f7dd503647c8ff2475bf90c3cf222ad74"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f06e5a9e99b7df44640767842f414ed5d7bedaaa78cd817ce04bbd6fd86e2dd6"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:396549cea79e8ca4ba65525470d534e8a41070e6b3500ce2414921099cb73e8d"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-win32.whl", hash = "sha256:f6be2d708a9d0e9b0054856f07ac7070fbe1754be40ca8525d5adccdbda8f475"}, + {file = "MarkupSafe-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:5045e892cfdaecc5b4c01822f353cf2c8feb88a6ec1c0adef2a2e705eef0f656"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7a07f40ef8f0fbc5ef1000d0c78771f4d5ca03b4953fc162749772916b298fc4"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d18b66fe626ac412d96c2ab536306c736c66cf2a31c243a45025156cc190dc8a"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:698e84142f3f884114ea8cf83e7a67ca8f4ace8454e78fe960646c6c91c63bfa"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a3b78a5af63ec10d8604180380c13dcd870aba7928c1fe04e881d5c792dc4e"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:15866d7f2dc60cfdde12ebb4e75e41be862348b4728300c36cdf405e258415ec"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6aa5e2e7fc9bc042ae82d8b79d795b9a62bd8f15ba1e7594e3db243f158b5565"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54635102ba3cf5da26eb6f96c4b8c53af8a9c0d97b64bdcb592596a6255d8518"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-win32.whl", hash = "sha256:3583a3a3ab7958e354dc1d25be74aee6228938312ee875a22330c4dc2e41beb0"}, + {file = "MarkupSafe-2.1.4-cp37-cp37m-win_amd64.whl", hash = "sha256:d6e427c7378c7f1b2bef6a344c925b8b63623d3321c09a237b7cc0e77dd98ceb"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:bf1196dcc239e608605b716e7b166eb5faf4bc192f8a44b81e85251e62584bd2"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df98d4a9cd6a88d6a585852f56f2155c9cdb6aec78361a19f938810aa020954"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b835aba863195269ea358cecc21b400276747cc977492319fd7682b8cd2c253d"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23984d1bdae01bee794267424af55eef4dfc038dc5d1272860669b2aa025c9e3"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c98c33ffe20e9a489145d97070a435ea0679fddaabcafe19982fe9c971987d5"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9896fca4a8eb246defc8b2a7ac77ef7553b638e04fbf170bff78a40fa8a91474"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b0fe73bac2fed83839dbdbe6da84ae2a31c11cfc1c777a40dbd8ac8a6ed1560f"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c7556bafeaa0a50e2fe7dc86e0382dea349ebcad8f010d5a7dc6ba568eaaa789"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-win32.whl", hash = "sha256:fc1a75aa8f11b87910ffd98de62b29d6520b6d6e8a3de69a70ca34dea85d2a8a"}, + {file = "MarkupSafe-2.1.4-cp38-cp38-win_amd64.whl", hash = "sha256:3a66c36a3864df95e4f62f9167c734b3b1192cb0851b43d7cc08040c074c6279"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:765f036a3d00395a326df2835d8f86b637dbaf9832f90f5d196c3b8a7a5080cb"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:21e7af8091007bf4bebf4521184f4880a6acab8df0df52ef9e513d8e5db23411"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5c31fe855c77cad679b302aabc42d724ed87c043b1432d457f4976add1c2c3e"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7653fa39578957bc42e5ebc15cf4361d9e0ee4b702d7d5ec96cdac860953c5b4"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47bb5f0142b8b64ed1399b6b60f700a580335c8e1c57f2f15587bd072012decc"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fe8512ed897d5daf089e5bd010c3dc03bb1bdae00b35588c49b98268d4a01e00"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:36d7626a8cca4d34216875aee5a1d3d654bb3dac201c1c003d182283e3205949"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b6f14a9cd50c3cb100eb94b3273131c80d102e19bb20253ac7bd7336118a673a"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-win32.whl", hash = "sha256:c8f253a84dbd2c63c19590fa86a032ef3d8cc18923b8049d91bcdeeb2581fbf6"}, + {file = "MarkupSafe-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:8b570a1537367b52396e53325769608f2a687ec9a4363647af1cded8928af959"}, + {file = "MarkupSafe-2.1.4.tar.gz", hash = "sha256:3aae9af4cac263007fd6309c64c6ab4506dd2b79382d9d19a1994f9240b8db4f"}, ] [[package]] @@ -1104,47 +1166,47 @@ test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "numpy" -version = "1.26.2" +version = "1.26.3" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "numpy-1.26.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3703fc9258a4a122d17043e57b35e5ef1c5a5837c3db8be396c82e04c1cf9b0f"}, - {file = "numpy-1.26.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc392fdcbd21d4be6ae1bb4475a03ce3b025cd49a9be5345d76d7585aea69440"}, - {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36340109af8da8805d8851ef1d74761b3b88e81a9bd80b290bbfed61bd2b4f75"}, - {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcc008217145b3d77abd3e4d5ef586e3bdfba8fe17940769f8aa09b99e856c00"}, - {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ced40d4e9e18242f70dd02d739e44698df3dcb010d31f495ff00a31ef6014fe"}, - {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b272d4cecc32c9e19911891446b72e986157e6a1809b7b56518b4f3755267523"}, - {file = "numpy-1.26.2-cp310-cp310-win32.whl", hash = "sha256:22f8fc02fdbc829e7a8c578dd8d2e15a9074b630d4da29cda483337e300e3ee9"}, - {file = "numpy-1.26.2-cp310-cp310-win_amd64.whl", hash = "sha256:26c9d33f8e8b846d5a65dd068c14e04018d05533b348d9eaeef6c1bd787f9919"}, - {file = "numpy-1.26.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b96e7b9c624ef3ae2ae0e04fa9b460f6b9f17ad8b4bec6d7756510f1f6c0c841"}, - {file = "numpy-1.26.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aa18428111fb9a591d7a9cc1b48150097ba6a7e8299fb56bdf574df650e7d1f1"}, - {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06fa1ed84aa60ea6ef9f91ba57b5ed963c3729534e6e54055fc151fad0423f0a"}, - {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96ca5482c3dbdd051bcd1fce8034603d6ebfc125a7bd59f55b40d8f5d246832b"}, - {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:854ab91a2906ef29dc3925a064fcd365c7b4da743f84b123002f6139bcb3f8a7"}, - {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f43740ab089277d403aa07567be138fc2a89d4d9892d113b76153e0e412409f8"}, - {file = "numpy-1.26.2-cp311-cp311-win32.whl", hash = "sha256:a2bbc29fcb1771cd7b7425f98b05307776a6baf43035d3b80c4b0f29e9545186"}, - {file = "numpy-1.26.2-cp311-cp311-win_amd64.whl", hash = "sha256:2b3fca8a5b00184828d12b073af4d0fc5fdd94b1632c2477526f6bd7842d700d"}, - {file = "numpy-1.26.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a4cd6ed4a339c21f1d1b0fdf13426cb3b284555c27ac2f156dfdaaa7e16bfab0"}, - {file = "numpy-1.26.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d5244aabd6ed7f312268b9247be47343a654ebea52a60f002dc70c769048e75"}, - {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a3cdb4d9c70e6b8c0814239ead47da00934666f668426fc6e94cce869e13fd7"}, - {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa317b2325f7aa0a9471663e6093c210cb2ae9c0ad824732b307d2c51983d5b6"}, - {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:174a8880739c16c925799c018f3f55b8130c1f7c8e75ab0a6fa9d41cab092fd6"}, - {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f79b231bf5c16b1f39c7f4875e1ded36abee1591e98742b05d8a0fb55d8a3eec"}, - {file = "numpy-1.26.2-cp312-cp312-win32.whl", hash = "sha256:4a06263321dfd3598cacb252f51e521a8cb4b6df471bb12a7ee5cbab20ea9167"}, - {file = "numpy-1.26.2-cp312-cp312-win_amd64.whl", hash = "sha256:b04f5dc6b3efdaab541f7857351aac359e6ae3c126e2edb376929bd3b7f92d7e"}, - {file = "numpy-1.26.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4eb8df4bf8d3d90d091e0146f6c28492b0be84da3e409ebef54349f71ed271ef"}, - {file = "numpy-1.26.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1a13860fdcd95de7cf58bd6f8bc5a5ef81c0b0625eb2c9a783948847abbef2c2"}, - {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64308ebc366a8ed63fd0bf426b6a9468060962f1a4339ab1074c228fa6ade8e3"}, - {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baf8aab04a2c0e859da118f0b38617e5ee65d75b83795055fb66c0d5e9e9b818"}, - {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d73a3abcac238250091b11caef9ad12413dab01669511779bc9b29261dd50210"}, - {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b361d369fc7e5e1714cf827b731ca32bff8d411212fccd29ad98ad622449cc36"}, - {file = "numpy-1.26.2-cp39-cp39-win32.whl", hash = "sha256:bd3f0091e845164a20bd5a326860c840fe2af79fa12e0469a12768a3ec578d80"}, - {file = "numpy-1.26.2-cp39-cp39-win_amd64.whl", hash = "sha256:2beef57fb031dcc0dc8fa4fe297a742027b954949cabb52a2a376c144e5e6060"}, - {file = "numpy-1.26.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1cc3d5029a30fb5f06704ad6b23b35e11309491c999838c31f124fee32107c79"}, - {file = "numpy-1.26.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94cc3c222bb9fb5a12e334d0479b97bb2df446fbe622b470928f5284ffca3f8d"}, - {file = "numpy-1.26.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe6b44fb8fcdf7eda4ef4461b97b3f63c466b27ab151bec2366db8b197387841"}, - {file = "numpy-1.26.2.tar.gz", hash = "sha256:f65738447676ab5777f11e6bbbdb8ce11b785e105f690bc45966574816b6d3ea"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485"}, + {file = "numpy-1.26.3-cp310-cp310-win32.whl", hash = "sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3"}, + {file = "numpy-1.26.3-cp310-cp310-win_amd64.whl", hash = "sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00"}, + {file = "numpy-1.26.3-cp311-cp311-win32.whl", hash = "sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b"}, + {file = "numpy-1.26.3-cp311-cp311-win_amd64.whl", hash = "sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36"}, + {file = "numpy-1.26.3-cp312-cp312-win32.whl", hash = "sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511"}, + {file = "numpy-1.26.3-cp312-cp312-win_amd64.whl", hash = "sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb"}, + {file = "numpy-1.26.3-cp39-cp39-win32.whl", hash = "sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03"}, + {file = "numpy-1.26.3-cp39-cp39-win_amd64.whl", hash = "sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5"}, + {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, ] [[package]] @@ -1455,36 +1517,40 @@ files = [ [[package]] name = "pandas" -version = "2.1.4" +version = "2.2.0" description = "Powerful data structures for data analysis, time series, and statistics" optional = true python-versions = ">=3.9" files = [ - {file = "pandas-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bdec823dc6ec53f7a6339a0e34c68b144a7a1fd28d80c260534c39c62c5bf8c9"}, - {file = "pandas-2.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:294d96cfaf28d688f30c918a765ea2ae2e0e71d3536754f4b6de0ea4a496d034"}, - {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b728fb8deba8905b319f96447a27033969f3ea1fea09d07d296c9030ab2ed1d"}, - {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00028e6737c594feac3c2df15636d73ace46b8314d236100b57ed7e4b9ebe8d9"}, - {file = "pandas-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:426dc0f1b187523c4db06f96fb5c8d1a845e259c99bda74f7de97bd8a3bb3139"}, - {file = "pandas-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:f237e6ca6421265643608813ce9793610ad09b40154a3344a088159590469e46"}, - {file = "pandas-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7d852d16c270e4331f6f59b3e9aa23f935f5c4b0ed2d0bc77637a8890a5d092"}, - {file = "pandas-2.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7d5f2f54f78164b3d7a40f33bf79a74cdee72c31affec86bfcabe7e0789821"}, - {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa6e92e639da0d6e2017d9ccff563222f4eb31e4b2c3cf32a2a392fc3103c0d"}, - {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d797591b6846b9db79e65dc2d0d48e61f7db8d10b2a9480b4e3faaddc421a171"}, - {file = "pandas-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2d3e7b00f703aea3945995ee63375c61b2e6aa5aa7871c5d622870e5e137623"}, - {file = "pandas-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc9bf7ade01143cddc0074aa6995edd05323974e6e40d9dbde081021ded8510e"}, - {file = "pandas-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:482d5076e1791777e1571f2e2d789e940dedd927325cc3cb6d0800c6304082f6"}, - {file = "pandas-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a706cfe7955c4ca59af8c7a0517370eafbd98593155b48f10f9811da440248b"}, - {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0513a132a15977b4a5b89aabd304647919bc2169eac4c8536afb29c07c23540"}, - {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f17f2b6fc076b2a0078862547595d66244db0f41bf79fc5f64a5c4d635bead"}, - {file = "pandas-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:45d63d2a9b1b37fa6c84a68ba2422dc9ed018bdaa668c7f47566a01188ceeec1"}, - {file = "pandas-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f69b0c9bb174a2342818d3e2778584e18c740d56857fc5cdb944ec8bbe4082cf"}, - {file = "pandas-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3f06bda01a143020bad20f7a85dd5f4a1600112145f126bc9e3e42077c24ef34"}, - {file = "pandas-2.1.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab5796839eb1fd62a39eec2916d3e979ec3130509930fea17fe6f81e18108f6a"}, - {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbaf9e8d3a63a9276d707b4d25930a262341bca9874fcb22eff5e3da5394732"}, - {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ebfd771110b50055712b3b711b51bee5d50135429364d0498e1213a7adc2be8"}, - {file = "pandas-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8ea107e0be2aba1da619cc6ba3f999b2bfc9669a83554b1904ce3dd9507f0860"}, - {file = "pandas-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:d65148b14788b3758daf57bf42725caa536575da2b64df9964c563b015230984"}, - {file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"}, + {file = "pandas-2.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8108ee1712bb4fa2c16981fba7e68b3f6ea330277f5ca34fa8d557e986a11670"}, + {file = "pandas-2.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:736da9ad4033aeab51d067fc3bd69a0ba36f5a60f66a527b3d72e2030e63280a"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38e0b4fc3ddceb56ec8a287313bc22abe17ab0eb184069f08fc6a9352a769b18"}, + {file = "pandas-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20404d2adefe92aed3b38da41d0847a143a09be982a31b85bc7dd565bdba0f4e"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ea3ee3f125032bfcade3a4cf85131ed064b4f8dd23e5ce6fa16473e48ebcaf5"}, + {file = "pandas-2.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9670b3ac00a387620489dfc1bca66db47a787f4e55911f1293063a78b108df1"}, + {file = "pandas-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:5a946f210383c7e6d16312d30b238fd508d80d927014f3b33fb5b15c2f895430"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a1b438fa26b208005c997e78672f1aa8138f67002e833312e6230f3e57fa87d5"}, + {file = "pandas-2.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8ce2fbc8d9bf303ce54a476116165220a1fedf15985b09656b4b4275300e920b"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2707514a7bec41a4ab81f2ccce8b382961a29fbe9492eab1305bb075b2b1ff4f"}, + {file = "pandas-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85793cbdc2d5bc32620dc8ffa715423f0c680dacacf55056ba13454a5be5de88"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cfd6c2491dc821b10c716ad6776e7ab311f7df5d16038d0b7458bc0b67dc10f3"}, + {file = "pandas-2.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a146b9dcacc3123aa2b399df1a284de5f46287a4ab4fbfc237eac98a92ebcb71"}, + {file = "pandas-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbc1b53c0e1fdf16388c33c3cca160f798d38aea2978004dd3f4d3dec56454c9"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a41d06f308a024981dcaa6c41f2f2be46a6b186b902c94c2674e8cb5c42985bc"}, + {file = "pandas-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:159205c99d7a5ce89ecfc37cb08ed179de7783737cea403b295b5eda8e9c56d1"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1e1f3861ea9132b32f2133788f3b14911b68102d562715d71bd0013bc45440"}, + {file = "pandas-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:761cb99b42a69005dec2b08854fb1d4888fdf7b05db23a8c5a099e4b886a2106"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a20628faaf444da122b2a64b1e5360cde100ee6283ae8effa0d8745153809a2e"}, + {file = "pandas-2.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f5be5d03ea2073627e7111f61b9f1f0d9625dc3c4d8dda72cc827b0c58a1d042"}, + {file = "pandas-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:a626795722d893ed6aacb64d2401d017ddc8a2341b49e0384ab9bf7112bdec30"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9f66419d4a41132eb7e9a73dcec9486cf5019f52d90dd35547af11bc58f8637d"}, + {file = "pandas-2.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:57abcaeda83fb80d447f28ab0cc7b32b13978f6f733875ebd1ed14f8fbc0f4ab"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60f1f7dba3c2d5ca159e18c46a34e7ca7247a73b5dd1a22b6d59707ed6b899a"}, + {file = "pandas-2.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb61dc8567b798b969bcc1fc964788f5a68214d333cade8319c7ab33e2b5d88a"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:52826b5f4ed658fa2b729264d63f6732b8b29949c7fd234510d57c61dbeadfcd"}, + {file = "pandas-2.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bde2bc699dbd80d7bc7f9cab1e23a95c4375de615860ca089f34e7c64f4a8de7"}, + {file = "pandas-2.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:3de918a754bbf2da2381e8a3dcc45eede8cd7775b047b923f9006d5f876802ae"}, + {file = "pandas-2.2.0.tar.gz", hash = "sha256:30b83f7c3eb217fb4d1b494a57a2fda5444f17834f5df2de6b2ffff68dc3c8e2"}, ] [package.dependencies] @@ -1495,31 +1561,31 @@ numpy = [ ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" -tzdata = ">=2022.1" +tzdata = ">=2022.7" [package.extras] -all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] -aws = ["s3fs (>=2022.05.0)"] -clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] -compression = ["zstandard (>=0.17.0)"] -computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] -feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2022.05.0)"] -gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] -hdf5 = ["tables (>=3.7.0)"] -html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] -mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] -parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] -plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] -spss = ["pyreadstat (>=1.1.5)"] -sql-other = ["SQLAlchemy (>=1.4.36)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.8.0)"] +xml = ["lxml (>=4.9.2)"] [[package]] name = "peft" @@ -1550,80 +1616,98 @@ test = ["black (>=22.0,<23.0)", "datasets", "diffusers", "hf-doc-builder", "para [[package]] name = "pillow" -version = "10.1.0" +version = "10.2.0" description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.8" files = [ - {file = "Pillow-10.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106"}, - {file = "Pillow-10.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273"}, - {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666"}, - {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2"}, - {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593"}, - {file = "Pillow-10.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db"}, - {file = "Pillow-10.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f"}, - {file = "Pillow-10.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818"}, - {file = "Pillow-10.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57"}, - {file = "Pillow-10.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7"}, - {file = "Pillow-10.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7"}, - {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610"}, - {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839"}, - {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172"}, - {file = "Pillow-10.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061"}, - {file = "Pillow-10.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262"}, - {file = "Pillow-10.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992"}, - {file = "Pillow-10.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a"}, - {file = "Pillow-10.1.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b"}, - {file = "Pillow-10.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d"}, - {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27"}, - {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312"}, - {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de"}, - {file = "Pillow-10.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651"}, - {file = "Pillow-10.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b"}, - {file = "Pillow-10.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f"}, - {file = "Pillow-10.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996"}, - {file = "Pillow-10.1.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793"}, - {file = "Pillow-10.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e"}, - {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2"}, - {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a"}, - {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01"}, - {file = "Pillow-10.1.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d"}, - {file = "Pillow-10.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80"}, - {file = "Pillow-10.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212"}, - {file = "Pillow-10.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14"}, - {file = "Pillow-10.1.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099"}, - {file = "Pillow-10.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616"}, - {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb"}, - {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219"}, - {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34"}, - {file = "Pillow-10.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd"}, - {file = "Pillow-10.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28"}, - {file = "Pillow-10.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2"}, - {file = "Pillow-10.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256"}, - {file = "Pillow-10.1.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7"}, - {file = "Pillow-10.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba"}, - {file = "Pillow-10.1.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4"}, - {file = "Pillow-10.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9"}, - {file = "Pillow-10.1.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e"}, - {file = "Pillow-10.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412"}, - {file = "Pillow-10.1.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b"}, - {file = "Pillow-10.1.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f"}, - {file = "Pillow-10.1.0.tar.gz", hash = "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38"}, + {file = "pillow-10.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:7823bdd049099efa16e4246bdf15e5a13dbb18a51b68fa06d6c1d4d8b99a796e"}, + {file = "pillow-10.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:83b2021f2ade7d1ed556bc50a399127d7fb245e725aa0113ebd05cfe88aaf588"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fad5ff2f13d69b7e74ce5b4ecd12cc0ec530fcee76356cac6742785ff71c452"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da2b52b37dad6d9ec64e653637a096905b258d2fc2b984c41ae7d08b938a67e4"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:47c0995fc4e7f79b5cfcab1fc437ff2890b770440f7696a3ba065ee0fd496563"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:322bdf3c9b556e9ffb18f93462e5f749d3444ce081290352c6070d014c93feb2"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:51f1a1bffc50e2e9492e87d8e09a17c5eea8409cda8d3f277eb6edc82813c17c"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:69ffdd6120a4737710a9eee73e1d2e37db89b620f702754b8f6e62594471dee0"}, + {file = "pillow-10.2.0-cp310-cp310-win32.whl", hash = "sha256:c6dafac9e0f2b3c78df97e79af707cdc5ef8e88208d686a4847bab8266870023"}, + {file = "pillow-10.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:aebb6044806f2e16ecc07b2a2637ee1ef67a11840a66752751714a0d924adf72"}, + {file = "pillow-10.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:7049e301399273a0136ff39b84c3678e314f2158f50f517bc50285fb5ec847ad"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35bb52c37f256f662abdfa49d2dfa6ce5d93281d323a9af377a120e89a9eafb5"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c23f307202661071d94b5e384e1e1dc7dfb972a28a2310e4ee16103e66ddb67"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:773efe0603db30c281521a7c0214cad7836c03b8ccff897beae9b47c0b657d61"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11fa2e5984b949b0dd6d7a94d967743d87c577ff0b83392f17cb3990d0d2fd6e"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:716d30ed977be8b37d3ef185fecb9e5a1d62d110dfbdcd1e2a122ab46fddb03f"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a086c2af425c5f62a65e12fbf385f7c9fcb8f107d0849dba5839461a129cf311"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c8de2789052ed501dd829e9cae8d3dcce7acb4777ea4a479c14521c942d395b1"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:609448742444d9290fd687940ac0b57fb35e6fd92bdb65386e08e99af60bf757"}, + {file = "pillow-10.2.0-cp311-cp311-win32.whl", hash = "sha256:823ef7a27cf86df6597fa0671066c1b596f69eba53efa3d1e1cb8b30f3533068"}, + {file = "pillow-10.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:1da3b2703afd040cf65ec97efea81cfba59cdbed9c11d8efc5ab09df9509fc56"}, + {file = "pillow-10.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:edca80cbfb2b68d7b56930b84a0e45ae1694aeba0541f798e908a49d66b837f1"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:1b5e1b74d1bd1b78bc3477528919414874748dd363e6272efd5abf7654e68bef"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0eae2073305f451d8ecacb5474997c08569fb4eb4ac231ffa4ad7d342fdc25ac"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7c2286c23cd350b80d2fc9d424fc797575fb16f854b831d16fd47ceec078f2c"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e23412b5c41e58cec602f1135c57dfcf15482013ce6e5f093a86db69646a5aa"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:52a50aa3fb3acb9cf7213573ef55d31d6eca37f5709c69e6858fe3bc04a5c2a2"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:127cee571038f252a552760076407f9cff79761c3d436a12af6000cd182a9d04"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8d12251f02d69d8310b046e82572ed486685c38f02176bd08baf216746eb947f"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:54f1852cd531aa981bc0965b7d609f5f6cc8ce8c41b1139f6ed6b3c54ab82bfb"}, + {file = "pillow-10.2.0-cp312-cp312-win32.whl", hash = "sha256:257d8788df5ca62c980314053197f4d46eefedf4e6175bc9412f14412ec4ea2f"}, + {file = "pillow-10.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:154e939c5f0053a383de4fd3d3da48d9427a7e985f58af8e94d0b3c9fcfcf4f9"}, + {file = "pillow-10.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8373c6c251f7ef8bda6675dd6d2b3a0fcc31edf1201266b5cf608b62a37407f9"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:870ea1ada0899fd0b79643990809323b389d4d1d46c192f97342eeb6ee0b8483"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b6b1e20608493548b1f32bce8cca185bf0480983890403d3b8753e44077129"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3031709084b6e7852d00479fd1d310b07d0ba82765f973b543c8af5061cf990e"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:3ff074fc97dd4e80543a3e91f69d58889baf2002b6be64347ea8cf5533188213"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:cb4c38abeef13c61d6916f264d4845fab99d7b711be96c326b84df9e3e0ff62d"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b1b3020d90c2d8e1dae29cf3ce54f8094f7938460fb5ce8bc5c01450b01fbaf6"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:170aeb00224ab3dc54230c797f8404507240dd868cf52066f66a41b33169bdbe"}, + {file = "pillow-10.2.0-cp38-cp38-win32.whl", hash = "sha256:c4225f5220f46b2fde568c74fca27ae9771536c2e29d7c04f4fb62c83275ac4e"}, + {file = "pillow-10.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0689b5a8c5288bc0504d9fcee48f61a6a586b9b98514d7d29b840143d6734f39"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:b792a349405fbc0163190fde0dc7b3fef3c9268292586cf5645598b48e63dc67"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c570f24be1e468e3f0ce7ef56a89a60f0e05b30a3669a459e419c6eac2c35364"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8ecd059fdaf60c1963c58ceb8997b32e9dc1b911f5da5307aab614f1ce5c2fb"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c365fd1703040de1ec284b176d6af5abe21b427cb3a5ff68e0759e1e313a5e7e"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:70c61d4c475835a19b3a5aa42492409878bbca7438554a1f89d20d58a7c75c01"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b6f491cdf80ae540738859d9766783e3b3c8e5bd37f5dfa0b76abdecc5081f13"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d189550615b4948f45252d7f005e53c2040cea1af5b60d6f79491a6e147eef7"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:49d9ba1ed0ef3e061088cd1e7538a0759aab559e2e0a80a36f9fd9d8c0c21591"}, + {file = "pillow-10.2.0-cp39-cp39-win32.whl", hash = "sha256:babf5acfede515f176833ed6028754cbcd0d206f7f614ea3447d67c33be12516"}, + {file = "pillow-10.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:0304004f8067386b477d20a518b50f3fa658a28d44e4116970abfcd94fac34a8"}, + {file = "pillow-10.2.0-cp39-cp39-win_arm64.whl", hash = "sha256:0fb3e7fc88a14eacd303e90481ad983fd5b69c761e9e6ef94c983f91025da869"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:322209c642aabdd6207517e9739c704dc9f9db943015535783239022002f054a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eedd52442c0a5ff4f887fab0c1c0bb164d8635b32c894bc1faf4c618dd89df2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb28c753fd5eb3dd859b4ee95de66cc62af91bcff5db5f2571d32a520baf1f04"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:33870dc4653c5017bf4c8873e5488d8f8d5f8935e2f1fb9a2208c47cdd66efd2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3c31822339516fb3c82d03f30e22b1d038da87ef27b6a78c9549888f8ceda39a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a2b56ba36e05f973d450582fb015594aaa78834fefe8dfb8fcd79b93e64ba4c6"}, + {file = "pillow-10.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d8e6aeb9201e655354b3ad049cb77d19813ad4ece0df1249d3c793de3774f8c7"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:2247178effb34a77c11c0e8ac355c7a741ceca0a732b27bf11e747bbc950722f"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15587643b9e5eb26c48e49a7b33659790d28f190fc514a322d55da2fb5c2950e"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753cd8f2086b2b80180d9b3010dd4ed147efc167c90d3bf593fe2af21265e5a5"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7c8f97e8e7a9009bcacbe3766a36175056c12f9a44e6e6f2d5caad06dcfbf03b"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d1b35bcd6c5543b9cb547dee3150c93008f8dd0f1fef78fc0cd2b141c5baf58a"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868"}, + {file = "pillow-10.2.0.tar.gz", hash = "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e"}, ] [package.extras] docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] [[package]] name = "pluggy" -version = "1.3.0" +version = "1.4.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, - {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, ] [package.extras] @@ -1632,47 +1716,47 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.25.1" +version = "4.25.2" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-4.25.1-cp310-abi3-win32.whl", hash = "sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7"}, - {file = "protobuf-4.25.1-cp310-abi3-win_amd64.whl", hash = "sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b"}, - {file = "protobuf-4.25.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd"}, - {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb"}, - {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7"}, - {file = "protobuf-4.25.1-cp38-cp38-win32.whl", hash = "sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd"}, - {file = "protobuf-4.25.1-cp38-cp38-win_amd64.whl", hash = "sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0"}, - {file = "protobuf-4.25.1-cp39-cp39-win32.whl", hash = "sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510"}, - {file = "protobuf-4.25.1-cp39-cp39-win_amd64.whl", hash = "sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10"}, - {file = "protobuf-4.25.1-py3-none-any.whl", hash = "sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6"}, - {file = "protobuf-4.25.1.tar.gz", hash = "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2"}, + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, ] [[package]] name = "psutil" -version = "5.9.6" +version = "5.9.8" description = "Cross-platform lib for process and system monitoring in Python." optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "psutil-5.9.6-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d"}, - {file = "psutil-5.9.6-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c"}, - {file = "psutil-5.9.6-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28"}, - {file = "psutil-5.9.6-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017"}, - {file = "psutil-5.9.6-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c"}, - {file = "psutil-5.9.6-cp27-none-win32.whl", hash = "sha256:70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9"}, - {file = "psutil-5.9.6-cp27-none-win_amd64.whl", hash = "sha256:51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac"}, - {file = "psutil-5.9.6-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a"}, - {file = "psutil-5.9.6-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c"}, - {file = "psutil-5.9.6-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4"}, - {file = "psutil-5.9.6-cp36-cp36m-win32.whl", hash = "sha256:3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602"}, - {file = "psutil-5.9.6-cp36-cp36m-win_amd64.whl", hash = "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"}, - {file = "psutil-5.9.6-cp37-abi3-win32.whl", hash = "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c"}, - {file = "psutil-5.9.6-cp37-abi3-win_amd64.whl", hash = "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a"}, - {file = "psutil-5.9.6-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57"}, - {file = "psutil-5.9.6.tar.gz", hash = "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a"}, + {file = "psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"}, + {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36"}, + {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d"}, + {file = "psutil-5.9.8-cp27-none-win32.whl", hash = "sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e"}, + {file = "psutil-5.9.8-cp27-none-win_amd64.whl", hash = "sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631"}, + {file = "psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421"}, + {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4"}, + {file = "psutil-5.9.8-cp36-cp36m-win32.whl", hash = "sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee"}, + {file = "psutil-5.9.8-cp36-cp36m-win_amd64.whl", hash = "sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2"}, + {file = "psutil-5.9.8-cp37-abi3-win32.whl", hash = "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0"}, + {file = "psutil-5.9.8-cp37-abi3-win_amd64.whl", hash = "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf"}, + {file = "psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"}, + {file = "psutil-5.9.8.tar.gz", hash = "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c"}, ] [package.extras] @@ -1680,51 +1764,51 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "pyarrow" -version = "14.0.1" +version = "15.0.0" description = "Python library for Apache Arrow" optional = true python-versions = ">=3.8" files = [ - {file = "pyarrow-14.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:96d64e5ba7dceb519a955e5eeb5c9adcfd63f73a56aea4722e2cc81364fc567a"}, - {file = "pyarrow-14.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a8ae88c0038d1bc362a682320112ee6774f006134cd5afc291591ee4bc06505"}, - {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f6f053cb66dc24091f5511e5920e45c83107f954a21032feadc7b9e3a8e7851"}, - {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:906b0dc25f2be12e95975722f1e60e162437023f490dbd80d0deb7375baf3171"}, - {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:78d4a77a46a7de9388b653af1c4ce539350726cd9af62e0831e4f2bd0c95a2f4"}, - {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:06ca79080ef89d6529bb8e5074d4b4f6086143b2520494fcb7cf8a99079cde93"}, - {file = "pyarrow-14.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:32542164d905002c42dff896efdac79b3bdd7291b1b74aa292fac8450d0e4dcd"}, - {file = "pyarrow-14.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c7331b4ed3401b7ee56f22c980608cf273f0380f77d0f73dd3c185f78f5a6220"}, - {file = "pyarrow-14.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:922e8b49b88da8633d6cac0e1b5a690311b6758d6f5d7c2be71acb0f1e14cd61"}, - {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58c889851ca33f992ea916b48b8540735055201b177cb0dcf0596a495a667b00"}, - {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30d8494870d9916bb53b2a4384948491444741cb9a38253c590e21f836b01222"}, - {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:be28e1a07f20391bb0b15ea03dcac3aade29fc773c5eb4bee2838e9b2cdde0cb"}, - {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:981670b4ce0110d8dcb3246410a4aabf5714db5d8ea63b15686bce1c914b1f83"}, - {file = "pyarrow-14.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:4756a2b373a28f6166c42711240643fb8bd6322467e9aacabd26b488fa41ec23"}, - {file = "pyarrow-14.0.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:cf87e2cec65dd5cf1aa4aba918d523ef56ef95597b545bbaad01e6433851aa10"}, - {file = "pyarrow-14.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:470ae0194fbfdfbf4a6b65b4f9e0f6e1fa0ea5b90c1ee6b65b38aecee53508c8"}, - {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6263cffd0c3721c1e348062997babdf0151301f7353010c9c9a8ed47448f82ab"}, - {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a8089d7e77d1455d529dbd7cff08898bbb2666ee48bc4085203af1d826a33cc"}, - {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fada8396bc739d958d0b81d291cfd201126ed5e7913cb73de6bc606befc30226"}, - {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:2a145dab9ed7849fc1101bf03bcdc69913547f10513fdf70fc3ab6c0a50c7eee"}, - {file = "pyarrow-14.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:05fe7994745b634c5fb16ce5717e39a1ac1fac3e2b0795232841660aa76647cd"}, - {file = "pyarrow-14.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:a8eeef015ae69d104c4c3117a6011e7e3ecd1abec79dc87fd2fac6e442f666ee"}, - {file = "pyarrow-14.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3c76807540989fe8fcd02285dd15e4f2a3da0b09d27781abec3adc265ddbeba1"}, - {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:450e4605e3c20e558485f9161a79280a61c55efe585d51513c014de9ae8d393f"}, - {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:323cbe60210173ffd7db78bfd50b80bdd792c4c9daca8843ef3cd70b186649db"}, - {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0140c7e2b740e08c5a459439d87acd26b747fc408bde0a8806096ee0baaa0c15"}, - {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:e592e482edd9f1ab32f18cd6a716c45b2c0f2403dc2af782f4e9674952e6dd27"}, - {file = "pyarrow-14.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d264ad13605b61959f2ae7c1d25b1a5b8505b112715c961418c8396433f213ad"}, - {file = "pyarrow-14.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:01e44de9749cddc486169cb632f3c99962318e9dacac7778315a110f4bf8a450"}, - {file = "pyarrow-14.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d0351fecf0e26e152542bc164c22ea2a8e8c682726fce160ce4d459ea802d69c"}, - {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33c1f6110c386464fd2e5e4ea3624466055bbe681ff185fd6c9daa98f30a3f9a"}, - {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11e045dfa09855b6d3e7705a37c42e2dc2c71d608fab34d3c23df2e02df9aec3"}, - {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:097828b55321897db0e1dbfc606e3ff8101ae5725673498cbfa7754ee0da80e4"}, - {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:1daab52050a1c48506c029e6fa0944a7b2436334d7e44221c16f6f1b2cc9c510"}, - {file = "pyarrow-14.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3f6d5faf4f1b0d5a7f97be987cf9e9f8cd39902611e818fe134588ee99bf0283"}, - {file = "pyarrow-14.0.1.tar.gz", hash = "sha256:b8b3f4fe8d4ec15e1ef9b599b94683c5216adaed78d5cb4c606180546d1e2ee1"}, + {file = "pyarrow-15.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:0a524532fd6dd482edaa563b686d754c70417c2f72742a8c990b322d4c03a15d"}, + {file = "pyarrow-15.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:60a6bdb314affa9c2e0d5dddf3d9cbb9ef4a8dddaa68669975287d47ece67642"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:66958fd1771a4d4b754cd385835e66a3ef6b12611e001d4e5edfcef5f30391e2"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f500956a49aadd907eaa21d4fff75f73954605eaa41f61cb94fb008cf2e00c6"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6f87d9c4f09e049c2cade559643424da84c43a35068f2a1c4653dc5b1408a929"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:85239b9f93278e130d86c0e6bb455dcb66fc3fd891398b9d45ace8799a871a1e"}, + {file = "pyarrow-15.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b8d43e31ca16aa6e12402fcb1e14352d0d809de70edd185c7650fe80e0769e3"}, + {file = "pyarrow-15.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:fa7cd198280dbd0c988df525e50e35b5d16873e2cdae2aaaa6363cdb64e3eec5"}, + {file = "pyarrow-15.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8780b1a29d3c8b21ba6b191305a2a607de2e30dab399776ff0aa09131e266340"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe0ec198ccc680f6c92723fadcb97b74f07c45ff3fdec9dd765deb04955ccf19"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:036a7209c235588c2f07477fe75c07e6caced9b7b61bb897c8d4e52c4b5f9555"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2bd8a0e5296797faf9a3294e9fa2dc67aa7f10ae2207920dbebb785c77e9dbe5"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e8ebed6053dbe76883a822d4e8da36860f479d55a762bd9e70d8494aed87113e"}, + {file = "pyarrow-15.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:17d53a9d1b2b5bd7d5e4cd84d018e2a45bc9baaa68f7e6e3ebed45649900ba99"}, + {file = "pyarrow-15.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9950a9c9df24090d3d558b43b97753b8f5867fb8e521f29876aa021c52fda351"}, + {file = "pyarrow-15.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:003d680b5e422d0204e7287bb3fa775b332b3fce2996aa69e9adea23f5c8f970"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f75fce89dad10c95f4bf590b765e3ae98bcc5ba9f6ce75adb828a334e26a3d40"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca9cb0039923bec49b4fe23803807e4ef39576a2bec59c32b11296464623dc2"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ed5a78ed29d171d0acc26a305a4b7f83c122d54ff5270810ac23c75813585e4"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6eda9e117f0402dfcd3cd6ec9bfee89ac5071c48fc83a84f3075b60efa96747f"}, + {file = "pyarrow-15.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:9a3a6180c0e8f2727e6f1b1c87c72d3254cac909e609f35f22532e4115461177"}, + {file = "pyarrow-15.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:19a8918045993349b207de72d4576af0191beef03ea655d8bdb13762f0cd6eac"}, + {file = "pyarrow-15.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d0ec076b32bacb6666e8813a22e6e5a7ef1314c8069d4ff345efa6246bc38593"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5db1769e5d0a77eb92344c7382d6543bea1164cca3704f84aa44e26c67e320fb"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2617e3bf9df2a00020dd1c1c6dce5cc343d979efe10bc401c0632b0eef6ef5b"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:d31c1d45060180131caf10f0f698e3a782db333a422038bf7fe01dace18b3a31"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:c8c287d1d479de8269398b34282e206844abb3208224dbdd7166d580804674b7"}, + {file = "pyarrow-15.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:07eb7f07dc9ecbb8dace0f58f009d3a29ee58682fcdc91337dfeb51ea618a75b"}, + {file = "pyarrow-15.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:47af7036f64fce990bb8a5948c04722e4e3ea3e13b1007ef52dfe0aa8f23cf7f"}, + {file = "pyarrow-15.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93768ccfff85cf044c418bfeeafce9a8bb0cee091bd8fd19011aff91e58de540"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6ee87fd6892700960d90abb7b17a72a5abb3b64ee0fe8db6c782bcc2d0dc0b4"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:001fca027738c5f6be0b7a3159cc7ba16a5c52486db18160909a0831b063c4e4"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:d1c48648f64aec09accf44140dccb92f4f94394b8d79976c426a5b79b11d4fa7"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:972a0141be402bb18e3201448c8ae62958c9c7923dfaa3b3d4530c835ac81aed"}, + {file = "pyarrow-15.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:f01fc5cf49081426429127aa2d427d9d98e1cb94a32cb961d583a70b7c4504e6"}, + {file = "pyarrow-15.0.0.tar.gz", hash = "sha256:876858f549d540898f927eba4ef77cd549ad8d24baa3207cf1b72e5788b50e83"}, ] [package.dependencies] -numpy = ">=1.16.6" +numpy = ">=1.16.6,<2" [[package]] name = "pyarrow-hotfix" @@ -1739,13 +1823,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.3" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, - {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] @@ -1845,99 +1929,104 @@ files = [ [[package]] name = "regex" -version = "2023.10.3" +version = "2023.12.25" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.7" files = [ - {file = "regex-2023.10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4c34d4f73ea738223a094d8e0ffd6d2c1a1b4c175da34d6b0de3d8d69bee6bcc"}, - {file = "regex-2023.10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8f4e49fc3ce020f65411432183e6775f24e02dff617281094ba6ab079ef0915"}, - {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cd1bccf99d3ef1ab6ba835308ad85be040e6a11b0977ef7ea8c8005f01a3c29"}, - {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81dce2ddc9f6e8f543d94b05d56e70d03a0774d32f6cca53e978dc01e4fc75b8"}, - {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c6b4d23c04831e3ab61717a707a5d763b300213db49ca680edf8bf13ab5d91b"}, - {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c15ad0aee158a15e17e0495e1e18741573d04eb6da06d8b84af726cfc1ed02ee"}, - {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6239d4e2e0b52c8bd38c51b760cd870069f0bdf99700a62cd509d7a031749a55"}, - {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4a8bf76e3182797c6b1afa5b822d1d5802ff30284abe4599e1247be4fd6b03be"}, - {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9c727bbcf0065cbb20f39d2b4f932f8fa1631c3e01fcedc979bd4f51fe051c5"}, - {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3ccf2716add72f80714b9a63899b67fa711b654be3fcdd34fa391d2d274ce767"}, - {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:107ac60d1bfdc3edb53be75e2a52aff7481b92817cfdddd9b4519ccf0e54a6ff"}, - {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:00ba3c9818e33f1fa974693fb55d24cdc8ebafcb2e4207680669d8f8d7cca79a"}, - {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f0a47efb1dbef13af9c9a54a94a0b814902e547b7f21acb29434504d18f36e3a"}, - {file = "regex-2023.10.3-cp310-cp310-win32.whl", hash = "sha256:36362386b813fa6c9146da6149a001b7bd063dabc4d49522a1f7aa65b725c7ec"}, - {file = "regex-2023.10.3-cp310-cp310-win_amd64.whl", hash = "sha256:c65a3b5330b54103e7d21cac3f6bf3900d46f6d50138d73343d9e5b2900b2353"}, - {file = "regex-2023.10.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90a79bce019c442604662d17bf69df99090e24cdc6ad95b18b6725c2988a490e"}, - {file = "regex-2023.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c7964c2183c3e6cce3f497e3a9f49d182e969f2dc3aeeadfa18945ff7bdd7051"}, - {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ef80829117a8061f974b2fda8ec799717242353bff55f8a29411794d635d964"}, - {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5addc9d0209a9afca5fc070f93b726bf7003bd63a427f65ef797a931782e7edc"}, - {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c148bec483cc4b421562b4bcedb8e28a3b84fcc8f0aa4418e10898f3c2c0eb9b"}, - {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d1f21af4c1539051049796a0f50aa342f9a27cde57318f2fc41ed50b0dbc4ac"}, - {file = "regex-2023.10.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b9ac09853b2a3e0d0082104036579809679e7715671cfbf89d83c1cb2a30f58"}, - {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ebedc192abbc7fd13c5ee800e83a6df252bec691eb2c4bedc9f8b2e2903f5e2a"}, - {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d8a993c0a0ffd5f2d3bda23d0cd75e7086736f8f8268de8a82fbc4bd0ac6791e"}, - {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:be6b7b8d42d3090b6c80793524fa66c57ad7ee3fe9722b258aec6d0672543fd0"}, - {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4023e2efc35a30e66e938de5aef42b520c20e7eda7bb5fb12c35e5d09a4c43f6"}, - {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d47840dc05e0ba04fe2e26f15126de7c755496d5a8aae4a08bda4dd8d646c54"}, - {file = "regex-2023.10.3-cp311-cp311-win32.whl", hash = "sha256:9145f092b5d1977ec8c0ab46e7b3381b2fd069957b9862a43bd383e5c01d18c2"}, - {file = "regex-2023.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:b6104f9a46bd8743e4f738afef69b153c4b8b592d35ae46db07fc28ae3d5fb7c"}, - {file = "regex-2023.10.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff507ae210371d4b1fe316d03433ac099f184d570a1a611e541923f78f05037"}, - {file = "regex-2023.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be5e22bbb67924dea15039c3282fa4cc6cdfbe0cbbd1c0515f9223186fc2ec5f"}, - {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a992f702c9be9c72fa46f01ca6e18d131906a7180950958f766c2aa294d4b41"}, - {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7434a61b158be563c1362d9071358f8ab91b8d928728cd2882af060481244c9e"}, - {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2169b2dcabf4e608416f7f9468737583ce5f0a6e8677c4efbf795ce81109d7c"}, - {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9e908ef5889cda4de038892b9accc36d33d72fb3e12c747e2799a0e806ec841"}, - {file = "regex-2023.10.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12bd4bc2c632742c7ce20db48e0d99afdc05e03f0b4c1af90542e05b809a03d9"}, - {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bc72c231f5449d86d6c7d9cc7cd819b6eb30134bb770b8cfdc0765e48ef9c420"}, - {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bce8814b076f0ce5766dc87d5a056b0e9437b8e0cd351b9a6c4e1134a7dfbda9"}, - {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:ba7cd6dc4d585ea544c1412019921570ebd8a597fabf475acc4528210d7c4a6f"}, - {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b0c7d2f698e83f15228ba41c135501cfe7d5740181d5903e250e47f617eb4292"}, - {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5a8f91c64f390ecee09ff793319f30a0f32492e99f5dc1c72bc361f23ccd0a9a"}, - {file = "regex-2023.10.3-cp312-cp312-win32.whl", hash = "sha256:ad08a69728ff3c79866d729b095872afe1e0557251da4abb2c5faff15a91d19a"}, - {file = "regex-2023.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:39cdf8d141d6d44e8d5a12a8569d5a227f645c87df4f92179bd06e2e2705e76b"}, - {file = "regex-2023.10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4a3ee019a9befe84fa3e917a2dd378807e423d013377a884c1970a3c2792d293"}, - {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76066d7ff61ba6bf3cb5efe2428fc82aac91802844c022d849a1f0f53820502d"}, - {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe50b61bab1b1ec260fa7cd91106fa9fece57e6beba05630afe27c71259c59b"}, - {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fd88f373cb71e6b59b7fa597e47e518282455c2734fd4306a05ca219a1991b0"}, - {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ab05a182c7937fb374f7e946f04fb23a0c0699c0450e9fb02ef567412d2fa3"}, - {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dac37cf08fcf2094159922edc7a2784cfcc5c70f8354469f79ed085f0328ebdf"}, - {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e54ddd0bb8fb626aa1f9ba7b36629564544954fff9669b15da3610c22b9a0991"}, - {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3367007ad1951fde612bf65b0dffc8fd681a4ab98ac86957d16491400d661302"}, - {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:16f8740eb6dbacc7113e3097b0a36065a02e37b47c936b551805d40340fb9971"}, - {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f4f2ca6df64cbdd27f27b34f35adb640b5d2d77264228554e68deda54456eb11"}, - {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:39807cbcbe406efca2a233884e169d056c35aa7e9f343d4e78665246a332f597"}, - {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7eece6fbd3eae4a92d7c748ae825cbc1ee41a89bb1c3db05b5578ed3cfcfd7cb"}, - {file = "regex-2023.10.3-cp37-cp37m-win32.whl", hash = "sha256:ce615c92d90df8373d9e13acddd154152645c0dc060871abf6bd43809673d20a"}, - {file = "regex-2023.10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0f649fa32fe734c4abdfd4edbb8381c74abf5f34bc0b3271ce687b23729299ed"}, - {file = "regex-2023.10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b98b7681a9437262947f41c7fac567c7e1f6eddd94b0483596d320092004533"}, - {file = "regex-2023.10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:91dc1d531f80c862441d7b66c4505cd6ea9d312f01fb2f4654f40c6fdf5cc37a"}, - {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82fcc1f1cc3ff1ab8a57ba619b149b907072e750815c5ba63e7aa2e1163384a4"}, - {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7979b834ec7a33aafae34a90aad9f914c41fd6eaa8474e66953f3f6f7cbd4368"}, - {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef71561f82a89af6cfcbee47f0fabfdb6e63788a9258e913955d89fdd96902ab"}, - {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd829712de97753367153ed84f2de752b86cd1f7a88b55a3a775eb52eafe8a94"}, - {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00e871d83a45eee2f8688d7e6849609c2ca2a04a6d48fba3dff4deef35d14f07"}, - {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:706e7b739fdd17cb89e1fbf712d9dc21311fc2333f6d435eac2d4ee81985098c"}, - {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cc3f1c053b73f20c7ad88b0d1d23be7e7b3901229ce89f5000a8399746a6e039"}, - {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6f85739e80d13644b981a88f529d79c5bdf646b460ba190bffcaf6d57b2a9863"}, - {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:741ba2f511cc9626b7561a440f87d658aabb3d6b744a86a3c025f866b4d19e7f"}, - {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e77c90ab5997e85901da85131fd36acd0ed2221368199b65f0d11bca44549711"}, - {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:979c24cbefaf2420c4e377ecd1f165ea08cc3d1fbb44bdc51bccbbf7c66a2cb4"}, - {file = "regex-2023.10.3-cp38-cp38-win32.whl", hash = "sha256:58837f9d221744d4c92d2cf7201c6acd19623b50c643b56992cbd2b745485d3d"}, - {file = "regex-2023.10.3-cp38-cp38-win_amd64.whl", hash = "sha256:c55853684fe08d4897c37dfc5faeff70607a5f1806c8be148f1695be4a63414b"}, - {file = "regex-2023.10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2c54e23836650bdf2c18222c87f6f840d4943944146ca479858404fedeb9f9af"}, - {file = "regex-2023.10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:69c0771ca5653c7d4b65203cbfc5e66db9375f1078689459fe196fe08b7b4930"}, - {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ac965a998e1388e6ff2e9781f499ad1eaa41e962a40d11c7823c9952c77123e"}, - {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c0e8fae5b27caa34177bdfa5a960c46ff2f78ee2d45c6db15ae3f64ecadde14"}, - {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6c56c3d47da04f921b73ff9415fbaa939f684d47293f071aa9cbb13c94afc17d"}, - {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ef1e014eed78ab650bef9a6a9cbe50b052c0aebe553fb2881e0453717573f52"}, - {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d29338556a59423d9ff7b6eb0cb89ead2b0875e08fe522f3e068b955c3e7b59b"}, - {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9c6d0ced3c06d0f183b73d3c5920727268d2201aa0fe6d55c60d68c792ff3588"}, - {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:994645a46c6a740ee8ce8df7911d4aee458d9b1bc5639bc968226763d07f00fa"}, - {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:66e2fe786ef28da2b28e222c89502b2af984858091675044d93cb50e6f46d7af"}, - {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:11175910f62b2b8c055f2b089e0fedd694fe2be3941b3e2633653bc51064c528"}, - {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:06e9abc0e4c9ab4779c74ad99c3fc10d3967d03114449acc2c2762ad4472b8ca"}, - {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fb02e4257376ae25c6dd95a5aec377f9b18c09be6ebdefa7ad209b9137b73d48"}, - {file = "regex-2023.10.3-cp39-cp39-win32.whl", hash = "sha256:3b2c3502603fab52d7619b882c25a6850b766ebd1b18de3df23b2f939360e1bd"}, - {file = "regex-2023.10.3-cp39-cp39-win_amd64.whl", hash = "sha256:adbccd17dcaff65704c856bd29951c58a1bd4b2b0f8ad6b826dbd543fe740988"}, - {file = "regex-2023.10.3.tar.gz", hash = "sha256:3fef4f844d2290ee0ba57addcec17eec9e3df73f10a2748485dfd6a3a188cc0f"}, + {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0694219a1d54336fd0445ea382d49d36882415c0134ee1e8332afd1529f0baa5"}, + {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b014333bd0217ad3d54c143de9d4b9a3ca1c5a29a6d0d554952ea071cff0f1f8"}, + {file = "regex-2023.12.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d865984b3f71f6d0af64d0d88f5733521698f6c16f445bb09ce746c92c97c586"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e0eabac536b4cc7f57a5f3d095bfa557860ab912f25965e08fe1545e2ed8b4c"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c25a8ad70e716f96e13a637802813f65d8a6760ef48672aa3502f4c24ea8b400"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9b6d73353f777630626f403b0652055ebfe8ff142a44ec2cf18ae470395766e"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9cc99d6946d750eb75827cb53c4371b8b0fe89c733a94b1573c9dd16ea6c9e4"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88d1f7bef20c721359d8675f7d9f8e414ec5003d8f642fdfd8087777ff7f94b5"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cb3fe77aec8f1995611f966d0c656fdce398317f850d0e6e7aebdfe61f40e1cd"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7aa47c2e9ea33a4a2a05f40fcd3ea36d73853a2aae7b4feab6fc85f8bf2c9704"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:df26481f0c7a3f8739fecb3e81bc9da3fcfae34d6c094563b9d4670b047312e1"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c40281f7d70baf6e0db0c2f7472b31609f5bc2748fe7275ea65a0b4601d9b392"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:d94a1db462d5690ebf6ae86d11c5e420042b9898af5dcf278bd97d6bda065423"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba1b30765a55acf15dce3f364e4928b80858fa8f979ad41f862358939bdd1f2f"}, + {file = "regex-2023.12.25-cp310-cp310-win32.whl", hash = "sha256:150c39f5b964e4d7dba46a7962a088fbc91f06e606f023ce57bb347a3b2d4630"}, + {file = "regex-2023.12.25-cp310-cp310-win_amd64.whl", hash = "sha256:09da66917262d9481c719599116c7dc0c321ffcec4b1f510c4f8a066f8768105"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b9d811f72210fa9306aeb88385b8f8bcef0dfbf3873410413c00aa94c56c2b6"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d902a43085a308cef32c0d3aea962524b725403fd9373dea18110904003bac97"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d166eafc19f4718df38887b2bbe1467a4f74a9830e8605089ea7a30dd4da8887"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7ad32824b7f02bb3c9f80306d405a1d9b7bb89362d68b3c5a9be53836caebdb"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:636ba0a77de609d6510235b7f0e77ec494d2657108f777e8765efc060094c98c"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fda75704357805eb953a3ee15a2b240694a9a514548cd49b3c5124b4e2ad01b"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f72cbae7f6b01591f90814250e636065850c5926751af02bb48da94dfced7baa"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db2a0b1857f18b11e3b0e54ddfefc96af46b0896fb678c85f63fb8c37518b3e7"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7502534e55c7c36c0978c91ba6f61703faf7ce733715ca48f499d3dbbd7657e0"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e8c7e08bb566de4faaf11984af13f6bcf6a08f327b13631d41d62592681d24fe"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:283fc8eed679758de38fe493b7d7d84a198b558942b03f017b1f94dda8efae80"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f44dd4d68697559d007462b0a3a1d9acd61d97072b71f6d1968daef26bc744bd"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:67d3ccfc590e5e7197750fcb3a2915b416a53e2de847a728cfa60141054123d4"}, + {file = "regex-2023.12.25-cp311-cp311-win32.whl", hash = "sha256:68191f80a9bad283432385961d9efe09d783bcd36ed35a60fb1ff3f1ec2efe87"}, + {file = "regex-2023.12.25-cp311-cp311-win_amd64.whl", hash = "sha256:7d2af3f6b8419661a0c421584cfe8aaec1c0e435ce7e47ee2a97e344b98f794f"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8a0ccf52bb37d1a700375a6b395bff5dd15c50acb745f7db30415bae3c2b0715"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c3c4a78615b7762740531c27cf46e2f388d8d727d0c0c739e72048beb26c8a9d"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad83e7545b4ab69216cef4cc47e344d19622e28aabec61574b20257c65466d6a"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7a635871143661feccce3979e1727c4e094f2bdfd3ec4b90dfd4f16f571a87a"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d498eea3f581fbe1b34b59c697512a8baef88212f92e4c7830fcc1499f5b45a5"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43f7cd5754d02a56ae4ebb91b33461dc67be8e3e0153f593c509e21d219c5060"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51f4b32f793812714fd5307222a7f77e739b9bc566dc94a18126aba3b92b98a3"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba99d8077424501b9616b43a2d208095746fb1284fc5ba490139651f971d39d9"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4bfc2b16e3ba8850e0e262467275dd4d62f0d045e0e9eda2bc65078c0110a11f"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8c2c19dae8a3eb0ea45a8448356ed561be843b13cbc34b840922ddf565498c1c"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:60080bb3d8617d96f0fb7e19796384cc2467447ef1c491694850ebd3670bc457"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b77e27b79448e34c2c51c09836033056a0547aa360c45eeeb67803da7b0eedaf"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:518440c991f514331f4850a63560321f833979d145d7d81186dbe2f19e27ae3d"}, + {file = "regex-2023.12.25-cp312-cp312-win32.whl", hash = "sha256:e2610e9406d3b0073636a3a2e80db05a02f0c3169b5632022b4e81c0364bcda5"}, + {file = "regex-2023.12.25-cp312-cp312-win_amd64.whl", hash = "sha256:cc37b9aeebab425f11f27e5e9e6cf580be7206c6582a64467a14dda211abc232"}, + {file = "regex-2023.12.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:da695d75ac97cb1cd725adac136d25ca687da4536154cdc2815f576e4da11c69"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d126361607b33c4eb7b36debc173bf25d7805847346dd4d99b5499e1fef52bc7"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4719bb05094d7d8563a450cf8738d2e1061420f79cfcc1fa7f0a44744c4d8f73"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5dd58946bce44b53b06d94aa95560d0b243eb2fe64227cba50017a8d8b3cd3e2"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22a86d9fff2009302c440b9d799ef2fe322416d2d58fc124b926aa89365ec482"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2aae8101919e8aa05ecfe6322b278f41ce2994c4a430303c4cd163fef746e04f"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e692296c4cc2873967771345a876bcfc1c547e8dd695c6b89342488b0ea55cd8"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:263ef5cc10979837f243950637fffb06e8daed7f1ac1e39d5910fd29929e489a"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d6f7e255e5fa94642a0724e35406e6cb7001c09d476ab5fce002f652b36d0c39"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:88ad44e220e22b63b0f8f81f007e8abbb92874d8ced66f32571ef8beb0643b2b"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:3a17d3ede18f9cedcbe23d2daa8a2cd6f59fe2bf082c567e43083bba3fb00347"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d15b274f9e15b1a0b7a45d2ac86d1f634d983ca40d6b886721626c47a400bf39"}, + {file = "regex-2023.12.25-cp37-cp37m-win32.whl", hash = "sha256:ed19b3a05ae0c97dd8f75a5d8f21f7723a8c33bbc555da6bbe1f96c470139d3c"}, + {file = "regex-2023.12.25-cp37-cp37m-win_amd64.whl", hash = "sha256:a6d1047952c0b8104a1d371f88f4ab62e6275567d4458c1e26e9627ad489b445"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b43523d7bc2abd757119dbfb38af91b5735eea45537ec6ec3a5ec3f9562a1c53"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:efb2d82f33b2212898f1659fb1c2e9ac30493ac41e4d53123da374c3b5541e64"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b7fca9205b59c1a3d5031f7e64ed627a1074730a51c2a80e97653e3e9fa0d415"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086dd15e9435b393ae06f96ab69ab2d333f5d65cbe65ca5a3ef0ec9564dfe770"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e81469f7d01efed9b53740aedd26085f20d49da65f9c1f41e822a33992cb1590"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:34e4af5b27232f68042aa40a91c3b9bb4da0eeb31b7632e0091afc4310afe6cb"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9852b76ab558e45b20bf1893b59af64a28bd3820b0c2efc80e0a70a4a3ea51c1"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff100b203092af77d1a5a7abe085b3506b7eaaf9abf65b73b7d6905b6cb76988"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cc038b2d8b1470364b1888a98fd22d616fba2b6309c5b5f181ad4483e0017861"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:094ba386bb5c01e54e14434d4caabf6583334090865b23ef58e0424a6286d3dc"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5cd05d0f57846d8ba4b71d9c00f6f37d6b97d5e5ef8b3c3840426a475c8f70f4"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:9aa1a67bbf0f957bbe096375887b2505f5d8ae16bf04488e8b0f334c36e31360"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:98a2636994f943b871786c9e82bfe7883ecdaba2ef5df54e1450fa9869d1f756"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37f8e93a81fc5e5bd8db7e10e62dc64261bcd88f8d7e6640aaebe9bc180d9ce2"}, + {file = "regex-2023.12.25-cp38-cp38-win32.whl", hash = "sha256:d78bd484930c1da2b9679290a41cdb25cc127d783768a0369d6b449e72f88beb"}, + {file = "regex-2023.12.25-cp38-cp38-win_amd64.whl", hash = "sha256:b521dcecebc5b978b447f0f69b5b7f3840eac454862270406a39837ffae4e697"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f7bc09bc9c29ebead055bcba136a67378f03d66bf359e87d0f7c759d6d4ffa31"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e14b73607d6231f3cc4622809c196b540a6a44e903bcfad940779c80dffa7be7"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9eda5f7a50141291beda3edd00abc2d4a5b16c29c92daf8d5bd76934150f3edc"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc6bb9aa69aacf0f6032c307da718f61a40cf970849e471254e0e91c56ffca95"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:298dc6354d414bc921581be85695d18912bea163a8b23cac9a2562bbcd5088b1"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f4e475a80ecbd15896a976aa0b386c5525d0ed34d5c600b6d3ebac0a67c7ddf"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:531ac6cf22b53e0696f8e1d56ce2396311254eb806111ddd3922c9d937151dae"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22f3470f7524b6da61e2020672df2f3063676aff444db1daa283c2ea4ed259d6"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:89723d2112697feaa320c9d351e5f5e7b841e83f8b143dba8e2d2b5f04e10923"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0ecf44ddf9171cd7566ef1768047f6e66975788258b1c6c6ca78098b95cf9a3d"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:905466ad1702ed4acfd67a902af50b8db1feeb9781436372261808df7a2a7bca"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:4558410b7a5607a645e9804a3e9dd509af12fb72b9825b13791a37cd417d73a5"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:7e316026cc1095f2a3e8cc012822c99f413b702eaa2ca5408a513609488cb62f"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3b1de218d5375cd6ac4b5493e0b9f3df2be331e86520f23382f216c137913d20"}, + {file = "regex-2023.12.25-cp39-cp39-win32.whl", hash = "sha256:11a963f8e25ab5c61348d090bf1b07f1953929c13bd2309a0662e9ff680763c9"}, + {file = "regex-2023.12.25-cp39-cp39-win_amd64.whl", hash = "sha256:e693e233ac92ba83a87024e1d32b5f9ab15ca55ddd916d878146f4e3406b5c91"}, + {file = "regex-2023.12.25.tar.gz", hash = "sha256:29171aa128da69afdf4bde412d5bedc335f2ca8fcfe4489038577d05f16181e5"}, ] [[package]] @@ -2042,45 +2131,45 @@ torch = ["numpy (>=1.21.6)", "torch (>=1.10)"] [[package]] name = "scipy" -version = "1.11.4" +version = "1.12.0" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "scipy-1.11.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc9a714581f561af0848e6b69947fda0614915f072dfd14142ed1bfe1b806710"}, - {file = "scipy-1.11.4-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:cf00bd2b1b0211888d4dc75656c0412213a8b25e80d73898083f402b50f47e41"}, - {file = "scipy-1.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9999c008ccf00e8fbcce1236f85ade5c569d13144f77a1946bef8863e8f6eb4"}, - {file = "scipy-1.11.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:933baf588daa8dc9a92c20a0be32f56d43faf3d1a60ab11b3f08c356430f6e56"}, - {file = "scipy-1.11.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8fce70f39076a5aa62e92e69a7f62349f9574d8405c0a5de6ed3ef72de07f446"}, - {file = "scipy-1.11.4-cp310-cp310-win_amd64.whl", hash = "sha256:6550466fbeec7453d7465e74d4f4b19f905642c89a7525571ee91dd7adabb5a3"}, - {file = "scipy-1.11.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be"}, - {file = "scipy-1.11.4-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:1b7c3dca977f30a739e0409fb001056484661cb2541a01aba0bb0029f7b68db8"}, - {file = "scipy-1.11.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c"}, - {file = "scipy-1.11.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:530f9ad26440e85766509dbf78edcfe13ffd0ab7fec2560ee5c36ff74d6269ff"}, - {file = "scipy-1.11.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5e347b14fe01003d3b78e196e84bd3f48ffe4c8a7b8a1afbcb8f5505cb710993"}, - {file = "scipy-1.11.4-cp311-cp311-win_amd64.whl", hash = "sha256:acf8ed278cc03f5aff035e69cb511741e0418681d25fbbb86ca65429c4f4d9cd"}, - {file = "scipy-1.11.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:028eccd22e654b3ea01ee63705681ee79933652b2d8f873e7949898dda6d11b6"}, - {file = "scipy-1.11.4-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2c6ff6ef9cc27f9b3db93a6f8b38f97387e6e0591600369a297a50a8e96e835d"}, - {file = "scipy-1.11.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b030c6674b9230d37c5c60ab456e2cf12f6784596d15ce8da9365e70896effc4"}, - {file = "scipy-1.11.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad669df80528aeca5f557712102538f4f37e503f0c5b9541655016dd0932ca79"}, - {file = "scipy-1.11.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ce7fff2e23ab2cc81ff452a9444c215c28e6305f396b2ba88343a567feec9660"}, - {file = "scipy-1.11.4-cp312-cp312-win_amd64.whl", hash = "sha256:36750b7733d960d7994888f0d148d31ea3017ac15eef664194b4ef68d36a4a97"}, - {file = "scipy-1.11.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e619aba2df228a9b34718efb023966da781e89dd3d21637b27f2e54db0410d7"}, - {file = "scipy-1.11.4-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec"}, - {file = "scipy-1.11.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d10e45a6c50211fe256da61a11c34927c68f277e03138777bdebedd933712fea"}, - {file = "scipy-1.11.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91af76a68eeae0064887a48e25c4e616fa519fa0d38602eda7e0f97d65d57937"}, - {file = "scipy-1.11.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6df1468153a31cf55ed5ed39647279beb9cfb5d3f84369453b49e4b8502394fd"}, - {file = "scipy-1.11.4-cp39-cp39-win_amd64.whl", hash = "sha256:ee410e6de8f88fd5cf6eadd73c135020bfbbbdfcd0f6162c36a7638a1ea8cc65"}, - {file = "scipy-1.11.4.tar.gz", hash = "sha256:90a2b78e7f5733b9de748f589f09225013685f9b218275257f8a8168ededaeaa"}, + {file = "scipy-1.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:78e4402e140879387187f7f25d91cc592b3501a2e51dfb320f48dfb73565f10b"}, + {file = "scipy-1.12.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:f5f00ebaf8de24d14b8449981a2842d404152774c1a1d880c901bf454cb8e2a1"}, + {file = "scipy-1.12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e53958531a7c695ff66c2e7bb7b79560ffdc562e2051644c5576c39ff8efb563"}, + {file = "scipy-1.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e32847e08da8d895ce09d108a494d9eb78974cf6de23063f93306a3e419960c"}, + {file = "scipy-1.12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4c1020cad92772bf44b8e4cdabc1df5d87376cb219742549ef69fc9fd86282dd"}, + {file = "scipy-1.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:75ea2a144096b5e39402e2ff53a36fecfd3b960d786b7efd3c180e29c39e53f2"}, + {file = "scipy-1.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:408c68423f9de16cb9e602528be4ce0d6312b05001f3de61fe9ec8b1263cad08"}, + {file = "scipy-1.12.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5adfad5dbf0163397beb4aca679187d24aec085343755fcdbdeb32b3679f254c"}, + {file = "scipy-1.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3003652496f6e7c387b1cf63f4bb720951cfa18907e998ea551e6de51a04467"}, + {file = "scipy-1.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b8066bce124ee5531d12a74b617d9ac0ea59245246410e19bca549656d9a40a"}, + {file = "scipy-1.12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8bee4993817e204d761dba10dbab0774ba5a8612e57e81319ea04d84945375ba"}, + {file = "scipy-1.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:a24024d45ce9a675c1fb8494e8e5244efea1c7a09c60beb1eeb80373d0fecc70"}, + {file = "scipy-1.12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e7e76cc48638228212c747ada851ef355c2bb5e7f939e10952bc504c11f4e372"}, + {file = "scipy-1.12.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f7ce148dffcd64ade37b2df9315541f9adad6efcaa86866ee7dd5db0c8f041c3"}, + {file = "scipy-1.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c39f92041f490422924dfdb782527a4abddf4707616e07b021de33467f917bc"}, + {file = "scipy-1.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7ebda398f86e56178c2fa94cad15bf457a218a54a35c2a7b4490b9f9cb2676c"}, + {file = "scipy-1.12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:95e5c750d55cf518c398a8240571b0e0782c2d5a703250872f36eaf737751338"}, + {file = "scipy-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:e646d8571804a304e1da01040d21577685ce8e2db08ac58e543eaca063453e1c"}, + {file = "scipy-1.12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:913d6e7956c3a671de3b05ccb66b11bc293f56bfdef040583a7221d9e22a2e35"}, + {file = "scipy-1.12.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba1b0c7256ad75401c73e4b3cf09d1f176e9bd4248f0d3112170fb2ec4db067"}, + {file = "scipy-1.12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:730badef9b827b368f351eacae2e82da414e13cf8bd5051b4bdfd720271a5371"}, + {file = "scipy-1.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6546dc2c11a9df6926afcbdd8a3edec28566e4e785b915e849348c6dd9f3f490"}, + {file = "scipy-1.12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:196ebad3a4882081f62a5bf4aeb7326aa34b110e533aab23e4374fcccb0890dc"}, + {file = "scipy-1.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:b360f1b6b2f742781299514e99ff560d1fe9bd1bff2712894b52abe528d1fd1e"}, + {file = "scipy-1.12.0.tar.gz", hash = "sha256:4bf5abab8a36d20193c698b0f1fc282c1d083c94723902c447e5d2f1780936a3"}, ] [package.dependencies] -numpy = ">=1.21.6,<1.28.0" +numpy = ">=1.22.4,<1.29.0" [package.extras] dev = ["click", "cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pydevtool", "rich-click", "ruff", "types-psutil", "typing_extensions"] doc = ["jupytext", "matplotlib (>2)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-design (>=0.2.0)"] -test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +test = ["asv", "gmpy2", "hypothesis", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] [[package]] name = "sentencepiece" @@ -2138,13 +2227,13 @@ files = [ [[package]] name = "setuptools" -version = "69.0.2" +version = "69.0.3" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.0.2-py3-none-any.whl", hash = "sha256:1e8fdff6797d3865f37397be788a4e3cba233608e9b509382a2777d25ebde7f2"}, - {file = "setuptools-69.0.2.tar.gz", hash = "sha256:735896e78a4742605974de002ac60562d286fa8051a7e2299445e8e8fbb01aa6"}, + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, ] [package.extras] @@ -2190,109 +2279,121 @@ files = [ [[package]] name = "tokenizers" -version = "0.15.0" +version = "0.15.1" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "tokenizers-0.15.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:cd3cd0299aaa312cd2988957598f80becd04d5a07338741eca076057a2b37d6e"}, - {file = "tokenizers-0.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a922c492c721744ee175f15b91704be2d305569d25f0547c77cd6c9f210f9dc"}, - {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:331dd786d02fc38698f835fff61c99480f98b73ce75a4c65bd110c9af5e4609a"}, - {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88dd0961c437d413ab027f8b115350c121d49902cfbadf08bb8f634b15fa1814"}, - {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6fdcc55339df7761cd52e1fbe8185d3b3963bc9e3f3545faa6c84f9e8818259a"}, - {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1480b0051d8ab5408e8e4db2dc832f7082ea24aa0722c427bde2418c6f3bd07"}, - {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9855e6c258918f9cf62792d4f6ddfa6c56dccd8c8118640f867f6393ecaf8bd7"}, - {file = "tokenizers-0.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de9529fe75efcd54ba8d516aa725e1851df9199f0669b665c55e90df08f5af86"}, - {file = "tokenizers-0.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8edcc90a36eab0705fe9121d6c77c6e42eeef25c7399864fd57dfb27173060bf"}, - {file = "tokenizers-0.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ae17884aafb3e94f34fb7cfedc29054f5f54e142475ebf8a265a4e388fee3f8b"}, - {file = "tokenizers-0.15.0-cp310-none-win32.whl", hash = "sha256:9a3241acdc9b44cff6e95c4a55b9be943ef3658f8edb3686034d353734adba05"}, - {file = "tokenizers-0.15.0-cp310-none-win_amd64.whl", hash = "sha256:4b31807cb393d6ea31926b307911c89a1209d5e27629aa79553d1599c8ffdefe"}, - {file = "tokenizers-0.15.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:af7e9be8c05d30bb137b9fd20f9d99354816599e5fd3d58a4b1e28ba3b36171f"}, - {file = "tokenizers-0.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c3d7343fa562ea29661783344a2d83662db0d3d17a6fa6a403cac8e512d2d9fd"}, - {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:32371008788aeeb0309a9244809a23e4c0259625e6b74a103700f6421373f395"}, - {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9db64c7c9954fbae698884c5bb089764edc549731e5f9b7fa1dd4e4d78d77f"}, - {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dbed5944c31195514669cf6381a0d8d47f164943000d10f93d6d02f0d45c25e0"}, - {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aab16c4a26d351d63e965b0c792f5da7227a37b69a6dc6d922ff70aa595b1b0c"}, - {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3c2b60b12fdd310bf85ce5d7d3f823456b9b65eed30f5438dd7761879c495983"}, - {file = "tokenizers-0.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0344d6602740e44054a9e5bbe9775a5e149c4dddaff15959bb07dcce95a5a859"}, - {file = "tokenizers-0.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4525f6997d81d9b6d9140088f4f5131f6627e4c960c2c87d0695ae7304233fc3"}, - {file = "tokenizers-0.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:65975094fef8cc68919644936764efd2ce98cf1bacbe8db2687155d2b0625bee"}, - {file = "tokenizers-0.15.0-cp311-none-win32.whl", hash = "sha256:ff5d2159c5d93015f5a4542aac6c315506df31853123aa39042672031768c301"}, - {file = "tokenizers-0.15.0-cp311-none-win_amd64.whl", hash = "sha256:2dd681b53cf615e60a31a115a3fda3980e543d25ca183797f797a6c3600788a3"}, - {file = "tokenizers-0.15.0-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:c9cce6ee149a3d703f86877bc2a6d997e34874b2d5a2d7839e36b2273f31d3d9"}, - {file = "tokenizers-0.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a0a94bc3370e6f1cc8a07a8ae867ce13b7c1b4291432a773931a61f256d44ea"}, - {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:309cfcccfc7e502cb1f1de2c9c1c94680082a65bfd3a912d5a5b2c90c677eb60"}, - {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8413e994dd7d875ab13009127fc85633916c71213917daf64962bafd488f15dc"}, - {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0ebf9430f901dbdc3dcb06b493ff24a3644c9f88c08e6a1d6d0ae2228b9b818"}, - {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10361e9c7864b22dd791ec5126327f6c9292fb1d23481d4895780688d5e298ac"}, - {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:babe42635b8a604c594bdc56d205755f73414fce17ba8479d142a963a6c25cbc"}, - {file = "tokenizers-0.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3768829861e964c7a4556f5f23307fce6a23872c2ebf030eb9822dbbbf7e9b2a"}, - {file = "tokenizers-0.15.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9c91588a630adc88065e1c03ac6831e3e2112558869b9ebcb2b8afd8a14c944d"}, - {file = "tokenizers-0.15.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:77606994e793ca54ecf3a3619adc8a906a28ca223d9354b38df41cb8766a0ed6"}, - {file = "tokenizers-0.15.0-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:6fe143939f3b596681922b2df12a591a5b010e7dcfbee2202482cd0c1c2f2459"}, - {file = "tokenizers-0.15.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:b7bee0f1795e3e3561e9a557061b1539e5255b8221e3f928f58100282407e090"}, - {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5d37e7f4439b4c46192ab4f2ff38ab815e4420f153caa13dec9272ef14403d34"}, - {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caadf255cf7f951b38d10097836d1f3bcff4aeaaffadfdf748bab780bf5bff95"}, - {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05accb9162bf711a941b1460b743d62fec61c160daf25e53c5eea52c74d77814"}, - {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26a2ef890740127cb115ee5260878f4a677e36a12831795fd7e85887c53b430b"}, - {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e54c5f26df14913620046b33e822cb3bcd091a332a55230c0e63cc77135e2169"}, - {file = "tokenizers-0.15.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669b8ed653a578bcff919566631156f5da3aab84c66f3c0b11a6281e8b4731c7"}, - {file = "tokenizers-0.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0ea480d943297df26f06f508dab6e012b07f42bf3dffdd36e70799368a5f5229"}, - {file = "tokenizers-0.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc80a0a565ebfc7cd89de7dd581da8c2b3238addfca6280572d27d763f135f2f"}, - {file = "tokenizers-0.15.0-cp37-none-win32.whl", hash = "sha256:cdd945e678bbdf4517d5d8de66578a5030aeefecdb46f5320b034de9cad8d4dd"}, - {file = "tokenizers-0.15.0-cp37-none-win_amd64.whl", hash = "sha256:1ab96ab7dc706e002c32b2ea211a94c1c04b4f4de48354728c3a6e22401af322"}, - {file = "tokenizers-0.15.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:f21c9eb71c9a671e2a42f18b456a3d118e50c7f0fc4dd9fa8f4eb727fea529bf"}, - {file = "tokenizers-0.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a5f4543a35889679fc3052086e69e81880b2a5a28ff2a52c5a604be94b77a3f"}, - {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f8aa81afec893e952bd39692b2d9ef60575ed8c86fce1fd876a06d2e73e82dca"}, - {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1574a5a4af22c3def93fe8fe4adcc90a39bf5797ed01686a4c46d1c3bc677d2f"}, - {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c7982fd0ec9e9122d03b209dac48cebfea3de0479335100ef379a9a959b9a5a"}, - {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d16b647032df2ce2c1f9097236e046ea9fedd969b25637b9d5d734d78aa53b"}, - {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b3cdf29e6f9653da330515dc8fa414be5a93aae79e57f8acc50d4028dd843edf"}, - {file = "tokenizers-0.15.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7286f3df10de840867372e3e64b99ef58c677210e3ceb653cd0e740a5c53fe78"}, - {file = "tokenizers-0.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aabc83028baa5a36ce7a94e7659250f0309c47fa4a639e5c2c38e6d5ea0de564"}, - {file = "tokenizers-0.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:72f78b0e0e276b1fc14a672fa73f3acca034ba8db4e782124a2996734a9ba9cf"}, - {file = "tokenizers-0.15.0-cp38-none-win32.whl", hash = "sha256:9680b0ecc26e7e42f16680c1aa62e924d58d1c2dd992707081cc10a374896ea2"}, - {file = "tokenizers-0.15.0-cp38-none-win_amd64.whl", hash = "sha256:f17cbd88dab695911cbdd385a5a7e3709cc61dff982351f5d1b5939f074a2466"}, - {file = "tokenizers-0.15.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:3661862df7382c5eb23ac4fbf7c75e69b02dc4f5784e4c5a734db406b5b24596"}, - {file = "tokenizers-0.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3045d191dad49647f5a5039738ecf1c77087945c7a295f7bcf051c37067e883"}, - {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a9fcaad9ab0801f14457d7c820d9f246b5ab590c407fc6b073819b1573097aa7"}, - {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79f17027f24fe9485701c8dbb269b9c713954ec3bdc1e7075a66086c0c0cd3c"}, - {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:01a3aa332abc4bee7640563949fcfedca4de8f52691b3b70f2fc6ca71bfc0f4e"}, - {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:05b83896a893cdfedad8785250daa3ba9f0504848323471524d4783d7291661e"}, - {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbbf2489fcf25d809731ba2744ff278dd07d9eb3f8b7482726bd6cae607073a4"}, - {file = "tokenizers-0.15.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab806ad521a5e9de38078b7add97589c313915f6f5fec6b2f9f289d14d607bd6"}, - {file = "tokenizers-0.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a522612d5c88a41563e3463226af64e2fa00629f65cdcc501d1995dd25d23f5"}, - {file = "tokenizers-0.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e58a38c4e6075810bdfb861d9c005236a72a152ebc7005941cc90d1bbf16aca9"}, - {file = "tokenizers-0.15.0-cp39-none-win32.whl", hash = "sha256:b8034f1041fd2bd2b84ff9f4dc4ae2e1c3b71606820a9cd5c562ebd291a396d1"}, - {file = "tokenizers-0.15.0-cp39-none-win_amd64.whl", hash = "sha256:edde9aa964145d528d0e0dbf14f244b8a85ebf276fb76869bc02e2530fa37a96"}, - {file = "tokenizers-0.15.0-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:309445d10d442b7521b98083dc9f0b5df14eca69dbbfebeb98d781ee2cef5d30"}, - {file = "tokenizers-0.15.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d3125a6499226d4d48efc54f7498886b94c418e93a205b673bc59364eecf0804"}, - {file = "tokenizers-0.15.0-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ed56ddf0d54877bb9c6d885177db79b41576e61b5ef6defeb579dcb803c04ad5"}, - {file = "tokenizers-0.15.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b22cd714706cc5b18992a232b023f736e539495f5cc61d2d28d176e55046f6c"}, - {file = "tokenizers-0.15.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fac2719b1e9bc8e8e7f6599b99d0a8e24f33d023eb8ef644c0366a596f0aa926"}, - {file = "tokenizers-0.15.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:85ddae17570ec7e5bfaf51ffa78d044f444a8693e1316e1087ee6150596897ee"}, - {file = "tokenizers-0.15.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:76f1bed992e396bf6f83e3df97b64ff47885e45e8365f8983afed8556a0bc51f"}, - {file = "tokenizers-0.15.0-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:3bb0f4df6dce41a1c7482087b60d18c372ef4463cb99aa8195100fcd41e0fd64"}, - {file = "tokenizers-0.15.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:22c27672c27a059a5f39ff4e49feed8c7f2e1525577c8a7e3978bd428eb5869d"}, - {file = "tokenizers-0.15.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78104f5d035c9991f92831fc0efe9e64a05d4032194f2a69f67aaa05a4d75bbb"}, - {file = "tokenizers-0.15.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a40b73dc19d82c3e3ffb40abdaacca8fbc95eeb26c66b7f9f860aebc07a73998"}, - {file = "tokenizers-0.15.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d801d1368188c74552cd779b1286e67cb9fd96f4c57a9f9a2a09b6def9e1ab37"}, - {file = "tokenizers-0.15.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82641ffb13a4da1293fcc9f437d457647e60ed0385a9216cd135953778b3f0a1"}, - {file = "tokenizers-0.15.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:160f9d1810f2c18fffa94aa98bf17632f6bd2dabc67fcb01a698ca80c37d52ee"}, - {file = "tokenizers-0.15.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:8d7d6eea831ed435fdeeb9bcd26476226401d7309d115a710c65da4088841948"}, - {file = "tokenizers-0.15.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f6456bec6c557d63d8ec0023758c32f589e1889ed03c055702e84ce275488bed"}, - {file = "tokenizers-0.15.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1eef39a502fad3bf104b9e1906b4fb0cee20e44e755e51df9a98f8922c3bf6d4"}, - {file = "tokenizers-0.15.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1e4664c5b797e093c19b794bbecc19d2367e782b4a577d8b7c1821db5dc150d"}, - {file = "tokenizers-0.15.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:ca003fb5f3995ff5cf676db6681b8ea5d54d3b30bea36af1120e78ee1a4a4cdf"}, - {file = "tokenizers-0.15.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7f17363141eb0c53752c89e10650b85ef059a52765d0802ba9613dbd2d21d425"}, - {file = "tokenizers-0.15.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:8a765db05581c7d7e1280170f2888cda351760d196cc059c37ea96f121125799"}, - {file = "tokenizers-0.15.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:2a0dd641a72604486cd7302dd8f87a12c8a9b45e1755e47d2682733f097c1af5"}, - {file = "tokenizers-0.15.0-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0a1a3c973e4dc97797fc19e9f11546c95278ffc55c4492acb742f69e035490bc"}, - {file = "tokenizers-0.15.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4fab75642aae4e604e729d6f78e0addb9d7e7d49e28c8f4d16b24da278e5263"}, - {file = "tokenizers-0.15.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65f80be77f6327a86d8fd35a4467adcfe6174c159b4ab52a1a8dd4c6f2d7d9e1"}, - {file = "tokenizers-0.15.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a8da7533dbe66b88afd430c56a2f2ce1fd82e2681868f857da38eeb3191d7498"}, - {file = "tokenizers-0.15.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:fa8eb4584fc6cbe6a84d7a7864be3ed28e23e9fd2146aa8ef1814d579df91958"}, - {file = "tokenizers-0.15.0.tar.gz", hash = "sha256:10c7e6e7b4cabd757da59e93f5f8d1126291d16f8b54f28510825ef56a3e5d0e"}, + {file = "tokenizers-0.15.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:32c9491dd1bcb33172c26b454dbd607276af959b9e78fa766e2694cafab3103c"}, + {file = "tokenizers-0.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29a1b784b870a097e7768f8c20c2dd851e2c75dad3efdae69a79d3e7f1d614d5"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0049fbe648af04148b08cb211994ce8365ee628ce49724b56aaefd09a3007a78"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e84b3c235219e75e24de6b71e6073cd2c8d740b14d88e4c6d131b90134e3a338"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8cc575769ea11d074308c6d71cb10b036cdaec941562c07fc7431d956c502f0e"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22bf28f299c4158e6d0b5eaebddfd500c4973d947ffeaca8bcbe2e8c137dff0b"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:506555f98361db9c74e1323a862d77dcd7d64c2058829a368bf4159d986e339f"}, + {file = "tokenizers-0.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7061b0a28ade15906f5b2ec8c48d3bdd6e24eca6b427979af34954fbe31d5cef"}, + {file = "tokenizers-0.15.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ed5e35507b7a0e2aac3285c4f5e37d4ec5cfc0e5825b862b68a0aaf2757af52"}, + {file = "tokenizers-0.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1c9df9247df0de6509dd751b1c086e5f124b220133b5c883bb691cb6fb3d786f"}, + {file = "tokenizers-0.15.1-cp310-none-win32.whl", hash = "sha256:dd999af1b4848bef1b11d289f04edaf189c269d5e6afa7a95fa1058644c3f021"}, + {file = "tokenizers-0.15.1-cp310-none-win_amd64.whl", hash = "sha256:39d06a57f7c06940d602fad98702cf7024c4eee7f6b9fe76b9f2197d5a4cc7e2"}, + {file = "tokenizers-0.15.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8ad034eb48bf728af06915e9294871f72fcc5254911eddec81d6df8dba1ce055"}, + {file = "tokenizers-0.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea9ede7c42f8fa90f31bfc40376fd91a7d83a4aa6ad38e6076de961d48585b26"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b85d6fe1a20d903877aa0ef32ef6b96e81e0e48b71c206d6046ce16094de6970"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a7d44f656320137c7d643b9c7dcc1814763385de737fb98fd2643880910f597"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd244bd0793cdacf27ee65ec3db88c21f5815460e8872bbeb32b040469d6774e"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0f3f4a36e371b3cb1123adac8aeeeeab207ad32f15ed686d9d71686a093bb140"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2921a53966afb29444da98d56a6ccbef23feb3b0c0f294b4e502370a0a64f25"}, + {file = "tokenizers-0.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f49068cf51f49c231067f1a8c9fc075ff960573f6b2a956e8e1b0154fb638ea5"}, + {file = "tokenizers-0.15.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0ab1a22f20eaaab832ab3b00a0709ca44a0eb04721e580277579411b622c741c"}, + {file = "tokenizers-0.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:671268f24b607c4adc6fa2b5b580fd4211b9f84b16bd7f46d62f8e5be0aa7ba4"}, + {file = "tokenizers-0.15.1-cp311-none-win32.whl", hash = "sha256:a4f03e33d2bf7df39c8894032aba599bf90f6f6378e683a19d28871f09bb07fc"}, + {file = "tokenizers-0.15.1-cp311-none-win_amd64.whl", hash = "sha256:30f689537bcc7576d8bd4daeeaa2cb8f36446ba2f13f421b173e88f2d8289c4e"}, + {file = "tokenizers-0.15.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f3a379dd0898a82ea3125e8f9c481373f73bffce6430d4315f0b6cd5547e409"}, + {file = "tokenizers-0.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d870ae58bba347d38ac3fc8b1f662f51e9c95272d776dd89f30035c83ee0a4f"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d6d28e0143ec2e253a8a39e94bf1d24776dbe73804fa748675dbffff4a5cd6d8"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61ae9ac9f44e2da128ee35db69489883b522f7abe033733fa54eb2de30dac23d"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d8e322a47e29128300b3f7749a03c0ec2bce0a3dc8539ebff738d3f59e233542"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:760334f475443bc13907b1a8e1cb0aeaf88aae489062546f9704dce6c498bfe2"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1b173753d4aca1e7d0d4cb52b5e3ffecfb0ca014e070e40391b6bb4c1d6af3f2"}, + {file = "tokenizers-0.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82c1f13d457c8f0ab17e32e787d03470067fe8a3b4d012e7cc57cb3264529f4a"}, + {file = "tokenizers-0.15.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:425b46ceff4505f20191df54b50ac818055d9d55023d58ae32a5d895b6f15bb0"}, + {file = "tokenizers-0.15.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:681ac6ba3b4fdaf868ead8971221a061f580961c386e9732ea54d46c7b72f286"}, + {file = "tokenizers-0.15.1-cp312-none-win32.whl", hash = "sha256:f2272656063ccfba2044df2115095223960d80525d208e7a32f6c01c351a6f4a"}, + {file = "tokenizers-0.15.1-cp312-none-win_amd64.whl", hash = "sha256:9abe103203b1c6a2435d248d5ff4cceebcf46771bfbc4957a98a74da6ed37674"}, + {file = "tokenizers-0.15.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:2ce9ed5c8ef26b026a66110e3c7b73d93ec2d26a0b1d0ea55ddce61c0e5f446f"}, + {file = "tokenizers-0.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:89b24d366137986c3647baac29ef902d2d5445003d11c30df52f1bd304689aeb"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0faebedd01b413ab777ca0ee85914ed8b031ea5762ab0ea60b707ce8b9be6842"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdbd9dfcdad4f3b95d801f768e143165165055c18e44ca79a8a26de889cd8e85"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:97194324c12565b07e9993ca9aa813b939541185682e859fb45bb8d7d99b3193"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:485e43e2cc159580e0d83fc919ec3a45ae279097f634b1ffe371869ffda5802c"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:191d084d60e3589d6420caeb3f9966168269315f8ec7fbc3883122dc9d99759d"}, + {file = "tokenizers-0.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01c28cc8d7220634a75b14c53f4fc9d1b485f99a5a29306a999c115921de2897"}, + {file = "tokenizers-0.15.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:325212027745d3f8d5d5006bb9e5409d674eb80a184f19873f4f83494e1fdd26"}, + {file = "tokenizers-0.15.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3c5573603c36ce12dbe318bcfb490a94cad2d250f34deb2f06cb6937957bbb71"}, + {file = "tokenizers-0.15.1-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:1441161adb6d71a15a630d5c1d8659d5ebe41b6b209586fbeea64738e58fcbb2"}, + {file = "tokenizers-0.15.1-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:382a8d0c31afcfb86571afbfefa37186df90865ce3f5b731842dab4460e53a38"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e76959783e3f4ec73b3f3d24d4eec5aa9225f0bee565c48e77f806ed1e048f12"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:401df223e5eb927c5961a0fc6b171818a2bba01fb36ef18c3e1b69b8cd80e591"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c52606c233c759561a16e81b2290a7738c3affac7a0b1f0a16fe58dc22e04c7d"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72c658bbe5a05ed8bc2ac5ad782385bfd743ffa4bc87d9b5026341e709c6f44"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25f5643a2f005c42f0737a326c6c6bdfedfdc9a994b10a1923d9c3e792e4d6a6"}, + {file = "tokenizers-0.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c5b6f633999d6b42466bbfe21be2e26ad1760b6f106967a591a41d8cbca980e"}, + {file = "tokenizers-0.15.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ceb5c9ad11a015150b545c1a11210966a45b8c3d68a942e57cf8938c578a77ca"}, + {file = "tokenizers-0.15.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bedd4ce0c4872db193444c395b11c7697260ce86a635ab6d48102d76be07d324"}, + {file = "tokenizers-0.15.1-cp37-none-win32.whl", hash = "sha256:cd6caef6c14f5ed6d35f0ddb78eab8ca6306d0cd9870330bccff72ad014a6f42"}, + {file = "tokenizers-0.15.1-cp37-none-win_amd64.whl", hash = "sha256:d2bd7af78f58d75a55e5df61efae164ab9200c04b76025f9cc6eeb7aff3219c2"}, + {file = "tokenizers-0.15.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:59b3ca6c02e0bd5704caee274978bd055de2dff2e2f39dadf536c21032dfd432"}, + {file = "tokenizers-0.15.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:48fe21b67c22583bed71933a025fd66b1f5cfae1baefa423c3d40379b5a6e74e"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:3d190254c66a20fb1efbdf035e6333c5e1f1c73b1f7bfad88f9c31908ac2c2c4"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fef90c8f5abf17d48d6635f5fd92ad258acd1d0c2d920935c8bf261782cfe7c8"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fac011ef7da3357aa7eb19efeecf3d201ede9618f37ddedddc5eb809ea0963ca"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:574ec5b3e71d1feda6b0ecac0e0445875729b4899806efbe2b329909ec75cb50"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aca16c3c0637c051a59ea99c4253f16fbb43034fac849076a7e7913b2b9afd2d"}, + {file = "tokenizers-0.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a6f238fc2bbfd3e12e8529980ec1624c7e5b69d4e959edb3d902f36974f725a"}, + {file = "tokenizers-0.15.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:587e11a26835b73c31867a728f32ca8a93c9ded4a6cd746516e68b9d51418431"}, + {file = "tokenizers-0.15.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6456e7ad397352775e2efdf68a9ec5d6524bbc4543e926eef428d36de627aed4"}, + {file = "tokenizers-0.15.1-cp38-none-win32.whl", hash = "sha256:614f0da7dd73293214bd143e6221cafd3f7790d06b799f33a987e29d057ca658"}, + {file = "tokenizers-0.15.1-cp38-none-win_amd64.whl", hash = "sha256:a4fa0a20d9f69cc2bf1cfce41aa40588598e77ec1d6f56bf0eb99769969d1ede"}, + {file = "tokenizers-0.15.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8d3f18a45e0cf03ce193d5900460dc2430eec4e14c786e5d79bddba7ea19034f"}, + {file = "tokenizers-0.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:38dbd6c38f88ad7d5dc5d70c764415d38fe3bcd99dc81638b572d093abc54170"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:777286b1f7e52de92aa4af49fe31046cfd32885d1bbaae918fab3bba52794c33"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58d4d550a3862a47dd249892d03a025e32286eb73cbd6bc887fb8fb64bc97165"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4eda68ce0344f35042ae89220b40a0007f721776b727806b5c95497b35714bb7"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cd33d15f7a3a784c3b665cfe807b8de3c6779e060349bd5005bb4ae5bdcb437"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a1aa370f978ac0bfb50374c3a40daa93fd56d47c0c70f0c79607fdac2ccbb42"}, + {file = "tokenizers-0.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:241482b940340fff26a2708cb9ba383a5bb8a2996d67a0ff2c4367bf4b86cc3a"}, + {file = "tokenizers-0.15.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:68f30b05f46a4d9aba88489eadd021904afe90e10a7950e28370d6e71b9db021"}, + {file = "tokenizers-0.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5a3c5d8025529670462b881b7b2527aacb6257398c9ec8e170070432c3ae3a82"}, + {file = "tokenizers-0.15.1-cp39-none-win32.whl", hash = "sha256:74d1827830f60a9d78da8f6d49a1fbea5422ce0eea42e2617877d23380a7efbc"}, + {file = "tokenizers-0.15.1-cp39-none-win_amd64.whl", hash = "sha256:9ff499923e4d6876d6b6a63ea84a56805eb35e91dd89b933a7aee0c56a3838c6"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b3aa007a0f4408f62a8471bdaa3faccad644cbf2622639f2906b4f9b5339e8b8"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f3d4176fa93d8b2070db8f3c70dc21106ae6624fcaaa334be6bdd3a0251e729e"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1d0e463655ef8b2064df07bd4a445ed7f76f6da3b286b4590812587d42f80e89"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:089138fd0351b62215c462a501bd68b8df0e213edcf99ab9efd5dba7b4cb733e"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e563ac628f5175ed08e950430e2580e544b3e4b606a0995bb6b52b3a3165728"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:244dcc28c5fde221cb4373961b20da30097669005b122384d7f9f22752487a46"}, + {file = "tokenizers-0.15.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d82951d46052dddae1369e68ff799a0e6e29befa9a0b46e387ae710fd4daefb0"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7b14296bc9059849246ceb256ffbe97f8806a9b5d707e0095c22db312f4fc014"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0309357bb9b6c8d86cdf456053479d7112074b470651a997a058cd7ad1c4ea57"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083f06e9d8d01b70b67bcbcb7751b38b6005512cce95808be6bf34803534a7e7"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85288aea86ada579789447f0dcec108ebef8da4b450037eb4813d83e4da9371e"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:385e6fcb01e8de90c1d157ae2a5338b23368d0b1c4cc25088cdca90147e35d17"}, + {file = "tokenizers-0.15.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:60067edfcbf7d6cd448ac47af41ec6e84377efbef7be0c06f15a7c1dd069e044"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f7e37f89acfe237d4eaf93c3b69b0f01f407a7a5d0b5a8f06ba91943ea3cf10"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:6a63a15b523d42ebc1f4028e5a568013388c2aefa4053a263e511cb10aaa02f1"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2417d9e4958a6c2fbecc34c27269e74561c55d8823bf914b422e261a11fdd5fd"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8550974bace6210e41ab04231e06408cf99ea4279e0862c02b8d47e7c2b2828"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:194ba82129b171bcd29235a969e5859a93e491e9b0f8b2581f500f200c85cfdd"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:1bfd95eef8b01e6c0805dbccc8eaf41d8c5a84f0cce72c0ab149fe76aae0bce6"}, + {file = "tokenizers-0.15.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b87a15dd72f8216b03c151e3dace00c75c3fe7b0ee9643c25943f31e582f1a34"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6ac22f358a0c2a6c685be49136ce7ea7054108986ad444f567712cf274b34cd8"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e9d1f046a9b9d9a95faa103f07db5921d2c1c50f0329ebba4359350ee02b18b"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a0fd30a4b74485f6a7af89fffb5fb84d6d5f649b3e74f8d37f624cc9e9e97cf"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80e45dc206b9447fa48795a1247c69a1732d890b53e2cc51ba42bc2fefa22407"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4eaff56ef3e218017fa1d72007184401f04cb3a289990d2b6a0a76ce71c95f96"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:b41dc107e4a4e9c95934e79b025228bbdda37d9b153d8b084160e88d5e48ad6f"}, + {file = "tokenizers-0.15.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1922b8582d0c33488764bcf32e80ef6054f515369e70092729c928aae2284bc2"}, + {file = "tokenizers-0.15.1.tar.gz", hash = "sha256:c0a331d6d5a3d6e97b7f99f562cee8d56797180797bc55f12070e495e717c980"}, ] [package.dependencies] @@ -2389,13 +2490,13 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.36.1" +version = "4.37.1" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" optional = false python-versions = ">=3.8.0" files = [ - {file = "transformers-4.36.1-py3-none-any.whl", hash = "sha256:0e309d03634885f02d46801ec4f2c3fc1d614a5b9ebde608181f3e842bac53b8"}, - {file = "transformers-4.36.1.tar.gz", hash = "sha256:28e55952d9bed68f06cf45a3d29cc480679b528afe944e68f8cf6c799e428759"}, + {file = "transformers-4.37.1-py3-none-any.whl", hash = "sha256:05e4c4bf94f74addeb716bc83517f49d55df1e9022db3d5b027c801e9a410ebf"}, + {file = "transformers-4.37.1.tar.gz", hash = "sha256:9843368d97fd7ac30126664743adc65e8e5be930da7d66342172e97bd1243e2d"}, ] [package.dependencies] @@ -2412,16 +2513,16 @@ tqdm = ">=4.27" [package.extras] accelerate = ["accelerate (>=0.21.0)"] -agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.10,!=1.12.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] +agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.11,!=1.12.0)"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "torchaudio", "torchvision"] audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] codecarbon = ["codecarbon (==1.2.0)"] deepspeed = ["accelerate (>=0.21.0)", "deepspeed (>=0.9.3)"] deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.21.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.14,<0.19)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -docs = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "torchaudio", "torchvision"] docs-specific = ["hf-doc-builder"] flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] @@ -2429,7 +2530,7 @@ ftfy = ["ftfy"] integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] modelcreation = ["cookiecutter (==1.7.3)"] -natten = ["natten (>=0.14.6)"] +natten = ["natten (>=0.14.6,<0.15.0)"] onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] optuna = ["optuna"] @@ -2448,10 +2549,10 @@ tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6, tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] timm = ["timm"] tokenizers = ["tokenizers (>=0.14,<0.19)"] -torch = ["accelerate (>=0.21.0)", "torch (>=1.10,!=1.12.0)"] +torch = ["accelerate (>=0.21.0)", "torch (>=1.11,!=1.12.0)"] torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.19.3,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "tqdm (>=4.27)"] +torchhub = ["filelock", "huggingface-hub (>=0.19.3,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.14,<0.19)", "torch (>=1.11,!=1.12.0)", "tqdm (>=4.27)"] video = ["av (==9.2.0)", "decord (==0.6.0)"] vision = ["Pillow (>=10.0.1,<=15.0)"] @@ -2513,13 +2614,13 @@ files = [ [[package]] name = "tzdata" -version = "2023.3" +version = "2023.4" description = "Provider of IANA time zone data" optional = true python-versions = ">=2" files = [ - {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, - {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, ] [[package]] @@ -2861,4 +2962,4 @@ torch = ["torch"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "d314f7dc9ea4d4e7581552f340d9171f24f709bf98de8b8c01c449c23026e7a3" +content-hash = "33d533d21d14c258678a8c4bb28e2a15e8ebe5ca35d8589cbfe4a7b7d2e79a90" diff --git a/server/pyproject.toml b/server/pyproject.toml index d1452678..72a7afb0 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "text-generation-server" -version = "1.3.4" +version = "1.4.0" description = "Text Generation Inference Python gRPC Server" authors = ["Olivier Dehaene "] diff --git a/server/requirements_cuda.txt b/server/requirements_cuda.txt index 694242e1..e9267512 100644 --- a/server/requirements_cuda.txt +++ b/server/requirements_cuda.txt @@ -13,11 +13,11 @@ grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13" grpcio-reflection==1.60.0 ; python_version >= "3.9" and python_version < "3.13" grpcio-status==1.60.0 ; python_version >= "3.9" and python_version < "3.13" grpcio==1.60.0 ; python_version >= "3.9" and python_version < "3.13" -hf-transfer==0.1.4 ; python_version >= "3.9" and python_version < "3.13" +hf-transfer==0.1.5 ; python_version >= "3.9" and python_version < "3.13" huggingface-hub==0.19.4 ; python_version >= "3.9" and python_version < "3.13" idna==3.6 ; python_version >= "3.9" and python_version < "3.13" loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" -numpy==1.26.2 ; python_version >= "3.9" and python_version < "3.13" +numpy==1.26.3 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "3.13" @@ -28,18 +28,18 @@ opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13 opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13" packaging==23.2 ; python_version >= "3.9" and python_version < "3.13" -pillow==10.1.0 ; python_version >= "3.9" and python_version < "3.13" -protobuf==4.25.1 ; python_version >= "3.9" and python_version < "3.13" +pillow==10.2.0 ; python_version >= "3.9" and python_version < "3.13" +protobuf==4.25.2 ; python_version >= "3.9" and python_version < "3.13" pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" -regex==2023.10.3 ; python_version >= "3.9" and python_version < "3.13" +regex==2023.12.25 ; python_version >= "3.9" and python_version < "3.13" requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" safetensors==0.3.3 ; python_version >= "3.9" and python_version < "3.13" -scipy==1.11.4 ; python_version >= "3.9" and python_version < "3.13" +scipy==1.12.0 ; python_version >= "3.9" and python_version < "3.13" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" -setuptools==69.0.2 ; python_version >= "3.9" and python_version < "3.13" -tokenizers==0.15.0 ; python_version >= "3.9" and python_version < "3.13" +setuptools==69.0.3 ; python_version >= "3.9" and python_version < "3.13" +tokenizers==0.15.1 ; python_version >= "3.9" and python_version < "3.13" tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" -transformers==4.36.1 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.37.1 ; python_version >= "3.9" and python_version < "3.13" typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" typing-extensions==4.9.0 ; python_version >= "3.9" and python_version < "3.13" urllib3==2.1.0 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/requirements_rocm.txt b/server/requirements_rocm.txt index e0495fde..053429c9 100644 --- a/server/requirements_rocm.txt +++ b/server/requirements_rocm.txt @@ -12,11 +12,11 @@ grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13" grpcio-reflection==1.60.0 ; python_version >= "3.9" and python_version < "3.13" grpcio-status==1.60.0 ; python_version >= "3.9" and python_version < "3.13" grpcio==1.60.0 ; python_version >= "3.9" and python_version < "3.13" -hf-transfer==0.1.4 ; python_version >= "3.9" and python_version < "3.13" +hf-transfer==0.1.5 ; python_version >= "3.9" and python_version < "3.13" huggingface-hub==0.19.4 ; python_version >= "3.9" and python_version < "3.13" idna==3.6 ; python_version >= "3.9" and python_version < "3.13" loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13" -numpy==1.26.2 ; python_version >= "3.9" and python_version < "3.13" +numpy==1.26.3 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "3.13" @@ -27,18 +27,18 @@ opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13 opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13" opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13" packaging==23.2 ; python_version >= "3.9" and python_version < "3.13" -pillow==10.1.0 ; python_version >= "3.9" and python_version < "3.13" -protobuf==4.25.1 ; python_version >= "3.9" and python_version < "3.13" +pillow==10.2.0 ; python_version >= "3.9" and python_version < "3.13" +protobuf==4.25.2 ; python_version >= "3.9" and python_version < "3.13" pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13" -regex==2023.10.3 ; python_version >= "3.9" and python_version < "3.13" +regex==2023.12.25 ; python_version >= "3.9" and python_version < "3.13" requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13" safetensors==0.3.3 ; python_version >= "3.9" and python_version < "3.13" -scipy==1.11.4 ; python_version >= "3.9" and python_version < "3.13" +scipy==1.12.0 ; python_version >= "3.9" and python_version < "3.13" sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13" -setuptools==69.0.2 ; python_version >= "3.9" and python_version < "3.13" -tokenizers==0.15.0 ; python_version >= "3.9" and python_version < "3.13" +setuptools==69.0.3 ; python_version >= "3.9" and python_version < "3.13" +tokenizers==0.15.1 ; python_version >= "3.9" and python_version < "3.13" tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13" -transformers==4.36.1 ; python_version >= "3.9" and python_version < "3.13" +transformers==4.37.1 ; python_version >= "3.9" and python_version < "3.13" typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13" typing-extensions==4.9.0 ; python_version >= "3.9" and python_version < "3.13" urllib3==2.1.0 ; python_version >= "3.9" and python_version < "3.13" diff --git a/server/tests/utils/test_layers.py b/server/tests/utils/test_layers.py index 0a9fecd1..93a0e982 100644 --- a/server/tests/utils/test_layers.py +++ b/server/tests/utils/test_layers.py @@ -3,24 +3,27 @@ TensorParallelEmbedding, ) + class ProcessGroup: def __init__(self, rank: int, world_size: int): self._rank = rank self.world_size = world_size - def size(self)->int: + def size(self) -> int: return self.world_size - def rank(self)->int: + def rank(self) -> int: return self._rank + class Weights: def __init__(self, rank: int, world_size: int, vocab_size: int, hidden_dim: int): - self.weight = torch.arange(vocab_size*hidden_dim).float().view(vocab_size, hidden_dim) + self.weight = ( + torch.arange(vocab_size * hidden_dim).float().view(vocab_size, hidden_dim) + ) self.process_group = ProcessGroup(rank, world_size) - - def get_partial_sharded(self, name:str, dim: int): + def get_partial_sharded(self, name: str, dim: int): assert dim == 0 rank = self.process_group.rank() @@ -35,10 +38,11 @@ def get_partial_sharded(self, name:str, dim: int): def get_shape(self, name: str): return self.weight.shape + def test_weight_hub_files_offline_error(): - vocab_size= 17 - weights = Weights(rank=0, world_size=1, vocab_size = vocab_size,hidden_dim = 256) + vocab_size = 17 + weights = Weights(rank=0, world_size=1, vocab_size=vocab_size, hidden_dim=256) embeddings = TensorParallelEmbedding("", weights) input_ids = torch.arange(vocab_size) @@ -47,18 +51,27 @@ def test_weight_hub_files_offline_error(): assert embeddings.max_id == 17 torch.testing.assert_close(output, torch.arange(256 * 17).float().view(17, 256)) - weights_0_2 = Weights(rank=0, world_size=2, vocab_size = vocab_size,hidden_dim = 256) - weights_1_2 = Weights(rank=1, world_size=2, vocab_size = vocab_size,hidden_dim = 256) + weights_0_2 = Weights(rank=0, world_size=2, vocab_size=vocab_size, hidden_dim=256) + weights_1_2 = Weights(rank=1, world_size=2, vocab_size=vocab_size, hidden_dim=256) embeddings_0_2 = TensorParallelEmbedding("", weights_0_2, reduce=False) assert embeddings_0_2.min_id == 0 assert embeddings_0_2.max_id == 9 - torch.testing.assert_close(embeddings_0_2.weight , torch.cat([torch.arange(9 * 256), torch.zeros(256)], dim=0).view(10, 256).float()) + torch.testing.assert_close( + embeddings_0_2.weight, + torch.cat([torch.arange(9 * 256), torch.zeros(256)], dim=0) + .view(10, 256) + .float(), + ) embeddings_1_2 = TensorParallelEmbedding("", weights_1_2, reduce=False) assert embeddings_1_2.min_id == 9 assert embeddings_1_2.max_id == 17 - torch.testing.assert_close(embeddings_1_2.weight , torch.cat([torch.arange(8 * 256) + 9 * 256, torch.zeros(256)], dim=0).view(9, 256).float()) + torch.testing.assert_close( + embeddings_1_2.weight, + torch.cat([torch.arange(8 * 256) + 9 * 256, torch.zeros(256)], dim=0) + .view(9, 256) + .float(), + ) output_tp_0 = embeddings_0_2.forward(input_ids) output_tp_1 = embeddings_1_2.forward(input_ids) torch.testing.assert_close(output, output_tp_0 + output_tp_1) - diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py index 99be6c7e..b74fbe36 100644 --- a/server/text_generation_server/cli.py +++ b/server/text_generation_server/cli.py @@ -226,7 +226,7 @@ def download_weights( pass except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError): pass - + elif (Path(model_id) / "adapter_config.json").exists(): # Try to load as a local PEFT model try: diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 679e1e2f..68096709 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -230,7 +230,7 @@ def get_model( dtype=dtype, trust_remote_code=trust_remote_code, ) - + elif model_type == "phi": if FLASH_ATTENTION: return FlashPhi( @@ -252,7 +252,9 @@ def get_model( elif model_type == "phi-msft": if FLASH_ATTENTION: - raise NotImplementedError("Legacy phi-msft is not supported with Flash Attention") + raise NotImplementedError( + "Legacy phi-msft is not supported with Flash Attention" + ) else: return Phi( model_id, diff --git a/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py index d103973f..96701794 100644 --- a/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py @@ -17,6 +17,7 @@ FastLayerNorm, ) + class PhiConfig(PretrainedConfig): def __init__( self, @@ -25,15 +26,15 @@ def __init__( num_hidden_layers=32, num_attention_heads=32, num_key_value_heads=32, - hidden_act="gelu_fast", # llama uses silu - layer_norm_eps=1e-05, # rms in llama, + hidden_act="gelu_fast", # llama uses silu + layer_norm_eps=1e-05, # rms in llama, pad_token_id=0, bos_token_id=1, eos_token_id=2, tie_word_embeddings=False, rope_theta=10000.0, - resid_pdrop=0.1, # llama doesn't have this - partial_rotary_factor=0.5, # important difference between llama and phi + resid_pdrop=0.1, # llama doesn't have this + partial_rotary_factor=0.5, # important difference between llama and phi **kwargs, ): self.vocab_size = vocab_size @@ -55,6 +56,7 @@ def __init__( **kwargs, ) + # this is the same as llama except for Phi uses bias=True def load_attention(config, prefix, weights): if config.num_attention_heads != config.num_key_value_heads: @@ -68,6 +70,7 @@ def load_attention(config, prefix, weights): bias=True, ) + def _load_gqa(config, prefix: str, weights): assert config.hidden_size % config.num_attention_heads == 0 assert config.num_attention_heads % weights.process_group.size() == 0 @@ -94,6 +97,7 @@ def _load_gqa(config, prefix: str, weights): get_linear(weight, bias=True, quantize=config.quantize) ) + class FlashPhiAttention(torch.nn.Module): def __init__( self, @@ -173,8 +177,7 @@ def forward( # # Apply partial positional embeddings in place self.rotary_emb( - query[:, :, :self.rotary_dim], kv[:, 0, :, :self.rotary_dim], - cos, sin + query[:, :, : self.rotary_dim], kv[:, 0, :, : self.rotary_dim], cos, sin ) # Reshape key and value and cache @@ -210,7 +213,8 @@ def forward( max_s, ) - return self.dense(attn_output.view(-1, self.num_heads*self.head_size)) + return self.dense(attn_output.view(-1, self.num_heads * self.head_size)) + class PhiMLP(nn.Module): def __init__(self, prefix, config, weights): @@ -256,7 +260,9 @@ def __init__(self, layer_id, config, weights): ) self.mlp = PhiMLP(prefix=f"{prefix}.mlp", config=config, weights=weights) self.input_layernorm = FastLayerNorm.load( - prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.layer_norm_eps + prefix=f"{prefix}.input_layernorm", + weights=weights, + eps=config.layer_norm_eps, ) self.resid_dropout = torch.nn.Dropout(config.resid_pdrop) @@ -287,10 +293,13 @@ def forward( max_s, ) - hidden_states = self.resid_dropout(attn_output).add(self.resid_dropout(self.mlp(hidden_states))) + hidden_states = self.resid_dropout(attn_output).add( + self.resid_dropout(self.mlp(hidden_states)) + ) return hidden_states, res + class FlashPhiModel(torch.nn.Module): def __init__(self, config, weights): super().__init__() @@ -361,6 +370,7 @@ def forward( return hidden_states + class FlashPhiForCausalLM(torch.nn.Module): def __init__(self, config, weights): super().__init__() @@ -380,7 +390,7 @@ def forward( kv_cache: List[Tuple[torch.Tensor, torch.Tensor]], block_tables: torch.Tensor, slots: torch.Tensor, - input_lengths: torch.Tensor, + input_lengths: torch.Tensor, max_s: int, lm_head_indices: Optional[torch.Tensor] = None, ) -> torch.Tensor: diff --git a/server/text_generation_server/models/custom_modeling/mpt_modeling.py b/server/text_generation_server/models/custom_modeling/mpt_modeling.py index 1a9aef74..2c2fec48 100644 --- a/server/text_generation_server/models/custom_modeling/mpt_modeling.py +++ b/server/text_generation_server/models/custom_modeling/mpt_modeling.py @@ -54,9 +54,19 @@ def load_col(config, prefix, weights, bias): bias_h = bias_h[0] bias_block_size = bias_h // bias_size - bias_q_part = bias_slice_[bias_rank * bias_block_size : (bias_rank + 1) * bias_block_size] - bias_k_part = bias_slice_[bias_h + bias_rank * bias_block_size : bias_h + (bias_rank + 1) * bias_block_size] - bias_v_part = bias_slice_[2 * bias_h + bias_rank * bias_block_size : 2 * bias_h + (bias_rank + 1) * bias_block_size] + bias_q_part = bias_slice_[ + bias_rank * bias_block_size : (bias_rank + 1) * bias_block_size + ] + bias_k_part = bias_slice_[ + bias_h + + bias_rank * bias_block_size : bias_h + + (bias_rank + 1) * bias_block_size + ] + bias_v_part = bias_slice_[ + 2 * bias_h + + bias_rank * bias_block_size : 2 * bias_h + + (bias_rank + 1) * bias_block_size + ] bias = torch.cat([bias_q_part, bias_k_part, bias_v_part], dim=0) if bias.dtype != torch.int32: @@ -352,8 +362,12 @@ def __init__( hidden_size = config.d_model head_dim = hidden_size // self.n_heads - self.q_ln = LPLayerNorm(d_model, bias=bias, prefix=f"{prefix}.q_ln", weights=weights) - self.k_ln = LPLayerNorm(self.n_heads * head_dim, prefix=f"{prefix}.k_ln", weights=weights) + self.q_ln = LPLayerNorm( + d_model, bias=bias, prefix=f"{prefix}.q_ln", weights=weights + ) + self.k_ln = LPLayerNorm( + self.n_heads * head_dim, prefix=f"{prefix}.k_ln", weights=weights + ) if self.attn_impl == "flash": self.attn_fn = flash_attn_fn elif self.attn_impl == "triton": @@ -684,7 +698,6 @@ def __init__( self.bias = nn.Parameter(weights.get_sharded(f"{prefix}.bias", dim=0)) self.normalized_shape = self.weight.shape - def forward(self, x): module_device = x.device downcast_x = _cast_if_autocast_enabled(x) @@ -798,7 +811,7 @@ def __init__(self, config, weights): self.wte = TensorParallelEmbedding("transformer.wte", weights) if not self.alibi: - self.wpe = TensorParallelEmbedding("transformer.wpe", weights) + self.wpe = TensorParallelEmbedding("transformer.wpe", weights) self.blocks = nn.ModuleList( [ MPTBlock(config, prefix=f"transformer.blocks.{i}", weights=weights) diff --git a/server/text_generation_server/models/custom_modeling/phi_modeling.py b/server/text_generation_server/models/custom_modeling/phi_modeling.py index f9999537..e5c09728 100644 --- a/server/text_generation_server/models/custom_modeling/phi_modeling.py +++ b/server/text_generation_server/models/custom_modeling/phi_modeling.py @@ -62,14 +62,12 @@ def __init__( **kwargs, ) + # RotaryEmbedding is a class that implements the rotary embedding. class RotaryEmbedding(nn.Module): def __init__(self, dim, max_seq_len): super().__init__() - inv_freq = [ - 1.0 / 10000.0 ** (i / dim) - for i in range(0, dim, 2) - ] + inv_freq = [1.0 / 10000.0 ** (i / dim) for i in range(0, dim, 2)] inv_freq_len = len(inv_freq) inv_freq = torch.tensor(inv_freq).view(1, inv_freq_len) t = torch.arange(0, max_seq_len, dtype=torch.float).view(max_seq_len, 1) @@ -131,6 +129,7 @@ def forward(self, hidden_states): hidden_states = self.linear(hidden_states) return hidden_states + # PhiMHA is a multi-head attention layer. This layer uses an attention mask to prevent tokens from attending to subsequent tokens. class PhiMHA(nn.Module): def __init__(self, prefix, config, weights): @@ -172,19 +171,27 @@ def forward( v = torch.cat([prev_v, v], dim=1) past_kv_cache = [k, v] - attn_weights = torch.einsum('bthd,bshd->bhts', q, k * self.softmax_scale) + attn_weights = torch.einsum("bthd,bshd->bhts", q, k * self.softmax_scale) if attention_mask is not None: seqlen_k = k.shape[1] seqlen_q = q.shape[1] - causal_mask = torch.triu(torch.full((seqlen_q, seqlen_k), -10000.0, device=attn_weights.device), 1) + causal_mask = torch.triu( + torch.full((seqlen_q, seqlen_k), -10000.0, device=attn_weights.device), + 1, + ) attn_weights = attn_weights + causal_mask.to(dtype=attn_weights.dtype) - + attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1) attn_output = attn_weights.matmul(v.transpose(1, 2)).squeeze(0) - attn_output = attn_output.view((b_size, self.num_heads, seq_len, self.head_dim)).transpose(1, 2).flatten(-2) + attn_output = ( + attn_output.view((b_size, self.num_heads, seq_len, self.head_dim)) + .transpose(1, 2) + .flatten(-2) + ) return self.out_proj(attn_output), past_kv_cache + # PhiMLP is a multi-layer perceptron. It contains two linear layers with a gelu activation function. class PhiMLP(nn.Module): def __init__(self, prefix, config, weights): @@ -204,19 +211,22 @@ def __init__(self, prefix, config, weights): bias=False, ) self.activation = torch.nn.functional.gelu - + def forward(self, hidden_states): hidden_states = self.fc1(hidden_states) hidden_states = self.activation(hidden_states) hidden_states = self.fc2(hidden_states) return hidden_states + # PhiBlock is a single transformer block. It contains a layer norm, a multi-head attention layer and an multi-layer perceptron. class PhiBlock(nn.Module): def __init__(self, layer_id, config, weights): super().__init__() self.layer_id = layer_id - self.layer_norm = nn.LayerNorm.load(prefix=f"{layer_id}.ln", weights=weights, eps=config.layer_norm_epsilon) + self.layer_norm = nn.LayerNorm.load( + prefix=f"{layer_id}.ln", weights=weights, eps=config.layer_norm_epsilon + ) self.mixer = PhiMHA(prefix=f"{layer_id}.mixer", config=config, weights=weights) self.mlp = PhiMLP(prefix=f"{layer_id}.mlp", config=config, weights=weights) @@ -228,11 +238,14 @@ def forward( ): residual = hidden_states hidden_states = self.layer_norm(hidden_states) - attn_outputs, past_kv_cache = self.mixer(hidden_states, kv_cache, attention_mask) + attn_outputs, past_kv_cache = self.mixer( + hidden_states, kv_cache, attention_mask + ) feed_forward_hidden_states = self.mlp(hidden_states) out = attn_outputs + feed_forward_hidden_states + residual return out, past_kv_cache + # PhiModel implements the embedding layer and the transformer blocks. class PhiModel(nn.Module): def __init__(self, config, weights): @@ -241,9 +254,12 @@ def __init__(self, config, weights): self.tp_world_size = weights.process_group.size() self.embed_tokens = TensorParallelEmbedding( prefix="transformer.embd.wte", weights=weights - ) + ) self.blocks = nn.ModuleList( - [PhiBlock(f"transformer.h.{layer_id}", config, weights) for layer_id in range(config.n_layer)] + [ + PhiBlock(f"transformer.h.{layer_id}", config, weights) + for layer_id in range(config.n_layer) + ] ) def forward( @@ -258,14 +274,19 @@ def forward( seq_len = hidden_states.shape[1] mask = None if seq_len <= 1 else attention_mask - past_key_values = [None] * len(self.blocks) if past_key_values is None else past_key_values + past_key_values = ( + [None] * len(self.blocks) if past_key_values is None else past_key_values + ) for index, block in enumerate(self.blocks): - hidden_states, new_key_values = block(hidden_states, past_key_values[index], mask) + hidden_states, new_key_values = block( + hidden_states, past_key_values[index], mask + ) past_key_values[index] = new_key_values return hidden_states, past_key_values + # PhiForCausalLM wraps the PhiModel and PhiCausalLMHead together and returns a CausalLMOutputWithPast object. class PhiForCausalLM(torch.nn.Module): def __init__(self, config, weights): @@ -290,12 +311,15 @@ def forward( loss = None if labels is not None: loss = nn.CrossEntropyLoss()( - logits[:, :-1].view(-1, logits.size(-1)), - labels[:, 1:].view(-1) + logits[:, :-1].view(-1, logits.size(-1)), labels[:, 1:].view(-1) ) if not return_dict: - return ((loss,) + (logits,) + model_output[1:]) if loss is not None else (logits,) + model_output[1:] + return ( + ((loss,) + (logits,) + model_output[1:]) + if loss is not None + else (logits,) + model_output[1:] + ) return CausalLMOutputWithPast( loss=loss, @@ -304,5 +328,3 @@ def forward( hidden_states=None, attentions=None, ) - - diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index 7be61906..94bd58f4 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -73,11 +73,11 @@ def __init__( import json import os from pathlib import Path - - is_local_model = (Path(use_medusa).exists() and Path(use_medusa).is_dir()) or os.getenv( - "WEIGHTS_CACHE_OVERRIDE", None - ) is not None - + + is_local_model = ( + Path(use_medusa).exists() and Path(use_medusa).is_dir() + ) or os.getenv("WEIGHTS_CACHE_OVERRIDE", None) is not None + if not is_local_model: medusa_config = hf_hub_download( use_medusa, revision=revision, filename="config.json" @@ -88,7 +88,7 @@ def __init__( else: medusa_config = str(Path(use_medusa) / "config.json") medusa_head = str(Path(use_medusa) / "medusa_lm_head.pt") - + with open(medusa_config, "r") as f: config = json.load(f) medusa_sf = medusa_head[: -len(".pt")] + ".safetensors" diff --git a/server/text_generation_server/models/flash_phi.py b/server/text_generation_server/models/flash_phi.py index 1c49f2a9..061b9740 100644 --- a/server/text_generation_server/models/flash_phi.py +++ b/server/text_generation_server/models/flash_phi.py @@ -63,11 +63,11 @@ def __init__( import json import os from pathlib import Path - - is_local_model = (Path(use_medusa).exists() and Path(use_medusa).is_dir()) or os.getenv( - "WEIGHTS_CACHE_OVERRIDE", None - ) is not None - + + is_local_model = ( + Path(use_medusa).exists() and Path(use_medusa).is_dir() + ) or os.getenv("WEIGHTS_CACHE_OVERRIDE", None) is not None + if not is_local_model: medusa_config = hf_hub_download( use_medusa, revision=revision, filename="config.json" @@ -78,7 +78,7 @@ def __init__( else: medusa_config = str(Path(use_medusa) / "config.json") medusa_head = str(Path(use_medusa) / "medusa_lm_head.pt") - + with open(medusa_config, "r") as f: config = json.load(f) medusa_sf = medusa_head[: -len(".pt")] + ".safetensors" diff --git a/server/text_generation_server/models/phi.py b/server/text_generation_server/models/phi.py index d477478a..79aa3fb9 100644 --- a/server/text_generation_server/models/phi.py +++ b/server/text_generation_server/models/phi.py @@ -5,13 +5,17 @@ from typing import Optional, List, Tuple from text_generation_server.models import CausalLM -from text_generation_server.models.custom_modeling.phi_modeling import PhiConfig, PhiForCausalLM +from text_generation_server.models.custom_modeling.phi_modeling import ( + PhiConfig, + PhiForCausalLM, +) from text_generation_server.utils import ( initialize_torch_distributed, weight_files, Weights, ) + class Phi(CausalLM): def __init__( self, @@ -60,4 +64,3 @@ def __init__( dtype=dtype, device=device, ) - diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 6ddfd6f4..010d6143 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -510,7 +510,9 @@ def __init__(self, prefix: str, weights, reduce=True): block_size = (num_embeddings + world_size - 1) // world_size self.min_id = rank * block_size self.max_id = min(num_embeddings, (rank + 1) * block_size) - self.null_idx = weight.shape[0] # Usually block_size, might be less in non even vocab_size. + self.null_idx = weight.shape[ + 0 + ] # Usually block_size, might be less in non even vocab_size. self.process_group = weights.process_group self.reduce = reduce From 069895b9859b776cab8145f3fa6f6d16ac40af47 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Fri, 26 Jan 2024 20:13:47 +0100 Subject: [PATCH 541/793] Fixing top_n_tokens. (#1497) # What does this PR do? Superseeds #1459 The fix works as follows. We updated next_token_chooser to return all logprbs, then batch_top_n_tokens, now also gets accepted_ids + speculated_length (so it knows how to interpret the flat logprobs). We then update the code to return lists ot `Tokens` that it expects. Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/tests/utils/test_tokens.py | 44 +++++++++---- .../models/causal_lm.py | 35 ++++++---- .../models/flash_causal_lm.py | 39 ++++++----- .../models/seq2seq_lm.py | 35 ++++++---- server/text_generation_server/models/types.py | 2 +- server/text_generation_server/utils/tokens.py | 65 +++++++++++++------ 6 files changed, 143 insertions(+), 77 deletions(-) diff --git a/server/tests/utils/test_tokens.py b/server/tests/utils/test_tokens.py index 0585f1fb..d3f2d766 100644 --- a/server/tests/utils/test_tokens.py +++ b/server/tests/utils/test_tokens.py @@ -50,19 +50,39 @@ def test_batch_top_tokens(): top_n_tokens = [0, 2, 3, 4, 5] top_n_tokens_tensor = torch.tensor(top_n_tokens) inp_logprobs = torch.tensor([[-1.0, -3.0, -4.0, -2.0, -3.0]] * 5) + accepted_ids = torch.ones_like(top_n_tokens_tensor) topn_tok_ids, topn_tok_logprobs = batch_top_tokens( - top_n_tokens, top_n_tokens_tensor, inp_logprobs + top_n_tokens, top_n_tokens_tensor, inp_logprobs, accepted_ids ) - assert topn_tok_ids[0] == [] - assert topn_tok_ids[1] == [0, 3] - assert topn_tok_ids[2] == [0, 3, 1, 4] - assert topn_tok_ids[3] == [0, 3, 1, 4] - assert topn_tok_ids[4] == [0, 3, 1, 4, 2] - - assert topn_tok_logprobs[0] == [] - assert topn_tok_logprobs[1] == [-1, -2] - assert topn_tok_logprobs[2] == [-1, -2, -3, -3] - assert topn_tok_logprobs[3] == [-1, -2, -3, -3] - assert topn_tok_logprobs[4] == [-1, -2, -3, -3, -4] + assert topn_tok_ids[0] == [[]] + assert topn_tok_ids[1] == [[0, 3]] + assert topn_tok_ids[2] == [[0, 3, 1, 4]] + assert topn_tok_ids[3] == [[0, 3, 1, 4]] + assert topn_tok_ids[4] == [[0, 3, 1, 4, 2]] + + assert topn_tok_logprobs[0] == [[]] + assert topn_tok_logprobs[1] == [[-1, -2]] + assert topn_tok_logprobs[2] == [[-1, -2, -3, -3]] + assert topn_tok_logprobs[3] == [[-1, -2, -3, -3]] + assert topn_tok_logprobs[4] == [[-1, -2, -3, -3, -4]] + + # Now let's make second member of the batch be speculated + inp_logprobs = torch.tensor([[-1.0, -3.0, -4.0, -2.0, -3.0]] * 5 * 2) + accepted_ids[1] = 2 + topn_tok_ids, topn_tok_logprobs = batch_top_tokens( + top_n_tokens, top_n_tokens_tensor, inp_logprobs, accepted_ids + ) + + assert topn_tok_ids[0] == [[]] + assert topn_tok_ids[1] == [[0, 3], [0, 3]] + assert topn_tok_ids[2] == [[0, 3, 1, 4]] + assert topn_tok_ids[3] == [[0, 3, 1, 4]] + assert topn_tok_ids[4] == [[0, 3, 1, 4, 2]] + + assert topn_tok_logprobs[0] == [[]] + assert topn_tok_logprobs[1] == [[-1, -2], [-1, -2]] + assert topn_tok_logprobs[2] == [[-1, -2, -3, -3]] + assert topn_tok_logprobs[3] == [[-1, -2, -3, -3]] + assert topn_tok_logprobs[4] == [[-1, -2, -3, -3, -4]] diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 7b10256c..29e9f8b1 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -580,10 +580,13 @@ def generate_token( generations: List[Generation] = [] stopped = True + # Speculation is not active for causal + accepted_ids = torch.ones_like(batch.input_ids)[:, 0] batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( batch.top_n_tokens, batch.top_n_tokens_tensor, torch.log_softmax(logits[:, -1], -1), + accepted_ids, ) start_decode = time.time_ns() @@ -692,20 +695,24 @@ def generate_token( prefill_tokens = None if top_n_tokens > 0: - toptoken_texts = self.tokenizer.batch_decode( - top_token_ids, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) - special_toptokens = [ - token_id in self.all_special_ids for token_id in top_token_ids - ] - top_tokens = Tokens( - top_token_ids, - top_token_logprobs, - toptoken_texts, - special_toptokens, - ) + all_top_tokens = [] + for (top_token_ids, top_token_logprobs) in zip(top_token_ids, top_token_logprobs): + toptoken_texts = self.tokenizer.batch_decode( + top_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + special_toptokens = [ + token_id in self.all_special_ids for token_id in top_token_ids + ] + top_tokens = Tokens( + top_token_ids, + top_token_logprobs, + toptoken_texts, + special_toptokens, + ) + all_top_tokens.append(top_tokens) + top_tokens = all_top_tokens else: top_tokens = None diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 930082cd..53a3d582 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -842,6 +842,8 @@ def generate_token( else: next_token_logits = out + + speculate = get_speculate() ( next_input_ids, next_token_logprobs, @@ -851,16 +853,15 @@ def generate_token( ) = batch.next_token_chooser( batch.all_input_ids_tensor[:, : batch.max_seqlen], next_token_logits, - get_speculate(), + speculate, batch.speculative_ids, speculative_logits, ) batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( - batch.top_n_tokens, batch.top_n_tokens_tensor, logprobs + batch.top_n_tokens, batch.top_n_tokens_tensor, logprobs, accepted_ids ) - speculative_length = 0 if speculative_ids is None else speculative_ids.shape[1] if prefill: if len(batch) > 1 and prefill_logprobs: # We create the prefill_tokens_indices tensor that will be used to gather prefill logprobs @@ -1062,20 +1063,24 @@ def generate_token( prefill_tokens = None if top_n_tokens > 0: - toptoken_texts = self.tokenizer.batch_decode( - top_token_ids, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) - special_toptokens = [ - token_id in self.all_special_ids for token_id in top_token_ids - ] - top_tokens = Tokens( - top_token_ids, - top_token_logprobs, - toptoken_texts, - special_toptokens, - ) + all_top_tokens = [] + for (top_token_ids, top_token_logprobs) in zip(top_token_ids, top_token_logprobs): + toptoken_texts = self.tokenizer.batch_decode( + top_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + special_toptokens = [ + token_id in self.all_special_ids for token_id in top_token_ids + ] + top_tokens = Tokens( + top_token_ids, + top_token_logprobs, + toptoken_texts, + special_toptokens, + ) + all_top_tokens.append(top_tokens) + top_tokens = all_top_tokens else: top_tokens = None diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index f2e4cec6..8b93aecd 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -640,10 +640,13 @@ def generate_token( batch.past_key_values, ) + # Speculation is not active for seq2seq + accepted_ids = torch.ones_like(batch.decoder_input_ids)[:, 0] batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( batch.top_n_tokens, batch.top_n_tokens_tensor, torch.log_softmax(logits[:, -1], -1), + accepted_ids, ) start_decode = time.time_ns() @@ -746,20 +749,24 @@ def generate_token( prefill_tokens = None if top_n_tokens > 0: - toptoken_texts = self.tokenizer.batch_decode( - top_token_ids, - clean_up_tokenization_spaces=False, - skip_special_tokens=False, - ) - special_toptokens = [ - token_id in self.all_special_ids for token_id in top_token_ids - ] - top_tokens = Tokens( - top_token_ids, - top_token_logprobs, - toptoken_texts, - special_toptokens, - ) + all_top_tokens = [] + for (top_token_ids, top_token_logprobs) in zip(top_token_ids, top_token_logprobs): + toptoken_texts = self.tokenizer.batch_decode( + top_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + special_toptokens = [ + token_id in self.all_special_ids for token_id in top_token_ids + ] + top_tokens = Tokens( + top_token_ids, + top_token_logprobs, + toptoken_texts, + special_toptokens, + ) + all_top_tokens.append(top_tokens) + top_tokens = all_top_tokens else: top_tokens = None diff --git a/server/text_generation_server/models/types.py b/server/text_generation_server/models/types.py index f85f27e5..bc68812e 100644 --- a/server/text_generation_server/models/types.py +++ b/server/text_generation_server/models/types.py @@ -95,5 +95,5 @@ def to_pb(self) -> generate_pb2.Generation: generated_text=self.generated_text.to_pb() if self.generated_text is not None else None, - top_tokens=self.top_tokens.to_pb() if self.top_tokens is not None else None, + top_tokens=[top_tokens.to_pb() for top_tokens in self.top_tokens] if self.top_tokens is not None else None, ) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 04cc8d97..270a6990 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -277,7 +277,8 @@ def __call__( scores[:, j] = _scores next_ids[:, j] = _next_ids next_ids = next_ids.view(B * S) - scores = scores.view(B * S, -1) + allscores = scores.view(B * S, -1) + alllogprobs = torch.log_softmax(allscores, -1) if speculated_ids is not None: accepted_ids = [] @@ -305,16 +306,17 @@ def __call__( accepted_ids, device=input_ids.device, dtype=input_ids.dtype ) next_ids = next_ids[indices] - scores = scores[indices] + logprobs = alllogprobs[indices] indices = torch.arange(B, device=input_ids.device) * S if speculative_scores is not None: speculative_scores = speculative_scores[indices + accepted_ids - 1] else: accepted_ids = torch.ones_like(next_ids) + logprobs = alllogprobs - logprobs = torch.log_softmax(scores, -1) next_logprobs = torch.gather(logprobs, 1, next_ids.view(-1, 1)).view(-1) + if speculate > 0: if speculative_scores is not None: # Medusa provided some scores @@ -327,7 +329,7 @@ def __call__( else: speculative_ids = None - return next_ids, next_logprobs, logprobs, accepted_ids, speculative_ids + return next_ids, next_logprobs, alllogprobs, accepted_ids, speculative_ids def filter(self, indices): if self.watermark_processor is not None: @@ -436,8 +438,8 @@ def filter(self, indices): def batch_top_tokens( - top_n_tokens: List[int], top_n_tokens_tensor: torch.Tensor, logprobs: torch.Tensor -) -> Tuple[List[List[int]], List[List[float]]]: + top_n_tokens: List[int], top_n_tokens_tensor: torch.Tensor, logprobs: torch.Tensor, accepted_ids: torch.Tensor +) -> Tuple[List[List[List[int]]], List[List[List[float]]]]: """Find the top n most likely tokens for a batch of generations. When multiple tokens have equal probabilities and they don't all fit, the @@ -446,14 +448,19 @@ def batch_top_tokens( max_top_n = max(top_n_tokens) # Early exit when top_n_tokens is not used if max_top_n == 0: - return [[]] * len(top_n_tokens), [[]] * len(top_n_tokens) + return [[[]]] * len(top_n_tokens), [[[]]] * len(top_n_tokens) + + batch_size = accepted_ids.shape[0] + speculate_size = logprobs.shape[0] // batch_size + top_n_tokens_tensor = top_n_tokens_tensor.repeat_interleave(speculate_size) # Ensure top_n doesn't exceed vocab size - top_n_tokens = [min(tok, logprobs.size(-1)) for tok in top_n_tokens] + top_n_tokens = [min(tok, logprobs.size(-1)) for tok in top_n_tokens for _ in range(speculate_size)] # Parallel kthvalue adapted from https://discuss.pytorch.org/t/how-to-efficiently-get-the-k-th-largest-values-in-parallel/160529/2 # Sorted topk is faster than torch.sort() since we only need a small subset - sorted_top_k = torch.topk(logprobs, k=max_top_n, dim=1, sorted=True).values + sorted_top_k = torch.topk(logprobs, k=max_top_n, dim=-1, sorted=True).values + nth_highest = torch.gather( sorted_top_k, 1, (top_n_tokens_tensor - 1).clip(min=0).unsqueeze(1) ) @@ -471,13 +478,33 @@ def batch_top_tokens( top_indices = top_k.indices.tolist() top_values = top_k.values.tolist() - return ( - [ - idxs[:n] if req_n > 0 else [] - for idxs, n, req_n in zip(top_indices, top_n_ishes, top_n_tokens) - ], - [ - vals[:n] if req_n > 0 else [] - for vals, n, req_n in zip(top_values, top_n_ishes, top_n_tokens) - ], - ) + batch_top_token_ids = [] + batch_top_token_logprobs = [] + accepted_ids_list = accepted_ids.tolist() + for i, n_accepted_ids in enumerate(accepted_ids_list): + start = speculate_size * i + stop = speculate_size * (i + 1) + _top_indices = top_indices[start: stop] + _top_values = top_values[start: stop] + _top_n_ishes = top_n_ishes[start: stop] + _top_n_tokens = top_n_tokens[start: stop] + + _top_indices = _top_indices[:n_accepted_ids] + _top_values = _top_values[:n_accepted_ids] + _top_n_ishes = _top_n_ishes[:n_accepted_ids] + _top_n_tokens = _top_n_tokens[:n_accepted_ids] + + row_top_token_ids = [] + row_top_token_logprobs = [] + + for idxs, vals, n, req_n in zip(_top_indices, _top_values, _top_n_ishes, _top_n_tokens): + indices = idxs[:n] if req_n > 0 else [] + values = vals[:n] if req_n > 0 else [] + + row_top_token_ids.append(indices) + row_top_token_logprobs.append(values) + + batch_top_token_ids.append(row_top_token_ids) + batch_top_token_logprobs.append(row_top_token_logprobs) + + return batch_top_token_ids, batch_top_token_logprobs From 0424dabb0179a0b6b76186244d716cf43e034cfd Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 29 Jan 2024 11:20:08 +0100 Subject: [PATCH 542/793] Sending compute type from the environment instead of hardcoded string (#1504) # What does this PR do? Sending compute type from the environment instead of hardcoded string Using env is slow, therefore getting it from global state instead. Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- router/src/server.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/router/src/server.rs b/router/src/server.rs index 998d6265..39d1de38 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -57,6 +57,7 @@ example = json ! ({"error": "Incomplete generation"})), async fn compat_generate( Extension(default_return_full_text): Extension, infer: Extension, + compute_type: Extension, Json(mut req): Json, ) -> Result)> { // default return_full_text given the pipeline_tag @@ -66,11 +67,11 @@ async fn compat_generate( // switch on stream if req.stream { - Ok(generate_stream(infer, Json(req.into())) + Ok(generate_stream(infer,compute_type, Json(req.into())) .await .into_response()) } else { - let (headers, Json(generation)) = generate(infer, Json(req.into())).await?; + let (headers, Json(generation)) = generate(infer, compute_type, Json(req.into())).await?; // wrap generation inside a Vec to match api-inference Ok((headers, Json(vec![generation])).into_response()) } @@ -145,6 +146,7 @@ seed, )] async fn generate( infer: Extension, + Extension(ComputeType(compute_type)): Extension, Json(req): Json, ) -> Result<(HeaderMap, Json), (StatusCode, Json)> { let span = tracing::Span::current(); @@ -230,7 +232,7 @@ async fn generate( // Headers let mut headers = HeaderMap::new(); - headers.insert("x-compute-type", "gpu+optimized".parse().unwrap()); + headers.insert("x-compute-type", compute_type.parse().unwrap()); headers.insert( "x-compute-time", total_time.as_millis().to_string().parse().unwrap(), @@ -339,6 +341,7 @@ seed, )] async fn generate_stream( Extension(infer): Extension, + Extension(compute_type): Extension, Json(req): Json, ) -> ( HeaderMap, @@ -349,13 +352,14 @@ async fn generate_stream( event.json_data(stream_token).unwrap() }; let (headers, response_stream) = - generate_stream_internal(infer, Json(req), on_message_callback).await; + generate_stream_internal(infer, compute_type, Json(req), on_message_callback).await; let sse = Sse::new(response_stream).keep_alive(KeepAlive::default()); (headers, sse) } async fn generate_stream_internal( infer: Infer, + ComputeType(compute_type): ComputeType, Json(req): Json, on_message_callback: impl Fn(StreamResponse) -> Event, ) -> (HeaderMap, impl Stream>) { @@ -368,7 +372,7 @@ async fn generate_stream_internal( let compute_characters = req.inputs.chars().count(); let mut headers = HeaderMap::new(); - headers.insert("x-compute-type", "gpu+optimized".parse().unwrap()); + headers.insert("x-compute-type",compute_type.parse().unwrap()); headers.insert( "x-compute-characters", compute_characters.to_string().parse().unwrap(), @@ -557,6 +561,7 @@ async fn generate_stream_internal( )] async fn chat_completions( Extension(infer): Extension, + Extension(compute_type): Extension, Extension(info): Extension, Json(req): Json, ) -> Result)> { @@ -645,12 +650,12 @@ async fn chat_completions( }; let (headers, response_stream) = - generate_stream_internal(infer, Json(generate_request), on_message_callback).await; + generate_stream_internal(infer, compute_type, Json(generate_request), on_message_callback).await; let sse = Sse::new(response_stream).keep_alive(KeepAlive::default()); Ok((headers, sse).into_response()) } else { let (headers, Json(generation)) = - generate(Extension(infer), Json(generate_request)).await?; + generate(Extension(infer), Extension(compute_type), Json(generate_request)).await?; let current_time = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -729,6 +734,9 @@ async fn metrics(prom_handle: Extension) -> String { prom_handle.render() } +#[derive(Clone, Debug)] +pub(crate) struct ComputeType(String); + /// Serving method #[allow(clippy::too_many_arguments)] pub async fn run( @@ -935,6 +943,8 @@ pub async fn run( Router::new().route("/invocations", post(compat_generate)) // Use 'compat_generate' otherwise }; + let compute_type = ComputeType(std::env::var("COMPUTE_TYPE").unwrap_or("gpu+optimized".to_string())); + // Combine routes and layers let app = Router::new() .merge(swagger_ui) @@ -944,6 +954,7 @@ pub async fn run( .layer(Extension(health_ext.clone())) .layer(Extension(compat_return_full_text)) .layer(Extension(infer)) + .layer(Extension(compute_type)) .layer(Extension(prom_handle.clone())) .layer(OtelAxumLayer::default()) .layer(cors_layer); From a9ea60684b6445b2507e147c6aeed0edb0b25eb7 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 29 Jan 2024 12:30:50 +0100 Subject: [PATCH 543/793] Create the compute type at launch time (if not provided in the env). (#1505) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- launcher/src/main.rs | 24 ++++++++++++++++++++++-- router/src/server.rs | 24 +++++++++++++++++------- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index f0e45141..054e546c 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -982,7 +982,20 @@ fn spawn_shards( Ok(()) } +fn compute_type(num_shard: usize) -> Option { + let output = Command::new("nvidia-smi") + .args(["--query-gpu=gpu_name", "--format=csv"]) + .output() + .ok()?; + let output = String::from_utf8(output.stdout).ok()?; + let fullname = output.split('\n').nth(1)?; + let cardname = fullname.replace(' ', "-").to_lowercase(); + let compute_type = format!("{num_shard}-{cardname}"); + Some(compute_type) +} + fn spawn_webserver( + num_shard: usize, args: Args, shutdown: Arc, shutdown_receiver: &mpsc::Receiver<()>, @@ -1072,6 +1085,13 @@ fn spawn_webserver( envs.push(("HUGGING_FACE_HUB_TOKEN".into(), api_token.into())) }; + // Parse Compute type + if let Ok(compute_type) = env::var("COMPUTE_TYPE") { + envs.push(("COMPUTE_TYPE".into(), compute_type.into())) + } else if let Some(compute_type) = compute_type(num_shard) { + envs.push(("COMPUTE_TYPE".into(), compute_type.into())) + } + let mut webserver = match Command::new("text-generation-router") .args(router_args) .envs(envs) @@ -1265,8 +1285,8 @@ fn main() -> Result<(), LauncherError> { return Ok(()); } - let mut webserver = - spawn_webserver(args, shutdown.clone(), &shutdown_receiver).map_err(|err| { + let mut webserver = spawn_webserver(num_shard, args, shutdown.clone(), &shutdown_receiver) + .map_err(|err| { shutdown_shards(shutdown.clone(), &shutdown_receiver); err })?; diff --git a/router/src/server.rs b/router/src/server.rs index 39d1de38..52ed03df 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -67,7 +67,7 @@ async fn compat_generate( // switch on stream if req.stream { - Ok(generate_stream(infer,compute_type, Json(req.into())) + Ok(generate_stream(infer, compute_type, Json(req.into())) .await .into_response()) } else { @@ -372,7 +372,7 @@ async fn generate_stream_internal( let compute_characters = req.inputs.chars().count(); let mut headers = HeaderMap::new(); - headers.insert("x-compute-type",compute_type.parse().unwrap()); + headers.insert("x-compute-type", compute_type.parse().unwrap()); headers.insert( "x-compute-characters", compute_characters.to_string().parse().unwrap(), @@ -649,13 +649,22 @@ async fn chat_completions( ) }; - let (headers, response_stream) = - generate_stream_internal(infer, compute_type, Json(generate_request), on_message_callback).await; + let (headers, response_stream) = generate_stream_internal( + infer, + compute_type, + Json(generate_request), + on_message_callback, + ) + .await; let sse = Sse::new(response_stream).keep_alive(KeepAlive::default()); Ok((headers, sse).into_response()) } else { - let (headers, Json(generation)) = - generate(Extension(infer), Extension(compute_type), Json(generate_request)).await?; + let (headers, Json(generation)) = generate( + Extension(infer), + Extension(compute_type), + Json(generate_request), + ) + .await?; let current_time = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -943,7 +952,8 @@ pub async fn run( Router::new().route("/invocations", post(compat_generate)) // Use 'compat_generate' otherwise }; - let compute_type = ComputeType(std::env::var("COMPUTE_TYPE").unwrap_or("gpu+optimized".to_string())); + let compute_type = + ComputeType(std::env::var("COMPUTE_TYPE").unwrap_or("gpu+optimized".to_string())); // Combine routes and layers let app = Router::new() From 2d56f106a60c7b698705494e7539f8a7e4c85dd9 Mon Sep 17 00:00:00 2001 From: freitng <153592523+freitng@users.noreply.github.com> Date: Mon, 29 Jan 2024 17:02:57 +0100 Subject: [PATCH 544/793] Modify default for max_new_tokens in python client (#1336) # What does this PR do? Since ([#1097](https://github.com/huggingface/text-generation-inference/pull/1097)) the clients do not need to specify a max_length anymore. However, the python client in this repo had not yet been adapted to these changes. This PR makes it possible to use the python client and not provide max_new_tokens. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [x] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- clients/python/tests/test_client.py | 16 ++++++++++++++++ clients/python/text_generation/client.py | 8 ++++---- clients/python/text_generation/types.py | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 1e25e1b1..775e7a6c 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -21,6 +21,22 @@ def test_generate(flan_t5_xxl_url, hf_headers): assert not response.details.tokens[0].special +def test_generate_max_new_tokens_not_set(flan_t5_xxl_url, hf_headers): + client = Client(flan_t5_xxl_url, hf_headers) + response = client.generate("test", decoder_input_details=True) + + assert response.generated_text != "" + assert response.details.finish_reason == FinishReason.EndOfSequenceToken + assert response.details.generated_tokens > 1 + assert response.details.seed is None + assert len(response.details.prefill) == 1 + assert response.details.prefill[0] == InputToken(id=0, text="", logprob=None) + assert len(response.details.tokens) > 1 + assert response.details.tokens[0].id == 3 + assert response.details.tokens[0].text == " " + assert not response.details.tokens[0].special + + def test_generate_best_of(flan_t5_xxl_url, hf_headers): client = Client(flan_t5_xxl_url, hf_headers) response = client.generate( diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index 0bf80f8c..63b5258d 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -62,7 +62,7 @@ def generate( self, prompt: str, do_sample: bool = False, - max_new_tokens: int = 20, + max_new_tokens: Optional[int] = None, best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, @@ -157,7 +157,7 @@ def generate_stream( self, prompt: str, do_sample: bool = False, - max_new_tokens: int = 20, + max_new_tokens: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, @@ -312,7 +312,7 @@ async def generate( self, prompt: str, do_sample: bool = False, - max_new_tokens: int = 20, + max_new_tokens: Optional[int] = None, best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, @@ -405,7 +405,7 @@ async def generate_stream( self, prompt: str, do_sample: bool = False, - max_new_tokens: int = 20, + max_new_tokens: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index aa02d8d8..7fa8033e 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -9,7 +9,7 @@ class Parameters(BaseModel): # Activate logits sampling do_sample: bool = False # Maximum number of generated tokens - max_new_tokens: int = 20 + max_new_tokens: Optional[int] = None # The parameter for repetition penalty. 1.0 means no penalty. # See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. repetition_penalty: Optional[float] = None From 0595bf3e9a15807d038e41f09c0f7e1c595d2417 Mon Sep 17 00:00:00 2001 From: dtlzhuangz <139844877+dtlzhuangz@users.noreply.github.com> Date: Wed, 31 Jan 2024 19:05:49 +0800 Subject: [PATCH 545/793] feat: eetq gemv optimization when batch_size <= 4 (#1502) # What does this PR do? Add TensorRT-LLM weight-only GEMV kernel support. We extract GEMV kernel from [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM/tree/main/cpp/tensorrt_llm/kernels/weightOnlyBatchedGemv) to accelerate the decode speed of EETQ when batch_size is smaller or equal to 4. - Features 1. There is almost no loss of quantization accuracy. 2. The speed of decoding is 13% - 27% faster than original EETQ which utilizes GEMM kernel. - Test Below is our test on 3090. Environment: torch=2.0.1, cuda=11.8, nvidia driver: 525.78.01 prompt=1024, max_new_tokens=50 ![image](https://github.com/huggingface/text-generation-inference/assets/139844877/98e63b23-23cd-452f-91bd-55ccdc9b7021) ![image](https://github.com/huggingface/text-generation-inference/assets/139844877/5c3132ff-fc1c-4b20-a83f-59b3d5f586b7) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/Makefile-eetq | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/Makefile-eetq b/server/Makefile-eetq index 5e8e9830..8c060987 100644 --- a/server/Makefile-eetq +++ b/server/Makefile-eetq @@ -1,4 +1,4 @@ -eetq_commit := 323827dd471458a84e9c840f614e4592b157a4b1 +eetq_commit := 71adb5e191bb8290069a580abff0355d7b2dd5c9 eetq: # Clone eetq @@ -6,7 +6,7 @@ eetq: git clone https://github.com/NetEase-FuXi/EETQ.git eetq build-eetq: eetq - cd eetq && git fetch && git checkout $(eetq_commit) + cd eetq && git fetch && git checkout $(eetq_commit) && git submodule update --init --recursive cd eetq && python setup.py build install-eetq: build-eetq From 2ae36a97fd9439de883bb5600db26634ee34d494 Mon Sep 17 00:00:00 2001 From: drbh Date: Wed, 31 Jan 2024 11:26:22 -0500 Subject: [PATCH 546/793] fix: improve messages api docs content and formatting (#1506) This PR simply updates the messages api docs to address content changes and make format consistent --- docs/source/messages_api.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/source/messages_api.md b/docs/source/messages_api.md index 899de865..1e342686 100644 --- a/docs/source/messages_api.md +++ b/docs/source/messages_api.md @@ -1,8 +1,8 @@ # Messages API -_Messages API is compatible to OpenAI Chat Completion API_ +Text Generation Inference (TGI) now supports the Messages API, which is fully compatible with the OpenAI Chat Completion API. This feature is available starting from version 1.4.0. You can use OpenAI's client libraries or third-party libraries expecting OpenAI schema to interact with TGI's Messages API. Below are some examples of how to utilize this compatibility. -Text Generation Inference (TGI) now supports the Message API which is fully compatible with the OpenAI Chat Completion API. This means you can use OpenAI's client libraries to interact with TGI's Messages API. Below are some examples of how to utilize this compatibility. +> **Note:** The Messages API is supported from TGI version 1.4.0 and above. Ensure you are using a compatible version to access this feature. ## Making a Request @@ -87,7 +87,7 @@ TGI can be deployed on various cloud providers for scalable and robust text gene ## Amazon SageMaker -To enable the Messages API in Amazon SageMaker you need to set the environment variable `MESSAGES_API_ENABLED=true`. +To enable the Messages API in Amazon SageMaker you need to set the environment variable `MESSAGES_API_ENABLED=true`. This will modify the `/invocations` route to accept Messages dictonaries consisting out of role and content. See the example below on how to deploy Llama with the new Messages API. @@ -98,30 +98,30 @@ import boto3 from sagemaker.huggingface import HuggingFaceModel, get_huggingface_llm_image_uri try: - role = sagemaker.get_execution_role() + role = sagemaker.get_execution_role() except ValueError: - iam = boto3.client('iam') - role = iam.get_role(RoleName='sagemaker_execution_role')['Role']['Arn'] + iam = boto3.client('iam') + role = iam.get_role(RoleName='sagemaker_execution_role')['Role']['Arn'] # Hub Model configuration. https://huggingface.co/models hub = { - 'HF_MODEL_ID':'HuggingFaceH4/zephyr-7b-beta', - 'SM_NUM_GPUS': json.dumps(1), - 'MESSAGES_API_ENABLED': True + 'HF_MODEL_ID':'HuggingFaceH4/zephyr-7b-beta', + 'SM_NUM_GPUS': json.dumps(1), + 'MESSAGES_API_ENABLED': True } # create Hugging Face Model Class huggingface_model = HuggingFaceModel( - image_uri=get_huggingface_llm_image_uri("huggingface",version="1.4.0"), - env=hub, - role=role, + image_uri=get_huggingface_llm_image_uri("huggingface",version="1.4.0"), + env=hub, + role=role, ) # deploy model to SageMaker Inference predictor = huggingface_model.deploy( - initial_instance_count=1, - instance_type="ml.g5.2xlarge", - container_startup_health_check_timeout=300, + initial_instance_count=1, + instance_type="ml.g5.2xlarge", + container_startup_health_check_timeout=300, ) # send request @@ -131,4 +131,4 @@ predictor.predict({ {"role": "user", "content": "What is deep learning?"} ] }) -``` \ No newline at end of file +``` From 13c62be467953c4762eeb15fa418bc0f6c716acd Mon Sep 17 00:00:00 2001 From: Dean Wyatte <2512762+dwyatte@users.noreply.github.com> Date: Thu, 1 Feb 2024 01:34:11 -0700 Subject: [PATCH 547/793] GPTNeoX: Use static rotary embedding (#1498) # What does this PR do? `transformers` 4.35 removed rotary embeddings from GPTNeoX's weights ([link to line diff](https://github.com/huggingface/transformers/commit/253f9a3f9716d08a81fb305fe71f983122eb608b#diff-0e2a05d86c82e96f516db8c14070ceb36f53ca44c6bc21a9cd92ad2e777b9cf1R298)). This applies the same fix as https://github.com/huggingface/text-generation-inference/pull/793 which generates them on-the-fly using the appropriate value from the config file Fixes https://github.com/huggingface/text-generation-inference/issues/1460 ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [x] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? @OlivierDehaene OR @Narsil --- .../models/custom_modeling/flash_neox_modeling.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py index eea5f787..3ee344e4 100644 --- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py @@ -91,6 +91,8 @@ def __init__(self, config, prefix, weights): self.hidden_size = hidden_size self.head_size = hidden_size // num_heads + self.rotary_dim = int(config.rotary_pct * self.head_size) + if self.num_heads % weights.process_group.size() != 0: raise ValueError( f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} " @@ -98,8 +100,11 @@ def __init__(self, config, prefix, weights): ) self.num_heads = self.num_heads // weights.process_group.size() - self.rotary_emb = PositionRotaryEmbedding.load( - config=config, prefix=f"{prefix}.rotary_emb", weights=weights + self.rotary_emb = PositionRotaryEmbedding.static( + config=config, + dim=self.rotary_dim, + base=config.rotary_emb_base, + device=weights.device, ) self.softmax_scale = self.head_size ** (-0.5) From 94d243b3d7916879f1735d4a6f231e915765c1c4 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 1 Feb 2024 10:23:37 +0100 Subject: [PATCH 548/793] Freshen up the README. --- README.md | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5fdb9f14..c4d84efa 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ to power Hugging Chat, the Inference API and Inference Endpoint. - [Local Install](#local-install) - [CUDA Kernels](#cuda-kernels) - [Optimized architectures](#optimized-architectures) -- [Run Falcon](#run-falcon) +- [Run Mistral](#run-a-model) - [Run](#run) - [Quantization](#quantization) - [Develop](#develop) @@ -42,7 +42,11 @@ Text Generation Inference (TGI) is a toolkit for deploying and serving Large Lan - Token streaming using Server-Sent Events (SSE) - Continuous batching of incoming requests for increased total throughput - Optimized transformers code for inference using [Flash Attention](https://github.com/HazyResearch/flash-attention) and [Paged Attention](https://github.com/vllm-project/vllm) on the most popular architectures -- Quantization with [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) and [GPT-Q](https://arxiv.org/abs/2210.17323) +- Quantization with : + - [bitsandbytes](https://github.com/TimDettmers/bitsandbytes) + - [GPT-Q](https://arxiv.org/abs/2210.17323) + - [EETQ](https://github.com/NetEase-FuXi/EETQ) + - [AWQ](https://github.com/casper-hansen/AutoAWQ) - [Safetensors](https://github.com/huggingface/safetensors) weight loading - Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226) - Logits warper (temperature scaling, top-p, top-k, repetition penalty, more details see [transformers.LogitsProcessor](https://huggingface.co/docs/transformers/internal/generation_utils#transformers.LogitsProcessor)) @@ -51,6 +55,14 @@ Text Generation Inference (TGI) is a toolkit for deploying and serving Large Lan - Custom Prompt Generation: Easily generate text by providing custom prompts to guide the model's output - Fine-tuning Support: Utilize fine-tuned models for specific tasks to achieve higher accuracy and performance +### Hardware support + +- [Nvidia](https://github.com/huggingface/text-generation-inference/pkgs/container/text-generation-inference) +- [AMD](https://github.com/huggingface/text-generation-inference/pkgs/container/text-generation-inference) (-rocm) +- [Inferentia](https://github.com/huggingface/optimum-neuron/tree/main/text-generation-inference) +- [Intel GPU](https://github.com/huggingface/text-generation-inference/pull/1475) +- [Gaudi](https://github.com/huggingface/tgi-gaudi) + ## Get Started @@ -154,7 +166,7 @@ Python 3.9, e.g. using `conda`: ```shell curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -conda create -n text-generation-inference python=3.9 +conda create -n text-generation-inference python=3.11 conda activate text-generation-inference ``` @@ -180,7 +192,7 @@ Then run: ```shell BUILD_EXTENSIONS=True make install # Install repository and HF/transformer fork with CUDA kernels -make run-falcon-7b-instruct +text-generation-launcher --model-id mistralai/Mistral-7B-Instruct-v0.2 ``` **Note:** on some machines, you may also need the OpenSSL libraries and gcc. On Linux machines, run: @@ -189,13 +201,6 @@ make run-falcon-7b-instruct sudo apt-get install libssl-dev gcc -y ``` -### CUDA Kernels - -The custom CUDA kernels are only tested on NVIDIA A100, AMD MI210 and AMD MI250. If you have any installation or runtime issues, you can remove -the kernels by using the `DISABLE_CUSTOM_KERNELS=True` environment variable. - -Be aware that the official Docker image has them enabled by default. - ## Optimized architectures TGI works out of the box to serve optimized models for all modern models. They can be found in [this list](https://huggingface.co/docs/text-generation-inference/supported_models). @@ -210,12 +215,12 @@ or -## Run Falcon +## Run locally ### Run ```shell -make run-falcon-7b-instruct +text-generation-launcher --model-id mistralai/Mistral-7B-Instruct-v0.2 ``` ### Quantization @@ -223,7 +228,7 @@ make run-falcon-7b-instruct You can also quantize the weights with bitsandbytes to reduce the VRAM requirement: ```shell -make run-falcon-7b-instruct-quantize +text-generation-launcher --model-id mistralai/Mistral-7B-Instruct-v0.2 --quantize ``` 4bit quantization is available using the [NF4 and FP4 data types from bitsandbytes](https://arxiv.org/pdf/2305.14314.pdf). It can be enabled by providing `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` as a command line argument to `text-generation-launcher`. From 9ad7b6a1a12f8cd6b715be9f0ca85603e0a2b002 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 1 Feb 2024 13:29:04 +0100 Subject: [PATCH 549/793] Hotfix the / health - route. (#1515) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- router/src/server.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/router/src/server.rs b/router/src/server.rs index 52ed03df..b4d26158 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -936,6 +936,7 @@ pub async fn run( // Define base and health routes let base_routes = Router::new() .route("/", post(compat_generate)) + .route("/", get(health)) .route("/info", get(get_model_info)) .route("/generate", post(generate)) .route("/generate_stream", post(generate_stream)) From 1e03b61b5c56e2ed5c723457df21cc18d48c1854 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 1 Feb 2024 14:36:10 +0000 Subject: [PATCH 550/793] Revert "Modify default for max_new_tokens in python client (#1336)" This reverts commit 2d56f106a60c7b698705494e7539f8a7e4c85dd9. It causes a breaking in our integrations-tests. --- clients/python/tests/test_client.py | 16 ---------------- clients/python/text_generation/client.py | 8 ++++---- clients/python/text_generation/types.py | 2 +- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 775e7a6c..1e25e1b1 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -21,22 +21,6 @@ def test_generate(flan_t5_xxl_url, hf_headers): assert not response.details.tokens[0].special -def test_generate_max_new_tokens_not_set(flan_t5_xxl_url, hf_headers): - client = Client(flan_t5_xxl_url, hf_headers) - response = client.generate("test", decoder_input_details=True) - - assert response.generated_text != "" - assert response.details.finish_reason == FinishReason.EndOfSequenceToken - assert response.details.generated_tokens > 1 - assert response.details.seed is None - assert len(response.details.prefill) == 1 - assert response.details.prefill[0] == InputToken(id=0, text="", logprob=None) - assert len(response.details.tokens) > 1 - assert response.details.tokens[0].id == 3 - assert response.details.tokens[0].text == " " - assert not response.details.tokens[0].special - - def test_generate_best_of(flan_t5_xxl_url, hf_headers): client = Client(flan_t5_xxl_url, hf_headers) response = client.generate( diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index 63b5258d..0bf80f8c 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -62,7 +62,7 @@ def generate( self, prompt: str, do_sample: bool = False, - max_new_tokens: Optional[int] = None, + max_new_tokens: int = 20, best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, @@ -157,7 +157,7 @@ def generate_stream( self, prompt: str, do_sample: bool = False, - max_new_tokens: Optional[int] = None, + max_new_tokens: int = 20, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, @@ -312,7 +312,7 @@ async def generate( self, prompt: str, do_sample: bool = False, - max_new_tokens: Optional[int] = None, + max_new_tokens: int = 20, best_of: Optional[int] = None, repetition_penalty: Optional[float] = None, return_full_text: bool = False, @@ -405,7 +405,7 @@ async def generate_stream( self, prompt: str, do_sample: bool = False, - max_new_tokens: Optional[int] = None, + max_new_tokens: int = 20, repetition_penalty: Optional[float] = None, return_full_text: bool = False, seed: Optional[int] = None, diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index 7fa8033e..aa02d8d8 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -9,7 +9,7 @@ class Parameters(BaseModel): # Activate logits sampling do_sample: bool = False # Maximum number of generated tokens - max_new_tokens: Optional[int] = None + max_new_tokens: int = 20 # The parameter for repetition penalty. 1.0 means no penalty. # See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. repetition_penalty: Optional[float] = None From ee1cf51ce796e4b034eedaf3e909b4c902eae70c Mon Sep 17 00:00:00 2001 From: drbh Date: Thu, 1 Feb 2024 09:39:32 -0500 Subject: [PATCH 551/793] fix: tokenizer config should use local model path when possible (#1518) This PR fixes the issue with loading a local tokenizer config. Previously the default functionality would look in the current working directory. Now if a local model path is specified we will check that directory for the tokenizer_config. ## Examples of valid commands uses tokenizer_config from hub ``` text-generation-launcher --model-id HuggingFaceH4/zephyr-7b-beta ``` use tokenizer_config from local model path ``` text-generation-launcher \ --model-id ~/.cache/huggingface/hub/models--HuggingFaceH4--zephyr-7b-beta/snapshots/dc24cabd13eacd3ae3a5fe574bd645483a335a4a/ ``` use specific tokenizer_config file ``` text-generation-launcher \ --model-id ~/.cache/huggingface/hub/models--HuggingFaceH4--zephyr-7b-beta/snapshots/dc24cabd13eacd3ae3a5fe574bd645483a335a4a/ \ --tokenizer-config-path ~/.cache/huggingface/hub/models--HuggingFaceH4--zephyr-7b-beta/snapshots/dc24cabd13eacd3ae3a5fe574bd645483a335a4a/tokenizer_config.json ``` --------- Co-authored-by: Nicolas Patry --- router/src/lib.rs | 2 +- router/src/main.rs | 49 +++++++++++++++++++++++++--------------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index fc5670a0..07360e78 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -37,7 +37,7 @@ pub struct HubTokenizerConfig { } impl HubTokenizerConfig { - pub fn from_file(filename: &str) -> Self { + pub fn from_file(filename: &std::path::Path) -> Self { let content = std::fs::read_to_string(filename).unwrap(); serde_json::from_str(&content).unwrap_or_default() } diff --git a/router/src/main.rs b/router/src/main.rs index 495fd5bc..2a080468 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -154,12 +154,6 @@ async fn main() -> Result<(), RouterError> { let local_path = Path::new(&tokenizer_name); let local_model = local_path.exists() && local_path.is_dir(); - // Load tokenizer config - // This will be used to format the chat template - let local_tokenizer_config_path = - tokenizer_config_path.unwrap_or("tokenizer_config.json".to_string()); - let local_tokenizer_config = Path::new(&local_tokenizer_config_path).exists(); - // Shared API builder initialization let api_builder = || { let mut builder = ApiBuilder::new() @@ -230,24 +224,35 @@ async fn main() -> Result<(), RouterError> { }; // Load tokenizer config if found locally, or check if we can get it from the API if needed - let tokenizer_config = if local_tokenizer_config { + let tokenizer_config = if let Some(path) = tokenizer_config_path { + tracing::info!("Using local tokenizer config from user specified path"); + HubTokenizerConfig::from_file(&std::path::PathBuf::from(path)) + } else if local_model { tracing::info!("Using local tokenizer config"); - HubTokenizerConfig::from_file(&local_tokenizer_config_path) - } else if let Some(api) = api { - tracing::info!("Using the Hugging Face API to retrieve tokenizer config"); - get_tokenizer_config(&api.repo(Repo::with_revision( - tokenizer_name.to_string(), - RepoType::Model, - revision.unwrap_or_else(|| "main".to_string()), - ))) - .await - .unwrap_or_else(|| { - tracing::warn!("Could not retrieve tokenizer config from the Hugging Face hub."); - HubTokenizerConfig::default() - }) + HubTokenizerConfig::from_file(&local_path.join("tokenizer_config.json")) } else { - tracing::warn!("Could not find tokenizer config locally and no revision specified"); - HubTokenizerConfig::default() + match api { + Some(api) => { + tracing::info!("Using the Hugging Face API to retrieve tokenizer config"); + let repo = Repo::with_revision( + tokenizer_name.to_string(), + RepoType::Model, + revision.unwrap_or("main".to_string()), + ); + get_tokenizer_config(&api.repo(repo)) + .await + .unwrap_or_else(|| { + tracing::warn!( + "Could not retrieve tokenizer config from the Hugging Face hub." + ); + HubTokenizerConfig::default() + }) + } + None => { + tracing::warn!("Could not find tokenizer config locally and no API specified"); + HubTokenizerConfig::default() + } + } }; if tokenizer.is_none() { From 0e97af456af3102ed4f927f7b7e870ec976079ae Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 1 Feb 2024 16:26:48 +0100 Subject: [PATCH 552/793] Updating tokenizers. (#1517) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- router/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/Cargo.toml b/router/Cargo.toml index f6f16dae..1a7ceb70 100644 --- a/router/Cargo.toml +++ b/router/Cargo.toml @@ -32,7 +32,7 @@ reqwest = { version = "0.11.20", features = [] } serde = "1.0.188" serde_json = "1.0.107" thiserror = "1.0.48" -tokenizers = { version = "0.14.0", features = ["http"] } +tokenizers = { version = "0.15.1", features = ["http"] } tokio = { version = "1.32.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] } tokio-stream = "0.1.14" tower-http = { version = "0.4.4", features = ["cors"] } From 3ab578b4160b200ad601bbd30bd8ecf39b979326 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Fri, 2 Feb 2024 14:05:30 +0100 Subject: [PATCH 553/793] [docs] Fix link to Install CLI (#1526) # What does this PR do? Attempts to fix a link from Using TGI CLI to Installation. ## Before submitting - [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? --- docs/source/basic_tutorials/using_cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/basic_tutorials/using_cli.md b/docs/source/basic_tutorials/using_cli.md index 82c10e6b..a3a65f60 100644 --- a/docs/source/basic_tutorials/using_cli.md +++ b/docs/source/basic_tutorials/using_cli.md @@ -1,6 +1,6 @@ # Using TGI CLI -You can use TGI command-line interface (CLI) to download weights, serve and quantize models, or get information on serving parameters. To install the CLI, please refer to [the installation section](./installation#install-cli). +You can use TGI command-line interface (CLI) to download weights, serve and quantize models, or get information on serving parameters. To install the CLI, please refer to [the installation section](../installation#install-cli). `text-generation-server` lets you download the model with `download-weights` command like below 👇 From 0da00be52c9e591f8890ab07eea05cc15b9b127b Mon Sep 17 00:00:00 2001 From: drbh Date: Fri, 2 Feb 2024 10:31:11 -0500 Subject: [PATCH 554/793] feat: add ie update to message docs (#1523) update messages api docs and add Hugging Face Inference Endpoints integrations section/instructions --------- Co-authored-by: Philipp Schmid <32632186+philschmid@users.noreply.github.com> --- docs/source/messages_api.md | 45 +++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/docs/source/messages_api.md b/docs/source/messages_api.md index 1e342686..939850aa 100644 --- a/docs/source/messages_api.md +++ b/docs/source/messages_api.md @@ -4,6 +4,15 @@ Text Generation Inference (TGI) now supports the Messages API, which is fully co > **Note:** The Messages API is supported from TGI version 1.4.0 and above. Ensure you are using a compatible version to access this feature. +#### Table of Contents + +- [Making a Request](#making-a-request) +- [Streaming](#streaming) +- [Synchronous](#synchronous) +- [Hugging Face Inference Endpoints](#hugging-face-inference-endpoints) +- [Cloud Providers](#cloud-providers) + - [Amazon SageMaker](#amazon-sagemaker) + ## Making a Request You can make a request to TGI's Messages API using `curl`. Here's an example: @@ -81,6 +90,38 @@ chat_completion = client.chat.completions.create( print(chat_completion) ``` +## Hugging Face Inference Endpoints + +The Messages API is integrated with [Inference Endpoints](https://huggingface.co/inference-endpoints/dedicated). +Every endpoint that uses "Text Generation Inference" with an LLM, which has a chat template can now be used. Below is an example of how to use IE with TGI using OpenAI's Python client library: + +> **Note:** Make sure to replace `base_url` with your endpoint URL and to include `v1/` at the end of the URL. The `api_key` should be replaced with your Hugging Face API key. + +```python +from openai import OpenAI + +# init the client but point it to TGI +client = OpenAI( + # replace with your endpoint url, make sure to include "v1/" at the end + base_url="https://vlzz10eq3fol3429.us-east-1.aws.endpoints.huggingface.cloud/v1/", + # replace with your API key + api_key="hf_XXX" +) + +chat_completion = client.chat.completions.create( + model="tgi", + messages=[ + {"role": "system", "content": "You are a helpful assistant." }, + {"role": "user", "content": "What is deep learning?"} + ], + stream=True +) + +# iterate and print stream +for message in chat_completion: + print(message.choices[0].delta.content, end="") +``` + ## Cloud Providers TGI can be deployed on various cloud providers for scalable and robust text generation. One such provider is Amazon SageMaker, which has recently added support for TGI. Here's how you can deploy TGI on Amazon SageMaker: @@ -114,7 +155,7 @@ hub = { huggingface_model = HuggingFaceModel( image_uri=get_huggingface_llm_image_uri("huggingface",version="1.4.0"), env=hub, - role=role, + role=role, ) # deploy model to SageMaker Inference @@ -123,7 +164,7 @@ predictor = huggingface_model.deploy( instance_type="ml.g5.2xlarge", container_startup_health_check_timeout=300, ) - + # send request predictor.predict({ "messages": [ From 17345402114e3d4a1f1c0a7fd650c734f7a992f9 Mon Sep 17 00:00:00 2001 From: drbh Date: Wed, 7 Feb 2024 03:35:53 -0500 Subject: [PATCH 555/793] =?UTF-8?q?feat:=20use=20existing=20add=5Fgenerati?= =?UTF-8?q?on=5Fprompt=20variable=20from=20config=20in=20temp=E2=80=A6=20(?= =?UTF-8?q?#1533)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds support to read the `add_generation_prompt` from the config and use it in the chat template. If `add_generation_prompt` does not exist we default to false --- router/src/infer.rs | 64 ++++++++++++++++++++++++++++++++++++++------- router/src/lib.rs | 1 + 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index 5f078ba0..4da0da0a 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -198,6 +198,7 @@ impl Infer { messages, eos_token: eos_token.as_deref(), bos_token: bos_token.as_deref(), + add_generation_prompt: true, }) .map_err(|e| { metrics::increment_counter!("tgi_request_failure", "err" => "template"); @@ -806,21 +807,14 @@ mod tests { ], bos_token: Some("[BOS]"), eos_token: Some("[EOS]"), + add_generation_prompt: true, }; let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); assert_eq!( result, - r#"### User: -Hi! - -### Assistant: -Hello how can I help?### User: -What is Deep Learning? - -### Assistant: -magic!"# + "### User:\nHi!\n\n### Assistant:\nHello how can I help?### User:\nWhat is Deep Learning?\n\n### Assistant:\nmagic!### Assistant:\n" ); } @@ -878,6 +872,7 @@ magic!"# ], bos_token: Some("[BOS]"), eos_token: Some("[EOS]"), + add_generation_prompt: true, }; let result = tmpl.unwrap().render(chat_template_inputs); //.err().unwrap(); @@ -943,9 +938,60 @@ magic!"# ], bos_token: Some("[BOS]"), eos_token: Some("[EOS]"), + add_generation_prompt: true, }; let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); assert_eq!(result, "[BOS][INST] Hi! [/INST]Hello how can I help?[EOS][INST] What is Deep Learning? [/INST]magic![EOS]"); } + + #[test] + fn test_chat_template_valid_with_add_generation_prompt() { + let mut env = Environment::new(); + env.add_function("raise_exception", raise_exception); + + let source = r#" + {% for message in messages %} + {{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}} + {% endfor %} + {% if add_generation_prompt %} + {{ '<|im_start|>assistant\n' }} + {% endif %}"#; + + // trim all the whitespace + let source = source + .lines() + .map(|line| line.trim()) + .collect::>() + .join(""); + + let tmpl = env.template_from_str(&source); + + let chat_template_inputs = ChatTemplateInputs { + messages: vec![ + Message { + role: "user".to_string(), + content: "Hi!".to_string(), + }, + Message { + role: "assistant".to_string(), + content: "Hello how can I help?".to_string(), + }, + Message { + role: "user".to_string(), + content: "What is Deep Learning?".to_string(), + }, + Message { + role: "assistant".to_string(), + content: "magic!".to_string(), + }, + ], + bos_token: Some("[BOS]"), + eos_token: Some("[EOS]"), + add_generation_prompt: true, + }; + + let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); + assert_eq!(result, "<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\nHello how can I help?<|im_end|>\n<|im_start|>user\nWhat is Deep Learning?<|im_end|>\n<|im_start|>assistant\nmagic!<|im_end|>\n<|im_start|>assistant\n"); + } } diff --git a/router/src/lib.rs b/router/src/lib.rs index 07360e78..e85519cc 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -398,6 +398,7 @@ pub(crate) struct ChatTemplateInputs<'a> { messages: Vec, bos_token: Option<&'a str>, eos_token: Option<&'a str>, + add_generation_prompt: bool, } #[derive(Clone, Deserialize, ToSchema, Serialize)] From bd405e035b9e05d4b1e74e029ff1d5de86854ea0 Mon Sep 17 00:00:00 2001 From: drbh Date: Thu, 8 Feb 2024 04:19:45 -0500 Subject: [PATCH 556/793] Impl simple mamba model (#1480) This draft PR is a work in progress implementation of the mamba model. This PR currently loads weights, and produces correct logits after a single pass. This PR still needs to correctly integrate this model so it produces tokens as expected, and apply optimization to avoid all copies during runtime/unnecessary operations. #### Helpful resources [Mamba: Linear-Time Sequence Modeling with Selective State Spaces (Albert Gu and Tri Dao)](https://arxiv.org/abs/2312.00752) https://github.com/johnma2006/mamba-minimal https://github.com/huggingface/candle/blob/main/candle-examples/examples/mamba-minimal/model.rs https://github.com/huggingface/transformers/pull/28094 Notes: this dev work is currently targeting `state-spaces/mamba-130m`, so if you want to test please use that model. Additionally when starting the router the prefill needs to be limited: `cargo run -- --max-batch-prefill-tokens 768 --max-input-length 768` ## Update / Current State Integration tests have been added and basic functionality such as model loading is supported. ```bash cd integration-tests pytest -vv models/test_fused_kernel_mamba.py ``` - [x] add tests - [x] load model - [x] make simple request - [ ] resolve warmup issue - [ ] resolve output issues fetching models tested during dev ```bash text-generation-server download-weights state-spaces/mamba-130m text-generation-server download-weights state-spaces/mamba-1.4b text-generation-server download-weights state-spaces/mamba-2.8b ``` The server can be run ```bash cd server MASTER_ADDR=127.0.0.1 MASTER_PORT=5555 python text_generation_server/cli.py serve state-spaces/mamba-2.8b ``` router ```bash cargo run ``` make a request ```bash curl -s localhost:3000/generate \ -X POST \ -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \ -H 'Content-Type: application/json' | jq ``` response ```json { "generated_text": "\n\nDeep learning is a machine learning technique that uses a deep neural network to learn from data." } ``` --------- Co-authored-by: Nicolas Patry --- Dockerfile | 10 + .../__snapshots__/test_mamba/test_mamba.json | 73 ++ .../test_mamba/test_mamba_all_params.json | 99 +++ .../test_mamba/test_mamba_load.json | 398 +++++++++++ integration-tests/models/test_mamba.py | 59 ++ server/.gitignore | 1 + server/Makefile | 1 + server/Makefile-selective-scan | 28 + .../text_generation_server/models/__init__.py | 29 +- .../models/custom_modeling/mamba_modeling.py | 194 ++++++ server/text_generation_server/models/mamba.py | 656 ++++++++++++++++++ 11 files changed, 1547 insertions(+), 1 deletion(-) create mode 100644 integration-tests/models/__snapshots__/test_mamba/test_mamba.json create mode 100644 integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json create mode 100644 integration-tests/models/__snapshots__/test_mamba/test_mamba_load.json create mode 100644 integration-tests/models/test_mamba.py create mode 100644 server/Makefile-selective-scan create mode 100644 server/text_generation_server/models/custom_modeling/mamba_modeling.py create mode 100644 server/text_generation_server/models/mamba.py diff --git a/Dockerfile b/Dockerfile index b6c5b2ed..6818005f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -154,6 +154,12 @@ COPY server/Makefile-vllm Makefile # Build specific version of vllm RUN make build-vllm-cuda +# Build mamba kernels +FROM kernel-builder as mamba-builder +WORKDIR /usr/src +COPY server/Makefile-selective-scan Makefile +RUN make build-all + # Build megablocks FROM kernel-builder as megablocks-builder @@ -205,6 +211,10 @@ COPY --from=eetq-kernels-builder /usr/src/eetq/build/lib.linux-x86_64-cpython-31 # Copy builds artifacts from vllm builder COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages +# Copy build artifacts from mamba builder +COPY --from=mamba-builder /usr/src/mamba/build/lib.linux-x86_64-cpython-310/ /opt/conda/lib/python3.10/site-packages +COPY --from=mamba-builder /usr/src/causal-conv1d/build/lib.linux-x86_64-cpython-310/ /opt/conda/lib/python3.10/site-packages + # Install flash-attention dependencies RUN pip install einops --no-cache-dir diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba.json new file mode 100644 index 00000000..4435f215 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba.json @@ -0,0 +1,73 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [], + "seed": null, + "tokens": [ + { + "id": 187, + "logprob": -0.3552246, + "special": false, + "text": "\n" + }, + { + "id": 187, + "logprob": -0.38378906, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.140625, + "special": false, + "text": "Deep" + }, + { + "id": 4715, + "logprob": -0.5551758, + "special": false, + "text": " learning" + }, + { + "id": 310, + "logprob": -0.59033203, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.70654297, + "special": false, + "text": " a" + }, + { + "id": 747, + "logprob": -2.0410156, + "special": false, + "text": " new" + }, + { + "id": 1511, + "logprob": -2.3789062, + "special": false, + "text": " type" + }, + { + "id": 273, + "logprob": -0.0026435852, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.2841797, + "special": false, + "text": " machine" + } + ], + "top_tokens": null + }, + "generated_text": "\n\nDeep learning is a new type of machine" +} diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json new file mode 100644 index 00000000..052c1c69 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json @@ -0,0 +1,99 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 2502, + "logprob": null, + "text": " red" + }, + { + "id": 13, + "logprob": -2.5234375, + "text": "," + }, + { + "id": 8862, + "logprob": -3.4433594, + "text": " yellow" + }, + { + "id": 13, + "logprob": -0.43017578, + "text": "," + }, + { + "id": 209, + "logprob": -8.21875, + "text": " " + } + ], + "seed": 0, + "tokens": [ + { + "id": 187, + "logprob": 0.0, + "special": false, + "text": "\n" + }, + { + "id": 395, + "logprob": -0.46411133, + "special": false, + "text": "and" + }, + { + "id": 13735, + "logprob": -2.1132812, + "special": false, + "text": " orange" + }, + { + "id": 313, + "logprob": -1.2128906, + "special": false, + "text": " (" + }, + { + "id": 249, + "logprob": -2.3671875, + "special": false, + "text": "in" + }, + { + "id": 253, + "logprob": 0.0, + "special": false, + "text": " the" + }, + { + "id": 1340, + "logprob": -1.640625, + "special": false, + "text": " order" + }, + { + "id": 597, + "logprob": -0.5488281, + "special": false, + "text": " they" + }, + { + "id": 3176, + "logprob": -0.48608398, + "special": false, + "text": " appear" + }, + { + "id": 275, + "logprob": 0.0, + "special": false, + "text": " in" + } + ], + "top_tokens": null + }, + "generated_text": "blue, red, yellow, \nand orange (in the order they appear in" +} diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba_load.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba_load.json new file mode 100644 index 00000000..014210b2 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba_load.json @@ -0,0 +1,398 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1276, + "logprob": null, + "text": "What" + }, + { + "id": 310, + "logprob": -0.8125, + "text": " is" + }, + { + "id": 18147, + "logprob": -12.828125, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -3.0, + "text": " Learning" + }, + { + "id": 32, + "logprob": -1.1484375, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 187, + "logprob": -0.3552246, + "special": false, + "text": "\n" + }, + { + "id": 187, + "logprob": -0.38378906, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.1279297, + "special": false, + "text": "Deep" + }, + { + "id": 4715, + "logprob": -0.5595703, + "special": false, + "text": " learning" + }, + { + "id": 310, + "logprob": -0.60253906, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.7050781, + "special": false, + "text": " a" + }, + { + "id": 747, + "logprob": -2.0488281, + "special": false, + "text": " new" + }, + { + "id": 1511, + "logprob": -2.3808594, + "special": false, + "text": " type" + }, + { + "id": 273, + "logprob": -0.0026416779, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.2851562, + "special": false, + "text": " machine" + } + ], + "top_tokens": null + }, + "generated_text": "\n\nDeep learning is a new type of machine" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1276, + "logprob": null, + "text": "What" + }, + { + "id": 310, + "logprob": -0.78027344, + "text": " is" + }, + { + "id": 18147, + "logprob": -12.8203125, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -2.9902344, + "text": " Learning" + }, + { + "id": 32, + "logprob": -1.1523438, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 187, + "logprob": -0.35351562, + "special": false, + "text": "\n" + }, + { + "id": 187, + "logprob": -0.38256836, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.1269531, + "special": false, + "text": "Deep" + }, + { + "id": 4715, + "logprob": -0.54541016, + "special": false, + "text": " learning" + }, + { + "id": 310, + "logprob": -0.59765625, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.7001953, + "special": false, + "text": " a" + }, + { + "id": 747, + "logprob": -2.0585938, + "special": false, + "text": " new" + }, + { + "id": 1511, + "logprob": -2.3789062, + "special": false, + "text": " type" + }, + { + "id": 273, + "logprob": -0.0027446747, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.2851562, + "special": false, + "text": " machine" + } + ], + "top_tokens": null + }, + "generated_text": "\n\nDeep learning is a new type of machine" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1276, + "logprob": null, + "text": "What" + }, + { + "id": 310, + "logprob": -0.78027344, + "text": " is" + }, + { + "id": 18147, + "logprob": -12.8203125, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -2.9902344, + "text": " Learning" + }, + { + "id": 32, + "logprob": -1.1523438, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 187, + "logprob": -0.35351562, + "special": false, + "text": "\n" + }, + { + "id": 187, + "logprob": -0.38256836, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.1269531, + "special": false, + "text": "Deep" + }, + { + "id": 4715, + "logprob": -0.54541016, + "special": false, + "text": " learning" + }, + { + "id": 310, + "logprob": -0.59765625, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.7001953, + "special": false, + "text": " a" + }, + { + "id": 747, + "logprob": -2.0585938, + "special": false, + "text": " new" + }, + { + "id": 1511, + "logprob": -2.3789062, + "special": false, + "text": " type" + }, + { + "id": 273, + "logprob": -0.0027446747, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.2851562, + "special": false, + "text": " machine" + } + ], + "top_tokens": null + }, + "generated_text": "\n\nDeep learning is a new type of machine" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1276, + "logprob": null, + "text": "What" + }, + { + "id": 310, + "logprob": -0.78027344, + "text": " is" + }, + { + "id": 18147, + "logprob": -12.8203125, + "text": " Deep" + }, + { + "id": 20727, + "logprob": -2.9902344, + "text": " Learning" + }, + { + "id": 32, + "logprob": -1.1523438, + "text": "?" + } + ], + "seed": null, + "tokens": [ + { + "id": 187, + "logprob": -0.35351562, + "special": false, + "text": "\n" + }, + { + "id": 187, + "logprob": -0.38256836, + "special": false, + "text": "\n" + }, + { + "id": 30763, + "logprob": -1.1269531, + "special": false, + "text": "Deep" + }, + { + "id": 4715, + "logprob": -0.54541016, + "special": false, + "text": " learning" + }, + { + "id": 310, + "logprob": -0.59765625, + "special": false, + "text": " is" + }, + { + "id": 247, + "logprob": -0.7001953, + "special": false, + "text": " a" + }, + { + "id": 747, + "logprob": -2.0585938, + "special": false, + "text": " new" + }, + { + "id": 1511, + "logprob": -2.3789062, + "special": false, + "text": " type" + }, + { + "id": 273, + "logprob": -0.0027446747, + "special": false, + "text": " of" + }, + { + "id": 5145, + "logprob": -1.2851562, + "special": false, + "text": " machine" + } + ], + "top_tokens": null + }, + "generated_text": "\n\nDeep learning is a new type of machine" + } +] diff --git a/integration-tests/models/test_mamba.py b/integration-tests/models/test_mamba.py new file mode 100644 index 00000000..d86faeff --- /dev/null +++ b/integration-tests/models/test_mamba.py @@ -0,0 +1,59 @@ +import pytest + + +@pytest.fixture(scope="module") +def fused_kernel_mamba_handle(launcher): + with launcher("state-spaces/mamba-130m", num_shard=1) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def fused_kernel_mamba(fused_kernel_mamba_handle): + await fused_kernel_mamba_handle.health(300) + return fused_kernel_mamba_handle.client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_mamba(fused_kernel_mamba, response_snapshot): + response = await fused_kernel_mamba.generate( + "What is Deep Learning?", max_new_tokens=10 + ) + + assert response.details.generated_tokens == 10 + assert response.generated_text == "\n\nDeep learning is a new type of machine" + assert response == response_snapshot + +@pytest.mark.asyncio +@pytest.mark.private +async def test_mamba_all_params(fused_kernel_mamba, response_snapshot): + response = await fused_kernel_mamba.generate( + "blue, red, yellow, ", + max_new_tokens=10, + repetition_penalty=1.2, + return_full_text=True, + stop_sequences=["test"], + temperature=0.5, + top_p=0.9, + top_k=10, + truncate=5, + typical_p=0.9, + watermark=True, + decoder_input_details=True, + seed=0, + ) + + assert response.details.generated_tokens == 10 + assert response.generated_text == "blue, red, yellow, \nand orange (in the order they appear in" + assert response == response_snapshot + +@pytest.mark.asyncio +@pytest.mark.private +async def test_mamba_load(fused_kernel_mamba, generate_load, response_snapshot): + responses = await generate_load(fused_kernel_mamba, "What is Deep Learning?", max_new_tokens=10, n=4) + + assert len(responses) == 4 + assert all([r.generated_text == responses[0].generated_text for r in responses]) + assert responses[0].generated_text == "\n\nDeep learning is a new type of machine" + + assert responses == response_snapshot diff --git a/server/.gitignore b/server/.gitignore index dcb8fe67..576746ee 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -161,3 +161,4 @@ flash-attention-v2/ vllm/ llm-awq/ eetq/ +mamba/ diff --git a/server/Makefile b/server/Makefile index b1926828..31d55c41 100644 --- a/server/Makefile +++ b/server/Makefile @@ -3,6 +3,7 @@ include Makefile-flash-att-v2 include Makefile-vllm include Makefile-awq include Makefile-eetq +include Makefile-selective-scan unit-tests: pytest -s -vv -m "not private" tests diff --git a/server/Makefile-selective-scan b/server/Makefile-selective-scan new file mode 100644 index 00000000..f4dec868 --- /dev/null +++ b/server/Makefile-selective-scan @@ -0,0 +1,28 @@ +selective_scan_commit := 2a3704fd47ba817b415627b06fd796b971fdc137 + +causal-conv1d: + rm -rf causal-conv1d + git clone https://github.com/Dao-AILab/causal-conv1d.git + +build-causal-conv1d: causal-conv1d + cd causal-conv1d/ && git checkout v1.1.1 # known latest working version tag + cd causal-conv1d/ && CAUSAL_CONV1D_FORCE_BUILD=TRUE python setup.py build + +install-causal-conv1d: build-causal-conv1d + pip uninstall causal-conv1d -y || true + cd causal-conv1d/ && pip install . + +# selective-scan dependends on causal-conv1d +selective-scan: + rm -rf mamba + git clone https://github.com/state-spaces/mamba.git mamba + +build-selective-scan: selective-scan + cd mamba/ && git fetch && git checkout $(selective_scan_commit) + cd mamba && python setup.py build + +install-selective-scan: install-causal-conv1d build-selective-scan + pip uninstall selective-scan-cuda -y || true + cd mamba && pip install . + +build-all: build-causal-conv1d build-selective-scan \ No newline at end of file diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index 68096709..a952f060 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -76,6 +76,15 @@ __all__.append(FlashMixtral) __all__.append(FlashPhi) +MAMBA_AVAILABLE = True +try: + from text_generation_server.models.mamba import Mamba +except ImportError as e: + logger.warning(f"Could not import Mamba: {e}") + MAMBA_AVAILABLE = False + +if MAMBA_AVAILABLE: + __all__.append(Mamba) def get_model( model_id: str, @@ -164,7 +173,25 @@ def get_model( if speculate > 0: logger.info(f"Using speculation {method} with {speculate} input ids.") - model_type = config_dict["model_type"] + model_type = config_dict.get("model_type", None) + if model_type is None: + # TODO: fix how we determine model type for Mamba + if "ssm_cfg" in config_dict: + # *only happens in Mamba case + model_type = "ssm" + else: + raise RuntimeError( + f"Could not determine model type for {model_id} revision {revision}" + ) + + if model_type == "ssm": + return Mamba( + model_id, + revision, + quantize=quantize, + dtype=dtype, + trust_remote_code=trust_remote_code, + ) if model_type == "gpt_bigcode": if FLASH_ATTENTION: diff --git a/server/text_generation_server/models/custom_modeling/mamba_modeling.py b/server/text_generation_server/models/custom_modeling/mamba_modeling.py new file mode 100644 index 00000000..1773f04d --- /dev/null +++ b/server/text_generation_server/models/custom_modeling/mamba_modeling.py @@ -0,0 +1,194 @@ +import torch +import torch.distributed + +from mamba_ssm.ops.triton.selective_state_update import selective_state_update +from mamba_ssm.ops.selective_scan_interface import selective_scan_fn +from mamba_ssm.utils.generation import InferenceParams +from torch import nn +from typing import Optional, Tuple, Any +from transformers.configuration_utils import PretrainedConfig +import torch.nn.functional as F + +from text_generation_server.utils.layers import ( + TensorParallelEmbedding, + FastRMSNorm, + FastLinear, +) + +from einops import rearrange +from causal_conv1d import causal_conv1d_fn, causal_conv1d_update +import math + +class MambaConfig(PretrainedConfig): + def __init__( + self, + vocab_size=50280, + d_model=768, + d_state=16, + n_layer=32, + layer_norm_epsilon=1e-5, + tie_word_embeddings=False, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + expand=2, + dt_rank="auto", + **kwargs, + ): + self.vocab_size = vocab_size + self.n_layer = n_layer + self.layer_norm_epsilon = layer_norm_epsilon + self.d_model = d_model + self.d_inner = d_model * 2 + self.d_conv = 4 + self.d_state = d_state + self.expand = expand + self.dt_rank = math.ceil(self.d_model / 16) if dt_rank == "auto" else dt_rank + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + +class MambaBlock(nn.Module): + def __init__(self, prefix, config, weights): + super().__init__() + self.layer_idx = int(prefix.split(".")[2]) + self.in_proj = FastLinear.load(config, f"{prefix}.in_proj", weights, bias=False) + self.x_proj = FastLinear.load(config, f"{prefix}.x_proj", weights, bias=False) + self.dt_proj = FastLinear.load(config, f"{prefix}.dt_proj", weights, bias=True) + self.dt_proj_no_bias = FastLinear.load(config, f"{prefix}.dt_proj", weights, bias=False) + self.out_proj = FastLinear.load(config, f"{prefix}.out_proj", weights, bias=False) + self.conv1d = FastLinear.load(config, f"{prefix}.conv1d", weights, bias=True) + self.negA = -torch.exp(weights.get_tensor(f"{prefix}.A_log").float()) + self.D = weights.get_tensor(f"{prefix}.D") + self.activation = "silu" + self.dt_rank = config.dt_rank + self.d_state = config.d_state + self.d_conv = config.d_conv + self.act = nn.SiLU() + + # inference_params + def forward(self, hidden_states: torch.Tensor, inference_params=None): + _, seqlen, _ = hidden_states.shape + conv_state, ssm_state = inference_params.key_value_memory_dict[self.layer_idx] + + if inference_params.seqlen_offset > 0: + out, conv_state, ssm_state = self.step(hidden_states, conv_state, ssm_state) + return out, conv_state, ssm_state + + projected_states = self.in_proj(hidden_states).transpose(1,2) + x, z = projected_states.chunk(2, dim=1) + conv_state = F.pad(x, (self.d_conv - seqlen, 0)) + x = causal_conv1d_fn( + x=x, + weight=self.conv1d.weight.view(self.conv1d.weight.size(0), self.conv1d.weight.size(2)), + bias=self.conv1d.bias, + activation=self.activation, + ) + + # We're careful here about the layout, to avoid extra transposes. + # We want dt to have d as the slowest moving dimension + # and L as the fastest moving dimension, since those are what the ssm_scan kernel expects. + x_dbl = self.x_proj(rearrange(x, "b d l -> (b l) d")) # (bl d) + dt, B, C = torch.split(x_dbl, [self.dt_rank, self.d_state, self.d_state], dim=-1) + dt = self.dt_proj.weight @ dt.t() + dt = rearrange(dt, "d (b l) -> b d l", l=seqlen) + B = rearrange(B, "(b l) dstate -> b dstate l", l=seqlen).contiguous() + C = rearrange(C, "(b l) dstate -> b dstate l", l=seqlen).contiguous() + y, last_state = selective_scan_fn( + x, + dt, + self.negA, + B, + C, + self.D.float(), + z=z, + delta_bias=self.dt_proj.bias.float(), + delta_softplus=True, + return_last_state=True, + ) + y = rearrange(y, "b d l -> b l d") + attn_outputs = self.out_proj(y) + return attn_outputs, conv_state, last_state + + def step(self, hidden_states, conv_state, ssm_state): + _xz = self.in_proj(hidden_states) + _x, _z = _xz.chunk(2, dim=-1) # (B D) + conv_state_new = torch.cat([conv_state, _x.transpose(1,2)], dim=-1) + conv_out = causal_conv1d_fn( + x=conv_state_new, + weight=self.conv1d.weight.view(self.conv1d.weight.size(0), self.conv1d.weight.size(2)), + bias=self.conv1d.bias, + activation=self.activation + ) + conv_state = conv_state_new[:, :, 1:] + bsz, seqlen, dim = hidden_states.shape + output_tensor = torch.zeros( + (bsz, seqlen, dim), + device=hidden_states.device, + dtype=hidden_states.dtype + ) + for i in range(0, bsz): + x = conv_out[i:i+1,:,-1] + z = _z[i:i+1, -1, :] + x_db = self.x_proj(x) + dt, B, C = torch.split(x_db, [self.dt_rank, self.d_state, self.d_state], dim=-1) + dt = F.linear(dt, self.dt_proj.weight) + y = selective_state_update( + ssm_state[i:i+1,:,:], x, dt, self.negA, B, C, self.D, z=z, dt_bias=self.dt_proj.bias, dt_softplus=True + ) + out = self.out_proj(y) + output_tensor[i] = out + + return output_tensor, conv_state, ssm_state + + + +class ResidualBlock(nn.Module): + def __init__(self, layer_id, config, weights): + super().__init__() + self.mamba_block = MambaBlock(prefix=f"{layer_id}.mixer", config=config, weights=weights) + self.layer_norm = FastRMSNorm.load(prefix=f"{layer_id}.norm", weights=weights, eps=config.layer_norm_epsilon) + + def forward( + self, + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor] = None, + inference_params: Optional[Any] = None, + ): + residual = (hidden_states + residual) if residual is not None else hidden_states + shape = residual.shape + hidden_states, _ = self.layer_norm(residual.view(-1, shape[-1])) + hidden_states, conv_state, last_ssm_state = self.mamba_block(hidden_states.view(*shape), inference_params) + return hidden_states, residual, conv_state, last_ssm_state + +class MambaModel(nn.Module): + def __init__(self, config, weights): + super().__init__() + prefix = "backbone" + self.embed_tokens = TensorParallelEmbedding(f"{prefix}.embedding", weights) + self.blocks = nn.ModuleList( + [ResidualBlock(f"{prefix}.layers.{i}", config, weights) for i in range(config.n_layer)] + ) + self.norm_f = FastRMSNorm.load(f"{prefix}.norm_f", weights, eps=config.layer_norm_epsilon) + self.lm_head = FastLinear.load(config, f"{prefix}.embedding", weights, bias=False) + self.config = config + + def forward(self, input_ids: torch.Tensor, inference_params=None, residual=None) -> Tuple[torch.Tensor, torch.Tensor, InferenceParams]: + hidden_states = self.embed_tokens(input_ids) + for block in self.blocks: + hidden_states, residual, conv_state, ssm_state = block(hidden_states, residual, inference_params) + inference_params.key_value_memory_dict[block.mamba_block.layer_idx] = (conv_state, ssm_state) + + hidden_states = hidden_states + residual if residual is not None else hidden_states + hidden_states, _ = self.norm_f(hidden_states.view(-1, hidden_states.size(-1))) + hidden_states = hidden_states.view(residual.shape) + logits = self.lm_head(hidden_states) + + # update the offset for the next inference using these params + inference_params.seqlen_offset += input_ids.size(1) + return logits, input_ids, inference_params \ No newline at end of file diff --git a/server/text_generation_server/models/mamba.py b/server/text_generation_server/models/mamba.py new file mode 100644 index 00000000..c10910aa --- /dev/null +++ b/server/text_generation_server/models/mamba.py @@ -0,0 +1,656 @@ +import torch +import torch.distributed +from transformers import AutoTokenizer, PreTrainedTokenizerBase +from typing import Optional +from text_generation_server.models.custom_modeling.mamba_modeling import ( + MambaConfig, +) +from text_generation_server.pb import generate_pb2 +from text_generation_server.utils import ( + initialize_torch_distributed, + weight_files, + Weights, +) +import time +from text_generation_server.models.custom_modeling.mamba_modeling import MambaModel +from text_generation_server.models import Model +from typing import Any, List, Optional, Tuple, Type, Dict +from text_generation_server.models.types import ( + Batch, + Tokens, + Generation, + GeneratedText, +) +from text_generation_server.utils.tokens import batch_top_tokens, Sampling +from dataclasses import dataclass +from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling +from mamba_ssm.utils.generation import InferenceParams + +@dataclass +class MambaBatch(Batch): + batch_id: int + requests: List[generate_pb2.Request] + requests_idx_mapping: Dict[int, int] + + # Decoder values + input_ids: torch.Tensor + + # All tokens + all_input_ids: List[torch.Tensor] + + # Lengths of all generations present in the batch + input_lengths: List[int] + prefix_offsets: List[int] + read_offsets: List[int] + + # Generation helpers + next_token_choosers: List[NextTokenChooser] + stopping_criterias: List[StoppingCriteria] + top_n_tokens: List[int] + top_n_tokens_tensor: torch.Tensor + + # Metadata used for padding + max_input_length: int + padding_right_offset: int + + # Maximum number of tokens this batch will grow to + max_tokens: int + + # Past metadata + keys_head_dim_last: bool = True + + # Inference params + inference_params: Optional[Dict[str, Any]] = None + + def to_pb(self) -> generate_pb2.CachedBatch: + return generate_pb2.CachedBatch( + id=self.batch_id, + request_ids=[r.id for r in self.requests], + size=len(self), + max_tokens=self.max_tokens, + ) + + @classmethod + def from_pb( + cls, + pb: generate_pb2.Batch, + tokenizer: PreTrainedTokenizerBase, + dtype: torch.dtype, + device: torch.device, + ) -> "MambaBatch": + inputs = [] + next_token_choosers = [] + stopping_criterias = [] + top_n_tokens = [] + prefix_offsets = [] + read_offsets = [] + requests_idx_mapping = {} + + # Parse batch + max_truncation = 0 + padding_right_offset = 0 + max_decode_tokens = 0 + for i, r in enumerate(pb.requests): + requests_idx_mapping[r.id] = i + inputs.append(r.inputs) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + stopping_criteria = StoppingCriteria.from_pb( + r.stopping_parameters, tokenizer + ) + stopping_criterias.append(stopping_criteria) + top_n_tokens.append(r.top_n_tokens) + max_truncation = max(max_truncation, r.truncate) + max_decode_tokens += stopping_criteria.max_new_tokens + padding_right_offset = max( + padding_right_offset, stopping_criteria.max_new_tokens + ) + + tokenized_inputs = tokenizer( + inputs, + return_tensors="pt", + padding=True, + return_token_type_ids=False, + truncation=True, + max_length=max_truncation, + ).to(device) + for _ in pb.requests: + input_len = tokenized_inputs["input_ids"].shape[1] + prefix_offsets.append(input_len - 5) + read_offsets.append(input_len) + + input_lengths = tokenized_inputs["attention_mask"].sum(1) + max_input_length = input_lengths.max() + input_ids = tokenized_inputs["input_ids"] + all_input_ids = tokenized_inputs["input_ids"].T.split(1, dim=1) + top_n_tokens_tensor = torch.tensor( + top_n_tokens, device=device, dtype=torch.int64 + ) + max_tokens = len(inputs) * (max_input_length + max_decode_tokens) + return cls( + batch_id=pb.id, + requests=pb.requests, + requests_idx_mapping=requests_idx_mapping, + input_ids=input_ids, + # past_input_ids=None, + all_input_ids=list(all_input_ids), + input_lengths=input_lengths.tolist(), + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, + max_input_length=max_input_length.item(), + padding_right_offset=padding_right_offset, + max_tokens=max_tokens, + ) + + def filter(self, request_ids: List[int]) -> Optional["MambaBatch"]: + if len(request_ids) == 0: + raise ValueError("Batch must have at least one request") + if len(request_ids) == len(self): + return self + + keep_indices = [] + + # New values after filtering + requests_idx_mapping = {} + requests = [] + input_lengths = [] + prefix_offsets = [] + read_offsets = [] + all_input_ids = [] + max_input_length = 0 + + next_token_choosers = [] + stopping_criterias = [] + top_n_tokens = [] + + total_remaining_decode_tokens = 0 + new_padding_right_offset = 0 + + indices = [] + for i, request_id in enumerate(request_ids): + idx = self.requests_idx_mapping[request_id] + requests_idx_mapping[request_id] = i + keep_indices.append(idx) + + requests.append(self.requests[idx]) + prefix_offsets.append(self.prefix_offsets[idx]) + read_offsets.append(self.read_offsets[idx]) + all_input_ids.append(self.all_input_ids[idx]) + + request_input_length = self.input_lengths[idx] + input_lengths.append(request_input_length) + max_input_length = max(max_input_length, request_input_length) + indices.append(idx) + + next_token_choosers.append(self.next_token_choosers[idx]) + stopping_criteria = self.stopping_criterias[idx] + stopping_criterias.append(stopping_criteria) + top_n_tokens.append(self.top_n_tokens[idx]) + remaining_decode_tokens = ( + stopping_criteria.max_new_tokens - stopping_criteria.current_tokens + ) + total_remaining_decode_tokens += remaining_decode_tokens + new_padding_right_offset = max( + new_padding_right_offset, remaining_decode_tokens + ) + + # Apply indices to input_ids, attention mask, past key values and other items that need to be cached + input_ids = self.input_ids[keep_indices] + + top_n_tokens_tensor = self.top_n_tokens_tensor[keep_indices] + max_tokens = len(request_ids) * max_input_length + total_remaining_decode_tokens + + self.requests = requests + self.requests_idx_mapping = requests_idx_mapping + self.input_ids = input_ids + self.all_input_ids = all_input_ids + self.input_lengths = input_lengths + self.prefix_offsets = prefix_offsets + self.read_offsets = read_offsets + self.next_token_choosers = next_token_choosers + self.stopping_criterias = stopping_criterias + self.top_n_tokens = top_n_tokens + self.top_n_tokens_tensor = top_n_tokens_tensor + self.max_input_length = max_input_length + self.padding_right_offset = new_padding_right_offset + self.max_tokens = max_tokens + + # TODO + # Kept it simple by just updating the state, maybe updating the other CPU values is necessary. + key_value_memory_dict = {} + for i, (conv_state, ssm_state) in self.inference_params.key_value_memory_dict.items(): + key_value_memory_dict[i] = (conv_state[indices], ssm_state[indices]) + self.inference_params.key_value_memory_dict = key_value_memory_dict + + return self + + @classmethod + def concatenate(cls, batches: List["MambaBatch"]) -> "MambaBatch": + # Used for padding + total_batch_size = 0 + max_input_length = 0 + padding_right_offset = 0 + for batch in batches: + total_batch_size += len(batch) + max_input_length = max(max_input_length, batch.max_input_length) + padding_right_offset = max(padding_right_offset, batch.padding_right_offset) + + # Batch attributes + requests = [] + requests_idx_mapping = {} + input_lengths = [] + prefix_offsets = [] + read_offsets = [] + all_input_ids = [] + next_token_choosers = [] + stopping_criterias = [] + top_n_tokens = [] + max_tokens = 0 + max_seqlen = 0 + batch_size = 0 + seqlen_offset = 0 + + # Batch tensors + input_ids = None + top_n_tokens_tensor = None + + # Used for slicing correctly inside the tensors + # Equivalent to a cumsum on batch sizes + start_index = 0 + for i, batch in enumerate(batches): + requests.extend(batch.requests) + input_lengths.extend(batch.input_lengths) + prefix_offsets.extend(batch.prefix_offsets) + read_offsets.extend(batch.read_offsets) + all_input_ids.extend(batch.all_input_ids) + next_token_choosers.extend(batch.next_token_choosers) + stopping_criterias.extend(batch.stopping_criterias) + top_n_tokens.extend(batch.top_n_tokens) + + if i == 0: + requests_idx_mapping = batch.requests_idx_mapping + else: + # We need to offset the mapping for each batch by the cumulative batch size + for k, v in batch.requests_idx_mapping.items(): + requests_idx_mapping[k] = v + start_index + + # Slicing end index for this batch + end_index = start_index + len(batch) + + # Create empty tensor + # input_ids is always of shape [batch_size, 1] + # We do not need to pad it + if input_ids is None: + input_ids = batch.input_ids.new_empty((total_batch_size, 1)) + # Copy to correct indices + input_ids[start_index:end_index] = batch.input_ids + + if top_n_tokens_tensor is None: + top_n_tokens_tensor = batches[0].top_n_tokens_tensor.new_zeros( + total_batch_size, + ) + top_n_tokens_tensor[start_index:end_index] = batch.top_n_tokens_tensor + + # Add eventual padding tokens that were added while concatenating + max_tokens += batch.max_tokens + ( + max_input_length - batch.max_input_length + ) * len(batch) + + max_seqlen = max(max_seqlen, batch.inference_params.max_seqlen) + seqlen_offset = max(seqlen_offset, batch.inference_params.seqlen_offset) + batch_size += batch.inference_params.max_batch_size + + start_index = end_index + + + (_, d_model, d_conv) = batches[0].inference_params.key_value_memory_dict[0][0].shape + (_, _, d_state) = batches[0].inference_params.key_value_memory_dict[0][1].shape + n_blocks = len(batches[0].inference_params.key_value_memory_dict) + dtype = batches[0].inference_params.key_value_memory_dict[0][0].dtype + device = batches[0].inference_params.key_value_memory_dict[0][0].device + + key_value_memory_dict = {} + for i in range(n_blocks): + conv_state = torch.zeros( + batch_size, + d_model, + d_conv, + device=device, + dtype=dtype, + ) + ssm_state = torch.zeros( + batch_size, + d_model, + d_state, + device=device, + dtype=dtype, + ) + key_value_memory_dict[i] = (conv_state, ssm_state) + lengths_per_sample = torch.zeros(batch_size, dtype=torch.int32, device=device) + + inference_params = InferenceParams( + max_seqlen=max_seqlen, + max_batch_size=batch_size, + seqlen_offset=seqlen_offset, + key_value_memory_dict=key_value_memory_dict, + lengths_per_sample=lengths_per_sample, + ) + + current_batch = 0 + for batch in batches: + for i in range(n_blocks): + conv_state, ssm_state = batch.inference_params.key_value_memory_dict[i] + batch_size = batch.inference_params.max_batch_size + inference_params.key_value_memory_dict[i][0][current_batch:current_batch + batch_size] = conv_state + inference_params.key_value_memory_dict[i][1][current_batch:current_batch + batch_size] = ssm_state + inference_params.lengths_per_sample[current_batch: current_batch + batch_size] = batch.inference_params.lengths_per_sample + current_batch += batch_size + + return cls( + batch_id=batches[0].batch_id, + requests=requests, + requests_idx_mapping=requests_idx_mapping, + input_ids=input_ids, + all_input_ids=all_input_ids, + input_lengths=input_lengths, + prefix_offsets=prefix_offsets, + read_offsets=read_offsets, + next_token_choosers=next_token_choosers, + stopping_criterias=stopping_criterias, + top_n_tokens=top_n_tokens, + top_n_tokens_tensor=top_n_tokens_tensor, + max_input_length=max_input_length, + padding_right_offset=padding_right_offset, + keys_head_dim_last=batches[0].keys_head_dim_last, + max_tokens=max_tokens, + inference_params=inference_params + ) + + def __len__(self): + return len(self.requests) + +class Mamba(Model): + def __init__( + self, + model_id: str, + revision: Optional[str] = None, + quantize: Optional[str] = None, + dtype: Optional[torch.dtype] = None, + trust_remote_code: bool = False, + ): + self.process_group, _rank, _world_size = initialize_torch_distributed() + if torch.cuda.is_available(): + device = torch.device("cuda") + dtype = torch.float16 if dtype is None else dtype + else: + if quantize: + raise ValueError("quantization is not available on CPU") + + device = torch.device("cpu") + dtype = torch.float32 if dtype is None else dtype + + tokenizer = AutoTokenizer.from_pretrained( + "EleutherAI/gpt-neox-20b", + revision=revision, + padding_side="left", + truncation_side="left", + trust_remote_code=trust_remote_code, + ) + config = MambaConfig.from_pretrained( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) + + tokenizer.bos_token_id = config.bos_token_id + tokenizer.eos_token_id = config.eos_token_id + tokenizer.pad_token = tokenizer.eos_token + + config.quantize = quantize + torch.distributed.barrier(group=self.process_group) + filenames = weight_files(model_id, revision=revision, extension=".safetensors") + weights = Weights(filenames, device, dtype, process_group=self.process_group) + model = MambaModel(config, weights) + torch.distributed.barrier(group=self.process_group) + super(Mamba, self).__init__( + model=model, + tokenizer=tokenizer, + requires_padding=True, + dtype=dtype, + device=device, + ) + + @property + def batch_type(self) -> Type[MambaBatch]: + return MambaBatch + + def warmup(self, batch) -> Optional[int]: + # TODO: implement warmup for Mamba if needed + return None + + def forward( + self, + input_ids: torch.Tensor, + past: Optional[List[torch.Tensor]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + return self.model( + input_ids, + past=past, + ) + + def generate_token(self, batch) -> Tuple[List[Any], Optional[Any], Tuple[int, int]]: + start = time.time_ns() + input_ids = batch.input_ids # batch.past_input_ids if batch.past_input_ids is not None else batch.input_ids + + batch_size = input_ids.shape[0] + max_seqlen = input_ids.shape[1] + dtype = input_ids.dtype + + # Inference params + seqlen_og = 0 + inf_cache = {} + lengths_per_sample = torch.ones(batch_size, dtype=torch.int32, device=input_ids.device) * max_seqlen + + if batch.inference_params is None: + inference_params = InferenceParams( + max_seqlen=max_seqlen, + max_batch_size=batch_size, + seqlen_offset=seqlen_og, + key_value_memory_dict=inf_cache, + lengths_per_sample=lengths_per_sample, + ) + + # Allocate inference cache + for res_block in self.model.blocks: + block = res_block.mamba_block + conv_state = torch.zeros( + batch_size, + self.model.config.d_model * self.model.config.expand, + self.model.config.d_conv, + device=block.conv1d.weight.device, + dtype=block.conv1d.weight.dtype, + ) + ssm_state = torch.zeros( + batch_size, + self.model.config.d_model * self.model.config.expand, + self.model.config.d_state, + device=block.dt_proj.weight.device, + dtype=block.dt_proj.weight.dtype, + ) + inference_params.key_value_memory_dict[block.layer_idx] = (conv_state, ssm_state) + batch.inference_params = inference_params + + # Forward pass + logits, past_input_ids, new_inference_params = self.model(input_ids, batch.inference_params) + + batch.inference_params = new_inference_params + # Results + generations: List[Generation] = [] + stopped = True + + # Speculation is not active for causal + accepted_ids = torch.ones_like(batch.input_ids)[:, 0] + batch_top_token_ids, batch_top_token_logprobs = batch_top_tokens( + batch.top_n_tokens, + batch.top_n_tokens_tensor, + torch.log_softmax(logits[:, -1], -1), + accepted_ids, + ) + + start_decode = time.time_ns() + + # Zipped iterator + iterator = zip( + batch.requests, + batch.input_lengths, + batch.prefix_offsets, + batch.read_offsets, + logits, + batch.next_token_choosers, + batch.stopping_criterias, + batch.all_input_ids, + batch.top_n_tokens, + batch_top_token_ids, + batch_top_token_logprobs, + ) + + # For each member of the batch + for i, ( + request, + input_length, + prefix_offset, + read_offset, + logits, + next_token_chooser, + stopping_criteria, + all_input_ids, + top_n_tokens, + top_token_ids, + top_token_logprobs, + ) in enumerate(iterator): + # Select next token + next_token_id, logprobs = next_token_chooser( + all_input_ids.view(1, -1), logits[-1:, :] + ) + + # Append next token to all tokens + all_input_ids = torch.cat([all_input_ids, next_token_id]) + new_input_length = input_length + 1 + + # Generated token + next_token_logprob = logprobs[-1, next_token_id] + next_token_id_squeezed = next_token_id.squeeze() + next_token_text, prefix_offset, read_offset = self.decode_token( + all_input_ids[:, 0], prefix_offset, read_offset + ) + + # Evaluate stopping criteria + stop, reason = stopping_criteria( + next_token_id_squeezed, + next_token_text, + ) + + if not stop: + stopped = False + + # Shard generations + # All generations will be appended in the rust sharded client + if i % self.world_size == self.rank: + if stop: + # Decode generated tokens + output_text, _, _ = self.decode_token( + all_input_ids[:, 0], + prefix_offset=len(all_input_ids) + - stopping_criteria.current_tokens + - 1, + read_offset=len(all_input_ids) - stopping_criteria.current_tokens, + skip_special_tokens=True, + ) + # Get seed + if isinstance(next_token_chooser.choice, Sampling): + seed = next_token_chooser.choice.seed + else: + seed = None + + generated_text = GeneratedText( + output_text, stopping_criteria.current_tokens, reason, seed + ) + else: + generated_text = None + + if stopping_criteria.current_tokens == 1 and request.prefill_logprobs: + # Remove generated token to only have prefill and add nan for first prompt token + prefill_logprobs = [float("nan")] + torch.log_softmax( + logits, -1 + ).gather(1, all_input_ids[1:]).squeeze(1)[ + -new_input_length:-1 + ].tolist() + prefill_token_ids = all_input_ids[-new_input_length:-1] + prefill_texts = self.tokenizer.batch_decode( + prefill_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + prefill_tokens = Tokens( + prefill_token_ids, + prefill_logprobs, + prefill_texts, + is_special=[], + ) + else: + prefill_tokens = None + + if top_n_tokens > 0: + toptoken_texts = self.tokenizer.batch_decode( + top_token_ids, + clean_up_tokenization_spaces=False, + skip_special_tokens=False, + ) + special_toptokens = [ + token_id in self.all_special_ids for token_id in top_token_ids + ] + top_tokens = Tokens( + top_token_ids, + top_token_logprobs, + toptoken_texts, + special_toptokens, + ) + else: + top_tokens = None + + generation = Generation( + request.id, + prefill_tokens, + Tokens( + [next_token_id_squeezed], + [next_token_logprob], + [next_token_text], + [next_token_id_squeezed.item() in self.all_special_ids], + ), + generated_text, + top_tokens, + ) + + generations.append(generation) + + # Update values + batch.input_ids[i, 0] = next_token_id + batch.all_input_ids[i] = all_input_ids + batch.input_lengths[i] = new_input_length + batch.prefix_offsets[i] = prefix_offset + batch.read_offsets[i] = read_offset + batch.max_input_length = max(batch.max_input_length, new_input_length) + + # We finished all generations in the batch; there is no next batch + if stopped: + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, None, (forward_ns, decode_ns) + + # Slice unused values from prefill + batch.input_ids = batch.input_ids[:, :1] + + forward_ns = start_decode - start + decode_ns = time.time_ns() - start_decode + return generations, batch, (forward_ns, decode_ns) From 39af000cb9bfa12628c21c6e65d3ce8930250171 Mon Sep 17 00:00:00 2001 From: Jason Stillerman Date: Thu, 8 Feb 2024 06:44:04 -0500 Subject: [PATCH 557/793] Update to peft 0.8.2 (#1537) # What does this PR do? ## Before submitting - [x] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [x] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [x] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [x] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. @OlivierDehaene OR @Narsil --- server/poetry.lock | 17 ++++++++++------- server/pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/server/poetry.lock b/server/poetry.lock index 64b1b74f..32031f89 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "accelerate" @@ -1589,30 +1589,32 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "peft" -version = "0.4.0" +version = "0.8.2" description = "Parameter-Efficient Fine-Tuning (PEFT)" optional = true python-versions = ">=3.8.0" files = [ - {file = "peft-0.4.0-py3-none-any.whl", hash = "sha256:2cf992772a6d703814477e0bdcdadd68cb8ea388111ce2d793dd2ff0e438f357"}, - {file = "peft-0.4.0.tar.gz", hash = "sha256:e768fa22d6e9f32aa7e891f0d06f355960278ca4dc0cdd96bff71f6f06269207"}, + {file = "peft-0.8.2-py3-none-any.whl", hash = "sha256:4a9c81c38e689fd4043b2757cd0e2b526a9b8b8fd04f8442df2c4824b32c2505"}, + {file = "peft-0.8.2.tar.gz", hash = "sha256:bbdf61db2d8ca503e894edc64016038e6f34b7b522374bad09a22af41882e7ac"}, ] [package.dependencies] -accelerate = "*" +accelerate = ">=0.21.0" +huggingface-hub = ">=0.17.0" numpy = ">=1.17" packaging = ">=20.0" psutil = "*" pyyaml = "*" safetensors = "*" torch = ">=1.13.0" +tqdm = "*" transformers = "*" [package.extras] dev = ["black (>=22.0,<23.0)", "hf-doc-builder", "ruff (>=0.0.241)", "urllib3 (<=2.0.0)"] docs-specific = ["hf-doc-builder"] quality = ["black (>=22.0,<23.0)", "ruff (>=0.0.241)", "urllib3 (<=2.0.0)"] -test = ["black (>=22.0,<23.0)", "datasets", "diffusers", "hf-doc-builder", "parameterized", "pytest", "pytest-cov", "pytest-xdist", "ruff (>=0.0.241)", "urllib3 (<=2.0.0)"] +test = ["black (>=22.0,<23.0)", "datasets", "diffusers (<0.21.0)", "hf-doc-builder", "parameterized", "pytest", "pytest-cov", "pytest-xdist", "ruff (>=0.0.241)", "scipy", "urllib3 (<=2.0.0)"] [[package]] name = "pillow" @@ -1893,6 +1895,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2962,4 +2965,4 @@ torch = ["torch"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "33d533d21d14c258678a8c4bb28e2a15e8ebe5ca35d8589cbfe4a7b7d2e79a90" +content-hash = "f7529125bdd7ce142082ce4969edbda5d9b67b6209f199194c54198829f5dc64" diff --git a/server/pyproject.toml b/server/pyproject.toml index 72a7afb0..b8ebf2e3 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -30,7 +30,7 @@ transformers = "^4.37.1" einops = "^0.6.1" texttable = { version = "^1.6.7", optional = true } datasets = { version = "^2.14.0", optional = true } -peft = { version = "^0.4.0", optional = true } +peft = { version = "^0.8.2", optional = true } torch = { version = "^2.1.1", optional = true } scipy = "^1.11.1" pillow = "^10.0.0" From 09b7c26bbdb940e4e0d2216e14fd437f89fcdeb2 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Thu, 8 Feb 2024 18:41:25 +0100 Subject: [PATCH 558/793] feat(server): add frequency penalty (#1541) --- Cargo.lock | 38 ++++++- benchmark/src/lib.rs | 3 + benchmark/src/main.rs | 7 ++ benchmark/src/table.rs | 2 + integration-tests/models/test_mamba.py | 11 +- proto/generate.proto | 2 + router/client/src/client.rs | 1 + router/src/health.rs | 1 + router/src/lib.rs | 86 ++++++++++++-- router/src/queue.rs | 1 + router/src/server.rs | 18 ++- router/src/validation.rs | 9 ++ server/tests/utils/test_tokens.py | 2 +- .../text_generation_server/models/__init__.py | 1 + .../models/causal_lm.py | 7 +- .../models/custom_modeling/mamba_modeling.py | 107 ++++++++++++------ .../models/flash_causal_lm.py | 8 +- server/text_generation_server/models/mamba.py | 59 +++++++--- .../models/seq2seq_lm.py | 7 +- server/text_generation_server/models/types.py | 4 +- .../utils/logits_process.py | 56 +++++++++ server/text_generation_server/utils/tokens.py | 55 +++++++-- 22 files changed, 396 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7fdf301a..3318f3b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2787,7 +2787,7 @@ dependencies = [ "tabled", "text-generation-client", "thiserror", - "tokenizers", + "tokenizers 0.14.1", "tokio", "tracing", "tracing-subscriber", @@ -2850,7 +2850,7 @@ dependencies = [ "serde_json", "text-generation-client", "thiserror", - "tokenizers", + "tokenizers 0.15.1", "tokio", "tokio-stream", "tower-http", @@ -2972,6 +2972,40 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "tokenizers" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db445cceba5dfeb0f9702be7d6bfd91801ddcbe8fe8722defe7f2e96da75812" +dependencies = [ + "aho-corasick", + "clap", + "derive_builder", + "esaxx-rs", + "getrandom", + "hf-hub", + "indicatif", + "itertools 0.11.0", + "lazy_static", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax 0.7.5", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.35.1" diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs index 1875652c..6deae48d 100644 --- a/benchmark/src/lib.rs +++ b/benchmark/src/lib.rs @@ -30,6 +30,7 @@ pub async fn run( top_p: Option, typical_p: Option, repetition_penalty: Option, + frequency_penalty: Option, watermark: bool, do_sample: bool, client: ShardedClient, @@ -42,6 +43,7 @@ pub async fn run( do_sample, seed: 0, repetition_penalty: repetition_penalty.unwrap_or(1.0), + frequency_penalty: frequency_penalty.unwrap_or(0.0), watermark, }; @@ -140,6 +142,7 @@ pub async fn run( top_p, typical_p, repetition_penalty, + frequency_penalty, watermark, do_sample, ); diff --git a/benchmark/src/main.rs b/benchmark/src/main.rs index 97c8af1c..2d89e045 100644 --- a/benchmark/src/main.rs +++ b/benchmark/src/main.rs @@ -84,6 +84,11 @@ struct Args { #[clap(long, env)] repetition_penalty: Option, + /// Generation parameter in case you want to specifically test/debug particular + /// decoding strategies, for full doc refer to the `text-generation-server` + #[clap(long, env)] + frequency_penalty: Option, + /// Generation parameter in case you want to specifically test/debug particular /// decoding strategies, for full doc refer to the `text-generation-server` #[clap(long, env)] @@ -119,6 +124,7 @@ fn main() -> Result<(), Box> { top_p, typical_p, repetition_penalty, + frequency_penalty, watermark, do_sample, master_shard_uds_path, @@ -187,6 +193,7 @@ fn main() -> Result<(), Box> { top_p, typical_p, repetition_penalty, + frequency_penalty, watermark, do_sample, sharded_client, diff --git a/benchmark/src/table.rs b/benchmark/src/table.rs index 9e36717b..c4819ff3 100644 --- a/benchmark/src/table.rs +++ b/benchmark/src/table.rs @@ -15,6 +15,7 @@ pub(crate) fn parameters_table( top_p: Option, typical_p: Option, repetition_penalty: Option, + frequency_penalty: Option, watermark: bool, do_sample: bool, ) -> Table { @@ -33,6 +34,7 @@ pub(crate) fn parameters_table( builder.push_record(["Top P", &format!("{top_p:?}")]); builder.push_record(["Typical P", &format!("{typical_p:?}")]); builder.push_record(["Repetition Penalty", &format!("{repetition_penalty:?}")]); + builder.push_record(["Frequency Penalty", &format!("{frequency_penalty:?}")]); builder.push_record(["Watermark", &watermark.to_string()]); builder.push_record(["Do Sample", &do_sample.to_string()]); diff --git a/integration-tests/models/test_mamba.py b/integration-tests/models/test_mamba.py index d86faeff..bf398999 100644 --- a/integration-tests/models/test_mamba.py +++ b/integration-tests/models/test_mamba.py @@ -24,6 +24,7 @@ async def test_mamba(fused_kernel_mamba, response_snapshot): assert response.generated_text == "\n\nDeep learning is a new type of machine" assert response == response_snapshot + @pytest.mark.asyncio @pytest.mark.private async def test_mamba_all_params(fused_kernel_mamba, response_snapshot): @@ -44,13 +45,19 @@ async def test_mamba_all_params(fused_kernel_mamba, response_snapshot): ) assert response.details.generated_tokens == 10 - assert response.generated_text == "blue, red, yellow, \nand orange (in the order they appear in" + assert ( + response.generated_text + == "blue, red, yellow, \nand orange (in the order they appear in" + ) assert response == response_snapshot + @pytest.mark.asyncio @pytest.mark.private async def test_mamba_load(fused_kernel_mamba, generate_load, response_snapshot): - responses = await generate_load(fused_kernel_mamba, "What is Deep Learning?", max_new_tokens=10, n=4) + responses = await generate_load( + fused_kernel_mamba, "What is Deep Learning?", max_new_tokens=10, n=4 + ) assert len(responses) == 4 assert all([r.generated_text == responses[0].generated_text for r in responses]) diff --git a/proto/generate.proto b/proto/generate.proto index 1f30df38..5140fdaa 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -66,6 +66,8 @@ message NextTokenChooserParameters { uint64 seed = 6; /// repetition penalty float repetition_penalty = 7; + /// frequency penalty + float frequency_penalty = 9; /// token watermarking using "A Watermark for Large Language Models" bool watermark = 8; } diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 023c5671..fde5c402 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -125,6 +125,7 @@ impl Client { do_sample: false, seed: 0, repetition_penalty: 1.2, + frequency_penalty: 0.1, watermark: true, }), stopping_parameters: Some(StoppingCriteriaParameters { diff --git a/router/src/health.rs b/router/src/health.rs index ab290fc1..e830a3c3 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -43,6 +43,7 @@ impl Health { do_sample: false, seed: 0, repetition_penalty: 1.0, + frequency_penalty: 0.0, watermark: false, }), stopping_parameters: Some(StoppingCriteriaParameters { diff --git a/router/src/lib.rs b/router/src/lib.rs index e85519cc..7c44d642 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -106,6 +106,14 @@ pub(crate) struct GenerateParameters { )] pub repetition_penalty: Option, #[serde(default)] + #[schema( + exclusive_minimum = -2.0, + nullable = true, + default = "null", + example = 0.1 + )] + pub frequency_penalty: Option, + #[serde(default)] #[schema(exclusive_minimum = 0, nullable = true, default = "null", example = 10)] pub top_k: Option, #[serde(default)] @@ -172,6 +180,7 @@ fn default_parameters() -> GenerateParameters { best_of: None, temperature: None, repetition_penalty: None, + frequency_penalty: None, top_k: None, top_p: None, typical_p: None, @@ -205,10 +214,71 @@ pub(crate) struct ChatCompletion { pub(crate) struct ChatCompletionComplete { pub index: u32, pub message: Message, - pub logprobs: Option>, + pub logprobs: Option, pub finish_reason: String, } +#[derive(Clone, Deserialize, Serialize, ToSchema)] +pub(crate) struct ChatCompletionLogprobs { + content: Vec, +} + +impl From<(Token, Vec)> for ChatCompletionLogprobs { + fn from(value: (Token, Vec)) -> Self { + let (token, top_tokens) = value; + + Self { + content: vec![ChatCompletionLogprob { + token: token.text, + logprob: token.logprob, + top_logprobs: top_tokens + .into_iter() + .map(|t| ChatCompletionTopLogprob { + token: t.text, + logprob: t.logprob, + }) + .collect(), + }], + } + } +} + +impl From<(Vec, Vec>)> for ChatCompletionLogprobs { + fn from(value: (Vec, Vec>)) -> Self { + let (tokens, top_tokens) = value; + Self { + content: tokens + .into_iter() + .zip(top_tokens) + .map(|(t, top_t)| ChatCompletionLogprob { + token: t.text, + logprob: t.logprob, + top_logprobs: top_t + .into_iter() + .map(|t| ChatCompletionTopLogprob { + token: t.text, + logprob: t.logprob, + }) + .collect(), + }) + .collect(), + } + } +} + +#[derive(Clone, Deserialize, Serialize, ToSchema)] +pub(crate) struct ChatCompletionLogprob { + token: String, + logprob: f32, + top_logprobs: Vec, +} + +#[derive(Clone, Deserialize, Serialize, ToSchema)] +pub(crate) struct ChatCompletionTopLogprob { + token: String, + logprob: f32, +} + #[derive(Clone, Deserialize, Serialize)] pub(crate) struct Usage { pub prompt_tokens: u32, @@ -238,7 +308,7 @@ impl ChatCompletion { content: output, }, logprobs: return_logprobs - .then(|| details.tokens.iter().map(|t| t.logprob).collect()), + .then(|| ChatCompletionLogprobs::from((details.tokens, details.top_tokens))), finish_reason: details.finish_reason.to_string(), }], usage: Usage { @@ -266,7 +336,7 @@ pub(crate) struct ChatCompletionChunk { pub(crate) struct ChatCompletionChoice { pub index: u32, pub delta: ChatCompletionDelta, - pub logprobs: Option, + pub logprobs: Option, pub finish_reason: Option, } @@ -285,7 +355,7 @@ impl ChatCompletionChunk { delta: String, created: u64, index: u32, - logprobs: Option, + logprobs: Option, finish_reason: Option, ) -> Self { Self { @@ -319,8 +389,8 @@ pub(crate) struct ChatRequest { /// UNUSED #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")] /// ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API. - pub model: String, /* NOTE: UNUSED */ - + pub model: String, + /* NOTE: UNUSED */ /// A list of messages comprising the conversation so far. #[serde(default = "default_request_messages")] pub messages: Vec, @@ -346,7 +416,6 @@ pub(crate) struct ChatRequest { #[schema(example = "false")] pub logprobs: Option, - /// UNUSED /// An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with /// an associated log probability. logprobs must be set to true if this parameter is used. #[serde(default)] @@ -365,7 +434,6 @@ pub(crate) struct ChatRequest { #[schema(nullable = true, example = "2")] pub n: Option, - /// UNUSED /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, /// increasing the model's likelihood to talk about new topics #[serde(default)] @@ -447,7 +515,7 @@ pub struct PrefillToken { logprob: f32, } -#[derive(Debug, Serialize, ToSchema)] +#[derive(Debug, Serialize, ToSchema, Clone)] pub struct Token { #[schema(example = 0)] id: u32, diff --git a/router/src/queue.rs b/router/src/queue.rs index 106cacc4..73a7169b 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -355,6 +355,7 @@ mod tests { do_sample: false, seed: 0, repetition_penalty: 0.0, + frequency_penalty: 0.0, watermark: false, }, stopping_parameters: StoppingCriteriaParameters { diff --git a/router/src/server.rs b/router/src/server.rs index b4d26158..acfdef91 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -4,9 +4,10 @@ use crate::infer::{InferError, InferResponse, InferStreamResponse}; use crate::validation::ValidationError; use crate::{ BestOfSequence, ChatCompletion, ChatCompletionChoice, ChatCompletionChunk, ChatCompletionDelta, - ChatRequest, CompatGenerateRequest, Details, ErrorResponse, FinishReason, GenerateParameters, - GenerateRequest, GenerateResponse, HubModelInfo, HubTokenizerConfig, Infer, Info, Message, - PrefillToken, SimpleToken, StreamDetails, StreamResponse, Token, TokenizeResponse, Validation, + ChatCompletionLogprobs, ChatRequest, CompatGenerateRequest, Details, ErrorResponse, + FinishReason, GenerateParameters, GenerateRequest, GenerateResponse, HubModelInfo, + HubTokenizerConfig, Infer, Info, Message, PrefillToken, SimpleToken, StreamDetails, + StreamResponse, Token, TokenizeResponse, Validation, }; use axum::extract::Extension; use axum::http::{HeaderMap, Method, StatusCode}; @@ -570,8 +571,8 @@ async fn chat_completions( let stream = req.stream; let max_new_tokens = req.max_tokens.or(Some(100)); let repetition_penalty = req - .frequency_penalty - // rescale frequency_penalty from (-2.0, 2.0) to (0.0, 4.0) + .presence_penalty + // rescale repetition_penalty from (-2.0, 2.0) to (0.0, 4.0) .map(|x| x + 2.0); let logprobs = req.logprobs.unwrap_or(false); let seed = req.seed; @@ -599,6 +600,7 @@ async fn chat_completions( best_of: None, temperature: req.temperature, repetition_penalty, + frequency_penalty: req.frequency_penalty, top_k: None, top_p: req.top_p, typical_p: None, @@ -630,6 +632,10 @@ async fn chat_completions( .unwrap_or_else(|_| std::time::Duration::from_secs(0)) .as_secs(); + let logprobs = logprobs.then(|| { + ChatCompletionLogprobs::from((stream_token.token.clone(), stream_token.top_tokens)) + }); + event .json_data(ChatCompletionChunk::new( model_id.clone(), @@ -637,7 +643,7 @@ async fn chat_completions( stream_token.token.text, current_time, stream_token.index, - logprobs.then_some(stream_token.token.logprob), + logprobs, stream_token.details.map(|d| d.finish_reason.to_string()), )) .map_or_else( diff --git a/router/src/validation.rs b/router/src/validation.rs index 750b98e5..e6874b11 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -170,6 +170,7 @@ impl Validation { best_of, temperature, repetition_penalty, + frequency_penalty, top_k, top_p, typical_p, @@ -206,6 +207,11 @@ impl Validation { return Err(ValidationError::RepetitionPenalty); } + let frequency_penalty = frequency_penalty.unwrap_or(0.0); + if !(-2.0..=2.0).contains(&frequency_penalty) { + return Err(ValidationError::FrequencyPenalty); + } + // Different because the proto default value is not a valid value // for the user let top_p = top_p @@ -289,6 +295,7 @@ impl Validation { let parameters = NextTokenChooserParameters { temperature, repetition_penalty, + frequency_penalty, top_k, top_p, typical_p, @@ -420,6 +427,8 @@ pub enum ValidationError { Temperature, #[error("`repetition_penalty` must be strictly positive")] RepetitionPenalty, + #[error("`frequency_penalty` must be >= -2.0 and <= 2.0")] + FrequencyPenalty, #[error("`top_p` must be > 0.0 and < 1.0")] TopP, #[error("`top_k` must be strictly positive")] diff --git a/server/tests/utils/test_tokens.py b/server/tests/utils/test_tokens.py index d3f2d766..5db32776 100644 --- a/server/tests/utils/test_tokens.py +++ b/server/tests/utils/test_tokens.py @@ -70,7 +70,7 @@ def test_batch_top_tokens(): # Now let's make second member of the batch be speculated inp_logprobs = torch.tensor([[-1.0, -3.0, -4.0, -2.0, -3.0]] * 5 * 2) - accepted_ids[1] = 2 + accepted_ids[1] = 2 topn_tok_ids, topn_tok_logprobs = batch_top_tokens( top_n_tokens, top_n_tokens_tensor, inp_logprobs, accepted_ids ) diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py index a952f060..da7d8416 100644 --- a/server/text_generation_server/models/__init__.py +++ b/server/text_generation_server/models/__init__.py @@ -86,6 +86,7 @@ if MAMBA_AVAILABLE: __all__.append(Mamba) + def get_model( model_id: str, revision: Optional[str], diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index 29e9f8b1..a7a16212 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -696,14 +696,17 @@ def generate_token( if top_n_tokens > 0: all_top_tokens = [] - for (top_token_ids, top_token_logprobs) in zip(top_token_ids, top_token_logprobs): + for (top_token_ids, top_token_logprobs) in zip( + top_token_ids, top_token_logprobs + ): toptoken_texts = self.tokenizer.batch_decode( top_token_ids, clean_up_tokenization_spaces=False, skip_special_tokens=False, ) special_toptokens = [ - token_id in self.all_special_ids for token_id in top_token_ids + token_id in self.all_special_ids + for token_id in top_token_ids ] top_tokens = Tokens( top_token_ids, diff --git a/server/text_generation_server/models/custom_modeling/mamba_modeling.py b/server/text_generation_server/models/custom_modeling/mamba_modeling.py index 1773f04d..017c0341 100644 --- a/server/text_generation_server/models/custom_modeling/mamba_modeling.py +++ b/server/text_generation_server/models/custom_modeling/mamba_modeling.py @@ -19,6 +19,7 @@ from causal_conv1d import causal_conv1d_fn, causal_conv1d_update import math + class MambaConfig(PretrainedConfig): def __init__( self, @@ -53,6 +54,7 @@ def __init__( **kwargs, ) + class MambaBlock(nn.Module): def __init__(self, prefix, config, weights): super().__init__() @@ -60,10 +62,14 @@ def __init__(self, prefix, config, weights): self.in_proj = FastLinear.load(config, f"{prefix}.in_proj", weights, bias=False) self.x_proj = FastLinear.load(config, f"{prefix}.x_proj", weights, bias=False) self.dt_proj = FastLinear.load(config, f"{prefix}.dt_proj", weights, bias=True) - self.dt_proj_no_bias = FastLinear.load(config, f"{prefix}.dt_proj", weights, bias=False) - self.out_proj = FastLinear.load(config, f"{prefix}.out_proj", weights, bias=False) + self.dt_proj_no_bias = FastLinear.load( + config, f"{prefix}.dt_proj", weights, bias=False + ) + self.out_proj = FastLinear.load( + config, f"{prefix}.out_proj", weights, bias=False + ) self.conv1d = FastLinear.load(config, f"{prefix}.conv1d", weights, bias=True) - self.negA = -torch.exp(weights.get_tensor(f"{prefix}.A_log").float()) + self.negA = -torch.exp(weights.get_tensor(f"{prefix}.A_log").float()) self.D = weights.get_tensor(f"{prefix}.D") self.activation = "silu" self.dt_rank = config.dt_rank @@ -80,12 +86,14 @@ def forward(self, hidden_states: torch.Tensor, inference_params=None): out, conv_state, ssm_state = self.step(hidden_states, conv_state, ssm_state) return out, conv_state, ssm_state - projected_states = self.in_proj(hidden_states).transpose(1,2) + projected_states = self.in_proj(hidden_states).transpose(1, 2) x, z = projected_states.chunk(2, dim=1) conv_state = F.pad(x, (self.d_conv - seqlen, 0)) x = causal_conv1d_fn( x=x, - weight=self.conv1d.weight.view(self.conv1d.weight.size(0), self.conv1d.weight.size(2)), + weight=self.conv1d.weight.view( + self.conv1d.weight.size(0), self.conv1d.weight.size(2) + ), bias=self.conv1d.bias, activation=self.activation, ) @@ -94,7 +102,9 @@ def forward(self, hidden_states: torch.Tensor, inference_params=None): # We want dt to have d as the slowest moving dimension # and L as the fastest moving dimension, since those are what the ssm_scan kernel expects. x_dbl = self.x_proj(rearrange(x, "b d l -> (b l) d")) # (bl d) - dt, B, C = torch.split(x_dbl, [self.dt_rank, self.d_state, self.d_state], dim=-1) + dt, B, C = torch.split( + x_dbl, [self.dt_rank, self.d_state, self.d_state], dim=-1 + ) dt = self.dt_proj.weight @ dt.t() dt = rearrange(dt, "d (b l) -> b d l", l=seqlen) B = rearrange(B, "(b l) dstate -> b dstate l", l=seqlen).contiguous() @@ -118,28 +128,39 @@ def forward(self, hidden_states: torch.Tensor, inference_params=None): def step(self, hidden_states, conv_state, ssm_state): _xz = self.in_proj(hidden_states) _x, _z = _xz.chunk(2, dim=-1) # (B D) - conv_state_new = torch.cat([conv_state, _x.transpose(1,2)], dim=-1) - conv_out = causal_conv1d_fn( - x=conv_state_new, - weight=self.conv1d.weight.view(self.conv1d.weight.size(0), self.conv1d.weight.size(2)), - bias=self.conv1d.bias, - activation=self.activation + conv_state_new = torch.cat([conv_state, _x.transpose(1, 2)], dim=-1) + conv_out = causal_conv1d_fn( + x=conv_state_new, + weight=self.conv1d.weight.view( + self.conv1d.weight.size(0), self.conv1d.weight.size(2) + ), + bias=self.conv1d.bias, + activation=self.activation, ) conv_state = conv_state_new[:, :, 1:] bsz, seqlen, dim = hidden_states.shape output_tensor = torch.zeros( - (bsz, seqlen, dim), - device=hidden_states.device, - dtype=hidden_states.dtype + (bsz, seqlen, dim), device=hidden_states.device, dtype=hidden_states.dtype ) for i in range(0, bsz): - x = conv_out[i:i+1,:,-1] - z = _z[i:i+1, -1, :] + x = conv_out[i : i + 1, :, -1] + z = _z[i : i + 1, -1, :] x_db = self.x_proj(x) - dt, B, C = torch.split(x_db, [self.dt_rank, self.d_state, self.d_state], dim=-1) + dt, B, C = torch.split( + x_db, [self.dt_rank, self.d_state, self.d_state], dim=-1 + ) dt = F.linear(dt, self.dt_proj.weight) y = selective_state_update( - ssm_state[i:i+1,:,:], x, dt, self.negA, B, C, self.D, z=z, dt_bias=self.dt_proj.bias, dt_softplus=True + ssm_state[i : i + 1, :, :], + x, + dt, + self.negA, + B, + C, + self.D, + z=z, + dt_bias=self.dt_proj.bias, + dt_softplus=True, ) out = self.out_proj(y) output_tensor[i] = out @@ -147,48 +168,70 @@ def step(self, hidden_states, conv_state, ssm_state): return output_tensor, conv_state, ssm_state - class ResidualBlock(nn.Module): def __init__(self, layer_id, config, weights): super().__init__() - self.mamba_block = MambaBlock(prefix=f"{layer_id}.mixer", config=config, weights=weights) - self.layer_norm = FastRMSNorm.load(prefix=f"{layer_id}.norm", weights=weights, eps=config.layer_norm_epsilon) + self.mamba_block = MambaBlock( + prefix=f"{layer_id}.mixer", config=config, weights=weights + ) + self.layer_norm = FastRMSNorm.load( + prefix=f"{layer_id}.norm", weights=weights, eps=config.layer_norm_epsilon + ) def forward( self, hidden_states: torch.Tensor, residual: Optional[torch.Tensor] = None, inference_params: Optional[Any] = None, - ): + ): residual = (hidden_states + residual) if residual is not None else hidden_states shape = residual.shape hidden_states, _ = self.layer_norm(residual.view(-1, shape[-1])) - hidden_states, conv_state, last_ssm_state = self.mamba_block(hidden_states.view(*shape), inference_params) + hidden_states, conv_state, last_ssm_state = self.mamba_block( + hidden_states.view(*shape), inference_params + ) return hidden_states, residual, conv_state, last_ssm_state + class MambaModel(nn.Module): def __init__(self, config, weights): super().__init__() prefix = "backbone" self.embed_tokens = TensorParallelEmbedding(f"{prefix}.embedding", weights) self.blocks = nn.ModuleList( - [ResidualBlock(f"{prefix}.layers.{i}", config, weights) for i in range(config.n_layer)] + [ + ResidualBlock(f"{prefix}.layers.{i}", config, weights) + for i in range(config.n_layer) + ] + ) + self.norm_f = FastRMSNorm.load( + f"{prefix}.norm_f", weights, eps=config.layer_norm_epsilon + ) + self.lm_head = FastLinear.load( + config, f"{prefix}.embedding", weights, bias=False ) - self.norm_f = FastRMSNorm.load(f"{prefix}.norm_f", weights, eps=config.layer_norm_epsilon) - self.lm_head = FastLinear.load(config, f"{prefix}.embedding", weights, bias=False) self.config = config - def forward(self, input_ids: torch.Tensor, inference_params=None, residual=None) -> Tuple[torch.Tensor, torch.Tensor, InferenceParams]: + def forward( + self, input_ids: torch.Tensor, inference_params=None, residual=None + ) -> Tuple[torch.Tensor, torch.Tensor, InferenceParams]: hidden_states = self.embed_tokens(input_ids) for block in self.blocks: - hidden_states, residual, conv_state, ssm_state = block(hidden_states, residual, inference_params) - inference_params.key_value_memory_dict[block.mamba_block.layer_idx] = (conv_state, ssm_state) + hidden_states, residual, conv_state, ssm_state = block( + hidden_states, residual, inference_params + ) + inference_params.key_value_memory_dict[block.mamba_block.layer_idx] = ( + conv_state, + ssm_state, + ) - hidden_states = hidden_states + residual if residual is not None else hidden_states + hidden_states = ( + hidden_states + residual if residual is not None else hidden_states + ) hidden_states, _ = self.norm_f(hidden_states.view(-1, hidden_states.size(-1))) hidden_states = hidden_states.view(residual.shape) logits = self.lm_head(hidden_states) # update the offset for the next inference using these params inference_params.seqlen_offset += input_ids.size(1) - return logits, input_ids, inference_params \ No newline at end of file + return logits, input_ids, inference_params diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 53a3d582..90776654 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -842,7 +842,6 @@ def generate_token( else: next_token_logits = out - speculate = get_speculate() ( next_input_ids, @@ -1064,14 +1063,17 @@ def generate_token( if top_n_tokens > 0: all_top_tokens = [] - for (top_token_ids, top_token_logprobs) in zip(top_token_ids, top_token_logprobs): + for (top_token_ids, top_token_logprobs) in zip( + top_token_ids, top_token_logprobs + ): toptoken_texts = self.tokenizer.batch_decode( top_token_ids, clean_up_tokenization_spaces=False, skip_special_tokens=False, ) special_toptokens = [ - token_id in self.all_special_ids for token_id in top_token_ids + token_id in self.all_special_ids + for token_id in top_token_ids ] top_tokens = Tokens( top_token_ids, diff --git a/server/text_generation_server/models/mamba.py b/server/text_generation_server/models/mamba.py index c10910aa..c51e1e20 100644 --- a/server/text_generation_server/models/mamba.py +++ b/server/text_generation_server/models/mamba.py @@ -26,6 +26,7 @@ from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling from mamba_ssm.utils.generation import InferenceParams + @dataclass class MambaBatch(Batch): batch_id: int @@ -69,7 +70,7 @@ def to_pb(self) -> generate_pb2.CachedBatch: size=len(self), max_tokens=self.max_tokens, ) - + @classmethod def from_pb( cls, @@ -196,7 +197,7 @@ def filter(self, request_ids: List[int]) -> Optional["MambaBatch"]: new_padding_right_offset = max( new_padding_right_offset, remaining_decode_tokens ) - + # Apply indices to input_ids, attention mask, past key values and other items that need to be cached input_ids = self.input_ids[keep_indices] @@ -218,10 +219,13 @@ def filter(self, request_ids: List[int]) -> Optional["MambaBatch"]: self.padding_right_offset = new_padding_right_offset self.max_tokens = max_tokens - # TODO + # TODO # Kept it simple by just updating the state, maybe updating the other CPU values is necessary. key_value_memory_dict = {} - for i, (conv_state, ssm_state) in self.inference_params.key_value_memory_dict.items(): + for i, ( + conv_state, + ssm_state, + ) in self.inference_params.key_value_memory_dict.items(): key_value_memory_dict[i] = (conv_state[indices], ssm_state[indices]) self.inference_params.key_value_memory_dict = key_value_memory_dict @@ -305,8 +309,9 @@ def concatenate(cls, batches: List["MambaBatch"]) -> "MambaBatch": start_index = end_index - - (_, d_model, d_conv) = batches[0].inference_params.key_value_memory_dict[0][0].shape + (_, d_model, d_conv) = ( + batches[0].inference_params.key_value_memory_dict[0][0].shape + ) (_, _, d_state) = batches[0].inference_params.key_value_memory_dict[0][1].shape n_blocks = len(batches[0].inference_params.key_value_memory_dict) dtype = batches[0].inference_params.key_value_memory_dict[0][0].dtype @@ -344,9 +349,15 @@ def concatenate(cls, batches: List["MambaBatch"]) -> "MambaBatch": for i in range(n_blocks): conv_state, ssm_state = batch.inference_params.key_value_memory_dict[i] batch_size = batch.inference_params.max_batch_size - inference_params.key_value_memory_dict[i][0][current_batch:current_batch + batch_size] = conv_state - inference_params.key_value_memory_dict[i][1][current_batch:current_batch + batch_size] = ssm_state - inference_params.lengths_per_sample[current_batch: current_batch + batch_size] = batch.inference_params.lengths_per_sample + inference_params.key_value_memory_dict[i][0][ + current_batch : current_batch + batch_size + ] = conv_state + inference_params.key_value_memory_dict[i][1][ + current_batch : current_batch + batch_size + ] = ssm_state + inference_params.lengths_per_sample[ + current_batch : current_batch + batch_size + ] = batch.inference_params.lengths_per_sample current_batch += batch_size return cls( @@ -366,12 +377,13 @@ def concatenate(cls, batches: List["MambaBatch"]) -> "MambaBatch": padding_right_offset=padding_right_offset, keys_head_dim_last=batches[0].keys_head_dim_last, max_tokens=max_tokens, - inference_params=inference_params + inference_params=inference_params, ) def __len__(self): return len(self.requests) + class Mamba(Model): def __init__( self, @@ -428,7 +440,7 @@ def batch_type(self) -> Type[MambaBatch]: def warmup(self, batch) -> Optional[int]: # TODO: implement warmup for Mamba if needed return None - + def forward( self, input_ids: torch.Tensor, @@ -441,7 +453,9 @@ def forward( def generate_token(self, batch) -> Tuple[List[Any], Optional[Any], Tuple[int, int]]: start = time.time_ns() - input_ids = batch.input_ids # batch.past_input_ids if batch.past_input_ids is not None else batch.input_ids + input_ids = ( + batch.input_ids + ) # batch.past_input_ids if batch.past_input_ids is not None else batch.input_ids batch_size = input_ids.shape[0] max_seqlen = input_ids.shape[1] @@ -450,8 +464,11 @@ def generate_token(self, batch) -> Tuple[List[Any], Optional[Any], Tuple[int, in # Inference params seqlen_og = 0 inf_cache = {} - lengths_per_sample = torch.ones(batch_size, dtype=torch.int32, device=input_ids.device) * max_seqlen - + lengths_per_sample = ( + torch.ones(batch_size, dtype=torch.int32, device=input_ids.device) + * max_seqlen + ) + if batch.inference_params is None: inference_params = InferenceParams( max_seqlen=max_seqlen, @@ -478,11 +495,16 @@ def generate_token(self, batch) -> Tuple[List[Any], Optional[Any], Tuple[int, in device=block.dt_proj.weight.device, dtype=block.dt_proj.weight.dtype, ) - inference_params.key_value_memory_dict[block.layer_idx] = (conv_state, ssm_state) + inference_params.key_value_memory_dict[block.layer_idx] = ( + conv_state, + ssm_state, + ) batch.inference_params = inference_params - + # Forward pass - logits, past_input_ids, new_inference_params = self.model(input_ids, batch.inference_params) + logits, past_input_ids, new_inference_params = self.model( + input_ids, batch.inference_params + ) batch.inference_params = new_inference_params # Results @@ -564,7 +586,8 @@ def generate_token(self, batch) -> Tuple[List[Any], Optional[Any], Tuple[int, in prefix_offset=len(all_input_ids) - stopping_criteria.current_tokens - 1, - read_offset=len(all_input_ids) - stopping_criteria.current_tokens, + read_offset=len(all_input_ids) + - stopping_criteria.current_tokens, skip_special_tokens=True, ) # Get seed diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 8b93aecd..25042a32 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -750,14 +750,17 @@ def generate_token( if top_n_tokens > 0: all_top_tokens = [] - for (top_token_ids, top_token_logprobs) in zip(top_token_ids, top_token_logprobs): + for (top_token_ids, top_token_logprobs) in zip( + top_token_ids, top_token_logprobs + ): toptoken_texts = self.tokenizer.batch_decode( top_token_ids, clean_up_tokenization_spaces=False, skip_special_tokens=False, ) special_toptokens = [ - token_id in self.all_special_ids for token_id in top_token_ids + token_id in self.all_special_ids + for token_id in top_token_ids ] top_tokens = Tokens( top_token_ids, diff --git a/server/text_generation_server/models/types.py b/server/text_generation_server/models/types.py index bc68812e..da71b0ec 100644 --- a/server/text_generation_server/models/types.py +++ b/server/text_generation_server/models/types.py @@ -95,5 +95,7 @@ def to_pb(self) -> generate_pb2.Generation: generated_text=self.generated_text.to_pb() if self.generated_text is not None else None, - top_tokens=[top_tokens.to_pb() for top_tokens in self.top_tokens] if self.top_tokens is not None else None, + top_tokens=[top_tokens.to_pb() for top_tokens in self.top_tokens] + if self.top_tokens is not None + else None, ) diff --git a/server/text_generation_server/utils/logits_process.py b/server/text_generation_server/utils/logits_process.py index f424eae4..291c522f 100644 --- a/server/text_generation_server/utils/logits_process.py +++ b/server/text_generation_server/utils/logits_process.py @@ -118,6 +118,62 @@ def filter(self, indices): return None +class FrequencyPenaltyLogitsProcessor(LogitsProcessor): + r""" + Frequency penalty as defined by OpenAI + + Args: + penalty (`float`): + The parameter for frequency penalty. 0.0 means no penalty. + """ + + def __init__(self, penalty: float): + self.penalty = penalty + + def __call__( + self, input_ids: torch.LongTensor, scores: torch.FloatTensor + ) -> torch.FloatTensor: + score = torch.gather(scores, 1, input_ids) + # if score < 0 then penalty has to be multiplied to reduce the previous token probability + score = -torch.where( + score < 0, score * self.penalty, score / self.penalty + ) + + return scores.scatter_add_(1, input_ids, score) + + +class HeterogeneousFrequencyPenaltyLogitsProcessor(LogitsProcessor): + r""" + Frequency penalty as defined by OpenAI + + Args: + frequency_penalty (`List[float]`): + The parameter for frequency penalty. 0.0 means no penalty. + """ + + def __init__(self, penalty: List[float], dtype: torch.dtype, device: torch.device): + self.penalty = penalty + self.penalty_tensor = torch.tensor( + penalty, dtype=dtype, device=device + ).unsqueeze(1) + + def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor: + score = torch.gather(scores, 1, input_ids) + # if score < 0 then penalty has to be multiplied to reduce the previous token probability + score = -torch.where( + score < 0, score * self.penalty_tensor, score / self.penalty_tensor + ) + + return scores.scatter_add_(1, input_ids, score) + + def filter(self, indices): + self.penalty = [self.penalty[i] for i in indices] + if any([x != 0.0 for x in self.penalty]): + self.penalty_tensor = self.penalty_tensor[indices] + return self + return None + + class HeterogeneousTemperatureLogitsWarper: r""" [`LogitsWarper`] for temperature (exponential scaling output probability distribution). diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index 270a6990..d6ca10c7 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -1,12 +1,14 @@ import re -from typing import Callable, List, Optional, Tuple +from typing import List, Optional, Tuple import torch from text_generation_server.pb import generate_pb2 from text_generation_server.pb.generate_pb2 import FinishReason from text_generation_server.utils.logits_process import ( + FrequencyPenaltyLogitsProcessor, HeterogeneousProcessorWrapper, HeterogeneousRepetitionPenaltyLogitsProcessor, + HeterogeneousFrequencyPenaltyLogitsProcessor, HeterogeneousTemperatureLogitsWarper, HeterogeneousTopKLogitsWarper, HeterogeneousTopPLogitsWarper, @@ -23,6 +25,7 @@ def __init__( watermark=False, temperature=1.0, repetition_penalty=1.0, + frequency_penalty=0.0, top_k=None, top_p=None, typical_p=None, @@ -35,7 +38,12 @@ def __init__( ) self.repetition_processor = ( RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty) - if repetition_penalty + if repetition_penalty and repetition_penalty != 1.0 + else None + ) + self.frequency_processor = ( + FrequencyPenaltyLogitsProcessor(penalty=frequency_penalty) + if frequency_penalty and frequency_penalty != 0.0 else None ) @@ -60,6 +68,8 @@ def __call__(self, input_ids, scores): scores = self.watermark_processor(input_ids, scores) if self.repetition_processor is not None: scores = self.repetition_processor(input_ids, scores) + if self.frequency_processor is not None: + scores = self.frequency_processor(input_ids, scores) if self.static_warper is None: next_logprob = torch.log_softmax(scores, -1) @@ -80,6 +90,7 @@ def from_pb( watermark=pb.watermark, temperature=pb.temperature, repetition_penalty=pb.repetition_penalty, + frequency_penalty=pb.frequency_penalty, top_k=pb.top_k, top_p=pb.top_p, typical_p=pb.typical_p, @@ -184,6 +195,7 @@ def __init__( watermark: List[bool], temperature: List[float], repetition_penalty: List[float], + frequency_penalty: List[float], top_k: List[int], top_p: List[float], typical_p: List[float], @@ -212,6 +224,14 @@ def __init__( else None ) + self.frequency_processor = ( + HeterogeneousFrequencyPenaltyLogitsProcessor( + frequency_penalty, dtype, device + ) + if any([x != 0.0 for x in frequency_penalty]) + else None + ) + if any([x != 1.0 for x in temperature]): do_sample = [ sample or x != 1.0 for x, sample in zip(temperature, do_sample) @@ -269,6 +289,8 @@ def __call__( _scores = self.watermark_processor(input_ids, _scores) if self.repetition_processor is not None: _scores = self.repetition_processor(input_ids, _scores) + if self.frequency_processor is not None: + _scores = self.frequency_processor(input_ids, _scores) for warper in self.warpers: _scores = warper(input_ids, _scores) @@ -316,7 +338,6 @@ def __call__( next_logprobs = torch.gather(logprobs, 1, next_ids.view(-1, 1)).view(-1) - if speculate > 0: if speculative_scores is not None: # Medusa provided some scores @@ -338,6 +359,9 @@ def filter(self, indices): if self.repetition_processor is not None: self.repetition_processor = self.repetition_processor.filter(indices) + if self.frequency_processor is not None: + self.frequency_processor = self.frequency_processor.filter(indices) + filtered_warpers = [] for warper in self.warpers: filtered_warper = warper.filter(indices) @@ -366,6 +390,7 @@ def from_pb( watermark=[pb_.watermark for pb_ in pb], temperature=[pb_.temperature for pb_ in pb], repetition_penalty=[pb_.repetition_penalty for pb_ in pb], + frequency_penalty=[pb_.frequency_penalty for pb_ in pb], top_k=[pb_.top_k for pb_ in pb], top_p=[pb_.top_p for pb_ in pb], typical_p=[pb_.typical_p for pb_ in pb], @@ -438,7 +463,10 @@ def filter(self, indices): def batch_top_tokens( - top_n_tokens: List[int], top_n_tokens_tensor: torch.Tensor, logprobs: torch.Tensor, accepted_ids: torch.Tensor + top_n_tokens: List[int], + top_n_tokens_tensor: torch.Tensor, + logprobs: torch.Tensor, + accepted_ids: torch.Tensor, ) -> Tuple[List[List[List[int]]], List[List[List[float]]]]: """Find the top n most likely tokens for a batch of generations. @@ -450,12 +478,15 @@ def batch_top_tokens( if max_top_n == 0: return [[[]]] * len(top_n_tokens), [[[]]] * len(top_n_tokens) - batch_size = accepted_ids.shape[0] speculate_size = logprobs.shape[0] // batch_size top_n_tokens_tensor = top_n_tokens_tensor.repeat_interleave(speculate_size) # Ensure top_n doesn't exceed vocab size - top_n_tokens = [min(tok, logprobs.size(-1)) for tok in top_n_tokens for _ in range(speculate_size)] + top_n_tokens = [ + min(tok, logprobs.size(-1)) + for tok in top_n_tokens + for _ in range(speculate_size) + ] # Parallel kthvalue adapted from https://discuss.pytorch.org/t/how-to-efficiently-get-the-k-th-largest-values-in-parallel/160529/2 # Sorted topk is faster than torch.sort() since we only need a small subset @@ -484,10 +515,10 @@ def batch_top_tokens( for i, n_accepted_ids in enumerate(accepted_ids_list): start = speculate_size * i stop = speculate_size * (i + 1) - _top_indices = top_indices[start: stop] - _top_values = top_values[start: stop] - _top_n_ishes = top_n_ishes[start: stop] - _top_n_tokens = top_n_tokens[start: stop] + _top_indices = top_indices[start:stop] + _top_values = top_values[start:stop] + _top_n_ishes = top_n_ishes[start:stop] + _top_n_tokens = top_n_tokens[start:stop] _top_indices = _top_indices[:n_accepted_ids] _top_values = _top_values[:n_accepted_ids] @@ -497,7 +528,9 @@ def batch_top_tokens( row_top_token_ids = [] row_top_token_logprobs = [] - for idxs, vals, n, req_n in zip(_top_indices, _top_values, _top_n_ishes, _top_n_tokens): + for idxs, vals, n, req_n in zip( + _top_indices, _top_values, _top_n_ishes, _top_n_tokens + ): indices = idxs[:n] if req_n > 0 else [] values = vals[:n] if req_n > 0 else [] From c5ef81bed5a1cf24c5048a9589ba74d84e150f13 Mon Sep 17 00:00:00 2001 From: drbh Date: Fri, 9 Feb 2024 04:32:04 -0500 Subject: [PATCH 559/793] chore: bump ci rust version (#1543) This PR bumps the rust toolchain in CI to resolve the CI build issue ```bash Downloaded crossbeam-utils v0.8.19 Downloaded crc32fast v1.3.2 error: failed to compile `text-generation-router v1.4.0 (/home/runner/work/text-generation-inference/text-generation-inference/router)`, intermediate artifacts can be found at `/home/runner/work/text-generation-inference/text-generation-inference/target` Caused by: package `clap_lex v0.7.0` cannot be built because it requires rustc 1.74 or newer, while the currently active rustc version is 1.71.0 Either upgrade to rustc 1.74 or newer, or use cargo update -p clap_lex@0.7.0 --precise ver where `ver` is the latest version of `clap_lex` supporting rustc 1.71.0 make: *** [Makefile:12: install-router] Error 101 ``` --- .github/workflows/tests.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 311ee6b9..ecc8eb4d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -33,7 +33,10 @@ jobs: - name: Install Rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.71.0 + # Released on: 28 December, 2023 + # Branched from master on: 10 November, 2023 + # https://releases.rs/docs/1.75.0/ + toolchain: 1.75.0 override: true components: rustfmt, clippy - name: Install Protoc From a4e5801684ea2b34bc14dbbacffc08ca2f7f71af Mon Sep 17 00:00:00 2001 From: Ilyas Moutawwakil <57442720+IlyasMoutawwakil@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:45:16 +0100 Subject: [PATCH 560/793] ROCm AWQ support (#1514) # What does this PR do? This PR adds the possibility to run AWQ models with Exllama/GPTQ kernels, specifically for ROCm devices that support Exllama kernels but not AWQ's GEMM. This is done by : - un-packing, reordering and re-packing AWQ weights when `--quantize gptq` but the model's `quant_method=awq`. - avoiding overflows when adding 1 to zeros in exllama and triton. Ref: https://github.com/casper-hansen/AutoAWQ/pull/313 ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --------- Co-authored-by: Nicolas Patry --- .../test_flash_llama_gptq.json | 57 +-- .../test_flash_llama_gptq_all_params.json | 25 +- .../test_flash_llama_gptq_load.json | 228 ++++++------ .../test_flash_starcoder_gptq.json | 215 ++++++------ ...t_flash_starcoder_gptq_default_params.json | 59 ++-- .../test_flash_starcoder_gptq_load.json | 332 +++++++++--------- .../exllama_kernels/cuda_func/q4_matmul.cu | 8 +- .../exllama_kernels/cuda_func/q4_matrix.cu | 2 +- .../cuda/q_gemm_kernel_gptq.cuh | 16 +- .../exllamav2_kernels/cuda/q_matrix.cu | 16 +- .../flash_santacoder_modeling.py | 14 +- .../utils/awq/conversion_utils.py | 97 +++++ .../utils/gptq/quant_linear.py | 2 +- server/text_generation_server/utils/layers.py | 7 + .../text_generation_server/utils/weights.py | 112 ++++-- 15 files changed, 688 insertions(+), 502 deletions(-) create mode 100644 server/text_generation_server/utils/awq/conversion_utils.py diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json index e4ffb83b..7797cc6c 100644 --- a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json +++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json @@ -11,78 +11,79 @@ }, { "id": 4321, - "logprob": -9.59375, + "logprob": -9.7890625, "text": "Test" }, { "id": 2009, - "logprob": -9.6640625, + "logprob": -9.625, "text": "request" } ], "seed": null, "tokens": [ - { - "id": 29918, - "logprob": -2.3867188, - "special": false, - "text": "_" - }, - { - "id": 5338, - "logprob": -2.8183594, - "special": false, - "text": "uri" - }, { "id": 13, - "logprob": -1.6367188, + "logprob": -2.3359375, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -1.0527344, + "logprob": -1.8779297, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.6542969, + "logprob": -1.2744141, "special": false, "text": " request" }, { - "id": 29918, - "logprob": -0.056121826, + "id": 13, + "logprob": -1.6933594, "special": false, - "text": "_" + "text": "\n" }, { - "id": 5338, - "logprob": -0.01600647, + "id": 3057, + "logprob": -1.4648438, "special": false, - "text": "uri" + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.15600586, + "special": false, + "text": " request" }, { "id": 13, - "logprob": -0.87939453, + "logprob": -0.8027344, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -0.7529297, + "logprob": -0.23022461, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.2980957, + "logprob": -0.0069885254, "special": false, "text": " request" + }, + { + "id": 13, + "logprob": -0.02218628, + "special": false, + "text": "\n" } - ] + ], + "top_tokens": null }, - "generated_text": "_uri\nTest request_uri\nTest request" + "generated_text": "\nTest request\nTest request\nTest request\n" } diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json index 02713a00..fa2fd4a2 100644 --- a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json +++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json @@ -11,12 +11,12 @@ }, { "id": 4321, - "logprob": -9.6015625, + "logprob": -9.84375, "text": "Test" }, { "id": 2009, - "logprob": -9.6640625, + "logprob": -9.6015625, "text": "request" } ], @@ -24,13 +24,13 @@ "tokens": [ { "id": 29899, - "logprob": -1.1640625, + "logprob": -1.5625, "special": false, "text": "-" }, { "id": 1454, - "logprob": -0.07543945, + "logprob": -0.20410156, "special": false, "text": "for" }, @@ -54,19 +54,19 @@ }, { "id": 396, - "logprob": -0.2956543, + "logprob": -0.27685547, "special": false, "text": " #" }, { "id": 29906, - "logprob": -0.52734375, + "logprob": -0.4970703, "special": false, "text": "2" }, { "id": 29900, - "logprob": -0.6899414, + "logprob": -0.80615234, "special": false, "text": "0" }, @@ -77,12 +77,13 @@ "text": "1" }, { - "id": 29946, - "logprob": -1.5068359, + "id": 29955, + "logprob": -1.0751953, "special": false, - "text": "4" + "text": "7" } - ] + ], + "top_tokens": null }, - "generated_text": "Test request-for-comment: #2014" + "generated_text": "Test request-for-comment: #2017" } diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json index 88bfa4f9..594b7351 100644 --- a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json +++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json @@ -12,80 +12,81 @@ }, { "id": 4321, - "logprob": -9.6015625, + "logprob": -9.828125, "text": "Test" }, { "id": 2009, - "logprob": -9.671875, + "logprob": -9.609375, "text": "request" } ], "seed": null, "tokens": [ - { - "id": 29918, - "logprob": -2.3828125, - "special": false, - "text": "_" - }, - { - "id": 5338, - "logprob": -2.8105469, - "special": false, - "text": "uri" - }, { "id": 13, - "logprob": -1.6396484, + "logprob": -2.3300781, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -1.0546875, + "logprob": -1.8740234, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.6513672, + "logprob": -1.2646484, "special": false, "text": " request" }, { - "id": 29918, - "logprob": -0.056365967, + "id": 13, + "logprob": -1.7158203, "special": false, - "text": "_" + "text": "\n" }, { - "id": 5338, - "logprob": -0.016082764, + "id": 3057, + "logprob": -1.4667969, "special": false, - "text": "uri" + "text": "Test" + }, + { + "id": 2009, + "logprob": -0.15344238, + "special": false, + "text": " request" }, { "id": 13, - "logprob": -0.87841797, + "logprob": -0.81591797, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -0.7548828, + "logprob": -0.22973633, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.29711914, + "logprob": -0.007045746, "special": false, "text": " request" + }, + { + "id": 13, + "logprob": -0.021957397, + "special": false, + "text": "\n" } - ] + ], + "top_tokens": null }, - "generated_text": "_uri\nTest request_uri\nTest request" + "generated_text": "\nTest request\nTest request\nTest request\n" }, { "details": { @@ -100,80 +101,81 @@ }, { "id": 4321, - "logprob": -9.6015625, + "logprob": -9.84375, "text": "Test" }, { "id": 2009, - "logprob": -9.6640625, + "logprob": -9.59375, "text": "request" } ], "seed": null, "tokens": [ - { - "id": 29918, - "logprob": -2.3828125, - "special": false, - "text": "_" - }, - { - "id": 5338, - "logprob": -2.828125, - "special": false, - "text": "uri" - }, { "id": 13, - "logprob": -1.6386719, + "logprob": -2.3378906, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -1.0527344, + "logprob": -1.8779297, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.6542969, + "logprob": -1.2636719, "special": false, "text": " request" }, { - "id": 29918, - "logprob": -0.055877686, + "id": 13, + "logprob": -1.6992188, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -1.4589844, "special": false, - "text": "_" + "text": "Test" }, { - "id": 5338, - "logprob": -0.016021729, + "id": 2009, + "logprob": -0.15344238, "special": false, - "text": "uri" + "text": " request" }, { "id": 13, - "logprob": -0.8769531, + "logprob": -0.79052734, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -0.7583008, + "logprob": -0.22937012, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.29833984, + "logprob": -0.007041931, "special": false, "text": " request" + }, + { + "id": 13, + "logprob": -0.022140503, + "special": false, + "text": "\n" } - ] + ], + "top_tokens": null }, - "generated_text": "_uri\nTest request_uri\nTest request" + "generated_text": "\nTest request\nTest request\nTest request\n" }, { "details": { @@ -188,80 +190,81 @@ }, { "id": 4321, - "logprob": -9.6015625, + "logprob": -9.84375, "text": "Test" }, { "id": 2009, - "logprob": -9.671875, + "logprob": -9.609375, "text": "request" } ], "seed": null, "tokens": [ - { - "id": 29918, - "logprob": -2.3847656, - "special": false, - "text": "_" - }, - { - "id": 5338, - "logprob": -2.8144531, - "special": false, - "text": "uri" - }, { "id": 13, - "logprob": -1.6396484, + "logprob": -2.3261719, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -1.0527344, + "logprob": -1.8730469, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.65478516, + "logprob": -1.2587891, "special": false, "text": " request" }, { - "id": 29918, - "logprob": -0.056243896, + "id": 13, + "logprob": -1.6894531, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -1.46875, "special": false, - "text": "_" + "text": "Test" }, { - "id": 5338, - "logprob": -0.016143799, + "id": 2009, + "logprob": -0.1541748, "special": false, - "text": "uri" + "text": " request" }, { "id": 13, - "logprob": -0.8808594, + "logprob": -0.80322266, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -0.75341797, + "logprob": -0.22912598, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.2956543, + "logprob": -0.0070495605, "special": false, "text": " request" + }, + { + "id": 13, + "logprob": -0.021606445, + "special": false, + "text": "\n" } - ] + ], + "top_tokens": null }, - "generated_text": "_uri\nTest request_uri\nTest request" + "generated_text": "\nTest request\nTest request\nTest request\n" }, { "details": { @@ -276,79 +279,80 @@ }, { "id": 4321, - "logprob": -9.6015625, + "logprob": -9.84375, "text": "Test" }, { "id": 2009, - "logprob": -9.6640625, + "logprob": -9.6015625, "text": "request" } ], "seed": null, "tokens": [ - { - "id": 29918, - "logprob": -2.3769531, - "special": false, - "text": "_" - }, - { - "id": 5338, - "logprob": -2.8183594, - "special": false, - "text": "uri" - }, { "id": 13, - "logprob": -1.6396484, + "logprob": -2.3320312, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -1.0546875, + "logprob": -1.875, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.65478516, + "logprob": -1.2646484, "special": false, "text": " request" }, { - "id": 29918, - "logprob": -0.05557251, + "id": 13, + "logprob": -1.6884766, + "special": false, + "text": "\n" + }, + { + "id": 3057, + "logprob": -1.4589844, "special": false, - "text": "_" + "text": "Test" }, { - "id": 5338, - "logprob": -0.01612854, + "id": 2009, + "logprob": -0.15185547, "special": false, - "text": "uri" + "text": " request" }, { "id": 13, - "logprob": -0.8730469, + "logprob": -0.79833984, "special": false, "text": "\n" }, { "id": 3057, - "logprob": -0.7519531, + "logprob": -0.22827148, "special": false, "text": "Test" }, { "id": 2009, - "logprob": -0.29785156, + "logprob": -0.006996155, "special": false, "text": " request" + }, + { + "id": 13, + "logprob": -0.021560669, + "special": false, + "text": "\n" } - ] + ], + "top_tokens": null }, - "generated_text": "_uri\nTest request_uri\nTest request" + "generated_text": "\nTest request\nTest request\nTest request\n" } ] diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json index 53055e42..5e537bb7 100644 --- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json +++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json @@ -1,193 +1,194 @@ { - "generated_text": "\n return sum(L) / len(L)\n\n\ndef geometric_mean(L", "details": { "best_of_sequences": null, "finish_reason": "length", "generated_tokens": 20, - "seed": null, "prefill": [ { "id": 589, - "text": "def", - "logprob": null + "logprob": null, + "text": "def" }, { "id": 3226, - "text": " ge", - "logprob": -9.0234375 + "logprob": -8.5859375, + "text": " ge" }, { "id": 21017, - "text": "ometric", - "logprob": -9.0859375 + "logprob": -7.5859375, + "text": "ometric" }, { "id": 81, - "text": "_", - "logprob": -0.25878906 + "logprob": -0.2668457, + "text": "_" }, { "id": 6009, - "text": "mean", - "logprob": -2.2109375 + "logprob": -1.6416016, + "text": "mean" }, { "id": 26, - "text": "(", - "logprob": -0.30371094 + "logprob": -0.22705078, + "text": "(" }, { "id": 62, - "text": "L", - "logprob": -5.6054688 + "logprob": -5.2304688, + "text": "L" }, { "id": 44, - "text": ":", - "logprob": -3.0722656 + "logprob": -3.0976562, + "text": ":" }, { "id": 1682, - "text": " List", - "logprob": -0.6879883 + "logprob": -1.1044922, + "text": " List" }, { "id": 77, - "text": "[", - "logprob": -0.38500977 + "logprob": -0.14294434, + "text": "[" }, { "id": 1808, - "text": "float", - "logprob": -0.984375 + "logprob": -0.32299805, + "text": "float" }, { "id": 10794, - "text": "]):", - "logprob": -2.5351562 + "logprob": -2.8164062, + "text": "]):" } ], + "seed": null, "tokens": [ { "id": 284, - "text": "\n ", - "logprob": -1.1738281, - "special": false + "logprob": -0.1282959, + "special": false, + "text": "\n " }, { - "id": 442, - "text": " return", - "logprob": -0.95947266, - "special": false + "id": 1524, + "logprob": -0.97998047, + "special": false, + "text": " \"\"\"" }, { - "id": 3632, - "text": " sum", - "logprob": -1.4199219, - "special": false + "id": 284, + "logprob": -0.7006836, + "special": false, + "text": "\n " }, { - "id": 26, - "text": "(", - "logprob": -0.085876465, - "special": false + "id": 14883, + "logprob": -2.1933594, + "special": false, + "text": " Calculate" }, { - "id": 62, - "text": "L", - "logprob": -0.09875488, - "special": false + "id": 322, + "logprob": -0.2697754, + "special": false, + "text": " the" }, { - "id": 27, - "text": ")", - "logprob": -0.30517578, - "special": false + "id": 3226, + "logprob": -0.0836792, + "special": false, + "text": " ge" }, { - "id": 517, - "text": " /", - "logprob": -0.42089844, - "special": false + "id": 21017, + "logprob": -0.018737793, + "special": false, + "text": "ometric" }, { - "id": 2069, - "text": " len", - "logprob": -0.042053223, - "special": false + "id": 5651, + "logprob": -0.028640747, + "special": false, + "text": " mean" }, { - "id": 26, - "text": "(", - "logprob": -0.0011806488, - "special": false + "id": 432, + "logprob": -0.29467773, + "special": false, + "text": " of" }, { - "id": 62, - "text": "L", - "logprob": -0.0005259514, - "special": false + "id": 312, + "logprob": -0.31518555, + "special": false, + "text": " a" }, { - "id": 27, - "text": ")", - "logprob": -0.0017633438, - "special": false + "id": 1149, + "logprob": -0.20605469, + "special": false, + "text": " list" }, { - "id": 478, - "text": "\n\n", - "logprob": -0.69189453, - "special": false + "id": 432, + "logprob": -0.23254395, + "special": false, + "text": " of" }, { - "id": 203, - "text": "\n", - "logprob": -0.041870117, - "special": false + "id": 7515, + "logprob": -0.4489746, + "special": false, + "text": " numbers" }, { - "id": 589, - "text": "def", - "logprob": -0.27856445, - "special": false + "id": 32, + "logprob": -0.6044922, + "special": false, + "text": "." }, { - "id": 3226, - "text": " ge", - "logprob": -1.7255859, - "special": false + "id": 446, + "logprob": -0.63964844, + "special": false, + "text": "\n\n " }, { - "id": 21017, - "text": "ometric", - "logprob": -0.011291504, - "special": false + "id": 499, + "logprob": -1.1953125, + "special": false, + "text": " :" }, { - "id": 81, - "text": "_", - "logprob": -0.008430481, - "special": false + "id": 753, + "logprob": -0.03515625, + "special": false, + "text": "param" }, { - "id": 6009, - "text": "mean", - "logprob": -0.025787354, - "special": false + "id": 498, + "logprob": -0.06311035, + "special": false, + "text": " L" }, { - "id": 26, - "text": "(", - "logprob": -0.073913574, - "special": false + "id": 44, + "logprob": -0.003414154, + "special": false, + "text": ":" }, { - "id": 62, - "text": "L", - "logprob": -0.09967041, - "special": false + "id": 1682, + "logprob": -1.3310547, + "special": false, + "text": " List" } - ] - } + ], + "top_tokens": null + }, + "generated_text": "\n \"\"\"\n Calculate the geometric mean of a list of numbers.\n\n :param L: List" } diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json index 1ace3814..bf0f5146 100644 --- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json +++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json @@ -11,57 +11,57 @@ }, { "id": 3226, - "logprob": -9.0234375, + "logprob": -8.5859375, "text": " ge" }, { "id": 21017, - "logprob": -9.0859375, + "logprob": -7.5898438, "text": "ometric" }, { "id": 81, - "logprob": -0.25830078, + "logprob": -0.26586914, "text": "_" }, { "id": 6009, - "logprob": -2.1875, + "logprob": -1.6347656, "text": "mean" }, { "id": 26, - "logprob": -0.30004883, + "logprob": -0.22705078, "text": "(" }, { "id": 62, - "logprob": -5.6171875, + "logprob": -5.2382812, "text": "L" }, { "id": 44, - "logprob": -3.078125, + "logprob": -3.0996094, "text": ":" }, { "id": 1682, - "logprob": -0.68066406, + "logprob": -1.1025391, "text": " List" }, { "id": 77, - "logprob": -0.38745117, + "logprob": -0.14294434, "text": "[" }, { "id": 1808, - "logprob": -0.9453125, + "logprob": -0.32226562, "text": "float" }, { "id": 10794, - "logprob": -2.5371094, + "logprob": -2.8164062, "text": "]):" } ], @@ -69,19 +69,19 @@ "tokens": [ { "id": 284, - "logprob": -0.051635742, + "logprob": 0.0, "special": false, "text": "\n " }, { "id": 442, - "logprob": 0.0, + "logprob": -1.3134766, "special": false, "text": " return" }, { "id": 11665, - "logprob": -1.2236328, + "logprob": -0.10021973, "special": false, "text": " reduce" }, @@ -129,7 +129,7 @@ }, { "id": 319, - "logprob": 0.0, + "logprob": -0.42871094, "special": false, "text": " *" }, @@ -158,36 +158,37 @@ "text": ")" }, { - "id": 203, - "logprob": -0.12695312, + "id": 1115, + "logprob": 0.0, "special": false, - "text": "\n" + "text": " **" }, { - "id": 203, + "id": 308, "logprob": 0.0, "special": false, - "text": "\n" + "text": " (" }, { - "id": 589, + "id": 35, "logprob": 0.0, "special": false, - "text": "def" + "text": "1" }, { - "id": 3226, - "logprob": 0.0, + "id": 32, + "logprob": -0.31323242, "special": false, - "text": " ge" + "text": "." }, { - "id": 21017, + "id": 34, "logprob": 0.0, "special": false, - "text": "ometric" + "text": "0" } - ] + ], + "top_tokens": null }, - "generated_text": "\n return reduce(lambda x, y: x * y, L)\n\ndef geometric" + "generated_text": "\n return reduce(lambda x, y: x * y, L) ** (1.0" } diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json index 5381ce5a..46a21ed8 100644 --- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json +++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json @@ -12,57 +12,57 @@ }, { "id": 3226, - "logprob": -9.0234375, + "logprob": -8.5859375, "text": " ge" }, { "id": 21017, - "logprob": -9.0859375, + "logprob": -7.5820312, "text": "ometric" }, { "id": 81, - "logprob": -0.25927734, + "logprob": -0.26708984, "text": "_" }, { "id": 6009, - "logprob": -2.25, + "logprob": -1.6386719, "text": "mean" }, { "id": 26, - "logprob": -0.30126953, + "logprob": -0.22717285, "text": "(" }, { "id": 62, - "logprob": -5.7539062, + "logprob": -5.234375, "text": "L" }, { "id": 44, - "logprob": -3.0878906, + "logprob": -3.1015625, "text": ":" }, { "id": 1682, - "logprob": -0.6845703, + "logprob": -1.1083984, "text": " List" }, { "id": 77, - "logprob": -0.3918457, + "logprob": -0.14294434, "text": "[" }, { "id": 1808, - "logprob": -0.8798828, + "logprob": -0.32592773, "text": "float" }, { "id": 10794, - "logprob": -2.4980469, + "logprob": -2.8164062, "text": "]):" } ], @@ -70,67 +70,68 @@ "tokens": [ { "id": 284, - "logprob": -1.1533203, + "logprob": -0.12817383, "special": false, "text": "\n " }, { - "id": 442, - "logprob": -0.91796875, + "id": 1524, + "logprob": -0.9863281, "special": false, - "text": " return" + "text": " \"\"\"" }, { - "id": 3632, - "logprob": -1.3291016, + "id": 284, + "logprob": -0.7011719, "special": false, - "text": " sum" + "text": "\n " }, { - "id": 26, - "logprob": -0.08062744, + "id": 14883, + "logprob": -2.2050781, "special": false, - "text": "(" + "text": " Calculate" }, { - "id": 62, - "logprob": -0.097717285, + "id": 322, + "logprob": -0.2668457, "special": false, - "text": "L" + "text": " the" }, { - "id": 27, - "logprob": -0.29003906, + "id": 3226, + "logprob": -0.08465576, "special": false, - "text": ")" + "text": " ge" }, { - "id": 517, - "logprob": -0.34958984, + "id": 21017, + "logprob": -0.019012451, "special": false, - "text": " /" + "text": "ometric" }, { - "id": 2069, - "logprob": -0.03829956, + "id": 5651, + "logprob": -0.028625488, "special": false, - "text": " len" + "text": " mean" }, { - "id": 26, - "logprob": -0.0011987686, + "id": 432, + "logprob": -0.29418945, "special": false, - "text": "(" + "text": " of" }, { - "id": 62, - "logprob": -0.00050878525, + "id": 312, + "logprob": -0.3161621, "special": false, - "text": "L" + "text": " a" } - ] + ], + "top_tokens": null }, - "generated_text": "\n return sum(L) / len(L" + "generated_text": "\n \"\"\"\n Calculate the geometric mean of a" }, { "details": { @@ -145,57 +146,57 @@ }, { "id": 3226, - "logprob": -9.0234375, + "logprob": -8.5859375, "text": " ge" }, { "id": 21017, - "logprob": -9.0859375, + "logprob": -7.59375, "text": "ometric" }, { "id": 81, - "logprob": -0.25878906, + "logprob": -0.26953125, "text": "_" }, { "id": 6009, - "logprob": -2.2109375, + "logprob": -1.640625, "text": "mean" }, { "id": 26, - "logprob": -0.30371094, + "logprob": -0.22705078, "text": "(" }, { "id": 62, - "logprob": -5.6054688, + "logprob": -5.234375, "text": "L" }, { "id": 44, - "logprob": -3.0722656, + "logprob": -3.1132812, "text": ":" }, { "id": 1682, - "logprob": -0.6879883, + "logprob": -1.1123047, "text": " List" }, { "id": 77, - "logprob": -0.38500977, + "logprob": -0.14294434, "text": "[" }, { "id": 1808, - "logprob": -0.984375, + "logprob": -0.32299805, "text": "float" }, { "id": 10794, - "logprob": -2.5351562, + "logprob": -2.8164062, "text": "]):" } ], @@ -203,67 +204,68 @@ "tokens": [ { "id": 284, - "logprob": -1.1738281, + "logprob": -0.12854004, "special": false, "text": "\n " }, { - "id": 442, - "logprob": -0.9584961, + "id": 1524, + "logprob": -0.9897461, "special": false, - "text": " return" + "text": " \"\"\"" }, { - "id": 3632, - "logprob": -1.4169922, + "id": 284, + "logprob": -0.69970703, "special": false, - "text": " sum" + "text": "\n " }, { - "id": 26, - "logprob": -0.085876465, + "id": 14883, + "logprob": -2.2050781, "special": false, - "text": "(" + "text": " Calculate" }, { - "id": 62, - "logprob": -0.0982666, + "id": 322, + "logprob": -0.2668457, "special": false, - "text": "L" + "text": " the" }, { - "id": 27, - "logprob": -0.3022461, + "id": 3226, + "logprob": -0.08496094, "special": false, - "text": ")" + "text": " ge" }, { - "id": 517, - "logprob": -0.40504883, + "id": 21017, + "logprob": -0.019012451, "special": false, - "text": " /" + "text": "ometric" }, { - "id": 2069, - "logprob": -0.041656494, + "id": 5651, + "logprob": -0.029037476, "special": false, - "text": " len" + "text": " mean" }, { - "id": 26, - "logprob": -0.0011844635, + "id": 432, + "logprob": -0.2939453, "special": false, - "text": "(" + "text": " of" }, { - "id": 62, - "logprob": -0.0005264282, + "id": 312, + "logprob": -0.31591797, "special": false, - "text": "L" + "text": " a" } - ] + ], + "top_tokens": null }, - "generated_text": "\n return sum(L) / len(L" + "generated_text": "\n \"\"\"\n Calculate the geometric mean of a" }, { "details": { @@ -278,57 +280,57 @@ }, { "id": 3226, - "logprob": -9.0234375, + "logprob": -8.5859375, "text": " ge" }, { "id": 21017, - "logprob": -9.0859375, + "logprob": -7.5859375, "text": "ometric" }, { "id": 81, - "logprob": -0.25927734, + "logprob": -0.26586914, "text": "_" }, { "id": 6009, - "logprob": -2.25, + "logprob": -1.6347656, "text": "mean" }, { "id": 26, - "logprob": -0.30126953, + "logprob": -0.22766113, "text": "(" }, { "id": 62, - "logprob": -5.7539062, + "logprob": -5.2265625, "text": "L" }, { "id": 44, - "logprob": -3.0878906, + "logprob": -3.0976562, "text": ":" }, { "id": 1682, - "logprob": -0.6845703, + "logprob": -1.1025391, "text": " List" }, { "id": 77, - "logprob": -0.3918457, + "logprob": -0.1427002, "text": "[" }, { "id": 1808, - "logprob": -0.8798828, + "logprob": -0.32592773, "text": "float" }, { "id": 10794, - "logprob": -2.4980469, + "logprob": -2.8164062, "text": "]):" } ], @@ -336,67 +338,68 @@ "tokens": [ { "id": 284, - "logprob": -1.1533203, + "logprob": -0.13012695, "special": false, "text": "\n " }, { - "id": 442, - "logprob": -0.9165039, + "id": 1524, + "logprob": -0.98046875, "special": false, - "text": " return" + "text": " \"\"\"" }, { - "id": 3632, - "logprob": -1.328125, + "id": 284, + "logprob": -0.69921875, "special": false, - "text": " sum" + "text": "\n " }, { - "id": 26, - "logprob": -0.07946777, + "id": 14883, + "logprob": -2.1992188, "special": false, - "text": "(" + "text": " Calculate" }, { - "id": 62, - "logprob": -0.09820557, + "id": 322, + "logprob": -0.2668457, "special": false, - "text": "L" + "text": " the" }, { - "id": 27, - "logprob": -0.28930664, + "id": 3226, + "logprob": -0.083496094, "special": false, - "text": ")" + "text": " ge" }, { - "id": 517, - "logprob": -0.34592773, + "id": 21017, + "logprob": -0.01902771, "special": false, - "text": " /" + "text": "ometric" }, { - "id": 2069, - "logprob": -0.038330078, + "id": 5651, + "logprob": -0.029006958, "special": false, - "text": " len" + "text": " mean" }, { - "id": 26, - "logprob": -0.0011940002, + "id": 432, + "logprob": -0.29248047, "special": false, - "text": "(" + "text": " of" }, { - "id": 62, - "logprob": -0.00050878525, + "id": 312, + "logprob": -0.3161621, "special": false, - "text": "L" + "text": " a" } - ] + ], + "top_tokens": null }, - "generated_text": "\n return sum(L) / len(L" + "generated_text": "\n \"\"\"\n Calculate the geometric mean of a" }, { "details": { @@ -411,57 +414,57 @@ }, { "id": 3226, - "logprob": -9.0234375, + "logprob": -8.5859375, "text": " ge" }, { "id": 21017, - "logprob": -9.0859375, + "logprob": -7.5859375, "text": "ometric" }, { "id": 81, - "logprob": -0.25927734, + "logprob": -0.26904297, "text": "_" }, { "id": 6009, - "logprob": -2.25, + "logprob": -1.6386719, "text": "mean" }, { "id": 26, - "logprob": -0.30126953, + "logprob": -0.22705078, "text": "(" }, { "id": 62, - "logprob": -5.7539062, + "logprob": -5.234375, "text": "L" }, { "id": 44, - "logprob": -3.0878906, + "logprob": -3.1132812, "text": ":" }, { "id": 1682, - "logprob": -0.6845703, + "logprob": -1.1074219, "text": " List" }, { "id": 77, - "logprob": -0.3918457, + "logprob": -0.14477539, "text": "[" }, { "id": 1808, - "logprob": -0.8798828, + "logprob": -0.3256836, "text": "float" }, { "id": 10794, - "logprob": -2.4980469, + "logprob": -2.8027344, "text": "]):" } ], @@ -469,66 +472,67 @@ "tokens": [ { "id": 284, - "logprob": -1.1533203, + "logprob": -0.12915039, "special": false, "text": "\n " }, { - "id": 442, - "logprob": -0.91259766, + "id": 1524, + "logprob": -0.98535156, "special": false, - "text": " return" + "text": " \"\"\"" }, { - "id": 3632, - "logprob": -1.3251953, + "id": 284, + "logprob": -0.69921875, "special": false, - "text": " sum" + "text": "\n " }, { - "id": 26, - "logprob": -0.08062744, + "id": 14883, + "logprob": -2.2011719, "special": false, - "text": "(" + "text": " Calculate" }, { - "id": 62, - "logprob": -0.09906006, + "id": 322, + "logprob": -0.26708984, "special": false, - "text": "L" + "text": " the" }, { - "id": 27, - "logprob": -0.28979492, + "id": 3226, + "logprob": -0.08502197, "special": false, - "text": ")" + "text": " ge" }, { - "id": 517, - "logprob": -0.35958984, + "id": 21017, + "logprob": -0.019012451, "special": false, - "text": " /" + "text": "ometric" }, { - "id": 2069, - "logprob": -0.038604736, + "id": 5651, + "logprob": -0.028625488, "special": false, - "text": " len" + "text": " mean" }, { - "id": 26, - "logprob": -0.0011901855, + "id": 432, + "logprob": -0.29589844, "special": false, - "text": "(" + "text": " of" }, { - "id": 62, - "logprob": -0.0005078316, + "id": 312, + "logprob": -0.31591797, "special": false, - "text": "L" + "text": " a" } - ] + ], + "top_tokens": null }, - "generated_text": "\n return sum(L) / len(L" + "generated_text": "\n \"\"\"\n Calculate the geometric mean of a" } ] diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu index 61380f42..09126efe 100644 --- a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu @@ -85,7 +85,7 @@ __global__ void q4_matmul_kernel if constexpr (use_half2) { half2 w_scale = w_scales_.item_half2half2(group, w_column); - uint32_t w_zero = w_zeros_.item(group, w_column) + 1; + uint32_t w_zero = (w_zeros_.item(group, w_column) + 1) & 0x0F; if constexpr (use_x_map) acc = dot_product_8_x_map(acc, x_, x_row, k, w_, k, w_column, w_scale, w_zero, groupsize / 8, x_map); else acc = dot_product_8 (acc, x_, x_row, k, w_, k, w_column, w_scale, w_zero, groupsize / 8); @@ -93,7 +93,7 @@ __global__ void q4_matmul_kernel else { half w_scale = w_scales_.item(group, w_column); - uint32_t w_zero = w_zeros_.item(group, w_column) + 1; + uint32_t w_zero = (w_zeros_.item(group, w_column) + 1) & 0x0F; if constexpr (use_x_map) acc_h = dot_product_8_x_map_h(acc_h, x_, x_row, k, w_, k, w_column, w_scale, w_zero, groupsize / 8, x_map); else acc_h = dot_product_8_h (acc_h, x_, x_row, k, w_, k, w_column, w_scale, w_zero, groupsize / 8); @@ -110,7 +110,7 @@ __global__ void q4_matmul_kernel { int group = k / groupsize; half2 w_scale = w_scales_.item_half2half2(group, w_column); - uint32_t w_zero = w_zeros_.item(group, w_column) + 1; + uint32_t w_zero = (w_zeros_.item(group, w_column) + 1) & 0x0F; if constexpr (use_x_map) acc = dot_product_8_x_map(acc, x_, x_row, k, w_, k, w_column, w_scale, w_zero, 1, x_map); else acc = dot_product_8 (acc, x_, x_row, k, w_, k, w_column, w_scale, w_zero, 1); @@ -119,7 +119,7 @@ __global__ void q4_matmul_kernel { int group = k / groupsize; half w_scale = w_scales_.item(group, w_column); - uint32_t w_zero = w_zeros_.item(group, w_column) + 1; + uint32_t w_zero = (w_zeros_.item(group, w_column) + 1) & 0x0F; if constexpr (use_x_map) acc_h = dot_product_8_x_map_h(acc_h, x_, x_row, k, w_, k, w_column, w_scale, w_zero, 1, x_map); else acc_h = dot_product_8_h (acc_h, x_, x_row, k, w_, k, w_column, w_scale, w_zero, 1); diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu index f3d1564f..2867a8d0 100644 --- a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu @@ -189,7 +189,7 @@ __global__ void reconstruct_kernel int group = row / groupsize; half w_scale = w_scales_.item(group, column); - uint32_t w_zero = w_zeros_.item(group, column) + 1; + uint32_t w_zero = (w_zeros_.item(group, column) + 1) & 0x0F; uint32_t w_read = w_.item_uint32_t(row, column); half* out_ptr = out_.item_ptr(row, column); diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh index 74b0db2b..f816fd9d 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm_kernel_gptq.cuh @@ -152,10 +152,10 @@ __global__ void gemm_half_q_half_gptq_kernel half2 y1y16[4][2]; b_gptq_qzeros_.item4(zeros, group, n); b_gptq_scales_.item4_h2(scales, group, n); - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + dequant_4bit_8_prep_zero((zeros[0] + 1) & 0x0F, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero((zeros[1] + 1) & 0x0F, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero((zeros[2] + 1) & 0x0F, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero((zeros[3] + 1) & 0x0F, z1z16[3], y1y16[3]); // __syncthreads(); @@ -174,10 +174,10 @@ __global__ void gemm_half_q_half_gptq_kernel nextgroup += groupsize; b_gptq_qzeros_.item4(zeros, group, n); b_gptq_scales_.item4_h2(scales, group, n); - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + dequant_4bit_8_prep_zero((zeros[0] + 1) & 0x0F, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero((zeros[1] + 1) & 0x0F, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero((zeros[2] + 1) & 0x0F, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero((zeros[3] + 1) & 0x0F, z1z16[3], y1y16[3]); } #pragma unroll diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu index ae08cc1f..7a0038b4 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu @@ -237,10 +237,10 @@ __global__ void reconstruct_gptq_kernel half2 y1y16[4][2]; b_gptq_qzeros_.item4(zeros, group, n); b_gptq_scales_.item4_h2(scales, group, n); - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + dequant_4bit_8_prep_zero((zeros[0] + 1) & 0x0F, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero((zeros[1] + 1) & 0x0F, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero((zeros[2] + 1) & 0x0F, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero((zeros[3] + 1) & 0x0F, z1z16[3], y1y16[3]); __syncthreads(); @@ -255,10 +255,10 @@ __global__ void reconstruct_gptq_kernel nextgroup += groupsize; b_gptq_qzeros_.item4(zeros, group, n); b_gptq_scales_.item4_h2(scales, group, n); - dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); - dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); - dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); - dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + dequant_4bit_8_prep_zero((zeros[0] + 1) & 0x0F, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero((zeros[1] + 1) & 0x0F, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero((zeros[2] + 1) & 0x0F, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero((zeros[3] + 1) & 0x0F, z1z16[3], y1y16[3]); } for (int p = 0; p < 4; p++) diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py index 22d03adf..81041046 100644 --- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py @@ -69,9 +69,17 @@ def _load_multi_mqa_gptq( qzeros = torch.cat([q_tensor, kv_tensor], dim=1) qzeros = qzeros.to(device=weights.device) - g_idx = weights.get_tensor(f"{prefix}.c_attn.g_idx") - g_idx = g_idx.to(device=weights.device) - bits, groupsize, _ = weights._get_gptq_params() + bits, groupsize, _, quant_method, = weights._get_gptq_params() + if quant_method == "gptq": + g_idx = weights.get_tensor(f"{prefix}.c_attn.g_idx") + g_idx = g_idx.to(device=weights.device) + elif quant_method == "awq": + g_idx = None + from text_generation_server.utils.awq.conversion_utils import ( + fast_awq_to_gptq, + ) + + qweight, qzeros = fast_awq_to_gptq(qweight, qzeros) from text_generation_server.utils.layers import HAS_EXLLAMA diff --git a/server/text_generation_server/utils/awq/conversion_utils.py b/server/text_generation_server/utils/awq/conversion_utils.py new file mode 100644 index 00000000..b19eafbb --- /dev/null +++ b/server/text_generation_server/utils/awq/conversion_utils.py @@ -0,0 +1,97 @@ +import torch +from typing import List + + +AWQ_PACK_ORDER = [0, 2, 4, 6, 1, 3, 5, 7] +REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7] + + +def pack(imatrix: torch.Tensor, direction: str = "column"): + """ + Packs a 4-bit integer matrix into a packed 32-bit integer matrix. + Args: + imatrix (torch.Tensor): matrix of integers + direction (str): direction of packing, either "column" or "row" + Returns: + qmatrix (torch.Tensor): packed matrix of integers + """ + shifts = torch.arange(0, 32, 4, dtype=torch.int32, device=imatrix.device) + + imatrix = imatrix.to(torch.int8) & 0x0F # eventually correct overflow + + if direction == "column": + imatrix = imatrix.view(-1, imatrix.shape[1] // (32 // 4), (32 // 4)) + qmatrix = torch.bitwise_left_shift(imatrix, shifts[None, None, :]).sum(dim=-1) + + elif direction == "row": + imatrix = imatrix.view(imatrix.shape[0] // (32 // 4), (32 // 4), -1) + qmatrix = torch.bitwise_left_shift(imatrix, shifts[None, :, None]).sum(dim=1) + + qmatrix = qmatrix.to(torch.int32) + + return qmatrix + + +def unpack(qmatrix: torch.Tensor, direction: str = "column"): + """ + Unpacks a 32-bit packed integer matrix into a 4-bit integer matrix. + Args: + qmatrix (torch.Tensor): matrix of packed integers + direction (str): direction of unpacking, either "column" or "row" + Returns: + imatrix (torch.Tensor): matrix of integers + """ + shifts = torch.arange(0, 32, 4, device=qmatrix.device) + + if direction == "column": + imatrix = torch.bitwise_right_shift( + qmatrix[:, :, None], shifts[None, None, :] + ).view(qmatrix.shape[0], -1) + + elif direction == "row": + imatrix = torch.bitwise_right_shift( + qmatrix[:, None, :], shifts[None, :, None] + ).view(-1, qmatrix.shape[-1]) + + imatrix = imatrix.to(torch.int8) & 0x0F # eventually correct overflow + + return imatrix + + +def apply_order( + imatrix: torch.Tensor, + direction: str = "column", + order: List[int] = AWQ_PACK_ORDER, +): + """ + Applies the order to a 4-bit integer matrix. + Args: + imatrix (torch.Tensor): matrix of integers + direction (str): direction of applying order, either "column" or "row" + order (List[int]): order to apply, default is AWQ_PACK_ORDER + Returns: + imatrix (torch.Tensor): matrix of integers + """ + if direction == "column": + imatrix = imatrix.view(-1, (32 // 4))[:, order].view(imatrix.shape) + elif direction == "row": + imatrix = imatrix.view((32 // 4), -1)[order, :].view(imatrix.shape) + + return imatrix + + +def fast_awq_to_gptq(qweight, qzeros): + # awq uses column packing for both weights and zeros + izeros = unpack(qzeros, direction="column") + iweights = unpack(qweight, direction="column") + + # Reverse the order of the iweight and izeros tensors + izeros = apply_order(izeros, direction="column", order=REVERSE_AWQ_PACK_ORDER) + iweights = apply_order(iweights, direction="column", order=REVERSE_AWQ_PACK_ORDER) + # Subtract 1 from the izeros tensor (gptq adds 1 to the zeros) + izeros = izeros - 1 + # exllama uses row packing for weights and column packing for zeros + qzeros = pack(izeros, direction="column") + qweight = pack(iweights, direction="row") + + return qweight, qzeros diff --git a/server/text_generation_server/utils/gptq/quant_linear.py b/server/text_generation_server/utils/gptq/quant_linear.py index bfc91c00..8ad0dd80 100644 --- a/server/text_generation_server/utils/gptq/quant_linear.py +++ b/server/text_generation_server/utils/gptq/quant_linear.py @@ -182,7 +182,7 @@ def matmul_248_kernel( ) # (BLOCK_SIZE_K, BLOCK_SIZE_N,) zeros = (zeros >> zeros_shifter[None, :]) & maxq - zeros = zeros + 1 + zeros = (zeros + 1) & maxq # eventually avoid overflow a = tl.load(a_ptrs, mask=a_mask, other=0.0) # (BLOCK_SIZE_M, BLOCK_SIZE_K) b = tl.load(b_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N), but repeated diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py index 010d6143..01e32588 100644 --- a/server/text_generation_server/utils/layers.py +++ b/server/text_generation_server/utils/layers.py @@ -349,6 +349,13 @@ def get_linear(weight, bias, quantize): raise NotImplementedError( f"The passed weight is not `awq` compatible, loader needs to be updated." ) + if IS_ROCM_SYSTEM: + raise NotImplementedError( + "AWQ GEMM kernel can't be used on ROCm systems, please use `--quantize gptq` instead " + "to use Exllama/GPTQ kernels for AWQ inference." + ) + if not HAS_AWQ: + raise NotImplementedError("You do not seem to have awq installed, either install it (cd server && make install-awq), or try using GPTQ `---quantize gptq` a conversion AWQ->GPTQ will happen on the fly") linear = WQLinear( w_bit=bits, group_size=groupsize, diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 186733f3..8f7e1f10 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -46,7 +46,6 @@ def _get_handle(self, filename): return self._handles[filename] def get_filename(self, tensor_name: str) -> (str, str): - names = [tensor_name] if self.prefix is not None: prefixed = f"{self.prefix}.{tensor_name}" @@ -154,15 +153,30 @@ def get_weights_col_packed_qkv(self, prefix: str, quantize: str): f"Cannot load `{quantize}` weight, make sure the model is already quantized." ) + bits, groupsize, _, quant_method = self._get_gptq_params() + qzeros = self._get_qweight(f"{prefix}.qzeros") scales = self._get_qweight(f"{prefix}.scales") scales = scales.to(dtype=self.dtype) - if quantize == "gptq": + + if quantize == "gptq" and quant_method == "gptq": g_idx = self.get_tensor(f"{prefix}.g_idx") + elif quantize == "gptq" and quant_method == "awq": + log_once( + logger.info, "Converting AWQ model to Exllama/GPTQ packing format." + ) + from text_generation_server.utils.awq.conversion_utils import ( + fast_awq_to_gptq, + ) + + qweight, qzeros = fast_awq_to_gptq(qweight, qzeros) + g_idx = ( + torch.arange(qweight.shape[0] * (32 // bits), device=qweight.device) + // groupsize + ).to(dtype=torch.int32) else: g_idx = None - bits, groupsize, _ = self._get_gptq_params() weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) else: slice_ = self._get_slice(f"{prefix}.weight") @@ -204,20 +218,40 @@ def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int): [self.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1 ) - if quantize == "gptq": + bits, groupsize, desc_act, quant_method = self._get_gptq_params() + + from text_generation_server.utils.layers import HAS_EXLLAMA + + use_exllama = ( + bits == 4 and HAS_EXLLAMA and quantize == "gptq" and not desc_act + ) + + if quantize == "gptq" and quant_method == "gptq": w = [self.get_tensor(f"{p}.g_idx") for p in prefixes] for w2 in w[1:]: torch.testing.assert_close(w2, w[0]) g_idx = w[0] + elif quantize == "gptq" and quant_method == "awq": + log_once( + logger.info, "Converting AWQ model to Exllama/GPTQ packing format." + ) + from text_generation_server.utils.awq.conversion_utils import ( + fast_awq_to_gptq, + ) + + qweight, qzeros = fast_awq_to_gptq(qweight, qzeros) + if use_exllama: + g_idx = None + else: + g_idx = ( + torch.arange( + qweight.shape[0] * (32 // bits), device=qweight.device + ) + // groupsize + ).to(dtype=torch.int32) else: g_idx = None - bits, groupsize, desc_act = self._get_gptq_params() - from text_generation_server.utils.layers import HAS_EXLLAMA - - use_exllama = ( - bits == 4 and HAS_EXLLAMA and quantize == "gptq" and not desc_act - ) weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) else: w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes] @@ -243,7 +277,7 @@ def get_tensor_shard(self, var, dim): def get_multi_weights_row(self, prefix: str, quantize: str): if quantize == "gptq": use_exllama = True - bits, groupsize, desc_act = self._get_gptq_params() + bits, groupsize, desc_act, quant_method = self._get_gptq_params() if bits != 4: use_exllama = False @@ -252,8 +286,19 @@ def get_multi_weights_row(self, prefix: str, quantize: str): log_once(logger.warning, "Disabling exllama because desc_act=True") use_exllama = False + try: + qweight = self.get_sharded(f"{prefix}.qweight", dim=0) + except RuntimeError: + raise RuntimeError( + "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" + ) + + if quant_method == "gptq": + g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) + elif quant_method == "awq": + g_idx = None + if self.process_group.size() > 1: - g_idx = self.get_tensor(f"{prefix}.g_idx") if g_idx is not None: if ( not torch.equal( @@ -269,13 +314,6 @@ def get_multi_weights_row(self, prefix: str, quantize: str): # it would require to reorder input activations that are split unto several GPUs use_exllama = False - try: - qweight = self.get_sharded(f"{prefix}.qweight", dim=0) - except RuntimeError: - raise RuntimeError( - "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`" - ) - from text_generation_server.utils.layers import HAS_EXLLAMA, CAN_EXLLAMA if use_exllama: @@ -289,8 +327,6 @@ def get_multi_weights_row(self, prefix: str, quantize: str): else: log_once(logger.info, f"Using exllama kernels v{HAS_EXLLAMA}") - g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0) - if use_exllama and groupsize != -1: qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0) scales = self.get_sharded(f"{prefix}.scales", dim=0) @@ -298,12 +334,31 @@ def get_multi_weights_row(self, prefix: str, quantize: str): qzeros = self.get_tensor(f"{prefix}.qzeros") scales = self.get_tensor(f"{prefix}.scales") - if use_exllama: + if use_exllama and g_idx is not None: g_idx = g_idx - g_idx[0] + if quant_method == "awq": + log_once( + logger.info, "Converting AWQ model to Exllama/GPTQ packing format." + ) + from text_generation_server.utils.awq.conversion_utils import ( + fast_awq_to_gptq, + ) + + qweight, qzeros = fast_awq_to_gptq(qweight, qzeros) + if use_exllama: + g_idx = None + else: + g_idx = ( + torch.arange( + qweight.shape[0] * (32 // bits), device=qweight.device + ) + // groupsize + ).to(dtype=torch.int32) + weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama) elif quantize == "awq": - bits, groupsize, _ = self._get_gptq_params() + bits, groupsize, _, _ = self._get_gptq_params() try: qweight = self.get_sharded(f"{prefix}.qweight", dim=0) @@ -322,20 +377,22 @@ def get_multi_weights_row(self, prefix: str, quantize: str): weight = self.get_sharded(f"{prefix}.weight", dim=1) return weight - def _get_gptq_params(self) -> Tuple[int, int, int]: + def _get_gptq_params(self) -> Tuple[int, int, int, str]: try: bits = self.get_tensor("gptq_bits").item() groupsize = self.get_tensor("gptq_groupsize").item() desc_act = False + quant_method = "gptq" except (SafetensorError, RuntimeError) as e: try: bits = self.gptq_bits groupsize = self.gptq_groupsize desc_act = getattr(self, "gptq_desc_act", False) + quant_method = getattr(self, "quant_method", "gptq") except Exception: raise e - return bits, groupsize, desc_act + return bits, groupsize, desc_act, quant_method def _set_gptq_params(self, model_id, revision): filename = "config.json" @@ -351,6 +408,7 @@ def _set_gptq_params(self, model_id, revision): self.gptq_bits = data["quantization_config"]["bits"] self.gptq_groupsize = data["quantization_config"]["group_size"] self.gptq_desc_act = data["quantization_config"]["desc_act"] + self.quant_method = data["quantization_config"]["quant_method"] except Exception: filename = "quantize_config.json" try: @@ -365,6 +423,8 @@ def _set_gptq_params(self, model_id, revision): self.gptq_bits = data["bits"] self.gptq_groupsize = data["group_size"] self.gptq_desc_act = data["desc_act"] + if "version" in data and data["version"] == "GEMM": + self.quant_method = "awq" except Exception: filename = "quant_config.json" try: @@ -379,5 +439,7 @@ def _set_gptq_params(self, model_id, revision): self.gptq_bits = data["w_bit"] self.gptq_groupsize = data["q_group_size"] self.gptq_desc_act = data["desc_act"] + if "version" in data and data["version"] == "GEMM": + self.quant_method = "awq" except Exception: pass From 532146338bfcc6af86efbe61825206a9e913f37f Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 9 Feb 2024 12:38:41 +0100 Subject: [PATCH 561/793] feat(router): add max_batch_size (#1542) Some hardware require a maximum batch size. --- docs/source/basic_tutorials/launcher.md | 8 +++ launcher/src/main.rs | 11 ++++ router/client/src/client.rs | 6 ++ router/client/src/sharded_client.rs | 8 ++- router/src/infer.rs | 13 +++- router/src/lib.rs | 2 + router/src/main.rs | 5 ++ router/src/queue.rs | 83 +++++++++++++++++++------ router/src/server.rs | 3 + 9 files changed, 118 insertions(+), 21 deletions(-) diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index 712b4fc4..ba54f058 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -197,6 +197,14 @@ Options: [env: MAX_WAITING_TOKENS=] [default: 20] +``` +## MAX_BATCH_SIZE +```shell + --max-batch-size + Enforce a maximum number of requests per batch Specific flag for hardware targets that do not support unpadded inference + + [env: MAX_BATCH_SIZE=] + ``` ## HOSTNAME ```shell diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 054e546c..a51742e6 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -279,6 +279,11 @@ struct Args { #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, + /// Enforce a maximum number of requests per batch + /// Specific flag for hardware targets that do not support unpadded inference + #[clap(long, env)] + max_batch_size: Option, + /// The IP address to listen on #[clap(default_value = "0.0.0.0", long, env)] hostname: String, @@ -1046,6 +1051,12 @@ fn spawn_webserver( router_args.push(max_batch_total_tokens.to_string()); } + // Router optional max batch size + if let Some(max_batch_size) = args.max_batch_size { + router_args.push("--max-batch-size".to_string()); + router_args.push(max_batch_size.to_string()); + } + // Model optional revision if let Some(ref revision) = args.revision { router_args.push("--revision".to_string()); diff --git a/router/client/src/client.rs b/router/client/src/client.rs index fde5c402..7b9f90fb 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -105,6 +105,7 @@ impl Client { max_input_length: u32, max_prefill_tokens: u32, max_total_tokens: u32, + max_batch_size: Option, ) -> Result> { let mut n_tokens = 0; let mut requests = Vec::new(); @@ -137,6 +138,11 @@ impl Client { top_n_tokens: 20, }); n_tokens += max_input_length; + + // Check max_batch_size + if Some(requests.len()) == max_batch_size { + break; + } } let batch = Batch { diff --git a/router/client/src/sharded_client.rs b/router/client/src/sharded_client.rs index f0e65ce5..e1e52d59 100644 --- a/router/client/src/sharded_client.rs +++ b/router/client/src/sharded_client.rs @@ -97,12 +97,18 @@ impl ShardedClient { max_input_length: u32, max_prefill_tokens: u32, max_total_tokens: u32, + max_batch_size: Option, ) -> Result> { let futures: Vec<_> = self .clients .iter_mut() .map(|client| { - Box::pin(client.warmup(max_input_length, max_prefill_tokens, max_total_tokens)) + Box::pin(client.warmup( + max_input_length, + max_prefill_tokens, + max_total_tokens, + max_batch_size, + )) }) .collect(); // Take the minimum value diff --git a/router/src/infer.rs b/router/src/infer.rs index 4da0da0a..d7b9b52b 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -61,6 +61,7 @@ impl Infer { max_batch_prefill_tokens: u32, max_batch_total_tokens: u32, max_waiting_tokens: usize, + max_batch_size: Option, max_concurrent_requests: usize, requires_padding: bool, window_size: Option, @@ -81,6 +82,7 @@ impl Infer { max_batch_prefill_tokens, max_batch_total_tokens, max_waiting_tokens, + max_batch_size, queue.clone(), shared.clone(), generation_health, @@ -338,6 +340,7 @@ async fn batching_task( max_batch_prefill_tokens: u32, max_batch_total_tokens: u32, max_waiting_tokens: usize, + max_batch_size: Option, queue: Queue, shared: Arc, generation_health: Arc, @@ -351,7 +354,12 @@ async fn batching_task( // This batch might be smaller than the maximum batch size if there are not enough requests // waiting in the queue while let Some((mut entries, batch, span)) = queue - .next_batch(None, max_batch_prefill_tokens, max_batch_total_tokens) + .next_batch( + None, + max_batch_size, + max_batch_prefill_tokens, + max_batch_total_tokens, + ) .await { let mut cached_batch = prefill(&mut client, batch, &mut entries, &generation_health) @@ -379,10 +387,11 @@ async fn batching_task( }; let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens); + let max_size = max_batch_size.map(|max_size| max_size - batch_size as usize); // Try to get a new batch if let Some((mut new_entries, new_batch, span)) = queue - .next_batch(min_size, max_batch_prefill_tokens, token_budget) + .next_batch(min_size, max_size, max_batch_prefill_tokens, token_budget) .await { // Tracking metrics diff --git a/router/src/lib.rs b/router/src/lib.rs index 7c44d642..3ce9eca8 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -73,6 +73,8 @@ pub struct Info { pub max_batch_total_tokens: u32, #[schema(example = "20")] pub max_waiting_tokens: usize, + #[schema(nullable = true, example = "null")] + pub max_batch_size: Option, #[schema(example = "2")] pub validation_workers: usize, /// Router Info diff --git a/router/src/main.rs b/router/src/main.rs index 2a080468..a1f8d97b 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -45,6 +45,8 @@ struct Args { max_batch_total_tokens: Option, #[clap(default_value = "20", long, env)] max_waiting_tokens: usize, + #[clap(long, env)] + max_batch_size: Option, #[clap(default_value = "0.0.0.0", long, env)] hostname: String, #[clap(default_value = "3000", long, short, env)] @@ -91,6 +93,7 @@ async fn main() -> Result<(), RouterError> { max_batch_prefill_tokens, max_batch_total_tokens, max_waiting_tokens, + max_batch_size, hostname, port, master_shard_uds_path, @@ -288,6 +291,7 @@ async fn main() -> Result<(), RouterError> { max_input_length as u32, max_batch_prefill_tokens, max_total_tokens as u32, + max_batch_size, ) .await .map_err(RouterError::Warmup)? @@ -344,6 +348,7 @@ async fn main() -> Result<(), RouterError> { max_batch_prefill_tokens, max_supported_batch_total_tokens, max_waiting_tokens, + max_batch_size, sharded_client, tokenizer, validation_workers, diff --git a/router/src/queue.rs b/router/src/queue.rs index 73a7169b..3675e0f5 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -70,6 +70,7 @@ impl Queue { pub(crate) async fn next_batch( &self, min_size: Option, + max_size: Option, prefill_token_budget: u32, token_budget: u32, ) -> Option { @@ -80,6 +81,7 @@ impl Queue { self.queue_sender .send(QueueCommand::NextBatch { min_size, + max_size, prefill_token_budget, token_budget, response_sender, @@ -110,12 +112,14 @@ async fn queue_task( } QueueCommand::NextBatch { min_size, + max_size, prefill_token_budget, token_budget, response_sender, span, } => span.in_scope(|| { - let next_batch = state.next_batch(min_size, prefill_token_budget, token_budget); + let next_batch = + state.next_batch(min_size, max_size, prefill_token_budget, token_budget); response_sender.send(next_batch).unwrap(); metrics::gauge!("tgi_queue_size", state.entries.len() as f64); }), @@ -181,6 +185,7 @@ impl State { fn next_batch( &mut self, min_size: Option, + max_size: Option, prefill_token_budget: u32, token_budget: u32, ) -> Option { @@ -274,6 +279,11 @@ impl State { entry.batch_time = Some(Instant::now()); // Insert in batch_entries IntMap batch_entries.insert(id, entry); + + // Check if max_size + if Some(batch_requests.len()) == max_size { + break; + } } // Empty batch @@ -322,6 +332,7 @@ enum QueueCommand { Append(Box, Span), NextBatch { min_size: Option, + max_size: Option, prefill_token_budget: u32, token_budget: u32, response_sender: oneshot::Sender>, @@ -394,8 +405,8 @@ mod tests { fn test_next_batch_empty() { let mut state = State::new(false, 1, None, 0); - assert!(state.next_batch(None, 1, 1).is_none()); - assert!(state.next_batch(Some(1), 1, 1).is_none()); + assert!(state.next_batch(None, None, 1, 1).is_none()); + assert!(state.next_batch(Some(1), None, 1, 1).is_none()); } #[test] @@ -406,7 +417,7 @@ mod tests { state.append(entry1); state.append(entry2); - let (entries, batch, _) = state.next_batch(None, 2, 2).unwrap(); + let (entries, batch, _) = state.next_batch(None, None, 2, 2).unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&0)); assert!(entries.contains_key(&1)); @@ -422,7 +433,7 @@ mod tests { let (entry3, _guard3) = default_entry(); state.append(entry3); - assert!(state.next_batch(Some(2), 2, 2).is_none()); + assert!(state.next_batch(Some(2), None, 2, 2).is_none()); assert_eq!(state.next_id, 3); assert_eq!(state.entries.len(), 1); @@ -430,6 +441,26 @@ mod tests { assert_eq!(id, 2); } + #[test] + fn test_next_batch_max_size() { + let mut state = State::new(false, 1, None, 0); + let (entry1, _guard1) = default_entry(); + let (entry2, _guard2) = default_entry(); + state.append(entry1); + state.append(entry2); + + let (entries, batch, _) = state.next_batch(None, Some(1), 2, 2).unwrap(); + assert_eq!(entries.len(), 1); + assert!(entries.contains_key(&0)); + assert!(entries.get(&0).unwrap().batch_time.is_some()); + assert_eq!(batch.id, 0); + assert_eq!(batch.size, 1); + + assert_eq!(state.next_id, 2); + assert_eq!(state.entries.len(), 1); + assert_eq!(state.next_batch_id, 1); + } + #[test] fn test_next_batch_token_budget() { let mut state = State::new(false, 1, None, 0); @@ -438,7 +469,7 @@ mod tests { state.append(entry1); state.append(entry2); - let (entries, batch, _) = state.next_batch(None, 1, 1).unwrap(); + let (entries, batch, _) = state.next_batch(None, None, 1, 1).unwrap(); assert_eq!(entries.len(), 1); assert!(entries.contains_key(&0)); assert_eq!(batch.id, 0); @@ -451,7 +482,7 @@ mod tests { let (entry3, _guard3) = default_entry(); state.append(entry3); - let (entries, batch, _) = state.next_batch(None, 3, 3).unwrap(); + let (entries, batch, _) = state.next_batch(None, None, 3, 3).unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&1)); assert!(entries.contains_key(&2)); @@ -474,8 +505,8 @@ mod tests { async fn test_queue_next_batch_empty() { let queue = Queue::new(false, 1, None, 0); - assert!(queue.next_batch(None, 1, 1).await.is_none()); - assert!(queue.next_batch(Some(1), 1, 1).await.is_none()); + assert!(queue.next_batch(None, None, 1, 1).await.is_none()); + assert!(queue.next_batch(Some(1), None, 1, 1).await.is_none()); } #[tokio::test] @@ -486,7 +517,7 @@ mod tests { queue.append(entry1); queue.append(entry2); - let (entries, batch, _) = queue.next_batch(None, 2, 2).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, None, 2, 2).await.unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&0)); assert!(entries.contains_key(&1)); @@ -499,11 +530,11 @@ mod tests { queue.append(entry3); // Not enough requests pending - assert!(queue.next_batch(Some(2), 2, 2).await.is_none()); + assert!(queue.next_batch(Some(2), None, 2, 2).await.is_none()); // Not enough token budget - assert!(queue.next_batch(Some(1), 0, 0).await.is_none()); + assert!(queue.next_batch(Some(1), None, 0, 0).await.is_none()); // Ok - let (entries2, batch2, _) = queue.next_batch(Some(1), 2, 2).await.unwrap(); + let (entries2, batch2, _) = queue.next_batch(Some(1), None, 2, 2).await.unwrap(); assert_eq!(entries2.len(), 1); assert!(entries2.contains_key(&2)); assert!(entries2.get(&2).unwrap().batch_time.is_some()); @@ -511,6 +542,22 @@ mod tests { assert_eq!(batch2.size, 1); } + #[tokio::test] + async fn test_queue_next_batch_max_size() { + let queue = Queue::new(false, 1, None, 0); + let (entry1, _guard1) = default_entry(); + let (entry2, _guard2) = default_entry(); + queue.append(entry1); + queue.append(entry2); + + let (entries, batch, _) = queue.next_batch(None, Some(1), 2, 2).await.unwrap(); + assert_eq!(entries.len(), 1); + assert!(entries.contains_key(&0)); + assert!(entries.get(&0).unwrap().batch_time.is_some()); + assert_eq!(batch.id, 0); + assert_eq!(batch.size, 1); + } + #[tokio::test] async fn test_queue_next_batch_token_budget() { let queue = Queue::new(false, 1, None, 0); @@ -519,7 +566,7 @@ mod tests { queue.append(entry1); queue.append(entry2); - let (entries, batch, _) = queue.next_batch(None, 1, 1).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, None, 1, 1).await.unwrap(); assert_eq!(entries.len(), 1); assert!(entries.contains_key(&0)); assert_eq!(batch.id, 0); @@ -528,7 +575,7 @@ mod tests { let (entry3, _guard3) = default_entry(); queue.append(entry3); - let (entries, batch, _) = queue.next_batch(None, 3, 3).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, None, 3, 3).await.unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&1)); assert!(entries.contains_key(&2)); @@ -545,9 +592,9 @@ mod tests { queue.append(entry2); // Budget of 1 is not enough - assert!(queue.next_batch(None, 1, 1).await.is_none()); + assert!(queue.next_batch(None, None, 1, 1).await.is_none()); - let (entries, batch, _) = queue.next_batch(None, 6, 6).await.unwrap(); + let (entries, batch, _) = queue.next_batch(None, None, 6, 6).await.unwrap(); assert_eq!(entries.len(), 2); assert!(entries.contains_key(&0)); assert!(entries.contains_key(&1)); @@ -561,6 +608,6 @@ mod tests { let (entry, _) = default_entry(); queue.append(entry); - assert!(queue.next_batch(None, 1, 1).await.is_none()); + assert!(queue.next_batch(None, None, 1, 1).await.is_none()); } } diff --git a/router/src/server.rs b/router/src/server.rs index acfdef91..00b793e3 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -768,6 +768,7 @@ pub async fn run( max_batch_prefill_tokens: u32, max_batch_total_tokens: u32, max_waiting_tokens: usize, + max_batch_size: Option, client: ShardedClient, tokenizer: Option, validation_workers: usize, @@ -849,6 +850,7 @@ pub async fn run( max_batch_prefill_tokens, max_batch_total_tokens, max_waiting_tokens, + max_batch_size, max_concurrent_requests, shard_info.requires_padding, shard_info.window_size, @@ -930,6 +932,7 @@ pub async fn run( waiting_served_ratio, max_batch_total_tokens, max_waiting_tokens, + max_batch_size, validation_workers, version: env!("CARGO_PKG_VERSION"), sha: option_env!("VERGEN_GIT_SHA"), From 0d794af6a5730efcfb23aa898232c6df3afec347 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Mon, 12 Feb 2024 10:09:29 +0100 Subject: [PATCH 562/793] feat: experimental support for cuda graphs (#1428) Co-authored-by: Nicolas Patry --- Dockerfile | 4 +- Dockerfile_amd | 2 +- docs/source/basic_tutorials/launcher.md | 8 ++ integration-tests/conftest.py | 5 +- launcher/src/main.rs | 14 +- server/Makefile-awq | 6 +- .../exllama_kernels/cuda_func/q4_matmul.cu | 5 +- .../exllama_kernels/cuda_func/q4_matmul.cuh | 8 +- .../exllama_kernels/cuda_func/q4_matrix.cu | 11 +- .../exllama_kernels/exllama_ext.cpp | 6 +- .../exllamav2_kernels/cuda/q_gemm.cu | 5 +- .../exllamav2_kernels/cuda/q_matrix.cu | 11 +- .../custom_modeling/flash_mistral_modeling.py | 8 +- .../custom_modeling/flash_mixtral_modeling.py | 8 +- .../models/flash_causal_lm.py | 131 ++++++++++++++++-- .../models/flash_mistral.py | 123 ++++++++++++++-- .../text_generation_server/utils/weights.py | 3 +- 17 files changed, 300 insertions(+), 58 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6818005f..252c5885 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Rust builder -FROM lukemathwalker/cargo-chef:latest-rust-1.71 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.75 AS chef WORKDIR /usr/src ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse @@ -166,7 +166,7 @@ FROM kernel-builder as megablocks-builder RUN pip install git+https://github.com/OlivierDehaene/megablocks@181709df192de9a941fdf3a641cdc65a0462996e # Text Generation Inference base image -FROM nvidia/cuda:12.1.0-base-ubuntu20.04 as base +FROM nvidia/cuda:12.1.0-base-ubuntu22.04 as base # Conda env ENV PATH=/opt/conda/bin:$PATH \ diff --git a/Dockerfile_amd b/Dockerfile_amd index d2b6f897..c2ec4a6d 100644 --- a/Dockerfile_amd +++ b/Dockerfile_amd @@ -1,5 +1,5 @@ # Rust builder -FROM lukemathwalker/cargo-chef:latest-rust-1.71 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.75 AS chef WORKDIR /usr/src ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index ba54f058..be31a7a4 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -205,6 +205,14 @@ Options: [env: MAX_BATCH_SIZE=] +``` +## ENABLE_CUDA_GRAPHS +```shell + --enable-cuda-graphs + Enable experimental support for cuda graphs + + [env: ENABLE_CUDA_GRAPHS=] + ``` ## HOSTNAME ```shell diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index 4cb4ca59..efeda08d 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -317,7 +317,10 @@ def docker_launcher( gpu_count = num_shard if num_shard is not None else 1 - env = {"LOG_LEVEL": "info,text_generation_router=debug"} + env = { + "LOG_LEVEL": "info,text_generation_router=debug", + "ENABLE_CUDA_GRAPHS": "true", + } if not use_flash_attention: env["USE_FLASH_ATTENTION"] = "false" diff --git a/launcher/src/main.rs b/launcher/src/main.rs index a51742e6..8367ef81 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -284,6 +284,10 @@ struct Args { #[clap(long, env)] max_batch_size: Option, + /// Enable experimental support for cuda graphs + #[clap(long, env)] + enable_cuda_graphs: bool, + /// The IP address to listen on #[clap(default_value = "0.0.0.0", long, env)] hostname: String, @@ -407,6 +411,7 @@ fn shard_manager( disable_custom_kernels: bool, watermark_gamma: Option, watermark_delta: Option, + enable_cuda_graphs: bool, cuda_memory_fraction: f32, rope_scaling: Option, rope_factor: Option, @@ -488,7 +493,7 @@ fn shard_manager( envs.push(("WORLD_SIZE".into(), world_size.to_string().into())); envs.push(("MASTER_ADDR".into(), master_addr.into())); envs.push(("MASTER_PORT".into(), master_port.to_string().into())); - envs.push(("NCCL_ASYNC_ERROR_HANDLING".into(), "1".into())); + envs.push(("TORCH_NCCL_AVOID_RECORD_STREAMS".into(), "1".into())); // CUDA memory fraction envs.push(( @@ -538,6 +543,11 @@ fn shard_manager( )); }; + // Enable experimental support for cuda graphs + if enable_cuda_graphs { + envs.push(("ENABLE_CUDA_GRAPHS".into(), "True".into())) + } + // If disable_custom_kernels is true, pass it to the shard as an env var if disable_custom_kernels { envs.push(("DISABLE_CUSTOM_KERNELS".into(), "True".into())) @@ -926,6 +936,7 @@ fn spawn_shards( let disable_custom_kernels = args.disable_custom_kernels; let watermark_gamma = args.watermark_gamma; let watermark_delta = args.watermark_delta; + let enable_cuda_graphs = args.enable_cuda_graphs; let cuda_memory_fraction = args.cuda_memory_fraction; let rope_scaling = args.rope_scaling; let rope_factor = args.rope_factor; @@ -947,6 +958,7 @@ fn spawn_shards( disable_custom_kernels, watermark_gamma, watermark_delta, + enable_cuda_graphs, cuda_memory_fraction, rope_scaling, rope_factor, diff --git a/server/Makefile-awq b/server/Makefile-awq index 80e78c08..5dd9dbaa 100644 --- a/server/Makefile-awq +++ b/server/Makefile-awq @@ -1,8 +1,10 @@ -awq_commit := f084f40bd996f3cf3a0633c1ad7d9d476c318aaa +# Fork that adds only the correct stream to this kernel in order +# to make cuda graphs work. +awq_commit := bd1dc2d5254345cc76ab71894651fb821275bdd4 awq: rm -rf llm-awq - git clone https://github.com/mit-han-lab/llm-awq + git clone https://github.com/huggingface/llm-awq build-awq: awq cd llm-awq/ && git fetch && git checkout $(awq_commit) diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu index 09126efe..1b0f7956 100644 --- a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cu @@ -1,5 +1,6 @@ #include "q4_matmul.cuh" #include "column_remap.cuh" +#include #include "../util.cuh" #include "../matrix.cuh" #include "../cu_compat.cuh" @@ -224,8 +225,8 @@ void q4_matmul_recons_cuda const int x_height, Q4Matrix* w, half* out, - const cublasHandle_t handle, - bool no_zero + bool no_zero, + const cublasHandle_t handle ) { int height = x_height; diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cuh b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cuh index 63611790..4c7a6669 100644 --- a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cuh +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matmul.cuh @@ -19,8 +19,8 @@ void q4_matmul_cuda const int x_height, const Q4Matrix* w, half* out, - bool no_zero = false, - cudaStream_t alt_stream = NULL + bool no_zero, + cudaStream_t alt_stream ); void q4_matmul_recons_cuda @@ -30,8 +30,8 @@ void q4_matmul_recons_cuda const int x_height, Q4Matrix* w, half* out, - const cublasHandle_t handle, - bool no_zero = false + bool no_zero, + const cublasHandle_t handle ); #endif diff --git a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu index 2867a8d0..1f32e6b8 100644 --- a/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu +++ b/server/exllama_kernels/exllama_kernels/cuda_func/q4_matrix.cu @@ -1,5 +1,6 @@ // Adapted from turboderp exllama: https://github.com/turboderp/exllama +#include #include "q4_matrix.cuh" #include #include "../util.cuh" @@ -90,7 +91,7 @@ __global__ void make_sequential_kernel int w2_row_shift = w2_subrow << 2; int wnew2_row_shift = i << 2; - uint64_t src = w2[w2_row * w2_stride + w2_column]; + uint64_t src = w2[w2_row * w2_stride + w2_column]; src >>= w2_row_shift; src &= 0x0000000f0000000f; src <<= wnew2_row_shift; @@ -146,7 +147,8 @@ void Q4Matrix::make_sequential(const uint32_t* cpu_g_idx) dim3 threads(UNSHUF_BLOCKSIZE_X, 1, 1); dim3 blocks(width / UNSHUF_BLOCKSIZE_X / 2, height / 8, 1); - make_sequential_kernel<<>>(cuda_qweight, cuda_new_qweight, cuda_x_map, height / 8, width); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + make_sequential_kernel<<>>(cuda_qweight, cuda_new_qweight, cuda_x_map, height / 8, width); // Replace qweights @@ -213,5 +215,6 @@ void Q4Matrix::reconstruct(half* out) 1 ); - reconstruct_kernel<<>>(cuda_qweight, out, cuda_scales, cuda_qzeros, height / 8, width, groupsize); -} \ No newline at end of file + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + reconstruct_kernel<<>>(cuda_qweight, out, cuda_scales, cuda_qzeros, height / 8, width, groupsize); +} diff --git a/server/exllama_kernels/exllama_kernels/exllama_ext.cpp b/server/exllama_kernels/exllama_kernels/exllama_ext.cpp index b786988b..f2df80e8 100644 --- a/server/exllama_kernels/exllama_kernels/exllama_ext.cpp +++ b/server/exllama_kernels/exllama_kernels/exllama_ext.cpp @@ -183,6 +183,7 @@ void q4_matmul int x_height = x.size(0); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); if (tuningParams.matmul_recons_thd == 0 || x_height < tuningParams.matmul_recons_thd) { q4_matmul_cuda @@ -191,7 +192,9 @@ void q4_matmul (half*) x.data_ptr(), x_height, wm, - (half*) out.data_ptr() + (half*) out.data_ptr(), + false, + stream ); } else @@ -203,6 +206,7 @@ void q4_matmul x_height, wm, (half*) out.data_ptr(), + false, at::cuda::getCurrentCUDABlasHandle() ); } diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu index b4e4cf22..5b99f1ba 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_gemm.cu @@ -38,6 +38,7 @@ void gemm_half_q_half_cuda_part bool mul_r_weights ) { + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); if (!b->is_gptq) { dim3 blockDim, gridDim; @@ -50,7 +51,7 @@ void gemm_half_q_half_cuda_part fp_gemm_half_q_half_kernel kernel = pick_gemm_half_q_half_kernel(m_count, r_weights != NULL, mul_r_weights); - kernel<<>> + kernel<<>> ( a, b->cuda_q_weight, @@ -91,7 +92,7 @@ void gemm_half_q_half_cuda_part // print_global_mem(r_weights, 1, 1, 1); // DBGI(r_weights_stride); - kernel<<>> + kernel<<>> ( a, b->cuda_q_weight, diff --git a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu index 7a0038b4..f7a91e29 100644 --- a/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu +++ b/server/exllamav2_kernels/exllamav2_kernels/cuda/q_matrix.cu @@ -168,8 +168,9 @@ QMatrix::QMatrix blockDim.y = 1; gridDim.x = DIVIDE(width, THREADS_X); gridDim.y = 1; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); - shuffle_kernel<<>>(cuda_q_weight, height, width, rows_8, rows_6, rows_5, rows_4, rows_3, rows_2); + shuffle_kernel<<>>(cuda_q_weight, height, width, rows_8, rows_6, rows_5, rows_4, rows_3, rows_2); } QMatrix::~QMatrix() @@ -475,11 +476,12 @@ void QMatrix::reconstruct(half* out) blockDim.x = BLOCK_KN_SIZE; blockDim.y = 1; gridDim.y = DIVIDE(height, BLOCK_KN_SIZE); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); if (!is_gptq) { gridDim.x = DIVIDE(width, BLOCK_KN_SIZE); - reconstruct_kernel<<>> + reconstruct_kernel<<>> ( cuda_q_weight, cuda_q_perm, @@ -502,7 +504,7 @@ void QMatrix::reconstruct(half* out) else { gridDim.x = DIVIDE(width, BLOCK_KN_SIZE * 4); - reconstruct_gptq_kernel<<>> + reconstruct_gptq_kernel<<>> ( cuda_q_weight, cuda_q_perm, @@ -563,6 +565,7 @@ __global__ void make_sequential_kernel bool QMatrix::make_sequential(const uint32_t* cpu_g_idx) { + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); uint32_t* cuda_new_qweight = NULL; cudaError_t err = cudaMalloc(&cuda_new_qweight, height / 8 * width * sizeof(uint32_t)); if (err != cudaSuccess) { @@ -621,7 +624,7 @@ bool QMatrix::make_sequential(const uint32_t* cpu_g_idx) gridDim.x = DIVIDE(width, THREADS_X); gridDim.y = height / 8; - make_sequential_kernel<<>> + make_sequential_kernel<<>> ( cuda_q_weight, cuda_new_qweight, diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py index 0fc4e1b3..7b45be57 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py @@ -425,6 +425,11 @@ def __init__(self, config, weights): weights=weights, ) self.max_past = config.sliding_window + self.max_past_tensor = ( + torch.tensor(config.sliding_window, device=weights.device) + if self.max_past is not None + else None + ) def forward( self, @@ -446,8 +451,7 @@ def forward( elif self.max_past is not None: # Clamp in decode mode as paged attention requires clamped values whereas the flash attention # kernel requires the true values - max_s = min(self.max_past, max_s) - input_lengths = torch.clamp(input_lengths, max=self.max_past) + input_lengths = torch.clamp(input_lengths, max=self.max_past_tensor) hidden_states = self.model( input_ids, diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py index 61488ec4..c91b2224 100644 --- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py +++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py @@ -816,6 +816,11 @@ def __init__(self, config, weights): weights=weights, ) self.max_past = config.sliding_window + self.max_past_tensor = ( + torch.tensor(config.sliding_window, device=weights.device) + if self.max_past is not None + else None + ) def forward( self, @@ -837,8 +842,7 @@ def forward( elif self.max_past is not None: # Clamp in decode mode as paged attention requires clamped values whereas the flash attention # kernel requires the true values - max_s = min(self.max_past, max_s) - input_lengths = torch.clamp(input_lengths, max=self.max_past) + input_lengths = torch.clamp(input_lengths, max=self.max_past_tensor) hidden_states = self.model( input_ids, diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 90776654..c7fda516 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -1,4 +1,5 @@ import math +import os import time import itertools import torch @@ -6,6 +7,7 @@ import numpy as np +from loguru import logger from dataclasses import dataclass from opentelemetry import trace from transformers import PreTrainedTokenizerBase @@ -31,6 +33,8 @@ tracer = trace.get_tracer(__name__) +MEM_POOL = torch.cuda.graph_pool_handle() + @dataclass class FlashCausalLMBatch(Batch): @@ -62,7 +66,7 @@ class FlashCausalLMBatch(Batch): # Set in prefill by the CacheManager # list of length b of list of length s_i // block_size block_tables: Optional[List[List[int]]] - # tensor of size [b, max_seqlen // block_size] holding the paged attention block tables for all sequences + # tensor of size [b, max_total_seqlen // block_size] holding the paged attention block tables for all sequences block_tables_tensor: Optional[torch.Tensor] # tensor of length \sum_{i=0}^{b} max_s_i holding the paged attention slots for all sequences slots: Optional[torch.Tensor] @@ -663,6 +667,8 @@ def __init__( self.num_kv_heads = num_kv_heads self.head_size = head_size + self.cuda_graphs = {} + super(FlashCausalLM, self).__init__( model=model, tokenizer=tokenizer, @@ -678,7 +684,60 @@ def __init__( def batch_type(self) -> Type[FlashCausalLMBatch]: return FlashCausalLMBatch + def cuda_graph_warmup(self, bs: int, max_s: int, max_bt: int): + input_ids = torch.zeros(bs, dtype=torch.int64, device=self.device) + position_ids = torch.zeros(bs, dtype=torch.int32, device=self.device) + slots = torch.arange(bs, dtype=torch.int32, device=self.device) + input_lengths = torch.ones(bs, dtype=torch.int32, device=self.device) * max_s + block_tables = ( + torch.arange(max_bt, dtype=torch.int32, device=self.device) + .repeat(bs) + .reshape((bs, max_bt)) + ) + kv_cache = get_cache_manager().kv_cache + + self.cuda_graphs[bs] = { + "input_ids": input_ids, + "position_ids": position_ids, + "kv_cache": kv_cache, + "block_tables": block_tables, + "slots": slots, + "input_lengths": input_lengths, + } + graph = torch.cuda.CUDAGraph() + self.cuda_graphs[bs]["graph"] = graph + + torch.cuda.synchronize() + # Run once outside to warmup + self.model.forward( + input_ids=input_ids, + position_ids=position_ids, + cu_seqlen_prefill=None, + kv_cache=kv_cache, + block_tables=block_tables, + slots=slots, + input_lengths=input_lengths, + max_s=max_s, + lm_head_indices=None, + ) + torch.cuda.synchronize() + + with torch.cuda.graph(graph, pool=MEM_POOL): + self.cuda_graphs[bs]["logits"] = self.model.forward( + input_ids=input_ids, + position_ids=position_ids, + cu_seqlen_prefill=None, + kv_cache=kv_cache, + block_tables=block_tables, + slots=slots, + input_lengths=input_lengths, + max_s=max_s, + lm_head_indices=None, + ) + torch.cuda.synchronize() + def warmup(self, batch: FlashCausalLMBatch): + # The warmup batch is the biggest batch we could ever receive torch.cuda.empty_cache() try: cache_manager = set_cache_manager( @@ -690,6 +749,8 @@ def warmup(self, batch: FlashCausalLMBatch): self.dtype, self.device, ) + max_bt = batch.max_blocks + max_s = max_bt * get_cache_manager().block_size _, batch, _ = self.generate_token(batch) except torch.cuda.OutOfMemoryError as e: raise RuntimeError( @@ -713,7 +774,8 @@ def warmup(self, batch: FlashCausalLMBatch): ) num_blocks = ( - int(free_memory // total_cache_size) + # Leave 5% for some wiggle room + int((free_memory * 0.95) // total_cache_size) # Add batch.blocks as we allocated it above, so it is included in the peak memory. + cache_manager.num_blocks ) @@ -731,9 +793,19 @@ def warmup(self, batch: FlashCausalLMBatch): self.device, ) + if os.getenv("ENABLE_CUDA_GRAPHS", "False") == "True": + try: + logger.info("Experimental support for Cuda Graphs is enabled") + # Warmup cuda graphs + for bs in [1, 2, 4] + [8 * i for i in range(8)]: + if self.speculate is None or self.speculate + 1 <= bs: + self.cuda_graph_warmup(bs, max_s, max_bt) + except Exception: + logger.exception(f"Decode cuda graph warmup failed") + return int(num_blocks * BLOCK_SIZE) - def forward(self, batch: FlashCausalLMBatch) -> Tuple[torch.Tensor, torch.Tensor]: + def forward(self, batch: FlashCausalLMBatch) -> torch.Tensor: # Model Forward if batch.speculative_ids is not None: input_ids = batch.input_ids @@ -785,17 +857,48 @@ def forward(self, batch: FlashCausalLMBatch) -> Tuple[torch.Tensor, torch.Tensor max_s = batch.max_seqlen lm_head_indices = batch.prefill_head_indices - return self.model.forward( - input_ids=input_ids, - position_ids=position_ids, - cu_seqlen_prefill=cu_seqlen_prefill, - kv_cache=kv_cache, - block_tables=block_tables, - slots=slots, - input_lengths=input_lengths, - max_s=max_s, - lm_head_indices=lm_head_indices, - ) + bs = input_ids.shape[0] + padded_bs = bs + if bs == 3: + padded_bs = 4 + elif 3 < bs <= 8: + padded_bs = 8 + elif bs > 8: + padded_bs = (bs + 7) // 8 * 8 + + # Try to find an associated cuda graph + cuda_graph = self.cuda_graphs.get(padded_bs, None) + + if cu_seqlen_prefill is not None or cuda_graph is None or batch.speculative_ids is not None: + return self.model.forward( + input_ids=input_ids, + position_ids=position_ids, + cu_seqlen_prefill=cu_seqlen_prefill, + kv_cache=kv_cache, + block_tables=block_tables, + slots=slots, + input_lengths=input_lengths, + max_s=max_s, + lm_head_indices=lm_head_indices, + ) + + # Copy inputs to the static inputs of the cuda graph + # Static inputs are potentially padded + cuda_graph["input_ids"][: input_ids.shape[0]] = input_ids + cuda_graph["position_ids"][: position_ids.shape[0]] = position_ids + cuda_graph["block_tables"][ + : block_tables.shape[0], : block_tables.shape[1] + ] = block_tables + cuda_graph["slots"].fill_(-1) + cuda_graph["slots"][: slots.shape[0]] = slots + cuda_graph["input_lengths"].zero_() + cuda_graph["input_lengths"][: input_lengths.shape[0]] = input_lengths + + # Replay the graph + cuda_graph["graph"].replay() + + # Slice output to the correct shape + return cuda_graph["logits"][:bs] @tracer.start_as_current_span("generate_token") def generate_token( diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py index 8c6cb025..34a50194 100644 --- a/server/text_generation_server/models/flash_mistral.py +++ b/server/text_generation_server/models/flash_mistral.py @@ -35,6 +35,8 @@ SLIDING_WINDOW: Optional[int] = None SLIDING_WINDOW_BLOCKS: Optional[int] = None +MEM_POOL = torch.cuda.graph_pool_handle() + # Adds windowing logic to FlashCausalLMBatch @dataclass @@ -332,6 +334,8 @@ def __init__( model = model_cls(config, weights) + self.cuda_graphs = {} + torch.distributed.barrier(group=self.process_group) super(BaseFlashMistral, self).__init__( model=model, @@ -350,6 +354,60 @@ def __init__( def batch_type(self) -> Type[FlashMistralBatch]: return FlashMistralBatch + def cuda_graph_warmup(self, bs: int, max_s: int, max_bt: int): + input_ids = torch.zeros(bs, dtype=torch.int64, device=self.device) + position_ids = torch.zeros(bs, dtype=torch.int32, device=self.device) + slots = torch.arange(bs, dtype=torch.int32, device=self.device) + input_lengths = torch.ones(bs, dtype=torch.int32, device=self.device) * max_s + block_tables = ( + torch.arange(max_bt, dtype=torch.int32, device=self.device) + .repeat(bs) + .reshape((bs, max_bt)) + ) + kv_cache = get_cache_manager().kv_cache + + self.cuda_graphs[bs] = { + "input_ids": input_ids, + "position_ids": position_ids, + "kv_cache": kv_cache, + "block_tables": block_tables, + "slots": slots, + "input_lengths": input_lengths, + } + graph = torch.cuda.CUDAGraph() + self.cuda_graphs[bs]["graph"] = graph + + torch.cuda.synchronize() + # Run once outside to warmup + self.model.forward( + input_ids=input_ids, + position_ids=position_ids, + cu_seqlen_prefill=None, + kv_cache=kv_cache, + block_tables=block_tables, + slots=slots, + input_lengths=input_lengths, + max_s=max_s, + prefill_cache_indices=None, + lm_head_indices=None, + ) + torch.cuda.synchronize() + + with torch.cuda.graph(graph, pool=MEM_POOL): + self.cuda_graphs[bs]["logits"] = self.model.forward( + input_ids=input_ids, + position_ids=position_ids, + cu_seqlen_prefill=None, + kv_cache=kv_cache, + block_tables=block_tables, + slots=slots, + input_lengths=input_lengths, + max_s=max_s, + prefill_cache_indices=None, + lm_head_indices=None, + ) + torch.cuda.synchronize() + def forward(self, batch: FlashMistralBatch) -> Tuple[torch.Tensor, torch.Tensor]: # Model Forward if batch.speculative_ids is not None: @@ -401,21 +459,56 @@ def forward(self, batch: FlashMistralBatch) -> Tuple[torch.Tensor, torch.Tensor] input_lengths = batch.input_lengths_tensor max_s = batch.max_seqlen lm_head_indices = batch.prefill_head_indices - logits = self.model.forward( - input_ids=input_ids, - position_ids=position_ids, - cu_seqlen_prefill=cu_seqlen_prefill, - kv_cache=kv_cache, - block_tables=block_tables, - slots=slots, - input_lengths=input_lengths, - max_s=max_s, - prefill_cache_indices=batch.prefill_cache_indices, - lm_head_indices=lm_head_indices, - ) - if batch.prefill_cache_indices is not None: - batch.prefill_cache_indices = None - return logits + + if self.model.max_past is not None: + max_s = min(self.model.max_past, max_s) + + bs = input_ids.shape[0] + padded_bs = bs + if bs == 3: + padded_bs = 4 + elif 3 < bs <= 8: + padded_bs = 8 + elif bs > 8: + padded_bs = (bs + 7) // 8 * 8 + + # Try to find an associated cuda graph + cuda_graph = self.cuda_graphs.get(padded_bs, None) + + if cu_seqlen_prefill is not None or cuda_graph is None: + logits = self.model.forward( + input_ids=input_ids, + position_ids=position_ids, + cu_seqlen_prefill=cu_seqlen_prefill, + kv_cache=kv_cache, + block_tables=block_tables, + slots=slots, + input_lengths=input_lengths, + max_s=max_s, + prefill_cache_indices=batch.prefill_cache_indices, + lm_head_indices=lm_head_indices, + ) + if batch.prefill_cache_indices is not None: + batch.prefill_cache_indices = None + return logits + + # Copy inputs to the static inputs of the cuda graph + # Static inputs are potentially padded + cuda_graph["input_ids"][: input_ids.shape[0]] = input_ids + cuda_graph["position_ids"][: position_ids.shape[0]] = position_ids + cuda_graph["block_tables"][ + : block_tables.shape[0], : block_tables.shape[1] + ] = block_tables + cuda_graph["slots"].fill_(-1) + cuda_graph["slots"][: slots.shape[0]] = slots + cuda_graph["input_lengths"].zero_() + cuda_graph["input_lengths"][: input_lengths.shape[0]] = input_lengths + + # Replay the graph + cuda_graph["graph"].replay() + + # Slice output to the correct shape + return cuda_graph["logits"][:bs] class FlashMistral(BaseFlashMistral): diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 8f7e1f10..d0614346 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -407,8 +407,9 @@ def _set_gptq_params(self, model_id, revision): data = json.load(f) self.gptq_bits = data["quantization_config"]["bits"] self.gptq_groupsize = data["quantization_config"]["group_size"] - self.gptq_desc_act = data["quantization_config"]["desc_act"] + # Order is important here, desc_act is missing on some real models self.quant_method = data["quantization_config"]["quant_method"] + self.gptq_desc_act = data["quantization_config"]["desc_act"] except Exception: filename = "quantize_config.json" try: From 246ad39d041054545f2483c6adf29a6ab69cfc69 Mon Sep 17 00:00:00 2001 From: drbh Date: Tue, 13 Feb 2024 10:01:02 -0500 Subject: [PATCH 563/793] feat: add deserialize_with that handles strings or objects with content (#1550) This PR adds a simple custom `deserialize_with` function that parses a string or an object with a content property. This should help support more token configuration files stored on the hub --- router/src/lib.rs | 85 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/router/src/lib.rs b/router/src/lib.rs index 3ce9eca8..a9d783bb 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -32,7 +32,9 @@ pub struct HubModelInfo { #[derive(Clone, Deserialize, Default)] pub struct HubTokenizerConfig { pub chat_template: Option, + #[serde(deserialize_with = "token_serde::deserialize")] pub bos_token: Option, + #[serde(deserialize_with = "token_serde::deserialize")] pub eos_token: Option, } @@ -43,6 +45,34 @@ impl HubTokenizerConfig { } } +mod token_serde { + use super::*; + use serde::de; + use serde::Deserializer; + use serde_json::Value; + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + + match value { + Value::String(s) => Ok(Some(s)), + Value::Object(map) => { + if let Some(content) = map.get("content").and_then(|v| v.as_str()) { + Ok(Some(content.to_string())) + } else { + Err(de::Error::custom( + "content key not found in structured token", + )) + } + } + _ => Err(de::Error::custom("invalid token format")), + } + } +} + #[derive(Clone, Debug, Serialize, ToSchema)] pub struct Info { /// Model info @@ -638,6 +668,8 @@ pub(crate) struct ErrorResponse { #[cfg(test)] mod tests { + use super::*; + use tokenizers::Tokenizer; pub(crate) async fn get_tokenizer() -> Tokenizer { @@ -646,4 +678,57 @@ mod tests { let filename = repo.get("tokenizer.json").unwrap(); Tokenizer::from_file(filename).unwrap() } + + #[test] + fn test_hub_nested_tokens_tokenizer_config() { + // this is a subset of the tokenizer.json file + // in this case we expect the tokens to be encoded as simple strings + let json_content = r#"{ + "chat_template": "test", + "bos_token": "<｜begin▁of▁sentence｜>", + "eos_token": "<｜end▁of▁sentence｜>" + }"#; + + let config: HubTokenizerConfig = serde_json::from_str(json_content).unwrap(); + + // check that we successfully parsed the tokens + assert_eq!(config.chat_template, Some("test".to_string())); + assert_eq!( + config.bos_token, + Some("<｜begin▁of▁sentence｜>".to_string()) + ); + assert_eq!(config.eos_token, Some("<｜end▁of▁sentence｜>".to_string())); + + // in this case we expect the tokens to be encoded as structured tokens + // we want the content of the structured token + let json_content = r#"{ + "chat_template": "test", + "bos_token": { + "__type": "AddedToken", + "content": "<｜begin▁of▁sentence｜>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "__type": "AddedToken", + "content": "<｜end▁of▁sentence｜>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } + }"#; + + let config: HubTokenizerConfig = serde_json::from_str(json_content).unwrap(); + + // check that we successfully parsed the tokens + assert_eq!(config.chat_template, Some("test".to_string())); + assert_eq!( + config.bos_token, + Some("<｜begin▁of▁sentence｜>".to_string()) + ); + assert_eq!(config.eos_token, Some("<｜end▁of▁sentence｜>".to_string())); + } } From 6f68bb14c77d4845b032be1ffbf06f622c05f467 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 13 Feb 2024 17:43:47 +0100 Subject: [PATCH 564/793] Fixing glibc version in the runtime. (#1556) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- Dockerfile_amd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile_amd b/Dockerfile_amd index c2ec4a6d..9a5e3568 100644 --- a/Dockerfile_amd +++ b/Dockerfile_amd @@ -36,7 +36,7 @@ COPY launcher launcher RUN cargo build --release # Text Generation Inference base image for RoCm -FROM rocm/dev-ubuntu-20.04:5.7 as base +FROM rocm/dev-ubuntu-22.04:5.7 as base RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ From 7671a419a088ce0e832b88217149bf9ee5c448b0 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 13 Feb 2024 22:46:16 +0100 Subject: [PATCH 565/793] Upgrade intermediary layer for nvidia too. (#1557) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 252c5885..65e2a6f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN cargo build --release # Python builder # Adapted from: https://github.com/pytorch/pytorch/blob/master/Dockerfile -FROM nvidia/cuda:12.1.0-devel-ubuntu20.04 as pytorch-install +FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 as pytorch-install ARG PYTORCH_VERSION=2.1.1 ARG PYTHON_VERSION=3.10 From d6b0fb9e25f98fabb56e1865e8ba890aa27b4279 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 14 Feb 2024 09:54:10 +0100 Subject: [PATCH 566/793] Improving mamba runtime by using updates (#1552) - Move float16 to bfloat16, which has less imprecisions (load test are failing with the update kernels + f16, all working under bf16). Another note, is that we are not respecting the layer norm in f32 defined in the configuration (this is OK in my book, but that could impact the f16 precision) - Moved to update kernels. Triton overhead is super high, removed by switching to cuda graphs works great (update cuda graph is available in TRT-LLM if needed, seems *exactly* like the regular ssm kernel. - Moved inference_params struct in order to make only 2 tensors, to reduce the overhead of copying back and forth to the cuda graphs. - Left over overhead seems entirely in the tokenization bit. (Still 4 copies are paid before launching the graph) # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- .../__snapshots__/test_mamba/test_mamba.json | 20 +- .../test_mamba/test_mamba_all_params.json | 60 ++-- .../test_mamba/test_mamba_load.json | 112 ++++---- integration-tests/models/test_mamba.py | 6 +- .../models/custom_modeling/mamba_modeling.py | 95 +++---- .../models/flash_causal_lm.py | 2 +- .../text_generation_server/models/globals.py | 3 + server/text_generation_server/models/mamba.py | 258 ++++++++++-------- 8 files changed, 286 insertions(+), 270 deletions(-) create mode 100644 server/text_generation_server/models/globals.py diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba.json index 4435f215..eaba5078 100644 --- a/integration-tests/models/__snapshots__/test_mamba/test_mamba.json +++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba.json @@ -8,61 +8,61 @@ "tokens": [ { "id": 187, - "logprob": -0.3552246, + "logprob": -0.37890625, "special": false, "text": "\n" }, { "id": 187, - "logprob": -0.38378906, + "logprob": -0.26953125, "special": false, "text": "\n" }, { "id": 30763, - "logprob": -1.140625, + "logprob": -1.1953125, "special": false, "text": "Deep" }, { "id": 4715, - "logprob": -0.5551758, + "logprob": -0.53515625, "special": false, "text": " learning" }, { "id": 310, - "logprob": -0.59033203, + "logprob": -0.625, "special": false, "text": " is" }, { "id": 247, - "logprob": -0.70654297, + "logprob": -0.6796875, "special": false, "text": " a" }, { "id": 747, - "logprob": -2.0410156, + "logprob": -2.0, "special": false, "text": " new" }, { "id": 1511, - "logprob": -2.3789062, + "logprob": -2.3125, "special": false, "text": " type" }, { "id": 273, - "logprob": -0.0026435852, + "logprob": -0.0028533936, "special": false, "text": " of" }, { "id": 5145, - "logprob": -1.2841797, + "logprob": -1.265625, "special": false, "text": " machine" } diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json index 052c1c69..85e9a9e0 100644 --- a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json +++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json @@ -11,22 +11,22 @@ }, { "id": 13, - "logprob": -2.5234375, + "logprob": -2.734375, "text": "," }, { "id": 8862, - "logprob": -3.4433594, + "logprob": -3.6875, "text": " yellow" }, { "id": 13, - "logprob": -0.43017578, + "logprob": -0.40234375, "text": "," }, { "id": 209, - "logprob": -8.21875, + "logprob": -8.25, "text": " " } ], @@ -40,60 +40,60 @@ }, { "id": 395, - "logprob": -0.46411133, + "logprob": -0.3125, "special": false, "text": "and" }, { - "id": 13735, - "logprob": -2.1132812, + "id": 4797, + "logprob": 0.0, "special": false, - "text": " orange" + "text": " blue" }, { - "id": 313, - "logprob": -1.2128906, + "id": 9830, + "logprob": -1.65625, "special": false, - "text": " (" + "text": " colors" }, { - "id": 249, - "logprob": -2.3671875, + "id": 15, + "logprob": 0.0, "special": false, - "text": "in" + "text": "." }, { - "id": 253, - "logprob": 0.0, + "id": 329, + "logprob": -2.4375, "special": false, - "text": " the" + "text": " A" }, { - "id": 1340, - "logprob": -1.640625, + "id": 1180, + "logprob": -1.953125, "special": false, - "text": " order" + "text": " number" }, { - "id": 597, - "logprob": -0.5488281, + "id": 273, + "logprob": 0.0, "special": false, - "text": " they" + "text": " of" }, { - "id": 3176, - "logprob": -0.48608398, + "id": 1027, + "logprob": -1.5546875, "special": false, - "text": " appear" + "text": " different" }, { - "id": 275, - "logprob": 0.0, + "id": 3295, + "logprob": -0.97265625, "special": false, - "text": " in" + "text": " color" } ], "top_tokens": null }, - "generated_text": "blue, red, yellow, \nand orange (in the order they appear in" + "generated_text": "blue, red, yellow, \nand blue colors. A number of different color" } diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba_load.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba_load.json index 014210b2..4921c14b 100644 --- a/integration-tests/models/__snapshots__/test_mamba/test_mamba_load.json +++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba_load.json @@ -12,22 +12,22 @@ }, { "id": 310, - "logprob": -0.8125, + "logprob": -0.83984375, "text": " is" }, { "id": 18147, - "logprob": -12.828125, + "logprob": -12.8125, "text": " Deep" }, { "id": 20727, - "logprob": -3.0, + "logprob": -2.84375, "text": " Learning" }, { "id": 32, - "logprob": -1.1484375, + "logprob": -1.25, "text": "?" } ], @@ -35,61 +35,61 @@ "tokens": [ { "id": 187, - "logprob": -0.3552246, + "logprob": -0.37890625, "special": false, "text": "\n" }, { "id": 187, - "logprob": -0.38378906, + "logprob": -0.4296875, "special": false, "text": "\n" }, { "id": 30763, - "logprob": -1.1279297, + "logprob": -1.078125, "special": false, "text": "Deep" }, { "id": 4715, - "logprob": -0.5595703, + "logprob": -0.515625, "special": false, "text": " learning" }, { "id": 310, - "logprob": -0.60253906, + "logprob": -0.6015625, "special": false, "text": " is" }, { "id": 247, - "logprob": -0.7050781, + "logprob": -0.65625, "special": false, "text": " a" }, { "id": 747, - "logprob": -2.0488281, + "logprob": -2.109375, "special": false, "text": " new" }, { "id": 1511, - "logprob": -2.3808594, + "logprob": -2.328125, "special": false, "text": " type" }, { "id": 273, - "logprob": -0.0026416779, + "logprob": -0.0032653809, "special": false, "text": " of" }, { "id": 5145, - "logprob": -1.2851562, + "logprob": -1.28125, "special": false, "text": " machine" } @@ -111,22 +111,22 @@ }, { "id": 310, - "logprob": -0.78027344, + "logprob": -0.80078125, "text": " is" }, { "id": 18147, - "logprob": -12.8203125, + "logprob": -13.25, "text": " Deep" }, { "id": 20727, - "logprob": -2.9902344, + "logprob": -2.828125, "text": " Learning" }, { "id": 32, - "logprob": -1.1523438, + "logprob": -1.1953125, "text": "?" } ], @@ -134,61 +134,61 @@ "tokens": [ { "id": 187, - "logprob": -0.35351562, + "logprob": -0.296875, "special": false, "text": "\n" }, { "id": 187, - "logprob": -0.38256836, + "logprob": -0.3359375, "special": false, "text": "\n" }, { "id": 30763, - "logprob": -1.1269531, + "logprob": -1.2578125, "special": false, "text": "Deep" }, { "id": 4715, - "logprob": -0.54541016, + "logprob": -0.5546875, "special": false, "text": " learning" }, { "id": 310, - "logprob": -0.59765625, + "logprob": -0.62890625, "special": false, "text": " is" }, { "id": 247, - "logprob": -0.7001953, + "logprob": -0.64453125, "special": false, "text": " a" }, { "id": 747, - "logprob": -2.0585938, + "logprob": -2.078125, "special": false, "text": " new" }, { "id": 1511, - "logprob": -2.3789062, + "logprob": -2.28125, "special": false, "text": " type" }, { "id": 273, - "logprob": -0.0027446747, + "logprob": -0.0030670166, "special": false, "text": " of" }, { "id": 5145, - "logprob": -1.2851562, + "logprob": -1.3125, "special": false, "text": " machine" } @@ -210,22 +210,22 @@ }, { "id": 310, - "logprob": -0.78027344, + "logprob": -0.80078125, "text": " is" }, { "id": 18147, - "logprob": -12.8203125, + "logprob": -13.25, "text": " Deep" }, { "id": 20727, - "logprob": -2.9902344, + "logprob": -2.828125, "text": " Learning" }, { "id": 32, - "logprob": -1.1523438, + "logprob": -1.1953125, "text": "?" } ], @@ -233,61 +233,61 @@ "tokens": [ { "id": 187, - "logprob": -0.35351562, + "logprob": -0.296875, "special": false, "text": "\n" }, { "id": 187, - "logprob": -0.38256836, + "logprob": -0.3359375, "special": false, "text": "\n" }, { "id": 30763, - "logprob": -1.1269531, + "logprob": -1.2578125, "special": false, "text": "Deep" }, { "id": 4715, - "logprob": -0.54541016, + "logprob": -0.5546875, "special": false, "text": " learning" }, { "id": 310, - "logprob": -0.59765625, + "logprob": -0.62890625, "special": false, "text": " is" }, { "id": 247, - "logprob": -0.7001953, + "logprob": -0.64453125, "special": false, "text": " a" }, { "id": 747, - "logprob": -2.0585938, + "logprob": -2.078125, "special": false, "text": " new" }, { "id": 1511, - "logprob": -2.3789062, + "logprob": -2.28125, "special": false, "text": " type" }, { "id": 273, - "logprob": -0.0027446747, + "logprob": -0.0030670166, "special": false, "text": " of" }, { "id": 5145, - "logprob": -1.2851562, + "logprob": -1.3125, "special": false, "text": " machine" } @@ -309,22 +309,22 @@ }, { "id": 310, - "logprob": -0.78027344, + "logprob": -0.80078125, "text": " is" }, { "id": 18147, - "logprob": -12.8203125, + "logprob": -13.25, "text": " Deep" }, { "id": 20727, - "logprob": -2.9902344, + "logprob": -2.828125, "text": " Learning" }, { "id": 32, - "logprob": -1.1523438, + "logprob": -1.1953125, "text": "?" } ], @@ -332,61 +332,61 @@ "tokens": [ { "id": 187, - "logprob": -0.35351562, + "logprob": -0.296875, "special": false, "text": "\n" }, { "id": 187, - "logprob": -0.38256836, + "logprob": -0.3359375, "special": false, "text": "\n" }, { "id": 30763, - "logprob": -1.1269531, + "logprob": -1.2578125, "special": false, "text": "Deep" }, { "id": 4715, - "logprob": -0.54541016, + "logprob": -0.5546875, "special": false, "text": " learning" }, { "id": 310, - "logprob": -0.59765625, + "logprob": -0.62890625, "special": false, "text": " is" }, { "id": 247, - "logprob": -0.7001953, + "logprob": -0.64453125, "special": false, "text": " a" }, { "id": 747, - "logprob": -2.0585938, + "logprob": -2.078125, "special": false, "text": " new" }, { "id": 1511, - "logprob": -2.3789062, + "logprob": -2.28125, "special": false, "text": " type" }, { "id": 273, - "logprob": -0.0027446747, + "logprob": -0.0030670166, "special": false, "text": " of" }, { "id": 5145, - "logprob": -1.2851562, + "logprob": -1.3125, "special": false, "text": " machine" } diff --git a/integration-tests/models/test_mamba.py b/integration-tests/models/test_mamba.py index bf398999..4cc863f0 100644 --- a/integration-tests/models/test_mamba.py +++ b/integration-tests/models/test_mamba.py @@ -47,14 +47,14 @@ async def test_mamba_all_params(fused_kernel_mamba, response_snapshot): assert response.details.generated_tokens == 10 assert ( response.generated_text - == "blue, red, yellow, \nand orange (in the order they appear in" + == "blue, red, yellow, \nand blue colors. A number of different color" ) assert response == response_snapshot @pytest.mark.asyncio @pytest.mark.private -async def test_mamba_load(fused_kernel_mamba, generate_load, response_snapshot): +async def test_mamba_load(fused_kernel_mamba, generate_load, generous_response_snapshot): responses = await generate_load( fused_kernel_mamba, "What is Deep Learning?", max_new_tokens=10, n=4 ) @@ -63,4 +63,4 @@ async def test_mamba_load(fused_kernel_mamba, generate_load, response_snapshot): assert all([r.generated_text == responses[0].generated_text for r in responses]) assert responses[0].generated_text == "\n\nDeep learning is a new type of machine" - assert responses == response_snapshot + assert responses == generous_response_snapshot diff --git a/server/text_generation_server/models/custom_modeling/mamba_modeling.py b/server/text_generation_server/models/custom_modeling/mamba_modeling.py index 017c0341..53e939bb 100644 --- a/server/text_generation_server/models/custom_modeling/mamba_modeling.py +++ b/server/text_generation_server/models/custom_modeling/mamba_modeling.py @@ -3,7 +3,6 @@ from mamba_ssm.ops.triton.selective_state_update import selective_state_update from mamba_ssm.ops.selective_scan_interface import selective_scan_fn -from mamba_ssm.utils.generation import InferenceParams from torch import nn from typing import Optional, Tuple, Any from transformers.configuration_utils import PretrainedConfig @@ -18,6 +17,17 @@ from einops import rearrange from causal_conv1d import causal_conv1d_fn, causal_conv1d_update import math +from dataclasses import dataclass + +@dataclass +class InferenceParams: + """Inference parameters that are passed to the main model in order + to efficienly calculate and store the context during inference.""" + max_seqlen: int + max_batch_size: int + conv_states: torch.Tensor + ssm_states: torch.Tensor + seqlen_offset: int class MambaConfig(PretrainedConfig): @@ -56,9 +66,9 @@ def __init__( class MambaBlock(nn.Module): - def __init__(self, prefix, config, weights): + def __init__(self, prefix, config, weights, layer_id): super().__init__() - self.layer_idx = int(prefix.split(".")[2]) + self.layer_id = layer_id self.in_proj = FastLinear.load(config, f"{prefix}.in_proj", weights, bias=False) self.x_proj = FastLinear.load(config, f"{prefix}.x_proj", weights, bias=False) self.dt_proj = FastLinear.load(config, f"{prefix}.dt_proj", weights, bias=True) @@ -79,21 +89,20 @@ def __init__(self, prefix, config, weights): # inference_params def forward(self, hidden_states: torch.Tensor, inference_params=None): - _, seqlen, _ = hidden_states.shape - conv_state, ssm_state = inference_params.key_value_memory_dict[self.layer_idx] - if inference_params.seqlen_offset > 0: + conv_state = inference_params.conv_states[self.layer_id] + ssm_state = inference_params.ssm_states[self.layer_id] out, conv_state, ssm_state = self.step(hidden_states, conv_state, ssm_state) return out, conv_state, ssm_state + _, seqlen, _ = hidden_states.shape projected_states = self.in_proj(hidden_states).transpose(1, 2) + # assert projected_states.shape == [batch_size, 2 * dstate, seqlen], f"{projected_states.shape} [{batch_size}, {dstate}, {seqlen}]" x, z = projected_states.chunk(2, dim=1) conv_state = F.pad(x, (self.d_conv - seqlen, 0)) x = causal_conv1d_fn( x=x, - weight=self.conv1d.weight.view( - self.conv1d.weight.size(0), self.conv1d.weight.size(2) - ), + weight=self.conv1d.weight.squeeze(1), bias=self.conv1d.bias, activation=self.activation, ) @@ -126,56 +135,28 @@ def forward(self, hidden_states: torch.Tensor, inference_params=None): return attn_outputs, conv_state, last_state def step(self, hidden_states, conv_state, ssm_state): - _xz = self.in_proj(hidden_states) - _x, _z = _xz.chunk(2, dim=-1) # (B D) - conv_state_new = torch.cat([conv_state, _x.transpose(1, 2)], dim=-1) - conv_out = causal_conv1d_fn( - x=conv_state_new, - weight=self.conv1d.weight.view( - self.conv1d.weight.size(0), self.conv1d.weight.size(2) - ), - bias=self.conv1d.bias, - activation=self.activation, + xz = self.in_proj(hidden_states.squeeze(1)) + x, z = xz.chunk(2, dim=-1) # (B D) + x = causal_conv1d_update(x, conv_state, self.conv1d.weight.squeeze(1), self.conv1d.bias, self.activation) + x_db = self.x_proj(x) # (B dt_rank+2*d_state) + dt, B, C = torch.split(x_db, [self.dt_rank, self.d_state, self.d_state], dim=-1) + dt = F.linear(dt, self.dt_proj.weight) + A = self.negA + y = selective_state_update( + ssm_state, x, dt, A, B, C, self.D, z=z, dt_bias=self.dt_proj.bias, dt_softplus=True ) - conv_state = conv_state_new[:, :, 1:] - bsz, seqlen, dim = hidden_states.shape - output_tensor = torch.zeros( - (bsz, seqlen, dim), device=hidden_states.device, dtype=hidden_states.dtype - ) - for i in range(0, bsz): - x = conv_out[i : i + 1, :, -1] - z = _z[i : i + 1, -1, :] - x_db = self.x_proj(x) - dt, B, C = torch.split( - x_db, [self.dt_rank, self.d_state, self.d_state], dim=-1 - ) - dt = F.linear(dt, self.dt_proj.weight) - y = selective_state_update( - ssm_state[i : i + 1, :, :], - x, - dt, - self.negA, - B, - C, - self.D, - z=z, - dt_bias=self.dt_proj.bias, - dt_softplus=True, - ) - out = self.out_proj(y) - output_tensor[i] = out - - return output_tensor, conv_state, ssm_state + out = self.out_proj(y) + return out.unsqueeze(1), conv_state.clone(), ssm_state.clone() class ResidualBlock(nn.Module): - def __init__(self, layer_id, config, weights): + def __init__(self, prefix, config, weights, layer_id): super().__init__() self.mamba_block = MambaBlock( - prefix=f"{layer_id}.mixer", config=config, weights=weights + prefix=f"{prefix}.mixer", config=config, weights=weights, layer_id=layer_id ) self.layer_norm = FastRMSNorm.load( - prefix=f"{layer_id}.norm", weights=weights, eps=config.layer_norm_epsilon + prefix=f"{prefix}.norm", weights=weights, eps=config.layer_norm_epsilon ) def forward( @@ -200,7 +181,7 @@ def __init__(self, config, weights): self.embed_tokens = TensorParallelEmbedding(f"{prefix}.embedding", weights) self.blocks = nn.ModuleList( [ - ResidualBlock(f"{prefix}.layers.{i}", config, weights) + ResidualBlock(f"{prefix}.layers.{i}", config, weights, layer_id=i) for i in range(config.n_layer) ] ) @@ -216,14 +197,12 @@ def forward( self, input_ids: torch.Tensor, inference_params=None, residual=None ) -> Tuple[torch.Tensor, torch.Tensor, InferenceParams]: hidden_states = self.embed_tokens(input_ids) - for block in self.blocks: + for i, block in enumerate(self.blocks): hidden_states, residual, conv_state, ssm_state = block( hidden_states, residual, inference_params ) - inference_params.key_value_memory_dict[block.mamba_block.layer_idx] = ( - conv_state, - ssm_state, - ) + inference_params.conv_states[i].copy_(conv_state) + inference_params.ssm_states[i].copy_(ssm_state) hidden_states = ( hidden_states + residual if residual is not None else hidden_states @@ -234,4 +213,4 @@ def forward( # update the offset for the next inference using these params inference_params.seqlen_offset += input_ids.size(1) - return logits, input_ids, inference_params + return logits diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index c7fda516..e04a9719 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -28,12 +28,12 @@ BLOCK_SIZE, ) from text_generation_server.pb import generate_pb2 +from text_generation_server.models.globals import MEM_POOL from text_generation_server.utils import StoppingCriteria, HeterogeneousNextTokenChooser from text_generation_server.utils.dist import MEMORY_FRACTION tracer = trace.get_tracer(__name__) -MEM_POOL = torch.cuda.graph_pool_handle() @dataclass diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py new file mode 100644 index 00000000..b0dca376 --- /dev/null +++ b/server/text_generation_server/models/globals.py @@ -0,0 +1,3 @@ +import torch + +MEM_POOL = torch.cuda.graph_pool_handle() diff --git a/server/text_generation_server/models/mamba.py b/server/text_generation_server/models/mamba.py index c51e1e20..8f18e475 100644 --- a/server/text_generation_server/models/mamba.py +++ b/server/text_generation_server/models/mamba.py @@ -2,17 +2,20 @@ import torch.distributed from transformers import AutoTokenizer, PreTrainedTokenizerBase from typing import Optional +import os from text_generation_server.models.custom_modeling.mamba_modeling import ( MambaConfig, ) +from loguru import logger from text_generation_server.pb import generate_pb2 from text_generation_server.utils import ( initialize_torch_distributed, weight_files, Weights, ) +from text_generation_server.models.globals import MEM_POOL import time -from text_generation_server.models.custom_modeling.mamba_modeling import MambaModel +from text_generation_server.models.custom_modeling.mamba_modeling import MambaModel, InferenceParams from text_generation_server.models import Model from typing import Any, List, Optional, Tuple, Type, Dict from text_generation_server.models.types import ( @@ -24,7 +27,34 @@ from text_generation_server.utils.tokens import batch_top_tokens, Sampling from dataclasses import dataclass from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling -from mamba_ssm.utils.generation import InferenceParams + +def new_inference_params(n_blocks: int, batch_size: int, d_inner: int, d_conv: int, d_state: int, seqlen_offset: int, dtype: torch.dtype, device: torch.device): + max_seqlen = 0 + conv_states = torch.zeros( + (n_blocks, + batch_size, + d_inner, + d_conv,), + device=device, + dtype=dtype, + ) + ssm_states = torch.zeros( + (n_blocks, + batch_size, + d_inner, + d_state,), + device=device, + dtype=dtype, + ) + inference_params = InferenceParams( + max_seqlen=max_seqlen, + max_batch_size=batch_size, + seqlen_offset=seqlen_offset, + conv_states=conv_states, + ssm_states=ssm_states, + + ) + return inference_params @dataclass @@ -221,14 +251,8 @@ def filter(self, request_ids: List[int]) -> Optional["MambaBatch"]: # TODO # Kept it simple by just updating the state, maybe updating the other CPU values is necessary. - key_value_memory_dict = {} - for i, ( - conv_state, - ssm_state, - ) in self.inference_params.key_value_memory_dict.items(): - key_value_memory_dict[i] = (conv_state[indices], ssm_state[indices]) - self.inference_params.key_value_memory_dict = key_value_memory_dict - + self.inference_params.conv_states = self.inference_params.conv_states[:, indices] + self.inference_params.ssm_states = self.inference_params.ssm_states[:, indices] return self @classmethod @@ -254,9 +278,16 @@ def concatenate(cls, batches: List["MambaBatch"]) -> "MambaBatch": top_n_tokens = [] max_tokens = 0 max_seqlen = 0 - batch_size = 0 seqlen_offset = 0 + (n_blocks, _, d_inner, d_conv) = ( + batches[0].inference_params.conv_states.shape + ) + (_, _, _, d_state) = batches[0].inference_params.ssm_states.shape + dtype = batches[0].inference_params.conv_states.dtype + device = batches[0].inference_params.conv_states.device + inference_params = new_inference_params(n_blocks=n_blocks, batch_size=total_batch_size, d_state=d_state, d_conv=d_conv, d_inner=d_inner, seqlen_offset=seqlen_offset, device=device, dtype=dtype) + # Batch tensors input_ids = None top_n_tokens_tensor = None @@ -303,62 +334,15 @@ def concatenate(cls, batches: List["MambaBatch"]) -> "MambaBatch": max_input_length - batch.max_input_length ) * len(batch) - max_seqlen = max(max_seqlen, batch.inference_params.max_seqlen) - seqlen_offset = max(seqlen_offset, batch.inference_params.seqlen_offset) - batch_size += batch.inference_params.max_batch_size + inference_params.max_seqlen = max(inference_params.max_seqlen, batch.inference_params.max_seqlen) + assert batch.inference_params.seqlen_offset != 0, "Invalid seqlen offset" + inference_params.seqlen_offset = max(inference_params.seqlen_offset, batch.inference_params.seqlen_offset) - start_index = end_index - (_, d_model, d_conv) = ( - batches[0].inference_params.key_value_memory_dict[0][0].shape - ) - (_, _, d_state) = batches[0].inference_params.key_value_memory_dict[0][1].shape - n_blocks = len(batches[0].inference_params.key_value_memory_dict) - dtype = batches[0].inference_params.key_value_memory_dict[0][0].dtype - device = batches[0].inference_params.key_value_memory_dict[0][0].device - - key_value_memory_dict = {} - for i in range(n_blocks): - conv_state = torch.zeros( - batch_size, - d_model, - d_conv, - device=device, - dtype=dtype, - ) - ssm_state = torch.zeros( - batch_size, - d_model, - d_state, - device=device, - dtype=dtype, - ) - key_value_memory_dict[i] = (conv_state, ssm_state) - lengths_per_sample = torch.zeros(batch_size, dtype=torch.int32, device=device) - - inference_params = InferenceParams( - max_seqlen=max_seqlen, - max_batch_size=batch_size, - seqlen_offset=seqlen_offset, - key_value_memory_dict=key_value_memory_dict, - lengths_per_sample=lengths_per_sample, - ) + inference_params.conv_states[:, start_index:end_index] = batch.inference_params.conv_states + inference_params.ssm_states[:, start_index:end_index] = batch.inference_params.ssm_states - current_batch = 0 - for batch in batches: - for i in range(n_blocks): - conv_state, ssm_state = batch.inference_params.key_value_memory_dict[i] - batch_size = batch.inference_params.max_batch_size - inference_params.key_value_memory_dict[i][0][ - current_batch : current_batch + batch_size - ] = conv_state - inference_params.key_value_memory_dict[i][1][ - current_batch : current_batch + batch_size - ] = ssm_state - inference_params.lengths_per_sample[ - current_batch : current_batch + batch_size - ] = batch.inference_params.lengths_per_sample - current_batch += batch_size + start_index = end_index return cls( batch_id=batches[0].batch_id, @@ -394,9 +378,13 @@ def __init__( trust_remote_code: bool = False, ): self.process_group, _rank, _world_size = initialize_torch_distributed() + self.cuda_graphs = {} if torch.cuda.is_available(): device = torch.device("cuda") - dtype = torch.float16 if dtype is None else dtype + # Bf16 is important. In f16 accumulations in the matmul are causing + # differences while the server is under load. + # This is detectable by the integration load test + dtype = torch.bfloat16 if dtype is None else dtype else: if quantize: raise ValueError("quantization is not available on CPU") @@ -439,17 +427,93 @@ def batch_type(self) -> Type[MambaBatch]: def warmup(self, batch) -> Optional[int]: # TODO: implement warmup for Mamba if needed + if os.getenv("ENABLE_CUDA_GRAPHS", "False") == "True": + if self.speculate is None or self.speculate == 0: + try: + logger.info("Experimental support for Cuda Graphs is enabled") + # Warmup cuda graphs + for bs in [1, 2, 4] + [8 * i for i in range(1, 9)]: + self.cuda_graph_warmup(bs) + except Exception: + logger.exception(f"Decode cuda graph warmup failed") + return None + def cuda_graph_warmup(self, batch_size: int): + input_ids = torch.zeros((batch_size, 1), dtype=torch.int64, device=self.device) + n_blocks = len(self.model.blocks) + + d_state = self.model.config.d_state + d_conv = self.model.config.d_conv + # Inner takes the expand multiplication + d_inner = self.model.config.d_inner + + # Important seqlen_offset to go through the update mecanism with the state + seqlen_offset = 1 + inference_params = new_inference_params(n_blocks=n_blocks, batch_size=batch_size, d_state=d_state, d_conv=d_conv, d_inner=d_inner, seqlen_offset=seqlen_offset, device=self.device, dtype=self.dtype) + + graph = torch.cuda.CUDAGraph() + + torch.cuda.synchronize() + # Run once outside to warmup + self.model.forward( + input_ids=input_ids, + inference_params=inference_params + ) + torch.cuda.synchronize() + + with torch.cuda.graph(graph, pool=MEM_POOL): + logits = self.model.forward( + input_ids=input_ids, + inference_params=inference_params + ) + torch.cuda.synchronize() + graph_dict = { + "input_ids": input_ids, + "inference_params": inference_params, + "graph": graph, + "logits": logits + } + self.cuda_graphs[batch_size] = graph_dict + def forward( self, input_ids: torch.Tensor, - past: Optional[List[torch.Tensor]] = None, + inference_params: Any ) -> Tuple[torch.Tensor, torch.Tensor]: - return self.model( - input_ids, - past=past, - ) + bs = input_ids.shape[0] + padded_bs = bs + if bs == 3: + padded_bs = 4 + elif 3 < bs <= 8: + padded_bs = 8 + elif bs > 8: + padded_bs = (bs + 7) // 8 * 8 + + # Try to find an associated cuda graph + cuda_graph = self.cuda_graphs.get(padded_bs, None) + is_prefill = inference_params is None or inference_params.seqlen_offset == 0 + + if is_prefill or cuda_graph is None: + return self.model( + input_ids, + inference_params=inference_params, + ) + + # Copy inputs to the static inputs of the cuda graph + # Static inputs are potentially padded + cuda_graph["input_ids"][: bs] = input_ids + cuda_graph["inference_params"].conv_states[:, : bs] = inference_params.conv_states + cuda_graph["inference_params"].ssm_states[:, : bs] = inference_params.ssm_states + + # Replay the graph + cuda_graph["graph"].replay() + + inference_params.conv_states.copy_(cuda_graph["inference_params"].conv_states[:, :bs]) + inference_params.ssm_states.copy_(cuda_graph["inference_params"].ssm_states[:, :bs]) + + # Slice output to the correct shape + return cuda_graph["logits"][:bs] def generate_token(self, batch) -> Tuple[List[Any], Optional[Any], Tuple[int, int]]: start = time.time_ns() @@ -457,56 +521,26 @@ def generate_token(self, batch) -> Tuple[List[Any], Optional[Any], Tuple[int, in batch.input_ids ) # batch.past_input_ids if batch.past_input_ids is not None else batch.input_ids - batch_size = input_ids.shape[0] - max_seqlen = input_ids.shape[1] - dtype = input_ids.dtype - + batch_size, max_seqlen = input_ids.shape # Inference params - seqlen_og = 0 - inf_cache = {} - lengths_per_sample = ( - torch.ones(batch_size, dtype=torch.int32, device=input_ids.device) - * max_seqlen - ) if batch.inference_params is None: - inference_params = InferenceParams( - max_seqlen=max_seqlen, - max_batch_size=batch_size, - seqlen_offset=seqlen_og, - key_value_memory_dict=inf_cache, - lengths_per_sample=lengths_per_sample, - ) - - # Allocate inference cache - for res_block in self.model.blocks: - block = res_block.mamba_block - conv_state = torch.zeros( - batch_size, - self.model.config.d_model * self.model.config.expand, - self.model.config.d_conv, - device=block.conv1d.weight.device, - dtype=block.conv1d.weight.dtype, - ) - ssm_state = torch.zeros( - batch_size, - self.model.config.d_model * self.model.config.expand, - self.model.config.d_state, - device=block.dt_proj.weight.device, - dtype=block.dt_proj.weight.dtype, - ) - inference_params.key_value_memory_dict[block.layer_idx] = ( - conv_state, - ssm_state, - ) + # 0 is important here + seqlen_offset = 0 + n_blocks = len(self.model.blocks) + d_state = self.model.config.d_state + d_conv = self.model.config.d_conv + d_inner = self.model.config.d_inner + inference_params = new_inference_params(n_blocks=n_blocks, batch_size=batch_size, d_state=d_state, d_conv=d_conv, d_inner=d_inner, seqlen_offset=seqlen_offset, device=self.device, dtype=self.dtype) batch.inference_params = inference_params # Forward pass - logits, past_input_ids, new_inference_params = self.model( - input_ids, batch.inference_params + logits = self.forward( + input_ids, inference_params=batch.inference_params ) - batch.inference_params = new_inference_params + + # batch.inference_params = new_inference_params # Results generations: List[Generation] = [] stopped = True From 4c2848b24bf67076fffc17fc98a260ee6a32fbf0 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 14 Feb 2024 15:30:07 +0100 Subject: [PATCH 567/793] Small cleanup. (#1560) Using a single `os.getenv` statement instead of multiple. Should make truthful values easier to catch In the end didn't move towards full CLI because modifying globals in Python is error prone (depends on code import order). Added an error when mamba is launched with TP. # What does this PR do? Fixes # (issue) ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? Anyone in the community is free to review the PR once the tests have passed. Feel free to tag members/contributors who may be interested in your PR. --- server/text_generation_server/models/flash_causal_lm.py | 4 ++-- server/text_generation_server/models/globals.py | 3 +++ server/text_generation_server/models/mamba.py | 8 +++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index e04a9719..886fe486 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -28,7 +28,7 @@ BLOCK_SIZE, ) from text_generation_server.pb import generate_pb2 -from text_generation_server.models.globals import MEM_POOL +from text_generation_server.models.globals import MEM_POOL, ENABLE_CUDA_GRAPHS from text_generation_server.utils import StoppingCriteria, HeterogeneousNextTokenChooser from text_generation_server.utils.dist import MEMORY_FRACTION @@ -793,7 +793,7 @@ def warmup(self, batch: FlashCausalLMBatch): self.device, ) - if os.getenv("ENABLE_CUDA_GRAPHS", "False") == "True": + if ENABLE_CUDA_GRAPHS: try: logger.info("Experimental support for Cuda Graphs is enabled") # Warmup cuda graphs diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py index b0dca376..3b8a70bc 100644 --- a/server/text_generation_server/models/globals.py +++ b/server/text_generation_server/models/globals.py @@ -1,3 +1,6 @@ import torch +import os MEM_POOL = torch.cuda.graph_pool_handle() +# This is overridden by the cli +ENABLE_CUDA_GRAPHS = os.getenv("ENABLE_CUDA_GRAPHS", "false").lower() in {"1", "true"} diff --git a/server/text_generation_server/models/mamba.py b/server/text_generation_server/models/mamba.py index 8f18e475..868db6aa 100644 --- a/server/text_generation_server/models/mamba.py +++ b/server/text_generation_server/models/mamba.py @@ -13,7 +13,7 @@ weight_files, Weights, ) -from text_generation_server.models.globals import MEM_POOL +from text_generation_server.models.globals import ENABLE_CUDA_GRAPHS, MEM_POOL import time from text_generation_server.models.custom_modeling.mamba_modeling import MambaModel, InferenceParams from text_generation_server.models import Model @@ -377,7 +377,9 @@ def __init__( dtype: Optional[torch.dtype] = None, trust_remote_code: bool = False, ): - self.process_group, _rank, _world_size = initialize_torch_distributed() + self.process_group, _rank, world_size = initialize_torch_distributed() + if world_size > 1: + raise RuntimeError("Mamba does not support Tensor Parallelism (TP)") self.cuda_graphs = {} if torch.cuda.is_available(): device = torch.device("cuda") @@ -427,7 +429,7 @@ def batch_type(self) -> Type[MambaBatch]: def warmup(self, batch) -> Optional[int]: # TODO: implement warmup for Mamba if needed - if os.getenv("ENABLE_CUDA_GRAPHS", "False") == "True": + if ENABLE_CUDA_GRAPHS: if self.speculate is None or self.speculate == 0: try: logger.info("Experimental support for Cuda Graphs is enabled") From cef0553d59713a5e5842b9a1d79334d4ffc066b9 Mon Sep 17 00:00:00 2001 From: drbh Date: Thu, 15 Feb 2024 04:28:10 -0500 Subject: [PATCH 568/793] Outlines guided generation (#1539) This WIP PR starts to add grammar support via outlines, currently this PR supports very simple regex grammars and does not optimize for precompiling or caching grammar fsm's. todo: - [X] add simple outlines guidance to `NextTokenChooser` - [X] update protos for grammar - [X] update generation params API - [X] constrain simple grammar - [ ] support parsing more complex grammar into fsm - [ ] support all outline support grammar types - [ ] explore optimizations to avoid recompiling grammars guided request ```bash curl -s 'http://localhost:3000/generate' \ --header 'Content-Type: application/json' \ --data-raw '{ "inputs": "make an email for david: \n", "parameters": { "max_new_tokens": 6, "grammar": "[\\w-]+@([\\w-]+\\.)+[\\w-]+" } }' | jq ``` response ```json { "generated_text": "david@example.com" } ``` unguided request ```bash curl -s 'http://localhost:3000/generate' \ --header 'Content-Type: application/json' \ --data '{ "inputs": "make an email for david: \n", "parameters": { "max_new_tokens": 6 } }' | jq ``` response ```json { "generated_text": " email = 'david" } ``` --- benchmark/src/lib.rs | 4 +- clients/python/text_generation/client.py | 10 + clients/python/text_generation/types.py | 28 +- docs/source/basic_tutorials/launcher.md | 8 + integration-tests/conftest.py | 30 +- .../test_flash_llama_grammar.json | 89 ++++ .../test_flash_llama_grammar_json.json | 274 ++++++++++ .../test_flash_llama_grammar_load.json | 478 ++++++++++++++++++ .../test_flash_llama_grammar_regex.json | 109 ++++ ...sh_llama_grammar_single_load_instance.json | 73 +++ .../models/test_grammar_llama.py | 151 ++++++ launcher/src/main.rs | 10 + proto/generate.proto | 10 + router/client/src/client.rs | 2 + router/client/src/lib.rs | 4 +- router/src/health.rs | 3 + router/src/lib.rs | 40 ++ router/src/main.rs | 4 + router/src/queue.rs | 8 +- router/src/server.rs | 3 + router/src/validation.rs | 46 +- server/pyproject.toml | 1 + .../models/causal_lm.py | 35 +- .../models/flash_causal_lm.py | 24 +- .../models/flash_mistral.py | 2 +- .../models/galactica.py | 2 +- .../models/idefics_causal_lm.py | 5 +- server/text_generation_server/models/mamba.py | 5 +- .../models/seq2seq_lm.py | 5 +- .../utils/logits_process.py | 142 +++++- server/text_generation_server/utils/tokens.py | 108 +++- 31 files changed, 1660 insertions(+), 53 deletions(-) create mode 100644 integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar.json create mode 100644 integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_json.json create mode 100644 integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_load.json create mode 100644 integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_regex.json create mode 100644 integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_single_load_instance.json create mode 100644 integration-tests/models/test_grammar_llama.py diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs index 6deae48d..638c6514 100644 --- a/benchmark/src/lib.rs +++ b/benchmark/src/lib.rs @@ -8,7 +8,7 @@ use crate::app::App; use crate::event::Event; use crossterm::ExecutableCommand; use std::io; -use text_generation_client::{NextTokenChooserParameters, ShardedClient}; +use text_generation_client::{GrammarType, NextTokenChooserParameters, ShardedClient}; use tokenizers::Tokenizer; use tokio::sync::{broadcast, mpsc}; use tui::backend::CrosstermBackend; @@ -45,6 +45,8 @@ pub async fn run( repetition_penalty: repetition_penalty.unwrap_or(1.0), frequency_penalty: frequency_penalty.unwrap_or(0.0), watermark, + grammar: String::new(), + grammar_type: GrammarType::None as i32, }; // Initialize terminal properties diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py index 0bf80f8c..bbccbf1d 100644 --- a/clients/python/text_generation/client.py +++ b/clients/python/text_generation/client.py @@ -10,6 +10,7 @@ Response, Request, Parameters, + Grammar, ) from text_generation.errors import parse_error @@ -76,6 +77,7 @@ def generate( watermark: bool = False, decoder_input_details: bool = False, top_n_tokens: Optional[int] = None, + grammar: Optional[Grammar] = None, ) -> Response: """ Given a prompt, generate the following text @@ -138,6 +140,7 @@ def generate( watermark=watermark, decoder_input_details=decoder_input_details, top_n_tokens=top_n_tokens, + grammar=grammar, ) request = Request(inputs=prompt, stream=False, parameters=parameters) @@ -169,6 +172,7 @@ def generate_stream( typical_p: Optional[float] = None, watermark: bool = False, top_n_tokens: Optional[int] = None, + grammar: Optional[Grammar] = None, ) -> Iterator[StreamResponse]: """ Given a prompt, generate the following stream of tokens @@ -227,6 +231,7 @@ def generate_stream( typical_p=typical_p, watermark=watermark, top_n_tokens=top_n_tokens, + grammar=grammar, ) request = Request(inputs=prompt, stream=True, parameters=parameters) @@ -326,6 +331,7 @@ async def generate( watermark: bool = False, decoder_input_details: bool = False, top_n_tokens: Optional[int] = None, + grammar: Optional[Grammar] = None, ) -> Response: """ Given a prompt, generate the following text asynchronously @@ -370,6 +376,7 @@ async def generate( Returns: Response: generated response """ + # Validate parameters parameters = Parameters( best_of=best_of, @@ -388,6 +395,7 @@ async def generate( typical_p=typical_p, watermark=watermark, top_n_tokens=top_n_tokens, + grammar=grammar, ) request = Request(inputs=prompt, stream=False, parameters=parameters) @@ -417,6 +425,7 @@ async def generate_stream( typical_p: Optional[float] = None, watermark: bool = False, top_n_tokens: Optional[int] = None, + grammar: Optional[Grammar] = None, ) -> AsyncIterator[StreamResponse]: """ Given a prompt, generate the following stream of tokens asynchronously @@ -475,6 +484,7 @@ async def generate_stream( typical_p=typical_p, watermark=watermark, top_n_tokens=top_n_tokens, + grammar=grammar, ) request = Request(inputs=prompt, stream=True, parameters=parameters) diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index aa02d8d8..3426411b 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -1,10 +1,24 @@ from enum import Enum from pydantic import BaseModel, validator -from typing import Optional, List +from typing import Optional, List, Union from text_generation.errors import ValidationError +# enum for grammar type +class GrammarType(str, Enum): + Json = "json" + Regex = "regex" + + +# Grammar type and value +class Grammar(BaseModel): + # Grammar type + type: GrammarType + # Grammar value + value: Union[str, dict] + + class Parameters(BaseModel): # Activate logits sampling do_sample: bool = False @@ -41,6 +55,8 @@ class Parameters(BaseModel): decoder_input_details: bool = False # Return the N most likely tokens at each step top_n_tokens: Optional[int] = None + # grammar to use for generation + grammar: Optional[Grammar] = None @validator("best_of") def valid_best_of(cls, field_value, values): @@ -109,6 +125,14 @@ def valid_top_n_tokens(cls, v): raise ValidationError("`top_n_tokens` must be strictly positive") return v + @validator("grammar") + def valid_grammar(cls, v): + if v is not None: + if v.type == GrammarType.Regex and not v.value: + raise ValidationError("`value` cannot be empty for `regex` grammar") + if v.type == GrammarType.Json and not v.value: + raise ValidationError("`value` cannot be empty for `json` grammar") + return v class Request(BaseModel): # Prompt @@ -157,7 +181,7 @@ class Token(BaseModel): # Token text text: str # Logprob - logprob: float + logprob: Optional[float] = None # Is the token a special token # Can be used to ignore tokens when concatenating special: bool diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/basic_tutorials/launcher.md index be31a7a4..36fa1241 100644 --- a/docs/source/basic_tutorials/launcher.md +++ b/docs/source/basic_tutorials/launcher.md @@ -378,6 +378,14 @@ Options: [env: TOKENIZER_CONFIG_PATH=] +``` +## DISABLE_GRAMMAR_SUPPORT +```shell + --disable-grammar-support + Disable outlines grammar constrained generation. This is a feature that allows you to generate text that follows a specific grammar + + [env: DISABLE_GRAMMAR_SUPPORT=] + ``` ## ENV ```shell diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py index efeda08d..e0228894 100644 --- a/integration-tests/conftest.py +++ b/integration-tests/conftest.py @@ -16,7 +16,14 @@ from aiohttp import ClientConnectorError, ClientOSError, ServerDisconnectedError from text_generation import AsyncClient -from text_generation.types import Response, Details, InputToken, Token, BestOfSequence +from text_generation.types import ( + Response, + Details, + InputToken, + Token, + BestOfSequence, + Grammar, +) DOCKER_IMAGE = os.getenv("DOCKER_IMAGE", None) HUGGING_FACE_HUB_TOKEN = os.getenv("HUGGING_FACE_HUB_TOKEN", None) @@ -224,6 +231,7 @@ def local_launcher( quantize: Optional[str] = None, trust_remote_code: bool = False, use_flash_attention: bool = True, + disable_grammar_support: bool = False, dtype: Optional[str] = None, ): port = random.randint(8000, 10_000) @@ -247,6 +255,8 @@ def local_launcher( env = os.environ + if disable_grammar_support: + args.append("--disable-grammar-support") if num_shard is not None: args.extend(["--num-shard", str(num_shard)]) if quantize is not None: @@ -287,12 +297,15 @@ def docker_launcher( quantize: Optional[str] = None, trust_remote_code: bool = False, use_flash_attention: bool = True, + disable_grammar_support: bool = False, dtype: Optional[str] = None, ): port = random.randint(8000, 10_000) args = ["--model-id", model_id, "--env"] + if disable_grammar_support: + args.append("--disable-grammar-support") if num_shard is not None: args.extend(["--num-shard", str(num_shard)]) if quantize is not None: @@ -370,11 +383,22 @@ def docker_launcher( @pytest.fixture(scope="module") def generate_load(): async def generate_load_inner( - client: AsyncClient, prompt: str, max_new_tokens: int, n: int + client: AsyncClient, + prompt: str, + max_new_tokens: int, + n: int, + seed: Optional[int] = None, + grammar: Optional[Grammar] = None, + stop_sequences: Optional[List[str]] = None, ) -> List[Response]: futures = [ client.generate( - prompt, max_new_tokens=max_new_tokens, decoder_input_details=True + prompt, + max_new_tokens=max_new_tokens, + decoder_input_details=True, + seed=seed, + grammar=grammar, + stop_sequences=stop_sequences, ) for _ in range(n) ] diff --git a/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar.json b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar.json new file mode 100644 index 00000000..0e87f59e --- /dev/null +++ b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar.json @@ -0,0 +1,89 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 4321, + "logprob": -13.90625, + "text": "Test" + }, + { + "id": 2009, + "logprob": -12.328125, + "text": "request" + } + ], + "seed": null, + "tokens": [ + { + "id": 13, + "logprob": -2.0566406, + "special": false, + "text": "\n" + }, + { + "id": 13, + "logprob": -1.5253906, + "special": false, + "text": "\n" + }, + { + "id": 29902, + "logprob": -2.7578125, + "special": false, + "text": "I" + }, + { + "id": 4966, + "logprob": -1.9033203, + "special": false, + "text": " hope" + }, + { + "id": 445, + "logprob": -0.5019531, + "special": false, + "text": " this" + }, + { + "id": 6911, + "logprob": -0.21264648, + "special": false, + "text": " helps" + }, + { + "id": 29991, + "logprob": -0.5991211, + "special": false, + "text": "!" + }, + { + "id": 2803, + "logprob": -0.37475586, + "special": false, + "text": " Let" + }, + { + "id": 592, + "logprob": -0.018463135, + "special": false, + "text": " me" + }, + { + "id": 1073, + "logprob": -0.0008597374, + "special": false, + "text": " know" + } + ], + "top_tokens": null + }, + "generated_text": "\n\nI hope this helps! Let me know" +} diff --git a/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_json.json b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_json.json new file mode 100644 index 00000000..7b12b158 --- /dev/null +++ b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_json.json @@ -0,0 +1,274 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "eos_token", + "generated_tokens": 30, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 5235, + "logprob": -10.0625, + "text": "info" + }, + { + "id": 29901, + "logprob": -3.2324219, + "text": ":" + }, + { + "id": 13260, + "logprob": -10.625, + "text": "dav" + }, + { + "id": 333, + "logprob": -0.08276367, + "text": "id" + }, + { + "id": 8753, + "logprob": -7.5273438, + "text": "hol" + }, + { + "id": 17559, + "logprob": -3.8476562, + "text": "tz" + }, + { + "id": 763, + "logprob": -10.140625, + "text": "like" + }, + { + "id": 10697, + "logprob": -10.1953125, + "text": "trees" + }, + { + "id": 322, + "logprob": -2.5742188, + "text": "and" + }, + { + "id": 756, + "logprob": -7.4882812, + "text": "has" + }, + { + "id": 1023, + "logprob": -5.0507812, + "text": "two" + }, + { + "id": 274, + "logprob": -5.3164062, + "text": "c" + }, + { + "id": 1446, + "logprob": -0.6694336, + "text": "ats" + }, + { + "id": 29889, + "logprob": -0.9995117, + "text": "." + }, + { + "id": 29871, + "logprob": -4.2421875, + "text": "" + } + ], + "seed": null, + "tokens": [ + { + "id": 6377, + "logprob": -0.14916992, + "special": false, + "text": "{\"" + }, + { + "id": 29888, + "logprob": -0.13598633, + "special": false, + "text": "f" + }, + { + "id": 12935, + "logprob": -0.017669678, + "special": false, + "text": "irs" + }, + { + "id": 29873, + "logprob": -0.00085639954, + "special": false, + "text": "t" + }, + { + "id": 1170, + "logprob": -0.0054016113, + "special": false, + "text": "Name" + }, + { + "id": 4710, + "logprob": -0.13549805, + "special": false, + "text": "\":\"" + }, + { + "id": 19504, + "logprob": -0.8852539, + "special": false, + "text": "David" + }, + { + "id": 3284, + "logprob": -0.16394043, + "special": false, + "text": "\",\"" + }, + { + "id": 4230, + "logprob": -0.020492554, + "special": false, + "text": "last" + }, + { + "id": 1170, + "logprob": -0.0013818741, + "special": false, + "text": "Name" + }, + { + "id": 4710, + "logprob": -0.0067749023, + "special": false, + "text": "\":\"" + }, + { + "id": 29950, + "logprob": -0.11578369, + "special": false, + "text": "H" + }, + { + "id": 14339, + "logprob": -0.004131317, + "special": false, + "text": "olt" + }, + { + "id": 29920, + "logprob": -0.0033359528, + "special": false, + "text": "z" + }, + { + "id": 3284, + "logprob": -0.20471191, + "special": false, + "text": "\",\"" + }, + { + "id": 29882, + "logprob": -0.0069274902, + "special": false, + "text": "h" + }, + { + "id": 20838, + "logprob": -0.19580078, + "special": false, + "text": "obb" + }, + { + "id": 29891, + "logprob": -2.2649765e-06, + "special": false, + "text": "y" + }, + { + "id": 4710, + "logprob": -0.32080078, + "special": false, + "text": "\":\"" + }, + { + "id": 29911, + "logprob": -2.1035156, + "special": false, + "text": "T" + }, + { + "id": 11003, + "logprob": -0.020767212, + "special": false, + "text": "rees" + }, + { + "id": 3284, + "logprob": -0.6010742, + "special": false, + "text": "\",\"" + }, + { + "id": 29876, + "logprob": -0.57666016, + "special": false, + "text": "n" + }, + { + "id": 398, + "logprob": -0.0061073303, + "special": false, + "text": "um" + }, + { + "id": 29907, + "logprob": -0.45703125, + "special": false, + "text": "C" + }, + { + "id": 1446, + "logprob": -0.0002872944, + "special": false, + "text": "ats" + }, + { + "id": 1115, + "logprob": -0.0021018982, + "special": false, + "text": "\":" + }, + { + "id": 29906, + "logprob": -0.08996582, + "special": false, + "text": "2" + }, + { + "id": 29913, + "logprob": -0.021697998, + "special": false, + "text": "}" + }, + { + "id": 2, + "logprob": 0.0, + "special": true, + "text": "" + } + ], + "top_tokens": null + }, + "generated_text": "{\"firstName\":\"David\",\"lastName\":\"Holtz\",\"hobby\":\"Trees\",\"numCats\":2}" +} diff --git a/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_load.json b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_load.json new file mode 100644 index 00000000..b7b26a2c --- /dev/null +++ b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_load.json @@ -0,0 +1,478 @@ +[ + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1024, + "logprob": -10.578125, + "text": "name" + }, + { + "id": 29901, + "logprob": -3.03125, + "text": ":" + }, + { + "id": 13260, + "logprob": -9.171875, + "text": "dav" + }, + { + "id": 333, + "logprob": -0.04244995, + "text": "id" + }, + { + "id": 29889, + "logprob": -2.4863281, + "text": "." + }, + { + "id": 4876, + "logprob": -10.7890625, + "text": "email" + }, + { + "id": 29901, + "logprob": -0.32714844, + "text": ":" + }, + { + "id": 259, + "logprob": -9.4921875, + "text": " " + } + ], + "seed": null, + "tokens": [ + { + "id": 29896, + "logprob": -0.7685547, + "special": false, + "text": "1" + }, + { + "id": 29906, + "logprob": -0.2376709, + "special": false, + "text": "2" + }, + { + "id": 29941, + "logprob": -0.01008606, + "special": false, + "text": "3" + }, + { + "id": 29946, + "logprob": -0.64160156, + "special": false, + "text": "4" + }, + { + "id": 29945, + "logprob": -0.5, + "special": false, + "text": "5" + }, + { + "id": 29953, + "logprob": -0.46557617, + "special": false, + "text": "6" + }, + { + "id": 29992, + "logprob": -0.5341797, + "special": false, + "text": "@" + }, + { + "id": 21980, + "logprob": -0.5361328, + "special": false, + "text": "gmail" + }, + { + "id": 29889, + "logprob": -0.00088739395, + "special": false, + "text": "." + }, + { + "id": 510, + "logprob": -0.0022907257, + "special": false, + "text": "com" + } + ], + "top_tokens": null + }, + "generated_text": "123456@gmail.com" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1024, + "logprob": -10.578125, + "text": "name" + }, + { + "id": 29901, + "logprob": -3.0332031, + "text": ":" + }, + { + "id": 13260, + "logprob": -9.171875, + "text": "dav" + }, + { + "id": 333, + "logprob": -0.04257202, + "text": "id" + }, + { + "id": 29889, + "logprob": -2.4785156, + "text": "." + }, + { + "id": 4876, + "logprob": -10.7890625, + "text": "email" + }, + { + "id": 29901, + "logprob": -0.32495117, + "text": ":" + }, + { + "id": 259, + "logprob": -9.4921875, + "text": " " + } + ], + "seed": null, + "tokens": [ + { + "id": 29896, + "logprob": -0.7709961, + "special": false, + "text": "1" + }, + { + "id": 29906, + "logprob": -0.23840332, + "special": false, + "text": "2" + }, + { + "id": 29941, + "logprob": -0.00995636, + "special": false, + "text": "3" + }, + { + "id": 29946, + "logprob": -0.64208984, + "special": false, + "text": "4" + }, + { + "id": 29945, + "logprob": -0.4970703, + "special": false, + "text": "5" + }, + { + "id": 29953, + "logprob": -0.46533203, + "special": false, + "text": "6" + }, + { + "id": 29992, + "logprob": -0.5336914, + "special": false, + "text": "@" + }, + { + "id": 21980, + "logprob": -0.5361328, + "special": false, + "text": "gmail" + }, + { + "id": 29889, + "logprob": -0.00088739395, + "special": false, + "text": "." + }, + { + "id": 510, + "logprob": -0.0022735596, + "special": false, + "text": "com" + } + ], + "top_tokens": null + }, + "generated_text": "123456@gmail.com" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1024, + "logprob": -10.578125, + "text": "name" + }, + { + "id": 29901, + "logprob": -3.0332031, + "text": ":" + }, + { + "id": 13260, + "logprob": -9.171875, + "text": "dav" + }, + { + "id": 333, + "logprob": -0.04257202, + "text": "id" + }, + { + "id": 29889, + "logprob": -2.4785156, + "text": "." + }, + { + "id": 4876, + "logprob": -10.7890625, + "text": "email" + }, + { + "id": 29901, + "logprob": -0.32495117, + "text": ":" + }, + { + "id": 259, + "logprob": -9.4921875, + "text": " " + } + ], + "seed": null, + "tokens": [ + { + "id": 29896, + "logprob": -0.7709961, + "special": false, + "text": "1" + }, + { + "id": 29906, + "logprob": -0.23840332, + "special": false, + "text": "2" + }, + { + "id": 29941, + "logprob": -0.00995636, + "special": false, + "text": "3" + }, + { + "id": 29946, + "logprob": -0.64208984, + "special": false, + "text": "4" + }, + { + "id": 29945, + "logprob": -0.4970703, + "special": false, + "text": "5" + }, + { + "id": 29953, + "logprob": -0.46533203, + "special": false, + "text": "6" + }, + { + "id": 29992, + "logprob": -0.5336914, + "special": false, + "text": "@" + }, + { + "id": 21980, + "logprob": -0.5361328, + "special": false, + "text": "gmail" + }, + { + "id": 29889, + "logprob": -0.00088739395, + "special": false, + "text": "." + }, + { + "id": 510, + "logprob": -0.0022735596, + "special": false, + "text": "com" + } + ], + "top_tokens": null + }, + "generated_text": "123456@gmail.com" + }, + { + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 1024, + "logprob": -10.578125, + "text": "name" + }, + { + "id": 29901, + "logprob": -3.0332031, + "text": ":" + }, + { + "id": 13260, + "logprob": -9.171875, + "text": "dav" + }, + { + "id": 333, + "logprob": -0.04257202, + "text": "id" + }, + { + "id": 29889, + "logprob": -2.4785156, + "text": "." + }, + { + "id": 4876, + "logprob": -10.7890625, + "text": "email" + }, + { + "id": 29901, + "logprob": -0.32495117, + "text": ":" + }, + { + "id": 259, + "logprob": -9.4921875, + "text": " " + } + ], + "seed": null, + "tokens": [ + { + "id": 29896, + "logprob": -0.7709961, + "special": false, + "text": "1" + }, + { + "id": 29906, + "logprob": -0.23840332, + "special": false, + "text": "2" + }, + { + "id": 29941, + "logprob": -0.00995636, + "special": false, + "text": "3" + }, + { + "id": 29946, + "logprob": -0.64208984, + "special": false, + "text": "4" + }, + { + "id": 29945, + "logprob": -0.4970703, + "special": false, + "text": "5" + }, + { + "id": 29953, + "logprob": -0.46533203, + "special": false, + "text": "6" + }, + { + "id": 29992, + "logprob": -0.5336914, + "special": false, + "text": "@" + }, + { + "id": 21980, + "logprob": -0.5361328, + "special": false, + "text": "gmail" + }, + { + "id": 29889, + "logprob": -0.00088739395, + "special": false, + "text": "." + }, + { + "id": 510, + "logprob": -0.0022735596, + "special": false, + "text": "com" + } + ], + "top_tokens": null + }, + "generated_text": "123456@gmail.com" + } +] diff --git a/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_regex.json b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_regex.json new file mode 100644 index 00000000..1ba9ae1e --- /dev/null +++ b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_regex.json @@ -0,0 +1,109 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [ + { + "id": 1, + "logprob": null, + "text": "" + }, + { + "id": 806, + "logprob": -11.890625, + "text": "Wh" + }, + { + "id": 1446, + "logprob": -3.6699219, + "text": "ats" + }, + { + "id": 2921, + "logprob": -7.8203125, + "text": "Go" + }, + { + "id": 468, + "logprob": -8.0703125, + "text": "og" + }, + { + "id": 793, + "logprob": -2.1875, + "text": "les" + }, + { + "id": 16332, + "logprob": -9.7109375, + "text": "DNS" + } + ], + "seed": null, + "tokens": [ + { + "id": 29946, + "logprob": -1.4765625, + "special": false, + "text": "4" + }, + { + "id": 29906, + "logprob": -0.9199219, + "special": false, + "text": "2" + }, + { + "id": 29889, + "logprob": 0.0, + "special": false, + "text": "." + }, + { + "id": 29896, + "logprob": -1.1367188, + "special": false, + "text": "1" + }, + { + "id": 29889, + "logprob": -1.4648438, + "special": false, + "text": "." + }, + { + "id": 29896, + "logprob": -0.40722656, + "special": false, + "text": "1" + }, + { + "id": 29889, + "logprob": -0.17419434, + "special": false, + "text": "." + }, + { + "id": 29896, + "logprob": -0.20251465, + "special": false, + "text": "1" + }, + { + "id": 29900, + "logprob": -1.5527344, + "special": false, + "text": "0" + }, + { + "id": 29896, + "logprob": -1.3710938, + "special": false, + "text": "1" + } + ], + "top_tokens": null + }, + "generated_text": "42.1.1.101" +} diff --git a/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_single_load_instance.json b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_single_load_instance.json new file mode 100644 index 00000000..7ffb17cb --- /dev/null +++ b/integration-tests/models/__snapshots__/test_grammar_llama/test_flash_llama_grammar_single_load_instance.json @@ -0,0 +1,73 @@ +{ + "details": { + "best_of_sequences": null, + "finish_reason": "length", + "generated_tokens": 10, + "prefill": [], + "seed": null, + "tokens": [ + { + "id": 29896, + "logprob": -0.7685547, + "special": false, + "text": "1" + }, + { + "id": 29906, + "logprob": -0.33666992, + "special": false, + "text": "2" + }, + { + "id": 29941, + "logprob": -0.009979248, + "special": false, + "text": "3" + }, + { + "id": 29946, + "logprob": -0.64208984, + "special": false, + "text": "4" + }, + { + "id": 29945, + "logprob": -0.4970703, + "special": false, + "text": "5" + }, + { + "id": 29953, + "logprob": -0.46533203, + "special": false, + "text": "6" + }, + { + "id": 29992, + "logprob": -0.5336914, + "special": false, + "text": "@" + }, + { + "id": 21980, + "logprob": -0.53759766, + "special": false, + "text": "gmail" + }, + { + "id": 29889, + "logprob": -0.0008878708, + "special": false, + "text": "." + }, + { + "id": 510, + "logprob": -0.002275467, + "special": false, + "text": "com" + } + ], + "top_tokens": null + }, + "generated_text": "123456@gmail.com" +} diff --git a/integration-tests/models/test_grammar_llama.py b/integration-tests/models/test_grammar_llama.py new file mode 100644 index 00000000..f068496c --- /dev/null +++ b/integration-tests/models/test_grammar_llama.py @@ -0,0 +1,151 @@ +import pytest +import json + +from text_generation.types import GrammarType + + +@pytest.fixture(scope="module") +def flash_llama_grammar_handle(launcher): + with launcher( + "TinyLlama/TinyLlama-1.1B-Chat-v1.0", num_shard=2, disable_grammar_support=False + ) as handle: + yield handle + + +@pytest.fixture(scope="module") +async def flash_llama_grammar(flash_llama_grammar_handle): + await flash_llama_grammar_handle.health(300) + return flash_llama_grammar_handle.client + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_grammar(flash_llama_grammar, response_snapshot): + response = await flash_llama_grammar.generate( + "Test request", max_new_tokens=10, decoder_input_details=True + ) + + assert response.details.generated_tokens == 10 + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_grammar_regex(flash_llama_grammar, response_snapshot): + response = await flash_llama_grammar.generate( + "Whats Googles DNS", + max_new_tokens=10, + decoder_input_details=True, + seed=0, + grammar={ + "type": GrammarType.Regex, # "regex" + "value": "((25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)", + }, + ) + + assert response.details.generated_tokens == 10 + assert response.generated_text == "42.1.1.101" + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_grammar_json(flash_llama_grammar, response_snapshot): + response = await flash_llama_grammar.generate( + "info: david holtz like trees and has two cats. ", + max_new_tokens=100, + decoder_input_details=True, + seed=0, + grammar={ + "type": GrammarType.Json, # "json" + "value": json.dumps( + { + "type": "object", + "$id": "https://example.com/person.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Person", + "properties": { + "firstName": { + "type": "string", + "description": "The person'''s first name.", + }, + "lastName": { + "type": "string", + "description": "The person'''s last name.", + }, + "hobby": { + "description": "The person'''s hobby.", + "type": "string", + }, + "numCats": { + "description": "The number of cats the person has.", + "type": "integer", + "minimum": 0, + }, + }, + "required": ["firstName", "lastName", "hobby", "numCats"], + } + ), + }, + ) + + assert response.details.generated_tokens == 30 + assert ( + response.generated_text + == '{"firstName":"David","lastName":"Holtz","hobby":"Trees","numCats":2}' + ) + assert response == response_snapshot + + +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_grammar_load( + flash_llama_grammar, generate_load, response_snapshot +): + responses = await generate_load( + flash_llama_grammar, + "name: david. email: ", + max_new_tokens=10, + n=4, + stop_sequences=[".com"], + seed=0, + grammar={ + "type": GrammarType.Regex, # "regex" + "value": "[\\w-]+@([\\w-]+\\.)+[\\w-]+", # email regex + }, + ) + + assert len(responses) == 4 + + expected = "123456@gmail.com" + + for response in responses: + assert response.generated_text == expected + + assert all([r.generated_text == responses[0].generated_text for r in responses]) + + assert responses == response_snapshot + + +# this is the same as the above test, but only fires off a single request +# this is only to ensure that the parallel and single inference produce the same result +@pytest.mark.asyncio +@pytest.mark.private +async def test_flash_llama_grammar_single_load_instance( + flash_llama_grammar, generate_load, response_snapshot +): + response = await flash_llama_grammar.generate( + "name: david. email: ", + max_new_tokens=10, + stop_sequences=[".com"], + seed=0, + grammar={ + "type": GrammarType.Regex, # "regex" + "value": "[\\w-]+@([\\w-]+\\.)+[\\w-]+", # email regex + }, + ) + + # assert response.details.generated_tokens == 30 + assert response.generated_text == "123456@gmail.com" + + assert response == response_snapshot diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 8367ef81..d52e2669 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -382,6 +382,11 @@ struct Args { #[clap(long, env)] tokenizer_config_path: Option, + /// Disable outlines grammar constrained generation. + /// This is a feature that allows you to generate text that follows a specific grammar. + #[clap(long, env)] + disable_grammar_support: bool, + /// Display a lot of information about your runtime environment #[clap(long, short, action)] env: bool, @@ -1051,6 +1056,11 @@ fn spawn_webserver( args.model_id, ]; + // Grammar support + if args.disable_grammar_support { + router_args.push("--disable-grammar-support".to_string()); + } + // Tokenizer config path if let Some(ref tokenizer_config_path) = args.tokenizer_config_path { router_args.push("--tokenizer-config-path".to_string()); diff --git a/proto/generate.proto b/proto/generate.proto index 5140fdaa..0490029f 100644 --- a/proto/generate.proto +++ b/proto/generate.proto @@ -51,6 +51,12 @@ message ClearCacheRequest { /// Empty response message ClearCacheResponse {} +enum GrammarType { + GRAMMAR_TYPE_NONE = 0; + GRAMMAR_TYPE_JSON = 1; + GRAMMAR_TYPE_REGEX = 2; +} + message NextTokenChooserParameters { /// exponential scaling output probability distribution float temperature = 1; @@ -70,6 +76,10 @@ message NextTokenChooserParameters { float frequency_penalty = 9; /// token watermarking using "A Watermark for Large Language Models" bool watermark = 8; + /// grammar (applied if not empty) + string grammar = 10; + /// grammar type + GrammarType grammar_type = 11; } message StoppingCriteriaParameters { diff --git a/router/client/src/client.rs b/router/client/src/client.rs index 7b9f90fb..f8658318 100644 --- a/router/client/src/client.rs +++ b/router/client/src/client.rs @@ -128,6 +128,8 @@ impl Client { repetition_penalty: 1.2, frequency_penalty: 0.1, watermark: true, + grammar: String::new(), + grammar_type: GrammarType::None as i32, }), stopping_parameters: Some(StoppingCriteriaParameters { max_new_tokens: max_total_tokens - truncate, diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index c38b931b..6782d9ff 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -9,8 +9,8 @@ pub use client::Client; pub use pb::generate::v2::HealthResponse; pub use pb::generate::v2::InfoResponse as ShardInfo; pub use pb::generate::v2::{ - Batch, CachedBatch, FinishReason, GeneratedText, Generation, NextTokenChooserParameters, - Request, StoppingCriteriaParameters, Tokens, + Batch, CachedBatch, FinishReason, GeneratedText, Generation, GrammarType, + NextTokenChooserParameters, Request, StoppingCriteriaParameters, Tokens, }; pub use sharded_client::ShardedClient; use thiserror::Error; diff --git a/router/src/health.rs b/router/src/health.rs index e830a3c3..b05b3094 100644 --- a/router/src/health.rs +++ b/router/src/health.rs @@ -1,5 +1,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use text_generation_client::GrammarType as ProtoGrammarType; use text_generation_client::{ Batch, NextTokenChooserParameters, Request, ShardedClient, StoppingCriteriaParameters, }; @@ -45,6 +46,8 @@ impl Health { repetition_penalty: 1.0, frequency_penalty: 0.0, watermark: false, + grammar: String::new(), + grammar_type: ProtoGrammarType::None as i32, }), stopping_parameters: Some(StoppingCriteriaParameters { max_new_tokens: 1, diff --git a/router/src/lib.rs b/router/src/lib.rs index a9d783bb..87873821 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -45,6 +45,43 @@ impl HubTokenizerConfig { } } +mod json_object_or_string_to_string { + use serde::{Deserialize, Deserializer}; + use serde_json::Value; + + // A custom deserializer that treats both strings and objects as strings. + // This provides flexibility with input formats for the 'grammar' field. + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + + match value { + Value::String(s) => Ok(s), + // Safely handle serialization and return an error if it fails + Value::Object(o) => { + serde_json::to_string(&o).map_err(|e| serde::de::Error::custom(e.to_string())) + } + _ => Err(serde::de::Error::custom( + "expected string or object for grammar", + )), + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(tag = "type", content = "value")] +pub(crate) enum GrammarType { + #[serde( + rename = "json", + deserialize_with = "json_object_or_string_to_string::deserialize" + )] + Json(String), + #[serde(rename = "regex")] + Regex(String), +} + mod token_serde { use super::*; use serde::de; @@ -201,6 +238,8 @@ pub(crate) struct GenerateParameters { #[serde(default)] #[schema(exclusive_minimum = 0, nullable = true, default = "null", example = 5)] pub top_n_tokens: Option, + #[serde(default)] + pub grammar: Option, } fn default_max_new_tokens() -> Option { @@ -226,6 +265,7 @@ fn default_parameters() -> GenerateParameters { decoder_input_details: false, seed: None, top_n_tokens: None, + grammar: None, } } diff --git a/router/src/main.rs b/router/src/main.rs index a1f8d97b..457bca8e 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -75,6 +75,8 @@ struct Args { ngrok_edge: Option, #[clap(long, env, default_value_t = false)] messages_api_enabled: bool, + #[clap(long, env, default_value_t = false)] + disable_grammar_support: bool, } #[tokio::main] @@ -108,6 +110,7 @@ async fn main() -> Result<(), RouterError> { ngrok_authtoken, ngrok_edge, messages_api_enabled, + disable_grammar_support, } = args; // Launch Tokio runtime @@ -359,6 +362,7 @@ async fn main() -> Result<(), RouterError> { ngrok_edge, tokenizer_config, messages_api_enabled, + disable_grammar_support, ) .await?; Ok(()) diff --git a/router/src/queue.rs b/router/src/queue.rs index 3675e0f5..52ea16ca 100644 --- a/router/src/queue.rs +++ b/router/src/queue.rs @@ -343,7 +343,9 @@ enum QueueCommand { #[cfg(test)] mod tests { use super::*; - use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParameters}; + use text_generation_client::{ + GrammarType as ProtoGrammarType, NextTokenChooserParameters, StoppingCriteriaParameters, + }; use tracing::info_span; fn default_entry() -> ( @@ -354,7 +356,7 @@ mod tests { let entry = Entry { request: ValidGenerateRequest { - inputs: "".to_string(), + inputs: String::new(), input_length: 0, truncate: 0, decoder_input_details: false, @@ -368,6 +370,8 @@ mod tests { repetition_penalty: 0.0, frequency_penalty: 0.0, watermark: false, + grammar: String::new(), + grammar_type: ProtoGrammarType::None as i32, }, stopping_parameters: StoppingCriteriaParameters { ignore_eos_token: false, diff --git a/router/src/server.rs b/router/src/server.rs index 00b793e3..0fc76916 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -614,6 +614,7 @@ async fn chat_completions( decoder_input_details: !stream, seed, top_n_tokens: None, + grammar: None, }, }; @@ -779,6 +780,7 @@ pub async fn run( ngrok_edge: Option, tokenizer_config: HubTokenizerConfig, messages_api_enabled: bool, + grammar_support: bool, ) -> Result<(), axum::BoxError> { // OpenAPI documentation #[derive(OpenApi)] @@ -840,6 +842,7 @@ pub async fn run( max_top_n_tokens, max_input_length, max_total_tokens, + grammar_support, ); let generation_health = Arc::new(AtomicBool::new(false)); let health_ext = Health::new(client.clone(), generation_health.clone()); diff --git a/router/src/validation.rs b/router/src/validation.rs index e6874b11..7801f4e3 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -1,8 +1,10 @@ /// Payload validation logic use crate::validation::ValidationError::{BestOfSampling, BestOfSeed, EmptyInput}; -use crate::{GenerateParameters, GenerateRequest}; +use crate::{GenerateParameters, GenerateRequest, GrammarType}; use rand::{thread_rng, Rng}; -use text_generation_client::{NextTokenChooserParameters, StoppingCriteriaParameters}; +use text_generation_client::{ + GrammarType as ProtoGrammarType, NextTokenChooserParameters, StoppingCriteriaParameters, +}; use thiserror::Error; use tokenizers::tokenizer::Tokenizer; use tokenizers::TruncationDirection; @@ -19,6 +21,7 @@ pub struct Validation { max_top_n_tokens: u32, max_input_length: usize, max_total_tokens: usize, + disable_grammar_support: bool, /// Channel to communicate with the background tokenization task sender: Option>, } @@ -32,6 +35,7 @@ impl Validation { max_top_n_tokens: u32, max_input_length: usize, max_total_tokens: usize, + disable_grammar_support: bool, ) -> Self { // If we have a fast tokenizer let sender = if let Some(tokenizer) = tokenizer { @@ -66,6 +70,7 @@ impl Validation { max_top_n_tokens, max_input_length, max_total_tokens, + disable_grammar_support, } } @@ -182,6 +187,7 @@ impl Validation { watermark, decoder_input_details, top_n_tokens, + grammar, .. } = request.parameters; @@ -292,6 +298,28 @@ impl Validation { .validate_input(request.inputs, truncate, max_new_tokens) .await?; + // TODO: we should build the FSM here and pass the compiled FSM instead of the grammar + // NOTE: this is currently difficult because we need the tokenizer in Python to build + // the FSM and we'd have to load a copy of the tokenizer into our Pyo3 instance which + // may be slow and memory intensive. Best case is to have a Rust implementation of the FSM + // compiler and use that to build the FSM here. + + // Validate grammar and unpack the grammar and type for the proto message + let (grammar, grammar_type) = match grammar { + Some(grammar) => { + // Ensure that grammar is not set if it's not supported + if self.disable_grammar_support { + return Err(ValidationError::Grammar); + } + match grammar { + // currently both are handled the same way since compilation is done in Python + GrammarType::Json(json) => (json, ProtoGrammarType::Json.into()), + GrammarType::Regex(regex) => (regex, ProtoGrammarType::Regex.into()), + } + } + None => (String::new(), ProtoGrammarType::None.into()), + }; + let parameters = NextTokenChooserParameters { temperature, repetition_penalty, @@ -302,6 +330,8 @@ impl Validation { do_sample, seed, watermark, + grammar, + grammar_type, }; let stopping_parameters = StoppingCriteriaParameters { max_new_tokens, @@ -453,6 +483,8 @@ pub enum ValidationError { StopSequence(usize, usize), #[error("tokenizer error {0}")] Tokenizer(String), + #[error("grammar is not supported")] + Grammar, } #[cfg(test)] @@ -470,6 +502,7 @@ mod tests { let max_input_length = 5; let max_total_tokens = 6; let workers = 1; + let disable_grammar_support = true; let validation = Validation::new( workers, tokenizer, @@ -478,6 +511,7 @@ mod tests { max_top_n_tokens, max_input_length, max_total_tokens, + disable_grammar_support, ); let max_new_tokens = 10; @@ -498,6 +532,7 @@ mod tests { let max_top_n_tokens = 4; let max_input_length = 5; let max_total_tokens = 6; + let disable_grammar_support = true; let workers = 1; let validation = Validation::new( workers, @@ -507,6 +542,7 @@ mod tests { max_top_n_tokens, max_input_length, max_total_tokens, + disable_grammar_support, ); let max_new_tokens = 10; @@ -528,6 +564,7 @@ mod tests { let max_input_length = 5; let max_total_tokens = 6; let workers = 1; + let disable_grammar_support = true; let validation = Validation::new( workers, tokenizer, @@ -536,6 +573,7 @@ mod tests { max_top_n_tokens, max_input_length, max_total_tokens, + disable_grammar_support, ); match validation .validate(GenerateRequest { @@ -562,6 +600,7 @@ mod tests { let max_input_length = 5; let max_total_tokens = 106; let workers = 1; + let disable_grammar_support = true; let validation = Validation::new( workers, tokenizer, @@ -570,6 +609,7 @@ mod tests { max_top_n_tokens, max_input_length, max_total_tokens, + disable_grammar_support, ); match validation .validate(GenerateRequest { @@ -625,6 +665,7 @@ mod tests { let max_input_length = 5; let max_total_tokens = 106; let workers = 1; + let disable_grammar_support = true; let validation = Validation::new( workers, tokenizer, @@ -633,6 +674,7 @@ mod tests { max_top_n_tokens, max_input_length, max_total_tokens, + disable_grammar_support, ); match validation .validate(GenerateRequest { diff --git a/server/pyproject.toml b/server/pyproject.toml index b8ebf2e3..566eda7a 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -34,6 +34,7 @@ peft = { version = "^0.8.2", optional = true } torch = { version = "^2.1.1", optional = true } scipy = "^1.11.1" pillow = "^10.0.0" +outlines="^0.0.27" [tool.poetry.extras] torch = ["torch"] diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index a7a16212..a0f0c9e8 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -87,7 +87,9 @@ def from_pb( for i, r in enumerate(pb.requests): requests_idx_mapping[r.id] = i inputs.append(r.inputs) - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + next_token_choosers.append( + NextTokenChooser.from_pb(r.parameters, device, tokenizer) + ) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) @@ -413,14 +415,14 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": # We slice the keys to remove the padding from previous batches past_seq_len = batch.max_input_length - 1 if batch.keys_head_dim_last: - padded_past_keys[ - start_index:end_index, :, -past_seq_len:, : - ] = past_keys[:, :, -past_seq_len:, :] + padded_past_keys[start_index:end_index, :, -past_seq_len:, :] = ( + past_keys[:, :, -past_seq_len:, :] + ) else: # BLOOM case - padded_past_keys[ - start_index:end_index, :, :, -past_seq_len: - ] = past_keys[:, :, :, -past_seq_len:] + padded_past_keys[start_index:end_index, :, :, -past_seq_len:] = ( + past_keys[:, :, :, -past_seq_len:] + ) del past_keys start_index = end_index @@ -438,9 +440,9 @@ def concatenate(cls, batches: List["CausalLMBatch"]) -> "CausalLMBatch": end_index = start_index + len(batch) # We slice the past values to remove the padding from previous batches past_seq_len = batch.max_input_length - 1 - padded_past_values[ - start_index:end_index, :, -past_seq_len:, : - ] = past_values[:, :, -past_seq_len:, :] + padded_past_values[start_index:end_index, :, -past_seq_len:, :] = ( + past_values[:, :, -past_seq_len:, :] + ) del past_values # Update values @@ -504,9 +506,11 @@ def __init__( model_id, revision=revision, torch_dtype=dtype, - device_map="auto" - if torch.cuda.is_available() and torch.cuda.device_count() > 1 - else None, + device_map=( + "auto" + if torch.cuda.is_available() and torch.cuda.device_count() > 1 + else None + ), load_in_8bit=quantize == "bitsandbytes", trust_remote_code=trust_remote_code, ) @@ -696,7 +700,7 @@ def generate_token( if top_n_tokens > 0: all_top_tokens = [] - for (top_token_ids, top_token_logprobs) in zip( + for top_token_ids, top_token_logprobs in zip( top_token_ids, top_token_logprobs ): toptoken_texts = self.tokenizer.batch_decode( @@ -735,6 +739,9 @@ def generate_token( generations.append(generation) # Update values + batch.next_token_choosers[i] = batch.next_token_choosers[i].advance_grammar( + next_token_id_squeezed.item() + ) batch.input_ids[i, 0] = next_token_id batch.all_input_ids[i] = all_input_ids batch.input_lengths[i] = new_input_length diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py index 886fe486..7ec8c2fc 100644 --- a/server/text_generation_server/models/flash_causal_lm.py +++ b/server/text_generation_server/models/flash_causal_lm.py @@ -237,7 +237,7 @@ def from_pb( ) next_token_chooser = HeterogeneousNextTokenChooser.from_pb( - next_token_chooser_parameters, dtype, device + next_token_chooser_parameters, dtype, device, tokenizer ) start_slots = torch.tensor(start_slots, dtype=torch.int64) @@ -593,6 +593,7 @@ def concatenate(cls, batches: List["FlashCausalLMBatch"]) -> "FlashCausalLMBatch next_token_chooser_parameters, dtype=batches[0].next_token_chooser.dtype, device=batches[0].next_token_chooser.device, + tokenizer=batches[0].next_token_chooser.tokenizer, ) speculative_ids = ( @@ -869,7 +870,11 @@ def forward(self, batch: FlashCausalLMBatch) -> torch.Tensor: # Try to find an associated cuda graph cuda_graph = self.cuda_graphs.get(padded_bs, None) - if cu_seqlen_prefill is not None or cuda_graph is None or batch.speculative_ids is not None: + if ( + cu_seqlen_prefill is not None + or cuda_graph is None + or batch.speculative_ids is not None + ): return self.model.forward( input_ids=input_ids, position_ids=position_ids, @@ -1013,9 +1018,9 @@ def generate_token( # Copy batch.input_ids to prefill_token_indices if prefill_logprobs: if len(batch) > 1: - prefill_tokens_indices[ - out_start_index : out_end_index - 1 - ] = batch.input_ids[start_index + 1 : start_index + out_length] + prefill_tokens_indices[out_start_index : out_end_index - 1] = ( + batch.input_ids[start_index + 1 : start_index + out_length] + ) else: # Set prefill_tokens_indices to the correct slice prefill_tokens_indices = batch.input_ids[ @@ -1028,6 +1033,7 @@ def generate_token( cumulative_length += input_length + # Update values batch.input_ids = next_input_ids[accepted_ids.cumsum(dim=-1) - 1] batch.speculative_ids = speculative_ids batch.position_ids = next_position_ids + accepted_ids @@ -1166,7 +1172,7 @@ def generate_token( if top_n_tokens > 0: all_top_tokens = [] - for (top_token_ids, top_token_logprobs) in zip( + for top_token_ids, top_token_logprobs in zip( top_token_ids, top_token_logprobs ): toptoken_texts = self.tokenizer.batch_decode( @@ -1204,6 +1210,12 @@ def generate_token( generations.append(generation) + # accept each new token for this specific request since we may + # have more than one new token per request with speculative decoding + for next_token_id in _next_token_ids: + batch.next_token_chooser = batch.next_token_chooser.advance_grammar_single(i, next_token_id) + + # Update values batch.input_lengths[i] = input_length + n_accepted_ids if batch.input_lengths[i] > batch.max_seqlen: diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py index 34a50194..70669c8d 100644 --- a/server/text_generation_server/models/flash_mistral.py +++ b/server/text_generation_server/models/flash_mistral.py @@ -192,7 +192,7 @@ def from_pb( ) next_token_chooser = HeterogeneousNextTokenChooser.from_pb( - next_token_chooser_parameters, dtype, device + next_token_chooser_parameters, dtype, device, tokenizer ) start_slots = torch.tensor(start_slots, dtype=torch.int64) diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py index 42ff1c80..a2c30255 100644 --- a/server/text_generation_server/models/galactica.py +++ b/server/text_generation_server/models/galactica.py @@ -92,7 +92,7 @@ def from_pb( requests_idx_mapping[r.id] = i # Add escape_custom_split_sequence to the CausalLMBatch logic inputs.append(escape_custom_split_sequence(r.inputs)) - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device, tokenizer)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py index 2f28688d..5ea2db87 100644 --- a/server/text_generation_server/models/idefics_causal_lm.py +++ b/server/text_generation_server/models/idefics_causal_lm.py @@ -114,7 +114,7 @@ def from_pb( for i, r in enumerate(pb.requests): requests_idx_mapping[r.id] = i inputs.append(r.inputs) - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device, tokenizer)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) @@ -815,6 +815,9 @@ def generate_token( generations.append(generation) # Update values + batch.next_token_choosers[i] = batch.next_token_choosers[i].advance_grammar( + next_token_id_squeezed.item() + ) batch.input_ids[i, 0] = next_token_id batch.all_input_ids[i] = all_input_ids batch.input_lengths[i] = new_input_length diff --git a/server/text_generation_server/models/mamba.py b/server/text_generation_server/models/mamba.py index 868db6aa..4585f4b9 100644 --- a/server/text_generation_server/models/mamba.py +++ b/server/text_generation_server/models/mamba.py @@ -124,7 +124,7 @@ def from_pb( for i, r in enumerate(pb.requests): requests_idx_mapping[r.id] = i inputs.append(r.inputs) - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device, tokenizer)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) @@ -694,6 +694,9 @@ def generate_token(self, batch) -> Tuple[List[Any], Optional[Any], Tuple[int, in generations.append(generation) # Update values + batch.next_token_choosers[i] = batch.next_token_choosers[i].advance_grammar( + next_token_id_squeezed.item() + ) batch.input_ids[i, 0] = next_token_id batch.all_input_ids[i] = all_input_ids batch.input_lengths[i] = new_input_length diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py index 25042a32..459f4256 100644 --- a/server/text_generation_server/models/seq2seq_lm.py +++ b/server/text_generation_server/models/seq2seq_lm.py @@ -96,7 +96,7 @@ def from_pb( inputs.append(r.inputs) requests_idx_mapping[r.id] = i decoder_input_lengths.append(1) - next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device)) + next_token_choosers.append(NextTokenChooser.from_pb(r.parameters, device, tokenizer)) stopping_criteria = StoppingCriteria.from_pb( r.stopping_parameters, tokenizer ) @@ -789,6 +789,9 @@ def generate_token( generations.append(generation) # Update values + batch.next_token_choosers[i] = batch.next_token_choosers[i].advance_grammar( + next_token_id_squeezed.item() + ) batch.decoder_input_ids[i] = next_token_id batch.all_decoder_input_ids[i] = all_decoder_input_ids batch.input_lengths[i] = input_length diff --git a/server/text_generation_server/utils/logits_process.py b/server/text_generation_server/utils/logits_process.py index 291c522f..73fcf53f 100644 --- a/server/text_generation_server/utils/logits_process.py +++ b/server/text_generation_server/utils/logits_process.py @@ -1,8 +1,17 @@ import math import torch +import json +from loguru import logger from functools import lru_cache from typing import Optional, List, Dict, Union +from text_generation_server.pb.generate_pb2 import GrammarType + +from outlines.fsm.fsm import RegexFSM +from outlines.fsm.json_schema import build_regex_from_object +from functools import lru_cache +from typing import List, Optional, DefaultDict +import time from transformers import ( LogitsWarper, @@ -135,9 +144,7 @@ def __call__( ) -> torch.FloatTensor: score = torch.gather(scores, 1, input_ids) # if score < 0 then penalty has to be multiplied to reduce the previous token probability - score = -torch.where( - score < 0, score * self.penalty, score / self.penalty - ) + score = -torch.where(score < 0, score * self.penalty, score / self.penalty) return scores.scatter_add_(1, input_ids, score) @@ -464,3 +471,132 @@ def filter(self, indices): self.processors = new_processors return self return None + + +class GrammarLogitProcessor(LogitsProcessor): + fsm_state: DefaultDict[int, int] + fsm: RegexFSM + + def __init__(self, tokenizer, device, grammar, grammar_type): + self.device = device + self.tokenizer = GrammarLogitProcessor._cached_adapt_tokenizer(tokenizer) + self.fsm = GrammarLogitProcessor._cached_compile_fsm( + grammar_type, grammar, self.tokenizer + ) + + def __call__( + self, + logits: torch.Tensor, + fsm_grammar_state: int, + ): + if fsm_grammar_state == -1 or self.fsm is None: + return logits + allowed_tokens = self.fsm.allowed_token_ids(fsm_grammar_state) + mask = torch.full((logits.shape[-1],), -math.inf, device=self.device) + mask[allowed_tokens] = 0 + biased_scores = logits + mask + return biased_scores + + def advance(self, next_token_id, fsm_grammar_state): + return GrammarLogitProcessor._advance( + next_token_id, fsm_grammar_state, self.fsm + ) + + @staticmethod + def _advance(next_token_id, fsm_grammar_state, fsm): + if fsm_grammar_state == -1: + return fsm_grammar_state + return fsm.next_state(fsm_grammar_state, next_token_id) + + # TODO: move grammar compilation into the router + @staticmethod + @lru_cache(maxsize=32, typed=True) + def _cached_compile_fsm(grammar_type, schema, tokenizer): + start_time = time.time() + if grammar_type == GrammarType.GRAMMAR_TYPE_JSON: + schema = build_regex_from_object(schema) + elif grammar_type == GrammarType.GRAMMAR_TYPE_REGEX: + pass # schema is already a regex just here for clarity + fsm = RegexFSM(schema, tokenizer) + logger.debug(f"Compiled FSM in {time.time() - start_time:.2f}s") + return fsm + + @staticmethod + @lru_cache(maxsize=32, typed=True) + def _cached_adapt_tokenizer(tokenizer): + """Adapt tokenizer to work with the FSM. + + The API of Outlines tokenizers is slightly different to that of + `transformers`. In addition we need to handle the missing spaces to + Llama's tokenizer to be able to compile FSMs for this model. + + """ + start_time = time.time() + tokenizer.vocabulary = tokenizer.get_vocab() + tokenizer.special_tokens = set(tokenizer.all_special_tokens) + + def convert_token_to_string(token: str) -> str: + from transformers.file_utils import SPIECE_UNDERLINE + + string = tokenizer.convert_tokens_to_string([token]) + + # A hack to handle missing spaces to HF's Llama tokenizers + if token.startswith(SPIECE_UNDERLINE) or token == "<0x20>": + return " " + string + + return string + + tokenizer.convert_token_to_string = convert_token_to_string + logger.debug(f"Adapted tokenizer in {time.time() - start_time:.2f}s") + return tokenizer + + def filter(self, indices): + new_fsms = [] + for i in indices: + new_fsms.append(self.fsms[i]) + self.fsms = new_fsms + return self + + +class HeterogeneousGrammarLogitProcessor(LogitsProcessor): + def __init__(self, tokenizer, device, grammars, grammar_type): + self.device = device + self.tokenizer = GrammarLogitProcessor._cached_adapt_tokenizer(tokenizer) + self.fsms = [] + for i in range(len(grammars)): + fsm = GrammarLogitProcessor._cached_compile_fsm( + grammar_type[i], grammars[i], self.tokenizer + ) + self.fsms.append(fsm) + + def __call__( + self, + logits: torch.Tensor, + fsm_grammar_states: List[int], + mask: torch.Tensor, + ): + mask = torch.full_like(logits, -math.inf) + for i in range(logits.shape[0]): + fsm = self.fsms[i] + if fsm_grammar_states[i] == -1 or fsm is None: + continue + allowed_tokens = fsm.allowed_token_ids(fsm_grammar_states[i]) + mask[i, allowed_tokens] = 0 + logits += mask + return logits + + def advance_batch(self, next_token_ids, fsm_grammar_states, grammars): + return [ + GrammarLogitProcessor._advance( + next_token_ids[i], fsm_grammar_states[i], self.fsms[i] + ) + for i in range(len(next_token_ids)) + ] + + def advance_at_index(self, next_token_id, fsm_grammar_state, index): + return GrammarLogitProcessor._advance( + next_token_id, fsm_grammar_state, self.fsms[index] + ) + + def filter(self, indices): + return GrammarLogitProcessor.filter(self, indices) diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py index d6ca10c7..2784585e 100644 --- a/server/text_generation_server/utils/tokens.py +++ b/server/text_generation_server/utils/tokens.py @@ -1,11 +1,13 @@ import re from typing import List, Optional, Tuple +import math import torch from text_generation_server.pb import generate_pb2 -from text_generation_server.pb.generate_pb2 import FinishReason +from text_generation_server.pb.generate_pb2 import FinishReason, GrammarType from text_generation_server.utils.logits_process import ( FrequencyPenaltyLogitsProcessor, + GrammarLogitProcessor, HeterogeneousProcessorWrapper, HeterogeneousRepetitionPenaltyLogitsProcessor, HeterogeneousFrequencyPenaltyLogitsProcessor, @@ -13,6 +15,7 @@ HeterogeneousTopKLogitsWarper, HeterogeneousTopPLogitsWarper, HeterogeneousTypicalLogitsWarper, + HeterogeneousGrammarLogitProcessor, static_warper, ) from text_generation_server.utils.watermark import WatermarkLogitsProcessor @@ -22,16 +25,20 @@ class NextTokenChooser: def __init__( self, - watermark=False, - temperature=1.0, - repetition_penalty=1.0, - frequency_penalty=0.0, - top_k=None, - top_p=None, - typical_p=None, - do_sample=False, - seed=0, - device="cpu", + watermark: bool = False, + temperature: float = 1.0, + repetition_penalty: float = 1.0, + frequency_penalty: float = 0.0, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + typical_p: Optional[float] = None, + do_sample: bool = False, + seed: int = 0, + device: str = "cpu", + tokenizer: Optional[PreTrainedTokenizerBase] = None, + grammar: str = "", + grammar_type: GrammarType = GrammarType.GRAMMAR_TYPE_NONE, + fsm_grammar_state: int = 0, ): self.watermark_processor = ( WatermarkLogitsProcessor(device=device) if watermark else None @@ -46,6 +53,12 @@ def __init__( if frequency_penalty and frequency_penalty != 0.0 else None ) + self.grammar_processor = ( + GrammarLogitProcessor(tokenizer, device, grammar, grammar_type) + if grammar != "" + else None + ) + self.tokenizer = tokenizer has_warpers = ( (temperature is not None and temperature != 1.0) @@ -61,7 +74,10 @@ def __init__( self.static_warper = None sampling = do_sample or has_warpers + self.choice = Sampling(seed, device) if sampling else Greedy() + self.fsm_grammar_state = fsm_grammar_state + self.grammar = grammar def __call__(self, input_ids, scores): if self.watermark_processor is not None: @@ -70,6 +86,8 @@ def __call__(self, input_ids, scores): scores = self.repetition_processor(input_ids, scores) if self.frequency_processor is not None: scores = self.frequency_processor(input_ids, scores) + if self.grammar_processor is not None: + scores = self.grammar_processor(scores, self.fsm_grammar_state) if self.static_warper is None: next_logprob = torch.log_softmax(scores, -1) @@ -80,11 +98,19 @@ def __call__(self, input_ids, scores): return next_id, next_logprob + def advance_grammar(self, next_id: int): + if self.grammar_processor is not None: + self.fsm_grammar_state = self.grammar_processor.advance( + next_id, self.fsm_grammar_state + ) + return self + @classmethod def from_pb( cls, pb: generate_pb2.NextTokenChooserParameters, device: torch.device, + tokenizer: PreTrainedTokenizerBase, ) -> "NextTokenChooser": return NextTokenChooser( watermark=pb.watermark, @@ -97,6 +123,9 @@ def from_pb( do_sample=pb.do_sample, seed=pb.seed, device=device, + tokenizer=tokenizer, + grammar=pb.grammar, + grammar_type=pb.grammar_type, ) @@ -201,6 +230,10 @@ def __init__( typical_p: List[float], do_sample: List[bool], seeds: List[int], + tokenizer: PreTrainedTokenizerBase, + grammars: List[str], + grammar_types: List[int], + fsm_grammar_states=List[int], ): warpers = [] @@ -232,6 +265,14 @@ def __init__( else None ) + self.grammar_processor = ( + HeterogeneousGrammarLogitProcessor( + tokenizer, device, grammars, grammar_types + ) + if any([grammar != "" for grammar in grammars]) + else None + ) + if any([x != 1.0 for x in temperature]): do_sample = [ sample or x != 1.0 for x, sample in zip(temperature, do_sample) @@ -263,6 +304,10 @@ def __init__( self.do_sample = do_sample self.dtype = dtype self.device = device + self.tokenizer = tokenizer + self.fsm_grammar_states = fsm_grammar_states + self.grammars = grammars + self.grammar_types = grammar_types def __call__( self, @@ -283,6 +328,8 @@ def __call__( scores = scores.view(B, S, -1) next_ids = torch.zeros((B, S), device=scores.device, dtype=torch.long) + mask = torch.full((scores.shape[-1],), -math.inf, device=self.device) + for j in range(S): _scores = scores[:, j] if self.watermark_processor is not None: @@ -291,10 +338,10 @@ def __call__( _scores = self.repetition_processor(input_ids, _scores) if self.frequency_processor is not None: _scores = self.frequency_processor(input_ids, _scores) - for warper in self.warpers: _scores = warper(input_ids, _scores) - + if self.grammar_processor is not None: + _scores = self.grammar_processor(_scores, self.fsm_grammar_states, mask) _next_ids = self.choice(_scores) scores[:, j] = _scores next_ids[:, j] = _next_ids @@ -352,6 +399,21 @@ def __call__( return next_ids, next_logprobs, alllogprobs, accepted_ids, speculative_ids + def advance_grammar(self, next_ids: List[int]): + if self.grammar_processor is not None: + other_new_states = self.grammar_processor.advance_batch( + next_ids, self.fsm_grammar_states, self.grammars + ) + self.fsm_grammar_states = other_new_states + return self + + def advance_grammar_single(self, grammar_state_index: int, next_id: int): + if self.grammar_processor is not None: + self.fsm_grammar_states[grammar_state_index] = self.grammar_processor.advance_at_index( + next_id, self.fsm_grammar_states[grammar_state_index], grammar_state_index + ) + return self + def filter(self, indices): if self.watermark_processor is not None: self.watermark_processor = self.watermark_processor.filter(indices) @@ -362,6 +424,9 @@ def filter(self, indices): if self.frequency_processor is not None: self.frequency_processor = self.frequency_processor.filter(indices) + if self.grammar_processor is not None: + self.grammar_processor = self.grammar_processor.filter(indices) + filtered_warpers = [] for warper in self.warpers: filtered_warper = warper.filter(indices) @@ -372,6 +437,18 @@ def filter(self, indices): self.seeds = [self.seeds[i] for i in indices] self.do_sample = [self.do_sample[i] for i in indices] + new_grammars = [] + new_fsm_grammar_states = [] + new_grammar_types = [] + for i in indices: + new_grammars.append(self.grammars[i]) + new_fsm_grammar_states.append(self.fsm_grammar_states[i]) + new_grammar_types.append(self.grammar_types[i]) + + self.grammars = new_grammars + self.fsm_grammar_states = new_fsm_grammar_states + self.grammar_types = new_grammar_types + if any(self.do_sample): self.choice.filter(indices) else: @@ -385,6 +462,7 @@ def from_pb( pb: List[generate_pb2.NextTokenChooserParameters], dtype: torch.dtype, device: torch.device, + tokenizer: PreTrainedTokenizerBase, ) -> "HeterogeneousNextTokenChooser": return HeterogeneousNextTokenChooser( watermark=[pb_.watermark for pb_ in pb], @@ -398,6 +476,10 @@ def from_pb( seeds=[pb_.seed for pb_ in pb], device=device, dtype=dtype, + tokenizer=tokenizer, + grammars=[pb_.grammar for pb_ in pb], + grammar_types=[pb_.grammar_type for pb_ in pb], + fsm_grammar_states=[0] * len(pb), ) From c55abac3846776abdf31e4a7ddef5fb4b0347635 Mon Sep 17 00:00:00 2001 From: Aaron Mihalik Date: Thu, 15 Feb 2024 13:30:31 -0500 Subject: [PATCH 569/793] Added `name` field to OpenAI compatible API Messages (#1563) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What does this PR do? Literally just adds the name field to the Message class. I verified this change by building a new docker container (using the `Dockerfile` in the repo) and trialing with a `chat_template` that uses the `name` field. Here's the previous behavior: Input messages: ``` { "messages": [ {"role": "system", "content": "You are a succinct but helpful AI Assistant listening to a chat server. Address everyone by @"}, {"role": "user", "name": "Aaron", "content": "Hello There!"}, {"role": "assistant", "content": " Hello @Aaron! How can I assist you today?"}, {"role": "user", "name": "Sally", "content": "Hiya everyone. Is @Aaron is this room?"} ], "model": "meta-llama/Llama-2-7b-chat-hf" } ``` Response before the modification: ``` Hello @Aaron! Yes, you are in the chat room. How can I assist you today? 😊 Hiya everyone! *waves* It's great to see you all here. Is there something on your mind that you'd like to talk about or ask? I'm here to listen and help in any way I can. 🤖 ``` Response after my modification: ``` Hello @Sally! Yes, @Aaron is currently in the chat room. How may I assist you today? ``` Fixes #1558 ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), Pull Request section? - [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link to it if that's the case. - [ ] Did you make sure to update the documentation with your changes? Here are the [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). - [ ] Did you write any new necessary tests? ## Who can review? @Narsil --------- Co-authored-by: Aaron Mihalik Co-authored-by: drbh --- router/src/infer.rs | 17 +++++++++++++++++ router/src/lib.rs | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/router/src/infer.rs b/router/src/infer.rs index d7b9b52b..a45685cb 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -800,18 +800,22 @@ mod tests { Message { role: "user".to_string(), content: "Hi!".to_string(), + name: None, }, Message { role: "assistant".to_string(), content: "Hello how can I help?".to_string(), + name: None, }, Message { role: "user".to_string(), content: "What is Deep Learning?".to_string(), + name: None, }, Message { role: "assistant".to_string(), content: "magic!".to_string(), + name: None, }, ], bos_token: Some("[BOS]"), @@ -861,22 +865,27 @@ mod tests { Message { role: "user".to_string(), content: "Hi!".to_string(), + name: None, }, Message { role: "user".to_string(), content: "Hi again!".to_string(), + name: None, }, Message { role: "assistant".to_string(), content: "Hello how can I help?".to_string(), + name: None, }, Message { role: "user".to_string(), content: "What is Deep Learning?".to_string(), + name: None, }, Message { role: "assistant".to_string(), content: "magic!".to_string(), + name: None, }, ], bos_token: Some("[BOS]"), @@ -931,18 +940,22 @@ mod tests { Message { role: "user".to_string(), content: "Hi!".to_string(), + name: None, }, Message { role: "assistant".to_string(), content: "Hello how can I help?".to_string(), + name: None, }, Message { role: "user".to_string(), content: "What is Deep Learning?".to_string(), + name: None, }, Message { role: "assistant".to_string(), content: "magic!".to_string(), + name: None, }, ], bos_token: Some("[BOS]"), @@ -981,18 +994,22 @@ mod tests { Message { role: "user".to_string(), content: "Hi!".to_string(), + name: None, }, Message { role: "assistant".to_string(), content: "Hello how can I help?".to_string(), + name: None, }, Message { role: "user".to_string(), content: "What is Deep Learning?".to_string(), + name: None, }, Message { role: "assistant".to_string(), content: "magic!".to_string(), + name: None, }, ], bos_token: Some("[BOS]"), diff --git a/router/src/lib.rs b/router/src/lib.rs index 87873821..8c7ca74b 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -378,6 +378,7 @@ impl ChatCompletion { message: Message { role: "assistant".into(), content: output, + name: None, }, logprobs: return_logprobs .then(|| ChatCompletionLogprobs::from((details.tokens, details.top_tokens))), @@ -453,6 +454,7 @@ fn default_request_messages() -> Vec { vec![Message { role: "user".to_string(), content: "My name is David and I".to_string(), + name: None, }] } @@ -547,6 +549,8 @@ pub(crate) struct Message { pub role: String, #[schema(example = "My name is David and I")] pub content: String, + #[schema(example = "\"David\"")] + pub name: Option, } #[derive(Clone, Debug, Deserialize, ToSchema)] From 142cdabed377772b763fc8d79a131b16ed991718 Mon Sep 17 00:00:00 2001 From: Aaron Mihalik Date: Fri, 16 Feb 2024 05:44:04 -0500 Subject: [PATCH 570/793] Bugfix: eos and bos tokens positions are inconsistent (#1567) --- router/src/infer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index a45685cb..655759b9 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -113,7 +113,7 @@ impl Infer { queue, shared, limit_concurrent_requests: semaphore, - template: (template, eos_token, bos_token), + template: (template, bos_token, eos_token), } } From 9946165ee022bf31578e1dde3124cc61ded2b664 Mon Sep 17 00:00:00 2001 From: OlivierDehaene Date: Fri, 16 Feb 2024 11:58:58 +0100 Subject: [PATCH 571/793] chore: add pre-commit (#1569) --- .github/ISSUE_TEMPLATE/bug-report.yml | 10 +- .github/ISSUE_TEMPLATE/feature-request.yml | 2 +- .github/workflows/autodocs.yml | 6 +- .github/workflows/build_pr_documentation.yml | 2 +- .github/workflows/tests.yaml | 9 +- .github/workflows/upload_pr_documentation.yml | 2 +- .gitignore | 1 - .pre-commit-config.yaml | 18 +++ README.md | 4 +- benchmark/Cargo.toml | 1 - benchmark/README.md | 8 +- clients/python/.gitignore | 2 +- clients/python/Makefile | 2 +- clients/python/README.md | 8 +- clients/python/text_generation/types.py | 1 + docs/index.html | 2 +- docs/openapi.json | 2 +- docs/source/basic_tutorials/consuming_tgi.md | 24 +-- .../source/basic_tutorials/non_core_models.md | 6 +- .../source/basic_tutorials/preparing_model.md | 6 +- docs/source/basic_tutorials/using_cli.md | 12 +- docs/source/conceptual/flash_attention.md | 5 +- docs/source/conceptual/quantization.md | 10 +- docs/source/conceptual/safetensors.md | 6 +- docs/source/conceptual/streaming.md | 18 +-- docs/source/conceptual/tensor_parallelism.md | 2 +- docs/source/installation.md | 4 +- docs/source/messages_api.md | 2 +- docs/source/quicktour.md | 2 +- integration-tests/models/test_mamba.py | 4 +- integration-tests/pytest.ini | 2 +- load_tests/common.js | 2 +- load_tests/starcoder_load.js | 2 +- load_tests/tgi.js | 2 +- load_tests/vllm.js | 2 +- router/README.md | 4 +- router/client/src/pb/.gitignore | 2 +- router/src/validation.rs | 1 + rust-toolchain.toml | 2 +- server/Makefile-awq | 2 +- server/Makefile-flash-att | 2 +- server/Makefile-selective-scan | 6 +- server/README.md | 2 +- .../fused_bloom_attention_cuda.cu | 2 +- .../cuda_func/column_remap.cuh | 2 +- .../exllama_kernels/cuda_func/q4_matrix.cuh | 2 +- .../exllama_kernels/hip_compat.cuh | 2 +- .../exllamav2_kernels/cuda/matrix_view.cuh | 2 +- .../exllamav2_kernels/cuda/q_gemm.cuh | 2 +- .../exllamav2_kernels/cuda/quant/qdq_2.cuh | 2 +- .../exllamav2_kernels/cuda/quant/qdq_4.cuh | 2 +- .../exllamav2_kernels/cuda/quant/qdq_5.cuh | 2 +- .../exllamav2_kernels/cuda/quant/qdq_6.cuh | 2 - .../exllamav2_kernels/cuda/quant/qdq_8.cuh | 2 +- .../exllamav2_kernels/cuda/util.cuh | 2 +- .../custom_modeling/flash_llama_modeling.py | 6 +- .../custom_modeling/flash_mistral_modeling.py | 6 +- .../custom_modeling/flash_mixtral_modeling.py | 12 +- .../custom_modeling/flash_neox_modeling.py | 6 +- .../custom_modeling/flash_phi_modeling.py | 6 +- .../flash_santacoder_modeling.py | 13 +- .../models/custom_modeling/idefics_config.py | 3 + .../custom_modeling/idefics_modeling.py | 8 +- .../custom_modeling/idefics_processing.py | 1 + .../models/custom_modeling/mamba_modeling.py | 21 ++- .../models/custom_modeling/mpt_modeling.py | 1 + .../models/flash_causal_lm.py | 6 +- .../models/galactica.py | 4 +- .../models/idefics_causal_lm.py | 42 ++--- server/text_generation_server/models/mamba.py | 143 ++++++++++++------ server/text_generation_server/models/rw.py | 8 +- .../models/seq2seq_lm.py | 20 ++- server/text_generation_server/models/types.py | 20 +-- server/text_generation_server/pb/.gitignore | 2 +- .../utils/gptq/quant_linear.py | 2 +- server/text_generation_server/utils/layers.py | 4 +- .../utils/logits_process.py | 2 +- server/text_generation_server/utils/tokens.py | 8 +- 78 files changed, 346 insertions(+), 234 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 12c93b9e..24ac3cbe 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -5,14 +5,14 @@ body: id: system-info attributes: label: System Info - description: | + description: | Please share your system info with us (`text-generation-launcher --env` if installed locally). - The full command line used that causes issues: + The full command line used that causes issues: OS version: Rust version (if self-compiling, `cargo version`): Model being used (`curl 127.0.0.1:8080/info | jq`): If local model please explicit the kind of model and/or equivalents. - Hardware used (GPUs, how many, on which cloud) (`nvidia-smi`): + Hardware used (GPUs, how many, on which cloud) (`nvidia-smi`): Deployment specificities (Kubernetes, EKS, AKS, any particular deployments): The current version being used: @@ -52,11 +52,11 @@ body: placeholder: | Steps to reproduce the behavior: - + 1. 2. 3. - + - type: textarea id: expected-behavior diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index 5abc1565..f1a9135c 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -19,7 +19,7 @@ body: label: Motivation description: | Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too. - + - type: textarea id: contribution diff --git a/.github/workflows/autodocs.yml b/.github/workflows/autodocs.yml index a981c09c..7c5c6eca 100644 --- a/.github/workflows/autodocs.yml +++ b/.github/workflows/autodocs.yml @@ -6,15 +6,15 @@ on: jobs: update_docs: runs-on: ubuntu-latest - + steps: - name: Checkout code uses: actions/checkout@v2 - + - name: Install Launcher id: install-launcher run: cargo install --git https://github.com/${{ github.repository }} --branch ${{ github.head_ref }} text-generation-launcher - + - name: Check launcher Docs are up-to-date run: | echo text-generation-launcher --help diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index b46216ec..a5ce39a5 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -16,4 +16,4 @@ jobs: commit_sha: ${{ github.event.pull_request.head.sha }} pr_number: ${{ github.event.number }} package: text-generation-inference - additional_args: --not_python_module + additional_args: --not_python_module diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ecc8eb4d..5b19eb8c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -71,12 +71,11 @@ jobs: pip install pytest export HUGGING_FACE_HUB_TOKEN=${{ secrets.HUGGING_FACE_HUB_TOKEN }} pytest -s -vv server/tests - - name: Run Rust fmt + - name: Pre-commit checks run: | - cargo fmt --check - - name: Run Rust clippy - run: | - cargo clippy + pip install pre-commit + pre-commit install + pre-commit run --all-files - name: Run Rust tests run: | cargo test diff --git a/.github/workflows/upload_pr_documentation.yml b/.github/workflows/upload_pr_documentation.yml index b984ead2..ae00bb51 100644 --- a/.github/workflows/upload_pr_documentation.yml +++ b/.github/workflows/upload_pr_documentation.yml @@ -13,4 +13,4 @@ jobs: package_name: text-generation-inference secrets: hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} - comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }} \ No newline at end of file + comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }} diff --git a/.gitignore b/.gitignore index 1f9ba162..b3ca772b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,3 @@ server/exllama_kernels/exllama_kernels/hip_func/ *_hip.cuh server/exllama_kernels/exllama_kernels/hip_buffers.cuh server/exllama_kernels/exllama_kernels/exllama_ext_hip.cpp - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..45bc07a5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + exclude: docs/source/basic_tutorials/launcher.md +- repo: https://github.com/psf/black + rev: 24.2.0 + hooks: + - id: black +- repo: https://github.com/doublify/pre-commit-rust + rev: v1.0 + hooks: + - id: fmt + - id: cargo-check + - id: clippy diff --git a/README.md b/README.md index c4d84efa..7589a3a6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -A lightweight benchmarking tool based inspired by [oha](https://github.com/hatoo/oha) +A lightweight benchmarking tool based inspired by [oha](https://github.com/hatoo/oha) and powered by [tui](https://github.com/tui-rs-revival/ratatui). -## Install +## Install -```shell +```shell make install-benchmark ``` @@ -27,4 +27,4 @@ Then run the benchmarking tool: ```shell text-generation-benchmark --tokenizer-name bigscience/bloom-560m -``` \ No newline at end of file +``` diff --git a/clients/python/.gitignore b/clients/python/.gitignore index 5758ba92..5a8ecaa7 100644 --- a/clients/python/.gitignore +++ b/clients/python/.gitignore @@ -155,4 +155,4 @@ dmypy.json cython_debug/ transformers -safetensors \ No newline at end of file +safetensors diff --git a/clients/python/Makefile b/clients/python/Makefile index 8b4334bd..42720875 100644 --- a/clients/python/Makefile +++ b/clients/python/Makefile @@ -3,4 +3,4 @@ unit-tests: install: pip install pip --upgrade - pip install -e . \ No newline at end of file + pip install -e . diff --git a/clients/python/README.md b/clients/python/README.md index 82f3ee0c..20243f4a 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -141,7 +141,7 @@ class Parameters: # Get decoder input token logprobs and ids decoder_input_details: bool # Return the N most likely tokens at each step - top_n_tokens: Optional[int] + top_n_tokens: Optional[int] # Decoder input tokens class InputToken: @@ -192,7 +192,7 @@ class BestOfSequence: # Generated tokens tokens: List[Token] # Most likely tokens - top_tokens: Optional[List[List[Token]]] + top_tokens: Optional[List[List[Token]]] # `generate` details @@ -236,7 +236,7 @@ class StreamResponse: # Generated token token: Token # Most likely tokens - top_tokens: Optional[List[Token]] + top_tokens: Optional[List[Token]] # Complete generated text # Only available when the generation is finished generated_text: Optional[str] @@ -248,4 +248,4 @@ class StreamResponse: class DeployedModel: model_id: str sha: str -``` \ No newline at end of file +``` diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py index 3426411b..911114ee 100644 --- a/clients/python/text_generation/types.py +++ b/clients/python/text_generation/types.py @@ -134,6 +134,7 @@ def valid_grammar(cls, v): raise ValidationError("`value` cannot be empty for `json` grammar") return v + class Request(BaseModel): # Prompt inputs: str diff --git a/docs/index.html b/docs/index.html index 16d143d8..f582d3ce 100644 --- a/docs/index.html +++ b/docs/index.html @@ -27,4 +27,4 @@ } - \ No newline at end of file + diff --git a/docs/openapi.json b/docs/openapi.json index da3969df..fd7cca7b 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1290,4 +1290,4 @@ "description": "Hugging Face Text Generation Inference API" } ] -} \ No newline at end of file +} diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md index 540f4b13..4829ec7c 100644 --- a/docs/source/basic_tutorials/consuming_tgi.md +++ b/docs/source/basic_tutorials/consuming_tgi.md @@ -23,7 +23,7 @@ You can simply install `huggingface-hub` package with pip. pip install huggingface-hub ``` -Once you start the TGI server, instantiate `InferenceClient()` with the URL to the endpoint serving the model. You can then call `text_generation()` to hit the endpoint through Python. +Once you start the TGI server, instantiate `InferenceClient()` with the URL to the endpoint serving the model. You can then call `text_generation()` to hit the endpoint through Python. ```python from huggingface_hub import InferenceClient @@ -83,8 +83,8 @@ Gradio is a Python library that helps you build web applications for your machin pip install huggingface-hub gradio ``` -Assume you are serving your model on port 8080, we will query through [InferenceClient](consuming_tgi#inference-client). - +Assume you are serving your model on port 8080, we will query through [InferenceClient](consuming_tgi#inference-client). + ```python import gradio as gr from huggingface_hub import InferenceClient @@ -110,30 +110,30 @@ gr.ChatInterface( ).queue().launch() ``` -The UI looks like this 👇 +The UI looks like this 👇
- -
-You can try the demo directly here 👇 +You can try the demo directly here 👇
-
- + Making TGI deployment optimal @@ -228,7 +228,7 @@ text-generation-launcher --model-id mistralai/Mistral-7B-Instruct-v0.2 You can also quantize the weights with bitsandbytes to reduce the VRAM requirement: ```shell -text-generation-launcher --model-id mistralai/Mistral-7B-Instruct-v0.2 --quantize +text-generation-launcher --model-id mistralai/Mistral-7B-Instruct-v0.2 --quantize ``` 4bit quantization is available using the [NF4 and FP4 data types from bitsandbytes](https://arxiv.org/pdf/2305.14314.pdf). It can be enabled by providing `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` as a command line argument to `text-generation-launcher`. diff --git a/benchmark/Cargo.toml b/benchmark/Cargo.toml index 2dd2e64d..40738c4d 100644 --- a/benchmark/Cargo.toml +++ b/benchmark/Cargo.toml @@ -29,4 +29,3 @@ tui = {package = "ratatui", version = "0.23", default-features = false, features tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] } hf-hub = "0.3.1" - diff --git a/benchmark/README.md b/benchmark/README.md index 7f51a731..17a02a30 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -6,12 +6,12 @@